1 #define pr_fmt(fmt) "SVM: " fmt
3 #include <linux/kvm_host.h>
7 #include "kvm_cache_regs.h"
12 #include <linux/module.h>
13 #include <linux/mod_devicetable.h>
14 #include <linux/kernel.h>
15 #include <linux/vmalloc.h>
16 #include <linux/highmem.h>
17 #include <linux/amd-iommu.h>
18 #include <linux/sched.h>
19 #include <linux/trace_events.h>
20 #include <linux/slab.h>
21 #include <linux/hashtable.h>
22 #include <linux/objtool.h>
23 #include <linux/psp-sev.h>
24 #include <linux/file.h>
25 #include <linux/pagemap.h>
26 #include <linux/swap.h>
27 #include <linux/rwsem.h>
28 #include <linux/cc_platform.h>
31 #include <asm/perf_event.h>
32 #include <asm/tlbflush.h>
34 #include <asm/debugreg.h>
35 #include <asm/kvm_para.h>
36 #include <asm/irq_remapping.h>
37 #include <asm/spec-ctrl.h>
38 #include <asm/cpu_device_id.h>
39 #include <asm/traps.h>
40 #include <asm/fpu/api.h>
42 #include <asm/virtext.h>
48 #include "kvm_onhyperv.h"
49 #include "svm_onhyperv.h"
51 MODULE_AUTHOR("Qumranet");
52 MODULE_LICENSE("GPL");
55 static const struct x86_cpu_id svm_cpu_id
[] = {
56 X86_MATCH_FEATURE(X86_FEATURE_SVM
, NULL
),
59 MODULE_DEVICE_TABLE(x86cpu
, svm_cpu_id
);
62 #define SEG_TYPE_LDT 2
63 #define SEG_TYPE_BUSY_TSS16 3
65 #define SVM_FEATURE_LBRV (1 << 1)
66 #define SVM_FEATURE_SVML (1 << 2)
67 #define SVM_FEATURE_TSC_RATE (1 << 4)
68 #define SVM_FEATURE_VMCB_CLEAN (1 << 5)
69 #define SVM_FEATURE_FLUSH_ASID (1 << 6)
70 #define SVM_FEATURE_DECODE_ASSIST (1 << 7)
71 #define SVM_FEATURE_PAUSE_FILTER (1 << 10)
73 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
75 #define TSC_RATIO_RSVD 0xffffff0000000000ULL
76 #define TSC_RATIO_MIN 0x0000000000000001ULL
77 #define TSC_RATIO_MAX 0x000000ffffffffffULL
79 static bool erratum_383_found __read_mostly
;
81 u32 msrpm_offsets
[MSRPM_OFFSETS
] __read_mostly
;
84 * Set osvw_len to higher value when updated Revision Guides
85 * are published and we know what the new status bits are
87 static uint64_t osvw_len
= 4, osvw_status
;
89 static DEFINE_PER_CPU(u64
, current_tsc_ratio
);
90 #define TSC_RATIO_DEFAULT 0x0100000000ULL
92 static const struct svm_direct_access_msrs
{
93 u32 index
; /* Index of the MSR */
94 bool always
; /* True if intercept is initially cleared */
95 } direct_access_msrs
[MAX_DIRECT_ACCESS_MSRS
] = {
96 { .index
= MSR_STAR
, .always
= true },
97 { .index
= MSR_IA32_SYSENTER_CS
, .always
= true },
98 { .index
= MSR_IA32_SYSENTER_EIP
, .always
= false },
99 { .index
= MSR_IA32_SYSENTER_ESP
, .always
= false },
101 { .index
= MSR_GS_BASE
, .always
= true },
102 { .index
= MSR_FS_BASE
, .always
= true },
103 { .index
= MSR_KERNEL_GS_BASE
, .always
= true },
104 { .index
= MSR_LSTAR
, .always
= true },
105 { .index
= MSR_CSTAR
, .always
= true },
106 { .index
= MSR_SYSCALL_MASK
, .always
= true },
108 { .index
= MSR_IA32_SPEC_CTRL
, .always
= false },
109 { .index
= MSR_IA32_PRED_CMD
, .always
= false },
110 { .index
= MSR_IA32_LASTBRANCHFROMIP
, .always
= false },
111 { .index
= MSR_IA32_LASTBRANCHTOIP
, .always
= false },
112 { .index
= MSR_IA32_LASTINTFROMIP
, .always
= false },
113 { .index
= MSR_IA32_LASTINTTOIP
, .always
= false },
114 { .index
= MSR_EFER
, .always
= false },
115 { .index
= MSR_IA32_CR_PAT
, .always
= false },
116 { .index
= MSR_AMD64_SEV_ES_GHCB
, .always
= true },
117 { .index
= MSR_INVALID
, .always
= false },
121 * These 2 parameters are used to config the controls for Pause-Loop Exiting:
122 * pause_filter_count: On processors that support Pause filtering(indicated
123 * by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter
124 * count value. On VMRUN this value is loaded into an internal counter.
125 * Each time a pause instruction is executed, this counter is decremented
126 * until it reaches zero at which time a #VMEXIT is generated if pause
127 * intercept is enabled. Refer to AMD APM Vol 2 Section 15.14.4 Pause
128 * Intercept Filtering for more details.
129 * This also indicate if ple logic enabled.
131 * pause_filter_thresh: In addition, some processor families support advanced
132 * pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on
133 * the amount of time a guest is allowed to execute in a pause loop.
134 * In this mode, a 16-bit pause filter threshold field is added in the
135 * VMCB. The threshold value is a cycle count that is used to reset the
136 * pause counter. As with simple pause filtering, VMRUN loads the pause
137 * count value from VMCB into an internal counter. Then, on each pause
138 * instruction the hardware checks the elapsed number of cycles since
139 * the most recent pause instruction against the pause filter threshold.
140 * If the elapsed cycle count is greater than the pause filter threshold,
141 * then the internal pause count is reloaded from the VMCB and execution
142 * continues. If the elapsed cycle count is less than the pause filter
143 * threshold, then the internal pause count is decremented. If the count
144 * value is less than zero and PAUSE intercept is enabled, a #VMEXIT is
145 * triggered. If advanced pause filtering is supported and pause filter
146 * threshold field is set to zero, the filter will operate in the simpler,
150 static unsigned short pause_filter_thresh
= KVM_DEFAULT_PLE_GAP
;
151 module_param(pause_filter_thresh
, ushort
, 0444);
153 static unsigned short pause_filter_count
= KVM_SVM_DEFAULT_PLE_WINDOW
;
154 module_param(pause_filter_count
, ushort
, 0444);
156 /* Default doubles per-vcpu window every exit. */
157 static unsigned short pause_filter_count_grow
= KVM_DEFAULT_PLE_WINDOW_GROW
;
158 module_param(pause_filter_count_grow
, ushort
, 0444);
160 /* Default resets per-vcpu window every exit to pause_filter_count. */
161 static unsigned short pause_filter_count_shrink
= KVM_DEFAULT_PLE_WINDOW_SHRINK
;
162 module_param(pause_filter_count_shrink
, ushort
, 0444);
164 /* Default is to compute the maximum so we can never overflow. */
165 static unsigned short pause_filter_count_max
= KVM_SVM_DEFAULT_PLE_WINDOW_MAX
;
166 module_param(pause_filter_count_max
, ushort
, 0444);
169 * Use nested page tables by default. Note, NPT may get forced off by
170 * svm_hardware_setup() if it's unsupported by hardware or the host kernel.
172 bool npt_enabled
= true;
173 module_param_named(npt
, npt_enabled
, bool, 0444);
175 /* allow nested virtualization in KVM/SVM */
176 static int nested
= true;
177 module_param(nested
, int, S_IRUGO
);
179 /* enable/disable Next RIP Save */
180 static int nrips
= true;
181 module_param(nrips
, int, 0444);
183 /* enable/disable Virtual VMLOAD VMSAVE */
184 static int vls
= true;
185 module_param(vls
, int, 0444);
187 /* enable/disable Virtual GIF */
188 static int vgif
= true;
189 module_param(vgif
, int, 0444);
191 /* enable/disable LBR virtualization */
192 static int lbrv
= true;
193 module_param(lbrv
, int, 0444);
195 static int tsc_scaling
= true;
196 module_param(tsc_scaling
, int, 0444);
199 * enable / disable AVIC. Because the defaults differ for APICv
200 * support between VMX and SVM we cannot use module_param_named.
203 module_param(avic
, bool, 0444);
205 bool __read_mostly dump_invalid_vmcb
;
206 module_param(dump_invalid_vmcb
, bool, 0644);
209 bool intercept_smi
= true;
210 module_param(intercept_smi
, bool, 0444);
213 static bool svm_gp_erratum_intercept
= true;
215 static u8 rsm_ins_bytes
[] = "\x0f\xaa";
217 static unsigned long iopm_base
;
219 struct kvm_ldttss_desc
{
222 unsigned base1
:8, type
:5, dpl
:2, p
:1;
223 unsigned limit1
:4, zero0
:3, g
:1, base2
:8;
226 } __attribute__((packed
));
228 DEFINE_PER_CPU(struct svm_cpu_data
*, svm_data
);
231 * Only MSR_TSC_AUX is switched via the user return hook. EFER is switched via
232 * the VMCB, and the SYSCALL/SYSENTER MSRs are handled by VMLOAD/VMSAVE.
234 * RDTSCP and RDPID are not used in the kernel, specifically to allow KVM to
235 * defer the restoration of TSC_AUX until the CPU returns to userspace.
237 static int tsc_aux_uret_slot __read_mostly
= -1;
239 static const u32 msrpm_ranges
[] = {0, 0xc0000000, 0xc0010000};
241 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
242 #define MSRS_RANGE_SIZE 2048
243 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
245 u32
svm_msrpm_offset(u32 msr
)
250 for (i
= 0; i
< NUM_MSR_MAPS
; i
++) {
251 if (msr
< msrpm_ranges
[i
] ||
252 msr
>= msrpm_ranges
[i
] + MSRS_IN_RANGE
)
255 offset
= (msr
- msrpm_ranges
[i
]) / 4; /* 4 msrs per u8 */
256 offset
+= (i
* MSRS_RANGE_SIZE
); /* add range offset */
258 /* Now we have the u8 offset - but need the u32 offset */
262 /* MSR not in any range */
266 #define MAX_INST_SIZE 15
268 static int get_npt_level(void)
271 return pgtable_l5_enabled() ? PT64_ROOT_5LEVEL
: PT64_ROOT_4LEVEL
;
273 return PT32E_ROOT_LEVEL
;
277 int svm_set_efer(struct kvm_vcpu
*vcpu
, u64 efer
)
279 struct vcpu_svm
*svm
= to_svm(vcpu
);
280 u64 old_efer
= vcpu
->arch
.efer
;
281 vcpu
->arch
.efer
= efer
;
284 /* Shadow paging assumes NX to be available. */
287 if (!(efer
& EFER_LMA
))
291 if ((old_efer
& EFER_SVME
) != (efer
& EFER_SVME
)) {
292 if (!(efer
& EFER_SVME
)) {
293 svm_leave_nested(vcpu
);
294 svm_set_gif(svm
, true);
295 /* #GP intercept is still needed for vmware backdoor */
296 if (!enable_vmware_backdoor
)
297 clr_exception_intercept(svm
, GP_VECTOR
);
300 * Free the nested guest state, unless we are in SMM.
301 * In this case we will return to the nested guest
302 * as soon as we leave SMM.
305 svm_free_nested(svm
);
308 int ret
= svm_allocate_nested(svm
);
311 vcpu
->arch
.efer
= old_efer
;
316 * Never intercept #GP for SEV guests, KVM can't
317 * decrypt guest memory to workaround the erratum.
319 if (svm_gp_erratum_intercept
&& !sev_guest(vcpu
->kvm
))
320 set_exception_intercept(svm
, GP_VECTOR
);
324 svm
->vmcb
->save
.efer
= efer
| EFER_SVME
;
325 vmcb_mark_dirty(svm
->vmcb
, VMCB_CR
);
329 static int is_external_interrupt(u32 info
)
331 info
&= SVM_EVTINJ_TYPE_MASK
| SVM_EVTINJ_VALID
;
332 return info
== (SVM_EVTINJ_VALID
| SVM_EVTINJ_TYPE_INTR
);
335 static u32
svm_get_interrupt_shadow(struct kvm_vcpu
*vcpu
)
337 struct vcpu_svm
*svm
= to_svm(vcpu
);
340 if (svm
->vmcb
->control
.int_state
& SVM_INTERRUPT_SHADOW_MASK
)
341 ret
= KVM_X86_SHADOW_INT_STI
| KVM_X86_SHADOW_INT_MOV_SS
;
345 static void svm_set_interrupt_shadow(struct kvm_vcpu
*vcpu
, int mask
)
347 struct vcpu_svm
*svm
= to_svm(vcpu
);
350 svm
->vmcb
->control
.int_state
&= ~SVM_INTERRUPT_SHADOW_MASK
;
352 svm
->vmcb
->control
.int_state
|= SVM_INTERRUPT_SHADOW_MASK
;
356 static int skip_emulated_instruction(struct kvm_vcpu
*vcpu
)
358 struct vcpu_svm
*svm
= to_svm(vcpu
);
361 * SEV-ES does not expose the next RIP. The RIP update is controlled by
362 * the type of exit and the #VC handler in the guest.
364 if (sev_es_guest(vcpu
->kvm
))
367 if (nrips
&& svm
->vmcb
->control
.next_rip
!= 0) {
368 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS
));
369 svm
->next_rip
= svm
->vmcb
->control
.next_rip
;
372 if (!svm
->next_rip
) {
373 if (!kvm_emulate_instruction(vcpu
, EMULTYPE_SKIP
))
376 kvm_rip_write(vcpu
, svm
->next_rip
);
380 svm_set_interrupt_shadow(vcpu
, 0);
385 static void svm_queue_exception(struct kvm_vcpu
*vcpu
)
387 struct vcpu_svm
*svm
= to_svm(vcpu
);
388 unsigned nr
= vcpu
->arch
.exception
.nr
;
389 bool has_error_code
= vcpu
->arch
.exception
.has_error_code
;
390 u32 error_code
= vcpu
->arch
.exception
.error_code
;
392 kvm_deliver_exception_payload(vcpu
);
394 if (nr
== BP_VECTOR
&& !nrips
) {
395 unsigned long rip
, old_rip
= kvm_rip_read(vcpu
);
398 * For guest debugging where we have to reinject #BP if some
399 * INT3 is guest-owned:
400 * Emulate nRIP by moving RIP forward. Will fail if injection
401 * raises a fault that is not intercepted. Still better than
402 * failing in all cases.
404 (void)skip_emulated_instruction(vcpu
);
405 rip
= kvm_rip_read(vcpu
);
406 svm
->int3_rip
= rip
+ svm
->vmcb
->save
.cs
.base
;
407 svm
->int3_injected
= rip
- old_rip
;
410 svm
->vmcb
->control
.event_inj
= nr
412 | (has_error_code
? SVM_EVTINJ_VALID_ERR
: 0)
413 | SVM_EVTINJ_TYPE_EXEPT
;
414 svm
->vmcb
->control
.event_inj_err
= error_code
;
417 static void svm_init_erratum_383(void)
423 if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH
))
426 /* Use _safe variants to not break nested virtualization */
427 val
= native_read_msr_safe(MSR_AMD64_DC_CFG
, &err
);
433 low
= lower_32_bits(val
);
434 high
= upper_32_bits(val
);
436 native_write_msr_safe(MSR_AMD64_DC_CFG
, low
, high
);
438 erratum_383_found
= true;
441 static void svm_init_osvw(struct kvm_vcpu
*vcpu
)
444 * Guests should see errata 400 and 415 as fixed (assuming that
445 * HLT and IO instructions are intercepted).
447 vcpu
->arch
.osvw
.length
= (osvw_len
>= 3) ? (osvw_len
) : 3;
448 vcpu
->arch
.osvw
.status
= osvw_status
& ~(6ULL);
451 * By increasing VCPU's osvw.length to 3 we are telling the guest that
452 * all osvw.status bits inside that length, including bit 0 (which is
453 * reserved for erratum 298), are valid. However, if host processor's
454 * osvw_len is 0 then osvw_status[0] carries no information. We need to
455 * be conservative here and therefore we tell the guest that erratum 298
456 * is present (because we really don't know).
458 if (osvw_len
== 0 && boot_cpu_data
.x86
== 0x10)
459 vcpu
->arch
.osvw
.status
|= 1;
462 static int has_svm(void)
466 if (!cpu_has_svm(&msg
)) {
467 printk(KERN_INFO
"has_svm: %s\n", msg
);
471 if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT
)) {
472 pr_info("KVM is unsupported when running as an SEV guest\n");
479 static void svm_hardware_disable(void)
481 /* Make sure we clean up behind us */
483 wrmsrl(MSR_AMD64_TSC_RATIO
, TSC_RATIO_DEFAULT
);
487 amd_pmu_disable_virt();
490 static int svm_hardware_enable(void)
493 struct svm_cpu_data
*sd
;
495 struct desc_struct
*gdt
;
496 int me
= raw_smp_processor_id();
498 rdmsrl(MSR_EFER
, efer
);
499 if (efer
& EFER_SVME
)
503 pr_err("%s: err EOPNOTSUPP on %d\n", __func__
, me
);
506 sd
= per_cpu(svm_data
, me
);
508 pr_err("%s: svm_data is NULL on %d\n", __func__
, me
);
512 sd
->asid_generation
= 1;
513 sd
->max_asid
= cpuid_ebx(SVM_CPUID_FUNC
) - 1;
514 sd
->next_asid
= sd
->max_asid
+ 1;
515 sd
->min_asid
= max_sev_asid
+ 1;
517 gdt
= get_current_gdt_rw();
518 sd
->tss_desc
= (struct kvm_ldttss_desc
*)(gdt
+ GDT_ENTRY_TSS
);
520 wrmsrl(MSR_EFER
, efer
| EFER_SVME
);
522 wrmsrl(MSR_VM_HSAVE_PA
, __sme_page_pa(sd
->save_area
));
524 if (static_cpu_has(X86_FEATURE_TSCRATEMSR
)) {
526 * Set the default value, even if we don't use TSC scaling
527 * to avoid having stale value in the msr
529 wrmsrl(MSR_AMD64_TSC_RATIO
, TSC_RATIO_DEFAULT
);
530 __this_cpu_write(current_tsc_ratio
, TSC_RATIO_DEFAULT
);
537 * Note that it is possible to have a system with mixed processor
538 * revisions and therefore different OSVW bits. If bits are not the same
539 * on different processors then choose the worst case (i.e. if erratum
540 * is present on one processor and not on another then assume that the
541 * erratum is present everywhere).
543 if (cpu_has(&boot_cpu_data
, X86_FEATURE_OSVW
)) {
544 uint64_t len
, status
= 0;
547 len
= native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH
, &err
);
549 status
= native_read_msr_safe(MSR_AMD64_OSVW_STATUS
,
553 osvw_status
= osvw_len
= 0;
557 osvw_status
|= status
;
558 osvw_status
&= (1ULL << osvw_len
) - 1;
561 osvw_status
= osvw_len
= 0;
563 svm_init_erratum_383();
565 amd_pmu_enable_virt();
570 static void svm_cpu_uninit(int cpu
)
572 struct svm_cpu_data
*sd
= per_cpu(svm_data
, cpu
);
577 per_cpu(svm_data
, cpu
) = NULL
;
578 kfree(sd
->sev_vmcbs
);
579 __free_page(sd
->save_area
);
583 static int svm_cpu_init(int cpu
)
585 struct svm_cpu_data
*sd
;
588 sd
= kzalloc(sizeof(struct svm_cpu_data
), GFP_KERNEL
);
592 sd
->save_area
= alloc_page(GFP_KERNEL
| __GFP_ZERO
);
596 ret
= sev_cpu_init(sd
);
600 per_cpu(svm_data
, cpu
) = sd
;
605 __free_page(sd
->save_area
);
612 static int direct_access_msr_slot(u32 msr
)
616 for (i
= 0; direct_access_msrs
[i
].index
!= MSR_INVALID
; i
++)
617 if (direct_access_msrs
[i
].index
== msr
)
623 static void set_shadow_msr_intercept(struct kvm_vcpu
*vcpu
, u32 msr
, int read
,
626 struct vcpu_svm
*svm
= to_svm(vcpu
);
627 int slot
= direct_access_msr_slot(msr
);
632 /* Set the shadow bitmaps to the desired intercept states */
634 set_bit(slot
, svm
->shadow_msr_intercept
.read
);
636 clear_bit(slot
, svm
->shadow_msr_intercept
.read
);
639 set_bit(slot
, svm
->shadow_msr_intercept
.write
);
641 clear_bit(slot
, svm
->shadow_msr_intercept
.write
);
644 static bool valid_msr_intercept(u32 index
)
646 return direct_access_msr_slot(index
) != -ENOENT
;
649 static bool msr_write_intercepted(struct kvm_vcpu
*vcpu
, u32 msr
)
656 msrpm
= is_guest_mode(vcpu
) ? to_svm(vcpu
)->nested
.msrpm
:
659 offset
= svm_msrpm_offset(msr
);
660 bit_write
= 2 * (msr
& 0x0f) + 1;
663 BUG_ON(offset
== MSR_INVALID
);
665 return !!test_bit(bit_write
, &tmp
);
668 static void set_msr_interception_bitmap(struct kvm_vcpu
*vcpu
, u32
*msrpm
,
669 u32 msr
, int read
, int write
)
671 u8 bit_read
, bit_write
;
676 * If this warning triggers extend the direct_access_msrs list at the
677 * beginning of the file
679 WARN_ON(!valid_msr_intercept(msr
));
681 /* Enforce non allowed MSRs to trap */
682 if (read
&& !kvm_msr_allowed(vcpu
, msr
, KVM_MSR_FILTER_READ
))
685 if (write
&& !kvm_msr_allowed(vcpu
, msr
, KVM_MSR_FILTER_WRITE
))
688 offset
= svm_msrpm_offset(msr
);
689 bit_read
= 2 * (msr
& 0x0f);
690 bit_write
= 2 * (msr
& 0x0f) + 1;
693 BUG_ON(offset
== MSR_INVALID
);
695 read
? clear_bit(bit_read
, &tmp
) : set_bit(bit_read
, &tmp
);
696 write
? clear_bit(bit_write
, &tmp
) : set_bit(bit_write
, &tmp
);
700 svm_hv_vmcb_dirty_nested_enlightenments(vcpu
);
704 void set_msr_interception(struct kvm_vcpu
*vcpu
, u32
*msrpm
, u32 msr
,
707 set_shadow_msr_intercept(vcpu
, msr
, read
, write
);
708 set_msr_interception_bitmap(vcpu
, msrpm
, msr
, read
, write
);
711 u32
*svm_vcpu_alloc_msrpm(void)
713 unsigned int order
= get_order(MSRPM_SIZE
);
714 struct page
*pages
= alloc_pages(GFP_KERNEL_ACCOUNT
, order
);
720 msrpm
= page_address(pages
);
721 memset(msrpm
, 0xff, PAGE_SIZE
* (1 << order
));
726 void svm_vcpu_init_msrpm(struct kvm_vcpu
*vcpu
, u32
*msrpm
)
730 for (i
= 0; direct_access_msrs
[i
].index
!= MSR_INVALID
; i
++) {
731 if (!direct_access_msrs
[i
].always
)
733 set_msr_interception(vcpu
, msrpm
, direct_access_msrs
[i
].index
, 1, 1);
738 void svm_vcpu_free_msrpm(u32
*msrpm
)
740 __free_pages(virt_to_page(msrpm
), get_order(MSRPM_SIZE
));
743 static void svm_msr_filter_changed(struct kvm_vcpu
*vcpu
)
745 struct vcpu_svm
*svm
= to_svm(vcpu
);
749 * Set intercept permissions for all direct access MSRs again. They
750 * will automatically get filtered through the MSR filter, so we are
751 * back in sync after this.
753 for (i
= 0; direct_access_msrs
[i
].index
!= MSR_INVALID
; i
++) {
754 u32 msr
= direct_access_msrs
[i
].index
;
755 u32 read
= test_bit(i
, svm
->shadow_msr_intercept
.read
);
756 u32 write
= test_bit(i
, svm
->shadow_msr_intercept
.write
);
758 set_msr_interception_bitmap(vcpu
, svm
->msrpm
, msr
, read
, write
);
762 static void add_msr_offset(u32 offset
)
766 for (i
= 0; i
< MSRPM_OFFSETS
; ++i
) {
768 /* Offset already in list? */
769 if (msrpm_offsets
[i
] == offset
)
772 /* Slot used by another offset? */
773 if (msrpm_offsets
[i
] != MSR_INVALID
)
776 /* Add offset to list */
777 msrpm_offsets
[i
] = offset
;
783 * If this BUG triggers the msrpm_offsets table has an overflow. Just
784 * increase MSRPM_OFFSETS in this case.
789 static void init_msrpm_offsets(void)
793 memset(msrpm_offsets
, 0xff, sizeof(msrpm_offsets
));
795 for (i
= 0; direct_access_msrs
[i
].index
!= MSR_INVALID
; i
++) {
798 offset
= svm_msrpm_offset(direct_access_msrs
[i
].index
);
799 BUG_ON(offset
== MSR_INVALID
);
801 add_msr_offset(offset
);
805 static void svm_enable_lbrv(struct kvm_vcpu
*vcpu
)
807 struct vcpu_svm
*svm
= to_svm(vcpu
);
809 svm
->vmcb
->control
.virt_ext
|= LBR_CTL_ENABLE_MASK
;
810 set_msr_interception(vcpu
, svm
->msrpm
, MSR_IA32_LASTBRANCHFROMIP
, 1, 1);
811 set_msr_interception(vcpu
, svm
->msrpm
, MSR_IA32_LASTBRANCHTOIP
, 1, 1);
812 set_msr_interception(vcpu
, svm
->msrpm
, MSR_IA32_LASTINTFROMIP
, 1, 1);
813 set_msr_interception(vcpu
, svm
->msrpm
, MSR_IA32_LASTINTTOIP
, 1, 1);
816 static void svm_disable_lbrv(struct kvm_vcpu
*vcpu
)
818 struct vcpu_svm
*svm
= to_svm(vcpu
);
820 svm
->vmcb
->control
.virt_ext
&= ~LBR_CTL_ENABLE_MASK
;
821 set_msr_interception(vcpu
, svm
->msrpm
, MSR_IA32_LASTBRANCHFROMIP
, 0, 0);
822 set_msr_interception(vcpu
, svm
->msrpm
, MSR_IA32_LASTBRANCHTOIP
, 0, 0);
823 set_msr_interception(vcpu
, svm
->msrpm
, MSR_IA32_LASTINTFROMIP
, 0, 0);
824 set_msr_interception(vcpu
, svm
->msrpm
, MSR_IA32_LASTINTTOIP
, 0, 0);
827 void disable_nmi_singlestep(struct vcpu_svm
*svm
)
829 svm
->nmi_singlestep
= false;
831 if (!(svm
->vcpu
.guest_debug
& KVM_GUESTDBG_SINGLESTEP
)) {
832 /* Clear our flags if they were not set by the guest */
833 if (!(svm
->nmi_singlestep_guest_rflags
& X86_EFLAGS_TF
))
834 svm
->vmcb
->save
.rflags
&= ~X86_EFLAGS_TF
;
835 if (!(svm
->nmi_singlestep_guest_rflags
& X86_EFLAGS_RF
))
836 svm
->vmcb
->save
.rflags
&= ~X86_EFLAGS_RF
;
840 static void grow_ple_window(struct kvm_vcpu
*vcpu
)
842 struct vcpu_svm
*svm
= to_svm(vcpu
);
843 struct vmcb_control_area
*control
= &svm
->vmcb
->control
;
844 int old
= control
->pause_filter_count
;
846 control
->pause_filter_count
= __grow_ple_window(old
,
848 pause_filter_count_grow
,
849 pause_filter_count_max
);
851 if (control
->pause_filter_count
!= old
) {
852 vmcb_mark_dirty(svm
->vmcb
, VMCB_INTERCEPTS
);
853 trace_kvm_ple_window_update(vcpu
->vcpu_id
,
854 control
->pause_filter_count
, old
);
858 static void shrink_ple_window(struct kvm_vcpu
*vcpu
)
860 struct vcpu_svm
*svm
= to_svm(vcpu
);
861 struct vmcb_control_area
*control
= &svm
->vmcb
->control
;
862 int old
= control
->pause_filter_count
;
864 control
->pause_filter_count
=
865 __shrink_ple_window(old
,
867 pause_filter_count_shrink
,
869 if (control
->pause_filter_count
!= old
) {
870 vmcb_mark_dirty(svm
->vmcb
, VMCB_INTERCEPTS
);
871 trace_kvm_ple_window_update(vcpu
->vcpu_id
,
872 control
->pause_filter_count
, old
);
876 static void svm_hardware_teardown(void)
880 sev_hardware_teardown();
882 for_each_possible_cpu(cpu
)
885 __free_pages(pfn_to_page(iopm_base
>> PAGE_SHIFT
),
886 get_order(IOPM_SIZE
));
890 static void init_seg(struct vmcb_seg
*seg
)
893 seg
->attrib
= SVM_SELECTOR_P_MASK
| SVM_SELECTOR_S_MASK
|
894 SVM_SELECTOR_WRITE_MASK
; /* Read/Write Data Segment */
899 static void init_sys_seg(struct vmcb_seg
*seg
, uint32_t type
)
902 seg
->attrib
= SVM_SELECTOR_P_MASK
| type
;
907 static u64
svm_get_l2_tsc_offset(struct kvm_vcpu
*vcpu
)
909 struct vcpu_svm
*svm
= to_svm(vcpu
);
911 return svm
->nested
.ctl
.tsc_offset
;
914 static u64
svm_get_l2_tsc_multiplier(struct kvm_vcpu
*vcpu
)
916 struct vcpu_svm
*svm
= to_svm(vcpu
);
918 return svm
->tsc_ratio_msr
;
921 static void svm_write_tsc_offset(struct kvm_vcpu
*vcpu
, u64 offset
)
923 struct vcpu_svm
*svm
= to_svm(vcpu
);
925 svm
->vmcb01
.ptr
->control
.tsc_offset
= vcpu
->arch
.l1_tsc_offset
;
926 svm
->vmcb
->control
.tsc_offset
= offset
;
927 vmcb_mark_dirty(svm
->vmcb
, VMCB_INTERCEPTS
);
930 void svm_write_tsc_multiplier(struct kvm_vcpu
*vcpu
, u64 multiplier
)
932 wrmsrl(MSR_AMD64_TSC_RATIO
, multiplier
);
935 /* Evaluate instruction intercepts that depend on guest CPUID features. */
936 static void svm_recalc_instruction_intercepts(struct kvm_vcpu
*vcpu
,
937 struct vcpu_svm
*svm
)
940 * Intercept INVPCID if shadow paging is enabled to sync/free shadow
941 * roots, or if INVPCID is disabled in the guest to inject #UD.
943 if (kvm_cpu_cap_has(X86_FEATURE_INVPCID
)) {
945 !guest_cpuid_has(&svm
->vcpu
, X86_FEATURE_INVPCID
))
946 svm_set_intercept(svm
, INTERCEPT_INVPCID
);
948 svm_clr_intercept(svm
, INTERCEPT_INVPCID
);
951 if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP
)) {
952 if (guest_cpuid_has(vcpu
, X86_FEATURE_RDTSCP
))
953 svm_clr_intercept(svm
, INTERCEPT_RDTSCP
);
955 svm_set_intercept(svm
, INTERCEPT_RDTSCP
);
959 static inline void init_vmcb_after_set_cpuid(struct kvm_vcpu
*vcpu
)
961 struct vcpu_svm
*svm
= to_svm(vcpu
);
963 if (guest_cpuid_is_intel(vcpu
)) {
965 * We must intercept SYSENTER_EIP and SYSENTER_ESP
966 * accesses because the processor only stores 32 bits.
967 * For the same reason we cannot use virtual VMLOAD/VMSAVE.
969 svm_set_intercept(svm
, INTERCEPT_VMLOAD
);
970 svm_set_intercept(svm
, INTERCEPT_VMSAVE
);
971 svm
->vmcb
->control
.virt_ext
&= ~VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK
;
973 set_msr_interception(vcpu
, svm
->msrpm
, MSR_IA32_SYSENTER_EIP
, 0, 0);
974 set_msr_interception(vcpu
, svm
->msrpm
, MSR_IA32_SYSENTER_ESP
, 0, 0);
977 * If hardware supports Virtual VMLOAD VMSAVE then enable it
978 * in VMCB and clear intercepts to avoid #VMEXIT.
981 svm_clr_intercept(svm
, INTERCEPT_VMLOAD
);
982 svm_clr_intercept(svm
, INTERCEPT_VMSAVE
);
983 svm
->vmcb
->control
.virt_ext
|= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK
;
985 /* No need to intercept these MSRs */
986 set_msr_interception(vcpu
, svm
->msrpm
, MSR_IA32_SYSENTER_EIP
, 1, 1);
987 set_msr_interception(vcpu
, svm
->msrpm
, MSR_IA32_SYSENTER_ESP
, 1, 1);
991 static void init_vmcb(struct kvm_vcpu
*vcpu
)
993 struct vcpu_svm
*svm
= to_svm(vcpu
);
994 struct vmcb_control_area
*control
= &svm
->vmcb
->control
;
995 struct vmcb_save_area
*save
= &svm
->vmcb
->save
;
997 svm_set_intercept(svm
, INTERCEPT_CR0_READ
);
998 svm_set_intercept(svm
, INTERCEPT_CR3_READ
);
999 svm_set_intercept(svm
, INTERCEPT_CR4_READ
);
1000 svm_set_intercept(svm
, INTERCEPT_CR0_WRITE
);
1001 svm_set_intercept(svm
, INTERCEPT_CR3_WRITE
);
1002 svm_set_intercept(svm
, INTERCEPT_CR4_WRITE
);
1003 if (!kvm_vcpu_apicv_active(vcpu
))
1004 svm_set_intercept(svm
, INTERCEPT_CR8_WRITE
);
1006 set_dr_intercepts(svm
);
1008 set_exception_intercept(svm
, PF_VECTOR
);
1009 set_exception_intercept(svm
, UD_VECTOR
);
1010 set_exception_intercept(svm
, MC_VECTOR
);
1011 set_exception_intercept(svm
, AC_VECTOR
);
1012 set_exception_intercept(svm
, DB_VECTOR
);
1014 * Guest access to VMware backdoor ports could legitimately
1015 * trigger #GP because of TSS I/O permission bitmap.
1016 * We intercept those #GP and allow access to them anyway
1017 * as VMware does. Don't intercept #GP for SEV guests as KVM can't
1018 * decrypt guest memory to decode the faulting instruction.
1020 if (enable_vmware_backdoor
&& !sev_guest(vcpu
->kvm
))
1021 set_exception_intercept(svm
, GP_VECTOR
);
1023 svm_set_intercept(svm
, INTERCEPT_INTR
);
1024 svm_set_intercept(svm
, INTERCEPT_NMI
);
1027 svm_set_intercept(svm
, INTERCEPT_SMI
);
1029 svm_set_intercept(svm
, INTERCEPT_SELECTIVE_CR0
);
1030 svm_set_intercept(svm
, INTERCEPT_RDPMC
);
1031 svm_set_intercept(svm
, INTERCEPT_CPUID
);
1032 svm_set_intercept(svm
, INTERCEPT_INVD
);
1033 svm_set_intercept(svm
, INTERCEPT_INVLPG
);
1034 svm_set_intercept(svm
, INTERCEPT_INVLPGA
);
1035 svm_set_intercept(svm
, INTERCEPT_IOIO_PROT
);
1036 svm_set_intercept(svm
, INTERCEPT_MSR_PROT
);
1037 svm_set_intercept(svm
, INTERCEPT_TASK_SWITCH
);
1038 svm_set_intercept(svm
, INTERCEPT_SHUTDOWN
);
1039 svm_set_intercept(svm
, INTERCEPT_VMRUN
);
1040 svm_set_intercept(svm
, INTERCEPT_VMMCALL
);
1041 svm_set_intercept(svm
, INTERCEPT_VMLOAD
);
1042 svm_set_intercept(svm
, INTERCEPT_VMSAVE
);
1043 svm_set_intercept(svm
, INTERCEPT_STGI
);
1044 svm_set_intercept(svm
, INTERCEPT_CLGI
);
1045 svm_set_intercept(svm
, INTERCEPT_SKINIT
);
1046 svm_set_intercept(svm
, INTERCEPT_WBINVD
);
1047 svm_set_intercept(svm
, INTERCEPT_XSETBV
);
1048 svm_set_intercept(svm
, INTERCEPT_RDPRU
);
1049 svm_set_intercept(svm
, INTERCEPT_RSM
);
1051 if (!kvm_mwait_in_guest(vcpu
->kvm
)) {
1052 svm_set_intercept(svm
, INTERCEPT_MONITOR
);
1053 svm_set_intercept(svm
, INTERCEPT_MWAIT
);
1056 if (!kvm_hlt_in_guest(vcpu
->kvm
))
1057 svm_set_intercept(svm
, INTERCEPT_HLT
);
1059 control
->iopm_base_pa
= __sme_set(iopm_base
);
1060 control
->msrpm_base_pa
= __sme_set(__pa(svm
->msrpm
));
1061 control
->int_ctl
= V_INTR_MASKING_MASK
;
1063 init_seg(&save
->es
);
1064 init_seg(&save
->ss
);
1065 init_seg(&save
->ds
);
1066 init_seg(&save
->fs
);
1067 init_seg(&save
->gs
);
1069 save
->cs
.selector
= 0xf000;
1070 save
->cs
.base
= 0xffff0000;
1071 /* Executable/Readable Code Segment */
1072 save
->cs
.attrib
= SVM_SELECTOR_READ_MASK
| SVM_SELECTOR_P_MASK
|
1073 SVM_SELECTOR_S_MASK
| SVM_SELECTOR_CODE_MASK
;
1074 save
->cs
.limit
= 0xffff;
1076 save
->gdtr
.base
= 0;
1077 save
->gdtr
.limit
= 0xffff;
1078 save
->idtr
.base
= 0;
1079 save
->idtr
.limit
= 0xffff;
1081 init_sys_seg(&save
->ldtr
, SEG_TYPE_LDT
);
1082 init_sys_seg(&save
->tr
, SEG_TYPE_BUSY_TSS16
);
1085 /* Setup VMCB for Nested Paging */
1086 control
->nested_ctl
|= SVM_NESTED_CTL_NP_ENABLE
;
1087 svm_clr_intercept(svm
, INTERCEPT_INVLPG
);
1088 clr_exception_intercept(svm
, PF_VECTOR
);
1089 svm_clr_intercept(svm
, INTERCEPT_CR3_READ
);
1090 svm_clr_intercept(svm
, INTERCEPT_CR3_WRITE
);
1091 save
->g_pat
= vcpu
->arch
.pat
;
1094 svm
->current_vmcb
->asid_generation
= 0;
1097 svm
->nested
.vmcb12_gpa
= INVALID_GPA
;
1098 svm
->nested
.last_vmcb12_gpa
= INVALID_GPA
;
1100 if (!kvm_pause_in_guest(vcpu
->kvm
)) {
1101 control
->pause_filter_count
= pause_filter_count
;
1102 if (pause_filter_thresh
)
1103 control
->pause_filter_thresh
= pause_filter_thresh
;
1104 svm_set_intercept(svm
, INTERCEPT_PAUSE
);
1106 svm_clr_intercept(svm
, INTERCEPT_PAUSE
);
1109 svm_recalc_instruction_intercepts(vcpu
, svm
);
1112 * If the host supports V_SPEC_CTRL then disable the interception
1113 * of MSR_IA32_SPEC_CTRL.
1115 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL
))
1116 set_msr_interception(vcpu
, svm
->msrpm
, MSR_IA32_SPEC_CTRL
, 1, 1);
1118 if (kvm_vcpu_apicv_active(vcpu
))
1119 avic_init_vmcb(svm
);
1122 svm_clr_intercept(svm
, INTERCEPT_STGI
);
1123 svm_clr_intercept(svm
, INTERCEPT_CLGI
);
1124 svm
->vmcb
->control
.int_ctl
|= V_GIF_ENABLE_MASK
;
1127 if (sev_guest(vcpu
->kvm
)) {
1128 svm
->vmcb
->control
.nested_ctl
|= SVM_NESTED_CTL_SEV_ENABLE
;
1129 clr_exception_intercept(svm
, UD_VECTOR
);
1131 if (sev_es_guest(vcpu
->kvm
)) {
1132 /* Perform SEV-ES specific VMCB updates */
1133 sev_es_init_vmcb(svm
);
1137 svm_hv_init_vmcb(svm
->vmcb
);
1138 init_vmcb_after_set_cpuid(vcpu
);
1140 vmcb_mark_all_dirty(svm
->vmcb
);
1145 static void __svm_vcpu_reset(struct kvm_vcpu
*vcpu
)
1147 struct vcpu_svm
*svm
= to_svm(vcpu
);
1149 svm_vcpu_init_msrpm(vcpu
, svm
->msrpm
);
1151 svm_init_osvw(vcpu
);
1152 vcpu
->arch
.microcode_version
= 0x01000065;
1153 svm
->tsc_ratio_msr
= kvm_default_tsc_scaling_ratio
;
1155 if (sev_es_guest(vcpu
->kvm
))
1156 sev_es_vcpu_reset(svm
);
1159 static void svm_vcpu_reset(struct kvm_vcpu
*vcpu
, bool init_event
)
1161 struct vcpu_svm
*svm
= to_svm(vcpu
);
1164 svm
->virt_spec_ctrl
= 0;
1169 __svm_vcpu_reset(vcpu
);
1172 void svm_switch_vmcb(struct vcpu_svm
*svm
, struct kvm_vmcb_info
*target_vmcb
)
1174 svm
->current_vmcb
= target_vmcb
;
1175 svm
->vmcb
= target_vmcb
->ptr
;
1178 static int svm_create_vcpu(struct kvm_vcpu
*vcpu
)
1180 struct vcpu_svm
*svm
;
1181 struct page
*vmcb01_page
;
1182 struct page
*vmsa_page
= NULL
;
1185 BUILD_BUG_ON(offsetof(struct vcpu_svm
, vcpu
) != 0);
1189 vmcb01_page
= alloc_page(GFP_KERNEL_ACCOUNT
| __GFP_ZERO
);
1193 if (sev_es_guest(vcpu
->kvm
)) {
1195 * SEV-ES guests require a separate VMSA page used to contain
1196 * the encrypted register state of the guest.
1198 vmsa_page
= alloc_page(GFP_KERNEL_ACCOUNT
| __GFP_ZERO
);
1200 goto error_free_vmcb_page
;
1203 * SEV-ES guests maintain an encrypted version of their FPU
1204 * state which is restored and saved on VMRUN and VMEXIT.
1205 * Mark vcpu->arch.guest_fpu->fpstate as scratch so it won't
1206 * do xsave/xrstor on it.
1208 fpstate_set_confidential(&vcpu
->arch
.guest_fpu
);
1211 err
= avic_init_vcpu(svm
);
1213 goto error_free_vmsa_page
;
1215 svm
->msrpm
= svm_vcpu_alloc_msrpm();
1218 goto error_free_vmsa_page
;
1221 svm
->vmcb01
.ptr
= page_address(vmcb01_page
);
1222 svm
->vmcb01
.pa
= __sme_set(page_to_pfn(vmcb01_page
) << PAGE_SHIFT
);
1223 svm_switch_vmcb(svm
, &svm
->vmcb01
);
1226 svm
->sev_es
.vmsa
= page_address(vmsa_page
);
1228 svm
->guest_state_loaded
= false;
1232 error_free_vmsa_page
:
1234 __free_page(vmsa_page
);
1235 error_free_vmcb_page
:
1236 __free_page(vmcb01_page
);
1241 static void svm_clear_current_vmcb(struct vmcb
*vmcb
)
1245 for_each_online_cpu(i
)
1246 cmpxchg(&per_cpu(svm_data
, i
)->current_vmcb
, vmcb
, NULL
);
1249 static void svm_free_vcpu(struct kvm_vcpu
*vcpu
)
1251 struct vcpu_svm
*svm
= to_svm(vcpu
);
1254 * The vmcb page can be recycled, causing a false negative in
1255 * svm_vcpu_load(). So, ensure that no logical CPU has this
1256 * vmcb page recorded as its current vmcb.
1258 svm_clear_current_vmcb(svm
->vmcb
);
1260 svm_free_nested(svm
);
1262 sev_free_vcpu(vcpu
);
1264 __free_page(pfn_to_page(__sme_clr(svm
->vmcb01
.pa
) >> PAGE_SHIFT
));
1265 __free_pages(virt_to_page(svm
->msrpm
), get_order(MSRPM_SIZE
));
1268 static void svm_prepare_guest_switch(struct kvm_vcpu
*vcpu
)
1270 struct vcpu_svm
*svm
= to_svm(vcpu
);
1271 struct svm_cpu_data
*sd
= per_cpu(svm_data
, vcpu
->cpu
);
1273 if (sev_es_guest(vcpu
->kvm
))
1274 sev_es_unmap_ghcb(svm
);
1276 if (svm
->guest_state_loaded
)
1280 * Save additional host state that will be restored on VMEXIT (sev-es)
1281 * or subsequent vmload of host save area.
1283 if (sev_es_guest(vcpu
->kvm
)) {
1284 sev_es_prepare_guest_switch(svm
, vcpu
->cpu
);
1286 vmsave(__sme_page_pa(sd
->save_area
));
1290 u64 tsc_ratio
= vcpu
->arch
.tsc_scaling_ratio
;
1291 if (tsc_ratio
!= __this_cpu_read(current_tsc_ratio
)) {
1292 __this_cpu_write(current_tsc_ratio
, tsc_ratio
);
1293 wrmsrl(MSR_AMD64_TSC_RATIO
, tsc_ratio
);
1297 if (likely(tsc_aux_uret_slot
>= 0))
1298 kvm_set_user_return_msr(tsc_aux_uret_slot
, svm
->tsc_aux
, -1ull);
1300 svm
->guest_state_loaded
= true;
1303 static void svm_prepare_host_switch(struct kvm_vcpu
*vcpu
)
1305 to_svm(vcpu
)->guest_state_loaded
= false;
1308 static void svm_vcpu_load(struct kvm_vcpu
*vcpu
, int cpu
)
1310 struct vcpu_svm
*svm
= to_svm(vcpu
);
1311 struct svm_cpu_data
*sd
= per_cpu(svm_data
, cpu
);
1313 if (sd
->current_vmcb
!= svm
->vmcb
) {
1314 sd
->current_vmcb
= svm
->vmcb
;
1315 indirect_branch_prediction_barrier();
1317 if (kvm_vcpu_apicv_active(vcpu
))
1318 avic_vcpu_load(vcpu
, cpu
);
1321 static void svm_vcpu_put(struct kvm_vcpu
*vcpu
)
1323 if (kvm_vcpu_apicv_active(vcpu
))
1324 avic_vcpu_put(vcpu
);
1326 svm_prepare_host_switch(vcpu
);
1328 ++vcpu
->stat
.host_state_reload
;
1331 static unsigned long svm_get_rflags(struct kvm_vcpu
*vcpu
)
1333 struct vcpu_svm
*svm
= to_svm(vcpu
);
1334 unsigned long rflags
= svm
->vmcb
->save
.rflags
;
1336 if (svm
->nmi_singlestep
) {
1337 /* Hide our flags if they were not set by the guest */
1338 if (!(svm
->nmi_singlestep_guest_rflags
& X86_EFLAGS_TF
))
1339 rflags
&= ~X86_EFLAGS_TF
;
1340 if (!(svm
->nmi_singlestep_guest_rflags
& X86_EFLAGS_RF
))
1341 rflags
&= ~X86_EFLAGS_RF
;
1346 static void svm_set_rflags(struct kvm_vcpu
*vcpu
, unsigned long rflags
)
1348 if (to_svm(vcpu
)->nmi_singlestep
)
1349 rflags
|= (X86_EFLAGS_TF
| X86_EFLAGS_RF
);
1352 * Any change of EFLAGS.VM is accompanied by a reload of SS
1353 * (caused by either a task switch or an inter-privilege IRET),
1354 * so we do not need to update the CPL here.
1356 to_svm(vcpu
)->vmcb
->save
.rflags
= rflags
;
1359 static bool svm_get_if_flag(struct kvm_vcpu
*vcpu
)
1361 struct vmcb
*vmcb
= to_svm(vcpu
)->vmcb
;
1363 return sev_es_guest(vcpu
->kvm
)
1364 ? vmcb
->control
.int_state
& SVM_GUEST_INTERRUPT_MASK
1365 : kvm_get_rflags(vcpu
) & X86_EFLAGS_IF
;
1368 static void svm_cache_reg(struct kvm_vcpu
*vcpu
, enum kvm_reg reg
)
1370 kvm_register_mark_available(vcpu
, reg
);
1373 case VCPU_EXREG_PDPTR
:
1375 * When !npt_enabled, mmu->pdptrs[] is already available since
1376 * it is always updated per SDM when moving to CRs.
1379 load_pdptrs(vcpu
, kvm_read_cr3(vcpu
));
1382 KVM_BUG_ON(1, vcpu
->kvm
);
1386 static void svm_set_vintr(struct vcpu_svm
*svm
)
1388 struct vmcb_control_area
*control
;
1391 * The following fields are ignored when AVIC is enabled
1393 WARN_ON(kvm_apicv_activated(svm
->vcpu
.kvm
));
1395 svm_set_intercept(svm
, INTERCEPT_VINTR
);
1398 * This is just a dummy VINTR to actually cause a vmexit to happen.
1399 * Actual injection of virtual interrupts happens through EVENTINJ.
1401 control
= &svm
->vmcb
->control
;
1402 control
->int_vector
= 0x0;
1403 control
->int_ctl
&= ~V_INTR_PRIO_MASK
;
1404 control
->int_ctl
|= V_IRQ_MASK
|
1405 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT
);
1406 vmcb_mark_dirty(svm
->vmcb
, VMCB_INTR
);
1409 static void svm_clear_vintr(struct vcpu_svm
*svm
)
1411 svm_clr_intercept(svm
, INTERCEPT_VINTR
);
1413 /* Drop int_ctl fields related to VINTR injection. */
1414 svm
->vmcb
->control
.int_ctl
&= ~V_IRQ_INJECTION_BITS_MASK
;
1415 if (is_guest_mode(&svm
->vcpu
)) {
1416 svm
->vmcb01
.ptr
->control
.int_ctl
&= ~V_IRQ_INJECTION_BITS_MASK
;
1418 WARN_ON((svm
->vmcb
->control
.int_ctl
& V_TPR_MASK
) !=
1419 (svm
->nested
.ctl
.int_ctl
& V_TPR_MASK
));
1421 svm
->vmcb
->control
.int_ctl
|= svm
->nested
.ctl
.int_ctl
&
1422 V_IRQ_INJECTION_BITS_MASK
;
1424 svm
->vmcb
->control
.int_vector
= svm
->nested
.ctl
.int_vector
;
1427 vmcb_mark_dirty(svm
->vmcb
, VMCB_INTR
);
1430 static struct vmcb_seg
*svm_seg(struct kvm_vcpu
*vcpu
, int seg
)
1432 struct vmcb_save_area
*save
= &to_svm(vcpu
)->vmcb
->save
;
1433 struct vmcb_save_area
*save01
= &to_svm(vcpu
)->vmcb01
.ptr
->save
;
1436 case VCPU_SREG_CS
: return &save
->cs
;
1437 case VCPU_SREG_DS
: return &save
->ds
;
1438 case VCPU_SREG_ES
: return &save
->es
;
1439 case VCPU_SREG_FS
: return &save01
->fs
;
1440 case VCPU_SREG_GS
: return &save01
->gs
;
1441 case VCPU_SREG_SS
: return &save
->ss
;
1442 case VCPU_SREG_TR
: return &save01
->tr
;
1443 case VCPU_SREG_LDTR
: return &save01
->ldtr
;
1449 static u64
svm_get_segment_base(struct kvm_vcpu
*vcpu
, int seg
)
1451 struct vmcb_seg
*s
= svm_seg(vcpu
, seg
);
1456 static void svm_get_segment(struct kvm_vcpu
*vcpu
,
1457 struct kvm_segment
*var
, int seg
)
1459 struct vmcb_seg
*s
= svm_seg(vcpu
, seg
);
1461 var
->base
= s
->base
;
1462 var
->limit
= s
->limit
;
1463 var
->selector
= s
->selector
;
1464 var
->type
= s
->attrib
& SVM_SELECTOR_TYPE_MASK
;
1465 var
->s
= (s
->attrib
>> SVM_SELECTOR_S_SHIFT
) & 1;
1466 var
->dpl
= (s
->attrib
>> SVM_SELECTOR_DPL_SHIFT
) & 3;
1467 var
->present
= (s
->attrib
>> SVM_SELECTOR_P_SHIFT
) & 1;
1468 var
->avl
= (s
->attrib
>> SVM_SELECTOR_AVL_SHIFT
) & 1;
1469 var
->l
= (s
->attrib
>> SVM_SELECTOR_L_SHIFT
) & 1;
1470 var
->db
= (s
->attrib
>> SVM_SELECTOR_DB_SHIFT
) & 1;
1473 * AMD CPUs circa 2014 track the G bit for all segments except CS.
1474 * However, the SVM spec states that the G bit is not observed by the
1475 * CPU, and some VMware virtual CPUs drop the G bit for all segments.
1476 * So let's synthesize a legal G bit for all segments, this helps
1477 * running KVM nested. It also helps cross-vendor migration, because
1478 * Intel's vmentry has a check on the 'G' bit.
1480 var
->g
= s
->limit
> 0xfffff;
1483 * AMD's VMCB does not have an explicit unusable field, so emulate it
1484 * for cross vendor migration purposes by "not present"
1486 var
->unusable
= !var
->present
;
1491 * Work around a bug where the busy flag in the tr selector
1501 * The accessed bit must always be set in the segment
1502 * descriptor cache, although it can be cleared in the
1503 * descriptor, the cached bit always remains at 1. Since
1504 * Intel has a check on this, set it here to support
1505 * cross-vendor migration.
1512 * On AMD CPUs sometimes the DB bit in the segment
1513 * descriptor is left as 1, although the whole segment has
1514 * been made unusable. Clear it here to pass an Intel VMX
1515 * entry check when cross vendor migrating.
1519 /* This is symmetric with svm_set_segment() */
1520 var
->dpl
= to_svm(vcpu
)->vmcb
->save
.cpl
;
1525 static int svm_get_cpl(struct kvm_vcpu
*vcpu
)
1527 struct vmcb_save_area
*save
= &to_svm(vcpu
)->vmcb
->save
;
1532 static void svm_get_idt(struct kvm_vcpu
*vcpu
, struct desc_ptr
*dt
)
1534 struct vcpu_svm
*svm
= to_svm(vcpu
);
1536 dt
->size
= svm
->vmcb
->save
.idtr
.limit
;
1537 dt
->address
= svm
->vmcb
->save
.idtr
.base
;
1540 static void svm_set_idt(struct kvm_vcpu
*vcpu
, struct desc_ptr
*dt
)
1542 struct vcpu_svm
*svm
= to_svm(vcpu
);
1544 svm
->vmcb
->save
.idtr
.limit
= dt
->size
;
1545 svm
->vmcb
->save
.idtr
.base
= dt
->address
;
1546 vmcb_mark_dirty(svm
->vmcb
, VMCB_DT
);
1549 static void svm_get_gdt(struct kvm_vcpu
*vcpu
, struct desc_ptr
*dt
)
1551 struct vcpu_svm
*svm
= to_svm(vcpu
);
1553 dt
->size
= svm
->vmcb
->save
.gdtr
.limit
;
1554 dt
->address
= svm
->vmcb
->save
.gdtr
.base
;
1557 static void svm_set_gdt(struct kvm_vcpu
*vcpu
, struct desc_ptr
*dt
)
1559 struct vcpu_svm
*svm
= to_svm(vcpu
);
1561 svm
->vmcb
->save
.gdtr
.limit
= dt
->size
;
1562 svm
->vmcb
->save
.gdtr
.base
= dt
->address
;
1563 vmcb_mark_dirty(svm
->vmcb
, VMCB_DT
);
1566 static void svm_post_set_cr3(struct kvm_vcpu
*vcpu
, unsigned long cr3
)
1568 struct vcpu_svm
*svm
= to_svm(vcpu
);
1571 * For guests that don't set guest_state_protected, the cr3 update is
1572 * handled via kvm_mmu_load() while entering the guest. For guests
1573 * that do (SEV-ES/SEV-SNP), the cr3 update needs to be written to
1574 * VMCB save area now, since the save area will become the initial
1575 * contents of the VMSA, and future VMCB save area updates won't be
1578 if (sev_es_guest(vcpu
->kvm
)) {
1579 svm
->vmcb
->save
.cr3
= cr3
;
1580 vmcb_mark_dirty(svm
->vmcb
, VMCB_CR
);
1584 void svm_set_cr0(struct kvm_vcpu
*vcpu
, unsigned long cr0
)
1586 struct vcpu_svm
*svm
= to_svm(vcpu
);
1589 #ifdef CONFIG_X86_64
1590 if (vcpu
->arch
.efer
& EFER_LME
&& !vcpu
->arch
.guest_state_protected
) {
1591 if (!is_paging(vcpu
) && (cr0
& X86_CR0_PG
)) {
1592 vcpu
->arch
.efer
|= EFER_LMA
;
1593 svm
->vmcb
->save
.efer
|= EFER_LMA
| EFER_LME
;
1596 if (is_paging(vcpu
) && !(cr0
& X86_CR0_PG
)) {
1597 vcpu
->arch
.efer
&= ~EFER_LMA
;
1598 svm
->vmcb
->save
.efer
&= ~(EFER_LMA
| EFER_LME
);
1602 vcpu
->arch
.cr0
= cr0
;
1605 hcr0
|= X86_CR0_PG
| X86_CR0_WP
;
1608 * re-enable caching here because the QEMU bios
1609 * does not do it - this results in some delay at
1612 if (kvm_check_has_quirk(vcpu
->kvm
, KVM_X86_QUIRK_CD_NW_CLEARED
))
1613 hcr0
&= ~(X86_CR0_CD
| X86_CR0_NW
);
1615 svm
->vmcb
->save
.cr0
= hcr0
;
1616 vmcb_mark_dirty(svm
->vmcb
, VMCB_CR
);
1619 * SEV-ES guests must always keep the CR intercepts cleared. CR
1620 * tracking is done using the CR write traps.
1622 if (sev_es_guest(vcpu
->kvm
))
1626 /* Selective CR0 write remains on. */
1627 svm_clr_intercept(svm
, INTERCEPT_CR0_READ
);
1628 svm_clr_intercept(svm
, INTERCEPT_CR0_WRITE
);
1630 svm_set_intercept(svm
, INTERCEPT_CR0_READ
);
1631 svm_set_intercept(svm
, INTERCEPT_CR0_WRITE
);
1635 static bool svm_is_valid_cr4(struct kvm_vcpu
*vcpu
, unsigned long cr4
)
1640 void svm_set_cr4(struct kvm_vcpu
*vcpu
, unsigned long cr4
)
1642 unsigned long host_cr4_mce
= cr4_read_shadow() & X86_CR4_MCE
;
1643 unsigned long old_cr4
= vcpu
->arch
.cr4
;
1645 if (npt_enabled
&& ((old_cr4
^ cr4
) & X86_CR4_PGE
))
1646 svm_flush_tlb(vcpu
);
1648 vcpu
->arch
.cr4
= cr4
;
1651 cr4
|= host_cr4_mce
;
1652 to_svm(vcpu
)->vmcb
->save
.cr4
= cr4
;
1653 vmcb_mark_dirty(to_svm(vcpu
)->vmcb
, VMCB_CR
);
1655 if ((cr4
^ old_cr4
) & (X86_CR4_OSXSAVE
| X86_CR4_PKE
))
1656 kvm_update_cpuid_runtime(vcpu
);
1659 static void svm_set_segment(struct kvm_vcpu
*vcpu
,
1660 struct kvm_segment
*var
, int seg
)
1662 struct vcpu_svm
*svm
= to_svm(vcpu
);
1663 struct vmcb_seg
*s
= svm_seg(vcpu
, seg
);
1665 s
->base
= var
->base
;
1666 s
->limit
= var
->limit
;
1667 s
->selector
= var
->selector
;
1668 s
->attrib
= (var
->type
& SVM_SELECTOR_TYPE_MASK
);
1669 s
->attrib
|= (var
->s
& 1) << SVM_SELECTOR_S_SHIFT
;
1670 s
->attrib
|= (var
->dpl
& 3) << SVM_SELECTOR_DPL_SHIFT
;
1671 s
->attrib
|= ((var
->present
& 1) && !var
->unusable
) << SVM_SELECTOR_P_SHIFT
;
1672 s
->attrib
|= (var
->avl
& 1) << SVM_SELECTOR_AVL_SHIFT
;
1673 s
->attrib
|= (var
->l
& 1) << SVM_SELECTOR_L_SHIFT
;
1674 s
->attrib
|= (var
->db
& 1) << SVM_SELECTOR_DB_SHIFT
;
1675 s
->attrib
|= (var
->g
& 1) << SVM_SELECTOR_G_SHIFT
;
1678 * This is always accurate, except if SYSRET returned to a segment
1679 * with SS.DPL != 3. Intel does not have this quirk, and always
1680 * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
1681 * would entail passing the CPL to userspace and back.
1683 if (seg
== VCPU_SREG_SS
)
1684 /* This is symmetric with svm_get_segment() */
1685 svm
->vmcb
->save
.cpl
= (var
->dpl
& 3);
1687 vmcb_mark_dirty(svm
->vmcb
, VMCB_SEG
);
1690 static void svm_update_exception_bitmap(struct kvm_vcpu
*vcpu
)
1692 struct vcpu_svm
*svm
= to_svm(vcpu
);
1694 clr_exception_intercept(svm
, BP_VECTOR
);
1696 if (vcpu
->guest_debug
& KVM_GUESTDBG_ENABLE
) {
1697 if (vcpu
->guest_debug
& KVM_GUESTDBG_USE_SW_BP
)
1698 set_exception_intercept(svm
, BP_VECTOR
);
1702 static void new_asid(struct vcpu_svm
*svm
, struct svm_cpu_data
*sd
)
1704 if (sd
->next_asid
> sd
->max_asid
) {
1705 ++sd
->asid_generation
;
1706 sd
->next_asid
= sd
->min_asid
;
1707 svm
->vmcb
->control
.tlb_ctl
= TLB_CONTROL_FLUSH_ALL_ASID
;
1708 vmcb_mark_dirty(svm
->vmcb
, VMCB_ASID
);
1711 svm
->current_vmcb
->asid_generation
= sd
->asid_generation
;
1712 svm
->asid
= sd
->next_asid
++;
1715 static void svm_set_dr6(struct vcpu_svm
*svm
, unsigned long value
)
1717 struct vmcb
*vmcb
= svm
->vmcb
;
1719 if (svm
->vcpu
.arch
.guest_state_protected
)
1722 if (unlikely(value
!= vmcb
->save
.dr6
)) {
1723 vmcb
->save
.dr6
= value
;
1724 vmcb_mark_dirty(vmcb
, VMCB_DR
);
1728 static void svm_sync_dirty_debug_regs(struct kvm_vcpu
*vcpu
)
1730 struct vcpu_svm
*svm
= to_svm(vcpu
);
1732 if (vcpu
->arch
.guest_state_protected
)
1735 get_debugreg(vcpu
->arch
.db
[0], 0);
1736 get_debugreg(vcpu
->arch
.db
[1], 1);
1737 get_debugreg(vcpu
->arch
.db
[2], 2);
1738 get_debugreg(vcpu
->arch
.db
[3], 3);
1740 * We cannot reset svm->vmcb->save.dr6 to DR6_ACTIVE_LOW here,
1741 * because db_interception might need it. We can do it before vmentry.
1743 vcpu
->arch
.dr6
= svm
->vmcb
->save
.dr6
;
1744 vcpu
->arch
.dr7
= svm
->vmcb
->save
.dr7
;
1745 vcpu
->arch
.switch_db_regs
&= ~KVM_DEBUGREG_WONT_EXIT
;
1746 set_dr_intercepts(svm
);
1749 static void svm_set_dr7(struct kvm_vcpu
*vcpu
, unsigned long value
)
1751 struct vcpu_svm
*svm
= to_svm(vcpu
);
1753 if (vcpu
->arch
.guest_state_protected
)
1756 svm
->vmcb
->save
.dr7
= value
;
1757 vmcb_mark_dirty(svm
->vmcb
, VMCB_DR
);
1760 static int pf_interception(struct kvm_vcpu
*vcpu
)
1762 struct vcpu_svm
*svm
= to_svm(vcpu
);
1764 u64 fault_address
= svm
->vmcb
->control
.exit_info_2
;
1765 u64 error_code
= svm
->vmcb
->control
.exit_info_1
;
1767 return kvm_handle_page_fault(vcpu
, error_code
, fault_address
,
1768 static_cpu_has(X86_FEATURE_DECODEASSISTS
) ?
1769 svm
->vmcb
->control
.insn_bytes
: NULL
,
1770 svm
->vmcb
->control
.insn_len
);
1773 static int npf_interception(struct kvm_vcpu
*vcpu
)
1775 struct vcpu_svm
*svm
= to_svm(vcpu
);
1777 u64 fault_address
= svm
->vmcb
->control
.exit_info_2
;
1778 u64 error_code
= svm
->vmcb
->control
.exit_info_1
;
1780 trace_kvm_page_fault(fault_address
, error_code
);
1781 return kvm_mmu_page_fault(vcpu
, fault_address
, error_code
,
1782 static_cpu_has(X86_FEATURE_DECODEASSISTS
) ?
1783 svm
->vmcb
->control
.insn_bytes
: NULL
,
1784 svm
->vmcb
->control
.insn_len
);
1787 static int db_interception(struct kvm_vcpu
*vcpu
)
1789 struct kvm_run
*kvm_run
= vcpu
->run
;
1790 struct vcpu_svm
*svm
= to_svm(vcpu
);
1792 if (!(vcpu
->guest_debug
&
1793 (KVM_GUESTDBG_SINGLESTEP
| KVM_GUESTDBG_USE_HW_BP
)) &&
1794 !svm
->nmi_singlestep
) {
1795 u32 payload
= svm
->vmcb
->save
.dr6
^ DR6_ACTIVE_LOW
;
1796 kvm_queue_exception_p(vcpu
, DB_VECTOR
, payload
);
1800 if (svm
->nmi_singlestep
) {
1801 disable_nmi_singlestep(svm
);
1802 /* Make sure we check for pending NMIs upon entry */
1803 kvm_make_request(KVM_REQ_EVENT
, vcpu
);
1806 if (vcpu
->guest_debug
&
1807 (KVM_GUESTDBG_SINGLESTEP
| KVM_GUESTDBG_USE_HW_BP
)) {
1808 kvm_run
->exit_reason
= KVM_EXIT_DEBUG
;
1809 kvm_run
->debug
.arch
.dr6
= svm
->vmcb
->save
.dr6
;
1810 kvm_run
->debug
.arch
.dr7
= svm
->vmcb
->save
.dr7
;
1811 kvm_run
->debug
.arch
.pc
=
1812 svm
->vmcb
->save
.cs
.base
+ svm
->vmcb
->save
.rip
;
1813 kvm_run
->debug
.arch
.exception
= DB_VECTOR
;
1820 static int bp_interception(struct kvm_vcpu
*vcpu
)
1822 struct vcpu_svm
*svm
= to_svm(vcpu
);
1823 struct kvm_run
*kvm_run
= vcpu
->run
;
1825 kvm_run
->exit_reason
= KVM_EXIT_DEBUG
;
1826 kvm_run
->debug
.arch
.pc
= svm
->vmcb
->save
.cs
.base
+ svm
->vmcb
->save
.rip
;
1827 kvm_run
->debug
.arch
.exception
= BP_VECTOR
;
1831 static int ud_interception(struct kvm_vcpu
*vcpu
)
1833 return handle_ud(vcpu
);
1836 static int ac_interception(struct kvm_vcpu
*vcpu
)
1838 kvm_queue_exception_e(vcpu
, AC_VECTOR
, 0);
1842 static bool is_erratum_383(void)
1847 if (!erratum_383_found
)
1850 value
= native_read_msr_safe(MSR_IA32_MC0_STATUS
, &err
);
1854 /* Bit 62 may or may not be set for this mce */
1855 value
&= ~(1ULL << 62);
1857 if (value
!= 0xb600000000010015ULL
)
1860 /* Clear MCi_STATUS registers */
1861 for (i
= 0; i
< 6; ++i
)
1862 native_write_msr_safe(MSR_IA32_MCx_STATUS(i
), 0, 0);
1864 value
= native_read_msr_safe(MSR_IA32_MCG_STATUS
, &err
);
1868 value
&= ~(1ULL << 2);
1869 low
= lower_32_bits(value
);
1870 high
= upper_32_bits(value
);
1872 native_write_msr_safe(MSR_IA32_MCG_STATUS
, low
, high
);
1875 /* Flush tlb to evict multi-match entries */
1881 static void svm_handle_mce(struct kvm_vcpu
*vcpu
)
1883 if (is_erratum_383()) {
1885 * Erratum 383 triggered. Guest state is corrupt so kill the
1888 pr_err("KVM: Guest triggered AMD Erratum 383\n");
1890 kvm_make_request(KVM_REQ_TRIPLE_FAULT
, vcpu
);
1896 * On an #MC intercept the MCE handler is not called automatically in
1897 * the host. So do it by hand here.
1899 kvm_machine_check();
1902 static int mc_interception(struct kvm_vcpu
*vcpu
)
1907 static int shutdown_interception(struct kvm_vcpu
*vcpu
)
1909 struct kvm_run
*kvm_run
= vcpu
->run
;
1910 struct vcpu_svm
*svm
= to_svm(vcpu
);
1913 * The VM save area has already been encrypted so it
1914 * cannot be reinitialized - just terminate.
1916 if (sev_es_guest(vcpu
->kvm
))
1920 * VMCB is undefined after a SHUTDOWN intercept. INIT the vCPU to put
1921 * the VMCB in a known good state. Unfortuately, KVM doesn't have
1922 * KVM_MP_STATE_SHUTDOWN and can't add it without potentially breaking
1923 * userspace. At a platform view, INIT is acceptable behavior as
1924 * there exist bare metal platforms that automatically INIT the CPU
1925 * in response to shutdown.
1927 clear_page(svm
->vmcb
);
1928 kvm_vcpu_reset(vcpu
, true);
1930 kvm_run
->exit_reason
= KVM_EXIT_SHUTDOWN
;
1934 static int io_interception(struct kvm_vcpu
*vcpu
)
1936 struct vcpu_svm
*svm
= to_svm(vcpu
);
1937 u32 io_info
= svm
->vmcb
->control
.exit_info_1
; /* address size bug? */
1938 int size
, in
, string
;
1941 ++vcpu
->stat
.io_exits
;
1942 string
= (io_info
& SVM_IOIO_STR_MASK
) != 0;
1943 in
= (io_info
& SVM_IOIO_TYPE_MASK
) != 0;
1944 port
= io_info
>> 16;
1945 size
= (io_info
& SVM_IOIO_SIZE_MASK
) >> SVM_IOIO_SIZE_SHIFT
;
1948 if (sev_es_guest(vcpu
->kvm
))
1949 return sev_es_string_io(svm
, size
, port
, in
);
1951 return kvm_emulate_instruction(vcpu
, 0);
1954 svm
->next_rip
= svm
->vmcb
->control
.exit_info_2
;
1956 return kvm_fast_pio(vcpu
, size
, port
, in
);
1959 static int nmi_interception(struct kvm_vcpu
*vcpu
)
1964 static int smi_interception(struct kvm_vcpu
*vcpu
)
1969 static int intr_interception(struct kvm_vcpu
*vcpu
)
1971 ++vcpu
->stat
.irq_exits
;
1975 static int vmload_vmsave_interception(struct kvm_vcpu
*vcpu
, bool vmload
)
1977 struct vcpu_svm
*svm
= to_svm(vcpu
);
1978 struct vmcb
*vmcb12
;
1979 struct kvm_host_map map
;
1982 if (nested_svm_check_permissions(vcpu
))
1985 ret
= kvm_vcpu_map(vcpu
, gpa_to_gfn(svm
->vmcb
->save
.rax
), &map
);
1988 kvm_inject_gp(vcpu
, 0);
1994 ret
= kvm_skip_emulated_instruction(vcpu
);
1997 svm_copy_vmloadsave_state(svm
->vmcb
, vmcb12
);
1998 svm
->sysenter_eip_hi
= 0;
1999 svm
->sysenter_esp_hi
= 0;
2001 svm_copy_vmloadsave_state(vmcb12
, svm
->vmcb
);
2004 kvm_vcpu_unmap(vcpu
, &map
, true);
2009 static int vmload_interception(struct kvm_vcpu
*vcpu
)
2011 return vmload_vmsave_interception(vcpu
, true);
2014 static int vmsave_interception(struct kvm_vcpu
*vcpu
)
2016 return vmload_vmsave_interception(vcpu
, false);
2019 static int vmrun_interception(struct kvm_vcpu
*vcpu
)
2021 if (nested_svm_check_permissions(vcpu
))
2024 return nested_svm_vmrun(vcpu
);
2034 /* Return NONE_SVM_INSTR if not SVM instrs, otherwise return decode result */
2035 static int svm_instr_opcode(struct kvm_vcpu
*vcpu
)
2037 struct x86_emulate_ctxt
*ctxt
= vcpu
->arch
.emulate_ctxt
;
2039 if (ctxt
->b
!= 0x1 || ctxt
->opcode_len
!= 2)
2040 return NONE_SVM_INSTR
;
2042 switch (ctxt
->modrm
) {
2043 case 0xd8: /* VMRUN */
2044 return SVM_INSTR_VMRUN
;
2045 case 0xda: /* VMLOAD */
2046 return SVM_INSTR_VMLOAD
;
2047 case 0xdb: /* VMSAVE */
2048 return SVM_INSTR_VMSAVE
;
2053 return NONE_SVM_INSTR
;
2056 static int emulate_svm_instr(struct kvm_vcpu
*vcpu
, int opcode
)
2058 const int guest_mode_exit_codes
[] = {
2059 [SVM_INSTR_VMRUN
] = SVM_EXIT_VMRUN
,
2060 [SVM_INSTR_VMLOAD
] = SVM_EXIT_VMLOAD
,
2061 [SVM_INSTR_VMSAVE
] = SVM_EXIT_VMSAVE
,
2063 int (*const svm_instr_handlers
[])(struct kvm_vcpu
*vcpu
) = {
2064 [SVM_INSTR_VMRUN
] = vmrun_interception
,
2065 [SVM_INSTR_VMLOAD
] = vmload_interception
,
2066 [SVM_INSTR_VMSAVE
] = vmsave_interception
,
2068 struct vcpu_svm
*svm
= to_svm(vcpu
);
2071 if (is_guest_mode(vcpu
)) {
2072 /* Returns '1' or -errno on failure, '0' on success. */
2073 ret
= nested_svm_simple_vmexit(svm
, guest_mode_exit_codes
[opcode
]);
2078 return svm_instr_handlers
[opcode
](vcpu
);
2082 * #GP handling code. Note that #GP can be triggered under the following two
2084 * 1) SVM VM-related instructions (VMRUN/VMSAVE/VMLOAD) that trigger #GP on
2085 * some AMD CPUs when EAX of these instructions are in the reserved memory
2086 * regions (e.g. SMM memory on host).
2087 * 2) VMware backdoor
2089 static int gp_interception(struct kvm_vcpu
*vcpu
)
2091 struct vcpu_svm
*svm
= to_svm(vcpu
);
2092 u32 error_code
= svm
->vmcb
->control
.exit_info_1
;
2095 /* Both #GP cases have zero error_code */
2099 /* Decode the instruction for usage later */
2100 if (x86_decode_emulated_instruction(vcpu
, 0, NULL
, 0) != EMULATION_OK
)
2103 opcode
= svm_instr_opcode(vcpu
);
2105 if (opcode
== NONE_SVM_INSTR
) {
2106 if (!enable_vmware_backdoor
)
2110 * VMware backdoor emulation on #GP interception only handles
2111 * IN{S}, OUT{S}, and RDPMC.
2113 if (!is_guest_mode(vcpu
))
2114 return kvm_emulate_instruction(vcpu
,
2115 EMULTYPE_VMWARE_GP
| EMULTYPE_NO_DECODE
);
2117 /* All SVM instructions expect page aligned RAX */
2118 if (svm
->vmcb
->save
.rax
& ~PAGE_MASK
)
2121 return emulate_svm_instr(vcpu
, opcode
);
2125 kvm_queue_exception_e(vcpu
, GP_VECTOR
, error_code
);
2129 void svm_set_gif(struct vcpu_svm
*svm
, bool value
)
2133 * If VGIF is enabled, the STGI intercept is only added to
2134 * detect the opening of the SMI/NMI window; remove it now.
2135 * Likewise, clear the VINTR intercept, we will set it
2136 * again while processing KVM_REQ_EVENT if needed.
2138 if (vgif_enabled(svm
))
2139 svm_clr_intercept(svm
, INTERCEPT_STGI
);
2140 if (svm_is_intercept(svm
, INTERCEPT_VINTR
))
2141 svm_clear_vintr(svm
);
2144 if (svm
->vcpu
.arch
.smi_pending
||
2145 svm
->vcpu
.arch
.nmi_pending
||
2146 kvm_cpu_has_injectable_intr(&svm
->vcpu
))
2147 kvm_make_request(KVM_REQ_EVENT
, &svm
->vcpu
);
2152 * After a CLGI no interrupts should come. But if vGIF is
2153 * in use, we still rely on the VINTR intercept (rather than
2154 * STGI) to detect an open interrupt window.
2156 if (!vgif_enabled(svm
))
2157 svm_clear_vintr(svm
);
2161 static int stgi_interception(struct kvm_vcpu
*vcpu
)
2165 if (nested_svm_check_permissions(vcpu
))
2168 ret
= kvm_skip_emulated_instruction(vcpu
);
2169 svm_set_gif(to_svm(vcpu
), true);
2173 static int clgi_interception(struct kvm_vcpu
*vcpu
)
2177 if (nested_svm_check_permissions(vcpu
))
2180 ret
= kvm_skip_emulated_instruction(vcpu
);
2181 svm_set_gif(to_svm(vcpu
), false);
2185 static int invlpga_interception(struct kvm_vcpu
*vcpu
)
2187 gva_t gva
= kvm_rax_read(vcpu
);
2188 u32 asid
= kvm_rcx_read(vcpu
);
2190 /* FIXME: Handle an address size prefix. */
2191 if (!is_long_mode(vcpu
))
2194 trace_kvm_invlpga(to_svm(vcpu
)->vmcb
->save
.rip
, asid
, gva
);
2196 /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
2197 kvm_mmu_invlpg(vcpu
, gva
);
2199 return kvm_skip_emulated_instruction(vcpu
);
2202 static int skinit_interception(struct kvm_vcpu
*vcpu
)
2204 trace_kvm_skinit(to_svm(vcpu
)->vmcb
->save
.rip
, kvm_rax_read(vcpu
));
2206 kvm_queue_exception(vcpu
, UD_VECTOR
);
2210 static int task_switch_interception(struct kvm_vcpu
*vcpu
)
2212 struct vcpu_svm
*svm
= to_svm(vcpu
);
2215 int int_type
= svm
->vmcb
->control
.exit_int_info
&
2216 SVM_EXITINTINFO_TYPE_MASK
;
2217 int int_vec
= svm
->vmcb
->control
.exit_int_info
& SVM_EVTINJ_VEC_MASK
;
2219 svm
->vmcb
->control
.exit_int_info
& SVM_EXITINTINFO_TYPE_MASK
;
2221 svm
->vmcb
->control
.exit_int_info
& SVM_EXITINTINFO_VALID
;
2222 bool has_error_code
= false;
2225 tss_selector
= (u16
)svm
->vmcb
->control
.exit_info_1
;
2227 if (svm
->vmcb
->control
.exit_info_2
&
2228 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET
))
2229 reason
= TASK_SWITCH_IRET
;
2230 else if (svm
->vmcb
->control
.exit_info_2
&
2231 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP
))
2232 reason
= TASK_SWITCH_JMP
;
2234 reason
= TASK_SWITCH_GATE
;
2236 reason
= TASK_SWITCH_CALL
;
2238 if (reason
== TASK_SWITCH_GATE
) {
2240 case SVM_EXITINTINFO_TYPE_NMI
:
2241 vcpu
->arch
.nmi_injected
= false;
2243 case SVM_EXITINTINFO_TYPE_EXEPT
:
2244 if (svm
->vmcb
->control
.exit_info_2
&
2245 (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE
)) {
2246 has_error_code
= true;
2248 (u32
)svm
->vmcb
->control
.exit_info_2
;
2250 kvm_clear_exception_queue(vcpu
);
2252 case SVM_EXITINTINFO_TYPE_INTR
:
2253 kvm_clear_interrupt_queue(vcpu
);
2260 if (reason
!= TASK_SWITCH_GATE
||
2261 int_type
== SVM_EXITINTINFO_TYPE_SOFT
||
2262 (int_type
== SVM_EXITINTINFO_TYPE_EXEPT
&&
2263 (int_vec
== OF_VECTOR
|| int_vec
== BP_VECTOR
))) {
2264 if (!skip_emulated_instruction(vcpu
))
2268 if (int_type
!= SVM_EXITINTINFO_TYPE_SOFT
)
2271 return kvm_task_switch(vcpu
, tss_selector
, int_vec
, reason
,
2272 has_error_code
, error_code
);
2275 static int iret_interception(struct kvm_vcpu
*vcpu
)
2277 struct vcpu_svm
*svm
= to_svm(vcpu
);
2279 ++vcpu
->stat
.nmi_window_exits
;
2280 vcpu
->arch
.hflags
|= HF_IRET_MASK
;
2281 if (!sev_es_guest(vcpu
->kvm
)) {
2282 svm_clr_intercept(svm
, INTERCEPT_IRET
);
2283 svm
->nmi_iret_rip
= kvm_rip_read(vcpu
);
2285 kvm_make_request(KVM_REQ_EVENT
, vcpu
);
2289 static int invlpg_interception(struct kvm_vcpu
*vcpu
)
2291 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS
))
2292 return kvm_emulate_instruction(vcpu
, 0);
2294 kvm_mmu_invlpg(vcpu
, to_svm(vcpu
)->vmcb
->control
.exit_info_1
);
2295 return kvm_skip_emulated_instruction(vcpu
);
2298 static int emulate_on_interception(struct kvm_vcpu
*vcpu
)
2300 return kvm_emulate_instruction(vcpu
, 0);
2303 static int rsm_interception(struct kvm_vcpu
*vcpu
)
2305 return kvm_emulate_instruction_from_buffer(vcpu
, rsm_ins_bytes
, 2);
2308 static bool check_selective_cr0_intercepted(struct kvm_vcpu
*vcpu
,
2311 struct vcpu_svm
*svm
= to_svm(vcpu
);
2312 unsigned long cr0
= vcpu
->arch
.cr0
;
2315 if (!is_guest_mode(vcpu
) ||
2316 (!(vmcb12_is_intercept(&svm
->nested
.ctl
, INTERCEPT_SELECTIVE_CR0
))))
2319 cr0
&= ~SVM_CR0_SELECTIVE_MASK
;
2320 val
&= ~SVM_CR0_SELECTIVE_MASK
;
2323 svm
->vmcb
->control
.exit_code
= SVM_EXIT_CR0_SEL_WRITE
;
2324 ret
= (nested_svm_exit_handled(svm
) == NESTED_EXIT_DONE
);
2330 #define CR_VALID (1ULL << 63)
2332 static int cr_interception(struct kvm_vcpu
*vcpu
)
2334 struct vcpu_svm
*svm
= to_svm(vcpu
);
2339 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS
))
2340 return emulate_on_interception(vcpu
);
2342 if (unlikely((svm
->vmcb
->control
.exit_info_1
& CR_VALID
) == 0))
2343 return emulate_on_interception(vcpu
);
2345 reg
= svm
->vmcb
->control
.exit_info_1
& SVM_EXITINFO_REG_MASK
;
2346 if (svm
->vmcb
->control
.exit_code
== SVM_EXIT_CR0_SEL_WRITE
)
2347 cr
= SVM_EXIT_WRITE_CR0
- SVM_EXIT_READ_CR0
;
2349 cr
= svm
->vmcb
->control
.exit_code
- SVM_EXIT_READ_CR0
;
2352 if (cr
>= 16) { /* mov to cr */
2354 val
= kvm_register_read(vcpu
, reg
);
2355 trace_kvm_cr_write(cr
, val
);
2358 if (!check_selective_cr0_intercepted(vcpu
, val
))
2359 err
= kvm_set_cr0(vcpu
, val
);
2365 err
= kvm_set_cr3(vcpu
, val
);
2368 err
= kvm_set_cr4(vcpu
, val
);
2371 err
= kvm_set_cr8(vcpu
, val
);
2374 WARN(1, "unhandled write to CR%d", cr
);
2375 kvm_queue_exception(vcpu
, UD_VECTOR
);
2378 } else { /* mov from cr */
2381 val
= kvm_read_cr0(vcpu
);
2384 val
= vcpu
->arch
.cr2
;
2387 val
= kvm_read_cr3(vcpu
);
2390 val
= kvm_read_cr4(vcpu
);
2393 val
= kvm_get_cr8(vcpu
);
2396 WARN(1, "unhandled read from CR%d", cr
);
2397 kvm_queue_exception(vcpu
, UD_VECTOR
);
2400 kvm_register_write(vcpu
, reg
, val
);
2401 trace_kvm_cr_read(cr
, val
);
2403 return kvm_complete_insn_gp(vcpu
, err
);
2406 static int cr_trap(struct kvm_vcpu
*vcpu
)
2408 struct vcpu_svm
*svm
= to_svm(vcpu
);
2409 unsigned long old_value
, new_value
;
2413 new_value
= (unsigned long)svm
->vmcb
->control
.exit_info_1
;
2415 cr
= svm
->vmcb
->control
.exit_code
- SVM_EXIT_CR0_WRITE_TRAP
;
2418 old_value
= kvm_read_cr0(vcpu
);
2419 svm_set_cr0(vcpu
, new_value
);
2421 kvm_post_set_cr0(vcpu
, old_value
, new_value
);
2424 old_value
= kvm_read_cr4(vcpu
);
2425 svm_set_cr4(vcpu
, new_value
);
2427 kvm_post_set_cr4(vcpu
, old_value
, new_value
);
2430 ret
= kvm_set_cr8(vcpu
, new_value
);
2433 WARN(1, "unhandled CR%d write trap", cr
);
2434 kvm_queue_exception(vcpu
, UD_VECTOR
);
2438 return kvm_complete_insn_gp(vcpu
, ret
);
2441 static int dr_interception(struct kvm_vcpu
*vcpu
)
2443 struct vcpu_svm
*svm
= to_svm(vcpu
);
2448 if (vcpu
->guest_debug
== 0) {
2450 * No more DR vmexits; force a reload of the debug registers
2451 * and reenter on this instruction. The next vmexit will
2452 * retrieve the full state of the debug registers.
2454 clr_dr_intercepts(svm
);
2455 vcpu
->arch
.switch_db_regs
|= KVM_DEBUGREG_WONT_EXIT
;
2459 if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS
))
2460 return emulate_on_interception(vcpu
);
2462 reg
= svm
->vmcb
->control
.exit_info_1
& SVM_EXITINFO_REG_MASK
;
2463 dr
= svm
->vmcb
->control
.exit_code
- SVM_EXIT_READ_DR0
;
2464 if (dr
>= 16) { /* mov to DRn */
2466 val
= kvm_register_read(vcpu
, reg
);
2467 err
= kvm_set_dr(vcpu
, dr
, val
);
2469 kvm_get_dr(vcpu
, dr
, &val
);
2470 kvm_register_write(vcpu
, reg
, val
);
2473 return kvm_complete_insn_gp(vcpu
, err
);
2476 static int cr8_write_interception(struct kvm_vcpu
*vcpu
)
2480 u8 cr8_prev
= kvm_get_cr8(vcpu
);
2481 /* instruction emulation calls kvm_set_cr8() */
2482 r
= cr_interception(vcpu
);
2483 if (lapic_in_kernel(vcpu
))
2485 if (cr8_prev
<= kvm_get_cr8(vcpu
))
2487 vcpu
->run
->exit_reason
= KVM_EXIT_SET_TPR
;
2491 static int efer_trap(struct kvm_vcpu
*vcpu
)
2493 struct msr_data msr_info
;
2497 * Clear the EFER_SVME bit from EFER. The SVM code always sets this
2498 * bit in svm_set_efer(), but __kvm_valid_efer() checks it against
2499 * whether the guest has X86_FEATURE_SVM - this avoids a failure if
2500 * the guest doesn't have X86_FEATURE_SVM.
2502 msr_info
.host_initiated
= false;
2503 msr_info
.index
= MSR_EFER
;
2504 msr_info
.data
= to_svm(vcpu
)->vmcb
->control
.exit_info_1
& ~EFER_SVME
;
2505 ret
= kvm_set_msr_common(vcpu
, &msr_info
);
2507 return kvm_complete_insn_gp(vcpu
, ret
);
2510 static int svm_get_msr_feature(struct kvm_msr_entry
*msr
)
2514 switch (msr
->index
) {
2515 case MSR_F10H_DECFG
:
2516 if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC
))
2517 msr
->data
|= MSR_F10H_DECFG_LFENCE_SERIALIZE
;
2519 case MSR_IA32_PERF_CAPABILITIES
:
2522 return KVM_MSR_RET_INVALID
;
2528 static int svm_get_msr(struct kvm_vcpu
*vcpu
, struct msr_data
*msr_info
)
2530 struct vcpu_svm
*svm
= to_svm(vcpu
);
2532 switch (msr_info
->index
) {
2533 case MSR_AMD64_TSC_RATIO
:
2534 if (!msr_info
->host_initiated
&& !svm
->tsc_scaling_enabled
)
2536 msr_info
->data
= svm
->tsc_ratio_msr
;
2539 msr_info
->data
= svm
->vmcb01
.ptr
->save
.star
;
2541 #ifdef CONFIG_X86_64
2543 msr_info
->data
= svm
->vmcb01
.ptr
->save
.lstar
;
2546 msr_info
->data
= svm
->vmcb01
.ptr
->save
.cstar
;
2548 case MSR_KERNEL_GS_BASE
:
2549 msr_info
->data
= svm
->vmcb01
.ptr
->save
.kernel_gs_base
;
2551 case MSR_SYSCALL_MASK
:
2552 msr_info
->data
= svm
->vmcb01
.ptr
->save
.sfmask
;
2555 case MSR_IA32_SYSENTER_CS
:
2556 msr_info
->data
= svm
->vmcb01
.ptr
->save
.sysenter_cs
;
2558 case MSR_IA32_SYSENTER_EIP
:
2559 msr_info
->data
= (u32
)svm
->vmcb01
.ptr
->save
.sysenter_eip
;
2560 if (guest_cpuid_is_intel(vcpu
))
2561 msr_info
->data
|= (u64
)svm
->sysenter_eip_hi
<< 32;
2563 case MSR_IA32_SYSENTER_ESP
:
2564 msr_info
->data
= svm
->vmcb01
.ptr
->save
.sysenter_esp
;
2565 if (guest_cpuid_is_intel(vcpu
))
2566 msr_info
->data
|= (u64
)svm
->sysenter_esp_hi
<< 32;
2569 msr_info
->data
= svm
->tsc_aux
;
2572 * Nobody will change the following 5 values in the VMCB so we can
2573 * safely return them on rdmsr. They will always be 0 until LBRV is
2576 case MSR_IA32_DEBUGCTLMSR
:
2577 msr_info
->data
= svm
->vmcb
->save
.dbgctl
;
2579 case MSR_IA32_LASTBRANCHFROMIP
:
2580 msr_info
->data
= svm
->vmcb
->save
.br_from
;
2582 case MSR_IA32_LASTBRANCHTOIP
:
2583 msr_info
->data
= svm
->vmcb
->save
.br_to
;
2585 case MSR_IA32_LASTINTFROMIP
:
2586 msr_info
->data
= svm
->vmcb
->save
.last_excp_from
;
2588 case MSR_IA32_LASTINTTOIP
:
2589 msr_info
->data
= svm
->vmcb
->save
.last_excp_to
;
2591 case MSR_VM_HSAVE_PA
:
2592 msr_info
->data
= svm
->nested
.hsave_msr
;
2595 msr_info
->data
= svm
->nested
.vm_cr_msr
;
2597 case MSR_IA32_SPEC_CTRL
:
2598 if (!msr_info
->host_initiated
&&
2599 !guest_has_spec_ctrl_msr(vcpu
))
2602 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL
))
2603 msr_info
->data
= svm
->vmcb
->save
.spec_ctrl
;
2605 msr_info
->data
= svm
->spec_ctrl
;
2607 case MSR_AMD64_VIRT_SPEC_CTRL
:
2608 if (!msr_info
->host_initiated
&&
2609 !guest_cpuid_has(vcpu
, X86_FEATURE_VIRT_SSBD
))
2612 msr_info
->data
= svm
->virt_spec_ctrl
;
2614 case MSR_F15H_IC_CFG
: {
2618 family
= guest_cpuid_family(vcpu
);
2619 model
= guest_cpuid_model(vcpu
);
2621 if (family
< 0 || model
< 0)
2622 return kvm_get_msr_common(vcpu
, msr_info
);
2626 if (family
== 0x15 &&
2627 (model
>= 0x2 && model
< 0x20))
2628 msr_info
->data
= 0x1E;
2631 case MSR_F10H_DECFG
:
2632 msr_info
->data
= svm
->msr_decfg
;
2635 return kvm_get_msr_common(vcpu
, msr_info
);
2640 static int svm_complete_emulated_msr(struct kvm_vcpu
*vcpu
, int err
)
2642 struct vcpu_svm
*svm
= to_svm(vcpu
);
2643 if (!err
|| !sev_es_guest(vcpu
->kvm
) || WARN_ON_ONCE(!svm
->sev_es
.ghcb
))
2644 return kvm_complete_insn_gp(vcpu
, err
);
2646 ghcb_set_sw_exit_info_1(svm
->sev_es
.ghcb
, 1);
2647 ghcb_set_sw_exit_info_2(svm
->sev_es
.ghcb
,
2649 SVM_EVTINJ_TYPE_EXEPT
|
2654 static int svm_set_vm_cr(struct kvm_vcpu
*vcpu
, u64 data
)
2656 struct vcpu_svm
*svm
= to_svm(vcpu
);
2657 int svm_dis
, chg_mask
;
2659 if (data
& ~SVM_VM_CR_VALID_MASK
)
2662 chg_mask
= SVM_VM_CR_VALID_MASK
;
2664 if (svm
->nested
.vm_cr_msr
& SVM_VM_CR_SVM_DIS_MASK
)
2665 chg_mask
&= ~(SVM_VM_CR_SVM_LOCK_MASK
| SVM_VM_CR_SVM_DIS_MASK
);
2667 svm
->nested
.vm_cr_msr
&= ~chg_mask
;
2668 svm
->nested
.vm_cr_msr
|= (data
& chg_mask
);
2670 svm_dis
= svm
->nested
.vm_cr_msr
& SVM_VM_CR_SVM_DIS_MASK
;
2672 /* check for svm_disable while efer.svme is set */
2673 if (svm_dis
&& (vcpu
->arch
.efer
& EFER_SVME
))
2679 static int svm_set_msr(struct kvm_vcpu
*vcpu
, struct msr_data
*msr
)
2681 struct vcpu_svm
*svm
= to_svm(vcpu
);
2684 u32 ecx
= msr
->index
;
2685 u64 data
= msr
->data
;
2687 case MSR_AMD64_TSC_RATIO
:
2688 if (!msr
->host_initiated
&& !svm
->tsc_scaling_enabled
)
2691 if (data
& TSC_RATIO_RSVD
)
2694 svm
->tsc_ratio_msr
= data
;
2696 if (svm
->tsc_scaling_enabled
&& is_guest_mode(vcpu
))
2697 nested_svm_update_tsc_ratio_msr(vcpu
);
2700 case MSR_IA32_CR_PAT
:
2701 if (!kvm_mtrr_valid(vcpu
, MSR_IA32_CR_PAT
, data
))
2703 vcpu
->arch
.pat
= data
;
2704 svm
->vmcb01
.ptr
->save
.g_pat
= data
;
2705 if (is_guest_mode(vcpu
))
2706 nested_vmcb02_compute_g_pat(svm
);
2707 vmcb_mark_dirty(svm
->vmcb
, VMCB_NPT
);
2709 case MSR_IA32_SPEC_CTRL
:
2710 if (!msr
->host_initiated
&&
2711 !guest_has_spec_ctrl_msr(vcpu
))
2714 if (kvm_spec_ctrl_test_value(data
))
2717 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL
))
2718 svm
->vmcb
->save
.spec_ctrl
= data
;
2720 svm
->spec_ctrl
= data
;
2726 * When it's written (to non-zero) for the first time, pass
2730 * The handling of the MSR bitmap for L2 guests is done in
2731 * nested_svm_vmrun_msrpm.
2732 * We update the L1 MSR bit as well since it will end up
2733 * touching the MSR anyway now.
2735 set_msr_interception(vcpu
, svm
->msrpm
, MSR_IA32_SPEC_CTRL
, 1, 1);
2737 case MSR_IA32_PRED_CMD
:
2738 if (!msr
->host_initiated
&&
2739 !guest_has_pred_cmd_msr(vcpu
))
2742 if (data
& ~PRED_CMD_IBPB
)
2744 if (!boot_cpu_has(X86_FEATURE_IBPB
))
2749 wrmsrl(MSR_IA32_PRED_CMD
, PRED_CMD_IBPB
);
2750 set_msr_interception(vcpu
, svm
->msrpm
, MSR_IA32_PRED_CMD
, 0, 1);
2752 case MSR_AMD64_VIRT_SPEC_CTRL
:
2753 if (!msr
->host_initiated
&&
2754 !guest_cpuid_has(vcpu
, X86_FEATURE_VIRT_SSBD
))
2757 if (data
& ~SPEC_CTRL_SSBD
)
2760 svm
->virt_spec_ctrl
= data
;
2763 svm
->vmcb01
.ptr
->save
.star
= data
;
2765 #ifdef CONFIG_X86_64
2767 svm
->vmcb01
.ptr
->save
.lstar
= data
;
2770 svm
->vmcb01
.ptr
->save
.cstar
= data
;
2772 case MSR_KERNEL_GS_BASE
:
2773 svm
->vmcb01
.ptr
->save
.kernel_gs_base
= data
;
2775 case MSR_SYSCALL_MASK
:
2776 svm
->vmcb01
.ptr
->save
.sfmask
= data
;
2779 case MSR_IA32_SYSENTER_CS
:
2780 svm
->vmcb01
.ptr
->save
.sysenter_cs
= data
;
2782 case MSR_IA32_SYSENTER_EIP
:
2783 svm
->vmcb01
.ptr
->save
.sysenter_eip
= (u32
)data
;
2785 * We only intercept the MSR_IA32_SYSENTER_{EIP|ESP} msrs
2786 * when we spoof an Intel vendor ID (for cross vendor migration).
2787 * In this case we use this intercept to track the high
2788 * 32 bit part of these msrs to support Intel's
2789 * implementation of SYSENTER/SYSEXIT.
2791 svm
->sysenter_eip_hi
= guest_cpuid_is_intel(vcpu
) ? (data
>> 32) : 0;
2793 case MSR_IA32_SYSENTER_ESP
:
2794 svm
->vmcb01
.ptr
->save
.sysenter_esp
= (u32
)data
;
2795 svm
->sysenter_esp_hi
= guest_cpuid_is_intel(vcpu
) ? (data
>> 32) : 0;
2799 * TSC_AUX is usually changed only during boot and never read
2800 * directly. Intercept TSC_AUX instead of exposing it to the
2801 * guest via direct_access_msrs, and switch it via user return.
2804 r
= kvm_set_user_return_msr(tsc_aux_uret_slot
, data
, -1ull);
2809 svm
->tsc_aux
= data
;
2811 case MSR_IA32_DEBUGCTLMSR
:
2813 vcpu_unimpl(vcpu
, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
2817 if (data
& DEBUGCTL_RESERVED_BITS
)
2820 svm
->vmcb
->save
.dbgctl
= data
;
2821 vmcb_mark_dirty(svm
->vmcb
, VMCB_LBR
);
2822 if (data
& (1ULL<<0))
2823 svm_enable_lbrv(vcpu
);
2825 svm_disable_lbrv(vcpu
);
2827 case MSR_VM_HSAVE_PA
:
2829 * Old kernels did not validate the value written to
2830 * MSR_VM_HSAVE_PA. Allow KVM_SET_MSR to set an invalid
2831 * value to allow live migrating buggy or malicious guests
2832 * originating from those kernels.
2834 if (!msr
->host_initiated
&& !page_address_valid(vcpu
, data
))
2837 svm
->nested
.hsave_msr
= data
& PAGE_MASK
;
2840 return svm_set_vm_cr(vcpu
, data
);
2842 vcpu_unimpl(vcpu
, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx
, data
);
2844 case MSR_F10H_DECFG
: {
2845 struct kvm_msr_entry msr_entry
;
2847 msr_entry
.index
= msr
->index
;
2848 if (svm_get_msr_feature(&msr_entry
))
2851 /* Check the supported bits */
2852 if (data
& ~msr_entry
.data
)
2855 /* Don't allow the guest to change a bit, #GP */
2856 if (!msr
->host_initiated
&& (data
^ msr_entry
.data
))
2859 svm
->msr_decfg
= data
;
2863 return kvm_set_msr_common(vcpu
, msr
);
2868 static int msr_interception(struct kvm_vcpu
*vcpu
)
2870 if (to_svm(vcpu
)->vmcb
->control
.exit_info_1
)
2871 return kvm_emulate_wrmsr(vcpu
);
2873 return kvm_emulate_rdmsr(vcpu
);
2876 static int interrupt_window_interception(struct kvm_vcpu
*vcpu
)
2878 kvm_make_request(KVM_REQ_EVENT
, vcpu
);
2879 svm_clear_vintr(to_svm(vcpu
));
2882 * For AVIC, the only reason to end up here is ExtINTs.
2883 * In this case AVIC was temporarily disabled for
2884 * requesting the IRQ window and we have to re-enable it.
2886 kvm_request_apicv_update(vcpu
->kvm
, true, APICV_INHIBIT_REASON_IRQWIN
);
2888 ++vcpu
->stat
.irq_window_exits
;
2892 static int pause_interception(struct kvm_vcpu
*vcpu
)
2897 * CPL is not made available for an SEV-ES guest, therefore
2898 * vcpu->arch.preempted_in_kernel can never be true. Just
2899 * set in_kernel to false as well.
2901 in_kernel
= !sev_es_guest(vcpu
->kvm
) && svm_get_cpl(vcpu
) == 0;
2903 if (!kvm_pause_in_guest(vcpu
->kvm
))
2904 grow_ple_window(vcpu
);
2906 kvm_vcpu_on_spin(vcpu
, in_kernel
);
2907 return kvm_skip_emulated_instruction(vcpu
);
2910 static int invpcid_interception(struct kvm_vcpu
*vcpu
)
2912 struct vcpu_svm
*svm
= to_svm(vcpu
);
2916 if (!guest_cpuid_has(vcpu
, X86_FEATURE_INVPCID
)) {
2917 kvm_queue_exception(vcpu
, UD_VECTOR
);
2922 * For an INVPCID intercept:
2923 * EXITINFO1 provides the linear address of the memory operand.
2924 * EXITINFO2 provides the contents of the register operand.
2926 type
= svm
->vmcb
->control
.exit_info_2
;
2927 gva
= svm
->vmcb
->control
.exit_info_1
;
2929 return kvm_handle_invpcid(vcpu
, type
, gva
);
2932 static int (*const svm_exit_handlers
[])(struct kvm_vcpu
*vcpu
) = {
2933 [SVM_EXIT_READ_CR0
] = cr_interception
,
2934 [SVM_EXIT_READ_CR3
] = cr_interception
,
2935 [SVM_EXIT_READ_CR4
] = cr_interception
,
2936 [SVM_EXIT_READ_CR8
] = cr_interception
,
2937 [SVM_EXIT_CR0_SEL_WRITE
] = cr_interception
,
2938 [SVM_EXIT_WRITE_CR0
] = cr_interception
,
2939 [SVM_EXIT_WRITE_CR3
] = cr_interception
,
2940 [SVM_EXIT_WRITE_CR4
] = cr_interception
,
2941 [SVM_EXIT_WRITE_CR8
] = cr8_write_interception
,
2942 [SVM_EXIT_READ_DR0
] = dr_interception
,
2943 [SVM_EXIT_READ_DR1
] = dr_interception
,
2944 [SVM_EXIT_READ_DR2
] = dr_interception
,
2945 [SVM_EXIT_READ_DR3
] = dr_interception
,
2946 [SVM_EXIT_READ_DR4
] = dr_interception
,
2947 [SVM_EXIT_READ_DR5
] = dr_interception
,
2948 [SVM_EXIT_READ_DR6
] = dr_interception
,
2949 [SVM_EXIT_READ_DR7
] = dr_interception
,
2950 [SVM_EXIT_WRITE_DR0
] = dr_interception
,
2951 [SVM_EXIT_WRITE_DR1
] = dr_interception
,
2952 [SVM_EXIT_WRITE_DR2
] = dr_interception
,
2953 [SVM_EXIT_WRITE_DR3
] = dr_interception
,
2954 [SVM_EXIT_WRITE_DR4
] = dr_interception
,
2955 [SVM_EXIT_WRITE_DR5
] = dr_interception
,
2956 [SVM_EXIT_WRITE_DR6
] = dr_interception
,
2957 [SVM_EXIT_WRITE_DR7
] = dr_interception
,
2958 [SVM_EXIT_EXCP_BASE
+ DB_VECTOR
] = db_interception
,
2959 [SVM_EXIT_EXCP_BASE
+ BP_VECTOR
] = bp_interception
,
2960 [SVM_EXIT_EXCP_BASE
+ UD_VECTOR
] = ud_interception
,
2961 [SVM_EXIT_EXCP_BASE
+ PF_VECTOR
] = pf_interception
,
2962 [SVM_EXIT_EXCP_BASE
+ MC_VECTOR
] = mc_interception
,
2963 [SVM_EXIT_EXCP_BASE
+ AC_VECTOR
] = ac_interception
,
2964 [SVM_EXIT_EXCP_BASE
+ GP_VECTOR
] = gp_interception
,
2965 [SVM_EXIT_INTR
] = intr_interception
,
2966 [SVM_EXIT_NMI
] = nmi_interception
,
2967 [SVM_EXIT_SMI
] = smi_interception
,
2968 [SVM_EXIT_VINTR
] = interrupt_window_interception
,
2969 [SVM_EXIT_RDPMC
] = kvm_emulate_rdpmc
,
2970 [SVM_EXIT_CPUID
] = kvm_emulate_cpuid
,
2971 [SVM_EXIT_IRET
] = iret_interception
,
2972 [SVM_EXIT_INVD
] = kvm_emulate_invd
,
2973 [SVM_EXIT_PAUSE
] = pause_interception
,
2974 [SVM_EXIT_HLT
] = kvm_emulate_halt
,
2975 [SVM_EXIT_INVLPG
] = invlpg_interception
,
2976 [SVM_EXIT_INVLPGA
] = invlpga_interception
,
2977 [SVM_EXIT_IOIO
] = io_interception
,
2978 [SVM_EXIT_MSR
] = msr_interception
,
2979 [SVM_EXIT_TASK_SWITCH
] = task_switch_interception
,
2980 [SVM_EXIT_SHUTDOWN
] = shutdown_interception
,
2981 [SVM_EXIT_VMRUN
] = vmrun_interception
,
2982 [SVM_EXIT_VMMCALL
] = kvm_emulate_hypercall
,
2983 [SVM_EXIT_VMLOAD
] = vmload_interception
,
2984 [SVM_EXIT_VMSAVE
] = vmsave_interception
,
2985 [SVM_EXIT_STGI
] = stgi_interception
,
2986 [SVM_EXIT_CLGI
] = clgi_interception
,
2987 [SVM_EXIT_SKINIT
] = skinit_interception
,
2988 [SVM_EXIT_RDTSCP
] = kvm_handle_invalid_op
,
2989 [SVM_EXIT_WBINVD
] = kvm_emulate_wbinvd
,
2990 [SVM_EXIT_MONITOR
] = kvm_emulate_monitor
,
2991 [SVM_EXIT_MWAIT
] = kvm_emulate_mwait
,
2992 [SVM_EXIT_XSETBV
] = kvm_emulate_xsetbv
,
2993 [SVM_EXIT_RDPRU
] = kvm_handle_invalid_op
,
2994 [SVM_EXIT_EFER_WRITE_TRAP
] = efer_trap
,
2995 [SVM_EXIT_CR0_WRITE_TRAP
] = cr_trap
,
2996 [SVM_EXIT_CR4_WRITE_TRAP
] = cr_trap
,
2997 [SVM_EXIT_CR8_WRITE_TRAP
] = cr_trap
,
2998 [SVM_EXIT_INVPCID
] = invpcid_interception
,
2999 [SVM_EXIT_NPF
] = npf_interception
,
3000 [SVM_EXIT_RSM
] = rsm_interception
,
3001 [SVM_EXIT_AVIC_INCOMPLETE_IPI
] = avic_incomplete_ipi_interception
,
3002 [SVM_EXIT_AVIC_UNACCELERATED_ACCESS
] = avic_unaccelerated_access_interception
,
3003 [SVM_EXIT_VMGEXIT
] = sev_handle_vmgexit
,
3006 static void dump_vmcb(struct kvm_vcpu
*vcpu
)
3008 struct vcpu_svm
*svm
= to_svm(vcpu
);
3009 struct vmcb_control_area
*control
= &svm
->vmcb
->control
;
3010 struct vmcb_save_area
*save
= &svm
->vmcb
->save
;
3011 struct vmcb_save_area
*save01
= &svm
->vmcb01
.ptr
->save
;
3013 if (!dump_invalid_vmcb
) {
3014 pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n");
3018 pr_err("VMCB %p, last attempted VMRUN on CPU %d\n",
3019 svm
->current_vmcb
->ptr
, vcpu
->arch
.last_vmentry_cpu
);
3020 pr_err("VMCB Control Area:\n");
3021 pr_err("%-20s%04x\n", "cr_read:", control
->intercepts
[INTERCEPT_CR
] & 0xffff);
3022 pr_err("%-20s%04x\n", "cr_write:", control
->intercepts
[INTERCEPT_CR
] >> 16);
3023 pr_err("%-20s%04x\n", "dr_read:", control
->intercepts
[INTERCEPT_DR
] & 0xffff);
3024 pr_err("%-20s%04x\n", "dr_write:", control
->intercepts
[INTERCEPT_DR
] >> 16);
3025 pr_err("%-20s%08x\n", "exceptions:", control
->intercepts
[INTERCEPT_EXCEPTION
]);
3026 pr_err("%-20s%08x %08x\n", "intercepts:",
3027 control
->intercepts
[INTERCEPT_WORD3
],
3028 control
->intercepts
[INTERCEPT_WORD4
]);
3029 pr_err("%-20s%d\n", "pause filter count:", control
->pause_filter_count
);
3030 pr_err("%-20s%d\n", "pause filter threshold:",
3031 control
->pause_filter_thresh
);
3032 pr_err("%-20s%016llx\n", "iopm_base_pa:", control
->iopm_base_pa
);
3033 pr_err("%-20s%016llx\n", "msrpm_base_pa:", control
->msrpm_base_pa
);
3034 pr_err("%-20s%016llx\n", "tsc_offset:", control
->tsc_offset
);
3035 pr_err("%-20s%d\n", "asid:", control
->asid
);
3036 pr_err("%-20s%d\n", "tlb_ctl:", control
->tlb_ctl
);
3037 pr_err("%-20s%08x\n", "int_ctl:", control
->int_ctl
);
3038 pr_err("%-20s%08x\n", "int_vector:", control
->int_vector
);
3039 pr_err("%-20s%08x\n", "int_state:", control
->int_state
);
3040 pr_err("%-20s%08x\n", "exit_code:", control
->exit_code
);
3041 pr_err("%-20s%016llx\n", "exit_info1:", control
->exit_info_1
);
3042 pr_err("%-20s%016llx\n", "exit_info2:", control
->exit_info_2
);
3043 pr_err("%-20s%08x\n", "exit_int_info:", control
->exit_int_info
);
3044 pr_err("%-20s%08x\n", "exit_int_info_err:", control
->exit_int_info_err
);
3045 pr_err("%-20s%lld\n", "nested_ctl:", control
->nested_ctl
);
3046 pr_err("%-20s%016llx\n", "nested_cr3:", control
->nested_cr3
);
3047 pr_err("%-20s%016llx\n", "avic_vapic_bar:", control
->avic_vapic_bar
);
3048 pr_err("%-20s%016llx\n", "ghcb:", control
->ghcb_gpa
);
3049 pr_err("%-20s%08x\n", "event_inj:", control
->event_inj
);
3050 pr_err("%-20s%08x\n", "event_inj_err:", control
->event_inj_err
);
3051 pr_err("%-20s%lld\n", "virt_ext:", control
->virt_ext
);
3052 pr_err("%-20s%016llx\n", "next_rip:", control
->next_rip
);
3053 pr_err("%-20s%016llx\n", "avic_backing_page:", control
->avic_backing_page
);
3054 pr_err("%-20s%016llx\n", "avic_logical_id:", control
->avic_logical_id
);
3055 pr_err("%-20s%016llx\n", "avic_physical_id:", control
->avic_physical_id
);
3056 pr_err("%-20s%016llx\n", "vmsa_pa:", control
->vmsa_pa
);
3057 pr_err("VMCB State Save Area:\n");
3058 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3060 save
->es
.selector
, save
->es
.attrib
,
3061 save
->es
.limit
, save
->es
.base
);
3062 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3064 save
->cs
.selector
, save
->cs
.attrib
,
3065 save
->cs
.limit
, save
->cs
.base
);
3066 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3068 save
->ss
.selector
, save
->ss
.attrib
,
3069 save
->ss
.limit
, save
->ss
.base
);
3070 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3072 save
->ds
.selector
, save
->ds
.attrib
,
3073 save
->ds
.limit
, save
->ds
.base
);
3074 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3076 save01
->fs
.selector
, save01
->fs
.attrib
,
3077 save01
->fs
.limit
, save01
->fs
.base
);
3078 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3080 save01
->gs
.selector
, save01
->gs
.attrib
,
3081 save01
->gs
.limit
, save01
->gs
.base
);
3082 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3084 save
->gdtr
.selector
, save
->gdtr
.attrib
,
3085 save
->gdtr
.limit
, save
->gdtr
.base
);
3086 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3088 save01
->ldtr
.selector
, save01
->ldtr
.attrib
,
3089 save01
->ldtr
.limit
, save01
->ldtr
.base
);
3090 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3092 save
->idtr
.selector
, save
->idtr
.attrib
,
3093 save
->idtr
.limit
, save
->idtr
.base
);
3094 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3096 save01
->tr
.selector
, save01
->tr
.attrib
,
3097 save01
->tr
.limit
, save01
->tr
.base
);
3098 pr_err("cpl: %d efer: %016llx\n",
3099 save
->cpl
, save
->efer
);
3100 pr_err("%-15s %016llx %-13s %016llx\n",
3101 "cr0:", save
->cr0
, "cr2:", save
->cr2
);
3102 pr_err("%-15s %016llx %-13s %016llx\n",
3103 "cr3:", save
->cr3
, "cr4:", save
->cr4
);
3104 pr_err("%-15s %016llx %-13s %016llx\n",
3105 "dr6:", save
->dr6
, "dr7:", save
->dr7
);
3106 pr_err("%-15s %016llx %-13s %016llx\n",
3107 "rip:", save
->rip
, "rflags:", save
->rflags
);
3108 pr_err("%-15s %016llx %-13s %016llx\n",
3109 "rsp:", save
->rsp
, "rax:", save
->rax
);
3110 pr_err("%-15s %016llx %-13s %016llx\n",
3111 "star:", save01
->star
, "lstar:", save01
->lstar
);
3112 pr_err("%-15s %016llx %-13s %016llx\n",
3113 "cstar:", save01
->cstar
, "sfmask:", save01
->sfmask
);
3114 pr_err("%-15s %016llx %-13s %016llx\n",
3115 "kernel_gs_base:", save01
->kernel_gs_base
,
3116 "sysenter_cs:", save01
->sysenter_cs
);
3117 pr_err("%-15s %016llx %-13s %016llx\n",
3118 "sysenter_esp:", save01
->sysenter_esp
,
3119 "sysenter_eip:", save01
->sysenter_eip
);
3120 pr_err("%-15s %016llx %-13s %016llx\n",
3121 "gpat:", save
->g_pat
, "dbgctl:", save
->dbgctl
);
3122 pr_err("%-15s %016llx %-13s %016llx\n",
3123 "br_from:", save
->br_from
, "br_to:", save
->br_to
);
3124 pr_err("%-15s %016llx %-13s %016llx\n",
3125 "excp_from:", save
->last_excp_from
,
3126 "excp_to:", save
->last_excp_to
);
3129 static bool svm_check_exit_valid(struct kvm_vcpu
*vcpu
, u64 exit_code
)
3131 return (exit_code
< ARRAY_SIZE(svm_exit_handlers
) &&
3132 svm_exit_handlers
[exit_code
]);
3135 static int svm_handle_invalid_exit(struct kvm_vcpu
*vcpu
, u64 exit_code
)
3137 vcpu_unimpl(vcpu
, "svm: unexpected exit reason 0x%llx\n", exit_code
);
3139 vcpu
->run
->exit_reason
= KVM_EXIT_INTERNAL_ERROR
;
3140 vcpu
->run
->internal
.suberror
= KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON
;
3141 vcpu
->run
->internal
.ndata
= 2;
3142 vcpu
->run
->internal
.data
[0] = exit_code
;
3143 vcpu
->run
->internal
.data
[1] = vcpu
->arch
.last_vmentry_cpu
;
3147 int svm_invoke_exit_handler(struct kvm_vcpu
*vcpu
, u64 exit_code
)
3149 if (!svm_check_exit_valid(vcpu
, exit_code
))
3150 return svm_handle_invalid_exit(vcpu
, exit_code
);
3152 #ifdef CONFIG_RETPOLINE
3153 if (exit_code
== SVM_EXIT_MSR
)
3154 return msr_interception(vcpu
);
3155 else if (exit_code
== SVM_EXIT_VINTR
)
3156 return interrupt_window_interception(vcpu
);
3157 else if (exit_code
== SVM_EXIT_INTR
)
3158 return intr_interception(vcpu
);
3159 else if (exit_code
== SVM_EXIT_HLT
)
3160 return kvm_emulate_halt(vcpu
);
3161 else if (exit_code
== SVM_EXIT_NPF
)
3162 return npf_interception(vcpu
);
3164 return svm_exit_handlers
[exit_code
](vcpu
);
3167 static void svm_get_exit_info(struct kvm_vcpu
*vcpu
, u32
*reason
,
3168 u64
*info1
, u64
*info2
,
3169 u32
*intr_info
, u32
*error_code
)
3171 struct vmcb_control_area
*control
= &to_svm(vcpu
)->vmcb
->control
;
3173 *reason
= control
->exit_code
;
3174 *info1
= control
->exit_info_1
;
3175 *info2
= control
->exit_info_2
;
3176 *intr_info
= control
->exit_int_info
;
3177 if ((*intr_info
& SVM_EXITINTINFO_VALID
) &&
3178 (*intr_info
& SVM_EXITINTINFO_VALID_ERR
))
3179 *error_code
= control
->exit_int_info_err
;
3184 static int handle_exit(struct kvm_vcpu
*vcpu
, fastpath_t exit_fastpath
)
3186 struct vcpu_svm
*svm
= to_svm(vcpu
);
3187 struct kvm_run
*kvm_run
= vcpu
->run
;
3188 u32 exit_code
= svm
->vmcb
->control
.exit_code
;
3190 trace_kvm_exit(vcpu
, KVM_ISA_SVM
);
3192 /* SEV-ES guests must use the CR write traps to track CR registers. */
3193 if (!sev_es_guest(vcpu
->kvm
)) {
3194 if (!svm_is_intercept(svm
, INTERCEPT_CR0_WRITE
))
3195 vcpu
->arch
.cr0
= svm
->vmcb
->save
.cr0
;
3197 vcpu
->arch
.cr3
= svm
->vmcb
->save
.cr3
;
3200 if (is_guest_mode(vcpu
)) {
3203 trace_kvm_nested_vmexit(vcpu
, KVM_ISA_SVM
);
3205 vmexit
= nested_svm_exit_special(svm
);
3207 if (vmexit
== NESTED_EXIT_CONTINUE
)
3208 vmexit
= nested_svm_exit_handled(svm
);
3210 if (vmexit
== NESTED_EXIT_DONE
)
3214 if (svm
->vmcb
->control
.exit_code
== SVM_EXIT_ERR
) {
3215 kvm_run
->exit_reason
= KVM_EXIT_FAIL_ENTRY
;
3216 kvm_run
->fail_entry
.hardware_entry_failure_reason
3217 = svm
->vmcb
->control
.exit_code
;
3218 kvm_run
->fail_entry
.cpu
= vcpu
->arch
.last_vmentry_cpu
;
3223 if (is_external_interrupt(svm
->vmcb
->control
.exit_int_info
) &&
3224 exit_code
!= SVM_EXIT_EXCP_BASE
+ PF_VECTOR
&&
3225 exit_code
!= SVM_EXIT_NPF
&& exit_code
!= SVM_EXIT_TASK_SWITCH
&&
3226 exit_code
!= SVM_EXIT_INTR
&& exit_code
!= SVM_EXIT_NMI
)
3227 printk(KERN_ERR
"%s: unexpected exit_int_info 0x%x "
3229 __func__
, svm
->vmcb
->control
.exit_int_info
,
3232 if (exit_fastpath
!= EXIT_FASTPATH_NONE
)
3235 return svm_invoke_exit_handler(vcpu
, exit_code
);
3238 static void reload_tss(struct kvm_vcpu
*vcpu
)
3240 struct svm_cpu_data
*sd
= per_cpu(svm_data
, vcpu
->cpu
);
3242 sd
->tss_desc
->type
= 9; /* available 32/64-bit TSS */
3246 static void pre_svm_run(struct kvm_vcpu
*vcpu
)
3248 struct svm_cpu_data
*sd
= per_cpu(svm_data
, vcpu
->cpu
);
3249 struct vcpu_svm
*svm
= to_svm(vcpu
);
3252 * If the previous vmrun of the vmcb occurred on a different physical
3253 * cpu, then mark the vmcb dirty and assign a new asid. Hardware's
3254 * vmcb clean bits are per logical CPU, as are KVM's asid assignments.
3256 if (unlikely(svm
->current_vmcb
->cpu
!= vcpu
->cpu
)) {
3257 svm
->current_vmcb
->asid_generation
= 0;
3258 vmcb_mark_all_dirty(svm
->vmcb
);
3259 svm
->current_vmcb
->cpu
= vcpu
->cpu
;
3262 if (sev_guest(vcpu
->kvm
))
3263 return pre_sev_run(svm
, vcpu
->cpu
);
3265 /* FIXME: handle wraparound of asid_generation */
3266 if (svm
->current_vmcb
->asid_generation
!= sd
->asid_generation
)
3270 static void svm_inject_nmi(struct kvm_vcpu
*vcpu
)
3272 struct vcpu_svm
*svm
= to_svm(vcpu
);
3274 svm
->vmcb
->control
.event_inj
= SVM_EVTINJ_VALID
| SVM_EVTINJ_TYPE_NMI
;
3275 vcpu
->arch
.hflags
|= HF_NMI_MASK
;
3276 if (!sev_es_guest(vcpu
->kvm
))
3277 svm_set_intercept(svm
, INTERCEPT_IRET
);
3278 ++vcpu
->stat
.nmi_injections
;
3281 static void svm_set_irq(struct kvm_vcpu
*vcpu
)
3283 struct vcpu_svm
*svm
= to_svm(vcpu
);
3285 BUG_ON(!(gif_set(svm
)));
3287 trace_kvm_inj_virq(vcpu
->arch
.interrupt
.nr
);
3288 ++vcpu
->stat
.irq_injections
;
3290 svm
->vmcb
->control
.event_inj
= vcpu
->arch
.interrupt
.nr
|
3291 SVM_EVTINJ_VALID
| SVM_EVTINJ_TYPE_INTR
;
3294 static void svm_deliver_interrupt(struct kvm_lapic
*apic
, int delivery_mode
,
3295 int trig_mode
, int vector
)
3297 struct kvm_vcpu
*vcpu
= apic
->vcpu
;
3299 if (svm_deliver_avic_intr(vcpu
, vector
)) {
3300 kvm_lapic_set_irr(vector
, apic
);
3301 kvm_make_request(KVM_REQ_EVENT
, vcpu
);
3302 kvm_vcpu_kick(vcpu
);
3304 trace_kvm_apicv_accept_irq(vcpu
->vcpu_id
, delivery_mode
,
3309 static void svm_update_cr8_intercept(struct kvm_vcpu
*vcpu
, int tpr
, int irr
)
3311 struct vcpu_svm
*svm
= to_svm(vcpu
);
3314 * SEV-ES guests must always keep the CR intercepts cleared. CR
3315 * tracking is done using the CR write traps.
3317 if (sev_es_guest(vcpu
->kvm
))
3320 if (nested_svm_virtualize_tpr(vcpu
))
3323 svm_clr_intercept(svm
, INTERCEPT_CR8_WRITE
);
3329 svm_set_intercept(svm
, INTERCEPT_CR8_WRITE
);
3332 bool svm_nmi_blocked(struct kvm_vcpu
*vcpu
)
3334 struct vcpu_svm
*svm
= to_svm(vcpu
);
3335 struct vmcb
*vmcb
= svm
->vmcb
;
3341 if (is_guest_mode(vcpu
) && nested_exit_on_nmi(svm
))
3344 ret
= (vmcb
->control
.int_state
& SVM_INTERRUPT_SHADOW_MASK
) ||
3345 (vcpu
->arch
.hflags
& HF_NMI_MASK
);
3350 static int svm_nmi_allowed(struct kvm_vcpu
*vcpu
, bool for_injection
)
3352 struct vcpu_svm
*svm
= to_svm(vcpu
);
3353 if (svm
->nested
.nested_run_pending
)
3356 /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */
3357 if (for_injection
&& is_guest_mode(vcpu
) && nested_exit_on_nmi(svm
))
3360 return !svm_nmi_blocked(vcpu
);
3363 static bool svm_get_nmi_mask(struct kvm_vcpu
*vcpu
)
3365 return !!(vcpu
->arch
.hflags
& HF_NMI_MASK
);
3368 static void svm_set_nmi_mask(struct kvm_vcpu
*vcpu
, bool masked
)
3370 struct vcpu_svm
*svm
= to_svm(vcpu
);
3373 vcpu
->arch
.hflags
|= HF_NMI_MASK
;
3374 if (!sev_es_guest(vcpu
->kvm
))
3375 svm_set_intercept(svm
, INTERCEPT_IRET
);
3377 vcpu
->arch
.hflags
&= ~HF_NMI_MASK
;
3378 if (!sev_es_guest(vcpu
->kvm
))
3379 svm_clr_intercept(svm
, INTERCEPT_IRET
);
3383 bool svm_interrupt_blocked(struct kvm_vcpu
*vcpu
)
3385 struct vcpu_svm
*svm
= to_svm(vcpu
);
3386 struct vmcb
*vmcb
= svm
->vmcb
;
3391 if (is_guest_mode(vcpu
)) {
3392 /* As long as interrupts are being delivered... */
3393 if ((svm
->nested
.ctl
.int_ctl
& V_INTR_MASKING_MASK
)
3394 ? !(svm
->vmcb01
.ptr
->save
.rflags
& X86_EFLAGS_IF
)
3395 : !(kvm_get_rflags(vcpu
) & X86_EFLAGS_IF
))
3398 /* ... vmexits aren't blocked by the interrupt shadow */
3399 if (nested_exit_on_intr(svm
))
3402 if (!svm_get_if_flag(vcpu
))
3406 return (vmcb
->control
.int_state
& SVM_INTERRUPT_SHADOW_MASK
);
3409 static int svm_interrupt_allowed(struct kvm_vcpu
*vcpu
, bool for_injection
)
3411 struct vcpu_svm
*svm
= to_svm(vcpu
);
3412 if (svm
->nested
.nested_run_pending
)
3416 * An IRQ must not be injected into L2 if it's supposed to VM-Exit,
3417 * e.g. if the IRQ arrived asynchronously after checking nested events.
3419 if (for_injection
&& is_guest_mode(vcpu
) && nested_exit_on_intr(svm
))
3422 return !svm_interrupt_blocked(vcpu
);
3425 static void svm_enable_irq_window(struct kvm_vcpu
*vcpu
)
3427 struct vcpu_svm
*svm
= to_svm(vcpu
);
3430 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
3431 * 1, because that's a separate STGI/VMRUN intercept. The next time we
3432 * get that intercept, this function will be called again though and
3433 * we'll get the vintr intercept. However, if the vGIF feature is
3434 * enabled, the STGI interception will not occur. Enable the irq
3435 * window under the assumption that the hardware will set the GIF.
3437 if (vgif_enabled(svm
) || gif_set(svm
)) {
3439 * IRQ window is not needed when AVIC is enabled,
3440 * unless we have pending ExtINT since it cannot be injected
3441 * via AVIC. In such case, we need to temporarily disable AVIC,
3442 * and fallback to injecting IRQ via V_IRQ.
3444 kvm_request_apicv_update(vcpu
->kvm
, false, APICV_INHIBIT_REASON_IRQWIN
);
3449 static void svm_enable_nmi_window(struct kvm_vcpu
*vcpu
)
3451 struct vcpu_svm
*svm
= to_svm(vcpu
);
3453 if ((vcpu
->arch
.hflags
& (HF_NMI_MASK
| HF_IRET_MASK
)) == HF_NMI_MASK
)
3454 return; /* IRET will cause a vm exit */
3456 if (!gif_set(svm
)) {
3457 if (vgif_enabled(svm
))
3458 svm_set_intercept(svm
, INTERCEPT_STGI
);
3459 return; /* STGI will cause a vm exit */
3463 * Something prevents NMI from been injected. Single step over possible
3464 * problem (IRET or exception injection or interrupt shadow)
3466 svm
->nmi_singlestep_guest_rflags
= svm_get_rflags(vcpu
);
3467 svm
->nmi_singlestep
= true;
3468 svm
->vmcb
->save
.rflags
|= (X86_EFLAGS_TF
| X86_EFLAGS_RF
);
3471 static int svm_set_tss_addr(struct kvm
*kvm
, unsigned int addr
)
3476 static int svm_set_identity_map_addr(struct kvm
*kvm
, u64 ident_addr
)
3481 void svm_flush_tlb(struct kvm_vcpu
*vcpu
)
3483 struct vcpu_svm
*svm
= to_svm(vcpu
);
3486 * Flush only the current ASID even if the TLB flush was invoked via
3487 * kvm_flush_remote_tlbs(). Although flushing remote TLBs requires all
3488 * ASIDs to be flushed, KVM uses a single ASID for L1 and L2, and
3489 * unconditionally does a TLB flush on both nested VM-Enter and nested
3490 * VM-Exit (via kvm_mmu_reset_context()).
3492 if (static_cpu_has(X86_FEATURE_FLUSHBYASID
))
3493 svm
->vmcb
->control
.tlb_ctl
= TLB_CONTROL_FLUSH_ASID
;
3495 svm
->current_vmcb
->asid_generation
--;
3498 static void svm_flush_tlb_gva(struct kvm_vcpu
*vcpu
, gva_t gva
)
3500 struct vcpu_svm
*svm
= to_svm(vcpu
);
3502 invlpga(gva
, svm
->vmcb
->control
.asid
);
3505 static inline void sync_cr8_to_lapic(struct kvm_vcpu
*vcpu
)
3507 struct vcpu_svm
*svm
= to_svm(vcpu
);
3509 if (nested_svm_virtualize_tpr(vcpu
))
3512 if (!svm_is_intercept(svm
, INTERCEPT_CR8_WRITE
)) {
3513 int cr8
= svm
->vmcb
->control
.int_ctl
& V_TPR_MASK
;
3514 kvm_set_cr8(vcpu
, cr8
);
3518 static inline void sync_lapic_to_cr8(struct kvm_vcpu
*vcpu
)
3520 struct vcpu_svm
*svm
= to_svm(vcpu
);
3523 if (nested_svm_virtualize_tpr(vcpu
) ||
3524 kvm_vcpu_apicv_active(vcpu
))
3527 cr8
= kvm_get_cr8(vcpu
);
3528 svm
->vmcb
->control
.int_ctl
&= ~V_TPR_MASK
;
3529 svm
->vmcb
->control
.int_ctl
|= cr8
& V_TPR_MASK
;
3532 static void svm_complete_interrupts(struct kvm_vcpu
*vcpu
)
3534 struct vcpu_svm
*svm
= to_svm(vcpu
);
3537 u32 exitintinfo
= svm
->vmcb
->control
.exit_int_info
;
3538 unsigned int3_injected
= svm
->int3_injected
;
3540 svm
->int3_injected
= 0;
3543 * If we've made progress since setting HF_IRET_MASK, we've
3544 * executed an IRET and can allow NMI injection.
3546 if ((vcpu
->arch
.hflags
& HF_IRET_MASK
) &&
3547 (sev_es_guest(vcpu
->kvm
) ||
3548 kvm_rip_read(vcpu
) != svm
->nmi_iret_rip
)) {
3549 vcpu
->arch
.hflags
&= ~(HF_NMI_MASK
| HF_IRET_MASK
);
3550 kvm_make_request(KVM_REQ_EVENT
, vcpu
);
3553 vcpu
->arch
.nmi_injected
= false;
3554 kvm_clear_exception_queue(vcpu
);
3555 kvm_clear_interrupt_queue(vcpu
);
3557 if (!(exitintinfo
& SVM_EXITINTINFO_VALID
))
3560 kvm_make_request(KVM_REQ_EVENT
, vcpu
);
3562 vector
= exitintinfo
& SVM_EXITINTINFO_VEC_MASK
;
3563 type
= exitintinfo
& SVM_EXITINTINFO_TYPE_MASK
;
3566 case SVM_EXITINTINFO_TYPE_NMI
:
3567 vcpu
->arch
.nmi_injected
= true;
3569 case SVM_EXITINTINFO_TYPE_EXEPT
:
3571 * Never re-inject a #VC exception.
3573 if (vector
== X86_TRAP_VC
)
3577 * In case of software exceptions, do not reinject the vector,
3578 * but re-execute the instruction instead. Rewind RIP first
3579 * if we emulated INT3 before.
3581 if (kvm_exception_is_soft(vector
)) {
3582 if (vector
== BP_VECTOR
&& int3_injected
&&
3583 kvm_is_linear_rip(vcpu
, svm
->int3_rip
))
3585 kvm_rip_read(vcpu
) - int3_injected
);
3588 if (exitintinfo
& SVM_EXITINTINFO_VALID_ERR
) {
3589 u32 err
= svm
->vmcb
->control
.exit_int_info_err
;
3590 kvm_requeue_exception_e(vcpu
, vector
, err
);
3593 kvm_requeue_exception(vcpu
, vector
);
3595 case SVM_EXITINTINFO_TYPE_INTR
:
3596 kvm_queue_interrupt(vcpu
, vector
, false);
3603 static void svm_cancel_injection(struct kvm_vcpu
*vcpu
)
3605 struct vcpu_svm
*svm
= to_svm(vcpu
);
3606 struct vmcb_control_area
*control
= &svm
->vmcb
->control
;
3608 control
->exit_int_info
= control
->event_inj
;
3609 control
->exit_int_info_err
= control
->event_inj_err
;
3610 control
->event_inj
= 0;
3611 svm_complete_interrupts(vcpu
);
3614 static int svm_vcpu_pre_run(struct kvm_vcpu
*vcpu
)
3619 static fastpath_t
svm_exit_handlers_fastpath(struct kvm_vcpu
*vcpu
)
3621 if (to_svm(vcpu
)->vmcb
->control
.exit_code
== SVM_EXIT_MSR
&&
3622 to_svm(vcpu
)->vmcb
->control
.exit_info_1
)
3623 return handle_fastpath_set_msr_irqoff(vcpu
);
3625 return EXIT_FASTPATH_NONE
;
3628 static noinstr
void svm_vcpu_enter_exit(struct kvm_vcpu
*vcpu
)
3630 struct vcpu_svm
*svm
= to_svm(vcpu
);
3631 unsigned long vmcb_pa
= svm
->current_vmcb
->pa
;
3633 guest_state_enter_irqoff();
3635 if (sev_es_guest(vcpu
->kvm
)) {
3636 __svm_sev_es_vcpu_run(vmcb_pa
);
3638 struct svm_cpu_data
*sd
= per_cpu(svm_data
, vcpu
->cpu
);
3641 * Use a single vmcb (vmcb01 because it's always valid) for
3642 * context switching guest state via VMLOAD/VMSAVE, that way
3643 * the state doesn't need to be copied between vmcb01 and
3644 * vmcb02 when switching vmcbs for nested virtualization.
3646 vmload(svm
->vmcb01
.pa
);
3647 __svm_vcpu_run(vmcb_pa
, (unsigned long *)&vcpu
->arch
.regs
);
3648 vmsave(svm
->vmcb01
.pa
);
3650 vmload(__sme_page_pa(sd
->save_area
));
3653 guest_state_exit_irqoff();
3656 static __no_kcsan fastpath_t
svm_vcpu_run(struct kvm_vcpu
*vcpu
)
3658 struct vcpu_svm
*svm
= to_svm(vcpu
);
3660 trace_kvm_entry(vcpu
);
3662 svm
->vmcb
->save
.rax
= vcpu
->arch
.regs
[VCPU_REGS_RAX
];
3663 svm
->vmcb
->save
.rsp
= vcpu
->arch
.regs
[VCPU_REGS_RSP
];
3664 svm
->vmcb
->save
.rip
= vcpu
->arch
.regs
[VCPU_REGS_RIP
];
3667 * Disable singlestep if we're injecting an interrupt/exception.
3668 * We don't want our modified rflags to be pushed on the stack where
3669 * we might not be able to easily reset them if we disabled NMI
3672 if (svm
->nmi_singlestep
&& svm
->vmcb
->control
.event_inj
) {
3674 * Event injection happens before external interrupts cause a
3675 * vmexit and interrupts are disabled here, so smp_send_reschedule
3676 * is enough to force an immediate vmexit.
3678 disable_nmi_singlestep(svm
);
3679 smp_send_reschedule(vcpu
->cpu
);
3684 sync_lapic_to_cr8(vcpu
);
3686 if (unlikely(svm
->asid
!= svm
->vmcb
->control
.asid
)) {
3687 svm
->vmcb
->control
.asid
= svm
->asid
;
3688 vmcb_mark_dirty(svm
->vmcb
, VMCB_ASID
);
3690 svm
->vmcb
->save
.cr2
= vcpu
->arch
.cr2
;
3692 svm_hv_update_vp_id(svm
->vmcb
, vcpu
);
3695 * Run with all-zero DR6 unless needed, so that we can get the exact cause
3698 if (unlikely(vcpu
->arch
.switch_db_regs
& KVM_DEBUGREG_WONT_EXIT
))
3699 svm_set_dr6(svm
, vcpu
->arch
.dr6
);
3701 svm_set_dr6(svm
, DR6_ACTIVE_LOW
);
3704 kvm_load_guest_xsave_state(vcpu
);
3706 kvm_wait_lapic_expire(vcpu
);
3709 * If this vCPU has touched SPEC_CTRL, restore the guest's value if
3710 * it's non-zero. Since vmentry is serialising on affected CPUs, there
3711 * is no need to worry about the conditional branch over the wrmsr
3712 * being speculatively taken.
3714 if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL
))
3715 x86_spec_ctrl_set_guest(svm
->spec_ctrl
, svm
->virt_spec_ctrl
);
3717 svm_vcpu_enter_exit(vcpu
);
3720 * We do not use IBRS in the kernel. If this vCPU has used the
3721 * SPEC_CTRL MSR it may have left it on; save the value and
3722 * turn it off. This is much more efficient than blindly adding
3723 * it to the atomic save/restore list. Especially as the former
3724 * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
3726 * For non-nested case:
3727 * If the L01 MSR bitmap does not intercept the MSR, then we need to
3731 * If the L02 MSR bitmap does not intercept the MSR, then we need to
3734 if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL
) &&
3735 unlikely(!msr_write_intercepted(vcpu
, MSR_IA32_SPEC_CTRL
)))
3736 svm
->spec_ctrl
= native_read_msr(MSR_IA32_SPEC_CTRL
);
3738 if (!sev_es_guest(vcpu
->kvm
))
3741 if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL
))
3742 x86_spec_ctrl_restore_host(svm
->spec_ctrl
, svm
->virt_spec_ctrl
);
3744 if (!sev_es_guest(vcpu
->kvm
)) {
3745 vcpu
->arch
.cr2
= svm
->vmcb
->save
.cr2
;
3746 vcpu
->arch
.regs
[VCPU_REGS_RAX
] = svm
->vmcb
->save
.rax
;
3747 vcpu
->arch
.regs
[VCPU_REGS_RSP
] = svm
->vmcb
->save
.rsp
;
3748 vcpu
->arch
.regs
[VCPU_REGS_RIP
] = svm
->vmcb
->save
.rip
;
3750 vcpu
->arch
.regs_dirty
= 0;
3752 if (unlikely(svm
->vmcb
->control
.exit_code
== SVM_EXIT_NMI
))
3753 kvm_before_interrupt(vcpu
, KVM_HANDLING_NMI
);
3755 kvm_load_host_xsave_state(vcpu
);
3758 /* Any pending NMI will happen here */
3760 if (unlikely(svm
->vmcb
->control
.exit_code
== SVM_EXIT_NMI
))
3761 kvm_after_interrupt(vcpu
);
3763 sync_cr8_to_lapic(vcpu
);
3766 if (is_guest_mode(vcpu
)) {
3767 nested_sync_control_from_vmcb02(svm
);
3769 /* Track VMRUNs that have made past consistency checking */
3770 if (svm
->nested
.nested_run_pending
&&
3771 svm
->vmcb
->control
.exit_code
!= SVM_EXIT_ERR
)
3772 ++vcpu
->stat
.nested_run
;
3774 svm
->nested
.nested_run_pending
= 0;
3777 svm
->vmcb
->control
.tlb_ctl
= TLB_CONTROL_DO_NOTHING
;
3778 vmcb_mark_all_clean(svm
->vmcb
);
3780 /* if exit due to PF check for async PF */
3781 if (svm
->vmcb
->control
.exit_code
== SVM_EXIT_EXCP_BASE
+ PF_VECTOR
)
3782 vcpu
->arch
.apf
.host_apf_flags
=
3783 kvm_read_and_reset_apf_flags();
3785 vcpu
->arch
.regs_avail
&= ~SVM_REGS_LAZY_LOAD_SET
;
3788 * We need to handle MC intercepts here before the vcpu has a chance to
3789 * change the physical cpu
3791 if (unlikely(svm
->vmcb
->control
.exit_code
==
3792 SVM_EXIT_EXCP_BASE
+ MC_VECTOR
))
3793 svm_handle_mce(vcpu
);
3795 svm_complete_interrupts(vcpu
);
3797 if (is_guest_mode(vcpu
))
3798 return EXIT_FASTPATH_NONE
;
3800 return svm_exit_handlers_fastpath(vcpu
);
3803 static void svm_load_mmu_pgd(struct kvm_vcpu
*vcpu
, hpa_t root_hpa
,
3806 struct vcpu_svm
*svm
= to_svm(vcpu
);
3810 svm
->vmcb
->control
.nested_cr3
= __sme_set(root_hpa
);
3811 vmcb_mark_dirty(svm
->vmcb
, VMCB_NPT
);
3813 hv_track_root_tdp(vcpu
, root_hpa
);
3815 cr3
= vcpu
->arch
.cr3
;
3816 } else if (vcpu
->arch
.mmu
->shadow_root_level
>= PT64_ROOT_4LEVEL
) {
3817 cr3
= __sme_set(root_hpa
) | kvm_get_active_pcid(vcpu
);
3819 /* PCID in the guest should be impossible with a 32-bit MMU. */
3820 WARN_ON_ONCE(kvm_get_active_pcid(vcpu
));
3824 svm
->vmcb
->save
.cr3
= cr3
;
3825 vmcb_mark_dirty(svm
->vmcb
, VMCB_CR
);
3828 static int is_disabled(void)
3832 rdmsrl(MSR_VM_CR
, vm_cr
);
3833 if (vm_cr
& (1 << SVM_VM_CR_SVM_DISABLE
))
3840 svm_patch_hypercall(struct kvm_vcpu
*vcpu
, unsigned char *hypercall
)
3843 * Patch in the VMMCALL instruction:
3845 hypercall
[0] = 0x0f;
3846 hypercall
[1] = 0x01;
3847 hypercall
[2] = 0xd9;
3850 static int __init
svm_check_processor_compat(void)
3855 static bool svm_cpu_has_accelerated_tpr(void)
3861 * The kvm parameter can be NULL (module initialization, or invocation before
3862 * VM creation). Be sure to check the kvm parameter before using it.
3864 static bool svm_has_emulated_msr(struct kvm
*kvm
, u32 index
)
3867 case MSR_IA32_MCG_EXT_CTL
:
3868 case MSR_IA32_VMX_BASIC
... MSR_IA32_VMX_VMFUNC
:
3870 case MSR_IA32_SMBASE
:
3871 /* SEV-ES guests do not support SMM, so report false */
3872 if (kvm
&& sev_es_guest(kvm
))
3882 static u64
svm_get_mt_mask(struct kvm_vcpu
*vcpu
, gfn_t gfn
, bool is_mmio
)
3887 static void svm_vcpu_after_set_cpuid(struct kvm_vcpu
*vcpu
)
3889 struct vcpu_svm
*svm
= to_svm(vcpu
);
3890 struct kvm_cpuid_entry2
*best
;
3892 vcpu
->arch
.xsaves_enabled
= guest_cpuid_has(vcpu
, X86_FEATURE_XSAVE
) &&
3893 boot_cpu_has(X86_FEATURE_XSAVE
) &&
3894 boot_cpu_has(X86_FEATURE_XSAVES
);
3896 /* Update nrips enabled cache */
3897 svm
->nrips_enabled
= kvm_cpu_cap_has(X86_FEATURE_NRIPS
) &&
3898 guest_cpuid_has(vcpu
, X86_FEATURE_NRIPS
);
3900 svm
->tsc_scaling_enabled
= tsc_scaling
&& guest_cpuid_has(vcpu
, X86_FEATURE_TSCRATEMSR
);
3902 svm_recalc_instruction_intercepts(vcpu
, svm
);
3904 /* For sev guests, the memory encryption bit is not reserved in CR3. */
3905 if (sev_guest(vcpu
->kvm
)) {
3906 best
= kvm_find_cpuid_entry(vcpu
, 0x8000001F, 0);
3908 vcpu
->arch
.reserved_gpa_bits
&= ~(1UL << (best
->ebx
& 0x3f));
3911 if (kvm_vcpu_apicv_active(vcpu
)) {
3913 * AVIC does not work with an x2APIC mode guest. If the X2APIC feature
3914 * is exposed to the guest, disable AVIC.
3916 if (guest_cpuid_has(vcpu
, X86_FEATURE_X2APIC
))
3917 kvm_request_apicv_update(vcpu
->kvm
, false,
3918 APICV_INHIBIT_REASON_X2APIC
);
3921 * Currently, AVIC does not work with nested virtualization.
3922 * So, we disable AVIC when cpuid for SVM is set in the L1 guest.
3924 if (nested
&& guest_cpuid_has(vcpu
, X86_FEATURE_SVM
))
3925 kvm_request_apicv_update(vcpu
->kvm
, false,
3926 APICV_INHIBIT_REASON_NESTED
);
3928 init_vmcb_after_set_cpuid(vcpu
);
3931 static bool svm_has_wbinvd_exit(void)
3936 #define PRE_EX(exit) { .exit_code = (exit), \
3937 .stage = X86_ICPT_PRE_EXCEPT, }
3938 #define POST_EX(exit) { .exit_code = (exit), \
3939 .stage = X86_ICPT_POST_EXCEPT, }
3940 #define POST_MEM(exit) { .exit_code = (exit), \
3941 .stage = X86_ICPT_POST_MEMACCESS, }
3943 static const struct __x86_intercept
{
3945 enum x86_intercept_stage stage
;
3946 } x86_intercept_map
[] = {
3947 [x86_intercept_cr_read
] = POST_EX(SVM_EXIT_READ_CR0
),
3948 [x86_intercept_cr_write
] = POST_EX(SVM_EXIT_WRITE_CR0
),
3949 [x86_intercept_clts
] = POST_EX(SVM_EXIT_WRITE_CR0
),
3950 [x86_intercept_lmsw
] = POST_EX(SVM_EXIT_WRITE_CR0
),
3951 [x86_intercept_smsw
] = POST_EX(SVM_EXIT_READ_CR0
),
3952 [x86_intercept_dr_read
] = POST_EX(SVM_EXIT_READ_DR0
),
3953 [x86_intercept_dr_write
] = POST_EX(SVM_EXIT_WRITE_DR0
),
3954 [x86_intercept_sldt
] = POST_EX(SVM_EXIT_LDTR_READ
),
3955 [x86_intercept_str
] = POST_EX(SVM_EXIT_TR_READ
),
3956 [x86_intercept_lldt
] = POST_EX(SVM_EXIT_LDTR_WRITE
),
3957 [x86_intercept_ltr
] = POST_EX(SVM_EXIT_TR_WRITE
),
3958 [x86_intercept_sgdt
] = POST_EX(SVM_EXIT_GDTR_READ
),
3959 [x86_intercept_sidt
] = POST_EX(SVM_EXIT_IDTR_READ
),
3960 [x86_intercept_lgdt
] = POST_EX(SVM_EXIT_GDTR_WRITE
),
3961 [x86_intercept_lidt
] = POST_EX(SVM_EXIT_IDTR_WRITE
),
3962 [x86_intercept_vmrun
] = POST_EX(SVM_EXIT_VMRUN
),
3963 [x86_intercept_vmmcall
] = POST_EX(SVM_EXIT_VMMCALL
),
3964 [x86_intercept_vmload
] = POST_EX(SVM_EXIT_VMLOAD
),
3965 [x86_intercept_vmsave
] = POST_EX(SVM_EXIT_VMSAVE
),
3966 [x86_intercept_stgi
] = POST_EX(SVM_EXIT_STGI
),
3967 [x86_intercept_clgi
] = POST_EX(SVM_EXIT_CLGI
),
3968 [x86_intercept_skinit
] = POST_EX(SVM_EXIT_SKINIT
),
3969 [x86_intercept_invlpga
] = POST_EX(SVM_EXIT_INVLPGA
),
3970 [x86_intercept_rdtscp
] = POST_EX(SVM_EXIT_RDTSCP
),
3971 [x86_intercept_monitor
] = POST_MEM(SVM_EXIT_MONITOR
),
3972 [x86_intercept_mwait
] = POST_EX(SVM_EXIT_MWAIT
),
3973 [x86_intercept_invlpg
] = POST_EX(SVM_EXIT_INVLPG
),
3974 [x86_intercept_invd
] = POST_EX(SVM_EXIT_INVD
),
3975 [x86_intercept_wbinvd
] = POST_EX(SVM_EXIT_WBINVD
),
3976 [x86_intercept_wrmsr
] = POST_EX(SVM_EXIT_MSR
),
3977 [x86_intercept_rdtsc
] = POST_EX(SVM_EXIT_RDTSC
),
3978 [x86_intercept_rdmsr
] = POST_EX(SVM_EXIT_MSR
),
3979 [x86_intercept_rdpmc
] = POST_EX(SVM_EXIT_RDPMC
),
3980 [x86_intercept_cpuid
] = PRE_EX(SVM_EXIT_CPUID
),
3981 [x86_intercept_rsm
] = PRE_EX(SVM_EXIT_RSM
),
3982 [x86_intercept_pause
] = PRE_EX(SVM_EXIT_PAUSE
),
3983 [x86_intercept_pushf
] = PRE_EX(SVM_EXIT_PUSHF
),
3984 [x86_intercept_popf
] = PRE_EX(SVM_EXIT_POPF
),
3985 [x86_intercept_intn
] = PRE_EX(SVM_EXIT_SWINT
),
3986 [x86_intercept_iret
] = PRE_EX(SVM_EXIT_IRET
),
3987 [x86_intercept_icebp
] = PRE_EX(SVM_EXIT_ICEBP
),
3988 [x86_intercept_hlt
] = POST_EX(SVM_EXIT_HLT
),
3989 [x86_intercept_in
] = POST_EX(SVM_EXIT_IOIO
),
3990 [x86_intercept_ins
] = POST_EX(SVM_EXIT_IOIO
),
3991 [x86_intercept_out
] = POST_EX(SVM_EXIT_IOIO
),
3992 [x86_intercept_outs
] = POST_EX(SVM_EXIT_IOIO
),
3993 [x86_intercept_xsetbv
] = PRE_EX(SVM_EXIT_XSETBV
),
4000 static int svm_check_intercept(struct kvm_vcpu
*vcpu
,
4001 struct x86_instruction_info
*info
,
4002 enum x86_intercept_stage stage
,
4003 struct x86_exception
*exception
)
4005 struct vcpu_svm
*svm
= to_svm(vcpu
);
4006 int vmexit
, ret
= X86EMUL_CONTINUE
;
4007 struct __x86_intercept icpt_info
;
4008 struct vmcb
*vmcb
= svm
->vmcb
;
4010 if (info
->intercept
>= ARRAY_SIZE(x86_intercept_map
))
4013 icpt_info
= x86_intercept_map
[info
->intercept
];
4015 if (stage
!= icpt_info
.stage
)
4018 switch (icpt_info
.exit_code
) {
4019 case SVM_EXIT_READ_CR0
:
4020 if (info
->intercept
== x86_intercept_cr_read
)
4021 icpt_info
.exit_code
+= info
->modrm_reg
;
4023 case SVM_EXIT_WRITE_CR0
: {
4024 unsigned long cr0
, val
;
4026 if (info
->intercept
== x86_intercept_cr_write
)
4027 icpt_info
.exit_code
+= info
->modrm_reg
;
4029 if (icpt_info
.exit_code
!= SVM_EXIT_WRITE_CR0
||
4030 info
->intercept
== x86_intercept_clts
)
4033 if (!(vmcb12_is_intercept(&svm
->nested
.ctl
,
4034 INTERCEPT_SELECTIVE_CR0
)))
4037 cr0
= vcpu
->arch
.cr0
& ~SVM_CR0_SELECTIVE_MASK
;
4038 val
= info
->src_val
& ~SVM_CR0_SELECTIVE_MASK
;
4040 if (info
->intercept
== x86_intercept_lmsw
) {
4043 /* lmsw can't clear PE - catch this here */
4044 if (cr0
& X86_CR0_PE
)
4049 icpt_info
.exit_code
= SVM_EXIT_CR0_SEL_WRITE
;
4053 case SVM_EXIT_READ_DR0
:
4054 case SVM_EXIT_WRITE_DR0
:
4055 icpt_info
.exit_code
+= info
->modrm_reg
;
4058 if (info
->intercept
== x86_intercept_wrmsr
)
4059 vmcb
->control
.exit_info_1
= 1;
4061 vmcb
->control
.exit_info_1
= 0;
4063 case SVM_EXIT_PAUSE
:
4065 * We get this for NOP only, but pause
4066 * is rep not, check this here
4068 if (info
->rep_prefix
!= REPE_PREFIX
)
4071 case SVM_EXIT_IOIO
: {
4075 if (info
->intercept
== x86_intercept_in
||
4076 info
->intercept
== x86_intercept_ins
) {
4077 exit_info
= ((info
->src_val
& 0xffff) << 16) |
4079 bytes
= info
->dst_bytes
;
4081 exit_info
= (info
->dst_val
& 0xffff) << 16;
4082 bytes
= info
->src_bytes
;
4085 if (info
->intercept
== x86_intercept_outs
||
4086 info
->intercept
== x86_intercept_ins
)
4087 exit_info
|= SVM_IOIO_STR_MASK
;
4089 if (info
->rep_prefix
)
4090 exit_info
|= SVM_IOIO_REP_MASK
;
4092 bytes
= min(bytes
, 4u);
4094 exit_info
|= bytes
<< SVM_IOIO_SIZE_SHIFT
;
4096 exit_info
|= (u32
)info
->ad_bytes
<< (SVM_IOIO_ASIZE_SHIFT
- 1);
4098 vmcb
->control
.exit_info_1
= exit_info
;
4099 vmcb
->control
.exit_info_2
= info
->next_rip
;
4107 /* TODO: Advertise NRIPS to guest hypervisor unconditionally */
4108 if (static_cpu_has(X86_FEATURE_NRIPS
))
4109 vmcb
->control
.next_rip
= info
->next_rip
;
4110 vmcb
->control
.exit_code
= icpt_info
.exit_code
;
4111 vmexit
= nested_svm_exit_handled(svm
);
4113 ret
= (vmexit
== NESTED_EXIT_DONE
) ? X86EMUL_INTERCEPTED
4120 static void svm_handle_exit_irqoff(struct kvm_vcpu
*vcpu
)
4124 static void svm_sched_in(struct kvm_vcpu
*vcpu
, int cpu
)
4126 if (!kvm_pause_in_guest(vcpu
->kvm
))
4127 shrink_ple_window(vcpu
);
4130 static void svm_setup_mce(struct kvm_vcpu
*vcpu
)
4132 /* [63:9] are reserved. */
4133 vcpu
->arch
.mcg_cap
&= 0x1ff;
4136 bool svm_smi_blocked(struct kvm_vcpu
*vcpu
)
4138 struct vcpu_svm
*svm
= to_svm(vcpu
);
4140 /* Per APM Vol.2 15.22.2 "Response to SMI" */
4144 return is_smm(vcpu
);
4147 static int svm_smi_allowed(struct kvm_vcpu
*vcpu
, bool for_injection
)
4149 struct vcpu_svm
*svm
= to_svm(vcpu
);
4150 if (svm
->nested
.nested_run_pending
)
4153 /* An SMI must not be injected into L2 if it's supposed to VM-Exit. */
4154 if (for_injection
&& is_guest_mode(vcpu
) && nested_exit_on_smi(svm
))
4157 return !svm_smi_blocked(vcpu
);
4160 static int svm_enter_smm(struct kvm_vcpu
*vcpu
, char *smstate
)
4162 struct vcpu_svm
*svm
= to_svm(vcpu
);
4163 struct kvm_host_map map_save
;
4166 if (!is_guest_mode(vcpu
))
4169 /* FED8h - SVM Guest */
4170 put_smstate(u64
, smstate
, 0x7ed8, 1);
4171 /* FEE0h - SVM Guest VMCB Physical Address */
4172 put_smstate(u64
, smstate
, 0x7ee0, svm
->nested
.vmcb12_gpa
);
4174 svm
->vmcb
->save
.rax
= vcpu
->arch
.regs
[VCPU_REGS_RAX
];
4175 svm
->vmcb
->save
.rsp
= vcpu
->arch
.regs
[VCPU_REGS_RSP
];
4176 svm
->vmcb
->save
.rip
= vcpu
->arch
.regs
[VCPU_REGS_RIP
];
4178 ret
= nested_svm_vmexit(svm
);
4183 * KVM uses VMCB01 to store L1 host state while L2 runs but
4184 * VMCB01 is going to be used during SMM and thus the state will
4185 * be lost. Temporary save non-VMLOAD/VMSAVE state to the host save
4186 * area pointed to by MSR_VM_HSAVE_PA. APM guarantees that the
4187 * format of the area is identical to guest save area offsetted
4188 * by 0x400 (matches the offset of 'struct vmcb_save_area'
4189 * within 'struct vmcb'). Note: HSAVE area may also be used by
4190 * L1 hypervisor to save additional host context (e.g. KVM does
4191 * that, see svm_prepare_guest_switch()) which must be
4194 if (kvm_vcpu_map(vcpu
, gpa_to_gfn(svm
->nested
.hsave_msr
),
4195 &map_save
) == -EINVAL
)
4198 BUILD_BUG_ON(offsetof(struct vmcb
, save
) != 0x400);
4200 svm_copy_vmrun_state(map_save
.hva
+ 0x400,
4201 &svm
->vmcb01
.ptr
->save
);
4203 kvm_vcpu_unmap(vcpu
, &map_save
, true);
4207 static int svm_leave_smm(struct kvm_vcpu
*vcpu
, const char *smstate
)
4209 struct vcpu_svm
*svm
= to_svm(vcpu
);
4210 struct kvm_host_map map
, map_save
;
4211 u64 saved_efer
, vmcb12_gpa
;
4212 struct vmcb
*vmcb12
;
4215 if (!guest_cpuid_has(vcpu
, X86_FEATURE_LM
))
4218 /* Non-zero if SMI arrived while vCPU was in guest mode. */
4219 if (!GET_SMSTATE(u64
, smstate
, 0x7ed8))
4222 if (!guest_cpuid_has(vcpu
, X86_FEATURE_SVM
))
4225 saved_efer
= GET_SMSTATE(u64
, smstate
, 0x7ed0);
4226 if (!(saved_efer
& EFER_SVME
))
4229 vmcb12_gpa
= GET_SMSTATE(u64
, smstate
, 0x7ee0);
4230 if (kvm_vcpu_map(vcpu
, gpa_to_gfn(vmcb12_gpa
), &map
) == -EINVAL
)
4234 if (kvm_vcpu_map(vcpu
, gpa_to_gfn(svm
->nested
.hsave_msr
), &map_save
) == -EINVAL
)
4237 if (svm_allocate_nested(svm
))
4241 * Restore L1 host state from L1 HSAVE area as VMCB01 was
4242 * used during SMM (see svm_enter_smm())
4245 svm_copy_vmrun_state(&svm
->vmcb01
.ptr
->save
, map_save
.hva
+ 0x400);
4248 * Enter the nested guest now
4252 nested_copy_vmcb_control_to_cache(svm
, &vmcb12
->control
);
4253 nested_copy_vmcb_save_to_cache(svm
, &vmcb12
->save
);
4254 ret
= enter_svm_guest_mode(vcpu
, vmcb12_gpa
, vmcb12
, false);
4257 kvm_vcpu_unmap(vcpu
, &map_save
, true);
4259 kvm_vcpu_unmap(vcpu
, &map
, true);
4263 static void svm_enable_smi_window(struct kvm_vcpu
*vcpu
)
4265 struct vcpu_svm
*svm
= to_svm(vcpu
);
4267 if (!gif_set(svm
)) {
4268 if (vgif_enabled(svm
))
4269 svm_set_intercept(svm
, INTERCEPT_STGI
);
4270 /* STGI will cause a vm exit */
4272 /* We must be in SMM; RSM will cause a vmexit anyway. */
4276 static bool svm_can_emulate_instruction(struct kvm_vcpu
*vcpu
, int emul_type
,
4277 void *insn
, int insn_len
)
4279 bool smep
, smap
, is_user
;
4283 /* Emulation is always possible when KVM has access to all guest state. */
4284 if (!sev_guest(vcpu
->kvm
))
4287 /* #UD and #GP should never be intercepted for SEV guests. */
4288 WARN_ON_ONCE(emul_type
& (EMULTYPE_TRAP_UD
|
4289 EMULTYPE_TRAP_UD_FORCED
|
4290 EMULTYPE_VMWARE_GP
));
4293 * Emulation is impossible for SEV-ES guests as KVM doesn't have access
4294 * to guest register state.
4296 if (sev_es_guest(vcpu
->kvm
))
4300 * Emulation is possible if the instruction is already decoded, e.g.
4301 * when completing I/O after returning from userspace.
4303 if (emul_type
& EMULTYPE_NO_DECODE
)
4307 * Emulation is possible for SEV guests if and only if a prefilled
4308 * buffer containing the bytes of the intercepted instruction is
4309 * available. SEV guest memory is encrypted with a guest specific key
4310 * and cannot be decrypted by KVM, i.e. KVM would read cyphertext and
4313 * Inject #UD if KVM reached this point without an instruction buffer.
4314 * In practice, this path should never be hit by a well-behaved guest,
4315 * e.g. KVM doesn't intercept #UD or #GP for SEV guests, but this path
4316 * is still theoretically reachable, e.g. via unaccelerated fault-like
4317 * AVIC access, and needs to be handled by KVM to avoid putting the
4318 * guest into an infinite loop. Injecting #UD is somewhat arbitrary,
4319 * but its the least awful option given lack of insight into the guest.
4321 if (unlikely(!insn
)) {
4322 kvm_queue_exception(vcpu
, UD_VECTOR
);
4327 * Emulate for SEV guests if the insn buffer is not empty. The buffer
4328 * will be empty if the DecodeAssist microcode cannot fetch bytes for
4329 * the faulting instruction because the code fetch itself faulted, e.g.
4330 * the guest attempted to fetch from emulated MMIO or a guest page
4331 * table used to translate CS:RIP resides in emulated MMIO.
4333 if (likely(insn_len
))
4337 * Detect and workaround Errata 1096 Fam_17h_00_0Fh.
4340 * When CPU raises #NPF on guest data access and vCPU CR4.SMAP=1, it is
4341 * possible that CPU microcode implementing DecodeAssist will fail to
4342 * read guest memory at CS:RIP and vmcb.GuestIntrBytes will incorrectly
4343 * be '0'. This happens because microcode reads CS:RIP using a _data_
4344 * loap uop with CPL=0 privileges. If the load hits a SMAP #PF, ucode
4345 * gives up and does not fill the instruction bytes buffer.
4347 * As above, KVM reaches this point iff the VM is an SEV guest, the CPU
4348 * supports DecodeAssist, a #NPF was raised, KVM's page fault handler
4349 * triggered emulation (e.g. for MMIO), and the CPU returned 0 in the
4350 * GuestIntrBytes field of the VMCB.
4352 * This does _not_ mean that the erratum has been encountered, as the
4353 * DecodeAssist will also fail if the load for CS:RIP hits a legitimate
4354 * #PF, e.g. if the guest attempt to execute from emulated MMIO and
4355 * encountered a reserved/not-present #PF.
4357 * To hit the erratum, the following conditions must be true:
4358 * 1. CR4.SMAP=1 (obviously).
4359 * 2. CR4.SMEP=0 || CPL=3. If SMEP=1 and CPL<3, the erratum cannot
4360 * have been hit as the guest would have encountered a SMEP
4361 * violation #PF, not a #NPF.
4362 * 3. The #NPF is not due to a code fetch, in which case failure to
4363 * retrieve the instruction bytes is legitimate (see abvoe).
4365 * In addition, don't apply the erratum workaround if the #NPF occurred
4366 * while translating guest page tables (see below).
4368 error_code
= to_svm(vcpu
)->vmcb
->control
.exit_info_1
;
4369 if (error_code
& (PFERR_GUEST_PAGE_MASK
| PFERR_FETCH_MASK
))
4372 cr4
= kvm_read_cr4(vcpu
);
4373 smep
= cr4
& X86_CR4_SMEP
;
4374 smap
= cr4
& X86_CR4_SMAP
;
4375 is_user
= svm_get_cpl(vcpu
) == 3;
4376 if (smap
&& (!smep
|| is_user
)) {
4377 pr_err_ratelimited("KVM: SEV Guest triggered AMD Erratum 1096\n");
4380 * If the fault occurred in userspace, arbitrarily inject #GP
4381 * to avoid killing the guest and to hopefully avoid confusing
4382 * the guest kernel too much, e.g. injecting #PF would not be
4383 * coherent with respect to the guest's page tables. Request
4384 * triple fault if the fault occurred in the kernel as there's
4385 * no fault that KVM can inject without confusing the guest.
4386 * In practice, the triple fault is moot as no sane SEV kernel
4387 * will execute from user memory while also running with SMAP=1.
4390 kvm_inject_gp(vcpu
, 0);
4392 kvm_make_request(KVM_REQ_TRIPLE_FAULT
, vcpu
);
4397 * If the erratum was not hit, simply resume the guest and let it fault
4398 * again. While awful, e.g. the vCPU may get stuck in an infinite loop
4399 * if the fault is at CPL=0, it's the lesser of all evils. Exiting to
4400 * userspace will kill the guest, and letting the emulator read garbage
4401 * will yield random behavior and potentially corrupt the guest.
4403 * Simply resuming the guest is technically not a violation of the SEV
4404 * architecture. AMD's APM states that all code fetches and page table
4405 * accesses for SEV guest are encrypted, regardless of the C-Bit. The
4406 * APM also states that encrypted accesses to MMIO are "ignored", but
4407 * doesn't explicitly define "ignored", i.e. doing nothing and letting
4408 * the guest spin is technically "ignoring" the access.
4413 static bool svm_apic_init_signal_blocked(struct kvm_vcpu
*vcpu
)
4415 struct vcpu_svm
*svm
= to_svm(vcpu
);
4418 * TODO: Last condition latch INIT signals on vCPU when
4419 * vCPU is in guest-mode and vmcb12 defines intercept on INIT.
4420 * To properly emulate the INIT intercept,
4421 * svm_check_nested_events() should call nested_svm_vmexit()
4422 * if an INIT signal is pending.
4424 return !gif_set(svm
) ||
4425 (vmcb_is_intercept(&svm
->vmcb
->control
, INTERCEPT_INIT
));
4428 static void svm_vcpu_deliver_sipi_vector(struct kvm_vcpu
*vcpu
, u8 vector
)
4430 if (!sev_es_guest(vcpu
->kvm
))
4431 return kvm_vcpu_deliver_sipi_vector(vcpu
, vector
);
4433 sev_vcpu_deliver_sipi_vector(vcpu
, vector
);
4436 static void svm_vm_destroy(struct kvm
*kvm
)
4438 avic_vm_destroy(kvm
);
4439 sev_vm_destroy(kvm
);
4442 static int svm_vm_init(struct kvm
*kvm
)
4444 if (!pause_filter_count
|| !pause_filter_thresh
)
4445 kvm
->arch
.pause_in_guest
= true;
4448 int ret
= avic_vm_init(kvm
);
4456 static struct kvm_x86_ops svm_x86_ops __initdata
= {
4459 .hardware_unsetup
= svm_hardware_teardown
,
4460 .hardware_enable
= svm_hardware_enable
,
4461 .hardware_disable
= svm_hardware_disable
,
4462 .cpu_has_accelerated_tpr
= svm_cpu_has_accelerated_tpr
,
4463 .has_emulated_msr
= svm_has_emulated_msr
,
4465 .vcpu_create
= svm_create_vcpu
,
4466 .vcpu_free
= svm_free_vcpu
,
4467 .vcpu_reset
= svm_vcpu_reset
,
4469 .vm_size
= sizeof(struct kvm_svm
),
4470 .vm_init
= svm_vm_init
,
4471 .vm_destroy
= svm_vm_destroy
,
4473 .prepare_guest_switch
= svm_prepare_guest_switch
,
4474 .vcpu_load
= svm_vcpu_load
,
4475 .vcpu_put
= svm_vcpu_put
,
4476 .vcpu_blocking
= avic_vcpu_blocking
,
4477 .vcpu_unblocking
= avic_vcpu_unblocking
,
4479 .update_exception_bitmap
= svm_update_exception_bitmap
,
4480 .get_msr_feature
= svm_get_msr_feature
,
4481 .get_msr
= svm_get_msr
,
4482 .set_msr
= svm_set_msr
,
4483 .get_segment_base
= svm_get_segment_base
,
4484 .get_segment
= svm_get_segment
,
4485 .set_segment
= svm_set_segment
,
4486 .get_cpl
= svm_get_cpl
,
4487 .get_cs_db_l_bits
= kvm_get_cs_db_l_bits
,
4488 .set_cr0
= svm_set_cr0
,
4489 .post_set_cr3
= svm_post_set_cr3
,
4490 .is_valid_cr4
= svm_is_valid_cr4
,
4491 .set_cr4
= svm_set_cr4
,
4492 .set_efer
= svm_set_efer
,
4493 .get_idt
= svm_get_idt
,
4494 .set_idt
= svm_set_idt
,
4495 .get_gdt
= svm_get_gdt
,
4496 .set_gdt
= svm_set_gdt
,
4497 .set_dr7
= svm_set_dr7
,
4498 .sync_dirty_debug_regs
= svm_sync_dirty_debug_regs
,
4499 .cache_reg
= svm_cache_reg
,
4500 .get_rflags
= svm_get_rflags
,
4501 .set_rflags
= svm_set_rflags
,
4502 .get_if_flag
= svm_get_if_flag
,
4504 .tlb_flush_all
= svm_flush_tlb
,
4505 .tlb_flush_current
= svm_flush_tlb
,
4506 .tlb_flush_gva
= svm_flush_tlb_gva
,
4507 .tlb_flush_guest
= svm_flush_tlb
,
4509 .vcpu_pre_run
= svm_vcpu_pre_run
,
4510 .run
= svm_vcpu_run
,
4511 .handle_exit
= handle_exit
,
4512 .skip_emulated_instruction
= skip_emulated_instruction
,
4513 .update_emulated_instruction
= NULL
,
4514 .set_interrupt_shadow
= svm_set_interrupt_shadow
,
4515 .get_interrupt_shadow
= svm_get_interrupt_shadow
,
4516 .patch_hypercall
= svm_patch_hypercall
,
4517 .set_irq
= svm_set_irq
,
4518 .set_nmi
= svm_inject_nmi
,
4519 .queue_exception
= svm_queue_exception
,
4520 .cancel_injection
= svm_cancel_injection
,
4521 .interrupt_allowed
= svm_interrupt_allowed
,
4522 .nmi_allowed
= svm_nmi_allowed
,
4523 .get_nmi_mask
= svm_get_nmi_mask
,
4524 .set_nmi_mask
= svm_set_nmi_mask
,
4525 .enable_nmi_window
= svm_enable_nmi_window
,
4526 .enable_irq_window
= svm_enable_irq_window
,
4527 .update_cr8_intercept
= svm_update_cr8_intercept
,
4528 .set_virtual_apic_mode
= svm_set_virtual_apic_mode
,
4529 .refresh_apicv_exec_ctrl
= svm_refresh_apicv_exec_ctrl
,
4530 .check_apicv_inhibit_reasons
= svm_check_apicv_inhibit_reasons
,
4531 .load_eoi_exitmap
= svm_load_eoi_exitmap
,
4532 .hwapic_irr_update
= svm_hwapic_irr_update
,
4533 .hwapic_isr_update
= svm_hwapic_isr_update
,
4534 .apicv_post_state_restore
= avic_post_state_restore
,
4536 .set_tss_addr
= svm_set_tss_addr
,
4537 .set_identity_map_addr
= svm_set_identity_map_addr
,
4538 .get_mt_mask
= svm_get_mt_mask
,
4540 .get_exit_info
= svm_get_exit_info
,
4542 .vcpu_after_set_cpuid
= svm_vcpu_after_set_cpuid
,
4544 .has_wbinvd_exit
= svm_has_wbinvd_exit
,
4546 .get_l2_tsc_offset
= svm_get_l2_tsc_offset
,
4547 .get_l2_tsc_multiplier
= svm_get_l2_tsc_multiplier
,
4548 .write_tsc_offset
= svm_write_tsc_offset
,
4549 .write_tsc_multiplier
= svm_write_tsc_multiplier
,
4551 .load_mmu_pgd
= svm_load_mmu_pgd
,
4553 .check_intercept
= svm_check_intercept
,
4554 .handle_exit_irqoff
= svm_handle_exit_irqoff
,
4556 .request_immediate_exit
= __kvm_request_immediate_exit
,
4558 .sched_in
= svm_sched_in
,
4560 .pmu_ops
= &amd_pmu_ops
,
4561 .nested_ops
= &svm_nested_ops
,
4563 .deliver_interrupt
= svm_deliver_interrupt
,
4564 .dy_apicv_has_pending_interrupt
= svm_dy_apicv_has_pending_interrupt
,
4565 .update_pi_irte
= svm_update_pi_irte
,
4566 .setup_mce
= svm_setup_mce
,
4568 .smi_allowed
= svm_smi_allowed
,
4569 .enter_smm
= svm_enter_smm
,
4570 .leave_smm
= svm_leave_smm
,
4571 .enable_smi_window
= svm_enable_smi_window
,
4573 .mem_enc_op
= svm_mem_enc_op
,
4574 .mem_enc_reg_region
= svm_register_enc_region
,
4575 .mem_enc_unreg_region
= svm_unregister_enc_region
,
4577 .vm_copy_enc_context_from
= svm_vm_copy_asid_from
,
4578 .vm_move_enc_context_from
= svm_vm_migrate_from
,
4580 .can_emulate_instruction
= svm_can_emulate_instruction
,
4582 .apic_init_signal_blocked
= svm_apic_init_signal_blocked
,
4584 .msr_filter_changed
= svm_msr_filter_changed
,
4585 .complete_emulated_msr
= svm_complete_emulated_msr
,
4587 .vcpu_deliver_sipi_vector
= svm_vcpu_deliver_sipi_vector
,
4591 * The default MMIO mask is a single bit (excluding the present bit),
4592 * which could conflict with the memory encryption bit. Check for
4593 * memory encryption support and override the default MMIO mask if
4594 * memory encryption is enabled.
4596 static __init
void svm_adjust_mmio_mask(void)
4598 unsigned int enc_bit
, mask_bit
;
4601 /* If there is no memory encryption support, use existing mask */
4602 if (cpuid_eax(0x80000000) < 0x8000001f)
4605 /* If memory encryption is not enabled, use existing mask */
4606 rdmsrl(MSR_AMD64_SYSCFG
, msr
);
4607 if (!(msr
& MSR_AMD64_SYSCFG_MEM_ENCRYPT
))
4610 enc_bit
= cpuid_ebx(0x8000001f) & 0x3f;
4611 mask_bit
= boot_cpu_data
.x86_phys_bits
;
4613 /* Increment the mask bit if it is the same as the encryption bit */
4614 if (enc_bit
== mask_bit
)
4618 * If the mask bit location is below 52, then some bits above the
4619 * physical addressing limit will always be reserved, so use the
4620 * rsvd_bits() function to generate the mask. This mask, along with
4621 * the present bit, will be used to generate a page fault with
4624 * If the mask bit location is 52 (or above), then clear the mask.
4626 mask
= (mask_bit
< 52) ? rsvd_bits(mask_bit
, 51) | PT_PRESENT_MASK
: 0;
4628 kvm_mmu_set_mmio_spte_mask(mask
, mask
, PT_WRITABLE_MASK
| PT_USER_MASK
);
4631 static __init
void svm_set_cpu_caps(void)
4637 /* CPUID 0x80000001 and 0x8000000A (SVM features) */
4639 kvm_cpu_cap_set(X86_FEATURE_SVM
);
4642 kvm_cpu_cap_set(X86_FEATURE_NRIPS
);
4645 kvm_cpu_cap_set(X86_FEATURE_NPT
);
4648 kvm_cpu_cap_set(X86_FEATURE_TSCRATEMSR
);
4650 /* Nested VM can receive #VMEXIT instead of triggering #GP */
4651 kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK
);
4654 /* CPUID 0x80000008 */
4655 if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD
) ||
4656 boot_cpu_has(X86_FEATURE_AMD_SSBD
))
4657 kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD
);
4659 /* AMD PMU PERFCTR_CORE CPUID */
4660 if (enable_pmu
&& boot_cpu_has(X86_FEATURE_PERFCTR_CORE
))
4661 kvm_cpu_cap_set(X86_FEATURE_PERFCTR_CORE
);
4663 /* CPUID 0x8000001F (SME/SEV features) */
4667 static __init
int svm_hardware_setup(void)
4670 struct page
*iopm_pages
;
4673 unsigned int order
= get_order(IOPM_SIZE
);
4676 * NX is required for shadow paging and for NPT if the NX huge pages
4677 * mitigation is enabled.
4679 if (!boot_cpu_has(X86_FEATURE_NX
)) {
4680 pr_err_ratelimited("NX (Execute Disable) not supported\n");
4683 kvm_enable_efer_bits(EFER_NX
);
4685 iopm_pages
= alloc_pages(GFP_KERNEL
, order
);
4690 iopm_va
= page_address(iopm_pages
);
4691 memset(iopm_va
, 0xff, PAGE_SIZE
* (1 << order
));
4692 iopm_base
= page_to_pfn(iopm_pages
) << PAGE_SHIFT
;
4694 init_msrpm_offsets();
4696 supported_xcr0
&= ~(XFEATURE_MASK_BNDREGS
| XFEATURE_MASK_BNDCSR
);
4698 if (boot_cpu_has(X86_FEATURE_FXSR_OPT
))
4699 kvm_enable_efer_bits(EFER_FFXSR
);
4702 if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR
)) {
4703 tsc_scaling
= false;
4705 pr_info("TSC scaling supported\n");
4706 kvm_has_tsc_control
= true;
4707 kvm_max_tsc_scaling_ratio
= TSC_RATIO_MAX
;
4708 kvm_tsc_scaling_ratio_frac_bits
= 32;
4712 tsc_aux_uret_slot
= kvm_add_user_return_msr(MSR_TSC_AUX
);
4714 /* Check for pause filtering support */
4715 if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER
)) {
4716 pause_filter_count
= 0;
4717 pause_filter_thresh
= 0;
4718 } else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD
)) {
4719 pause_filter_thresh
= 0;
4723 printk(KERN_INFO
"kvm: Nested Virtualization enabled\n");
4724 kvm_enable_efer_bits(EFER_SVME
| EFER_LMSLE
);
4728 * KVM's MMU doesn't support using 2-level paging for itself, and thus
4729 * NPT isn't supported if the host is using 2-level paging since host
4730 * CR4 is unchanged on VMRUN.
4732 if (!IS_ENABLED(CONFIG_X86_64
) && !IS_ENABLED(CONFIG_X86_PAE
))
4733 npt_enabled
= false;
4735 if (!boot_cpu_has(X86_FEATURE_NPT
))
4736 npt_enabled
= false;
4738 /* Force VM NPT level equal to the host's paging level */
4739 kvm_configure_mmu(npt_enabled
, get_npt_level(),
4740 get_npt_level(), PG_LEVEL_1G
);
4741 pr_info("kvm: Nested Paging %sabled\n", npt_enabled
? "en" : "dis");
4743 /* Note, SEV setup consumes npt_enabled. */
4744 sev_hardware_setup();
4746 svm_hv_hardware_setup();
4748 svm_adjust_mmio_mask();
4750 for_each_possible_cpu(cpu
) {
4751 r
= svm_cpu_init(cpu
);
4757 if (!boot_cpu_has(X86_FEATURE_NRIPS
))
4761 enable_apicv
= avic
= avic
&& npt_enabled
&& boot_cpu_has(X86_FEATURE_AVIC
);
4764 pr_info("AVIC enabled\n");
4766 amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier
);
4768 svm_x86_ops
.vcpu_blocking
= NULL
;
4769 svm_x86_ops
.vcpu_unblocking
= NULL
;
4774 !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD
) ||
4775 !IS_ENABLED(CONFIG_X86_64
)) {
4778 pr_info("Virtual VMLOAD VMSAVE supported\n");
4782 if (boot_cpu_has(X86_FEATURE_SVME_ADDR_CHK
))
4783 svm_gp_erratum_intercept
= false;
4786 if (!boot_cpu_has(X86_FEATURE_VGIF
))
4789 pr_info("Virtual GIF supported\n");
4793 if (!boot_cpu_has(X86_FEATURE_LBRV
))
4796 pr_info("LBR virtualization supported\n");
4800 pr_info("PMU virtualization is disabled\n");
4805 * It seems that on AMD processors PTE's accessed bit is
4806 * being set by the CPU hardware before the NPF vmexit.
4807 * This is not expected behaviour and our tests fail because
4809 * A workaround here is to disable support for
4810 * GUEST_MAXPHYADDR < HOST_MAXPHYADDR if NPT is enabled.
4811 * In this case userspace can know if there is support using
4812 * KVM_CAP_SMALLER_MAXPHYADDR extension and decide how to handle
4814 * If future AMD CPU models change the behaviour described above,
4815 * this variable can be changed accordingly
4817 allow_smaller_maxphyaddr
= !npt_enabled
;
4822 svm_hardware_teardown();
4827 static struct kvm_x86_init_ops svm_init_ops __initdata
= {
4828 .cpu_has_kvm_support
= has_svm
,
4829 .disabled_by_bios
= is_disabled
,
4830 .hardware_setup
= svm_hardware_setup
,
4831 .check_processor_compatibility
= svm_check_processor_compat
,
4833 .runtime_ops
= &svm_x86_ops
,
4836 static int __init
svm_init(void)
4838 __unused_size_checks();
4840 return kvm_init(&svm_init_ops
, sizeof(struct vcpu_svm
),
4841 __alignof__(struct vcpu_svm
), THIS_MODULE
);
4844 static void __exit
svm_exit(void)
4849 module_init(svm_init
)
4850 module_exit(svm_exit
)