KVM: VMX: Prevent RSB underflow before vmenter

[mirror_ubuntu-jammy-kernel.git] / arch / x86 / kvm / vmx / vmenter.S

diff --git a/arch/x86/kvm/vmx/vmenter.S b/arch/x86/kvm/vmx/vmenter.S
index 1c1a3d6c2af251b4c8aee0a3df8b6c36418f71d7..8a37ca74d9cf2ec7e62d667748cc65e5e9ab639f 100644 (file)
--- a/arch/x86/kvm/vmx/vmenter.S
+++ b/arch/x86/kvm/vmx/vmenter.S
@@ -1,9 +1,11 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #include <linux/linkage.h>
 #include <asm/asm.h>
+#include <asm/asm-offsets.h>
 #include <asm/bitsperlong.h>
 #include <asm/kvm_vcpu_regs.h>
 #include <asm/nospec-branch.h>
+#include <asm/percpu.h>
 #include <asm/segment.h>
 #include "run_flags.h"
 
@@ -73,6 +75,33 @@ SYM_FUNC_START(__vmx_vcpu_run)
        lea (%_ASM_SP), %_ASM_ARG2
        call vmx_update_host_rsp
 
+       ALTERNATIVE "jmp .Lspec_ctrl_done", "", X86_FEATURE_MSR_SPEC_CTRL
+
+       /*
+        * SPEC_CTRL handling: if the guest's SPEC_CTRL value differs from the
+        * host's, write the MSR.
+        *
+        * IMPORTANT: To avoid RSB underflow attacks and any other nastiness,
+        * there must not be any returns or indirect branches between this code
+        * and vmentry.
+        */
+       mov 2*WORD_SIZE(%_ASM_SP), %_ASM_DI
+       movl VMX_spec_ctrl(%_ASM_DI), %edi
+       movl PER_CPU_VAR(x86_spec_ctrl_current), %esi
+       cmp %edi, %esi
+       je .Lspec_ctrl_done
+       mov $MSR_IA32_SPEC_CTRL, %ecx
+       xor %edx, %edx
+       mov %edi, %eax
+       wrmsr
+
+.Lspec_ctrl_done:
+
+       /*
+        * Since vmentry is serializing on affected CPUs, there's no need for
+        * an LFENCE to stop speculation from skipping the wrmsr.
+        */
+
        /* Load @regs to RAX. */
        mov (%_ASM_SP), %_ASM_AX
 
@@ -193,15 +222,15 @@ SYM_INNER_LABEL(vmx_vmexit, SYM_L_GLOBAL)
         * IMPORTANT: RSB filling and SPEC_CTRL handling must be done before
         * the first unbalanced RET after vmexit!
         *
-        * For retpoline, RSB filling is needed to prevent poisoned RSB entries
-        * and (in some cases) RSB underflow.
+        * For retpoline or IBRS, RSB filling is needed to prevent poisoned RSB
+        * entries and (in some cases) RSB underflow.
         *
         * eIBRS has its own protection against poisoned RSB, so it doesn't
         * need the RSB filling sequence.  But it does need to be enabled
         * before the first unbalanced RET.
          */
 
-       FILL_RETURN_BUFFER %_ASM_CX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE
+       FILL_RETURN_BUFFER %_ASM_CX, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_VMEXIT
 
        pop %_ASM_ARG2  /* @flags */
        pop %_ASM_ARG1  /* @vmx */