arm64: KVM: Use per-CPU vector when BP hardening is enabled

[mirror_ubuntu-artful-kernel.git] / arch / arm64 / include / asm / kvm_mmu.h

diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index a89cc22abadcaced0ee272ec38eb8e490292fd8d..2d6d4bd9de52b48cbaaeae1cea86544c3f406bec 100644 (file)
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -175,18 +175,15 @@ static inline pmd_t kvm_s2pmd_mkwrite(pmd_t pmd)
 
 static inline void kvm_set_s2pte_readonly(pte_t *pte)
 {
-       pteval_t pteval;
-       unsigned long tmp;
-
-       asm volatile("//        kvm_set_s2pte_readonly\n"
-       "       prfm    pstl1strm, %2\n"
-       "1:     ldxr    %0, %2\n"
-       "       and     %0, %0, %3              // clear PTE_S2_RDWR\n"
-       "       orr     %0, %0, %4              // set PTE_S2_RDONLY\n"
-       "       stxr    %w1, %0, %2\n"
-       "       cbnz    %w1, 1b\n"
-       : "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*pte))
-       : "L" (~PTE_S2_RDWR), "L" (PTE_S2_RDONLY));
+       pteval_t old_pteval, pteval;
+
+       pteval = READ_ONCE(pte_val(*pte));
+       do {
+               old_pteval = pteval;
+               pteval &= ~PTE_S2_RDWR;
+               pteval |= PTE_S2_RDONLY;
+               pteval = cmpxchg_relaxed(&pte_val(*pte), old_pteval, pteval);
+       } while (pteval != old_pteval);
 }
 
 static inline bool kvm_s2pte_readonly(pte_t *pte)
@@ -312,5 +309,43 @@ static inline unsigned int kvm_get_vmid_bits(void)
        return (cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR1_VMIDBITS_SHIFT) == 2) ? 16 : 8;
 }
 
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+#include <asm/mmu.h>
+
+static inline void *kvm_get_hyp_vector(void)
+{
+       struct bp_hardening_data *data = arm64_get_bp_hardening_data();
+       void *vect = kvm_ksym_ref(__kvm_hyp_vector);
+
+       if (data->fn) {
+               vect = __bp_harden_hyp_vecs_start +
+                      data->hyp_vectors_slot * SZ_2K;
+
+               if (!has_vhe())
+                       vect = lm_alias(vect);
+       }
+
+       return vect;
+}
+
+static inline int kvm_map_vectors(void)
+{
+       return create_hyp_mappings(kvm_ksym_ref(__bp_harden_hyp_vecs_start),
+                                  kvm_ksym_ref(__bp_harden_hyp_vecs_end),
+                                  PAGE_HYP_EXEC);
+}
+
+#else
+static inline void *kvm_get_hyp_vector(void)
+{
+       return kvm_ksym_ref(__kvm_hyp_vector);
+}
+
+static inline int kvm_map_vectors(void)
+{
+       return 0;
+}
+#endif
+
 #endif /* __ASSEMBLY__ */
 #endif /* __ARM64_KVM_MMU_H__ */