[mirror_ubuntu-bionic-kernel.git] / arch / arm64 / include / asm / kvm_mmu.h

/*
 * Copyright (C) 2012,2013 - ARM Ltd
 * Author: Marc Zyngier <marc.zyngier@arm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef __ARM64_KVM_MMU_H__
#define __ARM64_KVM_MMU_H__

#include <asm/page.h>
#include <asm/memory.h>
#include <asm/cpufeature.h>

/*
 * As ARMv8.0 only has the TTBR0_EL2 register, we cannot express
 * "negative" addresses. This makes it impossible to directly share
 * mappings with the kernel.
 *
 * Instead, give the HYP mode its own VA region at a fixed offset from
 * the kernel by just masking the top bits (which are all ones for a
 * kernel address). We need to find out how many bits to mask.
 *
 * We want to build a set of page tables that cover both parts of the
 * idmap (the trampoline page used to initialize EL2), and our normal
 * runtime VA space, at the same time.
 *
 * Given that the kernel uses VA_BITS for its entire address space,
 * and that half of that space (VA_BITS - 1) is used for the linear
 * mapping, we can also limit the EL2 space to (VA_BITS - 1).
 *
 * The main question is "Within the VA_BITS space, does EL2 use the
 * top or the bottom half of that space to shadow the kernel's linear
 * mapping?". As we need to idmap the trampoline page, this is
 * determined by the range in which this page lives.
 *
 * If the page is in the bottom half, we have to use the top half. If
 * the page is in the top half, we have to use the bottom half:
 *
 * T = __pa_symbol(__hyp_idmap_text_start)
 * if (T & BIT(VA_BITS - 1))
 *	HYP_VA_MIN = 0  //idmap in upper half
 * else
 *	HYP_VA_MIN = 1 << (VA_BITS - 1)
 * HYP_VA_MAX = HYP_VA_MIN + (1 << (VA_BITS - 1)) - 1
 *
 * This of course assumes that the trampoline page exists within the
 * VA_BITS range. If it doesn't, then it means we're in the odd case
 * where the kernel idmap (as well as HYP) uses more levels than the
 * kernel runtime page tables (as seen when the kernel is configured
 * for 4k pages, 39bits VA, and yet memory lives just above that
 * limit, forcing the idmap to use 4 levels of page tables while the
 * kernel itself only uses 3). In this particular case, it doesn't
 * matter which side of VA_BITS we use, as we're guaranteed not to
 * conflict with anything.
 *
 * When using VHE, there are no separate hyp mappings and all KVM
 * functionality is already mapped as part of the main kernel
 * mappings, and none of this applies in that case.
 */

#define HYP_PAGE_OFFSET_HIGH_MASK	((UL(1) << VA_BITS) - 1)
#define HYP_PAGE_OFFSET_LOW_MASK	((UL(1) << (VA_BITS - 1)) - 1)

#ifdef __ASSEMBLY__

#include <asm/alternative.h>
#include <asm/cpufeature.h>

/*
 * Convert a kernel VA into a HYP VA.
 * reg: VA to be converted.
 *
 * This generates the following sequences:
 * - High mask:
 *		and x0, x0, #HYP_PAGE_OFFSET_HIGH_MASK
 *		nop
 * - Low mask:
 *		and x0, x0, #HYP_PAGE_OFFSET_HIGH_MASK
 *		and x0, x0, #HYP_PAGE_OFFSET_LOW_MASK
 * - VHE:
 *		nop
 *		nop
 *
 * The "low mask" version works because the mask is a strict subset of
 * the "high mask", hence performing the first mask for nothing.
 * Should be completely invisible on any viable CPU.
 */
.macro kern_hyp_va	reg
alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
	and     \reg, \reg, #HYP_PAGE_OFFSET_HIGH_MASK
alternative_else_nop_endif
alternative_if ARM64_HYP_OFFSET_LOW
	and     \reg, \reg, #HYP_PAGE_OFFSET_LOW_MASK
alternative_else_nop_endif
.endm

#else

#include <asm/pgalloc.h>
#include <asm/cache.h>
#include <asm/cacheflush.h>
#include <asm/mmu_context.h>
#include <asm/pgtable.h>

static inline unsigned long __kern_hyp_va(unsigned long v)
{
	asm volatile(ALTERNATIVE("and %0, %0, %1",
				 "nop",
				 ARM64_HAS_VIRT_HOST_EXTN)
		     : "+r" (v)
		     : "i" (HYP_PAGE_OFFSET_HIGH_MASK));
	asm volatile(ALTERNATIVE("nop",
				 "and %0, %0, %1",
				 ARM64_HYP_OFFSET_LOW)
		     : "+r" (v)
		     : "i" (HYP_PAGE_OFFSET_LOW_MASK));
	return v;
}

#define kern_hyp_va(v) 	((typeof(v))(__kern_hyp_va((unsigned long)(v))))

/*
 * Obtain the PC-relative address of a kernel symbol
 * s: symbol
 *
 * The goal of this macro is to return a symbol's address based on a
 * PC-relative computation, as opposed to a loading the VA from a
 * constant pool or something similar. This works well for HYP, as an
 * absolute VA is guaranteed to be wrong. Only use this if trying to
 * obtain the address of a symbol (i.e. not something you obtained by
 * following a pointer).
 */
#define hyp_symbol_addr(s)						\
	({								\
		typeof(s) *addr;					\
		asm("adrp	%0, %1\n"				\
		    "add	%0, %0, :lo12:%1\n"			\
		    : "=r" (addr) : "S" (&s));				\
		addr;							\
	})

/*
 * We currently only support a 40bit IPA.
 */
#define KVM_PHYS_SHIFT	(40)
#define KVM_PHYS_SIZE	(1UL << KVM_PHYS_SHIFT)
#define KVM_PHYS_MASK	(KVM_PHYS_SIZE - 1UL)

#include <asm/stage2_pgtable.h>

int create_hyp_mappings(void *from, void *to, pgprot_t prot);
int create_hyp_io_mappings(void *from, void *to, phys_addr_t);
void free_hyp_pgds(void);

void stage2_unmap_vm(struct kvm *kvm);
int kvm_alloc_stage2_pgd(struct kvm *kvm);
void kvm_free_stage2_pgd(struct kvm *kvm);
int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
			  phys_addr_t pa, unsigned long size, bool writable);

int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run);

void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu);

phys_addr_t kvm_mmu_get_httbr(void);
phys_addr_t kvm_get_idmap_vector(void);
int kvm_mmu_init(void);
void kvm_clear_hyp_idmap(void);

#define	kvm_set_pte(ptep, pte)		set_pte(ptep, pte)
#define	kvm_set_pmd(pmdp, pmd)		set_pmd(pmdp, pmd)

static inline pte_t kvm_s2pte_mkwrite(pte_t pte)
{
	pte_val(pte) |= PTE_S2_RDWR;
	return pte;
}

static inline pmd_t kvm_s2pmd_mkwrite(pmd_t pmd)
{
	pmd_val(pmd) |= PMD_S2_RDWR;
	return pmd;
}

static inline void kvm_set_s2pte_readonly(pte_t *pte)
{
	pteval_t old_pteval, pteval;

	pteval = READ_ONCE(pte_val(*pte));
	do {
		old_pteval = pteval;
		pteval &= ~PTE_S2_RDWR;
		pteval |= PTE_S2_RDONLY;
		pteval = cmpxchg_relaxed(&pte_val(*pte), old_pteval, pteval);
	} while (pteval != old_pteval);
}

static inline bool kvm_s2pte_readonly(pte_t *pte)
{
	return (pte_val(*pte) & PTE_S2_RDWR) == PTE_S2_RDONLY;
}

static inline void kvm_set_s2pmd_readonly(pmd_t *pmd)
{
	kvm_set_s2pte_readonly((pte_t *)pmd);
}

static inline bool kvm_s2pmd_readonly(pmd_t *pmd)
{
	return kvm_s2pte_readonly((pte_t *)pmd);
}

static inline bool kvm_page_empty(void *ptr)
{
	struct page *ptr_page = virt_to_page(ptr);
	return page_count(ptr_page) == 1;
}

#define hyp_pte_table_empty(ptep) kvm_page_empty(ptep)

#ifdef __PAGETABLE_PMD_FOLDED
#define hyp_pmd_table_empty(pmdp) (0)
#else
#define hyp_pmd_table_empty(pmdp) kvm_page_empty(pmdp)
#endif

#ifdef __PAGETABLE_PUD_FOLDED
#define hyp_pud_table_empty(pudp) (0)
#else
#define hyp_pud_table_empty(pudp) kvm_page_empty(pudp)
#endif

struct kvm;

#define kvm_flush_dcache_to_poc(a,l)	__flush_dcache_area((a), (l))

static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu)
{
	return (vcpu_sys_reg(vcpu, SCTLR_EL1) & 0b101) == 0b101;
}

static inline void __coherent_cache_guest_page(struct kvm_vcpu *vcpu,
					       kvm_pfn_t pfn,
					       unsigned long size)
{
	void *va = page_address(pfn_to_page(pfn));

	kvm_flush_dcache_to_poc(va, size);

	if (icache_is_aliasing()) {
		/* any kind of VIPT cache */
		__flush_icache_all();
	} else if (is_kernel_in_hyp_mode() || !icache_is_vpipt()) {
		/* PIPT or VPIPT at EL2 (see comment in __kvm_tlb_flush_vmid_ipa) */
		flush_icache_range((unsigned long)va,
				   (unsigned long)va + size);
	}
}

static inline void __kvm_flush_dcache_pte(pte_t pte)
{
	struct page *page = pte_page(pte);
	kvm_flush_dcache_to_poc(page_address(page), PAGE_SIZE);
}

static inline void __kvm_flush_dcache_pmd(pmd_t pmd)
{
	struct page *page = pmd_page(pmd);
	kvm_flush_dcache_to_poc(page_address(page), PMD_SIZE);
}

static inline void __kvm_flush_dcache_pud(pud_t pud)
{
	struct page *page = pud_page(pud);
	kvm_flush_dcache_to_poc(page_address(page), PUD_SIZE);
}

#define kvm_virt_to_phys(x)		__pa_symbol(x)

void kvm_set_way_flush(struct kvm_vcpu *vcpu);
void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled);

static inline bool __kvm_cpu_uses_extended_idmap(void)
{
	return __cpu_uses_extended_idmap();
}

static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd,
				       pgd_t *hyp_pgd,
				       pgd_t *merged_hyp_pgd,
				       unsigned long hyp_idmap_start)
{
	int idmap_idx;

	/*
	 * Use the first entry to access the HYP mappings. It is
	 * guaranteed to be free, otherwise we wouldn't use an
	 * extended idmap.
	 */
	VM_BUG_ON(pgd_val(merged_hyp_pgd[0]));
	merged_hyp_pgd[0] = __pgd(__pa(hyp_pgd) | PMD_TYPE_TABLE);

	/*
	 * Create another extended level entry that points to the boot HYP map,
	 * which contains an ID mapping of the HYP init code. We essentially
	 * merge the boot and runtime HYP maps by doing so, but they don't
	 * overlap anyway, so this is fine.
	 */
	idmap_idx = hyp_idmap_start >> VA_BITS;
	VM_BUG_ON(pgd_val(merged_hyp_pgd[idmap_idx]));
	merged_hyp_pgd[idmap_idx] = __pgd(__pa(boot_hyp_pgd) | PMD_TYPE_TABLE);
}

static inline unsigned int kvm_get_vmid_bits(void)
{
	int reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);

	return (cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR1_VMIDBITS_SHIFT) == 2) ? 16 : 8;
}

/*
 * We are not in the kvm->srcu critical section most of the time, so we take
 * the SRCU read lock here. Since we copy the data from the user page, we
 * can immediately drop the lock again.
 */
static inline int kvm_read_guest_lock(struct kvm *kvm,
				      gpa_t gpa, void *data, unsigned long len)
{
	int srcu_idx = srcu_read_lock(&kvm->srcu);
	int ret = kvm_read_guest(kvm, gpa, data, len);

	srcu_read_unlock(&kvm->srcu, srcu_idx);

	return ret;
}

static inline int kvm_write_guest_lock(struct kvm *kvm, gpa_t gpa,
				       const void *data, unsigned long len)
{
	int srcu_idx = srcu_read_lock(&kvm->srcu);
	int ret = kvm_write_guest(kvm, gpa, data, len);

	srcu_read_unlock(&kvm->srcu, srcu_idx);

	return ret;
}

#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
#include <asm/mmu.h>

static inline void *kvm_get_hyp_vector(void)
{
	struct bp_hardening_data *data = arm64_get_bp_hardening_data();
	void *vect = kvm_ksym_ref(__kvm_hyp_vector);

	if (data->fn) {
		vect = __bp_harden_hyp_vecs_start +
		       data->hyp_vectors_slot * SZ_2K;

		if (!has_vhe())
			vect = lm_alias(vect);
	}

	return vect;
}

static inline int kvm_map_vectors(void)
{
	return create_hyp_mappings(kvm_ksym_ref(__bp_harden_hyp_vecs_start),
				   kvm_ksym_ref(__bp_harden_hyp_vecs_end),
				   PAGE_HYP_EXEC);
}

#else
static inline void *kvm_get_hyp_vector(void)
{
	return kvm_ksym_ref(__kvm_hyp_vector);
}

static inline int kvm_map_vectors(void)
{
	return 0;
}
#endif

#ifdef CONFIG_ARM64_SSBD
DECLARE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required);

static inline int hyp_map_aux_data(void)
{
	int cpu, err;

	for_each_possible_cpu(cpu) {
		u64 *ptr;

		ptr = per_cpu_ptr(&arm64_ssbd_callback_required, cpu);
		err = create_hyp_mappings(ptr, ptr + 1, PAGE_HYP);
		if (err)
			return err;
	}
	return 0;
}
#else
static inline int hyp_map_aux_data(void)
{
	return 0;
}
#endif

#endif /* __ASSEMBLY__ */
#endif /* __ARM64_KVM_MMU_H__ */
Commit	Line	Data
37c43753 MZ	1	/*
	2	* Copyright (C) 2012,2013 - ARM Ltd
	3	* Author: Marc Zyngier <marc.zyngier@arm.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License version 2 as
	7	* published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
	18	#ifndef __ARM64_KVM_MMU_H__
	19	#define __ARM64_KVM_MMU_H__
	20
	21	#include <asm/page.h>
	22	#include <asm/memory.h>
20475f78	23	#include <asm/cpufeature.h>
37c43753 MZ	24
37c43753 MZ	25	/*
cedbb8b7	26	* As ARMv8.0 only has the TTBR0_EL2 register, we cannot express
37c43753 MZ	27	* "negative" addresses. This makes it impossible to directly share
	28	* mappings with the kernel.
	29	*
	30	* Instead, give the HYP mode its own VA region at a fixed offset from
	31	* the kernel by just masking the top bits (which are all ones for a
82a81bff	32	* kernel address). We need to find out how many bits to mask.
cedbb8b7	33	*
82a81bff MZ	34	* We want to build a set of page tables that cover both parts of the
	35	* idmap (the trampoline page used to initialize EL2), and our normal
	36	* runtime VA space, at the same time.
	37	*
	38	* Given that the kernel uses VA_BITS for its entire address space,
	39	* and that half of that space (VA_BITS - 1) is used for the linear
	40	* mapping, we can also limit the EL2 space to (VA_BITS - 1).
	41	*
	42	* The main question is "Within the VA_BITS space, does EL2 use the
	43	* top or the bottom half of that space to shadow the kernel's linear
	44	* mapping?". As we need to idmap the trampoline page, this is
	45	* determined by the range in which this page lives.
	46	*
	47	* If the page is in the bottom half, we have to use the top half. If
	48	* the page is in the top half, we have to use the bottom half:
	49	*
2077be67	50	* T = __pa_symbol(__hyp_idmap_text_start)
82a81bff MZ	51	* if (T & BIT(VA_BITS - 1))
	52	* HYP_VA_MIN = 0 //idmap in upper half
	53	* else
	54	* HYP_VA_MIN = 1 << (VA_BITS - 1)
	55	* HYP_VA_MAX = HYP_VA_MIN + (1 << (VA_BITS - 1)) - 1
	56	*
	57	* This of course assumes that the trampoline page exists within the
	58	* VA_BITS range. If it doesn't, then it means we're in the odd case
	59	* where the kernel idmap (as well as HYP) uses more levels than the
	60	* kernel runtime page tables (as seen when the kernel is configured
	61	* for 4k pages, 39bits VA, and yet memory lives just above that
	62	* limit, forcing the idmap to use 4 levels of page tables while the
	63	* kernel itself only uses 3). In this particular case, it doesn't
	64	* matter which side of VA_BITS we use, as we're guaranteed not to
	65	* conflict with anything.
	66	*
	67	* When using VHE, there are no separate hyp mappings and all KVM
	68	* functionality is already mapped as part of the main kernel
	69	* mappings, and none of this applies in that case.
37c43753	70	*/
d53d9bc6 MZ	71
	72	#define HYP_PAGE_OFFSET_HIGH_MASK ((UL(1) << VA_BITS) - 1)
	73	#define HYP_PAGE_OFFSET_LOW_MASK ((UL(1) << (VA_BITS - 1)) - 1)
	74
37c43753 MZ	75	#ifdef __ASSEMBLY__
37c43753 MZ	76
cedbb8b7 MZ	77	#include <asm/alternative.h>
	78	#include <asm/cpufeature.h>
	79
37c43753 MZ	80	/*
	81	* Convert a kernel VA into a HYP VA.
	82	* reg: VA to be converted.
fd81e6bf MZ	83	*
	84	* This generates the following sequences:
	85	* - High mask:
	86	* and x0, x0, #HYP_PAGE_OFFSET_HIGH_MASK
	87	* nop
	88	* - Low mask:
	89	* and x0, x0, #HYP_PAGE_OFFSET_HIGH_MASK
	90	* and x0, x0, #HYP_PAGE_OFFSET_LOW_MASK
	91	* - VHE:
	92	* nop
	93	* nop
	94	*
	95	* The "low mask" version works because the mask is a strict subset of
	96	* the "high mask", hence performing the first mask for nothing.
	97	* Should be completely invisible on any viable CPU.
37c43753 MZ	98	*/
37c43753 MZ	99	.macro kern_hyp_va reg
fd81e6bf MZ	100	alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
fd81e6bf MZ	101	and \reg, \reg, #HYP_PAGE_OFFSET_HIGH_MASK
e506236a MR	102	alternative_else_nop_endif
e506236a MR	103	alternative_if ARM64_HYP_OFFSET_LOW
fd81e6bf	104	and \reg, \reg, #HYP_PAGE_OFFSET_LOW_MASK
e506236a	105	alternative_else_nop_endif
37c43753 MZ	106	.endm
	107
	108	#else
	109
38f791a4	110	#include <asm/pgalloc.h>
02f7760e	111	#include <asm/cache.h>
37c43753	112	#include <asm/cacheflush.h>
e4c5a685 AB	113	#include <asm/mmu_context.h>
e4c5a685 AB	114	#include <asm/pgtable.h>
37c43753	115
fd81e6bf MZ	116	static inline unsigned long __kern_hyp_va(unsigned long v)
	117	{
	118	asm volatile(ALTERNATIVE("and %0, %0, %1",
	119	"nop",
	120	ARM64_HAS_VIRT_HOST_EXTN)
	121	: "+r" (v)
	122	: "i" (HYP_PAGE_OFFSET_HIGH_MASK));
	123	asm volatile(ALTERNATIVE("nop",
	124	"and %0, %0, %1",
	125	ARM64_HYP_OFFSET_LOW)
	126	: "+r" (v)
	127	: "i" (HYP_PAGE_OFFSET_LOW_MASK));
	128	return v;
	129	}
	130
94d0e598	131	#define kern_hyp_va(v) ((typeof(v))(__kern_hyp_va((unsigned long)(v))))
37c43753	132
c7588eb4 MZ	133	/*
	134	* Obtain the PC-relative address of a kernel symbol
	135	* s: symbol
	136	*
	137	* The goal of this macro is to return a symbol's address based on a
	138	* PC-relative computation, as opposed to a loading the VA from a
	139	* constant pool or something similar. This works well for HYP, as an
	140	* absolute VA is guaranteed to be wrong. Only use this if trying to
	141	* obtain the address of a symbol (i.e. not something you obtained by
	142	* following a pointer).
	143	*/
	144	#define hyp_symbol_addr(s) \
	145	({ \
	146	typeof(s) *addr; \
	147	asm("adrp %0, %1\n" \
	148	"add %0, %0, :lo12:%1\n" \
	149	: "=r" (addr) : "S" (&s)); \
	150	addr; \
	151	})
	152
37c43753	153	/*
dbff124e	154	* We currently only support a 40bit IPA.
37c43753	155	*/
dbff124e	156	#define KVM_PHYS_SHIFT (40)
37c43753 MZ	157	#define KVM_PHYS_SIZE (1UL << KVM_PHYS_SHIFT)
	158	#define KVM_PHYS_MASK (KVM_PHYS_SIZE - 1UL)
	159
c0ef6326 SP	160	#include <asm/stage2_pgtable.h>
c0ef6326 SP	161
c8dddecd	162	int create_hyp_mappings(void from, void to, pgprot_t prot);
37c43753	163	int create_hyp_io_mappings(void from, void to, phys_addr_t);
37c43753 MZ	164	void free_hyp_pgds(void);
37c43753 MZ	165
957db105	166	void stage2_unmap_vm(struct kvm *kvm);
37c43753 MZ	167	int kvm_alloc_stage2_pgd(struct kvm *kvm);
	168	void kvm_free_stage2_pgd(struct kvm *kvm);
	169	int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
c40f2f8f	170	phys_addr_t pa, unsigned long size, bool writable);
37c43753 MZ	171
	172	int kvm_handle_guest_abort(struct kvm_vcpu vcpu, struct kvm_run run);
	173
	174	void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu);
	175
	176	phys_addr_t kvm_mmu_get_httbr(void);
37c43753 MZ	177	phys_addr_t kvm_get_idmap_vector(void);
	178	int kvm_mmu_init(void);
	179	void kvm_clear_hyp_idmap(void);
	180
	181	#define kvm_set_pte(ptep, pte) set_pte(ptep, pte)
ad361f09	182	#define kvm_set_pmd(pmdp, pmd) set_pmd(pmdp, pmd)
37c43753	183
06485053	184	static inline pte_t kvm_s2pte_mkwrite(pte_t pte)
37c43753	185	{
06485053 CM	186	pte_val(pte) \|= PTE_S2_RDWR;
06485053 CM	187	return pte;
37c43753 MZ	188	}
37c43753 MZ	189
06485053	190	static inline pmd_t kvm_s2pmd_mkwrite(pmd_t pmd)
ad361f09	191	{
06485053 CM	192	pmd_val(pmd) \|= PMD_S2_RDWR;
06485053 CM	193	return pmd;
ad361f09 CD	194	}
ad361f09 CD	195
8199ed0e MS	196	static inline void kvm_set_s2pte_readonly(pte_t *pte)
8199ed0e MS	197	{
0966253d CM	198	pteval_t old_pteval, pteval;
	199
	200	pteval = READ_ONCE(pte_val(*pte));
	201	do {
	202	old_pteval = pteval;
	203	pteval &= ~PTE_S2_RDWR;
	204	pteval \|= PTE_S2_RDONLY;
	205	pteval = cmpxchg_relaxed(&pte_val(*pte), old_pteval, pteval);
	206	} while (pteval != old_pteval);
8199ed0e MS	207	}
	208
	209	static inline bool kvm_s2pte_readonly(pte_t *pte)
	210	{
	211	return (pte_val(*pte) & PTE_S2_RDWR) == PTE_S2_RDONLY;
	212	}
	213
	214	static inline void kvm_set_s2pmd_readonly(pmd_t *pmd)
	215	{
06485053	216	kvm_set_s2pte_readonly((pte_t *)pmd);
8199ed0e MS	217	}
	218
	219	static inline bool kvm_s2pmd_readonly(pmd_t *pmd)
	220	{
06485053	221	return kvm_s2pte_readonly((pte_t *)pmd);
38f791a4 CD	222	}
38f791a4 CD	223
4f853a71 CD	224	static inline bool kvm_page_empty(void *ptr)
	225	{
	226	struct page *ptr_page = virt_to_page(ptr);
	227	return page_count(ptr_page) == 1;
	228	}
	229
66f877fa	230	#define hyp_pte_table_empty(ptep) kvm_page_empty(ptep)
38f791a4 CD	231
38f791a4 CD	232	#ifdef __PAGETABLE_PMD_FOLDED
66f877fa	233	#define hyp_pmd_table_empty(pmdp) (0)
38f791a4	234	#else
66f877fa	235	#define hyp_pmd_table_empty(pmdp) kvm_page_empty(pmdp)
38f791a4 CD	236	#endif
	237
	238	#ifdef __PAGETABLE_PUD_FOLDED
66f877fa	239	#define hyp_pud_table_empty(pudp) (0)
4f853a71	240	#else
66f877fa	241	#define hyp_pud_table_empty(pudp) kvm_page_empty(pudp)
4f853a71	242	#endif
4f853a71	243
37c43753 MZ	244	struct kvm;
37c43753 MZ	245
2d58b733 MZ	246	#define kvm_flush_dcache_to_poc(a,l) __flush_dcache_area((a), (l))
	247
	248	static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu)
37c43753	249	{
2d58b733 MZ	250	return (vcpu_sys_reg(vcpu, SCTLR_EL1) & 0b101) == 0b101;
	251	}
	252
ba049e93 DW	253	static inline void __coherent_cache_guest_page(struct kvm_vcpu *vcpu,
ba049e93 DW	254	kvm_pfn_t pfn,
13b7756c	255	unsigned long size)
2d58b733	256	{
0d3e4d4f MZ	257	void *va = page_address(pfn_to_page(pfn));
0d3e4d4f MZ	258
8f36ebaf	259	kvm_flush_dcache_to_poc(va, size);
2d58b733	260
87da236e	261	if (icache_is_aliasing()) {
37c43753 MZ	262	/* any kind of VIPT cache */
37c43753 MZ	263	__flush_icache_all();
87da236e WD	264	} else if (is_kernel_in_hyp_mode() \|\| !icache_is_vpipt()) {
	265	/* PIPT or VPIPT at EL2 (see comment in __kvm_tlb_flush_vmid_ipa) */
	266	flush_icache_range((unsigned long)va,
	267	(unsigned long)va + size);
37c43753 MZ	268	}
	269	}
	270
363ef89f MZ	271	static inline void __kvm_flush_dcache_pte(pte_t pte)
	272	{
	273	struct page *page = pte_page(pte);
	274	kvm_flush_dcache_to_poc(page_address(page), PAGE_SIZE);
	275	}
	276
	277	static inline void __kvm_flush_dcache_pmd(pmd_t pmd)
	278	{
	279	struct page *page = pmd_page(pmd);
	280	kvm_flush_dcache_to_poc(page_address(page), PMD_SIZE);
	281	}
	282
	283	static inline void __kvm_flush_dcache_pud(pud_t pud)
	284	{
	285	struct page *page = pud_page(pud);
	286	kvm_flush_dcache_to_poc(page_address(page), PUD_SIZE);
	287	}
	288
2077be67	289	#define kvm_virt_to_phys(x) __pa_symbol(x)
37c43753	290
3c1e7165 MZ	291	void kvm_set_way_flush(struct kvm_vcpu *vcpu);
3c1e7165 MZ	292	void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled);
9d218a1f	293
e4c5a685 AB	294	static inline bool __kvm_cpu_uses_extended_idmap(void)
	295	{
	296	return __cpu_uses_extended_idmap();
	297	}
	298
	299	static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd,
	300	pgd_t *hyp_pgd,
	301	pgd_t *merged_hyp_pgd,
	302	unsigned long hyp_idmap_start)
	303	{
	304	int idmap_idx;
	305
	306	/*
	307	* Use the first entry to access the HYP mappings. It is
	308	* guaranteed to be free, otherwise we wouldn't use an
	309	* extended idmap.
	310	*/
	311	VM_BUG_ON(pgd_val(merged_hyp_pgd[0]));
	312	merged_hyp_pgd[0] = __pgd(__pa(hyp_pgd) \| PMD_TYPE_TABLE);
	313
	314	/*
	315	* Create another extended level entry that points to the boot HYP map,
	316	* which contains an ID mapping of the HYP init code. We essentially
	317	* merge the boot and runtime HYP maps by doing so, but they don't
	318	* overlap anyway, so this is fine.
	319	*/
	320	idmap_idx = hyp_idmap_start >> VA_BITS;
	321	VM_BUG_ON(pgd_val(merged_hyp_pgd[idmap_idx]));
	322	merged_hyp_pgd[idmap_idx] = __pgd(__pa(boot_hyp_pgd) \| PMD_TYPE_TABLE);
	323	}
	324
20475f78 VM	325	static inline unsigned int kvm_get_vmid_bits(void)
20475f78 VM	326	{
46823dd1	327	int reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
20475f78	328
28c5dcb2	329	return (cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR1_VMIDBITS_SHIFT) == 2) ? 16 : 8;
20475f78 VM	330	}
20475f78 VM	331
b9c628f6 AP	332	/*
	333	* We are not in the kvm->srcu critical section most of the time, so we take
	334	* the SRCU read lock here. Since we copy the data from the user page, we
	335	* can immediately drop the lock again.
	336	*/
	337	static inline int kvm_read_guest_lock(struct kvm *kvm,
	338	gpa_t gpa, void *data, unsigned long len)
	339	{
	340	int srcu_idx = srcu_read_lock(&kvm->srcu);
	341	int ret = kvm_read_guest(kvm, gpa, data, len);
	342
	343	srcu_read_unlock(&kvm->srcu, srcu_idx);
	344
	345	return ret;
	346	}
	347
3f401957 MZ	348	static inline int kvm_write_guest_lock(struct kvm *kvm, gpa_t gpa,
	349	const void *data, unsigned long len)
	350	{
	351	int srcu_idx = srcu_read_lock(&kvm->srcu);
	352	int ret = kvm_write_guest(kvm, gpa, data, len);
	353
	354	srcu_read_unlock(&kvm->srcu, srcu_idx);
	355
	356	return ret;
	357	}
	358
bc3eb3e9 MZ	359	#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
	360	#include <asm/mmu.h>
	361
	362	static inline void *kvm_get_hyp_vector(void)
	363	{
	364	struct bp_hardening_data *data = arm64_get_bp_hardening_data();
	365	void *vect = kvm_ksym_ref(__kvm_hyp_vector);
	366
	367	if (data->fn) {
	368	vect = __bp_harden_hyp_vecs_start +
	369	data->hyp_vectors_slot * SZ_2K;
	370
	371	if (!has_vhe())
	372	vect = lm_alias(vect);
	373	}
	374
	375	return vect;
	376	}
	377
	378	static inline int kvm_map_vectors(void)
	379	{
	380	return create_hyp_mappings(kvm_ksym_ref(__bp_harden_hyp_vecs_start),
	381	kvm_ksym_ref(__bp_harden_hyp_vecs_end),
	382	PAGE_HYP_EXEC);
	383	}
	384
	385	#else
	386	static inline void *kvm_get_hyp_vector(void)
	387	{
	388	return kvm_ksym_ref(__kvm_hyp_vector);
	389	}
	390
	391	static inline int kvm_map_vectors(void)
	392	{
	393	return 0;
	394	}
	395	#endif
	396
9ae6a587 MZ	397	#ifdef CONFIG_ARM64_SSBD
	398	DECLARE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required);
	399
	400	static inline int hyp_map_aux_data(void)
	401	{
	402	int cpu, err;
	403
	404	for_each_possible_cpu(cpu) {
	405	u64 *ptr;
	406
	407	ptr = per_cpu_ptr(&arm64_ssbd_callback_required, cpu);
	408	err = create_hyp_mappings(ptr, ptr + 1, PAGE_HYP);
	409	if (err)
	410	return err;
	411	}
	412	return 0;
	413	}
	414	#else
	415	static inline int hyp_map_aux_data(void)
	416	{
	417	return 0;
	418	}
	419	#endif
	420
37c43753 MZ	421	#endif /* __ASSEMBLY__ */
37c43753 MZ	422	#endif /* __ARM64_KVM_MMU_H__ */