[mirror_ubuntu-kernels.git] / arch / arm64 / include / asm / kvm_hyp.h

/*
 * Copyright (C) 2015 - ARM Ltd
 * Author: Marc Zyngier <marc.zyngier@arm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef __ARM64_KVM_HYP_H__
#define __ARM64_KVM_HYP_H__

#include <linux/compiler.h>
#include <linux/kvm_host.h>
#include <asm/alternative.h>
#include <asm/kvm_mmu.h>
#include <asm/sysreg.h>

#define __hyp_text __section(.hyp.text) notrace

#define read_sysreg_elx(r,nvh,vh)					\
	({								\
		u64 reg;						\
		asm volatile(ALTERNATIVE("mrs %0, " __stringify(r##nvh),\
					 "mrs_s %0, " __stringify(r##vh),\
					 ARM64_HAS_VIRT_HOST_EXTN)	\
			     : "=r" (reg));				\
		reg;							\
	})

#define write_sysreg_elx(v,r,nvh,vh)					\
	do {								\
		u64 __val = (u64)(v);					\
		asm volatile(ALTERNATIVE("msr " __stringify(r##nvh) ", %x0",\
					 "msr_s " __stringify(r##vh) ", %x0",\
					 ARM64_HAS_VIRT_HOST_EXTN)	\
					 : : "rZ" (__val));		\
	} while (0)

/*
 * Unified accessors for registers that have a different encoding
 * between VHE and non-VHE. They must be specified without their "ELx"
 * encoding.
 */
#define read_sysreg_el2(r)						\
	({								\
		u64 reg;						\
		asm volatile(ALTERNATIVE("mrs %0, " __stringify(r##_EL2),\
					 "mrs %0, " __stringify(r##_EL1),\
					 ARM64_HAS_VIRT_HOST_EXTN)	\
			     : "=r" (reg));				\
		reg;							\
	})

#define write_sysreg_el2(v,r)						\
	do {								\
		u64 __val = (u64)(v);					\
		asm volatile(ALTERNATIVE("msr " __stringify(r##_EL2) ", %x0",\
					 "msr " __stringify(r##_EL1) ", %x0",\
					 ARM64_HAS_VIRT_HOST_EXTN)	\
					 : : "rZ" (__val));		\
	} while (0)

#define read_sysreg_el0(r)	read_sysreg_elx(r, _EL0, _EL02)
#define write_sysreg_el0(v,r)	write_sysreg_elx(v, r, _EL0, _EL02)
#define read_sysreg_el1(r)	read_sysreg_elx(r, _EL1, _EL12)
#define write_sysreg_el1(v,r)	write_sysreg_elx(v, r, _EL1, _EL12)

/* The VHE specific system registers and their encoding */
#define sctlr_EL12              sys_reg(3, 5, 1, 0, 0)
#define cpacr_EL12              sys_reg(3, 5, 1, 0, 2)
#define ttbr0_EL12              sys_reg(3, 5, 2, 0, 0)
#define ttbr1_EL12              sys_reg(3, 5, 2, 0, 1)
#define tcr_EL12                sys_reg(3, 5, 2, 0, 2)
#define afsr0_EL12              sys_reg(3, 5, 5, 1, 0)
#define afsr1_EL12              sys_reg(3, 5, 5, 1, 1)
#define esr_EL12                sys_reg(3, 5, 5, 2, 0)
#define far_EL12                sys_reg(3, 5, 6, 0, 0)
#define mair_EL12               sys_reg(3, 5, 10, 2, 0)
#define amair_EL12              sys_reg(3, 5, 10, 3, 0)
#define vbar_EL12               sys_reg(3, 5, 12, 0, 0)
#define contextidr_EL12         sys_reg(3, 5, 13, 0, 1)
#define cntkctl_EL12            sys_reg(3, 5, 14, 1, 0)
#define cntp_tval_EL02          sys_reg(3, 5, 14, 2, 0)
#define cntp_ctl_EL02           sys_reg(3, 5, 14, 2, 1)
#define cntp_cval_EL02          sys_reg(3, 5, 14, 2, 2)
#define cntv_tval_EL02          sys_reg(3, 5, 14, 3, 0)
#define cntv_ctl_EL02           sys_reg(3, 5, 14, 3, 1)
#define cntv_cval_EL02          sys_reg(3, 5, 14, 3, 2)
#define spsr_EL12               sys_reg(3, 5, 4, 0, 0)
#define elr_EL12                sys_reg(3, 5, 4, 0, 1)

/**
 * hyp_alternate_select - Generates patchable code sequences that are
 * used to switch between two implementations of a function, depending
 * on the availability of a feature.
 *
 * @fname: a symbol name that will be defined as a function returning a
 * function pointer whose type will match @orig and @alt
 * @orig: A pointer to the default function, as returned by @fname when
 * @cond doesn't hold
 * @alt: A pointer to the alternate function, as returned by @fname
 * when @cond holds
 * @cond: a CPU feature (as described in asm/cpufeature.h)
 */
#define hyp_alternate_select(fname, orig, alt, cond)			\
typeof(orig) * __hyp_text fname(void)					\
{									\
	typeof(alt) *val = orig;					\
	asm volatile(ALTERNATIVE("nop		\n",			\
				 "mov	%0, %1	\n",			\
				 cond)					\
		     : "+r" (val) : "r" (alt));				\
	return val;							\
}

int __vgic_v2_perform_cpuif_access(struct kvm_vcpu *vcpu);

void __vgic_v3_save_state(struct kvm_vcpu *vcpu);
void __vgic_v3_restore_state(struct kvm_vcpu *vcpu);
void __vgic_v3_activate_traps(struct kvm_vcpu *vcpu);
void __vgic_v3_deactivate_traps(struct kvm_vcpu *vcpu);
void __vgic_v3_save_aprs(struct kvm_vcpu *vcpu);
void __vgic_v3_restore_aprs(struct kvm_vcpu *vcpu);
int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu);

void __timer_enable_traps(struct kvm_vcpu *vcpu);
void __timer_disable_traps(struct kvm_vcpu *vcpu);

void __sysreg_save_state_nvhe(struct kvm_cpu_context *ctxt);
void __sysreg_restore_state_nvhe(struct kvm_cpu_context *ctxt);
void sysreg_save_host_state_vhe(struct kvm_cpu_context *ctxt);
void sysreg_restore_host_state_vhe(struct kvm_cpu_context *ctxt);
void sysreg_save_guest_state_vhe(struct kvm_cpu_context *ctxt);
void sysreg_restore_guest_state_vhe(struct kvm_cpu_context *ctxt);
void __sysreg32_save_state(struct kvm_vcpu *vcpu);
void __sysreg32_restore_state(struct kvm_vcpu *vcpu);

void __debug_switch_to_guest(struct kvm_vcpu *vcpu);
void __debug_switch_to_host(struct kvm_vcpu *vcpu);

void __fpsimd_save_state(struct user_fpsimd_state *fp_regs);
void __fpsimd_restore_state(struct user_fpsimd_state *fp_regs);
bool __fpsimd_enabled(void);

void activate_traps_vhe_load(struct kvm_vcpu *vcpu);
void deactivate_traps_vhe_put(void);

u64 __guest_enter(struct kvm_vcpu *vcpu, struct kvm_cpu_context *host_ctxt);
void __noreturn __hyp_do_panic(unsigned long, ...);

/*
 * Must be called from hyp code running at EL2 with an updated VTTBR
 * and interrupts disabled.
 */
static __always_inline void __hyp_text __load_guest_stage2(struct kvm *kvm)
{
	write_sysreg(kvm->arch.vtcr, vtcr_el2);
	write_sysreg(kvm_get_vttbr(kvm), vttbr_el2);

	/*
	 * ARM erratum 1165522 requires the actual execution of the above
	 * before we can switch to the EL1/EL0 translation regime used by
	 * the guest.
	 */
	asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_1165522));
}

#endif /* __ARM64_KVM_HYP_H__ */
Commit	Line	Data
c76a0a66 MZ	1	/*
	2	* Copyright (C) 2015 - ARM Ltd
	3	* Author: Marc Zyngier <marc.zyngier@arm.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License version 2 as
	7	* published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
	18	#ifndef __ARM64_KVM_HYP_H__
	19	#define __ARM64_KVM_HYP_H__
	20
	21	#include <linux/compiler.h>
	22	#include <linux/kvm_host.h>
1e4448c5	23	#include <asm/alternative.h>
e329fb75	24	#include <asm/kvm_mmu.h>
c76a0a66 MZ	25	#include <asm/sysreg.h>
	26
	27	#define __hyp_text __section(.hyp.text) notrace
	28
915ccd1d MZ	29	#define read_sysreg_elx(r,nvh,vh) \
	30	({ \
	31	u64 reg; \
	32	asm volatile(ALTERNATIVE("mrs %0, " __stringify(r##nvh),\
	33	"mrs_s %0, " __stringify(r##vh),\
	34	ARM64_HAS_VIRT_HOST_EXTN) \
	35	: "=r" (reg)); \
	36	reg; \
	37	})
	38
	39	#define write_sysreg_elx(v,r,nvh,vh) \
	40	do { \
	41	u64 __val = (u64)(v); \
	42	asm volatile(ALTERNATIVE("msr " __stringify(r##nvh) ", %x0",\
	43	"msr_s " __stringify(r##vh) ", %x0",\
	44	ARM64_HAS_VIRT_HOST_EXTN) \
	45	: : "rZ" (__val)); \
	46	} while (0)
	47
	48	/*
	49	* Unified accessors for registers that have a different encoding
	50	* between VHE and non-VHE. They must be specified without their "ELx"
	51	* encoding.
	52	*/
	53	#define read_sysreg_el2(r) \
	54	({ \
	55	u64 reg; \
	56	asm volatile(ALTERNATIVE("mrs %0, " __stringify(r##_EL2),\
	57	"mrs %0, " __stringify(r##_EL1),\
	58	ARM64_HAS_VIRT_HOST_EXTN) \
	59	: "=r" (reg)); \
	60	reg; \
	61	})
	62
	63	#define write_sysreg_el2(v,r) \
	64	do { \
	65	u64 __val = (u64)(v); \
	66	asm volatile(ALTERNATIVE("msr " __stringify(r##_EL2) ", %x0",\
	67	"msr " __stringify(r##_EL1) ", %x0",\
	68	ARM64_HAS_VIRT_HOST_EXTN) \
	69	: : "rZ" (__val)); \
	70	} while (0)
	71
	72	#define read_sysreg_el0(r) read_sysreg_elx(r, _EL0, _EL02)
	73	#define write_sysreg_el0(v,r) write_sysreg_elx(v, r, _EL0, _EL02)
	74	#define read_sysreg_el1(r) read_sysreg_elx(r, _EL1, _EL12)
	75	#define write_sysreg_el1(v,r) write_sysreg_elx(v, r, _EL1, _EL12)
	76
	77	/* The VHE specific system registers and their encoding */
	78	#define sctlr_EL12 sys_reg(3, 5, 1, 0, 0)
	79	#define cpacr_EL12 sys_reg(3, 5, 1, 0, 2)
	80	#define ttbr0_EL12 sys_reg(3, 5, 2, 0, 0)
	81	#define ttbr1_EL12 sys_reg(3, 5, 2, 0, 1)
	82	#define tcr_EL12 sys_reg(3, 5, 2, 0, 2)
	83	#define afsr0_EL12 sys_reg(3, 5, 5, 1, 0)
	84	#define afsr1_EL12 sys_reg(3, 5, 5, 1, 1)
	85	#define esr_EL12 sys_reg(3, 5, 5, 2, 0)
	86	#define far_EL12 sys_reg(3, 5, 6, 0, 0)
	87	#define mair_EL12 sys_reg(3, 5, 10, 2, 0)
	88	#define amair_EL12 sys_reg(3, 5, 10, 3, 0)
	89	#define vbar_EL12 sys_reg(3, 5, 12, 0, 0)
	90	#define contextidr_EL12 sys_reg(3, 5, 13, 0, 1)
	91	#define cntkctl_EL12 sys_reg(3, 5, 14, 1, 0)
	92	#define cntp_tval_EL02 sys_reg(3, 5, 14, 2, 0)
93	#define cntp_ctl_EL02 sys_reg(3, 5, 14, 2, 1)
94	#define cntp_cval_EL02 sys_reg(3, 5, 14, 2, 2)
95	#define cntv_tval_EL02 sys_reg(3, 5, 14, 3, 0)
96	#define cntv_ctl_EL02 sys_reg(3, 5, 14, 3, 1)
97	#define cntv_cval_EL02 sys_reg(3, 5, 14, 3, 2)
98	#define spsr_EL12 sys_reg(3, 5, 4, 0, 0)
99	#define elr_EL12 sys_reg(3, 5, 4, 0, 1)
100
c1bf6e18 MZ	101	/**
	102	* hyp_alternate_select - Generates patchable code sequences that are
	103	* used to switch between two implementations of a function, depending
	104	* on the availability of a feature.
	105	*
	106	* @fname: a symbol name that will be defined as a function returning a
	107	* function pointer whose type will match @orig and @alt
	108	* @orig: A pointer to the default function, as returned by @fname when
	109	* @cond doesn't hold
	110	* @alt: A pointer to the alternate function, as returned by @fname
	111	* when @cond holds
	112	* @cond: a CPU feature (as described in asm/cpufeature.h)
	113	*/
	114	#define hyp_alternate_select(fname, orig, alt, cond) \
	115	typeof(orig) * __hyp_text fname(void) \
	116	{ \
	117	typeof(alt) *val = orig; \
	118	asm volatile(ALTERNATIVE("nop \n", \
	119	"mov %0, %1 \n", \
	120	cond) \
	121	: "+r" (val) : "r" (alt)); \
	122	return val; \
	123	}
	124
3272f0d0	125	int __vgic_v2_perform_cpuif_access(struct kvm_vcpu *vcpu);
06282fd2	126
f68d2b1b MZ	127	void __vgic_v3_save_state(struct kvm_vcpu *vcpu);
f68d2b1b MZ	128	void __vgic_v3_restore_state(struct kvm_vcpu *vcpu);
2d0e63e0 CD	129	void __vgic_v3_activate_traps(struct kvm_vcpu *vcpu);
2d0e63e0 CD	130	void __vgic_v3_deactivate_traps(struct kvm_vcpu *vcpu);
923a2e30 CD	131	void __vgic_v3_save_aprs(struct kvm_vcpu *vcpu);
923a2e30 CD	132	void __vgic_v3_restore_aprs(struct kvm_vcpu *vcpu);
59da1cbf	133	int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu);
f68d2b1b	134
688c50aa CD	135	void __timer_enable_traps(struct kvm_vcpu *vcpu);
688c50aa CD	136	void __timer_disable_traps(struct kvm_vcpu *vcpu);
1431af36	137
4cdecaba CD	138	void __sysreg_save_state_nvhe(struct kvm_cpu_context *ctxt);
4cdecaba CD	139	void __sysreg_restore_state_nvhe(struct kvm_cpu_context *ctxt);
f837453d CD	140	void sysreg_save_host_state_vhe(struct kvm_cpu_context *ctxt);
	141	void sysreg_restore_host_state_vhe(struct kvm_cpu_context *ctxt);
	142	void sysreg_save_guest_state_vhe(struct kvm_cpu_context *ctxt);
	143	void sysreg_restore_guest_state_vhe(struct kvm_cpu_context *ctxt);
c209ec85 MZ	144	void __sysreg32_save_state(struct kvm_vcpu *vcpu);
c209ec85 MZ	145	void __sysreg32_restore_state(struct kvm_vcpu *vcpu);
6d6ec20f	146
014c4c77 CD	147	void __debug_switch_to_guest(struct kvm_vcpu *vcpu);
014c4c77 CD	148	void __debug_switch_to_host(struct kvm_vcpu *vcpu);
8eb99267	149
c13d1683 MZ	150	void __fpsimd_save_state(struct user_fpsimd_state *fp_regs);
c13d1683 MZ	151	void __fpsimd_restore_state(struct user_fpsimd_state *fp_regs);
32876224	152	bool __fpsimd_enabled(void);
c13d1683	153
a2465629 CD	154	void activate_traps_vhe_load(struct kvm_vcpu *vcpu);
	155	void deactivate_traps_vhe_put(void);
	156
b97b66c1	157	u64 __guest_enter(struct kvm_vcpu vcpu, struct kvm_cpu_context host_ctxt);
53fd5b64	158	void __noreturn __hyp_do_panic(unsigned long, ...);
b97b66c1	159
9f98ddd6 SP	160	/*
	161	* Must be called from hyp code running at EL2 with an updated VTTBR
	162	* and interrupts disabled.
	163	*/
	164	static __always_inline void __hyp_text __load_guest_stage2(struct kvm *kvm)
	165	{
7665f3a8	166	write_sysreg(kvm->arch.vtcr, vtcr_el2);
e329fb75	167	write_sysreg(kvm_get_vttbr(kvm), vttbr_el2);
1e4448c5 MZ	168
	169	/*
	170	* ARM erratum 1165522 requires the actual execution of the above
	171	* before we can switch to the EL1/EL0 translation regime used by
	172	* the guest.
	173	*/
	174	asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_1165522));
9f98ddd6 SP	175	}
9f98ddd6 SP	176
c76a0a66 MZ	177	#endif /* __ARM64_KVM_HYP_H__ */
c76a0a66 MZ	178