[mirror_ubuntu-artful-kernel.git] / arch / arm64 / kvm / hyp / switch.c

/*
 * Copyright (C) 2015 - ARM Ltd
 * Author: Marc Zyngier <marc.zyngier@arm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/types.h>
#include <linux/jump_label.h>
#include <uapi/linux/psci.h>

#include <kvm/arm_psci.h>

#include <asm/kvm_asm.h>
#include <asm/kvm_emulate.h>
#include <asm/kvm_hyp.h>
#include <asm/fpsimd.h>

static bool __hyp_text __fpsimd_enabled_nvhe(void)
{
	return !(read_sysreg(cptr_el2) & CPTR_EL2_TFP);
}

static bool __hyp_text __fpsimd_enabled_vhe(void)
{
	return !!(read_sysreg(cpacr_el1) & CPACR_EL1_FPEN);
}

static hyp_alternate_select(__fpsimd_is_enabled,
			    __fpsimd_enabled_nvhe, __fpsimd_enabled_vhe,
			    ARM64_HAS_VIRT_HOST_EXTN);

bool __hyp_text __fpsimd_enabled(void)
{
	return __fpsimd_is_enabled()();
}

static void __hyp_text __activate_traps_vhe(void)
{
	u64 val;

	val = read_sysreg(cpacr_el1);
	val |= CPACR_EL1_TTA;
	val &= ~CPACR_EL1_FPEN;
	write_sysreg(val, cpacr_el1);

	write_sysreg(kvm_get_hyp_vector(), vbar_el1);
}

static void __hyp_text __activate_traps_nvhe(void)
{
	u64 val;

	val = CPTR_EL2_DEFAULT;
	val |= CPTR_EL2_TTA | CPTR_EL2_TFP;
	write_sysreg(val, cptr_el2);
}

static hyp_alternate_select(__activate_traps_arch,
			    __activate_traps_nvhe, __activate_traps_vhe,
			    ARM64_HAS_VIRT_HOST_EXTN);

static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)
{
	u64 val;

	/*
	 * We are about to set CPTR_EL2.TFP to trap all floating point
	 * register accesses to EL2, however, the ARM ARM clearly states that
	 * traps are only taken to EL2 if the operation would not otherwise
	 * trap to EL1.  Therefore, always make sure that for 32-bit guests,
	 * we set FPEXC.EN to prevent traps to EL1, when setting the TFP bit.
	 * If FP/ASIMD is not implemented, FPEXC is UNDEFINED and any access to
	 * it will cause an exception.
	 */
	val = vcpu->arch.hcr_el2;
	if (!(val & HCR_RW) && system_supports_fpsimd()) {
		write_sysreg(1 << 30, fpexc32_el2);
		isb();
	}
	write_sysreg(val, hcr_el2);
	/* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */
	write_sysreg(1 << 15, hstr_el2);
	/*
	 * Make sure we trap PMU access from EL0 to EL2. Also sanitize
	 * PMSELR_EL0 to make sure it never contains the cycle
	 * counter, which could make a PMXEVCNTR_EL0 access UNDEF at
	 * EL1 instead of being trapped to EL2.
	 */
	write_sysreg(0, pmselr_el0);
	write_sysreg(ARMV8_PMU_USERENR_MASK, pmuserenr_el0);
	write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2);
	__activate_traps_arch()();
}

static void __hyp_text __deactivate_traps_vhe(void)
{
	extern char vectors[];	/* kernel exception vectors */
	u64 mdcr_el2 = read_sysreg(mdcr_el2);

	mdcr_el2 &= MDCR_EL2_HPMN_MASK |
		    MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT |
		    MDCR_EL2_TPMS;

	write_sysreg(mdcr_el2, mdcr_el2);
	write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
	write_sysreg(CPACR_EL1_FPEN, cpacr_el1);
	write_sysreg(vectors, vbar_el1);
}

static void __hyp_text __deactivate_traps_nvhe(void)
{
	u64 mdcr_el2 = read_sysreg(mdcr_el2);

	mdcr_el2 &= MDCR_EL2_HPMN_MASK;
	mdcr_el2 |= MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT;

	write_sysreg(mdcr_el2, mdcr_el2);
	write_sysreg(HCR_RW, hcr_el2);
	write_sysreg(CPTR_EL2_DEFAULT, cptr_el2);
}

static hyp_alternate_select(__deactivate_traps_arch,
			    __deactivate_traps_nvhe, __deactivate_traps_vhe,
			    ARM64_HAS_VIRT_HOST_EXTN);

static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
{
	/*
	 * If we pended a virtual abort, preserve it until it gets
	 * cleared. See D1.14.3 (Virtual Interrupts) for details, but
	 * the crucial bit is "On taking a vSError interrupt,
	 * HCR_EL2.VSE is cleared to 0."
	 */
	if (vcpu->arch.hcr_el2 & HCR_VSE)
		vcpu->arch.hcr_el2 = read_sysreg(hcr_el2);

	__deactivate_traps_arch()();
	write_sysreg(0, hstr_el2);
	write_sysreg(0, pmuserenr_el0);
}

static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu)
{
	struct kvm *kvm = kern_hyp_va(vcpu->kvm);
	write_sysreg(kvm->arch.vttbr, vttbr_el2);
}

static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu)
{
	write_sysreg(0, vttbr_el2);
}

static void __hyp_text __vgic_save_state(struct kvm_vcpu *vcpu)
{
	if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
		__vgic_v3_save_state(vcpu);
	else
		__vgic_v2_save_state(vcpu);

	write_sysreg(read_sysreg(hcr_el2) & ~HCR_INT_OVERRIDE, hcr_el2);
}

static void __hyp_text __vgic_restore_state(struct kvm_vcpu *vcpu)
{
	u64 val;

	val = read_sysreg(hcr_el2);
	val |= 	HCR_INT_OVERRIDE;
	val |= vcpu->arch.irq_lines;
	write_sysreg(val, hcr_el2);

	if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
		__vgic_v3_restore_state(vcpu);
	else
		__vgic_v2_restore_state(vcpu);
}

static bool __hyp_text __true_value(void)
{
	return true;
}

static bool __hyp_text __false_value(void)
{
	return false;
}

static hyp_alternate_select(__check_arm_834220,
			    __false_value, __true_value,
			    ARM64_WORKAROUND_834220);

static bool __hyp_text __translate_far_to_hpfar(u64 far, u64 *hpfar)
{
	u64 par, tmp;

	/*
	 * Resolve the IPA the hard way using the guest VA.
	 *
	 * Stage-1 translation already validated the memory access
	 * rights. As such, we can use the EL1 translation regime, and
	 * don't have to distinguish between EL0 and EL1 access.
	 *
	 * We do need to save/restore PAR_EL1 though, as we haven't
	 * saved the guest context yet, and we may return early...
	 */
	par = read_sysreg(par_el1);
	asm volatile("at s1e1r, %0" : : "r" (far));
	isb();

	tmp = read_sysreg(par_el1);
	write_sysreg(par, par_el1);

	if (unlikely(tmp & 1))
		return false; /* Translation failed, back to guest */

	/* Convert PAR to HPFAR format */
	*hpfar = ((tmp >> 12) & ((1UL << 36) - 1)) << 4;
	return true;
}

static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu)
{
	u64 esr = read_sysreg_el2(esr);
	u8 ec = ESR_ELx_EC(esr);
	u64 hpfar, far;

	vcpu->arch.fault.esr_el2 = esr;

	if (ec != ESR_ELx_EC_DABT_LOW && ec != ESR_ELx_EC_IABT_LOW)
		return true;

	far = read_sysreg_el2(far);

	/*
	 * The HPFAR can be invalid if the stage 2 fault did not
	 * happen during a stage 1 page table walk (the ESR_EL2.S1PTW
	 * bit is clear) and one of the two following cases are true:
	 *   1. The fault was due to a permission fault
	 *   2. The processor carries errata 834220
	 *
	 * Therefore, for all non S1PTW faults where we either have a
	 * permission fault or the errata workaround is enabled, we
	 * resolve the IPA using the AT instruction.
	 */
	if (!(esr & ESR_ELx_S1PTW) &&
	    (__check_arm_834220()() || (esr & ESR_ELx_FSC_TYPE) == FSC_PERM)) {
		if (!__translate_far_to_hpfar(far, &hpfar))
			return false;
	} else {
		hpfar = read_sysreg(hpfar_el2);
	}

	vcpu->arch.fault.far_el2 = far;
	vcpu->arch.fault.hpfar_el2 = hpfar;
	return true;
}

static void __hyp_text __skip_instr(struct kvm_vcpu *vcpu)
{
	*vcpu_pc(vcpu) = read_sysreg_el2(elr);

	if (vcpu_mode_is_32bit(vcpu)) {
		vcpu->arch.ctxt.gp_regs.regs.pstate = read_sysreg_el2(spsr);
		kvm_skip_instr32(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
		write_sysreg_el2(vcpu->arch.ctxt.gp_regs.regs.pstate, spsr);
	} else {
		*vcpu_pc(vcpu) += 4;
	}

	write_sysreg_el2(*vcpu_pc(vcpu), elr);
}

int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
{
	struct kvm_cpu_context *host_ctxt;
	struct kvm_cpu_context *guest_ctxt;
	bool fp_enabled;
	u64 exit_code;

	vcpu = kern_hyp_va(vcpu);
	write_sysreg(vcpu, tpidr_el2);

	host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
	guest_ctxt = &vcpu->arch.ctxt;

	__sysreg_save_host_state(host_ctxt);
	__debug_cond_save_host_state(vcpu);

	__activate_traps(vcpu);
	__activate_vm(vcpu);

	__vgic_restore_state(vcpu);
	__timer_restore_state(vcpu);

	/*
	 * We must restore the 32-bit state before the sysregs, thanks
	 * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
	 */
	__sysreg32_restore_state(vcpu);
	__sysreg_restore_guest_state(guest_ctxt);
	__debug_restore_state(vcpu, kern_hyp_va(vcpu->arch.debug_ptr), guest_ctxt);

	/* Jump in the fire! */
again:
	exit_code = __guest_enter(vcpu, host_ctxt);
	/* And we're baaack! */

	/*
	 * We're using the raw exception code in order to only process
	 * the trap if no SError is pending. We will come back to the
	 * same PC once the SError has been injected, and replay the
	 * trapping instruction.
	 */
	if (exit_code == ARM_EXCEPTION_TRAP && !__populate_fault_info(vcpu))
		goto again;

	if (static_branch_unlikely(&vgic_v2_cpuif_trap) &&
	    exit_code == ARM_EXCEPTION_TRAP) {
		bool valid;

		valid = kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_DABT_LOW &&
			kvm_vcpu_trap_get_fault_type(vcpu) == FSC_FAULT &&
			kvm_vcpu_dabt_isvalid(vcpu) &&
			!kvm_vcpu_dabt_isextabt(vcpu) &&
			!kvm_vcpu_dabt_iss1tw(vcpu);

		if (valid) {
			int ret = __vgic_v2_perform_cpuif_access(vcpu);

			if (ret == 1) {
				__skip_instr(vcpu);
				goto again;
			}

			if (ret == -1) {
				/* Promote an illegal access to an SError */
				__skip_instr(vcpu);
				exit_code = ARM_EXCEPTION_EL1_SERROR;
			}

			/* 0 falls through to be handler out of EL2 */
		}
	}

	if (static_branch_unlikely(&vgic_v3_cpuif_trap) &&
	    exit_code == ARM_EXCEPTION_TRAP &&
	    (kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_SYS64 ||
	     kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_CP15_32)) {
		int ret = __vgic_v3_perform_cpuif_access(vcpu);

		if (ret == 1) {
			__skip_instr(vcpu);
			goto again;
		}

		/* 0 falls through to be handled out of EL2 */
	}

	if (cpus_have_const_cap(ARM64_HARDEN_BP_POST_GUEST_EXIT)) {
		u32 midr = read_cpuid_id();

		/* Apply BTAC predictors mitigation to all Falkor chips */
		if (((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR) ||
		    ((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR_V1)) {
			__qcom_hyp_sanitize_btac_predictors();
		}
	}

	fp_enabled = __fpsimd_enabled();

	__sysreg_save_guest_state(guest_ctxt);
	__sysreg32_save_state(vcpu);
	__timer_save_state(vcpu);
	__vgic_save_state(vcpu);

	__deactivate_traps(vcpu);
	__deactivate_vm(vcpu);

	__sysreg_restore_host_state(host_ctxt);

	if (fp_enabled) {
		__fpsimd_save_state(&guest_ctxt->gp_regs.fp_regs);
		__fpsimd_restore_state(&host_ctxt->gp_regs.fp_regs);
	}

	__debug_save_state(vcpu, kern_hyp_va(vcpu->arch.debug_ptr), guest_ctxt);
	/*
	 * This must come after restoring the host sysregs, since a non-VHE
	 * system may enable SPE here and make use of the TTBRs.
	 */
	__debug_cond_restore_host_state(vcpu);

	return exit_code;
}

static const char __hyp_panic_string[] = "HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n";

static void __hyp_text __hyp_call_panic_nvhe(u64 spsr, u64 elr, u64 par)
{
	unsigned long str_va;

	/*
	 * Force the panic string to be loaded from the literal pool,
	 * making sure it is a kernel address and not a PC-relative
	 * reference.
	 */
	asm volatile("ldr %0, =__hyp_panic_string" : "=r" (str_va));

	__hyp_do_panic(str_va,
		       spsr,  elr,
		       read_sysreg(esr_el2),   read_sysreg_el2(far),
		       read_sysreg(hpfar_el2), par,
		       (void *)read_sysreg(tpidr_el2));
}

static void __hyp_text __hyp_call_panic_vhe(u64 spsr, u64 elr, u64 par)
{
	panic(__hyp_panic_string,
	      spsr,  elr,
	      read_sysreg_el2(esr),   read_sysreg_el2(far),
	      read_sysreg(hpfar_el2), par,
	      (void *)read_sysreg(tpidr_el2));
}

static hyp_alternate_select(__hyp_call_panic,
			    __hyp_call_panic_nvhe, __hyp_call_panic_vhe,
			    ARM64_HAS_VIRT_HOST_EXTN);

void __hyp_text __noreturn __hyp_panic(void)
{
	u64 spsr = read_sysreg_el2(spsr);
	u64 elr = read_sysreg_el2(elr);
	u64 par = read_sysreg(par_el1);

	if (read_sysreg(vttbr_el2)) {
		struct kvm_vcpu *vcpu;
		struct kvm_cpu_context *host_ctxt;

		vcpu = (struct kvm_vcpu *)read_sysreg(tpidr_el2);
		host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
		__timer_save_state(vcpu);
		__deactivate_traps(vcpu);
		__deactivate_vm(vcpu);
		__sysreg_restore_host_state(host_ctxt);
	}

	/* Call panic for real */
	__hyp_call_panic()(spsr, elr, par);

	unreachable();
}
Commit	Line	Data
be901e9b MZ	1	/*
	2	* Copyright (C) 2015 - ARM Ltd
	3	* Author: Marc Zyngier <marc.zyngier@arm.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License version 2 as
	7	* published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
5f05a72a	18	#include <linux/types.h>
5a7a8426	19	#include <linux/jump_label.h>
74cf77a2	20	#include <uapi/linux/psci.h>
5a7a8426	21
b0eeb723 MZ	22	#include <kvm/arm_psci.h>
b0eeb723 MZ	23
68908bf7	24	#include <asm/kvm_asm.h>
fb5ee369	25	#include <asm/kvm_emulate.h>
13720a56	26	#include <asm/kvm_hyp.h>
82e0191a	27	#include <asm/fpsimd.h>
be901e9b	28
32876224 MZ	29	static bool __hyp_text __fpsimd_enabled_nvhe(void)
	30	{
	31	return !(read_sysreg(cptr_el2) & CPTR_EL2_TFP);
	32	}
	33
	34	static bool __hyp_text __fpsimd_enabled_vhe(void)
	35	{
	36	return !!(read_sysreg(cpacr_el1) & CPACR_EL1_FPEN);
	37	}
	38
	39	static hyp_alternate_select(__fpsimd_is_enabled,
	40	__fpsimd_enabled_nvhe, __fpsimd_enabled_vhe,
	41	ARM64_HAS_VIRT_HOST_EXTN);
	42
	43	bool __hyp_text __fpsimd_enabled(void)
	44	{
	45	return __fpsimd_is_enabled()();
	46	}
	47
68908bf7 MZ	48	static void __hyp_text __activate_traps_vhe(void)
	49	{
	50	u64 val;
	51
	52	val = read_sysreg(cpacr_el1);
	53	val \|= CPACR_EL1_TTA;
	54	val &= ~CPACR_EL1_FPEN;
	55	write_sysreg(val, cpacr_el1);
	56
59cbf79e	57	write_sysreg(kvm_get_hyp_vector(), vbar_el1);
68908bf7 MZ	58	}
	59
	60	static void __hyp_text __activate_traps_nvhe(void)
	61	{
	62	u64 val;
	63
	64	val = CPTR_EL2_DEFAULT;
	65	val \|= CPTR_EL2_TTA \| CPTR_EL2_TFP;
	66	write_sysreg(val, cptr_el2);
	67	}
	68
	69	static hyp_alternate_select(__activate_traps_arch,
	70	__activate_traps_nvhe, __activate_traps_vhe,
	71	ARM64_HAS_VIRT_HOST_EXTN);
	72
be901e9b MZ	73	static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)
	74	{
	75	u64 val;
	76
	77	/*
	78	* We are about to set CPTR_EL2.TFP to trap all floating point
	79	* register accesses to EL2, however, the ARM ARM clearly states that
	80	* traps are only taken to EL2 if the operation would not otherwise
	81	* trap to EL1. Therefore, always make sure that for 32-bit guests,
	82	* we set FPEXC.EN to prevent traps to EL1, when setting the TFP bit.
82e0191a SP	83	* If FP/ASIMD is not implemented, FPEXC is UNDEFINED and any access to
82e0191a SP	84	* it will cause an exception.
be901e9b MZ	85	*/
be901e9b MZ	86	val = vcpu->arch.hcr_el2;
82e0191a	87	if (!(val & HCR_RW) && system_supports_fpsimd()) {
be901e9b MZ	88	write_sysreg(1 << 30, fpexc32_el2);
	89	isb();
	90	}
	91	write_sysreg(val, hcr_el2);
	92	/* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */
	93	write_sysreg(1 << 15, hstr_el2);
21cbe3cc MZ	94	/*
	95	* Make sure we trap PMU access from EL0 to EL2. Also sanitize
	96	* PMSELR_EL0 to make sure it never contains the cycle
	97	* counter, which could make a PMXEVCNTR_EL0 access UNDEF at
	98	* EL1 instead of being trapped to EL2.
	99	*/
	100	write_sysreg(0, pmselr_el0);
d692b8ad	101	write_sysreg(ARMV8_PMU_USERENR_MASK, pmuserenr_el0);
68908bf7 MZ	102	write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2);
	103	__activate_traps_arch()();
	104	}
a7e0ac29	105
68908bf7 MZ	106	static void __hyp_text __deactivate_traps_vhe(void)
	107	{
	108	extern char vectors[]; /* kernel exception vectors */
f85279b4	109	u64 mdcr_el2 = read_sysreg(mdcr_el2);
a7e0ac29	110
f85279b4 WD	111	mdcr_el2 &= MDCR_EL2_HPMN_MASK \|
	112	MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT \|
	113	MDCR_EL2_TPMS;
	114
	115	write_sysreg(mdcr_el2, mdcr_el2);
68908bf7 MZ	116	write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
	117	write_sysreg(CPACR_EL1_FPEN, cpacr_el1);
	118	write_sysreg(vectors, vbar_el1);
be901e9b MZ	119	}
be901e9b MZ	120
68908bf7	121	static void __hyp_text __deactivate_traps_nvhe(void)
be901e9b	122	{
f85279b4 WD	123	u64 mdcr_el2 = read_sysreg(mdcr_el2);
	124
	125	mdcr_el2 &= MDCR_EL2_HPMN_MASK;
	126	mdcr_el2 \|= MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT;
	127
	128	write_sysreg(mdcr_el2, mdcr_el2);
be901e9b	129	write_sysreg(HCR_RW, hcr_el2);
68908bf7 MZ	130	write_sysreg(CPTR_EL2_DEFAULT, cptr_el2);
	131	}
	132
	133	static hyp_alternate_select(__deactivate_traps_arch,
	134	__deactivate_traps_nvhe, __deactivate_traps_vhe,
	135	ARM64_HAS_VIRT_HOST_EXTN);
	136
	137	static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
	138	{
44636f97 MZ	139	/*
	140	* If we pended a virtual abort, preserve it until it gets
	141	* cleared. See D1.14.3 (Virtual Interrupts) for details, but
	142	* the crucial bit is "On taking a vSError interrupt,
	143	* HCR_EL2.VSE is cleared to 0."
	144	*/
	145	if (vcpu->arch.hcr_el2 & HCR_VSE)
	146	vcpu->arch.hcr_el2 = read_sysreg(hcr_el2);
	147
68908bf7	148	__deactivate_traps_arch()();
be901e9b	149	write_sysreg(0, hstr_el2);
d692b8ad	150	write_sysreg(0, pmuserenr_el0);
be901e9b MZ	151	}
	152
	153	static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu)
	154	{
	155	struct kvm *kvm = kern_hyp_va(vcpu->kvm);
	156	write_sysreg(kvm->arch.vttbr, vttbr_el2);
	157	}
	158
	159	static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu)
	160	{
	161	write_sysreg(0, vttbr_el2);
	162	}
	163
be901e9b MZ	164	static void __hyp_text __vgic_save_state(struct kvm_vcpu *vcpu)
be901e9b MZ	165	{
5a7a8426 VM	166	if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
	167	__vgic_v3_save_state(vcpu);
	168	else
	169	__vgic_v2_save_state(vcpu);
	170
be901e9b MZ	171	write_sysreg(read_sysreg(hcr_el2) & ~HCR_INT_OVERRIDE, hcr_el2);
	172	}
	173
	174	static void __hyp_text __vgic_restore_state(struct kvm_vcpu *vcpu)
	175	{
	176	u64 val;
	177
	178	val = read_sysreg(hcr_el2);
	179	val \|= HCR_INT_OVERRIDE;
	180	val \|= vcpu->arch.irq_lines;
	181	write_sysreg(val, hcr_el2);
	182
5a7a8426 VM	183	if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
	184	__vgic_v3_restore_state(vcpu);
	185	else
	186	__vgic_v2_restore_state(vcpu);
be901e9b MZ	187	}
be901e9b MZ	188
5f05a72a MZ	189	static bool __hyp_text __true_value(void)
	190	{
	191	return true;
	192	}
	193
	194	static bool __hyp_text __false_value(void)
	195	{
	196	return false;
	197	}
	198
	199	static hyp_alternate_select(__check_arm_834220,
	200	__false_value, __true_value,
	201	ARM64_WORKAROUND_834220);
	202
	203	static bool __hyp_text __translate_far_to_hpfar(u64 far, u64 *hpfar)
	204	{
	205	u64 par, tmp;
	206
	207	/*
	208	* Resolve the IPA the hard way using the guest VA.
	209	*
	210	* Stage-1 translation already validated the memory access
	211	* rights. As such, we can use the EL1 translation regime, and
	212	* don't have to distinguish between EL0 and EL1 access.
	213	*
	214	* We do need to save/restore PAR_EL1 though, as we haven't
	215	* saved the guest context yet, and we may return early...
	216	*/
	217	par = read_sysreg(par_el1);
	218	asm volatile("at s1e1r, %0" : : "r" (far));
	219	isb();
	220
	221	tmp = read_sysreg(par_el1);
	222	write_sysreg(par, par_el1);
	223
	224	if (unlikely(tmp & 1))
	225	return false; /* Translation failed, back to guest */
	226
	227	/* Convert PAR to HPFAR format */
	228	*hpfar = ((tmp >> 12) & ((1UL << 36) - 1)) << 4;
	229	return true;
	230	}
	231
	232	static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu)
	233	{
	234	u64 esr = read_sysreg_el2(esr);
561454e2	235	u8 ec = ESR_ELx_EC(esr);
5f05a72a MZ	236	u64 hpfar, far;
	237
	238	vcpu->arch.fault.esr_el2 = esr;
	239
	240	if (ec != ESR_ELx_EC_DABT_LOW && ec != ESR_ELx_EC_IABT_LOW)
	241	return true;
	242
	243	far = read_sysreg_el2(far);
	244
	245	/*
	246	* The HPFAR can be invalid if the stage 2 fault did not
	247	* happen during a stage 1 page table walk (the ESR_EL2.S1PTW
	248	* bit is clear) and one of the two following cases are true:
	249	* 1. The fault was due to a permission fault
	250	* 2. The processor carries errata 834220
	251	*
	252	* Therefore, for all non S1PTW faults where we either have a
	253	* permission fault or the errata workaround is enabled, we
	254	* resolve the IPA using the AT instruction.
	255	*/
	256	if (!(esr & ESR_ELx_S1PTW) &&
	257	(__check_arm_834220()() \|\| (esr & ESR_ELx_FSC_TYPE) == FSC_PERM)) {
	258	if (!__translate_far_to_hpfar(far, &hpfar))
	259	return false;
	260	} else {
	261	hpfar = read_sysreg(hpfar_el2);
	262	}
	263
	264	vcpu->arch.fault.far_el2 = far;
	265	vcpu->arch.fault.hpfar_el2 = hpfar;
	266	return true;
	267	}
	268
fb5ee369 MZ	269	static void __hyp_text __skip_instr(struct kvm_vcpu *vcpu)
	270	{
	271	*vcpu_pc(vcpu) = read_sysreg_el2(elr);
	272
	273	if (vcpu_mode_is_32bit(vcpu)) {
	274	vcpu->arch.ctxt.gp_regs.regs.pstate = read_sysreg_el2(spsr);
	275	kvm_skip_instr32(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
	276	write_sysreg_el2(vcpu->arch.ctxt.gp_regs.regs.pstate, spsr);
	277	} else {
	278	*vcpu_pc(vcpu) += 4;
	279	}
	280
	281	write_sysreg_el2(*vcpu_pc(vcpu), elr);
	282	}
	283
cf0ba18a	284	int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
be901e9b MZ	285	{
	286	struct kvm_cpu_context *host_ctxt;
	287	struct kvm_cpu_context *guest_ctxt;
c13d1683	288	bool fp_enabled;
be901e9b MZ	289	u64 exit_code;
	290
	291	vcpu = kern_hyp_va(vcpu);
	292	write_sysreg(vcpu, tpidr_el2);
	293
	294	host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
	295	guest_ctxt = &vcpu->arch.ctxt;
	296
edef528d	297	__sysreg_save_host_state(host_ctxt);
be901e9b MZ	298	__debug_cond_save_host_state(vcpu);
	299
	300	__activate_traps(vcpu);
	301	__activate_vm(vcpu);
	302
	303	__vgic_restore_state(vcpu);
	304	__timer_restore_state(vcpu);
	305
	306	/*
	307	* We must restore the 32-bit state before the sysregs, thanks
674e7012	308	* to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
be901e9b MZ	309	*/
be901e9b MZ	310	__sysreg32_restore_state(vcpu);
edef528d	311	__sysreg_restore_guest_state(guest_ctxt);
be901e9b MZ	312	__debug_restore_state(vcpu, kern_hyp_va(vcpu->arch.debug_ptr), guest_ctxt);
	313
	314	/* Jump in the fire! */
5f05a72a	315	again:
be901e9b MZ	316	exit_code = __guest_enter(vcpu, host_ctxt);
	317	/* And we're baaack! */
	318
395ea79e MZ	319	/*
	320	* We're using the raw exception code in order to only process
	321	* the trap if no SError is pending. We will come back to the
	322	* same PC once the SError has been injected, and replay the
	323	* trapping instruction.
	324	*/
5f05a72a MZ	325	if (exit_code == ARM_EXCEPTION_TRAP && !__populate_fault_info(vcpu))
	326	goto again;
	327
fb5ee369 MZ	328	if (static_branch_unlikely(&vgic_v2_cpuif_trap) &&
	329	exit_code == ARM_EXCEPTION_TRAP) {
	330	bool valid;
	331
	332	valid = kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_DABT_LOW &&
	333	kvm_vcpu_trap_get_fault_type(vcpu) == FSC_FAULT &&
	334	kvm_vcpu_dabt_isvalid(vcpu) &&
	335	!kvm_vcpu_dabt_isextabt(vcpu) &&
	336	!kvm_vcpu_dabt_iss1tw(vcpu);
	337
3272f0d0 MZ	338	if (valid) {
	339	int ret = __vgic_v2_perform_cpuif_access(vcpu);
	340
	341	if (ret == 1) {
	342	__skip_instr(vcpu);
	343	goto again;
	344	}
	345
	346	if (ret == -1) {
	347	/* Promote an illegal access to an SError */
	348	__skip_instr(vcpu);
	349	exit_code = ARM_EXCEPTION_EL1_SERROR;
	350	}
	351
	352	/* 0 falls through to be handler out of EL2 */
fb5ee369 MZ	353	}
	354	}
	355
59da1cbf MZ	356	if (static_branch_unlikely(&vgic_v3_cpuif_trap) &&
	357	exit_code == ARM_EXCEPTION_TRAP &&
	358	(kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_SYS64 \|\|
	359	kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_CP15_32)) {
	360	int ret = __vgic_v3_perform_cpuif_access(vcpu);
	361
	362	if (ret == 1) {
	363	__skip_instr(vcpu);
	364	goto again;
	365	}
	366
	367	/* 0 falls through to be handled out of EL2 */
	368	}
	369
a516894a SD	370	if (cpus_have_const_cap(ARM64_HARDEN_BP_POST_GUEST_EXIT)) {
	371	u32 midr = read_cpuid_id();
	372
	373	/* Apply BTAC predictors mitigation to all Falkor chips */
dcd07d4d SD	374	if (((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR) \|\|
dcd07d4d SD	375	((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR_V1)) {
a516894a	376	__qcom_hyp_sanitize_btac_predictors();
dcd07d4d	377	}
a516894a SD	378	}
a516894a SD	379
c13d1683 MZ	380	fp_enabled = __fpsimd_enabled();
c13d1683 MZ	381
edef528d	382	__sysreg_save_guest_state(guest_ctxt);
be901e9b MZ	383	__sysreg32_save_state(vcpu);
	384	__timer_save_state(vcpu);
	385	__vgic_save_state(vcpu);
	386
	387	__deactivate_traps(vcpu);
	388	__deactivate_vm(vcpu);
	389
edef528d	390	__sysreg_restore_host_state(host_ctxt);
be901e9b	391
c13d1683 MZ	392	if (fp_enabled) {
	393	__fpsimd_save_state(&guest_ctxt->gp_regs.fp_regs);
	394	__fpsimd_restore_state(&host_ctxt->gp_regs.fp_regs);
	395	}
	396
be901e9b	397	__debug_save_state(vcpu, kern_hyp_va(vcpu->arch.debug_ptr), guest_ctxt);
f85279b4 WD	398	/*
	399	* This must come after restoring the host sysregs, since a non-VHE
	400	* system may enable SPE here and make use of the TTBRs.
	401	*/
be901e9b MZ	402	__debug_cond_restore_host_state(vcpu);
	403
	404	return exit_code;
	405	}
53fd5b64 MZ	406
	407	static const char __hyp_panic_string[] = "HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n";
	408
253dcbd3	409	static void __hyp_text __hyp_call_panic_nvhe(u64 spsr, u64 elr, u64 par)
53fd5b64	410	{
cf7df13d	411	unsigned long str_va;
253dcbd3	412
cf7df13d MZ	413	/*
	414	* Force the panic string to be loaded from the literal pool,
	415	* making sure it is a kernel address and not a PC-relative
	416	* reference.
	417	*/
	418	asm volatile("ldr %0, =__hyp_panic_string" : "=r" (str_va));
	419
	420	__hyp_do_panic(str_va,
253dcbd3 MZ	421	spsr, elr,
	422	read_sysreg(esr_el2), read_sysreg_el2(far),
	423	read_sysreg(hpfar_el2), par,
	424	(void *)read_sysreg(tpidr_el2));
	425	}
	426
	427	static void __hyp_text __hyp_call_panic_vhe(u64 spsr, u64 elr, u64 par)
	428	{
	429	panic(__hyp_panic_string,
	430	spsr, elr,
	431	read_sysreg_el2(esr), read_sysreg_el2(far),
	432	read_sysreg(hpfar_el2), par,
	433	(void *)read_sysreg(tpidr_el2));
	434	}
	435
	436	static hyp_alternate_select(__hyp_call_panic,
	437	__hyp_call_panic_nvhe, __hyp_call_panic_vhe,
	438	ARM64_HAS_VIRT_HOST_EXTN);
	439
	440	void __hyp_text __noreturn __hyp_panic(void)
	441	{
	442	u64 spsr = read_sysreg_el2(spsr);
	443	u64 elr = read_sysreg_el2(elr);
53fd5b64 MZ	444	u64 par = read_sysreg(par_el1);
	445
	446	if (read_sysreg(vttbr_el2)) {
	447	struct kvm_vcpu *vcpu;
	448	struct kvm_cpu_context *host_ctxt;
	449
	450	vcpu = (struct kvm_vcpu *)read_sysreg(tpidr_el2);
	451	host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
e8ec032b	452	__timer_save_state(vcpu);
53fd5b64 MZ	453	__deactivate_traps(vcpu);
53fd5b64 MZ	454	__deactivate_vm(vcpu);
edef528d	455	__sysreg_restore_host_state(host_ctxt);
53fd5b64 MZ	456	}
	457
	458	/* Call panic for real */
253dcbd3	459	__hyp_call_panic()(spsr, elr, par);
53fd5b64 MZ	460
	461	unreachable();
	462	}