arch/arm64/kvm/hyp/switch.c

   1 /*
   2  * Copyright (C) 2015 - ARM Ltd
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include <linux/types.h>
  19 #include <asm/kvm_asm.h>
  20 #include <asm/kvm_hyp.h>
  21
  22 static bool __hyp_text __fpsimd_enabled_nvhe(void)
  23 {
  24         return !(read_sysreg(cptr_el2) & CPTR_EL2_TFP);
  25 }
  26
  27 static bool __hyp_text __fpsimd_enabled_vhe(void)
  28 {
  29         return !!(read_sysreg(cpacr_el1) & CPACR_EL1_FPEN);
  30 }
  31
  32 static hyp_alternate_select(__fpsimd_is_enabled,
  33                             __fpsimd_enabled_nvhe, __fpsimd_enabled_vhe,
  34                             ARM64_HAS_VIRT_HOST_EXTN);
  35
  36 bool __hyp_text __fpsimd_enabled(void)
  37 {
  38         return __fpsimd_is_enabled()();
  39 }
  40
  41 static void __hyp_text __activate_traps_vhe(void)
  42 {
  43         u64 val;
  44
  45         val = read_sysreg(cpacr_el1);
  46         val |= CPACR_EL1_TTA;
  47         val &= ~CPACR_EL1_FPEN;
  48         write_sysreg(val, cpacr_el1);
  49
  50         write_sysreg(__kvm_hyp_vector, vbar_el1);
  51 }
  52
  53 static void __hyp_text __activate_traps_nvhe(void)
  54 {
  55         u64 val;
  56
  57         val = CPTR_EL2_DEFAULT;
  58         val |= CPTR_EL2_TTA | CPTR_EL2_TFP;
  59         write_sysreg(val, cptr_el2);
  60 }
  61
  62 static hyp_alternate_select(__activate_traps_arch,
  63                             __activate_traps_nvhe, __activate_traps_vhe,
  64                             ARM64_HAS_VIRT_HOST_EXTN);
  65
  66 static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)
  67 {
  68         u64 val;
  69
  70         /*
  71          * We are about to set CPTR_EL2.TFP to trap all floating point
  72          * register accesses to EL2, however, the ARM ARM clearly states that
  73          * traps are only taken to EL2 if the operation would not otherwise
  74          * trap to EL1.  Therefore, always make sure that for 32-bit guests,
  75          * we set FPEXC.EN to prevent traps to EL1, when setting the TFP bit.
  76          */
  77         val = vcpu->arch.hcr_el2;
  78         if (!(val & HCR_RW)) {
  79                 write_sysreg(1 << 30, fpexc32_el2);
  80                 isb();
  81         }
  82         write_sysreg(val, hcr_el2);
  83         /* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */
  84         write_sysreg(1 << 15, hstr_el2);
  85         write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2);
  86         __activate_traps_arch()();
  87 }
  88
  89 static void __hyp_text __deactivate_traps_vhe(void)
  90 {
  91         extern char vectors[];  /* kernel exception vectors */
  92
  93         write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
  94         write_sysreg(CPACR_EL1_FPEN, cpacr_el1);
  95         write_sysreg(vectors, vbar_el1);
  96 }
  97
  98 static void __hyp_text __deactivate_traps_nvhe(void)
  99 {
 100         write_sysreg(HCR_RW, hcr_el2);
 101         write_sysreg(CPTR_EL2_DEFAULT, cptr_el2);
 102 }
 103
 104 static hyp_alternate_select(__deactivate_traps_arch,
 105                             __deactivate_traps_nvhe, __deactivate_traps_vhe,
 106                             ARM64_HAS_VIRT_HOST_EXTN);
 107
 108 static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
 109 {
 110         __deactivate_traps_arch()();
 111         write_sysreg(0, hstr_el2);
 112         write_sysreg(read_sysreg(mdcr_el2) & MDCR_EL2_HPMN_MASK, mdcr_el2);
 113 }
 114
 115 static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu)
 116 {
 117         struct kvm *kvm = kern_hyp_va(vcpu->kvm);
 118         write_sysreg(kvm->arch.vttbr, vttbr_el2);
 119 }
 120
 121 static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu)
 122 {
 123         write_sysreg(0, vttbr_el2);
 124 }
 125
 126 static hyp_alternate_select(__vgic_call_save_state,
 127                             __vgic_v2_save_state, __vgic_v3_save_state,
 128                             ARM64_HAS_SYSREG_GIC_CPUIF);
 129
 130 static hyp_alternate_select(__vgic_call_restore_state,
 131                             __vgic_v2_restore_state, __vgic_v3_restore_state,
 132                             ARM64_HAS_SYSREG_GIC_CPUIF);
 133
 134 static void __hyp_text __vgic_save_state(struct kvm_vcpu *vcpu)
 135 {
 136         __vgic_call_save_state()(vcpu);
 137         write_sysreg(read_sysreg(hcr_el2) & ~HCR_INT_OVERRIDE, hcr_el2);
 138 }
 139
 140 static void __hyp_text __vgic_restore_state(struct kvm_vcpu *vcpu)
 141 {
 142         u64 val;
 143
 144         val = read_sysreg(hcr_el2);
 145         val |=  HCR_INT_OVERRIDE;
 146         val |= vcpu->arch.irq_lines;
 147         write_sysreg(val, hcr_el2);
 148
 149         __vgic_call_restore_state()(vcpu);
 150 }
 151
 152 static bool __hyp_text __true_value(void)
 153 {
 154         return true;
 155 }
 156
 157 static bool __hyp_text __false_value(void)
 158 {
 159         return false;
 160 }
 161
 162 static hyp_alternate_select(__check_arm_834220,
 163                             __false_value, __true_value,
 164                             ARM64_WORKAROUND_834220);
 165
 166 static bool __hyp_text __translate_far_to_hpfar(u64 far, u64 *hpfar)
 167 {
 168         u64 par, tmp;
 169
 170         /*
 171          * Resolve the IPA the hard way using the guest VA.
 172          *
 173          * Stage-1 translation already validated the memory access
 174          * rights. As such, we can use the EL1 translation regime, and
 175          * don't have to distinguish between EL0 and EL1 access.
 176          *
 177          * We do need to save/restore PAR_EL1 though, as we haven't
 178          * saved the guest context yet, and we may return early...
 179          */
 180         par = read_sysreg(par_el1);
 181         asm volatile("at s1e1r, %0" : : "r" (far));
 182         isb();
 183
 184         tmp = read_sysreg(par_el1);
 185         write_sysreg(par, par_el1);
 186
 187         if (unlikely(tmp & 1))
 188                 return false; /* Translation failed, back to guest */
 189
 190         /* Convert PAR to HPFAR format */
 191         *hpfar = ((tmp >> 12) & ((1UL << 36) - 1)) << 4;
 192         return true;
 193 }
 194
 195 static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu)
 196 {
 197         u64 esr = read_sysreg_el2(esr);
 198         u8 ec = esr >> ESR_ELx_EC_SHIFT;
 199         u64 hpfar, far;
 200
 201         vcpu->arch.fault.esr_el2 = esr;
 202
 203         if (ec != ESR_ELx_EC_DABT_LOW && ec != ESR_ELx_EC_IABT_LOW)
 204                 return true;
 205
 206         far = read_sysreg_el2(far);
 207
 208         /*
 209          * The HPFAR can be invalid if the stage 2 fault did not
 210          * happen during a stage 1 page table walk (the ESR_EL2.S1PTW
 211          * bit is clear) and one of the two following cases are true:
 212          *   1. The fault was due to a permission fault
 213          *   2. The processor carries errata 834220
 214          *
 215          * Therefore, for all non S1PTW faults where we either have a
 216          * permission fault or the errata workaround is enabled, we
 217          * resolve the IPA using the AT instruction.
 218          */
 219         if (!(esr & ESR_ELx_S1PTW) &&
 220             (__check_arm_834220()() || (esr & ESR_ELx_FSC_TYPE) == FSC_PERM)) {
 221                 if (!__translate_far_to_hpfar(far, &hpfar))
 222                         return false;
 223         } else {
 224                 hpfar = read_sysreg(hpfar_el2);
 225         }
 226
 227         vcpu->arch.fault.far_el2 = far;
 228         vcpu->arch.fault.hpfar_el2 = hpfar;
 229         return true;
 230 }
 231
 232 static int __hyp_text __guest_run(struct kvm_vcpu *vcpu)
 233 {
 234         struct kvm_cpu_context *host_ctxt;
 235         struct kvm_cpu_context *guest_ctxt;
 236         bool fp_enabled;
 237         u64 exit_code;
 238
 239         vcpu = kern_hyp_va(vcpu);
 240         write_sysreg(vcpu, tpidr_el2);
 241
 242         host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
 243         guest_ctxt = &vcpu->arch.ctxt;
 244
 245         __sysreg_save_host_state(host_ctxt);
 246         __debug_cond_save_host_state(vcpu);
 247
 248         __activate_traps(vcpu);
 249         __activate_vm(vcpu);
 250
 251         __vgic_restore_state(vcpu);
 252         __timer_restore_state(vcpu);
 253
 254         /*
 255          * We must restore the 32-bit state before the sysregs, thanks
 256          * to Cortex-A57 erratum #852523.
 257          */
 258         __sysreg32_restore_state(vcpu);
 259         __sysreg_restore_guest_state(guest_ctxt);
 260         __debug_restore_state(vcpu, kern_hyp_va(vcpu->arch.debug_ptr), guest_ctxt);
 261
 262         /* Jump in the fire! */
 263 again:
 264         exit_code = __guest_enter(vcpu, host_ctxt);
 265         /* And we're baaack! */
 266
 267         if (exit_code == ARM_EXCEPTION_TRAP && !__populate_fault_info(vcpu))
 268                 goto again;
 269
 270         fp_enabled = __fpsimd_enabled();
 271
 272         __sysreg_save_guest_state(guest_ctxt);
 273         __sysreg32_save_state(vcpu);
 274         __timer_save_state(vcpu);
 275         __vgic_save_state(vcpu);
 276
 277         __deactivate_traps(vcpu);
 278         __deactivate_vm(vcpu);
 279
 280         __sysreg_restore_host_state(host_ctxt);
 281
 282         if (fp_enabled) {
 283                 __fpsimd_save_state(&guest_ctxt->gp_regs.fp_regs);
 284                 __fpsimd_restore_state(&host_ctxt->gp_regs.fp_regs);
 285         }
 286
 287         __debug_save_state(vcpu, kern_hyp_va(vcpu->arch.debug_ptr), guest_ctxt);
 288         __debug_cond_restore_host_state(vcpu);
 289
 290         return exit_code;
 291 }
 292
 293 __alias(__guest_run) int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
 294
 295 static const char __hyp_panic_string[] = "HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n";
 296
 297 static void __hyp_text __hyp_call_panic_nvhe(u64 spsr, u64 elr, u64 par)
 298 {
 299         unsigned long str_va = (unsigned long)__hyp_panic_string;
 300
 301         __hyp_do_panic(hyp_kern_va(str_va),
 302                        spsr,  elr,
 303                        read_sysreg(esr_el2),   read_sysreg_el2(far),
 304                        read_sysreg(hpfar_el2), par,
 305                        (void *)read_sysreg(tpidr_el2));
 306 }
 307
 308 static void __hyp_text __hyp_call_panic_vhe(u64 spsr, u64 elr, u64 par)
 309 {
 310         panic(__hyp_panic_string,
 311               spsr,  elr,
 312               read_sysreg_el2(esr),   read_sysreg_el2(far),
 313               read_sysreg(hpfar_el2), par,
 314               (void *)read_sysreg(tpidr_el2));
 315 }
 316
 317 static hyp_alternate_select(__hyp_call_panic,
 318                             __hyp_call_panic_nvhe, __hyp_call_panic_vhe,
 319                             ARM64_HAS_VIRT_HOST_EXTN);
 320
 321 void __hyp_text __noreturn __hyp_panic(void)
 322 {
 323         u64 spsr = read_sysreg_el2(spsr);
 324         u64 elr = read_sysreg_el2(elr);
 325         u64 par = read_sysreg(par_el1);
 326
 327         if (read_sysreg(vttbr_el2)) {
 328                 struct kvm_vcpu *vcpu;
 329                 struct kvm_cpu_context *host_ctxt;
 330
 331                 vcpu = (struct kvm_vcpu *)read_sysreg(tpidr_el2);
 332                 host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
 333                 __deactivate_traps(vcpu);
 334                 __deactivate_vm(vcpu);
 335                 __sysreg_restore_host_state(host_ctxt);
 336         }
 337
 338         /* Call panic for real */
 339         __hyp_call_panic()(spsr, elr, par);
 340
 341         unreachable();
 342 }