]>
Commit | Line | Data |
---|---|---|
e6b673b7 DM |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * arch/arm64/kvm/fpsimd.c: Guest/host FPSIMD context coordination helpers | |
4 | * | |
5 | * Copyright 2018 Arm Limited | |
6 | * Author: Dave Martin <Dave.Martin@arm.com> | |
7 | */ | |
b045e4d0 | 8 | #include <linux/irqflags.h> |
e6b673b7 DM |
9 | #include <linux/sched.h> |
10 | #include <linux/thread_info.h> | |
11 | #include <linux/kvm_host.h> | |
12 | #include <asm/kvm_asm.h> | |
13 | #include <asm/kvm_host.h> | |
14 | #include <asm/kvm_mmu.h> | |
b3eb56b6 | 15 | #include <asm/sysreg.h> |
e6b673b7 DM |
16 | |
17 | /* | |
18 | * Called on entry to KVM_RUN unless this vcpu previously ran at least | |
19 | * once and the most recent prior KVM_RUN for this vcpu was called from | |
20 | * the same task as current (highly likely). | |
21 | * | |
22 | * This is guaranteed to execute before kvm_arch_vcpu_load_fp(vcpu), | |
23 | * such that on entering hyp the relevant parts of current are already | |
24 | * mapped. | |
25 | */ | |
26 | int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu) | |
27 | { | |
28 | int ret; | |
29 | ||
30 | struct thread_info *ti = ¤t->thread_info; | |
31 | struct user_fpsimd_state *fpsimd = ¤t->thread.uw.fpsimd_state; | |
32 | ||
33 | /* | |
34 | * Make sure the host task thread flags and fpsimd state are | |
35 | * visible to hyp: | |
36 | */ | |
37 | ret = create_hyp_mappings(ti, ti + 1, PAGE_HYP); | |
38 | if (ret) | |
39 | goto error; | |
40 | ||
41 | ret = create_hyp_mappings(fpsimd, fpsimd + 1, PAGE_HYP); | |
42 | if (ret) | |
43 | goto error; | |
44 | ||
45 | vcpu->arch.host_thread_info = kern_hyp_va(ti); | |
46 | vcpu->arch.host_fpsimd_state = kern_hyp_va(fpsimd); | |
47 | error: | |
48 | return ret; | |
49 | } | |
50 | ||
51 | /* | |
52 | * Prepare vcpu for saving the host's FPSIMD state and loading the guest's. | |
53 | * The actual loading is done by the FPSIMD access trap taken to hyp. | |
54 | * | |
55 | * Here, we just set the correct metadata to indicate that the FPSIMD | |
56 | * state in the cpu regs (if any) belongs to current on the host. | |
57 | * | |
58 | * TIF_SVE is backed up here, since it may get clobbered with guest state. | |
59 | * This flag is restored by kvm_arch_vcpu_put_fp(vcpu). | |
60 | */ | |
61 | void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu) | |
62 | { | |
e6b673b7 DM |
63 | BUG_ON(!current->mm); |
64 | ||
b3eb56b6 DM |
65 | vcpu->arch.flags &= ~(KVM_ARM64_FP_ENABLED | |
66 | KVM_ARM64_HOST_SVE_IN_USE | | |
67 | KVM_ARM64_HOST_SVE_ENABLED); | |
e6b673b7 | 68 | vcpu->arch.flags |= KVM_ARM64_FP_HOST; |
b3eb56b6 | 69 | |
e6b673b7 DM |
70 | if (test_thread_flag(TIF_SVE)) |
71 | vcpu->arch.flags |= KVM_ARM64_HOST_SVE_IN_USE; | |
b3eb56b6 DM |
72 | |
73 | if (read_sysreg(cpacr_el1) & CPACR_EL1_ZEN_EL0EN) | |
74 | vcpu->arch.flags |= KVM_ARM64_HOST_SVE_ENABLED; | |
e6b673b7 DM |
75 | } |
76 | ||
77 | /* | |
78 | * If the guest FPSIMD state was loaded, update the host's context | |
79 | * tracking data mark the CPU FPSIMD regs as dirty and belonging to vcpu | |
80 | * so that they will be written back if the kernel clobbers them due to | |
81 | * kernel-mode NEON before re-entry into the guest. | |
82 | */ | |
83 | void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu) | |
84 | { | |
85 | WARN_ON_ONCE(!irqs_disabled()); | |
86 | ||
87 | if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) { | |
88 | fpsimd_bind_state_to_cpu(&vcpu->arch.ctxt.gp_regs.fp_regs); | |
89 | clear_thread_flag(TIF_FOREIGN_FPSTATE); | |
90 | clear_thread_flag(TIF_SVE); | |
91 | } | |
92 | } | |
93 | ||
94 | /* | |
95 | * Write back the vcpu FPSIMD regs if they are dirty, and invalidate the | |
96 | * cpu FPSIMD regs so that they can't be spuriously reused if this vcpu | |
97 | * disappears and another task or vcpu appears that recycles the same | |
98 | * struct fpsimd_state. | |
99 | */ | |
100 | void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu) | |
101 | { | |
b045e4d0 DM |
102 | unsigned long flags; |
103 | ||
104 | local_irq_save(flags); | |
e6b673b7 | 105 | |
e6b673b7 DM |
106 | if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) { |
107 | /* Clean guest FP state to memory and invalidate cpu view */ | |
108 | fpsimd_save(); | |
109 | fpsimd_flush_cpu_state(); | |
b3eb56b6 DM |
110 | } else if (system_supports_sve()) { |
111 | /* | |
112 | * The FPSIMD/SVE state in the CPU has not been touched, and we | |
113 | * have SVE (and VHE): CPACR_EL1 (alias CPTR_EL2) has been | |
114 | * reset to CPACR_EL1_DEFAULT by the Hyp code, disabling SVE | |
115 | * for EL0. To avoid spurious traps, restore the trap state | |
116 | * seen by kvm_arch_vcpu_load_fp(): | |
117 | */ | |
118 | if (vcpu->arch.flags & KVM_ARM64_HOST_SVE_ENABLED) | |
119 | sysreg_clear_set(CPACR_EL1, 0, CPACR_EL1_ZEN_EL0EN); | |
120 | else | |
121 | sysreg_clear_set(CPACR_EL1, CPACR_EL1_ZEN_EL0EN, 0); | |
e6b673b7 DM |
122 | } |
123 | ||
2955bcc8 DM |
124 | update_thread_flag(TIF_SVE, |
125 | vcpu->arch.flags & KVM_ARM64_HOST_SVE_IN_USE); | |
126 | ||
b045e4d0 | 127 | local_irq_restore(flags); |
e6b673b7 | 128 | } |