]>
Commit | Line | Data |
---|---|---|
74fe55dc MZ |
1 | /* |
2 | * Copyright (C) 2017 ARM Ltd. | |
3 | * Author: Marc Zyngier <marc.zyngier@arm.com> | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify | |
6 | * it under the terms of the GNU General Public License version 2 as | |
7 | * published by the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | * GNU General Public License for more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License | |
15 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
16 | */ | |
17 | ||
18 | #include <linux/interrupt.h> | |
bdb2d2cc | 19 | #include <linux/irq.h> |
74fe55dc MZ |
20 | #include <linux/irqdomain.h> |
21 | #include <linux/kvm_host.h> | |
196b1364 | 22 | #include <linux/irqchip/arm-gic-v3.h> |
74fe55dc MZ |
23 | |
24 | #include "vgic.h" | |
25 | ||
ed8703a5 MZ |
26 | /* |
27 | * How KVM uses GICv4 (insert rude comments here): | |
28 | * | |
29 | * The vgic-v4 layer acts as a bridge between several entities: | |
30 | * - The GICv4 ITS representation offered by the ITS driver | |
31 | * - VFIO, which is in charge of the PCI endpoint | |
32 | * - The virtual ITS, which is the only thing the guest sees | |
33 | * | |
34 | * The configuration of VLPIs is triggered by a callback from VFIO, | |
35 | * instructing KVM that a PCI device has been configured to deliver | |
36 | * MSIs to a vITS. | |
37 | * | |
38 | * kvm_vgic_v4_set_forwarding() is thus called with the routing entry, | |
39 | * and this is used to find the corresponding vITS data structures | |
40 | * (ITS instance, device, event and irq) using a process that is | |
41 | * extremely similar to the injection of an MSI. | |
42 | * | |
43 | * At this stage, we can link the guest's view of an LPI (uniquely | |
44 | * identified by the routing entry) and the host irq, using the GICv4 | |
45 | * driver mapping operation. Should the mapping succeed, we've then | |
46 | * successfully upgraded the guest's LPI to a VLPI. We can then start | |
47 | * with updating GICv4's view of the property table and generating an | |
48 | * INValidation in order to kickstart the delivery of this VLPI to the | |
49 | * guest directly, without software intervention. Well, almost. | |
50 | * | |
51 | * When the PCI endpoint is deconfigured, this operation is reversed | |
52 | * with VFIO calling kvm_vgic_v4_unset_forwarding(). | |
53 | * | |
54 | * Once the VLPI has been mapped, it needs to follow any change the | |
55 | * guest performs on its LPI through the vITS. For that, a number of | |
56 | * command handlers have hooks to communicate these changes to the HW: | |
57 | * - Any invalidation triggers a call to its_prop_update_vlpi() | |
58 | * - The INT command results in a irq_set_irqchip_state(), which | |
59 | * generates an INT on the corresponding VLPI. | |
60 | * - The CLEAR command results in a irq_set_irqchip_state(), which | |
61 | * generates an CLEAR on the corresponding VLPI. | |
62 | * - DISCARD translates into an unmap, similar to a call to | |
63 | * kvm_vgic_v4_unset_forwarding(). | |
64 | * - MOVI is translated by an update of the existing mapping, changing | |
65 | * the target vcpu, resulting in a VMOVI being generated. | |
66 | * - MOVALL is translated by a string of mapping updates (similar to | |
67 | * the handling of MOVI). MOVALL is horrible. | |
68 | * | |
69 | * Note that a DISCARD/MAPTI sequence emitted from the guest without | |
70 | * reprogramming the PCI endpoint after MAPTI does not result in a | |
71 | * VLPI being mapped, as there is no callback from VFIO (the guest | |
72 | * will get the interrupt via the normal SW injection). Fixing this is | |
73 | * not trivial, and requires some horrible messing with the VFIO | |
74 | * internals. Not fun. Don't do that. | |
75 | * | |
76 | * Then there is the scheduling. Each time a vcpu is about to run on a | |
77 | * physical CPU, KVM must tell the corresponding redistributor about | |
78 | * it. And if we've migrated our vcpu from one CPU to another, we must | |
79 | * tell the ITS (so that the messages reach the right redistributor). | |
80 | * This is done in two steps: first issue a irq_set_affinity() on the | |
81 | * irq corresponding to the vcpu, then call its_schedule_vpe(). You | |
82 | * must be in a non-preemptible context. On exit, another call to | |
83 | * its_schedule_vpe() tells the redistributor that we're done with the | |
84 | * vcpu. | |
85 | * | |
86 | * Finally, the doorbell handling: Each vcpu is allocated an interrupt | |
87 | * which will fire each time a VLPI is made pending whilst the vcpu is | |
88 | * not running. Each time the vcpu gets blocked, the doorbell | |
89 | * interrupt gets enabled. When the vcpu is unblocked (for whatever | |
90 | * reason), the doorbell interrupt is disabled. | |
91 | */ | |
92 | ||
d3d83f7f MZ |
93 | #define DB_IRQ_FLAGS (IRQ_NOAUTOEN | IRQ_DISABLE_UNLAZY | IRQ_NO_BALANCING) |
94 | ||
bdb2d2cc MZ |
95 | static irqreturn_t vgic_v4_doorbell_handler(int irq, void *info) |
96 | { | |
97 | struct kvm_vcpu *vcpu = info; | |
98 | ||
99 | vcpu->arch.vgic_cpu.vgic_v3.its_vpe.pending_last = true; | |
100 | kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu); | |
101 | kvm_vcpu_kick(vcpu); | |
102 | ||
103 | return IRQ_HANDLED; | |
104 | } | |
105 | ||
74fe55dc MZ |
106 | /** |
107 | * vgic_v4_init - Initialize the GICv4 data structures | |
108 | * @kvm: Pointer to the VM being initialized | |
109 | * | |
110 | * We may be called each time a vITS is created, or when the | |
111 | * vgic is initialized. This relies on kvm->lock to be | |
112 | * held. In both cases, the number of vcpus should now be | |
113 | * fixed. | |
114 | */ | |
115 | int vgic_v4_init(struct kvm *kvm) | |
116 | { | |
117 | struct vgic_dist *dist = &kvm->arch.vgic; | |
118 | struct kvm_vcpu *vcpu; | |
119 | int i, nr_vcpus, ret; | |
120 | ||
f8f85dc0 | 121 | if (!kvm_vgic_global_state.has_gicv4) |
3d1ad640 CD |
122 | return 0; /* Nothing to see here... move along. */ |
123 | ||
74fe55dc MZ |
124 | if (dist->its_vm.vpes) |
125 | return 0; | |
126 | ||
127 | nr_vcpus = atomic_read(&kvm->online_vcpus); | |
128 | ||
129 | dist->its_vm.vpes = kzalloc(sizeof(*dist->its_vm.vpes) * nr_vcpus, | |
130 | GFP_KERNEL); | |
131 | if (!dist->its_vm.vpes) | |
132 | return -ENOMEM; | |
133 | ||
134 | dist->its_vm.nr_vpes = nr_vcpus; | |
135 | ||
136 | kvm_for_each_vcpu(i, vcpu, kvm) | |
137 | dist->its_vm.vpes[i] = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe; | |
138 | ||
139 | ret = its_alloc_vcpu_irqs(&dist->its_vm); | |
140 | if (ret < 0) { | |
141 | kvm_err("VPE IRQ allocation failure\n"); | |
142 | kfree(dist->its_vm.vpes); | |
143 | dist->its_vm.nr_vpes = 0; | |
144 | dist->its_vm.vpes = NULL; | |
145 | return ret; | |
146 | } | |
147 | ||
bdb2d2cc MZ |
148 | kvm_for_each_vcpu(i, vcpu, kvm) { |
149 | int irq = dist->its_vm.vpes[i]->irq; | |
150 | ||
151 | /* | |
152 | * Don't automatically enable the doorbell, as we're | |
153 | * flipping it back and forth when the vcpu gets | |
154 | * blocked. Also disable the lazy disabling, as the | |
155 | * doorbell could kick us out of the guest too | |
156 | * early... | |
157 | */ | |
d3d83f7f | 158 | irq_set_status_flags(irq, DB_IRQ_FLAGS); |
bdb2d2cc MZ |
159 | ret = request_irq(irq, vgic_v4_doorbell_handler, |
160 | 0, "vcpu", vcpu); | |
161 | if (ret) { | |
162 | kvm_err("failed to allocate vcpu IRQ%d\n", irq); | |
163 | /* | |
164 | * Trick: adjust the number of vpes so we know | |
165 | * how many to nuke on teardown... | |
166 | */ | |
167 | dist->its_vm.nr_vpes = i; | |
168 | break; | |
169 | } | |
170 | } | |
171 | ||
172 | if (ret) | |
173 | vgic_v4_teardown(kvm); | |
174 | ||
74fe55dc MZ |
175 | return ret; |
176 | } | |
177 | ||
178 | /** | |
179 | * vgic_v4_teardown - Free the GICv4 data structures | |
180 | * @kvm: Pointer to the VM being destroyed | |
181 | * | |
182 | * Relies on kvm->lock to be held. | |
183 | */ | |
184 | void vgic_v4_teardown(struct kvm *kvm) | |
185 | { | |
186 | struct its_vm *its_vm = &kvm->arch.vgic.its_vm; | |
bdb2d2cc | 187 | int i; |
74fe55dc MZ |
188 | |
189 | if (!its_vm->vpes) | |
190 | return; | |
191 | ||
bdb2d2cc MZ |
192 | for (i = 0; i < its_vm->nr_vpes; i++) { |
193 | struct kvm_vcpu *vcpu = kvm_get_vcpu(kvm, i); | |
194 | int irq = its_vm->vpes[i]->irq; | |
195 | ||
d3d83f7f | 196 | irq_clear_status_flags(irq, DB_IRQ_FLAGS); |
bdb2d2cc MZ |
197 | free_irq(irq, vcpu); |
198 | } | |
199 | ||
74fe55dc MZ |
200 | its_free_vcpu_irqs(its_vm); |
201 | kfree(its_vm->vpes); | |
202 | its_vm->nr_vpes = 0; | |
203 | its_vm->vpes = NULL; | |
204 | } | |
196b1364 | 205 | |
62775797 MZ |
206 | int vgic_v4_sync_hwstate(struct kvm_vcpu *vcpu) |
207 | { | |
208 | if (!vgic_supports_direct_msis(vcpu->kvm)) | |
209 | return 0; | |
210 | ||
211 | return its_schedule_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe, false); | |
212 | } | |
213 | ||
214 | int vgic_v4_flush_hwstate(struct kvm_vcpu *vcpu) | |
215 | { | |
216 | int irq = vcpu->arch.vgic_cpu.vgic_v3.its_vpe.irq; | |
217 | int err; | |
218 | ||
219 | if (!vgic_supports_direct_msis(vcpu->kvm)) | |
220 | return 0; | |
221 | ||
222 | /* | |
223 | * Before making the VPE resident, make sure the redistributor | |
224 | * corresponding to our current CPU expects us here. See the | |
225 | * doc in drivers/irqchip/irq-gic-v4.c to understand how this | |
226 | * turns into a VMOVP command at the ITS level. | |
227 | */ | |
228 | err = irq_set_affinity(irq, cpumask_of(smp_processor_id())); | |
229 | if (err) | |
230 | return err; | |
231 | ||
232 | err = its_schedule_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe, true); | |
233 | if (err) | |
234 | return err; | |
235 | ||
236 | /* | |
237 | * Now that the VPE is resident, let's get rid of a potential | |
238 | * doorbell interrupt that would still be pending. | |
239 | */ | |
240 | err = irq_set_irqchip_state(irq, IRQCHIP_STATE_PENDING, false); | |
241 | ||
242 | return err; | |
243 | } | |
244 | ||
196b1364 MZ |
245 | static struct vgic_its *vgic_get_its(struct kvm *kvm, |
246 | struct kvm_kernel_irq_routing_entry *irq_entry) | |
247 | { | |
248 | struct kvm_msi msi = (struct kvm_msi) { | |
249 | .address_lo = irq_entry->msi.address_lo, | |
250 | .address_hi = irq_entry->msi.address_hi, | |
251 | .data = irq_entry->msi.data, | |
252 | .flags = irq_entry->msi.flags, | |
253 | .devid = irq_entry->msi.devid, | |
254 | }; | |
255 | ||
256 | return vgic_msi_to_its(kvm, &msi); | |
257 | } | |
258 | ||
259 | int kvm_vgic_v4_set_forwarding(struct kvm *kvm, int virq, | |
260 | struct kvm_kernel_irq_routing_entry *irq_entry) | |
261 | { | |
262 | struct vgic_its *its; | |
263 | struct vgic_irq *irq; | |
264 | struct its_vlpi_map map; | |
265 | int ret; | |
266 | ||
267 | if (!vgic_supports_direct_msis(kvm)) | |
268 | return 0; | |
269 | ||
270 | /* | |
271 | * Get the ITS, and escape early on error (not a valid | |
272 | * doorbell for any of our vITSs). | |
273 | */ | |
274 | its = vgic_get_its(kvm, irq_entry); | |
275 | if (IS_ERR(its)) | |
276 | return 0; | |
277 | ||
278 | mutex_lock(&its->its_lock); | |
279 | ||
280 | /* Perform then actual DevID/EventID -> LPI translation. */ | |
281 | ret = vgic_its_resolve_lpi(kvm, its, irq_entry->msi.devid, | |
282 | irq_entry->msi.data, &irq); | |
283 | if (ret) | |
284 | goto out; | |
285 | ||
286 | /* | |
287 | * Emit the mapping request. If it fails, the ITS probably | |
288 | * isn't v4 compatible, so let's silently bail out. Holding | |
289 | * the ITS lock should ensure that nothing can modify the | |
290 | * target vcpu. | |
291 | */ | |
292 | map = (struct its_vlpi_map) { | |
293 | .vm = &kvm->arch.vgic.its_vm, | |
294 | .vpe = &irq->target_vcpu->arch.vgic_cpu.vgic_v3.its_vpe, | |
295 | .vintid = irq->intid, | |
296 | .properties = ((irq->priority & 0xfc) | | |
297 | (irq->enabled ? LPI_PROP_ENABLED : 0) | | |
298 | LPI_PROP_GROUP1), | |
299 | .db_enabled = true, | |
300 | }; | |
301 | ||
302 | ret = its_map_vlpi(virq, &map); | |
303 | if (ret) | |
304 | goto out; | |
305 | ||
306 | irq->hw = true; | |
307 | irq->host_irq = virq; | |
308 | ||
309 | out: | |
310 | mutex_unlock(&its->its_lock); | |
311 | return ret; | |
312 | } | |
313 | ||
314 | int kvm_vgic_v4_unset_forwarding(struct kvm *kvm, int virq, | |
315 | struct kvm_kernel_irq_routing_entry *irq_entry) | |
316 | { | |
317 | struct vgic_its *its; | |
318 | struct vgic_irq *irq; | |
319 | int ret; | |
320 | ||
321 | if (!vgic_supports_direct_msis(kvm)) | |
322 | return 0; | |
323 | ||
324 | /* | |
325 | * Get the ITS, and escape early on error (not a valid | |
326 | * doorbell for any of our vITSs). | |
327 | */ | |
328 | its = vgic_get_its(kvm, irq_entry); | |
329 | if (IS_ERR(its)) | |
330 | return 0; | |
331 | ||
332 | mutex_lock(&its->its_lock); | |
333 | ||
334 | ret = vgic_its_resolve_lpi(kvm, its, irq_entry->msi.devid, | |
335 | irq_entry->msi.data, &irq); | |
336 | if (ret) | |
337 | goto out; | |
338 | ||
339 | WARN_ON(!(irq->hw && irq->host_irq == virq)); | |
a05d1c0d MZ |
340 | if (irq->hw) { |
341 | irq->hw = false; | |
342 | ret = its_unmap_vlpi(virq); | |
343 | } | |
196b1364 MZ |
344 | |
345 | out: | |
346 | mutex_unlock(&its->its_lock); | |
347 | return ret; | |
348 | } | |
df9ba959 MZ |
349 | |
350 | void kvm_vgic_v4_enable_doorbell(struct kvm_vcpu *vcpu) | |
351 | { | |
352 | if (vgic_supports_direct_msis(vcpu->kvm)) { | |
353 | int irq = vcpu->arch.vgic_cpu.vgic_v3.its_vpe.irq; | |
354 | if (irq) | |
355 | enable_irq(irq); | |
356 | } | |
357 | } | |
358 | ||
359 | void kvm_vgic_v4_disable_doorbell(struct kvm_vcpu *vcpu) | |
360 | { | |
361 | if (vgic_supports_direct_msis(vcpu->kvm)) { | |
362 | int irq = vcpu->arch.vgic_cpu.vgic_v3.its_vpe.irq; | |
363 | if (irq) | |
364 | disable_irq(irq); | |
365 | } | |
366 | } |