]>
Commit | Line | Data |
---|---|---|
64a959d6 CD |
1 | /* |
2 | * Copyright (C) 2015, 2016 ARM Ltd. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License version 2 as | |
6 | * published by the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, | |
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
11 | * GNU General Public License for more details. | |
12 | * | |
13 | * You should have received a copy of the GNU General Public License | |
14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
15 | */ | |
16 | ||
17 | #include <linux/kvm.h> | |
18 | #include <linux/kvm_host.h> | |
8e444745 | 19 | #include <linux/list_sort.h> |
64a959d6 CD |
20 | |
21 | #include "vgic.h" | |
22 | ||
81eeb95d CD |
23 | #define CREATE_TRACE_POINTS |
24 | #include "../trace.h" | |
25 | ||
26 | #ifdef CONFIG_DEBUG_SPINLOCK | |
27 | #define DEBUG_SPINLOCK_BUG_ON(p) BUG_ON(p) | |
28 | #else | |
29 | #define DEBUG_SPINLOCK_BUG_ON(p) | |
30 | #endif | |
31 | ||
5a7a8426 | 32 | struct vgic_global __section(.hyp.text) kvm_vgic_global_state = {.gicv3_cpuif = STATIC_KEY_FALSE_INIT,}; |
64a959d6 | 33 | |
81eeb95d CD |
34 | /* |
35 | * Locking order is always: | |
424c3383 AP |
36 | * its->cmd_lock (mutex) |
37 | * its->its_lock (mutex) | |
38 | * vgic_cpu->ap_list_lock | |
3802411d AP |
39 | * kvm->lpi_list_lock |
40 | * vgic_irq->irq_lock | |
81eeb95d | 41 | * |
424c3383 AP |
42 | * If you need to take multiple locks, always take the upper lock first, |
43 | * then the lower ones, e.g. first take the its_lock, then the irq_lock. | |
44 | * If you are already holding a lock and need to take a higher one, you | |
45 | * have to drop the lower ranking lock first and re-aquire it after having | |
46 | * taken the upper one. | |
81eeb95d CD |
47 | * |
48 | * When taking more than one ap_list_lock at the same time, always take the | |
49 | * lowest numbered VCPU's ap_list_lock first, so: | |
50 | * vcpuX->vcpu_id < vcpuY->vcpu_id: | |
51 | * spin_lock(vcpuX->arch.vgic_cpu.ap_list_lock); | |
52 | * spin_lock(vcpuY->arch.vgic_cpu.ap_list_lock); | |
53 | */ | |
54 | ||
3802411d AP |
55 | /* |
56 | * Iterate over the VM's list of mapped LPIs to find the one with a | |
57 | * matching interrupt ID and return a reference to the IRQ structure. | |
58 | */ | |
59 | static struct vgic_irq *vgic_get_lpi(struct kvm *kvm, u32 intid) | |
60 | { | |
61 | struct vgic_dist *dist = &kvm->arch.vgic; | |
62 | struct vgic_irq *irq = NULL; | |
63 | ||
64 | spin_lock(&dist->lpi_list_lock); | |
65 | ||
66 | list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) { | |
67 | if (irq->intid != intid) | |
68 | continue; | |
69 | ||
70 | /* | |
71 | * This increases the refcount, the caller is expected to | |
72 | * call vgic_put_irq() later once it's finished with the IRQ. | |
73 | */ | |
d97594e6 | 74 | vgic_get_irq_kref(irq); |
3802411d AP |
75 | goto out_unlock; |
76 | } | |
77 | irq = NULL; | |
78 | ||
79 | out_unlock: | |
80 | spin_unlock(&dist->lpi_list_lock); | |
81 | ||
82 | return irq; | |
83 | } | |
84 | ||
85 | /* | |
86 | * This looks up the virtual interrupt ID to get the corresponding | |
87 | * struct vgic_irq. It also increases the refcount, so any caller is expected | |
88 | * to call vgic_put_irq() once it's finished with this IRQ. | |
89 | */ | |
64a959d6 CD |
90 | struct vgic_irq *vgic_get_irq(struct kvm *kvm, struct kvm_vcpu *vcpu, |
91 | u32 intid) | |
92 | { | |
93 | /* SGIs and PPIs */ | |
94 | if (intid <= VGIC_MAX_PRIVATE) | |
95 | return &vcpu->arch.vgic_cpu.private_irqs[intid]; | |
96 | ||
97 | /* SPIs */ | |
98 | if (intid <= VGIC_MAX_SPI) | |
99 | return &kvm->arch.vgic.spis[intid - VGIC_NR_PRIVATE_IRQS]; | |
100 | ||
3802411d | 101 | /* LPIs */ |
64a959d6 | 102 | if (intid >= VGIC_MIN_LPI) |
3802411d | 103 | return vgic_get_lpi(kvm, intid); |
64a959d6 CD |
104 | |
105 | WARN(1, "Looking up struct vgic_irq for reserved INTID"); | |
106 | return NULL; | |
107 | } | |
81eeb95d | 108 | |
3802411d AP |
109 | /* |
110 | * We can't do anything in here, because we lack the kvm pointer to | |
111 | * lock and remove the item from the lpi_list. So we keep this function | |
112 | * empty and use the return value of kref_put() to trigger the freeing. | |
113 | */ | |
5dd4b924 AP |
114 | static void vgic_irq_release(struct kref *ref) |
115 | { | |
5dd4b924 AP |
116 | } |
117 | ||
118 | void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq) | |
119 | { | |
2cccbb36 | 120 | struct vgic_dist *dist = &kvm->arch.vgic; |
3802411d | 121 | |
5dd4b924 AP |
122 | if (irq->intid < VGIC_MIN_LPI) |
123 | return; | |
124 | ||
2cccbb36 CD |
125 | spin_lock(&dist->lpi_list_lock); |
126 | if (!kref_put(&irq->refcount, vgic_irq_release)) { | |
127 | spin_unlock(&dist->lpi_list_lock); | |
3802411d | 128 | return; |
2cccbb36 | 129 | }; |
3802411d | 130 | |
3802411d AP |
131 | list_del(&irq->lpi_list); |
132 | dist->lpi_list_count--; | |
133 | spin_unlock(&dist->lpi_list_lock); | |
134 | ||
135 | kfree(irq); | |
5dd4b924 AP |
136 | } |
137 | ||
81eeb95d CD |
138 | /** |
139 | * kvm_vgic_target_oracle - compute the target vcpu for an irq | |
140 | * | |
141 | * @irq: The irq to route. Must be already locked. | |
142 | * | |
143 | * Based on the current state of the interrupt (enabled, pending, | |
144 | * active, vcpu and target_vcpu), compute the next vcpu this should be | |
145 | * given to. Return NULL if this shouldn't be injected at all. | |
146 | * | |
147 | * Requires the IRQ lock to be held. | |
148 | */ | |
149 | static struct kvm_vcpu *vgic_target_oracle(struct vgic_irq *irq) | |
150 | { | |
151 | DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&irq->irq_lock)); | |
152 | ||
153 | /* If the interrupt is active, it must stay on the current vcpu */ | |
154 | if (irq->active) | |
155 | return irq->vcpu ? : irq->target_vcpu; | |
156 | ||
157 | /* | |
158 | * If the IRQ is not active but enabled and pending, we should direct | |
159 | * it to its configured target VCPU. | |
160 | * If the distributor is disabled, pending interrupts shouldn't be | |
161 | * forwarded. | |
162 | */ | |
163 | if (irq->enabled && irq->pending) { | |
164 | if (unlikely(irq->target_vcpu && | |
165 | !irq->target_vcpu->kvm->arch.vgic.enabled)) | |
166 | return NULL; | |
167 | ||
168 | return irq->target_vcpu; | |
169 | } | |
170 | ||
171 | /* If neither active nor pending and enabled, then this IRQ should not | |
172 | * be queued to any VCPU. | |
173 | */ | |
174 | return NULL; | |
175 | } | |
176 | ||
8e444745 CD |
177 | /* |
178 | * The order of items in the ap_lists defines how we'll pack things in LRs as | |
179 | * well, the first items in the list being the first things populated in the | |
180 | * LRs. | |
181 | * | |
182 | * A hard rule is that active interrupts can never be pushed out of the LRs | |
183 | * (and therefore take priority) since we cannot reliably trap on deactivation | |
184 | * of IRQs and therefore they have to be present in the LRs. | |
185 | * | |
186 | * Otherwise things should be sorted by the priority field and the GIC | |
187 | * hardware support will take care of preemption of priority groups etc. | |
188 | * | |
189 | * Return negative if "a" sorts before "b", 0 to preserve order, and positive | |
190 | * to sort "b" before "a". | |
191 | */ | |
192 | static int vgic_irq_cmp(void *priv, struct list_head *a, struct list_head *b) | |
193 | { | |
194 | struct vgic_irq *irqa = container_of(a, struct vgic_irq, ap_list); | |
195 | struct vgic_irq *irqb = container_of(b, struct vgic_irq, ap_list); | |
196 | bool penda, pendb; | |
197 | int ret; | |
198 | ||
199 | spin_lock(&irqa->irq_lock); | |
200 | spin_lock_nested(&irqb->irq_lock, SINGLE_DEPTH_NESTING); | |
201 | ||
202 | if (irqa->active || irqb->active) { | |
203 | ret = (int)irqb->active - (int)irqa->active; | |
204 | goto out; | |
205 | } | |
206 | ||
207 | penda = irqa->enabled && irqa->pending; | |
208 | pendb = irqb->enabled && irqb->pending; | |
209 | ||
210 | if (!penda || !pendb) { | |
211 | ret = (int)pendb - (int)penda; | |
212 | goto out; | |
213 | } | |
214 | ||
215 | /* Both pending and enabled, sort by priority */ | |
216 | ret = irqa->priority - irqb->priority; | |
217 | out: | |
218 | spin_unlock(&irqb->irq_lock); | |
219 | spin_unlock(&irqa->irq_lock); | |
220 | return ret; | |
221 | } | |
222 | ||
223 | /* Must be called with the ap_list_lock held */ | |
224 | static void vgic_sort_ap_list(struct kvm_vcpu *vcpu) | |
225 | { | |
226 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
227 | ||
228 | DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&vgic_cpu->ap_list_lock)); | |
229 | ||
230 | list_sort(NULL, &vgic_cpu->ap_list_head, vgic_irq_cmp); | |
231 | } | |
232 | ||
81eeb95d CD |
233 | /* |
234 | * Only valid injection if changing level for level-triggered IRQs or for a | |
235 | * rising edge. | |
236 | */ | |
237 | static bool vgic_validate_injection(struct vgic_irq *irq, bool level) | |
238 | { | |
239 | switch (irq->config) { | |
240 | case VGIC_CONFIG_LEVEL: | |
241 | return irq->line_level != level; | |
242 | case VGIC_CONFIG_EDGE: | |
243 | return level; | |
244 | } | |
245 | ||
246 | return false; | |
247 | } | |
248 | ||
249 | /* | |
250 | * Check whether an IRQ needs to (and can) be queued to a VCPU's ap list. | |
251 | * Do the queuing if necessary, taking the right locks in the right order. | |
252 | * Returns true when the IRQ was queued, false otherwise. | |
253 | * | |
254 | * Needs to be entered with the IRQ lock already held, but will return | |
255 | * with all locks dropped. | |
256 | */ | |
257 | bool vgic_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq) | |
258 | { | |
259 | struct kvm_vcpu *vcpu; | |
260 | ||
261 | DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&irq->irq_lock)); | |
262 | ||
263 | retry: | |
264 | vcpu = vgic_target_oracle(irq); | |
265 | if (irq->vcpu || !vcpu) { | |
266 | /* | |
267 | * If this IRQ is already on a VCPU's ap_list, then it | |
268 | * cannot be moved or modified and there is no more work for | |
269 | * us to do. | |
270 | * | |
271 | * Otherwise, if the irq is not pending and enabled, it does | |
272 | * not need to be inserted into an ap_list and there is also | |
273 | * no more work for us to do. | |
274 | */ | |
275 | spin_unlock(&irq->irq_lock); | |
d42c7970 SWL |
276 | |
277 | /* | |
278 | * We have to kick the VCPU here, because we could be | |
279 | * queueing an edge-triggered interrupt for which we | |
280 | * get no EOI maintenance interrupt. In that case, | |
281 | * while the IRQ is already on the VCPU's AP list, the | |
282 | * VCPU could have EOI'ed the original interrupt and | |
283 | * won't see this one until it exits for some other | |
284 | * reason. | |
285 | */ | |
286 | if (vcpu) | |
287 | kvm_vcpu_kick(vcpu); | |
81eeb95d CD |
288 | return false; |
289 | } | |
290 | ||
291 | /* | |
292 | * We must unlock the irq lock to take the ap_list_lock where | |
293 | * we are going to insert this new pending interrupt. | |
294 | */ | |
295 | spin_unlock(&irq->irq_lock); | |
296 | ||
297 | /* someone can do stuff here, which we re-check below */ | |
298 | ||
299 | spin_lock(&vcpu->arch.vgic_cpu.ap_list_lock); | |
300 | spin_lock(&irq->irq_lock); | |
301 | ||
302 | /* | |
303 | * Did something change behind our backs? | |
304 | * | |
305 | * There are two cases: | |
306 | * 1) The irq lost its pending state or was disabled behind our | |
307 | * backs and/or it was queued to another VCPU's ap_list. | |
308 | * 2) Someone changed the affinity on this irq behind our | |
309 | * backs and we are now holding the wrong ap_list_lock. | |
310 | * | |
311 | * In both cases, drop the locks and retry. | |
312 | */ | |
313 | ||
314 | if (unlikely(irq->vcpu || vcpu != vgic_target_oracle(irq))) { | |
315 | spin_unlock(&irq->irq_lock); | |
316 | spin_unlock(&vcpu->arch.vgic_cpu.ap_list_lock); | |
317 | ||
318 | spin_lock(&irq->irq_lock); | |
319 | goto retry; | |
320 | } | |
321 | ||
5dd4b924 AP |
322 | /* |
323 | * Grab a reference to the irq to reflect the fact that it is | |
324 | * now in the ap_list. | |
325 | */ | |
326 | vgic_get_irq_kref(irq); | |
81eeb95d CD |
327 | list_add_tail(&irq->ap_list, &vcpu->arch.vgic_cpu.ap_list_head); |
328 | irq->vcpu = vcpu; | |
329 | ||
330 | spin_unlock(&irq->irq_lock); | |
331 | spin_unlock(&vcpu->arch.vgic_cpu.ap_list_lock); | |
332 | ||
333 | kvm_vcpu_kick(vcpu); | |
334 | ||
335 | return true; | |
336 | } | |
337 | ||
338 | static int vgic_update_irq_pending(struct kvm *kvm, int cpuid, | |
339 | unsigned int intid, bool level, | |
340 | bool mapped_irq) | |
341 | { | |
342 | struct kvm_vcpu *vcpu; | |
343 | struct vgic_irq *irq; | |
344 | int ret; | |
345 | ||
346 | trace_vgic_update_irq_pending(cpuid, intid, level); | |
347 | ||
ad275b8b EA |
348 | ret = vgic_lazy_init(kvm); |
349 | if (ret) | |
350 | return ret; | |
351 | ||
81eeb95d CD |
352 | vcpu = kvm_get_vcpu(kvm, cpuid); |
353 | if (!vcpu && intid < VGIC_NR_PRIVATE_IRQS) | |
354 | return -EINVAL; | |
355 | ||
356 | irq = vgic_get_irq(kvm, vcpu, intid); | |
357 | if (!irq) | |
358 | return -EINVAL; | |
359 | ||
5dd4b924 AP |
360 | if (irq->hw != mapped_irq) { |
361 | vgic_put_irq(kvm, irq); | |
81eeb95d | 362 | return -EINVAL; |
5dd4b924 | 363 | } |
81eeb95d CD |
364 | |
365 | spin_lock(&irq->irq_lock); | |
366 | ||
367 | if (!vgic_validate_injection(irq, level)) { | |
368 | /* Nothing to see here, move along... */ | |
369 | spin_unlock(&irq->irq_lock); | |
5dd4b924 | 370 | vgic_put_irq(kvm, irq); |
81eeb95d CD |
371 | return 0; |
372 | } | |
373 | ||
374 | if (irq->config == VGIC_CONFIG_LEVEL) { | |
375 | irq->line_level = level; | |
376 | irq->pending = level || irq->soft_pending; | |
377 | } else { | |
378 | irq->pending = true; | |
379 | } | |
380 | ||
381 | vgic_queue_irq_unlock(kvm, irq); | |
5dd4b924 | 382 | vgic_put_irq(kvm, irq); |
81eeb95d CD |
383 | |
384 | return 0; | |
385 | } | |
386 | ||
387 | /** | |
388 | * kvm_vgic_inject_irq - Inject an IRQ from a device to the vgic | |
389 | * @kvm: The VM structure pointer | |
390 | * @cpuid: The CPU for PPIs | |
391 | * @intid: The INTID to inject a new state to. | |
392 | * @level: Edge-triggered: true: to trigger the interrupt | |
393 | * false: to ignore the call | |
394 | * Level-sensitive true: raise the input signal | |
395 | * false: lower the input signal | |
396 | * | |
397 | * The VGIC is not concerned with devices being active-LOW or active-HIGH for | |
398 | * level-sensitive interrupts. You can think of the level parameter as 1 | |
399 | * being HIGH and 0 being LOW and all devices being active-HIGH. | |
400 | */ | |
401 | int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int intid, | |
402 | bool level) | |
403 | { | |
404 | return vgic_update_irq_pending(kvm, cpuid, intid, level, false); | |
405 | } | |
0919e84c | 406 | |
568e8c90 AP |
407 | int kvm_vgic_inject_mapped_irq(struct kvm *kvm, int cpuid, unsigned int intid, |
408 | bool level) | |
409 | { | |
410 | return vgic_update_irq_pending(kvm, cpuid, intid, level, true); | |
411 | } | |
412 | ||
413 | int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, u32 virt_irq, u32 phys_irq) | |
414 | { | |
415 | struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, virt_irq); | |
416 | ||
417 | BUG_ON(!irq); | |
418 | ||
419 | spin_lock(&irq->irq_lock); | |
420 | ||
421 | irq->hw = true; | |
422 | irq->hwintid = phys_irq; | |
423 | ||
424 | spin_unlock(&irq->irq_lock); | |
5dd4b924 | 425 | vgic_put_irq(vcpu->kvm, irq); |
568e8c90 AP |
426 | |
427 | return 0; | |
428 | } | |
429 | ||
430 | int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, unsigned int virt_irq) | |
431 | { | |
5dd4b924 | 432 | struct vgic_irq *irq; |
568e8c90 AP |
433 | |
434 | if (!vgic_initialized(vcpu->kvm)) | |
435 | return -EAGAIN; | |
436 | ||
5dd4b924 AP |
437 | irq = vgic_get_irq(vcpu->kvm, vcpu, virt_irq); |
438 | BUG_ON(!irq); | |
439 | ||
568e8c90 AP |
440 | spin_lock(&irq->irq_lock); |
441 | ||
442 | irq->hw = false; | |
443 | irq->hwintid = 0; | |
444 | ||
445 | spin_unlock(&irq->irq_lock); | |
5dd4b924 | 446 | vgic_put_irq(vcpu->kvm, irq); |
568e8c90 AP |
447 | |
448 | return 0; | |
449 | } | |
450 | ||
0919e84c MZ |
451 | /** |
452 | * vgic_prune_ap_list - Remove non-relevant interrupts from the list | |
453 | * | |
454 | * @vcpu: The VCPU pointer | |
455 | * | |
456 | * Go over the list of "interesting" interrupts, and prune those that we | |
457 | * won't have to consider in the near future. | |
458 | */ | |
459 | static void vgic_prune_ap_list(struct kvm_vcpu *vcpu) | |
460 | { | |
461 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
462 | struct vgic_irq *irq, *tmp; | |
463 | ||
464 | retry: | |
465 | spin_lock(&vgic_cpu->ap_list_lock); | |
466 | ||
467 | list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) { | |
468 | struct kvm_vcpu *target_vcpu, *vcpuA, *vcpuB; | |
469 | ||
470 | spin_lock(&irq->irq_lock); | |
471 | ||
472 | BUG_ON(vcpu != irq->vcpu); | |
473 | ||
474 | target_vcpu = vgic_target_oracle(irq); | |
475 | ||
476 | if (!target_vcpu) { | |
477 | /* | |
478 | * We don't need to process this interrupt any | |
479 | * further, move it off the list. | |
480 | */ | |
481 | list_del(&irq->ap_list); | |
482 | irq->vcpu = NULL; | |
483 | spin_unlock(&irq->irq_lock); | |
5dd4b924 AP |
484 | |
485 | /* | |
486 | * This vgic_put_irq call matches the | |
487 | * vgic_get_irq_kref in vgic_queue_irq_unlock, | |
488 | * where we added the LPI to the ap_list. As | |
489 | * we remove the irq from the list, we drop | |
490 | * also drop the refcount. | |
491 | */ | |
492 | vgic_put_irq(vcpu->kvm, irq); | |
0919e84c MZ |
493 | continue; |
494 | } | |
495 | ||
496 | if (target_vcpu == vcpu) { | |
497 | /* We're on the right CPU */ | |
498 | spin_unlock(&irq->irq_lock); | |
499 | continue; | |
500 | } | |
501 | ||
502 | /* This interrupt looks like it has to be migrated. */ | |
503 | ||
504 | spin_unlock(&irq->irq_lock); | |
505 | spin_unlock(&vgic_cpu->ap_list_lock); | |
506 | ||
507 | /* | |
508 | * Ensure locking order by always locking the smallest | |
509 | * ID first. | |
510 | */ | |
511 | if (vcpu->vcpu_id < target_vcpu->vcpu_id) { | |
512 | vcpuA = vcpu; | |
513 | vcpuB = target_vcpu; | |
514 | } else { | |
515 | vcpuA = target_vcpu; | |
516 | vcpuB = vcpu; | |
517 | } | |
518 | ||
519 | spin_lock(&vcpuA->arch.vgic_cpu.ap_list_lock); | |
520 | spin_lock_nested(&vcpuB->arch.vgic_cpu.ap_list_lock, | |
521 | SINGLE_DEPTH_NESTING); | |
522 | spin_lock(&irq->irq_lock); | |
523 | ||
524 | /* | |
525 | * If the affinity has been preserved, move the | |
526 | * interrupt around. Otherwise, it means things have | |
527 | * changed while the interrupt was unlocked, and we | |
528 | * need to replay this. | |
529 | * | |
530 | * In all cases, we cannot trust the list not to have | |
531 | * changed, so we restart from the beginning. | |
532 | */ | |
533 | if (target_vcpu == vgic_target_oracle(irq)) { | |
534 | struct vgic_cpu *new_cpu = &target_vcpu->arch.vgic_cpu; | |
535 | ||
536 | list_del(&irq->ap_list); | |
537 | irq->vcpu = target_vcpu; | |
538 | list_add_tail(&irq->ap_list, &new_cpu->ap_list_head); | |
539 | } | |
540 | ||
541 | spin_unlock(&irq->irq_lock); | |
542 | spin_unlock(&vcpuB->arch.vgic_cpu.ap_list_lock); | |
543 | spin_unlock(&vcpuA->arch.vgic_cpu.ap_list_lock); | |
544 | goto retry; | |
545 | } | |
546 | ||
547 | spin_unlock(&vgic_cpu->ap_list_lock); | |
548 | } | |
549 | ||
550 | static inline void vgic_process_maintenance_interrupt(struct kvm_vcpu *vcpu) | |
551 | { | |
59529f69 MZ |
552 | if (kvm_vgic_global_state.type == VGIC_V2) |
553 | vgic_v2_process_maintenance(vcpu); | |
554 | else | |
555 | vgic_v3_process_maintenance(vcpu); | |
0919e84c MZ |
556 | } |
557 | ||
558 | static inline void vgic_fold_lr_state(struct kvm_vcpu *vcpu) | |
559 | { | |
59529f69 MZ |
560 | if (kvm_vgic_global_state.type == VGIC_V2) |
561 | vgic_v2_fold_lr_state(vcpu); | |
562 | else | |
563 | vgic_v3_fold_lr_state(vcpu); | |
0919e84c MZ |
564 | } |
565 | ||
566 | /* Requires the irq_lock to be held. */ | |
567 | static inline void vgic_populate_lr(struct kvm_vcpu *vcpu, | |
568 | struct vgic_irq *irq, int lr) | |
569 | { | |
570 | DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&irq->irq_lock)); | |
140b086d | 571 | |
59529f69 MZ |
572 | if (kvm_vgic_global_state.type == VGIC_V2) |
573 | vgic_v2_populate_lr(vcpu, irq, lr); | |
574 | else | |
575 | vgic_v3_populate_lr(vcpu, irq, lr); | |
0919e84c MZ |
576 | } |
577 | ||
578 | static inline void vgic_clear_lr(struct kvm_vcpu *vcpu, int lr) | |
579 | { | |
59529f69 MZ |
580 | if (kvm_vgic_global_state.type == VGIC_V2) |
581 | vgic_v2_clear_lr(vcpu, lr); | |
582 | else | |
583 | vgic_v3_clear_lr(vcpu, lr); | |
0919e84c MZ |
584 | } |
585 | ||
586 | static inline void vgic_set_underflow(struct kvm_vcpu *vcpu) | |
587 | { | |
59529f69 MZ |
588 | if (kvm_vgic_global_state.type == VGIC_V2) |
589 | vgic_v2_set_underflow(vcpu); | |
590 | else | |
591 | vgic_v3_set_underflow(vcpu); | |
0919e84c MZ |
592 | } |
593 | ||
594 | /* Requires the ap_list_lock to be held. */ | |
595 | static int compute_ap_list_depth(struct kvm_vcpu *vcpu) | |
596 | { | |
597 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
598 | struct vgic_irq *irq; | |
599 | int count = 0; | |
600 | ||
601 | DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&vgic_cpu->ap_list_lock)); | |
602 | ||
603 | list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) { | |
604 | spin_lock(&irq->irq_lock); | |
605 | /* GICv2 SGIs can count for more than one... */ | |
606 | if (vgic_irq_is_sgi(irq->intid) && irq->source) | |
607 | count += hweight8(irq->source); | |
608 | else | |
609 | count++; | |
610 | spin_unlock(&irq->irq_lock); | |
611 | } | |
612 | return count; | |
613 | } | |
614 | ||
615 | /* Requires the VCPU's ap_list_lock to be held. */ | |
616 | static void vgic_flush_lr_state(struct kvm_vcpu *vcpu) | |
617 | { | |
618 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
619 | struct vgic_irq *irq; | |
620 | int count = 0; | |
621 | ||
622 | DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&vgic_cpu->ap_list_lock)); | |
623 | ||
624 | if (compute_ap_list_depth(vcpu) > kvm_vgic_global_state.nr_lr) { | |
625 | vgic_set_underflow(vcpu); | |
626 | vgic_sort_ap_list(vcpu); | |
627 | } | |
628 | ||
629 | list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) { | |
630 | spin_lock(&irq->irq_lock); | |
631 | ||
632 | if (unlikely(vgic_target_oracle(irq) != vcpu)) | |
633 | goto next; | |
634 | ||
635 | /* | |
636 | * If we get an SGI with multiple sources, try to get | |
637 | * them in all at once. | |
638 | */ | |
639 | do { | |
640 | vgic_populate_lr(vcpu, irq, count++); | |
641 | } while (irq->source && count < kvm_vgic_global_state.nr_lr); | |
642 | ||
643 | next: | |
644 | spin_unlock(&irq->irq_lock); | |
645 | ||
646 | if (count == kvm_vgic_global_state.nr_lr) | |
647 | break; | |
648 | } | |
649 | ||
650 | vcpu->arch.vgic_cpu.used_lrs = count; | |
651 | ||
652 | /* Nuke remaining LRs */ | |
653 | for ( ; count < kvm_vgic_global_state.nr_lr; count++) | |
654 | vgic_clear_lr(vcpu, count); | |
655 | } | |
656 | ||
657 | /* Sync back the hardware VGIC state into our emulation after a guest's run. */ | |
658 | void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu) | |
659 | { | |
0099b770 CD |
660 | if (unlikely(!vgic_initialized(vcpu->kvm))) |
661 | return; | |
662 | ||
0919e84c MZ |
663 | vgic_process_maintenance_interrupt(vcpu); |
664 | vgic_fold_lr_state(vcpu); | |
665 | vgic_prune_ap_list(vcpu); | |
666 | } | |
667 | ||
668 | /* Flush our emulation state into the GIC hardware before entering the guest. */ | |
669 | void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu) | |
670 | { | |
0099b770 CD |
671 | if (unlikely(!vgic_initialized(vcpu->kvm))) |
672 | return; | |
673 | ||
0919e84c MZ |
674 | spin_lock(&vcpu->arch.vgic_cpu.ap_list_lock); |
675 | vgic_flush_lr_state(vcpu); | |
676 | spin_unlock(&vcpu->arch.vgic_cpu.ap_list_lock); | |
677 | } | |
90eee56c EA |
678 | |
679 | int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu) | |
680 | { | |
681 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
682 | struct vgic_irq *irq; | |
683 | bool pending = false; | |
684 | ||
685 | if (!vcpu->kvm->arch.vgic.enabled) | |
686 | return false; | |
687 | ||
688 | spin_lock(&vgic_cpu->ap_list_lock); | |
689 | ||
690 | list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) { | |
691 | spin_lock(&irq->irq_lock); | |
692 | pending = irq->pending && irq->enabled; | |
693 | spin_unlock(&irq->irq_lock); | |
694 | ||
695 | if (pending) | |
696 | break; | |
697 | } | |
698 | ||
699 | spin_unlock(&vgic_cpu->ap_list_lock); | |
700 | ||
701 | return pending; | |
702 | } | |
2b0cda87 MZ |
703 | |
704 | void vgic_kick_vcpus(struct kvm *kvm) | |
705 | { | |
706 | struct kvm_vcpu *vcpu; | |
707 | int c; | |
708 | ||
709 | /* | |
710 | * We've injected an interrupt, time to find out who deserves | |
711 | * a good kick... | |
712 | */ | |
713 | kvm_for_each_vcpu(c, vcpu, kvm) { | |
714 | if (kvm_vgic_vcpu_pending_irq(vcpu)) | |
715 | kvm_vcpu_kick(vcpu); | |
716 | } | |
717 | } | |
568e8c90 AP |
718 | |
719 | bool kvm_vgic_map_is_active(struct kvm_vcpu *vcpu, unsigned int virt_irq) | |
720 | { | |
721 | struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, virt_irq); | |
722 | bool map_is_active; | |
723 | ||
724 | spin_lock(&irq->irq_lock); | |
725 | map_is_active = irq->hw && irq->active; | |
726 | spin_unlock(&irq->irq_lock); | |
5dd4b924 | 727 | vgic_put_irq(vcpu->kvm, irq); |
568e8c90 AP |
728 | |
729 | return map_is_active; | |
730 | } | |
0e4e82f1 | 731 |