]>
Commit | Line | Data |
---|---|---|
64a959d6 CD |
1 | /* |
2 | * Copyright (C) 2015, 2016 ARM Ltd. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License version 2 as | |
6 | * published by the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, | |
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
11 | * GNU General Public License for more details. | |
12 | * | |
13 | * You should have received a copy of the GNU General Public License | |
14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
15 | */ | |
16 | ||
17 | #include <linux/kvm.h> | |
18 | #include <linux/kvm_host.h> | |
8e444745 | 19 | #include <linux/list_sort.h> |
64a959d6 CD |
20 | |
21 | #include "vgic.h" | |
22 | ||
81eeb95d CD |
23 | #define CREATE_TRACE_POINTS |
24 | #include "../trace.h" | |
25 | ||
26 | #ifdef CONFIG_DEBUG_SPINLOCK | |
27 | #define DEBUG_SPINLOCK_BUG_ON(p) BUG_ON(p) | |
28 | #else | |
29 | #define DEBUG_SPINLOCK_BUG_ON(p) | |
30 | #endif | |
31 | ||
5a7a8426 | 32 | struct vgic_global __section(.hyp.text) kvm_vgic_global_state = {.gicv3_cpuif = STATIC_KEY_FALSE_INIT,}; |
64a959d6 | 33 | |
81eeb95d CD |
34 | /* |
35 | * Locking order is always: | |
424c3383 AP |
36 | * its->cmd_lock (mutex) |
37 | * its->its_lock (mutex) | |
38 | * vgic_cpu->ap_list_lock | |
3802411d AP |
39 | * kvm->lpi_list_lock |
40 | * vgic_irq->irq_lock | |
81eeb95d | 41 | * |
424c3383 AP |
42 | * If you need to take multiple locks, always take the upper lock first, |
43 | * then the lower ones, e.g. first take the its_lock, then the irq_lock. | |
44 | * If you are already holding a lock and need to take a higher one, you | |
45 | * have to drop the lower ranking lock first and re-aquire it after having | |
46 | * taken the upper one. | |
81eeb95d CD |
47 | * |
48 | * When taking more than one ap_list_lock at the same time, always take the | |
49 | * lowest numbered VCPU's ap_list_lock first, so: | |
50 | * vcpuX->vcpu_id < vcpuY->vcpu_id: | |
51 | * spin_lock(vcpuX->arch.vgic_cpu.ap_list_lock); | |
52 | * spin_lock(vcpuY->arch.vgic_cpu.ap_list_lock); | |
53 | */ | |
54 | ||
3802411d AP |
55 | /* |
56 | * Iterate over the VM's list of mapped LPIs to find the one with a | |
57 | * matching interrupt ID and return a reference to the IRQ structure. | |
58 | */ | |
59 | static struct vgic_irq *vgic_get_lpi(struct kvm *kvm, u32 intid) | |
60 | { | |
61 | struct vgic_dist *dist = &kvm->arch.vgic; | |
62 | struct vgic_irq *irq = NULL; | |
63 | ||
64 | spin_lock(&dist->lpi_list_lock); | |
65 | ||
66 | list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) { | |
67 | if (irq->intid != intid) | |
68 | continue; | |
69 | ||
70 | /* | |
71 | * This increases the refcount, the caller is expected to | |
72 | * call vgic_put_irq() later once it's finished with the IRQ. | |
73 | */ | |
d97594e6 | 74 | vgic_get_irq_kref(irq); |
3802411d AP |
75 | goto out_unlock; |
76 | } | |
77 | irq = NULL; | |
78 | ||
79 | out_unlock: | |
80 | spin_unlock(&dist->lpi_list_lock); | |
81 | ||
82 | return irq; | |
83 | } | |
84 | ||
85 | /* | |
86 | * This looks up the virtual interrupt ID to get the corresponding | |
87 | * struct vgic_irq. It also increases the refcount, so any caller is expected | |
88 | * to call vgic_put_irq() once it's finished with this IRQ. | |
89 | */ | |
64a959d6 CD |
90 | struct vgic_irq *vgic_get_irq(struct kvm *kvm, struct kvm_vcpu *vcpu, |
91 | u32 intid) | |
92 | { | |
93 | /* SGIs and PPIs */ | |
94 | if (intid <= VGIC_MAX_PRIVATE) | |
95 | return &vcpu->arch.vgic_cpu.private_irqs[intid]; | |
96 | ||
97 | /* SPIs */ | |
98 | if (intid <= VGIC_MAX_SPI) | |
99 | return &kvm->arch.vgic.spis[intid - VGIC_NR_PRIVATE_IRQS]; | |
100 | ||
3802411d | 101 | /* LPIs */ |
64a959d6 | 102 | if (intid >= VGIC_MIN_LPI) |
3802411d | 103 | return vgic_get_lpi(kvm, intid); |
64a959d6 CD |
104 | |
105 | WARN(1, "Looking up struct vgic_irq for reserved INTID"); | |
106 | return NULL; | |
107 | } | |
81eeb95d | 108 | |
3802411d AP |
109 | /* |
110 | * We can't do anything in here, because we lack the kvm pointer to | |
111 | * lock and remove the item from the lpi_list. So we keep this function | |
112 | * empty and use the return value of kref_put() to trigger the freeing. | |
113 | */ | |
5dd4b924 AP |
114 | static void vgic_irq_release(struct kref *ref) |
115 | { | |
5dd4b924 AP |
116 | } |
117 | ||
118 | void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq) | |
119 | { | |
2cccbb36 | 120 | struct vgic_dist *dist = &kvm->arch.vgic; |
3802411d | 121 | |
5dd4b924 AP |
122 | if (irq->intid < VGIC_MIN_LPI) |
123 | return; | |
124 | ||
2cccbb36 CD |
125 | spin_lock(&dist->lpi_list_lock); |
126 | if (!kref_put(&irq->refcount, vgic_irq_release)) { | |
127 | spin_unlock(&dist->lpi_list_lock); | |
3802411d | 128 | return; |
2cccbb36 | 129 | }; |
3802411d | 130 | |
3802411d AP |
131 | list_del(&irq->lpi_list); |
132 | dist->lpi_list_count--; | |
133 | spin_unlock(&dist->lpi_list_lock); | |
134 | ||
135 | kfree(irq); | |
5dd4b924 AP |
136 | } |
137 | ||
81eeb95d CD |
138 | /** |
139 | * kvm_vgic_target_oracle - compute the target vcpu for an irq | |
140 | * | |
141 | * @irq: The irq to route. Must be already locked. | |
142 | * | |
143 | * Based on the current state of the interrupt (enabled, pending, | |
144 | * active, vcpu and target_vcpu), compute the next vcpu this should be | |
145 | * given to. Return NULL if this shouldn't be injected at all. | |
146 | * | |
147 | * Requires the IRQ lock to be held. | |
148 | */ | |
149 | static struct kvm_vcpu *vgic_target_oracle(struct vgic_irq *irq) | |
150 | { | |
151 | DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&irq->irq_lock)); | |
152 | ||
153 | /* If the interrupt is active, it must stay on the current vcpu */ | |
154 | if (irq->active) | |
155 | return irq->vcpu ? : irq->target_vcpu; | |
156 | ||
157 | /* | |
158 | * If the IRQ is not active but enabled and pending, we should direct | |
159 | * it to its configured target VCPU. | |
160 | * If the distributor is disabled, pending interrupts shouldn't be | |
161 | * forwarded. | |
162 | */ | |
8694e4da | 163 | if (irq->enabled && irq_is_pending(irq)) { |
81eeb95d CD |
164 | if (unlikely(irq->target_vcpu && |
165 | !irq->target_vcpu->kvm->arch.vgic.enabled)) | |
166 | return NULL; | |
167 | ||
168 | return irq->target_vcpu; | |
169 | } | |
170 | ||
171 | /* If neither active nor pending and enabled, then this IRQ should not | |
172 | * be queued to any VCPU. | |
173 | */ | |
174 | return NULL; | |
175 | } | |
176 | ||
8e444745 CD |
177 | /* |
178 | * The order of items in the ap_lists defines how we'll pack things in LRs as | |
179 | * well, the first items in the list being the first things populated in the | |
180 | * LRs. | |
181 | * | |
182 | * A hard rule is that active interrupts can never be pushed out of the LRs | |
183 | * (and therefore take priority) since we cannot reliably trap on deactivation | |
184 | * of IRQs and therefore they have to be present in the LRs. | |
185 | * | |
186 | * Otherwise things should be sorted by the priority field and the GIC | |
187 | * hardware support will take care of preemption of priority groups etc. | |
188 | * | |
189 | * Return negative if "a" sorts before "b", 0 to preserve order, and positive | |
190 | * to sort "b" before "a". | |
191 | */ | |
192 | static int vgic_irq_cmp(void *priv, struct list_head *a, struct list_head *b) | |
193 | { | |
194 | struct vgic_irq *irqa = container_of(a, struct vgic_irq, ap_list); | |
195 | struct vgic_irq *irqb = container_of(b, struct vgic_irq, ap_list); | |
196 | bool penda, pendb; | |
197 | int ret; | |
198 | ||
199 | spin_lock(&irqa->irq_lock); | |
200 | spin_lock_nested(&irqb->irq_lock, SINGLE_DEPTH_NESTING); | |
201 | ||
202 | if (irqa->active || irqb->active) { | |
203 | ret = (int)irqb->active - (int)irqa->active; | |
204 | goto out; | |
205 | } | |
206 | ||
8694e4da CD |
207 | penda = irqa->enabled && irq_is_pending(irqa); |
208 | pendb = irqb->enabled && irq_is_pending(irqb); | |
8e444745 CD |
209 | |
210 | if (!penda || !pendb) { | |
211 | ret = (int)pendb - (int)penda; | |
212 | goto out; | |
213 | } | |
214 | ||
215 | /* Both pending and enabled, sort by priority */ | |
216 | ret = irqa->priority - irqb->priority; | |
217 | out: | |
218 | spin_unlock(&irqb->irq_lock); | |
219 | spin_unlock(&irqa->irq_lock); | |
220 | return ret; | |
221 | } | |
222 | ||
223 | /* Must be called with the ap_list_lock held */ | |
224 | static void vgic_sort_ap_list(struct kvm_vcpu *vcpu) | |
225 | { | |
226 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
227 | ||
228 | DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&vgic_cpu->ap_list_lock)); | |
229 | ||
230 | list_sort(NULL, &vgic_cpu->ap_list_head, vgic_irq_cmp); | |
231 | } | |
232 | ||
81eeb95d CD |
233 | /* |
234 | * Only valid injection if changing level for level-triggered IRQs or for a | |
235 | * rising edge. | |
236 | */ | |
237 | static bool vgic_validate_injection(struct vgic_irq *irq, bool level) | |
238 | { | |
239 | switch (irq->config) { | |
240 | case VGIC_CONFIG_LEVEL: | |
241 | return irq->line_level != level; | |
242 | case VGIC_CONFIG_EDGE: | |
243 | return level; | |
244 | } | |
245 | ||
246 | return false; | |
247 | } | |
248 | ||
249 | /* | |
250 | * Check whether an IRQ needs to (and can) be queued to a VCPU's ap list. | |
251 | * Do the queuing if necessary, taking the right locks in the right order. | |
252 | * Returns true when the IRQ was queued, false otherwise. | |
253 | * | |
254 | * Needs to be entered with the IRQ lock already held, but will return | |
255 | * with all locks dropped. | |
256 | */ | |
257 | bool vgic_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq) | |
258 | { | |
259 | struct kvm_vcpu *vcpu; | |
260 | ||
261 | DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&irq->irq_lock)); | |
262 | ||
263 | retry: | |
264 | vcpu = vgic_target_oracle(irq); | |
265 | if (irq->vcpu || !vcpu) { | |
266 | /* | |
267 | * If this IRQ is already on a VCPU's ap_list, then it | |
268 | * cannot be moved or modified and there is no more work for | |
269 | * us to do. | |
270 | * | |
271 | * Otherwise, if the irq is not pending and enabled, it does | |
272 | * not need to be inserted into an ap_list and there is also | |
273 | * no more work for us to do. | |
274 | */ | |
275 | spin_unlock(&irq->irq_lock); | |
d42c7970 SWL |
276 | |
277 | /* | |
278 | * We have to kick the VCPU here, because we could be | |
279 | * queueing an edge-triggered interrupt for which we | |
280 | * get no EOI maintenance interrupt. In that case, | |
281 | * while the IRQ is already on the VCPU's AP list, the | |
282 | * VCPU could have EOI'ed the original interrupt and | |
283 | * won't see this one until it exits for some other | |
284 | * reason. | |
285 | */ | |
286 | if (vcpu) | |
287 | kvm_vcpu_kick(vcpu); | |
81eeb95d CD |
288 | return false; |
289 | } | |
290 | ||
291 | /* | |
292 | * We must unlock the irq lock to take the ap_list_lock where | |
293 | * we are going to insert this new pending interrupt. | |
294 | */ | |
295 | spin_unlock(&irq->irq_lock); | |
296 | ||
297 | /* someone can do stuff here, which we re-check below */ | |
298 | ||
299 | spin_lock(&vcpu->arch.vgic_cpu.ap_list_lock); | |
300 | spin_lock(&irq->irq_lock); | |
301 | ||
302 | /* | |
303 | * Did something change behind our backs? | |
304 | * | |
305 | * There are two cases: | |
306 | * 1) The irq lost its pending state or was disabled behind our | |
307 | * backs and/or it was queued to another VCPU's ap_list. | |
308 | * 2) Someone changed the affinity on this irq behind our | |
309 | * backs and we are now holding the wrong ap_list_lock. | |
310 | * | |
311 | * In both cases, drop the locks and retry. | |
312 | */ | |
313 | ||
314 | if (unlikely(irq->vcpu || vcpu != vgic_target_oracle(irq))) { | |
315 | spin_unlock(&irq->irq_lock); | |
316 | spin_unlock(&vcpu->arch.vgic_cpu.ap_list_lock); | |
317 | ||
318 | spin_lock(&irq->irq_lock); | |
319 | goto retry; | |
320 | } | |
321 | ||
5dd4b924 AP |
322 | /* |
323 | * Grab a reference to the irq to reflect the fact that it is | |
324 | * now in the ap_list. | |
325 | */ | |
326 | vgic_get_irq_kref(irq); | |
81eeb95d CD |
327 | list_add_tail(&irq->ap_list, &vcpu->arch.vgic_cpu.ap_list_head); |
328 | irq->vcpu = vcpu; | |
329 | ||
330 | spin_unlock(&irq->irq_lock); | |
331 | spin_unlock(&vcpu->arch.vgic_cpu.ap_list_lock); | |
332 | ||
333 | kvm_vcpu_kick(vcpu); | |
334 | ||
335 | return true; | |
336 | } | |
337 | ||
11710dec CD |
338 | /** |
339 | * kvm_vgic_inject_irq - Inject an IRQ from a device to the vgic | |
340 | * @kvm: The VM structure pointer | |
341 | * @cpuid: The CPU for PPIs | |
342 | * @intid: The INTID to inject a new state to. | |
343 | * @level: Edge-triggered: true: to trigger the interrupt | |
344 | * false: to ignore the call | |
345 | * Level-sensitive true: raise the input signal | |
346 | * false: lower the input signal | |
347 | * | |
348 | * The VGIC is not concerned with devices being active-LOW or active-HIGH for | |
349 | * level-sensitive interrupts. You can think of the level parameter as 1 | |
350 | * being HIGH and 0 being LOW and all devices being active-HIGH. | |
351 | */ | |
352 | int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int intid, | |
353 | bool level) | |
81eeb95d CD |
354 | { |
355 | struct kvm_vcpu *vcpu; | |
356 | struct vgic_irq *irq; | |
357 | int ret; | |
358 | ||
359 | trace_vgic_update_irq_pending(cpuid, intid, level); | |
360 | ||
ad275b8b EA |
361 | ret = vgic_lazy_init(kvm); |
362 | if (ret) | |
363 | return ret; | |
364 | ||
81eeb95d CD |
365 | vcpu = kvm_get_vcpu(kvm, cpuid); |
366 | if (!vcpu && intid < VGIC_NR_PRIVATE_IRQS) | |
367 | return -EINVAL; | |
368 | ||
369 | irq = vgic_get_irq(kvm, vcpu, intid); | |
370 | if (!irq) | |
371 | return -EINVAL; | |
372 | ||
81eeb95d CD |
373 | spin_lock(&irq->irq_lock); |
374 | ||
375 | if (!vgic_validate_injection(irq, level)) { | |
376 | /* Nothing to see here, move along... */ | |
377 | spin_unlock(&irq->irq_lock); | |
5dd4b924 | 378 | vgic_put_irq(kvm, irq); |
81eeb95d CD |
379 | return 0; |
380 | } | |
381 | ||
8694e4da | 382 | if (irq->config == VGIC_CONFIG_LEVEL) |
81eeb95d | 383 | irq->line_level = level; |
8694e4da CD |
384 | else |
385 | irq->pending_latch = true; | |
81eeb95d CD |
386 | |
387 | vgic_queue_irq_unlock(kvm, irq); | |
5dd4b924 | 388 | vgic_put_irq(kvm, irq); |
81eeb95d CD |
389 | |
390 | return 0; | |
391 | } | |
392 | ||
568e8c90 AP |
393 | int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, u32 virt_irq, u32 phys_irq) |
394 | { | |
395 | struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, virt_irq); | |
396 | ||
397 | BUG_ON(!irq); | |
398 | ||
399 | spin_lock(&irq->irq_lock); | |
400 | ||
401 | irq->hw = true; | |
402 | irq->hwintid = phys_irq; | |
403 | ||
404 | spin_unlock(&irq->irq_lock); | |
5dd4b924 | 405 | vgic_put_irq(vcpu->kvm, irq); |
568e8c90 AP |
406 | |
407 | return 0; | |
408 | } | |
409 | ||
410 | int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, unsigned int virt_irq) | |
411 | { | |
5dd4b924 | 412 | struct vgic_irq *irq; |
568e8c90 AP |
413 | |
414 | if (!vgic_initialized(vcpu->kvm)) | |
415 | return -EAGAIN; | |
416 | ||
5dd4b924 AP |
417 | irq = vgic_get_irq(vcpu->kvm, vcpu, virt_irq); |
418 | BUG_ON(!irq); | |
419 | ||
568e8c90 AP |
420 | spin_lock(&irq->irq_lock); |
421 | ||
422 | irq->hw = false; | |
423 | irq->hwintid = 0; | |
424 | ||
425 | spin_unlock(&irq->irq_lock); | |
5dd4b924 | 426 | vgic_put_irq(vcpu->kvm, irq); |
568e8c90 AP |
427 | |
428 | return 0; | |
429 | } | |
430 | ||
0919e84c MZ |
431 | /** |
432 | * vgic_prune_ap_list - Remove non-relevant interrupts from the list | |
433 | * | |
434 | * @vcpu: The VCPU pointer | |
435 | * | |
436 | * Go over the list of "interesting" interrupts, and prune those that we | |
437 | * won't have to consider in the near future. | |
438 | */ | |
439 | static void vgic_prune_ap_list(struct kvm_vcpu *vcpu) | |
440 | { | |
441 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
442 | struct vgic_irq *irq, *tmp; | |
443 | ||
444 | retry: | |
445 | spin_lock(&vgic_cpu->ap_list_lock); | |
446 | ||
447 | list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) { | |
448 | struct kvm_vcpu *target_vcpu, *vcpuA, *vcpuB; | |
449 | ||
450 | spin_lock(&irq->irq_lock); | |
451 | ||
452 | BUG_ON(vcpu != irq->vcpu); | |
453 | ||
454 | target_vcpu = vgic_target_oracle(irq); | |
455 | ||
456 | if (!target_vcpu) { | |
457 | /* | |
458 | * We don't need to process this interrupt any | |
459 | * further, move it off the list. | |
460 | */ | |
461 | list_del(&irq->ap_list); | |
462 | irq->vcpu = NULL; | |
463 | spin_unlock(&irq->irq_lock); | |
5dd4b924 AP |
464 | |
465 | /* | |
466 | * This vgic_put_irq call matches the | |
467 | * vgic_get_irq_kref in vgic_queue_irq_unlock, | |
468 | * where we added the LPI to the ap_list. As | |
469 | * we remove the irq from the list, we drop | |
470 | * also drop the refcount. | |
471 | */ | |
472 | vgic_put_irq(vcpu->kvm, irq); | |
0919e84c MZ |
473 | continue; |
474 | } | |
475 | ||
476 | if (target_vcpu == vcpu) { | |
477 | /* We're on the right CPU */ | |
478 | spin_unlock(&irq->irq_lock); | |
479 | continue; | |
480 | } | |
481 | ||
482 | /* This interrupt looks like it has to be migrated. */ | |
483 | ||
484 | spin_unlock(&irq->irq_lock); | |
485 | spin_unlock(&vgic_cpu->ap_list_lock); | |
486 | ||
487 | /* | |
488 | * Ensure locking order by always locking the smallest | |
489 | * ID first. | |
490 | */ | |
491 | if (vcpu->vcpu_id < target_vcpu->vcpu_id) { | |
492 | vcpuA = vcpu; | |
493 | vcpuB = target_vcpu; | |
494 | } else { | |
495 | vcpuA = target_vcpu; | |
496 | vcpuB = vcpu; | |
497 | } | |
498 | ||
499 | spin_lock(&vcpuA->arch.vgic_cpu.ap_list_lock); | |
500 | spin_lock_nested(&vcpuB->arch.vgic_cpu.ap_list_lock, | |
501 | SINGLE_DEPTH_NESTING); | |
502 | spin_lock(&irq->irq_lock); | |
503 | ||
504 | /* | |
505 | * If the affinity has been preserved, move the | |
506 | * interrupt around. Otherwise, it means things have | |
507 | * changed while the interrupt was unlocked, and we | |
508 | * need to replay this. | |
509 | * | |
510 | * In all cases, we cannot trust the list not to have | |
511 | * changed, so we restart from the beginning. | |
512 | */ | |
513 | if (target_vcpu == vgic_target_oracle(irq)) { | |
514 | struct vgic_cpu *new_cpu = &target_vcpu->arch.vgic_cpu; | |
515 | ||
516 | list_del(&irq->ap_list); | |
517 | irq->vcpu = target_vcpu; | |
518 | list_add_tail(&irq->ap_list, &new_cpu->ap_list_head); | |
519 | } | |
520 | ||
521 | spin_unlock(&irq->irq_lock); | |
522 | spin_unlock(&vcpuB->arch.vgic_cpu.ap_list_lock); | |
523 | spin_unlock(&vcpuA->arch.vgic_cpu.ap_list_lock); | |
524 | goto retry; | |
525 | } | |
526 | ||
527 | spin_unlock(&vgic_cpu->ap_list_lock); | |
528 | } | |
529 | ||
530 | static inline void vgic_process_maintenance_interrupt(struct kvm_vcpu *vcpu) | |
531 | { | |
59529f69 MZ |
532 | if (kvm_vgic_global_state.type == VGIC_V2) |
533 | vgic_v2_process_maintenance(vcpu); | |
534 | else | |
535 | vgic_v3_process_maintenance(vcpu); | |
0919e84c MZ |
536 | } |
537 | ||
538 | static inline void vgic_fold_lr_state(struct kvm_vcpu *vcpu) | |
539 | { | |
59529f69 MZ |
540 | if (kvm_vgic_global_state.type == VGIC_V2) |
541 | vgic_v2_fold_lr_state(vcpu); | |
542 | else | |
543 | vgic_v3_fold_lr_state(vcpu); | |
0919e84c MZ |
544 | } |
545 | ||
546 | /* Requires the irq_lock to be held. */ | |
547 | static inline void vgic_populate_lr(struct kvm_vcpu *vcpu, | |
548 | struct vgic_irq *irq, int lr) | |
549 | { | |
550 | DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&irq->irq_lock)); | |
140b086d | 551 | |
59529f69 MZ |
552 | if (kvm_vgic_global_state.type == VGIC_V2) |
553 | vgic_v2_populate_lr(vcpu, irq, lr); | |
554 | else | |
555 | vgic_v3_populate_lr(vcpu, irq, lr); | |
0919e84c MZ |
556 | } |
557 | ||
558 | static inline void vgic_clear_lr(struct kvm_vcpu *vcpu, int lr) | |
559 | { | |
59529f69 MZ |
560 | if (kvm_vgic_global_state.type == VGIC_V2) |
561 | vgic_v2_clear_lr(vcpu, lr); | |
562 | else | |
563 | vgic_v3_clear_lr(vcpu, lr); | |
0919e84c MZ |
564 | } |
565 | ||
566 | static inline void vgic_set_underflow(struct kvm_vcpu *vcpu) | |
567 | { | |
59529f69 MZ |
568 | if (kvm_vgic_global_state.type == VGIC_V2) |
569 | vgic_v2_set_underflow(vcpu); | |
570 | else | |
571 | vgic_v3_set_underflow(vcpu); | |
0919e84c MZ |
572 | } |
573 | ||
574 | /* Requires the ap_list_lock to be held. */ | |
575 | static int compute_ap_list_depth(struct kvm_vcpu *vcpu) | |
576 | { | |
577 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
578 | struct vgic_irq *irq; | |
579 | int count = 0; | |
580 | ||
581 | DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&vgic_cpu->ap_list_lock)); | |
582 | ||
583 | list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) { | |
584 | spin_lock(&irq->irq_lock); | |
585 | /* GICv2 SGIs can count for more than one... */ | |
586 | if (vgic_irq_is_sgi(irq->intid) && irq->source) | |
587 | count += hweight8(irq->source); | |
588 | else | |
589 | count++; | |
590 | spin_unlock(&irq->irq_lock); | |
591 | } | |
592 | return count; | |
593 | } | |
594 | ||
595 | /* Requires the VCPU's ap_list_lock to be held. */ | |
596 | static void vgic_flush_lr_state(struct kvm_vcpu *vcpu) | |
597 | { | |
598 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
599 | struct vgic_irq *irq; | |
600 | int count = 0; | |
601 | ||
602 | DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&vgic_cpu->ap_list_lock)); | |
603 | ||
604 | if (compute_ap_list_depth(vcpu) > kvm_vgic_global_state.nr_lr) { | |
605 | vgic_set_underflow(vcpu); | |
606 | vgic_sort_ap_list(vcpu); | |
607 | } | |
608 | ||
609 | list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) { | |
610 | spin_lock(&irq->irq_lock); | |
611 | ||
612 | if (unlikely(vgic_target_oracle(irq) != vcpu)) | |
613 | goto next; | |
614 | ||
615 | /* | |
616 | * If we get an SGI with multiple sources, try to get | |
617 | * them in all at once. | |
618 | */ | |
619 | do { | |
620 | vgic_populate_lr(vcpu, irq, count++); | |
621 | } while (irq->source && count < kvm_vgic_global_state.nr_lr); | |
622 | ||
623 | next: | |
624 | spin_unlock(&irq->irq_lock); | |
625 | ||
626 | if (count == kvm_vgic_global_state.nr_lr) | |
627 | break; | |
628 | } | |
629 | ||
630 | vcpu->arch.vgic_cpu.used_lrs = count; | |
631 | ||
632 | /* Nuke remaining LRs */ | |
633 | for ( ; count < kvm_vgic_global_state.nr_lr; count++) | |
634 | vgic_clear_lr(vcpu, count); | |
635 | } | |
636 | ||
637 | /* Sync back the hardware VGIC state into our emulation after a guest's run. */ | |
638 | void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu) | |
639 | { | |
0099b770 CD |
640 | if (unlikely(!vgic_initialized(vcpu->kvm))) |
641 | return; | |
642 | ||
0919e84c MZ |
643 | vgic_process_maintenance_interrupt(vcpu); |
644 | vgic_fold_lr_state(vcpu); | |
645 | vgic_prune_ap_list(vcpu); | |
646 | } | |
647 | ||
648 | /* Flush our emulation state into the GIC hardware before entering the guest. */ | |
649 | void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu) | |
650 | { | |
0099b770 CD |
651 | if (unlikely(!vgic_initialized(vcpu->kvm))) |
652 | return; | |
653 | ||
0919e84c MZ |
654 | spin_lock(&vcpu->arch.vgic_cpu.ap_list_lock); |
655 | vgic_flush_lr_state(vcpu); | |
656 | spin_unlock(&vcpu->arch.vgic_cpu.ap_list_lock); | |
657 | } | |
90eee56c EA |
658 | |
659 | int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu) | |
660 | { | |
661 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
662 | struct vgic_irq *irq; | |
663 | bool pending = false; | |
664 | ||
665 | if (!vcpu->kvm->arch.vgic.enabled) | |
666 | return false; | |
667 | ||
668 | spin_lock(&vgic_cpu->ap_list_lock); | |
669 | ||
670 | list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) { | |
671 | spin_lock(&irq->irq_lock); | |
8694e4da | 672 | pending = irq_is_pending(irq) && irq->enabled; |
90eee56c EA |
673 | spin_unlock(&irq->irq_lock); |
674 | ||
675 | if (pending) | |
676 | break; | |
677 | } | |
678 | ||
679 | spin_unlock(&vgic_cpu->ap_list_lock); | |
680 | ||
681 | return pending; | |
682 | } | |
2b0cda87 MZ |
683 | |
684 | void vgic_kick_vcpus(struct kvm *kvm) | |
685 | { | |
686 | struct kvm_vcpu *vcpu; | |
687 | int c; | |
688 | ||
689 | /* | |
690 | * We've injected an interrupt, time to find out who deserves | |
691 | * a good kick... | |
692 | */ | |
693 | kvm_for_each_vcpu(c, vcpu, kvm) { | |
694 | if (kvm_vgic_vcpu_pending_irq(vcpu)) | |
695 | kvm_vcpu_kick(vcpu); | |
696 | } | |
697 | } | |
568e8c90 AP |
698 | |
699 | bool kvm_vgic_map_is_active(struct kvm_vcpu *vcpu, unsigned int virt_irq) | |
700 | { | |
701 | struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, virt_irq); | |
702 | bool map_is_active; | |
703 | ||
704 | spin_lock(&irq->irq_lock); | |
705 | map_is_active = irq->hw && irq->active; | |
706 | spin_unlock(&irq->irq_lock); | |
5dd4b924 | 707 | vgic_put_irq(vcpu->kvm, irq); |
568e8c90 AP |
708 | |
709 | return map_is_active; | |
710 | } | |
0e4e82f1 | 711 |