2 * VGIC MMIO handling functions
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
14 #include <linux/bitops.h>
15 #include <linux/bsearch.h>
16 #include <linux/kvm.h>
17 #include <linux/kvm_host.h>
18 #include <kvm/iodev.h>
19 #include <kvm/arm_vgic.h>
22 #include "vgic-mmio.h"
24 unsigned long vgic_mmio_read_raz(struct kvm_vcpu
*vcpu
,
25 gpa_t addr
, unsigned int len
)
30 unsigned long vgic_mmio_read_rao(struct kvm_vcpu
*vcpu
,
31 gpa_t addr
, unsigned int len
)
36 void vgic_mmio_write_wi(struct kvm_vcpu
*vcpu
, gpa_t addr
,
37 unsigned int len
, unsigned long val
)
43 * Read accesses to both GICD_ICENABLER and GICD_ISENABLER return the value
44 * of the enabled bit, so there is only one function for both here.
46 unsigned long vgic_mmio_read_enable(struct kvm_vcpu
*vcpu
,
47 gpa_t addr
, unsigned int len
)
49 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 1);
53 /* Loop over all IRQs affected by this read */
54 for (i
= 0; i
< len
* 8; i
++) {
55 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
60 vgic_put_irq(vcpu
->kvm
, irq
);
66 void vgic_mmio_write_senable(struct kvm_vcpu
*vcpu
,
67 gpa_t addr
, unsigned int len
,
70 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 1);
73 for_each_set_bit(i
, &val
, len
* 8) {
74 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
76 spin_lock(&irq
->irq_lock
);
78 vgic_queue_irq_unlock(vcpu
->kvm
, irq
);
80 vgic_put_irq(vcpu
->kvm
, irq
);
84 void vgic_mmio_write_cenable(struct kvm_vcpu
*vcpu
,
85 gpa_t addr
, unsigned int len
,
88 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 1);
91 for_each_set_bit(i
, &val
, len
* 8) {
92 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
94 spin_lock(&irq
->irq_lock
);
98 spin_unlock(&irq
->irq_lock
);
99 vgic_put_irq(vcpu
->kvm
, irq
);
103 unsigned long vgic_mmio_read_pending(struct kvm_vcpu
*vcpu
,
104 gpa_t addr
, unsigned int len
)
106 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 1);
110 /* Loop over all IRQs affected by this read */
111 for (i
= 0; i
< len
* 8; i
++) {
112 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
114 if (irq_is_pending(irq
))
117 vgic_put_irq(vcpu
->kvm
, irq
);
123 void vgic_mmio_write_spending(struct kvm_vcpu
*vcpu
,
124 gpa_t addr
, unsigned int len
,
127 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 1);
130 for_each_set_bit(i
, &val
, len
* 8) {
131 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
133 spin_lock(&irq
->irq_lock
);
134 irq
->pending_latch
= true;
136 vgic_queue_irq_unlock(vcpu
->kvm
, irq
);
137 vgic_put_irq(vcpu
->kvm
, irq
);
141 void vgic_mmio_write_cpending(struct kvm_vcpu
*vcpu
,
142 gpa_t addr
, unsigned int len
,
145 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 1);
148 for_each_set_bit(i
, &val
, len
* 8) {
149 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
151 spin_lock(&irq
->irq_lock
);
153 irq
->pending_latch
= false;
155 spin_unlock(&irq
->irq_lock
);
156 vgic_put_irq(vcpu
->kvm
, irq
);
160 unsigned long vgic_mmio_read_active(struct kvm_vcpu
*vcpu
,
161 gpa_t addr
, unsigned int len
)
163 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 1);
167 /* Loop over all IRQs affected by this read */
168 for (i
= 0; i
< len
* 8; i
++) {
169 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
174 vgic_put_irq(vcpu
->kvm
, irq
);
180 static void vgic_mmio_change_active(struct kvm_vcpu
*vcpu
, struct vgic_irq
*irq
,
181 bool new_active_state
)
183 struct kvm_vcpu
*requester_vcpu
;
184 spin_lock(&irq
->irq_lock
);
187 * The vcpu parameter here can mean multiple things depending on how
188 * this function is called; when handling a trap from the kernel it
189 * depends on the GIC version, and these functions are also called as
190 * part of save/restore from userspace.
192 * Therefore, we have to figure out the requester in a reliable way.
194 * When accessing VGIC state from user space, the requester_vcpu is
195 * NULL, which is fine, because we guarantee that no VCPUs are running
196 * when accessing VGIC state from user space so irq->vcpu->cpu is
199 requester_vcpu
= kvm_arm_get_running_vcpu();
202 * If this virtual IRQ was written into a list register, we
203 * have to make sure the CPU that runs the VCPU thread has
204 * synced back the LR state to the struct vgic_irq.
206 * As long as the conditions below are true, we know the VCPU thread
207 * may be on its way back from the guest (we kicked the VCPU thread in
208 * vgic_change_active_prepare) and still has to sync back this IRQ,
209 * so we release and re-acquire the spin_lock to let the other thread
212 while (irq
->vcpu
&& /* IRQ may have state in an LR somewhere */
213 irq
->vcpu
!= requester_vcpu
&& /* Current thread is not the VCPU thread */
214 irq
->vcpu
->cpu
!= -1) /* VCPU thread is running */
215 cond_resched_lock(&irq
->irq_lock
);
217 irq
->active
= new_active_state
;
218 if (new_active_state
)
219 vgic_queue_irq_unlock(vcpu
->kvm
, irq
);
221 spin_unlock(&irq
->irq_lock
);
225 * If we are fiddling with an IRQ's active state, we have to make sure the IRQ
226 * is not queued on some running VCPU's LRs, because then the change to the
227 * active state can be overwritten when the VCPU's state is synced coming back
230 * For shared interrupts, we have to stop all the VCPUs because interrupts can
231 * be migrated while we don't hold the IRQ locks and we don't want to be
232 * chasing moving targets.
234 * For private interrupts we don't have to do anything because userspace
235 * accesses to the VGIC state already require all VCPUs to be stopped, and
236 * only the VCPU itself can modify its private interrupts active state, which
237 * guarantees that the VCPU is not running.
239 static void vgic_change_active_prepare(struct kvm_vcpu
*vcpu
, u32 intid
)
241 if (intid
> VGIC_NR_PRIVATE_IRQS
)
242 kvm_arm_halt_guest(vcpu
->kvm
);
245 /* See vgic_change_active_prepare */
246 static void vgic_change_active_finish(struct kvm_vcpu
*vcpu
, u32 intid
)
248 if (intid
> VGIC_NR_PRIVATE_IRQS
)
249 kvm_arm_resume_guest(vcpu
->kvm
);
252 static void __vgic_mmio_write_cactive(struct kvm_vcpu
*vcpu
,
253 gpa_t addr
, unsigned int len
,
256 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 1);
259 for_each_set_bit(i
, &val
, len
* 8) {
260 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
261 vgic_mmio_change_active(vcpu
, irq
, false);
262 vgic_put_irq(vcpu
->kvm
, irq
);
266 void vgic_mmio_write_cactive(struct kvm_vcpu
*vcpu
,
267 gpa_t addr
, unsigned int len
,
270 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 1);
272 mutex_lock(&vcpu
->kvm
->lock
);
273 vgic_change_active_prepare(vcpu
, intid
);
275 __vgic_mmio_write_cactive(vcpu
, addr
, len
, val
);
277 vgic_change_active_finish(vcpu
, intid
);
278 mutex_unlock(&vcpu
->kvm
->lock
);
281 void vgic_mmio_uaccess_write_cactive(struct kvm_vcpu
*vcpu
,
282 gpa_t addr
, unsigned int len
,
285 __vgic_mmio_write_cactive(vcpu
, addr
, len
, val
);
288 static void __vgic_mmio_write_sactive(struct kvm_vcpu
*vcpu
,
289 gpa_t addr
, unsigned int len
,
292 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 1);
295 for_each_set_bit(i
, &val
, len
* 8) {
296 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
297 vgic_mmio_change_active(vcpu
, irq
, true);
298 vgic_put_irq(vcpu
->kvm
, irq
);
302 void vgic_mmio_write_sactive(struct kvm_vcpu
*vcpu
,
303 gpa_t addr
, unsigned int len
,
306 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 1);
308 mutex_lock(&vcpu
->kvm
->lock
);
309 vgic_change_active_prepare(vcpu
, intid
);
311 __vgic_mmio_write_sactive(vcpu
, addr
, len
, val
);
313 vgic_change_active_finish(vcpu
, intid
);
314 mutex_unlock(&vcpu
->kvm
->lock
);
317 void vgic_mmio_uaccess_write_sactive(struct kvm_vcpu
*vcpu
,
318 gpa_t addr
, unsigned int len
,
321 __vgic_mmio_write_sactive(vcpu
, addr
, len
, val
);
324 unsigned long vgic_mmio_read_priority(struct kvm_vcpu
*vcpu
,
325 gpa_t addr
, unsigned int len
)
327 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 8);
331 for (i
= 0; i
< len
; i
++) {
332 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
334 val
|= (u64
)irq
->priority
<< (i
* 8);
336 vgic_put_irq(vcpu
->kvm
, irq
);
343 * We currently don't handle changing the priority of an interrupt that
344 * is already pending on a VCPU. If there is a need for this, we would
345 * need to make this VCPU exit and re-evaluate the priorities, potentially
346 * leading to this interrupt getting presented now to the guest (if it has
347 * been masked by the priority mask before).
349 void vgic_mmio_write_priority(struct kvm_vcpu
*vcpu
,
350 gpa_t addr
, unsigned int len
,
353 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 8);
356 for (i
= 0; i
< len
; i
++) {
357 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
359 spin_lock(&irq
->irq_lock
);
360 /* Narrow the priority range to what we actually support */
361 irq
->priority
= (val
>> (i
* 8)) & GENMASK(7, 8 - VGIC_PRI_BITS
);
362 spin_unlock(&irq
->irq_lock
);
364 vgic_put_irq(vcpu
->kvm
, irq
);
368 unsigned long vgic_mmio_read_config(struct kvm_vcpu
*vcpu
,
369 gpa_t addr
, unsigned int len
)
371 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 2);
375 for (i
= 0; i
< len
* 4; i
++) {
376 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
378 if (irq
->config
== VGIC_CONFIG_EDGE
)
379 value
|= (2U << (i
* 2));
381 vgic_put_irq(vcpu
->kvm
, irq
);
387 void vgic_mmio_write_config(struct kvm_vcpu
*vcpu
,
388 gpa_t addr
, unsigned int len
,
391 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 2);
394 for (i
= 0; i
< len
* 4; i
++) {
395 struct vgic_irq
*irq
;
398 * The configuration cannot be changed for SGIs in general,
399 * for PPIs this is IMPLEMENTATION DEFINED. The arch timer
400 * code relies on PPIs being level triggered, so we also
401 * make them read-only here.
403 if (intid
+ i
< VGIC_NR_PRIVATE_IRQS
)
406 irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
407 spin_lock(&irq
->irq_lock
);
409 if (test_bit(i
* 2 + 1, &val
))
410 irq
->config
= VGIC_CONFIG_EDGE
;
412 irq
->config
= VGIC_CONFIG_LEVEL
;
414 spin_unlock(&irq
->irq_lock
);
415 vgic_put_irq(vcpu
->kvm
, irq
);
419 u64
vgic_read_irq_line_level_info(struct kvm_vcpu
*vcpu
, u32 intid
)
423 int nr_irqs
= vcpu
->kvm
->arch
.vgic
.nr_spis
+ VGIC_NR_PRIVATE_IRQS
;
425 for (i
= 0; i
< 32; i
++) {
426 struct vgic_irq
*irq
;
428 if ((intid
+ i
) < VGIC_NR_SGIS
|| (intid
+ i
) >= nr_irqs
)
431 irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
432 if (irq
->config
== VGIC_CONFIG_LEVEL
&& irq
->line_level
)
435 vgic_put_irq(vcpu
->kvm
, irq
);
441 void vgic_write_irq_line_level_info(struct kvm_vcpu
*vcpu
, u32 intid
,
445 int nr_irqs
= vcpu
->kvm
->arch
.vgic
.nr_spis
+ VGIC_NR_PRIVATE_IRQS
;
447 for (i
= 0; i
< 32; i
++) {
448 struct vgic_irq
*irq
;
451 if ((intid
+ i
) < VGIC_NR_SGIS
|| (intid
+ i
) >= nr_irqs
)
454 irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
457 * Line level is set irrespective of irq type
458 * (level or edge) to avoid dependency that VM should
459 * restore irq config before line level.
461 new_level
= !!(val
& (1U << i
));
462 spin_lock(&irq
->irq_lock
);
463 irq
->line_level
= new_level
;
465 vgic_queue_irq_unlock(vcpu
->kvm
, irq
);
467 spin_unlock(&irq
->irq_lock
);
469 vgic_put_irq(vcpu
->kvm
, irq
);
473 static int match_region(const void *key
, const void *elt
)
475 const unsigned int offset
= (unsigned long)key
;
476 const struct vgic_register_region
*region
= elt
;
478 if (offset
< region
->reg_offset
)
481 if (offset
>= region
->reg_offset
+ region
->len
)
487 const struct vgic_register_region
*
488 vgic_find_mmio_region(const struct vgic_register_region
*regions
,
489 int nr_regions
, unsigned int offset
)
491 return bsearch((void *)(uintptr_t)offset
, regions
, nr_regions
,
492 sizeof(regions
[0]), match_region
);
495 void vgic_set_vmcr(struct kvm_vcpu
*vcpu
, struct vgic_vmcr
*vmcr
)
497 if (kvm_vgic_global_state
.type
== VGIC_V2
)
498 vgic_v2_set_vmcr(vcpu
, vmcr
);
500 vgic_v3_set_vmcr(vcpu
, vmcr
);
503 void vgic_get_vmcr(struct kvm_vcpu
*vcpu
, struct vgic_vmcr
*vmcr
)
505 if (kvm_vgic_global_state
.type
== VGIC_V2
)
506 vgic_v2_get_vmcr(vcpu
, vmcr
);
508 vgic_v3_get_vmcr(vcpu
, vmcr
);
512 * kvm_mmio_read_buf() returns a value in a format where it can be converted
513 * to a byte array and be directly observed as the guest wanted it to appear
514 * in memory if it had done the store itself, which is LE for the GIC, as the
515 * guest knows the GIC is always LE.
517 * We convert this value to the CPUs native format to deal with it as a data
520 unsigned long vgic_data_mmio_bus_to_host(const void *val
, unsigned int len
)
522 unsigned long data
= kvm_mmio_read_buf(val
, len
);
528 return le16_to_cpu(data
);
530 return le32_to_cpu(data
);
532 return le64_to_cpu(data
);
537 * kvm_mmio_write_buf() expects a value in a format such that if converted to
538 * a byte array it is observed as the guest would see it if it could perform
539 * the load directly. Since the GIC is LE, and the guest knows this, the
540 * guest expects a value in little endian format.
542 * We convert the data value from the CPUs native format to LE so that the
543 * value is returned in the proper format.
545 void vgic_data_host_to_mmio_bus(void *buf
, unsigned int len
,
552 data
= cpu_to_le16(data
);
555 data
= cpu_to_le32(data
);
558 data
= cpu_to_le64(data
);
561 kvm_mmio_write_buf(buf
, len
, data
);
565 struct vgic_io_device
*kvm_to_vgic_iodev(const struct kvm_io_device
*dev
)
567 return container_of(dev
, struct vgic_io_device
, dev
);
570 static bool check_region(const struct kvm
*kvm
,
571 const struct vgic_register_region
*region
,
574 int flags
, nr_irqs
= kvm
->arch
.vgic
.nr_spis
+ VGIC_NR_PRIVATE_IRQS
;
578 flags
= VGIC_ACCESS_8bit
;
581 flags
= VGIC_ACCESS_32bit
;
584 flags
= VGIC_ACCESS_64bit
;
590 if ((region
->access_flags
& flags
) && IS_ALIGNED(addr
, len
)) {
591 if (!region
->bits_per_irq
)
594 /* Do we access a non-allocated IRQ? */
595 return VGIC_ADDR_TO_INTID(addr
, region
->bits_per_irq
) < nr_irqs
;
601 const struct vgic_register_region
*
602 vgic_get_mmio_region(struct kvm_vcpu
*vcpu
, struct vgic_io_device
*iodev
,
605 const struct vgic_register_region
*region
;
607 region
= vgic_find_mmio_region(iodev
->regions
, iodev
->nr_regions
,
608 addr
- iodev
->base_addr
);
609 if (!region
|| !check_region(vcpu
->kvm
, region
, addr
, len
))
615 static int vgic_uaccess_read(struct kvm_vcpu
*vcpu
, struct kvm_io_device
*dev
,
616 gpa_t addr
, u32
*val
)
618 struct vgic_io_device
*iodev
= kvm_to_vgic_iodev(dev
);
619 const struct vgic_register_region
*region
;
620 struct kvm_vcpu
*r_vcpu
;
622 region
= vgic_get_mmio_region(vcpu
, iodev
, addr
, sizeof(u32
));
628 r_vcpu
= iodev
->redist_vcpu
? iodev
->redist_vcpu
: vcpu
;
629 if (region
->uaccess_read
)
630 *val
= region
->uaccess_read(r_vcpu
, addr
, sizeof(u32
));
632 *val
= region
->read(r_vcpu
, addr
, sizeof(u32
));
637 static int vgic_uaccess_write(struct kvm_vcpu
*vcpu
, struct kvm_io_device
*dev
,
638 gpa_t addr
, const u32
*val
)
640 struct vgic_io_device
*iodev
= kvm_to_vgic_iodev(dev
);
641 const struct vgic_register_region
*region
;
642 struct kvm_vcpu
*r_vcpu
;
644 region
= vgic_get_mmio_region(vcpu
, iodev
, addr
, sizeof(u32
));
648 r_vcpu
= iodev
->redist_vcpu
? iodev
->redist_vcpu
: vcpu
;
649 if (region
->uaccess_write
)
650 region
->uaccess_write(r_vcpu
, addr
, sizeof(u32
), *val
);
652 region
->write(r_vcpu
, addr
, sizeof(u32
), *val
);
658 * Userland access to VGIC registers.
660 int vgic_uaccess(struct kvm_vcpu
*vcpu
, struct vgic_io_device
*dev
,
661 bool is_write
, int offset
, u32
*val
)
664 return vgic_uaccess_write(vcpu
, &dev
->dev
, offset
, val
);
666 return vgic_uaccess_read(vcpu
, &dev
->dev
, offset
, val
);
669 static int dispatch_mmio_read(struct kvm_vcpu
*vcpu
, struct kvm_io_device
*dev
,
670 gpa_t addr
, int len
, void *val
)
672 struct vgic_io_device
*iodev
= kvm_to_vgic_iodev(dev
);
673 const struct vgic_register_region
*region
;
674 unsigned long data
= 0;
676 region
= vgic_get_mmio_region(vcpu
, iodev
, addr
, len
);
682 switch (iodev
->iodev_type
) {
684 data
= region
->read(vcpu
, addr
, len
);
687 data
= region
->read(vcpu
, addr
, len
);
690 data
= region
->read(iodev
->redist_vcpu
, addr
, len
);
693 data
= region
->its_read(vcpu
->kvm
, iodev
->its
, addr
, len
);
697 vgic_data_host_to_mmio_bus(val
, len
, data
);
701 static int dispatch_mmio_write(struct kvm_vcpu
*vcpu
, struct kvm_io_device
*dev
,
702 gpa_t addr
, int len
, const void *val
)
704 struct vgic_io_device
*iodev
= kvm_to_vgic_iodev(dev
);
705 const struct vgic_register_region
*region
;
706 unsigned long data
= vgic_data_mmio_bus_to_host(val
, len
);
708 region
= vgic_get_mmio_region(vcpu
, iodev
, addr
, len
);
712 switch (iodev
->iodev_type
) {
714 region
->write(vcpu
, addr
, len
, data
);
717 region
->write(vcpu
, addr
, len
, data
);
720 region
->write(iodev
->redist_vcpu
, addr
, len
, data
);
723 region
->its_write(vcpu
->kvm
, iodev
->its
, addr
, len
, data
);
730 struct kvm_io_device_ops kvm_io_gic_ops
= {
731 .read
= dispatch_mmio_read
,
732 .write
= dispatch_mmio_write
,
735 int vgic_register_dist_iodev(struct kvm
*kvm
, gpa_t dist_base_address
,
738 struct vgic_io_device
*io_device
= &kvm
->arch
.vgic
.dist_iodev
;
744 len
= vgic_v2_init_dist_iodev(io_device
);
747 len
= vgic_v3_init_dist_iodev(io_device
);
753 io_device
->base_addr
= dist_base_address
;
754 io_device
->iodev_type
= IODEV_DIST
;
755 io_device
->redist_vcpu
= NULL
;
757 mutex_lock(&kvm
->slots_lock
);
758 ret
= kvm_io_bus_register_dev(kvm
, KVM_MMIO_BUS
, dist_base_address
,
759 len
, &io_device
->dev
);
760 mutex_unlock(&kvm
->slots_lock
);