2 * VGIC MMIO handling functions
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
14 #include <linux/bitops.h>
15 #include <linux/bsearch.h>
16 #include <linux/kvm.h>
17 #include <linux/kvm_host.h>
18 #include <kvm/iodev.h>
19 #include <kvm/arm_vgic.h>
22 #include "vgic-mmio.h"
24 unsigned long vgic_mmio_read_raz(struct kvm_vcpu
*vcpu
,
25 gpa_t addr
, unsigned int len
)
30 unsigned long vgic_mmio_read_rao(struct kvm_vcpu
*vcpu
,
31 gpa_t addr
, unsigned int len
)
36 void vgic_mmio_write_wi(struct kvm_vcpu
*vcpu
, gpa_t addr
,
37 unsigned int len
, unsigned long val
)
43 * Read accesses to both GICD_ICENABLER and GICD_ISENABLER return the value
44 * of the enabled bit, so there is only one function for both here.
46 unsigned long vgic_mmio_read_enable(struct kvm_vcpu
*vcpu
,
47 gpa_t addr
, unsigned int len
)
49 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 1);
53 /* Loop over all IRQs affected by this read */
54 for (i
= 0; i
< len
* 8; i
++) {
55 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
60 vgic_put_irq(vcpu
->kvm
, irq
);
66 void vgic_mmio_write_senable(struct kvm_vcpu
*vcpu
,
67 gpa_t addr
, unsigned int len
,
70 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 1);
73 for_each_set_bit(i
, &val
, len
* 8) {
74 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
76 spin_lock(&irq
->irq_lock
);
78 vgic_queue_irq_unlock(vcpu
->kvm
, irq
);
80 vgic_put_irq(vcpu
->kvm
, irq
);
84 void vgic_mmio_write_cenable(struct kvm_vcpu
*vcpu
,
85 gpa_t addr
, unsigned int len
,
88 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 1);
91 for_each_set_bit(i
, &val
, len
* 8) {
92 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
94 spin_lock(&irq
->irq_lock
);
98 spin_unlock(&irq
->irq_lock
);
99 vgic_put_irq(vcpu
->kvm
, irq
);
103 unsigned long vgic_mmio_read_pending(struct kvm_vcpu
*vcpu
,
104 gpa_t addr
, unsigned int len
)
106 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 1);
110 /* Loop over all IRQs affected by this read */
111 for (i
= 0; i
< len
* 8; i
++) {
112 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
114 if (irq_is_pending(irq
))
117 vgic_put_irq(vcpu
->kvm
, irq
);
123 void vgic_mmio_write_spending(struct kvm_vcpu
*vcpu
,
124 gpa_t addr
, unsigned int len
,
127 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 1);
130 for_each_set_bit(i
, &val
, len
* 8) {
131 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
133 spin_lock(&irq
->irq_lock
);
134 irq
->pending_latch
= true;
136 vgic_queue_irq_unlock(vcpu
->kvm
, irq
);
137 vgic_put_irq(vcpu
->kvm
, irq
);
141 void vgic_mmio_write_cpending(struct kvm_vcpu
*vcpu
,
142 gpa_t addr
, unsigned int len
,
145 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 1);
148 for_each_set_bit(i
, &val
, len
* 8) {
149 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
151 spin_lock(&irq
->irq_lock
);
153 irq
->pending_latch
= false;
155 spin_unlock(&irq
->irq_lock
);
156 vgic_put_irq(vcpu
->kvm
, irq
);
160 unsigned long vgic_mmio_read_active(struct kvm_vcpu
*vcpu
,
161 gpa_t addr
, unsigned int len
)
163 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 1);
167 /* Loop over all IRQs affected by this read */
168 for (i
= 0; i
< len
* 8; i
++) {
169 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
174 vgic_put_irq(vcpu
->kvm
, irq
);
180 static void vgic_mmio_change_active(struct kvm_vcpu
*vcpu
, struct vgic_irq
*irq
,
181 bool new_active_state
)
183 struct kvm_vcpu
*requester_vcpu
;
184 spin_lock(&irq
->irq_lock
);
187 * The vcpu parameter here can mean multiple things depending on how
188 * this function is called; when handling a trap from the kernel it
189 * depends on the GIC version, and these functions are also called as
190 * part of save/restore from userspace.
192 * Therefore, we have to figure out the requester in a reliable way.
194 * When accessing VGIC state from user space, the requester_vcpu is
195 * NULL, which is fine, because we guarantee that no VCPUs are running
196 * when accessing VGIC state from user space so irq->vcpu->cpu is
199 requester_vcpu
= kvm_arm_get_running_vcpu();
202 * If this virtual IRQ was written into a list register, we
203 * have to make sure the CPU that runs the VCPU thread has
204 * synced back the LR state to the struct vgic_irq.
206 * As long as the conditions below are true, we know the VCPU thread
207 * may be on its way back from the guest (we kicked the VCPU thread in
208 * vgic_change_active_prepare) and still has to sync back this IRQ,
209 * so we release and re-acquire the spin_lock to let the other thread
212 while (irq
->vcpu
&& /* IRQ may have state in an LR somewhere */
213 irq
->vcpu
!= requester_vcpu
&& /* Current thread is not the VCPU thread */
214 irq
->vcpu
->cpu
!= -1) /* VCPU thread is running */
215 cond_resched_lock(&irq
->irq_lock
);
217 irq
->active
= new_active_state
;
218 if (new_active_state
)
219 vgic_queue_irq_unlock(vcpu
->kvm
, irq
);
221 spin_unlock(&irq
->irq_lock
);
225 * If we are fiddling with an IRQ's active state, we have to make sure the IRQ
226 * is not queued on some running VCPU's LRs, because then the change to the
227 * active state can be overwritten when the VCPU's state is synced coming back
230 * For shared interrupts, we have to stop all the VCPUs because interrupts can
231 * be migrated while we don't hold the IRQ locks and we don't want to be
232 * chasing moving targets.
234 * For private interrupts, we only have to make sure the single and only VCPU
235 * that can potentially queue the IRQ is stopped.
237 static void vgic_change_active_prepare(struct kvm_vcpu
*vcpu
, u32 intid
)
239 if (intid
< VGIC_NR_PRIVATE_IRQS
)
240 kvm_arm_halt_vcpu(vcpu
);
242 kvm_arm_halt_guest(vcpu
->kvm
);
245 /* See vgic_change_active_prepare */
246 static void vgic_change_active_finish(struct kvm_vcpu
*vcpu
, u32 intid
)
248 if (intid
< VGIC_NR_PRIVATE_IRQS
)
249 kvm_arm_resume_vcpu(vcpu
);
251 kvm_arm_resume_guest(vcpu
->kvm
);
254 void vgic_mmio_write_cactive(struct kvm_vcpu
*vcpu
,
255 gpa_t addr
, unsigned int len
,
258 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 1);
261 vgic_change_active_prepare(vcpu
, intid
);
262 for_each_set_bit(i
, &val
, len
* 8) {
263 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
264 vgic_mmio_change_active(vcpu
, irq
, false);
265 vgic_put_irq(vcpu
->kvm
, irq
);
267 vgic_change_active_finish(vcpu
, intid
);
270 void vgic_mmio_write_sactive(struct kvm_vcpu
*vcpu
,
271 gpa_t addr
, unsigned int len
,
274 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 1);
277 vgic_change_active_prepare(vcpu
, intid
);
278 for_each_set_bit(i
, &val
, len
* 8) {
279 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
280 vgic_mmio_change_active(vcpu
, irq
, true);
281 vgic_put_irq(vcpu
->kvm
, irq
);
283 vgic_change_active_finish(vcpu
, intid
);
286 unsigned long vgic_mmio_read_priority(struct kvm_vcpu
*vcpu
,
287 gpa_t addr
, unsigned int len
)
289 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 8);
293 for (i
= 0; i
< len
; i
++) {
294 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
296 val
|= (u64
)irq
->priority
<< (i
* 8);
298 vgic_put_irq(vcpu
->kvm
, irq
);
305 * We currently don't handle changing the priority of an interrupt that
306 * is already pending on a VCPU. If there is a need for this, we would
307 * need to make this VCPU exit and re-evaluate the priorities, potentially
308 * leading to this interrupt getting presented now to the guest (if it has
309 * been masked by the priority mask before).
311 void vgic_mmio_write_priority(struct kvm_vcpu
*vcpu
,
312 gpa_t addr
, unsigned int len
,
315 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 8);
318 for (i
= 0; i
< len
; i
++) {
319 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
321 spin_lock(&irq
->irq_lock
);
322 /* Narrow the priority range to what we actually support */
323 irq
->priority
= (val
>> (i
* 8)) & GENMASK(7, 8 - VGIC_PRI_BITS
);
324 spin_unlock(&irq
->irq_lock
);
326 vgic_put_irq(vcpu
->kvm
, irq
);
330 unsigned long vgic_mmio_read_config(struct kvm_vcpu
*vcpu
,
331 gpa_t addr
, unsigned int len
)
333 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 2);
337 for (i
= 0; i
< len
* 4; i
++) {
338 struct vgic_irq
*irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
340 if (irq
->config
== VGIC_CONFIG_EDGE
)
341 value
|= (2U << (i
* 2));
343 vgic_put_irq(vcpu
->kvm
, irq
);
349 void vgic_mmio_write_config(struct kvm_vcpu
*vcpu
,
350 gpa_t addr
, unsigned int len
,
353 u32 intid
= VGIC_ADDR_TO_INTID(addr
, 2);
356 for (i
= 0; i
< len
* 4; i
++) {
357 struct vgic_irq
*irq
;
360 * The configuration cannot be changed for SGIs in general,
361 * for PPIs this is IMPLEMENTATION DEFINED. The arch timer
362 * code relies on PPIs being level triggered, so we also
363 * make them read-only here.
365 if (intid
+ i
< VGIC_NR_PRIVATE_IRQS
)
368 irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
369 spin_lock(&irq
->irq_lock
);
371 if (test_bit(i
* 2 + 1, &val
))
372 irq
->config
= VGIC_CONFIG_EDGE
;
374 irq
->config
= VGIC_CONFIG_LEVEL
;
376 spin_unlock(&irq
->irq_lock
);
377 vgic_put_irq(vcpu
->kvm
, irq
);
381 u64
vgic_read_irq_line_level_info(struct kvm_vcpu
*vcpu
, u32 intid
)
385 int nr_irqs
= vcpu
->kvm
->arch
.vgic
.nr_spis
+ VGIC_NR_PRIVATE_IRQS
;
387 for (i
= 0; i
< 32; i
++) {
388 struct vgic_irq
*irq
;
390 if ((intid
+ i
) < VGIC_NR_SGIS
|| (intid
+ i
) >= nr_irqs
)
393 irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
394 if (irq
->config
== VGIC_CONFIG_LEVEL
&& irq
->line_level
)
397 vgic_put_irq(vcpu
->kvm
, irq
);
403 void vgic_write_irq_line_level_info(struct kvm_vcpu
*vcpu
, u32 intid
,
407 int nr_irqs
= vcpu
->kvm
->arch
.vgic
.nr_spis
+ VGIC_NR_PRIVATE_IRQS
;
409 for (i
= 0; i
< 32; i
++) {
410 struct vgic_irq
*irq
;
413 if ((intid
+ i
) < VGIC_NR_SGIS
|| (intid
+ i
) >= nr_irqs
)
416 irq
= vgic_get_irq(vcpu
->kvm
, vcpu
, intid
+ i
);
419 * Line level is set irrespective of irq type
420 * (level or edge) to avoid dependency that VM should
421 * restore irq config before line level.
423 new_level
= !!(val
& (1U << i
));
424 spin_lock(&irq
->irq_lock
);
425 irq
->line_level
= new_level
;
427 vgic_queue_irq_unlock(vcpu
->kvm
, irq
);
429 spin_unlock(&irq
->irq_lock
);
431 vgic_put_irq(vcpu
->kvm
, irq
);
435 static int match_region(const void *key
, const void *elt
)
437 const unsigned int offset
= (unsigned long)key
;
438 const struct vgic_register_region
*region
= elt
;
440 if (offset
< region
->reg_offset
)
443 if (offset
>= region
->reg_offset
+ region
->len
)
449 /* Find the proper register handler entry given a certain address offset. */
450 static const struct vgic_register_region
*
451 vgic_find_mmio_region(const struct vgic_register_region
*region
, int nr_regions
,
454 return bsearch((void *)(uintptr_t)offset
, region
, nr_regions
,
455 sizeof(region
[0]), match_region
);
458 void vgic_set_vmcr(struct kvm_vcpu
*vcpu
, struct vgic_vmcr
*vmcr
)
460 if (kvm_vgic_global_state
.type
== VGIC_V2
)
461 vgic_v2_set_vmcr(vcpu
, vmcr
);
463 vgic_v3_set_vmcr(vcpu
, vmcr
);
466 void vgic_get_vmcr(struct kvm_vcpu
*vcpu
, struct vgic_vmcr
*vmcr
)
468 if (kvm_vgic_global_state
.type
== VGIC_V2
)
469 vgic_v2_get_vmcr(vcpu
, vmcr
);
471 vgic_v3_get_vmcr(vcpu
, vmcr
);
475 * kvm_mmio_read_buf() returns a value in a format where it can be converted
476 * to a byte array and be directly observed as the guest wanted it to appear
477 * in memory if it had done the store itself, which is LE for the GIC, as the
478 * guest knows the GIC is always LE.
480 * We convert this value to the CPUs native format to deal with it as a data
483 unsigned long vgic_data_mmio_bus_to_host(const void *val
, unsigned int len
)
485 unsigned long data
= kvm_mmio_read_buf(val
, len
);
491 return le16_to_cpu(data
);
493 return le32_to_cpu(data
);
495 return le64_to_cpu(data
);
500 * kvm_mmio_write_buf() expects a value in a format such that if converted to
501 * a byte array it is observed as the guest would see it if it could perform
502 * the load directly. Since the GIC is LE, and the guest knows this, the
503 * guest expects a value in little endian format.
505 * We convert the data value from the CPUs native format to LE so that the
506 * value is returned in the proper format.
508 void vgic_data_host_to_mmio_bus(void *buf
, unsigned int len
,
515 data
= cpu_to_le16(data
);
518 data
= cpu_to_le32(data
);
521 data
= cpu_to_le64(data
);
524 kvm_mmio_write_buf(buf
, len
, data
);
528 struct vgic_io_device
*kvm_to_vgic_iodev(const struct kvm_io_device
*dev
)
530 return container_of(dev
, struct vgic_io_device
, dev
);
533 static bool check_region(const struct kvm
*kvm
,
534 const struct vgic_register_region
*region
,
537 int flags
, nr_irqs
= kvm
->arch
.vgic
.nr_spis
+ VGIC_NR_PRIVATE_IRQS
;
541 flags
= VGIC_ACCESS_8bit
;
544 flags
= VGIC_ACCESS_32bit
;
547 flags
= VGIC_ACCESS_64bit
;
553 if ((region
->access_flags
& flags
) && IS_ALIGNED(addr
, len
)) {
554 if (!region
->bits_per_irq
)
557 /* Do we access a non-allocated IRQ? */
558 return VGIC_ADDR_TO_INTID(addr
, region
->bits_per_irq
) < nr_irqs
;
564 const struct vgic_register_region
*
565 vgic_get_mmio_region(struct kvm_vcpu
*vcpu
, struct vgic_io_device
*iodev
,
568 const struct vgic_register_region
*region
;
570 region
= vgic_find_mmio_region(iodev
->regions
, iodev
->nr_regions
,
571 addr
- iodev
->base_addr
);
572 if (!region
|| !check_region(vcpu
->kvm
, region
, addr
, len
))
578 static int vgic_uaccess_read(struct kvm_vcpu
*vcpu
, struct kvm_io_device
*dev
,
579 gpa_t addr
, u32
*val
)
581 struct vgic_io_device
*iodev
= kvm_to_vgic_iodev(dev
);
582 const struct vgic_register_region
*region
;
583 struct kvm_vcpu
*r_vcpu
;
585 region
= vgic_get_mmio_region(vcpu
, iodev
, addr
, sizeof(u32
));
591 r_vcpu
= iodev
->redist_vcpu
? iodev
->redist_vcpu
: vcpu
;
592 if (region
->uaccess_read
)
593 *val
= region
->uaccess_read(r_vcpu
, addr
, sizeof(u32
));
595 *val
= region
->read(r_vcpu
, addr
, sizeof(u32
));
600 static int vgic_uaccess_write(struct kvm_vcpu
*vcpu
, struct kvm_io_device
*dev
,
601 gpa_t addr
, const u32
*val
)
603 struct vgic_io_device
*iodev
= kvm_to_vgic_iodev(dev
);
604 const struct vgic_register_region
*region
;
605 struct kvm_vcpu
*r_vcpu
;
607 region
= vgic_get_mmio_region(vcpu
, iodev
, addr
, sizeof(u32
));
611 r_vcpu
= iodev
->redist_vcpu
? iodev
->redist_vcpu
: vcpu
;
612 if (region
->uaccess_write
)
613 region
->uaccess_write(r_vcpu
, addr
, sizeof(u32
), *val
);
615 region
->write(r_vcpu
, addr
, sizeof(u32
), *val
);
621 * Userland access to VGIC registers.
623 int vgic_uaccess(struct kvm_vcpu
*vcpu
, struct vgic_io_device
*dev
,
624 bool is_write
, int offset
, u32
*val
)
627 return vgic_uaccess_write(vcpu
, &dev
->dev
, offset
, val
);
629 return vgic_uaccess_read(vcpu
, &dev
->dev
, offset
, val
);
632 static int dispatch_mmio_read(struct kvm_vcpu
*vcpu
, struct kvm_io_device
*dev
,
633 gpa_t addr
, int len
, void *val
)
635 struct vgic_io_device
*iodev
= kvm_to_vgic_iodev(dev
);
636 const struct vgic_register_region
*region
;
637 unsigned long data
= 0;
639 region
= vgic_get_mmio_region(vcpu
, iodev
, addr
, len
);
645 switch (iodev
->iodev_type
) {
647 data
= region
->read(vcpu
, addr
, len
);
650 data
= region
->read(vcpu
, addr
, len
);
653 data
= region
->read(iodev
->redist_vcpu
, addr
, len
);
656 data
= region
->its_read(vcpu
->kvm
, iodev
->its
, addr
, len
);
660 vgic_data_host_to_mmio_bus(val
, len
, data
);
664 static int dispatch_mmio_write(struct kvm_vcpu
*vcpu
, struct kvm_io_device
*dev
,
665 gpa_t addr
, int len
, const void *val
)
667 struct vgic_io_device
*iodev
= kvm_to_vgic_iodev(dev
);
668 const struct vgic_register_region
*region
;
669 unsigned long data
= vgic_data_mmio_bus_to_host(val
, len
);
671 region
= vgic_get_mmio_region(vcpu
, iodev
, addr
, len
);
675 switch (iodev
->iodev_type
) {
677 region
->write(vcpu
, addr
, len
, data
);
680 region
->write(vcpu
, addr
, len
, data
);
683 region
->write(iodev
->redist_vcpu
, addr
, len
, data
);
686 region
->its_write(vcpu
->kvm
, iodev
->its
, addr
, len
, data
);
693 struct kvm_io_device_ops kvm_io_gic_ops
= {
694 .read
= dispatch_mmio_read
,
695 .write
= dispatch_mmio_write
,
698 int vgic_register_dist_iodev(struct kvm
*kvm
, gpa_t dist_base_address
,
701 struct vgic_io_device
*io_device
= &kvm
->arch
.vgic
.dist_iodev
;
707 len
= vgic_v2_init_dist_iodev(io_device
);
710 len
= vgic_v3_init_dist_iodev(io_device
);
716 io_device
->base_addr
= dist_base_address
;
717 io_device
->iodev_type
= IODEV_DIST
;
718 io_device
->redist_vcpu
= NULL
;
720 mutex_lock(&kvm
->slots_lock
);
721 ret
= kvm_io_bus_register_dev(kvm
, KVM_MMIO_BUS
, dist_base_address
,
722 len
, &io_device
->dev
);
723 mutex_unlock(&kvm
->slots_lock
);