]>
Commit | Line | Data |
---|---|---|
ed9b8cef AP |
1 | /* |
2 | * VGICv3 MMIO handling functions | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License version 2 as | |
6 | * published by the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, | |
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
11 | * GNU General Public License for more details. | |
12 | */ | |
13 | ||
14 | #include <linux/irqchip/arm-gic-v3.h> | |
15 | #include <linux/kvm.h> | |
16 | #include <linux/kvm_host.h> | |
17 | #include <kvm/iodev.h> | |
18 | #include <kvm/arm_vgic.h> | |
19 | ||
20 | #include <asm/kvm_emulate.h> | |
94574c94 VK |
21 | #include <asm/kvm_arm.h> |
22 | #include <asm/kvm_mmu.h> | |
ed9b8cef AP |
23 | |
24 | #include "vgic.h" | |
25 | #include "vgic-mmio.h" | |
26 | ||
741972d8 | 27 | /* extract @num bytes at @offset bytes offset in data */ |
d7d0a11e | 28 | unsigned long extract_bytes(u64 data, unsigned int offset, |
424c3383 | 29 | unsigned int num) |
741972d8 AP |
30 | { |
31 | return (data >> (offset * 8)) & GENMASK_ULL(num * 8 - 1, 0); | |
32 | } | |
33 | ||
0aa1de57 | 34 | /* allows updates of any half of a 64-bit register (or the whole thing) */ |
424c3383 AP |
35 | u64 update_64bit_reg(u64 reg, unsigned int offset, unsigned int len, |
36 | unsigned long val) | |
0aa1de57 AP |
37 | { |
38 | int lower = (offset & 4) * 8; | |
39 | int upper = lower + 8 * len - 1; | |
40 | ||
41 | reg &= ~GENMASK_ULL(upper, lower); | |
42 | val &= GENMASK_ULL(len * 8 - 1, 0); | |
43 | ||
44 | return reg | ((u64)val << lower); | |
45 | } | |
46 | ||
59c5ab40 AP |
47 | bool vgic_has_its(struct kvm *kvm) |
48 | { | |
49 | struct vgic_dist *dist = &kvm->arch.vgic; | |
50 | ||
51 | if (dist->vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3) | |
52 | return false; | |
53 | ||
1085fdc6 | 54 | return dist->has_its; |
59c5ab40 AP |
55 | } |
56 | ||
fd59ed3b AP |
57 | static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu, |
58 | gpa_t addr, unsigned int len) | |
59 | { | |
60 | u32 value = 0; | |
61 | ||
62 | switch (addr & 0x0c) { | |
63 | case GICD_CTLR: | |
64 | if (vcpu->kvm->arch.vgic.enabled) | |
65 | value |= GICD_CTLR_ENABLE_SS_G1; | |
66 | value |= GICD_CTLR_ARE_NS | GICD_CTLR_DS; | |
67 | break; | |
68 | case GICD_TYPER: | |
69 | value = vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS; | |
70 | value = (value >> 5) - 1; | |
0e4e82f1 AP |
71 | if (vgic_has_its(vcpu->kvm)) { |
72 | value |= (INTERRUPT_ID_BITS_ITS - 1) << 19; | |
73 | value |= GICD_TYPER_LPIS; | |
74 | } else { | |
75 | value |= (INTERRUPT_ID_BITS_SPIS - 1) << 19; | |
76 | } | |
fd59ed3b AP |
77 | break; |
78 | case GICD_IIDR: | |
79 | value = (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0); | |
80 | break; | |
81 | default: | |
82 | return 0; | |
83 | } | |
84 | ||
85 | return value; | |
86 | } | |
87 | ||
88 | static void vgic_mmio_write_v3_misc(struct kvm_vcpu *vcpu, | |
89 | gpa_t addr, unsigned int len, | |
90 | unsigned long val) | |
91 | { | |
92 | struct vgic_dist *dist = &vcpu->kvm->arch.vgic; | |
93 | bool was_enabled = dist->enabled; | |
94 | ||
95 | switch (addr & 0x0c) { | |
96 | case GICD_CTLR: | |
97 | dist->enabled = val & GICD_CTLR_ENABLE_SS_G1; | |
98 | ||
99 | if (!was_enabled && dist->enabled) | |
100 | vgic_kick_vcpus(vcpu->kvm); | |
101 | break; | |
102 | case GICD_TYPER: | |
103 | case GICD_IIDR: | |
104 | return; | |
105 | } | |
106 | } | |
107 | ||
78a714ab AP |
108 | static unsigned long vgic_mmio_read_irouter(struct kvm_vcpu *vcpu, |
109 | gpa_t addr, unsigned int len) | |
110 | { | |
111 | int intid = VGIC_ADDR_TO_INTID(addr, 64); | |
112 | struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, NULL, intid); | |
5dd4b924 | 113 | unsigned long ret = 0; |
78a714ab AP |
114 | |
115 | if (!irq) | |
116 | return 0; | |
117 | ||
118 | /* The upper word is RAZ for us. */ | |
5dd4b924 AP |
119 | if (!(addr & 4)) |
120 | ret = extract_bytes(READ_ONCE(irq->mpidr), addr & 7, len); | |
78a714ab | 121 | |
5dd4b924 AP |
122 | vgic_put_irq(vcpu->kvm, irq); |
123 | return ret; | |
78a714ab AP |
124 | } |
125 | ||
126 | static void vgic_mmio_write_irouter(struct kvm_vcpu *vcpu, | |
127 | gpa_t addr, unsigned int len, | |
128 | unsigned long val) | |
129 | { | |
130 | int intid = VGIC_ADDR_TO_INTID(addr, 64); | |
5dd4b924 | 131 | struct vgic_irq *irq; |
78a714ab AP |
132 | |
133 | /* The upper word is WI for us since we don't implement Aff3. */ | |
134 | if (addr & 4) | |
135 | return; | |
136 | ||
5dd4b924 AP |
137 | irq = vgic_get_irq(vcpu->kvm, NULL, intid); |
138 | ||
139 | if (!irq) | |
140 | return; | |
141 | ||
78a714ab AP |
142 | spin_lock(&irq->irq_lock); |
143 | ||
144 | /* We only care about and preserve Aff0, Aff1 and Aff2. */ | |
145 | irq->mpidr = val & GENMASK(23, 0); | |
146 | irq->target_vcpu = kvm_mpidr_to_vcpu(vcpu->kvm, irq->mpidr); | |
147 | ||
148 | spin_unlock(&irq->irq_lock); | |
5dd4b924 | 149 | vgic_put_irq(vcpu->kvm, irq); |
78a714ab AP |
150 | } |
151 | ||
59c5ab40 AP |
152 | static unsigned long vgic_mmio_read_v3r_ctlr(struct kvm_vcpu *vcpu, |
153 | gpa_t addr, unsigned int len) | |
154 | { | |
155 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
156 | ||
157 | return vgic_cpu->lpis_enabled ? GICR_CTLR_ENABLE_LPIS : 0; | |
158 | } | |
159 | ||
160 | ||
161 | static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu, | |
162 | gpa_t addr, unsigned int len, | |
163 | unsigned long val) | |
164 | { | |
165 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
166 | bool was_enabled = vgic_cpu->lpis_enabled; | |
167 | ||
168 | if (!vgic_has_its(vcpu->kvm)) | |
169 | return; | |
170 | ||
171 | vgic_cpu->lpis_enabled = val & GICR_CTLR_ENABLE_LPIS; | |
172 | ||
0e4e82f1 AP |
173 | if (!was_enabled && vgic_cpu->lpis_enabled) |
174 | vgic_enable_lpis(vcpu); | |
59c5ab40 AP |
175 | } |
176 | ||
741972d8 AP |
177 | static unsigned long vgic_mmio_read_v3r_typer(struct kvm_vcpu *vcpu, |
178 | gpa_t addr, unsigned int len) | |
179 | { | |
180 | unsigned long mpidr = kvm_vcpu_get_mpidr_aff(vcpu); | |
181 | int target_vcpu_id = vcpu->vcpu_id; | |
182 | u64 value; | |
183 | ||
e533a37f | 184 | value = (u64)(mpidr & GENMASK(23, 0)) << 32; |
741972d8 AP |
185 | value |= ((target_vcpu_id & 0xffff) << 8); |
186 | if (target_vcpu_id == atomic_read(&vcpu->kvm->online_vcpus) - 1) | |
187 | value |= GICR_TYPER_LAST; | |
0e4e82f1 AP |
188 | if (vgic_has_its(vcpu->kvm)) |
189 | value |= GICR_TYPER_PLPIS; | |
741972d8 AP |
190 | |
191 | return extract_bytes(value, addr & 7, len); | |
192 | } | |
193 | ||
194 | static unsigned long vgic_mmio_read_v3r_iidr(struct kvm_vcpu *vcpu, | |
195 | gpa_t addr, unsigned int len) | |
196 | { | |
197 | return (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0); | |
198 | } | |
199 | ||
54f59d2b AP |
200 | static unsigned long vgic_mmio_read_v3_idregs(struct kvm_vcpu *vcpu, |
201 | gpa_t addr, unsigned int len) | |
202 | { | |
203 | switch (addr & 0xffff) { | |
204 | case GICD_PIDR2: | |
205 | /* report a GICv3 compliant implementation */ | |
206 | return 0x3b; | |
207 | } | |
208 | ||
209 | return 0; | |
210 | } | |
211 | ||
2df903a8 VK |
212 | static unsigned long vgic_v3_uaccess_read_pending(struct kvm_vcpu *vcpu, |
213 | gpa_t addr, unsigned int len) | |
214 | { | |
215 | u32 intid = VGIC_ADDR_TO_INTID(addr, 1); | |
216 | u32 value = 0; | |
217 | int i; | |
218 | ||
219 | /* | |
220 | * pending state of interrupt is latched in pending_latch variable. | |
221 | * Userspace will save and restore pending state and line_level | |
222 | * separately. | |
223 | * Refer to Documentation/virtual/kvm/devices/arm-vgic-v3.txt | |
224 | * for handling of ISPENDR and ICPENDR. | |
225 | */ | |
226 | for (i = 0; i < len * 8; i++) { | |
227 | struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); | |
228 | ||
229 | if (irq->pending_latch) | |
230 | value |= (1U << i); | |
231 | ||
232 | vgic_put_irq(vcpu->kvm, irq); | |
233 | } | |
234 | ||
235 | return value; | |
236 | } | |
237 | ||
238 | static void vgic_v3_uaccess_write_pending(struct kvm_vcpu *vcpu, | |
239 | gpa_t addr, unsigned int len, | |
240 | unsigned long val) | |
241 | { | |
242 | u32 intid = VGIC_ADDR_TO_INTID(addr, 1); | |
243 | int i; | |
244 | ||
245 | for (i = 0; i < len * 8; i++) { | |
246 | struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); | |
247 | ||
248 | spin_lock(&irq->irq_lock); | |
249 | if (test_bit(i, &val)) { | |
250 | /* | |
251 | * pending_latch is set irrespective of irq type | |
252 | * (level or edge) to avoid dependency that VM should | |
253 | * restore irq config before pending info. | |
254 | */ | |
255 | irq->pending_latch = true; | |
256 | vgic_queue_irq_unlock(vcpu->kvm, irq); | |
257 | } else { | |
258 | irq->pending_latch = false; | |
259 | spin_unlock(&irq->irq_lock); | |
260 | } | |
261 | ||
262 | vgic_put_irq(vcpu->kvm, irq); | |
263 | } | |
264 | } | |
265 | ||
0aa1de57 AP |
266 | /* We want to avoid outer shareable. */ |
267 | u64 vgic_sanitise_shareability(u64 field) | |
268 | { | |
269 | switch (field) { | |
270 | case GIC_BASER_OuterShareable: | |
271 | return GIC_BASER_InnerShareable; | |
272 | default: | |
273 | return field; | |
274 | } | |
275 | } | |
276 | ||
277 | /* Avoid any inner non-cacheable mapping. */ | |
278 | u64 vgic_sanitise_inner_cacheability(u64 field) | |
279 | { | |
280 | switch (field) { | |
281 | case GIC_BASER_CACHE_nCnB: | |
282 | case GIC_BASER_CACHE_nC: | |
283 | return GIC_BASER_CACHE_RaWb; | |
284 | default: | |
285 | return field; | |
286 | } | |
287 | } | |
288 | ||
289 | /* Non-cacheable or same-as-inner are OK. */ | |
290 | u64 vgic_sanitise_outer_cacheability(u64 field) | |
291 | { | |
292 | switch (field) { | |
293 | case GIC_BASER_CACHE_SameAsInner: | |
294 | case GIC_BASER_CACHE_nC: | |
295 | return field; | |
296 | default: | |
297 | return GIC_BASER_CACHE_nC; | |
298 | } | |
299 | } | |
300 | ||
301 | u64 vgic_sanitise_field(u64 reg, u64 field_mask, int field_shift, | |
302 | u64 (*sanitise_fn)(u64)) | |
303 | { | |
304 | u64 field = (reg & field_mask) >> field_shift; | |
305 | ||
306 | field = sanitise_fn(field) << field_shift; | |
307 | return (reg & ~field_mask) | field; | |
308 | } | |
309 | ||
310 | #define PROPBASER_RES0_MASK \ | |
311 | (GENMASK_ULL(63, 59) | GENMASK_ULL(55, 52) | GENMASK_ULL(6, 5)) | |
312 | #define PENDBASER_RES0_MASK \ | |
313 | (BIT_ULL(63) | GENMASK_ULL(61, 59) | GENMASK_ULL(55, 52) | \ | |
314 | GENMASK_ULL(15, 12) | GENMASK_ULL(6, 0)) | |
315 | ||
316 | static u64 vgic_sanitise_pendbaser(u64 reg) | |
317 | { | |
318 | reg = vgic_sanitise_field(reg, GICR_PENDBASER_SHAREABILITY_MASK, | |
319 | GICR_PENDBASER_SHAREABILITY_SHIFT, | |
320 | vgic_sanitise_shareability); | |
321 | reg = vgic_sanitise_field(reg, GICR_PENDBASER_INNER_CACHEABILITY_MASK, | |
322 | GICR_PENDBASER_INNER_CACHEABILITY_SHIFT, | |
323 | vgic_sanitise_inner_cacheability); | |
324 | reg = vgic_sanitise_field(reg, GICR_PENDBASER_OUTER_CACHEABILITY_MASK, | |
325 | GICR_PENDBASER_OUTER_CACHEABILITY_SHIFT, | |
326 | vgic_sanitise_outer_cacheability); | |
327 | ||
328 | reg &= ~PENDBASER_RES0_MASK; | |
329 | reg &= ~GENMASK_ULL(51, 48); | |
330 | ||
331 | return reg; | |
332 | } | |
333 | ||
334 | static u64 vgic_sanitise_propbaser(u64 reg) | |
335 | { | |
336 | reg = vgic_sanitise_field(reg, GICR_PROPBASER_SHAREABILITY_MASK, | |
337 | GICR_PROPBASER_SHAREABILITY_SHIFT, | |
338 | vgic_sanitise_shareability); | |
339 | reg = vgic_sanitise_field(reg, GICR_PROPBASER_INNER_CACHEABILITY_MASK, | |
340 | GICR_PROPBASER_INNER_CACHEABILITY_SHIFT, | |
341 | vgic_sanitise_inner_cacheability); | |
342 | reg = vgic_sanitise_field(reg, GICR_PROPBASER_OUTER_CACHEABILITY_MASK, | |
343 | GICR_PROPBASER_OUTER_CACHEABILITY_SHIFT, | |
344 | vgic_sanitise_outer_cacheability); | |
345 | ||
346 | reg &= ~PROPBASER_RES0_MASK; | |
347 | reg &= ~GENMASK_ULL(51, 48); | |
348 | return reg; | |
349 | } | |
350 | ||
351 | static unsigned long vgic_mmio_read_propbase(struct kvm_vcpu *vcpu, | |
352 | gpa_t addr, unsigned int len) | |
353 | { | |
354 | struct vgic_dist *dist = &vcpu->kvm->arch.vgic; | |
355 | ||
356 | return extract_bytes(dist->propbaser, addr & 7, len); | |
357 | } | |
358 | ||
359 | static void vgic_mmio_write_propbase(struct kvm_vcpu *vcpu, | |
360 | gpa_t addr, unsigned int len, | |
361 | unsigned long val) | |
362 | { | |
363 | struct vgic_dist *dist = &vcpu->kvm->arch.vgic; | |
364 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
d9ae449b | 365 | u64 old_propbaser, propbaser; |
0aa1de57 AP |
366 | |
367 | /* Storing a value with LPIs already enabled is undefined */ | |
368 | if (vgic_cpu->lpis_enabled) | |
369 | return; | |
370 | ||
d9ae449b | 371 | do { |
3af4e414 | 372 | old_propbaser = READ_ONCE(dist->propbaser); |
d9ae449b CD |
373 | propbaser = old_propbaser; |
374 | propbaser = update_64bit_reg(propbaser, addr & 4, len, val); | |
375 | propbaser = vgic_sanitise_propbaser(propbaser); | |
376 | } while (cmpxchg64(&dist->propbaser, old_propbaser, | |
377 | propbaser) != old_propbaser); | |
0aa1de57 AP |
378 | } |
379 | ||
380 | static unsigned long vgic_mmio_read_pendbase(struct kvm_vcpu *vcpu, | |
381 | gpa_t addr, unsigned int len) | |
382 | { | |
383 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
384 | ||
385 | return extract_bytes(vgic_cpu->pendbaser, addr & 7, len); | |
386 | } | |
387 | ||
388 | static void vgic_mmio_write_pendbase(struct kvm_vcpu *vcpu, | |
389 | gpa_t addr, unsigned int len, | |
390 | unsigned long val) | |
391 | { | |
392 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
d9ae449b | 393 | u64 old_pendbaser, pendbaser; |
0aa1de57 AP |
394 | |
395 | /* Storing a value with LPIs already enabled is undefined */ | |
396 | if (vgic_cpu->lpis_enabled) | |
397 | return; | |
398 | ||
d9ae449b | 399 | do { |
3af4e414 | 400 | old_pendbaser = READ_ONCE(vgic_cpu->pendbaser); |
d9ae449b CD |
401 | pendbaser = old_pendbaser; |
402 | pendbaser = update_64bit_reg(pendbaser, addr & 4, len, val); | |
403 | pendbaser = vgic_sanitise_pendbaser(pendbaser); | |
404 | } while (cmpxchg64(&vgic_cpu->pendbaser, old_pendbaser, | |
405 | pendbaser) != old_pendbaser); | |
0aa1de57 AP |
406 | } |
407 | ||
ed9b8cef AP |
408 | /* |
409 | * The GICv3 per-IRQ registers are split to control PPIs and SGIs in the | |
410 | * redistributors, while SPIs are covered by registers in the distributor | |
411 | * block. Trying to set private IRQs in this block gets ignored. | |
412 | * We take some special care here to fix the calculation of the register | |
413 | * offset. | |
414 | */ | |
2df903a8 | 415 | #define REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(off, rd, wr, ur, uw, bpi, acc) \ |
ed9b8cef AP |
416 | { \ |
417 | .reg_offset = off, \ | |
418 | .bits_per_irq = bpi, \ | |
419 | .len = (bpi * VGIC_NR_PRIVATE_IRQS) / 8, \ | |
420 | .access_flags = acc, \ | |
421 | .read = vgic_mmio_read_raz, \ | |
422 | .write = vgic_mmio_write_wi, \ | |
423 | }, { \ | |
424 | .reg_offset = off + (bpi * VGIC_NR_PRIVATE_IRQS) / 8, \ | |
425 | .bits_per_irq = bpi, \ | |
426 | .len = (bpi * (1024 - VGIC_NR_PRIVATE_IRQS)) / 8, \ | |
427 | .access_flags = acc, \ | |
428 | .read = rd, \ | |
429 | .write = wr, \ | |
2df903a8 VK |
430 | .uaccess_read = ur, \ |
431 | .uaccess_write = uw, \ | |
ed9b8cef AP |
432 | } |
433 | ||
434 | static const struct vgic_register_region vgic_v3_dist_registers[] = { | |
435 | REGISTER_DESC_WITH_LENGTH(GICD_CTLR, | |
fd59ed3b | 436 | vgic_mmio_read_v3_misc, vgic_mmio_write_v3_misc, 16, |
ed9b8cef | 437 | VGIC_ACCESS_32bit), |
94574c94 VK |
438 | REGISTER_DESC_WITH_LENGTH(GICD_STATUSR, |
439 | vgic_mmio_read_rao, vgic_mmio_write_wi, 4, | |
440 | VGIC_ACCESS_32bit), | |
ed9b8cef | 441 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IGROUPR, |
2df903a8 | 442 | vgic_mmio_read_rao, vgic_mmio_write_wi, NULL, NULL, 1, |
ed9b8cef AP |
443 | VGIC_ACCESS_32bit), |
444 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISENABLER, | |
2df903a8 | 445 | vgic_mmio_read_enable, vgic_mmio_write_senable, NULL, NULL, 1, |
ed9b8cef AP |
446 | VGIC_ACCESS_32bit), |
447 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICENABLER, | |
2df903a8 | 448 | vgic_mmio_read_enable, vgic_mmio_write_cenable, NULL, NULL, 1, |
ed9b8cef AP |
449 | VGIC_ACCESS_32bit), |
450 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISPENDR, | |
2df903a8 VK |
451 | vgic_mmio_read_pending, vgic_mmio_write_spending, |
452 | vgic_v3_uaccess_read_pending, vgic_v3_uaccess_write_pending, 1, | |
ed9b8cef AP |
453 | VGIC_ACCESS_32bit), |
454 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICPENDR, | |
2df903a8 VK |
455 | vgic_mmio_read_pending, vgic_mmio_write_cpending, |
456 | vgic_mmio_read_raz, vgic_mmio_write_wi, 1, | |
ed9b8cef AP |
457 | VGIC_ACCESS_32bit), |
458 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISACTIVER, | |
3197191e CD |
459 | vgic_mmio_read_active, vgic_mmio_write_sactive, |
460 | NULL, vgic_mmio_uaccess_write_sactive, 1, | |
ed9b8cef AP |
461 | VGIC_ACCESS_32bit), |
462 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICACTIVER, | |
3197191e CD |
463 | vgic_mmio_read_active, vgic_mmio_write_cactive, |
464 | NULL, vgic_mmio_uaccess_write_cactive, | |
465 | 1, VGIC_ACCESS_32bit), | |
ed9b8cef | 466 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IPRIORITYR, |
2df903a8 VK |
467 | vgic_mmio_read_priority, vgic_mmio_write_priority, NULL, NULL, |
468 | 8, VGIC_ACCESS_32bit | VGIC_ACCESS_8bit), | |
ed9b8cef | 469 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ITARGETSR, |
2df903a8 | 470 | vgic_mmio_read_raz, vgic_mmio_write_wi, NULL, NULL, 8, |
ed9b8cef AP |
471 | VGIC_ACCESS_32bit | VGIC_ACCESS_8bit), |
472 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICFGR, | |
2df903a8 | 473 | vgic_mmio_read_config, vgic_mmio_write_config, NULL, NULL, 2, |
ed9b8cef AP |
474 | VGIC_ACCESS_32bit), |
475 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IGRPMODR, | |
2df903a8 | 476 | vgic_mmio_read_raz, vgic_mmio_write_wi, NULL, NULL, 1, |
ed9b8cef AP |
477 | VGIC_ACCESS_32bit), |
478 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IROUTER, | |
2df903a8 | 479 | vgic_mmio_read_irouter, vgic_mmio_write_irouter, NULL, NULL, 64, |
ed9b8cef AP |
480 | VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), |
481 | REGISTER_DESC_WITH_LENGTH(GICD_IDREGS, | |
54f59d2b | 482 | vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48, |
ed9b8cef AP |
483 | VGIC_ACCESS_32bit), |
484 | }; | |
485 | ||
486 | static const struct vgic_register_region vgic_v3_rdbase_registers[] = { | |
487 | REGISTER_DESC_WITH_LENGTH(GICR_CTLR, | |
59c5ab40 | 488 | vgic_mmio_read_v3r_ctlr, vgic_mmio_write_v3r_ctlr, 4, |
ed9b8cef | 489 | VGIC_ACCESS_32bit), |
94574c94 VK |
490 | REGISTER_DESC_WITH_LENGTH(GICR_STATUSR, |
491 | vgic_mmio_read_raz, vgic_mmio_write_wi, 4, | |
492 | VGIC_ACCESS_32bit), | |
ed9b8cef | 493 | REGISTER_DESC_WITH_LENGTH(GICR_IIDR, |
741972d8 | 494 | vgic_mmio_read_v3r_iidr, vgic_mmio_write_wi, 4, |
ed9b8cef AP |
495 | VGIC_ACCESS_32bit), |
496 | REGISTER_DESC_WITH_LENGTH(GICR_TYPER, | |
741972d8 | 497 | vgic_mmio_read_v3r_typer, vgic_mmio_write_wi, 8, |
ed9b8cef | 498 | VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), |
94574c94 VK |
499 | REGISTER_DESC_WITH_LENGTH(GICR_WAKER, |
500 | vgic_mmio_read_raz, vgic_mmio_write_wi, 4, | |
501 | VGIC_ACCESS_32bit), | |
ed9b8cef | 502 | REGISTER_DESC_WITH_LENGTH(GICR_PROPBASER, |
0aa1de57 | 503 | vgic_mmio_read_propbase, vgic_mmio_write_propbase, 8, |
ed9b8cef AP |
504 | VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), |
505 | REGISTER_DESC_WITH_LENGTH(GICR_PENDBASER, | |
0aa1de57 | 506 | vgic_mmio_read_pendbase, vgic_mmio_write_pendbase, 8, |
ed9b8cef AP |
507 | VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), |
508 | REGISTER_DESC_WITH_LENGTH(GICR_IDREGS, | |
54f59d2b | 509 | vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48, |
ed9b8cef AP |
510 | VGIC_ACCESS_32bit), |
511 | }; | |
512 | ||
513 | static const struct vgic_register_region vgic_v3_sgibase_registers[] = { | |
514 | REGISTER_DESC_WITH_LENGTH(GICR_IGROUPR0, | |
515 | vgic_mmio_read_rao, vgic_mmio_write_wi, 4, | |
516 | VGIC_ACCESS_32bit), | |
517 | REGISTER_DESC_WITH_LENGTH(GICR_ISENABLER0, | |
518 | vgic_mmio_read_enable, vgic_mmio_write_senable, 4, | |
519 | VGIC_ACCESS_32bit), | |
520 | REGISTER_DESC_WITH_LENGTH(GICR_ICENABLER0, | |
521 | vgic_mmio_read_enable, vgic_mmio_write_cenable, 4, | |
522 | VGIC_ACCESS_32bit), | |
2df903a8 VK |
523 | REGISTER_DESC_WITH_LENGTH_UACCESS(GICR_ISPENDR0, |
524 | vgic_mmio_read_pending, vgic_mmio_write_spending, | |
525 | vgic_v3_uaccess_read_pending, vgic_v3_uaccess_write_pending, 4, | |
ed9b8cef | 526 | VGIC_ACCESS_32bit), |
2df903a8 VK |
527 | REGISTER_DESC_WITH_LENGTH_UACCESS(GICR_ICPENDR0, |
528 | vgic_mmio_read_pending, vgic_mmio_write_cpending, | |
529 | vgic_mmio_read_raz, vgic_mmio_write_wi, 4, | |
ed9b8cef | 530 | VGIC_ACCESS_32bit), |
0710f9a6 CD |
531 | REGISTER_DESC_WITH_LENGTH_UACCESS(GICR_ISACTIVER0, |
532 | vgic_mmio_read_active, vgic_mmio_write_sactive, | |
533 | NULL, vgic_mmio_uaccess_write_sactive, | |
534 | 4, VGIC_ACCESS_32bit), | |
535 | REGISTER_DESC_WITH_LENGTH_UACCESS(GICR_ICACTIVER0, | |
536 | vgic_mmio_read_active, vgic_mmio_write_cactive, | |
537 | NULL, vgic_mmio_uaccess_write_cactive, | |
538 | 4, VGIC_ACCESS_32bit), | |
ed9b8cef AP |
539 | REGISTER_DESC_WITH_LENGTH(GICR_IPRIORITYR0, |
540 | vgic_mmio_read_priority, vgic_mmio_write_priority, 32, | |
541 | VGIC_ACCESS_32bit | VGIC_ACCESS_8bit), | |
542 | REGISTER_DESC_WITH_LENGTH(GICR_ICFGR0, | |
543 | vgic_mmio_read_config, vgic_mmio_write_config, 8, | |
544 | VGIC_ACCESS_32bit), | |
545 | REGISTER_DESC_WITH_LENGTH(GICR_IGRPMODR0, | |
546 | vgic_mmio_read_raz, vgic_mmio_write_wi, 4, | |
547 | VGIC_ACCESS_32bit), | |
548 | REGISTER_DESC_WITH_LENGTH(GICR_NSACR, | |
549 | vgic_mmio_read_raz, vgic_mmio_write_wi, 4, | |
550 | VGIC_ACCESS_32bit), | |
551 | }; | |
552 | ||
553 | unsigned int vgic_v3_init_dist_iodev(struct vgic_io_device *dev) | |
554 | { | |
555 | dev->regions = vgic_v3_dist_registers; | |
556 | dev->nr_regions = ARRAY_SIZE(vgic_v3_dist_registers); | |
557 | ||
558 | kvm_iodevice_init(&dev->dev, &kvm_io_gic_ops); | |
559 | ||
560 | return SZ_64K; | |
561 | } | |
562 | ||
7fadcd3a CD |
563 | /** |
564 | * vgic_register_redist_iodev - register a single redist iodev | |
565 | * @vcpu: The VCPU to which the redistributor belongs | |
566 | * | |
567 | * Register a KVM iodev for this VCPU's redistributor using the address | |
568 | * provided. | |
569 | * | |
570 | * Return 0 on success, -ERRNO otherwise. | |
571 | */ | |
1aab6f46 | 572 | int vgic_register_redist_iodev(struct kvm_vcpu *vcpu) |
7fadcd3a CD |
573 | { |
574 | struct kvm *kvm = vcpu->kvm; | |
575 | struct vgic_dist *vgic = &kvm->arch.vgic; | |
576 | struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev; | |
577 | struct vgic_io_device *sgi_dev = &vcpu->arch.vgic_cpu.sgi_iodev; | |
578 | gpa_t rd_base, sgi_base; | |
579 | int ret; | |
580 | ||
1aab6f46 CD |
581 | /* |
582 | * We may be creating VCPUs before having set the base address for the | |
583 | * redistributor region, in which case we will come back to this | |
584 | * function for all VCPUs when the base address is set. Just return | |
585 | * without doing any work for now. | |
586 | */ | |
587 | if (IS_VGIC_ADDR_UNDEF(vgic->vgic_redist_base)) | |
588 | return 0; | |
589 | ||
590 | if (!vgic_v3_check_base(kvm)) | |
591 | return -EINVAL; | |
592 | ||
552c9f47 | 593 | rd_base = vgic->vgic_redist_base + vgic->vgic_redist_free_offset; |
7fadcd3a CD |
594 | sgi_base = rd_base + SZ_64K; |
595 | ||
596 | kvm_iodevice_init(&rd_dev->dev, &kvm_io_gic_ops); | |
597 | rd_dev->base_addr = rd_base; | |
598 | rd_dev->iodev_type = IODEV_REDIST; | |
599 | rd_dev->regions = vgic_v3_rdbase_registers; | |
600 | rd_dev->nr_regions = ARRAY_SIZE(vgic_v3_rdbase_registers); | |
601 | rd_dev->redist_vcpu = vcpu; | |
602 | ||
603 | mutex_lock(&kvm->slots_lock); | |
604 | ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, rd_base, | |
605 | SZ_64K, &rd_dev->dev); | |
606 | mutex_unlock(&kvm->slots_lock); | |
607 | ||
608 | if (ret) | |
609 | return ret; | |
610 | ||
611 | kvm_iodevice_init(&sgi_dev->dev, &kvm_io_gic_ops); | |
612 | sgi_dev->base_addr = sgi_base; | |
613 | sgi_dev->iodev_type = IODEV_REDIST; | |
614 | sgi_dev->regions = vgic_v3_sgibase_registers; | |
615 | sgi_dev->nr_regions = ARRAY_SIZE(vgic_v3_sgibase_registers); | |
616 | sgi_dev->redist_vcpu = vcpu; | |
617 | ||
618 | mutex_lock(&kvm->slots_lock); | |
619 | ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, sgi_base, | |
620 | SZ_64K, &sgi_dev->dev); | |
552c9f47 | 621 | if (ret) { |
7fadcd3a CD |
622 | kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, |
623 | &rd_dev->dev); | |
fa472fa9 | 624 | goto out; |
552c9f47 | 625 | } |
7fadcd3a | 626 | |
552c9f47 | 627 | vgic->vgic_redist_free_offset += 2 * SZ_64K; |
fa472fa9 CD |
628 | out: |
629 | mutex_unlock(&kvm->slots_lock); | |
7fadcd3a CD |
630 | return ret; |
631 | } | |
632 | ||
633 | static void vgic_unregister_redist_iodev(struct kvm_vcpu *vcpu) | |
634 | { | |
635 | struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev; | |
636 | struct vgic_io_device *sgi_dev = &vcpu->arch.vgic_cpu.sgi_iodev; | |
637 | ||
638 | kvm_io_bus_unregister_dev(vcpu->kvm, KVM_MMIO_BUS, &rd_dev->dev); | |
639 | kvm_io_bus_unregister_dev(vcpu->kvm, KVM_MMIO_BUS, &sgi_dev->dev); | |
640 | } | |
641 | ||
1aab6f46 | 642 | static int vgic_register_all_redist_iodevs(struct kvm *kvm) |
ed9b8cef | 643 | { |
ed9b8cef | 644 | struct kvm_vcpu *vcpu; |
ed9b8cef AP |
645 | int c, ret = 0; |
646 | ||
ed9b8cef | 647 | kvm_for_each_vcpu(c, vcpu, kvm) { |
7fadcd3a | 648 | ret = vgic_register_redist_iodev(vcpu); |
ed9b8cef AP |
649 | if (ret) |
650 | break; | |
ed9b8cef AP |
651 | } |
652 | ||
653 | if (ret) { | |
654 | /* The current c failed, so we start with the previous one. */ | |
fa472fa9 | 655 | mutex_lock(&kvm->slots_lock); |
ed9b8cef | 656 | for (c--; c >= 0; c--) { |
8f6cdc1c | 657 | vcpu = kvm_get_vcpu(kvm, c); |
7fadcd3a | 658 | vgic_unregister_redist_iodev(vcpu); |
ed9b8cef | 659 | } |
fa472fa9 | 660 | mutex_unlock(&kvm->slots_lock); |
ed9b8cef AP |
661 | } |
662 | ||
663 | return ret; | |
664 | } | |
621ecd8d | 665 | |
1aab6f46 CD |
666 | int vgic_v3_set_redist_base(struct kvm *kvm, u64 addr) |
667 | { | |
668 | struct vgic_dist *vgic = &kvm->arch.vgic; | |
669 | int ret; | |
670 | ||
671 | /* vgic_check_ioaddr makes sure we don't do this twice */ | |
672 | ret = vgic_check_ioaddr(kvm, &vgic->vgic_redist_base, addr, SZ_64K); | |
673 | if (ret) | |
674 | return ret; | |
675 | ||
676 | vgic->vgic_redist_base = addr; | |
677 | if (!vgic_v3_check_base(kvm)) { | |
678 | vgic->vgic_redist_base = VGIC_ADDR_UNDEF; | |
679 | return -EINVAL; | |
680 | } | |
681 | ||
682 | /* | |
683 | * Register iodevs for each existing VCPU. Adding more VCPUs | |
684 | * afterwards will register the iodevs when needed. | |
685 | */ | |
686 | ret = vgic_register_all_redist_iodevs(kvm); | |
687 | if (ret) | |
688 | return ret; | |
689 | ||
690 | return 0; | |
691 | } | |
692 | ||
94574c94 VK |
693 | int vgic_v3_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr) |
694 | { | |
695 | const struct vgic_register_region *region; | |
696 | struct vgic_io_device iodev; | |
697 | struct vgic_reg_attr reg_attr; | |
698 | struct kvm_vcpu *vcpu; | |
699 | gpa_t addr; | |
700 | int ret; | |
701 | ||
702 | ret = vgic_v3_parse_attr(dev, attr, ®_attr); | |
703 | if (ret) | |
704 | return ret; | |
705 | ||
706 | vcpu = reg_attr.vcpu; | |
707 | addr = reg_attr.addr; | |
708 | ||
709 | switch (attr->group) { | |
710 | case KVM_DEV_ARM_VGIC_GRP_DIST_REGS: | |
711 | iodev.regions = vgic_v3_dist_registers; | |
712 | iodev.nr_regions = ARRAY_SIZE(vgic_v3_dist_registers); | |
713 | iodev.base_addr = 0; | |
714 | break; | |
715 | case KVM_DEV_ARM_VGIC_GRP_REDIST_REGS:{ | |
716 | iodev.regions = vgic_v3_rdbase_registers; | |
717 | iodev.nr_regions = ARRAY_SIZE(vgic_v3_rdbase_registers); | |
718 | iodev.base_addr = 0; | |
719 | break; | |
720 | } | |
d017d7b0 VK |
721 | case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: { |
722 | u64 reg, id; | |
723 | ||
724 | id = (attr->attr & KVM_DEV_ARM_VGIC_SYSREG_INSTR_MASK); | |
725 | return vgic_v3_has_cpu_sysregs_attr(vcpu, 0, id, ®); | |
726 | } | |
94574c94 VK |
727 | default: |
728 | return -ENXIO; | |
729 | } | |
730 | ||
731 | /* We only support aligned 32-bit accesses. */ | |
732 | if (addr & 3) | |
733 | return -ENXIO; | |
734 | ||
735 | region = vgic_get_mmio_region(vcpu, &iodev, addr, sizeof(u32)); | |
736 | if (!region) | |
737 | return -ENXIO; | |
738 | ||
739 | return 0; | |
740 | } | |
621ecd8d AP |
741 | /* |
742 | * Compare a given affinity (level 1-3 and a level 0 mask, from the SGI | |
743 | * generation register ICC_SGI1R_EL1) with a given VCPU. | |
744 | * If the VCPU's MPIDR matches, return the level0 affinity, otherwise | |
745 | * return -1. | |
746 | */ | |
747 | static int match_mpidr(u64 sgi_aff, u16 sgi_cpu_mask, struct kvm_vcpu *vcpu) | |
748 | { | |
749 | unsigned long affinity; | |
750 | int level0; | |
751 | ||
752 | /* | |
753 | * Split the current VCPU's MPIDR into affinity level 0 and the | |
754 | * rest as this is what we have to compare against. | |
755 | */ | |
756 | affinity = kvm_vcpu_get_mpidr_aff(vcpu); | |
757 | level0 = MPIDR_AFFINITY_LEVEL(affinity, 0); | |
758 | affinity &= ~MPIDR_LEVEL_MASK; | |
759 | ||
760 | /* bail out if the upper three levels don't match */ | |
761 | if (sgi_aff != affinity) | |
762 | return -1; | |
763 | ||
764 | /* Is this VCPU's bit set in the mask ? */ | |
765 | if (!(sgi_cpu_mask & BIT(level0))) | |
766 | return -1; | |
767 | ||
768 | return level0; | |
769 | } | |
770 | ||
771 | /* | |
772 | * The ICC_SGI* registers encode the affinity differently from the MPIDR, | |
773 | * so provide a wrapper to use the existing defines to isolate a certain | |
774 | * affinity level. | |
775 | */ | |
776 | #define SGI_AFFINITY_LEVEL(reg, level) \ | |
777 | ((((reg) & ICC_SGI1R_AFFINITY_## level ##_MASK) \ | |
778 | >> ICC_SGI1R_AFFINITY_## level ##_SHIFT) << MPIDR_LEVEL_SHIFT(level)) | |
779 | ||
780 | /** | |
781 | * vgic_v3_dispatch_sgi - handle SGI requests from VCPUs | |
782 | * @vcpu: The VCPU requesting a SGI | |
783 | * @reg: The value written into the ICC_SGI1R_EL1 register by that VCPU | |
784 | * | |
785 | * With GICv3 (and ARE=1) CPUs trigger SGIs by writing to a system register. | |
786 | * This will trap in sys_regs.c and call this function. | |
787 | * This ICC_SGI1R_EL1 register contains the upper three affinity levels of the | |
788 | * target processors as well as a bitmask of 16 Aff0 CPUs. | |
789 | * If the interrupt routing mode bit is not set, we iterate over all VCPUs to | |
790 | * check for matching ones. If this bit is set, we signal all, but not the | |
791 | * calling VCPU. | |
792 | */ | |
793 | void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg) | |
794 | { | |
795 | struct kvm *kvm = vcpu->kvm; | |
796 | struct kvm_vcpu *c_vcpu; | |
797 | u16 target_cpus; | |
798 | u64 mpidr; | |
799 | int sgi, c; | |
800 | int vcpu_id = vcpu->vcpu_id; | |
801 | bool broadcast; | |
802 | ||
803 | sgi = (reg & ICC_SGI1R_SGI_ID_MASK) >> ICC_SGI1R_SGI_ID_SHIFT; | |
e533a37f | 804 | broadcast = reg & BIT_ULL(ICC_SGI1R_IRQ_ROUTING_MODE_BIT); |
621ecd8d AP |
805 | target_cpus = (reg & ICC_SGI1R_TARGET_LIST_MASK) >> ICC_SGI1R_TARGET_LIST_SHIFT; |
806 | mpidr = SGI_AFFINITY_LEVEL(reg, 3); | |
807 | mpidr |= SGI_AFFINITY_LEVEL(reg, 2); | |
808 | mpidr |= SGI_AFFINITY_LEVEL(reg, 1); | |
809 | ||
810 | /* | |
811 | * We iterate over all VCPUs to find the MPIDRs matching the request. | |
812 | * If we have handled one CPU, we clear its bit to detect early | |
813 | * if we are already finished. This avoids iterating through all | |
814 | * VCPUs when most of the times we just signal a single VCPU. | |
815 | */ | |
816 | kvm_for_each_vcpu(c, c_vcpu, kvm) { | |
817 | struct vgic_irq *irq; | |
818 | ||
819 | /* Exit early if we have dealt with all requested CPUs */ | |
820 | if (!broadcast && target_cpus == 0) | |
821 | break; | |
822 | ||
823 | /* Don't signal the calling VCPU */ | |
824 | if (broadcast && c == vcpu_id) | |
825 | continue; | |
826 | ||
827 | if (!broadcast) { | |
828 | int level0; | |
829 | ||
830 | level0 = match_mpidr(mpidr, target_cpus, c_vcpu); | |
831 | if (level0 == -1) | |
832 | continue; | |
833 | ||
834 | /* remove this matching VCPU from the mask */ | |
835 | target_cpus &= ~BIT(level0); | |
836 | } | |
837 | ||
838 | irq = vgic_get_irq(vcpu->kvm, c_vcpu, sgi); | |
839 | ||
840 | spin_lock(&irq->irq_lock); | |
8694e4da | 841 | irq->pending_latch = true; |
621ecd8d AP |
842 | |
843 | vgic_queue_irq_unlock(vcpu->kvm, irq); | |
5dd4b924 | 844 | vgic_put_irq(vcpu->kvm, irq); |
621ecd8d AP |
845 | } |
846 | } | |
94574c94 VK |
847 | |
848 | int vgic_v3_dist_uaccess(struct kvm_vcpu *vcpu, bool is_write, | |
849 | int offset, u32 *val) | |
850 | { | |
851 | struct vgic_io_device dev = { | |
852 | .regions = vgic_v3_dist_registers, | |
853 | .nr_regions = ARRAY_SIZE(vgic_v3_dist_registers), | |
854 | }; | |
855 | ||
856 | return vgic_uaccess(vcpu, &dev, is_write, offset, val); | |
857 | } | |
858 | ||
859 | int vgic_v3_redist_uaccess(struct kvm_vcpu *vcpu, bool is_write, | |
860 | int offset, u32 *val) | |
861 | { | |
862 | struct vgic_io_device rd_dev = { | |
863 | .regions = vgic_v3_rdbase_registers, | |
864 | .nr_regions = ARRAY_SIZE(vgic_v3_rdbase_registers), | |
865 | }; | |
866 | ||
867 | struct vgic_io_device sgi_dev = { | |
868 | .regions = vgic_v3_sgibase_registers, | |
869 | .nr_regions = ARRAY_SIZE(vgic_v3_sgibase_registers), | |
870 | }; | |
871 | ||
872 | /* SGI_base is the next 64K frame after RD_base */ | |
873 | if (offset >= SZ_64K) | |
874 | return vgic_uaccess(vcpu, &sgi_dev, is_write, offset - SZ_64K, | |
875 | val); | |
876 | else | |
877 | return vgic_uaccess(vcpu, &rd_dev, is_write, offset, val); | |
878 | } | |
e96a006c VK |
879 | |
880 | int vgic_v3_line_level_info_uaccess(struct kvm_vcpu *vcpu, bool is_write, | |
881 | u32 intid, u64 *val) | |
882 | { | |
883 | if (intid % 32) | |
884 | return -EINVAL; | |
885 | ||
886 | if (is_write) | |
887 | vgic_write_irq_line_level_info(vcpu, intid, *val); | |
888 | else | |
889 | *val = vgic_read_irq_line_level_info(vcpu, intid); | |
890 | ||
891 | return 0; | |
892 | } |