]>
Commit | Line | Data |
---|---|---|
a7bf3034 PF |
1 | /* |
2 | * ARM Generic Interrupt Controller using KVM in-kernel support | |
3 | * | |
4 | * Copyright (c) 2015 Samsung Electronics Co., Ltd. | |
5 | * Written by Pavel Fedin | |
6 | * Based on vGICv2 code by Peter Maydell | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or modify | |
9 | * it under the terms of the GNU General Public License as published by | |
10 | * the Free Software Foundation, either version 2 of the License, or | |
11 | * (at your option) any later version. | |
12 | * | |
13 | * This program is distributed in the hope that it will be useful, | |
14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
16 | * GNU General Public License for more details. | |
17 | * | |
18 | * You should have received a copy of the GNU General Public License along | |
19 | * with this program; if not, see <http://www.gnu.org/licenses/>. | |
20 | */ | |
21 | ||
8ef94f0b | 22 | #include "qemu/osdep.h" |
da34e65c | 23 | #include "qapi/error.h" |
a7bf3034 PF |
24 | #include "hw/intc/arm_gicv3_common.h" |
25 | #include "hw/sysbus.h" | |
367b9f52 | 26 | #include "qemu/error-report.h" |
a7bf3034 | 27 | #include "sysemu/kvm.h" |
d5aa0c22 | 28 | #include "sysemu/sysemu.h" |
a7bf3034 | 29 | #include "kvm_arm.h" |
367b9f52 | 30 | #include "gicv3_internal.h" |
a7bf3034 | 31 | #include "vgic_common.h" |
795c40b8 | 32 | #include "migration/blocker.h" |
a7bf3034 PF |
33 | |
34 | #ifdef DEBUG_GICV3_KVM | |
35 | #define DPRINTF(fmt, ...) \ | |
36 | do { fprintf(stderr, "kvm_gicv3: " fmt, ## __VA_ARGS__); } while (0) | |
37 | #else | |
38 | #define DPRINTF(fmt, ...) \ | |
39 | do { } while (0) | |
40 | #endif | |
41 | ||
42 | #define TYPE_KVM_ARM_GICV3 "kvm-arm-gicv3" | |
43 | #define KVM_ARM_GICV3(obj) \ | |
44 | OBJECT_CHECK(GICv3State, (obj), TYPE_KVM_ARM_GICV3) | |
45 | #define KVM_ARM_GICV3_CLASS(klass) \ | |
46 | OBJECT_CLASS_CHECK(KVMARMGICv3Class, (klass), TYPE_KVM_ARM_GICV3) | |
47 | #define KVM_ARM_GICV3_GET_CLASS(obj) \ | |
48 | OBJECT_GET_CLASS(KVMARMGICv3Class, (obj), TYPE_KVM_ARM_GICV3) | |
49 | ||
367b9f52 VK |
50 | #define KVM_DEV_ARM_VGIC_SYSREG(op0, op1, crn, crm, op2) \ |
51 | (ARM64_SYS_REG_SHIFT_MASK(op0, OP0) | \ | |
52 | ARM64_SYS_REG_SHIFT_MASK(op1, OP1) | \ | |
53 | ARM64_SYS_REG_SHIFT_MASK(crn, CRN) | \ | |
54 | ARM64_SYS_REG_SHIFT_MASK(crm, CRM) | \ | |
55 | ARM64_SYS_REG_SHIFT_MASK(op2, OP2)) | |
56 | ||
57 | #define ICC_PMR_EL1 \ | |
58 | KVM_DEV_ARM_VGIC_SYSREG(3, 0, 4, 6, 0) | |
59 | #define ICC_BPR0_EL1 \ | |
60 | KVM_DEV_ARM_VGIC_SYSREG(3, 0, 12, 8, 3) | |
61 | #define ICC_AP0R_EL1(n) \ | |
62 | KVM_DEV_ARM_VGIC_SYSREG(3, 0, 12, 8, 4 | n) | |
63 | #define ICC_AP1R_EL1(n) \ | |
64 | KVM_DEV_ARM_VGIC_SYSREG(3, 0, 12, 9, n) | |
65 | #define ICC_BPR1_EL1 \ | |
66 | KVM_DEV_ARM_VGIC_SYSREG(3, 0, 12, 12, 3) | |
67 | #define ICC_CTLR_EL1 \ | |
68 | KVM_DEV_ARM_VGIC_SYSREG(3, 0, 12, 12, 4) | |
69 | #define ICC_SRE_EL1 \ | |
70 | KVM_DEV_ARM_VGIC_SYSREG(3, 0, 12, 12, 5) | |
71 | #define ICC_IGRPEN0_EL1 \ | |
72 | KVM_DEV_ARM_VGIC_SYSREG(3, 0, 12, 12, 6) | |
73 | #define ICC_IGRPEN1_EL1 \ | |
74 | KVM_DEV_ARM_VGIC_SYSREG(3, 0, 12, 12, 7) | |
75 | ||
a7bf3034 PF |
76 | typedef struct KVMARMGICv3Class { |
77 | ARMGICv3CommonClass parent_class; | |
78 | DeviceRealize parent_realize; | |
79 | void (*parent_reset)(DeviceState *dev); | |
80 | } KVMARMGICv3Class; | |
81 | ||
82 | static void kvm_arm_gicv3_set_irq(void *opaque, int irq, int level) | |
83 | { | |
84 | GICv3State *s = (GICv3State *)opaque; | |
85 | ||
86 | kvm_arm_gic_set_irq(s->num_irq, irq, level); | |
87 | } | |
88 | ||
367b9f52 VK |
89 | #define KVM_VGIC_ATTR(reg, typer) \ |
90 | ((typer & KVM_DEV_ARM_VGIC_V3_MPIDR_MASK) | (reg)) | |
91 | ||
92 | static inline void kvm_gicd_access(GICv3State *s, int offset, | |
93 | uint32_t *val, bool write) | |
94 | { | |
95 | kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_DIST_REGS, | |
96 | KVM_VGIC_ATTR(offset, 0), | |
556969e9 | 97 | val, write, &error_abort); |
367b9f52 VK |
98 | } |
99 | ||
100 | static inline void kvm_gicr_access(GICv3State *s, int offset, int cpu, | |
101 | uint32_t *val, bool write) | |
102 | { | |
103 | kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_REDIST_REGS, | |
104 | KVM_VGIC_ATTR(offset, s->cpu[cpu].gicr_typer), | |
556969e9 | 105 | val, write, &error_abort); |
367b9f52 VK |
106 | } |
107 | ||
108 | static inline void kvm_gicc_access(GICv3State *s, uint64_t reg, int cpu, | |
109 | uint64_t *val, bool write) | |
110 | { | |
111 | kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, | |
112 | KVM_VGIC_ATTR(reg, s->cpu[cpu].gicr_typer), | |
556969e9 | 113 | val, write, &error_abort); |
367b9f52 VK |
114 | } |
115 | ||
116 | static inline void kvm_gic_line_level_access(GICv3State *s, int irq, int cpu, | |
117 | uint32_t *val, bool write) | |
118 | { | |
119 | kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO, | |
120 | KVM_VGIC_ATTR(irq, s->cpu[cpu].gicr_typer) | | |
121 | (VGIC_LEVEL_INFO_LINE_LEVEL << | |
122 | KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT), | |
556969e9 | 123 | val, write, &error_abort); |
367b9f52 VK |
124 | } |
125 | ||
126 | /* Loop through each distributor IRQ related register; since bits | |
127 | * corresponding to SPIs and PPIs are RAZ/WI when affinity routing | |
128 | * is enabled, we skip those. | |
129 | */ | |
130 | #define for_each_dist_irq_reg(_irq, _max, _field_width) \ | |
131 | for (_irq = GIC_INTERNAL; _irq < _max; _irq += (32 / _field_width)) | |
132 | ||
133 | static void kvm_dist_get_priority(GICv3State *s, uint32_t offset, uint8_t *bmp) | |
134 | { | |
135 | uint32_t reg, *field; | |
136 | int irq; | |
137 | ||
138 | field = (uint32_t *)bmp; | |
139 | for_each_dist_irq_reg(irq, s->num_irq, 8) { | |
140 | kvm_gicd_access(s, offset, ®, false); | |
141 | *field = reg; | |
142 | offset += 4; | |
143 | field++; | |
144 | } | |
145 | } | |
146 | ||
147 | static void kvm_dist_put_priority(GICv3State *s, uint32_t offset, uint8_t *bmp) | |
148 | { | |
149 | uint32_t reg, *field; | |
150 | int irq; | |
151 | ||
152 | field = (uint32_t *)bmp; | |
153 | for_each_dist_irq_reg(irq, s->num_irq, 8) { | |
154 | reg = *field; | |
155 | kvm_gicd_access(s, offset, ®, true); | |
156 | offset += 4; | |
157 | field++; | |
158 | } | |
159 | } | |
160 | ||
161 | static void kvm_dist_get_edge_trigger(GICv3State *s, uint32_t offset, | |
162 | uint32_t *bmp) | |
163 | { | |
164 | uint32_t reg; | |
165 | int irq; | |
166 | ||
5d639531 SZ |
167 | /* For the KVM GICv3, affinity routing is always enabled, and the first 2 |
168 | * GICD_ICFGR<n> registers are always RAZ/WI. The corresponding | |
169 | * functionality is replaced by GICR_ICFGR<n>. It doesn't need to sync | |
170 | * them. So it should increase the offset to skip GIC_INTERNAL irqs. | |
171 | * This matches the for_each_dist_irq_reg() macro which also skips the | |
172 | * first GIC_INTERNAL irqs. | |
173 | */ | |
174 | offset += (GIC_INTERNAL * 2) / 8; | |
367b9f52 VK |
175 | for_each_dist_irq_reg(irq, s->num_irq, 2) { |
176 | kvm_gicd_access(s, offset, ®, false); | |
177 | reg = half_unshuffle32(reg >> 1); | |
178 | if (irq % 32 != 0) { | |
179 | reg = (reg << 16); | |
180 | } | |
181 | *gic_bmp_ptr32(bmp, irq) |= reg; | |
182 | offset += 4; | |
183 | } | |
184 | } | |
185 | ||
186 | static void kvm_dist_put_edge_trigger(GICv3State *s, uint32_t offset, | |
187 | uint32_t *bmp) | |
188 | { | |
189 | uint32_t reg; | |
190 | int irq; | |
191 | ||
5d639531 SZ |
192 | /* For the KVM GICv3, affinity routing is always enabled, and the first 2 |
193 | * GICD_ICFGR<n> registers are always RAZ/WI. The corresponding | |
194 | * functionality is replaced by GICR_ICFGR<n>. It doesn't need to sync | |
195 | * them. So it should increase the offset to skip GIC_INTERNAL irqs. | |
196 | * This matches the for_each_dist_irq_reg() macro which also skips the | |
197 | * first GIC_INTERNAL irqs. | |
198 | */ | |
199 | offset += (GIC_INTERNAL * 2) / 8; | |
367b9f52 VK |
200 | for_each_dist_irq_reg(irq, s->num_irq, 2) { |
201 | reg = *gic_bmp_ptr32(bmp, irq); | |
202 | if (irq % 32 != 0) { | |
203 | reg = (reg & 0xffff0000) >> 16; | |
204 | } else { | |
205 | reg = reg & 0xffff; | |
206 | } | |
207 | reg = half_shuffle32(reg) << 1; | |
208 | kvm_gicd_access(s, offset, ®, true); | |
209 | offset += 4; | |
210 | } | |
211 | } | |
212 | ||
213 | static void kvm_gic_get_line_level_bmp(GICv3State *s, uint32_t *bmp) | |
214 | { | |
215 | uint32_t reg; | |
216 | int irq; | |
217 | ||
218 | for_each_dist_irq_reg(irq, s->num_irq, 1) { | |
219 | kvm_gic_line_level_access(s, irq, 0, ®, false); | |
220 | *gic_bmp_ptr32(bmp, irq) = reg; | |
221 | } | |
222 | } | |
223 | ||
224 | static void kvm_gic_put_line_level_bmp(GICv3State *s, uint32_t *bmp) | |
225 | { | |
226 | uint32_t reg; | |
227 | int irq; | |
228 | ||
229 | for_each_dist_irq_reg(irq, s->num_irq, 1) { | |
230 | reg = *gic_bmp_ptr32(bmp, irq); | |
231 | kvm_gic_line_level_access(s, irq, 0, ®, true); | |
232 | } | |
233 | } | |
234 | ||
235 | /* Read a bitmap register group from the kernel VGIC. */ | |
236 | static void kvm_dist_getbmp(GICv3State *s, uint32_t offset, uint32_t *bmp) | |
237 | { | |
238 | uint32_t reg; | |
239 | int irq; | |
240 | ||
5d639531 SZ |
241 | /* For the KVM GICv3, affinity routing is always enabled, and the |
242 | * GICD_IGROUPR0/GICD_IGRPMODR0/GICD_ISENABLER0/GICD_ISPENDR0/ | |
243 | * GICD_ISACTIVER0 registers are always RAZ/WI. The corresponding | |
244 | * functionality is replaced by the GICR registers. It doesn't need to sync | |
245 | * them. So it should increase the offset to skip GIC_INTERNAL irqs. | |
246 | * This matches the for_each_dist_irq_reg() macro which also skips the | |
247 | * first GIC_INTERNAL irqs. | |
248 | */ | |
249 | offset += (GIC_INTERNAL * 1) / 8; | |
367b9f52 VK |
250 | for_each_dist_irq_reg(irq, s->num_irq, 1) { |
251 | kvm_gicd_access(s, offset, ®, false); | |
252 | *gic_bmp_ptr32(bmp, irq) = reg; | |
253 | offset += 4; | |
254 | } | |
255 | } | |
256 | ||
257 | static void kvm_dist_putbmp(GICv3State *s, uint32_t offset, | |
258 | uint32_t clroffset, uint32_t *bmp) | |
259 | { | |
260 | uint32_t reg; | |
261 | int irq; | |
262 | ||
5d639531 SZ |
263 | /* For the KVM GICv3, affinity routing is always enabled, and the |
264 | * GICD_IGROUPR0/GICD_IGRPMODR0/GICD_ISENABLER0/GICD_ISPENDR0/ | |
265 | * GICD_ISACTIVER0 registers are always RAZ/WI. The corresponding | |
266 | * functionality is replaced by the GICR registers. It doesn't need to sync | |
267 | * them. So it should increase the offset and clroffset to skip GIC_INTERNAL | |
268 | * irqs. This matches the for_each_dist_irq_reg() macro which also skips the | |
269 | * first GIC_INTERNAL irqs. | |
270 | */ | |
271 | offset += (GIC_INTERNAL * 1) / 8; | |
272 | if (clroffset != 0) { | |
273 | clroffset += (GIC_INTERNAL * 1) / 8; | |
274 | } | |
275 | ||
367b9f52 VK |
276 | for_each_dist_irq_reg(irq, s->num_irq, 1) { |
277 | /* If this bitmap is a set/clear register pair, first write to the | |
278 | * clear-reg to clear all bits before using the set-reg to write | |
279 | * the 1 bits. | |
280 | */ | |
281 | if (clroffset != 0) { | |
282 | reg = 0; | |
283 | kvm_gicd_access(s, clroffset, ®, true); | |
e55c6b67 | 284 | clroffset += 4; |
367b9f52 VK |
285 | } |
286 | reg = *gic_bmp_ptr32(bmp, irq); | |
287 | kvm_gicd_access(s, offset, ®, true); | |
288 | offset += 4; | |
289 | } | |
290 | } | |
291 | ||
292 | static void kvm_arm_gicv3_check(GICv3State *s) | |
293 | { | |
294 | uint32_t reg; | |
295 | uint32_t num_irq; | |
296 | ||
297 | /* Sanity checking s->num_irq */ | |
298 | kvm_gicd_access(s, GICD_TYPER, ®, false); | |
299 | num_irq = ((reg & 0x1f) + 1) * 32; | |
300 | ||
301 | if (num_irq < s->num_irq) { | |
302 | error_report("Model requests %u IRQs, but kernel supports max %u", | |
303 | s->num_irq, num_irq); | |
304 | abort(); | |
305 | } | |
306 | } | |
307 | ||
a7bf3034 PF |
308 | static void kvm_arm_gicv3_put(GICv3State *s) |
309 | { | |
367b9f52 VK |
310 | uint32_t regl, regh, reg; |
311 | uint64_t reg64, redist_typer; | |
312 | int ncpu, i; | |
313 | ||
314 | kvm_arm_gicv3_check(s); | |
315 | ||
316 | kvm_gicr_access(s, GICR_TYPER, 0, ®l, false); | |
317 | kvm_gicr_access(s, GICR_TYPER + 4, 0, ®h, false); | |
318 | redist_typer = ((uint64_t)regh << 32) | regl; | |
319 | ||
320 | reg = s->gicd_ctlr; | |
321 | kvm_gicd_access(s, GICD_CTLR, ®, true); | |
322 | ||
323 | if (redist_typer & GICR_TYPER_PLPIS) { | |
324 | /* Set base addresses before LPIs are enabled by GICR_CTLR write */ | |
325 | for (ncpu = 0; ncpu < s->num_cpu; ncpu++) { | |
326 | GICv3CPUState *c = &s->cpu[ncpu]; | |
327 | ||
328 | reg64 = c->gicr_propbaser; | |
329 | regl = (uint32_t)reg64; | |
330 | kvm_gicr_access(s, GICR_PROPBASER, ncpu, ®l, true); | |
331 | regh = (uint32_t)(reg64 >> 32); | |
332 | kvm_gicr_access(s, GICR_PROPBASER + 4, ncpu, ®h, true); | |
333 | ||
334 | reg64 = c->gicr_pendbaser; | |
7229ec58 | 335 | if (!(c->gicr_ctlr & GICR_CTLR_ENABLE_LPIS)) { |
367b9f52 VK |
336 | /* Setting PTZ is advised if LPIs are disabled, to reduce |
337 | * GIC initialization time. | |
338 | */ | |
339 | reg64 |= GICR_PENDBASER_PTZ; | |
340 | } | |
341 | regl = (uint32_t)reg64; | |
342 | kvm_gicr_access(s, GICR_PENDBASER, ncpu, ®l, true); | |
343 | regh = (uint32_t)(reg64 >> 32); | |
344 | kvm_gicr_access(s, GICR_PENDBASER + 4, ncpu, ®h, true); | |
345 | } | |
346 | } | |
347 | ||
348 | /* Redistributor state (one per CPU) */ | |
349 | ||
350 | for (ncpu = 0; ncpu < s->num_cpu; ncpu++) { | |
351 | GICv3CPUState *c = &s->cpu[ncpu]; | |
352 | ||
353 | reg = c->gicr_ctlr; | |
354 | kvm_gicr_access(s, GICR_CTLR, ncpu, ®, true); | |
355 | ||
356 | reg = c->gicr_statusr[GICV3_NS]; | |
357 | kvm_gicr_access(s, GICR_STATUSR, ncpu, ®, true); | |
358 | ||
359 | reg = c->gicr_waker; | |
360 | kvm_gicr_access(s, GICR_WAKER, ncpu, ®, true); | |
361 | ||
362 | reg = c->gicr_igroupr0; | |
363 | kvm_gicr_access(s, GICR_IGROUPR0, ncpu, ®, true); | |
364 | ||
365 | reg = ~0; | |
366 | kvm_gicr_access(s, GICR_ICENABLER0, ncpu, ®, true); | |
367 | reg = c->gicr_ienabler0; | |
368 | kvm_gicr_access(s, GICR_ISENABLER0, ncpu, ®, true); | |
369 | ||
370 | /* Restore config before pending so we treat level/edge correctly */ | |
371 | reg = half_shuffle32(c->edge_trigger >> 16) << 1; | |
372 | kvm_gicr_access(s, GICR_ICFGR1, ncpu, ®, true); | |
373 | ||
374 | reg = c->level; | |
375 | kvm_gic_line_level_access(s, 0, ncpu, ®, true); | |
376 | ||
377 | reg = ~0; | |
378 | kvm_gicr_access(s, GICR_ICPENDR0, ncpu, ®, true); | |
379 | reg = c->gicr_ipendr0; | |
380 | kvm_gicr_access(s, GICR_ISPENDR0, ncpu, ®, true); | |
381 | ||
382 | reg = ~0; | |
383 | kvm_gicr_access(s, GICR_ICACTIVER0, ncpu, ®, true); | |
384 | reg = c->gicr_iactiver0; | |
385 | kvm_gicr_access(s, GICR_ISACTIVER0, ncpu, ®, true); | |
386 | ||
387 | for (i = 0; i < GIC_INTERNAL; i += 4) { | |
388 | reg = c->gicr_ipriorityr[i] | | |
389 | (c->gicr_ipriorityr[i + 1] << 8) | | |
390 | (c->gicr_ipriorityr[i + 2] << 16) | | |
391 | (c->gicr_ipriorityr[i + 3] << 24); | |
392 | kvm_gicr_access(s, GICR_IPRIORITYR + i, ncpu, ®, true); | |
393 | } | |
394 | } | |
395 | ||
396 | /* Distributor state (shared between all CPUs */ | |
397 | reg = s->gicd_statusr[GICV3_NS]; | |
398 | kvm_gicd_access(s, GICD_STATUSR, ®, true); | |
399 | ||
400 | /* s->enable bitmap -> GICD_ISENABLERn */ | |
401 | kvm_dist_putbmp(s, GICD_ISENABLER, GICD_ICENABLER, s->enabled); | |
402 | ||
403 | /* s->group bitmap -> GICD_IGROUPRn */ | |
404 | kvm_dist_putbmp(s, GICD_IGROUPR, 0, s->group); | |
405 | ||
406 | /* Restore targets before pending to ensure the pending state is set on | |
407 | * the appropriate CPU interfaces in the kernel | |
408 | */ | |
409 | ||
410 | /* s->gicd_irouter[irq] -> GICD_IROUTERn | |
411 | * We can't use kvm_dist_put() here because the registers are 64-bit | |
412 | */ | |
413 | for (i = GIC_INTERNAL; i < s->num_irq; i++) { | |
414 | uint32_t offset; | |
415 | ||
416 | offset = GICD_IROUTER + (sizeof(uint32_t) * i); | |
417 | reg = (uint32_t)s->gicd_irouter[i]; | |
418 | kvm_gicd_access(s, offset, ®, true); | |
419 | ||
420 | offset = GICD_IROUTER + (sizeof(uint32_t) * i) + 4; | |
421 | reg = (uint32_t)(s->gicd_irouter[i] >> 32); | |
422 | kvm_gicd_access(s, offset, ®, true); | |
423 | } | |
424 | ||
425 | /* s->trigger bitmap -> GICD_ICFGRn | |
426 | * (restore configuration registers before pending IRQs so we treat | |
427 | * level/edge correctly) | |
428 | */ | |
429 | kvm_dist_put_edge_trigger(s, GICD_ICFGR, s->edge_trigger); | |
430 | ||
431 | /* s->level bitmap -> line_level */ | |
432 | kvm_gic_put_line_level_bmp(s, s->level); | |
433 | ||
434 | /* s->pending bitmap -> GICD_ISPENDRn */ | |
435 | kvm_dist_putbmp(s, GICD_ISPENDR, GICD_ICPENDR, s->pending); | |
436 | ||
437 | /* s->active bitmap -> GICD_ISACTIVERn */ | |
438 | kvm_dist_putbmp(s, GICD_ISACTIVER, GICD_ICACTIVER, s->active); | |
439 | ||
440 | /* s->gicd_ipriority[] -> GICD_IPRIORITYRn */ | |
441 | kvm_dist_put_priority(s, GICD_IPRIORITYR, s->gicd_ipriority); | |
442 | ||
443 | /* CPU Interface state (one per CPU) */ | |
444 | ||
445 | for (ncpu = 0; ncpu < s->num_cpu; ncpu++) { | |
446 | GICv3CPUState *c = &s->cpu[ncpu]; | |
447 | int num_pri_bits; | |
448 | ||
449 | kvm_gicc_access(s, ICC_SRE_EL1, ncpu, &c->icc_sre_el1, true); | |
450 | kvm_gicc_access(s, ICC_CTLR_EL1, ncpu, | |
451 | &c->icc_ctlr_el1[GICV3_NS], true); | |
452 | kvm_gicc_access(s, ICC_IGRPEN0_EL1, ncpu, | |
453 | &c->icc_igrpen[GICV3_G0], true); | |
454 | kvm_gicc_access(s, ICC_IGRPEN1_EL1, ncpu, | |
455 | &c->icc_igrpen[GICV3_G1NS], true); | |
456 | kvm_gicc_access(s, ICC_PMR_EL1, ncpu, &c->icc_pmr_el1, true); | |
457 | kvm_gicc_access(s, ICC_BPR0_EL1, ncpu, &c->icc_bpr[GICV3_G0], true); | |
458 | kvm_gicc_access(s, ICC_BPR1_EL1, ncpu, &c->icc_bpr[GICV3_G1NS], true); | |
459 | ||
460 | num_pri_bits = ((c->icc_ctlr_el1[GICV3_NS] & | |
461 | ICC_CTLR_EL1_PRIBITS_MASK) >> | |
462 | ICC_CTLR_EL1_PRIBITS_SHIFT) + 1; | |
463 | ||
464 | switch (num_pri_bits) { | |
465 | case 7: | |
466 | reg64 = c->icc_apr[GICV3_G0][3]; | |
467 | kvm_gicc_access(s, ICC_AP0R_EL1(3), ncpu, ®64, true); | |
468 | reg64 = c->icc_apr[GICV3_G0][2]; | |
469 | kvm_gicc_access(s, ICC_AP0R_EL1(2), ncpu, ®64, true); | |
470 | case 6: | |
471 | reg64 = c->icc_apr[GICV3_G0][1]; | |
472 | kvm_gicc_access(s, ICC_AP0R_EL1(1), ncpu, ®64, true); | |
473 | default: | |
474 | reg64 = c->icc_apr[GICV3_G0][0]; | |
475 | kvm_gicc_access(s, ICC_AP0R_EL1(0), ncpu, ®64, true); | |
476 | } | |
477 | ||
478 | switch (num_pri_bits) { | |
479 | case 7: | |
480 | reg64 = c->icc_apr[GICV3_G1NS][3]; | |
481 | kvm_gicc_access(s, ICC_AP1R_EL1(3), ncpu, ®64, true); | |
482 | reg64 = c->icc_apr[GICV3_G1NS][2]; | |
483 | kvm_gicc_access(s, ICC_AP1R_EL1(2), ncpu, ®64, true); | |
484 | case 6: | |
485 | reg64 = c->icc_apr[GICV3_G1NS][1]; | |
486 | kvm_gicc_access(s, ICC_AP1R_EL1(1), ncpu, ®64, true); | |
487 | default: | |
488 | reg64 = c->icc_apr[GICV3_G1NS][0]; | |
489 | kvm_gicc_access(s, ICC_AP1R_EL1(0), ncpu, ®64, true); | |
490 | } | |
491 | } | |
a7bf3034 PF |
492 | } |
493 | ||
494 | static void kvm_arm_gicv3_get(GICv3State *s) | |
495 | { | |
367b9f52 VK |
496 | uint32_t regl, regh, reg; |
497 | uint64_t reg64, redist_typer; | |
498 | int ncpu, i; | |
499 | ||
500 | kvm_arm_gicv3_check(s); | |
501 | ||
502 | kvm_gicr_access(s, GICR_TYPER, 0, ®l, false); | |
503 | kvm_gicr_access(s, GICR_TYPER + 4, 0, ®h, false); | |
504 | redist_typer = ((uint64_t)regh << 32) | regl; | |
505 | ||
506 | kvm_gicd_access(s, GICD_CTLR, ®, false); | |
507 | s->gicd_ctlr = reg; | |
508 | ||
509 | /* Redistributor state (one per CPU) */ | |
510 | ||
511 | for (ncpu = 0; ncpu < s->num_cpu; ncpu++) { | |
512 | GICv3CPUState *c = &s->cpu[ncpu]; | |
513 | ||
514 | kvm_gicr_access(s, GICR_CTLR, ncpu, ®, false); | |
515 | c->gicr_ctlr = reg; | |
516 | ||
517 | kvm_gicr_access(s, GICR_STATUSR, ncpu, ®, false); | |
518 | c->gicr_statusr[GICV3_NS] = reg; | |
519 | ||
520 | kvm_gicr_access(s, GICR_WAKER, ncpu, ®, false); | |
521 | c->gicr_waker = reg; | |
522 | ||
523 | kvm_gicr_access(s, GICR_IGROUPR0, ncpu, ®, false); | |
524 | c->gicr_igroupr0 = reg; | |
525 | kvm_gicr_access(s, GICR_ISENABLER0, ncpu, ®, false); | |
526 | c->gicr_ienabler0 = reg; | |
527 | kvm_gicr_access(s, GICR_ICFGR1, ncpu, ®, false); | |
528 | c->edge_trigger = half_unshuffle32(reg >> 1) << 16; | |
529 | kvm_gic_line_level_access(s, 0, ncpu, ®, false); | |
530 | c->level = reg; | |
531 | kvm_gicr_access(s, GICR_ISPENDR0, ncpu, ®, false); | |
532 | c->gicr_ipendr0 = reg; | |
533 | kvm_gicr_access(s, GICR_ISACTIVER0, ncpu, ®, false); | |
534 | c->gicr_iactiver0 = reg; | |
535 | ||
536 | for (i = 0; i < GIC_INTERNAL; i += 4) { | |
537 | kvm_gicr_access(s, GICR_IPRIORITYR + i, ncpu, ®, false); | |
538 | c->gicr_ipriorityr[i] = extract32(reg, 0, 8); | |
539 | c->gicr_ipriorityr[i + 1] = extract32(reg, 8, 8); | |
540 | c->gicr_ipriorityr[i + 2] = extract32(reg, 16, 8); | |
541 | c->gicr_ipriorityr[i + 3] = extract32(reg, 24, 8); | |
542 | } | |
543 | } | |
544 | ||
545 | if (redist_typer & GICR_TYPER_PLPIS) { | |
546 | for (ncpu = 0; ncpu < s->num_cpu; ncpu++) { | |
547 | GICv3CPUState *c = &s->cpu[ncpu]; | |
548 | ||
549 | kvm_gicr_access(s, GICR_PROPBASER, ncpu, ®l, false); | |
550 | kvm_gicr_access(s, GICR_PROPBASER + 4, ncpu, ®h, false); | |
551 | c->gicr_propbaser = ((uint64_t)regh << 32) | regl; | |
552 | ||
553 | kvm_gicr_access(s, GICR_PENDBASER, ncpu, ®l, false); | |
554 | kvm_gicr_access(s, GICR_PENDBASER + 4, ncpu, ®h, false); | |
555 | c->gicr_pendbaser = ((uint64_t)regh << 32) | regl; | |
556 | } | |
557 | } | |
558 | ||
559 | /* Distributor state (shared between all CPUs */ | |
560 | ||
561 | kvm_gicd_access(s, GICD_STATUSR, ®, false); | |
562 | s->gicd_statusr[GICV3_NS] = reg; | |
563 | ||
564 | /* GICD_IGROUPRn -> s->group bitmap */ | |
565 | kvm_dist_getbmp(s, GICD_IGROUPR, s->group); | |
566 | ||
567 | /* GICD_ISENABLERn -> s->enabled bitmap */ | |
568 | kvm_dist_getbmp(s, GICD_ISENABLER, s->enabled); | |
569 | ||
570 | /* Line level of irq */ | |
571 | kvm_gic_get_line_level_bmp(s, s->level); | |
572 | /* GICD_ISPENDRn -> s->pending bitmap */ | |
573 | kvm_dist_getbmp(s, GICD_ISPENDR, s->pending); | |
574 | ||
575 | /* GICD_ISACTIVERn -> s->active bitmap */ | |
576 | kvm_dist_getbmp(s, GICD_ISACTIVER, s->active); | |
577 | ||
578 | /* GICD_ICFGRn -> s->trigger bitmap */ | |
579 | kvm_dist_get_edge_trigger(s, GICD_ICFGR, s->edge_trigger); | |
580 | ||
581 | /* GICD_IPRIORITYRn -> s->gicd_ipriority[] */ | |
582 | kvm_dist_get_priority(s, GICD_IPRIORITYR, s->gicd_ipriority); | |
583 | ||
584 | /* GICD_IROUTERn -> s->gicd_irouter[irq] */ | |
585 | for (i = GIC_INTERNAL; i < s->num_irq; i++) { | |
586 | uint32_t offset; | |
587 | ||
588 | offset = GICD_IROUTER + (sizeof(uint32_t) * i); | |
589 | kvm_gicd_access(s, offset, ®l, false); | |
590 | offset = GICD_IROUTER + (sizeof(uint32_t) * i) + 4; | |
591 | kvm_gicd_access(s, offset, ®h, false); | |
592 | s->gicd_irouter[i] = ((uint64_t)regh << 32) | regl; | |
593 | } | |
594 | ||
595 | /***************************************************************** | |
596 | * CPU Interface(s) State | |
597 | */ | |
598 | ||
599 | for (ncpu = 0; ncpu < s->num_cpu; ncpu++) { | |
600 | GICv3CPUState *c = &s->cpu[ncpu]; | |
601 | int num_pri_bits; | |
602 | ||
603 | kvm_gicc_access(s, ICC_SRE_EL1, ncpu, &c->icc_sre_el1, false); | |
604 | kvm_gicc_access(s, ICC_CTLR_EL1, ncpu, | |
605 | &c->icc_ctlr_el1[GICV3_NS], false); | |
606 | kvm_gicc_access(s, ICC_IGRPEN0_EL1, ncpu, | |
607 | &c->icc_igrpen[GICV3_G0], false); | |
608 | kvm_gicc_access(s, ICC_IGRPEN1_EL1, ncpu, | |
609 | &c->icc_igrpen[GICV3_G1NS], false); | |
610 | kvm_gicc_access(s, ICC_PMR_EL1, ncpu, &c->icc_pmr_el1, false); | |
611 | kvm_gicc_access(s, ICC_BPR0_EL1, ncpu, &c->icc_bpr[GICV3_G0], false); | |
612 | kvm_gicc_access(s, ICC_BPR1_EL1, ncpu, &c->icc_bpr[GICV3_G1NS], false); | |
613 | num_pri_bits = ((c->icc_ctlr_el1[GICV3_NS] & | |
614 | ICC_CTLR_EL1_PRIBITS_MASK) >> | |
615 | ICC_CTLR_EL1_PRIBITS_SHIFT) + 1; | |
616 | ||
617 | switch (num_pri_bits) { | |
618 | case 7: | |
619 | kvm_gicc_access(s, ICC_AP0R_EL1(3), ncpu, ®64, false); | |
620 | c->icc_apr[GICV3_G0][3] = reg64; | |
621 | kvm_gicc_access(s, ICC_AP0R_EL1(2), ncpu, ®64, false); | |
622 | c->icc_apr[GICV3_G0][2] = reg64; | |
623 | case 6: | |
624 | kvm_gicc_access(s, ICC_AP0R_EL1(1), ncpu, ®64, false); | |
625 | c->icc_apr[GICV3_G0][1] = reg64; | |
626 | default: | |
627 | kvm_gicc_access(s, ICC_AP0R_EL1(0), ncpu, ®64, false); | |
628 | c->icc_apr[GICV3_G0][0] = reg64; | |
629 | } | |
630 | ||
631 | switch (num_pri_bits) { | |
632 | case 7: | |
633 | kvm_gicc_access(s, ICC_AP1R_EL1(3), ncpu, ®64, false); | |
634 | c->icc_apr[GICV3_G1NS][3] = reg64; | |
635 | kvm_gicc_access(s, ICC_AP1R_EL1(2), ncpu, ®64, false); | |
636 | c->icc_apr[GICV3_G1NS][2] = reg64; | |
637 | case 6: | |
638 | kvm_gicc_access(s, ICC_AP1R_EL1(1), ncpu, ®64, false); | |
639 | c->icc_apr[GICV3_G1NS][1] = reg64; | |
640 | default: | |
641 | kvm_gicc_access(s, ICC_AP1R_EL1(0), ncpu, ®64, false); | |
642 | c->icc_apr[GICV3_G1NS][0] = reg64; | |
643 | } | |
644 | } | |
a7bf3034 PF |
645 | } |
646 | ||
07a5628c VK |
647 | static void arm_gicv3_icc_reset(CPUARMState *env, const ARMCPRegInfo *ri) |
648 | { | |
649 | ARMCPU *cpu; | |
650 | GICv3State *s; | |
651 | GICv3CPUState *c; | |
652 | ||
653 | c = (GICv3CPUState *)env->gicv3state; | |
654 | s = c->gic; | |
655 | cpu = ARM_CPU(c->cpu); | |
656 | ||
07a5628c VK |
657 | c->icc_pmr_el1 = 0; |
658 | c->icc_bpr[GICV3_G0] = GIC_MIN_BPR; | |
659 | c->icc_bpr[GICV3_G1] = GIC_MIN_BPR; | |
660 | c->icc_bpr[GICV3_G1NS] = GIC_MIN_BPR; | |
661 | ||
662 | c->icc_sre_el1 = 0x7; | |
663 | memset(c->icc_apr, 0, sizeof(c->icc_apr)); | |
664 | memset(c->icc_igrpen, 0, sizeof(c->icc_igrpen)); | |
e7d54416 EA |
665 | |
666 | if (s->migration_blocker) { | |
667 | return; | |
668 | } | |
669 | ||
670 | /* Initialize to actual HW supported configuration */ | |
671 | kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, | |
672 | KVM_VGIC_ATTR(ICC_CTLR_EL1, cpu->mp_affinity), | |
556969e9 | 673 | &c->icc_ctlr_el1[GICV3_NS], false, &error_abort); |
e7d54416 EA |
674 | |
675 | c->icc_ctlr_el1[GICV3_S] = c->icc_ctlr_el1[GICV3_NS]; | |
07a5628c VK |
676 | } |
677 | ||
a7bf3034 PF |
678 | static void kvm_arm_gicv3_reset(DeviceState *dev) |
679 | { | |
680 | GICv3State *s = ARM_GICV3_COMMON(dev); | |
681 | KVMARMGICv3Class *kgc = KVM_ARM_GICV3_GET_CLASS(s); | |
682 | ||
683 | DPRINTF("Reset\n"); | |
684 | ||
685 | kgc->parent_reset(dev); | |
367b9f52 VK |
686 | |
687 | if (s->migration_blocker) { | |
688 | DPRINTF("Cannot put kernel gic state, no kernel interface\n"); | |
689 | return; | |
690 | } | |
691 | ||
a7bf3034 PF |
692 | kvm_arm_gicv3_put(s); |
693 | } | |
694 | ||
07a5628c VK |
695 | /* |
696 | * CPU interface registers of GIC needs to be reset on CPU reset. | |
697 | * For the calling arm_gicv3_icc_reset() on CPU reset, we register | |
698 | * below ARMCPRegInfo. As we reset the whole cpu interface under single | |
699 | * register reset, we define only one register of CPU interface instead | |
700 | * of defining all the registers. | |
701 | */ | |
702 | static const ARMCPRegInfo gicv3_cpuif_reginfo[] = { | |
703 | { .name = "ICC_CTLR_EL1", .state = ARM_CP_STATE_BOTH, | |
704 | .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 12, .opc2 = 4, | |
705 | /* | |
706 | * If ARM_CP_NOP is used, resetfn is not called, | |
707 | * So ARM_CP_NO_RAW is appropriate type. | |
708 | */ | |
709 | .type = ARM_CP_NO_RAW, | |
710 | .access = PL1_RW, | |
711 | .readfn = arm_cp_read_zero, | |
712 | .writefn = arm_cp_write_ignore, | |
713 | /* | |
714 | * We hang the whole cpu interface reset routine off here | |
715 | * rather than parcelling it out into one little function | |
716 | * per register | |
717 | */ | |
718 | .resetfn = arm_gicv3_icc_reset, | |
719 | }, | |
720 | REGINFO_SENTINEL | |
721 | }; | |
722 | ||
d5aa0c22 EA |
723 | /** |
724 | * vm_change_state_handler - VM change state callback aiming at flushing | |
725 | * RDIST pending tables into guest RAM | |
726 | * | |
727 | * The tables get flushed to guest RAM whenever the VM gets stopped. | |
728 | */ | |
729 | static void vm_change_state_handler(void *opaque, int running, | |
730 | RunState state) | |
731 | { | |
732 | GICv3State *s = (GICv3State *)opaque; | |
733 | Error *err = NULL; | |
734 | int ret; | |
735 | ||
736 | if (running) { | |
737 | return; | |
738 | } | |
739 | ||
740 | ret = kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, | |
741 | KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES, | |
742 | NULL, true, &err); | |
743 | if (err) { | |
744 | error_report_err(err); | |
745 | } | |
746 | if (ret < 0 && ret != -EFAULT) { | |
747 | abort(); | |
748 | } | |
749 | } | |
750 | ||
751 | ||
a7bf3034 PF |
752 | static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp) |
753 | { | |
754 | GICv3State *s = KVM_ARM_GICV3(dev); | |
755 | KVMARMGICv3Class *kgc = KVM_ARM_GICV3_GET_CLASS(s); | |
756 | Error *local_err = NULL; | |
d19a4d4e | 757 | int i; |
a7bf3034 PF |
758 | |
759 | DPRINTF("kvm_arm_gicv3_realize\n"); | |
760 | ||
761 | kgc->parent_realize(dev, &local_err); | |
762 | if (local_err) { | |
763 | error_propagate(errp, local_err); | |
764 | return; | |
765 | } | |
766 | ||
767 | if (s->security_extn) { | |
768 | error_setg(errp, "the in-kernel VGICv3 does not implement the " | |
769 | "security extensions"); | |
770 | return; | |
771 | } | |
772 | ||
773 | gicv3_init_irqs_and_mmio(s, kvm_arm_gicv3_set_irq, NULL); | |
774 | ||
07a5628c VK |
775 | for (i = 0; i < s->num_cpu; i++) { |
776 | ARMCPU *cpu = ARM_CPU(qemu_get_cpu(i)); | |
777 | ||
778 | define_arm_cp_regs(cpu, gicv3_cpuif_reginfo); | |
779 | } | |
780 | ||
a7bf3034 PF |
781 | /* Try to create the device via the device control API */ |
782 | s->dev_fd = kvm_create_device(kvm_state, KVM_DEV_TYPE_ARM_VGIC_V3, false); | |
783 | if (s->dev_fd < 0) { | |
784 | error_setg_errno(errp, -s->dev_fd, "error creating in-kernel VGIC"); | |
785 | return; | |
786 | } | |
787 | ||
788 | kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, | |
556969e9 | 789 | 0, &s->num_irq, true, &error_abort); |
a7bf3034 PF |
790 | |
791 | /* Tell the kernel to complete VGIC initialization now */ | |
792 | kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, | |
556969e9 | 793 | KVM_DEV_ARM_VGIC_CTRL_INIT, NULL, true, &error_abort); |
a7bf3034 PF |
794 | |
795 | kvm_arm_register_device(&s->iomem_dist, -1, KVM_DEV_ARM_VGIC_GRP_ADDR, | |
796 | KVM_VGIC_V3_ADDR_TYPE_DIST, s->dev_fd); | |
797 | kvm_arm_register_device(&s->iomem_redist, -1, KVM_DEV_ARM_VGIC_GRP_ADDR, | |
798 | KVM_VGIC_V3_ADDR_TYPE_REDIST, s->dev_fd); | |
757caeed | 799 | |
d19a4d4e EA |
800 | if (kvm_has_gsi_routing()) { |
801 | /* set up irq routing */ | |
802 | kvm_init_irq_routing(kvm_state); | |
803 | for (i = 0; i < s->num_irq - GIC_INTERNAL; ++i) { | |
804 | kvm_irqchip_add_irq_route(kvm_state, i, 0, i); | |
805 | } | |
806 | ||
807 | kvm_gsi_routing_allowed = true; | |
808 | ||
809 | kvm_irqchip_commit_routes(kvm_state); | |
810 | } | |
367b9f52 VK |
811 | |
812 | if (!kvm_device_check_attr(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_DIST_REGS, | |
813 | GICD_CTLR)) { | |
814 | error_setg(&s->migration_blocker, "This operating system kernel does " | |
815 | "not support vGICv3 migration"); | |
816 | migrate_add_blocker(s->migration_blocker, &local_err); | |
817 | if (local_err) { | |
818 | error_propagate(errp, local_err); | |
819 | error_free(s->migration_blocker); | |
820 | return; | |
821 | } | |
822 | } | |
d5aa0c22 EA |
823 | if (kvm_device_check_attr(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, |
824 | KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES)) { | |
825 | qemu_add_vm_change_state_handler(vm_change_state_handler, s); | |
826 | } | |
a7bf3034 PF |
827 | } |
828 | ||
829 | static void kvm_arm_gicv3_class_init(ObjectClass *klass, void *data) | |
830 | { | |
831 | DeviceClass *dc = DEVICE_CLASS(klass); | |
832 | ARMGICv3CommonClass *agcc = ARM_GICV3_COMMON_CLASS(klass); | |
833 | KVMARMGICv3Class *kgc = KVM_ARM_GICV3_CLASS(klass); | |
834 | ||
835 | agcc->pre_save = kvm_arm_gicv3_get; | |
836 | agcc->post_load = kvm_arm_gicv3_put; | |
837 | kgc->parent_realize = dc->realize; | |
838 | kgc->parent_reset = dc->reset; | |
839 | dc->realize = kvm_arm_gicv3_realize; | |
840 | dc->reset = kvm_arm_gicv3_reset; | |
841 | } | |
842 | ||
843 | static const TypeInfo kvm_arm_gicv3_info = { | |
844 | .name = TYPE_KVM_ARM_GICV3, | |
845 | .parent = TYPE_ARM_GICV3_COMMON, | |
846 | .instance_size = sizeof(GICv3State), | |
847 | .class_init = kvm_arm_gicv3_class_init, | |
848 | .class_size = sizeof(KVMARMGICv3Class), | |
849 | }; | |
850 | ||
851 | static void kvm_arm_gicv3_register_types(void) | |
852 | { | |
853 | type_register_static(&kvm_arm_gicv3_info); | |
854 | } | |
855 | ||
856 | type_init(kvm_arm_gicv3_register_types) |