]>
Commit | Line | Data |
---|---|---|
90977732 EA |
1 | /* |
2 | * Copyright (C) 2015, 2016 ARM Ltd. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License version 2 as | |
6 | * published by the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, | |
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
11 | * GNU General Public License for more details. | |
12 | * | |
13 | * You should have received a copy of the GNU General Public License | |
14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
15 | */ | |
16 | ||
17 | #include <linux/uaccess.h> | |
18 | #include <linux/interrupt.h> | |
19 | #include <linux/cpu.h> | |
20 | #include <linux/kvm_host.h> | |
21 | #include <kvm/arm_vgic.h> | |
22 | #include <asm/kvm_mmu.h> | |
23 | #include "vgic.h" | |
24 | ||
ad275b8b EA |
25 | /* |
26 | * Initialization rules: there are multiple stages to the vgic | |
27 | * initialization, both for the distributor and the CPU interfaces. | |
28 | * | |
29 | * Distributor: | |
30 | * | |
31 | * - kvm_vgic_early_init(): initialization of static data that doesn't | |
32 | * depend on any sizing information or emulation type. No allocation | |
33 | * is allowed there. | |
34 | * | |
35 | * - vgic_init(): allocation and initialization of the generic data | |
36 | * structures that depend on sizing information (number of CPUs, | |
37 | * number of interrupts). Also initializes the vcpu specific data | |
38 | * structures. Can be executed lazily for GICv2. | |
39 | * | |
40 | * CPU Interface: | |
41 | * | |
42 | * - kvm_vgic_cpu_early_init(): initialization of static data that | |
43 | * doesn't depend on any sizing information or emulation type. No | |
44 | * allocation is allowed there. | |
45 | */ | |
46 | ||
47 | /* EARLY INIT */ | |
48 | ||
49 | /* | |
50 | * Those 2 functions should not be needed anymore but they | |
51 | * still are called from arm.c | |
52 | */ | |
53 | void kvm_vgic_early_init(struct kvm *kvm) | |
54 | { | |
55 | } | |
56 | ||
57 | void kvm_vgic_vcpu_early_init(struct kvm_vcpu *vcpu) | |
58 | { | |
59 | } | |
60 | ||
5e6431da EA |
61 | /* CREATION */ |
62 | ||
63 | /** | |
64 | * kvm_vgic_create: triggered by the instantiation of the VGIC device by | |
65 | * user space, either through the legacy KVM_CREATE_IRQCHIP ioctl (v2 only) | |
66 | * or through the generic KVM_CREATE_DEVICE API ioctl. | |
67 | * irqchip_in_kernel() tells you if this function succeeded or not. | |
ad275b8b EA |
68 | * @kvm: kvm struct pointer |
69 | * @type: KVM_DEV_TYPE_ARM_VGIC_V[23] | |
5e6431da EA |
70 | */ |
71 | int kvm_vgic_create(struct kvm *kvm, u32 type) | |
72 | { | |
73 | int i, vcpu_lock_idx = -1, ret; | |
74 | struct kvm_vcpu *vcpu; | |
75 | ||
a28ebea2 CD |
76 | if (irqchip_in_kernel(kvm)) |
77 | return -EEXIST; | |
5e6431da EA |
78 | |
79 | /* | |
80 | * This function is also called by the KVM_CREATE_IRQCHIP handler, | |
81 | * which had no chance yet to check the availability of the GICv2 | |
82 | * emulation. So check this here again. KVM_CREATE_DEVICE does | |
83 | * the proper checks already. | |
84 | */ | |
85 | if (type == KVM_DEV_TYPE_ARM_VGIC_V2 && | |
a28ebea2 CD |
86 | !kvm_vgic_global_state.can_emulate_gicv2) |
87 | return -ENODEV; | |
5e6431da EA |
88 | |
89 | /* | |
90 | * Any time a vcpu is run, vcpu_load is called which tries to grab the | |
91 | * vcpu->mutex. By grabbing the vcpu->mutex of all VCPUs we ensure | |
92 | * that no other VCPUs are run while we create the vgic. | |
93 | */ | |
94 | ret = -EBUSY; | |
95 | kvm_for_each_vcpu(i, vcpu, kvm) { | |
96 | if (!mutex_trylock(&vcpu->mutex)) | |
97 | goto out_unlock; | |
98 | vcpu_lock_idx = i; | |
99 | } | |
100 | ||
101 | kvm_for_each_vcpu(i, vcpu, kvm) { | |
102 | if (vcpu->arch.has_run_once) | |
103 | goto out_unlock; | |
104 | } | |
105 | ret = 0; | |
106 | ||
107 | if (type == KVM_DEV_TYPE_ARM_VGIC_V2) | |
108 | kvm->arch.max_vcpus = VGIC_V2_MAX_CPUS; | |
109 | else | |
110 | kvm->arch.max_vcpus = VGIC_V3_MAX_CPUS; | |
111 | ||
112 | if (atomic_read(&kvm->online_vcpus) > kvm->arch.max_vcpus) { | |
113 | ret = -E2BIG; | |
114 | goto out_unlock; | |
115 | } | |
116 | ||
117 | kvm->arch.vgic.in_kernel = true; | |
118 | kvm->arch.vgic.vgic_model = type; | |
119 | ||
120 | /* | |
121 | * kvm_vgic_global_state.vctrl_base is set on vgic probe (kvm_arch_init) | |
122 | * it is stored in distributor struct for asm save/restore purpose | |
123 | */ | |
124 | kvm->arch.vgic.vctrl_base = kvm_vgic_global_state.vctrl_base; | |
125 | ||
126 | kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF; | |
127 | kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF; | |
128 | kvm->arch.vgic.vgic_redist_base = VGIC_ADDR_UNDEF; | |
129 | ||
130 | out_unlock: | |
131 | for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) { | |
132 | vcpu = kvm_get_vcpu(kvm, vcpu_lock_idx); | |
133 | mutex_unlock(&vcpu->mutex); | |
134 | } | |
5e6431da EA |
135 | return ret; |
136 | } | |
137 | ||
ad275b8b EA |
138 | /* INIT/DESTROY */ |
139 | ||
140 | /** | |
141 | * kvm_vgic_dist_init: initialize the dist data structures | |
142 | * @kvm: kvm struct pointer | |
143 | * @nr_spis: number of spis, frozen by caller | |
144 | */ | |
145 | static int kvm_vgic_dist_init(struct kvm *kvm, unsigned int nr_spis) | |
146 | { | |
147 | struct vgic_dist *dist = &kvm->arch.vgic; | |
148 | struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0); | |
149 | int i; | |
150 | ||
3802411d AP |
151 | INIT_LIST_HEAD(&dist->lpi_list_head); |
152 | spin_lock_init(&dist->lpi_list_lock); | |
153 | ||
ad275b8b EA |
154 | dist->spis = kcalloc(nr_spis, sizeof(struct vgic_irq), GFP_KERNEL); |
155 | if (!dist->spis) | |
156 | return -ENOMEM; | |
157 | ||
158 | /* | |
159 | * In the following code we do not take the irq struct lock since | |
160 | * no other action on irq structs can happen while the VGIC is | |
161 | * not initialized yet: | |
162 | * If someone wants to inject an interrupt or does a MMIO access, we | |
163 | * require prior initialization in case of a virtual GICv3 or trigger | |
164 | * initialization when using a virtual GICv2. | |
165 | */ | |
166 | for (i = 0; i < nr_spis; i++) { | |
167 | struct vgic_irq *irq = &dist->spis[i]; | |
168 | ||
169 | irq->intid = i + VGIC_NR_PRIVATE_IRQS; | |
170 | INIT_LIST_HEAD(&irq->ap_list); | |
171 | spin_lock_init(&irq->irq_lock); | |
172 | irq->vcpu = NULL; | |
173 | irq->target_vcpu = vcpu0; | |
5dd4b924 | 174 | kref_init(&irq->refcount); |
ad275b8b EA |
175 | if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2) |
176 | irq->targets = 0; | |
177 | else | |
178 | irq->mpidr = 0; | |
179 | } | |
180 | return 0; | |
181 | } | |
182 | ||
183 | /** | |
184 | * kvm_vgic_vcpu_init: initialize the vcpu data structures and | |
185 | * enable the VCPU interface | |
186 | * @vcpu: the VCPU which's VGIC should be initialized | |
187 | */ | |
188 | static void kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu) | |
189 | { | |
190 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
191 | int i; | |
192 | ||
193 | INIT_LIST_HEAD(&vgic_cpu->ap_list_head); | |
194 | spin_lock_init(&vgic_cpu->ap_list_lock); | |
195 | ||
196 | /* | |
197 | * Enable and configure all SGIs to be edge-triggered and | |
198 | * configure all PPIs as level-triggered. | |
199 | */ | |
200 | for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) { | |
201 | struct vgic_irq *irq = &vgic_cpu->private_irqs[i]; | |
202 | ||
203 | INIT_LIST_HEAD(&irq->ap_list); | |
204 | spin_lock_init(&irq->irq_lock); | |
205 | irq->intid = i; | |
206 | irq->vcpu = NULL; | |
207 | irq->target_vcpu = vcpu; | |
208 | irq->targets = 1U << vcpu->vcpu_id; | |
5dd4b924 | 209 | kref_init(&irq->refcount); |
ad275b8b EA |
210 | if (vgic_irq_is_sgi(i)) { |
211 | /* SGIs */ | |
212 | irq->enabled = 1; | |
213 | irq->config = VGIC_CONFIG_EDGE; | |
214 | } else { | |
215 | /* PPIs */ | |
216 | irq->config = VGIC_CONFIG_LEVEL; | |
217 | } | |
218 | } | |
219 | if (kvm_vgic_global_state.type == VGIC_V2) | |
220 | vgic_v2_enable(vcpu); | |
221 | else | |
222 | vgic_v3_enable(vcpu); | |
223 | } | |
224 | ||
225 | /* | |
226 | * vgic_init: allocates and initializes dist and vcpu data structures | |
227 | * depending on two dimensioning parameters: | |
228 | * - the number of spis | |
229 | * - the number of vcpus | |
230 | * The function is generally called when nr_spis has been explicitly set | |
231 | * by the guest through the KVM DEVICE API. If not nr_spis is set to 256. | |
232 | * vgic_initialized() returns true when this function has succeeded. | |
233 | * Must be called with kvm->lock held! | |
234 | */ | |
235 | int vgic_init(struct kvm *kvm) | |
236 | { | |
237 | struct vgic_dist *dist = &kvm->arch.vgic; | |
238 | struct kvm_vcpu *vcpu; | |
239 | int ret = 0, i; | |
240 | ||
241 | if (vgic_initialized(kvm)) | |
242 | return 0; | |
243 | ||
244 | /* freeze the number of spis */ | |
245 | if (!dist->nr_spis) | |
246 | dist->nr_spis = VGIC_NR_IRQS_LEGACY - VGIC_NR_PRIVATE_IRQS; | |
247 | ||
248 | ret = kvm_vgic_dist_init(kvm, dist->nr_spis); | |
249 | if (ret) | |
250 | goto out; | |
251 | ||
0e4e82f1 AP |
252 | if (vgic_has_its(kvm)) |
253 | dist->msis_require_devid = true; | |
254 | ||
ad275b8b EA |
255 | kvm_for_each_vcpu(i, vcpu, kvm) |
256 | kvm_vgic_vcpu_init(vcpu); | |
257 | ||
180ae7b1 EA |
258 | ret = kvm_vgic_setup_default_irq_routing(kvm); |
259 | if (ret) | |
260 | goto out; | |
261 | ||
ad275b8b EA |
262 | dist->initialized = true; |
263 | out: | |
264 | return ret; | |
265 | } | |
266 | ||
267 | static void kvm_vgic_dist_destroy(struct kvm *kvm) | |
268 | { | |
269 | struct vgic_dist *dist = &kvm->arch.vgic; | |
270 | ||
271 | mutex_lock(&kvm->lock); | |
272 | ||
273 | dist->ready = false; | |
274 | dist->initialized = false; | |
275 | ||
276 | kfree(dist->spis); | |
ad275b8b EA |
277 | dist->nr_spis = 0; |
278 | ||
279 | mutex_unlock(&kvm->lock); | |
280 | } | |
281 | ||
282 | void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu) | |
283 | { | |
284 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
285 | ||
286 | INIT_LIST_HEAD(&vgic_cpu->ap_list_head); | |
287 | } | |
288 | ||
289 | void kvm_vgic_destroy(struct kvm *kvm) | |
290 | { | |
291 | struct kvm_vcpu *vcpu; | |
292 | int i; | |
293 | ||
294 | kvm_vgic_dist_destroy(kvm); | |
295 | ||
296 | kvm_for_each_vcpu(i, vcpu, kvm) | |
297 | kvm_vgic_vcpu_destroy(vcpu); | |
298 | } | |
299 | ||
300 | /** | |
301 | * vgic_lazy_init: Lazy init is only allowed if the GIC exposed to the guest | |
302 | * is a GICv2. A GICv3 must be explicitly initialized by the guest using the | |
303 | * KVM_DEV_ARM_VGIC_GRP_CTRL KVM_DEVICE group. | |
304 | * @kvm: kvm struct pointer | |
305 | */ | |
306 | int vgic_lazy_init(struct kvm *kvm) | |
307 | { | |
308 | int ret = 0; | |
309 | ||
310 | if (unlikely(!vgic_initialized(kvm))) { | |
311 | /* | |
312 | * We only provide the automatic initialization of the VGIC | |
313 | * for the legacy case of a GICv2. Any other type must | |
314 | * be explicitly initialized once setup with the respective | |
315 | * KVM device call. | |
316 | */ | |
317 | if (kvm->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V2) | |
318 | return -EBUSY; | |
319 | ||
320 | mutex_lock(&kvm->lock); | |
321 | ret = vgic_init(kvm); | |
322 | mutex_unlock(&kvm->lock); | |
323 | } | |
324 | ||
325 | return ret; | |
326 | } | |
327 | ||
b0442ee2 EA |
328 | /* RESOURCE MAPPING */ |
329 | ||
330 | /** | |
331 | * Map the MMIO regions depending on the VGIC model exposed to the guest | |
332 | * called on the first VCPU run. | |
333 | * Also map the virtual CPU interface into the VM. | |
334 | * v2/v3 derivatives call vgic_init if not already done. | |
335 | * vgic_ready() returns true if this function has succeeded. | |
336 | * @kvm: kvm struct pointer | |
337 | */ | |
338 | int kvm_vgic_map_resources(struct kvm *kvm) | |
339 | { | |
340 | struct vgic_dist *dist = &kvm->arch.vgic; | |
341 | int ret = 0; | |
342 | ||
343 | mutex_lock(&kvm->lock); | |
344 | if (!irqchip_in_kernel(kvm)) | |
345 | goto out; | |
346 | ||
347 | if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2) | |
348 | ret = vgic_v2_map_resources(kvm); | |
349 | else | |
350 | ret = vgic_v3_map_resources(kvm); | |
351 | out: | |
352 | mutex_unlock(&kvm->lock); | |
353 | return ret; | |
354 | } | |
355 | ||
90977732 EA |
356 | /* GENERIC PROBE */ |
357 | ||
15d7e3d3 | 358 | static int vgic_init_cpu_starting(unsigned int cpu) |
90977732 EA |
359 | { |
360 | enable_percpu_irq(kvm_vgic_global_state.maint_irq, 0); | |
15d7e3d3 | 361 | return 0; |
90977732 EA |
362 | } |
363 | ||
90977732 | 364 | |
15d7e3d3 AMG |
365 | static int vgic_init_cpu_dying(unsigned int cpu) |
366 | { | |
367 | disable_percpu_irq(kvm_vgic_global_state.maint_irq); | |
368 | return 0; | |
90977732 EA |
369 | } |
370 | ||
90977732 EA |
371 | static irqreturn_t vgic_maintenance_handler(int irq, void *data) |
372 | { | |
373 | /* | |
374 | * We cannot rely on the vgic maintenance interrupt to be | |
375 | * delivered synchronously. This means we can only use it to | |
376 | * exit the VM, and we perform the handling of EOIed | |
377 | * interrupts on the exit path (see vgic_process_maintenance). | |
378 | */ | |
379 | return IRQ_HANDLED; | |
380 | } | |
381 | ||
382 | /** | |
383 | * kvm_vgic_hyp_init: populates the kvm_vgic_global_state variable | |
384 | * according to the host GIC model. Accordingly calls either | |
385 | * vgic_v2/v3_probe which registers the KVM_DEVICE that can be | |
386 | * instantiated by a guest later on . | |
387 | */ | |
388 | int kvm_vgic_hyp_init(void) | |
389 | { | |
390 | const struct gic_kvm_info *gic_kvm_info; | |
391 | int ret; | |
392 | ||
393 | gic_kvm_info = gic_get_kvm_info(); | |
394 | if (!gic_kvm_info) | |
395 | return -ENODEV; | |
396 | ||
397 | if (!gic_kvm_info->maint_irq) { | |
398 | kvm_err("No vgic maintenance irq\n"); | |
399 | return -ENXIO; | |
400 | } | |
401 | ||
402 | switch (gic_kvm_info->type) { | |
403 | case GIC_V2: | |
404 | ret = vgic_v2_probe(gic_kvm_info); | |
405 | break; | |
406 | case GIC_V3: | |
407 | ret = vgic_v3_probe(gic_kvm_info); | |
5a7a8426 VM |
408 | if (!ret) { |
409 | static_branch_enable(&kvm_vgic_global_state.gicv3_cpuif); | |
410 | kvm_info("GIC system register CPU interface enabled\n"); | |
411 | } | |
90977732 EA |
412 | break; |
413 | default: | |
414 | ret = -ENODEV; | |
415 | }; | |
416 | ||
417 | if (ret) | |
418 | return ret; | |
419 | ||
420 | kvm_vgic_global_state.maint_irq = gic_kvm_info->maint_irq; | |
421 | ret = request_percpu_irq(kvm_vgic_global_state.maint_irq, | |
422 | vgic_maintenance_handler, | |
423 | "vgic", kvm_get_running_vcpus()); | |
424 | if (ret) { | |
425 | kvm_err("Cannot register interrupt %d\n", | |
426 | kvm_vgic_global_state.maint_irq); | |
427 | return ret; | |
428 | } | |
429 | ||
15d7e3d3 | 430 | ret = cpuhp_setup_state(CPUHP_AP_KVM_ARM_VGIC_INIT_STARTING, |
73c1b41e | 431 | "kvm/arm/vgic:starting", |
15d7e3d3 | 432 | vgic_init_cpu_starting, vgic_init_cpu_dying); |
90977732 EA |
433 | if (ret) { |
434 | kvm_err("Cannot register vgic CPU notifier\n"); | |
435 | goto out_free_irq; | |
436 | } | |
437 | ||
90977732 EA |
438 | kvm_info("vgic interrupt IRQ%d\n", kvm_vgic_global_state.maint_irq); |
439 | return 0; | |
440 | ||
441 | out_free_irq: | |
442 | free_percpu_irq(kvm_vgic_global_state.maint_irq, | |
443 | kvm_get_running_vcpus()); | |
444 | return ret; | |
445 | } |