]>
Commit | Line | Data |
---|---|---|
53e72406 MZ |
1 | /* |
2 | * Copyright (C) 2012 ARM Ltd. | |
3 | * Author: Marc Zyngier <marc.zyngier@arm.com> | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify | |
6 | * it under the terms of the GNU General Public License version 2 as | |
7 | * published by the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | * GNU General Public License for more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License | |
15 | * along with this program; if not, write to the Free Software | |
16 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | |
17 | */ | |
18 | ||
19 | #include <linux/cpu.h> | |
20 | #include <linux/of_irq.h> | |
21 | #include <linux/kvm.h> | |
22 | #include <linux/kvm_host.h> | |
23 | #include <linux/interrupt.h> | |
24 | ||
372b7c1b | 25 | #include <clocksource/arm_arch_timer.h> |
53e72406 MZ |
26 | #include <asm/arch_timer.h> |
27 | ||
7275acdf MZ |
28 | #include <kvm/arm_vgic.h> |
29 | #include <kvm/arm_arch_timer.h> | |
53e72406 MZ |
30 | |
31 | static struct timecounter *timecounter; | |
32 | static struct workqueue_struct *wqueue; | |
5ae7f87a | 33 | static unsigned int host_vtimer_irq; |
53e72406 MZ |
34 | |
35 | static cycle_t kvm_phys_timer_read(void) | |
36 | { | |
37 | return timecounter->cc->read(timecounter->cc); | |
38 | } | |
39 | ||
40 | static bool timer_is_armed(struct arch_timer_cpu *timer) | |
41 | { | |
42 | return timer->armed; | |
43 | } | |
44 | ||
45 | /* timer_arm: as in "arm the timer", not as in ARM the company */ | |
46 | static void timer_arm(struct arch_timer_cpu *timer, u64 ns) | |
47 | { | |
48 | timer->armed = true; | |
49 | hrtimer_start(&timer->timer, ktime_add_ns(ktime_get(), ns), | |
50 | HRTIMER_MODE_ABS); | |
51 | } | |
52 | ||
53 | static void timer_disarm(struct arch_timer_cpu *timer) | |
54 | { | |
55 | if (timer_is_armed(timer)) { | |
56 | hrtimer_cancel(&timer->timer); | |
57 | cancel_work_sync(&timer->expired); | |
58 | timer->armed = false; | |
59 | } | |
60 | } | |
61 | ||
62 | static void kvm_timer_inject_irq(struct kvm_vcpu *vcpu) | |
63 | { | |
05971120 | 64 | int ret; |
53e72406 MZ |
65 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; |
66 | ||
f120cd65 MZ |
67 | kvm_vgic_set_phys_irq_active(timer->map, true); |
68 | ret = kvm_vgic_inject_mapped_irq(vcpu->kvm, vcpu->vcpu_id, | |
69 | timer->map, | |
70 | timer->irq->level); | |
05971120 | 71 | WARN_ON(ret); |
53e72406 MZ |
72 | } |
73 | ||
74 | static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id) | |
75 | { | |
76 | struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id; | |
77 | ||
78 | /* | |
79 | * We disable the timer in the world switch and let it be | |
80 | * handled by kvm_timer_sync_hwstate(). Getting a timer | |
81 | * interrupt at this point is a sure sign of some major | |
82 | * breakage. | |
83 | */ | |
84 | pr_warn("Unexpected interrupt %d on vcpu %p\n", irq, vcpu); | |
85 | return IRQ_HANDLED; | |
86 | } | |
87 | ||
1a748478 CD |
88 | /* |
89 | * Work function for handling the backup timer that we schedule when a vcpu is | |
90 | * no longer running, but had a timer programmed to fire in the future. | |
91 | */ | |
53e72406 MZ |
92 | static void kvm_timer_inject_irq_work(struct work_struct *work) |
93 | { | |
94 | struct kvm_vcpu *vcpu; | |
95 | ||
96 | vcpu = container_of(work, struct kvm_vcpu, arch.timer_cpu.expired); | |
97 | vcpu->arch.timer_cpu.armed = false; | |
1a748478 CD |
98 | |
99 | /* | |
100 | * If the vcpu is blocked we want to wake it up so that it will see | |
101 | * the timer has expired when entering the guest. | |
102 | */ | |
103 | kvm_vcpu_kick(vcpu); | |
53e72406 MZ |
104 | } |
105 | ||
106 | static enum hrtimer_restart kvm_timer_expire(struct hrtimer *hrt) | |
107 | { | |
108 | struct arch_timer_cpu *timer; | |
109 | timer = container_of(hrt, struct arch_timer_cpu, timer); | |
110 | queue_work(wqueue, &timer->expired); | |
111 | return HRTIMER_NORESTART; | |
112 | } | |
113 | ||
d35268da CD |
114 | static bool kvm_timer_irq_can_fire(struct kvm_vcpu *vcpu) |
115 | { | |
116 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; | |
117 | ||
118 | return !(timer->cntv_ctl & ARCH_TIMER_CTRL_IT_MASK) && | |
119 | (timer->cntv_ctl & ARCH_TIMER_CTRL_ENABLE) && | |
120 | !kvm_vgic_get_phys_irq_active(timer->map); | |
121 | } | |
122 | ||
1a748478 CD |
123 | bool kvm_timer_should_fire(struct kvm_vcpu *vcpu) |
124 | { | |
125 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; | |
126 | cycle_t cval, now; | |
127 | ||
d35268da | 128 | if (!kvm_timer_irq_can_fire(vcpu)) |
1a748478 CD |
129 | return false; |
130 | ||
131 | cval = timer->cntv_cval; | |
132 | now = kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff; | |
133 | ||
134 | return cval <= now; | |
135 | } | |
136 | ||
d35268da CD |
137 | /* |
138 | * Schedule the background timer before calling kvm_vcpu_block, so that this | |
139 | * thread is removed from its waitqueue and made runnable when there's a timer | |
140 | * interrupt to handle. | |
141 | */ | |
142 | void kvm_timer_schedule(struct kvm_vcpu *vcpu) | |
143 | { | |
144 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; | |
145 | u64 ns; | |
146 | cycle_t cval, now; | |
147 | ||
148 | BUG_ON(timer_is_armed(timer)); | |
149 | ||
150 | /* | |
151 | * No need to schedule a background timer if the guest timer has | |
152 | * already expired, because kvm_vcpu_block will return before putting | |
153 | * the thread to sleep. | |
154 | */ | |
155 | if (kvm_timer_should_fire(vcpu)) | |
156 | return; | |
157 | ||
158 | /* | |
159 | * If the timer is not capable of raising interrupts (disabled or | |
160 | * masked), then there's no more work for us to do. | |
161 | */ | |
162 | if (!kvm_timer_irq_can_fire(vcpu)) | |
163 | return; | |
164 | ||
165 | /* The timer has not yet expired, schedule a background timer */ | |
166 | cval = timer->cntv_cval; | |
167 | now = kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff; | |
168 | ||
169 | ns = cyclecounter_cyc2ns(timecounter->cc, | |
170 | cval - now, | |
171 | timecounter->mask, | |
172 | &timecounter->frac); | |
173 | timer_arm(timer, ns); | |
174 | } | |
175 | ||
176 | void kvm_timer_unschedule(struct kvm_vcpu *vcpu) | |
177 | { | |
178 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; | |
179 | timer_disarm(timer); | |
180 | } | |
181 | ||
53e72406 MZ |
182 | /** |
183 | * kvm_timer_flush_hwstate - prepare to move the virt timer to the cpu | |
184 | * @vcpu: The vcpu pointer | |
185 | * | |
d35268da CD |
186 | * Check if the virtual timer has expired while we were running in the host, |
187 | * and inject an interrupt if that was the case. | |
53e72406 MZ |
188 | */ |
189 | void kvm_timer_flush_hwstate(struct kvm_vcpu *vcpu) | |
190 | { | |
191 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; | |
cff9211e CD |
192 | bool phys_active; |
193 | int ret; | |
53e72406 | 194 | |
1a748478 CD |
195 | if (kvm_timer_should_fire(vcpu)) |
196 | kvm_timer_inject_irq(vcpu); | |
cff9211e CD |
197 | |
198 | /* | |
199 | * We keep track of whether the edge-triggered interrupt has been | |
200 | * signalled to the vgic/guest, and if so, we mask the interrupt and | |
201 | * the physical distributor to prevent the timer from raising a | |
202 | * physical interrupt whenever we run a guest, preventing forward | |
203 | * VCPU progress. | |
204 | */ | |
205 | if (kvm_vgic_get_phys_irq_active(timer->map)) | |
206 | phys_active = true; | |
207 | else | |
208 | phys_active = false; | |
209 | ||
210 | ret = irq_set_irqchip_state(timer->map->irq, | |
211 | IRQCHIP_STATE_ACTIVE, | |
212 | phys_active); | |
213 | WARN_ON(ret); | |
53e72406 MZ |
214 | } |
215 | ||
216 | /** | |
217 | * kvm_timer_sync_hwstate - sync timer state from cpu | |
218 | * @vcpu: The vcpu pointer | |
219 | * | |
d35268da CD |
220 | * Check if the virtual timer has expired while we were running in the guest, |
221 | * and inject an interrupt if that was the case. | |
53e72406 MZ |
222 | */ |
223 | void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu) | |
224 | { | |
225 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; | |
53e72406 | 226 | |
53e72406 MZ |
227 | BUG_ON(timer_is_armed(timer)); |
228 | ||
d35268da | 229 | if (kvm_timer_should_fire(vcpu)) |
53e72406 | 230 | kvm_timer_inject_irq(vcpu); |
53e72406 MZ |
231 | } |
232 | ||
f120cd65 MZ |
233 | int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu, |
234 | const struct kvm_irq_level *irq) | |
5ae7f87a AP |
235 | { |
236 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; | |
f120cd65 | 237 | struct irq_phys_map *map; |
5ae7f87a AP |
238 | |
239 | /* | |
240 | * The vcpu timer irq number cannot be determined in | |
241 | * kvm_timer_vcpu_init() because it is called much before | |
242 | * kvm_vcpu_set_target(). To handle this, we determine | |
243 | * vcpu timer irq number when the vcpu is reset. | |
244 | */ | |
245 | timer->irq = irq; | |
f120cd65 | 246 | |
4ad9e16a CD |
247 | /* |
248 | * The bits in CNTV_CTL are architecturally reset to UNKNOWN for ARMv8 | |
249 | * and to 0 for ARMv7. We provide an implementation that always | |
250 | * resets the timer to be disabled and unmasked and is compliant with | |
251 | * the ARMv7 architecture. | |
252 | */ | |
253 | timer->cntv_ctl = 0; | |
254 | ||
f120cd65 MZ |
255 | /* |
256 | * Tell the VGIC that the virtual interrupt is tied to a | |
257 | * physical interrupt. We do that once per VCPU. | |
258 | */ | |
259 | map = kvm_vgic_map_phys_irq(vcpu, irq->irq, host_vtimer_irq); | |
260 | if (WARN_ON(IS_ERR(map))) | |
261 | return PTR_ERR(map); | |
262 | ||
263 | timer->map = map; | |
264 | return 0; | |
5ae7f87a AP |
265 | } |
266 | ||
53e72406 MZ |
267 | void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu) |
268 | { | |
269 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; | |
270 | ||
271 | INIT_WORK(&timer->expired, kvm_timer_inject_irq_work); | |
272 | hrtimer_init(&timer->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); | |
273 | timer->timer.function = kvm_timer_expire; | |
53e72406 MZ |
274 | } |
275 | ||
276 | static void kvm_timer_init_interrupt(void *info) | |
277 | { | |
5ae7f87a | 278 | enable_percpu_irq(host_vtimer_irq, 0); |
53e72406 MZ |
279 | } |
280 | ||
39735a3a AP |
281 | int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value) |
282 | { | |
283 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; | |
284 | ||
285 | switch (regid) { | |
286 | case KVM_REG_ARM_TIMER_CTL: | |
287 | timer->cntv_ctl = value; | |
288 | break; | |
289 | case KVM_REG_ARM_TIMER_CNT: | |
290 | vcpu->kvm->arch.timer.cntvoff = kvm_phys_timer_read() - value; | |
291 | break; | |
292 | case KVM_REG_ARM_TIMER_CVAL: | |
293 | timer->cntv_cval = value; | |
294 | break; | |
295 | default: | |
296 | return -1; | |
297 | } | |
298 | return 0; | |
299 | } | |
300 | ||
301 | u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid) | |
302 | { | |
303 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; | |
304 | ||
305 | switch (regid) { | |
306 | case KVM_REG_ARM_TIMER_CTL: | |
307 | return timer->cntv_ctl; | |
308 | case KVM_REG_ARM_TIMER_CNT: | |
309 | return kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff; | |
310 | case KVM_REG_ARM_TIMER_CVAL: | |
311 | return timer->cntv_cval; | |
312 | } | |
313 | return (u64)-1; | |
314 | } | |
53e72406 MZ |
315 | |
316 | static int kvm_timer_cpu_notify(struct notifier_block *self, | |
317 | unsigned long action, void *cpu) | |
318 | { | |
319 | switch (action) { | |
320 | case CPU_STARTING: | |
321 | case CPU_STARTING_FROZEN: | |
322 | kvm_timer_init_interrupt(NULL); | |
323 | break; | |
324 | case CPU_DYING: | |
325 | case CPU_DYING_FROZEN: | |
5ae7f87a | 326 | disable_percpu_irq(host_vtimer_irq); |
53e72406 MZ |
327 | break; |
328 | } | |
329 | ||
330 | return NOTIFY_OK; | |
331 | } | |
332 | ||
333 | static struct notifier_block kvm_timer_cpu_nb = { | |
334 | .notifier_call = kvm_timer_cpu_notify, | |
335 | }; | |
336 | ||
337 | static const struct of_device_id arch_timer_of_match[] = { | |
338 | { .compatible = "arm,armv7-timer", }, | |
f61701e0 | 339 | { .compatible = "arm,armv8-timer", }, |
53e72406 MZ |
340 | {}, |
341 | }; | |
342 | ||
343 | int kvm_timer_hyp_init(void) | |
344 | { | |
345 | struct device_node *np; | |
346 | unsigned int ppi; | |
347 | int err; | |
348 | ||
349 | timecounter = arch_timer_get_timecounter(); | |
350 | if (!timecounter) | |
351 | return -ENODEV; | |
352 | ||
353 | np = of_find_matching_node(NULL, arch_timer_of_match); | |
354 | if (!np) { | |
355 | kvm_err("kvm_arch_timer: can't find DT node\n"); | |
356 | return -ENODEV; | |
357 | } | |
358 | ||
359 | ppi = irq_of_parse_and_map(np, 2); | |
360 | if (!ppi) { | |
361 | kvm_err("kvm_arch_timer: no virtual timer interrupt\n"); | |
362 | err = -EINVAL; | |
363 | goto out; | |
364 | } | |
365 | ||
366 | err = request_percpu_irq(ppi, kvm_arch_timer_handler, | |
367 | "kvm guest timer", kvm_get_running_vcpus()); | |
368 | if (err) { | |
369 | kvm_err("kvm_arch_timer: can't request interrupt %d (%d)\n", | |
370 | ppi, err); | |
371 | goto out; | |
372 | } | |
373 | ||
5ae7f87a | 374 | host_vtimer_irq = ppi; |
53e72406 | 375 | |
553f809e | 376 | err = __register_cpu_notifier(&kvm_timer_cpu_nb); |
53e72406 MZ |
377 | if (err) { |
378 | kvm_err("Cannot register timer CPU notifier\n"); | |
379 | goto out_free; | |
380 | } | |
381 | ||
382 | wqueue = create_singlethread_workqueue("kvm_arch_timer"); | |
383 | if (!wqueue) { | |
384 | err = -ENOMEM; | |
385 | goto out_free; | |
386 | } | |
387 | ||
388 | kvm_info("%s IRQ%d\n", np->name, ppi); | |
389 | on_each_cpu(kvm_timer_init_interrupt, NULL, 1); | |
390 | ||
391 | goto out; | |
392 | out_free: | |
393 | free_percpu_irq(ppi, kvm_get_running_vcpus()); | |
394 | out: | |
395 | of_node_put(np); | |
396 | return err; | |
397 | } | |
398 | ||
399 | void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu) | |
400 | { | |
401 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; | |
402 | ||
403 | timer_disarm(timer); | |
f120cd65 MZ |
404 | if (timer->map) |
405 | kvm_vgic_unmap_phys_irq(vcpu, timer->map); | |
53e72406 MZ |
406 | } |
407 | ||
05971120 | 408 | void kvm_timer_enable(struct kvm *kvm) |
53e72406 | 409 | { |
05971120 CD |
410 | if (kvm->arch.timer.enabled) |
411 | return; | |
412 | ||
413 | /* | |
414 | * There is a potential race here between VCPUs starting for the first | |
415 | * time, which may be enabling the timer multiple times. That doesn't | |
416 | * hurt though, because we're just setting a variable to the same | |
417 | * variable that it already was. The important thing is that all | |
418 | * VCPUs have the enabled variable set, before entering the guest, if | |
419 | * the arch timers are enabled. | |
420 | */ | |
421 | if (timecounter && wqueue) | |
53e72406 | 422 | kvm->arch.timer.enabled = 1; |
05971120 | 423 | } |
53e72406 | 424 | |
05971120 CD |
425 | void kvm_timer_init(struct kvm *kvm) |
426 | { | |
427 | kvm->arch.timer.cntvoff = kvm_phys_timer_read(); | |
53e72406 | 428 | } |