]>
Commit | Line | Data |
---|---|---|
53e72406 MZ |
1 | /* |
2 | * Copyright (C) 2012 ARM Ltd. | |
3 | * Author: Marc Zyngier <marc.zyngier@arm.com> | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify | |
6 | * it under the terms of the GNU General Public License version 2 as | |
7 | * published by the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | * GNU General Public License for more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License | |
15 | * along with this program; if not, write to the Free Software | |
16 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | |
17 | */ | |
18 | ||
19 | #include <linux/cpu.h> | |
53e72406 MZ |
20 | #include <linux/kvm.h> |
21 | #include <linux/kvm_host.h> | |
22 | #include <linux/interrupt.h> | |
b452cb52 | 23 | #include <linux/irq.h> |
99a1db7a | 24 | #include <linux/uaccess.h> |
53e72406 | 25 | |
372b7c1b | 26 | #include <clocksource/arm_arch_timer.h> |
53e72406 | 27 | #include <asm/arch_timer.h> |
488f94d7 | 28 | #include <asm/kvm_hyp.h> |
53e72406 | 29 | |
7275acdf MZ |
30 | #include <kvm/arm_vgic.h> |
31 | #include <kvm/arm_arch_timer.h> | |
53e72406 | 32 | |
e21f0910 CD |
33 | #include "trace.h" |
34 | ||
53e72406 | 35 | static struct timecounter *timecounter; |
5ae7f87a | 36 | static unsigned int host_vtimer_irq; |
cabdc5c5 | 37 | static u32 host_vtimer_irq_flags; |
53e72406 | 38 | |
85e69ad7 CD |
39 | static const struct kvm_irq_level default_ptimer_irq = { |
40 | .irq = 30, | |
41 | .level = 1, | |
42 | }; | |
43 | ||
44 | static const struct kvm_irq_level default_vtimer_irq = { | |
45 | .irq = 27, | |
46 | .level = 1, | |
47 | }; | |
48 | ||
b103cc3f CD |
49 | static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx); |
50 | static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level, | |
51 | struct arch_timer_context *timer_ctx); | |
1c88ab7e | 52 | static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx); |
9b4a3004 | 53 | |
7b6b4631 | 54 | u64 kvm_phys_timer_read(void) |
53e72406 MZ |
55 | { |
56 | return timecounter->cc->read(timecounter->cc); | |
57 | } | |
58 | ||
8409a06f | 59 | static void soft_timer_start(struct hrtimer *hrt, u64 ns) |
53e72406 | 60 | { |
8409a06f | 61 | hrtimer_start(hrt, ktime_add_ns(ktime_get(), ns), |
53e72406 MZ |
62 | HRTIMER_MODE_ABS); |
63 | } | |
64 | ||
8409a06f | 65 | static void soft_timer_cancel(struct hrtimer *hrt, struct work_struct *work) |
53e72406 | 66 | { |
8409a06f | 67 | hrtimer_cancel(hrt); |
f2a2129e CD |
68 | if (work) |
69 | cancel_work_sync(work); | |
53e72406 MZ |
70 | } |
71 | ||
b103cc3f | 72 | static void kvm_vtimer_update_mask_user(struct kvm_vcpu *vcpu) |
53e72406 | 73 | { |
b103cc3f | 74 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); |
53e72406 MZ |
75 | |
76 | /* | |
b103cc3f CD |
77 | * When using a userspace irqchip with the architected timers, we must |
78 | * prevent continuously exiting from the guest, and therefore mask the | |
79 | * physical interrupt by disabling it on the host interrupt controller | |
80 | * when the virtual level is high, such that the guest can make | |
81 | * forward progress. Once we detect the output level being | |
82 | * de-asserted, we unmask the interrupt again so that we exit from the | |
83 | * guest when the timer fires. | |
53e72406 | 84 | */ |
b103cc3f CD |
85 | if (vtimer->irq.level) |
86 | disable_percpu_irq(host_vtimer_irq); | |
87 | else | |
88 | enable_percpu_irq(host_vtimer_irq, 0); | |
89 | } | |
90 | ||
91 | static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id) | |
92 | { | |
93 | struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id; | |
94 | struct arch_timer_context *vtimer; | |
95 | ||
96 | if (!vcpu) { | |
97 | pr_warn_once("Spurious arch timer IRQ on non-VCPU thread\n"); | |
98 | return IRQ_NONE; | |
99 | } | |
100 | vtimer = vcpu_vtimer(vcpu); | |
101 | ||
102 | if (!vtimer->irq.level) { | |
103 | vtimer->cnt_ctl = read_sysreg_el0(cntv_ctl); | |
104 | if (kvm_timer_irq_can_fire(vtimer)) | |
105 | kvm_timer_update_irq(vcpu, true, vtimer); | |
106 | } | |
107 | ||
108 | if (unlikely(!irqchip_in_kernel(vcpu->kvm))) | |
109 | kvm_vtimer_update_mask_user(vcpu); | |
110 | ||
53e72406 MZ |
111 | return IRQ_HANDLED; |
112 | } | |
113 | ||
1a748478 CD |
114 | /* |
115 | * Work function for handling the backup timer that we schedule when a vcpu is | |
116 | * no longer running, but had a timer programmed to fire in the future. | |
117 | */ | |
53e72406 MZ |
118 | static void kvm_timer_inject_irq_work(struct work_struct *work) |
119 | { | |
120 | struct kvm_vcpu *vcpu; | |
121 | ||
122 | vcpu = container_of(work, struct kvm_vcpu, arch.timer_cpu.expired); | |
1c5631c7 | 123 | |
1a748478 CD |
124 | /* |
125 | * If the vcpu is blocked we want to wake it up so that it will see | |
126 | * the timer has expired when entering the guest. | |
127 | */ | |
1b6502e5 | 128 | kvm_vcpu_wake_up(vcpu); |
53e72406 MZ |
129 | } |
130 | ||
9171fa2e | 131 | static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx) |
1c5631c7 | 132 | { |
a5a1d1c2 | 133 | u64 cval, now; |
1c5631c7 | 134 | |
9171fa2e JL |
135 | cval = timer_ctx->cnt_cval; |
136 | now = kvm_phys_timer_read() - timer_ctx->cntvoff; | |
1c5631c7 MZ |
137 | |
138 | if (now < cval) { | |
139 | u64 ns; | |
140 | ||
141 | ns = cyclecounter_cyc2ns(timecounter->cc, | |
142 | cval - now, | |
143 | timecounter->mask, | |
144 | &timecounter->frac); | |
145 | return ns; | |
146 | } | |
147 | ||
148 | return 0; | |
149 | } | |
150 | ||
fb280e97 JL |
151 | static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx) |
152 | { | |
153 | return !(timer_ctx->cnt_ctl & ARCH_TIMER_CTRL_IT_MASK) && | |
154 | (timer_ctx->cnt_ctl & ARCH_TIMER_CTRL_ENABLE); | |
155 | } | |
156 | ||
157 | /* | |
158 | * Returns the earliest expiration time in ns among guest timers. | |
159 | * Note that it will return 0 if none of timers can fire. | |
160 | */ | |
161 | static u64 kvm_timer_earliest_exp(struct kvm_vcpu *vcpu) | |
162 | { | |
163 | u64 min_virt = ULLONG_MAX, min_phys = ULLONG_MAX; | |
164 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); | |
165 | struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); | |
166 | ||
167 | if (kvm_timer_irq_can_fire(vtimer)) | |
168 | min_virt = kvm_timer_compute_delta(vtimer); | |
169 | ||
170 | if (kvm_timer_irq_can_fire(ptimer)) | |
171 | min_phys = kvm_timer_compute_delta(ptimer); | |
172 | ||
173 | /* If none of timers can fire, then return 0 */ | |
174 | if ((min_virt == ULLONG_MAX) && (min_phys == ULLONG_MAX)) | |
175 | return 0; | |
176 | ||
177 | return min(min_virt, min_phys); | |
178 | } | |
179 | ||
14d61fa9 | 180 | static enum hrtimer_restart kvm_bg_timer_expire(struct hrtimer *hrt) |
53e72406 MZ |
181 | { |
182 | struct arch_timer_cpu *timer; | |
1c5631c7 MZ |
183 | struct kvm_vcpu *vcpu; |
184 | u64 ns; | |
185 | ||
14d61fa9 | 186 | timer = container_of(hrt, struct arch_timer_cpu, bg_timer); |
1c5631c7 MZ |
187 | vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu); |
188 | ||
189 | /* | |
190 | * Check that the timer has really expired from the guest's | |
191 | * PoV (NTP on the host may have forced it to expire | |
192 | * early). If we should have slept longer, restart it. | |
193 | */ | |
fb280e97 | 194 | ns = kvm_timer_earliest_exp(vcpu); |
1c5631c7 MZ |
195 | if (unlikely(ns)) { |
196 | hrtimer_forward_now(hrt, ns_to_ktime(ns)); | |
197 | return HRTIMER_RESTART; | |
198 | } | |
199 | ||
3706feac | 200 | schedule_work(&timer->expired); |
53e72406 MZ |
201 | return HRTIMER_NORESTART; |
202 | } | |
203 | ||
f2a2129e CD |
204 | static enum hrtimer_restart kvm_phys_timer_expire(struct hrtimer *hrt) |
205 | { | |
bbdd52cf CD |
206 | struct arch_timer_context *ptimer; |
207 | struct arch_timer_cpu *timer; | |
208 | struct kvm_vcpu *vcpu; | |
209 | u64 ns; | |
210 | ||
211 | timer = container_of(hrt, struct arch_timer_cpu, phys_timer); | |
212 | vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu); | |
213 | ptimer = vcpu_ptimer(vcpu); | |
214 | ||
215 | /* | |
216 | * Check that the timer has really expired from the guest's | |
217 | * PoV (NTP on the host may have forced it to expire | |
218 | * early). If not ready, schedule for a later time. | |
219 | */ | |
220 | ns = kvm_timer_compute_delta(ptimer); | |
221 | if (unlikely(ns)) { | |
222 | hrtimer_forward_now(hrt, ns_to_ktime(ns)); | |
223 | return HRTIMER_RESTART; | |
224 | } | |
225 | ||
226 | kvm_timer_update_irq(vcpu, true, ptimer); | |
f2a2129e CD |
227 | return HRTIMER_NORESTART; |
228 | } | |
229 | ||
1c88ab7e | 230 | static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx) |
1a748478 | 231 | { |
a5a1d1c2 | 232 | u64 cval, now; |
1a748478 | 233 | |
9171fa2e | 234 | if (!kvm_timer_irq_can_fire(timer_ctx)) |
1a748478 CD |
235 | return false; |
236 | ||
9171fa2e JL |
237 | cval = timer_ctx->cnt_cval; |
238 | now = kvm_phys_timer_read() - timer_ctx->cntvoff; | |
1a748478 CD |
239 | |
240 | return cval <= now; | |
241 | } | |
242 | ||
1c88ab7e CD |
243 | bool kvm_timer_is_pending(struct kvm_vcpu *vcpu) |
244 | { | |
245 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); | |
246 | struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); | |
247 | ||
248 | if (vtimer->irq.level || ptimer->irq.level) | |
249 | return true; | |
250 | ||
251 | /* | |
252 | * When this is called from withing the wait loop of kvm_vcpu_block(), | |
253 | * the software view of the timer state is up to date (timer->loaded | |
254 | * is false), and so we can simply check if the timer should fire now. | |
255 | */ | |
256 | if (!vtimer->loaded && kvm_timer_should_fire(vtimer)) | |
257 | return true; | |
258 | ||
259 | return kvm_timer_should_fire(ptimer); | |
260 | } | |
261 | ||
d9e13977 AG |
262 | /* |
263 | * Reflect the timer output level into the kvm_run structure | |
264 | */ | |
265 | void kvm_timer_update_run(struct kvm_vcpu *vcpu) | |
266 | { | |
267 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); | |
268 | struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); | |
269 | struct kvm_sync_regs *regs = &vcpu->run->s.regs; | |
270 | ||
d9e13977 AG |
271 | /* Populate the device bitmap with the timer states */ |
272 | regs->device_irq_level &= ~(KVM_ARM_DEV_EL1_VTIMER | | |
273 | KVM_ARM_DEV_EL1_PTIMER); | |
274 | if (vtimer->irq.level) | |
275 | regs->device_irq_level |= KVM_ARM_DEV_EL1_VTIMER; | |
276 | if (ptimer->irq.level) | |
277 | regs->device_irq_level |= KVM_ARM_DEV_EL1_PTIMER; | |
278 | } | |
279 | ||
9171fa2e JL |
280 | static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level, |
281 | struct arch_timer_context *timer_ctx) | |
4b4b4512 CD |
282 | { |
283 | int ret; | |
4b4b4512 | 284 | |
9171fa2e JL |
285 | timer_ctx->irq.level = new_level; |
286 | trace_kvm_timer_update_irq(vcpu->vcpu_id, timer_ctx->irq.irq, | |
287 | timer_ctx->irq.level); | |
11710dec | 288 | |
d9e13977 AG |
289 | if (likely(irqchip_in_kernel(vcpu->kvm))) { |
290 | ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id, | |
291 | timer_ctx->irq.irq, | |
cb3f0ad8 CD |
292 | timer_ctx->irq.level, |
293 | timer_ctx); | |
d9e13977 AG |
294 | WARN_ON(ret); |
295 | } | |
4b4b4512 CD |
296 | } |
297 | ||
cda93b7a | 298 | /* Schedule the background timer for the emulated timer. */ |
bbdd52cf | 299 | static void phys_timer_emulate(struct kvm_vcpu *vcpu) |
cda93b7a CD |
300 | { |
301 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; | |
bbdd52cf | 302 | struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); |
cda93b7a | 303 | |
bbdd52cf CD |
304 | /* |
305 | * If the timer can fire now we have just raised the IRQ line and we | |
306 | * don't need to have a soft timer scheduled for the future. If the | |
307 | * timer cannot fire at all, then we also don't need a soft timer. | |
308 | */ | |
309 | if (kvm_timer_should_fire(ptimer) || !kvm_timer_irq_can_fire(ptimer)) { | |
310 | soft_timer_cancel(&timer->phys_timer, NULL); | |
cda93b7a | 311 | return; |
bbdd52cf | 312 | } |
cda93b7a | 313 | |
bbdd52cf | 314 | soft_timer_start(&timer->phys_timer, kvm_timer_compute_delta(ptimer)); |
cda93b7a CD |
315 | } |
316 | ||
4b4b4512 | 317 | /* |
bbdd52cf CD |
318 | * Check if there was a change in the timer state, so that we should either |
319 | * raise or lower the line level to the GIC or schedule a background timer to | |
320 | * emulate the physical timer. | |
4b4b4512 | 321 | */ |
b22e7df2 | 322 | static void kvm_timer_update_state(struct kvm_vcpu *vcpu) |
4b4b4512 CD |
323 | { |
324 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; | |
fbb4aeec | 325 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); |
58e0c973 | 326 | struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); |
4b4b4512 | 327 | |
d9e13977 | 328 | if (unlikely(!timer->enabled)) |
b22e7df2 | 329 | return; |
4b4b4512 | 330 | |
9171fa2e JL |
331 | if (kvm_timer_should_fire(vtimer) != vtimer->irq.level) |
332 | kvm_timer_update_irq(vcpu, !vtimer->irq.level, vtimer); | |
b3aff6cc | 333 | |
58e0c973 JL |
334 | if (kvm_timer_should_fire(ptimer) != ptimer->irq.level) |
335 | kvm_timer_update_irq(vcpu, !ptimer->irq.level, ptimer); | |
bbdd52cf CD |
336 | |
337 | phys_timer_emulate(vcpu); | |
4b4b4512 CD |
338 | } |
339 | ||
b103cc3f | 340 | static void vtimer_save_state(struct kvm_vcpu *vcpu) |
688c50aa CD |
341 | { |
342 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; | |
343 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); | |
b103cc3f CD |
344 | unsigned long flags; |
345 | ||
346 | local_irq_save(flags); | |
347 | ||
348 | if (!vtimer->loaded) | |
349 | goto out; | |
688c50aa CD |
350 | |
351 | if (timer->enabled) { | |
352 | vtimer->cnt_ctl = read_sysreg_el0(cntv_ctl); | |
353 | vtimer->cnt_cval = read_sysreg_el0(cntv_cval); | |
354 | } | |
355 | ||
356 | /* Disable the virtual timer */ | |
357 | write_sysreg_el0(0, cntv_ctl); | |
b103cc3f CD |
358 | |
359 | vtimer->loaded = false; | |
360 | out: | |
361 | local_irq_restore(flags); | |
688c50aa CD |
362 | } |
363 | ||
d35268da CD |
364 | /* |
365 | * Schedule the background timer before calling kvm_vcpu_block, so that this | |
366 | * thread is removed from its waitqueue and made runnable when there's a timer | |
367 | * interrupt to handle. | |
368 | */ | |
369 | void kvm_timer_schedule(struct kvm_vcpu *vcpu) | |
370 | { | |
371 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; | |
9171fa2e | 372 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); |
fb280e97 | 373 | struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); |
d35268da | 374 | |
b103cc3f CD |
375 | vtimer_save_state(vcpu); |
376 | ||
d35268da | 377 | /* |
fb280e97 | 378 | * No need to schedule a background timer if any guest timer has |
d35268da CD |
379 | * already expired, because kvm_vcpu_block will return before putting |
380 | * the thread to sleep. | |
381 | */ | |
fb280e97 | 382 | if (kvm_timer_should_fire(vtimer) || kvm_timer_should_fire(ptimer)) |
d35268da CD |
383 | return; |
384 | ||
385 | /* | |
fb280e97 | 386 | * If both timers are not capable of raising interrupts (disabled or |
d35268da CD |
387 | * masked), then there's no more work for us to do. |
388 | */ | |
fb280e97 | 389 | if (!kvm_timer_irq_can_fire(vtimer) && !kvm_timer_irq_can_fire(ptimer)) |
d35268da CD |
390 | return; |
391 | ||
fb280e97 JL |
392 | /* |
393 | * The guest timers have not yet expired, schedule a background timer. | |
394 | * Set the earliest expiration time among the guest timers. | |
395 | */ | |
14d61fa9 | 396 | soft_timer_start(&timer->bg_timer, kvm_timer_earliest_exp(vcpu)); |
d35268da CD |
397 | } |
398 | ||
b103cc3f | 399 | static void vtimer_restore_state(struct kvm_vcpu *vcpu) |
688c50aa CD |
400 | { |
401 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; | |
402 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); | |
b103cc3f CD |
403 | unsigned long flags; |
404 | ||
405 | local_irq_save(flags); | |
406 | ||
407 | if (vtimer->loaded) | |
408 | goto out; | |
688c50aa CD |
409 | |
410 | if (timer->enabled) { | |
411 | write_sysreg_el0(vtimer->cnt_cval, cntv_cval); | |
412 | isb(); | |
413 | write_sysreg_el0(vtimer->cnt_ctl, cntv_ctl); | |
414 | } | |
b103cc3f CD |
415 | |
416 | vtimer->loaded = true; | |
417 | out: | |
418 | local_irq_restore(flags); | |
688c50aa CD |
419 | } |
420 | ||
d35268da CD |
421 | void kvm_timer_unschedule(struct kvm_vcpu *vcpu) |
422 | { | |
423 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; | |
8409a06f | 424 | |
b103cc3f CD |
425 | vtimer_restore_state(vcpu); |
426 | ||
14d61fa9 | 427 | soft_timer_cancel(&timer->bg_timer, &timer->expired); |
d35268da CD |
428 | } |
429 | ||
688c50aa CD |
430 | static void set_cntvoff(u64 cntvoff) |
431 | { | |
432 | u32 low = lower_32_bits(cntvoff); | |
433 | u32 high = upper_32_bits(cntvoff); | |
434 | ||
435 | /* | |
436 | * Since kvm_call_hyp doesn't fully support the ARM PCS especially on | |
437 | * 32-bit systems, but rather passes register by register shifted one | |
438 | * place (we put the function address in r0/x0), we cannot simply pass | |
439 | * a 64-bit value as an argument, but have to split the value in two | |
440 | * 32-bit halves. | |
441 | */ | |
442 | kvm_call_hyp(__kvm_timer_set_cntvoff, low, high); | |
443 | } | |
444 | ||
b103cc3f | 445 | static void kvm_timer_vcpu_load_vgic(struct kvm_vcpu *vcpu) |
53e72406 | 446 | { |
fbb4aeec | 447 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); |
cff9211e CD |
448 | bool phys_active; |
449 | int ret; | |
53e72406 | 450 | |
fbb4aeec | 451 | phys_active = vtimer->irq.level || |
b103cc3f | 452 | kvm_vgic_map_is_active(vcpu, vtimer->irq.irq); |
9b4a3004 | 453 | |
b452cb52 | 454 | ret = irq_set_irqchip_state(host_vtimer_irq, |
cff9211e CD |
455 | IRQCHIP_STATE_ACTIVE, |
456 | phys_active); | |
457 | WARN_ON(ret); | |
b103cc3f | 458 | } |
9b4a3004 | 459 | |
b103cc3f CD |
460 | static void kvm_timer_vcpu_load_user(struct kvm_vcpu *vcpu) |
461 | { | |
462 | kvm_vtimer_update_mask_user(vcpu); | |
463 | } | |
464 | ||
465 | void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu) | |
466 | { | |
467 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; | |
468 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); | |
469 | ||
470 | if (unlikely(!timer->enabled)) | |
471 | return; | |
472 | ||
473 | if (unlikely(!irqchip_in_kernel(vcpu->kvm))) | |
474 | kvm_timer_vcpu_load_user(vcpu); | |
475 | else | |
476 | kvm_timer_vcpu_load_vgic(vcpu); | |
477 | ||
478 | set_cntvoff(vtimer->cntvoff); | |
479 | ||
480 | vtimer_restore_state(vcpu); | |
481 | ||
bbdd52cf CD |
482 | /* Set the background timer for the physical timer emulation. */ |
483 | phys_timer_emulate(vcpu); | |
53e72406 MZ |
484 | } |
485 | ||
d9e13977 AG |
486 | bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu) |
487 | { | |
488 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); | |
489 | struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); | |
490 | struct kvm_sync_regs *sregs = &vcpu->run->s.regs; | |
491 | bool vlevel, plevel; | |
492 | ||
493 | if (likely(irqchip_in_kernel(vcpu->kvm))) | |
494 | return false; | |
495 | ||
496 | vlevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_VTIMER; | |
497 | plevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_PTIMER; | |
498 | ||
499 | return vtimer->irq.level != vlevel || | |
500 | ptimer->irq.level != plevel; | |
501 | } | |
502 | ||
b103cc3f CD |
503 | void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu) |
504 | { | |
505 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; | |
688c50aa | 506 | |
b103cc3f CD |
507 | if (unlikely(!timer->enabled)) |
508 | return; | |
509 | ||
b103cc3f CD |
510 | vtimer_save_state(vcpu); |
511 | ||
bbdd52cf CD |
512 | /* |
513 | * Cancel the physical timer emulation, because the only case where we | |
514 | * need it after a vcpu_put is in the context of a sleeping VCPU, and | |
515 | * in that case we already factor in the deadline for the physical | |
516 | * timer when scheduling the bg_timer. | |
517 | * | |
518 | * In any case, we re-schedule the hrtimer for the physical timer when | |
519 | * coming back to the VCPU thread in kvm_timer_vcpu_load(). | |
520 | */ | |
521 | soft_timer_cancel(&timer->phys_timer, NULL); | |
522 | ||
b103cc3f CD |
523 | /* |
524 | * The kernel may decide to run userspace after calling vcpu_put, so | |
525 | * we reset cntvoff to 0 to ensure a consistent read between user | |
526 | * accesses to the virtual counter and kernel access to the physical | |
527 | * counter. | |
528 | */ | |
529 | set_cntvoff(0); | |
530 | } | |
531 | ||
532 | static void unmask_vtimer_irq(struct kvm_vcpu *vcpu) | |
533 | { | |
534 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); | |
535 | ||
536 | if (unlikely(!irqchip_in_kernel(vcpu->kvm))) { | |
537 | kvm_vtimer_update_mask_user(vcpu); | |
538 | return; | |
539 | } | |
540 | ||
541 | /* | |
542 | * If the guest disabled the timer without acking the interrupt, then | |
543 | * we must make sure the physical and virtual active states are in | |
544 | * sync by deactivating the physical interrupt, because otherwise we | |
545 | * wouldn't see the next timer interrupt in the host. | |
546 | */ | |
547 | if (!kvm_vgic_map_is_active(vcpu, vtimer->irq.irq)) { | |
548 | int ret; | |
549 | ret = irq_set_irqchip_state(host_vtimer_irq, | |
550 | IRQCHIP_STATE_ACTIVE, | |
551 | false); | |
552 | WARN_ON(ret); | |
553 | } | |
d9e13977 AG |
554 | } |
555 | ||
53e72406 MZ |
556 | /** |
557 | * kvm_timer_sync_hwstate - sync timer state from cpu | |
558 | * @vcpu: The vcpu pointer | |
559 | * | |
d9e13977 | 560 | * Check if any of the timers have expired while we were running in the guest, |
d35268da | 561 | * and inject an interrupt if that was the case. |
53e72406 MZ |
562 | */ |
563 | void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu) | |
564 | { | |
b103cc3f | 565 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); |
53e72406 | 566 | |
4b4b4512 | 567 | /* |
b103cc3f CD |
568 | * If we entered the guest with the vtimer output asserted we have to |
569 | * check if the guest has modified the timer so that we should lower | |
570 | * the line at this point. | |
4b4b4512 | 571 | */ |
b103cc3f CD |
572 | if (vtimer->irq.level) { |
573 | vtimer->cnt_ctl = read_sysreg_el0(cntv_ctl); | |
574 | vtimer->cnt_cval = read_sysreg_el0(cntv_cval); | |
575 | if (!kvm_timer_should_fire(vtimer)) { | |
576 | kvm_timer_update_irq(vcpu, false, vtimer); | |
577 | unmask_vtimer_irq(vcpu); | |
578 | } | |
579 | } | |
53e72406 MZ |
580 | } |
581 | ||
85e69ad7 | 582 | int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu) |
5ae7f87a | 583 | { |
fbb4aeec | 584 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); |
a91d1855 | 585 | struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); |
5ae7f87a | 586 | |
4ad9e16a CD |
587 | /* |
588 | * The bits in CNTV_CTL are architecturally reset to UNKNOWN for ARMv8 | |
589 | * and to 0 for ARMv7. We provide an implementation that always | |
590 | * resets the timer to be disabled and unmasked and is compliant with | |
591 | * the ARMv7 architecture. | |
592 | */ | |
fbb4aeec | 593 | vtimer->cnt_ctl = 0; |
a91d1855 | 594 | ptimer->cnt_ctl = 0; |
4b4b4512 | 595 | kvm_timer_update_state(vcpu); |
4ad9e16a | 596 | |
41a54482 | 597 | return 0; |
5ae7f87a AP |
598 | } |
599 | ||
90de943a JL |
600 | /* Make the updates of cntvoff for all vtimer contexts atomic */ |
601 | static void update_vtimer_cntvoff(struct kvm_vcpu *vcpu, u64 cntvoff) | |
602 | { | |
603 | int i; | |
604 | struct kvm *kvm = vcpu->kvm; | |
605 | struct kvm_vcpu *tmp; | |
606 | ||
607 | mutex_lock(&kvm->lock); | |
608 | kvm_for_each_vcpu(i, tmp, kvm) | |
609 | vcpu_vtimer(tmp)->cntvoff = cntvoff; | |
610 | ||
611 | /* | |
612 | * When called from the vcpu create path, the CPU being created is not | |
613 | * included in the loop above, so we just set it here as well. | |
614 | */ | |
615 | vcpu_vtimer(vcpu)->cntvoff = cntvoff; | |
616 | mutex_unlock(&kvm->lock); | |
617 | } | |
618 | ||
53e72406 MZ |
619 | void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu) |
620 | { | |
621 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; | |
85e69ad7 CD |
622 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); |
623 | struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); | |
53e72406 | 624 | |
90de943a JL |
625 | /* Synchronize cntvoff across all vtimers of a VM. */ |
626 | update_vtimer_cntvoff(vcpu, kvm_phys_timer_read()); | |
a91d1855 | 627 | vcpu_ptimer(vcpu)->cntvoff = 0; |
90de943a | 628 | |
53e72406 | 629 | INIT_WORK(&timer->expired, kvm_timer_inject_irq_work); |
14d61fa9 CD |
630 | hrtimer_init(&timer->bg_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); |
631 | timer->bg_timer.function = kvm_bg_timer_expire; | |
85e69ad7 | 632 | |
f2a2129e CD |
633 | hrtimer_init(&timer->phys_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); |
634 | timer->phys_timer.function = kvm_phys_timer_expire; | |
635 | ||
85e69ad7 CD |
636 | vtimer->irq.irq = default_vtimer_irq.irq; |
637 | ptimer->irq.irq = default_ptimer_irq.irq; | |
53e72406 MZ |
638 | } |
639 | ||
640 | static void kvm_timer_init_interrupt(void *info) | |
641 | { | |
cabdc5c5 | 642 | enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags); |
53e72406 MZ |
643 | } |
644 | ||
39735a3a AP |
645 | int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value) |
646 | { | |
fbb4aeec | 647 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); |
5c5196da | 648 | struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); |
39735a3a AP |
649 | |
650 | switch (regid) { | |
651 | case KVM_REG_ARM_TIMER_CTL: | |
5c5196da | 652 | vtimer->cnt_ctl = value & ~ARCH_TIMER_CTRL_IT_STAT; |
39735a3a AP |
653 | break; |
654 | case KVM_REG_ARM_TIMER_CNT: | |
90de943a | 655 | update_vtimer_cntvoff(vcpu, kvm_phys_timer_read() - value); |
39735a3a AP |
656 | break; |
657 | case KVM_REG_ARM_TIMER_CVAL: | |
fbb4aeec | 658 | vtimer->cnt_cval = value; |
39735a3a | 659 | break; |
5c5196da CD |
660 | case KVM_REG_ARM_PTIMER_CTL: |
661 | ptimer->cnt_ctl = value & ~ARCH_TIMER_CTRL_IT_STAT; | |
662 | break; | |
663 | case KVM_REG_ARM_PTIMER_CVAL: | |
664 | ptimer->cnt_cval = value; | |
665 | break; | |
666 | ||
39735a3a AP |
667 | default: |
668 | return -1; | |
669 | } | |
4b4b4512 CD |
670 | |
671 | kvm_timer_update_state(vcpu); | |
39735a3a AP |
672 | return 0; |
673 | } | |
674 | ||
5c5196da CD |
675 | static u64 read_timer_ctl(struct arch_timer_context *timer) |
676 | { | |
677 | /* | |
678 | * Set ISTATUS bit if it's expired. | |
679 | * Note that according to ARMv8 ARM Issue A.k, ISTATUS bit is | |
680 | * UNKNOWN when ENABLE bit is 0, so we chose to set ISTATUS bit | |
681 | * regardless of ENABLE bit for our implementation convenience. | |
682 | */ | |
683 | if (!kvm_timer_compute_delta(timer)) | |
684 | return timer->cnt_ctl | ARCH_TIMER_CTRL_IT_STAT; | |
685 | else | |
686 | return timer->cnt_ctl; | |
687 | } | |
688 | ||
39735a3a AP |
689 | u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid) |
690 | { | |
5c5196da | 691 | struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); |
fbb4aeec | 692 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); |
39735a3a AP |
693 | |
694 | switch (regid) { | |
695 | case KVM_REG_ARM_TIMER_CTL: | |
5c5196da | 696 | return read_timer_ctl(vtimer); |
39735a3a | 697 | case KVM_REG_ARM_TIMER_CNT: |
90de943a | 698 | return kvm_phys_timer_read() - vtimer->cntvoff; |
39735a3a | 699 | case KVM_REG_ARM_TIMER_CVAL: |
fbb4aeec | 700 | return vtimer->cnt_cval; |
5c5196da CD |
701 | case KVM_REG_ARM_PTIMER_CTL: |
702 | return read_timer_ctl(ptimer); | |
703 | case KVM_REG_ARM_PTIMER_CVAL: | |
704 | return ptimer->cnt_cval; | |
705 | case KVM_REG_ARM_PTIMER_CNT: | |
706 | return kvm_phys_timer_read(); | |
39735a3a AP |
707 | } |
708 | return (u64)-1; | |
709 | } | |
53e72406 | 710 | |
b3c9950a | 711 | static int kvm_timer_starting_cpu(unsigned int cpu) |
53e72406 | 712 | { |
b3c9950a RC |
713 | kvm_timer_init_interrupt(NULL); |
714 | return 0; | |
53e72406 MZ |
715 | } |
716 | ||
b3c9950a RC |
717 | static int kvm_timer_dying_cpu(unsigned int cpu) |
718 | { | |
719 | disable_percpu_irq(host_vtimer_irq); | |
720 | return 0; | |
721 | } | |
53e72406 | 722 | |
53e72406 MZ |
723 | int kvm_timer_hyp_init(void) |
724 | { | |
29c2d6ff | 725 | struct arch_timer_kvm_info *info; |
53e72406 MZ |
726 | int err; |
727 | ||
29c2d6ff JG |
728 | info = arch_timer_get_kvm_info(); |
729 | timecounter = &info->timecounter; | |
53e72406 | 730 | |
8e1a0476 CD |
731 | if (!timecounter->cc) { |
732 | kvm_err("kvm_arch_timer: uninitialized timecounter\n"); | |
733 | return -ENODEV; | |
734 | } | |
735 | ||
29c2d6ff JG |
736 | if (info->virtual_irq <= 0) { |
737 | kvm_err("kvm_arch_timer: invalid virtual timer IRQ: %d\n", | |
738 | info->virtual_irq); | |
53e72406 MZ |
739 | return -ENODEV; |
740 | } | |
29c2d6ff | 741 | host_vtimer_irq = info->virtual_irq; |
53e72406 | 742 | |
cabdc5c5 MZ |
743 | host_vtimer_irq_flags = irq_get_trigger_type(host_vtimer_irq); |
744 | if (host_vtimer_irq_flags != IRQF_TRIGGER_HIGH && | |
745 | host_vtimer_irq_flags != IRQF_TRIGGER_LOW) { | |
746 | kvm_err("Invalid trigger for IRQ%d, assuming level low\n", | |
747 | host_vtimer_irq); | |
748 | host_vtimer_irq_flags = IRQF_TRIGGER_LOW; | |
749 | } | |
750 | ||
29c2d6ff | 751 | err = request_percpu_irq(host_vtimer_irq, kvm_arch_timer_handler, |
53e72406 MZ |
752 | "kvm guest timer", kvm_get_running_vcpus()); |
753 | if (err) { | |
754 | kvm_err("kvm_arch_timer: can't request interrupt %d (%d)\n", | |
29c2d6ff | 755 | host_vtimer_irq, err); |
5d947a14 | 756 | return err; |
53e72406 MZ |
757 | } |
758 | ||
40f4cba9 CD |
759 | err = irq_set_vcpu_affinity(host_vtimer_irq, kvm_get_running_vcpus()); |
760 | if (err) { | |
761 | kvm_err("kvm_arch_timer: error setting vcpu affinity\n"); | |
762 | goto out_free_irq; | |
763 | } | |
764 | ||
29c2d6ff | 765 | kvm_info("virtual timer IRQ%d\n", host_vtimer_irq); |
53e72406 | 766 | |
b3c9950a | 767 | cpuhp_setup_state(CPUHP_AP_KVM_ARM_TIMER_STARTING, |
73c1b41e | 768 | "kvm/arm/timer:starting", kvm_timer_starting_cpu, |
b3c9950a | 769 | kvm_timer_dying_cpu); |
40f4cba9 CD |
770 | return 0; |
771 | out_free_irq: | |
772 | free_percpu_irq(host_vtimer_irq, kvm_get_running_vcpus()); | |
53e72406 MZ |
773 | return err; |
774 | } | |
775 | ||
776 | void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu) | |
777 | { | |
778 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; | |
fbb4aeec | 779 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); |
53e72406 | 780 | |
14d61fa9 | 781 | soft_timer_cancel(&timer->bg_timer, &timer->expired); |
f2a2129e | 782 | soft_timer_cancel(&timer->phys_timer, NULL); |
fbb4aeec | 783 | kvm_vgic_unmap_phys_irq(vcpu, vtimer->irq.irq); |
53e72406 MZ |
784 | } |
785 | ||
abcb851d | 786 | static bool timer_irqs_are_valid(struct kvm_vcpu *vcpu) |
99a1db7a | 787 | { |
99a1db7a | 788 | int vtimer_irq, ptimer_irq; |
abcb851d | 789 | int i, ret; |
99a1db7a | 790 | |
99a1db7a | 791 | vtimer_irq = vcpu_vtimer(vcpu)->irq.irq; |
abcb851d CD |
792 | ret = kvm_vgic_set_owner(vcpu, vtimer_irq, vcpu_vtimer(vcpu)); |
793 | if (ret) | |
794 | return false; | |
99a1db7a | 795 | |
abcb851d CD |
796 | ptimer_irq = vcpu_ptimer(vcpu)->irq.irq; |
797 | ret = kvm_vgic_set_owner(vcpu, ptimer_irq, vcpu_ptimer(vcpu)); | |
798 | if (ret) | |
99a1db7a CD |
799 | return false; |
800 | ||
abcb851d | 801 | kvm_for_each_vcpu(i, vcpu, vcpu->kvm) { |
99a1db7a CD |
802 | if (vcpu_vtimer(vcpu)->irq.irq != vtimer_irq || |
803 | vcpu_ptimer(vcpu)->irq.irq != ptimer_irq) | |
804 | return false; | |
805 | } | |
806 | ||
807 | return true; | |
808 | } | |
809 | ||
41a54482 | 810 | int kvm_timer_enable(struct kvm_vcpu *vcpu) |
53e72406 | 811 | { |
41a54482 | 812 | struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; |
fbb4aeec | 813 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); |
41a54482 CD |
814 | int ret; |
815 | ||
816 | if (timer->enabled) | |
817 | return 0; | |
818 | ||
d9e13977 AG |
819 | /* Without a VGIC we do not map virtual IRQs to physical IRQs */ |
820 | if (!irqchip_in_kernel(vcpu->kvm)) | |
821 | goto no_vgic; | |
822 | ||
823 | if (!vgic_initialized(vcpu->kvm)) | |
824 | return -ENODEV; | |
825 | ||
abcb851d | 826 | if (!timer_irqs_are_valid(vcpu)) { |
99a1db7a CD |
827 | kvm_debug("incorrectly configured timer irqs\n"); |
828 | return -EINVAL; | |
829 | } | |
830 | ||
47bbd31f | 831 | ret = kvm_vgic_map_phys_irq(vcpu, host_vtimer_irq, vtimer->irq.irq); |
41a54482 CD |
832 | if (ret) |
833 | return ret; | |
834 | ||
d9e13977 | 835 | no_vgic: |
4a2c4da1 | 836 | preempt_disable(); |
fd5ebf99 | 837 | timer->enabled = 1; |
22601127 CD |
838 | if (!irqchip_in_kernel(vcpu->kvm)) |
839 | kvm_timer_vcpu_load_user(vcpu); | |
840 | else | |
841 | kvm_timer_vcpu_load_vgic(vcpu); | |
4a2c4da1 CD |
842 | preempt_enable(); |
843 | ||
41a54482 | 844 | return 0; |
05971120 | 845 | } |
53e72406 | 846 | |
488f94d7 JL |
847 | /* |
848 | * On VHE system, we only need to configure trap on physical timer and counter | |
849 | * accesses in EL0 and EL1 once, not for every world switch. | |
850 | * The host kernel runs at EL2 with HCR_EL2.TGE == 1, | |
851 | * and this makes those bits have no effect for the host kernel execution. | |
852 | */ | |
853 | void kvm_timer_init_vhe(void) | |
854 | { | |
855 | /* When HCR_EL2.E2H ==1, EL1PCEN and EL1PCTEN are shifted by 10 */ | |
856 | u32 cnthctl_shift = 10; | |
857 | u64 val; | |
858 | ||
859 | /* | |
860 | * Disallow physical timer access for the guest. | |
861 | * Physical counter access is allowed. | |
862 | */ | |
863 | val = read_sysreg(cnthctl_el2); | |
864 | val &= ~(CNTHCTL_EL1PCEN << cnthctl_shift); | |
865 | val |= (CNTHCTL_EL1PCTEN << cnthctl_shift); | |
866 | write_sysreg(val, cnthctl_el2); | |
867 | } | |
99a1db7a CD |
868 | |
869 | static void set_timer_irqs(struct kvm *kvm, int vtimer_irq, int ptimer_irq) | |
870 | { | |
871 | struct kvm_vcpu *vcpu; | |
872 | int i; | |
873 | ||
874 | kvm_for_each_vcpu(i, vcpu, kvm) { | |
875 | vcpu_vtimer(vcpu)->irq.irq = vtimer_irq; | |
876 | vcpu_ptimer(vcpu)->irq.irq = ptimer_irq; | |
877 | } | |
878 | } | |
879 | ||
880 | int kvm_arm_timer_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) | |
881 | { | |
882 | int __user *uaddr = (int __user *)(long)attr->addr; | |
883 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); | |
884 | struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); | |
885 | int irq; | |
886 | ||
887 | if (!irqchip_in_kernel(vcpu->kvm)) | |
888 | return -EINVAL; | |
889 | ||
890 | if (get_user(irq, uaddr)) | |
891 | return -EFAULT; | |
892 | ||
893 | if (!(irq_is_ppi(irq))) | |
894 | return -EINVAL; | |
895 | ||
896 | if (vcpu->arch.timer_cpu.enabled) | |
897 | return -EBUSY; | |
898 | ||
899 | switch (attr->attr) { | |
900 | case KVM_ARM_VCPU_TIMER_IRQ_VTIMER: | |
901 | set_timer_irqs(vcpu->kvm, irq, ptimer->irq.irq); | |
902 | break; | |
903 | case KVM_ARM_VCPU_TIMER_IRQ_PTIMER: | |
904 | set_timer_irqs(vcpu->kvm, vtimer->irq.irq, irq); | |
905 | break; | |
906 | default: | |
907 | return -ENXIO; | |
908 | } | |
909 | ||
910 | return 0; | |
911 | } | |
912 | ||
913 | int kvm_arm_timer_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) | |
914 | { | |
915 | int __user *uaddr = (int __user *)(long)attr->addr; | |
916 | struct arch_timer_context *timer; | |
917 | int irq; | |
918 | ||
919 | switch (attr->attr) { | |
920 | case KVM_ARM_VCPU_TIMER_IRQ_VTIMER: | |
921 | timer = vcpu_vtimer(vcpu); | |
922 | break; | |
923 | case KVM_ARM_VCPU_TIMER_IRQ_PTIMER: | |
924 | timer = vcpu_ptimer(vcpu); | |
925 | break; | |
926 | default: | |
927 | return -ENXIO; | |
928 | } | |
929 | ||
930 | irq = timer->irq.irq; | |
931 | return put_user(irq, uaddr); | |
932 | } | |
933 | ||
934 | int kvm_arm_timer_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) | |
935 | { | |
936 | switch (attr->attr) { | |
937 | case KVM_ARM_VCPU_TIMER_IRQ_VTIMER: | |
938 | case KVM_ARM_VCPU_TIMER_IRQ_PTIMER: | |
939 | return 0; | |
940 | } | |
941 | ||
942 | return -ENXIO; | |
943 | } |