]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - arch/x86/kvm/lapic.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: LAPIC: Apply change to TDCR right away to the timer
[mirror_ubuntu-bionic-kernel.git] / arch / x86 / kvm / lapic.c
CommitLineData
97222cc8
ED		 1
2/*
3 * Local APIC virtualization
4 *
5 * Copyright (C) 2006 Qumranet, Inc.
6 * Copyright (C) 2007 Novell
7 * Copyright (C) 2007 Intel
9611c187 8 * Copyright 2009 Red Hat, Inc. and/or its affiliates.
97222cc8
ED		 9 *
10 * Authors:
11 * Dor Laor <dor.laor@qumranet.com>
12 * Gregory Haskins <ghaskins@novell.com>
13 * Yaozu (Eddie) Dong <eddie.dong@intel.com>
14 *
15 * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation.
16 *
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
19 */
20
edf88417 21#include <linux/kvm_host.h>
97222cc8
ED		 22#include <linux/kvm.h>
23#include <linux/mm.h>
24#include <linux/highmem.h>
25#include <linux/smp.h>
26#include <linux/hrtimer.h>
27#include <linux/io.h>
1767e931 28#include <linux/export.h>
6f6d6a1a 29#include <linux/math64.h>
5a0e3ad6 30#include <linux/slab.h>
97222cc8
ED		 31#include <asm/processor.h>
32#include <asm/msr.h>
33#include <asm/page.h>
34#include <asm/current.h>
35#include <asm/apicdef.h>
d0659d94 36#include <asm/delay.h>
60063497 37#include <linux/atomic.h>
c5cc421b 38#include <linux/jump_label.h>
5fdbf976 39#include "kvm_cache_regs.h"
97222cc8 40#include "irq.h"
229456fc 41#include "trace.h"
fc61b800 42#include "x86.h"
00b27a3e 43#include "cpuid.h"
5c919412 44#include "hyperv.h"
97222cc8 45
b682b814
MT		 46#ifndef CONFIG_X86_64
47#define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
48#else
49#define mod_64(x, y) ((x) % (y))
50#endif
51
97222cc8
ED		 52#define PRId64 "d"
53#define PRIx64 "llx"
54#define PRIu64 "u"
55#define PRIo64 "o"
56
97222cc8
ED		 57/* #define apic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) */
58#define apic_debug(fmt, arg...)
59
97222cc8 60/* 14 is the version for Xeon and Pentium 8.4.8*/
1e6e2755 61#define APIC_VERSION (0x14UL | ((KVM_APIC_LVT_NUM - 1) << 16))
97222cc8
ED		 62#define LAPIC_MMIO_LENGTH (1 << 12)
63/* followed define is not in apicdef.h */
64#define APIC_SHORT_MASK 0xc0000
65#define APIC_DEST_NOSHORT 0x0
66#define APIC_DEST_MASK 0x800
67#define MAX_APIC_VECTOR 256
ecba9a52 68#define APIC_VECTORS_PER_REG 32
97222cc8 69
394457a9
NA		 70#define APIC_BROADCAST 0xFF
71#define X2APIC_BROADCAST 0xFFFFFFFFul
72
a0c9a822
MT		 73static inline int apic_test_vector(int vec, void *bitmap)
74{
75 return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
76}
77
10606919
YZ		 78bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector)
79{
80 struct kvm_lapic *apic = vcpu->arch.apic;
81
82 return apic_test_vector(vector, apic->regs + APIC_ISR) ||
83 apic_test_vector(vector, apic->regs + APIC_IRR);
84}
85
97222cc8
ED		 86static inline void apic_clear_vector(int vec, void *bitmap)
87{
88 clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
89}
90
8680b94b
MT		 91static inline int __apic_test_and_set_vector(int vec, void *bitmap)
92{
93 return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
94}
95
96static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
97{
98 return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
99}
100
c5cc421b 101struct static_key_deferred apic_hw_disabled __read_mostly;
f8c1ea10
GN		 102struct static_key_deferred apic_sw_disabled __read_mostly;
103
97222cc8
ED		 104static inline int apic_enabled(struct kvm_lapic *apic)
105{
c48f1496 106 return kvm_apic_sw_enabled(apic) && kvm_apic_hw_enabled(apic);
54e9818f
GN		 107}
108
97222cc8
ED		 109#define LVT_MASK \
110 (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK)
111
112#define LINT_MASK \
113 (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
114 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)
115
6e500439
RK		 116static inline u8 kvm_xapic_id(struct kvm_lapic *apic)
117{
118 return kvm_lapic_get_reg(apic, APIC_ID) >> 24;
119}
120
121static inline u32 kvm_x2apic_id(struct kvm_lapic *apic)
122{
123 return apic->vcpu->vcpu_id;
124}
125
e45115b6
RK		 126static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map,
127 u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) {
128 switch (map->mode) {
129 case KVM_APIC_MODE_X2APIC: {
130 u32 offset = (dest_id >> 16) * 16;
0ca52e7b 131 u32 max_apic_id = map->max_apic_id;
e45115b6
RK		 132
133 if (offset <= max_apic_id) {
134 u8 cluster_size = min(max_apic_id - offset + 1, 16U);
135
136 *cluster = &map->phys_map[offset];
137 *mask = dest_id & (0xffff >> (16 - cluster_size));
138 } else {
139 *mask = 0;
140 }
3b5a5ffa 141
e45115b6
RK		 142 return true;
143 }
144 case KVM_APIC_MODE_XAPIC_FLAT:
145 *cluster = map->xapic_flat_map;
146 *mask = dest_id & 0xff;
147 return true;
148 case KVM_APIC_MODE_XAPIC_CLUSTER:
444fdad8 149 *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf];
e45115b6
RK		 150 *mask = dest_id & 0xf;
151 return true;
152 default:
153 /* Not optimized. */
154 return false;
155 }
3548a259
RK		 156}
157
af1bae54 158static void kvm_apic_map_free(struct rcu_head *rcu)
3b5a5ffa 159{
af1bae54 160 struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu);
3b5a5ffa 161
af1bae54 162 kvfree(map);
3b5a5ffa
RK		 163}
164
1e08ec4a
GN		 165static void recalculate_apic_map(struct kvm *kvm)
166{
167 struct kvm_apic_map *new, *old = NULL;
168 struct kvm_vcpu *vcpu;
169 int i;
6e500439 170 u32 max_id = 255; /* enough space for any xAPIC ID */
1e08ec4a
GN		 171
172 mutex_lock(&kvm->arch.apic_map_lock);
173
0ca52e7b
RK		 174 kvm_for_each_vcpu(i, vcpu, kvm)
175 if (kvm_apic_present(vcpu))
6e500439 176 max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic));
0ca52e7b 177
a7c3e901
MH		 178 new = kvzalloc(sizeof(struct kvm_apic_map) +
179 sizeof(struct kvm_lapic *) * ((u64)max_id + 1), GFP_KERNEL);
0ca52e7b 180
1e08ec4a
GN		 181 if (!new)
182 goto out;
183
0ca52e7b
RK		 184 new->max_apic_id = max_id;
185
173beedc
NA		 186 kvm_for_each_vcpu(i, vcpu, kvm) {
187 struct kvm_lapic *apic = vcpu->arch.apic;
e45115b6
RK		 188 struct kvm_lapic **cluster;
189 u16 mask;
5bd5db38
RK		 190 u32 ldr;
191 u8 xapic_id;
192 u32 x2apic_id;
1e08ec4a 193
df04d1d1
RK		 194 if (!kvm_apic_present(vcpu))
195 continue;
196
5bd5db38
RK		 197 xapic_id = kvm_xapic_id(apic);
198 x2apic_id = kvm_x2apic_id(apic);
199
200 /* Hotplug hack: see kvm_apic_match_physical_addr(), ... */
201 if ((apic_x2apic_mode(apic) || x2apic_id > 0xff) &&
202 x2apic_id <= new->max_apic_id)
203 new->phys_map[x2apic_id] = apic;
204 /*
205 * ... xAPIC ID of VCPUs with APIC ID > 0xff will wrap-around,
206 * prevent them from masking VCPUs with APIC ID <= 0xff.
207 */
208 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id])
209 new->phys_map[xapic_id] = apic;
3548a259 210
6e500439
RK		 211 ldr = kvm_lapic_get_reg(apic, APIC_LDR);
212
3b5a5ffa
RK		 213 if (apic_x2apic_mode(apic)) {
214 new->mode |= KVM_APIC_MODE_X2APIC;
215 } else if (ldr) {
216 ldr = GET_APIC_LOGICAL_ID(ldr);
dfb95954 217 if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT)
3b5a5ffa
RK		 218 new->mode |= KVM_APIC_MODE_XAPIC_FLAT;
219 else
220 new->mode |= KVM_APIC_MODE_XAPIC_CLUSTER;
221 }
222
e45115b6 223 if (!kvm_apic_map_get_logical_dest(new, ldr, &cluster, &mask))
3548a259
RK		 224 continue;
225
e45115b6
RK		 226 if (mask)
227 cluster[ffs(mask) - 1] = apic;
1e08ec4a
GN		 228 }
229out:
230 old = rcu_dereference_protected(kvm->arch.apic_map,
231 lockdep_is_held(&kvm->arch.apic_map_lock));
232 rcu_assign_pointer(kvm->arch.apic_map, new);
233 mutex_unlock(&kvm->arch.apic_map_lock);
234
235 if (old)
af1bae54 236 call_rcu(&old->rcu, kvm_apic_map_free);
c7c9c56c 237
b053b2ae 238 kvm_make_scan_ioapic_request(kvm);
1e08ec4a
GN		 239}
240
1e1b6c26
NA		 241static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
242{
e462755c 243 bool enabled = val & APIC_SPIV_APIC_ENABLED;
1e1b6c26 244
1e6e2755 245 kvm_lapic_set_reg(apic, APIC_SPIV, val);
e462755c
RK		 246
247 if (enabled != apic->sw_enabled) {
248 apic->sw_enabled = enabled;
249 if (enabled) {
1e1b6c26
NA		 250 static_key_slow_dec_deferred(&apic_sw_disabled);
251 recalculate_apic_map(apic->vcpu->kvm);
252 } else
253 static_key_slow_inc(&apic_sw_disabled.key);
254 }
255}
256
a92e2543 257static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id)
1e08ec4a 258{
1e6e2755 259 kvm_lapic_set_reg(apic, APIC_ID, id << 24);
1e08ec4a
GN		 260 recalculate_apic_map(apic->vcpu->kvm);
261}
262
263static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id)
264{
1e6e2755 265 kvm_lapic_set_reg(apic, APIC_LDR, id);
1e08ec4a
GN		 266 recalculate_apic_map(apic->vcpu->kvm);
267}
268
a92e2543 269static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id)
257b9a5f
RK		 270{
271 u32 ldr = ((id >> 4) << 16) | (1 << (id & 0xf));
272
6e500439
RK		 273 WARN_ON_ONCE(id != apic->vcpu->vcpu_id);
274
a92e2543 275 kvm_lapic_set_reg(apic, APIC_ID, id);
1e6e2755 276 kvm_lapic_set_reg(apic, APIC_LDR, ldr);
257b9a5f
RK		 277 recalculate_apic_map(apic->vcpu->kvm);
278}
279
97222cc8
ED		 280static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type)
281{
dfb95954 282 return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED);
97222cc8
ED		 283}
284
285static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type)
286{
dfb95954 287 return kvm_lapic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK;
97222cc8
ED		 288}
289
a3e06bbe
LJ		 290static inline int apic_lvtt_oneshot(struct kvm_lapic *apic)
291{
f30ebc31 292 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT;
a3e06bbe
LJ		 293}
294
97222cc8
ED		 295static inline int apic_lvtt_period(struct kvm_lapic *apic)
296{
f30ebc31 297 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC;
a3e06bbe
LJ		 298}
299
300static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic)
301{
f30ebc31 302 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE;
97222cc8
ED		 303}
304
cc6e462c
JK		 305static inline int apic_lvt_nmi_mode(u32 lvt_val)
306{
307 return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI;
308}
309
fc61b800
GN		 310void kvm_apic_set_version(struct kvm_vcpu *vcpu)
311{
312 struct kvm_lapic *apic = vcpu->arch.apic;
313 struct kvm_cpuid_entry2 *feat;
314 u32 v = APIC_VERSION;
315
bce87cce 316 if (!lapic_in_kernel(vcpu))
fc61b800
GN		 317 return;
318
319 feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0);
320 if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31))))
321 v |= APIC_LVR_DIRECTED_EOI;
1e6e2755 322 kvm_lapic_set_reg(apic, APIC_LVR, v);
fc61b800
GN		 323}
324
1e6e2755 325static const unsigned int apic_lvt_mask[KVM_APIC_LVT_NUM] = {
a3e06bbe 326 LVT_MASK , /* part LVTT mask, timer mode mask added at runtime */
97222cc8
ED		 327 LVT_MASK | APIC_MODE_MASK, /* LVTTHMR */
328 LVT_MASK | APIC_MODE_MASK, /* LVTPC */
329 LINT_MASK, LINT_MASK, /* LVT0-1 */
330 LVT_MASK /* LVTERR */
331};
332
333static int find_highest_vector(void *bitmap)
334{
ecba9a52
TY		 335 int vec;
336 u32 *reg;
97222cc8 337
ecba9a52
TY		 338 for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG;
339 vec >= 0; vec -= APIC_VECTORS_PER_REG) {
340 reg = bitmap + REG_POS(vec);
341 if (*reg)
810e6def 342 return __fls(*reg) + vec;
ecba9a52 343 }
97222cc8 344
ecba9a52 345 return -1;
97222cc8
ED		 346}
347
8680b94b
MT		 348static u8 count_vectors(void *bitmap)
349{
ecba9a52
TY		 350 int vec;
351 u32 *reg;
8680b94b 352 u8 count = 0;
ecba9a52
TY		 353
354 for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) {
355 reg = bitmap + REG_POS(vec);
356 count += hweight32(*reg);
357 }
358
8680b94b
MT		 359 return count;
360}
361
810e6def 362int __kvm_apic_update_irr(u32 *pir, void *regs)
a20ed54d 363{
810e6def
PB		 364 u32 i, vec;
365 u32 pir_val, irr_val;
366 int max_irr = -1;
a20ed54d 367
810e6def 368 for (i = vec = 0; i <= 7; i++, vec += 32) {
ad361091 369 pir_val = READ_ONCE(pir[i]);
810e6def 370 irr_val = *((u32 *)(regs + APIC_IRR + i * 0x10));
ad361091 371 if (pir_val) {
810e6def
PB		 372 irr_val |= xchg(&pir[i], 0);
373 *((u32 *)(regs + APIC_IRR + i * 0x10)) = irr_val;
ad361091 374 }
810e6def
PB		 375 if (irr_val)
376 max_irr = __fls(irr_val) + vec;
a20ed54d 377 }
810e6def
PB		 378
379 return max_irr;
a20ed54d 380}
705699a1
WV		 381EXPORT_SYMBOL_GPL(__kvm_apic_update_irr);
382
810e6def 383int kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir)
705699a1
WV		 384{
385 struct kvm_lapic *apic = vcpu->arch.apic;
386
b95234c8 387 return __kvm_apic_update_irr(pir, apic->regs);
705699a1 388}
a20ed54d
YZ		 389EXPORT_SYMBOL_GPL(kvm_apic_update_irr);
390
33e4c686 391static inline int apic_search_irr(struct kvm_lapic *apic)
97222cc8 392{
33e4c686 393 return find_highest_vector(apic->regs + APIC_IRR);
97222cc8
ED		 394}
395
396static inline int apic_find_highest_irr(struct kvm_lapic *apic)
397{
398 int result;
399
c7c9c56c
YZ		 400 /*
401 * Note that irr_pending is just a hint. It will be always
402 * true with virtual interrupt delivery enabled.
403 */
33e4c686
GN		 404 if (!apic->irr_pending)
405 return -1;
406
407 result = apic_search_irr(apic);
97222cc8
ED		 408 ASSERT(result == -1 || result >= 16);
409
410 return result;
411}
412
33e4c686
GN		 413static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
414{
56cc2406
WL		 415 struct kvm_vcpu *vcpu;
416
417 vcpu = apic->vcpu;
418
d62caabb 419 if (unlikely(vcpu->arch.apicv_active)) {
b95234c8 420 /* need to update RVI */
f210f757 421 apic_clear_vector(vec, apic->regs + APIC_IRR);
b95234c8
PB		 422 kvm_x86_ops->hwapic_irr_update(vcpu,
423 apic_find_highest_irr(apic));
f210f757
NA		 424 } else {
425 apic->irr_pending = false;
426 apic_clear_vector(vec, apic->regs + APIC_IRR);
427 if (apic_search_irr(apic) != -1)
428 apic->irr_pending = true;
56cc2406 429 }
33e4c686
GN		 430}
431
8680b94b
MT		 432static inline void apic_set_isr(int vec, struct kvm_lapic *apic)
433{
56cc2406
WL		 434 struct kvm_vcpu *vcpu;
435
436 if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR))
437 return;
438
439 vcpu = apic->vcpu;
fc57ac2c 440
8680b94b 441 /*
56cc2406
WL		 442 * With APIC virtualization enabled, all caching is disabled
443 * because the processor can modify ISR under the hood. Instead
444 * just set SVI.
8680b94b 445 */
d62caabb 446 if (unlikely(vcpu->arch.apicv_active))
67c9dddc 447 kvm_x86_ops->hwapic_isr_update(vcpu, vec);
56cc2406
WL		 448 else {
449 ++apic->isr_count;
450 BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
451 /*
452 * ISR (in service register) bit is set when injecting an interrupt.
453 * The highest vector is injected. Thus the latest bit set matches
454 * the highest bit in ISR.
455 */
456 apic->highest_isr_cache = vec;
457 }
8680b94b
MT		 458}
459
fc57ac2c
PB		 460static inline int apic_find_highest_isr(struct kvm_lapic *apic)
461{
462 int result;
463
464 /*
465 * Note that isr_count is always 1, and highest_isr_cache
466 * is always -1, with APIC virtualization enabled.
467 */
468 if (!apic->isr_count)
469 return -1;
470 if (likely(apic->highest_isr_cache != -1))
471 return apic->highest_isr_cache;
472
473 result = find_highest_vector(apic->regs + APIC_ISR);
474 ASSERT(result == -1 || result >= 16);
475
476 return result;
477}
478
8680b94b
MT		 479static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
480{
fc57ac2c
PB		 481 struct kvm_vcpu *vcpu;
482 if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR))
483 return;
484
485 vcpu = apic->vcpu;
486
487 /*
488 * We do get here for APIC virtualization enabled if the guest
489 * uses the Hyper-V APIC enlightenment. In this case we may need
490 * to trigger a new interrupt delivery by writing the SVI field;
491 * on the other hand isr_count and highest_isr_cache are unused
492 * and must be left alone.
493 */
d62caabb 494 if (unlikely(vcpu->arch.apicv_active))
67c9dddc 495 kvm_x86_ops->hwapic_isr_update(vcpu,
fc57ac2c
PB		 496 apic_find_highest_isr(apic));
497 else {
8680b94b 498 --apic->isr_count;
fc57ac2c
PB		 499 BUG_ON(apic->isr_count < 0);
500 apic->highest_isr_cache = -1;
501 }
8680b94b
MT		 502}
503
6e5d865c
YS		 504int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
505{
33e4c686
GN		 506 /* This may race with setting of irr in __apic_accept_irq() and
507 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq
508 * will cause vmexit immediately and the value will be recalculated
509 * on the next vmentry.
510 */
f8543d6a 511 return apic_find_highest_irr(vcpu->arch.apic);
6e5d865c 512}
76dfafd5 513EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr);
6e5d865c 514
6da7e3f6 515static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
b4f2225c 516 int vector, int level, int trig_mode,
9e4aabe2 517 struct dest_map *dest_map);
6da7e3f6 518
b4f2225c 519int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq,
9e4aabe2 520 struct dest_map *dest_map)
97222cc8 521{
ad312c7c 522 struct kvm_lapic *apic = vcpu->arch.apic;
8be5453f 523
58c2dde1 524 return __apic_accept_irq(apic, irq->delivery_mode, irq->vector,
b4f2225c 525 irq->level, irq->trig_mode, dest_map);
97222cc8
ED		 526}
527
ae7a2a3f
MT		 528static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val)
529{
4e335d9e
PB		 530
531 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val,
532 sizeof(val));
ae7a2a3f
MT		 533}
534
535static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val)
536{
4e335d9e
PB		 537
538 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val,
539 sizeof(*val));
ae7a2a3f
MT		 540}
541
542static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu)
543{
544 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;
545}
546
547static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu)
548{
549 u8 val;
550 if (pv_eoi_get_user(vcpu, &val) < 0)
551 apic_debug("Can't read EOI MSR value: 0x%llx\n",
96893977 552 (unsigned long long)vcpu->arch.pv_eoi.msr_val);
ae7a2a3f
MT		 553 return val & 0x1;
554}
555
556static void pv_eoi_set_pending(struct kvm_vcpu *vcpu)
557{
558 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) {
559 apic_debug("Can't set EOI MSR value: 0x%llx\n",
96893977 560 (unsigned long long)vcpu->arch.pv_eoi.msr_val);
ae7a2a3f
MT		 561 return;
562 }
563 __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
564}
565
566static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu)
567{
568 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) {
569 apic_debug("Can't clear EOI MSR value: 0x%llx\n",
96893977 570 (unsigned long long)vcpu->arch.pv_eoi.msr_val);
ae7a2a3f
MT		 571 return;
572 }
573 __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
574}
575
b3c045d3
PB		 576static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr)
577{
3d92789f 578 int highest_irr;
76dfafd5
PB		 579 if (kvm_x86_ops->sync_pir_to_irr && apic->vcpu->arch.apicv_active)
580 highest_irr = kvm_x86_ops->sync_pir_to_irr(apic->vcpu);
581 else
582 highest_irr = apic_find_highest_irr(apic);
b3c045d3
PB		 583 if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr)
584 return -1;
585 return highest_irr;
586}
587
588static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr)
97222cc8 589{
3842d135 590 u32 tpr, isrv, ppr, old_ppr;
97222cc8
ED		 591 int isr;
592
dfb95954
SS		 593 old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI);
594 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI);
97222cc8
ED		 595 isr = apic_find_highest_isr(apic);
596 isrv = (isr != -1) ? isr : 0;
597
598 if ((tpr & 0xf0) >= (isrv & 0xf0))
599 ppr = tpr & 0xff;
600 else
601 ppr = isrv & 0xf0;
602
603 apic_debug("vlapic %p, ppr 0x%x, isr 0x%x, isrv 0x%x",
604 apic, ppr, isr, isrv);
605
b3c045d3
PB		 606 *new_ppr = ppr;
607 if (old_ppr != ppr)
1e6e2755 608 kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr);
b3c045d3
PB		 609
610 return ppr < old_ppr;
611}
612
613static void apic_update_ppr(struct kvm_lapic *apic)
614{
615 u32 ppr;
616
26fbbee5
PB		 617 if (__apic_update_ppr(apic, &ppr) &&
618 apic_has_interrupt_for_ppr(apic, ppr) != -1)
b3c045d3 619 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
97222cc8
ED		 620}
621
eb90f341
PB		 622void kvm_apic_update_ppr(struct kvm_vcpu *vcpu)
623{
624 apic_update_ppr(vcpu->arch.apic);
625}
626EXPORT_SYMBOL_GPL(kvm_apic_update_ppr);
627
97222cc8
ED		 628static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr)
629{
1e6e2755 630 kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr);
97222cc8
ED		 631 apic_update_ppr(apic);
632}
633
03d2249e 634static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda)
394457a9 635{
b4535b58
RK		 636 return mda == (apic_x2apic_mode(apic) ?
637 X2APIC_BROADCAST : APIC_BROADCAST);
394457a9
NA		 638}
639
03d2249e 640static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda)
97222cc8 641{
03d2249e
RK		 642 if (kvm_apic_broadcast(apic, mda))
643 return true;
644
645 if (apic_x2apic_mode(apic))
6e500439 646 return mda == kvm_x2apic_id(apic);
03d2249e 647
5bd5db38
RK		 648 /*
649 * Hotplug hack: Make LAPIC in xAPIC mode also accept interrupts as if
650 * it were in x2APIC mode. Hotplugged VCPUs start in xAPIC mode and
651 * this allows unique addressing of VCPUs with APIC ID over 0xff.
652 * The 0xff condition is needed because writeable xAPIC ID.
653 */
654 if (kvm_x2apic_id(apic) > 0xff && mda == kvm_x2apic_id(apic))
655 return true;
656
b4535b58 657 return mda == kvm_xapic_id(apic);
97222cc8
ED		 658}
659
52c233a4 660static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
97222cc8 661{
0105d1a5
GN		 662 u32 logical_id;
663
394457a9 664 if (kvm_apic_broadcast(apic, mda))
9368b567 665 return true;
394457a9 666
dfb95954 667 logical_id = kvm_lapic_get_reg(apic, APIC_LDR);
97222cc8 668
9368b567 669 if (apic_x2apic_mode(apic))
8a395363
RK		 670 return ((logical_id >> 16) == (mda >> 16))
671 && (logical_id & mda & 0xffff) != 0;
97222cc8 672
9368b567 673 logical_id = GET_APIC_LOGICAL_ID(logical_id);
97222cc8 674
dfb95954 675 switch (kvm_lapic_get_reg(apic, APIC_DFR)) {
97222cc8 676 case APIC_DFR_FLAT:
9368b567 677 return (logical_id & mda) != 0;
97222cc8 678 case APIC_DFR_CLUSTER:
9368b567
RK		 679 return ((logical_id >> 4) == (mda >> 4))
680 && (logical_id & mda & 0xf) != 0;
97222cc8 681 default:
7712de87 682 apic_debug("Bad DFR vcpu %d: %08x\n",
dfb95954 683 apic->vcpu->vcpu_id, kvm_lapic_get_reg(apic, APIC_DFR));
9368b567 684 return false;
97222cc8 685 }
97222cc8
ED		 686}
687
c519265f
RK		 688/* The KVM local APIC implementation has two quirks:
689 *
b4535b58
RK		 690 * - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs
691 * in xAPIC mode if the "destination & 0xff" matches its xAPIC ID.
692 * KVM doesn't do that aliasing.
c519265f
RK		 693 *
694 * - in-kernel IOAPIC messages have to be delivered directly to
695 * x2APIC, because the kernel does not support interrupt remapping.
696 * In order to support broadcast without interrupt remapping, x2APIC
697 * rewrites the destination of non-IPI messages from APIC_BROADCAST
698 * to X2APIC_BROADCAST.
699 *
700 * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API. This is
701 * important when userspace wants to use x2APIC-format MSIs, because
702 * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7".
03d2249e 703 */
c519265f
RK		 704static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id,
705 struct kvm_lapic *source, struct kvm_lapic *target)
03d2249e
RK		 706{
707 bool ipi = source != NULL;
03d2249e 708
c519265f 709 if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled &&
b4535b58 710 !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target))
03d2249e
RK		 711 return X2APIC_BROADCAST;
712
b4535b58 713 return dest_id;
03d2249e
RK		 714}
715
52c233a4 716bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
394457a9 717 int short_hand, unsigned int dest, int dest_mode)
97222cc8 718{
ad312c7c 719 struct kvm_lapic *target = vcpu->arch.apic;
c519265f 720 u32 mda = kvm_apic_mda(vcpu, dest, source, target);
97222cc8
ED		 721
722 apic_debug("target %p, source %p, dest 0x%x, "
343f94fe 723 "dest_mode 0x%x, short_hand 0x%x\n",
97222cc8
ED		 724 target, source, dest, dest_mode, short_hand);
725
bd371396 726 ASSERT(target);
97222cc8
ED		 727 switch (short_hand) {
728 case APIC_DEST_NOSHORT:
3697f302 729 if (dest_mode == APIC_DEST_PHYSICAL)
03d2249e 730 return kvm_apic_match_physical_addr(target, mda);
343f94fe 731 else
03d2249e 732 return kvm_apic_match_logical_addr(target, mda);
97222cc8 733 case APIC_DEST_SELF:
9368b567 734 return target == source;
97222cc8 735 case APIC_DEST_ALLINC:
9368b567 736 return true;
97222cc8 737 case APIC_DEST_ALLBUT:
9368b567 738 return target != source;
97222cc8 739 default:
7712de87
JK		 740 apic_debug("kvm: apic: Bad dest shorthand value %x\n",
741 short_hand);
9368b567 742 return false;
97222cc8 743 }
97222cc8 744}
1e6e2755 745EXPORT_SYMBOL_GPL(kvm_apic_match_dest);
97222cc8 746
52004014
FW		 747int kvm_vector_to_index(u32 vector, u32 dest_vcpus,
748 const unsigned long *bitmap, u32 bitmap_size)
749{
750 u32 mod;
751 int i, idx = -1;
752
753 mod = vector % dest_vcpus;
754
755 for (i = 0; i <= mod; i++) {
756 idx = find_next_bit(bitmap, bitmap_size, idx + 1);
757 BUG_ON(idx == bitmap_size);
758 }
759
760 return idx;
761}
762
4efd805f
RK		 763static void kvm_apic_disabled_lapic_found(struct kvm *kvm)
764{
765 if (!kvm->arch.disabled_lapic_found) {
766 kvm->arch.disabled_lapic_found = true;
767 printk(KERN_INFO
768 "Disabled LAPIC found during irq injection\n");
769 }
770}
771
c519265f
RK		 772static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src,
773 struct kvm_lapic_irq *irq, struct kvm_apic_map *map)
1e08ec4a 774{
c519265f
RK		 775 if (kvm->arch.x2apic_broadcast_quirk_disabled) {
776 if ((irq->dest_id == APIC_BROADCAST &&
777 map->mode != KVM_APIC_MODE_X2APIC))
778 return true;
779 if (irq->dest_id == X2APIC_BROADCAST)
780 return true;
781 } else {
782 bool x2apic_ipi = src && *src && apic_x2apic_mode(*src);
783 if (irq->dest_id == (x2apic_ipi ?
784 X2APIC_BROADCAST : APIC_BROADCAST))
785 return true;
786 }
1e08ec4a 787
c519265f
RK		 788 return false;
789}
1e08ec4a 790
64aa47bf
RK		 791/* Return true if the interrupt can be handled by using *bitmap as index mask
792 * for valid destinations in *dst array.
793 * Return false if kvm_apic_map_get_dest_lapic did nothing useful.
794 * Note: we may have zero kvm_lapic destinations when we return true, which
795 * means that the interrupt should be dropped. In this case, *bitmap would be
796 * zero and *dst undefined.
797 */
798static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm,
799 struct kvm_lapic **src, struct kvm_lapic_irq *irq,
800 struct kvm_apic_map *map, struct kvm_lapic ***dst,
801 unsigned long *bitmap)
802{
803 int i, lowest;
1e08ec4a 804
64aa47bf
RK		 805 if (irq->shorthand == APIC_DEST_SELF && src) {
806 *dst = src;
807 *bitmap = 1;
808 return true;
809 } else if (irq->shorthand)
1e08ec4a
GN		 810 return false;
811
c519265f 812 if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map))
9ea369b0
RK		 813 return false;
814
64aa47bf 815 if (irq->dest_mode == APIC_DEST_PHYSICAL) {
0ca52e7b 816 if (irq->dest_id > map->max_apic_id) {
64aa47bf
RK		 817 *bitmap = 0;
818 } else {
819 *dst = &map->phys_map[irq->dest_id];
820 *bitmap = 1;
821 }
1e08ec4a 822 return true;
bea15428 823 }
698f9755 824
e45115b6
RK		 825 *bitmap = 0;
826 if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst,
827 (u16 *)bitmap))
1e08ec4a 828 return false;
fa834e91 829
64aa47bf
RK		 830 if (!kvm_lowest_prio_delivery(irq))
831 return true;
3548a259 832
64aa47bf
RK		 833 if (!kvm_vector_hashing_enabled()) {
834 lowest = -1;
835 for_each_set_bit(i, bitmap, 16) {
836 if (!(*dst)[i])
837 continue;
838 if (lowest < 0)
839 lowest = i;
840 else if (kvm_apic_compare_prio((*dst)[i]->vcpu,
841 (*dst)[lowest]->vcpu) < 0)
842 lowest = i;
3548a259 843 }
64aa47bf
RK		 844 } else {
845 if (!*bitmap)
846 return true;
3548a259 847
64aa47bf
RK		 848 lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap),
849 bitmap, 16);
45c3094a 850
64aa47bf
RK		 851 if (!(*dst)[lowest]) {
852 kvm_apic_disabled_lapic_found(kvm);
853 *bitmap = 0;
854 return true;
855 }
856 }
1e08ec4a 857
64aa47bf 858 *bitmap = (lowest >= 0) ? 1 << lowest : 0;
1e08ec4a 859
64aa47bf
RK		 860 return true;
861}
52004014 862
64aa47bf
RK		 863bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
864 struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map)
865{
866 struct kvm_apic_map *map;
867 unsigned long bitmap;
868 struct kvm_lapic **dst = NULL;
869 int i;
870 bool ret;
52004014 871
64aa47bf 872 *r = -1;
52004014 873
64aa47bf
RK		 874 if (irq->shorthand == APIC_DEST_SELF) {
875 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map);
876 return true;
877 }
52004014 878
64aa47bf
RK		 879 rcu_read_lock();
880 map = rcu_dereference(kvm->arch.apic_map);
52004014 881
64aa47bf
RK		 882 ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap);
883 if (ret)
884 for_each_set_bit(i, &bitmap, 16) {
885 if (!dst[i])
886 continue;
887 if (*r < 0)
888 *r = 0;
889 *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map);
1e08ec4a 890 }
1e08ec4a 891
1e08ec4a
GN		 892 rcu_read_unlock();
893 return ret;
894}
895
6228a0da
FW		 896/*
897 * This routine tries to handler interrupts in posted mode, here is how
898 * it deals with different cases:
899 * - For single-destination interrupts, handle it in posted mode
900 * - Else if vector hashing is enabled and it is a lowest-priority
901 * interrupt, handle it in posted mode and use the following mechanism
902 * to find the destinaiton vCPU.
903 * 1. For lowest-priority interrupts, store all the possible
904 * destination vCPUs in an array.
905 * 2. Use "guest vector % max number of destination vCPUs" to find
906 * the right destination vCPU in the array for the lowest-priority
907 * interrupt.
908 * - Otherwise, use remapped mode to inject the interrupt.
909 */
8feb4a04
FW		 910bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq,
911 struct kvm_vcpu **dest_vcpu)
912{
913 struct kvm_apic_map *map;
64aa47bf
RK		 914 unsigned long bitmap;
915 struct kvm_lapic **dst = NULL;
8feb4a04 916 bool ret = false;
8feb4a04
FW		 917
918 if (irq->shorthand)
919 return false;
920
921 rcu_read_lock();
922 map = rcu_dereference(kvm->arch.apic_map);
923
64aa47bf
RK		 924 if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) &&
925 hweight16(bitmap) == 1) {
926 unsigned long i = find_first_bit(&bitmap, 16);
6228a0da 927
64aa47bf
RK		 928 if (dst[i]) {
929 *dest_vcpu = dst[i]->vcpu;
930 ret = true;
6228a0da 931 }
8feb4a04
FW		 932 }
933
8feb4a04
FW		 934 rcu_read_unlock();
935 return ret;
936}
937
97222cc8
ED		 938/*
939 * Add a pending IRQ into lapic.
940 * Return 1 if successfully added and 0 if discarded.
941 */
942static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
b4f2225c 943 int vector, int level, int trig_mode,
9e4aabe2 944 struct dest_map *dest_map)
97222cc8 945{
6da7e3f6 946 int result = 0;
c5ec1534 947 struct kvm_vcpu *vcpu = apic->vcpu;
97222cc8 948
a183b638
PB		 949 trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode,
950 trig_mode, vector);
97222cc8 951 switch (delivery_mode) {
97222cc8 952 case APIC_DM_LOWEST:
e1035715
GN		 953 vcpu->arch.apic_arb_prio++;
954 case APIC_DM_FIXED:
bdaffe1d
PB		 955 if (unlikely(trig_mode && !level))
956 break;
957
97222cc8
ED		 958 /* FIXME add logic for vcpu on reset */
959 if (unlikely(!apic_enabled(apic)))
960 break;
961
11f5cc05
JK		 962 result = 1;
963
9daa5007 964 if (dest_map) {
9e4aabe2 965 __set_bit(vcpu->vcpu_id, dest_map->map);
9daa5007
JR		 966 dest_map->vectors[vcpu->vcpu_id] = vector;
967 }
a5d36f82 968
bdaffe1d
PB		 969 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) {
970 if (trig_mode)
1e6e2755 971 kvm_lapic_set_vector(vector, apic->regs + APIC_TMR);
bdaffe1d
PB		 972 else
973 apic_clear_vector(vector, apic->regs + APIC_TMR);
974 }
975
d62caabb 976 if (vcpu->arch.apicv_active)
5a71785d 977 kvm_x86_ops->deliver_posted_interrupt(vcpu, vector);
11f5cc05 978 else {
1e6e2755 979 kvm_lapic_set_irr(vector, apic);
5a71785d
YZ		 980
981 kvm_make_request(KVM_REQ_EVENT, vcpu);
982 kvm_vcpu_kick(vcpu);
983 }
97222cc8
ED		 984 break;
985
986 case APIC_DM_REMRD:
24d2166b
R		 987 result = 1;
988 vcpu->arch.pv.pv_unhalted = 1;
989 kvm_make_request(KVM_REQ_EVENT, vcpu);
990 kvm_vcpu_kick(vcpu);
97222cc8
ED		 991 break;
992
993 case APIC_DM_SMI:
64d60670
PB		 994 result = 1;
995 kvm_make_request(KVM_REQ_SMI, vcpu);
996 kvm_vcpu_kick(vcpu);
97222cc8 997 break;
3419ffc8 998
97222cc8 999 case APIC_DM_NMI:
6da7e3f6 1000 result = 1;
3419ffc8 1001 kvm_inject_nmi(vcpu);
26df99c6 1002 kvm_vcpu_kick(vcpu);
97222cc8
ED		 1003 break;
1004
1005 case APIC_DM_INIT:
a52315e1 1006 if (!trig_mode || level) {
6da7e3f6 1007 result = 1;
66450a21
JK		 1008 /* assumes that there are only KVM_APIC_INIT/SIPI */
1009 apic->pending_events = (1UL << KVM_APIC_INIT);
1010 /* make sure pending_events is visible before sending
1011 * the request */
1012 smp_wmb();
3842d135 1013 kvm_make_request(KVM_REQ_EVENT, vcpu);
c5ec1534
HQ		 1014 kvm_vcpu_kick(vcpu);
1015 } else {
1b10bf31
JK		 1016 apic_debug("Ignoring de-assert INIT to vcpu %d\n",
1017 vcpu->vcpu_id);
c5ec1534 1018 }
97222cc8
ED		 1019 break;
1020
1021 case APIC_DM_STARTUP:
1b10bf31
JK		 1022 apic_debug("SIPI to vcpu %d vector 0x%02x\n",
1023 vcpu->vcpu_id, vector);
66450a21
JK		 1024 result = 1;
1025 apic->sipi_vector = vector;
1026 /* make sure sipi_vector is visible for the receiver */
1027 smp_wmb();
1028 set_bit(KVM_APIC_SIPI, &apic->pending_events);
1029 kvm_make_request(KVM_REQ_EVENT, vcpu);
1030 kvm_vcpu_kick(vcpu);
97222cc8
ED		 1031 break;
1032
23930f95
JK		 1033 case APIC_DM_EXTINT:
1034 /*
1035 * Should only be called by kvm_apic_local_deliver() with LVT0,
1036 * before NMI watchdog was enabled. Already handled by
1037 * kvm_apic_accept_pic_intr().
1038 */
1039 break;
1040
97222cc8
ED		 1041 default:
1042 printk(KERN_ERR "TODO: unsupported delivery mode %x\n",
1043 delivery_mode);
1044 break;
1045 }
1046 return result;
1047}
1048
e1035715 1049int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
8be5453f 1050{
e1035715 1051 return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio;
8be5453f
ZX		 1052}
1053
3bb345f3
PB		 1054static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector)
1055{
6308630b 1056 return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors);
3bb345f3
PB		 1057}
1058
c7c9c56c
YZ		 1059static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector)
1060{
7543a635
SR		 1061 int trigger_mode;
1062
1063 /* Eoi the ioapic only if the ioapic doesn't own the vector. */
1064 if (!kvm_ioapic_handles_vector(apic, vector))
1065 return;
3bb345f3 1066
7543a635
SR		 1067 /* Request a KVM exit to inform the userspace IOAPIC. */
1068 if (irqchip_split(apic->vcpu->kvm)) {
1069 apic->vcpu->arch.pending_ioapic_eoi = vector;
1070 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu);
1071 return;
c7c9c56c 1072 }
7543a635
SR		 1073
1074 if (apic_test_vector(vector, apic->regs + APIC_TMR))
1075 trigger_mode = IOAPIC_LEVEL_TRIG;
1076 else
1077 trigger_mode = IOAPIC_EDGE_TRIG;
1078
1079 kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode);
c7c9c56c
YZ		 1080}
1081
ae7a2a3f 1082static int apic_set_eoi(struct kvm_lapic *apic)
97222cc8
ED		 1083{
1084 int vector = apic_find_highest_isr(apic);
ae7a2a3f
MT		 1085
1086 trace_kvm_eoi(apic, vector);
1087
97222cc8
ED		 1088 /*
1089 * Not every write EOI will has corresponding ISR,
1090 * one example is when Kernel check timer on setup_IO_APIC
1091 */
1092 if (vector == -1)
ae7a2a3f 1093 return vector;
97222cc8 1094
8680b94b 1095 apic_clear_isr(vector, apic);
97222cc8
ED		 1096 apic_update_ppr(apic);
1097
5c919412
AS		 1098 if (test_bit(vector, vcpu_to_synic(apic->vcpu)->vec_bitmap))
1099 kvm_hv_synic_send_eoi(apic->vcpu, vector);
1100
c7c9c56c 1101 kvm_ioapic_send_eoi(apic, vector);
3842d135 1102 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
ae7a2a3f 1103 return vector;
97222cc8
ED		 1104}
1105
c7c9c56c
YZ		 1106/*
1107 * this interface assumes a trap-like exit, which has already finished
1108 * desired side effect including vISR and vPPR update.
1109 */
1110void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector)
1111{
1112 struct kvm_lapic *apic = vcpu->arch.apic;
1113
1114 trace_kvm_eoi(apic, vector);
1115
1116 kvm_ioapic_send_eoi(apic, vector);
1117 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
1118}
1119EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated);
1120
97222cc8
ED		 1121static void apic_send_ipi(struct kvm_lapic *apic)
1122{
dfb95954
SS		 1123 u32 icr_low = kvm_lapic_get_reg(apic, APIC_ICR);
1124 u32 icr_high = kvm_lapic_get_reg(apic, APIC_ICR2);
58c2dde1 1125 struct kvm_lapic_irq irq;
97222cc8 1126
58c2dde1
GN		 1127 irq.vector = icr_low & APIC_VECTOR_MASK;
1128 irq.delivery_mode = icr_low & APIC_MODE_MASK;
1129 irq.dest_mode = icr_low & APIC_DEST_MASK;
b7cb2231 1130 irq.level = (icr_low & APIC_INT_ASSERT) != 0;
58c2dde1
GN		 1131 irq.trig_mode = icr_low & APIC_INT_LEVELTRIG;
1132 irq.shorthand = icr_low & APIC_SHORT_MASK;
93bbf0b8 1133 irq.msi_redir_hint = false;
0105d1a5
GN		 1134 if (apic_x2apic_mode(apic))
1135 irq.dest_id = icr_high;
1136 else
1137 irq.dest_id = GET_APIC_DEST_FIELD(icr_high);
97222cc8 1138
1000ff8d
GN		 1139 trace_kvm_apic_ipi(icr_low, irq.dest_id);
1140
97222cc8
ED		 1141 apic_debug("icr_high 0x%x, icr_low 0x%x, "
1142 "short_hand 0x%x, dest 0x%x, trig_mode 0x%x, level 0x%x, "
93bbf0b8
JS		 1143 "dest_mode 0x%x, delivery_mode 0x%x, vector 0x%x, "
1144 "msi_redir_hint 0x%x\n",
9b5843dd 1145 icr_high, icr_low, irq.shorthand, irq.dest_id,
58c2dde1 1146 irq.trig_mode, irq.level, irq.dest_mode, irq.delivery_mode,
93bbf0b8 1147 irq.vector, irq.msi_redir_hint);
58c2dde1 1148
b4f2225c 1149 kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL);
97222cc8
ED		 1150}
1151
1152static u32 apic_get_tmcct(struct kvm_lapic *apic)
1153{
8003c9ae 1154 ktime_t remaining, now;
b682b814 1155 s64 ns;
9da8f4e8 1156 u32 tmcct;
97222cc8
ED		 1157
1158 ASSERT(apic != NULL);
1159
9da8f4e8 1160 /* if initial count is 0, current count should also be 0 */
dfb95954 1161 if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 ||
b963a22e 1162 apic->lapic_timer.period == 0)
9da8f4e8
KP		 1163 return 0;
1164
5587859f 1165 now = ktime_get();
8003c9ae 1166 remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
b682b814 1167 if (ktime_to_ns(remaining) < 0)
8b0e1953 1168 remaining = 0;
b682b814 1169
d3c7b77d
MT		 1170 ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period);
1171 tmcct = div64_u64(ns,
1172 (APIC_BUS_CYCLE_NS * apic->divide_count));
97222cc8
ED		 1173
1174 return tmcct;
1175}
1176
b209749f
AK		 1177static void __report_tpr_access(struct kvm_lapic *apic, bool write)
1178{
1179 struct kvm_vcpu *vcpu = apic->vcpu;
1180 struct kvm_run *run = vcpu->run;
1181
a8eeb04a 1182 kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu);
5fdbf976 1183 run->tpr_access.rip = kvm_rip_read(vcpu);
b209749f
AK		 1184 run->tpr_access.is_write = write;
1185}
1186
1187static inline void report_tpr_access(struct kvm_lapic *apic, bool write)
1188{
1189 if (apic->vcpu->arch.tpr_access_reporting)
1190 __report_tpr_access(apic, write);
1191}
1192
97222cc8
ED		 1193static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
1194{
1195 u32 val = 0;
1196
1197 if (offset >= LAPIC_MMIO_LENGTH)
1198 return 0;
1199
1200 switch (offset) {
1201 case APIC_ARBPRI:
7712de87 1202 apic_debug("Access APIC ARBPRI register which is for P6\n");
97222cc8
ED		 1203 break;
1204
1205 case APIC_TMCCT: /* Timer CCR */
a3e06bbe
LJ		 1206 if (apic_lvtt_tscdeadline(apic))
1207 return 0;
1208
97222cc8
ED		 1209 val = apic_get_tmcct(apic);
1210 break;
4a4541a4
AK		 1211 case APIC_PROCPRI:
1212 apic_update_ppr(apic);
dfb95954 1213 val = kvm_lapic_get_reg(apic, offset);
4a4541a4 1214 break;
b209749f
AK		 1215 case APIC_TASKPRI:
1216 report_tpr_access(apic, false);
1217 /* fall thru */
97222cc8 1218 default:
dfb95954 1219 val = kvm_lapic_get_reg(apic, offset);
97222cc8
ED		 1220 break;
1221 }
1222
1223 return val;
1224}
1225
d76685c4
GH		 1226static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev)
1227{
1228 return container_of(dev, struct kvm_lapic, dev);
1229}
1230
1e6e2755 1231int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
0105d1a5 1232 void *data)
97222cc8 1233{
97222cc8
ED		 1234 unsigned char alignment = offset & 0xf;
1235 u32 result;
d5b0b5b1 1236 /* this bitmask has a bit cleared for each reserved register */
0105d1a5 1237 static const u64 rmask = 0x43ff01ffffffe70cULL;
97222cc8
ED		 1238
1239 if ((alignment + len) > 4) {
4088bb3c
GN		 1240 apic_debug("KVM_APIC_READ: alignment error %x %d\n",
1241 offset, len);
0105d1a5 1242 return 1;
97222cc8 1243 }
0105d1a5
GN		 1244
1245 if (offset > 0x3f0 || !(rmask & (1ULL << (offset >> 4)))) {
4088bb3c
GN		 1246 apic_debug("KVM_APIC_READ: read reserved register %x\n",
1247 offset);
0105d1a5
GN		 1248 return 1;
1249 }
1250
97222cc8
ED		 1251 result = __apic_read(apic, offset & ~0xf);
1252
229456fc
MT		 1253 trace_kvm_apic_read(offset, result);
1254
97222cc8
ED		 1255 switch (len) {
1256 case 1:
1257 case 2:
1258 case 4:
1259 memcpy(data, (char *)&result + alignment, len);
1260 break;
1261 default:
1262 printk(KERN_ERR "Local APIC read with len = %x, "
1263 "should be 1,2, or 4 instead\n", len);
1264 break;
1265 }
bda9020e 1266 return 0;
97222cc8 1267}
1e6e2755 1268EXPORT_SYMBOL_GPL(kvm_lapic_reg_read);
97222cc8 1269
0105d1a5
GN		 1270static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
1271{
c48f1496 1272 return kvm_apic_hw_enabled(apic) &&
0105d1a5
GN		 1273 addr >= apic->base_address &&
1274 addr < apic->base_address + LAPIC_MMIO_LENGTH;
1275}
1276
e32edf4f 1277static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
0105d1a5
GN		 1278 gpa_t address, int len, void *data)
1279{
1280 struct kvm_lapic *apic = to_lapic(this);
1281 u32 offset = address - apic->base_address;
1282
1283 if (!apic_mmio_in_range(apic, address))
1284 return -EOPNOTSUPP;
1285
1e6e2755 1286 kvm_lapic_reg_read(apic, offset, len, data);
0105d1a5
GN		 1287
1288 return 0;
1289}
1290
97222cc8
ED		 1291static void update_divide_count(struct kvm_lapic *apic)
1292{
1293 u32 tmp1, tmp2, tdcr;
1294
dfb95954 1295 tdcr = kvm_lapic_get_reg(apic, APIC_TDCR);
97222cc8
ED		 1296 tmp1 = tdcr & 0xf;
1297 tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1;
d3c7b77d 1298 apic->divide_count = 0x1 << (tmp2 & 0x7);
97222cc8
ED		 1299
1300 apic_debug("timer divide count is 0x%x\n",
9b5843dd 1301 apic->divide_count);
97222cc8
ED		 1302}
1303
ccbfa1d3
WL		 1304static void limit_periodic_timer_frequency(struct kvm_lapic *apic)
1305{
1306 /*
1307 * Do not allow the guest to program periodic timers with small
1308 * interval, since the hrtimers are not throttled by the host
1309 * scheduler.
1310 */
dedf9c5e 1311 if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
ccbfa1d3
WL		 1312 s64 min_period = min_timer_period_us * 1000LL;
1313
1314 if (apic->lapic_timer.period < min_period) {
1315 pr_info_ratelimited(
1316 "kvm: vcpu %i: requested %lld ns "
1317 "lapic timer period limited to %lld ns\n",
1318 apic->vcpu->vcpu_id,
1319 apic->lapic_timer.period, min_period);
1320 apic->lapic_timer.period = min_period;
1321 }
1322 }
1323}
1324
b6ac0695
RK		 1325static void apic_update_lvtt(struct kvm_lapic *apic)
1326{
dfb95954 1327 u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) &
b6ac0695
RK		 1328 apic->lapic_timer.timer_mode_mask;
1329
1330 if (apic->lapic_timer.timer_mode != timer_mode) {
c69518c8 1331 if (apic_lvtt_tscdeadline(apic) != (timer_mode ==
dedf9c5e 1332 APIC_LVT_TIMER_TSCDEADLINE)) {
c69518c8 1333 kvm_lapic_set_reg(apic, APIC_TMICT, 0);
dedf9c5e
WL		 1334 hrtimer_cancel(&apic->lapic_timer.timer);
1335 }
b6ac0695 1336 apic->lapic_timer.timer_mode = timer_mode;
dedf9c5e 1337 limit_periodic_timer_frequency(apic);
b6ac0695
RK		 1338 }
1339}
1340
5d87db71
RK		 1341static void apic_timer_expired(struct kvm_lapic *apic)
1342{
1343 struct kvm_vcpu *vcpu = apic->vcpu;
8577370f 1344 struct swait_queue_head *q = &vcpu->wq;
d0659d94 1345 struct kvm_timer *ktimer = &apic->lapic_timer;
5d87db71 1346
5d87db71
RK		 1347 if (atomic_read(&apic->lapic_timer.pending))
1348 return;
1349
1350 atomic_inc(&apic->lapic_timer.pending);
bab5bb39 1351 kvm_set_pending_timer(vcpu);
5d87db71 1352
cc1b4680
DB		 1353 /*
1354 * For x86, the atomic_inc() is serialized, thus
1355 * using swait_active() is safe.
1356 */
8577370f
MT		 1357 if (swait_active(q))
1358 swake_up(q);
d0659d94
MT		 1359
1360 if (apic_lvtt_tscdeadline(apic))
1361 ktimer->expired_tscdeadline = ktimer->tscdeadline;
1362}
1363
1364/*
1365 * On APICv, this test will cause a busy wait
1366 * during a higher-priority task.
1367 */
1368
1369static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu)
1370{
1371 struct kvm_lapic *apic = vcpu->arch.apic;
dfb95954 1372 u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT);
d0659d94
MT		 1373
1374 if (kvm_apic_hw_enabled(apic)) {
1375 int vec = reg & APIC_VECTOR_MASK;
f9339860 1376 void *bitmap = apic->regs + APIC_ISR;
d0659d94 1377
d62caabb 1378 if (vcpu->arch.apicv_active)
f9339860
MT		 1379 bitmap = apic->regs + APIC_IRR;
1380
1381 if (apic_test_vector(vec, bitmap))
1382 return true;
d0659d94
MT		 1383 }
1384 return false;
1385}
1386
1387void wait_lapic_expire(struct kvm_vcpu *vcpu)
1388{
1389 struct kvm_lapic *apic = vcpu->arch.apic;
1390 u64 guest_tsc, tsc_deadline;
1391
bce87cce 1392 if (!lapic_in_kernel(vcpu))
d0659d94
MT		 1393 return;
1394
1395 if (apic->lapic_timer.expired_tscdeadline == 0)
1396 return;
1397
1398 if (!lapic_timer_int_injected(vcpu))
1399 return;
1400
1401 tsc_deadline = apic->lapic_timer.expired_tscdeadline;
1402 apic->lapic_timer.expired_tscdeadline = 0;
4ba76538 1403 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
6c19b753 1404 trace_kvm_wait_lapic_expire(vcpu->vcpu_id, guest_tsc - tsc_deadline);
d0659d94
MT		 1405
1406 /* __delay is delay_tsc whenever the hardware has TSC, thus always. */
1407 if (guest_tsc < tsc_deadline)
b606f189
MT		 1408 __delay(min(tsc_deadline - guest_tsc,
1409 nsec_to_cycles(vcpu, lapic_timer_advance_ns)));
5d87db71
RK		 1410}
1411
53f9eedf
YJ		 1412static void start_sw_tscdeadline(struct kvm_lapic *apic)
1413{
1414 u64 guest_tsc, tscdeadline = apic->lapic_timer.tscdeadline;
1415 u64 ns = 0;
1416 ktime_t expire;
1417 struct kvm_vcpu *vcpu = apic->vcpu;
1418 unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz;
1419 unsigned long flags;
1420 ktime_t now;
1421
1422 if (unlikely(!tscdeadline || !this_tsc_khz))
1423 return;
1424
1425 local_irq_save(flags);
1426
5587859f 1427 now = ktime_get();
53f9eedf
YJ		 1428 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1429 if (likely(tscdeadline > guest_tsc)) {
1430 ns = (tscdeadline - guest_tsc) * 1000000ULL;
1431 do_div(ns, this_tsc_khz);
1432 expire = ktime_add_ns(now, ns);
1433 expire = ktime_sub_ns(expire, lapic_timer_advance_ns);
1434 hrtimer_start(&apic->lapic_timer.timer,
1435 expire, HRTIMER_MODE_ABS_PINNED);
1436 } else
1437 apic_timer_expired(apic);
1438
1439 local_irq_restore(flags);
1440}
1441
7d7f7da2 1442static void start_sw_period(struct kvm_lapic *apic)
8003c9ae
WL		 1443{
1444 if (!apic->lapic_timer.period)
1445 return;
1446
1447 if (apic_lvtt_oneshot(apic) &&
5587859f 1448 ktime_after(ktime_get(),
8003c9ae
WL		 1449 apic->lapic_timer.target_expiration)) {
1450 apic_timer_expired(apic);
1451 return;
1452 }
1453
1454 hrtimer_start(&apic->lapic_timer.timer,
1455 apic->lapic_timer.target_expiration,
1456 HRTIMER_MODE_ABS_PINNED);
1457}
1458
c301b909
WL		 1459static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor)
1460{
1461 ktime_t now, remaining;
1462 u64 ns_remaining_old, ns_remaining_new;
1463
1464 apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT)
1465 * APIC_BUS_CYCLE_NS * apic->divide_count;
1466 limit_periodic_timer_frequency(apic);
1467
1468 now = ktime_get();
1469 remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
1470 if (ktime_to_ns(remaining) < 0)
1471 remaining = 0;
1472
1473 ns_remaining_old = ktime_to_ns(remaining);
1474 ns_remaining_new = mul_u64_u32_div(ns_remaining_old,
1475 apic->divide_count, old_divisor);
1476
1477 apic->lapic_timer.tscdeadline +=
1478 nsec_to_cycles(apic->vcpu, ns_remaining_new) -
1479 nsec_to_cycles(apic->vcpu, ns_remaining_old);
1480 apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new);
1481}
1482
8003c9ae 1483static bool set_target_expiration(struct kvm_lapic *apic)
7d7f7da2
WL		 1484{
1485 ktime_t now;
8003c9ae 1486 u64 tscl = rdtsc();
7d7f7da2 1487
5587859f 1488 now = ktime_get();
7d7f7da2 1489 apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT)
8003c9ae 1490 * APIC_BUS_CYCLE_NS * apic->divide_count;
7d7f7da2
WL		 1491
1492 if (!apic->lapic_timer.period)
8003c9ae
WL		 1493 return false;
1494
ccbfa1d3 1495 limit_periodic_timer_frequency(apic);
7d7f7da2 1496
7d7f7da2
WL		 1497 apic_debug("%s: bus cycle is %" PRId64 "ns, now 0x%016"
1498 PRIx64 ", "
1499 "timer initial count 0x%x, period %lldns, "
1500 "expire @ 0x%016" PRIx64 ".\n", __func__,
1501 APIC_BUS_CYCLE_NS, ktime_to_ns(now),
1502 kvm_lapic_get_reg(apic, APIC_TMICT),
1503 apic->lapic_timer.period,
1504 ktime_to_ns(ktime_add_ns(now,
1505 apic->lapic_timer.period)));
8003c9ae
WL		 1506
1507 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
1508 nsec_to_cycles(apic->vcpu, apic->lapic_timer.period);
1509 apic->lapic_timer.target_expiration = ktime_add_ns(now, apic->lapic_timer.period);
1510
1511 return true;
1512}
1513
1514static void advance_periodic_target_expiration(struct kvm_lapic *apic)
1515{
1516 apic->lapic_timer.tscdeadline +=
1517 nsec_to_cycles(apic->vcpu, apic->lapic_timer.period);
1518 apic->lapic_timer.target_expiration =
1519 ktime_add_ns(apic->lapic_timer.target_expiration,
1520 apic->lapic_timer.period);
7d7f7da2
WL		 1521}
1522
ce7a058a
YJ		 1523bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu)
1524{
91005300
WL		 1525 if (!lapic_in_kernel(vcpu))
1526 return false;
1527
ce7a058a
YJ		 1528 return vcpu->arch.apic->lapic_timer.hv_timer_in_use;
1529}
1530EXPORT_SYMBOL_GPL(kvm_lapic_hv_timer_in_use);
1531
7e810a38 1532static void cancel_hv_timer(struct kvm_lapic *apic)
bd97ad0e 1533{
1d518c68 1534 WARN_ON(preemptible());
a749e247 1535 WARN_ON(!apic->lapic_timer.hv_timer_in_use);
bd97ad0e
WL		 1536 kvm_x86_ops->cancel_hv_timer(apic->vcpu);
1537 apic->lapic_timer.hv_timer_in_use = false;
1538}
1539
a749e247 1540static bool start_hv_timer(struct kvm_lapic *apic)
196f20ca 1541{
35ee9e48
PB		 1542 struct kvm_timer *ktimer = &apic->lapic_timer;
1543 int r;
196f20ca 1544
1d518c68 1545 WARN_ON(preemptible());
a749e247
PB		 1546 if (!kvm_x86_ops->set_hv_timer)
1547 return false;
1548
35ee9e48
PB		 1549 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending))
1550 return false;
1551
1552 r = kvm_x86_ops->set_hv_timer(apic->vcpu, ktimer->tscdeadline);
1553 if (r < 0)
1554 return false;
1555
1556 ktimer->hv_timer_in_use = true;
1557 hrtimer_cancel(&ktimer->timer);
196f20ca 1558
35ee9e48
PB		 1559 /*
1560 * Also recheck ktimer->pending, in case the sw timer triggered in
1561 * the window. For periodic timer, leave the hv timer running for
1562 * simplicity, and the deadline will be recomputed on the next vmexit.
1563 */
c8533544
WL		 1564 if (!apic_lvtt_period(apic) && (r || atomic_read(&ktimer->pending))) {
1565 if (r)
1566 apic_timer_expired(apic);
35ee9e48 1567 return false;
c8533544 1568 }
a749e247
PB		 1569
1570 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, true);
35ee9e48
PB		 1571 return true;
1572}
1573
a749e247 1574static void start_sw_timer(struct kvm_lapic *apic)
35ee9e48 1575{
a749e247 1576 struct kvm_timer *ktimer = &apic->lapic_timer;
1d518c68
WL		 1577
1578 WARN_ON(preemptible());
a749e247
PB		 1579 if (apic->lapic_timer.hv_timer_in_use)
1580 cancel_hv_timer(apic);
1581 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending))
1582 return;
1583
1584 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
1585 start_sw_period(apic);
1586 else if (apic_lvtt_tscdeadline(apic))
1587 start_sw_tscdeadline(apic);
1588 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false);
1589}
35ee9e48 1590
a749e247
PB		 1591static void restart_apic_timer(struct kvm_lapic *apic)
1592{
1d518c68 1593 preempt_disable();
a749e247
PB		 1594 if (!start_hv_timer(apic))
1595 start_sw_timer(apic);
1d518c68 1596 preempt_enable();
196f20ca
WL		 1597}
1598
8003c9ae
WL		 1599void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu)
1600{
1601 struct kvm_lapic *apic = vcpu->arch.apic;
1602
1d518c68
WL		 1603 preempt_disable();
1604 /* If the preempt notifier has already run, it also called apic_timer_expired */
1605 if (!apic->lapic_timer.hv_timer_in_use)
1606 goto out;
8003c9ae
WL		 1607 WARN_ON(swait_active(&vcpu->wq));
1608 cancel_hv_timer(apic);
1609 apic_timer_expired(apic);
1610
1611 if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
1612 advance_periodic_target_expiration(apic);
a749e247 1613 restart_apic_timer(apic);
8003c9ae 1614 }
1d518c68
WL		 1615out:
1616 preempt_enable();
8003c9ae
WL		 1617}
1618EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer);
1619
ce7a058a
YJ		 1620void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu)
1621{
a749e247 1622 restart_apic_timer(vcpu->arch.apic);
ce7a058a
YJ		 1623}
1624EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_hv_timer);
1625
1626void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu)
1627{
1628 struct kvm_lapic *apic = vcpu->arch.apic;
1629
1d518c68 1630 preempt_disable();
ce7a058a 1631 /* Possibly the TSC deadline timer is not enabled yet */
a749e247
PB		 1632 if (apic->lapic_timer.hv_timer_in_use)
1633 start_sw_timer(apic);
1d518c68 1634 preempt_enable();
a749e247
PB		 1635}
1636EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_sw_timer);
ce7a058a 1637
a749e247
PB		 1638void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu)
1639{
1640 struct kvm_lapic *apic = vcpu->arch.apic;
ce7a058a 1641
a749e247
PB		 1642 WARN_ON(!apic->lapic_timer.hv_timer_in_use);
1643 restart_apic_timer(apic);
ce7a058a 1644}
ce7a058a 1645
97222cc8
ED		 1646static void start_apic_timer(struct kvm_lapic *apic)
1647{
d3c7b77d 1648 atomic_set(&apic->lapic_timer.pending, 0);
0b975a3c 1649
a749e247
PB		 1650 if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
1651 && !set_target_expiration(apic))
1652 return;
1653
1654 restart_apic_timer(apic);
97222cc8
ED		 1655}
1656
cc6e462c
JK		 1657static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
1658{
59fd1323 1659 bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val);
cc6e462c 1660
59fd1323
RK		 1661 if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) {
1662 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode;
1663 if (lvt0_in_nmi_mode) {
cc6e462c
JK		 1664 apic_debug("Receive NMI setting on APIC_LVT0 "
1665 "for cpu %d\n", apic->vcpu->vcpu_id);
42720138 1666 atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
59fd1323
RK		 1667 } else
1668 atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
1669 }
cc6e462c
JK		 1670}
1671
1e6e2755 1672int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
97222cc8 1673{
0105d1a5 1674 int ret = 0;
97222cc8 1675
0105d1a5 1676 trace_kvm_apic_write(reg, val);
97222cc8 1677
0105d1a5 1678 switch (reg) {
97222cc8 1679 case APIC_ID: /* Local APIC ID */
0105d1a5 1680 if (!apic_x2apic_mode(apic))
a92e2543 1681 kvm_apic_set_xapic_id(apic, val >> 24);
0105d1a5
GN		 1682 else
1683 ret = 1;
97222cc8
ED		 1684 break;
1685
1686 case APIC_TASKPRI:
b209749f 1687 report_tpr_access(apic, true);
97222cc8
ED		 1688 apic_set_tpr(apic, val & 0xff);
1689 break;
1690
1691 case APIC_EOI:
1692 apic_set_eoi(apic);
1693 break;
1694
1695 case APIC_LDR:
0105d1a5 1696 if (!apic_x2apic_mode(apic))
1e08ec4a 1697 kvm_apic_set_ldr(apic, val & APIC_LDR_MASK);
0105d1a5
GN		 1698 else
1699 ret = 1;
97222cc8
ED		 1700 break;
1701
1702 case APIC_DFR:
1e08ec4a 1703 if (!apic_x2apic_mode(apic)) {
1e6e2755 1704 kvm_lapic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF);
1e08ec4a
GN		 1705 recalculate_apic_map(apic->vcpu->kvm);
1706 } else
0105d1a5 1707 ret = 1;
97222cc8
ED		 1708 break;
1709
fc61b800
GN		 1710 case APIC_SPIV: {
1711 u32 mask = 0x3ff;
dfb95954 1712 if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI)
fc61b800 1713 mask |= APIC_SPIV_DIRECTED_EOI;
f8c1ea10 1714 apic_set_spiv(apic, val & mask);
97222cc8
ED		 1715 if (!(val & APIC_SPIV_APIC_ENABLED)) {
1716 int i;
1717 u32 lvt_val;
1718
1e6e2755 1719 for (i = 0; i < KVM_APIC_LVT_NUM; i++) {
dfb95954 1720 lvt_val = kvm_lapic_get_reg(apic,
97222cc8 1721 APIC_LVTT + 0x10 * i);
1e6e2755 1722 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i,
97222cc8
ED		 1723 lvt_val | APIC_LVT_MASKED);
1724 }
b6ac0695 1725 apic_update_lvtt(apic);
d3c7b77d 1726 atomic_set(&apic->lapic_timer.pending, 0);
97222cc8
ED		 1727
1728 }
1729 break;
fc61b800 1730 }
97222cc8
ED		 1731 case APIC_ICR:
1732 /* No delay here, so we always clear the pending bit */
1e6e2755 1733 kvm_lapic_set_reg(apic, APIC_ICR, val & ~(1 << 12));
97222cc8
ED		 1734 apic_send_ipi(apic);
1735 break;
1736
1737 case APIC_ICR2:
0105d1a5
GN		 1738 if (!apic_x2apic_mode(apic))
1739 val &= 0xff000000;
1e6e2755 1740 kvm_lapic_set_reg(apic, APIC_ICR2, val);
97222cc8
ED		 1741 break;
1742
23930f95 1743 case APIC_LVT0:
cc6e462c 1744 apic_manage_nmi_watchdog(apic, val);
97222cc8
ED		 1745 case APIC_LVTTHMR:
1746 case APIC_LVTPC:
97222cc8
ED		 1747 case APIC_LVT1:
1748 case APIC_LVTERR:
1749 /* TODO: Check vector */
c48f1496 1750 if (!kvm_apic_sw_enabled(apic))
97222cc8
ED		 1751 val |= APIC_LVT_MASKED;
1752
0105d1a5 1753 val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4];
1e6e2755 1754 kvm_lapic_set_reg(apic, reg, val);
97222cc8
ED		 1755
1756 break;
1757
b6ac0695 1758 case APIC_LVTT:
c48f1496 1759 if (!kvm_apic_sw_enabled(apic))
a3e06bbe
LJ		 1760 val |= APIC_LVT_MASKED;
1761 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask);
1e6e2755 1762 kvm_lapic_set_reg(apic, APIC_LVTT, val);
b6ac0695 1763 apic_update_lvtt(apic);
a3e06bbe
LJ		 1764 break;
1765
97222cc8 1766 case APIC_TMICT:
a3e06bbe
LJ		 1767 if (apic_lvtt_tscdeadline(apic))
1768 break;
1769
d3c7b77d 1770 hrtimer_cancel(&apic->lapic_timer.timer);
1e6e2755 1771 kvm_lapic_set_reg(apic, APIC_TMICT, val);
97222cc8 1772 start_apic_timer(apic);
0105d1a5 1773 break;
97222cc8 1774
c301b909
WL		 1775 case APIC_TDCR: {
1776 uint32_t old_divisor = apic->divide_count;
1777
97222cc8 1778 if (val & 4)
7712de87 1779 apic_debug("KVM_WRITE:TDCR %x\n", val);
1e6e2755 1780 kvm_lapic_set_reg(apic, APIC_TDCR, val);
97222cc8 1781 update_divide_count(apic);
c301b909
WL		 1782 if (apic->divide_count != old_divisor &&
1783 apic->lapic_timer.period) {
1784 hrtimer_cancel(&apic->lapic_timer.timer);
1785 update_target_expiration(apic, old_divisor);
1786 restart_apic_timer(apic);
1787 }
97222cc8 1788 break;
c301b909 1789 }
0105d1a5
GN		 1790 case APIC_ESR:
1791 if (apic_x2apic_mode(apic) && val != 0) {
7712de87 1792 apic_debug("KVM_WRITE:ESR not zero %x\n", val);
0105d1a5
GN		 1793 ret = 1;
1794 }
1795 break;
1796
1797 case APIC_SELF_IPI:
1798 if (apic_x2apic_mode(apic)) {
1e6e2755 1799 kvm_lapic_reg_write(apic, APIC_ICR, 0x40000 | (val & 0xff));
0105d1a5
GN		 1800 } else
1801 ret = 1;
1802 break;
97222cc8 1803 default:
0105d1a5 1804 ret = 1;
97222cc8
ED		 1805 break;
1806 }
0105d1a5
GN		 1807 if (ret)
1808 apic_debug("Local APIC Write to read-only register %x\n", reg);
1809 return ret;
1810}
1e6e2755 1811EXPORT_SYMBOL_GPL(kvm_lapic_reg_write);
0105d1a5 1812
e32edf4f 1813static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
0105d1a5
GN		 1814 gpa_t address, int len, const void *data)
1815{
1816 struct kvm_lapic *apic = to_lapic(this);
1817 unsigned int offset = address - apic->base_address;
1818 u32 val;
1819
1820 if (!apic_mmio_in_range(apic, address))
1821 return -EOPNOTSUPP;
1822
1823 /*
1824 * APIC register must be aligned on 128-bits boundary.
1825 * 32/64/128 bits registers must be accessed thru 32 bits.
1826 * Refer SDM 8.4.1
1827 */
1828 if (len != 4 || (offset & 0xf)) {
1829 /* Don't shout loud, $infamous_os would cause only noise. */
1830 apic_debug("apic write: bad size=%d %lx\n", len, (long)address);
756975bb 1831 return 0;
0105d1a5
GN		 1832 }
1833
1834 val = *(u32*)data;
1835
1836 /* too common printing */
1837 if (offset != APIC_EOI)
1838 apic_debug("%s: offset 0x%x with length 0x%x, and value is "
1839 "0x%x\n", __func__, offset, len, val);
1840
1e6e2755 1841 kvm_lapic_reg_write(apic, offset & 0xff0, val);
0105d1a5 1842
bda9020e 1843 return 0;
97222cc8
ED		 1844}
1845
58fbbf26
KT		 1846void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu)
1847{
1e6e2755 1848 kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0);
58fbbf26
KT		 1849}
1850EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi);
1851
83d4c286
YZ		 1852/* emulate APIC access in a trap manner */
1853void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset)
1854{
1855 u32 val = 0;
1856
1857 /* hw has done the conditional check and inst decode */
1858 offset &= 0xff0;
1859
1e6e2755 1860 kvm_lapic_reg_read(vcpu->arch.apic, offset, 4, &val);
83d4c286
YZ		 1861
1862 /* TODO: optimize to just emulate side effect w/o one more write */
1e6e2755 1863 kvm_lapic_reg_write(vcpu->arch.apic, offset, val);
83d4c286
YZ		 1864}
1865EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode);
1866
d589444e 1867void kvm_free_lapic(struct kvm_vcpu *vcpu)
97222cc8 1868{
f8c1ea10
GN		 1869 struct kvm_lapic *apic = vcpu->arch.apic;
1870
ad312c7c 1871 if (!vcpu->arch.apic)
97222cc8
ED		 1872 return;
1873
f8c1ea10 1874 hrtimer_cancel(&apic->lapic_timer.timer);
97222cc8 1875
c5cc421b
GN		 1876 if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE))
1877 static_key_slow_dec_deferred(&apic_hw_disabled);
1878
e462755c 1879 if (!apic->sw_enabled)
f8c1ea10 1880 static_key_slow_dec_deferred(&apic_sw_disabled);
97222cc8 1881
f8c1ea10
GN		 1882 if (apic->regs)
1883 free_page((unsigned long)apic->regs);
1884
1885 kfree(apic);
97222cc8
ED		 1886}
1887
1888/*
1889 *----------------------------------------------------------------------
1890 * LAPIC interface
1891 *----------------------------------------------------------------------
1892 */
a3e06bbe
LJ		 1893u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu)
1894{
1895 struct kvm_lapic *apic = vcpu->arch.apic;
a3e06bbe 1896
a10388e1
WL		 1897 if (!lapic_in_kernel(vcpu) ||
1898 !apic_lvtt_tscdeadline(apic))
a3e06bbe
LJ		 1899 return 0;
1900
1901 return apic->lapic_timer.tscdeadline;
1902}
1903
1904void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data)
1905{
1906 struct kvm_lapic *apic = vcpu->arch.apic;
a3e06bbe 1907
bce87cce 1908 if (!lapic_in_kernel(vcpu) || apic_lvtt_oneshot(apic) ||
54e9818f 1909 apic_lvtt_period(apic))
a3e06bbe
LJ		 1910 return;
1911
1912 hrtimer_cancel(&apic->lapic_timer.timer);
1913 apic->lapic_timer.tscdeadline = data;
1914 start_apic_timer(apic);
1915}
1916
97222cc8
ED		 1917void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8)
1918{
ad312c7c 1919 struct kvm_lapic *apic = vcpu->arch.apic;
97222cc8 1920
b93463aa 1921 apic_set_tpr(apic, ((cr8 & 0x0f) << 4)
dfb95954 1922 | (kvm_lapic_get_reg(apic, APIC_TASKPRI) & 4));
97222cc8
ED		 1923}
1924
1925u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
1926{
97222cc8
ED		 1927 u64 tpr;
1928
dfb95954 1929 tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI);
97222cc8
ED		 1930
1931 return (tpr & 0xf0) >> 4;
1932}
1933
1934void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
1935{
8d14695f 1936 u64 old_value = vcpu->arch.apic_base;
ad312c7c 1937 struct kvm_lapic *apic = vcpu->arch.apic;
97222cc8 1938
c7dd15b3 1939 if (!apic)
97222cc8 1940 value |= MSR_IA32_APICBASE_BSP;
c5af89b6 1941
e66d2ae7
JK		 1942 vcpu->arch.apic_base = value;
1943
c7dd15b3
JM		 1944 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE)
1945 kvm_update_cpuid(vcpu);
1946
1947 if (!apic)
1948 return;
1949
c5cc421b 1950 /* update jump label if enable bit changes */
0dce7cd6 1951 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) {
49bd29ba
RK		 1952 if (value & MSR_IA32_APICBASE_ENABLE) {
1953 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
c5cc421b 1954 static_key_slow_dec_deferred(&apic_hw_disabled);
187ca84b 1955 } else {
c5cc421b 1956 static_key_slow_inc(&apic_hw_disabled.key);
187ca84b
WL		 1957 recalculate_apic_map(vcpu->kvm);
1958 }
c5cc421b
GN		 1959 }
1960
8d14695f
YZ		 1961 if ((old_value ^ value) & X2APIC_ENABLE) {
1962 if (value & X2APIC_ENABLE) {
257b9a5f 1963 kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
8d14695f
YZ		 1964 kvm_x86_ops->set_virtual_x2apic_mode(vcpu, true);
1965 } else
1966 kvm_x86_ops->set_virtual_x2apic_mode(vcpu, false);
0105d1a5 1967 }
8d14695f 1968
ad312c7c 1969 apic->base_address = apic->vcpu->arch.apic_base &
97222cc8
ED		 1970 MSR_IA32_APICBASE_BASE;
1971
db324fe6
NA		 1972 if ((value & MSR_IA32_APICBASE_ENABLE) &&
1973 apic->base_address != APIC_DEFAULT_PHYS_BASE)
1974 pr_warn_once("APIC base relocation is unsupported by KVM");
1975
97222cc8
ED		 1976 /* with FSB delivery interrupt, we can restart APIC functionality */
1977 apic_debug("apic base msr is 0x%016" PRIx64 ", and base address is "
ad312c7c 1978 "0x%lx.\n", apic->vcpu->arch.apic_base, apic->base_address);
97222cc8
ED		 1979
1980}
1981
d28bc9dd 1982void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event)
97222cc8
ED		 1983{
1984 struct kvm_lapic *apic;
1985 int i;
1986
b8688d51 1987 apic_debug("%s\n", __func__);
97222cc8
ED		 1988
1989 ASSERT(vcpu);
ad312c7c 1990 apic = vcpu->arch.apic;
97222cc8
ED		 1991 ASSERT(apic != NULL);
1992
1993 /* Stop the timer in case it's a reset to an active apic */
d3c7b77d 1994 hrtimer_cancel(&apic->lapic_timer.timer);
97222cc8 1995
4d8e772b
RK		 1996 if (!init_event) {
1997 kvm_lapic_set_base(vcpu, APIC_DEFAULT_PHYS_BASE |
1998 MSR_IA32_APICBASE_ENABLE);
a92e2543 1999 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
4d8e772b 2000 }
fc61b800 2001 kvm_apic_set_version(apic->vcpu);
97222cc8 2002
1e6e2755
SS		 2003 for (i = 0; i < KVM_APIC_LVT_NUM; i++)
2004 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED);
b6ac0695 2005 apic_update_lvtt(apic);
52b54190
JS		 2006 if (kvm_vcpu_is_reset_bsp(vcpu) &&
2007 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED))
1e6e2755 2008 kvm_lapic_set_reg(apic, APIC_LVT0,
90de4a18 2009 SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT));
dfb95954 2010 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
97222cc8 2011
1e6e2755 2012 kvm_lapic_set_reg(apic, APIC_DFR, 0xffffffffU);
f8c1ea10 2013 apic_set_spiv(apic, 0xff);
1e6e2755 2014 kvm_lapic_set_reg(apic, APIC_TASKPRI, 0);
c028dd6b
RK		 2015 if (!apic_x2apic_mode(apic))
2016 kvm_apic_set_ldr(apic, 0);
1e6e2755
SS		 2017 kvm_lapic_set_reg(apic, APIC_ESR, 0);
2018 kvm_lapic_set_reg(apic, APIC_ICR, 0);
2019 kvm_lapic_set_reg(apic, APIC_ICR2, 0);
2020 kvm_lapic_set_reg(apic, APIC_TDCR, 0);
2021 kvm_lapic_set_reg(apic, APIC_TMICT, 0);
97222cc8 2022 for (i = 0; i < 8; i++) {
1e6e2755
SS		 2023 kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0);
2024 kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0);
2025 kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
97222cc8 2026 }
d62caabb
AS		 2027 apic->irr_pending = vcpu->arch.apicv_active;
2028 apic->isr_count = vcpu->arch.apicv_active ? 1 : 0;
8680b94b 2029 apic->highest_isr_cache = -1;
b33ac88b 2030 update_divide_count(apic);
d3c7b77d 2031 atomic_set(&apic->lapic_timer.pending, 0);
c5af89b6 2032 if (kvm_vcpu_is_bsp(vcpu))
5dbc8f3f
GN		 2033 kvm_lapic_set_base(vcpu,
2034 vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP);
ae7a2a3f 2035 vcpu->arch.pv_eoi.msr_val = 0;
97222cc8
ED		 2036 apic_update_ppr(apic);
2037
e1035715 2038 vcpu->arch.apic_arb_prio = 0;
41383771 2039 vcpu->arch.apic_attention = 0;
e1035715 2040
6e500439 2041 apic_debug("%s: vcpu=%p, id=0x%x, base_msr="
b8688d51 2042 "0x%016" PRIx64 ", base_address=0x%0lx.\n", __func__,
6e500439 2043 vcpu, kvm_lapic_get_reg(apic, APIC_ID),
ad312c7c 2044 vcpu->arch.apic_base, apic->base_address);
97222cc8
ED		 2045}
2046
97222cc8
ED		 2047/*
2048 *----------------------------------------------------------------------
2049 * timer interface
2050 *----------------------------------------------------------------------
2051 */
1b9778da 2052
2a6eac96 2053static bool lapic_is_periodic(struct kvm_lapic *apic)
97222cc8 2054{
d3c7b77d 2055 return apic_lvtt_period(apic);
97222cc8
ED		 2056}
2057
3d80840d
MT		 2058int apic_has_pending_timer(struct kvm_vcpu *vcpu)
2059{
54e9818f 2060 struct kvm_lapic *apic = vcpu->arch.apic;
3d80840d 2061
1e3161b4 2062 if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT))
54e9818f 2063 return atomic_read(&apic->lapic_timer.pending);
3d80840d
MT		 2064
2065 return 0;
2066}
2067
89342082 2068int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type)
1b9778da 2069{
dfb95954 2070 u32 reg = kvm_lapic_get_reg(apic, lvt_type);
23930f95 2071 int vector, mode, trig_mode;
23930f95 2072
c48f1496 2073 if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) {
23930f95
JK		 2074 vector = reg & APIC_VECTOR_MASK;
2075 mode = reg & APIC_MODE_MASK;
2076 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER;
b4f2225c
YZ		 2077 return __apic_accept_irq(apic, mode, vector, 1, trig_mode,
2078 NULL);
23930f95
JK		 2079 }
2080 return 0;
2081}
1b9778da 2082
8fdb2351 2083void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu)
23930f95 2084{
8fdb2351
JK		 2085 struct kvm_lapic *apic = vcpu->arch.apic;
2086
2087 if (apic)
2088 kvm_apic_local_deliver(apic, APIC_LVT0);
1b9778da
ED		 2089}
2090
d76685c4
GH		 2091static const struct kvm_io_device_ops apic_mmio_ops = {
2092 .read = apic_mmio_read,
2093 .write = apic_mmio_write,
d76685c4
GH		 2094};
2095
e9d90d47
AK		 2096static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
2097{
2098 struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
2a6eac96 2099 struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer);
e9d90d47 2100
5d87db71 2101 apic_timer_expired(apic);
e9d90d47 2102
2a6eac96 2103 if (lapic_is_periodic(apic)) {
8003c9ae 2104 advance_periodic_target_expiration(apic);
e9d90d47
AK		 2105 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
2106 return HRTIMER_RESTART;
2107 } else
2108 return HRTIMER_NORESTART;
2109}
2110
97222cc8
ED		 2111int kvm_create_lapic(struct kvm_vcpu *vcpu)
2112{
2113 struct kvm_lapic *apic;
2114
2115 ASSERT(vcpu != NULL);
2116 apic_debug("apic_init %d\n", vcpu->vcpu_id);
2117
2118 apic = kzalloc(sizeof(*apic), GFP_KERNEL);
2119 if (!apic)
2120 goto nomem;
2121
ad312c7c 2122 vcpu->arch.apic = apic;
97222cc8 2123
afc20184
TY		 2124 apic->regs = (void *)get_zeroed_page(GFP_KERNEL);
2125 if (!apic->regs) {
97222cc8
ED		 2126 printk(KERN_ERR "malloc apic regs error for vcpu %x\n",
2127 vcpu->vcpu_id);
d589444e 2128 goto nomem_free_apic;
97222cc8 2129 }
97222cc8
ED		 2130 apic->vcpu = vcpu;
2131
d3c7b77d 2132 hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
61abdbe0 2133 HRTIMER_MODE_ABS_PINNED);
e9d90d47 2134 apic->lapic_timer.timer.function = apic_timer_fn;
d3c7b77d 2135
c5cc421b
GN		 2136 /*
2137 * APIC is created enabled. This will prevent kvm_lapic_set_base from
2138 * thinking that APIC satet has changed.
2139 */
2140 vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE;
f8c1ea10 2141 static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */
d28bc9dd 2142 kvm_lapic_reset(vcpu, false);
d76685c4 2143 kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
97222cc8
ED		 2144
2145 return 0;
d589444e
RR		 2146nomem_free_apic:
2147 kfree(apic);
97222cc8 2148nomem:
97222cc8
ED		 2149 return -ENOMEM;
2150}
97222cc8
ED		 2151
2152int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu)
2153{
ad312c7c 2154 struct kvm_lapic *apic = vcpu->arch.apic;
b3c045d3 2155 u32 ppr;
97222cc8 2156
f8543d6a 2157 if (!apic_enabled(apic))
97222cc8
ED		 2158 return -1;
2159
b3c045d3
PB		 2160 __apic_update_ppr(apic, &ppr);
2161 return apic_has_interrupt_for_ppr(apic, ppr);
97222cc8
ED		 2162}
2163
40487c68
QH		 2164int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu)
2165{
dfb95954 2166 u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0);
40487c68
QH		 2167 int r = 0;
2168
c48f1496 2169 if (!kvm_apic_hw_enabled(vcpu->arch.apic))
e7dca5c0
CL		 2170 r = 1;
2171 if ((lvt0 & APIC_LVT_MASKED) == 0 &&
2172 GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT)
2173 r = 1;
40487c68
QH		 2174 return r;
2175}
2176
1b9778da
ED		 2177void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu)
2178{
ad312c7c 2179 struct kvm_lapic *apic = vcpu->arch.apic;
1b9778da 2180
54e9818f 2181 if (atomic_read(&apic->lapic_timer.pending) > 0) {
f1ed0450 2182 kvm_apic_local_deliver(apic, APIC_LVTT);
fae0ba21
NA		 2183 if (apic_lvtt_tscdeadline(apic))
2184 apic->lapic_timer.tscdeadline = 0;
8003c9ae
WL		 2185 if (apic_lvtt_oneshot(apic)) {
2186 apic->lapic_timer.tscdeadline = 0;
8b0e1953 2187 apic->lapic_timer.target_expiration = 0;
8003c9ae 2188 }
f1ed0450 2189 atomic_set(&apic->lapic_timer.pending, 0);
1b9778da
ED		 2190 }
2191}
2192
97222cc8
ED		 2193int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu)
2194{
2195 int vector = kvm_apic_has_interrupt(vcpu);
ad312c7c 2196 struct kvm_lapic *apic = vcpu->arch.apic;
4d82d12b 2197 u32 ppr;
97222cc8
ED		 2198
2199 if (vector == -1)
2200 return -1;
2201
56cc2406
WL		 2202 /*
2203 * We get here even with APIC virtualization enabled, if doing
2204 * nested virtualization and L1 runs with the "acknowledge interrupt
2205 * on exit" mode. Then we cannot inject the interrupt via RVI,
2206 * because the process would deliver it through the IDT.
2207 */
2208
97222cc8 2209 apic_clear_irr(vector, apic);
5c919412 2210 if (test_bit(vector, vcpu_to_synic(vcpu)->auto_eoi_bitmap)) {
4d82d12b
PB		 2211 /*
2212 * For auto-EOI interrupts, there might be another pending
2213 * interrupt above PPR, so check whether to raise another
2214 * KVM_REQ_EVENT.
2215 */
5c919412 2216 apic_update_ppr(apic);
4d82d12b
PB		 2217 } else {
2218 /*
2219 * For normal interrupts, PPR has been raised and there cannot
2220 * be a higher-priority pending interrupt---except if there was
2221 * a concurrent interrupt injection, but that would have
2222 * triggered KVM_REQ_EVENT already.
2223 */
2224 apic_set_isr(vector, apic);
2225 __apic_update_ppr(apic, &ppr);
5c919412
AS		 2226 }
2227
97222cc8
ED		 2228 return vector;
2229}
96ad2cc6 2230
a92e2543
RK		 2231static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu,
2232 struct kvm_lapic_state *s, bool set)
2233{
2234 if (apic_x2apic_mode(vcpu->arch.apic)) {
2235 u32 *id = (u32 *)(s->regs + APIC_ID);
2236
37131313
RK		 2237 if (vcpu->kvm->arch.x2apic_format) {
2238 if (*id != vcpu->vcpu_id)
2239 return -EINVAL;
2240 } else {
2241 if (set)
2242 *id >>= 24;
2243 else
2244 *id <<= 24;
2245 }
a92e2543
RK		 2246 }
2247
2248 return 0;
2249}
2250
2251int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
2252{
2253 memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s));
2254 return kvm_apic_state_fixup(vcpu, s, false);
2255}
2256
2257int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
96ad2cc6 2258{
ad312c7c 2259 struct kvm_lapic *apic = vcpu->arch.apic;
a92e2543
RK		 2260 int r;
2261
96ad2cc6 2262
5dbc8f3f 2263 kvm_lapic_set_base(vcpu, vcpu->arch.apic_base);
64eb0620
GN		 2264 /* set SPIV separately to get count of SW disabled APICs right */
2265 apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV)));
a92e2543
RK		 2266
2267 r = kvm_apic_state_fixup(vcpu, s, true);
2268 if (r)
2269 return r;
64eb0620 2270 memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s);
a92e2543
RK		 2271
2272 recalculate_apic_map(vcpu->kvm);
fc61b800
GN		 2273 kvm_apic_set_version(vcpu);
2274
96ad2cc6 2275 apic_update_ppr(apic);
d3c7b77d 2276 hrtimer_cancel(&apic->lapic_timer.timer);
b6ac0695 2277 apic_update_lvtt(apic);
dfb95954 2278 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
96ad2cc6
ED		 2279 update_divide_count(apic);
2280 start_apic_timer(apic);
6e24a6ef 2281 apic->irr_pending = true;
d62caabb 2282 apic->isr_count = vcpu->arch.apicv_active ?
c7c9c56c 2283 1 : count_vectors(apic->regs + APIC_ISR);
8680b94b 2284 apic->highest_isr_cache = -1;
d62caabb 2285 if (vcpu->arch.apicv_active) {
967235d3 2286 kvm_x86_ops->apicv_post_state_restore(vcpu);
4114c27d
WW		 2287 kvm_x86_ops->hwapic_irr_update(vcpu,
2288 apic_find_highest_irr(apic));
67c9dddc 2289 kvm_x86_ops->hwapic_isr_update(vcpu,
b4eef9b3 2290 apic_find_highest_isr(apic));
d62caabb 2291 }
3842d135 2292 kvm_make_request(KVM_REQ_EVENT, vcpu);
49df6397
SR		 2293 if (ioapic_in_kernel(vcpu->kvm))
2294 kvm_rtc_eoi_tracking_restore_one(vcpu);
0669a510
RK		 2295
2296 vcpu->arch.apic_arb_prio = 0;
a92e2543
RK		 2297
2298 return 0;
96ad2cc6 2299}
a3d7f85f 2300
2f52d58c 2301void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
a3d7f85f 2302{
a3d7f85f
ED		 2303 struct hrtimer *timer;
2304
bce87cce 2305 if (!lapic_in_kernel(vcpu))
a3d7f85f
ED		 2306 return;
2307
54e9818f 2308 timer = &vcpu->arch.apic->lapic_timer.timer;
a3d7f85f 2309 if (hrtimer_cancel(timer))
61abdbe0 2310 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED);
a3d7f85f 2311}
b93463aa 2312
ae7a2a3f
MT		 2313/*
2314 * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt
2315 *
2316 * Detect whether guest triggered PV EOI since the
2317 * last entry. If yes, set EOI on guests's behalf.
2318 * Clear PV EOI in guest memory in any case.
2319 */
2320static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu,
2321 struct kvm_lapic *apic)
2322{
2323 bool pending;
2324 int vector;
2325 /*
2326 * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host
2327 * and KVM_PV_EOI_ENABLED in guest memory as follows:
2328 *
2329 * KVM_APIC_PV_EOI_PENDING is unset:
2330 * -> host disabled PV EOI.
2331 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set:
2332 * -> host enabled PV EOI, guest did not execute EOI yet.
2333 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset:
2334 * -> host enabled PV EOI, guest executed EOI.
2335 */
2336 BUG_ON(!pv_eoi_enabled(vcpu));
2337 pending = pv_eoi_get_pending(vcpu);
2338 /*
2339 * Clear pending bit in any case: it will be set again on vmentry.
2340 * While this might not be ideal from performance point of view,
2341 * this makes sure pv eoi is only enabled when we know it's safe.
2342 */
2343 pv_eoi_clr_pending(vcpu);
2344 if (pending)
2345 return;
2346 vector = apic_set_eoi(apic);
2347 trace_kvm_pv_eoi(apic, vector);
2348}
2349
b93463aa
AK		 2350void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu)
2351{
2352 u32 data;
b93463aa 2353
ae7a2a3f
MT		 2354 if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention))
2355 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic);
2356
41383771 2357 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
b93463aa
AK		 2358 return;
2359
4e335d9e
PB		 2360 if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
2361 sizeof(u32)))
603242a8 2362 return;
b93463aa
AK		 2363
2364 apic_set_tpr(vcpu->arch.apic, data & 0xff);
2365}
2366
ae7a2a3f
MT		 2367/*
2368 * apic_sync_pv_eoi_to_guest - called before vmentry
2369 *
2370 * Detect whether it's safe to enable PV EOI and
2371 * if yes do so.
2372 */
2373static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu,
2374 struct kvm_lapic *apic)
2375{
2376 if (!pv_eoi_enabled(vcpu) ||
2377 /* IRR set or many bits in ISR: could be nested. */
2378 apic->irr_pending ||
2379 /* Cache not set: could be safe but we don't bother. */
2380 apic->highest_isr_cache == -1 ||
2381 /* Need EOI to update ioapic. */
3bb345f3 2382 kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) {
ae7a2a3f
MT		 2383 /*
2384 * PV EOI was disabled by apic_sync_pv_eoi_from_guest
2385 * so we need not do anything here.
2386 */
2387 return;
2388 }
2389
2390 pv_eoi_set_pending(apic->vcpu);
2391}
2392
b93463aa
AK		 2393void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
2394{
2395 u32 data, tpr;
2396 int max_irr, max_isr;
ae7a2a3f 2397 struct kvm_lapic *apic = vcpu->arch.apic;
b93463aa 2398
ae7a2a3f
MT		 2399 apic_sync_pv_eoi_to_guest(vcpu, apic);
2400
41383771 2401 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
b93463aa
AK		 2402 return;
2403
dfb95954 2404 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff;
b93463aa
AK		 2405 max_irr = apic_find_highest_irr(apic);
2406 if (max_irr < 0)
2407 max_irr = 0;
2408 max_isr = apic_find_highest_isr(apic);
2409 if (max_isr < 0)
2410 max_isr = 0;
2411 data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24);
2412
4e335d9e
PB		 2413 kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
2414 sizeof(u32));
b93463aa
AK		 2415}
2416
fda4e2e8 2417int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr)
b93463aa 2418{
fda4e2e8 2419 if (vapic_addr) {
4e335d9e 2420 if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
fda4e2e8
AH		 2421 &vcpu->arch.apic->vapic_cache,
2422 vapic_addr, sizeof(u32)))
2423 return -EINVAL;
41383771 2424 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
fda4e2e8 2425 } else {
41383771 2426 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
fda4e2e8
AH		 2427 }
2428
2429 vcpu->arch.apic->vapic_addr = vapic_addr;
2430 return 0;
b93463aa 2431}
0105d1a5
GN		 2432
2433int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data)
2434{
2435 struct kvm_lapic *apic = vcpu->arch.apic;
2436 u32 reg = (msr - APIC_BASE_MSR) << 4;
2437
35754c98 2438 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
0105d1a5
GN		 2439 return 1;
2440
c69d3d9b
NA		 2441 if (reg == APIC_ICR2)
2442 return 1;
2443
0105d1a5 2444 /* if this is ICR write vector before command */
decdc283 2445 if (reg == APIC_ICR)
1e6e2755
SS		 2446 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
2447 return kvm_lapic_reg_write(apic, reg, (u32)data);
0105d1a5
GN		 2448}
2449
2450int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data)
2451{
2452 struct kvm_lapic *apic = vcpu->arch.apic;
2453 u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0;
2454
35754c98 2455 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
0105d1a5
GN		 2456 return 1;
2457
c69d3d9b
NA		 2458 if (reg == APIC_DFR || reg == APIC_ICR2) {
2459 apic_debug("KVM_APIC_READ: read x2apic reserved register %x\n",
2460 reg);
2461 return 1;
2462 }
2463
1e6e2755 2464 if (kvm_lapic_reg_read(apic, reg, 4, &low))
0105d1a5 2465 return 1;
decdc283 2466 if (reg == APIC_ICR)
1e6e2755 2467 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
0105d1a5
GN		 2468
2469 *data = (((u64)high) << 32) | low;
2470
2471 return 0;
2472}
10388a07
GN		 2473
2474int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data)
2475{
2476 struct kvm_lapic *apic = vcpu->arch.apic;
2477
bce87cce 2478 if (!lapic_in_kernel(vcpu))
10388a07
GN		 2479 return 1;
2480
2481 /* if this is ICR write vector before command */
2482 if (reg == APIC_ICR)
1e6e2755
SS		 2483 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
2484 return kvm_lapic_reg_write(apic, reg, (u32)data);
10388a07
GN		 2485}
2486
2487int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data)
2488{
2489 struct kvm_lapic *apic = vcpu->arch.apic;
2490 u32 low, high = 0;
2491
bce87cce 2492 if (!lapic_in_kernel(vcpu))
10388a07
GN		 2493 return 1;
2494
1e6e2755 2495 if (kvm_lapic_reg_read(apic, reg, 4, &low))
10388a07
GN		 2496 return 1;
2497 if (reg == APIC_ICR)
1e6e2755 2498 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
10388a07
GN		 2499
2500 *data = (((u64)high) << 32) | low;
2501
2502 return 0;
2503}
ae7a2a3f
MT		 2504
2505int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data)
2506{
2507 u64 addr = data & ~KVM_MSR_ENABLED;
2508 if (!IS_ALIGNED(addr, 4))
2509 return 1;
2510
2511 vcpu->arch.pv_eoi.msr_val = data;
2512 if (!pv_eoi_enabled(vcpu))
2513 return 0;
4e335d9e 2514 return kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.pv_eoi.data,
8f964525 2515 addr, sizeof(u8));
ae7a2a3f 2516}
c5cc421b 2517
66450a21
JK		 2518void kvm_apic_accept_events(struct kvm_vcpu *vcpu)
2519{
2520 struct kvm_lapic *apic = vcpu->arch.apic;
2b4a273b 2521 u8 sipi_vector;
299018f4 2522 unsigned long pe;
66450a21 2523
bce87cce 2524 if (!lapic_in_kernel(vcpu) || !apic->pending_events)
66450a21
JK		 2525 return;
2526
cd7764fe
PB		 2527 /*
2528 * INITs are latched while in SMM. Because an SMM CPU cannot
2529 * be in KVM_MP_STATE_INIT_RECEIVED state, just eat SIPIs
2530 * and delay processing of INIT until the next RSM.
2531 */
2532 if (is_smm(vcpu)) {
2533 WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED);
2534 if (test_bit(KVM_APIC_SIPI, &apic->pending_events))
2535 clear_bit(KVM_APIC_SIPI, &apic->pending_events);
2536 return;
2537 }
299018f4 2538
cd7764fe 2539 pe = xchg(&apic->pending_events, 0);
299018f4 2540 if (test_bit(KVM_APIC_INIT, &pe)) {
d28bc9dd
NA		 2541 kvm_lapic_reset(vcpu, true);
2542 kvm_vcpu_reset(vcpu, true);
66450a21
JK		 2543 if (kvm_vcpu_is_bsp(apic->vcpu))
2544 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
2545 else
2546 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
2547 }
299018f4 2548 if (test_bit(KVM_APIC_SIPI, &pe) &&
66450a21
JK		 2549 vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
2550 /* evaluate pending_events before reading the vector */
2551 smp_rmb();
2552 sipi_vector = apic->sipi_vector;
98eff52a 2553 apic_debug("vcpu %d received sipi with vector # %x\n",
66450a21
JK		 2554 vcpu->vcpu_id, sipi_vector);
2555 kvm_vcpu_deliver_sipi_vector(vcpu, sipi_vector);
2556 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
2557 }
2558}
2559
c5cc421b
GN		 2560void kvm_lapic_init(void)
2561{
2562 /* do not patch jump label more than once per second */
2563 jump_label_rate_limit(&apic_hw_disabled, HZ);
f8c1ea10 2564 jump_label_rate_limit(&apic_sw_disabled, HZ);
c5cc421b 2565}
cef84c30
DM		 2566
2567void kvm_lapic_exit(void)
2568{
2569 static_key_deferred_flush(&apic_hw_disabled);
2570 static_key_deferred_flush(&apic_sw_disabled);
2571}
ubuntu-bionic-kernel mirror