]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - arch/x86/kvm/lapic.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Revert "KVM: Support vCPU-based gfn->hva cache"
[mirror_ubuntu-bionic-kernel.git] / arch / x86 / kvm / lapic.c
CommitLineData
97222cc8
ED		 1
2/*
3 * Local APIC virtualization
4 *
5 * Copyright (C) 2006 Qumranet, Inc.
6 * Copyright (C) 2007 Novell
7 * Copyright (C) 2007 Intel
9611c187 8 * Copyright 2009 Red Hat, Inc. and/or its affiliates.
97222cc8
ED		 9 *
10 * Authors:
11 * Dor Laor <dor.laor@qumranet.com>
12 * Gregory Haskins <ghaskins@novell.com>
13 * Yaozu (Eddie) Dong <eddie.dong@intel.com>
14 *
15 * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation.
16 *
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
19 */
20
edf88417 21#include <linux/kvm_host.h>
97222cc8
ED		 22#include <linux/kvm.h>
23#include <linux/mm.h>
24#include <linux/highmem.h>
25#include <linux/smp.h>
26#include <linux/hrtimer.h>
27#include <linux/io.h>
1767e931 28#include <linux/export.h>
6f6d6a1a 29#include <linux/math64.h>
5a0e3ad6 30#include <linux/slab.h>
97222cc8
ED		 31#include <asm/processor.h>
32#include <asm/msr.h>
33#include <asm/page.h>
34#include <asm/current.h>
35#include <asm/apicdef.h>
d0659d94 36#include <asm/delay.h>
60063497 37#include <linux/atomic.h>
c5cc421b 38#include <linux/jump_label.h>
5fdbf976 39#include "kvm_cache_regs.h"
97222cc8 40#include "irq.h"
229456fc 41#include "trace.h"
fc61b800 42#include "x86.h"
00b27a3e 43#include "cpuid.h"
5c919412 44#include "hyperv.h"
97222cc8 45
b682b814
MT		 46#ifndef CONFIG_X86_64
47#define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
48#else
49#define mod_64(x, y) ((x) % (y))
50#endif
51
97222cc8
ED		 52#define PRId64 "d"
53#define PRIx64 "llx"
54#define PRIu64 "u"
55#define PRIo64 "o"
56
57#define APIC_BUS_CYCLE_NS 1
58
59/* #define apic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) */
60#define apic_debug(fmt, arg...)
61
97222cc8 62/* 14 is the version for Xeon and Pentium 8.4.8*/
1e6e2755 63#define APIC_VERSION (0x14UL | ((KVM_APIC_LVT_NUM - 1) << 16))
97222cc8
ED		 64#define LAPIC_MMIO_LENGTH (1 << 12)
65/* followed define is not in apicdef.h */
66#define APIC_SHORT_MASK 0xc0000
67#define APIC_DEST_NOSHORT 0x0
68#define APIC_DEST_MASK 0x800
69#define MAX_APIC_VECTOR 256
ecba9a52 70#define APIC_VECTORS_PER_REG 32
97222cc8 71
394457a9
NA		 72#define APIC_BROADCAST 0xFF
73#define X2APIC_BROADCAST 0xFFFFFFFFul
74
a0c9a822
MT		 75static inline int apic_test_vector(int vec, void *bitmap)
76{
77 return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
78}
79
10606919
YZ		 80bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector)
81{
82 struct kvm_lapic *apic = vcpu->arch.apic;
83
84 return apic_test_vector(vector, apic->regs + APIC_ISR) ||
85 apic_test_vector(vector, apic->regs + APIC_IRR);
86}
87
97222cc8
ED		 88static inline void apic_clear_vector(int vec, void *bitmap)
89{
90 clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
91}
92
8680b94b
MT		 93static inline int __apic_test_and_set_vector(int vec, void *bitmap)
94{
95 return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
96}
97
98static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
99{
100 return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
101}
102
c5cc421b 103struct static_key_deferred apic_hw_disabled __read_mostly;
f8c1ea10
GN		 104struct static_key_deferred apic_sw_disabled __read_mostly;
105
97222cc8
ED		 106static inline int apic_enabled(struct kvm_lapic *apic)
107{
c48f1496 108 return kvm_apic_sw_enabled(apic) && kvm_apic_hw_enabled(apic);
54e9818f
GN		 109}
110
97222cc8
ED		 111#define LVT_MASK \
112 (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK)
113
114#define LINT_MASK \
115 (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
116 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)
117
6e500439
RK		 118static inline u8 kvm_xapic_id(struct kvm_lapic *apic)
119{
120 return kvm_lapic_get_reg(apic, APIC_ID) >> 24;
121}
122
123static inline u32 kvm_x2apic_id(struct kvm_lapic *apic)
124{
125 return apic->vcpu->vcpu_id;
126}
127
e45115b6
RK		 128static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map,
129 u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) {
130 switch (map->mode) {
131 case KVM_APIC_MODE_X2APIC: {
132 u32 offset = (dest_id >> 16) * 16;
0ca52e7b 133 u32 max_apic_id = map->max_apic_id;
e45115b6
RK		 134
135 if (offset <= max_apic_id) {
136 u8 cluster_size = min(max_apic_id - offset + 1, 16U);
137
138 *cluster = &map->phys_map[offset];
139 *mask = dest_id & (0xffff >> (16 - cluster_size));
140 } else {
141 *mask = 0;
142 }
3b5a5ffa 143
e45115b6
RK		 144 return true;
145 }
146 case KVM_APIC_MODE_XAPIC_FLAT:
147 *cluster = map->xapic_flat_map;
148 *mask = dest_id & 0xff;
149 return true;
150 case KVM_APIC_MODE_XAPIC_CLUSTER:
444fdad8 151 *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf];
e45115b6
RK		 152 *mask = dest_id & 0xf;
153 return true;
154 default:
155 /* Not optimized. */
156 return false;
157 }
3548a259
RK		 158}
159
af1bae54 160static void kvm_apic_map_free(struct rcu_head *rcu)
3b5a5ffa 161{
af1bae54 162 struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu);
3b5a5ffa 163
af1bae54 164 kvfree(map);
3b5a5ffa
RK		 165}
166
1e08ec4a
GN		 167static void recalculate_apic_map(struct kvm *kvm)
168{
169 struct kvm_apic_map *new, *old = NULL;
170 struct kvm_vcpu *vcpu;
171 int i;
6e500439 172 u32 max_id = 255; /* enough space for any xAPIC ID */
1e08ec4a
GN		 173
174 mutex_lock(&kvm->arch.apic_map_lock);
175
0ca52e7b
RK		 176 kvm_for_each_vcpu(i, vcpu, kvm)
177 if (kvm_apic_present(vcpu))
6e500439 178 max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic));
0ca52e7b 179
af1bae54
RK		 180 new = kvm_kvzalloc(sizeof(struct kvm_apic_map) +
181 sizeof(struct kvm_lapic *) * ((u64)max_id + 1));
0ca52e7b 182
1e08ec4a
GN		 183 if (!new)
184 goto out;
185
0ca52e7b
RK		 186 new->max_apic_id = max_id;
187
173beedc
NA		 188 kvm_for_each_vcpu(i, vcpu, kvm) {
189 struct kvm_lapic *apic = vcpu->arch.apic;
e45115b6
RK		 190 struct kvm_lapic **cluster;
191 u16 mask;
5bd5db38
RK		 192 u32 ldr;
193 u8 xapic_id;
194 u32 x2apic_id;
1e08ec4a 195
df04d1d1
RK		 196 if (!kvm_apic_present(vcpu))
197 continue;
198
5bd5db38
RK		 199 xapic_id = kvm_xapic_id(apic);
200 x2apic_id = kvm_x2apic_id(apic);
201
202 /* Hotplug hack: see kvm_apic_match_physical_addr(), ... */
203 if ((apic_x2apic_mode(apic) || x2apic_id > 0xff) &&
204 x2apic_id <= new->max_apic_id)
205 new->phys_map[x2apic_id] = apic;
206 /*
207 * ... xAPIC ID of VCPUs with APIC ID > 0xff will wrap-around,
208 * prevent them from masking VCPUs with APIC ID <= 0xff.
209 */
210 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id])
211 new->phys_map[xapic_id] = apic;
3548a259 212
6e500439
RK		 213 ldr = kvm_lapic_get_reg(apic, APIC_LDR);
214
3b5a5ffa
RK		 215 if (apic_x2apic_mode(apic)) {
216 new->mode |= KVM_APIC_MODE_X2APIC;
217 } else if (ldr) {
218 ldr = GET_APIC_LOGICAL_ID(ldr);
dfb95954 219 if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT)
3b5a5ffa
RK		 220 new->mode |= KVM_APIC_MODE_XAPIC_FLAT;
221 else
222 new->mode |= KVM_APIC_MODE_XAPIC_CLUSTER;
223 }
224
e45115b6 225 if (!kvm_apic_map_get_logical_dest(new, ldr, &cluster, &mask))
3548a259
RK		 226 continue;
227
e45115b6
RK		 228 if (mask)
229 cluster[ffs(mask) - 1] = apic;
1e08ec4a
GN		 230 }
231out:
232 old = rcu_dereference_protected(kvm->arch.apic_map,
233 lockdep_is_held(&kvm->arch.apic_map_lock));
234 rcu_assign_pointer(kvm->arch.apic_map, new);
235 mutex_unlock(&kvm->arch.apic_map_lock);
236
237 if (old)
af1bae54 238 call_rcu(&old->rcu, kvm_apic_map_free);
c7c9c56c 239
b053b2ae 240 kvm_make_scan_ioapic_request(kvm);
1e08ec4a
GN		 241}
242
1e1b6c26
NA		 243static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
244{
e462755c 245 bool enabled = val & APIC_SPIV_APIC_ENABLED;
1e1b6c26 246
1e6e2755 247 kvm_lapic_set_reg(apic, APIC_SPIV, val);
e462755c
RK		 248
249 if (enabled != apic->sw_enabled) {
250 apic->sw_enabled = enabled;
251 if (enabled) {
1e1b6c26
NA		 252 static_key_slow_dec_deferred(&apic_sw_disabled);
253 recalculate_apic_map(apic->vcpu->kvm);
254 } else
255 static_key_slow_inc(&apic_sw_disabled.key);
256 }
257}
258
a92e2543 259static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id)
1e08ec4a 260{
1e6e2755 261 kvm_lapic_set_reg(apic, APIC_ID, id << 24);
1e08ec4a
GN		 262 recalculate_apic_map(apic->vcpu->kvm);
263}
264
265static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id)
266{
1e6e2755 267 kvm_lapic_set_reg(apic, APIC_LDR, id);
1e08ec4a
GN		 268 recalculate_apic_map(apic->vcpu->kvm);
269}
270
a92e2543 271static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id)
257b9a5f
RK		 272{
273 u32 ldr = ((id >> 4) << 16) | (1 << (id & 0xf));
274
6e500439
RK		 275 WARN_ON_ONCE(id != apic->vcpu->vcpu_id);
276
a92e2543 277 kvm_lapic_set_reg(apic, APIC_ID, id);
1e6e2755 278 kvm_lapic_set_reg(apic, APIC_LDR, ldr);
257b9a5f
RK		 279 recalculate_apic_map(apic->vcpu->kvm);
280}
281
97222cc8
ED		 282static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type)
283{
dfb95954 284 return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED);
97222cc8
ED		 285}
286
287static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type)
288{
dfb95954 289 return kvm_lapic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK;
97222cc8
ED		 290}
291
a3e06bbe
LJ		 292static inline int apic_lvtt_oneshot(struct kvm_lapic *apic)
293{
f30ebc31 294 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT;
a3e06bbe
LJ		 295}
296
97222cc8
ED		 297static inline int apic_lvtt_period(struct kvm_lapic *apic)
298{
f30ebc31 299 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC;
a3e06bbe
LJ		 300}
301
302static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic)
303{
f30ebc31 304 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE;
97222cc8
ED		 305}
306
cc6e462c
JK		 307static inline int apic_lvt_nmi_mode(u32 lvt_val)
308{
309 return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI;
310}
311
fc61b800
GN		 312void kvm_apic_set_version(struct kvm_vcpu *vcpu)
313{
314 struct kvm_lapic *apic = vcpu->arch.apic;
315 struct kvm_cpuid_entry2 *feat;
316 u32 v = APIC_VERSION;
317
bce87cce 318 if (!lapic_in_kernel(vcpu))
fc61b800
GN		 319 return;
320
321 feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0);
322 if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31))))
323 v |= APIC_LVR_DIRECTED_EOI;
1e6e2755 324 kvm_lapic_set_reg(apic, APIC_LVR, v);
fc61b800
GN		 325}
326
1e6e2755 327static const unsigned int apic_lvt_mask[KVM_APIC_LVT_NUM] = {
a3e06bbe 328 LVT_MASK , /* part LVTT mask, timer mode mask added at runtime */
97222cc8
ED		 329 LVT_MASK | APIC_MODE_MASK, /* LVTTHMR */
330 LVT_MASK | APIC_MODE_MASK, /* LVTPC */
331 LINT_MASK, LINT_MASK, /* LVT0-1 */
332 LVT_MASK /* LVTERR */
333};
334
335static int find_highest_vector(void *bitmap)
336{
ecba9a52
TY		 337 int vec;
338 u32 *reg;
97222cc8 339
ecba9a52
TY		 340 for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG;
341 vec >= 0; vec -= APIC_VECTORS_PER_REG) {
342 reg = bitmap + REG_POS(vec);
343 if (*reg)
810e6def 344 return __fls(*reg) + vec;
ecba9a52 345 }
97222cc8 346
ecba9a52 347 return -1;
97222cc8
ED		 348}
349
8680b94b
MT		 350static u8 count_vectors(void *bitmap)
351{
ecba9a52
TY		 352 int vec;
353 u32 *reg;
8680b94b 354 u8 count = 0;
ecba9a52
TY		 355
356 for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) {
357 reg = bitmap + REG_POS(vec);
358 count += hweight32(*reg);
359 }
360
8680b94b
MT		 361 return count;
362}
363
810e6def 364int __kvm_apic_update_irr(u32 *pir, void *regs)
a20ed54d 365{
810e6def
PB		 366 u32 i, vec;
367 u32 pir_val, irr_val;
368 int max_irr = -1;
a20ed54d 369
810e6def 370 for (i = vec = 0; i <= 7; i++, vec += 32) {
ad361091 371 pir_val = READ_ONCE(pir[i]);
810e6def 372 irr_val = *((u32 *)(regs + APIC_IRR + i * 0x10));
ad361091 373 if (pir_val) {
810e6def
PB		 374 irr_val |= xchg(&pir[i], 0);
375 *((u32 *)(regs + APIC_IRR + i * 0x10)) = irr_val;
ad361091 376 }
810e6def
PB		 377 if (irr_val)
378 max_irr = __fls(irr_val) + vec;
a20ed54d 379 }
810e6def
PB		 380
381 return max_irr;
a20ed54d 382}
705699a1
WV		 383EXPORT_SYMBOL_GPL(__kvm_apic_update_irr);
384
810e6def 385int kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir)
705699a1
WV		 386{
387 struct kvm_lapic *apic = vcpu->arch.apic;
388
b95234c8 389 return __kvm_apic_update_irr(pir, apic->regs);
705699a1 390}
a20ed54d
YZ		 391EXPORT_SYMBOL_GPL(kvm_apic_update_irr);
392
33e4c686 393static inline int apic_search_irr(struct kvm_lapic *apic)
97222cc8 394{
33e4c686 395 return find_highest_vector(apic->regs + APIC_IRR);
97222cc8
ED		 396}
397
398static inline int apic_find_highest_irr(struct kvm_lapic *apic)
399{
400 int result;
401
c7c9c56c
YZ		 402 /*
403 * Note that irr_pending is just a hint. It will be always
404 * true with virtual interrupt delivery enabled.
405 */
33e4c686
GN		 406 if (!apic->irr_pending)
407 return -1;
408
409 result = apic_search_irr(apic);
97222cc8
ED		 410 ASSERT(result == -1 || result >= 16);
411
412 return result;
413}
414
33e4c686
GN		 415static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
416{
56cc2406
WL		 417 struct kvm_vcpu *vcpu;
418
419 vcpu = apic->vcpu;
420
d62caabb 421 if (unlikely(vcpu->arch.apicv_active)) {
b95234c8 422 /* need to update RVI */
f210f757 423 apic_clear_vector(vec, apic->regs + APIC_IRR);
b95234c8
PB		 424 kvm_x86_ops->hwapic_irr_update(vcpu,
425 apic_find_highest_irr(apic));
f210f757
NA		 426 } else {
427 apic->irr_pending = false;
428 apic_clear_vector(vec, apic->regs + APIC_IRR);
429 if (apic_search_irr(apic) != -1)
430 apic->irr_pending = true;
56cc2406 431 }
33e4c686
GN		 432}
433
8680b94b
MT		 434static inline void apic_set_isr(int vec, struct kvm_lapic *apic)
435{
56cc2406
WL		 436 struct kvm_vcpu *vcpu;
437
438 if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR))
439 return;
440
441 vcpu = apic->vcpu;
fc57ac2c 442
8680b94b 443 /*
56cc2406
WL		 444 * With APIC virtualization enabled, all caching is disabled
445 * because the processor can modify ISR under the hood. Instead
446 * just set SVI.
8680b94b 447 */
d62caabb 448 if (unlikely(vcpu->arch.apicv_active))
67c9dddc 449 kvm_x86_ops->hwapic_isr_update(vcpu, vec);
56cc2406
WL		 450 else {
451 ++apic->isr_count;
452 BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
453 /*
454 * ISR (in service register) bit is set when injecting an interrupt.
455 * The highest vector is injected. Thus the latest bit set matches
456 * the highest bit in ISR.
457 */
458 apic->highest_isr_cache = vec;
459 }
8680b94b
MT		 460}
461
fc57ac2c
PB		 462static inline int apic_find_highest_isr(struct kvm_lapic *apic)
463{
464 int result;
465
466 /*
467 * Note that isr_count is always 1, and highest_isr_cache
468 * is always -1, with APIC virtualization enabled.
469 */
470 if (!apic->isr_count)
471 return -1;
472 if (likely(apic->highest_isr_cache != -1))
473 return apic->highest_isr_cache;
474
475 result = find_highest_vector(apic->regs + APIC_ISR);
476 ASSERT(result == -1 || result >= 16);
477
478 return result;
479}
480
8680b94b
MT		 481static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
482{
fc57ac2c
PB		 483 struct kvm_vcpu *vcpu;
484 if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR))
485 return;
486
487 vcpu = apic->vcpu;
488
489 /*
490 * We do get here for APIC virtualization enabled if the guest
491 * uses the Hyper-V APIC enlightenment. In this case we may need
492 * to trigger a new interrupt delivery by writing the SVI field;
493 * on the other hand isr_count and highest_isr_cache are unused
494 * and must be left alone.
495 */
d62caabb 496 if (unlikely(vcpu->arch.apicv_active))
67c9dddc 497 kvm_x86_ops->hwapic_isr_update(vcpu,
fc57ac2c
PB		 498 apic_find_highest_isr(apic));
499 else {
8680b94b 500 --apic->isr_count;
fc57ac2c
PB		 501 BUG_ON(apic->isr_count < 0);
502 apic->highest_isr_cache = -1;
503 }
8680b94b
MT		 504}
505
6e5d865c
YS		 506int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
507{
33e4c686
GN		 508 /* This may race with setting of irr in __apic_accept_irq() and
509 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq
510 * will cause vmexit immediately and the value will be recalculated
511 * on the next vmentry.
512 */
f8543d6a 513 return apic_find_highest_irr(vcpu->arch.apic);
6e5d865c 514}
76dfafd5 515EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr);
6e5d865c 516
6da7e3f6 517static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
b4f2225c 518 int vector, int level, int trig_mode,
9e4aabe2 519 struct dest_map *dest_map);
6da7e3f6 520
b4f2225c 521int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq,
9e4aabe2 522 struct dest_map *dest_map)
97222cc8 523{
ad312c7c 524 struct kvm_lapic *apic = vcpu->arch.apic;
8be5453f 525
58c2dde1 526 return __apic_accept_irq(apic, irq->delivery_mode, irq->vector,
b4f2225c 527 irq->level, irq->trig_mode, dest_map);
97222cc8
ED		 528}
529
ae7a2a3f
MT		 530static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val)
531{
4e335d9e
PB		 532
533 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val,
534 sizeof(val));
ae7a2a3f
MT		 535}
536
537static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val)
538{
4e335d9e
PB		 539
540 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val,
541 sizeof(*val));
ae7a2a3f
MT		 542}
543
544static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu)
545{
546 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;
547}
548
549static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu)
550{
551 u8 val;
552 if (pv_eoi_get_user(vcpu, &val) < 0)
553 apic_debug("Can't read EOI MSR value: 0x%llx\n",
96893977 554 (unsigned long long)vcpu->arch.pv_eoi.msr_val);
ae7a2a3f
MT		 555 return val & 0x1;
556}
557
558static void pv_eoi_set_pending(struct kvm_vcpu *vcpu)
559{
560 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) {
561 apic_debug("Can't set EOI MSR value: 0x%llx\n",
96893977 562 (unsigned long long)vcpu->arch.pv_eoi.msr_val);
ae7a2a3f
MT		 563 return;
564 }
565 __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
566}
567
568static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu)
569{
570 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) {
571 apic_debug("Can't clear EOI MSR value: 0x%llx\n",
96893977 572 (unsigned long long)vcpu->arch.pv_eoi.msr_val);
ae7a2a3f
MT		 573 return;
574 }
575 __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
576}
577
b3c045d3
PB		 578static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr)
579{
3d92789f 580 int highest_irr;
76dfafd5
PB		 581 if (kvm_x86_ops->sync_pir_to_irr && apic->vcpu->arch.apicv_active)
582 highest_irr = kvm_x86_ops->sync_pir_to_irr(apic->vcpu);
583 else
584 highest_irr = apic_find_highest_irr(apic);
b3c045d3
PB		 585 if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr)
586 return -1;
587 return highest_irr;
588}
589
590static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr)
97222cc8 591{
3842d135 592 u32 tpr, isrv, ppr, old_ppr;
97222cc8
ED		 593 int isr;
594
dfb95954
SS		 595 old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI);
596 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI);
97222cc8
ED		 597 isr = apic_find_highest_isr(apic);
598 isrv = (isr != -1) ? isr : 0;
599
600 if ((tpr & 0xf0) >= (isrv & 0xf0))
601 ppr = tpr & 0xff;
602 else
603 ppr = isrv & 0xf0;
604
605 apic_debug("vlapic %p, ppr 0x%x, isr 0x%x, isrv 0x%x",
606 apic, ppr, isr, isrv);
607
b3c045d3
PB		 608 *new_ppr = ppr;
609 if (old_ppr != ppr)
1e6e2755 610 kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr);
b3c045d3
PB		 611
612 return ppr < old_ppr;
613}
614
615static void apic_update_ppr(struct kvm_lapic *apic)
616{
617 u32 ppr;
618
26fbbee5
PB		 619 if (__apic_update_ppr(apic, &ppr) &&
620 apic_has_interrupt_for_ppr(apic, ppr) != -1)
b3c045d3 621 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
97222cc8
ED		 622}
623
eb90f341
PB		 624void kvm_apic_update_ppr(struct kvm_vcpu *vcpu)
625{
626 apic_update_ppr(vcpu->arch.apic);
627}
628EXPORT_SYMBOL_GPL(kvm_apic_update_ppr);
629
97222cc8
ED		 630static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr)
631{
1e6e2755 632 kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr);
97222cc8
ED		 633 apic_update_ppr(apic);
634}
635
03d2249e 636static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda)
394457a9 637{
b4535b58
RK		 638 return mda == (apic_x2apic_mode(apic) ?
639 X2APIC_BROADCAST : APIC_BROADCAST);
394457a9
NA		 640}
641
03d2249e 642static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda)
97222cc8 643{
03d2249e
RK		 644 if (kvm_apic_broadcast(apic, mda))
645 return true;
646
647 if (apic_x2apic_mode(apic))
6e500439 648 return mda == kvm_x2apic_id(apic);
03d2249e 649
5bd5db38
RK		 650 /*
651 * Hotplug hack: Make LAPIC in xAPIC mode also accept interrupts as if
652 * it were in x2APIC mode. Hotplugged VCPUs start in xAPIC mode and
653 * this allows unique addressing of VCPUs with APIC ID over 0xff.
654 * The 0xff condition is needed because writeable xAPIC ID.
655 */
656 if (kvm_x2apic_id(apic) > 0xff && mda == kvm_x2apic_id(apic))
657 return true;
658
b4535b58 659 return mda == kvm_xapic_id(apic);
97222cc8
ED		 660}
661
52c233a4 662static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
97222cc8 663{
0105d1a5
GN		 664 u32 logical_id;
665
394457a9 666 if (kvm_apic_broadcast(apic, mda))
9368b567 667 return true;
394457a9 668
dfb95954 669 logical_id = kvm_lapic_get_reg(apic, APIC_LDR);
97222cc8 670
9368b567 671 if (apic_x2apic_mode(apic))
8a395363
RK		 672 return ((logical_id >> 16) == (mda >> 16))
673 && (logical_id & mda & 0xffff) != 0;
97222cc8 674
9368b567 675 logical_id = GET_APIC_LOGICAL_ID(logical_id);
97222cc8 676
dfb95954 677 switch (kvm_lapic_get_reg(apic, APIC_DFR)) {
97222cc8 678 case APIC_DFR_FLAT:
9368b567 679 return (logical_id & mda) != 0;
97222cc8 680 case APIC_DFR_CLUSTER:
9368b567
RK		 681 return ((logical_id >> 4) == (mda >> 4))
682 && (logical_id & mda & 0xf) != 0;
97222cc8 683 default:
7712de87 684 apic_debug("Bad DFR vcpu %d: %08x\n",
dfb95954 685 apic->vcpu->vcpu_id, kvm_lapic_get_reg(apic, APIC_DFR));
9368b567 686 return false;
97222cc8 687 }
97222cc8
ED		 688}
689
c519265f
RK		 690/* The KVM local APIC implementation has two quirks:
691 *
b4535b58
RK		 692 * - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs
693 * in xAPIC mode if the "destination & 0xff" matches its xAPIC ID.
694 * KVM doesn't do that aliasing.
c519265f
RK		 695 *
696 * - in-kernel IOAPIC messages have to be delivered directly to
697 * x2APIC, because the kernel does not support interrupt remapping.
698 * In order to support broadcast without interrupt remapping, x2APIC
699 * rewrites the destination of non-IPI messages from APIC_BROADCAST
700 * to X2APIC_BROADCAST.
701 *
702 * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API. This is
703 * important when userspace wants to use x2APIC-format MSIs, because
704 * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7".
03d2249e 705 */
c519265f
RK		 706static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id,
707 struct kvm_lapic *source, struct kvm_lapic *target)
03d2249e
RK		 708{
709 bool ipi = source != NULL;
03d2249e 710
c519265f 711 if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled &&
b4535b58 712 !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target))
03d2249e
RK		 713 return X2APIC_BROADCAST;
714
b4535b58 715 return dest_id;
03d2249e
RK		 716}
717
52c233a4 718bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
394457a9 719 int short_hand, unsigned int dest, int dest_mode)
97222cc8 720{
ad312c7c 721 struct kvm_lapic *target = vcpu->arch.apic;
c519265f 722 u32 mda = kvm_apic_mda(vcpu, dest, source, target);
97222cc8
ED		 723
724 apic_debug("target %p, source %p, dest 0x%x, "
343f94fe 725 "dest_mode 0x%x, short_hand 0x%x\n",
97222cc8
ED		 726 target, source, dest, dest_mode, short_hand);
727
bd371396 728 ASSERT(target);
97222cc8
ED		 729 switch (short_hand) {
730 case APIC_DEST_NOSHORT:
3697f302 731 if (dest_mode == APIC_DEST_PHYSICAL)
03d2249e 732 return kvm_apic_match_physical_addr(target, mda);
343f94fe 733 else
03d2249e 734 return kvm_apic_match_logical_addr(target, mda);
97222cc8 735 case APIC_DEST_SELF:
9368b567 736 return target == source;
97222cc8 737 case APIC_DEST_ALLINC:
9368b567 738 return true;
97222cc8 739 case APIC_DEST_ALLBUT:
9368b567 740 return target != source;
97222cc8 741 default:
7712de87
JK		 742 apic_debug("kvm: apic: Bad dest shorthand value %x\n",
743 short_hand);
9368b567 744 return false;
97222cc8 745 }
97222cc8 746}
1e6e2755 747EXPORT_SYMBOL_GPL(kvm_apic_match_dest);
97222cc8 748
52004014
FW		 749int kvm_vector_to_index(u32 vector, u32 dest_vcpus,
750 const unsigned long *bitmap, u32 bitmap_size)
751{
752 u32 mod;
753 int i, idx = -1;
754
755 mod = vector % dest_vcpus;
756
757 for (i = 0; i <= mod; i++) {
758 idx = find_next_bit(bitmap, bitmap_size, idx + 1);
759 BUG_ON(idx == bitmap_size);
760 }
761
762 return idx;
763}
764
4efd805f
RK		 765static void kvm_apic_disabled_lapic_found(struct kvm *kvm)
766{
767 if (!kvm->arch.disabled_lapic_found) {
768 kvm->arch.disabled_lapic_found = true;
769 printk(KERN_INFO
770 "Disabled LAPIC found during irq injection\n");
771 }
772}
773
c519265f
RK		 774static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src,
775 struct kvm_lapic_irq *irq, struct kvm_apic_map *map)
1e08ec4a 776{
c519265f
RK		 777 if (kvm->arch.x2apic_broadcast_quirk_disabled) {
778 if ((irq->dest_id == APIC_BROADCAST &&
779 map->mode != KVM_APIC_MODE_X2APIC))
780 return true;
781 if (irq->dest_id == X2APIC_BROADCAST)
782 return true;
783 } else {
784 bool x2apic_ipi = src && *src && apic_x2apic_mode(*src);
785 if (irq->dest_id == (x2apic_ipi ?
786 X2APIC_BROADCAST : APIC_BROADCAST))
787 return true;
788 }
1e08ec4a 789
c519265f
RK		 790 return false;
791}
1e08ec4a 792
64aa47bf
RK		 793/* Return true if the interrupt can be handled by using *bitmap as index mask
794 * for valid destinations in *dst array.
795 * Return false if kvm_apic_map_get_dest_lapic did nothing useful.
796 * Note: we may have zero kvm_lapic destinations when we return true, which
797 * means that the interrupt should be dropped. In this case, *bitmap would be
798 * zero and *dst undefined.
799 */
800static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm,
801 struct kvm_lapic **src, struct kvm_lapic_irq *irq,
802 struct kvm_apic_map *map, struct kvm_lapic ***dst,
803 unsigned long *bitmap)
804{
805 int i, lowest;
1e08ec4a 806
64aa47bf
RK		 807 if (irq->shorthand == APIC_DEST_SELF && src) {
808 *dst = src;
809 *bitmap = 1;
810 return true;
811 } else if (irq->shorthand)
1e08ec4a
GN		 812 return false;
813
c519265f 814 if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map))
9ea369b0
RK		 815 return false;
816
64aa47bf 817 if (irq->dest_mode == APIC_DEST_PHYSICAL) {
0ca52e7b 818 if (irq->dest_id > map->max_apic_id) {
64aa47bf
RK		 819 *bitmap = 0;
820 } else {
821 *dst = &map->phys_map[irq->dest_id];
822 *bitmap = 1;
823 }
1e08ec4a 824 return true;
bea15428 825 }
698f9755 826
e45115b6
RK		 827 *bitmap = 0;
828 if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst,
829 (u16 *)bitmap))
1e08ec4a 830 return false;
fa834e91 831
64aa47bf
RK		 832 if (!kvm_lowest_prio_delivery(irq))
833 return true;
3548a259 834
64aa47bf
RK		 835 if (!kvm_vector_hashing_enabled()) {
836 lowest = -1;
837 for_each_set_bit(i, bitmap, 16) {
838 if (!(*dst)[i])
839 continue;
840 if (lowest < 0)
841 lowest = i;
842 else if (kvm_apic_compare_prio((*dst)[i]->vcpu,
843 (*dst)[lowest]->vcpu) < 0)
844 lowest = i;
3548a259 845 }
64aa47bf
RK		 846 } else {
847 if (!*bitmap)
848 return true;
3548a259 849
64aa47bf
RK		 850 lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap),
851 bitmap, 16);
45c3094a 852
64aa47bf
RK		 853 if (!(*dst)[lowest]) {
854 kvm_apic_disabled_lapic_found(kvm);
855 *bitmap = 0;
856 return true;
857 }
858 }
1e08ec4a 859
64aa47bf 860 *bitmap = (lowest >= 0) ? 1 << lowest : 0;
1e08ec4a 861
64aa47bf
RK		 862 return true;
863}
52004014 864
64aa47bf
RK		 865bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
866 struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map)
867{
868 struct kvm_apic_map *map;
869 unsigned long bitmap;
870 struct kvm_lapic **dst = NULL;
871 int i;
872 bool ret;
52004014 873
64aa47bf 874 *r = -1;
52004014 875
64aa47bf
RK		 876 if (irq->shorthand == APIC_DEST_SELF) {
877 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map);
878 return true;
879 }
52004014 880
64aa47bf
RK		 881 rcu_read_lock();
882 map = rcu_dereference(kvm->arch.apic_map);
52004014 883
64aa47bf
RK		 884 ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap);
885 if (ret)
886 for_each_set_bit(i, &bitmap, 16) {
887 if (!dst[i])
888 continue;
889 if (*r < 0)
890 *r = 0;
891 *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map);
1e08ec4a 892 }
1e08ec4a 893
1e08ec4a
GN		 894 rcu_read_unlock();
895 return ret;
896}
897
6228a0da
FW		 898/*
899 * This routine tries to handler interrupts in posted mode, here is how
900 * it deals with different cases:
901 * - For single-destination interrupts, handle it in posted mode
902 * - Else if vector hashing is enabled and it is a lowest-priority
903 * interrupt, handle it in posted mode and use the following mechanism
904 * to find the destinaiton vCPU.
905 * 1. For lowest-priority interrupts, store all the possible
906 * destination vCPUs in an array.
907 * 2. Use "guest vector % max number of destination vCPUs" to find
908 * the right destination vCPU in the array for the lowest-priority
909 * interrupt.
910 * - Otherwise, use remapped mode to inject the interrupt.
911 */
8feb4a04
FW		 912bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq,
913 struct kvm_vcpu **dest_vcpu)
914{
915 struct kvm_apic_map *map;
64aa47bf
RK		 916 unsigned long bitmap;
917 struct kvm_lapic **dst = NULL;
8feb4a04 918 bool ret = false;
8feb4a04
FW		 919
920 if (irq->shorthand)
921 return false;
922
923 rcu_read_lock();
924 map = rcu_dereference(kvm->arch.apic_map);
925
64aa47bf
RK		 926 if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) &&
927 hweight16(bitmap) == 1) {
928 unsigned long i = find_first_bit(&bitmap, 16);
6228a0da 929
64aa47bf
RK		 930 if (dst[i]) {
931 *dest_vcpu = dst[i]->vcpu;
932 ret = true;
6228a0da 933 }
8feb4a04
FW		 934 }
935
8feb4a04
FW		 936 rcu_read_unlock();
937 return ret;
938}
939
97222cc8
ED		 940/*
941 * Add a pending IRQ into lapic.
942 * Return 1 if successfully added and 0 if discarded.
943 */
944static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
b4f2225c 945 int vector, int level, int trig_mode,
9e4aabe2 946 struct dest_map *dest_map)
97222cc8 947{
6da7e3f6 948 int result = 0;
c5ec1534 949 struct kvm_vcpu *vcpu = apic->vcpu;
97222cc8 950
a183b638
PB		 951 trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode,
952 trig_mode, vector);
97222cc8 953 switch (delivery_mode) {
97222cc8 954 case APIC_DM_LOWEST:
e1035715
GN		 955 vcpu->arch.apic_arb_prio++;
956 case APIC_DM_FIXED:
bdaffe1d
PB		 957 if (unlikely(trig_mode && !level))
958 break;
959
97222cc8
ED		 960 /* FIXME add logic for vcpu on reset */
961 if (unlikely(!apic_enabled(apic)))
962 break;
963
11f5cc05
JK		 964 result = 1;
965
9daa5007 966 if (dest_map) {
9e4aabe2 967 __set_bit(vcpu->vcpu_id, dest_map->map);
9daa5007
JR		 968 dest_map->vectors[vcpu->vcpu_id] = vector;
969 }
a5d36f82 970
bdaffe1d
PB		 971 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) {
972 if (trig_mode)
1e6e2755 973 kvm_lapic_set_vector(vector, apic->regs + APIC_TMR);
bdaffe1d
PB		 974 else
975 apic_clear_vector(vector, apic->regs + APIC_TMR);
976 }
977
d62caabb 978 if (vcpu->arch.apicv_active)
5a71785d 979 kvm_x86_ops->deliver_posted_interrupt(vcpu, vector);
11f5cc05 980 else {
1e6e2755 981 kvm_lapic_set_irr(vector, apic);
5a71785d
YZ		 982
983 kvm_make_request(KVM_REQ_EVENT, vcpu);
984 kvm_vcpu_kick(vcpu);
985 }
97222cc8
ED		 986 break;
987
988 case APIC_DM_REMRD:
24d2166b
R		 989 result = 1;
990 vcpu->arch.pv.pv_unhalted = 1;
991 kvm_make_request(KVM_REQ_EVENT, vcpu);
992 kvm_vcpu_kick(vcpu);
97222cc8
ED		 993 break;
994
995 case APIC_DM_SMI:
64d60670
PB		 996 result = 1;
997 kvm_make_request(KVM_REQ_SMI, vcpu);
998 kvm_vcpu_kick(vcpu);
97222cc8 999 break;
3419ffc8 1000
97222cc8 1001 case APIC_DM_NMI:
6da7e3f6 1002 result = 1;
3419ffc8 1003 kvm_inject_nmi(vcpu);
26df99c6 1004 kvm_vcpu_kick(vcpu);
97222cc8
ED		 1005 break;
1006
1007 case APIC_DM_INIT:
a52315e1 1008 if (!trig_mode || level) {
6da7e3f6 1009 result = 1;
66450a21
JK		 1010 /* assumes that there are only KVM_APIC_INIT/SIPI */
1011 apic->pending_events = (1UL << KVM_APIC_INIT);
1012 /* make sure pending_events is visible before sending
1013 * the request */
1014 smp_wmb();
3842d135 1015 kvm_make_request(KVM_REQ_EVENT, vcpu);
c5ec1534
HQ		 1016 kvm_vcpu_kick(vcpu);
1017 } else {
1b10bf31
JK		 1018 apic_debug("Ignoring de-assert INIT to vcpu %d\n",
1019 vcpu->vcpu_id);
c5ec1534 1020 }
97222cc8
ED		 1021 break;
1022
1023 case APIC_DM_STARTUP:
1b10bf31
JK		 1024 apic_debug("SIPI to vcpu %d vector 0x%02x\n",
1025 vcpu->vcpu_id, vector);
66450a21
JK		 1026 result = 1;
1027 apic->sipi_vector = vector;
1028 /* make sure sipi_vector is visible for the receiver */
1029 smp_wmb();
1030 set_bit(KVM_APIC_SIPI, &apic->pending_events);
1031 kvm_make_request(KVM_REQ_EVENT, vcpu);
1032 kvm_vcpu_kick(vcpu);
97222cc8
ED		 1033 break;
1034
23930f95
JK		 1035 case APIC_DM_EXTINT:
1036 /*
1037 * Should only be called by kvm_apic_local_deliver() with LVT0,
1038 * before NMI watchdog was enabled. Already handled by
1039 * kvm_apic_accept_pic_intr().
1040 */
1041 break;
1042
97222cc8
ED		 1043 default:
1044 printk(KERN_ERR "TODO: unsupported delivery mode %x\n",
1045 delivery_mode);
1046 break;
1047 }
1048 return result;
1049}
1050
e1035715 1051int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
8be5453f 1052{
e1035715 1053 return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio;
8be5453f
ZX		 1054}
1055
3bb345f3
PB		 1056static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector)
1057{
6308630b 1058 return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors);
3bb345f3
PB		 1059}
1060
c7c9c56c
YZ		 1061static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector)
1062{
7543a635
SR		 1063 int trigger_mode;
1064
1065 /* Eoi the ioapic only if the ioapic doesn't own the vector. */
1066 if (!kvm_ioapic_handles_vector(apic, vector))
1067 return;
3bb345f3 1068
7543a635
SR		 1069 /* Request a KVM exit to inform the userspace IOAPIC. */
1070 if (irqchip_split(apic->vcpu->kvm)) {
1071 apic->vcpu->arch.pending_ioapic_eoi = vector;
1072 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu);
1073 return;
c7c9c56c 1074 }
7543a635
SR		 1075
1076 if (apic_test_vector(vector, apic->regs + APIC_TMR))
1077 trigger_mode = IOAPIC_LEVEL_TRIG;
1078 else
1079 trigger_mode = IOAPIC_EDGE_TRIG;
1080
1081 kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode);
c7c9c56c
YZ		 1082}
1083
ae7a2a3f 1084static int apic_set_eoi(struct kvm_lapic *apic)
97222cc8
ED		 1085{
1086 int vector = apic_find_highest_isr(apic);
ae7a2a3f
MT		 1087
1088 trace_kvm_eoi(apic, vector);
1089
97222cc8
ED		 1090 /*
1091 * Not every write EOI will has corresponding ISR,
1092 * one example is when Kernel check timer on setup_IO_APIC
1093 */
1094 if (vector == -1)
ae7a2a3f 1095 return vector;
97222cc8 1096
8680b94b 1097 apic_clear_isr(vector, apic);
97222cc8
ED		 1098 apic_update_ppr(apic);
1099
5c919412
AS		 1100 if (test_bit(vector, vcpu_to_synic(apic->vcpu)->vec_bitmap))
1101 kvm_hv_synic_send_eoi(apic->vcpu, vector);
1102
c7c9c56c 1103 kvm_ioapic_send_eoi(apic, vector);
3842d135 1104 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
ae7a2a3f 1105 return vector;
97222cc8
ED		 1106}
1107
c7c9c56c
YZ		 1108/*
1109 * this interface assumes a trap-like exit, which has already finished
1110 * desired side effect including vISR and vPPR update.
1111 */
1112void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector)
1113{
1114 struct kvm_lapic *apic = vcpu->arch.apic;
1115
1116 trace_kvm_eoi(apic, vector);
1117
1118 kvm_ioapic_send_eoi(apic, vector);
1119 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
1120}
1121EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated);
1122
97222cc8
ED		 1123static void apic_send_ipi(struct kvm_lapic *apic)
1124{
dfb95954
SS		 1125 u32 icr_low = kvm_lapic_get_reg(apic, APIC_ICR);
1126 u32 icr_high = kvm_lapic_get_reg(apic, APIC_ICR2);
58c2dde1 1127 struct kvm_lapic_irq irq;
97222cc8 1128
58c2dde1
GN		 1129 irq.vector = icr_low & APIC_VECTOR_MASK;
1130 irq.delivery_mode = icr_low & APIC_MODE_MASK;
1131 irq.dest_mode = icr_low & APIC_DEST_MASK;
b7cb2231 1132 irq.level = (icr_low & APIC_INT_ASSERT) != 0;
58c2dde1
GN		 1133 irq.trig_mode = icr_low & APIC_INT_LEVELTRIG;
1134 irq.shorthand = icr_low & APIC_SHORT_MASK;
93bbf0b8 1135 irq.msi_redir_hint = false;
0105d1a5
GN		 1136 if (apic_x2apic_mode(apic))
1137 irq.dest_id = icr_high;
1138 else
1139 irq.dest_id = GET_APIC_DEST_FIELD(icr_high);
97222cc8 1140
1000ff8d
GN		 1141 trace_kvm_apic_ipi(icr_low, irq.dest_id);
1142
97222cc8
ED		 1143 apic_debug("icr_high 0x%x, icr_low 0x%x, "
1144 "short_hand 0x%x, dest 0x%x, trig_mode 0x%x, level 0x%x, "
93bbf0b8
JS		 1145 "dest_mode 0x%x, delivery_mode 0x%x, vector 0x%x, "
1146 "msi_redir_hint 0x%x\n",
9b5843dd 1147 icr_high, icr_low, irq.shorthand, irq.dest_id,
58c2dde1 1148 irq.trig_mode, irq.level, irq.dest_mode, irq.delivery_mode,
93bbf0b8 1149 irq.vector, irq.msi_redir_hint);
58c2dde1 1150
b4f2225c 1151 kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL);
97222cc8
ED		 1152}
1153
1154static u32 apic_get_tmcct(struct kvm_lapic *apic)
1155{
8003c9ae 1156 ktime_t remaining, now;
b682b814 1157 s64 ns;
9da8f4e8 1158 u32 tmcct;
97222cc8
ED		 1159
1160 ASSERT(apic != NULL);
1161
9da8f4e8 1162 /* if initial count is 0, current count should also be 0 */
dfb95954 1163 if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 ||
b963a22e 1164 apic->lapic_timer.period == 0)
9da8f4e8
KP		 1165 return 0;
1166
5587859f 1167 now = ktime_get();
8003c9ae 1168 remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
b682b814 1169 if (ktime_to_ns(remaining) < 0)
8b0e1953 1170 remaining = 0;
b682b814 1171
d3c7b77d
MT		 1172 ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period);
1173 tmcct = div64_u64(ns,
1174 (APIC_BUS_CYCLE_NS * apic->divide_count));
97222cc8
ED		 1175
1176 return tmcct;
1177}
1178
b209749f
AK		 1179static void __report_tpr_access(struct kvm_lapic *apic, bool write)
1180{
1181 struct kvm_vcpu *vcpu = apic->vcpu;
1182 struct kvm_run *run = vcpu->run;
1183
a8eeb04a 1184 kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu);
5fdbf976 1185 run->tpr_access.rip = kvm_rip_read(vcpu);
b209749f
AK		 1186 run->tpr_access.is_write = write;
1187}
1188
1189static inline void report_tpr_access(struct kvm_lapic *apic, bool write)
1190{
1191 if (apic->vcpu->arch.tpr_access_reporting)
1192 __report_tpr_access(apic, write);
1193}
1194
97222cc8
ED		 1195static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
1196{
1197 u32 val = 0;
1198
1199 if (offset >= LAPIC_MMIO_LENGTH)
1200 return 0;
1201
1202 switch (offset) {
1203 case APIC_ARBPRI:
7712de87 1204 apic_debug("Access APIC ARBPRI register which is for P6\n");
97222cc8
ED		 1205 break;
1206
1207 case APIC_TMCCT: /* Timer CCR */
a3e06bbe
LJ		 1208 if (apic_lvtt_tscdeadline(apic))
1209 return 0;
1210
97222cc8
ED		 1211 val = apic_get_tmcct(apic);
1212 break;
4a4541a4
AK		 1213 case APIC_PROCPRI:
1214 apic_update_ppr(apic);
dfb95954 1215 val = kvm_lapic_get_reg(apic, offset);
4a4541a4 1216 break;
b209749f
AK		 1217 case APIC_TASKPRI:
1218 report_tpr_access(apic, false);
1219 /* fall thru */
97222cc8 1220 default:
dfb95954 1221 val = kvm_lapic_get_reg(apic, offset);
97222cc8
ED		 1222 break;
1223 }
1224
1225 return val;
1226}
1227
d76685c4
GH		 1228static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev)
1229{
1230 return container_of(dev, struct kvm_lapic, dev);
1231}
1232
1e6e2755 1233int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
0105d1a5 1234 void *data)
97222cc8 1235{
97222cc8
ED		 1236 unsigned char alignment = offset & 0xf;
1237 u32 result;
d5b0b5b1 1238 /* this bitmask has a bit cleared for each reserved register */
0105d1a5 1239 static const u64 rmask = 0x43ff01ffffffe70cULL;
97222cc8
ED		 1240
1241 if ((alignment + len) > 4) {
4088bb3c
GN		 1242 apic_debug("KVM_APIC_READ: alignment error %x %d\n",
1243 offset, len);
0105d1a5 1244 return 1;
97222cc8 1245 }
0105d1a5
GN		 1246
1247 if (offset > 0x3f0 || !(rmask & (1ULL << (offset >> 4)))) {
4088bb3c
GN		 1248 apic_debug("KVM_APIC_READ: read reserved register %x\n",
1249 offset);
0105d1a5
GN		 1250 return 1;
1251 }
1252
97222cc8
ED		 1253 result = __apic_read(apic, offset & ~0xf);
1254
229456fc
MT		 1255 trace_kvm_apic_read(offset, result);
1256
97222cc8
ED		 1257 switch (len) {
1258 case 1:
1259 case 2:
1260 case 4:
1261 memcpy(data, (char *)&result + alignment, len);
1262 break;
1263 default:
1264 printk(KERN_ERR "Local APIC read with len = %x, "
1265 "should be 1,2, or 4 instead\n", len);
1266 break;
1267 }
bda9020e 1268 return 0;
97222cc8 1269}
1e6e2755 1270EXPORT_SYMBOL_GPL(kvm_lapic_reg_read);
97222cc8 1271
0105d1a5
GN		 1272static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
1273{
c48f1496 1274 return kvm_apic_hw_enabled(apic) &&
0105d1a5
GN		 1275 addr >= apic->base_address &&
1276 addr < apic->base_address + LAPIC_MMIO_LENGTH;
1277}
1278
e32edf4f 1279static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
0105d1a5
GN		 1280 gpa_t address, int len, void *data)
1281{
1282 struct kvm_lapic *apic = to_lapic(this);
1283 u32 offset = address - apic->base_address;
1284
1285 if (!apic_mmio_in_range(apic, address))
1286 return -EOPNOTSUPP;
1287
1e6e2755 1288 kvm_lapic_reg_read(apic, offset, len, data);
0105d1a5
GN		 1289
1290 return 0;
1291}
1292
97222cc8
ED		 1293static void update_divide_count(struct kvm_lapic *apic)
1294{
1295 u32 tmp1, tmp2, tdcr;
1296
dfb95954 1297 tdcr = kvm_lapic_get_reg(apic, APIC_TDCR);
97222cc8
ED		 1298 tmp1 = tdcr & 0xf;
1299 tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1;
d3c7b77d 1300 apic->divide_count = 0x1 << (tmp2 & 0x7);
97222cc8
ED		 1301
1302 apic_debug("timer divide count is 0x%x\n",
9b5843dd 1303 apic->divide_count);
97222cc8
ED		 1304}
1305
b6ac0695
RK		 1306static void apic_update_lvtt(struct kvm_lapic *apic)
1307{
dfb95954 1308 u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) &
b6ac0695
RK		 1309 apic->lapic_timer.timer_mode_mask;
1310
1311 if (apic->lapic_timer.timer_mode != timer_mode) {
1312 apic->lapic_timer.timer_mode = timer_mode;
1313 hrtimer_cancel(&apic->lapic_timer.timer);
1314 }
1315}
1316
5d87db71
RK		 1317static void apic_timer_expired(struct kvm_lapic *apic)
1318{
1319 struct kvm_vcpu *vcpu = apic->vcpu;
8577370f 1320 struct swait_queue_head *q = &vcpu->wq;
d0659d94 1321 struct kvm_timer *ktimer = &apic->lapic_timer;
5d87db71 1322
5d87db71
RK		 1323 if (atomic_read(&apic->lapic_timer.pending))
1324 return;
1325
1326 atomic_inc(&apic->lapic_timer.pending);
bab5bb39 1327 kvm_set_pending_timer(vcpu);
5d87db71 1328
8577370f
MT		 1329 if (swait_active(q))
1330 swake_up(q);
d0659d94
MT		 1331
1332 if (apic_lvtt_tscdeadline(apic))
1333 ktimer->expired_tscdeadline = ktimer->tscdeadline;
1334}
1335
1336/*
1337 * On APICv, this test will cause a busy wait
1338 * during a higher-priority task.
1339 */
1340
1341static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu)
1342{
1343 struct kvm_lapic *apic = vcpu->arch.apic;
dfb95954 1344 u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT);
d0659d94
MT		 1345
1346 if (kvm_apic_hw_enabled(apic)) {
1347 int vec = reg & APIC_VECTOR_MASK;
f9339860 1348 void *bitmap = apic->regs + APIC_ISR;
d0659d94 1349
d62caabb 1350 if (vcpu->arch.apicv_active)
f9339860
MT		 1351 bitmap = apic->regs + APIC_IRR;
1352
1353 if (apic_test_vector(vec, bitmap))
1354 return true;
d0659d94
MT		 1355 }
1356 return false;
1357}
1358
1359void wait_lapic_expire(struct kvm_vcpu *vcpu)
1360{
1361 struct kvm_lapic *apic = vcpu->arch.apic;
1362 u64 guest_tsc, tsc_deadline;
1363
bce87cce 1364 if (!lapic_in_kernel(vcpu))
d0659d94
MT		 1365 return;
1366
1367 if (apic->lapic_timer.expired_tscdeadline == 0)
1368 return;
1369
1370 if (!lapic_timer_int_injected(vcpu))
1371 return;
1372
1373 tsc_deadline = apic->lapic_timer.expired_tscdeadline;
1374 apic->lapic_timer.expired_tscdeadline = 0;
4ba76538 1375 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
6c19b753 1376 trace_kvm_wait_lapic_expire(vcpu->vcpu_id, guest_tsc - tsc_deadline);
d0659d94
MT		 1377
1378 /* __delay is delay_tsc whenever the hardware has TSC, thus always. */
1379 if (guest_tsc < tsc_deadline)
b606f189
MT		 1380 __delay(min(tsc_deadline - guest_tsc,
1381 nsec_to_cycles(vcpu, lapic_timer_advance_ns)));
5d87db71
RK		 1382}
1383
53f9eedf
YJ		 1384static void start_sw_tscdeadline(struct kvm_lapic *apic)
1385{
1386 u64 guest_tsc, tscdeadline = apic->lapic_timer.tscdeadline;
1387 u64 ns = 0;
1388 ktime_t expire;
1389 struct kvm_vcpu *vcpu = apic->vcpu;
1390 unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz;
1391 unsigned long flags;
1392 ktime_t now;
1393
1394 if (unlikely(!tscdeadline || !this_tsc_khz))
1395 return;
1396
1397 local_irq_save(flags);
1398
5587859f 1399 now = ktime_get();
53f9eedf
YJ		 1400 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1401 if (likely(tscdeadline > guest_tsc)) {
1402 ns = (tscdeadline - guest_tsc) * 1000000ULL;
1403 do_div(ns, this_tsc_khz);
1404 expire = ktime_add_ns(now, ns);
1405 expire = ktime_sub_ns(expire, lapic_timer_advance_ns);
1406 hrtimer_start(&apic->lapic_timer.timer,
1407 expire, HRTIMER_MODE_ABS_PINNED);
1408 } else
1409 apic_timer_expired(apic);
1410
1411 local_irq_restore(flags);
1412}
1413
7d7f7da2 1414static void start_sw_period(struct kvm_lapic *apic)
8003c9ae
WL		 1415{
1416 if (!apic->lapic_timer.period)
1417 return;
1418
1419 if (apic_lvtt_oneshot(apic) &&
5587859f 1420 ktime_after(ktime_get(),
8003c9ae
WL		 1421 apic->lapic_timer.target_expiration)) {
1422 apic_timer_expired(apic);
1423 return;
1424 }
1425
1426 hrtimer_start(&apic->lapic_timer.timer,
1427 apic->lapic_timer.target_expiration,
1428 HRTIMER_MODE_ABS_PINNED);
1429}
1430
1431static bool set_target_expiration(struct kvm_lapic *apic)
7d7f7da2
WL		 1432{
1433 ktime_t now;
8003c9ae 1434 u64 tscl = rdtsc();
7d7f7da2 1435
5587859f 1436 now = ktime_get();
7d7f7da2 1437 apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT)
8003c9ae 1438 * APIC_BUS_CYCLE_NS * apic->divide_count;
7d7f7da2
WL		 1439
1440 if (!apic->lapic_timer.period)
8003c9ae
WL		 1441 return false;
1442
7d7f7da2
WL		 1443 /*
1444 * Do not allow the guest to program periodic timers with small
1445 * interval, since the hrtimers are not throttled by the host
1446 * scheduler.
1447 */
1448 if (apic_lvtt_period(apic)) {
1449 s64 min_period = min_timer_period_us * 1000LL;
1450
1451 if (apic->lapic_timer.period < min_period) {
1452 pr_info_ratelimited(
1453 "kvm: vcpu %i: requested %lld ns "
1454 "lapic timer period limited to %lld ns\n",
1455 apic->vcpu->vcpu_id,
1456 apic->lapic_timer.period, min_period);
1457 apic->lapic_timer.period = min_period;
1458 }
1459 }
1460
7d7f7da2
WL		 1461 apic_debug("%s: bus cycle is %" PRId64 "ns, now 0x%016"
1462 PRIx64 ", "
1463 "timer initial count 0x%x, period %lldns, "
1464 "expire @ 0x%016" PRIx64 ".\n", __func__,
1465 APIC_BUS_CYCLE_NS, ktime_to_ns(now),
1466 kvm_lapic_get_reg(apic, APIC_TMICT),
1467 apic->lapic_timer.period,
1468 ktime_to_ns(ktime_add_ns(now,
1469 apic->lapic_timer.period)));
8003c9ae
WL		 1470
1471 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
1472 nsec_to_cycles(apic->vcpu, apic->lapic_timer.period);
1473 apic->lapic_timer.target_expiration = ktime_add_ns(now, apic->lapic_timer.period);
1474
1475 return true;
1476}
1477
1478static void advance_periodic_target_expiration(struct kvm_lapic *apic)
1479{
1480 apic->lapic_timer.tscdeadline +=
1481 nsec_to_cycles(apic->vcpu, apic->lapic_timer.period);
1482 apic->lapic_timer.target_expiration =
1483 ktime_add_ns(apic->lapic_timer.target_expiration,
1484 apic->lapic_timer.period);
7d7f7da2
WL		 1485}
1486
ce7a058a
YJ		 1487bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu)
1488{
91005300
WL		 1489 if (!lapic_in_kernel(vcpu))
1490 return false;
1491
ce7a058a
YJ		 1492 return vcpu->arch.apic->lapic_timer.hv_timer_in_use;
1493}
1494EXPORT_SYMBOL_GPL(kvm_lapic_hv_timer_in_use);
1495
7e810a38 1496static void cancel_hv_timer(struct kvm_lapic *apic)
bd97ad0e
WL		 1497{
1498 kvm_x86_ops->cancel_hv_timer(apic->vcpu);
1499 apic->lapic_timer.hv_timer_in_use = false;
1500}
1501
7e810a38 1502static bool start_hv_timer(struct kvm_lapic *apic)
196f20ca
WL		 1503{
1504 u64 tscdeadline = apic->lapic_timer.tscdeadline;
1505
8003c9ae
WL		 1506 if ((atomic_read(&apic->lapic_timer.pending) &&
1507 !apic_lvtt_period(apic)) ||
196f20ca
WL		 1508 kvm_x86_ops->set_hv_timer(apic->vcpu, tscdeadline)) {
1509 if (apic->lapic_timer.hv_timer_in_use)
7e810a38 1510 cancel_hv_timer(apic);
196f20ca
WL		 1511 } else {
1512 apic->lapic_timer.hv_timer_in_use = true;
1513 hrtimer_cancel(&apic->lapic_timer.timer);
1514
1515 /* In case the sw timer triggered in the window */
8003c9ae
WL		 1516 if (atomic_read(&apic->lapic_timer.pending) &&
1517 !apic_lvtt_period(apic))
7e810a38 1518 cancel_hv_timer(apic);
196f20ca
WL		 1519 }
1520 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id,
1521 apic->lapic_timer.hv_timer_in_use);
1522 return apic->lapic_timer.hv_timer_in_use;
1523}
1524
8003c9ae
WL		 1525void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu)
1526{
1527 struct kvm_lapic *apic = vcpu->arch.apic;
1528
1529 WARN_ON(!apic->lapic_timer.hv_timer_in_use);
1530 WARN_ON(swait_active(&vcpu->wq));
1531 cancel_hv_timer(apic);
1532 apic_timer_expired(apic);
1533
1534 if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
1535 advance_periodic_target_expiration(apic);
1536 if (!start_hv_timer(apic))
1537 start_sw_period(apic);
1538 }
1539}
1540EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer);
1541
ce7a058a
YJ		 1542void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu)
1543{
1544 struct kvm_lapic *apic = vcpu->arch.apic;
1545
1546 WARN_ON(apic->lapic_timer.hv_timer_in_use);
1547
8003c9ae 1548 start_hv_timer(apic);
ce7a058a
YJ		 1549}
1550EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_hv_timer);
1551
1552void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu)
1553{
1554 struct kvm_lapic *apic = vcpu->arch.apic;
1555
1556 /* Possibly the TSC deadline timer is not enabled yet */
1557 if (!apic->lapic_timer.hv_timer_in_use)
1558 return;
1559
7e810a38 1560 cancel_hv_timer(apic);
ce7a058a
YJ		 1561
1562 if (atomic_read(&apic->lapic_timer.pending))
1563 return;
1564
8003c9ae
WL		 1565 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
1566 start_sw_period(apic);
1567 else if (apic_lvtt_tscdeadline(apic))
1568 start_sw_tscdeadline(apic);
ce7a058a
YJ		 1569}
1570EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_sw_timer);
1571
97222cc8
ED		 1572static void start_apic_timer(struct kvm_lapic *apic)
1573{
d3c7b77d 1574 atomic_set(&apic->lapic_timer.pending, 0);
0b975a3c 1575
a3e06bbe 1576 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) {
8003c9ae
WL		 1577 if (set_target_expiration(apic) &&
1578 !(kvm_x86_ops->set_hv_timer && start_hv_timer(apic)))
1579 start_sw_period(apic);
a3e06bbe 1580 } else if (apic_lvtt_tscdeadline(apic)) {
7e810a38 1581 if (!(kvm_x86_ops->set_hv_timer && start_hv_timer(apic)))
ce7a058a 1582 start_sw_tscdeadline(apic);
a3e06bbe 1583 }
97222cc8
ED		 1584}
1585
cc6e462c
JK		 1586static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
1587{
59fd1323 1588 bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val);
cc6e462c 1589
59fd1323
RK		 1590 if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) {
1591 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode;
1592 if (lvt0_in_nmi_mode) {
cc6e462c
JK		 1593 apic_debug("Receive NMI setting on APIC_LVT0 "
1594 "for cpu %d\n", apic->vcpu->vcpu_id);
42720138 1595 atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
59fd1323
RK		 1596 } else
1597 atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
1598 }
cc6e462c
JK		 1599}
1600
1e6e2755 1601int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
97222cc8 1602{
0105d1a5 1603 int ret = 0;
97222cc8 1604
0105d1a5 1605 trace_kvm_apic_write(reg, val);
97222cc8 1606
0105d1a5 1607 switch (reg) {
97222cc8 1608 case APIC_ID: /* Local APIC ID */
0105d1a5 1609 if (!apic_x2apic_mode(apic))
a92e2543 1610 kvm_apic_set_xapic_id(apic, val >> 24);
0105d1a5
GN		 1611 else
1612 ret = 1;
97222cc8
ED		 1613 break;
1614
1615 case APIC_TASKPRI:
b209749f 1616 report_tpr_access(apic, true);
97222cc8
ED		 1617 apic_set_tpr(apic, val & 0xff);
1618 break;
1619
1620 case APIC_EOI:
1621 apic_set_eoi(apic);
1622 break;
1623
1624 case APIC_LDR:
0105d1a5 1625 if (!apic_x2apic_mode(apic))
1e08ec4a 1626 kvm_apic_set_ldr(apic, val & APIC_LDR_MASK);
0105d1a5
GN		 1627 else
1628 ret = 1;
97222cc8
ED		 1629 break;
1630
1631 case APIC_DFR:
1e08ec4a 1632 if (!apic_x2apic_mode(apic)) {
1e6e2755 1633 kvm_lapic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF);
1e08ec4a
GN		 1634 recalculate_apic_map(apic->vcpu->kvm);
1635 } else
0105d1a5 1636 ret = 1;
97222cc8
ED		 1637 break;
1638
fc61b800
GN		 1639 case APIC_SPIV: {
1640 u32 mask = 0x3ff;
dfb95954 1641 if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI)
fc61b800 1642 mask |= APIC_SPIV_DIRECTED_EOI;
f8c1ea10 1643 apic_set_spiv(apic, val & mask);
97222cc8
ED		 1644 if (!(val & APIC_SPIV_APIC_ENABLED)) {
1645 int i;
1646 u32 lvt_val;
1647
1e6e2755 1648 for (i = 0; i < KVM_APIC_LVT_NUM; i++) {
dfb95954 1649 lvt_val = kvm_lapic_get_reg(apic,
97222cc8 1650 APIC_LVTT + 0x10 * i);
1e6e2755 1651 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i,
97222cc8
ED		 1652 lvt_val | APIC_LVT_MASKED);
1653 }
b6ac0695 1654 apic_update_lvtt(apic);
d3c7b77d 1655 atomic_set(&apic->lapic_timer.pending, 0);
97222cc8
ED		 1656
1657 }
1658 break;
fc61b800 1659 }
97222cc8
ED		 1660 case APIC_ICR:
1661 /* No delay here, so we always clear the pending bit */
1e6e2755 1662 kvm_lapic_set_reg(apic, APIC_ICR, val & ~(1 << 12));
97222cc8
ED		 1663 apic_send_ipi(apic);
1664 break;
1665
1666 case APIC_ICR2:
0105d1a5
GN		 1667 if (!apic_x2apic_mode(apic))
1668 val &= 0xff000000;
1e6e2755 1669 kvm_lapic_set_reg(apic, APIC_ICR2, val);
97222cc8
ED		 1670 break;
1671
23930f95 1672 case APIC_LVT0:
cc6e462c 1673 apic_manage_nmi_watchdog(apic, val);
97222cc8
ED		 1674 case APIC_LVTTHMR:
1675 case APIC_LVTPC:
97222cc8
ED		 1676 case APIC_LVT1:
1677 case APIC_LVTERR:
1678 /* TODO: Check vector */
c48f1496 1679 if (!kvm_apic_sw_enabled(apic))
97222cc8
ED		 1680 val |= APIC_LVT_MASKED;
1681
0105d1a5 1682 val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4];
1e6e2755 1683 kvm_lapic_set_reg(apic, reg, val);
97222cc8
ED		 1684
1685 break;
1686
b6ac0695 1687 case APIC_LVTT:
c48f1496 1688 if (!kvm_apic_sw_enabled(apic))
a3e06bbe
LJ		 1689 val |= APIC_LVT_MASKED;
1690 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask);
1e6e2755 1691 kvm_lapic_set_reg(apic, APIC_LVTT, val);
b6ac0695 1692 apic_update_lvtt(apic);
a3e06bbe
LJ		 1693 break;
1694
97222cc8 1695 case APIC_TMICT:
a3e06bbe
LJ		 1696 if (apic_lvtt_tscdeadline(apic))
1697 break;
1698
d3c7b77d 1699 hrtimer_cancel(&apic->lapic_timer.timer);
1e6e2755 1700 kvm_lapic_set_reg(apic, APIC_TMICT, val);
97222cc8 1701 start_apic_timer(apic);
0105d1a5 1702 break;
97222cc8
ED		 1703
1704 case APIC_TDCR:
1705 if (val & 4)
7712de87 1706 apic_debug("KVM_WRITE:TDCR %x\n", val);
1e6e2755 1707 kvm_lapic_set_reg(apic, APIC_TDCR, val);
97222cc8
ED		 1708 update_divide_count(apic);
1709 break;
1710
0105d1a5
GN		 1711 case APIC_ESR:
1712 if (apic_x2apic_mode(apic) && val != 0) {
7712de87 1713 apic_debug("KVM_WRITE:ESR not zero %x\n", val);
0105d1a5
GN		 1714 ret = 1;
1715 }
1716 break;
1717
1718 case APIC_SELF_IPI:
1719 if (apic_x2apic_mode(apic)) {
1e6e2755 1720 kvm_lapic_reg_write(apic, APIC_ICR, 0x40000 | (val & 0xff));
0105d1a5
GN		 1721 } else
1722 ret = 1;
1723 break;
97222cc8 1724 default:
0105d1a5 1725 ret = 1;
97222cc8
ED		 1726 break;
1727 }
0105d1a5
GN		 1728 if (ret)
1729 apic_debug("Local APIC Write to read-only register %x\n", reg);
1730 return ret;
1731}
1e6e2755 1732EXPORT_SYMBOL_GPL(kvm_lapic_reg_write);
0105d1a5 1733
e32edf4f 1734static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
0105d1a5
GN		 1735 gpa_t address, int len, const void *data)
1736{
1737 struct kvm_lapic *apic = to_lapic(this);
1738 unsigned int offset = address - apic->base_address;
1739 u32 val;
1740
1741 if (!apic_mmio_in_range(apic, address))
1742 return -EOPNOTSUPP;
1743
1744 /*
1745 * APIC register must be aligned on 128-bits boundary.
1746 * 32/64/128 bits registers must be accessed thru 32 bits.
1747 * Refer SDM 8.4.1
1748 */
1749 if (len != 4 || (offset & 0xf)) {
1750 /* Don't shout loud, $infamous_os would cause only noise. */
1751 apic_debug("apic write: bad size=%d %lx\n", len, (long)address);
756975bb 1752 return 0;
0105d1a5
GN		 1753 }
1754
1755 val = *(u32*)data;
1756
1757 /* too common printing */
1758 if (offset != APIC_EOI)
1759 apic_debug("%s: offset 0x%x with length 0x%x, and value is "
1760 "0x%x\n", __func__, offset, len, val);
1761
1e6e2755 1762 kvm_lapic_reg_write(apic, offset & 0xff0, val);
0105d1a5 1763
bda9020e 1764 return 0;
97222cc8
ED		 1765}
1766
58fbbf26
KT		 1767void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu)
1768{
1e6e2755 1769 kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0);
58fbbf26
KT		 1770}
1771EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi);
1772
83d4c286
YZ		 1773/* emulate APIC access in a trap manner */
1774void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset)
1775{
1776 u32 val = 0;
1777
1778 /* hw has done the conditional check and inst decode */
1779 offset &= 0xff0;
1780
1e6e2755 1781 kvm_lapic_reg_read(vcpu->arch.apic, offset, 4, &val);
83d4c286
YZ		 1782
1783 /* TODO: optimize to just emulate side effect w/o one more write */
1e6e2755 1784 kvm_lapic_reg_write(vcpu->arch.apic, offset, val);
83d4c286
YZ		 1785}
1786EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode);
1787
d589444e 1788void kvm_free_lapic(struct kvm_vcpu *vcpu)
97222cc8 1789{
f8c1ea10
GN		 1790 struct kvm_lapic *apic = vcpu->arch.apic;
1791
ad312c7c 1792 if (!vcpu->arch.apic)
97222cc8
ED		 1793 return;
1794
f8c1ea10 1795 hrtimer_cancel(&apic->lapic_timer.timer);
97222cc8 1796
c5cc421b
GN		 1797 if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE))
1798 static_key_slow_dec_deferred(&apic_hw_disabled);
1799
e462755c 1800 if (!apic->sw_enabled)
f8c1ea10 1801 static_key_slow_dec_deferred(&apic_sw_disabled);
97222cc8 1802
f8c1ea10
GN		 1803 if (apic->regs)
1804 free_page((unsigned long)apic->regs);
1805
1806 kfree(apic);
97222cc8
ED		 1807}
1808
1809/*
1810 *----------------------------------------------------------------------
1811 * LAPIC interface
1812 *----------------------------------------------------------------------
1813 */
498f8162
WL		 1814u64 kvm_get_lapic_target_expiration_tsc(struct kvm_vcpu *vcpu)
1815{
1816 struct kvm_lapic *apic = vcpu->arch.apic;
1817
1818 if (!lapic_in_kernel(vcpu))
1819 return 0;
1820
1821 return apic->lapic_timer.tscdeadline;
1822}
97222cc8 1823
a3e06bbe
LJ		 1824u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu)
1825{
1826 struct kvm_lapic *apic = vcpu->arch.apic;
a3e06bbe 1827
a10388e1
WL		 1828 if (!lapic_in_kernel(vcpu) ||
1829 !apic_lvtt_tscdeadline(apic))
a3e06bbe
LJ		 1830 return 0;
1831
1832 return apic->lapic_timer.tscdeadline;
1833}
1834
1835void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data)
1836{
1837 struct kvm_lapic *apic = vcpu->arch.apic;
a3e06bbe 1838
bce87cce 1839 if (!lapic_in_kernel(vcpu) || apic_lvtt_oneshot(apic) ||
54e9818f 1840 apic_lvtt_period(apic))
a3e06bbe
LJ		 1841 return;
1842
1843 hrtimer_cancel(&apic->lapic_timer.timer);
1844 apic->lapic_timer.tscdeadline = data;
1845 start_apic_timer(apic);
1846}
1847
97222cc8
ED		 1848void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8)
1849{
ad312c7c 1850 struct kvm_lapic *apic = vcpu->arch.apic;
97222cc8 1851
b93463aa 1852 apic_set_tpr(apic, ((cr8 & 0x0f) << 4)
dfb95954 1853 | (kvm_lapic_get_reg(apic, APIC_TASKPRI) & 4));
97222cc8
ED		 1854}
1855
1856u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
1857{
97222cc8
ED		 1858 u64 tpr;
1859
dfb95954 1860 tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI);
97222cc8
ED		 1861
1862 return (tpr & 0xf0) >> 4;
1863}
1864
1865void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
1866{
8d14695f 1867 u64 old_value = vcpu->arch.apic_base;
ad312c7c 1868 struct kvm_lapic *apic = vcpu->arch.apic;
97222cc8 1869
c7dd15b3 1870 if (!apic)
97222cc8 1871 value |= MSR_IA32_APICBASE_BSP;
c5af89b6 1872
e66d2ae7
JK		 1873 vcpu->arch.apic_base = value;
1874
c7dd15b3
JM		 1875 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE)
1876 kvm_update_cpuid(vcpu);
1877
1878 if (!apic)
1879 return;
1880
c5cc421b 1881 /* update jump label if enable bit changes */
0dce7cd6 1882 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) {
49bd29ba
RK		 1883 if (value & MSR_IA32_APICBASE_ENABLE) {
1884 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
c5cc421b 1885 static_key_slow_dec_deferred(&apic_hw_disabled);
187ca84b 1886 } else {
c5cc421b 1887 static_key_slow_inc(&apic_hw_disabled.key);
187ca84b
WL		 1888 recalculate_apic_map(vcpu->kvm);
1889 }
c5cc421b
GN		 1890 }
1891
8d14695f
YZ		 1892 if ((old_value ^ value) & X2APIC_ENABLE) {
1893 if (value & X2APIC_ENABLE) {
257b9a5f 1894 kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
8d14695f
YZ		 1895 kvm_x86_ops->set_virtual_x2apic_mode(vcpu, true);
1896 } else
1897 kvm_x86_ops->set_virtual_x2apic_mode(vcpu, false);
0105d1a5 1898 }
8d14695f 1899
ad312c7c 1900 apic->base_address = apic->vcpu->arch.apic_base &
97222cc8
ED		 1901 MSR_IA32_APICBASE_BASE;
1902
db324fe6
NA		 1903 if ((value & MSR_IA32_APICBASE_ENABLE) &&
1904 apic->base_address != APIC_DEFAULT_PHYS_BASE)
1905 pr_warn_once("APIC base relocation is unsupported by KVM");
1906
97222cc8
ED		 1907 /* with FSB delivery interrupt, we can restart APIC functionality */
1908 apic_debug("apic base msr is 0x%016" PRIx64 ", and base address is "
ad312c7c 1909 "0x%lx.\n", apic->vcpu->arch.apic_base, apic->base_address);
97222cc8
ED		 1910
1911}
1912
d28bc9dd 1913void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event)
97222cc8
ED		 1914{
1915 struct kvm_lapic *apic;
1916 int i;
1917
b8688d51 1918 apic_debug("%s\n", __func__);
97222cc8
ED		 1919
1920 ASSERT(vcpu);
ad312c7c 1921 apic = vcpu->arch.apic;
97222cc8
ED		 1922 ASSERT(apic != NULL);
1923
1924 /* Stop the timer in case it's a reset to an active apic */
d3c7b77d 1925 hrtimer_cancel(&apic->lapic_timer.timer);
97222cc8 1926
4d8e772b
RK		 1927 if (!init_event) {
1928 kvm_lapic_set_base(vcpu, APIC_DEFAULT_PHYS_BASE |
1929 MSR_IA32_APICBASE_ENABLE);
a92e2543 1930 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
4d8e772b 1931 }
fc61b800 1932 kvm_apic_set_version(apic->vcpu);
97222cc8 1933
1e6e2755
SS		 1934 for (i = 0; i < KVM_APIC_LVT_NUM; i++)
1935 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED);
b6ac0695 1936 apic_update_lvtt(apic);
0da029ed 1937 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED))
1e6e2755 1938 kvm_lapic_set_reg(apic, APIC_LVT0,
90de4a18 1939 SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT));
dfb95954 1940 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
97222cc8 1941
1e6e2755 1942 kvm_lapic_set_reg(apic, APIC_DFR, 0xffffffffU);
f8c1ea10 1943 apic_set_spiv(apic, 0xff);
1e6e2755 1944 kvm_lapic_set_reg(apic, APIC_TASKPRI, 0);
c028dd6b
RK		 1945 if (!apic_x2apic_mode(apic))
1946 kvm_apic_set_ldr(apic, 0);
1e6e2755
SS		 1947 kvm_lapic_set_reg(apic, APIC_ESR, 0);
1948 kvm_lapic_set_reg(apic, APIC_ICR, 0);
1949 kvm_lapic_set_reg(apic, APIC_ICR2, 0);
1950 kvm_lapic_set_reg(apic, APIC_TDCR, 0);
1951 kvm_lapic_set_reg(apic, APIC_TMICT, 0);
97222cc8 1952 for (i = 0; i < 8; i++) {
1e6e2755
SS		 1953 kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0);
1954 kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0);
1955 kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
97222cc8 1956 }
d62caabb
AS		 1957 apic->irr_pending = vcpu->arch.apicv_active;
1958 apic->isr_count = vcpu->arch.apicv_active ? 1 : 0;
8680b94b 1959 apic->highest_isr_cache = -1;
b33ac88b 1960 update_divide_count(apic);
d3c7b77d 1961 atomic_set(&apic->lapic_timer.pending, 0);
c5af89b6 1962 if (kvm_vcpu_is_bsp(vcpu))
5dbc8f3f
GN		 1963 kvm_lapic_set_base(vcpu,
1964 vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP);
ae7a2a3f 1965 vcpu->arch.pv_eoi.msr_val = 0;
97222cc8
ED		 1966 apic_update_ppr(apic);
1967
e1035715 1968 vcpu->arch.apic_arb_prio = 0;
41383771 1969 vcpu->arch.apic_attention = 0;
e1035715 1970
6e500439 1971 apic_debug("%s: vcpu=%p, id=0x%x, base_msr="
b8688d51 1972 "0x%016" PRIx64 ", base_address=0x%0lx.\n", __func__,
6e500439 1973 vcpu, kvm_lapic_get_reg(apic, APIC_ID),
ad312c7c 1974 vcpu->arch.apic_base, apic->base_address);
97222cc8
ED		 1975}
1976
97222cc8
ED		 1977/*
1978 *----------------------------------------------------------------------
1979 * timer interface
1980 *----------------------------------------------------------------------
1981 */
1b9778da 1982
2a6eac96 1983static bool lapic_is_periodic(struct kvm_lapic *apic)
97222cc8 1984{
d3c7b77d 1985 return apic_lvtt_period(apic);
97222cc8
ED		 1986}
1987
3d80840d
MT		 1988int apic_has_pending_timer(struct kvm_vcpu *vcpu)
1989{
54e9818f 1990 struct kvm_lapic *apic = vcpu->arch.apic;
3d80840d 1991
1e3161b4 1992 if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT))
54e9818f 1993 return atomic_read(&apic->lapic_timer.pending);
3d80840d
MT		 1994
1995 return 0;
1996}
1997
89342082 1998int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type)
1b9778da 1999{
dfb95954 2000 u32 reg = kvm_lapic_get_reg(apic, lvt_type);
23930f95 2001 int vector, mode, trig_mode;
23930f95 2002
c48f1496 2003 if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) {
23930f95
JK		 2004 vector = reg & APIC_VECTOR_MASK;
2005 mode = reg & APIC_MODE_MASK;
2006 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER;
b4f2225c
YZ		 2007 return __apic_accept_irq(apic, mode, vector, 1, trig_mode,
2008 NULL);
23930f95
JK		 2009 }
2010 return 0;
2011}
1b9778da 2012
8fdb2351 2013void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu)
23930f95 2014{
8fdb2351
JK		 2015 struct kvm_lapic *apic = vcpu->arch.apic;
2016
2017 if (apic)
2018 kvm_apic_local_deliver(apic, APIC_LVT0);
1b9778da
ED		 2019}
2020
d76685c4
GH		 2021static const struct kvm_io_device_ops apic_mmio_ops = {
2022 .read = apic_mmio_read,
2023 .write = apic_mmio_write,
d76685c4
GH		 2024};
2025
e9d90d47
AK		 2026static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
2027{
2028 struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
2a6eac96 2029 struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer);
e9d90d47 2030
5d87db71 2031 apic_timer_expired(apic);
e9d90d47 2032
2a6eac96 2033 if (lapic_is_periodic(apic)) {
8003c9ae 2034 advance_periodic_target_expiration(apic);
e9d90d47
AK		 2035 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
2036 return HRTIMER_RESTART;
2037 } else
2038 return HRTIMER_NORESTART;
2039}
2040
97222cc8
ED		 2041int kvm_create_lapic(struct kvm_vcpu *vcpu)
2042{
2043 struct kvm_lapic *apic;
2044
2045 ASSERT(vcpu != NULL);
2046 apic_debug("apic_init %d\n", vcpu->vcpu_id);
2047
2048 apic = kzalloc(sizeof(*apic), GFP_KERNEL);
2049 if (!apic)
2050 goto nomem;
2051
ad312c7c 2052 vcpu->arch.apic = apic;
97222cc8 2053
afc20184
TY		 2054 apic->regs = (void *)get_zeroed_page(GFP_KERNEL);
2055 if (!apic->regs) {
97222cc8
ED		 2056 printk(KERN_ERR "malloc apic regs error for vcpu %x\n",
2057 vcpu->vcpu_id);
d589444e 2058 goto nomem_free_apic;
97222cc8 2059 }
97222cc8
ED		 2060 apic->vcpu = vcpu;
2061
d3c7b77d 2062 hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
61abdbe0 2063 HRTIMER_MODE_ABS_PINNED);
e9d90d47 2064 apic->lapic_timer.timer.function = apic_timer_fn;
d3c7b77d 2065
c5cc421b
GN		 2066 /*
2067 * APIC is created enabled. This will prevent kvm_lapic_set_base from
2068 * thinking that APIC satet has changed.
2069 */
2070 vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE;
f8c1ea10 2071 static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */
d28bc9dd 2072 kvm_lapic_reset(vcpu, false);
d76685c4 2073 kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
97222cc8
ED		 2074
2075 return 0;
d589444e
RR		 2076nomem_free_apic:
2077 kfree(apic);
97222cc8 2078nomem:
97222cc8
ED		 2079 return -ENOMEM;
2080}
97222cc8
ED		 2081
2082int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu)
2083{
ad312c7c 2084 struct kvm_lapic *apic = vcpu->arch.apic;
b3c045d3 2085 u32 ppr;
97222cc8 2086
f8543d6a 2087 if (!apic_enabled(apic))
97222cc8
ED		 2088 return -1;
2089
b3c045d3
PB		 2090 __apic_update_ppr(apic, &ppr);
2091 return apic_has_interrupt_for_ppr(apic, ppr);
97222cc8
ED		 2092}
2093
40487c68
QH		 2094int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu)
2095{
dfb95954 2096 u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0);
40487c68
QH		 2097 int r = 0;
2098
c48f1496 2099 if (!kvm_apic_hw_enabled(vcpu->arch.apic))
e7dca5c0
CL		 2100 r = 1;
2101 if ((lvt0 & APIC_LVT_MASKED) == 0 &&
2102 GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT)
2103 r = 1;
40487c68
QH		 2104 return r;
2105}
2106
1b9778da
ED		 2107void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu)
2108{
ad312c7c 2109 struct kvm_lapic *apic = vcpu->arch.apic;
1b9778da 2110
54e9818f 2111 if (atomic_read(&apic->lapic_timer.pending) > 0) {
f1ed0450 2112 kvm_apic_local_deliver(apic, APIC_LVTT);
fae0ba21
NA		 2113 if (apic_lvtt_tscdeadline(apic))
2114 apic->lapic_timer.tscdeadline = 0;
8003c9ae
WL		 2115 if (apic_lvtt_oneshot(apic)) {
2116 apic->lapic_timer.tscdeadline = 0;
8b0e1953 2117 apic->lapic_timer.target_expiration = 0;
8003c9ae 2118 }
f1ed0450 2119 atomic_set(&apic->lapic_timer.pending, 0);
1b9778da
ED		 2120 }
2121}
2122
97222cc8
ED		 2123int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu)
2124{
2125 int vector = kvm_apic_has_interrupt(vcpu);
ad312c7c 2126 struct kvm_lapic *apic = vcpu->arch.apic;
4d82d12b 2127 u32 ppr;
97222cc8
ED		 2128
2129 if (vector == -1)
2130 return -1;
2131
56cc2406
WL		 2132 /*
2133 * We get here even with APIC virtualization enabled, if doing
2134 * nested virtualization and L1 runs with the "acknowledge interrupt
2135 * on exit" mode. Then we cannot inject the interrupt via RVI,
2136 * because the process would deliver it through the IDT.
2137 */
2138
97222cc8 2139 apic_clear_irr(vector, apic);
5c919412 2140 if (test_bit(vector, vcpu_to_synic(vcpu)->auto_eoi_bitmap)) {
4d82d12b
PB		 2141 /*
2142 * For auto-EOI interrupts, there might be another pending
2143 * interrupt above PPR, so check whether to raise another
2144 * KVM_REQ_EVENT.
2145 */
5c919412 2146 apic_update_ppr(apic);
4d82d12b
PB		 2147 } else {
2148 /*
2149 * For normal interrupts, PPR has been raised and there cannot
2150 * be a higher-priority pending interrupt---except if there was
2151 * a concurrent interrupt injection, but that would have
2152 * triggered KVM_REQ_EVENT already.
2153 */
2154 apic_set_isr(vector, apic);
2155 __apic_update_ppr(apic, &ppr);
5c919412
AS		 2156 }
2157
97222cc8
ED		 2158 return vector;
2159}
96ad2cc6 2160
a92e2543
RK		 2161static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu,
2162 struct kvm_lapic_state *s, bool set)
2163{
2164 if (apic_x2apic_mode(vcpu->arch.apic)) {
2165 u32 *id = (u32 *)(s->regs + APIC_ID);
2166
37131313
RK		 2167 if (vcpu->kvm->arch.x2apic_format) {
2168 if (*id != vcpu->vcpu_id)
2169 return -EINVAL;
2170 } else {
2171 if (set)
2172 *id >>= 24;
2173 else
2174 *id <<= 24;
2175 }
a92e2543
RK		 2176 }
2177
2178 return 0;
2179}
2180
2181int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
2182{
2183 memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s));
2184 return kvm_apic_state_fixup(vcpu, s, false);
2185}
2186
2187int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
96ad2cc6 2188{
ad312c7c 2189 struct kvm_lapic *apic = vcpu->arch.apic;
a92e2543
RK		 2190 int r;
2191
96ad2cc6 2192
5dbc8f3f 2193 kvm_lapic_set_base(vcpu, vcpu->arch.apic_base);
64eb0620
GN		 2194 /* set SPIV separately to get count of SW disabled APICs right */
2195 apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV)));
a92e2543
RK		 2196
2197 r = kvm_apic_state_fixup(vcpu, s, true);
2198 if (r)
2199 return r;
64eb0620 2200 memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s);
a92e2543
RK		 2201
2202 recalculate_apic_map(vcpu->kvm);
fc61b800
GN		 2203 kvm_apic_set_version(vcpu);
2204
96ad2cc6 2205 apic_update_ppr(apic);
d3c7b77d 2206 hrtimer_cancel(&apic->lapic_timer.timer);
b6ac0695 2207 apic_update_lvtt(apic);
dfb95954 2208 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
96ad2cc6
ED		 2209 update_divide_count(apic);
2210 start_apic_timer(apic);
6e24a6ef 2211 apic->irr_pending = true;
d62caabb 2212 apic->isr_count = vcpu->arch.apicv_active ?
c7c9c56c 2213 1 : count_vectors(apic->regs + APIC_ISR);
8680b94b 2214 apic->highest_isr_cache = -1;
d62caabb 2215 if (vcpu->arch.apicv_active) {
967235d3 2216 kvm_x86_ops->apicv_post_state_restore(vcpu);
4114c27d
WW		 2217 kvm_x86_ops->hwapic_irr_update(vcpu,
2218 apic_find_highest_irr(apic));
67c9dddc 2219 kvm_x86_ops->hwapic_isr_update(vcpu,
b4eef9b3 2220 apic_find_highest_isr(apic));
d62caabb 2221 }
3842d135 2222 kvm_make_request(KVM_REQ_EVENT, vcpu);
49df6397
SR		 2223 if (ioapic_in_kernel(vcpu->kvm))
2224 kvm_rtc_eoi_tracking_restore_one(vcpu);
0669a510
RK		 2225
2226 vcpu->arch.apic_arb_prio = 0;
a92e2543
RK		 2227
2228 return 0;
96ad2cc6 2229}
a3d7f85f 2230
2f52d58c 2231void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
a3d7f85f 2232{
a3d7f85f
ED		 2233 struct hrtimer *timer;
2234
bce87cce 2235 if (!lapic_in_kernel(vcpu))
a3d7f85f
ED		 2236 return;
2237
54e9818f 2238 timer = &vcpu->arch.apic->lapic_timer.timer;
a3d7f85f 2239 if (hrtimer_cancel(timer))
61abdbe0 2240 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED);
a3d7f85f 2241}
b93463aa 2242
ae7a2a3f
MT		 2243/*
2244 * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt
2245 *
2246 * Detect whether guest triggered PV EOI since the
2247 * last entry. If yes, set EOI on guests's behalf.
2248 * Clear PV EOI in guest memory in any case.
2249 */
2250static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu,
2251 struct kvm_lapic *apic)
2252{
2253 bool pending;
2254 int vector;
2255 /*
2256 * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host
2257 * and KVM_PV_EOI_ENABLED in guest memory as follows:
2258 *
2259 * KVM_APIC_PV_EOI_PENDING is unset:
2260 * -> host disabled PV EOI.
2261 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set:
2262 * -> host enabled PV EOI, guest did not execute EOI yet.
2263 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset:
2264 * -> host enabled PV EOI, guest executed EOI.
2265 */
2266 BUG_ON(!pv_eoi_enabled(vcpu));
2267 pending = pv_eoi_get_pending(vcpu);
2268 /*
2269 * Clear pending bit in any case: it will be set again on vmentry.
2270 * While this might not be ideal from performance point of view,
2271 * this makes sure pv eoi is only enabled when we know it's safe.
2272 */
2273 pv_eoi_clr_pending(vcpu);
2274 if (pending)
2275 return;
2276 vector = apic_set_eoi(apic);
2277 trace_kvm_pv_eoi(apic, vector);
2278}
2279
b93463aa
AK		 2280void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu)
2281{
2282 u32 data;
b93463aa 2283
ae7a2a3f
MT		 2284 if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention))
2285 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic);
2286
41383771 2287 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
b93463aa
AK		 2288 return;
2289
4e335d9e
PB		 2290 if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
2291 sizeof(u32)))
603242a8 2292 return;
b93463aa
AK		 2293
2294 apic_set_tpr(vcpu->arch.apic, data & 0xff);
2295}
2296
ae7a2a3f
MT		 2297/*
2298 * apic_sync_pv_eoi_to_guest - called before vmentry
2299 *
2300 * Detect whether it's safe to enable PV EOI and
2301 * if yes do so.
2302 */
2303static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu,
2304 struct kvm_lapic *apic)
2305{
2306 if (!pv_eoi_enabled(vcpu) ||
2307 /* IRR set or many bits in ISR: could be nested. */
2308 apic->irr_pending ||
2309 /* Cache not set: could be safe but we don't bother. */
2310 apic->highest_isr_cache == -1 ||
2311 /* Need EOI to update ioapic. */
3bb345f3 2312 kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) {
ae7a2a3f
MT		 2313 /*
2314 * PV EOI was disabled by apic_sync_pv_eoi_from_guest
2315 * so we need not do anything here.
2316 */
2317 return;
2318 }
2319
2320 pv_eoi_set_pending(apic->vcpu);
2321}
2322
b93463aa
AK		 2323void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
2324{
2325 u32 data, tpr;
2326 int max_irr, max_isr;
ae7a2a3f 2327 struct kvm_lapic *apic = vcpu->arch.apic;
b93463aa 2328
ae7a2a3f
MT		 2329 apic_sync_pv_eoi_to_guest(vcpu, apic);
2330
41383771 2331 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
b93463aa
AK		 2332 return;
2333
dfb95954 2334 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff;
b93463aa
AK		 2335 max_irr = apic_find_highest_irr(apic);
2336 if (max_irr < 0)
2337 max_irr = 0;
2338 max_isr = apic_find_highest_isr(apic);
2339 if (max_isr < 0)
2340 max_isr = 0;
2341 data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24);
2342
4e335d9e
PB		 2343 kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
2344 sizeof(u32));
b93463aa
AK		 2345}
2346
fda4e2e8 2347int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr)
b93463aa 2348{
fda4e2e8 2349 if (vapic_addr) {
4e335d9e 2350 if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
fda4e2e8
AH		 2351 &vcpu->arch.apic->vapic_cache,
2352 vapic_addr, sizeof(u32)))
2353 return -EINVAL;
41383771 2354 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
fda4e2e8 2355 } else {
41383771 2356 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
fda4e2e8
AH		 2357 }
2358
2359 vcpu->arch.apic->vapic_addr = vapic_addr;
2360 return 0;
b93463aa 2361}
0105d1a5
GN		 2362
2363int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data)
2364{
2365 struct kvm_lapic *apic = vcpu->arch.apic;
2366 u32 reg = (msr - APIC_BASE_MSR) << 4;
2367
35754c98 2368 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
0105d1a5
GN		 2369 return 1;
2370
c69d3d9b
NA		 2371 if (reg == APIC_ICR2)
2372 return 1;
2373
0105d1a5 2374 /* if this is ICR write vector before command */
decdc283 2375 if (reg == APIC_ICR)
1e6e2755
SS		 2376 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
2377 return kvm_lapic_reg_write(apic, reg, (u32)data);
0105d1a5
GN		 2378}
2379
2380int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data)
2381{
2382 struct kvm_lapic *apic = vcpu->arch.apic;
2383 u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0;
2384
35754c98 2385 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
0105d1a5
GN		 2386 return 1;
2387
c69d3d9b
NA		 2388 if (reg == APIC_DFR || reg == APIC_ICR2) {
2389 apic_debug("KVM_APIC_READ: read x2apic reserved register %x\n",
2390 reg);
2391 return 1;
2392 }
2393
1e6e2755 2394 if (kvm_lapic_reg_read(apic, reg, 4, &low))
0105d1a5 2395 return 1;
decdc283 2396 if (reg == APIC_ICR)
1e6e2755 2397 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
0105d1a5
GN		 2398
2399 *data = (((u64)high) << 32) | low;
2400
2401 return 0;
2402}
10388a07
GN		 2403
2404int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data)
2405{
2406 struct kvm_lapic *apic = vcpu->arch.apic;
2407
bce87cce 2408 if (!lapic_in_kernel(vcpu))
10388a07
GN		 2409 return 1;
2410
2411 /* if this is ICR write vector before command */
2412 if (reg == APIC_ICR)
1e6e2755
SS		 2413 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
2414 return kvm_lapic_reg_write(apic, reg, (u32)data);
10388a07
GN		 2415}
2416
2417int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data)
2418{
2419 struct kvm_lapic *apic = vcpu->arch.apic;
2420 u32 low, high = 0;
2421
bce87cce 2422 if (!lapic_in_kernel(vcpu))
10388a07
GN		 2423 return 1;
2424
1e6e2755 2425 if (kvm_lapic_reg_read(apic, reg, 4, &low))
10388a07
GN		 2426 return 1;
2427 if (reg == APIC_ICR)
1e6e2755 2428 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
10388a07
GN		 2429
2430 *data = (((u64)high) << 32) | low;
2431
2432 return 0;
2433}
ae7a2a3f
MT		 2434
2435int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data)
2436{
2437 u64 addr = data & ~KVM_MSR_ENABLED;
2438 if (!IS_ALIGNED(addr, 4))
2439 return 1;
2440
2441 vcpu->arch.pv_eoi.msr_val = data;
2442 if (!pv_eoi_enabled(vcpu))
2443 return 0;
4e335d9e 2444 return kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.pv_eoi.data,
8f964525 2445 addr, sizeof(u8));
ae7a2a3f 2446}
c5cc421b 2447
66450a21
JK		 2448void kvm_apic_accept_events(struct kvm_vcpu *vcpu)
2449{
2450 struct kvm_lapic *apic = vcpu->arch.apic;
2b4a273b 2451 u8 sipi_vector;
299018f4 2452 unsigned long pe;
66450a21 2453
bce87cce 2454 if (!lapic_in_kernel(vcpu) || !apic->pending_events)
66450a21
JK		 2455 return;
2456
cd7764fe
PB		 2457 /*
2458 * INITs are latched while in SMM. Because an SMM CPU cannot
2459 * be in KVM_MP_STATE_INIT_RECEIVED state, just eat SIPIs
2460 * and delay processing of INIT until the next RSM.
2461 */
2462 if (is_smm(vcpu)) {
2463 WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED);
2464 if (test_bit(KVM_APIC_SIPI, &apic->pending_events))
2465 clear_bit(KVM_APIC_SIPI, &apic->pending_events);
2466 return;
2467 }
299018f4 2468
cd7764fe 2469 pe = xchg(&apic->pending_events, 0);
299018f4 2470 if (test_bit(KVM_APIC_INIT, &pe)) {
d28bc9dd
NA		 2471 kvm_lapic_reset(vcpu, true);
2472 kvm_vcpu_reset(vcpu, true);
66450a21
JK		 2473 if (kvm_vcpu_is_bsp(apic->vcpu))
2474 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
2475 else
2476 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
2477 }
299018f4 2478 if (test_bit(KVM_APIC_SIPI, &pe) &&
66450a21
JK		 2479 vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
2480 /* evaluate pending_events before reading the vector */
2481 smp_rmb();
2482 sipi_vector = apic->sipi_vector;
98eff52a 2483 apic_debug("vcpu %d received sipi with vector # %x\n",
66450a21
JK		 2484 vcpu->vcpu_id, sipi_vector);
2485 kvm_vcpu_deliver_sipi_vector(vcpu, sipi_vector);
2486 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
2487 }
2488}
2489
c5cc421b
GN		 2490void kvm_lapic_init(void)
2491{
2492 /* do not patch jump label more than once per second */
2493 jump_label_rate_limit(&apic_hw_disabled, HZ);
f8c1ea10 2494 jump_label_rate_limit(&apic_sw_disabled, HZ);
c5cc421b 2495}
cef84c30
DM		 2496
2497void kvm_lapic_exit(void)
2498{
2499 static_key_deferred_flush(&apic_hw_disabled);
2500 static_key_deferred_flush(&apic_sw_disabled);
2501}
ubuntu-bionic-kernel mirror