]> git.proxmox.com Git - mirror_ubuntu-focal-kernel.git/blame - arch/x86/kvm/lapic.c
projects / mirror_ubuntu-focal-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: x86: add VCPU stat for KVM_REQ_EVENT processing
[mirror_ubuntu-focal-kernel.git] / arch / x86 / kvm / lapic.c
CommitLineData
97222cc8
ED		 1
2/*
3 * Local APIC virtualization
4 *
5 * Copyright (C) 2006 Qumranet, Inc.
6 * Copyright (C) 2007 Novell
7 * Copyright (C) 2007 Intel
9611c187 8 * Copyright 2009 Red Hat, Inc. and/or its affiliates.
97222cc8
ED		 9 *
10 * Authors:
11 * Dor Laor <dor.laor@qumranet.com>
12 * Gregory Haskins <ghaskins@novell.com>
13 * Yaozu (Eddie) Dong <eddie.dong@intel.com>
14 *
15 * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation.
16 *
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
19 */
20
edf88417 21#include <linux/kvm_host.h>
97222cc8
ED		 22#include <linux/kvm.h>
23#include <linux/mm.h>
24#include <linux/highmem.h>
25#include <linux/smp.h>
26#include <linux/hrtimer.h>
27#include <linux/io.h>
1767e931 28#include <linux/export.h>
6f6d6a1a 29#include <linux/math64.h>
5a0e3ad6 30#include <linux/slab.h>
97222cc8
ED		 31#include <asm/processor.h>
32#include <asm/msr.h>
33#include <asm/page.h>
34#include <asm/current.h>
35#include <asm/apicdef.h>
d0659d94 36#include <asm/delay.h>
60063497 37#include <linux/atomic.h>
c5cc421b 38#include <linux/jump_label.h>
5fdbf976 39#include "kvm_cache_regs.h"
97222cc8 40#include "irq.h"
229456fc 41#include "trace.h"
fc61b800 42#include "x86.h"
00b27a3e 43#include "cpuid.h"
5c919412 44#include "hyperv.h"
97222cc8 45
b682b814
MT		 46#ifndef CONFIG_X86_64
47#define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
48#else
49#define mod_64(x, y) ((x) % (y))
50#endif
51
97222cc8
ED		 52#define PRId64 "d"
53#define PRIx64 "llx"
54#define PRIu64 "u"
55#define PRIo64 "o"
56
57#define APIC_BUS_CYCLE_NS 1
58
59/* #define apic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) */
60#define apic_debug(fmt, arg...)
61
97222cc8 62/* 14 is the version for Xeon and Pentium 8.4.8*/
1e6e2755 63#define APIC_VERSION (0x14UL | ((KVM_APIC_LVT_NUM - 1) << 16))
97222cc8
ED		 64#define LAPIC_MMIO_LENGTH (1 << 12)
65/* followed define is not in apicdef.h */
66#define APIC_SHORT_MASK 0xc0000
67#define APIC_DEST_NOSHORT 0x0
68#define APIC_DEST_MASK 0x800
69#define MAX_APIC_VECTOR 256
ecba9a52 70#define APIC_VECTORS_PER_REG 32
97222cc8 71
394457a9
NA		 72#define APIC_BROADCAST 0xFF
73#define X2APIC_BROADCAST 0xFFFFFFFFul
74
a0c9a822
MT		 75static inline int apic_test_vector(int vec, void *bitmap)
76{
77 return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
78}
79
10606919
YZ		 80bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector)
81{
82 struct kvm_lapic *apic = vcpu->arch.apic;
83
84 return apic_test_vector(vector, apic->regs + APIC_ISR) ||
85 apic_test_vector(vector, apic->regs + APIC_IRR);
86}
87
97222cc8
ED		 88static inline void apic_clear_vector(int vec, void *bitmap)
89{
90 clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
91}
92
8680b94b
MT		 93static inline int __apic_test_and_set_vector(int vec, void *bitmap)
94{
95 return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
96}
97
98static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
99{
100 return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
101}
102
c5cc421b 103struct static_key_deferred apic_hw_disabled __read_mostly;
f8c1ea10
GN		 104struct static_key_deferred apic_sw_disabled __read_mostly;
105
97222cc8
ED		 106static inline int apic_enabled(struct kvm_lapic *apic)
107{
c48f1496 108 return kvm_apic_sw_enabled(apic) && kvm_apic_hw_enabled(apic);
54e9818f
GN		 109}
110
97222cc8
ED		 111#define LVT_MASK \
112 (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK)
113
114#define LINT_MASK \
115 (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
116 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)
117
6e500439
RK		 118static inline u8 kvm_xapic_id(struct kvm_lapic *apic)
119{
120 return kvm_lapic_get_reg(apic, APIC_ID) >> 24;
121}
122
123static inline u32 kvm_x2apic_id(struct kvm_lapic *apic)
124{
125 return apic->vcpu->vcpu_id;
126}
127
e45115b6
RK		 128static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map,
129 u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) {
130 switch (map->mode) {
131 case KVM_APIC_MODE_X2APIC: {
132 u32 offset = (dest_id >> 16) * 16;
0ca52e7b 133 u32 max_apic_id = map->max_apic_id;
e45115b6
RK		 134
135 if (offset <= max_apic_id) {
136 u8 cluster_size = min(max_apic_id - offset + 1, 16U);
137
138 *cluster = &map->phys_map[offset];
139 *mask = dest_id & (0xffff >> (16 - cluster_size));
140 } else {
141 *mask = 0;
142 }
3b5a5ffa 143
e45115b6
RK		 144 return true;
145 }
146 case KVM_APIC_MODE_XAPIC_FLAT:
147 *cluster = map->xapic_flat_map;
148 *mask = dest_id & 0xff;
149 return true;
150 case KVM_APIC_MODE_XAPIC_CLUSTER:
444fdad8 151 *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf];
e45115b6
RK		 152 *mask = dest_id & 0xf;
153 return true;
154 default:
155 /* Not optimized. */
156 return false;
157 }
3548a259
RK		 158}
159
af1bae54 160static void kvm_apic_map_free(struct rcu_head *rcu)
3b5a5ffa 161{
af1bae54 162 struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu);
3b5a5ffa 163
af1bae54 164 kvfree(map);
3b5a5ffa
RK		 165}
166
1e08ec4a
GN		 167static void recalculate_apic_map(struct kvm *kvm)
168{
169 struct kvm_apic_map *new, *old = NULL;
170 struct kvm_vcpu *vcpu;
171 int i;
6e500439 172 u32 max_id = 255; /* enough space for any xAPIC ID */
1e08ec4a
GN		 173
174 mutex_lock(&kvm->arch.apic_map_lock);
175
0ca52e7b
RK		 176 kvm_for_each_vcpu(i, vcpu, kvm)
177 if (kvm_apic_present(vcpu))
6e500439 178 max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic));
0ca52e7b 179
af1bae54
RK		 180 new = kvm_kvzalloc(sizeof(struct kvm_apic_map) +
181 sizeof(struct kvm_lapic *) * ((u64)max_id + 1));
0ca52e7b 182
1e08ec4a
GN		 183 if (!new)
184 goto out;
185
0ca52e7b
RK		 186 new->max_apic_id = max_id;
187
173beedc
NA		 188 kvm_for_each_vcpu(i, vcpu, kvm) {
189 struct kvm_lapic *apic = vcpu->arch.apic;
e45115b6
RK		 190 struct kvm_lapic **cluster;
191 u16 mask;
5bd5db38
RK		 192 u32 ldr;
193 u8 xapic_id;
194 u32 x2apic_id;
1e08ec4a 195
df04d1d1
RK		 196 if (!kvm_apic_present(vcpu))
197 continue;
198
5bd5db38
RK		 199 xapic_id = kvm_xapic_id(apic);
200 x2apic_id = kvm_x2apic_id(apic);
201
202 /* Hotplug hack: see kvm_apic_match_physical_addr(), ... */
203 if ((apic_x2apic_mode(apic) || x2apic_id > 0xff) &&
204 x2apic_id <= new->max_apic_id)
205 new->phys_map[x2apic_id] = apic;
206 /*
207 * ... xAPIC ID of VCPUs with APIC ID > 0xff will wrap-around,
208 * prevent them from masking VCPUs with APIC ID <= 0xff.
209 */
210 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id])
211 new->phys_map[xapic_id] = apic;
3548a259 212
6e500439
RK		 213 ldr = kvm_lapic_get_reg(apic, APIC_LDR);
214
3b5a5ffa
RK		 215 if (apic_x2apic_mode(apic)) {
216 new->mode |= KVM_APIC_MODE_X2APIC;
217 } else if (ldr) {
218 ldr = GET_APIC_LOGICAL_ID(ldr);
dfb95954 219 if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT)
3b5a5ffa
RK		 220 new->mode |= KVM_APIC_MODE_XAPIC_FLAT;
221 else
222 new->mode |= KVM_APIC_MODE_XAPIC_CLUSTER;
223 }
224
e45115b6 225 if (!kvm_apic_map_get_logical_dest(new, ldr, &cluster, &mask))
3548a259
RK		 226 continue;
227
e45115b6
RK		 228 if (mask)
229 cluster[ffs(mask) - 1] = apic;
1e08ec4a
GN		 230 }
231out:
232 old = rcu_dereference_protected(kvm->arch.apic_map,
233 lockdep_is_held(&kvm->arch.apic_map_lock));
234 rcu_assign_pointer(kvm->arch.apic_map, new);
235 mutex_unlock(&kvm->arch.apic_map_lock);
236
237 if (old)
af1bae54 238 call_rcu(&old->rcu, kvm_apic_map_free);
c7c9c56c 239
b053b2ae 240 kvm_make_scan_ioapic_request(kvm);
1e08ec4a
GN		 241}
242
1e1b6c26
NA		 243static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
244{
e462755c 245 bool enabled = val & APIC_SPIV_APIC_ENABLED;
1e1b6c26 246
1e6e2755 247 kvm_lapic_set_reg(apic, APIC_SPIV, val);
e462755c
RK		 248
249 if (enabled != apic->sw_enabled) {
250 apic->sw_enabled = enabled;
251 if (enabled) {
1e1b6c26
NA		 252 static_key_slow_dec_deferred(&apic_sw_disabled);
253 recalculate_apic_map(apic->vcpu->kvm);
254 } else
255 static_key_slow_inc(&apic_sw_disabled.key);
256 }
257}
258
a92e2543 259static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id)
1e08ec4a 260{
1e6e2755 261 kvm_lapic_set_reg(apic, APIC_ID, id << 24);
1e08ec4a
GN		 262 recalculate_apic_map(apic->vcpu->kvm);
263}
264
265static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id)
266{
1e6e2755 267 kvm_lapic_set_reg(apic, APIC_LDR, id);
1e08ec4a
GN		 268 recalculate_apic_map(apic->vcpu->kvm);
269}
270
a92e2543 271static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id)
257b9a5f
RK		 272{
273 u32 ldr = ((id >> 4) << 16) | (1 << (id & 0xf));
274
6e500439
RK		 275 WARN_ON_ONCE(id != apic->vcpu->vcpu_id);
276
a92e2543 277 kvm_lapic_set_reg(apic, APIC_ID, id);
1e6e2755 278 kvm_lapic_set_reg(apic, APIC_LDR, ldr);
257b9a5f
RK		 279 recalculate_apic_map(apic->vcpu->kvm);
280}
281
97222cc8
ED		 282static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type)
283{
dfb95954 284 return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED);
97222cc8
ED		 285}
286
287static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type)
288{
dfb95954 289 return kvm_lapic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK;
97222cc8
ED		 290}
291
a3e06bbe
LJ		 292static inline int apic_lvtt_oneshot(struct kvm_lapic *apic)
293{
f30ebc31 294 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT;
a3e06bbe
LJ		 295}
296
97222cc8
ED		 297static inline int apic_lvtt_period(struct kvm_lapic *apic)
298{
f30ebc31 299 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC;
a3e06bbe
LJ		 300}
301
302static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic)
303{
f30ebc31 304 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE;
97222cc8
ED		 305}
306
cc6e462c
JK		 307static inline int apic_lvt_nmi_mode(u32 lvt_val)
308{
309 return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI;
310}
311
fc61b800
GN		 312void kvm_apic_set_version(struct kvm_vcpu *vcpu)
313{
314 struct kvm_lapic *apic = vcpu->arch.apic;
315 struct kvm_cpuid_entry2 *feat;
316 u32 v = APIC_VERSION;
317
bce87cce 318 if (!lapic_in_kernel(vcpu))
fc61b800
GN		 319 return;
320
321 feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0);
322 if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31))))
323 v |= APIC_LVR_DIRECTED_EOI;
1e6e2755 324 kvm_lapic_set_reg(apic, APIC_LVR, v);
fc61b800
GN		 325}
326
1e6e2755 327static const unsigned int apic_lvt_mask[KVM_APIC_LVT_NUM] = {
a3e06bbe 328 LVT_MASK , /* part LVTT mask, timer mode mask added at runtime */
97222cc8
ED		 329 LVT_MASK | APIC_MODE_MASK, /* LVTTHMR */
330 LVT_MASK | APIC_MODE_MASK, /* LVTPC */
331 LINT_MASK, LINT_MASK, /* LVT0-1 */
332 LVT_MASK /* LVTERR */
333};
334
335static int find_highest_vector(void *bitmap)
336{
ecba9a52
TY		 337 int vec;
338 u32 *reg;
97222cc8 339
ecba9a52
TY		 340 for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG;
341 vec >= 0; vec -= APIC_VECTORS_PER_REG) {
342 reg = bitmap + REG_POS(vec);
343 if (*reg)
344 return fls(*reg) - 1 + vec;
345 }
97222cc8 346
ecba9a52 347 return -1;
97222cc8
ED		 348}
349
8680b94b
MT		 350static u8 count_vectors(void *bitmap)
351{
ecba9a52
TY		 352 int vec;
353 u32 *reg;
8680b94b 354 u8 count = 0;
ecba9a52
TY		 355
356 for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) {
357 reg = bitmap + REG_POS(vec);
358 count += hweight32(*reg);
359 }
360
8680b94b
MT		 361 return count;
362}
363
705699a1 364void __kvm_apic_update_irr(u32 *pir, void *regs)
a20ed54d
YZ		 365{
366 u32 i, pir_val;
a20ed54d
YZ		 367
368 for (i = 0; i <= 7; i++) {
ad361091
PB		 369 pir_val = READ_ONCE(pir[i]);
370 if (pir_val) {
371 pir_val = xchg(&pir[i], 0);
705699a1 372 *((u32 *)(regs + APIC_IRR + i * 0x10)) |= pir_val;
ad361091 373 }
a20ed54d
YZ		 374 }
375}
705699a1
WV		 376EXPORT_SYMBOL_GPL(__kvm_apic_update_irr);
377
378void kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir)
379{
380 struct kvm_lapic *apic = vcpu->arch.apic;
381
382 __kvm_apic_update_irr(pir, apic->regs);
c77f3fab
RK		 383
384 kvm_make_request(KVM_REQ_EVENT, vcpu);
705699a1 385}
a20ed54d
YZ		 386EXPORT_SYMBOL_GPL(kvm_apic_update_irr);
387
33e4c686 388static inline int apic_search_irr(struct kvm_lapic *apic)
97222cc8 389{
33e4c686 390 return find_highest_vector(apic->regs + APIC_IRR);
97222cc8
ED		 391}
392
393static inline int apic_find_highest_irr(struct kvm_lapic *apic)
394{
395 int result;
396
c7c9c56c
YZ		 397 /*
398 * Note that irr_pending is just a hint. It will be always
399 * true with virtual interrupt delivery enabled.
400 */
33e4c686
GN		 401 if (!apic->irr_pending)
402 return -1;
403
d62caabb
AS		 404 if (apic->vcpu->arch.apicv_active)
405 kvm_x86_ops->sync_pir_to_irr(apic->vcpu);
33e4c686 406 result = apic_search_irr(apic);
97222cc8
ED		 407 ASSERT(result == -1 || result >= 16);
408
409 return result;
410}
411
33e4c686
GN		 412static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
413{
56cc2406
WL		 414 struct kvm_vcpu *vcpu;
415
416 vcpu = apic->vcpu;
417
d62caabb 418 if (unlikely(vcpu->arch.apicv_active)) {
56cc2406 419 /* try to update RVI */
f210f757 420 apic_clear_vector(vec, apic->regs + APIC_IRR);
56cc2406 421 kvm_make_request(KVM_REQ_EVENT, vcpu);
f210f757
NA		 422 } else {
423 apic->irr_pending = false;
424 apic_clear_vector(vec, apic->regs + APIC_IRR);
425 if (apic_search_irr(apic) != -1)
426 apic->irr_pending = true;
56cc2406 427 }
33e4c686
GN		 428}
429
8680b94b
MT		 430static inline void apic_set_isr(int vec, struct kvm_lapic *apic)
431{
56cc2406
WL		 432 struct kvm_vcpu *vcpu;
433
434 if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR))
435 return;
436
437 vcpu = apic->vcpu;
fc57ac2c 438
8680b94b 439 /*
56cc2406
WL		 440 * With APIC virtualization enabled, all caching is disabled
441 * because the processor can modify ISR under the hood. Instead
442 * just set SVI.
8680b94b 443 */
d62caabb 444 if (unlikely(vcpu->arch.apicv_active))
67c9dddc 445 kvm_x86_ops->hwapic_isr_update(vcpu, vec);
56cc2406
WL		 446 else {
447 ++apic->isr_count;
448 BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
449 /*
450 * ISR (in service register) bit is set when injecting an interrupt.
451 * The highest vector is injected. Thus the latest bit set matches
452 * the highest bit in ISR.
453 */
454 apic->highest_isr_cache = vec;
455 }
8680b94b
MT		 456}
457
fc57ac2c
PB		 458static inline int apic_find_highest_isr(struct kvm_lapic *apic)
459{
460 int result;
461
462 /*
463 * Note that isr_count is always 1, and highest_isr_cache
464 * is always -1, with APIC virtualization enabled.
465 */
466 if (!apic->isr_count)
467 return -1;
468 if (likely(apic->highest_isr_cache != -1))
469 return apic->highest_isr_cache;
470
471 result = find_highest_vector(apic->regs + APIC_ISR);
472 ASSERT(result == -1 || result >= 16);
473
474 return result;
475}
476
8680b94b
MT		 477static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
478{
fc57ac2c
PB		 479 struct kvm_vcpu *vcpu;
480 if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR))
481 return;
482
483 vcpu = apic->vcpu;
484
485 /*
486 * We do get here for APIC virtualization enabled if the guest
487 * uses the Hyper-V APIC enlightenment. In this case we may need
488 * to trigger a new interrupt delivery by writing the SVI field;
489 * on the other hand isr_count and highest_isr_cache are unused
490 * and must be left alone.
491 */
d62caabb 492 if (unlikely(vcpu->arch.apicv_active))
67c9dddc 493 kvm_x86_ops->hwapic_isr_update(vcpu,
fc57ac2c
PB		 494 apic_find_highest_isr(apic));
495 else {
8680b94b 496 --apic->isr_count;
fc57ac2c
PB		 497 BUG_ON(apic->isr_count < 0);
498 apic->highest_isr_cache = -1;
499 }
8680b94b
MT		 500}
501
6e5d865c
YS		 502int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
503{
33e4c686
GN		 504 /* This may race with setting of irr in __apic_accept_irq() and
505 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq
506 * will cause vmexit immediately and the value will be recalculated
507 * on the next vmentry.
508 */
f8543d6a 509 return apic_find_highest_irr(vcpu->arch.apic);
6e5d865c 510}
6e5d865c 511
6da7e3f6 512static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
b4f2225c 513 int vector, int level, int trig_mode,
9e4aabe2 514 struct dest_map *dest_map);
6da7e3f6 515
b4f2225c 516int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq,
9e4aabe2 517 struct dest_map *dest_map)
97222cc8 518{
ad312c7c 519 struct kvm_lapic *apic = vcpu->arch.apic;
8be5453f 520
58c2dde1 521 return __apic_accept_irq(apic, irq->delivery_mode, irq->vector,
b4f2225c 522 irq->level, irq->trig_mode, dest_map);
97222cc8
ED		 523}
524
ae7a2a3f
MT		 525static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val)
526{
527
528 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val,
529 sizeof(val));
530}
531
532static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val)
533{
534
535 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val,
536 sizeof(*val));
537}
538
539static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu)
540{
541 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;
542}
543
544static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu)
545{
546 u8 val;
547 if (pv_eoi_get_user(vcpu, &val) < 0)
548 apic_debug("Can't read EOI MSR value: 0x%llx\n",
96893977 549 (unsigned long long)vcpu->arch.pv_eoi.msr_val);
ae7a2a3f
MT		 550 return val & 0x1;
551}
552
553static void pv_eoi_set_pending(struct kvm_vcpu *vcpu)
554{
555 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) {
556 apic_debug("Can't set EOI MSR value: 0x%llx\n",
96893977 557 (unsigned long long)vcpu->arch.pv_eoi.msr_val);
ae7a2a3f
MT		 558 return;
559 }
560 __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
561}
562
563static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu)
564{
565 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) {
566 apic_debug("Can't clear EOI MSR value: 0x%llx\n",
96893977 567 (unsigned long long)vcpu->arch.pv_eoi.msr_val);
ae7a2a3f
MT		 568 return;
569 }
570 __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
571}
572
97222cc8
ED		 573static void apic_update_ppr(struct kvm_lapic *apic)
574{
3842d135 575 u32 tpr, isrv, ppr, old_ppr;
97222cc8
ED		 576 int isr;
577
dfb95954
SS		 578 old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI);
579 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI);
97222cc8
ED		 580 isr = apic_find_highest_isr(apic);
581 isrv = (isr != -1) ? isr : 0;
582
583 if ((tpr & 0xf0) >= (isrv & 0xf0))
584 ppr = tpr & 0xff;
585 else
586 ppr = isrv & 0xf0;
587
588 apic_debug("vlapic %p, ppr 0x%x, isr 0x%x, isrv 0x%x",
589 apic, ppr, isr, isrv);
590
3842d135 591 if (old_ppr != ppr) {
1e6e2755 592 kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr);
83bcacb1
AK		 593 if (ppr < old_ppr)
594 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
3842d135 595 }
97222cc8
ED		 596}
597
598static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr)
599{
1e6e2755 600 kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr);
97222cc8
ED		 601 apic_update_ppr(apic);
602}
603
03d2249e 604static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda)
394457a9 605{
b4535b58
RK		 606 return mda == (apic_x2apic_mode(apic) ?
607 X2APIC_BROADCAST : APIC_BROADCAST);
394457a9
NA		 608}
609
03d2249e 610static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda)
97222cc8 611{
03d2249e
RK		 612 if (kvm_apic_broadcast(apic, mda))
613 return true;
614
615 if (apic_x2apic_mode(apic))
6e500439 616 return mda == kvm_x2apic_id(apic);
03d2249e 617
5bd5db38
RK		 618 /*
619 * Hotplug hack: Make LAPIC in xAPIC mode also accept interrupts as if
620 * it were in x2APIC mode. Hotplugged VCPUs start in xAPIC mode and
621 * this allows unique addressing of VCPUs with APIC ID over 0xff.
622 * The 0xff condition is needed because writeable xAPIC ID.
623 */
624 if (kvm_x2apic_id(apic) > 0xff && mda == kvm_x2apic_id(apic))
625 return true;
626
b4535b58 627 return mda == kvm_xapic_id(apic);
97222cc8
ED		 628}
629
52c233a4 630static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
97222cc8 631{
0105d1a5
GN		 632 u32 logical_id;
633
394457a9 634 if (kvm_apic_broadcast(apic, mda))
9368b567 635 return true;
394457a9 636
dfb95954 637 logical_id = kvm_lapic_get_reg(apic, APIC_LDR);
97222cc8 638
9368b567 639 if (apic_x2apic_mode(apic))
8a395363
RK		 640 return ((logical_id >> 16) == (mda >> 16))
641 && (logical_id & mda & 0xffff) != 0;
97222cc8 642
9368b567 643 logical_id = GET_APIC_LOGICAL_ID(logical_id);
97222cc8 644
dfb95954 645 switch (kvm_lapic_get_reg(apic, APIC_DFR)) {
97222cc8 646 case APIC_DFR_FLAT:
9368b567 647 return (logical_id & mda) != 0;
97222cc8 648 case APIC_DFR_CLUSTER:
9368b567
RK		 649 return ((logical_id >> 4) == (mda >> 4))
650 && (logical_id & mda & 0xf) != 0;
97222cc8 651 default:
7712de87 652 apic_debug("Bad DFR vcpu %d: %08x\n",
dfb95954 653 apic->vcpu->vcpu_id, kvm_lapic_get_reg(apic, APIC_DFR));
9368b567 654 return false;
97222cc8 655 }
97222cc8
ED		 656}
657
c519265f
RK		 658/* The KVM local APIC implementation has two quirks:
659 *
b4535b58
RK		 660 * - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs
661 * in xAPIC mode if the "destination & 0xff" matches its xAPIC ID.
662 * KVM doesn't do that aliasing.
c519265f
RK		 663 *
664 * - in-kernel IOAPIC messages have to be delivered directly to
665 * x2APIC, because the kernel does not support interrupt remapping.
666 * In order to support broadcast without interrupt remapping, x2APIC
667 * rewrites the destination of non-IPI messages from APIC_BROADCAST
668 * to X2APIC_BROADCAST.
669 *
670 * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API. This is
671 * important when userspace wants to use x2APIC-format MSIs, because
672 * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7".
03d2249e 673 */
c519265f
RK		 674static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id,
675 struct kvm_lapic *source, struct kvm_lapic *target)
03d2249e
RK		 676{
677 bool ipi = source != NULL;
03d2249e 678
c519265f 679 if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled &&
b4535b58 680 !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target))
03d2249e
RK		 681 return X2APIC_BROADCAST;
682
b4535b58 683 return dest_id;
03d2249e
RK		 684}
685
52c233a4 686bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
394457a9 687 int short_hand, unsigned int dest, int dest_mode)
97222cc8 688{
ad312c7c 689 struct kvm_lapic *target = vcpu->arch.apic;
c519265f 690 u32 mda = kvm_apic_mda(vcpu, dest, source, target);
97222cc8
ED		 691
692 apic_debug("target %p, source %p, dest 0x%x, "
343f94fe 693 "dest_mode 0x%x, short_hand 0x%x\n",
97222cc8
ED		 694 target, source, dest, dest_mode, short_hand);
695
bd371396 696 ASSERT(target);
97222cc8
ED		 697 switch (short_hand) {
698 case APIC_DEST_NOSHORT:
3697f302 699 if (dest_mode == APIC_DEST_PHYSICAL)
03d2249e 700 return kvm_apic_match_physical_addr(target, mda);
343f94fe 701 else
03d2249e 702 return kvm_apic_match_logical_addr(target, mda);
97222cc8 703 case APIC_DEST_SELF:
9368b567 704 return target == source;
97222cc8 705 case APIC_DEST_ALLINC:
9368b567 706 return true;
97222cc8 707 case APIC_DEST_ALLBUT:
9368b567 708 return target != source;
97222cc8 709 default:
7712de87
JK		 710 apic_debug("kvm: apic: Bad dest shorthand value %x\n",
711 short_hand);
9368b567 712 return false;
97222cc8 713 }
97222cc8 714}
1e6e2755 715EXPORT_SYMBOL_GPL(kvm_apic_match_dest);
97222cc8 716
52004014
FW		 717int kvm_vector_to_index(u32 vector, u32 dest_vcpus,
718 const unsigned long *bitmap, u32 bitmap_size)
719{
720 u32 mod;
721 int i, idx = -1;
722
723 mod = vector % dest_vcpus;
724
725 for (i = 0; i <= mod; i++) {
726 idx = find_next_bit(bitmap, bitmap_size, idx + 1);
727 BUG_ON(idx == bitmap_size);
728 }
729
730 return idx;
731}
732
4efd805f
RK		 733static void kvm_apic_disabled_lapic_found(struct kvm *kvm)
734{
735 if (!kvm->arch.disabled_lapic_found) {
736 kvm->arch.disabled_lapic_found = true;
737 printk(KERN_INFO
738 "Disabled LAPIC found during irq injection\n");
739 }
740}
741
c519265f
RK		 742static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src,
743 struct kvm_lapic_irq *irq, struct kvm_apic_map *map)
1e08ec4a 744{
c519265f
RK		 745 if (kvm->arch.x2apic_broadcast_quirk_disabled) {
746 if ((irq->dest_id == APIC_BROADCAST &&
747 map->mode != KVM_APIC_MODE_X2APIC))
748 return true;
749 if (irq->dest_id == X2APIC_BROADCAST)
750 return true;
751 } else {
752 bool x2apic_ipi = src && *src && apic_x2apic_mode(*src);
753 if (irq->dest_id == (x2apic_ipi ?
754 X2APIC_BROADCAST : APIC_BROADCAST))
755 return true;
756 }
1e08ec4a 757
c519265f
RK		 758 return false;
759}
1e08ec4a 760
64aa47bf
RK		 761/* Return true if the interrupt can be handled by using *bitmap as index mask
762 * for valid destinations in *dst array.
763 * Return false if kvm_apic_map_get_dest_lapic did nothing useful.
764 * Note: we may have zero kvm_lapic destinations when we return true, which
765 * means that the interrupt should be dropped. In this case, *bitmap would be
766 * zero and *dst undefined.
767 */
768static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm,
769 struct kvm_lapic **src, struct kvm_lapic_irq *irq,
770 struct kvm_apic_map *map, struct kvm_lapic ***dst,
771 unsigned long *bitmap)
772{
773 int i, lowest;
1e08ec4a 774
64aa47bf
RK		 775 if (irq->shorthand == APIC_DEST_SELF && src) {
776 *dst = src;
777 *bitmap = 1;
778 return true;
779 } else if (irq->shorthand)
1e08ec4a
GN		 780 return false;
781
c519265f 782 if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map))
9ea369b0
RK		 783 return false;
784
64aa47bf 785 if (irq->dest_mode == APIC_DEST_PHYSICAL) {
0ca52e7b 786 if (irq->dest_id > map->max_apic_id) {
64aa47bf
RK		 787 *bitmap = 0;
788 } else {
789 *dst = &map->phys_map[irq->dest_id];
790 *bitmap = 1;
791 }
1e08ec4a 792 return true;
bea15428 793 }
698f9755 794
e45115b6
RK		 795 *bitmap = 0;
796 if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst,
797 (u16 *)bitmap))
1e08ec4a 798 return false;
fa834e91 799
64aa47bf
RK		 800 if (!kvm_lowest_prio_delivery(irq))
801 return true;
3548a259 802
64aa47bf
RK		 803 if (!kvm_vector_hashing_enabled()) {
804 lowest = -1;
805 for_each_set_bit(i, bitmap, 16) {
806 if (!(*dst)[i])
807 continue;
808 if (lowest < 0)
809 lowest = i;
810 else if (kvm_apic_compare_prio((*dst)[i]->vcpu,
811 (*dst)[lowest]->vcpu) < 0)
812 lowest = i;
3548a259 813 }
64aa47bf
RK		 814 } else {
815 if (!*bitmap)
816 return true;
3548a259 817
64aa47bf
RK		 818 lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap),
819 bitmap, 16);
45c3094a 820
64aa47bf
RK		 821 if (!(*dst)[lowest]) {
822 kvm_apic_disabled_lapic_found(kvm);
823 *bitmap = 0;
824 return true;
825 }
826 }
1e08ec4a 827
64aa47bf 828 *bitmap = (lowest >= 0) ? 1 << lowest : 0;
1e08ec4a 829
64aa47bf
RK		 830 return true;
831}
52004014 832
64aa47bf
RK		 833bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
834 struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map)
835{
836 struct kvm_apic_map *map;
837 unsigned long bitmap;
838 struct kvm_lapic **dst = NULL;
839 int i;
840 bool ret;
52004014 841
64aa47bf 842 *r = -1;
52004014 843
64aa47bf
RK		 844 if (irq->shorthand == APIC_DEST_SELF) {
845 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map);
846 return true;
847 }
52004014 848
64aa47bf
RK		 849 rcu_read_lock();
850 map = rcu_dereference(kvm->arch.apic_map);
52004014 851
64aa47bf
RK		 852 ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap);
853 if (ret)
854 for_each_set_bit(i, &bitmap, 16) {
855 if (!dst[i])
856 continue;
857 if (*r < 0)
858 *r = 0;
859 *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map);
1e08ec4a 860 }
1e08ec4a 861
1e08ec4a
GN		 862 rcu_read_unlock();
863 return ret;
864}
865
6228a0da
FW		 866/*
867 * This routine tries to handler interrupts in posted mode, here is how
868 * it deals with different cases:
869 * - For single-destination interrupts, handle it in posted mode
870 * - Else if vector hashing is enabled and it is a lowest-priority
871 * interrupt, handle it in posted mode and use the following mechanism
872 * to find the destinaiton vCPU.
873 * 1. For lowest-priority interrupts, store all the possible
874 * destination vCPUs in an array.
875 * 2. Use "guest vector % max number of destination vCPUs" to find
876 * the right destination vCPU in the array for the lowest-priority
877 * interrupt.
878 * - Otherwise, use remapped mode to inject the interrupt.
879 */
8feb4a04
FW		 880bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq,
881 struct kvm_vcpu **dest_vcpu)
882{
883 struct kvm_apic_map *map;
64aa47bf
RK		 884 unsigned long bitmap;
885 struct kvm_lapic **dst = NULL;
8feb4a04 886 bool ret = false;
8feb4a04
FW		 887
888 if (irq->shorthand)
889 return false;
890
891 rcu_read_lock();
892 map = rcu_dereference(kvm->arch.apic_map);
893
64aa47bf
RK		 894 if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) &&
895 hweight16(bitmap) == 1) {
896 unsigned long i = find_first_bit(&bitmap, 16);
6228a0da 897
64aa47bf
RK		 898 if (dst[i]) {
899 *dest_vcpu = dst[i]->vcpu;
900 ret = true;
6228a0da 901 }
8feb4a04
FW		 902 }
903
8feb4a04
FW		 904 rcu_read_unlock();
905 return ret;
906}
907
97222cc8
ED		 908/*
909 * Add a pending IRQ into lapic.
910 * Return 1 if successfully added and 0 if discarded.
911 */
912static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
b4f2225c 913 int vector, int level, int trig_mode,
9e4aabe2 914 struct dest_map *dest_map)
97222cc8 915{
6da7e3f6 916 int result = 0;
c5ec1534 917 struct kvm_vcpu *vcpu = apic->vcpu;
97222cc8 918
a183b638
PB		 919 trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode,
920 trig_mode, vector);
97222cc8 921 switch (delivery_mode) {
97222cc8 922 case APIC_DM_LOWEST:
e1035715
GN		 923 vcpu->arch.apic_arb_prio++;
924 case APIC_DM_FIXED:
bdaffe1d
PB		 925 if (unlikely(trig_mode && !level))
926 break;
927
97222cc8
ED		 928 /* FIXME add logic for vcpu on reset */
929 if (unlikely(!apic_enabled(apic)))
930 break;
931
11f5cc05
JK		 932 result = 1;
933
9daa5007 934 if (dest_map) {
9e4aabe2 935 __set_bit(vcpu->vcpu_id, dest_map->map);
9daa5007
JR		 936 dest_map->vectors[vcpu->vcpu_id] = vector;
937 }
a5d36f82 938
bdaffe1d
PB		 939 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) {
940 if (trig_mode)
1e6e2755 941 kvm_lapic_set_vector(vector, apic->regs + APIC_TMR);
bdaffe1d
PB		 942 else
943 apic_clear_vector(vector, apic->regs + APIC_TMR);
944 }
945
d62caabb 946 if (vcpu->arch.apicv_active)
5a71785d 947 kvm_x86_ops->deliver_posted_interrupt(vcpu, vector);
11f5cc05 948 else {
1e6e2755 949 kvm_lapic_set_irr(vector, apic);
5a71785d
YZ		 950
951 kvm_make_request(KVM_REQ_EVENT, vcpu);
952 kvm_vcpu_kick(vcpu);
953 }
97222cc8
ED		 954 break;
955
956 case APIC_DM_REMRD:
24d2166b
R		 957 result = 1;
958 vcpu->arch.pv.pv_unhalted = 1;
959 kvm_make_request(KVM_REQ_EVENT, vcpu);
960 kvm_vcpu_kick(vcpu);
97222cc8
ED		 961 break;
962
963 case APIC_DM_SMI:
64d60670
PB		 964 result = 1;
965 kvm_make_request(KVM_REQ_SMI, vcpu);
966 kvm_vcpu_kick(vcpu);
97222cc8 967 break;
3419ffc8 968
97222cc8 969 case APIC_DM_NMI:
6da7e3f6 970 result = 1;
3419ffc8 971 kvm_inject_nmi(vcpu);
26df99c6 972 kvm_vcpu_kick(vcpu);
97222cc8
ED		 973 break;
974
975 case APIC_DM_INIT:
a52315e1 976 if (!trig_mode || level) {
6da7e3f6 977 result = 1;
66450a21
JK		 978 /* assumes that there are only KVM_APIC_INIT/SIPI */
979 apic->pending_events = (1UL << KVM_APIC_INIT);
980 /* make sure pending_events is visible before sending
981 * the request */
982 smp_wmb();
3842d135 983 kvm_make_request(KVM_REQ_EVENT, vcpu);
c5ec1534
HQ		 984 kvm_vcpu_kick(vcpu);
985 } else {
1b10bf31
JK		 986 apic_debug("Ignoring de-assert INIT to vcpu %d\n",
987 vcpu->vcpu_id);
c5ec1534 988 }
97222cc8
ED		 989 break;
990
991 case APIC_DM_STARTUP:
1b10bf31
JK		 992 apic_debug("SIPI to vcpu %d vector 0x%02x\n",
993 vcpu->vcpu_id, vector);
66450a21
JK		 994 result = 1;
995 apic->sipi_vector = vector;
996 /* make sure sipi_vector is visible for the receiver */
997 smp_wmb();
998 set_bit(KVM_APIC_SIPI, &apic->pending_events);
999 kvm_make_request(KVM_REQ_EVENT, vcpu);
1000 kvm_vcpu_kick(vcpu);
97222cc8
ED		 1001 break;
1002
23930f95
JK		 1003 case APIC_DM_EXTINT:
1004 /*
1005 * Should only be called by kvm_apic_local_deliver() with LVT0,
1006 * before NMI watchdog was enabled. Already handled by
1007 * kvm_apic_accept_pic_intr().
1008 */
1009 break;
1010
97222cc8
ED		 1011 default:
1012 printk(KERN_ERR "TODO: unsupported delivery mode %x\n",
1013 delivery_mode);
1014 break;
1015 }
1016 return result;
1017}
1018
e1035715 1019int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
8be5453f 1020{
e1035715 1021 return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio;
8be5453f
ZX		 1022}
1023
3bb345f3
PB		 1024static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector)
1025{
6308630b 1026 return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors);
3bb345f3
PB		 1027}
1028
c7c9c56c
YZ		 1029static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector)
1030{
7543a635
SR		 1031 int trigger_mode;
1032
1033 /* Eoi the ioapic only if the ioapic doesn't own the vector. */
1034 if (!kvm_ioapic_handles_vector(apic, vector))
1035 return;
3bb345f3 1036
7543a635
SR		 1037 /* Request a KVM exit to inform the userspace IOAPIC. */
1038 if (irqchip_split(apic->vcpu->kvm)) {
1039 apic->vcpu->arch.pending_ioapic_eoi = vector;
1040 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu);
1041 return;
c7c9c56c 1042 }
7543a635
SR		 1043
1044 if (apic_test_vector(vector, apic->regs + APIC_TMR))
1045 trigger_mode = IOAPIC_LEVEL_TRIG;
1046 else
1047 trigger_mode = IOAPIC_EDGE_TRIG;
1048
1049 kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode);
c7c9c56c
YZ		 1050}
1051
ae7a2a3f 1052static int apic_set_eoi(struct kvm_lapic *apic)
97222cc8
ED		 1053{
1054 int vector = apic_find_highest_isr(apic);
ae7a2a3f
MT		 1055
1056 trace_kvm_eoi(apic, vector);
1057
97222cc8
ED		 1058 /*
1059 * Not every write EOI will has corresponding ISR,
1060 * one example is when Kernel check timer on setup_IO_APIC
1061 */
1062 if (vector == -1)
ae7a2a3f 1063 return vector;
97222cc8 1064
8680b94b 1065 apic_clear_isr(vector, apic);
97222cc8
ED		 1066 apic_update_ppr(apic);
1067
5c919412
AS		 1068 if (test_bit(vector, vcpu_to_synic(apic->vcpu)->vec_bitmap))
1069 kvm_hv_synic_send_eoi(apic->vcpu, vector);
1070
c7c9c56c 1071 kvm_ioapic_send_eoi(apic, vector);
3842d135 1072 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
ae7a2a3f 1073 return vector;
97222cc8
ED		 1074}
1075
c7c9c56c
YZ		 1076/*
1077 * this interface assumes a trap-like exit, which has already finished
1078 * desired side effect including vISR and vPPR update.
1079 */
1080void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector)
1081{
1082 struct kvm_lapic *apic = vcpu->arch.apic;
1083
1084 trace_kvm_eoi(apic, vector);
1085
1086 kvm_ioapic_send_eoi(apic, vector);
1087 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
1088}
1089EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated);
1090
97222cc8
ED		 1091static void apic_send_ipi(struct kvm_lapic *apic)
1092{
dfb95954
SS		 1093 u32 icr_low = kvm_lapic_get_reg(apic, APIC_ICR);
1094 u32 icr_high = kvm_lapic_get_reg(apic, APIC_ICR2);
58c2dde1 1095 struct kvm_lapic_irq irq;
97222cc8 1096
58c2dde1
GN		 1097 irq.vector = icr_low & APIC_VECTOR_MASK;
1098 irq.delivery_mode = icr_low & APIC_MODE_MASK;
1099 irq.dest_mode = icr_low & APIC_DEST_MASK;
b7cb2231 1100 irq.level = (icr_low & APIC_INT_ASSERT) != 0;
58c2dde1
GN		 1101 irq.trig_mode = icr_low & APIC_INT_LEVELTRIG;
1102 irq.shorthand = icr_low & APIC_SHORT_MASK;
93bbf0b8 1103 irq.msi_redir_hint = false;
0105d1a5
GN		 1104 if (apic_x2apic_mode(apic))
1105 irq.dest_id = icr_high;
1106 else
1107 irq.dest_id = GET_APIC_DEST_FIELD(icr_high);
97222cc8 1108
1000ff8d
GN		 1109 trace_kvm_apic_ipi(icr_low, irq.dest_id);
1110
97222cc8
ED		 1111 apic_debug("icr_high 0x%x, icr_low 0x%x, "
1112 "short_hand 0x%x, dest 0x%x, trig_mode 0x%x, level 0x%x, "
93bbf0b8
JS		 1113 "dest_mode 0x%x, delivery_mode 0x%x, vector 0x%x, "
1114 "msi_redir_hint 0x%x\n",
9b5843dd 1115 icr_high, icr_low, irq.shorthand, irq.dest_id,
58c2dde1 1116 irq.trig_mode, irq.level, irq.dest_mode, irq.delivery_mode,
93bbf0b8 1117 irq.vector, irq.msi_redir_hint);
58c2dde1 1118
b4f2225c 1119 kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL);
97222cc8
ED		 1120}
1121
1122static u32 apic_get_tmcct(struct kvm_lapic *apic)
1123{
8003c9ae 1124 ktime_t remaining, now;
b682b814 1125 s64 ns;
9da8f4e8 1126 u32 tmcct;
97222cc8
ED		 1127
1128 ASSERT(apic != NULL);
1129
9da8f4e8 1130 /* if initial count is 0, current count should also be 0 */
dfb95954 1131 if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 ||
b963a22e 1132 apic->lapic_timer.period == 0)
9da8f4e8
KP		 1133 return 0;
1134
5587859f 1135 now = ktime_get();
8003c9ae 1136 remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
b682b814 1137 if (ktime_to_ns(remaining) < 0)
8b0e1953 1138 remaining = 0;
b682b814 1139
d3c7b77d
MT		 1140 ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period);
1141 tmcct = div64_u64(ns,
1142 (APIC_BUS_CYCLE_NS * apic->divide_count));
97222cc8
ED		 1143
1144 return tmcct;
1145}
1146
b209749f
AK		 1147static void __report_tpr_access(struct kvm_lapic *apic, bool write)
1148{
1149 struct kvm_vcpu *vcpu = apic->vcpu;
1150 struct kvm_run *run = vcpu->run;
1151
a8eeb04a 1152 kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu);
5fdbf976 1153 run->tpr_access.rip = kvm_rip_read(vcpu);
b209749f
AK		 1154 run->tpr_access.is_write = write;
1155}
1156
1157static inline void report_tpr_access(struct kvm_lapic *apic, bool write)
1158{
1159 if (apic->vcpu->arch.tpr_access_reporting)
1160 __report_tpr_access(apic, write);
1161}
1162
97222cc8
ED		 1163static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
1164{
1165 u32 val = 0;
1166
1167 if (offset >= LAPIC_MMIO_LENGTH)
1168 return 0;
1169
1170 switch (offset) {
1171 case APIC_ARBPRI:
7712de87 1172 apic_debug("Access APIC ARBPRI register which is for P6\n");
97222cc8
ED		 1173 break;
1174
1175 case APIC_TMCCT: /* Timer CCR */
a3e06bbe
LJ		 1176 if (apic_lvtt_tscdeadline(apic))
1177 return 0;
1178
97222cc8
ED		 1179 val = apic_get_tmcct(apic);
1180 break;
4a4541a4
AK		 1181 case APIC_PROCPRI:
1182 apic_update_ppr(apic);
dfb95954 1183 val = kvm_lapic_get_reg(apic, offset);
4a4541a4 1184 break;
b209749f
AK		 1185 case APIC_TASKPRI:
1186 report_tpr_access(apic, false);
1187 /* fall thru */
97222cc8 1188 default:
dfb95954 1189 val = kvm_lapic_get_reg(apic, offset);
97222cc8
ED		 1190 break;
1191 }
1192
1193 return val;
1194}
1195
d76685c4
GH		 1196static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev)
1197{
1198 return container_of(dev, struct kvm_lapic, dev);
1199}
1200
1e6e2755 1201int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
0105d1a5 1202 void *data)
97222cc8 1203{
97222cc8
ED		 1204 unsigned char alignment = offset & 0xf;
1205 u32 result;
d5b0b5b1 1206 /* this bitmask has a bit cleared for each reserved register */
0105d1a5 1207 static const u64 rmask = 0x43ff01ffffffe70cULL;
97222cc8
ED		 1208
1209 if ((alignment + len) > 4) {
4088bb3c
GN		 1210 apic_debug("KVM_APIC_READ: alignment error %x %d\n",
1211 offset, len);
0105d1a5 1212 return 1;
97222cc8 1213 }
0105d1a5
GN		 1214
1215 if (offset > 0x3f0 || !(rmask & (1ULL << (offset >> 4)))) {
4088bb3c
GN		 1216 apic_debug("KVM_APIC_READ: read reserved register %x\n",
1217 offset);
0105d1a5
GN		 1218 return 1;
1219 }
1220
97222cc8
ED		 1221 result = __apic_read(apic, offset & ~0xf);
1222
229456fc
MT		 1223 trace_kvm_apic_read(offset, result);
1224
97222cc8
ED		 1225 switch (len) {
1226 case 1:
1227 case 2:
1228 case 4:
1229 memcpy(data, (char *)&result + alignment, len);
1230 break;
1231 default:
1232 printk(KERN_ERR "Local APIC read with len = %x, "
1233 "should be 1,2, or 4 instead\n", len);
1234 break;
1235 }
bda9020e 1236 return 0;
97222cc8 1237}
1e6e2755 1238EXPORT_SYMBOL_GPL(kvm_lapic_reg_read);
97222cc8 1239
0105d1a5
GN		 1240static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
1241{
c48f1496 1242 return kvm_apic_hw_enabled(apic) &&
0105d1a5
GN		 1243 addr >= apic->base_address &&
1244 addr < apic->base_address + LAPIC_MMIO_LENGTH;
1245}
1246
e32edf4f 1247static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
0105d1a5
GN		 1248 gpa_t address, int len, void *data)
1249{
1250 struct kvm_lapic *apic = to_lapic(this);
1251 u32 offset = address - apic->base_address;
1252
1253 if (!apic_mmio_in_range(apic, address))
1254 return -EOPNOTSUPP;
1255
1e6e2755 1256 kvm_lapic_reg_read(apic, offset, len, data);
0105d1a5
GN		 1257
1258 return 0;
1259}
1260
97222cc8
ED		 1261static void update_divide_count(struct kvm_lapic *apic)
1262{
1263 u32 tmp1, tmp2, tdcr;
1264
dfb95954 1265 tdcr = kvm_lapic_get_reg(apic, APIC_TDCR);
97222cc8
ED		 1266 tmp1 = tdcr & 0xf;
1267 tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1;
d3c7b77d 1268 apic->divide_count = 0x1 << (tmp2 & 0x7);
97222cc8
ED		 1269
1270 apic_debug("timer divide count is 0x%x\n",
9b5843dd 1271 apic->divide_count);
97222cc8
ED		 1272}
1273
b6ac0695
RK		 1274static void apic_update_lvtt(struct kvm_lapic *apic)
1275{
dfb95954 1276 u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) &
b6ac0695
RK		 1277 apic->lapic_timer.timer_mode_mask;
1278
1279 if (apic->lapic_timer.timer_mode != timer_mode) {
1280 apic->lapic_timer.timer_mode = timer_mode;
1281 hrtimer_cancel(&apic->lapic_timer.timer);
1282 }
1283}
1284
5d87db71
RK		 1285static void apic_timer_expired(struct kvm_lapic *apic)
1286{
1287 struct kvm_vcpu *vcpu = apic->vcpu;
8577370f 1288 struct swait_queue_head *q = &vcpu->wq;
d0659d94 1289 struct kvm_timer *ktimer = &apic->lapic_timer;
5d87db71 1290
5d87db71
RK		 1291 if (atomic_read(&apic->lapic_timer.pending))
1292 return;
1293
1294 atomic_inc(&apic->lapic_timer.pending);
bab5bb39 1295 kvm_set_pending_timer(vcpu);
5d87db71 1296
8577370f
MT		 1297 if (swait_active(q))
1298 swake_up(q);
d0659d94
MT		 1299
1300 if (apic_lvtt_tscdeadline(apic))
1301 ktimer->expired_tscdeadline = ktimer->tscdeadline;
1302}
1303
1304/*
1305 * On APICv, this test will cause a busy wait
1306 * during a higher-priority task.
1307 */
1308
1309static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu)
1310{
1311 struct kvm_lapic *apic = vcpu->arch.apic;
dfb95954 1312 u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT);
d0659d94
MT		 1313
1314 if (kvm_apic_hw_enabled(apic)) {
1315 int vec = reg & APIC_VECTOR_MASK;
f9339860 1316 void *bitmap = apic->regs + APIC_ISR;
d0659d94 1317
d62caabb 1318 if (vcpu->arch.apicv_active)
f9339860
MT		 1319 bitmap = apic->regs + APIC_IRR;
1320
1321 if (apic_test_vector(vec, bitmap))
1322 return true;
d0659d94
MT		 1323 }
1324 return false;
1325}
1326
1327void wait_lapic_expire(struct kvm_vcpu *vcpu)
1328{
1329 struct kvm_lapic *apic = vcpu->arch.apic;
1330 u64 guest_tsc, tsc_deadline;
1331
bce87cce 1332 if (!lapic_in_kernel(vcpu))
d0659d94
MT		 1333 return;
1334
1335 if (apic->lapic_timer.expired_tscdeadline == 0)
1336 return;
1337
1338 if (!lapic_timer_int_injected(vcpu))
1339 return;
1340
1341 tsc_deadline = apic->lapic_timer.expired_tscdeadline;
1342 apic->lapic_timer.expired_tscdeadline = 0;
4ba76538 1343 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
6c19b753 1344 trace_kvm_wait_lapic_expire(vcpu->vcpu_id, guest_tsc - tsc_deadline);
d0659d94
MT		 1345
1346 /* __delay is delay_tsc whenever the hardware has TSC, thus always. */
1347 if (guest_tsc < tsc_deadline)
b606f189
MT		 1348 __delay(min(tsc_deadline - guest_tsc,
1349 nsec_to_cycles(vcpu, lapic_timer_advance_ns)));
5d87db71
RK		 1350}
1351
53f9eedf
YJ		 1352static void start_sw_tscdeadline(struct kvm_lapic *apic)
1353{
1354 u64 guest_tsc, tscdeadline = apic->lapic_timer.tscdeadline;
1355 u64 ns = 0;
1356 ktime_t expire;
1357 struct kvm_vcpu *vcpu = apic->vcpu;
1358 unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz;
1359 unsigned long flags;
1360 ktime_t now;
1361
1362 if (unlikely(!tscdeadline || !this_tsc_khz))
1363 return;
1364
1365 local_irq_save(flags);
1366
5587859f 1367 now = ktime_get();
53f9eedf
YJ		 1368 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1369 if (likely(tscdeadline > guest_tsc)) {
1370 ns = (tscdeadline - guest_tsc) * 1000000ULL;
1371 do_div(ns, this_tsc_khz);
1372 expire = ktime_add_ns(now, ns);
1373 expire = ktime_sub_ns(expire, lapic_timer_advance_ns);
1374 hrtimer_start(&apic->lapic_timer.timer,
1375 expire, HRTIMER_MODE_ABS_PINNED);
1376 } else
1377 apic_timer_expired(apic);
1378
1379 local_irq_restore(flags);
1380}
1381
7d7f7da2 1382static void start_sw_period(struct kvm_lapic *apic)
8003c9ae
WL		 1383{
1384 if (!apic->lapic_timer.period)
1385 return;
1386
1387 if (apic_lvtt_oneshot(apic) &&
5587859f 1388 ktime_after(ktime_get(),
8003c9ae
WL		 1389 apic->lapic_timer.target_expiration)) {
1390 apic_timer_expired(apic);
1391 return;
1392 }
1393
1394 hrtimer_start(&apic->lapic_timer.timer,
1395 apic->lapic_timer.target_expiration,
1396 HRTIMER_MODE_ABS_PINNED);
1397}
1398
1399static bool set_target_expiration(struct kvm_lapic *apic)
7d7f7da2
WL		 1400{
1401 ktime_t now;
8003c9ae 1402 u64 tscl = rdtsc();
7d7f7da2 1403
5587859f 1404 now = ktime_get();
7d7f7da2 1405 apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT)
8003c9ae 1406 * APIC_BUS_CYCLE_NS * apic->divide_count;
7d7f7da2
WL		 1407
1408 if (!apic->lapic_timer.period)
8003c9ae
WL		 1409 return false;
1410
7d7f7da2
WL		 1411 /*
1412 * Do not allow the guest to program periodic timers with small
1413 * interval, since the hrtimers are not throttled by the host
1414 * scheduler.
1415 */
1416 if (apic_lvtt_period(apic)) {
1417 s64 min_period = min_timer_period_us * 1000LL;
1418
1419 if (apic->lapic_timer.period < min_period) {
1420 pr_info_ratelimited(
1421 "kvm: vcpu %i: requested %lld ns "
1422 "lapic timer period limited to %lld ns\n",
1423 apic->vcpu->vcpu_id,
1424 apic->lapic_timer.period, min_period);
1425 apic->lapic_timer.period = min_period;
1426 }
1427 }
1428
7d7f7da2
WL		 1429 apic_debug("%s: bus cycle is %" PRId64 "ns, now 0x%016"
1430 PRIx64 ", "
1431 "timer initial count 0x%x, period %lldns, "
1432 "expire @ 0x%016" PRIx64 ".\n", __func__,
1433 APIC_BUS_CYCLE_NS, ktime_to_ns(now),
1434 kvm_lapic_get_reg(apic, APIC_TMICT),
1435 apic->lapic_timer.period,
1436 ktime_to_ns(ktime_add_ns(now,
1437 apic->lapic_timer.period)));
8003c9ae
WL		 1438
1439 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
1440 nsec_to_cycles(apic->vcpu, apic->lapic_timer.period);
1441 apic->lapic_timer.target_expiration = ktime_add_ns(now, apic->lapic_timer.period);
1442
1443 return true;
1444}
1445
1446static void advance_periodic_target_expiration(struct kvm_lapic *apic)
1447{
1448 apic->lapic_timer.tscdeadline +=
1449 nsec_to_cycles(apic->vcpu, apic->lapic_timer.period);
1450 apic->lapic_timer.target_expiration =
1451 ktime_add_ns(apic->lapic_timer.target_expiration,
1452 apic->lapic_timer.period);
7d7f7da2
WL		 1453}
1454
ce7a058a
YJ		 1455bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu)
1456{
91005300
WL		 1457 if (!lapic_in_kernel(vcpu))
1458 return false;
1459
ce7a058a
YJ		 1460 return vcpu->arch.apic->lapic_timer.hv_timer_in_use;
1461}
1462EXPORT_SYMBOL_GPL(kvm_lapic_hv_timer_in_use);
1463
7e810a38 1464static void cancel_hv_timer(struct kvm_lapic *apic)
bd97ad0e
WL		 1465{
1466 kvm_x86_ops->cancel_hv_timer(apic->vcpu);
1467 apic->lapic_timer.hv_timer_in_use = false;
1468}
1469
7e810a38 1470static bool start_hv_timer(struct kvm_lapic *apic)
196f20ca
WL		 1471{
1472 u64 tscdeadline = apic->lapic_timer.tscdeadline;
1473
8003c9ae
WL		 1474 if ((atomic_read(&apic->lapic_timer.pending) &&
1475 !apic_lvtt_period(apic)) ||
196f20ca
WL		 1476 kvm_x86_ops->set_hv_timer(apic->vcpu, tscdeadline)) {
1477 if (apic->lapic_timer.hv_timer_in_use)
7e810a38 1478 cancel_hv_timer(apic);
196f20ca
WL		 1479 } else {
1480 apic->lapic_timer.hv_timer_in_use = true;
1481 hrtimer_cancel(&apic->lapic_timer.timer);
1482
1483 /* In case the sw timer triggered in the window */
8003c9ae
WL		 1484 if (atomic_read(&apic->lapic_timer.pending) &&
1485 !apic_lvtt_period(apic))
7e810a38 1486 cancel_hv_timer(apic);
196f20ca
WL		 1487 }
1488 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id,
1489 apic->lapic_timer.hv_timer_in_use);
1490 return apic->lapic_timer.hv_timer_in_use;
1491}
1492
8003c9ae
WL		 1493void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu)
1494{
1495 struct kvm_lapic *apic = vcpu->arch.apic;
1496
1497 WARN_ON(!apic->lapic_timer.hv_timer_in_use);
1498 WARN_ON(swait_active(&vcpu->wq));
1499 cancel_hv_timer(apic);
1500 apic_timer_expired(apic);
1501
1502 if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
1503 advance_periodic_target_expiration(apic);
1504 if (!start_hv_timer(apic))
1505 start_sw_period(apic);
1506 }
1507}
1508EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer);
1509
ce7a058a
YJ		 1510void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu)
1511{
1512 struct kvm_lapic *apic = vcpu->arch.apic;
1513
1514 WARN_ON(apic->lapic_timer.hv_timer_in_use);
1515
8003c9ae 1516 start_hv_timer(apic);
ce7a058a
YJ		 1517}
1518EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_hv_timer);
1519
1520void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu)
1521{
1522 struct kvm_lapic *apic = vcpu->arch.apic;
1523
1524 /* Possibly the TSC deadline timer is not enabled yet */
1525 if (!apic->lapic_timer.hv_timer_in_use)
1526 return;
1527
7e810a38 1528 cancel_hv_timer(apic);
ce7a058a
YJ		 1529
1530 if (atomic_read(&apic->lapic_timer.pending))
1531 return;
1532
8003c9ae
WL		 1533 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
1534 start_sw_period(apic);
1535 else if (apic_lvtt_tscdeadline(apic))
1536 start_sw_tscdeadline(apic);
ce7a058a
YJ		 1537}
1538EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_sw_timer);
1539
97222cc8
ED		 1540static void start_apic_timer(struct kvm_lapic *apic)
1541{
d3c7b77d 1542 atomic_set(&apic->lapic_timer.pending, 0);
0b975a3c 1543
a3e06bbe 1544 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) {
8003c9ae
WL		 1545 if (set_target_expiration(apic) &&
1546 !(kvm_x86_ops->set_hv_timer && start_hv_timer(apic)))
1547 start_sw_period(apic);
a3e06bbe 1548 } else if (apic_lvtt_tscdeadline(apic)) {
7e810a38 1549 if (!(kvm_x86_ops->set_hv_timer && start_hv_timer(apic)))
ce7a058a 1550 start_sw_tscdeadline(apic);
a3e06bbe 1551 }
97222cc8
ED		 1552}
1553
cc6e462c
JK		 1554static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
1555{
59fd1323 1556 bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val);
cc6e462c 1557
59fd1323
RK		 1558 if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) {
1559 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode;
1560 if (lvt0_in_nmi_mode) {
cc6e462c
JK		 1561 apic_debug("Receive NMI setting on APIC_LVT0 "
1562 "for cpu %d\n", apic->vcpu->vcpu_id);
42720138 1563 atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
59fd1323
RK		 1564 } else
1565 atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
1566 }
cc6e462c
JK		 1567}
1568
1e6e2755 1569int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
97222cc8 1570{
0105d1a5 1571 int ret = 0;
97222cc8 1572
0105d1a5 1573 trace_kvm_apic_write(reg, val);
97222cc8 1574
0105d1a5 1575 switch (reg) {
97222cc8 1576 case APIC_ID: /* Local APIC ID */
0105d1a5 1577 if (!apic_x2apic_mode(apic))
a92e2543 1578 kvm_apic_set_xapic_id(apic, val >> 24);
0105d1a5
GN		 1579 else
1580 ret = 1;
97222cc8
ED		 1581 break;
1582
1583 case APIC_TASKPRI:
b209749f 1584 report_tpr_access(apic, true);
97222cc8
ED		 1585 apic_set_tpr(apic, val & 0xff);
1586 break;
1587
1588 case APIC_EOI:
1589 apic_set_eoi(apic);
1590 break;
1591
1592 case APIC_LDR:
0105d1a5 1593 if (!apic_x2apic_mode(apic))
1e08ec4a 1594 kvm_apic_set_ldr(apic, val & APIC_LDR_MASK);
0105d1a5
GN		 1595 else
1596 ret = 1;
97222cc8
ED		 1597 break;
1598
1599 case APIC_DFR:
1e08ec4a 1600 if (!apic_x2apic_mode(apic)) {
1e6e2755 1601 kvm_lapic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF);
1e08ec4a
GN		 1602 recalculate_apic_map(apic->vcpu->kvm);
1603 } else
0105d1a5 1604 ret = 1;
97222cc8
ED		 1605 break;
1606
fc61b800
GN		 1607 case APIC_SPIV: {
1608 u32 mask = 0x3ff;
dfb95954 1609 if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI)
fc61b800 1610 mask |= APIC_SPIV_DIRECTED_EOI;
f8c1ea10 1611 apic_set_spiv(apic, val & mask);
97222cc8
ED		 1612 if (!(val & APIC_SPIV_APIC_ENABLED)) {
1613 int i;
1614 u32 lvt_val;
1615
1e6e2755 1616 for (i = 0; i < KVM_APIC_LVT_NUM; i++) {
dfb95954 1617 lvt_val = kvm_lapic_get_reg(apic,
97222cc8 1618 APIC_LVTT + 0x10 * i);
1e6e2755 1619 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i,
97222cc8
ED		 1620 lvt_val | APIC_LVT_MASKED);
1621 }
b6ac0695 1622 apic_update_lvtt(apic);
d3c7b77d 1623 atomic_set(&apic->lapic_timer.pending, 0);
97222cc8
ED		 1624
1625 }
1626 break;
fc61b800 1627 }
97222cc8
ED		 1628 case APIC_ICR:
1629 /* No delay here, so we always clear the pending bit */
1e6e2755 1630 kvm_lapic_set_reg(apic, APIC_ICR, val & ~(1 << 12));
97222cc8
ED		 1631 apic_send_ipi(apic);
1632 break;
1633
1634 case APIC_ICR2:
0105d1a5
GN		 1635 if (!apic_x2apic_mode(apic))
1636 val &= 0xff000000;
1e6e2755 1637 kvm_lapic_set_reg(apic, APIC_ICR2, val);
97222cc8
ED		 1638 break;
1639
23930f95 1640 case APIC_LVT0:
cc6e462c 1641 apic_manage_nmi_watchdog(apic, val);
97222cc8
ED		 1642 case APIC_LVTTHMR:
1643 case APIC_LVTPC:
97222cc8
ED		 1644 case APIC_LVT1:
1645 case APIC_LVTERR:
1646 /* TODO: Check vector */
c48f1496 1647 if (!kvm_apic_sw_enabled(apic))
97222cc8
ED		 1648 val |= APIC_LVT_MASKED;
1649
0105d1a5 1650 val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4];
1e6e2755 1651 kvm_lapic_set_reg(apic, reg, val);
97222cc8
ED		 1652
1653 break;
1654
b6ac0695 1655 case APIC_LVTT:
c48f1496 1656 if (!kvm_apic_sw_enabled(apic))
a3e06bbe
LJ		 1657 val |= APIC_LVT_MASKED;
1658 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask);
1e6e2755 1659 kvm_lapic_set_reg(apic, APIC_LVTT, val);
b6ac0695 1660 apic_update_lvtt(apic);
a3e06bbe
LJ		 1661 break;
1662
97222cc8 1663 case APIC_TMICT:
a3e06bbe
LJ		 1664 if (apic_lvtt_tscdeadline(apic))
1665 break;
1666
d3c7b77d 1667 hrtimer_cancel(&apic->lapic_timer.timer);
1e6e2755 1668 kvm_lapic_set_reg(apic, APIC_TMICT, val);
97222cc8 1669 start_apic_timer(apic);
0105d1a5 1670 break;
97222cc8
ED		 1671
1672 case APIC_TDCR:
1673 if (val & 4)
7712de87 1674 apic_debug("KVM_WRITE:TDCR %x\n", val);
1e6e2755 1675 kvm_lapic_set_reg(apic, APIC_TDCR, val);
97222cc8
ED		 1676 update_divide_count(apic);
1677 break;
1678
0105d1a5
GN		 1679 case APIC_ESR:
1680 if (apic_x2apic_mode(apic) && val != 0) {
7712de87 1681 apic_debug("KVM_WRITE:ESR not zero %x\n", val);
0105d1a5
GN		 1682 ret = 1;
1683 }
1684 break;
1685
1686 case APIC_SELF_IPI:
1687 if (apic_x2apic_mode(apic)) {
1e6e2755 1688 kvm_lapic_reg_write(apic, APIC_ICR, 0x40000 | (val & 0xff));
0105d1a5
GN		 1689 } else
1690 ret = 1;
1691 break;
97222cc8 1692 default:
0105d1a5 1693 ret = 1;
97222cc8
ED		 1694 break;
1695 }
0105d1a5
GN		 1696 if (ret)
1697 apic_debug("Local APIC Write to read-only register %x\n", reg);
1698 return ret;
1699}
1e6e2755 1700EXPORT_SYMBOL_GPL(kvm_lapic_reg_write);
0105d1a5 1701
e32edf4f 1702static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
0105d1a5
GN		 1703 gpa_t address, int len, const void *data)
1704{
1705 struct kvm_lapic *apic = to_lapic(this);
1706 unsigned int offset = address - apic->base_address;
1707 u32 val;
1708
1709 if (!apic_mmio_in_range(apic, address))
1710 return -EOPNOTSUPP;
1711
1712 /*
1713 * APIC register must be aligned on 128-bits boundary.
1714 * 32/64/128 bits registers must be accessed thru 32 bits.
1715 * Refer SDM 8.4.1
1716 */
1717 if (len != 4 || (offset & 0xf)) {
1718 /* Don't shout loud, $infamous_os would cause only noise. */
1719 apic_debug("apic write: bad size=%d %lx\n", len, (long)address);
756975bb 1720 return 0;
0105d1a5
GN		 1721 }
1722
1723 val = *(u32*)data;
1724
1725 /* too common printing */
1726 if (offset != APIC_EOI)
1727 apic_debug("%s: offset 0x%x with length 0x%x, and value is "
1728 "0x%x\n", __func__, offset, len, val);
1729
1e6e2755 1730 kvm_lapic_reg_write(apic, offset & 0xff0, val);
0105d1a5 1731
bda9020e 1732 return 0;
97222cc8
ED		 1733}
1734
58fbbf26
KT		 1735void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu)
1736{
1e6e2755 1737 kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0);
58fbbf26
KT		 1738}
1739EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi);
1740
83d4c286
YZ		 1741/* emulate APIC access in a trap manner */
1742void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset)
1743{
1744 u32 val = 0;
1745
1746 /* hw has done the conditional check and inst decode */
1747 offset &= 0xff0;
1748
1e6e2755 1749 kvm_lapic_reg_read(vcpu->arch.apic, offset, 4, &val);
83d4c286
YZ		 1750
1751 /* TODO: optimize to just emulate side effect w/o one more write */
1e6e2755 1752 kvm_lapic_reg_write(vcpu->arch.apic, offset, val);
83d4c286
YZ		 1753}
1754EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode);
1755
d589444e 1756void kvm_free_lapic(struct kvm_vcpu *vcpu)
97222cc8 1757{
f8c1ea10
GN		 1758 struct kvm_lapic *apic = vcpu->arch.apic;
1759
ad312c7c 1760 if (!vcpu->arch.apic)
97222cc8
ED		 1761 return;
1762
f8c1ea10 1763 hrtimer_cancel(&apic->lapic_timer.timer);
97222cc8 1764
c5cc421b
GN		 1765 if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE))
1766 static_key_slow_dec_deferred(&apic_hw_disabled);
1767
e462755c 1768 if (!apic->sw_enabled)
f8c1ea10 1769 static_key_slow_dec_deferred(&apic_sw_disabled);
97222cc8 1770
f8c1ea10
GN		 1771 if (apic->regs)
1772 free_page((unsigned long)apic->regs);
1773
1774 kfree(apic);
97222cc8
ED		 1775}
1776
1777/*
1778 *----------------------------------------------------------------------
1779 * LAPIC interface
1780 *----------------------------------------------------------------------
1781 */
498f8162
WL		 1782u64 kvm_get_lapic_target_expiration_tsc(struct kvm_vcpu *vcpu)
1783{
1784 struct kvm_lapic *apic = vcpu->arch.apic;
1785
1786 if (!lapic_in_kernel(vcpu))
1787 return 0;
1788
1789 return apic->lapic_timer.tscdeadline;
1790}
97222cc8 1791
a3e06bbe
LJ		 1792u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu)
1793{
1794 struct kvm_lapic *apic = vcpu->arch.apic;
a3e06bbe 1795
a10388e1
WL		 1796 if (!lapic_in_kernel(vcpu) ||
1797 !apic_lvtt_tscdeadline(apic))
a3e06bbe
LJ		 1798 return 0;
1799
1800 return apic->lapic_timer.tscdeadline;
1801}
1802
1803void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data)
1804{
1805 struct kvm_lapic *apic = vcpu->arch.apic;
a3e06bbe 1806
bce87cce 1807 if (!lapic_in_kernel(vcpu) || apic_lvtt_oneshot(apic) ||
54e9818f 1808 apic_lvtt_period(apic))
a3e06bbe
LJ		 1809 return;
1810
1811 hrtimer_cancel(&apic->lapic_timer.timer);
1812 apic->lapic_timer.tscdeadline = data;
1813 start_apic_timer(apic);
1814}
1815
97222cc8
ED		 1816void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8)
1817{
ad312c7c 1818 struct kvm_lapic *apic = vcpu->arch.apic;
97222cc8 1819
b93463aa 1820 apic_set_tpr(apic, ((cr8 & 0x0f) << 4)
dfb95954 1821 | (kvm_lapic_get_reg(apic, APIC_TASKPRI) & 4));
97222cc8
ED		 1822}
1823
1824u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
1825{
97222cc8
ED		 1826 u64 tpr;
1827
dfb95954 1828 tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI);
97222cc8
ED		 1829
1830 return (tpr & 0xf0) >> 4;
1831}
1832
1833void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
1834{
8d14695f 1835 u64 old_value = vcpu->arch.apic_base;
ad312c7c 1836 struct kvm_lapic *apic = vcpu->arch.apic;
97222cc8 1837
c7dd15b3 1838 if (!apic)
97222cc8 1839 value |= MSR_IA32_APICBASE_BSP;
c5af89b6 1840
e66d2ae7
JK		 1841 vcpu->arch.apic_base = value;
1842
c7dd15b3
JM		 1843 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE)
1844 kvm_update_cpuid(vcpu);
1845
1846 if (!apic)
1847 return;
1848
c5cc421b 1849 /* update jump label if enable bit changes */
0dce7cd6 1850 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) {
49bd29ba
RK		 1851 if (value & MSR_IA32_APICBASE_ENABLE) {
1852 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
c5cc421b 1853 static_key_slow_dec_deferred(&apic_hw_disabled);
187ca84b 1854 } else {
c5cc421b 1855 static_key_slow_inc(&apic_hw_disabled.key);
187ca84b
WL		 1856 recalculate_apic_map(vcpu->kvm);
1857 }
c5cc421b
GN		 1858 }
1859
8d14695f
YZ		 1860 if ((old_value ^ value) & X2APIC_ENABLE) {
1861 if (value & X2APIC_ENABLE) {
257b9a5f 1862 kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
8d14695f
YZ		 1863 kvm_x86_ops->set_virtual_x2apic_mode(vcpu, true);
1864 } else
1865 kvm_x86_ops->set_virtual_x2apic_mode(vcpu, false);
0105d1a5 1866 }
8d14695f 1867
ad312c7c 1868 apic->base_address = apic->vcpu->arch.apic_base &
97222cc8
ED		 1869 MSR_IA32_APICBASE_BASE;
1870
db324fe6
NA		 1871 if ((value & MSR_IA32_APICBASE_ENABLE) &&
1872 apic->base_address != APIC_DEFAULT_PHYS_BASE)
1873 pr_warn_once("APIC base relocation is unsupported by KVM");
1874
97222cc8
ED		 1875 /* with FSB delivery interrupt, we can restart APIC functionality */
1876 apic_debug("apic base msr is 0x%016" PRIx64 ", and base address is "
ad312c7c 1877 "0x%lx.\n", apic->vcpu->arch.apic_base, apic->base_address);
97222cc8
ED		 1878
1879}
1880
d28bc9dd 1881void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event)
97222cc8
ED		 1882{
1883 struct kvm_lapic *apic;
1884 int i;
1885
b8688d51 1886 apic_debug("%s\n", __func__);
97222cc8
ED		 1887
1888 ASSERT(vcpu);
ad312c7c 1889 apic = vcpu->arch.apic;
97222cc8
ED		 1890 ASSERT(apic != NULL);
1891
1892 /* Stop the timer in case it's a reset to an active apic */
d3c7b77d 1893 hrtimer_cancel(&apic->lapic_timer.timer);
97222cc8 1894
4d8e772b
RK		 1895 if (!init_event) {
1896 kvm_lapic_set_base(vcpu, APIC_DEFAULT_PHYS_BASE |
1897 MSR_IA32_APICBASE_ENABLE);
a92e2543 1898 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
4d8e772b 1899 }
fc61b800 1900 kvm_apic_set_version(apic->vcpu);
97222cc8 1901
1e6e2755
SS		 1902 for (i = 0; i < KVM_APIC_LVT_NUM; i++)
1903 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED);
b6ac0695 1904 apic_update_lvtt(apic);
0da029ed 1905 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED))
1e6e2755 1906 kvm_lapic_set_reg(apic, APIC_LVT0,
90de4a18 1907 SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT));
dfb95954 1908 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
97222cc8 1909
1e6e2755 1910 kvm_lapic_set_reg(apic, APIC_DFR, 0xffffffffU);
f8c1ea10 1911 apic_set_spiv(apic, 0xff);
1e6e2755 1912 kvm_lapic_set_reg(apic, APIC_TASKPRI, 0);
c028dd6b
RK		 1913 if (!apic_x2apic_mode(apic))
1914 kvm_apic_set_ldr(apic, 0);
1e6e2755
SS		 1915 kvm_lapic_set_reg(apic, APIC_ESR, 0);
1916 kvm_lapic_set_reg(apic, APIC_ICR, 0);
1917 kvm_lapic_set_reg(apic, APIC_ICR2, 0);
1918 kvm_lapic_set_reg(apic, APIC_TDCR, 0);
1919 kvm_lapic_set_reg(apic, APIC_TMICT, 0);
97222cc8 1920 for (i = 0; i < 8; i++) {
1e6e2755
SS		 1921 kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0);
1922 kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0);
1923 kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
97222cc8 1924 }
d62caabb
AS		 1925 apic->irr_pending = vcpu->arch.apicv_active;
1926 apic->isr_count = vcpu->arch.apicv_active ? 1 : 0;
8680b94b 1927 apic->highest_isr_cache = -1;
b33ac88b 1928 update_divide_count(apic);
d3c7b77d 1929 atomic_set(&apic->lapic_timer.pending, 0);
c5af89b6 1930 if (kvm_vcpu_is_bsp(vcpu))
5dbc8f3f
GN		 1931 kvm_lapic_set_base(vcpu,
1932 vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP);
ae7a2a3f 1933 vcpu->arch.pv_eoi.msr_val = 0;
97222cc8
ED		 1934 apic_update_ppr(apic);
1935
e1035715 1936 vcpu->arch.apic_arb_prio = 0;
41383771 1937 vcpu->arch.apic_attention = 0;
e1035715 1938
6e500439 1939 apic_debug("%s: vcpu=%p, id=0x%x, base_msr="
b8688d51 1940 "0x%016" PRIx64 ", base_address=0x%0lx.\n", __func__,
6e500439 1941 vcpu, kvm_lapic_get_reg(apic, APIC_ID),
ad312c7c 1942 vcpu->arch.apic_base, apic->base_address);
97222cc8
ED		 1943}
1944
97222cc8
ED		 1945/*
1946 *----------------------------------------------------------------------
1947 * timer interface
1948 *----------------------------------------------------------------------
1949 */
1b9778da 1950
2a6eac96 1951static bool lapic_is_periodic(struct kvm_lapic *apic)
97222cc8 1952{
d3c7b77d 1953 return apic_lvtt_period(apic);
97222cc8
ED		 1954}
1955
3d80840d
MT		 1956int apic_has_pending_timer(struct kvm_vcpu *vcpu)
1957{
54e9818f 1958 struct kvm_lapic *apic = vcpu->arch.apic;
3d80840d 1959
1e3161b4 1960 if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT))
54e9818f 1961 return atomic_read(&apic->lapic_timer.pending);
3d80840d
MT		 1962
1963 return 0;
1964}
1965
89342082 1966int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type)
1b9778da 1967{
dfb95954 1968 u32 reg = kvm_lapic_get_reg(apic, lvt_type);
23930f95 1969 int vector, mode, trig_mode;
23930f95 1970
c48f1496 1971 if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) {
23930f95
JK		 1972 vector = reg & APIC_VECTOR_MASK;
1973 mode = reg & APIC_MODE_MASK;
1974 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER;
b4f2225c
YZ		 1975 return __apic_accept_irq(apic, mode, vector, 1, trig_mode,
1976 NULL);
23930f95
JK		 1977 }
1978 return 0;
1979}
1b9778da 1980
8fdb2351 1981void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu)
23930f95 1982{
8fdb2351
JK		 1983 struct kvm_lapic *apic = vcpu->arch.apic;
1984
1985 if (apic)
1986 kvm_apic_local_deliver(apic, APIC_LVT0);
1b9778da
ED		 1987}
1988
d76685c4
GH		 1989static const struct kvm_io_device_ops apic_mmio_ops = {
1990 .read = apic_mmio_read,
1991 .write = apic_mmio_write,
d76685c4
GH		 1992};
1993
e9d90d47
AK		 1994static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
1995{
1996 struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
2a6eac96 1997 struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer);
e9d90d47 1998
5d87db71 1999 apic_timer_expired(apic);
e9d90d47 2000
2a6eac96 2001 if (lapic_is_periodic(apic)) {
8003c9ae 2002 advance_periodic_target_expiration(apic);
e9d90d47
AK		 2003 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
2004 return HRTIMER_RESTART;
2005 } else
2006 return HRTIMER_NORESTART;
2007}
2008
97222cc8
ED		 2009int kvm_create_lapic(struct kvm_vcpu *vcpu)
2010{
2011 struct kvm_lapic *apic;
2012
2013 ASSERT(vcpu != NULL);
2014 apic_debug("apic_init %d\n", vcpu->vcpu_id);
2015
2016 apic = kzalloc(sizeof(*apic), GFP_KERNEL);
2017 if (!apic)
2018 goto nomem;
2019
ad312c7c 2020 vcpu->arch.apic = apic;
97222cc8 2021
afc20184
TY		 2022 apic->regs = (void *)get_zeroed_page(GFP_KERNEL);
2023 if (!apic->regs) {
97222cc8
ED		 2024 printk(KERN_ERR "malloc apic regs error for vcpu %x\n",
2025 vcpu->vcpu_id);
d589444e 2026 goto nomem_free_apic;
97222cc8 2027 }
97222cc8
ED		 2028 apic->vcpu = vcpu;
2029
d3c7b77d 2030 hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
61abdbe0 2031 HRTIMER_MODE_ABS_PINNED);
e9d90d47 2032 apic->lapic_timer.timer.function = apic_timer_fn;
d3c7b77d 2033
c5cc421b
GN		 2034 /*
2035 * APIC is created enabled. This will prevent kvm_lapic_set_base from
2036 * thinking that APIC satet has changed.
2037 */
2038 vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE;
f8c1ea10 2039 static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */
d28bc9dd 2040 kvm_lapic_reset(vcpu, false);
d76685c4 2041 kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
97222cc8
ED		 2042
2043 return 0;
d589444e
RR		 2044nomem_free_apic:
2045 kfree(apic);
97222cc8 2046nomem:
97222cc8
ED		 2047 return -ENOMEM;
2048}
97222cc8
ED		 2049
2050int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu)
2051{
ad312c7c 2052 struct kvm_lapic *apic = vcpu->arch.apic;
97222cc8
ED		 2053 int highest_irr;
2054
f8543d6a 2055 if (!apic_enabled(apic))
97222cc8
ED		 2056 return -1;
2057
6e5d865c 2058 apic_update_ppr(apic);
97222cc8
ED		 2059 highest_irr = apic_find_highest_irr(apic);
2060 if ((highest_irr == -1) ||
dfb95954 2061 ((highest_irr & 0xF0) <= kvm_lapic_get_reg(apic, APIC_PROCPRI)))
97222cc8
ED		 2062 return -1;
2063 return highest_irr;
2064}
2065
40487c68
QH		 2066int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu)
2067{
dfb95954 2068 u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0);
40487c68
QH		 2069 int r = 0;
2070
c48f1496 2071 if (!kvm_apic_hw_enabled(vcpu->arch.apic))
e7dca5c0
CL		 2072 r = 1;
2073 if ((lvt0 & APIC_LVT_MASKED) == 0 &&
2074 GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT)
2075 r = 1;
40487c68
QH		 2076 return r;
2077}
2078
1b9778da
ED		 2079void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu)
2080{
ad312c7c 2081 struct kvm_lapic *apic = vcpu->arch.apic;
1b9778da 2082
54e9818f 2083 if (atomic_read(&apic->lapic_timer.pending) > 0) {
f1ed0450 2084 kvm_apic_local_deliver(apic, APIC_LVTT);
fae0ba21
NA		 2085 if (apic_lvtt_tscdeadline(apic))
2086 apic->lapic_timer.tscdeadline = 0;
8003c9ae
WL		 2087 if (apic_lvtt_oneshot(apic)) {
2088 apic->lapic_timer.tscdeadline = 0;
8b0e1953 2089 apic->lapic_timer.target_expiration = 0;
8003c9ae 2090 }
f1ed0450 2091 atomic_set(&apic->lapic_timer.pending, 0);
1b9778da
ED		 2092 }
2093}
2094
97222cc8
ED		 2095int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu)
2096{
2097 int vector = kvm_apic_has_interrupt(vcpu);
ad312c7c 2098 struct kvm_lapic *apic = vcpu->arch.apic;
97222cc8
ED		 2099
2100 if (vector == -1)
2101 return -1;
2102
56cc2406
WL		 2103 /*
2104 * We get here even with APIC virtualization enabled, if doing
2105 * nested virtualization and L1 runs with the "acknowledge interrupt
2106 * on exit" mode. Then we cannot inject the interrupt via RVI,
2107 * because the process would deliver it through the IDT.
2108 */
2109
8680b94b 2110 apic_set_isr(vector, apic);
97222cc8
ED		 2111 apic_update_ppr(apic);
2112 apic_clear_irr(vector, apic);
5c919412
AS		 2113
2114 if (test_bit(vector, vcpu_to_synic(vcpu)->auto_eoi_bitmap)) {
2115 apic_clear_isr(vector, apic);
2116 apic_update_ppr(apic);
2117 }
2118
97222cc8
ED		 2119 return vector;
2120}
96ad2cc6 2121
a92e2543
RK		 2122static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu,
2123 struct kvm_lapic_state *s, bool set)
2124{
2125 if (apic_x2apic_mode(vcpu->arch.apic)) {
2126 u32 *id = (u32 *)(s->regs + APIC_ID);
2127
37131313
RK		 2128 if (vcpu->kvm->arch.x2apic_format) {
2129 if (*id != vcpu->vcpu_id)
2130 return -EINVAL;
2131 } else {
2132 if (set)
2133 *id >>= 24;
2134 else
2135 *id <<= 24;
2136 }
a92e2543
RK		 2137 }
2138
2139 return 0;
2140}
2141
2142int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
2143{
2144 memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s));
2145 return kvm_apic_state_fixup(vcpu, s, false);
2146}
2147
2148int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
96ad2cc6 2149{
ad312c7c 2150 struct kvm_lapic *apic = vcpu->arch.apic;
a92e2543
RK		 2151 int r;
2152
96ad2cc6 2153
5dbc8f3f 2154 kvm_lapic_set_base(vcpu, vcpu->arch.apic_base);
64eb0620
GN		 2155 /* set SPIV separately to get count of SW disabled APICs right */
2156 apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV)));
a92e2543
RK		 2157
2158 r = kvm_apic_state_fixup(vcpu, s, true);
2159 if (r)
2160 return r;
64eb0620 2161 memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s);
a92e2543
RK		 2162
2163 recalculate_apic_map(vcpu->kvm);
fc61b800
GN		 2164 kvm_apic_set_version(vcpu);
2165
96ad2cc6 2166 apic_update_ppr(apic);
d3c7b77d 2167 hrtimer_cancel(&apic->lapic_timer.timer);
b6ac0695 2168 apic_update_lvtt(apic);
dfb95954 2169 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
96ad2cc6
ED		 2170 update_divide_count(apic);
2171 start_apic_timer(apic);
6e24a6ef 2172 apic->irr_pending = true;
d62caabb 2173 apic->isr_count = vcpu->arch.apicv_active ?
c7c9c56c 2174 1 : count_vectors(apic->regs + APIC_ISR);
8680b94b 2175 apic->highest_isr_cache = -1;
d62caabb 2176 if (vcpu->arch.apicv_active) {
be8ca170
SS		 2177 if (kvm_x86_ops->apicv_post_state_restore)
2178 kvm_x86_ops->apicv_post_state_restore(vcpu);
4114c27d
WW		 2179 kvm_x86_ops->hwapic_irr_update(vcpu,
2180 apic_find_highest_irr(apic));
67c9dddc 2181 kvm_x86_ops->hwapic_isr_update(vcpu,
b4eef9b3 2182 apic_find_highest_isr(apic));
d62caabb 2183 }
3842d135 2184 kvm_make_request(KVM_REQ_EVENT, vcpu);
49df6397
SR		 2185 if (ioapic_in_kernel(vcpu->kvm))
2186 kvm_rtc_eoi_tracking_restore_one(vcpu);
0669a510
RK		 2187
2188 vcpu->arch.apic_arb_prio = 0;
a92e2543
RK		 2189
2190 return 0;
96ad2cc6 2191}
a3d7f85f 2192
2f52d58c 2193void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
a3d7f85f 2194{
a3d7f85f
ED		 2195 struct hrtimer *timer;
2196
bce87cce 2197 if (!lapic_in_kernel(vcpu))
a3d7f85f
ED		 2198 return;
2199
54e9818f 2200 timer = &vcpu->arch.apic->lapic_timer.timer;
a3d7f85f 2201 if (hrtimer_cancel(timer))
61abdbe0 2202 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED);
a3d7f85f 2203}
b93463aa 2204
ae7a2a3f
MT		 2205/*
2206 * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt
2207 *
2208 * Detect whether guest triggered PV EOI since the
2209 * last entry. If yes, set EOI on guests's behalf.
2210 * Clear PV EOI in guest memory in any case.
2211 */
2212static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu,
2213 struct kvm_lapic *apic)
2214{
2215 bool pending;
2216 int vector;
2217 /*
2218 * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host
2219 * and KVM_PV_EOI_ENABLED in guest memory as follows:
2220 *
2221 * KVM_APIC_PV_EOI_PENDING is unset:
2222 * -> host disabled PV EOI.
2223 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set:
2224 * -> host enabled PV EOI, guest did not execute EOI yet.
2225 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset:
2226 * -> host enabled PV EOI, guest executed EOI.
2227 */
2228 BUG_ON(!pv_eoi_enabled(vcpu));
2229 pending = pv_eoi_get_pending(vcpu);
2230 /*
2231 * Clear pending bit in any case: it will be set again on vmentry.
2232 * While this might not be ideal from performance point of view,
2233 * this makes sure pv eoi is only enabled when we know it's safe.
2234 */
2235 pv_eoi_clr_pending(vcpu);
2236 if (pending)
2237 return;
2238 vector = apic_set_eoi(apic);
2239 trace_kvm_pv_eoi(apic, vector);
2240}
2241
b93463aa
AK		 2242void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu)
2243{
2244 u32 data;
b93463aa 2245
ae7a2a3f
MT		 2246 if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention))
2247 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic);
2248
41383771 2249 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
b93463aa
AK		 2250 return;
2251
603242a8
NK		 2252 if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
2253 sizeof(u32)))
2254 return;
b93463aa
AK		 2255
2256 apic_set_tpr(vcpu->arch.apic, data & 0xff);
2257}
2258
ae7a2a3f
MT		 2259/*
2260 * apic_sync_pv_eoi_to_guest - called before vmentry
2261 *
2262 * Detect whether it's safe to enable PV EOI and
2263 * if yes do so.
2264 */
2265static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu,
2266 struct kvm_lapic *apic)
2267{
2268 if (!pv_eoi_enabled(vcpu) ||
2269 /* IRR set or many bits in ISR: could be nested. */
2270 apic->irr_pending ||
2271 /* Cache not set: could be safe but we don't bother. */
2272 apic->highest_isr_cache == -1 ||
2273 /* Need EOI to update ioapic. */
3bb345f3 2274 kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) {
ae7a2a3f
MT		 2275 /*
2276 * PV EOI was disabled by apic_sync_pv_eoi_from_guest
2277 * so we need not do anything here.
2278 */
2279 return;
2280 }
2281
2282 pv_eoi_set_pending(apic->vcpu);
2283}
2284
b93463aa
AK		 2285void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
2286{
2287 u32 data, tpr;
2288 int max_irr, max_isr;
ae7a2a3f 2289 struct kvm_lapic *apic = vcpu->arch.apic;
b93463aa 2290
ae7a2a3f
MT		 2291 apic_sync_pv_eoi_to_guest(vcpu, apic);
2292
41383771 2293 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
b93463aa
AK		 2294 return;
2295
dfb95954 2296 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff;
b93463aa
AK		 2297 max_irr = apic_find_highest_irr(apic);
2298 if (max_irr < 0)
2299 max_irr = 0;
2300 max_isr = apic_find_highest_isr(apic);
2301 if (max_isr < 0)
2302 max_isr = 0;
2303 data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24);
2304
fda4e2e8
AH		 2305 kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
2306 sizeof(u32));
b93463aa
AK		 2307}
2308
fda4e2e8 2309int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr)
b93463aa 2310{
fda4e2e8
AH		 2311 if (vapic_addr) {
2312 if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
2313 &vcpu->arch.apic->vapic_cache,
2314 vapic_addr, sizeof(u32)))
2315 return -EINVAL;
41383771 2316 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
fda4e2e8 2317 } else {
41383771 2318 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
fda4e2e8
AH		 2319 }
2320
2321 vcpu->arch.apic->vapic_addr = vapic_addr;
2322 return 0;
b93463aa 2323}
0105d1a5
GN		 2324
2325int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data)
2326{
2327 struct kvm_lapic *apic = vcpu->arch.apic;
2328 u32 reg = (msr - APIC_BASE_MSR) << 4;
2329
35754c98 2330 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
0105d1a5
GN		 2331 return 1;
2332
c69d3d9b
NA		 2333 if (reg == APIC_ICR2)
2334 return 1;
2335
0105d1a5 2336 /* if this is ICR write vector before command */
decdc283 2337 if (reg == APIC_ICR)
1e6e2755
SS		 2338 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
2339 return kvm_lapic_reg_write(apic, reg, (u32)data);
0105d1a5
GN		 2340}
2341
2342int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data)
2343{
2344 struct kvm_lapic *apic = vcpu->arch.apic;
2345 u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0;
2346
35754c98 2347 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
0105d1a5
GN		 2348 return 1;
2349
c69d3d9b
NA		 2350 if (reg == APIC_DFR || reg == APIC_ICR2) {
2351 apic_debug("KVM_APIC_READ: read x2apic reserved register %x\n",
2352 reg);
2353 return 1;
2354 }
2355
1e6e2755 2356 if (kvm_lapic_reg_read(apic, reg, 4, &low))
0105d1a5 2357 return 1;
decdc283 2358 if (reg == APIC_ICR)
1e6e2755 2359 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
0105d1a5
GN		 2360
2361 *data = (((u64)high) << 32) | low;
2362
2363 return 0;
2364}
10388a07
GN		 2365
2366int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data)
2367{
2368 struct kvm_lapic *apic = vcpu->arch.apic;
2369
bce87cce 2370 if (!lapic_in_kernel(vcpu))
10388a07
GN		 2371 return 1;
2372
2373 /* if this is ICR write vector before command */
2374 if (reg == APIC_ICR)
1e6e2755
SS		 2375 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
2376 return kvm_lapic_reg_write(apic, reg, (u32)data);
10388a07
GN		 2377}
2378
2379int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data)
2380{
2381 struct kvm_lapic *apic = vcpu->arch.apic;
2382 u32 low, high = 0;
2383
bce87cce 2384 if (!lapic_in_kernel(vcpu))
10388a07
GN		 2385 return 1;
2386
1e6e2755 2387 if (kvm_lapic_reg_read(apic, reg, 4, &low))
10388a07
GN		 2388 return 1;
2389 if (reg == APIC_ICR)
1e6e2755 2390 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
10388a07
GN		 2391
2392 *data = (((u64)high) << 32) | low;
2393
2394 return 0;
2395}
ae7a2a3f
MT		 2396
2397int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data)
2398{
2399 u64 addr = data & ~KVM_MSR_ENABLED;
2400 if (!IS_ALIGNED(addr, 4))
2401 return 1;
2402
2403 vcpu->arch.pv_eoi.msr_val = data;
2404 if (!pv_eoi_enabled(vcpu))
2405 return 0;
2406 return kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.pv_eoi.data,
8f964525 2407 addr, sizeof(u8));
ae7a2a3f 2408}
c5cc421b 2409
66450a21
JK		 2410void kvm_apic_accept_events(struct kvm_vcpu *vcpu)
2411{
2412 struct kvm_lapic *apic = vcpu->arch.apic;
2b4a273b 2413 u8 sipi_vector;
299018f4 2414 unsigned long pe;
66450a21 2415
bce87cce 2416 if (!lapic_in_kernel(vcpu) || !apic->pending_events)
66450a21
JK		 2417 return;
2418
cd7764fe
PB		 2419 /*
2420 * INITs are latched while in SMM. Because an SMM CPU cannot
2421 * be in KVM_MP_STATE_INIT_RECEIVED state, just eat SIPIs
2422 * and delay processing of INIT until the next RSM.
2423 */
2424 if (is_smm(vcpu)) {
2425 WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED);
2426 if (test_bit(KVM_APIC_SIPI, &apic->pending_events))
2427 clear_bit(KVM_APIC_SIPI, &apic->pending_events);
2428 return;
2429 }
299018f4 2430
cd7764fe 2431 pe = xchg(&apic->pending_events, 0);
299018f4 2432 if (test_bit(KVM_APIC_INIT, &pe)) {
d28bc9dd
NA		 2433 kvm_lapic_reset(vcpu, true);
2434 kvm_vcpu_reset(vcpu, true);
66450a21
JK		 2435 if (kvm_vcpu_is_bsp(apic->vcpu))
2436 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
2437 else
2438 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
2439 }
299018f4 2440 if (test_bit(KVM_APIC_SIPI, &pe) &&
66450a21
JK		 2441 vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
2442 /* evaluate pending_events before reading the vector */
2443 smp_rmb();
2444 sipi_vector = apic->sipi_vector;
98eff52a 2445 apic_debug("vcpu %d received sipi with vector # %x\n",
66450a21
JK		 2446 vcpu->vcpu_id, sipi_vector);
2447 kvm_vcpu_deliver_sipi_vector(vcpu, sipi_vector);
2448 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
2449 }
2450}
2451
c5cc421b
GN		 2452void kvm_lapic_init(void)
2453{
2454 /* do not patch jump label more than once per second */
2455 jump_label_rate_limit(&apic_hw_disabled, HZ);
f8c1ea10 2456 jump_label_rate_limit(&apic_sw_disabled, HZ);
c5cc421b 2457}
Ubuntu Focal Linux kernel mirror