]>
Commit | Line | Data |
---|---|---|
53aaf262 TG |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _ASM_X86_IDTENTRY_H | |
3 | #define _ASM_X86_IDTENTRY_H | |
4 | ||
5 | /* Interrupts/Exceptions */ | |
6 | #include <asm/trapnr.h> | |
7 | ||
8 | #ifndef __ASSEMBLY__ | |
27d6b4d1 | 9 | #include <linux/entry-common.h> |
6368558c TG |
10 | #include <linux/hardirq.h> |
11 | ||
12 | #include <asm/irq_stack.h> | |
53aaf262 TG |
13 | |
14 | /** | |
15 | * DECLARE_IDTENTRY - Declare functions for simple IDT entry points | |
16 | * No error code pushed by hardware | |
17 | * @vector: Vector number (ignored for C) | |
18 | * @func: Function name of the entry point | |
19 | * | |
20 | * Declares three functions: | |
21 | * - The ASM entry point: asm_##func | |
22 | * - The XEN PV trap entry point: xen_##func (maybe unused) | |
23 | * - The C handler called from the ASM entry point | |
24 | * | |
25 | * Note: This is the C variant of DECLARE_IDTENTRY(). As the name says it | |
26 | * declares the entry points for usage in C code. There is an ASM variant | |
27 | * as well which is used to emit the entry stubs in entry_32/64.S. | |
28 | */ | |
29 | #define DECLARE_IDTENTRY(vector, func) \ | |
30 | asmlinkage void asm_##func(void); \ | |
31 | asmlinkage void xen_asm_##func(void); \ | |
32 | __visible void func(struct pt_regs *regs) | |
33 | ||
34 | /** | |
35 | * DEFINE_IDTENTRY - Emit code for simple IDT entry points | |
36 | * @func: Function name of the entry point | |
37 | * | |
38 | * @func is called from ASM entry code with interrupts disabled. | |
39 | * | |
40 | * The macro is written so it acts as function definition. Append the | |
41 | * body with a pair of curly brackets. | |
42 | * | |
a27a0a55 TG |
43 | * irqentry_enter() contains common code which has to be invoked before |
44 | * arbitrary code in the body. irqentry_exit() contains common code | |
53aaf262 TG |
45 | * which has to run before returning to the low level assembly code. |
46 | */ | |
47 | #define DEFINE_IDTENTRY(func) \ | |
48 | static __always_inline void __##func(struct pt_regs *regs); \ | |
49 | \ | |
50 | __visible noinstr void func(struct pt_regs *regs) \ | |
51 | { \ | |
a27a0a55 | 52 | irqentry_state_t state = irqentry_enter(regs); \ |
fa95d7dc | 53 | \ |
53aaf262 TG |
54 | instrumentation_begin(); \ |
55 | __##func (regs); \ | |
56 | instrumentation_end(); \ | |
a27a0a55 | 57 | irqentry_exit(regs, state); \ |
53aaf262 TG |
58 | } \ |
59 | \ | |
60 | static __always_inline void __##func(struct pt_regs *regs) | |
61 | ||
d7729050 TG |
62 | /* Special case for 32bit IRET 'trap' */ |
63 | #define DECLARE_IDTENTRY_SW DECLARE_IDTENTRY | |
64 | #define DEFINE_IDTENTRY_SW DEFINE_IDTENTRY | |
65 | ||
aabfe538 TG |
66 | /** |
67 | * DECLARE_IDTENTRY_ERRORCODE - Declare functions for simple IDT entry points | |
68 | * Error code pushed by hardware | |
69 | * @vector: Vector number (ignored for C) | |
70 | * @func: Function name of the entry point | |
71 | * | |
72 | * Declares three functions: | |
73 | * - The ASM entry point: asm_##func | |
74 | * - The XEN PV trap entry point: xen_##func (maybe unused) | |
75 | * - The C handler called from the ASM entry point | |
76 | * | |
77 | * Same as DECLARE_IDTENTRY, but has an extra error_code argument for the | |
78 | * C-handler. | |
79 | */ | |
80 | #define DECLARE_IDTENTRY_ERRORCODE(vector, func) \ | |
81 | asmlinkage void asm_##func(void); \ | |
82 | asmlinkage void xen_asm_##func(void); \ | |
83 | __visible void func(struct pt_regs *regs, unsigned long error_code) | |
84 | ||
85 | /** | |
86 | * DEFINE_IDTENTRY_ERRORCODE - Emit code for simple IDT entry points | |
87 | * Error code pushed by hardware | |
88 | * @func: Function name of the entry point | |
89 | * | |
90 | * Same as DEFINE_IDTENTRY, but has an extra error_code argument | |
91 | */ | |
92 | #define DEFINE_IDTENTRY_ERRORCODE(func) \ | |
93 | static __always_inline void __##func(struct pt_regs *regs, \ | |
94 | unsigned long error_code); \ | |
95 | \ | |
96 | __visible noinstr void func(struct pt_regs *regs, \ | |
97 | unsigned long error_code) \ | |
98 | { \ | |
a27a0a55 | 99 | irqentry_state_t state = irqentry_enter(regs); \ |
fa95d7dc | 100 | \ |
aabfe538 TG |
101 | instrumentation_begin(); \ |
102 | __##func (regs, error_code); \ | |
103 | instrumentation_end(); \ | |
a27a0a55 | 104 | irqentry_exit(regs, state); \ |
aabfe538 TG |
105 | } \ |
106 | \ | |
107 | static __always_inline void __##func(struct pt_regs *regs, \ | |
108 | unsigned long error_code) | |
109 | ||
0dc6cdc2 TG |
110 | /** |
111 | * DECLARE_IDTENTRY_RAW - Declare functions for raw IDT entry points | |
112 | * No error code pushed by hardware | |
113 | * @vector: Vector number (ignored for C) | |
114 | * @func: Function name of the entry point | |
115 | * | |
116 | * Maps to DECLARE_IDTENTRY(). | |
117 | */ | |
118 | #define DECLARE_IDTENTRY_RAW(vector, func) \ | |
119 | DECLARE_IDTENTRY(vector, func) | |
120 | ||
121 | /** | |
122 | * DEFINE_IDTENTRY_RAW - Emit code for raw IDT entry points | |
123 | * @func: Function name of the entry point | |
124 | * | |
125 | * @func is called from ASM entry code with interrupts disabled. | |
126 | * | |
127 | * The macro is written so it acts as function definition. Append the | |
128 | * body with a pair of curly brackets. | |
129 | * | |
130 | * Contrary to DEFINE_IDTENTRY() this does not invoke the | |
131 | * idtentry_enter/exit() helpers before and after the body invocation. This | |
132 | * needs to be done in the body itself if applicable. Use if extra work | |
133 | * is required before the enter/exit() helpers are invoked. | |
134 | */ | |
135 | #define DEFINE_IDTENTRY_RAW(func) \ | |
136 | __visible noinstr void func(struct pt_regs *regs) | |
137 | ||
6a8dfa8e TG |
138 | /** |
139 | * DECLARE_IDTENTRY_RAW_ERRORCODE - Declare functions for raw IDT entry points | |
140 | * Error code pushed by hardware | |
141 | * @vector: Vector number (ignored for C) | |
142 | * @func: Function name of the entry point | |
143 | * | |
144 | * Maps to DECLARE_IDTENTRY_ERRORCODE() | |
145 | */ | |
146 | #define DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func) \ | |
147 | DECLARE_IDTENTRY_ERRORCODE(vector, func) | |
148 | ||
149 | /** | |
150 | * DEFINE_IDTENTRY_RAW_ERRORCODE - Emit code for raw IDT entry points | |
151 | * @func: Function name of the entry point | |
152 | * | |
153 | * @func is called from ASM entry code with interrupts disabled. | |
154 | * | |
155 | * The macro is written so it acts as function definition. Append the | |
156 | * body with a pair of curly brackets. | |
157 | * | |
158 | * Contrary to DEFINE_IDTENTRY_ERRORCODE() this does not invoke the | |
a27a0a55 | 159 | * irqentry_enter/exit() helpers before and after the body invocation. This |
6a8dfa8e TG |
160 | * needs to be done in the body itself if applicable. Use if extra work |
161 | * is required before the enter/exit() helpers are invoked. | |
162 | */ | |
163 | #define DEFINE_IDTENTRY_RAW_ERRORCODE(func) \ | |
164 | __visible noinstr void func(struct pt_regs *regs, unsigned long error_code) | |
165 | ||
0bf7c314 TG |
166 | /** |
167 | * DECLARE_IDTENTRY_IRQ - Declare functions for device interrupt IDT entry | |
168 | * points (common/spurious) | |
169 | * @vector: Vector number (ignored for C) | |
170 | * @func: Function name of the entry point | |
171 | * | |
172 | * Maps to DECLARE_IDTENTRY_ERRORCODE() | |
173 | */ | |
174 | #define DECLARE_IDTENTRY_IRQ(vector, func) \ | |
175 | DECLARE_IDTENTRY_ERRORCODE(vector, func) | |
176 | ||
177 | /** | |
178 | * DEFINE_IDTENTRY_IRQ - Emit code for device interrupt IDT entry points | |
179 | * @func: Function name of the entry point | |
180 | * | |
181 | * The vector number is pushed by the low level entry stub and handed | |
182 | * to the function as error_code argument which needs to be truncated | |
183 | * to an u8 because the push is sign extending. | |
184 | * | |
0bf7c314 | 185 | * irq_enter/exit_rcu() are invoked before the function body and the |
790ce3b4 TG |
186 | * KVM L1D flush request is set. Stack switching to the interrupt stack |
187 | * has to be done in the function body if necessary. | |
0bf7c314 TG |
188 | */ |
189 | #define DEFINE_IDTENTRY_IRQ(func) \ | |
5b51e1db | 190 | static void __##func(struct pt_regs *regs, u32 vector); \ |
0bf7c314 TG |
191 | \ |
192 | __visible noinstr void func(struct pt_regs *regs, \ | |
193 | unsigned long error_code) \ | |
194 | { \ | |
a27a0a55 | 195 | irqentry_state_t state = irqentry_enter(regs); \ |
5b51e1db | 196 | u32 vector = (u32)(u8)error_code; \ |
0bf7c314 TG |
197 | \ |
198 | instrumentation_begin(); \ | |
0bf7c314 | 199 | kvm_set_cpu_l1tf_flush_l1d(); \ |
5b51e1db | 200 | run_irq_on_irqstack_cond(__##func, regs, vector); \ |
0bf7c314 | 201 | instrumentation_end(); \ |
a27a0a55 | 202 | irqentry_exit(regs, state); \ |
0bf7c314 TG |
203 | } \ |
204 | \ | |
5b51e1db | 205 | static noinline void __##func(struct pt_regs *regs, u32 vector) |
0bf7c314 | 206 | |
6368558c TG |
207 | /** |
208 | * DECLARE_IDTENTRY_SYSVEC - Declare functions for system vector entry points | |
209 | * @vector: Vector number (ignored for C) | |
210 | * @func: Function name of the entry point | |
211 | * | |
212 | * Declares three functions: | |
213 | * - The ASM entry point: asm_##func | |
214 | * - The XEN PV trap entry point: xen_##func (maybe unused) | |
215 | * - The C handler called from the ASM entry point | |
216 | * | |
217 | * Maps to DECLARE_IDTENTRY(). | |
218 | */ | |
219 | #define DECLARE_IDTENTRY_SYSVEC(vector, func) \ | |
220 | DECLARE_IDTENTRY(vector, func) | |
221 | ||
222 | /** | |
223 | * DEFINE_IDTENTRY_SYSVEC - Emit code for system vector IDT entry points | |
224 | * @func: Function name of the entry point | |
225 | * | |
a27a0a55 | 226 | * irqentry_enter/exit() and irq_enter/exit_rcu() are invoked before the |
6368558c TG |
227 | * function body. KVM L1D flush request is set. |
228 | * | |
229 | * Runs the function on the interrupt stack if the entry hit kernel mode | |
230 | */ | |
231 | #define DEFINE_IDTENTRY_SYSVEC(func) \ | |
232 | static void __##func(struct pt_regs *regs); \ | |
233 | \ | |
234 | __visible noinstr void func(struct pt_regs *regs) \ | |
235 | { \ | |
a27a0a55 | 236 | irqentry_state_t state = irqentry_enter(regs); \ |
6368558c TG |
237 | \ |
238 | instrumentation_begin(); \ | |
6368558c | 239 | kvm_set_cpu_l1tf_flush_l1d(); \ |
a7b3474c | 240 | run_sysvec_on_irqstack_cond(__##func, regs); \ |
6368558c | 241 | instrumentation_end(); \ |
a27a0a55 | 242 | irqentry_exit(regs, state); \ |
6368558c TG |
243 | } \ |
244 | \ | |
245 | static noinline void __##func(struct pt_regs *regs) | |
246 | ||
247 | /** | |
248 | * DEFINE_IDTENTRY_SYSVEC_SIMPLE - Emit code for simple system vector IDT | |
249 | * entry points | |
250 | * @func: Function name of the entry point | |
251 | * | |
252 | * Runs the function on the interrupted stack. No switch to IRQ stack and | |
253 | * only the minimal __irq_enter/exit() handling. | |
254 | * | |
255 | * Only use for 'empty' vectors like reschedule IPI and KVM posted | |
256 | * interrupt vectors. | |
257 | */ | |
258 | #define DEFINE_IDTENTRY_SYSVEC_SIMPLE(func) \ | |
259 | static __always_inline void __##func(struct pt_regs *regs); \ | |
260 | \ | |
261 | __visible noinstr void func(struct pt_regs *regs) \ | |
262 | { \ | |
a27a0a55 | 263 | irqentry_state_t state = irqentry_enter(regs); \ |
6368558c TG |
264 | \ |
265 | instrumentation_begin(); \ | |
266 | __irq_enter_raw(); \ | |
267 | kvm_set_cpu_l1tf_flush_l1d(); \ | |
268 | __##func (regs); \ | |
269 | __irq_exit_raw(); \ | |
270 | instrumentation_end(); \ | |
a27a0a55 | 271 | irqentry_exit(regs, state); \ |
6368558c TG |
272 | } \ |
273 | \ | |
274 | static __always_inline void __##func(struct pt_regs *regs) | |
275 | ||
2f6474e4 TG |
276 | /** |
277 | * DECLARE_IDTENTRY_XENCB - Declare functions for XEN HV callback entry point | |
278 | * @vector: Vector number (ignored for C) | |
279 | * @func: Function name of the entry point | |
280 | * | |
281 | * Declares three functions: | |
282 | * - The ASM entry point: asm_##func | |
283 | * - The XEN PV trap entry point: xen_##func (maybe unused) | |
284 | * - The C handler called from the ASM entry point | |
285 | * | |
286 | * Maps to DECLARE_IDTENTRY(). Distinct entry point to handle the 32/64-bit | |
287 | * difference | |
288 | */ | |
289 | #define DECLARE_IDTENTRY_XENCB(vector, func) \ | |
290 | DECLARE_IDTENTRY(vector, func) | |
6a8dfa8e | 291 | |
2c058b03 TG |
292 | #ifdef CONFIG_X86_64 |
293 | /** | |
294 | * DECLARE_IDTENTRY_IST - Declare functions for IST handling IDT entry points | |
295 | * @vector: Vector number (ignored for C) | |
296 | * @func: Function name of the entry point | |
297 | * | |
f08e32ec TG |
298 | * Maps to DECLARE_IDTENTRY_RAW, but declares also the NOIST C handler |
299 | * which is called from the ASM entry point on user mode entry | |
2c058b03 TG |
300 | */ |
301 | #define DECLARE_IDTENTRY_IST(vector, func) \ | |
f08e32ec TG |
302 | DECLARE_IDTENTRY_RAW(vector, func); \ |
303 | __visible void noist_##func(struct pt_regs *regs) | |
2c058b03 | 304 | |
a13644f3 JR |
305 | /** |
306 | * DECLARE_IDTENTRY_VC - Declare functions for the VC entry point | |
307 | * @vector: Vector number (ignored for C) | |
308 | * @func: Function name of the entry point | |
309 | * | |
310 | * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE, but declares also the | |
311 | * safe_stack C handler. | |
312 | */ | |
313 | #define DECLARE_IDTENTRY_VC(vector, func) \ | |
314 | DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func); \ | |
0786138c | 315 | __visible noinstr void ist_##func(struct pt_regs *regs, unsigned long error_code); \ |
a13644f3 JR |
316 | __visible noinstr void safe_stack_##func(struct pt_regs *regs, unsigned long error_code) |
317 | ||
2c058b03 TG |
318 | /** |
319 | * DEFINE_IDTENTRY_IST - Emit code for IST entry points | |
320 | * @func: Function name of the entry point | |
321 | * | |
322 | * Maps to DEFINE_IDTENTRY_RAW | |
323 | */ | |
324 | #define DEFINE_IDTENTRY_IST(func) \ | |
325 | DEFINE_IDTENTRY_RAW(func) | |
326 | ||
f08e32ec TG |
327 | /** |
328 | * DEFINE_IDTENTRY_NOIST - Emit code for NOIST entry points which | |
329 | * belong to a IST entry point (MCE, DB) | |
330 | * @func: Function name of the entry point. Must be the same as | |
331 | * the function name of the corresponding IST variant | |
332 | * | |
333 | * Maps to DEFINE_IDTENTRY_RAW(). | |
334 | */ | |
335 | #define DEFINE_IDTENTRY_NOIST(func) \ | |
336 | DEFINE_IDTENTRY_RAW(noist_##func) | |
337 | ||
6a8dfa8e TG |
338 | /** |
339 | * DECLARE_IDTENTRY_DF - Declare functions for double fault | |
340 | * @vector: Vector number (ignored for C) | |
341 | * @func: Function name of the entry point | |
342 | * | |
343 | * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE | |
344 | */ | |
345 | #define DECLARE_IDTENTRY_DF(vector, func) \ | |
346 | DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func) | |
347 | ||
348 | /** | |
349 | * DEFINE_IDTENTRY_DF - Emit code for double fault | |
350 | * @func: Function name of the entry point | |
351 | * | |
352 | * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE | |
353 | */ | |
354 | #define DEFINE_IDTENTRY_DF(func) \ | |
355 | DEFINE_IDTENTRY_RAW_ERRORCODE(func) | |
356 | ||
a13644f3 JR |
357 | /** |
358 | * DEFINE_IDTENTRY_VC_SAFE_STACK - Emit code for VMM communication handler | |
359 | which runs on a safe stack. | |
360 | * @func: Function name of the entry point | |
361 | * | |
362 | * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE | |
363 | */ | |
364 | #define DEFINE_IDTENTRY_VC_SAFE_STACK(func) \ | |
365 | DEFINE_IDTENTRY_RAW_ERRORCODE(safe_stack_##func) | |
366 | ||
367 | /** | |
368 | * DEFINE_IDTENTRY_VC_IST - Emit code for VMM communication handler | |
369 | which runs on the VC fall-back stack | |
370 | * @func: Function name of the entry point | |
371 | * | |
372 | * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE | |
373 | */ | |
374 | #define DEFINE_IDTENTRY_VC_IST(func) \ | |
375 | DEFINE_IDTENTRY_RAW_ERRORCODE(ist_##func) | |
376 | ||
377 | /** | |
378 | * DEFINE_IDTENTRY_VC - Emit code for VMM communication handler | |
379 | * @func: Function name of the entry point | |
380 | * | |
381 | * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE | |
382 | */ | |
383 | #define DEFINE_IDTENTRY_VC(func) \ | |
384 | DEFINE_IDTENTRY_RAW_ERRORCODE(func) | |
385 | ||
2c058b03 | 386 | #else /* CONFIG_X86_64 */ |
6a8dfa8e | 387 | |
6a8dfa8e TG |
388 | /** |
389 | * DECLARE_IDTENTRY_DF - Declare functions for double fault 32bit variant | |
390 | * @vector: Vector number (ignored for C) | |
391 | * @func: Function name of the entry point | |
392 | * | |
393 | * Declares two functions: | |
394 | * - The ASM entry point: asm_##func | |
395 | * - The C handler called from the C shim | |
396 | */ | |
397 | #define DECLARE_IDTENTRY_DF(vector, func) \ | |
398 | asmlinkage void asm_##func(void); \ | |
399 | __visible void func(struct pt_regs *regs, \ | |
400 | unsigned long error_code, \ | |
401 | unsigned long address) | |
402 | ||
403 | /** | |
404 | * DEFINE_IDTENTRY_DF - Emit code for double fault on 32bit | |
405 | * @func: Function name of the entry point | |
406 | * | |
407 | * This is called through the doublefault shim which already provides | |
408 | * cr2 in the address argument. | |
409 | */ | |
410 | #define DEFINE_IDTENTRY_DF(func) \ | |
411 | __visible noinstr void func(struct pt_regs *regs, \ | |
412 | unsigned long error_code, \ | |
413 | unsigned long address) | |
414 | ||
2c058b03 TG |
415 | #endif /* !CONFIG_X86_64 */ |
416 | ||
417 | /* C-Code mapping */ | |
13cbc0cd AL |
418 | #define DECLARE_IDTENTRY_NMI DECLARE_IDTENTRY_RAW |
419 | #define DEFINE_IDTENTRY_NMI DEFINE_IDTENTRY_RAW | |
420 | ||
421 | #ifdef CONFIG_X86_64 | |
2c058b03 TG |
422 | #define DECLARE_IDTENTRY_MCE DECLARE_IDTENTRY_IST |
423 | #define DEFINE_IDTENTRY_MCE DEFINE_IDTENTRY_IST | |
f08e32ec | 424 | #define DEFINE_IDTENTRY_MCE_USER DEFINE_IDTENTRY_NOIST |
2c058b03 | 425 | |
2c058b03 TG |
426 | #define DECLARE_IDTENTRY_DEBUG DECLARE_IDTENTRY_IST |
427 | #define DEFINE_IDTENTRY_DEBUG DEFINE_IDTENTRY_IST | |
f08e32ec | 428 | #define DEFINE_IDTENTRY_DEBUG_USER DEFINE_IDTENTRY_NOIST |
13cbc0cd | 429 | #endif |
2c058b03 | 430 | |
53aaf262 TG |
431 | #else /* !__ASSEMBLY__ */ |
432 | ||
433 | /* | |
434 | * The ASM variants for DECLARE_IDTENTRY*() which emit the ASM entry stubs. | |
435 | */ | |
436 | #define DECLARE_IDTENTRY(vector, func) \ | |
e2dcb5f1 | 437 | idtentry vector asm_##func func has_error_code=0 |
53aaf262 | 438 | |
aabfe538 | 439 | #define DECLARE_IDTENTRY_ERRORCODE(vector, func) \ |
e2dcb5f1 | 440 | idtentry vector asm_##func func has_error_code=1 |
aabfe538 | 441 | |
d7729050 TG |
442 | /* Special case for 32bit IRET 'trap'. Do not emit ASM code */ |
443 | #define DECLARE_IDTENTRY_SW(vector, func) | |
444 | ||
0dc6cdc2 TG |
445 | #define DECLARE_IDTENTRY_RAW(vector, func) \ |
446 | DECLARE_IDTENTRY(vector, func) | |
447 | ||
6a8dfa8e TG |
448 | #define DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func) \ |
449 | DECLARE_IDTENTRY_ERRORCODE(vector, func) | |
450 | ||
0bf7c314 TG |
451 | /* Entries for common/spurious (device) interrupts */ |
452 | #define DECLARE_IDTENTRY_IRQ(vector, func) \ | |
453 | idtentry_irq vector func | |
454 | ||
6368558c TG |
455 | /* System vector entries */ |
456 | #define DECLARE_IDTENTRY_SYSVEC(vector, func) \ | |
457 | idtentry_sysvec vector func | |
458 | ||
2c058b03 TG |
459 | #ifdef CONFIG_X86_64 |
460 | # define DECLARE_IDTENTRY_MCE(vector, func) \ | |
461 | idtentry_mce_db vector asm_##func func | |
462 | ||
463 | # define DECLARE_IDTENTRY_DEBUG(vector, func) \ | |
464 | idtentry_mce_db vector asm_##func func | |
465 | ||
6a8dfa8e TG |
466 | # define DECLARE_IDTENTRY_DF(vector, func) \ |
467 | idtentry_df vector asm_##func func | |
468 | ||
2f6474e4 TG |
469 | # define DECLARE_IDTENTRY_XENCB(vector, func) \ |
470 | DECLARE_IDTENTRY(vector, func) | |
471 | ||
a13644f3 JR |
472 | # define DECLARE_IDTENTRY_VC(vector, func) \ |
473 | idtentry_vc vector asm_##func func | |
474 | ||
2c058b03 TG |
475 | #else |
476 | # define DECLARE_IDTENTRY_MCE(vector, func) \ | |
477 | DECLARE_IDTENTRY(vector, func) | |
478 | ||
6a8dfa8e TG |
479 | /* No ASM emitted for DF as this goes through a C shim */ |
480 | # define DECLARE_IDTENTRY_DF(vector, func) | |
481 | ||
2f6474e4 TG |
482 | /* No ASM emitted for XEN hypervisor callback */ |
483 | # define DECLARE_IDTENTRY_XENCB(vector, func) | |
484 | ||
2c058b03 TG |
485 | #endif |
486 | ||
487 | /* No ASM code emitted for NMI */ | |
488 | #define DECLARE_IDTENTRY_NMI(vector, func) | |
489 | ||
633260fa TG |
490 | /* |
491 | * ASM code to emit the common vector entry stubs where each stub is | |
492 | * packed into 8 bytes. | |
493 | * | |
494 | * Note, that the 'pushq imm8' is emitted via '.byte 0x6a, vector' because | |
495 | * GCC treats the local vector variable as unsigned int and would expand | |
496 | * all vectors above 0x7F to a 5 byte push. The original code did an | |
497 | * adjustment of the vector number to be in the signed byte range to avoid | |
498 | * this. While clever it's mindboggling counterintuitive and requires the | |
499 | * odd conversion back to a real vector number in the C entry points. Using | |
500 | * .byte achieves the same thing and the only fixup needed in the C entry | |
501 | * point is to mask off the bits above bit 7 because the push is sign | |
502 | * extending. | |
503 | */ | |
504 | .align 8 | |
505 | SYM_CODE_START(irq_entries_start) | |
506 | vector=FIRST_EXTERNAL_VECTOR | |
633260fa TG |
507 | .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR) |
508 | UNWIND_HINT_IRET_REGS | |
6ee93f8d | 509 | 0 : |
633260fa | 510 | .byte 0x6a, vector |
fa5e5c40 | 511 | jmp asm_common_interrupt |
633260fa TG |
512 | nop |
513 | /* Ensure that the above is 8 bytes max */ | |
6ee93f8d JC |
514 | . = 0b + 8 |
515 | vector = vector+1 | |
633260fa TG |
516 | .endr |
517 | SYM_CODE_END(irq_entries_start) | |
518 | ||
519 | #ifdef CONFIG_X86_LOCAL_APIC | |
520 | .align 8 | |
521 | SYM_CODE_START(spurious_entries_start) | |
522 | vector=FIRST_SYSTEM_VECTOR | |
633260fa TG |
523 | .rept (NR_VECTORS - FIRST_SYSTEM_VECTOR) |
524 | UNWIND_HINT_IRET_REGS | |
6ee93f8d | 525 | 0 : |
633260fa | 526 | .byte 0x6a, vector |
fa5e5c40 | 527 | jmp asm_spurious_interrupt |
633260fa TG |
528 | nop |
529 | /* Ensure that the above is 8 bytes max */ | |
6ee93f8d JC |
530 | . = 0b + 8 |
531 | vector = vector+1 | |
633260fa TG |
532 | .endr |
533 | SYM_CODE_END(spurious_entries_start) | |
534 | #endif | |
535 | ||
53aaf262 TG |
536 | #endif /* __ASSEMBLY__ */ |
537 | ||
9d06c402 TG |
538 | /* |
539 | * The actual entry points. Note that DECLARE_IDTENTRY*() serves two | |
540 | * purposes: | |
541 | * - provide the function declarations when included from C-Code | |
542 | * - emit the ASM stubs when included from entry_32/64.S | |
543 | * | |
544 | * This avoids duplicate defines and ensures that everything is consistent. | |
545 | */ | |
546 | ||
2f6474e4 TG |
547 | /* |
548 | * Dummy trap number so the low level ASM macro vector number checks do not | |
549 | * match which results in emitting plain IDTENTRY stubs without bells and | |
d9f6e12f | 550 | * whistles. |
2f6474e4 TG |
551 | */ |
552 | #define X86_TRAP_OTHER 0xFFFF | |
553 | ||
9d06c402 TG |
554 | /* Simple exception entry points. No hardware error code */ |
555 | DECLARE_IDTENTRY(X86_TRAP_DE, exc_divide_error); | |
4b6b9111 | 556 | DECLARE_IDTENTRY(X86_TRAP_OF, exc_overflow); |
58d9c81f | 557 | DECLARE_IDTENTRY(X86_TRAP_BR, exc_bounds); |
866ae2cc | 558 | DECLARE_IDTENTRY(X86_TRAP_NM, exc_device_not_available); |
f95658fd | 559 | DECLARE_IDTENTRY(X86_TRAP_OLD_MF, exc_coproc_segment_overrun); |
dad7106f | 560 | DECLARE_IDTENTRY(X86_TRAP_SPURIOUS, exc_spurious_interrupt_bug); |
14a8bd2a | 561 | DECLARE_IDTENTRY(X86_TRAP_MF, exc_coprocessor_error); |
48227e21 | 562 | DECLARE_IDTENTRY(X86_TRAP_XF, exc_simd_coprocessor_error); |
9d06c402 | 563 | |
d7729050 TG |
564 | /* 32bit software IRET trap. Do not emit ASM code */ |
565 | DECLARE_IDTENTRY_SW(X86_TRAP_IRET, iret_error); | |
566 | ||
97b3d290 TG |
567 | /* Simple exception entries with error code pushed by hardware */ |
568 | DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_TS, exc_invalid_tss); | |
99a3fb8d | 569 | DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_NP, exc_segment_not_present); |
fd9689bf | 570 | DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_SS, exc_stack_segment); |
be4c11af | 571 | DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_GP, exc_general_protection); |
436608bb | 572 | DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_AC, exc_alignment_check); |
97b3d290 | 573 | |
8edd7e37 | 574 | /* Raw exception entries which need extra work */ |
15a416e8 | 575 | DECLARE_IDTENTRY_RAW(X86_TRAP_UD, exc_invalid_op); |
91eeafea TG |
576 | DECLARE_IDTENTRY_RAW(X86_TRAP_BP, exc_int3); |
577 | DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_PF, exc_page_fault); | |
8edd7e37 | 578 | |
8cd501c1 | 579 | #ifdef CONFIG_X86_MCE |
13cbc0cd | 580 | #ifdef CONFIG_X86_64 |
8cd501c1 | 581 | DECLARE_IDTENTRY_MCE(X86_TRAP_MC, exc_machine_check); |
13cbc0cd AL |
582 | #else |
583 | DECLARE_IDTENTRY_RAW(X86_TRAP_MC, exc_machine_check); | |
584 | #endif | |
c3d7fa66 JG |
585 | #ifdef CONFIG_XEN_PV |
586 | DECLARE_IDTENTRY_RAW(X86_TRAP_MC, xenpv_exc_machine_check); | |
587 | #endif | |
8cd501c1 TG |
588 | #endif |
589 | ||
6271fef0 | 590 | /* NMI */ |
a217a659 LJ |
591 | |
592 | #if defined(CONFIG_X86_64) && IS_ENABLED(CONFIG_KVM_INTEL) | |
593 | /* | |
594 | * Special NOIST entry point for VMX which invokes this on the kernel | |
595 | * stack. asm_exc_nmi() requires an IST to work correctly vs. the NMI | |
596 | * 'executing' marker. | |
597 | * | |
598 | * On 32bit this just uses the regular NMI entry point because 32-bit does | |
599 | * not have ISTs. | |
600 | */ | |
601 | DECLARE_IDTENTRY(X86_TRAP_NMI, exc_nmi_noist); | |
602 | #else | |
603 | #define asm_exc_nmi_noist asm_exc_nmi | |
604 | #endif | |
605 | ||
6271fef0 | 606 | DECLARE_IDTENTRY_NMI(X86_TRAP_NMI, exc_nmi); |
76fdb041 | 607 | #ifdef CONFIG_XEN_PV |
f41f0824 AL |
608 | DECLARE_IDTENTRY_RAW(X86_TRAP_NMI, xenpv_exc_nmi); |
609 | #endif | |
6271fef0 | 610 | |
2bbc68f8 | 611 | /* #DB */ |
13cbc0cd | 612 | #ifdef CONFIG_X86_64 |
2bbc68f8 | 613 | DECLARE_IDTENTRY_DEBUG(X86_TRAP_DB, exc_debug); |
13cbc0cd AL |
614 | #else |
615 | DECLARE_IDTENTRY_RAW(X86_TRAP_DB, exc_debug); | |
616 | #endif | |
76fdb041 | 617 | #ifdef CONFIG_XEN_PV |
f41f0824 AL |
618 | DECLARE_IDTENTRY_RAW(X86_TRAP_DB, xenpv_exc_debug); |
619 | #endif | |
2bbc68f8 | 620 | |
c29c775a TG |
621 | /* #DF */ |
622 | DECLARE_IDTENTRY_DF(X86_TRAP_DF, exc_double_fault); | |
5b4c6d65 JG |
623 | #ifdef CONFIG_XEN_PV |
624 | DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_DF, xenpv_exc_double_fault); | |
625 | #endif | |
c29c775a | 626 | |
0786138c TL |
627 | /* #VC */ |
628 | #ifdef CONFIG_AMD_MEM_ENCRYPT | |
629 | DECLARE_IDTENTRY_VC(X86_TRAP_VC, exc_vmm_communication); | |
630 | #endif | |
631 | ||
2f6474e4 TG |
632 | #ifdef CONFIG_XEN_PV |
633 | DECLARE_IDTENTRY_XENCB(X86_TRAP_OTHER, exc_xen_hypervisor_callback); | |
2e924936 | 634 | DECLARE_IDTENTRY_RAW(X86_TRAP_OTHER, exc_xen_unknown_trap); |
2f6474e4 TG |
635 | #endif |
636 | ||
fa5e5c40 TG |
637 | /* Device interrupts common/spurious */ |
638 | DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER, common_interrupt); | |
639 | #ifdef CONFIG_X86_LOCAL_APIC | |
640 | DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER, spurious_interrupt); | |
641 | #endif | |
642 | ||
db0338ee TG |
643 | /* System vector entry points */ |
644 | #ifdef CONFIG_X86_LOCAL_APIC | |
645 | DECLARE_IDTENTRY_SYSVEC(ERROR_APIC_VECTOR, sysvec_error_interrupt); | |
646 | DECLARE_IDTENTRY_SYSVEC(SPURIOUS_APIC_VECTOR, sysvec_spurious_apic_interrupt); | |
647 | DECLARE_IDTENTRY_SYSVEC(LOCAL_TIMER_VECTOR, sysvec_apic_timer_interrupt); | |
648 | DECLARE_IDTENTRY_SYSVEC(X86_PLATFORM_IPI_VECTOR, sysvec_x86_platform_ipi); | |
649 | #endif | |
650 | ||
582f9191 | 651 | #ifdef CONFIG_SMP |
13cad985 | 652 | DECLARE_IDTENTRY(RESCHEDULE_VECTOR, sysvec_reschedule_ipi); |
582f9191 TG |
653 | DECLARE_IDTENTRY_SYSVEC(IRQ_MOVE_CLEANUP_VECTOR, sysvec_irq_move_cleanup); |
654 | DECLARE_IDTENTRY_SYSVEC(REBOOT_VECTOR, sysvec_reboot); | |
655 | DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_SINGLE_VECTOR, sysvec_call_function_single); | |
656 | DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_VECTOR, sysvec_call_function); | |
657 | #endif | |
658 | ||
720909a7 | 659 | #ifdef CONFIG_X86_LOCAL_APIC |
720909a7 TG |
660 | # ifdef CONFIG_X86_MCE_THRESHOLD |
661 | DECLARE_IDTENTRY_SYSVEC(THRESHOLD_APIC_VECTOR, sysvec_threshold); | |
662 | # endif | |
663 | ||
664 | # ifdef CONFIG_X86_MCE_AMD | |
665 | DECLARE_IDTENTRY_SYSVEC(DEFERRED_ERROR_VECTOR, sysvec_deferred_error); | |
666 | # endif | |
667 | ||
668 | # ifdef CONFIG_X86_THERMAL_VECTOR | |
669 | DECLARE_IDTENTRY_SYSVEC(THERMAL_APIC_VECTOR, sysvec_thermal); | |
670 | # endif | |
671 | ||
672 | # ifdef CONFIG_IRQ_WORK | |
673 | DECLARE_IDTENTRY_SYSVEC(IRQ_WORK_VECTOR, sysvec_irq_work); | |
674 | # endif | |
675 | #endif | |
676 | ||
9c3b1f49 TG |
677 | #ifdef CONFIG_HAVE_KVM |
678 | DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_VECTOR, sysvec_kvm_posted_intr_ipi); | |
679 | DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_WAKEUP_VECTOR, sysvec_kvm_posted_intr_wakeup_ipi); | |
680 | DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_NESTED_VECTOR, sysvec_kvm_posted_intr_nested_ipi); | |
681 | #endif | |
682 | ||
a16be368 TG |
683 | #if IS_ENABLED(CONFIG_HYPERV) |
684 | DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_hyperv_callback); | |
5769fe26 SD |
685 | DECLARE_IDTENTRY_SYSVEC(HYPERV_REENLIGHTENMENT_VECTOR, sysvec_hyperv_reenlightenment); |
686 | DECLARE_IDTENTRY_SYSVEC(HYPERV_STIMER0_VECTOR, sysvec_hyperv_stimer0); | |
a16be368 TG |
687 | #endif |
688 | ||
689 | #if IS_ENABLED(CONFIG_ACRN_GUEST) | |
690 | DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_acrn_hv_callback); | |
691 | #endif | |
692 | ||
cb09ea29 TG |
693 | #ifdef CONFIG_XEN_PVHVM |
694 | DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_xen_hvm_callback); | |
695 | #endif | |
696 | ||
26d05b36 PB |
697 | #ifdef CONFIG_KVM_GUEST |
698 | DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_kvm_asyncpf_interrupt); | |
699 | #endif | |
700 | ||
2f6474e4 TG |
701 | #undef X86_TRAP_OTHER |
702 | ||
53aaf262 | 703 | #endif |