]>
Commit | Line | Data |
---|---|---|
82c73e0a | 1 | // SPDX-License-Identifier: GPL-2.0-only |
d8ed9d48 TG |
2 | /* |
3 | * Interrupt descriptor table related code | |
d8ed9d48 TG |
4 | */ |
5 | #include <linux/interrupt.h> | |
6 | ||
3318e974 TG |
7 | #include <asm/traps.h> |
8 | #include <asm/proto.h> | |
d8ed9d48 | 9 | #include <asm/desc.h> |
447ae316 | 10 | #include <asm/hw_irq.h> |
d8ed9d48 | 11 | |
3318e974 TG |
12 | struct idt_data { |
13 | unsigned int vector; | |
14 | unsigned int segment; | |
15 | struct idt_bits bits; | |
16 | const void *addr; | |
17 | }; | |
18 | ||
19 | #define DPL0 0x0 | |
20 | #define DPL3 0x3 | |
21 | ||
22 | #define DEFAULT_STACK 0 | |
23 | ||
24 | #define G(_vector, _addr, _ist, _type, _dpl, _segment) \ | |
25 | { \ | |
26 | .vector = _vector, \ | |
27 | .bits.ist = _ist, \ | |
28 | .bits.type = _type, \ | |
29 | .bits.dpl = _dpl, \ | |
30 | .bits.p = 1, \ | |
31 | .addr = _addr, \ | |
32 | .segment = _segment, \ | |
33 | } | |
34 | ||
35 | /* Interrupt gate */ | |
36 | #define INTG(_vector, _addr) \ | |
37 | G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL0, __KERNEL_CS) | |
38 | ||
39 | /* System interrupt gate */ | |
40 | #define SYSG(_vector, _addr) \ | |
41 | G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL3, __KERNEL_CS) | |
42 | ||
8f34c5b5 TG |
43 | /* |
44 | * Interrupt gate with interrupt stack. The _ist index is the index in | |
45 | * the tss.ist[] array, but for the descriptor it needs to start at 1. | |
46 | */ | |
3318e974 | 47 | #define ISTG(_vector, _addr, _ist) \ |
8f34c5b5 | 48 | G(_vector, _addr, _ist + 1, GATE_INTERRUPT, DPL0, __KERNEL_CS) |
3318e974 TG |
49 | |
50 | /* Task gate */ | |
51 | #define TSKG(_vector, _gdt) \ | |
52 | G(_vector, NULL, DEFAULT_STACK, GATE_TASK, DPL0, _gdt << 3) | |
53 | ||
06184325 VK |
54 | |
55 | static bool idt_setup_done __initdata; | |
56 | ||
433f8924 TG |
57 | /* |
58 | * Early traps running on the DEFAULT_STACK because the other interrupt | |
59 | * stacks work only after cpu_init(). | |
60 | */ | |
327867fa | 61 | static const __initconst struct idt_data early_idts[] = { |
2bbc68f8 | 62 | INTG(X86_TRAP_DB, asm_exc_debug), |
8edd7e37 | 63 | SYSG(X86_TRAP_BP, asm_exc_int3), |
433f8924 | 64 | #ifdef CONFIG_X86_32 |
91eeafea | 65 | INTG(X86_TRAP_PF, asm_exc_page_fault), |
433f8924 TG |
66 | #endif |
67 | }; | |
68 | ||
b70543a0 TG |
69 | /* |
70 | * The default IDT entries which are set up in trap_init() before | |
71 | * cpu_init() is invoked. Interrupt stacks cannot be used at that point and | |
72 | * the traps which use them are reinitialized with IST after cpu_init() has | |
73 | * set up TSS. | |
74 | */ | |
327867fa | 75 | static const __initconst struct idt_data def_idts[] = { |
9d06c402 | 76 | INTG(X86_TRAP_DE, asm_exc_divide_error), |
6271fef0 | 77 | INTG(X86_TRAP_NMI, asm_exc_nmi), |
58d9c81f | 78 | INTG(X86_TRAP_BR, asm_exc_bounds), |
49893c5c | 79 | INTG(X86_TRAP_UD, asm_exc_invalid_op), |
866ae2cc | 80 | INTG(X86_TRAP_NM, asm_exc_device_not_available), |
f95658fd | 81 | INTG(X86_TRAP_OLD_MF, asm_exc_coproc_segment_overrun), |
97b3d290 | 82 | INTG(X86_TRAP_TS, asm_exc_invalid_tss), |
99a3fb8d | 83 | INTG(X86_TRAP_NP, asm_exc_segment_not_present), |
fd9689bf | 84 | INTG(X86_TRAP_SS, asm_exc_stack_segment), |
be4c11af | 85 | INTG(X86_TRAP_GP, asm_exc_general_protection), |
dad7106f | 86 | INTG(X86_TRAP_SPURIOUS, asm_exc_spurious_interrupt_bug), |
14a8bd2a | 87 | INTG(X86_TRAP_MF, asm_exc_coprocessor_error), |
436608bb | 88 | INTG(X86_TRAP_AC, asm_exc_alignment_check), |
48227e21 | 89 | INTG(X86_TRAP_XF, asm_exc_simd_coprocessor_error), |
b70543a0 TG |
90 | |
91 | #ifdef CONFIG_X86_32 | |
92 | TSKG(X86_TRAP_DF, GDT_ENTRY_DOUBLEFAULT_TSS), | |
93 | #else | |
c29c775a | 94 | INTG(X86_TRAP_DF, asm_exc_double_fault), |
b70543a0 | 95 | #endif |
2bbc68f8 | 96 | INTG(X86_TRAP_DB, asm_exc_debug), |
b70543a0 TG |
97 | |
98 | #ifdef CONFIG_X86_MCE | |
8cd501c1 | 99 | INTG(X86_TRAP_MC, asm_exc_machine_check), |
b70543a0 TG |
100 | #endif |
101 | ||
4b6b9111 | 102 | SYSG(X86_TRAP_OF, asm_exc_overflow), |
b70543a0 TG |
103 | #if defined(CONFIG_IA32_EMULATION) |
104 | SYSG(IA32_SYSCALL_VECTOR, entry_INT80_compat), | |
105 | #elif defined(CONFIG_X86_32) | |
106 | SYSG(IA32_SYSCALL_VECTOR, entry_INT80_32), | |
107 | #endif | |
108 | }; | |
109 | ||
636a7598 TG |
110 | /* |
111 | * The APIC and SMP idt entries | |
112 | */ | |
327867fa | 113 | static const __initconst struct idt_data apic_idts[] = { |
636a7598 | 114 | #ifdef CONFIG_SMP |
582f9191 TG |
115 | INTG(RESCHEDULE_VECTOR, reschedule_interrupt), |
116 | INTG(CALL_FUNCTION_VECTOR, asm_sysvec_call_function), | |
117 | INTG(CALL_FUNCTION_SINGLE_VECTOR, asm_sysvec_call_function_single), | |
118 | INTG(IRQ_MOVE_CLEANUP_VECTOR, asm_sysvec_irq_move_cleanup), | |
119 | INTG(REBOOT_VECTOR, asm_sysvec_reboot), | |
636a7598 TG |
120 | #endif |
121 | ||
122 | #ifdef CONFIG_X86_THERMAL_VECTOR | |
720909a7 | 123 | INTG(THERMAL_APIC_VECTOR, asm_sysvec_thermal), |
636a7598 TG |
124 | #endif |
125 | ||
126 | #ifdef CONFIG_X86_MCE_THRESHOLD | |
720909a7 | 127 | INTG(THRESHOLD_APIC_VECTOR, asm_sysvec_threshold), |
636a7598 TG |
128 | #endif |
129 | ||
130 | #ifdef CONFIG_X86_MCE_AMD | |
720909a7 | 131 | INTG(DEFERRED_ERROR_VECTOR, asm_sysvec_deferred_error), |
636a7598 TG |
132 | #endif |
133 | ||
134 | #ifdef CONFIG_X86_LOCAL_APIC | |
720909a7 TG |
135 | INTG(LOCAL_TIMER_VECTOR, asm_sysvec_apic_timer_interrupt), |
136 | INTG(X86_PLATFORM_IPI_VECTOR, asm_sysvec_x86_platform_ipi), | |
636a7598 | 137 | # ifdef CONFIG_HAVE_KVM |
720909a7 TG |
138 | INTG(POSTED_INTR_VECTOR, kvm_posted_intr_ipi), |
139 | INTG(POSTED_INTR_WAKEUP_VECTOR, kvm_posted_intr_wakeup_ipi), | |
140 | INTG(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi), | |
636a7598 TG |
141 | # endif |
142 | # ifdef CONFIG_IRQ_WORK | |
720909a7 | 143 | INTG(IRQ_WORK_VECTOR, asm_sysvec_irq_work), |
636a7598 | 144 | # endif |
720909a7 TG |
145 | # ifdef CONFIG_X86_UV |
146 | INTG(UV_BAU_MESSAGE, asm_sysvec_uv_bau_message), | |
147 | # endif | |
148 | INTG(SPURIOUS_APIC_VECTOR, asm_sysvec_spurious_apic_interrupt), | |
149 | INTG(ERROR_APIC_VECTOR, asm_sysvec_error_interrupt), | |
636a7598 TG |
150 | #endif |
151 | }; | |
152 | ||
433f8924 TG |
153 | #ifdef CONFIG_X86_64 |
154 | /* | |
155 | * Early traps running on the DEFAULT_STACK because the other interrupt | |
156 | * stacks work only after cpu_init(). | |
157 | */ | |
327867fa | 158 | static const __initconst struct idt_data early_pf_idts[] = { |
91eeafea | 159 | INTG(X86_TRAP_PF, asm_exc_page_fault), |
433f8924 | 160 | }; |
0a30908b TG |
161 | |
162 | /* | |
163 | * Override for the debug_idt. Same as the default, but with interrupt | |
164 | * stack set to DEFAULT_STACK (0). Required for NMI trap handling. | |
165 | */ | |
327867fa | 166 | static const __initconst struct idt_data dbg_idts[] = { |
2bbc68f8 | 167 | INTG(X86_TRAP_DB, asm_exc_debug), |
0a30908b | 168 | }; |
433f8924 TG |
169 | #endif |
170 | ||
d8ed9d48 TG |
171 | /* Must be page-aligned because the real IDT is used in a fixmap. */ |
172 | gate_desc idt_table[IDT_ENTRIES] __page_aligned_bss; | |
173 | ||
d8ed9d48 | 174 | struct desc_ptr idt_descr __ro_after_init = { |
16bc18d8 | 175 | .size = (IDT_ENTRIES * 2 * sizeof(unsigned long)) - 1, |
d8ed9d48 TG |
176 | .address = (unsigned long) idt_table, |
177 | }; | |
178 | ||
16bc18d8 TG |
179 | #ifdef CONFIG_X86_64 |
180 | /* No need to be aligned, but done to keep all IDTs defined the same way. */ | |
181 | gate_desc debug_idt_table[IDT_ENTRIES] __page_aligned_bss; | |
182 | ||
90f6225f TG |
183 | /* |
184 | * The exceptions which use Interrupt stacks. They are setup after | |
185 | * cpu_init() when the TSS has been initialized. | |
186 | */ | |
327867fa | 187 | static const __initconst struct idt_data ist_idts[] = { |
2bbc68f8 | 188 | ISTG(X86_TRAP_DB, asm_exc_debug, IST_INDEX_DB), |
6271fef0 | 189 | ISTG(X86_TRAP_NMI, asm_exc_nmi, IST_INDEX_NMI), |
c29c775a | 190 | ISTG(X86_TRAP_DF, asm_exc_double_fault, IST_INDEX_DF), |
90f6225f | 191 | #ifdef CONFIG_X86_MCE |
8cd501c1 | 192 | ISTG(X86_TRAP_MC, asm_exc_machine_check, IST_INDEX_MCE), |
90f6225f TG |
193 | #endif |
194 | }; | |
195 | ||
0a30908b TG |
196 | /* |
197 | * Override for the debug_idt. Same as the default, but with interrupt | |
198 | * stack set to DEFAULT_STACK (0). Required for NMI trap handling. | |
199 | */ | |
d8ed9d48 TG |
200 | const struct desc_ptr debug_idt_descr = { |
201 | .size = IDT_ENTRIES * 16 - 1, | |
202 | .address = (unsigned long) debug_idt_table, | |
203 | }; | |
204 | #endif | |
e802a51e | 205 | |
3318e974 TG |
206 | static inline void idt_init_desc(gate_desc *gate, const struct idt_data *d) |
207 | { | |
208 | unsigned long addr = (unsigned long) d->addr; | |
209 | ||
210 | gate->offset_low = (u16) addr; | |
211 | gate->segment = (u16) d->segment; | |
212 | gate->bits = d->bits; | |
213 | gate->offset_middle = (u16) (addr >> 16); | |
214 | #ifdef CONFIG_X86_64 | |
215 | gate->offset_high = (u32) (addr >> 32); | |
216 | gate->reserved = 0; | |
217 | #endif | |
218 | } | |
219 | ||
db18da78 TG |
220 | static void |
221 | idt_setup_from_table(gate_desc *idt, const struct idt_data *t, int size, bool sys) | |
3318e974 TG |
222 | { |
223 | gate_desc desc; | |
224 | ||
225 | for (; size > 0; t++, size--) { | |
226 | idt_init_desc(&desc, t); | |
3318e974 | 227 | write_idt_entry(idt, t->vector, &desc); |
db18da78 | 228 | if (sys) |
7854f822 | 229 | set_bit(t->vector, system_vectors); |
3318e974 TG |
230 | } |
231 | } | |
232 | ||
facaa3e3 TG |
233 | static void set_intr_gate(unsigned int n, const void *addr) |
234 | { | |
235 | struct idt_data data; | |
236 | ||
237 | BUG_ON(n > 0xFF); | |
238 | ||
239 | memset(&data, 0, sizeof(data)); | |
240 | data.vector = n; | |
241 | data.addr = addr; | |
242 | data.segment = __KERNEL_CS; | |
243 | data.bits.type = GATE_INTERRUPT; | |
244 | data.bits.p = 1; | |
245 | ||
246 | idt_setup_from_table(idt_table, &data, 1, false); | |
247 | } | |
248 | ||
433f8924 TG |
249 | /** |
250 | * idt_setup_early_traps - Initialize the idt table with early traps | |
251 | * | |
252 | * On X8664 these traps do not use interrupt stacks as they can't work | |
253 | * before cpu_init() is invoked and sets up TSS. The IST variants are | |
254 | * installed after that. | |
255 | */ | |
256 | void __init idt_setup_early_traps(void) | |
257 | { | |
db18da78 TG |
258 | idt_setup_from_table(idt_table, early_idts, ARRAY_SIZE(early_idts), |
259 | true); | |
433f8924 TG |
260 | load_idt(&idt_descr); |
261 | } | |
262 | ||
b70543a0 TG |
263 | /** |
264 | * idt_setup_traps - Initialize the idt table with default traps | |
265 | */ | |
266 | void __init idt_setup_traps(void) | |
267 | { | |
db18da78 | 268 | idt_setup_from_table(idt_table, def_idts, ARRAY_SIZE(def_idts), true); |
b70543a0 TG |
269 | } |
270 | ||
433f8924 TG |
271 | #ifdef CONFIG_X86_64 |
272 | /** | |
273 | * idt_setup_early_pf - Initialize the idt table with early pagefault handler | |
274 | * | |
275 | * On X8664 this does not use interrupt stacks as they can't work before | |
276 | * cpu_init() is invoked and sets up TSS. The IST variant is installed | |
277 | * after that. | |
278 | * | |
279 | * FIXME: Why is 32bit and 64bit installing the PF handler at different | |
280 | * places in the early setup code? | |
281 | */ | |
282 | void __init idt_setup_early_pf(void) | |
283 | { | |
284 | idt_setup_from_table(idt_table, early_pf_idts, | |
db18da78 | 285 | ARRAY_SIZE(early_pf_idts), true); |
433f8924 | 286 | } |
0a30908b | 287 | |
90f6225f TG |
288 | /** |
289 | * idt_setup_ist_traps - Initialize the idt table with traps using IST | |
290 | */ | |
291 | void __init idt_setup_ist_traps(void) | |
292 | { | |
db18da78 | 293 | idt_setup_from_table(idt_table, ist_idts, ARRAY_SIZE(ist_idts), true); |
90f6225f TG |
294 | } |
295 | ||
0a30908b TG |
296 | /** |
297 | * idt_setup_debugidt_traps - Initialize the debug idt table with debug traps | |
298 | */ | |
299 | void __init idt_setup_debugidt_traps(void) | |
300 | { | |
301 | memcpy(&debug_idt_table, &idt_table, IDT_ENTRIES * 16); | |
302 | ||
db18da78 | 303 | idt_setup_from_table(debug_idt_table, dbg_idts, ARRAY_SIZE(dbg_idts), false); |
0a30908b | 304 | } |
433f8924 TG |
305 | #endif |
306 | ||
636a7598 TG |
307 | /** |
308 | * idt_setup_apic_and_irq_gates - Setup APIC/SMP and normal interrupt gates | |
309 | */ | |
310 | void __init idt_setup_apic_and_irq_gates(void) | |
311 | { | |
dc20b2d5 TG |
312 | int i = FIRST_EXTERNAL_VECTOR; |
313 | void *entry; | |
314 | ||
db18da78 | 315 | idt_setup_from_table(idt_table, apic_idts, ARRAY_SIZE(apic_idts), true); |
dc20b2d5 | 316 | |
7854f822 | 317 | for_each_clear_bit_from(i, system_vectors, FIRST_SYSTEM_VECTOR) { |
dc20b2d5 TG |
318 | entry = irq_entries_start + 8 * (i - FIRST_EXTERNAL_VECTOR); |
319 | set_intr_gate(i, entry); | |
320 | } | |
321 | ||
dc20b2d5 | 322 | #ifdef CONFIG_X86_LOCAL_APIC |
33662812 | 323 | for_each_clear_bit_from(i, system_vectors, NR_VECTORS) { |
1f1fbc70 VK |
324 | /* |
325 | * Don't set the non assigned system vectors in the | |
326 | * system_vectors bitmap. Otherwise they show up in | |
327 | * /proc/interrupts. | |
328 | */ | |
f8a8fe61 TG |
329 | entry = spurious_entries_start + 8 * (i - FIRST_SYSTEM_VECTOR); |
330 | set_intr_gate(i, entry); | |
dc20b2d5 | 331 | } |
33662812 | 332 | #endif |
06184325 | 333 | idt_setup_done = true; |
636a7598 TG |
334 | } |
335 | ||
588787fd TG |
336 | /** |
337 | * idt_setup_early_handler - Initializes the idt table with early handlers | |
338 | */ | |
339 | void __init idt_setup_early_handler(void) | |
340 | { | |
341 | int i; | |
342 | ||
343 | for (i = 0; i < NUM_EXCEPTION_VECTORS; i++) | |
344 | set_intr_gate(i, early_idt_handler_array[i]); | |
87e81786 TG |
345 | #ifdef CONFIG_X86_32 |
346 | for ( ; i < NR_VECTORS; i++) | |
347 | set_intr_gate(i, early_ignore_irq); | |
348 | #endif | |
588787fd TG |
349 | load_idt(&idt_descr); |
350 | } | |
351 | ||
e802a51e TG |
352 | /** |
353 | * idt_invalidate - Invalidate interrupt descriptor table | |
354 | * @addr: The virtual address of the 'invalid' IDT | |
355 | */ | |
356 | void idt_invalidate(void *addr) | |
357 | { | |
358 | struct desc_ptr idt = { .address = (unsigned long) addr, .size = 0 }; | |
359 | ||
360 | load_idt(&idt); | |
361 | } | |
db18da78 | 362 | |
06184325 | 363 | void __init alloc_intr_gate(unsigned int n, const void *addr) |
db18da78 | 364 | { |
06184325 VK |
365 | if (WARN_ON(n < FIRST_SYSTEM_VECTOR)) |
366 | return; | |
367 | ||
368 | if (WARN_ON(idt_setup_done)) | |
369 | return; | |
370 | ||
371 | if (!WARN_ON(test_and_set_bit(n, system_vectors))) | |
4447ac11 | 372 | set_intr_gate(n, addr); |
db18da78 | 373 | } |