]>
Commit | Line | Data |
---|---|---|
d8ed9d48 TG |
1 | /* |
2 | * Interrupt descriptor table related code | |
3 | * | |
4 | * This file is licensed under the GPL V2 | |
5 | */ | |
6 | #include <linux/interrupt.h> | |
7 | ||
3318e974 TG |
8 | #include <asm/traps.h> |
9 | #include <asm/proto.h> | |
d8ed9d48 | 10 | #include <asm/desc.h> |
f901f138 | 11 | #include <asm/hw_irq.h> |
d8ed9d48 | 12 | |
3318e974 TG |
13 | struct idt_data { |
14 | unsigned int vector; | |
15 | unsigned int segment; | |
16 | struct idt_bits bits; | |
17 | const void *addr; | |
18 | }; | |
19 | ||
20 | #define DPL0 0x0 | |
21 | #define DPL3 0x3 | |
22 | ||
23 | #define DEFAULT_STACK 0 | |
24 | ||
25 | #define G(_vector, _addr, _ist, _type, _dpl, _segment) \ | |
26 | { \ | |
27 | .vector = _vector, \ | |
28 | .bits.ist = _ist, \ | |
29 | .bits.type = _type, \ | |
30 | .bits.dpl = _dpl, \ | |
31 | .bits.p = 1, \ | |
32 | .addr = _addr, \ | |
33 | .segment = _segment, \ | |
34 | } | |
35 | ||
36 | /* Interrupt gate */ | |
37 | #define INTG(_vector, _addr) \ | |
38 | G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL0, __KERNEL_CS) | |
39 | ||
40 | /* System interrupt gate */ | |
41 | #define SYSG(_vector, _addr) \ | |
42 | G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL3, __KERNEL_CS) | |
43 | ||
44 | /* Interrupt gate with interrupt stack */ | |
45 | #define ISTG(_vector, _addr, _ist) \ | |
46 | G(_vector, _addr, _ist, GATE_INTERRUPT, DPL0, __KERNEL_CS) | |
47 | ||
c6ef8942 IM |
48 | /* System interrupt gate with interrupt stack */ |
49 | #define SISTG(_vector, _addr, _ist) \ | |
50 | G(_vector, _addr, _ist, GATE_INTERRUPT, DPL3, __KERNEL_CS) | |
51 | ||
3318e974 TG |
52 | /* Task gate */ |
53 | #define TSKG(_vector, _gdt) \ | |
54 | G(_vector, NULL, DEFAULT_STACK, GATE_TASK, DPL0, _gdt << 3) | |
55 | ||
433f8924 TG |
56 | /* |
57 | * Early traps running on the DEFAULT_STACK because the other interrupt | |
58 | * stacks work only after cpu_init(). | |
59 | */ | |
327867fa | 60 | static const __initconst struct idt_data early_idts[] = { |
433f8924 TG |
61 | INTG(X86_TRAP_DB, debug), |
62 | SYSG(X86_TRAP_BP, int3), | |
63 | #ifdef CONFIG_X86_32 | |
64 | INTG(X86_TRAP_PF, page_fault), | |
65 | #endif | |
66 | }; | |
67 | ||
b70543a0 TG |
68 | /* |
69 | * The default IDT entries which are set up in trap_init() before | |
70 | * cpu_init() is invoked. Interrupt stacks cannot be used at that point and | |
71 | * the traps which use them are reinitialized with IST after cpu_init() has | |
72 | * set up TSS. | |
73 | */ | |
327867fa | 74 | static const __initconst struct idt_data def_idts[] = { |
b70543a0 TG |
75 | INTG(X86_TRAP_DE, divide_error), |
76 | INTG(X86_TRAP_NMI, nmi), | |
77 | INTG(X86_TRAP_BR, bounds), | |
78 | INTG(X86_TRAP_UD, invalid_op), | |
79 | INTG(X86_TRAP_NM, device_not_available), | |
80 | INTG(X86_TRAP_OLD_MF, coprocessor_segment_overrun), | |
81 | INTG(X86_TRAP_TS, invalid_TSS), | |
82 | INTG(X86_TRAP_NP, segment_not_present), | |
83 | INTG(X86_TRAP_SS, stack_segment), | |
84 | INTG(X86_TRAP_GP, general_protection), | |
85 | INTG(X86_TRAP_SPURIOUS, spurious_interrupt_bug), | |
86 | INTG(X86_TRAP_MF, coprocessor_error), | |
87 | INTG(X86_TRAP_AC, alignment_check), | |
88 | INTG(X86_TRAP_XF, simd_coprocessor_error), | |
89 | ||
90 | #ifdef CONFIG_X86_32 | |
91 | TSKG(X86_TRAP_DF, GDT_ENTRY_DOUBLEFAULT_TSS), | |
92 | #else | |
93 | INTG(X86_TRAP_DF, double_fault), | |
94 | #endif | |
95 | INTG(X86_TRAP_DB, debug), | |
b70543a0 TG |
96 | |
97 | #ifdef CONFIG_X86_MCE | |
98 | INTG(X86_TRAP_MC, &machine_check), | |
99 | #endif | |
100 | ||
101 | SYSG(X86_TRAP_OF, overflow), | |
102 | #if defined(CONFIG_IA32_EMULATION) | |
103 | SYSG(IA32_SYSCALL_VECTOR, entry_INT80_compat), | |
104 | #elif defined(CONFIG_X86_32) | |
105 | SYSG(IA32_SYSCALL_VECTOR, entry_INT80_32), | |
106 | #endif | |
107 | }; | |
108 | ||
636a7598 TG |
109 | /* |
110 | * The APIC and SMP idt entries | |
111 | */ | |
327867fa | 112 | static const __initconst struct idt_data apic_idts[] = { |
636a7598 TG |
113 | #ifdef CONFIG_SMP |
114 | INTG(RESCHEDULE_VECTOR, reschedule_interrupt), | |
115 | INTG(CALL_FUNCTION_VECTOR, call_function_interrupt), | |
116 | INTG(CALL_FUNCTION_SINGLE_VECTOR, call_function_single_interrupt), | |
117 | INTG(IRQ_MOVE_CLEANUP_VECTOR, irq_move_cleanup_interrupt), | |
118 | INTG(REBOOT_VECTOR, reboot_interrupt), | |
119 | #endif | |
120 | ||
121 | #ifdef CONFIG_X86_THERMAL_VECTOR | |
122 | INTG(THERMAL_APIC_VECTOR, thermal_interrupt), | |
123 | #endif | |
124 | ||
125 | #ifdef CONFIG_X86_MCE_THRESHOLD | |
126 | INTG(THRESHOLD_APIC_VECTOR, threshold_interrupt), | |
127 | #endif | |
128 | ||
129 | #ifdef CONFIG_X86_MCE_AMD | |
130 | INTG(DEFERRED_ERROR_VECTOR, deferred_error_interrupt), | |
131 | #endif | |
132 | ||
133 | #ifdef CONFIG_X86_LOCAL_APIC | |
134 | INTG(LOCAL_TIMER_VECTOR, apic_timer_interrupt), | |
135 | INTG(X86_PLATFORM_IPI_VECTOR, x86_platform_ipi), | |
136 | # ifdef CONFIG_HAVE_KVM | |
137 | INTG(POSTED_INTR_VECTOR, kvm_posted_intr_ipi), | |
138 | INTG(POSTED_INTR_WAKEUP_VECTOR, kvm_posted_intr_wakeup_ipi), | |
139 | INTG(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi), | |
140 | # endif | |
141 | # ifdef CONFIG_IRQ_WORK | |
142 | INTG(IRQ_WORK_VECTOR, irq_work_interrupt), | |
143 | # endif | |
4273e73a AB |
144 | #ifdef CONFIG_X86_UV |
145 | INTG(UV_BAU_MESSAGE, uv_bau_message_intr1), | |
146 | #endif | |
636a7598 TG |
147 | INTG(SPURIOUS_APIC_VECTOR, spurious_interrupt), |
148 | INTG(ERROR_APIC_VECTOR, error_interrupt), | |
149 | #endif | |
150 | }; | |
151 | ||
433f8924 TG |
152 | #ifdef CONFIG_X86_64 |
153 | /* | |
154 | * Early traps running on the DEFAULT_STACK because the other interrupt | |
155 | * stacks work only after cpu_init(). | |
156 | */ | |
327867fa | 157 | static const __initconst struct idt_data early_pf_idts[] = { |
433f8924 TG |
158 | INTG(X86_TRAP_PF, page_fault), |
159 | }; | |
0a30908b TG |
160 | |
161 | /* | |
162 | * Override for the debug_idt. Same as the default, but with interrupt | |
163 | * stack set to DEFAULT_STACK (0). Required for NMI trap handling. | |
164 | */ | |
327867fa | 165 | static const __initconst struct idt_data dbg_idts[] = { |
0a30908b | 166 | INTG(X86_TRAP_DB, debug), |
0a30908b | 167 | }; |
433f8924 TG |
168 | #endif |
169 | ||
d8ed9d48 TG |
170 | /* Must be page-aligned because the real IDT is used in a fixmap. */ |
171 | gate_desc idt_table[IDT_ENTRIES] __page_aligned_bss; | |
172 | ||
d8ed9d48 | 173 | struct desc_ptr idt_descr __ro_after_init = { |
16bc18d8 | 174 | .size = (IDT_ENTRIES * 2 * sizeof(unsigned long)) - 1, |
d8ed9d48 TG |
175 | .address = (unsigned long) idt_table, |
176 | }; | |
177 | ||
16bc18d8 TG |
178 | #ifdef CONFIG_X86_64 |
179 | /* No need to be aligned, but done to keep all IDTs defined the same way. */ | |
180 | gate_desc debug_idt_table[IDT_ENTRIES] __page_aligned_bss; | |
181 | ||
90f6225f TG |
182 | /* |
183 | * The exceptions which use Interrupt stacks. They are setup after | |
184 | * cpu_init() when the TSS has been initialized. | |
185 | */ | |
327867fa | 186 | static const __initconst struct idt_data ist_idts[] = { |
90f6225f TG |
187 | ISTG(X86_TRAP_DB, debug, DEBUG_STACK), |
188 | ISTG(X86_TRAP_NMI, nmi, NMI_STACK), | |
90f6225f TG |
189 | ISTG(X86_TRAP_DF, double_fault, DOUBLEFAULT_STACK), |
190 | #ifdef CONFIG_X86_MCE | |
191 | ISTG(X86_TRAP_MC, &machine_check, MCE_STACK), | |
192 | #endif | |
193 | }; | |
194 | ||
0a30908b TG |
195 | /* |
196 | * Override for the debug_idt. Same as the default, but with interrupt | |
197 | * stack set to DEFAULT_STACK (0). Required for NMI trap handling. | |
198 | */ | |
d8ed9d48 TG |
199 | const struct desc_ptr debug_idt_descr = { |
200 | .size = IDT_ENTRIES * 16 - 1, | |
201 | .address = (unsigned long) debug_idt_table, | |
202 | }; | |
203 | #endif | |
e802a51e | 204 | |
3318e974 TG |
205 | static inline void idt_init_desc(gate_desc *gate, const struct idt_data *d) |
206 | { | |
207 | unsigned long addr = (unsigned long) d->addr; | |
208 | ||
209 | gate->offset_low = (u16) addr; | |
210 | gate->segment = (u16) d->segment; | |
211 | gate->bits = d->bits; | |
212 | gate->offset_middle = (u16) (addr >> 16); | |
213 | #ifdef CONFIG_X86_64 | |
214 | gate->offset_high = (u32) (addr >> 32); | |
215 | gate->reserved = 0; | |
216 | #endif | |
217 | } | |
218 | ||
db18da78 TG |
219 | static void |
220 | idt_setup_from_table(gate_desc *idt, const struct idt_data *t, int size, bool sys) | |
3318e974 TG |
221 | { |
222 | gate_desc desc; | |
223 | ||
224 | for (; size > 0; t++, size--) { | |
225 | idt_init_desc(&desc, t); | |
3318e974 | 226 | write_idt_entry(idt, t->vector, &desc); |
db18da78 | 227 | if (sys) |
7854f822 | 228 | set_bit(t->vector, system_vectors); |
3318e974 TG |
229 | } |
230 | } | |
231 | ||
facaa3e3 TG |
232 | static void set_intr_gate(unsigned int n, const void *addr) |
233 | { | |
234 | struct idt_data data; | |
235 | ||
236 | BUG_ON(n > 0xFF); | |
237 | ||
238 | memset(&data, 0, sizeof(data)); | |
239 | data.vector = n; | |
240 | data.addr = addr; | |
241 | data.segment = __KERNEL_CS; | |
242 | data.bits.type = GATE_INTERRUPT; | |
243 | data.bits.p = 1; | |
244 | ||
245 | idt_setup_from_table(idt_table, &data, 1, false); | |
246 | } | |
247 | ||
433f8924 TG |
248 | /** |
249 | * idt_setup_early_traps - Initialize the idt table with early traps | |
250 | * | |
251 | * On X8664 these traps do not use interrupt stacks as they can't work | |
252 | * before cpu_init() is invoked and sets up TSS. The IST variants are | |
253 | * installed after that. | |
254 | */ | |
255 | void __init idt_setup_early_traps(void) | |
256 | { | |
db18da78 TG |
257 | idt_setup_from_table(idt_table, early_idts, ARRAY_SIZE(early_idts), |
258 | true); | |
433f8924 TG |
259 | load_idt(&idt_descr); |
260 | } | |
261 | ||
b70543a0 TG |
262 | /** |
263 | * idt_setup_traps - Initialize the idt table with default traps | |
264 | */ | |
265 | void __init idt_setup_traps(void) | |
266 | { | |
db18da78 | 267 | idt_setup_from_table(idt_table, def_idts, ARRAY_SIZE(def_idts), true); |
b70543a0 TG |
268 | } |
269 | ||
433f8924 TG |
270 | #ifdef CONFIG_X86_64 |
271 | /** | |
272 | * idt_setup_early_pf - Initialize the idt table with early pagefault handler | |
273 | * | |
274 | * On X8664 this does not use interrupt stacks as they can't work before | |
275 | * cpu_init() is invoked and sets up TSS. The IST variant is installed | |
276 | * after that. | |
277 | * | |
278 | * FIXME: Why is 32bit and 64bit installing the PF handler at different | |
279 | * places in the early setup code? | |
280 | */ | |
281 | void __init idt_setup_early_pf(void) | |
282 | { | |
283 | idt_setup_from_table(idt_table, early_pf_idts, | |
db18da78 | 284 | ARRAY_SIZE(early_pf_idts), true); |
433f8924 | 285 | } |
0a30908b | 286 | |
90f6225f TG |
287 | /** |
288 | * idt_setup_ist_traps - Initialize the idt table with traps using IST | |
289 | */ | |
290 | void __init idt_setup_ist_traps(void) | |
291 | { | |
db18da78 | 292 | idt_setup_from_table(idt_table, ist_idts, ARRAY_SIZE(ist_idts), true); |
90f6225f TG |
293 | } |
294 | ||
0a30908b TG |
295 | /** |
296 | * idt_setup_debugidt_traps - Initialize the debug idt table with debug traps | |
297 | */ | |
298 | void __init idt_setup_debugidt_traps(void) | |
299 | { | |
300 | memcpy(&debug_idt_table, &idt_table, IDT_ENTRIES * 16); | |
301 | ||
db18da78 | 302 | idt_setup_from_table(debug_idt_table, dbg_idts, ARRAY_SIZE(dbg_idts), false); |
0a30908b | 303 | } |
433f8924 TG |
304 | #endif |
305 | ||
636a7598 TG |
306 | /** |
307 | * idt_setup_apic_and_irq_gates - Setup APIC/SMP and normal interrupt gates | |
308 | */ | |
309 | void __init idt_setup_apic_and_irq_gates(void) | |
310 | { | |
dc20b2d5 TG |
311 | int i = FIRST_EXTERNAL_VECTOR; |
312 | void *entry; | |
313 | ||
db18da78 | 314 | idt_setup_from_table(idt_table, apic_idts, ARRAY_SIZE(apic_idts), true); |
dc20b2d5 | 315 | |
7854f822 | 316 | for_each_clear_bit_from(i, system_vectors, FIRST_SYSTEM_VECTOR) { |
dc20b2d5 TG |
317 | entry = irq_entries_start + 8 * (i - FIRST_EXTERNAL_VECTOR); |
318 | set_intr_gate(i, entry); | |
319 | } | |
320 | ||
7854f822 | 321 | for_each_clear_bit_from(i, system_vectors, NR_VECTORS) { |
dc20b2d5 | 322 | #ifdef CONFIG_X86_LOCAL_APIC |
7854f822 | 323 | set_bit(i, system_vectors); |
dc20b2d5 TG |
324 | set_intr_gate(i, spurious_interrupt); |
325 | #else | |
326 | entry = irq_entries_start + 8 * (i - FIRST_EXTERNAL_VECTOR); | |
327 | set_intr_gate(i, entry); | |
328 | #endif | |
329 | } | |
636a7598 TG |
330 | } |
331 | ||
588787fd TG |
332 | /** |
333 | * idt_setup_early_handler - Initializes the idt table with early handlers | |
334 | */ | |
335 | void __init idt_setup_early_handler(void) | |
336 | { | |
337 | int i; | |
338 | ||
339 | for (i = 0; i < NUM_EXCEPTION_VECTORS; i++) | |
340 | set_intr_gate(i, early_idt_handler_array[i]); | |
87e81786 TG |
341 | #ifdef CONFIG_X86_32 |
342 | for ( ; i < NR_VECTORS; i++) | |
343 | set_intr_gate(i, early_ignore_irq); | |
344 | #endif | |
588787fd TG |
345 | load_idt(&idt_descr); |
346 | } | |
347 | ||
e802a51e TG |
348 | /** |
349 | * idt_invalidate - Invalidate interrupt descriptor table | |
350 | * @addr: The virtual address of the 'invalid' IDT | |
351 | */ | |
352 | void idt_invalidate(void *addr) | |
353 | { | |
354 | struct desc_ptr idt = { .address = (unsigned long) addr, .size = 0 }; | |
355 | ||
356 | load_idt(&idt); | |
357 | } | |
db18da78 | 358 | |
facaa3e3 | 359 | void __init update_intr_gate(unsigned int n, const void *addr) |
db18da78 | 360 | { |
7854f822 | 361 | if (WARN_ON_ONCE(!test_bit(n, system_vectors))) |
facaa3e3 TG |
362 | return; |
363 | set_intr_gate(n, addr); | |
db18da78 TG |
364 | } |
365 | ||
366 | void alloc_intr_gate(unsigned int n, const void *addr) | |
367 | { | |
4447ac11 | 368 | BUG_ON(n < FIRST_SYSTEM_VECTOR); |
7854f822 | 369 | if (!test_and_set_bit(n, system_vectors)) |
4447ac11 | 370 | set_intr_gate(n, addr); |
db18da78 | 371 | } |