1 // SPDX-License-Identifier: GPL-2.0-only
3 * Interrupt descriptor table related code
5 #include <linux/interrupt.h>
10 #include <asm/hw_irq.h>
22 #define DEFAULT_STACK 0
24 #define G(_vector, _addr, _ist, _type, _dpl, _segment) \
32 .segment = _segment, \
36 #define INTG(_vector, _addr) \
37 G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL0, __KERNEL_CS)
39 /* System interrupt gate */
40 #define SYSG(_vector, _addr) \
41 G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL3, __KERNEL_CS)
44 * Interrupt gate with interrupt stack. The _ist index is the index in
45 * the tss.ist[] array, but for the descriptor it needs to start at 1.
47 #define ISTG(_vector, _addr, _ist) \
48 G(_vector, _addr, _ist + 1, GATE_INTERRUPT, DPL0, __KERNEL_CS)
51 #define TSKG(_vector, _gdt) \
52 G(_vector, NULL, DEFAULT_STACK, GATE_TASK, DPL0, _gdt << 3)
55 static bool idt_setup_done __initdata
;
58 * Early traps running on the DEFAULT_STACK because the other interrupt
59 * stacks work only after cpu_init().
61 static const __initconst
struct idt_data early_idts
[] = {
62 INTG(X86_TRAP_DB
, debug
),
63 SYSG(X86_TRAP_BP
, int3
),
65 INTG(X86_TRAP_PF
, page_fault
),
70 * The default IDT entries which are set up in trap_init() before
71 * cpu_init() is invoked. Interrupt stacks cannot be used at that point and
72 * the traps which use them are reinitialized with IST after cpu_init() has
75 static const __initconst
struct idt_data def_idts
[] = {
76 INTG(X86_TRAP_DE
, asm_exc_divide_error
),
77 INTG(X86_TRAP_NMI
, nmi
),
78 INTG(X86_TRAP_BR
, asm_exc_bounds
),
79 INTG(X86_TRAP_UD
, asm_exc_invalid_op
),
80 INTG(X86_TRAP_NM
, asm_exc_device_not_available
),
81 INTG(X86_TRAP_OLD_MF
, asm_exc_coproc_segment_overrun
),
82 INTG(X86_TRAP_TS
, asm_exc_invalid_tss
),
83 INTG(X86_TRAP_NP
, asm_exc_segment_not_present
),
84 INTG(X86_TRAP_SS
, asm_exc_stack_segment
),
85 INTG(X86_TRAP_GP
, asm_exc_general_protection
),
86 INTG(X86_TRAP_SPURIOUS
, asm_exc_spurious_interrupt_bug
),
87 INTG(X86_TRAP_MF
, asm_exc_coprocessor_error
),
88 INTG(X86_TRAP_AC
, alignment_check
),
89 INTG(X86_TRAP_XF
, simd_coprocessor_error
),
92 TSKG(X86_TRAP_DF
, GDT_ENTRY_DOUBLEFAULT_TSS
),
94 INTG(X86_TRAP_DF
, double_fault
),
96 INTG(X86_TRAP_DB
, debug
),
99 INTG(X86_TRAP_MC
, machine_check
),
102 SYSG(X86_TRAP_OF
, asm_exc_overflow
),
103 #if defined(CONFIG_IA32_EMULATION)
104 SYSG(IA32_SYSCALL_VECTOR
, entry_INT80_compat
),
105 #elif defined(CONFIG_X86_32)
106 SYSG(IA32_SYSCALL_VECTOR
, entry_INT80_32
),
111 * The APIC and SMP idt entries
113 static const __initconst
struct idt_data apic_idts
[] = {
115 INTG(RESCHEDULE_VECTOR
, reschedule_interrupt
),
116 INTG(CALL_FUNCTION_VECTOR
, call_function_interrupt
),
117 INTG(CALL_FUNCTION_SINGLE_VECTOR
, call_function_single_interrupt
),
118 INTG(IRQ_MOVE_CLEANUP_VECTOR
, irq_move_cleanup_interrupt
),
119 INTG(REBOOT_VECTOR
, reboot_interrupt
),
122 #ifdef CONFIG_X86_THERMAL_VECTOR
123 INTG(THERMAL_APIC_VECTOR
, thermal_interrupt
),
126 #ifdef CONFIG_X86_MCE_THRESHOLD
127 INTG(THRESHOLD_APIC_VECTOR
, threshold_interrupt
),
130 #ifdef CONFIG_X86_MCE_AMD
131 INTG(DEFERRED_ERROR_VECTOR
, deferred_error_interrupt
),
134 #ifdef CONFIG_X86_LOCAL_APIC
135 INTG(LOCAL_TIMER_VECTOR
, apic_timer_interrupt
),
136 INTG(X86_PLATFORM_IPI_VECTOR
, x86_platform_ipi
),
137 # ifdef CONFIG_HAVE_KVM
138 INTG(POSTED_INTR_VECTOR
, kvm_posted_intr_ipi
),
139 INTG(POSTED_INTR_WAKEUP_VECTOR
, kvm_posted_intr_wakeup_ipi
),
140 INTG(POSTED_INTR_NESTED_VECTOR
, kvm_posted_intr_nested_ipi
),
142 # ifdef CONFIG_IRQ_WORK
143 INTG(IRQ_WORK_VECTOR
, irq_work_interrupt
),
146 INTG(UV_BAU_MESSAGE
, uv_bau_message_intr1
),
148 INTG(SPURIOUS_APIC_VECTOR
, spurious_interrupt
),
149 INTG(ERROR_APIC_VECTOR
, error_interrupt
),
155 * Early traps running on the DEFAULT_STACK because the other interrupt
156 * stacks work only after cpu_init().
158 static const __initconst
struct idt_data early_pf_idts
[] = {
159 INTG(X86_TRAP_PF
, page_fault
),
163 * Override for the debug_idt. Same as the default, but with interrupt
164 * stack set to DEFAULT_STACK (0). Required for NMI trap handling.
166 static const __initconst
struct idt_data dbg_idts
[] = {
167 INTG(X86_TRAP_DB
, debug
),
171 /* Must be page-aligned because the real IDT is used in a fixmap. */
172 gate_desc idt_table
[IDT_ENTRIES
] __page_aligned_bss
;
174 struct desc_ptr idt_descr __ro_after_init
= {
175 .size
= (IDT_ENTRIES
* 2 * sizeof(unsigned long)) - 1,
176 .address
= (unsigned long) idt_table
,
180 /* No need to be aligned, but done to keep all IDTs defined the same way. */
181 gate_desc debug_idt_table
[IDT_ENTRIES
] __page_aligned_bss
;
184 * The exceptions which use Interrupt stacks. They are setup after
185 * cpu_init() when the TSS has been initialized.
187 static const __initconst
struct idt_data ist_idts
[] = {
188 ISTG(X86_TRAP_DB
, debug
, IST_INDEX_DB
),
189 ISTG(X86_TRAP_NMI
, nmi
, IST_INDEX_NMI
),
190 ISTG(X86_TRAP_DF
, double_fault
, IST_INDEX_DF
),
191 #ifdef CONFIG_X86_MCE
192 ISTG(X86_TRAP_MC
, machine_check
, IST_INDEX_MCE
),
197 * Override for the debug_idt. Same as the default, but with interrupt
198 * stack set to DEFAULT_STACK (0). Required for NMI trap handling.
200 const struct desc_ptr debug_idt_descr
= {
201 .size
= IDT_ENTRIES
* 16 - 1,
202 .address
= (unsigned long) debug_idt_table
,
206 static inline void idt_init_desc(gate_desc
*gate
, const struct idt_data
*d
)
208 unsigned long addr
= (unsigned long) d
->addr
;
210 gate
->offset_low
= (u16
) addr
;
211 gate
->segment
= (u16
) d
->segment
;
212 gate
->bits
= d
->bits
;
213 gate
->offset_middle
= (u16
) (addr
>> 16);
215 gate
->offset_high
= (u32
) (addr
>> 32);
221 idt_setup_from_table(gate_desc
*idt
, const struct idt_data
*t
, int size
, bool sys
)
225 for (; size
> 0; t
++, size
--) {
226 idt_init_desc(&desc
, t
);
227 write_idt_entry(idt
, t
->vector
, &desc
);
229 set_bit(t
->vector
, system_vectors
);
233 static void set_intr_gate(unsigned int n
, const void *addr
)
235 struct idt_data data
;
239 memset(&data
, 0, sizeof(data
));
242 data
.segment
= __KERNEL_CS
;
243 data
.bits
.type
= GATE_INTERRUPT
;
246 idt_setup_from_table(idt_table
, &data
, 1, false);
250 * idt_setup_early_traps - Initialize the idt table with early traps
252 * On X8664 these traps do not use interrupt stacks as they can't work
253 * before cpu_init() is invoked and sets up TSS. The IST variants are
254 * installed after that.
256 void __init
idt_setup_early_traps(void)
258 idt_setup_from_table(idt_table
, early_idts
, ARRAY_SIZE(early_idts
),
260 load_idt(&idt_descr
);
264 * idt_setup_traps - Initialize the idt table with default traps
266 void __init
idt_setup_traps(void)
268 idt_setup_from_table(idt_table
, def_idts
, ARRAY_SIZE(def_idts
), true);
273 * idt_setup_early_pf - Initialize the idt table with early pagefault handler
275 * On X8664 this does not use interrupt stacks as they can't work before
276 * cpu_init() is invoked and sets up TSS. The IST variant is installed
279 * FIXME: Why is 32bit and 64bit installing the PF handler at different
280 * places in the early setup code?
282 void __init
idt_setup_early_pf(void)
284 idt_setup_from_table(idt_table
, early_pf_idts
,
285 ARRAY_SIZE(early_pf_idts
), true);
289 * idt_setup_ist_traps - Initialize the idt table with traps using IST
291 void __init
idt_setup_ist_traps(void)
293 idt_setup_from_table(idt_table
, ist_idts
, ARRAY_SIZE(ist_idts
), true);
297 * idt_setup_debugidt_traps - Initialize the debug idt table with debug traps
299 void __init
idt_setup_debugidt_traps(void)
301 memcpy(&debug_idt_table
, &idt_table
, IDT_ENTRIES
* 16);
303 idt_setup_from_table(debug_idt_table
, dbg_idts
, ARRAY_SIZE(dbg_idts
), false);
308 * idt_setup_apic_and_irq_gates - Setup APIC/SMP and normal interrupt gates
310 void __init
idt_setup_apic_and_irq_gates(void)
312 int i
= FIRST_EXTERNAL_VECTOR
;
315 idt_setup_from_table(idt_table
, apic_idts
, ARRAY_SIZE(apic_idts
), true);
317 for_each_clear_bit_from(i
, system_vectors
, FIRST_SYSTEM_VECTOR
) {
318 entry
= irq_entries_start
+ 8 * (i
- FIRST_EXTERNAL_VECTOR
);
319 set_intr_gate(i
, entry
);
322 #ifdef CONFIG_X86_LOCAL_APIC
323 for_each_clear_bit_from(i
, system_vectors
, NR_VECTORS
) {
325 * Don't set the non assigned system vectors in the
326 * system_vectors bitmap. Otherwise they show up in
329 entry
= spurious_entries_start
+ 8 * (i
- FIRST_SYSTEM_VECTOR
);
330 set_intr_gate(i
, entry
);
333 idt_setup_done
= true;
337 * idt_setup_early_handler - Initializes the idt table with early handlers
339 void __init
idt_setup_early_handler(void)
343 for (i
= 0; i
< NUM_EXCEPTION_VECTORS
; i
++)
344 set_intr_gate(i
, early_idt_handler_array
[i
]);
346 for ( ; i
< NR_VECTORS
; i
++)
347 set_intr_gate(i
, early_ignore_irq
);
349 load_idt(&idt_descr
);
353 * idt_invalidate - Invalidate interrupt descriptor table
354 * @addr: The virtual address of the 'invalid' IDT
356 void idt_invalidate(void *addr
)
358 struct desc_ptr idt
= { .address
= (unsigned long) addr
, .size
= 0 };
363 void __init
alloc_intr_gate(unsigned int n
, const void *addr
)
365 if (WARN_ON(n
< FIRST_SYSTEM_VECTOR
))
368 if (WARN_ON(idt_setup_done
))
371 if (!WARN_ON(test_and_set_bit(n
, system_vectors
)))
372 set_intr_gate(n
, addr
);