arch/x86/kernel/idt.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Interrupt descriptor table related code
   4  */
   5 #include <linux/interrupt.h>
   6
   7 #include <asm/traps.h>
   8 #include <asm/proto.h>
   9 #include <asm/desc.h>
  10 #include <asm/hw_irq.h>
  11
  12 struct idt_data {
  13         unsigned int    vector;
  14         unsigned int    segment;
  15         struct idt_bits bits;
  16         const void      *addr;
  17 };
  18
  19 #define DPL0            0x0
  20 #define DPL3            0x3
  21
  22 #define DEFAULT_STACK   0
  23
  24 #define G(_vector, _addr, _ist, _type, _dpl, _segment)  \
  25         {                                               \
  26                 .vector         = _vector,              \
  27                 .bits.ist       = _ist,                 \
  28                 .bits.type      = _type,                \
  29                 .bits.dpl       = _dpl,                 \
  30                 .bits.p         = 1,                    \
  31                 .addr           = _addr,                \
  32                 .segment        = _segment,             \
  33         }
  34
  35 /* Interrupt gate */
  36 #define INTG(_vector, _addr)                            \
  37         G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL0, __KERNEL_CS)
  38
  39 /* System interrupt gate */
  40 #define SYSG(_vector, _addr)                            \
  41         G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL3, __KERNEL_CS)
  42
  43 /*
  44  * Interrupt gate with interrupt stack. The _ist index is the index in
  45  * the tss.ist[] array, but for the descriptor it needs to start at 1.
  46  */
  47 #define ISTG(_vector, _addr, _ist)                      \
  48         G(_vector, _addr, _ist + 1, GATE_INTERRUPT, DPL0, __KERNEL_CS)
  49
  50 /* Task gate */
  51 #define TSKG(_vector, _gdt)                             \
  52         G(_vector, NULL, DEFAULT_STACK, GATE_TASK, DPL0, _gdt << 3)
  53
  54
  55 static bool idt_setup_done __initdata;
  56
  57 /*
  58  * Early traps running on the DEFAULT_STACK because the other interrupt
  59  * stacks work only after cpu_init().
  60  */
  61 static const __initconst struct idt_data early_idts[] = {
  62         INTG(X86_TRAP_DB,               debug),
  63         SYSG(X86_TRAP_BP,               int3),
  64 #ifdef CONFIG_X86_32
  65         INTG(X86_TRAP_PF,               page_fault),
  66 #endif
  67 };
  68
  69 /*
  70  * The default IDT entries which are set up in trap_init() before
  71  * cpu_init() is invoked. Interrupt stacks cannot be used at that point and
  72  * the traps which use them are reinitialized with IST after cpu_init() has
  73  * set up TSS.
  74  */
  75 static const __initconst struct idt_data def_idts[] = {
  76         INTG(X86_TRAP_DE,               asm_exc_divide_error),
  77         INTG(X86_TRAP_NMI,              nmi),
  78         INTG(X86_TRAP_BR,               asm_exc_bounds),
  79         INTG(X86_TRAP_UD,               asm_exc_invalid_op),
  80         INTG(X86_TRAP_NM,               asm_exc_device_not_available),
  81         INTG(X86_TRAP_OLD_MF,           asm_exc_coproc_segment_overrun),
  82         INTG(X86_TRAP_TS,               asm_exc_invalid_tss),
  83         INTG(X86_TRAP_NP,               asm_exc_segment_not_present),
  84         INTG(X86_TRAP_SS,               asm_exc_stack_segment),
  85         INTG(X86_TRAP_GP,               asm_exc_general_protection),
  86         INTG(X86_TRAP_SPURIOUS,         asm_exc_spurious_interrupt_bug),
  87         INTG(X86_TRAP_MF,               asm_exc_coprocessor_error),
  88         INTG(X86_TRAP_AC,               asm_exc_alignment_check),
  89         INTG(X86_TRAP_XF,               simd_coprocessor_error),
  90
  91 #ifdef CONFIG_X86_32
  92         TSKG(X86_TRAP_DF,               GDT_ENTRY_DOUBLEFAULT_TSS),
  93 #else
  94         INTG(X86_TRAP_DF,               double_fault),
  95 #endif
  96         INTG(X86_TRAP_DB,               debug),
  97
  98 #ifdef CONFIG_X86_MCE
  99         INTG(X86_TRAP_MC,               machine_check),
 100 #endif
 101
 102         SYSG(X86_TRAP_OF,               asm_exc_overflow),
 103 #if defined(CONFIG_IA32_EMULATION)
 104         SYSG(IA32_SYSCALL_VECTOR,       entry_INT80_compat),
 105 #elif defined(CONFIG_X86_32)
 106         SYSG(IA32_SYSCALL_VECTOR,       entry_INT80_32),
 107 #endif
 108 };
 109
 110 /*
 111  * The APIC and SMP idt entries
 112  */
 113 static const __initconst struct idt_data apic_idts[] = {
 114 #ifdef CONFIG_SMP
 115         INTG(RESCHEDULE_VECTOR,         reschedule_interrupt),
 116         INTG(CALL_FUNCTION_VECTOR,      call_function_interrupt),
 117         INTG(CALL_FUNCTION_SINGLE_VECTOR, call_function_single_interrupt),
 118         INTG(IRQ_MOVE_CLEANUP_VECTOR,   irq_move_cleanup_interrupt),
 119         INTG(REBOOT_VECTOR,             reboot_interrupt),
 120 #endif
 121
 122 #ifdef CONFIG_X86_THERMAL_VECTOR
 123         INTG(THERMAL_APIC_VECTOR,       thermal_interrupt),
 124 #endif
 125
 126 #ifdef CONFIG_X86_MCE_THRESHOLD
 127         INTG(THRESHOLD_APIC_VECTOR,     threshold_interrupt),
 128 #endif
 129
 130 #ifdef CONFIG_X86_MCE_AMD
 131         INTG(DEFERRED_ERROR_VECTOR,     deferred_error_interrupt),
 132 #endif
 133
 134 #ifdef CONFIG_X86_LOCAL_APIC
 135         INTG(LOCAL_TIMER_VECTOR,        apic_timer_interrupt),
 136         INTG(X86_PLATFORM_IPI_VECTOR,   x86_platform_ipi),
 137 # ifdef CONFIG_HAVE_KVM
 138         INTG(POSTED_INTR_VECTOR,        kvm_posted_intr_ipi),
 139         INTG(POSTED_INTR_WAKEUP_VECTOR, kvm_posted_intr_wakeup_ipi),
 140         INTG(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi),
 141 # endif
 142 # ifdef CONFIG_IRQ_WORK
 143         INTG(IRQ_WORK_VECTOR,           irq_work_interrupt),
 144 # endif
 145 #ifdef CONFIG_X86_UV
 146         INTG(UV_BAU_MESSAGE,            uv_bau_message_intr1),
 147 #endif
 148         INTG(SPURIOUS_APIC_VECTOR,      spurious_interrupt),
 149         INTG(ERROR_APIC_VECTOR,         error_interrupt),
 150 #endif
 151 };
 152
 153 #ifdef CONFIG_X86_64
 154 /*
 155  * Early traps running on the DEFAULT_STACK because the other interrupt
 156  * stacks work only after cpu_init().
 157  */
 158 static const __initconst struct idt_data early_pf_idts[] = {
 159         INTG(X86_TRAP_PF,               page_fault),
 160 };
 161
 162 /*
 163  * Override for the debug_idt. Same as the default, but with interrupt
 164  * stack set to DEFAULT_STACK (0). Required for NMI trap handling.
 165  */
 166 static const __initconst struct idt_data dbg_idts[] = {
 167         INTG(X86_TRAP_DB,       debug),
 168 };
 169 #endif
 170
 171 /* Must be page-aligned because the real IDT is used in a fixmap. */
 172 gate_desc idt_table[IDT_ENTRIES] __page_aligned_bss;
 173
 174 struct desc_ptr idt_descr __ro_after_init = {
 175         .size           = (IDT_ENTRIES * 2 * sizeof(unsigned long)) - 1,
 176         .address        = (unsigned long) idt_table,
 177 };
 178
 179 #ifdef CONFIG_X86_64
 180 /* No need to be aligned, but done to keep all IDTs defined the same way. */
 181 gate_desc debug_idt_table[IDT_ENTRIES] __page_aligned_bss;
 182
 183 /*
 184  * The exceptions which use Interrupt stacks. They are setup after
 185  * cpu_init() when the TSS has been initialized.
 186  */
 187 static const __initconst struct idt_data ist_idts[] = {
 188         ISTG(X86_TRAP_DB,       debug,          IST_INDEX_DB),
 189         ISTG(X86_TRAP_NMI,      nmi,            IST_INDEX_NMI),
 190         ISTG(X86_TRAP_DF,       double_fault,   IST_INDEX_DF),
 191 #ifdef CONFIG_X86_MCE
 192         ISTG(X86_TRAP_MC,       machine_check,  IST_INDEX_MCE),
 193 #endif
 194 };
 195
 196 /*
 197  * Override for the debug_idt. Same as the default, but with interrupt
 198  * stack set to DEFAULT_STACK (0). Required for NMI trap handling.
 199  */
 200 const struct desc_ptr debug_idt_descr = {
 201         .size           = IDT_ENTRIES * 16 - 1,
 202         .address        = (unsigned long) debug_idt_table,
 203 };
 204 #endif
 205
 206 static inline void idt_init_desc(gate_desc *gate, const struct idt_data *d)
 207 {
 208         unsigned long addr = (unsigned long) d->addr;
 209
 210         gate->offset_low        = (u16) addr;
 211         gate->segment           = (u16) d->segment;
 212         gate->bits              = d->bits;
 213         gate->offset_middle     = (u16) (addr >> 16);
 214 #ifdef CONFIG_X86_64
 215         gate->offset_high       = (u32) (addr >> 32);
 216         gate->reserved          = 0;
 217 #endif
 218 }
 219
 220 static void
 221 idt_setup_from_table(gate_desc *idt, const struct idt_data *t, int size, bool sys)
 222 {
 223         gate_desc desc;
 224
 225         for (; size > 0; t++, size--) {
 226                 idt_init_desc(&desc, t);
 227                 write_idt_entry(idt, t->vector, &desc);
 228                 if (sys)
 229                         set_bit(t->vector, system_vectors);
 230         }
 231 }
 232
 233 static void set_intr_gate(unsigned int n, const void *addr)
 234 {
 235         struct idt_data data;
 236
 237         BUG_ON(n > 0xFF);
 238
 239         memset(&data, 0, sizeof(data));
 240         data.vector     = n;
 241         data.addr       = addr;
 242         data.segment    = __KERNEL_CS;
 243         data.bits.type  = GATE_INTERRUPT;
 244         data.bits.p     = 1;
 245
 246         idt_setup_from_table(idt_table, &data, 1, false);
 247 }
 248
 249 /**
 250  * idt_setup_early_traps - Initialize the idt table with early traps
 251  *
 252  * On X8664 these traps do not use interrupt stacks as they can't work
 253  * before cpu_init() is invoked and sets up TSS. The IST variants are
 254  * installed after that.
 255  */
 256 void __init idt_setup_early_traps(void)
 257 {
 258         idt_setup_from_table(idt_table, early_idts, ARRAY_SIZE(early_idts),
 259                              true);
 260         load_idt(&idt_descr);
 261 }
 262
 263 /**
 264  * idt_setup_traps - Initialize the idt table with default traps
 265  */
 266 void __init idt_setup_traps(void)
 267 {
 268         idt_setup_from_table(idt_table, def_idts, ARRAY_SIZE(def_idts), true);
 269 }
 270
 271 #ifdef CONFIG_X86_64
 272 /**
 273  * idt_setup_early_pf - Initialize the idt table with early pagefault handler
 274  *
 275  * On X8664 this does not use interrupt stacks as they can't work before
 276  * cpu_init() is invoked and sets up TSS. The IST variant is installed
 277  * after that.
 278  *
 279  * FIXME: Why is 32bit and 64bit installing the PF handler at different
 280  * places in the early setup code?
 281  */
 282 void __init idt_setup_early_pf(void)
 283 {
 284         idt_setup_from_table(idt_table, early_pf_idts,
 285                              ARRAY_SIZE(early_pf_idts), true);
 286 }
 287
 288 /**
 289  * idt_setup_ist_traps - Initialize the idt table with traps using IST
 290  */
 291 void __init idt_setup_ist_traps(void)
 292 {
 293         idt_setup_from_table(idt_table, ist_idts, ARRAY_SIZE(ist_idts), true);
 294 }
 295
 296 /**
 297  * idt_setup_debugidt_traps - Initialize the debug idt table with debug traps
 298  */
 299 void __init idt_setup_debugidt_traps(void)
 300 {
 301         memcpy(&debug_idt_table, &idt_table, IDT_ENTRIES * 16);
 302
 303         idt_setup_from_table(debug_idt_table, dbg_idts, ARRAY_SIZE(dbg_idts), false);
 304 }
 305 #endif
 306
 307 /**
 308  * idt_setup_apic_and_irq_gates - Setup APIC/SMP and normal interrupt gates
 309  */
 310 void __init idt_setup_apic_and_irq_gates(void)
 311 {
 312         int i = FIRST_EXTERNAL_VECTOR;
 313         void *entry;
 314
 315         idt_setup_from_table(idt_table, apic_idts, ARRAY_SIZE(apic_idts), true);
 316
 317         for_each_clear_bit_from(i, system_vectors, FIRST_SYSTEM_VECTOR) {
 318                 entry = irq_entries_start + 8 * (i - FIRST_EXTERNAL_VECTOR);
 319                 set_intr_gate(i, entry);
 320         }
 321
 322 #ifdef CONFIG_X86_LOCAL_APIC
 323         for_each_clear_bit_from(i, system_vectors, NR_VECTORS) {
 324                 /*
 325                  * Don't set the non assigned system vectors in the
 326                  * system_vectors bitmap. Otherwise they show up in
 327                  * /proc/interrupts.
 328                  */
 329                 entry = spurious_entries_start + 8 * (i - FIRST_SYSTEM_VECTOR);
 330                 set_intr_gate(i, entry);
 331         }
 332 #endif
 333         idt_setup_done = true;
 334 }
 335
 336 /**
 337  * idt_setup_early_handler - Initializes the idt table with early handlers
 338  */
 339 void __init idt_setup_early_handler(void)
 340 {
 341         int i;
 342
 343         for (i = 0; i < NUM_EXCEPTION_VECTORS; i++)
 344                 set_intr_gate(i, early_idt_handler_array[i]);
 345 #ifdef CONFIG_X86_32
 346         for ( ; i < NR_VECTORS; i++)
 347                 set_intr_gate(i, early_ignore_irq);
 348 #endif
 349         load_idt(&idt_descr);
 350 }
 351
 352 /**
 353  * idt_invalidate - Invalidate interrupt descriptor table
 354  * @addr:       The virtual address of the 'invalid' IDT
 355  */
 356 void idt_invalidate(void *addr)
 357 {
 358         struct desc_ptr idt = { .address = (unsigned long) addr, .size = 0 };
 359
 360         load_idt(&idt);
 361 }
 362
 363 void __init alloc_intr_gate(unsigned int n, const void *addr)
 364 {
 365         if (WARN_ON(n < FIRST_SYSTEM_VECTOR))
 366                 return;
 367
 368         if (WARN_ON(idt_setup_done))
 369                 return;
 370
 371         if (!WARN_ON(test_and_set_bit(n, system_vectors)))
 372                 set_intr_gate(n, addr);
 373 }