1 // SPDX-License-Identifier: GPL-2.0-only
3 * Local APIC handling, local APIC timers
5 * (c) 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
8 * Maciej W. Rozycki : Bits for genuine 82489DX APICs;
9 * thanks to Eric Gilmore
11 * for testing these extensively.
12 * Maciej W. Rozycki : Various updates and fixes.
13 * Mikael Pettersson : Power Management for UP-APIC.
15 * Mikael Pettersson : PM converted to driver model.
18 #include <linux/perf_event.h>
19 #include <linux/kernel_stat.h>
20 #include <linux/mc146818rtc.h>
21 #include <linux/acpi_pmtmr.h>
22 #include <linux/clockchips.h>
23 #include <linux/interrupt.h>
24 #include <linux/memblock.h>
25 #include <linux/ftrace.h>
26 #include <linux/ioport.h>
27 #include <linux/export.h>
28 #include <linux/syscore_ops.h>
29 #include <linux/delay.h>
30 #include <linux/timex.h>
31 #include <linux/i8253.h>
32 #include <linux/dmar.h>
33 #include <linux/init.h>
34 #include <linux/cpu.h>
35 #include <linux/dmi.h>
36 #include <linux/smp.h>
39 #include <asm/trace/irq_vectors.h>
40 #include <asm/irq_remapping.h>
41 #include <asm/perf_event.h>
42 #include <asm/x86_init.h>
43 #include <linux/atomic.h>
44 #include <asm/barrier.h>
45 #include <asm/mpspec.h>
46 #include <asm/i8259.h>
47 #include <asm/proto.h>
48 #include <asm/traps.h>
51 #include <asm/io_apic.h>
59 #include <asm/hypervisor.h>
60 #include <asm/cpu_device_id.h>
61 #include <asm/intel-family.h>
62 #include <asm/irq_regs.h>
64 unsigned int num_processors
;
66 unsigned disabled_cpus
;
68 /* Processor that is doing the boot up */
69 unsigned int boot_cpu_physical_apicid __ro_after_init
= -1U;
70 EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid
);
72 u8 boot_cpu_apic_version __ro_after_init
;
75 * The highest APIC ID seen during enumeration.
77 static unsigned int max_physical_apicid
;
80 * Bitmask of physically existing CPUs:
82 physid_mask_t phys_cpu_present_map
;
85 * Processor to be disabled specified by kernel parameter
86 * disable_cpu_apicid=<int>, mostly used for the kdump 2nd kernel to
87 * avoid undefined behaviour caused by sending INIT from AP to BSP.
89 static unsigned int disabled_cpu_apicid __ro_after_init
= BAD_APICID
;
92 * This variable controls which CPUs receive external NMIs. By default,
93 * external NMIs are delivered only to the BSP.
95 static int apic_extnmi __ro_after_init
= APIC_EXTNMI_BSP
;
98 * Hypervisor supports 15 bits of APIC ID in MSI Extended Destination ID
100 static bool virt_ext_dest_id __ro_after_init
;
103 * Map cpu index to physical APIC ID
105 DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16
, x86_cpu_to_apicid
, BAD_APICID
);
106 DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16
, x86_bios_cpu_apicid
, BAD_APICID
);
107 DEFINE_EARLY_PER_CPU_READ_MOSTLY(u32
, x86_cpu_to_acpiid
, U32_MAX
);
108 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid
);
109 EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid
);
110 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_acpiid
);
115 * On x86_32, the mapping between cpu and logical apicid may vary
116 * depending on apic in use. The following early percpu variable is
117 * used for the mapping. This is where the behaviors of x86_64 and 32
118 * actually diverge. Let's keep it ugly for now.
120 DEFINE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid
, BAD_APICID
);
122 /* Local APIC was disabled by the BIOS and enabled by the kernel */
123 static int enabled_via_apicbase __ro_after_init
;
126 * Handle interrupt mode configuration register (IMCR).
127 * This register controls whether the interrupt signals
128 * that reach the BSP come from the master PIC or from the
129 * local APIC. Before entering Symmetric I/O Mode, either
130 * the BIOS or the operating system must switch out of
131 * PIC Mode by changing the IMCR.
133 static inline void imcr_pic_to_apic(void)
135 /* select IMCR register */
137 /* NMI and 8259 INTR go through APIC */
141 static inline void imcr_apic_to_pic(void)
143 /* select IMCR register */
145 /* NMI and 8259 INTR go directly to BSP */
151 * Knob to control our willingness to enable the local APIC.
155 static int force_enable_local_apic __initdata
;
158 * APIC command line parameters
160 static int __init
parse_lapic(char *arg
)
162 if (IS_ENABLED(CONFIG_X86_32
) && !arg
)
163 force_enable_local_apic
= 1;
164 else if (arg
&& !strncmp(arg
, "notscdeadline", 13))
165 setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER
);
168 early_param("lapic", parse_lapic
);
171 static int apic_calibrate_pmtmr __initdata
;
172 static __init
int setup_apicpmtimer(char *s
)
174 apic_calibrate_pmtmr
= 1;
178 __setup("apicpmtimer", setup_apicpmtimer
);
181 unsigned long mp_lapic_addr __ro_after_init
;
182 int disable_apic __ro_after_init
;
183 /* Disable local APIC timer from the kernel commandline or via dmi quirk */
184 static int disable_apic_timer __initdata
;
185 /* Local APIC timer works in C2 */
186 int local_apic_timer_c2_ok __ro_after_init
;
187 EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok
);
190 * Debug level, exported for io_apic.c
192 int apic_verbosity __ro_after_init
;
194 int pic_mode __ro_after_init
;
196 /* Have we found an MP table */
197 int smp_found_config __ro_after_init
;
199 static struct resource lapic_resource
= {
200 .name
= "Local APIC",
201 .flags
= IORESOURCE_MEM
| IORESOURCE_BUSY
,
204 unsigned int lapic_timer_period
= 0;
206 static void apic_pm_activate(void);
208 static unsigned long apic_phys __ro_after_init
;
211 * Get the LAPIC version
213 static inline int lapic_get_version(void)
215 return GET_APIC_VERSION(apic_read(APIC_LVR
));
219 * Check, if the APIC is integrated or a separate chip
221 static inline int lapic_is_integrated(void)
223 return APIC_INTEGRATED(lapic_get_version());
227 * Check, whether this is a modern or a first generation APIC
229 static int modern_apic(void)
231 /* AMD systems use old APIC versions, so check the CPU */
232 if (boot_cpu_data
.x86_vendor
== X86_VENDOR_AMD
&&
233 boot_cpu_data
.x86
>= 0xf)
236 /* Hygon systems use modern APIC */
237 if (boot_cpu_data
.x86_vendor
== X86_VENDOR_HYGON
)
240 return lapic_get_version() >= 0x14;
244 * right after this call apic become NOOP driven
245 * so apic->write/read doesn't do anything
247 static void __init
apic_disable(void)
249 pr_info("APIC: switched to apic NOOP\n");
253 void native_apic_wait_icr_idle(void)
255 while (apic_read(APIC_ICR
) & APIC_ICR_BUSY
)
259 u32
native_safe_apic_wait_icr_idle(void)
266 send_status
= apic_read(APIC_ICR
) & APIC_ICR_BUSY
;
269 inc_irq_stat(icr_read_retry_count
);
271 } while (timeout
++ < 1000);
276 void native_apic_icr_write(u32 low
, u32 id
)
280 local_irq_save(flags
);
281 apic_write(APIC_ICR2
, SET_APIC_DEST_FIELD(id
));
282 apic_write(APIC_ICR
, low
);
283 local_irq_restore(flags
);
286 u64
native_apic_icr_read(void)
290 icr2
= apic_read(APIC_ICR2
);
291 icr1
= apic_read(APIC_ICR
);
293 return icr1
| ((u64
)icr2
<< 32);
298 * get_physical_broadcast - Get number of physical broadcast IDs
300 int get_physical_broadcast(void)
302 return modern_apic() ? 0xff : 0xf;
307 * lapic_get_maxlvt - get the maximum number of local vector table entries
309 int lapic_get_maxlvt(void)
312 * - we always have APIC integrated on 64bit mode
313 * - 82489DXs do not report # of LVT entries
315 return lapic_is_integrated() ? GET_APIC_MAXLVT(apic_read(APIC_LVR
)) : 2;
323 #define APIC_DIVISOR 16
324 #define TSC_DIVISOR 8
327 * This function sets up the local APIC timer, with a timeout of
328 * 'clocks' APIC bus clock. During calibration we actually call
329 * this function twice on the boot CPU, once with a bogus timeout
330 * value, second time for real. The other (noncalibrating) CPUs
331 * call this function only once, with the real, calibrated value.
333 * We do reads before writes even if unnecessary, to get around the
334 * P5 APIC double write bug.
336 static void __setup_APIC_LVTT(unsigned int clocks
, int oneshot
, int irqen
)
338 unsigned int lvtt_value
, tmp_value
;
340 lvtt_value
= LOCAL_TIMER_VECTOR
;
342 lvtt_value
|= APIC_LVT_TIMER_PERIODIC
;
343 else if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER
))
344 lvtt_value
|= APIC_LVT_TIMER_TSCDEADLINE
;
346 if (!lapic_is_integrated())
347 lvtt_value
|= SET_APIC_TIMER_BASE(APIC_TIMER_BASE_DIV
);
350 lvtt_value
|= APIC_LVT_MASKED
;
352 apic_write(APIC_LVTT
, lvtt_value
);
354 if (lvtt_value
& APIC_LVT_TIMER_TSCDEADLINE
) {
356 * See Intel SDM: TSC-Deadline Mode chapter. In xAPIC mode,
357 * writing to the APIC LVTT and TSC_DEADLINE MSR isn't serialized.
358 * According to Intel, MFENCE can do the serialization here.
360 asm volatile("mfence" : : : "memory");
367 tmp_value
= apic_read(APIC_TDCR
);
368 apic_write(APIC_TDCR
,
369 (tmp_value
& ~(APIC_TDR_DIV_1
| APIC_TDR_DIV_TMBASE
)) |
373 apic_write(APIC_TMICT
, clocks
/ APIC_DIVISOR
);
377 * Setup extended LVT, AMD specific
379 * Software should use the LVT offsets the BIOS provides. The offsets
380 * are determined by the subsystems using it like those for MCE
381 * threshold or IBS. On K8 only offset 0 (APIC500) and MCE interrupts
382 * are supported. Beginning with family 10h at least 4 offsets are
385 * Since the offsets must be consistent for all cores, we keep track
386 * of the LVT offsets in software and reserve the offset for the same
387 * vector also to be used on other cores. An offset is freed by
388 * setting the entry to APIC_EILVT_MASKED.
390 * If the BIOS is right, there should be no conflicts. Otherwise a
391 * "[Firmware Bug]: ..." error message is generated. However, if
392 * software does not properly determines the offsets, it is not
393 * necessarily a BIOS bug.
396 static atomic_t eilvt_offsets
[APIC_EILVT_NR_MAX
];
398 static inline int eilvt_entry_is_changeable(unsigned int old
, unsigned int new)
400 return (old
& APIC_EILVT_MASKED
)
401 || (new == APIC_EILVT_MASKED
)
402 || ((new & ~APIC_EILVT_MASKED
) == old
);
405 static unsigned int reserve_eilvt_offset(int offset
, unsigned int new)
407 unsigned int rsvd
, vector
;
409 if (offset
>= APIC_EILVT_NR_MAX
)
412 rsvd
= atomic_read(&eilvt_offsets
[offset
]);
414 vector
= rsvd
& ~APIC_EILVT_MASKED
; /* 0: unassigned */
415 if (vector
&& !eilvt_entry_is_changeable(vector
, new))
416 /* may not change if vectors are different */
418 rsvd
= atomic_cmpxchg(&eilvt_offsets
[offset
], rsvd
, new);
419 } while (rsvd
!= new);
421 rsvd
&= ~APIC_EILVT_MASKED
;
422 if (rsvd
&& rsvd
!= vector
)
423 pr_info("LVT offset %d assigned for vector 0x%02x\n",
430 * If mask=1, the LVT entry does not generate interrupts while mask=0
431 * enables the vector. See also the BKDGs. Must be called with
432 * preemption disabled.
435 int setup_APIC_eilvt(u8 offset
, u8 vector
, u8 msg_type
, u8 mask
)
437 unsigned long reg
= APIC_EILVTn(offset
);
438 unsigned int new, old
, reserved
;
440 new = (mask
<< 16) | (msg_type
<< 8) | vector
;
441 old
= apic_read(reg
);
442 reserved
= reserve_eilvt_offset(offset
, new);
444 if (reserved
!= new) {
445 pr_err(FW_BUG
"cpu %d, try to use APIC%lX (LVT offset %d) for "
446 "vector 0x%x, but the register is already in use for "
447 "vector 0x%x on another cpu\n",
448 smp_processor_id(), reg
, offset
, new, reserved
);
452 if (!eilvt_entry_is_changeable(old
, new)) {
453 pr_err(FW_BUG
"cpu %d, try to use APIC%lX (LVT offset %d) for "
454 "vector 0x%x, but the register is already in use for "
455 "vector 0x%x on this cpu\n",
456 smp_processor_id(), reg
, offset
, new, old
);
460 apic_write(reg
, new);
464 EXPORT_SYMBOL_GPL(setup_APIC_eilvt
);
467 * Program the next event, relative to now
469 static int lapic_next_event(unsigned long delta
,
470 struct clock_event_device
*evt
)
472 apic_write(APIC_TMICT
, delta
);
476 static int lapic_next_deadline(unsigned long delta
,
477 struct clock_event_device
*evt
)
481 /* This MSR is special and need a special fence: */
485 wrmsrl(MSR_IA32_TSC_DEADLINE
, tsc
+ (((u64
) delta
) * TSC_DIVISOR
));
489 static int lapic_timer_shutdown(struct clock_event_device
*evt
)
493 /* Lapic used as dummy for broadcast ? */
494 if (evt
->features
& CLOCK_EVT_FEAT_DUMMY
)
497 v
= apic_read(APIC_LVTT
);
498 v
|= (APIC_LVT_MASKED
| LOCAL_TIMER_VECTOR
);
499 apic_write(APIC_LVTT
, v
);
500 apic_write(APIC_TMICT
, 0);
505 lapic_timer_set_periodic_oneshot(struct clock_event_device
*evt
, bool oneshot
)
507 /* Lapic used as dummy for broadcast ? */
508 if (evt
->features
& CLOCK_EVT_FEAT_DUMMY
)
511 __setup_APIC_LVTT(lapic_timer_period
, oneshot
, 1);
515 static int lapic_timer_set_periodic(struct clock_event_device
*evt
)
517 return lapic_timer_set_periodic_oneshot(evt
, false);
520 static int lapic_timer_set_oneshot(struct clock_event_device
*evt
)
522 return lapic_timer_set_periodic_oneshot(evt
, true);
526 * Local APIC timer broadcast function
528 static void lapic_timer_broadcast(const struct cpumask
*mask
)
531 apic
->send_IPI_mask(mask
, LOCAL_TIMER_VECTOR
);
537 * The local apic timer can be used for any function which is CPU local.
539 static struct clock_event_device lapic_clockevent
= {
541 .features
= CLOCK_EVT_FEAT_PERIODIC
|
542 CLOCK_EVT_FEAT_ONESHOT
| CLOCK_EVT_FEAT_C3STOP
543 | CLOCK_EVT_FEAT_DUMMY
,
545 .set_state_shutdown
= lapic_timer_shutdown
,
546 .set_state_periodic
= lapic_timer_set_periodic
,
547 .set_state_oneshot
= lapic_timer_set_oneshot
,
548 .set_state_oneshot_stopped
= lapic_timer_shutdown
,
549 .set_next_event
= lapic_next_event
,
550 .broadcast
= lapic_timer_broadcast
,
554 static DEFINE_PER_CPU(struct clock_event_device
, lapic_events
);
556 static const struct x86_cpu_id deadline_match
[] __initconst
= {
557 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(HASWELL_X
, X86_STEPPINGS(0x2, 0x2), 0x3a), /* EP */
558 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(HASWELL_X
, X86_STEPPINGS(0x4, 0x4), 0x0f), /* EX */
560 X86_MATCH_INTEL_FAM6_MODEL( BROADWELL_X
, 0x0b000020),
562 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D
, X86_STEPPINGS(0x2, 0x2), 0x00000011),
563 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D
, X86_STEPPINGS(0x3, 0x3), 0x0700000e),
564 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D
, X86_STEPPINGS(0x4, 0x4), 0x0f00000c),
565 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D
, X86_STEPPINGS(0x5, 0x5), 0x0e000003),
567 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X
, X86_STEPPINGS(0x3, 0x3), 0x01000136),
568 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X
, X86_STEPPINGS(0x4, 0x4), 0x02000014),
569 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X
, X86_STEPPINGS(0x5, 0xf), 0),
571 X86_MATCH_INTEL_FAM6_MODEL( HASWELL
, 0x22),
572 X86_MATCH_INTEL_FAM6_MODEL( HASWELL_L
, 0x20),
573 X86_MATCH_INTEL_FAM6_MODEL( HASWELL_G
, 0x17),
575 X86_MATCH_INTEL_FAM6_MODEL( BROADWELL
, 0x25),
576 X86_MATCH_INTEL_FAM6_MODEL( BROADWELL_G
, 0x17),
578 X86_MATCH_INTEL_FAM6_MODEL( SKYLAKE_L
, 0xb2),
579 X86_MATCH_INTEL_FAM6_MODEL( SKYLAKE
, 0xb2),
581 X86_MATCH_INTEL_FAM6_MODEL( KABYLAKE_L
, 0x52),
582 X86_MATCH_INTEL_FAM6_MODEL( KABYLAKE
, 0x52),
587 static __init
bool apic_validate_deadline_timer(void)
589 const struct x86_cpu_id
*m
;
592 if (!boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER
))
594 if (boot_cpu_has(X86_FEATURE_HYPERVISOR
))
597 m
= x86_match_cpu(deadline_match
);
601 rev
= (u32
)m
->driver_data
;
603 if (boot_cpu_data
.microcode
>= rev
)
606 setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER
);
607 pr_err(FW_BUG
"TSC_DEADLINE disabled due to Errata; "
608 "please update microcode to version: 0x%x (or later)\n", rev
);
613 * Setup the local APIC timer for this CPU. Copy the initialized values
614 * of the boot CPU and register the clock event in the framework.
616 static void setup_APIC_timer(void)
618 struct clock_event_device
*levt
= this_cpu_ptr(&lapic_events
);
620 if (this_cpu_has(X86_FEATURE_ARAT
)) {
621 lapic_clockevent
.features
&= ~CLOCK_EVT_FEAT_C3STOP
;
622 /* Make LAPIC timer preferrable over percpu HPET */
623 lapic_clockevent
.rating
= 150;
626 memcpy(levt
, &lapic_clockevent
, sizeof(*levt
));
627 levt
->cpumask
= cpumask_of(smp_processor_id());
629 if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER
)) {
630 levt
->name
= "lapic-deadline";
631 levt
->features
&= ~(CLOCK_EVT_FEAT_PERIODIC
|
632 CLOCK_EVT_FEAT_DUMMY
);
633 levt
->set_next_event
= lapic_next_deadline
;
634 clockevents_config_and_register(levt
,
635 tsc_khz
* (1000 / TSC_DIVISOR
),
638 clockevents_register_device(levt
);
642 * Install the updated TSC frequency from recalibration at the TSC
643 * deadline clockevent devices.
645 static void __lapic_update_tsc_freq(void *info
)
647 struct clock_event_device
*levt
= this_cpu_ptr(&lapic_events
);
649 if (!this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER
))
652 clockevents_update_freq(levt
, tsc_khz
* (1000 / TSC_DIVISOR
));
655 void lapic_update_tsc_freq(void)
658 * The clockevent device's ->mult and ->shift can both be
659 * changed. In order to avoid races, schedule the frequency
660 * update code on each CPU.
662 on_each_cpu(__lapic_update_tsc_freq
, NULL
, 0);
666 * In this functions we calibrate APIC bus clocks to the external timer.
668 * We want to do the calibration only once since we want to have local timer
669 * irqs syncron. CPUs connected by the same APIC bus have the very same bus
672 * This was previously done by reading the PIT/HPET and waiting for a wrap
673 * around to find out, that a tick has elapsed. I have a box, where the PIT
674 * readout is broken, so it never gets out of the wait loop again. This was
675 * also reported by others.
677 * Monitoring the jiffies value is inaccurate and the clockevents
678 * infrastructure allows us to do a simple substitution of the interrupt
681 * The calibration routine also uses the pm_timer when possible, as the PIT
682 * happens to run way too slow (factor 2.3 on my VAIO CoreDuo, which goes
683 * back to normal later in the boot process).
686 #define LAPIC_CAL_LOOPS (HZ/10)
688 static __initdata
int lapic_cal_loops
= -1;
689 static __initdata
long lapic_cal_t1
, lapic_cal_t2
;
690 static __initdata
unsigned long long lapic_cal_tsc1
, lapic_cal_tsc2
;
691 static __initdata
unsigned long lapic_cal_pm1
, lapic_cal_pm2
;
692 static __initdata
unsigned long lapic_cal_j1
, lapic_cal_j2
;
695 * Temporary interrupt handler and polled calibration function.
697 static void __init
lapic_cal_handler(struct clock_event_device
*dev
)
699 unsigned long long tsc
= 0;
700 long tapic
= apic_read(APIC_TMCCT
);
701 unsigned long pm
= acpi_pm_read_early();
703 if (boot_cpu_has(X86_FEATURE_TSC
))
706 switch (lapic_cal_loops
++) {
708 lapic_cal_t1
= tapic
;
709 lapic_cal_tsc1
= tsc
;
711 lapic_cal_j1
= jiffies
;
714 case LAPIC_CAL_LOOPS
:
715 lapic_cal_t2
= tapic
;
716 lapic_cal_tsc2
= tsc
;
717 if (pm
< lapic_cal_pm1
)
718 pm
+= ACPI_PM_OVRRUN
;
720 lapic_cal_j2
= jiffies
;
726 calibrate_by_pmtimer(long deltapm
, long *delta
, long *deltatsc
)
728 const long pm_100ms
= PMTMR_TICKS_PER_SEC
/ 10;
729 const long pm_thresh
= pm_100ms
/ 100;
733 #ifndef CONFIG_X86_PM_TIMER
737 apic_printk(APIC_VERBOSE
, "... PM-Timer delta = %ld\n", deltapm
);
739 /* Check, if the PM timer is available */
743 mult
= clocksource_hz2mult(PMTMR_TICKS_PER_SEC
, 22);
745 if (deltapm
> (pm_100ms
- pm_thresh
) &&
746 deltapm
< (pm_100ms
+ pm_thresh
)) {
747 apic_printk(APIC_VERBOSE
, "... PM-Timer result ok\n");
751 res
= (((u64
)deltapm
) * mult
) >> 22;
752 do_div(res
, 1000000);
753 pr_warn("APIC calibration not consistent "
754 "with PM-Timer: %ldms instead of 100ms\n", (long)res
);
756 /* Correct the lapic counter value */
757 res
= (((u64
)(*delta
)) * pm_100ms
);
758 do_div(res
, deltapm
);
759 pr_info("APIC delta adjusted to PM-Timer: "
760 "%lu (%ld)\n", (unsigned long)res
, *delta
);
763 /* Correct the tsc counter value */
764 if (boot_cpu_has(X86_FEATURE_TSC
)) {
765 res
= (((u64
)(*deltatsc
)) * pm_100ms
);
766 do_div(res
, deltapm
);
767 apic_printk(APIC_VERBOSE
, "TSC delta adjusted to "
768 "PM-Timer: %lu (%ld)\n",
769 (unsigned long)res
, *deltatsc
);
770 *deltatsc
= (long)res
;
776 static int __init
lapic_init_clockevent(void)
778 if (!lapic_timer_period
)
781 /* Calculate the scaled math multiplication factor */
782 lapic_clockevent
.mult
= div_sc(lapic_timer_period
/APIC_DIVISOR
,
783 TICK_NSEC
, lapic_clockevent
.shift
);
784 lapic_clockevent
.max_delta_ns
=
785 clockevent_delta2ns(0x7FFFFFFF, &lapic_clockevent
);
786 lapic_clockevent
.max_delta_ticks
= 0x7FFFFFFF;
787 lapic_clockevent
.min_delta_ns
=
788 clockevent_delta2ns(0xF, &lapic_clockevent
);
789 lapic_clockevent
.min_delta_ticks
= 0xF;
794 bool __init
apic_needs_pit(void)
797 * If the frequencies are not known, PIT is required for both TSC
798 * and apic timer calibration.
800 if (!tsc_khz
|| !cpu_khz
)
803 /* Is there an APIC at all or is it disabled? */
804 if (!boot_cpu_has(X86_FEATURE_APIC
) || disable_apic
)
808 * If interrupt delivery mode is legacy PIC or virtual wire without
809 * configuration, the local APIC timer wont be set up. Make sure
810 * that the PIT is initialized.
812 if (apic_intr_mode
== APIC_PIC
||
813 apic_intr_mode
== APIC_VIRTUAL_WIRE_NO_CONFIG
)
816 /* Virt guests may lack ARAT, but still have DEADLINE */
817 if (!boot_cpu_has(X86_FEATURE_ARAT
))
820 /* Deadline timer is based on TSC so no further PIT action required */
821 if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER
))
824 /* APIC timer disabled? */
825 if (disable_apic_timer
)
828 * The APIC timer frequency is known already, no PIT calibration
829 * required. If unknown, let the PIT be initialized.
831 return lapic_timer_period
== 0;
834 static int __init
calibrate_APIC_clock(void)
836 struct clock_event_device
*levt
= this_cpu_ptr(&lapic_events
);
837 u64 tsc_perj
= 0, tsc_start
= 0;
838 unsigned long jif_start
;
839 unsigned long deltaj
;
840 long delta
, deltatsc
;
841 int pm_referenced
= 0;
843 if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER
))
847 * Check if lapic timer has already been calibrated by platform
848 * specific routine, such as tsc calibration code. If so just fill
849 * in the clockevent structure and return.
851 if (!lapic_init_clockevent()) {
852 apic_printk(APIC_VERBOSE
, "lapic timer already calibrated %d\n",
855 * Direct calibration methods must have an always running
856 * local APIC timer, no need for broadcast timer.
858 lapic_clockevent
.features
&= ~CLOCK_EVT_FEAT_DUMMY
;
862 apic_printk(APIC_VERBOSE
, "Using local APIC timer interrupts.\n"
863 "calibrating APIC timer ...\n");
866 * There are platforms w/o global clockevent devices. Instead of
867 * making the calibration conditional on that, use a polling based
868 * approach everywhere.
873 * Setup the APIC counter to maximum. There is no way the lapic
874 * can underflow in the 100ms detection time frame
876 __setup_APIC_LVTT(0xffffffff, 0, 0);
879 * Methods to terminate the calibration loop:
880 * 1) Global clockevent if available (jiffies)
881 * 2) TSC if available and frequency is known
883 jif_start
= READ_ONCE(jiffies
);
887 tsc_perj
= div_u64((u64
)tsc_khz
* 1000, HZ
);
891 * Enable interrupts so the tick can fire, if a global
892 * clockevent device is available
896 while (lapic_cal_loops
<= LAPIC_CAL_LOOPS
) {
897 /* Wait for a tick to elapse */
900 u64 tsc_now
= rdtsc();
901 if ((tsc_now
- tsc_start
) >= tsc_perj
) {
902 tsc_start
+= tsc_perj
;
906 unsigned long jif_now
= READ_ONCE(jiffies
);
908 if (time_after(jif_now
, jif_start
)) {
916 /* Invoke the calibration routine */
918 lapic_cal_handler(NULL
);
924 /* Build delta t1-t2 as apic timer counts down */
925 delta
= lapic_cal_t1
- lapic_cal_t2
;
926 apic_printk(APIC_VERBOSE
, "... lapic delta = %ld\n", delta
);
928 deltatsc
= (long)(lapic_cal_tsc2
- lapic_cal_tsc1
);
930 /* we trust the PM based calibration if possible */
931 pm_referenced
= !calibrate_by_pmtimer(lapic_cal_pm2
- lapic_cal_pm1
,
934 lapic_timer_period
= (delta
* APIC_DIVISOR
) / LAPIC_CAL_LOOPS
;
935 lapic_init_clockevent();
937 apic_printk(APIC_VERBOSE
, "..... delta %ld\n", delta
);
938 apic_printk(APIC_VERBOSE
, "..... mult: %u\n", lapic_clockevent
.mult
);
939 apic_printk(APIC_VERBOSE
, "..... calibration result: %u\n",
942 if (boot_cpu_has(X86_FEATURE_TSC
)) {
943 apic_printk(APIC_VERBOSE
, "..... CPU clock speed is "
945 (deltatsc
/ LAPIC_CAL_LOOPS
) / (1000000 / HZ
),
946 (deltatsc
/ LAPIC_CAL_LOOPS
) % (1000000 / HZ
));
949 apic_printk(APIC_VERBOSE
, "..... host bus clock speed is "
951 lapic_timer_period
/ (1000000 / HZ
),
952 lapic_timer_period
% (1000000 / HZ
));
955 * Do a sanity check on the APIC calibration result
957 if (lapic_timer_period
< (1000000 / HZ
)) {
959 pr_warn("APIC frequency too slow, disabling apic timer\n");
963 levt
->features
&= ~CLOCK_EVT_FEAT_DUMMY
;
966 * PM timer calibration failed or not turned on so lets try APIC
967 * timer based calibration, if a global clockevent device is
970 if (!pm_referenced
&& global_clock_event
) {
971 apic_printk(APIC_VERBOSE
, "... verify APIC timer\n");
974 * Setup the apic timer manually
976 levt
->event_handler
= lapic_cal_handler
;
977 lapic_timer_set_periodic(levt
);
978 lapic_cal_loops
= -1;
980 /* Let the interrupts run */
983 while (lapic_cal_loops
<= LAPIC_CAL_LOOPS
)
986 /* Stop the lapic timer */
988 lapic_timer_shutdown(levt
);
991 deltaj
= lapic_cal_j2
- lapic_cal_j1
;
992 apic_printk(APIC_VERBOSE
, "... jiffies delta = %lu\n", deltaj
);
994 /* Check, if the jiffies result is consistent */
995 if (deltaj
>= LAPIC_CAL_LOOPS
-2 && deltaj
<= LAPIC_CAL_LOOPS
+2)
996 apic_printk(APIC_VERBOSE
, "... jiffies result ok\n");
998 levt
->features
|= CLOCK_EVT_FEAT_DUMMY
;
1002 if (levt
->features
& CLOCK_EVT_FEAT_DUMMY
) {
1003 pr_warn("APIC timer disabled due to verification failure\n");
1011 * Setup the boot APIC
1013 * Calibrate and verify the result.
1015 void __init
setup_boot_APIC_clock(void)
1018 * The local apic timer can be disabled via the kernel
1019 * commandline or from the CPU detection code. Register the lapic
1020 * timer as a dummy clock event source on SMP systems, so the
1021 * broadcast mechanism is used. On UP systems simply ignore it.
1023 if (disable_apic_timer
) {
1024 pr_info("Disabling APIC timer\n");
1025 /* No broadcast on UP ! */
1026 if (num_possible_cpus() > 1) {
1027 lapic_clockevent
.mult
= 1;
1033 if (calibrate_APIC_clock()) {
1034 /* No broadcast on UP ! */
1035 if (num_possible_cpus() > 1)
1041 * If nmi_watchdog is set to IO_APIC, we need the
1042 * PIT/HPET going. Otherwise register lapic as a dummy
1045 lapic_clockevent
.features
&= ~CLOCK_EVT_FEAT_DUMMY
;
1047 /* Setup the lapic or request the broadcast */
1049 amd_e400_c1e_apic_setup();
1052 void setup_secondary_APIC_clock(void)
1055 amd_e400_c1e_apic_setup();
1059 * The guts of the apic timer interrupt
1061 static void local_apic_timer_interrupt(void)
1063 struct clock_event_device
*evt
= this_cpu_ptr(&lapic_events
);
1066 * Normally we should not be here till LAPIC has been initialized but
1067 * in some cases like kdump, its possible that there is a pending LAPIC
1068 * timer interrupt from previous kernel's context and is delivered in
1069 * new kernel the moment interrupts are enabled.
1071 * Interrupts are enabled early and LAPIC is setup much later, hence
1072 * its possible that when we get here evt->event_handler is NULL.
1073 * Check for event_handler being NULL and discard the interrupt as
1076 if (!evt
->event_handler
) {
1077 pr_warn("Spurious LAPIC timer interrupt on cpu %d\n",
1078 smp_processor_id());
1080 lapic_timer_shutdown(evt
);
1085 * the NMI deadlock-detector uses this.
1087 inc_irq_stat(apic_timer_irqs
);
1089 evt
->event_handler(evt
);
1093 * Local APIC timer interrupt. This is the most natural way for doing
1094 * local interrupts, but local timer interrupts can be emulated by
1095 * broadcast interrupts too. [in case the hw doesn't support APIC timers]
1097 * [ if a single-CPU system runs an SMP kernel then we call the local
1098 * interrupt as well. Thus we cannot inline the local irq ... ]
1100 DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_interrupt
)
1102 struct pt_regs
*old_regs
= set_irq_regs(regs
);
1105 trace_local_timer_entry(LOCAL_TIMER_VECTOR
);
1106 local_apic_timer_interrupt();
1107 trace_local_timer_exit(LOCAL_TIMER_VECTOR
);
1109 set_irq_regs(old_regs
);
1112 int setup_profiling_timer(unsigned int multiplier
)
1118 * Local APIC start and shutdown
1122 * clear_local_APIC - shutdown the local APIC
1124 * This is called, when a CPU is disabled and before rebooting, so the state of
1125 * the local APIC has no dangling leftovers. Also used to cleanout any BIOS
1126 * leftovers during boot.
1128 void clear_local_APIC(void)
1133 /* APIC hasn't been mapped yet */
1134 if (!x2apic_mode
&& !apic_phys
)
1137 maxlvt
= lapic_get_maxlvt();
1139 * Masking an LVT entry can trigger a local APIC error
1140 * if the vector is zero. Mask LVTERR first to prevent this.
1143 v
= ERROR_APIC_VECTOR
; /* any non-zero vector will do */
1144 apic_write(APIC_LVTERR
, v
| APIC_LVT_MASKED
);
1147 * Careful: we have to set masks only first to deassert
1148 * any level-triggered sources.
1150 v
= apic_read(APIC_LVTT
);
1151 apic_write(APIC_LVTT
, v
| APIC_LVT_MASKED
);
1152 v
= apic_read(APIC_LVT0
);
1153 apic_write(APIC_LVT0
, v
| APIC_LVT_MASKED
);
1154 v
= apic_read(APIC_LVT1
);
1155 apic_write(APIC_LVT1
, v
| APIC_LVT_MASKED
);
1157 v
= apic_read(APIC_LVTPC
);
1158 apic_write(APIC_LVTPC
, v
| APIC_LVT_MASKED
);
1161 /* lets not touch this if we didn't frob it */
1162 #ifdef CONFIG_X86_THERMAL_VECTOR
1164 v
= apic_read(APIC_LVTTHMR
);
1165 apic_write(APIC_LVTTHMR
, v
| APIC_LVT_MASKED
);
1168 #ifdef CONFIG_X86_MCE_INTEL
1170 v
= apic_read(APIC_LVTCMCI
);
1171 if (!(v
& APIC_LVT_MASKED
))
1172 apic_write(APIC_LVTCMCI
, v
| APIC_LVT_MASKED
);
1177 * Clean APIC state for other OSs:
1179 apic_write(APIC_LVTT
, APIC_LVT_MASKED
);
1180 apic_write(APIC_LVT0
, APIC_LVT_MASKED
);
1181 apic_write(APIC_LVT1
, APIC_LVT_MASKED
);
1183 apic_write(APIC_LVTERR
, APIC_LVT_MASKED
);
1185 apic_write(APIC_LVTPC
, APIC_LVT_MASKED
);
1187 /* Integrated APIC (!82489DX) ? */
1188 if (lapic_is_integrated()) {
1190 /* Clear ESR due to Pentium errata 3AP and 11AP */
1191 apic_write(APIC_ESR
, 0);
1192 apic_read(APIC_ESR
);
1197 * apic_soft_disable - Clears and software disables the local APIC on hotplug
1199 * Contrary to disable_local_APIC() this does not touch the enable bit in
1200 * MSR_IA32_APICBASE. Clearing that bit on systems based on the 3 wire APIC
1201 * bus would require a hardware reset as the APIC would lose track of bus
1202 * arbitration. On systems with FSB delivery APICBASE could be disabled,
1203 * but it has to be guaranteed that no interrupt is sent to the APIC while
1204 * in that state and it's not clear from the SDM whether it still responds
1205 * to INIT/SIPI messages. Stay on the safe side and use software disable.
1207 void apic_soft_disable(void)
1213 /* Soft disable APIC (implies clearing of registers for 82489DX!). */
1214 value
= apic_read(APIC_SPIV
);
1215 value
&= ~APIC_SPIV_APIC_ENABLED
;
1216 apic_write(APIC_SPIV
, value
);
1220 * disable_local_APIC - clear and disable the local APIC
1222 void disable_local_APIC(void)
1224 /* APIC hasn't been mapped yet */
1225 if (!x2apic_mode
&& !apic_phys
)
1228 apic_soft_disable();
1230 #ifdef CONFIG_X86_32
1232 * When LAPIC was disabled by the BIOS and enabled by the kernel,
1233 * restore the disabled state.
1235 if (enabled_via_apicbase
) {
1238 rdmsr(MSR_IA32_APICBASE
, l
, h
);
1239 l
&= ~MSR_IA32_APICBASE_ENABLE
;
1240 wrmsr(MSR_IA32_APICBASE
, l
, h
);
1246 * If Linux enabled the LAPIC against the BIOS default disable it down before
1247 * re-entering the BIOS on shutdown. Otherwise the BIOS may get confused and
1248 * not power-off. Additionally clear all LVT entries before disable_local_APIC
1249 * for the case where Linux didn't enable the LAPIC.
1251 void lapic_shutdown(void)
1253 unsigned long flags
;
1255 if (!boot_cpu_has(X86_FEATURE_APIC
) && !apic_from_smp_config())
1258 local_irq_save(flags
);
1260 #ifdef CONFIG_X86_32
1261 if (!enabled_via_apicbase
)
1265 disable_local_APIC();
1268 local_irq_restore(flags
);
1272 * sync_Arb_IDs - synchronize APIC bus arbitration IDs
1274 void __init
sync_Arb_IDs(void)
1277 * Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 And not
1280 if (modern_apic() || boot_cpu_data
.x86_vendor
== X86_VENDOR_AMD
)
1286 apic_wait_icr_idle();
1288 apic_printk(APIC_DEBUG
, "Synchronizing Arb IDs.\n");
1289 apic_write(APIC_ICR
, APIC_DEST_ALLINC
|
1290 APIC_INT_LEVELTRIG
| APIC_DM_INIT
);
1293 enum apic_intr_mode_id apic_intr_mode __ro_after_init
;
1295 static int __init
__apic_intr_mode_select(void)
1297 /* Check kernel option */
1299 pr_info("APIC disabled via kernel command line\n");
1304 #ifdef CONFIG_X86_64
1305 /* On 64-bit, the APIC must be integrated, Check local APIC only */
1306 if (!boot_cpu_has(X86_FEATURE_APIC
)) {
1308 pr_info("APIC disabled by BIOS\n");
1312 /* On 32-bit, the APIC may be integrated APIC or 82489DX */
1314 /* Neither 82489DX nor integrated APIC ? */
1315 if (!boot_cpu_has(X86_FEATURE_APIC
) && !smp_found_config
) {
1320 /* If the BIOS pretends there is an integrated APIC ? */
1321 if (!boot_cpu_has(X86_FEATURE_APIC
) &&
1322 APIC_INTEGRATED(boot_cpu_apic_version
)) {
1324 pr_err(FW_BUG
"Local APIC %d not detected, force emulation\n",
1325 boot_cpu_physical_apicid
);
1330 /* Check MP table or ACPI MADT configuration */
1331 if (!smp_found_config
) {
1332 disable_ioapic_support();
1334 pr_info("APIC: ACPI MADT or MP tables are not detected\n");
1335 return APIC_VIRTUAL_WIRE_NO_CONFIG
;
1337 return APIC_VIRTUAL_WIRE
;
1341 /* If SMP should be disabled, then really disable it! */
1342 if (!setup_max_cpus
) {
1343 pr_info("APIC: SMP mode deactivated\n");
1344 return APIC_SYMMETRIC_IO_NO_ROUTING
;
1347 if (read_apic_id() != boot_cpu_physical_apicid
) {
1348 panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
1349 read_apic_id(), boot_cpu_physical_apicid
);
1350 /* Or can we switch back to PIC here? */
1354 return APIC_SYMMETRIC_IO
;
1357 /* Select the interrupt delivery mode for the BSP */
1358 void __init
apic_intr_mode_select(void)
1360 apic_intr_mode
= __apic_intr_mode_select();
1364 * An initial setup of the virtual wire mode.
1366 void __init
init_bsp_APIC(void)
1371 * Don't do the setup now if we have a SMP BIOS as the
1372 * through-I/O-APIC virtual wire mode might be active.
1374 if (smp_found_config
|| !boot_cpu_has(X86_FEATURE_APIC
))
1378 * Do not trust the local APIC being empty at bootup.
1385 value
= apic_read(APIC_SPIV
);
1386 value
&= ~APIC_VECTOR_MASK
;
1387 value
|= APIC_SPIV_APIC_ENABLED
;
1389 #ifdef CONFIG_X86_32
1390 /* This bit is reserved on P4/Xeon and should be cleared */
1391 if ((boot_cpu_data
.x86_vendor
== X86_VENDOR_INTEL
) &&
1392 (boot_cpu_data
.x86
== 15))
1393 value
&= ~APIC_SPIV_FOCUS_DISABLED
;
1396 value
|= APIC_SPIV_FOCUS_DISABLED
;
1397 value
|= SPURIOUS_APIC_VECTOR
;
1398 apic_write(APIC_SPIV
, value
);
1401 * Set up the virtual wire mode.
1403 apic_write(APIC_LVT0
, APIC_DM_EXTINT
);
1404 value
= APIC_DM_NMI
;
1405 if (!lapic_is_integrated()) /* 82489DX */
1406 value
|= APIC_LVT_LEVEL_TRIGGER
;
1407 if (apic_extnmi
== APIC_EXTNMI_NONE
)
1408 value
|= APIC_LVT_MASKED
;
1409 apic_write(APIC_LVT1
, value
);
1412 static void __init
apic_bsp_setup(bool upmode
);
1414 /* Init the interrupt delivery mode for the BSP */
1415 void __init
apic_intr_mode_init(void)
1417 bool upmode
= IS_ENABLED(CONFIG_UP_LATE_INIT
);
1419 switch (apic_intr_mode
) {
1421 pr_info("APIC: Keep in PIC mode(8259)\n");
1423 case APIC_VIRTUAL_WIRE
:
1424 pr_info("APIC: Switch to virtual wire mode setup\n");
1425 default_setup_apic_routing();
1427 case APIC_VIRTUAL_WIRE_NO_CONFIG
:
1428 pr_info("APIC: Switch to virtual wire mode setup with no configuration\n");
1430 default_setup_apic_routing();
1432 case APIC_SYMMETRIC_IO
:
1433 pr_info("APIC: Switch to symmetric I/O mode setup\n");
1434 default_setup_apic_routing();
1436 case APIC_SYMMETRIC_IO_NO_ROUTING
:
1437 pr_info("APIC: Switch to symmetric I/O mode setup in no SMP routine\n");
1441 if (x86_platform
.apic_post_init
)
1442 x86_platform
.apic_post_init();
1444 apic_bsp_setup(upmode
);
1447 static void lapic_setup_esr(void)
1449 unsigned int oldvalue
, value
, maxlvt
;
1451 if (!lapic_is_integrated()) {
1452 pr_info("No ESR for 82489DX.\n");
1456 if (apic
->disable_esr
) {
1458 * Something untraceable is creating bad interrupts on
1459 * secondary quads ... for the moment, just leave the
1460 * ESR disabled - we can't do anything useful with the
1461 * errors anyway - mbligh
1463 pr_info("Leaving ESR disabled.\n");
1467 maxlvt
= lapic_get_maxlvt();
1468 if (maxlvt
> 3) /* Due to the Pentium erratum 3AP. */
1469 apic_write(APIC_ESR
, 0);
1470 oldvalue
= apic_read(APIC_ESR
);
1472 /* enables sending errors */
1473 value
= ERROR_APIC_VECTOR
;
1474 apic_write(APIC_LVTERR
, value
);
1477 * spec says clear errors after enabling vector.
1480 apic_write(APIC_ESR
, 0);
1481 value
= apic_read(APIC_ESR
);
1482 if (value
!= oldvalue
)
1483 apic_printk(APIC_VERBOSE
, "ESR value before enabling "
1484 "vector: 0x%08x after: 0x%08x\n",
1488 #define APIC_IR_REGS APIC_ISR_NR
1489 #define APIC_IR_BITS (APIC_IR_REGS * 32)
1490 #define APIC_IR_MAPSIZE (APIC_IR_BITS / BITS_PER_LONG)
1493 unsigned long map
[APIC_IR_MAPSIZE
];
1494 u32 regs
[APIC_IR_REGS
];
1497 static bool apic_check_and_ack(union apic_ir
*irr
, union apic_ir
*isr
)
1502 for (i
= 0; i
< APIC_IR_REGS
; i
++)
1503 irr
->regs
[i
] = apic_read(APIC_IRR
+ i
* 0x10);
1506 for (i
= 0; i
< APIC_IR_REGS
; i
++)
1507 isr
->regs
[i
] = apic_read(APIC_ISR
+ i
* 0x10);
1510 * If the ISR map is not empty. ACK the APIC and run another round
1511 * to verify whether a pending IRR has been unblocked and turned
1514 if (!bitmap_empty(isr
->map
, APIC_IR_BITS
)) {
1516 * There can be multiple ISR bits set when a high priority
1517 * interrupt preempted a lower priority one. Issue an ACK
1520 for_each_set_bit(bit
, isr
->map
, APIC_IR_BITS
)
1525 return !bitmap_empty(irr
->map
, APIC_IR_BITS
);
1529 * After a crash, we no longer service the interrupts and a pending
1530 * interrupt from previous kernel might still have ISR bit set.
1532 * Most probably by now the CPU has serviced that pending interrupt and it
1533 * might not have done the ack_APIC_irq() because it thought, interrupt
1534 * came from i8259 as ExtInt. LAPIC did not get EOI so it does not clear
1535 * the ISR bit and cpu thinks it has already serivced the interrupt. Hence
1536 * a vector might get locked. It was noticed for timer irq (vector
1537 * 0x31). Issue an extra EOI to clear ISR.
1539 * If there are pending IRR bits they turn into ISR bits after a higher
1540 * priority ISR bit has been acked.
1542 static void apic_pending_intr_clear(void)
1544 union apic_ir irr
, isr
;
1547 /* 512 loops are way oversized and give the APIC a chance to obey. */
1548 for (i
= 0; i
< 512; i
++) {
1549 if (!apic_check_and_ack(&irr
, &isr
))
1552 /* Dump the IRR/ISR content if that failed */
1553 pr_warn("APIC: Stale IRR: %256pb ISR: %256pb\n", irr
.map
, isr
.map
);
1557 * setup_local_APIC - setup the local APIC
1559 * Used to setup local APIC while initializing BSP or bringing up APs.
1560 * Always called with preemption disabled.
1562 static void setup_local_APIC(void)
1564 int cpu
= smp_processor_id();
1568 disable_ioapic_support();
1573 * If this comes from kexec/kcrash the APIC might be enabled in
1574 * SPIV. Soft disable it before doing further initialization.
1576 value
= apic_read(APIC_SPIV
);
1577 value
&= ~APIC_SPIV_APIC_ENABLED
;
1578 apic_write(APIC_SPIV
, value
);
1580 #ifdef CONFIG_X86_32
1581 /* Pound the ESR really hard over the head with a big hammer - mbligh */
1582 if (lapic_is_integrated() && apic
->disable_esr
) {
1583 apic_write(APIC_ESR
, 0);
1584 apic_write(APIC_ESR
, 0);
1585 apic_write(APIC_ESR
, 0);
1586 apic_write(APIC_ESR
, 0);
1590 * Double-check whether this APIC is really registered.
1591 * This is meaningless in clustered apic mode, so we skip it.
1593 BUG_ON(!apic
->apic_id_registered());
1596 * Intel recommends to set DFR, LDR and TPR before enabling
1597 * an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel
1598 * document number 292116). So here it goes...
1600 apic
->init_apic_ldr();
1602 #ifdef CONFIG_X86_32
1603 if (apic
->dest_mode_logical
) {
1604 int logical_apicid
, ldr_apicid
;
1607 * APIC LDR is initialized. If logical_apicid mapping was
1608 * initialized during get_smp_config(), make sure it matches
1611 logical_apicid
= early_per_cpu(x86_cpu_to_logical_apicid
, cpu
);
1612 ldr_apicid
= GET_APIC_LOGICAL_ID(apic_read(APIC_LDR
));
1613 if (logical_apicid
!= BAD_APICID
)
1614 WARN_ON(logical_apicid
!= ldr_apicid
);
1615 /* Always use the value from LDR. */
1616 early_per_cpu(x86_cpu_to_logical_apicid
, cpu
) = ldr_apicid
;
1621 * Set Task Priority to 'accept all except vectors 0-31'. An APIC
1622 * vector in the 16-31 range could be delivered if TPR == 0, but we
1623 * would think it's an exception and terrible things will happen. We
1624 * never change this later on.
1626 value
= apic_read(APIC_TASKPRI
);
1627 value
&= ~APIC_TPRI_MASK
;
1629 apic_write(APIC_TASKPRI
, value
);
1631 /* Clear eventually stale ISR/IRR bits */
1632 apic_pending_intr_clear();
1635 * Now that we are all set up, enable the APIC
1637 value
= apic_read(APIC_SPIV
);
1638 value
&= ~APIC_VECTOR_MASK
;
1642 value
|= APIC_SPIV_APIC_ENABLED
;
1644 #ifdef CONFIG_X86_32
1646 * Some unknown Intel IO/APIC (or APIC) errata is biting us with
1647 * certain networking cards. If high frequency interrupts are
1648 * happening on a particular IOAPIC pin, plus the IOAPIC routing
1649 * entry is masked/unmasked at a high rate as well then sooner or
1650 * later IOAPIC line gets 'stuck', no more interrupts are received
1651 * from the device. If focus CPU is disabled then the hang goes
1654 * [ This bug can be reproduced easily with a level-triggered
1655 * PCI Ne2000 networking cards and PII/PIII processors, dual
1659 * Actually disabling the focus CPU check just makes the hang less
1660 * frequent as it makes the interrupt distributon model be more
1661 * like LRU than MRU (the short-term load is more even across CPUs).
1665 * - enable focus processor (bit==0)
1666 * - 64bit mode always use processor focus
1667 * so no need to set it
1669 value
&= ~APIC_SPIV_FOCUS_DISABLED
;
1673 * Set spurious IRQ vector
1675 value
|= SPURIOUS_APIC_VECTOR
;
1676 apic_write(APIC_SPIV
, value
);
1678 perf_events_lapic_init();
1681 * Set up LVT0, LVT1:
1683 * set up through-local-APIC on the boot CPU's LINT0. This is not
1684 * strictly necessary in pure symmetric-IO mode, but sometimes
1685 * we delegate interrupts to the 8259A.
1688 * TODO: set up through-local-APIC from through-I/O-APIC? --macro
1690 value
= apic_read(APIC_LVT0
) & APIC_LVT_MASKED
;
1691 if (!cpu
&& (pic_mode
|| !value
|| skip_ioapic_setup
)) {
1692 value
= APIC_DM_EXTINT
;
1693 apic_printk(APIC_VERBOSE
, "enabled ExtINT on CPU#%d\n", cpu
);
1695 value
= APIC_DM_EXTINT
| APIC_LVT_MASKED
;
1696 apic_printk(APIC_VERBOSE
, "masked ExtINT on CPU#%d\n", cpu
);
1698 apic_write(APIC_LVT0
, value
);
1701 * Only the BSP sees the LINT1 NMI signal by default. This can be
1702 * modified by apic_extnmi= boot option.
1704 if ((!cpu
&& apic_extnmi
!= APIC_EXTNMI_NONE
) ||
1705 apic_extnmi
== APIC_EXTNMI_ALL
)
1706 value
= APIC_DM_NMI
;
1708 value
= APIC_DM_NMI
| APIC_LVT_MASKED
;
1711 if (!lapic_is_integrated())
1712 value
|= APIC_LVT_LEVEL_TRIGGER
;
1713 apic_write(APIC_LVT1
, value
);
1715 #ifdef CONFIG_X86_MCE_INTEL
1716 /* Recheck CMCI information after local APIC is up on CPU #0 */
1722 static void end_local_APIC_setup(void)
1726 #ifdef CONFIG_X86_32
1729 /* Disable the local apic timer */
1730 value
= apic_read(APIC_LVTT
);
1731 value
|= (APIC_LVT_MASKED
| LOCAL_TIMER_VECTOR
);
1732 apic_write(APIC_LVTT
, value
);
1740 * APIC setup function for application processors. Called from smpboot.c
1742 void apic_ap_setup(void)
1745 end_local_APIC_setup();
1748 #ifdef CONFIG_X86_X2APIC
1750 EXPORT_SYMBOL_GPL(x2apic_mode
);
1757 static int x2apic_state
;
1759 static void __x2apic_disable(void)
1763 if (!boot_cpu_has(X86_FEATURE_APIC
))
1766 rdmsrl(MSR_IA32_APICBASE
, msr
);
1767 if (!(msr
& X2APIC_ENABLE
))
1769 /* Disable xapic and x2apic first and then reenable xapic mode */
1770 wrmsrl(MSR_IA32_APICBASE
, msr
& ~(X2APIC_ENABLE
| XAPIC_ENABLE
));
1771 wrmsrl(MSR_IA32_APICBASE
, msr
& ~X2APIC_ENABLE
);
1772 printk_once(KERN_INFO
"x2apic disabled\n");
1775 static void __x2apic_enable(void)
1779 rdmsrl(MSR_IA32_APICBASE
, msr
);
1780 if (msr
& X2APIC_ENABLE
)
1782 wrmsrl(MSR_IA32_APICBASE
, msr
| X2APIC_ENABLE
);
1783 printk_once(KERN_INFO
"x2apic enabled\n");
1786 static int __init
setup_nox2apic(char *str
)
1788 if (x2apic_enabled()) {
1789 int apicid
= native_apic_msr_read(APIC_ID
);
1791 if (apicid
>= 255) {
1792 pr_warn("Apicid: %08x, cannot enforce nox2apic\n",
1796 pr_warn("x2apic already enabled.\n");
1799 setup_clear_cpu_cap(X86_FEATURE_X2APIC
);
1800 x2apic_state
= X2APIC_DISABLED
;
1804 early_param("nox2apic", setup_nox2apic
);
1806 /* Called from cpu_init() to enable x2apic on (secondary) cpus */
1807 void x2apic_setup(void)
1810 * If x2apic is not in ON state, disable it if already enabled
1813 if (x2apic_state
!= X2APIC_ON
) {
1820 static __init
void x2apic_disable(void)
1822 u32 x2apic_id
, state
= x2apic_state
;
1825 x2apic_state
= X2APIC_DISABLED
;
1827 if (state
!= X2APIC_ON
)
1830 x2apic_id
= read_apic_id();
1831 if (x2apic_id
>= 255)
1832 panic("Cannot disable x2apic, id: %08x\n", x2apic_id
);
1835 register_lapic_address(mp_lapic_addr
);
1838 static __init
void x2apic_enable(void)
1840 if (x2apic_state
!= X2APIC_OFF
)
1844 x2apic_state
= X2APIC_ON
;
1848 static __init
void try_to_enable_x2apic(int remap_mode
)
1850 if (x2apic_state
== X2APIC_DISABLED
)
1853 if (remap_mode
!= IRQ_REMAP_X2APIC_MODE
) {
1854 u32 apic_limit
= 255;
1857 * Using X2APIC without IR is not architecturally supported
1858 * on bare metal but may be supported in guests.
1860 if (!x86_init
.hyper
.x2apic_available()) {
1861 pr_info("x2apic: IRQ remapping doesn't support X2APIC mode\n");
1867 * If the hypervisor supports extended destination ID in
1868 * MSI, that increases the maximum APIC ID that can be
1869 * used for non-remapped IRQ domains.
1871 if (x86_init
.hyper
.msi_ext_dest_id()) {
1872 virt_ext_dest_id
= 1;
1877 * Without IR, all CPUs can be addressed by IOAPIC/MSI only
1878 * in physical mode, and CPUs with an APIC ID that cannnot
1879 * be addressed must not be brought online.
1881 x2apic_set_max_apicid(apic_limit
);
1887 void __init
check_x2apic(void)
1889 if (x2apic_enabled()) {
1890 pr_info("x2apic: enabled by BIOS, switching to x2apic ops\n");
1892 x2apic_state
= X2APIC_ON
;
1893 } else if (!boot_cpu_has(X86_FEATURE_X2APIC
)) {
1894 x2apic_state
= X2APIC_DISABLED
;
1897 #else /* CONFIG_X86_X2APIC */
1898 static int __init
validate_x2apic(void)
1900 if (!apic_is_x2apic_enabled())
1903 * Checkme: Can we simply turn off x2apic here instead of panic?
1905 panic("BIOS has enabled x2apic but kernel doesn't support x2apic, please disable x2apic in BIOS.\n");
1907 early_initcall(validate_x2apic
);
1909 static inline void try_to_enable_x2apic(int remap_mode
) { }
1910 static inline void __x2apic_enable(void) { }
1911 #endif /* !CONFIG_X86_X2APIC */
1913 void __init
enable_IR_x2apic(void)
1915 unsigned long flags
;
1918 if (skip_ioapic_setup
) {
1919 pr_info("Not enabling interrupt remapping due to skipped IO-APIC setup\n");
1923 ir_stat
= irq_remapping_prepare();
1924 if (ir_stat
< 0 && !x2apic_supported())
1927 ret
= save_ioapic_entries();
1929 pr_info("Saving IO-APIC state failed: %d\n", ret
);
1933 local_irq_save(flags
);
1934 legacy_pic
->mask_all();
1935 mask_ioapic_entries();
1937 /* If irq_remapping_prepare() succeeded, try to enable it */
1939 ir_stat
= irq_remapping_enable();
1940 /* ir_stat contains the remap mode or an error code */
1941 try_to_enable_x2apic(ir_stat
);
1944 restore_ioapic_entries();
1945 legacy_pic
->restore_mask();
1946 local_irq_restore(flags
);
1949 #ifdef CONFIG_X86_64
1951 * Detect and enable local APICs on non-SMP boards.
1952 * Original code written by Keir Fraser.
1953 * On AMD64 we trust the BIOS - if it says no APIC it is likely
1954 * not correctly set up (usually the APIC timer won't work etc.)
1956 static int __init
detect_init_APIC(void)
1958 if (!boot_cpu_has(X86_FEATURE_APIC
)) {
1959 pr_info("No local APIC present\n");
1963 mp_lapic_addr
= APIC_DEFAULT_PHYS_BASE
;
1968 static int __init
apic_verify(void)
1973 * The APIC feature bit should now be enabled
1976 features
= cpuid_edx(1);
1977 if (!(features
& (1 << X86_FEATURE_APIC
))) {
1978 pr_warn("Could not enable APIC!\n");
1981 set_cpu_cap(&boot_cpu_data
, X86_FEATURE_APIC
);
1982 mp_lapic_addr
= APIC_DEFAULT_PHYS_BASE
;
1984 /* The BIOS may have set up the APIC at some other address */
1985 if (boot_cpu_data
.x86
>= 6) {
1986 rdmsr(MSR_IA32_APICBASE
, l
, h
);
1987 if (l
& MSR_IA32_APICBASE_ENABLE
)
1988 mp_lapic_addr
= l
& MSR_IA32_APICBASE_BASE
;
1991 pr_info("Found and enabled local APIC!\n");
1995 int __init
apic_force_enable(unsigned long addr
)
2003 * Some BIOSes disable the local APIC in the APIC_BASE
2004 * MSR. This can only be done in software for Intel P6 or later
2005 * and AMD K7 (Model > 1) or later.
2007 if (boot_cpu_data
.x86
>= 6) {
2008 rdmsr(MSR_IA32_APICBASE
, l
, h
);
2009 if (!(l
& MSR_IA32_APICBASE_ENABLE
)) {
2010 pr_info("Local APIC disabled by BIOS -- reenabling.\n");
2011 l
&= ~MSR_IA32_APICBASE_BASE
;
2012 l
|= MSR_IA32_APICBASE_ENABLE
| addr
;
2013 wrmsr(MSR_IA32_APICBASE
, l
, h
);
2014 enabled_via_apicbase
= 1;
2017 return apic_verify();
2021 * Detect and initialize APIC
2023 static int __init
detect_init_APIC(void)
2025 /* Disabled by kernel option? */
2029 switch (boot_cpu_data
.x86_vendor
) {
2030 case X86_VENDOR_AMD
:
2031 if ((boot_cpu_data
.x86
== 6 && boot_cpu_data
.x86_model
> 1) ||
2032 (boot_cpu_data
.x86
>= 15))
2035 case X86_VENDOR_HYGON
:
2037 case X86_VENDOR_INTEL
:
2038 if (boot_cpu_data
.x86
== 6 || boot_cpu_data
.x86
== 15 ||
2039 (boot_cpu_data
.x86
== 5 && boot_cpu_has(X86_FEATURE_APIC
)))
2046 if (!boot_cpu_has(X86_FEATURE_APIC
)) {
2048 * Over-ride BIOS and try to enable the local APIC only if
2049 * "lapic" specified.
2051 if (!force_enable_local_apic
) {
2052 pr_info("Local APIC disabled by BIOS -- "
2053 "you can enable it with \"lapic\"\n");
2056 if (apic_force_enable(APIC_DEFAULT_PHYS_BASE
))
2068 pr_info("No local APIC present or hardware disabled\n");
2074 * init_apic_mappings - initialize APIC mappings
2076 void __init
init_apic_mappings(void)
2078 unsigned int new_apicid
;
2080 if (apic_validate_deadline_timer())
2081 pr_info("TSC deadline timer available\n");
2084 boot_cpu_physical_apicid
= read_apic_id();
2088 /* If no local APIC can be found return early */
2089 if (!smp_found_config
&& detect_init_APIC()) {
2090 /* lets NOP'ify apic operations */
2091 pr_info("APIC: disable apic facility\n");
2094 apic_phys
= mp_lapic_addr
;
2097 * If the system has ACPI MADT tables or MP info, the LAPIC
2098 * address is already registered.
2100 if (!acpi_lapic
&& !smp_found_config
)
2101 register_lapic_address(apic_phys
);
2105 * Fetch the APIC ID of the BSP in case we have a
2106 * default configuration (or the MP table is broken).
2108 new_apicid
= read_apic_id();
2109 if (boot_cpu_physical_apicid
!= new_apicid
) {
2110 boot_cpu_physical_apicid
= new_apicid
;
2112 * yeah -- we lie about apic_version
2113 * in case if apic was disabled via boot option
2114 * but it's not a problem for SMP compiled kernel
2115 * since apic_intr_mode_select is prepared for such
2116 * a case and disable smp mode
2118 boot_cpu_apic_version
= GET_APIC_VERSION(apic_read(APIC_LVR
));
2122 void __init
register_lapic_address(unsigned long address
)
2124 mp_lapic_addr
= address
;
2127 set_fixmap_nocache(FIX_APIC_BASE
, address
);
2128 apic_printk(APIC_VERBOSE
, "mapped APIC to %16lx (%16lx)\n",
2129 APIC_BASE
, address
);
2131 if (boot_cpu_physical_apicid
== -1U) {
2132 boot_cpu_physical_apicid
= read_apic_id();
2133 boot_cpu_apic_version
= GET_APIC_VERSION(apic_read(APIC_LVR
));
2138 * Local APIC interrupts
2142 * Common handling code for spurious_interrupt and spurious_vector entry
2143 * points below. No point in allowing the compiler to inline it twice.
2145 static noinline
void handle_spurious_interrupt(u8 vector
)
2149 trace_spurious_apic_entry(vector
);
2151 inc_irq_stat(irq_spurious_count
);
2154 * If this is a spurious interrupt then do not acknowledge
2156 if (vector
== SPURIOUS_APIC_VECTOR
) {
2158 pr_info("Spurious APIC interrupt (vector 0xFF) on CPU#%d, should never happen.\n",
2159 smp_processor_id());
2164 * If it is a vectored one, verify it's set in the ISR. If set,
2167 v
= apic_read(APIC_ISR
+ ((vector
& ~0x1f) >> 1));
2168 if (v
& (1 << (vector
& 0x1f))) {
2169 pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Acked\n",
2170 vector
, smp_processor_id());
2173 pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Not pending!\n",
2174 vector
, smp_processor_id());
2177 trace_spurious_apic_exit(vector
);
2181 * spurious_interrupt - Catch all for interrupts raised on unused vectors
2182 * @regs: Pointer to pt_regs on stack
2183 * @vector: The vector number
2185 * This is invoked from ASM entry code to catch all interrupts which
2186 * trigger on an entry which is routed to the common_spurious idtentry
2189 DEFINE_IDTENTRY_IRQ(spurious_interrupt
)
2191 handle_spurious_interrupt(vector
);
2194 DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt
)
2196 handle_spurious_interrupt(SPURIOUS_APIC_VECTOR
);
2200 * This interrupt should never happen with our APIC/SMP architecture
2202 DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt
)
2204 static const char * const error_interrupt_reason
[] = {
2205 "Send CS error", /* APIC Error Bit 0 */
2206 "Receive CS error", /* APIC Error Bit 1 */
2207 "Send accept error", /* APIC Error Bit 2 */
2208 "Receive accept error", /* APIC Error Bit 3 */
2209 "Redirectable IPI", /* APIC Error Bit 4 */
2210 "Send illegal vector", /* APIC Error Bit 5 */
2211 "Received illegal vector", /* APIC Error Bit 6 */
2212 "Illegal register address", /* APIC Error Bit 7 */
2216 trace_error_apic_entry(ERROR_APIC_VECTOR
);
2218 /* First tickle the hardware, only then report what went on. -- REW */
2219 if (lapic_get_maxlvt() > 3) /* Due to the Pentium erratum 3AP. */
2220 apic_write(APIC_ESR
, 0);
2221 v
= apic_read(APIC_ESR
);
2223 atomic_inc(&irq_err_count
);
2225 apic_printk(APIC_DEBUG
, KERN_DEBUG
"APIC error on CPU%d: %02x",
2226 smp_processor_id(), v
);
2231 apic_printk(APIC_DEBUG
, KERN_CONT
" : %s", error_interrupt_reason
[i
]);
2236 apic_printk(APIC_DEBUG
, KERN_CONT
"\n");
2238 trace_error_apic_exit(ERROR_APIC_VECTOR
);
2242 * connect_bsp_APIC - attach the APIC to the interrupt system
2244 static void __init
connect_bsp_APIC(void)
2246 #ifdef CONFIG_X86_32
2249 * Do not trust the local APIC being empty at bootup.
2253 * PIC mode, enable APIC mode in the IMCR, i.e. connect BSP's
2254 * local APIC to INT and NMI lines.
2256 apic_printk(APIC_VERBOSE
, "leaving PIC mode, "
2257 "enabling APIC mode.\n");
2264 * disconnect_bsp_APIC - detach the APIC from the interrupt system
2265 * @virt_wire_setup: indicates, whether virtual wire mode is selected
2267 * Virtual wire mode is necessary to deliver legacy interrupts even when the
2270 void disconnect_bsp_APIC(int virt_wire_setup
)
2274 #ifdef CONFIG_X86_32
2277 * Put the board back into PIC mode (has an effect only on
2278 * certain older boards). Note that APIC interrupts, including
2279 * IPIs, won't work beyond this point! The only exception are
2282 apic_printk(APIC_VERBOSE
, "disabling APIC mode, "
2283 "entering PIC mode.\n");
2289 /* Go back to Virtual Wire compatibility mode */
2291 /* For the spurious interrupt use vector F, and enable it */
2292 value
= apic_read(APIC_SPIV
);
2293 value
&= ~APIC_VECTOR_MASK
;
2294 value
|= APIC_SPIV_APIC_ENABLED
;
2296 apic_write(APIC_SPIV
, value
);
2298 if (!virt_wire_setup
) {
2300 * For LVT0 make it edge triggered, active high,
2301 * external and enabled
2303 value
= apic_read(APIC_LVT0
);
2304 value
&= ~(APIC_MODE_MASK
| APIC_SEND_PENDING
|
2305 APIC_INPUT_POLARITY
| APIC_LVT_REMOTE_IRR
|
2306 APIC_LVT_LEVEL_TRIGGER
| APIC_LVT_MASKED
);
2307 value
|= APIC_LVT_REMOTE_IRR
| APIC_SEND_PENDING
;
2308 value
= SET_APIC_DELIVERY_MODE(value
, APIC_MODE_EXTINT
);
2309 apic_write(APIC_LVT0
, value
);
2312 apic_write(APIC_LVT0
, APIC_LVT_MASKED
);
2316 * For LVT1 make it edge triggered, active high,
2319 value
= apic_read(APIC_LVT1
);
2320 value
&= ~(APIC_MODE_MASK
| APIC_SEND_PENDING
|
2321 APIC_INPUT_POLARITY
| APIC_LVT_REMOTE_IRR
|
2322 APIC_LVT_LEVEL_TRIGGER
| APIC_LVT_MASKED
);
2323 value
|= APIC_LVT_REMOTE_IRR
| APIC_SEND_PENDING
;
2324 value
= SET_APIC_DELIVERY_MODE(value
, APIC_MODE_NMI
);
2325 apic_write(APIC_LVT1
, value
);
2329 * The number of allocated logical CPU IDs. Since logical CPU IDs are allocated
2330 * contiguously, it equals to current allocated max logical CPU ID plus 1.
2331 * All allocated CPU IDs should be in the [0, nr_logical_cpuids) range,
2332 * so the maximum of nr_logical_cpuids is nr_cpu_ids.
2334 * NOTE: Reserve 0 for BSP.
2336 static int nr_logical_cpuids
= 1;
2339 * Used to store mapping between logical CPU IDs and APIC IDs.
2341 static int cpuid_to_apicid
[] = {
2342 [0 ... NR_CPUS
- 1] = -1,
2345 bool arch_match_cpu_phys_id(int cpu
, u64 phys_id
)
2347 return phys_id
== cpuid_to_apicid
[cpu
];
2352 * apic_id_is_primary_thread - Check whether APIC ID belongs to a primary thread
2353 * @apicid: APIC ID to check
2355 bool apic_id_is_primary_thread(unsigned int apicid
)
2359 if (smp_num_siblings
== 1)
2361 /* Isolate the SMT bit(s) in the APICID and check for 0 */
2362 mask
= (1U << (fls(smp_num_siblings
) - 1)) - 1;
2363 return !(apicid
& mask
);
2368 * Should use this API to allocate logical CPU IDs to keep nr_logical_cpuids
2369 * and cpuid_to_apicid[] synchronized.
2371 static int allocate_logical_cpuid(int apicid
)
2376 * cpuid <-> apicid mapping is persistent, so when a cpu is up,
2377 * check if the kernel has allocated a cpuid for it.
2379 for (i
= 0; i
< nr_logical_cpuids
; i
++) {
2380 if (cpuid_to_apicid
[i
] == apicid
)
2384 /* Allocate a new cpuid. */
2385 if (nr_logical_cpuids
>= nr_cpu_ids
) {
2386 WARN_ONCE(1, "APIC: NR_CPUS/possible_cpus limit of %u reached. "
2387 "Processor %d/0x%x and the rest are ignored.\n",
2388 nr_cpu_ids
, nr_logical_cpuids
, apicid
);
2392 cpuid_to_apicid
[nr_logical_cpuids
] = apicid
;
2393 return nr_logical_cpuids
++;
2396 int generic_processor_info(int apicid
, int version
)
2398 int cpu
, max
= nr_cpu_ids
;
2399 bool boot_cpu_detected
= physid_isset(boot_cpu_physical_apicid
,
2400 phys_cpu_present_map
);
2403 * boot_cpu_physical_apicid is designed to have the apicid
2404 * returned by read_apic_id(), i.e, the apicid of the
2405 * currently booting-up processor. However, on some platforms,
2406 * it is temporarily modified by the apicid reported as BSP
2407 * through MP table. Concretely:
2409 * - arch/x86/kernel/mpparse.c: MP_processor_info()
2410 * - arch/x86/mm/amdtopology.c: amd_numa_init()
2412 * This function is executed with the modified
2413 * boot_cpu_physical_apicid. So, disabled_cpu_apicid kernel
2414 * parameter doesn't work to disable APs on kdump 2nd kernel.
2416 * Since fixing handling of boot_cpu_physical_apicid requires
2417 * another discussion and tests on each platform, we leave it
2418 * for now and here we use read_apic_id() directly in this
2419 * function, generic_processor_info().
2421 if (disabled_cpu_apicid
!= BAD_APICID
&&
2422 disabled_cpu_apicid
!= read_apic_id() &&
2423 disabled_cpu_apicid
== apicid
) {
2424 int thiscpu
= num_processors
+ disabled_cpus
;
2426 pr_warn("APIC: Disabling requested cpu."
2427 " Processor %d/0x%x ignored.\n", thiscpu
, apicid
);
2434 * If boot cpu has not been detected yet, then only allow upto
2435 * nr_cpu_ids - 1 processors and keep one slot free for boot cpu
2437 if (!boot_cpu_detected
&& num_processors
>= nr_cpu_ids
- 1 &&
2438 apicid
!= boot_cpu_physical_apicid
) {
2439 int thiscpu
= max
+ disabled_cpus
- 1;
2441 pr_warn("APIC: NR_CPUS/possible_cpus limit of %i almost"
2442 " reached. Keeping one slot for boot cpu."
2443 " Processor %d/0x%x ignored.\n", max
, thiscpu
, apicid
);
2449 if (num_processors
>= nr_cpu_ids
) {
2450 int thiscpu
= max
+ disabled_cpus
;
2452 pr_warn("APIC: NR_CPUS/possible_cpus limit of %i reached. "
2453 "Processor %d/0x%x ignored.\n", max
, thiscpu
, apicid
);
2459 if (apicid
== boot_cpu_physical_apicid
) {
2461 * x86_bios_cpu_apicid is required to have processors listed
2462 * in same order as logical cpu numbers. Hence the first
2463 * entry is BSP, and so on.
2464 * boot_cpu_init() already hold bit 0 in cpu_present_mask
2469 /* Logical cpuid 0 is reserved for BSP. */
2470 cpuid_to_apicid
[0] = apicid
;
2472 cpu
= allocate_logical_cpuid(apicid
);
2482 if (version
== 0x0) {
2483 pr_warn("BIOS bug: APIC version is 0 for CPU %d/0x%x, fixing up to 0x10\n",
2488 if (version
!= boot_cpu_apic_version
) {
2489 pr_warn("BIOS bug: APIC version mismatch, boot CPU: %x, CPU %d: version %x\n",
2490 boot_cpu_apic_version
, cpu
, version
);
2493 if (apicid
> max_physical_apicid
)
2494 max_physical_apicid
= apicid
;
2496 #if defined(CONFIG_SMP) || defined(CONFIG_X86_64)
2497 early_per_cpu(x86_cpu_to_apicid
, cpu
) = apicid
;
2498 early_per_cpu(x86_bios_cpu_apicid
, cpu
) = apicid
;
2500 #ifdef CONFIG_X86_32
2501 early_per_cpu(x86_cpu_to_logical_apicid
, cpu
) =
2502 apic
->x86_32_early_logical_apicid(cpu
);
2504 set_cpu_possible(cpu
, true);
2505 physid_set(apicid
, phys_cpu_present_map
);
2506 set_cpu_present(cpu
, true);
2512 int hard_smp_processor_id(void)
2514 return read_apic_id();
2517 void __irq_msi_compose_msg(struct irq_cfg
*cfg
, struct msi_msg
*msg
,
2520 memset(msg
, 0, sizeof(*msg
));
2522 msg
->arch_addr_lo
.base_address
= X86_MSI_BASE_ADDRESS_LOW
;
2523 msg
->arch_addr_lo
.dest_mode_logical
= apic
->dest_mode_logical
;
2524 msg
->arch_addr_lo
.destid_0_7
= cfg
->dest_apicid
& 0xFF;
2526 msg
->arch_data
.delivery_mode
= APIC_DELIVERY_MODE_FIXED
;
2527 msg
->arch_data
.vector
= cfg
->vector
;
2529 msg
->address_hi
= X86_MSI_BASE_ADDRESS_HIGH
;
2531 * Only the IOMMU itself can use the trick of putting destination
2532 * APIC ID into the high bits of the address. Anything else would
2533 * just be writing to memory if it tried that, and needs IR to
2534 * address APICs which can't be addressed in the normal 32-bit
2535 * address range at 0xFFExxxxx. That is typically just 8 bits, but
2536 * some hypervisors allow the extended destination ID field in bits
2537 * 5-11 to be used, giving support for 15 bits of APIC IDs in total.
2540 msg
->arch_addr_hi
.destid_8_31
= cfg
->dest_apicid
>> 8;
2541 else if (virt_ext_dest_id
&& cfg
->dest_apicid
< 0x8000)
2542 msg
->arch_addr_lo
.virt_destid_8_14
= cfg
->dest_apicid
>> 8;
2544 WARN_ON_ONCE(cfg
->dest_apicid
> 0xFF);
2547 u32
x86_msi_msg_get_destid(struct msi_msg
*msg
, bool extid
)
2549 u32 dest
= msg
->arch_addr_lo
.destid_0_7
;
2552 dest
|= msg
->arch_addr_hi
.destid_8_31
<< 8;
2555 EXPORT_SYMBOL_GPL(x86_msi_msg_get_destid
);
2558 * Override the generic EOI implementation with an optimized version.
2559 * Only called during early boot when only one CPU is active and with
2560 * interrupts disabled, so we know this does not race with actual APIC driver
2563 void __init
apic_set_eoi_write(void (*eoi_write
)(u32 reg
, u32 v
))
2567 for (drv
= __apicdrivers
; drv
< __apicdrivers_end
; drv
++) {
2568 /* Should happen once for each apic */
2569 WARN_ON((*drv
)->eoi_write
== eoi_write
);
2570 (*drv
)->native_eoi_write
= (*drv
)->eoi_write
;
2571 (*drv
)->eoi_write
= eoi_write
;
2575 static void __init
apic_bsp_up_setup(void)
2577 #ifdef CONFIG_X86_64
2578 apic_write(APIC_ID
, apic
->set_apic_id(boot_cpu_physical_apicid
));
2581 * Hack: In case of kdump, after a crash, kernel might be booting
2582 * on a cpu with non-zero lapic id. But boot_cpu_physical_apicid
2583 * might be zero if read from MP tables. Get it from LAPIC.
2585 # ifdef CONFIG_CRASH_DUMP
2586 boot_cpu_physical_apicid
= read_apic_id();
2589 physid_set_mask_of_physid(boot_cpu_physical_apicid
, &phys_cpu_present_map
);
2593 * apic_bsp_setup - Setup function for local apic and io-apic
2594 * @upmode: Force UP mode (for APIC_init_uniprocessor)
2596 static void __init
apic_bsp_setup(bool upmode
)
2600 apic_bsp_up_setup();
2604 end_local_APIC_setup();
2605 irq_remap_enable_fault_handling();
2609 #ifdef CONFIG_UP_LATE_INIT
2610 void __init
up_late_init(void)
2612 if (apic_intr_mode
== APIC_PIC
)
2615 /* Setup local timer */
2616 x86_init
.timers
.setup_percpu_clockev();
2627 * 'active' is true if the local APIC was enabled by us and
2628 * not the BIOS; this signifies that we are also responsible
2629 * for disabling it before entering apm/acpi suspend
2632 /* r/w apic fields */
2633 unsigned int apic_id
;
2634 unsigned int apic_taskpri
;
2635 unsigned int apic_ldr
;
2636 unsigned int apic_dfr
;
2637 unsigned int apic_spiv
;
2638 unsigned int apic_lvtt
;
2639 unsigned int apic_lvtpc
;
2640 unsigned int apic_lvt0
;
2641 unsigned int apic_lvt1
;
2642 unsigned int apic_lvterr
;
2643 unsigned int apic_tmict
;
2644 unsigned int apic_tdcr
;
2645 unsigned int apic_thmr
;
2646 unsigned int apic_cmci
;
2649 static int lapic_suspend(void)
2651 unsigned long flags
;
2654 if (!apic_pm_state
.active
)
2657 maxlvt
= lapic_get_maxlvt();
2659 apic_pm_state
.apic_id
= apic_read(APIC_ID
);
2660 apic_pm_state
.apic_taskpri
= apic_read(APIC_TASKPRI
);
2661 apic_pm_state
.apic_ldr
= apic_read(APIC_LDR
);
2662 apic_pm_state
.apic_dfr
= apic_read(APIC_DFR
);
2663 apic_pm_state
.apic_spiv
= apic_read(APIC_SPIV
);
2664 apic_pm_state
.apic_lvtt
= apic_read(APIC_LVTT
);
2666 apic_pm_state
.apic_lvtpc
= apic_read(APIC_LVTPC
);
2667 apic_pm_state
.apic_lvt0
= apic_read(APIC_LVT0
);
2668 apic_pm_state
.apic_lvt1
= apic_read(APIC_LVT1
);
2669 apic_pm_state
.apic_lvterr
= apic_read(APIC_LVTERR
);
2670 apic_pm_state
.apic_tmict
= apic_read(APIC_TMICT
);
2671 apic_pm_state
.apic_tdcr
= apic_read(APIC_TDCR
);
2672 #ifdef CONFIG_X86_THERMAL_VECTOR
2674 apic_pm_state
.apic_thmr
= apic_read(APIC_LVTTHMR
);
2676 #ifdef CONFIG_X86_MCE_INTEL
2678 apic_pm_state
.apic_cmci
= apic_read(APIC_LVTCMCI
);
2681 local_irq_save(flags
);
2684 * Mask IOAPIC before disabling the local APIC to prevent stale IRR
2685 * entries on some implementations.
2687 mask_ioapic_entries();
2689 disable_local_APIC();
2691 irq_remapping_disable();
2693 local_irq_restore(flags
);
2697 static void lapic_resume(void)
2700 unsigned long flags
;
2703 if (!apic_pm_state
.active
)
2706 local_irq_save(flags
);
2709 * IO-APIC and PIC have their own resume routines.
2710 * We just mask them here to make sure the interrupt
2711 * subsystem is completely quiet while we enable x2apic
2712 * and interrupt-remapping.
2714 mask_ioapic_entries();
2715 legacy_pic
->mask_all();
2721 * Make sure the APICBASE points to the right address
2723 * FIXME! This will be wrong if we ever support suspend on
2724 * SMP! We'll need to do this as part of the CPU restore!
2726 if (boot_cpu_data
.x86
>= 6) {
2727 rdmsr(MSR_IA32_APICBASE
, l
, h
);
2728 l
&= ~MSR_IA32_APICBASE_BASE
;
2729 l
|= MSR_IA32_APICBASE_ENABLE
| mp_lapic_addr
;
2730 wrmsr(MSR_IA32_APICBASE
, l
, h
);
2734 maxlvt
= lapic_get_maxlvt();
2735 apic_write(APIC_LVTERR
, ERROR_APIC_VECTOR
| APIC_LVT_MASKED
);
2736 apic_write(APIC_ID
, apic_pm_state
.apic_id
);
2737 apic_write(APIC_DFR
, apic_pm_state
.apic_dfr
);
2738 apic_write(APIC_LDR
, apic_pm_state
.apic_ldr
);
2739 apic_write(APIC_TASKPRI
, apic_pm_state
.apic_taskpri
);
2740 apic_write(APIC_SPIV
, apic_pm_state
.apic_spiv
);
2741 apic_write(APIC_LVT0
, apic_pm_state
.apic_lvt0
);
2742 apic_write(APIC_LVT1
, apic_pm_state
.apic_lvt1
);
2743 #ifdef CONFIG_X86_THERMAL_VECTOR
2745 apic_write(APIC_LVTTHMR
, apic_pm_state
.apic_thmr
);
2747 #ifdef CONFIG_X86_MCE_INTEL
2749 apic_write(APIC_LVTCMCI
, apic_pm_state
.apic_cmci
);
2752 apic_write(APIC_LVTPC
, apic_pm_state
.apic_lvtpc
);
2753 apic_write(APIC_LVTT
, apic_pm_state
.apic_lvtt
);
2754 apic_write(APIC_TDCR
, apic_pm_state
.apic_tdcr
);
2755 apic_write(APIC_TMICT
, apic_pm_state
.apic_tmict
);
2756 apic_write(APIC_ESR
, 0);
2757 apic_read(APIC_ESR
);
2758 apic_write(APIC_LVTERR
, apic_pm_state
.apic_lvterr
);
2759 apic_write(APIC_ESR
, 0);
2760 apic_read(APIC_ESR
);
2762 irq_remapping_reenable(x2apic_mode
);
2764 local_irq_restore(flags
);
2768 * This device has no shutdown method - fully functioning local APICs
2769 * are needed on every CPU up until machine_halt/restart/poweroff.
2772 static struct syscore_ops lapic_syscore_ops
= {
2773 .resume
= lapic_resume
,
2774 .suspend
= lapic_suspend
,
2777 static void apic_pm_activate(void)
2779 apic_pm_state
.active
= 1;
2782 static int __init
init_lapic_sysfs(void)
2784 /* XXX: remove suspend/resume procs if !apic_pm_state.active? */
2785 if (boot_cpu_has(X86_FEATURE_APIC
))
2786 register_syscore_ops(&lapic_syscore_ops
);
2791 /* local apic needs to resume before other devices access its registers. */
2792 core_initcall(init_lapic_sysfs
);
2794 #else /* CONFIG_PM */
2796 static void apic_pm_activate(void) { }
2798 #endif /* CONFIG_PM */
2800 #ifdef CONFIG_X86_64
2802 static int multi_checked
;
2805 static int set_multi(const struct dmi_system_id
*d
)
2809 pr_info("APIC: %s detected, Multi Chassis\n", d
->ident
);
2814 static const struct dmi_system_id multi_dmi_table
[] = {
2816 .callback
= set_multi
,
2817 .ident
= "IBM System Summit2",
2819 DMI_MATCH(DMI_SYS_VENDOR
, "IBM"),
2820 DMI_MATCH(DMI_PRODUCT_NAME
, "Summit2"),
2826 static void dmi_check_multi(void)
2831 dmi_check_system(multi_dmi_table
);
2836 * apic_is_clustered_box() -- Check if we can expect good TSC
2838 * Thus far, the major user of this is IBM's Summit2 series:
2839 * Clustered boxes may have unsynced TSC problems if they are
2841 * Use DMI to check them
2843 int apic_is_clustered_box(void)
2851 * APIC command line parameters
2853 static int __init
setup_disableapic(char *arg
)
2856 setup_clear_cpu_cap(X86_FEATURE_APIC
);
2859 early_param("disableapic", setup_disableapic
);
2861 /* same as disableapic, for compatibility */
2862 static int __init
setup_nolapic(char *arg
)
2864 return setup_disableapic(arg
);
2866 early_param("nolapic", setup_nolapic
);
2868 static int __init
parse_lapic_timer_c2_ok(char *arg
)
2870 local_apic_timer_c2_ok
= 1;
2873 early_param("lapic_timer_c2_ok", parse_lapic_timer_c2_ok
);
2875 static int __init
parse_disable_apic_timer(char *arg
)
2877 disable_apic_timer
= 1;
2880 early_param("noapictimer", parse_disable_apic_timer
);
2882 static int __init
parse_nolapic_timer(char *arg
)
2884 disable_apic_timer
= 1;
2887 early_param("nolapic_timer", parse_nolapic_timer
);
2889 static int __init
apic_set_verbosity(char *arg
)
2892 #ifdef CONFIG_X86_64
2893 skip_ioapic_setup
= 0;
2899 if (strcmp("debug", arg
) == 0)
2900 apic_verbosity
= APIC_DEBUG
;
2901 else if (strcmp("verbose", arg
) == 0)
2902 apic_verbosity
= APIC_VERBOSE
;
2903 #ifdef CONFIG_X86_64
2905 pr_warn("APIC Verbosity level %s not recognised"
2906 " use apic=verbose or apic=debug\n", arg
);
2913 early_param("apic", apic_set_verbosity
);
2915 static int __init
lapic_insert_resource(void)
2920 /* Put local APIC into the resource map. */
2921 lapic_resource
.start
= apic_phys
;
2922 lapic_resource
.end
= lapic_resource
.start
+ PAGE_SIZE
- 1;
2923 insert_resource(&iomem_resource
, &lapic_resource
);
2929 * need call insert after e820__reserve_resources()
2930 * that is using request_resource
2932 late_initcall(lapic_insert_resource
);
2934 static int __init
apic_set_disabled_cpu_apicid(char *arg
)
2936 if (!arg
|| !get_option(&arg
, &disabled_cpu_apicid
))
2941 early_param("disable_cpu_apicid", apic_set_disabled_cpu_apicid
);
2943 static int __init
apic_set_extnmi(char *arg
)
2948 if (!strncmp("all", arg
, 3))
2949 apic_extnmi
= APIC_EXTNMI_ALL
;
2950 else if (!strncmp("none", arg
, 4))
2951 apic_extnmi
= APIC_EXTNMI_NONE
;
2952 else if (!strncmp("bsp", arg
, 3))
2953 apic_extnmi
= APIC_EXTNMI_BSP
;
2955 pr_warn("Unknown external NMI delivery mode `%s' ignored\n", arg
);
2961 early_param("apic_extnmi", apic_set_extnmi
);