2 * intel_idle.c - native hardware idle loop for modern Intel processors
4 * Copyright (c) 2013, Intel Corporation.
5 * Len Brown <len.brown@intel.com>
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2, as published by the Free Software Foundation.
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * You should have received a copy of the GNU General Public License along with
17 * this program; if not, write to the Free Software Foundation, Inc.,
18 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
22 * intel_idle is a cpuidle driver that loads on specific Intel processors
23 * in lieu of the legacy ACPI processor_idle driver. The intent is to
24 * make Linux more efficient on these processors, as intel_idle knows
25 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs.
31 * All CPUs have same idle states as boot CPU
33 * Chipset BM_STS (bus master status) bit is a NOP
34 * for preventing entry into deep C-stats
40 * The driver currently initializes for_each_online_cpu() upon modprobe.
41 * It it unaware of subsequent processors hot-added to the system.
42 * This means that if you boot with maxcpus=n and later online
43 * processors above n, those processors will use C1 only.
45 * ACPI has a .suspend hack to turn off deep c-statees during suspend
46 * to avoid complications with the lapic timer workaround.
47 * Have not seen issues with suspend, but may need same workaround here.
49 * There is currently no kernel-based automatic probing/loading mechanism
50 * if the driver is built as a module.
53 /* un-comment DEBUG to enable pr_debug() statements */
56 #include <linux/kernel.h>
57 #include <linux/cpuidle.h>
58 #include <linux/tick.h>
59 #include <trace/events/power.h>
60 #include <linux/sched.h>
61 #include <linux/notifier.h>
62 #include <linux/cpu.h>
63 #include <linux/module.h>
64 #include <asm/cpu_device_id.h>
65 #include <asm/mwait.h>
68 #define INTEL_IDLE_VERSION "0.4"
69 #define PREFIX "intel_idle: "
71 static struct cpuidle_driver intel_idle_driver
= {
75 /* intel_idle.max_cstate=0 disables driver */
76 static int max_cstate
= CPUIDLE_STATE_MAX
- 1;
78 static unsigned int mwait_substates
;
80 #define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF
81 /* Reliable LAPIC Timer States, bit 1 for C1 etc. */
82 static unsigned int lapic_timer_reliable_states
= (1 << 1); /* Default to only C1 */
85 struct cpuidle_state
*state_table
;
88 * Hardware C-state auto-demotion may not always be optimal.
89 * Indicate which enable bits to clear here.
91 unsigned long auto_demotion_disable_flags
;
92 bool byt_auto_demotion_disable_flag
;
93 bool disable_promotion_to_c1e
;
96 static const struct idle_cpu
*icpu
;
97 static struct cpuidle_device __percpu
*intel_idle_cpuidle_devices
;
98 static int intel_idle(struct cpuidle_device
*dev
,
99 struct cpuidle_driver
*drv
, int index
);
100 static void intel_idle_freeze(struct cpuidle_device
*dev
,
101 struct cpuidle_driver
*drv
, int index
);
102 static int intel_idle_cpu_init(int cpu
);
104 static struct cpuidle_state
*cpuidle_state_table
;
107 * Set this flag for states where the HW flushes the TLB for us
108 * and so we don't need cross-calls to keep it consistent.
109 * If this flag is set, SW flushes the TLB, so even if the
110 * HW doesn't do the flushing, this flag is safe to use.
112 #define CPUIDLE_FLAG_TLB_FLUSHED 0x10000
115 * MWAIT takes an 8-bit "hint" in EAX "suggesting"
116 * the C-state (top nibble) and sub-state (bottom nibble)
117 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc.
119 * We store the hint at the top of our "flags" for each state.
121 #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF)
122 #define MWAIT2flg(eax) ((eax & 0xFF) << 24)
125 * States are indexed by the cstate number,
126 * which is also the index into the MWAIT hint array.
127 * Thus C0 is a dummy.
129 static struct cpuidle_state nehalem_cstates
[] = {
132 .desc
= "MWAIT 0x00",
133 .flags
= MWAIT2flg(0x00),
135 .target_residency
= 6,
136 .enter
= &intel_idle
,
137 .enter_freeze
= intel_idle_freeze
, },
140 .desc
= "MWAIT 0x01",
141 .flags
= MWAIT2flg(0x01),
143 .target_residency
= 20,
144 .enter
= &intel_idle
,
145 .enter_freeze
= intel_idle_freeze
, },
148 .desc
= "MWAIT 0x10",
149 .flags
= MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED
,
151 .target_residency
= 80,
152 .enter
= &intel_idle
,
153 .enter_freeze
= intel_idle_freeze
, },
156 .desc
= "MWAIT 0x20",
157 .flags
= MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED
,
159 .target_residency
= 800,
160 .enter
= &intel_idle
,
161 .enter_freeze
= intel_idle_freeze
, },
166 static struct cpuidle_state snb_cstates
[] = {
169 .desc
= "MWAIT 0x00",
170 .flags
= MWAIT2flg(0x00),
172 .target_residency
= 2,
173 .enter
= &intel_idle
,
174 .enter_freeze
= intel_idle_freeze
, },
177 .desc
= "MWAIT 0x01",
178 .flags
= MWAIT2flg(0x01),
180 .target_residency
= 20,
181 .enter
= &intel_idle
,
182 .enter_freeze
= intel_idle_freeze
, },
185 .desc
= "MWAIT 0x10",
186 .flags
= MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED
,
188 .target_residency
= 211,
189 .enter
= &intel_idle
,
190 .enter_freeze
= intel_idle_freeze
, },
193 .desc
= "MWAIT 0x20",
194 .flags
= MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED
,
196 .target_residency
= 345,
197 .enter
= &intel_idle
,
198 .enter_freeze
= intel_idle_freeze
, },
201 .desc
= "MWAIT 0x30",
202 .flags
= MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED
,
204 .target_residency
= 345,
205 .enter
= &intel_idle
,
206 .enter_freeze
= intel_idle_freeze
, },
211 static struct cpuidle_state byt_cstates
[] = {
214 .desc
= "MWAIT 0x00",
215 .flags
= MWAIT2flg(0x00),
217 .target_residency
= 1,
218 .enter
= &intel_idle
,
219 .enter_freeze
= intel_idle_freeze
, },
222 .desc
= "MWAIT 0x58",
223 .flags
= MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED
,
225 .target_residency
= 275,
226 .enter
= &intel_idle
,
227 .enter_freeze
= intel_idle_freeze
, },
230 .desc
= "MWAIT 0x52",
231 .flags
= MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED
,
233 .target_residency
= 560,
234 .enter
= &intel_idle
,
235 .enter_freeze
= intel_idle_freeze
, },
238 .desc
= "MWAIT 0x60",
239 .flags
= MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED
,
240 .exit_latency
= 1200,
241 .target_residency
= 4000,
242 .enter
= &intel_idle
,
243 .enter_freeze
= intel_idle_freeze
, },
246 .desc
= "MWAIT 0x64",
247 .flags
= MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED
,
248 .exit_latency
= 10000,
249 .target_residency
= 20000,
250 .enter
= &intel_idle
,
251 .enter_freeze
= intel_idle_freeze
, },
256 static struct cpuidle_state cht_cstates
[] = {
259 .desc
= "MWAIT 0x00",
260 .flags
= MWAIT2flg(0x00),
262 .target_residency
= 1,
263 .enter
= &intel_idle
,
264 .enter_freeze
= intel_idle_freeze
, },
267 .desc
= "MWAIT 0x58",
268 .flags
= MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED
,
270 .target_residency
= 275,
271 .enter
= &intel_idle
,
272 .enter_freeze
= intel_idle_freeze
, },
275 .desc
= "MWAIT 0x52",
276 .flags
= MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED
,
278 .target_residency
= 560,
279 .enter
= &intel_idle
,
280 .enter_freeze
= intel_idle_freeze
, },
283 .desc
= "MWAIT 0x60",
284 .flags
= MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED
,
285 .exit_latency
= 1200,
286 .target_residency
= 4000,
287 .enter
= &intel_idle
,
288 .enter_freeze
= intel_idle_freeze
, },
291 .desc
= "MWAIT 0x64",
292 .flags
= MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED
,
293 .exit_latency
= 10000,
294 .target_residency
= 20000,
295 .enter
= &intel_idle
,
296 .enter_freeze
= intel_idle_freeze
, },
301 static struct cpuidle_state ivb_cstates
[] = {
304 .desc
= "MWAIT 0x00",
305 .flags
= MWAIT2flg(0x00),
307 .target_residency
= 1,
308 .enter
= &intel_idle
,
309 .enter_freeze
= intel_idle_freeze
, },
312 .desc
= "MWAIT 0x01",
313 .flags
= MWAIT2flg(0x01),
315 .target_residency
= 20,
316 .enter
= &intel_idle
,
317 .enter_freeze
= intel_idle_freeze
, },
320 .desc
= "MWAIT 0x10",
321 .flags
= MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED
,
323 .target_residency
= 156,
324 .enter
= &intel_idle
,
325 .enter_freeze
= intel_idle_freeze
, },
328 .desc
= "MWAIT 0x20",
329 .flags
= MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED
,
331 .target_residency
= 300,
332 .enter
= &intel_idle
,
333 .enter_freeze
= intel_idle_freeze
, },
336 .desc
= "MWAIT 0x30",
337 .flags
= MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED
,
339 .target_residency
= 300,
340 .enter
= &intel_idle
,
341 .enter_freeze
= intel_idle_freeze
, },
346 static struct cpuidle_state ivt_cstates
[] = {
349 .desc
= "MWAIT 0x00",
350 .flags
= MWAIT2flg(0x00),
352 .target_residency
= 1,
353 .enter
= &intel_idle
,
354 .enter_freeze
= intel_idle_freeze
, },
357 .desc
= "MWAIT 0x01",
358 .flags
= MWAIT2flg(0x01),
360 .target_residency
= 80,
361 .enter
= &intel_idle
,
362 .enter_freeze
= intel_idle_freeze
, },
365 .desc
= "MWAIT 0x10",
366 .flags
= MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED
,
368 .target_residency
= 156,
369 .enter
= &intel_idle
,
370 .enter_freeze
= intel_idle_freeze
, },
373 .desc
= "MWAIT 0x20",
374 .flags
= MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED
,
376 .target_residency
= 300,
377 .enter
= &intel_idle
,
378 .enter_freeze
= intel_idle_freeze
, },
383 static struct cpuidle_state ivt_cstates_4s
[] = {
386 .desc
= "MWAIT 0x00",
387 .flags
= MWAIT2flg(0x00),
389 .target_residency
= 1,
390 .enter
= &intel_idle
,
391 .enter_freeze
= intel_idle_freeze
, },
393 .name
= "C1E-IVT-4S",
394 .desc
= "MWAIT 0x01",
395 .flags
= MWAIT2flg(0x01),
397 .target_residency
= 250,
398 .enter
= &intel_idle
,
399 .enter_freeze
= intel_idle_freeze
, },
402 .desc
= "MWAIT 0x10",
403 .flags
= MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED
,
405 .target_residency
= 300,
406 .enter
= &intel_idle
,
407 .enter_freeze
= intel_idle_freeze
, },
410 .desc
= "MWAIT 0x20",
411 .flags
= MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED
,
413 .target_residency
= 400,
414 .enter
= &intel_idle
,
415 .enter_freeze
= intel_idle_freeze
, },
420 static struct cpuidle_state ivt_cstates_8s
[] = {
423 .desc
= "MWAIT 0x00",
424 .flags
= MWAIT2flg(0x00),
426 .target_residency
= 1,
427 .enter
= &intel_idle
,
428 .enter_freeze
= intel_idle_freeze
, },
430 .name
= "C1E-IVT-8S",
431 .desc
= "MWAIT 0x01",
432 .flags
= MWAIT2flg(0x01),
434 .target_residency
= 500,
435 .enter
= &intel_idle
,
436 .enter_freeze
= intel_idle_freeze
, },
439 .desc
= "MWAIT 0x10",
440 .flags
= MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED
,
442 .target_residency
= 600,
443 .enter
= &intel_idle
,
444 .enter_freeze
= intel_idle_freeze
, },
447 .desc
= "MWAIT 0x20",
448 .flags
= MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED
,
450 .target_residency
= 700,
451 .enter
= &intel_idle
,
452 .enter_freeze
= intel_idle_freeze
, },
457 static struct cpuidle_state hsw_cstates
[] = {
460 .desc
= "MWAIT 0x00",
461 .flags
= MWAIT2flg(0x00),
463 .target_residency
= 2,
464 .enter
= &intel_idle
,
465 .enter_freeze
= intel_idle_freeze
, },
468 .desc
= "MWAIT 0x01",
469 .flags
= MWAIT2flg(0x01),
471 .target_residency
= 20,
472 .enter
= &intel_idle
,
473 .enter_freeze
= intel_idle_freeze
, },
476 .desc
= "MWAIT 0x10",
477 .flags
= MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED
,
479 .target_residency
= 100,
480 .enter
= &intel_idle
,
481 .enter_freeze
= intel_idle_freeze
, },
484 .desc
= "MWAIT 0x20",
485 .flags
= MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED
,
487 .target_residency
= 400,
488 .enter
= &intel_idle
,
489 .enter_freeze
= intel_idle_freeze
, },
492 .desc
= "MWAIT 0x32",
493 .flags
= MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED
,
495 .target_residency
= 500,
496 .enter
= &intel_idle
,
497 .enter_freeze
= intel_idle_freeze
, },
500 .desc
= "MWAIT 0x40",
501 .flags
= MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED
,
503 .target_residency
= 900,
504 .enter
= &intel_idle
,
505 .enter_freeze
= intel_idle_freeze
, },
508 .desc
= "MWAIT 0x50",
509 .flags
= MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED
,
511 .target_residency
= 1800,
512 .enter
= &intel_idle
,
513 .enter_freeze
= intel_idle_freeze
, },
516 .desc
= "MWAIT 0x60",
517 .flags
= MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED
,
518 .exit_latency
= 2600,
519 .target_residency
= 7700,
520 .enter
= &intel_idle
,
521 .enter_freeze
= intel_idle_freeze
, },
525 static struct cpuidle_state bdw_cstates
[] = {
528 .desc
= "MWAIT 0x00",
529 .flags
= MWAIT2flg(0x00),
531 .target_residency
= 2,
532 .enter
= &intel_idle
,
533 .enter_freeze
= intel_idle_freeze
, },
536 .desc
= "MWAIT 0x01",
537 .flags
= MWAIT2flg(0x01),
539 .target_residency
= 20,
540 .enter
= &intel_idle
,
541 .enter_freeze
= intel_idle_freeze
, },
544 .desc
= "MWAIT 0x10",
545 .flags
= MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED
,
547 .target_residency
= 100,
548 .enter
= &intel_idle
,
549 .enter_freeze
= intel_idle_freeze
, },
552 .desc
= "MWAIT 0x20",
553 .flags
= MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED
,
555 .target_residency
= 400,
556 .enter
= &intel_idle
,
557 .enter_freeze
= intel_idle_freeze
, },
560 .desc
= "MWAIT 0x32",
561 .flags
= MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED
,
563 .target_residency
= 500,
564 .enter
= &intel_idle
,
565 .enter_freeze
= intel_idle_freeze
, },
568 .desc
= "MWAIT 0x40",
569 .flags
= MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED
,
571 .target_residency
= 900,
572 .enter
= &intel_idle
,
573 .enter_freeze
= intel_idle_freeze
, },
576 .desc
= "MWAIT 0x50",
577 .flags
= MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED
,
579 .target_residency
= 1800,
580 .enter
= &intel_idle
,
581 .enter_freeze
= intel_idle_freeze
, },
584 .desc
= "MWAIT 0x60",
585 .flags
= MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED
,
586 .exit_latency
= 2600,
587 .target_residency
= 7700,
588 .enter
= &intel_idle
,
589 .enter_freeze
= intel_idle_freeze
, },
594 static struct cpuidle_state skl_cstates
[] = {
597 .desc
= "MWAIT 0x00",
598 .flags
= MWAIT2flg(0x00),
600 .target_residency
= 2,
601 .enter
= &intel_idle
,
602 .enter_freeze
= intel_idle_freeze
, },
605 .desc
= "MWAIT 0x01",
606 .flags
= MWAIT2flg(0x01),
608 .target_residency
= 20,
609 .enter
= &intel_idle
,
610 .enter_freeze
= intel_idle_freeze
, },
613 .desc
= "MWAIT 0x10",
614 .flags
= MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED
,
616 .target_residency
= 100,
617 .enter
= &intel_idle
,
618 .enter_freeze
= intel_idle_freeze
, },
621 .desc
= "MWAIT 0x20",
622 .flags
= MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED
,
624 .target_residency
= 200,
625 .enter
= &intel_idle
,
626 .enter_freeze
= intel_idle_freeze
, },
629 .desc
= "MWAIT 0x33",
630 .flags
= MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED
,
632 .target_residency
= 800,
633 .enter
= &intel_idle
,
634 .enter_freeze
= intel_idle_freeze
, },
637 .desc
= "MWAIT 0x40",
638 .flags
= MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED
,
640 .target_residency
= 800,
641 .enter
= &intel_idle
,
642 .enter_freeze
= intel_idle_freeze
, },
645 .desc
= "MWAIT 0x60",
646 .flags
= MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED
,
648 .target_residency
= 5000,
649 .enter
= &intel_idle
,
650 .enter_freeze
= intel_idle_freeze
, },
655 static struct cpuidle_state atom_cstates
[] = {
658 .desc
= "MWAIT 0x00",
659 .flags
= MWAIT2flg(0x00),
661 .target_residency
= 20,
662 .enter
= &intel_idle
,
663 .enter_freeze
= intel_idle_freeze
, },
666 .desc
= "MWAIT 0x10",
667 .flags
= MWAIT2flg(0x10),
669 .target_residency
= 80,
670 .enter
= &intel_idle
,
671 .enter_freeze
= intel_idle_freeze
, },
674 .desc
= "MWAIT 0x30",
675 .flags
= MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED
,
677 .target_residency
= 400,
678 .enter
= &intel_idle
,
679 .enter_freeze
= intel_idle_freeze
, },
682 .desc
= "MWAIT 0x52",
683 .flags
= MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED
,
685 .target_residency
= 560,
686 .enter
= &intel_idle
,
687 .enter_freeze
= intel_idle_freeze
, },
691 static struct cpuidle_state avn_cstates
[] = {
694 .desc
= "MWAIT 0x00",
695 .flags
= MWAIT2flg(0x00),
697 .target_residency
= 2,
698 .enter
= &intel_idle
,
699 .enter_freeze
= intel_idle_freeze
, },
702 .desc
= "MWAIT 0x51",
703 .flags
= MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED
,
705 .target_residency
= 45,
706 .enter
= &intel_idle
,
707 .enter_freeze
= intel_idle_freeze
, },
714 * @dev: cpuidle_device
715 * @drv: cpuidle driver
716 * @index: index of cpuidle state
718 * Must be called under local_irq_disable().
720 static int intel_idle(struct cpuidle_device
*dev
,
721 struct cpuidle_driver
*drv
, int index
)
723 unsigned long ecx
= 1; /* break on interrupt flag */
724 struct cpuidle_state
*state
= &drv
->states
[index
];
725 unsigned long eax
= flg2MWAIT(state
->flags
);
727 int cpu
= smp_processor_id();
729 cstate
= (((eax
) >> MWAIT_SUBSTATE_SIZE
) & MWAIT_CSTATE_MASK
) + 1;
732 * leave_mm() to avoid costly and often unnecessary wakeups
733 * for flushing the user TLB's associated with the active mm.
735 if (state
->flags
& CPUIDLE_FLAG_TLB_FLUSHED
)
738 if (!(lapic_timer_reliable_states
& (1 << (cstate
))))
739 tick_broadcast_enter();
741 mwait_idle_with_hints(eax
, ecx
);
743 if (!(lapic_timer_reliable_states
& (1 << (cstate
))))
744 tick_broadcast_exit();
750 * intel_idle_freeze - simplified "enter" callback routine for suspend-to-idle
751 * @dev: cpuidle_device
752 * @drv: cpuidle driver
753 * @index: state index
755 static void intel_idle_freeze(struct cpuidle_device
*dev
,
756 struct cpuidle_driver
*drv
, int index
)
758 unsigned long ecx
= 1; /* break on interrupt flag */
759 unsigned long eax
= flg2MWAIT(drv
->states
[index
].flags
);
761 mwait_idle_with_hints(eax
, ecx
);
764 static void __setup_broadcast_timer(void *arg
)
766 unsigned long on
= (unsigned long)arg
;
769 tick_broadcast_enable();
771 tick_broadcast_disable();
774 static int cpu_hotplug_notify(struct notifier_block
*n
,
775 unsigned long action
, void *hcpu
)
777 int hotcpu
= (unsigned long)hcpu
;
778 struct cpuidle_device
*dev
;
780 switch (action
& ~CPU_TASKS_FROZEN
) {
783 if (lapic_timer_reliable_states
!= LAPIC_TIMER_ALWAYS_RELIABLE
)
784 smp_call_function_single(hotcpu
, __setup_broadcast_timer
,
788 * Some systems can hotplug a cpu at runtime after
789 * the kernel has booted, we have to initialize the
790 * driver in this case
792 dev
= per_cpu_ptr(intel_idle_cpuidle_devices
, hotcpu
);
793 if (!dev
->registered
)
794 intel_idle_cpu_init(hotcpu
);
801 static struct notifier_block cpu_hotplug_notifier
= {
802 .notifier_call
= cpu_hotplug_notify
,
805 static void auto_demotion_disable(void *dummy
)
807 unsigned long long msr_bits
;
809 rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL
, msr_bits
);
810 msr_bits
&= ~(icpu
->auto_demotion_disable_flags
);
811 wrmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL
, msr_bits
);
813 static void c1e_promotion_disable(void *dummy
)
815 unsigned long long msr_bits
;
817 rdmsrl(MSR_IA32_POWER_CTL
, msr_bits
);
819 wrmsrl(MSR_IA32_POWER_CTL
, msr_bits
);
822 static const struct idle_cpu idle_cpu_nehalem
= {
823 .state_table
= nehalem_cstates
,
824 .auto_demotion_disable_flags
= NHM_C1_AUTO_DEMOTE
| NHM_C3_AUTO_DEMOTE
,
825 .disable_promotion_to_c1e
= true,
828 static const struct idle_cpu idle_cpu_atom
= {
829 .state_table
= atom_cstates
,
832 static const struct idle_cpu idle_cpu_lincroft
= {
833 .state_table
= atom_cstates
,
834 .auto_demotion_disable_flags
= ATM_LNC_C6_AUTO_DEMOTE
,
837 static const struct idle_cpu idle_cpu_snb
= {
838 .state_table
= snb_cstates
,
839 .disable_promotion_to_c1e
= true,
842 static const struct idle_cpu idle_cpu_byt
= {
843 .state_table
= byt_cstates
,
844 .disable_promotion_to_c1e
= true,
845 .byt_auto_demotion_disable_flag
= true,
848 static const struct idle_cpu idle_cpu_cht
= {
849 .state_table
= cht_cstates
,
850 .disable_promotion_to_c1e
= true,
851 .byt_auto_demotion_disable_flag
= true,
854 static const struct idle_cpu idle_cpu_ivb
= {
855 .state_table
= ivb_cstates
,
856 .disable_promotion_to_c1e
= true,
859 static const struct idle_cpu idle_cpu_ivt
= {
860 .state_table
= ivt_cstates
,
861 .disable_promotion_to_c1e
= true,
864 static const struct idle_cpu idle_cpu_hsw
= {
865 .state_table
= hsw_cstates
,
866 .disable_promotion_to_c1e
= true,
869 static const struct idle_cpu idle_cpu_bdw
= {
870 .state_table
= bdw_cstates
,
871 .disable_promotion_to_c1e
= true,
874 static const struct idle_cpu idle_cpu_skl
= {
875 .state_table
= skl_cstates
,
876 .disable_promotion_to_c1e
= true,
880 static const struct idle_cpu idle_cpu_avn
= {
881 .state_table
= avn_cstates
,
882 .disable_promotion_to_c1e
= true,
885 #define ICPU(model, cpu) \
886 { X86_VENDOR_INTEL, 6, model, X86_FEATURE_MWAIT, (unsigned long)&cpu }
888 static const struct x86_cpu_id intel_idle_ids
[] __initconst
= {
889 ICPU(0x1a, idle_cpu_nehalem
),
890 ICPU(0x1e, idle_cpu_nehalem
),
891 ICPU(0x1f, idle_cpu_nehalem
),
892 ICPU(0x25, idle_cpu_nehalem
),
893 ICPU(0x2c, idle_cpu_nehalem
),
894 ICPU(0x2e, idle_cpu_nehalem
),
895 ICPU(0x1c, idle_cpu_atom
),
896 ICPU(0x26, idle_cpu_lincroft
),
897 ICPU(0x2f, idle_cpu_nehalem
),
898 ICPU(0x2a, idle_cpu_snb
),
899 ICPU(0x2d, idle_cpu_snb
),
900 ICPU(0x36, idle_cpu_atom
),
901 ICPU(0x37, idle_cpu_byt
),
902 ICPU(0x4c, idle_cpu_cht
),
903 ICPU(0x3a, idle_cpu_ivb
),
904 ICPU(0x3e, idle_cpu_ivt
),
905 ICPU(0x3c, idle_cpu_hsw
),
906 ICPU(0x3f, idle_cpu_hsw
),
907 ICPU(0x45, idle_cpu_hsw
),
908 ICPU(0x46, idle_cpu_hsw
),
909 ICPU(0x4d, idle_cpu_avn
),
910 ICPU(0x3d, idle_cpu_bdw
),
911 ICPU(0x47, idle_cpu_bdw
),
912 ICPU(0x4f, idle_cpu_bdw
),
913 ICPU(0x56, idle_cpu_bdw
),
914 ICPU(0x4e, idle_cpu_skl
),
915 ICPU(0x5e, idle_cpu_skl
),
918 MODULE_DEVICE_TABLE(x86cpu
, intel_idle_ids
);
923 static int __init
intel_idle_probe(void)
925 unsigned int eax
, ebx
, ecx
;
926 const struct x86_cpu_id
*id
;
928 if (max_cstate
== 0) {
929 pr_debug(PREFIX
"disabled\n");
933 id
= x86_match_cpu(intel_idle_ids
);
935 if (boot_cpu_data
.x86_vendor
== X86_VENDOR_INTEL
&&
936 boot_cpu_data
.x86
== 6)
937 pr_debug(PREFIX
"does not run on family %d model %d\n",
938 boot_cpu_data
.x86
, boot_cpu_data
.x86_model
);
942 if (boot_cpu_data
.cpuid_level
< CPUID_MWAIT_LEAF
)
945 cpuid(CPUID_MWAIT_LEAF
, &eax
, &ebx
, &ecx
, &mwait_substates
);
947 if (!(ecx
& CPUID5_ECX_EXTENSIONS_SUPPORTED
) ||
948 !(ecx
& CPUID5_ECX_INTERRUPT_BREAK
) ||
952 pr_debug(PREFIX
"MWAIT substates: 0x%x\n", mwait_substates
);
954 icpu
= (const struct idle_cpu
*)id
->driver_data
;
955 cpuidle_state_table
= icpu
->state_table
;
957 if (boot_cpu_has(X86_FEATURE_ARAT
)) /* Always Reliable APIC Timer */
958 lapic_timer_reliable_states
= LAPIC_TIMER_ALWAYS_RELIABLE
;
960 on_each_cpu(__setup_broadcast_timer
, (void *)true, 1);
962 pr_debug(PREFIX
"v" INTEL_IDLE_VERSION
963 " model 0x%X\n", boot_cpu_data
.x86_model
);
965 pr_debug(PREFIX
"lapic_timer_reliable_states 0x%x\n",
966 lapic_timer_reliable_states
);
971 * intel_idle_cpuidle_devices_uninit()
972 * unregister, free cpuidle_devices
974 static void intel_idle_cpuidle_devices_uninit(void)
977 struct cpuidle_device
*dev
;
979 for_each_online_cpu(i
) {
980 dev
= per_cpu_ptr(intel_idle_cpuidle_devices
, i
);
981 cpuidle_unregister_device(dev
);
984 free_percpu(intel_idle_cpuidle_devices
);
989 * intel_idle_state_table_update()
991 * Update the default state_table for this CPU-id
993 * Currently used to access tuned IVT multi-socket targets
994 * Assumption: num_sockets == (max_package_num + 1)
996 void intel_idle_state_table_update(void)
998 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */
999 if (boot_cpu_data
.x86_model
== 0x3e) { /* IVT */
1000 int cpu
, package_num
, num_sockets
= 1;
1002 for_each_online_cpu(cpu
) {
1003 package_num
= topology_physical_package_id(cpu
);
1004 if (package_num
+ 1 > num_sockets
) {
1005 num_sockets
= package_num
+ 1;
1007 if (num_sockets
> 4) {
1008 cpuidle_state_table
= ivt_cstates_8s
;
1014 if (num_sockets
> 2)
1015 cpuidle_state_table
= ivt_cstates_4s
;
1016 /* else, 1 and 2 socket systems use default ivt_cstates */
1022 * intel_idle_cpuidle_driver_init()
1023 * allocate, initialize cpuidle_states
1025 static int __init
intel_idle_cpuidle_driver_init(void)
1028 struct cpuidle_driver
*drv
= &intel_idle_driver
;
1030 intel_idle_state_table_update();
1032 drv
->state_count
= 1;
1034 for (cstate
= 0; cstate
< CPUIDLE_STATE_MAX
; ++cstate
) {
1035 int num_substates
, mwait_hint
, mwait_cstate
;
1037 if ((cpuidle_state_table
[cstate
].enter
== NULL
) &&
1038 (cpuidle_state_table
[cstate
].enter_freeze
== NULL
))
1041 if (cstate
+ 1 > max_cstate
) {
1042 printk(PREFIX
"max_cstate %d reached\n",
1047 mwait_hint
= flg2MWAIT(cpuidle_state_table
[cstate
].flags
);
1048 mwait_cstate
= MWAIT_HINT2CSTATE(mwait_hint
);
1050 /* number of sub-states for this state in CPUID.MWAIT */
1051 num_substates
= (mwait_substates
>> ((mwait_cstate
+ 1) * 4))
1052 & MWAIT_SUBSTATE_MASK
;
1054 /* if NO sub-states for this state in CPUID, skip it */
1055 if (num_substates
== 0)
1058 if (((mwait_cstate
+ 1) > 2) &&
1059 !boot_cpu_has(X86_FEATURE_NONSTOP_TSC
))
1060 mark_tsc_unstable("TSC halts in idle"
1061 " states deeper than C2");
1063 drv
->states
[drv
->state_count
] = /* structure copy */
1064 cpuidle_state_table
[cstate
];
1066 drv
->state_count
+= 1;
1069 if (icpu
->auto_demotion_disable_flags
)
1070 on_each_cpu(auto_demotion_disable
, NULL
, 1);
1072 if (icpu
->byt_auto_demotion_disable_flag
) {
1073 wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG
, 0);
1074 wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG
, 0);
1077 if (icpu
->disable_promotion_to_c1e
) /* each-cpu is redundant */
1078 on_each_cpu(c1e_promotion_disable
, NULL
, 1);
1085 * intel_idle_cpu_init()
1086 * allocate, initialize, register cpuidle_devices
1087 * @cpu: cpu/core to initialize
1089 static int intel_idle_cpu_init(int cpu
)
1091 struct cpuidle_device
*dev
;
1093 dev
= per_cpu_ptr(intel_idle_cpuidle_devices
, cpu
);
1097 if (cpuidle_register_device(dev
)) {
1098 pr_debug(PREFIX
"cpuidle_register_device %d failed!\n", cpu
);
1099 intel_idle_cpuidle_devices_uninit();
1103 if (icpu
->auto_demotion_disable_flags
)
1104 smp_call_function_single(cpu
, auto_demotion_disable
, NULL
, 1);
1106 if (icpu
->disable_promotion_to_c1e
)
1107 smp_call_function_single(cpu
, c1e_promotion_disable
, NULL
, 1);
1112 static int __init
intel_idle_init(void)
1116 /* Do not load intel_idle at all for now if idle= is passed */
1117 if (boot_option_idle_override
!= IDLE_NO_OVERRIDE
)
1120 retval
= intel_idle_probe();
1124 intel_idle_cpuidle_driver_init();
1125 retval
= cpuidle_register_driver(&intel_idle_driver
);
1127 struct cpuidle_driver
*drv
= cpuidle_get_driver();
1128 printk(KERN_DEBUG PREFIX
"intel_idle yielding to %s",
1129 drv
? drv
->name
: "none");
1133 intel_idle_cpuidle_devices
= alloc_percpu(struct cpuidle_device
);
1134 if (intel_idle_cpuidle_devices
== NULL
)
1137 cpu_notifier_register_begin();
1139 for_each_online_cpu(i
) {
1140 retval
= intel_idle_cpu_init(i
);
1142 cpu_notifier_register_done();
1143 cpuidle_unregister_driver(&intel_idle_driver
);
1147 __register_cpu_notifier(&cpu_hotplug_notifier
);
1149 cpu_notifier_register_done();
1154 static void __exit
intel_idle_exit(void)
1156 intel_idle_cpuidle_devices_uninit();
1157 cpuidle_unregister_driver(&intel_idle_driver
);
1159 cpu_notifier_register_begin();
1161 if (lapic_timer_reliable_states
!= LAPIC_TIMER_ALWAYS_RELIABLE
)
1162 on_each_cpu(__setup_broadcast_timer
, (void *)false, 1);
1163 __unregister_cpu_notifier(&cpu_hotplug_notifier
);
1165 cpu_notifier_register_done();
1170 module_init(intel_idle_init
);
1171 module_exit(intel_idle_exit
);
1173 module_param(max_cstate
, int, 0444);
1175 MODULE_AUTHOR("Len Brown <len.brown@intel.com>");
1176 MODULE_DESCRIPTION("Cpuidle driver for Intel Hardware v" INTEL_IDLE_VERSION
);
1177 MODULE_LICENSE("GPL");