]> git.proxmox.com Git - mirror_ubuntu-focal-kernel.git/blob - drivers/idle/intel_idle.c
projects / mirror_ubuntu-focal-kernel.git / blob
? search:


347b08b56042f707203fd625e6e757b7b186ae6a
[mirror_ubuntu-focal-kernel.git] / drivers / idle / intel_idle.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * intel_idle.c - native hardware idle loop for modern Intel processors
4 *
5 * Copyright (c) 2013, Intel Corporation.
6 * Len Brown <len.brown@intel.com>
7 */
8
9 /*
10 * intel_idle is a cpuidle driver that loads on specific Intel processors
11 * in lieu of the legacy ACPI processor_idle driver. The intent is to
12 * make Linux more efficient on these processors, as intel_idle knows
13 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs.
14 */
15
16 /*
17 * Design Assumptions
18 *
19 * All CPUs have same idle states as boot CPU
20 *
21 * Chipset BM_STS (bus master status) bit is a NOP
22 * for preventing entry into deep C-stats
23 */
24
25 /*
26 * Known limitations
27 *
28 * The driver currently initializes for_each_online_cpu() upon modprobe.
29 * It it unaware of subsequent processors hot-added to the system.
30 * This means that if you boot with maxcpus=n and later online
31 * processors above n, those processors will use C1 only.
32 *
33 * ACPI has a .suspend hack to turn off deep c-statees during suspend
34 * to avoid complications with the lapic timer workaround.
35 * Have not seen issues with suspend, but may need same workaround here.
36 *
37 */
38
39 /* un-comment DEBUG to enable pr_debug() statements */
40 #define DEBUG
41
42 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
43
44 #include <linux/kernel.h>
45 #include <linux/cpuidle.h>
46 #include <linux/tick.h>
47 #include <trace/events/power.h>
48 #include <linux/sched.h>
49 #include <linux/notifier.h>
50 #include <linux/cpu.h>
51 #include <linux/moduleparam.h>
52 #include <asm/cpu_device_id.h>
53 #include <asm/intel-family.h>
54 #include <asm/mwait.h>
55 #include <asm/msr.h>
56
57 #define INTEL_IDLE_VERSION "0.4.1"
58
59 static struct cpuidle_driver intel_idle_driver = {
60 .name = "intel_idle",
61 .owner = THIS_MODULE,
62 };
63 /* intel_idle.max_cstate=0 disables driver */
64 static int max_cstate = CPUIDLE_STATE_MAX - 1;
65
66 static unsigned int mwait_substates;
67
68 #define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF
69 /* Reliable LAPIC Timer States, bit 1 for C1 etc. */
70 static unsigned int lapic_timer_reliable_states = (1 << 1); /* Default to only C1 */
71
72 struct idle_cpu {
73 struct cpuidle_state *state_table;
74
75 /*
76 * Hardware C-state auto-demotion may not always be optimal.
77 * Indicate which enable bits to clear here.
78 */
79 unsigned long auto_demotion_disable_flags;
80 bool byt_auto_demotion_disable_flag;
81 bool disable_promotion_to_c1e;
82 };
83
84 static const struct idle_cpu *icpu;
85 static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
86 static int intel_idle(struct cpuidle_device *dev,
87 struct cpuidle_driver *drv, int index);
88 static void intel_idle_s2idle(struct cpuidle_device *dev,
89 struct cpuidle_driver *drv, int index);
90 static struct cpuidle_state *cpuidle_state_table;
91
92 /*
93 * Set this flag for states where the HW flushes the TLB for us
94 * and so we don't need cross-calls to keep it consistent.
95 * If this flag is set, SW flushes the TLB, so even if the
96 * HW doesn't do the flushing, this flag is safe to use.
97 */
98 #define CPUIDLE_FLAG_TLB_FLUSHED 0x10000
99
100 /*
101 * MWAIT takes an 8-bit "hint" in EAX "suggesting"
102 * the C-state (top nibble) and sub-state (bottom nibble)
103 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc.
104 *
105 * We store the hint at the top of our "flags" for each state.
106 */
107 #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF)
108 #define MWAIT2flg(eax) ((eax & 0xFF) << 24)
109
110 /*
111 * States are indexed by the cstate number,
112 * which is also the index into the MWAIT hint array.
113 * Thus C0 is a dummy.
114 */
115 static struct cpuidle_state nehalem_cstates[] = {
116 {
117 .name = "C1",
118 .desc = "MWAIT 0x00",
119 .flags = MWAIT2flg(0x00),
120 .exit_latency = 3,
121 .target_residency = 6,
122 .enter = &intel_idle,
123 .enter_s2idle = intel_idle_s2idle, },
124 {
125 .name = "C1E",
126 .desc = "MWAIT 0x01",
127 .flags = MWAIT2flg(0x01),
128 .exit_latency = 10,
129 .target_residency = 20,
130 .enter = &intel_idle,
131 .enter_s2idle = intel_idle_s2idle, },
132 {
133 .name = "C3",
134 .desc = "MWAIT 0x10",
135 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
136 .exit_latency = 20,
137 .target_residency = 80,
138 .enter = &intel_idle,
139 .enter_s2idle = intel_idle_s2idle, },
140 {
141 .name = "C6",
142 .desc = "MWAIT 0x20",
143 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
144 .exit_latency = 200,
145 .target_residency = 800,
146 .enter = &intel_idle,
147 .enter_s2idle = intel_idle_s2idle, },
148 {
149 .enter = NULL }
150 };
151
152 static struct cpuidle_state snb_cstates[] = {
153 {
154 .name = "C1",
155 .desc = "MWAIT 0x00",
156 .flags = MWAIT2flg(0x00),
157 .exit_latency = 2,
158 .target_residency = 2,
159 .enter = &intel_idle,
160 .enter_s2idle = intel_idle_s2idle, },
161 {
162 .name = "C1E",
163 .desc = "MWAIT 0x01",
164 .flags = MWAIT2flg(0x01),
165 .exit_latency = 10,
166 .target_residency = 20,
167 .enter = &intel_idle,
168 .enter_s2idle = intel_idle_s2idle, },
169 {
170 .name = "C3",
171 .desc = "MWAIT 0x10",
172 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
173 .exit_latency = 80,
174 .target_residency = 211,
175 .enter = &intel_idle,
176 .enter_s2idle = intel_idle_s2idle, },
177 {
178 .name = "C6",
179 .desc = "MWAIT 0x20",
180 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
181 .exit_latency = 104,
182 .target_residency = 345,
183 .enter = &intel_idle,
184 .enter_s2idle = intel_idle_s2idle, },
185 {
186 .name = "C7",
187 .desc = "MWAIT 0x30",
188 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
189 .exit_latency = 109,
190 .target_residency = 345,
191 .enter = &intel_idle,
192 .enter_s2idle = intel_idle_s2idle, },
193 {
194 .enter = NULL }
195 };
196
197 static struct cpuidle_state byt_cstates[] = {
198 {
199 .name = "C1",
200 .desc = "MWAIT 0x00",
201 .flags = MWAIT2flg(0x00),
202 .exit_latency = 1,
203 .target_residency = 1,
204 .enter = &intel_idle,
205 .enter_s2idle = intel_idle_s2idle, },
206 {
207 .name = "C6N",
208 .desc = "MWAIT 0x58",
209 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
210 .exit_latency = 300,
211 .target_residency = 275,
212 .enter = &intel_idle,
213 .enter_s2idle = intel_idle_s2idle, },
214 {
215 .name = "C6S",
216 .desc = "MWAIT 0x52",
217 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
218 .exit_latency = 500,
219 .target_residency = 560,
220 .enter = &intel_idle,
221 .enter_s2idle = intel_idle_s2idle, },
222 {
223 .name = "C7",
224 .desc = "MWAIT 0x60",
225 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
226 .exit_latency = 1200,
227 .target_residency = 4000,
228 .enter = &intel_idle,
229 .enter_s2idle = intel_idle_s2idle, },
230 {
231 .name = "C7S",
232 .desc = "MWAIT 0x64",
233 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
234 .exit_latency = 10000,
235 .target_residency = 20000,
236 .enter = &intel_idle,
237 .enter_s2idle = intel_idle_s2idle, },
238 {
239 .enter = NULL }
240 };
241
242 static struct cpuidle_state cht_cstates[] = {
243 {
244 .name = "C1",
245 .desc = "MWAIT 0x00",
246 .flags = MWAIT2flg(0x00),
247 .exit_latency = 1,
248 .target_residency = 1,
249 .enter = &intel_idle,
250 .enter_s2idle = intel_idle_s2idle, },
251 {
252 .name = "C6N",
253 .desc = "MWAIT 0x58",
254 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
255 .exit_latency = 80,
256 .target_residency = 275,
257 .enter = &intel_idle,
258 .enter_s2idle = intel_idle_s2idle, },
259 {
260 .name = "C6S",
261 .desc = "MWAIT 0x52",
262 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
263 .exit_latency = 200,
264 .target_residency = 560,
265 .enter = &intel_idle,
266 .enter_s2idle = intel_idle_s2idle, },
267 {
268 .name = "C7",
269 .desc = "MWAIT 0x60",
270 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
271 .exit_latency = 1200,
272 .target_residency = 4000,
273 .enter = &intel_idle,
274 .enter_s2idle = intel_idle_s2idle, },
275 {
276 .name = "C7S",
277 .desc = "MWAIT 0x64",
278 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
279 .exit_latency = 10000,
280 .target_residency = 20000,
281 .enter = &intel_idle,
282 .enter_s2idle = intel_idle_s2idle, },
283 {
284 .enter = NULL }
285 };
286
287 static struct cpuidle_state ivb_cstates[] = {
288 {
289 .name = "C1",
290 .desc = "MWAIT 0x00",
291 .flags = MWAIT2flg(0x00),
292 .exit_latency = 1,
293 .target_residency = 1,
294 .enter = &intel_idle,
295 .enter_s2idle = intel_idle_s2idle, },
296 {
297 .name = "C1E",
298 .desc = "MWAIT 0x01",
299 .flags = MWAIT2flg(0x01),
300 .exit_latency = 10,
301 .target_residency = 20,
302 .enter = &intel_idle,
303 .enter_s2idle = intel_idle_s2idle, },
304 {
305 .name = "C3",
306 .desc = "MWAIT 0x10",
307 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
308 .exit_latency = 59,
309 .target_residency = 156,
310 .enter = &intel_idle,
311 .enter_s2idle = intel_idle_s2idle, },
312 {
313 .name = "C6",
314 .desc = "MWAIT 0x20",
315 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
316 .exit_latency = 80,
317 .target_residency = 300,
318 .enter = &intel_idle,
319 .enter_s2idle = intel_idle_s2idle, },
320 {
321 .name = "C7",
322 .desc = "MWAIT 0x30",
323 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
324 .exit_latency = 87,
325 .target_residency = 300,
326 .enter = &intel_idle,
327 .enter_s2idle = intel_idle_s2idle, },
328 {
329 .enter = NULL }
330 };
331
332 static struct cpuidle_state ivt_cstates[] = {
333 {
334 .name = "C1",
335 .desc = "MWAIT 0x00",
336 .flags = MWAIT2flg(0x00),
337 .exit_latency = 1,
338 .target_residency = 1,
339 .enter = &intel_idle,
340 .enter_s2idle = intel_idle_s2idle, },
341 {
342 .name = "C1E",
343 .desc = "MWAIT 0x01",
344 .flags = MWAIT2flg(0x01),
345 .exit_latency = 10,
346 .target_residency = 80,
347 .enter = &intel_idle,
348 .enter_s2idle = intel_idle_s2idle, },
349 {
350 .name = "C3",
351 .desc = "MWAIT 0x10",
352 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
353 .exit_latency = 59,
354 .target_residency = 156,
355 .enter = &intel_idle,
356 .enter_s2idle = intel_idle_s2idle, },
357 {
358 .name = "C6",
359 .desc = "MWAIT 0x20",
360 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
361 .exit_latency = 82,
362 .target_residency = 300,
363 .enter = &intel_idle,
364 .enter_s2idle = intel_idle_s2idle, },
365 {
366 .enter = NULL }
367 };
368
369 static struct cpuidle_state ivt_cstates_4s[] = {
370 {
371 .name = "C1",
372 .desc = "MWAIT 0x00",
373 .flags = MWAIT2flg(0x00),
374 .exit_latency = 1,
375 .target_residency = 1,
376 .enter = &intel_idle,
377 .enter_s2idle = intel_idle_s2idle, },
378 {
379 .name = "C1E",
380 .desc = "MWAIT 0x01",
381 .flags = MWAIT2flg(0x01),
382 .exit_latency = 10,
383 .target_residency = 250,
384 .enter = &intel_idle,
385 .enter_s2idle = intel_idle_s2idle, },
386 {
387 .name = "C3",
388 .desc = "MWAIT 0x10",
389 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
390 .exit_latency = 59,
391 .target_residency = 300,
392 .enter = &intel_idle,
393 .enter_s2idle = intel_idle_s2idle, },
394 {
395 .name = "C6",
396 .desc = "MWAIT 0x20",
397 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
398 .exit_latency = 84,
399 .target_residency = 400,
400 .enter = &intel_idle,
401 .enter_s2idle = intel_idle_s2idle, },
402 {
403 .enter = NULL }
404 };
405
406 static struct cpuidle_state ivt_cstates_8s[] = {
407 {
408 .name = "C1",
409 .desc = "MWAIT 0x00",
410 .flags = MWAIT2flg(0x00),
411 .exit_latency = 1,
412 .target_residency = 1,
413 .enter = &intel_idle,
414 .enter_s2idle = intel_idle_s2idle, },
415 {
416 .name = "C1E",
417 .desc = "MWAIT 0x01",
418 .flags = MWAIT2flg(0x01),
419 .exit_latency = 10,
420 .target_residency = 500,
421 .enter = &intel_idle,
422 .enter_s2idle = intel_idle_s2idle, },
423 {
424 .name = "C3",
425 .desc = "MWAIT 0x10",
426 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
427 .exit_latency = 59,
428 .target_residency = 600,
429 .enter = &intel_idle,
430 .enter_s2idle = intel_idle_s2idle, },
431 {
432 .name = "C6",
433 .desc = "MWAIT 0x20",
434 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
435 .exit_latency = 88,
436 .target_residency = 700,
437 .enter = &intel_idle,
438 .enter_s2idle = intel_idle_s2idle, },
439 {
440 .enter = NULL }
441 };
442
443 static struct cpuidle_state hsw_cstates[] = {
444 {
445 .name = "C1",
446 .desc = "MWAIT 0x00",
447 .flags = MWAIT2flg(0x00),
448 .exit_latency = 2,
449 .target_residency = 2,
450 .enter = &intel_idle,
451 .enter_s2idle = intel_idle_s2idle, },
452 {
453 .name = "C1E",
454 .desc = "MWAIT 0x01",
455 .flags = MWAIT2flg(0x01),
456 .exit_latency = 10,
457 .target_residency = 20,
458 .enter = &intel_idle,
459 .enter_s2idle = intel_idle_s2idle, },
460 {
461 .name = "C3",
462 .desc = "MWAIT 0x10",
463 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
464 .exit_latency = 33,
465 .target_residency = 100,
466 .enter = &intel_idle,
467 .enter_s2idle = intel_idle_s2idle, },
468 {
469 .name = "C6",
470 .desc = "MWAIT 0x20",
471 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
472 .exit_latency = 133,
473 .target_residency = 400,
474 .enter = &intel_idle,
475 .enter_s2idle = intel_idle_s2idle, },
476 {
477 .name = "C7s",
478 .desc = "MWAIT 0x32",
479 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
480 .exit_latency = 166,
481 .target_residency = 500,
482 .enter = &intel_idle,
483 .enter_s2idle = intel_idle_s2idle, },
484 {
485 .name = "C8",
486 .desc = "MWAIT 0x40",
487 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
488 .exit_latency = 300,
489 .target_residency = 900,
490 .enter = &intel_idle,
491 .enter_s2idle = intel_idle_s2idle, },
492 {
493 .name = "C9",
494 .desc = "MWAIT 0x50",
495 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
496 .exit_latency = 600,
497 .target_residency = 1800,
498 .enter = &intel_idle,
499 .enter_s2idle = intel_idle_s2idle, },
500 {
501 .name = "C10",
502 .desc = "MWAIT 0x60",
503 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
504 .exit_latency = 2600,
505 .target_residency = 7700,
506 .enter = &intel_idle,
507 .enter_s2idle = intel_idle_s2idle, },
508 {
509 .enter = NULL }
510 };
511 static struct cpuidle_state bdw_cstates[] = {
512 {
513 .name = "C1",
514 .desc = "MWAIT 0x00",
515 .flags = MWAIT2flg(0x00),
516 .exit_latency = 2,
517 .target_residency = 2,
518 .enter = &intel_idle,
519 .enter_s2idle = intel_idle_s2idle, },
520 {
521 .name = "C1E",
522 .desc = "MWAIT 0x01",
523 .flags = MWAIT2flg(0x01),
524 .exit_latency = 10,
525 .target_residency = 20,
526 .enter = &intel_idle,
527 .enter_s2idle = intel_idle_s2idle, },
528 {
529 .name = "C3",
530 .desc = "MWAIT 0x10",
531 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
532 .exit_latency = 40,
533 .target_residency = 100,
534 .enter = &intel_idle,
535 .enter_s2idle = intel_idle_s2idle, },
536 {
537 .name = "C6",
538 .desc = "MWAIT 0x20",
539 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
540 .exit_latency = 133,
541 .target_residency = 400,
542 .enter = &intel_idle,
543 .enter_s2idle = intel_idle_s2idle, },
544 {
545 .name = "C7s",
546 .desc = "MWAIT 0x32",
547 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
548 .exit_latency = 166,
549 .target_residency = 500,
550 .enter = &intel_idle,
551 .enter_s2idle = intel_idle_s2idle, },
552 {
553 .name = "C8",
554 .desc = "MWAIT 0x40",
555 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
556 .exit_latency = 300,
557 .target_residency = 900,
558 .enter = &intel_idle,
559 .enter_s2idle = intel_idle_s2idle, },
560 {
561 .name = "C9",
562 .desc = "MWAIT 0x50",
563 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
564 .exit_latency = 600,
565 .target_residency = 1800,
566 .enter = &intel_idle,
567 .enter_s2idle = intel_idle_s2idle, },
568 {
569 .name = "C10",
570 .desc = "MWAIT 0x60",
571 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
572 .exit_latency = 2600,
573 .target_residency = 7700,
574 .enter = &intel_idle,
575 .enter_s2idle = intel_idle_s2idle, },
576 {
577 .enter = NULL }
578 };
579
580 static struct cpuidle_state skl_cstates[] = {
581 {
582 .name = "C1",
583 .desc = "MWAIT 0x00",
584 .flags = MWAIT2flg(0x00),
585 .exit_latency = 2,
586 .target_residency = 2,
587 .enter = &intel_idle,
588 .enter_s2idle = intel_idle_s2idle, },
589 {
590 .name = "C1E",
591 .desc = "MWAIT 0x01",
592 .flags = MWAIT2flg(0x01),
593 .exit_latency = 10,
594 .target_residency = 20,
595 .enter = &intel_idle,
596 .enter_s2idle = intel_idle_s2idle, },
597 {
598 .name = "C3",
599 .desc = "MWAIT 0x10",
600 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
601 .exit_latency = 70,
602 .target_residency = 100,
603 .enter = &intel_idle,
604 .enter_s2idle = intel_idle_s2idle, },
605 {
606 .name = "C6",
607 .desc = "MWAIT 0x20",
608 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
609 .exit_latency = 85,
610 .target_residency = 200,
611 .enter = &intel_idle,
612 .enter_s2idle = intel_idle_s2idle, },
613 {
614 .name = "C7s",
615 .desc = "MWAIT 0x33",
616 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED,
617 .exit_latency = 124,
618 .target_residency = 800,
619 .enter = &intel_idle,
620 .enter_s2idle = intel_idle_s2idle, },
621 {
622 .name = "C8",
623 .desc = "MWAIT 0x40",
624 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
625 .exit_latency = 200,
626 .target_residency = 800,
627 .enter = &intel_idle,
628 .enter_s2idle = intel_idle_s2idle, },
629 {
630 .name = "C9",
631 .desc = "MWAIT 0x50",
632 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
633 .exit_latency = 480,
634 .target_residency = 5000,
635 .enter = &intel_idle,
636 .enter_s2idle = intel_idle_s2idle, },
637 {
638 .name = "C10",
639 .desc = "MWAIT 0x60",
640 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
641 .exit_latency = 890,
642 .target_residency = 5000,
643 .enter = &intel_idle,
644 .enter_s2idle = intel_idle_s2idle, },
645 {
646 .enter = NULL }
647 };
648
649 static struct cpuidle_state skx_cstates[] = {
650 {
651 .name = "C1",
652 .desc = "MWAIT 0x00",
653 .flags = MWAIT2flg(0x00),
654 .exit_latency = 2,
655 .target_residency = 2,
656 .enter = &intel_idle,
657 .enter_s2idle = intel_idle_s2idle, },
658 {
659 .name = "C1E",
660 .desc = "MWAIT 0x01",
661 .flags = MWAIT2flg(0x01),
662 .exit_latency = 10,
663 .target_residency = 20,
664 .enter = &intel_idle,
665 .enter_s2idle = intel_idle_s2idle, },
666 {
667 .name = "C6",
668 .desc = "MWAIT 0x20",
669 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
670 .exit_latency = 133,
671 .target_residency = 600,
672 .enter = &intel_idle,
673 .enter_s2idle = intel_idle_s2idle, },
674 {
675 .enter = NULL }
676 };
677
678 static struct cpuidle_state atom_cstates[] = {
679 {
680 .name = "C1E",
681 .desc = "MWAIT 0x00",
682 .flags = MWAIT2flg(0x00),
683 .exit_latency = 10,
684 .target_residency = 20,
685 .enter = &intel_idle,
686 .enter_s2idle = intel_idle_s2idle, },
687 {
688 .name = "C2",
689 .desc = "MWAIT 0x10",
690 .flags = MWAIT2flg(0x10),
691 .exit_latency = 20,
692 .target_residency = 80,
693 .enter = &intel_idle,
694 .enter_s2idle = intel_idle_s2idle, },
695 {
696 .name = "C4",
697 .desc = "MWAIT 0x30",
698 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
699 .exit_latency = 100,
700 .target_residency = 400,
701 .enter = &intel_idle,
702 .enter_s2idle = intel_idle_s2idle, },
703 {
704 .name = "C6",
705 .desc = "MWAIT 0x52",
706 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
707 .exit_latency = 140,
708 .target_residency = 560,
709 .enter = &intel_idle,
710 .enter_s2idle = intel_idle_s2idle, },
711 {
712 .enter = NULL }
713 };
714 static struct cpuidle_state tangier_cstates[] = {
715 {
716 .name = "C1",
717 .desc = "MWAIT 0x00",
718 .flags = MWAIT2flg(0x00),
719 .exit_latency = 1,
720 .target_residency = 4,
721 .enter = &intel_idle,
722 .enter_s2idle = intel_idle_s2idle, },
723 {
724 .name = "C4",
725 .desc = "MWAIT 0x30",
726 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
727 .exit_latency = 100,
728 .target_residency = 400,
729 .enter = &intel_idle,
730 .enter_s2idle = intel_idle_s2idle, },
731 {
732 .name = "C6",
733 .desc = "MWAIT 0x52",
734 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
735 .exit_latency = 140,
736 .target_residency = 560,
737 .enter = &intel_idle,
738 .enter_s2idle = intel_idle_s2idle, },
739 {
740 .name = "C7",
741 .desc = "MWAIT 0x60",
742 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
743 .exit_latency = 1200,
744 .target_residency = 4000,
745 .enter = &intel_idle,
746 .enter_s2idle = intel_idle_s2idle, },
747 {
748 .name = "C9",
749 .desc = "MWAIT 0x64",
750 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
751 .exit_latency = 10000,
752 .target_residency = 20000,
753 .enter = &intel_idle,
754 .enter_s2idle = intel_idle_s2idle, },
755 {
756 .enter = NULL }
757 };
758 static struct cpuidle_state avn_cstates[] = {
759 {
760 .name = "C1",
761 .desc = "MWAIT 0x00",
762 .flags = MWAIT2flg(0x00),
763 .exit_latency = 2,
764 .target_residency = 2,
765 .enter = &intel_idle,
766 .enter_s2idle = intel_idle_s2idle, },
767 {
768 .name = "C6",
769 .desc = "MWAIT 0x51",
770 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED,
771 .exit_latency = 15,
772 .target_residency = 45,
773 .enter = &intel_idle,
774 .enter_s2idle = intel_idle_s2idle, },
775 {
776 .enter = NULL }
777 };
778 static struct cpuidle_state knl_cstates[] = {
779 {
780 .name = "C1",
781 .desc = "MWAIT 0x00",
782 .flags = MWAIT2flg(0x00),
783 .exit_latency = 1,
784 .target_residency = 2,
785 .enter = &intel_idle,
786 .enter_s2idle = intel_idle_s2idle },
787 {
788 .name = "C6",
789 .desc = "MWAIT 0x10",
790 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
791 .exit_latency = 120,
792 .target_residency = 500,
793 .enter = &intel_idle,
794 .enter_s2idle = intel_idle_s2idle },
795 {
796 .enter = NULL }
797 };
798
799 static struct cpuidle_state bxt_cstates[] = {
800 {
801 .name = "C1",
802 .desc = "MWAIT 0x00",
803 .flags = MWAIT2flg(0x00),
804 .exit_latency = 2,
805 .target_residency = 2,
806 .enter = &intel_idle,
807 .enter_s2idle = intel_idle_s2idle, },
808 {
809 .name = "C1E",
810 .desc = "MWAIT 0x01",
811 .flags = MWAIT2flg(0x01),
812 .exit_latency = 10,
813 .target_residency = 20,
814 .enter = &intel_idle,
815 .enter_s2idle = intel_idle_s2idle, },
816 {
817 .name = "C6",
818 .desc = "MWAIT 0x20",
819 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
820 .exit_latency = 133,
821 .target_residency = 133,
822 .enter = &intel_idle,
823 .enter_s2idle = intel_idle_s2idle, },
824 {
825 .name = "C7s",
826 .desc = "MWAIT 0x31",
827 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED,
828 .exit_latency = 155,
829 .target_residency = 155,
830 .enter = &intel_idle,
831 .enter_s2idle = intel_idle_s2idle, },
832 {
833 .name = "C8",
834 .desc = "MWAIT 0x40",
835 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
836 .exit_latency = 1000,
837 .target_residency = 1000,
838 .enter = &intel_idle,
839 .enter_s2idle = intel_idle_s2idle, },
840 {
841 .name = "C9",
842 .desc = "MWAIT 0x50",
843 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
844 .exit_latency = 2000,
845 .target_residency = 2000,
846 .enter = &intel_idle,
847 .enter_s2idle = intel_idle_s2idle, },
848 {
849 .name = "C10",
850 .desc = "MWAIT 0x60",
851 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
852 .exit_latency = 10000,
853 .target_residency = 10000,
854 .enter = &intel_idle,
855 .enter_s2idle = intel_idle_s2idle, },
856 {
857 .enter = NULL }
858 };
859
860 static struct cpuidle_state dnv_cstates[] = {
861 {
862 .name = "C1",
863 .desc = "MWAIT 0x00",
864 .flags = MWAIT2flg(0x00),
865 .exit_latency = 2,
866 .target_residency = 2,
867 .enter = &intel_idle,
868 .enter_s2idle = intel_idle_s2idle, },
869 {
870 .name = "C1E",
871 .desc = "MWAIT 0x01",
872 .flags = MWAIT2flg(0x01),
873 .exit_latency = 10,
874 .target_residency = 20,
875 .enter = &intel_idle,
876 .enter_s2idle = intel_idle_s2idle, },
877 {
878 .name = "C6",
879 .desc = "MWAIT 0x20",
880 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
881 .exit_latency = 50,
882 .target_residency = 500,
883 .enter = &intel_idle,
884 .enter_s2idle = intel_idle_s2idle, },
885 {
886 .enter = NULL }
887 };
888
889 /**
890 * intel_idle
891 * @dev: cpuidle_device
892 * @drv: cpuidle driver
893 * @index: index of cpuidle state
894 *
895 * Must be called under local_irq_disable().
896 */
897 static __cpuidle int intel_idle(struct cpuidle_device *dev,
898 struct cpuidle_driver *drv, int index)
899 {
900 unsigned long ecx = 1; /* break on interrupt flag */
901 struct cpuidle_state *state = &drv->states[index];
902 unsigned long eax = flg2MWAIT(state->flags);
903 unsigned int cstate;
904 bool uninitialized_var(tick);
905 int cpu = smp_processor_id();
906
907 /*
908 * leave_mm() to avoid costly and often unnecessary wakeups
909 * for flushing the user TLB's associated with the active mm.
910 */
911 if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED)
912 leave_mm(cpu);
913
914 if (!static_cpu_has(X86_FEATURE_ARAT)) {
915 cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) &
916 MWAIT_CSTATE_MASK) + 1;
917 tick = false;
918 if (!(lapic_timer_reliable_states & (1 << (cstate)))) {
919 tick = true;
920 tick_broadcast_enter();
921 }
922 }
923
924 mwait_idle_with_hints(eax, ecx);
925
926 if (!static_cpu_has(X86_FEATURE_ARAT) && tick)
927 tick_broadcast_exit();
928
929 return index;
930 }
931
932 /**
933 * intel_idle_s2idle - simplified "enter" callback routine for suspend-to-idle
934 * @dev: cpuidle_device
935 * @drv: cpuidle driver
936 * @index: state index
937 */
938 static void intel_idle_s2idle(struct cpuidle_device *dev,
939 struct cpuidle_driver *drv, int index)
940 {
941 unsigned long ecx = 1; /* break on interrupt flag */
942 unsigned long eax = flg2MWAIT(drv->states[index].flags);
943
944 mwait_idle_with_hints(eax, ecx);
945 }
946
947 static void __setup_broadcast_timer(bool on)
948 {
949 if (on)
950 tick_broadcast_enable();
951 else
952 tick_broadcast_disable();
953 }
954
955 static void auto_demotion_disable(void)
956 {
957 unsigned long long msr_bits;
958
959 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
960 msr_bits &= ~(icpu->auto_demotion_disable_flags);
961 wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
962 }
963 static void c1e_promotion_disable(void)
964 {
965 unsigned long long msr_bits;
966
967 rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
968 msr_bits &= ~0x2;
969 wrmsrl(MSR_IA32_POWER_CTL, msr_bits);
970 }
971
972 static const struct idle_cpu idle_cpu_nehalem = {
973 .state_table = nehalem_cstates,
974 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
975 .disable_promotion_to_c1e = true,
976 };
977
978 static const struct idle_cpu idle_cpu_atom = {
979 .state_table = atom_cstates,
980 };
981
982 static const struct idle_cpu idle_cpu_tangier = {
983 .state_table = tangier_cstates,
984 };
985
986 static const struct idle_cpu idle_cpu_lincroft = {
987 .state_table = atom_cstates,
988 .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE,
989 };
990
991 static const struct idle_cpu idle_cpu_snb = {
992 .state_table = snb_cstates,
993 .disable_promotion_to_c1e = true,
994 };
995
996 static const struct idle_cpu idle_cpu_byt = {
997 .state_table = byt_cstates,
998 .disable_promotion_to_c1e = true,
999 .byt_auto_demotion_disable_flag = true,
1000 };
1001
1002 static const struct idle_cpu idle_cpu_cht = {
1003 .state_table = cht_cstates,
1004 .disable_promotion_to_c1e = true,
1005 .byt_auto_demotion_disable_flag = true,
1006 };
1007
1008 static const struct idle_cpu idle_cpu_ivb = {
1009 .state_table = ivb_cstates,
1010 .disable_promotion_to_c1e = true,
1011 };
1012
1013 static const struct idle_cpu idle_cpu_ivt = {
1014 .state_table = ivt_cstates,
1015 .disable_promotion_to_c1e = true,
1016 };
1017
1018 static const struct idle_cpu idle_cpu_hsw = {
1019 .state_table = hsw_cstates,
1020 .disable_promotion_to_c1e = true,
1021 };
1022
1023 static const struct idle_cpu idle_cpu_bdw = {
1024 .state_table = bdw_cstates,
1025 .disable_promotion_to_c1e = true,
1026 };
1027
1028 static const struct idle_cpu idle_cpu_skl = {
1029 .state_table = skl_cstates,
1030 .disable_promotion_to_c1e = true,
1031 };
1032
1033 static const struct idle_cpu idle_cpu_skx = {
1034 .state_table = skx_cstates,
1035 .disable_promotion_to_c1e = true,
1036 };
1037
1038 static const struct idle_cpu idle_cpu_avn = {
1039 .state_table = avn_cstates,
1040 .disable_promotion_to_c1e = true,
1041 };
1042
1043 static const struct idle_cpu idle_cpu_knl = {
1044 .state_table = knl_cstates,
1045 };
1046
1047 static const struct idle_cpu idle_cpu_bxt = {
1048 .state_table = bxt_cstates,
1049 .disable_promotion_to_c1e = true,
1050 };
1051
1052 static const struct idle_cpu idle_cpu_dnv = {
1053 .state_table = dnv_cstates,
1054 .disable_promotion_to_c1e = true,
1055 };
1056
1057 static const struct x86_cpu_id intel_idle_ids[] __initconst = {
1058 INTEL_CPU_FAM6(NEHALEM_EP, idle_cpu_nehalem),
1059 INTEL_CPU_FAM6(NEHALEM, idle_cpu_nehalem),
1060 INTEL_CPU_FAM6(NEHALEM_G, idle_cpu_nehalem),
1061 INTEL_CPU_FAM6(WESTMERE, idle_cpu_nehalem),
1062 INTEL_CPU_FAM6(WESTMERE_EP, idle_cpu_nehalem),
1063 INTEL_CPU_FAM6(NEHALEM_EX, idle_cpu_nehalem),
1064 INTEL_CPU_FAM6(ATOM_BONNELL, idle_cpu_atom),
1065 INTEL_CPU_FAM6(ATOM_BONNELL_MID, idle_cpu_lincroft),
1066 INTEL_CPU_FAM6(WESTMERE_EX, idle_cpu_nehalem),
1067 INTEL_CPU_FAM6(SANDYBRIDGE, idle_cpu_snb),
1068 INTEL_CPU_FAM6(SANDYBRIDGE_X, idle_cpu_snb),
1069 INTEL_CPU_FAM6(ATOM_SALTWELL, idle_cpu_atom),
1070 INTEL_CPU_FAM6(ATOM_SILVERMONT, idle_cpu_byt),
1071 INTEL_CPU_FAM6(ATOM_SILVERMONT_MID, idle_cpu_tangier),
1072 INTEL_CPU_FAM6(ATOM_AIRMONT, idle_cpu_cht),
1073 INTEL_CPU_FAM6(IVYBRIDGE, idle_cpu_ivb),
1074 INTEL_CPU_FAM6(IVYBRIDGE_X, idle_cpu_ivt),
1075 INTEL_CPU_FAM6(HASWELL, idle_cpu_hsw),
1076 INTEL_CPU_FAM6(HASWELL_X, idle_cpu_hsw),
1077 INTEL_CPU_FAM6(HASWELL_L, idle_cpu_hsw),
1078 INTEL_CPU_FAM6(HASWELL_G, idle_cpu_hsw),
1079 INTEL_CPU_FAM6(ATOM_SILVERMONT_D, idle_cpu_avn),
1080 INTEL_CPU_FAM6(BROADWELL, idle_cpu_bdw),
1081 INTEL_CPU_FAM6(BROADWELL_G, idle_cpu_bdw),
1082 INTEL_CPU_FAM6(BROADWELL_X, idle_cpu_bdw),
1083 INTEL_CPU_FAM6(BROADWELL_D, idle_cpu_bdw),
1084 INTEL_CPU_FAM6(SKYLAKE_L, idle_cpu_skl),
1085 INTEL_CPU_FAM6(SKYLAKE, idle_cpu_skl),
1086 INTEL_CPU_FAM6(KABYLAKE_L, idle_cpu_skl),
1087 INTEL_CPU_FAM6(KABYLAKE, idle_cpu_skl),
1088 INTEL_CPU_FAM6(SKYLAKE_X, idle_cpu_skx),
1089 INTEL_CPU_FAM6(XEON_PHI_KNL, idle_cpu_knl),
1090 INTEL_CPU_FAM6(XEON_PHI_KNM, idle_cpu_knl),
1091 INTEL_CPU_FAM6(ATOM_GOLDMONT, idle_cpu_bxt),
1092 INTEL_CPU_FAM6(ATOM_GOLDMONT_PLUS, idle_cpu_bxt),
1093 INTEL_CPU_FAM6(ATOM_GOLDMONT_D, idle_cpu_dnv),
1094 INTEL_CPU_FAM6(ATOM_TREMONT_D, idle_cpu_dnv),
1095 {}
1096 };
1097
1098 /*
1099 * intel_idle_probe()
1100 */
1101 static int __init intel_idle_probe(void)
1102 {
1103 unsigned int eax, ebx, ecx;
1104 const struct x86_cpu_id *id;
1105
1106 if (max_cstate == 0) {
1107 pr_debug("disabled\n");
1108 return -EPERM;
1109 }
1110
1111 id = x86_match_cpu(intel_idle_ids);
1112 if (!id) {
1113 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
1114 boot_cpu_data.x86 == 6)
1115 pr_debug("does not run on family %d model %d\n",
1116 boot_cpu_data.x86, boot_cpu_data.x86_model);
1117 return -ENODEV;
1118 }
1119
1120 if (!boot_cpu_has(X86_FEATURE_MWAIT)) {
1121 pr_debug("Please enable MWAIT in BIOS SETUP\n");
1122 return -ENODEV;
1123 }
1124
1125 if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
1126 return -ENODEV;
1127
1128 cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates);
1129
1130 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
1131 !(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||
1132 !mwait_substates)
1133 return -ENODEV;
1134
1135 pr_debug("MWAIT substates: 0x%x\n", mwait_substates);
1136
1137 icpu = (const struct idle_cpu *)id->driver_data;
1138 cpuidle_state_table = icpu->state_table;
1139
1140 pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n",
1141 boot_cpu_data.x86_model);
1142
1143 return 0;
1144 }
1145
1146 /*
1147 * intel_idle_cpuidle_devices_uninit()
1148 * Unregisters the cpuidle devices.
1149 */
1150 static void intel_idle_cpuidle_devices_uninit(void)
1151 {
1152 int i;
1153 struct cpuidle_device *dev;
1154
1155 for_each_online_cpu(i) {
1156 dev = per_cpu_ptr(intel_idle_cpuidle_devices, i);
1157 cpuidle_unregister_device(dev);
1158 }
1159 }
1160
1161 /*
1162 * ivt_idle_state_table_update(void)
1163 *
1164 * Tune IVT multi-socket targets
1165 * Assumption: num_sockets == (max_package_num + 1)
1166 */
1167 static void ivt_idle_state_table_update(void)
1168 {
1169 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */
1170 int cpu, package_num, num_sockets = 1;
1171
1172 for_each_online_cpu(cpu) {
1173 package_num = topology_physical_package_id(cpu);
1174 if (package_num + 1 > num_sockets) {
1175 num_sockets = package_num + 1;
1176
1177 if (num_sockets > 4) {
1178 cpuidle_state_table = ivt_cstates_8s;
1179 return;
1180 }
1181 }
1182 }
1183
1184 if (num_sockets > 2)
1185 cpuidle_state_table = ivt_cstates_4s;
1186
1187 /* else, 1 and 2 socket systems use default ivt_cstates */
1188 }
1189
1190 /*
1191 * Translate IRTL (Interrupt Response Time Limit) MSR to usec
1192 */
1193
1194 static unsigned int irtl_ns_units[] = {
1195 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 };
1196
1197 static unsigned long long irtl_2_usec(unsigned long long irtl)
1198 {
1199 unsigned long long ns;
1200
1201 if (!irtl)
1202 return 0;
1203
1204 ns = irtl_ns_units[(irtl >> 10) & 0x7];
1205
1206 return div64_u64((irtl & 0x3FF) * ns, 1000);
1207 }
1208 /*
1209 * bxt_idle_state_table_update(void)
1210 *
1211 * On BXT, we trust the IRTL to show the definitive maximum latency
1212 * We use the same value for target_residency.
1213 */
1214 static void bxt_idle_state_table_update(void)
1215 {
1216 unsigned long long msr;
1217 unsigned int usec;
1218
1219 rdmsrl(MSR_PKGC6_IRTL, msr);
1220 usec = irtl_2_usec(msr);
1221 if (usec) {
1222 bxt_cstates[2].exit_latency = usec;
1223 bxt_cstates[2].target_residency = usec;
1224 }
1225
1226 rdmsrl(MSR_PKGC7_IRTL, msr);
1227 usec = irtl_2_usec(msr);
1228 if (usec) {
1229 bxt_cstates[3].exit_latency = usec;
1230 bxt_cstates[3].target_residency = usec;
1231 }
1232
1233 rdmsrl(MSR_PKGC8_IRTL, msr);
1234 usec = irtl_2_usec(msr);
1235 if (usec) {
1236 bxt_cstates[4].exit_latency = usec;
1237 bxt_cstates[4].target_residency = usec;
1238 }
1239
1240 rdmsrl(MSR_PKGC9_IRTL, msr);
1241 usec = irtl_2_usec(msr);
1242 if (usec) {
1243 bxt_cstates[5].exit_latency = usec;
1244 bxt_cstates[5].target_residency = usec;
1245 }
1246
1247 rdmsrl(MSR_PKGC10_IRTL, msr);
1248 usec = irtl_2_usec(msr);
1249 if (usec) {
1250 bxt_cstates[6].exit_latency = usec;
1251 bxt_cstates[6].target_residency = usec;
1252 }
1253
1254 }
1255 /*
1256 * sklh_idle_state_table_update(void)
1257 *
1258 * On SKL-H (model 0x5e) disable C8 and C9 if:
1259 * C10 is enabled and SGX disabled
1260 */
1261 static void sklh_idle_state_table_update(void)
1262 {
1263 unsigned long long msr;
1264 unsigned int eax, ebx, ecx, edx;
1265
1266
1267 /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */
1268 if (max_cstate <= 7)
1269 return;
1270
1271 /* if PC10 not present in CPUID.MWAIT.EDX */
1272 if ((mwait_substates & (0xF << 28)) == 0)
1273 return;
1274
1275 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1276
1277 /* PC10 is not enabled in PKG C-state limit */
1278 if ((msr & 0xF) != 8)
1279 return;
1280
1281 ecx = 0;
1282 cpuid(7, &eax, &ebx, &ecx, &edx);
1283
1284 /* if SGX is present */
1285 if (ebx & (1 << 2)) {
1286
1287 rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
1288
1289 /* if SGX is enabled */
1290 if (msr & (1 << 18))
1291 return;
1292 }
1293
1294 skl_cstates[5].disabled = 1; /* C8-SKL */
1295 skl_cstates[6].disabled = 1; /* C9-SKL */
1296 }
1297 /*
1298 * intel_idle_state_table_update()
1299 *
1300 * Update the default state_table for this CPU-id
1301 */
1302
1303 static void intel_idle_state_table_update(void)
1304 {
1305 switch (boot_cpu_data.x86_model) {
1306
1307 case INTEL_FAM6_IVYBRIDGE_X:
1308 ivt_idle_state_table_update();
1309 break;
1310 case INTEL_FAM6_ATOM_GOLDMONT:
1311 case INTEL_FAM6_ATOM_GOLDMONT_PLUS:
1312 bxt_idle_state_table_update();
1313 break;
1314 case INTEL_FAM6_SKYLAKE:
1315 sklh_idle_state_table_update();
1316 break;
1317 }
1318 }
1319
1320 /*
1321 * intel_idle_cpuidle_driver_init()
1322 * allocate, initialize cpuidle_states
1323 */
1324 static void __init intel_idle_cpuidle_driver_init(void)
1325 {
1326 int cstate;
1327 struct cpuidle_driver *drv = &intel_idle_driver;
1328
1329 intel_idle_state_table_update();
1330
1331 cpuidle_poll_state_init(drv);
1332 drv->state_count = 1;
1333
1334 for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) {
1335 int num_substates, mwait_hint, mwait_cstate;
1336
1337 if ((cpuidle_state_table[cstate].enter == NULL) &&
1338 (cpuidle_state_table[cstate].enter_s2idle == NULL))
1339 break;
1340
1341 if (cstate + 1 > max_cstate) {
1342 pr_info("max_cstate %d reached\n", max_cstate);
1343 break;
1344 }
1345
1346 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags);
1347 mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint);
1348
1349 /* number of sub-states for this state in CPUID.MWAIT */
1350 num_substates = (mwait_substates >> ((mwait_cstate + 1) * 4))
1351 & MWAIT_SUBSTATE_MASK;
1352
1353 /* if NO sub-states for this state in CPUID, skip it */
1354 if (num_substates == 0)
1355 continue;
1356
1357 /* if state marked as disabled, skip it */
1358 if (cpuidle_state_table[cstate].disabled != 0) {
1359 pr_debug("state %s is disabled\n",
1360 cpuidle_state_table[cstate].name);
1361 continue;
1362 }
1363
1364
1365 if (((mwait_cstate + 1) > 2) &&
1366 !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
1367 mark_tsc_unstable("TSC halts in idle"
1368 " states deeper than C2");
1369
1370 drv->states[drv->state_count] = /* structure copy */
1371 cpuidle_state_table[cstate];
1372
1373 drv->state_count += 1;
1374 }
1375
1376 if (icpu->byt_auto_demotion_disable_flag) {
1377 wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0);
1378 wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0);
1379 }
1380 }
1381
1382
1383 /*
1384 * intel_idle_cpu_init()
1385 * allocate, initialize, register cpuidle_devices
1386 * @cpu: cpu/core to initialize
1387 */
1388 static int intel_idle_cpu_init(unsigned int cpu)
1389 {
1390 struct cpuidle_device *dev;
1391
1392 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
1393 dev->cpu = cpu;
1394
1395 if (cpuidle_register_device(dev)) {
1396 pr_debug("cpuidle_register_device %d failed!\n", cpu);
1397 return -EIO;
1398 }
1399
1400 if (icpu->auto_demotion_disable_flags)
1401 auto_demotion_disable();
1402
1403 if (icpu->disable_promotion_to_c1e)
1404 c1e_promotion_disable();
1405
1406 return 0;
1407 }
1408
1409 static int intel_idle_cpu_online(unsigned int cpu)
1410 {
1411 struct cpuidle_device *dev;
1412
1413 if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE)
1414 __setup_broadcast_timer(true);
1415
1416 /*
1417 * Some systems can hotplug a cpu at runtime after
1418 * the kernel has booted, we have to initialize the
1419 * driver in this case
1420 */
1421 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
1422 if (!dev->registered)
1423 return intel_idle_cpu_init(cpu);
1424
1425 return 0;
1426 }
1427
1428 static int __init intel_idle_init(void)
1429 {
1430 int retval;
1431
1432 /* Do not load intel_idle at all for now if idle= is passed */
1433 if (boot_option_idle_override != IDLE_NO_OVERRIDE)
1434 return -ENODEV;
1435
1436 retval = intel_idle_probe();
1437 if (retval)
1438 return retval;
1439
1440 intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);
1441 if (intel_idle_cpuidle_devices == NULL)
1442 return -ENOMEM;
1443
1444 intel_idle_cpuidle_driver_init();
1445 retval = cpuidle_register_driver(&intel_idle_driver);
1446 if (retval) {
1447 struct cpuidle_driver *drv = cpuidle_get_driver();
1448 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"),
1449 drv ? drv->name : "none");
1450 goto init_driver_fail;
1451 }
1452
1453 if (boot_cpu_has(X86_FEATURE_ARAT)) /* Always Reliable APIC Timer */
1454 lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE;
1455
1456 retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online",
1457 intel_idle_cpu_online, NULL);
1458 if (retval < 0)
1459 goto hp_setup_fail;
1460
1461 pr_debug("lapic_timer_reliable_states 0x%x\n",
1462 lapic_timer_reliable_states);
1463
1464 return 0;
1465
1466 hp_setup_fail:
1467 intel_idle_cpuidle_devices_uninit();
1468 cpuidle_unregister_driver(&intel_idle_driver);
1469 init_driver_fail:
1470 free_percpu(intel_idle_cpuidle_devices);
1471 return retval;
1472
1473 }
1474 device_initcall(intel_idle_init);
1475
1476 /*
1477 * We are not really modular, but we used to support that. Meaning we also
1478 * support "intel_idle.max_cstate=..." at boot and also a read-only export of
1479 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param
1480 * is the easiest way (currently) to continue doing that.
1481 */
1482 module_param(max_cstate, int, 0444);
Ubuntu Focal Linux kernel mirror