]>
Commit | Line | Data |
---|---|---|
a61127c2 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
26717172 LB |
2 | /* |
3 | * intel_idle.c - native hardware idle loop for modern Intel processors | |
4 | * | |
fab04b22 | 5 | * Copyright (c) 2013, Intel Corporation. |
26717172 | 6 | * Len Brown <len.brown@intel.com> |
26717172 LB |
7 | */ |
8 | ||
9 | /* | |
10 | * intel_idle is a cpuidle driver that loads on specific Intel processors | |
11 | * in lieu of the legacy ACPI processor_idle driver. The intent is to | |
12 | * make Linux more efficient on these processors, as intel_idle knows | |
13 | * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. | |
14 | */ | |
15 | ||
16 | /* | |
17 | * Design Assumptions | |
18 | * | |
19 | * All CPUs have same idle states as boot CPU | |
20 | * | |
21 | * Chipset BM_STS (bus master status) bit is a NOP | |
22 | * for preventing entry into deep C-stats | |
23 | */ | |
24 | ||
25 | /* | |
26 | * Known limitations | |
27 | * | |
28 | * The driver currently initializes for_each_online_cpu() upon modprobe. | |
29 | * It it unaware of subsequent processors hot-added to the system. | |
30 | * This means that if you boot with maxcpus=n and later online | |
31 | * processors above n, those processors will use C1 only. | |
32 | * | |
33 | * ACPI has a .suspend hack to turn off deep c-statees during suspend | |
34 | * to avoid complications with the lapic timer workaround. | |
35 | * Have not seen issues with suspend, but may need same workaround here. | |
36 | * | |
26717172 LB |
37 | */ |
38 | ||
39 | /* un-comment DEBUG to enable pr_debug() statements */ | |
40 | #define DEBUG | |
41 | ||
654d08a4 JP |
42 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
43 | ||
26717172 LB |
44 | #include <linux/kernel.h> |
45 | #include <linux/cpuidle.h> | |
76962caa | 46 | #include <linux/tick.h> |
26717172 LB |
47 | #include <trace/events/power.h> |
48 | #include <linux/sched.h> | |
2a2d31c8 SL |
49 | #include <linux/notifier.h> |
50 | #include <linux/cpu.h> | |
02c4fae9 | 51 | #include <linux/moduleparam.h> |
b66b8b9a | 52 | #include <asm/cpu_device_id.h> |
db73c5a8 | 53 | #include <asm/intel-family.h> |
bc83cccc | 54 | #include <asm/mwait.h> |
14796fca | 55 | #include <asm/msr.h> |
26717172 | 56 | |
d70e28f5 | 57 | #define INTEL_IDLE_VERSION "0.4.1" |
26717172 | 58 | |
26717172 LB |
59 | static struct cpuidle_driver intel_idle_driver = { |
60 | .name = "intel_idle", | |
61 | .owner = THIS_MODULE, | |
62 | }; | |
63 | /* intel_idle.max_cstate=0 disables driver */ | |
137ecc77 | 64 | static int max_cstate = CPUIDLE_STATE_MAX - 1; |
26717172 | 65 | |
c4236282 | 66 | static unsigned int mwait_substates; |
26717172 | 67 | |
2a2d31c8 | 68 | #define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF |
26717172 | 69 | /* Reliable LAPIC Timer States, bit 1 for C1 etc. */ |
d13780d4 | 70 | static unsigned int lapic_timer_reliable_states = (1 << 1); /* Default to only C1 */ |
26717172 | 71 | |
b66b8b9a AK |
72 | struct idle_cpu { |
73 | struct cpuidle_state *state_table; | |
74 | ||
75 | /* | |
76 | * Hardware C-state auto-demotion may not always be optimal. | |
77 | * Indicate which enable bits to clear here. | |
78 | */ | |
79 | unsigned long auto_demotion_disable_flags; | |
8c058d53 | 80 | bool byt_auto_demotion_disable_flag; |
32e95180 | 81 | bool disable_promotion_to_c1e; |
b66b8b9a AK |
82 | }; |
83 | ||
84 | static const struct idle_cpu *icpu; | |
3265eba0 | 85 | static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; |
46bcfad7 DD |
86 | static int intel_idle(struct cpuidle_device *dev, |
87 | struct cpuidle_driver *drv, int index); | |
28ba086e | 88 | static void intel_idle_s2idle(struct cpuidle_device *dev, |
5fe2e527 | 89 | struct cpuidle_driver *drv, int index); |
26717172 LB |
90 | static struct cpuidle_state *cpuidle_state_table; |
91 | ||
956d033f LB |
92 | /* |
93 | * Set this flag for states where the HW flushes the TLB for us | |
94 | * and so we don't need cross-calls to keep it consistent. | |
95 | * If this flag is set, SW flushes the TLB, so even if the | |
96 | * HW doesn't do the flushing, this flag is safe to use. | |
97 | */ | |
98 | #define CPUIDLE_FLAG_TLB_FLUSHED 0x10000 | |
99 | ||
b1beab48 LB |
100 | /* |
101 | * MWAIT takes an 8-bit "hint" in EAX "suggesting" | |
102 | * the C-state (top nibble) and sub-state (bottom nibble) | |
103 | * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. | |
104 | * | |
105 | * We store the hint at the top of our "flags" for each state. | |
106 | */ | |
107 | #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) | |
108 | #define MWAIT2flg(eax) ((eax & 0xFF) << 24) | |
109 | ||
26717172 LB |
110 | /* |
111 | * States are indexed by the cstate number, | |
112 | * which is also the index into the MWAIT hint array. | |
113 | * Thus C0 is a dummy. | |
114 | */ | |
ba0dc81e | 115 | static struct cpuidle_state nehalem_cstates[] = { |
e022e7eb | 116 | { |
de09cdd0 | 117 | .name = "C1", |
26717172 | 118 | .desc = "MWAIT 0x00", |
b82b6cca | 119 | .flags = MWAIT2flg(0x00), |
26717172 | 120 | .exit_latency = 3, |
26717172 | 121 | .target_residency = 6, |
5fe2e527 | 122 | .enter = &intel_idle, |
28ba086e | 123 | .enter_s2idle = intel_idle_s2idle, }, |
32e95180 | 124 | { |
de09cdd0 | 125 | .name = "C1E", |
32e95180 | 126 | .desc = "MWAIT 0x01", |
b82b6cca | 127 | .flags = MWAIT2flg(0x01), |
32e95180 LB |
128 | .exit_latency = 10, |
129 | .target_residency = 20, | |
5fe2e527 | 130 | .enter = &intel_idle, |
28ba086e | 131 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 132 | { |
de09cdd0 | 133 | .name = "C3", |
26717172 | 134 | .desc = "MWAIT 0x10", |
b82b6cca | 135 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
26717172 | 136 | .exit_latency = 20, |
26717172 | 137 | .target_residency = 80, |
5fe2e527 | 138 | .enter = &intel_idle, |
28ba086e | 139 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 140 | { |
de09cdd0 | 141 | .name = "C6", |
26717172 | 142 | .desc = "MWAIT 0x20", |
b82b6cca | 143 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
26717172 | 144 | .exit_latency = 200, |
26717172 | 145 | .target_residency = 800, |
5fe2e527 | 146 | .enter = &intel_idle, |
28ba086e | 147 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb LB |
148 | { |
149 | .enter = NULL } | |
26717172 LB |
150 | }; |
151 | ||
ba0dc81e | 152 | static struct cpuidle_state snb_cstates[] = { |
e022e7eb | 153 | { |
de09cdd0 | 154 | .name = "C1", |
d13780d4 | 155 | .desc = "MWAIT 0x00", |
b82b6cca | 156 | .flags = MWAIT2flg(0x00), |
32e95180 LB |
157 | .exit_latency = 2, |
158 | .target_residency = 2, | |
5fe2e527 | 159 | .enter = &intel_idle, |
28ba086e | 160 | .enter_s2idle = intel_idle_s2idle, }, |
32e95180 | 161 | { |
de09cdd0 | 162 | .name = "C1E", |
32e95180 | 163 | .desc = "MWAIT 0x01", |
b82b6cca | 164 | .flags = MWAIT2flg(0x01), |
32e95180 LB |
165 | .exit_latency = 10, |
166 | .target_residency = 20, | |
5fe2e527 | 167 | .enter = &intel_idle, |
28ba086e | 168 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 169 | { |
de09cdd0 | 170 | .name = "C3", |
d13780d4 | 171 | .desc = "MWAIT 0x10", |
b82b6cca | 172 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
d13780d4 | 173 | .exit_latency = 80, |
ddbd550d | 174 | .target_residency = 211, |
5fe2e527 | 175 | .enter = &intel_idle, |
28ba086e | 176 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 177 | { |
de09cdd0 | 178 | .name = "C6", |
d13780d4 | 179 | .desc = "MWAIT 0x20", |
b82b6cca | 180 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
d13780d4 | 181 | .exit_latency = 104, |
ddbd550d | 182 | .target_residency = 345, |
5fe2e527 | 183 | .enter = &intel_idle, |
28ba086e | 184 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 185 | { |
de09cdd0 | 186 | .name = "C7", |
d13780d4 | 187 | .desc = "MWAIT 0x30", |
b82b6cca | 188 | .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, |
d13780d4 | 189 | .exit_latency = 109, |
ddbd550d | 190 | .target_residency = 345, |
5fe2e527 | 191 | .enter = &intel_idle, |
28ba086e | 192 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb LB |
193 | { |
194 | .enter = NULL } | |
d13780d4 LB |
195 | }; |
196 | ||
718987d6 LB |
197 | static struct cpuidle_state byt_cstates[] = { |
198 | { | |
de09cdd0 | 199 | .name = "C1", |
718987d6 | 200 | .desc = "MWAIT 0x00", |
b82b6cca | 201 | .flags = MWAIT2flg(0x00), |
718987d6 LB |
202 | .exit_latency = 1, |
203 | .target_residency = 1, | |
5fe2e527 | 204 | .enter = &intel_idle, |
28ba086e | 205 | .enter_s2idle = intel_idle_s2idle, }, |
718987d6 | 206 | { |
de09cdd0 | 207 | .name = "C6N", |
718987d6 | 208 | .desc = "MWAIT 0x58", |
b82b6cca | 209 | .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, |
d7ef7671 | 210 | .exit_latency = 300, |
718987d6 | 211 | .target_residency = 275, |
5fe2e527 | 212 | .enter = &intel_idle, |
28ba086e | 213 | .enter_s2idle = intel_idle_s2idle, }, |
718987d6 | 214 | { |
de09cdd0 | 215 | .name = "C6S", |
718987d6 | 216 | .desc = "MWAIT 0x52", |
b82b6cca | 217 | .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, |
d7ef7671 | 218 | .exit_latency = 500, |
718987d6 | 219 | .target_residency = 560, |
5fe2e527 | 220 | .enter = &intel_idle, |
28ba086e | 221 | .enter_s2idle = intel_idle_s2idle, }, |
718987d6 | 222 | { |
de09cdd0 | 223 | .name = "C7", |
718987d6 | 224 | .desc = "MWAIT 0x60", |
b82b6cca | 225 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, |
718987d6 | 226 | .exit_latency = 1200, |
d7ef7671 | 227 | .target_residency = 4000, |
5fe2e527 | 228 | .enter = &intel_idle, |
28ba086e | 229 | .enter_s2idle = intel_idle_s2idle, }, |
718987d6 | 230 | { |
de09cdd0 | 231 | .name = "C7S", |
718987d6 | 232 | .desc = "MWAIT 0x64", |
b82b6cca | 233 | .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, |
718987d6 LB |
234 | .exit_latency = 10000, |
235 | .target_residency = 20000, | |
5fe2e527 | 236 | .enter = &intel_idle, |
28ba086e | 237 | .enter_s2idle = intel_idle_s2idle, }, |
718987d6 LB |
238 | { |
239 | .enter = NULL } | |
240 | }; | |
241 | ||
cab07a56 LB |
242 | static struct cpuidle_state cht_cstates[] = { |
243 | { | |
de09cdd0 | 244 | .name = "C1", |
cab07a56 LB |
245 | .desc = "MWAIT 0x00", |
246 | .flags = MWAIT2flg(0x00), | |
247 | .exit_latency = 1, | |
248 | .target_residency = 1, | |
249 | .enter = &intel_idle, | |
28ba086e | 250 | .enter_s2idle = intel_idle_s2idle, }, |
cab07a56 | 251 | { |
de09cdd0 | 252 | .name = "C6N", |
cab07a56 LB |
253 | .desc = "MWAIT 0x58", |
254 | .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, | |
255 | .exit_latency = 80, | |
256 | .target_residency = 275, | |
257 | .enter = &intel_idle, | |
28ba086e | 258 | .enter_s2idle = intel_idle_s2idle, }, |
cab07a56 | 259 | { |
de09cdd0 | 260 | .name = "C6S", |
cab07a56 LB |
261 | .desc = "MWAIT 0x52", |
262 | .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, | |
263 | .exit_latency = 200, | |
264 | .target_residency = 560, | |
265 | .enter = &intel_idle, | |
28ba086e | 266 | .enter_s2idle = intel_idle_s2idle, }, |
cab07a56 | 267 | { |
de09cdd0 | 268 | .name = "C7", |
cab07a56 LB |
269 | .desc = "MWAIT 0x60", |
270 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, | |
271 | .exit_latency = 1200, | |
272 | .target_residency = 4000, | |
273 | .enter = &intel_idle, | |
28ba086e | 274 | .enter_s2idle = intel_idle_s2idle, }, |
cab07a56 | 275 | { |
de09cdd0 | 276 | .name = "C7S", |
cab07a56 LB |
277 | .desc = "MWAIT 0x64", |
278 | .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, | |
279 | .exit_latency = 10000, | |
280 | .target_residency = 20000, | |
281 | .enter = &intel_idle, | |
28ba086e | 282 | .enter_s2idle = intel_idle_s2idle, }, |
cab07a56 LB |
283 | { |
284 | .enter = NULL } | |
285 | }; | |
286 | ||
ba0dc81e | 287 | static struct cpuidle_state ivb_cstates[] = { |
e022e7eb | 288 | { |
de09cdd0 | 289 | .name = "C1", |
6edab08c | 290 | .desc = "MWAIT 0x00", |
b82b6cca | 291 | .flags = MWAIT2flg(0x00), |
6edab08c LB |
292 | .exit_latency = 1, |
293 | .target_residency = 1, | |
5fe2e527 | 294 | .enter = &intel_idle, |
28ba086e | 295 | .enter_s2idle = intel_idle_s2idle, }, |
32e95180 | 296 | { |
de09cdd0 | 297 | .name = "C1E", |
32e95180 | 298 | .desc = "MWAIT 0x01", |
b82b6cca | 299 | .flags = MWAIT2flg(0x01), |
32e95180 LB |
300 | .exit_latency = 10, |
301 | .target_residency = 20, | |
5fe2e527 | 302 | .enter = &intel_idle, |
28ba086e | 303 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 304 | { |
de09cdd0 | 305 | .name = "C3", |
6edab08c | 306 | .desc = "MWAIT 0x10", |
b82b6cca | 307 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
6edab08c LB |
308 | .exit_latency = 59, |
309 | .target_residency = 156, | |
5fe2e527 | 310 | .enter = &intel_idle, |
28ba086e | 311 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 312 | { |
de09cdd0 | 313 | .name = "C6", |
6edab08c | 314 | .desc = "MWAIT 0x20", |
b82b6cca | 315 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
6edab08c LB |
316 | .exit_latency = 80, |
317 | .target_residency = 300, | |
5fe2e527 | 318 | .enter = &intel_idle, |
28ba086e | 319 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 320 | { |
de09cdd0 | 321 | .name = "C7", |
6edab08c | 322 | .desc = "MWAIT 0x30", |
b82b6cca | 323 | .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, |
6edab08c LB |
324 | .exit_latency = 87, |
325 | .target_residency = 300, | |
5fe2e527 | 326 | .enter = &intel_idle, |
28ba086e | 327 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb LB |
328 | { |
329 | .enter = NULL } | |
6edab08c LB |
330 | }; |
331 | ||
0138d8f0 LB |
332 | static struct cpuidle_state ivt_cstates[] = { |
333 | { | |
de09cdd0 | 334 | .name = "C1", |
0138d8f0 | 335 | .desc = "MWAIT 0x00", |
b82b6cca | 336 | .flags = MWAIT2flg(0x00), |
0138d8f0 LB |
337 | .exit_latency = 1, |
338 | .target_residency = 1, | |
5fe2e527 | 339 | .enter = &intel_idle, |
28ba086e | 340 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 341 | { |
de09cdd0 | 342 | .name = "C1E", |
0138d8f0 | 343 | .desc = "MWAIT 0x01", |
b82b6cca | 344 | .flags = MWAIT2flg(0x01), |
0138d8f0 LB |
345 | .exit_latency = 10, |
346 | .target_residency = 80, | |
5fe2e527 | 347 | .enter = &intel_idle, |
28ba086e | 348 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 349 | { |
de09cdd0 | 350 | .name = "C3", |
0138d8f0 | 351 | .desc = "MWAIT 0x10", |
b82b6cca | 352 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
353 | .exit_latency = 59, |
354 | .target_residency = 156, | |
5fe2e527 | 355 | .enter = &intel_idle, |
28ba086e | 356 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 357 | { |
de09cdd0 | 358 | .name = "C6", |
0138d8f0 | 359 | .desc = "MWAIT 0x20", |
b82b6cca | 360 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
361 | .exit_latency = 82, |
362 | .target_residency = 300, | |
5fe2e527 | 363 | .enter = &intel_idle, |
28ba086e | 364 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 LB |
365 | { |
366 | .enter = NULL } | |
367 | }; | |
368 | ||
369 | static struct cpuidle_state ivt_cstates_4s[] = { | |
370 | { | |
de09cdd0 | 371 | .name = "C1", |
0138d8f0 | 372 | .desc = "MWAIT 0x00", |
b82b6cca | 373 | .flags = MWAIT2flg(0x00), |
0138d8f0 LB |
374 | .exit_latency = 1, |
375 | .target_residency = 1, | |
5fe2e527 | 376 | .enter = &intel_idle, |
28ba086e | 377 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 378 | { |
de09cdd0 | 379 | .name = "C1E", |
0138d8f0 | 380 | .desc = "MWAIT 0x01", |
b82b6cca | 381 | .flags = MWAIT2flg(0x01), |
0138d8f0 LB |
382 | .exit_latency = 10, |
383 | .target_residency = 250, | |
5fe2e527 | 384 | .enter = &intel_idle, |
28ba086e | 385 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 386 | { |
de09cdd0 | 387 | .name = "C3", |
0138d8f0 | 388 | .desc = "MWAIT 0x10", |
b82b6cca | 389 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
390 | .exit_latency = 59, |
391 | .target_residency = 300, | |
5fe2e527 | 392 | .enter = &intel_idle, |
28ba086e | 393 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 394 | { |
de09cdd0 | 395 | .name = "C6", |
0138d8f0 | 396 | .desc = "MWAIT 0x20", |
b82b6cca | 397 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
398 | .exit_latency = 84, |
399 | .target_residency = 400, | |
5fe2e527 | 400 | .enter = &intel_idle, |
28ba086e | 401 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 LB |
402 | { |
403 | .enter = NULL } | |
404 | }; | |
405 | ||
406 | static struct cpuidle_state ivt_cstates_8s[] = { | |
407 | { | |
de09cdd0 | 408 | .name = "C1", |
0138d8f0 | 409 | .desc = "MWAIT 0x00", |
b82b6cca | 410 | .flags = MWAIT2flg(0x00), |
0138d8f0 LB |
411 | .exit_latency = 1, |
412 | .target_residency = 1, | |
5fe2e527 | 413 | .enter = &intel_idle, |
28ba086e | 414 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 415 | { |
de09cdd0 | 416 | .name = "C1E", |
0138d8f0 | 417 | .desc = "MWAIT 0x01", |
b82b6cca | 418 | .flags = MWAIT2flg(0x01), |
0138d8f0 LB |
419 | .exit_latency = 10, |
420 | .target_residency = 500, | |
5fe2e527 | 421 | .enter = &intel_idle, |
28ba086e | 422 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 423 | { |
de09cdd0 | 424 | .name = "C3", |
0138d8f0 | 425 | .desc = "MWAIT 0x10", |
b82b6cca | 426 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
427 | .exit_latency = 59, |
428 | .target_residency = 600, | |
5fe2e527 | 429 | .enter = &intel_idle, |
28ba086e | 430 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 431 | { |
de09cdd0 | 432 | .name = "C6", |
0138d8f0 | 433 | .desc = "MWAIT 0x20", |
b82b6cca | 434 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
435 | .exit_latency = 88, |
436 | .target_residency = 700, | |
5fe2e527 | 437 | .enter = &intel_idle, |
28ba086e | 438 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 LB |
439 | { |
440 | .enter = NULL } | |
441 | }; | |
442 | ||
ba0dc81e | 443 | static struct cpuidle_state hsw_cstates[] = { |
e022e7eb | 444 | { |
de09cdd0 | 445 | .name = "C1", |
85a4d2d4 | 446 | .desc = "MWAIT 0x00", |
b82b6cca | 447 | .flags = MWAIT2flg(0x00), |
85a4d2d4 LB |
448 | .exit_latency = 2, |
449 | .target_residency = 2, | |
5fe2e527 | 450 | .enter = &intel_idle, |
28ba086e | 451 | .enter_s2idle = intel_idle_s2idle, }, |
32e95180 | 452 | { |
de09cdd0 | 453 | .name = "C1E", |
32e95180 | 454 | .desc = "MWAIT 0x01", |
b82b6cca | 455 | .flags = MWAIT2flg(0x01), |
32e95180 LB |
456 | .exit_latency = 10, |
457 | .target_residency = 20, | |
5fe2e527 | 458 | .enter = &intel_idle, |
28ba086e | 459 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 460 | { |
de09cdd0 | 461 | .name = "C3", |
85a4d2d4 | 462 | .desc = "MWAIT 0x10", |
b82b6cca | 463 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
85a4d2d4 LB |
464 | .exit_latency = 33, |
465 | .target_residency = 100, | |
5fe2e527 | 466 | .enter = &intel_idle, |
28ba086e | 467 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 468 | { |
de09cdd0 | 469 | .name = "C6", |
85a4d2d4 | 470 | .desc = "MWAIT 0x20", |
b82b6cca | 471 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
85a4d2d4 LB |
472 | .exit_latency = 133, |
473 | .target_residency = 400, | |
5fe2e527 | 474 | .enter = &intel_idle, |
28ba086e | 475 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 476 | { |
de09cdd0 | 477 | .name = "C7s", |
85a4d2d4 | 478 | .desc = "MWAIT 0x32", |
b82b6cca | 479 | .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, |
85a4d2d4 LB |
480 | .exit_latency = 166, |
481 | .target_residency = 500, | |
5fe2e527 | 482 | .enter = &intel_idle, |
28ba086e | 483 | .enter_s2idle = intel_idle_s2idle, }, |
86239ceb | 484 | { |
de09cdd0 | 485 | .name = "C8", |
86239ceb | 486 | .desc = "MWAIT 0x40", |
b82b6cca | 487 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, |
86239ceb LB |
488 | .exit_latency = 300, |
489 | .target_residency = 900, | |
5fe2e527 | 490 | .enter = &intel_idle, |
28ba086e | 491 | .enter_s2idle = intel_idle_s2idle, }, |
86239ceb | 492 | { |
de09cdd0 | 493 | .name = "C9", |
86239ceb | 494 | .desc = "MWAIT 0x50", |
b82b6cca | 495 | .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, |
86239ceb LB |
496 | .exit_latency = 600, |
497 | .target_residency = 1800, | |
5fe2e527 | 498 | .enter = &intel_idle, |
28ba086e | 499 | .enter_s2idle = intel_idle_s2idle, }, |
86239ceb | 500 | { |
de09cdd0 | 501 | .name = "C10", |
86239ceb | 502 | .desc = "MWAIT 0x60", |
b82b6cca | 503 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, |
86239ceb LB |
504 | .exit_latency = 2600, |
505 | .target_residency = 7700, | |
5fe2e527 | 506 | .enter = &intel_idle, |
28ba086e | 507 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb LB |
508 | { |
509 | .enter = NULL } | |
85a4d2d4 | 510 | }; |
a138b568 LB |
511 | static struct cpuidle_state bdw_cstates[] = { |
512 | { | |
de09cdd0 | 513 | .name = "C1", |
a138b568 | 514 | .desc = "MWAIT 0x00", |
b82b6cca | 515 | .flags = MWAIT2flg(0x00), |
a138b568 LB |
516 | .exit_latency = 2, |
517 | .target_residency = 2, | |
5fe2e527 | 518 | .enter = &intel_idle, |
28ba086e | 519 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 | 520 | { |
de09cdd0 | 521 | .name = "C1E", |
a138b568 | 522 | .desc = "MWAIT 0x01", |
b82b6cca | 523 | .flags = MWAIT2flg(0x01), |
a138b568 LB |
524 | .exit_latency = 10, |
525 | .target_residency = 20, | |
5fe2e527 | 526 | .enter = &intel_idle, |
28ba086e | 527 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 | 528 | { |
de09cdd0 | 529 | .name = "C3", |
a138b568 | 530 | .desc = "MWAIT 0x10", |
b82b6cca | 531 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
532 | .exit_latency = 40, |
533 | .target_residency = 100, | |
5fe2e527 | 534 | .enter = &intel_idle, |
28ba086e | 535 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 | 536 | { |
de09cdd0 | 537 | .name = "C6", |
a138b568 | 538 | .desc = "MWAIT 0x20", |
b82b6cca | 539 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
540 | .exit_latency = 133, |
541 | .target_residency = 400, | |
5fe2e527 | 542 | .enter = &intel_idle, |
28ba086e | 543 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 | 544 | { |
de09cdd0 | 545 | .name = "C7s", |
a138b568 | 546 | .desc = "MWAIT 0x32", |
b82b6cca | 547 | .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
548 | .exit_latency = 166, |
549 | .target_residency = 500, | |
5fe2e527 | 550 | .enter = &intel_idle, |
28ba086e | 551 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 | 552 | { |
de09cdd0 | 553 | .name = "C8", |
a138b568 | 554 | .desc = "MWAIT 0x40", |
b82b6cca | 555 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
556 | .exit_latency = 300, |
557 | .target_residency = 900, | |
5fe2e527 | 558 | .enter = &intel_idle, |
28ba086e | 559 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 | 560 | { |
de09cdd0 | 561 | .name = "C9", |
a138b568 | 562 | .desc = "MWAIT 0x50", |
b82b6cca | 563 | .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
564 | .exit_latency = 600, |
565 | .target_residency = 1800, | |
5fe2e527 | 566 | .enter = &intel_idle, |
28ba086e | 567 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 | 568 | { |
de09cdd0 | 569 | .name = "C10", |
a138b568 | 570 | .desc = "MWAIT 0x60", |
b82b6cca | 571 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
572 | .exit_latency = 2600, |
573 | .target_residency = 7700, | |
5fe2e527 | 574 | .enter = &intel_idle, |
28ba086e | 575 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 LB |
576 | { |
577 | .enter = NULL } | |
578 | }; | |
85a4d2d4 | 579 | |
493f133f LB |
580 | static struct cpuidle_state skl_cstates[] = { |
581 | { | |
de09cdd0 | 582 | .name = "C1", |
493f133f LB |
583 | .desc = "MWAIT 0x00", |
584 | .flags = MWAIT2flg(0x00), | |
585 | .exit_latency = 2, | |
586 | .target_residency = 2, | |
587 | .enter = &intel_idle, | |
28ba086e | 588 | .enter_s2idle = intel_idle_s2idle, }, |
493f133f | 589 | { |
de09cdd0 | 590 | .name = "C1E", |
493f133f LB |
591 | .desc = "MWAIT 0x01", |
592 | .flags = MWAIT2flg(0x01), | |
593 | .exit_latency = 10, | |
594 | .target_residency = 20, | |
595 | .enter = &intel_idle, | |
28ba086e | 596 | .enter_s2idle = intel_idle_s2idle, }, |
493f133f | 597 | { |
de09cdd0 | 598 | .name = "C3", |
493f133f LB |
599 | .desc = "MWAIT 0x10", |
600 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, | |
601 | .exit_latency = 70, | |
602 | .target_residency = 100, | |
603 | .enter = &intel_idle, | |
28ba086e | 604 | .enter_s2idle = intel_idle_s2idle, }, |
493f133f | 605 | { |
de09cdd0 | 606 | .name = "C6", |
493f133f LB |
607 | .desc = "MWAIT 0x20", |
608 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, | |
135919a3 | 609 | .exit_latency = 85, |
493f133f LB |
610 | .target_residency = 200, |
611 | .enter = &intel_idle, | |
28ba086e | 612 | .enter_s2idle = intel_idle_s2idle, }, |
493f133f | 613 | { |
de09cdd0 | 614 | .name = "C7s", |
493f133f LB |
615 | .desc = "MWAIT 0x33", |
616 | .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED, | |
617 | .exit_latency = 124, | |
618 | .target_residency = 800, | |
619 | .enter = &intel_idle, | |
28ba086e | 620 | .enter_s2idle = intel_idle_s2idle, }, |
493f133f | 621 | { |
de09cdd0 | 622 | .name = "C8", |
493f133f LB |
623 | .desc = "MWAIT 0x40", |
624 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, | |
135919a3 | 625 | .exit_latency = 200, |
493f133f LB |
626 | .target_residency = 800, |
627 | .enter = &intel_idle, | |
28ba086e | 628 | .enter_s2idle = intel_idle_s2idle, }, |
135919a3 | 629 | { |
de09cdd0 | 630 | .name = "C9", |
135919a3 LB |
631 | .desc = "MWAIT 0x50", |
632 | .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, | |
633 | .exit_latency = 480, | |
634 | .target_residency = 5000, | |
635 | .enter = &intel_idle, | |
28ba086e | 636 | .enter_s2idle = intel_idle_s2idle, }, |
493f133f | 637 | { |
de09cdd0 | 638 | .name = "C10", |
493f133f LB |
639 | .desc = "MWAIT 0x60", |
640 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, | |
641 | .exit_latency = 890, | |
642 | .target_residency = 5000, | |
643 | .enter = &intel_idle, | |
28ba086e | 644 | .enter_s2idle = intel_idle_s2idle, }, |
493f133f LB |
645 | { |
646 | .enter = NULL } | |
647 | }; | |
648 | ||
f9e71657 LB |
649 | static struct cpuidle_state skx_cstates[] = { |
650 | { | |
de09cdd0 | 651 | .name = "C1", |
f9e71657 LB |
652 | .desc = "MWAIT 0x00", |
653 | .flags = MWAIT2flg(0x00), | |
654 | .exit_latency = 2, | |
655 | .target_residency = 2, | |
656 | .enter = &intel_idle, | |
28ba086e | 657 | .enter_s2idle = intel_idle_s2idle, }, |
f9e71657 | 658 | { |
de09cdd0 | 659 | .name = "C1E", |
f9e71657 LB |
660 | .desc = "MWAIT 0x01", |
661 | .flags = MWAIT2flg(0x01), | |
662 | .exit_latency = 10, | |
663 | .target_residency = 20, | |
664 | .enter = &intel_idle, | |
28ba086e | 665 | .enter_s2idle = intel_idle_s2idle, }, |
f9e71657 | 666 | { |
de09cdd0 | 667 | .name = "C6", |
f9e71657 LB |
668 | .desc = "MWAIT 0x20", |
669 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, | |
670 | .exit_latency = 133, | |
671 | .target_residency = 600, | |
672 | .enter = &intel_idle, | |
28ba086e | 673 | .enter_s2idle = intel_idle_s2idle, }, |
f9e71657 LB |
674 | { |
675 | .enter = NULL } | |
676 | }; | |
677 | ||
ba0dc81e | 678 | static struct cpuidle_state atom_cstates[] = { |
e022e7eb | 679 | { |
de09cdd0 | 680 | .name = "C1E", |
26717172 | 681 | .desc = "MWAIT 0x00", |
b82b6cca | 682 | .flags = MWAIT2flg(0x00), |
32e95180 LB |
683 | .exit_latency = 10, |
684 | .target_residency = 20, | |
5fe2e527 | 685 | .enter = &intel_idle, |
28ba086e | 686 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 687 | { |
de09cdd0 | 688 | .name = "C2", |
26717172 | 689 | .desc = "MWAIT 0x10", |
b82b6cca | 690 | .flags = MWAIT2flg(0x10), |
26717172 | 691 | .exit_latency = 20, |
26717172 | 692 | .target_residency = 80, |
5fe2e527 | 693 | .enter = &intel_idle, |
28ba086e | 694 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 695 | { |
de09cdd0 | 696 | .name = "C4", |
26717172 | 697 | .desc = "MWAIT 0x30", |
b82b6cca | 698 | .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, |
26717172 | 699 | .exit_latency = 100, |
26717172 | 700 | .target_residency = 400, |
5fe2e527 | 701 | .enter = &intel_idle, |
28ba086e | 702 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 703 | { |
de09cdd0 | 704 | .name = "C6", |
7fcca7d9 | 705 | .desc = "MWAIT 0x52", |
b82b6cca | 706 | .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, |
7fcca7d9 | 707 | .exit_latency = 140, |
7fcca7d9 | 708 | .target_residency = 560, |
5fe2e527 | 709 | .enter = &intel_idle, |
28ba086e | 710 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb LB |
711 | { |
712 | .enter = NULL } | |
26717172 | 713 | }; |
5e7ec268 AS |
714 | static struct cpuidle_state tangier_cstates[] = { |
715 | { | |
de09cdd0 | 716 | .name = "C1", |
5e7ec268 AS |
717 | .desc = "MWAIT 0x00", |
718 | .flags = MWAIT2flg(0x00), | |
719 | .exit_latency = 1, | |
720 | .target_residency = 4, | |
721 | .enter = &intel_idle, | |
28ba086e | 722 | .enter_s2idle = intel_idle_s2idle, }, |
5e7ec268 | 723 | { |
de09cdd0 | 724 | .name = "C4", |
5e7ec268 AS |
725 | .desc = "MWAIT 0x30", |
726 | .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, | |
727 | .exit_latency = 100, | |
728 | .target_residency = 400, | |
729 | .enter = &intel_idle, | |
28ba086e | 730 | .enter_s2idle = intel_idle_s2idle, }, |
5e7ec268 | 731 | { |
de09cdd0 | 732 | .name = "C6", |
5e7ec268 AS |
733 | .desc = "MWAIT 0x52", |
734 | .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, | |
735 | .exit_latency = 140, | |
736 | .target_residency = 560, | |
737 | .enter = &intel_idle, | |
28ba086e | 738 | .enter_s2idle = intel_idle_s2idle, }, |
5e7ec268 | 739 | { |
de09cdd0 | 740 | .name = "C7", |
5e7ec268 AS |
741 | .desc = "MWAIT 0x60", |
742 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, | |
743 | .exit_latency = 1200, | |
744 | .target_residency = 4000, | |
745 | .enter = &intel_idle, | |
28ba086e | 746 | .enter_s2idle = intel_idle_s2idle, }, |
5e7ec268 | 747 | { |
de09cdd0 | 748 | .name = "C9", |
5e7ec268 AS |
749 | .desc = "MWAIT 0x64", |
750 | .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, | |
751 | .exit_latency = 10000, | |
752 | .target_residency = 20000, | |
753 | .enter = &intel_idle, | |
28ba086e | 754 | .enter_s2idle = intel_idle_s2idle, }, |
5e7ec268 AS |
755 | { |
756 | .enter = NULL } | |
757 | }; | |
88390996 | 758 | static struct cpuidle_state avn_cstates[] = { |
fab04b22 | 759 | { |
de09cdd0 | 760 | .name = "C1", |
fab04b22 | 761 | .desc = "MWAIT 0x00", |
b82b6cca | 762 | .flags = MWAIT2flg(0x00), |
fab04b22 LB |
763 | .exit_latency = 2, |
764 | .target_residency = 2, | |
5fe2e527 | 765 | .enter = &intel_idle, |
28ba086e | 766 | .enter_s2idle = intel_idle_s2idle, }, |
fab04b22 | 767 | { |
de09cdd0 | 768 | .name = "C6", |
fab04b22 | 769 | .desc = "MWAIT 0x51", |
b82b6cca | 770 | .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, |
fab04b22 LB |
771 | .exit_latency = 15, |
772 | .target_residency = 45, | |
5fe2e527 | 773 | .enter = &intel_idle, |
28ba086e | 774 | .enter_s2idle = intel_idle_s2idle, }, |
88390996 JL |
775 | { |
776 | .enter = NULL } | |
fab04b22 | 777 | }; |
281baf7a DC |
778 | static struct cpuidle_state knl_cstates[] = { |
779 | { | |
de09cdd0 | 780 | .name = "C1", |
281baf7a DC |
781 | .desc = "MWAIT 0x00", |
782 | .flags = MWAIT2flg(0x00), | |
783 | .exit_latency = 1, | |
784 | .target_residency = 2, | |
785 | .enter = &intel_idle, | |
28ba086e | 786 | .enter_s2idle = intel_idle_s2idle }, |
281baf7a | 787 | { |
de09cdd0 | 788 | .name = "C6", |
281baf7a DC |
789 | .desc = "MWAIT 0x10", |
790 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, | |
791 | .exit_latency = 120, | |
792 | .target_residency = 500, | |
793 | .enter = &intel_idle, | |
28ba086e | 794 | .enter_s2idle = intel_idle_s2idle }, |
281baf7a DC |
795 | { |
796 | .enter = NULL } | |
797 | }; | |
26717172 | 798 | |
5dcef694 LB |
799 | static struct cpuidle_state bxt_cstates[] = { |
800 | { | |
de09cdd0 | 801 | .name = "C1", |
5dcef694 LB |
802 | .desc = "MWAIT 0x00", |
803 | .flags = MWAIT2flg(0x00), | |
804 | .exit_latency = 2, | |
805 | .target_residency = 2, | |
806 | .enter = &intel_idle, | |
28ba086e | 807 | .enter_s2idle = intel_idle_s2idle, }, |
5dcef694 | 808 | { |
de09cdd0 | 809 | .name = "C1E", |
5dcef694 LB |
810 | .desc = "MWAIT 0x01", |
811 | .flags = MWAIT2flg(0x01), | |
812 | .exit_latency = 10, | |
813 | .target_residency = 20, | |
814 | .enter = &intel_idle, | |
28ba086e | 815 | .enter_s2idle = intel_idle_s2idle, }, |
5dcef694 | 816 | { |
de09cdd0 | 817 | .name = "C6", |
5dcef694 LB |
818 | .desc = "MWAIT 0x20", |
819 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, | |
820 | .exit_latency = 133, | |
821 | .target_residency = 133, | |
822 | .enter = &intel_idle, | |
28ba086e | 823 | .enter_s2idle = intel_idle_s2idle, }, |
5dcef694 | 824 | { |
de09cdd0 | 825 | .name = "C7s", |
5dcef694 LB |
826 | .desc = "MWAIT 0x31", |
827 | .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED, | |
828 | .exit_latency = 155, | |
829 | .target_residency = 155, | |
830 | .enter = &intel_idle, | |
28ba086e | 831 | .enter_s2idle = intel_idle_s2idle, }, |
5dcef694 | 832 | { |
de09cdd0 | 833 | .name = "C8", |
5dcef694 LB |
834 | .desc = "MWAIT 0x40", |
835 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, | |
836 | .exit_latency = 1000, | |
837 | .target_residency = 1000, | |
838 | .enter = &intel_idle, | |
28ba086e | 839 | .enter_s2idle = intel_idle_s2idle, }, |
5dcef694 | 840 | { |
de09cdd0 | 841 | .name = "C9", |
5dcef694 LB |
842 | .desc = "MWAIT 0x50", |
843 | .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, | |
844 | .exit_latency = 2000, | |
845 | .target_residency = 2000, | |
846 | .enter = &intel_idle, | |
28ba086e | 847 | .enter_s2idle = intel_idle_s2idle, }, |
5dcef694 | 848 | { |
de09cdd0 | 849 | .name = "C10", |
5dcef694 LB |
850 | .desc = "MWAIT 0x60", |
851 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, | |
852 | .exit_latency = 10000, | |
853 | .target_residency = 10000, | |
854 | .enter = &intel_idle, | |
28ba086e | 855 | .enter_s2idle = intel_idle_s2idle, }, |
5dcef694 LB |
856 | { |
857 | .enter = NULL } | |
858 | }; | |
859 | ||
0080d65b JP |
860 | static struct cpuidle_state dnv_cstates[] = { |
861 | { | |
de09cdd0 | 862 | .name = "C1", |
0080d65b JP |
863 | .desc = "MWAIT 0x00", |
864 | .flags = MWAIT2flg(0x00), | |
865 | .exit_latency = 2, | |
866 | .target_residency = 2, | |
867 | .enter = &intel_idle, | |
28ba086e | 868 | .enter_s2idle = intel_idle_s2idle, }, |
0080d65b | 869 | { |
de09cdd0 | 870 | .name = "C1E", |
0080d65b JP |
871 | .desc = "MWAIT 0x01", |
872 | .flags = MWAIT2flg(0x01), | |
873 | .exit_latency = 10, | |
874 | .target_residency = 20, | |
875 | .enter = &intel_idle, | |
28ba086e | 876 | .enter_s2idle = intel_idle_s2idle, }, |
0080d65b | 877 | { |
de09cdd0 | 878 | .name = "C6", |
0080d65b JP |
879 | .desc = "MWAIT 0x20", |
880 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, | |
881 | .exit_latency = 50, | |
882 | .target_residency = 500, | |
883 | .enter = &intel_idle, | |
28ba086e | 884 | .enter_s2idle = intel_idle_s2idle, }, |
0080d65b JP |
885 | { |
886 | .enter = NULL } | |
887 | }; | |
888 | ||
26717172 LB |
889 | /** |
890 | * intel_idle | |
891 | * @dev: cpuidle_device | |
46bcfad7 | 892 | * @drv: cpuidle driver |
e978aa7d | 893 | * @index: index of cpuidle state |
26717172 | 894 | * |
63ff07be | 895 | * Must be called under local_irq_disable(). |
26717172 | 896 | */ |
6727ad9e CM |
897 | static __cpuidle int intel_idle(struct cpuidle_device *dev, |
898 | struct cpuidle_driver *drv, int index) | |
26717172 LB |
899 | { |
900 | unsigned long ecx = 1; /* break on interrupt flag */ | |
46bcfad7 | 901 | struct cpuidle_state *state = &drv->states[index]; |
b1beab48 | 902 | unsigned long eax = flg2MWAIT(state->flags); |
26717172 | 903 | unsigned int cstate; |
0563bb7b | 904 | bool uninitialized_var(tick); |
67535736 | 905 | int cpu = smp_processor_id(); |
26717172 | 906 | |
6110a1f4 | 907 | /* |
67535736 AL |
908 | * leave_mm() to avoid costly and often unnecessary wakeups |
909 | * for flushing the user TLB's associated with the active mm. | |
6110a1f4 | 910 | */ |
67535736 AL |
911 | if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED) |
912 | leave_mm(cpu); | |
6110a1f4 | 913 | |
0563bb7b JB |
914 | if (!static_cpu_has(X86_FEATURE_ARAT)) { |
915 | cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) & | |
916 | MWAIT_CSTATE_MASK) + 1; | |
917 | tick = false; | |
918 | if (!(lapic_timer_reliable_states & (1 << (cstate)))) { | |
919 | tick = true; | |
920 | tick_broadcast_enter(); | |
921 | } | |
922 | } | |
26717172 | 923 | |
16824255 | 924 | mwait_idle_with_hints(eax, ecx); |
26717172 | 925 | |
0563bb7b | 926 | if (!static_cpu_has(X86_FEATURE_ARAT) && tick) |
f6cee191 | 927 | tick_broadcast_exit(); |
26717172 | 928 | |
e978aa7d | 929 | return index; |
26717172 LB |
930 | } |
931 | ||
5fe2e527 | 932 | /** |
28ba086e | 933 | * intel_idle_s2idle - simplified "enter" callback routine for suspend-to-idle |
5fe2e527 RW |
934 | * @dev: cpuidle_device |
935 | * @drv: cpuidle driver | |
936 | * @index: state index | |
937 | */ | |
28ba086e | 938 | static void intel_idle_s2idle(struct cpuidle_device *dev, |
5fe2e527 RW |
939 | struct cpuidle_driver *drv, int index) |
940 | { | |
941 | unsigned long ecx = 1; /* break on interrupt flag */ | |
942 | unsigned long eax = flg2MWAIT(drv->states[index].flags); | |
943 | ||
944 | mwait_idle_with_hints(eax, ecx); | |
945 | } | |
946 | ||
fb1013a0 | 947 | static void __setup_broadcast_timer(bool on) |
2a2d31c8 | 948 | { |
76962caa TG |
949 | if (on) |
950 | tick_broadcast_enable(); | |
951 | else | |
952 | tick_broadcast_disable(); | |
2a2d31c8 SL |
953 | } |
954 | ||
fb1013a0 | 955 | static void auto_demotion_disable(void) |
14796fca LB |
956 | { |
957 | unsigned long long msr_bits; | |
958 | ||
6cfb2374 | 959 | rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); |
b66b8b9a | 960 | msr_bits &= ~(icpu->auto_demotion_disable_flags); |
6cfb2374 | 961 | wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); |
14796fca | 962 | } |
fb1013a0 | 963 | static void c1e_promotion_disable(void) |
32e95180 LB |
964 | { |
965 | unsigned long long msr_bits; | |
966 | ||
967 | rdmsrl(MSR_IA32_POWER_CTL, msr_bits); | |
968 | msr_bits &= ~0x2; | |
969 | wrmsrl(MSR_IA32_POWER_CTL, msr_bits); | |
970 | } | |
14796fca | 971 | |
b66b8b9a AK |
972 | static const struct idle_cpu idle_cpu_nehalem = { |
973 | .state_table = nehalem_cstates, | |
b66b8b9a | 974 | .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, |
32e95180 | 975 | .disable_promotion_to_c1e = true, |
b66b8b9a AK |
976 | }; |
977 | ||
978 | static const struct idle_cpu idle_cpu_atom = { | |
979 | .state_table = atom_cstates, | |
980 | }; | |
981 | ||
5e7ec268 AS |
982 | static const struct idle_cpu idle_cpu_tangier = { |
983 | .state_table = tangier_cstates, | |
984 | }; | |
985 | ||
b66b8b9a AK |
986 | static const struct idle_cpu idle_cpu_lincroft = { |
987 | .state_table = atom_cstates, | |
988 | .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, | |
989 | }; | |
990 | ||
991 | static const struct idle_cpu idle_cpu_snb = { | |
992 | .state_table = snb_cstates, | |
32e95180 | 993 | .disable_promotion_to_c1e = true, |
b66b8b9a AK |
994 | }; |
995 | ||
718987d6 LB |
996 | static const struct idle_cpu idle_cpu_byt = { |
997 | .state_table = byt_cstates, | |
998 | .disable_promotion_to_c1e = true, | |
8c058d53 | 999 | .byt_auto_demotion_disable_flag = true, |
718987d6 LB |
1000 | }; |
1001 | ||
cab07a56 LB |
1002 | static const struct idle_cpu idle_cpu_cht = { |
1003 | .state_table = cht_cstates, | |
1004 | .disable_promotion_to_c1e = true, | |
1005 | .byt_auto_demotion_disable_flag = true, | |
1006 | }; | |
1007 | ||
6edab08c LB |
1008 | static const struct idle_cpu idle_cpu_ivb = { |
1009 | .state_table = ivb_cstates, | |
32e95180 | 1010 | .disable_promotion_to_c1e = true, |
6edab08c LB |
1011 | }; |
1012 | ||
0138d8f0 LB |
1013 | static const struct idle_cpu idle_cpu_ivt = { |
1014 | .state_table = ivt_cstates, | |
1015 | .disable_promotion_to_c1e = true, | |
1016 | }; | |
1017 | ||
85a4d2d4 LB |
1018 | static const struct idle_cpu idle_cpu_hsw = { |
1019 | .state_table = hsw_cstates, | |
32e95180 | 1020 | .disable_promotion_to_c1e = true, |
85a4d2d4 LB |
1021 | }; |
1022 | ||
a138b568 LB |
1023 | static const struct idle_cpu idle_cpu_bdw = { |
1024 | .state_table = bdw_cstates, | |
1025 | .disable_promotion_to_c1e = true, | |
1026 | }; | |
1027 | ||
493f133f LB |
1028 | static const struct idle_cpu idle_cpu_skl = { |
1029 | .state_table = skl_cstates, | |
1030 | .disable_promotion_to_c1e = true, | |
1031 | }; | |
1032 | ||
f9e71657 LB |
1033 | static const struct idle_cpu idle_cpu_skx = { |
1034 | .state_table = skx_cstates, | |
1035 | .disable_promotion_to_c1e = true, | |
1036 | }; | |
493f133f | 1037 | |
fab04b22 LB |
1038 | static const struct idle_cpu idle_cpu_avn = { |
1039 | .state_table = avn_cstates, | |
1040 | .disable_promotion_to_c1e = true, | |
1041 | }; | |
1042 | ||
281baf7a DC |
1043 | static const struct idle_cpu idle_cpu_knl = { |
1044 | .state_table = knl_cstates, | |
1045 | }; | |
1046 | ||
5dcef694 LB |
1047 | static const struct idle_cpu idle_cpu_bxt = { |
1048 | .state_table = bxt_cstates, | |
1049 | .disable_promotion_to_c1e = true, | |
1050 | }; | |
1051 | ||
0080d65b JP |
1052 | static const struct idle_cpu idle_cpu_dnv = { |
1053 | .state_table = dnv_cstates, | |
1054 | .disable_promotion_to_c1e = true, | |
1055 | }; | |
1056 | ||
d5cdc3c4 | 1057 | static const struct x86_cpu_id intel_idle_ids[] __initconst = { |
a4a008e5 AS |
1058 | INTEL_CPU_FAM6(NEHALEM_EP, idle_cpu_nehalem), |
1059 | INTEL_CPU_FAM6(NEHALEM, idle_cpu_nehalem), | |
1060 | INTEL_CPU_FAM6(NEHALEM_G, idle_cpu_nehalem), | |
1061 | INTEL_CPU_FAM6(WESTMERE, idle_cpu_nehalem), | |
1062 | INTEL_CPU_FAM6(WESTMERE_EP, idle_cpu_nehalem), | |
1063 | INTEL_CPU_FAM6(NEHALEM_EX, idle_cpu_nehalem), | |
c05f3642 LT |
1064 | INTEL_CPU_FAM6(ATOM_BONNELL, idle_cpu_atom), |
1065 | INTEL_CPU_FAM6(ATOM_BONNELL_MID, idle_cpu_lincroft), | |
a4a008e5 AS |
1066 | INTEL_CPU_FAM6(WESTMERE_EX, idle_cpu_nehalem), |
1067 | INTEL_CPU_FAM6(SANDYBRIDGE, idle_cpu_snb), | |
1068 | INTEL_CPU_FAM6(SANDYBRIDGE_X, idle_cpu_snb), | |
c05f3642 LT |
1069 | INTEL_CPU_FAM6(ATOM_SALTWELL, idle_cpu_atom), |
1070 | INTEL_CPU_FAM6(ATOM_SILVERMONT, idle_cpu_byt), | |
1071 | INTEL_CPU_FAM6(ATOM_SILVERMONT_MID, idle_cpu_tangier), | |
a4a008e5 AS |
1072 | INTEL_CPU_FAM6(ATOM_AIRMONT, idle_cpu_cht), |
1073 | INTEL_CPU_FAM6(IVYBRIDGE, idle_cpu_ivb), | |
1074 | INTEL_CPU_FAM6(IVYBRIDGE_X, idle_cpu_ivt), | |
c66f78a6 | 1075 | INTEL_CPU_FAM6(HASWELL, idle_cpu_hsw), |
a4a008e5 | 1076 | INTEL_CPU_FAM6(HASWELL_X, idle_cpu_hsw), |
af239c44 | 1077 | INTEL_CPU_FAM6(HASWELL_L, idle_cpu_hsw), |
5e741407 | 1078 | INTEL_CPU_FAM6(HASWELL_G, idle_cpu_hsw), |
5ebb34ed | 1079 | INTEL_CPU_FAM6(ATOM_SILVERMONT_D, idle_cpu_avn), |
c66f78a6 | 1080 | INTEL_CPU_FAM6(BROADWELL, idle_cpu_bdw), |
5e741407 | 1081 | INTEL_CPU_FAM6(BROADWELL_G, idle_cpu_bdw), |
a4a008e5 | 1082 | INTEL_CPU_FAM6(BROADWELL_X, idle_cpu_bdw), |
5ebb34ed | 1083 | INTEL_CPU_FAM6(BROADWELL_D, idle_cpu_bdw), |
af239c44 | 1084 | INTEL_CPU_FAM6(SKYLAKE_L, idle_cpu_skl), |
c66f78a6 | 1085 | INTEL_CPU_FAM6(SKYLAKE, idle_cpu_skl), |
af239c44 | 1086 | INTEL_CPU_FAM6(KABYLAKE_L, idle_cpu_skl), |
c66f78a6 | 1087 | INTEL_CPU_FAM6(KABYLAKE, idle_cpu_skl), |
a4a008e5 AS |
1088 | INTEL_CPU_FAM6(SKYLAKE_X, idle_cpu_skx), |
1089 | INTEL_CPU_FAM6(XEON_PHI_KNL, idle_cpu_knl), | |
1090 | INTEL_CPU_FAM6(XEON_PHI_KNM, idle_cpu_knl), | |
1091 | INTEL_CPU_FAM6(ATOM_GOLDMONT, idle_cpu_bxt), | |
c05f3642 | 1092 | INTEL_CPU_FAM6(ATOM_GOLDMONT_PLUS, idle_cpu_bxt), |
5ebb34ed PZ |
1093 | INTEL_CPU_FAM6(ATOM_GOLDMONT_D, idle_cpu_dnv), |
1094 | INTEL_CPU_FAM6(ATOM_TREMONT_D, idle_cpu_dnv), | |
b66b8b9a AK |
1095 | {} |
1096 | }; | |
b66b8b9a | 1097 | |
26717172 LB |
1098 | /* |
1099 | * intel_idle_probe() | |
1100 | */ | |
00f3e755 | 1101 | static int __init intel_idle_probe(void) |
26717172 | 1102 | { |
c4236282 | 1103 | unsigned int eax, ebx, ecx; |
b66b8b9a | 1104 | const struct x86_cpu_id *id; |
26717172 LB |
1105 | |
1106 | if (max_cstate == 0) { | |
654d08a4 | 1107 | pr_debug("disabled\n"); |
26717172 LB |
1108 | return -EPERM; |
1109 | } | |
1110 | ||
b66b8b9a AK |
1111 | id = x86_match_cpu(intel_idle_ids); |
1112 | if (!id) { | |
1113 | if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && | |
1114 | boot_cpu_data.x86 == 6) | |
654d08a4 JP |
1115 | pr_debug("does not run on family %d model %d\n", |
1116 | boot_cpu_data.x86, boot_cpu_data.x86_model); | |
26717172 | 1117 | return -ENODEV; |
b66b8b9a | 1118 | } |
26717172 | 1119 | |
a4c44753 LB |
1120 | if (!boot_cpu_has(X86_FEATURE_MWAIT)) { |
1121 | pr_debug("Please enable MWAIT in BIOS SETUP\n"); | |
1122 | return -ENODEV; | |
1123 | } | |
1124 | ||
26717172 LB |
1125 | if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) |
1126 | return -ENODEV; | |
1127 | ||
c4236282 | 1128 | cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); |
26717172 LB |
1129 | |
1130 | if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || | |
5c2a9f06 TR |
1131 | !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || |
1132 | !mwait_substates) | |
26717172 | 1133 | return -ENODEV; |
26717172 | 1134 | |
654d08a4 | 1135 | pr_debug("MWAIT substates: 0x%x\n", mwait_substates); |
26717172 | 1136 | |
b66b8b9a AK |
1137 | icpu = (const struct idle_cpu *)id->driver_data; |
1138 | cpuidle_state_table = icpu->state_table; | |
26717172 | 1139 | |
654d08a4 JP |
1140 | pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n", |
1141 | boot_cpu_data.x86_model); | |
26717172 | 1142 | |
26717172 LB |
1143 | return 0; |
1144 | } | |
1145 | ||
1146 | /* | |
1147 | * intel_idle_cpuidle_devices_uninit() | |
ca42489d | 1148 | * Unregisters the cpuidle devices. |
26717172 LB |
1149 | */ |
1150 | static void intel_idle_cpuidle_devices_uninit(void) | |
1151 | { | |
1152 | int i; | |
1153 | struct cpuidle_device *dev; | |
1154 | ||
1155 | for_each_online_cpu(i) { | |
1156 | dev = per_cpu_ptr(intel_idle_cpuidle_devices, i); | |
1157 | cpuidle_unregister_device(dev); | |
1158 | } | |
26717172 | 1159 | } |
0138d8f0 LB |
1160 | |
1161 | /* | |
d70e28f5 | 1162 | * ivt_idle_state_table_update(void) |
0138d8f0 | 1163 | * |
d70e28f5 | 1164 | * Tune IVT multi-socket targets |
0138d8f0 LB |
1165 | * Assumption: num_sockets == (max_package_num + 1) |
1166 | */ | |
d70e28f5 | 1167 | static void ivt_idle_state_table_update(void) |
0138d8f0 LB |
1168 | { |
1169 | /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ | |
d70e28f5 LB |
1170 | int cpu, package_num, num_sockets = 1; |
1171 | ||
1172 | for_each_online_cpu(cpu) { | |
1173 | package_num = topology_physical_package_id(cpu); | |
1174 | if (package_num + 1 > num_sockets) { | |
1175 | num_sockets = package_num + 1; | |
1176 | ||
1177 | if (num_sockets > 4) { | |
1178 | cpuidle_state_table = ivt_cstates_8s; | |
1179 | return; | |
0138d8f0 LB |
1180 | } |
1181 | } | |
d70e28f5 LB |
1182 | } |
1183 | ||
1184 | if (num_sockets > 2) | |
1185 | cpuidle_state_table = ivt_cstates_4s; | |
1186 | ||
1187 | /* else, 1 and 2 socket systems use default ivt_cstates */ | |
1188 | } | |
5dcef694 LB |
1189 | |
1190 | /* | |
1191 | * Translate IRTL (Interrupt Response Time Limit) MSR to usec | |
1192 | */ | |
1193 | ||
1194 | static unsigned int irtl_ns_units[] = { | |
1195 | 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 }; | |
1196 | ||
1197 | static unsigned long long irtl_2_usec(unsigned long long irtl) | |
1198 | { | |
1199 | unsigned long long ns; | |
1200 | ||
3451ab3e JB |
1201 | if (!irtl) |
1202 | return 0; | |
1203 | ||
bef45096 | 1204 | ns = irtl_ns_units[(irtl >> 10) & 0x7]; |
5dcef694 LB |
1205 | |
1206 | return div64_u64((irtl & 0x3FF) * ns, 1000); | |
1207 | } | |
1208 | /* | |
1209 | * bxt_idle_state_table_update(void) | |
1210 | * | |
1211 | * On BXT, we trust the IRTL to show the definitive maximum latency | |
1212 | * We use the same value for target_residency. | |
1213 | */ | |
1214 | static void bxt_idle_state_table_update(void) | |
1215 | { | |
1216 | unsigned long long msr; | |
3451ab3e | 1217 | unsigned int usec; |
5dcef694 LB |
1218 | |
1219 | rdmsrl(MSR_PKGC6_IRTL, msr); | |
3451ab3e JB |
1220 | usec = irtl_2_usec(msr); |
1221 | if (usec) { | |
5dcef694 LB |
1222 | bxt_cstates[2].exit_latency = usec; |
1223 | bxt_cstates[2].target_residency = usec; | |
1224 | } | |
1225 | ||
1226 | rdmsrl(MSR_PKGC7_IRTL, msr); | |
3451ab3e JB |
1227 | usec = irtl_2_usec(msr); |
1228 | if (usec) { | |
5dcef694 LB |
1229 | bxt_cstates[3].exit_latency = usec; |
1230 | bxt_cstates[3].target_residency = usec; | |
1231 | } | |
1232 | ||
1233 | rdmsrl(MSR_PKGC8_IRTL, msr); | |
3451ab3e JB |
1234 | usec = irtl_2_usec(msr); |
1235 | if (usec) { | |
5dcef694 LB |
1236 | bxt_cstates[4].exit_latency = usec; |
1237 | bxt_cstates[4].target_residency = usec; | |
1238 | } | |
1239 | ||
1240 | rdmsrl(MSR_PKGC9_IRTL, msr); | |
3451ab3e JB |
1241 | usec = irtl_2_usec(msr); |
1242 | if (usec) { | |
5dcef694 LB |
1243 | bxt_cstates[5].exit_latency = usec; |
1244 | bxt_cstates[5].target_residency = usec; | |
1245 | } | |
1246 | ||
1247 | rdmsrl(MSR_PKGC10_IRTL, msr); | |
3451ab3e JB |
1248 | usec = irtl_2_usec(msr); |
1249 | if (usec) { | |
5dcef694 LB |
1250 | bxt_cstates[6].exit_latency = usec; |
1251 | bxt_cstates[6].target_residency = usec; | |
1252 | } | |
1253 | ||
1254 | } | |
d70e28f5 LB |
1255 | /* |
1256 | * sklh_idle_state_table_update(void) | |
1257 | * | |
1258 | * On SKL-H (model 0x5e) disable C8 and C9 if: | |
1259 | * C10 is enabled and SGX disabled | |
1260 | */ | |
1261 | static void sklh_idle_state_table_update(void) | |
1262 | { | |
1263 | unsigned long long msr; | |
1264 | unsigned int eax, ebx, ecx, edx; | |
1265 | ||
1266 | ||
1267 | /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ | |
1268 | if (max_cstate <= 7) | |
1269 | return; | |
1270 | ||
1271 | /* if PC10 not present in CPUID.MWAIT.EDX */ | |
1272 | if ((mwait_substates & (0xF << 28)) == 0) | |
1273 | return; | |
1274 | ||
6cfb2374 | 1275 | rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); |
d70e28f5 LB |
1276 | |
1277 | /* PC10 is not enabled in PKG C-state limit */ | |
1278 | if ((msr & 0xF) != 8) | |
1279 | return; | |
1280 | ||
1281 | ecx = 0; | |
1282 | cpuid(7, &eax, &ebx, &ecx, &edx); | |
1283 | ||
1284 | /* if SGX is present */ | |
1285 | if (ebx & (1 << 2)) { | |
0138d8f0 | 1286 | |
d70e28f5 LB |
1287 | rdmsrl(MSR_IA32_FEATURE_CONTROL, msr); |
1288 | ||
1289 | /* if SGX is enabled */ | |
1290 | if (msr & (1 << 18)) | |
1291 | return; | |
1292 | } | |
1293 | ||
1294 | skl_cstates[5].disabled = 1; /* C8-SKL */ | |
1295 | skl_cstates[6].disabled = 1; /* C9-SKL */ | |
1296 | } | |
1297 | /* | |
1298 | * intel_idle_state_table_update() | |
1299 | * | |
1300 | * Update the default state_table for this CPU-id | |
1301 | */ | |
1302 | ||
1303 | static void intel_idle_state_table_update(void) | |
1304 | { | |
1305 | switch (boot_cpu_data.x86_model) { | |
1306 | ||
db73c5a8 | 1307 | case INTEL_FAM6_IVYBRIDGE_X: |
d70e28f5 LB |
1308 | ivt_idle_state_table_update(); |
1309 | break; | |
db73c5a8 | 1310 | case INTEL_FAM6_ATOM_GOLDMONT: |
f2c4db1b | 1311 | case INTEL_FAM6_ATOM_GOLDMONT_PLUS: |
5dcef694 LB |
1312 | bxt_idle_state_table_update(); |
1313 | break; | |
c66f78a6 | 1314 | case INTEL_FAM6_SKYLAKE: |
d70e28f5 LB |
1315 | sklh_idle_state_table_update(); |
1316 | break; | |
0138d8f0 | 1317 | } |
0138d8f0 LB |
1318 | } |
1319 | ||
46bcfad7 DD |
1320 | /* |
1321 | * intel_idle_cpuidle_driver_init() | |
1322 | * allocate, initialize cpuidle_states | |
1323 | */ | |
5469c827 | 1324 | static void __init intel_idle_cpuidle_driver_init(void) |
46bcfad7 DD |
1325 | { |
1326 | int cstate; | |
1327 | struct cpuidle_driver *drv = &intel_idle_driver; | |
1328 | ||
0138d8f0 LB |
1329 | intel_idle_state_table_update(); |
1330 | ||
1b39e3f8 | 1331 | cpuidle_poll_state_init(drv); |
46bcfad7 DD |
1332 | drv->state_count = 1; |
1333 | ||
e022e7eb | 1334 | for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { |
24bfa950 | 1335 | int num_substates, mwait_hint, mwait_cstate; |
46bcfad7 | 1336 | |
7dd0e0af | 1337 | if ((cpuidle_state_table[cstate].enter == NULL) && |
28ba086e | 1338 | (cpuidle_state_table[cstate].enter_s2idle == NULL)) |
e022e7eb LB |
1339 | break; |
1340 | ||
1341 | if (cstate + 1 > max_cstate) { | |
654d08a4 | 1342 | pr_info("max_cstate %d reached\n", max_cstate); |
46bcfad7 DD |
1343 | break; |
1344 | } | |
1345 | ||
e022e7eb LB |
1346 | mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); |
1347 | mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint); | |
e022e7eb | 1348 | |
24bfa950 | 1349 | /* number of sub-states for this state in CPUID.MWAIT */ |
e022e7eb | 1350 | num_substates = (mwait_substates >> ((mwait_cstate + 1) * 4)) |
46bcfad7 | 1351 | & MWAIT_SUBSTATE_MASK; |
e022e7eb | 1352 | |
24bfa950 LB |
1353 | /* if NO sub-states for this state in CPUID, skip it */ |
1354 | if (num_substates == 0) | |
46bcfad7 | 1355 | continue; |
46bcfad7 | 1356 | |
d70e28f5 LB |
1357 | /* if state marked as disabled, skip it */ |
1358 | if (cpuidle_state_table[cstate].disabled != 0) { | |
654d08a4 JP |
1359 | pr_debug("state %s is disabled\n", |
1360 | cpuidle_state_table[cstate].name); | |
d70e28f5 LB |
1361 | continue; |
1362 | } | |
1363 | ||
1364 | ||
e022e7eb | 1365 | if (((mwait_cstate + 1) > 2) && |
46bcfad7 DD |
1366 | !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) |
1367 | mark_tsc_unstable("TSC halts in idle" | |
1368 | " states deeper than C2"); | |
1369 | ||
1370 | drv->states[drv->state_count] = /* structure copy */ | |
1371 | cpuidle_state_table[cstate]; | |
1372 | ||
1373 | drv->state_count += 1; | |
1374 | } | |
1375 | ||
8c058d53 LB |
1376 | if (icpu->byt_auto_demotion_disable_flag) { |
1377 | wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); | |
1378 | wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); | |
1379 | } | |
46bcfad7 DD |
1380 | } |
1381 | ||
1382 | ||
26717172 | 1383 | /* |
65b7f839 | 1384 | * intel_idle_cpu_init() |
26717172 | 1385 | * allocate, initialize, register cpuidle_devices |
65b7f839 | 1386 | * @cpu: cpu/core to initialize |
26717172 | 1387 | */ |
fb1013a0 | 1388 | static int intel_idle_cpu_init(unsigned int cpu) |
26717172 | 1389 | { |
26717172 LB |
1390 | struct cpuidle_device *dev; |
1391 | ||
65b7f839 | 1392 | dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); |
65b7f839 | 1393 | dev->cpu = cpu; |
26717172 | 1394 | |
65b7f839 | 1395 | if (cpuidle_register_device(dev)) { |
654d08a4 | 1396 | pr_debug("cpuidle_register_device %d failed!\n", cpu); |
65b7f839 | 1397 | return -EIO; |
26717172 LB |
1398 | } |
1399 | ||
b66b8b9a | 1400 | if (icpu->auto_demotion_disable_flags) |
fb1013a0 | 1401 | auto_demotion_disable(); |
65b7f839 | 1402 | |
dbf87ab8 | 1403 | if (icpu->disable_promotion_to_c1e) |
fb1013a0 SAS |
1404 | c1e_promotion_disable(); |
1405 | ||
1406 | return 0; | |
1407 | } | |
1408 | ||
1409 | static int intel_idle_cpu_online(unsigned int cpu) | |
1410 | { | |
1411 | struct cpuidle_device *dev; | |
1412 | ||
1413 | if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) | |
1414 | __setup_broadcast_timer(true); | |
1415 | ||
1416 | /* | |
1417 | * Some systems can hotplug a cpu at runtime after | |
1418 | * the kernel has booted, we have to initialize the | |
1419 | * driver in this case | |
1420 | */ | |
1421 | dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); | |
1422 | if (!dev->registered) | |
1423 | return intel_idle_cpu_init(cpu); | |
dbf87ab8 | 1424 | |
26717172 LB |
1425 | return 0; |
1426 | } | |
26717172 LB |
1427 | |
1428 | static int __init intel_idle_init(void) | |
1429 | { | |
fb1013a0 | 1430 | int retval; |
26717172 | 1431 | |
d1896049 TR |
1432 | /* Do not load intel_idle at all for now if idle= is passed */ |
1433 | if (boot_option_idle_override != IDLE_NO_OVERRIDE) | |
1434 | return -ENODEV; | |
1435 | ||
26717172 LB |
1436 | retval = intel_idle_probe(); |
1437 | if (retval) | |
1438 | return retval; | |
1439 | ||
e9df69cc RC |
1440 | intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); |
1441 | if (intel_idle_cpuidle_devices == NULL) | |
1442 | return -ENOMEM; | |
1443 | ||
46bcfad7 | 1444 | intel_idle_cpuidle_driver_init(); |
26717172 LB |
1445 | retval = cpuidle_register_driver(&intel_idle_driver); |
1446 | if (retval) { | |
3735d524 | 1447 | struct cpuidle_driver *drv = cpuidle_get_driver(); |
654d08a4 JP |
1448 | printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"), |
1449 | drv ? drv->name : "none"); | |
fb1013a0 | 1450 | goto init_driver_fail; |
26717172 LB |
1451 | } |
1452 | ||
2259a819 RC |
1453 | if (boot_cpu_has(X86_FEATURE_ARAT)) /* Always Reliable APIC Timer */ |
1454 | lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE; | |
2259a819 | 1455 | |
fb1013a0 SAS |
1456 | retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", |
1457 | intel_idle_cpu_online, NULL); | |
1458 | if (retval < 0) | |
1459 | goto hp_setup_fail; | |
26717172 | 1460 | |
654d08a4 JP |
1461 | pr_debug("lapic_timer_reliable_states 0x%x\n", |
1462 | lapic_timer_reliable_states); | |
2259a819 | 1463 | |
26717172 | 1464 | return 0; |
fb1013a0 SAS |
1465 | |
1466 | hp_setup_fail: | |
1467 | intel_idle_cpuidle_devices_uninit(); | |
1468 | cpuidle_unregister_driver(&intel_idle_driver); | |
1469 | init_driver_fail: | |
1470 | free_percpu(intel_idle_cpuidle_devices); | |
1471 | return retval; | |
1472 | ||
26717172 | 1473 | } |
02c4fae9 | 1474 | device_initcall(intel_idle_init); |
26717172 | 1475 | |
02c4fae9 PG |
1476 | /* |
1477 | * We are not really modular, but we used to support that. Meaning we also | |
1478 | * support "intel_idle.max_cstate=..." at boot and also a read-only export of | |
1479 | * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param | |
1480 | * is the easiest way (currently) to continue doing that. | |
1481 | */ | |
26717172 | 1482 | module_param(max_cstate, int, 0444); |