]>
Commit | Line | Data |
---|---|---|
26717172 LB |
1 | /* |
2 | * intel_idle.c - native hardware idle loop for modern Intel processors | |
3 | * | |
fab04b22 | 4 | * Copyright (c) 2013, Intel Corporation. |
26717172 LB |
5 | * Len Brown <len.brown@intel.com> |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify it | |
8 | * under the terms and conditions of the GNU General Public License, | |
9 | * version 2, as published by the Free Software Foundation. | |
10 | * | |
11 | * This program is distributed in the hope it will be useful, but WITHOUT | |
12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
13 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
14 | * more details. | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License along with | |
17 | * this program; if not, write to the Free Software Foundation, Inc., | |
18 | * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. | |
19 | */ | |
20 | ||
21 | /* | |
22 | * intel_idle is a cpuidle driver that loads on specific Intel processors | |
23 | * in lieu of the legacy ACPI processor_idle driver. The intent is to | |
24 | * make Linux more efficient on these processors, as intel_idle knows | |
25 | * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. | |
26 | */ | |
27 | ||
28 | /* | |
29 | * Design Assumptions | |
30 | * | |
31 | * All CPUs have same idle states as boot CPU | |
32 | * | |
33 | * Chipset BM_STS (bus master status) bit is a NOP | |
34 | * for preventing entry into deep C-stats | |
35 | */ | |
36 | ||
37 | /* | |
38 | * Known limitations | |
39 | * | |
40 | * The driver currently initializes for_each_online_cpu() upon modprobe. | |
41 | * It it unaware of subsequent processors hot-added to the system. | |
42 | * This means that if you boot with maxcpus=n and later online | |
43 | * processors above n, those processors will use C1 only. | |
44 | * | |
45 | * ACPI has a .suspend hack to turn off deep c-statees during suspend | |
46 | * to avoid complications with the lapic timer workaround. | |
47 | * Have not seen issues with suspend, but may need same workaround here. | |
48 | * | |
49 | * There is currently no kernel-based automatic probing/loading mechanism | |
50 | * if the driver is built as a module. | |
51 | */ | |
52 | ||
53 | /* un-comment DEBUG to enable pr_debug() statements */ | |
54 | #define DEBUG | |
55 | ||
56 | #include <linux/kernel.h> | |
57 | #include <linux/cpuidle.h> | |
76962caa | 58 | #include <linux/tick.h> |
26717172 LB |
59 | #include <trace/events/power.h> |
60 | #include <linux/sched.h> | |
2a2d31c8 SL |
61 | #include <linux/notifier.h> |
62 | #include <linux/cpu.h> | |
7c52d551 | 63 | #include <linux/module.h> |
b66b8b9a | 64 | #include <asm/cpu_device_id.h> |
bc83cccc | 65 | #include <asm/mwait.h> |
14796fca | 66 | #include <asm/msr.h> |
26717172 | 67 | |
d70e28f5 | 68 | #define INTEL_IDLE_VERSION "0.4.1" |
26717172 LB |
69 | #define PREFIX "intel_idle: " |
70 | ||
26717172 LB |
71 | static struct cpuidle_driver intel_idle_driver = { |
72 | .name = "intel_idle", | |
73 | .owner = THIS_MODULE, | |
74 | }; | |
75 | /* intel_idle.max_cstate=0 disables driver */ | |
137ecc77 | 76 | static int max_cstate = CPUIDLE_STATE_MAX - 1; |
26717172 | 77 | |
c4236282 | 78 | static unsigned int mwait_substates; |
26717172 | 79 | |
2a2d31c8 | 80 | #define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF |
26717172 | 81 | /* Reliable LAPIC Timer States, bit 1 for C1 etc. */ |
d13780d4 | 82 | static unsigned int lapic_timer_reliable_states = (1 << 1); /* Default to only C1 */ |
26717172 | 83 | |
b66b8b9a AK |
84 | struct idle_cpu { |
85 | struct cpuidle_state *state_table; | |
86 | ||
87 | /* | |
88 | * Hardware C-state auto-demotion may not always be optimal. | |
89 | * Indicate which enable bits to clear here. | |
90 | */ | |
91 | unsigned long auto_demotion_disable_flags; | |
8c058d53 | 92 | bool byt_auto_demotion_disable_flag; |
32e95180 | 93 | bool disable_promotion_to_c1e; |
b66b8b9a AK |
94 | }; |
95 | ||
96 | static const struct idle_cpu *icpu; | |
3265eba0 | 97 | static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; |
46bcfad7 DD |
98 | static int intel_idle(struct cpuidle_device *dev, |
99 | struct cpuidle_driver *drv, int index); | |
5fe2e527 RW |
100 | static void intel_idle_freeze(struct cpuidle_device *dev, |
101 | struct cpuidle_driver *drv, int index); | |
25ac7761 | 102 | static int intel_idle_cpu_init(int cpu); |
26717172 LB |
103 | |
104 | static struct cpuidle_state *cpuidle_state_table; | |
105 | ||
956d033f LB |
106 | /* |
107 | * Set this flag for states where the HW flushes the TLB for us | |
108 | * and so we don't need cross-calls to keep it consistent. | |
109 | * If this flag is set, SW flushes the TLB, so even if the | |
110 | * HW doesn't do the flushing, this flag is safe to use. | |
111 | */ | |
112 | #define CPUIDLE_FLAG_TLB_FLUSHED 0x10000 | |
113 | ||
b1beab48 LB |
114 | /* |
115 | * MWAIT takes an 8-bit "hint" in EAX "suggesting" | |
116 | * the C-state (top nibble) and sub-state (bottom nibble) | |
117 | * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. | |
118 | * | |
119 | * We store the hint at the top of our "flags" for each state. | |
120 | */ | |
121 | #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) | |
122 | #define MWAIT2flg(eax) ((eax & 0xFF) << 24) | |
123 | ||
26717172 LB |
124 | /* |
125 | * States are indexed by the cstate number, | |
126 | * which is also the index into the MWAIT hint array. | |
127 | * Thus C0 is a dummy. | |
128 | */ | |
ba0dc81e | 129 | static struct cpuidle_state nehalem_cstates[] = { |
e022e7eb | 130 | { |
15e123e5 | 131 | .name = "C1-NHM", |
26717172 | 132 | .desc = "MWAIT 0x00", |
b82b6cca | 133 | .flags = MWAIT2flg(0x00), |
26717172 | 134 | .exit_latency = 3, |
26717172 | 135 | .target_residency = 6, |
5fe2e527 RW |
136 | .enter = &intel_idle, |
137 | .enter_freeze = intel_idle_freeze, }, | |
32e95180 LB |
138 | { |
139 | .name = "C1E-NHM", | |
140 | .desc = "MWAIT 0x01", | |
b82b6cca | 141 | .flags = MWAIT2flg(0x01), |
32e95180 LB |
142 | .exit_latency = 10, |
143 | .target_residency = 20, | |
5fe2e527 RW |
144 | .enter = &intel_idle, |
145 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 146 | { |
15e123e5 | 147 | .name = "C3-NHM", |
26717172 | 148 | .desc = "MWAIT 0x10", |
b82b6cca | 149 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
26717172 | 150 | .exit_latency = 20, |
26717172 | 151 | .target_residency = 80, |
5fe2e527 RW |
152 | .enter = &intel_idle, |
153 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 154 | { |
15e123e5 | 155 | .name = "C6-NHM", |
26717172 | 156 | .desc = "MWAIT 0x20", |
b82b6cca | 157 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
26717172 | 158 | .exit_latency = 200, |
26717172 | 159 | .target_residency = 800, |
5fe2e527 RW |
160 | .enter = &intel_idle, |
161 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb LB |
162 | { |
163 | .enter = NULL } | |
26717172 LB |
164 | }; |
165 | ||
ba0dc81e | 166 | static struct cpuidle_state snb_cstates[] = { |
e022e7eb | 167 | { |
15e123e5 | 168 | .name = "C1-SNB", |
d13780d4 | 169 | .desc = "MWAIT 0x00", |
b82b6cca | 170 | .flags = MWAIT2flg(0x00), |
32e95180 LB |
171 | .exit_latency = 2, |
172 | .target_residency = 2, | |
5fe2e527 RW |
173 | .enter = &intel_idle, |
174 | .enter_freeze = intel_idle_freeze, }, | |
32e95180 LB |
175 | { |
176 | .name = "C1E-SNB", | |
177 | .desc = "MWAIT 0x01", | |
b82b6cca | 178 | .flags = MWAIT2flg(0x01), |
32e95180 LB |
179 | .exit_latency = 10, |
180 | .target_residency = 20, | |
5fe2e527 RW |
181 | .enter = &intel_idle, |
182 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 183 | { |
15e123e5 | 184 | .name = "C3-SNB", |
d13780d4 | 185 | .desc = "MWAIT 0x10", |
b82b6cca | 186 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
d13780d4 | 187 | .exit_latency = 80, |
ddbd550d | 188 | .target_residency = 211, |
5fe2e527 RW |
189 | .enter = &intel_idle, |
190 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 191 | { |
15e123e5 | 192 | .name = "C6-SNB", |
d13780d4 | 193 | .desc = "MWAIT 0x20", |
b82b6cca | 194 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
d13780d4 | 195 | .exit_latency = 104, |
ddbd550d | 196 | .target_residency = 345, |
5fe2e527 RW |
197 | .enter = &intel_idle, |
198 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 199 | { |
15e123e5 | 200 | .name = "C7-SNB", |
d13780d4 | 201 | .desc = "MWAIT 0x30", |
b82b6cca | 202 | .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, |
d13780d4 | 203 | .exit_latency = 109, |
ddbd550d | 204 | .target_residency = 345, |
5fe2e527 RW |
205 | .enter = &intel_idle, |
206 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb LB |
207 | { |
208 | .enter = NULL } | |
d13780d4 LB |
209 | }; |
210 | ||
718987d6 LB |
211 | static struct cpuidle_state byt_cstates[] = { |
212 | { | |
213 | .name = "C1-BYT", | |
214 | .desc = "MWAIT 0x00", | |
b82b6cca | 215 | .flags = MWAIT2flg(0x00), |
718987d6 LB |
216 | .exit_latency = 1, |
217 | .target_residency = 1, | |
5fe2e527 RW |
218 | .enter = &intel_idle, |
219 | .enter_freeze = intel_idle_freeze, }, | |
718987d6 LB |
220 | { |
221 | .name = "C6N-BYT", | |
222 | .desc = "MWAIT 0x58", | |
b82b6cca | 223 | .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, |
d7ef7671 | 224 | .exit_latency = 300, |
718987d6 | 225 | .target_residency = 275, |
5fe2e527 RW |
226 | .enter = &intel_idle, |
227 | .enter_freeze = intel_idle_freeze, }, | |
718987d6 LB |
228 | { |
229 | .name = "C6S-BYT", | |
230 | .desc = "MWAIT 0x52", | |
b82b6cca | 231 | .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, |
d7ef7671 | 232 | .exit_latency = 500, |
718987d6 | 233 | .target_residency = 560, |
5fe2e527 RW |
234 | .enter = &intel_idle, |
235 | .enter_freeze = intel_idle_freeze, }, | |
718987d6 LB |
236 | { |
237 | .name = "C7-BYT", | |
238 | .desc = "MWAIT 0x60", | |
b82b6cca | 239 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, |
718987d6 | 240 | .exit_latency = 1200, |
d7ef7671 | 241 | .target_residency = 4000, |
5fe2e527 RW |
242 | .enter = &intel_idle, |
243 | .enter_freeze = intel_idle_freeze, }, | |
718987d6 LB |
244 | { |
245 | .name = "C7S-BYT", | |
246 | .desc = "MWAIT 0x64", | |
b82b6cca | 247 | .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, |
718987d6 LB |
248 | .exit_latency = 10000, |
249 | .target_residency = 20000, | |
5fe2e527 RW |
250 | .enter = &intel_idle, |
251 | .enter_freeze = intel_idle_freeze, }, | |
718987d6 LB |
252 | { |
253 | .enter = NULL } | |
254 | }; | |
255 | ||
cab07a56 LB |
256 | static struct cpuidle_state cht_cstates[] = { |
257 | { | |
258 | .name = "C1-CHT", | |
259 | .desc = "MWAIT 0x00", | |
260 | .flags = MWAIT2flg(0x00), | |
261 | .exit_latency = 1, | |
262 | .target_residency = 1, | |
263 | .enter = &intel_idle, | |
264 | .enter_freeze = intel_idle_freeze, }, | |
265 | { | |
266 | .name = "C6N-CHT", | |
267 | .desc = "MWAIT 0x58", | |
268 | .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, | |
269 | .exit_latency = 80, | |
270 | .target_residency = 275, | |
271 | .enter = &intel_idle, | |
272 | .enter_freeze = intel_idle_freeze, }, | |
273 | { | |
274 | .name = "C6S-CHT", | |
275 | .desc = "MWAIT 0x52", | |
276 | .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, | |
277 | .exit_latency = 200, | |
278 | .target_residency = 560, | |
279 | .enter = &intel_idle, | |
280 | .enter_freeze = intel_idle_freeze, }, | |
281 | { | |
282 | .name = "C7-CHT", | |
283 | .desc = "MWAIT 0x60", | |
284 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, | |
285 | .exit_latency = 1200, | |
286 | .target_residency = 4000, | |
287 | .enter = &intel_idle, | |
288 | .enter_freeze = intel_idle_freeze, }, | |
289 | { | |
290 | .name = "C7S-CHT", | |
291 | .desc = "MWAIT 0x64", | |
292 | .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, | |
293 | .exit_latency = 10000, | |
294 | .target_residency = 20000, | |
295 | .enter = &intel_idle, | |
296 | .enter_freeze = intel_idle_freeze, }, | |
297 | { | |
298 | .enter = NULL } | |
299 | }; | |
300 | ||
ba0dc81e | 301 | static struct cpuidle_state ivb_cstates[] = { |
e022e7eb | 302 | { |
6edab08c LB |
303 | .name = "C1-IVB", |
304 | .desc = "MWAIT 0x00", | |
b82b6cca | 305 | .flags = MWAIT2flg(0x00), |
6edab08c LB |
306 | .exit_latency = 1, |
307 | .target_residency = 1, | |
5fe2e527 RW |
308 | .enter = &intel_idle, |
309 | .enter_freeze = intel_idle_freeze, }, | |
32e95180 LB |
310 | { |
311 | .name = "C1E-IVB", | |
312 | .desc = "MWAIT 0x01", | |
b82b6cca | 313 | .flags = MWAIT2flg(0x01), |
32e95180 LB |
314 | .exit_latency = 10, |
315 | .target_residency = 20, | |
5fe2e527 RW |
316 | .enter = &intel_idle, |
317 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 318 | { |
6edab08c LB |
319 | .name = "C3-IVB", |
320 | .desc = "MWAIT 0x10", | |
b82b6cca | 321 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
6edab08c LB |
322 | .exit_latency = 59, |
323 | .target_residency = 156, | |
5fe2e527 RW |
324 | .enter = &intel_idle, |
325 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 326 | { |
6edab08c LB |
327 | .name = "C6-IVB", |
328 | .desc = "MWAIT 0x20", | |
b82b6cca | 329 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
6edab08c LB |
330 | .exit_latency = 80, |
331 | .target_residency = 300, | |
5fe2e527 RW |
332 | .enter = &intel_idle, |
333 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 334 | { |
6edab08c LB |
335 | .name = "C7-IVB", |
336 | .desc = "MWAIT 0x30", | |
b82b6cca | 337 | .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, |
6edab08c LB |
338 | .exit_latency = 87, |
339 | .target_residency = 300, | |
5fe2e527 RW |
340 | .enter = &intel_idle, |
341 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb LB |
342 | { |
343 | .enter = NULL } | |
6edab08c LB |
344 | }; |
345 | ||
0138d8f0 LB |
346 | static struct cpuidle_state ivt_cstates[] = { |
347 | { | |
348 | .name = "C1-IVT", | |
349 | .desc = "MWAIT 0x00", | |
b82b6cca | 350 | .flags = MWAIT2flg(0x00), |
0138d8f0 LB |
351 | .exit_latency = 1, |
352 | .target_residency = 1, | |
5fe2e527 RW |
353 | .enter = &intel_idle, |
354 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
355 | { |
356 | .name = "C1E-IVT", | |
357 | .desc = "MWAIT 0x01", | |
b82b6cca | 358 | .flags = MWAIT2flg(0x01), |
0138d8f0 LB |
359 | .exit_latency = 10, |
360 | .target_residency = 80, | |
5fe2e527 RW |
361 | .enter = &intel_idle, |
362 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
363 | { |
364 | .name = "C3-IVT", | |
365 | .desc = "MWAIT 0x10", | |
b82b6cca | 366 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
367 | .exit_latency = 59, |
368 | .target_residency = 156, | |
5fe2e527 RW |
369 | .enter = &intel_idle, |
370 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
371 | { |
372 | .name = "C6-IVT", | |
373 | .desc = "MWAIT 0x20", | |
b82b6cca | 374 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
375 | .exit_latency = 82, |
376 | .target_residency = 300, | |
5fe2e527 RW |
377 | .enter = &intel_idle, |
378 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
379 | { |
380 | .enter = NULL } | |
381 | }; | |
382 | ||
383 | static struct cpuidle_state ivt_cstates_4s[] = { | |
384 | { | |
385 | .name = "C1-IVT-4S", | |
386 | .desc = "MWAIT 0x00", | |
b82b6cca | 387 | .flags = MWAIT2flg(0x00), |
0138d8f0 LB |
388 | .exit_latency = 1, |
389 | .target_residency = 1, | |
5fe2e527 RW |
390 | .enter = &intel_idle, |
391 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
392 | { |
393 | .name = "C1E-IVT-4S", | |
394 | .desc = "MWAIT 0x01", | |
b82b6cca | 395 | .flags = MWAIT2flg(0x01), |
0138d8f0 LB |
396 | .exit_latency = 10, |
397 | .target_residency = 250, | |
5fe2e527 RW |
398 | .enter = &intel_idle, |
399 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
400 | { |
401 | .name = "C3-IVT-4S", | |
402 | .desc = "MWAIT 0x10", | |
b82b6cca | 403 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
404 | .exit_latency = 59, |
405 | .target_residency = 300, | |
5fe2e527 RW |
406 | .enter = &intel_idle, |
407 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
408 | { |
409 | .name = "C6-IVT-4S", | |
410 | .desc = "MWAIT 0x20", | |
b82b6cca | 411 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
412 | .exit_latency = 84, |
413 | .target_residency = 400, | |
5fe2e527 RW |
414 | .enter = &intel_idle, |
415 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
416 | { |
417 | .enter = NULL } | |
418 | }; | |
419 | ||
420 | static struct cpuidle_state ivt_cstates_8s[] = { | |
421 | { | |
422 | .name = "C1-IVT-8S", | |
423 | .desc = "MWAIT 0x00", | |
b82b6cca | 424 | .flags = MWAIT2flg(0x00), |
0138d8f0 LB |
425 | .exit_latency = 1, |
426 | .target_residency = 1, | |
5fe2e527 RW |
427 | .enter = &intel_idle, |
428 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
429 | { |
430 | .name = "C1E-IVT-8S", | |
431 | .desc = "MWAIT 0x01", | |
b82b6cca | 432 | .flags = MWAIT2flg(0x01), |
0138d8f0 LB |
433 | .exit_latency = 10, |
434 | .target_residency = 500, | |
5fe2e527 RW |
435 | .enter = &intel_idle, |
436 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
437 | { |
438 | .name = "C3-IVT-8S", | |
439 | .desc = "MWAIT 0x10", | |
b82b6cca | 440 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
441 | .exit_latency = 59, |
442 | .target_residency = 600, | |
5fe2e527 RW |
443 | .enter = &intel_idle, |
444 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
445 | { |
446 | .name = "C6-IVT-8S", | |
447 | .desc = "MWAIT 0x20", | |
b82b6cca | 448 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
449 | .exit_latency = 88, |
450 | .target_residency = 700, | |
5fe2e527 RW |
451 | .enter = &intel_idle, |
452 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
453 | { |
454 | .enter = NULL } | |
455 | }; | |
456 | ||
ba0dc81e | 457 | static struct cpuidle_state hsw_cstates[] = { |
e022e7eb | 458 | { |
85a4d2d4 LB |
459 | .name = "C1-HSW", |
460 | .desc = "MWAIT 0x00", | |
b82b6cca | 461 | .flags = MWAIT2flg(0x00), |
85a4d2d4 LB |
462 | .exit_latency = 2, |
463 | .target_residency = 2, | |
5fe2e527 RW |
464 | .enter = &intel_idle, |
465 | .enter_freeze = intel_idle_freeze, }, | |
32e95180 LB |
466 | { |
467 | .name = "C1E-HSW", | |
468 | .desc = "MWAIT 0x01", | |
b82b6cca | 469 | .flags = MWAIT2flg(0x01), |
32e95180 LB |
470 | .exit_latency = 10, |
471 | .target_residency = 20, | |
5fe2e527 RW |
472 | .enter = &intel_idle, |
473 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 474 | { |
85a4d2d4 LB |
475 | .name = "C3-HSW", |
476 | .desc = "MWAIT 0x10", | |
b82b6cca | 477 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
85a4d2d4 LB |
478 | .exit_latency = 33, |
479 | .target_residency = 100, | |
5fe2e527 RW |
480 | .enter = &intel_idle, |
481 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 482 | { |
85a4d2d4 LB |
483 | .name = "C6-HSW", |
484 | .desc = "MWAIT 0x20", | |
b82b6cca | 485 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
85a4d2d4 LB |
486 | .exit_latency = 133, |
487 | .target_residency = 400, | |
5fe2e527 RW |
488 | .enter = &intel_idle, |
489 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 490 | { |
85a4d2d4 LB |
491 | .name = "C7s-HSW", |
492 | .desc = "MWAIT 0x32", | |
b82b6cca | 493 | .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, |
85a4d2d4 LB |
494 | .exit_latency = 166, |
495 | .target_residency = 500, | |
5fe2e527 RW |
496 | .enter = &intel_idle, |
497 | .enter_freeze = intel_idle_freeze, }, | |
86239ceb LB |
498 | { |
499 | .name = "C8-HSW", | |
500 | .desc = "MWAIT 0x40", | |
b82b6cca | 501 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, |
86239ceb LB |
502 | .exit_latency = 300, |
503 | .target_residency = 900, | |
5fe2e527 RW |
504 | .enter = &intel_idle, |
505 | .enter_freeze = intel_idle_freeze, }, | |
86239ceb LB |
506 | { |
507 | .name = "C9-HSW", | |
508 | .desc = "MWAIT 0x50", | |
b82b6cca | 509 | .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, |
86239ceb LB |
510 | .exit_latency = 600, |
511 | .target_residency = 1800, | |
5fe2e527 RW |
512 | .enter = &intel_idle, |
513 | .enter_freeze = intel_idle_freeze, }, | |
86239ceb LB |
514 | { |
515 | .name = "C10-HSW", | |
516 | .desc = "MWAIT 0x60", | |
b82b6cca | 517 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, |
86239ceb LB |
518 | .exit_latency = 2600, |
519 | .target_residency = 7700, | |
5fe2e527 RW |
520 | .enter = &intel_idle, |
521 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb LB |
522 | { |
523 | .enter = NULL } | |
85a4d2d4 | 524 | }; |
a138b568 LB |
525 | static struct cpuidle_state bdw_cstates[] = { |
526 | { | |
527 | .name = "C1-BDW", | |
528 | .desc = "MWAIT 0x00", | |
b82b6cca | 529 | .flags = MWAIT2flg(0x00), |
a138b568 LB |
530 | .exit_latency = 2, |
531 | .target_residency = 2, | |
5fe2e527 RW |
532 | .enter = &intel_idle, |
533 | .enter_freeze = intel_idle_freeze, }, | |
a138b568 LB |
534 | { |
535 | .name = "C1E-BDW", | |
536 | .desc = "MWAIT 0x01", | |
b82b6cca | 537 | .flags = MWAIT2flg(0x01), |
a138b568 LB |
538 | .exit_latency = 10, |
539 | .target_residency = 20, | |
5fe2e527 RW |
540 | .enter = &intel_idle, |
541 | .enter_freeze = intel_idle_freeze, }, | |
a138b568 LB |
542 | { |
543 | .name = "C3-BDW", | |
544 | .desc = "MWAIT 0x10", | |
b82b6cca | 545 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
546 | .exit_latency = 40, |
547 | .target_residency = 100, | |
5fe2e527 RW |
548 | .enter = &intel_idle, |
549 | .enter_freeze = intel_idle_freeze, }, | |
a138b568 LB |
550 | { |
551 | .name = "C6-BDW", | |
552 | .desc = "MWAIT 0x20", | |
b82b6cca | 553 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
554 | .exit_latency = 133, |
555 | .target_residency = 400, | |
5fe2e527 RW |
556 | .enter = &intel_idle, |
557 | .enter_freeze = intel_idle_freeze, }, | |
a138b568 LB |
558 | { |
559 | .name = "C7s-BDW", | |
560 | .desc = "MWAIT 0x32", | |
b82b6cca | 561 | .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
562 | .exit_latency = 166, |
563 | .target_residency = 500, | |
5fe2e527 RW |
564 | .enter = &intel_idle, |
565 | .enter_freeze = intel_idle_freeze, }, | |
a138b568 LB |
566 | { |
567 | .name = "C8-BDW", | |
568 | .desc = "MWAIT 0x40", | |
b82b6cca | 569 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
570 | .exit_latency = 300, |
571 | .target_residency = 900, | |
5fe2e527 RW |
572 | .enter = &intel_idle, |
573 | .enter_freeze = intel_idle_freeze, }, | |
a138b568 LB |
574 | { |
575 | .name = "C9-BDW", | |
576 | .desc = "MWAIT 0x50", | |
b82b6cca | 577 | .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
578 | .exit_latency = 600, |
579 | .target_residency = 1800, | |
5fe2e527 RW |
580 | .enter = &intel_idle, |
581 | .enter_freeze = intel_idle_freeze, }, | |
a138b568 LB |
582 | { |
583 | .name = "C10-BDW", | |
584 | .desc = "MWAIT 0x60", | |
b82b6cca | 585 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
586 | .exit_latency = 2600, |
587 | .target_residency = 7700, | |
5fe2e527 RW |
588 | .enter = &intel_idle, |
589 | .enter_freeze = intel_idle_freeze, }, | |
a138b568 LB |
590 | { |
591 | .enter = NULL } | |
592 | }; | |
85a4d2d4 | 593 | |
493f133f LB |
594 | static struct cpuidle_state skl_cstates[] = { |
595 | { | |
596 | .name = "C1-SKL", | |
597 | .desc = "MWAIT 0x00", | |
598 | .flags = MWAIT2flg(0x00), | |
599 | .exit_latency = 2, | |
600 | .target_residency = 2, | |
601 | .enter = &intel_idle, | |
602 | .enter_freeze = intel_idle_freeze, }, | |
603 | { | |
604 | .name = "C1E-SKL", | |
605 | .desc = "MWAIT 0x01", | |
606 | .flags = MWAIT2flg(0x01), | |
607 | .exit_latency = 10, | |
608 | .target_residency = 20, | |
609 | .enter = &intel_idle, | |
610 | .enter_freeze = intel_idle_freeze, }, | |
611 | { | |
612 | .name = "C3-SKL", | |
613 | .desc = "MWAIT 0x10", | |
614 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, | |
615 | .exit_latency = 70, | |
616 | .target_residency = 100, | |
617 | .enter = &intel_idle, | |
618 | .enter_freeze = intel_idle_freeze, }, | |
619 | { | |
620 | .name = "C6-SKL", | |
621 | .desc = "MWAIT 0x20", | |
622 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, | |
135919a3 | 623 | .exit_latency = 85, |
493f133f LB |
624 | .target_residency = 200, |
625 | .enter = &intel_idle, | |
626 | .enter_freeze = intel_idle_freeze, }, | |
627 | { | |
628 | .name = "C7s-SKL", | |
629 | .desc = "MWAIT 0x33", | |
630 | .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED, | |
631 | .exit_latency = 124, | |
632 | .target_residency = 800, | |
633 | .enter = &intel_idle, | |
634 | .enter_freeze = intel_idle_freeze, }, | |
635 | { | |
636 | .name = "C8-SKL", | |
637 | .desc = "MWAIT 0x40", | |
638 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, | |
135919a3 | 639 | .exit_latency = 200, |
493f133f LB |
640 | .target_residency = 800, |
641 | .enter = &intel_idle, | |
642 | .enter_freeze = intel_idle_freeze, }, | |
135919a3 LB |
643 | { |
644 | .name = "C9-SKL", | |
645 | .desc = "MWAIT 0x50", | |
646 | .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, | |
647 | .exit_latency = 480, | |
648 | .target_residency = 5000, | |
649 | .enter = &intel_idle, | |
650 | .enter_freeze = intel_idle_freeze, }, | |
493f133f LB |
651 | { |
652 | .name = "C10-SKL", | |
653 | .desc = "MWAIT 0x60", | |
654 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, | |
655 | .exit_latency = 890, | |
656 | .target_residency = 5000, | |
657 | .enter = &intel_idle, | |
658 | .enter_freeze = intel_idle_freeze, }, | |
659 | { | |
660 | .enter = NULL } | |
661 | }; | |
662 | ||
ba0dc81e | 663 | static struct cpuidle_state atom_cstates[] = { |
e022e7eb | 664 | { |
32e95180 | 665 | .name = "C1E-ATM", |
26717172 | 666 | .desc = "MWAIT 0x00", |
b82b6cca | 667 | .flags = MWAIT2flg(0x00), |
32e95180 LB |
668 | .exit_latency = 10, |
669 | .target_residency = 20, | |
5fe2e527 RW |
670 | .enter = &intel_idle, |
671 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 672 | { |
15e123e5 | 673 | .name = "C2-ATM", |
26717172 | 674 | .desc = "MWAIT 0x10", |
b82b6cca | 675 | .flags = MWAIT2flg(0x10), |
26717172 | 676 | .exit_latency = 20, |
26717172 | 677 | .target_residency = 80, |
5fe2e527 RW |
678 | .enter = &intel_idle, |
679 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 680 | { |
15e123e5 | 681 | .name = "C4-ATM", |
26717172 | 682 | .desc = "MWAIT 0x30", |
b82b6cca | 683 | .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, |
26717172 | 684 | .exit_latency = 100, |
26717172 | 685 | .target_residency = 400, |
5fe2e527 RW |
686 | .enter = &intel_idle, |
687 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 688 | { |
15e123e5 | 689 | .name = "C6-ATM", |
7fcca7d9 | 690 | .desc = "MWAIT 0x52", |
b82b6cca | 691 | .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, |
7fcca7d9 | 692 | .exit_latency = 140, |
7fcca7d9 | 693 | .target_residency = 560, |
5fe2e527 RW |
694 | .enter = &intel_idle, |
695 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb LB |
696 | { |
697 | .enter = NULL } | |
26717172 | 698 | }; |
88390996 | 699 | static struct cpuidle_state avn_cstates[] = { |
fab04b22 LB |
700 | { |
701 | .name = "C1-AVN", | |
702 | .desc = "MWAIT 0x00", | |
b82b6cca | 703 | .flags = MWAIT2flg(0x00), |
fab04b22 LB |
704 | .exit_latency = 2, |
705 | .target_residency = 2, | |
5fe2e527 RW |
706 | .enter = &intel_idle, |
707 | .enter_freeze = intel_idle_freeze, }, | |
fab04b22 LB |
708 | { |
709 | .name = "C6-AVN", | |
710 | .desc = "MWAIT 0x51", | |
b82b6cca | 711 | .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, |
fab04b22 LB |
712 | .exit_latency = 15, |
713 | .target_residency = 45, | |
5fe2e527 RW |
714 | .enter = &intel_idle, |
715 | .enter_freeze = intel_idle_freeze, }, | |
88390996 JL |
716 | { |
717 | .enter = NULL } | |
fab04b22 | 718 | }; |
281baf7a DC |
719 | static struct cpuidle_state knl_cstates[] = { |
720 | { | |
721 | .name = "C1-KNL", | |
722 | .desc = "MWAIT 0x00", | |
723 | .flags = MWAIT2flg(0x00), | |
724 | .exit_latency = 1, | |
725 | .target_residency = 2, | |
726 | .enter = &intel_idle, | |
727 | .enter_freeze = intel_idle_freeze }, | |
728 | { | |
729 | .name = "C6-KNL", | |
730 | .desc = "MWAIT 0x10", | |
731 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, | |
732 | .exit_latency = 120, | |
733 | .target_residency = 500, | |
734 | .enter = &intel_idle, | |
735 | .enter_freeze = intel_idle_freeze }, | |
736 | { | |
737 | .enter = NULL } | |
738 | }; | |
26717172 | 739 | |
26717172 LB |
740 | /** |
741 | * intel_idle | |
742 | * @dev: cpuidle_device | |
46bcfad7 | 743 | * @drv: cpuidle driver |
e978aa7d | 744 | * @index: index of cpuidle state |
26717172 | 745 | * |
63ff07be | 746 | * Must be called under local_irq_disable(). |
26717172 | 747 | */ |
46bcfad7 DD |
748 | static int intel_idle(struct cpuidle_device *dev, |
749 | struct cpuidle_driver *drv, int index) | |
26717172 LB |
750 | { |
751 | unsigned long ecx = 1; /* break on interrupt flag */ | |
46bcfad7 | 752 | struct cpuidle_state *state = &drv->states[index]; |
b1beab48 | 753 | unsigned long eax = flg2MWAIT(state->flags); |
26717172 | 754 | unsigned int cstate; |
26717172 LB |
755 | int cpu = smp_processor_id(); |
756 | ||
757 | cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK) + 1; | |
758 | ||
6110a1f4 | 759 | /* |
c8381cc3 LB |
760 | * leave_mm() to avoid costly and often unnecessary wakeups |
761 | * for flushing the user TLB's associated with the active mm. | |
6110a1f4 | 762 | */ |
c8381cc3 | 763 | if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED) |
6110a1f4 SS |
764 | leave_mm(cpu); |
765 | ||
26717172 | 766 | if (!(lapic_timer_reliable_states & (1 << (cstate)))) |
f6cee191 | 767 | tick_broadcast_enter(); |
26717172 | 768 | |
16824255 | 769 | mwait_idle_with_hints(eax, ecx); |
26717172 | 770 | |
26717172 | 771 | if (!(lapic_timer_reliable_states & (1 << (cstate)))) |
f6cee191 | 772 | tick_broadcast_exit(); |
26717172 | 773 | |
e978aa7d | 774 | return index; |
26717172 LB |
775 | } |
776 | ||
5fe2e527 RW |
777 | /** |
778 | * intel_idle_freeze - simplified "enter" callback routine for suspend-to-idle | |
779 | * @dev: cpuidle_device | |
780 | * @drv: cpuidle driver | |
781 | * @index: state index | |
782 | */ | |
783 | static void intel_idle_freeze(struct cpuidle_device *dev, | |
784 | struct cpuidle_driver *drv, int index) | |
785 | { | |
786 | unsigned long ecx = 1; /* break on interrupt flag */ | |
787 | unsigned long eax = flg2MWAIT(drv->states[index].flags); | |
788 | ||
789 | mwait_idle_with_hints(eax, ecx); | |
790 | } | |
791 | ||
2a2d31c8 SL |
792 | static void __setup_broadcast_timer(void *arg) |
793 | { | |
76962caa | 794 | unsigned long on = (unsigned long)arg; |
2a2d31c8 | 795 | |
76962caa TG |
796 | if (on) |
797 | tick_broadcast_enable(); | |
798 | else | |
799 | tick_broadcast_disable(); | |
2a2d31c8 SL |
800 | } |
801 | ||
25ac7761 DL |
802 | static int cpu_hotplug_notify(struct notifier_block *n, |
803 | unsigned long action, void *hcpu) | |
2a2d31c8 SL |
804 | { |
805 | int hotcpu = (unsigned long)hcpu; | |
25ac7761 | 806 | struct cpuidle_device *dev; |
2a2d31c8 | 807 | |
e2401453 | 808 | switch (action & ~CPU_TASKS_FROZEN) { |
2a2d31c8 | 809 | case CPU_ONLINE: |
25ac7761 DL |
810 | |
811 | if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) | |
812 | smp_call_function_single(hotcpu, __setup_broadcast_timer, | |
813 | (void *)true, 1); | |
814 | ||
815 | /* | |
816 | * Some systems can hotplug a cpu at runtime after | |
817 | * the kernel has booted, we have to initialize the | |
818 | * driver in this case | |
819 | */ | |
820 | dev = per_cpu_ptr(intel_idle_cpuidle_devices, hotcpu); | |
08820546 RC |
821 | if (dev->registered) |
822 | break; | |
823 | ||
824 | if (intel_idle_cpu_init(hotcpu)) | |
825 | return NOTIFY_BAD; | |
25ac7761 | 826 | |
2a2d31c8 | 827 | break; |
2a2d31c8 SL |
828 | } |
829 | return NOTIFY_OK; | |
830 | } | |
831 | ||
25ac7761 DL |
832 | static struct notifier_block cpu_hotplug_notifier = { |
833 | .notifier_call = cpu_hotplug_notify, | |
2a2d31c8 SL |
834 | }; |
835 | ||
14796fca LB |
836 | static void auto_demotion_disable(void *dummy) |
837 | { | |
838 | unsigned long long msr_bits; | |
839 | ||
840 | rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits); | |
b66b8b9a | 841 | msr_bits &= ~(icpu->auto_demotion_disable_flags); |
14796fca LB |
842 | wrmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits); |
843 | } | |
32e95180 LB |
844 | static void c1e_promotion_disable(void *dummy) |
845 | { | |
846 | unsigned long long msr_bits; | |
847 | ||
848 | rdmsrl(MSR_IA32_POWER_CTL, msr_bits); | |
849 | msr_bits &= ~0x2; | |
850 | wrmsrl(MSR_IA32_POWER_CTL, msr_bits); | |
851 | } | |
14796fca | 852 | |
b66b8b9a AK |
853 | static const struct idle_cpu idle_cpu_nehalem = { |
854 | .state_table = nehalem_cstates, | |
b66b8b9a | 855 | .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, |
32e95180 | 856 | .disable_promotion_to_c1e = true, |
b66b8b9a AK |
857 | }; |
858 | ||
859 | static const struct idle_cpu idle_cpu_atom = { | |
860 | .state_table = atom_cstates, | |
861 | }; | |
862 | ||
863 | static const struct idle_cpu idle_cpu_lincroft = { | |
864 | .state_table = atom_cstates, | |
865 | .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, | |
866 | }; | |
867 | ||
868 | static const struct idle_cpu idle_cpu_snb = { | |
869 | .state_table = snb_cstates, | |
32e95180 | 870 | .disable_promotion_to_c1e = true, |
b66b8b9a AK |
871 | }; |
872 | ||
718987d6 LB |
873 | static const struct idle_cpu idle_cpu_byt = { |
874 | .state_table = byt_cstates, | |
875 | .disable_promotion_to_c1e = true, | |
8c058d53 | 876 | .byt_auto_demotion_disable_flag = true, |
718987d6 LB |
877 | }; |
878 | ||
cab07a56 LB |
879 | static const struct idle_cpu idle_cpu_cht = { |
880 | .state_table = cht_cstates, | |
881 | .disable_promotion_to_c1e = true, | |
882 | .byt_auto_demotion_disable_flag = true, | |
883 | }; | |
884 | ||
6edab08c LB |
885 | static const struct idle_cpu idle_cpu_ivb = { |
886 | .state_table = ivb_cstates, | |
32e95180 | 887 | .disable_promotion_to_c1e = true, |
6edab08c LB |
888 | }; |
889 | ||
0138d8f0 LB |
890 | static const struct idle_cpu idle_cpu_ivt = { |
891 | .state_table = ivt_cstates, | |
892 | .disable_promotion_to_c1e = true, | |
893 | }; | |
894 | ||
85a4d2d4 LB |
895 | static const struct idle_cpu idle_cpu_hsw = { |
896 | .state_table = hsw_cstates, | |
32e95180 | 897 | .disable_promotion_to_c1e = true, |
85a4d2d4 LB |
898 | }; |
899 | ||
a138b568 LB |
900 | static const struct idle_cpu idle_cpu_bdw = { |
901 | .state_table = bdw_cstates, | |
902 | .disable_promotion_to_c1e = true, | |
903 | }; | |
904 | ||
493f133f LB |
905 | static const struct idle_cpu idle_cpu_skl = { |
906 | .state_table = skl_cstates, | |
907 | .disable_promotion_to_c1e = true, | |
908 | }; | |
909 | ||
910 | ||
fab04b22 LB |
911 | static const struct idle_cpu idle_cpu_avn = { |
912 | .state_table = avn_cstates, | |
913 | .disable_promotion_to_c1e = true, | |
914 | }; | |
915 | ||
281baf7a DC |
916 | static const struct idle_cpu idle_cpu_knl = { |
917 | .state_table = knl_cstates, | |
918 | }; | |
919 | ||
b66b8b9a AK |
920 | #define ICPU(model, cpu) \ |
921 | { X86_VENDOR_INTEL, 6, model, X86_FEATURE_MWAIT, (unsigned long)&cpu } | |
922 | ||
d5cdc3c4 | 923 | static const struct x86_cpu_id intel_idle_ids[] __initconst = { |
b66b8b9a AK |
924 | ICPU(0x1a, idle_cpu_nehalem), |
925 | ICPU(0x1e, idle_cpu_nehalem), | |
926 | ICPU(0x1f, idle_cpu_nehalem), | |
8bf11938 BH |
927 | ICPU(0x25, idle_cpu_nehalem), |
928 | ICPU(0x2c, idle_cpu_nehalem), | |
929 | ICPU(0x2e, idle_cpu_nehalem), | |
b66b8b9a AK |
930 | ICPU(0x1c, idle_cpu_atom), |
931 | ICPU(0x26, idle_cpu_lincroft), | |
8bf11938 | 932 | ICPU(0x2f, idle_cpu_nehalem), |
b66b8b9a AK |
933 | ICPU(0x2a, idle_cpu_snb), |
934 | ICPU(0x2d, idle_cpu_snb), | |
acead1b0 | 935 | ICPU(0x36, idle_cpu_atom), |
718987d6 | 936 | ICPU(0x37, idle_cpu_byt), |
cab07a56 | 937 | ICPU(0x4c, idle_cpu_cht), |
6edab08c | 938 | ICPU(0x3a, idle_cpu_ivb), |
0138d8f0 | 939 | ICPU(0x3e, idle_cpu_ivt), |
85a4d2d4 LB |
940 | ICPU(0x3c, idle_cpu_hsw), |
941 | ICPU(0x3f, idle_cpu_hsw), | |
942 | ICPU(0x45, idle_cpu_hsw), | |
0b15841b | 943 | ICPU(0x46, idle_cpu_hsw), |
a138b568 LB |
944 | ICPU(0x4d, idle_cpu_avn), |
945 | ICPU(0x3d, idle_cpu_bdw), | |
bea57077 | 946 | ICPU(0x47, idle_cpu_bdw), |
a138b568 LB |
947 | ICPU(0x4f, idle_cpu_bdw), |
948 | ICPU(0x56, idle_cpu_bdw), | |
493f133f LB |
949 | ICPU(0x4e, idle_cpu_skl), |
950 | ICPU(0x5e, idle_cpu_skl), | |
281baf7a | 951 | ICPU(0x57, idle_cpu_knl), |
b66b8b9a AK |
952 | {} |
953 | }; | |
954 | MODULE_DEVICE_TABLE(x86cpu, intel_idle_ids); | |
955 | ||
26717172 LB |
956 | /* |
957 | * intel_idle_probe() | |
958 | */ | |
00f3e755 | 959 | static int __init intel_idle_probe(void) |
26717172 | 960 | { |
c4236282 | 961 | unsigned int eax, ebx, ecx; |
b66b8b9a | 962 | const struct x86_cpu_id *id; |
26717172 LB |
963 | |
964 | if (max_cstate == 0) { | |
965 | pr_debug(PREFIX "disabled\n"); | |
966 | return -EPERM; | |
967 | } | |
968 | ||
b66b8b9a AK |
969 | id = x86_match_cpu(intel_idle_ids); |
970 | if (!id) { | |
971 | if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && | |
972 | boot_cpu_data.x86 == 6) | |
973 | pr_debug(PREFIX "does not run on family %d model %d\n", | |
974 | boot_cpu_data.x86, boot_cpu_data.x86_model); | |
26717172 | 975 | return -ENODEV; |
b66b8b9a | 976 | } |
26717172 LB |
977 | |
978 | if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) | |
979 | return -ENODEV; | |
980 | ||
c4236282 | 981 | cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); |
26717172 LB |
982 | |
983 | if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || | |
5c2a9f06 TR |
984 | !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || |
985 | !mwait_substates) | |
26717172 | 986 | return -ENODEV; |
26717172 | 987 | |
c4236282 | 988 | pr_debug(PREFIX "MWAIT substates: 0x%x\n", mwait_substates); |
26717172 | 989 | |
b66b8b9a AK |
990 | icpu = (const struct idle_cpu *)id->driver_data; |
991 | cpuidle_state_table = icpu->state_table; | |
26717172 LB |
992 | |
993 | pr_debug(PREFIX "v" INTEL_IDLE_VERSION | |
994 | " model 0x%X\n", boot_cpu_data.x86_model); | |
995 | ||
26717172 LB |
996 | return 0; |
997 | } | |
998 | ||
999 | /* | |
1000 | * intel_idle_cpuidle_devices_uninit() | |
ca42489d | 1001 | * Unregisters the cpuidle devices. |
26717172 LB |
1002 | */ |
1003 | static void intel_idle_cpuidle_devices_uninit(void) | |
1004 | { | |
1005 | int i; | |
1006 | struct cpuidle_device *dev; | |
1007 | ||
1008 | for_each_online_cpu(i) { | |
1009 | dev = per_cpu_ptr(intel_idle_cpuidle_devices, i); | |
1010 | cpuidle_unregister_device(dev); | |
1011 | } | |
26717172 | 1012 | } |
0138d8f0 LB |
1013 | |
1014 | /* | |
d70e28f5 | 1015 | * ivt_idle_state_table_update(void) |
0138d8f0 | 1016 | * |
d70e28f5 | 1017 | * Tune IVT multi-socket targets |
0138d8f0 LB |
1018 | * Assumption: num_sockets == (max_package_num + 1) |
1019 | */ | |
d70e28f5 | 1020 | static void ivt_idle_state_table_update(void) |
0138d8f0 LB |
1021 | { |
1022 | /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ | |
d70e28f5 LB |
1023 | int cpu, package_num, num_sockets = 1; |
1024 | ||
1025 | for_each_online_cpu(cpu) { | |
1026 | package_num = topology_physical_package_id(cpu); | |
1027 | if (package_num + 1 > num_sockets) { | |
1028 | num_sockets = package_num + 1; | |
1029 | ||
1030 | if (num_sockets > 4) { | |
1031 | cpuidle_state_table = ivt_cstates_8s; | |
1032 | return; | |
0138d8f0 LB |
1033 | } |
1034 | } | |
d70e28f5 LB |
1035 | } |
1036 | ||
1037 | if (num_sockets > 2) | |
1038 | cpuidle_state_table = ivt_cstates_4s; | |
1039 | ||
1040 | /* else, 1 and 2 socket systems use default ivt_cstates */ | |
1041 | } | |
1042 | /* | |
1043 | * sklh_idle_state_table_update(void) | |
1044 | * | |
1045 | * On SKL-H (model 0x5e) disable C8 and C9 if: | |
1046 | * C10 is enabled and SGX disabled | |
1047 | */ | |
1048 | static void sklh_idle_state_table_update(void) | |
1049 | { | |
1050 | unsigned long long msr; | |
1051 | unsigned int eax, ebx, ecx, edx; | |
1052 | ||
1053 | ||
1054 | /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ | |
1055 | if (max_cstate <= 7) | |
1056 | return; | |
1057 | ||
1058 | /* if PC10 not present in CPUID.MWAIT.EDX */ | |
1059 | if ((mwait_substates & (0xF << 28)) == 0) | |
1060 | return; | |
1061 | ||
1062 | rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr); | |
1063 | ||
1064 | /* PC10 is not enabled in PKG C-state limit */ | |
1065 | if ((msr & 0xF) != 8) | |
1066 | return; | |
1067 | ||
1068 | ecx = 0; | |
1069 | cpuid(7, &eax, &ebx, &ecx, &edx); | |
1070 | ||
1071 | /* if SGX is present */ | |
1072 | if (ebx & (1 << 2)) { | |
0138d8f0 | 1073 | |
d70e28f5 LB |
1074 | rdmsrl(MSR_IA32_FEATURE_CONTROL, msr); |
1075 | ||
1076 | /* if SGX is enabled */ | |
1077 | if (msr & (1 << 18)) | |
1078 | return; | |
1079 | } | |
1080 | ||
1081 | skl_cstates[5].disabled = 1; /* C8-SKL */ | |
1082 | skl_cstates[6].disabled = 1; /* C9-SKL */ | |
1083 | } | |
1084 | /* | |
1085 | * intel_idle_state_table_update() | |
1086 | * | |
1087 | * Update the default state_table for this CPU-id | |
1088 | */ | |
1089 | ||
1090 | static void intel_idle_state_table_update(void) | |
1091 | { | |
1092 | switch (boot_cpu_data.x86_model) { | |
1093 | ||
1094 | case 0x3e: /* IVT */ | |
1095 | ivt_idle_state_table_update(); | |
1096 | break; | |
1097 | case 0x5e: /* SKL-H */ | |
1098 | sklh_idle_state_table_update(); | |
1099 | break; | |
0138d8f0 | 1100 | } |
0138d8f0 LB |
1101 | } |
1102 | ||
46bcfad7 DD |
1103 | /* |
1104 | * intel_idle_cpuidle_driver_init() | |
1105 | * allocate, initialize cpuidle_states | |
1106 | */ | |
5469c827 | 1107 | static void __init intel_idle_cpuidle_driver_init(void) |
46bcfad7 DD |
1108 | { |
1109 | int cstate; | |
1110 | struct cpuidle_driver *drv = &intel_idle_driver; | |
1111 | ||
0138d8f0 LB |
1112 | intel_idle_state_table_update(); |
1113 | ||
46bcfad7 DD |
1114 | drv->state_count = 1; |
1115 | ||
e022e7eb | 1116 | for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { |
24bfa950 | 1117 | int num_substates, mwait_hint, mwait_cstate; |
46bcfad7 | 1118 | |
7dd0e0af LB |
1119 | if ((cpuidle_state_table[cstate].enter == NULL) && |
1120 | (cpuidle_state_table[cstate].enter_freeze == NULL)) | |
e022e7eb LB |
1121 | break; |
1122 | ||
1123 | if (cstate + 1 > max_cstate) { | |
46bcfad7 DD |
1124 | printk(PREFIX "max_cstate %d reached\n", |
1125 | max_cstate); | |
1126 | break; | |
1127 | } | |
1128 | ||
e022e7eb LB |
1129 | mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); |
1130 | mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint); | |
e022e7eb | 1131 | |
24bfa950 | 1132 | /* number of sub-states for this state in CPUID.MWAIT */ |
e022e7eb | 1133 | num_substates = (mwait_substates >> ((mwait_cstate + 1) * 4)) |
46bcfad7 | 1134 | & MWAIT_SUBSTATE_MASK; |
e022e7eb | 1135 | |
24bfa950 LB |
1136 | /* if NO sub-states for this state in CPUID, skip it */ |
1137 | if (num_substates == 0) | |
46bcfad7 | 1138 | continue; |
46bcfad7 | 1139 | |
d70e28f5 LB |
1140 | /* if state marked as disabled, skip it */ |
1141 | if (cpuidle_state_table[cstate].disabled != 0) { | |
1142 | pr_debug(PREFIX "state %s is disabled", | |
1143 | cpuidle_state_table[cstate].name); | |
1144 | continue; | |
1145 | } | |
1146 | ||
1147 | ||
e022e7eb | 1148 | if (((mwait_cstate + 1) > 2) && |
46bcfad7 DD |
1149 | !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) |
1150 | mark_tsc_unstable("TSC halts in idle" | |
1151 | " states deeper than C2"); | |
1152 | ||
1153 | drv->states[drv->state_count] = /* structure copy */ | |
1154 | cpuidle_state_table[cstate]; | |
1155 | ||
1156 | drv->state_count += 1; | |
1157 | } | |
1158 | ||
8c058d53 LB |
1159 | if (icpu->byt_auto_demotion_disable_flag) { |
1160 | wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); | |
1161 | wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); | |
1162 | } | |
46bcfad7 DD |
1163 | } |
1164 | ||
1165 | ||
26717172 | 1166 | /* |
65b7f839 | 1167 | * intel_idle_cpu_init() |
26717172 | 1168 | * allocate, initialize, register cpuidle_devices |
65b7f839 | 1169 | * @cpu: cpu/core to initialize |
26717172 | 1170 | */ |
25ac7761 | 1171 | static int intel_idle_cpu_init(int cpu) |
26717172 | 1172 | { |
26717172 LB |
1173 | struct cpuidle_device *dev; |
1174 | ||
65b7f839 | 1175 | dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); |
26717172 | 1176 | |
65b7f839 | 1177 | dev->cpu = cpu; |
26717172 | 1178 | |
65b7f839 TR |
1179 | if (cpuidle_register_device(dev)) { |
1180 | pr_debug(PREFIX "cpuidle_register_device %d failed!\n", cpu); | |
65b7f839 | 1181 | return -EIO; |
26717172 LB |
1182 | } |
1183 | ||
b66b8b9a | 1184 | if (icpu->auto_demotion_disable_flags) |
65b7f839 TR |
1185 | smp_call_function_single(cpu, auto_demotion_disable, NULL, 1); |
1186 | ||
dbf87ab8 BZ |
1187 | if (icpu->disable_promotion_to_c1e) |
1188 | smp_call_function_single(cpu, c1e_promotion_disable, NULL, 1); | |
1189 | ||
26717172 LB |
1190 | return 0; |
1191 | } | |
26717172 LB |
1192 | |
1193 | static int __init intel_idle_init(void) | |
1194 | { | |
65b7f839 | 1195 | int retval, i; |
26717172 | 1196 | |
d1896049 TR |
1197 | /* Do not load intel_idle at all for now if idle= is passed */ |
1198 | if (boot_option_idle_override != IDLE_NO_OVERRIDE) | |
1199 | return -ENODEV; | |
1200 | ||
26717172 LB |
1201 | retval = intel_idle_probe(); |
1202 | if (retval) | |
1203 | return retval; | |
1204 | ||
e9df69cc RC |
1205 | intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); |
1206 | if (intel_idle_cpuidle_devices == NULL) | |
1207 | return -ENOMEM; | |
1208 | ||
46bcfad7 | 1209 | intel_idle_cpuidle_driver_init(); |
26717172 LB |
1210 | retval = cpuidle_register_driver(&intel_idle_driver); |
1211 | if (retval) { | |
3735d524 | 1212 | struct cpuidle_driver *drv = cpuidle_get_driver(); |
26717172 | 1213 | printk(KERN_DEBUG PREFIX "intel_idle yielding to %s", |
3735d524 | 1214 | drv ? drv->name : "none"); |
e9df69cc | 1215 | free_percpu(intel_idle_cpuidle_devices); |
26717172 LB |
1216 | return retval; |
1217 | } | |
1218 | ||
07494d54 SB |
1219 | cpu_notifier_register_begin(); |
1220 | ||
65b7f839 TR |
1221 | for_each_online_cpu(i) { |
1222 | retval = intel_idle_cpu_init(i); | |
1223 | if (retval) { | |
b69ef2c0 | 1224 | intel_idle_cpuidle_devices_uninit(); |
07494d54 | 1225 | cpu_notifier_register_done(); |
65b7f839 | 1226 | cpuidle_unregister_driver(&intel_idle_driver); |
ca42489d | 1227 | free_percpu(intel_idle_cpuidle_devices); |
65b7f839 TR |
1228 | return retval; |
1229 | } | |
26717172 | 1230 | } |
07494d54 SB |
1231 | __register_cpu_notifier(&cpu_hotplug_notifier); |
1232 | ||
2259a819 RC |
1233 | if (boot_cpu_has(X86_FEATURE_ARAT)) /* Always Reliable APIC Timer */ |
1234 | lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE; | |
1235 | else | |
1236 | on_each_cpu(__setup_broadcast_timer, (void *)true, 1); | |
1237 | ||
07494d54 | 1238 | cpu_notifier_register_done(); |
26717172 | 1239 | |
2259a819 RC |
1240 | pr_debug(PREFIX "lapic_timer_reliable_states 0x%x\n", |
1241 | lapic_timer_reliable_states); | |
1242 | ||
26717172 LB |
1243 | return 0; |
1244 | } | |
1245 | ||
1246 | static void __exit intel_idle_exit(void) | |
1247 | { | |
3e66a9ab RC |
1248 | struct cpuidle_device *dev; |
1249 | int i; | |
1250 | ||
07494d54 | 1251 | cpu_notifier_register_begin(); |
25ac7761 DL |
1252 | |
1253 | if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) | |
39a74fde | 1254 | on_each_cpu(__setup_broadcast_timer, (void *)false, 1); |
07494d54 | 1255 | __unregister_cpu_notifier(&cpu_hotplug_notifier); |
3e66a9ab RC |
1256 | |
1257 | for_each_possible_cpu(i) { | |
1258 | dev = per_cpu_ptr(intel_idle_cpuidle_devices, i); | |
1259 | cpuidle_unregister_device(dev); | |
1260 | } | |
07494d54 SB |
1261 | |
1262 | cpu_notifier_register_done(); | |
51319918 RC |
1263 | |
1264 | cpuidle_unregister_driver(&intel_idle_driver); | |
ca42489d | 1265 | free_percpu(intel_idle_cpuidle_devices); |
26717172 LB |
1266 | } |
1267 | ||
1268 | module_init(intel_idle_init); | |
1269 | module_exit(intel_idle_exit); | |
1270 | ||
26717172 | 1271 | module_param(max_cstate, int, 0444); |
26717172 LB |
1272 | |
1273 | MODULE_AUTHOR("Len Brown <len.brown@intel.com>"); | |
1274 | MODULE_DESCRIPTION("Cpuidle driver for Intel Hardware v" INTEL_IDLE_VERSION); | |
1275 | MODULE_LICENSE("GPL"); |