]>
Commit | Line | Data |
---|---|---|
d405a98c SP |
1 | /* |
2 | * PowerNV cpuidle code | |
3 | * | |
4 | * Copyright 2015 IBM Corp. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or | |
7 | * modify it under the terms of the GNU General Public License | |
8 | * as published by the Free Software Foundation; either version | |
9 | * 2 of the License, or (at your option) any later version. | |
10 | */ | |
11 | ||
12 | #include <linux/types.h> | |
13 | #include <linux/mm.h> | |
14 | #include <linux/slab.h> | |
15 | #include <linux/of.h> | |
5703d2f4 SP |
16 | #include <linux/device.h> |
17 | #include <linux/cpu.h> | |
d405a98c SP |
18 | |
19 | #include <asm/firmware.h> | |
4bece972 | 20 | #include <asm/machdep.h> |
d405a98c SP |
21 | #include <asm/opal.h> |
22 | #include <asm/cputhreads.h> | |
23 | #include <asm/cpuidle.h> | |
24 | #include <asm/code-patching.h> | |
25 | #include <asm/smp.h> | |
2201f994 | 26 | #include <asm/runlatch.h> |
d405a98c SP |
27 | |
28 | #include "powernv.h" | |
29 | #include "subcore.h" | |
30 | ||
bcef83a0 SP |
31 | /* Power ISA 3.0 allows for stop states 0x0 - 0xF */ |
32 | #define MAX_STOP_STATE 0xF | |
33 | ||
1e1601b3 AA |
34 | #define P9_STOP_SPR_MSR 2000 |
35 | #define P9_STOP_SPR_PSSCR 855 | |
36 | ||
d405a98c SP |
37 | static u32 supported_cpuidle_states; |
38 | ||
1e1601b3 AA |
39 | /* |
40 | * The default stop state that will be used by ppc_md.power_save | |
41 | * function on platforms that support stop instruction. | |
42 | */ | |
43 | static u64 pnv_default_stop_val; | |
44 | static u64 pnv_default_stop_mask; | |
45 | static bool default_stop_found; | |
46 | ||
47 | /* | |
48 | * First deep stop state. Used to figure out when to save/restore | |
49 | * hypervisor context. | |
50 | */ | |
51 | u64 pnv_first_deep_stop_state = MAX_STOP_STATE; | |
52 | ||
53 | /* | |
54 | * psscr value and mask of the deepest stop idle state. | |
55 | * Used when a cpu is offlined. | |
56 | */ | |
57 | static u64 pnv_deepest_stop_psscr_val; | |
58 | static u64 pnv_deepest_stop_psscr_mask; | |
785a12af | 59 | static u64 pnv_deepest_stop_flag; |
1e1601b3 AA |
60 | static bool deepest_stop_found; |
61 | ||
bcef83a0 | 62 | static int pnv_save_sprs_for_deep_states(void) |
d405a98c SP |
63 | { |
64 | int cpu; | |
65 | int rc; | |
66 | ||
67 | /* | |
446957ba | 68 | * hid0, hid1, hid4, hid5, hmeer and lpcr values are symmetric across |
d405a98c | 69 | * all cpus at boot. Get these reg values of current cpu and use the |
446957ba | 70 | * same across all cpus. |
d405a98c | 71 | */ |
24be85a2 | 72 | uint64_t lpcr_val = mfspr(SPRN_LPCR); |
d405a98c SP |
73 | uint64_t hid0_val = mfspr(SPRN_HID0); |
74 | uint64_t hid1_val = mfspr(SPRN_HID1); | |
75 | uint64_t hid4_val = mfspr(SPRN_HID4); | |
76 | uint64_t hid5_val = mfspr(SPRN_HID5); | |
77 | uint64_t hmeer_val = mfspr(SPRN_HMEER); | |
1e1601b3 AA |
78 | uint64_t msr_val = MSR_IDLE; |
79 | uint64_t psscr_val = pnv_deepest_stop_psscr_val; | |
d405a98c SP |
80 | |
81 | for_each_possible_cpu(cpu) { | |
82 | uint64_t pir = get_hard_smp_processor_id(cpu); | |
83 | uint64_t hsprg0_val = (uint64_t)&paca[cpu]; | |
84 | ||
d405a98c SP |
85 | rc = opal_slw_set_reg(pir, SPRN_HSPRG0, hsprg0_val); |
86 | if (rc != 0) | |
87 | return rc; | |
88 | ||
89 | rc = opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val); | |
90 | if (rc != 0) | |
91 | return rc; | |
92 | ||
1e1601b3 AA |
93 | if (cpu_has_feature(CPU_FTR_ARCH_300)) { |
94 | rc = opal_slw_set_reg(pir, P9_STOP_SPR_MSR, msr_val); | |
95 | if (rc) | |
96 | return rc; | |
97 | ||
98 | rc = opal_slw_set_reg(pir, | |
99 | P9_STOP_SPR_PSSCR, psscr_val); | |
100 | ||
101 | if (rc) | |
102 | return rc; | |
103 | } | |
104 | ||
d405a98c SP |
105 | /* HIDs are per core registers */ |
106 | if (cpu_thread_in_core(cpu) == 0) { | |
107 | ||
108 | rc = opal_slw_set_reg(pir, SPRN_HMEER, hmeer_val); | |
109 | if (rc != 0) | |
110 | return rc; | |
111 | ||
112 | rc = opal_slw_set_reg(pir, SPRN_HID0, hid0_val); | |
113 | if (rc != 0) | |
114 | return rc; | |
115 | ||
1e1601b3 AA |
116 | /* Only p8 needs to set extra HID regiters */ |
117 | if (!cpu_has_feature(CPU_FTR_ARCH_300)) { | |
d405a98c | 118 | |
1e1601b3 AA |
119 | rc = opal_slw_set_reg(pir, SPRN_HID1, hid1_val); |
120 | if (rc != 0) | |
121 | return rc; | |
d405a98c | 122 | |
1e1601b3 AA |
123 | rc = opal_slw_set_reg(pir, SPRN_HID4, hid4_val); |
124 | if (rc != 0) | |
125 | return rc; | |
126 | ||
127 | rc = opal_slw_set_reg(pir, SPRN_HID5, hid5_val); | |
128 | if (rc != 0) | |
129 | return rc; | |
130 | } | |
d405a98c SP |
131 | } |
132 | } | |
133 | ||
134 | return 0; | |
135 | } | |
136 | ||
137 | static void pnv_alloc_idle_core_states(void) | |
138 | { | |
139 | int i, j; | |
140 | int nr_cores = cpu_nr_cores(); | |
141 | u32 *core_idle_state; | |
142 | ||
143 | /* | |
5f221c3c GS |
144 | * core_idle_state - The lower 8 bits track the idle state of |
145 | * each thread of the core. | |
146 | * | |
147 | * The most significant bit is the lock bit. | |
148 | * | |
149 | * Initially all the bits corresponding to threads_per_core | |
150 | * are set. They are cleared when the thread enters deep idle | |
151 | * state like sleep and winkle/stop. | |
152 | * | |
153 | * Initially the lock bit is cleared. The lock bit has 2 | |
154 | * purposes: | |
155 | * a. While the first thread in the core waking up from | |
156 | * idle is restoring core state, it prevents other | |
157 | * threads in the core from switching to process | |
158 | * context. | |
159 | * b. While the last thread in the core is saving the | |
160 | * core state, it prevents a different thread from | |
161 | * waking up. | |
d405a98c SP |
162 | */ |
163 | for (i = 0; i < nr_cores; i++) { | |
164 | int first_cpu = i * threads_per_core; | |
165 | int node = cpu_to_node(first_cpu); | |
17ed4c8f | 166 | size_t paca_ptr_array_size; |
d405a98c SP |
167 | |
168 | core_idle_state = kmalloc_node(sizeof(u32), GFP_KERNEL, node); | |
5f221c3c | 169 | *core_idle_state = (1 << threads_per_core) - 1; |
17ed4c8f GS |
170 | paca_ptr_array_size = (threads_per_core * |
171 | sizeof(struct paca_struct *)); | |
d405a98c SP |
172 | |
173 | for (j = 0; j < threads_per_core; j++) { | |
174 | int cpu = first_cpu + j; | |
175 | ||
176 | paca[cpu].core_idle_state_ptr = core_idle_state; | |
177 | paca[cpu].thread_idle_state = PNV_THREAD_RUNNING; | |
178 | paca[cpu].thread_mask = 1 << j; | |
17ed4c8f GS |
179 | if (!cpu_has_feature(CPU_FTR_POWER9_DD1)) |
180 | continue; | |
181 | paca[cpu].thread_sibling_pacas = | |
182 | kmalloc_node(paca_ptr_array_size, | |
183 | GFP_KERNEL, node); | |
d405a98c SP |
184 | } |
185 | } | |
186 | ||
187 | update_subcore_sibling_mask(); | |
188 | ||
785a12af GS |
189 | if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT) { |
190 | int rc = pnv_save_sprs_for_deep_states(); | |
191 | ||
192 | if (likely(!rc)) | |
193 | return; | |
194 | ||
195 | /* | |
196 | * The stop-api is unable to restore hypervisor | |
197 | * resources on wakeup from platform idle states which | |
198 | * lose full context. So disable such states. | |
199 | */ | |
200 | supported_cpuidle_states &= ~OPAL_PM_LOSE_FULL_CONTEXT; | |
201 | pr_warn("cpuidle-powernv: Disabling idle states that lose full context\n"); | |
202 | pr_warn("cpuidle-powernv: Idle power-savings, CPU-Hotplug affected\n"); | |
203 | ||
204 | if (cpu_has_feature(CPU_FTR_ARCH_300) && | |
205 | (pnv_deepest_stop_flag & OPAL_PM_LOSE_FULL_CONTEXT)) { | |
206 | /* | |
207 | * Use the default stop state for CPU-Hotplug | |
208 | * if available. | |
209 | */ | |
210 | if (default_stop_found) { | |
211 | pnv_deepest_stop_psscr_val = | |
212 | pnv_default_stop_val; | |
213 | pnv_deepest_stop_psscr_mask = | |
214 | pnv_default_stop_mask; | |
215 | pr_warn("cpuidle-powernv: Offlined CPUs will stop with psscr = 0x%016llx\n", | |
216 | pnv_deepest_stop_psscr_val); | |
217 | } else { /* Fallback to snooze loop for CPU-Hotplug */ | |
218 | deepest_stop_found = false; | |
219 | pr_warn("cpuidle-powernv: Offlined CPUs will busy wait\n"); | |
220 | } | |
221 | } | |
222 | } | |
d405a98c SP |
223 | } |
224 | ||
225 | u32 pnv_get_supported_cpuidle_states(void) | |
226 | { | |
227 | return supported_cpuidle_states; | |
228 | } | |
229 | EXPORT_SYMBOL_GPL(pnv_get_supported_cpuidle_states); | |
230 | ||
5703d2f4 SP |
231 | static void pnv_fastsleep_workaround_apply(void *info) |
232 | ||
233 | { | |
234 | int rc; | |
235 | int *err = info; | |
236 | ||
237 | rc = opal_config_cpu_idle_state(OPAL_CONFIG_IDLE_FASTSLEEP, | |
238 | OPAL_CONFIG_IDLE_APPLY); | |
239 | if (rc) | |
240 | *err = 1; | |
241 | } | |
242 | ||
243 | /* | |
244 | * Used to store fastsleep workaround state | |
245 | * 0 - Workaround applied/undone at fastsleep entry/exit path (Default) | |
246 | * 1 - Workaround applied once, never undone. | |
247 | */ | |
248 | static u8 fastsleep_workaround_applyonce; | |
249 | ||
250 | static ssize_t show_fastsleep_workaround_applyonce(struct device *dev, | |
251 | struct device_attribute *attr, char *buf) | |
252 | { | |
253 | return sprintf(buf, "%u\n", fastsleep_workaround_applyonce); | |
254 | } | |
255 | ||
256 | static ssize_t store_fastsleep_workaround_applyonce(struct device *dev, | |
257 | struct device_attribute *attr, const char *buf, | |
258 | size_t count) | |
259 | { | |
260 | cpumask_t primary_thread_mask; | |
261 | int err; | |
262 | u8 val; | |
263 | ||
264 | if (kstrtou8(buf, 0, &val) || val != 1) | |
265 | return -EINVAL; | |
266 | ||
267 | if (fastsleep_workaround_applyonce == 1) | |
268 | return count; | |
269 | ||
270 | /* | |
271 | * fastsleep_workaround_applyonce = 1 implies | |
272 | * fastsleep workaround needs to be left in 'applied' state on all | |
273 | * the cores. Do this by- | |
274 | * 1. Patching out the call to 'undo' workaround in fastsleep exit path | |
446957ba | 275 | * 2. Sending ipi to all the cores which have at least one online thread |
5703d2f4 SP |
276 | * 3. Patching out the call to 'apply' workaround in fastsleep entry |
277 | * path | |
278 | * There is no need to send ipi to cores which have all threads | |
279 | * offlined, as last thread of the core entering fastsleep or deeper | |
280 | * state would have applied workaround. | |
281 | */ | |
282 | err = patch_instruction( | |
283 | (unsigned int *)pnv_fastsleep_workaround_at_exit, | |
284 | PPC_INST_NOP); | |
285 | if (err) { | |
286 | pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_exit"); | |
287 | goto fail; | |
288 | } | |
289 | ||
290 | get_online_cpus(); | |
291 | primary_thread_mask = cpu_online_cores_map(); | |
292 | on_each_cpu_mask(&primary_thread_mask, | |
293 | pnv_fastsleep_workaround_apply, | |
294 | &err, 1); | |
295 | put_online_cpus(); | |
296 | if (err) { | |
297 | pr_err("fastsleep_workaround_applyonce change failed while running pnv_fastsleep_workaround_apply"); | |
298 | goto fail; | |
299 | } | |
300 | ||
301 | err = patch_instruction( | |
302 | (unsigned int *)pnv_fastsleep_workaround_at_entry, | |
303 | PPC_INST_NOP); | |
304 | if (err) { | |
305 | pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_entry"); | |
306 | goto fail; | |
307 | } | |
308 | ||
309 | fastsleep_workaround_applyonce = 1; | |
310 | ||
311 | return count; | |
312 | fail: | |
313 | return -EIO; | |
314 | } | |
315 | ||
316 | static DEVICE_ATTR(fastsleep_workaround_applyonce, 0600, | |
317 | show_fastsleep_workaround_applyonce, | |
318 | store_fastsleep_workaround_applyonce); | |
319 | ||
2201f994 NP |
320 | static unsigned long __power7_idle_type(unsigned long type) |
321 | { | |
322 | unsigned long srr1; | |
323 | ||
324 | if (!prep_irq_for_idle_irqsoff()) | |
325 | return 0; | |
326 | ||
40d24343 | 327 | __ppc64_runlatch_off(); |
2201f994 | 328 | srr1 = power7_idle_insn(type); |
40d24343 | 329 | __ppc64_runlatch_on(); |
2201f994 NP |
330 | |
331 | fini_irq_for_idle_irqsoff(); | |
332 | ||
333 | return srr1; | |
334 | } | |
335 | ||
336 | void power7_idle_type(unsigned long type) | |
337 | { | |
771d4304 NP |
338 | unsigned long srr1; |
339 | ||
340 | srr1 = __power7_idle_type(type); | |
341 | irq_set_pending_from_srr1(srr1); | |
2201f994 NP |
342 | } |
343 | ||
344 | void power7_idle(void) | |
345 | { | |
346 | if (!powersave_nap) | |
347 | return; | |
348 | ||
349 | power7_idle_type(PNV_THREAD_NAP); | |
350 | } | |
351 | ||
352 | static unsigned long __power9_idle_type(unsigned long stop_psscr_val, | |
353 | unsigned long stop_psscr_mask) | |
354 | { | |
355 | unsigned long psscr; | |
356 | unsigned long srr1; | |
357 | ||
358 | if (!prep_irq_for_idle_irqsoff()) | |
359 | return 0; | |
360 | ||
361 | psscr = mfspr(SPRN_PSSCR); | |
362 | psscr = (psscr & ~stop_psscr_mask) | stop_psscr_val; | |
363 | ||
40d24343 | 364 | __ppc64_runlatch_off(); |
2201f994 | 365 | srr1 = power9_idle_stop(psscr); |
40d24343 | 366 | __ppc64_runlatch_on(); |
2201f994 NP |
367 | |
368 | fini_irq_for_idle_irqsoff(); | |
369 | ||
370 | return srr1; | |
371 | } | |
372 | ||
373 | void power9_idle_type(unsigned long stop_psscr_val, | |
374 | unsigned long stop_psscr_mask) | |
375 | { | |
771d4304 NP |
376 | unsigned long srr1; |
377 | ||
378 | srr1 = __power9_idle_type(stop_psscr_val, stop_psscr_mask); | |
379 | irq_set_pending_from_srr1(srr1); | |
2201f994 NP |
380 | } |
381 | ||
bcef83a0 SP |
382 | /* |
383 | * Used for ppc_md.power_save which needs a function with no parameters | |
384 | */ | |
2201f994 | 385 | void power9_idle(void) |
d405a98c | 386 | { |
2201f994 | 387 | power9_idle_type(pnv_default_stop_val, pnv_default_stop_mask); |
bcef83a0 | 388 | } |
09206b60 | 389 | |
67d20418 | 390 | #ifdef CONFIG_HOTPLUG_CPU |
24be85a2 GS |
391 | static void pnv_program_cpu_hotplug_lpcr(unsigned int cpu, u64 lpcr_val) |
392 | { | |
393 | u64 pir = get_hard_smp_processor_id(cpu); | |
394 | ||
395 | mtspr(SPRN_LPCR, lpcr_val); | |
5d298baa GS |
396 | |
397 | /* | |
398 | * Program the LPCR via stop-api only if the deepest stop state | |
399 | * can lose hypervisor context. | |
400 | */ | |
401 | if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT) | |
402 | opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val); | |
24be85a2 GS |
403 | } |
404 | ||
a7cd88da GS |
405 | /* |
406 | * pnv_cpu_offline: A function that puts the CPU into the deepest | |
407 | * available platform idle state on a CPU-Offline. | |
2525db04 | 408 | * interrupts hard disabled and no lazy irq pending. |
a7cd88da GS |
409 | */ |
410 | unsigned long pnv_cpu_offline(unsigned int cpu) | |
411 | { | |
412 | unsigned long srr1; | |
a7cd88da | 413 | u32 idle_states = pnv_get_supported_cpuidle_states(); |
24be85a2 GS |
414 | u64 lpcr_val; |
415 | ||
416 | /* | |
417 | * We don't want to take decrementer interrupts while we are | |
418 | * offline, so clear LPCR:PECE1. We keep PECE2 (and | |
419 | * LPCR_PECE_HVEE on P9) enabled as to let IPIs in. | |
420 | * | |
421 | * If the CPU gets woken up by a special wakeup, ensure that | |
422 | * the SLW engine sets LPCR with decrementer bit cleared, else | |
423 | * the CPU will come back to the kernel due to a spurious | |
424 | * wakeup. | |
425 | */ | |
426 | lpcr_val = mfspr(SPRN_LPCR) & ~(u64)LPCR_PECE1; | |
427 | pnv_program_cpu_hotplug_lpcr(cpu, lpcr_val); | |
a7cd88da | 428 | |
40d24343 | 429 | __ppc64_runlatch_off(); |
2525db04 | 430 | |
f3b3f284 | 431 | if (cpu_has_feature(CPU_FTR_ARCH_300) && deepest_stop_found) { |
2525db04 NP |
432 | unsigned long psscr; |
433 | ||
434 | psscr = mfspr(SPRN_PSSCR); | |
435 | psscr = (psscr & ~pnv_deepest_stop_psscr_mask) | | |
436 | pnv_deepest_stop_psscr_val; | |
437 | srr1 = power9_idle_stop(psscr); | |
438 | ||
785a12af GS |
439 | } else if ((idle_states & OPAL_PM_WINKLE_ENABLED) && |
440 | (idle_states & OPAL_PM_LOSE_FULL_CONTEXT)) { | |
2525db04 | 441 | srr1 = power7_idle_insn(PNV_THREAD_WINKLE); |
a7cd88da GS |
442 | } else if ((idle_states & OPAL_PM_SLEEP_ENABLED) || |
443 | (idle_states & OPAL_PM_SLEEP_ENABLED_ER1)) { | |
2525db04 | 444 | srr1 = power7_idle_insn(PNV_THREAD_SLEEP); |
90061231 | 445 | } else if (idle_states & OPAL_PM_NAP_ENABLED) { |
2525db04 | 446 | srr1 = power7_idle_insn(PNV_THREAD_NAP); |
90061231 GS |
447 | } else { |
448 | /* This is the fallback method. We emulate snooze */ | |
449 | while (!generic_check_cpu_restart(cpu)) { | |
450 | HMT_low(); | |
451 | HMT_very_low(); | |
452 | } | |
453 | srr1 = 0; | |
454 | HMT_medium(); | |
a7cd88da GS |
455 | } |
456 | ||
40d24343 | 457 | __ppc64_runlatch_on(); |
2525db04 | 458 | |
24be85a2 GS |
459 | /* |
460 | * Re-enable decrementer interrupts in LPCR. | |
461 | * | |
462 | * Further, we want stop states to be woken up by decrementer | |
463 | * for non-hotplug cases. So program the LPCR via stop api as | |
464 | * well. | |
465 | */ | |
466 | lpcr_val = mfspr(SPRN_LPCR) | (u64)LPCR_PECE1; | |
467 | pnv_program_cpu_hotplug_lpcr(cpu, lpcr_val); | |
468 | ||
a7cd88da GS |
469 | return srr1; |
470 | } | |
67d20418 | 471 | #endif |
a7cd88da | 472 | |
bcef83a0 SP |
473 | /* |
474 | * Power ISA 3.0 idle initialization. | |
475 | * | |
476 | * POWER ISA 3.0 defines a new SPR Processor stop Status and Control | |
477 | * Register (PSSCR) to control idle behavior. | |
478 | * | |
479 | * PSSCR layout: | |
480 | * ---------------------------------------------------------- | |
481 | * | PLS | /// | SD | ESL | EC | PSLL | /// | TR | MTL | RL | | |
482 | * ---------------------------------------------------------- | |
483 | * 0 4 41 42 43 44 48 54 56 60 | |
484 | * | |
485 | * PSSCR key fields: | |
486 | * Bits 0:3 - Power-Saving Level Status (PLS). This field indicates the | |
487 | * lowest power-saving state the thread entered since stop instruction was | |
488 | * last executed. | |
489 | * | |
490 | * Bit 41 - Status Disable(SD) | |
491 | * 0 - Shows PLS entries | |
492 | * 1 - PLS entries are all 0 | |
493 | * | |
494 | * Bit 42 - Enable State Loss | |
495 | * 0 - No state is lost irrespective of other fields | |
496 | * 1 - Allows state loss | |
497 | * | |
498 | * Bit 43 - Exit Criterion | |
499 | * 0 - Exit from power-save mode on any interrupt | |
500 | * 1 - Exit from power-save mode controlled by LPCR's PECE bits | |
501 | * | |
502 | * Bits 44:47 - Power-Saving Level Limit | |
503 | * This limits the power-saving level that can be entered into. | |
504 | * | |
505 | * Bits 60:63 - Requested Level | |
506 | * Used to specify which power-saving level must be entered on executing | |
507 | * stop instruction | |
09206b60 GS |
508 | */ |
509 | ||
510 | int validate_psscr_val_mask(u64 *psscr_val, u64 *psscr_mask, u32 flags) | |
511 | { | |
512 | int err = 0; | |
513 | ||
514 | /* | |
515 | * psscr_mask == 0xf indicates an older firmware. | |
516 | * Set remaining fields of psscr to the default values. | |
517 | * See NOTE above definition of PSSCR_HV_DEFAULT_VAL | |
518 | */ | |
519 | if (*psscr_mask == 0xf) { | |
520 | *psscr_val = *psscr_val | PSSCR_HV_DEFAULT_VAL; | |
521 | *psscr_mask = PSSCR_HV_DEFAULT_MASK; | |
522 | return err; | |
523 | } | |
524 | ||
525 | /* | |
526 | * New firmware is expected to set the psscr_val bits correctly. | |
527 | * Validate that the following invariants are correctly maintained by | |
528 | * the new firmware. | |
529 | * - ESL bit value matches the EC bit value. | |
530 | * - ESL bit is set for all the deep stop states. | |
531 | */ | |
532 | if (GET_PSSCR_ESL(*psscr_val) != GET_PSSCR_EC(*psscr_val)) { | |
533 | err = ERR_EC_ESL_MISMATCH; | |
534 | } else if ((flags & OPAL_PM_LOSE_FULL_CONTEXT) && | |
535 | GET_PSSCR_ESL(*psscr_val) == 0) { | |
536 | err = ERR_DEEP_STATE_ESL_MISMATCH; | |
537 | } | |
538 | ||
539 | return err; | |
540 | } | |
541 | ||
542 | /* | |
543 | * pnv_arch300_idle_init: Initializes the default idle state, first | |
544 | * deep idle state and deepest idle state on | |
545 | * ISA 3.0 CPUs. | |
bcef83a0 SP |
546 | * |
547 | * @np: /ibm,opal/power-mgt device node | |
548 | * @flags: cpu-idle-state-flags array | |
549 | * @dt_idle_states: Number of idle state entries | |
550 | * Returns 0 on success | |
551 | */ | |
dd34c74c | 552 | static int __init pnv_power9_idle_init(struct device_node *np, u32 *flags, |
bcef83a0 SP |
553 | int dt_idle_states) |
554 | { | |
555 | u64 *psscr_val = NULL; | |
09206b60 GS |
556 | u64 *psscr_mask = NULL; |
557 | u32 *residency_ns = NULL; | |
558 | u64 max_residency_ns = 0; | |
bcef83a0 | 559 | int rc = 0, i; |
d405a98c | 560 | |
09206b60 GS |
561 | psscr_val = kcalloc(dt_idle_states, sizeof(*psscr_val), GFP_KERNEL); |
562 | psscr_mask = kcalloc(dt_idle_states, sizeof(*psscr_mask), GFP_KERNEL); | |
563 | residency_ns = kcalloc(dt_idle_states, sizeof(*residency_ns), | |
564 | GFP_KERNEL); | |
565 | ||
566 | if (!psscr_val || !psscr_mask || !residency_ns) { | |
bcef83a0 | 567 | rc = -1; |
d405a98c | 568 | goto out; |
bcef83a0 | 569 | } |
09206b60 | 570 | |
bcef83a0 SP |
571 | if (of_property_read_u64_array(np, |
572 | "ibm,cpu-idle-state-psscr", | |
573 | psscr_val, dt_idle_states)) { | |
09206b60 GS |
574 | pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr in DT\n"); |
575 | rc = -1; | |
576 | goto out; | |
577 | } | |
578 | ||
579 | if (of_property_read_u64_array(np, | |
580 | "ibm,cpu-idle-state-psscr-mask", | |
581 | psscr_mask, dt_idle_states)) { | |
582 | pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr-mask in DT\n"); | |
583 | rc = -1; | |
584 | goto out; | |
585 | } | |
586 | ||
587 | if (of_property_read_u32_array(np, | |
588 | "ibm,cpu-idle-state-residency-ns", | |
589 | residency_ns, dt_idle_states)) { | |
590 | pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-residency-ns in DT\n"); | |
bcef83a0 | 591 | rc = -1; |
d405a98c | 592 | goto out; |
bcef83a0 | 593 | } |
d405a98c | 594 | |
bcef83a0 | 595 | /* |
09206b60 GS |
596 | * Set pnv_first_deep_stop_state, pnv_deepest_stop_psscr_{val,mask}, |
597 | * and the pnv_default_stop_{val,mask}. | |
598 | * | |
c0691f9d SP |
599 | * pnv_first_deep_stop_state should be set to the first stop |
600 | * level to cause hypervisor state loss. | |
09206b60 GS |
601 | * |
602 | * pnv_deepest_stop_{val,mask} should be set to values corresponding to | |
603 | * the deepest stop state. | |
604 | * | |
605 | * pnv_default_stop_{val,mask} should be set to values corresponding to | |
606 | * the shallowest (OPAL_PM_STOP_INST_FAST) loss-less stop state. | |
bcef83a0 SP |
607 | */ |
608 | pnv_first_deep_stop_state = MAX_STOP_STATE; | |
609 | for (i = 0; i < dt_idle_states; i++) { | |
09206b60 | 610 | int err; |
bcef83a0 SP |
611 | u64 psscr_rl = psscr_val[i] & PSSCR_RL_MASK; |
612 | ||
613 | if ((flags[i] & OPAL_PM_LOSE_FULL_CONTEXT) && | |
614 | (pnv_first_deep_stop_state > psscr_rl)) | |
615 | pnv_first_deep_stop_state = psscr_rl; | |
c0691f9d | 616 | |
09206b60 GS |
617 | err = validate_psscr_val_mask(&psscr_val[i], &psscr_mask[i], |
618 | flags[i]); | |
619 | if (err) { | |
620 | report_invalid_psscr_val(psscr_val[i], err); | |
621 | continue; | |
622 | } | |
623 | ||
624 | if (max_residency_ns < residency_ns[i]) { | |
625 | max_residency_ns = residency_ns[i]; | |
626 | pnv_deepest_stop_psscr_val = psscr_val[i]; | |
627 | pnv_deepest_stop_psscr_mask = psscr_mask[i]; | |
785a12af | 628 | pnv_deepest_stop_flag = flags[i]; |
09206b60 GS |
629 | deepest_stop_found = true; |
630 | } | |
631 | ||
632 | if (!default_stop_found && | |
633 | (flags[i] & OPAL_PM_STOP_INST_FAST)) { | |
634 | pnv_default_stop_val = psscr_val[i]; | |
635 | pnv_default_stop_mask = psscr_mask[i]; | |
636 | default_stop_found = true; | |
637 | } | |
638 | } | |
639 | ||
f3b3f284 GS |
640 | if (unlikely(!default_stop_found)) { |
641 | pr_warn("cpuidle-powernv: No suitable default stop state found. Disabling platform idle.\n"); | |
642 | } else { | |
643 | ppc_md.power_save = power9_idle; | |
644 | pr_info("cpuidle-powernv: Default stop: psscr = 0x%016llx,mask=0x%016llx\n", | |
09206b60 GS |
645 | pnv_default_stop_val, pnv_default_stop_mask); |
646 | } | |
647 | ||
f3b3f284 GS |
648 | if (unlikely(!deepest_stop_found)) { |
649 | pr_warn("cpuidle-powernv: No suitable stop state for CPU-Hotplug. Offlined CPUs will busy wait"); | |
650 | } else { | |
651 | pr_info("cpuidle-powernv: Deepest stop: psscr = 0x%016llx,mask=0x%016llx\n", | |
09206b60 GS |
652 | pnv_deepest_stop_psscr_val, |
653 | pnv_deepest_stop_psscr_mask); | |
bcef83a0 SP |
654 | } |
655 | ||
f3b3f284 GS |
656 | pr_info("cpuidle-powernv: Requested Level (RL) value of first deep stop = 0x%llx\n", |
657 | pnv_first_deep_stop_state); | |
bcef83a0 SP |
658 | out: |
659 | kfree(psscr_val); | |
09206b60 GS |
660 | kfree(psscr_mask); |
661 | kfree(residency_ns); | |
bcef83a0 SP |
662 | return rc; |
663 | } | |
664 | ||
665 | /* | |
666 | * Probe device tree for supported idle states | |
667 | */ | |
668 | static void __init pnv_probe_idle_states(void) | |
669 | { | |
670 | struct device_node *np; | |
671 | int dt_idle_states; | |
672 | u32 *flags = NULL; | |
673 | int i; | |
674 | ||
675 | np = of_find_node_by_path("/ibm,opal/power-mgt"); | |
676 | if (!np) { | |
d405a98c SP |
677 | pr_warn("opal: PowerMgmt Node not found\n"); |
678 | goto out; | |
679 | } | |
bcef83a0 | 680 | dt_idle_states = of_property_count_u32_elems(np, |
d405a98c SP |
681 | "ibm,cpu-idle-state-flags"); |
682 | if (dt_idle_states < 0) { | |
683 | pr_warn("cpuidle-powernv: no idle states found in the DT\n"); | |
684 | goto out; | |
685 | } | |
686 | ||
bcef83a0 SP |
687 | flags = kcalloc(dt_idle_states, sizeof(*flags), GFP_KERNEL); |
688 | ||
689 | if (of_property_read_u32_array(np, | |
d405a98c SP |
690 | "ibm,cpu-idle-state-flags", flags, dt_idle_states)) { |
691 | pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-flags in DT\n"); | |
bcef83a0 SP |
692 | goto out; |
693 | } | |
694 | ||
695 | if (cpu_has_feature(CPU_FTR_ARCH_300)) { | |
dd34c74c | 696 | if (pnv_power9_idle_init(np, flags, dt_idle_states)) |
bcef83a0 | 697 | goto out; |
d405a98c SP |
698 | } |
699 | ||
700 | for (i = 0; i < dt_idle_states; i++) | |
701 | supported_cpuidle_states |= flags[i]; | |
702 | ||
bcef83a0 SP |
703 | out: |
704 | kfree(flags); | |
705 | } | |
706 | static int __init pnv_init_idle_states(void) | |
707 | { | |
708 | ||
709 | supported_cpuidle_states = 0; | |
710 | ||
711 | if (cpuidle_disable != IDLE_NO_OVERRIDE) | |
712 | goto out; | |
713 | ||
714 | pnv_probe_idle_states(); | |
715 | ||
d405a98c SP |
716 | if (!(supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1)) { |
717 | patch_instruction( | |
718 | (unsigned int *)pnv_fastsleep_workaround_at_entry, | |
719 | PPC_INST_NOP); | |
720 | patch_instruction( | |
721 | (unsigned int *)pnv_fastsleep_workaround_at_exit, | |
722 | PPC_INST_NOP); | |
5703d2f4 SP |
723 | } else { |
724 | /* | |
725 | * OPAL_PM_SLEEP_ENABLED_ER1 is set. It indicates that | |
726 | * workaround is needed to use fastsleep. Provide sysfs | |
727 | * control to choose how this workaround has to be applied. | |
728 | */ | |
729 | device_create_file(cpu_subsys.dev_root, | |
730 | &dev_attr_fastsleep_workaround_applyonce); | |
d405a98c | 731 | } |
5703d2f4 | 732 | |
d405a98c | 733 | pnv_alloc_idle_core_states(); |
5593e303 | 734 | |
17ed4c8f GS |
735 | /* |
736 | * For each CPU, record its PACA address in each of it's | |
737 | * sibling thread's PACA at the slot corresponding to this | |
738 | * CPU's index in the core. | |
739 | */ | |
740 | if (cpu_has_feature(CPU_FTR_POWER9_DD1)) { | |
741 | int cpu; | |
742 | ||
743 | pr_info("powernv: idle: Saving PACA pointers of all CPUs in their thread sibling PACA\n"); | |
744 | for_each_possible_cpu(cpu) { | |
745 | int base_cpu = cpu_first_thread_sibling(cpu); | |
746 | int idx = cpu_thread_in_core(cpu); | |
747 | int i; | |
748 | ||
749 | for (i = 0; i < threads_per_core; i++) { | |
750 | int j = base_cpu + i; | |
751 | ||
752 | paca[j].thread_sibling_pacas[idx] = &paca[cpu]; | |
753 | } | |
754 | } | |
755 | } | |
756 | ||
5593e303 SP |
757 | if (supported_cpuidle_states & OPAL_PM_NAP_ENABLED) |
758 | ppc_md.power_save = power7_idle; | |
bcef83a0 | 759 | |
d405a98c SP |
760 | out: |
761 | return 0; | |
762 | } | |
4bece972 | 763 | machine_subsys_initcall(powernv, pnv_init_idle_states); |