]>
Commit | Line | Data |
---|---|---|
948cf67c | 1 | /* |
7230c564 | 2 | * This file contains the power_save function for Power7 CPUs. |
948cf67c BH |
3 | * |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation; either version | |
7 | * 2 of the License, or (at your option) any later version. | |
8 | */ | |
9 | ||
10 | #include <linux/threads.h> | |
11 | #include <asm/processor.h> | |
12 | #include <asm/page.h> | |
13 | #include <asm/cputable.h> | |
14 | #include <asm/thread_info.h> | |
15 | #include <asm/ppc_asm.h> | |
16 | #include <asm/asm-offsets.h> | |
17 | #include <asm/ppc-opcode.h> | |
7230c564 | 18 | #include <asm/hw_irq.h> |
f0888f70 | 19 | #include <asm/kvm_book3s_asm.h> |
97eb001f | 20 | #include <asm/opal.h> |
7cba160a | 21 | #include <asm/cpuidle.h> |
77b54e9f | 22 | #include <asm/mmu-hash64.h> |
948cf67c BH |
23 | |
24 | #undef DEBUG | |
25 | ||
77b54e9f SP |
26 | /* |
27 | * Use unused space in the interrupt stack to save and restore | |
28 | * registers for winkle support. | |
29 | */ | |
30 | #define _SDR1 GPR3 | |
31 | #define _RPR GPR4 | |
32 | #define _SPURR GPR5 | |
33 | #define _PURR GPR6 | |
34 | #define _TSCR GPR7 | |
35 | #define _DSCR GPR8 | |
36 | #define _AMOR GPR9 | |
37 | #define _WORT GPR10 | |
38 | #define _WORC GPR11 | |
39 | ||
aca79d2b | 40 | /* Idle state entry routines */ |
948cf67c | 41 | |
aca79d2b VS |
42 | #define IDLE_STATE_ENTER_SEQ(IDLE_INST) \ |
43 | /* Magic NAP/SLEEP/WINKLE mode enter sequence */ \ | |
44 | std r0,0(r1); \ | |
45 | ptesync; \ | |
46 | ld r0,0(r1); \ | |
47 | 1: cmp cr0,r0,r0; \ | |
48 | bne 1b; \ | |
49 | IDLE_INST; \ | |
50 | b . | |
948cf67c | 51 | |
aca79d2b VS |
52 | .text |
53 | ||
54 | /* | |
55 | * Pass requested state in r3: | |
7cba160a | 56 | * r3 - PNV_THREAD_NAP/SLEEP/WINKLE |
8d6f7c5a ME |
57 | * |
58 | * To check IRQ_HAPPENED in r4 | |
59 | * 0 - don't check | |
60 | * 1 - check | |
aca79d2b VS |
61 | */ |
62 | _GLOBAL(power7_powersave_common) | |
63 | /* Use r3 to pass state nap/sleep/winkle */ | |
948cf67c BH |
64 | /* NAP is a state loss, we create a regs frame on the |
65 | * stack, fill it up with the state we care about and | |
66 | * stick a pointer to it in PACAR1. We really only | |
67 | * need to save PC, some CR bits and the NV GPRs, | |
68 | * but for now an interrupt frame will do. | |
69 | */ | |
70 | mflr r0 | |
71 | std r0,16(r1) | |
72 | stdu r1,-INT_FRAME_SIZE(r1) | |
73 | std r0,_LINK(r1) | |
74 | std r0,_NIP(r1) | |
75 | ||
76 | #ifndef CONFIG_SMP | |
77 | /* Make sure FPU, VSX etc... are flushed as we may lose | |
78 | * state when going to nap mode | |
79 | */ | |
b1576fec | 80 | bl discard_lazy_cpu_state |
948cf67c BH |
81 | #endif /* CONFIG_SMP */ |
82 | ||
83 | /* Hard disable interrupts */ | |
84 | mfmsr r9 | |
85 | rldicl r9,r9,48,1 | |
86 | rotldi r9,r9,16 | |
87 | mtmsrd r9,1 /* hard-disable interrupts */ | |
7230c564 BH |
88 | |
89 | /* Check if something happened while soft-disabled */ | |
90 | lbz r0,PACAIRQHAPPENED(r13) | |
d6a4f709 | 91 | andi. r0,r0,~PACA_IRQ_HARD_DIS@l |
7230c564 | 92 | beq 1f |
8d6f7c5a ME |
93 | cmpwi cr0,r4,0 |
94 | beq 1f | |
7230c564 BH |
95 | addi r1,r1,INT_FRAME_SIZE |
96 | ld r0,16(r1) | |
f57333a7 | 97 | li r3,0 /* Return 0 (no nap) */ |
7230c564 BH |
98 | mtlr r0 |
99 | blr | |
100 | ||
101 | 1: /* We mark irqs hard disabled as this is the state we'll | |
102 | * be in when returning and we need to tell arch_local_irq_restore() | |
103 | * about it | |
104 | */ | |
105 | li r0,PACA_IRQ_HARD_DIS | |
106 | stb r0,PACAIRQHAPPENED(r13) | |
107 | ||
108 | /* We haven't lost state ... yet */ | |
948cf67c | 109 | li r0,0 |
2fde6d20 | 110 | stb r0,PACA_NAPSTATELOST(r13) |
948cf67c BH |
111 | |
112 | /* Continue saving state */ | |
113 | SAVE_GPR(2, r1) | |
114 | SAVE_NVGPRS(r1) | |
aca79d2b VS |
115 | mfcr r4 |
116 | std r4,_CCR(r1) | |
948cf67c BH |
117 | std r9,_MSR(r1) |
118 | std r1,PACAR1(r13) | |
119 | ||
8117ac6a PM |
120 | /* |
121 | * Go to real mode to do the nap, as required by the architecture. | |
122 | * Also, we need to be in real mode before setting hwthread_state, | |
123 | * because as soon as we do that, another thread can switch | |
124 | * the MMU context to the guest. | |
125 | */ | |
126 | LOAD_REG_IMMEDIATE(r5, MSR_IDLE) | |
127 | li r6, MSR_RI | |
128 | andc r6, r9, r6 | |
129 | LOAD_REG_ADDR(r7, power7_enter_nap_mode) | |
130 | mtmsrd r6, 1 /* clear RI before setting SRR0/1 */ | |
131 | mtspr SPRN_SRR0, r7 | |
132 | mtspr SPRN_SRR1, r5 | |
133 | rfid | |
134 | ||
135 | .globl power7_enter_nap_mode | |
136 | power7_enter_nap_mode: | |
9975f5e3 | 137 | #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE |
f0888f70 PM |
138 | /* Tell KVM we're napping */ |
139 | li r4,KVM_HWTHREAD_IN_NAP | |
140 | stb r4,HSTATE_HWTHREAD_STATE(r13) | |
141 | #endif | |
7cba160a | 142 | stb r3,PACA_THREAD_IDLE_STATE(r13) |
77b54e9f SP |
143 | cmpwi cr3,r3,PNV_THREAD_SLEEP |
144 | bge cr3,2f | |
aca79d2b VS |
145 | IDLE_STATE_ENTER_SEQ(PPC_NAP) |
146 | /* No return */ | |
7cba160a SP |
147 | 2: |
148 | /* Sleep or winkle */ | |
149 | lbz r7,PACA_THREAD_MASK(r13) | |
150 | ld r14,PACA_CORE_IDLE_STATE_PTR(r13) | |
151 | lwarx_loop1: | |
152 | lwarx r15,0,r14 | |
153 | andc r15,r15,r7 /* Clear thread bit */ | |
154 | ||
155 | andi. r15,r15,PNV_CORE_IDLE_THREAD_BITS | |
156 | ||
157 | /* | |
158 | * If cr0 = 0, then current thread is the last thread of the core entering | |
159 | * sleep. Last thread needs to execute the hardware bug workaround code if | |
160 | * required by the platform. | |
161 | * Make the workaround call unconditionally here. The below branch call is | |
162 | * patched out when the idle states are discovered if the platform does not | |
163 | * require it. | |
164 | */ | |
165 | .global pnv_fastsleep_workaround_at_entry | |
166 | pnv_fastsleep_workaround_at_entry: | |
167 | beq fastsleep_workaround_at_entry | |
168 | ||
169 | stwcx. r15,0,r14 | |
170 | bne- lwarx_loop1 | |
171 | isync | |
172 | ||
77b54e9f SP |
173 | common_enter: /* common code for all the threads entering sleep or winkle */ |
174 | bgt cr3,enter_winkle | |
7cba160a SP |
175 | IDLE_STATE_ENTER_SEQ(PPC_SLEEP) |
176 | ||
177 | fastsleep_workaround_at_entry: | |
178 | ori r15,r15,PNV_CORE_IDLE_LOCK_BIT | |
179 | stwcx. r15,0,r14 | |
180 | bne- lwarx_loop1 | |
181 | isync | |
182 | ||
183 | /* Fast sleep workaround */ | |
184 | li r3,1 | |
185 | li r4,1 | |
186 | li r0,OPAL_CONFIG_CPU_IDLE_STATE | |
187 | bl opal_call_realmode | |
188 | ||
189 | /* Clear Lock bit */ | |
190 | li r0,0 | |
191 | lwsync | |
192 | stw r0,0(r14) | |
193 | b common_enter | |
194 | ||
77b54e9f SP |
195 | enter_winkle: |
196 | /* | |
197 | * Note all register i.e per-core, per-subcore or per-thread is saved | |
198 | * here since any thread in the core might wake up first | |
199 | */ | |
200 | mfspr r3,SPRN_SDR1 | |
201 | std r3,_SDR1(r1) | |
202 | mfspr r3,SPRN_RPR | |
203 | std r3,_RPR(r1) | |
204 | mfspr r3,SPRN_SPURR | |
205 | std r3,_SPURR(r1) | |
206 | mfspr r3,SPRN_PURR | |
207 | std r3,_PURR(r1) | |
208 | mfspr r3,SPRN_TSCR | |
209 | std r3,_TSCR(r1) | |
210 | mfspr r3,SPRN_DSCR | |
211 | std r3,_DSCR(r1) | |
212 | mfspr r3,SPRN_AMOR | |
213 | std r3,_AMOR(r1) | |
214 | mfspr r3,SPRN_WORT | |
215 | std r3,_WORT(r1) | |
216 | mfspr r3,SPRN_WORC | |
217 | std r3,_WORC(r1) | |
218 | IDLE_STATE_ENTER_SEQ(PPC_WINKLE) | |
f0888f70 | 219 | |
aca79d2b VS |
220 | _GLOBAL(power7_idle) |
221 | /* Now check if user or arch enabled NAP mode */ | |
222 | LOAD_REG_ADDRBASE(r3,powersave_nap) | |
223 | lwz r4,ADDROFF(powersave_nap)(r3) | |
224 | cmpwi 0,r4,0 | |
225 | beqlr | |
8d6f7c5a | 226 | li r3, 1 |
aca79d2b VS |
227 | /* fall through */ |
228 | ||
229 | _GLOBAL(power7_nap) | |
8d6f7c5a | 230 | mr r4,r3 |
7cba160a | 231 | li r3,PNV_THREAD_NAP |
aca79d2b VS |
232 | b power7_powersave_common |
233 | /* No return */ | |
234 | ||
235 | _GLOBAL(power7_sleep) | |
7cba160a | 236 | li r3,PNV_THREAD_SLEEP |
c733cf83 | 237 | li r4,1 |
aca79d2b VS |
238 | b power7_powersave_common |
239 | /* No return */ | |
948cf67c | 240 | |
77b54e9f SP |
241 | _GLOBAL(power7_winkle) |
242 | li r3,3 | |
243 | li r4,1 | |
244 | b power7_powersave_common | |
245 | /* No return */ | |
246 | ||
bbdb760d MS |
247 | #define CHECK_HMI_INTERRUPT \ |
248 | mfspr r0,SPRN_SRR1; \ | |
249 | BEGIN_FTR_SECTION_NESTED(66); \ | |
250 | rlwinm r0,r0,45-31,0xf; /* extract wake reason field (P8) */ \ | |
251 | FTR_SECTION_ELSE_NESTED(66); \ | |
252 | rlwinm r0,r0,45-31,0xe; /* P7 wake reason field is 3 bits */ \ | |
253 | ALT_FTR_SECTION_END_NESTED_IFSET(CPU_FTR_ARCH_207S, 66); \ | |
254 | cmpwi r0,0xa; /* Hypervisor maintenance ? */ \ | |
255 | bne 20f; \ | |
256 | /* Invoke opal call to handle hmi */ \ | |
257 | ld r2,PACATOC(r13); \ | |
258 | ld r1,PACAR1(r13); \ | |
259 | std r3,ORIG_GPR3(r1); /* Save original r3 */ \ | |
7cba160a | 260 | li r0,OPAL_HANDLE_HMI; /* Pass opal token argument*/ \ |
db97efff | 261 | bl opal_call_realmode; \ |
bbdb760d MS |
262 | ld r3,ORIG_GPR3(r1); /* Restore original r3 */ \ |
263 | 20: nop; | |
264 | ||
265 | ||
97eb001f VS |
266 | _GLOBAL(power7_wakeup_tb_loss) |
267 | ld r2,PACATOC(r13); | |
268 | ld r1,PACAR1(r13) | |
7cba160a SP |
269 | /* |
270 | * Before entering any idle state, the NVGPRs are saved in the stack | |
271 | * and they are restored before switching to the process context. Hence | |
272 | * until they are restored, they are free to be used. | |
273 | * | |
274 | * Save SRR1 in a NVGPR as it might be clobbered in opal_call_realmode | |
275 | * (called in CHECK_HMI_INTERRUPT). SRR1 is required to determine the | |
276 | * wakeup reason if we branch to kvm_start_guest. | |
277 | */ | |
97eb001f | 278 | |
7cba160a | 279 | mfspr r16,SPRN_SRR1 |
bbdb760d MS |
280 | BEGIN_FTR_SECTION |
281 | CHECK_HMI_INTERRUPT | |
282 | END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) | |
7cba160a SP |
283 | |
284 | lbz r7,PACA_THREAD_MASK(r13) | |
285 | ld r14,PACA_CORE_IDLE_STATE_PTR(r13) | |
286 | lwarx_loop2: | |
287 | lwarx r15,0,r14 | |
288 | andi. r9,r15,PNV_CORE_IDLE_LOCK_BIT | |
289 | /* | |
290 | * Lock bit is set in one of the 2 cases- | |
291 | * a. In the sleep/winkle enter path, the last thread is executing | |
292 | * fastsleep workaround code. | |
293 | * b. In the wake up path, another thread is executing fastsleep | |
294 | * workaround undo code or resyncing timebase or restoring context | |
295 | * In either case loop until the lock bit is cleared. | |
296 | */ | |
297 | bne core_idle_lock_held | |
298 | ||
299 | cmpwi cr2,r15,0 | |
77b54e9f SP |
300 | lbz r4,PACA_SUBCORE_SIBLING_MASK(r13) |
301 | and r4,r4,r15 | |
302 | cmpwi cr1,r4,0 /* Check if first in subcore */ | |
303 | ||
304 | /* | |
305 | * At this stage | |
306 | * cr1 - 0b0100 if first thread to wakeup in subcore | |
307 | * cr2 - 0b0100 if first thread to wakeup in core | |
308 | * cr3- 0b0010 if waking up from sleep or winkle | |
309 | * cr4 - 0b0100 if waking up from winkle | |
310 | */ | |
311 | ||
7cba160a SP |
312 | or r15,r15,r7 /* Set thread bit */ |
313 | ||
77b54e9f | 314 | beq cr1,first_thread_in_subcore |
7cba160a | 315 | |
77b54e9f | 316 | /* Not first thread in subcore to wake up */ |
7cba160a SP |
317 | stwcx. r15,0,r14 |
318 | bne- lwarx_loop2 | |
319 | isync | |
320 | b common_exit | |
321 | ||
322 | core_idle_lock_held: | |
323 | HMT_LOW | |
324 | core_idle_lock_loop: | |
325 | lwz r15,0(14) | |
326 | andi. r9,r15,PNV_CORE_IDLE_LOCK_BIT | |
327 | bne core_idle_lock_loop | |
328 | HMT_MEDIUM | |
329 | b lwarx_loop2 | |
330 | ||
77b54e9f SP |
331 | first_thread_in_subcore: |
332 | /* First thread in subcore to wakeup */ | |
7cba160a SP |
333 | ori r15,r15,PNV_CORE_IDLE_LOCK_BIT |
334 | stwcx. r15,0,r14 | |
335 | bne- lwarx_loop2 | |
336 | isync | |
337 | ||
77b54e9f SP |
338 | /* |
339 | * If waking up from sleep, subcore state is not lost. Hence | |
340 | * skip subcore state restore | |
341 | */ | |
342 | bne cr4,subcore_state_restored | |
343 | ||
344 | /* Restore per-subcore state */ | |
345 | ld r4,_SDR1(r1) | |
346 | mtspr SPRN_SDR1,r4 | |
347 | ld r4,_RPR(r1) | |
348 | mtspr SPRN_RPR,r4 | |
349 | ld r4,_AMOR(r1) | |
350 | mtspr SPRN_AMOR,r4 | |
351 | ||
352 | subcore_state_restored: | |
353 | /* | |
354 | * Check if the thread is also the first thread in the core. If not, | |
355 | * skip to clear_lock. | |
356 | */ | |
357 | bne cr2,clear_lock | |
358 | ||
359 | first_thread_in_core: | |
360 | ||
7cba160a SP |
361 | /* |
362 | * First thread in the core waking up from fastsleep. It needs to | |
363 | * call the fastsleep workaround code if the platform requires it. | |
364 | * Call it unconditionally here. The below branch instruction will | |
365 | * be patched out when the idle states are discovered if platform | |
366 | * does not require workaround. | |
367 | */ | |
368 | .global pnv_fastsleep_workaround_at_exit | |
369 | pnv_fastsleep_workaround_at_exit: | |
370 | b fastsleep_workaround_at_exit | |
371 | ||
372 | timebase_resync: | |
373 | /* Do timebase resync if we are waking up from sleep. Use cr3 value | |
374 | * set in exceptions-64s.S */ | |
375 | ble cr3,clear_lock | |
97eb001f | 376 | /* Time base re-sync */ |
7cba160a | 377 | li r0,OPAL_RESYNC_TIMEBASE |
db97efff | 378 | bl opal_call_realmode; |
97eb001f VS |
379 | /* TODO: Check r3 for failure */ |
380 | ||
77b54e9f SP |
381 | /* |
382 | * If waking up from sleep, per core state is not lost, skip to | |
383 | * clear_lock. | |
384 | */ | |
385 | bne cr4,clear_lock | |
386 | ||
387 | /* Restore per core state */ | |
388 | ld r4,_TSCR(r1) | |
389 | mtspr SPRN_TSCR,r4 | |
390 | ld r4,_WORC(r1) | |
391 | mtspr SPRN_WORC,r4 | |
392 | ||
7cba160a SP |
393 | clear_lock: |
394 | andi. r15,r15,PNV_CORE_IDLE_THREAD_BITS | |
395 | lwsync | |
396 | stw r15,0(r14) | |
397 | ||
398 | common_exit: | |
77b54e9f SP |
399 | /* |
400 | * Common to all threads. | |
401 | * | |
402 | * If waking up from sleep, hypervisor state is not lost. Hence | |
403 | * skip hypervisor state restore. | |
404 | */ | |
405 | bne cr4,hypervisor_state_restored | |
406 | ||
407 | /* Waking up from winkle */ | |
408 | ||
409 | /* Restore per thread state */ | |
410 | bl __restore_cpu_power8 | |
411 | ||
412 | /* Restore SLB from PACA */ | |
413 | ld r8,PACA_SLBSHADOWPTR(r13) | |
414 | ||
415 | .rept SLB_NUM_BOLTED | |
416 | li r3, SLBSHADOW_SAVEAREA | |
417 | LDX_BE r5, r8, r3 | |
418 | addi r3, r3, 8 | |
419 | LDX_BE r6, r8, r3 | |
420 | andis. r7,r5,SLB_ESID_V@h | |
421 | beq 1f | |
422 | slbmte r6,r5 | |
423 | 1: addi r8,r8,16 | |
424 | .endr | |
425 | ||
426 | ld r4,_SPURR(r1) | |
427 | mtspr SPRN_SPURR,r4 | |
428 | ld r4,_PURR(r1) | |
429 | mtspr SPRN_PURR,r4 | |
430 | ld r4,_DSCR(r1) | |
431 | mtspr SPRN_DSCR,r4 | |
432 | ld r4,_WORT(r1) | |
433 | mtspr SPRN_WORT,r4 | |
434 | ||
435 | hypervisor_state_restored: | |
436 | ||
7cba160a SP |
437 | li r5,PNV_THREAD_RUNNING |
438 | stb r5,PACA_THREAD_IDLE_STATE(r13) | |
439 | ||
440 | mtspr SPRN_SRR1,r16 | |
441 | #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE | |
442 | li r0,KVM_HWTHREAD_IN_KERNEL | |
443 | stb r0,HSTATE_HWTHREAD_STATE(r13) | |
444 | /* Order setting hwthread_state vs. testing hwthread_req */ | |
445 | sync | |
446 | lbz r0,HSTATE_HWTHREAD_REQ(r13) | |
447 | cmpwi r0,0 | |
448 | beq 6f | |
449 | b kvm_start_guest | |
450 | 6: | |
451 | #endif | |
452 | ||
97eb001f VS |
453 | REST_NVGPRS(r1) |
454 | REST_GPR(2, r1) | |
455 | ld r3,_CCR(r1) | |
456 | ld r4,_MSR(r1) | |
457 | ld r5,_NIP(r1) | |
458 | addi r1,r1,INT_FRAME_SIZE | |
459 | mtcr r3 | |
460 | mfspr r3,SPRN_SRR1 /* Return SRR1 */ | |
461 | mtspr SPRN_SRR1,r4 | |
462 | mtspr SPRN_SRR0,r5 | |
463 | rfid | |
464 | ||
7cba160a SP |
465 | fastsleep_workaround_at_exit: |
466 | li r3,1 | |
467 | li r4,0 | |
468 | li r0,OPAL_CONFIG_CPU_IDLE_STATE | |
469 | bl opal_call_realmode | |
470 | b timebase_resync | |
471 | ||
56548fc0 PM |
472 | /* |
473 | * R3 here contains the value that will be returned to the caller | |
474 | * of power7_nap. | |
475 | */ | |
948cf67c | 476 | _GLOBAL(power7_wakeup_loss) |
948cf67c | 477 | ld r1,PACAR1(r13) |
bbdb760d MS |
478 | BEGIN_FTR_SECTION |
479 | CHECK_HMI_INTERRUPT | |
480 | END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) | |
948cf67c BH |
481 | REST_NVGPRS(r1) |
482 | REST_GPR(2, r1) | |
56548fc0 | 483 | ld r6,_CCR(r1) |
948cf67c BH |
484 | ld r4,_MSR(r1) |
485 | ld r5,_NIP(r1) | |
486 | addi r1,r1,INT_FRAME_SIZE | |
56548fc0 | 487 | mtcr r6 |
948cf67c BH |
488 | mtspr SPRN_SRR1,r4 |
489 | mtspr SPRN_SRR0,r5 | |
490 | rfid | |
491 | ||
56548fc0 PM |
492 | /* |
493 | * R3 here contains the value that will be returned to the caller | |
494 | * of power7_nap. | |
495 | */ | |
948cf67c | 496 | _GLOBAL(power7_wakeup_noloss) |
2fde6d20 PM |
497 | lbz r0,PACA_NAPSTATELOST(r13) |
498 | cmpwi r0,0 | |
b1576fec | 499 | bne power7_wakeup_loss |
bbdb760d MS |
500 | BEGIN_FTR_SECTION |
501 | CHECK_HMI_INTERRUPT | |
502 | END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) | |
948cf67c BH |
503 | ld r1,PACAR1(r13) |
504 | ld r4,_MSR(r1) | |
505 | ld r5,_NIP(r1) | |
506 | addi r1,r1,INT_FRAME_SIZE | |
507 | mtspr SPRN_SRR1,r4 | |
508 | mtspr SPRN_SRR0,r5 | |
509 | rfid |