]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
707827f3 | 2 | /* |
962e7bd4 | 3 | * cpuidle-pseries - idle state cpuidle driver. |
707827f3 DD |
4 | * Adapted from drivers/idle/intel_idle.c and |
5 | * drivers/acpi/processor_idle.c | |
6 | * | |
7 | */ | |
8 | ||
9 | #include <linux/kernel.h> | |
10 | #include <linux/module.h> | |
11 | #include <linux/init.h> | |
12 | #include <linux/moduleparam.h> | |
13 | #include <linux/cpuidle.h> | |
14 | #include <linux/cpu.h> | |
16aaaff6 | 15 | #include <linux/notifier.h> |
707827f3 DD |
16 | |
17 | #include <asm/paca.h> | |
18 | #include <asm/reg.h> | |
707827f3 DD |
19 | #include <asm/machdep.h> |
20 | #include <asm/firmware.h> | |
3f67d962 | 21 | #include <asm/runlatch.h> |
e4a884cc | 22 | #include <asm/idle.h> |
212bebb4 | 23 | #include <asm/plpar_wrappers.h> |
707827f3 | 24 | |
92fe8483 | 25 | static struct cpuidle_driver pseries_idle_driver = { |
1ca80944 DL |
26 | .name = "pseries_idle", |
27 | .owner = THIS_MODULE, | |
707827f3 DD |
28 | }; |
29 | ||
624e46d0 NP |
30 | static int max_idle_state __read_mostly; |
31 | static struct cpuidle_state *cpuidle_state_table __read_mostly; | |
32 | static u64 snooze_timeout __read_mostly; | |
33 | static bool snooze_timeout_en __read_mostly; | |
707827f3 | 34 | |
707827f3 DD |
35 | static int snooze_loop(struct cpuidle_device *dev, |
36 | struct cpuidle_driver *drv, | |
37 | int index) | |
38 | { | |
78eaa10f | 39 | u64 snooze_exit_time; |
707827f3 | 40 | |
3fc5ee92 NP |
41 | set_thread_flag(TIF_POLLING_NRFLAG); |
42 | ||
c4019198 | 43 | pseries_idle_prolog(); |
83dac594 | 44 | local_irq_enable(); |
78eaa10f | 45 | snooze_exit_time = get_tb() + snooze_timeout; |
707827f3 | 46 | |
b69dbba0 | 47 | while (!need_resched()) { |
83dac594 DD |
48 | HMT_low(); |
49 | HMT_very_low(); | |
7ded4291 NP |
50 | if (likely(snooze_timeout_en) && get_tb() > snooze_exit_time) { |
51 | /* | |
52 | * Task has not woken up but we are exiting the polling | |
53 | * loop anyway. Require a barrier after polling is | |
54 | * cleared to order subsequent test of need_resched(). | |
55 | */ | |
56 | clear_thread_flag(TIF_POLLING_NRFLAG); | |
57 | smp_mb(); | |
78eaa10f | 58 | break; |
7ded4291 | 59 | } |
707827f3 DD |
60 | } |
61 | ||
707827f3 | 62 | HMT_medium(); |
83dac594 | 63 | clear_thread_flag(TIF_POLLING_NRFLAG); |
83dac594 | 64 | |
ced54c08 NP |
65 | local_irq_disable(); |
66 | ||
c4019198 | 67 | pseries_idle_epilog(); |
1ca80944 | 68 | |
707827f3 DD |
69 | return index; |
70 | } | |
71 | ||
7230c564 BH |
72 | static void check_and_cede_processor(void) |
73 | { | |
74 | /* | |
be2cf20a BH |
75 | * Ensure our interrupt state is properly tracked, |
76 | * also checks if no interrupt has occurred while we | |
77 | * were soft-disabled | |
7230c564 | 78 | */ |
be2cf20a | 79 | if (prep_irq_for_idle()) { |
7230c564 | 80 | cede_processor(); |
be2cf20a BH |
81 | #ifdef CONFIG_TRACE_IRQFLAGS |
82 | /* Ensure that H_CEDE returns with IRQs on */ | |
83 | if (WARN_ON(!(mfmsr() & MSR_EE))) | |
84 | __hard_irq_enable(); | |
85 | #endif | |
86 | } | |
7230c564 BH |
87 | } |
88 | ||
707827f3 DD |
89 | static int dedicated_cede_loop(struct cpuidle_device *dev, |
90 | struct cpuidle_driver *drv, | |
91 | int index) | |
92 | { | |
707827f3 | 93 | |
c4019198 | 94 | pseries_idle_prolog(); |
707827f3 DD |
95 | get_lppaca()->donate_dedicated_cpu = 1; |
96 | ||
707827f3 | 97 | HMT_medium(); |
7230c564 | 98 | check_and_cede_processor(); |
707827f3 | 99 | |
ced54c08 | 100 | local_irq_disable(); |
707827f3 | 101 | get_lppaca()->donate_dedicated_cpu = 0; |
1ca80944 | 102 | |
c4019198 | 103 | pseries_idle_epilog(); |
1ca80944 | 104 | |
707827f3 DD |
105 | return index; |
106 | } | |
107 | ||
108 | static int shared_cede_loop(struct cpuidle_device *dev, | |
109 | struct cpuidle_driver *drv, | |
110 | int index) | |
111 | { | |
707827f3 | 112 | |
c4019198 | 113 | pseries_idle_prolog(); |
707827f3 DD |
114 | |
115 | /* | |
116 | * Yield the processor to the hypervisor. We return if | |
117 | * an external interrupt occurs (which are driven prior | |
118 | * to returning here) or if a prod occurs from another | |
119 | * processor. When returning here, external interrupts | |
120 | * are enabled. | |
121 | */ | |
7230c564 | 122 | check_and_cede_processor(); |
707827f3 | 123 | |
ced54c08 | 124 | local_irq_disable(); |
c4019198 | 125 | pseries_idle_epilog(); |
1ca80944 | 126 | |
707827f3 DD |
127 | return index; |
128 | } | |
129 | ||
130 | /* | |
131 | * States for dedicated partition case. | |
132 | */ | |
bf7f61f2 | 133 | static struct cpuidle_state dedicated_states[] = { |
707827f3 DD |
134 | { /* Snooze */ |
135 | .name = "snooze", | |
136 | .desc = "snooze", | |
707827f3 DD |
137 | .exit_latency = 0, |
138 | .target_residency = 0, | |
139 | .enter = &snooze_loop }, | |
140 | { /* CEDE */ | |
141 | .name = "CEDE", | |
142 | .desc = "CEDE", | |
83dac594 DD |
143 | .exit_latency = 10, |
144 | .target_residency = 100, | |
707827f3 DD |
145 | .enter = &dedicated_cede_loop }, |
146 | }; | |
147 | ||
148 | /* | |
149 | * States for shared partition case. | |
150 | */ | |
bf7f61f2 | 151 | static struct cpuidle_state shared_states[] = { |
f2ac428e NP |
152 | { /* Snooze */ |
153 | .name = "snooze", | |
154 | .desc = "snooze", | |
155 | .exit_latency = 0, | |
156 | .target_residency = 0, | |
157 | .enter = &snooze_loop }, | |
707827f3 DD |
158 | { /* Shared Cede */ |
159 | .name = "Shared Cede", | |
160 | .desc = "Shared Cede", | |
f2ac428e NP |
161 | .exit_latency = 10, |
162 | .target_residency = 100, | |
707827f3 DD |
163 | .enter = &shared_cede_loop }, |
164 | }; | |
165 | ||
529351fd | 166 | static int pseries_cpuidle_cpu_online(unsigned int cpu) |
707827f3 | 167 | { |
529351fd | 168 | struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu); |
16aaaff6 | 169 | |
852d8cb1 | 170 | if (dev && cpuidle_get_driver()) { |
529351fd SAS |
171 | cpuidle_pause_and_lock(); |
172 | cpuidle_enable_device(dev); | |
173 | cpuidle_resume_and_unlock(); | |
174 | } | |
175 | return 0; | |
176 | } | |
852d8cb1 | 177 | |
529351fd SAS |
178 | static int pseries_cpuidle_cpu_dead(unsigned int cpu) |
179 | { | |
180 | struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu); | |
852d8cb1 | 181 | |
529351fd SAS |
182 | if (dev && cpuidle_get_driver()) { |
183 | cpuidle_pause_and_lock(); | |
184 | cpuidle_disable_device(dev); | |
185 | cpuidle_resume_and_unlock(); | |
707827f3 | 186 | } |
529351fd | 187 | return 0; |
707827f3 DD |
188 | } |
189 | ||
190 | /* | |
191 | * pseries_cpuidle_driver_init() | |
192 | */ | |
193 | static int pseries_cpuidle_driver_init(void) | |
194 | { | |
195 | int idle_state; | |
196 | struct cpuidle_driver *drv = &pseries_idle_driver; | |
197 | ||
198 | drv->state_count = 0; | |
199 | ||
bf7f61f2 DD |
200 | for (idle_state = 0; idle_state < max_idle_state; ++idle_state) { |
201 | /* Is the state not enabled? */ | |
707827f3 DD |
202 | if (cpuidle_state_table[idle_state].enter == NULL) |
203 | continue; | |
204 | ||
205 | drv->states[drv->state_count] = /* structure copy */ | |
206 | cpuidle_state_table[idle_state]; | |
207 | ||
707827f3 DD |
208 | drv->state_count += 1; |
209 | } | |
210 | ||
211 | return 0; | |
212 | } | |
213 | ||
707827f3 DD |
214 | /* |
215 | * pseries_idle_probe() | |
216 | * Choose state table for shared versus dedicated partition | |
217 | */ | |
218 | static int pseries_idle_probe(void) | |
219 | { | |
220 | ||
e8bb3e00 DD |
221 | if (cpuidle_disable != IDLE_NO_OVERRIDE) |
222 | return -ENODEV; | |
223 | ||
b69dbba0 | 224 | if (firmware_has_feature(FW_FEATURE_SPLPAR)) { |
2b038cbc BL |
225 | /* |
226 | * Use local_paca instead of get_lppaca() since | |
227 | * preemption is not disabled, and it is not required in | |
228 | * fact, since lppaca_ptr does not need to be the value | |
229 | * associated to the current CPU, it can be from any CPU. | |
230 | */ | |
231 | if (lppaca_shared_proc(local_paca->lppaca_ptr)) { | |
b69dbba0 | 232 | cpuidle_state_table = shared_states; |
bf7f61f2 DD |
233 | max_idle_state = ARRAY_SIZE(shared_states); |
234 | } else { | |
b69dbba0 | 235 | cpuidle_state_table = dedicated_states; |
bf7f61f2 DD |
236 | max_idle_state = ARRAY_SIZE(dedicated_states); |
237 | } | |
b69dbba0 DD |
238 | } else |
239 | return -ENODEV; | |
707827f3 | 240 | |
78eaa10f SB |
241 | if (max_idle_state > 1) { |
242 | snooze_timeout_en = true; | |
243 | snooze_timeout = cpuidle_state_table[1].target_residency * | |
244 | tb_ticks_per_usec; | |
245 | } | |
707827f3 DD |
246 | return 0; |
247 | } | |
248 | ||
249 | static int __init pseries_processor_idle_init(void) | |
250 | { | |
251 | int retval; | |
252 | ||
253 | retval = pseries_idle_probe(); | |
254 | if (retval) | |
255 | return retval; | |
256 | ||
257 | pseries_cpuidle_driver_init(); | |
b69dbba0 | 258 | retval = cpuidle_register(&pseries_idle_driver, NULL); |
707827f3 DD |
259 | if (retval) { |
260 | printk(KERN_DEBUG "Registration of pseries driver failed.\n"); | |
261 | return retval; | |
262 | } | |
263 | ||
529351fd SAS |
264 | retval = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, |
265 | "cpuidle/pseries:online", | |
266 | pseries_cpuidle_cpu_online, NULL); | |
267 | WARN_ON(retval < 0); | |
268 | retval = cpuhp_setup_state_nocalls(CPUHP_CPUIDLE_DEAD, | |
269 | "cpuidle/pseries:DEAD", NULL, | |
270 | pseries_cpuidle_cpu_dead); | |
271 | WARN_ON(retval < 0); | |
707827f3 | 272 | printk(KERN_DEBUG "pseries_idle_driver registered\n"); |
707827f3 DD |
273 | return 0; |
274 | } | |
275 | ||
12431c64 | 276 | device_initcall(pseries_processor_idle_init); |