]>
Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
55190f88 BH |
2 | /* |
3 | * SMP support for PowerNV machines. | |
4 | * | |
5 | * Copyright 2011 IBM Corp. | |
55190f88 BH |
6 | */ |
7 | ||
8 | #include <linux/kernel.h> | |
9 | #include <linux/module.h> | |
10 | #include <linux/sched.h> | |
ef8bd77f | 11 | #include <linux/sched/hotplug.h> |
55190f88 BH |
12 | #include <linux/smp.h> |
13 | #include <linux/interrupt.h> | |
14 | #include <linux/delay.h> | |
15 | #include <linux/init.h> | |
16 | #include <linux/spinlock.h> | |
17 | #include <linux/cpu.h> | |
18 | ||
19 | #include <asm/irq.h> | |
20 | #include <asm/smp.h> | |
21 | #include <asm/paca.h> | |
22 | #include <asm/machdep.h> | |
23 | #include <asm/cputable.h> | |
24 | #include <asm/firmware.h> | |
55190f88 BH |
25 | #include <asm/vdso_datapage.h> |
26 | #include <asm/cputhreads.h> | |
27 | #include <asm/xics.h> | |
243e2511 | 28 | #include <asm/xive.h> |
14a43e69 | 29 | #include <asm/opal.h> |
f2038911 | 30 | #include <asm/runlatch.h> |
2751b628 | 31 | #include <asm/code-patching.h> |
d4e58e59 | 32 | #include <asm/dbell.h> |
755563bc PM |
33 | #include <asm/kvm_ppc.h> |
34 | #include <asm/ppc-opcode.h> | |
a7cd88da | 35 | #include <asm/cpuidle.h> |
4145f358 BS |
36 | #include <asm/kexec.h> |
37 | #include <asm/reg.h> | |
19f8a5b5 | 38 | #include <asm/powernv.h> |
55190f88 BH |
39 | |
40 | #include "powernv.h" | |
41 | ||
344eb010 BH |
42 | #ifdef DEBUG |
43 | #include <asm/udbg.h> | |
44 | #define DBG(fmt...) udbg_printf(fmt) | |
45 | #else | |
46 | #define DBG(fmt...) | |
47 | #endif | |
48 | ||
061d19f2 | 49 | static void pnv_smp_setup_cpu(int cpu) |
55190f88 | 50 | { |
5080332c MN |
51 | /* |
52 | * P9 workaround for CI vector load (see traps.c), | |
53 | * enable the corresponding HMI interrupt | |
54 | */ | |
55 | if (pvr_version_is(PVR_POWER9)) | |
56 | mtspr(SPRN_HMEER, mfspr(SPRN_HMEER) | PPC_BIT(17)); | |
57 | ||
243e2511 BH |
58 | if (xive_enabled()) |
59 | xive_smp_setup_cpu(); | |
60 | else if (cpu != boot_cpuid) | |
55190f88 BH |
61 | xics_setup_cpu(); |
62 | } | |
63 | ||
e51df2c1 | 64 | static int pnv_smp_kick_cpu(int nr) |
14a43e69 | 65 | { |
76d98ab4 | 66 | unsigned int pcpu; |
2751b628 AB |
67 | unsigned long start_here = |
68 | __pa(ppc_function_entry(generic_secondary_smp_init)); | |
14a43e69 | 69 | long rc; |
e4d54f71 | 70 | uint8_t status; |
14a43e69 | 71 | |
c642af9c | 72 | if (nr < 0 || nr >= nr_cpu_ids) |
f8d0d5dc | 73 | return -EINVAL; |
14a43e69 | 74 | |
76d98ab4 | 75 | pcpu = get_hard_smp_processor_id(nr); |
b2b48584 | 76 | /* |
e4d54f71 | 77 | * If we already started or OPAL is not supported, we just |
b2b48584 | 78 | * kick the CPU via the PACA |
14a43e69 | 79 | */ |
d2e60075 | 80 | if (paca_ptrs[nr]->cpu_start || !firmware_has_feature(FW_FEATURE_OPAL)) |
b2b48584 BH |
81 | goto kick; |
82 | ||
83 | /* | |
84 | * At this point, the CPU can either be spinning on the way in | |
85 | * from kexec or be inside OPAL waiting to be started for the | |
86 | * first time. OPAL v3 allows us to query OPAL to know if it | |
87 | * has the CPUs, so we do that | |
88 | */ | |
e4d54f71 SS |
89 | rc = opal_query_cpu_status(pcpu, &status); |
90 | if (rc != OPAL_SUCCESS) { | |
91 | pr_warn("OPAL Error %ld querying CPU %d state\n", rc, nr); | |
92 | return -ENODEV; | |
93 | } | |
b2b48584 | 94 | |
e4d54f71 SS |
95 | /* |
96 | * Already started, just kick it, probably coming from | |
97 | * kexec and spinning | |
98 | */ | |
99 | if (status == OPAL_THREAD_STARTED) | |
100 | goto kick; | |
b2b48584 | 101 | |
e4d54f71 SS |
102 | /* |
103 | * Available/inactive, let's kick it | |
104 | */ | |
105 | if (status == OPAL_THREAD_INACTIVE) { | |
106 | pr_devel("OPAL: Starting CPU %d (HW 0x%x)...\n", nr, pcpu); | |
107 | rc = opal_start_cpu(pcpu, start_here); | |
108 | if (rc != OPAL_SUCCESS) { | |
109 | pr_warn("OPAL Error %ld starting CPU %d\n", rc, nr); | |
b2b48584 BH |
110 | return -ENODEV; |
111 | } | |
112 | } else { | |
113 | /* | |
e4d54f71 SS |
114 | * An unavailable CPU (or any other unknown status) |
115 | * shouldn't be started. It should also | |
116 | * not be in the possible map but currently it can | |
117 | * happen | |
b2b48584 | 118 | */ |
e4d54f71 SS |
119 | pr_devel("OPAL: CPU %d (HW 0x%x) is unavailable" |
120 | " (status %d)...\n", nr, pcpu, status); | |
121 | return -ENODEV; | |
14a43e69 | 122 | } |
e4d54f71 SS |
123 | |
124 | kick: | |
14a43e69 BH |
125 | return smp_generic_kick_cpu(nr); |
126 | } | |
127 | ||
344eb010 BH |
128 | #ifdef CONFIG_HOTPLUG_CPU |
129 | ||
130 | static int pnv_smp_cpu_disable(void) | |
131 | { | |
132 | int cpu = smp_processor_id(); | |
133 | ||
134 | /* This is identical to pSeries... might consolidate by | |
135 | * moving migrate_irqs_away to a ppc_md with default to | |
136 | * the generic fixup_irqs. --BenH. | |
137 | */ | |
138 | set_cpu_online(cpu, false); | |
139 | vdso_data->processorCount--; | |
140 | if (cpu == boot_cpuid) | |
141 | boot_cpuid = cpumask_any(cpu_online_mask); | |
243e2511 BH |
142 | if (xive_enabled()) |
143 | xive_smp_disable_cpu(); | |
144 | else | |
145 | xics_migrate_irqs_away(); | |
344eb010 BH |
146 | return 0; |
147 | } | |
148 | ||
149 | static void pnv_smp_cpu_kill_self(void) | |
150 | { | |
151 | unsigned int cpu; | |
755563bc | 152 | unsigned long srr1, wmask; |
19f8a5b5 | 153 | u64 lpcr_val; |
344eb010 | 154 | |
344eb010 | 155 | /* Standard hot unplug procedure */ |
2525db04 NP |
156 | /* |
157 | * This hard disables local interurpts, ensuring we have no lazy | |
158 | * irqs pending. | |
159 | */ | |
160 | WARN_ON(irqs_disabled()); | |
161 | hard_irq_disable(); | |
162 | WARN_ON(lazy_irq_pending()); | |
163 | ||
344eb010 BH |
164 | idle_task_exit(); |
165 | current->active_mm = NULL; /* for sanity */ | |
166 | cpu = smp_processor_id(); | |
167 | DBG("CPU%d offline\n", cpu); | |
168 | generic_set_cpu_dead(cpu); | |
169 | smp_wmb(); | |
170 | ||
755563bc PM |
171 | wmask = SRR1_WAKEMASK; |
172 | if (cpu_has_feature(CPU_FTR_ARCH_207S)) | |
173 | wmask = SRR1_WAKEMASK_P8; | |
174 | ||
19f8a5b5 PM |
175 | /* |
176 | * We don't want to take decrementer interrupts while we are | |
177 | * offline, so clear LPCR:PECE1. We keep PECE2 (and | |
178 | * LPCR_PECE_HVEE on P9) enabled so as to let IPIs in. | |
179 | * | |
180 | * If the CPU gets woken up by a special wakeup, ensure that | |
181 | * the SLW engine sets LPCR with decrementer bit cleared, else | |
182 | * the CPU will come back to the kernel due to a spurious | |
183 | * wakeup. | |
184 | */ | |
185 | lpcr_val = mfspr(SPRN_LPCR) & ~(u64)LPCR_PECE1; | |
186 | pnv_program_cpu_hotplug_lpcr(cpu, lpcr_val); | |
187 | ||
344eb010 | 188 | while (!generic_check_cpu_restart(cpu)) { |
53c656c4 PM |
189 | /* |
190 | * Clear IPI flag, since we don't handle IPIs while | |
191 | * offline, except for those when changing micro-threading | |
192 | * mode, which are handled explicitly below, and those | |
193 | * for coming online, which are handled via | |
194 | * generic_check_cpu_restart() calls. | |
195 | */ | |
196 | kvmppc_set_host_ipi(cpu, 0); | |
77b54e9f | 197 | |
a7cd88da | 198 | srr1 = pnv_cpu_offline(cpu); |
e2186023 | 199 | |
2525db04 NP |
200 | WARN_ON(lazy_irq_pending()); |
201 | ||
56548fc0 PM |
202 | /* |
203 | * If the SRR1 value indicates that we woke up due to | |
204 | * an external interrupt, then clear the interrupt. | |
205 | * We clear the interrupt before checking for the | |
206 | * reason, so as to avoid a race where we wake up for | |
207 | * some other reason, find nothing and clear the interrupt | |
208 | * just as some other cpu is sending us an interrupt. | |
209 | * If we returned from power7_nap as a result of | |
210 | * having finished executing in a KVM guest, then srr1 | |
211 | * contains 0. | |
212 | */ | |
53c656c4 | 213 | if (((srr1 & wmask) == SRR1_WAKEEE) || |
2525db04 | 214 | ((srr1 & wmask) == SRR1_WAKEHVI)) { |
243e2511 BH |
215 | if (cpu_has_feature(CPU_FTR_ARCH_300)) { |
216 | if (xive_enabled()) | |
217 | xive_flush_interrupt(); | |
218 | else | |
219 | icp_opal_flush_interrupt(); | |
220 | } else | |
9b256714 | 221 | icp_native_flush_interrupt(); |
755563bc PM |
222 | } else if ((srr1 & wmask) == SRR1_WAKEHDBELL) { |
223 | unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER); | |
224 | asm volatile(PPC_MSGCLR(%0) : : "r" (msg)); | |
4145f358 BS |
225 | } else if ((srr1 & wmask) == SRR1_WAKERESET) { |
226 | irq_set_pending_from_srr1(srr1); | |
227 | /* Does not return */ | |
56548fc0 | 228 | } |
4145f358 | 229 | |
53c656c4 | 230 | smp_mb(); |
e2186023 | 231 | |
4145f358 BS |
232 | /* |
233 | * For kdump kernels, we process the ipi and jump to | |
234 | * crash_ipi_callback | |
235 | */ | |
236 | if (kdump_in_progress()) { | |
237 | /* | |
238 | * If we got to this point, we've not used | |
239 | * NMI's, otherwise we would have gone | |
240 | * via the SRR1_WAKERESET path. We are | |
241 | * using regular IPI's for waking up offline | |
242 | * threads. | |
243 | */ | |
244 | struct pt_regs regs; | |
245 | ||
246 | ppc_save_regs(®s); | |
247 | crash_ipi_callback(®s); | |
248 | /* Does not return */ | |
249 | } | |
250 | ||
e2186023 ME |
251 | if (cpu_core_split_required()) |
252 | continue; | |
253 | ||
53c656c4 | 254 | if (srr1 && !generic_check_cpu_restart(cpu)) |
2525db04 NP |
255 | DBG("CPU%d Unexpected exit while offline srr1=%lx!\n", |
256 | cpu, srr1); | |
257 | ||
344eb010 | 258 | } |
9b256714 | 259 | |
19f8a5b5 PM |
260 | /* |
261 | * Re-enable decrementer interrupts in LPCR. | |
262 | * | |
263 | * Further, we want stop states to be woken up by decrementer | |
264 | * for non-hotplug cases. So program the LPCR via stop api as | |
265 | * well. | |
266 | */ | |
267 | lpcr_val = mfspr(SPRN_LPCR) | (u64)LPCR_PECE1; | |
268 | pnv_program_cpu_hotplug_lpcr(cpu, lpcr_val); | |
269 | ||
344eb010 BH |
270 | DBG("CPU%d coming online...\n", cpu); |
271 | } | |
272 | ||
273 | #endif /* CONFIG_HOTPLUG_CPU */ | |
274 | ||
d70a54e2 GK |
275 | static int pnv_cpu_bootable(unsigned int nr) |
276 | { | |
277 | /* | |
278 | * Starting with POWER8, the subcore logic relies on all threads of a | |
279 | * core being booted so that they can participate in split mode | |
280 | * switches. So on those machines we ignore the smt_enabled_at_boot | |
281 | * setting (smt-enabled on the kernel command line). | |
282 | */ | |
283 | if (cpu_has_feature(CPU_FTR_ARCH_207S)) | |
284 | return 1; | |
285 | ||
286 | return smp_generic_cpu_bootable(nr); | |
287 | } | |
288 | ||
243e2511 BH |
289 | static int pnv_smp_prepare_cpu(int cpu) |
290 | { | |
291 | if (xive_enabled()) | |
292 | return xive_smp_prepare_cpu(cpu); | |
293 | return 0; | |
294 | } | |
295 | ||
45b21cfe ME |
296 | /* Cause IPI as setup by the interrupt controller (xics or xive) */ |
297 | static void (*ic_cause_ipi)(int cpu); | |
298 | ||
b866cc21 NP |
299 | static void pnv_cause_ipi(int cpu) |
300 | { | |
301 | if (doorbell_try_core_ipi(cpu)) | |
302 | return; | |
303 | ||
45b21cfe | 304 | ic_cause_ipi(cpu); |
b866cc21 NP |
305 | } |
306 | ||
243e2511 BH |
307 | static void __init pnv_smp_probe(void) |
308 | { | |
309 | if (xive_enabled()) | |
310 | xive_smp_probe(); | |
311 | else | |
312 | xics_smp_probe(); | |
b866cc21 | 313 | |
6b3edefe | 314 | if (cpu_has_feature(CPU_FTR_DBELL)) { |
45b21cfe ME |
315 | ic_cause_ipi = smp_ops->cause_ipi; |
316 | WARN_ON(!ic_cause_ipi); | |
317 | ||
2bf1071a NP |
318 | if (cpu_has_feature(CPU_FTR_ARCH_300)) |
319 | smp_ops->cause_ipi = doorbell_global_ipi; | |
320 | else | |
6b3edefe | 321 | smp_ops->cause_ipi = pnv_cause_ipi; |
b866cc21 | 322 | } |
243e2511 BH |
323 | } |
324 | ||
e36d0a2e NP |
325 | static int pnv_system_reset_exception(struct pt_regs *regs) |
326 | { | |
327 | if (smp_handle_nmi_ipi(regs)) | |
328 | return 1; | |
329 | return 0; | |
330 | } | |
331 | ||
332 | static int pnv_cause_nmi_ipi(int cpu) | |
333 | { | |
334 | int64_t rc; | |
335 | ||
336 | if (cpu >= 0) { | |
ee03b9b4 NP |
337 | int h = get_hard_smp_processor_id(cpu); |
338 | ||
339 | if (opal_check_token(OPAL_QUIESCE)) | |
340 | opal_quiesce(QUIESCE_HOLD, h); | |
341 | ||
342 | rc = opal_signal_system_reset(h); | |
343 | ||
344 | if (opal_check_token(OPAL_QUIESCE)) | |
345 | opal_quiesce(QUIESCE_RESUME, h); | |
346 | ||
e36d0a2e NP |
347 | if (rc != OPAL_SUCCESS) |
348 | return 0; | |
349 | return 1; | |
350 | ||
351 | } else if (cpu == NMI_IPI_ALL_OTHERS) { | |
352 | bool success = true; | |
353 | int c; | |
354 | ||
ee03b9b4 NP |
355 | if (opal_check_token(OPAL_QUIESCE)) |
356 | opal_quiesce(QUIESCE_HOLD, -1); | |
e36d0a2e NP |
357 | |
358 | /* | |
359 | * We do not use broadcasts (yet), because it's not clear | |
360 | * exactly what semantics Linux wants or the firmware should | |
361 | * provide. | |
362 | */ | |
363 | for_each_online_cpu(c) { | |
364 | if (c == smp_processor_id()) | |
365 | continue; | |
366 | ||
367 | rc = opal_signal_system_reset( | |
368 | get_hard_smp_processor_id(c)); | |
369 | if (rc != OPAL_SUCCESS) | |
370 | success = false; | |
371 | } | |
ee03b9b4 NP |
372 | |
373 | if (opal_check_token(OPAL_QUIESCE)) | |
374 | opal_quiesce(QUIESCE_RESUME, -1); | |
375 | ||
e36d0a2e NP |
376 | if (success) |
377 | return 1; | |
378 | ||
379 | /* | |
380 | * Caller will fall back to doorbells, which may pick | |
381 | * up the remainders. | |
382 | */ | |
383 | } | |
384 | ||
385 | return 0; | |
386 | } | |
387 | ||
55190f88 | 388 | static struct smp_ops_t pnv_smp_ops = { |
b866cc21 NP |
389 | .message_pass = NULL, /* Use smp_muxed_ipi_message_pass */ |
390 | .cause_ipi = NULL, /* Filled at runtime by pnv_smp_probe() */ | |
c64af645 | 391 | .cause_nmi_ipi = NULL, |
243e2511 BH |
392 | .probe = pnv_smp_probe, |
393 | .prepare_cpu = pnv_smp_prepare_cpu, | |
14a43e69 | 394 | .kick_cpu = pnv_smp_kick_cpu, |
55190f88 | 395 | .setup_cpu = pnv_smp_setup_cpu, |
d70a54e2 | 396 | .cpu_bootable = pnv_cpu_bootable, |
344eb010 BH |
397 | #ifdef CONFIG_HOTPLUG_CPU |
398 | .cpu_disable = pnv_smp_cpu_disable, | |
399 | .cpu_die = generic_cpu_die, | |
400 | #endif /* CONFIG_HOTPLUG_CPU */ | |
55190f88 BH |
401 | }; |
402 | ||
403 | /* This is called very early during platform setup_arch */ | |
404 | void __init pnv_smp_init(void) | |
405 | { | |
e36d0a2e NP |
406 | if (opal_check_token(OPAL_SIGNAL_SYSTEM_RESET)) { |
407 | ppc_md.system_reset_exception = pnv_system_reset_exception; | |
408 | pnv_smp_ops.cause_nmi_ipi = pnv_cause_nmi_ipi; | |
409 | } | |
55190f88 BH |
410 | smp_ops = &pnv_smp_ops; |
411 | ||
344eb010 BH |
412 | #ifdef CONFIG_HOTPLUG_CPU |
413 | ppc_md.cpu_die = pnv_smp_cpu_kill_self; | |
4145f358 BS |
414 | #ifdef CONFIG_KEXEC_CORE |
415 | crash_wake_offline = 1; | |
416 | #endif | |
344eb010 | 417 | #endif |
55190f88 | 418 | } |