]>
Commit | Line | Data |
---|---|---|
5a61ef74 NP |
1 | /* |
2 | * Copyright 2017, Nicholas Piggin, IBM Corporation | |
3 | * Licensed under GPLv2. | |
4 | */ | |
5 | ||
6 | #define pr_fmt(fmt) "dt-cpu-ftrs: " fmt | |
7 | ||
8 | #include <linux/export.h> | |
9 | #include <linux/init.h> | |
10 | #include <linux/jump_label.h> | |
a2b05b7a | 11 | #include <linux/libfdt.h> |
5a61ef74 NP |
12 | #include <linux/memblock.h> |
13 | #include <linux/printk.h> | |
14 | #include <linux/sched.h> | |
15 | #include <linux/string.h> | |
16 | #include <linux/threads.h> | |
17 | ||
18 | #include <asm/cputable.h> | |
19 | #include <asm/dt_cpu_ftrs.h> | |
20 | #include <asm/mmu.h> | |
21 | #include <asm/oprofile_impl.h> | |
22 | #include <asm/prom.h> | |
23 | #include <asm/setup.h> | |
24 | ||
25 | ||
26 | /* Device-tree visible constants follow */ | |
27 | #define ISA_V2_07B 2070 | |
28 | #define ISA_V3_0B 3000 | |
29 | ||
30 | #define USABLE_PR (1U << 0) | |
31 | #define USABLE_OS (1U << 1) | |
32 | #define USABLE_HV (1U << 2) | |
33 | ||
34 | #define HV_SUPPORT_HFSCR (1U << 0) | |
35 | #define OS_SUPPORT_FSCR (1U << 0) | |
36 | ||
37 | /* For parsing, we define all bits set as "NONE" case */ | |
38 | #define HV_SUPPORT_NONE 0xffffffffU | |
39 | #define OS_SUPPORT_NONE 0xffffffffU | |
40 | ||
41 | struct dt_cpu_feature { | |
42 | const char *name; | |
43 | uint32_t isa; | |
44 | uint32_t usable_privilege; | |
45 | uint32_t hv_support; | |
46 | uint32_t os_support; | |
47 | uint32_t hfscr_bit_nr; | |
48 | uint32_t fscr_bit_nr; | |
49 | uint32_t hwcap_bit_nr; | |
50 | /* fdt parsing */ | |
51 | unsigned long node; | |
52 | int enabled; | |
53 | int disabled; | |
54 | }; | |
55 | ||
56 | #define CPU_FTRS_BASE \ | |
57 | (CPU_FTR_USE_TB | \ | |
58 | CPU_FTR_LWSYNC | \ | |
59 | CPU_FTR_FPU_UNAVAILABLE |\ | |
60 | CPU_FTR_NODSISRALIGN |\ | |
61 | CPU_FTR_NOEXECUTE |\ | |
62 | CPU_FTR_COHERENT_ICACHE | \ | |
63 | CPU_FTR_STCX_CHECKS_ADDRESS |\ | |
64 | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \ | |
65 | CPU_FTR_DAWR | \ | |
66 | CPU_FTR_ARCH_206 |\ | |
67 | CPU_FTR_ARCH_207S) | |
68 | ||
69 | #define MMU_FTRS_HASH_BASE (MMU_FTRS_POWER8) | |
70 | ||
71 | #define COMMON_USER_BASE (PPC_FEATURE_32 | PPC_FEATURE_64 | \ | |
72 | PPC_FEATURE_ARCH_2_06 |\ | |
73 | PPC_FEATURE_ICACHE_SNOOP) | |
74 | #define COMMON_USER2_BASE (PPC_FEATURE2_ARCH_2_07 | \ | |
75 | PPC_FEATURE2_ISEL) | |
76 | /* | |
77 | * Set up the base CPU | |
78 | */ | |
79 | ||
80 | extern void __flush_tlb_power8(unsigned int action); | |
81 | extern void __flush_tlb_power9(unsigned int action); | |
82 | extern long __machine_check_early_realmode_p8(struct pt_regs *regs); | |
83 | extern long __machine_check_early_realmode_p9(struct pt_regs *regs); | |
84 | ||
85 | static int hv_mode; | |
86 | ||
87 | static struct { | |
88 | u64 lpcr; | |
89 | u64 hfscr; | |
90 | u64 fscr; | |
91 | } system_registers; | |
92 | ||
93 | static void (*init_pmu_registers)(void); | |
94 | ||
95 | static void cpufeatures_flush_tlb(void) | |
96 | { | |
5a61ef74 NP |
97 | /* |
98 | * This is a temporary measure to keep equivalent TLB flush as the | |
99 | * cputable based setup code. | |
100 | */ | |
101 | switch (PVR_VER(mfspr(SPRN_PVR))) { | |
102 | case PVR_POWER8: | |
103 | case PVR_POWER8E: | |
104 | case PVR_POWER8NVL: | |
3b7af5c0 | 105 | __flush_tlb_power8(TLB_INVAL_SCOPE_GLOBAL); |
5a61ef74 NP |
106 | break; |
107 | case PVR_POWER9: | |
3b7af5c0 | 108 | __flush_tlb_power9(TLB_INVAL_SCOPE_GLOBAL); |
5a61ef74 NP |
109 | break; |
110 | default: | |
5a61ef74 NP |
111 | pr_err("unknown CPU version for boot TLB flush\n"); |
112 | break; | |
113 | } | |
5a61ef74 NP |
114 | } |
115 | ||
116 | static void __restore_cpu_cpufeatures(void) | |
117 | { | |
118 | /* | |
119 | * LPCR is restored by the power on engine already. It can be changed | |
120 | * after early init e.g., by radix enable, and we have no unified API | |
121 | * for saving and restoring such SPRs. | |
122 | * | |
123 | * This ->restore hook should really be removed from idle and register | |
124 | * restore moved directly into the idle restore code, because this code | |
125 | * doesn't know how idle is implemented or what it needs restored here. | |
126 | * | |
127 | * The best we can do to accommodate secondary boot and idle restore | |
128 | * for now is "or" LPCR with existing. | |
129 | */ | |
130 | ||
131 | mtspr(SPRN_LPCR, system_registers.lpcr | mfspr(SPRN_LPCR)); | |
132 | if (hv_mode) { | |
133 | mtspr(SPRN_LPID, 0); | |
134 | mtspr(SPRN_HFSCR, system_registers.hfscr); | |
135 | } | |
136 | mtspr(SPRN_FSCR, system_registers.fscr); | |
137 | ||
138 | if (init_pmu_registers) | |
139 | init_pmu_registers(); | |
140 | ||
141 | cpufeatures_flush_tlb(); | |
142 | } | |
143 | ||
144 | static char dt_cpu_name[64]; | |
145 | ||
146 | static struct cpu_spec __initdata base_cpu_spec = { | |
147 | .cpu_name = NULL, | |
148 | .cpu_features = CPU_FTRS_BASE, | |
149 | .cpu_user_features = COMMON_USER_BASE, | |
150 | .cpu_user_features2 = COMMON_USER2_BASE, | |
151 | .mmu_features = 0, | |
152 | .icache_bsize = 32, /* minimum block size, fixed by */ | |
153 | .dcache_bsize = 32, /* cache info init. */ | |
154 | .num_pmcs = 0, | |
155 | .pmc_type = PPC_PMC_DEFAULT, | |
156 | .oprofile_cpu_type = NULL, | |
157 | .oprofile_type = PPC_OPROFILE_INVALID, | |
158 | .cpu_setup = NULL, | |
159 | .cpu_restore = __restore_cpu_cpufeatures, | |
160 | .flush_tlb = NULL, | |
161 | .machine_check_early = NULL, | |
162 | .platform = NULL, | |
163 | }; | |
164 | ||
165 | static void __init cpufeatures_setup_cpu(void) | |
166 | { | |
167 | set_cur_cpu_spec(&base_cpu_spec); | |
168 | ||
169 | cur_cpu_spec->pvr_mask = -1; | |
170 | cur_cpu_spec->pvr_value = mfspr(SPRN_PVR); | |
171 | ||
172 | /* Initialize the base environment -- clear FSCR/HFSCR. */ | |
173 | hv_mode = !!(mfmsr() & MSR_HV); | |
174 | if (hv_mode) { | |
175 | /* CPU_FTR_HVMODE is used early in PACA setup */ | |
176 | cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; | |
177 | mtspr(SPRN_HFSCR, 0); | |
178 | } | |
179 | mtspr(SPRN_FSCR, 0); | |
180 | ||
181 | /* | |
182 | * LPCR does not get cleared, to match behaviour with secondaries | |
183 | * in __restore_cpu_cpufeatures. Once the idle code is fixed, this | |
184 | * could clear LPCR too. | |
185 | */ | |
186 | } | |
187 | ||
188 | static int __init feat_try_enable_unknown(struct dt_cpu_feature *f) | |
189 | { | |
190 | if (f->hv_support == HV_SUPPORT_NONE) { | |
191 | } else if (f->hv_support & HV_SUPPORT_HFSCR) { | |
192 | u64 hfscr = mfspr(SPRN_HFSCR); | |
193 | hfscr |= 1UL << f->hfscr_bit_nr; | |
194 | mtspr(SPRN_HFSCR, hfscr); | |
195 | } else { | |
196 | /* Does not have a known recipe */ | |
197 | return 0; | |
198 | } | |
199 | ||
200 | if (f->os_support == OS_SUPPORT_NONE) { | |
201 | } else if (f->os_support & OS_SUPPORT_FSCR) { | |
202 | u64 fscr = mfspr(SPRN_FSCR); | |
203 | fscr |= 1UL << f->fscr_bit_nr; | |
204 | mtspr(SPRN_FSCR, fscr); | |
205 | } else { | |
206 | /* Does not have a known recipe */ | |
207 | return 0; | |
208 | } | |
209 | ||
210 | if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { | |
211 | uint32_t word = f->hwcap_bit_nr / 32; | |
212 | uint32_t bit = f->hwcap_bit_nr % 32; | |
213 | ||
214 | if (word == 0) | |
215 | cur_cpu_spec->cpu_user_features |= 1U << bit; | |
216 | else if (word == 1) | |
217 | cur_cpu_spec->cpu_user_features2 |= 1U << bit; | |
218 | else | |
219 | pr_err("%s could not advertise to user (no hwcap bits)\n", f->name); | |
220 | } | |
221 | ||
222 | return 1; | |
223 | } | |
224 | ||
225 | static int __init feat_enable(struct dt_cpu_feature *f) | |
226 | { | |
227 | if (f->hv_support != HV_SUPPORT_NONE) { | |
228 | if (f->hfscr_bit_nr != -1) { | |
229 | u64 hfscr = mfspr(SPRN_HFSCR); | |
230 | hfscr |= 1UL << f->hfscr_bit_nr; | |
231 | mtspr(SPRN_HFSCR, hfscr); | |
232 | } | |
233 | } | |
234 | ||
235 | if (f->os_support != OS_SUPPORT_NONE) { | |
236 | if (f->fscr_bit_nr != -1) { | |
237 | u64 fscr = mfspr(SPRN_FSCR); | |
238 | fscr |= 1UL << f->fscr_bit_nr; | |
239 | mtspr(SPRN_FSCR, fscr); | |
240 | } | |
241 | } | |
242 | ||
243 | if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { | |
244 | uint32_t word = f->hwcap_bit_nr / 32; | |
245 | uint32_t bit = f->hwcap_bit_nr % 32; | |
246 | ||
247 | if (word == 0) | |
248 | cur_cpu_spec->cpu_user_features |= 1U << bit; | |
249 | else if (word == 1) | |
250 | cur_cpu_spec->cpu_user_features2 |= 1U << bit; | |
251 | else | |
252 | pr_err("CPU feature: %s could not advertise to user (no hwcap bits)\n", f->name); | |
253 | } | |
254 | ||
255 | return 1; | |
256 | } | |
257 | ||
258 | static int __init feat_disable(struct dt_cpu_feature *f) | |
259 | { | |
260 | return 0; | |
261 | } | |
262 | ||
263 | static int __init feat_enable_hv(struct dt_cpu_feature *f) | |
264 | { | |
265 | u64 lpcr; | |
266 | ||
267 | if (!hv_mode) { | |
268 | pr_err("CPU feature hypervisor present in device tree but HV mode not enabled in the CPU. Ignoring.\n"); | |
269 | return 0; | |
270 | } | |
271 | ||
272 | mtspr(SPRN_LPID, 0); | |
273 | ||
274 | lpcr = mfspr(SPRN_LPCR); | |
275 | lpcr &= ~LPCR_LPES0; /* HV external interrupts */ | |
276 | mtspr(SPRN_LPCR, lpcr); | |
277 | ||
278 | cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; | |
279 | ||
280 | return 1; | |
281 | } | |
282 | ||
283 | static int __init feat_enable_le(struct dt_cpu_feature *f) | |
284 | { | |
285 | cur_cpu_spec->cpu_user_features |= PPC_FEATURE_TRUE_LE; | |
286 | return 1; | |
287 | } | |
288 | ||
289 | static int __init feat_enable_smt(struct dt_cpu_feature *f) | |
290 | { | |
291 | cur_cpu_spec->cpu_features |= CPU_FTR_SMT; | |
292 | cur_cpu_spec->cpu_user_features |= PPC_FEATURE_SMT; | |
293 | return 1; | |
294 | } | |
295 | ||
296 | static int __init feat_enable_idle_nap(struct dt_cpu_feature *f) | |
297 | { | |
298 | u64 lpcr; | |
299 | ||
300 | /* Set PECE wakeup modes for ISA 207 */ | |
301 | lpcr = mfspr(SPRN_LPCR); | |
302 | lpcr |= LPCR_PECE0; | |
303 | lpcr |= LPCR_PECE1; | |
304 | lpcr |= LPCR_PECE2; | |
305 | mtspr(SPRN_LPCR, lpcr); | |
306 | ||
307 | return 1; | |
308 | } | |
309 | ||
310 | static int __init feat_enable_align_dsisr(struct dt_cpu_feature *f) | |
311 | { | |
312 | cur_cpu_spec->cpu_features &= ~CPU_FTR_NODSISRALIGN; | |
313 | ||
314 | return 1; | |
315 | } | |
316 | ||
317 | static int __init feat_enable_idle_stop(struct dt_cpu_feature *f) | |
318 | { | |
319 | u64 lpcr; | |
320 | ||
321 | /* Set PECE wakeup modes for ISAv3.0B */ | |
322 | lpcr = mfspr(SPRN_LPCR); | |
323 | lpcr |= LPCR_PECE0; | |
324 | lpcr |= LPCR_PECE1; | |
325 | lpcr |= LPCR_PECE2; | |
326 | mtspr(SPRN_LPCR, lpcr); | |
327 | ||
328 | return 1; | |
329 | } | |
330 | ||
331 | static int __init feat_enable_mmu_hash(struct dt_cpu_feature *f) | |
332 | { | |
333 | u64 lpcr; | |
334 | ||
335 | lpcr = mfspr(SPRN_LPCR); | |
336 | lpcr &= ~LPCR_ISL; | |
337 | ||
338 | /* VRMASD */ | |
339 | lpcr |= LPCR_VPM0; | |
340 | lpcr &= ~LPCR_VPM1; | |
341 | lpcr |= 0x10UL << LPCR_VRMASD_SH; /* L=1 LP=00 */ | |
342 | mtspr(SPRN_LPCR, lpcr); | |
343 | ||
344 | cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; | |
345 | cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; | |
346 | ||
347 | return 1; | |
348 | } | |
349 | ||
350 | static int __init feat_enable_mmu_hash_v3(struct dt_cpu_feature *f) | |
351 | { | |
352 | u64 lpcr; | |
353 | ||
354 | lpcr = mfspr(SPRN_LPCR); | |
355 | lpcr &= ~LPCR_ISL; | |
356 | mtspr(SPRN_LPCR, lpcr); | |
357 | ||
358 | cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; | |
359 | cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; | |
360 | ||
361 | return 1; | |
362 | } | |
363 | ||
364 | ||
365 | static int __init feat_enable_mmu_radix(struct dt_cpu_feature *f) | |
366 | { | |
367 | #ifdef CONFIG_PPC_RADIX_MMU | |
368 | cur_cpu_spec->mmu_features |= MMU_FTR_TYPE_RADIX; | |
369 | cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; | |
370 | cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; | |
371 | ||
372 | return 1; | |
373 | #endif | |
374 | return 0; | |
375 | } | |
376 | ||
377 | static int __init feat_enable_dscr(struct dt_cpu_feature *f) | |
378 | { | |
379 | u64 lpcr; | |
380 | ||
381 | feat_enable(f); | |
382 | ||
383 | lpcr = mfspr(SPRN_LPCR); | |
384 | lpcr &= ~LPCR_DPFD; | |
385 | lpcr |= (4UL << LPCR_DPFD_SH); | |
386 | mtspr(SPRN_LPCR, lpcr); | |
387 | ||
388 | return 1; | |
389 | } | |
390 | ||
391 | static void hfscr_pmu_enable(void) | |
392 | { | |
393 | u64 hfscr = mfspr(SPRN_HFSCR); | |
394 | hfscr |= PPC_BIT(60); | |
395 | mtspr(SPRN_HFSCR, hfscr); | |
396 | } | |
397 | ||
398 | static void init_pmu_power8(void) | |
399 | { | |
400 | if (hv_mode) { | |
401 | mtspr(SPRN_MMCRC, 0); | |
402 | mtspr(SPRN_MMCRH, 0); | |
403 | } | |
404 | ||
405 | mtspr(SPRN_MMCRA, 0); | |
406 | mtspr(SPRN_MMCR0, 0); | |
407 | mtspr(SPRN_MMCR1, 0); | |
408 | mtspr(SPRN_MMCR2, 0); | |
409 | mtspr(SPRN_MMCRS, 0); | |
410 | } | |
411 | ||
412 | static int __init feat_enable_mce_power8(struct dt_cpu_feature *f) | |
413 | { | |
414 | cur_cpu_spec->platform = "power8"; | |
415 | cur_cpu_spec->flush_tlb = __flush_tlb_power8; | |
416 | cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p8; | |
417 | ||
418 | return 1; | |
419 | } | |
420 | ||
421 | static int __init feat_enable_pmu_power8(struct dt_cpu_feature *f) | |
422 | { | |
423 | hfscr_pmu_enable(); | |
424 | ||
425 | init_pmu_power8(); | |
426 | init_pmu_registers = init_pmu_power8; | |
427 | ||
428 | cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; | |
429 | cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; | |
430 | if (pvr_version_is(PVR_POWER8E)) | |
431 | cur_cpu_spec->cpu_features |= CPU_FTR_PMAO_BUG; | |
432 | ||
433 | cur_cpu_spec->num_pmcs = 6; | |
434 | cur_cpu_spec->pmc_type = PPC_PMC_IBM; | |
435 | cur_cpu_spec->oprofile_cpu_type = "ppc64/power8"; | |
436 | ||
437 | return 1; | |
438 | } | |
439 | ||
440 | static void init_pmu_power9(void) | |
441 | { | |
442 | if (hv_mode) | |
443 | mtspr(SPRN_MMCRC, 0); | |
444 | ||
445 | mtspr(SPRN_MMCRA, 0); | |
446 | mtspr(SPRN_MMCR0, 0); | |
447 | mtspr(SPRN_MMCR1, 0); | |
448 | mtspr(SPRN_MMCR2, 0); | |
449 | } | |
450 | ||
451 | static int __init feat_enable_mce_power9(struct dt_cpu_feature *f) | |
452 | { | |
453 | cur_cpu_spec->platform = "power9"; | |
454 | cur_cpu_spec->flush_tlb = __flush_tlb_power9; | |
455 | cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p9; | |
456 | ||
457 | return 1; | |
458 | } | |
459 | ||
460 | static int __init feat_enable_pmu_power9(struct dt_cpu_feature *f) | |
461 | { | |
462 | hfscr_pmu_enable(); | |
463 | ||
464 | init_pmu_power9(); | |
465 | init_pmu_registers = init_pmu_power9; | |
466 | ||
467 | cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; | |
468 | cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; | |
469 | ||
470 | cur_cpu_spec->num_pmcs = 6; | |
471 | cur_cpu_spec->pmc_type = PPC_PMC_IBM; | |
472 | cur_cpu_spec->oprofile_cpu_type = "ppc64/power9"; | |
473 | ||
474 | return 1; | |
475 | } | |
476 | ||
477 | static int __init feat_enable_tm(struct dt_cpu_feature *f) | |
478 | { | |
479 | #ifdef CONFIG_PPC_TRANSACTIONAL_MEM | |
480 | feat_enable(f); | |
481 | cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_HTM_NOSC; | |
482 | return 1; | |
483 | #endif | |
484 | return 0; | |
485 | } | |
486 | ||
487 | static int __init feat_enable_fp(struct dt_cpu_feature *f) | |
488 | { | |
489 | feat_enable(f); | |
490 | cur_cpu_spec->cpu_features &= ~CPU_FTR_FPU_UNAVAILABLE; | |
491 | ||
492 | return 1; | |
493 | } | |
494 | ||
495 | static int __init feat_enable_vector(struct dt_cpu_feature *f) | |
496 | { | |
497 | #ifdef CONFIG_ALTIVEC | |
498 | feat_enable(f); | |
499 | cur_cpu_spec->cpu_features |= CPU_FTR_ALTIVEC; | |
500 | cur_cpu_spec->cpu_features |= CPU_FTR_VMX_COPY; | |
501 | cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_ALTIVEC; | |
502 | ||
503 | return 1; | |
504 | #endif | |
505 | return 0; | |
506 | } | |
507 | ||
508 | static int __init feat_enable_vsx(struct dt_cpu_feature *f) | |
509 | { | |
510 | #ifdef CONFIG_VSX | |
511 | feat_enable(f); | |
512 | cur_cpu_spec->cpu_features |= CPU_FTR_VSX; | |
513 | cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_VSX; | |
514 | ||
515 | return 1; | |
516 | #endif | |
517 | return 0; | |
518 | } | |
519 | ||
520 | static int __init feat_enable_purr(struct dt_cpu_feature *f) | |
521 | { | |
522 | cur_cpu_spec->cpu_features |= CPU_FTR_PURR | CPU_FTR_SPURR; | |
523 | ||
524 | return 1; | |
525 | } | |
526 | ||
527 | static int __init feat_enable_ebb(struct dt_cpu_feature *f) | |
528 | { | |
529 | /* | |
530 | * PPC_FEATURE2_EBB is enabled in PMU init code because it has | |
531 | * historically been related to the PMU facility. This may have | |
532 | * to be decoupled if EBB becomes more generic. For now, follow | |
533 | * existing convention. | |
534 | */ | |
535 | f->hwcap_bit_nr = -1; | |
536 | feat_enable(f); | |
537 | ||
538 | return 1; | |
539 | } | |
540 | ||
541 | static int __init feat_enable_dbell(struct dt_cpu_feature *f) | |
542 | { | |
543 | u64 lpcr; | |
544 | ||
545 | /* P9 has an HFSCR for privileged state */ | |
546 | feat_enable(f); | |
547 | ||
548 | cur_cpu_spec->cpu_features |= CPU_FTR_DBELL; | |
549 | ||
550 | lpcr = mfspr(SPRN_LPCR); | |
551 | lpcr |= LPCR_PECEDH; /* hyp doorbell wakeup */ | |
552 | mtspr(SPRN_LPCR, lpcr); | |
553 | ||
554 | return 1; | |
555 | } | |
556 | ||
557 | static int __init feat_enable_hvi(struct dt_cpu_feature *f) | |
558 | { | |
559 | u64 lpcr; | |
560 | ||
561 | /* | |
562 | * POWER9 XIVE interrupts including in OPAL XICS compatibility | |
563 | * are always delivered as hypervisor virtualization interrupts (HVI) | |
564 | * rather than EE. | |
565 | * | |
566 | * However LPES0 is not set here, in the chance that an EE does get | |
567 | * delivered to the host somehow, the EE handler would not expect it | |
568 | * to be delivered in LPES0 mode (e.g., using SRR[01]). This could | |
569 | * happen if there is a bug in interrupt controller code, or IC is | |
570 | * misconfigured in systemsim. | |
571 | */ | |
572 | ||
573 | lpcr = mfspr(SPRN_LPCR); | |
574 | lpcr |= LPCR_HVICE; /* enable hvi interrupts */ | |
575 | lpcr |= LPCR_HEIC; /* disable ee interrupts when MSR_HV */ | |
576 | lpcr |= LPCR_PECE_HVEE; /* hvi can wake from stop */ | |
577 | mtspr(SPRN_LPCR, lpcr); | |
578 | ||
579 | return 1; | |
580 | } | |
581 | ||
582 | static int __init feat_enable_large_ci(struct dt_cpu_feature *f) | |
583 | { | |
584 | cur_cpu_spec->mmu_features |= MMU_FTR_CI_LARGE_PAGE; | |
585 | ||
586 | return 1; | |
587 | } | |
588 | ||
589 | struct dt_cpu_feature_match { | |
590 | const char *name; | |
591 | int (*enable)(struct dt_cpu_feature *f); | |
592 | u64 cpu_ftr_bit_mask; | |
593 | }; | |
594 | ||
595 | static struct dt_cpu_feature_match __initdata | |
596 | dt_cpu_feature_match_table[] = { | |
597 | {"hypervisor", feat_enable_hv, 0}, | |
598 | {"big-endian", feat_enable, 0}, | |
599 | {"little-endian", feat_enable_le, CPU_FTR_REAL_LE}, | |
600 | {"smt", feat_enable_smt, 0}, | |
601 | {"interrupt-facilities", feat_enable, 0}, | |
602 | {"timer-facilities", feat_enable, 0}, | |
603 | {"timer-facilities-v3", feat_enable, 0}, | |
604 | {"debug-facilities", feat_enable, 0}, | |
605 | {"come-from-address-register", feat_enable, CPU_FTR_CFAR}, | |
606 | {"branch-tracing", feat_enable, 0}, | |
607 | {"floating-point", feat_enable_fp, 0}, | |
608 | {"vector", feat_enable_vector, 0}, | |
609 | {"vector-scalar", feat_enable_vsx, 0}, | |
610 | {"vector-scalar-v3", feat_enable, 0}, | |
611 | {"decimal-floating-point", feat_enable, 0}, | |
612 | {"decimal-integer", feat_enable, 0}, | |
613 | {"quadword-load-store", feat_enable, 0}, | |
614 | {"vector-crypto", feat_enable, 0}, | |
615 | {"mmu-hash", feat_enable_mmu_hash, 0}, | |
616 | {"mmu-radix", feat_enable_mmu_radix, 0}, | |
617 | {"mmu-hash-v3", feat_enable_mmu_hash_v3, 0}, | |
618 | {"virtual-page-class-key-protection", feat_enable, 0}, | |
619 | {"transactional-memory", feat_enable_tm, CPU_FTR_TM}, | |
620 | {"transactional-memory-v3", feat_enable_tm, 0}, | |
621 | {"idle-nap", feat_enable_idle_nap, 0}, | |
622 | {"alignment-interrupt-dsisr", feat_enable_align_dsisr, 0}, | |
623 | {"idle-stop", feat_enable_idle_stop, 0}, | |
624 | {"machine-check-power8", feat_enable_mce_power8, 0}, | |
625 | {"performance-monitor-power8", feat_enable_pmu_power8, 0}, | |
626 | {"data-stream-control-register", feat_enable_dscr, CPU_FTR_DSCR}, | |
627 | {"event-based-branch", feat_enable_ebb, 0}, | |
628 | {"target-address-register", feat_enable, 0}, | |
629 | {"branch-history-rolling-buffer", feat_enable, 0}, | |
630 | {"control-register", feat_enable, CPU_FTR_CTRL}, | |
631 | {"processor-control-facility", feat_enable_dbell, CPU_FTR_DBELL}, | |
632 | {"processor-control-facility-v3", feat_enable_dbell, CPU_FTR_DBELL}, | |
633 | {"processor-utilization-of-resources-register", feat_enable_purr, 0}, | |
5a61ef74 NP |
634 | {"no-execute", feat_enable, 0}, |
635 | {"strong-access-ordering", feat_enable, CPU_FTR_SAO}, | |
636 | {"cache-inhibited-large-page", feat_enable_large_ci, 0}, | |
c1807e3f | 637 | {"coprocessor-icswx", feat_enable, 0}, |
5a61ef74 NP |
638 | {"hypervisor-virtualization-interrupt", feat_enable_hvi, 0}, |
639 | {"program-priority-register", feat_enable, CPU_FTR_HAS_PPR}, | |
640 | {"wait", feat_enable, 0}, | |
641 | {"atomic-memory-operations", feat_enable, 0}, | |
642 | {"branch-v3", feat_enable, 0}, | |
643 | {"copy-paste", feat_enable, 0}, | |
644 | {"decimal-floating-point-v3", feat_enable, 0}, | |
645 | {"decimal-integer-v3", feat_enable, 0}, | |
646 | {"fixed-point-v3", feat_enable, 0}, | |
647 | {"floating-point-v3", feat_enable, 0}, | |
648 | {"group-start-register", feat_enable, 0}, | |
649 | {"pc-relative-addressing", feat_enable, 0}, | |
650 | {"machine-check-power9", feat_enable_mce_power9, 0}, | |
651 | {"performance-monitor-power9", feat_enable_pmu_power9, 0}, | |
652 | {"event-based-branch-v3", feat_enable, 0}, | |
653 | {"random-number-generator", feat_enable, 0}, | |
654 | {"system-call-vectored", feat_disable, 0}, | |
655 | {"trace-interrupt-v3", feat_enable, 0}, | |
656 | {"vector-v3", feat_enable, 0}, | |
657 | {"vector-binary128", feat_enable, 0}, | |
658 | {"vector-binary16", feat_enable, 0}, | |
659 | {"wait-v3", feat_enable, 0}, | |
660 | }; | |
661 | ||
a2b05b7a NP |
662 | static bool __initdata using_dt_cpu_ftrs; |
663 | static bool __initdata enable_unknown = true; | |
664 | ||
665 | static int __init dt_cpu_ftrs_parse(char *str) | |
666 | { | |
667 | if (!str) | |
668 | return 0; | |
669 | ||
670 | if (!strcmp(str, "off")) | |
671 | using_dt_cpu_ftrs = false; | |
672 | else if (!strcmp(str, "known")) | |
673 | enable_unknown = false; | |
674 | else | |
675 | return 1; | |
676 | ||
677 | return 0; | |
678 | } | |
679 | early_param("dt_cpu_ftrs", dt_cpu_ftrs_parse); | |
5a61ef74 NP |
680 | |
681 | static void __init cpufeatures_setup_start(u32 isa) | |
682 | { | |
683 | pr_info("setup for ISA %d\n", isa); | |
684 | ||
685 | if (isa >= 3000) { | |
686 | cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_300; | |
687 | cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_00; | |
688 | } | |
689 | } | |
690 | ||
691 | static bool __init cpufeatures_process_feature(struct dt_cpu_feature *f) | |
692 | { | |
693 | const struct dt_cpu_feature_match *m; | |
694 | bool known = false; | |
695 | int i; | |
696 | ||
697 | for (i = 0; i < ARRAY_SIZE(dt_cpu_feature_match_table); i++) { | |
698 | m = &dt_cpu_feature_match_table[i]; | |
699 | if (!strcmp(f->name, m->name)) { | |
700 | known = true; | |
701 | if (m->enable(f)) | |
702 | break; | |
703 | ||
704 | pr_info("not enabling: %s (disabled or unsupported by kernel)\n", | |
705 | f->name); | |
706 | return false; | |
707 | } | |
708 | } | |
709 | ||
a2b05b7a | 710 | if (!known && enable_unknown) { |
5a61ef74 NP |
711 | if (!feat_try_enable_unknown(f)) { |
712 | pr_info("not enabling: %s (unknown and unsupported by kernel)\n", | |
713 | f->name); | |
714 | return false; | |
715 | } | |
716 | } | |
717 | ||
718 | if (m->cpu_ftr_bit_mask) | |
719 | cur_cpu_spec->cpu_features |= m->cpu_ftr_bit_mask; | |
720 | ||
721 | if (known) | |
722 | pr_debug("enabling: %s\n", f->name); | |
723 | else | |
724 | pr_debug("enabling: %s (unknown)\n", f->name); | |
725 | ||
726 | return true; | |
727 | } | |
728 | ||
729 | static __init void cpufeatures_cpu_quirks(void) | |
730 | { | |
731 | int version = mfspr(SPRN_PVR); | |
732 | ||
733 | /* | |
734 | * Not all quirks can be derived from the cpufeatures device tree. | |
735 | */ | |
736 | if ((version & 0xffffff00) == 0x004e0100) | |
737 | cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD1; | |
4d6c51b1 ME |
738 | else if ((version & 0xffffefff) == 0x004e0201) |
739 | cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; | |
5a61ef74 NP |
740 | } |
741 | ||
742 | static void __init cpufeatures_setup_finished(void) | |
743 | { | |
744 | cpufeatures_cpu_quirks(); | |
745 | ||
746 | if (hv_mode && !(cur_cpu_spec->cpu_features & CPU_FTR_HVMODE)) { | |
747 | pr_err("hypervisor not present in device tree but HV mode is enabled in the CPU. Enabling.\n"); | |
748 | cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; | |
749 | } | |
750 | ||
751 | system_registers.lpcr = mfspr(SPRN_LPCR); | |
752 | system_registers.hfscr = mfspr(SPRN_HFSCR); | |
753 | system_registers.fscr = mfspr(SPRN_FSCR); | |
754 | ||
755 | cpufeatures_flush_tlb(); | |
756 | ||
757 | pr_info("final cpu/mmu features = 0x%016lx 0x%08x\n", | |
758 | cur_cpu_spec->cpu_features, cur_cpu_spec->mmu_features); | |
759 | } | |
760 | ||
a2b05b7a NP |
761 | static int __init disabled_on_cmdline(void) |
762 | { | |
763 | unsigned long root, chosen; | |
764 | const char *p; | |
765 | ||
766 | root = of_get_flat_dt_root(); | |
767 | chosen = of_get_flat_dt_subnode_by_name(root, "chosen"); | |
768 | if (chosen == -FDT_ERR_NOTFOUND) | |
769 | return false; | |
770 | ||
771 | p = of_get_flat_dt_prop(chosen, "bootargs", NULL); | |
772 | if (!p) | |
773 | return false; | |
774 | ||
775 | if (strstr(p, "dt_cpu_ftrs=off")) | |
776 | return true; | |
777 | ||
778 | return false; | |
779 | } | |
780 | ||
5a61ef74 NP |
781 | static int __init fdt_find_cpu_features(unsigned long node, const char *uname, |
782 | int depth, void *data) | |
783 | { | |
784 | if (of_flat_dt_is_compatible(node, "ibm,powerpc-cpu-features") | |
785 | && of_get_flat_dt_prop(node, "isa", NULL)) | |
786 | return 1; | |
787 | ||
788 | return 0; | |
789 | } | |
790 | ||
5a61ef74 NP |
791 | bool __init dt_cpu_ftrs_in_use(void) |
792 | { | |
793 | return using_dt_cpu_ftrs; | |
794 | } | |
795 | ||
796 | bool __init dt_cpu_ftrs_init(void *fdt) | |
797 | { | |
a2b05b7a NP |
798 | using_dt_cpu_ftrs = false; |
799 | ||
5a61ef74 NP |
800 | /* Setup and verify the FDT, if it fails we just bail */ |
801 | if (!early_init_dt_verify(fdt)) | |
802 | return false; | |
803 | ||
804 | if (!of_scan_flat_dt(fdt_find_cpu_features, NULL)) | |
805 | return false; | |
806 | ||
a2b05b7a NP |
807 | if (disabled_on_cmdline()) |
808 | return false; | |
809 | ||
5a61ef74 NP |
810 | cpufeatures_setup_cpu(); |
811 | ||
812 | using_dt_cpu_ftrs = true; | |
813 | return true; | |
814 | } | |
815 | ||
816 | static int nr_dt_cpu_features; | |
817 | static struct dt_cpu_feature *dt_cpu_features; | |
818 | ||
819 | static int __init process_cpufeatures_node(unsigned long node, | |
820 | const char *uname, int i) | |
821 | { | |
822 | const __be32 *prop; | |
823 | struct dt_cpu_feature *f; | |
824 | int len; | |
825 | ||
826 | f = &dt_cpu_features[i]; | |
827 | memset(f, 0, sizeof(struct dt_cpu_feature)); | |
828 | ||
829 | f->node = node; | |
830 | ||
831 | f->name = uname; | |
832 | ||
833 | prop = of_get_flat_dt_prop(node, "isa", &len); | |
834 | if (!prop) { | |
835 | pr_warn("%s: missing isa property\n", uname); | |
836 | return 0; | |
837 | } | |
838 | f->isa = be32_to_cpup(prop); | |
839 | ||
840 | prop = of_get_flat_dt_prop(node, "usable-privilege", &len); | |
841 | if (!prop) { | |
842 | pr_warn("%s: missing usable-privilege property", uname); | |
843 | return 0; | |
844 | } | |
845 | f->usable_privilege = be32_to_cpup(prop); | |
846 | ||
847 | prop = of_get_flat_dt_prop(node, "hv-support", &len); | |
848 | if (prop) | |
849 | f->hv_support = be32_to_cpup(prop); | |
850 | else | |
851 | f->hv_support = HV_SUPPORT_NONE; | |
852 | ||
853 | prop = of_get_flat_dt_prop(node, "os-support", &len); | |
854 | if (prop) | |
855 | f->os_support = be32_to_cpup(prop); | |
856 | else | |
857 | f->os_support = OS_SUPPORT_NONE; | |
858 | ||
859 | prop = of_get_flat_dt_prop(node, "hfscr-bit-nr", &len); | |
860 | if (prop) | |
861 | f->hfscr_bit_nr = be32_to_cpup(prop); | |
862 | else | |
863 | f->hfscr_bit_nr = -1; | |
864 | prop = of_get_flat_dt_prop(node, "fscr-bit-nr", &len); | |
865 | if (prop) | |
866 | f->fscr_bit_nr = be32_to_cpup(prop); | |
867 | else | |
868 | f->fscr_bit_nr = -1; | |
869 | prop = of_get_flat_dt_prop(node, "hwcap-bit-nr", &len); | |
870 | if (prop) | |
871 | f->hwcap_bit_nr = be32_to_cpup(prop); | |
872 | else | |
873 | f->hwcap_bit_nr = -1; | |
874 | ||
875 | if (f->usable_privilege & USABLE_HV) { | |
876 | if (!(mfmsr() & MSR_HV)) { | |
877 | pr_warn("%s: HV feature passed to guest\n", uname); | |
878 | return 0; | |
879 | } | |
880 | ||
881 | if (f->hv_support == HV_SUPPORT_NONE && f->hfscr_bit_nr != -1) { | |
882 | pr_warn("%s: unwanted hfscr_bit_nr\n", uname); | |
883 | return 0; | |
884 | } | |
885 | ||
886 | if (f->hv_support == HV_SUPPORT_HFSCR) { | |
887 | if (f->hfscr_bit_nr == -1) { | |
888 | pr_warn("%s: missing hfscr_bit_nr\n", uname); | |
889 | return 0; | |
890 | } | |
891 | } | |
892 | } else { | |
893 | if (f->hv_support != HV_SUPPORT_NONE || f->hfscr_bit_nr != -1) { | |
894 | pr_warn("%s: unwanted hv_support/hfscr_bit_nr\n", uname); | |
895 | return 0; | |
896 | } | |
897 | } | |
898 | ||
899 | if (f->usable_privilege & USABLE_OS) { | |
900 | if (f->os_support == OS_SUPPORT_NONE && f->fscr_bit_nr != -1) { | |
901 | pr_warn("%s: unwanted fscr_bit_nr\n", uname); | |
902 | return 0; | |
903 | } | |
904 | ||
905 | if (f->os_support == OS_SUPPORT_FSCR) { | |
906 | if (f->fscr_bit_nr == -1) { | |
907 | pr_warn("%s: missing fscr_bit_nr\n", uname); | |
908 | return 0; | |
909 | } | |
910 | } | |
911 | } else { | |
912 | if (f->os_support != OS_SUPPORT_NONE || f->fscr_bit_nr != -1) { | |
913 | pr_warn("%s: unwanted os_support/fscr_bit_nr\n", uname); | |
914 | return 0; | |
915 | } | |
916 | } | |
917 | ||
918 | if (!(f->usable_privilege & USABLE_PR)) { | |
919 | if (f->hwcap_bit_nr != -1) { | |
920 | pr_warn("%s: unwanted hwcap_bit_nr\n", uname); | |
921 | return 0; | |
922 | } | |
923 | } | |
924 | ||
925 | /* Do all the independent features in the first pass */ | |
926 | if (!of_get_flat_dt_prop(node, "dependencies", &len)) { | |
927 | if (cpufeatures_process_feature(f)) | |
928 | f->enabled = 1; | |
929 | else | |
930 | f->disabled = 1; | |
931 | } | |
932 | ||
933 | return 0; | |
934 | } | |
935 | ||
936 | static void __init cpufeatures_deps_enable(struct dt_cpu_feature *f) | |
937 | { | |
938 | const __be32 *prop; | |
939 | int len; | |
940 | int nr_deps; | |
941 | int i; | |
942 | ||
943 | if (f->enabled || f->disabled) | |
944 | return; | |
945 | ||
946 | prop = of_get_flat_dt_prop(f->node, "dependencies", &len); | |
947 | if (!prop) { | |
948 | pr_warn("%s: missing dependencies property", f->name); | |
949 | return; | |
950 | } | |
951 | ||
952 | nr_deps = len / sizeof(int); | |
953 | ||
954 | for (i = 0; i < nr_deps; i++) { | |
955 | unsigned long phandle = be32_to_cpu(prop[i]); | |
956 | int j; | |
957 | ||
958 | for (j = 0; j < nr_dt_cpu_features; j++) { | |
959 | struct dt_cpu_feature *d = &dt_cpu_features[j]; | |
960 | ||
961 | if (of_get_flat_dt_phandle(d->node) == phandle) { | |
962 | cpufeatures_deps_enable(d); | |
963 | if (d->disabled) { | |
964 | f->disabled = 1; | |
965 | return; | |
966 | } | |
967 | } | |
968 | } | |
969 | } | |
970 | ||
971 | if (cpufeatures_process_feature(f)) | |
972 | f->enabled = 1; | |
973 | else | |
974 | f->disabled = 1; | |
975 | } | |
976 | ||
977 | static int __init scan_cpufeatures_subnodes(unsigned long node, | |
978 | const char *uname, | |
979 | void *data) | |
980 | { | |
981 | int *count = data; | |
982 | ||
983 | process_cpufeatures_node(node, uname, *count); | |
984 | ||
985 | (*count)++; | |
986 | ||
987 | return 0; | |
988 | } | |
989 | ||
990 | static int __init count_cpufeatures_subnodes(unsigned long node, | |
991 | const char *uname, | |
992 | void *data) | |
993 | { | |
994 | int *count = data; | |
995 | ||
996 | (*count)++; | |
997 | ||
998 | return 0; | |
999 | } | |
1000 | ||
1001 | static int __init dt_cpu_ftrs_scan_callback(unsigned long node, const char | |
1002 | *uname, int depth, void *data) | |
1003 | { | |
1004 | const __be32 *prop; | |
1005 | int count, i; | |
1006 | u32 isa; | |
1007 | ||
1008 | /* We are scanning "ibm,powerpc-cpu-features" nodes only */ | |
1009 | if (!of_flat_dt_is_compatible(node, "ibm,powerpc-cpu-features")) | |
1010 | return 0; | |
1011 | ||
1012 | prop = of_get_flat_dt_prop(node, "isa", NULL); | |
1013 | if (!prop) | |
1014 | /* We checked before, "can't happen" */ | |
1015 | return 0; | |
1016 | ||
1017 | isa = be32_to_cpup(prop); | |
1018 | ||
1019 | /* Count and allocate space for cpu features */ | |
1020 | of_scan_flat_dt_subnodes(node, count_cpufeatures_subnodes, | |
1021 | &nr_dt_cpu_features); | |
1022 | dt_cpu_features = __va( | |
1023 | memblock_alloc(sizeof(struct dt_cpu_feature)* | |
1024 | nr_dt_cpu_features, PAGE_SIZE)); | |
1025 | ||
1026 | cpufeatures_setup_start(isa); | |
1027 | ||
1028 | /* Scan nodes into dt_cpu_features and enable those without deps */ | |
1029 | count = 0; | |
1030 | of_scan_flat_dt_subnodes(node, scan_cpufeatures_subnodes, &count); | |
1031 | ||
1032 | /* Recursive enable remaining features with dependencies */ | |
1033 | for (i = 0; i < nr_dt_cpu_features; i++) { | |
1034 | struct dt_cpu_feature *f = &dt_cpu_features[i]; | |
1035 | ||
1036 | cpufeatures_deps_enable(f); | |
1037 | } | |
1038 | ||
1039 | prop = of_get_flat_dt_prop(node, "display-name", NULL); | |
1040 | if (prop && strlen((char *)prop) != 0) { | |
1041 | strlcpy(dt_cpu_name, (char *)prop, sizeof(dt_cpu_name)); | |
1042 | cur_cpu_spec->cpu_name = dt_cpu_name; | |
1043 | } | |
1044 | ||
1045 | cpufeatures_setup_finished(); | |
1046 | ||
1047 | memblock_free(__pa(dt_cpu_features), | |
1048 | sizeof(struct dt_cpu_feature)*nr_dt_cpu_features); | |
1049 | ||
1050 | return 0; | |
1051 | } | |
1052 | ||
1053 | void __init dt_cpu_ftrs_scan(void) | |
1054 | { | |
a2b05b7a NP |
1055 | if (!using_dt_cpu_ftrs) |
1056 | return; | |
1057 | ||
5a61ef74 NP |
1058 | of_scan_flat_dt(dt_cpu_ftrs_scan_callback, NULL); |
1059 | } |