]>
Commit | Line | Data |
---|---|---|
51c52e86 ME |
1 | /* |
2 | * Copyright (C) 2001 Ben. Herrenschmidt (benh@kernel.crashing.org) | |
3 | * | |
4 | * Modifications for ppc64: | |
5 | * Copyright (C) 2003 Dave Engebretsen <engebret@us.ibm.com> | |
6 | * | |
7 | * Copyright 2008 Michael Ellerman, IBM Corporation. | |
8 | * | |
9 | * This program is free software; you can redistribute it and/or | |
10 | * modify it under the terms of the GNU General Public License | |
11 | * as published by the Free Software Foundation; either version | |
12 | * 2 of the License, or (at your option) any later version. | |
13 | */ | |
14 | ||
3880ecb0 | 15 | #include <linux/types.h> |
309b315b | 16 | #include <linux/jump_label.h> |
51c52e86 | 17 | #include <linux/kernel.h> |
362e7701 ME |
18 | #include <linux/string.h> |
19 | #include <linux/init.h> | |
589ee628 | 20 | #include <linux/sched/mm.h> |
51c52e86 ME |
21 | #include <asm/cputable.h> |
22 | #include <asm/code-patching.h> | |
d715e433 AB |
23 | #include <asm/page.h> |
24 | #include <asm/sections.h> | |
9402c684 BH |
25 | #include <asm/setup.h> |
26 | #include <asm/firmware.h> | |
51c52e86 ME |
27 | |
28 | struct fixup_entry { | |
29 | unsigned long mask; | |
30 | unsigned long value; | |
31 | long start_off; | |
32 | long end_off; | |
fac23fe4 ME |
33 | long alt_start_off; |
34 | long alt_end_off; | |
51c52e86 ME |
35 | }; |
36 | ||
9b1a735d | 37 | static unsigned int *calc_addr(struct fixup_entry *fcur, long offset) |
51c52e86 | 38 | { |
9b1a735d ME |
39 | /* |
40 | * We store the offset to the code as a negative offset from | |
41 | * the start of the alt_entry, to support the VDSO. This | |
42 | * routine converts that back into an actual address. | |
43 | */ | |
44 | return (unsigned int *)((unsigned long)fcur + offset); | |
45 | } | |
46 | ||
47 | static int patch_alt_instruction(unsigned int *src, unsigned int *dest, | |
48 | unsigned int *alt_start, unsigned int *alt_end) | |
49 | { | |
50 | unsigned int instr; | |
51 | ||
52 | instr = *src; | |
53 | ||
54 | if (instr_is_relative_branch(*src)) { | |
55 | unsigned int *target = (unsigned int *)branch_target(src); | |
56 | ||
57 | /* Branch within the section doesn't need translating */ | |
58 | if (target < alt_start || target >= alt_end) { | |
59 | instr = translate_branch(dest, src); | |
60 | if (!instr) | |
61 | return 1; | |
62 | } | |
63 | } | |
64 | ||
65 | patch_instruction(dest, instr); | |
66 | ||
67 | return 0; | |
68 | } | |
69 | ||
70 | static int patch_feature_section(unsigned long value, struct fixup_entry *fcur) | |
71 | { | |
72 | unsigned int *start, *end, *alt_start, *alt_end, *src, *dest; | |
73 | ||
74 | start = calc_addr(fcur, fcur->start_off); | |
75 | end = calc_addr(fcur, fcur->end_off); | |
76 | alt_start = calc_addr(fcur, fcur->alt_start_off); | |
77 | alt_end = calc_addr(fcur, fcur->alt_end_off); | |
78 | ||
79 | if ((alt_end - alt_start) > (end - start)) | |
80 | return 1; | |
51c52e86 ME |
81 | |
82 | if ((value & fcur->mask) == fcur->value) | |
9b1a735d | 83 | return 0; |
51c52e86 | 84 | |
9b1a735d ME |
85 | src = alt_start; |
86 | dest = start; | |
51c52e86 | 87 | |
9b1a735d ME |
88 | for (; src < alt_end; src++, dest++) { |
89 | if (patch_alt_instruction(src, dest, alt_start, alt_end)) | |
90 | return 1; | |
51c52e86 | 91 | } |
9b1a735d ME |
92 | |
93 | for (; dest < end; dest++) | |
16c57b36 | 94 | patch_instruction(dest, PPC_INST_NOP); |
9b1a735d ME |
95 | |
96 | return 0; | |
51c52e86 ME |
97 | } |
98 | ||
99 | void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end) | |
100 | { | |
101 | struct fixup_entry *fcur, *fend; | |
102 | ||
103 | fcur = fixup_start; | |
104 | fend = fixup_end; | |
105 | ||
9b1a735d ME |
106 | for (; fcur < fend; fcur++) { |
107 | if (patch_feature_section(value, fcur)) { | |
1856c020 | 108 | WARN_ON(1); |
9b1a735d ME |
109 | printk("Unable to patch feature section at %p - %p" \ |
110 | " with %p - %p\n", | |
111 | calc_addr(fcur, fcur->start_off), | |
112 | calc_addr(fcur, fcur->end_off), | |
113 | calc_addr(fcur, fcur->alt_start_off), | |
114 | calc_addr(fcur, fcur->alt_end_off)); | |
115 | } | |
116 | } | |
51c52e86 | 117 | } |
362e7701 | 118 | |
2d1b2027 KG |
119 | void do_lwsync_fixups(unsigned long value, void *fixup_start, void *fixup_end) |
120 | { | |
3d98ffbf BH |
121 | long *start, *end; |
122 | unsigned int *dest; | |
2d1b2027 KG |
123 | |
124 | if (!(value & CPU_FTR_LWSYNC)) | |
125 | return ; | |
126 | ||
127 | start = fixup_start; | |
128 | end = fixup_end; | |
129 | ||
130 | for (; start < end; start++) { | |
131 | dest = (void *)start + *start; | |
16c57b36 | 132 | patch_instruction(dest, PPC_INST_LWSYNC); |
2d1b2027 KG |
133 | } |
134 | } | |
135 | ||
9402c684 | 136 | static void do_final_fixups(void) |
d715e433 AB |
137 | { |
138 | #if defined(CONFIG_PPC64) && defined(CONFIG_RELOCATABLE) | |
139 | int *src, *dest; | |
140 | unsigned long length; | |
141 | ||
142 | if (PHYSICAL_START == 0) | |
143 | return; | |
144 | ||
145 | src = (int *)(KERNELBASE + PHYSICAL_START); | |
146 | dest = (int *)KERNELBASE; | |
147 | length = (__end_interrupts - _stext) / sizeof(int); | |
148 | ||
149 | while (length--) { | |
150 | patch_instruction(dest, *src); | |
151 | src++; | |
152 | dest++; | |
153 | } | |
154 | #endif | |
155 | } | |
156 | ||
a28e46f1 ME |
157 | static unsigned long __initdata saved_cpu_features; |
158 | static unsigned int __initdata saved_mmu_features; | |
159 | #ifdef CONFIG_PPC64 | |
160 | static unsigned long __initdata saved_firmware_features; | |
161 | #endif | |
162 | ||
163 | void __init apply_feature_fixups(void) | |
9402c684 | 164 | { |
2c0f9951 | 165 | struct cpu_spec *spec = PTRRELOC(*PTRRELOC(&cur_cpu_spec)); |
9402c684 | 166 | |
a28e46f1 ME |
167 | *PTRRELOC(&saved_cpu_features) = spec->cpu_features; |
168 | *PTRRELOC(&saved_mmu_features) = spec->mmu_features; | |
169 | ||
9402c684 BH |
170 | /* |
171 | * Apply the CPU-specific and firmware specific fixups to kernel text | |
172 | * (nop out sections not relevant to this CPU or this firmware). | |
173 | */ | |
174 | do_feature_fixups(spec->cpu_features, | |
175 | PTRRELOC(&__start___ftr_fixup), | |
176 | PTRRELOC(&__stop___ftr_fixup)); | |
177 | ||
178 | do_feature_fixups(spec->mmu_features, | |
179 | PTRRELOC(&__start___mmu_ftr_fixup), | |
180 | PTRRELOC(&__stop___mmu_ftr_fixup)); | |
181 | ||
182 | do_lwsync_fixups(spec->cpu_features, | |
183 | PTRRELOC(&__start___lwsync_fixup), | |
184 | PTRRELOC(&__stop___lwsync_fixup)); | |
185 | ||
186 | #ifdef CONFIG_PPC64 | |
a28e46f1 | 187 | saved_firmware_features = powerpc_firmware_features; |
9402c684 BH |
188 | do_feature_fixups(powerpc_firmware_features, |
189 | &__start___fw_ftr_fixup, &__stop___fw_ftr_fixup); | |
190 | #endif | |
191 | do_final_fixups(); | |
97f6e0cc | 192 | } |
309b315b | 193 | |
97f6e0cc BH |
194 | void __init setup_feature_keys(void) |
195 | { | |
309b315b AK |
196 | /* |
197 | * Initialise jump label. This causes all the cpu/mmu_has_feature() | |
198 | * checks to take on their correct polarity based on the current set of | |
199 | * CPU/MMU features. | |
200 | */ | |
201 | jump_label_init(); | |
4db73271 | 202 | cpu_feature_keys_init(); |
c12e6f24 | 203 | mmu_feature_keys_init(); |
9402c684 BH |
204 | } |
205 | ||
a28e46f1 ME |
206 | static int __init check_features(void) |
207 | { | |
208 | WARN(saved_cpu_features != cur_cpu_spec->cpu_features, | |
209 | "CPU features changed after feature patching!\n"); | |
210 | WARN(saved_mmu_features != cur_cpu_spec->mmu_features, | |
211 | "MMU features changed after feature patching!\n"); | |
212 | #ifdef CONFIG_PPC64 | |
213 | WARN(saved_firmware_features != powerpc_firmware_features, | |
214 | "Firmware features changed after feature patching!\n"); | |
215 | #endif | |
216 | ||
217 | return 0; | |
218 | } | |
219 | late_initcall(check_features); | |
220 | ||
362e7701 ME |
221 | #ifdef CONFIG_FTR_FIXUP_SELFTEST |
222 | ||
223 | #define check(x) \ | |
224 | if (!(x)) printk("feature-fixups: test failed at line %d\n", __LINE__); | |
225 | ||
226 | /* This must be after the text it fixes up, vmlinux.lds.S enforces that atm */ | |
227 | static struct fixup_entry fixup; | |
228 | ||
229 | static long calc_offset(struct fixup_entry *entry, unsigned int *p) | |
230 | { | |
231 | return (unsigned long)p - (unsigned long)entry; | |
232 | } | |
233 | ||
e51df2c1 | 234 | static void test_basic_patching(void) |
362e7701 | 235 | { |
c69a48cd DA |
236 | extern unsigned int ftr_fixup_test1[]; |
237 | extern unsigned int end_ftr_fixup_test1[]; | |
238 | extern unsigned int ftr_fixup_test1_orig[]; | |
239 | extern unsigned int ftr_fixup_test1_expected[]; | |
240 | int size = end_ftr_fixup_test1 - ftr_fixup_test1; | |
362e7701 ME |
241 | |
242 | fixup.value = fixup.mask = 8; | |
c69a48cd DA |
243 | fixup.start_off = calc_offset(&fixup, ftr_fixup_test1 + 1); |
244 | fixup.end_off = calc_offset(&fixup, ftr_fixup_test1 + 2); | |
362e7701 ME |
245 | fixup.alt_start_off = fixup.alt_end_off = 0; |
246 | ||
247 | /* Sanity check */ | |
c69a48cd | 248 | check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0); |
362e7701 ME |
249 | |
250 | /* Check we don't patch if the value matches */ | |
251 | patch_feature_section(8, &fixup); | |
c69a48cd | 252 | check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0); |
362e7701 ME |
253 | |
254 | /* Check we do patch if the value doesn't match */ | |
255 | patch_feature_section(0, &fixup); | |
c69a48cd | 256 | check(memcmp(ftr_fixup_test1, ftr_fixup_test1_expected, size) == 0); |
362e7701 ME |
257 | |
258 | /* Check we do patch if the mask doesn't match */ | |
c69a48cd DA |
259 | memcpy(ftr_fixup_test1, ftr_fixup_test1_orig, size); |
260 | check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0); | |
362e7701 | 261 | patch_feature_section(~8, &fixup); |
c69a48cd | 262 | check(memcmp(ftr_fixup_test1, ftr_fixup_test1_expected, size) == 0); |
362e7701 ME |
263 | } |
264 | ||
265 | static void test_alternative_patching(void) | |
266 | { | |
c69a48cd DA |
267 | extern unsigned int ftr_fixup_test2[]; |
268 | extern unsigned int end_ftr_fixup_test2[]; | |
269 | extern unsigned int ftr_fixup_test2_orig[]; | |
270 | extern unsigned int ftr_fixup_test2_alt[]; | |
271 | extern unsigned int ftr_fixup_test2_expected[]; | |
272 | int size = end_ftr_fixup_test2 - ftr_fixup_test2; | |
362e7701 ME |
273 | |
274 | fixup.value = fixup.mask = 0xF; | |
c69a48cd DA |
275 | fixup.start_off = calc_offset(&fixup, ftr_fixup_test2 + 1); |
276 | fixup.end_off = calc_offset(&fixup, ftr_fixup_test2 + 2); | |
277 | fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_test2_alt); | |
278 | fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_test2_alt + 1); | |
362e7701 ME |
279 | |
280 | /* Sanity check */ | |
c69a48cd | 281 | check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0); |
362e7701 ME |
282 | |
283 | /* Check we don't patch if the value matches */ | |
284 | patch_feature_section(0xF, &fixup); | |
c69a48cd | 285 | check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0); |
362e7701 ME |
286 | |
287 | /* Check we do patch if the value doesn't match */ | |
288 | patch_feature_section(0, &fixup); | |
c69a48cd | 289 | check(memcmp(ftr_fixup_test2, ftr_fixup_test2_expected, size) == 0); |
362e7701 ME |
290 | |
291 | /* Check we do patch if the mask doesn't match */ | |
c69a48cd DA |
292 | memcpy(ftr_fixup_test2, ftr_fixup_test2_orig, size); |
293 | check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0); | |
362e7701 | 294 | patch_feature_section(~0xF, &fixup); |
c69a48cd | 295 | check(memcmp(ftr_fixup_test2, ftr_fixup_test2_expected, size) == 0); |
362e7701 ME |
296 | } |
297 | ||
298 | static void test_alternative_case_too_big(void) | |
299 | { | |
c69a48cd DA |
300 | extern unsigned int ftr_fixup_test3[]; |
301 | extern unsigned int end_ftr_fixup_test3[]; | |
302 | extern unsigned int ftr_fixup_test3_orig[]; | |
303 | extern unsigned int ftr_fixup_test3_alt[]; | |
304 | int size = end_ftr_fixup_test3 - ftr_fixup_test3; | |
362e7701 ME |
305 | |
306 | fixup.value = fixup.mask = 0xC; | |
c69a48cd DA |
307 | fixup.start_off = calc_offset(&fixup, ftr_fixup_test3 + 1); |
308 | fixup.end_off = calc_offset(&fixup, ftr_fixup_test3 + 2); | |
309 | fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_test3_alt); | |
310 | fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_test3_alt + 2); | |
362e7701 ME |
311 | |
312 | /* Sanity check */ | |
c69a48cd | 313 | check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); |
362e7701 ME |
314 | |
315 | /* Expect nothing to be patched, and the error returned to us */ | |
316 | check(patch_feature_section(0xF, &fixup) == 1); | |
c69a48cd | 317 | check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); |
362e7701 | 318 | check(patch_feature_section(0, &fixup) == 1); |
c69a48cd | 319 | check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); |
362e7701 | 320 | check(patch_feature_section(~0xF, &fixup) == 1); |
c69a48cd | 321 | check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); |
362e7701 ME |
322 | } |
323 | ||
324 | static void test_alternative_case_too_small(void) | |
325 | { | |
c69a48cd DA |
326 | extern unsigned int ftr_fixup_test4[]; |
327 | extern unsigned int end_ftr_fixup_test4[]; | |
328 | extern unsigned int ftr_fixup_test4_orig[]; | |
329 | extern unsigned int ftr_fixup_test4_alt[]; | |
330 | extern unsigned int ftr_fixup_test4_expected[]; | |
331 | int size = end_ftr_fixup_test4 - ftr_fixup_test4; | |
362e7701 ME |
332 | unsigned long flag; |
333 | ||
334 | /* Check a high-bit flag */ | |
335 | flag = 1UL << ((sizeof(unsigned long) - 1) * 8); | |
336 | fixup.value = fixup.mask = flag; | |
c69a48cd DA |
337 | fixup.start_off = calc_offset(&fixup, ftr_fixup_test4 + 1); |
338 | fixup.end_off = calc_offset(&fixup, ftr_fixup_test4 + 5); | |
339 | fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_test4_alt); | |
340 | fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_test4_alt + 2); | |
362e7701 ME |
341 | |
342 | /* Sanity check */ | |
c69a48cd | 343 | check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0); |
362e7701 ME |
344 | |
345 | /* Check we don't patch if the value matches */ | |
346 | patch_feature_section(flag, &fixup); | |
c69a48cd | 347 | check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0); |
362e7701 ME |
348 | |
349 | /* Check we do patch if the value doesn't match */ | |
350 | patch_feature_section(0, &fixup); | |
c69a48cd | 351 | check(memcmp(ftr_fixup_test4, ftr_fixup_test4_expected, size) == 0); |
362e7701 ME |
352 | |
353 | /* Check we do patch if the mask doesn't match */ | |
c69a48cd DA |
354 | memcpy(ftr_fixup_test4, ftr_fixup_test4_orig, size); |
355 | check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0); | |
362e7701 | 356 | patch_feature_section(~flag, &fixup); |
c69a48cd | 357 | check(memcmp(ftr_fixup_test4, ftr_fixup_test4_expected, size) == 0); |
362e7701 ME |
358 | } |
359 | ||
360 | static void test_alternative_case_with_branch(void) | |
361 | { | |
c69a48cd DA |
362 | extern unsigned int ftr_fixup_test5[]; |
363 | extern unsigned int end_ftr_fixup_test5[]; | |
364 | extern unsigned int ftr_fixup_test5_expected[]; | |
365 | int size = end_ftr_fixup_test5 - ftr_fixup_test5; | |
362e7701 | 366 | |
c69a48cd | 367 | check(memcmp(ftr_fixup_test5, ftr_fixup_test5_expected, size) == 0); |
362e7701 ME |
368 | } |
369 | ||
370 | static void test_alternative_case_with_external_branch(void) | |
371 | { | |
c69a48cd DA |
372 | extern unsigned int ftr_fixup_test6[]; |
373 | extern unsigned int end_ftr_fixup_test6[]; | |
374 | extern unsigned int ftr_fixup_test6_expected[]; | |
375 | int size = end_ftr_fixup_test6 - ftr_fixup_test6; | |
362e7701 | 376 | |
c69a48cd | 377 | check(memcmp(ftr_fixup_test6, ftr_fixup_test6_expected, size) == 0); |
362e7701 ME |
378 | } |
379 | ||
380 | static void test_cpu_macros(void) | |
381 | { | |
c69a48cd DA |
382 | extern u8 ftr_fixup_test_FTR_macros[]; |
383 | extern u8 ftr_fixup_test_FTR_macros_expected[]; | |
384 | unsigned long size = ftr_fixup_test_FTR_macros_expected - | |
385 | ftr_fixup_test_FTR_macros; | |
362e7701 ME |
386 | |
387 | /* The fixups have already been done for us during boot */ | |
c69a48cd DA |
388 | check(memcmp(ftr_fixup_test_FTR_macros, |
389 | ftr_fixup_test_FTR_macros_expected, size) == 0); | |
362e7701 ME |
390 | } |
391 | ||
392 | static void test_fw_macros(void) | |
393 | { | |
394 | #ifdef CONFIG_PPC64 | |
c69a48cd DA |
395 | extern u8 ftr_fixup_test_FW_FTR_macros[]; |
396 | extern u8 ftr_fixup_test_FW_FTR_macros_expected[]; | |
397 | unsigned long size = ftr_fixup_test_FW_FTR_macros_expected - | |
398 | ftr_fixup_test_FW_FTR_macros; | |
362e7701 ME |
399 | |
400 | /* The fixups have already been done for us during boot */ | |
c69a48cd DA |
401 | check(memcmp(ftr_fixup_test_FW_FTR_macros, |
402 | ftr_fixup_test_FW_FTR_macros_expected, size) == 0); | |
362e7701 ME |
403 | #endif |
404 | } | |
405 | ||
2d1b2027 KG |
406 | static void test_lwsync_macros(void) |
407 | { | |
c69a48cd DA |
408 | extern u8 lwsync_fixup_test[]; |
409 | extern u8 end_lwsync_fixup_test[]; | |
410 | extern u8 lwsync_fixup_test_expected_LWSYNC[]; | |
411 | extern u8 lwsync_fixup_test_expected_SYNC[]; | |
412 | unsigned long size = end_lwsync_fixup_test - | |
413 | lwsync_fixup_test; | |
2d1b2027 KG |
414 | |
415 | /* The fixups have already been done for us during boot */ | |
416 | if (cur_cpu_spec->cpu_features & CPU_FTR_LWSYNC) { | |
c69a48cd DA |
417 | check(memcmp(lwsync_fixup_test, |
418 | lwsync_fixup_test_expected_LWSYNC, size) == 0); | |
2d1b2027 | 419 | } else { |
c69a48cd DA |
420 | check(memcmp(lwsync_fixup_test, |
421 | lwsync_fixup_test_expected_SYNC, size) == 0); | |
2d1b2027 KG |
422 | } |
423 | } | |
424 | ||
362e7701 ME |
425 | static int __init test_feature_fixups(void) |
426 | { | |
427 | printk(KERN_DEBUG "Running feature fixup self-tests ...\n"); | |
428 | ||
429 | test_basic_patching(); | |
430 | test_alternative_patching(); | |
431 | test_alternative_case_too_big(); | |
432 | test_alternative_case_too_small(); | |
433 | test_alternative_case_with_branch(); | |
434 | test_alternative_case_with_external_branch(); | |
435 | test_cpu_macros(); | |
436 | test_fw_macros(); | |
2d1b2027 | 437 | test_lwsync_macros(); |
362e7701 ME |
438 | |
439 | return 0; | |
440 | } | |
441 | late_initcall(test_feature_fixups); | |
442 | ||
443 | #endif /* CONFIG_FTR_FIXUP_SELFTEST */ |