]>
Commit | Line | Data |
---|---|---|
51c52e86 ME |
1 | /* |
2 | * Copyright (C) 2001 Ben. Herrenschmidt (benh@kernel.crashing.org) | |
3 | * | |
4 | * Modifications for ppc64: | |
5 | * Copyright (C) 2003 Dave Engebretsen <engebret@us.ibm.com> | |
6 | * | |
7 | * Copyright 2008 Michael Ellerman, IBM Corporation. | |
8 | * | |
9 | * This program is free software; you can redistribute it and/or | |
10 | * modify it under the terms of the GNU General Public License | |
11 | * as published by the Free Software Foundation; either version | |
12 | * 2 of the License, or (at your option) any later version. | |
13 | */ | |
14 | ||
3880ecb0 | 15 | #include <linux/types.h> |
51c52e86 | 16 | #include <linux/kernel.h> |
362e7701 ME |
17 | #include <linux/string.h> |
18 | #include <linux/init.h> | |
51c52e86 ME |
19 | #include <asm/cputable.h> |
20 | #include <asm/code-patching.h> | |
d715e433 AB |
21 | #include <asm/page.h> |
22 | #include <asm/sections.h> | |
9402c684 BH |
23 | #include <asm/setup.h> |
24 | #include <asm/firmware.h> | |
51c52e86 ME |
25 | |
26 | struct fixup_entry { | |
27 | unsigned long mask; | |
28 | unsigned long value; | |
29 | long start_off; | |
30 | long end_off; | |
fac23fe4 ME |
31 | long alt_start_off; |
32 | long alt_end_off; | |
51c52e86 ME |
33 | }; |
34 | ||
9b1a735d | 35 | static unsigned int *calc_addr(struct fixup_entry *fcur, long offset) |
51c52e86 | 36 | { |
9b1a735d ME |
37 | /* |
38 | * We store the offset to the code as a negative offset from | |
39 | * the start of the alt_entry, to support the VDSO. This | |
40 | * routine converts that back into an actual address. | |
41 | */ | |
42 | return (unsigned int *)((unsigned long)fcur + offset); | |
43 | } | |
44 | ||
45 | static int patch_alt_instruction(unsigned int *src, unsigned int *dest, | |
46 | unsigned int *alt_start, unsigned int *alt_end) | |
47 | { | |
48 | unsigned int instr; | |
49 | ||
50 | instr = *src; | |
51 | ||
52 | if (instr_is_relative_branch(*src)) { | |
53 | unsigned int *target = (unsigned int *)branch_target(src); | |
54 | ||
55 | /* Branch within the section doesn't need translating */ | |
56 | if (target < alt_start || target >= alt_end) { | |
57 | instr = translate_branch(dest, src); | |
58 | if (!instr) | |
59 | return 1; | |
60 | } | |
61 | } | |
62 | ||
63 | patch_instruction(dest, instr); | |
64 | ||
65 | return 0; | |
66 | } | |
67 | ||
68 | static int patch_feature_section(unsigned long value, struct fixup_entry *fcur) | |
69 | { | |
70 | unsigned int *start, *end, *alt_start, *alt_end, *src, *dest; | |
71 | ||
72 | start = calc_addr(fcur, fcur->start_off); | |
73 | end = calc_addr(fcur, fcur->end_off); | |
74 | alt_start = calc_addr(fcur, fcur->alt_start_off); | |
75 | alt_end = calc_addr(fcur, fcur->alt_end_off); | |
76 | ||
77 | if ((alt_end - alt_start) > (end - start)) | |
78 | return 1; | |
51c52e86 ME |
79 | |
80 | if ((value & fcur->mask) == fcur->value) | |
9b1a735d | 81 | return 0; |
51c52e86 | 82 | |
9b1a735d ME |
83 | src = alt_start; |
84 | dest = start; | |
51c52e86 | 85 | |
9b1a735d ME |
86 | for (; src < alt_end; src++, dest++) { |
87 | if (patch_alt_instruction(src, dest, alt_start, alt_end)) | |
88 | return 1; | |
51c52e86 | 89 | } |
9b1a735d ME |
90 | |
91 | for (; dest < end; dest++) | |
16c57b36 | 92 | patch_instruction(dest, PPC_INST_NOP); |
9b1a735d ME |
93 | |
94 | return 0; | |
51c52e86 ME |
95 | } |
96 | ||
97 | void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end) | |
98 | { | |
99 | struct fixup_entry *fcur, *fend; | |
100 | ||
101 | fcur = fixup_start; | |
102 | fend = fixup_end; | |
103 | ||
9b1a735d ME |
104 | for (; fcur < fend; fcur++) { |
105 | if (patch_feature_section(value, fcur)) { | |
1856c020 | 106 | WARN_ON(1); |
9b1a735d ME |
107 | printk("Unable to patch feature section at %p - %p" \ |
108 | " with %p - %p\n", | |
109 | calc_addr(fcur, fcur->start_off), | |
110 | calc_addr(fcur, fcur->end_off), | |
111 | calc_addr(fcur, fcur->alt_start_off), | |
112 | calc_addr(fcur, fcur->alt_end_off)); | |
113 | } | |
114 | } | |
51c52e86 | 115 | } |
362e7701 | 116 | |
2d1b2027 KG |
117 | void do_lwsync_fixups(unsigned long value, void *fixup_start, void *fixup_end) |
118 | { | |
3d98ffbf BH |
119 | long *start, *end; |
120 | unsigned int *dest; | |
2d1b2027 KG |
121 | |
122 | if (!(value & CPU_FTR_LWSYNC)) | |
123 | return ; | |
124 | ||
125 | start = fixup_start; | |
126 | end = fixup_end; | |
127 | ||
128 | for (; start < end; start++) { | |
129 | dest = (void *)start + *start; | |
16c57b36 | 130 | patch_instruction(dest, PPC_INST_LWSYNC); |
2d1b2027 KG |
131 | } |
132 | } | |
133 | ||
9402c684 | 134 | static void do_final_fixups(void) |
d715e433 AB |
135 | { |
136 | #if defined(CONFIG_PPC64) && defined(CONFIG_RELOCATABLE) | |
137 | int *src, *dest; | |
138 | unsigned long length; | |
139 | ||
140 | if (PHYSICAL_START == 0) | |
141 | return; | |
142 | ||
143 | src = (int *)(KERNELBASE + PHYSICAL_START); | |
144 | dest = (int *)KERNELBASE; | |
145 | length = (__end_interrupts - _stext) / sizeof(int); | |
146 | ||
147 | while (length--) { | |
148 | patch_instruction(dest, *src); | |
149 | src++; | |
150 | dest++; | |
151 | } | |
152 | #endif | |
153 | } | |
154 | ||
a28e46f1 ME |
155 | static unsigned long __initdata saved_cpu_features; |
156 | static unsigned int __initdata saved_mmu_features; | |
157 | #ifdef CONFIG_PPC64 | |
158 | static unsigned long __initdata saved_firmware_features; | |
159 | #endif | |
160 | ||
161 | void __init apply_feature_fixups(void) | |
9402c684 BH |
162 | { |
163 | struct cpu_spec *spec = *PTRRELOC(&cur_cpu_spec); | |
164 | ||
a28e46f1 ME |
165 | *PTRRELOC(&saved_cpu_features) = spec->cpu_features; |
166 | *PTRRELOC(&saved_mmu_features) = spec->mmu_features; | |
167 | ||
9402c684 BH |
168 | /* |
169 | * Apply the CPU-specific and firmware specific fixups to kernel text | |
170 | * (nop out sections not relevant to this CPU or this firmware). | |
171 | */ | |
172 | do_feature_fixups(spec->cpu_features, | |
173 | PTRRELOC(&__start___ftr_fixup), | |
174 | PTRRELOC(&__stop___ftr_fixup)); | |
175 | ||
176 | do_feature_fixups(spec->mmu_features, | |
177 | PTRRELOC(&__start___mmu_ftr_fixup), | |
178 | PTRRELOC(&__stop___mmu_ftr_fixup)); | |
179 | ||
180 | do_lwsync_fixups(spec->cpu_features, | |
181 | PTRRELOC(&__start___lwsync_fixup), | |
182 | PTRRELOC(&__stop___lwsync_fixup)); | |
183 | ||
184 | #ifdef CONFIG_PPC64 | |
a28e46f1 | 185 | saved_firmware_features = powerpc_firmware_features; |
9402c684 BH |
186 | do_feature_fixups(powerpc_firmware_features, |
187 | &__start___fw_ftr_fixup, &__stop___fw_ftr_fixup); | |
188 | #endif | |
189 | do_final_fixups(); | |
190 | } | |
191 | ||
a28e46f1 ME |
192 | static int __init check_features(void) |
193 | { | |
194 | WARN(saved_cpu_features != cur_cpu_spec->cpu_features, | |
195 | "CPU features changed after feature patching!\n"); | |
196 | WARN(saved_mmu_features != cur_cpu_spec->mmu_features, | |
197 | "MMU features changed after feature patching!\n"); | |
198 | #ifdef CONFIG_PPC64 | |
199 | WARN(saved_firmware_features != powerpc_firmware_features, | |
200 | "Firmware features changed after feature patching!\n"); | |
201 | #endif | |
202 | ||
203 | return 0; | |
204 | } | |
205 | late_initcall(check_features); | |
206 | ||
362e7701 ME |
207 | #ifdef CONFIG_FTR_FIXUP_SELFTEST |
208 | ||
209 | #define check(x) \ | |
210 | if (!(x)) printk("feature-fixups: test failed at line %d\n", __LINE__); | |
211 | ||
212 | /* This must be after the text it fixes up, vmlinux.lds.S enforces that atm */ | |
213 | static struct fixup_entry fixup; | |
214 | ||
215 | static long calc_offset(struct fixup_entry *entry, unsigned int *p) | |
216 | { | |
217 | return (unsigned long)p - (unsigned long)entry; | |
218 | } | |
219 | ||
e51df2c1 | 220 | static void test_basic_patching(void) |
362e7701 ME |
221 | { |
222 | extern unsigned int ftr_fixup_test1; | |
223 | extern unsigned int end_ftr_fixup_test1; | |
224 | extern unsigned int ftr_fixup_test1_orig; | |
225 | extern unsigned int ftr_fixup_test1_expected; | |
226 | int size = &end_ftr_fixup_test1 - &ftr_fixup_test1; | |
227 | ||
228 | fixup.value = fixup.mask = 8; | |
229 | fixup.start_off = calc_offset(&fixup, &ftr_fixup_test1 + 1); | |
230 | fixup.end_off = calc_offset(&fixup, &ftr_fixup_test1 + 2); | |
231 | fixup.alt_start_off = fixup.alt_end_off = 0; | |
232 | ||
233 | /* Sanity check */ | |
234 | check(memcmp(&ftr_fixup_test1, &ftr_fixup_test1_orig, size) == 0); | |
235 | ||
236 | /* Check we don't patch if the value matches */ | |
237 | patch_feature_section(8, &fixup); | |
238 | check(memcmp(&ftr_fixup_test1, &ftr_fixup_test1_orig, size) == 0); | |
239 | ||
240 | /* Check we do patch if the value doesn't match */ | |
241 | patch_feature_section(0, &fixup); | |
242 | check(memcmp(&ftr_fixup_test1, &ftr_fixup_test1_expected, size) == 0); | |
243 | ||
244 | /* Check we do patch if the mask doesn't match */ | |
245 | memcpy(&ftr_fixup_test1, &ftr_fixup_test1_orig, size); | |
246 | check(memcmp(&ftr_fixup_test1, &ftr_fixup_test1_orig, size) == 0); | |
247 | patch_feature_section(~8, &fixup); | |
248 | check(memcmp(&ftr_fixup_test1, &ftr_fixup_test1_expected, size) == 0); | |
249 | } | |
250 | ||
251 | static void test_alternative_patching(void) | |
252 | { | |
253 | extern unsigned int ftr_fixup_test2; | |
254 | extern unsigned int end_ftr_fixup_test2; | |
255 | extern unsigned int ftr_fixup_test2_orig; | |
256 | extern unsigned int ftr_fixup_test2_alt; | |
257 | extern unsigned int ftr_fixup_test2_expected; | |
258 | int size = &end_ftr_fixup_test2 - &ftr_fixup_test2; | |
259 | ||
260 | fixup.value = fixup.mask = 0xF; | |
261 | fixup.start_off = calc_offset(&fixup, &ftr_fixup_test2 + 1); | |
262 | fixup.end_off = calc_offset(&fixup, &ftr_fixup_test2 + 2); | |
263 | fixup.alt_start_off = calc_offset(&fixup, &ftr_fixup_test2_alt); | |
264 | fixup.alt_end_off = calc_offset(&fixup, &ftr_fixup_test2_alt + 1); | |
265 | ||
266 | /* Sanity check */ | |
267 | check(memcmp(&ftr_fixup_test2, &ftr_fixup_test2_orig, size) == 0); | |
268 | ||
269 | /* Check we don't patch if the value matches */ | |
270 | patch_feature_section(0xF, &fixup); | |
271 | check(memcmp(&ftr_fixup_test2, &ftr_fixup_test2_orig, size) == 0); | |
272 | ||
273 | /* Check we do patch if the value doesn't match */ | |
274 | patch_feature_section(0, &fixup); | |
275 | check(memcmp(&ftr_fixup_test2, &ftr_fixup_test2_expected, size) == 0); | |
276 | ||
277 | /* Check we do patch if the mask doesn't match */ | |
278 | memcpy(&ftr_fixup_test2, &ftr_fixup_test2_orig, size); | |
279 | check(memcmp(&ftr_fixup_test2, &ftr_fixup_test2_orig, size) == 0); | |
280 | patch_feature_section(~0xF, &fixup); | |
281 | check(memcmp(&ftr_fixup_test2, &ftr_fixup_test2_expected, size) == 0); | |
282 | } | |
283 | ||
284 | static void test_alternative_case_too_big(void) | |
285 | { | |
286 | extern unsigned int ftr_fixup_test3; | |
287 | extern unsigned int end_ftr_fixup_test3; | |
288 | extern unsigned int ftr_fixup_test3_orig; | |
289 | extern unsigned int ftr_fixup_test3_alt; | |
290 | int size = &end_ftr_fixup_test3 - &ftr_fixup_test3; | |
291 | ||
292 | fixup.value = fixup.mask = 0xC; | |
293 | fixup.start_off = calc_offset(&fixup, &ftr_fixup_test3 + 1); | |
294 | fixup.end_off = calc_offset(&fixup, &ftr_fixup_test3 + 2); | |
295 | fixup.alt_start_off = calc_offset(&fixup, &ftr_fixup_test3_alt); | |
296 | fixup.alt_end_off = calc_offset(&fixup, &ftr_fixup_test3_alt + 2); | |
297 | ||
298 | /* Sanity check */ | |
299 | check(memcmp(&ftr_fixup_test3, &ftr_fixup_test3_orig, size) == 0); | |
300 | ||
301 | /* Expect nothing to be patched, and the error returned to us */ | |
302 | check(patch_feature_section(0xF, &fixup) == 1); | |
303 | check(memcmp(&ftr_fixup_test3, &ftr_fixup_test3_orig, size) == 0); | |
304 | check(patch_feature_section(0, &fixup) == 1); | |
305 | check(memcmp(&ftr_fixup_test3, &ftr_fixup_test3_orig, size) == 0); | |
306 | check(patch_feature_section(~0xF, &fixup) == 1); | |
307 | check(memcmp(&ftr_fixup_test3, &ftr_fixup_test3_orig, size) == 0); | |
308 | } | |
309 | ||
310 | static void test_alternative_case_too_small(void) | |
311 | { | |
312 | extern unsigned int ftr_fixup_test4; | |
313 | extern unsigned int end_ftr_fixup_test4; | |
314 | extern unsigned int ftr_fixup_test4_orig; | |
315 | extern unsigned int ftr_fixup_test4_alt; | |
316 | extern unsigned int ftr_fixup_test4_expected; | |
317 | int size = &end_ftr_fixup_test4 - &ftr_fixup_test4; | |
318 | unsigned long flag; | |
319 | ||
320 | /* Check a high-bit flag */ | |
321 | flag = 1UL << ((sizeof(unsigned long) - 1) * 8); | |
322 | fixup.value = fixup.mask = flag; | |
323 | fixup.start_off = calc_offset(&fixup, &ftr_fixup_test4 + 1); | |
324 | fixup.end_off = calc_offset(&fixup, &ftr_fixup_test4 + 5); | |
325 | fixup.alt_start_off = calc_offset(&fixup, &ftr_fixup_test4_alt); | |
326 | fixup.alt_end_off = calc_offset(&fixup, &ftr_fixup_test4_alt + 2); | |
327 | ||
328 | /* Sanity check */ | |
329 | check(memcmp(&ftr_fixup_test4, &ftr_fixup_test4_orig, size) == 0); | |
330 | ||
331 | /* Check we don't patch if the value matches */ | |
332 | patch_feature_section(flag, &fixup); | |
333 | check(memcmp(&ftr_fixup_test4, &ftr_fixup_test4_orig, size) == 0); | |
334 | ||
335 | /* Check we do patch if the value doesn't match */ | |
336 | patch_feature_section(0, &fixup); | |
337 | check(memcmp(&ftr_fixup_test4, &ftr_fixup_test4_expected, size) == 0); | |
338 | ||
339 | /* Check we do patch if the mask doesn't match */ | |
340 | memcpy(&ftr_fixup_test4, &ftr_fixup_test4_orig, size); | |
341 | check(memcmp(&ftr_fixup_test4, &ftr_fixup_test4_orig, size) == 0); | |
342 | patch_feature_section(~flag, &fixup); | |
343 | check(memcmp(&ftr_fixup_test4, &ftr_fixup_test4_expected, size) == 0); | |
344 | } | |
345 | ||
346 | static void test_alternative_case_with_branch(void) | |
347 | { | |
348 | extern unsigned int ftr_fixup_test5; | |
349 | extern unsigned int end_ftr_fixup_test5; | |
350 | extern unsigned int ftr_fixup_test5_expected; | |
351 | int size = &end_ftr_fixup_test5 - &ftr_fixup_test5; | |
352 | ||
353 | check(memcmp(&ftr_fixup_test5, &ftr_fixup_test5_expected, size) == 0); | |
354 | } | |
355 | ||
356 | static void test_alternative_case_with_external_branch(void) | |
357 | { | |
358 | extern unsigned int ftr_fixup_test6; | |
359 | extern unsigned int end_ftr_fixup_test6; | |
360 | extern unsigned int ftr_fixup_test6_expected; | |
361 | int size = &end_ftr_fixup_test6 - &ftr_fixup_test6; | |
362 | ||
363 | check(memcmp(&ftr_fixup_test6, &ftr_fixup_test6_expected, size) == 0); | |
364 | } | |
365 | ||
366 | static void test_cpu_macros(void) | |
367 | { | |
3880ecb0 SR |
368 | extern u8 ftr_fixup_test_FTR_macros; |
369 | extern u8 ftr_fixup_test_FTR_macros_expected; | |
362e7701 ME |
370 | unsigned long size = &ftr_fixup_test_FTR_macros_expected - |
371 | &ftr_fixup_test_FTR_macros; | |
372 | ||
373 | /* The fixups have already been done for us during boot */ | |
374 | check(memcmp(&ftr_fixup_test_FTR_macros, | |
375 | &ftr_fixup_test_FTR_macros_expected, size) == 0); | |
376 | } | |
377 | ||
378 | static void test_fw_macros(void) | |
379 | { | |
380 | #ifdef CONFIG_PPC64 | |
3880ecb0 SR |
381 | extern u8 ftr_fixup_test_FW_FTR_macros; |
382 | extern u8 ftr_fixup_test_FW_FTR_macros_expected; | |
362e7701 ME |
383 | unsigned long size = &ftr_fixup_test_FW_FTR_macros_expected - |
384 | &ftr_fixup_test_FW_FTR_macros; | |
385 | ||
386 | /* The fixups have already been done for us during boot */ | |
387 | check(memcmp(&ftr_fixup_test_FW_FTR_macros, | |
388 | &ftr_fixup_test_FW_FTR_macros_expected, size) == 0); | |
389 | #endif | |
390 | } | |
391 | ||
2d1b2027 KG |
392 | static void test_lwsync_macros(void) |
393 | { | |
3880ecb0 SR |
394 | extern u8 lwsync_fixup_test; |
395 | extern u8 end_lwsync_fixup_test; | |
396 | extern u8 lwsync_fixup_test_expected_LWSYNC; | |
397 | extern u8 lwsync_fixup_test_expected_SYNC; | |
2d1b2027 KG |
398 | unsigned long size = &end_lwsync_fixup_test - |
399 | &lwsync_fixup_test; | |
400 | ||
401 | /* The fixups have already been done for us during boot */ | |
402 | if (cur_cpu_spec->cpu_features & CPU_FTR_LWSYNC) { | |
403 | check(memcmp(&lwsync_fixup_test, | |
404 | &lwsync_fixup_test_expected_LWSYNC, size) == 0); | |
405 | } else { | |
406 | check(memcmp(&lwsync_fixup_test, | |
407 | &lwsync_fixup_test_expected_SYNC, size) == 0); | |
408 | } | |
409 | } | |
410 | ||
362e7701 ME |
411 | static int __init test_feature_fixups(void) |
412 | { | |
413 | printk(KERN_DEBUG "Running feature fixup self-tests ...\n"); | |
414 | ||
415 | test_basic_patching(); | |
416 | test_alternative_patching(); | |
417 | test_alternative_case_too_big(); | |
418 | test_alternative_case_too_small(); | |
419 | test_alternative_case_with_branch(); | |
420 | test_alternative_case_with_external_branch(); | |
421 | test_cpu_macros(); | |
422 | test_fw_macros(); | |
2d1b2027 | 423 | test_lwsync_macros(); |
362e7701 ME |
424 | |
425 | return 0; | |
426 | } | |
427 | late_initcall(test_feature_fixups); | |
428 | ||
429 | #endif /* CONFIG_FTR_FIXUP_SELFTEST */ |