]>
Commit | Line | Data |
---|---|---|
51c52e86 ME |
1 | /* |
2 | * Copyright (C) 2001 Ben. Herrenschmidt (benh@kernel.crashing.org) | |
3 | * | |
4 | * Modifications for ppc64: | |
5 | * Copyright (C) 2003 Dave Engebretsen <engebret@us.ibm.com> | |
6 | * | |
7 | * Copyright 2008 Michael Ellerman, IBM Corporation. | |
8 | * | |
9 | * This program is free software; you can redistribute it and/or | |
10 | * modify it under the terms of the GNU General Public License | |
11 | * as published by the Free Software Foundation; either version | |
12 | * 2 of the License, or (at your option) any later version. | |
13 | */ | |
14 | ||
3880ecb0 | 15 | #include <linux/types.h> |
309b315b | 16 | #include <linux/jump_label.h> |
51c52e86 | 17 | #include <linux/kernel.h> |
362e7701 ME |
18 | #include <linux/string.h> |
19 | #include <linux/init.h> | |
51c52e86 ME |
20 | #include <asm/cputable.h> |
21 | #include <asm/code-patching.h> | |
d715e433 AB |
22 | #include <asm/page.h> |
23 | #include <asm/sections.h> | |
9402c684 BH |
24 | #include <asm/setup.h> |
25 | #include <asm/firmware.h> | |
51c52e86 ME |
26 | |
27 | struct fixup_entry { | |
28 | unsigned long mask; | |
29 | unsigned long value; | |
30 | long start_off; | |
31 | long end_off; | |
fac23fe4 ME |
32 | long alt_start_off; |
33 | long alt_end_off; | |
51c52e86 ME |
34 | }; |
35 | ||
9b1a735d | 36 | static unsigned int *calc_addr(struct fixup_entry *fcur, long offset) |
51c52e86 | 37 | { |
9b1a735d ME |
38 | /* |
39 | * We store the offset to the code as a negative offset from | |
40 | * the start of the alt_entry, to support the VDSO. This | |
41 | * routine converts that back into an actual address. | |
42 | */ | |
43 | return (unsigned int *)((unsigned long)fcur + offset); | |
44 | } | |
45 | ||
46 | static int patch_alt_instruction(unsigned int *src, unsigned int *dest, | |
47 | unsigned int *alt_start, unsigned int *alt_end) | |
48 | { | |
49 | unsigned int instr; | |
50 | ||
51 | instr = *src; | |
52 | ||
53 | if (instr_is_relative_branch(*src)) { | |
54 | unsigned int *target = (unsigned int *)branch_target(src); | |
55 | ||
56 | /* Branch within the section doesn't need translating */ | |
57 | if (target < alt_start || target >= alt_end) { | |
58 | instr = translate_branch(dest, src); | |
59 | if (!instr) | |
60 | return 1; | |
61 | } | |
62 | } | |
63 | ||
64 | patch_instruction(dest, instr); | |
65 | ||
66 | return 0; | |
67 | } | |
68 | ||
69 | static int patch_feature_section(unsigned long value, struct fixup_entry *fcur) | |
70 | { | |
71 | unsigned int *start, *end, *alt_start, *alt_end, *src, *dest; | |
72 | ||
73 | start = calc_addr(fcur, fcur->start_off); | |
74 | end = calc_addr(fcur, fcur->end_off); | |
75 | alt_start = calc_addr(fcur, fcur->alt_start_off); | |
76 | alt_end = calc_addr(fcur, fcur->alt_end_off); | |
77 | ||
78 | if ((alt_end - alt_start) > (end - start)) | |
79 | return 1; | |
51c52e86 ME |
80 | |
81 | if ((value & fcur->mask) == fcur->value) | |
9b1a735d | 82 | return 0; |
51c52e86 | 83 | |
9b1a735d ME |
84 | src = alt_start; |
85 | dest = start; | |
51c52e86 | 86 | |
9b1a735d ME |
87 | for (; src < alt_end; src++, dest++) { |
88 | if (patch_alt_instruction(src, dest, alt_start, alt_end)) | |
89 | return 1; | |
51c52e86 | 90 | } |
9b1a735d ME |
91 | |
92 | for (; dest < end; dest++) | |
16c57b36 | 93 | patch_instruction(dest, PPC_INST_NOP); |
9b1a735d ME |
94 | |
95 | return 0; | |
51c52e86 ME |
96 | } |
97 | ||
98 | void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end) | |
99 | { | |
100 | struct fixup_entry *fcur, *fend; | |
101 | ||
102 | fcur = fixup_start; | |
103 | fend = fixup_end; | |
104 | ||
9b1a735d ME |
105 | for (; fcur < fend; fcur++) { |
106 | if (patch_feature_section(value, fcur)) { | |
1856c020 | 107 | WARN_ON(1); |
9b1a735d ME |
108 | printk("Unable to patch feature section at %p - %p" \ |
109 | " with %p - %p\n", | |
110 | calc_addr(fcur, fcur->start_off), | |
111 | calc_addr(fcur, fcur->end_off), | |
112 | calc_addr(fcur, fcur->alt_start_off), | |
113 | calc_addr(fcur, fcur->alt_end_off)); | |
114 | } | |
115 | } | |
51c52e86 | 116 | } |
362e7701 | 117 | |
2d1b2027 KG |
118 | void do_lwsync_fixups(unsigned long value, void *fixup_start, void *fixup_end) |
119 | { | |
3d98ffbf BH |
120 | long *start, *end; |
121 | unsigned int *dest; | |
2d1b2027 KG |
122 | |
123 | if (!(value & CPU_FTR_LWSYNC)) | |
124 | return ; | |
125 | ||
126 | start = fixup_start; | |
127 | end = fixup_end; | |
128 | ||
129 | for (; start < end; start++) { | |
130 | dest = (void *)start + *start; | |
16c57b36 | 131 | patch_instruction(dest, PPC_INST_LWSYNC); |
2d1b2027 KG |
132 | } |
133 | } | |
134 | ||
9402c684 | 135 | static void do_final_fixups(void) |
d715e433 AB |
136 | { |
137 | #if defined(CONFIG_PPC64) && defined(CONFIG_RELOCATABLE) | |
138 | int *src, *dest; | |
139 | unsigned long length; | |
140 | ||
141 | if (PHYSICAL_START == 0) | |
142 | return; | |
143 | ||
144 | src = (int *)(KERNELBASE + PHYSICAL_START); | |
145 | dest = (int *)KERNELBASE; | |
146 | length = (__end_interrupts - _stext) / sizeof(int); | |
147 | ||
148 | while (length--) { | |
149 | patch_instruction(dest, *src); | |
150 | src++; | |
151 | dest++; | |
152 | } | |
153 | #endif | |
154 | } | |
155 | ||
a28e46f1 ME |
156 | static unsigned long __initdata saved_cpu_features; |
157 | static unsigned int __initdata saved_mmu_features; | |
158 | #ifdef CONFIG_PPC64 | |
159 | static unsigned long __initdata saved_firmware_features; | |
160 | #endif | |
161 | ||
162 | void __init apply_feature_fixups(void) | |
9402c684 BH |
163 | { |
164 | struct cpu_spec *spec = *PTRRELOC(&cur_cpu_spec); | |
165 | ||
a28e46f1 ME |
166 | *PTRRELOC(&saved_cpu_features) = spec->cpu_features; |
167 | *PTRRELOC(&saved_mmu_features) = spec->mmu_features; | |
168 | ||
9402c684 BH |
169 | /* |
170 | * Apply the CPU-specific and firmware specific fixups to kernel text | |
171 | * (nop out sections not relevant to this CPU or this firmware). | |
172 | */ | |
173 | do_feature_fixups(spec->cpu_features, | |
174 | PTRRELOC(&__start___ftr_fixup), | |
175 | PTRRELOC(&__stop___ftr_fixup)); | |
176 | ||
177 | do_feature_fixups(spec->mmu_features, | |
178 | PTRRELOC(&__start___mmu_ftr_fixup), | |
179 | PTRRELOC(&__stop___mmu_ftr_fixup)); | |
180 | ||
181 | do_lwsync_fixups(spec->cpu_features, | |
182 | PTRRELOC(&__start___lwsync_fixup), | |
183 | PTRRELOC(&__stop___lwsync_fixup)); | |
184 | ||
185 | #ifdef CONFIG_PPC64 | |
a28e46f1 | 186 | saved_firmware_features = powerpc_firmware_features; |
9402c684 BH |
187 | do_feature_fixups(powerpc_firmware_features, |
188 | &__start___fw_ftr_fixup, &__stop___fw_ftr_fixup); | |
189 | #endif | |
190 | do_final_fixups(); | |
309b315b AK |
191 | |
192 | /* | |
193 | * Initialise jump label. This causes all the cpu/mmu_has_feature() | |
194 | * checks to take on their correct polarity based on the current set of | |
195 | * CPU/MMU features. | |
196 | */ | |
197 | jump_label_init(); | |
4db73271 | 198 | cpu_feature_keys_init(); |
c12e6f24 | 199 | mmu_feature_keys_init(); |
9402c684 BH |
200 | } |
201 | ||
a28e46f1 ME |
202 | static int __init check_features(void) |
203 | { | |
204 | WARN(saved_cpu_features != cur_cpu_spec->cpu_features, | |
205 | "CPU features changed after feature patching!\n"); | |
206 | WARN(saved_mmu_features != cur_cpu_spec->mmu_features, | |
207 | "MMU features changed after feature patching!\n"); | |
208 | #ifdef CONFIG_PPC64 | |
209 | WARN(saved_firmware_features != powerpc_firmware_features, | |
210 | "Firmware features changed after feature patching!\n"); | |
211 | #endif | |
212 | ||
213 | return 0; | |
214 | } | |
215 | late_initcall(check_features); | |
216 | ||
362e7701 ME |
217 | #ifdef CONFIG_FTR_FIXUP_SELFTEST |
218 | ||
219 | #define check(x) \ | |
220 | if (!(x)) printk("feature-fixups: test failed at line %d\n", __LINE__); | |
221 | ||
222 | /* This must be after the text it fixes up, vmlinux.lds.S enforces that atm */ | |
223 | static struct fixup_entry fixup; | |
224 | ||
225 | static long calc_offset(struct fixup_entry *entry, unsigned int *p) | |
226 | { | |
227 | return (unsigned long)p - (unsigned long)entry; | |
228 | } | |
229 | ||
e51df2c1 | 230 | static void test_basic_patching(void) |
362e7701 ME |
231 | { |
232 | extern unsigned int ftr_fixup_test1; | |
233 | extern unsigned int end_ftr_fixup_test1; | |
234 | extern unsigned int ftr_fixup_test1_orig; | |
235 | extern unsigned int ftr_fixup_test1_expected; | |
236 | int size = &end_ftr_fixup_test1 - &ftr_fixup_test1; | |
237 | ||
238 | fixup.value = fixup.mask = 8; | |
239 | fixup.start_off = calc_offset(&fixup, &ftr_fixup_test1 + 1); | |
240 | fixup.end_off = calc_offset(&fixup, &ftr_fixup_test1 + 2); | |
241 | fixup.alt_start_off = fixup.alt_end_off = 0; | |
242 | ||
243 | /* Sanity check */ | |
244 | check(memcmp(&ftr_fixup_test1, &ftr_fixup_test1_orig, size) == 0); | |
245 | ||
246 | /* Check we don't patch if the value matches */ | |
247 | patch_feature_section(8, &fixup); | |
248 | check(memcmp(&ftr_fixup_test1, &ftr_fixup_test1_orig, size) == 0); | |
249 | ||
250 | /* Check we do patch if the value doesn't match */ | |
251 | patch_feature_section(0, &fixup); | |
252 | check(memcmp(&ftr_fixup_test1, &ftr_fixup_test1_expected, size) == 0); | |
253 | ||
254 | /* Check we do patch if the mask doesn't match */ | |
255 | memcpy(&ftr_fixup_test1, &ftr_fixup_test1_orig, size); | |
256 | check(memcmp(&ftr_fixup_test1, &ftr_fixup_test1_orig, size) == 0); | |
257 | patch_feature_section(~8, &fixup); | |
258 | check(memcmp(&ftr_fixup_test1, &ftr_fixup_test1_expected, size) == 0); | |
259 | } | |
260 | ||
261 | static void test_alternative_patching(void) | |
262 | { | |
263 | extern unsigned int ftr_fixup_test2; | |
264 | extern unsigned int end_ftr_fixup_test2; | |
265 | extern unsigned int ftr_fixup_test2_orig; | |
266 | extern unsigned int ftr_fixup_test2_alt; | |
267 | extern unsigned int ftr_fixup_test2_expected; | |
268 | int size = &end_ftr_fixup_test2 - &ftr_fixup_test2; | |
269 | ||
270 | fixup.value = fixup.mask = 0xF; | |
271 | fixup.start_off = calc_offset(&fixup, &ftr_fixup_test2 + 1); | |
272 | fixup.end_off = calc_offset(&fixup, &ftr_fixup_test2 + 2); | |
273 | fixup.alt_start_off = calc_offset(&fixup, &ftr_fixup_test2_alt); | |
274 | fixup.alt_end_off = calc_offset(&fixup, &ftr_fixup_test2_alt + 1); | |
275 | ||
276 | /* Sanity check */ | |
277 | check(memcmp(&ftr_fixup_test2, &ftr_fixup_test2_orig, size) == 0); | |
278 | ||
279 | /* Check we don't patch if the value matches */ | |
280 | patch_feature_section(0xF, &fixup); | |
281 | check(memcmp(&ftr_fixup_test2, &ftr_fixup_test2_orig, size) == 0); | |
282 | ||
283 | /* Check we do patch if the value doesn't match */ | |
284 | patch_feature_section(0, &fixup); | |
285 | check(memcmp(&ftr_fixup_test2, &ftr_fixup_test2_expected, size) == 0); | |
286 | ||
287 | /* Check we do patch if the mask doesn't match */ | |
288 | memcpy(&ftr_fixup_test2, &ftr_fixup_test2_orig, size); | |
289 | check(memcmp(&ftr_fixup_test2, &ftr_fixup_test2_orig, size) == 0); | |
290 | patch_feature_section(~0xF, &fixup); | |
291 | check(memcmp(&ftr_fixup_test2, &ftr_fixup_test2_expected, size) == 0); | |
292 | } | |
293 | ||
294 | static void test_alternative_case_too_big(void) | |
295 | { | |
296 | extern unsigned int ftr_fixup_test3; | |
297 | extern unsigned int end_ftr_fixup_test3; | |
298 | extern unsigned int ftr_fixup_test3_orig; | |
299 | extern unsigned int ftr_fixup_test3_alt; | |
300 | int size = &end_ftr_fixup_test3 - &ftr_fixup_test3; | |
301 | ||
302 | fixup.value = fixup.mask = 0xC; | |
303 | fixup.start_off = calc_offset(&fixup, &ftr_fixup_test3 + 1); | |
304 | fixup.end_off = calc_offset(&fixup, &ftr_fixup_test3 + 2); | |
305 | fixup.alt_start_off = calc_offset(&fixup, &ftr_fixup_test3_alt); | |
306 | fixup.alt_end_off = calc_offset(&fixup, &ftr_fixup_test3_alt + 2); | |
307 | ||
308 | /* Sanity check */ | |
309 | check(memcmp(&ftr_fixup_test3, &ftr_fixup_test3_orig, size) == 0); | |
310 | ||
311 | /* Expect nothing to be patched, and the error returned to us */ | |
312 | check(patch_feature_section(0xF, &fixup) == 1); | |
313 | check(memcmp(&ftr_fixup_test3, &ftr_fixup_test3_orig, size) == 0); | |
314 | check(patch_feature_section(0, &fixup) == 1); | |
315 | check(memcmp(&ftr_fixup_test3, &ftr_fixup_test3_orig, size) == 0); | |
316 | check(patch_feature_section(~0xF, &fixup) == 1); | |
317 | check(memcmp(&ftr_fixup_test3, &ftr_fixup_test3_orig, size) == 0); | |
318 | } | |
319 | ||
320 | static void test_alternative_case_too_small(void) | |
321 | { | |
322 | extern unsigned int ftr_fixup_test4; | |
323 | extern unsigned int end_ftr_fixup_test4; | |
324 | extern unsigned int ftr_fixup_test4_orig; | |
325 | extern unsigned int ftr_fixup_test4_alt; | |
326 | extern unsigned int ftr_fixup_test4_expected; | |
327 | int size = &end_ftr_fixup_test4 - &ftr_fixup_test4; | |
328 | unsigned long flag; | |
329 | ||
330 | /* Check a high-bit flag */ | |
331 | flag = 1UL << ((sizeof(unsigned long) - 1) * 8); | |
332 | fixup.value = fixup.mask = flag; | |
333 | fixup.start_off = calc_offset(&fixup, &ftr_fixup_test4 + 1); | |
334 | fixup.end_off = calc_offset(&fixup, &ftr_fixup_test4 + 5); | |
335 | fixup.alt_start_off = calc_offset(&fixup, &ftr_fixup_test4_alt); | |
336 | fixup.alt_end_off = calc_offset(&fixup, &ftr_fixup_test4_alt + 2); | |
337 | ||
338 | /* Sanity check */ | |
339 | check(memcmp(&ftr_fixup_test4, &ftr_fixup_test4_orig, size) == 0); | |
340 | ||
341 | /* Check we don't patch if the value matches */ | |
342 | patch_feature_section(flag, &fixup); | |
343 | check(memcmp(&ftr_fixup_test4, &ftr_fixup_test4_orig, size) == 0); | |
344 | ||
345 | /* Check we do patch if the value doesn't match */ | |
346 | patch_feature_section(0, &fixup); | |
347 | check(memcmp(&ftr_fixup_test4, &ftr_fixup_test4_expected, size) == 0); | |
348 | ||
349 | /* Check we do patch if the mask doesn't match */ | |
350 | memcpy(&ftr_fixup_test4, &ftr_fixup_test4_orig, size); | |
351 | check(memcmp(&ftr_fixup_test4, &ftr_fixup_test4_orig, size) == 0); | |
352 | patch_feature_section(~flag, &fixup); | |
353 | check(memcmp(&ftr_fixup_test4, &ftr_fixup_test4_expected, size) == 0); | |
354 | } | |
355 | ||
356 | static void test_alternative_case_with_branch(void) | |
357 | { | |
358 | extern unsigned int ftr_fixup_test5; | |
359 | extern unsigned int end_ftr_fixup_test5; | |
360 | extern unsigned int ftr_fixup_test5_expected; | |
361 | int size = &end_ftr_fixup_test5 - &ftr_fixup_test5; | |
362 | ||
363 | check(memcmp(&ftr_fixup_test5, &ftr_fixup_test5_expected, size) == 0); | |
364 | } | |
365 | ||
366 | static void test_alternative_case_with_external_branch(void) | |
367 | { | |
368 | extern unsigned int ftr_fixup_test6; | |
369 | extern unsigned int end_ftr_fixup_test6; | |
370 | extern unsigned int ftr_fixup_test6_expected; | |
371 | int size = &end_ftr_fixup_test6 - &ftr_fixup_test6; | |
372 | ||
373 | check(memcmp(&ftr_fixup_test6, &ftr_fixup_test6_expected, size) == 0); | |
374 | } | |
375 | ||
376 | static void test_cpu_macros(void) | |
377 | { | |
3880ecb0 SR |
378 | extern u8 ftr_fixup_test_FTR_macros; |
379 | extern u8 ftr_fixup_test_FTR_macros_expected; | |
362e7701 ME |
380 | unsigned long size = &ftr_fixup_test_FTR_macros_expected - |
381 | &ftr_fixup_test_FTR_macros; | |
382 | ||
383 | /* The fixups have already been done for us during boot */ | |
384 | check(memcmp(&ftr_fixup_test_FTR_macros, | |
385 | &ftr_fixup_test_FTR_macros_expected, size) == 0); | |
386 | } | |
387 | ||
388 | static void test_fw_macros(void) | |
389 | { | |
390 | #ifdef CONFIG_PPC64 | |
3880ecb0 SR |
391 | extern u8 ftr_fixup_test_FW_FTR_macros; |
392 | extern u8 ftr_fixup_test_FW_FTR_macros_expected; | |
362e7701 ME |
393 | unsigned long size = &ftr_fixup_test_FW_FTR_macros_expected - |
394 | &ftr_fixup_test_FW_FTR_macros; | |
395 | ||
396 | /* The fixups have already been done for us during boot */ | |
397 | check(memcmp(&ftr_fixup_test_FW_FTR_macros, | |
398 | &ftr_fixup_test_FW_FTR_macros_expected, size) == 0); | |
399 | #endif | |
400 | } | |
401 | ||
2d1b2027 KG |
402 | static void test_lwsync_macros(void) |
403 | { | |
3880ecb0 SR |
404 | extern u8 lwsync_fixup_test; |
405 | extern u8 end_lwsync_fixup_test; | |
406 | extern u8 lwsync_fixup_test_expected_LWSYNC; | |
407 | extern u8 lwsync_fixup_test_expected_SYNC; | |
2d1b2027 KG |
408 | unsigned long size = &end_lwsync_fixup_test - |
409 | &lwsync_fixup_test; | |
410 | ||
411 | /* The fixups have already been done for us during boot */ | |
412 | if (cur_cpu_spec->cpu_features & CPU_FTR_LWSYNC) { | |
413 | check(memcmp(&lwsync_fixup_test, | |
414 | &lwsync_fixup_test_expected_LWSYNC, size) == 0); | |
415 | } else { | |
416 | check(memcmp(&lwsync_fixup_test, | |
417 | &lwsync_fixup_test_expected_SYNC, size) == 0); | |
418 | } | |
419 | } | |
420 | ||
362e7701 ME |
421 | static int __init test_feature_fixups(void) |
422 | { | |
423 | printk(KERN_DEBUG "Running feature fixup self-tests ...\n"); | |
424 | ||
425 | test_basic_patching(); | |
426 | test_alternative_patching(); | |
427 | test_alternative_case_too_big(); | |
428 | test_alternative_case_too_small(); | |
429 | test_alternative_case_with_branch(); | |
430 | test_alternative_case_with_external_branch(); | |
431 | test_cpu_macros(); | |
432 | test_fw_macros(); | |
2d1b2027 | 433 | test_lwsync_macros(); |
362e7701 ME |
434 | |
435 | return 0; | |
436 | } | |
437 | late_initcall(test_feature_fixups); | |
438 | ||
439 | #endif /* CONFIG_FTR_FIXUP_SELFTEST */ |