2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
6 * Support for Kernel relocation at boot time
8 * Copyright (C) 2015, Imagination Technologies Ltd.
9 * Authors: Matt Redfearn (matt.redfearn@imgtec.com)
11 #include <asm/bootinfo.h>
12 #include <asm/cacheflush.h>
13 #include <asm/fw/fw.h>
14 #include <asm/sections.h>
15 #include <asm/setup.h>
16 #include <asm/timex.h>
17 #include <linux/elf.h>
18 #include <linux/kernel.h>
19 #include <linux/libfdt.h>
20 #include <linux/of_fdt.h>
21 #include <linux/sched.h>
22 #include <linux/start_kernel.h>
23 #include <linux/string.h>
24 #include <linux/printk.h>
26 #define RELOCATED(x) ((void *)((long)x + offset))
28 extern u32 _relocation_start
[]; /* End kernel image / start relocation table */
29 extern u32 _relocation_end
[]; /* End relocation table */
31 extern long __start___ex_table
; /* Start exception table */
32 extern long __stop___ex_table
; /* End exception table */
34 extern void __weak
plat_fdt_relocated(void *new_location
);
37 * This function may be defined for a platform to perform any post-relocation
39 * Return non-zero to abort relocation
41 int __weak
plat_post_relocation(long offset
)
46 static inline u32 __init
get_synci_step(void)
50 __asm__("rdhwr %0, $1" : "=r" (res
));
55 static void __init
sync_icache(void *kbase
, unsigned long kernel_length
)
57 void *kend
= kbase
+ kernel_length
;
58 u32 step
= get_synci_step();
67 } while (kbase
< kend
);
69 /* Completion barrier */
73 static int __init
apply_r_mips_64_rel(u32
*loc_orig
, u32
*loc_new
, long offset
)
75 *(u64
*)loc_new
+= offset
;
80 static int __init
apply_r_mips_32_rel(u32
*loc_orig
, u32
*loc_new
, long offset
)
87 static int __init
apply_r_mips_26_rel(u32
*loc_orig
, u32
*loc_new
, long offset
)
89 unsigned long target_addr
= (*loc_orig
) & 0x03ffffff;
92 pr_err("Dangerous R_MIPS_26 REL relocation\n");
96 /* Original target address */
98 target_addr
+= (unsigned long)loc_orig
& ~0x03ffffff;
100 /* Get the new target address */
101 target_addr
+= offset
;
103 if ((target_addr
& 0xf0000000) != ((unsigned long)loc_new
& 0xf0000000)) {
104 pr_err("R_MIPS_26 REL relocation overflow\n");
108 target_addr
-= (unsigned long)loc_new
& ~0x03ffffff;
111 *loc_new
= (*loc_new
& ~0x03ffffff) | (target_addr
& 0x03ffffff);
117 static int __init
apply_r_mips_hi16_rel(u32
*loc_orig
, u32
*loc_new
, long offset
)
119 unsigned long insn
= *loc_orig
;
120 unsigned long target
= (insn
& 0xffff) << 16; /* high 16bits of target */
124 *loc_new
= (insn
& ~0xffff) | ((target
>> 16) & 0xffff);
128 static int (*reloc_handlers_rel
[]) (u32
*, u32
*, long) __initdata
= {
129 [R_MIPS_64
] = apply_r_mips_64_rel
,
130 [R_MIPS_32
] = apply_r_mips_32_rel
,
131 [R_MIPS_26
] = apply_r_mips_26_rel
,
132 [R_MIPS_HI16
] = apply_r_mips_hi16_rel
,
135 int __init
do_relocations(void *kbase_old
, void *kbase_new
, long offset
)
143 for (r
= _relocation_start
; r
< _relocation_end
; r
++) {
144 /* Sentinel for last relocation */
148 type
= (*r
>> 24) & 0xff;
149 loc_orig
= (void *)(kbase_old
+ ((*r
& 0x00ffffff) << 2));
150 loc_new
= RELOCATED(loc_orig
);
152 if (reloc_handlers_rel
[type
] == NULL
) {
153 /* Unsupported relocation */
154 pr_err("Unhandled relocation type %d at 0x%pK\n",
159 res
= reloc_handlers_rel
[type
](loc_orig
, loc_new
, offset
);
168 * The exception table is filled in by the relocs tool after vmlinux is linked.
169 * It must be relocated separately since there will not be any relocation
170 * information for it filled in by the linker.
172 static int __init
relocate_exception_table(long offset
)
174 unsigned long *etable_start
, *etable_end
, *e
;
176 etable_start
= RELOCATED(&__start___ex_table
);
177 etable_end
= RELOCATED(&__stop___ex_table
);
179 for (e
= etable_start
; e
< etable_end
; e
++)
185 #ifdef CONFIG_RANDOMIZE_BASE
187 static inline __init
unsigned long rotate_xor(unsigned long hash
,
188 const void *area
, size_t size
)
191 unsigned long *ptr
= (unsigned long *)area
;
193 for (i
= 0; i
< size
/ sizeof(hash
); i
++) {
194 /* Rotate by odd number of bits and XOR. */
195 hash
= (hash
<< ((sizeof(hash
) * 8) - 7)) | (hash
>> 7);
202 static inline __init
unsigned long get_random_boot(void)
204 unsigned long entropy
= random_get_entropy();
205 unsigned long hash
= 0;
207 /* Attempt to create a simple but unpredictable starting entropy. */
208 hash
= rotate_xor(hash
, linux_banner
, strlen(linux_banner
));
210 /* Add in any runtime entropy we can get */
211 hash
= rotate_xor(hash
, &entropy
, sizeof(entropy
));
213 #if defined(CONFIG_USE_OF)
214 /* Get any additional entropy passed in device tree */
215 if (initial_boot_params
) {
219 node
= fdt_path_offset(initial_boot_params
, "/chosen");
221 prop
= fdt_getprop_w(initial_boot_params
, node
,
223 if (prop
&& (len
== sizeof(u64
)))
224 hash
= rotate_xor(hash
, prop
, sizeof(*prop
));
227 #endif /* CONFIG_USE_OF */
232 static inline __init
bool kaslr_disabled(void)
236 #if defined(CONFIG_CMDLINE_BOOL)
237 const char *builtin_cmdline
= CONFIG_CMDLINE
;
239 str
= strstr(builtin_cmdline
, "nokaslr");
240 if (str
== builtin_cmdline
||
241 (str
> builtin_cmdline
&& *(str
- 1) == ' '))
244 str
= strstr(arcs_cmdline
, "nokaslr");
245 if (str
== arcs_cmdline
|| (str
> arcs_cmdline
&& *(str
- 1) == ' '))
251 static inline void __init
*determine_relocation_address(void)
253 /* Choose a new address for the kernel */
254 unsigned long kernel_length
;
256 unsigned long offset
;
258 if (kaslr_disabled())
261 kernel_length
= (long)_end
- (long)(&_text
);
263 offset
= get_random_boot() << 16;
264 offset
&= (CONFIG_RANDOMIZE_BASE_MAX_OFFSET
- 1);
265 if (offset
< kernel_length
)
266 offset
+= ALIGN(kernel_length
, 0xffff);
268 return RELOCATED(dest
);
273 static inline void __init
*determine_relocation_address(void)
276 * Choose a new address for the kernel
277 * For now we'll hard code the destination
279 return (void *)0xffffffff81000000;
284 static inline int __init
relocation_addr_valid(void *loc_new
)
286 if ((unsigned long)loc_new
& 0x0000ffff) {
287 /* Inappropriately aligned new location */
290 if ((unsigned long)loc_new
< (unsigned long)&_end
) {
291 /* New location overlaps original kernel */
297 void *__init
relocate_kernel(void)
300 unsigned long kernel_length
;
301 unsigned long bss_length
;
304 /* Default to original kernel entry point */
305 void *kernel_entry
= start_kernel
;
308 /* Get the command line */
310 #if defined(CONFIG_USE_OF)
311 /* Deal with the device tree */
312 fdt
= plat_get_fdt();
313 early_init_dt_scan(fdt
);
314 if (boot_command_line
[0]) {
315 /* Boot command line was passed in device tree */
316 strlcpy(arcs_cmdline
, boot_command_line
, COMMAND_LINE_SIZE
);
318 #endif /* CONFIG_USE_OF */
320 kernel_length
= (long)(&_relocation_start
) - (long)(&_text
);
321 bss_length
= (long)&__bss_stop
- (long)&__bss_start
;
323 loc_new
= determine_relocation_address();
325 /* Sanity check relocation address */
326 if (relocation_addr_valid(loc_new
))
327 offset
= (unsigned long)loc_new
- (unsigned long)(&_text
);
329 /* Reset the command line now so we don't end up with a duplicate */
330 arcs_cmdline
[0] = '\0';
333 void (*fdt_relocated_
)(void *) = NULL
;
334 #if defined(CONFIG_USE_OF)
335 unsigned long fdt_phys
= virt_to_phys(fdt
);
338 * If built-in dtb is used then it will have been relocated
339 * during kernel _text relocation. If appended DTB is used
340 * then it will not be relocated, but it should remain
341 * intact in the original location. If dtb is loaded by
342 * the bootloader then it may need to be moved if it crosses
343 * the target memory area
346 if (fdt_phys
>= virt_to_phys(RELOCATED(&_text
)) &&
347 fdt_phys
<= virt_to_phys(RELOCATED(&_end
))) {
348 void *fdt_relocated
=
349 RELOCATED(ALIGN((long)&_end
, PAGE_SIZE
));
350 memcpy(fdt_relocated
, fdt
, fdt_totalsize(fdt
));
352 fdt_relocated_
= RELOCATED(&plat_fdt_relocated
);
354 #endif /* CONFIG_USE_OF */
356 /* Copy the kernel to it's new location */
357 memcpy(loc_new
, &_text
, kernel_length
);
359 /* Perform relocations on the new kernel */
360 res
= do_relocations(&_text
, loc_new
, offset
);
364 /* Sync the caches ready for execution of new kernel */
365 sync_icache(loc_new
, kernel_length
);
367 res
= relocate_exception_table(offset
);
372 * The original .bss has already been cleared, and
373 * some variables such as command line parameters
374 * stored to it so make a copy in the new location.
376 memcpy(RELOCATED(&__bss_start
), &__bss_start
, bss_length
);
379 * If fdt was stored outside of the kernel image and
380 * had to be moved then update platform's state data
381 * with the new fdt location
387 * Last chance for the platform to abort relocation.
388 * This may also be used by the platform to perform any
389 * initialisation required now that the new kernel is
390 * resident in memory and ready to be executed.
392 if (plat_post_relocation(offset
))
395 /* The current thread is now within the relocated image */
396 __current_thread_info
= RELOCATED(&init_thread_union
);
398 /* Return the new kernel's entry point */
399 kernel_entry
= RELOCATED(start_kernel
);
406 * Show relocation information on panic.
408 void show_kernel_relocation(const char *level
)
410 unsigned long offset
;
412 offset
= __pa_symbol(_text
) - __pa_symbol(VMLINUX_LOAD_ADDRESS
);
414 if (IS_ENABLED(CONFIG_RELOCATABLE
) && offset
> 0) {
416 pr_cont("Kernel relocated by 0x%pK\n", (void *)offset
);
417 pr_cont(" .text @ 0x%pK\n", _text
);
418 pr_cont(" .data @ 0x%pK\n", _sdata
);
419 pr_cont(" .bss @ 0x%pK\n", __bss_start
);
423 static int kernel_location_notifier_fn(struct notifier_block
*self
,
424 unsigned long v
, void *p
)
426 show_kernel_relocation(KERN_EMERG
);
430 static struct notifier_block kernel_location_notifier
= {
431 .notifier_call
= kernel_location_notifier_fn
434 static int __init
register_kernel_offset_dumper(void)
436 atomic_notifier_chain_register(&panic_notifier_list
,
437 &kernel_location_notifier
);
440 __initcall(register_kernel_offset_dumper
);