]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
7de828df KC |
2 | /* |
3 | * kaslr.c | |
4 | * | |
5 | * This contains the routines needed to generate a reasonable level of | |
6 | * entropy to choose a randomized kernel base address offset in support | |
7 | * of Kernel Address Space Layout Randomization (KASLR). Additionally | |
8 | * handles walking the physical memory maps (and tracking memory regions | |
9 | * to avoid) in order to select a physical memory location that can | |
10 | * contain the entire properly aligned running kernel image. | |
11 | * | |
12 | */ | |
d52e7d5a BH |
13 | |
14 | /* | |
15 | * isspace() in linux/ctype.h is expected by next_args() to filter | |
16 | * out "space/lf/tab". While boot/ctype.h conflicts with linux/ctype.h, | |
17 | * since isdigit() is implemented in both of them. Hence disable it | |
18 | * here. | |
19 | */ | |
20 | #define BOOT_CTYPE_H | |
21 | ||
22 | /* | |
23 | * _ctype[] in lib/ctype.c is needed by isspace() of linux/ctype.h. | |
24 | * While both lib/ctype.c and lib/cmdline.c will bring EXPORT_SYMBOL | |
25 | * which is meaningless and will cause compiling error in some cases. | |
26 | * So do not include linux/export.h and define EXPORT_SYMBOL(sym) | |
27 | * as empty. | |
28 | */ | |
29 | #define _LINUX_EXPORT_H | |
30 | #define EXPORT_SYMBOL(sym) | |
31 | ||
8ab3820f | 32 | #include "misc.h" |
dc425a6e | 33 | #include "error.h" |
5b8b9cf7 | 34 | #include "../string.h" |
8ab3820f | 35 | |
a653f356 KC |
36 | #include <generated/compile.h> |
37 | #include <linux/module.h> | |
38 | #include <linux/uts.h> | |
39 | #include <linux/utsname.h> | |
d52e7d5a | 40 | #include <linux/ctype.h> |
c05cd797 | 41 | #include <linux/efi.h> |
a653f356 | 42 | #include <generated/utsrelease.h> |
c05cd797 | 43 | #include <asm/efi.h> |
a653f356 | 44 | |
d52e7d5a BH |
45 | /* Macros used by the included decompressor code below. */ |
46 | #define STATIC | |
47 | #include <linux/decompress/mm.h> | |
48 | ||
49 | extern unsigned long get_cmd_line_ptr(void); | |
50 | ||
a653f356 | 51 | /* Simplified build-specific string for starting entropy. */ |
327f7d72 | 52 | static const char build_str[] = UTS_RELEASE " (" LINUX_COMPILE_BY "@" |
a653f356 KC |
53 | LINUX_COMPILE_HOST ") (" LINUX_COMPILER ") " UTS_VERSION; |
54 | ||
a653f356 KC |
55 | static unsigned long rotate_xor(unsigned long hash, const void *area, |
56 | size_t size) | |
57 | { | |
58 | size_t i; | |
59 | unsigned long *ptr = (unsigned long *)area; | |
60 | ||
61 | for (i = 0; i < size / sizeof(hash); i++) { | |
62 | /* Rotate by odd number of bits and XOR. */ | |
63 | hash = (hash << ((sizeof(hash) * 8) - 7)) | (hash >> 7); | |
64 | hash ^= ptr[i]; | |
65 | } | |
66 | ||
67 | return hash; | |
68 | } | |
69 | ||
70 | /* Attempt to create a simple but unpredictable starting entropy. */ | |
d899a7d1 | 71 | static unsigned long get_boot_seed(void) |
a653f356 KC |
72 | { |
73 | unsigned long hash = 0; | |
74 | ||
75 | hash = rotate_xor(hash, build_str, sizeof(build_str)); | |
6655e0aa | 76 | hash = rotate_xor(hash, boot_params, sizeof(*boot_params)); |
a653f356 KC |
77 | |
78 | return hash; | |
79 | } | |
80 | ||
d899a7d1 TG |
81 | #define KASLR_COMPRESSED_BOOT |
82 | #include "../../lib/kaslr.c" | |
8ab3820f | 83 | |
82fa9637 | 84 | struct mem_vector { |
f2844249 DJ |
85 | unsigned long long start; |
86 | unsigned long long size; | |
82fa9637 KC |
87 | }; |
88 | ||
f2844249 DJ |
89 | /* Only supporting at most 4 unusable memmap regions with kaslr */ |
90 | #define MAX_MEMMAP_REGIONS 4 | |
91 | ||
92 | static bool memmap_too_large; | |
93 | ||
d52e7d5a | 94 | |
4cdba14f BH |
95 | /* Store memory limit specified by "mem=nn[KMG]" or "memmap=nn[KMG]" */ |
96 | unsigned long long mem_limit = ULLONG_MAX; | |
97 | ||
98 | ||
ed09acde KC |
99 | enum mem_avoid_index { |
100 | MEM_AVOID_ZO_RANGE = 0, | |
101 | MEM_AVOID_INITRD, | |
102 | MEM_AVOID_CMDLINE, | |
103 | MEM_AVOID_BOOTPARAMS, | |
f2844249 DJ |
104 | MEM_AVOID_MEMMAP_BEGIN, |
105 | MEM_AVOID_MEMMAP_END = MEM_AVOID_MEMMAP_BEGIN + MAX_MEMMAP_REGIONS - 1, | |
ed09acde KC |
106 | MEM_AVOID_MAX, |
107 | }; | |
108 | ||
e290e8c5 | 109 | static struct mem_vector mem_avoid[MEM_AVOID_MAX]; |
82fa9637 | 110 | |
82fa9637 KC |
111 | static bool mem_overlaps(struct mem_vector *one, struct mem_vector *two) |
112 | { | |
113 | /* Item one is entirely before item two. */ | |
114 | if (one->start + one->size <= two->start) | |
115 | return false; | |
116 | /* Item one is entirely after item two. */ | |
117 | if (one->start >= two->start + two->size) | |
118 | return false; | |
119 | return true; | |
120 | } | |
121 | ||
d52e7d5a | 122 | char *skip_spaces(const char *str) |
f2844249 | 123 | { |
d52e7d5a BH |
124 | while (isspace(*str)) |
125 | ++str; | |
126 | return (char *)str; | |
f2844249 | 127 | } |
d52e7d5a BH |
128 | #include "../../../../lib/ctype.c" |
129 | #include "../../../../lib/cmdline.c" | |
f2844249 DJ |
130 | |
131 | static int | |
132 | parse_memmap(char *p, unsigned long long *start, unsigned long long *size) | |
133 | { | |
134 | char *oldp; | |
135 | ||
136 | if (!p) | |
137 | return -EINVAL; | |
138 | ||
139 | /* We don't care about this option here */ | |
140 | if (!strncmp(p, "exactmap", 8)) | |
141 | return -EINVAL; | |
142 | ||
143 | oldp = p; | |
d52e7d5a | 144 | *size = memparse(p, &p); |
f2844249 DJ |
145 | if (p == oldp) |
146 | return -EINVAL; | |
147 | ||
148 | switch (*p) { | |
f2844249 DJ |
149 | case '#': |
150 | case '$': | |
151 | case '!': | |
d52e7d5a | 152 | *start = memparse(p + 1, &p); |
f2844249 | 153 | return 0; |
4cdba14f BH |
154 | case '@': |
155 | /* memmap=nn@ss specifies usable region, should be skipped */ | |
156 | *size = 0; | |
157 | /* Fall through */ | |
158 | default: | |
159 | /* | |
160 | * If w/o offset, only size specified, memmap=nn[KMG] has the | |
161 | * same behaviour as mem=nn[KMG]. It limits the max address | |
162 | * system can use. Region above the limit should be avoided. | |
163 | */ | |
164 | *start = 0; | |
f2844249 DJ |
165 | return 0; |
166 | } | |
167 | ||
168 | return -EINVAL; | |
169 | } | |
170 | ||
d52e7d5a | 171 | static void mem_avoid_memmap(char *str) |
f2844249 | 172 | { |
d52e7d5a | 173 | static int i; |
f2844249 | 174 | int rc; |
f2844249 | 175 | |
d52e7d5a | 176 | if (i >= MAX_MEMMAP_REGIONS) |
f2844249 DJ |
177 | return; |
178 | ||
f2844249 DJ |
179 | while (str && (i < MAX_MEMMAP_REGIONS)) { |
180 | int rc; | |
181 | unsigned long long start, size; | |
182 | char *k = strchr(str, ','); | |
183 | ||
184 | if (k) | |
185 | *k++ = 0; | |
186 | ||
187 | rc = parse_memmap(str, &start, &size); | |
188 | if (rc < 0) | |
189 | break; | |
190 | str = k; | |
4cdba14f BH |
191 | |
192 | if (start == 0) { | |
193 | /* Store the specified memory limit if size > 0 */ | |
194 | if (size > 0) | |
195 | mem_limit = size; | |
196 | ||
f2844249 | 197 | continue; |
4cdba14f | 198 | } |
f2844249 DJ |
199 | |
200 | mem_avoid[MEM_AVOID_MEMMAP_BEGIN + i].start = start; | |
201 | mem_avoid[MEM_AVOID_MEMMAP_BEGIN + i].size = size; | |
202 | i++; | |
203 | } | |
204 | ||
205 | /* More than 4 memmaps, fail kaslr */ | |
206 | if ((i >= MAX_MEMMAP_REGIONS) && str) | |
207 | memmap_too_large = true; | |
208 | } | |
209 | ||
d52e7d5a BH |
210 | static int handle_mem_memmap(void) |
211 | { | |
212 | char *args = (char *)get_cmd_line_ptr(); | |
213 | size_t len = strlen((char *)args); | |
214 | char *tmp_cmdline; | |
215 | char *param, *val; | |
4cdba14f | 216 | u64 mem_size; |
d52e7d5a | 217 | |
4cdba14f | 218 | if (!strstr(args, "memmap=") && !strstr(args, "mem=")) |
d52e7d5a BH |
219 | return 0; |
220 | ||
221 | tmp_cmdline = malloc(len + 1); | |
222 | if (!tmp_cmdline ) | |
223 | error("Failed to allocate space for tmp_cmdline"); | |
224 | ||
225 | memcpy(tmp_cmdline, args, len); | |
226 | tmp_cmdline[len] = 0; | |
227 | args = tmp_cmdline; | |
228 | ||
229 | /* Chew leading spaces */ | |
230 | args = skip_spaces(args); | |
231 | ||
232 | while (*args) { | |
233 | args = next_arg(args, ¶m, &val); | |
234 | /* Stop at -- */ | |
235 | if (!val && strcmp(param, "--") == 0) { | |
236 | warn("Only '--' specified in cmdline"); | |
237 | free(tmp_cmdline); | |
238 | return -1; | |
239 | } | |
240 | ||
4cdba14f | 241 | if (!strcmp(param, "memmap")) { |
d52e7d5a | 242 | mem_avoid_memmap(val); |
4cdba14f BH |
243 | } else if (!strcmp(param, "mem")) { |
244 | char *p = val; | |
245 | ||
246 | if (!strcmp(p, "nopentium")) | |
247 | continue; | |
248 | mem_size = memparse(p, &p); | |
249 | if (mem_size == 0) { | |
250 | free(tmp_cmdline); | |
251 | return -EINVAL; | |
252 | } | |
253 | mem_limit = mem_size; | |
254 | } | |
d52e7d5a BH |
255 | } |
256 | ||
257 | free(tmp_cmdline); | |
258 | return 0; | |
259 | } | |
260 | ||
9dc1969c | 261 | /* |
ed09acde KC |
262 | * In theory, KASLR can put the kernel anywhere in the range of [16M, 64T). |
263 | * The mem_avoid array is used to store the ranges that need to be avoided | |
264 | * when KASLR searches for an appropriate random address. We must avoid any | |
9dc1969c | 265 | * regions that are unsafe to overlap with during decompression, and other |
ed09acde KC |
266 | * things like the initrd, cmdline and boot_params. This comment seeks to |
267 | * explain mem_avoid as clearly as possible since incorrect mem_avoid | |
268 | * memory ranges lead to really hard to debug boot failures. | |
269 | * | |
270 | * The initrd, cmdline, and boot_params are trivial to identify for | |
cb18ef0d | 271 | * avoiding. They are MEM_AVOID_INITRD, MEM_AVOID_CMDLINE, and |
ed09acde KC |
272 | * MEM_AVOID_BOOTPARAMS respectively below. |
273 | * | |
274 | * What is not obvious how to avoid is the range of memory that is used | |
275 | * during decompression (MEM_AVOID_ZO_RANGE below). This range must cover | |
276 | * the compressed kernel (ZO) and its run space, which is used to extract | |
277 | * the uncompressed kernel (VO) and relocs. | |
278 | * | |
279 | * ZO's full run size sits against the end of the decompression buffer, so | |
280 | * we can calculate where text, data, bss, etc of ZO are positioned more | |
281 | * easily. | |
282 | * | |
283 | * For additional background, the decompression calculations can be found | |
284 | * in header.S, and the memory diagram is based on the one found in misc.c. | |
285 | * | |
286 | * The following conditions are already enforced by the image layouts and | |
287 | * associated code: | |
288 | * - input + input_size >= output + output_size | |
289 | * - kernel_total_size <= init_size | |
290 | * - kernel_total_size <= output_size (see Note below) | |
291 | * - output + init_size >= output + output_size | |
9dc1969c | 292 | * |
ed09acde KC |
293 | * (Note that kernel_total_size and output_size have no fundamental |
294 | * relationship, but output_size is passed to choose_random_location | |
295 | * as a maximum of the two. The diagram is showing a case where | |
296 | * kernel_total_size is larger than output_size, but this case is | |
297 | * handled by bumping output_size.) | |
9dc1969c | 298 | * |
ed09acde | 299 | * The above conditions can be illustrated by a diagram: |
9dc1969c | 300 | * |
ed09acde KC |
301 | * 0 output input input+input_size output+init_size |
302 | * | | | | | | |
303 | * | | | | | | |
304 | * |-----|--------|--------|--------------|-----------|--|-------------| | |
305 | * | | | | |
306 | * | | | | |
307 | * output+init_size-ZO_INIT_SIZE output+output_size output+kernel_total_size | |
9dc1969c | 308 | * |
ed09acde KC |
309 | * [output, output+init_size) is the entire memory range used for |
310 | * extracting the compressed image. | |
9dc1969c | 311 | * |
ed09acde KC |
312 | * [output, output+kernel_total_size) is the range needed for the |
313 | * uncompressed kernel (VO) and its run size (bss, brk, etc). | |
9dc1969c | 314 | * |
ed09acde KC |
315 | * [output, output+output_size) is VO plus relocs (i.e. the entire |
316 | * uncompressed payload contained by ZO). This is the area of the buffer | |
317 | * written to during decompression. | |
9dc1969c | 318 | * |
ed09acde KC |
319 | * [output+init_size-ZO_INIT_SIZE, output+init_size) is the worst-case |
320 | * range of the copied ZO and decompression code. (i.e. the range | |
321 | * covered backwards of size ZO_INIT_SIZE, starting from output+init_size.) | |
9dc1969c | 322 | * |
ed09acde KC |
323 | * [input, input+input_size) is the original copied compressed image (ZO) |
324 | * (i.e. it does not include its run size). This range must be avoided | |
325 | * because it contains the data used for decompression. | |
9dc1969c | 326 | * |
ed09acde KC |
327 | * [input+input_size, output+init_size) is [_text, _end) for ZO. This |
328 | * range includes ZO's heap and stack, and must be avoided since it | |
329 | * performs the decompression. | |
9dc1969c | 330 | * |
ed09acde KC |
331 | * Since the above two ranges need to be avoided and they are adjacent, |
332 | * they can be merged, resulting in: [input, output+init_size) which | |
333 | * becomes the MEM_AVOID_ZO_RANGE below. | |
9dc1969c | 334 | */ |
82fa9637 | 335 | static void mem_avoid_init(unsigned long input, unsigned long input_size, |
9dc1969c | 336 | unsigned long output) |
82fa9637 | 337 | { |
9dc1969c | 338 | unsigned long init_size = boot_params->hdr.init_size; |
82fa9637 KC |
339 | u64 initrd_start, initrd_size; |
340 | u64 cmd_line, cmd_line_size; | |
82fa9637 KC |
341 | char *ptr; |
342 | ||
343 | /* | |
344 | * Avoid the region that is unsafe to overlap during | |
9dc1969c | 345 | * decompression. |
82fa9637 | 346 | */ |
ed09acde KC |
347 | mem_avoid[MEM_AVOID_ZO_RANGE].start = input; |
348 | mem_avoid[MEM_AVOID_ZO_RANGE].size = (output + init_size) - input; | |
3a94707d KC |
349 | add_identity_map(mem_avoid[MEM_AVOID_ZO_RANGE].start, |
350 | mem_avoid[MEM_AVOID_ZO_RANGE].size); | |
82fa9637 KC |
351 | |
352 | /* Avoid initrd. */ | |
6655e0aa KC |
353 | initrd_start = (u64)boot_params->ext_ramdisk_image << 32; |
354 | initrd_start |= boot_params->hdr.ramdisk_image; | |
355 | initrd_size = (u64)boot_params->ext_ramdisk_size << 32; | |
356 | initrd_size |= boot_params->hdr.ramdisk_size; | |
ed09acde KC |
357 | mem_avoid[MEM_AVOID_INITRD].start = initrd_start; |
358 | mem_avoid[MEM_AVOID_INITRD].size = initrd_size; | |
3a94707d | 359 | /* No need to set mapping for initrd, it will be handled in VO. */ |
82fa9637 KC |
360 | |
361 | /* Avoid kernel command line. */ | |
6655e0aa KC |
362 | cmd_line = (u64)boot_params->ext_cmd_line_ptr << 32; |
363 | cmd_line |= boot_params->hdr.cmd_line_ptr; | |
82fa9637 KC |
364 | /* Calculate size of cmd_line. */ |
365 | ptr = (char *)(unsigned long)cmd_line; | |
366 | for (cmd_line_size = 0; ptr[cmd_line_size++]; ) | |
367 | ; | |
ed09acde KC |
368 | mem_avoid[MEM_AVOID_CMDLINE].start = cmd_line; |
369 | mem_avoid[MEM_AVOID_CMDLINE].size = cmd_line_size; | |
3a94707d KC |
370 | add_identity_map(mem_avoid[MEM_AVOID_CMDLINE].start, |
371 | mem_avoid[MEM_AVOID_CMDLINE].size); | |
82fa9637 | 372 | |
ed09acde KC |
373 | /* Avoid boot parameters. */ |
374 | mem_avoid[MEM_AVOID_BOOTPARAMS].start = (unsigned long)boot_params; | |
375 | mem_avoid[MEM_AVOID_BOOTPARAMS].size = sizeof(*boot_params); | |
3a94707d KC |
376 | add_identity_map(mem_avoid[MEM_AVOID_BOOTPARAMS].start, |
377 | mem_avoid[MEM_AVOID_BOOTPARAMS].size); | |
378 | ||
379 | /* We don't need to set a mapping for setup_data. */ | |
380 | ||
f2844249 | 381 | /* Mark the memmap regions we need to avoid */ |
d52e7d5a | 382 | handle_mem_memmap(); |
f2844249 | 383 | |
3a94707d KC |
384 | #ifdef CONFIG_X86_VERBOSE_BOOTUP |
385 | /* Make sure video RAM can be used. */ | |
386 | add_identity_map(0, PMD_SIZE); | |
387 | #endif | |
82fa9637 KC |
388 | } |
389 | ||
06486d6c KC |
390 | /* |
391 | * Does this memory vector overlap a known avoided area? If so, record the | |
392 | * overlap region with the lowest address. | |
393 | */ | |
394 | static bool mem_avoid_overlap(struct mem_vector *img, | |
395 | struct mem_vector *overlap) | |
82fa9637 KC |
396 | { |
397 | int i; | |
0cacbfbe | 398 | struct setup_data *ptr; |
06486d6c KC |
399 | unsigned long earliest = img->start + img->size; |
400 | bool is_overlapping = false; | |
82fa9637 KC |
401 | |
402 | for (i = 0; i < MEM_AVOID_MAX; i++) { | |
06486d6c KC |
403 | if (mem_overlaps(img, &mem_avoid[i]) && |
404 | mem_avoid[i].start < earliest) { | |
405 | *overlap = mem_avoid[i]; | |
6daa2ec0 | 406 | earliest = overlap->start; |
06486d6c KC |
407 | is_overlapping = true; |
408 | } | |
82fa9637 KC |
409 | } |
410 | ||
0cacbfbe | 411 | /* Avoid all entries in the setup_data linked list. */ |
6655e0aa | 412 | ptr = (struct setup_data *)(unsigned long)boot_params->hdr.setup_data; |
0cacbfbe KC |
413 | while (ptr) { |
414 | struct mem_vector avoid; | |
415 | ||
20cc2888 | 416 | avoid.start = (unsigned long)ptr; |
0cacbfbe KC |
417 | avoid.size = sizeof(*ptr) + ptr->len; |
418 | ||
06486d6c KC |
419 | if (mem_overlaps(img, &avoid) && (avoid.start < earliest)) { |
420 | *overlap = avoid; | |
6daa2ec0 | 421 | earliest = overlap->start; |
06486d6c KC |
422 | is_overlapping = true; |
423 | } | |
0cacbfbe KC |
424 | |
425 | ptr = (struct setup_data *)(unsigned long)ptr->next; | |
426 | } | |
427 | ||
06486d6c | 428 | return is_overlapping; |
82fa9637 KC |
429 | } |
430 | ||
c401cf15 BH |
431 | struct slot_area { |
432 | unsigned long addr; | |
433 | int num; | |
434 | }; | |
435 | ||
436 | #define MAX_SLOT_AREA 100 | |
437 | ||
438 | static struct slot_area slot_areas[MAX_SLOT_AREA]; | |
439 | ||
e290e8c5 | 440 | static unsigned long slot_max; |
82fa9637 | 441 | |
c401cf15 BH |
442 | static unsigned long slot_area_index; |
443 | ||
444 | static void store_slot_info(struct mem_vector *region, unsigned long image_size) | |
445 | { | |
446 | struct slot_area slot_area; | |
447 | ||
448 | if (slot_area_index == MAX_SLOT_AREA) | |
449 | return; | |
450 | ||
451 | slot_area.addr = region->start; | |
452 | slot_area.num = (region->size - image_size) / | |
453 | CONFIG_PHYSICAL_ALIGN + 1; | |
454 | ||
455 | if (slot_area.num > 0) { | |
456 | slot_areas[slot_area_index++] = slot_area; | |
457 | slot_max += slot_area.num; | |
458 | } | |
459 | } | |
460 | ||
82fa9637 KC |
461 | static unsigned long slots_fetch_random(void) |
462 | { | |
ed9f007e KC |
463 | unsigned long slot; |
464 | int i; | |
465 | ||
82fa9637 KC |
466 | /* Handle case of no slots stored. */ |
467 | if (slot_max == 0) | |
468 | return 0; | |
469 | ||
d899a7d1 | 470 | slot = kaslr_get_random_long("Physical") % slot_max; |
ed9f007e KC |
471 | |
472 | for (i = 0; i < slot_area_index; i++) { | |
473 | if (slot >= slot_areas[i].num) { | |
474 | slot -= slot_areas[i].num; | |
475 | continue; | |
476 | } | |
477 | return slot_areas[i].addr + slot * CONFIG_PHYSICAL_ALIGN; | |
478 | } | |
479 | ||
480 | if (i == slot_area_index) | |
481 | debug_putstr("slots_fetch_random() failed!?\n"); | |
482 | return 0; | |
82fa9637 KC |
483 | } |
484 | ||
27aac205 | 485 | static void process_mem_region(struct mem_vector *entry, |
82fa9637 KC |
486 | unsigned long minimum, |
487 | unsigned long image_size) | |
488 | { | |
ed9f007e KC |
489 | struct mem_vector region, overlap; |
490 | struct slot_area slot_area; | |
4cdba14f | 491 | unsigned long start_orig, end; |
87891b01 | 492 | struct mem_vector cur_entry; |
82fa9637 | 493 | |
ed9f007e | 494 | /* On 32-bit, ignore entries entirely above our maximum. */ |
87891b01 | 495 | if (IS_ENABLED(CONFIG_X86_32) && entry->start >= KERNEL_IMAGE_SIZE) |
82fa9637 KC |
496 | return; |
497 | ||
498 | /* Ignore entries entirely below our minimum. */ | |
87891b01 | 499 | if (entry->start + entry->size < minimum) |
82fa9637 KC |
500 | return; |
501 | ||
4cdba14f | 502 | /* Ignore entries above memory limit */ |
87891b01 BH |
503 | end = min(entry->size + entry->start, mem_limit); |
504 | if (entry->start >= end) | |
4cdba14f | 505 | return; |
87891b01 BH |
506 | cur_entry.start = entry->start; |
507 | cur_entry.size = end - entry->start; | |
4cdba14f | 508 | |
87891b01 | 509 | region.start = cur_entry.start; |
4cdba14f | 510 | region.size = cur_entry.size; |
82fa9637 | 511 | |
ed9f007e KC |
512 | /* Give up if slot area array is full. */ |
513 | while (slot_area_index < MAX_SLOT_AREA) { | |
514 | start_orig = region.start; | |
82fa9637 | 515 | |
ed9f007e KC |
516 | /* Potentially raise address to minimum location. */ |
517 | if (region.start < minimum) | |
518 | region.start = minimum; | |
82fa9637 | 519 | |
ed9f007e KC |
520 | /* Potentially raise address to meet alignment needs. */ |
521 | region.start = ALIGN(region.start, CONFIG_PHYSICAL_ALIGN); | |
82fa9637 | 522 | |
27aac205 | 523 | /* Did we raise the address above the passed in memory entry? */ |
87891b01 | 524 | if (region.start > cur_entry.start + cur_entry.size) |
ed9f007e | 525 | return; |
82fa9637 | 526 | |
ed9f007e KC |
527 | /* Reduce size by any delta from the original address. */ |
528 | region.size -= region.start - start_orig; | |
82fa9637 | 529 | |
ed9f007e KC |
530 | /* On 32-bit, reduce region size to fit within max size. */ |
531 | if (IS_ENABLED(CONFIG_X86_32) && | |
532 | region.start + region.size > KERNEL_IMAGE_SIZE) | |
533 | region.size = KERNEL_IMAGE_SIZE - region.start; | |
534 | ||
535 | /* Return if region can't contain decompressed kernel */ | |
536 | if (region.size < image_size) | |
537 | return; | |
538 | ||
539 | /* If nothing overlaps, store the region and return. */ | |
540 | if (!mem_avoid_overlap(®ion, &overlap)) { | |
541 | store_slot_info(®ion, image_size); | |
542 | return; | |
543 | } | |
544 | ||
545 | /* Store beginning of region if holds at least image_size. */ | |
546 | if (overlap.start > region.start + image_size) { | |
547 | struct mem_vector beginning; | |
548 | ||
549 | beginning.start = region.start; | |
550 | beginning.size = overlap.start - region.start; | |
551 | store_slot_info(&beginning, image_size); | |
552 | } | |
553 | ||
554 | /* Return if overlap extends to or past end of region. */ | |
555 | if (overlap.start + overlap.size >= region.start + region.size) | |
556 | return; | |
557 | ||
558 | /* Clip off the overlapping region and start over. */ | |
559 | region.size -= overlap.start - region.start + overlap.size; | |
560 | region.start = overlap.start + overlap.size; | |
82fa9637 KC |
561 | } |
562 | } | |
563 | ||
c05cd797 BH |
564 | #ifdef CONFIG_EFI |
565 | /* | |
566 | * Returns true if mirror region found (and must have been processed | |
567 | * for slots adding) | |
568 | */ | |
569 | static bool | |
570 | process_efi_entries(unsigned long minimum, unsigned long image_size) | |
571 | { | |
572 | struct efi_info *e = &boot_params->efi_info; | |
573 | bool efi_mirror_found = false; | |
574 | struct mem_vector region; | |
575 | efi_memory_desc_t *md; | |
576 | unsigned long pmap; | |
577 | char *signature; | |
578 | u32 nr_desc; | |
579 | int i; | |
580 | ||
581 | signature = (char *)&e->efi_loader_signature; | |
582 | if (strncmp(signature, EFI32_LOADER_SIGNATURE, 4) && | |
583 | strncmp(signature, EFI64_LOADER_SIGNATURE, 4)) | |
584 | return false; | |
585 | ||
586 | #ifdef CONFIG_X86_32 | |
587 | /* Can't handle data above 4GB at this time */ | |
588 | if (e->efi_memmap_hi) { | |
589 | warn("EFI memmap is above 4GB, can't be handled now on x86_32. EFI should be disabled.\n"); | |
590 | return false; | |
591 | } | |
592 | pmap = e->efi_memmap; | |
593 | #else | |
594 | pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32)); | |
595 | #endif | |
596 | ||
597 | nr_desc = e->efi_memmap_size / e->efi_memdesc_size; | |
598 | for (i = 0; i < nr_desc; i++) { | |
599 | md = efi_early_memdesc_ptr(pmap, e->efi_memdesc_size, i); | |
600 | if (md->attribute & EFI_MEMORY_MORE_RELIABLE) { | |
c05cd797 | 601 | efi_mirror_found = true; |
0982adc7 | 602 | break; |
c05cd797 BH |
603 | } |
604 | } | |
605 | ||
0982adc7 NH |
606 | for (i = 0; i < nr_desc; i++) { |
607 | md = efi_early_memdesc_ptr(pmap, e->efi_memdesc_size, i); | |
608 | ||
609 | /* | |
610 | * Here we are more conservative in picking free memory than | |
611 | * the EFI spec allows: | |
612 | * | |
613 | * According to the spec, EFI_BOOT_SERVICES_{CODE|DATA} are also | |
614 | * free memory and thus available to place the kernel image into, | |
615 | * but in practice there's firmware where using that memory leads | |
616 | * to crashes. | |
617 | * | |
618 | * Only EFI_CONVENTIONAL_MEMORY is guaranteed to be free. | |
619 | */ | |
620 | if (md->type != EFI_CONVENTIONAL_MEMORY) | |
621 | continue; | |
622 | ||
623 | if (efi_mirror_found && | |
624 | !(md->attribute & EFI_MEMORY_MORE_RELIABLE)) | |
625 | continue; | |
626 | ||
627 | region.start = md->phys_addr; | |
628 | region.size = md->num_pages << EFI_PAGE_SHIFT; | |
629 | process_mem_region(®ion, minimum, image_size); | |
630 | if (slot_area_index == MAX_SLOT_AREA) { | |
631 | debug_putstr("Aborted EFI scan (slot_areas full)!\n"); | |
632 | break; | |
633 | } | |
634 | } | |
635 | return true; | |
c05cd797 BH |
636 | } |
637 | #else | |
638 | static inline bool | |
639 | process_efi_entries(unsigned long minimum, unsigned long image_size) | |
640 | { | |
641 | return false; | |
642 | } | |
643 | #endif | |
644 | ||
f62995c9 BH |
645 | static void process_e820_entries(unsigned long minimum, |
646 | unsigned long image_size) | |
82fa9637 KC |
647 | { |
648 | int i; | |
87891b01 | 649 | struct mem_vector region; |
f62995c9 BH |
650 | struct boot_e820_entry *entry; |
651 | ||
652 | /* Verify potential e820 positions, appending to slots list. */ | |
653 | for (i = 0; i < boot_params->e820_entries; i++) { | |
654 | entry = &boot_params->e820_table[i]; | |
655 | /* Skip non-RAM entries. */ | |
656 | if (entry->type != E820_TYPE_RAM) | |
657 | continue; | |
87891b01 BH |
658 | region.start = entry->addr; |
659 | region.size = entry->size; | |
27aac205 | 660 | process_mem_region(®ion, minimum, image_size); |
f62995c9 BH |
661 | if (slot_area_index == MAX_SLOT_AREA) { |
662 | debug_putstr("Aborted e820 scan (slot_areas full)!\n"); | |
663 | break; | |
664 | } | |
665 | } | |
666 | } | |
82fa9637 | 667 | |
f62995c9 BH |
668 | static unsigned long find_random_phys_addr(unsigned long minimum, |
669 | unsigned long image_size) | |
670 | { | |
f2844249 DJ |
671 | /* Check if we had too many memmaps. */ |
672 | if (memmap_too_large) { | |
c05cd797 | 673 | debug_putstr("Aborted memory entries scan (more than 4 memmap= args)!\n"); |
f2844249 DJ |
674 | return 0; |
675 | } | |
676 | ||
82fa9637 KC |
677 | /* Make sure minimum is aligned. */ |
678 | minimum = ALIGN(minimum, CONFIG_PHYSICAL_ALIGN); | |
679 | ||
c05cd797 BH |
680 | if (process_efi_entries(minimum, image_size)) |
681 | return slots_fetch_random(); | |
682 | ||
f62995c9 | 683 | process_e820_entries(minimum, image_size); |
82fa9637 KC |
684 | return slots_fetch_random(); |
685 | } | |
686 | ||
071a7493 BH |
687 | static unsigned long find_random_virt_addr(unsigned long minimum, |
688 | unsigned long image_size) | |
689 | { | |
690 | unsigned long slots, random_addr; | |
691 | ||
692 | /* Make sure minimum is aligned. */ | |
693 | minimum = ALIGN(minimum, CONFIG_PHYSICAL_ALIGN); | |
694 | /* Align image_size for easy slot calculations. */ | |
695 | image_size = ALIGN(image_size, CONFIG_PHYSICAL_ALIGN); | |
696 | ||
697 | /* | |
698 | * There are how many CONFIG_PHYSICAL_ALIGN-sized slots | |
699 | * that can hold image_size within the range of minimum to | |
700 | * KERNEL_IMAGE_SIZE? | |
701 | */ | |
702 | slots = (KERNEL_IMAGE_SIZE - minimum - image_size) / | |
703 | CONFIG_PHYSICAL_ALIGN + 1; | |
704 | ||
d899a7d1 | 705 | random_addr = kaslr_get_random_long("Virtual") % slots; |
071a7493 BH |
706 | |
707 | return random_addr * CONFIG_PHYSICAL_ALIGN + minimum; | |
708 | } | |
709 | ||
549f90db BP |
710 | /* |
711 | * Since this function examines addresses much more numerically, | |
712 | * it takes the input and output pointers as 'unsigned long'. | |
713 | */ | |
8391c73c BH |
714 | void choose_random_location(unsigned long input, |
715 | unsigned long input_size, | |
716 | unsigned long *output, | |
717 | unsigned long output_size, | |
718 | unsigned long *virt_addr) | |
8ab3820f | 719 | { |
e066cc47 | 720 | unsigned long random_addr, min_addr; |
8ab3820f KC |
721 | |
722 | if (cmdline_find_option_bool("nokaslr")) { | |
0f8ede1b | 723 | warn("KASLR disabled: 'nokaslr' on cmdline."); |
8391c73c | 724 | return; |
8ab3820f KC |
725 | } |
726 | ||
6655e0aa | 727 | boot_params->hdr.loadflags |= KASLR_FLAG; |
78cac48c | 728 | |
11fdf97a KC |
729 | /* Prepare to add new identity pagetables on demand. */ |
730 | initialize_identity_maps(); | |
731 | ||
82fa9637 | 732 | /* Record the various known unsafe memory ranges. */ |
8391c73c | 733 | mem_avoid_init(input, input_size, *output); |
82fa9637 | 734 | |
e066cc47 YL |
735 | /* |
736 | * Low end of the randomization range should be the | |
737 | * smaller of 512M or the initial kernel image | |
738 | * location: | |
739 | */ | |
740 | min_addr = min(*output, 512UL << 20); | |
741 | ||
c05cd797 | 742 | /* Walk available memory entries to find a random address. */ |
e066cc47 | 743 | random_addr = find_random_phys_addr(min_addr, output_size); |
9016875d | 744 | if (!random_addr) { |
f2844249 | 745 | warn("Physical KASLR disabled: no suitable memory region!"); |
8391c73c BH |
746 | } else { |
747 | /* Update the new physical address location. */ | |
748 | if (*output != random_addr) { | |
749 | add_identity_map(random_addr, output_size); | |
750 | *output = random_addr; | |
751 | } | |
da63b6b2 BH |
752 | |
753 | /* | |
754 | * This loads the identity mapping page table. | |
755 | * This should only be done if a new physical address | |
756 | * is found for the kernel, otherwise we should keep | |
757 | * the old page table to make it be like the "nokaslr" | |
758 | * case. | |
759 | */ | |
760 | finalize_identity_maps(); | |
82fa9637 KC |
761 | } |
762 | ||
8391c73c BH |
763 | |
764 | /* Pick random virtual address starting from LOAD_PHYSICAL_ADDR. */ | |
765 | if (IS_ENABLED(CONFIG_X86_64)) | |
766 | random_addr = find_random_virt_addr(LOAD_PHYSICAL_ADDR, output_size); | |
767 | *virt_addr = random_addr; | |
8ab3820f | 768 | } |