]>
Commit | Line | Data |
---|---|---|
457c8996 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
b79cd8f1 | 2 | /* |
640e1b38 | 3 | * Low level x86 E820 memory map handling functions. |
b79cd8f1 | 4 | * |
640e1b38 IM |
5 | * The firmware and bootloader passes us the "E820 table", which is the primary |
6 | * physical memory layout description available about x86 systems. | |
b79cd8f1 | 7 | * |
640e1b38 IM |
8 | * The kernel takes the E820 memory layout and optionally modifies it with |
9 | * quirks and other tweaks, and feeds that into the generic Linux memory | |
10 | * allocation code routines via a platform independent interface (memblock, etc.). | |
b79cd8f1 | 11 | */ |
93a72052 | 12 | #include <linux/crash_dump.h> |
57c8a661 | 13 | #include <linux/memblock.h> |
bf62f398 | 14 | #include <linux/suspend.h> |
976513db | 15 | #include <linux/acpi.h> |
5dfcf14d | 16 | #include <linux/firmware-map.h> |
d1bbdd66 | 17 | #include <linux/sort.h> |
357b4da5 | 18 | #include <linux/memory_hotplug.h> |
b79cd8f1 | 19 | |
66441bd3 | 20 | #include <asm/e820/api.h> |
b79cd8f1 YL |
21 | #include <asm/setup.h> |
22 | ||
5dfcf14d | 23 | /* |
12df216c | 24 | * We organize the E820 table into three main data structures: |
544a0f47 | 25 | * |
12df216c CY |
26 | * - 'e820_table_firmware': the original firmware version passed to us by the |
27 | * bootloader - not modified by the kernel. It is composed of two parts: | |
28 | * the first 128 E820 memory entries in boot_params.e820_table and the remaining | |
29 | * (if any) entries of the SETUP_E820_EXT nodes. We use this to: | |
544a0f47 IM |
30 | * |
31 | * - inform the user about the firmware's notion of memory layout | |
32 | * via /sys/firmware/memmap | |
33 | * | |
34 | * - the hibernation code uses it to generate a kernel-independent MD5 | |
35 | * fingerprint of the physical memory layout of a system. | |
36 | * | |
12df216c CY |
37 | * - 'e820_table_kexec': a slightly modified (by the kernel) firmware version |
38 | * passed to us by the bootloader - the major difference between | |
39 | * e820_table_firmware[] and this one is that, the latter marks the setup_data | |
40 | * list created by the EFI boot stub as reserved, so that kexec can reuse the | |
41 | * setup_data information in the second kernel. Besides, e820_table_kexec[] | |
42 | * might also be modified by the kexec itself to fake a mptable. | |
43 | * We use this to: | |
44 | * | |
640e1b38 | 45 | * - kexec, which is a bootloader in disguise, uses the original E820 |
544a0f47 | 46 | * layout to pass to the kexec-ed kernel. This way the original kernel |
640e1b38 | 47 | * can have a restricted E820 map while the kexec()-ed kexec-kernel |
544a0f47 IM |
48 | * can have access to full memory - etc. |
49 | * | |
640e1b38 | 50 | * - 'e820_table': this is the main E820 table that is massaged by the |
544a0f47 IM |
51 | * low level x86 platform code, or modified by boot parameters, before |
52 | * passed on to higher level MM layers. | |
53 | * | |
640e1b38 | 54 | * Once the E820 map has been converted to the standard Linux memory layout |
544a0f47 IM |
55 | * information its role stops - modifying it has no effect and does not get |
56 | * re-propagated. So itsmain role is a temporary bootstrap storage of firmware | |
57 | * specific memory layout data during early bootup. | |
5dfcf14d | 58 | */ |
544a0f47 | 59 | static struct e820_table e820_table_init __initdata; |
a09bae0f | 60 | static struct e820_table e820_table_kexec_init __initdata; |
12df216c | 61 | static struct e820_table e820_table_firmware_init __initdata; |
544a0f47 IM |
62 | |
63 | struct e820_table *e820_table __refdata = &e820_table_init; | |
a09bae0f | 64 | struct e820_table *e820_table_kexec __refdata = &e820_table_kexec_init; |
12df216c | 65 | struct e820_table *e820_table_firmware __refdata = &e820_table_firmware_init; |
b79cd8f1 YL |
66 | |
67 | /* For PCI or other memory-mapped resources */ | |
68 | unsigned long pci_mem_start = 0xaeedbabe; | |
69 | #ifdef CONFIG_PCI | |
70 | EXPORT_SYMBOL(pci_mem_start); | |
71 | #endif | |
72 | ||
73 | /* | |
74 | * This function checks if any part of the range <start,end> is mapped | |
75 | * with type. | |
76 | */ | |
0c55671f KA |
77 | static bool _e820__mapped_any(struct e820_table *table, |
78 | u64 start, u64 end, enum e820_type type) | |
b79cd8f1 YL |
79 | { |
80 | int i; | |
81 | ||
0c55671f KA |
82 | for (i = 0; i < table->nr_entries; i++) { |
83 | struct e820_entry *entry = &table->entries[i]; | |
b79cd8f1 | 84 | |
e5540f87 | 85 | if (type && entry->type != type) |
b79cd8f1 | 86 | continue; |
e5540f87 | 87 | if (entry->addr >= end || entry->addr + entry->size <= start) |
b79cd8f1 YL |
88 | continue; |
89 | return 1; | |
90 | } | |
91 | return 0; | |
92 | } | |
0c55671f KA |
93 | |
94 | bool e820__mapped_raw_any(u64 start, u64 end, enum e820_type type) | |
95 | { | |
96 | return _e820__mapped_any(e820_table_firmware, start, end, type); | |
97 | } | |
98 | EXPORT_SYMBOL_GPL(e820__mapped_raw_any); | |
99 | ||
100 | bool e820__mapped_any(u64 start, u64 end, enum e820_type type) | |
101 | { | |
102 | return _e820__mapped_any(e820_table, start, end, type); | |
103 | } | |
3bce64f0 | 104 | EXPORT_SYMBOL_GPL(e820__mapped_any); |
b79cd8f1 YL |
105 | |
106 | /* | |
640e1b38 | 107 | * This function checks if the entire <start,end> range is mapped with 'type'. |
b79cd8f1 | 108 | * |
640e1b38 IM |
109 | * Note: this function only works correctly once the E820 table is sorted and |
110 | * not-overlapping (at least for the range specified), which is the case normally. | |
b79cd8f1 | 111 | */ |
d68baa3f TL |
112 | static struct e820_entry *__e820__mapped_all(u64 start, u64 end, |
113 | enum e820_type type) | |
b79cd8f1 YL |
114 | { |
115 | int i; | |
116 | ||
bf495573 | 117 | for (i = 0; i < e820_table->nr_entries; i++) { |
e5540f87 | 118 | struct e820_entry *entry = &e820_table->entries[i]; |
b79cd8f1 | 119 | |
e5540f87 | 120 | if (type && entry->type != type) |
b79cd8f1 | 121 | continue; |
640e1b38 IM |
122 | |
123 | /* Is the region (part) in overlap with the current region? */ | |
e5540f87 | 124 | if (entry->addr >= end || entry->addr + entry->size <= start) |
b79cd8f1 YL |
125 | continue; |
126 | ||
640e1b38 IM |
127 | /* |
128 | * If the region is at the beginning of <start,end> we move | |
129 | * 'start' to the end of the region since it's ok until there | |
b79cd8f1 | 130 | */ |
e5540f87 IM |
131 | if (entry->addr <= start) |
132 | start = entry->addr + entry->size; | |
640e1b38 | 133 | |
b79cd8f1 | 134 | /* |
640e1b38 IM |
135 | * If 'start' is now at or beyond 'end', we're done, full |
136 | * coverage of the desired range exists: | |
b79cd8f1 YL |
137 | */ |
138 | if (start >= end) | |
d68baa3f | 139 | return entry; |
b79cd8f1 | 140 | } |
d68baa3f TL |
141 | |
142 | return NULL; | |
143 | } | |
144 | ||
145 | /* | |
146 | * This function checks if the entire range <start,end> is mapped with type. | |
147 | */ | |
148 | bool __init e820__mapped_all(u64 start, u64 end, enum e820_type type) | |
149 | { | |
150 | return __e820__mapped_all(start, end, type); | |
151 | } | |
152 | ||
153 | /* | |
154 | * This function returns the type associated with the range <start,end>. | |
155 | */ | |
156 | int e820__get_entry_type(u64 start, u64 end) | |
157 | { | |
158 | struct e820_entry *entry = __e820__mapped_all(start, end, 0); | |
159 | ||
160 | return entry ? entry->type : -EINVAL; | |
b79cd8f1 YL |
161 | } |
162 | ||
163 | /* | |
640e1b38 | 164 | * Add a memory region to the kernel E820 map. |
b79cd8f1 | 165 | */ |
6afc03b8 | 166 | static void __init __e820__range_add(struct e820_table *table, u64 start, u64 size, enum e820_type type) |
b79cd8f1 | 167 | { |
bf495573 | 168 | int x = table->nr_entries; |
b79cd8f1 | 169 | |
bf495573 | 170 | if (x >= ARRAY_SIZE(table->entries)) { |
1de392f5 JP |
171 | pr_err("too many entries; ignoring [mem %#010llx-%#010llx]\n", |
172 | start, start + size - 1); | |
b79cd8f1 YL |
173 | return; |
174 | } | |
175 | ||
bf495573 IM |
176 | table->entries[x].addr = start; |
177 | table->entries[x].size = size; | |
178 | table->entries[x].type = type; | |
179 | table->nr_entries++; | |
773e673d YL |
180 | } |
181 | ||
6afc03b8 | 182 | void __init e820__range_add(u64 start, u64 size, enum e820_type type) |
773e673d | 183 | { |
ab6bc04c | 184 | __e820__range_add(e820_table, start, size, type); |
b79cd8f1 YL |
185 | } |
186 | ||
6afc03b8 | 187 | static void __init e820_print_type(enum e820_type type) |
c61cf4cf YL |
188 | { |
189 | switch (type) { | |
09821ff1 IM |
190 | case E820_TYPE_RAM: /* Fall through: */ |
191 | case E820_TYPE_RESERVED_KERN: pr_cont("usable"); break; | |
192 | case E820_TYPE_RESERVED: pr_cont("reserved"); break; | |
193 | case E820_TYPE_ACPI: pr_cont("ACPI data"); break; | |
194 | case E820_TYPE_NVS: pr_cont("ACPI NVS"); break; | |
195 | case E820_TYPE_UNUSABLE: pr_cont("unusable"); break; | |
196 | case E820_TYPE_PMEM: /* Fall through: */ | |
197 | case E820_TYPE_PRAM: pr_cont("persistent (type %u)", type); break; | |
01259ef1 | 198 | default: pr_cont("type %u", type); break; |
c61cf4cf YL |
199 | } |
200 | } | |
201 | ||
be0c3f0f | 202 | void __init e820__print_table(char *who) |
b79cd8f1 YL |
203 | { |
204 | int i; | |
205 | ||
bf495573 | 206 | for (i = 0; i < e820_table->nr_entries; i++) { |
1de392f5 JP |
207 | pr_info("%s: [mem %#018Lx-%#018Lx] ", |
208 | who, | |
209 | e820_table->entries[i].addr, | |
210 | e820_table->entries[i].addr + e820_table->entries[i].size - 1); | |
640e1b38 | 211 | |
bf495573 | 212 | e820_print_type(e820_table->entries[i].type); |
01259ef1 | 213 | pr_cont("\n"); |
b79cd8f1 YL |
214 | } |
215 | } | |
216 | ||
217 | /* | |
9a02fd0f | 218 | * Sanitize an E820 map. |
b79cd8f1 | 219 | * |
9a02fd0f | 220 | * Some E820 layouts include overlapping entries. The following |
640e1b38 | 221 | * replaces the original E820 map with a new one, removing overlaps, |
5b7eb2e9 PJ |
222 | * and resolving conflicting memory types in favor of highest |
223 | * numbered type. | |
b79cd8f1 | 224 | * |
9a02fd0f IM |
225 | * The input parameter 'entries' points to an array of 'struct |
226 | * e820_entry' which on entry has elements in the range [0, *nr_entries) | |
227 | * valid, and which has space for up to max_nr_entries entries. | |
640e1b38 | 228 | * On return, the resulting sanitized E820 map entries will be in |
9a02fd0f | 229 | * overwritten in the same location, starting at 'entries'. |
5b7eb2e9 | 230 | * |
9a02fd0f IM |
231 | * The integer pointed to by nr_entries must be valid on entry (the |
232 | * current number of valid entries located at 'entries'). If the | |
233 | * sanitizing succeeds the *nr_entries will be updated with the new | |
234 | * number of valid entries (something no more than max_nr_entries). | |
5b7eb2e9 | 235 | * |
f52355a9 | 236 | * The return value from e820__update_table() is zero if it |
5b7eb2e9 PJ |
237 | * successfully 'sanitized' the map entries passed in, and is -1 |
238 | * if it did nothing, which can happen if either of (1) it was | |
239 | * only passed one map entry, or (2) any of the input map entries | |
240 | * were invalid (start + size < start, meaning that the size was | |
241 | * so big the described memory range wrapped around through zero.) | |
242 | * | |
243 | * Visually we're performing the following | |
244 | * (1,2,3,4 = memory types)... | |
245 | * | |
246 | * Sample memory map (w/overlaps): | |
247 | * ____22__________________ | |
248 | * ______________________4_ | |
249 | * ____1111________________ | |
250 | * _44_____________________ | |
251 | * 11111111________________ | |
252 | * ____________________33__ | |
253 | * ___________44___________ | |
254 | * __________33333_________ | |
255 | * ______________22________ | |
256 | * ___________________2222_ | |
257 | * _________111111111______ | |
258 | * _____________________11_ | |
259 | * _________________4______ | |
260 | * | |
261 | * Sanitized equivalent (no overlap): | |
262 | * 1_______________________ | |
263 | * _44_____________________ | |
264 | * ___1____________________ | |
265 | * ____22__________________ | |
266 | * ______11________________ | |
267 | * _________1______________ | |
268 | * __________3_____________ | |
269 | * ___________44___________ | |
270 | * _____________33_________ | |
271 | * _______________2________ | |
272 | * ________________1_______ | |
273 | * _________________4______ | |
274 | * ___________________2____ | |
275 | * ____________________33__ | |
276 | * ______________________4_ | |
b79cd8f1 | 277 | */ |
d1bbdd66 | 278 | struct change_member { |
9a02fd0f IM |
279 | /* Pointer to the original entry: */ |
280 | struct e820_entry *entry; | |
640e1b38 IM |
281 | /* Address for this change point: */ |
282 | unsigned long long addr; | |
d1bbdd66 MD |
283 | }; |
284 | ||
441ac2f3 IM |
285 | static struct change_member change_point_list[2*E820_MAX_ENTRIES] __initdata; |
286 | static struct change_member *change_point[2*E820_MAX_ENTRIES] __initdata; | |
287 | static struct e820_entry *overlap_list[E820_MAX_ENTRIES] __initdata; | |
288 | static struct e820_entry new_entries[E820_MAX_ENTRIES] __initdata; | |
289 | ||
d1bbdd66 MD |
290 | static int __init cpcompare(const void *a, const void *b) |
291 | { | |
292 | struct change_member * const *app = a, * const *bpp = b; | |
293 | const struct change_member *ap = *app, *bp = *bpp; | |
294 | ||
295 | /* | |
296 | * Inputs are pointers to two elements of change_point[]. If their | |
640e1b38 | 297 | * addresses are not equal, their difference dominates. If the addresses |
d1bbdd66 MD |
298 | * are equal, then consider one that represents the end of its region |
299 | * to be greater than one that does not. | |
300 | */ | |
301 | if (ap->addr != bp->addr) | |
302 | return ap->addr > bp->addr ? 1 : -1; | |
303 | ||
9a02fd0f | 304 | return (ap->addr != ap->entry->addr) - (bp->addr != bp->entry->addr); |
d1bbdd66 | 305 | } |
5b7eb2e9 | 306 | |
441ac2f3 | 307 | int __init e820__update_table(struct e820_table *table) |
b79cd8f1 | 308 | { |
441ac2f3 IM |
309 | struct e820_entry *entries = table->entries; |
310 | u32 max_nr_entries = ARRAY_SIZE(table->entries); | |
6afc03b8 | 311 | enum e820_type current_type, last_type; |
b79cd8f1 | 312 | unsigned long long last_addr; |
441ac2f3 IM |
313 | u32 new_nr_entries, overlap_entries; |
314 | u32 i, chg_idx, chg_nr; | |
b79cd8f1 | 315 | |
640e1b38 | 316 | /* If there's only one memory region, don't bother: */ |
441ac2f3 | 317 | if (table->nr_entries < 2) |
b79cd8f1 YL |
318 | return -1; |
319 | ||
441ac2f3 | 320 | BUG_ON(table->nr_entries > max_nr_entries); |
b79cd8f1 | 321 | |
9a02fd0f | 322 | /* Bail out if we find any unreasonable addresses in the map: */ |
441ac2f3 | 323 | for (i = 0; i < table->nr_entries; i++) { |
9a02fd0f | 324 | if (entries[i].addr + entries[i].size < entries[i].addr) |
b79cd8f1 | 325 | return -1; |
640e1b38 | 326 | } |
b79cd8f1 | 327 | |
640e1b38 | 328 | /* Create pointers for initial change-point information (for sorting): */ |
441ac2f3 | 329 | for (i = 0; i < 2 * table->nr_entries; i++) |
b79cd8f1 YL |
330 | change_point[i] = &change_point_list[i]; |
331 | ||
640e1b38 IM |
332 | /* |
333 | * Record all known change-points (starting and ending addresses), | |
334 | * omitting empty memory regions: | |
335 | */ | |
441ac2f3 IM |
336 | chg_idx = 0; |
337 | for (i = 0; i < table->nr_entries; i++) { | |
9a02fd0f | 338 | if (entries[i].size != 0) { |
441ac2f3 IM |
339 | change_point[chg_idx]->addr = entries[i].addr; |
340 | change_point[chg_idx++]->entry = &entries[i]; | |
341 | change_point[chg_idx]->addr = entries[i].addr + entries[i].size; | |
342 | change_point[chg_idx++]->entry = &entries[i]; | |
b79cd8f1 YL |
343 | } |
344 | } | |
441ac2f3 | 345 | chg_nr = chg_idx; |
b79cd8f1 | 346 | |
640e1b38 | 347 | /* Sort change-point list by memory addresses (low -> high): */ |
d88961b5 | 348 | sort(change_point, chg_nr, sizeof(*change_point), cpcompare, NULL); |
b79cd8f1 | 349 | |
9a02fd0f | 350 | /* Create a new memory map, removing overlaps: */ |
640e1b38 | 351 | overlap_entries = 0; /* Number of entries in the overlap table */ |
9a02fd0f | 352 | new_nr_entries = 0; /* Index for creating new map entries */ |
640e1b38 IM |
353 | last_type = 0; /* Start with undefined memory type */ |
354 | last_addr = 0; /* Start with 0 as last starting address */ | |
b79cd8f1 | 355 | |
9a02fd0f | 356 | /* Loop through change-points, determining effect on the new map: */ |
441ac2f3 | 357 | for (chg_idx = 0; chg_idx < chg_nr; chg_idx++) { |
9a02fd0f | 358 | /* Keep track of all overlapping entries */ |
441ac2f3 | 359 | if (change_point[chg_idx]->addr == change_point[chg_idx]->entry->addr) { |
640e1b38 | 360 | /* Add map entry to overlap list (> 1 entry implies an overlap) */ |
441ac2f3 | 361 | overlap_list[overlap_entries++] = change_point[chg_idx]->entry; |
b79cd8f1 | 362 | } else { |
640e1b38 | 363 | /* Remove entry from list (order independent, so swap with last): */ |
b79cd8f1 | 364 | for (i = 0; i < overlap_entries; i++) { |
441ac2f3 | 365 | if (overlap_list[i] == change_point[chg_idx]->entry) |
640e1b38 | 366 | overlap_list[i] = overlap_list[overlap_entries-1]; |
b79cd8f1 YL |
367 | } |
368 | overlap_entries--; | |
369 | } | |
370 | /* | |
640e1b38 | 371 | * If there are overlapping entries, decide which |
b79cd8f1 YL |
372 | * "type" to use (larger value takes precedence -- |
373 | * 1=usable, 2,3,4,4+=unusable) | |
374 | */ | |
375 | current_type = 0; | |
640e1b38 | 376 | for (i = 0; i < overlap_entries; i++) { |
b79cd8f1 YL |
377 | if (overlap_list[i]->type > current_type) |
378 | current_type = overlap_list[i]->type; | |
640e1b38 IM |
379 | } |
380 | ||
9a02fd0f | 381 | /* Continue building up new map based on this information: */ |
09821ff1 | 382 | if (current_type != last_type || current_type == E820_TYPE_PRAM) { |
b79cd8f1 | 383 | if (last_type != 0) { |
441ac2f3 | 384 | new_entries[new_nr_entries].size = change_point[chg_idx]->addr - last_addr; |
640e1b38 | 385 | /* Move forward only if the new size was non-zero: */ |
9a02fd0f IM |
386 | if (new_entries[new_nr_entries].size != 0) |
387 | /* No more space left for new entries? */ | |
388 | if (++new_nr_entries >= max_nr_entries) | |
b79cd8f1 YL |
389 | break; |
390 | } | |
391 | if (current_type != 0) { | |
441ac2f3 | 392 | new_entries[new_nr_entries].addr = change_point[chg_idx]->addr; |
9a02fd0f | 393 | new_entries[new_nr_entries].type = current_type; |
441ac2f3 | 394 | last_addr = change_point[chg_idx]->addr; |
b79cd8f1 YL |
395 | } |
396 | last_type = current_type; | |
397 | } | |
398 | } | |
640e1b38 | 399 | |
9a02fd0f | 400 | /* Copy the new entries into the original location: */ |
441ac2f3 IM |
401 | memcpy(entries, new_entries, new_nr_entries*sizeof(*entries)); |
402 | table->nr_entries = new_nr_entries; | |
b79cd8f1 YL |
403 | |
404 | return 0; | |
405 | } | |
406 | ||
7410aa1c | 407 | static int __init __append_e820_table(struct boot_e820_entry *entries, u32 nr_entries) |
8c5beb50 | 408 | { |
7410aa1c | 409 | struct boot_e820_entry *entry = entries; |
9a02fd0f IM |
410 | |
411 | while (nr_entries) { | |
412 | u64 start = entry->addr; | |
413 | u64 size = entry->size; | |
3ec97965 | 414 | u64 end = start + size - 1; |
9a02fd0f | 415 | u32 type = entry->type; |
8c5beb50 | 416 | |
640e1b38 | 417 | /* Ignore the entry on 64-bit overflow: */ |
3ec97965 | 418 | if (start > end && likely(size)) |
8c5beb50 HY |
419 | return -1; |
420 | ||
ab6bc04c | 421 | e820__range_add(start, size, type); |
8c5beb50 | 422 | |
9a02fd0f IM |
423 | entry++; |
424 | nr_entries--; | |
8c5beb50 HY |
425 | } |
426 | return 0; | |
427 | } | |
428 | ||
b79cd8f1 | 429 | /* |
640e1b38 | 430 | * Copy the BIOS E820 map into a safe place. |
b79cd8f1 YL |
431 | * |
432 | * Sanity-check it while we're at it.. | |
433 | * | |
434 | * If we're lucky and live on a modern system, the setup code | |
435 | * will have given us a memory map that we can use to properly | |
436 | * set up memory. If we aren't, we'll fake a memory map. | |
437 | */ | |
7410aa1c | 438 | static int __init append_e820_table(struct boot_e820_entry *entries, u32 nr_entries) |
b79cd8f1 YL |
439 | { |
440 | /* Only one memory region (or negative)? Ignore it */ | |
9a02fd0f | 441 | if (nr_entries < 2) |
b79cd8f1 YL |
442 | return -1; |
443 | ||
9a02fd0f | 444 | return __append_e820_table(entries, nr_entries); |
b79cd8f1 YL |
445 | } |
446 | ||
640e1b38 | 447 | static u64 __init |
6afc03b8 | 448 | __e820__range_update(struct e820_table *table, u64 start, u64 size, enum e820_type old_type, enum e820_type new_type) |
b79cd8f1 | 449 | { |
78a8b35b | 450 | u64 end; |
773e673d | 451 | unsigned int i; |
b79cd8f1 YL |
452 | u64 real_updated_size = 0; |
453 | ||
454 | BUG_ON(old_type == new_type); | |
455 | ||
232b957a YL |
456 | if (size > (ULLONG_MAX - start)) |
457 | size = ULLONG_MAX - start; | |
458 | ||
78a8b35b | 459 | end = start + size; |
e22af0be | 460 | printk(KERN_DEBUG "e820: update [mem %#010Lx-%#010Lx] ", start, end - 1); |
c61cf4cf | 461 | e820_print_type(old_type); |
01259ef1 | 462 | pr_cont(" ==> "); |
c61cf4cf | 463 | e820_print_type(new_type); |
01259ef1 | 464 | pr_cont("\n"); |
c61cf4cf | 465 | |
bf495573 | 466 | for (i = 0; i < table->nr_entries; i++) { |
e5540f87 | 467 | struct e820_entry *entry = &table->entries[i]; |
b79cd8f1 | 468 | u64 final_start, final_end; |
e5540f87 | 469 | u64 entry_end; |
78a8b35b | 470 | |
e5540f87 | 471 | if (entry->type != old_type) |
b79cd8f1 | 472 | continue; |
78a8b35b | 473 | |
e5540f87 | 474 | entry_end = entry->addr + entry->size; |
640e1b38 IM |
475 | |
476 | /* Completely covered by new range? */ | |
e5540f87 IM |
477 | if (entry->addr >= start && entry_end <= end) { |
478 | entry->type = new_type; | |
479 | real_updated_size += entry->size; | |
b79cd8f1 YL |
480 | continue; |
481 | } | |
78a8b35b | 482 | |
640e1b38 | 483 | /* New range is completely covered? */ |
e5540f87 | 484 | if (entry->addr < start && entry_end > end) { |
ab6bc04c IM |
485 | __e820__range_add(table, start, size, new_type); |
486 | __e820__range_add(table, end, entry_end - end, entry->type); | |
e5540f87 | 487 | entry->size = start - entry->addr; |
78a8b35b YL |
488 | real_updated_size += size; |
489 | continue; | |
490 | } | |
491 | ||
640e1b38 | 492 | /* Partially covered: */ |
e5540f87 IM |
493 | final_start = max(start, entry->addr); |
494 | final_end = min(end, entry_end); | |
b79cd8f1 YL |
495 | if (final_start >= final_end) |
496 | continue; | |
5c0e6f03 | 497 | |
ab6bc04c | 498 | __e820__range_add(table, final_start, final_end - final_start, new_type); |
5c0e6f03 | 499 | |
b79cd8f1 | 500 | real_updated_size += final_end - final_start; |
976dd4dc | 501 | |
773e673d | 502 | /* |
640e1b38 IM |
503 | * Left range could be head or tail, so need to update |
504 | * its size first: | |
773e673d | 505 | */ |
e5540f87 IM |
506 | entry->size -= final_end - final_start; |
507 | if (entry->addr < final_start) | |
976dd4dc | 508 | continue; |
640e1b38 | 509 | |
e5540f87 | 510 | entry->addr = final_end; |
b79cd8f1 YL |
511 | } |
512 | return real_updated_size; | |
513 | } | |
514 | ||
6afc03b8 | 515 | u64 __init e820__range_update(u64 start, u64 size, enum e820_type old_type, enum e820_type new_type) |
fc9036ea | 516 | { |
ab6bc04c | 517 | return __e820__range_update(e820_table, start, size, old_type, new_type); |
fc9036ea YL |
518 | } |
519 | ||
a09bae0f | 520 | static u64 __init e820__range_update_kexec(u64 start, u64 size, enum e820_type old_type, enum e820_type new_type) |
fc9036ea | 521 | { |
a09bae0f | 522 | return __e820__range_update(e820_table_kexec, start, size, old_type, new_type); |
fc9036ea YL |
523 | } |
524 | ||
640e1b38 | 525 | /* Remove a range of memory from the E820 table: */ |
81b3e090 | 526 | u64 __init e820__range_remove(u64 start, u64 size, enum e820_type old_type, bool check_type) |
7a1fd986 YL |
527 | { |
528 | int i; | |
1b5576e6 | 529 | u64 end; |
7a1fd986 YL |
530 | u64 real_removed_size = 0; |
531 | ||
232b957a YL |
532 | if (size > (ULLONG_MAX - start)) |
533 | size = ULLONG_MAX - start; | |
534 | ||
1b5576e6 | 535 | end = start + size; |
e22af0be | 536 | printk(KERN_DEBUG "e820: remove [mem %#010Lx-%#010Lx] ", start, end - 1); |
81b3e090 | 537 | if (check_type) |
9f3a5f52 | 538 | e820_print_type(old_type); |
01259ef1 | 539 | pr_cont("\n"); |
1b5576e6 | 540 | |
bf495573 | 541 | for (i = 0; i < e820_table->nr_entries; i++) { |
e5540f87 | 542 | struct e820_entry *entry = &e820_table->entries[i]; |
7a1fd986 | 543 | u64 final_start, final_end; |
e5540f87 | 544 | u64 entry_end; |
7a1fd986 | 545 | |
81b3e090 | 546 | if (check_type && entry->type != old_type) |
7a1fd986 | 547 | continue; |
9f3a5f52 | 548 | |
e5540f87 | 549 | entry_end = entry->addr + entry->size; |
640e1b38 IM |
550 | |
551 | /* Completely covered? */ | |
e5540f87 IM |
552 | if (entry->addr >= start && entry_end <= end) { |
553 | real_removed_size += entry->size; | |
d88961b5 | 554 | memset(entry, 0, sizeof(*entry)); |
7a1fd986 YL |
555 | continue; |
556 | } | |
9f3a5f52 | 557 | |
640e1b38 | 558 | /* Is the new range completely covered? */ |
e5540f87 | 559 | if (entry->addr < start && entry_end > end) { |
ab6bc04c | 560 | e820__range_add(end, entry_end - end, entry->type); |
e5540f87 | 561 | entry->size = start - entry->addr; |
9f3a5f52 YL |
562 | real_removed_size += size; |
563 | continue; | |
564 | } | |
565 | ||
640e1b38 | 566 | /* Partially covered: */ |
e5540f87 IM |
567 | final_start = max(start, entry->addr); |
568 | final_end = min(end, entry_end); | |
7a1fd986 YL |
569 | if (final_start >= final_end) |
570 | continue; | |
640e1b38 | 571 | |
7a1fd986 YL |
572 | real_removed_size += final_end - final_start; |
573 | ||
9f3a5f52 | 574 | /* |
640e1b38 IM |
575 | * Left range could be head or tail, so need to update |
576 | * the size first: | |
9f3a5f52 | 577 | */ |
e5540f87 IM |
578 | entry->size -= final_end - final_start; |
579 | if (entry->addr < final_start) | |
7a1fd986 | 580 | continue; |
640e1b38 | 581 | |
e5540f87 | 582 | entry->addr = final_end; |
7a1fd986 YL |
583 | } |
584 | return real_removed_size; | |
585 | } | |
586 | ||
6464d294 | 587 | void __init e820__update_table_print(void) |
b79cd8f1 | 588 | { |
f9748fa0 | 589 | if (e820__update_table(e820_table)) |
b79cd8f1 | 590 | return; |
640e1b38 | 591 | |
1de392f5 | 592 | pr_info("modified physical RAM map:\n"); |
be0c3f0f | 593 | e820__print_table("modified"); |
b79cd8f1 | 594 | } |
640e1b38 | 595 | |
a09bae0f | 596 | static void __init e820__update_table_kexec(void) |
fc9036ea | 597 | { |
a09bae0f | 598 | e820__update_table(e820_table_kexec); |
fc9036ea | 599 | } |
640e1b38 | 600 | |
fd6493e1 | 601 | #define MAX_GAP_END 0x100000000ull |
640e1b38 | 602 | |
b79cd8f1 | 603 | /* |
640e1b38 | 604 | * Search for a gap in the E820 memory space from 0 to MAX_GAP_END (4GB). |
b79cd8f1 | 605 | */ |
640e1b38 | 606 | static int __init e820_search_gap(unsigned long *gapstart, unsigned long *gapsize) |
b79cd8f1 | 607 | { |
b4ed1d15 | 608 | unsigned long long last = MAX_GAP_END; |
bf495573 | 609 | int i = e820_table->nr_entries; |
b79cd8f1 YL |
610 | int found = 0; |
611 | ||
b79cd8f1 | 612 | while (--i >= 0) { |
bf495573 IM |
613 | unsigned long long start = e820_table->entries[i].addr; |
614 | unsigned long long end = start + e820_table->entries[i].size; | |
b79cd8f1 YL |
615 | |
616 | /* | |
617 | * Since "last" is at most 4GB, we know we'll | |
640e1b38 | 618 | * fit in 32 bits if this condition is true: |
b79cd8f1 YL |
619 | */ |
620 | if (last > end) { | |
621 | unsigned long gap = last - end; | |
622 | ||
3381959d AK |
623 | if (gap >= *gapsize) { |
624 | *gapsize = gap; | |
625 | *gapstart = end; | |
b79cd8f1 YL |
626 | found = 1; |
627 | } | |
628 | } | |
629 | if (start < last) | |
630 | last = start; | |
631 | } | |
3381959d AK |
632 | return found; |
633 | } | |
634 | ||
635 | /* | |
640e1b38 IM |
636 | * Search for the biggest gap in the low 32 bits of the E820 |
637 | * memory space. We pass this space to the PCI subsystem, so | |
638 | * that it can assign MMIO resources for hotplug or | |
639 | * unconfigured devices in. | |
640 | * | |
3381959d AK |
641 | * Hopefully the BIOS let enough space left. |
642 | */ | |
2df908ba | 643 | __init void e820__setup_pci_gap(void) |
3381959d | 644 | { |
5d423ccd | 645 | unsigned long gapstart, gapsize; |
3381959d AK |
646 | int found; |
647 | ||
3381959d | 648 | gapsize = 0x400000; |
b4ed1d15 | 649 | found = e820_search_gap(&gapstart, &gapsize); |
b79cd8f1 | 650 | |
b79cd8f1 | 651 | if (!found) { |
c19a5f35 | 652 | #ifdef CONFIG_X86_64 |
c987d12f | 653 | gapstart = (max_pfn << PAGE_SHIFT) + 1024*1024; |
1de392f5 JP |
654 | pr_err("Cannot find an available gap in the 32-bit address range\n"); |
655 | pr_err("PCI devices with unassigned 32-bit BARs may not work!\n"); | |
c19a5f35 AB |
656 | #else |
657 | gapstart = 0x10000000; | |
b79cd8f1 | 658 | #endif |
c19a5f35 | 659 | } |
b79cd8f1 YL |
660 | |
661 | /* | |
1506c8dc | 662 | * e820__reserve_resources_late() protects stolen RAM already: |
b79cd8f1 | 663 | */ |
5d423ccd | 664 | pci_mem_start = gapstart; |
b79cd8f1 | 665 | |
1de392f5 JP |
666 | pr_info("[mem %#010lx-%#010lx] available for PCI devices\n", |
667 | gapstart, gapstart + gapsize - 1); | |
b79cd8f1 YL |
668 | } |
669 | ||
47533968 DV |
670 | /* |
671 | * Called late during init, in free_initmem(). | |
672 | * | |
a09bae0f | 673 | * Initial e820_table and e820_table_kexec are largish __initdata arrays. |
640e1b38 IM |
674 | * |
675 | * Copy them to a (usually much smaller) dynamically allocated area that is | |
676 | * sized precisely after the number of e820 entries. | |
677 | * | |
678 | * This is done after we've performed all the fixes and tweaks to the tables. | |
679 | * All functions which modify them are __init functions, which won't exist | |
680 | * after free_initmem(). | |
47533968 | 681 | */ |
0c6fc11a | 682 | __init void e820__reallocate_tables(void) |
47533968 | 683 | { |
61a50101 | 684 | struct e820_table *n; |
47533968 DV |
685 | int size; |
686 | ||
640e1b38 | 687 | size = offsetof(struct e820_table, entries) + sizeof(struct e820_entry)*e820_table->nr_entries; |
345dca4c | 688 | n = kmemdup(e820_table, size, GFP_KERNEL); |
47533968 | 689 | BUG_ON(!n); |
61a50101 | 690 | e820_table = n; |
47533968 | 691 | |
a09bae0f | 692 | size = offsetof(struct e820_table, entries) + sizeof(struct e820_entry)*e820_table_kexec->nr_entries; |
345dca4c | 693 | n = kmemdup(e820_table_kexec, size, GFP_KERNEL); |
47533968 | 694 | BUG_ON(!n); |
a09bae0f | 695 | e820_table_kexec = n; |
12df216c CY |
696 | |
697 | size = offsetof(struct e820_table, entries) + sizeof(struct e820_entry)*e820_table_firmware->nr_entries; | |
345dca4c | 698 | n = kmemdup(e820_table_firmware, size, GFP_KERNEL); |
12df216c | 699 | BUG_ON(!n); |
12df216c | 700 | e820_table_firmware = n; |
47533968 DV |
701 | } |
702 | ||
640e1b38 IM |
703 | /* |
704 | * Because of the small fixed size of struct boot_params, only the first | |
705 | * 128 E820 memory entries are passed to the kernel via boot_params.e820_table, | |
706 | * the remaining (if any) entries are passed via the SETUP_E820_EXT node of | |
707 | * struct setup_data, which is parsed here. | |
8c5beb50 | 708 | */ |
914053c0 | 709 | void __init e820__memory_setup_extended(u64 phys_addr, u32 data_len) |
8c5beb50 | 710 | { |
8c5beb50 | 711 | int entries; |
7410aa1c | 712 | struct boot_e820_entry *extmap; |
30e46b57 | 713 | struct setup_data *sdata; |
8c5beb50 | 714 | |
30e46b57 | 715 | sdata = early_memremap(phys_addr, data_len); |
d88961b5 | 716 | entries = sdata->len / sizeof(*extmap); |
7410aa1c | 717 | extmap = (struct boot_e820_entry *)(sdata->data); |
640e1b38 | 718 | |
61a50101 | 719 | __append_e820_table(extmap, entries); |
f9748fa0 | 720 | e820__update_table(e820_table); |
640e1b38 | 721 | |
a09bae0f | 722 | memcpy(e820_table_kexec, e820_table, sizeof(*e820_table_kexec)); |
12df216c | 723 | memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware)); |
b7a67e02 | 724 | |
8d4a40bc | 725 | early_memunmap(sdata, data_len); |
1de392f5 | 726 | pr_info("extended physical RAM map:\n"); |
be0c3f0f | 727 | e820__print_table("extended"); |
8c5beb50 HY |
728 | } |
729 | ||
090d7171 | 730 | /* |
bf62f398 | 731 | * Find the ranges of physical addresses that do not correspond to |
090d7171 | 732 | * E820 RAM areas and register the corresponding pages as 'nosave' for |
640e1b38 | 733 | * hibernation (32-bit) or software suspend and suspend to RAM (64-bit). |
bf62f398 | 734 | * |
640e1b38 | 735 | * This function requires the E820 map to be sorted and without any |
84779575 | 736 | * overlapping entries. |
bf62f398 | 737 | */ |
090d7171 | 738 | void __init e820__register_nosave_regions(unsigned long limit_pfn) |
bf62f398 YL |
739 | { |
740 | int i; | |
84779575 | 741 | unsigned long pfn = 0; |
bf62f398 | 742 | |
bf495573 | 743 | for (i = 0; i < e820_table->nr_entries; i++) { |
e5540f87 | 744 | struct e820_entry *entry = &e820_table->entries[i]; |
bf62f398 | 745 | |
e5540f87 IM |
746 | if (pfn < PFN_UP(entry->addr)) |
747 | register_nosave_region(pfn, PFN_UP(entry->addr)); | |
bf62f398 | 748 | |
e5540f87 | 749 | pfn = PFN_DOWN(entry->addr + entry->size); |
ec776ef6 | 750 | |
09821ff1 | 751 | if (entry->type != E820_TYPE_RAM && entry->type != E820_TYPE_RESERVED_KERN) |
e5540f87 | 752 | register_nosave_region(PFN_UP(entry->addr), pfn); |
bf62f398 YL |
753 | |
754 | if (pfn >= limit_pfn) | |
755 | break; | |
756 | } | |
757 | } | |
a4c81cf6 | 758 | |
b54ac6d2 | 759 | #ifdef CONFIG_ACPI |
640e1b38 IM |
760 | /* |
761 | * Register ACPI NVS memory regions, so that we can save/restore them during | |
762 | * hibernation and the subsequent resume: | |
b69edc76 | 763 | */ |
090d7171 | 764 | static int __init e820__register_nvs_regions(void) |
b69edc76 RW |
765 | { |
766 | int i; | |
767 | ||
bf495573 | 768 | for (i = 0; i < e820_table->nr_entries; i++) { |
e5540f87 | 769 | struct e820_entry *entry = &e820_table->entries[i]; |
b69edc76 | 770 | |
09821ff1 | 771 | if (entry->type == E820_TYPE_NVS) |
e5540f87 | 772 | acpi_nvs_register(entry->addr, entry->size); |
b69edc76 RW |
773 | } |
774 | ||
775 | return 0; | |
776 | } | |
090d7171 | 777 | core_initcall(e820__register_nvs_regions); |
b69edc76 RW |
778 | #endif |
779 | ||
2944e16b | 780 | /* |
5da217ca IM |
781 | * Allocate the requested number of bytes with the requsted alignment |
782 | * and return (the physical address) to the caller. Also register this | |
a09bae0f | 783 | * range in the 'kexec' E820 table as a reserved range. |
5da217ca IM |
784 | * |
785 | * This allows kexec to fake a new mptable, as if it came from the real | |
786 | * system. | |
2944e16b | 787 | */ |
5da217ca | 788 | u64 __init e820__memblock_alloc_reserved(u64 size, u64 align) |
2944e16b | 789 | { |
2944e16b | 790 | u64 addr; |
2944e16b | 791 | |
42b46aef | 792 | addr = memblock_phys_alloc(size, align); |
ab5d140b | 793 | if (addr) { |
a09bae0f | 794 | e820__range_update_kexec(addr, size, E820_TYPE_RAM, E820_TYPE_RESERVED); |
1de392f5 | 795 | pr_info("update e820_table_kexec for e820__memblock_alloc_reserved()\n"); |
a09bae0f | 796 | e820__update_table_kexec(); |
61438766 | 797 | } |
2944e16b | 798 | |
2944e16b YL |
799 | return addr; |
800 | } | |
801 | ||
ee0c80fa YL |
802 | #ifdef CONFIG_X86_32 |
803 | # ifdef CONFIG_X86_PAE | |
804 | # define MAX_ARCH_PFN (1ULL<<(36-PAGE_SHIFT)) | |
805 | # else | |
806 | # define MAX_ARCH_PFN (1ULL<<(32-PAGE_SHIFT)) | |
807 | # endif | |
808 | #else /* CONFIG_X86_32 */ | |
bd70e522 | 809 | # define MAX_ARCH_PFN MAXMEM>>PAGE_SHIFT |
ee0c80fa YL |
810 | #endif |
811 | ||
ee0c80fa YL |
812 | /* |
813 | * Find the highest page frame number we have available | |
814 | */ | |
6afc03b8 | 815 | static unsigned long __init e820_end_pfn(unsigned long limit_pfn, enum e820_type type) |
ee0c80fa | 816 | { |
2dc807b3 YL |
817 | int i; |
818 | unsigned long last_pfn = 0; | |
ee0c80fa YL |
819 | unsigned long max_arch_pfn = MAX_ARCH_PFN; |
820 | ||
bf495573 | 821 | for (i = 0; i < e820_table->nr_entries; i++) { |
e5540f87 | 822 | struct e820_entry *entry = &e820_table->entries[i]; |
f361a450 | 823 | unsigned long start_pfn; |
2dc807b3 YL |
824 | unsigned long end_pfn; |
825 | ||
e5540f87 | 826 | if (entry->type != type) |
c22d4c18 | 827 | continue; |
c22d4c18 | 828 | |
e5540f87 IM |
829 | start_pfn = entry->addr >> PAGE_SHIFT; |
830 | end_pfn = (entry->addr + entry->size) >> PAGE_SHIFT; | |
f361a450 YL |
831 | |
832 | if (start_pfn >= limit_pfn) | |
833 | continue; | |
834 | if (end_pfn > limit_pfn) { | |
835 | last_pfn = limit_pfn; | |
836 | break; | |
837 | } | |
2dc807b3 YL |
838 | if (end_pfn > last_pfn) |
839 | last_pfn = end_pfn; | |
840 | } | |
ee0c80fa YL |
841 | |
842 | if (last_pfn > max_arch_pfn) | |
843 | last_pfn = max_arch_pfn; | |
ee0c80fa | 844 | |
1de392f5 JP |
845 | pr_info("last_pfn = %#lx max_arch_pfn = %#lx\n", |
846 | last_pfn, max_arch_pfn); | |
ee0c80fa YL |
847 | return last_pfn; |
848 | } | |
640e1b38 | 849 | |
0c6fc11a | 850 | unsigned long __init e820__end_of_ram_pfn(void) |
f361a450 | 851 | { |
09821ff1 | 852 | return e820_end_pfn(MAX_ARCH_PFN, E820_TYPE_RAM); |
f361a450 | 853 | } |
ee0c80fa | 854 | |
0c6fc11a | 855 | unsigned long __init e820__end_of_low_ram_pfn(void) |
f361a450 | 856 | { |
09821ff1 | 857 | return e820_end_pfn(1UL << (32 - PAGE_SHIFT), E820_TYPE_RAM); |
f361a450 | 858 | } |
ee0c80fa | 859 | |
8c2103f2 | 860 | static void __init early_panic(char *msg) |
ab4a465e YL |
861 | { |
862 | early_printk(msg); | |
863 | panic(msg); | |
864 | } | |
865 | ||
69a7704d YL |
866 | static int userdef __initdata; |
867 | ||
640e1b38 | 868 | /* The "mem=nopentium" boot option disables 4MB page tables on 32-bit kernels: */ |
ab4a465e YL |
869 | static int __init parse_memopt(char *p) |
870 | { | |
871 | u64 mem_size; | |
872 | ||
873 | if (!p) | |
874 | return -EINVAL; | |
875 | ||
ab4a465e | 876 | if (!strcmp(p, "nopentium")) { |
9a6d44b9 | 877 | #ifdef CONFIG_X86_32 |
ab4a465e YL |
878 | setup_clear_cpu_cap(X86_FEATURE_PSE); |
879 | return 0; | |
9a6d44b9 | 880 | #else |
01259ef1 | 881 | pr_warn("mem=nopentium ignored! (only supported on x86_32)\n"); |
9a6d44b9 | 882 | return -EINVAL; |
ab4a465e | 883 | #endif |
9a6d44b9 | 884 | } |
ab4a465e | 885 | |
69a7704d | 886 | userdef = 1; |
ab4a465e | 887 | mem_size = memparse(p, &p); |
640e1b38 IM |
888 | |
889 | /* Don't remove all memory when getting "mem={invalid}" parameter: */ | |
77eed821 KM |
890 | if (mem_size == 0) |
891 | return -EINVAL; | |
640e1b38 | 892 | |
09821ff1 | 893 | e820__range_remove(mem_size, ULLONG_MAX - mem_size, E820_TYPE_RAM, 1); |
611dfd78 | 894 | |
357b4da5 JG |
895 | #ifdef CONFIG_MEMORY_HOTPLUG |
896 | max_mem_size = mem_size; | |
897 | #endif | |
898 | ||
ab4a465e YL |
899 | return 0; |
900 | } | |
901 | early_param("mem", parse_memopt); | |
902 | ||
9710f581 | 903 | static int __init parse_memmap_one(char *p) |
ab4a465e YL |
904 | { |
905 | char *oldp; | |
906 | u64 start_at, mem_size; | |
907 | ||
a737abd1 CG |
908 | if (!p) |
909 | return -EINVAL; | |
910 | ||
d6be118a | 911 | if (!strncmp(p, "exactmap", 8)) { |
ab4a465e YL |
912 | #ifdef CONFIG_CRASH_DUMP |
913 | /* | |
914 | * If we are doing a crash dump, we still need to know | |
640e1b38 | 915 | * the real memory size before the original memory map is |
ab4a465e YL |
916 | * reset. |
917 | */ | |
0c6fc11a | 918 | saved_max_pfn = e820__end_of_ram_pfn(); |
ab4a465e | 919 | #endif |
bf495573 | 920 | e820_table->nr_entries = 0; |
ab4a465e YL |
921 | userdef = 1; |
922 | return 0; | |
923 | } | |
924 | ||
925 | oldp = p; | |
926 | mem_size = memparse(p, &p); | |
927 | if (p == oldp) | |
928 | return -EINVAL; | |
929 | ||
930 | userdef = 1; | |
931 | if (*p == '@') { | |
932 | start_at = memparse(p+1, &p); | |
09821ff1 | 933 | e820__range_add(start_at, mem_size, E820_TYPE_RAM); |
ab4a465e YL |
934 | } else if (*p == '#') { |
935 | start_at = memparse(p+1, &p); | |
09821ff1 | 936 | e820__range_add(start_at, mem_size, E820_TYPE_ACPI); |
ab4a465e YL |
937 | } else if (*p == '$') { |
938 | start_at = memparse(p+1, &p); | |
09821ff1 | 939 | e820__range_add(start_at, mem_size, E820_TYPE_RESERVED); |
ec776ef6 CH |
940 | } else if (*p == '!') { |
941 | start_at = memparse(p+1, &p); | |
09821ff1 | 942 | e820__range_add(start_at, mem_size, E820_TYPE_PRAM); |
ef61f8a3 JS |
943 | } else if (*p == '%') { |
944 | enum e820_type from = 0, to = 0; | |
945 | ||
946 | start_at = memparse(p + 1, &p); | |
947 | if (*p == '-') | |
948 | from = simple_strtoull(p + 1, &p, 0); | |
949 | if (*p == '+') | |
950 | to = simple_strtoull(p + 1, &p, 0); | |
951 | if (*p != '\0') | |
952 | return -EINVAL; | |
953 | if (from && to) | |
954 | e820__range_update(start_at, mem_size, from, to); | |
955 | else if (to) | |
956 | e820__range_add(start_at, mem_size, to); | |
957 | else if (from) | |
958 | e820__range_remove(start_at, mem_size, from, 1); | |
959 | else | |
960 | e820__range_remove(start_at, mem_size, 0, 0); | |
640e1b38 | 961 | } else { |
09821ff1 | 962 | e820__range_remove(mem_size, ULLONG_MAX - mem_size, E820_TYPE_RAM, 1); |
640e1b38 | 963 | } |
7b479bec | 964 | |
ab4a465e YL |
965 | return *p == '\0' ? 0 : -EINVAL; |
966 | } | |
640e1b38 | 967 | |
9710f581 YL |
968 | static int __init parse_memmap_opt(char *str) |
969 | { | |
970 | while (str) { | |
971 | char *k = strchr(str, ','); | |
972 | ||
973 | if (k) | |
974 | *k++ = 0; | |
975 | ||
976 | parse_memmap_one(str); | |
977 | str = k; | |
978 | } | |
979 | ||
980 | return 0; | |
981 | } | |
ab4a465e YL |
982 | early_param("memmap", parse_memmap_opt); |
983 | ||
1a127034 IM |
984 | /* |
985 | * Reserve all entries from the bootloader's extensible data nodes list, | |
986 | * because if present we are going to use it later on to fetch e820 | |
987 | * entries from it: | |
988 | */ | |
989 | void __init e820__reserve_setup_data(void) | |
da92139b IM |
990 | { |
991 | struct setup_data *data; | |
992 | u64 pa_data; | |
993 | ||
994 | pa_data = boot_params.hdr.setup_data; | |
995 | if (!pa_data) | |
996 | return; | |
997 | ||
998 | while (pa_data) { | |
999 | data = early_memremap(pa_data, sizeof(*data)); | |
09821ff1 | 1000 | e820__range_update(pa_data, sizeof(*data)+data->len, E820_TYPE_RAM, E820_TYPE_RESERVED_KERN); |
a09bae0f | 1001 | e820__range_update_kexec(pa_data, sizeof(*data)+data->len, E820_TYPE_RAM, E820_TYPE_RESERVED_KERN); |
da92139b IM |
1002 | pa_data = data->next; |
1003 | early_memunmap(data, sizeof(*data)); | |
1004 | } | |
1005 | ||
f9748fa0 | 1006 | e820__update_table(e820_table); |
a09bae0f | 1007 | e820__update_table(e820_table_kexec); |
1a127034 IM |
1008 | |
1009 | pr_info("extended physical RAM map:\n"); | |
be0c3f0f | 1010 | e820__print_table("reserve setup_data"); |
da92139b IM |
1011 | } |
1012 | ||
9641bdaf IM |
1013 | /* |
1014 | * Called after parse_early_param(), after early parameters (such as mem=) | |
1015 | * have been processed, in which case we already have an E820 table filled in | |
1016 | * via the parameter callback function(s), but it's not sorted and printed yet: | |
1017 | */ | |
1018 | void __init e820__finish_early_params(void) | |
ab4a465e YL |
1019 | { |
1020 | if (userdef) { | |
f9748fa0 | 1021 | if (e820__update_table(e820_table) < 0) |
ab4a465e | 1022 | early_panic("Invalid user supplied memory map"); |
ab4a465e | 1023 | |
1de392f5 | 1024 | pr_info("user-defined physical RAM map:\n"); |
be0c3f0f | 1025 | e820__print_table("user"); |
ab4a465e YL |
1026 | } |
1027 | } | |
41c094fd | 1028 | |
c594761d | 1029 | static const char *__init e820_type_to_string(struct e820_entry *entry) |
5dfcf14d | 1030 | { |
c594761d | 1031 | switch (entry->type) { |
09821ff1 IM |
1032 | case E820_TYPE_RESERVED_KERN: /* Fall-through: */ |
1033 | case E820_TYPE_RAM: return "System RAM"; | |
1034 | case E820_TYPE_ACPI: return "ACPI Tables"; | |
1035 | case E820_TYPE_NVS: return "ACPI Non-volatile Storage"; | |
1036 | case E820_TYPE_UNUSABLE: return "Unusable memory"; | |
1037 | case E820_TYPE_PRAM: return "Persistent Memory (legacy)"; | |
1038 | case E820_TYPE_PMEM: return "Persistent Memory"; | |
c5231a57 IM |
1039 | case E820_TYPE_RESERVED: return "Reserved"; |
1040 | default: return "Unknown E820 type"; | |
5dfcf14d BW |
1041 | } |
1042 | } | |
1043 | ||
c594761d | 1044 | static unsigned long __init e820_type_to_iomem_type(struct e820_entry *entry) |
f33b14a4 | 1045 | { |
c594761d | 1046 | switch (entry->type) { |
09821ff1 IM |
1047 | case E820_TYPE_RESERVED_KERN: /* Fall-through: */ |
1048 | case E820_TYPE_RAM: return IORESOURCE_SYSTEM_RAM; | |
1049 | case E820_TYPE_ACPI: /* Fall-through: */ | |
1050 | case E820_TYPE_NVS: /* Fall-through: */ | |
1051 | case E820_TYPE_UNUSABLE: /* Fall-through: */ | |
1052 | case E820_TYPE_PRAM: /* Fall-through: */ | |
1053 | case E820_TYPE_PMEM: /* Fall-through: */ | |
c5231a57 | 1054 | case E820_TYPE_RESERVED: /* Fall-through: */ |
09821ff1 | 1055 | default: return IORESOURCE_MEM; |
f33b14a4 TK |
1056 | } |
1057 | } | |
1058 | ||
c594761d | 1059 | static unsigned long __init e820_type_to_iores_desc(struct e820_entry *entry) |
f33b14a4 | 1060 | { |
c594761d | 1061 | switch (entry->type) { |
09821ff1 IM |
1062 | case E820_TYPE_ACPI: return IORES_DESC_ACPI_TABLES; |
1063 | case E820_TYPE_NVS: return IORES_DESC_ACPI_NV_STORAGE; | |
1064 | case E820_TYPE_PMEM: return IORES_DESC_PERSISTENT_MEMORY; | |
1065 | case E820_TYPE_PRAM: return IORES_DESC_PERSISTENT_MEMORY_LEGACY; | |
1066 | case E820_TYPE_RESERVED_KERN: /* Fall-through: */ | |
1067 | case E820_TYPE_RAM: /* Fall-through: */ | |
1068 | case E820_TYPE_UNUSABLE: /* Fall-through: */ | |
c5231a57 | 1069 | case E820_TYPE_RESERVED: /* Fall-through: */ |
09821ff1 | 1070 | default: return IORES_DESC_NONE; |
f33b14a4 TK |
1071 | } |
1072 | } | |
1073 | ||
c5231a57 | 1074 | static bool __init do_mark_busy(enum e820_type type, struct resource *res) |
ad5fb870 DW |
1075 | { |
1076 | /* this is the legacy bios/dos rom-shadow + mmio region */ | |
1077 | if (res->start < (1ULL<<20)) | |
1078 | return true; | |
1079 | ||
1080 | /* | |
1081 | * Treat persistent memory like device memory, i.e. reserve it | |
1082 | * for exclusive use of a driver | |
1083 | */ | |
1084 | switch (type) { | |
09821ff1 IM |
1085 | case E820_TYPE_RESERVED: |
1086 | case E820_TYPE_PRAM: | |
1087 | case E820_TYPE_PMEM: | |
ad5fb870 | 1088 | return false; |
c5231a57 IM |
1089 | case E820_TYPE_RESERVED_KERN: |
1090 | case E820_TYPE_RAM: | |
1091 | case E820_TYPE_ACPI: | |
1092 | case E820_TYPE_NVS: | |
1093 | case E820_TYPE_UNUSABLE: | |
ad5fb870 DW |
1094 | default: |
1095 | return true; | |
1096 | } | |
1097 | } | |
1098 | ||
41c094fd | 1099 | /* |
640e1b38 | 1100 | * Mark E820 reserved areas as busy for the resource manager: |
41c094fd | 1101 | */ |
640e1b38 | 1102 | |
a5444d15 | 1103 | static struct resource __initdata *e820_res; |
640e1b38 | 1104 | |
1506c8dc | 1105 | void __init e820__reserve_resources(void) |
41c094fd YL |
1106 | { |
1107 | int i; | |
58f7c988 | 1108 | struct resource *res; |
a5444d15 | 1109 | u64 end; |
41c094fd | 1110 | |
7e1c4e27 MR |
1111 | res = memblock_alloc(sizeof(*res) * e820_table->nr_entries, |
1112 | SMP_CACHE_BYTES); | |
8a7f97b9 MR |
1113 | if (!res) |
1114 | panic("%s: Failed to allocate %zu bytes\n", __func__, | |
1115 | sizeof(*res) * e820_table->nr_entries); | |
58f7c988 | 1116 | e820_res = res; |
c594761d | 1117 | |
bf495573 | 1118 | for (i = 0; i < e820_table->nr_entries; i++) { |
c594761d IM |
1119 | struct e820_entry *entry = e820_table->entries + i; |
1120 | ||
1121 | end = entry->addr + entry->size - 1; | |
8308c54d | 1122 | if (end != (resource_size_t)end) { |
41c094fd YL |
1123 | res++; |
1124 | continue; | |
1125 | } | |
c594761d IM |
1126 | res->start = entry->addr; |
1127 | res->end = end; | |
1128 | res->name = e820_type_to_string(entry); | |
1129 | res->flags = e820_type_to_iomem_type(entry); | |
1130 | res->desc = e820_type_to_iores_desc(entry); | |
a5444d15 IM |
1131 | |
1132 | /* | |
1506c8dc IM |
1133 | * Don't register the region that could be conflicted with |
1134 | * PCI device BAR resources and insert them later in | |
1135 | * pcibios_resource_survey(): | |
a5444d15 | 1136 | */ |
c594761d | 1137 | if (do_mark_busy(entry->type, res)) { |
1f987577 | 1138 | res->flags |= IORESOURCE_BUSY; |
58f7c988 | 1139 | insert_resource(&iomem_resource, res); |
1f987577 | 1140 | } |
41c094fd YL |
1141 | res++; |
1142 | } | |
5dfcf14d | 1143 | |
12df216c CY |
1144 | /* Expose the bootloader-provided memory layout to the sysfs. */ |
1145 | for (i = 0; i < e820_table_firmware->nr_entries; i++) { | |
1146 | struct e820_entry *entry = e820_table_firmware->entries + i; | |
640e1b38 | 1147 | |
c594761d | 1148 | firmware_map_add_early(entry->addr, entry->addr + entry->size, e820_type_to_string(entry)); |
5dfcf14d | 1149 | } |
41c094fd YL |
1150 | } |
1151 | ||
1506c8dc IM |
1152 | /* |
1153 | * How much should we pad the end of RAM, depending on where it is? | |
1154 | */ | |
8c2103f2 | 1155 | static unsigned long __init ram_alignment(resource_size_t pos) |
45fbe3ee LT |
1156 | { |
1157 | unsigned long mb = pos >> 20; | |
1158 | ||
1159 | /* To 64kB in the first megabyte */ | |
1160 | if (!mb) | |
1161 | return 64*1024; | |
1162 | ||
1163 | /* To 1MB in the first 16MB */ | |
1164 | if (mb < 16) | |
1165 | return 1024*1024; | |
1166 | ||
15b812f1 YL |
1167 | /* To 64MB for anything above that */ |
1168 | return 64*1024*1024; | |
45fbe3ee LT |
1169 | } |
1170 | ||
7c5371c4 YL |
1171 | #define MAX_RESOURCE_SIZE ((resource_size_t)-1) |
1172 | ||
1506c8dc | 1173 | void __init e820__reserve_resources_late(void) |
58f7c988 YL |
1174 | { |
1175 | int i; | |
1176 | struct resource *res; | |
1177 | ||
1178 | res = e820_res; | |
bf495573 | 1179 | for (i = 0; i < e820_table->nr_entries; i++) { |
a5444d15 | 1180 | if (!res->parent && res->end) |
1f987577 | 1181 | insert_resource_expand_to_fit(&iomem_resource, res); |
58f7c988 YL |
1182 | res++; |
1183 | } | |
45fbe3ee LT |
1184 | |
1185 | /* | |
640e1b38 | 1186 | * Try to bump up RAM regions to reasonable boundaries, to |
45fbe3ee LT |
1187 | * avoid stolen RAM: |
1188 | */ | |
bf495573 IM |
1189 | for (i = 0; i < e820_table->nr_entries; i++) { |
1190 | struct e820_entry *entry = &e820_table->entries[i]; | |
7c5371c4 | 1191 | u64 start, end; |
45fbe3ee | 1192 | |
09821ff1 | 1193 | if (entry->type != E820_TYPE_RAM) |
45fbe3ee | 1194 | continue; |
640e1b38 | 1195 | |
45fbe3ee | 1196 | start = entry->addr + entry->size; |
7c5371c4 YL |
1197 | end = round_up(start, ram_alignment(start)) - 1; |
1198 | if (end > MAX_RESOURCE_SIZE) | |
1199 | end = MAX_RESOURCE_SIZE; | |
1200 | if (start >= end) | |
45fbe3ee | 1201 | continue; |
640e1b38 | 1202 | |
e22af0be | 1203 | printk(KERN_DEBUG "e820: reserve RAM buffer [mem %#010llx-%#010llx]\n", start, end); |
640e1b38 | 1204 | reserve_region_with_split(&iomem_resource, start, end, "RAM buffer"); |
45fbe3ee | 1205 | } |
58f7c988 YL |
1206 | } |
1207 | ||
640e1b38 IM |
1208 | /* |
1209 | * Pass the firmware (bootloader) E820 map to the kernel and process it: | |
1210 | */ | |
103e2063 | 1211 | char *__init e820__memory_setup_default(void) |
064d25f1 YL |
1212 | { |
1213 | char *who = "BIOS-e820"; | |
640e1b38 | 1214 | |
064d25f1 YL |
1215 | /* |
1216 | * Try to copy the BIOS-supplied E820-map. | |
1217 | * | |
1218 | * Otherwise fake a memory map; one section from 0k->640k, | |
1219 | * the next section from 1mb->appropriate_mem_k | |
1220 | */ | |
640e1b38 | 1221 | if (append_e820_table(boot_params.e820_table, boot_params.e820_entries) < 0) { |
95a71a45 | 1222 | u64 mem_size; |
064d25f1 | 1223 | |
640e1b38 IM |
1224 | /* Compare results from other methods and take the one that gives more RAM: */ |
1225 | if (boot_params.alt_mem_k < boot_params.screen_info.ext_mem_k) { | |
064d25f1 YL |
1226 | mem_size = boot_params.screen_info.ext_mem_k; |
1227 | who = "BIOS-88"; | |
1228 | } else { | |
1229 | mem_size = boot_params.alt_mem_k; | |
1230 | who = "BIOS-e801"; | |
1231 | } | |
1232 | ||
bf495573 | 1233 | e820_table->nr_entries = 0; |
09821ff1 IM |
1234 | e820__range_add(0, LOWMEMSIZE(), E820_TYPE_RAM); |
1235 | e820__range_add(HIGH_MEMORY, mem_size << 10, E820_TYPE_RAM); | |
064d25f1 YL |
1236 | } |
1237 | ||
7410aa1c IM |
1238 | /* We just appended a lot of ranges, sanitize the table: */ |
1239 | e820__update_table(e820_table); | |
1240 | ||
064d25f1 YL |
1241 | return who; |
1242 | } | |
1243 | ||
103e2063 IM |
1244 | /* |
1245 | * Calls e820__memory_setup_default() in essence to pick up the firmware/bootloader | |
1246 | * E820 map - with an optional platform quirk available for virtual platforms | |
1247 | * to override this method of boot environment processing: | |
1248 | */ | |
1249 | void __init e820__memory_setup(void) | |
064d25f1 | 1250 | { |
0be15526 YL |
1251 | char *who; |
1252 | ||
09c51513 | 1253 | /* This is a firmware interface ABI - make sure we don't break it: */ |
7410aa1c | 1254 | BUILD_BUG_ON(sizeof(struct boot_e820_entry) != 20); |
09c51513 | 1255 | |
6b18ae3e | 1256 | who = x86_init.resources.memory_setup(); |
640e1b38 | 1257 | |
a09bae0f | 1258 | memcpy(e820_table_kexec, e820_table, sizeof(*e820_table_kexec)); |
12df216c | 1259 | memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware)); |
640e1b38 | 1260 | |
1de392f5 | 1261 | pr_info("BIOS-provided physical RAM map:\n"); |
be0c3f0f | 1262 | e820__print_table(who); |
064d25f1 | 1263 | } |
72d7c3b3 | 1264 | |
4918e228 | 1265 | void __init e820__memblock_setup(void) |
72d7c3b3 YL |
1266 | { |
1267 | int i; | |
1268 | u64 end; | |
1269 | ||
1270 | /* | |
4918e228 IM |
1271 | * The bootstrap memblock region count maximum is 128 entries |
1272 | * (INIT_MEMBLOCK_REGIONS), but EFI might pass us more E820 entries | |
1273 | * than that - so allow memblock resizing. | |
1274 | * | |
1275 | * This is safe, because this call happens pretty late during x86 setup, | |
1276 | * so we know about reserved memory regions already. (This is important | |
1277 | * so that memblock resizing does no stomp over reserved areas.) | |
72d7c3b3 | 1278 | */ |
1aadc056 | 1279 | memblock_allow_resize(); |
72d7c3b3 | 1280 | |
bf495573 | 1281 | for (i = 0; i < e820_table->nr_entries; i++) { |
e5540f87 | 1282 | struct e820_entry *entry = &e820_table->entries[i]; |
72d7c3b3 | 1283 | |
e5540f87 | 1284 | end = entry->addr + entry->size; |
72d7c3b3 YL |
1285 | if (end != (resource_size_t)end) |
1286 | continue; | |
1287 | ||
09821ff1 | 1288 | if (entry->type != E820_TYPE_RAM && entry->type != E820_TYPE_RESERVED_KERN) |
9fd61bc9 MM |
1289 | continue; |
1290 | ||
1291 | memblock_add(entry->addr, entry->size); | |
72d7c3b3 YL |
1292 | } |
1293 | ||
4918e228 | 1294 | /* Throw away partial pages: */ |
6ede1fd3 YL |
1295 | memblock_trim_memory(PAGE_SIZE); |
1296 | ||
72d7c3b3 YL |
1297 | memblock_dump_all(); |
1298 | } |