]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - mm/nobootmem.c
mm: add a helper function to check may oom condition
[mirror_ubuntu-bionic-kernel.git] / mm / nobootmem.c
CommitLineData
09325873
YL
1/*
2 * bootmem - A boot-time physical memory allocator and configurator
3 *
4 * Copyright (C) 1999 Ingo Molnar
5 * 1999 Kanoj Sarcar, SGI
6 * 2008 Johannes Weiner
7 *
8 * Access to this subsystem has to be serialized externally (which is true
9 * for the boot process anyway).
10 */
11#include <linux/init.h>
12#include <linux/pfn.h>
13#include <linux/slab.h>
14#include <linux/bootmem.h>
b95f1b31 15#include <linux/export.h>
09325873
YL
16#include <linux/kmemleak.h>
17#include <linux/range.h>
18#include <linux/memblock.h>
19
20#include <asm/bug.h>
21#include <asm/io.h>
22#include <asm/processor.h>
23
24#include "internal.h"
25
e782ab42
YL
26#ifndef CONFIG_NEED_MULTIPLE_NODES
27struct pglist_data __refdata contig_page_data;
28EXPORT_SYMBOL(contig_page_data);
29#endif
30
09325873
YL
31unsigned long max_low_pfn;
32unsigned long min_low_pfn;
33unsigned long max_pfn;
34
8bc1f91e
YL
35static void * __init __alloc_memory_core_early(int nid, u64 size, u64 align,
36 u64 goal, u64 limit)
37{
38 void *ptr;
39 u64 addr;
40
41 if (limit > memblock.current_limit)
42 limit = memblock.current_limit;
43
eb40c4c2 44 addr = memblock_find_in_range_node(goal, limit, size, align, nid);
1f5026a7 45 if (!addr)
8bc1f91e
YL
46 return NULL;
47
b476e295 48 memblock_reserve(addr, size);
8bc1f91e
YL
49 ptr = phys_to_virt(addr);
50 memset(ptr, 0, size);
8bc1f91e
YL
51 /*
52 * The min_count is set to 0 so that bootmem allocated blocks
53 * are never reported as leaks.
54 */
55 kmemleak_alloc(ptr, size, 0, 0);
56 return ptr;
57}
58
09325873
YL
59/*
60 * free_bootmem_late - free bootmem pages directly to page allocator
61 * @addr: starting address of the range
62 * @size: size of the range in bytes
63 *
64 * This is only useful when the bootmem allocator has already been torn
65 * down, but we are still initializing the system. Pages are given directly
66 * to the page allocator, no bootmem metadata is updated because it is gone.
67 */
68void __init free_bootmem_late(unsigned long addr, unsigned long size)
69{
70 unsigned long cursor, end;
71
72 kmemleak_free_part(__va(addr), size);
73
74 cursor = PFN_UP(addr);
75 end = PFN_DOWN(addr + size);
76
77 for (; cursor < end; cursor++) {
78 __free_pages_bootmem(pfn_to_page(cursor), 0);
79 totalram_pages++;
80 }
81}
82
83static void __init __free_pages_memory(unsigned long start, unsigned long end)
84{
6bc2e853 85 unsigned long i, start_aligned, end_aligned;
09325873
YL
86 int order = ilog2(BITS_PER_LONG);
87
88 start_aligned = (start + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1);
89 end_aligned = end & ~(BITS_PER_LONG - 1);
90
91 if (end_aligned <= start_aligned) {
92 for (i = start; i < end; i++)
93 __free_pages_bootmem(pfn_to_page(i), 0);
94
95 return;
96 }
97
98 for (i = start; i < start_aligned; i++)
99 __free_pages_bootmem(pfn_to_page(i), 0);
100
101 for (i = start_aligned; i < end_aligned; i += BITS_PER_LONG)
102 __free_pages_bootmem(pfn_to_page(i), order);
103
104 for (i = end_aligned; i < end; i++)
105 __free_pages_bootmem(pfn_to_page(i), 0);
106}
107
29f67386
YL
108static unsigned long __init __free_memory_core(phys_addr_t start,
109 phys_addr_t end)
110{
111 unsigned long start_pfn = PFN_UP(start);
112 unsigned long end_pfn = min_t(unsigned long,
113 PFN_DOWN(end), max_low_pfn);
114
115 if (start_pfn > end_pfn)
116 return 0;
117
118 __free_pages_memory(start_pfn, end_pfn);
119
120 return end_pfn - start_pfn;
121}
122
b4def350 123static unsigned long __init free_low_memory_core_early(void)
09325873 124{
09325873 125 unsigned long count = 0;
29f67386 126 phys_addr_t start, end, size;
8a9ca34c
TH
127 u64 i;
128
29f67386
YL
129 for_each_free_mem_range(i, MAX_NUMNODES, &start, &end, NULL)
130 count += __free_memory_core(start, end);
131
132 /* free range that is used for reserved array if we allocate it */
133 size = get_allocated_memblock_reserved_regions_info(&start);
134 if (size)
135 count += __free_memory_core(start, start + size);
09325873
YL
136
137 return count;
138}
139
7b4b2a0d
JL
140static int reset_managed_pages_done __initdata;
141
142static inline void __init reset_node_managed_pages(pg_data_t *pgdat)
9feedc9d
JL
143{
144 struct zone *z;
145
7b4b2a0d
JL
146 if (reset_managed_pages_done)
147 return;
9feedc9d 148 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
7b4b2a0d
JL
149 z->managed_pages = 0;
150}
151
152void __init reset_all_zones_managed_pages(void)
153{
154 struct pglist_data *pgdat;
155
156 for_each_online_pgdat(pgdat)
157 reset_node_managed_pages(pgdat);
158 reset_managed_pages_done = 1;
9feedc9d
JL
159}
160
09325873
YL
161/**
162 * free_all_bootmem - release free pages to the buddy allocator
163 *
164 * Returns the number of pages actually released.
165 */
166unsigned long __init free_all_bootmem(void)
167{
0c988534
JL
168 unsigned long pages;
169
7b4b2a0d 170 reset_all_zones_managed_pages();
9feedc9d 171
09325873
YL
172 /*
173 * We need to use MAX_NUMNODES instead of NODE_DATA(0)->node_id
25985edc 174 * because in some case like Node0 doesn't have RAM installed
09325873 175 * low ram will be on Node1
09325873 176 */
0c988534
JL
177 pages = free_low_memory_core_early();
178 totalram_pages += pages;
179
180 return pages;
09325873
YL
181}
182
183/**
184 * free_bootmem_node - mark a page range as usable
185 * @pgdat: node the range resides on
186 * @physaddr: starting address of the range
187 * @size: size of the range in bytes
188 *
189 * Partial pages will be considered reserved and left as they are.
190 *
191 * The range must reside completely on the specified node.
192 */
193void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
194 unsigned long size)
195{
196 kmemleak_free_part(__va(physaddr), size);
24aa0788 197 memblock_free(physaddr, size);
09325873
YL
198}
199
200/**
201 * free_bootmem - mark a page range as usable
202 * @addr: starting address of the range
203 * @size: size of the range in bytes
204 *
205 * Partial pages will be considered reserved and left as they are.
206 *
207 * The range must be contiguous but may span node boundaries.
208 */
209void __init free_bootmem(unsigned long addr, unsigned long size)
210{
211 kmemleak_free_part(__va(addr), size);
24aa0788 212 memblock_free(addr, size);
09325873
YL
213}
214
215static void * __init ___alloc_bootmem_nopanic(unsigned long size,
216 unsigned long align,
217 unsigned long goal,
218 unsigned long limit)
219{
220 void *ptr;
221
222 if (WARN_ON_ONCE(slab_is_available()))
223 return kzalloc(size, GFP_NOWAIT);
224
225restart:
226
227 ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align, goal, limit);
228
229 if (ptr)
230 return ptr;
231
232 if (goal != 0) {
233 goal = 0;
234 goto restart;
235 }
236
237 return NULL;
238}
239
240/**
241 * __alloc_bootmem_nopanic - allocate boot memory without panicking
242 * @size: size of the request in bytes
243 * @align: alignment of the region
244 * @goal: preferred starting address of the region
245 *
246 * The goal is dropped if it can not be satisfied and the allocation will
247 * fall back to memory below @goal.
248 *
249 * Allocation may happen on any node in the system.
250 *
251 * Returns NULL on failure.
252 */
253void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
254 unsigned long goal)
255{
256 unsigned long limit = -1UL;
257
258 return ___alloc_bootmem_nopanic(size, align, goal, limit);
259}
260
261static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
262 unsigned long goal, unsigned long limit)
263{
264 void *mem = ___alloc_bootmem_nopanic(size, align, goal, limit);
265
266 if (mem)
267 return mem;
268 /*
269 * Whoops, we cannot satisfy the allocation request.
270 */
271 printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
272 panic("Out of memory");
273 return NULL;
274}
275
276/**
277 * __alloc_bootmem - allocate boot memory
278 * @size: size of the request in bytes
279 * @align: alignment of the region
280 * @goal: preferred starting address of the region
281 *
282 * The goal is dropped if it can not be satisfied and the allocation will
283 * fall back to memory below @goal.
284 *
285 * Allocation may happen on any node in the system.
286 *
287 * The function panics if the request can not be satisfied.
288 */
289void * __init __alloc_bootmem(unsigned long size, unsigned long align,
290 unsigned long goal)
291{
292 unsigned long limit = -1UL;
293
294 return ___alloc_bootmem(size, align, goal, limit);
295}
296
99ab7b19 297void * __init ___alloc_bootmem_node_nopanic(pg_data_t *pgdat,
ba539868
JW
298 unsigned long size,
299 unsigned long align,
300 unsigned long goal,
301 unsigned long limit)
302{
303 void *ptr;
304
305again:
306 ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
307 goal, limit);
308 if (ptr)
309 return ptr;
310
311 ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align,
312 goal, limit);
313 if (ptr)
314 return ptr;
315
316 if (goal) {
317 goal = 0;
318 goto again;
319 }
320
321 return NULL;
322}
323
324void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
325 unsigned long align, unsigned long goal)
326{
327 if (WARN_ON_ONCE(slab_is_available()))
328 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
329
330 return ___alloc_bootmem_node_nopanic(pgdat, size, align, goal, 0);
331}
332
333void * __init ___alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
334 unsigned long align, unsigned long goal,
335 unsigned long limit)
336{
337 void *ptr;
338
339 ptr = ___alloc_bootmem_node_nopanic(pgdat, size, align, goal, limit);
340 if (ptr)
341 return ptr;
342
343 printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
344 panic("Out of memory");
345 return NULL;
346}
347
09325873
YL
348/**
349 * __alloc_bootmem_node - allocate boot memory from a specific node
350 * @pgdat: node to allocate from
351 * @size: size of the request in bytes
352 * @align: alignment of the region
353 * @goal: preferred starting address of the region
354 *
355 * The goal is dropped if it can not be satisfied and the allocation will
356 * fall back to memory below @goal.
357 *
358 * Allocation may fall back to any node in the system if the specified node
359 * can not hold the requested memory.
360 *
361 * The function panics if the request can not be satisfied.
362 */
363void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
364 unsigned long align, unsigned long goal)
365{
09325873
YL
366 if (WARN_ON_ONCE(slab_is_available()))
367 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
368
ba539868 369 return ___alloc_bootmem_node(pgdat, size, align, goal, 0);
09325873
YL
370}
371
372void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size,
373 unsigned long align, unsigned long goal)
374{
09325873 375 return __alloc_bootmem_node(pgdat, size, align, goal);
09325873
YL
376}
377
09325873
YL
378#ifndef ARCH_LOW_ADDRESS_LIMIT
379#define ARCH_LOW_ADDRESS_LIMIT 0xffffffffUL
380#endif
381
382/**
383 * __alloc_bootmem_low - allocate low boot memory
384 * @size: size of the request in bytes
385 * @align: alignment of the region
386 * @goal: preferred starting address of the region
387 *
388 * The goal is dropped if it can not be satisfied and the allocation will
389 * fall back to memory below @goal.
390 *
391 * Allocation may happen on any node in the system.
392 *
393 * The function panics if the request can not be satisfied.
394 */
395void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
396 unsigned long goal)
397{
398 return ___alloc_bootmem(size, align, goal, ARCH_LOW_ADDRESS_LIMIT);
399}
400
38fa4175
YL
401void * __init __alloc_bootmem_low_nopanic(unsigned long size,
402 unsigned long align,
403 unsigned long goal)
404{
405 return ___alloc_bootmem_nopanic(size, align, goal,
406 ARCH_LOW_ADDRESS_LIMIT);
407}
408
09325873
YL
409/**
410 * __alloc_bootmem_low_node - allocate low boot memory from a specific node
411 * @pgdat: node to allocate from
412 * @size: size of the request in bytes
413 * @align: alignment of the region
414 * @goal: preferred starting address of the region
415 *
416 * The goal is dropped if it can not be satisfied and the allocation will
417 * fall back to memory below @goal.
418 *
419 * Allocation may fall back to any node in the system if the specified node
420 * can not hold the requested memory.
421 *
422 * The function panics if the request can not be satisfied.
423 */
424void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
425 unsigned long align, unsigned long goal)
426{
09325873
YL
427 if (WARN_ON_ONCE(slab_is_available()))
428 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
429
ba539868
JW
430 return ___alloc_bootmem_node(pgdat, size, align, goal,
431 ARCH_LOW_ADDRESS_LIMIT);
09325873 432}