2 * bootmem - A boot-time physical memory allocator and configurator
4 * Copyright (C) 1999 Ingo Molnar
5 * 1999 Kanoj Sarcar, SGI
8 * Access to this subsystem has to be serialized externally (which is true
9 * for the boot process anyway).
11 #include <linux/init.h>
12 #include <linux/pfn.h>
13 #include <linux/slab.h>
14 #include <linux/bootmem.h>
15 #include <linux/module.h>
16 #include <linux/kmemleak.h>
17 #include <linux/range.h>
21 #include <asm/processor.h>
25 unsigned long max_low_pfn
;
26 unsigned long min_low_pfn
;
27 unsigned long max_pfn
;
29 #ifdef CONFIG_CRASH_DUMP
31 * If we have booted due to a crash, max_pfn will be a very low value. We need
32 * to know the amount of memory that the previous kernel used.
34 unsigned long saved_max_pfn
;
37 #ifndef CONFIG_NO_BOOTMEM
38 bootmem_data_t bootmem_node_data
[MAX_NUMNODES
] __initdata
;
40 static struct list_head bdata_list __initdata
= LIST_HEAD_INIT(bdata_list
);
42 static int bootmem_debug
;
44 static int __init
bootmem_debug_setup(char *buf
)
49 early_param("bootmem_debug", bootmem_debug_setup
);
51 #define bdebug(fmt, args...) ({ \
52 if (unlikely(bootmem_debug)) \
58 static unsigned long __init
bootmap_bytes(unsigned long pages
)
60 unsigned long bytes
= (pages
+ 7) / 8;
62 return ALIGN(bytes
, sizeof(long));
66 * bootmem_bootmap_pages - calculate bitmap size in pages
67 * @pages: number of pages the bitmap has to represent
69 unsigned long __init
bootmem_bootmap_pages(unsigned long pages
)
71 unsigned long bytes
= bootmap_bytes(pages
);
73 return PAGE_ALIGN(bytes
) >> PAGE_SHIFT
;
79 static void __init
link_bootmem(bootmem_data_t
*bdata
)
81 struct list_head
*iter
;
83 list_for_each(iter
, &bdata_list
) {
86 ent
= list_entry(iter
, bootmem_data_t
, list
);
87 if (bdata
->node_min_pfn
< ent
->node_min_pfn
)
90 list_add_tail(&bdata
->list
, iter
);
94 * Called once to set up the allocator itself.
96 static unsigned long __init
init_bootmem_core(bootmem_data_t
*bdata
,
97 unsigned long mapstart
, unsigned long start
, unsigned long end
)
99 unsigned long mapsize
;
101 mminit_validate_memmodel_limits(&start
, &end
);
102 bdata
->node_bootmem_map
= phys_to_virt(PFN_PHYS(mapstart
));
103 bdata
->node_min_pfn
= start
;
104 bdata
->node_low_pfn
= end
;
108 * Initially all pages are reserved - setup_arch() has to
109 * register free RAM areas explicitly.
111 mapsize
= bootmap_bytes(end
- start
);
112 memset(bdata
->node_bootmem_map
, 0xff, mapsize
);
114 bdebug("nid=%td start=%lx map=%lx end=%lx mapsize=%lx\n",
115 bdata
- bootmem_node_data
, start
, mapstart
, end
, mapsize
);
121 * init_bootmem_node - register a node as boot memory
122 * @pgdat: node to register
123 * @freepfn: pfn where the bitmap for this node is to be placed
124 * @startpfn: first pfn on the node
125 * @endpfn: first pfn after the node
127 * Returns the number of bytes needed to hold the bitmap for this node.
129 unsigned long __init
init_bootmem_node(pg_data_t
*pgdat
, unsigned long freepfn
,
130 unsigned long startpfn
, unsigned long endpfn
)
132 return init_bootmem_core(pgdat
->bdata
, freepfn
, startpfn
, endpfn
);
136 * init_bootmem - register boot memory
137 * @start: pfn where the bitmap is to be placed
138 * @pages: number of available physical pages
140 * Returns the number of bytes needed to hold the bitmap.
142 unsigned long __init
init_bootmem(unsigned long start
, unsigned long pages
)
146 return init_bootmem_core(NODE_DATA(0)->bdata
, start
, 0, pages
);
150 * free_bootmem_late - free bootmem pages directly to page allocator
151 * @addr: starting address of the range
152 * @size: size of the range in bytes
154 * This is only useful when the bootmem allocator has already been torn
155 * down, but we are still initializing the system. Pages are given directly
156 * to the page allocator, no bootmem metadata is updated because it is gone.
158 void __init
free_bootmem_late(unsigned long addr
, unsigned long size
)
160 unsigned long cursor
, end
;
162 kmemleak_free_part(__va(addr
), size
);
164 cursor
= PFN_UP(addr
);
165 end
= PFN_DOWN(addr
+ size
);
167 for (; cursor
< end
; cursor
++) {
168 __free_pages_bootmem(pfn_to_page(cursor
), 0);
173 #ifdef CONFIG_NO_BOOTMEM
174 static void __init
__free_pages_memory(unsigned long start
, unsigned long end
)
177 unsigned long start_aligned
, end_aligned
;
178 int order
= ilog2(BITS_PER_LONG
);
180 start_aligned
= (start
+ (BITS_PER_LONG
- 1)) & ~(BITS_PER_LONG
- 1);
181 end_aligned
= end
& ~(BITS_PER_LONG
- 1);
183 if (end_aligned
<= start_aligned
) {
184 for (i
= start
; i
< end
; i
++)
185 __free_pages_bootmem(pfn_to_page(i
), 0);
190 for (i
= start
; i
< start_aligned
; i
++)
191 __free_pages_bootmem(pfn_to_page(i
), 0);
193 for (i
= start_aligned
; i
< end_aligned
; i
+= BITS_PER_LONG
)
194 __free_pages_bootmem(pfn_to_page(i
), order
);
196 for (i
= end_aligned
; i
< end
; i
++)
197 __free_pages_bootmem(pfn_to_page(i
), 0);
200 unsigned long __init
free_all_memory_core_early(int nodeid
)
204 unsigned long count
= 0;
205 struct range
*range
= NULL
;
208 nr_range
= get_free_all_memory_range(&range
, nodeid
);
210 for (i
= 0; i
< nr_range
; i
++) {
211 start
= range
[i
].start
;
213 count
+= end
- start
;
214 __free_pages_memory(start
, end
);
220 static unsigned long __init
free_all_bootmem_core(bootmem_data_t
*bdata
)
224 unsigned long start
, end
, pages
, count
= 0;
226 if (!bdata
->node_bootmem_map
)
229 start
= bdata
->node_min_pfn
;
230 end
= bdata
->node_low_pfn
;
233 * If the start is aligned to the machines wordsize, we might
234 * be able to free pages in bulks of that order.
236 aligned
= !(start
& (BITS_PER_LONG
- 1));
238 bdebug("nid=%td start=%lx end=%lx aligned=%d\n",
239 bdata
- bootmem_node_data
, start
, end
, aligned
);
241 while (start
< end
) {
242 unsigned long *map
, idx
, vec
;
244 map
= bdata
->node_bootmem_map
;
245 idx
= start
- bdata
->node_min_pfn
;
246 vec
= ~map
[idx
/ BITS_PER_LONG
];
248 if (aligned
&& vec
== ~0UL && start
+ BITS_PER_LONG
< end
) {
249 int order
= ilog2(BITS_PER_LONG
);
251 __free_pages_bootmem(pfn_to_page(start
), order
);
252 count
+= BITS_PER_LONG
;
254 unsigned long off
= 0;
256 while (vec
&& off
< BITS_PER_LONG
) {
258 page
= pfn_to_page(start
+ off
);
259 __free_pages_bootmem(page
, 0);
266 start
+= BITS_PER_LONG
;
269 page
= virt_to_page(bdata
->node_bootmem_map
);
270 pages
= bdata
->node_low_pfn
- bdata
->node_min_pfn
;
271 pages
= bootmem_bootmap_pages(pages
);
274 __free_pages_bootmem(page
++, 0);
276 bdebug("nid=%td released=%lx\n", bdata
- bootmem_node_data
, count
);
283 * free_all_bootmem_node - release a node's free pages to the buddy allocator
284 * @pgdat: node to be released
286 * Returns the number of pages actually released.
288 unsigned long __init
free_all_bootmem_node(pg_data_t
*pgdat
)
290 register_page_bootmem_info_node(pgdat
);
291 #ifdef CONFIG_NO_BOOTMEM
292 /* free_all_memory_core_early(MAX_NUMNODES) will be called later */
295 return free_all_bootmem_core(pgdat
->bdata
);
300 * free_all_bootmem - release free pages to the buddy allocator
302 * Returns the number of pages actually released.
304 unsigned long __init
free_all_bootmem(void)
306 #ifdef CONFIG_NO_BOOTMEM
307 return free_all_memory_core_early(NODE_DATA(0)->node_id
);
309 return free_all_bootmem_core(NODE_DATA(0)->bdata
);
313 #ifndef CONFIG_NO_BOOTMEM
314 static void __init
__free(bootmem_data_t
*bdata
,
315 unsigned long sidx
, unsigned long eidx
)
319 bdebug("nid=%td start=%lx end=%lx\n", bdata
- bootmem_node_data
,
320 sidx
+ bdata
->node_min_pfn
,
321 eidx
+ bdata
->node_min_pfn
);
323 if (bdata
->hint_idx
> sidx
)
324 bdata
->hint_idx
= sidx
;
326 for (idx
= sidx
; idx
< eidx
; idx
++)
327 if (!test_and_clear_bit(idx
, bdata
->node_bootmem_map
))
331 static int __init
__reserve(bootmem_data_t
*bdata
, unsigned long sidx
,
332 unsigned long eidx
, int flags
)
335 int exclusive
= flags
& BOOTMEM_EXCLUSIVE
;
337 bdebug("nid=%td start=%lx end=%lx flags=%x\n",
338 bdata
- bootmem_node_data
,
339 sidx
+ bdata
->node_min_pfn
,
340 eidx
+ bdata
->node_min_pfn
,
343 for (idx
= sidx
; idx
< eidx
; idx
++)
344 if (test_and_set_bit(idx
, bdata
->node_bootmem_map
)) {
346 __free(bdata
, sidx
, idx
);
349 bdebug("silent double reserve of PFN %lx\n",
350 idx
+ bdata
->node_min_pfn
);
355 static int __init
mark_bootmem_node(bootmem_data_t
*bdata
,
356 unsigned long start
, unsigned long end
,
357 int reserve
, int flags
)
359 unsigned long sidx
, eidx
;
361 bdebug("nid=%td start=%lx end=%lx reserve=%d flags=%x\n",
362 bdata
- bootmem_node_data
, start
, end
, reserve
, flags
);
364 BUG_ON(start
< bdata
->node_min_pfn
);
365 BUG_ON(end
> bdata
->node_low_pfn
);
367 sidx
= start
- bdata
->node_min_pfn
;
368 eidx
= end
- bdata
->node_min_pfn
;
371 return __reserve(bdata
, sidx
, eidx
, flags
);
373 __free(bdata
, sidx
, eidx
);
377 static int __init
mark_bootmem(unsigned long start
, unsigned long end
,
378 int reserve
, int flags
)
381 bootmem_data_t
*bdata
;
384 list_for_each_entry(bdata
, &bdata_list
, list
) {
388 if (pos
< bdata
->node_min_pfn
||
389 pos
>= bdata
->node_low_pfn
) {
390 BUG_ON(pos
!= start
);
394 max
= min(bdata
->node_low_pfn
, end
);
396 err
= mark_bootmem_node(bdata
, pos
, max
, reserve
, flags
);
397 if (reserve
&& err
) {
398 mark_bootmem(start
, pos
, 0, 0);
404 pos
= bdata
->node_low_pfn
;
411 * free_bootmem_node - mark a page range as usable
412 * @pgdat: node the range resides on
413 * @physaddr: starting address of the range
414 * @size: size of the range in bytes
416 * Partial pages will be considered reserved and left as they are.
418 * The range must reside completely on the specified node.
420 void __init
free_bootmem_node(pg_data_t
*pgdat
, unsigned long physaddr
,
423 #ifdef CONFIG_NO_BOOTMEM
424 free_early(physaddr
, physaddr
+ size
);
426 unsigned long start
, end
;
428 kmemleak_free_part(__va(physaddr
), size
);
430 start
= PFN_UP(physaddr
);
431 end
= PFN_DOWN(physaddr
+ size
);
433 mark_bootmem_node(pgdat
->bdata
, start
, end
, 0, 0);
438 * free_bootmem - mark a page range as usable
439 * @addr: starting address of the range
440 * @size: size of the range in bytes
442 * Partial pages will be considered reserved and left as they are.
444 * The range must be contiguous but may span node boundaries.
446 void __init
free_bootmem(unsigned long addr
, unsigned long size
)
448 #ifdef CONFIG_NO_BOOTMEM
449 free_early(addr
, addr
+ size
);
451 unsigned long start
, end
;
453 kmemleak_free_part(__va(addr
), size
);
455 start
= PFN_UP(addr
);
456 end
= PFN_DOWN(addr
+ size
);
458 mark_bootmem(start
, end
, 0, 0);
463 * reserve_bootmem_node - mark a page range as reserved
464 * @pgdat: node the range resides on
465 * @physaddr: starting address of the range
466 * @size: size of the range in bytes
467 * @flags: reservation flags (see linux/bootmem.h)
469 * Partial pages will be reserved.
471 * The range must reside completely on the specified node.
473 int __init
reserve_bootmem_node(pg_data_t
*pgdat
, unsigned long physaddr
,
474 unsigned long size
, int flags
)
476 #ifdef CONFIG_NO_BOOTMEM
480 unsigned long start
, end
;
482 start
= PFN_DOWN(physaddr
);
483 end
= PFN_UP(physaddr
+ size
);
485 return mark_bootmem_node(pgdat
->bdata
, start
, end
, 1, flags
);
490 * reserve_bootmem - mark a page range as usable
491 * @addr: starting address of the range
492 * @size: size of the range in bytes
493 * @flags: reservation flags (see linux/bootmem.h)
495 * Partial pages will be reserved.
497 * The range must be contiguous but may span node boundaries.
499 int __init
reserve_bootmem(unsigned long addr
, unsigned long size
,
502 #ifdef CONFIG_NO_BOOTMEM
506 unsigned long start
, end
;
508 start
= PFN_DOWN(addr
);
509 end
= PFN_UP(addr
+ size
);
511 return mark_bootmem(start
, end
, 1, flags
);
515 #ifndef CONFIG_NO_BOOTMEM
516 static unsigned long __init
align_idx(struct bootmem_data
*bdata
,
517 unsigned long idx
, unsigned long step
)
519 unsigned long base
= bdata
->node_min_pfn
;
522 * Align the index with respect to the node start so that the
523 * combination of both satisfies the requested alignment.
526 return ALIGN(base
+ idx
, step
) - base
;
529 static unsigned long __init
align_off(struct bootmem_data
*bdata
,
530 unsigned long off
, unsigned long align
)
532 unsigned long base
= PFN_PHYS(bdata
->node_min_pfn
);
534 /* Same as align_idx for byte offsets */
536 return ALIGN(base
+ off
, align
) - base
;
539 static void * __init
alloc_bootmem_core(struct bootmem_data
*bdata
,
540 unsigned long size
, unsigned long align
,
541 unsigned long goal
, unsigned long limit
)
543 unsigned long fallback
= 0;
544 unsigned long min
, max
, start
, sidx
, midx
, step
;
546 bdebug("nid=%td size=%lx [%lu pages] align=%lx goal=%lx limit=%lx\n",
547 bdata
- bootmem_node_data
, size
, PAGE_ALIGN(size
) >> PAGE_SHIFT
,
551 BUG_ON(align
& (align
- 1));
552 BUG_ON(limit
&& goal
+ size
> limit
);
554 if (!bdata
->node_bootmem_map
)
557 min
= bdata
->node_min_pfn
;
558 max
= bdata
->node_low_pfn
;
561 limit
>>= PAGE_SHIFT
;
563 if (limit
&& max
> limit
)
568 step
= max(align
>> PAGE_SHIFT
, 1UL);
570 if (goal
&& min
< goal
&& goal
< max
)
571 start
= ALIGN(goal
, step
);
573 start
= ALIGN(min
, step
);
575 sidx
= start
- bdata
->node_min_pfn
;
576 midx
= max
- bdata
->node_min_pfn
;
578 if (bdata
->hint_idx
> sidx
) {
580 * Handle the valid case of sidx being zero and still
581 * catch the fallback below.
584 sidx
= align_idx(bdata
, bdata
->hint_idx
, step
);
590 unsigned long eidx
, i
, start_off
, end_off
;
592 sidx
= find_next_zero_bit(bdata
->node_bootmem_map
, midx
, sidx
);
593 sidx
= align_idx(bdata
, sidx
, step
);
594 eidx
= sidx
+ PFN_UP(size
);
596 if (sidx
>= midx
|| eidx
> midx
)
599 for (i
= sidx
; i
< eidx
; i
++)
600 if (test_bit(i
, bdata
->node_bootmem_map
)) {
601 sidx
= align_idx(bdata
, i
, step
);
607 if (bdata
->last_end_off
& (PAGE_SIZE
- 1) &&
608 PFN_DOWN(bdata
->last_end_off
) + 1 == sidx
)
609 start_off
= align_off(bdata
, bdata
->last_end_off
, align
);
611 start_off
= PFN_PHYS(sidx
);
613 merge
= PFN_DOWN(start_off
) < sidx
;
614 end_off
= start_off
+ size
;
616 bdata
->last_end_off
= end_off
;
617 bdata
->hint_idx
= PFN_UP(end_off
);
620 * Reserve the area now:
622 if (__reserve(bdata
, PFN_DOWN(start_off
) + merge
,
623 PFN_UP(end_off
), BOOTMEM_EXCLUSIVE
))
626 region
= phys_to_virt(PFN_PHYS(bdata
->node_min_pfn
) +
628 memset(region
, 0, size
);
630 * The min_count is set to 0 so that bootmem allocated blocks
631 * are never reported as leaks.
633 kmemleak_alloc(region
, size
, 0, 0);
638 sidx
= align_idx(bdata
, fallback
- 1, step
);
646 static void * __init
alloc_arch_preferred_bootmem(bootmem_data_t
*bdata
,
647 unsigned long size
, unsigned long align
,
648 unsigned long goal
, unsigned long limit
)
650 if (WARN_ON_ONCE(slab_is_available()))
651 return kzalloc(size
, GFP_NOWAIT
);
653 #ifdef CONFIG_HAVE_ARCH_BOOTMEM
655 bootmem_data_t
*p_bdata
;
657 p_bdata
= bootmem_arch_preferred_node(bdata
, size
, align
,
660 return alloc_bootmem_core(p_bdata
, size
, align
,
668 static void * __init
___alloc_bootmem_nopanic(unsigned long size
,
673 #ifdef CONFIG_NO_BOOTMEM
676 if (WARN_ON_ONCE(slab_is_available()))
677 return kzalloc(size
, GFP_NOWAIT
);
681 ptr
= __alloc_memory_core_early(MAX_NUMNODES
, size
, align
, goal
, limit
);
693 bootmem_data_t
*bdata
;
697 region
= alloc_arch_preferred_bootmem(NULL
, size
, align
, goal
, limit
);
701 list_for_each_entry(bdata
, &bdata_list
, list
) {
702 if (goal
&& bdata
->node_low_pfn
<= PFN_DOWN(goal
))
704 if (limit
&& bdata
->node_min_pfn
>= PFN_DOWN(limit
))
707 region
= alloc_bootmem_core(bdata
, size
, align
, goal
, limit
);
722 * __alloc_bootmem_nopanic - allocate boot memory without panicking
723 * @size: size of the request in bytes
724 * @align: alignment of the region
725 * @goal: preferred starting address of the region
727 * The goal is dropped if it can not be satisfied and the allocation will
728 * fall back to memory below @goal.
730 * Allocation may happen on any node in the system.
732 * Returns NULL on failure.
734 void * __init
__alloc_bootmem_nopanic(unsigned long size
, unsigned long align
,
737 unsigned long limit
= 0;
739 #ifdef CONFIG_NO_BOOTMEM
743 return ___alloc_bootmem_nopanic(size
, align
, goal
, limit
);
746 static void * __init
___alloc_bootmem(unsigned long size
, unsigned long align
,
747 unsigned long goal
, unsigned long limit
)
749 void *mem
= ___alloc_bootmem_nopanic(size
, align
, goal
, limit
);
754 * Whoops, we cannot satisfy the allocation request.
756 printk(KERN_ALERT
"bootmem alloc of %lu bytes failed!\n", size
);
757 panic("Out of memory");
762 * __alloc_bootmem - allocate boot memory
763 * @size: size of the request in bytes
764 * @align: alignment of the region
765 * @goal: preferred starting address of the region
767 * The goal is dropped if it can not be satisfied and the allocation will
768 * fall back to memory below @goal.
770 * Allocation may happen on any node in the system.
772 * The function panics if the request can not be satisfied.
774 void * __init
__alloc_bootmem(unsigned long size
, unsigned long align
,
777 unsigned long limit
= 0;
779 #ifdef CONFIG_NO_BOOTMEM
783 return ___alloc_bootmem(size
, align
, goal
, limit
);
786 #ifndef CONFIG_NO_BOOTMEM
787 static void * __init
___alloc_bootmem_node(bootmem_data_t
*bdata
,
788 unsigned long size
, unsigned long align
,
789 unsigned long goal
, unsigned long limit
)
793 ptr
= alloc_arch_preferred_bootmem(bdata
, size
, align
, goal
, limit
);
797 ptr
= alloc_bootmem_core(bdata
, size
, align
, goal
, limit
);
801 return ___alloc_bootmem(size
, align
, goal
, limit
);
806 * __alloc_bootmem_node - allocate boot memory from a specific node
807 * @pgdat: node to allocate from
808 * @size: size of the request in bytes
809 * @align: alignment of the region
810 * @goal: preferred starting address of the region
812 * The goal is dropped if it can not be satisfied and the allocation will
813 * fall back to memory below @goal.
815 * Allocation may fall back to any node in the system if the specified node
816 * can not hold the requested memory.
818 * The function panics if the request can not be satisfied.
820 void * __init
__alloc_bootmem_node(pg_data_t
*pgdat
, unsigned long size
,
821 unsigned long align
, unsigned long goal
)
823 if (WARN_ON_ONCE(slab_is_available()))
824 return kzalloc_node(size
, GFP_NOWAIT
, pgdat
->node_id
);
826 #ifdef CONFIG_NO_BOOTMEM
827 return __alloc_memory_core_early(pgdat
->node_id
, size
, align
,
830 return ___alloc_bootmem_node(pgdat
->bdata
, size
, align
, goal
, 0);
834 void * __init
__alloc_bootmem_node_high(pg_data_t
*pgdat
, unsigned long size
,
835 unsigned long align
, unsigned long goal
)
838 unsigned long end_pfn
;
840 if (WARN_ON_ONCE(slab_is_available()))
841 return kzalloc_node(size
, GFP_NOWAIT
, pgdat
->node_id
);
843 /* update goal according ...MAX_DMA32_PFN */
844 end_pfn
= pgdat
->node_start_pfn
+ pgdat
->node_spanned_pages
;
846 if (end_pfn
> MAX_DMA32_PFN
+ (128 >> (20 - PAGE_SHIFT
)) &&
847 (goal
>> PAGE_SHIFT
) < MAX_DMA32_PFN
) {
849 unsigned long new_goal
;
851 new_goal
= MAX_DMA32_PFN
<< PAGE_SHIFT
;
852 #ifdef CONFIG_NO_BOOTMEM
853 ptr
= __alloc_memory_core_early(pgdat
->node_id
, size
, align
,
856 ptr
= alloc_bootmem_core(pgdat
->bdata
, size
, align
,
864 return __alloc_bootmem_node(pgdat
, size
, align
, goal
);
868 #ifdef CONFIG_SPARSEMEM
870 * alloc_bootmem_section - allocate boot memory from a specific section
871 * @size: size of the request in bytes
872 * @section_nr: sparse map section to allocate from
874 * Return NULL on failure.
876 void * __init
alloc_bootmem_section(unsigned long size
,
877 unsigned long section_nr
)
879 #ifdef CONFIG_NO_BOOTMEM
880 unsigned long pfn
, goal
, limit
;
882 pfn
= section_nr_to_pfn(section_nr
);
883 goal
= pfn
<< PAGE_SHIFT
;
884 limit
= section_nr_to_pfn(section_nr
+ 1) << PAGE_SHIFT
;
886 return __alloc_memory_core_early(early_pfn_to_nid(pfn
), size
,
887 SMP_CACHE_BYTES
, goal
, limit
);
889 bootmem_data_t
*bdata
;
890 unsigned long pfn
, goal
, limit
;
892 pfn
= section_nr_to_pfn(section_nr
);
893 goal
= pfn
<< PAGE_SHIFT
;
894 limit
= section_nr_to_pfn(section_nr
+ 1) << PAGE_SHIFT
;
895 bdata
= &bootmem_node_data
[early_pfn_to_nid(pfn
)];
897 return alloc_bootmem_core(bdata
, size
, SMP_CACHE_BYTES
, goal
, limit
);
902 void * __init
__alloc_bootmem_node_nopanic(pg_data_t
*pgdat
, unsigned long size
,
903 unsigned long align
, unsigned long goal
)
907 if (WARN_ON_ONCE(slab_is_available()))
908 return kzalloc_node(size
, GFP_NOWAIT
, pgdat
->node_id
);
910 #ifdef CONFIG_NO_BOOTMEM
911 ptr
= __alloc_memory_core_early(pgdat
->node_id
, size
, align
,
914 ptr
= alloc_arch_preferred_bootmem(pgdat
->bdata
, size
, align
, goal
, 0);
918 ptr
= alloc_bootmem_core(pgdat
->bdata
, size
, align
, goal
, 0);
923 return __alloc_bootmem_nopanic(size
, align
, goal
);
926 #ifndef ARCH_LOW_ADDRESS_LIMIT
927 #define ARCH_LOW_ADDRESS_LIMIT 0xffffffffUL
931 * __alloc_bootmem_low - allocate low boot memory
932 * @size: size of the request in bytes
933 * @align: alignment of the region
934 * @goal: preferred starting address of the region
936 * The goal is dropped if it can not be satisfied and the allocation will
937 * fall back to memory below @goal.
939 * Allocation may happen on any node in the system.
941 * The function panics if the request can not be satisfied.
943 void * __init
__alloc_bootmem_low(unsigned long size
, unsigned long align
,
946 return ___alloc_bootmem(size
, align
, goal
, ARCH_LOW_ADDRESS_LIMIT
);
950 * __alloc_bootmem_low_node - allocate low boot memory from a specific node
951 * @pgdat: node to allocate from
952 * @size: size of the request in bytes
953 * @align: alignment of the region
954 * @goal: preferred starting address of the region
956 * The goal is dropped if it can not be satisfied and the allocation will
957 * fall back to memory below @goal.
959 * Allocation may fall back to any node in the system if the specified node
960 * can not hold the requested memory.
962 * The function panics if the request can not be satisfied.
964 void * __init
__alloc_bootmem_low_node(pg_data_t
*pgdat
, unsigned long size
,
965 unsigned long align
, unsigned long goal
)
967 if (WARN_ON_ONCE(slab_is_available()))
968 return kzalloc_node(size
, GFP_NOWAIT
, pgdat
->node_id
);
970 #ifdef CONFIG_NO_BOOTMEM
971 return __alloc_memory_core_early(pgdat
->node_id
, size
, align
,
972 goal
, ARCH_LOW_ADDRESS_LIMIT
);
974 return ___alloc_bootmem_node(pgdat
->bdata
, size
, align
,
975 goal
, ARCH_LOW_ADDRESS_LIMIT
);