2 * sparse memory mappings.
5 #include <linux/slab.h>
6 #include <linux/mmzone.h>
7 #include <linux/bootmem.h>
8 #include <linux/compiler.h>
9 #include <linux/highmem.h>
10 #include <linux/export.h>
11 #include <linux/spinlock.h>
12 #include <linux/vmalloc.h>
16 #include <asm/pgalloc.h>
17 #include <asm/pgtable.h>
20 * Permanent SPARSEMEM data:
22 * 1) mem_section - memory sections, mem_map's for valid memory
24 #ifdef CONFIG_SPARSEMEM_EXTREME
25 struct mem_section
**mem_section
;
27 struct mem_section mem_section
[NR_SECTION_ROOTS
][SECTIONS_PER_ROOT
]
28 ____cacheline_internodealigned_in_smp
;
30 EXPORT_SYMBOL(mem_section
);
32 #ifdef NODE_NOT_IN_PAGE_FLAGS
34 * If we did not store the node number in the page then we have to
35 * do a lookup in the section_to_node_table in order to find which
36 * node the page belongs to.
38 #if MAX_NUMNODES <= 256
39 static u8 section_to_node_table
[NR_MEM_SECTIONS
] __cacheline_aligned
;
41 static u16 section_to_node_table
[NR_MEM_SECTIONS
] __cacheline_aligned
;
44 int page_to_nid(const struct page
*page
)
46 return section_to_node_table
[page_to_section(page
)];
48 EXPORT_SYMBOL(page_to_nid
);
50 static void set_section_nid(unsigned long section_nr
, int nid
)
52 section_to_node_table
[section_nr
] = nid
;
54 #else /* !NODE_NOT_IN_PAGE_FLAGS */
55 static inline void set_section_nid(unsigned long section_nr
, int nid
)
60 #ifdef CONFIG_SPARSEMEM_EXTREME
61 static noinline
struct mem_section __ref
*sparse_index_alloc(int nid
)
63 struct mem_section
*section
= NULL
;
64 unsigned long array_size
= SECTIONS_PER_ROOT
*
65 sizeof(struct mem_section
);
67 if (slab_is_available())
68 section
= kzalloc_node(array_size
, GFP_KERNEL
, nid
);
70 section
= memblock_virt_alloc_node(array_size
, nid
);
75 static int __meminit
sparse_index_init(unsigned long section_nr
, int nid
)
77 unsigned long root
= SECTION_NR_TO_ROOT(section_nr
);
78 struct mem_section
*section
;
80 if (mem_section
[root
])
83 section
= sparse_index_alloc(nid
);
87 mem_section
[root
] = section
;
91 #else /* !SPARSEMEM_EXTREME */
92 static inline int sparse_index_init(unsigned long section_nr
, int nid
)
98 #ifdef CONFIG_SPARSEMEM_EXTREME
99 int __section_nr(struct mem_section
* ms
)
101 unsigned long root_nr
;
102 struct mem_section
*root
= NULL
;
104 for (root_nr
= 0; root_nr
< NR_SECTION_ROOTS
; root_nr
++) {
105 root
= __nr_to_section(root_nr
* SECTIONS_PER_ROOT
);
109 if ((ms
>= root
) && (ms
< (root
+ SECTIONS_PER_ROOT
)))
115 return (root_nr
* SECTIONS_PER_ROOT
) + (ms
- root
);
118 int __section_nr(struct mem_section
* ms
)
120 return (int)(ms
- mem_section
[0]);
125 * During early boot, before section_mem_map is used for an actual
126 * mem_map, we use section_mem_map to store the section's NUMA
127 * node. This keeps us from having to use another data structure. The
128 * node information is cleared just before we store the real mem_map.
130 static inline unsigned long sparse_encode_early_nid(int nid
)
132 return (nid
<< SECTION_NID_SHIFT
);
135 static inline int sparse_early_nid(struct mem_section
*section
)
137 return (section
->section_mem_map
>> SECTION_NID_SHIFT
);
140 /* Validate the physical addressing limitations of the model */
141 void __meminit
mminit_validate_memmodel_limits(unsigned long *start_pfn
,
142 unsigned long *end_pfn
)
144 unsigned long max_sparsemem_pfn
= 1UL << (MAX_PHYSMEM_BITS
-PAGE_SHIFT
);
147 * Sanity checks - do not allow an architecture to pass
148 * in larger pfns than the maximum scope of sparsemem:
150 if (*start_pfn
> max_sparsemem_pfn
) {
151 mminit_dprintk(MMINIT_WARNING
, "pfnvalidation",
152 "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
153 *start_pfn
, *end_pfn
, max_sparsemem_pfn
);
155 *start_pfn
= max_sparsemem_pfn
;
156 *end_pfn
= max_sparsemem_pfn
;
157 } else if (*end_pfn
> max_sparsemem_pfn
) {
158 mminit_dprintk(MMINIT_WARNING
, "pfnvalidation",
159 "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
160 *start_pfn
, *end_pfn
, max_sparsemem_pfn
);
162 *end_pfn
= max_sparsemem_pfn
;
167 * There are a number of times that we loop over NR_MEM_SECTIONS,
168 * looking for section_present() on each. But, when we have very
169 * large physical address spaces, NR_MEM_SECTIONS can also be
170 * very large which makes the loops quite long.
172 * Keeping track of this gives us an easy way to break out of
175 int __highest_present_section_nr
;
176 static void section_mark_present(struct mem_section
*ms
)
178 int section_nr
= __section_nr(ms
);
180 if (section_nr
> __highest_present_section_nr
)
181 __highest_present_section_nr
= section_nr
;
183 ms
->section_mem_map
|= SECTION_MARKED_PRESENT
;
186 static inline int next_present_section_nr(int section_nr
)
190 if (present_section_nr(section_nr
))
192 } while ((section_nr
< NR_MEM_SECTIONS
) &&
193 (section_nr
<= __highest_present_section_nr
));
197 #define for_each_present_section_nr(start, section_nr) \
198 for (section_nr = next_present_section_nr(start-1); \
199 ((section_nr >= 0) && \
200 (section_nr < NR_MEM_SECTIONS) && \
201 (section_nr <= __highest_present_section_nr)); \
202 section_nr = next_present_section_nr(section_nr))
204 /* Record a memory area against a node. */
205 void __init
memory_present(int nid
, unsigned long start
, unsigned long end
)
209 #ifdef CONFIG_SPARSEMEM_EXTREME
210 if (unlikely(!mem_section
)) {
211 unsigned long size
, align
;
213 size
= sizeof(struct mem_section
) * NR_SECTION_ROOTS
;
214 align
= 1 << (INTERNODE_CACHE_SHIFT
);
215 mem_section
= memblock_virt_alloc(size
, align
);
219 start
&= PAGE_SECTION_MASK
;
220 mminit_validate_memmodel_limits(&start
, &end
);
221 for (pfn
= start
; pfn
< end
; pfn
+= PAGES_PER_SECTION
) {
222 unsigned long section
= pfn_to_section_nr(pfn
);
223 struct mem_section
*ms
;
225 sparse_index_init(section
, nid
);
226 set_section_nid(section
, nid
);
228 ms
= __nr_to_section(section
);
229 if (!ms
->section_mem_map
) {
230 ms
->section_mem_map
= sparse_encode_early_nid(nid
) |
232 section_mark_present(ms
);
238 * Only used by the i386 NUMA architecures, but relatively
241 unsigned long __init
node_memmap_size_bytes(int nid
, unsigned long start_pfn
,
242 unsigned long end_pfn
)
245 unsigned long nr_pages
= 0;
247 mminit_validate_memmodel_limits(&start_pfn
, &end_pfn
);
248 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= PAGES_PER_SECTION
) {
249 if (nid
!= early_pfn_to_nid(pfn
))
252 if (pfn_present(pfn
))
253 nr_pages
+= PAGES_PER_SECTION
;
256 return nr_pages
* sizeof(struct page
);
260 * Subtle, we encode the real pfn into the mem_map such that
261 * the identity pfn - section_mem_map will return the actual
262 * physical page frame number.
264 static unsigned long sparse_encode_mem_map(struct page
*mem_map
, unsigned long pnum
)
266 return (unsigned long)(mem_map
- (section_nr_to_pfn(pnum
)));
270 * Decode mem_map from the coded memmap
272 struct page
*sparse_decode_mem_map(unsigned long coded_mem_map
, unsigned long pnum
)
274 /* mask off the extra low bits of information */
275 coded_mem_map
&= SECTION_MAP_MASK
;
276 return ((struct page
*)coded_mem_map
) + section_nr_to_pfn(pnum
);
279 static int __meminit
sparse_init_one_section(struct mem_section
*ms
,
280 unsigned long pnum
, struct page
*mem_map
,
281 unsigned long *pageblock_bitmap
)
283 if (!present_section(ms
))
286 ms
->section_mem_map
&= ~SECTION_MAP_MASK
;
287 ms
->section_mem_map
|= sparse_encode_mem_map(mem_map
, pnum
) |
289 ms
->pageblock_flags
= pageblock_bitmap
;
294 unsigned long usemap_size(void)
296 return BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS
) * sizeof(unsigned long);
299 #ifdef CONFIG_MEMORY_HOTPLUG
300 static unsigned long *__kmalloc_section_usemap(void)
302 return kmalloc(usemap_size(), GFP_KERNEL
);
304 #endif /* CONFIG_MEMORY_HOTPLUG */
306 #ifdef CONFIG_MEMORY_HOTREMOVE
307 static unsigned long * __init
308 sparse_early_usemaps_alloc_pgdat_section(struct pglist_data
*pgdat
,
311 unsigned long goal
, limit
;
315 * A page may contain usemaps for other sections preventing the
316 * page being freed and making a section unremovable while
317 * other sections referencing the usemap remain active. Similarly,
318 * a pgdat can prevent a section being removed. If section A
319 * contains a pgdat and section B contains the usemap, both
320 * sections become inter-dependent. This allocates usemaps
321 * from the same section as the pgdat where possible to avoid
324 goal
= __pa(pgdat
) & (PAGE_SECTION_MASK
<< PAGE_SHIFT
);
325 limit
= goal
+ (1UL << PA_SECTION_SHIFT
);
326 nid
= early_pfn_to_nid(goal
>> PAGE_SHIFT
);
328 p
= memblock_virt_alloc_try_nid_nopanic(size
,
329 SMP_CACHE_BYTES
, goal
, limit
,
338 static void __init
check_usemap_section_nr(int nid
, unsigned long *usemap
)
340 unsigned long usemap_snr
, pgdat_snr
;
341 static unsigned long old_usemap_snr
;
342 static unsigned long old_pgdat_snr
;
343 struct pglist_data
*pgdat
= NODE_DATA(nid
);
347 if (!old_usemap_snr
) {
348 old_usemap_snr
= NR_MEM_SECTIONS
;
349 old_pgdat_snr
= NR_MEM_SECTIONS
;
352 usemap_snr
= pfn_to_section_nr(__pa(usemap
) >> PAGE_SHIFT
);
353 pgdat_snr
= pfn_to_section_nr(__pa(pgdat
) >> PAGE_SHIFT
);
354 if (usemap_snr
== pgdat_snr
)
357 if (old_usemap_snr
== usemap_snr
&& old_pgdat_snr
== pgdat_snr
)
358 /* skip redundant message */
361 old_usemap_snr
= usemap_snr
;
362 old_pgdat_snr
= pgdat_snr
;
364 usemap_nid
= sparse_early_nid(__nr_to_section(usemap_snr
));
365 if (usemap_nid
!= nid
) {
366 pr_info("node %d must be removed before remove section %ld\n",
371 * There is a circular dependency.
372 * Some platforms allow un-removable section because they will just
373 * gather other removable sections for dynamic partitioning.
374 * Just notify un-removable section's number here.
376 pr_info("Section %ld and %ld (node %d) have a circular dependency on usemap and pgdat allocations\n",
377 usemap_snr
, pgdat_snr
, nid
);
380 static unsigned long * __init
381 sparse_early_usemaps_alloc_pgdat_section(struct pglist_data
*pgdat
,
384 return memblock_virt_alloc_node_nopanic(size
, pgdat
->node_id
);
387 static void __init
check_usemap_section_nr(int nid
, unsigned long *usemap
)
390 #endif /* CONFIG_MEMORY_HOTREMOVE */
392 static void __init
sparse_early_usemaps_alloc_node(void *data
,
393 unsigned long pnum_begin
,
394 unsigned long pnum_end
,
395 unsigned long usemap_count
, int nodeid
)
399 unsigned long **usemap_map
= (unsigned long **)data
;
400 int size
= usemap_size();
402 usemap
= sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nodeid
),
403 size
* usemap_count
);
405 pr_warn("%s: allocation failed\n", __func__
);
409 for (pnum
= pnum_begin
; pnum
< pnum_end
; pnum
++) {
410 if (!present_section_nr(pnum
))
412 usemap_map
[pnum
] = usemap
;
414 check_usemap_section_nr(nodeid
, usemap_map
[pnum
]);
418 #ifndef CONFIG_SPARSEMEM_VMEMMAP
419 struct page __init
*sparse_mem_map_populate(unsigned long pnum
, int nid
)
424 map
= alloc_remap(nid
, sizeof(struct page
) * PAGES_PER_SECTION
);
428 size
= PAGE_ALIGN(sizeof(struct page
) * PAGES_PER_SECTION
);
429 map
= memblock_virt_alloc_try_nid(size
,
430 PAGE_SIZE
, __pa(MAX_DMA_ADDRESS
),
431 BOOTMEM_ALLOC_ACCESSIBLE
, nid
);
434 void __init
sparse_mem_maps_populate_node(struct page
**map_map
,
435 unsigned long pnum_begin
,
436 unsigned long pnum_end
,
437 unsigned long map_count
, int nodeid
)
441 unsigned long size
= sizeof(struct page
) * PAGES_PER_SECTION
;
443 map
= alloc_remap(nodeid
, size
* map_count
);
445 for (pnum
= pnum_begin
; pnum
< pnum_end
; pnum
++) {
446 if (!present_section_nr(pnum
))
454 size
= PAGE_ALIGN(size
);
455 map
= memblock_virt_alloc_try_nid(size
* map_count
,
456 PAGE_SIZE
, __pa(MAX_DMA_ADDRESS
),
457 BOOTMEM_ALLOC_ACCESSIBLE
, nodeid
);
459 for (pnum
= pnum_begin
; pnum
< pnum_end
; pnum
++) {
460 if (!present_section_nr(pnum
))
469 for (pnum
= pnum_begin
; pnum
< pnum_end
; pnum
++) {
470 struct mem_section
*ms
;
472 if (!present_section_nr(pnum
))
474 map_map
[pnum
] = sparse_mem_map_populate(pnum
, nodeid
);
477 ms
= __nr_to_section(pnum
);
478 pr_err("%s: sparsemem memory map backing failed some memory will not be available\n",
480 ms
->section_mem_map
= 0;
483 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */
485 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
486 static void __init
sparse_early_mem_maps_alloc_node(void *data
,
487 unsigned long pnum_begin
,
488 unsigned long pnum_end
,
489 unsigned long map_count
, int nodeid
)
491 struct page
**map_map
= (struct page
**)data
;
492 sparse_mem_maps_populate_node(map_map
, pnum_begin
, pnum_end
,
496 static struct page __init
*sparse_early_mem_map_alloc(unsigned long pnum
)
499 struct mem_section
*ms
= __nr_to_section(pnum
);
500 int nid
= sparse_early_nid(ms
);
502 map
= sparse_mem_map_populate(pnum
, nid
);
506 pr_err("%s: sparsemem memory map backing failed some memory will not be available\n",
508 ms
->section_mem_map
= 0;
513 void __weak __meminit
vmemmap_populate_print_last(void)
518 * alloc_usemap_and_memmap - memory alloction for pageblock flags and vmemmap
519 * @map: usemap_map for pageblock flags or mmap_map for vmemmap
521 static void __init
alloc_usemap_and_memmap(void (*alloc_func
)
522 (void *, unsigned long, unsigned long,
523 unsigned long, int), void *data
)
526 unsigned long map_count
;
527 int nodeid_begin
= 0;
528 unsigned long pnum_begin
= 0;
530 for_each_present_section_nr(0, pnum
) {
531 struct mem_section
*ms
;
533 ms
= __nr_to_section(pnum
);
534 nodeid_begin
= sparse_early_nid(ms
);
539 for_each_present_section_nr(pnum_begin
+ 1, pnum
) {
540 struct mem_section
*ms
;
543 ms
= __nr_to_section(pnum
);
544 nodeid
= sparse_early_nid(ms
);
545 if (nodeid
== nodeid_begin
) {
549 /* ok, we need to take cake of from pnum_begin to pnum - 1*/
550 alloc_func(data
, pnum_begin
, pnum
,
551 map_count
, nodeid_begin
);
552 /* new start, update count etc*/
553 nodeid_begin
= nodeid
;
558 alloc_func(data
, pnum_begin
, NR_MEM_SECTIONS
,
559 map_count
, nodeid_begin
);
563 * Allocate the accumulated non-linear sections, allocate a mem_map
564 * for each and record the physical to section mapping.
566 void __init
sparse_init(void)
570 unsigned long *usemap
;
571 unsigned long **usemap_map
;
573 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
575 struct page
**map_map
;
578 /* see include/linux/mmzone.h 'struct mem_section' definition */
579 BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section
)));
581 /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */
582 set_pageblock_order();
585 * map is using big page (aka 2M in x86 64 bit)
586 * usemap is less one page (aka 24 bytes)
587 * so alloc 2M (with 2M align) and 24 bytes in turn will
588 * make next 2M slip to one more 2M later.
589 * then in big system, the memory will have a lot of holes...
590 * here try to allocate 2M pages continuously.
592 * powerpc need to call sparse_init_one_section right after each
593 * sparse_early_mem_map_alloc, so allocate usemap_map at first.
595 size
= sizeof(unsigned long *) * NR_MEM_SECTIONS
;
596 usemap_map
= memblock_virt_alloc(size
, 0);
598 panic("can not allocate usemap_map\n");
599 alloc_usemap_and_memmap(sparse_early_usemaps_alloc_node
,
602 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
603 size2
= sizeof(struct page
*) * NR_MEM_SECTIONS
;
604 map_map
= memblock_virt_alloc(size2
, 0);
606 panic("can not allocate map_map\n");
607 alloc_usemap_and_memmap(sparse_early_mem_maps_alloc_node
,
611 for_each_present_section_nr(0, pnum
) {
612 usemap
= usemap_map
[pnum
];
616 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
619 map
= sparse_early_mem_map_alloc(pnum
);
624 sparse_init_one_section(__nr_to_section(pnum
), pnum
, map
,
628 vmemmap_populate_print_last();
630 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
631 memblock_free_early(__pa(map_map
), size2
);
633 memblock_free_early(__pa(usemap_map
), size
);
636 #ifdef CONFIG_MEMORY_HOTPLUG
638 /* Mark all memory sections within the pfn range as online */
639 void online_mem_sections(unsigned long start_pfn
, unsigned long end_pfn
)
643 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= PAGES_PER_SECTION
) {
644 unsigned long section_nr
= pfn_to_section_nr(pfn
);
645 struct mem_section
*ms
;
647 /* onlining code should never touch invalid ranges */
648 if (WARN_ON(!valid_section_nr(section_nr
)))
651 ms
= __nr_to_section(section_nr
);
652 ms
->section_mem_map
|= SECTION_IS_ONLINE
;
656 #ifdef CONFIG_MEMORY_HOTREMOVE
657 /* Mark all memory sections within the pfn range as online */
658 void offline_mem_sections(unsigned long start_pfn
, unsigned long end_pfn
)
662 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= PAGES_PER_SECTION
) {
663 unsigned long section_nr
= pfn_to_section_nr(start_pfn
);
664 struct mem_section
*ms
;
667 * TODO this needs some double checking. Offlining code makes
668 * sure to check pfn_valid but those checks might be just bogus
670 if (WARN_ON(!valid_section_nr(section_nr
)))
673 ms
= __nr_to_section(section_nr
);
674 ms
->section_mem_map
&= ~SECTION_IS_ONLINE
;
679 #ifdef CONFIG_SPARSEMEM_VMEMMAP
680 static inline struct page
*kmalloc_section_memmap(unsigned long pnum
, int nid
)
682 /* This will make the necessary allocations eventually. */
683 return sparse_mem_map_populate(pnum
, nid
);
685 static void __kfree_section_memmap(struct page
*memmap
)
687 unsigned long start
= (unsigned long)memmap
;
688 unsigned long end
= (unsigned long)(memmap
+ PAGES_PER_SECTION
);
690 vmemmap_free(start
, end
);
692 #ifdef CONFIG_MEMORY_HOTREMOVE
693 static void free_map_bootmem(struct page
*memmap
)
695 unsigned long start
= (unsigned long)memmap
;
696 unsigned long end
= (unsigned long)(memmap
+ PAGES_PER_SECTION
);
698 vmemmap_free(start
, end
);
700 #endif /* CONFIG_MEMORY_HOTREMOVE */
702 static struct page
*__kmalloc_section_memmap(void)
704 struct page
*page
, *ret
;
705 unsigned long memmap_size
= sizeof(struct page
) * PAGES_PER_SECTION
;
707 page
= alloc_pages(GFP_KERNEL
|__GFP_NOWARN
, get_order(memmap_size
));
711 ret
= vmalloc(memmap_size
);
717 ret
= (struct page
*)pfn_to_kaddr(page_to_pfn(page
));
723 static inline struct page
*kmalloc_section_memmap(unsigned long pnum
, int nid
)
725 return __kmalloc_section_memmap();
728 static void __kfree_section_memmap(struct page
*memmap
)
730 if (is_vmalloc_addr(memmap
))
733 free_pages((unsigned long)memmap
,
734 get_order(sizeof(struct page
) * PAGES_PER_SECTION
));
737 #ifdef CONFIG_MEMORY_HOTREMOVE
738 static void free_map_bootmem(struct page
*memmap
)
740 unsigned long maps_section_nr
, removing_section_nr
, i
;
741 unsigned long magic
, nr_pages
;
742 struct page
*page
= virt_to_page(memmap
);
744 nr_pages
= PAGE_ALIGN(PAGES_PER_SECTION
* sizeof(struct page
))
747 for (i
= 0; i
< nr_pages
; i
++, page
++) {
748 magic
= (unsigned long) page
->freelist
;
750 BUG_ON(magic
== NODE_INFO
);
752 maps_section_nr
= pfn_to_section_nr(page_to_pfn(page
));
753 removing_section_nr
= page_private(page
);
756 * When this function is called, the removing section is
757 * logical offlined state. This means all pages are isolated
758 * from page allocator. If removing section's memmap is placed
759 * on the same section, it must not be freed.
760 * If it is freed, page allocator may allocate it which will
761 * be removed physically soon.
763 if (maps_section_nr
!= removing_section_nr
)
764 put_page_bootmem(page
);
767 #endif /* CONFIG_MEMORY_HOTREMOVE */
768 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
771 * returns the number of sections whose mem_maps were properly
772 * set. If this is <=0, then that means that the passed-in
773 * map was not consumed and must be freed.
775 int __meminit
sparse_add_one_section(struct pglist_data
*pgdat
, unsigned long start_pfn
)
777 unsigned long section_nr
= pfn_to_section_nr(start_pfn
);
778 struct mem_section
*ms
;
780 unsigned long *usemap
;
785 * no locking for this, because it does its own
786 * plus, it does a kmalloc
788 ret
= sparse_index_init(section_nr
, pgdat
->node_id
);
789 if (ret
< 0 && ret
!= -EEXIST
)
791 memmap
= kmalloc_section_memmap(section_nr
, pgdat
->node_id
);
794 usemap
= __kmalloc_section_usemap();
796 __kfree_section_memmap(memmap
);
800 pgdat_resize_lock(pgdat
, &flags
);
802 ms
= __pfn_to_section(start_pfn
);
803 if (ms
->section_mem_map
& SECTION_MARKED_PRESENT
) {
808 memset(memmap
, 0, sizeof(struct page
) * PAGES_PER_SECTION
);
810 section_mark_present(ms
);
812 ret
= sparse_init_one_section(ms
, section_nr
, memmap
, usemap
);
815 pgdat_resize_unlock(pgdat
, &flags
);
818 __kfree_section_memmap(memmap
);
823 #ifdef CONFIG_MEMORY_HOTREMOVE
824 #ifdef CONFIG_MEMORY_FAILURE
825 static void clear_hwpoisoned_pages(struct page
*memmap
, int nr_pages
)
832 for (i
= 0; i
< nr_pages
; i
++) {
833 if (PageHWPoison(&memmap
[i
])) {
834 atomic_long_sub(1, &num_poisoned_pages
);
835 ClearPageHWPoison(&memmap
[i
]);
840 static inline void clear_hwpoisoned_pages(struct page
*memmap
, int nr_pages
)
845 static void free_section_usemap(struct page
*memmap
, unsigned long *usemap
)
847 struct page
*usemap_page
;
852 usemap_page
= virt_to_page(usemap
);
854 * Check to see if allocation came from hot-plug-add
856 if (PageSlab(usemap_page
) || PageCompound(usemap_page
)) {
859 __kfree_section_memmap(memmap
);
864 * The usemap came from bootmem. This is packed with other usemaps
865 * on the section which has pgdat at boot time. Just keep it as is now.
869 free_map_bootmem(memmap
);
872 void sparse_remove_one_section(struct zone
*zone
, struct mem_section
*ms
,
873 unsigned long map_offset
)
875 struct page
*memmap
= NULL
;
876 unsigned long *usemap
= NULL
, flags
;
877 struct pglist_data
*pgdat
= zone
->zone_pgdat
;
879 pgdat_resize_lock(pgdat
, &flags
);
880 if (ms
->section_mem_map
) {
881 usemap
= ms
->pageblock_flags
;
882 memmap
= sparse_decode_mem_map(ms
->section_mem_map
,
884 ms
->section_mem_map
= 0;
885 ms
->pageblock_flags
= NULL
;
887 pgdat_resize_unlock(pgdat
, &flags
);
889 clear_hwpoisoned_pages(memmap
+ map_offset
,
890 PAGES_PER_SECTION
- map_offset
);
891 free_section_usemap(memmap
, usemap
);
893 #endif /* CONFIG_MEMORY_HOTREMOVE */
894 #endif /* CONFIG_MEMORY_HOTPLUG */