1 // SPDX-License-Identifier: GPL-2.0
3 * sparse memory mappings.
6 #include <linux/slab.h>
7 #include <linux/mmzone.h>
8 #include <linux/bootmem.h>
9 #include <linux/compiler.h>
10 #include <linux/highmem.h>
11 #include <linux/export.h>
12 #include <linux/spinlock.h>
13 #include <linux/vmalloc.h>
17 #include <asm/pgalloc.h>
18 #include <asm/pgtable.h>
21 * Permanent SPARSEMEM data:
23 * 1) mem_section - memory sections, mem_map's for valid memory
25 #ifdef CONFIG_SPARSEMEM_EXTREME
26 struct mem_section
**mem_section
;
28 struct mem_section mem_section
[NR_SECTION_ROOTS
][SECTIONS_PER_ROOT
]
29 ____cacheline_internodealigned_in_smp
;
31 EXPORT_SYMBOL(mem_section
);
33 #ifdef NODE_NOT_IN_PAGE_FLAGS
35 * If we did not store the node number in the page then we have to
36 * do a lookup in the section_to_node_table in order to find which
37 * node the page belongs to.
39 #if MAX_NUMNODES <= 256
40 static u8 section_to_node_table
[NR_MEM_SECTIONS
] __cacheline_aligned
;
42 static u16 section_to_node_table
[NR_MEM_SECTIONS
] __cacheline_aligned
;
45 int page_to_nid(const struct page
*page
)
47 return section_to_node_table
[page_to_section(page
)];
49 EXPORT_SYMBOL(page_to_nid
);
51 static void set_section_nid(unsigned long section_nr
, int nid
)
53 section_to_node_table
[section_nr
] = nid
;
55 #else /* !NODE_NOT_IN_PAGE_FLAGS */
56 static inline void set_section_nid(unsigned long section_nr
, int nid
)
61 #ifdef CONFIG_SPARSEMEM_EXTREME
62 static noinline
struct mem_section __ref
*sparse_index_alloc(int nid
)
64 struct mem_section
*section
= NULL
;
65 unsigned long array_size
= SECTIONS_PER_ROOT
*
66 sizeof(struct mem_section
);
68 if (slab_is_available())
69 section
= kzalloc_node(array_size
, GFP_KERNEL
, nid
);
71 section
= memblock_virt_alloc_node(array_size
, nid
);
76 static int __meminit
sparse_index_init(unsigned long section_nr
, int nid
)
78 unsigned long root
= SECTION_NR_TO_ROOT(section_nr
);
79 struct mem_section
*section
;
81 if (mem_section
[root
])
84 section
= sparse_index_alloc(nid
);
88 mem_section
[root
] = section
;
92 #else /* !SPARSEMEM_EXTREME */
93 static inline int sparse_index_init(unsigned long section_nr
, int nid
)
99 #ifdef CONFIG_SPARSEMEM_EXTREME
100 int __section_nr(struct mem_section
* ms
)
102 unsigned long root_nr
;
103 struct mem_section
*root
= NULL
;
105 for (root_nr
= 0; root_nr
< NR_SECTION_ROOTS
; root_nr
++) {
106 root
= __nr_to_section(root_nr
* SECTIONS_PER_ROOT
);
110 if ((ms
>= root
) && (ms
< (root
+ SECTIONS_PER_ROOT
)))
116 return (root_nr
* SECTIONS_PER_ROOT
) + (ms
- root
);
119 int __section_nr(struct mem_section
* ms
)
121 return (int)(ms
- mem_section
[0]);
126 * During early boot, before section_mem_map is used for an actual
127 * mem_map, we use section_mem_map to store the section's NUMA
128 * node. This keeps us from having to use another data structure. The
129 * node information is cleared just before we store the real mem_map.
131 static inline unsigned long sparse_encode_early_nid(int nid
)
133 return (nid
<< SECTION_NID_SHIFT
);
136 static inline int sparse_early_nid(struct mem_section
*section
)
138 return (section
->section_mem_map
>> SECTION_NID_SHIFT
);
141 /* Validate the physical addressing limitations of the model */
142 void __meminit
mminit_validate_memmodel_limits(unsigned long *start_pfn
,
143 unsigned long *end_pfn
)
145 unsigned long max_sparsemem_pfn
= 1UL << (MAX_PHYSMEM_BITS
-PAGE_SHIFT
);
148 * Sanity checks - do not allow an architecture to pass
149 * in larger pfns than the maximum scope of sparsemem:
151 if (*start_pfn
> max_sparsemem_pfn
) {
152 mminit_dprintk(MMINIT_WARNING
, "pfnvalidation",
153 "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
154 *start_pfn
, *end_pfn
, max_sparsemem_pfn
);
156 *start_pfn
= max_sparsemem_pfn
;
157 *end_pfn
= max_sparsemem_pfn
;
158 } else if (*end_pfn
> max_sparsemem_pfn
) {
159 mminit_dprintk(MMINIT_WARNING
, "pfnvalidation",
160 "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
161 *start_pfn
, *end_pfn
, max_sparsemem_pfn
);
163 *end_pfn
= max_sparsemem_pfn
;
168 * There are a number of times that we loop over NR_MEM_SECTIONS,
169 * looking for section_present() on each. But, when we have very
170 * large physical address spaces, NR_MEM_SECTIONS can also be
171 * very large which makes the loops quite long.
173 * Keeping track of this gives us an easy way to break out of
176 int __highest_present_section_nr
;
177 static void section_mark_present(struct mem_section
*ms
)
179 int section_nr
= __section_nr(ms
);
181 if (section_nr
> __highest_present_section_nr
)
182 __highest_present_section_nr
= section_nr
;
184 ms
->section_mem_map
|= SECTION_MARKED_PRESENT
;
187 static inline int next_present_section_nr(int section_nr
)
191 if (present_section_nr(section_nr
))
193 } while ((section_nr
< NR_MEM_SECTIONS
) &&
194 (section_nr
<= __highest_present_section_nr
));
198 #define for_each_present_section_nr(start, section_nr) \
199 for (section_nr = next_present_section_nr(start-1); \
200 ((section_nr >= 0) && \
201 (section_nr < NR_MEM_SECTIONS) && \
202 (section_nr <= __highest_present_section_nr)); \
203 section_nr = next_present_section_nr(section_nr))
205 /* Record a memory area against a node. */
206 void __init
memory_present(int nid
, unsigned long start
, unsigned long end
)
210 start
&= PAGE_SECTION_MASK
;
211 mminit_validate_memmodel_limits(&start
, &end
);
212 for (pfn
= start
; pfn
< end
; pfn
+= PAGES_PER_SECTION
) {
213 unsigned long section
= pfn_to_section_nr(pfn
);
214 struct mem_section
*ms
;
216 sparse_index_init(section
, nid
);
217 set_section_nid(section
, nid
);
219 ms
= __nr_to_section(section
);
220 if (!ms
->section_mem_map
) {
221 ms
->section_mem_map
= sparse_encode_early_nid(nid
) |
223 section_mark_present(ms
);
229 * Only used by the i386 NUMA architecures, but relatively
232 unsigned long __init
node_memmap_size_bytes(int nid
, unsigned long start_pfn
,
233 unsigned long end_pfn
)
236 unsigned long nr_pages
= 0;
238 mminit_validate_memmodel_limits(&start_pfn
, &end_pfn
);
239 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= PAGES_PER_SECTION
) {
240 if (nid
!= early_pfn_to_nid(pfn
))
243 if (pfn_present(pfn
))
244 nr_pages
+= PAGES_PER_SECTION
;
247 return nr_pages
* sizeof(struct page
);
251 * Subtle, we encode the real pfn into the mem_map such that
252 * the identity pfn - section_mem_map will return the actual
253 * physical page frame number.
255 static unsigned long sparse_encode_mem_map(struct page
*mem_map
, unsigned long pnum
)
257 return (unsigned long)(mem_map
- (section_nr_to_pfn(pnum
)));
261 * Decode mem_map from the coded memmap
263 struct page
*sparse_decode_mem_map(unsigned long coded_mem_map
, unsigned long pnum
)
265 /* mask off the extra low bits of information */
266 coded_mem_map
&= SECTION_MAP_MASK
;
267 return ((struct page
*)coded_mem_map
) + section_nr_to_pfn(pnum
);
270 static int __meminit
sparse_init_one_section(struct mem_section
*ms
,
271 unsigned long pnum
, struct page
*mem_map
,
272 unsigned long *pageblock_bitmap
)
274 if (!present_section(ms
))
277 ms
->section_mem_map
&= ~SECTION_MAP_MASK
;
278 ms
->section_mem_map
|= sparse_encode_mem_map(mem_map
, pnum
) |
280 ms
->pageblock_flags
= pageblock_bitmap
;
285 unsigned long usemap_size(void)
287 return BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS
) * sizeof(unsigned long);
290 #ifdef CONFIG_MEMORY_HOTPLUG
291 static unsigned long *__kmalloc_section_usemap(void)
293 return kmalloc(usemap_size(), GFP_KERNEL
);
295 #endif /* CONFIG_MEMORY_HOTPLUG */
297 #ifdef CONFIG_MEMORY_HOTREMOVE
298 static unsigned long * __init
299 sparse_early_usemaps_alloc_pgdat_section(struct pglist_data
*pgdat
,
302 unsigned long goal
, limit
;
306 * A page may contain usemaps for other sections preventing the
307 * page being freed and making a section unremovable while
308 * other sections referencing the usemap remain active. Similarly,
309 * a pgdat can prevent a section being removed. If section A
310 * contains a pgdat and section B contains the usemap, both
311 * sections become inter-dependent. This allocates usemaps
312 * from the same section as the pgdat where possible to avoid
315 goal
= __pa(pgdat
) & (PAGE_SECTION_MASK
<< PAGE_SHIFT
);
316 limit
= goal
+ (1UL << PA_SECTION_SHIFT
);
317 nid
= early_pfn_to_nid(goal
>> PAGE_SHIFT
);
319 p
= memblock_virt_alloc_try_nid_nopanic(size
,
320 SMP_CACHE_BYTES
, goal
, limit
,
329 static void __init
check_usemap_section_nr(int nid
, unsigned long *usemap
)
331 unsigned long usemap_snr
, pgdat_snr
;
332 static unsigned long old_usemap_snr
;
333 static unsigned long old_pgdat_snr
;
334 struct pglist_data
*pgdat
= NODE_DATA(nid
);
338 if (!old_usemap_snr
) {
339 old_usemap_snr
= NR_MEM_SECTIONS
;
340 old_pgdat_snr
= NR_MEM_SECTIONS
;
343 usemap_snr
= pfn_to_section_nr(__pa(usemap
) >> PAGE_SHIFT
);
344 pgdat_snr
= pfn_to_section_nr(__pa(pgdat
) >> PAGE_SHIFT
);
345 if (usemap_snr
== pgdat_snr
)
348 if (old_usemap_snr
== usemap_snr
&& old_pgdat_snr
== pgdat_snr
)
349 /* skip redundant message */
352 old_usemap_snr
= usemap_snr
;
353 old_pgdat_snr
= pgdat_snr
;
355 usemap_nid
= sparse_early_nid(__nr_to_section(usemap_snr
));
356 if (usemap_nid
!= nid
) {
357 pr_info("node %d must be removed before remove section %ld\n",
362 * There is a circular dependency.
363 * Some platforms allow un-removable section because they will just
364 * gather other removable sections for dynamic partitioning.
365 * Just notify un-removable section's number here.
367 pr_info("Section %ld and %ld (node %d) have a circular dependency on usemap and pgdat allocations\n",
368 usemap_snr
, pgdat_snr
, nid
);
371 static unsigned long * __init
372 sparse_early_usemaps_alloc_pgdat_section(struct pglist_data
*pgdat
,
375 return memblock_virt_alloc_node_nopanic(size
, pgdat
->node_id
);
378 static void __init
check_usemap_section_nr(int nid
, unsigned long *usemap
)
381 #endif /* CONFIG_MEMORY_HOTREMOVE */
383 static void __init
sparse_early_usemaps_alloc_node(void *data
,
384 unsigned long pnum_begin
,
385 unsigned long pnum_end
,
386 unsigned long usemap_count
, int nodeid
)
390 unsigned long **usemap_map
= (unsigned long **)data
;
391 int size
= usemap_size();
393 usemap
= sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nodeid
),
394 size
* usemap_count
);
396 pr_warn("%s: allocation failed\n", __func__
);
400 for (pnum
= pnum_begin
; pnum
< pnum_end
; pnum
++) {
401 if (!present_section_nr(pnum
))
403 usemap_map
[pnum
] = usemap
;
405 check_usemap_section_nr(nodeid
, usemap_map
[pnum
]);
409 #ifndef CONFIG_SPARSEMEM_VMEMMAP
410 struct page __init
*sparse_mem_map_populate(unsigned long pnum
, int nid
)
415 map
= alloc_remap(nid
, sizeof(struct page
) * PAGES_PER_SECTION
);
419 size
= PAGE_ALIGN(sizeof(struct page
) * PAGES_PER_SECTION
);
420 map
= memblock_virt_alloc_try_nid(size
,
421 PAGE_SIZE
, __pa(MAX_DMA_ADDRESS
),
422 BOOTMEM_ALLOC_ACCESSIBLE
, nid
);
425 void __init
sparse_mem_maps_populate_node(struct page
**map_map
,
426 unsigned long pnum_begin
,
427 unsigned long pnum_end
,
428 unsigned long map_count
, int nodeid
)
432 unsigned long size
= sizeof(struct page
) * PAGES_PER_SECTION
;
434 map
= alloc_remap(nodeid
, size
* map_count
);
436 for (pnum
= pnum_begin
; pnum
< pnum_end
; pnum
++) {
437 if (!present_section_nr(pnum
))
445 size
= PAGE_ALIGN(size
);
446 map
= memblock_virt_alloc_try_nid(size
* map_count
,
447 PAGE_SIZE
, __pa(MAX_DMA_ADDRESS
),
448 BOOTMEM_ALLOC_ACCESSIBLE
, nodeid
);
450 for (pnum
= pnum_begin
; pnum
< pnum_end
; pnum
++) {
451 if (!present_section_nr(pnum
))
460 for (pnum
= pnum_begin
; pnum
< pnum_end
; pnum
++) {
461 struct mem_section
*ms
;
463 if (!present_section_nr(pnum
))
465 map_map
[pnum
] = sparse_mem_map_populate(pnum
, nodeid
);
468 ms
= __nr_to_section(pnum
);
469 pr_err("%s: sparsemem memory map backing failed some memory will not be available\n",
471 ms
->section_mem_map
= 0;
474 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */
476 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
477 static void __init
sparse_early_mem_maps_alloc_node(void *data
,
478 unsigned long pnum_begin
,
479 unsigned long pnum_end
,
480 unsigned long map_count
, int nodeid
)
482 struct page
**map_map
= (struct page
**)data
;
483 sparse_mem_maps_populate_node(map_map
, pnum_begin
, pnum_end
,
487 static struct page __init
*sparse_early_mem_map_alloc(unsigned long pnum
)
490 struct mem_section
*ms
= __nr_to_section(pnum
);
491 int nid
= sparse_early_nid(ms
);
493 map
= sparse_mem_map_populate(pnum
, nid
);
497 pr_err("%s: sparsemem memory map backing failed some memory will not be available\n",
499 ms
->section_mem_map
= 0;
504 void __weak __meminit
vmemmap_populate_print_last(void)
509 * alloc_usemap_and_memmap - memory alloction for pageblock flags and vmemmap
510 * @map: usemap_map for pageblock flags or mmap_map for vmemmap
512 static void __init
alloc_usemap_and_memmap(void (*alloc_func
)
513 (void *, unsigned long, unsigned long,
514 unsigned long, int), void *data
)
517 unsigned long map_count
;
518 int nodeid_begin
= 0;
519 unsigned long pnum_begin
= 0;
521 for_each_present_section_nr(0, pnum
) {
522 struct mem_section
*ms
;
524 ms
= __nr_to_section(pnum
);
525 nodeid_begin
= sparse_early_nid(ms
);
530 for_each_present_section_nr(pnum_begin
+ 1, pnum
) {
531 struct mem_section
*ms
;
534 ms
= __nr_to_section(pnum
);
535 nodeid
= sparse_early_nid(ms
);
536 if (nodeid
== nodeid_begin
) {
540 /* ok, we need to take cake of from pnum_begin to pnum - 1*/
541 alloc_func(data
, pnum_begin
, pnum
,
542 map_count
, nodeid_begin
);
543 /* new start, update count etc*/
544 nodeid_begin
= nodeid
;
549 alloc_func(data
, pnum_begin
, NR_MEM_SECTIONS
,
550 map_count
, nodeid_begin
);
554 * Allocate the accumulated non-linear sections, allocate a mem_map
555 * for each and record the physical to section mapping.
557 void __init
sparse_init(void)
561 unsigned long *usemap
;
562 unsigned long **usemap_map
;
564 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
566 struct page
**map_map
;
569 /* see include/linux/mmzone.h 'struct mem_section' definition */
570 BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section
)));
572 /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */
573 set_pageblock_order();
576 * map is using big page (aka 2M in x86 64 bit)
577 * usemap is less one page (aka 24 bytes)
578 * so alloc 2M (with 2M align) and 24 bytes in turn will
579 * make next 2M slip to one more 2M later.
580 * then in big system, the memory will have a lot of holes...
581 * here try to allocate 2M pages continuously.
583 * powerpc need to call sparse_init_one_section right after each
584 * sparse_early_mem_map_alloc, so allocate usemap_map at first.
586 size
= sizeof(unsigned long *) * NR_MEM_SECTIONS
;
587 usemap_map
= memblock_virt_alloc(size
, 0);
589 panic("can not allocate usemap_map\n");
590 alloc_usemap_and_memmap(sparse_early_usemaps_alloc_node
,
593 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
594 size2
= sizeof(struct page
*) * NR_MEM_SECTIONS
;
595 map_map
= memblock_virt_alloc(size2
, 0);
597 panic("can not allocate map_map\n");
598 alloc_usemap_and_memmap(sparse_early_mem_maps_alloc_node
,
602 for_each_present_section_nr(0, pnum
) {
603 usemap
= usemap_map
[pnum
];
607 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
610 map
= sparse_early_mem_map_alloc(pnum
);
615 sparse_init_one_section(__nr_to_section(pnum
), pnum
, map
,
619 vmemmap_populate_print_last();
621 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
622 memblock_free_early(__pa(map_map
), size2
);
624 memblock_free_early(__pa(usemap_map
), size
);
627 #ifdef CONFIG_MEMORY_HOTPLUG
629 /* Mark all memory sections within the pfn range as online */
630 void online_mem_sections(unsigned long start_pfn
, unsigned long end_pfn
)
634 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= PAGES_PER_SECTION
) {
635 unsigned long section_nr
= pfn_to_section_nr(pfn
);
636 struct mem_section
*ms
;
638 /* onlining code should never touch invalid ranges */
639 if (WARN_ON(!valid_section_nr(section_nr
)))
642 ms
= __nr_to_section(section_nr
);
643 ms
->section_mem_map
|= SECTION_IS_ONLINE
;
647 #ifdef CONFIG_MEMORY_HOTREMOVE
648 /* Mark all memory sections within the pfn range as online */
649 void offline_mem_sections(unsigned long start_pfn
, unsigned long end_pfn
)
653 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= PAGES_PER_SECTION
) {
654 unsigned long section_nr
= pfn_to_section_nr(start_pfn
);
655 struct mem_section
*ms
;
658 * TODO this needs some double checking. Offlining code makes
659 * sure to check pfn_valid but those checks might be just bogus
661 if (WARN_ON(!valid_section_nr(section_nr
)))
664 ms
= __nr_to_section(section_nr
);
665 ms
->section_mem_map
&= ~SECTION_IS_ONLINE
;
670 #ifdef CONFIG_SPARSEMEM_VMEMMAP
671 static inline struct page
*kmalloc_section_memmap(unsigned long pnum
, int nid
)
673 /* This will make the necessary allocations eventually. */
674 return sparse_mem_map_populate(pnum
, nid
);
676 static void __kfree_section_memmap(struct page
*memmap
)
678 unsigned long start
= (unsigned long)memmap
;
679 unsigned long end
= (unsigned long)(memmap
+ PAGES_PER_SECTION
);
681 vmemmap_free(start
, end
);
683 #ifdef CONFIG_MEMORY_HOTREMOVE
684 static void free_map_bootmem(struct page
*memmap
)
686 unsigned long start
= (unsigned long)memmap
;
687 unsigned long end
= (unsigned long)(memmap
+ PAGES_PER_SECTION
);
689 vmemmap_free(start
, end
);
691 #endif /* CONFIG_MEMORY_HOTREMOVE */
693 static struct page
*__kmalloc_section_memmap(void)
695 struct page
*page
, *ret
;
696 unsigned long memmap_size
= sizeof(struct page
) * PAGES_PER_SECTION
;
698 page
= alloc_pages(GFP_KERNEL
|__GFP_NOWARN
, get_order(memmap_size
));
702 ret
= vmalloc(memmap_size
);
708 ret
= (struct page
*)pfn_to_kaddr(page_to_pfn(page
));
714 static inline struct page
*kmalloc_section_memmap(unsigned long pnum
, int nid
)
716 return __kmalloc_section_memmap();
719 static void __kfree_section_memmap(struct page
*memmap
)
721 if (is_vmalloc_addr(memmap
))
724 free_pages((unsigned long)memmap
,
725 get_order(sizeof(struct page
) * PAGES_PER_SECTION
));
728 #ifdef CONFIG_MEMORY_HOTREMOVE
729 static void free_map_bootmem(struct page
*memmap
)
731 unsigned long maps_section_nr
, removing_section_nr
, i
;
732 unsigned long magic
, nr_pages
;
733 struct page
*page
= virt_to_page(memmap
);
735 nr_pages
= PAGE_ALIGN(PAGES_PER_SECTION
* sizeof(struct page
))
738 for (i
= 0; i
< nr_pages
; i
++, page
++) {
739 magic
= (unsigned long) page
->freelist
;
741 BUG_ON(magic
== NODE_INFO
);
743 maps_section_nr
= pfn_to_section_nr(page_to_pfn(page
));
744 removing_section_nr
= page_private(page
);
747 * When this function is called, the removing section is
748 * logical offlined state. This means all pages are isolated
749 * from page allocator. If removing section's memmap is placed
750 * on the same section, it must not be freed.
751 * If it is freed, page allocator may allocate it which will
752 * be removed physically soon.
754 if (maps_section_nr
!= removing_section_nr
)
755 put_page_bootmem(page
);
758 #endif /* CONFIG_MEMORY_HOTREMOVE */
759 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
762 * returns the number of sections whose mem_maps were properly
763 * set. If this is <=0, then that means that the passed-in
764 * map was not consumed and must be freed.
766 int __meminit
sparse_add_one_section(struct pglist_data
*pgdat
, unsigned long start_pfn
)
768 unsigned long section_nr
= pfn_to_section_nr(start_pfn
);
769 struct mem_section
*ms
;
771 unsigned long *usemap
;
776 * no locking for this, because it does its own
777 * plus, it does a kmalloc
779 ret
= sparse_index_init(section_nr
, pgdat
->node_id
);
780 if (ret
< 0 && ret
!= -EEXIST
)
782 memmap
= kmalloc_section_memmap(section_nr
, pgdat
->node_id
);
785 usemap
= __kmalloc_section_usemap();
787 __kfree_section_memmap(memmap
);
791 pgdat_resize_lock(pgdat
, &flags
);
793 ms
= __pfn_to_section(start_pfn
);
794 if (ms
->section_mem_map
& SECTION_MARKED_PRESENT
) {
799 memset(memmap
, 0, sizeof(struct page
) * PAGES_PER_SECTION
);
801 section_mark_present(ms
);
803 ret
= sparse_init_one_section(ms
, section_nr
, memmap
, usemap
);
806 pgdat_resize_unlock(pgdat
, &flags
);
809 __kfree_section_memmap(memmap
);
814 #ifdef CONFIG_MEMORY_HOTREMOVE
815 #ifdef CONFIG_MEMORY_FAILURE
816 static void clear_hwpoisoned_pages(struct page
*memmap
, int nr_pages
)
823 for (i
= 0; i
< nr_pages
; i
++) {
824 if (PageHWPoison(&memmap
[i
])) {
825 atomic_long_sub(1, &num_poisoned_pages
);
826 ClearPageHWPoison(&memmap
[i
]);
831 static inline void clear_hwpoisoned_pages(struct page
*memmap
, int nr_pages
)
836 static void free_section_usemap(struct page
*memmap
, unsigned long *usemap
)
838 struct page
*usemap_page
;
843 usemap_page
= virt_to_page(usemap
);
845 * Check to see if allocation came from hot-plug-add
847 if (PageSlab(usemap_page
) || PageCompound(usemap_page
)) {
850 __kfree_section_memmap(memmap
);
855 * The usemap came from bootmem. This is packed with other usemaps
856 * on the section which has pgdat at boot time. Just keep it as is now.
860 free_map_bootmem(memmap
);
863 void sparse_remove_one_section(struct zone
*zone
, struct mem_section
*ms
,
864 unsigned long map_offset
)
866 struct page
*memmap
= NULL
;
867 unsigned long *usemap
= NULL
, flags
;
868 struct pglist_data
*pgdat
= zone
->zone_pgdat
;
870 pgdat_resize_lock(pgdat
, &flags
);
871 if (ms
->section_mem_map
) {
872 usemap
= ms
->pageblock_flags
;
873 memmap
= sparse_decode_mem_map(ms
->section_mem_map
,
875 ms
->section_mem_map
= 0;
876 ms
->pageblock_flags
= NULL
;
878 pgdat_resize_unlock(pgdat
, &flags
);
880 clear_hwpoisoned_pages(memmap
+ map_offset
,
881 PAGES_PER_SECTION
- map_offset
);
882 free_section_usemap(memmap
, usemap
);
884 #endif /* CONFIG_MEMORY_HOTREMOVE */
885 #endif /* CONFIG_MEMORY_HOTPLUG */