1 // SPDX-License-Identifier: GPL-2.0
3 * sparse memory mappings.
6 #include <linux/slab.h>
7 #include <linux/mmzone.h>
8 #include <linux/bootmem.h>
9 #include <linux/compiler.h>
10 #include <linux/highmem.h>
11 #include <linux/export.h>
12 #include <linux/spinlock.h>
13 #include <linux/vmalloc.h>
17 #include <asm/pgalloc.h>
18 #include <asm/pgtable.h>
21 * Permanent SPARSEMEM data:
23 * 1) mem_section - memory sections, mem_map's for valid memory
25 #ifdef CONFIG_SPARSEMEM_EXTREME
26 struct mem_section
**mem_section
;
28 struct mem_section mem_section
[NR_SECTION_ROOTS
][SECTIONS_PER_ROOT
]
29 ____cacheline_internodealigned_in_smp
;
31 EXPORT_SYMBOL(mem_section
);
33 #ifdef NODE_NOT_IN_PAGE_FLAGS
35 * If we did not store the node number in the page then we have to
36 * do a lookup in the section_to_node_table in order to find which
37 * node the page belongs to.
39 #if MAX_NUMNODES <= 256
40 static u8 section_to_node_table
[NR_MEM_SECTIONS
] __cacheline_aligned
;
42 static u16 section_to_node_table
[NR_MEM_SECTIONS
] __cacheline_aligned
;
45 int page_to_nid(const struct page
*page
)
47 return section_to_node_table
[page_to_section(page
)];
49 EXPORT_SYMBOL(page_to_nid
);
51 static void set_section_nid(unsigned long section_nr
, int nid
)
53 section_to_node_table
[section_nr
] = nid
;
55 #else /* !NODE_NOT_IN_PAGE_FLAGS */
56 static inline void set_section_nid(unsigned long section_nr
, int nid
)
61 #ifdef CONFIG_SPARSEMEM_EXTREME
62 static noinline
struct mem_section __ref
*sparse_index_alloc(int nid
)
64 struct mem_section
*section
= NULL
;
65 unsigned long array_size
= SECTIONS_PER_ROOT
*
66 sizeof(struct mem_section
);
68 if (slab_is_available())
69 section
= kzalloc_node(array_size
, GFP_KERNEL
, nid
);
71 section
= memblock_virt_alloc_node(array_size
, nid
);
76 static int __meminit
sparse_index_init(unsigned long section_nr
, int nid
)
78 unsigned long root
= SECTION_NR_TO_ROOT(section_nr
);
79 struct mem_section
*section
;
81 if (mem_section
[root
])
84 section
= sparse_index_alloc(nid
);
88 mem_section
[root
] = section
;
92 #else /* !SPARSEMEM_EXTREME */
93 static inline int sparse_index_init(unsigned long section_nr
, int nid
)
99 #ifdef CONFIG_SPARSEMEM_EXTREME
100 int __section_nr(struct mem_section
* ms
)
102 unsigned long root_nr
;
103 struct mem_section
*root
= NULL
;
105 for (root_nr
= 0; root_nr
< NR_SECTION_ROOTS
; root_nr
++) {
106 root
= __nr_to_section(root_nr
* SECTIONS_PER_ROOT
);
110 if ((ms
>= root
) && (ms
< (root
+ SECTIONS_PER_ROOT
)))
116 return (root_nr
* SECTIONS_PER_ROOT
) + (ms
- root
);
119 int __section_nr(struct mem_section
* ms
)
121 return (int)(ms
- mem_section
[0]);
126 * During early boot, before section_mem_map is used for an actual
127 * mem_map, we use section_mem_map to store the section's NUMA
128 * node. This keeps us from having to use another data structure. The
129 * node information is cleared just before we store the real mem_map.
131 static inline unsigned long sparse_encode_early_nid(int nid
)
133 return (nid
<< SECTION_NID_SHIFT
);
136 static inline int sparse_early_nid(struct mem_section
*section
)
138 return (section
->section_mem_map
>> SECTION_NID_SHIFT
);
141 /* Validate the physical addressing limitations of the model */
142 void __meminit
mminit_validate_memmodel_limits(unsigned long *start_pfn
,
143 unsigned long *end_pfn
)
145 unsigned long max_sparsemem_pfn
= 1UL << (MAX_PHYSMEM_BITS
-PAGE_SHIFT
);
148 * Sanity checks - do not allow an architecture to pass
149 * in larger pfns than the maximum scope of sparsemem:
151 if (*start_pfn
> max_sparsemem_pfn
) {
152 mminit_dprintk(MMINIT_WARNING
, "pfnvalidation",
153 "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
154 *start_pfn
, *end_pfn
, max_sparsemem_pfn
);
156 *start_pfn
= max_sparsemem_pfn
;
157 *end_pfn
= max_sparsemem_pfn
;
158 } else if (*end_pfn
> max_sparsemem_pfn
) {
159 mminit_dprintk(MMINIT_WARNING
, "pfnvalidation",
160 "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
161 *start_pfn
, *end_pfn
, max_sparsemem_pfn
);
163 *end_pfn
= max_sparsemem_pfn
;
168 * There are a number of times that we loop over NR_MEM_SECTIONS,
169 * looking for section_present() on each. But, when we have very
170 * large physical address spaces, NR_MEM_SECTIONS can also be
171 * very large which makes the loops quite long.
173 * Keeping track of this gives us an easy way to break out of
176 int __highest_present_section_nr
;
177 static void section_mark_present(struct mem_section
*ms
)
179 int section_nr
= __section_nr(ms
);
181 if (section_nr
> __highest_present_section_nr
)
182 __highest_present_section_nr
= section_nr
;
184 ms
->section_mem_map
|= SECTION_MARKED_PRESENT
;
187 static inline int next_present_section_nr(int section_nr
)
191 if (present_section_nr(section_nr
))
193 } while ((section_nr
< NR_MEM_SECTIONS
) &&
194 (section_nr
<= __highest_present_section_nr
));
198 #define for_each_present_section_nr(start, section_nr) \
199 for (section_nr = next_present_section_nr(start-1); \
200 ((section_nr >= 0) && \
201 (section_nr < NR_MEM_SECTIONS) && \
202 (section_nr <= __highest_present_section_nr)); \
203 section_nr = next_present_section_nr(section_nr))
205 /* Record a memory area against a node. */
206 void __init
memory_present(int nid
, unsigned long start
, unsigned long end
)
210 #ifdef CONFIG_SPARSEMEM_EXTREME
211 if (unlikely(!mem_section
)) {
212 unsigned long size
, align
;
214 size
= sizeof(struct mem_section
*) * NR_SECTION_ROOTS
;
215 align
= 1 << (INTERNODE_CACHE_SHIFT
);
216 mem_section
= memblock_virt_alloc(size
, align
);
220 start
&= PAGE_SECTION_MASK
;
221 mminit_validate_memmodel_limits(&start
, &end
);
222 for (pfn
= start
; pfn
< end
; pfn
+= PAGES_PER_SECTION
) {
223 unsigned long section
= pfn_to_section_nr(pfn
);
224 struct mem_section
*ms
;
226 sparse_index_init(section
, nid
);
227 set_section_nid(section
, nid
);
229 ms
= __nr_to_section(section
);
230 if (!ms
->section_mem_map
) {
231 ms
->section_mem_map
= sparse_encode_early_nid(nid
) |
233 section_mark_present(ms
);
239 * Only used by the i386 NUMA architecures, but relatively
242 unsigned long __init
node_memmap_size_bytes(int nid
, unsigned long start_pfn
,
243 unsigned long end_pfn
)
246 unsigned long nr_pages
= 0;
248 mminit_validate_memmodel_limits(&start_pfn
, &end_pfn
);
249 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= PAGES_PER_SECTION
) {
250 if (nid
!= early_pfn_to_nid(pfn
))
253 if (pfn_present(pfn
))
254 nr_pages
+= PAGES_PER_SECTION
;
257 return nr_pages
* sizeof(struct page
);
261 * Subtle, we encode the real pfn into the mem_map such that
262 * the identity pfn - section_mem_map will return the actual
263 * physical page frame number.
265 static unsigned long sparse_encode_mem_map(struct page
*mem_map
, unsigned long pnum
)
267 unsigned long coded_mem_map
=
268 (unsigned long)(mem_map
- (section_nr_to_pfn(pnum
)));
269 BUILD_BUG_ON(SECTION_MAP_LAST_BIT
> (1UL<<PFN_SECTION_SHIFT
));
270 BUG_ON(coded_mem_map
& ~SECTION_MAP_MASK
);
271 return coded_mem_map
;
275 * Decode mem_map from the coded memmap
277 struct page
*sparse_decode_mem_map(unsigned long coded_mem_map
, unsigned long pnum
)
279 /* mask off the extra low bits of information */
280 coded_mem_map
&= SECTION_MAP_MASK
;
281 return ((struct page
*)coded_mem_map
) + section_nr_to_pfn(pnum
);
284 static int __meminit
sparse_init_one_section(struct mem_section
*ms
,
285 unsigned long pnum
, struct page
*mem_map
,
286 unsigned long *pageblock_bitmap
)
288 if (!present_section(ms
))
291 ms
->section_mem_map
&= ~SECTION_MAP_MASK
;
292 ms
->section_mem_map
|= sparse_encode_mem_map(mem_map
, pnum
) |
294 ms
->pageblock_flags
= pageblock_bitmap
;
299 unsigned long usemap_size(void)
301 return BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS
) * sizeof(unsigned long);
304 #ifdef CONFIG_MEMORY_HOTPLUG
305 static unsigned long *__kmalloc_section_usemap(void)
307 return kmalloc(usemap_size(), GFP_KERNEL
);
309 #endif /* CONFIG_MEMORY_HOTPLUG */
311 #ifdef CONFIG_MEMORY_HOTREMOVE
312 static unsigned long * __init
313 sparse_early_usemaps_alloc_pgdat_section(struct pglist_data
*pgdat
,
316 unsigned long goal
, limit
;
320 * A page may contain usemaps for other sections preventing the
321 * page being freed and making a section unremovable while
322 * other sections referencing the usemap remain active. Similarly,
323 * a pgdat can prevent a section being removed. If section A
324 * contains a pgdat and section B contains the usemap, both
325 * sections become inter-dependent. This allocates usemaps
326 * from the same section as the pgdat where possible to avoid
329 goal
= __pa(pgdat
) & (PAGE_SECTION_MASK
<< PAGE_SHIFT
);
330 limit
= goal
+ (1UL << PA_SECTION_SHIFT
);
331 nid
= early_pfn_to_nid(goal
>> PAGE_SHIFT
);
333 p
= memblock_virt_alloc_try_nid_nopanic(size
,
334 SMP_CACHE_BYTES
, goal
, limit
,
343 static void __init
check_usemap_section_nr(int nid
, unsigned long *usemap
)
345 unsigned long usemap_snr
, pgdat_snr
;
346 static unsigned long old_usemap_snr
;
347 static unsigned long old_pgdat_snr
;
348 struct pglist_data
*pgdat
= NODE_DATA(nid
);
352 if (!old_usemap_snr
) {
353 old_usemap_snr
= NR_MEM_SECTIONS
;
354 old_pgdat_snr
= NR_MEM_SECTIONS
;
357 usemap_snr
= pfn_to_section_nr(__pa(usemap
) >> PAGE_SHIFT
);
358 pgdat_snr
= pfn_to_section_nr(__pa(pgdat
) >> PAGE_SHIFT
);
359 if (usemap_snr
== pgdat_snr
)
362 if (old_usemap_snr
== usemap_snr
&& old_pgdat_snr
== pgdat_snr
)
363 /* skip redundant message */
366 old_usemap_snr
= usemap_snr
;
367 old_pgdat_snr
= pgdat_snr
;
369 usemap_nid
= sparse_early_nid(__nr_to_section(usemap_snr
));
370 if (usemap_nid
!= nid
) {
371 pr_info("node %d must be removed before remove section %ld\n",
376 * There is a circular dependency.
377 * Some platforms allow un-removable section because they will just
378 * gather other removable sections for dynamic partitioning.
379 * Just notify un-removable section's number here.
381 pr_info("Section %ld and %ld (node %d) have a circular dependency on usemap and pgdat allocations\n",
382 usemap_snr
, pgdat_snr
, nid
);
385 static unsigned long * __init
386 sparse_early_usemaps_alloc_pgdat_section(struct pglist_data
*pgdat
,
389 return memblock_virt_alloc_node_nopanic(size
, pgdat
->node_id
);
392 static void __init
check_usemap_section_nr(int nid
, unsigned long *usemap
)
395 #endif /* CONFIG_MEMORY_HOTREMOVE */
397 static void __init
sparse_early_usemaps_alloc_node(void *data
,
398 unsigned long pnum_begin
,
399 unsigned long pnum_end
,
400 unsigned long usemap_count
, int nodeid
)
404 unsigned long **usemap_map
= (unsigned long **)data
;
405 int size
= usemap_size();
407 usemap
= sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nodeid
),
408 size
* usemap_count
);
410 pr_warn("%s: allocation failed\n", __func__
);
414 for (pnum
= pnum_begin
; pnum
< pnum_end
; pnum
++) {
415 if (!present_section_nr(pnum
))
417 usemap_map
[pnum
] = usemap
;
419 check_usemap_section_nr(nodeid
, usemap_map
[pnum
]);
423 #ifndef CONFIG_SPARSEMEM_VMEMMAP
424 struct page __init
*sparse_mem_map_populate(unsigned long pnum
, int nid
)
429 map
= alloc_remap(nid
, sizeof(struct page
) * PAGES_PER_SECTION
);
433 size
= PAGE_ALIGN(sizeof(struct page
) * PAGES_PER_SECTION
);
434 map
= memblock_virt_alloc_try_nid(size
,
435 PAGE_SIZE
, __pa(MAX_DMA_ADDRESS
),
436 BOOTMEM_ALLOC_ACCESSIBLE
, nid
);
439 void __init
sparse_mem_maps_populate_node(struct page
**map_map
,
440 unsigned long pnum_begin
,
441 unsigned long pnum_end
,
442 unsigned long map_count
, int nodeid
)
446 unsigned long size
= sizeof(struct page
) * PAGES_PER_SECTION
;
448 map
= alloc_remap(nodeid
, size
* map_count
);
450 for (pnum
= pnum_begin
; pnum
< pnum_end
; pnum
++) {
451 if (!present_section_nr(pnum
))
459 size
= PAGE_ALIGN(size
);
460 map
= memblock_virt_alloc_try_nid_raw(size
* map_count
,
461 PAGE_SIZE
, __pa(MAX_DMA_ADDRESS
),
462 BOOTMEM_ALLOC_ACCESSIBLE
, nodeid
);
464 for (pnum
= pnum_begin
; pnum
< pnum_end
; pnum
++) {
465 if (!present_section_nr(pnum
))
474 for (pnum
= pnum_begin
; pnum
< pnum_end
; pnum
++) {
475 struct mem_section
*ms
;
477 if (!present_section_nr(pnum
))
479 map_map
[pnum
] = sparse_mem_map_populate(pnum
, nodeid
);
482 ms
= __nr_to_section(pnum
);
483 pr_err("%s: sparsemem memory map backing failed some memory will not be available\n",
485 ms
->section_mem_map
= 0;
488 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */
490 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
491 static void __init
sparse_early_mem_maps_alloc_node(void *data
,
492 unsigned long pnum_begin
,
493 unsigned long pnum_end
,
494 unsigned long map_count
, int nodeid
)
496 struct page
**map_map
= (struct page
**)data
;
497 sparse_mem_maps_populate_node(map_map
, pnum_begin
, pnum_end
,
501 static struct page __init
*sparse_early_mem_map_alloc(unsigned long pnum
)
504 struct mem_section
*ms
= __nr_to_section(pnum
);
505 int nid
= sparse_early_nid(ms
);
507 map
= sparse_mem_map_populate(pnum
, nid
);
511 pr_err("%s: sparsemem memory map backing failed some memory will not be available\n",
513 ms
->section_mem_map
= 0;
518 void __weak __meminit
vmemmap_populate_print_last(void)
523 * alloc_usemap_and_memmap - memory alloction for pageblock flags and vmemmap
524 * @map: usemap_map for pageblock flags or mmap_map for vmemmap
526 static void __init
alloc_usemap_and_memmap(void (*alloc_func
)
527 (void *, unsigned long, unsigned long,
528 unsigned long, int), void *data
)
531 unsigned long map_count
;
532 int nodeid_begin
= 0;
533 unsigned long pnum_begin
= 0;
535 for_each_present_section_nr(0, pnum
) {
536 struct mem_section
*ms
;
538 ms
= __nr_to_section(pnum
);
539 nodeid_begin
= sparse_early_nid(ms
);
544 for_each_present_section_nr(pnum_begin
+ 1, pnum
) {
545 struct mem_section
*ms
;
548 ms
= __nr_to_section(pnum
);
549 nodeid
= sparse_early_nid(ms
);
550 if (nodeid
== nodeid_begin
) {
554 /* ok, we need to take cake of from pnum_begin to pnum - 1*/
555 alloc_func(data
, pnum_begin
, pnum
,
556 map_count
, nodeid_begin
);
557 /* new start, update count etc*/
558 nodeid_begin
= nodeid
;
563 alloc_func(data
, pnum_begin
, NR_MEM_SECTIONS
,
564 map_count
, nodeid_begin
);
568 * Allocate the accumulated non-linear sections, allocate a mem_map
569 * for each and record the physical to section mapping.
571 void __init
sparse_init(void)
575 unsigned long *usemap
;
576 unsigned long **usemap_map
;
578 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
580 struct page
**map_map
;
583 /* see include/linux/mmzone.h 'struct mem_section' definition */
584 BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section
)));
586 /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */
587 set_pageblock_order();
590 * map is using big page (aka 2M in x86 64 bit)
591 * usemap is less one page (aka 24 bytes)
592 * so alloc 2M (with 2M align) and 24 bytes in turn will
593 * make next 2M slip to one more 2M later.
594 * then in big system, the memory will have a lot of holes...
595 * here try to allocate 2M pages continuously.
597 * powerpc need to call sparse_init_one_section right after each
598 * sparse_early_mem_map_alloc, so allocate usemap_map at first.
600 size
= sizeof(unsigned long *) * NR_MEM_SECTIONS
;
601 usemap_map
= memblock_virt_alloc(size
, 0);
603 panic("can not allocate usemap_map\n");
604 alloc_usemap_and_memmap(sparse_early_usemaps_alloc_node
,
607 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
608 size2
= sizeof(struct page
*) * NR_MEM_SECTIONS
;
609 map_map
= memblock_virt_alloc(size2
, 0);
611 panic("can not allocate map_map\n");
612 alloc_usemap_and_memmap(sparse_early_mem_maps_alloc_node
,
616 for_each_present_section_nr(0, pnum
) {
617 usemap
= usemap_map
[pnum
];
621 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
624 map
= sparse_early_mem_map_alloc(pnum
);
629 sparse_init_one_section(__nr_to_section(pnum
), pnum
, map
,
633 vmemmap_populate_print_last();
635 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
636 memblock_free_early(__pa(map_map
), size2
);
638 memblock_free_early(__pa(usemap_map
), size
);
641 #ifdef CONFIG_MEMORY_HOTPLUG
643 /* Mark all memory sections within the pfn range as online */
644 void online_mem_sections(unsigned long start_pfn
, unsigned long end_pfn
)
648 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= PAGES_PER_SECTION
) {
649 unsigned long section_nr
= pfn_to_section_nr(pfn
);
650 struct mem_section
*ms
;
652 /* onlining code should never touch invalid ranges */
653 if (WARN_ON(!valid_section_nr(section_nr
)))
656 ms
= __nr_to_section(section_nr
);
657 ms
->section_mem_map
|= SECTION_IS_ONLINE
;
661 #ifdef CONFIG_MEMORY_HOTREMOVE
662 /* Mark all memory sections within the pfn range as online */
663 void offline_mem_sections(unsigned long start_pfn
, unsigned long end_pfn
)
667 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= PAGES_PER_SECTION
) {
668 unsigned long section_nr
= pfn_to_section_nr(start_pfn
);
669 struct mem_section
*ms
;
672 * TODO this needs some double checking. Offlining code makes
673 * sure to check pfn_valid but those checks might be just bogus
675 if (WARN_ON(!valid_section_nr(section_nr
)))
678 ms
= __nr_to_section(section_nr
);
679 ms
->section_mem_map
&= ~SECTION_IS_ONLINE
;
684 #ifdef CONFIG_SPARSEMEM_VMEMMAP
685 static inline struct page
*kmalloc_section_memmap(unsigned long pnum
, int nid
)
687 /* This will make the necessary allocations eventually. */
688 return sparse_mem_map_populate(pnum
, nid
);
690 static void __kfree_section_memmap(struct page
*memmap
)
692 unsigned long start
= (unsigned long)memmap
;
693 unsigned long end
= (unsigned long)(memmap
+ PAGES_PER_SECTION
);
695 vmemmap_free(start
, end
);
697 #ifdef CONFIG_MEMORY_HOTREMOVE
698 static void free_map_bootmem(struct page
*memmap
)
700 unsigned long start
= (unsigned long)memmap
;
701 unsigned long end
= (unsigned long)(memmap
+ PAGES_PER_SECTION
);
703 vmemmap_free(start
, end
);
705 #endif /* CONFIG_MEMORY_HOTREMOVE */
707 static struct page
*__kmalloc_section_memmap(void)
709 struct page
*page
, *ret
;
710 unsigned long memmap_size
= sizeof(struct page
) * PAGES_PER_SECTION
;
712 page
= alloc_pages(GFP_KERNEL
|__GFP_NOWARN
, get_order(memmap_size
));
716 ret
= vmalloc(memmap_size
);
722 ret
= (struct page
*)pfn_to_kaddr(page_to_pfn(page
));
728 static inline struct page
*kmalloc_section_memmap(unsigned long pnum
, int nid
)
730 return __kmalloc_section_memmap();
733 static void __kfree_section_memmap(struct page
*memmap
)
735 if (is_vmalloc_addr(memmap
))
738 free_pages((unsigned long)memmap
,
739 get_order(sizeof(struct page
) * PAGES_PER_SECTION
));
742 #ifdef CONFIG_MEMORY_HOTREMOVE
743 static void free_map_bootmem(struct page
*memmap
)
745 unsigned long maps_section_nr
, removing_section_nr
, i
;
746 unsigned long magic
, nr_pages
;
747 struct page
*page
= virt_to_page(memmap
);
749 nr_pages
= PAGE_ALIGN(PAGES_PER_SECTION
* sizeof(struct page
))
752 for (i
= 0; i
< nr_pages
; i
++, page
++) {
753 magic
= (unsigned long) page
->freelist
;
755 BUG_ON(magic
== NODE_INFO
);
757 maps_section_nr
= pfn_to_section_nr(page_to_pfn(page
));
758 removing_section_nr
= page_private(page
);
761 * When this function is called, the removing section is
762 * logical offlined state. This means all pages are isolated
763 * from page allocator. If removing section's memmap is placed
764 * on the same section, it must not be freed.
765 * If it is freed, page allocator may allocate it which will
766 * be removed physically soon.
768 if (maps_section_nr
!= removing_section_nr
)
769 put_page_bootmem(page
);
772 #endif /* CONFIG_MEMORY_HOTREMOVE */
773 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
776 * returns the number of sections whose mem_maps were properly
777 * set. If this is <=0, then that means that the passed-in
778 * map was not consumed and must be freed.
780 int __meminit
sparse_add_one_section(struct pglist_data
*pgdat
, unsigned long start_pfn
)
782 unsigned long section_nr
= pfn_to_section_nr(start_pfn
);
783 struct mem_section
*ms
;
785 unsigned long *usemap
;
790 * no locking for this, because it does its own
791 * plus, it does a kmalloc
793 ret
= sparse_index_init(section_nr
, pgdat
->node_id
);
794 if (ret
< 0 && ret
!= -EEXIST
)
796 memmap
= kmalloc_section_memmap(section_nr
, pgdat
->node_id
);
799 usemap
= __kmalloc_section_usemap();
801 __kfree_section_memmap(memmap
);
805 pgdat_resize_lock(pgdat
, &flags
);
807 ms
= __pfn_to_section(start_pfn
);
808 if (ms
->section_mem_map
& SECTION_MARKED_PRESENT
) {
813 memset(memmap
, 0, sizeof(struct page
) * PAGES_PER_SECTION
);
815 section_mark_present(ms
);
817 ret
= sparse_init_one_section(ms
, section_nr
, memmap
, usemap
);
820 pgdat_resize_unlock(pgdat
, &flags
);
823 __kfree_section_memmap(memmap
);
828 #ifdef CONFIG_MEMORY_HOTREMOVE
829 #ifdef CONFIG_MEMORY_FAILURE
830 static void clear_hwpoisoned_pages(struct page
*memmap
, int nr_pages
)
837 for (i
= 0; i
< nr_pages
; i
++) {
838 if (PageHWPoison(&memmap
[i
])) {
839 atomic_long_sub(1, &num_poisoned_pages
);
840 ClearPageHWPoison(&memmap
[i
]);
845 static inline void clear_hwpoisoned_pages(struct page
*memmap
, int nr_pages
)
850 static void free_section_usemap(struct page
*memmap
, unsigned long *usemap
)
852 struct page
*usemap_page
;
857 usemap_page
= virt_to_page(usemap
);
859 * Check to see if allocation came from hot-plug-add
861 if (PageSlab(usemap_page
) || PageCompound(usemap_page
)) {
864 __kfree_section_memmap(memmap
);
869 * The usemap came from bootmem. This is packed with other usemaps
870 * on the section which has pgdat at boot time. Just keep it as is now.
874 free_map_bootmem(memmap
);
877 void sparse_remove_one_section(struct zone
*zone
, struct mem_section
*ms
,
878 unsigned long map_offset
)
880 struct page
*memmap
= NULL
;
881 unsigned long *usemap
= NULL
, flags
;
882 struct pglist_data
*pgdat
= zone
->zone_pgdat
;
884 pgdat_resize_lock(pgdat
, &flags
);
885 if (ms
->section_mem_map
) {
886 usemap
= ms
->pageblock_flags
;
887 memmap
= sparse_decode_mem_map(ms
->section_mem_map
,
889 ms
->section_mem_map
= 0;
890 ms
->pageblock_flags
= NULL
;
892 pgdat_resize_unlock(pgdat
, &flags
);
894 clear_hwpoisoned_pages(memmap
+ map_offset
,
895 PAGES_PER_SECTION
- map_offset
);
896 free_section_usemap(memmap
, usemap
);
898 #endif /* CONFIG_MEMORY_HOTREMOVE */
899 #endif /* CONFIG_MEMORY_HOTPLUG */