1 // SPDX-License-Identifier: GPL-2.0-only
3 * linux/mm/memory_hotplug.c
8 #include <linux/stddef.h>
10 #include <linux/sched/signal.h>
11 #include <linux/swap.h>
12 #include <linux/interrupt.h>
13 #include <linux/pagemap.h>
14 #include <linux/compiler.h>
15 #include <linux/export.h>
16 #include <linux/pagevec.h>
17 #include <linux/writeback.h>
18 #include <linux/slab.h>
19 #include <linux/sysctl.h>
20 #include <linux/cpu.h>
21 #include <linux/memory.h>
22 #include <linux/memremap.h>
23 #include <linux/memory_hotplug.h>
24 #include <linux/highmem.h>
25 #include <linux/vmalloc.h>
26 #include <linux/ioport.h>
27 #include <linux/delay.h>
28 #include <linux/migrate.h>
29 #include <linux/page-isolation.h>
30 #include <linux/pfn.h>
31 #include <linux/suspend.h>
32 #include <linux/mm_inline.h>
33 #include <linux/firmware-map.h>
34 #include <linux/stop_machine.h>
35 #include <linux/hugetlb.h>
36 #include <linux/memblock.h>
37 #include <linux/compaction.h>
38 #include <linux/rmap.h>
40 #include <asm/tlbflush.h>
46 * online_page_callback contains pointer to current page onlining function.
47 * Initially it is generic_online_page(). If it is required it could be
48 * changed by calling set_online_page_callback() for callback registration
49 * and restore_online_page_callback() for generic callback restore.
52 static void generic_online_page(struct page
*page
, unsigned int order
);
54 static online_page_callback_t online_page_callback
= generic_online_page
;
55 static DEFINE_MUTEX(online_page_callback_lock
);
57 DEFINE_STATIC_PERCPU_RWSEM(mem_hotplug_lock
);
59 static int default_kernel_zone
= ZONE_NORMAL
;
61 void get_online_mems(void)
63 percpu_down_read(&mem_hotplug_lock
);
66 void put_online_mems(void)
68 percpu_up_read(&mem_hotplug_lock
);
71 bool movable_node_enabled
= false;
73 #ifndef CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE
74 bool memhp_auto_online
;
76 bool memhp_auto_online
= true;
78 EXPORT_SYMBOL_GPL(memhp_auto_online
);
80 static int __init
setup_memhp_default_state(char *str
)
82 if (!strcmp(str
, "online"))
83 memhp_auto_online
= true;
84 else if (!strcmp(str
, "offline"))
85 memhp_auto_online
= false;
89 __setup("memhp_default_state=", setup_memhp_default_state
);
91 void mem_hotplug_begin(void)
94 percpu_down_write(&mem_hotplug_lock
);
97 void mem_hotplug_done(void)
99 percpu_up_write(&mem_hotplug_lock
);
103 u64 max_mem_size
= U64_MAX
;
105 /* add this memory to iomem resource */
106 static struct resource
*register_memory_resource(u64 start
, u64 size
)
108 struct resource
*res
;
109 unsigned long flags
= IORESOURCE_SYSTEM_RAM
| IORESOURCE_BUSY
;
110 char *resource_name
= "System RAM";
112 if (start
+ size
> max_mem_size
)
113 return ERR_PTR(-E2BIG
);
116 * Request ownership of the new memory range. This might be
117 * a child of an existing resource that was present but
118 * not marked as busy.
120 res
= __request_region(&iomem_resource
, start
, size
,
121 resource_name
, flags
);
124 pr_debug("Unable to reserve System RAM region: %016llx->%016llx\n",
125 start
, start
+ size
);
126 return ERR_PTR(-EEXIST
);
131 static void release_memory_resource(struct resource
*res
)
135 release_resource(res
);
139 #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
140 void get_page_bootmem(unsigned long info
, struct page
*page
,
143 page
->freelist
= (void *)type
;
144 SetPagePrivate(page
);
145 set_page_private(page
, info
);
149 void put_page_bootmem(struct page
*page
)
153 type
= (unsigned long) page
->freelist
;
154 BUG_ON(type
< MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE
||
155 type
> MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE
);
157 if (page_ref_dec_return(page
) == 1) {
158 page
->freelist
= NULL
;
159 ClearPagePrivate(page
);
160 set_page_private(page
, 0);
161 INIT_LIST_HEAD(&page
->lru
);
162 free_reserved_page(page
);
166 #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE
167 #ifndef CONFIG_SPARSEMEM_VMEMMAP
168 static void register_page_bootmem_info_section(unsigned long start_pfn
)
170 unsigned long mapsize
, section_nr
, i
;
171 struct mem_section
*ms
;
172 struct page
*page
, *memmap
;
173 struct mem_section_usage
*usage
;
175 section_nr
= pfn_to_section_nr(start_pfn
);
176 ms
= __nr_to_section(section_nr
);
178 /* Get section's memmap address */
179 memmap
= sparse_decode_mem_map(ms
->section_mem_map
, section_nr
);
182 * Get page for the memmap's phys address
183 * XXX: need more consideration for sparse_vmemmap...
185 page
= virt_to_page(memmap
);
186 mapsize
= sizeof(struct page
) * PAGES_PER_SECTION
;
187 mapsize
= PAGE_ALIGN(mapsize
) >> PAGE_SHIFT
;
189 /* remember memmap's page */
190 for (i
= 0; i
< mapsize
; i
++, page
++)
191 get_page_bootmem(section_nr
, page
, SECTION_INFO
);
194 page
= virt_to_page(usage
);
196 mapsize
= PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT
;
198 for (i
= 0; i
< mapsize
; i
++, page
++)
199 get_page_bootmem(section_nr
, page
, MIX_SECTION_INFO
);
202 #else /* CONFIG_SPARSEMEM_VMEMMAP */
203 static void register_page_bootmem_info_section(unsigned long start_pfn
)
205 unsigned long mapsize
, section_nr
, i
;
206 struct mem_section
*ms
;
207 struct page
*page
, *memmap
;
208 struct mem_section_usage
*usage
;
210 section_nr
= pfn_to_section_nr(start_pfn
);
211 ms
= __nr_to_section(section_nr
);
213 memmap
= sparse_decode_mem_map(ms
->section_mem_map
, section_nr
);
215 register_page_bootmem_memmap(section_nr
, memmap
, PAGES_PER_SECTION
);
218 page
= virt_to_page(usage
);
220 mapsize
= PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT
;
222 for (i
= 0; i
< mapsize
; i
++, page
++)
223 get_page_bootmem(section_nr
, page
, MIX_SECTION_INFO
);
225 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */
227 void __init
register_page_bootmem_info_node(struct pglist_data
*pgdat
)
229 unsigned long i
, pfn
, end_pfn
, nr_pages
;
230 int node
= pgdat
->node_id
;
233 nr_pages
= PAGE_ALIGN(sizeof(struct pglist_data
)) >> PAGE_SHIFT
;
234 page
= virt_to_page(pgdat
);
236 for (i
= 0; i
< nr_pages
; i
++, page
++)
237 get_page_bootmem(node
, page
, NODE_INFO
);
239 pfn
= pgdat
->node_start_pfn
;
240 end_pfn
= pgdat_end_pfn(pgdat
);
242 /* register section info */
243 for (; pfn
< end_pfn
; pfn
+= PAGES_PER_SECTION
) {
245 * Some platforms can assign the same pfn to multiple nodes - on
246 * node0 as well as nodeN. To avoid registering a pfn against
247 * multiple nodes we check that this pfn does not already
248 * reside in some other nodes.
250 if (pfn_valid(pfn
) && (early_pfn_to_nid(pfn
) == node
))
251 register_page_bootmem_info_section(pfn
);
254 #endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */
256 static int check_pfn_span(unsigned long pfn
, unsigned long nr_pages
,
260 * Disallow all operations smaller than a sub-section and only
261 * allow operations smaller than a section for
262 * SPARSEMEM_VMEMMAP. Note that check_hotplug_memory_range()
263 * enforces a larger memory_block_size_bytes() granularity for
264 * memory that will be marked online, so this check should only
265 * fire for direct arch_{add,remove}_memory() users outside of
266 * add_memory_resource().
268 unsigned long min_align
;
270 if (IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP
))
271 min_align
= PAGES_PER_SUBSECTION
;
273 min_align
= PAGES_PER_SECTION
;
274 if (!IS_ALIGNED(pfn
, min_align
)
275 || !IS_ALIGNED(nr_pages
, min_align
)) {
276 WARN(1, "Misaligned __%s_pages start: %#lx end: #%lx\n",
277 reason
, pfn
, pfn
+ nr_pages
- 1);
284 * Reasonably generic function for adding memory. It is
285 * expected that archs that support memory hotplug will
286 * call this function after deciding the zone to which to
289 int __ref
__add_pages(int nid
, unsigned long pfn
, unsigned long nr_pages
,
290 struct mhp_restrictions
*restrictions
)
293 unsigned long nr
, start_sec
, end_sec
;
294 struct vmem_altmap
*altmap
= restrictions
->altmap
;
298 * Validate altmap is within bounds of the total request
300 if (altmap
->base_pfn
!= pfn
301 || vmem_altmap_offset(altmap
) > nr_pages
) {
302 pr_warn_once("memory add fail, invalid altmap\n");
308 err
= check_pfn_span(pfn
, nr_pages
, "add");
312 start_sec
= pfn_to_section_nr(pfn
);
313 end_sec
= pfn_to_section_nr(pfn
+ nr_pages
- 1);
314 for (nr
= start_sec
; nr
<= end_sec
; nr
++) {
317 pfns
= min(nr_pages
, PAGES_PER_SECTION
318 - (pfn
& ~PAGE_SECTION_MASK
));
319 err
= sparse_add_section(nid
, pfn
, pfns
, altmap
);
326 vmemmap_populate_print_last();
330 /* find the smallest valid pfn in the range [start_pfn, end_pfn) */
331 static unsigned long find_smallest_section_pfn(int nid
, struct zone
*zone
,
332 unsigned long start_pfn
,
333 unsigned long end_pfn
)
335 for (; start_pfn
< end_pfn
; start_pfn
+= PAGES_PER_SUBSECTION
) {
336 if (unlikely(!pfn_to_online_page(start_pfn
)))
339 if (unlikely(pfn_to_nid(start_pfn
) != nid
))
342 if (zone
&& zone
!= page_zone(pfn_to_page(start_pfn
)))
351 /* find the biggest valid pfn in the range [start_pfn, end_pfn). */
352 static unsigned long find_biggest_section_pfn(int nid
, struct zone
*zone
,
353 unsigned long start_pfn
,
354 unsigned long end_pfn
)
358 /* pfn is the end pfn of a memory section. */
360 for (; pfn
>= start_pfn
; pfn
-= PAGES_PER_SUBSECTION
) {
361 if (unlikely(!pfn_to_online_page(pfn
)))
364 if (unlikely(pfn_to_nid(pfn
) != nid
))
367 if (zone
&& zone
!= page_zone(pfn_to_page(pfn
)))
376 static void shrink_zone_span(struct zone
*zone
, unsigned long start_pfn
,
377 unsigned long end_pfn
)
379 unsigned long zone_start_pfn
= zone
->zone_start_pfn
;
380 unsigned long z
= zone_end_pfn(zone
); /* zone_end_pfn namespace clash */
381 unsigned long zone_end_pfn
= z
;
383 int nid
= zone_to_nid(zone
);
385 zone_span_writelock(zone
);
386 if (zone_start_pfn
== start_pfn
) {
388 * If the section is smallest section in the zone, it need
389 * shrink zone->zone_start_pfn and zone->zone_spanned_pages.
390 * In this case, we find second smallest valid mem_section
391 * for shrinking zone.
393 pfn
= find_smallest_section_pfn(nid
, zone
, end_pfn
,
396 zone
->zone_start_pfn
= pfn
;
397 zone
->spanned_pages
= zone_end_pfn
- pfn
;
399 } else if (zone_end_pfn
== end_pfn
) {
401 * If the section is biggest section in the zone, it need
402 * shrink zone->spanned_pages.
403 * In this case, we find second biggest valid mem_section for
406 pfn
= find_biggest_section_pfn(nid
, zone
, zone_start_pfn
,
409 zone
->spanned_pages
= pfn
- zone_start_pfn
+ 1;
413 * The section is not biggest or smallest mem_section in the zone, it
414 * only creates a hole in the zone. So in this case, we need not
415 * change the zone. But perhaps, the zone has only hole data. Thus
416 * it check the zone has only hole or not.
418 pfn
= zone_start_pfn
;
419 for (; pfn
< zone_end_pfn
; pfn
+= PAGES_PER_SUBSECTION
) {
420 if (unlikely(!pfn_to_online_page(pfn
)))
423 if (page_zone(pfn_to_page(pfn
)) != zone
)
426 /* Skip range to be removed */
427 if (pfn
>= start_pfn
&& pfn
< end_pfn
)
430 /* If we find valid section, we have nothing to do */
431 zone_span_writeunlock(zone
);
435 /* The zone has no valid section */
436 zone
->zone_start_pfn
= 0;
437 zone
->spanned_pages
= 0;
438 zone_span_writeunlock(zone
);
441 static void update_pgdat_span(struct pglist_data
*pgdat
)
443 unsigned long node_start_pfn
= 0, node_end_pfn
= 0;
446 for (zone
= pgdat
->node_zones
;
447 zone
< pgdat
->node_zones
+ MAX_NR_ZONES
; zone
++) {
448 unsigned long zone_end_pfn
= zone
->zone_start_pfn
+
451 /* No need to lock the zones, they can't change. */
452 if (!zone
->spanned_pages
)
455 node_start_pfn
= zone
->zone_start_pfn
;
456 node_end_pfn
= zone_end_pfn
;
460 if (zone_end_pfn
> node_end_pfn
)
461 node_end_pfn
= zone_end_pfn
;
462 if (zone
->zone_start_pfn
< node_start_pfn
)
463 node_start_pfn
= zone
->zone_start_pfn
;
466 pgdat
->node_start_pfn
= node_start_pfn
;
467 pgdat
->node_spanned_pages
= node_end_pfn
- node_start_pfn
;
470 void __ref
remove_pfn_range_from_zone(struct zone
*zone
,
471 unsigned long start_pfn
,
472 unsigned long nr_pages
)
474 struct pglist_data
*pgdat
= zone
->zone_pgdat
;
477 #ifdef CONFIG_ZONE_DEVICE
479 * Zone shrinking code cannot properly deal with ZONE_DEVICE. So
480 * we will not try to shrink the zones - which is okay as
481 * set_zone_contiguous() cannot deal with ZONE_DEVICE either way.
483 if (zone_idx(zone
) == ZONE_DEVICE
)
487 clear_zone_contiguous(zone
);
489 pgdat_resize_lock(zone
->zone_pgdat
, &flags
);
490 shrink_zone_span(zone
, start_pfn
, start_pfn
+ nr_pages
);
491 update_pgdat_span(pgdat
);
492 pgdat_resize_unlock(zone
->zone_pgdat
, &flags
);
494 set_zone_contiguous(zone
);
497 static void __remove_section(unsigned long pfn
, unsigned long nr_pages
,
498 unsigned long map_offset
,
499 struct vmem_altmap
*altmap
)
501 struct mem_section
*ms
= __nr_to_section(pfn_to_section_nr(pfn
));
503 if (WARN_ON_ONCE(!valid_section(ms
)))
506 sparse_remove_section(ms
, pfn
, nr_pages
, map_offset
, altmap
);
510 * __remove_pages() - remove sections of pages
511 * @pfn: starting pageframe (must be aligned to start of a section)
512 * @nr_pages: number of pages to remove (must be multiple of section size)
513 * @altmap: alternative device page map or %NULL if default memmap is used
515 * Generic helper function to remove section mappings and sysfs entries
516 * for the section of the memory we are removing. Caller needs to make
517 * sure that pages are marked reserved and zones are adjust properly by
518 * calling offline_pages().
520 void __remove_pages(unsigned long pfn
, unsigned long nr_pages
,
521 struct vmem_altmap
*altmap
)
523 unsigned long map_offset
= 0;
524 unsigned long nr
, start_sec
, end_sec
;
526 map_offset
= vmem_altmap_offset(altmap
);
528 if (check_pfn_span(pfn
, nr_pages
, "remove"))
531 start_sec
= pfn_to_section_nr(pfn
);
532 end_sec
= pfn_to_section_nr(pfn
+ nr_pages
- 1);
533 for (nr
= start_sec
; nr
<= end_sec
; nr
++) {
537 pfns
= min(nr_pages
, PAGES_PER_SECTION
538 - (pfn
& ~PAGE_SECTION_MASK
));
539 __remove_section(pfn
, pfns
, map_offset
, altmap
);
546 int set_online_page_callback(online_page_callback_t callback
)
551 mutex_lock(&online_page_callback_lock
);
553 if (online_page_callback
== generic_online_page
) {
554 online_page_callback
= callback
;
558 mutex_unlock(&online_page_callback_lock
);
563 EXPORT_SYMBOL_GPL(set_online_page_callback
);
565 int restore_online_page_callback(online_page_callback_t callback
)
570 mutex_lock(&online_page_callback_lock
);
572 if (online_page_callback
== callback
) {
573 online_page_callback
= generic_online_page
;
577 mutex_unlock(&online_page_callback_lock
);
582 EXPORT_SYMBOL_GPL(restore_online_page_callback
);
584 void __online_page_set_limits(struct page
*page
)
587 EXPORT_SYMBOL_GPL(__online_page_set_limits
);
589 void __online_page_increment_counters(struct page
*page
)
591 adjust_managed_page_count(page
, 1);
593 EXPORT_SYMBOL_GPL(__online_page_increment_counters
);
595 void __online_page_free(struct page
*page
)
597 __free_reserved_page(page
);
599 EXPORT_SYMBOL_GPL(__online_page_free
);
601 static void generic_online_page(struct page
*page
, unsigned int order
)
604 * Freeing the page with debug_pagealloc enabled will try to unmap it,
605 * so we should map it first. This is better than introducing a special
606 * case in page freeing fast path.
608 if (debug_pagealloc_enabled_static())
609 kernel_map_pages(page
, 1 << order
, 1);
610 __free_pages_core(page
, order
);
611 totalram_pages_add(1UL << order
);
612 #ifdef CONFIG_HIGHMEM
613 if (PageHighMem(page
))
614 totalhigh_pages_add(1UL << order
);
618 static int online_pages_range(unsigned long start_pfn
, unsigned long nr_pages
,
621 const unsigned long end_pfn
= start_pfn
+ nr_pages
;
626 * Online the pages. The callback might decide to keep some pages
627 * PG_reserved (to add them to the buddy later), but we still account
628 * them as being online/belonging to this zone ("present").
630 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= 1ul << order
) {
631 order
= min(MAX_ORDER
- 1, get_order(PFN_PHYS(end_pfn
- pfn
)));
632 /* __free_pages_core() wants pfns to be aligned to the order */
633 if (WARN_ON_ONCE(!IS_ALIGNED(pfn
, 1ul << order
)))
635 (*online_page_callback
)(pfn_to_page(pfn
), order
);
638 /* mark all involved sections as online */
639 online_mem_sections(start_pfn
, end_pfn
);
641 *(unsigned long *)arg
+= nr_pages
;
645 /* check which state of node_states will be changed when online memory */
646 static void node_states_check_changes_online(unsigned long nr_pages
,
647 struct zone
*zone
, struct memory_notify
*arg
)
649 int nid
= zone_to_nid(zone
);
651 arg
->status_change_nid
= NUMA_NO_NODE
;
652 arg
->status_change_nid_normal
= NUMA_NO_NODE
;
653 arg
->status_change_nid_high
= NUMA_NO_NODE
;
655 if (!node_state(nid
, N_MEMORY
))
656 arg
->status_change_nid
= nid
;
657 if (zone_idx(zone
) <= ZONE_NORMAL
&& !node_state(nid
, N_NORMAL_MEMORY
))
658 arg
->status_change_nid_normal
= nid
;
659 #ifdef CONFIG_HIGHMEM
660 if (zone_idx(zone
) <= ZONE_HIGHMEM
&& !node_state(nid
, N_HIGH_MEMORY
))
661 arg
->status_change_nid_high
= nid
;
665 static void node_states_set_node(int node
, struct memory_notify
*arg
)
667 if (arg
->status_change_nid_normal
>= 0)
668 node_set_state(node
, N_NORMAL_MEMORY
);
670 if (arg
->status_change_nid_high
>= 0)
671 node_set_state(node
, N_HIGH_MEMORY
);
673 if (arg
->status_change_nid
>= 0)
674 node_set_state(node
, N_MEMORY
);
677 static void __meminit
resize_zone_range(struct zone
*zone
, unsigned long start_pfn
,
678 unsigned long nr_pages
)
680 unsigned long old_end_pfn
= zone_end_pfn(zone
);
682 if (zone_is_empty(zone
) || start_pfn
< zone
->zone_start_pfn
)
683 zone
->zone_start_pfn
= start_pfn
;
685 zone
->spanned_pages
= max(start_pfn
+ nr_pages
, old_end_pfn
) - zone
->zone_start_pfn
;
688 static void __meminit
resize_pgdat_range(struct pglist_data
*pgdat
, unsigned long start_pfn
,
689 unsigned long nr_pages
)
691 unsigned long old_end_pfn
= pgdat_end_pfn(pgdat
);
693 if (!pgdat
->node_spanned_pages
|| start_pfn
< pgdat
->node_start_pfn
)
694 pgdat
->node_start_pfn
= start_pfn
;
696 pgdat
->node_spanned_pages
= max(start_pfn
+ nr_pages
, old_end_pfn
) - pgdat
->node_start_pfn
;
700 * Associate the pfn range with the given zone, initializing the memmaps
701 * and resizing the pgdat/zone data to span the added pages. After this
702 * call, all affected pages are PG_reserved.
704 void __ref
move_pfn_range_to_zone(struct zone
*zone
, unsigned long start_pfn
,
705 unsigned long nr_pages
, struct vmem_altmap
*altmap
)
707 struct pglist_data
*pgdat
= zone
->zone_pgdat
;
708 int nid
= pgdat
->node_id
;
711 clear_zone_contiguous(zone
);
713 /* TODO Huh pgdat is irqsave while zone is not. It used to be like that before */
714 pgdat_resize_lock(pgdat
, &flags
);
715 zone_span_writelock(zone
);
716 if (zone_is_empty(zone
))
717 init_currently_empty_zone(zone
, start_pfn
, nr_pages
);
718 resize_zone_range(zone
, start_pfn
, nr_pages
);
719 zone_span_writeunlock(zone
);
720 resize_pgdat_range(pgdat
, start_pfn
, nr_pages
);
721 pgdat_resize_unlock(pgdat
, &flags
);
724 * TODO now we have a visible range of pages which are not associated
725 * with their zone properly. Not nice but set_pfnblock_flags_mask
726 * expects the zone spans the pfn range. All the pages in the range
727 * are reserved so nobody should be touching them so we should be safe
729 memmap_init_zone(nr_pages
, nid
, zone_idx(zone
), start_pfn
,
730 MEMINIT_HOTPLUG
, altmap
);
732 set_zone_contiguous(zone
);
735 void set_default_mem_hotplug_zone(enum zone_type zone
)
737 default_kernel_zone
= zone
;
740 #ifdef CONFIG_HIGHMEM
741 #define MAX_KERNEL_ZONE ZONE_HIGHMEM
743 #define MAX_KERNEL_ZONE ZONE_NORMAL
747 * Returns a default kernel memory zone for the given pfn range.
748 * If no kernel zone covers this pfn range it will automatically go
749 * to the MAX_KERNEL_ZONE.
751 static struct zone
*default_kernel_zone_for_pfn(int nid
, unsigned long start_pfn
,
752 unsigned long nr_pages
)
754 struct pglist_data
*pgdat
= NODE_DATA(nid
);
757 for (zid
= 0; zid
<= MAX_KERNEL_ZONE
; zid
++) {
758 struct zone
*zone
= &pgdat
->node_zones
[zid
];
760 if (zone_intersects(zone
, start_pfn
, nr_pages
))
764 return &pgdat
->node_zones
[default_kernel_zone
];
767 static inline struct zone
*default_zone_for_pfn(int nid
, unsigned long start_pfn
,
768 unsigned long nr_pages
)
770 struct zone
*kernel_zone
= default_kernel_zone_for_pfn(nid
, start_pfn
,
772 struct zone
*movable_zone
= &NODE_DATA(nid
)->node_zones
[ZONE_MOVABLE
];
773 bool in_kernel
= zone_intersects(kernel_zone
, start_pfn
, nr_pages
);
774 bool in_movable
= zone_intersects(movable_zone
, start_pfn
, nr_pages
);
777 * We inherit the existing zone in a simple case where zones do not
778 * overlap in the given range
780 if (in_kernel
^ in_movable
)
781 return (in_kernel
) ? kernel_zone
: movable_zone
;
784 * If the range doesn't belong to any zone or two zones overlap in the
785 * given range then we use movable zone only if movable_node is
786 * enabled because we always online to a kernel zone by default.
788 return movable_node_enabled
? movable_zone
: kernel_zone
;
791 struct zone
*zone_for_pfn_range(int online_type
, int nid
,
792 unsigned long start_pfn
, unsigned long nr_pages
)
794 if (online_type
== MMOP_ONLINE_KERNEL
)
795 return default_kernel_zone_for_pfn(nid
, start_pfn
, nr_pages
);
797 if (online_type
== MMOP_ONLINE_MOVABLE
)
798 return &NODE_DATA(nid
)->node_zones
[ZONE_MOVABLE
];
800 return default_zone_for_pfn(nid
, start_pfn
, nr_pages
);
803 int __ref
online_pages(unsigned long pfn
, unsigned long nr_pages
, int online_type
)
806 unsigned long onlined_pages
= 0;
808 int need_zonelists_rebuild
= 0;
811 struct memory_notify arg
;
812 struct memory_block
*mem
;
817 * We can't use pfn_to_nid() because nid might be stored in struct page
818 * which is not yet initialized. Instead, we find nid from memory block.
820 mem
= find_memory_block(__pfn_to_section(pfn
));
822 put_device(&mem
->dev
);
824 /* associate pfn range with the zone */
825 zone
= zone_for_pfn_range(online_type
, nid
, pfn
, nr_pages
);
826 move_pfn_range_to_zone(zone
, pfn
, nr_pages
, NULL
);
829 arg
.nr_pages
= nr_pages
;
830 node_states_check_changes_online(nr_pages
, zone
, &arg
);
832 ret
= memory_notify(MEM_GOING_ONLINE
, &arg
);
833 ret
= notifier_to_errno(ret
);
835 goto failed_addition
;
838 * If this zone is not populated, then it is not in zonelist.
839 * This means the page allocator ignores this zone.
840 * So, zonelist must be updated after online.
842 if (!populated_zone(zone
)) {
843 need_zonelists_rebuild
= 1;
844 setup_zone_pageset(zone
);
847 ret
= walk_system_ram_range(pfn
, nr_pages
, &onlined_pages
,
850 /* not a single memory resource was applicable */
851 if (need_zonelists_rebuild
)
852 zone_pcp_reset(zone
);
853 goto failed_addition
;
856 zone
->present_pages
+= onlined_pages
;
858 pgdat_resize_lock(zone
->zone_pgdat
, &flags
);
859 zone
->zone_pgdat
->node_present_pages
+= onlined_pages
;
860 pgdat_resize_unlock(zone
->zone_pgdat
, &flags
);
864 node_states_set_node(nid
, &arg
);
865 if (need_zonelists_rebuild
)
866 build_all_zonelists(NULL
);
868 zone_pcp_update(zone
);
870 init_per_zone_wmark_min();
875 vm_total_pages
= nr_free_pagecache_pages();
877 writeback_set_ratelimit();
879 memory_notify(MEM_ONLINE
, &arg
);
884 pr_debug("online_pages [mem %#010llx-%#010llx] failed\n",
885 (unsigned long long) pfn
<< PAGE_SHIFT
,
886 (((unsigned long long) pfn
+ nr_pages
) << PAGE_SHIFT
) - 1);
887 memory_notify(MEM_CANCEL_ONLINE
, &arg
);
888 remove_pfn_range_from_zone(zone
, pfn
, nr_pages
);
892 #endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */
894 static void reset_node_present_pages(pg_data_t
*pgdat
)
898 for (z
= pgdat
->node_zones
; z
< pgdat
->node_zones
+ MAX_NR_ZONES
; z
++)
899 z
->present_pages
= 0;
901 pgdat
->node_present_pages
= 0;
904 /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */
905 static pg_data_t __ref
*hotadd_new_pgdat(int nid
, u64 start
)
907 struct pglist_data
*pgdat
;
908 unsigned long start_pfn
= PFN_DOWN(start
);
910 pgdat
= NODE_DATA(nid
);
912 pgdat
= arch_alloc_nodedata(nid
);
916 pgdat
->per_cpu_nodestats
=
917 alloc_percpu(struct per_cpu_nodestat
);
918 arch_refresh_nodedata(nid
, pgdat
);
922 * Reset the nr_zones, order and classzone_idx before reuse.
923 * Note that kswapd will init kswapd_classzone_idx properly
924 * when it starts in the near future.
927 pgdat
->kswapd_order
= 0;
928 pgdat
->kswapd_classzone_idx
= 0;
929 for_each_online_cpu(cpu
) {
930 struct per_cpu_nodestat
*p
;
932 p
= per_cpu_ptr(pgdat
->per_cpu_nodestats
, cpu
);
933 memset(p
, 0, sizeof(*p
));
937 /* we can use NODE_DATA(nid) from here */
939 pgdat
->node_id
= nid
;
940 pgdat
->node_start_pfn
= start_pfn
;
942 /* init node's zones as empty zones, we don't have any present pages.*/
943 free_area_init_core_hotplug(nid
);
946 * The node we allocated has no zone fallback lists. For avoiding
947 * to access not-initialized zonelist, build here.
949 build_all_zonelists(pgdat
);
952 * When memory is hot-added, all the memory is in offline state. So
953 * clear all zones' present_pages because they will be updated in
954 * online_pages() and offline_pages().
956 reset_node_managed_pages(pgdat
);
957 reset_node_present_pages(pgdat
);
962 static void rollback_node_hotadd(int nid
)
964 pg_data_t
*pgdat
= NODE_DATA(nid
);
966 arch_refresh_nodedata(nid
, NULL
);
967 free_percpu(pgdat
->per_cpu_nodestats
);
968 arch_free_nodedata(pgdat
);
973 * try_online_node - online a node if offlined
975 * @start: start addr of the node
976 * @set_node_online: Whether we want to online the node
977 * called by cpu_up() to online a node without onlined memory.
980 * 1 -> a new node has been allocated
981 * 0 -> the node is already online
982 * -ENOMEM -> the node could not be allocated
984 static int __try_online_node(int nid
, u64 start
, bool set_node_online
)
989 if (node_online(nid
))
992 pgdat
= hotadd_new_pgdat(nid
, start
);
994 pr_err("Cannot online node %d due to NULL pgdat\n", nid
);
999 if (set_node_online
) {
1000 node_set_online(nid
);
1001 ret
= register_one_node(nid
);
1009 * Users of this function always want to online/register the node
1011 int try_online_node(int nid
)
1015 mem_hotplug_begin();
1016 ret
= __try_online_node(nid
, 0, true);
1021 static int check_hotplug_memory_range(u64 start
, u64 size
)
1023 /* memory range must be block size aligned */
1024 if (!size
|| !IS_ALIGNED(start
, memory_block_size_bytes()) ||
1025 !IS_ALIGNED(size
, memory_block_size_bytes())) {
1026 pr_err("Block size [%#lx] unaligned hotplug range: start %#llx, size %#llx",
1027 memory_block_size_bytes(), start
, size
);
1034 static int online_memory_block(struct memory_block
*mem
, void *arg
)
1036 return device_online(&mem
->dev
);
1040 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
1041 * and online/offline operations (triggered e.g. by sysfs).
1043 * we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG
1045 int __ref
add_memory_resource(int nid
, struct resource
*res
)
1047 struct mhp_restrictions restrictions
= {};
1049 bool new_node
= false;
1053 size
= resource_size(res
);
1055 ret
= check_hotplug_memory_range(start
, size
);
1059 mem_hotplug_begin();
1062 * Add new range to memblock so that when hotadd_new_pgdat() is called
1063 * to allocate new pgdat, get_pfn_range_for_nid() will be able to find
1064 * this new range and calculate total pages correctly. The range will
1065 * be removed at hot-remove time.
1067 memblock_add_node(start
, size
, nid
);
1069 ret
= __try_online_node(nid
, start
, false);
1074 /* call arch's memory hotadd */
1075 ret
= arch_add_memory(nid
, start
, size
, &restrictions
);
1079 /* create memory block devices after memory was added */
1080 ret
= create_memory_block_devices(start
, size
);
1082 arch_remove_memory(nid
, start
, size
, NULL
);
1087 /* If sysfs file of new node can't be created, cpu on the node
1088 * can't be hot-added. There is no rollback way now.
1089 * So, check by BUG_ON() to catch it reluctantly..
1090 * We online node here. We can't roll back from here.
1092 node_set_online(nid
);
1093 ret
= __register_one_node(nid
);
1097 /* link memory sections under this node.*/
1098 ret
= link_mem_sections(nid
, PFN_DOWN(start
), PFN_UP(start
+ size
- 1),
1102 /* create new memmap entry */
1103 firmware_map_add_hotplug(start
, start
+ size
, "System RAM");
1105 /* device_online() will take the lock when calling online_pages() */
1108 /* online pages if requested */
1109 if (memhp_auto_online
)
1110 walk_memory_blocks(start
, size
, NULL
, online_memory_block
);
1114 /* rollback pgdat allocation and others */
1116 rollback_node_hotadd(nid
);
1117 memblock_remove(start
, size
);
1122 /* requires device_hotplug_lock, see add_memory_resource() */
1123 int __ref
__add_memory(int nid
, u64 start
, u64 size
)
1125 struct resource
*res
;
1128 res
= register_memory_resource(start
, size
);
1130 return PTR_ERR(res
);
1132 ret
= add_memory_resource(nid
, res
);
1134 release_memory_resource(res
);
1138 int add_memory(int nid
, u64 start
, u64 size
)
1142 lock_device_hotplug();
1143 rc
= __add_memory(nid
, start
, size
);
1144 unlock_device_hotplug();
1148 EXPORT_SYMBOL_GPL(add_memory
);
1150 #ifdef CONFIG_MEMORY_HOTREMOVE
1152 * A free page on the buddy free lists (not the per-cpu lists) has PageBuddy
1153 * set and the size of the free page is given by page_order(). Using this,
1154 * the function determines if the pageblock contains only free pages.
1155 * Due to buddy contraints, a free page at least the size of a pageblock will
1156 * be located at the start of the pageblock
1158 static inline int pageblock_free(struct page
*page
)
1160 return PageBuddy(page
) && page_order(page
) >= pageblock_order
;
1163 /* Return the pfn of the start of the next active pageblock after a given pfn */
1164 static unsigned long next_active_pageblock(unsigned long pfn
)
1166 struct page
*page
= pfn_to_page(pfn
);
1168 /* Ensure the starting page is pageblock-aligned */
1169 BUG_ON(pfn
& (pageblock_nr_pages
- 1));
1171 /* If the entire pageblock is free, move to the end of free page */
1172 if (pageblock_free(page
)) {
1174 /* be careful. we don't have locks, page_order can be changed.*/
1175 order
= page_order(page
);
1176 if ((order
< MAX_ORDER
) && (order
>= pageblock_order
))
1177 return pfn
+ (1 << order
);
1180 return pfn
+ pageblock_nr_pages
;
1183 static bool is_pageblock_removable_nolock(unsigned long pfn
)
1185 struct page
*page
= pfn_to_page(pfn
);
1189 * We have to be careful here because we are iterating over memory
1190 * sections which are not zone aware so we might end up outside of
1191 * the zone but still within the section.
1192 * We have to take care about the node as well. If the node is offline
1193 * its NODE_DATA will be NULL - see page_zone.
1195 if (!node_online(page_to_nid(page
)))
1198 zone
= page_zone(page
);
1199 pfn
= page_to_pfn(page
);
1200 if (!zone_spans_pfn(zone
, pfn
))
1203 return !has_unmovable_pages(zone
, page
, 0, MIGRATE_MOVABLE
, SKIP_HWPOISON
);
1206 /* Checks if this range of memory is likely to be hot-removable. */
1207 bool is_mem_section_removable(unsigned long start_pfn
, unsigned long nr_pages
)
1209 unsigned long end_pfn
, pfn
;
1211 end_pfn
= min(start_pfn
+ nr_pages
,
1212 zone_end_pfn(page_zone(pfn_to_page(start_pfn
))));
1214 /* Check the starting page of each pageblock within the range */
1215 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
= next_active_pageblock(pfn
)) {
1216 if (!is_pageblock_removable_nolock(pfn
))
1221 /* All pageblocks in the memory block are likely to be hot-removable */
1226 * Confirm all pages in a range [start, end) belong to the same zone.
1227 * When true, return its valid [start, end).
1229 int test_pages_in_a_zone(unsigned long start_pfn
, unsigned long end_pfn
,
1230 unsigned long *valid_start
, unsigned long *valid_end
)
1232 unsigned long pfn
, sec_end_pfn
;
1233 unsigned long start
, end
;
1234 struct zone
*zone
= NULL
;
1237 for (pfn
= start_pfn
, sec_end_pfn
= SECTION_ALIGN_UP(start_pfn
+ 1);
1239 pfn
= sec_end_pfn
, sec_end_pfn
+= PAGES_PER_SECTION
) {
1240 /* Make sure the memory section is present first */
1241 if (!present_section_nr(pfn_to_section_nr(pfn
)))
1243 for (; pfn
< sec_end_pfn
&& pfn
< end_pfn
;
1244 pfn
+= MAX_ORDER_NR_PAGES
) {
1246 /* This is just a CONFIG_HOLES_IN_ZONE check.*/
1247 while ((i
< MAX_ORDER_NR_PAGES
) &&
1248 !pfn_valid_within(pfn
+ i
))
1250 if (i
== MAX_ORDER_NR_PAGES
|| pfn
+ i
>= end_pfn
)
1252 /* Check if we got outside of the zone */
1253 if (zone
&& !zone_spans_pfn(zone
, pfn
+ i
))
1255 page
= pfn_to_page(pfn
+ i
);
1256 if (zone
&& page_zone(page
) != zone
)
1260 zone
= page_zone(page
);
1261 end
= pfn
+ MAX_ORDER_NR_PAGES
;
1266 *valid_start
= start
;
1267 *valid_end
= min(end
, end_pfn
);
1275 * Scan pfn range [start,end) to find movable/migratable pages (LRU pages,
1276 * non-lru movable pages and hugepages). We scan pfn because it's much
1277 * easier than scanning over linked list. This function returns the pfn
1278 * of the first found movable page if it's found, otherwise 0.
1280 static unsigned long scan_movable_pages(unsigned long start
, unsigned long end
)
1284 for (pfn
= start
; pfn
< end
; pfn
++) {
1285 struct page
*page
, *head
;
1288 if (!pfn_valid(pfn
))
1290 page
= pfn_to_page(pfn
);
1293 if (__PageMovable(page
))
1296 if (!PageHuge(page
))
1298 head
= compound_head(page
);
1299 if (page_huge_active(head
))
1301 skip
= compound_nr(head
) - (page
- head
);
1307 static struct page
*new_node_page(struct page
*page
, unsigned long private)
1309 int nid
= page_to_nid(page
);
1310 nodemask_t nmask
= node_states
[N_MEMORY
];
1313 * try to allocate from a different node but reuse this node if there
1314 * are no other online nodes to be used (e.g. we are offlining a part
1315 * of the only existing node)
1317 node_clear(nid
, nmask
);
1318 if (nodes_empty(nmask
))
1319 node_set(nid
, nmask
);
1321 return new_page_nodemask(page
, nid
, &nmask
);
1325 do_migrate_range(unsigned long start_pfn
, unsigned long end_pfn
)
1332 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
++) {
1333 if (!pfn_valid(pfn
))
1335 page
= pfn_to_page(pfn
);
1337 if (PageHuge(page
)) {
1338 struct page
*head
= compound_head(page
);
1339 pfn
= page_to_pfn(head
) + compound_nr(head
) - 1;
1340 isolate_huge_page(head
, &source
);
1342 } else if (PageTransHuge(page
))
1343 pfn
= page_to_pfn(compound_head(page
))
1344 + hpage_nr_pages(page
) - 1;
1347 * HWPoison pages have elevated reference counts so the migration would
1348 * fail on them. It also doesn't make any sense to migrate them in the
1349 * first place. Still try to unmap such a page in case it is still mapped
1350 * (e.g. current hwpoison implementation doesn't unmap KSM pages but keep
1351 * the unmap as the catch all safety net).
1353 if (PageHWPoison(page
)) {
1354 if (WARN_ON(PageLRU(page
)))
1355 isolate_lru_page(page
);
1356 if (page_mapped(page
))
1357 try_to_unmap(page
, TTU_IGNORE_MLOCK
| TTU_IGNORE_ACCESS
);
1361 if (!get_page_unless_zero(page
))
1364 * We can skip free pages. And we can deal with pages on
1365 * LRU and non-lru movable pages.
1368 ret
= isolate_lru_page(page
);
1370 ret
= isolate_movable_page(page
, ISOLATE_UNEVICTABLE
);
1371 if (!ret
) { /* Success */
1372 list_add_tail(&page
->lru
, &source
);
1373 if (!__PageMovable(page
))
1374 inc_node_page_state(page
, NR_ISOLATED_ANON
+
1375 page_is_file_cache(page
));
1378 pr_warn("failed to isolate pfn %lx\n", pfn
);
1379 dump_page(page
, "isolation failed");
1383 if (!list_empty(&source
)) {
1384 /* Allocate a new page from the nearest neighbor node */
1385 ret
= migrate_pages(&source
, new_node_page
, NULL
, 0,
1386 MIGRATE_SYNC
, MR_MEMORY_HOTPLUG
);
1388 list_for_each_entry(page
, &source
, lru
) {
1389 pr_warn("migrating pfn %lx failed ret:%d ",
1390 page_to_pfn(page
), ret
);
1391 dump_page(page
, "migration failure");
1393 putback_movable_pages(&source
);
1401 * remove from free_area[] and mark all as Reserved.
1404 offline_isolated_pages_cb(unsigned long start
, unsigned long nr_pages
,
1407 unsigned long *offlined_pages
= (unsigned long *)data
;
1409 *offlined_pages
+= __offline_isolated_pages(start
, start
+ nr_pages
);
1414 * Check all pages in range, recoreded as memory resource, are isolated.
1417 check_pages_isolated_cb(unsigned long start_pfn
, unsigned long nr_pages
,
1420 return test_pages_isolated(start_pfn
, start_pfn
+ nr_pages
, true);
1423 static int __init
cmdline_parse_movable_node(char *p
)
1425 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
1426 movable_node_enabled
= true;
1428 pr_warn("movable_node parameter depends on CONFIG_HAVE_MEMBLOCK_NODE_MAP to work properly\n");
1432 early_param("movable_node", cmdline_parse_movable_node
);
1434 /* check which state of node_states will be changed when offline memory */
1435 static void node_states_check_changes_offline(unsigned long nr_pages
,
1436 struct zone
*zone
, struct memory_notify
*arg
)
1438 struct pglist_data
*pgdat
= zone
->zone_pgdat
;
1439 unsigned long present_pages
= 0;
1442 arg
->status_change_nid
= NUMA_NO_NODE
;
1443 arg
->status_change_nid_normal
= NUMA_NO_NODE
;
1444 arg
->status_change_nid_high
= NUMA_NO_NODE
;
1447 * Check whether node_states[N_NORMAL_MEMORY] will be changed.
1448 * If the memory to be offline is within the range
1449 * [0..ZONE_NORMAL], and it is the last present memory there,
1450 * the zones in that range will become empty after the offlining,
1451 * thus we can determine that we need to clear the node from
1452 * node_states[N_NORMAL_MEMORY].
1454 for (zt
= 0; zt
<= ZONE_NORMAL
; zt
++)
1455 present_pages
+= pgdat
->node_zones
[zt
].present_pages
;
1456 if (zone_idx(zone
) <= ZONE_NORMAL
&& nr_pages
>= present_pages
)
1457 arg
->status_change_nid_normal
= zone_to_nid(zone
);
1459 #ifdef CONFIG_HIGHMEM
1461 * node_states[N_HIGH_MEMORY] contains nodes which
1462 * have normal memory or high memory.
1463 * Here we add the present_pages belonging to ZONE_HIGHMEM.
1464 * If the zone is within the range of [0..ZONE_HIGHMEM), and
1465 * we determine that the zones in that range become empty,
1466 * we need to clear the node for N_HIGH_MEMORY.
1468 present_pages
+= pgdat
->node_zones
[ZONE_HIGHMEM
].present_pages
;
1469 if (zone_idx(zone
) <= ZONE_HIGHMEM
&& nr_pages
>= present_pages
)
1470 arg
->status_change_nid_high
= zone_to_nid(zone
);
1474 * We have accounted the pages from [0..ZONE_NORMAL), and
1475 * in case of CONFIG_HIGHMEM the pages from ZONE_HIGHMEM
1477 * Here we count the possible pages from ZONE_MOVABLE.
1478 * If after having accounted all the pages, we see that the nr_pages
1479 * to be offlined is over or equal to the accounted pages,
1480 * we know that the node will become empty, and so, we can clear
1481 * it for N_MEMORY as well.
1483 present_pages
+= pgdat
->node_zones
[ZONE_MOVABLE
].present_pages
;
1485 if (nr_pages
>= present_pages
)
1486 arg
->status_change_nid
= zone_to_nid(zone
);
1489 static void node_states_clear_node(int node
, struct memory_notify
*arg
)
1491 if (arg
->status_change_nid_normal
>= 0)
1492 node_clear_state(node
, N_NORMAL_MEMORY
);
1494 if (arg
->status_change_nid_high
>= 0)
1495 node_clear_state(node
, N_HIGH_MEMORY
);
1497 if (arg
->status_change_nid
>= 0)
1498 node_clear_state(node
, N_MEMORY
);
1501 static int __ref
__offline_pages(unsigned long start_pfn
,
1502 unsigned long end_pfn
)
1504 unsigned long pfn
, nr_pages
;
1505 unsigned long offlined_pages
= 0;
1506 int ret
, node
, nr_isolate_pageblock
;
1507 unsigned long flags
;
1508 unsigned long valid_start
, valid_end
;
1510 struct memory_notify arg
;
1513 mem_hotplug_begin();
1515 /* This makes hotplug much easier...and readable.
1516 we assume this for now. .*/
1517 if (!test_pages_in_a_zone(start_pfn
, end_pfn
, &valid_start
,
1520 reason
= "multizone range";
1521 goto failed_removal
;
1524 zone
= page_zone(pfn_to_page(valid_start
));
1525 node
= zone_to_nid(zone
);
1526 nr_pages
= end_pfn
- start_pfn
;
1528 /* set above range as isolated */
1529 ret
= start_isolate_page_range(start_pfn
, end_pfn
,
1531 SKIP_HWPOISON
| REPORT_FAILURE
);
1533 reason
= "failure to isolate range";
1534 goto failed_removal
;
1536 nr_isolate_pageblock
= ret
;
1538 arg
.start_pfn
= start_pfn
;
1539 arg
.nr_pages
= nr_pages
;
1540 node_states_check_changes_offline(nr_pages
, zone
, &arg
);
1542 ret
= memory_notify(MEM_GOING_OFFLINE
, &arg
);
1543 ret
= notifier_to_errno(ret
);
1545 reason
= "notifier failure";
1546 goto failed_removal_isolated
;
1550 for (pfn
= start_pfn
; pfn
;) {
1551 if (signal_pending(current
)) {
1553 reason
= "signal backoff";
1554 goto failed_removal_isolated
;
1558 lru_add_drain_all();
1560 pfn
= scan_movable_pages(pfn
, end_pfn
);
1563 * TODO: fatal migration failures should bail
1566 do_migrate_range(pfn
, end_pfn
);
1571 * Dissolve free hugepages in the memory block before doing
1572 * offlining actually in order to make hugetlbfs's object
1573 * counting consistent.
1575 ret
= dissolve_free_huge_pages(start_pfn
, end_pfn
);
1577 reason
= "failure to dissolve huge pages";
1578 goto failed_removal_isolated
;
1581 ret
= walk_system_ram_range(start_pfn
, end_pfn
- start_pfn
,
1582 NULL
, check_pages_isolated_cb
);
1584 * per-cpu pages are drained in start_isolate_page_range, but if
1585 * there are still pages that are not free, make sure that we
1586 * drain again, because when we isolated range we might
1587 * have raced with another thread that was adding pages to pcp
1590 * Forward progress should be still guaranteed because
1591 * pages on the pcp list can only belong to MOVABLE_ZONE
1592 * because has_unmovable_pages explicitly checks for
1593 * PageBuddy on freed pages on other zones.
1596 drain_all_pages(zone
);
1599 /* Ok, all of our target is isolated.
1600 We cannot do rollback at this point. */
1601 walk_system_ram_range(start_pfn
, end_pfn
- start_pfn
,
1602 &offlined_pages
, offline_isolated_pages_cb
);
1603 pr_info("Offlined Pages %ld\n", offlined_pages
);
1605 * Onlining will reset pagetype flags and makes migrate type
1606 * MOVABLE, so just need to decrease the number of isolated
1607 * pageblocks zone counter here.
1609 spin_lock_irqsave(&zone
->lock
, flags
);
1610 zone
->nr_isolate_pageblock
-= nr_isolate_pageblock
;
1611 spin_unlock_irqrestore(&zone
->lock
, flags
);
1613 /* removal success */
1614 adjust_managed_page_count(pfn_to_page(start_pfn
), -offlined_pages
);
1615 zone
->present_pages
-= offlined_pages
;
1617 pgdat_resize_lock(zone
->zone_pgdat
, &flags
);
1618 zone
->zone_pgdat
->node_present_pages
-= offlined_pages
;
1619 pgdat_resize_unlock(zone
->zone_pgdat
, &flags
);
1621 init_per_zone_wmark_min();
1623 if (!populated_zone(zone
)) {
1624 zone_pcp_reset(zone
);
1625 build_all_zonelists(NULL
);
1627 zone_pcp_update(zone
);
1629 node_states_clear_node(node
, &arg
);
1630 if (arg
.status_change_nid
>= 0) {
1632 kcompactd_stop(node
);
1635 vm_total_pages
= nr_free_pagecache_pages();
1636 writeback_set_ratelimit();
1638 memory_notify(MEM_OFFLINE
, &arg
);
1639 remove_pfn_range_from_zone(zone
, start_pfn
, nr_pages
);
1643 failed_removal_isolated
:
1644 undo_isolate_page_range(start_pfn
, end_pfn
, MIGRATE_MOVABLE
);
1645 memory_notify(MEM_CANCEL_OFFLINE
, &arg
);
1647 pr_debug("memory offlining [mem %#010llx-%#010llx] failed due to %s\n",
1648 (unsigned long long) start_pfn
<< PAGE_SHIFT
,
1649 ((unsigned long long) end_pfn
<< PAGE_SHIFT
) - 1,
1651 /* pushback to free area */
1656 int offline_pages(unsigned long start_pfn
, unsigned long nr_pages
)
1658 return __offline_pages(start_pfn
, start_pfn
+ nr_pages
);
1661 static int check_memblock_offlined_cb(struct memory_block
*mem
, void *arg
)
1663 int ret
= !is_memblock_offlined(mem
);
1665 if (unlikely(ret
)) {
1666 phys_addr_t beginpa
, endpa
;
1668 beginpa
= PFN_PHYS(section_nr_to_pfn(mem
->start_section_nr
));
1669 endpa
= beginpa
+ memory_block_size_bytes() - 1;
1670 pr_warn("removing memory fails, because memory [%pa-%pa] is onlined\n",
1678 static int check_cpu_on_node(pg_data_t
*pgdat
)
1682 for_each_present_cpu(cpu
) {
1683 if (cpu_to_node(cpu
) == pgdat
->node_id
)
1685 * the cpu on this node isn't removed, and we can't
1686 * offline this node.
1694 static int check_no_memblock_for_node_cb(struct memory_block
*mem
, void *arg
)
1696 int nid
= *(int *)arg
;
1699 * If a memory block belongs to multiple nodes, the stored nid is not
1700 * reliable. However, such blocks are always online (e.g., cannot get
1701 * offlined) and, therefore, are still spanned by the node.
1703 return mem
->nid
== nid
? -EEXIST
: 0;
1710 * Offline a node if all memory sections and cpus of the node are removed.
1712 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
1713 * and online/offline operations before this call.
1715 void try_offline_node(int nid
)
1717 pg_data_t
*pgdat
= NODE_DATA(nid
);
1721 * If the node still spans pages (especially ZONE_DEVICE), don't
1722 * offline it. A node spans memory after move_pfn_range_to_zone(),
1723 * e.g., after the memory block was onlined.
1725 if (pgdat
->node_spanned_pages
)
1729 * Especially offline memory blocks might not be spanned by the
1730 * node. They will get spanned by the node once they get onlined.
1731 * However, they link to the node in sysfs and can get onlined later.
1733 rc
= for_each_memory_block(&nid
, check_no_memblock_for_node_cb
);
1737 if (check_cpu_on_node(pgdat
))
1741 * all memory/cpu of this node are removed, we can offline this
1744 node_set_offline(nid
);
1745 unregister_one_node(nid
);
1747 EXPORT_SYMBOL(try_offline_node
);
1749 static void __release_memory_resource(resource_size_t start
,
1750 resource_size_t size
)
1755 * When removing memory in the same granularity as it was added,
1756 * this function never fails. It might only fail if resources
1757 * have to be adjusted or split. We'll ignore the error, as
1758 * removing of memory cannot fail.
1760 ret
= release_mem_region_adjustable(&iomem_resource
, start
, size
);
1762 resource_size_t endres
= start
+ size
- 1;
1764 pr_warn("Unable to release resource <%pa-%pa> (%d)\n",
1765 &start
, &endres
, ret
);
1769 static int __ref
try_remove_memory(int nid
, u64 start
, u64 size
)
1773 BUG_ON(check_hotplug_memory_range(start
, size
));
1776 * All memory blocks must be offlined before removing memory. Check
1777 * whether all memory blocks in question are offline and return error
1778 * if this is not the case.
1780 rc
= walk_memory_blocks(start
, size
, NULL
, check_memblock_offlined_cb
);
1784 /* remove memmap entry */
1785 firmware_map_remove(start
, start
+ size
, "System RAM");
1786 memblock_free(start
, size
);
1787 memblock_remove(start
, size
);
1790 * Memory block device removal under the device_hotplug_lock is
1791 * a barrier against racing online attempts.
1793 remove_memory_block_devices(start
, size
);
1795 mem_hotplug_begin();
1797 arch_remove_memory(nid
, start
, size
, NULL
);
1798 __release_memory_resource(start
, size
);
1800 try_offline_node(nid
);
1809 * @start: physical address of the region to remove
1810 * @size: size of the region to remove
1812 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
1813 * and online/offline operations before this call, as required by
1814 * try_offline_node().
1816 void __remove_memory(int nid
, u64 start
, u64 size
)
1820 * trigger BUG() if some memory is not offlined prior to calling this
1823 if (try_remove_memory(nid
, start
, size
))
1828 * Remove memory if every memory block is offline, otherwise return -EBUSY is
1829 * some memory is not offline
1831 int remove_memory(int nid
, u64 start
, u64 size
)
1835 lock_device_hotplug();
1836 rc
= try_remove_memory(nid
, start
, size
);
1837 unlock_device_hotplug();
1841 EXPORT_SYMBOL_GPL(remove_memory
);
1842 #endif /* CONFIG_MEMORY_HOTREMOVE */