]>
git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blob - mm/page_isolation.c
1 // SPDX-License-Identifier: GPL-2.0
3 * linux/mm/page_isolation.c
7 #include <linux/page-isolation.h>
8 #include <linux/pageblock-flags.h>
9 #include <linux/memory.h>
10 #include <linux/hugetlb.h>
11 #include <linux/page_owner.h>
12 #include <linux/migrate.h>
15 #define CREATE_TRACE_POINTS
16 #include <trace/events/page_isolation.h>
18 static int set_migratetype_isolate(struct page
*page
, int migratetype
, int isol_flags
)
20 struct zone
*zone
= page_zone(page
);
21 struct page
*unmovable
;
24 spin_lock_irqsave(&zone
->lock
, flags
);
27 * We assume the caller intended to SET migrate type to isolate.
28 * If it is already set, then someone else must have raced and
31 if (is_migrate_isolate_page(page
)) {
32 spin_unlock_irqrestore(&zone
->lock
, flags
);
37 * FIXME: Now, memory hotplug doesn't call shrink_slab() by itself.
38 * We just check MOVABLE pages.
40 unmovable
= has_unmovable_pages(zone
, page
, migratetype
, isol_flags
);
42 unsigned long nr_pages
;
43 int mt
= get_pageblock_migratetype(page
);
45 set_pageblock_migratetype(page
, MIGRATE_ISOLATE
);
46 zone
->nr_isolate_pageblock
++;
47 nr_pages
= move_freepages_block(zone
, page
, MIGRATE_ISOLATE
,
50 __mod_zone_freepage_state(zone
, -nr_pages
, mt
);
51 spin_unlock_irqrestore(&zone
->lock
, flags
);
52 drain_all_pages(zone
);
56 spin_unlock_irqrestore(&zone
->lock
, flags
);
57 if (isol_flags
& REPORT_FAILURE
) {
59 * printk() with zone->lock held will likely trigger a
60 * lockdep splat, so defer it here.
62 dump_page(unmovable
, "unmovable page");
68 static void unset_migratetype_isolate(struct page
*page
, unsigned migratetype
)
71 unsigned long flags
, nr_pages
;
72 bool isolated_page
= false;
74 unsigned long pfn
, buddy_pfn
;
77 zone
= page_zone(page
);
78 spin_lock_irqsave(&zone
->lock
, flags
);
79 if (!is_migrate_isolate_page(page
))
83 * Because freepage with more than pageblock_order on isolated
84 * pageblock is restricted to merge due to freepage counting problem,
85 * it is possible that there is free buddy page.
86 * move_freepages_block() doesn't care of merge so we need other
87 * approach in order to merge them. Isolation and free will make
88 * these pages to be merged.
90 if (PageBuddy(page
)) {
91 order
= buddy_order(page
);
92 if (order
>= pageblock_order
) {
93 pfn
= page_to_pfn(page
);
94 buddy_pfn
= __find_buddy_pfn(pfn
, order
);
95 buddy
= page
+ (buddy_pfn
- pfn
);
97 if (pfn_valid_within(buddy_pfn
) &&
98 !is_migrate_isolate_page(buddy
)) {
99 __isolate_free_page(page
, order
);
100 isolated_page
= true;
106 * If we isolate freepage with more than pageblock_order, there
107 * should be no freepage in the range, so we could avoid costly
108 * pageblock scanning for freepage moving.
110 * We didn't actually touch any of the isolated pages, so place them
111 * to the tail of the freelist. This is an optimization for memory
112 * onlining - just onlined memory won't immediately be considered for
115 if (!isolated_page
) {
116 nr_pages
= move_freepages_block(zone
, page
, migratetype
, NULL
);
117 __mod_zone_freepage_state(zone
, nr_pages
, migratetype
);
119 set_pageblock_migratetype(page
, migratetype
);
121 __putback_isolated_page(page
, order
, migratetype
);
122 zone
->nr_isolate_pageblock
--;
124 spin_unlock_irqrestore(&zone
->lock
, flags
);
127 static inline struct page
*
128 __first_valid_page(unsigned long pfn
, unsigned long nr_pages
)
132 for (i
= 0; i
< nr_pages
; i
++) {
135 page
= pfn_to_online_page(pfn
+ i
);
144 * start_isolate_page_range() - make page-allocation-type of range of pages to
145 * be MIGRATE_ISOLATE.
146 * @start_pfn: The lower PFN of the range to be isolated.
147 * @end_pfn: The upper PFN of the range to be isolated.
148 * start_pfn/end_pfn must be aligned to pageblock_order.
149 * @migratetype: Migrate type to set in error recovery.
150 * @flags: The following flags are allowed (they can be combined in
152 * MEMORY_OFFLINE - isolate to offline (!allocate) memory
153 * e.g., skip over PageHWPoison() pages
154 * and PageOffline() pages.
155 * REPORT_FAILURE - report details about the failure to
158 * Making page-allocation-type to be MIGRATE_ISOLATE means free pages in
159 * the range will never be allocated. Any free pages and pages freed in the
160 * future will not be allocated again. If specified range includes migrate types
161 * other than MOVABLE or CMA, this will fail with -EBUSY. For isolating all
162 * pages in the range finally, the caller have to free all pages in the range.
163 * test_page_isolated() can be used for test it.
165 * There is no high level synchronization mechanism that prevents two threads
166 * from trying to isolate overlapping ranges. If this happens, one thread
167 * will notice pageblocks in the overlapping range already set to isolate.
168 * This happens in set_migratetype_isolate, and set_migratetype_isolate
169 * returns an error. We then clean up by restoring the migration type on
170 * pageblocks we may have modified and return -EBUSY to caller. This
171 * prevents two threads from simultaneously working on overlapping ranges.
173 * Please note that there is no strong synchronization with the page allocator
174 * either. Pages might be freed while their page blocks are marked ISOLATED.
175 * In some cases pages might still end up on pcp lists and that would allow
176 * for their allocation even when they are in fact isolated already. Depending
177 * on how strong of a guarantee the caller needs drain_all_pages might be needed
178 * (e.g. __offline_pages will need to call it after check for isolated range for
181 * Return: 0 on success and -EBUSY if any part of range cannot be isolated.
183 int start_isolate_page_range(unsigned long start_pfn
, unsigned long end_pfn
,
184 unsigned migratetype
, int flags
)
187 unsigned long undo_pfn
;
190 BUG_ON(!IS_ALIGNED(start_pfn
, pageblock_nr_pages
));
191 BUG_ON(!IS_ALIGNED(end_pfn
, pageblock_nr_pages
));
193 for (pfn
= start_pfn
;
195 pfn
+= pageblock_nr_pages
) {
196 page
= __first_valid_page(pfn
, pageblock_nr_pages
);
198 if (set_migratetype_isolate(page
, migratetype
, flags
)) {
206 for (pfn
= start_pfn
;
208 pfn
+= pageblock_nr_pages
) {
209 struct page
*page
= pfn_to_online_page(pfn
);
212 unset_migratetype_isolate(page
, migratetype
);
219 * Make isolated pages available again.
221 void undo_isolate_page_range(unsigned long start_pfn
, unsigned long end_pfn
,
222 unsigned migratetype
)
227 BUG_ON(!IS_ALIGNED(start_pfn
, pageblock_nr_pages
));
228 BUG_ON(!IS_ALIGNED(end_pfn
, pageblock_nr_pages
));
230 for (pfn
= start_pfn
;
232 pfn
+= pageblock_nr_pages
) {
233 page
= __first_valid_page(pfn
, pageblock_nr_pages
);
234 if (!page
|| !is_migrate_isolate_page(page
))
236 unset_migratetype_isolate(page
, migratetype
);
240 * Test all pages in the range is free(means isolated) or not.
241 * all pages in [start_pfn...end_pfn) must be in the same zone.
242 * zone->lock must be held before call this.
244 * Returns the last tested pfn.
247 __test_page_isolated_in_pageblock(unsigned long pfn
, unsigned long end_pfn
,
252 while (pfn
< end_pfn
) {
253 if (!pfn_valid_within(pfn
)) {
257 page
= pfn_to_page(pfn
);
260 * If the page is on a free list, it has to be on
261 * the correct MIGRATE_ISOLATE freelist. There is no
262 * simple way to verify that as VM_BUG_ON(), though.
264 pfn
+= 1 << buddy_order(page
);
265 else if ((flags
& MEMORY_OFFLINE
) && PageHWPoison(page
))
266 /* A HWPoisoned page cannot be also PageBuddy */
268 else if ((flags
& MEMORY_OFFLINE
) && PageOffline(page
) &&
271 * The responsible driver agreed to skip PageOffline()
272 * pages when offlining memory by dropping its
273 * reference in MEM_GOING_OFFLINE.
283 /* Caller should ensure that requested range is in a single zone */
284 int test_pages_isolated(unsigned long start_pfn
, unsigned long end_pfn
,
287 unsigned long pfn
, flags
;
292 * Note: pageblock_nr_pages != MAX_ORDER. Then, chunks of free pages
293 * are not aligned to pageblock_nr_pages.
294 * Then we just check migratetype first.
296 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= pageblock_nr_pages
) {
297 page
= __first_valid_page(pfn
, pageblock_nr_pages
);
298 if (page
&& !is_migrate_isolate_page(page
))
301 page
= __first_valid_page(start_pfn
, end_pfn
- start_pfn
);
302 if ((pfn
< end_pfn
) || !page
)
304 /* Check all pages are free or marked as ISOLATED */
305 zone
= page_zone(page
);
306 spin_lock_irqsave(&zone
->lock
, flags
);
307 pfn
= __test_page_isolated_in_pageblock(start_pfn
, end_pfn
, isol_flags
);
308 spin_unlock_irqrestore(&zone
->lock
, flags
);
310 trace_test_pages_isolated(start_pfn
, end_pfn
, pfn
);
312 return pfn
< end_pfn
? -EBUSY
: 0;