1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/debugfs.h>
4 #include <linux/slab.h>
5 #include <linux/uaccess.h>
6 #include <linux/memblock.h>
7 #include <linux/stacktrace.h>
8 #include <linux/page_owner.h>
9 #include <linux/jump_label.h>
10 #include <linux/migrate.h>
11 #include <linux/stackdepot.h>
12 #include <linux/seq_file.h>
17 * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack)
18 * to use off stack temporal storage
20 #define PAGE_OWNER_STACK_DEPTH (16)
24 short last_migrate_reason
;
26 depot_stack_handle_t handle
;
29 static bool page_owner_disabled
= true;
30 DEFINE_STATIC_KEY_FALSE(page_owner_inited
);
32 static depot_stack_handle_t dummy_handle
;
33 static depot_stack_handle_t failure_handle
;
34 static depot_stack_handle_t early_handle
;
36 static void init_early_allocated_pages(void);
38 static int __init
early_page_owner_param(char *buf
)
43 if (strcmp(buf
, "on") == 0)
44 page_owner_disabled
= false;
48 early_param("page_owner", early_page_owner_param
);
50 static bool need_page_owner(void)
52 if (page_owner_disabled
)
58 static __always_inline depot_stack_handle_t
create_dummy_stack(void)
60 unsigned long entries
[4];
61 unsigned int nr_entries
;
63 nr_entries
= stack_trace_save(entries
, ARRAY_SIZE(entries
), 0);
64 return stack_depot_save(entries
, nr_entries
, GFP_KERNEL
);
67 static noinline
void register_dummy_stack(void)
69 dummy_handle
= create_dummy_stack();
72 static noinline
void register_failure_stack(void)
74 failure_handle
= create_dummy_stack();
77 static noinline
void register_early_stack(void)
79 early_handle
= create_dummy_stack();
82 static void init_page_owner(void)
84 if (page_owner_disabled
)
87 register_dummy_stack();
88 register_failure_stack();
89 register_early_stack();
90 static_branch_enable(&page_owner_inited
);
91 init_early_allocated_pages();
94 struct page_ext_operations page_owner_ops
= {
95 .size
= sizeof(struct page_owner
),
96 .need
= need_page_owner
,
97 .init
= init_page_owner
,
100 static inline struct page_owner
*get_page_owner(struct page_ext
*page_ext
)
102 return (void *)page_ext
+ page_owner_ops
.offset
;
105 void __reset_page_owner(struct page
*page
, unsigned int order
)
108 struct page_ext
*page_ext
;
110 for (i
= 0; i
< (1 << order
); i
++) {
111 page_ext
= lookup_page_ext(page
+ i
);
112 if (unlikely(!page_ext
))
114 __clear_bit(PAGE_EXT_OWNER
, &page_ext
->flags
);
118 static inline bool check_recursive_alloc(unsigned long *entries
,
119 unsigned int nr_entries
,
124 for (i
= 0; i
< nr_entries
; i
++) {
125 if (entries
[i
] == ip
)
131 static noinline depot_stack_handle_t
save_stack(gfp_t flags
)
133 unsigned long entries
[PAGE_OWNER_STACK_DEPTH
];
134 depot_stack_handle_t handle
;
135 unsigned int nr_entries
;
137 nr_entries
= stack_trace_save(entries
, ARRAY_SIZE(entries
), 2);
140 * We need to check recursion here because our request to
141 * stackdepot could trigger memory allocation to save new
142 * entry. New memory allocation would reach here and call
143 * stack_depot_save_entries() again if we don't catch it. There is
144 * still not enough memory in stackdepot so it would try to
145 * allocate memory again and loop forever.
147 if (check_recursive_alloc(entries
, nr_entries
, _RET_IP_
))
150 handle
= stack_depot_save(entries
, nr_entries
, flags
);
152 handle
= failure_handle
;
157 static inline void __set_page_owner_handle(struct page_ext
*page_ext
,
158 depot_stack_handle_t handle
, unsigned int order
, gfp_t gfp_mask
)
160 struct page_owner
*page_owner
;
162 page_owner
= get_page_owner(page_ext
);
163 page_owner
->handle
= handle
;
164 page_owner
->order
= order
;
165 page_owner
->gfp_mask
= gfp_mask
;
166 page_owner
->last_migrate_reason
= -1;
168 __set_bit(PAGE_EXT_OWNER
, &page_ext
->flags
);
171 noinline
void __set_page_owner(struct page
*page
, unsigned int order
,
174 struct page_ext
*page_ext
= lookup_page_ext(page
);
175 depot_stack_handle_t handle
;
177 if (unlikely(!page_ext
))
180 handle
= save_stack(gfp_mask
);
181 __set_page_owner_handle(page_ext
, handle
, order
, gfp_mask
);
184 void __set_page_owner_migrate_reason(struct page
*page
, int reason
)
186 struct page_ext
*page_ext
= lookup_page_ext(page
);
187 struct page_owner
*page_owner
;
189 if (unlikely(!page_ext
))
192 page_owner
= get_page_owner(page_ext
);
193 page_owner
->last_migrate_reason
= reason
;
196 void __split_page_owner(struct page
*page
, unsigned int order
)
199 struct page_ext
*page_ext
= lookup_page_ext(page
);
200 struct page_owner
*page_owner
;
202 if (unlikely(!page_ext
))
205 page_owner
= get_page_owner(page_ext
);
206 page_owner
->order
= 0;
207 for (i
= 1; i
< (1 << order
); i
++)
208 __copy_page_owner(page
, page
+ i
);
211 void __copy_page_owner(struct page
*oldpage
, struct page
*newpage
)
213 struct page_ext
*old_ext
= lookup_page_ext(oldpage
);
214 struct page_ext
*new_ext
= lookup_page_ext(newpage
);
215 struct page_owner
*old_page_owner
, *new_page_owner
;
217 if (unlikely(!old_ext
|| !new_ext
))
220 old_page_owner
= get_page_owner(old_ext
);
221 new_page_owner
= get_page_owner(new_ext
);
222 new_page_owner
->order
= old_page_owner
->order
;
223 new_page_owner
->gfp_mask
= old_page_owner
->gfp_mask
;
224 new_page_owner
->last_migrate_reason
=
225 old_page_owner
->last_migrate_reason
;
226 new_page_owner
->handle
= old_page_owner
->handle
;
229 * We don't clear the bit on the oldpage as it's going to be freed
230 * after migration. Until then, the info can be useful in case of
231 * a bug, and the overal stats will be off a bit only temporarily.
232 * Also, migrate_misplaced_transhuge_page() can still fail the
233 * migration and then we want the oldpage to retain the info. But
234 * in that case we also don't need to explicitly clear the info from
235 * the new page, which will be freed.
237 __set_bit(PAGE_EXT_OWNER
, &new_ext
->flags
);
240 void pagetypeinfo_showmixedcount_print(struct seq_file
*m
,
241 pg_data_t
*pgdat
, struct zone
*zone
)
244 struct page_ext
*page_ext
;
245 struct page_owner
*page_owner
;
246 unsigned long pfn
= zone
->zone_start_pfn
, block_end_pfn
;
247 unsigned long end_pfn
= pfn
+ zone
->spanned_pages
;
248 unsigned long count
[MIGRATE_TYPES
] = { 0, };
249 int pageblock_mt
, page_mt
;
252 /* Scan block by block. First and last block may be incomplete */
253 pfn
= zone
->zone_start_pfn
;
256 * Walk the zone in pageblock_nr_pages steps. If a page block spans
257 * a zone boundary, it will be double counted between zones. This does
258 * not matter as the mixed block count will still be correct
260 for (; pfn
< end_pfn
; ) {
261 page
= pfn_to_online_page(pfn
);
263 pfn
= ALIGN(pfn
+ 1, MAX_ORDER_NR_PAGES
);
267 block_end_pfn
= ALIGN(pfn
+ 1, pageblock_nr_pages
);
268 block_end_pfn
= min(block_end_pfn
, end_pfn
);
270 pageblock_mt
= get_pageblock_migratetype(page
);
272 for (; pfn
< block_end_pfn
; pfn
++) {
273 if (!pfn_valid_within(pfn
))
276 /* The pageblock is online, no need to recheck. */
277 page
= pfn_to_page(pfn
);
279 if (page_zone(page
) != zone
)
282 if (PageBuddy(page
)) {
283 unsigned long freepage_order
;
285 freepage_order
= page_order_unsafe(page
);
286 if (freepage_order
< MAX_ORDER
)
287 pfn
+= (1UL << freepage_order
) - 1;
291 if (PageReserved(page
))
294 page_ext
= lookup_page_ext(page
);
295 if (unlikely(!page_ext
))
298 if (!test_bit(PAGE_EXT_OWNER
, &page_ext
->flags
))
301 page_owner
= get_page_owner(page_ext
);
302 page_mt
= gfpflags_to_migratetype(
303 page_owner
->gfp_mask
);
304 if (pageblock_mt
!= page_mt
) {
305 if (is_migrate_cma(pageblock_mt
))
306 count
[MIGRATE_MOVABLE
]++;
308 count
[pageblock_mt
]++;
313 pfn
+= (1UL << page_owner
->order
) - 1;
318 seq_printf(m
, "Node %d, zone %8s ", pgdat
->node_id
, zone
->name
);
319 for (i
= 0; i
< MIGRATE_TYPES
; i
++)
320 seq_printf(m
, "%12lu ", count
[i
]);
325 print_page_owner(char __user
*buf
, size_t count
, unsigned long pfn
,
326 struct page
*page
, struct page_owner
*page_owner
,
327 depot_stack_handle_t handle
)
329 int ret
, pageblock_mt
, page_mt
;
330 unsigned long *entries
;
331 unsigned int nr_entries
;
334 count
= min_t(size_t, count
, PAGE_SIZE
);
335 kbuf
= kmalloc(count
, GFP_KERNEL
);
339 ret
= snprintf(kbuf
, count
,
340 "Page allocated via order %u, mask %#x(%pGg)\n",
341 page_owner
->order
, page_owner
->gfp_mask
,
342 &page_owner
->gfp_mask
);
347 /* Print information relevant to grouping pages by mobility */
348 pageblock_mt
= get_pageblock_migratetype(page
);
349 page_mt
= gfpflags_to_migratetype(page_owner
->gfp_mask
);
350 ret
+= snprintf(kbuf
+ ret
, count
- ret
,
351 "PFN %lu type %s Block %lu type %s Flags %#lx(%pGp)\n",
353 migratetype_names
[page_mt
],
354 pfn
>> pageblock_order
,
355 migratetype_names
[pageblock_mt
],
356 page
->flags
, &page
->flags
);
361 nr_entries
= stack_depot_fetch(handle
, &entries
);
362 ret
+= stack_trace_snprint(kbuf
+ ret
, count
- ret
, entries
, nr_entries
, 0);
366 if (page_owner
->last_migrate_reason
!= -1) {
367 ret
+= snprintf(kbuf
+ ret
, count
- ret
,
368 "Page has been migrated, last migrate reason: %s\n",
369 migrate_reason_names
[page_owner
->last_migrate_reason
]);
374 ret
+= snprintf(kbuf
+ ret
, count
- ret
, "\n");
378 if (copy_to_user(buf
, kbuf
, ret
))
389 void __dump_page_owner(struct page
*page
)
391 struct page_ext
*page_ext
= lookup_page_ext(page
);
392 struct page_owner
*page_owner
;
393 depot_stack_handle_t handle
;
394 unsigned long *entries
;
395 unsigned int nr_entries
;
399 if (unlikely(!page_ext
)) {
400 pr_alert("There is not page extension available.\n");
404 page_owner
= get_page_owner(page_ext
);
405 gfp_mask
= page_owner
->gfp_mask
;
406 mt
= gfpflags_to_migratetype(gfp_mask
);
408 if (!test_bit(PAGE_EXT_OWNER
, &page_ext
->flags
)) {
409 pr_alert("page_owner info is not active (free page?)\n");
413 handle
= READ_ONCE(page_owner
->handle
);
415 pr_alert("page_owner info is not active (free page?)\n");
419 nr_entries
= stack_depot_fetch(handle
, &entries
);
420 pr_alert("page allocated via order %u, migratetype %s, gfp_mask %#x(%pGg)\n",
421 page_owner
->order
, migratetype_names
[mt
], gfp_mask
, &gfp_mask
);
422 stack_trace_print(entries
, nr_entries
, 0);
424 if (page_owner
->last_migrate_reason
!= -1)
425 pr_alert("page has been migrated, last migrate reason: %s\n",
426 migrate_reason_names
[page_owner
->last_migrate_reason
]);
430 read_page_owner(struct file
*file
, char __user
*buf
, size_t count
, loff_t
*ppos
)
434 struct page_ext
*page_ext
;
435 struct page_owner
*page_owner
;
436 depot_stack_handle_t handle
;
438 if (!static_branch_unlikely(&page_owner_inited
))
442 pfn
= min_low_pfn
+ *ppos
;
444 /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */
445 while (!pfn_valid(pfn
) && (pfn
& (MAX_ORDER_NR_PAGES
- 1)) != 0)
448 drain_all_pages(NULL
);
450 /* Find an allocated page */
451 for (; pfn
< max_pfn
; pfn
++) {
453 * If the new page is in a new MAX_ORDER_NR_PAGES area,
454 * validate the area as existing, skip it if not
456 if ((pfn
& (MAX_ORDER_NR_PAGES
- 1)) == 0 && !pfn_valid(pfn
)) {
457 pfn
+= MAX_ORDER_NR_PAGES
- 1;
461 /* Check for holes within a MAX_ORDER area */
462 if (!pfn_valid_within(pfn
))
465 page
= pfn_to_page(pfn
);
466 if (PageBuddy(page
)) {
467 unsigned long freepage_order
= page_order_unsafe(page
);
469 if (freepage_order
< MAX_ORDER
)
470 pfn
+= (1UL << freepage_order
) - 1;
474 page_ext
= lookup_page_ext(page
);
475 if (unlikely(!page_ext
))
479 * Some pages could be missed by concurrent allocation or free,
480 * because we don't hold the zone lock.
482 if (!test_bit(PAGE_EXT_OWNER
, &page_ext
->flags
))
485 page_owner
= get_page_owner(page_ext
);
488 * Access to page_ext->handle isn't synchronous so we should
489 * be careful to access it.
491 handle
= READ_ONCE(page_owner
->handle
);
495 /* Record the next PFN to read in the file offset */
496 *ppos
= (pfn
- min_low_pfn
) + 1;
498 return print_page_owner(buf
, count
, pfn
, page
,
505 static void init_pages_in_zone(pg_data_t
*pgdat
, struct zone
*zone
)
507 unsigned long pfn
= zone
->zone_start_pfn
;
508 unsigned long end_pfn
= zone_end_pfn(zone
);
509 unsigned long count
= 0;
512 * Walk the zone in pageblock_nr_pages steps. If a page block spans
513 * a zone boundary, it will be double counted between zones. This does
514 * not matter as the mixed block count will still be correct
516 for (; pfn
< end_pfn
; ) {
517 unsigned long block_end_pfn
;
519 if (!pfn_valid(pfn
)) {
520 pfn
= ALIGN(pfn
+ 1, MAX_ORDER_NR_PAGES
);
524 block_end_pfn
= ALIGN(pfn
+ 1, pageblock_nr_pages
);
525 block_end_pfn
= min(block_end_pfn
, end_pfn
);
527 for (; pfn
< block_end_pfn
; pfn
++) {
529 struct page_ext
*page_ext
;
531 if (!pfn_valid_within(pfn
))
534 page
= pfn_to_page(pfn
);
536 if (page_zone(page
) != zone
)
540 * To avoid having to grab zone->lock, be a little
541 * careful when reading buddy page order. The only
542 * danger is that we skip too much and potentially miss
543 * some early allocated pages, which is better than
544 * heavy lock contention.
546 if (PageBuddy(page
)) {
547 unsigned long order
= page_order_unsafe(page
);
549 if (order
> 0 && order
< MAX_ORDER
)
550 pfn
+= (1UL << order
) - 1;
554 if (PageReserved(page
))
557 page_ext
= lookup_page_ext(page
);
558 if (unlikely(!page_ext
))
561 /* Maybe overlapping zone */
562 if (test_bit(PAGE_EXT_OWNER
, &page_ext
->flags
))
565 /* Found early allocated page */
566 __set_page_owner_handle(page_ext
, early_handle
, 0, 0);
572 pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n",
573 pgdat
->node_id
, zone
->name
, count
);
576 static void init_zones_in_node(pg_data_t
*pgdat
)
579 struct zone
*node_zones
= pgdat
->node_zones
;
581 for (zone
= node_zones
; zone
- node_zones
< MAX_NR_ZONES
; ++zone
) {
582 if (!populated_zone(zone
))
585 init_pages_in_zone(pgdat
, zone
);
589 static void init_early_allocated_pages(void)
593 for_each_online_pgdat(pgdat
)
594 init_zones_in_node(pgdat
);
597 static const struct file_operations proc_page_owner_operations
= {
598 .read
= read_page_owner
,
601 static int __init
pageowner_init(void)
603 if (!static_branch_unlikely(&page_owner_inited
)) {
604 pr_info("page_owner is disabled\n");
608 debugfs_create_file("page_owner", 0400, NULL
, NULL
,
609 &proc_page_owner_operations
);
613 late_initcall(pageowner_init
)