1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/debugfs.h>
4 #include <linux/slab.h>
5 #include <linux/uaccess.h>
6 #include <linux/memblock.h>
7 #include <linux/stacktrace.h>
8 #include <linux/page_owner.h>
9 #include <linux/jump_label.h>
10 #include <linux/migrate.h>
11 #include <linux/stackdepot.h>
12 #include <linux/seq_file.h>
13 #include <linux/sched/clock.h>
18 * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack)
19 * to use off stack temporal storage
21 #define PAGE_OWNER_STACK_DEPTH (16)
25 short last_migrate_reason
;
27 depot_stack_handle_t handle
;
28 depot_stack_handle_t free_handle
;
34 static bool page_owner_enabled
= false;
35 DEFINE_STATIC_KEY_FALSE(page_owner_inited
);
37 static depot_stack_handle_t dummy_handle
;
38 static depot_stack_handle_t failure_handle
;
39 static depot_stack_handle_t early_handle
;
41 static void init_early_allocated_pages(void);
43 static int __init
early_page_owner_param(char *buf
)
45 return kstrtobool(buf
, &page_owner_enabled
);
47 early_param("page_owner", early_page_owner_param
);
49 static bool need_page_owner(void)
51 return page_owner_enabled
;
54 static __always_inline depot_stack_handle_t
create_dummy_stack(void)
56 unsigned long entries
[4];
57 unsigned int nr_entries
;
59 nr_entries
= stack_trace_save(entries
, ARRAY_SIZE(entries
), 0);
60 return stack_depot_save(entries
, nr_entries
, GFP_KERNEL
);
63 static noinline
void register_dummy_stack(void)
65 dummy_handle
= create_dummy_stack();
68 static noinline
void register_failure_stack(void)
70 failure_handle
= create_dummy_stack();
73 static noinline
void register_early_stack(void)
75 early_handle
= create_dummy_stack();
78 static void init_page_owner(void)
80 if (!page_owner_enabled
)
83 register_dummy_stack();
84 register_failure_stack();
85 register_early_stack();
86 static_branch_enable(&page_owner_inited
);
87 init_early_allocated_pages();
90 struct page_ext_operations page_owner_ops
= {
91 .size
= sizeof(struct page_owner
),
92 .need
= need_page_owner
,
93 .init
= init_page_owner
,
96 static inline struct page_owner
*get_page_owner(struct page_ext
*page_ext
)
98 return (void *)page_ext
+ page_owner_ops
.offset
;
101 static noinline depot_stack_handle_t
save_stack(gfp_t flags
)
103 unsigned long entries
[PAGE_OWNER_STACK_DEPTH
];
104 depot_stack_handle_t handle
;
105 unsigned int nr_entries
;
110 * Sometimes page metadata allocation tracking requires more
111 * memory to be allocated:
112 * - when new stack trace is saved to stack depot
113 * - when backtrace itself is calculated (ia64)
115 if (current
->in_page_owner
)
117 current
->in_page_owner
= 1;
119 nr_entries
= stack_trace_save(entries
, ARRAY_SIZE(entries
), 2);
120 handle
= stack_depot_save(entries
, nr_entries
, flags
);
122 handle
= failure_handle
;
124 current
->in_page_owner
= 0;
128 void __reset_page_owner(struct page
*page
, unsigned int order
)
131 struct page_ext
*page_ext
;
132 depot_stack_handle_t handle
;
133 struct page_owner
*page_owner
;
134 u64 free_ts_nsec
= local_clock();
136 page_ext
= lookup_page_ext(page
);
137 if (unlikely(!page_ext
))
140 handle
= save_stack(GFP_NOWAIT
| __GFP_NOWARN
);
141 for (i
= 0; i
< (1 << order
); i
++) {
142 __clear_bit(PAGE_EXT_OWNER_ALLOCATED
, &page_ext
->flags
);
143 page_owner
= get_page_owner(page_ext
);
144 page_owner
->free_handle
= handle
;
145 page_owner
->free_ts_nsec
= free_ts_nsec
;
146 page_ext
= page_ext_next(page_ext
);
150 static inline void __set_page_owner_handle(struct page_ext
*page_ext
,
151 depot_stack_handle_t handle
,
152 unsigned int order
, gfp_t gfp_mask
)
154 struct page_owner
*page_owner
;
157 for (i
= 0; i
< (1 << order
); i
++) {
158 page_owner
= get_page_owner(page_ext
);
159 page_owner
->handle
= handle
;
160 page_owner
->order
= order
;
161 page_owner
->gfp_mask
= gfp_mask
;
162 page_owner
->last_migrate_reason
= -1;
163 page_owner
->pid
= current
->pid
;
164 page_owner
->ts_nsec
= local_clock();
165 __set_bit(PAGE_EXT_OWNER
, &page_ext
->flags
);
166 __set_bit(PAGE_EXT_OWNER_ALLOCATED
, &page_ext
->flags
);
168 page_ext
= page_ext_next(page_ext
);
172 noinline
void __set_page_owner(struct page
*page
, unsigned int order
,
175 struct page_ext
*page_ext
= lookup_page_ext(page
);
176 depot_stack_handle_t handle
;
178 if (unlikely(!page_ext
))
181 handle
= save_stack(gfp_mask
);
182 __set_page_owner_handle(page_ext
, handle
, order
, gfp_mask
);
185 void __set_page_owner_migrate_reason(struct page
*page
, int reason
)
187 struct page_ext
*page_ext
= lookup_page_ext(page
);
188 struct page_owner
*page_owner
;
190 if (unlikely(!page_ext
))
193 page_owner
= get_page_owner(page_ext
);
194 page_owner
->last_migrate_reason
= reason
;
197 void __split_page_owner(struct page
*page
, unsigned int nr
)
200 struct page_ext
*page_ext
= lookup_page_ext(page
);
201 struct page_owner
*page_owner
;
203 if (unlikely(!page_ext
))
206 for (i
= 0; i
< nr
; i
++) {
207 page_owner
= get_page_owner(page_ext
);
208 page_owner
->order
= 0;
209 page_ext
= page_ext_next(page_ext
);
213 void __copy_page_owner(struct page
*oldpage
, struct page
*newpage
)
215 struct page_ext
*old_ext
= lookup_page_ext(oldpage
);
216 struct page_ext
*new_ext
= lookup_page_ext(newpage
);
217 struct page_owner
*old_page_owner
, *new_page_owner
;
219 if (unlikely(!old_ext
|| !new_ext
))
222 old_page_owner
= get_page_owner(old_ext
);
223 new_page_owner
= get_page_owner(new_ext
);
224 new_page_owner
->order
= old_page_owner
->order
;
225 new_page_owner
->gfp_mask
= old_page_owner
->gfp_mask
;
226 new_page_owner
->last_migrate_reason
=
227 old_page_owner
->last_migrate_reason
;
228 new_page_owner
->handle
= old_page_owner
->handle
;
229 new_page_owner
->pid
= old_page_owner
->pid
;
230 new_page_owner
->ts_nsec
= old_page_owner
->ts_nsec
;
231 new_page_owner
->free_ts_nsec
= old_page_owner
->ts_nsec
;
234 * We don't clear the bit on the oldpage as it's going to be freed
235 * after migration. Until then, the info can be useful in case of
236 * a bug, and the overall stats will be off a bit only temporarily.
237 * Also, migrate_misplaced_transhuge_page() can still fail the
238 * migration and then we want the oldpage to retain the info. But
239 * in that case we also don't need to explicitly clear the info from
240 * the new page, which will be freed.
242 __set_bit(PAGE_EXT_OWNER
, &new_ext
->flags
);
243 __set_bit(PAGE_EXT_OWNER_ALLOCATED
, &new_ext
->flags
);
246 void pagetypeinfo_showmixedcount_print(struct seq_file
*m
,
247 pg_data_t
*pgdat
, struct zone
*zone
)
250 struct page_ext
*page_ext
;
251 struct page_owner
*page_owner
;
252 unsigned long pfn
, block_end_pfn
;
253 unsigned long end_pfn
= zone_end_pfn(zone
);
254 unsigned long count
[MIGRATE_TYPES
] = { 0, };
255 int pageblock_mt
, page_mt
;
258 /* Scan block by block. First and last block may be incomplete */
259 pfn
= zone
->zone_start_pfn
;
262 * Walk the zone in pageblock_nr_pages steps. If a page block spans
263 * a zone boundary, it will be double counted between zones. This does
264 * not matter as the mixed block count will still be correct
266 for (; pfn
< end_pfn
; ) {
267 page
= pfn_to_online_page(pfn
);
269 pfn
= ALIGN(pfn
+ 1, MAX_ORDER_NR_PAGES
);
273 block_end_pfn
= ALIGN(pfn
+ 1, pageblock_nr_pages
);
274 block_end_pfn
= min(block_end_pfn
, end_pfn
);
276 pageblock_mt
= get_pageblock_migratetype(page
);
278 for (; pfn
< block_end_pfn
; pfn
++) {
279 if (!pfn_valid_within(pfn
))
282 /* The pageblock is online, no need to recheck. */
283 page
= pfn_to_page(pfn
);
285 if (page_zone(page
) != zone
)
288 if (PageBuddy(page
)) {
289 unsigned long freepage_order
;
291 freepage_order
= buddy_order_unsafe(page
);
292 if (freepage_order
< MAX_ORDER
)
293 pfn
+= (1UL << freepage_order
) - 1;
297 if (PageReserved(page
))
300 page_ext
= lookup_page_ext(page
);
301 if (unlikely(!page_ext
))
304 if (!test_bit(PAGE_EXT_OWNER_ALLOCATED
, &page_ext
->flags
))
307 page_owner
= get_page_owner(page_ext
);
308 page_mt
= gfp_migratetype(page_owner
->gfp_mask
);
309 if (pageblock_mt
!= page_mt
) {
310 if (is_migrate_cma(pageblock_mt
))
311 count
[MIGRATE_MOVABLE
]++;
313 count
[pageblock_mt
]++;
318 pfn
+= (1UL << page_owner
->order
) - 1;
323 seq_printf(m
, "Node %d, zone %8s ", pgdat
->node_id
, zone
->name
);
324 for (i
= 0; i
< MIGRATE_TYPES
; i
++)
325 seq_printf(m
, "%12lu ", count
[i
]);
330 print_page_owner(char __user
*buf
, size_t count
, unsigned long pfn
,
331 struct page
*page
, struct page_owner
*page_owner
,
332 depot_stack_handle_t handle
)
334 int ret
, pageblock_mt
, page_mt
;
335 unsigned long *entries
;
336 unsigned int nr_entries
;
339 count
= min_t(size_t, count
, PAGE_SIZE
);
340 kbuf
= kmalloc(count
, GFP_KERNEL
);
344 ret
= snprintf(kbuf
, count
,
345 "Page allocated via order %u, mask %#x(%pGg), pid %d, ts %llu ns, free_ts %llu ns\n",
346 page_owner
->order
, page_owner
->gfp_mask
,
347 &page_owner
->gfp_mask
, page_owner
->pid
,
348 page_owner
->ts_nsec
, page_owner
->free_ts_nsec
);
353 /* Print information relevant to grouping pages by mobility */
354 pageblock_mt
= get_pageblock_migratetype(page
);
355 page_mt
= gfp_migratetype(page_owner
->gfp_mask
);
356 ret
+= snprintf(kbuf
+ ret
, count
- ret
,
357 "PFN %lu type %s Block %lu type %s Flags %#lx(%pGp)\n",
359 migratetype_names
[page_mt
],
360 pfn
>> pageblock_order
,
361 migratetype_names
[pageblock_mt
],
362 page
->flags
, &page
->flags
);
367 nr_entries
= stack_depot_fetch(handle
, &entries
);
368 ret
+= stack_trace_snprint(kbuf
+ ret
, count
- ret
, entries
, nr_entries
, 0);
372 if (page_owner
->last_migrate_reason
!= -1) {
373 ret
+= snprintf(kbuf
+ ret
, count
- ret
,
374 "Page has been migrated, last migrate reason: %s\n",
375 migrate_reason_names
[page_owner
->last_migrate_reason
]);
380 ret
+= snprintf(kbuf
+ ret
, count
- ret
, "\n");
384 if (copy_to_user(buf
, kbuf
, ret
))
395 void __dump_page_owner(const struct page
*page
)
397 struct page_ext
*page_ext
= lookup_page_ext(page
);
398 struct page_owner
*page_owner
;
399 depot_stack_handle_t handle
;
400 unsigned long *entries
;
401 unsigned int nr_entries
;
405 if (unlikely(!page_ext
)) {
406 pr_alert("There is not page extension available.\n");
410 page_owner
= get_page_owner(page_ext
);
411 gfp_mask
= page_owner
->gfp_mask
;
412 mt
= gfp_migratetype(gfp_mask
);
414 if (!test_bit(PAGE_EXT_OWNER
, &page_ext
->flags
)) {
415 pr_alert("page_owner info is not present (never set?)\n");
419 if (test_bit(PAGE_EXT_OWNER_ALLOCATED
, &page_ext
->flags
))
420 pr_alert("page_owner tracks the page as allocated\n");
422 pr_alert("page_owner tracks the page as freed\n");
424 pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, ts %llu, free_ts %llu\n",
425 page_owner
->order
, migratetype_names
[mt
], gfp_mask
, &gfp_mask
,
426 page_owner
->pid
, page_owner
->ts_nsec
, page_owner
->free_ts_nsec
);
428 handle
= READ_ONCE(page_owner
->handle
);
430 pr_alert("page_owner allocation stack trace missing\n");
432 nr_entries
= stack_depot_fetch(handle
, &entries
);
433 stack_trace_print(entries
, nr_entries
, 0);
436 handle
= READ_ONCE(page_owner
->free_handle
);
438 pr_alert("page_owner free stack trace missing\n");
440 nr_entries
= stack_depot_fetch(handle
, &entries
);
441 pr_alert("page last free stack trace:\n");
442 stack_trace_print(entries
, nr_entries
, 0);
445 if (page_owner
->last_migrate_reason
!= -1)
446 pr_alert("page has been migrated, last migrate reason: %s\n",
447 migrate_reason_names
[page_owner
->last_migrate_reason
]);
451 read_page_owner(struct file
*file
, char __user
*buf
, size_t count
, loff_t
*ppos
)
455 struct page_ext
*page_ext
;
456 struct page_owner
*page_owner
;
457 depot_stack_handle_t handle
;
459 if (!static_branch_unlikely(&page_owner_inited
))
463 pfn
= min_low_pfn
+ *ppos
;
465 /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */
466 while (!pfn_valid(pfn
) && (pfn
& (MAX_ORDER_NR_PAGES
- 1)) != 0)
469 drain_all_pages(NULL
);
471 /* Find an allocated page */
472 for (; pfn
< max_pfn
; pfn
++) {
474 * If the new page is in a new MAX_ORDER_NR_PAGES area,
475 * validate the area as existing, skip it if not
477 if ((pfn
& (MAX_ORDER_NR_PAGES
- 1)) == 0 && !pfn_valid(pfn
)) {
478 pfn
+= MAX_ORDER_NR_PAGES
- 1;
482 /* Check for holes within a MAX_ORDER area */
483 if (!pfn_valid_within(pfn
))
486 page
= pfn_to_page(pfn
);
487 if (PageBuddy(page
)) {
488 unsigned long freepage_order
= buddy_order_unsafe(page
);
490 if (freepage_order
< MAX_ORDER
)
491 pfn
+= (1UL << freepage_order
) - 1;
495 page_ext
= lookup_page_ext(page
);
496 if (unlikely(!page_ext
))
500 * Some pages could be missed by concurrent allocation or free,
501 * because we don't hold the zone lock.
503 if (!test_bit(PAGE_EXT_OWNER
, &page_ext
->flags
))
507 * Although we do have the info about past allocation of free
508 * pages, it's not relevant for current memory usage.
510 if (!test_bit(PAGE_EXT_OWNER_ALLOCATED
, &page_ext
->flags
))
513 page_owner
= get_page_owner(page_ext
);
516 * Don't print "tail" pages of high-order allocations as that
517 * would inflate the stats.
519 if (!IS_ALIGNED(pfn
, 1 << page_owner
->order
))
523 * Access to page_ext->handle isn't synchronous so we should
524 * be careful to access it.
526 handle
= READ_ONCE(page_owner
->handle
);
530 /* Record the next PFN to read in the file offset */
531 *ppos
= (pfn
- min_low_pfn
) + 1;
533 return print_page_owner(buf
, count
, pfn
, page
,
540 static void init_pages_in_zone(pg_data_t
*pgdat
, struct zone
*zone
)
542 unsigned long pfn
= zone
->zone_start_pfn
;
543 unsigned long end_pfn
= zone_end_pfn(zone
);
544 unsigned long count
= 0;
547 * Walk the zone in pageblock_nr_pages steps. If a page block spans
548 * a zone boundary, it will be double counted between zones. This does
549 * not matter as the mixed block count will still be correct
551 for (; pfn
< end_pfn
; ) {
552 unsigned long block_end_pfn
;
554 if (!pfn_valid(pfn
)) {
555 pfn
= ALIGN(pfn
+ 1, MAX_ORDER_NR_PAGES
);
559 block_end_pfn
= ALIGN(pfn
+ 1, pageblock_nr_pages
);
560 block_end_pfn
= min(block_end_pfn
, end_pfn
);
562 for (; pfn
< block_end_pfn
; pfn
++) {
564 struct page_ext
*page_ext
;
566 if (!pfn_valid_within(pfn
))
569 page
= pfn_to_page(pfn
);
571 if (page_zone(page
) != zone
)
575 * To avoid having to grab zone->lock, be a little
576 * careful when reading buddy page order. The only
577 * danger is that we skip too much and potentially miss
578 * some early allocated pages, which is better than
579 * heavy lock contention.
581 if (PageBuddy(page
)) {
582 unsigned long order
= buddy_order_unsafe(page
);
584 if (order
> 0 && order
< MAX_ORDER
)
585 pfn
+= (1UL << order
) - 1;
589 if (PageReserved(page
))
592 page_ext
= lookup_page_ext(page
);
593 if (unlikely(!page_ext
))
596 /* Maybe overlapping zone */
597 if (test_bit(PAGE_EXT_OWNER
, &page_ext
->flags
))
600 /* Found early allocated page */
601 __set_page_owner_handle(page_ext
, early_handle
,
608 pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n",
609 pgdat
->node_id
, zone
->name
, count
);
612 static void init_zones_in_node(pg_data_t
*pgdat
)
615 struct zone
*node_zones
= pgdat
->node_zones
;
617 for (zone
= node_zones
; zone
- node_zones
< MAX_NR_ZONES
; ++zone
) {
618 if (!populated_zone(zone
))
621 init_pages_in_zone(pgdat
, zone
);
625 static void init_early_allocated_pages(void)
629 for_each_online_pgdat(pgdat
)
630 init_zones_in_node(pgdat
);
633 static const struct file_operations proc_page_owner_operations
= {
634 .read
= read_page_owner
,
637 static int __init
pageowner_init(void)
639 if (!static_branch_unlikely(&page_owner_inited
)) {
640 pr_info("page_owner is disabled\n");
644 debugfs_create_file("page_owner", 0400, NULL
, NULL
,
645 &proc_page_owner_operations
);
649 late_initcall(pageowner_init
)