[mirror_ubuntu-bionic-kernel.git] / mm / page_owner.c

#include <linux/debugfs.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/bootmem.h>
#include <linux/stacktrace.h>
#include <linux/page_owner.h>
#include <linux/jump_label.h>
#include <linux/migrate.h>
#include <linux/stackdepot.h>
#include <linux/seq_file.h>

#include "internal.h"

/*
 * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack)
 * to use off stack temporal storage
 */
#define PAGE_OWNER_STACK_DEPTH (16)

struct page_owner {
	unsigned int order;
	gfp_t gfp_mask;
	int last_migrate_reason;
	depot_stack_handle_t handle;
};

static bool page_owner_disabled = true;
DEFINE_STATIC_KEY_FALSE(page_owner_inited);

static depot_stack_handle_t dummy_handle;
static depot_stack_handle_t failure_handle;
static depot_stack_handle_t early_handle;

static void init_early_allocated_pages(void);

static int early_page_owner_param(char *buf)
{
	if (!buf)
		return -EINVAL;

	if (strcmp(buf, "on") == 0)
		page_owner_disabled = false;

	return 0;
}
early_param("page_owner", early_page_owner_param);

static bool need_page_owner(void)
{
	if (page_owner_disabled)
		return false;

	return true;
}

static __always_inline depot_stack_handle_t create_dummy_stack(void)
{
	unsigned long entries[4];
	struct stack_trace dummy;

	dummy.nr_entries = 0;
	dummy.max_entries = ARRAY_SIZE(entries);
	dummy.entries = &entries[0];
	dummy.skip = 0;

	save_stack_trace(&dummy);
	return depot_save_stack(&dummy, GFP_KERNEL);
}

static noinline void register_dummy_stack(void)
{
	dummy_handle = create_dummy_stack();
}

static noinline void register_failure_stack(void)
{
	failure_handle = create_dummy_stack();
}

static noinline void register_early_stack(void)
{
	early_handle = create_dummy_stack();
}

static void init_page_owner(void)
{
	if (page_owner_disabled)
		return;

	register_dummy_stack();
	register_failure_stack();
	register_early_stack();
	static_branch_enable(&page_owner_inited);
	init_early_allocated_pages();
}

struct page_ext_operations page_owner_ops = {
	.size = sizeof(struct page_owner),
	.need = need_page_owner,
	.init = init_page_owner,
};

static inline struct page_owner *get_page_owner(struct page_ext *page_ext)
{
	return (void *)page_ext + page_owner_ops.offset;
}

void __reset_page_owner(struct page *page, unsigned int order)
{
	int i;
	struct page_ext *page_ext;

	for (i = 0; i < (1 << order); i++) {
		page_ext = lookup_page_ext(page + i);
		if (unlikely(!page_ext))
			continue;
		__clear_bit(PAGE_EXT_OWNER, &page_ext->flags);
	}
}

static inline bool check_recursive_alloc(struct stack_trace *trace,
					unsigned long ip)
{
	int i, count;

	if (!trace->nr_entries)
		return false;

	for (i = 0, count = 0; i < trace->nr_entries; i++) {
		if (trace->entries[i] == ip && ++count == 2)
			return true;
	}

	return false;
}

static noinline depot_stack_handle_t save_stack(gfp_t flags)
{
	unsigned long entries[PAGE_OWNER_STACK_DEPTH];
	struct stack_trace trace = {
		.nr_entries = 0,
		.entries = entries,
		.max_entries = PAGE_OWNER_STACK_DEPTH,
		.skip = 0
	};
	depot_stack_handle_t handle;

	save_stack_trace(&trace);
	if (trace.nr_entries != 0 &&
	    trace.entries[trace.nr_entries-1] == ULONG_MAX)
		trace.nr_entries--;

	/*
	 * We need to check recursion here because our request to stackdepot
	 * could trigger memory allocation to save new entry. New memory
	 * allocation would reach here and call depot_save_stack() again
	 * if we don't catch it. There is still not enough memory in stackdepot
	 * so it would try to allocate memory again and loop forever.
	 */
	if (check_recursive_alloc(&trace, _RET_IP_))
		return dummy_handle;

	handle = depot_save_stack(&trace, flags);
	if (!handle)
		handle = failure_handle;

	return handle;
}

static inline void __set_page_owner_handle(struct page_ext *page_ext,
	depot_stack_handle_t handle, unsigned int order, gfp_t gfp_mask)
{
	struct page_owner *page_owner;

	page_owner = get_page_owner(page_ext);
	page_owner->handle = handle;
	page_owner->order = order;
	page_owner->gfp_mask = gfp_mask;
	page_owner->last_migrate_reason = -1;

	__set_bit(PAGE_EXT_OWNER, &page_ext->flags);
}

noinline void __set_page_owner(struct page *page, unsigned int order,
					gfp_t gfp_mask)
{
	struct page_ext *page_ext = lookup_page_ext(page);
	depot_stack_handle_t handle;

	if (unlikely(!page_ext))
		return;

	handle = save_stack(gfp_mask);
	__set_page_owner_handle(page_ext, handle, order, gfp_mask);
}

void __set_page_owner_migrate_reason(struct page *page, int reason)
{
	struct page_ext *page_ext = lookup_page_ext(page);
	struct page_owner *page_owner;

	if (unlikely(!page_ext))
		return;

	page_owner = get_page_owner(page_ext);
	page_owner->last_migrate_reason = reason;
}

void __split_page_owner(struct page *page, unsigned int order)
{
	int i;
	struct page_ext *page_ext = lookup_page_ext(page);
	struct page_owner *page_owner;

	if (unlikely(!page_ext))
		return;

	page_owner = get_page_owner(page_ext);
	page_owner->order = 0;
	for (i = 1; i < (1 << order); i++)
		__copy_page_owner(page, page + i);
}

void __copy_page_owner(struct page *oldpage, struct page *newpage)
{
	struct page_ext *old_ext = lookup_page_ext(oldpage);
	struct page_ext *new_ext = lookup_page_ext(newpage);
	struct page_owner *old_page_owner, *new_page_owner;

	if (unlikely(!old_ext || !new_ext))
		return;

	old_page_owner = get_page_owner(old_ext);
	new_page_owner = get_page_owner(new_ext);
	new_page_owner->order = old_page_owner->order;
	new_page_owner->gfp_mask = old_page_owner->gfp_mask;
	new_page_owner->last_migrate_reason =
		old_page_owner->last_migrate_reason;
	new_page_owner->handle = old_page_owner->handle;

	/*
	 * We don't clear the bit on the oldpage as it's going to be freed
	 * after migration. Until then, the info can be useful in case of
	 * a bug, and the overal stats will be off a bit only temporarily.
	 * Also, migrate_misplaced_transhuge_page() can still fail the
	 * migration and then we want the oldpage to retain the info. But
	 * in that case we also don't need to explicitly clear the info from
	 * the new page, which will be freed.
	 */
	__set_bit(PAGE_EXT_OWNER, &new_ext->flags);
}

void pagetypeinfo_showmixedcount_print(struct seq_file *m,
				       pg_data_t *pgdat, struct zone *zone)
{
	struct page *page;
	struct page_ext *page_ext;
	struct page_owner *page_owner;
	unsigned long pfn = zone->zone_start_pfn, block_end_pfn;
	unsigned long end_pfn = pfn + zone->spanned_pages;
	unsigned long count[MIGRATE_TYPES] = { 0, };
	int pageblock_mt, page_mt;
	int i;

	/* Scan block by block. First and last block may be incomplete */
	pfn = zone->zone_start_pfn;

	/*
	 * Walk the zone in pageblock_nr_pages steps. If a page block spans
	 * a zone boundary, it will be double counted between zones. This does
	 * not matter as the mixed block count will still be correct
	 */
	for (; pfn < end_pfn; ) {
		if (!pfn_valid(pfn)) {
			pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES);
			continue;
		}

		block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages);
		block_end_pfn = min(block_end_pfn, end_pfn);

		page = pfn_to_page(pfn);
		pageblock_mt = get_pageblock_migratetype(page);

		for (; pfn < block_end_pfn; pfn++) {
			if (!pfn_valid_within(pfn))
				continue;

			page = pfn_to_page(pfn);

			if (page_zone(page) != zone)
				continue;

			if (PageBuddy(page)) {
				unsigned long freepage_order;

				freepage_order = page_order_unsafe(page);
				if (freepage_order < MAX_ORDER)
					pfn += (1UL << freepage_order) - 1;
				continue;
			}

			if (PageReserved(page))
				continue;

			page_ext = lookup_page_ext(page);
			if (unlikely(!page_ext))
				continue;

			if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags))
				continue;

			page_owner = get_page_owner(page_ext);
			page_mt = gfpflags_to_migratetype(
					page_owner->gfp_mask);
			if (pageblock_mt != page_mt) {
				if (is_migrate_cma(pageblock_mt))
					count[MIGRATE_MOVABLE]++;
				else
					count[pageblock_mt]++;

				pfn = block_end_pfn;
				break;
			}
			pfn += (1UL << page_owner->order) - 1;
		}
	}

	/* Print counts */
	seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
	for (i = 0; i < MIGRATE_TYPES; i++)
		seq_printf(m, "%12lu ", count[i]);
	seq_putc(m, '\n');
}

static ssize_t
print_page_owner(char __user *buf, size_t count, unsigned long pfn,
		struct page *page, struct page_owner *page_owner,
		depot_stack_handle_t handle)
{
	int ret;
	int pageblock_mt, page_mt;
	char *kbuf;
	unsigned long entries[PAGE_OWNER_STACK_DEPTH];
	struct stack_trace trace = {
		.nr_entries = 0,
		.entries = entries,
		.max_entries = PAGE_OWNER_STACK_DEPTH,
		.skip = 0
	};

	kbuf = kmalloc(count, GFP_KERNEL);
	if (!kbuf)
		return -ENOMEM;

	ret = snprintf(kbuf, count,
			"Page allocated via order %u, mask %#x(%pGg)\n",
			page_owner->order, page_owner->gfp_mask,
			&page_owner->gfp_mask);

	if (ret >= count)
		goto err;

	/* Print information relevant to grouping pages by mobility */
	pageblock_mt = get_pageblock_migratetype(page);
	page_mt  = gfpflags_to_migratetype(page_owner->gfp_mask);
	ret += snprintf(kbuf + ret, count - ret,
			"PFN %lu type %s Block %lu type %s Flags %#lx(%pGp)\n",
			pfn,
			migratetype_names[page_mt],
			pfn >> pageblock_order,
			migratetype_names[pageblock_mt],
			page->flags, &page->flags);

	if (ret >= count)
		goto err;

	depot_fetch_stack(handle, &trace);
	ret += snprint_stack_trace(kbuf + ret, count - ret, &trace, 0);
	if (ret >= count)
		goto err;

	if (page_owner->last_migrate_reason != -1) {
		ret += snprintf(kbuf + ret, count - ret,
			"Page has been migrated, last migrate reason: %s\n",
			migrate_reason_names[page_owner->last_migrate_reason]);
		if (ret >= count)
			goto err;
	}

	ret += snprintf(kbuf + ret, count - ret, "\n");
	if (ret >= count)
		goto err;

	if (copy_to_user(buf, kbuf, ret))
		ret = -EFAULT;

	kfree(kbuf);
	return ret;

err:
	kfree(kbuf);
	return -ENOMEM;
}

void __dump_page_owner(struct page *page)
{
	struct page_ext *page_ext = lookup_page_ext(page);
	struct page_owner *page_owner;
	unsigned long entries[PAGE_OWNER_STACK_DEPTH];
	struct stack_trace trace = {
		.nr_entries = 0,
		.entries = entries,
		.max_entries = PAGE_OWNER_STACK_DEPTH,
		.skip = 0
	};
	depot_stack_handle_t handle;
	gfp_t gfp_mask;
	int mt;

	if (unlikely(!page_ext)) {
		pr_alert("There is not page extension available.\n");
		return;
	}

	page_owner = get_page_owner(page_ext);
	gfp_mask = page_owner->gfp_mask;
	mt = gfpflags_to_migratetype(gfp_mask);

	if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) {
		pr_alert("page_owner info is not active (free page?)\n");
		return;
	}

	handle = READ_ONCE(page_owner->handle);
	if (!handle) {
		pr_alert("page_owner info is not active (free page?)\n");
		return;
	}

	depot_fetch_stack(handle, &trace);
	pr_alert("page allocated via order %u, migratetype %s, gfp_mask %#x(%pGg)\n",
		 page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask);
	print_stack_trace(&trace, 0);

	if (page_owner->last_migrate_reason != -1)
		pr_alert("page has been migrated, last migrate reason: %s\n",
			migrate_reason_names[page_owner->last_migrate_reason]);
}

static ssize_t
read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos)
{
	unsigned long pfn;
	struct page *page;
	struct page_ext *page_ext;
	struct page_owner *page_owner;
	depot_stack_handle_t handle;

	if (!static_branch_unlikely(&page_owner_inited))
		return -EINVAL;

	page = NULL;
	pfn = min_low_pfn + *ppos;

	/* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */
	while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0)
		pfn++;

	drain_all_pages(NULL);

	/* Find an allocated page */
	for (; pfn < max_pfn; pfn++) {
		/*
		 * If the new page is in a new MAX_ORDER_NR_PAGES area,
		 * validate the area as existing, skip it if not
		 */
		if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) {
			pfn += MAX_ORDER_NR_PAGES - 1;
			continue;
		}

		/* Check for holes within a MAX_ORDER area */
		if (!pfn_valid_within(pfn))
			continue;

		page = pfn_to_page(pfn);
		if (PageBuddy(page)) {
			unsigned long freepage_order = page_order_unsafe(page);

			if (freepage_order < MAX_ORDER)
				pfn += (1UL << freepage_order) - 1;
			continue;
		}

		page_ext = lookup_page_ext(page);
		if (unlikely(!page_ext))
			continue;

		/*
		 * Some pages could be missed by concurrent allocation or free,
		 * because we don't hold the zone lock.
		 */
		if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags))
			continue;

		page_owner = get_page_owner(page_ext);

		/*
		 * Access to page_ext->handle isn't synchronous so we should
		 * be careful to access it.
		 */
		handle = READ_ONCE(page_owner->handle);
		if (!handle)
			continue;

		/* Record the next PFN to read in the file offset */
		*ppos = (pfn - min_low_pfn) + 1;

		return print_page_owner(buf, count, pfn, page,
				page_owner, handle);
	}

	return 0;
}

static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone)
{
	struct page *page;
	struct page_ext *page_ext;
	unsigned long pfn = zone->zone_start_pfn, block_end_pfn;
	unsigned long end_pfn = pfn + zone->spanned_pages;
	unsigned long count = 0;

	/* Scan block by block. First and last block may be incomplete */
	pfn = zone->zone_start_pfn;

	/*
	 * Walk the zone in pageblock_nr_pages steps. If a page block spans
	 * a zone boundary, it will be double counted between zones. This does
	 * not matter as the mixed block count will still be correct
	 */
	for (; pfn < end_pfn; ) {
		if (!pfn_valid(pfn)) {
			pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES);
			continue;
		}

		block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages);
		block_end_pfn = min(block_end_pfn, end_pfn);

		page = pfn_to_page(pfn);

		for (; pfn < block_end_pfn; pfn++) {
			if (!pfn_valid_within(pfn))
				continue;

			page = pfn_to_page(pfn);

			if (page_zone(page) != zone)
				continue;

			/*
			 * To avoid having to grab zone->lock, be a little
			 * careful when reading buddy page order. The only
			 * danger is that we skip too much and potentially miss
			 * some early allocated pages, which is better than
			 * heavy lock contention.
			 */
			if (PageBuddy(page)) {
				unsigned long order = page_order_unsafe(page);

				if (order > 0 && order < MAX_ORDER)
					pfn += (1UL << order) - 1;
				continue;
			}

			if (PageReserved(page))
				continue;

			page_ext = lookup_page_ext(page);
			if (unlikely(!page_ext))
				continue;

			/* Maybe overlapping zone */
			if (test_bit(PAGE_EXT_OWNER, &page_ext->flags))
				continue;

			/* Found early allocated page */
			__set_page_owner_handle(page_ext, early_handle, 0, 0);
			count++;
		}
		cond_resched();
	}

	pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n",
		pgdat->node_id, zone->name, count);
}

static void init_zones_in_node(pg_data_t *pgdat)
{
	struct zone *zone;
	struct zone *node_zones = pgdat->node_zones;

	for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
		if (!populated_zone(zone))
			continue;

		init_pages_in_zone(pgdat, zone);
	}
}

static void init_early_allocated_pages(void)
{
	pg_data_t *pgdat;

	drain_all_pages(NULL);
	for_each_online_pgdat(pgdat)
		init_zones_in_node(pgdat);
}

static const struct file_operations proc_page_owner_operations = {
	.read		= read_page_owner,
};

static int __init pageowner_init(void)
{
	struct dentry *dentry;

	if (!static_branch_unlikely(&page_owner_inited)) {
		pr_info("page_owner is disabled\n");
		return 0;
	}

	dentry = debugfs_create_file("page_owner", S_IRUSR, NULL,
			NULL, &proc_page_owner_operations);
	if (IS_ERR(dentry))
		return PTR_ERR(dentry);

	return 0;
}
late_initcall(pageowner_init)
Commit	Line	Data
48c96a36 JK	1	#include <linux/debugfs.h>
	2	#include <linux/mm.h>
	3	#include <linux/slab.h>
	4	#include <linux/uaccess.h>
	5	#include <linux/bootmem.h>
	6	#include <linux/stacktrace.h>
	7	#include <linux/page_owner.h>
7dd80b8a	8	#include <linux/jump_label.h>
7cd12b4a	9	#include <linux/migrate.h>
f2ca0b55	10	#include <linux/stackdepot.h>
e2f612e6	11	#include <linux/seq_file.h>
f2ca0b55	12
48c96a36 JK	13	#include "internal.h"
48c96a36 JK	14
f2ca0b55 JK	15	/*
	16	* TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack)
	17	* to use off stack temporal storage
	18	*/
	19	#define PAGE_OWNER_STACK_DEPTH (16)
	20
9300d8df JK	21	struct page_owner {
	22	unsigned int order;
	23	gfp_t gfp_mask;
	24	int last_migrate_reason;
	25	depot_stack_handle_t handle;
	26	};
	27
48c96a36	28	static bool page_owner_disabled = true;
7dd80b8a	29	DEFINE_STATIC_KEY_FALSE(page_owner_inited);
48c96a36	30
f2ca0b55 JK	31	static depot_stack_handle_t dummy_handle;
f2ca0b55 JK	32	static depot_stack_handle_t failure_handle;
dab4ead1	33	static depot_stack_handle_t early_handle;
f2ca0b55	34
61cf5feb JK	35	static void init_early_allocated_pages(void);
61cf5feb JK	36
48c96a36 JK	37	static int early_page_owner_param(char *buf)
	38	{
	39	if (!buf)
	40	return -EINVAL;
	41
	42	if (strcmp(buf, "on") == 0)
	43	page_owner_disabled = false;
	44
	45	return 0;
	46	}
	47	early_param("page_owner", early_page_owner_param);
	48
	49	static bool need_page_owner(void)
	50	{
	51	if (page_owner_disabled)
	52	return false;
	53
	54	return true;
	55	}
	56
dab4ead1	57	static __always_inline depot_stack_handle_t create_dummy_stack(void)
f2ca0b55 JK	58	{
	59	unsigned long entries[4];
	60	struct stack_trace dummy;
	61
	62	dummy.nr_entries = 0;
	63	dummy.max_entries = ARRAY_SIZE(entries);
	64	dummy.entries = &entries[0];
	65	dummy.skip = 0;
	66
	67	save_stack_trace(&dummy);
dab4ead1	68	return depot_save_stack(&dummy, GFP_KERNEL);
f2ca0b55 JK	69	}
f2ca0b55 JK	70
dab4ead1	71	static noinline void register_dummy_stack(void)
f2ca0b55	72	{
dab4ead1 VB	73	dummy_handle = create_dummy_stack();
dab4ead1 VB	74	}
f2ca0b55	75
dab4ead1 VB	76	static noinline void register_failure_stack(void)
	77	{
	78	failure_handle = create_dummy_stack();
	79	}
f2ca0b55	80
dab4ead1 VB	81	static noinline void register_early_stack(void)
	82	{
	83	early_handle = create_dummy_stack();
f2ca0b55 JK	84	}
f2ca0b55 JK	85
48c96a36 JK	86	static void init_page_owner(void)
	87	{
	88	if (page_owner_disabled)
	89	return;
	90
f2ca0b55 JK	91	register_dummy_stack();
f2ca0b55 JK	92	register_failure_stack();
dab4ead1	93	register_early_stack();
7dd80b8a	94	static_branch_enable(&page_owner_inited);
61cf5feb	95	init_early_allocated_pages();
48c96a36 JK	96	}
	97
	98	struct page_ext_operations page_owner_ops = {
9300d8df	99	.size = sizeof(struct page_owner),
48c96a36 JK	100	.need = need_page_owner,
	101	.init = init_page_owner,
	102	};
	103
9300d8df JK	104	static inline struct page_owner get_page_owner(struct page_ext page_ext)
	105	{
	106	return (void *)page_ext + page_owner_ops.offset;
	107	}
	108
48c96a36 JK	109	void __reset_page_owner(struct page *page, unsigned int order)
	110	{
	111	int i;
	112	struct page_ext *page_ext;
	113
	114	for (i = 0; i < (1 << order); i++) {
	115	page_ext = lookup_page_ext(page + i);
f86e4271 YS	116	if (unlikely(!page_ext))
f86e4271 YS	117	continue;
48c96a36 JK	118	__clear_bit(PAGE_EXT_OWNER, &page_ext->flags);
	119	}
	120	}
	121
f2ca0b55 JK	122	static inline bool check_recursive_alloc(struct stack_trace *trace,
f2ca0b55 JK	123	unsigned long ip)
48c96a36	124	{
f2ca0b55 JK	125	int i, count;
	126
	127	if (!trace->nr_entries)
	128	return false;
	129
	130	for (i = 0, count = 0; i < trace->nr_entries; i++) {
	131	if (trace->entries[i] == ip && ++count == 2)
	132	return true;
	133	}
f86e4271	134
f2ca0b55 JK	135	return false;
	136	}
	137
	138	static noinline depot_stack_handle_t save_stack(gfp_t flags)
	139	{
	140	unsigned long entries[PAGE_OWNER_STACK_DEPTH];
94f759d6 SR	141	struct stack_trace trace = {
94f759d6 SR	142	.nr_entries = 0,
f2ca0b55 JK	143	.entries = entries,
	144	.max_entries = PAGE_OWNER_STACK_DEPTH,
	145	.skip = 0
94f759d6	146	};
f2ca0b55 JK	147	depot_stack_handle_t handle;
	148
	149	save_stack_trace(&trace);
	150	if (trace.nr_entries != 0 &&
	151	trace.entries[trace.nr_entries-1] == ULONG_MAX)
	152	trace.nr_entries--;
	153
	154	/*
	155	* We need to check recursion here because our request to stackdepot
	156	* could trigger memory allocation to save new entry. New memory
	157	* allocation would reach here and call depot_save_stack() again
	158	* if we don't catch it. There is still not enough memory in stackdepot
	159	* so it would try to allocate memory again and loop forever.
	160	*/
	161	if (check_recursive_alloc(&trace, _RET_IP_))
	162	return dummy_handle;
	163
	164	handle = depot_save_stack(&trace, flags);
	165	if (!handle)
	166	handle = failure_handle;
	167
	168	return handle;
	169	}
	170
dab4ead1 VB	171	static inline void __set_page_owner_handle(struct page_ext *page_ext,
dab4ead1 VB	172	depot_stack_handle_t handle, unsigned int order, gfp_t gfp_mask)
f2ca0b55	173	{
9300d8df	174	struct page_owner *page_owner;
48c96a36	175
9300d8df	176	page_owner = get_page_owner(page_ext);
dab4ead1	177	page_owner->handle = handle;
9300d8df JK	178	page_owner->order = order;
	179	page_owner->gfp_mask = gfp_mask;
	180	page_owner->last_migrate_reason = -1;
48c96a36 JK	181
	182	__set_bit(PAGE_EXT_OWNER, &page_ext->flags);
	183	}
	184
dab4ead1 VB	185	noinline void __set_page_owner(struct page *page, unsigned int order,
	186	gfp_t gfp_mask)
	187	{
	188	struct page_ext *page_ext = lookup_page_ext(page);
	189	depot_stack_handle_t handle;
	190
	191	if (unlikely(!page_ext))
	192	return;
	193
	194	handle = save_stack(gfp_mask);
	195	__set_page_owner_handle(page_ext, handle, order, gfp_mask);
	196	}
	197
7cd12b4a VB	198	void __set_page_owner_migrate_reason(struct page *page, int reason)
	199	{
	200	struct page_ext *page_ext = lookup_page_ext(page);
9300d8df JK	201	struct page_owner *page_owner;
9300d8df JK	202
f86e4271 YS	203	if (unlikely(!page_ext))
f86e4271 YS	204	return;
7cd12b4a	205
9300d8df JK	206	page_owner = get_page_owner(page_ext);
9300d8df JK	207	page_owner->last_migrate_reason = reason;
7cd12b4a VB	208	}
7cd12b4a VB	209
a9627bc5	210	void __split_page_owner(struct page *page, unsigned int order)
e2cfc911	211	{
a9627bc5	212	int i;
e2cfc911	213	struct page_ext *page_ext = lookup_page_ext(page);
9300d8df	214	struct page_owner *page_owner;
a9627bc5	215
f86e4271	216	if (unlikely(!page_ext))
a9627bc5	217	return;
e2cfc911	218
9300d8df JK	219	page_owner = get_page_owner(page_ext);
9300d8df JK	220	page_owner->order = 0;
a9627bc5 JK	221	for (i = 1; i < (1 << order); i++)
a9627bc5 JK	222	__copy_page_owner(page, page + i);
e2cfc911 JK	223	}
e2cfc911 JK	224
d435edca VB	225	void __copy_page_owner(struct page oldpage, struct page newpage)
	226	{
	227	struct page_ext *old_ext = lookup_page_ext(oldpage);
	228	struct page_ext *new_ext = lookup_page_ext(newpage);
9300d8df	229	struct page_owner old_page_owner, new_page_owner;
d435edca	230
f86e4271 YS	231	if (unlikely(!old_ext \|\| !new_ext))
	232	return;
	233
9300d8df JK	234	old_page_owner = get_page_owner(old_ext);
	235	new_page_owner = get_page_owner(new_ext);
	236	new_page_owner->order = old_page_owner->order;
	237	new_page_owner->gfp_mask = old_page_owner->gfp_mask;
	238	new_page_owner->last_migrate_reason =
	239	old_page_owner->last_migrate_reason;
	240	new_page_owner->handle = old_page_owner->handle;
d435edca VB	241
	242	/*
	243	* We don't clear the bit on the oldpage as it's going to be freed
	244	* after migration. Until then, the info can be useful in case of
	245	* a bug, and the overal stats will be off a bit only temporarily.
	246	* Also, migrate_misplaced_transhuge_page() can still fail the
	247	* migration and then we want the oldpage to retain the info. But
	248	* in that case we also don't need to explicitly clear the info from
	249	* the new page, which will be freed.
	250	*/
	251	__set_bit(PAGE_EXT_OWNER, &new_ext->flags);
	252	}
	253
e2f612e6 JK	254	void pagetypeinfo_showmixedcount_print(struct seq_file *m,
	255	pg_data_t pgdat, struct zone zone)
	256	{
	257	struct page *page;
	258	struct page_ext *page_ext;
9300d8df	259	struct page_owner *page_owner;
e2f612e6 JK	260	unsigned long pfn = zone->zone_start_pfn, block_end_pfn;
	261	unsigned long end_pfn = pfn + zone->spanned_pages;
	262	unsigned long count[MIGRATE_TYPES] = { 0, };
	263	int pageblock_mt, page_mt;
	264	int i;
	265
	266	/* Scan block by block. First and last block may be incomplete */
	267	pfn = zone->zone_start_pfn;
	268
	269	/*
	270	* Walk the zone in pageblock_nr_pages steps. If a page block spans
	271	* a zone boundary, it will be double counted between zones. This does
	272	* not matter as the mixed block count will still be correct
	273	*/
	274	for (; pfn < end_pfn; ) {
	275	if (!pfn_valid(pfn)) {
	276	pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES);
	277	continue;
	278	}
	279
	280	block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages);
	281	block_end_pfn = min(block_end_pfn, end_pfn);
	282
	283	page = pfn_to_page(pfn);
	284	pageblock_mt = get_pageblock_migratetype(page);
	285
	286	for (; pfn < block_end_pfn; pfn++) {
	287	if (!pfn_valid_within(pfn))
	288	continue;
	289
	290	page = pfn_to_page(pfn);
	291
	292	if (page_zone(page) != zone)
	293	continue;
	294
	295	if (PageBuddy(page)) {
727c080f VM	296	unsigned long freepage_order;
	297
	298	freepage_order = page_order_unsafe(page);
	299	if (freepage_order < MAX_ORDER)
	300	pfn += (1UL << freepage_order) - 1;
e2f612e6 JK	301	continue;
	302	}
	303
	304	if (PageReserved(page))
	305	continue;
	306
	307	page_ext = lookup_page_ext(page);
	308	if (unlikely(!page_ext))
	309	continue;
	310
	311	if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags))
	312	continue;
	313
9300d8df JK	314	page_owner = get_page_owner(page_ext);
	315	page_mt = gfpflags_to_migratetype(
	316	page_owner->gfp_mask);
e2f612e6 JK	317	if (pageblock_mt != page_mt) {
	318	if (is_migrate_cma(pageblock_mt))
	319	count[MIGRATE_MOVABLE]++;
	320	else
	321	count[pageblock_mt]++;
	322
	323	pfn = block_end_pfn;
	324	break;
	325	}
9300d8df	326	pfn += (1UL << page_owner->order) - 1;
e2f612e6 JK	327	}
	328	}
	329
	330	/* Print counts */
	331	seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
	332	for (i = 0; i < MIGRATE_TYPES; i++)
	333	seq_printf(m, "%12lu ", count[i]);
	334	seq_putc(m, '\n');
	335	}
	336
48c96a36 JK	337	static ssize_t
48c96a36 JK	338	print_page_owner(char __user *buf, size_t count, unsigned long pfn,
9300d8df	339	struct page page, struct page_owner page_owner,
f2ca0b55	340	depot_stack_handle_t handle)
48c96a36 JK	341	{
	342	int ret;
	343	int pageblock_mt, page_mt;
	344	char *kbuf;
f2ca0b55	345	unsigned long entries[PAGE_OWNER_STACK_DEPTH];
94f759d6	346	struct stack_trace trace = {
f2ca0b55 JK	347	.nr_entries = 0,
	348	.entries = entries,
	349	.max_entries = PAGE_OWNER_STACK_DEPTH,
	350	.skip = 0
94f759d6	351	};
48c96a36 JK	352
	353	kbuf = kmalloc(count, GFP_KERNEL);
	354	if (!kbuf)
	355	return -ENOMEM;
	356
	357	ret = snprintf(kbuf, count,
60f30350	358	"Page allocated via order %u, mask %#x(%pGg)\n",
9300d8df JK	359	page_owner->order, page_owner->gfp_mask,
9300d8df JK	360	&page_owner->gfp_mask);
48c96a36 JK	361
	362	if (ret >= count)
	363	goto err;
	364
	365	/* Print information relevant to grouping pages by mobility */
0b423ca2	366	pageblock_mt = get_pageblock_migratetype(page);
9300d8df	367	page_mt = gfpflags_to_migratetype(page_owner->gfp_mask);
48c96a36	368	ret += snprintf(kbuf + ret, count - ret,
60f30350	369	"PFN %lu type %s Block %lu type %s Flags %#lx(%pGp)\n",
48c96a36	370	pfn,
60f30350	371	migratetype_names[page_mt],
48c96a36	372	pfn >> pageblock_order,
60f30350 VB	373	migratetype_names[pageblock_mt],
60f30350 VB	374	page->flags, &page->flags);
48c96a36 JK	375
	376	if (ret >= count)
	377	goto err;
	378
f2ca0b55	379	depot_fetch_stack(handle, &trace);
94f759d6	380	ret += snprint_stack_trace(kbuf + ret, count - ret, &trace, 0);
48c96a36 JK	381	if (ret >= count)
	382	goto err;
	383
9300d8df	384	if (page_owner->last_migrate_reason != -1) {
7cd12b4a VB	385	ret += snprintf(kbuf + ret, count - ret,
7cd12b4a VB	386	"Page has been migrated, last migrate reason: %s\n",
9300d8df	387	migrate_reason_names[page_owner->last_migrate_reason]);
7cd12b4a VB	388	if (ret >= count)
	389	goto err;
	390	}
	391
48c96a36 JK	392	ret += snprintf(kbuf + ret, count - ret, "\n");
	393	if (ret >= count)
	394	goto err;
	395
	396	if (copy_to_user(buf, kbuf, ret))
	397	ret = -EFAULT;
	398
	399	kfree(kbuf);
	400	return ret;
	401
	402	err:
	403	kfree(kbuf);
	404	return -ENOMEM;
	405	}
	406
4e462112 VB	407	void __dump_page_owner(struct page *page)
	408	{
	409	struct page_ext *page_ext = lookup_page_ext(page);
9300d8df	410	struct page_owner *page_owner;
f2ca0b55	411	unsigned long entries[PAGE_OWNER_STACK_DEPTH];
4e462112	412	struct stack_trace trace = {
f2ca0b55 JK	413	.nr_entries = 0,
	414	.entries = entries,
	415	.max_entries = PAGE_OWNER_STACK_DEPTH,
	416	.skip = 0
4e462112	417	};
f2ca0b55	418	depot_stack_handle_t handle;
8285027f SM	419	gfp_t gfp_mask;
8285027f SM	420	int mt;
4e462112	421
f86e4271 YS	422	if (unlikely(!page_ext)) {
	423	pr_alert("There is not page extension available.\n");
	424	return;
	425	}
9300d8df JK	426
	427	page_owner = get_page_owner(page_ext);
	428	gfp_mask = page_owner->gfp_mask;
8285027f	429	mt = gfpflags_to_migratetype(gfp_mask);
f86e4271	430
4e462112 VB	431	if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) {
	432	pr_alert("page_owner info is not active (free page?)\n");
	433	return;
	434	}
	435
9300d8df	436	handle = READ_ONCE(page_owner->handle);
f2ca0b55 JK	437	if (!handle) {
	438	pr_alert("page_owner info is not active (free page?)\n");
	439	return;
	440	}
	441
	442	depot_fetch_stack(handle, &trace);
756a025f	443	pr_alert("page allocated via order %u, migratetype %s, gfp_mask %#x(%pGg)\n",
9300d8df	444	page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask);
4e462112 VB	445	print_stack_trace(&trace, 0);
4e462112 VB	446
9300d8df	447	if (page_owner->last_migrate_reason != -1)
4e462112	448	pr_alert("page has been migrated, last migrate reason: %s\n",
9300d8df	449	migrate_reason_names[page_owner->last_migrate_reason]);
4e462112 VB	450	}
4e462112 VB	451
48c96a36 JK	452	static ssize_t
	453	read_page_owner(struct file file, char __user buf, size_t count, loff_t *ppos)
	454	{
	455	unsigned long pfn;
	456	struct page *page;
	457	struct page_ext *page_ext;
9300d8df	458	struct page_owner *page_owner;
f2ca0b55	459	depot_stack_handle_t handle;
48c96a36	460
7dd80b8a	461	if (!static_branch_unlikely(&page_owner_inited))
48c96a36 JK	462	return -EINVAL;
	463
	464	page = NULL;
	465	pfn = min_low_pfn + *ppos;
	466
	467	/* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */
	468	while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0)
	469	pfn++;
	470
	471	drain_all_pages(NULL);
	472
	473	/* Find an allocated page */
	474	for (; pfn < max_pfn; pfn++) {
	475	/*
	476	* If the new page is in a new MAX_ORDER_NR_PAGES area,
	477	* validate the area as existing, skip it if not
	478	*/
	479	if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) {
	480	pfn += MAX_ORDER_NR_PAGES - 1;
	481	continue;
	482	}
	483
	484	/* Check for holes within a MAX_ORDER area */
	485	if (!pfn_valid_within(pfn))
	486	continue;
	487
	488	page = pfn_to_page(pfn);
	489	if (PageBuddy(page)) {
	490	unsigned long freepage_order = page_order_unsafe(page);
	491
	492	if (freepage_order < MAX_ORDER)
	493	pfn += (1UL << freepage_order) - 1;
	494	continue;
	495	}
	496
	497	page_ext = lookup_page_ext(page);
f86e4271 YS	498	if (unlikely(!page_ext))
f86e4271 YS	499	continue;
48c96a36 JK	500
48c96a36 JK	501	/*
61cf5feb JK	502	* Some pages could be missed by concurrent allocation or free,
61cf5feb JK	503	* because we don't hold the zone lock.
48c96a36 JK	504	*/
	505	if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags))
	506	continue;
	507
9300d8df JK	508	page_owner = get_page_owner(page_ext);
9300d8df JK	509
f2ca0b55 JK	510	/*
	511	* Access to page_ext->handle isn't synchronous so we should
	512	* be careful to access it.
	513	*/
9300d8df	514	handle = READ_ONCE(page_owner->handle);
f2ca0b55 JK	515	if (!handle)
	516	continue;
	517
48c96a36 JK	518	/* Record the next PFN to read in the file offset */
	519	*ppos = (pfn - min_low_pfn) + 1;
	520
f2ca0b55	521	return print_page_owner(buf, count, pfn, page,
9300d8df	522	page_owner, handle);
48c96a36 JK	523	}
	524
	525	return 0;
	526	}
	527
61cf5feb JK	528	static void init_pages_in_zone(pg_data_t pgdat, struct zone zone)
	529	{
	530	struct page *page;
	531	struct page_ext *page_ext;
	532	unsigned long pfn = zone->zone_start_pfn, block_end_pfn;
	533	unsigned long end_pfn = pfn + zone->spanned_pages;
	534	unsigned long count = 0;
	535
	536	/* Scan block by block. First and last block may be incomplete */
	537	pfn = zone->zone_start_pfn;
	538
	539	/*
	540	* Walk the zone in pageblock_nr_pages steps. If a page block spans
	541	* a zone boundary, it will be double counted between zones. This does
	542	* not matter as the mixed block count will still be correct
	543	*/
	544	for (; pfn < end_pfn; ) {
	545	if (!pfn_valid(pfn)) {
	546	pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES);
	547	continue;
	548	}
	549
	550	block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages);
	551	block_end_pfn = min(block_end_pfn, end_pfn);
	552
	553	page = pfn_to_page(pfn);
	554
	555	for (; pfn < block_end_pfn; pfn++) {
	556	if (!pfn_valid_within(pfn))
	557	continue;
	558
	559	page = pfn_to_page(pfn);
	560
9d43f5ae JK	561	if (page_zone(page) != zone)
	562	continue;
	563
61cf5feb	564	/*
10903027 VB	565	* To avoid having to grab zone->lock, be a little
	566	* careful when reading buddy page order. The only
	567	* danger is that we skip too much and potentially miss
	568	* some early allocated pages, which is better than
	569	* heavy lock contention.
61cf5feb JK	570	*/
61cf5feb JK	571	if (PageBuddy(page)) {
10903027 VB	572	unsigned long order = page_order_unsafe(page);
	573
	574	if (order > 0 && order < MAX_ORDER)
	575	pfn += (1UL << order) - 1;
61cf5feb JK	576	continue;
	577	}
	578
	579	if (PageReserved(page))
	580	continue;
	581
	582	page_ext = lookup_page_ext(page);
f86e4271 YS	583	if (unlikely(!page_ext))
f86e4271 YS	584	continue;
61cf5feb	585
dab4ead1	586	/* Maybe overlapping zone */
61cf5feb JK	587	if (test_bit(PAGE_EXT_OWNER, &page_ext->flags))
	588	continue;
	589
	590	/* Found early allocated page */
dab4ead1	591	__set_page_owner_handle(page_ext, early_handle, 0, 0);
61cf5feb JK	592	count++;
61cf5feb JK	593	}
10903027	594	cond_resched();
61cf5feb JK	595	}
	596
	597	pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n",
	598	pgdat->node_id, zone->name, count);
	599	}
	600
	601	static void init_zones_in_node(pg_data_t *pgdat)
	602	{
	603	struct zone *zone;
	604	struct zone *node_zones = pgdat->node_zones;
61cf5feb JK	605
	606	for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
	607	if (!populated_zone(zone))
	608	continue;
	609
61cf5feb	610	init_pages_in_zone(pgdat, zone);
61cf5feb JK	611	}
	612	}
	613
	614	static void init_early_allocated_pages(void)
	615	{
	616	pg_data_t *pgdat;
	617
	618	drain_all_pages(NULL);
	619	for_each_online_pgdat(pgdat)
	620	init_zones_in_node(pgdat);
	621	}
	622
48c96a36 JK	623	static const struct file_operations proc_page_owner_operations = {
	624	.read = read_page_owner,
	625	};
	626
	627	static int __init pageowner_init(void)
	628	{
	629	struct dentry *dentry;
	630
7dd80b8a	631	if (!static_branch_unlikely(&page_owner_inited)) {
48c96a36 JK	632	pr_info("page_owner is disabled\n");
	633	return 0;
	634	}
	635
	636	dentry = debugfs_create_file("page_owner", S_IRUSR, NULL,
	637	NULL, &proc_page_owner_operations);
	638	if (IS_ERR(dentry))
	639	return PTR_ERR(dentry);
	640
	641	return 0;
	642	}
44c5af96	643	late_initcall(pageowner_init)