[mirror_ubuntu-artful-kernel.git] / mm / page_cgroup.c

#include <linux/mm.h>
#include <linux/mmzone.h>
#include <linux/bootmem.h>
#include <linux/bit_spinlock.h>
#include <linux/page_cgroup.h>
#include <linux/hash.h>
#include <linux/slab.h>
#include <linux/memory.h>
#include <linux/vmalloc.h>
#include <linux/cgroup.h>
#include <linux/swapops.h>
#include <linux/kmemleak.h>

static unsigned long total_usage;

#if !defined(CONFIG_SPARSEMEM)


void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat)
{
	pgdat->node_page_cgroup = NULL;
}

struct page_cgroup *lookup_page_cgroup(struct page *page)
{
	unsigned long pfn = page_to_pfn(page);
	unsigned long offset;
	struct page_cgroup *base;

	base = NODE_DATA(page_to_nid(page))->node_page_cgroup;
#ifdef CONFIG_DEBUG_VM
	/*
	 * The sanity checks the page allocator does upon freeing a
	 * page can reach here before the page_cgroup arrays are
	 * allocated when feeding a range of pages to the allocator
	 * for the first time during bootup or memory hotplug.
	 */
	if (unlikely(!base))
		return NULL;
#endif
	offset = pfn - NODE_DATA(page_to_nid(page))->node_start_pfn;
	return base + offset;
}

static int __init alloc_node_page_cgroup(int nid)
{
	struct page_cgroup *base;
	unsigned long table_size;
	unsigned long nr_pages;

	nr_pages = NODE_DATA(nid)->node_spanned_pages;
	if (!nr_pages)
		return 0;

	table_size = sizeof(struct page_cgroup) * nr_pages;

	base = memblock_virt_alloc_try_nid_nopanic(
			table_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS),
			BOOTMEM_ALLOC_ACCESSIBLE, nid);
	if (!base)
		return -ENOMEM;
	NODE_DATA(nid)->node_page_cgroup = base;
	total_usage += table_size;
	return 0;
}

void __init page_cgroup_init_flatmem(void)
{

	int nid, fail;

	if (mem_cgroup_disabled())
		return;

	for_each_online_node(nid)  {
		fail = alloc_node_page_cgroup(nid);
		if (fail)
			goto fail;
	}
	printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
	printk(KERN_INFO "please try 'cgroup_disable=memory' option if you"
	" don't want memory cgroups\n");
	return;
fail:
	printk(KERN_CRIT "allocation of page_cgroup failed.\n");
	printk(KERN_CRIT "please try 'cgroup_disable=memory' boot option\n");
	panic("Out of memory");
}

#else /* CONFIG_FLAT_NODE_MEM_MAP */

struct page_cgroup *lookup_page_cgroup(struct page *page)
{
	unsigned long pfn = page_to_pfn(page);
	struct mem_section *section = __pfn_to_section(pfn);
#ifdef CONFIG_DEBUG_VM
	/*
	 * The sanity checks the page allocator does upon freeing a
	 * page can reach here before the page_cgroup arrays are
	 * allocated when feeding a range of pages to the allocator
	 * for the first time during bootup or memory hotplug.
	 */
	if (!section->page_cgroup)
		return NULL;
#endif
	return section->page_cgroup + pfn;
}

static void *__meminit alloc_page_cgroup(size_t size, int nid)
{
	gfp_t flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN;
	void *addr = NULL;

	addr = alloc_pages_exact_nid(nid, size, flags);
	if (addr) {
		kmemleak_alloc(addr, size, 1, flags);
		return addr;
	}

	if (node_state(nid, N_HIGH_MEMORY))
		addr = vzalloc_node(size, nid);
	else
		addr = vzalloc(size);

	return addr;
}

static int __meminit init_section_page_cgroup(unsigned long pfn, int nid)
{
	struct mem_section *section;
	struct page_cgroup *base;
	unsigned long table_size;

	section = __pfn_to_section(pfn);

	if (section->page_cgroup)
		return 0;

	table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION;
	base = alloc_page_cgroup(table_size, nid);

	/*
	 * The value stored in section->page_cgroup is (base - pfn)
	 * and it does not point to the memory block allocated above,
	 * causing kmemleak false positives.
	 */
	kmemleak_not_leak(base);

	if (!base) {
		printk(KERN_ERR "page cgroup allocation failure\n");
		return -ENOMEM;
	}

	/*
	 * The passed "pfn" may not be aligned to SECTION.  For the calculation
	 * we need to apply a mask.
	 */
	pfn &= PAGE_SECTION_MASK;
	section->page_cgroup = base - pfn;
	total_usage += table_size;
	return 0;
}
#ifdef CONFIG_MEMORY_HOTPLUG
static void free_page_cgroup(void *addr)
{
	if (is_vmalloc_addr(addr)) {
		vfree(addr);
	} else {
		struct page *page = virt_to_page(addr);
		size_t table_size =
			sizeof(struct page_cgroup) * PAGES_PER_SECTION;

		BUG_ON(PageReserved(page));
		kmemleak_free(addr);
		free_pages_exact(addr, table_size);
	}
}

static void __free_page_cgroup(unsigned long pfn)
{
	struct mem_section *ms;
	struct page_cgroup *base;

	ms = __pfn_to_section(pfn);
	if (!ms || !ms->page_cgroup)
		return;
	base = ms->page_cgroup + pfn;
	free_page_cgroup(base);
	ms->page_cgroup = NULL;
}

static int __meminit online_page_cgroup(unsigned long start_pfn,
				unsigned long nr_pages,
				int nid)
{
	unsigned long start, end, pfn;
	int fail = 0;

	start = SECTION_ALIGN_DOWN(start_pfn);
	end = SECTION_ALIGN_UP(start_pfn + nr_pages);

	if (nid == -1) {
		/*
		 * In this case, "nid" already exists and contains valid memory.
		 * "start_pfn" passed to us is a pfn which is an arg for
		 * online__pages(), and start_pfn should exist.
		 */
		nid = pfn_to_nid(start_pfn);
		VM_BUG_ON(!node_state(nid, N_ONLINE));
	}

	for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION) {
		if (!pfn_present(pfn))
			continue;
		fail = init_section_page_cgroup(pfn, nid);
	}
	if (!fail)
		return 0;

	/* rollback */
	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
		__free_page_cgroup(pfn);

	return -ENOMEM;
}

static int __meminit offline_page_cgroup(unsigned long start_pfn,
				unsigned long nr_pages, int nid)
{
	unsigned long start, end, pfn;

	start = SECTION_ALIGN_DOWN(start_pfn);
	end = SECTION_ALIGN_UP(start_pfn + nr_pages);

	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
		__free_page_cgroup(pfn);
	return 0;

}

static int __meminit page_cgroup_callback(struct notifier_block *self,
			       unsigned long action, void *arg)
{
	struct memory_notify *mn = arg;
	int ret = 0;
	switch (action) {
	case MEM_GOING_ONLINE:
		ret = online_page_cgroup(mn->start_pfn,
				   mn->nr_pages, mn->status_change_nid);
		break;
	case MEM_OFFLINE:
		offline_page_cgroup(mn->start_pfn,
				mn->nr_pages, mn->status_change_nid);
		break;
	case MEM_CANCEL_ONLINE:
		offline_page_cgroup(mn->start_pfn,
				mn->nr_pages, mn->status_change_nid);
		break;
	case MEM_GOING_OFFLINE:
		break;
	case MEM_ONLINE:
	case MEM_CANCEL_OFFLINE:
		break;
	}

	return notifier_from_errno(ret);
}

#endif

void __init page_cgroup_init(void)
{
	unsigned long pfn;
	int nid;

	if (mem_cgroup_disabled())
		return;

	for_each_node_state(nid, N_MEMORY) {
		unsigned long start_pfn, end_pfn;

		start_pfn = node_start_pfn(nid);
		end_pfn = node_end_pfn(nid);
		/*
		 * start_pfn and end_pfn may not be aligned to SECTION and the
		 * page->flags of out of node pages are not initialized.  So we
		 * scan [start_pfn, the biggest section's pfn < end_pfn) here.
		 */
		for (pfn = start_pfn;
		     pfn < end_pfn;
                     pfn = ALIGN(pfn + 1, PAGES_PER_SECTION)) {

			if (!pfn_valid(pfn))
				continue;
			/*
			 * Nodes's pfns can be overlapping.
			 * We know some arch can have a nodes layout such as
			 * -------------pfn-------------->
			 * N0 | N1 | N2 | N0 | N1 | N2|....
			 */
			if (pfn_to_nid(pfn) != nid)
				continue;
			if (init_section_page_cgroup(pfn, nid))
				goto oom;
		}
	}
	hotplug_memory_notifier(page_cgroup_callback, 0);
	printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
	printk(KERN_INFO "please try 'cgroup_disable=memory' option if you "
			 "don't want memory cgroups\n");
	return;
oom:
	printk(KERN_CRIT "try 'cgroup_disable=memory' boot option\n");
	panic("Out of memory");
}

void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat)
{
	return;
}

#endif


#ifdef CONFIG_MEMCG_SWAP

static DEFINE_MUTEX(swap_cgroup_mutex);
struct swap_cgroup_ctrl {
	struct page **map;
	unsigned long length;
	spinlock_t	lock;
};

static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];

struct swap_cgroup {
	unsigned short		id;
};
#define SC_PER_PAGE	(PAGE_SIZE/sizeof(struct swap_cgroup))

/*
 * SwapCgroup implements "lookup" and "exchange" operations.
 * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
 * against SwapCache. At swap_free(), this is accessed directly from swap.
 *
 * This means,
 *  - we have no race in "exchange" when we're accessed via SwapCache because
 *    SwapCache(and its swp_entry) is under lock.
 *  - When called via swap_free(), there is no user of this entry and no race.
 * Then, we don't need lock around "exchange".
 *
 * TODO: we can push these buffers out to HIGHMEM.
 */

/*
 * allocate buffer for swap_cgroup.
 */
static int swap_cgroup_prepare(int type)
{
	struct page *page;
	struct swap_cgroup_ctrl *ctrl;
	unsigned long idx, max;

	ctrl = &swap_cgroup_ctrl[type];

	for (idx = 0; idx < ctrl->length; idx++) {
		page = alloc_page(GFP_KERNEL | __GFP_ZERO);
		if (!page)
			goto not_enough_page;
		ctrl->map[idx] = page;
	}
	return 0;
not_enough_page:
	max = idx;
	for (idx = 0; idx < max; idx++)
		__free_page(ctrl->map[idx]);

	return -ENOMEM;
}

static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent,
					struct swap_cgroup_ctrl **ctrlp)
{
	pgoff_t offset = swp_offset(ent);
	struct swap_cgroup_ctrl *ctrl;
	struct page *mappage;
	struct swap_cgroup *sc;

	ctrl = &swap_cgroup_ctrl[swp_type(ent)];
	if (ctrlp)
		*ctrlp = ctrl;

	mappage = ctrl->map[offset / SC_PER_PAGE];
	sc = page_address(mappage);
	return sc + offset % SC_PER_PAGE;
}

/**
 * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
 * @ent: swap entry to be cmpxchged
 * @old: old id
 * @new: new id
 *
 * Returns old id at success, 0 at failure.
 * (There is no mem_cgroup using 0 as its id)
 */
unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
					unsigned short old, unsigned short new)
{
	struct swap_cgroup_ctrl *ctrl;
	struct swap_cgroup *sc;
	unsigned long flags;
	unsigned short retval;

	sc = lookup_swap_cgroup(ent, &ctrl);

	spin_lock_irqsave(&ctrl->lock, flags);
	retval = sc->id;
	if (retval == old)
		sc->id = new;
	else
		retval = 0;
	spin_unlock_irqrestore(&ctrl->lock, flags);
	return retval;
}

/**
 * swap_cgroup_record - record mem_cgroup for this swp_entry.
 * @ent: swap entry to be recorded into
 * @id: mem_cgroup to be recorded
 *
 * Returns old value at success, 0 at failure.
 * (Of course, old value can be 0.)
 */
unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id)
{
	struct swap_cgroup_ctrl *ctrl;
	struct swap_cgroup *sc;
	unsigned short old;
	unsigned long flags;

	sc = lookup_swap_cgroup(ent, &ctrl);

	spin_lock_irqsave(&ctrl->lock, flags);
	old = sc->id;
	sc->id = id;
	spin_unlock_irqrestore(&ctrl->lock, flags);

	return old;
}

/**
 * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry
 * @ent: swap entry to be looked up.
 *
 * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
 */
unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
{
	return lookup_swap_cgroup(ent, NULL)->id;
}

int swap_cgroup_swapon(int type, unsigned long max_pages)
{
	void *array;
	unsigned long array_size;
	unsigned long length;
	struct swap_cgroup_ctrl *ctrl;

	if (!do_swap_account)
		return 0;

	length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
	array_size = length * sizeof(void *);

	array = vzalloc(array_size);
	if (!array)
		goto nomem;

	ctrl = &swap_cgroup_ctrl[type];
	mutex_lock(&swap_cgroup_mutex);
	ctrl->length = length;
	ctrl->map = array;
	spin_lock_init(&ctrl->lock);
	if (swap_cgroup_prepare(type)) {
		/* memory shortage */
		ctrl->map = NULL;
		ctrl->length = 0;
		mutex_unlock(&swap_cgroup_mutex);
		vfree(array);
		goto nomem;
	}
	mutex_unlock(&swap_cgroup_mutex);

	return 0;
nomem:
	printk(KERN_INFO "couldn't allocate enough memory for swap_cgroup.\n");
	printk(KERN_INFO
		"swap_cgroup can be disabled by swapaccount=0 boot option\n");
	return -ENOMEM;
}

void swap_cgroup_swapoff(int type)
{
	struct page **map;
	unsigned long i, length;
	struct swap_cgroup_ctrl *ctrl;

	if (!do_swap_account)
		return;

	mutex_lock(&swap_cgroup_mutex);
	ctrl = &swap_cgroup_ctrl[type];
	map = ctrl->map;
	length = ctrl->length;
	ctrl->map = NULL;
	ctrl->length = 0;
	mutex_unlock(&swap_cgroup_mutex);

	if (map) {
		for (i = 0; i < length; i++) {
			struct page *page = map[i];
			if (page)
				__free_page(page);
		}
		vfree(map);
	}
}

#endif
Commit	Line	Data
52d4b9ac KH	1	#include <linux/mm.h>
	2	#include <linux/mmzone.h>
	3	#include <linux/bootmem.h>
	4	#include <linux/bit_spinlock.h>
	5	#include <linux/page_cgroup.h>
	6	#include <linux/hash.h>
94b6da5a	7	#include <linux/slab.h>
52d4b9ac	8	#include <linux/memory.h>
4c821042	9	#include <linux/vmalloc.h>
94b6da5a	10	#include <linux/cgroup.h>
27a7faa0	11	#include <linux/swapops.h>
7952f988	12	#include <linux/kmemleak.h>
52d4b9ac	13
52d4b9ac KH	14	static unsigned long total_usage;
	15
	16	#if !defined(CONFIG_SPARSEMEM)
	17
	18
31168481	19	void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat)
52d4b9ac KH	20	{
	21	pgdat->node_page_cgroup = NULL;
	22	}
	23
	24	struct page_cgroup lookup_page_cgroup(struct page page)
	25	{
	26	unsigned long pfn = page_to_pfn(page);
	27	unsigned long offset;
	28	struct page_cgroup *base;
	29
	30	base = NODE_DATA(page_to_nid(page))->node_page_cgroup;
00c54c0b JW	31	#ifdef CONFIG_DEBUG_VM
	32	/*
	33	* The sanity checks the page allocator does upon freeing a
	34	* page can reach here before the page_cgroup arrays are
	35	* allocated when feeding a range of pages to the allocator
	36	* for the first time during bootup or memory hotplug.
	37	*/
52d4b9ac KH	38	if (unlikely(!base))
52d4b9ac KH	39	return NULL;
00c54c0b	40	#endif
52d4b9ac KH	41	offset = pfn - NODE_DATA(page_to_nid(page))->node_start_pfn;
	42	return base + offset;
	43	}
	44
	45	static int __init alloc_node_page_cgroup(int nid)
	46	{
6b208e3f	47	struct page_cgroup *base;
52d4b9ac	48	unsigned long table_size;
6b208e3f	49	unsigned long nr_pages;
52d4b9ac	50
52d4b9ac	51	nr_pages = NODE_DATA(nid)->node_spanned_pages;
653d22c0 KH	52	if (!nr_pages)
	53	return 0;
	54
52d4b9ac	55	table_size = sizeof(struct page_cgroup) * nr_pages;
ca371c0d	56
0d036e9e GS	57	base = memblock_virt_alloc_try_nid_nopanic(
	58	table_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS),
	59	BOOTMEM_ALLOC_ACCESSIBLE, nid);
ca371c0d	60	if (!base)
52d4b9ac	61	return -ENOMEM;
52d4b9ac KH	62	NODE_DATA(nid)->node_page_cgroup = base;
	63	total_usage += table_size;
	64	return 0;
	65	}
	66
ca371c0d	67	void __init page_cgroup_init_flatmem(void)
52d4b9ac KH	68	{
	69
	70	int nid, fail;
	71
f8d66542	72	if (mem_cgroup_disabled())
94b6da5a KH	73	return;
94b6da5a KH	74
52d4b9ac KH	75	for_each_online_node(nid) {
	76	fail = alloc_node_page_cgroup(nid);
	77	if (fail)
	78	goto fail;
	79	}
	80	printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
8ca739e3 RD	81	printk(KERN_INFO "please try 'cgroup_disable=memory' option if you"
8ca739e3 RD	82	" don't want memory cgroups\n");
52d4b9ac KH	83	return;
52d4b9ac KH	84	fail:
8ca739e3 RD	85	printk(KERN_CRIT "allocation of page_cgroup failed.\n");
8ca739e3 RD	86	printk(KERN_CRIT "please try 'cgroup_disable=memory' boot option\n");
52d4b9ac KH	87	panic("Out of memory");
	88	}
	89
	90	#else /* CONFIG_FLAT_NODE_MEM_MAP */
	91
	92	struct page_cgroup lookup_page_cgroup(struct page page)
	93	{
	94	unsigned long pfn = page_to_pfn(page);
	95	struct mem_section *section = __pfn_to_section(pfn);
00c54c0b JW	96	#ifdef CONFIG_DEBUG_VM
	97	/*
	98	* The sanity checks the page allocator does upon freeing a
	99	* page can reach here before the page_cgroup arrays are
	100	* allocated when feeding a range of pages to the allocator
	101	* for the first time during bootup or memory hotplug.
	102	*/
d69b042f BS	103	if (!section->page_cgroup)
d69b042f BS	104	return NULL;
00c54c0b	105	#endif
52d4b9ac KH	106	return section->page_cgroup + pfn;
	107	}
	108
268433b8	109	static void *__meminit alloc_page_cgroup(size_t size, int nid)
dde79e00	110	{
6b208e3f	111	gfp_t flags = GFP_KERNEL \| __GFP_ZERO \| __GFP_NOWARN;
dde79e00 MH	112	void *addr = NULL;
dde79e00 MH	113
ff7ee93f SR	114	addr = alloc_pages_exact_nid(nid, size, flags);
	115	if (addr) {
	116	kmemleak_alloc(addr, size, 1, flags);
dde79e00	117	return addr;
ff7ee93f	118	}
dde79e00 MH	119
dde79e00 MH	120	if (node_state(nid, N_HIGH_MEMORY))
6b208e3f	121	addr = vzalloc_node(size, nid);
dde79e00	122	else
6b208e3f	123	addr = vzalloc(size);
dde79e00 MH	124
	125	return addr;
	126	}
	127
37573e8c	128	static int __meminit init_section_page_cgroup(unsigned long pfn, int nid)
52d4b9ac	129	{
6b3ae58e	130	struct mem_section *section;
6b208e3f	131	struct page_cgroup *base;
52d4b9ac	132	unsigned long table_size;
52d4b9ac	133
6b208e3f	134	section = __pfn_to_section(pfn);
6b3ae58e JW	135
	136	if (section->page_cgroup)
	137	return 0;
	138
6b3ae58e	139	table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION;
dde79e00 MH	140	base = alloc_page_cgroup(table_size, nid);
dde79e00 MH	141
6b3ae58e JW	142	/*
	143	* The value stored in section->page_cgroup is (base - pfn)
	144	* and it does not point to the memory block allocated above,
	145	* causing kmemleak false positives.
	146	*/
	147	kmemleak_not_leak(base);
52d4b9ac KH	148
	149	if (!base) {
	150	printk(KERN_ERR "page cgroup allocation failure\n");
	151	return -ENOMEM;
	152	}
	153
37573e8c KH	154	/*
	155	* The passed "pfn" may not be aligned to SECTION. For the calculation
	156	* we need to apply a mask.
	157	*/
	158	pfn &= PAGE_SECTION_MASK;
52d4b9ac KH	159	section->page_cgroup = base - pfn;
	160	total_usage += table_size;
	161	return 0;
	162	}
	163	#ifdef CONFIG_MEMORY_HOTPLUG
0efc8eb9 BL	164	static void free_page_cgroup(void *addr)
	165	{
	166	if (is_vmalloc_addr(addr)) {
	167	vfree(addr);
	168	} else {
	169	struct page *page = virt_to_page(addr);
	170	size_t table_size =
	171	sizeof(struct page_cgroup) * PAGES_PER_SECTION;
	172
	173	BUG_ON(PageReserved(page));
401507d6	174	kmemleak_free(addr);
0efc8eb9 BL	175	free_pages_exact(addr, table_size);
	176	}
	177	}
	178
d20199e1	179	static void __free_page_cgroup(unsigned long pfn)
52d4b9ac KH	180	{
	181	struct mem_section *ms;
	182	struct page_cgroup *base;
	183
	184	ms = __pfn_to_section(pfn);
	185	if (!ms \|\| !ms->page_cgroup)
	186	return;
	187	base = ms->page_cgroup + pfn;
dde79e00 MH	188	free_page_cgroup(base);
dde79e00 MH	189	ms->page_cgroup = NULL;
52d4b9ac KH	190	}
52d4b9ac KH	191
d20199e1 RK	192	static int __meminit online_page_cgroup(unsigned long start_pfn,
	193	unsigned long nr_pages,
	194	int nid)
52d4b9ac KH	195	{
	196	unsigned long start, end, pfn;
	197	int fail = 0;
	198
1bb36fbd DK	199	start = SECTION_ALIGN_DOWN(start_pfn);
1bb36fbd DK	200	end = SECTION_ALIGN_UP(start_pfn + nr_pages);
52d4b9ac	201
37573e8c KH	202	if (nid == -1) {
	203	/*
	204	* In this case, "nid" already exists and contains valid memory.
	205	* "start_pfn" passed to us is a pfn which is an arg for
	206	* online__pages(), and start_pfn should exist.
	207	*/
	208	nid = pfn_to_nid(start_pfn);
	209	VM_BUG_ON(!node_state(nid, N_ONLINE));
	210	}
	211
52d4b9ac KH	212	for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION) {
	213	if (!pfn_present(pfn))
	214	continue;
37573e8c	215	fail = init_section_page_cgroup(pfn, nid);
52d4b9ac KH	216	}
	217	if (!fail)
	218	return 0;
	219
	220	/* rollback */
	221	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
	222	__free_page_cgroup(pfn);
	223
	224	return -ENOMEM;
	225	}
	226
d20199e1 RK	227	static int __meminit offline_page_cgroup(unsigned long start_pfn,
d20199e1 RK	228	unsigned long nr_pages, int nid)
52d4b9ac KH	229	{
	230	unsigned long start, end, pfn;
	231
1bb36fbd DK	232	start = SECTION_ALIGN_DOWN(start_pfn);
1bb36fbd DK	233	end = SECTION_ALIGN_UP(start_pfn + nr_pages);
52d4b9ac KH	234
	235	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
	236	__free_page_cgroup(pfn);
	237	return 0;
	238
	239	}
	240
31168481	241	static int __meminit page_cgroup_callback(struct notifier_block *self,
52d4b9ac KH	242	unsigned long action, void *arg)
	243	{
	244	struct memory_notify *mn = arg;
	245	int ret = 0;
	246	switch (action) {
	247	case MEM_GOING_ONLINE:
	248	ret = online_page_cgroup(mn->start_pfn,
	249	mn->nr_pages, mn->status_change_nid);
	250	break;
52d4b9ac KH	251	case MEM_OFFLINE:
	252	offline_page_cgroup(mn->start_pfn,
	253	mn->nr_pages, mn->status_change_nid);
	254	break;
dc19f9db	255	case MEM_CANCEL_ONLINE:
7c72eb32 WC	256	offline_page_cgroup(mn->start_pfn,
	257	mn->nr_pages, mn->status_change_nid);
	258	break;
52d4b9ac KH	259	case MEM_GOING_OFFLINE:
	260	break;
	261	case MEM_ONLINE:
	262	case MEM_CANCEL_OFFLINE:
	263	break;
	264	}
dc19f9db	265
5fda1bd5	266	return notifier_from_errno(ret);
52d4b9ac KH	267	}
	268
	269	#endif
	270
	271	void __init page_cgroup_init(void)
	272	{
	273	unsigned long pfn;
37573e8c	274	int nid;
52d4b9ac	275
f8d66542	276	if (mem_cgroup_disabled())
94b6da5a KH	277	return;
94b6da5a KH	278
31aaea4a	279	for_each_node_state(nid, N_MEMORY) {
37573e8c KH	280	unsigned long start_pfn, end_pfn;
	281
	282	start_pfn = node_start_pfn(nid);
	283	end_pfn = node_end_pfn(nid);
	284	/*
	285	* start_pfn and end_pfn may not be aligned to SECTION and the
	286	* page->flags of out of node pages are not initialized. So we
	287	* scan [start_pfn, the biggest section's pfn < end_pfn) here.
	288	*/
	289	for (pfn = start_pfn;
	290	pfn < end_pfn;
	291	pfn = ALIGN(pfn + 1, PAGES_PER_SECTION)) {
	292
	293	if (!pfn_valid(pfn))
	294	continue;
	295	/*
	296	* Nodes's pfns can be overlapping.
	297	* We know some arch can have a nodes layout such as
	298	* -------------pfn-------------->
	299	* N0 \| N1 \| N2 \| N0 \| N1 \| N2\|....
	300	*/
	301	if (pfn_to_nid(pfn) != nid)
	302	continue;
	303	if (init_section_page_cgroup(pfn, nid))
	304	goto oom;
	305	}
52d4b9ac	306	}
37573e8c	307	hotplug_memory_notifier(page_cgroup_callback, 0);
52d4b9ac	308	printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
37573e8c KH	309	printk(KERN_INFO "please try 'cgroup_disable=memory' option if you "
	310	"don't want memory cgroups\n");
	311	return;
	312	oom:
	313	printk(KERN_CRIT "try 'cgroup_disable=memory' boot option\n");
	314	panic("Out of memory");
52d4b9ac KH	315	}
52d4b9ac KH	316
31168481	317	void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat)
52d4b9ac KH	318	{
	319	return;
	320	}
	321
	322	#endif
27a7faa0 KH	323
27a7faa0 KH	324
c255a458	325	#ifdef CONFIG_MEMCG_SWAP
27a7faa0 KH	326
	327	static DEFINE_MUTEX(swap_cgroup_mutex);
	328	struct swap_cgroup_ctrl {
	329	struct page **map;
	330	unsigned long length;
e9e58a4e	331	spinlock_t lock;
27a7faa0 KH	332	};
27a7faa0 KH	333
61600f57	334	static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
27a7faa0	335
27a7faa0	336	struct swap_cgroup {
a3b2d692	337	unsigned short id;
27a7faa0 KH	338	};
27a7faa0 KH	339	#define SC_PER_PAGE (PAGE_SIZE/sizeof(struct swap_cgroup))
27a7faa0 KH	340
	341	/*
	342	* SwapCgroup implements "lookup" and "exchange" operations.
	343	* In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
	344	* against SwapCache. At swap_free(), this is accessed directly from swap.
	345	*
	346	* This means,
	347	* - we have no race in "exchange" when we're accessed via SwapCache because
	348	* SwapCache(and its swp_entry) is under lock.
	349	* - When called via swap_free(), there is no user of this entry and no race.
	350	* Then, we don't need lock around "exchange".
	351	*
	352	* TODO: we can push these buffers out to HIGHMEM.
	353	*/
	354
	355	/*
	356	* allocate buffer for swap_cgroup.
	357	*/
	358	static int swap_cgroup_prepare(int type)
	359	{
	360	struct page *page;
	361	struct swap_cgroup_ctrl *ctrl;
	362	unsigned long idx, max;
	363
27a7faa0 KH	364	ctrl = &swap_cgroup_ctrl[type];
	365
	366	for (idx = 0; idx < ctrl->length; idx++) {
	367	page = alloc_page(GFP_KERNEL \| __GFP_ZERO);
	368	if (!page)
	369	goto not_enough_page;
	370	ctrl->map[idx] = page;
	371	}
	372	return 0;
	373	not_enough_page:
	374	max = idx;
	375	for (idx = 0; idx < max; idx++)
	376	__free_page(ctrl->map[idx]);
	377
	378	return -ENOMEM;
	379	}
	380
9fb4b7cc BL	381	static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent,
	382	struct swap_cgroup_ctrl **ctrlp)
	383	{
	384	pgoff_t offset = swp_offset(ent);
	385	struct swap_cgroup_ctrl *ctrl;
	386	struct page *mappage;
c09ff089	387	struct swap_cgroup *sc;
9fb4b7cc BL	388
	389	ctrl = &swap_cgroup_ctrl[swp_type(ent)];
	390	if (ctrlp)
	391	*ctrlp = ctrl;
	392
	393	mappage = ctrl->map[offset / SC_PER_PAGE];
c09ff089 HD	394	sc = page_address(mappage);
c09ff089 HD	395	return sc + offset % SC_PER_PAGE;
9fb4b7cc BL	396	}
9fb4b7cc BL	397
02491447 DN	398	/**
02491447 DN	399	* swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
dad7557e	400	* @ent: swap entry to be cmpxchged
02491447 DN	401	* @old: old id
	402	* @new: new id
	403	*
	404	* Returns old id at success, 0 at failure.
25985edc	405	* (There is no mem_cgroup using 0 as its id)
02491447 DN	406	*/
	407	unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
	408	unsigned short old, unsigned short new)
	409	{
02491447	410	struct swap_cgroup_ctrl *ctrl;
02491447	411	struct swap_cgroup *sc;
e9e58a4e KH	412	unsigned long flags;
e9e58a4e KH	413	unsigned short retval;
02491447	414
9fb4b7cc	415	sc = lookup_swap_cgroup(ent, &ctrl);
02491447	416
e9e58a4e KH	417	spin_lock_irqsave(&ctrl->lock, flags);
	418	retval = sc->id;
	419	if (retval == old)
	420	sc->id = new;
02491447	421	else
e9e58a4e KH	422	retval = 0;
	423	spin_unlock_irqrestore(&ctrl->lock, flags);
	424	return retval;
02491447 DN	425	}
02491447 DN	426
27a7faa0 KH	427	/**
	428	* swap_cgroup_record - record mem_cgroup for this swp_entry.
	429	* @ent: swap entry to be recorded into
dad7557e	430	* @id: mem_cgroup to be recorded
27a7faa0	431	*
a3b2d692 KH	432	* Returns old value at success, 0 at failure.
a3b2d692 KH	433	* (Of course, old value can be 0.)
27a7faa0	434	*/
a3b2d692	435	unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id)
27a7faa0	436	{
27a7faa0	437	struct swap_cgroup_ctrl *ctrl;
27a7faa0	438	struct swap_cgroup *sc;
a3b2d692	439	unsigned short old;
e9e58a4e	440	unsigned long flags;
27a7faa0	441
9fb4b7cc	442	sc = lookup_swap_cgroup(ent, &ctrl);
27a7faa0	443
e9e58a4e KH	444	spin_lock_irqsave(&ctrl->lock, flags);
	445	old = sc->id;
	446	sc->id = id;
	447	spin_unlock_irqrestore(&ctrl->lock, flags);
27a7faa0 KH	448
	449	return old;
	450	}
	451
	452	/**
9fb4b7cc	453	* lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry
27a7faa0 KH	454	* @ent: swap entry to be looked up.
27a7faa0 KH	455	*
b3ff8a2f	456	* Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
27a7faa0	457	*/
9fb4b7cc	458	unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
27a7faa0	459	{
9fb4b7cc	460	return lookup_swap_cgroup(ent, NULL)->id;
27a7faa0 KH	461	}
	462
	463	int swap_cgroup_swapon(int type, unsigned long max_pages)
	464	{
	465	void *array;
	466	unsigned long array_size;
	467	unsigned long length;
	468	struct swap_cgroup_ctrl *ctrl;
	469
	470	if (!do_swap_account)
	471	return 0;
	472
33278f7f	473	length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
27a7faa0 KH	474	array_size = length * sizeof(void *);
27a7faa0 KH	475
8c1fec1b	476	array = vzalloc(array_size);
27a7faa0 KH	477	if (!array)
	478	goto nomem;
	479
27a7faa0 KH	480	ctrl = &swap_cgroup_ctrl[type];
	481	mutex_lock(&swap_cgroup_mutex);
	482	ctrl->length = length;
	483	ctrl->map = array;
e9e58a4e	484	spin_lock_init(&ctrl->lock);
27a7faa0 KH	485	if (swap_cgroup_prepare(type)) {
	486	/* memory shortage */
	487	ctrl->map = NULL;
	488	ctrl->length = 0;
27a7faa0	489	mutex_unlock(&swap_cgroup_mutex);
6a5b18d2	490	vfree(array);
27a7faa0 KH	491	goto nomem;
	492	}
	493	mutex_unlock(&swap_cgroup_mutex);
	494
27a7faa0 KH	495	return 0;
	496	nomem:
	497	printk(KERN_INFO "couldn't allocate enough memory for swap_cgroup.\n");
	498	printk(KERN_INFO
00a66d29	499	"swap_cgroup can be disabled by swapaccount=0 boot option\n");
27a7faa0 KH	500	return -ENOMEM;
	501	}
	502
	503	void swap_cgroup_swapoff(int type)
	504	{
6a5b18d2 NK	505	struct page **map;
6a5b18d2 NK	506	unsigned long i, length;
27a7faa0 KH	507	struct swap_cgroup_ctrl *ctrl;
	508
	509	if (!do_swap_account)
	510	return;
	511
	512	mutex_lock(&swap_cgroup_mutex);
	513	ctrl = &swap_cgroup_ctrl[type];
6a5b18d2 NK	514	map = ctrl->map;
	515	length = ctrl->length;
	516	ctrl->map = NULL;
	517	ctrl->length = 0;
	518	mutex_unlock(&swap_cgroup_mutex);
	519
	520	if (map) {
	521	for (i = 0; i < length; i++) {
	522	struct page *page = map[i];
27a7faa0 KH	523	if (page)
	524	__free_page(page);
	525	}
6a5b18d2	526	vfree(map);
27a7faa0	527	}
27a7faa0 KH	528	}
	529
	530	#endif