[mirror_ubuntu-jammy-kernel.git] / mm / page_cgroup.c

#include <linux/mm.h>
#include <linux/mmzone.h>
#include <linux/bootmem.h>
#include <linux/bit_spinlock.h>
#include <linux/page_cgroup.h>
#include <linux/hash.h>
#include <linux/slab.h>
#include <linux/memory.h>
#include <linux/vmalloc.h>
#include <linux/cgroup.h>
#include <linux/swapops.h>
#include <linux/kmemleak.h>

static void __meminit
__init_page_cgroup(struct page_cgroup *pc, unsigned long pfn)
{
	pc->flags = 0;
	pc->mem_cgroup = NULL;
	pc->page = pfn_to_page(pfn);
	INIT_LIST_HEAD(&pc->lru);
}
static unsigned long total_usage;

#if !defined(CONFIG_SPARSEMEM)


void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat)
{
	pgdat->node_page_cgroup = NULL;
}

struct page_cgroup *lookup_page_cgroup(struct page *page)
{
	unsigned long pfn = page_to_pfn(page);
	unsigned long offset;
	struct page_cgroup *base;

	base = NODE_DATA(page_to_nid(page))->node_page_cgroup;
	if (unlikely(!base))
		return NULL;

	offset = pfn - NODE_DATA(page_to_nid(page))->node_start_pfn;
	return base + offset;
}

static int __init alloc_node_page_cgroup(int nid)
{
	struct page_cgroup *base, *pc;
	unsigned long table_size;
	unsigned long start_pfn, nr_pages, index;

	start_pfn = NODE_DATA(nid)->node_start_pfn;
	nr_pages = NODE_DATA(nid)->node_spanned_pages;

	if (!nr_pages)
		return 0;

	table_size = sizeof(struct page_cgroup) * nr_pages;

	base = __alloc_bootmem_node_nopanic(NODE_DATA(nid),
			table_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
	if (!base)
		return -ENOMEM;
	for (index = 0; index < nr_pages; index++) {
		pc = base + index;
		__init_page_cgroup(pc, start_pfn + index);
	}
	NODE_DATA(nid)->node_page_cgroup = base;
	total_usage += table_size;
	return 0;
}

void __init page_cgroup_init_flatmem(void)
{

	int nid, fail;

	if (mem_cgroup_disabled())
		return;

	for_each_online_node(nid)  {
		fail = alloc_node_page_cgroup(nid);
		if (fail)
			goto fail;
	}
	printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
	printk(KERN_INFO "please try 'cgroup_disable=memory' option if you"
	" don't want memory cgroups\n");
	return;
fail:
	printk(KERN_CRIT "allocation of page_cgroup failed.\n");
	printk(KERN_CRIT "please try 'cgroup_disable=memory' boot option\n");
	panic("Out of memory");
}

#else /* CONFIG_FLAT_NODE_MEM_MAP */

struct page_cgroup *lookup_page_cgroup(struct page *page)
{
	unsigned long pfn = page_to_pfn(page);
	struct mem_section *section = __pfn_to_section(pfn);

	if (!section->page_cgroup)
		return NULL;
	return section->page_cgroup + pfn;
}

/* __alloc_bootmem...() is protected by !slab_available() */
static int __init_refok init_section_page_cgroup(unsigned long pfn)
{
	struct mem_section *section = __pfn_to_section(pfn);
	struct page_cgroup *base, *pc;
	unsigned long table_size;
	int nid, index;

	if (!section->page_cgroup) {
		nid = page_to_nid(pfn_to_page(pfn));
		table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION;
		VM_BUG_ON(!slab_is_available());
		if (node_state(nid, N_HIGH_MEMORY)) {
			base = kmalloc_node(table_size,
				GFP_KERNEL | __GFP_NOWARN, nid);
			if (!base)
				base = vmalloc_node(table_size, nid);
		} else {
			base = kmalloc(table_size, GFP_KERNEL | __GFP_NOWARN);
			if (!base)
				base = vmalloc(table_size);
		}
		/*
		 * The value stored in section->page_cgroup is (base - pfn)
		 * and it does not point to the memory block allocated above,
		 * causing kmemleak false positives.
		 */
		kmemleak_not_leak(base);
	} else {
		/*
 		 * We don't have to allocate page_cgroup again, but
		 * address of memmap may be changed. So, we have to initialize
		 * again.
		 */
		base = section->page_cgroup + pfn;
		table_size = 0;
		/* check address of memmap is changed or not. */
		if (base->page == pfn_to_page(pfn))
			return 0;
	}

	if (!base) {
		printk(KERN_ERR "page cgroup allocation failure\n");
		return -ENOMEM;
	}

	for (index = 0; index < PAGES_PER_SECTION; index++) {
		pc = base + index;
		__init_page_cgroup(pc, pfn + index);
	}

	section->page_cgroup = base - pfn;
	total_usage += table_size;
	return 0;
}
#ifdef CONFIG_MEMORY_HOTPLUG
void __free_page_cgroup(unsigned long pfn)
{
	struct mem_section *ms;
	struct page_cgroup *base;

	ms = __pfn_to_section(pfn);
	if (!ms || !ms->page_cgroup)
		return;
	base = ms->page_cgroup + pfn;
	if (is_vmalloc_addr(base)) {
		vfree(base);
		ms->page_cgroup = NULL;
	} else {
		struct page *page = virt_to_page(base);
		if (!PageReserved(page)) { /* Is bootmem ? */
			kfree(base);
			ms->page_cgroup = NULL;
		}
	}
}

int __meminit online_page_cgroup(unsigned long start_pfn,
			unsigned long nr_pages,
			int nid)
{
	unsigned long start, end, pfn;
	int fail = 0;

	start = start_pfn & ~(PAGES_PER_SECTION - 1);
	end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION);

	for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION) {
		if (!pfn_present(pfn))
			continue;
		fail = init_section_page_cgroup(pfn);
	}
	if (!fail)
		return 0;

	/* rollback */
	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
		__free_page_cgroup(pfn);

	return -ENOMEM;
}

int __meminit offline_page_cgroup(unsigned long start_pfn,
		unsigned long nr_pages, int nid)
{
	unsigned long start, end, pfn;

	start = start_pfn & ~(PAGES_PER_SECTION - 1);
	end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION);

	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
		__free_page_cgroup(pfn);
	return 0;

}

static int __meminit page_cgroup_callback(struct notifier_block *self,
			       unsigned long action, void *arg)
{
	struct memory_notify *mn = arg;
	int ret = 0;
	switch (action) {
	case MEM_GOING_ONLINE:
		ret = online_page_cgroup(mn->start_pfn,
				   mn->nr_pages, mn->status_change_nid);
		break;
	case MEM_OFFLINE:
		offline_page_cgroup(mn->start_pfn,
				mn->nr_pages, mn->status_change_nid);
		break;
	case MEM_CANCEL_ONLINE:
	case MEM_GOING_OFFLINE:
		break;
	case MEM_ONLINE:
	case MEM_CANCEL_OFFLINE:
		break;
	}

	if (ret)
		ret = notifier_from_errno(ret);
	else
		ret = NOTIFY_OK;

	return ret;
}

#endif

void __init page_cgroup_init(void)
{
	unsigned long pfn;
	int fail = 0;

	if (mem_cgroup_disabled())
		return;

	for (pfn = 0; !fail && pfn < max_pfn; pfn += PAGES_PER_SECTION) {
		if (!pfn_present(pfn))
			continue;
		fail = init_section_page_cgroup(pfn);
	}
	if (fail) {
		printk(KERN_CRIT "try 'cgroup_disable=memory' boot option\n");
		panic("Out of memory");
	} else {
		hotplug_memory_notifier(page_cgroup_callback, 0);
	}
	printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
	printk(KERN_INFO "please try 'cgroup_disable=memory' option if you don't"
	" want memory cgroups\n");
}

void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat)
{
	return;
}

#endif


#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP

static DEFINE_MUTEX(swap_cgroup_mutex);
struct swap_cgroup_ctrl {
	struct page **map;
	unsigned long length;
	spinlock_t	lock;
};

struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];

struct swap_cgroup {
	unsigned short		id;
};
#define SC_PER_PAGE	(PAGE_SIZE/sizeof(struct swap_cgroup))
#define SC_POS_MASK	(SC_PER_PAGE - 1)

/*
 * SwapCgroup implements "lookup" and "exchange" operations.
 * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
 * against SwapCache. At swap_free(), this is accessed directly from swap.
 *
 * This means,
 *  - we have no race in "exchange" when we're accessed via SwapCache because
 *    SwapCache(and its swp_entry) is under lock.
 *  - When called via swap_free(), there is no user of this entry and no race.
 * Then, we don't need lock around "exchange".
 *
 * TODO: we can push these buffers out to HIGHMEM.
 */

/*
 * allocate buffer for swap_cgroup.
 */
static int swap_cgroup_prepare(int type)
{
	struct page *page;
	struct swap_cgroup_ctrl *ctrl;
	unsigned long idx, max;

	ctrl = &swap_cgroup_ctrl[type];

	for (idx = 0; idx < ctrl->length; idx++) {
		page = alloc_page(GFP_KERNEL | __GFP_ZERO);
		if (!page)
			goto not_enough_page;
		ctrl->map[idx] = page;
	}
	return 0;
not_enough_page:
	max = idx;
	for (idx = 0; idx < max; idx++)
		__free_page(ctrl->map[idx]);

	return -ENOMEM;
}

/**
 * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
 * @end: swap entry to be cmpxchged
 * @old: old id
 * @new: new id
 *
 * Returns old id at success, 0 at failure.
 * (There is no mem_cgroup useing 0 as its id)
 */
unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
					unsigned short old, unsigned short new)
{
	int type = swp_type(ent);
	unsigned long offset = swp_offset(ent);
	unsigned long idx = offset / SC_PER_PAGE;
	unsigned long pos = offset & SC_POS_MASK;
	struct swap_cgroup_ctrl *ctrl;
	struct page *mappage;
	struct swap_cgroup *sc;
	unsigned long flags;
	unsigned short retval;

	ctrl = &swap_cgroup_ctrl[type];

	mappage = ctrl->map[idx];
	sc = page_address(mappage);
	sc += pos;
	spin_lock_irqsave(&ctrl->lock, flags);
	retval = sc->id;
	if (retval == old)
		sc->id = new;
	else
		retval = 0;
	spin_unlock_irqrestore(&ctrl->lock, flags);
	return retval;
}

/**
 * swap_cgroup_record - record mem_cgroup for this swp_entry.
 * @ent: swap entry to be recorded into
 * @mem: mem_cgroup to be recorded
 *
 * Returns old value at success, 0 at failure.
 * (Of course, old value can be 0.)
 */
unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id)
{
	int type = swp_type(ent);
	unsigned long offset = swp_offset(ent);
	unsigned long idx = offset / SC_PER_PAGE;
	unsigned long pos = offset & SC_POS_MASK;
	struct swap_cgroup_ctrl *ctrl;
	struct page *mappage;
	struct swap_cgroup *sc;
	unsigned short old;
	unsigned long flags;

	ctrl = &swap_cgroup_ctrl[type];

	mappage = ctrl->map[idx];
	sc = page_address(mappage);
	sc += pos;
	spin_lock_irqsave(&ctrl->lock, flags);
	old = sc->id;
	sc->id = id;
	spin_unlock_irqrestore(&ctrl->lock, flags);

	return old;
}

/**
 * lookup_swap_cgroup - lookup mem_cgroup tied to swap entry
 * @ent: swap entry to be looked up.
 *
 * Returns CSS ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
 */
unsigned short lookup_swap_cgroup(swp_entry_t ent)
{
	int type = swp_type(ent);
	unsigned long offset = swp_offset(ent);
	unsigned long idx = offset / SC_PER_PAGE;
	unsigned long pos = offset & SC_POS_MASK;
	struct swap_cgroup_ctrl *ctrl;
	struct page *mappage;
	struct swap_cgroup *sc;
	unsigned short ret;

	ctrl = &swap_cgroup_ctrl[type];
	mappage = ctrl->map[idx];
	sc = page_address(mappage);
	sc += pos;
	ret = sc->id;
	return ret;
}

int swap_cgroup_swapon(int type, unsigned long max_pages)
{
	void *array;
	unsigned long array_size;
	unsigned long length;
	struct swap_cgroup_ctrl *ctrl;

	if (!do_swap_account)
		return 0;

	length = ((max_pages/SC_PER_PAGE) + 1);
	array_size = length * sizeof(void *);

	array = vmalloc(array_size);
	if (!array)
		goto nomem;

	memset(array, 0, array_size);
	ctrl = &swap_cgroup_ctrl[type];
	mutex_lock(&swap_cgroup_mutex);
	ctrl->length = length;
	ctrl->map = array;
	spin_lock_init(&ctrl->lock);
	if (swap_cgroup_prepare(type)) {
		/* memory shortage */
		ctrl->map = NULL;
		ctrl->length = 0;
		vfree(array);
		mutex_unlock(&swap_cgroup_mutex);
		goto nomem;
	}
	mutex_unlock(&swap_cgroup_mutex);

	return 0;
nomem:
	printk(KERN_INFO "couldn't allocate enough memory for swap_cgroup.\n");
	printk(KERN_INFO
		"swap_cgroup can be disabled by noswapaccount boot option\n");
	return -ENOMEM;
}

void swap_cgroup_swapoff(int type)
{
	int i;
	struct swap_cgroup_ctrl *ctrl;

	if (!do_swap_account)
		return;

	mutex_lock(&swap_cgroup_mutex);
	ctrl = &swap_cgroup_ctrl[type];
	if (ctrl->map) {
		for (i = 0; i < ctrl->length; i++) {
			struct page *page = ctrl->map[i];
			if (page)
				__free_page(page);
		}
		vfree(ctrl->map);
		ctrl->map = NULL;
		ctrl->length = 0;
	}
	mutex_unlock(&swap_cgroup_mutex);
}

#endif
Commit	Line	Data
52d4b9ac KH	1	#include <linux/mm.h>
	2	#include <linux/mmzone.h>
	3	#include <linux/bootmem.h>
	4	#include <linux/bit_spinlock.h>
	5	#include <linux/page_cgroup.h>
	6	#include <linux/hash.h>
94b6da5a	7	#include <linux/slab.h>
52d4b9ac	8	#include <linux/memory.h>
4c821042	9	#include <linux/vmalloc.h>
94b6da5a	10	#include <linux/cgroup.h>
27a7faa0	11	#include <linux/swapops.h>
7952f988	12	#include <linux/kmemleak.h>
52d4b9ac KH	13
	14	static void __meminit
	15	__init_page_cgroup(struct page_cgroup *pc, unsigned long pfn)
	16	{
	17	pc->flags = 0;
	18	pc->mem_cgroup = NULL;
	19	pc->page = pfn_to_page(pfn);
08e552c6	20	INIT_LIST_HEAD(&pc->lru);
52d4b9ac KH	21	}
	22	static unsigned long total_usage;
	23
	24	#if !defined(CONFIG_SPARSEMEM)
	25
	26
31168481	27	void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat)
52d4b9ac KH	28	{
	29	pgdat->node_page_cgroup = NULL;
	30	}
	31
	32	struct page_cgroup lookup_page_cgroup(struct page page)
	33	{
	34	unsigned long pfn = page_to_pfn(page);
	35	unsigned long offset;
	36	struct page_cgroup *base;
	37
	38	base = NODE_DATA(page_to_nid(page))->node_page_cgroup;
	39	if (unlikely(!base))
	40	return NULL;
	41
	42	offset = pfn - NODE_DATA(page_to_nid(page))->node_start_pfn;
	43	return base + offset;
	44	}
	45
	46	static int __init alloc_node_page_cgroup(int nid)
	47	{
	48	struct page_cgroup base, pc;
	49	unsigned long table_size;
	50	unsigned long start_pfn, nr_pages, index;
	51
	52	start_pfn = NODE_DATA(nid)->node_start_pfn;
	53	nr_pages = NODE_DATA(nid)->node_spanned_pages;
	54
653d22c0 KH	55	if (!nr_pages)
	56	return 0;
	57
52d4b9ac	58	table_size = sizeof(struct page_cgroup) * nr_pages;
ca371c0d KH	59
	60	base = __alloc_bootmem_node_nopanic(NODE_DATA(nid),
	61	table_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
	62	if (!base)
52d4b9ac KH	63	return -ENOMEM;
	64	for (index = 0; index < nr_pages; index++) {
	65	pc = base + index;
	66	__init_page_cgroup(pc, start_pfn + index);
	67	}
	68	NODE_DATA(nid)->node_page_cgroup = base;
	69	total_usage += table_size;
	70	return 0;
	71	}
	72
ca371c0d	73	void __init page_cgroup_init_flatmem(void)
52d4b9ac KH	74	{
	75
	76	int nid, fail;
	77
f8d66542	78	if (mem_cgroup_disabled())
94b6da5a KH	79	return;
94b6da5a KH	80
52d4b9ac KH	81	for_each_online_node(nid) {
	82	fail = alloc_node_page_cgroup(nid);
	83	if (fail)
	84	goto fail;
	85	}
	86	printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
8ca739e3 RD	87	printk(KERN_INFO "please try 'cgroup_disable=memory' option if you"
8ca739e3 RD	88	" don't want memory cgroups\n");
52d4b9ac KH	89	return;
52d4b9ac KH	90	fail:
8ca739e3 RD	91	printk(KERN_CRIT "allocation of page_cgroup failed.\n");
8ca739e3 RD	92	printk(KERN_CRIT "please try 'cgroup_disable=memory' boot option\n");
52d4b9ac KH	93	panic("Out of memory");
	94	}
	95
	96	#else /* CONFIG_FLAT_NODE_MEM_MAP */
	97
	98	struct page_cgroup lookup_page_cgroup(struct page page)
	99	{
	100	unsigned long pfn = page_to_pfn(page);
	101	struct mem_section *section = __pfn_to_section(pfn);
	102
d69b042f BS	103	if (!section->page_cgroup)
d69b042f BS	104	return NULL;
52d4b9ac KH	105	return section->page_cgroup + pfn;
	106	}
	107
31168481	108	/* __alloc_bootmem...() is protected by !slab_available() */
feb16694	109	static int __init_refok init_section_page_cgroup(unsigned long pfn)
52d4b9ac	110	{
0753b0ef	111	struct mem_section *section = __pfn_to_section(pfn);
52d4b9ac KH	112	struct page_cgroup base, pc;
	113	unsigned long table_size;
	114	int nid, index;
	115
dc19f9db KH	116	if (!section->page_cgroup) {
	117	nid = page_to_nid(pfn_to_page(pfn));
	118	table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION;
ca371c0d	119	VM_BUG_ON(!slab_is_available());
f52407ce SL	120	if (node_state(nid, N_HIGH_MEMORY)) {
f52407ce SL	121	base = kmalloc_node(table_size,
ca371c0d	122	GFP_KERNEL \| __GFP_NOWARN, nid);
f52407ce SL	123	if (!base)
	124	base = vmalloc_node(table_size, nid);
	125	} else {
	126	base = kmalloc(table_size, GFP_KERNEL \| __GFP_NOWARN);
	127	if (!base)
	128	base = vmalloc(table_size);
	129	}
7952f988 CM	130	/*
	131	* The value stored in section->page_cgroup is (base - pfn)
	132	* and it does not point to the memory block allocated above,
	133	* causing kmemleak false positives.
	134	*/
	135	kmemleak_not_leak(base);
dc19f9db KH	136	} else {
	137	/*
	138	* We don't have to allocate page_cgroup again, but
	139	* address of memmap may be changed. So, we have to initialize
	140	* again.
	141	*/
	142	base = section->page_cgroup + pfn;
	143	table_size = 0;
	144	/* check address of memmap is changed or not. */
	145	if (base->page == pfn_to_page(pfn))
	146	return 0;
94b6da5a	147	}
52d4b9ac KH	148
	149	if (!base) {
	150	printk(KERN_ERR "page cgroup allocation failure\n");
	151	return -ENOMEM;
	152	}
	153
	154	for (index = 0; index < PAGES_PER_SECTION; index++) {
	155	pc = base + index;
	156	__init_page_cgroup(pc, pfn + index);
	157	}
	158
52d4b9ac KH	159	section->page_cgroup = base - pfn;
	160	total_usage += table_size;
	161	return 0;
	162	}
	163	#ifdef CONFIG_MEMORY_HOTPLUG
	164	void __free_page_cgroup(unsigned long pfn)
	165	{
	166	struct mem_section *ms;
	167	struct page_cgroup *base;
	168
	169	ms = __pfn_to_section(pfn);
	170	if (!ms \|\| !ms->page_cgroup)
	171	return;
	172	base = ms->page_cgroup + pfn;
94b6da5a	173	if (is_vmalloc_addr(base)) {
52d4b9ac	174	vfree(base);
94b6da5a KH	175	ms->page_cgroup = NULL;
	176	} else {
	177	struct page *page = virt_to_page(base);
	178	if (!PageReserved(page)) { /* Is bootmem ? */
	179	kfree(base);
	180	ms->page_cgroup = NULL;
	181	}
	182	}
52d4b9ac KH	183	}
52d4b9ac KH	184
31168481	185	int __meminit online_page_cgroup(unsigned long start_pfn,
52d4b9ac KH	186	unsigned long nr_pages,
	187	int nid)
	188	{
	189	unsigned long start, end, pfn;
	190	int fail = 0;
	191
33c5d3d6	192	start = start_pfn & ~(PAGES_PER_SECTION - 1);
52d4b9ac KH	193	end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION);
	194
	195	for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION) {
	196	if (!pfn_present(pfn))
	197	continue;
	198	fail = init_section_page_cgroup(pfn);
	199	}
	200	if (!fail)
	201	return 0;
	202
	203	/* rollback */
	204	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
	205	__free_page_cgroup(pfn);
	206
	207	return -ENOMEM;
	208	}
	209
31168481	210	int __meminit offline_page_cgroup(unsigned long start_pfn,
52d4b9ac KH	211	unsigned long nr_pages, int nid)
	212	{
	213	unsigned long start, end, pfn;
	214
33c5d3d6	215	start = start_pfn & ~(PAGES_PER_SECTION - 1);
52d4b9ac KH	216	end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION);
	217
	218	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
	219	__free_page_cgroup(pfn);
	220	return 0;
	221
	222	}
	223
31168481	224	static int __meminit page_cgroup_callback(struct notifier_block *self,
52d4b9ac KH	225	unsigned long action, void *arg)
	226	{
	227	struct memory_notify *mn = arg;
	228	int ret = 0;
	229	switch (action) {
	230	case MEM_GOING_ONLINE:
	231	ret = online_page_cgroup(mn->start_pfn,
	232	mn->nr_pages, mn->status_change_nid);
	233	break;
52d4b9ac KH	234	case MEM_OFFLINE:
	235	offline_page_cgroup(mn->start_pfn,
	236	mn->nr_pages, mn->status_change_nid);
	237	break;
dc19f9db	238	case MEM_CANCEL_ONLINE:
52d4b9ac KH	239	case MEM_GOING_OFFLINE:
	240	break;
	241	case MEM_ONLINE:
	242	case MEM_CANCEL_OFFLINE:
	243	break;
	244	}
dc19f9db KH	245
	246	if (ret)
	247	ret = notifier_from_errno(ret);
	248	else
	249	ret = NOTIFY_OK;
	250
52d4b9ac KH	251	return ret;
	252	}
	253
	254	#endif
	255
	256	void __init page_cgroup_init(void)
	257	{
	258	unsigned long pfn;
	259	int fail = 0;
	260
f8d66542	261	if (mem_cgroup_disabled())
94b6da5a KH	262	return;
94b6da5a KH	263
52d4b9ac KH	264	for (pfn = 0; !fail && pfn < max_pfn; pfn += PAGES_PER_SECTION) {
	265	if (!pfn_present(pfn))
	266	continue;
	267	fail = init_section_page_cgroup(pfn);
	268	}
	269	if (fail) {
8ca739e3	270	printk(KERN_CRIT "try 'cgroup_disable=memory' boot option\n");
52d4b9ac KH	271	panic("Out of memory");
	272	} else {
	273	hotplug_memory_notifier(page_cgroup_callback, 0);
	274	}
	275	printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
8ca739e3 RD	276	printk(KERN_INFO "please try 'cgroup_disable=memory' option if you don't"
8ca739e3 RD	277	" want memory cgroups\n");
52d4b9ac KH	278	}
52d4b9ac KH	279
31168481	280	void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat)
52d4b9ac KH	281	{
	282	return;
	283	}
	284
	285	#endif
27a7faa0 KH	286
	287
	288	#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
	289
	290	static DEFINE_MUTEX(swap_cgroup_mutex);
	291	struct swap_cgroup_ctrl {
	292	struct page **map;
	293	unsigned long length;
e9e58a4e	294	spinlock_t lock;
27a7faa0 KH	295	};
	296
	297	struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
	298
27a7faa0	299	struct swap_cgroup {
a3b2d692	300	unsigned short id;
27a7faa0 KH	301	};
	302	#define SC_PER_PAGE (PAGE_SIZE/sizeof(struct swap_cgroup))
	303	#define SC_POS_MASK (SC_PER_PAGE - 1)
	304
	305	/*
	306	* SwapCgroup implements "lookup" and "exchange" operations.
	307	* In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
	308	* against SwapCache. At swap_free(), this is accessed directly from swap.
	309	*
	310	* This means,
	311	* - we have no race in "exchange" when we're accessed via SwapCache because
	312	* SwapCache(and its swp_entry) is under lock.
	313	* - When called via swap_free(), there is no user of this entry and no race.
	314	* Then, we don't need lock around "exchange".
	315	*
	316	* TODO: we can push these buffers out to HIGHMEM.
	317	*/
	318
	319	/*
	320	* allocate buffer for swap_cgroup.
	321	*/
	322	static int swap_cgroup_prepare(int type)
	323	{
	324	struct page *page;
	325	struct swap_cgroup_ctrl *ctrl;
	326	unsigned long idx, max;
	327
27a7faa0 KH	328	ctrl = &swap_cgroup_ctrl[type];
	329
	330	for (idx = 0; idx < ctrl->length; idx++) {
	331	page = alloc_page(GFP_KERNEL \| __GFP_ZERO);
	332	if (!page)
	333	goto not_enough_page;
	334	ctrl->map[idx] = page;
	335	}
	336	return 0;
	337	not_enough_page:
	338	max = idx;
	339	for (idx = 0; idx < max; idx++)
	340	__free_page(ctrl->map[idx]);
	341
	342	return -ENOMEM;
	343	}
	344
02491447 DN	345	/**
	346	* swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
	347	* @end: swap entry to be cmpxchged
	348	* @old: old id
	349	* @new: new id
	350	*
	351	* Returns old id at success, 0 at failure.
	352	* (There is no mem_cgroup useing 0 as its id)
	353	*/
	354	unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
	355	unsigned short old, unsigned short new)
	356	{
	357	int type = swp_type(ent);
	358	unsigned long offset = swp_offset(ent);
	359	unsigned long idx = offset / SC_PER_PAGE;
	360	unsigned long pos = offset & SC_POS_MASK;
	361	struct swap_cgroup_ctrl *ctrl;
	362	struct page *mappage;
	363	struct swap_cgroup *sc;
e9e58a4e KH	364	unsigned long flags;
e9e58a4e KH	365	unsigned short retval;
02491447 DN	366
	367	ctrl = &swap_cgroup_ctrl[type];
	368
	369	mappage = ctrl->map[idx];
	370	sc = page_address(mappage);
	371	sc += pos;
e9e58a4e KH	372	spin_lock_irqsave(&ctrl->lock, flags);
	373	retval = sc->id;
	374	if (retval == old)
	375	sc->id = new;
02491447	376	else
e9e58a4e KH	377	retval = 0;
	378	spin_unlock_irqrestore(&ctrl->lock, flags);
	379	return retval;
02491447 DN	380	}
02491447 DN	381
27a7faa0 KH	382	/**
	383	* swap_cgroup_record - record mem_cgroup for this swp_entry.
	384	* @ent: swap entry to be recorded into
	385	* @mem: mem_cgroup to be recorded
	386	*
a3b2d692 KH	387	* Returns old value at success, 0 at failure.
a3b2d692 KH	388	* (Of course, old value can be 0.)
27a7faa0	389	*/
a3b2d692	390	unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id)
27a7faa0 KH	391	{
	392	int type = swp_type(ent);
	393	unsigned long offset = swp_offset(ent);
	394	unsigned long idx = offset / SC_PER_PAGE;
	395	unsigned long pos = offset & SC_POS_MASK;
	396	struct swap_cgroup_ctrl *ctrl;
	397	struct page *mappage;
	398	struct swap_cgroup *sc;
a3b2d692	399	unsigned short old;
e9e58a4e	400	unsigned long flags;
27a7faa0	401
27a7faa0 KH	402	ctrl = &swap_cgroup_ctrl[type];
	403
	404	mappage = ctrl->map[idx];
	405	sc = page_address(mappage);
	406	sc += pos;
e9e58a4e KH	407	spin_lock_irqsave(&ctrl->lock, flags);
	408	old = sc->id;
	409	sc->id = id;
	410	spin_unlock_irqrestore(&ctrl->lock, flags);
27a7faa0 KH	411
	412	return old;
	413	}
	414
	415	/**
	416	* lookup_swap_cgroup - lookup mem_cgroup tied to swap entry
	417	* @ent: swap entry to be looked up.
	418	*
a3b2d692	419	* Returns CSS ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
27a7faa0	420	*/
a3b2d692	421	unsigned short lookup_swap_cgroup(swp_entry_t ent)
27a7faa0 KH	422	{
	423	int type = swp_type(ent);
	424	unsigned long offset = swp_offset(ent);
	425	unsigned long idx = offset / SC_PER_PAGE;
	426	unsigned long pos = offset & SC_POS_MASK;
	427	struct swap_cgroup_ctrl *ctrl;
	428	struct page *mappage;
	429	struct swap_cgroup *sc;
a3b2d692	430	unsigned short ret;
27a7faa0	431
27a7faa0 KH	432	ctrl = &swap_cgroup_ctrl[type];
	433	mappage = ctrl->map[idx];
	434	sc = page_address(mappage);
	435	sc += pos;
a3b2d692	436	ret = sc->id;
27a7faa0 KH	437	return ret;
	438	}
	439
	440	int swap_cgroup_swapon(int type, unsigned long max_pages)
	441	{
	442	void *array;
	443	unsigned long array_size;
	444	unsigned long length;
	445	struct swap_cgroup_ctrl *ctrl;
	446
	447	if (!do_swap_account)
	448	return 0;
	449
	450	length = ((max_pages/SC_PER_PAGE) + 1);
	451	array_size = length * sizeof(void *);
	452
	453	array = vmalloc(array_size);
	454	if (!array)
	455	goto nomem;
	456
	457	memset(array, 0, array_size);
	458	ctrl = &swap_cgroup_ctrl[type];
	459	mutex_lock(&swap_cgroup_mutex);
	460	ctrl->length = length;
	461	ctrl->map = array;
e9e58a4e	462	spin_lock_init(&ctrl->lock);
27a7faa0 KH	463	if (swap_cgroup_prepare(type)) {
	464	/* memory shortage */
	465	ctrl->map = NULL;
	466	ctrl->length = 0;
	467	vfree(array);
	468	mutex_unlock(&swap_cgroup_mutex);
	469	goto nomem;
	470	}
	471	mutex_unlock(&swap_cgroup_mutex);
	472
27a7faa0 KH	473	return 0;
	474	nomem:
	475	printk(KERN_INFO "couldn't allocate enough memory for swap_cgroup.\n");
	476	printk(KERN_INFO
	477	"swap_cgroup can be disabled by noswapaccount boot option\n");
	478	return -ENOMEM;
	479	}
	480
	481	void swap_cgroup_swapoff(int type)
	482	{
	483	int i;
	484	struct swap_cgroup_ctrl *ctrl;
	485
	486	if (!do_swap_account)
	487	return;
	488
	489	mutex_lock(&swap_cgroup_mutex);
	490	ctrl = &swap_cgroup_ctrl[type];
	491	if (ctrl->map) {
	492	for (i = 0; i < ctrl->length; i++) {
	493	struct page *page = ctrl->map[i];
	494	if (page)
	495	__free_page(page);
	496	}
	497	vfree(ctrl->map);
	498	ctrl->map = NULL;
	499	ctrl->length = 0;
	500	}
	501	mutex_unlock(&swap_cgroup_mutex);
	502	}
	503
	504	#endif