[mirror_ubuntu-bionic-kernel.git] / net / ipv4 / tcp_memcontrol.c

#include <net/tcp.h>
#include <net/tcp_memcontrol.h>
#include <net/sock.h>
#include <net/ip.h>
#include <linux/nsproxy.h>
#include <linux/memcontrol.h>
#include <linux/module.h>

int tcp_init_cgroup(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
{
	/*
	 * The root cgroup does not use res_counters, but rather,
	 * rely on the data already collected by the network
	 * subsystem
	 */
	struct res_counter *res_parent = NULL;
	struct cg_proto *cg_proto, *parent_cg;
	struct mem_cgroup *parent = parent_mem_cgroup(memcg);

	cg_proto = tcp_prot.proto_cgroup(memcg);
	if (!cg_proto)
		return 0;

	cg_proto->sysctl_mem[0] = sysctl_tcp_mem[0];
	cg_proto->sysctl_mem[1] = sysctl_tcp_mem[1];
	cg_proto->sysctl_mem[2] = sysctl_tcp_mem[2];
	cg_proto->memory_pressure = 0;
	cg_proto->memcg = memcg;

	parent_cg = tcp_prot.proto_cgroup(parent);
	if (parent_cg)
		res_parent = &parent_cg->memory_allocated;

	res_counter_init(&cg_proto->memory_allocated, res_parent);
	percpu_counter_init(&cg_proto->sockets_allocated, 0);

	return 0;
}
EXPORT_SYMBOL(tcp_init_cgroup);

void tcp_destroy_cgroup(struct mem_cgroup *memcg)
{
	struct cg_proto *cg_proto;

	cg_proto = tcp_prot.proto_cgroup(memcg);
	if (!cg_proto)
		return;

	percpu_counter_destroy(&cg_proto->sockets_allocated);
}
EXPORT_SYMBOL(tcp_destroy_cgroup);

static int tcp_update_limit(struct mem_cgroup *memcg, u64 val)
{
	struct cg_proto *cg_proto;
	int i;
	int ret;

	cg_proto = tcp_prot.proto_cgroup(memcg);
	if (!cg_proto)
		return -EINVAL;

	if (val > RES_COUNTER_MAX)
		val = RES_COUNTER_MAX;

	ret = res_counter_set_limit(&cg_proto->memory_allocated, val);
	if (ret)
		return ret;

	for (i = 0; i < 3; i++)
		cg_proto->sysctl_mem[i] = min_t(long, val >> PAGE_SHIFT,
						sysctl_tcp_mem[i]);

	if (val == RES_COUNTER_MAX)
		clear_bit(MEMCG_SOCK_ACTIVE, &cg_proto->flags);
	else if (val != RES_COUNTER_MAX) {
		/*
		 * The active bit needs to be written after the static_key
		 * update. This is what guarantees that the socket activation
		 * function is the last one to run. See sock_update_memcg() for
		 * details, and note that we don't mark any socket as belonging
		 * to this memcg until that flag is up.
		 *
		 * We need to do this, because static_keys will span multiple
		 * sites, but we can't control their order. If we mark a socket
		 * as accounted, but the accounting functions are not patched in
		 * yet, we'll lose accounting.
		 *
		 * We never race with the readers in sock_update_memcg(),
		 * because when this value change, the code to process it is not
		 * patched in yet.
		 *
		 * The activated bit is used to guarantee that no two writers
		 * will do the update in the same memcg. Without that, we can't
		 * properly shutdown the static key.
		 */
		if (!test_and_set_bit(MEMCG_SOCK_ACTIVATED, &cg_proto->flags))
			static_key_slow_inc(&memcg_socket_limit_enabled);
		set_bit(MEMCG_SOCK_ACTIVE, &cg_proto->flags);
	}

	return 0;
}

static ssize_t tcp_cgroup_write(struct kernfs_open_file *of,
				char *buf, size_t nbytes, loff_t off)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
	unsigned long long val;
	int ret = 0;

	buf = strstrip(buf);

	switch (of_cft(of)->private) {
	case RES_LIMIT:
		/* see memcontrol.c */
		ret = res_counter_memparse_write_strategy(buf, &val);
		if (ret)
			break;
		ret = tcp_update_limit(memcg, val);
		break;
	default:
		ret = -EINVAL;
		break;
	}
	return ret ?: nbytes;
}

static u64 tcp_read_stat(struct mem_cgroup *memcg, int type, u64 default_val)
{
	struct cg_proto *cg_proto;

	cg_proto = tcp_prot.proto_cgroup(memcg);
	if (!cg_proto)
		return default_val;

	return res_counter_read_u64(&cg_proto->memory_allocated, type);
}

static u64 tcp_read_usage(struct mem_cgroup *memcg)
{
	struct cg_proto *cg_proto;

	cg_proto = tcp_prot.proto_cgroup(memcg);
	if (!cg_proto)
		return atomic_long_read(&tcp_memory_allocated) << PAGE_SHIFT;

	return res_counter_read_u64(&cg_proto->memory_allocated, RES_USAGE);
}

static u64 tcp_cgroup_read(struct cgroup_subsys_state *css, struct cftype *cft)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
	u64 val;

	switch (cft->private) {
	case RES_LIMIT:
		val = tcp_read_stat(memcg, RES_LIMIT, RES_COUNTER_MAX);
		break;
	case RES_USAGE:
		val = tcp_read_usage(memcg);
		break;
	case RES_FAILCNT:
	case RES_MAX_USAGE:
		val = tcp_read_stat(memcg, cft->private, 0);
		break;
	default:
		BUG();
	}
	return val;
}

static int tcp_cgroup_reset(struct cgroup_subsys_state *css, unsigned int event)
{
	struct mem_cgroup *memcg;
	struct cg_proto *cg_proto;

	memcg = mem_cgroup_from_css(css);
	cg_proto = tcp_prot.proto_cgroup(memcg);
	if (!cg_proto)
		return 0;

	switch (event) {
	case RES_MAX_USAGE:
		res_counter_reset_max(&cg_proto->memory_allocated);
		break;
	case RES_FAILCNT:
		res_counter_reset_failcnt(&cg_proto->memory_allocated);
		break;
	}

	return 0;
}

static struct cftype tcp_files[] = {
	{
		.name = "kmem.tcp.limit_in_bytes",
		.write = tcp_cgroup_write,
		.read_u64 = tcp_cgroup_read,
		.private = RES_LIMIT,
	},
	{
		.name = "kmem.tcp.usage_in_bytes",
		.read_u64 = tcp_cgroup_read,
		.private = RES_USAGE,
	},
	{
		.name = "kmem.tcp.failcnt",
		.private = RES_FAILCNT,
		.trigger = tcp_cgroup_reset,
		.read_u64 = tcp_cgroup_read,
	},
	{
		.name = "kmem.tcp.max_usage_in_bytes",
		.private = RES_MAX_USAGE,
		.trigger = tcp_cgroup_reset,
		.read_u64 = tcp_cgroup_read,
	},
	{ }	/* terminate */
};

static int __init tcp_memcontrol_init(void)
{
	WARN_ON(cgroup_add_cftypes(&memory_cgrp_subsys, tcp_files));
	return 0;
}
__initcall(tcp_memcontrol_init);
Commit	Line	Data
d1a4c0b3 GC	1	#include <net/tcp.h>
	2	#include <net/tcp_memcontrol.h>
	3	#include <net/sock.h>
3dc43e3e GC	4	#include <net/ip.h>
3dc43e3e GC	5	#include <linux/nsproxy.h>
d1a4c0b3 GC	6	#include <linux/memcontrol.h>
	7	#include <linux/module.h>
	8
1d62e436	9	int tcp_init_cgroup(struct mem_cgroup memcg, struct cgroup_subsys ss)
d1a4c0b3 GC	10	{
	11	/*
	12	* The root cgroup does not use res_counters, but rather,
	13	* rely on the data already collected by the network
	14	* subsystem
	15	*/
	16	struct res_counter *res_parent = NULL;
	17	struct cg_proto cg_proto, parent_cg;
d1a4c0b3 GC	18	struct mem_cgroup *parent = parent_mem_cgroup(memcg);
	19
	20	cg_proto = tcp_prot.proto_cgroup(memcg);
	21	if (!cg_proto)
6bc10349	22	return 0;
d1a4c0b3	23
2e685cad EB	24	cg_proto->sysctl_mem[0] = sysctl_tcp_mem[0];
	25	cg_proto->sysctl_mem[1] = sysctl_tcp_mem[1];
	26	cg_proto->sysctl_mem[2] = sysctl_tcp_mem[2];
	27	cg_proto->memory_pressure = 0;
	28	cg_proto->memcg = memcg;
d1a4c0b3 GC	29
	30	parent_cg = tcp_prot.proto_cgroup(parent);
	31	if (parent_cg)
2e685cad	32	res_parent = &parent_cg->memory_allocated;
d1a4c0b3	33
2e685cad EB	34	res_counter_init(&cg_proto->memory_allocated, res_parent);
2e685cad EB	35	percpu_counter_init(&cg_proto->sockets_allocated, 0);
d1a4c0b3	36
6bc10349	37	return 0;
d1a4c0b3 GC	38	}
	39	EXPORT_SYMBOL(tcp_init_cgroup);
	40
1d62e436	41	void tcp_destroy_cgroup(struct mem_cgroup *memcg)
d1a4c0b3	42	{
d1a4c0b3	43	struct cg_proto *cg_proto;
d1a4c0b3 GC	44
	45	cg_proto = tcp_prot.proto_cgroup(memcg);
	46	if (!cg_proto)
	47	return;
	48
2e685cad	49	percpu_counter_destroy(&cg_proto->sockets_allocated);
d1a4c0b3 GC	50	}
d1a4c0b3 GC	51	EXPORT_SYMBOL(tcp_destroy_cgroup);
3aaabe23 GC	52
	53	static int tcp_update_limit(struct mem_cgroup *memcg, u64 val)
	54	{
3aaabe23	55	struct cg_proto *cg_proto;
3aaabe23 GC	56	int i;
	57	int ret;
	58
	59	cg_proto = tcp_prot.proto_cgroup(memcg);
	60	if (!cg_proto)
	61	return -EINVAL;
	62
6de5a8bf SZ	63	if (val > RES_COUNTER_MAX)
6de5a8bf SZ	64	val = RES_COUNTER_MAX;
3aaabe23	65
2e685cad	66	ret = res_counter_set_limit(&cg_proto->memory_allocated, val);
3aaabe23 GC	67	if (ret)
	68	return ret;
	69
	70	for (i = 0; i < 3; i++)
2e685cad EB	71	cg_proto->sysctl_mem[i] = min_t(long, val >> PAGE_SHIFT,
2e685cad EB	72	sysctl_tcp_mem[i]);
3aaabe23	73
6de5a8bf	74	if (val == RES_COUNTER_MAX)
3f134619	75	clear_bit(MEMCG_SOCK_ACTIVE, &cg_proto->flags);
6de5a8bf	76	else if (val != RES_COUNTER_MAX) {
3f134619 GC	77	/*
	78	* The active bit needs to be written after the static_key
	79	* update. This is what guarantees that the socket activation
	80	* function is the last one to run. See sock_update_memcg() for
	81	* details, and note that we don't mark any socket as belonging
	82	* to this memcg until that flag is up.
	83	*
	84	* We need to do this, because static_keys will span multiple
	85	* sites, but we can't control their order. If we mark a socket
	86	* as accounted, but the accounting functions are not patched in
	87	* yet, we'll lose accounting.
	88	*
	89	* We never race with the readers in sock_update_memcg(),
	90	* because when this value change, the code to process it is not
	91	* patched in yet.
	92	*
	93	* The activated bit is used to guarantee that no two writers
	94	* will do the update in the same memcg. Without that, we can't
	95	* properly shutdown the static key.
	96	*/
	97	if (!test_and_set_bit(MEMCG_SOCK_ACTIVATED, &cg_proto->flags))
	98	static_key_slow_inc(&memcg_socket_limit_enabled);
	99	set_bit(MEMCG_SOCK_ACTIVE, &cg_proto->flags);
	100	}
3aaabe23 GC	101
	102	return 0;
	103	}
	104
451af504 TH	105	static ssize_t tcp_cgroup_write(struct kernfs_open_file *of,
451af504 TH	106	char *buf, size_t nbytes, loff_t off)
3aaabe23	107	{
451af504	108	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3aaabe23 GC	109	unsigned long long val;
	110	int ret = 0;
	111
451af504 TH	112	buf = strstrip(buf);
	113
	114	switch (of_cft(of)->private) {
3aaabe23 GC	115	case RES_LIMIT:
3aaabe23 GC	116	/* see memcontrol.c */
451af504	117	ret = res_counter_memparse_write_strategy(buf, &val);
3aaabe23 GC	118	if (ret)
	119	break;
	120	ret = tcp_update_limit(memcg, val);
	121	break;
	122	default:
	123	ret = -EINVAL;
	124	break;
	125	}
451af504	126	return ret ?: nbytes;
3aaabe23 GC	127	}
	128
	129	static u64 tcp_read_stat(struct mem_cgroup *memcg, int type, u64 default_val)
	130	{
3aaabe23 GC	131	struct cg_proto *cg_proto;
	132
	133	cg_proto = tcp_prot.proto_cgroup(memcg);
	134	if (!cg_proto)
	135	return default_val;
	136
2e685cad	137	return res_counter_read_u64(&cg_proto->memory_allocated, type);
3aaabe23 GC	138	}
3aaabe23 GC	139
5a6dd343 GC	140	static u64 tcp_read_usage(struct mem_cgroup *memcg)
5a6dd343 GC	141	{
5a6dd343 GC	142	struct cg_proto *cg_proto;
	143
	144	cg_proto = tcp_prot.proto_cgroup(memcg);
	145	if (!cg_proto)
	146	return atomic_long_read(&tcp_memory_allocated) << PAGE_SHIFT;
	147
2e685cad	148	return res_counter_read_u64(&cg_proto->memory_allocated, RES_USAGE);
5a6dd343 GC	149	}
5a6dd343 GC	150
182446d0	151	static u64 tcp_cgroup_read(struct cgroup_subsys_state css, struct cftype cft)
3aaabe23	152	{
182446d0	153	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3aaabe23 GC	154	u64 val;
	155
	156	switch (cft->private) {
	157	case RES_LIMIT:
6de5a8bf	158	val = tcp_read_stat(memcg, RES_LIMIT, RES_COUNTER_MAX);
3aaabe23	159	break;
5a6dd343 GC	160	case RES_USAGE:
	161	val = tcp_read_usage(memcg);
	162	break;
ffea59e5	163	case RES_FAILCNT:
0850f0f5 GC	164	case RES_MAX_USAGE:
0850f0f5 GC	165	val = tcp_read_stat(memcg, cft->private, 0);
ffea59e5	166	break;
3aaabe23 GC	167	default:
	168	BUG();
	169	}
	170	return val;
	171	}
	172
182446d0	173	static int tcp_cgroup_reset(struct cgroup_subsys_state *css, unsigned int event)
ffea59e5 GC	174	{
ffea59e5 GC	175	struct mem_cgroup *memcg;
ffea59e5 GC	176	struct cg_proto *cg_proto;
ffea59e5 GC	177
182446d0	178	memcg = mem_cgroup_from_css(css);
ffea59e5 GC	179	cg_proto = tcp_prot.proto_cgroup(memcg);
	180	if (!cg_proto)
	181	return 0;
ffea59e5 GC	182
ffea59e5 GC	183	switch (event) {
0850f0f5	184	case RES_MAX_USAGE:
2e685cad	185	res_counter_reset_max(&cg_proto->memory_allocated);
0850f0f5	186	break;
ffea59e5	187	case RES_FAILCNT:
2e685cad	188	res_counter_reset_failcnt(&cg_proto->memory_allocated);
ffea59e5 GC	189	break;
	190	}
	191
	192	return 0;
	193	}
	194
676f7c8f TH	195	static struct cftype tcp_files[] = {
	196	{
	197	.name = "kmem.tcp.limit_in_bytes",
451af504	198	.write = tcp_cgroup_write,
676f7c8f TH	199	.read_u64 = tcp_cgroup_read,
	200	.private = RES_LIMIT,
	201	},
	202	{
	203	.name = "kmem.tcp.usage_in_bytes",
	204	.read_u64 = tcp_cgroup_read,
	205	.private = RES_USAGE,
	206	},
	207	{
	208	.name = "kmem.tcp.failcnt",
	209	.private = RES_FAILCNT,
	210	.trigger = tcp_cgroup_reset,
	211	.read_u64 = tcp_cgroup_read,
	212	},
	213	{
	214	.name = "kmem.tcp.max_usage_in_bytes",
	215	.private = RES_MAX_USAGE,
	216	.trigger = tcp_cgroup_reset,
	217	.read_u64 = tcp_cgroup_read,
	218	},
6bc10349	219	{ } /* terminate */
676f7c8f	220	};
6bc10349 TH	221
	222	static int __init tcp_memcontrol_init(void)
	223	{
073219e9	224	WARN_ON(cgroup_add_cftypes(&memory_cgrp_subsys, tcp_files));
6bc10349 TH	225	return 0;
	226	}
	227	__initcall(tcp_memcontrol_init);