[mirror_ubuntu-bionic-kernel.git] / kernel / bpf / syscall.c

/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of version 2 of the GNU General Public
 * License as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * General Public License for more details.
 */
#include <linux/bpf.h>
#include <linux/syscalls.h>
#include <linux/slab.h>
#include <linux/anon_inodes.h>
#include <linux/file.h>
#include <linux/license.h>
#include <linux/filter.h>
#include <linux/version.h>

int sysctl_unprivileged_bpf_disabled __read_mostly;

static LIST_HEAD(bpf_map_types);

static struct bpf_map *find_and_alloc_map(union bpf_attr *attr)
{
	struct bpf_map_type_list *tl;
	struct bpf_map *map;

	list_for_each_entry(tl, &bpf_map_types, list_node) {
		if (tl->type == attr->map_type) {
			map = tl->ops->map_alloc(attr);
			if (IS_ERR(map))
				return map;
			map->ops = tl->ops;
			map->map_type = attr->map_type;
			return map;
		}
	}
	return ERR_PTR(-EINVAL);
}

/* boot time registration of different map implementations */
void bpf_register_map_type(struct bpf_map_type_list *tl)
{
	list_add(&tl->list_node, &bpf_map_types);
}

static int bpf_map_charge_memlock(struct bpf_map *map)
{
	struct user_struct *user = get_current_user();
	unsigned long memlock_limit;

	memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;

	atomic_long_add(map->pages, &user->locked_vm);

	if (atomic_long_read(&user->locked_vm) > memlock_limit) {
		atomic_long_sub(map->pages, &user->locked_vm);
		free_uid(user);
		return -EPERM;
	}
	map->user = user;
	return 0;
}

static void bpf_map_uncharge_memlock(struct bpf_map *map)
{
	struct user_struct *user = map->user;

	atomic_long_sub(map->pages, &user->locked_vm);
	free_uid(user);
}

/* called from workqueue */
static void bpf_map_free_deferred(struct work_struct *work)
{
	struct bpf_map *map = container_of(work, struct bpf_map, work);

	bpf_map_uncharge_memlock(map);
	/* implementation dependent freeing */
	map->ops->map_free(map);
}

/* decrement map refcnt and schedule it for freeing via workqueue
 * (unrelying map implementation ops->map_free() might sleep)
 */
void bpf_map_put(struct bpf_map *map)
{
	if (atomic_dec_and_test(&map->refcnt)) {
		INIT_WORK(&map->work, bpf_map_free_deferred);
		schedule_work(&map->work);
	}
}

static int bpf_map_release(struct inode *inode, struct file *filp)
{
	struct bpf_map *map = filp->private_data;

	if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY)
		/* prog_array stores refcnt-ed bpf_prog pointers
		 * release them all when user space closes prog_array_fd
		 */
		bpf_fd_array_map_clear(map);

	bpf_map_put(map);
	return 0;
}

static const struct file_operations bpf_map_fops = {
	.release = bpf_map_release,
};

/* helper macro to check that unused fields 'union bpf_attr' are zero */
#define CHECK_ATTR(CMD) \
	memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
		   sizeof(attr->CMD##_LAST_FIELD), 0, \
		   sizeof(*attr) - \
		   offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
		   sizeof(attr->CMD##_LAST_FIELD)) != NULL

#define BPF_MAP_CREATE_LAST_FIELD max_entries
/* called via syscall */
static int map_create(union bpf_attr *attr)
{
	struct bpf_map *map;
	int err;

	err = CHECK_ATTR(BPF_MAP_CREATE);
	if (err)
		return -EINVAL;

	/* find map type and init map: hashtable vs rbtree vs bloom vs ... */
	map = find_and_alloc_map(attr);
	if (IS_ERR(map))
		return PTR_ERR(map);

	atomic_set(&map->refcnt, 1);

	err = bpf_map_charge_memlock(map);
	if (err)
		goto free_map;

	err = anon_inode_getfd("bpf-map", &bpf_map_fops, map, O_RDWR | O_CLOEXEC);

	if (err < 0)
		/* failed to allocate fd */
		goto free_map;

	return err;

free_map:
	map->ops->map_free(map);
	return err;
}

/* if error is returned, fd is released.
 * On success caller should complete fd access with matching fdput()
 */
struct bpf_map *bpf_map_get(struct fd f)
{
	struct bpf_map *map;

	if (!f.file)
		return ERR_PTR(-EBADF);

	if (f.file->f_op != &bpf_map_fops) {
		fdput(f);
		return ERR_PTR(-EINVAL);
	}

	map = f.file->private_data;

	return map;
}

/* helper to convert user pointers passed inside __aligned_u64 fields */
static void __user *u64_to_ptr(__u64 val)
{
	return (void __user *) (unsigned long) val;
}

/* last field in 'union bpf_attr' used by this command */
#define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value

static int map_lookup_elem(union bpf_attr *attr)
{
	void __user *ukey = u64_to_ptr(attr->key);
	void __user *uvalue = u64_to_ptr(attr->value);
	int ufd = attr->map_fd;
	struct bpf_map *map;
	void *key, *value, *ptr;
	struct fd f;
	int err;

	if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
		return -EINVAL;

	f = fdget(ufd);
	map = bpf_map_get(f);
	if (IS_ERR(map))
		return PTR_ERR(map);

	err = -ENOMEM;
	key = kmalloc(map->key_size, GFP_USER);
	if (!key)
		goto err_put;

	err = -EFAULT;
	if (copy_from_user(key, ukey, map->key_size) != 0)
		goto free_key;

	err = -ENOMEM;
	value = kmalloc(map->value_size, GFP_USER);
	if (!value)
		goto free_key;

	rcu_read_lock();
	ptr = map->ops->map_lookup_elem(map, key);
	if (ptr)
		memcpy(value, ptr, map->value_size);
	rcu_read_unlock();

	err = -ENOENT;
	if (!ptr)
		goto free_value;

	err = -EFAULT;
	if (copy_to_user(uvalue, value, map->value_size) != 0)
		goto free_value;

	err = 0;

free_value:
	kfree(value);
free_key:
	kfree(key);
err_put:
	fdput(f);
	return err;
}

#define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags

static int map_update_elem(union bpf_attr *attr)
{
	void __user *ukey = u64_to_ptr(attr->key);
	void __user *uvalue = u64_to_ptr(attr->value);
	int ufd = attr->map_fd;
	struct bpf_map *map;
	void *key, *value;
	struct fd f;
	int err;

	if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
		return -EINVAL;

	f = fdget(ufd);
	map = bpf_map_get(f);
	if (IS_ERR(map))
		return PTR_ERR(map);

	err = -ENOMEM;
	key = kmalloc(map->key_size, GFP_USER);
	if (!key)
		goto err_put;

	err = -EFAULT;
	if (copy_from_user(key, ukey, map->key_size) != 0)
		goto free_key;

	err = -ENOMEM;
	value = kmalloc(map->value_size, GFP_USER);
	if (!value)
		goto free_key;

	err = -EFAULT;
	if (copy_from_user(value, uvalue, map->value_size) != 0)
		goto free_value;

	/* eBPF program that use maps are running under rcu_read_lock(),
	 * therefore all map accessors rely on this fact, so do the same here
	 */
	rcu_read_lock();
	err = map->ops->map_update_elem(map, key, value, attr->flags);
	rcu_read_unlock();

free_value:
	kfree(value);
free_key:
	kfree(key);
err_put:
	fdput(f);
	return err;
}

#define BPF_MAP_DELETE_ELEM_LAST_FIELD key

static int map_delete_elem(union bpf_attr *attr)
{
	void __user *ukey = u64_to_ptr(attr->key);
	int ufd = attr->map_fd;
	struct bpf_map *map;
	struct fd f;
	void *key;
	int err;

	if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
		return -EINVAL;

	f = fdget(ufd);
	map = bpf_map_get(f);
	if (IS_ERR(map))
		return PTR_ERR(map);

	err = -ENOMEM;
	key = kmalloc(map->key_size, GFP_USER);
	if (!key)
		goto err_put;

	err = -EFAULT;
	if (copy_from_user(key, ukey, map->key_size) != 0)
		goto free_key;

	rcu_read_lock();
	err = map->ops->map_delete_elem(map, key);
	rcu_read_unlock();

free_key:
	kfree(key);
err_put:
	fdput(f);
	return err;
}

/* last field in 'union bpf_attr' used by this command */
#define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key

static int map_get_next_key(union bpf_attr *attr)
{
	void __user *ukey = u64_to_ptr(attr->key);
	void __user *unext_key = u64_to_ptr(attr->next_key);
	int ufd = attr->map_fd;
	struct bpf_map *map;
	void *key, *next_key;
	struct fd f;
	int err;

	if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
		return -EINVAL;

	f = fdget(ufd);
	map = bpf_map_get(f);
	if (IS_ERR(map))
		return PTR_ERR(map);

	err = -ENOMEM;
	key = kmalloc(map->key_size, GFP_USER);
	if (!key)
		goto err_put;

	err = -EFAULT;
	if (copy_from_user(key, ukey, map->key_size) != 0)
		goto free_key;

	err = -ENOMEM;
	next_key = kmalloc(map->key_size, GFP_USER);
	if (!next_key)
		goto free_key;

	rcu_read_lock();
	err = map->ops->map_get_next_key(map, key, next_key);
	rcu_read_unlock();
	if (err)
		goto free_next_key;

	err = -EFAULT;
	if (copy_to_user(unext_key, next_key, map->key_size) != 0)
		goto free_next_key;

	err = 0;

free_next_key:
	kfree(next_key);
free_key:
	kfree(key);
err_put:
	fdput(f);
	return err;
}

static LIST_HEAD(bpf_prog_types);

static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
{
	struct bpf_prog_type_list *tl;

	list_for_each_entry(tl, &bpf_prog_types, list_node) {
		if (tl->type == type) {
			prog->aux->ops = tl->ops;
			prog->type = type;
			return 0;
		}
	}

	return -EINVAL;
}

void bpf_register_prog_type(struct bpf_prog_type_list *tl)
{
	list_add(&tl->list_node, &bpf_prog_types);
}

/* fixup insn->imm field of bpf_call instructions:
 * if (insn->imm == BPF_FUNC_map_lookup_elem)
 *      insn->imm = bpf_map_lookup_elem - __bpf_call_base;
 * else if (insn->imm == BPF_FUNC_map_update_elem)
 *      insn->imm = bpf_map_update_elem - __bpf_call_base;
 * else ...
 *
 * this function is called after eBPF program passed verification
 */
static void fixup_bpf_calls(struct bpf_prog *prog)
{
	const struct bpf_func_proto *fn;
	int i;

	for (i = 0; i < prog->len; i++) {
		struct bpf_insn *insn = &prog->insnsi[i];

		if (insn->code == (BPF_JMP | BPF_CALL)) {
			/* we reach here when program has bpf_call instructions
			 * and it passed bpf_check(), means that
			 * ops->get_func_proto must have been supplied, check it
			 */
			BUG_ON(!prog->aux->ops->get_func_proto);

			if (insn->imm == BPF_FUNC_get_route_realm)
				prog->dst_needed = 1;
			if (insn->imm == BPF_FUNC_get_prandom_u32)
				bpf_user_rnd_init_once();
			if (insn->imm == BPF_FUNC_tail_call) {
				/* mark bpf_tail_call as different opcode
				 * to avoid conditional branch in
				 * interpeter for every normal call
				 * and to prevent accidental JITing by
				 * JIT compiler that doesn't support
				 * bpf_tail_call yet
				 */
				insn->imm = 0;
				insn->code |= BPF_X;
				continue;
			}

			fn = prog->aux->ops->get_func_proto(insn->imm);
			/* all functions that have prototype and verifier allowed
			 * programs to call them, must be real in-kernel functions
			 */
			BUG_ON(!fn->func);
			insn->imm = fn->func - __bpf_call_base;
		}
	}
}

/* drop refcnt on maps used by eBPF program and free auxilary data */
static void free_used_maps(struct bpf_prog_aux *aux)
{
	int i;

	for (i = 0; i < aux->used_map_cnt; i++)
		bpf_map_put(aux->used_maps[i]);

	kfree(aux->used_maps);
}

static int bpf_prog_charge_memlock(struct bpf_prog *prog)
{
	struct user_struct *user = get_current_user();
	unsigned long memlock_limit;

	memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;

	atomic_long_add(prog->pages, &user->locked_vm);
	if (atomic_long_read(&user->locked_vm) > memlock_limit) {
		atomic_long_sub(prog->pages, &user->locked_vm);
		free_uid(user);
		return -EPERM;
	}
	prog->aux->user = user;
	return 0;
}

static void bpf_prog_uncharge_memlock(struct bpf_prog *prog)
{
	struct user_struct *user = prog->aux->user;

	atomic_long_sub(prog->pages, &user->locked_vm);
	free_uid(user);
}

static void __prog_put_rcu(struct rcu_head *rcu)
{
	struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);

	free_used_maps(aux);
	bpf_prog_uncharge_memlock(aux->prog);
	bpf_prog_free(aux->prog);
}

/* version of bpf_prog_put() that is called after a grace period */
void bpf_prog_put_rcu(struct bpf_prog *prog)
{
	if (atomic_dec_and_test(&prog->aux->refcnt)) {
		prog->aux->prog = prog;
		call_rcu(&prog->aux->rcu, __prog_put_rcu);
	}
}

void bpf_prog_put(struct bpf_prog *prog)
{
	if (atomic_dec_and_test(&prog->aux->refcnt)) {
		free_used_maps(prog->aux);
		bpf_prog_uncharge_memlock(prog);
		bpf_prog_free(prog);
	}
}
EXPORT_SYMBOL_GPL(bpf_prog_put);

static int bpf_prog_release(struct inode *inode, struct file *filp)
{
	struct bpf_prog *prog = filp->private_data;

	bpf_prog_put_rcu(prog);
	return 0;
}

static const struct file_operations bpf_prog_fops = {
        .release = bpf_prog_release,
};

static struct bpf_prog *get_prog(struct fd f)
{
	struct bpf_prog *prog;

	if (!f.file)
		return ERR_PTR(-EBADF);

	if (f.file->f_op != &bpf_prog_fops) {
		fdput(f);
		return ERR_PTR(-EINVAL);
	}

	prog = f.file->private_data;

	return prog;
}

/* called by sockets/tracing/seccomp before attaching program to an event
 * pairs with bpf_prog_put()
 */
struct bpf_prog *bpf_prog_get(u32 ufd)
{
	struct fd f = fdget(ufd);
	struct bpf_prog *prog;

	prog = get_prog(f);

	if (IS_ERR(prog))
		return prog;

	atomic_inc(&prog->aux->refcnt);
	fdput(f);
	return prog;
}
EXPORT_SYMBOL_GPL(bpf_prog_get);

/* last field in 'union bpf_attr' used by this command */
#define	BPF_PROG_LOAD_LAST_FIELD kern_version

static int bpf_prog_load(union bpf_attr *attr)
{
	enum bpf_prog_type type = attr->prog_type;
	struct bpf_prog *prog;
	int err;
	char license[128];
	bool is_gpl;

	if (CHECK_ATTR(BPF_PROG_LOAD))
		return -EINVAL;

	/* copy eBPF program license from user space */
	if (strncpy_from_user(license, u64_to_ptr(attr->license),
			      sizeof(license) - 1) < 0)
		return -EFAULT;
	license[sizeof(license) - 1] = 0;

	/* eBPF programs must be GPL compatible to use GPL-ed functions */
	is_gpl = license_is_gpl_compatible(license);

	if (attr->insn_cnt >= BPF_MAXINSNS)
		return -EINVAL;

	if (type == BPF_PROG_TYPE_KPROBE &&
	    attr->kern_version != LINUX_VERSION_CODE)
		return -EINVAL;

	if (type != BPF_PROG_TYPE_SOCKET_FILTER && !capable(CAP_SYS_ADMIN))
		return -EPERM;

	/* plain bpf_prog allocation */
	prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
	if (!prog)
		return -ENOMEM;

	err = bpf_prog_charge_memlock(prog);
	if (err)
		goto free_prog_nouncharge;

	prog->len = attr->insn_cnt;

	err = -EFAULT;
	if (copy_from_user(prog->insns, u64_to_ptr(attr->insns),
			   prog->len * sizeof(struct bpf_insn)) != 0)
		goto free_prog;

	prog->orig_prog = NULL;
	prog->jited = 0;

	atomic_set(&prog->aux->refcnt, 1);
	prog->gpl_compatible = is_gpl ? 1 : 0;

	/* find program type: socket_filter vs tracing_filter */
	err = find_prog_type(type, prog);
	if (err < 0)
		goto free_prog;

	/* run eBPF verifier */
	err = bpf_check(&prog, attr);
	if (err < 0)
		goto free_used_maps;

	/* fixup BPF_CALL->imm field */
	fixup_bpf_calls(prog);

	/* eBPF program is ready to be JITed */
	err = bpf_prog_select_runtime(prog);
	if (err < 0)
		goto free_used_maps;

	err = anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, O_RDWR | O_CLOEXEC);
	if (err < 0)
		/* failed to allocate fd */
		goto free_used_maps;

	return err;

free_used_maps:
	free_used_maps(prog->aux);
free_prog:
	bpf_prog_uncharge_memlock(prog);
free_prog_nouncharge:
	bpf_prog_free(prog);
	return err;
}

SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
{
	union bpf_attr attr = {};
	int err;

	if (!capable(CAP_SYS_ADMIN) && sysctl_unprivileged_bpf_disabled)
		return -EPERM;

	if (!access_ok(VERIFY_READ, uattr, 1))
		return -EFAULT;

	if (size > PAGE_SIZE)	/* silly large */
		return -E2BIG;

	/* If we're handed a bigger struct than we know of,
	 * ensure all the unknown bits are 0 - i.e. new
	 * user-space does not rely on any kernel feature
	 * extensions we dont know about yet.
	 */
	if (size > sizeof(attr)) {
		unsigned char __user *addr;
		unsigned char __user *end;
		unsigned char val;

		addr = (void __user *)uattr + sizeof(attr);
		end  = (void __user *)uattr + size;

		for (; addr < end; addr++) {
			err = get_user(val, addr);
			if (err)
				return err;
			if (val)
				return -E2BIG;
		}
		size = sizeof(attr);
	}

	/* copy attributes from user space, may be less than sizeof(bpf_attr) */
	if (copy_from_user(&attr, uattr, size) != 0)
		return -EFAULT;

	switch (cmd) {
	case BPF_MAP_CREATE:
		err = map_create(&attr);
		break;
	case BPF_MAP_LOOKUP_ELEM:
		err = map_lookup_elem(&attr);
		break;
	case BPF_MAP_UPDATE_ELEM:
		err = map_update_elem(&attr);
		break;
	case BPF_MAP_DELETE_ELEM:
		err = map_delete_elem(&attr);
		break;
	case BPF_MAP_GET_NEXT_KEY:
		err = map_get_next_key(&attr);
		break;
	case BPF_PROG_LOAD:
		err = bpf_prog_load(&attr);
		break;
	default:
		err = -EINVAL;
		break;
	}

	return err;
}
Commit	Line	Data
99c55f7d AS	1	/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
	2	*
	3	* This program is free software; you can redistribute it and/or
	4	* modify it under the terms of version 2 of the GNU General Public
	5	* License as published by the Free Software Foundation.
	6	*
	7	* This program is distributed in the hope that it will be useful, but
	8	* WITHOUT ANY WARRANTY; without even the implied warranty of
	9	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	10	* General Public License for more details.
	11	*/
	12	#include <linux/bpf.h>
	13	#include <linux/syscalls.h>
	14	#include <linux/slab.h>
	15	#include <linux/anon_inodes.h>
db20fd2b	16	#include <linux/file.h>
09756af4 AS	17	#include <linux/license.h>
09756af4 AS	18	#include <linux/filter.h>
2541517c	19	#include <linux/version.h>
99c55f7d	20
1be7f75d AS	21	int sysctl_unprivileged_bpf_disabled __read_mostly;
1be7f75d AS	22
99c55f7d AS	23	static LIST_HEAD(bpf_map_types);
	24
	25	static struct bpf_map find_and_alloc_map(union bpf_attr attr)
	26	{
	27	struct bpf_map_type_list *tl;
	28	struct bpf_map *map;
	29
	30	list_for_each_entry(tl, &bpf_map_types, list_node) {
	31	if (tl->type == attr->map_type) {
	32	map = tl->ops->map_alloc(attr);
	33	if (IS_ERR(map))
	34	return map;
	35	map->ops = tl->ops;
	36	map->map_type = attr->map_type;
	37	return map;
	38	}
	39	}
	40	return ERR_PTR(-EINVAL);
	41	}
	42
	43	/* boot time registration of different map implementations */
	44	void bpf_register_map_type(struct bpf_map_type_list *tl)
	45	{
	46	list_add(&tl->list_node, &bpf_map_types);
	47	}
	48
aaac3ba9 AS	49	static int bpf_map_charge_memlock(struct bpf_map *map)
	50	{
	51	struct user_struct *user = get_current_user();
	52	unsigned long memlock_limit;
	53
	54	memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
	55
	56	atomic_long_add(map->pages, &user->locked_vm);
	57
	58	if (atomic_long_read(&user->locked_vm) > memlock_limit) {
	59	atomic_long_sub(map->pages, &user->locked_vm);
	60	free_uid(user);
	61	return -EPERM;
	62	}
	63	map->user = user;
	64	return 0;
	65	}
	66
	67	static void bpf_map_uncharge_memlock(struct bpf_map *map)
	68	{
	69	struct user_struct *user = map->user;
	70
	71	atomic_long_sub(map->pages, &user->locked_vm);
	72	free_uid(user);
	73	}
	74
99c55f7d AS	75	/* called from workqueue */
	76	static void bpf_map_free_deferred(struct work_struct *work)
	77	{
	78	struct bpf_map *map = container_of(work, struct bpf_map, work);
	79
aaac3ba9	80	bpf_map_uncharge_memlock(map);
99c55f7d AS	81	/* implementation dependent freeing */
	82	map->ops->map_free(map);
	83	}
	84
	85	/* decrement map refcnt and schedule it for freeing via workqueue
	86	* (unrelying map implementation ops->map_free() might sleep)
	87	*/
	88	void bpf_map_put(struct bpf_map *map)
	89	{
	90	if (atomic_dec_and_test(&map->refcnt)) {
	91	INIT_WORK(&map->work, bpf_map_free_deferred);
	92	schedule_work(&map->work);
	93	}
	94	}
	95
	96	static int bpf_map_release(struct inode inode, struct file filp)
	97	{
	98	struct bpf_map *map = filp->private_data;
	99
04fd61ab AS	100	if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY)
	101	/* prog_array stores refcnt-ed bpf_prog pointers
	102	* release them all when user space closes prog_array_fd
	103	*/
2a36f0b9	104	bpf_fd_array_map_clear(map);
04fd61ab	105
99c55f7d AS	106	bpf_map_put(map);
	107	return 0;
	108	}
	109
	110	static const struct file_operations bpf_map_fops = {
	111	.release = bpf_map_release,
	112	};
	113
	114	/* helper macro to check that unused fields 'union bpf_attr' are zero */
	115	#define CHECK_ATTR(CMD) \
	116	memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
	117	sizeof(attr->CMD##_LAST_FIELD), 0, \
	118	sizeof(*attr) - \
	119	offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
	120	sizeof(attr->CMD##_LAST_FIELD)) != NULL
	121
	122	#define BPF_MAP_CREATE_LAST_FIELD max_entries
	123	/* called via syscall */
	124	static int map_create(union bpf_attr *attr)
	125	{
	126	struct bpf_map *map;
	127	int err;
	128
	129	err = CHECK_ATTR(BPF_MAP_CREATE);
	130	if (err)
	131	return -EINVAL;
	132
	133	/* find map type and init map: hashtable vs rbtree vs bloom vs ... */
	134	map = find_and_alloc_map(attr);
	135	if (IS_ERR(map))
	136	return PTR_ERR(map);
	137
	138	atomic_set(&map->refcnt, 1);
	139
aaac3ba9 AS	140	err = bpf_map_charge_memlock(map);
	141	if (err)
	142	goto free_map;
	143
99c55f7d AS	144	err = anon_inode_getfd("bpf-map", &bpf_map_fops, map, O_RDWR \| O_CLOEXEC);
	145
	146	if (err < 0)
	147	/* failed to allocate fd */
	148	goto free_map;
	149
	150	return err;
	151
	152	free_map:
	153	map->ops->map_free(map);
	154	return err;
	155	}
	156
db20fd2b AS	157	/* if error is returned, fd is released.
	158	* On success caller should complete fd access with matching fdput()
	159	*/
	160	struct bpf_map *bpf_map_get(struct fd f)
	161	{
	162	struct bpf_map *map;
	163
	164	if (!f.file)
	165	return ERR_PTR(-EBADF);
	166
	167	if (f.file->f_op != &bpf_map_fops) {
	168	fdput(f);
	169	return ERR_PTR(-EINVAL);
	170	}
	171
	172	map = f.file->private_data;
	173
	174	return map;
	175	}
	176
	177	/* helper to convert user pointers passed inside __aligned_u64 fields */
	178	static void __user *u64_to_ptr(__u64 val)
	179	{
	180	return (void __user *) (unsigned long) val;
	181	}
	182
	183	/* last field in 'union bpf_attr' used by this command */
	184	#define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value
	185
	186	static int map_lookup_elem(union bpf_attr *attr)
	187	{
	188	void __user *ukey = u64_to_ptr(attr->key);
	189	void __user *uvalue = u64_to_ptr(attr->value);
	190	int ufd = attr->map_fd;
db20fd2b	191	struct bpf_map *map;
8ebe667c	192	void key, value, *ptr;
592867bf	193	struct fd f;
db20fd2b AS	194	int err;
	195
	196	if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
	197	return -EINVAL;
	198
592867bf	199	f = fdget(ufd);
db20fd2b AS	200	map = bpf_map_get(f);
	201	if (IS_ERR(map))
	202	return PTR_ERR(map);
	203
	204	err = -ENOMEM;
	205	key = kmalloc(map->key_size, GFP_USER);
	206	if (!key)
	207	goto err_put;
	208
	209	err = -EFAULT;
	210	if (copy_from_user(key, ukey, map->key_size) != 0)
	211	goto free_key;
	212
8ebe667c AS	213	err = -ENOMEM;
8ebe667c AS	214	value = kmalloc(map->value_size, GFP_USER);
db20fd2b	215	if (!value)
8ebe667c AS	216	goto free_key;
	217
	218	rcu_read_lock();
	219	ptr = map->ops->map_lookup_elem(map, key);
	220	if (ptr)
	221	memcpy(value, ptr, map->value_size);
	222	rcu_read_unlock();
	223
	224	err = -ENOENT;
	225	if (!ptr)
	226	goto free_value;
db20fd2b AS	227
	228	err = -EFAULT;
	229	if (copy_to_user(uvalue, value, map->value_size) != 0)
8ebe667c	230	goto free_value;
db20fd2b AS	231
	232	err = 0;
	233
8ebe667c AS	234	free_value:
8ebe667c AS	235	kfree(value);
db20fd2b AS	236	free_key:
	237	kfree(key);
	238	err_put:
	239	fdput(f);
	240	return err;
	241	}
	242
3274f520	243	#define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
db20fd2b AS	244
	245	static int map_update_elem(union bpf_attr *attr)
	246	{
	247	void __user *ukey = u64_to_ptr(attr->key);
	248	void __user *uvalue = u64_to_ptr(attr->value);
	249	int ufd = attr->map_fd;
db20fd2b AS	250	struct bpf_map *map;
db20fd2b AS	251	void key, value;
592867bf	252	struct fd f;
db20fd2b AS	253	int err;
	254
	255	if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
	256	return -EINVAL;
	257
592867bf	258	f = fdget(ufd);
db20fd2b AS	259	map = bpf_map_get(f);
	260	if (IS_ERR(map))
	261	return PTR_ERR(map);
	262
	263	err = -ENOMEM;
	264	key = kmalloc(map->key_size, GFP_USER);
	265	if (!key)
	266	goto err_put;
	267
	268	err = -EFAULT;
	269	if (copy_from_user(key, ukey, map->key_size) != 0)
	270	goto free_key;
	271
	272	err = -ENOMEM;
	273	value = kmalloc(map->value_size, GFP_USER);
	274	if (!value)
	275	goto free_key;
	276
	277	err = -EFAULT;
	278	if (copy_from_user(value, uvalue, map->value_size) != 0)
	279	goto free_value;
	280
	281	/* eBPF program that use maps are running under rcu_read_lock(),
	282	* therefore all map accessors rely on this fact, so do the same here
	283	*/
	284	rcu_read_lock();
3274f520	285	err = map->ops->map_update_elem(map, key, value, attr->flags);
db20fd2b AS	286	rcu_read_unlock();
	287
	288	free_value:
	289	kfree(value);
	290	free_key:
	291	kfree(key);
	292	err_put:
	293	fdput(f);
	294	return err;
	295	}
	296
	297	#define BPF_MAP_DELETE_ELEM_LAST_FIELD key
	298
	299	static int map_delete_elem(union bpf_attr *attr)
	300	{
	301	void __user *ukey = u64_to_ptr(attr->key);
	302	int ufd = attr->map_fd;
db20fd2b	303	struct bpf_map *map;
592867bf	304	struct fd f;
db20fd2b AS	305	void *key;
	306	int err;
	307
	308	if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
	309	return -EINVAL;
	310
592867bf	311	f = fdget(ufd);
db20fd2b AS	312	map = bpf_map_get(f);
	313	if (IS_ERR(map))
	314	return PTR_ERR(map);
	315
	316	err = -ENOMEM;
	317	key = kmalloc(map->key_size, GFP_USER);
	318	if (!key)
	319	goto err_put;
	320
	321	err = -EFAULT;
	322	if (copy_from_user(key, ukey, map->key_size) != 0)
	323	goto free_key;
	324
	325	rcu_read_lock();
	326	err = map->ops->map_delete_elem(map, key);
	327	rcu_read_unlock();
	328
	329	free_key:
	330	kfree(key);
	331	err_put:
	332	fdput(f);
	333	return err;
	334	}
	335
	336	/* last field in 'union bpf_attr' used by this command */
	337	#define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
	338
	339	static int map_get_next_key(union bpf_attr *attr)
	340	{
	341	void __user *ukey = u64_to_ptr(attr->key);
	342	void __user *unext_key = u64_to_ptr(attr->next_key);
	343	int ufd = attr->map_fd;
db20fd2b AS	344	struct bpf_map *map;
db20fd2b AS	345	void key, next_key;
592867bf	346	struct fd f;
db20fd2b AS	347	int err;
	348
	349	if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
	350	return -EINVAL;
	351
592867bf	352	f = fdget(ufd);
db20fd2b AS	353	map = bpf_map_get(f);
	354	if (IS_ERR(map))
	355	return PTR_ERR(map);
	356
	357	err = -ENOMEM;
	358	key = kmalloc(map->key_size, GFP_USER);
	359	if (!key)
	360	goto err_put;
	361
	362	err = -EFAULT;
	363	if (copy_from_user(key, ukey, map->key_size) != 0)
	364	goto free_key;
	365
	366	err = -ENOMEM;
	367	next_key = kmalloc(map->key_size, GFP_USER);
	368	if (!next_key)
	369	goto free_key;
	370
	371	rcu_read_lock();
	372	err = map->ops->map_get_next_key(map, key, next_key);
	373	rcu_read_unlock();
	374	if (err)
	375	goto free_next_key;
	376
	377	err = -EFAULT;
	378	if (copy_to_user(unext_key, next_key, map->key_size) != 0)
	379	goto free_next_key;
	380
	381	err = 0;
	382
	383	free_next_key:
	384	kfree(next_key);
	385	free_key:
	386	kfree(key);
	387	err_put:
	388	fdput(f);
	389	return err;
	390	}
	391
09756af4 AS	392	static LIST_HEAD(bpf_prog_types);
	393
	394	static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
	395	{
	396	struct bpf_prog_type_list *tl;
	397
	398	list_for_each_entry(tl, &bpf_prog_types, list_node) {
	399	if (tl->type == type) {
	400	prog->aux->ops = tl->ops;
24701ece	401	prog->type = type;
09756af4 AS	402	return 0;
	403	}
	404	}
24701ece	405
09756af4 AS	406	return -EINVAL;
	407	}
	408
	409	void bpf_register_prog_type(struct bpf_prog_type_list *tl)
	410	{
	411	list_add(&tl->list_node, &bpf_prog_types);
	412	}
	413
0a542a86 AS	414	/* fixup insn->imm field of bpf_call instructions:
	415	* if (insn->imm == BPF_FUNC_map_lookup_elem)
	416	* insn->imm = bpf_map_lookup_elem - __bpf_call_base;
	417	* else if (insn->imm == BPF_FUNC_map_update_elem)
	418	* insn->imm = bpf_map_update_elem - __bpf_call_base;
	419	* else ...
	420	*
	421	* this function is called after eBPF program passed verification
	422	*/
	423	static void fixup_bpf_calls(struct bpf_prog *prog)
	424	{
	425	const struct bpf_func_proto *fn;
	426	int i;
	427
	428	for (i = 0; i < prog->len; i++) {
	429	struct bpf_insn *insn = &prog->insnsi[i];
	430
	431	if (insn->code == (BPF_JMP \| BPF_CALL)) {
	432	/* we reach here when program has bpf_call instructions
	433	* and it passed bpf_check(), means that
	434	* ops->get_func_proto must have been supplied, check it
	435	*/
	436	BUG_ON(!prog->aux->ops->get_func_proto);
	437
c46646d0 DB	438	if (insn->imm == BPF_FUNC_get_route_realm)
c46646d0 DB	439	prog->dst_needed = 1;
3ad00405 DB	440	if (insn->imm == BPF_FUNC_get_prandom_u32)
3ad00405 DB	441	bpf_user_rnd_init_once();
04fd61ab AS	442	if (insn->imm == BPF_FUNC_tail_call) {
	443	/* mark bpf_tail_call as different opcode
	444	* to avoid conditional branch in
	445	* interpeter for every normal call
	446	* and to prevent accidental JITing by
	447	* JIT compiler that doesn't support
	448	* bpf_tail_call yet
	449	*/
	450	insn->imm = 0;
	451	insn->code \|= BPF_X;
	452	continue;
	453	}
	454
0a542a86 AS	455	fn = prog->aux->ops->get_func_proto(insn->imm);
	456	/* all functions that have prototype and verifier allowed
	457	* programs to call them, must be real in-kernel functions
	458	*/
	459	BUG_ON(!fn->func);
	460	insn->imm = fn->func - __bpf_call_base;
	461	}
	462	}
	463	}
	464
09756af4 AS	465	/* drop refcnt on maps used by eBPF program and free auxilary data */
	466	static void free_used_maps(struct bpf_prog_aux *aux)
	467	{
	468	int i;
	469
	470	for (i = 0; i < aux->used_map_cnt; i++)
	471	bpf_map_put(aux->used_maps[i]);
	472
	473	kfree(aux->used_maps);
	474	}
	475
aaac3ba9 AS	476	static int bpf_prog_charge_memlock(struct bpf_prog *prog)
	477	{
	478	struct user_struct *user = get_current_user();
	479	unsigned long memlock_limit;
	480
	481	memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
	482
	483	atomic_long_add(prog->pages, &user->locked_vm);
	484	if (atomic_long_read(&user->locked_vm) > memlock_limit) {
	485	atomic_long_sub(prog->pages, &user->locked_vm);
	486	free_uid(user);
	487	return -EPERM;
	488	}
	489	prog->aux->user = user;
	490	return 0;
	491	}
	492
	493	static void bpf_prog_uncharge_memlock(struct bpf_prog *prog)
	494	{
	495	struct user_struct *user = prog->aux->user;
	496
	497	atomic_long_sub(prog->pages, &user->locked_vm);
	498	free_uid(user);
	499	}
	500
abf2e7d6 AS	501	static void __prog_put_rcu(struct rcu_head *rcu)
	502	{
	503	struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
	504
	505	free_used_maps(aux);
aaac3ba9	506	bpf_prog_uncharge_memlock(aux->prog);
abf2e7d6 AS	507	bpf_prog_free(aux->prog);
	508	}
	509
	510	/* version of bpf_prog_put() that is called after a grace period */
	511	void bpf_prog_put_rcu(struct bpf_prog *prog)
	512	{
	513	if (atomic_dec_and_test(&prog->aux->refcnt)) {
	514	prog->aux->prog = prog;
	515	call_rcu(&prog->aux->rcu, __prog_put_rcu);
	516	}
	517	}
	518
09756af4 AS	519	void bpf_prog_put(struct bpf_prog *prog)
	520	{
	521	if (atomic_dec_and_test(&prog->aux->refcnt)) {
	522	free_used_maps(prog->aux);
ac00737f	523	bpf_prog_uncharge_memlock(prog);
09756af4 AS	524	bpf_prog_free(prog);
	525	}
	526	}
e2e9b654	527	EXPORT_SYMBOL_GPL(bpf_prog_put);
09756af4 AS	528
	529	static int bpf_prog_release(struct inode inode, struct file filp)
	530	{
	531	struct bpf_prog *prog = filp->private_data;
	532
abf2e7d6	533	bpf_prog_put_rcu(prog);
09756af4 AS	534	return 0;
	535	}
	536
	537	static const struct file_operations bpf_prog_fops = {
	538	.release = bpf_prog_release,
	539	};
	540
	541	static struct bpf_prog *get_prog(struct fd f)
	542	{
	543	struct bpf_prog *prog;
	544
	545	if (!f.file)
	546	return ERR_PTR(-EBADF);
	547
	548	if (f.file->f_op != &bpf_prog_fops) {
	549	fdput(f);
	550	return ERR_PTR(-EINVAL);
	551	}
	552
	553	prog = f.file->private_data;
	554
	555	return prog;
	556	}
	557
	558	/* called by sockets/tracing/seccomp before attaching program to an event
	559	* pairs with bpf_prog_put()
	560	*/
	561	struct bpf_prog *bpf_prog_get(u32 ufd)
	562	{
	563	struct fd f = fdget(ufd);
	564	struct bpf_prog *prog;
	565
	566	prog = get_prog(f);
	567
	568	if (IS_ERR(prog))
	569	return prog;
	570
	571	atomic_inc(&prog->aux->refcnt);
	572	fdput(f);
	573	return prog;
	574	}
e2e9b654	575	EXPORT_SYMBOL_GPL(bpf_prog_get);
09756af4 AS	576
09756af4 AS	577	/* last field in 'union bpf_attr' used by this command */
2541517c	578	#define BPF_PROG_LOAD_LAST_FIELD kern_version
09756af4 AS	579
	580	static int bpf_prog_load(union bpf_attr *attr)
	581	{
	582	enum bpf_prog_type type = attr->prog_type;
	583	struct bpf_prog *prog;
	584	int err;
	585	char license[128];
	586	bool is_gpl;
	587
	588	if (CHECK_ATTR(BPF_PROG_LOAD))
	589	return -EINVAL;
	590
	591	/* copy eBPF program license from user space */
	592	if (strncpy_from_user(license, u64_to_ptr(attr->license),
	593	sizeof(license) - 1) < 0)
	594	return -EFAULT;
	595	license[sizeof(license) - 1] = 0;
	596
	597	/* eBPF programs must be GPL compatible to use GPL-ed functions */
	598	is_gpl = license_is_gpl_compatible(license);
	599
	600	if (attr->insn_cnt >= BPF_MAXINSNS)
	601	return -EINVAL;
	602
2541517c AS	603	if (type == BPF_PROG_TYPE_KPROBE &&
	604	attr->kern_version != LINUX_VERSION_CODE)
	605	return -EINVAL;
	606
1be7f75d AS	607	if (type != BPF_PROG_TYPE_SOCKET_FILTER && !capable(CAP_SYS_ADMIN))
	608	return -EPERM;
	609
09756af4 AS	610	/* plain bpf_prog allocation */
	611	prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
	612	if (!prog)
	613	return -ENOMEM;
	614
aaac3ba9 AS	615	err = bpf_prog_charge_memlock(prog);
	616	if (err)
	617	goto free_prog_nouncharge;
	618
09756af4 AS	619	prog->len = attr->insn_cnt;
	620
	621	err = -EFAULT;
	622	if (copy_from_user(prog->insns, u64_to_ptr(attr->insns),
	623	prog->len * sizeof(struct bpf_insn)) != 0)
	624	goto free_prog;
	625
	626	prog->orig_prog = NULL;
a91263d5	627	prog->jited = 0;
09756af4 AS	628
09756af4 AS	629	atomic_set(&prog->aux->refcnt, 1);
a91263d5	630	prog->gpl_compatible = is_gpl ? 1 : 0;
09756af4 AS	631
	632	/* find program type: socket_filter vs tracing_filter */
	633	err = find_prog_type(type, prog);
	634	if (err < 0)
	635	goto free_prog;
	636
	637	/* run eBPF verifier */
9bac3d6d	638	err = bpf_check(&prog, attr);
09756af4 AS	639	if (err < 0)
	640	goto free_used_maps;
	641
0a542a86 AS	642	/* fixup BPF_CALL->imm field */
	643	fixup_bpf_calls(prog);
	644
09756af4	645	/* eBPF program is ready to be JITed */
04fd61ab AS	646	err = bpf_prog_select_runtime(prog);
	647	if (err < 0)
	648	goto free_used_maps;
09756af4 AS	649
09756af4 AS	650	err = anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, O_RDWR \| O_CLOEXEC);
09756af4 AS	651	if (err < 0)
	652	/* failed to allocate fd */
	653	goto free_used_maps;
	654
	655	return err;
	656
	657	free_used_maps:
	658	free_used_maps(prog->aux);
	659	free_prog:
aaac3ba9 AS	660	bpf_prog_uncharge_memlock(prog);
aaac3ba9 AS	661	free_prog_nouncharge:
09756af4 AS	662	bpf_prog_free(prog);
	663	return err;
	664	}
	665
99c55f7d AS	666	SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
	667	{
	668	union bpf_attr attr = {};
	669	int err;
	670
1be7f75d	671	if (!capable(CAP_SYS_ADMIN) && sysctl_unprivileged_bpf_disabled)
99c55f7d AS	672	return -EPERM;
	673
	674	if (!access_ok(VERIFY_READ, uattr, 1))
	675	return -EFAULT;
	676
	677	if (size > PAGE_SIZE) /* silly large */
	678	return -E2BIG;
	679
	680	/* If we're handed a bigger struct than we know of,
	681	* ensure all the unknown bits are 0 - i.e. new
	682	* user-space does not rely on any kernel feature
	683	* extensions we dont know about yet.
	684	*/
	685	if (size > sizeof(attr)) {
	686	unsigned char __user *addr;
	687	unsigned char __user *end;
	688	unsigned char val;
	689
	690	addr = (void __user *)uattr + sizeof(attr);
	691	end = (void __user *)uattr + size;
	692
	693	for (; addr < end; addr++) {
	694	err = get_user(val, addr);
	695	if (err)
	696	return err;
	697	if (val)
	698	return -E2BIG;
	699	}
	700	size = sizeof(attr);
	701	}
	702
	703	/* copy attributes from user space, may be less than sizeof(bpf_attr) */
	704	if (copy_from_user(&attr, uattr, size) != 0)
	705	return -EFAULT;
	706
	707	switch (cmd) {
	708	case BPF_MAP_CREATE:
	709	err = map_create(&attr);
	710	break;
db20fd2b AS	711	case BPF_MAP_LOOKUP_ELEM:
	712	err = map_lookup_elem(&attr);
	713	break;
	714	case BPF_MAP_UPDATE_ELEM:
	715	err = map_update_elem(&attr);
	716	break;
	717	case BPF_MAP_DELETE_ELEM:
	718	err = map_delete_elem(&attr);
	719	break;
	720	case BPF_MAP_GET_NEXT_KEY:
	721	err = map_get_next_key(&attr);
	722	break;
09756af4 AS	723	case BPF_PROG_LOAD:
	724	err = bpf_prog_load(&attr);
	725	break;
99c55f7d AS	726	default:
	727	err = -EINVAL;
	728	break;
	729	}
	730
	731	return err;
	732	}