[mirror_iproute2.git] / tc / tc_bpf.c

/*
 * tc_bpf.c	BPF common code
 *
 *		This program is free software; you can distribute it and/or
 *		modify it under the terms of the GNU General Public License
 *		as published by the Free Software Foundation; either version
 *		2 of the License, or (at your option) any later version.
 *
 * Authors:	Daniel Borkmann <dborkman@redhat.com>
 *		Jiri Pirko <jiri@resnulli.us>
 *		Alexei Starovoitov <ast@plumgrid.com>
 */

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <stdbool.h>
#include <errno.h>
#include <fcntl.h>
#include <stdarg.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/un.h>
#include <linux/filter.h>
#include <linux/netlink.h>
#include <linux/rtnetlink.h>

#ifdef HAVE_ELF
#include <libelf.h>
#include <gelf.h>
#endif

#include "utils.h"

#include "bpf_elf.h"
#include "bpf_scm.h"

#include "tc_util.h"
#include "tc_bpf.h"

int bpf_parse_string(char *arg, bool from_file, __u16 *bpf_len,
		     char **bpf_string, bool *need_release,
		     const char separator)
{
	char sp;

	if (from_file) {
		size_t tmp_len, op_len = sizeof("65535 255 255 4294967295,");
		char *tmp_string;
		FILE *fp;

		tmp_len = sizeof("4096,") + BPF_MAXINSNS * op_len;
		tmp_string = malloc(tmp_len);
		if (tmp_string == NULL)
			return -ENOMEM;

		memset(tmp_string, 0, tmp_len);

		fp = fopen(arg, "r");
		if (fp == NULL) {
			perror("Cannot fopen");
			free(tmp_string);
			return -ENOENT;
		}

		if (!fgets(tmp_string, tmp_len, fp)) {
			free(tmp_string);
			fclose(fp);
			return -EIO;
		}

		fclose(fp);

		*need_release = true;
		*bpf_string = tmp_string;
	} else {
		*need_release = false;
		*bpf_string = arg;
	}

	if (sscanf(*bpf_string, "%hu%c", bpf_len, &sp) != 2 ||
	    sp != separator) {
		if (*need_release)
			free(*bpf_string);
		return -EINVAL;
	}

	return 0;
}

int bpf_parse_ops(int argc, char **argv, struct sock_filter *bpf_ops,
		  bool from_file)
{
	char *bpf_string, *token, separator = ',';
	int ret = 0, i = 0;
	bool need_release;
	__u16 bpf_len = 0;

	if (argc < 1)
		return -EINVAL;
	if (bpf_parse_string(argv[0], from_file, &bpf_len, &bpf_string,
			     &need_release, separator))
		return -EINVAL;
	if (bpf_len == 0 || bpf_len > BPF_MAXINSNS) {
		ret = -EINVAL;
		goto out;
	}

	token = bpf_string;
	while ((token = strchr(token, separator)) && (++token)[0]) {
		if (i >= bpf_len) {
			fprintf(stderr, "Real program length exceeds encoded "
				"length parameter!\n");
			ret = -EINVAL;
			goto out;
		}

		if (sscanf(token, "%hu %hhu %hhu %u,",
			   &bpf_ops[i].code, &bpf_ops[i].jt,
			   &bpf_ops[i].jf, &bpf_ops[i].k) != 4) {
			fprintf(stderr, "Error at instruction %d!\n", i);
			ret = -EINVAL;
			goto out;
		}

		i++;
	}

	if (i != bpf_len) {
		fprintf(stderr, "Parsed program length is less than encoded"
			"length parameter!\n");
		ret = -EINVAL;
		goto out;
	}
	ret = bpf_len;

out:
	if (need_release)
		free(bpf_string);

	return ret;
}

void bpf_print_ops(FILE *f, struct rtattr *bpf_ops, __u16 len)
{
	struct sock_filter *ops = (struct sock_filter *) RTA_DATA(bpf_ops);
	int i;

	if (len == 0)
		return;

	fprintf(f, "bytecode \'%u,", len);

	for (i = 0; i < len - 1; i++)
		fprintf(f, "%hu %hhu %hhu %u,", ops[i].code, ops[i].jt,
			ops[i].jf, ops[i].k);

	fprintf(f, "%hu %hhu %hhu %u\'", ops[i].code, ops[i].jt,
		ops[i].jf, ops[i].k);
}

const char *bpf_default_section(const enum bpf_prog_type type)
{
	switch (type) {
	case BPF_PROG_TYPE_SCHED_CLS:
		return ELF_SECTION_CLASSIFIER;
	case BPF_PROG_TYPE_SCHED_ACT:
		return ELF_SECTION_ACTION;
	default:
		return NULL;
	}
}

#ifdef HAVE_ELF
struct bpf_elf_sec_data {
	GElf_Shdr sec_hdr;
	char *sec_name;
	Elf_Data *sec_data;
};

struct bpf_map_data {
	int *fds;
	const char *obj;
	struct bpf_elf_st *st;
	struct bpf_elf_map *ent;
};

/* If we provide a small buffer with log level enabled, the kernel
 * could fail program load as no buffer space is available for the
 * log and thus verifier fails. In case something doesn't pass the
 * verifier we still want to hand something descriptive to the user.
 */
static char bpf_log_buf[65536];

static struct bpf_elf_st bpf_st;

static int map_fds[ELF_MAX_MAPS];
static struct bpf_elf_map map_ent[ELF_MAX_MAPS];

static void bpf_dump_error(const char *format, ...)  __check_format_string(1, 2);
static void bpf_dump_error(const char *format, ...)
{
	va_list vl;

	va_start(vl, format);
	vfprintf(stderr, format, vl);
	va_end(vl);

	fprintf(stderr, "%s\n", bpf_log_buf);
	memset(bpf_log_buf, 0, sizeof(bpf_log_buf));
}

static void bpf_save_finfo(int file_fd)
{
	struct stat st;
	int ret;

	memset(&bpf_st, 0, sizeof(bpf_st));

	ret = fstat(file_fd, &st);
	if (ret < 0) {
		fprintf(stderr, "Stat of elf file failed: %s\n",
			strerror(errno));
		return;
	}

	bpf_st.st_dev = st.st_dev;
	bpf_st.st_ino = st.st_ino;
}

static void bpf_clear_finfo(void)
{
	memset(&bpf_st, 0, sizeof(bpf_st));
}

static bool bpf_may_skip_map_creation(int file_fd)
{
	struct stat st;
	int ret;

	ret = fstat(file_fd, &st);
	if (ret < 0) {
		fprintf(stderr, "Stat of elf file failed: %s\n",
			strerror(errno));
		return false;
	}

	return (bpf_st.st_dev == st.st_dev) &&
	       (bpf_st.st_ino == st.st_ino);
}

static int bpf_create_map(enum bpf_map_type type, unsigned int size_key,
			  unsigned int size_value, unsigned int max_elem)
{
	union bpf_attr attr = {
		.map_type	= type,
		.key_size	= size_key,
		.value_size	= size_value,
		.max_entries	= max_elem,
	};

	return bpf(BPF_MAP_CREATE, &attr, sizeof(attr));
}

static int bpf_prog_load(enum bpf_prog_type type, const struct bpf_insn *insns,
			 unsigned int len, const char *license)
{
	union bpf_attr attr = {
		.prog_type	= type,
		.insns		= bpf_ptr_to_u64(insns),
		.insn_cnt	= len / sizeof(struct bpf_insn),
		.license	= bpf_ptr_to_u64(license),
		.log_buf	= bpf_ptr_to_u64(bpf_log_buf),
		.log_size	= sizeof(bpf_log_buf),
		.log_level	= 1,
	};

	return bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
}

static int bpf_prog_attach(enum bpf_prog_type type, const struct bpf_insn *insns,
			   unsigned int size, const char *license)
{
	int prog_fd = bpf_prog_load(type, insns, size, license);

	if (prog_fd < 0)
		bpf_dump_error("BPF program rejected: %s\n", strerror(errno));

	return prog_fd;
}

static int bpf_map_attach(enum bpf_map_type type, unsigned int size_key,
			  unsigned int size_value, unsigned int max_elem)
{
	int map_fd = bpf_create_map(type, size_key, size_value, max_elem);

	if (map_fd < 0)
		bpf_dump_error("BPF map rejected: %s\n", strerror(errno));

	return map_fd;
}

static void bpf_maps_init(void)
{
	int i;

	memset(map_ent, 0, sizeof(map_ent));
	for (i = 0; i < ARRAY_SIZE(map_fds); i++)
		map_fds[i] = -1;
}

static int bpf_maps_count(void)
{
	int i, count = 0;

	for (i = 0; i < ARRAY_SIZE(map_fds); i++) {
		if (map_fds[i] < 0)
			break;
		count++;
	}

	return count;
}

static void bpf_maps_destroy(void)
{
	int i;

	memset(map_ent, 0, sizeof(map_ent));
	for (i = 0; i < ARRAY_SIZE(map_fds); i++) {
		if (map_fds[i] >= 0)
			close(map_fds[i]);
	}
}

static int bpf_maps_attach(struct bpf_elf_map *maps, unsigned int num_maps)
{
	int i, ret;

	for (i = 0; (i < num_maps) && (num_maps <= ARRAY_SIZE(map_fds)); i++) {
		struct bpf_elf_map *map = &maps[i];

		ret = bpf_map_attach(map->type, map->size_key,
				     map->size_value, map->max_elem);
		if (ret < 0)
			goto err_unwind;

		map_fds[i] = ret;
	}

	return 0;

err_unwind:
	bpf_maps_destroy();
	return ret;
}

static int bpf_fill_section_data(Elf *elf_fd, GElf_Ehdr *elf_hdr, int sec_index,
				 struct bpf_elf_sec_data *sec_data)
{
	GElf_Shdr sec_hdr;
	Elf_Scn *sec_fd;
	Elf_Data *sec_edata;
	char *sec_name;

	memset(sec_data, 0, sizeof(*sec_data));

	sec_fd = elf_getscn(elf_fd, sec_index);
	if (!sec_fd)
		return -EINVAL;

	if (gelf_getshdr(sec_fd, &sec_hdr) != &sec_hdr)
		return -EIO;

	sec_name = elf_strptr(elf_fd, elf_hdr->e_shstrndx,
			      sec_hdr.sh_name);
	if (!sec_name || !sec_hdr.sh_size)
		return -ENOENT;

	sec_edata = elf_getdata(sec_fd, NULL);
	if (!sec_edata || elf_getdata(sec_fd, sec_edata))
		return -EIO;

	memcpy(&sec_data->sec_hdr, &sec_hdr, sizeof(sec_hdr));
	sec_data->sec_name = sec_name;
	sec_data->sec_data = sec_edata;

	return 0;
}

static int bpf_apply_relo_data(struct bpf_elf_sec_data *data_relo,
			       struct bpf_elf_sec_data *data_insn,
			       Elf_Data *sym_tab)
{
	Elf_Data *idata = data_insn->sec_data;
	GElf_Shdr *rhdr = &data_relo->sec_hdr;
	int relo_ent, relo_num = rhdr->sh_size / rhdr->sh_entsize;
	struct bpf_insn *insns = idata->d_buf;
	unsigned int num_insns = idata->d_size / sizeof(*insns);

	for (relo_ent = 0; relo_ent < relo_num; relo_ent++) {
		unsigned int ioff, fnum;
		GElf_Rel relo;
		GElf_Sym sym;

		if (gelf_getrel(data_relo->sec_data, relo_ent, &relo) != &relo)
			return -EIO;

		ioff = relo.r_offset / sizeof(struct bpf_insn);
		if (ioff >= num_insns)
			return -EINVAL;
		if (insns[ioff].code != (BPF_LD | BPF_IMM | BPF_DW))
			return -EINVAL;

		if (gelf_getsym(sym_tab, GELF_R_SYM(relo.r_info), &sym) != &sym)
			return -EIO;

		fnum = sym.st_value / sizeof(struct bpf_elf_map);
		if (fnum >= ARRAY_SIZE(map_fds))
			return -EINVAL;
		if (map_fds[fnum] < 0)
			return -EINVAL;

		insns[ioff].src_reg = BPF_PSEUDO_MAP_FD;
		insns[ioff].imm = map_fds[fnum];
	}

	return 0;
}

static int bpf_fetch_ancillary(int file_fd, Elf *elf_fd, GElf_Ehdr *elf_hdr,
			       bool *sec_seen, char *license, unsigned int lic_len,
			       Elf_Data **sym_tab)
{
	int sec_index, ret = -1;

	for (sec_index = 1; sec_index < elf_hdr->e_shnum; sec_index++) {
		struct bpf_elf_sec_data data_anc;

		ret = bpf_fill_section_data(elf_fd, elf_hdr, sec_index,
					    &data_anc);
		if (ret < 0)
			continue;

		/* Extract and load eBPF map fds. */
		if (!strcmp(data_anc.sec_name, ELF_SECTION_MAPS) &&
		    !bpf_may_skip_map_creation(file_fd)) {
			struct bpf_elf_map *maps;
			unsigned int maps_num;

			if (data_anc.sec_data->d_size % sizeof(*maps) != 0)
				return -EINVAL;

			maps = data_anc.sec_data->d_buf;
			maps_num = data_anc.sec_data->d_size / sizeof(*maps);
			memcpy(map_ent, maps, data_anc.sec_data->d_size);

			sec_seen[sec_index] = true;
			ret = bpf_maps_attach(maps, maps_num);
			if (ret < 0)
				return ret;
		}
		/* Extract eBPF license. */
		else if (!strcmp(data_anc.sec_name, ELF_SECTION_LICENSE)) {
			if (data_anc.sec_data->d_size > lic_len)
				return -ENOMEM;

			sec_seen[sec_index] = true;
			memcpy(license, data_anc.sec_data->d_buf,
			       data_anc.sec_data->d_size);
		}
		/* Extract symbol table for relocations (map fd fixups). */
		else if (data_anc.sec_hdr.sh_type == SHT_SYMTAB) {
			sec_seen[sec_index] = true;
			*sym_tab = data_anc.sec_data;
		}
	}

	return ret;
}

static int bpf_fetch_prog_relo(Elf *elf_fd, GElf_Ehdr *elf_hdr, bool *sec_seen,
			       enum bpf_prog_type type, const char *sec,
			       const char *license, Elf_Data *sym_tab)
{
	int sec_index, prog_fd = -1;

	for (sec_index = 1; sec_index < elf_hdr->e_shnum; sec_index++) {
		struct bpf_elf_sec_data data_relo, data_insn;
		int ins_index, ret;

		/* Attach eBPF programs with relocation data (maps). */
		ret = bpf_fill_section_data(elf_fd, elf_hdr, sec_index,
					    &data_relo);
		if (ret < 0 || data_relo.sec_hdr.sh_type != SHT_REL)
			continue;

		ins_index = data_relo.sec_hdr.sh_info;

		ret = bpf_fill_section_data(elf_fd, elf_hdr, ins_index,
					    &data_insn);
		if (ret < 0)
			continue;
		if (strcmp(data_insn.sec_name, sec))
			continue;

		sec_seen[sec_index] = true;
		sec_seen[ins_index] = true;

		ret = bpf_apply_relo_data(&data_relo, &data_insn, sym_tab);
		if (ret < 0)
			continue;

		prog_fd = bpf_prog_attach(type, data_insn.sec_data->d_buf,
					  data_insn.sec_data->d_size, license);
		if (prog_fd < 0)
			continue;

		break;
	}

	return prog_fd;
}

static int bpf_fetch_prog(Elf *elf_fd, GElf_Ehdr *elf_hdr, bool *sec_seen,
			  enum bpf_prog_type type, const char *sec,
			  const char *license)
{
	int sec_index, prog_fd = -1;

	for (sec_index = 1; sec_index < elf_hdr->e_shnum; sec_index++) {
		struct bpf_elf_sec_data data_insn;
		int ret;

		/* Attach eBPF programs without relocation data. */
		if (sec_seen[sec_index])
			continue;

		ret = bpf_fill_section_data(elf_fd, elf_hdr, sec_index,
					    &data_insn);
		if (ret < 0)
			continue;
		if (strcmp(data_insn.sec_name, sec))
			continue;

		prog_fd = bpf_prog_attach(type, data_insn.sec_data->d_buf,
					  data_insn.sec_data->d_size, license);
		if (prog_fd < 0)
			continue;

		break;
	}

	return prog_fd;
}

int bpf_open_object(const char *path, enum bpf_prog_type type, const char *sec)
{
	char license[ELF_MAX_LICENSE_LEN];
	int file_fd, prog_fd = -1, ret;
	Elf_Data *sym_tab = NULL;
	GElf_Ehdr elf_hdr;
	bool *sec_seen;
	Elf *elf_fd;

	if (elf_version(EV_CURRENT) == EV_NONE)
		return -EINVAL;

	file_fd = open(path, O_RDONLY, 0);
	if (file_fd < 0)
		return -errno;

	elf_fd = elf_begin(file_fd, ELF_C_READ, NULL);
	if (!elf_fd) {
		ret = -EINVAL;
		goto out;
	}

	if (gelf_getehdr(elf_fd, &elf_hdr) != &elf_hdr) {
		ret = -EIO;
		goto out_elf;
	}

	sec_seen = calloc(elf_hdr.e_shnum, sizeof(*sec_seen));
	if (!sec_seen) {
		ret = -ENOMEM;
		goto out_elf;
	}

	memset(license, 0, sizeof(license));
	if (!bpf_may_skip_map_creation(file_fd))
		bpf_maps_init();

	ret = bpf_fetch_ancillary(file_fd, elf_fd, &elf_hdr, sec_seen,
				  license, sizeof(license), &sym_tab);
	if (ret < 0)
		goto out_maps;
	if (sym_tab)
		prog_fd = bpf_fetch_prog_relo(elf_fd, &elf_hdr, sec_seen, type,
					      sec, license, sym_tab);
	if (prog_fd < 0)
		prog_fd = bpf_fetch_prog(elf_fd, &elf_hdr, sec_seen, type, sec,
					 license);
	if (prog_fd < 0)
		goto out_maps;

	bpf_save_finfo(file_fd);

	free(sec_seen);

	elf_end(elf_fd);
	close(file_fd);

	return prog_fd;

out_maps:
	bpf_maps_destroy();
	free(sec_seen);
out_elf:
	elf_end(elf_fd);
out:
	close(file_fd);
	bpf_clear_finfo();
	return prog_fd;
}

static int
bpf_map_set_xmit(int fd, struct sockaddr_un *addr, unsigned int addr_len,
		 const struct bpf_map_data *aux, unsigned int ents)
{
	struct bpf_map_set_msg msg;
	int *cmsg_buf, min_fd;
	char *amsg_buf;
	int i;

	memset(&msg, 0, sizeof(msg));

	msg.aux.uds_ver = BPF_SCM_AUX_VER;
	msg.aux.num_ent = ents;

	strncpy(msg.aux.obj_name, aux->obj, sizeof(msg.aux.obj_name));
	memcpy(&msg.aux.obj_st, aux->st, sizeof(msg.aux.obj_st));

	cmsg_buf = bpf_map_set_init(&msg, addr, addr_len);
	amsg_buf = (char *)msg.aux.ent;

	for (i = 0; i < ents; i += min_fd) {
		int ret;

		min_fd = min(BPF_SCM_MAX_FDS * 1U, ents - i);

		bpf_map_set_init_single(&msg, min_fd);

		memcpy(cmsg_buf, &aux->fds[i], sizeof(aux->fds[0]) * min_fd);
		memcpy(amsg_buf, &aux->ent[i], sizeof(aux->ent[0]) * min_fd);

		ret = sendmsg(fd, &msg.hdr, 0);
		if (ret <= 0)
			return ret ? : -1;
	}

	return 0;
}

int bpf_handoff_map_fds(const char *path, const char *obj)
{
	struct sockaddr_un addr;
	struct bpf_map_data bpf_aux;
	int fd, ret;

	fd = socket(AF_UNIX, SOCK_DGRAM, 0);
	if (fd < 0) {
		fprintf(stderr, "Cannot open socket: %s\n",
			strerror(errno));
		return -1;
	}

	memset(&addr, 0, sizeof(addr));
	addr.sun_family = AF_UNIX;
	strncpy(addr.sun_path, path, sizeof(addr.sun_path));

	ret = connect(fd, (struct sockaddr *)&addr, sizeof(addr));
	if (ret < 0) {
		fprintf(stderr, "Cannot connect to %s: %s\n",
			path, strerror(errno));
		return -1;
	}

	memset(&bpf_aux, 0, sizeof(bpf_aux));

	bpf_aux.fds = map_fds;
	bpf_aux.ent = map_ent;

	bpf_aux.obj = obj;
	bpf_aux.st = &bpf_st;

	ret = bpf_map_set_xmit(fd, &addr, sizeof(addr), &bpf_aux,
			       bpf_maps_count());
	if (ret < 0)
		fprintf(stderr, "Cannot xmit fds to %s: %s\n",
			path, strerror(errno));

	close(fd);
	return ret;
}
#endif /* HAVE_ELF */
Commit	Line	Data
1d129d19 JP	1	/*
	2	* tc_bpf.c BPF common code
	3	*
	4	* This program is free software; you can distribute it and/or
	5	* modify it under the terms of the GNU General Public License
	6	* as published by the Free Software Foundation; either version
	7	* 2 of the License, or (at your option) any later version.
	8	*
	9	* Authors: Daniel Borkmann <dborkman@redhat.com>
	10	* Jiri Pirko <jiri@resnulli.us>
11c39b5e	11	* Alexei Starovoitov <ast@plumgrid.com>
1d129d19 JP	12	*/
	13
	14	#include <stdio.h>
	15	#include <stdlib.h>
	16	#include <unistd.h>
	17	#include <string.h>
	18	#include <stdbool.h>
	19	#include <errno.h>
11c39b5e DB	20	#include <fcntl.h>
	21	#include <stdarg.h>
	22	#include <sys/types.h>
	23	#include <sys/stat.h>
6256f8c9	24	#include <sys/un.h>
1d129d19 JP	25	#include <linux/filter.h>
	26	#include <linux/netlink.h>
	27	#include <linux/rtnetlink.h>
	28
11c39b5e DB	29	#ifdef HAVE_ELF
	30	#include <libelf.h>
	31	#include <gelf.h>
	32	#endif
	33
1d129d19	34	#include "utils.h"
6256f8c9 DB	35
	36	#include "bpf_elf.h"
	37	#include "bpf_scm.h"
	38
1d129d19 JP	39	#include "tc_util.h"
	40	#include "tc_bpf.h"
	41
	42	int bpf_parse_string(char arg, bool from_file, __u16 bpf_len,
	43	char *bpf_string, bool need_release,
	44	const char separator)
	45	{
	46	char sp;
	47
	48	if (from_file) {
	49	size_t tmp_len, op_len = sizeof("65535 255 255 4294967295,");
	50	char *tmp_string;
	51	FILE *fp;
	52
	53	tmp_len = sizeof("4096,") + BPF_MAXINSNS * op_len;
	54	tmp_string = malloc(tmp_len);
	55	if (tmp_string == NULL)
	56	return -ENOMEM;
	57
	58	memset(tmp_string, 0, tmp_len);
	59
	60	fp = fopen(arg, "r");
	61	if (fp == NULL) {
	62	perror("Cannot fopen");
	63	free(tmp_string);
	64	return -ENOENT;
	65	}
	66
	67	if (!fgets(tmp_string, tmp_len, fp)) {
	68	free(tmp_string);
	69	fclose(fp);
	70	return -EIO;
	71	}
	72
	73	fclose(fp);
	74
	75	*need_release = true;
	76	*bpf_string = tmp_string;
	77	} else {
	78	*need_release = false;
	79	*bpf_string = arg;
	80	}
	81
	82	if (sscanf(*bpf_string, "%hu%c", bpf_len, &sp) != 2 \|\|
	83	sp != separator) {
	84	if (*need_release)
	85	free(*bpf_string);
	86	return -EINVAL;
	87	}
	88
	89	return 0;
	90	}
	91
	92	int bpf_parse_ops(int argc, char *argv, struct sock_filter bpf_ops,
	93	bool from_file)
	94	{
	95	char bpf_string, token, separator = ',';
	96	int ret = 0, i = 0;
	97	bool need_release;
	98	__u16 bpf_len = 0;
	99
	100	if (argc < 1)
	101	return -EINVAL;
	102	if (bpf_parse_string(argv[0], from_file, &bpf_len, &bpf_string,
103	&need_release, separator))
104	return -EINVAL;
105	if (bpf_len == 0 \|\| bpf_len > BPF_MAXINSNS) {
106	ret = -EINVAL;
107	goto out;
108	}
109
110	token = bpf_string;
111	while ((token = strchr(token, separator)) && (++token)[0]) {
112	if (i >= bpf_len) {
113	fprintf(stderr, "Real program length exceeds encoded "
114	"length parameter!\n");
115	ret = -EINVAL;
116	goto out;
117	}
118
119	if (sscanf(token, "%hu %hhu %hhu %u,",
120	&bpf_ops[i].code, &bpf_ops[i].jt,
121	&bpf_ops[i].jf, &bpf_ops[i].k) != 4) {
122	fprintf(stderr, "Error at instruction %d!\n", i);
123	ret = -EINVAL;
124	goto out;
125	}
126
127	i++;
128	}
129
130	if (i != bpf_len) {
131	fprintf(stderr, "Parsed program length is less than encoded"
132	"length parameter!\n");
133	ret = -EINVAL;
134	goto out;
135	}
136	ret = bpf_len;
137
138	out:
139	if (need_release)
140	free(bpf_string);
141
142	return ret;
143	}
144
145	void bpf_print_ops(FILE f, struct rtattr bpf_ops, __u16 len)
146	{
147	struct sock_filter ops = (struct sock_filter ) RTA_DATA(bpf_ops);
148	int i;
149
150	if (len == 0)
151	return;
152
153	fprintf(f, "bytecode \'%u,", len);
154
155	for (i = 0; i < len - 1; i++)
156	fprintf(f, "%hu %hhu %hhu %u,", ops[i].code, ops[i].jt,
157	ops[i].jf, ops[i].k);
158
6256f8c9	159	fprintf(f, "%hu %hhu %hhu %u\'", ops[i].code, ops[i].jt,
1d129d19 JP	160	ops[i].jf, ops[i].k);
1d129d19 JP	161	}
11c39b5e	162
6256f8c9	163	const char *bpf_default_section(const enum bpf_prog_type type)
11c39b5e DB	164	{
	165	switch (type) {
	166	case BPF_PROG_TYPE_SCHED_CLS:
	167	return ELF_SECTION_CLASSIFIER;
6256f8c9 DB	168	case BPF_PROG_TYPE_SCHED_ACT:
6256f8c9 DB	169	return ELF_SECTION_ACTION;
11c39b5e DB	170	default:
	171	return NULL;
	172	}
	173	}
	174
6256f8c9 DB	175	#ifdef HAVE_ELF
	176	struct bpf_elf_sec_data {
	177	GElf_Shdr sec_hdr;
	178	char *sec_name;
	179	Elf_Data *sec_data;
	180	};
	181
	182	struct bpf_map_data {
	183	int *fds;
	184	const char *obj;
	185	struct bpf_elf_st *st;
	186	struct bpf_elf_map *ent;
	187	};
	188
	189	/* If we provide a small buffer with log level enabled, the kernel
	190	* could fail program load as no buffer space is available for the
	191	* log and thus verifier fails. In case something doesn't pass the
	192	* verifier we still want to hand something descriptive to the user.
	193	*/
	194	static char bpf_log_buf[65536];
	195
	196	static struct bpf_elf_st bpf_st;
	197
	198	static int map_fds[ELF_MAX_MAPS];
	199	static struct bpf_elf_map map_ent[ELF_MAX_MAPS];
	200
11c39b5e DB	201	static void bpf_dump_error(const char *format, ...) __check_format_string(1, 2);
	202	static void bpf_dump_error(const char *format, ...)
	203	{
	204	va_list vl;
	205
	206	va_start(vl, format);
	207	vfprintf(stderr, format, vl);
	208	va_end(vl);
	209
6256f8c9	210	fprintf(stderr, "%s\n", bpf_log_buf);
11c39b5e DB	211	memset(bpf_log_buf, 0, sizeof(bpf_log_buf));
	212	}
	213
6256f8c9 DB	214	static void bpf_save_finfo(int file_fd)
	215	{
	216	struct stat st;
	217	int ret;
	218
	219	memset(&bpf_st, 0, sizeof(bpf_st));
	220
	221	ret = fstat(file_fd, &st);
	222	if (ret < 0) {
	223	fprintf(stderr, "Stat of elf file failed: %s\n",
	224	strerror(errno));
	225	return;
	226	}
	227
	228	bpf_st.st_dev = st.st_dev;
	229	bpf_st.st_ino = st.st_ino;
	230	}
	231
	232	static void bpf_clear_finfo(void)
	233	{
	234	memset(&bpf_st, 0, sizeof(bpf_st));
	235	}
	236
	237	static bool bpf_may_skip_map_creation(int file_fd)
	238	{
	239	struct stat st;
	240	int ret;
	241
	242	ret = fstat(file_fd, &st);
	243	if (ret < 0) {
	244	fprintf(stderr, "Stat of elf file failed: %s\n",
	245	strerror(errno));
	246	return false;
	247	}
	248
	249	return (bpf_st.st_dev == st.st_dev) &&
	250	(bpf_st.st_ino == st.st_ino);
	251	}
	252
11c39b5e DB	253	static int bpf_create_map(enum bpf_map_type type, unsigned int size_key,
	254	unsigned int size_value, unsigned int max_elem)
	255	{
	256	union bpf_attr attr = {
	257	.map_type = type,
	258	.key_size = size_key,
	259	.value_size = size_value,
	260	.max_entries = max_elem,
	261	};
	262
	263	return bpf(BPF_MAP_CREATE, &attr, sizeof(attr));
	264	}
	265
	266	static int bpf_prog_load(enum bpf_prog_type type, const struct bpf_insn *insns,
	267	unsigned int len, const char *license)
	268	{
	269	union bpf_attr attr = {
	270	.prog_type = type,
	271	.insns = bpf_ptr_to_u64(insns),
	272	.insn_cnt = len / sizeof(struct bpf_insn),
	273	.license = bpf_ptr_to_u64(license),
	274	.log_buf = bpf_ptr_to_u64(bpf_log_buf),
	275	.log_size = sizeof(bpf_log_buf),
	276	.log_level = 1,
	277	};
	278
	279	return bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
	280	}
	281
	282	static int bpf_prog_attach(enum bpf_prog_type type, const struct bpf_insn *insns,
	283	unsigned int size, const char *license)
	284	{
	285	int prog_fd = bpf_prog_load(type, insns, size, license);
	286
	287	if (prog_fd < 0)
	288	bpf_dump_error("BPF program rejected: %s\n", strerror(errno));
	289
	290	return prog_fd;
	291	}
	292
	293	static int bpf_map_attach(enum bpf_map_type type, unsigned int size_key,
	294	unsigned int size_value, unsigned int max_elem)
	295	{
	296	int map_fd = bpf_create_map(type, size_key, size_value, max_elem);
	297
	298	if (map_fd < 0)
	299	bpf_dump_error("BPF map rejected: %s\n", strerror(errno));
	300
	301	return map_fd;
	302	}
	303
6256f8c9	304	static void bpf_maps_init(void)
11c39b5e DB	305	{
	306	int i;
	307
6256f8c9 DB	308	memset(map_ent, 0, sizeof(map_ent));
6256f8c9 DB	309	for (i = 0; i < ARRAY_SIZE(map_fds); i++)
11c39b5e DB	310	map_fds[i] = -1;
	311	}
	312
6256f8c9 DB	313	static int bpf_maps_count(void)
	314	{
	315	int i, count = 0;
	316
	317	for (i = 0; i < ARRAY_SIZE(map_fds); i++) {
	318	if (map_fds[i] < 0)
	319	break;
	320	count++;
	321	}
	322
	323	return count;
	324	}
	325
	326	static void bpf_maps_destroy(void)
11c39b5e DB	327	{
	328	int i;
	329
6256f8c9 DB	330	memset(map_ent, 0, sizeof(map_ent));
6256f8c9 DB	331	for (i = 0; i < ARRAY_SIZE(map_fds); i++) {
11c39b5e DB	332	if (map_fds[i] >= 0)
	333	close(map_fds[i]);
	334	}
	335	}
	336
6256f8c9	337	static int bpf_maps_attach(struct bpf_elf_map *maps, unsigned int num_maps)
11c39b5e DB	338	{
	339	int i, ret;
	340
6256f8c9	341	for (i = 0; (i < num_maps) && (num_maps <= ARRAY_SIZE(map_fds)); i++) {
11c39b5e DB	342	struct bpf_elf_map *map = &maps[i];
	343
	344	ret = bpf_map_attach(map->type, map->size_key,
	345	map->size_value, map->max_elem);
	346	if (ret < 0)
	347	goto err_unwind;
	348
	349	map_fds[i] = ret;
	350	}
	351
	352	return 0;
	353
	354	err_unwind:
6256f8c9	355	bpf_maps_destroy();
11c39b5e DB	356	return ret;
	357	}
	358
	359	static int bpf_fill_section_data(Elf elf_fd, GElf_Ehdr elf_hdr, int sec_index,
	360	struct bpf_elf_sec_data *sec_data)
	361	{
	362	GElf_Shdr sec_hdr;
	363	Elf_Scn *sec_fd;
	364	Elf_Data *sec_edata;
	365	char *sec_name;
	366
	367	memset(sec_data, 0, sizeof(*sec_data));
	368
	369	sec_fd = elf_getscn(elf_fd, sec_index);
	370	if (!sec_fd)
	371	return -EINVAL;
	372
	373	if (gelf_getshdr(sec_fd, &sec_hdr) != &sec_hdr)
	374	return -EIO;
	375
	376	sec_name = elf_strptr(elf_fd, elf_hdr->e_shstrndx,
	377	sec_hdr.sh_name);
	378	if (!sec_name \|\| !sec_hdr.sh_size)
	379	return -ENOENT;
	380
	381	sec_edata = elf_getdata(sec_fd, NULL);
	382	if (!sec_edata \|\| elf_getdata(sec_fd, sec_edata))
	383	return -EIO;
	384
	385	memcpy(&sec_data->sec_hdr, &sec_hdr, sizeof(sec_hdr));
	386	sec_data->sec_name = sec_name;
	387	sec_data->sec_data = sec_edata;
	388
	389	return 0;
	390	}
	391
	392	static int bpf_apply_relo_data(struct bpf_elf_sec_data *data_relo,
	393	struct bpf_elf_sec_data *data_insn,
6256f8c9	394	Elf_Data *sym_tab)
11c39b5e DB	395	{
	396	Elf_Data *idata = data_insn->sec_data;
	397	GElf_Shdr *rhdr = &data_relo->sec_hdr;
	398	int relo_ent, relo_num = rhdr->sh_size / rhdr->sh_entsize;
	399	struct bpf_insn *insns = idata->d_buf;
	400	unsigned int num_insns = idata->d_size / sizeof(*insns);
	401
	402	for (relo_ent = 0; relo_ent < relo_num; relo_ent++) {
	403	unsigned int ioff, fnum;
	404	GElf_Rel relo;
	405	GElf_Sym sym;
	406
	407	if (gelf_getrel(data_relo->sec_data, relo_ent, &relo) != &relo)
	408	return -EIO;
	409
	410	ioff = relo.r_offset / sizeof(struct bpf_insn);
	411	if (ioff >= num_insns)
	412	return -EINVAL;
	413	if (insns[ioff].code != (BPF_LD \| BPF_IMM \| BPF_DW))
	414	return -EINVAL;
	415
	416	if (gelf_getsym(sym_tab, GELF_R_SYM(relo.r_info), &sym) != &sym)
	417	return -EIO;
	418
	419	fnum = sym.st_value / sizeof(struct bpf_elf_map);
6256f8c9 DB	420	if (fnum >= ARRAY_SIZE(map_fds))
	421	return -EINVAL;
	422	if (map_fds[fnum] < 0)
11c39b5e DB	423	return -EINVAL;
	424
	425	insns[ioff].src_reg = BPF_PSEUDO_MAP_FD;
	426	insns[ioff].imm = map_fds[fnum];
	427	}
	428
	429	return 0;
	430	}
	431
6256f8c9 DB	432	static int bpf_fetch_ancillary(int file_fd, Elf elf_fd, GElf_Ehdr elf_hdr,
6256f8c9 DB	433	bool sec_seen, char license, unsigned int lic_len,
11c39b5e DB	434	Elf_Data **sym_tab)
	435	{
	436	int sec_index, ret = -1;
	437
	438	for (sec_index = 1; sec_index < elf_hdr->e_shnum; sec_index++) {
	439	struct bpf_elf_sec_data data_anc;
	440
	441	ret = bpf_fill_section_data(elf_fd, elf_hdr, sec_index,
	442	&data_anc);
	443	if (ret < 0)
	444	continue;
	445
	446	/* Extract and load eBPF map fds. */
6256f8c9 DB	447	if (!strcmp(data_anc.sec_name, ELF_SECTION_MAPS) &&
	448	!bpf_may_skip_map_creation(file_fd)) {
	449	struct bpf_elf_map *maps;
	450	unsigned int maps_num;
	451
	452	if (data_anc.sec_data->d_size % sizeof(*maps) != 0)
	453	return -EINVAL;
	454
	455	maps = data_anc.sec_data->d_buf;
	456	maps_num = data_anc.sec_data->d_size / sizeof(*maps);
	457	memcpy(map_ent, maps, data_anc.sec_data->d_size);
11c39b5e DB	458
11c39b5e DB	459	sec_seen[sec_index] = true;
6256f8c9	460	ret = bpf_maps_attach(maps, maps_num);
11c39b5e DB	461	if (ret < 0)
	462	return ret;
	463	}
	464	/* Extract eBPF license. */
	465	else if (!strcmp(data_anc.sec_name, ELF_SECTION_LICENSE)) {
	466	if (data_anc.sec_data->d_size > lic_len)
	467	return -ENOMEM;
	468
	469	sec_seen[sec_index] = true;
	470	memcpy(license, data_anc.sec_data->d_buf,
	471	data_anc.sec_data->d_size);
	472	}
	473	/* Extract symbol table for relocations (map fd fixups). */
	474	else if (data_anc.sec_hdr.sh_type == SHT_SYMTAB) {
	475	sec_seen[sec_index] = true;
	476	*sym_tab = data_anc.sec_data;
	477	}
	478	}
	479
	480	return ret;
	481	}
	482
	483	static int bpf_fetch_prog_relo(Elf elf_fd, GElf_Ehdr elf_hdr, bool *sec_seen,
6256f8c9 DB	484	enum bpf_prog_type type, const char *sec,
6256f8c9 DB	485	const char license, Elf_Data sym_tab)
11c39b5e DB	486	{
	487	int sec_index, prog_fd = -1;
	488
	489	for (sec_index = 1; sec_index < elf_hdr->e_shnum; sec_index++) {
	490	struct bpf_elf_sec_data data_relo, data_insn;
	491	int ins_index, ret;
	492
	493	/* Attach eBPF programs with relocation data (maps). */
	494	ret = bpf_fill_section_data(elf_fd, elf_hdr, sec_index,
	495	&data_relo);
	496	if (ret < 0 \|\| data_relo.sec_hdr.sh_type != SHT_REL)
	497	continue;
	498
	499	ins_index = data_relo.sec_hdr.sh_info;
	500
	501	ret = bpf_fill_section_data(elf_fd, elf_hdr, ins_index,
	502	&data_insn);
	503	if (ret < 0)
	504	continue;
6256f8c9	505	if (strcmp(data_insn.sec_name, sec))
11c39b5e DB	506	continue;
	507
	508	sec_seen[sec_index] = true;
	509	sec_seen[ins_index] = true;
	510
6256f8c9	511	ret = bpf_apply_relo_data(&data_relo, &data_insn, sym_tab);
11c39b5e DB	512	if (ret < 0)
	513	continue;
	514
	515	prog_fd = bpf_prog_attach(type, data_insn.sec_data->d_buf,
	516	data_insn.sec_data->d_size, license);
	517	if (prog_fd < 0)
	518	continue;
	519
	520	break;
	521	}
	522
	523	return prog_fd;
	524	}
	525
	526	static int bpf_fetch_prog(Elf elf_fd, GElf_Ehdr elf_hdr, bool *sec_seen,
6256f8c9 DB	527	enum bpf_prog_type type, const char *sec,
6256f8c9 DB	528	const char *license)
11c39b5e DB	529	{
	530	int sec_index, prog_fd = -1;
	531
	532	for (sec_index = 1; sec_index < elf_hdr->e_shnum; sec_index++) {
	533	struct bpf_elf_sec_data data_insn;
	534	int ret;
	535
	536	/* Attach eBPF programs without relocation data. */
	537	if (sec_seen[sec_index])
	538	continue;
	539
	540	ret = bpf_fill_section_data(elf_fd, elf_hdr, sec_index,
	541	&data_insn);
	542	if (ret < 0)
	543	continue;
6256f8c9	544	if (strcmp(data_insn.sec_name, sec))
11c39b5e DB	545	continue;
	546
	547	prog_fd = bpf_prog_attach(type, data_insn.sec_data->d_buf,
	548	data_insn.sec_data->d_size, license);
	549	if (prog_fd < 0)
	550	continue;
	551
	552	break;
	553	}
	554
	555	return prog_fd;
	556	}
	557
6256f8c9	558	int bpf_open_object(const char path, enum bpf_prog_type type, const char sec)
11c39b5e	559	{
11c39b5e DB	560	char license[ELF_MAX_LICENSE_LEN];
	561	int file_fd, prog_fd = -1, ret;
	562	Elf_Data *sym_tab = NULL;
	563	GElf_Ehdr elf_hdr;
	564	bool *sec_seen;
	565	Elf *elf_fd;
	566
	567	if (elf_version(EV_CURRENT) == EV_NONE)
	568	return -EINVAL;
	569
	570	file_fd = open(path, O_RDONLY, 0);
	571	if (file_fd < 0)
	572	return -errno;
	573
	574	elf_fd = elf_begin(file_fd, ELF_C_READ, NULL);
	575	if (!elf_fd) {
	576	ret = -EINVAL;
	577	goto out;
	578	}
	579
	580	if (gelf_getehdr(elf_fd, &elf_hdr) != &elf_hdr) {
	581	ret = -EIO;
	582	goto out_elf;
	583	}
	584
	585	sec_seen = calloc(elf_hdr.e_shnum, sizeof(*sec_seen));
	586	if (!sec_seen) {
	587	ret = -ENOMEM;
	588	goto out_elf;
	589	}
	590
	591	memset(license, 0, sizeof(license));
6256f8c9 DB	592	if (!bpf_may_skip_map_creation(file_fd))
6256f8c9 DB	593	bpf_maps_init();
11c39b5e	594
6256f8c9	595	ret = bpf_fetch_ancillary(file_fd, elf_fd, &elf_hdr, sec_seen,
11c39b5e DB	596	license, sizeof(license), &sym_tab);
	597	if (ret < 0)
	598	goto out_maps;
	599	if (sym_tab)
	600	prog_fd = bpf_fetch_prog_relo(elf_fd, &elf_hdr, sec_seen, type,
6256f8c9	601	sec, license, sym_tab);
11c39b5e	602	if (prog_fd < 0)
6256f8c9	603	prog_fd = bpf_fetch_prog(elf_fd, &elf_hdr, sec_seen, type, sec,
11c39b5e DB	604	license);
	605	if (prog_fd < 0)
	606	goto out_maps;
6256f8c9 DB	607
	608	bpf_save_finfo(file_fd);
	609
	610	free(sec_seen);
	611
	612	elf_end(elf_fd);
	613	close(file_fd);
	614
	615	return prog_fd;
	616
	617	out_maps:
	618	bpf_maps_destroy();
11c39b5e DB	619	free(sec_seen);
	620	out_elf:
	621	elf_end(elf_fd);
	622	out:
	623	close(file_fd);
6256f8c9	624	bpf_clear_finfo();
11c39b5e	625	return prog_fd;
6256f8c9	626	}
11c39b5e	627
6256f8c9 DB	628	static int
	629	bpf_map_set_xmit(int fd, struct sockaddr_un *addr, unsigned int addr_len,
	630	const struct bpf_map_data *aux, unsigned int ents)
	631	{
	632	struct bpf_map_set_msg msg;
	633	int *cmsg_buf, min_fd;
	634	char *amsg_buf;
	635	int i;
	636
	637	memset(&msg, 0, sizeof(msg));
	638
	639	msg.aux.uds_ver = BPF_SCM_AUX_VER;
	640	msg.aux.num_ent = ents;
	641
	642	strncpy(msg.aux.obj_name, aux->obj, sizeof(msg.aux.obj_name));
	643	memcpy(&msg.aux.obj_st, aux->st, sizeof(msg.aux.obj_st));
	644
	645	cmsg_buf = bpf_map_set_init(&msg, addr, addr_len);
	646	amsg_buf = (char *)msg.aux.ent;
	647
	648	for (i = 0; i < ents; i += min_fd) {
	649	int ret;
	650
	651	min_fd = min(BPF_SCM_MAX_FDS * 1U, ents - i);
	652
	653	bpf_map_set_init_single(&msg, min_fd);
	654
	655	memcpy(cmsg_buf, &aux->fds[i], sizeof(aux->fds[0]) * min_fd);
	656	memcpy(amsg_buf, &aux->ent[i], sizeof(aux->ent[0]) * min_fd);
	657
	658	ret = sendmsg(fd, &msg.hdr, 0);
	659	if (ret <= 0)
	660	return ret ? : -1;
	661	}
	662
	663	return 0;
11c39b5e DB	664	}
11c39b5e DB	665
6256f8c9 DB	666	int bpf_handoff_map_fds(const char path, const char obj)
	667	{
	668	struct sockaddr_un addr;
	669	struct bpf_map_data bpf_aux;
	670	int fd, ret;
	671
	672	fd = socket(AF_UNIX, SOCK_DGRAM, 0);
	673	if (fd < 0) {
	674	fprintf(stderr, "Cannot open socket: %s\n",
	675	strerror(errno));
	676	return -1;
	677	}
	678
	679	memset(&addr, 0, sizeof(addr));
	680	addr.sun_family = AF_UNIX;
	681	strncpy(addr.sun_path, path, sizeof(addr.sun_path));
	682
	683	ret = connect(fd, (struct sockaddr *)&addr, sizeof(addr));
	684	if (ret < 0) {
	685	fprintf(stderr, "Cannot connect to %s: %s\n",
	686	path, strerror(errno));
	687	return -1;
	688	}
	689
	690	memset(&bpf_aux, 0, sizeof(bpf_aux));
	691
	692	bpf_aux.fds = map_fds;
	693	bpf_aux.ent = map_ent;
	694
	695	bpf_aux.obj = obj;
	696	bpf_aux.st = &bpf_st;
	697
	698	ret = bpf_map_set_xmit(fd, &addr, sizeof(addr), &bpf_aux,
	699	bpf_maps_count());
	700	if (ret < 0)
	701	fprintf(stderr, "Cannot xmit fds to %s: %s\n",
	702	path, strerror(errno));
	703
	704	close(fd);
	705	return ret;
	706	}
11c39b5e	707	#endif /* HAVE_ELF */