[mirror_ubuntu-jammy-kernel.git] / samples / bpf / xdp_redirect_cpu_user.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2017 Jesper Dangaard Brouer, Red Hat, Inc.
 */
static const char *__doc__ =
"XDP CPU redirect tool, using BPF_MAP_TYPE_CPUMAP\n"
"Usage: xdp_redirect_cpu -d <IFINDEX|IFNAME> -c 0 ... -c N\n"
"Valid specification for CPUMAP BPF program:\n"
"  --mprog-name/-e pass (use built-in XDP_PASS program)\n"
"  --mprog-name/-e drop (use built-in XDP_DROP program)\n"
"  --redirect-device/-r <ifindex|ifname> (use built-in DEVMAP redirect program)\n"
"  Custom CPUMAP BPF program:\n"
"    --mprog-filename/-f <filename> --mprog-name/-e <program>\n"
"    Optionally, also pass --redirect-map/-m and --redirect-device/-r together\n"
"    to configure DEVMAP in BPF object <filename>\n";

#include <errno.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <string.h>
#include <unistd.h>
#include <locale.h>
#include <sys/resource.h>
#include <sys/sysinfo.h>
#include <getopt.h>
#include <net/if.h>
#include <time.h>
#include <linux/limits.h>
#include <arpa/inet.h>
#include <linux/if_link.h>
#include <bpf/bpf.h>
#include <bpf/libbpf.h>
#include "bpf_util.h"
#include "xdp_sample_user.h"
#include "xdp_redirect_cpu.skel.h"

static int map_fd;
static int avail_fd;
static int count_fd;

static int mask = SAMPLE_RX_CNT | SAMPLE_REDIRECT_ERR_MAP_CNT |
		  SAMPLE_CPUMAP_ENQUEUE_CNT | SAMPLE_CPUMAP_KTHREAD_CNT |
		  SAMPLE_EXCEPTION_CNT;

DEFINE_SAMPLE_INIT(xdp_redirect_cpu);

static const struct option long_options[] = {
	{ "help", no_argument, NULL, 'h' },
	{ "dev", required_argument, NULL, 'd' },
	{ "skb-mode", no_argument, NULL, 'S' },
	{ "progname", required_argument, NULL, 'p' },
	{ "qsize", required_argument, NULL, 'q' },
	{ "cpu", required_argument, NULL, 'c' },
	{ "stress-mode", no_argument, NULL, 'x' },
	{ "force", no_argument, NULL, 'F' },
	{ "interval", required_argument, NULL, 'i' },
	{ "verbose", no_argument, NULL, 'v' },
	{ "stats", no_argument, NULL, 's' },
	{ "mprog-name", required_argument, NULL, 'e' },
	{ "mprog-filename", required_argument, NULL, 'f' },
	{ "redirect-device", required_argument, NULL, 'r' },
	{ "redirect-map", required_argument, NULL, 'm' },
	{}
};

static void print_avail_progs(struct bpf_object *obj)
{
	struct bpf_program *pos;

	printf(" Programs to be used for -p/--progname:\n");
	bpf_object__for_each_program(pos, obj) {
		if (bpf_program__is_xdp(pos)) {
			if (!strncmp(bpf_program__name(pos), "xdp_prognum",
				     sizeof("xdp_prognum") - 1))
				printf(" %s\n", bpf_program__name(pos));
		}
	}
}

static void usage(char *argv[], const struct option *long_options,
		  const char *doc, int mask, bool error, struct bpf_object *obj)
{
	sample_usage(argv, long_options, doc, mask, error);
	print_avail_progs(obj);
}

static int create_cpu_entry(__u32 cpu, struct bpf_cpumap_val *value,
			    __u32 avail_idx, bool new)
{
	__u32 curr_cpus_count = 0;
	__u32 key = 0;
	int ret;

	/* Add a CPU entry to cpumap, as this allocate a cpu entry in
	 * the kernel for the cpu.
	 */
	ret = bpf_map_update_elem(map_fd, &cpu, value, 0);
	if (ret < 0) {
		fprintf(stderr, "Create CPU entry failed: %s\n", strerror(errno));
		return ret;
	}

	/* Inform bpf_prog's that a new CPU is available to select
	 * from via some control maps.
	 */
	ret = bpf_map_update_elem(avail_fd, &avail_idx, &cpu, 0);
	if (ret < 0) {
		fprintf(stderr, "Add to avail CPUs failed: %s\n", strerror(errno));
		return ret;
	}

	/* When not replacing/updating existing entry, bump the count */
	ret = bpf_map_lookup_elem(count_fd, &key, &curr_cpus_count);
	if (ret < 0) {
		fprintf(stderr, "Failed reading curr cpus_count: %s\n",
			strerror(errno));
		return ret;
	}
	if (new) {
		curr_cpus_count++;
		ret = bpf_map_update_elem(count_fd, &key,
					  &curr_cpus_count, 0);
		if (ret < 0) {
			fprintf(stderr, "Failed write curr cpus_count: %s\n",
				strerror(errno));
			return ret;
		}
	}

	printf("%s CPU: %u as idx: %u qsize: %d cpumap_prog_fd: %d (cpus_count: %u)\n",
	       new ? "Add new" : "Replace", cpu, avail_idx,
	       value->qsize, value->bpf_prog.fd, curr_cpus_count);

	return 0;
}

/* CPUs are zero-indexed. Thus, add a special sentinel default value
 * in map cpus_available to mark CPU index'es not configured
 */
static int mark_cpus_unavailable(void)
{
	int ret, i, n_cpus = libbpf_num_possible_cpus();
	__u32 invalid_cpu = n_cpus;

	for (i = 0; i < n_cpus; i++) {
		ret = bpf_map_update_elem(avail_fd, &i,
					  &invalid_cpu, 0);
		if (ret < 0) {
			fprintf(stderr, "Failed marking CPU unavailable: %s\n",
				strerror(errno));
			return ret;
		}
	}

	return 0;
}

/* Stress cpumap management code by concurrently changing underlying cpumap */
static void stress_cpumap(void *ctx)
{
	struct bpf_cpumap_val *value = ctx;

	/* Changing qsize will cause kernel to free and alloc a new
	 * bpf_cpu_map_entry, with an associated/complicated tear-down
	 * procedure.
	 */
	value->qsize = 1024;
	create_cpu_entry(1, value, 0, false);
	value->qsize = 8;
	create_cpu_entry(1, value, 0, false);
	value->qsize = 16000;
	create_cpu_entry(1, value, 0, false);
}

static int set_cpumap_prog(struct xdp_redirect_cpu *skel,
			   const char *redir_interface, const char *redir_map,
			   const char *mprog_filename, const char *mprog_name)
{
	if (mprog_filename) {
		struct bpf_program *prog;
		struct bpf_object *obj;
		int ret;

		if (!mprog_name) {
			fprintf(stderr, "BPF program not specified for file %s\n",
				mprog_filename);
			goto end;
		}
		if ((redir_interface && !redir_map) || (!redir_interface && redir_map)) {
			fprintf(stderr, "--redirect-%s specified but --redirect-%s not specified\n",
				redir_interface ? "device" : "map", redir_interface ? "map" : "device");
			goto end;
		}

		/* Custom BPF program */
		obj = bpf_object__open_file(mprog_filename, NULL);
		if (!obj) {
			ret = -errno;
			fprintf(stderr, "Failed to bpf_prog_load_xattr: %s\n",
				strerror(errno));
			return ret;
		}

		ret = bpf_object__load(obj);
		if (ret < 0) {
			ret = -errno;
			fprintf(stderr, "Failed to bpf_object__load: %s\n",
				strerror(errno));
			return ret;
		}

		if (redir_map) {
			int err, redir_map_fd, ifindex_out, key = 0;

			redir_map_fd = bpf_object__find_map_fd_by_name(obj, redir_map);
			if (redir_map_fd < 0) {
				fprintf(stderr, "Failed to bpf_object__find_map_fd_by_name: %s\n",
					strerror(errno));
				return redir_map_fd;
			}

			ifindex_out = if_nametoindex(redir_interface);
			if (!ifindex_out)
				ifindex_out = strtoul(redir_interface, NULL, 0);
			if (!ifindex_out) {
				fprintf(stderr, "Bad interface name or index\n");
				return -EINVAL;
			}

			err = bpf_map_update_elem(redir_map_fd, &key, &ifindex_out, 0);
			if (err < 0)
				return err;
		}

		prog = bpf_object__find_program_by_name(obj, mprog_name);
		if (!prog) {
			ret = -errno;
			fprintf(stderr, "Failed to bpf_object__find_program_by_name: %s\n",
				strerror(errno));
			return ret;
		}

		return bpf_program__fd(prog);
	} else {
		if (mprog_name) {
			if (redir_interface || redir_map) {
				fprintf(stderr, "Need to specify --mprog-filename/-f\n");
				goto end;
			}
			if (!strcmp(mprog_name, "pass") || !strcmp(mprog_name, "drop")) {
				/* Use built-in pass/drop programs */
				return *mprog_name == 'p' ? bpf_program__fd(skel->progs.xdp_redirect_cpu_pass)
					: bpf_program__fd(skel->progs.xdp_redirect_cpu_drop);
			} else {
				fprintf(stderr, "Unknown name \"%s\" for built-in BPF program\n",
					mprog_name);
				goto end;
			}
		} else {
			if (redir_map) {
				fprintf(stderr, "Need to specify --mprog-filename, --mprog-name and"
					" --redirect-device with --redirect-map\n");
				goto end;
			}
			if (redir_interface) {
				/* Use built-in devmap redirect */
				struct bpf_devmap_val val = {};
				int ifindex_out, err;
				__u32 key = 0;

				if (!redir_interface)
					return 0;

				ifindex_out = if_nametoindex(redir_interface);
				if (!ifindex_out)
					ifindex_out = strtoul(redir_interface, NULL, 0);
				if (!ifindex_out) {
					fprintf(stderr, "Bad interface name or index\n");
					return -EINVAL;
				}

				if (get_mac_addr(ifindex_out, skel->bss->tx_mac_addr) < 0) {
					printf("Get interface %d mac failed\n", ifindex_out);
					return -EINVAL;
				}

				val.ifindex = ifindex_out;
				val.bpf_prog.fd = bpf_program__fd(skel->progs.xdp_redirect_egress_prog);
				err = bpf_map_update_elem(bpf_map__fd(skel->maps.tx_port), &key, &val, 0);
				if (err < 0)
					return -errno;

				return bpf_program__fd(skel->progs.xdp_redirect_cpu_devmap);
			}
		}
	}

	/* Disabled */
	return 0;
end:
	fprintf(stderr, "Invalid options for CPUMAP BPF program\n");
	return -EINVAL;
}

int main(int argc, char **argv)
{
	const char *redir_interface = NULL, *redir_map = NULL;
	const char *mprog_filename = NULL, *mprog_name = NULL;
	struct xdp_redirect_cpu *skel;
	struct bpf_map_info info = {};
	char ifname_buf[IF_NAMESIZE];
	struct bpf_cpumap_val value;
	__u32 infosz = sizeof(info);
	int ret = EXIT_FAIL_OPTION;
	unsigned long interval = 2;
	bool stress_mode = false;
	struct bpf_program *prog;
	const char *prog_name;
	bool generic = false;
	bool force = false;
	int added_cpus = 0;
	bool error = true;
	int longindex = 0;
	int add_cpu = -1;
	int ifindex = -1;
	int *cpu, i, opt;
	char *ifname;
	__u32 qsize;
	int n_cpus;

	n_cpus = libbpf_num_possible_cpus();

	/* Notice: Choosing the queue size is very important when CPU is
	 * configured with power-saving states.
	 *
	 * If deepest state take 133 usec to wakeup from (133/10^6). When link
	 * speed is 10Gbit/s ((10*10^9/8) in bytes/sec). How many bytes can
	 * arrive with in 133 usec at this speed: (10*10^9/8)*(133/10^6) =
	 * 166250 bytes. With MTU size packets this is 110 packets, and with
	 * minimum Ethernet (MAC-preamble + intergap) 84 bytes is 1979 packets.
	 *
	 * Setting default cpumap queue to 2048 as worst-case (small packet)
	 * should be +64 packet due kthread wakeup call (due to xdp_do_flush)
	 * worst-case is 2043 packets.
	 *
	 * Sysadm can configured system to avoid deep-sleep via:
	 *   tuned-adm profile network-latency
	 */
	qsize = 2048;

	skel = xdp_redirect_cpu__open();
	if (!skel) {
		fprintf(stderr, "Failed to xdp_redirect_cpu__open: %s\n",
			strerror(errno));
		ret = EXIT_FAIL_BPF;
		goto end;
	}

	ret = sample_init_pre_load(skel);
	if (ret < 0) {
		fprintf(stderr, "Failed to sample_init_pre_load: %s\n", strerror(-ret));
		ret = EXIT_FAIL_BPF;
		goto end_destroy;
	}

	if (bpf_map__set_max_entries(skel->maps.cpu_map, n_cpus) < 0) {
		fprintf(stderr, "Failed to set max entries for cpu_map map: %s",
			strerror(errno));
		ret = EXIT_FAIL_BPF;
		goto end_destroy;
	}

	if (bpf_map__set_max_entries(skel->maps.cpus_available, n_cpus) < 0) {
		fprintf(stderr, "Failed to set max entries for cpus_available map: %s",
			strerror(errno));
		ret = EXIT_FAIL_BPF;
		goto end_destroy;
	}

	cpu = calloc(n_cpus, sizeof(int));
	if (!cpu) {
		fprintf(stderr, "Failed to allocate cpu array\n");
		goto end_destroy;
	}

	prog = skel->progs.xdp_prognum5_lb_hash_ip_pairs;
	while ((opt = getopt_long(argc, argv, "d:si:Sxp:f:e:r:m:c:q:Fvh",
				  long_options, &longindex)) != -1) {
		switch (opt) {
		case 'd':
			if (strlen(optarg) >= IF_NAMESIZE) {
				fprintf(stderr, "-d/--dev name too long\n");
				goto end_cpu;
			}
			ifname = (char *)&ifname_buf;
			safe_strncpy(ifname, optarg, sizeof(ifname));
			ifindex = if_nametoindex(ifname);
			if (!ifindex)
				ifindex = strtoul(optarg, NULL, 0);
			if (!ifindex) {
				fprintf(stderr, "Bad interface index or name (%d): %s\n",
					errno, strerror(errno));
				usage(argv, long_options, __doc__, mask, true, skel->obj);
				goto end_cpu;
			}
			break;
		case 's':
			mask |= SAMPLE_REDIRECT_MAP_CNT;
			break;
		case 'i':
			interval = strtoul(optarg, NULL, 0);
			break;
		case 'S':
			generic = true;
			break;
		case 'x':
			stress_mode = true;
			break;
		case 'p':
			/* Selecting eBPF prog to load */
			prog_name = optarg;
			prog = bpf_object__find_program_by_name(skel->obj,
								prog_name);
			if (!prog) {
				fprintf(stderr,
					"Failed to find program %s specified by"
					" option -p/--progname\n",
					prog_name);
				print_avail_progs(skel->obj);
				goto end_cpu;
			}
			break;
		case 'f':
			mprog_filename = optarg;
			break;
		case 'e':
			mprog_name = optarg;
			break;
		case 'r':
			redir_interface = optarg;
			mask |= SAMPLE_DEVMAP_XMIT_CNT_MULTI;
			break;
		case 'm':
			redir_map = optarg;
			break;
		case 'c':
			/* Add multiple CPUs */
			add_cpu = strtoul(optarg, NULL, 0);
			if (add_cpu >= n_cpus) {
				fprintf(stderr,
				"--cpu nr too large for cpumap err (%d):%s\n",
					errno, strerror(errno));
				usage(argv, long_options, __doc__, mask, true, skel->obj);
				goto end_cpu;
			}
			cpu[added_cpus++] = add_cpu;
			break;
		case 'q':
			qsize = strtoul(optarg, NULL, 0);
			break;
		case 'F':
			force = true;
			break;
		case 'v':
			sample_switch_mode();
			break;
		case 'h':
			error = false;
		default:
			usage(argv, long_options, __doc__, mask, error, skel->obj);
			goto end_cpu;
		}
	}

	ret = EXIT_FAIL_OPTION;
	if (ifindex == -1) {
		fprintf(stderr, "Required option --dev missing\n");
		usage(argv, long_options, __doc__, mask, true, skel->obj);
		goto end_cpu;
	}

	if (add_cpu == -1) {
		fprintf(stderr, "Required option --cpu missing\n"
				"Specify multiple --cpu option to add more\n");
		usage(argv, long_options, __doc__, mask, true, skel->obj);
		goto end_cpu;
	}

	skel->rodata->from_match[0] = ifindex;
	if (redir_interface)
		skel->rodata->to_match[0] = if_nametoindex(redir_interface);

	ret = xdp_redirect_cpu__load(skel);
	if (ret < 0) {
		fprintf(stderr, "Failed to xdp_redirect_cpu__load: %s\n",
			strerror(errno));
		goto end_cpu;
	}

	ret = bpf_obj_get_info_by_fd(bpf_map__fd(skel->maps.cpu_map), &info, &infosz);
	if (ret < 0) {
		fprintf(stderr, "Failed bpf_obj_get_info_by_fd for cpumap: %s\n",
			strerror(errno));
		goto end_cpu;
	}

	skel->bss->cpumap_map_id = info.id;

	map_fd = bpf_map__fd(skel->maps.cpu_map);
	avail_fd = bpf_map__fd(skel->maps.cpus_available);
	count_fd = bpf_map__fd(skel->maps.cpus_count);

	ret = mark_cpus_unavailable();
	if (ret < 0) {
		fprintf(stderr, "Unable to mark CPUs as unavailable\n");
		goto end_cpu;
	}

	ret = sample_init(skel, mask);
	if (ret < 0) {
		fprintf(stderr, "Failed to initialize sample: %s\n", strerror(-ret));
		ret = EXIT_FAIL;
		goto end_cpu;
	}

	value.bpf_prog.fd = set_cpumap_prog(skel, redir_interface, redir_map,
					    mprog_filename, mprog_name);
	if (value.bpf_prog.fd < 0) {
		fprintf(stderr, "Failed to set CPUMAP BPF program: %s\n",
			strerror(-value.bpf_prog.fd));
		usage(argv, long_options, __doc__, mask, true, skel->obj);
		ret = EXIT_FAIL_BPF;
		goto end_cpu;
	}
	value.qsize = qsize;

	for (i = 0; i < added_cpus; i++) {
		if (create_cpu_entry(cpu[i], &value, i, true) < 0) {
			fprintf(stderr, "Cannot proceed, exiting\n");
			usage(argv, long_options, __doc__, mask, true, skel->obj);
			goto end_cpu;
		}
	}

	ret = EXIT_FAIL_XDP;
	if (sample_install_xdp(prog, ifindex, generic, force) < 0)
		goto end_cpu;

	ret = sample_run(interval, stress_mode ? stress_cpumap : NULL, &value);
	if (ret < 0) {
		fprintf(stderr, "Failed during sample run: %s\n", strerror(-ret));
		ret = EXIT_FAIL;
		goto end_cpu;
	}
	ret = EXIT_OK;
end_cpu:
	free(cpu);
end_destroy:
	xdp_redirect_cpu__destroy(skel);
end:
	sample_exit(ret);
}
Commit	Line	Data
7f904d7e TG	1	// SPDX-License-Identifier: GPL-2.0-only
7f904d7e TG	2	/* Copyright(c) 2017 Jesper Dangaard Brouer, Red Hat, Inc.
fad3917e JDB	3	*/
fad3917e JDB	4	static const char *__doc__ =
e531a220 KKD	5	"XDP CPU redirect tool, using BPF_MAP_TYPE_CPUMAP\n"
	6	"Usage: xdp_redirect_cpu -d <IFINDEX\|IFNAME> -c 0 ... -c N\n"
	7	"Valid specification for CPUMAP BPF program:\n"
	8	" --mprog-name/-e pass (use built-in XDP_PASS program)\n"
	9	" --mprog-name/-e drop (use built-in XDP_DROP program)\n"
	10	" --redirect-device/-r <ifindex\|ifname> (use built-in DEVMAP redirect program)\n"
	11	" Custom CPUMAP BPF program:\n"
	12	" --mprog-filename/-f <filename> --mprog-name/-e <program>\n"
	13	" Optionally, also pass --redirect-map/-m and --redirect-device/-r together\n"
	14	" to configure DEVMAP in BPF object <filename>\n";
fad3917e JDB	15
	16	#include <errno.h>
	17	#include <signal.h>
	18	#include <stdio.h>
	19	#include <stdlib.h>
	20	#include <stdbool.h>
	21	#include <string.h>
	22	#include <unistd.h>
	23	#include <locale.h>
	24	#include <sys/resource.h>
6a098154	25	#include <sys/sysinfo.h>
fad3917e JDB	26	#include <getopt.h>
	27	#include <net/if.h>
	28	#include <time.h>
f9e6bfdb	29	#include <linux/limits.h>
fad3917e JDB	30	#include <arpa/inet.h>
fad3917e JDB	31	#include <linux/if_link.h>
2bf3e2ef	32	#include <bpf/bpf.h>
7cf245a3	33	#include <bpf/libbpf.h>
fad3917e	34	#include "bpf_util.h"
e531a220 KKD	35	#include "xdp_sample_user.h"
e531a220 KKD	36	#include "xdp_redirect_cpu.skel.h"
fad3917e	37
e531a220 KKD	38	static int map_fd;
	39	static int avail_fd;
	40	static int count_fd;
151936bf	41
e531a220 KKD	42	static int mask = SAMPLE_RX_CNT \| SAMPLE_REDIRECT_ERR_MAP_CNT \|
	43	SAMPLE_CPUMAP_ENQUEUE_CNT \| SAMPLE_CPUMAP_KTHREAD_CNT \|
	44	SAMPLE_EXCEPTION_CNT;
fad3917e	45
e531a220	46	DEFINE_SAMPLE_INIT(xdp_redirect_cpu);
fad3917e JDB	47
fad3917e JDB	48	static const struct option long_options[] = {
e531a220 KKD	49	{ "help", no_argument, NULL, 'h' },
	50	{ "dev", required_argument, NULL, 'd' },
	51	{ "skb-mode", no_argument, NULL, 'S' },
	52	{ "progname", required_argument, NULL, 'p' },
	53	{ "qsize", required_argument, NULL, 'q' },
	54	{ "cpu", required_argument, NULL, 'c' },
	55	{ "stress-mode", no_argument, NULL, 'x' },
	56	{ "force", no_argument, NULL, 'F' },
	57	{ "interval", required_argument, NULL, 'i' },
	58	{ "verbose", no_argument, NULL, 'v' },
	59	{ "stats", no_argument, NULL, 's' },
	60	{ "mprog-name", required_argument, NULL, 'e' },
	61	{ "mprog-filename", required_argument, NULL, 'f' },
	62	{ "redirect-device", required_argument, NULL, 'r' },
	63	{ "redirect-map", required_argument, NULL, 'm' },
	64	{}
fad3917e JDB	65	};
fad3917e JDB	66
bbaf6029 MF	67	static void print_avail_progs(struct bpf_object *obj)
	68	{
	69	struct bpf_program *pos;
	70
e531a220	71	printf(" Programs to be used for -p/--progname:\n");
bbaf6029	72	bpf_object__for_each_program(pos, obj) {
e531a220 KKD	73	if (bpf_program__is_xdp(pos)) {
	74	if (!strncmp(bpf_program__name(pos), "xdp_prognum",
	75	sizeof("xdp_prognum") - 1))
	76	printf(" %s\n", bpf_program__name(pos));
fad3917e	77	}
fad3917e	78	}
fad3917e JDB	79	}
fad3917e JDB	80
e531a220 KKD	81	static void usage(char argv[], const struct option long_options,
e531a220 KKD	82	const char doc, int mask, bool error, struct bpf_object obj)
fad3917e	83	{
e531a220 KKD	84	sample_usage(argv, long_options, doc, mask, error);
e531a220 KKD	85	print_avail_progs(obj);
fad3917e JDB	86	}
fad3917e JDB	87
ce4dade7	88	static int create_cpu_entry(__u32 cpu, struct bpf_cpumap_val *value,
fad3917e JDB	89	__u32 avail_idx, bool new)
	90	{
	91	__u32 curr_cpus_count = 0;
	92	__u32 key = 0;
	93	int ret;
	94
	95	/* Add a CPU entry to cpumap, as this allocate a cpu entry in
	96	* the kernel for the cpu.
	97	*/
e531a220 KKD	98	ret = bpf_map_update_elem(map_fd, &cpu, value, 0);
	99	if (ret < 0) {
	100	fprintf(stderr, "Create CPU entry failed: %s\n", strerror(errno));
	101	return ret;
fad3917e JDB	102	}
	103
	104	/* Inform bpf_prog's that a new CPU is available to select
	105	* from via some control maps.
	106	*/
e531a220 KKD	107	ret = bpf_map_update_elem(avail_fd, &avail_idx, &cpu, 0);
	108	if (ret < 0) {
	109	fprintf(stderr, "Add to avail CPUs failed: %s\n", strerror(errno));
	110	return ret;
fad3917e JDB	111	}
	112
	113	/* When not replacing/updating existing entry, bump the count */
e531a220 KKD	114	ret = bpf_map_lookup_elem(count_fd, &key, &curr_cpus_count);
	115	if (ret < 0) {
	116	fprintf(stderr, "Failed reading curr cpus_count: %s\n",
	117	strerror(errno));
	118	return ret;
fad3917e JDB	119	}
	120	if (new) {
	121	curr_cpus_count++;
e531a220	122	ret = bpf_map_update_elem(count_fd, &key,
bbaf6029	123	&curr_cpus_count, 0);
e531a220 KKD	124	if (ret < 0) {
	125	fprintf(stderr, "Failed write curr cpus_count: %s\n",
	126	strerror(errno));
	127	return ret;
fad3917e JDB	128	}
fad3917e JDB	129	}
e531a220 KKD	130
	131	printf("%s CPU: %u as idx: %u qsize: %d cpumap_prog_fd: %d (cpus_count: %u)\n",
	132	new ? "Add new" : "Replace", cpu, avail_idx,
ce4dade7	133	value->qsize, value->bpf_prog.fd, curr_cpus_count);
fad3917e JDB	134
	135	return 0;
	136	}
	137
	138	/* CPUs are zero-indexed. Thus, add a special sentinel default value
	139	* in map cpus_available to mark CPU index'es not configured
	140	*/
e531a220	141	static int mark_cpus_unavailable(void)
fad3917e	142	{
e531a220	143	int ret, i, n_cpus = libbpf_num_possible_cpus();
48b2e71c	144	__u32 invalid_cpu = n_cpus;
fad3917e	145
6a098154	146	for (i = 0; i < n_cpus; i++) {
e531a220	147	ret = bpf_map_update_elem(avail_fd, &i,
bbaf6029	148	&invalid_cpu, 0);
e531a220 KKD	149	if (ret < 0) {
	150	fprintf(stderr, "Failed marking CPU unavailable: %s\n",
	151	strerror(errno));
	152	return ret;
fad3917e JDB	153	}
fad3917e JDB	154	}
e531a220 KKD	155
e531a220 KKD	156	return 0;
fad3917e JDB	157	}
	158
	159	/* Stress cpumap management code by concurrently changing underlying cpumap */
e531a220	160	static void stress_cpumap(void *ctx)
fad3917e	161	{
e531a220 KKD	162	struct bpf_cpumap_val *value = ctx;
e531a220 KKD	163
fad3917e JDB	164	/* Changing qsize will cause kernel to free and alloc a new
	165	* bpf_cpu_map_entry, with an associated/complicated tear-down
	166	* procedure.
	167	*/
ce4dade7 LB	168	value->qsize = 1024;
	169	create_cpu_entry(1, value, 0, false);
	170	value->qsize = 8;
	171	create_cpu_entry(1, value, 0, false);
	172	value->qsize = 16000;
	173	create_cpu_entry(1, value, 0, false);
fad3917e JDB	174	}
fad3917e JDB	175
e531a220 KKD	176	static int set_cpumap_prog(struct xdp_redirect_cpu *skel,
	177	const char redir_interface, const char redir_map,
	178	const char mprog_filename, const char mprog_name)
f9e6bfdb	179	{
e531a220 KKD	180	if (mprog_filename) {
	181	struct bpf_program *prog;
	182	struct bpf_object *obj;
	183	int ret;
	184
	185	if (!mprog_name) {
	186	fprintf(stderr, "BPF program not specified for file %s\n",
	187	mprog_filename);
	188	goto end;
	189	}
	190	if ((redir_interface && !redir_map) \|\| (!redir_interface && redir_map)) {
	191	fprintf(stderr, "--redirect-%s specified but --redirect-%s not specified\n",
	192	redir_interface ? "device" : "map", redir_interface ? "map" : "device");
	193	goto end;
151936bf	194	}
bbaf6029	195
e531a220 KKD	196	/* Custom BPF program */
	197	obj = bpf_object__open_file(mprog_filename, NULL);
	198	if (!obj) {
	199	ret = -errno;
	200	fprintf(stderr, "Failed to bpf_prog_load_xattr: %s\n",
	201	strerror(errno));
	202	return ret;
	203	}
ce4dade7	204
e531a220 KKD	205	ret = bpf_object__load(obj);
	206	if (ret < 0) {
	207	ret = -errno;
	208	fprintf(stderr, "Failed to bpf_object__load: %s\n",
	209	strerror(errno));
	210	return ret;
	211	}
ce4dade7	212
e531a220 KKD	213	if (redir_map) {
e531a220 KKD	214	int err, redir_map_fd, ifindex_out, key = 0;
ce4dade7	215
e531a220 KKD	216	redir_map_fd = bpf_object__find_map_fd_by_name(obj, redir_map);
	217	if (redir_map_fd < 0) {
	218	fprintf(stderr, "Failed to bpf_object__find_map_fd_by_name: %s\n",
	219	strerror(errno));
	220	return redir_map_fd;
	221	}
ce4dade7	222
e531a220 KKD	223	ifindex_out = if_nametoindex(redir_interface);
	224	if (!ifindex_out)
	225	ifindex_out = strtoul(redir_interface, NULL, 0);
	226	if (!ifindex_out) {
	227	fprintf(stderr, "Bad interface name or index\n");
	228	return -EINVAL;
	229	}
ce4dade7	230
e531a220 KKD	231	err = bpf_map_update_elem(redir_map_fd, &key, &ifindex_out, 0);
	232	if (err < 0)
	233	return err;
	234	}
ce4dade7	235
e531a220 KKD	236	prog = bpf_object__find_program_by_name(obj, mprog_name);
	237	if (!prog) {
	238	ret = -errno;
	239	fprintf(stderr, "Failed to bpf_object__find_program_by_name: %s\n",
	240	strerror(errno));
	241	return ret;
	242	}
ce4dade7	243
e531a220 KKD	244	return bpf_program__fd(prog);
	245	} else {
	246	if (mprog_name) {
	247	if (redir_interface \|\| redir_map) {
	248	fprintf(stderr, "Need to specify --mprog-filename/-f\n");
	249	goto end;
	250	}
	251	if (!strcmp(mprog_name, "pass") \|\| !strcmp(mprog_name, "drop")) {
	252	/* Use built-in pass/drop programs */
	253	return *mprog_name == 'p' ? bpf_program__fd(skel->progs.xdp_redirect_cpu_pass)
	254	: bpf_program__fd(skel->progs.xdp_redirect_cpu_drop);
	255	} else {
	256	fprintf(stderr, "Unknown name \"%s\" for built-in BPF program\n",
	257	mprog_name);
	258	goto end;
	259	}
	260	} else {
	261	if (redir_map) {
	262	fprintf(stderr, "Need to specify --mprog-filename, --mprog-name and"
	263	" --redirect-device with --redirect-map\n");
	264	goto end;
	265	}
	266	if (redir_interface) {
	267	/* Use built-in devmap redirect */
	268	struct bpf_devmap_val val = {};
	269	int ifindex_out, err;
	270	__u32 key = 0;
	271
	272	if (!redir_interface)
	273	return 0;
	274
	275	ifindex_out = if_nametoindex(redir_interface);
	276	if (!ifindex_out)
	277	ifindex_out = strtoul(redir_interface, NULL, 0);
	278	if (!ifindex_out) {
	279	fprintf(stderr, "Bad interface name or index\n");
	280	return -EINVAL;
	281	}
	282
	283	if (get_mac_addr(ifindex_out, skel->bss->tx_mac_addr) < 0) {
	284	printf("Get interface %d mac failed\n", ifindex_out);
	285	return -EINVAL;
	286	}
	287
	288	val.ifindex = ifindex_out;
	289	val.bpf_prog.fd = bpf_program__fd(skel->progs.xdp_redirect_egress_prog);
	290	err = bpf_map_update_elem(bpf_map__fd(skel->maps.tx_port), &key, &val, 0);
	291	if (err < 0)
	292	return -errno;
	293
	294	return bpf_program__fd(skel->progs.xdp_redirect_cpu_devmap);
	295	}
	296	}
ce4dade7 LB	297	}
ce4dade7 LB	298
e531a220 KKD	299	/* Disabled */
	300	return 0;
	301	end:
	302	fprintf(stderr, "Invalid options for CPUMAP BPF program\n");
	303	return -EINVAL;
ce4dade7 LB	304	}
ce4dade7 LB	305
fad3917e JDB	306	int main(int argc, char **argv)
fad3917e JDB	307	{
e531a220 KKD	308	const char redir_interface = NULL, redir_map = NULL;
	309	const char mprog_filename = NULL, mprog_name = NULL;
	310	struct xdp_redirect_cpu *skel;
	311	struct bpf_map_info info = {};
	312	char ifname_buf[IF_NAMESIZE];
ce4dade7	313	struct bpf_cpumap_val value;
e531a220 KKD	314	__u32 infosz = sizeof(info);
	315	int ret = EXIT_FAIL_OPTION;
	316	unsigned long interval = 2;
fad3917e	317	bool stress_mode = false;
bbaf6029	318	struct bpf_program *prog;
e531a220 KKD	319	const char *prog_name;
	320	bool generic = false;
	321	bool force = false;
fad3917e	322	int added_cpus = 0;
e531a220	323	bool error = true;
fad3917e	324	int longindex = 0;
fad3917e	325	int add_cpu = -1;
e531a220 KKD	326	int ifindex = -1;
	327	int *cpu, i, opt;
	328	char *ifname;
fad3917e	329	__u32 qsize;
e531a220	330	int n_cpus;
fad3917e	331
e531a220	332	n_cpus = libbpf_num_possible_cpus();
6a098154	333
eff94154 JDB	334	/* Notice: Choosing the queue size is very important when CPU is
	335	* configured with power-saving states.
	336	*
	337	* If deepest state take 133 usec to wakeup from (133/10^6). When link
	338	* speed is 10Gbit/s ((10*10^9/8) in bytes/sec). How many bytes can
	339	* arrive with in 133 usec at this speed: (1010^9/8)(133/10^6) =
	340	* 166250 bytes. With MTU size packets this is 110 packets, and with
	341	* minimum Ethernet (MAC-preamble + intergap) 84 bytes is 1979 packets.
	342	*
	343	* Setting default cpumap queue to 2048 as worst-case (small packet)
	344	* should be +64 packet due kthread wakeup call (due to xdp_do_flush)
	345	* worst-case is 2043 packets.
	346	*
	347	* Sysadm can configured system to avoid deep-sleep via:
	348	* tuned-adm profile network-latency
fad3917e	349	*/
eff94154	350	qsize = 2048;
fad3917e	351
e531a220 KKD	352	skel = xdp_redirect_cpu__open();
	353	if (!skel) {
	354	fprintf(stderr, "Failed to xdp_redirect_cpu__open: %s\n",
bbaf6029	355	strerror(errno));
e531a220 KKD	356	ret = EXIT_FAIL_BPF;
	357	goto end;
	358	}
	359
	360	ret = sample_init_pre_load(skel);
	361	if (ret < 0) {
	362	fprintf(stderr, "Failed to sample_init_pre_load: %s\n", strerror(-ret));
	363	ret = EXIT_FAIL_BPF;
	364	goto end_destroy;
bbaf6029	365	}
151936bf	366
e531a220 KKD	367	if (bpf_map__set_max_entries(skel->maps.cpu_map, n_cpus) < 0) {
	368	fprintf(stderr, "Failed to set max entries for cpu_map map: %s",
	369	strerror(errno));
	370	ret = EXIT_FAIL_BPF;
	371	goto end_destroy;
151936bf DL	372	}
151936bf DL	373
e531a220 KKD	374	if (bpf_map__set_max_entries(skel->maps.cpus_available, n_cpus) < 0) {
	375	fprintf(stderr, "Failed to set max entries for cpus_available map: %s",
	376	strerror(errno));
	377	ret = EXIT_FAIL_BPF;
	378	goto end_destroy;
fad3917e	379	}
fad3917e	380
e531a220	381	cpu = calloc(n_cpus, sizeof(int));
a4e76f1b	382	if (!cpu) {
e531a220 KKD	383	fprintf(stderr, "Failed to allocate cpu array\n");
e531a220 KKD	384	goto end_destroy;
a4e76f1b	385	}
a4e76f1b	386
e531a220 KKD	387	prog = skel->progs.xdp_prognum5_lb_hash_ip_pairs;
e531a220 KKD	388	while ((opt = getopt_long(argc, argv, "d:si:Sxp:f:e:r:m:c:q:Fvh",
fad3917e JDB	389	long_options, &longindex)) != -1) {
	390	switch (opt) {
	391	case 'd':
	392	if (strlen(optarg) >= IF_NAMESIZE) {
e531a220 KKD	393	fprintf(stderr, "-d/--dev name too long\n");
e531a220 KKD	394	goto end_cpu;
fad3917e JDB	395	}
fad3917e JDB	396	ifname = (char *)&ifname_buf;
e531a220	397	safe_strncpy(ifname, optarg, sizeof(ifname));
fad3917e	398	ifindex = if_nametoindex(ifname);
e531a220 KKD	399	if (!ifindex)
	400	ifindex = strtoul(optarg, NULL, 0);
	401	if (!ifindex) {
	402	fprintf(stderr, "Bad interface index or name (%d): %s\n",
fad3917e	403	errno, strerror(errno));
e531a220 KKD	404	usage(argv, long_options, __doc__, mask, true, skel->obj);
e531a220 KKD	405	goto end_cpu;
fad3917e JDB	406	}
	407	break;
	408	case 's':
e531a220 KKD	409	mask \|= SAMPLE_REDIRECT_MAP_CNT;
	410	break;
	411	case 'i':
	412	interval = strtoul(optarg, NULL, 0);
fad3917e JDB	413	break;
fad3917e JDB	414	case 'S':
e531a220	415	generic = true;
fad3917e	416	break;
fad3917e JDB	417	case 'x':
	418	stress_mode = true;
	419	break;
fad3917e JDB	420	case 'p':
fad3917e JDB	421	/* Selecting eBPF prog to load */
bbaf6029	422	prog_name = optarg;
e531a220 KKD	423	prog = bpf_object__find_program_by_name(skel->obj,
	424	prog_name);
	425	if (!prog) {
	426	fprintf(stderr,
	427	"Failed to find program %s specified by"
	428	" option -p/--progname\n",
	429	prog_name);
	430	print_avail_progs(skel->obj);
	431	goto end_cpu;
	432	}
ce4dade7 LB	433	break;
	434	case 'f':
	435	mprog_filename = optarg;
	436	break;
	437	case 'e':
	438	mprog_name = optarg;
	439	break;
	440	case 'r':
	441	redir_interface = optarg;
e531a220	442	mask \|= SAMPLE_DEVMAP_XMIT_CNT_MULTI;
ce4dade7 LB	443	break;
	444	case 'm':
	445	redir_map = optarg;
	446	break;
fad3917e JDB	447	case 'c':
	448	/* Add multiple CPUs */
	449	add_cpu = strtoul(optarg, NULL, 0);
6a098154	450	if (add_cpu >= n_cpus) {
fad3917e	451	fprintf(stderr,
48b2e71c	452	"--cpu nr too large for cpumap err (%d):%s\n",
fad3917e	453	errno, strerror(errno));
48b2e71c	454	usage(argv, long_options, __doc__, mask, true, skel->obj);
e531a220	455	goto end_cpu;
fad3917e	456	}
a4e76f1b	457	cpu[added_cpus++] = add_cpu;
fad3917e JDB	458	break;
fad3917e JDB	459	case 'q':
e531a220	460	qsize = strtoul(optarg, NULL, 0);
fad3917e	461	break;
743e568c	462	case 'F':
e531a220 KKD	463	force = true;
	464	break;
	465	case 'v':
	466	sample_switch_mode();
743e568c	467	break;
fad3917e	468	case 'h':
e531a220	469	error = false;
fad3917e	470	default:
e531a220 KKD	471	usage(argv, long_options, __doc__, mask, error, skel->obj);
e531a220 KKD	472	goto end_cpu;
fad3917e JDB	473	}
fad3917e JDB	474	}
d50ecc46	475
e531a220	476	ret = EXIT_FAIL_OPTION;
fad3917e	477	if (ifindex == -1) {
e531a220 KKD	478	fprintf(stderr, "Required option --dev missing\n");
	479	usage(argv, long_options, __doc__, mask, true, skel->obj);
	480	goto end_cpu;
fad3917e	481	}
e531a220	482
fad3917e	483	if (add_cpu == -1) {
e531a220 KKD	484	fprintf(stderr, "Required option --cpu missing\n"
	485	"Specify multiple --cpu option to add more\n");
	486	usage(argv, long_options, __doc__, mask, true, skel->obj);
	487	goto end_cpu;
fad3917e JDB	488	}
fad3917e JDB	489
e531a220 KKD	490	skel->rodata->from_match[0] = ifindex;
	491	if (redir_interface)
	492	skel->rodata->to_match[0] = if_nametoindex(redir_interface);
	493
	494	ret = xdp_redirect_cpu__load(skel);
	495	if (ret < 0) {
	496	fprintf(stderr, "Failed to xdp_redirect_cpu__load: %s\n",
	497	strerror(errno));
	498	goto end_cpu;
ce4dade7	499	}
ce4dade7	500
e531a220 KKD	501	ret = bpf_obj_get_info_by_fd(bpf_map__fd(skel->maps.cpu_map), &info, &infosz);
	502	if (ret < 0) {
	503	fprintf(stderr, "Failed bpf_obj_get_info_by_fd for cpumap: %s\n",
	504	strerror(errno));
	505	goto end_cpu;
	506	}
a4e76f1b	507
e531a220	508	skel->bss->cpumap_map_id = info.id;
fad3917e	509
e531a220 KKD	510	map_fd = bpf_map__fd(skel->maps.cpu_map);
	511	avail_fd = bpf_map__fd(skel->maps.cpus_available);
	512	count_fd = bpf_map__fd(skel->maps.cpus_count);
	513
	514	ret = mark_cpus_unavailable();
	515	if (ret < 0) {
	516	fprintf(stderr, "Unable to mark CPUs as unavailable\n");
	517	goto end_cpu;
bbaf6029 MF	518	}
bbaf6029 MF	519
e531a220 KKD	520	ret = sample_init(skel, mask);
	521	if (ret < 0) {
	522	fprintf(stderr, "Failed to initialize sample: %s\n", strerror(-ret));
	523	ret = EXIT_FAIL;
	524	goto end_cpu;
bbaf6029 MF	525	}
bbaf6029 MF	526
e531a220 KKD	527	value.bpf_prog.fd = set_cpumap_prog(skel, redir_interface, redir_map,
	528	mprog_filename, mprog_name);
	529	if (value.bpf_prog.fd < 0) {
	530	fprintf(stderr, "Failed to set CPUMAP BPF program: %s\n",
	531	strerror(-value.bpf_prog.fd));
	532	usage(argv, long_options, __doc__, mask, true, skel->obj);
	533	ret = EXIT_FAIL_BPF;
	534	goto end_cpu;
fad3917e	535	}
e531a220	536	value.qsize = qsize;
fad3917e	537
e531a220 KKD	538	for (i = 0; i < added_cpus; i++) {
	539	if (create_cpu_entry(cpu[i], &value, i, true) < 0) {
	540	fprintf(stderr, "Cannot proceed, exiting\n");
	541	usage(argv, long_options, __doc__, mask, true, skel->obj);
	542	goto end_cpu;
	543	}
3b7a8ec2	544	}
3b7a8ec2	545
e531a220 KKD	546	ret = EXIT_FAIL_XDP;
	547	if (sample_install_xdp(prog, ifindex, generic, force) < 0)
	548	goto end_cpu;
151936bf	549
e531a220 KKD	550	ret = sample_run(interval, stress_mode ? stress_cpumap : NULL, &value);
	551	if (ret < 0) {
	552	fprintf(stderr, "Failed during sample run: %s\n", strerror(-ret));
	553	ret = EXIT_FAIL;
	554	goto end_cpu;
	555	}
	556	ret = EXIT_OK;
	557	end_cpu:
a4e76f1b	558	free(cpu);
e531a220 KKD	559	end_destroy:
	560	xdp_redirect_cpu__destroy(skel);
	561	end:
	562	sample_exit(ret);
fad3917e	563	}