[mirror_ubuntu-hirsute-kernel.git] / samples / bpf / map_perf_test_user.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2016 Facebook
 */
#define _GNU_SOURCE
#include <sched.h>
#include <stdio.h>
#include <sys/types.h>
#include <asm/unistd.h>
#include <unistd.h>
#include <assert.h>
#include <sys/wait.h>
#include <stdlib.h>
#include <signal.h>
#include <string.h>
#include <time.h>
#include <sys/resource.h>
#include <arpa/inet.h>
#include <errno.h>

#include <bpf/bpf.h>
#include <bpf/libbpf.h>

#define TEST_BIT(t) (1U << (t))
#define MAX_NR_CPUS 1024

static __u64 time_get_ns(void)
{
	struct timespec ts;

	clock_gettime(CLOCK_MONOTONIC, &ts);
	return ts.tv_sec * 1000000000ull + ts.tv_nsec;
}

enum test_type {
	HASH_PREALLOC,
	PERCPU_HASH_PREALLOC,
	HASH_KMALLOC,
	PERCPU_HASH_KMALLOC,
	LRU_HASH_PREALLOC,
	NOCOMMON_LRU_HASH_PREALLOC,
	LPM_KMALLOC,
	HASH_LOOKUP,
	ARRAY_LOOKUP,
	INNER_LRU_HASH_PREALLOC,
	LRU_HASH_LOOKUP,
	NR_TESTS,
};

const char *test_map_names[NR_TESTS] = {
	[HASH_PREALLOC] = "hash_map",
	[PERCPU_HASH_PREALLOC] = "percpu_hash_map",
	[HASH_KMALLOC] = "hash_map_alloc",
	[PERCPU_HASH_KMALLOC] = "percpu_hash_map_alloc",
	[LRU_HASH_PREALLOC] = "lru_hash_map",
	[NOCOMMON_LRU_HASH_PREALLOC] = "nocommon_lru_hash_map",
	[LPM_KMALLOC] = "lpm_trie_map_alloc",
	[HASH_LOOKUP] = "hash_map",
	[ARRAY_LOOKUP] = "array_map",
	[INNER_LRU_HASH_PREALLOC] = "inner_lru_hash_map",
	[LRU_HASH_LOOKUP] = "lru_hash_lookup_map",
};

enum map_idx {
	array_of_lru_hashs_idx,
	hash_map_alloc_idx,
	lru_hash_lookup_idx,
	NR_IDXES,
};

static int map_fd[NR_IDXES];

static int test_flags = ~0;
static uint32_t num_map_entries;
static uint32_t inner_lru_hash_size;
static int lru_hash_lookup_test_entries = 32;
static uint32_t max_cnt = 1000000;

static int check_test_flags(enum test_type t)
{
	return test_flags & TEST_BIT(t);
}

static void test_hash_prealloc(int cpu)
{
	__u64 start_time;
	int i;

	start_time = time_get_ns();
	for (i = 0; i < max_cnt; i++)
		syscall(__NR_getuid);
	printf("%d:hash_map_perf pre-alloc %lld events per sec\n",
	       cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time));
}

static int pre_test_lru_hash_lookup(int tasks)
{
	int fd = map_fd[lru_hash_lookup_idx];
	uint32_t key;
	long val = 1;
	int ret;

	if (num_map_entries > lru_hash_lookup_test_entries)
		lru_hash_lookup_test_entries = num_map_entries;

	/* Populate the lru_hash_map for LRU_HASH_LOOKUP perf test.
	 *
	 * It is fine that the user requests for a map with
	 * num_map_entries < 32 and some of the later lru hash lookup
	 * may return not found.  For LRU map, we are not interested
	 * in such small map performance.
	 */
	for (key = 0; key < lru_hash_lookup_test_entries; key++) {
		ret = bpf_map_update_elem(fd, &key, &val, BPF_NOEXIST);
		if (ret)
			return ret;
	}

	return 0;
}

static void do_test_lru(enum test_type test, int cpu)
{
	static int inner_lru_map_fds[MAX_NR_CPUS];

	struct sockaddr_in6 in6 = { .sin6_family = AF_INET6 };
	const char *test_name;
	__u64 start_time;
	int i, ret;

	if (test == INNER_LRU_HASH_PREALLOC && cpu) {
		/* If CPU is not 0, create inner_lru hash map and insert the fd
		 * value into the array_of_lru_hash map. In case of CPU 0,
		 * 'inner_lru_hash_map' was statically inserted on the map init
		 */
		int outer_fd = map_fd[array_of_lru_hashs_idx];
		unsigned int mycpu, mynode;

		assert(cpu < MAX_NR_CPUS);

		ret = syscall(__NR_getcpu, &mycpu, &mynode, NULL);
		assert(!ret);

		inner_lru_map_fds[cpu] =
			bpf_create_map_node(BPF_MAP_TYPE_LRU_HASH,
					    test_map_names[INNER_LRU_HASH_PREALLOC],
					    sizeof(uint32_t),
					    sizeof(long),
					    inner_lru_hash_size, 0,
					    mynode);
		if (inner_lru_map_fds[cpu] == -1) {
			printf("cannot create BPF_MAP_TYPE_LRU_HASH %s(%d)\n",
			       strerror(errno), errno);
			exit(1);
		}

		ret = bpf_map_update_elem(outer_fd, &cpu,
					  &inner_lru_map_fds[cpu],
					  BPF_ANY);
		if (ret) {
			printf("cannot update ARRAY_OF_LRU_HASHS with key:%u. %s(%d)\n",
			       cpu, strerror(errno), errno);
			exit(1);
		}
	}

	in6.sin6_addr.s6_addr16[0] = 0xdead;
	in6.sin6_addr.s6_addr16[1] = 0xbeef;

	if (test == LRU_HASH_PREALLOC) {
		test_name = "lru_hash_map_perf";
		in6.sin6_addr.s6_addr16[2] = 0;
	} else if (test == NOCOMMON_LRU_HASH_PREALLOC) {
		test_name = "nocommon_lru_hash_map_perf";
		in6.sin6_addr.s6_addr16[2] = 1;
	} else if (test == INNER_LRU_HASH_PREALLOC) {
		test_name = "inner_lru_hash_map_perf";
		in6.sin6_addr.s6_addr16[2] = 2;
	} else if (test == LRU_HASH_LOOKUP) {
		test_name = "lru_hash_lookup_perf";
		in6.sin6_addr.s6_addr16[2] = 3;
		in6.sin6_addr.s6_addr32[3] = 0;
	} else {
		assert(0);
	}

	start_time = time_get_ns();
	for (i = 0; i < max_cnt; i++) {
		ret = connect(-1, (const struct sockaddr *)&in6, sizeof(in6));
		assert(ret == -1 && errno == EBADF);
		if (in6.sin6_addr.s6_addr32[3] <
		    lru_hash_lookup_test_entries - 32)
			in6.sin6_addr.s6_addr32[3] += 32;
		else
			in6.sin6_addr.s6_addr32[3] = 0;
	}
	printf("%d:%s pre-alloc %lld events per sec\n",
	       cpu, test_name,
	       max_cnt * 1000000000ll / (time_get_ns() - start_time));
}

static void test_lru_hash_prealloc(int cpu)
{
	do_test_lru(LRU_HASH_PREALLOC, cpu);
}

static void test_nocommon_lru_hash_prealloc(int cpu)
{
	do_test_lru(NOCOMMON_LRU_HASH_PREALLOC, cpu);
}

static void test_inner_lru_hash_prealloc(int cpu)
{
	do_test_lru(INNER_LRU_HASH_PREALLOC, cpu);
}

static void test_lru_hash_lookup(int cpu)
{
	do_test_lru(LRU_HASH_LOOKUP, cpu);
}

static void test_percpu_hash_prealloc(int cpu)
{
	__u64 start_time;
	int i;

	start_time = time_get_ns();
	for (i = 0; i < max_cnt; i++)
		syscall(__NR_geteuid);
	printf("%d:percpu_hash_map_perf pre-alloc %lld events per sec\n",
	       cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time));
}

static void test_hash_kmalloc(int cpu)
{
	__u64 start_time;
	int i;

	start_time = time_get_ns();
	for (i = 0; i < max_cnt; i++)
		syscall(__NR_getgid);
	printf("%d:hash_map_perf kmalloc %lld events per sec\n",
	       cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time));
}

static void test_percpu_hash_kmalloc(int cpu)
{
	__u64 start_time;
	int i;

	start_time = time_get_ns();
	for (i = 0; i < max_cnt; i++)
		syscall(__NR_getegid);
	printf("%d:percpu_hash_map_perf kmalloc %lld events per sec\n",
	       cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time));
}

static void test_lpm_kmalloc(int cpu)
{
	__u64 start_time;
	int i;

	start_time = time_get_ns();
	for (i = 0; i < max_cnt; i++)
		syscall(__NR_gettid);
	printf("%d:lpm_perf kmalloc %lld events per sec\n",
	       cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time));
}

static void test_hash_lookup(int cpu)
{
	__u64 start_time;
	int i;

	start_time = time_get_ns();
	for (i = 0; i < max_cnt; i++)
		syscall(__NR_getpgid, 0);
	printf("%d:hash_lookup %lld lookups per sec\n",
	       cpu, max_cnt * 1000000000ll * 64 / (time_get_ns() - start_time));
}

static void test_array_lookup(int cpu)
{
	__u64 start_time;
	int i;

	start_time = time_get_ns();
	for (i = 0; i < max_cnt; i++)
		syscall(__NR_getppid, 0);
	printf("%d:array_lookup %lld lookups per sec\n",
	       cpu, max_cnt * 1000000000ll * 64 / (time_get_ns() - start_time));
}

typedef int (*pre_test_func)(int tasks);
const pre_test_func pre_test_funcs[] = {
	[LRU_HASH_LOOKUP] = pre_test_lru_hash_lookup,
};

typedef void (*test_func)(int cpu);
const test_func test_funcs[] = {
	[HASH_PREALLOC] = test_hash_prealloc,
	[PERCPU_HASH_PREALLOC] = test_percpu_hash_prealloc,
	[HASH_KMALLOC] = test_hash_kmalloc,
	[PERCPU_HASH_KMALLOC] = test_percpu_hash_kmalloc,
	[LRU_HASH_PREALLOC] = test_lru_hash_prealloc,
	[NOCOMMON_LRU_HASH_PREALLOC] = test_nocommon_lru_hash_prealloc,
	[LPM_KMALLOC] = test_lpm_kmalloc,
	[HASH_LOOKUP] = test_hash_lookup,
	[ARRAY_LOOKUP] = test_array_lookup,
	[INNER_LRU_HASH_PREALLOC] = test_inner_lru_hash_prealloc,
	[LRU_HASH_LOOKUP] = test_lru_hash_lookup,
};

static int pre_test(int tasks)
{
	int i;

	for (i = 0; i < NR_TESTS; i++) {
		if (pre_test_funcs[i] && check_test_flags(i)) {
			int ret = pre_test_funcs[i](tasks);

			if (ret)
				return ret;
		}
	}

	return 0;
}

static void loop(int cpu)
{
	cpu_set_t cpuset;
	int i;

	CPU_ZERO(&cpuset);
	CPU_SET(cpu, &cpuset);
	sched_setaffinity(0, sizeof(cpuset), &cpuset);

	for (i = 0; i < NR_TESTS; i++) {
		if (check_test_flags(i))
			test_funcs[i](cpu);
	}
}

static void run_perf_test(int tasks)
{
	pid_t pid[tasks];
	int i;

	assert(!pre_test(tasks));

	for (i = 0; i < tasks; i++) {
		pid[i] = fork();
		if (pid[i] == 0) {
			loop(i);
			exit(0);
		} else if (pid[i] == -1) {
			printf("couldn't spawn #%d process\n", i);
			exit(1);
		}
	}
	for (i = 0; i < tasks; i++) {
		int status;

		assert(waitpid(pid[i], &status, 0) == pid[i]);
		assert(status == 0);
	}
}

static void fill_lpm_trie(void)
{
	struct bpf_lpm_trie_key *key;
	unsigned long value = 0;
	unsigned int i;
	int r;

	key = alloca(sizeof(*key) + 4);
	key->prefixlen = 32;

	for (i = 0; i < 512; ++i) {
		key->prefixlen = rand() % 33;
		key->data[0] = rand() & 0xff;
		key->data[1] = rand() & 0xff;
		key->data[2] = rand() & 0xff;
		key->data[3] = rand() & 0xff;
		r = bpf_map_update_elem(map_fd[hash_map_alloc_idx],
					key, &value, 0);
		assert(!r);
	}

	key->prefixlen = 32;
	key->data[0] = 192;
	key->data[1] = 168;
	key->data[2] = 0;
	key->data[3] = 1;
	value = 128;

	r = bpf_map_update_elem(map_fd[hash_map_alloc_idx], key, &value, 0);
	assert(!r);
}

static void fixup_map(struct bpf_object *obj)
{
	struct bpf_map *map;
	int i;

	bpf_object__for_each_map(map, obj) {
		const char *name = bpf_map__name(map);

		/* Only change the max_entries for the enabled test(s) */
		for (i = 0; i < NR_TESTS; i++) {
			if (!strcmp(test_map_names[i], name) &&
			    (check_test_flags(i))) {
				bpf_map__resize(map, num_map_entries);
				continue;
			}
		}
	}

	inner_lru_hash_size = num_map_entries;
}

int main(int argc, char **argv)
{
	int nr_cpus = sysconf(_SC_NPROCESSORS_ONLN);
	struct bpf_link *links[8];
	struct bpf_program *prog;
	struct bpf_object *obj;
	struct bpf_map *map;
	char filename[256];
	int i = 0;

	if (argc > 1)
		test_flags = atoi(argv[1]) ? : test_flags;

	if (argc > 2)
		nr_cpus = atoi(argv[2]) ? : nr_cpus;

	if (argc > 3)
		num_map_entries = atoi(argv[3]);

	if (argc > 4)
		max_cnt = atoi(argv[4]);

	snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);
	obj = bpf_object__open_file(filename, NULL);
	if (libbpf_get_error(obj)) {
		fprintf(stderr, "ERROR: opening BPF object file failed\n");
		return 0;
	}

	map = bpf_object__find_map_by_name(obj, "inner_lru_hash_map");
	if (libbpf_get_error(map)) {
		fprintf(stderr, "ERROR: finding a map in obj file failed\n");
		goto cleanup;
	}

	inner_lru_hash_size = bpf_map__max_entries(map);
	if (!inner_lru_hash_size) {
		fprintf(stderr, "ERROR: failed to get map attribute\n");
		goto cleanup;
	}

	/* resize BPF map prior to loading */
	if (num_map_entries > 0)
		fixup_map(obj);

	/* load BPF program */
	if (bpf_object__load(obj)) {
		fprintf(stderr, "ERROR: loading BPF object file failed\n");
		goto cleanup;
	}

	map_fd[0] = bpf_object__find_map_fd_by_name(obj, "array_of_lru_hashs");
	map_fd[1] = bpf_object__find_map_fd_by_name(obj, "hash_map_alloc");
	map_fd[2] = bpf_object__find_map_fd_by_name(obj, "lru_hash_lookup_map");
	if (map_fd[0] < 0 || map_fd[1] < 0 || map_fd[2] < 0) {
		fprintf(stderr, "ERROR: finding a map in obj file failed\n");
		goto cleanup;
	}

	bpf_object__for_each_program(prog, obj) {
		links[i] = bpf_program__attach(prog);
		if (libbpf_get_error(links[i])) {
			fprintf(stderr, "ERROR: bpf_program__attach failed\n");
			links[i] = NULL;
			goto cleanup;
		}
		i++;
	}

	fill_lpm_trie();

	run_perf_test(nr_cpus);

cleanup:
	for (i--; i >= 0; i--)
		bpf_link__destroy(links[i]);

	bpf_object__close(obj);
	return 0;
}
Commit	Line	Data
25763b3c	1	// SPDX-License-Identifier: GPL-2.0-only
26e90931	2	/* Copyright (c) 2016 Facebook
26e90931 AS	3	*/
	4	#define _GNU_SOURCE
	5	#include <sched.h>
	6	#include <stdio.h>
	7	#include <sys/types.h>
	8	#include <asm/unistd.h>
	9	#include <unistd.h>
	10	#include <assert.h>
	11	#include <sys/wait.h>
	12	#include <stdlib.h>
	13	#include <signal.h>
26e90931 AS	14	#include <string.h>
26e90931 AS	15	#include <time.h>
77e63534	16	#include <sys/resource.h>
bf8db5d2 MKL	17	#include <arpa/inet.h>
	18	#include <errno.h>
	19
2bf3e2ef	20	#include <bpf/bpf.h>
cc7f641d	21	#include <bpf/libbpf.h>
26e90931	22
9fd63d05	23	#define TEST_BIT(t) (1U << (t))
3a5795b8	24	#define MAX_NR_CPUS 1024
26e90931 AS	25
	26	static __u64 time_get_ns(void)
	27	{
	28	struct timespec ts;
	29
	30	clock_gettime(CLOCK_MONOTONIC, &ts);
	31	return ts.tv_sec * 1000000000ull + ts.tv_nsec;
	32	}
	33
9fd63d05 MKL	34	enum test_type {
	35	HASH_PREALLOC,
	36	PERCPU_HASH_PREALLOC,
	37	HASH_KMALLOC,
	38	PERCPU_HASH_KMALLOC,
	39	LRU_HASH_PREALLOC,
	40	NOCOMMON_LRU_HASH_PREALLOC,
	41	LPM_KMALLOC,
	42	HASH_LOOKUP,
	43	ARRAY_LOOKUP,
3a5795b8	44	INNER_LRU_HASH_PREALLOC,
637cd8c3	45	LRU_HASH_LOOKUP,
9fd63d05 MKL	46	NR_TESTS,
	47	};
	48
	49	const char *test_map_names[NR_TESTS] = {
	50	[HASH_PREALLOC] = "hash_map",
	51	[PERCPU_HASH_PREALLOC] = "percpu_hash_map",
	52	[HASH_KMALLOC] = "hash_map_alloc",
	53	[PERCPU_HASH_KMALLOC] = "percpu_hash_map_alloc",
	54	[LRU_HASH_PREALLOC] = "lru_hash_map",
	55	[NOCOMMON_LRU_HASH_PREALLOC] = "nocommon_lru_hash_map",
	56	[LPM_KMALLOC] = "lpm_trie_map_alloc",
	57	[HASH_LOOKUP] = "hash_map",
	58	[ARRAY_LOOKUP] = "array_map",
3a5795b8	59	[INNER_LRU_HASH_PREALLOC] = "inner_lru_hash_map",
637cd8c3	60	[LRU_HASH_LOOKUP] = "lru_hash_lookup_map",
9fd63d05	61	};
26e90931	62
cc7f641d DL	63	enum map_idx {
	64	array_of_lru_hashs_idx,
	65	hash_map_alloc_idx,
	66	lru_hash_lookup_idx,
	67	NR_IDXES,
	68	};
	69
	70	static int map_fd[NR_IDXES];
	71
26e90931	72	static int test_flags = ~0;
9fd63d05	73	static uint32_t num_map_entries;
3a5795b8	74	static uint32_t inner_lru_hash_size;
637cd8c3	75	static int lru_hash_lookup_test_entries = 32;
9fd63d05 MKL	76	static uint32_t max_cnt = 1000000;
	77
	78	static int check_test_flags(enum test_type t)
	79	{
	80	return test_flags & TEST_BIT(t);
	81	}
26e90931 AS	82
	83	static void test_hash_prealloc(int cpu)
	84	{
	85	__u64 start_time;
	86	int i;
	87
	88	start_time = time_get_ns();
9fd63d05	89	for (i = 0; i < max_cnt; i++)
26e90931 AS	90	syscall(__NR_getuid);
26e90931 AS	91	printf("%d:hash_map_perf pre-alloc %lld events per sec\n",
9fd63d05	92	cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time));
26e90931 AS	93	}
26e90931 AS	94
637cd8c3 MKL	95	static int pre_test_lru_hash_lookup(int tasks)
	96	{
	97	int fd = map_fd[lru_hash_lookup_idx];
	98	uint32_t key;
	99	long val = 1;
	100	int ret;
	101
	102	if (num_map_entries > lru_hash_lookup_test_entries)
	103	lru_hash_lookup_test_entries = num_map_entries;
	104
	105	/* Populate the lru_hash_map for LRU_HASH_LOOKUP perf test.
	106	*
	107	* It is fine that the user requests for a map with
	108	* num_map_entries < 32 and some of the later lru hash lookup
	109	* may return not found. For LRU map, we are not interested
	110	* in such small map performance.
	111	*/
	112	for (key = 0; key < lru_hash_lookup_test_entries; key++) {
	113	ret = bpf_map_update_elem(fd, &key, &val, BPF_NOEXIST);
	114	if (ret)
	115	return ret;
	116	}
	117
	118	return 0;
	119	}
	120
9fd63d05	121	static void do_test_lru(enum test_type test, int cpu)
5db58faf	122	{
3a5795b8 MKL	123	static int inner_lru_map_fds[MAX_NR_CPUS];
3a5795b8 MKL	124
bf8db5d2 MKL	125	struct sockaddr_in6 in6 = { .sin6_family = AF_INET6 };
bf8db5d2 MKL	126	const char *test_name;
5db58faf	127	__u64 start_time;
bf8db5d2 MKL	128	int i, ret;
bf8db5d2 MKL	129
cc7f641d DL	130	if (test == INNER_LRU_HASH_PREALLOC && cpu) {
	131	/* If CPU is not 0, create inner_lru hash map and insert the fd
	132	* value into the array_of_lru_hash map. In case of CPU 0,
	133	* 'inner_lru_hash_map' was statically inserted on the map init
	134	*/
3a5795b8	135	int outer_fd = map_fd[array_of_lru_hashs_idx];
ad17d0e6	136	unsigned int mycpu, mynode;
3a5795b8 MKL	137
	138	assert(cpu < MAX_NR_CPUS);
	139
cc7f641d DL	140	ret = syscall(__NR_getcpu, &mycpu, &mynode, NULL);
	141	assert(!ret);
	142
	143	inner_lru_map_fds[cpu] =
	144	bpf_create_map_node(BPF_MAP_TYPE_LRU_HASH,
	145	test_map_names[INNER_LRU_HASH_PREALLOC],
	146	sizeof(uint32_t),
	147	sizeof(long),
	148	inner_lru_hash_size, 0,
	149	mynode);
	150	if (inner_lru_map_fds[cpu] == -1) {
	151	printf("cannot create BPF_MAP_TYPE_LRU_HASH %s(%d)\n",
	152	strerror(errno), errno);
	153	exit(1);
3a5795b8 MKL	154	}
	155
	156	ret = bpf_map_update_elem(outer_fd, &cpu,
	157	&inner_lru_map_fds[cpu],
	158	BPF_ANY);
	159	if (ret) {
	160	printf("cannot update ARRAY_OF_LRU_HASHS with key:%u. %s(%d)\n",
	161	cpu, strerror(errno), errno);
	162	exit(1);
	163	}
	164	}
	165
bf8db5d2 MKL	166	in6.sin6_addr.s6_addr16[0] = 0xdead;
	167	in6.sin6_addr.s6_addr16[1] = 0xbeef;
	168
9fd63d05	169	if (test == LRU_HASH_PREALLOC) {
bf8db5d2	170	test_name = "lru_hash_map_perf";
637cd8c3	171	in6.sin6_addr.s6_addr16[2] = 0;
9fd63d05	172	} else if (test == NOCOMMON_LRU_HASH_PREALLOC) {
bf8db5d2	173	test_name = "nocommon_lru_hash_map_perf";
637cd8c3	174	in6.sin6_addr.s6_addr16[2] = 1;
3a5795b8 MKL	175	} else if (test == INNER_LRU_HASH_PREALLOC) {
3a5795b8 MKL	176	test_name = "inner_lru_hash_map_perf";
637cd8c3 MKL	177	in6.sin6_addr.s6_addr16[2] = 2;
	178	} else if (test == LRU_HASH_LOOKUP) {
	179	test_name = "lru_hash_lookup_perf";
	180	in6.sin6_addr.s6_addr16[2] = 3;
	181	in6.sin6_addr.s6_addr32[3] = 0;
bf8db5d2 MKL	182	} else {
	183	assert(0);
	184	}
5db58faf MKL	185
5db58faf MKL	186	start_time = time_get_ns();
9fd63d05	187	for (i = 0; i < max_cnt; i++) {
bf8db5d2 MKL	188	ret = connect(-1, (const struct sockaddr *)&in6, sizeof(in6));
bf8db5d2 MKL	189	assert(ret == -1 && errno == EBADF);
637cd8c3 MKL	190	if (in6.sin6_addr.s6_addr32[3] <
	191	lru_hash_lookup_test_entries - 32)
	192	in6.sin6_addr.s6_addr32[3] += 32;
	193	else
	194	in6.sin6_addr.s6_addr32[3] = 0;
bf8db5d2 MKL	195	}
	196	printf("%d:%s pre-alloc %lld events per sec\n",
	197	cpu, test_name,
9fd63d05	198	max_cnt * 1000000000ll / (time_get_ns() - start_time));
5db58faf MKL	199	}
5db58faf MKL	200
bf8db5d2	201	static void test_lru_hash_prealloc(int cpu)
5db58faf	202	{
bf8db5d2 MKL	203	do_test_lru(LRU_HASH_PREALLOC, cpu);
bf8db5d2 MKL	204	}
5db58faf	205
bf8db5d2 MKL	206	static void test_nocommon_lru_hash_prealloc(int cpu)
	207	{
	208	do_test_lru(NOCOMMON_LRU_HASH_PREALLOC, cpu);
5db58faf MKL	209	}
5db58faf MKL	210
3a5795b8 MKL	211	static void test_inner_lru_hash_prealloc(int cpu)
	212	{
	213	do_test_lru(INNER_LRU_HASH_PREALLOC, cpu);
	214	}
	215
637cd8c3 MKL	216	static void test_lru_hash_lookup(int cpu)
	217	{
	218	do_test_lru(LRU_HASH_LOOKUP, cpu);
	219	}
	220
26e90931 AS	221	static void test_percpu_hash_prealloc(int cpu)
	222	{
	223	__u64 start_time;
	224	int i;
	225
	226	start_time = time_get_ns();
9fd63d05	227	for (i = 0; i < max_cnt; i++)
26e90931 AS	228	syscall(__NR_geteuid);
26e90931 AS	229	printf("%d:percpu_hash_map_perf pre-alloc %lld events per sec\n",
9fd63d05	230	cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time));
26e90931 AS	231	}
	232
	233	static void test_hash_kmalloc(int cpu)
	234	{
	235	__u64 start_time;
	236	int i;
	237
	238	start_time = time_get_ns();
9fd63d05	239	for (i = 0; i < max_cnt; i++)
26e90931 AS	240	syscall(__NR_getgid);
26e90931 AS	241	printf("%d:hash_map_perf kmalloc %lld events per sec\n",
9fd63d05	242	cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time));
26e90931 AS	243	}
	244
	245	static void test_percpu_hash_kmalloc(int cpu)
	246	{
	247	__u64 start_time;
	248	int i;
	249
	250	start_time = time_get_ns();
9fd63d05	251	for (i = 0; i < max_cnt; i++)
26e90931 AS	252	syscall(__NR_getegid);
26e90931 AS	253	printf("%d:percpu_hash_map_perf kmalloc %lld events per sec\n",
9fd63d05	254	cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time));
26e90931 AS	255	}
26e90931 AS	256
b8a943e2 DH	257	static void test_lpm_kmalloc(int cpu)
	258	{
	259	__u64 start_time;
	260	int i;
	261
	262	start_time = time_get_ns();
9fd63d05	263	for (i = 0; i < max_cnt; i++)
b8a943e2 DH	264	syscall(__NR_gettid);
b8a943e2 DH	265	printf("%d:lpm_perf kmalloc %lld events per sec\n",
9fd63d05	266	cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time));
b8a943e2 DH	267	}
b8a943e2 DH	268
95ff141e AS	269	static void test_hash_lookup(int cpu)
	270	{
	271	__u64 start_time;
	272	int i;
	273
	274	start_time = time_get_ns();
9fd63d05	275	for (i = 0; i < max_cnt; i++)
95ff141e AS	276	syscall(__NR_getpgid, 0);
95ff141e AS	277	printf("%d:hash_lookup %lld lookups per sec\n",
9fd63d05	278	cpu, max_cnt * 1000000000ll * 64 / (time_get_ns() - start_time));
95ff141e AS	279	}
	280
	281	static void test_array_lookup(int cpu)
	282	{
	283	__u64 start_time;
	284	int i;
	285
	286	start_time = time_get_ns();
9fd63d05	287	for (i = 0; i < max_cnt; i++)
95ec6696	288	syscall(__NR_getppid, 0);
95ff141e	289	printf("%d:array_lookup %lld lookups per sec\n",
9fd63d05	290	cpu, max_cnt * 1000000000ll * 64 / (time_get_ns() - start_time));
95ff141e AS	291	}
95ff141e AS	292
637cd8c3 MKL	293	typedef int (*pre_test_func)(int tasks);
	294	const pre_test_func pre_test_funcs[] = {
	295	[LRU_HASH_LOOKUP] = pre_test_lru_hash_lookup,
	296	};
	297
9fd63d05 MKL	298	typedef void (*test_func)(int cpu);
	299	const test_func test_funcs[] = {
	300	[HASH_PREALLOC] = test_hash_prealloc,
	301	[PERCPU_HASH_PREALLOC] = test_percpu_hash_prealloc,
	302	[HASH_KMALLOC] = test_hash_kmalloc,
	303	[PERCPU_HASH_KMALLOC] = test_percpu_hash_kmalloc,
	304	[LRU_HASH_PREALLOC] = test_lru_hash_prealloc,
	305	[NOCOMMON_LRU_HASH_PREALLOC] = test_nocommon_lru_hash_prealloc,
	306	[LPM_KMALLOC] = test_lpm_kmalloc,
	307	[HASH_LOOKUP] = test_hash_lookup,
	308	[ARRAY_LOOKUP] = test_array_lookup,
3a5795b8	309	[INNER_LRU_HASH_PREALLOC] = test_inner_lru_hash_prealloc,
637cd8c3	310	[LRU_HASH_LOOKUP] = test_lru_hash_lookup,
9fd63d05 MKL	311	};
9fd63d05 MKL	312
637cd8c3 MKL	313	static int pre_test(int tasks)
	314	{
	315	int i;
	316
	317	for (i = 0; i < NR_TESTS; i++) {
	318	if (pre_test_funcs[i] && check_test_flags(i)) {
	319	int ret = pre_test_funcs[i](tasks);
	320
	321	if (ret)
	322	return ret;
	323	}
	324	}
	325
	326	return 0;
	327	}
	328
26e90931 AS	329	static void loop(int cpu)
	330	{
	331	cpu_set_t cpuset;
9fd63d05	332	int i;
26e90931 AS	333
	334	CPU_ZERO(&cpuset);
	335	CPU_SET(cpu, &cpuset);
	336	sched_setaffinity(0, sizeof(cpuset), &cpuset);
	337
9fd63d05 MKL	338	for (i = 0; i < NR_TESTS; i++) {
	339	if (check_test_flags(i))
	340	test_funcs[i](cpu);
	341	}
26e90931 AS	342	}
	343
	344	static void run_perf_test(int tasks)
	345	{
	346	pid_t pid[tasks];
	347	int i;
	348
637cd8c3 MKL	349	assert(!pre_test(tasks));
637cd8c3 MKL	350
26e90931 AS	351	for (i = 0; i < tasks; i++) {
	352	pid[i] = fork();
	353	if (pid[i] == 0) {
	354	loop(i);
	355	exit(0);
	356	} else if (pid[i] == -1) {
	357	printf("couldn't spawn #%d process\n", i);
	358	exit(1);
	359	}
	360	}
	361	for (i = 0; i < tasks; i++) {
	362	int status;
	363
	364	assert(waitpid(pid[i], &status, 0) == pid[i]);
	365	assert(status == 0);
	366	}
	367	}
	368
b8a943e2 DH	369	static void fill_lpm_trie(void)
	370	{
	371	struct bpf_lpm_trie_key *key;
	372	unsigned long value = 0;
	373	unsigned int i;
	374	int r;
	375
	376	key = alloca(sizeof(*key) + 4);
	377	key->prefixlen = 32;
	378
	379	for (i = 0; i < 512; ++i) {
	380	key->prefixlen = rand() % 33;
	381	key->data[0] = rand() & 0xff;
	382	key->data[1] = rand() & 0xff;
	383	key->data[2] = rand() & 0xff;
	384	key->data[3] = rand() & 0xff;
cc7f641d DL	385	r = bpf_map_update_elem(map_fd[hash_map_alloc_idx],
cc7f641d DL	386	key, &value, 0);
b8a943e2 DH	387	assert(!r);
	388	}
	389
	390	key->prefixlen = 32;
	391	key->data[0] = 192;
	392	key->data[1] = 168;
	393	key->data[2] = 0;
	394	key->data[3] = 1;
	395	value = 128;
	396
cc7f641d	397	r = bpf_map_update_elem(map_fd[hash_map_alloc_idx], key, &value, 0);
b8a943e2 DH	398	assert(!r);
	399	}
	400
cc7f641d	401	static void fixup_map(struct bpf_object *obj)
9fd63d05	402	{
cc7f641d	403	struct bpf_map *map;
9fd63d05 MKL	404	int i;
9fd63d05 MKL	405
cc7f641d DL	406	bpf_object__for_each_map(map, obj) {
cc7f641d DL	407	const char *name = bpf_map__name(map);
3a5795b8	408
cc7f641d DL	409	/* Only change the max_entries for the enabled test(s) */
	410	for (i = 0; i < NR_TESTS; i++) {
	411	if (!strcmp(test_map_names[i], name) &&
	412	(check_test_flags(i))) {
	413	bpf_map__resize(map, num_map_entries);
	414	continue;
	415	}
3a5795b8	416	}
3a5795b8 MKL	417	}
3a5795b8 MKL	418
3a5795b8	419	inner_lru_hash_size = num_map_entries;
9fd63d05 MKL	420	}
9fd63d05 MKL	421
26e90931 AS	422	int main(int argc, char **argv)
26e90931 AS	423	{
cc7f641d DL	424	int nr_cpus = sysconf(_SC_NPROCESSORS_ONLN);
	425	struct bpf_link *links[8];
	426	struct bpf_program *prog;
	427	struct bpf_object *obj;
	428	struct bpf_map *map;
26e90931	429	char filename[256];
cc7f641d	430	int i = 0;
26e90931	431
26e90931 AS	432	if (argc > 1)
	433	test_flags = atoi(argv[1]) ? : test_flags;
	434
	435	if (argc > 2)
cc7f641d	436	nr_cpus = atoi(argv[2]) ? : nr_cpus;
26e90931	437
9fd63d05 MKL	438	if (argc > 3)
	439	num_map_entries = atoi(argv[3]);
	440
	441	if (argc > 4)
	442	max_cnt = atoi(argv[4]);
	443
cc7f641d DL	444	snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);
	445	obj = bpf_object__open_file(filename, NULL);
	446	if (libbpf_get_error(obj)) {
	447	fprintf(stderr, "ERROR: opening BPF object file failed\n");
	448	return 0;
	449	}
	450
	451	map = bpf_object__find_map_by_name(obj, "inner_lru_hash_map");
	452	if (libbpf_get_error(map)) {
	453	fprintf(stderr, "ERROR: finding a map in obj file failed\n");
	454	goto cleanup;
	455	}
	456
	457	inner_lru_hash_size = bpf_map__max_entries(map);
	458	if (!inner_lru_hash_size) {
	459	fprintf(stderr, "ERROR: failed to get map attribute\n");
	460	goto cleanup;
	461	}
	462
	463	/* resize BPF map prior to loading */
	464	if (num_map_entries > 0)
	465	fixup_map(obj);
	466
	467	/* load BPF program */
	468	if (bpf_object__load(obj)) {
	469	fprintf(stderr, "ERROR: loading BPF object file failed\n");
	470	goto cleanup;
	471	}
	472
	473	map_fd[0] = bpf_object__find_map_fd_by_name(obj, "array_of_lru_hashs");
	474	map_fd[1] = bpf_object__find_map_fd_by_name(obj, "hash_map_alloc");
	475	map_fd[2] = bpf_object__find_map_fd_by_name(obj, "lru_hash_lookup_map");
	476	if (map_fd[0] < 0 \|\| map_fd[1] < 0 \|\| map_fd[2] < 0) {
	477	fprintf(stderr, "ERROR: finding a map in obj file failed\n");
	478	goto cleanup;
	479	}
	480
	481	bpf_object__for_each_program(prog, obj) {
	482	links[i] = bpf_program__attach(prog);
	483	if (libbpf_get_error(links[i])) {
	484	fprintf(stderr, "ERROR: bpf_program__attach failed\n");
	485	links[i] = NULL;
	486	goto cleanup;
	487	}
	488	i++;
26e90931 AS	489	}
26e90931 AS	490
b8a943e2 DH	491	fill_lpm_trie();
b8a943e2 DH	492
cc7f641d DL	493	run_perf_test(nr_cpus);
	494
	495	cleanup:
	496	for (i--; i >= 0; i--)
	497	bpf_link__destroy(links[i]);
26e90931	498
cc7f641d	499	bpf_object__close(obj);
26e90931 AS	500	return 0;
26e90931 AS	501	}