]>
Commit | Line | Data |
---|---|---|
b4b8faa1 | 1 | // SPDX-License-Identifier: GPL-2.0 |
dac09149 | 2 | /* Copyright(c) 2017 - 2018 Intel Corporation. */ |
b4b8faa1 | 3 | |
248c7f9c | 4 | #include <asm/barrier.h> |
b4b8faa1 MK |
5 | #include <errno.h> |
6 | #include <getopt.h> | |
7 | #include <libgen.h> | |
8 | #include <linux/bpf.h> | |
248c7f9c | 9 | #include <linux/compiler.h> |
b4b8faa1 MK |
10 | #include <linux/if_link.h> |
11 | #include <linux/if_xdp.h> | |
12 | #include <linux/if_ether.h> | |
248c7f9c MK |
13 | #include <locale.h> |
14 | #include <net/ethernet.h> | |
b4b8faa1 | 15 | #include <net/if.h> |
248c7f9c MK |
16 | #include <poll.h> |
17 | #include <pthread.h> | |
b4b8faa1 MK |
18 | #include <signal.h> |
19 | #include <stdbool.h> | |
20 | #include <stdio.h> | |
21 | #include <stdlib.h> | |
22 | #include <string.h> | |
248c7f9c | 23 | #include <sys/mman.h> |
b4b8faa1 MK |
24 | #include <sys/resource.h> |
25 | #include <sys/socket.h> | |
248c7f9c | 26 | #include <sys/types.h> |
b4b8faa1 MK |
27 | #include <time.h> |
28 | #include <unistd.h> | |
b4b8faa1 | 29 | |
4d18f6de DL |
30 | #include "libbpf.h" |
31 | #include "xsk.h" | |
2e5d72c1 | 32 | #include "xdpsock.h" |
2bf3e2ef | 33 | #include <bpf/bpf.h> |
b4b8faa1 | 34 | |
b4b8faa1 MK |
35 | #ifndef SOL_XDP |
36 | #define SOL_XDP 283 | |
37 | #endif | |
38 | ||
39 | #ifndef AF_XDP | |
40 | #define AF_XDP 44 | |
41 | #endif | |
42 | ||
43 | #ifndef PF_XDP | |
44 | #define PF_XDP AF_XDP | |
45 | #endif | |
46 | ||
248c7f9c MK |
47 | #define NUM_FRAMES (4 * 1024) |
48 | #define BATCH_SIZE 64 | |
b4b8faa1 MK |
49 | |
50 | #define DEBUG_HEXDUMP 0 | |
51 | ||
a412ef54 | 52 | typedef __u64 u64; |
b4b8faa1 MK |
53 | typedef __u32 u32; |
54 | ||
55 | static unsigned long prev_time; | |
56 | ||
57 | enum benchmark_type { | |
58 | BENCH_RXDROP = 0, | |
59 | BENCH_TXONLY = 1, | |
60 | BENCH_L2FWD = 2, | |
61 | }; | |
62 | ||
63 | static enum benchmark_type opt_bench = BENCH_RXDROP; | |
743e568c | 64 | static u32 opt_xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST; |
b4b8faa1 MK |
65 | static const char *opt_if = ""; |
66 | static int opt_ifindex; | |
67 | static int opt_queue; | |
68 | static int opt_poll; | |
b4b8faa1 | 69 | static int opt_interval = 1; |
46738f73 | 70 | static u32 opt_xdp_bind_flags = XDP_USE_NEED_WAKEUP; |
c543f546 KL |
71 | static u32 opt_umem_flags; |
72 | static int opt_unaligned_chunks; | |
3945b37a | 73 | static int opt_mmap_flags; |
c543f546 | 74 | static u32 opt_xdp_bind_flags; |
123e8da1 | 75 | static int opt_xsk_frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; |
46738f73 MK |
76 | static int opt_timeout = 1000; |
77 | static bool opt_need_wakeup = true; | |
2e5d72c1 MK |
78 | static u32 opt_num_xsks = 1; |
79 | static u32 prog_id; | |
b4b8faa1 | 80 | |
248c7f9c MK |
81 | struct xsk_umem_info { |
82 | struct xsk_ring_prod fq; | |
83 | struct xsk_ring_cons cq; | |
84 | struct xsk_umem *umem; | |
85 | void *buffer; | |
b4b8faa1 MK |
86 | }; |
87 | ||
248c7f9c MK |
88 | struct xsk_socket_info { |
89 | struct xsk_ring_cons rx; | |
90 | struct xsk_ring_prod tx; | |
91 | struct xsk_umem_info *umem; | |
92 | struct xsk_socket *xsk; | |
b4b8faa1 MK |
93 | unsigned long rx_npkts; |
94 | unsigned long tx_npkts; | |
95 | unsigned long prev_rx_npkts; | |
96 | unsigned long prev_tx_npkts; | |
248c7f9c | 97 | u32 outstanding_tx; |
b4b8faa1 MK |
98 | }; |
99 | ||
b4b8faa1 | 100 | static int num_socks; |
248c7f9c | 101 | struct xsk_socket_info *xsks[MAX_SOCKS]; |
b4b8faa1 MK |
102 | |
103 | static unsigned long get_nsecs(void) | |
104 | { | |
105 | struct timespec ts; | |
106 | ||
107 | clock_gettime(CLOCK_MONOTONIC, &ts); | |
108 | return ts.tv_sec * 1000000000UL + ts.tv_nsec; | |
109 | } | |
110 | ||
248c7f9c | 111 | static void print_benchmark(bool running) |
b4b8faa1 | 112 | { |
248c7f9c | 113 | const char *bench_str = "INVALID"; |
b4b8faa1 | 114 | |
248c7f9c MK |
115 | if (opt_bench == BENCH_RXDROP) |
116 | bench_str = "rxdrop"; | |
117 | else if (opt_bench == BENCH_TXONLY) | |
118 | bench_str = "txonly"; | |
119 | else if (opt_bench == BENCH_L2FWD) | |
120 | bench_str = "l2fwd"; | |
b4b8faa1 | 121 | |
248c7f9c MK |
122 | printf("%s:%d %s ", opt_if, opt_queue, bench_str); |
123 | if (opt_xdp_flags & XDP_FLAGS_SKB_MODE) | |
124 | printf("xdp-skb "); | |
125 | else if (opt_xdp_flags & XDP_FLAGS_DRV_MODE) | |
126 | printf("xdp-drv "); | |
127 | else | |
128 | printf(" "); | |
b4b8faa1 | 129 | |
248c7f9c MK |
130 | if (opt_poll) |
131 | printf("poll() "); | |
b4b8faa1 | 132 | |
248c7f9c MK |
133 | if (running) { |
134 | printf("running..."); | |
135 | fflush(stdout); | |
b4b8faa1 | 136 | } |
b4b8faa1 MK |
137 | } |
138 | ||
248c7f9c | 139 | static void dump_stats(void) |
b4b8faa1 | 140 | { |
248c7f9c MK |
141 | unsigned long now = get_nsecs(); |
142 | long dt = now - prev_time; | |
143 | int i; | |
b4b8faa1 | 144 | |
248c7f9c | 145 | prev_time = now; |
b4b8faa1 | 146 | |
248c7f9c MK |
147 | for (i = 0; i < num_socks && xsks[i]; i++) { |
148 | char *fmt = "%-15s %'-11.0f %'-11lu\n"; | |
149 | double rx_pps, tx_pps; | |
b4b8faa1 | 150 | |
248c7f9c MK |
151 | rx_pps = (xsks[i]->rx_npkts - xsks[i]->prev_rx_npkts) * |
152 | 1000000000. / dt; | |
153 | tx_pps = (xsks[i]->tx_npkts - xsks[i]->prev_tx_npkts) * | |
154 | 1000000000. / dt; | |
b4b8faa1 | 155 | |
248c7f9c MK |
156 | printf("\n sock%d@", i); |
157 | print_benchmark(false); | |
158 | printf("\n"); | |
b4b8faa1 | 159 | |
248c7f9c MK |
160 | printf("%-15s %-11s %-11s %-11.2f\n", "", "pps", "pkts", |
161 | dt / 1000000000.); | |
162 | printf(fmt, "rx", rx_pps, xsks[i]->rx_npkts); | |
163 | printf(fmt, "tx", tx_pps, xsks[i]->tx_npkts); | |
b4b8faa1 | 164 | |
248c7f9c MK |
165 | xsks[i]->prev_rx_npkts = xsks[i]->rx_npkts; |
166 | xsks[i]->prev_tx_npkts = xsks[i]->tx_npkts; | |
b4b8faa1 | 167 | } |
b4b8faa1 MK |
168 | } |
169 | ||
248c7f9c | 170 | static void *poller(void *arg) |
b4b8faa1 | 171 | { |
248c7f9c MK |
172 | (void)arg; |
173 | for (;;) { | |
174 | sleep(opt_interval); | |
175 | dump_stats(); | |
b4b8faa1 MK |
176 | } |
177 | ||
248c7f9c | 178 | return NULL; |
b4b8faa1 MK |
179 | } |
180 | ||
248c7f9c | 181 | static void remove_xdp_program(void) |
b4b8faa1 | 182 | { |
2e5d72c1 | 183 | u32 curr_prog_id = 0; |
b4b8faa1 | 184 | |
248c7f9c MK |
185 | if (bpf_get_link_xdp_id(opt_ifindex, &curr_prog_id, opt_xdp_flags)) { |
186 | printf("bpf_get_link_xdp_id failed\n"); | |
187 | exit(EXIT_FAILURE); | |
b4b8faa1 | 188 | } |
248c7f9c MK |
189 | if (prog_id == curr_prog_id) |
190 | bpf_set_link_xdp_fd(opt_ifindex, -1, opt_xdp_flags); | |
191 | else if (!curr_prog_id) | |
192 | printf("couldn't find a prog id on a given interface\n"); | |
193 | else | |
194 | printf("program on interface changed, not removing\n"); | |
b4b8faa1 MK |
195 | } |
196 | ||
248c7f9c | 197 | static void int_exit(int sig) |
b4b8faa1 | 198 | { |
248c7f9c | 199 | struct xsk_umem *umem = xsks[0]->umem->umem; |
2e5d72c1 | 200 | int i; |
b4b8faa1 | 201 | |
248c7f9c | 202 | dump_stats(); |
2e5d72c1 MK |
203 | for (i = 0; i < num_socks; i++) |
204 | xsk_socket__delete(xsks[i]->xsk); | |
248c7f9c MK |
205 | (void)xsk_umem__delete(umem); |
206 | remove_xdp_program(); | |
b4b8faa1 | 207 | |
248c7f9c | 208 | exit(EXIT_SUCCESS); |
b4b8faa1 MK |
209 | } |
210 | ||
248c7f9c MK |
211 | static void __exit_with_error(int error, const char *file, const char *func, |
212 | int line) | |
b4b8faa1 | 213 | { |
248c7f9c MK |
214 | fprintf(stderr, "%s:%s:%i: errno: %d/\"%s\"\n", file, func, |
215 | line, error, strerror(error)); | |
216 | dump_stats(); | |
217 | remove_xdp_program(); | |
218 | exit(EXIT_FAILURE); | |
b4b8faa1 MK |
219 | } |
220 | ||
248c7f9c MK |
221 | #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, \ |
222 | __LINE__) | |
b4b8faa1 | 223 | |
248c7f9c MK |
224 | static const char pkt_data[] = |
225 | "\x3c\xfd\xfe\x9e\x7f\x71\xec\xb1\xd7\x98\x3a\xc0\x08\x00\x45\x00" | |
226 | "\x00\x2e\x00\x00\x00\x00\x40\x11\x88\x97\x05\x08\x07\x08\xc8\x14" | |
227 | "\x1e\x04\x10\x92\x10\x92\x00\x1a\x6d\xa3\x34\x33\x1f\x69\x40\x6b" | |
228 | "\x54\x59\xb6\x14\x2d\x11\x44\xbf\xaf\xd9\xbe\xaa"; | |
b4b8faa1 MK |
229 | |
230 | static void swap_mac_addresses(void *data) | |
231 | { | |
232 | struct ether_header *eth = (struct ether_header *)data; | |
233 | struct ether_addr *src_addr = (struct ether_addr *)ð->ether_shost; | |
234 | struct ether_addr *dst_addr = (struct ether_addr *)ð->ether_dhost; | |
235 | struct ether_addr tmp; | |
236 | ||
237 | tmp = *src_addr; | |
238 | *src_addr = *dst_addr; | |
239 | *dst_addr = tmp; | |
240 | } | |
241 | ||
a412ef54 | 242 | static void hex_dump(void *pkt, size_t length, u64 addr) |
b4b8faa1 | 243 | { |
b4b8faa1 MK |
244 | const unsigned char *address = (unsigned char *)pkt; |
245 | const unsigned char *line = address; | |
246 | size_t line_size = 32; | |
247 | unsigned char c; | |
a412ef54 BT |
248 | char buf[32]; |
249 | int i = 0; | |
b4b8faa1 | 250 | |
a412ef54 BT |
251 | if (!DEBUG_HEXDUMP) |
252 | return; | |
253 | ||
254 | sprintf(buf, "addr=%llu", addr); | |
b4b8faa1 | 255 | printf("length = %zu\n", length); |
a412ef54 | 256 | printf("%s | ", buf); |
b4b8faa1 MK |
257 | while (length-- > 0) { |
258 | printf("%02X ", *address++); | |
259 | if (!(++i % line_size) || (length == 0 && i % line_size)) { | |
260 | if (length == 0) { | |
261 | while (i++ % line_size) | |
262 | printf("__ "); | |
263 | } | |
264 | printf(" | "); /* right close */ | |
265 | while (line < address) { | |
266 | c = *line++; | |
267 | printf("%c", (c < 33 || c == 255) ? 0x2E : c); | |
268 | } | |
269 | printf("\n"); | |
270 | if (length > 0) | |
a412ef54 | 271 | printf("%s | ", buf); |
b4b8faa1 MK |
272 | } |
273 | } | |
274 | printf("\n"); | |
275 | } | |
b4b8faa1 | 276 | |
248c7f9c | 277 | static size_t gen_eth_frame(struct xsk_umem_info *umem, u64 addr) |
b4b8faa1 | 278 | { |
248c7f9c MK |
279 | memcpy(xsk_umem__get_data(umem->buffer, addr), pkt_data, |
280 | sizeof(pkt_data) - 1); | |
b4b8faa1 MK |
281 | return sizeof(pkt_data) - 1; |
282 | } | |
283 | ||
248c7f9c | 284 | static struct xsk_umem_info *xsk_configure_umem(void *buffer, u64 size) |
b4b8faa1 | 285 | { |
248c7f9c | 286 | struct xsk_umem_info *umem; |
123e8da1 MM |
287 | struct xsk_umem_config cfg = { |
288 | .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS, | |
289 | .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, | |
290 | .frame_size = opt_xsk_frame_size, | |
291 | .frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM, | |
c543f546 | 292 | .flags = opt_umem_flags |
123e8da1 | 293 | }; |
661842c4 | 294 | int ret; |
b4b8faa1 MK |
295 | |
296 | umem = calloc(1, sizeof(*umem)); | |
248c7f9c MK |
297 | if (!umem) |
298 | exit_with_error(errno); | |
b4b8faa1 | 299 | |
248c7f9c | 300 | ret = xsk_umem__create(&umem->umem, buffer, size, &umem->fq, &umem->cq, |
123e8da1 | 301 | &cfg); |
248c7f9c MK |
302 | if (ret) |
303 | exit_with_error(-ret); | |
b4b8faa1 | 304 | |
661842c4 MK |
305 | umem->buffer = buffer; |
306 | return umem; | |
307 | } | |
308 | ||
309 | static void xsk_populate_fill_ring(struct xsk_umem_info *umem) | |
310 | { | |
311 | int ret, i; | |
312 | u32 idx; | |
313 | ||
2e5d72c1 MK |
314 | ret = xsk_ring_prod__reserve(&umem->fq, |
315 | XSK_RING_PROD__DEFAULT_NUM_DESCS, &idx); | |
316 | if (ret != XSK_RING_PROD__DEFAULT_NUM_DESCS) | |
317 | exit_with_error(-ret); | |
318 | for (i = 0; i < XSK_RING_PROD__DEFAULT_NUM_DESCS; i++) | |
319 | *xsk_ring_prod__fill_addr(&umem->fq, idx++) = | |
320 | i * opt_xsk_frame_size; | |
321 | xsk_ring_prod__submit(&umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS); | |
b4b8faa1 MK |
322 | } |
323 | ||
661842c4 MK |
324 | static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem, |
325 | bool rx, bool tx) | |
b4b8faa1 | 326 | { |
248c7f9c MK |
327 | struct xsk_socket_config cfg; |
328 | struct xsk_socket_info *xsk; | |
661842c4 MK |
329 | struct xsk_ring_cons *rxr; |
330 | struct xsk_ring_prod *txr; | |
248c7f9c | 331 | int ret; |
b4b8faa1 MK |
332 | |
333 | xsk = calloc(1, sizeof(*xsk)); | |
248c7f9c MK |
334 | if (!xsk) |
335 | exit_with_error(errno); | |
336 | ||
337 | xsk->umem = umem; | |
338 | cfg.rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS; | |
339 | cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; | |
2e5d72c1 MK |
340 | if (opt_num_xsks > 1) |
341 | cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD; | |
342 | else | |
343 | cfg.libbpf_flags = 0; | |
248c7f9c MK |
344 | cfg.xdp_flags = opt_xdp_flags; |
345 | cfg.bind_flags = opt_xdp_bind_flags; | |
2e5d72c1 | 346 | |
661842c4 MK |
347 | rxr = rx ? &xsk->rx : NULL; |
348 | txr = tx ? &xsk->tx : NULL; | |
349 | ret = xsk_socket__create(&xsk->xsk, opt_if, opt_queue, umem->umem, | |
350 | rxr, txr, &cfg); | |
248c7f9c MK |
351 | if (ret) |
352 | exit_with_error(-ret); | |
353 | ||
354 | ret = bpf_get_link_xdp_id(opt_ifindex, &prog_id, opt_xdp_flags); | |
355 | if (ret) | |
356 | exit_with_error(-ret); | |
357 | ||
b4b8faa1 MK |
358 | return xsk; |
359 | } | |
360 | ||
b4b8faa1 MK |
361 | static struct option long_options[] = { |
362 | {"rxdrop", no_argument, 0, 'r'}, | |
363 | {"txonly", no_argument, 0, 't'}, | |
364 | {"l2fwd", no_argument, 0, 'l'}, | |
365 | {"interface", required_argument, 0, 'i'}, | |
366 | {"queue", required_argument, 0, 'q'}, | |
367 | {"poll", no_argument, 0, 'p'}, | |
b4b8faa1 MK |
368 | {"xdp-skb", no_argument, 0, 'S'}, |
369 | {"xdp-native", no_argument, 0, 'N'}, | |
370 | {"interval", required_argument, 0, 'n'}, | |
58c50ae4 BT |
371 | {"zero-copy", no_argument, 0, 'z'}, |
372 | {"copy", no_argument, 0, 'c'}, | |
123e8da1 | 373 | {"frame-size", required_argument, 0, 'f'}, |
46738f73 | 374 | {"no-need-wakeup", no_argument, 0, 'm'}, |
c543f546 | 375 | {"unaligned", no_argument, 0, 'u'}, |
2e5d72c1 | 376 | {"shared-umem", no_argument, 0, 'M'}, |
b4b8faa1 MK |
377 | {0, 0, 0, 0} |
378 | }; | |
379 | ||
380 | static void usage(const char *prog) | |
381 | { | |
382 | const char *str = | |
383 | " Usage: %s [OPTIONS]\n" | |
384 | " Options:\n" | |
385 | " -r, --rxdrop Discard all incoming packets (default)\n" | |
386 | " -t, --txonly Only send packets\n" | |
387 | " -l, --l2fwd MAC swap L2 forwarding\n" | |
388 | " -i, --interface=n Run on interface n\n" | |
389 | " -q, --queue=n Use queue n (default 0)\n" | |
390 | " -p, --poll Use poll syscall\n" | |
b4b8faa1 | 391 | " -S, --xdp-skb=n Use XDP skb-mod\n" |
4564a8bb | 392 | " -N, --xdp-native=n Enforce XDP native mode\n" |
b4b8faa1 | 393 | " -n, --interval=n Specify statistics update interval (default 1 sec).\n" |
58c50ae4 BT |
394 | " -z, --zero-copy Force zero-copy mode.\n" |
395 | " -c, --copy Force copy mode.\n" | |
46738f73 | 396 | " -m, --no-need-wakeup Turn off use of driver need wakeup flag.\n" |
c543f546 KL |
397 | " -f, --frame-size=n Set the frame size (must be a power of two in aligned mode, default is %d).\n" |
398 | " -u, --unaligned Enable unaligned chunk placement\n" | |
2e5d72c1 | 399 | " -M, --shared-umem Enable XDP_SHARED_UMEM\n" |
b4b8faa1 | 400 | "\n"; |
123e8da1 | 401 | fprintf(stderr, str, prog, XSK_UMEM__DEFAULT_FRAME_SIZE); |
b4b8faa1 MK |
402 | exit(EXIT_FAILURE); |
403 | } | |
404 | ||
405 | static void parse_command_line(int argc, char **argv) | |
406 | { | |
407 | int option_index, c; | |
408 | ||
409 | opterr = 0; | |
410 | ||
411 | for (;;) { | |
2e5d72c1 | 412 | c = getopt_long(argc, argv, "Frtli:q:psSNn:czf:muM", |
46738f73 | 413 | long_options, &option_index); |
b4b8faa1 MK |
414 | if (c == -1) |
415 | break; | |
416 | ||
417 | switch (c) { | |
418 | case 'r': | |
419 | opt_bench = BENCH_RXDROP; | |
420 | break; | |
421 | case 't': | |
422 | opt_bench = BENCH_TXONLY; | |
423 | break; | |
424 | case 'l': | |
425 | opt_bench = BENCH_L2FWD; | |
426 | break; | |
427 | case 'i': | |
428 | opt_if = optarg; | |
429 | break; | |
430 | case 'q': | |
431 | opt_queue = atoi(optarg); | |
432 | break; | |
b4b8faa1 MK |
433 | case 'p': |
434 | opt_poll = 1; | |
435 | break; | |
436 | case 'S': | |
437 | opt_xdp_flags |= XDP_FLAGS_SKB_MODE; | |
9f5232cc | 438 | opt_xdp_bind_flags |= XDP_COPY; |
b4b8faa1 MK |
439 | break; |
440 | case 'N': | |
441 | opt_xdp_flags |= XDP_FLAGS_DRV_MODE; | |
442 | break; | |
443 | case 'n': | |
444 | opt_interval = atoi(optarg); | |
445 | break; | |
58c50ae4 BT |
446 | case 'z': |
447 | opt_xdp_bind_flags |= XDP_ZEROCOPY; | |
448 | break; | |
449 | case 'c': | |
450 | opt_xdp_bind_flags |= XDP_COPY; | |
451 | break; | |
c543f546 KL |
452 | case 'u': |
453 | opt_umem_flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG; | |
454 | opt_unaligned_chunks = 1; | |
3945b37a | 455 | opt_mmap_flags = MAP_HUGETLB; |
c543f546 | 456 | break; |
743e568c MF |
457 | case 'F': |
458 | opt_xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST; | |
459 | break; | |
123e8da1 MM |
460 | case 'f': |
461 | opt_xsk_frame_size = atoi(optarg); | |
2e5d72c1 | 462 | break; |
46738f73 MK |
463 | case 'm': |
464 | opt_need_wakeup = false; | |
465 | opt_xdp_bind_flags &= ~XDP_USE_NEED_WAKEUP; | |
123e8da1 | 466 | break; |
2e5d72c1 MK |
467 | case 'M': |
468 | opt_num_xsks = MAX_SOCKS; | |
469 | break; | |
b4b8faa1 MK |
470 | default: |
471 | usage(basename(argv[0])); | |
472 | } | |
473 | } | |
474 | ||
475 | opt_ifindex = if_nametoindex(opt_if); | |
476 | if (!opt_ifindex) { | |
477 | fprintf(stderr, "ERROR: interface \"%s\" does not exist\n", | |
478 | opt_if); | |
479 | usage(basename(argv[0])); | |
480 | } | |
248c7f9c | 481 | |
c543f546 KL |
482 | if ((opt_xsk_frame_size & (opt_xsk_frame_size - 1)) && |
483 | !opt_unaligned_chunks) { | |
123e8da1 MM |
484 | fprintf(stderr, "--frame-size=%d is not a power of two\n", |
485 | opt_xsk_frame_size); | |
486 | usage(basename(argv[0])); | |
487 | } | |
b4b8faa1 MK |
488 | } |
489 | ||
248c7f9c | 490 | static void kick_tx(struct xsk_socket_info *xsk) |
b4b8faa1 MK |
491 | { |
492 | int ret; | |
493 | ||
248c7f9c | 494 | ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); |
c03079c9 | 495 | if (ret >= 0 || errno == ENOBUFS || errno == EAGAIN || errno == EBUSY) |
b4b8faa1 | 496 | return; |
248c7f9c | 497 | exit_with_error(errno); |
b4b8faa1 MK |
498 | } |
499 | ||
46738f73 MK |
500 | static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk, |
501 | struct pollfd *fds) | |
b4b8faa1 | 502 | { |
03895e63 | 503 | struct xsk_umem_info *umem = xsk->umem; |
b74e21ab | 504 | u32 idx_cq = 0, idx_fq = 0; |
b4b8faa1 MK |
505 | unsigned int rcvd; |
506 | size_t ndescs; | |
507 | ||
508 | if (!xsk->outstanding_tx) | |
509 | return; | |
510 | ||
46738f73 MK |
511 | if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx)) |
512 | kick_tx(xsk); | |
513 | ||
b4b8faa1 | 514 | ndescs = (xsk->outstanding_tx > BATCH_SIZE) ? BATCH_SIZE : |
248c7f9c | 515 | xsk->outstanding_tx; |
b4b8faa1 MK |
516 | |
517 | /* re-add completed Tx buffers */ | |
03895e63 | 518 | rcvd = xsk_ring_cons__peek(&umem->cq, ndescs, &idx_cq); |
b4b8faa1 | 519 | if (rcvd > 0) { |
248c7f9c MK |
520 | unsigned int i; |
521 | int ret; | |
522 | ||
03895e63 | 523 | ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); |
248c7f9c MK |
524 | while (ret != rcvd) { |
525 | if (ret < 0) | |
526 | exit_with_error(-ret); | |
03895e63 | 527 | if (xsk_ring_prod__needs_wakeup(&umem->fq)) |
46738f73 | 528 | ret = poll(fds, num_socks, opt_timeout); |
03895e63 | 529 | ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); |
248c7f9c | 530 | } |
03895e63 | 531 | |
248c7f9c | 532 | for (i = 0; i < rcvd; i++) |
03895e63 KL |
533 | *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = |
534 | *xsk_ring_cons__comp_addr(&umem->cq, idx_cq++); | |
248c7f9c MK |
535 | |
536 | xsk_ring_prod__submit(&xsk->umem->fq, rcvd); | |
537 | xsk_ring_cons__release(&xsk->umem->cq, rcvd); | |
b4b8faa1 MK |
538 | xsk->outstanding_tx -= rcvd; |
539 | xsk->tx_npkts += rcvd; | |
540 | } | |
541 | } | |
542 | ||
248c7f9c | 543 | static inline void complete_tx_only(struct xsk_socket_info *xsk) |
b4b8faa1 | 544 | { |
b4b8faa1 | 545 | unsigned int rcvd; |
248c7f9c | 546 | u32 idx; |
b4b8faa1 MK |
547 | |
548 | if (!xsk->outstanding_tx) | |
549 | return; | |
550 | ||
46738f73 MK |
551 | if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx)) |
552 | kick_tx(xsk); | |
b4b8faa1 | 553 | |
248c7f9c | 554 | rcvd = xsk_ring_cons__peek(&xsk->umem->cq, BATCH_SIZE, &idx); |
b4b8faa1 | 555 | if (rcvd > 0) { |
248c7f9c | 556 | xsk_ring_cons__release(&xsk->umem->cq, rcvd); |
b4b8faa1 MK |
557 | xsk->outstanding_tx -= rcvd; |
558 | xsk->tx_npkts += rcvd; | |
559 | } | |
560 | } | |
561 | ||
46738f73 | 562 | static void rx_drop(struct xsk_socket_info *xsk, struct pollfd *fds) |
b4b8faa1 | 563 | { |
b4b8faa1 | 564 | unsigned int rcvd, i; |
b74e21ab | 565 | u32 idx_rx = 0, idx_fq = 0; |
248c7f9c | 566 | int ret; |
b4b8faa1 | 567 | |
248c7f9c | 568 | rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx); |
46738f73 MK |
569 | if (!rcvd) { |
570 | if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) | |
571 | ret = poll(fds, num_socks, opt_timeout); | |
b4b8faa1 | 572 | return; |
46738f73 | 573 | } |
b4b8faa1 | 574 | |
248c7f9c MK |
575 | ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq); |
576 | while (ret != rcvd) { | |
577 | if (ret < 0) | |
578 | exit_with_error(-ret); | |
46738f73 MK |
579 | if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) |
580 | ret = poll(fds, num_socks, opt_timeout); | |
248c7f9c MK |
581 | ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq); |
582 | } | |
583 | ||
b4b8faa1 | 584 | for (i = 0; i < rcvd; i++) { |
248c7f9c MK |
585 | u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr; |
586 | u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len; | |
03895e63 KL |
587 | u64 orig = xsk_umem__extract_addr(addr); |
588 | ||
589 | addr = xsk_umem__add_offset_to_addr(addr); | |
248c7f9c | 590 | char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr); |
b4b8faa1 | 591 | |
248c7f9c | 592 | hex_dump(pkt, len, addr); |
03895e63 | 593 | *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx_fq++) = orig; |
b4b8faa1 MK |
594 | } |
595 | ||
248c7f9c MK |
596 | xsk_ring_prod__submit(&xsk->umem->fq, rcvd); |
597 | xsk_ring_cons__release(&xsk->rx, rcvd); | |
b4b8faa1 | 598 | xsk->rx_npkts += rcvd; |
b4b8faa1 MK |
599 | } |
600 | ||
601 | static void rx_drop_all(void) | |
602 | { | |
2e5d72c1 | 603 | struct pollfd fds[MAX_SOCKS] = {}; |
46738f73 | 604 | int i, ret; |
b4b8faa1 | 605 | |
b4b8faa1 | 606 | for (i = 0; i < num_socks; i++) { |
248c7f9c | 607 | fds[i].fd = xsk_socket__fd(xsks[i]->xsk); |
b4b8faa1 | 608 | fds[i].events = POLLIN; |
b4b8faa1 MK |
609 | } |
610 | ||
611 | for (;;) { | |
612 | if (opt_poll) { | |
46738f73 | 613 | ret = poll(fds, num_socks, opt_timeout); |
b4b8faa1 MK |
614 | if (ret <= 0) |
615 | continue; | |
616 | } | |
617 | ||
618 | for (i = 0; i < num_socks; i++) | |
46738f73 MK |
619 | rx_drop(xsks[i], fds); |
620 | } | |
621 | } | |
622 | ||
623 | static void tx_only(struct xsk_socket_info *xsk, u32 frame_nb) | |
624 | { | |
625 | u32 idx; | |
626 | ||
627 | if (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) == BATCH_SIZE) { | |
628 | unsigned int i; | |
629 | ||
630 | for (i = 0; i < BATCH_SIZE; i++) { | |
631 | xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->addr = | |
632 | (frame_nb + i) << XSK_UMEM__DEFAULT_FRAME_SHIFT; | |
633 | xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->len = | |
634 | sizeof(pkt_data) - 1; | |
635 | } | |
636 | ||
637 | xsk_ring_prod__submit(&xsk->tx, BATCH_SIZE); | |
638 | xsk->outstanding_tx += BATCH_SIZE; | |
639 | frame_nb += BATCH_SIZE; | |
640 | frame_nb %= NUM_FRAMES; | |
b4b8faa1 | 641 | } |
46738f73 MK |
642 | |
643 | complete_tx_only(xsk); | |
b4b8faa1 MK |
644 | } |
645 | ||
46738f73 | 646 | static void tx_only_all(void) |
b4b8faa1 | 647 | { |
2e5d72c1 | 648 | struct pollfd fds[MAX_SOCKS] = {}; |
46738f73 MK |
649 | u32 frame_nb[MAX_SOCKS] = {}; |
650 | int i, ret; | |
b4b8faa1 | 651 | |
46738f73 MK |
652 | for (i = 0; i < num_socks; i++) { |
653 | fds[0].fd = xsk_socket__fd(xsks[i]->xsk); | |
654 | fds[0].events = POLLOUT; | |
655 | } | |
b4b8faa1 MK |
656 | |
657 | for (;;) { | |
658 | if (opt_poll) { | |
46738f73 | 659 | ret = poll(fds, num_socks, opt_timeout); |
b4b8faa1 MK |
660 | if (ret <= 0) |
661 | continue; | |
662 | ||
248c7f9c | 663 | if (!(fds[0].revents & POLLOUT)) |
b4b8faa1 MK |
664 | continue; |
665 | } | |
666 | ||
46738f73 MK |
667 | for (i = 0; i < num_socks; i++) |
668 | tx_only(xsks[i], frame_nb[i]); | |
b4b8faa1 MK |
669 | } |
670 | } | |
671 | ||
46738f73 | 672 | static void l2fwd(struct xsk_socket_info *xsk, struct pollfd *fds) |
b4b8faa1 | 673 | { |
46738f73 MK |
674 | unsigned int rcvd, i; |
675 | u32 idx_rx = 0, idx_tx = 0; | |
676 | int ret; | |
b4b8faa1 | 677 | |
46738f73 | 678 | complete_tx_l2fwd(xsk, fds); |
b4b8faa1 | 679 | |
46738f73 MK |
680 | rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx); |
681 | if (!rcvd) { | |
682 | if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) | |
683 | ret = poll(fds, num_socks, opt_timeout); | |
684 | return; | |
685 | } | |
b4b8faa1 | 686 | |
46738f73 MK |
687 | ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx); |
688 | while (ret != rcvd) { | |
689 | if (ret < 0) | |
690 | exit_with_error(-ret); | |
691 | if (xsk_ring_prod__needs_wakeup(&xsk->tx)) | |
692 | kick_tx(xsk); | |
248c7f9c | 693 | ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx); |
46738f73 MK |
694 | } |
695 | ||
696 | for (i = 0; i < rcvd; i++) { | |
697 | u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr; | |
698 | u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len; | |
5a712e13 | 699 | u64 orig = addr; |
03895e63 KL |
700 | |
701 | addr = xsk_umem__add_offset_to_addr(addr); | |
46738f73 MK |
702 | char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr); |
703 | ||
704 | swap_mac_addresses(pkt); | |
248c7f9c | 705 | |
46738f73 | 706 | hex_dump(pkt, len, addr); |
03895e63 | 707 | xsk_ring_prod__tx_desc(&xsk->tx, idx_tx)->addr = orig; |
46738f73 MK |
708 | xsk_ring_prod__tx_desc(&xsk->tx, idx_tx++)->len = len; |
709 | } | |
b4b8faa1 | 710 | |
46738f73 MK |
711 | xsk_ring_prod__submit(&xsk->tx, rcvd); |
712 | xsk_ring_cons__release(&xsk->rx, rcvd); | |
b4b8faa1 | 713 | |
46738f73 MK |
714 | xsk->rx_npkts += rcvd; |
715 | xsk->outstanding_tx += rcvd; | |
716 | } | |
717 | ||
718 | static void l2fwd_all(void) | |
719 | { | |
2e5d72c1 | 720 | struct pollfd fds[MAX_SOCKS] = {}; |
46738f73 MK |
721 | int i, ret; |
722 | ||
46738f73 MK |
723 | for (i = 0; i < num_socks; i++) { |
724 | fds[i].fd = xsk_socket__fd(xsks[i]->xsk); | |
725 | fds[i].events = POLLOUT | POLLIN; | |
726 | } | |
b4b8faa1 | 727 | |
46738f73 MK |
728 | for (;;) { |
729 | if (opt_poll) { | |
730 | ret = poll(fds, num_socks, opt_timeout); | |
731 | if (ret <= 0) | |
732 | continue; | |
733 | } | |
b4b8faa1 | 734 | |
46738f73 MK |
735 | for (i = 0; i < num_socks; i++) |
736 | l2fwd(xsks[i], fds); | |
b4b8faa1 MK |
737 | } |
738 | } | |
739 | ||
2e5d72c1 MK |
740 | static void load_xdp_program(char **argv, struct bpf_object **obj) |
741 | { | |
742 | struct bpf_prog_load_attr prog_load_attr = { | |
743 | .prog_type = BPF_PROG_TYPE_XDP, | |
744 | }; | |
745 | char xdp_filename[256]; | |
746 | int prog_fd; | |
747 | ||
748 | snprintf(xdp_filename, sizeof(xdp_filename), "%s_kern.o", argv[0]); | |
749 | prog_load_attr.file = xdp_filename; | |
750 | ||
751 | if (bpf_prog_load_xattr(&prog_load_attr, obj, &prog_fd)) | |
752 | exit(EXIT_FAILURE); | |
753 | if (prog_fd < 0) { | |
754 | fprintf(stderr, "ERROR: no program found: %s\n", | |
755 | strerror(prog_fd)); | |
756 | exit(EXIT_FAILURE); | |
757 | } | |
758 | ||
759 | if (bpf_set_link_xdp_fd(opt_ifindex, prog_fd, opt_xdp_flags) < 0) { | |
760 | fprintf(stderr, "ERROR: link set xdp fd failed\n"); | |
761 | exit(EXIT_FAILURE); | |
762 | } | |
763 | } | |
764 | ||
765 | static void enter_xsks_into_map(struct bpf_object *obj) | |
766 | { | |
767 | struct bpf_map *map; | |
768 | int i, xsks_map; | |
769 | ||
770 | map = bpf_object__find_map_by_name(obj, "xsks_map"); | |
771 | xsks_map = bpf_map__fd(map); | |
772 | if (xsks_map < 0) { | |
773 | fprintf(stderr, "ERROR: no xsks map found: %s\n", | |
774 | strerror(xsks_map)); | |
775 | exit(EXIT_FAILURE); | |
776 | } | |
777 | ||
778 | for (i = 0; i < num_socks; i++) { | |
779 | int fd = xsk_socket__fd(xsks[i]->xsk); | |
780 | int key, ret; | |
781 | ||
782 | key = i; | |
783 | ret = bpf_map_update_elem(xsks_map, &key, &fd, 0); | |
784 | if (ret) { | |
785 | fprintf(stderr, "ERROR: bpf_map_update_elem %d\n", i); | |
786 | exit(EXIT_FAILURE); | |
787 | } | |
788 | } | |
789 | } | |
790 | ||
b4b8faa1 MK |
791 | int main(int argc, char **argv) |
792 | { | |
793 | struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY}; | |
661842c4 | 794 | bool rx = false, tx = false; |
248c7f9c | 795 | struct xsk_umem_info *umem; |
2e5d72c1 | 796 | struct bpf_object *obj; |
b4b8faa1 | 797 | pthread_t pt; |
2e5d72c1 | 798 | int i, ret; |
248c7f9c | 799 | void *bufs; |
b4b8faa1 MK |
800 | |
801 | parse_command_line(argc, argv); | |
802 | ||
803 | if (setrlimit(RLIMIT_MEMLOCK, &r)) { | |
804 | fprintf(stderr, "ERROR: setrlimit(RLIMIT_MEMLOCK) \"%s\"\n", | |
805 | strerror(errno)); | |
806 | exit(EXIT_FAILURE); | |
807 | } | |
808 | ||
2e5d72c1 MK |
809 | if (opt_num_xsks > 1) |
810 | load_xdp_program(argv, &obj); | |
811 | ||
3945b37a KL |
812 | /* Reserve memory for the umem. Use hugepages if unaligned chunk mode */ |
813 | bufs = mmap(NULL, NUM_FRAMES * opt_xsk_frame_size, | |
814 | PROT_READ | PROT_WRITE, | |
815 | MAP_PRIVATE | MAP_ANONYMOUS | opt_mmap_flags, -1, 0); | |
816 | if (bufs == MAP_FAILED) { | |
817 | printf("ERROR: mmap failed\n"); | |
818 | exit(EXIT_FAILURE); | |
819 | } | |
2e5d72c1 MK |
820 | |
821 | /* Create sockets... */ | |
123e8da1 | 822 | umem = xsk_configure_umem(bufs, NUM_FRAMES * opt_xsk_frame_size); |
661842c4 MK |
823 | if (opt_bench == BENCH_RXDROP || opt_bench == BENCH_L2FWD) { |
824 | rx = true; | |
825 | xsk_populate_fill_ring(umem); | |
826 | } | |
827 | if (opt_bench == BENCH_L2FWD || opt_bench == BENCH_TXONLY) | |
828 | tx = true; | |
2e5d72c1 | 829 | for (i = 0; i < opt_num_xsks; i++) |
661842c4 | 830 | xsks[num_socks++] = xsk_configure_socket(umem, rx, tx); |
b4b8faa1 | 831 | |
661842c4 MK |
832 | if (opt_bench == BENCH_TXONLY) |
833 | for (i = 0; i < NUM_FRAMES; i++) | |
834 | gen_eth_frame(umem, i * opt_xsk_frame_size); | |
b4b8faa1 | 835 | |
2e5d72c1 MK |
836 | if (opt_num_xsks > 1 && opt_bench != BENCH_TXONLY) |
837 | enter_xsks_into_map(obj); | |
b4b8faa1 MK |
838 | |
839 | signal(SIGINT, int_exit); | |
840 | signal(SIGTERM, int_exit); | |
841 | signal(SIGABRT, int_exit); | |
842 | ||
843 | setlocale(LC_ALL, ""); | |
844 | ||
845 | ret = pthread_create(&pt, NULL, poller, NULL); | |
248c7f9c MK |
846 | if (ret) |
847 | exit_with_error(ret); | |
b4b8faa1 MK |
848 | |
849 | prev_time = get_nsecs(); | |
850 | ||
851 | if (opt_bench == BENCH_RXDROP) | |
852 | rx_drop_all(); | |
853 | else if (opt_bench == BENCH_TXONLY) | |
46738f73 | 854 | tx_only_all(); |
b4b8faa1 | 855 | else |
46738f73 | 856 | l2fwd_all(); |
b4b8faa1 MK |
857 | |
858 | return 0; | |
859 | } |