]>
Commit | Line | Data |
---|---|---|
b4b8faa1 | 1 | // SPDX-License-Identifier: GPL-2.0 |
dac09149 | 2 | /* Copyright(c) 2017 - 2018 Intel Corporation. */ |
b4b8faa1 | 3 | |
248c7f9c | 4 | #include <asm/barrier.h> |
b4b8faa1 MK |
5 | #include <errno.h> |
6 | #include <getopt.h> | |
7 | #include <libgen.h> | |
8 | #include <linux/bpf.h> | |
248c7f9c | 9 | #include <linux/compiler.h> |
b4b8faa1 MK |
10 | #include <linux/if_link.h> |
11 | #include <linux/if_xdp.h> | |
12 | #include <linux/if_ether.h> | |
248c7f9c MK |
13 | #include <locale.h> |
14 | #include <net/ethernet.h> | |
b4b8faa1 | 15 | #include <net/if.h> |
248c7f9c MK |
16 | #include <poll.h> |
17 | #include <pthread.h> | |
b4b8faa1 MK |
18 | #include <signal.h> |
19 | #include <stdbool.h> | |
20 | #include <stdio.h> | |
21 | #include <stdlib.h> | |
22 | #include <string.h> | |
248c7f9c | 23 | #include <sys/mman.h> |
b4b8faa1 MK |
24 | #include <sys/resource.h> |
25 | #include <sys/socket.h> | |
248c7f9c | 26 | #include <sys/types.h> |
b4b8faa1 MK |
27 | #include <time.h> |
28 | #include <unistd.h> | |
b4b8faa1 | 29 | |
4d18f6de DL |
30 | #include "libbpf.h" |
31 | #include "xsk.h" | |
2e5d72c1 | 32 | #include "xdpsock.h" |
2bf3e2ef | 33 | #include <bpf/bpf.h> |
b4b8faa1 | 34 | |
b4b8faa1 MK |
35 | #ifndef SOL_XDP |
36 | #define SOL_XDP 283 | |
37 | #endif | |
38 | ||
39 | #ifndef AF_XDP | |
40 | #define AF_XDP 44 | |
41 | #endif | |
42 | ||
43 | #ifndef PF_XDP | |
44 | #define PF_XDP AF_XDP | |
45 | #endif | |
46 | ||
248c7f9c | 47 | #define NUM_FRAMES (4 * 1024) |
b4b8faa1 MK |
48 | |
49 | #define DEBUG_HEXDUMP 0 | |
50 | ||
a412ef54 | 51 | typedef __u64 u64; |
b4b8faa1 MK |
52 | typedef __u32 u32; |
53 | ||
54 | static unsigned long prev_time; | |
55 | ||
56 | enum benchmark_type { | |
57 | BENCH_RXDROP = 0, | |
58 | BENCH_TXONLY = 1, | |
59 | BENCH_L2FWD = 2, | |
60 | }; | |
61 | ||
62 | static enum benchmark_type opt_bench = BENCH_RXDROP; | |
743e568c | 63 | static u32 opt_xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST; |
b4b8faa1 MK |
64 | static const char *opt_if = ""; |
65 | static int opt_ifindex; | |
66 | static int opt_queue; | |
d3f11b01 JJ |
67 | static unsigned long opt_duration; |
68 | static unsigned long start_time; | |
69 | static bool benchmark_done; | |
cd9e72b6 | 70 | static u32 opt_batch_size = 64; |
ece6e969 | 71 | static int opt_pkt_count; |
b4b8faa1 | 72 | static int opt_poll; |
b4b8faa1 | 73 | static int opt_interval = 1; |
46738f73 | 74 | static u32 opt_xdp_bind_flags = XDP_USE_NEED_WAKEUP; |
c543f546 KL |
75 | static u32 opt_umem_flags; |
76 | static int opt_unaligned_chunks; | |
3945b37a | 77 | static int opt_mmap_flags; |
c543f546 | 78 | static u32 opt_xdp_bind_flags; |
123e8da1 | 79 | static int opt_xsk_frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; |
46738f73 MK |
80 | static int opt_timeout = 1000; |
81 | static bool opt_need_wakeup = true; | |
2e5d72c1 MK |
82 | static u32 opt_num_xsks = 1; |
83 | static u32 prog_id; | |
b4b8faa1 | 84 | |
248c7f9c MK |
85 | struct xsk_umem_info { |
86 | struct xsk_ring_prod fq; | |
87 | struct xsk_ring_cons cq; | |
88 | struct xsk_umem *umem; | |
89 | void *buffer; | |
b4b8faa1 MK |
90 | }; |
91 | ||
248c7f9c MK |
92 | struct xsk_socket_info { |
93 | struct xsk_ring_cons rx; | |
94 | struct xsk_ring_prod tx; | |
95 | struct xsk_umem_info *umem; | |
96 | struct xsk_socket *xsk; | |
b4b8faa1 MK |
97 | unsigned long rx_npkts; |
98 | unsigned long tx_npkts; | |
99 | unsigned long prev_rx_npkts; | |
100 | unsigned long prev_tx_npkts; | |
248c7f9c | 101 | u32 outstanding_tx; |
b4b8faa1 MK |
102 | }; |
103 | ||
b4b8faa1 | 104 | static int num_socks; |
248c7f9c | 105 | struct xsk_socket_info *xsks[MAX_SOCKS]; |
b4b8faa1 MK |
106 | |
107 | static unsigned long get_nsecs(void) | |
108 | { | |
109 | struct timespec ts; | |
110 | ||
111 | clock_gettime(CLOCK_MONOTONIC, &ts); | |
112 | return ts.tv_sec * 1000000000UL + ts.tv_nsec; | |
113 | } | |
114 | ||
248c7f9c | 115 | static void print_benchmark(bool running) |
b4b8faa1 | 116 | { |
248c7f9c | 117 | const char *bench_str = "INVALID"; |
b4b8faa1 | 118 | |
248c7f9c MK |
119 | if (opt_bench == BENCH_RXDROP) |
120 | bench_str = "rxdrop"; | |
121 | else if (opt_bench == BENCH_TXONLY) | |
122 | bench_str = "txonly"; | |
123 | else if (opt_bench == BENCH_L2FWD) | |
124 | bench_str = "l2fwd"; | |
b4b8faa1 | 125 | |
248c7f9c MK |
126 | printf("%s:%d %s ", opt_if, opt_queue, bench_str); |
127 | if (opt_xdp_flags & XDP_FLAGS_SKB_MODE) | |
128 | printf("xdp-skb "); | |
129 | else if (opt_xdp_flags & XDP_FLAGS_DRV_MODE) | |
130 | printf("xdp-drv "); | |
131 | else | |
132 | printf(" "); | |
b4b8faa1 | 133 | |
248c7f9c MK |
134 | if (opt_poll) |
135 | printf("poll() "); | |
b4b8faa1 | 136 | |
248c7f9c MK |
137 | if (running) { |
138 | printf("running..."); | |
139 | fflush(stdout); | |
b4b8faa1 | 140 | } |
b4b8faa1 MK |
141 | } |
142 | ||
248c7f9c | 143 | static void dump_stats(void) |
b4b8faa1 | 144 | { |
248c7f9c MK |
145 | unsigned long now = get_nsecs(); |
146 | long dt = now - prev_time; | |
147 | int i; | |
b4b8faa1 | 148 | |
248c7f9c | 149 | prev_time = now; |
b4b8faa1 | 150 | |
248c7f9c MK |
151 | for (i = 0; i < num_socks && xsks[i]; i++) { |
152 | char *fmt = "%-15s %'-11.0f %'-11lu\n"; | |
153 | double rx_pps, tx_pps; | |
b4b8faa1 | 154 | |
248c7f9c MK |
155 | rx_pps = (xsks[i]->rx_npkts - xsks[i]->prev_rx_npkts) * |
156 | 1000000000. / dt; | |
157 | tx_pps = (xsks[i]->tx_npkts - xsks[i]->prev_tx_npkts) * | |
158 | 1000000000. / dt; | |
b4b8faa1 | 159 | |
248c7f9c MK |
160 | printf("\n sock%d@", i); |
161 | print_benchmark(false); | |
162 | printf("\n"); | |
b4b8faa1 | 163 | |
248c7f9c MK |
164 | printf("%-15s %-11s %-11s %-11.2f\n", "", "pps", "pkts", |
165 | dt / 1000000000.); | |
166 | printf(fmt, "rx", rx_pps, xsks[i]->rx_npkts); | |
167 | printf(fmt, "tx", tx_pps, xsks[i]->tx_npkts); | |
b4b8faa1 | 168 | |
248c7f9c MK |
169 | xsks[i]->prev_rx_npkts = xsks[i]->rx_npkts; |
170 | xsks[i]->prev_tx_npkts = xsks[i]->tx_npkts; | |
b4b8faa1 | 171 | } |
b4b8faa1 MK |
172 | } |
173 | ||
d3f11b01 JJ |
174 | static bool is_benchmark_done(void) |
175 | { | |
176 | if (opt_duration > 0) { | |
177 | unsigned long dt = (get_nsecs() - start_time); | |
178 | ||
179 | if (dt >= opt_duration) | |
180 | benchmark_done = true; | |
181 | } | |
182 | return benchmark_done; | |
183 | } | |
184 | ||
248c7f9c | 185 | static void *poller(void *arg) |
b4b8faa1 | 186 | { |
248c7f9c | 187 | (void)arg; |
d3f11b01 | 188 | while (!is_benchmark_done()) { |
248c7f9c MK |
189 | sleep(opt_interval); |
190 | dump_stats(); | |
b4b8faa1 MK |
191 | } |
192 | ||
248c7f9c | 193 | return NULL; |
b4b8faa1 MK |
194 | } |
195 | ||
248c7f9c | 196 | static void remove_xdp_program(void) |
b4b8faa1 | 197 | { |
2e5d72c1 | 198 | u32 curr_prog_id = 0; |
b4b8faa1 | 199 | |
248c7f9c MK |
200 | if (bpf_get_link_xdp_id(opt_ifindex, &curr_prog_id, opt_xdp_flags)) { |
201 | printf("bpf_get_link_xdp_id failed\n"); | |
202 | exit(EXIT_FAILURE); | |
b4b8faa1 | 203 | } |
248c7f9c MK |
204 | if (prog_id == curr_prog_id) |
205 | bpf_set_link_xdp_fd(opt_ifindex, -1, opt_xdp_flags); | |
206 | else if (!curr_prog_id) | |
207 | printf("couldn't find a prog id on a given interface\n"); | |
208 | else | |
209 | printf("program on interface changed, not removing\n"); | |
b4b8faa1 MK |
210 | } |
211 | ||
248c7f9c | 212 | static void int_exit(int sig) |
69525588 JJ |
213 | { |
214 | benchmark_done = true; | |
215 | } | |
216 | ||
217 | static void xdpsock_cleanup(void) | |
b4b8faa1 | 218 | { |
248c7f9c | 219 | struct xsk_umem *umem = xsks[0]->umem->umem; |
2e5d72c1 | 220 | int i; |
b4b8faa1 | 221 | |
248c7f9c | 222 | dump_stats(); |
2e5d72c1 MK |
223 | for (i = 0; i < num_socks; i++) |
224 | xsk_socket__delete(xsks[i]->xsk); | |
248c7f9c MK |
225 | (void)xsk_umem__delete(umem); |
226 | remove_xdp_program(); | |
b4b8faa1 MK |
227 | } |
228 | ||
248c7f9c MK |
229 | static void __exit_with_error(int error, const char *file, const char *func, |
230 | int line) | |
b4b8faa1 | 231 | { |
248c7f9c MK |
232 | fprintf(stderr, "%s:%s:%i: errno: %d/\"%s\"\n", file, func, |
233 | line, error, strerror(error)); | |
234 | dump_stats(); | |
235 | remove_xdp_program(); | |
236 | exit(EXIT_FAILURE); | |
b4b8faa1 MK |
237 | } |
238 | ||
248c7f9c MK |
239 | #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, \ |
240 | __LINE__) | |
248c7f9c MK |
241 | static const char pkt_data[] = |
242 | "\x3c\xfd\xfe\x9e\x7f\x71\xec\xb1\xd7\x98\x3a\xc0\x08\x00\x45\x00" | |
243 | "\x00\x2e\x00\x00\x00\x00\x40\x11\x88\x97\x05\x08\x07\x08\xc8\x14" | |
244 | "\x1e\x04\x10\x92\x10\x92\x00\x1a\x6d\xa3\x34\x33\x1f\x69\x40\x6b" | |
245 | "\x54\x59\xb6\x14\x2d\x11\x44\xbf\xaf\xd9\xbe\xaa"; | |
b4b8faa1 MK |
246 | |
247 | static void swap_mac_addresses(void *data) | |
248 | { | |
249 | struct ether_header *eth = (struct ether_header *)data; | |
250 | struct ether_addr *src_addr = (struct ether_addr *)ð->ether_shost; | |
251 | struct ether_addr *dst_addr = (struct ether_addr *)ð->ether_dhost; | |
252 | struct ether_addr tmp; | |
253 | ||
254 | tmp = *src_addr; | |
255 | *src_addr = *dst_addr; | |
256 | *dst_addr = tmp; | |
257 | } | |
258 | ||
a412ef54 | 259 | static void hex_dump(void *pkt, size_t length, u64 addr) |
b4b8faa1 | 260 | { |
b4b8faa1 MK |
261 | const unsigned char *address = (unsigned char *)pkt; |
262 | const unsigned char *line = address; | |
263 | size_t line_size = 32; | |
264 | unsigned char c; | |
a412ef54 BT |
265 | char buf[32]; |
266 | int i = 0; | |
b4b8faa1 | 267 | |
a412ef54 BT |
268 | if (!DEBUG_HEXDUMP) |
269 | return; | |
270 | ||
271 | sprintf(buf, "addr=%llu", addr); | |
b4b8faa1 | 272 | printf("length = %zu\n", length); |
a412ef54 | 273 | printf("%s | ", buf); |
b4b8faa1 MK |
274 | while (length-- > 0) { |
275 | printf("%02X ", *address++); | |
276 | if (!(++i % line_size) || (length == 0 && i % line_size)) { | |
277 | if (length == 0) { | |
278 | while (i++ % line_size) | |
279 | printf("__ "); | |
280 | } | |
281 | printf(" | "); /* right close */ | |
282 | while (line < address) { | |
283 | c = *line++; | |
284 | printf("%c", (c < 33 || c == 255) ? 0x2E : c); | |
285 | } | |
286 | printf("\n"); | |
287 | if (length > 0) | |
a412ef54 | 288 | printf("%s | ", buf); |
b4b8faa1 MK |
289 | } |
290 | } | |
291 | printf("\n"); | |
292 | } | |
b4b8faa1 | 293 | |
cd9e72b6 | 294 | static void gen_eth_frame(struct xsk_umem_info *umem, u64 addr) |
b4b8faa1 | 295 | { |
248c7f9c MK |
296 | memcpy(xsk_umem__get_data(umem->buffer, addr), pkt_data, |
297 | sizeof(pkt_data) - 1); | |
b4b8faa1 MK |
298 | } |
299 | ||
248c7f9c | 300 | static struct xsk_umem_info *xsk_configure_umem(void *buffer, u64 size) |
b4b8faa1 | 301 | { |
248c7f9c | 302 | struct xsk_umem_info *umem; |
123e8da1 MM |
303 | struct xsk_umem_config cfg = { |
304 | .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS, | |
305 | .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, | |
306 | .frame_size = opt_xsk_frame_size, | |
307 | .frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM, | |
c543f546 | 308 | .flags = opt_umem_flags |
123e8da1 | 309 | }; |
661842c4 | 310 | int ret; |
b4b8faa1 MK |
311 | |
312 | umem = calloc(1, sizeof(*umem)); | |
248c7f9c MK |
313 | if (!umem) |
314 | exit_with_error(errno); | |
b4b8faa1 | 315 | |
248c7f9c | 316 | ret = xsk_umem__create(&umem->umem, buffer, size, &umem->fq, &umem->cq, |
123e8da1 | 317 | &cfg); |
248c7f9c MK |
318 | if (ret) |
319 | exit_with_error(-ret); | |
b4b8faa1 | 320 | |
661842c4 MK |
321 | umem->buffer = buffer; |
322 | return umem; | |
323 | } | |
324 | ||
325 | static void xsk_populate_fill_ring(struct xsk_umem_info *umem) | |
326 | { | |
327 | int ret, i; | |
328 | u32 idx; | |
329 | ||
2e5d72c1 MK |
330 | ret = xsk_ring_prod__reserve(&umem->fq, |
331 | XSK_RING_PROD__DEFAULT_NUM_DESCS, &idx); | |
332 | if (ret != XSK_RING_PROD__DEFAULT_NUM_DESCS) | |
333 | exit_with_error(-ret); | |
334 | for (i = 0; i < XSK_RING_PROD__DEFAULT_NUM_DESCS; i++) | |
335 | *xsk_ring_prod__fill_addr(&umem->fq, idx++) = | |
336 | i * opt_xsk_frame_size; | |
337 | xsk_ring_prod__submit(&umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS); | |
b4b8faa1 MK |
338 | } |
339 | ||
661842c4 MK |
340 | static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem, |
341 | bool rx, bool tx) | |
b4b8faa1 | 342 | { |
248c7f9c MK |
343 | struct xsk_socket_config cfg; |
344 | struct xsk_socket_info *xsk; | |
661842c4 MK |
345 | struct xsk_ring_cons *rxr; |
346 | struct xsk_ring_prod *txr; | |
248c7f9c | 347 | int ret; |
b4b8faa1 MK |
348 | |
349 | xsk = calloc(1, sizeof(*xsk)); | |
248c7f9c MK |
350 | if (!xsk) |
351 | exit_with_error(errno); | |
352 | ||
353 | xsk->umem = umem; | |
354 | cfg.rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS; | |
355 | cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; | |
2e5d72c1 MK |
356 | if (opt_num_xsks > 1) |
357 | cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD; | |
358 | else | |
359 | cfg.libbpf_flags = 0; | |
248c7f9c MK |
360 | cfg.xdp_flags = opt_xdp_flags; |
361 | cfg.bind_flags = opt_xdp_bind_flags; | |
2e5d72c1 | 362 | |
661842c4 MK |
363 | rxr = rx ? &xsk->rx : NULL; |
364 | txr = tx ? &xsk->tx : NULL; | |
365 | ret = xsk_socket__create(&xsk->xsk, opt_if, opt_queue, umem->umem, | |
366 | rxr, txr, &cfg); | |
248c7f9c MK |
367 | if (ret) |
368 | exit_with_error(-ret); | |
369 | ||
370 | ret = bpf_get_link_xdp_id(opt_ifindex, &prog_id, opt_xdp_flags); | |
371 | if (ret) | |
372 | exit_with_error(-ret); | |
373 | ||
b4b8faa1 MK |
374 | return xsk; |
375 | } | |
376 | ||
b4b8faa1 MK |
377 | static struct option long_options[] = { |
378 | {"rxdrop", no_argument, 0, 'r'}, | |
379 | {"txonly", no_argument, 0, 't'}, | |
380 | {"l2fwd", no_argument, 0, 'l'}, | |
381 | {"interface", required_argument, 0, 'i'}, | |
382 | {"queue", required_argument, 0, 'q'}, | |
383 | {"poll", no_argument, 0, 'p'}, | |
b4b8faa1 MK |
384 | {"xdp-skb", no_argument, 0, 'S'}, |
385 | {"xdp-native", no_argument, 0, 'N'}, | |
386 | {"interval", required_argument, 0, 'n'}, | |
58c50ae4 BT |
387 | {"zero-copy", no_argument, 0, 'z'}, |
388 | {"copy", no_argument, 0, 'c'}, | |
123e8da1 | 389 | {"frame-size", required_argument, 0, 'f'}, |
46738f73 | 390 | {"no-need-wakeup", no_argument, 0, 'm'}, |
c543f546 | 391 | {"unaligned", no_argument, 0, 'u'}, |
2e5d72c1 | 392 | {"shared-umem", no_argument, 0, 'M'}, |
b3133329 | 393 | {"force", no_argument, 0, 'F'}, |
d3f11b01 | 394 | {"duration", required_argument, 0, 'd'}, |
cd9e72b6 | 395 | {"batch-size", required_argument, 0, 'b'}, |
ece6e969 | 396 | {"tx-pkt-count", required_argument, 0, 'C'}, |
b4b8faa1 MK |
397 | {0, 0, 0, 0} |
398 | }; | |
399 | ||
400 | static void usage(const char *prog) | |
401 | { | |
402 | const char *str = | |
403 | " Usage: %s [OPTIONS]\n" | |
404 | " Options:\n" | |
405 | " -r, --rxdrop Discard all incoming packets (default)\n" | |
406 | " -t, --txonly Only send packets\n" | |
407 | " -l, --l2fwd MAC swap L2 forwarding\n" | |
408 | " -i, --interface=n Run on interface n\n" | |
409 | " -q, --queue=n Use queue n (default 0)\n" | |
410 | " -p, --poll Use poll syscall\n" | |
b4b8faa1 | 411 | " -S, --xdp-skb=n Use XDP skb-mod\n" |
4564a8bb | 412 | " -N, --xdp-native=n Enforce XDP native mode\n" |
b4b8faa1 | 413 | " -n, --interval=n Specify statistics update interval (default 1 sec).\n" |
58c50ae4 BT |
414 | " -z, --zero-copy Force zero-copy mode.\n" |
415 | " -c, --copy Force copy mode.\n" | |
46738f73 | 416 | " -m, --no-need-wakeup Turn off use of driver need wakeup flag.\n" |
c543f546 KL |
417 | " -f, --frame-size=n Set the frame size (must be a power of two in aligned mode, default is %d).\n" |
418 | " -u, --unaligned Enable unaligned chunk placement\n" | |
2e5d72c1 | 419 | " -M, --shared-umem Enable XDP_SHARED_UMEM\n" |
b3133329 | 420 | " -F, --force Force loading the XDP prog\n" |
d3f11b01 JJ |
421 | " -d, --duration=n Duration in secs to run command.\n" |
422 | " Default: forever.\n" | |
cd9e72b6 JJ |
423 | " -b, --batch-size=n Batch size for sending or receiving\n" |
424 | " packets. Default: %d\n" | |
ece6e969 JJ |
425 | " -C, --tx-pkt-count=n Number of packets to send.\n" |
426 | " Default: Continuous packets.\n" | |
b4b8faa1 | 427 | "\n"; |
cd9e72b6 JJ |
428 | fprintf(stderr, str, prog, XSK_UMEM__DEFAULT_FRAME_SIZE, |
429 | opt_batch_size); | |
b4b8faa1 MK |
430 | exit(EXIT_FAILURE); |
431 | } | |
432 | ||
433 | static void parse_command_line(int argc, char **argv) | |
434 | { | |
435 | int option_index, c; | |
436 | ||
437 | opterr = 0; | |
438 | ||
439 | for (;;) { | |
ece6e969 | 440 | c = getopt_long(argc, argv, "Frtli:q:pSNn:czf:muMd:b:C:", |
46738f73 | 441 | long_options, &option_index); |
b4b8faa1 MK |
442 | if (c == -1) |
443 | break; | |
444 | ||
445 | switch (c) { | |
446 | case 'r': | |
447 | opt_bench = BENCH_RXDROP; | |
448 | break; | |
449 | case 't': | |
450 | opt_bench = BENCH_TXONLY; | |
451 | break; | |
452 | case 'l': | |
453 | opt_bench = BENCH_L2FWD; | |
454 | break; | |
455 | case 'i': | |
456 | opt_if = optarg; | |
457 | break; | |
458 | case 'q': | |
459 | opt_queue = atoi(optarg); | |
460 | break; | |
b4b8faa1 MK |
461 | case 'p': |
462 | opt_poll = 1; | |
463 | break; | |
464 | case 'S': | |
465 | opt_xdp_flags |= XDP_FLAGS_SKB_MODE; | |
9f5232cc | 466 | opt_xdp_bind_flags |= XDP_COPY; |
b4b8faa1 MK |
467 | break; |
468 | case 'N': | |
d50ecc46 | 469 | /* default, set below */ |
b4b8faa1 MK |
470 | break; |
471 | case 'n': | |
472 | opt_interval = atoi(optarg); | |
473 | break; | |
58c50ae4 BT |
474 | case 'z': |
475 | opt_xdp_bind_flags |= XDP_ZEROCOPY; | |
476 | break; | |
477 | case 'c': | |
478 | opt_xdp_bind_flags |= XDP_COPY; | |
479 | break; | |
c543f546 KL |
480 | case 'u': |
481 | opt_umem_flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG; | |
482 | opt_unaligned_chunks = 1; | |
3945b37a | 483 | opt_mmap_flags = MAP_HUGETLB; |
c543f546 | 484 | break; |
743e568c MF |
485 | case 'F': |
486 | opt_xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST; | |
487 | break; | |
123e8da1 MM |
488 | case 'f': |
489 | opt_xsk_frame_size = atoi(optarg); | |
2e5d72c1 | 490 | break; |
46738f73 MK |
491 | case 'm': |
492 | opt_need_wakeup = false; | |
493 | opt_xdp_bind_flags &= ~XDP_USE_NEED_WAKEUP; | |
123e8da1 | 494 | break; |
2e5d72c1 MK |
495 | case 'M': |
496 | opt_num_xsks = MAX_SOCKS; | |
497 | break; | |
d3f11b01 JJ |
498 | case 'd': |
499 | opt_duration = atoi(optarg); | |
500 | opt_duration *= 1000000000; | |
501 | break; | |
cd9e72b6 JJ |
502 | case 'b': |
503 | opt_batch_size = atoi(optarg); | |
504 | break; | |
ece6e969 JJ |
505 | case 'C': |
506 | opt_pkt_count = atoi(optarg); | |
507 | break; | |
b4b8faa1 MK |
508 | default: |
509 | usage(basename(argv[0])); | |
510 | } | |
511 | } | |
512 | ||
d50ecc46 THJ |
513 | if (!(opt_xdp_flags & XDP_FLAGS_SKB_MODE)) |
514 | opt_xdp_flags |= XDP_FLAGS_DRV_MODE; | |
515 | ||
b4b8faa1 MK |
516 | opt_ifindex = if_nametoindex(opt_if); |
517 | if (!opt_ifindex) { | |
518 | fprintf(stderr, "ERROR: interface \"%s\" does not exist\n", | |
519 | opt_if); | |
520 | usage(basename(argv[0])); | |
521 | } | |
248c7f9c | 522 | |
c543f546 KL |
523 | if ((opt_xsk_frame_size & (opt_xsk_frame_size - 1)) && |
524 | !opt_unaligned_chunks) { | |
123e8da1 MM |
525 | fprintf(stderr, "--frame-size=%d is not a power of two\n", |
526 | opt_xsk_frame_size); | |
527 | usage(basename(argv[0])); | |
528 | } | |
b4b8faa1 MK |
529 | } |
530 | ||
248c7f9c | 531 | static void kick_tx(struct xsk_socket_info *xsk) |
b4b8faa1 MK |
532 | { |
533 | int ret; | |
534 | ||
248c7f9c | 535 | ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); |
c03079c9 | 536 | if (ret >= 0 || errno == ENOBUFS || errno == EAGAIN || errno == EBUSY) |
b4b8faa1 | 537 | return; |
248c7f9c | 538 | exit_with_error(errno); |
b4b8faa1 MK |
539 | } |
540 | ||
46738f73 MK |
541 | static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk, |
542 | struct pollfd *fds) | |
b4b8faa1 | 543 | { |
03895e63 | 544 | struct xsk_umem_info *umem = xsk->umem; |
b74e21ab | 545 | u32 idx_cq = 0, idx_fq = 0; |
b4b8faa1 MK |
546 | unsigned int rcvd; |
547 | size_t ndescs; | |
548 | ||
549 | if (!xsk->outstanding_tx) | |
550 | return; | |
551 | ||
46738f73 MK |
552 | if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx)) |
553 | kick_tx(xsk); | |
554 | ||
cd9e72b6 | 555 | ndescs = (xsk->outstanding_tx > opt_batch_size) ? opt_batch_size : |
248c7f9c | 556 | xsk->outstanding_tx; |
b4b8faa1 MK |
557 | |
558 | /* re-add completed Tx buffers */ | |
03895e63 | 559 | rcvd = xsk_ring_cons__peek(&umem->cq, ndescs, &idx_cq); |
b4b8faa1 | 560 | if (rcvd > 0) { |
248c7f9c MK |
561 | unsigned int i; |
562 | int ret; | |
563 | ||
03895e63 | 564 | ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); |
248c7f9c MK |
565 | while (ret != rcvd) { |
566 | if (ret < 0) | |
567 | exit_with_error(-ret); | |
03895e63 | 568 | if (xsk_ring_prod__needs_wakeup(&umem->fq)) |
46738f73 | 569 | ret = poll(fds, num_socks, opt_timeout); |
03895e63 | 570 | ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); |
248c7f9c | 571 | } |
03895e63 | 572 | |
248c7f9c | 573 | for (i = 0; i < rcvd; i++) |
03895e63 KL |
574 | *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = |
575 | *xsk_ring_cons__comp_addr(&umem->cq, idx_cq++); | |
248c7f9c MK |
576 | |
577 | xsk_ring_prod__submit(&xsk->umem->fq, rcvd); | |
578 | xsk_ring_cons__release(&xsk->umem->cq, rcvd); | |
b4b8faa1 MK |
579 | xsk->outstanding_tx -= rcvd; |
580 | xsk->tx_npkts += rcvd; | |
581 | } | |
582 | } | |
583 | ||
ece6e969 JJ |
584 | static inline void complete_tx_only(struct xsk_socket_info *xsk, |
585 | int batch_size) | |
b4b8faa1 | 586 | { |
b4b8faa1 | 587 | unsigned int rcvd; |
248c7f9c | 588 | u32 idx; |
b4b8faa1 MK |
589 | |
590 | if (!xsk->outstanding_tx) | |
591 | return; | |
592 | ||
46738f73 MK |
593 | if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx)) |
594 | kick_tx(xsk); | |
b4b8faa1 | 595 | |
ece6e969 | 596 | rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx); |
b4b8faa1 | 597 | if (rcvd > 0) { |
248c7f9c | 598 | xsk_ring_cons__release(&xsk->umem->cq, rcvd); |
b4b8faa1 MK |
599 | xsk->outstanding_tx -= rcvd; |
600 | xsk->tx_npkts += rcvd; | |
601 | } | |
602 | } | |
603 | ||
46738f73 | 604 | static void rx_drop(struct xsk_socket_info *xsk, struct pollfd *fds) |
b4b8faa1 | 605 | { |
b4b8faa1 | 606 | unsigned int rcvd, i; |
b74e21ab | 607 | u32 idx_rx = 0, idx_fq = 0; |
248c7f9c | 608 | int ret; |
b4b8faa1 | 609 | |
cd9e72b6 | 610 | rcvd = xsk_ring_cons__peek(&xsk->rx, opt_batch_size, &idx_rx); |
46738f73 MK |
611 | if (!rcvd) { |
612 | if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) | |
613 | ret = poll(fds, num_socks, opt_timeout); | |
b4b8faa1 | 614 | return; |
46738f73 | 615 | } |
b4b8faa1 | 616 | |
248c7f9c MK |
617 | ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq); |
618 | while (ret != rcvd) { | |
619 | if (ret < 0) | |
620 | exit_with_error(-ret); | |
46738f73 MK |
621 | if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) |
622 | ret = poll(fds, num_socks, opt_timeout); | |
248c7f9c MK |
623 | ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq); |
624 | } | |
625 | ||
b4b8faa1 | 626 | for (i = 0; i < rcvd; i++) { |
248c7f9c MK |
627 | u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr; |
628 | u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len; | |
03895e63 KL |
629 | u64 orig = xsk_umem__extract_addr(addr); |
630 | ||
631 | addr = xsk_umem__add_offset_to_addr(addr); | |
248c7f9c | 632 | char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr); |
b4b8faa1 | 633 | |
248c7f9c | 634 | hex_dump(pkt, len, addr); |
03895e63 | 635 | *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx_fq++) = orig; |
b4b8faa1 MK |
636 | } |
637 | ||
248c7f9c MK |
638 | xsk_ring_prod__submit(&xsk->umem->fq, rcvd); |
639 | xsk_ring_cons__release(&xsk->rx, rcvd); | |
b4b8faa1 | 640 | xsk->rx_npkts += rcvd; |
b4b8faa1 MK |
641 | } |
642 | ||
643 | static void rx_drop_all(void) | |
644 | { | |
2e5d72c1 | 645 | struct pollfd fds[MAX_SOCKS] = {}; |
46738f73 | 646 | int i, ret; |
b4b8faa1 | 647 | |
b4b8faa1 | 648 | for (i = 0; i < num_socks; i++) { |
248c7f9c | 649 | fds[i].fd = xsk_socket__fd(xsks[i]->xsk); |
b4b8faa1 | 650 | fds[i].events = POLLIN; |
b4b8faa1 MK |
651 | } |
652 | ||
653 | for (;;) { | |
654 | if (opt_poll) { | |
46738f73 | 655 | ret = poll(fds, num_socks, opt_timeout); |
b4b8faa1 MK |
656 | if (ret <= 0) |
657 | continue; | |
658 | } | |
659 | ||
660 | for (i = 0; i < num_socks; i++) | |
46738f73 | 661 | rx_drop(xsks[i], fds); |
d3f11b01 JJ |
662 | |
663 | if (benchmark_done) | |
664 | break; | |
46738f73 MK |
665 | } |
666 | } | |
667 | ||
ece6e969 | 668 | static void tx_only(struct xsk_socket_info *xsk, u32 frame_nb, int batch_size) |
46738f73 MK |
669 | { |
670 | u32 idx; | |
cd9e72b6 | 671 | unsigned int i; |
46738f73 | 672 | |
ece6e969 JJ |
673 | while (xsk_ring_prod__reserve(&xsk->tx, batch_size, &idx) < |
674 | batch_size) { | |
675 | complete_tx_only(xsk, batch_size); | |
cd9e72b6 | 676 | } |
46738f73 | 677 | |
ece6e969 | 678 | for (i = 0; i < batch_size; i++) { |
cd9e72b6 JJ |
679 | struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, |
680 | idx + i); | |
681 | tx_desc->addr = (frame_nb + i) << XSK_UMEM__DEFAULT_FRAME_SHIFT; | |
682 | tx_desc->len = sizeof(pkt_data) - 1; | |
b4b8faa1 | 683 | } |
46738f73 | 684 | |
ece6e969 JJ |
685 | xsk_ring_prod__submit(&xsk->tx, batch_size); |
686 | xsk->outstanding_tx += batch_size; | |
687 | frame_nb += batch_size; | |
cd9e72b6 | 688 | frame_nb %= NUM_FRAMES; |
ece6e969 JJ |
689 | complete_tx_only(xsk, batch_size); |
690 | } | |
691 | ||
692 | static inline int get_batch_size(int pkt_cnt) | |
693 | { | |
694 | if (!opt_pkt_count) | |
695 | return opt_batch_size; | |
696 | ||
697 | if (pkt_cnt + opt_batch_size <= opt_pkt_count) | |
698 | return opt_batch_size; | |
699 | ||
700 | return opt_pkt_count - pkt_cnt; | |
701 | } | |
702 | ||
703 | static void complete_tx_only_all(void) | |
704 | { | |
705 | bool pending; | |
706 | int i; | |
707 | ||
708 | do { | |
709 | pending = false; | |
710 | for (i = 0; i < num_socks; i++) { | |
711 | if (xsks[i]->outstanding_tx) { | |
712 | complete_tx_only(xsks[i], opt_batch_size); | |
713 | pending = !!xsks[i]->outstanding_tx; | |
714 | } | |
715 | } | |
716 | } while (pending); | |
b4b8faa1 MK |
717 | } |
718 | ||
46738f73 | 719 | static void tx_only_all(void) |
b4b8faa1 | 720 | { |
2e5d72c1 | 721 | struct pollfd fds[MAX_SOCKS] = {}; |
46738f73 | 722 | u32 frame_nb[MAX_SOCKS] = {}; |
ece6e969 | 723 | int pkt_cnt = 0; |
46738f73 | 724 | int i, ret; |
b4b8faa1 | 725 | |
46738f73 MK |
726 | for (i = 0; i < num_socks; i++) { |
727 | fds[0].fd = xsk_socket__fd(xsks[i]->xsk); | |
728 | fds[0].events = POLLOUT; | |
729 | } | |
b4b8faa1 | 730 | |
ece6e969 JJ |
731 | while ((opt_pkt_count && pkt_cnt < opt_pkt_count) || !opt_pkt_count) { |
732 | int batch_size = get_batch_size(pkt_cnt); | |
733 | ||
b4b8faa1 | 734 | if (opt_poll) { |
46738f73 | 735 | ret = poll(fds, num_socks, opt_timeout); |
b4b8faa1 MK |
736 | if (ret <= 0) |
737 | continue; | |
738 | ||
248c7f9c | 739 | if (!(fds[0].revents & POLLOUT)) |
b4b8faa1 MK |
740 | continue; |
741 | } | |
742 | ||
46738f73 | 743 | for (i = 0; i < num_socks; i++) |
ece6e969 JJ |
744 | tx_only(xsks[i], frame_nb[i], batch_size); |
745 | ||
746 | pkt_cnt += batch_size; | |
d3f11b01 JJ |
747 | |
748 | if (benchmark_done) | |
749 | break; | |
b4b8faa1 | 750 | } |
ece6e969 JJ |
751 | |
752 | if (opt_pkt_count) | |
753 | complete_tx_only_all(); | |
b4b8faa1 MK |
754 | } |
755 | ||
46738f73 | 756 | static void l2fwd(struct xsk_socket_info *xsk, struct pollfd *fds) |
b4b8faa1 | 757 | { |
46738f73 MK |
758 | unsigned int rcvd, i; |
759 | u32 idx_rx = 0, idx_tx = 0; | |
760 | int ret; | |
b4b8faa1 | 761 | |
46738f73 | 762 | complete_tx_l2fwd(xsk, fds); |
b4b8faa1 | 763 | |
cd9e72b6 | 764 | rcvd = xsk_ring_cons__peek(&xsk->rx, opt_batch_size, &idx_rx); |
46738f73 MK |
765 | if (!rcvd) { |
766 | if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) | |
767 | ret = poll(fds, num_socks, opt_timeout); | |
768 | return; | |
769 | } | |
b4b8faa1 | 770 | |
46738f73 MK |
771 | ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx); |
772 | while (ret != rcvd) { | |
773 | if (ret < 0) | |
774 | exit_with_error(-ret); | |
775 | if (xsk_ring_prod__needs_wakeup(&xsk->tx)) | |
776 | kick_tx(xsk); | |
248c7f9c | 777 | ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx); |
46738f73 MK |
778 | } |
779 | ||
780 | for (i = 0; i < rcvd; i++) { | |
781 | u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr; | |
782 | u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len; | |
5a712e13 | 783 | u64 orig = addr; |
03895e63 KL |
784 | |
785 | addr = xsk_umem__add_offset_to_addr(addr); | |
46738f73 MK |
786 | char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr); |
787 | ||
788 | swap_mac_addresses(pkt); | |
248c7f9c | 789 | |
46738f73 | 790 | hex_dump(pkt, len, addr); |
03895e63 | 791 | xsk_ring_prod__tx_desc(&xsk->tx, idx_tx)->addr = orig; |
46738f73 MK |
792 | xsk_ring_prod__tx_desc(&xsk->tx, idx_tx++)->len = len; |
793 | } | |
b4b8faa1 | 794 | |
46738f73 MK |
795 | xsk_ring_prod__submit(&xsk->tx, rcvd); |
796 | xsk_ring_cons__release(&xsk->rx, rcvd); | |
b4b8faa1 | 797 | |
46738f73 MK |
798 | xsk->rx_npkts += rcvd; |
799 | xsk->outstanding_tx += rcvd; | |
800 | } | |
801 | ||
802 | static void l2fwd_all(void) | |
803 | { | |
2e5d72c1 | 804 | struct pollfd fds[MAX_SOCKS] = {}; |
46738f73 MK |
805 | int i, ret; |
806 | ||
46738f73 MK |
807 | for (i = 0; i < num_socks; i++) { |
808 | fds[i].fd = xsk_socket__fd(xsks[i]->xsk); | |
809 | fds[i].events = POLLOUT | POLLIN; | |
810 | } | |
b4b8faa1 | 811 | |
46738f73 MK |
812 | for (;;) { |
813 | if (opt_poll) { | |
814 | ret = poll(fds, num_socks, opt_timeout); | |
815 | if (ret <= 0) | |
816 | continue; | |
817 | } | |
b4b8faa1 | 818 | |
46738f73 MK |
819 | for (i = 0; i < num_socks; i++) |
820 | l2fwd(xsks[i], fds); | |
d3f11b01 JJ |
821 | |
822 | if (benchmark_done) | |
823 | break; | |
b4b8faa1 MK |
824 | } |
825 | } | |
826 | ||
2e5d72c1 MK |
827 | static void load_xdp_program(char **argv, struct bpf_object **obj) |
828 | { | |
829 | struct bpf_prog_load_attr prog_load_attr = { | |
830 | .prog_type = BPF_PROG_TYPE_XDP, | |
831 | }; | |
832 | char xdp_filename[256]; | |
833 | int prog_fd; | |
834 | ||
835 | snprintf(xdp_filename, sizeof(xdp_filename), "%s_kern.o", argv[0]); | |
836 | prog_load_attr.file = xdp_filename; | |
837 | ||
838 | if (bpf_prog_load_xattr(&prog_load_attr, obj, &prog_fd)) | |
839 | exit(EXIT_FAILURE); | |
840 | if (prog_fd < 0) { | |
841 | fprintf(stderr, "ERROR: no program found: %s\n", | |
842 | strerror(prog_fd)); | |
843 | exit(EXIT_FAILURE); | |
844 | } | |
845 | ||
846 | if (bpf_set_link_xdp_fd(opt_ifindex, prog_fd, opt_xdp_flags) < 0) { | |
847 | fprintf(stderr, "ERROR: link set xdp fd failed\n"); | |
848 | exit(EXIT_FAILURE); | |
849 | } | |
850 | } | |
851 | ||
852 | static void enter_xsks_into_map(struct bpf_object *obj) | |
853 | { | |
854 | struct bpf_map *map; | |
855 | int i, xsks_map; | |
856 | ||
857 | map = bpf_object__find_map_by_name(obj, "xsks_map"); | |
858 | xsks_map = bpf_map__fd(map); | |
859 | if (xsks_map < 0) { | |
860 | fprintf(stderr, "ERROR: no xsks map found: %s\n", | |
861 | strerror(xsks_map)); | |
862 | exit(EXIT_FAILURE); | |
863 | } | |
864 | ||
865 | for (i = 0; i < num_socks; i++) { | |
866 | int fd = xsk_socket__fd(xsks[i]->xsk); | |
867 | int key, ret; | |
868 | ||
869 | key = i; | |
870 | ret = bpf_map_update_elem(xsks_map, &key, &fd, 0); | |
871 | if (ret) { | |
872 | fprintf(stderr, "ERROR: bpf_map_update_elem %d\n", i); | |
873 | exit(EXIT_FAILURE); | |
874 | } | |
875 | } | |
876 | } | |
877 | ||
b4b8faa1 MK |
878 | int main(int argc, char **argv) |
879 | { | |
880 | struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY}; | |
661842c4 | 881 | bool rx = false, tx = false; |
248c7f9c | 882 | struct xsk_umem_info *umem; |
2e5d72c1 | 883 | struct bpf_object *obj; |
b4b8faa1 | 884 | pthread_t pt; |
2e5d72c1 | 885 | int i, ret; |
248c7f9c | 886 | void *bufs; |
b4b8faa1 MK |
887 | |
888 | parse_command_line(argc, argv); | |
889 | ||
890 | if (setrlimit(RLIMIT_MEMLOCK, &r)) { | |
891 | fprintf(stderr, "ERROR: setrlimit(RLIMIT_MEMLOCK) \"%s\"\n", | |
892 | strerror(errno)); | |
893 | exit(EXIT_FAILURE); | |
894 | } | |
895 | ||
2e5d72c1 MK |
896 | if (opt_num_xsks > 1) |
897 | load_xdp_program(argv, &obj); | |
898 | ||
3945b37a KL |
899 | /* Reserve memory for the umem. Use hugepages if unaligned chunk mode */ |
900 | bufs = mmap(NULL, NUM_FRAMES * opt_xsk_frame_size, | |
901 | PROT_READ | PROT_WRITE, | |
902 | MAP_PRIVATE | MAP_ANONYMOUS | opt_mmap_flags, -1, 0); | |
903 | if (bufs == MAP_FAILED) { | |
904 | printf("ERROR: mmap failed\n"); | |
905 | exit(EXIT_FAILURE); | |
906 | } | |
2e5d72c1 MK |
907 | |
908 | /* Create sockets... */ | |
123e8da1 | 909 | umem = xsk_configure_umem(bufs, NUM_FRAMES * opt_xsk_frame_size); |
661842c4 MK |
910 | if (opt_bench == BENCH_RXDROP || opt_bench == BENCH_L2FWD) { |
911 | rx = true; | |
912 | xsk_populate_fill_ring(umem); | |
913 | } | |
914 | if (opt_bench == BENCH_L2FWD || opt_bench == BENCH_TXONLY) | |
915 | tx = true; | |
2e5d72c1 | 916 | for (i = 0; i < opt_num_xsks; i++) |
661842c4 | 917 | xsks[num_socks++] = xsk_configure_socket(umem, rx, tx); |
b4b8faa1 | 918 | |
661842c4 MK |
919 | if (opt_bench == BENCH_TXONLY) |
920 | for (i = 0; i < NUM_FRAMES; i++) | |
921 | gen_eth_frame(umem, i * opt_xsk_frame_size); | |
b4b8faa1 | 922 | |
2e5d72c1 MK |
923 | if (opt_num_xsks > 1 && opt_bench != BENCH_TXONLY) |
924 | enter_xsks_into_map(obj); | |
b4b8faa1 MK |
925 | |
926 | signal(SIGINT, int_exit); | |
927 | signal(SIGTERM, int_exit); | |
928 | signal(SIGABRT, int_exit); | |
929 | ||
930 | setlocale(LC_ALL, ""); | |
931 | ||
932 | ret = pthread_create(&pt, NULL, poller, NULL); | |
248c7f9c MK |
933 | if (ret) |
934 | exit_with_error(ret); | |
b4b8faa1 MK |
935 | |
936 | prev_time = get_nsecs(); | |
d3f11b01 | 937 | start_time = prev_time; |
b4b8faa1 MK |
938 | |
939 | if (opt_bench == BENCH_RXDROP) | |
940 | rx_drop_all(); | |
941 | else if (opt_bench == BENCH_TXONLY) | |
46738f73 | 942 | tx_only_all(); |
b4b8faa1 | 943 | else |
46738f73 | 944 | l2fwd_all(); |
b4b8faa1 | 945 | |
ece6e969 JJ |
946 | benchmark_done = true; |
947 | ||
d3f11b01 JJ |
948 | pthread_join(pt, NULL); |
949 | ||
69525588 JJ |
950 | xdpsock_cleanup(); |
951 | ||
b4b8faa1 MK |
952 | return 0; |
953 | } |