]>
Commit | Line | Data |
---|---|---|
b4b8faa1 | 1 | // SPDX-License-Identifier: GPL-2.0 |
dac09149 | 2 | /* Copyright(c) 2017 - 2018 Intel Corporation. */ |
b4b8faa1 | 3 | |
248c7f9c | 4 | #include <asm/barrier.h> |
b4b8faa1 MK |
5 | #include <errno.h> |
6 | #include <getopt.h> | |
7 | #include <libgen.h> | |
8 | #include <linux/bpf.h> | |
248c7f9c | 9 | #include <linux/compiler.h> |
b4b8faa1 MK |
10 | #include <linux/if_link.h> |
11 | #include <linux/if_xdp.h> | |
12 | #include <linux/if_ether.h> | |
248c7f9c MK |
13 | #include <locale.h> |
14 | #include <net/ethernet.h> | |
b4b8faa1 | 15 | #include <net/if.h> |
248c7f9c MK |
16 | #include <poll.h> |
17 | #include <pthread.h> | |
b4b8faa1 MK |
18 | #include <signal.h> |
19 | #include <stdbool.h> | |
20 | #include <stdio.h> | |
21 | #include <stdlib.h> | |
22 | #include <string.h> | |
248c7f9c | 23 | #include <sys/mman.h> |
b4b8faa1 MK |
24 | #include <sys/resource.h> |
25 | #include <sys/socket.h> | |
248c7f9c | 26 | #include <sys/types.h> |
b4b8faa1 MK |
27 | #include <time.h> |
28 | #include <unistd.h> | |
b4b8faa1 | 29 | |
4d18f6de DL |
30 | #include "libbpf.h" |
31 | #include "xsk.h" | |
2bf3e2ef | 32 | #include <bpf/bpf.h> |
b4b8faa1 | 33 | |
b4b8faa1 MK |
34 | #ifndef SOL_XDP |
35 | #define SOL_XDP 283 | |
36 | #endif | |
37 | ||
38 | #ifndef AF_XDP | |
39 | #define AF_XDP 44 | |
40 | #endif | |
41 | ||
42 | #ifndef PF_XDP | |
43 | #define PF_XDP AF_XDP | |
44 | #endif | |
45 | ||
248c7f9c MK |
46 | #define NUM_FRAMES (4 * 1024) |
47 | #define BATCH_SIZE 64 | |
b4b8faa1 MK |
48 | |
49 | #define DEBUG_HEXDUMP 0 | |
248c7f9c | 50 | #define MAX_SOCKS 8 |
b4b8faa1 | 51 | |
a412ef54 | 52 | typedef __u64 u64; |
b4b8faa1 MK |
53 | typedef __u32 u32; |
54 | ||
55 | static unsigned long prev_time; | |
56 | ||
57 | enum benchmark_type { | |
58 | BENCH_RXDROP = 0, | |
59 | BENCH_TXONLY = 1, | |
60 | BENCH_L2FWD = 2, | |
61 | }; | |
62 | ||
63 | static enum benchmark_type opt_bench = BENCH_RXDROP; | |
743e568c | 64 | static u32 opt_xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST; |
b4b8faa1 MK |
65 | static const char *opt_if = ""; |
66 | static int opt_ifindex; | |
67 | static int opt_queue; | |
68 | static int opt_poll; | |
b4b8faa1 | 69 | static int opt_interval = 1; |
46738f73 | 70 | static u32 opt_xdp_bind_flags = XDP_USE_NEED_WAKEUP; |
c543f546 KL |
71 | static u32 opt_umem_flags; |
72 | static int opt_unaligned_chunks; | |
3945b37a | 73 | static int opt_mmap_flags; |
c543f546 | 74 | static u32 opt_xdp_bind_flags; |
123e8da1 | 75 | static int opt_xsk_frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; |
46738f73 MK |
76 | static int opt_timeout = 1000; |
77 | static bool opt_need_wakeup = true; | |
3b7a8ec2 | 78 | static __u32 prog_id; |
b4b8faa1 | 79 | |
248c7f9c MK |
80 | struct xsk_umem_info { |
81 | struct xsk_ring_prod fq; | |
82 | struct xsk_ring_cons cq; | |
83 | struct xsk_umem *umem; | |
84 | void *buffer; | |
b4b8faa1 MK |
85 | }; |
86 | ||
248c7f9c MK |
87 | struct xsk_socket_info { |
88 | struct xsk_ring_cons rx; | |
89 | struct xsk_ring_prod tx; | |
90 | struct xsk_umem_info *umem; | |
91 | struct xsk_socket *xsk; | |
b4b8faa1 MK |
92 | unsigned long rx_npkts; |
93 | unsigned long tx_npkts; | |
94 | unsigned long prev_rx_npkts; | |
95 | unsigned long prev_tx_npkts; | |
248c7f9c | 96 | u32 outstanding_tx; |
b4b8faa1 MK |
97 | }; |
98 | ||
b4b8faa1 | 99 | static int num_socks; |
248c7f9c | 100 | struct xsk_socket_info *xsks[MAX_SOCKS]; |
b4b8faa1 MK |
101 | |
102 | static unsigned long get_nsecs(void) | |
103 | { | |
104 | struct timespec ts; | |
105 | ||
106 | clock_gettime(CLOCK_MONOTONIC, &ts); | |
107 | return ts.tv_sec * 1000000000UL + ts.tv_nsec; | |
108 | } | |
109 | ||
248c7f9c | 110 | static void print_benchmark(bool running) |
b4b8faa1 | 111 | { |
248c7f9c | 112 | const char *bench_str = "INVALID"; |
b4b8faa1 | 113 | |
248c7f9c MK |
114 | if (opt_bench == BENCH_RXDROP) |
115 | bench_str = "rxdrop"; | |
116 | else if (opt_bench == BENCH_TXONLY) | |
117 | bench_str = "txonly"; | |
118 | else if (opt_bench == BENCH_L2FWD) | |
119 | bench_str = "l2fwd"; | |
b4b8faa1 | 120 | |
248c7f9c MK |
121 | printf("%s:%d %s ", opt_if, opt_queue, bench_str); |
122 | if (opt_xdp_flags & XDP_FLAGS_SKB_MODE) | |
123 | printf("xdp-skb "); | |
124 | else if (opt_xdp_flags & XDP_FLAGS_DRV_MODE) | |
125 | printf("xdp-drv "); | |
126 | else | |
127 | printf(" "); | |
b4b8faa1 | 128 | |
248c7f9c MK |
129 | if (opt_poll) |
130 | printf("poll() "); | |
b4b8faa1 | 131 | |
248c7f9c MK |
132 | if (running) { |
133 | printf("running..."); | |
134 | fflush(stdout); | |
b4b8faa1 | 135 | } |
b4b8faa1 MK |
136 | } |
137 | ||
248c7f9c | 138 | static void dump_stats(void) |
b4b8faa1 | 139 | { |
248c7f9c MK |
140 | unsigned long now = get_nsecs(); |
141 | long dt = now - prev_time; | |
142 | int i; | |
b4b8faa1 | 143 | |
248c7f9c | 144 | prev_time = now; |
b4b8faa1 | 145 | |
248c7f9c MK |
146 | for (i = 0; i < num_socks && xsks[i]; i++) { |
147 | char *fmt = "%-15s %'-11.0f %'-11lu\n"; | |
148 | double rx_pps, tx_pps; | |
b4b8faa1 | 149 | |
248c7f9c MK |
150 | rx_pps = (xsks[i]->rx_npkts - xsks[i]->prev_rx_npkts) * |
151 | 1000000000. / dt; | |
152 | tx_pps = (xsks[i]->tx_npkts - xsks[i]->prev_tx_npkts) * | |
153 | 1000000000. / dt; | |
b4b8faa1 | 154 | |
248c7f9c MK |
155 | printf("\n sock%d@", i); |
156 | print_benchmark(false); | |
157 | printf("\n"); | |
b4b8faa1 | 158 | |
248c7f9c MK |
159 | printf("%-15s %-11s %-11s %-11.2f\n", "", "pps", "pkts", |
160 | dt / 1000000000.); | |
161 | printf(fmt, "rx", rx_pps, xsks[i]->rx_npkts); | |
162 | printf(fmt, "tx", tx_pps, xsks[i]->tx_npkts); | |
b4b8faa1 | 163 | |
248c7f9c MK |
164 | xsks[i]->prev_rx_npkts = xsks[i]->rx_npkts; |
165 | xsks[i]->prev_tx_npkts = xsks[i]->tx_npkts; | |
b4b8faa1 | 166 | } |
b4b8faa1 MK |
167 | } |
168 | ||
248c7f9c | 169 | static void *poller(void *arg) |
b4b8faa1 | 170 | { |
248c7f9c MK |
171 | (void)arg; |
172 | for (;;) { | |
173 | sleep(opt_interval); | |
174 | dump_stats(); | |
b4b8faa1 MK |
175 | } |
176 | ||
248c7f9c | 177 | return NULL; |
b4b8faa1 MK |
178 | } |
179 | ||
248c7f9c | 180 | static void remove_xdp_program(void) |
b4b8faa1 | 181 | { |
248c7f9c | 182 | __u32 curr_prog_id = 0; |
b4b8faa1 | 183 | |
248c7f9c MK |
184 | if (bpf_get_link_xdp_id(opt_ifindex, &curr_prog_id, opt_xdp_flags)) { |
185 | printf("bpf_get_link_xdp_id failed\n"); | |
186 | exit(EXIT_FAILURE); | |
b4b8faa1 | 187 | } |
248c7f9c MK |
188 | if (prog_id == curr_prog_id) |
189 | bpf_set_link_xdp_fd(opt_ifindex, -1, opt_xdp_flags); | |
190 | else if (!curr_prog_id) | |
191 | printf("couldn't find a prog id on a given interface\n"); | |
192 | else | |
193 | printf("program on interface changed, not removing\n"); | |
b4b8faa1 MK |
194 | } |
195 | ||
248c7f9c | 196 | static void int_exit(int sig) |
b4b8faa1 | 197 | { |
248c7f9c | 198 | struct xsk_umem *umem = xsks[0]->umem->umem; |
b4b8faa1 | 199 | |
248c7f9c | 200 | (void)sig; |
b4b8faa1 | 201 | |
248c7f9c MK |
202 | dump_stats(); |
203 | xsk_socket__delete(xsks[0]->xsk); | |
204 | (void)xsk_umem__delete(umem); | |
205 | remove_xdp_program(); | |
b4b8faa1 | 206 | |
248c7f9c | 207 | exit(EXIT_SUCCESS); |
b4b8faa1 MK |
208 | } |
209 | ||
248c7f9c MK |
210 | static void __exit_with_error(int error, const char *file, const char *func, |
211 | int line) | |
b4b8faa1 | 212 | { |
248c7f9c MK |
213 | fprintf(stderr, "%s:%s:%i: errno: %d/\"%s\"\n", file, func, |
214 | line, error, strerror(error)); | |
215 | dump_stats(); | |
216 | remove_xdp_program(); | |
217 | exit(EXIT_FAILURE); | |
b4b8faa1 MK |
218 | } |
219 | ||
248c7f9c MK |
220 | #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, \ |
221 | __LINE__) | |
b4b8faa1 | 222 | |
248c7f9c MK |
223 | static const char pkt_data[] = |
224 | "\x3c\xfd\xfe\x9e\x7f\x71\xec\xb1\xd7\x98\x3a\xc0\x08\x00\x45\x00" | |
225 | "\x00\x2e\x00\x00\x00\x00\x40\x11\x88\x97\x05\x08\x07\x08\xc8\x14" | |
226 | "\x1e\x04\x10\x92\x10\x92\x00\x1a\x6d\xa3\x34\x33\x1f\x69\x40\x6b" | |
227 | "\x54\x59\xb6\x14\x2d\x11\x44\xbf\xaf\xd9\xbe\xaa"; | |
b4b8faa1 MK |
228 | |
229 | static void swap_mac_addresses(void *data) | |
230 | { | |
231 | struct ether_header *eth = (struct ether_header *)data; | |
232 | struct ether_addr *src_addr = (struct ether_addr *)ð->ether_shost; | |
233 | struct ether_addr *dst_addr = (struct ether_addr *)ð->ether_dhost; | |
234 | struct ether_addr tmp; | |
235 | ||
236 | tmp = *src_addr; | |
237 | *src_addr = *dst_addr; | |
238 | *dst_addr = tmp; | |
239 | } | |
240 | ||
a412ef54 | 241 | static void hex_dump(void *pkt, size_t length, u64 addr) |
b4b8faa1 | 242 | { |
b4b8faa1 MK |
243 | const unsigned char *address = (unsigned char *)pkt; |
244 | const unsigned char *line = address; | |
245 | size_t line_size = 32; | |
246 | unsigned char c; | |
a412ef54 BT |
247 | char buf[32]; |
248 | int i = 0; | |
b4b8faa1 | 249 | |
a412ef54 BT |
250 | if (!DEBUG_HEXDUMP) |
251 | return; | |
252 | ||
253 | sprintf(buf, "addr=%llu", addr); | |
b4b8faa1 | 254 | printf("length = %zu\n", length); |
a412ef54 | 255 | printf("%s | ", buf); |
b4b8faa1 MK |
256 | while (length-- > 0) { |
257 | printf("%02X ", *address++); | |
258 | if (!(++i % line_size) || (length == 0 && i % line_size)) { | |
259 | if (length == 0) { | |
260 | while (i++ % line_size) | |
261 | printf("__ "); | |
262 | } | |
263 | printf(" | "); /* right close */ | |
264 | while (line < address) { | |
265 | c = *line++; | |
266 | printf("%c", (c < 33 || c == 255) ? 0x2E : c); | |
267 | } | |
268 | printf("\n"); | |
269 | if (length > 0) | |
a412ef54 | 270 | printf("%s | ", buf); |
b4b8faa1 MK |
271 | } |
272 | } | |
273 | printf("\n"); | |
274 | } | |
b4b8faa1 | 275 | |
248c7f9c | 276 | static size_t gen_eth_frame(struct xsk_umem_info *umem, u64 addr) |
b4b8faa1 | 277 | { |
248c7f9c MK |
278 | memcpy(xsk_umem__get_data(umem->buffer, addr), pkt_data, |
279 | sizeof(pkt_data) - 1); | |
b4b8faa1 MK |
280 | return sizeof(pkt_data) - 1; |
281 | } | |
282 | ||
248c7f9c | 283 | static struct xsk_umem_info *xsk_configure_umem(void *buffer, u64 size) |
b4b8faa1 | 284 | { |
248c7f9c | 285 | struct xsk_umem_info *umem; |
123e8da1 MM |
286 | struct xsk_umem_config cfg = { |
287 | .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS, | |
288 | .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, | |
289 | .frame_size = opt_xsk_frame_size, | |
290 | .frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM, | |
c543f546 | 291 | .flags = opt_umem_flags |
123e8da1 | 292 | }; |
c543f546 | 293 | |
248c7f9c | 294 | int ret; |
b4b8faa1 MK |
295 | |
296 | umem = calloc(1, sizeof(*umem)); | |
248c7f9c MK |
297 | if (!umem) |
298 | exit_with_error(errno); | |
b4b8faa1 | 299 | |
248c7f9c | 300 | ret = xsk_umem__create(&umem->umem, buffer, size, &umem->fq, &umem->cq, |
123e8da1 | 301 | &cfg); |
c543f546 | 302 | |
248c7f9c MK |
303 | if (ret) |
304 | exit_with_error(-ret); | |
b4b8faa1 | 305 | |
248c7f9c | 306 | umem->buffer = buffer; |
b4b8faa1 MK |
307 | return umem; |
308 | } | |
309 | ||
248c7f9c | 310 | static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem) |
b4b8faa1 | 311 | { |
248c7f9c MK |
312 | struct xsk_socket_config cfg; |
313 | struct xsk_socket_info *xsk; | |
314 | int ret; | |
315 | u32 idx; | |
316 | int i; | |
b4b8faa1 MK |
317 | |
318 | xsk = calloc(1, sizeof(*xsk)); | |
248c7f9c MK |
319 | if (!xsk) |
320 | exit_with_error(errno); | |
321 | ||
322 | xsk->umem = umem; | |
323 | cfg.rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS; | |
324 | cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; | |
325 | cfg.libbpf_flags = 0; | |
326 | cfg.xdp_flags = opt_xdp_flags; | |
327 | cfg.bind_flags = opt_xdp_bind_flags; | |
328 | ret = xsk_socket__create(&xsk->xsk, opt_if, opt_queue, umem->umem, | |
329 | &xsk->rx, &xsk->tx, &cfg); | |
330 | if (ret) | |
331 | exit_with_error(-ret); | |
332 | ||
333 | ret = bpf_get_link_xdp_id(opt_ifindex, &prog_id, opt_xdp_flags); | |
334 | if (ret) | |
335 | exit_with_error(-ret); | |
336 | ||
337 | ret = xsk_ring_prod__reserve(&xsk->umem->fq, | |
338 | XSK_RING_PROD__DEFAULT_NUM_DESCS, | |
339 | &idx); | |
340 | if (ret != XSK_RING_PROD__DEFAULT_NUM_DESCS) | |
341 | exit_with_error(-ret); | |
123e8da1 MM |
342 | for (i = 0; i < XSK_RING_PROD__DEFAULT_NUM_DESCS; i++) |
343 | *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx++) = | |
344 | i * opt_xsk_frame_size; | |
248c7f9c MK |
345 | xsk_ring_prod__submit(&xsk->umem->fq, |
346 | XSK_RING_PROD__DEFAULT_NUM_DESCS); | |
b4b8faa1 MK |
347 | |
348 | return xsk; | |
349 | } | |
350 | ||
b4b8faa1 MK |
351 | static struct option long_options[] = { |
352 | {"rxdrop", no_argument, 0, 'r'}, | |
353 | {"txonly", no_argument, 0, 't'}, | |
354 | {"l2fwd", no_argument, 0, 'l'}, | |
355 | {"interface", required_argument, 0, 'i'}, | |
356 | {"queue", required_argument, 0, 'q'}, | |
357 | {"poll", no_argument, 0, 'p'}, | |
b4b8faa1 MK |
358 | {"xdp-skb", no_argument, 0, 'S'}, |
359 | {"xdp-native", no_argument, 0, 'N'}, | |
360 | {"interval", required_argument, 0, 'n'}, | |
58c50ae4 BT |
361 | {"zero-copy", no_argument, 0, 'z'}, |
362 | {"copy", no_argument, 0, 'c'}, | |
123e8da1 | 363 | {"frame-size", required_argument, 0, 'f'}, |
46738f73 | 364 | {"no-need-wakeup", no_argument, 0, 'm'}, |
c543f546 | 365 | {"unaligned", no_argument, 0, 'u'}, |
b4b8faa1 MK |
366 | {0, 0, 0, 0} |
367 | }; | |
368 | ||
369 | static void usage(const char *prog) | |
370 | { | |
371 | const char *str = | |
372 | " Usage: %s [OPTIONS]\n" | |
373 | " Options:\n" | |
374 | " -r, --rxdrop Discard all incoming packets (default)\n" | |
375 | " -t, --txonly Only send packets\n" | |
376 | " -l, --l2fwd MAC swap L2 forwarding\n" | |
377 | " -i, --interface=n Run on interface n\n" | |
378 | " -q, --queue=n Use queue n (default 0)\n" | |
379 | " -p, --poll Use poll syscall\n" | |
b4b8faa1 | 380 | " -S, --xdp-skb=n Use XDP skb-mod\n" |
4564a8bb | 381 | " -N, --xdp-native=n Enforce XDP native mode\n" |
b4b8faa1 | 382 | " -n, --interval=n Specify statistics update interval (default 1 sec).\n" |
58c50ae4 BT |
383 | " -z, --zero-copy Force zero-copy mode.\n" |
384 | " -c, --copy Force copy mode.\n" | |
123e8da1 | 385 | " -f, --frame-size=n Set the frame size (must be a power of two, default is %d).\n" |
46738f73 | 386 | " -m, --no-need-wakeup Turn off use of driver need wakeup flag.\n" |
c543f546 KL |
387 | " -f, --frame-size=n Set the frame size (must be a power of two in aligned mode, default is %d).\n" |
388 | " -u, --unaligned Enable unaligned chunk placement\n" | |
b4b8faa1 | 389 | "\n"; |
123e8da1 | 390 | fprintf(stderr, str, prog, XSK_UMEM__DEFAULT_FRAME_SIZE); |
b4b8faa1 MK |
391 | exit(EXIT_FAILURE); |
392 | } | |
393 | ||
394 | static void parse_command_line(int argc, char **argv) | |
395 | { | |
396 | int option_index, c; | |
397 | ||
398 | opterr = 0; | |
399 | ||
400 | for (;;) { | |
c543f546 | 401 | c = getopt_long(argc, argv, "Frtli:q:psSNn:czf:mu", |
46738f73 | 402 | long_options, &option_index); |
b4b8faa1 MK |
403 | if (c == -1) |
404 | break; | |
405 | ||
406 | switch (c) { | |
407 | case 'r': | |
408 | opt_bench = BENCH_RXDROP; | |
409 | break; | |
410 | case 't': | |
411 | opt_bench = BENCH_TXONLY; | |
412 | break; | |
413 | case 'l': | |
414 | opt_bench = BENCH_L2FWD; | |
415 | break; | |
416 | case 'i': | |
417 | opt_if = optarg; | |
418 | break; | |
419 | case 'q': | |
420 | opt_queue = atoi(optarg); | |
421 | break; | |
b4b8faa1 MK |
422 | case 'p': |
423 | opt_poll = 1; | |
424 | break; | |
425 | case 'S': | |
426 | opt_xdp_flags |= XDP_FLAGS_SKB_MODE; | |
9f5232cc | 427 | opt_xdp_bind_flags |= XDP_COPY; |
b4b8faa1 MK |
428 | break; |
429 | case 'N': | |
430 | opt_xdp_flags |= XDP_FLAGS_DRV_MODE; | |
431 | break; | |
432 | case 'n': | |
433 | opt_interval = atoi(optarg); | |
434 | break; | |
58c50ae4 BT |
435 | case 'z': |
436 | opt_xdp_bind_flags |= XDP_ZEROCOPY; | |
437 | break; | |
438 | case 'c': | |
439 | opt_xdp_bind_flags |= XDP_COPY; | |
440 | break; | |
c543f546 KL |
441 | case 'u': |
442 | opt_umem_flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG; | |
443 | opt_unaligned_chunks = 1; | |
3945b37a | 444 | opt_mmap_flags = MAP_HUGETLB; |
c543f546 | 445 | break; |
743e568c MF |
446 | case 'F': |
447 | opt_xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST; | |
448 | break; | |
123e8da1 MM |
449 | case 'f': |
450 | opt_xsk_frame_size = atoi(optarg); | |
46738f73 MK |
451 | case 'm': |
452 | opt_need_wakeup = false; | |
453 | opt_xdp_bind_flags &= ~XDP_USE_NEED_WAKEUP; | |
123e8da1 | 454 | break; |
c543f546 | 455 | |
b4b8faa1 MK |
456 | default: |
457 | usage(basename(argv[0])); | |
458 | } | |
459 | } | |
460 | ||
461 | opt_ifindex = if_nametoindex(opt_if); | |
462 | if (!opt_ifindex) { | |
463 | fprintf(stderr, "ERROR: interface \"%s\" does not exist\n", | |
464 | opt_if); | |
465 | usage(basename(argv[0])); | |
466 | } | |
248c7f9c | 467 | |
c543f546 KL |
468 | if ((opt_xsk_frame_size & (opt_xsk_frame_size - 1)) && |
469 | !opt_unaligned_chunks) { | |
123e8da1 MM |
470 | fprintf(stderr, "--frame-size=%d is not a power of two\n", |
471 | opt_xsk_frame_size); | |
472 | usage(basename(argv[0])); | |
473 | } | |
b4b8faa1 MK |
474 | } |
475 | ||
248c7f9c | 476 | static void kick_tx(struct xsk_socket_info *xsk) |
b4b8faa1 MK |
477 | { |
478 | int ret; | |
479 | ||
248c7f9c | 480 | ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); |
c03079c9 | 481 | if (ret >= 0 || errno == ENOBUFS || errno == EAGAIN || errno == EBUSY) |
b4b8faa1 | 482 | return; |
248c7f9c | 483 | exit_with_error(errno); |
b4b8faa1 MK |
484 | } |
485 | ||
46738f73 MK |
486 | static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk, |
487 | struct pollfd *fds) | |
b4b8faa1 | 488 | { |
03895e63 | 489 | struct xsk_umem_info *umem = xsk->umem; |
b74e21ab | 490 | u32 idx_cq = 0, idx_fq = 0; |
b4b8faa1 MK |
491 | unsigned int rcvd; |
492 | size_t ndescs; | |
493 | ||
494 | if (!xsk->outstanding_tx) | |
495 | return; | |
496 | ||
46738f73 MK |
497 | if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx)) |
498 | kick_tx(xsk); | |
499 | ||
b4b8faa1 | 500 | ndescs = (xsk->outstanding_tx > BATCH_SIZE) ? BATCH_SIZE : |
248c7f9c | 501 | xsk->outstanding_tx; |
b4b8faa1 MK |
502 | |
503 | /* re-add completed Tx buffers */ | |
03895e63 | 504 | rcvd = xsk_ring_cons__peek(&umem->cq, ndescs, &idx_cq); |
b4b8faa1 | 505 | if (rcvd > 0) { |
248c7f9c MK |
506 | unsigned int i; |
507 | int ret; | |
508 | ||
03895e63 | 509 | ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); |
248c7f9c MK |
510 | while (ret != rcvd) { |
511 | if (ret < 0) | |
512 | exit_with_error(-ret); | |
03895e63 | 513 | if (xsk_ring_prod__needs_wakeup(&umem->fq)) |
46738f73 | 514 | ret = poll(fds, num_socks, opt_timeout); |
03895e63 | 515 | ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); |
248c7f9c | 516 | } |
03895e63 | 517 | |
248c7f9c | 518 | for (i = 0; i < rcvd; i++) |
03895e63 KL |
519 | *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = |
520 | *xsk_ring_cons__comp_addr(&umem->cq, idx_cq++); | |
248c7f9c MK |
521 | |
522 | xsk_ring_prod__submit(&xsk->umem->fq, rcvd); | |
523 | xsk_ring_cons__release(&xsk->umem->cq, rcvd); | |
b4b8faa1 MK |
524 | xsk->outstanding_tx -= rcvd; |
525 | xsk->tx_npkts += rcvd; | |
526 | } | |
527 | } | |
528 | ||
248c7f9c | 529 | static inline void complete_tx_only(struct xsk_socket_info *xsk) |
b4b8faa1 | 530 | { |
b4b8faa1 | 531 | unsigned int rcvd; |
248c7f9c | 532 | u32 idx; |
b4b8faa1 MK |
533 | |
534 | if (!xsk->outstanding_tx) | |
535 | return; | |
536 | ||
46738f73 MK |
537 | if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx)) |
538 | kick_tx(xsk); | |
b4b8faa1 | 539 | |
248c7f9c | 540 | rcvd = xsk_ring_cons__peek(&xsk->umem->cq, BATCH_SIZE, &idx); |
b4b8faa1 | 541 | if (rcvd > 0) { |
248c7f9c | 542 | xsk_ring_cons__release(&xsk->umem->cq, rcvd); |
b4b8faa1 MK |
543 | xsk->outstanding_tx -= rcvd; |
544 | xsk->tx_npkts += rcvd; | |
545 | } | |
546 | } | |
547 | ||
46738f73 | 548 | static void rx_drop(struct xsk_socket_info *xsk, struct pollfd *fds) |
b4b8faa1 | 549 | { |
b4b8faa1 | 550 | unsigned int rcvd, i; |
b74e21ab | 551 | u32 idx_rx = 0, idx_fq = 0; |
248c7f9c | 552 | int ret; |
b4b8faa1 | 553 | |
248c7f9c | 554 | rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx); |
46738f73 MK |
555 | if (!rcvd) { |
556 | if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) | |
557 | ret = poll(fds, num_socks, opt_timeout); | |
b4b8faa1 | 558 | return; |
46738f73 | 559 | } |
b4b8faa1 | 560 | |
248c7f9c MK |
561 | ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq); |
562 | while (ret != rcvd) { | |
563 | if (ret < 0) | |
564 | exit_with_error(-ret); | |
46738f73 MK |
565 | if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) |
566 | ret = poll(fds, num_socks, opt_timeout); | |
248c7f9c MK |
567 | ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq); |
568 | } | |
569 | ||
b4b8faa1 | 570 | for (i = 0; i < rcvd; i++) { |
248c7f9c MK |
571 | u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr; |
572 | u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len; | |
03895e63 KL |
573 | u64 orig = xsk_umem__extract_addr(addr); |
574 | ||
575 | addr = xsk_umem__add_offset_to_addr(addr); | |
248c7f9c | 576 | char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr); |
b4b8faa1 | 577 | |
248c7f9c | 578 | hex_dump(pkt, len, addr); |
03895e63 | 579 | *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx_fq++) = orig; |
b4b8faa1 MK |
580 | } |
581 | ||
248c7f9c MK |
582 | xsk_ring_prod__submit(&xsk->umem->fq, rcvd); |
583 | xsk_ring_cons__release(&xsk->rx, rcvd); | |
b4b8faa1 | 584 | xsk->rx_npkts += rcvd; |
b4b8faa1 MK |
585 | } |
586 | ||
587 | static void rx_drop_all(void) | |
588 | { | |
589 | struct pollfd fds[MAX_SOCKS + 1]; | |
46738f73 | 590 | int i, ret; |
b4b8faa1 MK |
591 | |
592 | memset(fds, 0, sizeof(fds)); | |
593 | ||
594 | for (i = 0; i < num_socks; i++) { | |
248c7f9c | 595 | fds[i].fd = xsk_socket__fd(xsks[i]->xsk); |
b4b8faa1 | 596 | fds[i].events = POLLIN; |
b4b8faa1 MK |
597 | } |
598 | ||
599 | for (;;) { | |
600 | if (opt_poll) { | |
46738f73 | 601 | ret = poll(fds, num_socks, opt_timeout); |
b4b8faa1 MK |
602 | if (ret <= 0) |
603 | continue; | |
604 | } | |
605 | ||
606 | for (i = 0; i < num_socks; i++) | |
46738f73 MK |
607 | rx_drop(xsks[i], fds); |
608 | } | |
609 | } | |
610 | ||
611 | static void tx_only(struct xsk_socket_info *xsk, u32 frame_nb) | |
612 | { | |
613 | u32 idx; | |
614 | ||
615 | if (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) == BATCH_SIZE) { | |
616 | unsigned int i; | |
617 | ||
618 | for (i = 0; i < BATCH_SIZE; i++) { | |
619 | xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->addr = | |
620 | (frame_nb + i) << XSK_UMEM__DEFAULT_FRAME_SHIFT; | |
621 | xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->len = | |
622 | sizeof(pkt_data) - 1; | |
623 | } | |
624 | ||
625 | xsk_ring_prod__submit(&xsk->tx, BATCH_SIZE); | |
626 | xsk->outstanding_tx += BATCH_SIZE; | |
627 | frame_nb += BATCH_SIZE; | |
628 | frame_nb %= NUM_FRAMES; | |
b4b8faa1 | 629 | } |
46738f73 MK |
630 | |
631 | complete_tx_only(xsk); | |
b4b8faa1 MK |
632 | } |
633 | ||
46738f73 | 634 | static void tx_only_all(void) |
b4b8faa1 | 635 | { |
46738f73 MK |
636 | struct pollfd fds[MAX_SOCKS]; |
637 | u32 frame_nb[MAX_SOCKS] = {}; | |
638 | int i, ret; | |
b4b8faa1 MK |
639 | |
640 | memset(fds, 0, sizeof(fds)); | |
46738f73 MK |
641 | for (i = 0; i < num_socks; i++) { |
642 | fds[0].fd = xsk_socket__fd(xsks[i]->xsk); | |
643 | fds[0].events = POLLOUT; | |
644 | } | |
b4b8faa1 MK |
645 | |
646 | for (;;) { | |
647 | if (opt_poll) { | |
46738f73 | 648 | ret = poll(fds, num_socks, opt_timeout); |
b4b8faa1 MK |
649 | if (ret <= 0) |
650 | continue; | |
651 | ||
248c7f9c | 652 | if (!(fds[0].revents & POLLOUT)) |
b4b8faa1 MK |
653 | continue; |
654 | } | |
655 | ||
46738f73 MK |
656 | for (i = 0; i < num_socks; i++) |
657 | tx_only(xsks[i], frame_nb[i]); | |
b4b8faa1 MK |
658 | } |
659 | } | |
660 | ||
46738f73 | 661 | static void l2fwd(struct xsk_socket_info *xsk, struct pollfd *fds) |
b4b8faa1 | 662 | { |
46738f73 MK |
663 | unsigned int rcvd, i; |
664 | u32 idx_rx = 0, idx_tx = 0; | |
665 | int ret; | |
b4b8faa1 | 666 | |
46738f73 | 667 | complete_tx_l2fwd(xsk, fds); |
b4b8faa1 | 668 | |
46738f73 MK |
669 | rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx); |
670 | if (!rcvd) { | |
671 | if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) | |
672 | ret = poll(fds, num_socks, opt_timeout); | |
673 | return; | |
674 | } | |
b4b8faa1 | 675 | |
46738f73 MK |
676 | ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx); |
677 | while (ret != rcvd) { | |
678 | if (ret < 0) | |
679 | exit_with_error(-ret); | |
680 | if (xsk_ring_prod__needs_wakeup(&xsk->tx)) | |
681 | kick_tx(xsk); | |
248c7f9c | 682 | ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx); |
46738f73 MK |
683 | } |
684 | ||
685 | for (i = 0; i < rcvd; i++) { | |
686 | u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr; | |
687 | u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len; | |
5a712e13 | 688 | u64 orig = addr; |
03895e63 KL |
689 | |
690 | addr = xsk_umem__add_offset_to_addr(addr); | |
46738f73 MK |
691 | char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr); |
692 | ||
693 | swap_mac_addresses(pkt); | |
248c7f9c | 694 | |
46738f73 | 695 | hex_dump(pkt, len, addr); |
03895e63 | 696 | xsk_ring_prod__tx_desc(&xsk->tx, idx_tx)->addr = orig; |
46738f73 MK |
697 | xsk_ring_prod__tx_desc(&xsk->tx, idx_tx++)->len = len; |
698 | } | |
b4b8faa1 | 699 | |
46738f73 MK |
700 | xsk_ring_prod__submit(&xsk->tx, rcvd); |
701 | xsk_ring_cons__release(&xsk->rx, rcvd); | |
b4b8faa1 | 702 | |
46738f73 MK |
703 | xsk->rx_npkts += rcvd; |
704 | xsk->outstanding_tx += rcvd; | |
705 | } | |
706 | ||
707 | static void l2fwd_all(void) | |
708 | { | |
709 | struct pollfd fds[MAX_SOCKS]; | |
710 | int i, ret; | |
711 | ||
712 | memset(fds, 0, sizeof(fds)); | |
713 | ||
714 | for (i = 0; i < num_socks; i++) { | |
715 | fds[i].fd = xsk_socket__fd(xsks[i]->xsk); | |
716 | fds[i].events = POLLOUT | POLLIN; | |
717 | } | |
b4b8faa1 | 718 | |
46738f73 MK |
719 | for (;;) { |
720 | if (opt_poll) { | |
721 | ret = poll(fds, num_socks, opt_timeout); | |
722 | if (ret <= 0) | |
723 | continue; | |
724 | } | |
b4b8faa1 | 725 | |
46738f73 MK |
726 | for (i = 0; i < num_socks; i++) |
727 | l2fwd(xsks[i], fds); | |
b4b8faa1 MK |
728 | } |
729 | } | |
730 | ||
731 | int main(int argc, char **argv) | |
732 | { | |
733 | struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY}; | |
248c7f9c | 734 | struct xsk_umem_info *umem; |
b4b8faa1 | 735 | pthread_t pt; |
248c7f9c MK |
736 | void *bufs; |
737 | int ret; | |
b4b8faa1 MK |
738 | |
739 | parse_command_line(argc, argv); | |
740 | ||
741 | if (setrlimit(RLIMIT_MEMLOCK, &r)) { | |
742 | fprintf(stderr, "ERROR: setrlimit(RLIMIT_MEMLOCK) \"%s\"\n", | |
743 | strerror(errno)); | |
744 | exit(EXIT_FAILURE); | |
745 | } | |
746 | ||
3945b37a KL |
747 | /* Reserve memory for the umem. Use hugepages if unaligned chunk mode */ |
748 | bufs = mmap(NULL, NUM_FRAMES * opt_xsk_frame_size, | |
749 | PROT_READ | PROT_WRITE, | |
750 | MAP_PRIVATE | MAP_ANONYMOUS | opt_mmap_flags, -1, 0); | |
751 | if (bufs == MAP_FAILED) { | |
752 | printf("ERROR: mmap failed\n"); | |
753 | exit(EXIT_FAILURE); | |
754 | } | |
248c7f9c | 755 | /* Create sockets... */ |
123e8da1 | 756 | umem = xsk_configure_umem(bufs, NUM_FRAMES * opt_xsk_frame_size); |
248c7f9c | 757 | xsks[num_socks++] = xsk_configure_socket(umem); |
b4b8faa1 | 758 | |
248c7f9c MK |
759 | if (opt_bench == BENCH_TXONLY) { |
760 | int i; | |
b4b8faa1 | 761 | |
123e8da1 MM |
762 | for (i = 0; i < NUM_FRAMES; i++) |
763 | (void)gen_eth_frame(umem, i * opt_xsk_frame_size); | |
b4b8faa1 MK |
764 | } |
765 | ||
766 | signal(SIGINT, int_exit); | |
767 | signal(SIGTERM, int_exit); | |
768 | signal(SIGABRT, int_exit); | |
769 | ||
770 | setlocale(LC_ALL, ""); | |
771 | ||
772 | ret = pthread_create(&pt, NULL, poller, NULL); | |
248c7f9c MK |
773 | if (ret) |
774 | exit_with_error(ret); | |
b4b8faa1 MK |
775 | |
776 | prev_time = get_nsecs(); | |
777 | ||
778 | if (opt_bench == BENCH_RXDROP) | |
779 | rx_drop_all(); | |
780 | else if (opt_bench == BENCH_TXONLY) | |
46738f73 | 781 | tx_only_all(); |
b4b8faa1 | 782 | else |
46738f73 | 783 | l2fwd_all(); |
b4b8faa1 MK |
784 | |
785 | return 0; | |
786 | } |