]>
Commit | Line | Data |
---|---|---|
b4b8faa1 | 1 | // SPDX-License-Identifier: GPL-2.0 |
dac09149 | 2 | /* Copyright(c) 2017 - 2018 Intel Corporation. */ |
b4b8faa1 | 3 | |
248c7f9c | 4 | #include <asm/barrier.h> |
b4b8faa1 MK |
5 | #include <errno.h> |
6 | #include <getopt.h> | |
7 | #include <libgen.h> | |
8 | #include <linux/bpf.h> | |
248c7f9c | 9 | #include <linux/compiler.h> |
b4b8faa1 MK |
10 | #include <linux/if_link.h> |
11 | #include <linux/if_xdp.h> | |
12 | #include <linux/if_ether.h> | |
4a3c23ae JJ |
13 | #include <linux/ip.h> |
14 | #include <linux/udp.h> | |
15 | #include <arpa/inet.h> | |
248c7f9c MK |
16 | #include <locale.h> |
17 | #include <net/ethernet.h> | |
b4b8faa1 | 18 | #include <net/if.h> |
248c7f9c MK |
19 | #include <poll.h> |
20 | #include <pthread.h> | |
b4b8faa1 MK |
21 | #include <signal.h> |
22 | #include <stdbool.h> | |
23 | #include <stdio.h> | |
24 | #include <stdlib.h> | |
25 | #include <string.h> | |
248c7f9c | 26 | #include <sys/mman.h> |
b4b8faa1 MK |
27 | #include <sys/resource.h> |
28 | #include <sys/socket.h> | |
248c7f9c | 29 | #include <sys/types.h> |
b4b8faa1 MK |
30 | #include <time.h> |
31 | #include <unistd.h> | |
b4b8faa1 | 32 | |
7cf245a3 THJ |
33 | #include <bpf/libbpf.h> |
34 | #include <bpf/xsk.h> | |
2bf3e2ef | 35 | #include <bpf/bpf.h> |
7cf245a3 | 36 | #include "xdpsock.h" |
b4b8faa1 | 37 | |
b4b8faa1 MK |
38 | #ifndef SOL_XDP |
39 | #define SOL_XDP 283 | |
40 | #endif | |
41 | ||
42 | #ifndef AF_XDP | |
43 | #define AF_XDP 44 | |
44 | #endif | |
45 | ||
46 | #ifndef PF_XDP | |
47 | #define PF_XDP AF_XDP | |
48 | #endif | |
49 | ||
248c7f9c | 50 | #define NUM_FRAMES (4 * 1024) |
4a3c23ae | 51 | #define MIN_PKT_SIZE 64 |
b4b8faa1 MK |
52 | |
53 | #define DEBUG_HEXDUMP 0 | |
54 | ||
a412ef54 | 55 | typedef __u64 u64; |
b4b8faa1 | 56 | typedef __u32 u32; |
4a3c23ae JJ |
57 | typedef __u16 u16; |
58 | typedef __u8 u8; | |
b4b8faa1 MK |
59 | |
60 | static unsigned long prev_time; | |
61 | ||
62 | enum benchmark_type { | |
63 | BENCH_RXDROP = 0, | |
64 | BENCH_TXONLY = 1, | |
65 | BENCH_L2FWD = 2, | |
66 | }; | |
67 | ||
68 | static enum benchmark_type opt_bench = BENCH_RXDROP; | |
743e568c | 69 | static u32 opt_xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST; |
b4b8faa1 MK |
70 | static const char *opt_if = ""; |
71 | static int opt_ifindex; | |
72 | static int opt_queue; | |
d3f11b01 JJ |
73 | static unsigned long opt_duration; |
74 | static unsigned long start_time; | |
75 | static bool benchmark_done; | |
cd9e72b6 | 76 | static u32 opt_batch_size = 64; |
ece6e969 | 77 | static int opt_pkt_count; |
4a3c23ae | 78 | static u16 opt_pkt_size = MIN_PKT_SIZE; |
46e3268e | 79 | static u32 opt_pkt_fill_pattern = 0x12345678; |
b36c3206 | 80 | static bool opt_extra_stats; |
b4b8faa1 | 81 | static int opt_poll; |
b4b8faa1 | 82 | static int opt_interval = 1; |
46738f73 | 83 | static u32 opt_xdp_bind_flags = XDP_USE_NEED_WAKEUP; |
c543f546 KL |
84 | static u32 opt_umem_flags; |
85 | static int opt_unaligned_chunks; | |
3945b37a | 86 | static int opt_mmap_flags; |
123e8da1 | 87 | static int opt_xsk_frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; |
46738f73 MK |
88 | static int opt_timeout = 1000; |
89 | static bool opt_need_wakeup = true; | |
2e5d72c1 MK |
90 | static u32 opt_num_xsks = 1; |
91 | static u32 prog_id; | |
b4b8faa1 | 92 | |
248c7f9c MK |
93 | struct xsk_umem_info { |
94 | struct xsk_ring_prod fq; | |
95 | struct xsk_ring_cons cq; | |
96 | struct xsk_umem *umem; | |
97 | void *buffer; | |
b4b8faa1 MK |
98 | }; |
99 | ||
248c7f9c MK |
100 | struct xsk_socket_info { |
101 | struct xsk_ring_cons rx; | |
102 | struct xsk_ring_prod tx; | |
103 | struct xsk_umem_info *umem; | |
104 | struct xsk_socket *xsk; | |
b4b8faa1 MK |
105 | unsigned long rx_npkts; |
106 | unsigned long tx_npkts; | |
b36c3206 CL |
107 | unsigned long rx_dropped_npkts; |
108 | unsigned long rx_invalid_npkts; | |
109 | unsigned long tx_invalid_npkts; | |
110 | unsigned long rx_full_npkts; | |
111 | unsigned long rx_fill_empty_npkts; | |
112 | unsigned long tx_empty_npkts; | |
b4b8faa1 MK |
113 | unsigned long prev_rx_npkts; |
114 | unsigned long prev_tx_npkts; | |
b36c3206 CL |
115 | unsigned long prev_rx_dropped_npkts; |
116 | unsigned long prev_rx_invalid_npkts; | |
117 | unsigned long prev_tx_invalid_npkts; | |
118 | unsigned long prev_rx_full_npkts; | |
119 | unsigned long prev_rx_fill_empty_npkts; | |
120 | unsigned long prev_tx_empty_npkts; | |
248c7f9c | 121 | u32 outstanding_tx; |
b4b8faa1 MK |
122 | }; |
123 | ||
b4b8faa1 | 124 | static int num_socks; |
248c7f9c | 125 | struct xsk_socket_info *xsks[MAX_SOCKS]; |
b4b8faa1 MK |
126 | |
127 | static unsigned long get_nsecs(void) | |
128 | { | |
129 | struct timespec ts; | |
130 | ||
131 | clock_gettime(CLOCK_MONOTONIC, &ts); | |
132 | return ts.tv_sec * 1000000000UL + ts.tv_nsec; | |
133 | } | |
134 | ||
248c7f9c | 135 | static void print_benchmark(bool running) |
b4b8faa1 | 136 | { |
248c7f9c | 137 | const char *bench_str = "INVALID"; |
b4b8faa1 | 138 | |
248c7f9c MK |
139 | if (opt_bench == BENCH_RXDROP) |
140 | bench_str = "rxdrop"; | |
141 | else if (opt_bench == BENCH_TXONLY) | |
142 | bench_str = "txonly"; | |
143 | else if (opt_bench == BENCH_L2FWD) | |
144 | bench_str = "l2fwd"; | |
b4b8faa1 | 145 | |
248c7f9c MK |
146 | printf("%s:%d %s ", opt_if, opt_queue, bench_str); |
147 | if (opt_xdp_flags & XDP_FLAGS_SKB_MODE) | |
148 | printf("xdp-skb "); | |
149 | else if (opt_xdp_flags & XDP_FLAGS_DRV_MODE) | |
150 | printf("xdp-drv "); | |
151 | else | |
152 | printf(" "); | |
b4b8faa1 | 153 | |
248c7f9c MK |
154 | if (opt_poll) |
155 | printf("poll() "); | |
b4b8faa1 | 156 | |
248c7f9c MK |
157 | if (running) { |
158 | printf("running..."); | |
159 | fflush(stdout); | |
b4b8faa1 | 160 | } |
b4b8faa1 MK |
161 | } |
162 | ||
b36c3206 CL |
163 | static int xsk_get_xdp_stats(int fd, struct xsk_socket_info *xsk) |
164 | { | |
165 | struct xdp_statistics stats; | |
166 | socklen_t optlen; | |
167 | int err; | |
168 | ||
169 | optlen = sizeof(stats); | |
170 | err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen); | |
171 | if (err) | |
172 | return err; | |
173 | ||
174 | if (optlen == sizeof(struct xdp_statistics)) { | |
175 | xsk->rx_dropped_npkts = stats.rx_dropped; | |
176 | xsk->rx_invalid_npkts = stats.rx_invalid_descs; | |
177 | xsk->tx_invalid_npkts = stats.tx_invalid_descs; | |
178 | xsk->rx_full_npkts = stats.rx_ring_full; | |
179 | xsk->rx_fill_empty_npkts = stats.rx_fill_ring_empty_descs; | |
180 | xsk->tx_empty_npkts = stats.tx_ring_empty_descs; | |
181 | return 0; | |
182 | } | |
183 | ||
184 | return -EINVAL; | |
185 | } | |
186 | ||
248c7f9c | 187 | static void dump_stats(void) |
b4b8faa1 | 188 | { |
248c7f9c MK |
189 | unsigned long now = get_nsecs(); |
190 | long dt = now - prev_time; | |
191 | int i; | |
b4b8faa1 | 192 | |
248c7f9c | 193 | prev_time = now; |
b4b8faa1 | 194 | |
248c7f9c MK |
195 | for (i = 0; i < num_socks && xsks[i]; i++) { |
196 | char *fmt = "%-15s %'-11.0f %'-11lu\n"; | |
b36c3206 CL |
197 | double rx_pps, tx_pps, dropped_pps, rx_invalid_pps, full_pps, fill_empty_pps, |
198 | tx_invalid_pps, tx_empty_pps; | |
b4b8faa1 | 199 | |
248c7f9c MK |
200 | rx_pps = (xsks[i]->rx_npkts - xsks[i]->prev_rx_npkts) * |
201 | 1000000000. / dt; | |
202 | tx_pps = (xsks[i]->tx_npkts - xsks[i]->prev_tx_npkts) * | |
203 | 1000000000. / dt; | |
b4b8faa1 | 204 | |
248c7f9c MK |
205 | printf("\n sock%d@", i); |
206 | print_benchmark(false); | |
207 | printf("\n"); | |
b4b8faa1 | 208 | |
248c7f9c MK |
209 | printf("%-15s %-11s %-11s %-11.2f\n", "", "pps", "pkts", |
210 | dt / 1000000000.); | |
211 | printf(fmt, "rx", rx_pps, xsks[i]->rx_npkts); | |
212 | printf(fmt, "tx", tx_pps, xsks[i]->tx_npkts); | |
b4b8faa1 | 213 | |
248c7f9c MK |
214 | xsks[i]->prev_rx_npkts = xsks[i]->rx_npkts; |
215 | xsks[i]->prev_tx_npkts = xsks[i]->tx_npkts; | |
b36c3206 CL |
216 | |
217 | if (opt_extra_stats) { | |
218 | if (!xsk_get_xdp_stats(xsk_socket__fd(xsks[i]->xsk), xsks[i])) { | |
219 | dropped_pps = (xsks[i]->rx_dropped_npkts - | |
220 | xsks[i]->prev_rx_dropped_npkts) * 1000000000. / dt; | |
221 | rx_invalid_pps = (xsks[i]->rx_invalid_npkts - | |
222 | xsks[i]->prev_rx_invalid_npkts) * 1000000000. / dt; | |
223 | tx_invalid_pps = (xsks[i]->tx_invalid_npkts - | |
224 | xsks[i]->prev_tx_invalid_npkts) * 1000000000. / dt; | |
225 | full_pps = (xsks[i]->rx_full_npkts - | |
226 | xsks[i]->prev_rx_full_npkts) * 1000000000. / dt; | |
227 | fill_empty_pps = (xsks[i]->rx_fill_empty_npkts - | |
228 | xsks[i]->prev_rx_fill_empty_npkts) | |
229 | * 1000000000. / dt; | |
230 | tx_empty_pps = (xsks[i]->tx_empty_npkts - | |
231 | xsks[i]->prev_tx_empty_npkts) * 1000000000. / dt; | |
232 | ||
233 | printf(fmt, "rx dropped", dropped_pps, | |
234 | xsks[i]->rx_dropped_npkts); | |
235 | printf(fmt, "rx invalid", rx_invalid_pps, | |
236 | xsks[i]->rx_invalid_npkts); | |
237 | printf(fmt, "tx invalid", tx_invalid_pps, | |
238 | xsks[i]->tx_invalid_npkts); | |
239 | printf(fmt, "rx queue full", full_pps, | |
240 | xsks[i]->rx_full_npkts); | |
241 | printf(fmt, "fill ring empty", fill_empty_pps, | |
242 | xsks[i]->rx_fill_empty_npkts); | |
243 | printf(fmt, "tx ring empty", tx_empty_pps, | |
244 | xsks[i]->tx_empty_npkts); | |
245 | ||
246 | xsks[i]->prev_rx_dropped_npkts = xsks[i]->rx_dropped_npkts; | |
247 | xsks[i]->prev_rx_invalid_npkts = xsks[i]->rx_invalid_npkts; | |
248 | xsks[i]->prev_tx_invalid_npkts = xsks[i]->tx_invalid_npkts; | |
249 | xsks[i]->prev_rx_full_npkts = xsks[i]->rx_full_npkts; | |
250 | xsks[i]->prev_rx_fill_empty_npkts = xsks[i]->rx_fill_empty_npkts; | |
251 | xsks[i]->prev_tx_empty_npkts = xsks[i]->tx_empty_npkts; | |
252 | } else { | |
253 | printf("%-15s\n", "Error retrieving extra stats"); | |
254 | } | |
255 | } | |
b4b8faa1 | 256 | } |
b4b8faa1 MK |
257 | } |
258 | ||
d3f11b01 JJ |
259 | static bool is_benchmark_done(void) |
260 | { | |
261 | if (opt_duration > 0) { | |
262 | unsigned long dt = (get_nsecs() - start_time); | |
263 | ||
264 | if (dt >= opt_duration) | |
265 | benchmark_done = true; | |
266 | } | |
267 | return benchmark_done; | |
268 | } | |
269 | ||
248c7f9c | 270 | static void *poller(void *arg) |
b4b8faa1 | 271 | { |
248c7f9c | 272 | (void)arg; |
d3f11b01 | 273 | while (!is_benchmark_done()) { |
248c7f9c MK |
274 | sleep(opt_interval); |
275 | dump_stats(); | |
b4b8faa1 MK |
276 | } |
277 | ||
248c7f9c | 278 | return NULL; |
b4b8faa1 MK |
279 | } |
280 | ||
248c7f9c | 281 | static void remove_xdp_program(void) |
b4b8faa1 | 282 | { |
2e5d72c1 | 283 | u32 curr_prog_id = 0; |
b4b8faa1 | 284 | |
248c7f9c MK |
285 | if (bpf_get_link_xdp_id(opt_ifindex, &curr_prog_id, opt_xdp_flags)) { |
286 | printf("bpf_get_link_xdp_id failed\n"); | |
287 | exit(EXIT_FAILURE); | |
b4b8faa1 | 288 | } |
248c7f9c MK |
289 | if (prog_id == curr_prog_id) |
290 | bpf_set_link_xdp_fd(opt_ifindex, -1, opt_xdp_flags); | |
291 | else if (!curr_prog_id) | |
292 | printf("couldn't find a prog id on a given interface\n"); | |
293 | else | |
294 | printf("program on interface changed, not removing\n"); | |
b4b8faa1 MK |
295 | } |
296 | ||
248c7f9c | 297 | static void int_exit(int sig) |
69525588 JJ |
298 | { |
299 | benchmark_done = true; | |
300 | } | |
301 | ||
302 | static void xdpsock_cleanup(void) | |
b4b8faa1 | 303 | { |
248c7f9c | 304 | struct xsk_umem *umem = xsks[0]->umem->umem; |
2e5d72c1 | 305 | int i; |
b4b8faa1 | 306 | |
248c7f9c | 307 | dump_stats(); |
2e5d72c1 MK |
308 | for (i = 0; i < num_socks; i++) |
309 | xsk_socket__delete(xsks[i]->xsk); | |
248c7f9c MK |
310 | (void)xsk_umem__delete(umem); |
311 | remove_xdp_program(); | |
b4b8faa1 MK |
312 | } |
313 | ||
248c7f9c MK |
314 | static void __exit_with_error(int error, const char *file, const char *func, |
315 | int line) | |
b4b8faa1 | 316 | { |
248c7f9c MK |
317 | fprintf(stderr, "%s:%s:%i: errno: %d/\"%s\"\n", file, func, |
318 | line, error, strerror(error)); | |
319 | dump_stats(); | |
320 | remove_xdp_program(); | |
321 | exit(EXIT_FAILURE); | |
b4b8faa1 MK |
322 | } |
323 | ||
248c7f9c MK |
324 | #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, \ |
325 | __LINE__) | |
b4b8faa1 MK |
326 | static void swap_mac_addresses(void *data) |
327 | { | |
328 | struct ether_header *eth = (struct ether_header *)data; | |
329 | struct ether_addr *src_addr = (struct ether_addr *)ð->ether_shost; | |
330 | struct ether_addr *dst_addr = (struct ether_addr *)ð->ether_dhost; | |
331 | struct ether_addr tmp; | |
332 | ||
333 | tmp = *src_addr; | |
334 | *src_addr = *dst_addr; | |
335 | *dst_addr = tmp; | |
336 | } | |
337 | ||
a412ef54 | 338 | static void hex_dump(void *pkt, size_t length, u64 addr) |
b4b8faa1 | 339 | { |
b4b8faa1 MK |
340 | const unsigned char *address = (unsigned char *)pkt; |
341 | const unsigned char *line = address; | |
342 | size_t line_size = 32; | |
343 | unsigned char c; | |
a412ef54 BT |
344 | char buf[32]; |
345 | int i = 0; | |
b4b8faa1 | 346 | |
a412ef54 BT |
347 | if (!DEBUG_HEXDUMP) |
348 | return; | |
349 | ||
350 | sprintf(buf, "addr=%llu", addr); | |
b4b8faa1 | 351 | printf("length = %zu\n", length); |
a412ef54 | 352 | printf("%s | ", buf); |
b4b8faa1 MK |
353 | while (length-- > 0) { |
354 | printf("%02X ", *address++); | |
355 | if (!(++i % line_size) || (length == 0 && i % line_size)) { | |
356 | if (length == 0) { | |
357 | while (i++ % line_size) | |
358 | printf("__ "); | |
359 | } | |
360 | printf(" | "); /* right close */ | |
361 | while (line < address) { | |
362 | c = *line++; | |
363 | printf("%c", (c < 33 || c == 255) ? 0x2E : c); | |
364 | } | |
365 | printf("\n"); | |
366 | if (length > 0) | |
a412ef54 | 367 | printf("%s | ", buf); |
b4b8faa1 MK |
368 | } |
369 | } | |
370 | printf("\n"); | |
371 | } | |
b4b8faa1 | 372 | |
4a3c23ae JJ |
373 | static void *memset32_htonl(void *dest, u32 val, u32 size) |
374 | { | |
375 | u32 *ptr = (u32 *)dest; | |
376 | int i; | |
377 | ||
378 | val = htonl(val); | |
379 | ||
380 | for (i = 0; i < (size & (~0x3)); i += 4) | |
381 | ptr[i >> 2] = val; | |
382 | ||
383 | for (; i < size; i++) | |
384 | ((char *)dest)[i] = ((char *)&val)[i & 3]; | |
385 | ||
386 | return dest; | |
387 | } | |
388 | ||
389 | /* | |
390 | * This function code has been taken from | |
391 | * Linux kernel lib/checksum.c | |
392 | */ | |
393 | static inline unsigned short from32to16(unsigned int x) | |
394 | { | |
395 | /* add up 16-bit and 16-bit for 16+c bit */ | |
396 | x = (x & 0xffff) + (x >> 16); | |
397 | /* add up carry.. */ | |
398 | x = (x & 0xffff) + (x >> 16); | |
399 | return x; | |
400 | } | |
401 | ||
402 | /* | |
403 | * This function code has been taken from | |
404 | * Linux kernel lib/checksum.c | |
405 | */ | |
406 | static unsigned int do_csum(const unsigned char *buff, int len) | |
407 | { | |
408 | unsigned int result = 0; | |
409 | int odd; | |
410 | ||
411 | if (len <= 0) | |
412 | goto out; | |
413 | odd = 1 & (unsigned long)buff; | |
414 | if (odd) { | |
415 | #ifdef __LITTLE_ENDIAN | |
416 | result += (*buff << 8); | |
417 | #else | |
418 | result = *buff; | |
419 | #endif | |
420 | len--; | |
421 | buff++; | |
422 | } | |
423 | if (len >= 2) { | |
424 | if (2 & (unsigned long)buff) { | |
425 | result += *(unsigned short *)buff; | |
426 | len -= 2; | |
427 | buff += 2; | |
428 | } | |
429 | if (len >= 4) { | |
430 | const unsigned char *end = buff + | |
431 | ((unsigned int)len & ~3); | |
432 | unsigned int carry = 0; | |
433 | ||
434 | do { | |
435 | unsigned int w = *(unsigned int *)buff; | |
436 | ||
437 | buff += 4; | |
438 | result += carry; | |
439 | result += w; | |
440 | carry = (w > result); | |
441 | } while (buff < end); | |
442 | result += carry; | |
443 | result = (result & 0xffff) + (result >> 16); | |
444 | } | |
445 | if (len & 2) { | |
446 | result += *(unsigned short *)buff; | |
447 | buff += 2; | |
448 | } | |
449 | } | |
450 | if (len & 1) | |
451 | #ifdef __LITTLE_ENDIAN | |
452 | result += *buff; | |
453 | #else | |
454 | result += (*buff << 8); | |
455 | #endif | |
456 | result = from32to16(result); | |
457 | if (odd) | |
458 | result = ((result >> 8) & 0xff) | ((result & 0xff) << 8); | |
459 | out: | |
460 | return result; | |
461 | } | |
462 | ||
463 | __sum16 ip_fast_csum(const void *iph, unsigned int ihl); | |
464 | ||
465 | /* | |
466 | * This is a version of ip_compute_csum() optimized for IP headers, | |
467 | * which always checksum on 4 octet boundaries. | |
468 | * This function code has been taken from | |
469 | * Linux kernel lib/checksum.c | |
470 | */ | |
471 | __sum16 ip_fast_csum(const void *iph, unsigned int ihl) | |
472 | { | |
473 | return (__force __sum16)~do_csum(iph, ihl * 4); | |
474 | } | |
475 | ||
476 | /* | |
477 | * Fold a partial checksum | |
478 | * This function code has been taken from | |
479 | * Linux kernel include/asm-generic/checksum.h | |
480 | */ | |
481 | static inline __sum16 csum_fold(__wsum csum) | |
482 | { | |
483 | u32 sum = (__force u32)csum; | |
484 | ||
485 | sum = (sum & 0xffff) + (sum >> 16); | |
486 | sum = (sum & 0xffff) + (sum >> 16); | |
487 | return (__force __sum16)~sum; | |
488 | } | |
489 | ||
490 | /* | |
491 | * This function code has been taken from | |
492 | * Linux kernel lib/checksum.c | |
493 | */ | |
494 | static inline u32 from64to32(u64 x) | |
495 | { | |
496 | /* add up 32-bit and 32-bit for 32+c bit */ | |
497 | x = (x & 0xffffffff) + (x >> 32); | |
498 | /* add up carry.. */ | |
499 | x = (x & 0xffffffff) + (x >> 32); | |
500 | return (u32)x; | |
501 | } | |
502 | ||
503 | __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, | |
504 | __u32 len, __u8 proto, __wsum sum); | |
505 | ||
506 | /* | |
507 | * This function code has been taken from | |
508 | * Linux kernel lib/checksum.c | |
509 | */ | |
510 | __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, | |
511 | __u32 len, __u8 proto, __wsum sum) | |
512 | { | |
513 | unsigned long long s = (__force u32)sum; | |
514 | ||
515 | s += (__force u32)saddr; | |
516 | s += (__force u32)daddr; | |
517 | #ifdef __BIG_ENDIAN__ | |
518 | s += proto + len; | |
519 | #else | |
520 | s += (proto + len) << 8; | |
521 | #endif | |
522 | return (__force __wsum)from64to32(s); | |
523 | } | |
524 | ||
525 | /* | |
526 | * This function has been taken from | |
527 | * Linux kernel include/asm-generic/checksum.h | |
528 | */ | |
529 | static inline __sum16 | |
530 | csum_tcpudp_magic(__be32 saddr, __be32 daddr, __u32 len, | |
531 | __u8 proto, __wsum sum) | |
532 | { | |
533 | return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum)); | |
534 | } | |
535 | ||
536 | static inline u16 udp_csum(u32 saddr, u32 daddr, u32 len, | |
537 | u8 proto, u16 *udp_pkt) | |
538 | { | |
539 | u32 csum = 0; | |
540 | u32 cnt = 0; | |
541 | ||
542 | /* udp hdr and data */ | |
543 | for (; cnt < len; cnt += 2) | |
544 | csum += udp_pkt[cnt >> 1]; | |
545 | ||
546 | return csum_tcpudp_magic(saddr, daddr, len, proto, csum); | |
547 | } | |
548 | ||
549 | #define ETH_FCS_SIZE 4 | |
550 | ||
551 | #define PKT_HDR_SIZE (sizeof(struct ethhdr) + sizeof(struct iphdr) + \ | |
552 | sizeof(struct udphdr)) | |
553 | ||
554 | #define PKT_SIZE (opt_pkt_size - ETH_FCS_SIZE) | |
555 | #define IP_PKT_SIZE (PKT_SIZE - sizeof(struct ethhdr)) | |
556 | #define UDP_PKT_SIZE (IP_PKT_SIZE - sizeof(struct iphdr)) | |
557 | #define UDP_PKT_DATA_SIZE (UDP_PKT_SIZE - sizeof(struct udphdr)) | |
558 | ||
559 | static u8 pkt_data[XSK_UMEM__DEFAULT_FRAME_SIZE]; | |
560 | ||
561 | static void gen_eth_hdr_data(void) | |
562 | { | |
563 | struct udphdr *udp_hdr = (struct udphdr *)(pkt_data + | |
564 | sizeof(struct ethhdr) + | |
565 | sizeof(struct iphdr)); | |
566 | struct iphdr *ip_hdr = (struct iphdr *)(pkt_data + | |
567 | sizeof(struct ethhdr)); | |
568 | struct ethhdr *eth_hdr = (struct ethhdr *)pkt_data; | |
569 | ||
570 | /* ethernet header */ | |
571 | memcpy(eth_hdr->h_dest, "\x3c\xfd\xfe\x9e\x7f\x71", ETH_ALEN); | |
572 | memcpy(eth_hdr->h_source, "\xec\xb1\xd7\x98\x3a\xc0", ETH_ALEN); | |
573 | eth_hdr->h_proto = htons(ETH_P_IP); | |
574 | ||
575 | /* IP header */ | |
576 | ip_hdr->version = IPVERSION; | |
577 | ip_hdr->ihl = 0x5; /* 20 byte header */ | |
578 | ip_hdr->tos = 0x0; | |
579 | ip_hdr->tot_len = htons(IP_PKT_SIZE); | |
580 | ip_hdr->id = 0; | |
581 | ip_hdr->frag_off = 0; | |
582 | ip_hdr->ttl = IPDEFTTL; | |
583 | ip_hdr->protocol = IPPROTO_UDP; | |
584 | ip_hdr->saddr = htonl(0x0a0a0a10); | |
585 | ip_hdr->daddr = htonl(0x0a0a0a20); | |
586 | ||
587 | /* IP header checksum */ | |
588 | ip_hdr->check = 0; | |
589 | ip_hdr->check = ip_fast_csum((const void *)ip_hdr, ip_hdr->ihl); | |
590 | ||
591 | /* UDP header */ | |
592 | udp_hdr->source = htons(0x1000); | |
593 | udp_hdr->dest = htons(0x1000); | |
594 | udp_hdr->len = htons(UDP_PKT_SIZE); | |
595 | ||
596 | /* UDP data */ | |
46e3268e | 597 | memset32_htonl(pkt_data + PKT_HDR_SIZE, opt_pkt_fill_pattern, |
4a3c23ae JJ |
598 | UDP_PKT_DATA_SIZE); |
599 | ||
600 | /* UDP header checksum */ | |
601 | udp_hdr->check = 0; | |
602 | udp_hdr->check = udp_csum(ip_hdr->saddr, ip_hdr->daddr, UDP_PKT_SIZE, | |
603 | IPPROTO_UDP, (u16 *)udp_hdr); | |
604 | } | |
605 | ||
cd9e72b6 | 606 | static void gen_eth_frame(struct xsk_umem_info *umem, u64 addr) |
b4b8faa1 | 607 | { |
248c7f9c | 608 | memcpy(xsk_umem__get_data(umem->buffer, addr), pkt_data, |
4a3c23ae | 609 | PKT_SIZE); |
b4b8faa1 MK |
610 | } |
611 | ||
248c7f9c | 612 | static struct xsk_umem_info *xsk_configure_umem(void *buffer, u64 size) |
b4b8faa1 | 613 | { |
248c7f9c | 614 | struct xsk_umem_info *umem; |
123e8da1 | 615 | struct xsk_umem_config cfg = { |
c8a039a4 MK |
616 | /* We recommend that you set the fill ring size >= HW RX ring size + |
617 | * AF_XDP RX ring size. Make sure you fill up the fill ring | |
618 | * with buffers at regular intervals, and you will with this setting | |
619 | * avoid allocation failures in the driver. These are usually quite | |
620 | * expensive since drivers have not been written to assume that | |
621 | * allocation failures are common. For regular sockets, kernel | |
622 | * allocated memory is used that only runs out in OOM situations | |
623 | * that should be rare. | |
624 | */ | |
625 | .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS * 2, | |
123e8da1 MM |
626 | .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, |
627 | .frame_size = opt_xsk_frame_size, | |
628 | .frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM, | |
c543f546 | 629 | .flags = opt_umem_flags |
123e8da1 | 630 | }; |
661842c4 | 631 | int ret; |
b4b8faa1 MK |
632 | |
633 | umem = calloc(1, sizeof(*umem)); | |
248c7f9c MK |
634 | if (!umem) |
635 | exit_with_error(errno); | |
b4b8faa1 | 636 | |
248c7f9c | 637 | ret = xsk_umem__create(&umem->umem, buffer, size, &umem->fq, &umem->cq, |
123e8da1 | 638 | &cfg); |
248c7f9c MK |
639 | if (ret) |
640 | exit_with_error(-ret); | |
b4b8faa1 | 641 | |
661842c4 MK |
642 | umem->buffer = buffer; |
643 | return umem; | |
644 | } | |
645 | ||
646 | static void xsk_populate_fill_ring(struct xsk_umem_info *umem) | |
647 | { | |
648 | int ret, i; | |
649 | u32 idx; | |
650 | ||
2e5d72c1 | 651 | ret = xsk_ring_prod__reserve(&umem->fq, |
c8a039a4 MK |
652 | XSK_RING_PROD__DEFAULT_NUM_DESCS * 2, &idx); |
653 | if (ret != XSK_RING_PROD__DEFAULT_NUM_DESCS * 2) | |
2e5d72c1 | 654 | exit_with_error(-ret); |
c8a039a4 | 655 | for (i = 0; i < XSK_RING_PROD__DEFAULT_NUM_DESCS * 2; i++) |
2e5d72c1 MK |
656 | *xsk_ring_prod__fill_addr(&umem->fq, idx++) = |
657 | i * opt_xsk_frame_size; | |
c8a039a4 | 658 | xsk_ring_prod__submit(&umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS * 2); |
b4b8faa1 MK |
659 | } |
660 | ||
661842c4 MK |
661 | static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem, |
662 | bool rx, bool tx) | |
b4b8faa1 | 663 | { |
248c7f9c MK |
664 | struct xsk_socket_config cfg; |
665 | struct xsk_socket_info *xsk; | |
661842c4 MK |
666 | struct xsk_ring_cons *rxr; |
667 | struct xsk_ring_prod *txr; | |
248c7f9c | 668 | int ret; |
b4b8faa1 MK |
669 | |
670 | xsk = calloc(1, sizeof(*xsk)); | |
248c7f9c MK |
671 | if (!xsk) |
672 | exit_with_error(errno); | |
673 | ||
674 | xsk->umem = umem; | |
675 | cfg.rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS; | |
676 | cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; | |
2e5d72c1 MK |
677 | if (opt_num_xsks > 1) |
678 | cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD; | |
679 | else | |
680 | cfg.libbpf_flags = 0; | |
248c7f9c MK |
681 | cfg.xdp_flags = opt_xdp_flags; |
682 | cfg.bind_flags = opt_xdp_bind_flags; | |
2e5d72c1 | 683 | |
661842c4 MK |
684 | rxr = rx ? &xsk->rx : NULL; |
685 | txr = tx ? &xsk->tx : NULL; | |
686 | ret = xsk_socket__create(&xsk->xsk, opt_if, opt_queue, umem->umem, | |
687 | rxr, txr, &cfg); | |
248c7f9c MK |
688 | if (ret) |
689 | exit_with_error(-ret); | |
690 | ||
691 | ret = bpf_get_link_xdp_id(opt_ifindex, &prog_id, opt_xdp_flags); | |
692 | if (ret) | |
693 | exit_with_error(-ret); | |
694 | ||
b4b8faa1 MK |
695 | return xsk; |
696 | } | |
697 | ||
b4b8faa1 MK |
698 | static struct option long_options[] = { |
699 | {"rxdrop", no_argument, 0, 'r'}, | |
700 | {"txonly", no_argument, 0, 't'}, | |
701 | {"l2fwd", no_argument, 0, 'l'}, | |
702 | {"interface", required_argument, 0, 'i'}, | |
703 | {"queue", required_argument, 0, 'q'}, | |
704 | {"poll", no_argument, 0, 'p'}, | |
b4b8faa1 MK |
705 | {"xdp-skb", no_argument, 0, 'S'}, |
706 | {"xdp-native", no_argument, 0, 'N'}, | |
707 | {"interval", required_argument, 0, 'n'}, | |
58c50ae4 BT |
708 | {"zero-copy", no_argument, 0, 'z'}, |
709 | {"copy", no_argument, 0, 'c'}, | |
123e8da1 | 710 | {"frame-size", required_argument, 0, 'f'}, |
46738f73 | 711 | {"no-need-wakeup", no_argument, 0, 'm'}, |
c543f546 | 712 | {"unaligned", no_argument, 0, 'u'}, |
2e5d72c1 | 713 | {"shared-umem", no_argument, 0, 'M'}, |
b3133329 | 714 | {"force", no_argument, 0, 'F'}, |
d3f11b01 | 715 | {"duration", required_argument, 0, 'd'}, |
cd9e72b6 | 716 | {"batch-size", required_argument, 0, 'b'}, |
ece6e969 | 717 | {"tx-pkt-count", required_argument, 0, 'C'}, |
4a3c23ae | 718 | {"tx-pkt-size", required_argument, 0, 's'}, |
46e3268e | 719 | {"tx-pkt-pattern", required_argument, 0, 'P'}, |
b36c3206 | 720 | {"extra-stats", no_argument, 0, 'x'}, |
b4b8faa1 MK |
721 | {0, 0, 0, 0} |
722 | }; | |
723 | ||
724 | static void usage(const char *prog) | |
725 | { | |
726 | const char *str = | |
727 | " Usage: %s [OPTIONS]\n" | |
728 | " Options:\n" | |
729 | " -r, --rxdrop Discard all incoming packets (default)\n" | |
730 | " -t, --txonly Only send packets\n" | |
731 | " -l, --l2fwd MAC swap L2 forwarding\n" | |
732 | " -i, --interface=n Run on interface n\n" | |
733 | " -q, --queue=n Use queue n (default 0)\n" | |
734 | " -p, --poll Use poll syscall\n" | |
b4b8faa1 | 735 | " -S, --xdp-skb=n Use XDP skb-mod\n" |
4564a8bb | 736 | " -N, --xdp-native=n Enforce XDP native mode\n" |
b4b8faa1 | 737 | " -n, --interval=n Specify statistics update interval (default 1 sec).\n" |
58c50ae4 BT |
738 | " -z, --zero-copy Force zero-copy mode.\n" |
739 | " -c, --copy Force copy mode.\n" | |
46738f73 | 740 | " -m, --no-need-wakeup Turn off use of driver need wakeup flag.\n" |
c543f546 KL |
741 | " -f, --frame-size=n Set the frame size (must be a power of two in aligned mode, default is %d).\n" |
742 | " -u, --unaligned Enable unaligned chunk placement\n" | |
2e5d72c1 | 743 | " -M, --shared-umem Enable XDP_SHARED_UMEM\n" |
b3133329 | 744 | " -F, --force Force loading the XDP prog\n" |
d3f11b01 JJ |
745 | " -d, --duration=n Duration in secs to run command.\n" |
746 | " Default: forever.\n" | |
cd9e72b6 JJ |
747 | " -b, --batch-size=n Batch size for sending or receiving\n" |
748 | " packets. Default: %d\n" | |
ece6e969 JJ |
749 | " -C, --tx-pkt-count=n Number of packets to send.\n" |
750 | " Default: Continuous packets.\n" | |
4a3c23ae JJ |
751 | " -s, --tx-pkt-size=n Transmit packet size.\n" |
752 | " (Default: %d bytes)\n" | |
753 | " Min size: %d, Max size %d.\n" | |
46e3268e | 754 | " -P, --tx-pkt-pattern=nPacket fill pattern. Default: 0x%x\n" |
b36c3206 | 755 | " -x, --extra-stats Display extra statistics.\n" |
b4b8faa1 | 756 | "\n"; |
cd9e72b6 | 757 | fprintf(stderr, str, prog, XSK_UMEM__DEFAULT_FRAME_SIZE, |
4a3c23ae | 758 | opt_batch_size, MIN_PKT_SIZE, MIN_PKT_SIZE, |
46e3268e | 759 | XSK_UMEM__DEFAULT_FRAME_SIZE, opt_pkt_fill_pattern); |
4a3c23ae | 760 | |
b4b8faa1 MK |
761 | exit(EXIT_FAILURE); |
762 | } | |
763 | ||
764 | static void parse_command_line(int argc, char **argv) | |
765 | { | |
766 | int option_index, c; | |
767 | ||
768 | opterr = 0; | |
769 | ||
770 | for (;;) { | |
b36c3206 | 771 | c = getopt_long(argc, argv, "Frtli:q:pSNn:czf:muMd:b:C:s:P:x", |
46738f73 | 772 | long_options, &option_index); |
b4b8faa1 MK |
773 | if (c == -1) |
774 | break; | |
775 | ||
776 | switch (c) { | |
777 | case 'r': | |
778 | opt_bench = BENCH_RXDROP; | |
779 | break; | |
780 | case 't': | |
781 | opt_bench = BENCH_TXONLY; | |
782 | break; | |
783 | case 'l': | |
784 | opt_bench = BENCH_L2FWD; | |
785 | break; | |
786 | case 'i': | |
787 | opt_if = optarg; | |
788 | break; | |
789 | case 'q': | |
790 | opt_queue = atoi(optarg); | |
791 | break; | |
b4b8faa1 MK |
792 | case 'p': |
793 | opt_poll = 1; | |
794 | break; | |
795 | case 'S': | |
796 | opt_xdp_flags |= XDP_FLAGS_SKB_MODE; | |
9f5232cc | 797 | opt_xdp_bind_flags |= XDP_COPY; |
b4b8faa1 MK |
798 | break; |
799 | case 'N': | |
d50ecc46 | 800 | /* default, set below */ |
b4b8faa1 MK |
801 | break; |
802 | case 'n': | |
803 | opt_interval = atoi(optarg); | |
804 | break; | |
58c50ae4 BT |
805 | case 'z': |
806 | opt_xdp_bind_flags |= XDP_ZEROCOPY; | |
807 | break; | |
808 | case 'c': | |
809 | opt_xdp_bind_flags |= XDP_COPY; | |
810 | break; | |
c543f546 KL |
811 | case 'u': |
812 | opt_umem_flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG; | |
813 | opt_unaligned_chunks = 1; | |
3945b37a | 814 | opt_mmap_flags = MAP_HUGETLB; |
c543f546 | 815 | break; |
743e568c MF |
816 | case 'F': |
817 | opt_xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST; | |
818 | break; | |
123e8da1 MM |
819 | case 'f': |
820 | opt_xsk_frame_size = atoi(optarg); | |
2e5d72c1 | 821 | break; |
46738f73 MK |
822 | case 'm': |
823 | opt_need_wakeup = false; | |
824 | opt_xdp_bind_flags &= ~XDP_USE_NEED_WAKEUP; | |
123e8da1 | 825 | break; |
2e5d72c1 MK |
826 | case 'M': |
827 | opt_num_xsks = MAX_SOCKS; | |
828 | break; | |
d3f11b01 JJ |
829 | case 'd': |
830 | opt_duration = atoi(optarg); | |
831 | opt_duration *= 1000000000; | |
832 | break; | |
cd9e72b6 JJ |
833 | case 'b': |
834 | opt_batch_size = atoi(optarg); | |
835 | break; | |
ece6e969 JJ |
836 | case 'C': |
837 | opt_pkt_count = atoi(optarg); | |
838 | break; | |
4a3c23ae JJ |
839 | case 's': |
840 | opt_pkt_size = atoi(optarg); | |
841 | if (opt_pkt_size > (XSK_UMEM__DEFAULT_FRAME_SIZE) || | |
842 | opt_pkt_size < MIN_PKT_SIZE) { | |
843 | fprintf(stderr, | |
844 | "ERROR: Invalid frame size %d\n", | |
845 | opt_pkt_size); | |
846 | usage(basename(argv[0])); | |
847 | } | |
848 | break; | |
46e3268e JJ |
849 | case 'P': |
850 | opt_pkt_fill_pattern = strtol(optarg, NULL, 16); | |
851 | break; | |
b36c3206 CL |
852 | case 'x': |
853 | opt_extra_stats = 1; | |
854 | break; | |
b4b8faa1 MK |
855 | default: |
856 | usage(basename(argv[0])); | |
857 | } | |
858 | } | |
859 | ||
d50ecc46 THJ |
860 | if (!(opt_xdp_flags & XDP_FLAGS_SKB_MODE)) |
861 | opt_xdp_flags |= XDP_FLAGS_DRV_MODE; | |
862 | ||
b4b8faa1 MK |
863 | opt_ifindex = if_nametoindex(opt_if); |
864 | if (!opt_ifindex) { | |
865 | fprintf(stderr, "ERROR: interface \"%s\" does not exist\n", | |
866 | opt_if); | |
867 | usage(basename(argv[0])); | |
868 | } | |
248c7f9c | 869 | |
c543f546 KL |
870 | if ((opt_xsk_frame_size & (opt_xsk_frame_size - 1)) && |
871 | !opt_unaligned_chunks) { | |
123e8da1 MM |
872 | fprintf(stderr, "--frame-size=%d is not a power of two\n", |
873 | opt_xsk_frame_size); | |
874 | usage(basename(argv[0])); | |
875 | } | |
b4b8faa1 MK |
876 | } |
877 | ||
248c7f9c | 878 | static void kick_tx(struct xsk_socket_info *xsk) |
b4b8faa1 MK |
879 | { |
880 | int ret; | |
881 | ||
248c7f9c | 882 | ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); |
8ed47e14 MF |
883 | if (ret >= 0 || errno == ENOBUFS || errno == EAGAIN || |
884 | errno == EBUSY || errno == ENETDOWN) | |
b4b8faa1 | 885 | return; |
248c7f9c | 886 | exit_with_error(errno); |
b4b8faa1 MK |
887 | } |
888 | ||
46738f73 MK |
889 | static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk, |
890 | struct pollfd *fds) | |
b4b8faa1 | 891 | { |
03895e63 | 892 | struct xsk_umem_info *umem = xsk->umem; |
b74e21ab | 893 | u32 idx_cq = 0, idx_fq = 0; |
b4b8faa1 MK |
894 | unsigned int rcvd; |
895 | size_t ndescs; | |
896 | ||
897 | if (!xsk->outstanding_tx) | |
898 | return; | |
899 | ||
cd9e72b6 | 900 | ndescs = (xsk->outstanding_tx > opt_batch_size) ? opt_batch_size : |
248c7f9c | 901 | xsk->outstanding_tx; |
b4b8faa1 MK |
902 | |
903 | /* re-add completed Tx buffers */ | |
03895e63 | 904 | rcvd = xsk_ring_cons__peek(&umem->cq, ndescs, &idx_cq); |
b4b8faa1 | 905 | if (rcvd > 0) { |
248c7f9c MK |
906 | unsigned int i; |
907 | int ret; | |
908 | ||
03895e63 | 909 | ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); |
248c7f9c MK |
910 | while (ret != rcvd) { |
911 | if (ret < 0) | |
912 | exit_with_error(-ret); | |
03895e63 | 913 | if (xsk_ring_prod__needs_wakeup(&umem->fq)) |
46738f73 | 914 | ret = poll(fds, num_socks, opt_timeout); |
03895e63 | 915 | ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); |
248c7f9c | 916 | } |
03895e63 | 917 | |
248c7f9c | 918 | for (i = 0; i < rcvd; i++) |
03895e63 KL |
919 | *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = |
920 | *xsk_ring_cons__comp_addr(&umem->cq, idx_cq++); | |
248c7f9c MK |
921 | |
922 | xsk_ring_prod__submit(&xsk->umem->fq, rcvd); | |
923 | xsk_ring_cons__release(&xsk->umem->cq, rcvd); | |
b4b8faa1 MK |
924 | xsk->outstanding_tx -= rcvd; |
925 | xsk->tx_npkts += rcvd; | |
926 | } | |
927 | } | |
928 | ||
ece6e969 JJ |
929 | static inline void complete_tx_only(struct xsk_socket_info *xsk, |
930 | int batch_size) | |
b4b8faa1 | 931 | { |
b4b8faa1 | 932 | unsigned int rcvd; |
248c7f9c | 933 | u32 idx; |
b4b8faa1 MK |
934 | |
935 | if (!xsk->outstanding_tx) | |
936 | return; | |
937 | ||
46738f73 MK |
938 | if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx)) |
939 | kick_tx(xsk); | |
b4b8faa1 | 940 | |
ece6e969 | 941 | rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx); |
b4b8faa1 | 942 | if (rcvd > 0) { |
248c7f9c | 943 | xsk_ring_cons__release(&xsk->umem->cq, rcvd); |
b4b8faa1 MK |
944 | xsk->outstanding_tx -= rcvd; |
945 | xsk->tx_npkts += rcvd; | |
946 | } | |
947 | } | |
948 | ||
46738f73 | 949 | static void rx_drop(struct xsk_socket_info *xsk, struct pollfd *fds) |
b4b8faa1 | 950 | { |
b4b8faa1 | 951 | unsigned int rcvd, i; |
b74e21ab | 952 | u32 idx_rx = 0, idx_fq = 0; |
248c7f9c | 953 | int ret; |
b4b8faa1 | 954 | |
cd9e72b6 | 955 | rcvd = xsk_ring_cons__peek(&xsk->rx, opt_batch_size, &idx_rx); |
46738f73 MK |
956 | if (!rcvd) { |
957 | if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) | |
958 | ret = poll(fds, num_socks, opt_timeout); | |
b4b8faa1 | 959 | return; |
46738f73 | 960 | } |
b4b8faa1 | 961 | |
248c7f9c MK |
962 | ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq); |
963 | while (ret != rcvd) { | |
964 | if (ret < 0) | |
965 | exit_with_error(-ret); | |
46738f73 MK |
966 | if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) |
967 | ret = poll(fds, num_socks, opt_timeout); | |
248c7f9c MK |
968 | ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq); |
969 | } | |
970 | ||
b4b8faa1 | 971 | for (i = 0; i < rcvd; i++) { |
248c7f9c MK |
972 | u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr; |
973 | u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len; | |
03895e63 KL |
974 | u64 orig = xsk_umem__extract_addr(addr); |
975 | ||
976 | addr = xsk_umem__add_offset_to_addr(addr); | |
248c7f9c | 977 | char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr); |
b4b8faa1 | 978 | |
248c7f9c | 979 | hex_dump(pkt, len, addr); |
03895e63 | 980 | *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx_fq++) = orig; |
b4b8faa1 MK |
981 | } |
982 | ||
248c7f9c MK |
983 | xsk_ring_prod__submit(&xsk->umem->fq, rcvd); |
984 | xsk_ring_cons__release(&xsk->rx, rcvd); | |
b4b8faa1 | 985 | xsk->rx_npkts += rcvd; |
b4b8faa1 MK |
986 | } |
987 | ||
988 | static void rx_drop_all(void) | |
989 | { | |
2e5d72c1 | 990 | struct pollfd fds[MAX_SOCKS] = {}; |
46738f73 | 991 | int i, ret; |
b4b8faa1 | 992 | |
b4b8faa1 | 993 | for (i = 0; i < num_socks; i++) { |
248c7f9c | 994 | fds[i].fd = xsk_socket__fd(xsks[i]->xsk); |
b4b8faa1 | 995 | fds[i].events = POLLIN; |
b4b8faa1 MK |
996 | } |
997 | ||
998 | for (;;) { | |
999 | if (opt_poll) { | |
46738f73 | 1000 | ret = poll(fds, num_socks, opt_timeout); |
b4b8faa1 MK |
1001 | if (ret <= 0) |
1002 | continue; | |
1003 | } | |
1004 | ||
1005 | for (i = 0; i < num_socks; i++) | |
46738f73 | 1006 | rx_drop(xsks[i], fds); |
d3f11b01 JJ |
1007 | |
1008 | if (benchmark_done) | |
1009 | break; | |
46738f73 MK |
1010 | } |
1011 | } | |
1012 | ||
b69e56cf | 1013 | static void tx_only(struct xsk_socket_info *xsk, u32 *frame_nb, int batch_size) |
46738f73 MK |
1014 | { |
1015 | u32 idx; | |
cd9e72b6 | 1016 | unsigned int i; |
46738f73 | 1017 | |
ece6e969 JJ |
1018 | while (xsk_ring_prod__reserve(&xsk->tx, batch_size, &idx) < |
1019 | batch_size) { | |
1020 | complete_tx_only(xsk, batch_size); | |
cd9e72b6 | 1021 | } |
46738f73 | 1022 | |
ece6e969 | 1023 | for (i = 0; i < batch_size; i++) { |
cd9e72b6 JJ |
1024 | struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, |
1025 | idx + i); | |
b69e56cf | 1026 | tx_desc->addr = (*frame_nb + i) << XSK_UMEM__DEFAULT_FRAME_SHIFT; |
4a3c23ae | 1027 | tx_desc->len = PKT_SIZE; |
b4b8faa1 | 1028 | } |
46738f73 | 1029 | |
ece6e969 JJ |
1030 | xsk_ring_prod__submit(&xsk->tx, batch_size); |
1031 | xsk->outstanding_tx += batch_size; | |
b69e56cf WJ |
1032 | *frame_nb += batch_size; |
1033 | *frame_nb %= NUM_FRAMES; | |
ece6e969 JJ |
1034 | complete_tx_only(xsk, batch_size); |
1035 | } | |
1036 | ||
1037 | static inline int get_batch_size(int pkt_cnt) | |
1038 | { | |
1039 | if (!opt_pkt_count) | |
1040 | return opt_batch_size; | |
1041 | ||
1042 | if (pkt_cnt + opt_batch_size <= opt_pkt_count) | |
1043 | return opt_batch_size; | |
1044 | ||
1045 | return opt_pkt_count - pkt_cnt; | |
1046 | } | |
1047 | ||
1048 | static void complete_tx_only_all(void) | |
1049 | { | |
1050 | bool pending; | |
1051 | int i; | |
1052 | ||
1053 | do { | |
1054 | pending = false; | |
1055 | for (i = 0; i < num_socks; i++) { | |
1056 | if (xsks[i]->outstanding_tx) { | |
1057 | complete_tx_only(xsks[i], opt_batch_size); | |
1058 | pending = !!xsks[i]->outstanding_tx; | |
1059 | } | |
1060 | } | |
1061 | } while (pending); | |
b4b8faa1 MK |
1062 | } |
1063 | ||
46738f73 | 1064 | static void tx_only_all(void) |
b4b8faa1 | 1065 | { |
2e5d72c1 | 1066 | struct pollfd fds[MAX_SOCKS] = {}; |
46738f73 | 1067 | u32 frame_nb[MAX_SOCKS] = {}; |
ece6e969 | 1068 | int pkt_cnt = 0; |
46738f73 | 1069 | int i, ret; |
b4b8faa1 | 1070 | |
46738f73 MK |
1071 | for (i = 0; i < num_socks; i++) { |
1072 | fds[0].fd = xsk_socket__fd(xsks[i]->xsk); | |
1073 | fds[0].events = POLLOUT; | |
1074 | } | |
b4b8faa1 | 1075 | |
ece6e969 JJ |
1076 | while ((opt_pkt_count && pkt_cnt < opt_pkt_count) || !opt_pkt_count) { |
1077 | int batch_size = get_batch_size(pkt_cnt); | |
1078 | ||
b4b8faa1 | 1079 | if (opt_poll) { |
46738f73 | 1080 | ret = poll(fds, num_socks, opt_timeout); |
b4b8faa1 MK |
1081 | if (ret <= 0) |
1082 | continue; | |
1083 | ||
248c7f9c | 1084 | if (!(fds[0].revents & POLLOUT)) |
b4b8faa1 MK |
1085 | continue; |
1086 | } | |
1087 | ||
46738f73 | 1088 | for (i = 0; i < num_socks; i++) |
b69e56cf | 1089 | tx_only(xsks[i], &frame_nb[i], batch_size); |
ece6e969 JJ |
1090 | |
1091 | pkt_cnt += batch_size; | |
d3f11b01 JJ |
1092 | |
1093 | if (benchmark_done) | |
1094 | break; | |
b4b8faa1 | 1095 | } |
ece6e969 JJ |
1096 | |
1097 | if (opt_pkt_count) | |
1098 | complete_tx_only_all(); | |
b4b8faa1 MK |
1099 | } |
1100 | ||
46738f73 | 1101 | static void l2fwd(struct xsk_socket_info *xsk, struct pollfd *fds) |
b4b8faa1 | 1102 | { |
46738f73 MK |
1103 | unsigned int rcvd, i; |
1104 | u32 idx_rx = 0, idx_tx = 0; | |
1105 | int ret; | |
b4b8faa1 | 1106 | |
46738f73 | 1107 | complete_tx_l2fwd(xsk, fds); |
b4b8faa1 | 1108 | |
cd9e72b6 | 1109 | rcvd = xsk_ring_cons__peek(&xsk->rx, opt_batch_size, &idx_rx); |
46738f73 MK |
1110 | if (!rcvd) { |
1111 | if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) | |
1112 | ret = poll(fds, num_socks, opt_timeout); | |
1113 | return; | |
1114 | } | |
b4b8faa1 | 1115 | |
46738f73 MK |
1116 | ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx); |
1117 | while (ret != rcvd) { | |
1118 | if (ret < 0) | |
1119 | exit_with_error(-ret); | |
1120 | if (xsk_ring_prod__needs_wakeup(&xsk->tx)) | |
1121 | kick_tx(xsk); | |
248c7f9c | 1122 | ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx); |
46738f73 MK |
1123 | } |
1124 | ||
1125 | for (i = 0; i < rcvd; i++) { | |
1126 | u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr; | |
1127 | u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len; | |
5a712e13 | 1128 | u64 orig = addr; |
03895e63 KL |
1129 | |
1130 | addr = xsk_umem__add_offset_to_addr(addr); | |
46738f73 MK |
1131 | char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr); |
1132 | ||
1133 | swap_mac_addresses(pkt); | |
248c7f9c | 1134 | |
46738f73 | 1135 | hex_dump(pkt, len, addr); |
03895e63 | 1136 | xsk_ring_prod__tx_desc(&xsk->tx, idx_tx)->addr = orig; |
46738f73 MK |
1137 | xsk_ring_prod__tx_desc(&xsk->tx, idx_tx++)->len = len; |
1138 | } | |
b4b8faa1 | 1139 | |
46738f73 MK |
1140 | xsk_ring_prod__submit(&xsk->tx, rcvd); |
1141 | xsk_ring_cons__release(&xsk->rx, rcvd); | |
b4b8faa1 | 1142 | |
46738f73 MK |
1143 | xsk->rx_npkts += rcvd; |
1144 | xsk->outstanding_tx += rcvd; | |
1145 | } | |
1146 | ||
1147 | static void l2fwd_all(void) | |
1148 | { | |
2e5d72c1 | 1149 | struct pollfd fds[MAX_SOCKS] = {}; |
46738f73 MK |
1150 | int i, ret; |
1151 | ||
46738f73 MK |
1152 | for (i = 0; i < num_socks; i++) { |
1153 | fds[i].fd = xsk_socket__fd(xsks[i]->xsk); | |
1154 | fds[i].events = POLLOUT | POLLIN; | |
1155 | } | |
b4b8faa1 | 1156 | |
46738f73 MK |
1157 | for (;;) { |
1158 | if (opt_poll) { | |
1159 | ret = poll(fds, num_socks, opt_timeout); | |
1160 | if (ret <= 0) | |
1161 | continue; | |
1162 | } | |
b4b8faa1 | 1163 | |
46738f73 MK |
1164 | for (i = 0; i < num_socks; i++) |
1165 | l2fwd(xsks[i], fds); | |
d3f11b01 JJ |
1166 | |
1167 | if (benchmark_done) | |
1168 | break; | |
b4b8faa1 MK |
1169 | } |
1170 | } | |
1171 | ||
2e5d72c1 MK |
1172 | static void load_xdp_program(char **argv, struct bpf_object **obj) |
1173 | { | |
1174 | struct bpf_prog_load_attr prog_load_attr = { | |
1175 | .prog_type = BPF_PROG_TYPE_XDP, | |
1176 | }; | |
1177 | char xdp_filename[256]; | |
1178 | int prog_fd; | |
1179 | ||
1180 | snprintf(xdp_filename, sizeof(xdp_filename), "%s_kern.o", argv[0]); | |
1181 | prog_load_attr.file = xdp_filename; | |
1182 | ||
1183 | if (bpf_prog_load_xattr(&prog_load_attr, obj, &prog_fd)) | |
1184 | exit(EXIT_FAILURE); | |
1185 | if (prog_fd < 0) { | |
1186 | fprintf(stderr, "ERROR: no program found: %s\n", | |
1187 | strerror(prog_fd)); | |
1188 | exit(EXIT_FAILURE); | |
1189 | } | |
1190 | ||
1191 | if (bpf_set_link_xdp_fd(opt_ifindex, prog_fd, opt_xdp_flags) < 0) { | |
1192 | fprintf(stderr, "ERROR: link set xdp fd failed\n"); | |
1193 | exit(EXIT_FAILURE); | |
1194 | } | |
1195 | } | |
1196 | ||
1197 | static void enter_xsks_into_map(struct bpf_object *obj) | |
1198 | { | |
1199 | struct bpf_map *map; | |
1200 | int i, xsks_map; | |
1201 | ||
1202 | map = bpf_object__find_map_by_name(obj, "xsks_map"); | |
1203 | xsks_map = bpf_map__fd(map); | |
1204 | if (xsks_map < 0) { | |
1205 | fprintf(stderr, "ERROR: no xsks map found: %s\n", | |
1206 | strerror(xsks_map)); | |
1207 | exit(EXIT_FAILURE); | |
1208 | } | |
1209 | ||
1210 | for (i = 0; i < num_socks; i++) { | |
1211 | int fd = xsk_socket__fd(xsks[i]->xsk); | |
1212 | int key, ret; | |
1213 | ||
1214 | key = i; | |
1215 | ret = bpf_map_update_elem(xsks_map, &key, &fd, 0); | |
1216 | if (ret) { | |
1217 | fprintf(stderr, "ERROR: bpf_map_update_elem %d\n", i); | |
1218 | exit(EXIT_FAILURE); | |
1219 | } | |
1220 | } | |
1221 | } | |
1222 | ||
b4b8faa1 MK |
1223 | int main(int argc, char **argv) |
1224 | { | |
1225 | struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY}; | |
661842c4 | 1226 | bool rx = false, tx = false; |
248c7f9c | 1227 | struct xsk_umem_info *umem; |
2e5d72c1 | 1228 | struct bpf_object *obj; |
b4b8faa1 | 1229 | pthread_t pt; |
2e5d72c1 | 1230 | int i, ret; |
248c7f9c | 1231 | void *bufs; |
b4b8faa1 MK |
1232 | |
1233 | parse_command_line(argc, argv); | |
1234 | ||
1235 | if (setrlimit(RLIMIT_MEMLOCK, &r)) { | |
1236 | fprintf(stderr, "ERROR: setrlimit(RLIMIT_MEMLOCK) \"%s\"\n", | |
1237 | strerror(errno)); | |
1238 | exit(EXIT_FAILURE); | |
1239 | } | |
1240 | ||
2e5d72c1 MK |
1241 | if (opt_num_xsks > 1) |
1242 | load_xdp_program(argv, &obj); | |
1243 | ||
3945b37a KL |
1244 | /* Reserve memory for the umem. Use hugepages if unaligned chunk mode */ |
1245 | bufs = mmap(NULL, NUM_FRAMES * opt_xsk_frame_size, | |
1246 | PROT_READ | PROT_WRITE, | |
1247 | MAP_PRIVATE | MAP_ANONYMOUS | opt_mmap_flags, -1, 0); | |
1248 | if (bufs == MAP_FAILED) { | |
1249 | printf("ERROR: mmap failed\n"); | |
1250 | exit(EXIT_FAILURE); | |
1251 | } | |
2e5d72c1 MK |
1252 | |
1253 | /* Create sockets... */ | |
123e8da1 | 1254 | umem = xsk_configure_umem(bufs, NUM_FRAMES * opt_xsk_frame_size); |
661842c4 MK |
1255 | if (opt_bench == BENCH_RXDROP || opt_bench == BENCH_L2FWD) { |
1256 | rx = true; | |
1257 | xsk_populate_fill_ring(umem); | |
1258 | } | |
1259 | if (opt_bench == BENCH_L2FWD || opt_bench == BENCH_TXONLY) | |
1260 | tx = true; | |
2e5d72c1 | 1261 | for (i = 0; i < opt_num_xsks; i++) |
661842c4 | 1262 | xsks[num_socks++] = xsk_configure_socket(umem, rx, tx); |
b4b8faa1 | 1263 | |
4a3c23ae JJ |
1264 | if (opt_bench == BENCH_TXONLY) { |
1265 | gen_eth_hdr_data(); | |
1266 | ||
661842c4 MK |
1267 | for (i = 0; i < NUM_FRAMES; i++) |
1268 | gen_eth_frame(umem, i * opt_xsk_frame_size); | |
4a3c23ae | 1269 | } |
b4b8faa1 | 1270 | |
2e5d72c1 MK |
1271 | if (opt_num_xsks > 1 && opt_bench != BENCH_TXONLY) |
1272 | enter_xsks_into_map(obj); | |
b4b8faa1 MK |
1273 | |
1274 | signal(SIGINT, int_exit); | |
1275 | signal(SIGTERM, int_exit); | |
1276 | signal(SIGABRT, int_exit); | |
1277 | ||
1278 | setlocale(LC_ALL, ""); | |
1279 | ||
1280 | ret = pthread_create(&pt, NULL, poller, NULL); | |
248c7f9c MK |
1281 | if (ret) |
1282 | exit_with_error(ret); | |
b4b8faa1 MK |
1283 | |
1284 | prev_time = get_nsecs(); | |
d3f11b01 | 1285 | start_time = prev_time; |
b4b8faa1 MK |
1286 | |
1287 | if (opt_bench == BENCH_RXDROP) | |
1288 | rx_drop_all(); | |
1289 | else if (opt_bench == BENCH_TXONLY) | |
46738f73 | 1290 | tx_only_all(); |
b4b8faa1 | 1291 | else |
46738f73 | 1292 | l2fwd_all(); |
b4b8faa1 | 1293 | |
ece6e969 JJ |
1294 | benchmark_done = true; |
1295 | ||
d3f11b01 JJ |
1296 | pthread_join(pt, NULL); |
1297 | ||
69525588 JJ |
1298 | xdpsock_cleanup(); |
1299 | ||
b4b8faa1 MK |
1300 | return 0; |
1301 | } |