1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2017 - 2018 Intel Corporation. */
4 #include <asm/barrier.h>
9 #include <linux/compiler.h>
10 #include <linux/if_link.h>
11 #include <linux/if_xdp.h>
12 #include <linux/if_ether.h>
14 #include <net/ethernet.h>
24 #include <sys/resource.h>
25 #include <sys/socket.h>
26 #include <sys/types.h>
46 #define NUM_FRAMES (4 * 1024)
49 #define DEBUG_HEXDUMP 0
55 static unsigned long prev_time
;
63 static enum benchmark_type opt_bench
= BENCH_RXDROP
;
64 static u32 opt_xdp_flags
= XDP_FLAGS_UPDATE_IF_NOEXIST
;
65 static const char *opt_if
= "";
66 static int opt_ifindex
;
69 static int opt_interval
= 1;
70 static u32 opt_xdp_bind_flags
= XDP_USE_NEED_WAKEUP
;
71 static u32 opt_umem_flags
;
72 static int opt_unaligned_chunks
;
73 static int opt_mmap_flags
;
74 static u32 opt_xdp_bind_flags
;
75 static int opt_xsk_frame_size
= XSK_UMEM__DEFAULT_FRAME_SIZE
;
76 static int opt_timeout
= 1000;
77 static bool opt_need_wakeup
= true;
80 struct xsk_umem_info
{
81 struct xsk_ring_prod fq
;
82 struct xsk_ring_cons cq
;
83 struct xsk_umem
*umem
;
87 struct xsk_socket_info
{
88 struct xsk_ring_cons rx
;
89 struct xsk_ring_prod tx
;
90 struct xsk_umem_info
*umem
;
91 struct xsk_socket
*xsk
;
92 unsigned long rx_npkts
;
93 unsigned long tx_npkts
;
94 unsigned long prev_rx_npkts
;
95 unsigned long prev_tx_npkts
;
100 struct xsk_socket_info
*xsks
[MAX_SOCKS
];
102 static unsigned long get_nsecs(void)
106 clock_gettime(CLOCK_MONOTONIC
, &ts
);
107 return ts
.tv_sec
* 1000000000UL + ts
.tv_nsec
;
110 static void print_benchmark(bool running
)
112 const char *bench_str
= "INVALID";
114 if (opt_bench
== BENCH_RXDROP
)
115 bench_str
= "rxdrop";
116 else if (opt_bench
== BENCH_TXONLY
)
117 bench_str
= "txonly";
118 else if (opt_bench
== BENCH_L2FWD
)
121 printf("%s:%d %s ", opt_if
, opt_queue
, bench_str
);
122 if (opt_xdp_flags
& XDP_FLAGS_SKB_MODE
)
124 else if (opt_xdp_flags
& XDP_FLAGS_DRV_MODE
)
133 printf("running...");
138 static void dump_stats(void)
140 unsigned long now
= get_nsecs();
141 long dt
= now
- prev_time
;
146 for (i
= 0; i
< num_socks
&& xsks
[i
]; i
++) {
147 char *fmt
= "%-15s %'-11.0f %'-11lu\n";
148 double rx_pps
, tx_pps
;
150 rx_pps
= (xsks
[i
]->rx_npkts
- xsks
[i
]->prev_rx_npkts
) *
152 tx_pps
= (xsks
[i
]->tx_npkts
- xsks
[i
]->prev_tx_npkts
) *
155 printf("\n sock%d@", i
);
156 print_benchmark(false);
159 printf("%-15s %-11s %-11s %-11.2f\n", "", "pps", "pkts",
161 printf(fmt
, "rx", rx_pps
, xsks
[i
]->rx_npkts
);
162 printf(fmt
, "tx", tx_pps
, xsks
[i
]->tx_npkts
);
164 xsks
[i
]->prev_rx_npkts
= xsks
[i
]->rx_npkts
;
165 xsks
[i
]->prev_tx_npkts
= xsks
[i
]->tx_npkts
;
169 static void *poller(void *arg
)
180 static void remove_xdp_program(void)
182 __u32 curr_prog_id
= 0;
184 if (bpf_get_link_xdp_id(opt_ifindex
, &curr_prog_id
, opt_xdp_flags
)) {
185 printf("bpf_get_link_xdp_id failed\n");
188 if (prog_id
== curr_prog_id
)
189 bpf_set_link_xdp_fd(opt_ifindex
, -1, opt_xdp_flags
);
190 else if (!curr_prog_id
)
191 printf("couldn't find a prog id on a given interface\n");
193 printf("program on interface changed, not removing\n");
196 static void int_exit(int sig
)
198 struct xsk_umem
*umem
= xsks
[0]->umem
->umem
;
203 xsk_socket__delete(xsks
[0]->xsk
);
204 (void)xsk_umem__delete(umem
);
205 remove_xdp_program();
210 static void __exit_with_error(int error
, const char *file
, const char *func
,
213 fprintf(stderr
, "%s:%s:%i: errno: %d/\"%s\"\n", file
, func
,
214 line
, error
, strerror(error
));
216 remove_xdp_program();
220 #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, \
223 static const char pkt_data
[] =
224 "\x3c\xfd\xfe\x9e\x7f\x71\xec\xb1\xd7\x98\x3a\xc0\x08\x00\x45\x00"
225 "\x00\x2e\x00\x00\x00\x00\x40\x11\x88\x97\x05\x08\x07\x08\xc8\x14"
226 "\x1e\x04\x10\x92\x10\x92\x00\x1a\x6d\xa3\x34\x33\x1f\x69\x40\x6b"
227 "\x54\x59\xb6\x14\x2d\x11\x44\xbf\xaf\xd9\xbe\xaa";
229 static void swap_mac_addresses(void *data
)
231 struct ether_header
*eth
= (struct ether_header
*)data
;
232 struct ether_addr
*src_addr
= (struct ether_addr
*)ð
->ether_shost
;
233 struct ether_addr
*dst_addr
= (struct ether_addr
*)ð
->ether_dhost
;
234 struct ether_addr tmp
;
237 *src_addr
= *dst_addr
;
241 static void hex_dump(void *pkt
, size_t length
, u64 addr
)
243 const unsigned char *address
= (unsigned char *)pkt
;
244 const unsigned char *line
= address
;
245 size_t line_size
= 32;
253 sprintf(buf
, "addr=%llu", addr
);
254 printf("length = %zu\n", length
);
255 printf("%s | ", buf
);
256 while (length
-- > 0) {
257 printf("%02X ", *address
++);
258 if (!(++i
% line_size
) || (length
== 0 && i
% line_size
)) {
260 while (i
++ % line_size
)
263 printf(" | "); /* right close */
264 while (line
< address
) {
266 printf("%c", (c
< 33 || c
== 255) ? 0x2E : c
);
270 printf("%s | ", buf
);
276 static size_t gen_eth_frame(struct xsk_umem_info
*umem
, u64 addr
)
278 memcpy(xsk_umem__get_data(umem
->buffer
, addr
), pkt_data
,
279 sizeof(pkt_data
) - 1);
280 return sizeof(pkt_data
) - 1;
283 static struct xsk_umem_info
*xsk_configure_umem(void *buffer
, u64 size
)
285 struct xsk_umem_info
*umem
;
286 struct xsk_umem_config cfg
= {
287 .fill_size
= XSK_RING_PROD__DEFAULT_NUM_DESCS
,
288 .comp_size
= XSK_RING_CONS__DEFAULT_NUM_DESCS
,
289 .frame_size
= opt_xsk_frame_size
,
290 .frame_headroom
= XSK_UMEM__DEFAULT_FRAME_HEADROOM
,
291 .flags
= opt_umem_flags
296 umem
= calloc(1, sizeof(*umem
));
298 exit_with_error(errno
);
300 ret
= xsk_umem__create(&umem
->umem
, buffer
, size
, &umem
->fq
, &umem
->cq
,
304 exit_with_error(-ret
);
306 umem
->buffer
= buffer
;
310 static struct xsk_socket_info
*xsk_configure_socket(struct xsk_umem_info
*umem
)
312 struct xsk_socket_config cfg
;
313 struct xsk_socket_info
*xsk
;
318 xsk
= calloc(1, sizeof(*xsk
));
320 exit_with_error(errno
);
323 cfg
.rx_size
= XSK_RING_CONS__DEFAULT_NUM_DESCS
;
324 cfg
.tx_size
= XSK_RING_PROD__DEFAULT_NUM_DESCS
;
325 cfg
.libbpf_flags
= 0;
326 cfg
.xdp_flags
= opt_xdp_flags
;
327 cfg
.bind_flags
= opt_xdp_bind_flags
;
328 ret
= xsk_socket__create(&xsk
->xsk
, opt_if
, opt_queue
, umem
->umem
,
329 &xsk
->rx
, &xsk
->tx
, &cfg
);
331 exit_with_error(-ret
);
333 ret
= bpf_get_link_xdp_id(opt_ifindex
, &prog_id
, opt_xdp_flags
);
335 exit_with_error(-ret
);
337 ret
= xsk_ring_prod__reserve(&xsk
->umem
->fq
,
338 XSK_RING_PROD__DEFAULT_NUM_DESCS
,
340 if (ret
!= XSK_RING_PROD__DEFAULT_NUM_DESCS
)
341 exit_with_error(-ret
);
342 for (i
= 0; i
< XSK_RING_PROD__DEFAULT_NUM_DESCS
; i
++)
343 *xsk_ring_prod__fill_addr(&xsk
->umem
->fq
, idx
++) =
344 i
* opt_xsk_frame_size
;
345 xsk_ring_prod__submit(&xsk
->umem
->fq
,
346 XSK_RING_PROD__DEFAULT_NUM_DESCS
);
351 static struct option long_options
[] = {
352 {"rxdrop", no_argument
, 0, 'r'},
353 {"txonly", no_argument
, 0, 't'},
354 {"l2fwd", no_argument
, 0, 'l'},
355 {"interface", required_argument
, 0, 'i'},
356 {"queue", required_argument
, 0, 'q'},
357 {"poll", no_argument
, 0, 'p'},
358 {"xdp-skb", no_argument
, 0, 'S'},
359 {"xdp-native", no_argument
, 0, 'N'},
360 {"interval", required_argument
, 0, 'n'},
361 {"zero-copy", no_argument
, 0, 'z'},
362 {"copy", no_argument
, 0, 'c'},
363 {"frame-size", required_argument
, 0, 'f'},
364 {"no-need-wakeup", no_argument
, 0, 'm'},
365 {"unaligned", no_argument
, 0, 'u'},
369 static void usage(const char *prog
)
372 " Usage: %s [OPTIONS]\n"
374 " -r, --rxdrop Discard all incoming packets (default)\n"
375 " -t, --txonly Only send packets\n"
376 " -l, --l2fwd MAC swap L2 forwarding\n"
377 " -i, --interface=n Run on interface n\n"
378 " -q, --queue=n Use queue n (default 0)\n"
379 " -p, --poll Use poll syscall\n"
380 " -S, --xdp-skb=n Use XDP skb-mod\n"
381 " -N, --xdp-native=n Enfore XDP native mode\n"
382 " -n, --interval=n Specify statistics update interval (default 1 sec).\n"
383 " -z, --zero-copy Force zero-copy mode.\n"
384 " -c, --copy Force copy mode.\n"
385 " -f, --frame-size=n Set the frame size (must be a power of two, default is %d).\n"
386 " -m, --no-need-wakeup Turn off use of driver need wakeup flag.\n"
387 " -f, --frame-size=n Set the frame size (must be a power of two in aligned mode, default is %d).\n"
388 " -u, --unaligned Enable unaligned chunk placement\n"
390 fprintf(stderr
, str
, prog
, XSK_UMEM__DEFAULT_FRAME_SIZE
);
394 static void parse_command_line(int argc
, char **argv
)
401 c
= getopt_long(argc
, argv
, "Frtli:q:psSNn:czf:mu",
402 long_options
, &option_index
);
408 opt_bench
= BENCH_RXDROP
;
411 opt_bench
= BENCH_TXONLY
;
414 opt_bench
= BENCH_L2FWD
;
420 opt_queue
= atoi(optarg
);
426 opt_xdp_flags
|= XDP_FLAGS_SKB_MODE
;
427 opt_xdp_bind_flags
|= XDP_COPY
;
430 opt_xdp_flags
|= XDP_FLAGS_DRV_MODE
;
433 opt_interval
= atoi(optarg
);
436 opt_xdp_bind_flags
|= XDP_ZEROCOPY
;
439 opt_xdp_bind_flags
|= XDP_COPY
;
442 opt_umem_flags
|= XDP_UMEM_UNALIGNED_CHUNK_FLAG
;
443 opt_unaligned_chunks
= 1;
444 opt_mmap_flags
= MAP_HUGETLB
;
447 opt_xdp_flags
&= ~XDP_FLAGS_UPDATE_IF_NOEXIST
;
450 opt_xsk_frame_size
= atoi(optarg
);
452 opt_need_wakeup
= false;
453 opt_xdp_bind_flags
&= ~XDP_USE_NEED_WAKEUP
;
457 usage(basename(argv
[0]));
461 opt_ifindex
= if_nametoindex(opt_if
);
463 fprintf(stderr
, "ERROR: interface \"%s\" does not exist\n",
465 usage(basename(argv
[0]));
468 if ((opt_xsk_frame_size
& (opt_xsk_frame_size
- 1)) &&
469 !opt_unaligned_chunks
) {
470 fprintf(stderr
, "--frame-size=%d is not a power of two\n",
472 usage(basename(argv
[0]));
476 static void kick_tx(struct xsk_socket_info
*xsk
)
480 ret
= sendto(xsk_socket__fd(xsk
->xsk
), NULL
, 0, MSG_DONTWAIT
, NULL
, 0);
481 if (ret
>= 0 || errno
== ENOBUFS
|| errno
== EAGAIN
|| errno
== EBUSY
)
483 exit_with_error(errno
);
486 static inline void complete_tx_l2fwd(struct xsk_socket_info
*xsk
,
489 struct xsk_umem_info
*umem
= xsk
->umem
;
490 u32 idx_cq
= 0, idx_fq
= 0;
494 if (!xsk
->outstanding_tx
)
497 if (!opt_need_wakeup
|| xsk_ring_prod__needs_wakeup(&xsk
->tx
))
500 ndescs
= (xsk
->outstanding_tx
> BATCH_SIZE
) ? BATCH_SIZE
:
503 /* re-add completed Tx buffers */
504 rcvd
= xsk_ring_cons__peek(&umem
->cq
, ndescs
, &idx_cq
);
509 ret
= xsk_ring_prod__reserve(&umem
->fq
, rcvd
, &idx_fq
);
510 while (ret
!= rcvd
) {
512 exit_with_error(-ret
);
513 if (xsk_ring_prod__needs_wakeup(&umem
->fq
))
514 ret
= poll(fds
, num_socks
, opt_timeout
);
515 ret
= xsk_ring_prod__reserve(&umem
->fq
, rcvd
, &idx_fq
);
518 for (i
= 0; i
< rcvd
; i
++)
519 *xsk_ring_prod__fill_addr(&umem
->fq
, idx_fq
++) =
520 *xsk_ring_cons__comp_addr(&umem
->cq
, idx_cq
++);
522 xsk_ring_prod__submit(&xsk
->umem
->fq
, rcvd
);
523 xsk_ring_cons__release(&xsk
->umem
->cq
, rcvd
);
524 xsk
->outstanding_tx
-= rcvd
;
525 xsk
->tx_npkts
+= rcvd
;
529 static inline void complete_tx_only(struct xsk_socket_info
*xsk
)
534 if (!xsk
->outstanding_tx
)
537 if (!opt_need_wakeup
|| xsk_ring_prod__needs_wakeup(&xsk
->tx
))
540 rcvd
= xsk_ring_cons__peek(&xsk
->umem
->cq
, BATCH_SIZE
, &idx
);
542 xsk_ring_cons__release(&xsk
->umem
->cq
, rcvd
);
543 xsk
->outstanding_tx
-= rcvd
;
544 xsk
->tx_npkts
+= rcvd
;
548 static void rx_drop(struct xsk_socket_info
*xsk
, struct pollfd
*fds
)
550 unsigned int rcvd
, i
;
551 u32 idx_rx
= 0, idx_fq
= 0;
554 rcvd
= xsk_ring_cons__peek(&xsk
->rx
, BATCH_SIZE
, &idx_rx
);
556 if (xsk_ring_prod__needs_wakeup(&xsk
->umem
->fq
))
557 ret
= poll(fds
, num_socks
, opt_timeout
);
561 ret
= xsk_ring_prod__reserve(&xsk
->umem
->fq
, rcvd
, &idx_fq
);
562 while (ret
!= rcvd
) {
564 exit_with_error(-ret
);
565 if (xsk_ring_prod__needs_wakeup(&xsk
->umem
->fq
))
566 ret
= poll(fds
, num_socks
, opt_timeout
);
567 ret
= xsk_ring_prod__reserve(&xsk
->umem
->fq
, rcvd
, &idx_fq
);
570 for (i
= 0; i
< rcvd
; i
++) {
571 u64 addr
= xsk_ring_cons__rx_desc(&xsk
->rx
, idx_rx
)->addr
;
572 u32 len
= xsk_ring_cons__rx_desc(&xsk
->rx
, idx_rx
++)->len
;
573 u64 orig
= xsk_umem__extract_addr(addr
);
575 addr
= xsk_umem__add_offset_to_addr(addr
);
576 char *pkt
= xsk_umem__get_data(xsk
->umem
->buffer
, addr
);
578 hex_dump(pkt
, len
, addr
);
579 *xsk_ring_prod__fill_addr(&xsk
->umem
->fq
, idx_fq
++) = orig
;
582 xsk_ring_prod__submit(&xsk
->umem
->fq
, rcvd
);
583 xsk_ring_cons__release(&xsk
->rx
, rcvd
);
584 xsk
->rx_npkts
+= rcvd
;
587 static void rx_drop_all(void)
589 struct pollfd fds
[MAX_SOCKS
+ 1];
592 memset(fds
, 0, sizeof(fds
));
594 for (i
= 0; i
< num_socks
; i
++) {
595 fds
[i
].fd
= xsk_socket__fd(xsks
[i
]->xsk
);
596 fds
[i
].events
= POLLIN
;
601 ret
= poll(fds
, num_socks
, opt_timeout
);
606 for (i
= 0; i
< num_socks
; i
++)
607 rx_drop(xsks
[i
], fds
);
611 static void tx_only(struct xsk_socket_info
*xsk
, u32 frame_nb
)
615 if (xsk_ring_prod__reserve(&xsk
->tx
, BATCH_SIZE
, &idx
) == BATCH_SIZE
) {
618 for (i
= 0; i
< BATCH_SIZE
; i
++) {
619 xsk_ring_prod__tx_desc(&xsk
->tx
, idx
+ i
)->addr
=
620 (frame_nb
+ i
) << XSK_UMEM__DEFAULT_FRAME_SHIFT
;
621 xsk_ring_prod__tx_desc(&xsk
->tx
, idx
+ i
)->len
=
622 sizeof(pkt_data
) - 1;
625 xsk_ring_prod__submit(&xsk
->tx
, BATCH_SIZE
);
626 xsk
->outstanding_tx
+= BATCH_SIZE
;
627 frame_nb
+= BATCH_SIZE
;
628 frame_nb
%= NUM_FRAMES
;
631 complete_tx_only(xsk
);
634 static void tx_only_all(void)
636 struct pollfd fds
[MAX_SOCKS
];
637 u32 frame_nb
[MAX_SOCKS
] = {};
640 memset(fds
, 0, sizeof(fds
));
641 for (i
= 0; i
< num_socks
; i
++) {
642 fds
[0].fd
= xsk_socket__fd(xsks
[i
]->xsk
);
643 fds
[0].events
= POLLOUT
;
648 ret
= poll(fds
, num_socks
, opt_timeout
);
652 if (!(fds
[0].revents
& POLLOUT
))
656 for (i
= 0; i
< num_socks
; i
++)
657 tx_only(xsks
[i
], frame_nb
[i
]);
661 static void l2fwd(struct xsk_socket_info
*xsk
, struct pollfd
*fds
)
663 unsigned int rcvd
, i
;
664 u32 idx_rx
= 0, idx_tx
= 0;
667 complete_tx_l2fwd(xsk
, fds
);
669 rcvd
= xsk_ring_cons__peek(&xsk
->rx
, BATCH_SIZE
, &idx_rx
);
671 if (xsk_ring_prod__needs_wakeup(&xsk
->umem
->fq
))
672 ret
= poll(fds
, num_socks
, opt_timeout
);
676 ret
= xsk_ring_prod__reserve(&xsk
->tx
, rcvd
, &idx_tx
);
677 while (ret
!= rcvd
) {
679 exit_with_error(-ret
);
680 if (xsk_ring_prod__needs_wakeup(&xsk
->tx
))
682 ret
= xsk_ring_prod__reserve(&xsk
->tx
, rcvd
, &idx_tx
);
685 for (i
= 0; i
< rcvd
; i
++) {
686 u64 addr
= xsk_ring_cons__rx_desc(&xsk
->rx
, idx_rx
)->addr
;
687 u32 len
= xsk_ring_cons__rx_desc(&xsk
->rx
, idx_rx
++)->len
;
690 addr
= xsk_umem__add_offset_to_addr(addr
);
691 char *pkt
= xsk_umem__get_data(xsk
->umem
->buffer
, addr
);
693 swap_mac_addresses(pkt
);
695 hex_dump(pkt
, len
, addr
);
696 xsk_ring_prod__tx_desc(&xsk
->tx
, idx_tx
)->addr
= orig
;
697 xsk_ring_prod__tx_desc(&xsk
->tx
, idx_tx
++)->len
= len
;
700 xsk_ring_prod__submit(&xsk
->tx
, rcvd
);
701 xsk_ring_cons__release(&xsk
->rx
, rcvd
);
703 xsk
->rx_npkts
+= rcvd
;
704 xsk
->outstanding_tx
+= rcvd
;
707 static void l2fwd_all(void)
709 struct pollfd fds
[MAX_SOCKS
];
712 memset(fds
, 0, sizeof(fds
));
714 for (i
= 0; i
< num_socks
; i
++) {
715 fds
[i
].fd
= xsk_socket__fd(xsks
[i
]->xsk
);
716 fds
[i
].events
= POLLOUT
| POLLIN
;
721 ret
= poll(fds
, num_socks
, opt_timeout
);
726 for (i
= 0; i
< num_socks
; i
++)
731 int main(int argc
, char **argv
)
733 struct rlimit r
= {RLIM_INFINITY
, RLIM_INFINITY
};
734 struct xsk_umem_info
*umem
;
739 parse_command_line(argc
, argv
);
741 if (setrlimit(RLIMIT_MEMLOCK
, &r
)) {
742 fprintf(stderr
, "ERROR: setrlimit(RLIMIT_MEMLOCK) \"%s\"\n",
747 /* Reserve memory for the umem. Use hugepages if unaligned chunk mode */
748 bufs
= mmap(NULL
, NUM_FRAMES
* opt_xsk_frame_size
,
749 PROT_READ
| PROT_WRITE
,
750 MAP_PRIVATE
| MAP_ANONYMOUS
| opt_mmap_flags
, -1, 0);
751 if (bufs
== MAP_FAILED
) {
752 printf("ERROR: mmap failed\n");
755 /* Create sockets... */
756 umem
= xsk_configure_umem(bufs
, NUM_FRAMES
* opt_xsk_frame_size
);
757 xsks
[num_socks
++] = xsk_configure_socket(umem
);
759 if (opt_bench
== BENCH_TXONLY
) {
762 for (i
= 0; i
< NUM_FRAMES
; i
++)
763 (void)gen_eth_frame(umem
, i
* opt_xsk_frame_size
);
766 signal(SIGINT
, int_exit
);
767 signal(SIGTERM
, int_exit
);
768 signal(SIGABRT
, int_exit
);
770 setlocale(LC_ALL
, "");
772 ret
= pthread_create(&pt
, NULL
, poller
, NULL
);
774 exit_with_error(ret
);
776 prev_time
= get_nsecs();
778 if (opt_bench
== BENCH_RXDROP
)
780 else if (opt_bench
== BENCH_TXONLY
)