1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
4 * AF_XDP user-space access library.
6 * Copyright(c) 2018 - 2019 Intel Corporation.
8 * Author(s): Magnus Karlsson <magnus.karlsson@intel.com>
15 #include <arpa/inet.h>
16 #include <asm/barrier.h>
17 #include <linux/compiler.h>
18 #include <linux/ethtool.h>
19 #include <linux/filter.h>
20 #include <linux/if_ether.h>
21 #include <linux/if_packet.h>
22 #include <linux/if_xdp.h>
23 #include <linux/sockios.h>
25 #include <sys/ioctl.h>
27 #include <sys/socket.h>
28 #include <sys/types.h>
32 #include "libbpf_internal.h"
48 struct xsk_ring_prod
*fill
;
49 struct xsk_ring_cons
*comp
;
51 struct xsk_umem_config config
;
57 struct xsk_ring_cons
*rx
;
58 struct xsk_ring_prod
*tx
;
60 struct xsk_umem
*umem
;
61 struct xsk_socket_config config
;
67 char ifname
[IFNAMSIZ
];
71 bool xdp_prog_attached
;
76 /* Up until and including Linux 5.3 */
77 struct xdp_ring_offset_v1
{
83 /* Up until and including Linux 5.3 */
84 struct xdp_mmap_offsets_v1
{
85 struct xdp_ring_offset_v1 rx
;
86 struct xdp_ring_offset_v1 tx
;
87 struct xdp_ring_offset_v1 fr
;
88 struct xdp_ring_offset_v1 cr
;
91 int xsk_umem__fd(const struct xsk_umem
*umem
)
93 return umem
? umem
->fd
: -EINVAL
;
96 int xsk_socket__fd(const struct xsk_socket
*xsk
)
98 return xsk
? xsk
->fd
: -EINVAL
;
101 static bool xsk_page_aligned(void *buffer
)
103 unsigned long addr
= (unsigned long)buffer
;
105 return !(addr
& (getpagesize() - 1));
108 static void xsk_set_umem_config(struct xsk_umem_config
*cfg
,
109 const struct xsk_umem_config
*usr_cfg
)
112 cfg
->fill_size
= XSK_RING_PROD__DEFAULT_NUM_DESCS
;
113 cfg
->comp_size
= XSK_RING_CONS__DEFAULT_NUM_DESCS
;
114 cfg
->frame_size
= XSK_UMEM__DEFAULT_FRAME_SIZE
;
115 cfg
->frame_headroom
= XSK_UMEM__DEFAULT_FRAME_HEADROOM
;
116 cfg
->flags
= XSK_UMEM__DEFAULT_FLAGS
;
120 cfg
->fill_size
= usr_cfg
->fill_size
;
121 cfg
->comp_size
= usr_cfg
->comp_size
;
122 cfg
->frame_size
= usr_cfg
->frame_size
;
123 cfg
->frame_headroom
= usr_cfg
->frame_headroom
;
124 cfg
->flags
= usr_cfg
->flags
;
127 static int xsk_set_xdp_socket_config(struct xsk_socket_config
*cfg
,
128 const struct xsk_socket_config
*usr_cfg
)
131 cfg
->rx_size
= XSK_RING_CONS__DEFAULT_NUM_DESCS
;
132 cfg
->tx_size
= XSK_RING_PROD__DEFAULT_NUM_DESCS
;
133 cfg
->libbpf_flags
= 0;
139 if (usr_cfg
->libbpf_flags
& ~XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD
)
142 cfg
->rx_size
= usr_cfg
->rx_size
;
143 cfg
->tx_size
= usr_cfg
->tx_size
;
144 cfg
->libbpf_flags
= usr_cfg
->libbpf_flags
;
145 cfg
->xdp_flags
= usr_cfg
->xdp_flags
;
146 cfg
->bind_flags
= usr_cfg
->bind_flags
;
151 static void xsk_mmap_offsets_v1(struct xdp_mmap_offsets
*off
)
153 struct xdp_mmap_offsets_v1 off_v1
;
155 /* getsockopt on a kernel <= 5.3 has no flags fields.
156 * Copy over the offsets to the correct places in the >=5.4 format
157 * and put the flags where they would have been on that kernel.
159 memcpy(&off_v1
, off
, sizeof(off_v1
));
161 off
->rx
.producer
= off_v1
.rx
.producer
;
162 off
->rx
.consumer
= off_v1
.rx
.consumer
;
163 off
->rx
.desc
= off_v1
.rx
.desc
;
164 off
->rx
.flags
= off_v1
.rx
.consumer
+ sizeof(__u32
);
166 off
->tx
.producer
= off_v1
.tx
.producer
;
167 off
->tx
.consumer
= off_v1
.tx
.consumer
;
168 off
->tx
.desc
= off_v1
.tx
.desc
;
169 off
->tx
.flags
= off_v1
.tx
.consumer
+ sizeof(__u32
);
171 off
->fr
.producer
= off_v1
.fr
.producer
;
172 off
->fr
.consumer
= off_v1
.fr
.consumer
;
173 off
->fr
.desc
= off_v1
.fr
.desc
;
174 off
->fr
.flags
= off_v1
.fr
.consumer
+ sizeof(__u32
);
176 off
->cr
.producer
= off_v1
.cr
.producer
;
177 off
->cr
.consumer
= off_v1
.cr
.consumer
;
178 off
->cr
.desc
= off_v1
.cr
.desc
;
179 off
->cr
.flags
= off_v1
.cr
.consumer
+ sizeof(__u32
);
182 static int xsk_get_mmap_offsets(int fd
, struct xdp_mmap_offsets
*off
)
187 optlen
= sizeof(*off
);
188 err
= getsockopt(fd
, SOL_XDP
, XDP_MMAP_OFFSETS
, off
, &optlen
);
192 if (optlen
== sizeof(*off
))
195 if (optlen
== sizeof(struct xdp_mmap_offsets_v1
)) {
196 xsk_mmap_offsets_v1(off
);
203 int xsk_umem__create_v0_0_4(struct xsk_umem
**umem_ptr
, void *umem_area
,
204 __u64 size
, struct xsk_ring_prod
*fill
,
205 struct xsk_ring_cons
*comp
,
206 const struct xsk_umem_config
*usr_config
)
208 struct xdp_mmap_offsets off
;
209 struct xdp_umem_reg mr
;
210 struct xsk_umem
*umem
;
214 if (!umem_area
|| !umem_ptr
|| !fill
|| !comp
)
216 if (!size
&& !xsk_page_aligned(umem_area
))
219 umem
= calloc(1, sizeof(*umem
));
223 umem
->fd
= socket(AF_XDP
, SOCK_RAW
, 0);
229 umem
->umem_area
= umem_area
;
230 xsk_set_umem_config(&umem
->config
, usr_config
);
232 memset(&mr
, 0, sizeof(mr
));
233 mr
.addr
= (uintptr_t)umem_area
;
235 mr
.chunk_size
= umem
->config
.frame_size
;
236 mr
.headroom
= umem
->config
.frame_headroom
;
237 mr
.flags
= umem
->config
.flags
;
239 err
= setsockopt(umem
->fd
, SOL_XDP
, XDP_UMEM_REG
, &mr
, sizeof(mr
));
244 err
= setsockopt(umem
->fd
, SOL_XDP
, XDP_UMEM_FILL_RING
,
245 &umem
->config
.fill_size
,
246 sizeof(umem
->config
.fill_size
));
251 err
= setsockopt(umem
->fd
, SOL_XDP
, XDP_UMEM_COMPLETION_RING
,
252 &umem
->config
.comp_size
,
253 sizeof(umem
->config
.comp_size
));
259 err
= xsk_get_mmap_offsets(umem
->fd
, &off
);
265 map
= mmap(NULL
, off
.fr
.desc
+ umem
->config
.fill_size
* sizeof(__u64
),
266 PROT_READ
| PROT_WRITE
, MAP_SHARED
| MAP_POPULATE
, umem
->fd
,
267 XDP_UMEM_PGOFF_FILL_RING
);
268 if (map
== MAP_FAILED
) {
274 fill
->mask
= umem
->config
.fill_size
- 1;
275 fill
->size
= umem
->config
.fill_size
;
276 fill
->producer
= map
+ off
.fr
.producer
;
277 fill
->consumer
= map
+ off
.fr
.consumer
;
278 fill
->flags
= map
+ off
.fr
.flags
;
279 fill
->ring
= map
+ off
.fr
.desc
;
280 fill
->cached_cons
= umem
->config
.fill_size
;
282 map
= mmap(NULL
, off
.cr
.desc
+ umem
->config
.comp_size
* sizeof(__u64
),
283 PROT_READ
| PROT_WRITE
, MAP_SHARED
| MAP_POPULATE
, umem
->fd
,
284 XDP_UMEM_PGOFF_COMPLETION_RING
);
285 if (map
== MAP_FAILED
) {
291 comp
->mask
= umem
->config
.comp_size
- 1;
292 comp
->size
= umem
->config
.comp_size
;
293 comp
->producer
= map
+ off
.cr
.producer
;
294 comp
->consumer
= map
+ off
.cr
.consumer
;
295 comp
->flags
= map
+ off
.cr
.flags
;
296 comp
->ring
= map
+ off
.cr
.desc
;
302 munmap(map
, off
.fr
.desc
+ umem
->config
.fill_size
* sizeof(__u64
));
310 struct xsk_umem_config_v1
{
314 __u32 frame_headroom
;
317 int xsk_umem__create_v0_0_2(struct xsk_umem
**umem_ptr
, void *umem_area
,
318 __u64 size
, struct xsk_ring_prod
*fill
,
319 struct xsk_ring_cons
*comp
,
320 const struct xsk_umem_config
*usr_config
)
322 struct xsk_umem_config config
;
324 memcpy(&config
, usr_config
, sizeof(struct xsk_umem_config_v1
));
327 return xsk_umem__create_v0_0_4(umem_ptr
, umem_area
, size
, fill
, comp
,
330 COMPAT_VERSION(xsk_umem__create_v0_0_2
, xsk_umem__create
, LIBBPF_0
.0
.2)
331 DEFAULT_VERSION(xsk_umem__create_v0_0_4
, xsk_umem__create
, LIBBPF_0
.0
.4)
333 static int xsk_load_xdp_prog(struct xsk_socket
*xsk
)
335 static const int log_buf_size
= 16 * 1024;
336 char log_buf
[log_buf_size
];
339 /* This is the C-program:
340 * SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
342 * int ret, index = ctx->rx_queue_index;
344 * // A set entry here means that the correspnding queue_id
345 * // has an active AF_XDP socket bound to it.
346 * ret = bpf_redirect_map(&xsks_map, index, XDP_PASS);
350 * // Fallback for pre-5.3 kernels, not supporting default
351 * // action in the flags parameter.
352 * if (bpf_map_lookup_elem(&xsks_map, &index))
353 * return bpf_redirect_map(&xsks_map, index, 0);
357 struct bpf_insn prog
[] = {
358 /* r2 = *(u32 *)(r1 + 16) */
359 BPF_LDX_MEM(BPF_W
, BPF_REG_2
, BPF_REG_1
, 16),
360 /* *(u32 *)(r10 - 4) = r2 */
361 BPF_STX_MEM(BPF_W
, BPF_REG_10
, BPF_REG_2
, -4),
363 BPF_LD_MAP_FD(BPF_REG_1
, xsk
->xsks_map_fd
),
365 BPF_MOV64_IMM(BPF_REG_3
, 2),
366 /* call bpf_redirect_map */
367 BPF_EMIT_CALL(BPF_FUNC_redirect_map
),
368 /* if w0 != 0 goto pc+13 */
369 BPF_JMP32_IMM(BPF_JSGT
, BPF_REG_0
, 0, 13),
371 BPF_MOV64_REG(BPF_REG_2
, BPF_REG_10
),
373 BPF_ALU64_IMM(BPF_ADD
, BPF_REG_2
, -4),
375 BPF_LD_MAP_FD(BPF_REG_1
, xsk
->xsks_map_fd
),
376 /* call bpf_map_lookup_elem */
377 BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem
),
379 BPF_MOV64_REG(BPF_REG_1
, BPF_REG_0
),
381 BPF_MOV64_IMM(BPF_REG_0
, 2),
382 /* if r1 == 0 goto pc+5 */
383 BPF_JMP_IMM(BPF_JEQ
, BPF_REG_1
, 0, 5),
384 /* r2 = *(u32 *)(r10 - 4) */
385 BPF_LDX_MEM(BPF_W
, BPF_REG_2
, BPF_REG_10
, -4),
387 BPF_LD_MAP_FD(BPF_REG_1
, xsk
->xsks_map_fd
),
389 BPF_MOV64_IMM(BPF_REG_3
, 0),
390 /* call bpf_redirect_map */
391 BPF_EMIT_CALL(BPF_FUNC_redirect_map
),
392 /* The jumps are to this instruction */
395 size_t insns_cnt
= sizeof(prog
) / sizeof(struct bpf_insn
);
397 prog_fd
= bpf_load_program(BPF_PROG_TYPE_XDP
, prog
, insns_cnt
,
398 "LGPL-2.1 or BSD-2-Clause", 0, log_buf
,
401 pr_warn("BPF log buffer:\n%s", log_buf
);
405 err
= bpf_set_link_xdp_fd(xsk
->ifindex
, prog_fd
, xsk
->config
.xdp_flags
);
411 xsk
->prog_fd
= prog_fd
;
415 static int xsk_get_max_queues(struct xsk_socket
*xsk
)
417 struct ethtool_channels channels
= { .cmd
= ETHTOOL_GCHANNELS
};
418 struct ifreq ifr
= {};
421 fd
= socket(AF_INET
, SOCK_DGRAM
, 0);
425 ifr
.ifr_data
= (void *)&channels
;
426 memcpy(ifr
.ifr_name
, xsk
->ifname
, IFNAMSIZ
- 1);
427 ifr
.ifr_name
[IFNAMSIZ
- 1] = '\0';
428 err
= ioctl(fd
, SIOCETHTOOL
, &ifr
);
429 if (err
&& errno
!= EOPNOTSUPP
) {
435 /* If the device says it has no channels, then all traffic
436 * is sent to a single stream, so max queues = 1.
440 /* Take the max of rx, tx, combined. Drivers return
441 * the number of channels in different ways.
443 ret
= max(channels
.max_rx
, channels
.max_tx
);
444 ret
= max(ret
, (int)channels
.max_combined
);
452 static int xsk_create_bpf_maps(struct xsk_socket
*xsk
)
457 max_queues
= xsk_get_max_queues(xsk
);
461 fd
= bpf_create_map_name(BPF_MAP_TYPE_XSKMAP
, "xsks_map",
462 sizeof(int), sizeof(int), max_queues
, 0);
466 xsk
->xsks_map_fd
= fd
;
471 static void xsk_delete_bpf_maps(struct xsk_socket
*xsk
)
473 bpf_map_delete_elem(xsk
->xsks_map_fd
, &xsk
->queue_id
);
474 close(xsk
->xsks_map_fd
);
477 static int xsk_lookup_bpf_maps(struct xsk_socket
*xsk
)
479 __u32 i
, *map_ids
, num_maps
, prog_len
= sizeof(struct bpf_prog_info
);
480 __u32 map_len
= sizeof(struct bpf_map_info
);
481 struct bpf_prog_info prog_info
= {};
482 struct bpf_map_info map_info
;
485 err
= bpf_obj_get_info_by_fd(xsk
->prog_fd
, &prog_info
, &prog_len
);
489 num_maps
= prog_info
.nr_map_ids
;
491 map_ids
= calloc(prog_info
.nr_map_ids
, sizeof(*map_ids
));
495 memset(&prog_info
, 0, prog_len
);
496 prog_info
.nr_map_ids
= num_maps
;
497 prog_info
.map_ids
= (__u64
)(unsigned long)map_ids
;
499 err
= bpf_obj_get_info_by_fd(xsk
->prog_fd
, &prog_info
, &prog_len
);
503 xsk
->xsks_map_fd
= -1;
505 for (i
= 0; i
< prog_info
.nr_map_ids
; i
++) {
506 fd
= bpf_map_get_fd_by_id(map_ids
[i
]);
510 err
= bpf_obj_get_info_by_fd(fd
, &map_info
, &map_len
);
516 if (!strcmp(map_info
.name
, "xsks_map")) {
517 xsk
->xsks_map_fd
= fd
;
525 if (xsk
->xsks_map_fd
== -1)
533 static int xsk_set_bpf_maps(struct xsk_socket
*xsk
)
535 return bpf_map_update_elem(xsk
->xsks_map_fd
, &xsk
->queue_id
,
539 static int xsk_setup_xdp_prog(struct xsk_socket
*xsk
)
544 err
= bpf_get_link_xdp_id(xsk
->ifindex
, &prog_id
,
545 xsk
->config
.xdp_flags
);
550 err
= xsk_create_bpf_maps(xsk
);
554 err
= xsk_load_xdp_prog(xsk
);
556 xsk_delete_bpf_maps(xsk
);
560 xsk
->prog_fd
= bpf_prog_get_fd_by_id(prog_id
);
561 if (xsk
->prog_fd
< 0)
563 err
= xsk_lookup_bpf_maps(xsk
);
571 err
= xsk_set_bpf_maps(xsk
);
573 xsk_delete_bpf_maps(xsk
);
581 int xsk_socket__create(struct xsk_socket
**xsk_ptr
, const char *ifname
,
582 __u32 queue_id
, struct xsk_umem
*umem
,
583 struct xsk_ring_cons
*rx
, struct xsk_ring_prod
*tx
,
584 const struct xsk_socket_config
*usr_config
)
586 void *rx_map
= NULL
, *tx_map
= NULL
;
587 struct sockaddr_xdp sxdp
= {};
588 struct xdp_mmap_offsets off
;
589 struct xsk_socket
*xsk
;
592 if (!umem
|| !xsk_ptr
|| !(rx
|| tx
))
595 xsk
= calloc(1, sizeof(*xsk
));
599 err
= xsk_set_xdp_socket_config(&xsk
->config
, usr_config
);
603 if (umem
->refcount
&&
604 !(xsk
->config
.libbpf_flags
& XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD
)) {
605 pr_warn("Error: shared umems not supported by libbpf supplied XDP program.\n");
610 if (umem
->refcount
++ > 0) {
611 xsk
->fd
= socket(AF_XDP
, SOCK_RAW
, 0);
620 xsk
->outstanding_tx
= 0;
621 xsk
->queue_id
= queue_id
;
623 xsk
->ifindex
= if_nametoindex(ifname
);
628 memcpy(xsk
->ifname
, ifname
, IFNAMSIZ
- 1);
629 xsk
->ifname
[IFNAMSIZ
- 1] = '\0';
632 err
= setsockopt(xsk
->fd
, SOL_XDP
, XDP_RX_RING
,
633 &xsk
->config
.rx_size
,
634 sizeof(xsk
->config
.rx_size
));
641 err
= setsockopt(xsk
->fd
, SOL_XDP
, XDP_TX_RING
,
642 &xsk
->config
.tx_size
,
643 sizeof(xsk
->config
.tx_size
));
650 err
= xsk_get_mmap_offsets(xsk
->fd
, &off
);
657 rx_map
= mmap(NULL
, off
.rx
.desc
+
658 xsk
->config
.rx_size
* sizeof(struct xdp_desc
),
659 PROT_READ
| PROT_WRITE
, MAP_SHARED
| MAP_POPULATE
,
660 xsk
->fd
, XDP_PGOFF_RX_RING
);
661 if (rx_map
== MAP_FAILED
) {
666 rx
->mask
= xsk
->config
.rx_size
- 1;
667 rx
->size
= xsk
->config
.rx_size
;
668 rx
->producer
= rx_map
+ off
.rx
.producer
;
669 rx
->consumer
= rx_map
+ off
.rx
.consumer
;
670 rx
->flags
= rx_map
+ off
.rx
.flags
;
671 rx
->ring
= rx_map
+ off
.rx
.desc
;
676 tx_map
= mmap(NULL
, off
.tx
.desc
+
677 xsk
->config
.tx_size
* sizeof(struct xdp_desc
),
678 PROT_READ
| PROT_WRITE
, MAP_SHARED
| MAP_POPULATE
,
679 xsk
->fd
, XDP_PGOFF_TX_RING
);
680 if (tx_map
== MAP_FAILED
) {
685 tx
->mask
= xsk
->config
.tx_size
- 1;
686 tx
->size
= xsk
->config
.tx_size
;
687 tx
->producer
= tx_map
+ off
.tx
.producer
;
688 tx
->consumer
= tx_map
+ off
.tx
.consumer
;
689 tx
->flags
= tx_map
+ off
.tx
.flags
;
690 tx
->ring
= tx_map
+ off
.tx
.desc
;
691 tx
->cached_cons
= xsk
->config
.tx_size
;
695 sxdp
.sxdp_family
= PF_XDP
;
696 sxdp
.sxdp_ifindex
= xsk
->ifindex
;
697 sxdp
.sxdp_queue_id
= xsk
->queue_id
;
698 if (umem
->refcount
> 1) {
699 sxdp
.sxdp_flags
= XDP_SHARED_UMEM
;
700 sxdp
.sxdp_shared_umem_fd
= umem
->fd
;
702 sxdp
.sxdp_flags
= xsk
->config
.bind_flags
;
705 err
= bind(xsk
->fd
, (struct sockaddr
*)&sxdp
, sizeof(sxdp
));
713 if (!(xsk
->config
.libbpf_flags
& XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD
)) {
714 err
= xsk_setup_xdp_prog(xsk
);
724 munmap(tx_map
, off
.tx
.desc
+
725 xsk
->config
.tx_size
* sizeof(struct xdp_desc
));
728 munmap(rx_map
, off
.rx
.desc
+
729 xsk
->config
.rx_size
* sizeof(struct xdp_desc
));
731 if (--umem
->refcount
)
738 int xsk_umem__delete(struct xsk_umem
*umem
)
740 struct xdp_mmap_offsets off
;
749 err
= xsk_get_mmap_offsets(umem
->fd
, &off
);
751 munmap(umem
->fill
->ring
- off
.fr
.desc
,
752 off
.fr
.desc
+ umem
->config
.fill_size
* sizeof(__u64
));
753 munmap(umem
->comp
->ring
- off
.cr
.desc
,
754 off
.cr
.desc
+ umem
->config
.comp_size
* sizeof(__u64
));
763 void xsk_socket__delete(struct xsk_socket
*xsk
)
765 size_t desc_sz
= sizeof(struct xdp_desc
);
766 struct xdp_mmap_offsets off
;
772 if (xsk
->prog_fd
!= -1) {
773 xsk_delete_bpf_maps(xsk
);
777 err
= xsk_get_mmap_offsets(xsk
->fd
, &off
);
780 munmap(xsk
->rx
->ring
- off
.rx
.desc
,
781 off
.rx
.desc
+ xsk
->config
.rx_size
* desc_sz
);
784 munmap(xsk
->tx
->ring
- off
.tx
.desc
,
785 off
.tx
.desc
+ xsk
->config
.tx_size
* desc_sz
);
790 xsk
->umem
->refcount
--;
791 /* Do not close an fd that also has an associated umem connected
794 if (xsk
->fd
!= xsk
->umem
->fd
)