]>
Commit | Line | Data |
---|---|---|
ddc64d0a | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
aecd67b6 JDB |
2 | /* include/net/xdp.h |
3 | * | |
4 | * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc. | |
aecd67b6 JDB |
5 | */ |
6 | #ifndef __LINUX_NET_XDP_H__ | |
7 | #define __LINUX_NET_XDP_H__ | |
8 | ||
9 | /** | |
10 | * DOC: XDP RX-queue information | |
11 | * | |
12 | * The XDP RX-queue info (xdp_rxq_info) is associated with the driver | |
13 | * level RX-ring queues. It is information that is specific to how | |
14 | * the driver have configured a given RX-ring queue. | |
15 | * | |
16 | * Each xdp_buff frame received in the driver carry a (pointer) | |
17 | * reference to this xdp_rxq_info structure. This provides the XDP | |
18 | * data-path read-access to RX-info for both kernel and bpf-side | |
19 | * (limited subset). | |
20 | * | |
21 | * For now, direct access is only safe while running in NAPI/softirq | |
22 | * context. Contents is read-mostly and must not be updated during | |
23 | * driver NAPI/softirq poll. | |
24 | * | |
25 | * The driver usage API is a register and unregister API. | |
26 | * | |
27 | * The struct is not directly tied to the XDP prog. A new XDP prog | |
28 | * can be attached as long as it doesn't change the underlying | |
29 | * RX-ring. If the RX-ring does change significantly, the NIC driver | |
30 | * naturally need to stop the RX-ring before purging and reallocating | |
31 | * memory. In that process the driver MUST call unregistor (which | |
32 | * also apply for driver shutdown and unload). The register API is | |
33 | * also mandatory during RX-ring setup. | |
34 | */ | |
35 | ||
5ab073ff JDB |
36 | enum xdp_mem_type { |
37 | MEM_TYPE_PAGE_SHARED = 0, /* Split-page refcnt based model */ | |
38 | MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */ | |
57d0a1c1 | 39 | MEM_TYPE_PAGE_POOL, |
02b55e56 | 40 | MEM_TYPE_ZERO_COPY, |
5ab073ff JDB |
41 | MEM_TYPE_MAX, |
42 | }; | |
43 | ||
42b33468 | 44 | /* XDP flags for ndo_xdp_xmit */ |
42b33468 JDB |
45 | #define XDP_XMIT_FLUSH (1U << 0) /* doorbell signal consumer */ |
46 | #define XDP_XMIT_FLAGS_MASK XDP_XMIT_FLUSH | |
47 | ||
5ab073ff JDB |
48 | struct xdp_mem_info { |
49 | u32 type; /* enum xdp_mem_type, but known size type */ | |
8d5d8852 | 50 | u32 id; |
5ab073ff JDB |
51 | }; |
52 | ||
57d0a1c1 JDB |
53 | struct page_pool; |
54 | ||
02b55e56 BT |
55 | struct zero_copy_allocator { |
56 | void (*free)(struct zero_copy_allocator *zca, unsigned long handle); | |
57 | }; | |
58 | ||
aecd67b6 JDB |
59 | struct xdp_rxq_info { |
60 | struct net_device *dev; | |
61 | u32 queue_index; | |
62 | u32 reg_state; | |
5ab073ff | 63 | struct xdp_mem_info mem; |
aecd67b6 JDB |
64 | } ____cacheline_aligned; /* perf critical, avoid false-sharing */ |
65 | ||
106ca27f JDB |
66 | struct xdp_buff { |
67 | void *data; | |
68 | void *data_end; | |
69 | void *data_meta; | |
70 | void *data_hard_start; | |
02b55e56 | 71 | unsigned long handle; |
106ca27f JDB |
72 | struct xdp_rxq_info *rxq; |
73 | }; | |
5ab073ff | 74 | |
c0048cff JDB |
75 | struct xdp_frame { |
76 | void *data; | |
77 | u16 len; | |
78 | u16 headroom; | |
79 | u16 metasize; | |
80 | /* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time, | |
81 | * while mem info is valid on remote CPU. | |
82 | */ | |
83 | struct xdp_mem_info mem; | |
70280ed9 | 84 | struct net_device *dev_rx; /* used by cpumap */ |
c0048cff JDB |
85 | }; |
86 | ||
a8d5b4ab TM |
87 | /* Clear kernel pointers in xdp_frame */ |
88 | static inline void xdp_scrub_frame(struct xdp_frame *frame) | |
89 | { | |
90 | frame->data = NULL; | |
91 | frame->dev_rx = NULL; | |
92 | } | |
93 | ||
b0d1beef BT |
94 | struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp); |
95 | ||
c0048cff JDB |
96 | /* Convert xdp_buff to xdp_frame */ |
97 | static inline | |
98 | struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp) | |
99 | { | |
100 | struct xdp_frame *xdp_frame; | |
101 | int metasize; | |
102 | int headroom; | |
103 | ||
02b55e56 | 104 | if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) |
b0d1beef | 105 | return xdp_convert_zc_to_xdp_frame(xdp); |
02b55e56 | 106 | |
c0048cff JDB |
107 | /* Assure headroom is available for storing info */ |
108 | headroom = xdp->data - xdp->data_hard_start; | |
109 | metasize = xdp->data - xdp->data_meta; | |
110 | metasize = metasize > 0 ? metasize : 0; | |
111 | if (unlikely((headroom - metasize) < sizeof(*xdp_frame))) | |
112 | return NULL; | |
113 | ||
114 | /* Store info in top of packet */ | |
115 | xdp_frame = xdp->data_hard_start; | |
116 | ||
117 | xdp_frame->data = xdp->data; | |
118 | xdp_frame->len = xdp->data_end - xdp->data; | |
119 | xdp_frame->headroom = headroom - sizeof(*xdp_frame); | |
120 | xdp_frame->metasize = metasize; | |
121 | ||
122 | /* rxq only valid until napi_schedule ends, convert to xdp_mem_info */ | |
123 | xdp_frame->mem = xdp->rxq->mem; | |
124 | ||
125 | return xdp_frame; | |
126 | } | |
127 | ||
03993094 | 128 | void xdp_return_frame(struct xdp_frame *xdpf); |
389ab7f0 | 129 | void xdp_return_frame_rx_napi(struct xdp_frame *xdpf); |
c497176c | 130 | void xdp_return_buff(struct xdp_buff *xdp); |
5ab073ff | 131 | |
6bf071bf JDB |
132 | /* When sending xdp_frame into the network stack, then there is no |
133 | * return point callback, which is needed to release e.g. DMA-mapping | |
134 | * resources with page_pool. Thus, have explicit function to release | |
135 | * frame resources. | |
136 | */ | |
137 | void __xdp_release_frame(void *data, struct xdp_mem_info *mem); | |
138 | static inline void xdp_release_frame(struct xdp_frame *xdpf) | |
139 | { | |
140 | struct xdp_mem_info *mem = &xdpf->mem; | |
141 | ||
142 | /* Curr only page_pool needs this */ | |
143 | if (mem->type == MEM_TYPE_PAGE_POOL) | |
144 | __xdp_release_frame(xdpf->data, mem); | |
145 | } | |
146 | ||
aecd67b6 JDB |
147 | int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, |
148 | struct net_device *dev, u32 queue_index); | |
149 | void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq); | |
150 | void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq); | |
c0124f32 | 151 | bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq); |
5ab073ff JDB |
152 | int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq, |
153 | enum xdp_mem_type type, void *allocator); | |
dce5bd61 | 154 | void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq); |
aecd67b6 | 155 | |
106ca27f JDB |
156 | /* Drivers not supporting XDP metadata can use this helper, which |
157 | * rejects any room expansion for metadata as a result. | |
158 | */ | |
159 | static __always_inline void | |
160 | xdp_set_data_meta_invalid(struct xdp_buff *xdp) | |
161 | { | |
162 | xdp->data_meta = xdp->data + 1; | |
163 | } | |
164 | ||
165 | static __always_inline bool | |
166 | xdp_data_meta_unsupported(const struct xdp_buff *xdp) | |
167 | { | |
168 | return unlikely(xdp->data_meta > xdp->data); | |
169 | } | |
170 | ||
05296620 JK |
171 | struct xdp_attachment_info { |
172 | struct bpf_prog *prog; | |
173 | u32 flags; | |
174 | }; | |
175 | ||
176 | struct netdev_bpf; | |
177 | int xdp_attachment_query(struct xdp_attachment_info *info, | |
178 | struct netdev_bpf *bpf); | |
179 | bool xdp_attachment_flags_ok(struct xdp_attachment_info *info, | |
180 | struct netdev_bpf *bpf); | |
181 | void xdp_attachment_setup(struct xdp_attachment_info *info, | |
182 | struct netdev_bpf *bpf); | |
183 | ||
aecd67b6 | 184 | #endif /* __LINUX_NET_XDP_H__ */ |