]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | /* SPDX-License-Identifier: BSD-3-Clause |
2 | * Copyright(c) 2010-2014 Intel Corporation. | |
3 | * Copyright 2014 6WIND S.A. | |
7c673cae FG |
4 | */ |
5 | ||
6 | #ifndef _RTE_MBUF_H_ | |
7 | #define _RTE_MBUF_H_ | |
8 | ||
9 | /** | |
10 | * @file | |
11 | * RTE Mbuf | |
12 | * | |
13 | * The mbuf library provides the ability to create and destroy buffers | |
14 | * that may be used by the RTE application to store message | |
15 | * buffers. The message buffers are stored in a mempool, using the | |
16 | * RTE mempool library. | |
17 | * | |
11fdf7f2 TL |
18 | * The preferred way to create a mbuf pool is to use |
19 | * rte_pktmbuf_pool_create(). However, in some situations, an | |
20 | * application may want to have more control (ex: populate the pool with | |
21 | * specific memory), in this case it is possible to use functions from | |
22 | * rte_mempool. See how rte_pktmbuf_pool_create() is implemented for | |
23 | * details. | |
24 | * | |
7c673cae FG |
25 | * This library provides an API to allocate/free packet mbufs, which are |
26 | * used to carry network packets. | |
27 | * | |
28 | * To understand the concepts of packet buffers or mbufs, you | |
29 | * should read "TCP/IP Illustrated, Volume 2: The Implementation, | |
30 | * Addison-Wesley, 1995, ISBN 0-201-63354-X from Richard Stevens" | |
31 | * http://www.kohala.com/start/tcpipiv2.html | |
32 | */ | |
33 | ||
34 | #include <stdint.h> | |
11fdf7f2 | 35 | #include <rte_compat.h> |
7c673cae | 36 | #include <rte_common.h> |
11fdf7f2 | 37 | #include <rte_config.h> |
7c673cae FG |
38 | #include <rte_mempool.h> |
39 | #include <rte_memory.h> | |
40 | #include <rte_atomic.h> | |
41 | #include <rte_prefetch.h> | |
42 | #include <rte_branch_prediction.h> | |
9f95a23c | 43 | #include <rte_byteorder.h> |
7c673cae FG |
44 | #include <rte_mbuf_ptype.h> |
45 | ||
46 | #ifdef __cplusplus | |
47 | extern "C" { | |
48 | #endif | |
49 | ||
50 | /* | |
51 | * Packet Offload Features Flags. It also carry packet type information. | |
52 | * Critical resources. Both rx/tx shared these bits. Be cautious on any change | |
53 | * | |
54 | * - RX flags start at bit position zero, and get added to the left of previous | |
55 | * flags. | |
56 | * - The most-significant 3 bits are reserved for generic mbuf flags | |
57 | * - TX flags therefore start at bit position 60 (i.e. 63-3), and new flags get | |
58 | * added to the right of the previously defined flags i.e. they should count | |
59 | * downwards, not upwards. | |
60 | * | |
61 | * Keep these flags synchronized with rte_get_rx_ol_flag_name() and | |
62 | * rte_get_tx_ol_flag_name(). | |
63 | */ | |
64 | ||
65 | /** | |
11fdf7f2 TL |
66 | * The RX packet is a 802.1q VLAN packet, and the tci has been |
67 | * saved in in mbuf->vlan_tci. | |
68 | * If the flag PKT_RX_VLAN_STRIPPED is also present, the VLAN | |
69 | * header has been stripped from mbuf data, else it is still | |
70 | * present. | |
7c673cae | 71 | */ |
11fdf7f2 | 72 | #define PKT_RX_VLAN (1ULL << 0) |
7c673cae FG |
73 | |
74 | #define PKT_RX_RSS_HASH (1ULL << 1) /**< RX packet with RSS hash result. */ | |
75 | #define PKT_RX_FDIR (1ULL << 2) /**< RX packet with FDIR match indicate. */ | |
76 | ||
77 | /** | |
78 | * Deprecated. | |
79 | * Checking this flag alone is deprecated: check the 2 bits of | |
80 | * PKT_RX_L4_CKSUM_MASK. | |
81 | * This flag was set when the L4 checksum of a packet was detected as | |
82 | * wrong by the hardware. | |
83 | */ | |
84 | #define PKT_RX_L4_CKSUM_BAD (1ULL << 3) | |
85 | ||
86 | /** | |
87 | * Deprecated. | |
88 | * Checking this flag alone is deprecated: check the 2 bits of | |
89 | * PKT_RX_IP_CKSUM_MASK. | |
90 | * This flag was set when the IP checksum of a packet was detected as | |
91 | * wrong by the hardware. | |
92 | */ | |
93 | #define PKT_RX_IP_CKSUM_BAD (1ULL << 4) | |
94 | ||
95 | #define PKT_RX_EIP_CKSUM_BAD (1ULL << 5) /**< External IP header checksum error. */ | |
96 | ||
97 | /** | |
98 | * A vlan has been stripped by the hardware and its tci is saved in | |
99 | * mbuf->vlan_tci. This can only happen if vlan stripping is enabled | |
100 | * in the RX configuration of the PMD. | |
11fdf7f2 | 101 | * When PKT_RX_VLAN_STRIPPED is set, PKT_RX_VLAN must also be set. |
7c673cae FG |
102 | */ |
103 | #define PKT_RX_VLAN_STRIPPED (1ULL << 6) | |
104 | ||
105 | /** | |
106 | * Mask of bits used to determine the status of RX IP checksum. | |
107 | * - PKT_RX_IP_CKSUM_UNKNOWN: no information about the RX IP checksum | |
108 | * - PKT_RX_IP_CKSUM_BAD: the IP checksum in the packet is wrong | |
109 | * - PKT_RX_IP_CKSUM_GOOD: the IP checksum in the packet is valid | |
110 | * - PKT_RX_IP_CKSUM_NONE: the IP checksum is not correct in the packet | |
111 | * data, but the integrity of the IP header is verified. | |
112 | */ | |
113 | #define PKT_RX_IP_CKSUM_MASK ((1ULL << 4) | (1ULL << 7)) | |
114 | ||
115 | #define PKT_RX_IP_CKSUM_UNKNOWN 0 | |
116 | #define PKT_RX_IP_CKSUM_BAD (1ULL << 4) | |
117 | #define PKT_RX_IP_CKSUM_GOOD (1ULL << 7) | |
118 | #define PKT_RX_IP_CKSUM_NONE ((1ULL << 4) | (1ULL << 7)) | |
119 | ||
120 | /** | |
121 | * Mask of bits used to determine the status of RX L4 checksum. | |
122 | * - PKT_RX_L4_CKSUM_UNKNOWN: no information about the RX L4 checksum | |
123 | * - PKT_RX_L4_CKSUM_BAD: the L4 checksum in the packet is wrong | |
124 | * - PKT_RX_L4_CKSUM_GOOD: the L4 checksum in the packet is valid | |
125 | * - PKT_RX_L4_CKSUM_NONE: the L4 checksum is not correct in the packet | |
126 | * data, but the integrity of the L4 data is verified. | |
127 | */ | |
128 | #define PKT_RX_L4_CKSUM_MASK ((1ULL << 3) | (1ULL << 8)) | |
129 | ||
130 | #define PKT_RX_L4_CKSUM_UNKNOWN 0 | |
131 | #define PKT_RX_L4_CKSUM_BAD (1ULL << 3) | |
132 | #define PKT_RX_L4_CKSUM_GOOD (1ULL << 8) | |
133 | #define PKT_RX_L4_CKSUM_NONE ((1ULL << 3) | (1ULL << 8)) | |
134 | ||
135 | #define PKT_RX_IEEE1588_PTP (1ULL << 9) /**< RX IEEE1588 L2 Ethernet PT Packet. */ | |
136 | #define PKT_RX_IEEE1588_TMST (1ULL << 10) /**< RX IEEE1588 L2/L4 timestamped packet.*/ | |
137 | #define PKT_RX_FDIR_ID (1ULL << 13) /**< FD id reported if FDIR match. */ | |
138 | #define PKT_RX_FDIR_FLX (1ULL << 14) /**< Flexible bytes reported if FDIR match. */ | |
139 | ||
140 | /** | |
141 | * The 2 vlans have been stripped by the hardware and their tci are | |
142 | * saved in mbuf->vlan_tci (inner) and mbuf->vlan_tci_outer (outer). | |
143 | * This can only happen if vlan stripping is enabled in the RX | |
9f95a23c | 144 | * configuration of the PMD. |
11fdf7f2 TL |
145 | * When PKT_RX_QINQ_STRIPPED is set, the flags (PKT_RX_VLAN | |
146 | * PKT_RX_VLAN_STRIPPED | PKT_RX_QINQ) must also be set. | |
7c673cae FG |
147 | */ |
148 | #define PKT_RX_QINQ_STRIPPED (1ULL << 15) | |
149 | ||
7c673cae FG |
150 | /** |
151 | * When packets are coalesced by a hardware or virtual driver, this flag | |
152 | * can be set in the RX mbuf, meaning that the m->tso_segsz field is | |
153 | * valid and is set to the segment size of original packets. | |
154 | */ | |
155 | #define PKT_RX_LRO (1ULL << 16) | |
156 | ||
11fdf7f2 TL |
157 | /** |
158 | * Indicate that the timestamp field in the mbuf is valid. | |
159 | */ | |
160 | #define PKT_RX_TIMESTAMP (1ULL << 17) | |
161 | ||
162 | /** | |
163 | * Indicate that security offload processing was applied on the RX packet. | |
164 | */ | |
165 | #define PKT_RX_SEC_OFFLOAD (1ULL << 18) | |
166 | ||
167 | /** | |
168 | * Indicate that security offload processing failed on the RX packet. | |
169 | */ | |
170 | #define PKT_RX_SEC_OFFLOAD_FAILED (1ULL << 19) | |
171 | ||
172 | /** | |
173 | * The RX packet is a double VLAN, and the outer tci has been | |
9f95a23c TL |
174 | * saved in in mbuf->vlan_tci_outer. If PKT_RX_QINQ set, PKT_RX_VLAN |
175 | * also should be set and inner tci should be saved to mbuf->vlan_tci. | |
11fdf7f2 TL |
176 | * If the flag PKT_RX_QINQ_STRIPPED is also present, both VLANs |
177 | * headers have been stripped from mbuf data, else they are still | |
178 | * present. | |
179 | */ | |
180 | #define PKT_RX_QINQ (1ULL << 20) | |
181 | ||
9f95a23c TL |
182 | /** |
183 | * Mask of bits used to determine the status of outer RX L4 checksum. | |
184 | * - PKT_RX_OUTER_L4_CKSUM_UNKNOWN: no info about the outer RX L4 checksum | |
185 | * - PKT_RX_OUTER_L4_CKSUM_BAD: the outer L4 checksum in the packet is wrong | |
186 | * - PKT_RX_OUTER_L4_CKSUM_GOOD: the outer L4 checksum in the packet is valid | |
187 | * - PKT_RX_OUTER_L4_CKSUM_INVALID: invalid outer L4 checksum state. | |
188 | * | |
189 | * The detection of PKT_RX_OUTER_L4_CKSUM_GOOD shall be based on the given | |
190 | * HW capability, At minimum, the PMD should support | |
191 | * PKT_RX_OUTER_L4_CKSUM_UNKNOWN and PKT_RX_OUTER_L4_CKSUM_BAD states | |
192 | * if the DEV_RX_OFFLOAD_OUTER_UDP_CKSUM offload is available. | |
193 | */ | |
194 | #define PKT_RX_OUTER_L4_CKSUM_MASK ((1ULL << 21) | (1ULL << 22)) | |
195 | ||
196 | #define PKT_RX_OUTER_L4_CKSUM_UNKNOWN 0 | |
197 | #define PKT_RX_OUTER_L4_CKSUM_BAD (1ULL << 21) | |
198 | #define PKT_RX_OUTER_L4_CKSUM_GOOD (1ULL << 22) | |
199 | #define PKT_RX_OUTER_L4_CKSUM_INVALID ((1ULL << 21) | (1ULL << 22)) | |
200 | ||
7c673cae FG |
201 | /* add new RX flags here */ |
202 | ||
203 | /* add new TX flags here */ | |
204 | ||
9f95a23c TL |
205 | /** |
206 | * Indicate that the metadata field in the mbuf is in use. | |
207 | */ | |
208 | #define PKT_TX_METADATA (1ULL << 40) | |
209 | ||
210 | /** | |
211 | * Outer UDP checksum offload flag. This flag is used for enabling | |
212 | * outer UDP checksum in PMD. To use outer UDP checksum, the user needs to | |
213 | * 1) Enable the following in mbuf, | |
214 | * a) Fill outer_l2_len and outer_l3_len in mbuf. | |
215 | * b) Set the PKT_TX_OUTER_UDP_CKSUM flag. | |
216 | * c) Set the PKT_TX_OUTER_IPV4 or PKT_TX_OUTER_IPV6 flag. | |
217 | * 2) Configure DEV_TX_OFFLOAD_OUTER_UDP_CKSUM offload flag. | |
218 | */ | |
219 | #define PKT_TX_OUTER_UDP_CKSUM (1ULL << 41) | |
220 | ||
11fdf7f2 TL |
221 | /** |
222 | * UDP Fragmentation Offload flag. This flag is used for enabling UDP | |
223 | * fragmentation in SW or in HW. When use UFO, mbuf->tso_segsz is used | |
224 | * to store the MSS of UDP fragments. | |
225 | */ | |
226 | #define PKT_TX_UDP_SEG (1ULL << 42) | |
227 | ||
228 | /** | |
229 | * Request security offload processing on the TX packet. | |
230 | */ | |
231 | #define PKT_TX_SEC_OFFLOAD (1ULL << 43) | |
232 | ||
233 | /** | |
234 | * Offload the MACsec. This flag must be set by the application to enable | |
235 | * this offload feature for a packet to be transmitted. | |
236 | */ | |
237 | #define PKT_TX_MACSEC (1ULL << 44) | |
238 | ||
7c673cae FG |
239 | /** |
240 | * Bits 45:48 used for the tunnel type. | |
11fdf7f2 TL |
241 | * The tunnel type must be specified for TSO or checksum on the inner part |
242 | * of tunnel packets. | |
243 | * These flags can be used with PKT_TX_TCP_SEG for TSO, or PKT_TX_xxx_CKSUM. | |
244 | * The mbuf fields for inner and outer header lengths are required: | |
245 | * outer_l2_len, outer_l3_len, l2_len, l3_len, l4_len and tso_segsz for TSO. | |
7c673cae FG |
246 | */ |
247 | #define PKT_TX_TUNNEL_VXLAN (0x1ULL << 45) | |
248 | #define PKT_TX_TUNNEL_GRE (0x2ULL << 45) | |
249 | #define PKT_TX_TUNNEL_IPIP (0x3ULL << 45) | |
250 | #define PKT_TX_TUNNEL_GENEVE (0x4ULL << 45) | |
11fdf7f2 TL |
251 | /** TX packet with MPLS-in-UDP RFC 7510 header. */ |
252 | #define PKT_TX_TUNNEL_MPLSINUDP (0x5ULL << 45) | |
253 | #define PKT_TX_TUNNEL_VXLAN_GPE (0x6ULL << 45) | |
254 | /** | |
255 | * Generic IP encapsulated tunnel type, used for TSO and checksum offload. | |
256 | * It can be used for tunnels which are not standards or listed above. | |
257 | * It is preferred to use specific tunnel flags like PKT_TX_TUNNEL_GRE | |
258 | * or PKT_TX_TUNNEL_IPIP if possible. | |
259 | * The ethdev must be configured with DEV_TX_OFFLOAD_IP_TNL_TSO. | |
260 | * Outer and inner checksums are done according to the existing flags like | |
261 | * PKT_TX_xxx_CKSUM. | |
262 | * Specific tunnel headers that contain payload length, sequence id | |
263 | * or checksum are not expected to be updated. | |
264 | */ | |
265 | #define PKT_TX_TUNNEL_IP (0xDULL << 45) | |
266 | /** | |
267 | * Generic UDP encapsulated tunnel type, used for TSO and checksum offload. | |
268 | * UDP tunnel type implies outer IP layer. | |
269 | * It can be used for tunnels which are not standards or listed above. | |
270 | * It is preferred to use specific tunnel flags like PKT_TX_TUNNEL_VXLAN | |
271 | * if possible. | |
272 | * The ethdev must be configured with DEV_TX_OFFLOAD_UDP_TNL_TSO. | |
273 | * Outer and inner checksums are done according to the existing flags like | |
274 | * PKT_TX_xxx_CKSUM. | |
275 | * Specific tunnel headers that contain payload length, sequence id | |
276 | * or checksum are not expected to be updated. | |
277 | */ | |
278 | #define PKT_TX_TUNNEL_UDP (0xEULL << 45) | |
7c673cae FG |
279 | /* add new TX TUNNEL type here */ |
280 | #define PKT_TX_TUNNEL_MASK (0xFULL << 45) | |
281 | ||
282 | /** | |
9f95a23c TL |
283 | * Double VLAN insertion (QinQ) request to driver, driver may offload the |
284 | * insertion based on device capability. | |
285 | * mbuf 'vlan_tci' & 'vlan_tci_outer' must be valid when this flag is set. | |
7c673cae | 286 | */ |
9f95a23c | 287 | #define PKT_TX_QINQ (1ULL << 49) |
11fdf7f2 TL |
288 | /* this old name is deprecated */ |
289 | #define PKT_TX_QINQ_PKT PKT_TX_QINQ | |
7c673cae FG |
290 | |
291 | /** | |
292 | * TCP segmentation offload. To enable this offload feature for a | |
293 | * packet to be transmitted on hardware supporting TSO: | |
294 | * - set the PKT_TX_TCP_SEG flag in mbuf->ol_flags (this flag implies | |
295 | * PKT_TX_TCP_CKSUM) | |
296 | * - set the flag PKT_TX_IPV4 or PKT_TX_IPV6 | |
11fdf7f2 | 297 | * - if it's IPv4, set the PKT_TX_IP_CKSUM flag |
7c673cae | 298 | * - fill the mbuf offload information: l2_len, l3_len, l4_len, tso_segsz |
7c673cae FG |
299 | */ |
300 | #define PKT_TX_TCP_SEG (1ULL << 50) | |
301 | ||
302 | #define PKT_TX_IEEE1588_TMST (1ULL << 51) /**< TX IEEE1588 packet to timestamp. */ | |
303 | ||
304 | /** | |
305 | * Bits 52+53 used for L4 packet type with checksum enabled: 00: Reserved, | |
306 | * 01: TCP checksum, 10: SCTP checksum, 11: UDP checksum. To use hardware | |
307 | * L4 checksum offload, the user needs to: | |
308 | * - fill l2_len and l3_len in mbuf | |
309 | * - set the flags PKT_TX_TCP_CKSUM, PKT_TX_SCTP_CKSUM or PKT_TX_UDP_CKSUM | |
310 | * - set the flag PKT_TX_IPV4 or PKT_TX_IPV6 | |
7c673cae FG |
311 | */ |
312 | #define PKT_TX_L4_NO_CKSUM (0ULL << 52) /**< Disable L4 cksum of TX pkt. */ | |
313 | #define PKT_TX_TCP_CKSUM (1ULL << 52) /**< TCP cksum of TX pkt. computed by NIC. */ | |
314 | #define PKT_TX_SCTP_CKSUM (2ULL << 52) /**< SCTP cksum of TX pkt. computed by NIC. */ | |
315 | #define PKT_TX_UDP_CKSUM (3ULL << 52) /**< UDP cksum of TX pkt. computed by NIC. */ | |
316 | #define PKT_TX_L4_MASK (3ULL << 52) /**< Mask for L4 cksum offload request. */ | |
317 | ||
318 | /** | |
319 | * Offload the IP checksum in the hardware. The flag PKT_TX_IPV4 should | |
320 | * also be set by the application, although a PMD will only check | |
321 | * PKT_TX_IP_CKSUM. | |
7c673cae FG |
322 | * - fill the mbuf offload information: l2_len, l3_len |
323 | */ | |
324 | #define PKT_TX_IP_CKSUM (1ULL << 54) | |
325 | ||
326 | /** | |
327 | * Packet is IPv4. This flag must be set when using any offload feature | |
328 | * (TSO, L3 or L4 checksum) to tell the NIC that the packet is an IPv4 | |
329 | * packet. If the packet is a tunneled packet, this flag is related to | |
330 | * the inner headers. | |
331 | */ | |
332 | #define PKT_TX_IPV4 (1ULL << 55) | |
333 | ||
334 | /** | |
335 | * Packet is IPv6. This flag must be set when using an offload feature | |
336 | * (TSO or L4 checksum) to tell the NIC that the packet is an IPv6 | |
337 | * packet. If the packet is a tunneled packet, this flag is related to | |
338 | * the inner headers. | |
339 | */ | |
340 | #define PKT_TX_IPV6 (1ULL << 56) | |
341 | ||
11fdf7f2 | 342 | /** |
9f95a23c TL |
343 | * VLAN tag insertion request to driver, driver may offload the insertion |
344 | * based on the device capability. | |
345 | * mbuf 'vlan_tci' field must be valid when this flag is set. | |
11fdf7f2 TL |
346 | */ |
347 | #define PKT_TX_VLAN (1ULL << 57) | |
348 | /* this old name is deprecated */ | |
349 | #define PKT_TX_VLAN_PKT PKT_TX_VLAN | |
7c673cae FG |
350 | |
351 | /** | |
352 | * Offload the IP checksum of an external header in the hardware. The | |
11fdf7f2 TL |
353 | * flag PKT_TX_OUTER_IPV4 should also be set by the application, although |
354 | * a PMD will only check PKT_TX_OUTER_IP_CKSUM. | |
7c673cae FG |
355 | * - fill the mbuf offload information: outer_l2_len, outer_l3_len |
356 | */ | |
357 | #define PKT_TX_OUTER_IP_CKSUM (1ULL << 58) | |
358 | ||
359 | /** | |
360 | * Packet outer header is IPv4. This flag must be set when using any | |
361 | * outer offload feature (L3 or L4 checksum) to tell the NIC that the | |
362 | * outer header of the tunneled packet is an IPv4 packet. | |
363 | */ | |
364 | #define PKT_TX_OUTER_IPV4 (1ULL << 59) | |
365 | ||
366 | /** | |
367 | * Packet outer header is IPv6. This flag must be set when using any | |
368 | * outer offload feature (L4 checksum) to tell the NIC that the outer | |
369 | * header of the tunneled packet is an IPv6 packet. | |
370 | */ | |
371 | #define PKT_TX_OUTER_IPV6 (1ULL << 60) | |
372 | ||
11fdf7f2 TL |
373 | /** |
374 | * Bitmask of all supported packet Tx offload features flags, | |
375 | * which can be set for packet. | |
376 | */ | |
377 | #define PKT_TX_OFFLOAD_MASK ( \ | |
9f95a23c TL |
378 | PKT_TX_OUTER_IPV6 | \ |
379 | PKT_TX_OUTER_IPV4 | \ | |
380 | PKT_TX_OUTER_IP_CKSUM | \ | |
381 | PKT_TX_VLAN_PKT | \ | |
382 | PKT_TX_IPV6 | \ | |
383 | PKT_TX_IPV4 | \ | |
11fdf7f2 TL |
384 | PKT_TX_IP_CKSUM | \ |
385 | PKT_TX_L4_MASK | \ | |
11fdf7f2 | 386 | PKT_TX_IEEE1588_TMST | \ |
9f95a23c | 387 | PKT_TX_TCP_SEG | \ |
11fdf7f2 | 388 | PKT_TX_QINQ_PKT | \ |
11fdf7f2 TL |
389 | PKT_TX_TUNNEL_MASK | \ |
390 | PKT_TX_MACSEC | \ | |
9f95a23c TL |
391 | PKT_TX_SEC_OFFLOAD | \ |
392 | PKT_TX_UDP_SEG | \ | |
393 | PKT_TX_OUTER_UDP_CKSUM | \ | |
394 | PKT_TX_METADATA) | |
7c673cae | 395 | |
11fdf7f2 TL |
396 | /** |
397 | * Mbuf having an external buffer attached. shinfo in mbuf must be filled. | |
398 | */ | |
399 | #define EXT_ATTACHED_MBUF (1ULL << 61) | |
7c673cae | 400 | |
11fdf7f2 | 401 | #define IND_ATTACHED_MBUF (1ULL << 62) /**< Indirect attached mbuf */ |
7c673cae FG |
402 | |
403 | /** Alignment constraint of mbuf private area. */ | |
404 | #define RTE_MBUF_PRIV_ALIGN 8 | |
405 | ||
406 | /** | |
407 | * Get the name of a RX offload flag | |
408 | * | |
409 | * @param mask | |
410 | * The mask describing the flag. | |
411 | * @return | |
412 | * The name of this flag, or NULL if it's not a valid RX flag. | |
413 | */ | |
414 | const char *rte_get_rx_ol_flag_name(uint64_t mask); | |
415 | ||
416 | /** | |
417 | * Dump the list of RX offload flags in a buffer | |
418 | * | |
419 | * @param mask | |
420 | * The mask describing the RX flags. | |
421 | * @param buf | |
422 | * The output buffer. | |
423 | * @param buflen | |
424 | * The length of the buffer. | |
425 | * @return | |
426 | * 0 on success, (-1) on error. | |
427 | */ | |
428 | int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen); | |
429 | ||
430 | /** | |
431 | * Get the name of a TX offload flag | |
432 | * | |
433 | * @param mask | |
434 | * The mask describing the flag. Usually only one bit must be set. | |
435 | * Several bits can be given if they belong to the same mask. | |
436 | * Ex: PKT_TX_L4_MASK. | |
437 | * @return | |
438 | * The name of this flag, or NULL if it's not a valid TX flag. | |
439 | */ | |
440 | const char *rte_get_tx_ol_flag_name(uint64_t mask); | |
441 | ||
442 | /** | |
443 | * Dump the list of TX offload flags in a buffer | |
444 | * | |
445 | * @param mask | |
446 | * The mask describing the TX flags. | |
447 | * @param buf | |
448 | * The output buffer. | |
449 | * @param buflen | |
450 | * The length of the buffer. | |
451 | * @return | |
452 | * 0 on success, (-1) on error. | |
453 | */ | |
454 | int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen); | |
455 | ||
456 | /** | |
457 | * Some NICs need at least 2KB buffer to RX standard Ethernet frame without | |
458 | * splitting it into multiple segments. | |
459 | * So, for mbufs that planned to be involved into RX/TX, the recommended | |
460 | * minimal buffer length is 2KB + RTE_PKTMBUF_HEADROOM. | |
461 | */ | |
462 | #define RTE_MBUF_DEFAULT_DATAROOM 2048 | |
463 | #define RTE_MBUF_DEFAULT_BUF_SIZE \ | |
464 | (RTE_MBUF_DEFAULT_DATAROOM + RTE_PKTMBUF_HEADROOM) | |
465 | ||
466 | /* define a set of marker types that can be used to refer to set points in the | |
467 | * mbuf */ | |
468 | __extension__ | |
469 | typedef void *MARKER[0]; /**< generic marker for a point in a structure */ | |
470 | __extension__ | |
471 | typedef uint8_t MARKER8[0]; /**< generic marker with 1B alignment */ | |
472 | __extension__ | |
473 | typedef uint64_t MARKER64[0]; /**< marker that allows us to overwrite 8 bytes | |
474 | * with a single assignment */ | |
475 | ||
9f95a23c TL |
476 | struct rte_mbuf_sched { |
477 | uint32_t queue_id; /**< Queue ID. */ | |
478 | uint8_t traffic_class; | |
479 | /**< Traffic class ID. Traffic class 0 | |
480 | * is the highest priority traffic class. | |
481 | */ | |
482 | uint8_t color; | |
483 | /**< Color. @see enum rte_color.*/ | |
484 | uint16_t reserved; /**< Reserved. */ | |
485 | }; /**< Hierarchical scheduler */ | |
486 | ||
487 | /** | |
488 | * enum for the tx_offload bit-fields lengths and offsets. | |
489 | * defines the layout of rte_mbuf tx_offload field. | |
490 | */ | |
491 | enum { | |
492 | RTE_MBUF_L2_LEN_BITS = 7, | |
493 | RTE_MBUF_L3_LEN_BITS = 9, | |
494 | RTE_MBUF_L4_LEN_BITS = 8, | |
495 | RTE_MBUF_TSO_SEGSZ_BITS = 16, | |
496 | RTE_MBUF_OUTL3_LEN_BITS = 9, | |
497 | RTE_MBUF_OUTL2_LEN_BITS = 7, | |
498 | RTE_MBUF_TXOFLD_UNUSED_BITS = sizeof(uint64_t) * CHAR_BIT - | |
499 | RTE_MBUF_L2_LEN_BITS - | |
500 | RTE_MBUF_L3_LEN_BITS - | |
501 | RTE_MBUF_L4_LEN_BITS - | |
502 | RTE_MBUF_TSO_SEGSZ_BITS - | |
503 | RTE_MBUF_OUTL3_LEN_BITS - | |
504 | RTE_MBUF_OUTL2_LEN_BITS, | |
505 | #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN | |
506 | RTE_MBUF_L2_LEN_OFS = | |
507 | sizeof(uint64_t) * CHAR_BIT - RTE_MBUF_L2_LEN_BITS, | |
508 | RTE_MBUF_L3_LEN_OFS = RTE_MBUF_L2_LEN_OFS - RTE_MBUF_L3_LEN_BITS, | |
509 | RTE_MBUF_L4_LEN_OFS = RTE_MBUF_L3_LEN_OFS - RTE_MBUF_L4_LEN_BITS, | |
510 | RTE_MBUF_TSO_SEGSZ_OFS = RTE_MBUF_L4_LEN_OFS - RTE_MBUF_TSO_SEGSZ_BITS, | |
511 | RTE_MBUF_OUTL3_LEN_OFS = | |
512 | RTE_MBUF_TSO_SEGSZ_OFS - RTE_MBUF_OUTL3_LEN_BITS, | |
513 | RTE_MBUF_OUTL2_LEN_OFS = | |
514 | RTE_MBUF_OUTL3_LEN_OFS - RTE_MBUF_OUTL2_LEN_BITS, | |
515 | RTE_MBUF_TXOFLD_UNUSED_OFS = | |
516 | RTE_MBUF_OUTL2_LEN_OFS - RTE_MBUF_TXOFLD_UNUSED_BITS, | |
517 | #else | |
518 | RTE_MBUF_L2_LEN_OFS = 0, | |
519 | RTE_MBUF_L3_LEN_OFS = RTE_MBUF_L2_LEN_OFS + RTE_MBUF_L2_LEN_BITS, | |
520 | RTE_MBUF_L4_LEN_OFS = RTE_MBUF_L3_LEN_OFS + RTE_MBUF_L3_LEN_BITS, | |
521 | RTE_MBUF_TSO_SEGSZ_OFS = RTE_MBUF_L4_LEN_OFS + RTE_MBUF_L4_LEN_BITS, | |
522 | RTE_MBUF_OUTL3_LEN_OFS = | |
523 | RTE_MBUF_TSO_SEGSZ_OFS + RTE_MBUF_TSO_SEGSZ_BITS, | |
524 | RTE_MBUF_OUTL2_LEN_OFS = | |
525 | RTE_MBUF_OUTL3_LEN_OFS + RTE_MBUF_OUTL3_LEN_BITS, | |
526 | RTE_MBUF_TXOFLD_UNUSED_OFS = | |
527 | RTE_MBUF_OUTL2_LEN_OFS + RTE_MBUF_OUTL2_LEN_BITS, | |
528 | #endif | |
529 | }; | |
530 | ||
7c673cae FG |
531 | /** |
532 | * The generic rte_mbuf, containing a packet mbuf. | |
533 | */ | |
534 | struct rte_mbuf { | |
535 | MARKER cacheline0; | |
536 | ||
537 | void *buf_addr; /**< Virtual address of segment buffer. */ | |
11fdf7f2 TL |
538 | /** |
539 | * Physical address of segment buffer. | |
540 | * Force alignment to 8-bytes, so as to ensure we have the exact | |
541 | * same mbuf cacheline0 layout for 32-bit and 64-bit. This makes | |
542 | * working on vector drivers easier. | |
543 | */ | |
544 | RTE_STD_C11 | |
545 | union { | |
546 | rte_iova_t buf_iova; | |
547 | rte_iova_t buf_physaddr; /**< deprecated */ | |
548 | } __rte_aligned(sizeof(rte_iova_t)); | |
7c673cae | 549 | |
11fdf7f2 TL |
550 | /* next 8 bytes are initialised on RX descriptor rearm */ |
551 | MARKER64 rearm_data; | |
7c673cae FG |
552 | uint16_t data_off; |
553 | ||
554 | /** | |
11fdf7f2 TL |
555 | * Reference counter. Its size should at least equal to the size |
556 | * of port field (16 bits), to support zero-copy broadcast. | |
7c673cae FG |
557 | * It should only be accessed using the following functions: |
558 | * rte_mbuf_refcnt_update(), rte_mbuf_refcnt_read(), and | |
559 | * rte_mbuf_refcnt_set(). The functionality of these functions (atomic, | |
560 | * or non-atomic) is controlled by the CONFIG_RTE_MBUF_REFCNT_ATOMIC | |
561 | * config option. | |
562 | */ | |
563 | RTE_STD_C11 | |
564 | union { | |
565 | rte_atomic16_t refcnt_atomic; /**< Atomically accessed refcnt */ | |
566 | uint16_t refcnt; /**< Non-atomically accessed refcnt */ | |
567 | }; | |
11fdf7f2 TL |
568 | uint16_t nb_segs; /**< Number of segments. */ |
569 | ||
9f95a23c TL |
570 | /** Input port (16 bits to support more than 256 virtual ports). |
571 | * The event eth Tx adapter uses this field to specify the output port. | |
572 | */ | |
11fdf7f2 | 573 | uint16_t port; |
7c673cae FG |
574 | |
575 | uint64_t ol_flags; /**< Offload features. */ | |
576 | ||
577 | /* remaining bytes are set on RX when pulling packet from descriptor */ | |
578 | MARKER rx_descriptor_fields1; | |
579 | ||
580 | /* | |
581 | * The packet type, which is the combination of outer/inner L2, L3, L4 | |
582 | * and tunnel types. The packet_type is about data really present in the | |
583 | * mbuf. Example: if vlan stripping is enabled, a received vlan packet | |
584 | * would have RTE_PTYPE_L2_ETHER and not RTE_PTYPE_L2_VLAN because the | |
585 | * vlan is stripped from the data. | |
586 | */ | |
587 | RTE_STD_C11 | |
588 | union { | |
589 | uint32_t packet_type; /**< L2/L3/L4 and tunnel information. */ | |
590 | struct { | |
591 | uint32_t l2_type:4; /**< (Outer) L2 type. */ | |
592 | uint32_t l3_type:4; /**< (Outer) L3 type. */ | |
593 | uint32_t l4_type:4; /**< (Outer) L4 type. */ | |
594 | uint32_t tun_type:4; /**< Tunnel type. */ | |
11fdf7f2 TL |
595 | RTE_STD_C11 |
596 | union { | |
597 | uint8_t inner_esp_next_proto; | |
598 | /**< ESP next protocol type, valid if | |
599 | * RTE_PTYPE_TUNNEL_ESP tunnel type is set | |
600 | * on both Tx and Rx. | |
601 | */ | |
602 | __extension__ | |
603 | struct { | |
604 | uint8_t inner_l2_type:4; | |
605 | /**< Inner L2 type. */ | |
606 | uint8_t inner_l3_type:4; | |
607 | /**< Inner L3 type. */ | |
608 | }; | |
609 | }; | |
7c673cae FG |
610 | uint32_t inner_l4_type:4; /**< Inner L4 type. */ |
611 | }; | |
612 | }; | |
613 | ||
614 | uint32_t pkt_len; /**< Total pkt len: sum of all segments. */ | |
615 | uint16_t data_len; /**< Amount of data in segment buffer. */ | |
11fdf7f2 | 616 | /** VLAN TCI (CPU order), valid if PKT_RX_VLAN is set. */ |
7c673cae FG |
617 | uint16_t vlan_tci; |
618 | ||
9f95a23c | 619 | RTE_STD_C11 |
7c673cae | 620 | union { |
9f95a23c TL |
621 | union { |
622 | uint32_t rss; /**< RSS hash result if RSS enabled */ | |
623 | struct { | |
624 | union { | |
625 | struct { | |
626 | uint16_t hash; | |
627 | uint16_t id; | |
628 | }; | |
629 | uint32_t lo; | |
630 | /**< Second 4 flexible bytes */ | |
7c673cae | 631 | }; |
9f95a23c TL |
632 | uint32_t hi; |
633 | /**< First 4 flexible bytes or FD ID, dependent | |
634 | * on PKT_RX_FDIR_* flag in ol_flags. | |
635 | */ | |
636 | } fdir; /**< Filter identifier if FDIR enabled */ | |
637 | struct rte_mbuf_sched sched; | |
638 | /**< Hierarchical scheduler : 8 bytes */ | |
639 | struct { | |
640 | uint32_t reserved1; | |
641 | uint16_t reserved2; | |
642 | uint16_t txq; | |
643 | /**< The event eth Tx adapter uses this field | |
644 | * to store Tx queue id. | |
645 | * @see rte_event_eth_tx_adapter_txq_set() | |
646 | */ | |
647 | } txadapter; /**< Eventdev ethdev Tx adapter */ | |
648 | /**< User defined tags. See rte_distributor_process() */ | |
649 | uint32_t usr; | |
650 | } hash; /**< hash information */ | |
7c673cae | 651 | struct { |
9f95a23c TL |
652 | /** |
653 | * Application specific metadata value | |
654 | * for egress flow rule match. | |
655 | * Valid if PKT_TX_METADATA is set. | |
656 | * Located here to allow conjunct use | |
657 | * with hash.sched.hi. | |
658 | */ | |
659 | uint32_t tx_metadata; | |
660 | uint32_t reserved; | |
661 | }; | |
662 | }; | |
7c673cae | 663 | |
11fdf7f2 | 664 | /** Outer VLAN TCI (CPU order), valid if PKT_RX_QINQ is set. */ |
7c673cae FG |
665 | uint16_t vlan_tci_outer; |
666 | ||
11fdf7f2 TL |
667 | uint16_t buf_len; /**< Length of segment buffer. */ |
668 | ||
669 | /** Valid if PKT_RX_TIMESTAMP is set. The unit and time reference | |
670 | * are not normalized but are always the same for a given port. | |
671 | */ | |
672 | uint64_t timestamp; | |
673 | ||
7c673cae FG |
674 | /* second cache line - fields only used in slow path or on TX */ |
675 | MARKER cacheline1 __rte_cache_min_aligned; | |
676 | ||
677 | RTE_STD_C11 | |
678 | union { | |
679 | void *userdata; /**< Can be used for external metadata */ | |
680 | uint64_t udata64; /**< Allow 8-byte userdata on 32-bit */ | |
681 | }; | |
682 | ||
683 | struct rte_mempool *pool; /**< Pool from which mbuf was allocated. */ | |
684 | struct rte_mbuf *next; /**< Next segment of scattered packet. */ | |
685 | ||
686 | /* fields to support TX offloads */ | |
687 | RTE_STD_C11 | |
688 | union { | |
689 | uint64_t tx_offload; /**< combined for easy fetch */ | |
690 | __extension__ | |
691 | struct { | |
9f95a23c | 692 | uint64_t l2_len:RTE_MBUF_L2_LEN_BITS; |
7c673cae FG |
693 | /**< L2 (MAC) Header Length for non-tunneling pkt. |
694 | * Outer_L4_len + ... + Inner_L2_len for tunneling pkt. | |
695 | */ | |
9f95a23c TL |
696 | uint64_t l3_len:RTE_MBUF_L3_LEN_BITS; |
697 | /**< L3 (IP) Header Length. */ | |
698 | uint64_t l4_len:RTE_MBUF_L4_LEN_BITS; | |
699 | /**< L4 (TCP/UDP) Header Length. */ | |
700 | uint64_t tso_segsz:RTE_MBUF_TSO_SEGSZ_BITS; | |
701 | /**< TCP TSO segment size */ | |
7c673cae FG |
702 | |
703 | /* fields for TX offloading of tunnels */ | |
9f95a23c TL |
704 | uint64_t outer_l3_len:RTE_MBUF_OUTL3_LEN_BITS; |
705 | /**< Outer L3 (IP) Hdr Length. */ | |
706 | uint64_t outer_l2_len:RTE_MBUF_OUTL2_LEN_BITS; | |
707 | /**< Outer L2 (MAC) Hdr Length. */ | |
7c673cae | 708 | |
9f95a23c | 709 | /* uint64_t unused:RTE_MBUF_TXOFLD_UNUSED_BITS; */ |
7c673cae FG |
710 | }; |
711 | }; | |
712 | ||
713 | /** Size of the application private data. In case of an indirect | |
714 | * mbuf, it stores the direct mbuf private data size. */ | |
715 | uint16_t priv_size; | |
716 | ||
717 | /** Timesync flags for use with IEEE1588. */ | |
718 | uint16_t timesync; | |
11fdf7f2 TL |
719 | |
720 | /** Sequence number. See also rte_reorder_insert(). */ | |
721 | uint32_t seqn; | |
722 | ||
723 | /** Shared data for external buffer attached to mbuf. See | |
724 | * rte_pktmbuf_attach_extbuf(). | |
725 | */ | |
726 | struct rte_mbuf_ext_shared_info *shinfo; | |
727 | ||
7c673cae FG |
728 | } __rte_cache_aligned; |
729 | ||
11fdf7f2 TL |
730 | /** |
731 | * Function typedef of callback to free externally attached buffer. | |
732 | */ | |
733 | typedef void (*rte_mbuf_extbuf_free_callback_t)(void *addr, void *opaque); | |
734 | ||
735 | /** | |
736 | * Shared data at the end of an external buffer. | |
737 | */ | |
738 | struct rte_mbuf_ext_shared_info { | |
739 | rte_mbuf_extbuf_free_callback_t free_cb; /**< Free callback function */ | |
740 | void *fcb_opaque; /**< Free callback argument */ | |
741 | rte_atomic16_t refcnt_atomic; /**< Atomically accessed refcnt */ | |
742 | }; | |
743 | ||
744 | /**< Maximum number of nb_segs allowed. */ | |
745 | #define RTE_MBUF_MAX_NB_SEGS UINT16_MAX | |
746 | ||
7c673cae FG |
747 | /** |
748 | * Prefetch the first part of the mbuf | |
749 | * | |
750 | * The first 64 bytes of the mbuf corresponds to fields that are used early | |
751 | * in the receive path. If the cache line of the architecture is higher than | |
752 | * 64B, the second part will also be prefetched. | |
753 | * | |
754 | * @param m | |
755 | * The pointer to the mbuf. | |
756 | */ | |
757 | static inline void | |
758 | rte_mbuf_prefetch_part1(struct rte_mbuf *m) | |
759 | { | |
760 | rte_prefetch0(&m->cacheline0); | |
761 | } | |
762 | ||
763 | /** | |
764 | * Prefetch the second part of the mbuf | |
765 | * | |
766 | * The next 64 bytes of the mbuf corresponds to fields that are used in the | |
767 | * transmit path. If the cache line of the architecture is higher than 64B, | |
768 | * this function does nothing as it is expected that the full mbuf is | |
769 | * already in cache. | |
770 | * | |
771 | * @param m | |
772 | * The pointer to the mbuf. | |
773 | */ | |
774 | static inline void | |
775 | rte_mbuf_prefetch_part2(struct rte_mbuf *m) | |
776 | { | |
777 | #if RTE_CACHE_LINE_SIZE == 64 | |
778 | rte_prefetch0(&m->cacheline1); | |
779 | #else | |
780 | RTE_SET_USED(m); | |
781 | #endif | |
782 | } | |
783 | ||
784 | ||
785 | static inline uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp); | |
786 | ||
787 | /** | |
11fdf7f2 | 788 | * Return the IO address of the beginning of the mbuf data |
7c673cae FG |
789 | * |
790 | * @param mb | |
791 | * The pointer to the mbuf. | |
792 | * @return | |
11fdf7f2 | 793 | * The IO address of the beginning of the mbuf data |
7c673cae | 794 | */ |
11fdf7f2 TL |
795 | static inline rte_iova_t |
796 | rte_mbuf_data_iova(const struct rte_mbuf *mb) | |
797 | { | |
798 | return mb->buf_iova + mb->data_off; | |
799 | } | |
800 | ||
801 | __rte_deprecated | |
7c673cae FG |
802 | static inline phys_addr_t |
803 | rte_mbuf_data_dma_addr(const struct rte_mbuf *mb) | |
804 | { | |
11fdf7f2 | 805 | return rte_mbuf_data_iova(mb); |
7c673cae FG |
806 | } |
807 | ||
808 | /** | |
11fdf7f2 | 809 | * Return the default IO address of the beginning of the mbuf data |
7c673cae FG |
810 | * |
811 | * This function is used by drivers in their receive function, as it | |
812 | * returns the location where data should be written by the NIC, taking | |
813 | * the default headroom in account. | |
814 | * | |
815 | * @param mb | |
816 | * The pointer to the mbuf. | |
817 | * @return | |
11fdf7f2 | 818 | * The IO address of the beginning of the mbuf data |
7c673cae | 819 | */ |
11fdf7f2 TL |
820 | static inline rte_iova_t |
821 | rte_mbuf_data_iova_default(const struct rte_mbuf *mb) | |
822 | { | |
823 | return mb->buf_iova + RTE_PKTMBUF_HEADROOM; | |
824 | } | |
825 | ||
826 | __rte_deprecated | |
7c673cae FG |
827 | static inline phys_addr_t |
828 | rte_mbuf_data_dma_addr_default(const struct rte_mbuf *mb) | |
829 | { | |
11fdf7f2 | 830 | return rte_mbuf_data_iova_default(mb); |
7c673cae FG |
831 | } |
832 | ||
833 | /** | |
834 | * Return the mbuf owning the data buffer address of an indirect mbuf. | |
835 | * | |
836 | * @param mi | |
837 | * The pointer to the indirect mbuf. | |
838 | * @return | |
839 | * The address of the direct mbuf corresponding to buffer_addr. | |
840 | */ | |
841 | static inline struct rte_mbuf * | |
842 | rte_mbuf_from_indirect(struct rte_mbuf *mi) | |
843 | { | |
844 | return (struct rte_mbuf *)RTE_PTR_SUB(mi->buf_addr, sizeof(*mi) + mi->priv_size); | |
845 | } | |
846 | ||
847 | /** | |
9f95a23c TL |
848 | * Return address of buffer embedded in the given mbuf. |
849 | * | |
850 | * The return value shall be same as mb->buf_addr if the mbuf is already | |
851 | * initialized and direct. However, this API is useful if mempool of the | |
852 | * mbuf is already known because it doesn't need to access mbuf contents in | |
853 | * order to get the mempool pointer. | |
854 | * | |
855 | * @warning | |
856 | * @b EXPERIMENTAL: This API may change without prior notice. | |
857 | * This will be used by rte_mbuf_to_baddr() which has redundant code once | |
858 | * experimental tag is removed. | |
859 | * | |
860 | * @param mb | |
861 | * The pointer to the mbuf. | |
862 | * @param mp | |
863 | * The pointer to the mempool of the mbuf. | |
864 | * @return | |
865 | * The pointer of the mbuf buffer. | |
866 | */ | |
867 | static inline char * __rte_experimental | |
868 | rte_mbuf_buf_addr(struct rte_mbuf *mb, struct rte_mempool *mp) | |
869 | { | |
870 | return (char *)mb + sizeof(*mb) + rte_pktmbuf_priv_size(mp); | |
871 | } | |
872 | ||
873 | /** | |
874 | * Return the default address of the beginning of the mbuf data. | |
875 | * | |
876 | * @warning | |
877 | * @b EXPERIMENTAL: This API may change without prior notice. | |
878 | * | |
879 | * @param mb | |
880 | * The pointer to the mbuf. | |
881 | * @return | |
882 | * The pointer of the beginning of the mbuf data. | |
883 | */ | |
884 | static inline char * __rte_experimental | |
885 | rte_mbuf_data_addr_default(struct rte_mbuf *mb) | |
886 | { | |
887 | return rte_mbuf_buf_addr(mb, mb->pool) + RTE_PKTMBUF_HEADROOM; | |
888 | } | |
889 | ||
890 | /** | |
891 | * Return address of buffer embedded in the given mbuf. | |
892 | * | |
893 | * @note: Accessing mempool pointer of a mbuf is expensive because the | |
894 | * pointer is stored in the 2nd cache line of mbuf. If mempool is known, it | |
895 | * is better not to reference the mempool pointer in mbuf but calling | |
896 | * rte_mbuf_buf_addr() would be more efficient. | |
7c673cae FG |
897 | * |
898 | * @param md | |
899 | * The pointer to the mbuf. | |
900 | * @return | |
901 | * The address of the data buffer owned by the mbuf. | |
902 | */ | |
903 | static inline char * | |
904 | rte_mbuf_to_baddr(struct rte_mbuf *md) | |
905 | { | |
9f95a23c TL |
906 | #ifdef ALLOW_EXPERIMENTAL_API |
907 | return rte_mbuf_buf_addr(md, md->pool); | |
908 | #else | |
7c673cae FG |
909 | char *buffer_addr; |
910 | buffer_addr = (char *)md + sizeof(*md) + rte_pktmbuf_priv_size(md->pool); | |
911 | return buffer_addr; | |
9f95a23c | 912 | #endif |
7c673cae FG |
913 | } |
914 | ||
915 | /** | |
11fdf7f2 TL |
916 | * Return the starting address of the private data area embedded in |
917 | * the given mbuf. | |
918 | * | |
919 | * Note that no check is made to ensure that a private data area | |
920 | * actually exists in the supplied mbuf. | |
921 | * | |
922 | * @param m | |
923 | * The pointer to the mbuf. | |
924 | * @return | |
925 | * The starting address of the private data area of the given mbuf. | |
926 | */ | |
927 | static inline void * __rte_experimental | |
928 | rte_mbuf_to_priv(struct rte_mbuf *m) | |
929 | { | |
930 | return RTE_PTR_ADD(m, sizeof(struct rte_mbuf)); | |
931 | } | |
932 | ||
933 | /** | |
934 | * Returns TRUE if given mbuf is cloned by mbuf indirection, or FALSE | |
935 | * otherwise. | |
936 | * | |
937 | * If a mbuf has its data in another mbuf and references it by mbuf | |
938 | * indirection, this mbuf can be defined as a cloned mbuf. | |
939 | */ | |
940 | #define RTE_MBUF_CLONED(mb) ((mb)->ol_flags & IND_ATTACHED_MBUF) | |
941 | ||
11fdf7f2 TL |
942 | /** |
943 | * Returns TRUE if given mbuf has an external buffer, or FALSE otherwise. | |
944 | * | |
945 | * External buffer is a user-provided anonymous buffer. | |
946 | */ | |
947 | #define RTE_MBUF_HAS_EXTBUF(mb) ((mb)->ol_flags & EXT_ATTACHED_MBUF) | |
7c673cae FG |
948 | |
949 | /** | |
950 | * Returns TRUE if given mbuf is direct, or FALSE otherwise. | |
11fdf7f2 TL |
951 | * |
952 | * If a mbuf embeds its own data after the rte_mbuf structure, this mbuf | |
953 | * can be defined as a direct mbuf. | |
7c673cae | 954 | */ |
11fdf7f2 TL |
955 | #define RTE_MBUF_DIRECT(mb) \ |
956 | (!((mb)->ol_flags & (IND_ATTACHED_MBUF | EXT_ATTACHED_MBUF))) | |
7c673cae FG |
957 | |
958 | /** | |
959 | * Private data in case of pktmbuf pool. | |
960 | * | |
961 | * A structure that contains some pktmbuf_pool-specific data that are | |
962 | * appended after the mempool structure (in private data). | |
963 | */ | |
964 | struct rte_pktmbuf_pool_private { | |
965 | uint16_t mbuf_data_room_size; /**< Size of data space in each mbuf. */ | |
966 | uint16_t mbuf_priv_size; /**< Size of private area in each mbuf. */ | |
967 | }; | |
968 | ||
969 | #ifdef RTE_LIBRTE_MBUF_DEBUG | |
970 | ||
971 | /** check mbuf type in debug mode */ | |
972 | #define __rte_mbuf_sanity_check(m, is_h) rte_mbuf_sanity_check(m, is_h) | |
973 | ||
974 | #else /* RTE_LIBRTE_MBUF_DEBUG */ | |
975 | ||
976 | /** check mbuf type in debug mode */ | |
977 | #define __rte_mbuf_sanity_check(m, is_h) do { } while (0) | |
978 | ||
979 | #endif /* RTE_LIBRTE_MBUF_DEBUG */ | |
980 | ||
981 | #ifdef RTE_MBUF_REFCNT_ATOMIC | |
982 | ||
983 | /** | |
984 | * Reads the value of an mbuf's refcnt. | |
985 | * @param m | |
986 | * Mbuf to read | |
987 | * @return | |
988 | * Reference count number. | |
989 | */ | |
990 | static inline uint16_t | |
991 | rte_mbuf_refcnt_read(const struct rte_mbuf *m) | |
992 | { | |
993 | return (uint16_t)(rte_atomic16_read(&m->refcnt_atomic)); | |
994 | } | |
995 | ||
996 | /** | |
997 | * Sets an mbuf's refcnt to a defined value. | |
998 | * @param m | |
999 | * Mbuf to update | |
1000 | * @param new_value | |
1001 | * Value set | |
1002 | */ | |
1003 | static inline void | |
1004 | rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value) | |
1005 | { | |
11fdf7f2 TL |
1006 | rte_atomic16_set(&m->refcnt_atomic, (int16_t)new_value); |
1007 | } | |
1008 | ||
1009 | /* internal */ | |
1010 | static inline uint16_t | |
1011 | __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value) | |
1012 | { | |
1013 | return (uint16_t)(rte_atomic16_add_return(&m->refcnt_atomic, value)); | |
7c673cae FG |
1014 | } |
1015 | ||
1016 | /** | |
1017 | * Adds given value to an mbuf's refcnt and returns its new value. | |
1018 | * @param m | |
1019 | * Mbuf to update | |
1020 | * @param value | |
1021 | * Value to add/subtract | |
1022 | * @return | |
1023 | * Updated value | |
1024 | */ | |
1025 | static inline uint16_t | |
1026 | rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value) | |
1027 | { | |
1028 | /* | |
1029 | * The atomic_add is an expensive operation, so we don't want to | |
9f95a23c | 1030 | * call it in the case where we know we are the unique holder of |
7c673cae FG |
1031 | * this mbuf (i.e. ref_cnt == 1). Otherwise, an atomic |
1032 | * operation has to be used because concurrent accesses on the | |
1033 | * reference counter can occur. | |
1034 | */ | |
1035 | if (likely(rte_mbuf_refcnt_read(m) == 1)) { | |
11fdf7f2 TL |
1036 | ++value; |
1037 | rte_mbuf_refcnt_set(m, (uint16_t)value); | |
1038 | return (uint16_t)value; | |
7c673cae FG |
1039 | } |
1040 | ||
11fdf7f2 | 1041 | return __rte_mbuf_refcnt_update(m, value); |
7c673cae FG |
1042 | } |
1043 | ||
1044 | #else /* ! RTE_MBUF_REFCNT_ATOMIC */ | |
1045 | ||
11fdf7f2 TL |
1046 | /* internal */ |
1047 | static inline uint16_t | |
1048 | __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value) | |
1049 | { | |
1050 | m->refcnt = (uint16_t)(m->refcnt + value); | |
1051 | return m->refcnt; | |
1052 | } | |
1053 | ||
7c673cae FG |
1054 | /** |
1055 | * Adds given value to an mbuf's refcnt and returns its new value. | |
1056 | */ | |
1057 | static inline uint16_t | |
1058 | rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value) | |
1059 | { | |
11fdf7f2 | 1060 | return __rte_mbuf_refcnt_update(m, value); |
7c673cae FG |
1061 | } |
1062 | ||
1063 | /** | |
1064 | * Reads the value of an mbuf's refcnt. | |
1065 | */ | |
1066 | static inline uint16_t | |
1067 | rte_mbuf_refcnt_read(const struct rte_mbuf *m) | |
1068 | { | |
1069 | return m->refcnt; | |
1070 | } | |
1071 | ||
1072 | /** | |
1073 | * Sets an mbuf's refcnt to the defined value. | |
1074 | */ | |
1075 | static inline void | |
1076 | rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value) | |
1077 | { | |
1078 | m->refcnt = new_value; | |
1079 | } | |
1080 | ||
1081 | #endif /* RTE_MBUF_REFCNT_ATOMIC */ | |
1082 | ||
11fdf7f2 TL |
1083 | /** |
1084 | * Reads the refcnt of an external buffer. | |
1085 | * | |
1086 | * @param shinfo | |
1087 | * Shared data of the external buffer. | |
1088 | * @return | |
1089 | * Reference count number. | |
1090 | */ | |
1091 | static inline uint16_t | |
1092 | rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo) | |
1093 | { | |
1094 | return (uint16_t)(rte_atomic16_read(&shinfo->refcnt_atomic)); | |
1095 | } | |
1096 | ||
1097 | /** | |
1098 | * Set refcnt of an external buffer. | |
1099 | * | |
1100 | * @param shinfo | |
1101 | * Shared data of the external buffer. | |
1102 | * @param new_value | |
1103 | * Value set | |
1104 | */ | |
1105 | static inline void | |
1106 | rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo, | |
1107 | uint16_t new_value) | |
1108 | { | |
1109 | rte_atomic16_set(&shinfo->refcnt_atomic, (int16_t)new_value); | |
1110 | } | |
1111 | ||
1112 | /** | |
1113 | * Add given value to refcnt of an external buffer and return its new | |
1114 | * value. | |
1115 | * | |
1116 | * @param shinfo | |
1117 | * Shared data of the external buffer. | |
1118 | * @param value | |
1119 | * Value to add/subtract | |
1120 | * @return | |
1121 | * Updated value | |
1122 | */ | |
1123 | static inline uint16_t | |
1124 | rte_mbuf_ext_refcnt_update(struct rte_mbuf_ext_shared_info *shinfo, | |
1125 | int16_t value) | |
1126 | { | |
1127 | if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1)) { | |
1128 | ++value; | |
1129 | rte_mbuf_ext_refcnt_set(shinfo, (uint16_t)value); | |
1130 | return (uint16_t)value; | |
1131 | } | |
1132 | ||
1133 | return (uint16_t)rte_atomic16_add_return(&shinfo->refcnt_atomic, value); | |
1134 | } | |
1135 | ||
7c673cae FG |
1136 | /** Mbuf prefetch */ |
1137 | #define RTE_MBUF_PREFETCH_TO_FREE(m) do { \ | |
1138 | if ((m) != NULL) \ | |
1139 | rte_prefetch0(m); \ | |
1140 | } while (0) | |
1141 | ||
1142 | ||
1143 | /** | |
1144 | * Sanity checks on an mbuf. | |
1145 | * | |
1146 | * Check the consistency of the given mbuf. The function will cause a | |
1147 | * panic if corruption is detected. | |
1148 | * | |
1149 | * @param m | |
1150 | * The mbuf to be checked. | |
1151 | * @param is_header | |
1152 | * True if the mbuf is a packet header, false if it is a sub-segment | |
1153 | * of a packet (in this case, some fields like nb_segs are not checked) | |
1154 | */ | |
1155 | void | |
1156 | rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header); | |
1157 | ||
9f95a23c TL |
1158 | /** |
1159 | * Sanity checks on a mbuf. | |
1160 | * | |
1161 | * Almost like rte_mbuf_sanity_check(), but this function gives the reason | |
1162 | * if corruption is detected rather than panic. | |
1163 | * | |
1164 | * @param m | |
1165 | * The mbuf to be checked. | |
1166 | * @param is_header | |
1167 | * True if the mbuf is a packet header, false if it is a sub-segment | |
1168 | * of a packet (in this case, some fields like nb_segs are not checked) | |
1169 | * @param reason | |
1170 | * A reference to a string pointer where to store the reason why a mbuf is | |
1171 | * considered invalid. | |
1172 | * @return | |
1173 | * - 0 if no issue has been found, reason is left untouched. | |
1174 | * - -1 if a problem is detected, reason then points to a string describing | |
1175 | * the reason why the mbuf is deemed invalid. | |
1176 | */ | |
1177 | __rte_experimental | |
1178 | int rte_mbuf_check(const struct rte_mbuf *m, int is_header, | |
1179 | const char **reason); | |
1180 | ||
11fdf7f2 TL |
1181 | #define MBUF_RAW_ALLOC_CHECK(m) do { \ |
1182 | RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1); \ | |
1183 | RTE_ASSERT((m)->next == NULL); \ | |
1184 | RTE_ASSERT((m)->nb_segs == 1); \ | |
1185 | __rte_mbuf_sanity_check(m, 0); \ | |
1186 | } while (0) | |
1187 | ||
7c673cae | 1188 | /** |
11fdf7f2 | 1189 | * Allocate an uninitialized mbuf from mempool *mp*. |
7c673cae FG |
1190 | * |
1191 | * This function can be used by PMDs (especially in RX functions) to | |
11fdf7f2 | 1192 | * allocate an uninitialized mbuf. The driver is responsible of |
7c673cae FG |
1193 | * initializing all the required fields. See rte_pktmbuf_reset(). |
1194 | * For standard needs, prefer rte_pktmbuf_alloc(). | |
1195 | * | |
11fdf7f2 TL |
1196 | * The caller can expect that the following fields of the mbuf structure |
1197 | * are initialized: buf_addr, buf_iova, buf_len, refcnt=1, nb_segs=1, | |
1198 | * next=NULL, pool, priv_size. The other fields must be initialized | |
1199 | * by the caller. | |
1200 | * | |
7c673cae FG |
1201 | * @param mp |
1202 | * The mempool from which mbuf is allocated. | |
1203 | * @return | |
1204 | * - The pointer to the new mbuf on success. | |
1205 | * - NULL if allocation failed. | |
1206 | */ | |
1207 | static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp) | |
1208 | { | |
1209 | struct rte_mbuf *m; | |
7c673cae | 1210 | |
11fdf7f2 | 1211 | if (rte_mempool_get(mp, (void **)&m) < 0) |
7c673cae | 1212 | return NULL; |
11fdf7f2 | 1213 | MBUF_RAW_ALLOC_CHECK(m); |
7c673cae FG |
1214 | return m; |
1215 | } | |
1216 | ||
1217 | /** | |
11fdf7f2 TL |
1218 | * Put mbuf back into its original mempool. |
1219 | * | |
1220 | * The caller must ensure that the mbuf is direct and properly | |
1221 | * reinitialized (refcnt=1, next=NULL, nb_segs=1), as done by | |
1222 | * rte_pktmbuf_prefree_seg(). | |
1223 | * | |
1224 | * This function should be used with care, when optimization is | |
1225 | * required. For standard needs, prefer rte_pktmbuf_free() or | |
1226 | * rte_pktmbuf_free_seg(). | |
7c673cae FG |
1227 | * |
1228 | * @param m | |
1229 | * The mbuf to be freed. | |
1230 | */ | |
11fdf7f2 TL |
1231 | static __rte_always_inline void |
1232 | rte_mbuf_raw_free(struct rte_mbuf *m) | |
7c673cae | 1233 | { |
11fdf7f2 TL |
1234 | RTE_ASSERT(RTE_MBUF_DIRECT(m)); |
1235 | RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1); | |
1236 | RTE_ASSERT(m->next == NULL); | |
1237 | RTE_ASSERT(m->nb_segs == 1); | |
1238 | __rte_mbuf_sanity_check(m, 0); | |
7c673cae FG |
1239 | rte_mempool_put(m->pool, m); |
1240 | } | |
1241 | ||
7c673cae FG |
1242 | /** |
1243 | * The packet mbuf constructor. | |
1244 | * | |
1245 | * This function initializes some fields in the mbuf structure that are | |
1246 | * not modified by the user once created (origin pool, buffer start | |
1247 | * address, and so on). This function is given as a callback function to | |
11fdf7f2 | 1248 | * rte_mempool_obj_iter() or rte_mempool_create() at pool creation time. |
7c673cae FG |
1249 | * |
1250 | * @param mp | |
1251 | * The mempool from which mbufs originate. | |
1252 | * @param opaque_arg | |
1253 | * A pointer that can be used by the user to retrieve useful information | |
11fdf7f2 TL |
1254 | * for mbuf initialization. This pointer is the opaque argument passed to |
1255 | * rte_mempool_obj_iter() or rte_mempool_create(). | |
7c673cae FG |
1256 | * @param m |
1257 | * The mbuf to initialize. | |
1258 | * @param i | |
1259 | * The index of the mbuf in the pool table. | |
1260 | */ | |
1261 | void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg, | |
1262 | void *m, unsigned i); | |
1263 | ||
1264 | ||
1265 | /** | |
1266 | * A packet mbuf pool constructor. | |
1267 | * | |
1268 | * This function initializes the mempool private data in the case of a | |
1269 | * pktmbuf pool. This private data is needed by the driver. The | |
11fdf7f2 TL |
1270 | * function must be called on the mempool before it is used, or it |
1271 | * can be given as a callback function to rte_mempool_create() at | |
7c673cae FG |
1272 | * pool creation. It can be extended by the user, for example, to |
1273 | * provide another packet size. | |
1274 | * | |
1275 | * @param mp | |
1276 | * The mempool from which mbufs originate. | |
1277 | * @param opaque_arg | |
1278 | * A pointer that can be used by the user to retrieve useful information | |
11fdf7f2 TL |
1279 | * for mbuf initialization. This pointer is the opaque argument passed to |
1280 | * rte_mempool_create(). | |
7c673cae FG |
1281 | */ |
1282 | void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg); | |
1283 | ||
1284 | /** | |
1285 | * Create a mbuf pool. | |
1286 | * | |
1287 | * This function creates and initializes a packet mbuf pool. It is | |
11fdf7f2 | 1288 | * a wrapper to rte_mempool functions. |
7c673cae FG |
1289 | * |
1290 | * @param name | |
1291 | * The name of the mbuf pool. | |
1292 | * @param n | |
1293 | * The number of elements in the mbuf pool. The optimum size (in terms | |
1294 | * of memory usage) for a mempool is when n is a power of two minus one: | |
1295 | * n = (2^q - 1). | |
1296 | * @param cache_size | |
1297 | * Size of the per-core object cache. See rte_mempool_create() for | |
1298 | * details. | |
1299 | * @param priv_size | |
1300 | * Size of application private are between the rte_mbuf structure | |
1301 | * and the data buffer. This value must be aligned to RTE_MBUF_PRIV_ALIGN. | |
1302 | * @param data_room_size | |
1303 | * Size of data buffer in each mbuf, including RTE_PKTMBUF_HEADROOM. | |
1304 | * @param socket_id | |
1305 | * The socket identifier where the memory should be allocated. The | |
1306 | * value can be *SOCKET_ID_ANY* if there is no NUMA constraint for the | |
1307 | * reserved zone. | |
1308 | * @return | |
1309 | * The pointer to the new allocated mempool, on success. NULL on error | |
1310 | * with rte_errno set appropriately. Possible rte_errno values include: | |
1311 | * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure | |
1312 | * - E_RTE_SECONDARY - function was called from a secondary process instance | |
1313 | * - EINVAL - cache size provided is too large, or priv_size is not aligned. | |
1314 | * - ENOSPC - the maximum number of memzones has already been allocated | |
1315 | * - EEXIST - a memzone with the same name already exists | |
1316 | * - ENOMEM - no appropriate memory area found in which to create memzone | |
1317 | */ | |
1318 | struct rte_mempool * | |
1319 | rte_pktmbuf_pool_create(const char *name, unsigned n, | |
1320 | unsigned cache_size, uint16_t priv_size, uint16_t data_room_size, | |
1321 | int socket_id); | |
1322 | ||
11fdf7f2 TL |
1323 | /** |
1324 | * Create a mbuf pool with a given mempool ops name | |
1325 | * | |
1326 | * This function creates and initializes a packet mbuf pool. It is | |
1327 | * a wrapper to rte_mempool functions. | |
1328 | * | |
1329 | * @param name | |
1330 | * The name of the mbuf pool. | |
1331 | * @param n | |
1332 | * The number of elements in the mbuf pool. The optimum size (in terms | |
1333 | * of memory usage) for a mempool is when n is a power of two minus one: | |
1334 | * n = (2^q - 1). | |
1335 | * @param cache_size | |
1336 | * Size of the per-core object cache. See rte_mempool_create() for | |
1337 | * details. | |
1338 | * @param priv_size | |
1339 | * Size of application private are between the rte_mbuf structure | |
1340 | * and the data buffer. This value must be aligned to RTE_MBUF_PRIV_ALIGN. | |
1341 | * @param data_room_size | |
1342 | * Size of data buffer in each mbuf, including RTE_PKTMBUF_HEADROOM. | |
1343 | * @param socket_id | |
1344 | * The socket identifier where the memory should be allocated. The | |
1345 | * value can be *SOCKET_ID_ANY* if there is no NUMA constraint for the | |
1346 | * reserved zone. | |
1347 | * @param ops_name | |
1348 | * The mempool ops name to be used for this mempool instead of | |
1349 | * default mempool. The value can be *NULL* to use default mempool. | |
1350 | * @return | |
1351 | * The pointer to the new allocated mempool, on success. NULL on error | |
1352 | * with rte_errno set appropriately. Possible rte_errno values include: | |
1353 | * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure | |
1354 | * - E_RTE_SECONDARY - function was called from a secondary process instance | |
1355 | * - EINVAL - cache size provided is too large, or priv_size is not aligned. | |
1356 | * - ENOSPC - the maximum number of memzones has already been allocated | |
1357 | * - EEXIST - a memzone with the same name already exists | |
1358 | * - ENOMEM - no appropriate memory area found in which to create memzone | |
1359 | */ | |
1360 | struct rte_mempool * | |
1361 | rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n, | |
1362 | unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, | |
1363 | int socket_id, const char *ops_name); | |
1364 | ||
7c673cae FG |
1365 | /** |
1366 | * Get the data room size of mbufs stored in a pktmbuf_pool | |
1367 | * | |
1368 | * The data room size is the amount of data that can be stored in a | |
1369 | * mbuf including the headroom (RTE_PKTMBUF_HEADROOM). | |
1370 | * | |
1371 | * @param mp | |
1372 | * The packet mbuf pool. | |
1373 | * @return | |
1374 | * The data room size of mbufs stored in this mempool. | |
1375 | */ | |
1376 | static inline uint16_t | |
1377 | rte_pktmbuf_data_room_size(struct rte_mempool *mp) | |
1378 | { | |
1379 | struct rte_pktmbuf_pool_private *mbp_priv; | |
1380 | ||
1381 | mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp); | |
1382 | return mbp_priv->mbuf_data_room_size; | |
1383 | } | |
1384 | ||
1385 | /** | |
1386 | * Get the application private size of mbufs stored in a pktmbuf_pool | |
1387 | * | |
1388 | * The private size of mbuf is a zone located between the rte_mbuf | |
1389 | * structure and the data buffer where an application can store data | |
1390 | * associated to a packet. | |
1391 | * | |
1392 | * @param mp | |
1393 | * The packet mbuf pool. | |
1394 | * @return | |
1395 | * The private size of mbufs stored in this mempool. | |
1396 | */ | |
1397 | static inline uint16_t | |
1398 | rte_pktmbuf_priv_size(struct rte_mempool *mp) | |
1399 | { | |
1400 | struct rte_pktmbuf_pool_private *mbp_priv; | |
1401 | ||
1402 | mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp); | |
1403 | return mbp_priv->mbuf_priv_size; | |
1404 | } | |
1405 | ||
1406 | /** | |
1407 | * Reset the data_off field of a packet mbuf to its default value. | |
1408 | * | |
1409 | * The given mbuf must have only one segment, which should be empty. | |
1410 | * | |
1411 | * @param m | |
1412 | * The packet mbuf's data_off field has to be reset. | |
1413 | */ | |
1414 | static inline void rte_pktmbuf_reset_headroom(struct rte_mbuf *m) | |
1415 | { | |
11fdf7f2 TL |
1416 | m->data_off = (uint16_t)RTE_MIN((uint16_t)RTE_PKTMBUF_HEADROOM, |
1417 | (uint16_t)m->buf_len); | |
7c673cae FG |
1418 | } |
1419 | ||
1420 | /** | |
1421 | * Reset the fields of a packet mbuf to their default values. | |
1422 | * | |
1423 | * The given mbuf must have only one segment. | |
1424 | * | |
1425 | * @param m | |
9f95a23c | 1426 | * The packet mbuf to be reset. |
7c673cae | 1427 | */ |
11fdf7f2 TL |
1428 | #define MBUF_INVALID_PORT UINT16_MAX |
1429 | ||
7c673cae FG |
1430 | static inline void rte_pktmbuf_reset(struct rte_mbuf *m) |
1431 | { | |
1432 | m->next = NULL; | |
1433 | m->pkt_len = 0; | |
1434 | m->tx_offload = 0; | |
1435 | m->vlan_tci = 0; | |
1436 | m->vlan_tci_outer = 0; | |
1437 | m->nb_segs = 1; | |
11fdf7f2 | 1438 | m->port = MBUF_INVALID_PORT; |
7c673cae FG |
1439 | |
1440 | m->ol_flags = 0; | |
1441 | m->packet_type = 0; | |
1442 | rte_pktmbuf_reset_headroom(m); | |
1443 | ||
1444 | m->data_len = 0; | |
1445 | __rte_mbuf_sanity_check(m, 1); | |
1446 | } | |
1447 | ||
1448 | /** | |
1449 | * Allocate a new mbuf from a mempool. | |
1450 | * | |
1451 | * This new mbuf contains one segment, which has a length of 0. The pointer | |
1452 | * to data is initialized to have some bytes of headroom in the buffer | |
1453 | * (if buffer size allows). | |
1454 | * | |
1455 | * @param mp | |
1456 | * The mempool from which the mbuf is allocated. | |
1457 | * @return | |
1458 | * - The pointer to the new mbuf on success. | |
1459 | * - NULL if allocation failed. | |
1460 | */ | |
1461 | static inline struct rte_mbuf *rte_pktmbuf_alloc(struct rte_mempool *mp) | |
1462 | { | |
1463 | struct rte_mbuf *m; | |
1464 | if ((m = rte_mbuf_raw_alloc(mp)) != NULL) | |
1465 | rte_pktmbuf_reset(m); | |
1466 | return m; | |
1467 | } | |
1468 | ||
1469 | /** | |
1470 | * Allocate a bulk of mbufs, initialize refcnt and reset the fields to default | |
1471 | * values. | |
1472 | * | |
1473 | * @param pool | |
1474 | * The mempool from which mbufs are allocated. | |
1475 | * @param mbufs | |
1476 | * Array of pointers to mbufs | |
1477 | * @param count | |
1478 | * Array size | |
1479 | * @return | |
1480 | * - 0: Success | |
11fdf7f2 | 1481 | * - -ENOENT: Not enough entries in the mempool; no mbufs are retrieved. |
7c673cae FG |
1482 | */ |
1483 | static inline int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool, | |
1484 | struct rte_mbuf **mbufs, unsigned count) | |
1485 | { | |
1486 | unsigned idx = 0; | |
1487 | int rc; | |
1488 | ||
1489 | rc = rte_mempool_get_bulk(pool, (void **)mbufs, count); | |
1490 | if (unlikely(rc)) | |
1491 | return rc; | |
1492 | ||
1493 | /* To understand duff's device on loop unwinding optimization, see | |
1494 | * https://en.wikipedia.org/wiki/Duff's_device. | |
1495 | * Here while() loop is used rather than do() while{} to avoid extra | |
1496 | * check if count is zero. | |
1497 | */ | |
1498 | switch (count % 4) { | |
1499 | case 0: | |
1500 | while (idx != count) { | |
11fdf7f2 | 1501 | MBUF_RAW_ALLOC_CHECK(mbufs[idx]); |
7c673cae FG |
1502 | rte_pktmbuf_reset(mbufs[idx]); |
1503 | idx++; | |
11fdf7f2 | 1504 | /* fall-through */ |
7c673cae | 1505 | case 3: |
11fdf7f2 | 1506 | MBUF_RAW_ALLOC_CHECK(mbufs[idx]); |
7c673cae FG |
1507 | rte_pktmbuf_reset(mbufs[idx]); |
1508 | idx++; | |
11fdf7f2 | 1509 | /* fall-through */ |
7c673cae | 1510 | case 2: |
11fdf7f2 | 1511 | MBUF_RAW_ALLOC_CHECK(mbufs[idx]); |
7c673cae FG |
1512 | rte_pktmbuf_reset(mbufs[idx]); |
1513 | idx++; | |
11fdf7f2 | 1514 | /* fall-through */ |
7c673cae | 1515 | case 1: |
11fdf7f2 | 1516 | MBUF_RAW_ALLOC_CHECK(mbufs[idx]); |
7c673cae FG |
1517 | rte_pktmbuf_reset(mbufs[idx]); |
1518 | idx++; | |
11fdf7f2 | 1519 | /* fall-through */ |
7c673cae FG |
1520 | } |
1521 | } | |
1522 | return 0; | |
1523 | } | |
1524 | ||
11fdf7f2 TL |
1525 | /** |
1526 | * Initialize shared data at the end of an external buffer before attaching | |
1527 | * to a mbuf by ``rte_pktmbuf_attach_extbuf()``. This is not a mandatory | |
1528 | * initialization but a helper function to simply spare a few bytes at the | |
1529 | * end of the buffer for shared data. If shared data is allocated | |
1530 | * separately, this should not be called but application has to properly | |
1531 | * initialize the shared data according to its need. | |
1532 | * | |
1533 | * Free callback and its argument is saved and the refcnt is set to 1. | |
1534 | * | |
1535 | * @warning | |
1536 | * The value of buf_len will be reduced to RTE_PTR_DIFF(shinfo, buf_addr) | |
1537 | * after this initialization. This shall be used for | |
1538 | * ``rte_pktmbuf_attach_extbuf()`` | |
1539 | * | |
1540 | * @param buf_addr | |
1541 | * The pointer to the external buffer. | |
1542 | * @param [in,out] buf_len | |
1543 | * The pointer to length of the external buffer. Input value must be | |
1544 | * larger than the size of ``struct rte_mbuf_ext_shared_info`` and | |
1545 | * padding for alignment. If not enough, this function will return NULL. | |
1546 | * Adjusted buffer length will be returned through this pointer. | |
1547 | * @param free_cb | |
1548 | * Free callback function to call when the external buffer needs to be | |
1549 | * freed. | |
1550 | * @param fcb_opaque | |
1551 | * Argument for the free callback function. | |
1552 | * | |
1553 | * @return | |
1554 | * A pointer to the initialized shared data on success, return NULL | |
1555 | * otherwise. | |
1556 | */ | |
1557 | static inline struct rte_mbuf_ext_shared_info * | |
1558 | rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len, | |
1559 | rte_mbuf_extbuf_free_callback_t free_cb, void *fcb_opaque) | |
1560 | { | |
1561 | struct rte_mbuf_ext_shared_info *shinfo; | |
1562 | void *buf_end = RTE_PTR_ADD(buf_addr, *buf_len); | |
1563 | void *addr; | |
1564 | ||
1565 | addr = RTE_PTR_ALIGN_FLOOR(RTE_PTR_SUB(buf_end, sizeof(*shinfo)), | |
1566 | sizeof(uintptr_t)); | |
1567 | if (addr <= buf_addr) | |
1568 | return NULL; | |
1569 | ||
1570 | shinfo = (struct rte_mbuf_ext_shared_info *)addr; | |
1571 | shinfo->free_cb = free_cb; | |
1572 | shinfo->fcb_opaque = fcb_opaque; | |
1573 | rte_mbuf_ext_refcnt_set(shinfo, 1); | |
1574 | ||
1575 | *buf_len = (uint16_t)RTE_PTR_DIFF(shinfo, buf_addr); | |
1576 | return shinfo; | |
1577 | } | |
1578 | ||
1579 | /** | |
1580 | * Attach an external buffer to a mbuf. | |
1581 | * | |
1582 | * User-managed anonymous buffer can be attached to an mbuf. When attaching | |
1583 | * it, corresponding free callback function and its argument should be | |
1584 | * provided via shinfo. This callback function will be called once all the | |
1585 | * mbufs are detached from the buffer (refcnt becomes zero). | |
1586 | * | |
1587 | * The headroom for the attaching mbuf will be set to zero and this can be | |
1588 | * properly adjusted after attachment. For example, ``rte_pktmbuf_adj()`` | |
1589 | * or ``rte_pktmbuf_reset_headroom()`` might be used. | |
1590 | * | |
1591 | * More mbufs can be attached to the same external buffer by | |
1592 | * ``rte_pktmbuf_attach()`` once the external buffer has been attached by | |
1593 | * this API. | |
1594 | * | |
1595 | * Detachment can be done by either ``rte_pktmbuf_detach_extbuf()`` or | |
1596 | * ``rte_pktmbuf_detach()``. | |
1597 | * | |
1598 | * Memory for shared data must be provided and user must initialize all of | |
9f95a23c | 1599 | * the content properly, especially free callback and refcnt. The pointer |
11fdf7f2 TL |
1600 | * of shared data will be stored in m->shinfo. |
1601 | * ``rte_pktmbuf_ext_shinfo_init_helper`` can help to simply spare a few | |
1602 | * bytes at the end of buffer for the shared data, store free callback and | |
1603 | * its argument and set the refcnt to 1. The following is an example: | |
1604 | * | |
1605 | * struct rte_mbuf_ext_shared_info *shinfo = | |
1606 | * rte_pktmbuf_ext_shinfo_init_helper(buf_addr, &buf_len, | |
1607 | * free_cb, fcb_arg); | |
1608 | * rte_pktmbuf_attach_extbuf(m, buf_addr, buf_iova, buf_len, shinfo); | |
1609 | * rte_pktmbuf_reset_headroom(m); | |
1610 | * rte_pktmbuf_adj(m, data_len); | |
1611 | * | |
1612 | * Attaching an external buffer is quite similar to mbuf indirection in | |
1613 | * replacing buffer addresses and length of a mbuf, but a few differences: | |
1614 | * - When an indirect mbuf is attached, refcnt of the direct mbuf would be | |
1615 | * 2 as long as the direct mbuf itself isn't freed after the attachment. | |
1616 | * In such cases, the buffer area of a direct mbuf must be read-only. But | |
1617 | * external buffer has its own refcnt and it starts from 1. Unless | |
1618 | * multiple mbufs are attached to a mbuf having an external buffer, the | |
1619 | * external buffer is writable. | |
1620 | * - There's no need to allocate buffer from a mempool. Any buffer can be | |
1621 | * attached with appropriate free callback and its IO address. | |
1622 | * - Smaller metadata is required to maintain shared data such as refcnt. | |
1623 | * | |
11fdf7f2 TL |
1624 | * @param m |
1625 | * The pointer to the mbuf. | |
1626 | * @param buf_addr | |
1627 | * The pointer to the external buffer. | |
1628 | * @param buf_iova | |
1629 | * IO address of the external buffer. | |
1630 | * @param buf_len | |
1631 | * The size of the external buffer. | |
1632 | * @param shinfo | |
1633 | * User-provided memory for shared data of the external buffer. | |
1634 | */ | |
9f95a23c | 1635 | static inline void |
11fdf7f2 TL |
1636 | rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr, |
1637 | rte_iova_t buf_iova, uint16_t buf_len, | |
1638 | struct rte_mbuf_ext_shared_info *shinfo) | |
1639 | { | |
1640 | /* mbuf should not be read-only */ | |
1641 | RTE_ASSERT(RTE_MBUF_DIRECT(m) && rte_mbuf_refcnt_read(m) == 1); | |
1642 | RTE_ASSERT(shinfo->free_cb != NULL); | |
1643 | ||
1644 | m->buf_addr = buf_addr; | |
1645 | m->buf_iova = buf_iova; | |
1646 | m->buf_len = buf_len; | |
1647 | ||
1648 | m->data_len = 0; | |
1649 | m->data_off = 0; | |
1650 | ||
1651 | m->ol_flags |= EXT_ATTACHED_MBUF; | |
1652 | m->shinfo = shinfo; | |
1653 | } | |
1654 | ||
1655 | /** | |
1656 | * Detach the external buffer attached to a mbuf, same as | |
1657 | * ``rte_pktmbuf_detach()`` | |
1658 | * | |
1659 | * @param m | |
1660 | * The mbuf having external buffer. | |
1661 | */ | |
1662 | #define rte_pktmbuf_detach_extbuf(m) rte_pktmbuf_detach(m) | |
1663 | ||
7c673cae FG |
1664 | /** |
1665 | * Attach packet mbuf to another packet mbuf. | |
1666 | * | |
11fdf7f2 TL |
1667 | * If the mbuf we are attaching to isn't a direct buffer and is attached to |
1668 | * an external buffer, the mbuf being attached will be attached to the | |
1669 | * external buffer instead of mbuf indirection. | |
1670 | * | |
1671 | * Otherwise, the mbuf will be indirectly attached. After attachment we | |
1672 | * refer the mbuf we attached as 'indirect', while mbuf we attached to as | |
1673 | * 'direct'. The direct mbuf's reference counter is incremented. | |
7c673cae FG |
1674 | * |
1675 | * Right now, not supported: | |
1676 | * - attachment for already indirect mbuf (e.g. - mi has to be direct). | |
1677 | * - mbuf we trying to attach (mi) is used by someone else | |
1678 | * e.g. it's reference counter is greater then 1. | |
1679 | * | |
1680 | * @param mi | |
1681 | * The indirect packet mbuf. | |
1682 | * @param m | |
1683 | * The packet mbuf we're attaching to. | |
1684 | */ | |
1685 | static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m) | |
1686 | { | |
7c673cae FG |
1687 | RTE_ASSERT(RTE_MBUF_DIRECT(mi) && |
1688 | rte_mbuf_refcnt_read(mi) == 1); | |
1689 | ||
11fdf7f2 TL |
1690 | if (RTE_MBUF_HAS_EXTBUF(m)) { |
1691 | rte_mbuf_ext_refcnt_update(m->shinfo, 1); | |
1692 | mi->ol_flags = m->ol_flags; | |
1693 | mi->shinfo = m->shinfo; | |
1694 | } else { | |
1695 | /* if m is not direct, get the mbuf that embeds the data */ | |
1696 | rte_mbuf_refcnt_update(rte_mbuf_from_indirect(m), 1); | |
1697 | mi->priv_size = m->priv_size; | |
1698 | mi->ol_flags = m->ol_flags | IND_ATTACHED_MBUF; | |
1699 | } | |
7c673cae | 1700 | |
11fdf7f2 | 1701 | mi->buf_iova = m->buf_iova; |
7c673cae FG |
1702 | mi->buf_addr = m->buf_addr; |
1703 | mi->buf_len = m->buf_len; | |
1704 | ||
7c673cae FG |
1705 | mi->data_off = m->data_off; |
1706 | mi->data_len = m->data_len; | |
1707 | mi->port = m->port; | |
1708 | mi->vlan_tci = m->vlan_tci; | |
1709 | mi->vlan_tci_outer = m->vlan_tci_outer; | |
1710 | mi->tx_offload = m->tx_offload; | |
1711 | mi->hash = m->hash; | |
1712 | ||
1713 | mi->next = NULL; | |
1714 | mi->pkt_len = mi->data_len; | |
1715 | mi->nb_segs = 1; | |
7c673cae | 1716 | mi->packet_type = m->packet_type; |
11fdf7f2 | 1717 | mi->timestamp = m->timestamp; |
7c673cae FG |
1718 | |
1719 | __rte_mbuf_sanity_check(mi, 1); | |
1720 | __rte_mbuf_sanity_check(m, 0); | |
1721 | } | |
1722 | ||
1723 | /** | |
11fdf7f2 TL |
1724 | * @internal used by rte_pktmbuf_detach(). |
1725 | * | |
1726 | * Decrement the reference counter of the external buffer. When the | |
1727 | * reference counter becomes 0, the buffer is freed by pre-registered | |
1728 | * callback. | |
1729 | */ | |
1730 | static inline void | |
1731 | __rte_pktmbuf_free_extbuf(struct rte_mbuf *m) | |
1732 | { | |
1733 | RTE_ASSERT(RTE_MBUF_HAS_EXTBUF(m)); | |
1734 | RTE_ASSERT(m->shinfo != NULL); | |
1735 | ||
1736 | if (rte_mbuf_ext_refcnt_update(m->shinfo, -1) == 0) | |
1737 | m->shinfo->free_cb(m->buf_addr, m->shinfo->fcb_opaque); | |
1738 | } | |
1739 | ||
1740 | /** | |
1741 | * @internal used by rte_pktmbuf_detach(). | |
7c673cae | 1742 | * |
11fdf7f2 TL |
1743 | * Decrement the direct mbuf's reference counter. When the reference |
1744 | * counter becomes 0, the direct mbuf is freed. | |
1745 | */ | |
1746 | static inline void | |
1747 | __rte_pktmbuf_free_direct(struct rte_mbuf *m) | |
1748 | { | |
1749 | struct rte_mbuf *md; | |
1750 | ||
9f95a23c | 1751 | RTE_ASSERT(RTE_MBUF_CLONED(m)); |
11fdf7f2 TL |
1752 | |
1753 | md = rte_mbuf_from_indirect(m); | |
1754 | ||
1755 | if (rte_mbuf_refcnt_update(md, -1) == 0) { | |
1756 | md->next = NULL; | |
1757 | md->nb_segs = 1; | |
1758 | rte_mbuf_refcnt_set(md, 1); | |
1759 | rte_mbuf_raw_free(md); | |
1760 | } | |
1761 | } | |
1762 | ||
1763 | /** | |
1764 | * Detach a packet mbuf from external buffer or direct buffer. | |
1765 | * | |
1766 | * - decrement refcnt and free the external/direct buffer if refcnt | |
1767 | * becomes zero. | |
7c673cae FG |
1768 | * - restore original mbuf address and length values. |
1769 | * - reset pktmbuf data and data_len to their default values. | |
7c673cae FG |
1770 | * |
1771 | * All other fields of the given packet mbuf will be left intact. | |
1772 | * | |
1773 | * @param m | |
1774 | * The indirect attached packet mbuf. | |
1775 | */ | |
1776 | static inline void rte_pktmbuf_detach(struct rte_mbuf *m) | |
1777 | { | |
7c673cae | 1778 | struct rte_mempool *mp = m->pool; |
11fdf7f2 TL |
1779 | uint32_t mbuf_size, buf_len; |
1780 | uint16_t priv_size; | |
1781 | ||
1782 | if (RTE_MBUF_HAS_EXTBUF(m)) | |
1783 | __rte_pktmbuf_free_extbuf(m); | |
1784 | else | |
1785 | __rte_pktmbuf_free_direct(m); | |
7c673cae FG |
1786 | |
1787 | priv_size = rte_pktmbuf_priv_size(mp); | |
11fdf7f2 | 1788 | mbuf_size = (uint32_t)(sizeof(struct rte_mbuf) + priv_size); |
7c673cae FG |
1789 | buf_len = rte_pktmbuf_data_room_size(mp); |
1790 | ||
1791 | m->priv_size = priv_size; | |
1792 | m->buf_addr = (char *)m + mbuf_size; | |
11fdf7f2 | 1793 | m->buf_iova = rte_mempool_virt2iova(m) + mbuf_size; |
7c673cae FG |
1794 | m->buf_len = (uint16_t)buf_len; |
1795 | rte_pktmbuf_reset_headroom(m); | |
1796 | m->data_len = 0; | |
1797 | m->ol_flags = 0; | |
7c673cae FG |
1798 | } |
1799 | ||
11fdf7f2 TL |
1800 | /** |
1801 | * Decrease reference counter and unlink a mbuf segment | |
1802 | * | |
1803 | * This function does the same than a free, except that it does not | |
1804 | * return the segment to its pool. | |
1805 | * It decreases the reference counter, and if it reaches 0, it is | |
1806 | * detached from its parent for an indirect mbuf. | |
1807 | * | |
1808 | * @param m | |
1809 | * The mbuf to be unlinked | |
1810 | * @return | |
1811 | * - (m) if it is the last reference. It can be recycled or freed. | |
1812 | * - (NULL) if the mbuf still has remaining references on it. | |
1813 | */ | |
1814 | static __rte_always_inline struct rte_mbuf * | |
1815 | rte_pktmbuf_prefree_seg(struct rte_mbuf *m) | |
7c673cae FG |
1816 | { |
1817 | __rte_mbuf_sanity_check(m, 0); | |
1818 | ||
11fdf7f2 TL |
1819 | if (likely(rte_mbuf_refcnt_read(m) == 1)) { |
1820 | ||
1821 | if (!RTE_MBUF_DIRECT(m)) | |
7c673cae | 1822 | rte_pktmbuf_detach(m); |
11fdf7f2 TL |
1823 | |
1824 | if (m->next != NULL) { | |
1825 | m->next = NULL; | |
1826 | m->nb_segs = 1; | |
1827 | } | |
1828 | ||
1829 | return m; | |
1830 | ||
1831 | } else if (__rte_mbuf_refcnt_update(m, -1) == 0) { | |
1832 | ||
1833 | if (!RTE_MBUF_DIRECT(m)) | |
1834 | rte_pktmbuf_detach(m); | |
1835 | ||
1836 | if (m->next != NULL) { | |
1837 | m->next = NULL; | |
1838 | m->nb_segs = 1; | |
1839 | } | |
1840 | rte_mbuf_refcnt_set(m, 1); | |
1841 | ||
7c673cae FG |
1842 | return m; |
1843 | } | |
1844 | return NULL; | |
1845 | } | |
1846 | ||
1847 | /** | |
1848 | * Free a segment of a packet mbuf into its original mempool. | |
1849 | * | |
1850 | * Free an mbuf, without parsing other segments in case of chained | |
1851 | * buffers. | |
1852 | * | |
1853 | * @param m | |
1854 | * The packet mbuf segment to be freed. | |
1855 | */ | |
11fdf7f2 | 1856 | static __rte_always_inline void |
7c673cae FG |
1857 | rte_pktmbuf_free_seg(struct rte_mbuf *m) |
1858 | { | |
11fdf7f2 TL |
1859 | m = rte_pktmbuf_prefree_seg(m); |
1860 | if (likely(m != NULL)) | |
1861 | rte_mbuf_raw_free(m); | |
7c673cae FG |
1862 | } |
1863 | ||
1864 | /** | |
1865 | * Free a packet mbuf back into its original mempool. | |
1866 | * | |
1867 | * Free an mbuf, and all its segments in case of chained buffers. Each | |
1868 | * segment is added back into its original mempool. | |
1869 | * | |
1870 | * @param m | |
11fdf7f2 | 1871 | * The packet mbuf to be freed. If NULL, the function does nothing. |
7c673cae FG |
1872 | */ |
1873 | static inline void rte_pktmbuf_free(struct rte_mbuf *m) | |
1874 | { | |
1875 | struct rte_mbuf *m_next; | |
1876 | ||
11fdf7f2 TL |
1877 | if (m != NULL) |
1878 | __rte_mbuf_sanity_check(m, 1); | |
7c673cae FG |
1879 | |
1880 | while (m != NULL) { | |
1881 | m_next = m->next; | |
1882 | rte_pktmbuf_free_seg(m); | |
1883 | m = m_next; | |
1884 | } | |
1885 | } | |
1886 | ||
1887 | /** | |
1888 | * Creates a "clone" of the given packet mbuf. | |
1889 | * | |
1890 | * Walks through all segments of the given packet mbuf, and for each of them: | |
1891 | * - Creates a new packet mbuf from the given pool. | |
1892 | * - Attaches newly created mbuf to the segment. | |
1893 | * Then updates pkt_len and nb_segs of the "clone" packet mbuf to match values | |
1894 | * from the original packet mbuf. | |
1895 | * | |
1896 | * @param md | |
1897 | * The packet mbuf to be cloned. | |
1898 | * @param mp | |
1899 | * The mempool from which the "clone" mbufs are allocated. | |
1900 | * @return | |
1901 | * - The pointer to the new "clone" mbuf on success. | |
1902 | * - NULL if allocation fails. | |
1903 | */ | |
1904 | static inline struct rte_mbuf *rte_pktmbuf_clone(struct rte_mbuf *md, | |
1905 | struct rte_mempool *mp) | |
1906 | { | |
1907 | struct rte_mbuf *mc, *mi, **prev; | |
1908 | uint32_t pktlen; | |
11fdf7f2 | 1909 | uint16_t nseg; |
7c673cae FG |
1910 | |
1911 | if (unlikely ((mc = rte_pktmbuf_alloc(mp)) == NULL)) | |
1912 | return NULL; | |
1913 | ||
1914 | mi = mc; | |
1915 | prev = &mi->next; | |
1916 | pktlen = md->pkt_len; | |
1917 | nseg = 0; | |
1918 | ||
1919 | do { | |
1920 | nseg++; | |
1921 | rte_pktmbuf_attach(mi, md); | |
1922 | *prev = mi; | |
1923 | prev = &mi->next; | |
1924 | } while ((md = md->next) != NULL && | |
1925 | (mi = rte_pktmbuf_alloc(mp)) != NULL); | |
1926 | ||
1927 | *prev = NULL; | |
1928 | mc->nb_segs = nseg; | |
1929 | mc->pkt_len = pktlen; | |
1930 | ||
1931 | /* Allocation of new indirect segment failed */ | |
1932 | if (unlikely (mi == NULL)) { | |
1933 | rte_pktmbuf_free(mc); | |
1934 | return NULL; | |
1935 | } | |
1936 | ||
1937 | __rte_mbuf_sanity_check(mc, 1); | |
1938 | return mc; | |
1939 | } | |
1940 | ||
1941 | /** | |
1942 | * Adds given value to the refcnt of all packet mbuf segments. | |
1943 | * | |
1944 | * Walks through all segments of given packet mbuf and for each of them | |
1945 | * invokes rte_mbuf_refcnt_update(). | |
1946 | * | |
1947 | * @param m | |
1948 | * The packet mbuf whose refcnt to be updated. | |
1949 | * @param v | |
1950 | * The value to add to the mbuf's segments refcnt. | |
1951 | */ | |
1952 | static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v) | |
1953 | { | |
1954 | __rte_mbuf_sanity_check(m, 1); | |
1955 | ||
1956 | do { | |
1957 | rte_mbuf_refcnt_update(m, v); | |
1958 | } while ((m = m->next) != NULL); | |
1959 | } | |
1960 | ||
1961 | /** | |
1962 | * Get the headroom in a packet mbuf. | |
1963 | * | |
1964 | * @param m | |
1965 | * The packet mbuf. | |
1966 | * @return | |
1967 | * The length of the headroom. | |
1968 | */ | |
1969 | static inline uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m) | |
1970 | { | |
11fdf7f2 | 1971 | __rte_mbuf_sanity_check(m, 0); |
7c673cae FG |
1972 | return m->data_off; |
1973 | } | |
1974 | ||
1975 | /** | |
1976 | * Get the tailroom of a packet mbuf. | |
1977 | * | |
1978 | * @param m | |
1979 | * The packet mbuf. | |
1980 | * @return | |
1981 | * The length of the tailroom. | |
1982 | */ | |
1983 | static inline uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m) | |
1984 | { | |
11fdf7f2 | 1985 | __rte_mbuf_sanity_check(m, 0); |
7c673cae FG |
1986 | return (uint16_t)(m->buf_len - rte_pktmbuf_headroom(m) - |
1987 | m->data_len); | |
1988 | } | |
1989 | ||
1990 | /** | |
1991 | * Get the last segment of the packet. | |
1992 | * | |
1993 | * @param m | |
1994 | * The packet mbuf. | |
1995 | * @return | |
1996 | * The last segment of the given mbuf. | |
1997 | */ | |
1998 | static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m) | |
1999 | { | |
7c673cae | 2000 | __rte_mbuf_sanity_check(m, 1); |
11fdf7f2 TL |
2001 | while (m->next != NULL) |
2002 | m = m->next; | |
2003 | return m; | |
7c673cae FG |
2004 | } |
2005 | ||
2006 | /** | |
2007 | * A macro that points to an offset into the data in the mbuf. | |
2008 | * | |
2009 | * The returned pointer is cast to type t. Before using this | |
2010 | * function, the user must ensure that the first segment is large | |
2011 | * enough to accommodate its data. | |
2012 | * | |
2013 | * @param m | |
2014 | * The packet mbuf. | |
2015 | * @param o | |
2016 | * The offset into the mbuf data. | |
2017 | * @param t | |
2018 | * The type to cast the result into. | |
2019 | */ | |
2020 | #define rte_pktmbuf_mtod_offset(m, t, o) \ | |
2021 | ((t)((char *)(m)->buf_addr + (m)->data_off + (o))) | |
2022 | ||
2023 | /** | |
2024 | * A macro that points to the start of the data in the mbuf. | |
2025 | * | |
2026 | * The returned pointer is cast to type t. Before using this | |
2027 | * function, the user must ensure that the first segment is large | |
2028 | * enough to accommodate its data. | |
2029 | * | |
2030 | * @param m | |
2031 | * The packet mbuf. | |
2032 | * @param t | |
2033 | * The type to cast the result into. | |
2034 | */ | |
2035 | #define rte_pktmbuf_mtod(m, t) rte_pktmbuf_mtod_offset(m, t, 0) | |
2036 | ||
2037 | /** | |
11fdf7f2 | 2038 | * A macro that returns the IO address that points to an offset of the |
7c673cae FG |
2039 | * start of the data in the mbuf |
2040 | * | |
2041 | * @param m | |
2042 | * The packet mbuf. | |
2043 | * @param o | |
2044 | * The offset into the data to calculate address from. | |
2045 | */ | |
11fdf7f2 TL |
2046 | #define rte_pktmbuf_iova_offset(m, o) \ |
2047 | (rte_iova_t)((m)->buf_iova + (m)->data_off + (o)) | |
2048 | ||
2049 | /* deprecated */ | |
7c673cae | 2050 | #define rte_pktmbuf_mtophys_offset(m, o) \ |
11fdf7f2 | 2051 | rte_pktmbuf_iova_offset(m, o) |
7c673cae FG |
2052 | |
2053 | /** | |
11fdf7f2 | 2054 | * A macro that returns the IO address that points to the start of the |
7c673cae FG |
2055 | * data in the mbuf |
2056 | * | |
2057 | * @param m | |
2058 | * The packet mbuf. | |
2059 | */ | |
11fdf7f2 TL |
2060 | #define rte_pktmbuf_iova(m) rte_pktmbuf_iova_offset(m, 0) |
2061 | ||
2062 | /* deprecated */ | |
2063 | #define rte_pktmbuf_mtophys(m) rte_pktmbuf_iova(m) | |
7c673cae FG |
2064 | |
2065 | /** | |
2066 | * A macro that returns the length of the packet. | |
2067 | * | |
2068 | * The value can be read or assigned. | |
2069 | * | |
2070 | * @param m | |
2071 | * The packet mbuf. | |
2072 | */ | |
2073 | #define rte_pktmbuf_pkt_len(m) ((m)->pkt_len) | |
2074 | ||
2075 | /** | |
2076 | * A macro that returns the length of the segment. | |
2077 | * | |
2078 | * The value can be read or assigned. | |
2079 | * | |
2080 | * @param m | |
2081 | * The packet mbuf. | |
2082 | */ | |
2083 | #define rte_pktmbuf_data_len(m) ((m)->data_len) | |
2084 | ||
2085 | /** | |
2086 | * Prepend len bytes to an mbuf data area. | |
2087 | * | |
2088 | * Returns a pointer to the new | |
2089 | * data start address. If there is not enough headroom in the first | |
2090 | * segment, the function will return NULL, without modifying the mbuf. | |
2091 | * | |
2092 | * @param m | |
2093 | * The pkt mbuf. | |
2094 | * @param len | |
2095 | * The amount of data to prepend (in bytes). | |
2096 | * @return | |
2097 | * A pointer to the start of the newly prepended data, or | |
2098 | * NULL if there is not enough headroom space in the first segment | |
2099 | */ | |
2100 | static inline char *rte_pktmbuf_prepend(struct rte_mbuf *m, | |
2101 | uint16_t len) | |
2102 | { | |
2103 | __rte_mbuf_sanity_check(m, 1); | |
2104 | ||
2105 | if (unlikely(len > rte_pktmbuf_headroom(m))) | |
2106 | return NULL; | |
2107 | ||
11fdf7f2 TL |
2108 | /* NB: elaborating the subtraction like this instead of using |
2109 | * -= allows us to ensure the result type is uint16_t | |
2110 | * avoiding compiler warnings on gcc 8.1 at least */ | |
2111 | m->data_off = (uint16_t)(m->data_off - len); | |
7c673cae FG |
2112 | m->data_len = (uint16_t)(m->data_len + len); |
2113 | m->pkt_len = (m->pkt_len + len); | |
2114 | ||
2115 | return (char *)m->buf_addr + m->data_off; | |
2116 | } | |
2117 | ||
2118 | /** | |
2119 | * Append len bytes to an mbuf. | |
2120 | * | |
2121 | * Append len bytes to an mbuf and return a pointer to the start address | |
2122 | * of the added data. If there is not enough tailroom in the last | |
2123 | * segment, the function will return NULL, without modifying the mbuf. | |
2124 | * | |
2125 | * @param m | |
2126 | * The packet mbuf. | |
2127 | * @param len | |
2128 | * The amount of data to append (in bytes). | |
2129 | * @return | |
2130 | * A pointer to the start of the newly appended data, or | |
2131 | * NULL if there is not enough tailroom space in the last segment | |
2132 | */ | |
2133 | static inline char *rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len) | |
2134 | { | |
2135 | void *tail; | |
2136 | struct rte_mbuf *m_last; | |
2137 | ||
2138 | __rte_mbuf_sanity_check(m, 1); | |
2139 | ||
2140 | m_last = rte_pktmbuf_lastseg(m); | |
2141 | if (unlikely(len > rte_pktmbuf_tailroom(m_last))) | |
2142 | return NULL; | |
2143 | ||
2144 | tail = (char *)m_last->buf_addr + m_last->data_off + m_last->data_len; | |
2145 | m_last->data_len = (uint16_t)(m_last->data_len + len); | |
2146 | m->pkt_len = (m->pkt_len + len); | |
2147 | return (char*) tail; | |
2148 | } | |
2149 | ||
2150 | /** | |
2151 | * Remove len bytes at the beginning of an mbuf. | |
2152 | * | |
2153 | * Returns a pointer to the start address of the new data area. If the | |
2154 | * length is greater than the length of the first segment, then the | |
2155 | * function will fail and return NULL, without modifying the mbuf. | |
2156 | * | |
2157 | * @param m | |
2158 | * The packet mbuf. | |
2159 | * @param len | |
2160 | * The amount of data to remove (in bytes). | |
2161 | * @return | |
2162 | * A pointer to the new start of the data. | |
2163 | */ | |
2164 | static inline char *rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len) | |
2165 | { | |
2166 | __rte_mbuf_sanity_check(m, 1); | |
2167 | ||
2168 | if (unlikely(len > m->data_len)) | |
2169 | return NULL; | |
2170 | ||
11fdf7f2 TL |
2171 | /* NB: elaborating the addition like this instead of using |
2172 | * += allows us to ensure the result type is uint16_t | |
2173 | * avoiding compiler warnings on gcc 8.1 at least */ | |
7c673cae | 2174 | m->data_len = (uint16_t)(m->data_len - len); |
11fdf7f2 | 2175 | m->data_off = (uint16_t)(m->data_off + len); |
7c673cae FG |
2176 | m->pkt_len = (m->pkt_len - len); |
2177 | return (char *)m->buf_addr + m->data_off; | |
2178 | } | |
2179 | ||
2180 | /** | |
2181 | * Remove len bytes of data at the end of the mbuf. | |
2182 | * | |
2183 | * If the length is greater than the length of the last segment, the | |
2184 | * function will fail and return -1 without modifying the mbuf. | |
2185 | * | |
2186 | * @param m | |
2187 | * The packet mbuf. | |
2188 | * @param len | |
2189 | * The amount of data to remove (in bytes). | |
2190 | * @return | |
2191 | * - 0: On success. | |
2192 | * - -1: On error. | |
2193 | */ | |
2194 | static inline int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len) | |
2195 | { | |
2196 | struct rte_mbuf *m_last; | |
2197 | ||
2198 | __rte_mbuf_sanity_check(m, 1); | |
2199 | ||
2200 | m_last = rte_pktmbuf_lastseg(m); | |
2201 | if (unlikely(len > m_last->data_len)) | |
2202 | return -1; | |
2203 | ||
2204 | m_last->data_len = (uint16_t)(m_last->data_len - len); | |
2205 | m->pkt_len = (m->pkt_len - len); | |
2206 | return 0; | |
2207 | } | |
2208 | ||
2209 | /** | |
2210 | * Test if mbuf data is contiguous. | |
2211 | * | |
2212 | * @param m | |
2213 | * The packet mbuf. | |
2214 | * @return | |
2215 | * - 1, if all data is contiguous (one segment). | |
2216 | * - 0, if there is several segments. | |
2217 | */ | |
2218 | static inline int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m) | |
2219 | { | |
2220 | __rte_mbuf_sanity_check(m, 1); | |
2221 | return !!(m->nb_segs == 1); | |
2222 | } | |
2223 | ||
2224 | /** | |
2225 | * @internal used by rte_pktmbuf_read(). | |
2226 | */ | |
2227 | const void *__rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off, | |
2228 | uint32_t len, void *buf); | |
2229 | ||
2230 | /** | |
2231 | * Read len data bytes in a mbuf at specified offset. | |
2232 | * | |
2233 | * If the data is contiguous, return the pointer in the mbuf data, else | |
2234 | * copy the data in the buffer provided by the user and return its | |
2235 | * pointer. | |
2236 | * | |
2237 | * @param m | |
2238 | * The pointer to the mbuf. | |
2239 | * @param off | |
2240 | * The offset of the data in the mbuf. | |
2241 | * @param len | |
2242 | * The amount of bytes to read. | |
2243 | * @param buf | |
11fdf7f2 | 2244 | * The buffer where data is copied if it is not contiguous in mbuf |
7c673cae FG |
2245 | * data. Its length should be at least equal to the len parameter. |
2246 | * @return | |
2247 | * The pointer to the data, either in the mbuf if it is contiguous, | |
2248 | * or in the user buffer. If mbuf is too small, NULL is returned. | |
2249 | */ | |
2250 | static inline const void *rte_pktmbuf_read(const struct rte_mbuf *m, | |
2251 | uint32_t off, uint32_t len, void *buf) | |
2252 | { | |
2253 | if (likely(off + len <= rte_pktmbuf_data_len(m))) | |
2254 | return rte_pktmbuf_mtod_offset(m, char *, off); | |
2255 | else | |
2256 | return __rte_pktmbuf_read(m, off, len, buf); | |
2257 | } | |
2258 | ||
2259 | /** | |
2260 | * Chain an mbuf to another, thereby creating a segmented packet. | |
2261 | * | |
2262 | * Note: The implementation will do a linear walk over the segments to find | |
2263 | * the tail entry. For cases when there are many segments, it's better to | |
2264 | * chain the entries manually. | |
2265 | * | |
2266 | * @param head | |
2267 | * The head of the mbuf chain (the first packet) | |
2268 | * @param tail | |
2269 | * The mbuf to put last in the chain | |
2270 | * | |
2271 | * @return | |
2272 | * - 0, on success. | |
11fdf7f2 | 2273 | * - -EOVERFLOW, if the chain segment limit exceeded |
7c673cae FG |
2274 | */ |
2275 | static inline int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail) | |
2276 | { | |
2277 | struct rte_mbuf *cur_tail; | |
2278 | ||
2279 | /* Check for number-of-segments-overflow */ | |
11fdf7f2 | 2280 | if (head->nb_segs + tail->nb_segs > RTE_MBUF_MAX_NB_SEGS) |
7c673cae FG |
2281 | return -EOVERFLOW; |
2282 | ||
2283 | /* Chain 'tail' onto the old tail */ | |
2284 | cur_tail = rte_pktmbuf_lastseg(head); | |
2285 | cur_tail->next = tail; | |
2286 | ||
11fdf7f2 TL |
2287 | /* accumulate number of segments and total length. |
2288 | * NB: elaborating the addition like this instead of using | |
2289 | * -= allows us to ensure the result type is uint16_t | |
2290 | * avoiding compiler warnings on gcc 8.1 at least */ | |
2291 | head->nb_segs = (uint16_t)(head->nb_segs + tail->nb_segs); | |
7c673cae FG |
2292 | head->pkt_len += tail->pkt_len; |
2293 | ||
2294 | /* pkt_len is only set in the head */ | |
2295 | tail->pkt_len = tail->data_len; | |
2296 | ||
2297 | return 0; | |
2298 | } | |
2299 | ||
9f95a23c TL |
2300 | /* |
2301 | * @warning | |
2302 | * @b EXPERIMENTAL: This API may change without prior notice. | |
2303 | * | |
2304 | * For given input values generate raw tx_offload value. | |
2305 | * Note that it is caller responsibility to make sure that input parameters | |
2306 | * don't exceed maximum bit-field values. | |
2307 | * @param il2 | |
2308 | * l2_len value. | |
2309 | * @param il3 | |
2310 | * l3_len value. | |
2311 | * @param il4 | |
2312 | * l4_len value. | |
2313 | * @param tso | |
2314 | * tso_segsz value. | |
2315 | * @param ol3 | |
2316 | * outer_l3_len value. | |
2317 | * @param ol2 | |
2318 | * outer_l2_len value. | |
2319 | * @param unused | |
2320 | * unused value. | |
2321 | * @return | |
2322 | * raw tx_offload value. | |
2323 | */ | |
2324 | static __rte_always_inline uint64_t | |
2325 | rte_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso, | |
2326 | uint64_t ol3, uint64_t ol2, uint64_t unused) | |
2327 | { | |
2328 | return il2 << RTE_MBUF_L2_LEN_OFS | | |
2329 | il3 << RTE_MBUF_L3_LEN_OFS | | |
2330 | il4 << RTE_MBUF_L4_LEN_OFS | | |
2331 | tso << RTE_MBUF_TSO_SEGSZ_OFS | | |
2332 | ol3 << RTE_MBUF_OUTL3_LEN_OFS | | |
2333 | ol2 << RTE_MBUF_OUTL2_LEN_OFS | | |
2334 | unused << RTE_MBUF_TXOFLD_UNUSED_OFS; | |
2335 | } | |
2336 | ||
11fdf7f2 TL |
2337 | /** |
2338 | * Validate general requirements for Tx offload in mbuf. | |
2339 | * | |
2340 | * This function checks correctness and completeness of Tx offload settings. | |
2341 | * | |
2342 | * @param m | |
2343 | * The packet mbuf to be validated. | |
2344 | * @return | |
2345 | * 0 if packet is valid | |
2346 | */ | |
2347 | static inline int | |
2348 | rte_validate_tx_offload(const struct rte_mbuf *m) | |
2349 | { | |
2350 | uint64_t ol_flags = m->ol_flags; | |
11fdf7f2 TL |
2351 | |
2352 | /* Does packet set any of available offloads? */ | |
2353 | if (!(ol_flags & PKT_TX_OFFLOAD_MASK)) | |
2354 | return 0; | |
2355 | ||
11fdf7f2 TL |
2356 | /* IP checksum can be counted only for IPv4 packet */ |
2357 | if ((ol_flags & PKT_TX_IP_CKSUM) && (ol_flags & PKT_TX_IPV6)) | |
2358 | return -EINVAL; | |
2359 | ||
2360 | /* IP type not set when required */ | |
2361 | if (ol_flags & (PKT_TX_L4_MASK | PKT_TX_TCP_SEG)) | |
2362 | if (!(ol_flags & (PKT_TX_IPV4 | PKT_TX_IPV6))) | |
2363 | return -EINVAL; | |
2364 | ||
2365 | /* Check requirements for TSO packet */ | |
2366 | if (ol_flags & PKT_TX_TCP_SEG) | |
2367 | if ((m->tso_segsz == 0) || | |
2368 | ((ol_flags & PKT_TX_IPV4) && | |
2369 | !(ol_flags & PKT_TX_IP_CKSUM))) | |
2370 | return -EINVAL; | |
2371 | ||
2372 | /* PKT_TX_OUTER_IP_CKSUM set for non outer IPv4 packet. */ | |
2373 | if ((ol_flags & PKT_TX_OUTER_IP_CKSUM) && | |
2374 | !(ol_flags & PKT_TX_OUTER_IPV4)) | |
2375 | return -EINVAL; | |
2376 | ||
2377 | return 0; | |
2378 | } | |
2379 | ||
2380 | /** | |
2381 | * Linearize data in mbuf. | |
2382 | * | |
2383 | * This function moves the mbuf data in the first segment if there is enough | |
2384 | * tailroom. The subsequent segments are unchained and freed. | |
2385 | * | |
2386 | * @param mbuf | |
2387 | * mbuf to linearize | |
2388 | * @return | |
2389 | * - 0, on success | |
2390 | * - -1, on error | |
2391 | */ | |
2392 | static inline int | |
2393 | rte_pktmbuf_linearize(struct rte_mbuf *mbuf) | |
2394 | { | |
2395 | size_t seg_len, copy_len; | |
2396 | struct rte_mbuf *m; | |
2397 | struct rte_mbuf *m_next; | |
2398 | char *buffer; | |
2399 | ||
2400 | if (rte_pktmbuf_is_contiguous(mbuf)) | |
2401 | return 0; | |
2402 | ||
2403 | /* Extend first segment to the total packet length */ | |
2404 | copy_len = rte_pktmbuf_pkt_len(mbuf) - rte_pktmbuf_data_len(mbuf); | |
2405 | ||
2406 | if (unlikely(copy_len > rte_pktmbuf_tailroom(mbuf))) | |
2407 | return -1; | |
2408 | ||
2409 | buffer = rte_pktmbuf_mtod_offset(mbuf, char *, mbuf->data_len); | |
2410 | mbuf->data_len = (uint16_t)(mbuf->pkt_len); | |
2411 | ||
2412 | /* Append data from next segments to the first one */ | |
2413 | m = mbuf->next; | |
2414 | while (m != NULL) { | |
2415 | m_next = m->next; | |
2416 | ||
2417 | seg_len = rte_pktmbuf_data_len(m); | |
2418 | rte_memcpy(buffer, rte_pktmbuf_mtod(m, char *), seg_len); | |
2419 | buffer += seg_len; | |
2420 | ||
2421 | rte_pktmbuf_free_seg(m); | |
2422 | m = m_next; | |
2423 | } | |
2424 | ||
2425 | mbuf->next = NULL; | |
2426 | mbuf->nb_segs = 1; | |
2427 | ||
2428 | return 0; | |
2429 | } | |
2430 | ||
7c673cae FG |
2431 | /** |
2432 | * Dump an mbuf structure to a file. | |
2433 | * | |
2434 | * Dump all fields for the given packet mbuf and all its associated | |
2435 | * segments (in the case of a chained buffer). | |
2436 | * | |
2437 | * @param f | |
2438 | * A pointer to a file for output | |
2439 | * @param m | |
2440 | * The packet mbuf. | |
2441 | * @param dump_len | |
2442 | * If dump_len != 0, also dump the "dump_len" first data bytes of | |
2443 | * the packet. | |
2444 | */ | |
2445 | void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len); | |
2446 | ||
9f95a23c TL |
2447 | /** |
2448 | * Get the value of mbuf sched queue_id field. | |
2449 | */ | |
2450 | static inline uint32_t | |
2451 | rte_mbuf_sched_queue_get(const struct rte_mbuf *m) | |
2452 | { | |
2453 | return m->hash.sched.queue_id; | |
2454 | } | |
2455 | ||
2456 | /** | |
2457 | * Get the value of mbuf sched traffic_class field. | |
2458 | */ | |
2459 | static inline uint8_t | |
2460 | rte_mbuf_sched_traffic_class_get(const struct rte_mbuf *m) | |
2461 | { | |
2462 | return m->hash.sched.traffic_class; | |
2463 | } | |
2464 | ||
2465 | /** | |
2466 | * Get the value of mbuf sched color field. | |
2467 | */ | |
2468 | static inline uint8_t | |
2469 | rte_mbuf_sched_color_get(const struct rte_mbuf *m) | |
2470 | { | |
2471 | return m->hash.sched.color; | |
2472 | } | |
2473 | ||
2474 | /** | |
2475 | * Get the values of mbuf sched queue_id, traffic_class and color. | |
2476 | * | |
2477 | * @param m | |
2478 | * Mbuf to read | |
2479 | * @param queue_id | |
2480 | * Returns the queue id | |
2481 | * @param traffic_class | |
2482 | * Returns the traffic class id | |
2483 | * @param color | |
2484 | * Returns the colour id | |
2485 | */ | |
2486 | static inline void | |
2487 | rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id, | |
2488 | uint8_t *traffic_class, | |
2489 | uint8_t *color) | |
2490 | { | |
2491 | struct rte_mbuf_sched sched = m->hash.sched; | |
2492 | ||
2493 | *queue_id = sched.queue_id; | |
2494 | *traffic_class = sched.traffic_class; | |
2495 | *color = sched.color; | |
2496 | } | |
2497 | ||
2498 | /** | |
2499 | * Set the mbuf sched queue_id to the defined value. | |
2500 | */ | |
2501 | static inline void | |
2502 | rte_mbuf_sched_queue_set(struct rte_mbuf *m, uint32_t queue_id) | |
2503 | { | |
2504 | m->hash.sched.queue_id = queue_id; | |
2505 | } | |
2506 | ||
2507 | /** | |
2508 | * Set the mbuf sched traffic_class id to the defined value. | |
2509 | */ | |
2510 | static inline void | |
2511 | rte_mbuf_sched_traffic_class_set(struct rte_mbuf *m, uint8_t traffic_class) | |
2512 | { | |
2513 | m->hash.sched.traffic_class = traffic_class; | |
2514 | } | |
2515 | ||
2516 | /** | |
2517 | * Set the mbuf sched color id to the defined value. | |
2518 | */ | |
2519 | static inline void | |
2520 | rte_mbuf_sched_color_set(struct rte_mbuf *m, uint8_t color) | |
2521 | { | |
2522 | m->hash.sched.color = color; | |
2523 | } | |
2524 | ||
2525 | /** | |
2526 | * Set the mbuf sched queue_id, traffic_class and color. | |
2527 | * | |
2528 | * @param m | |
2529 | * Mbuf to set | |
2530 | * @param queue_id | |
2531 | * Queue id value to be set | |
2532 | * @param traffic_class | |
2533 | * Traffic class id value to be set | |
2534 | * @param color | |
2535 | * Color id to be set | |
2536 | */ | |
2537 | static inline void | |
2538 | rte_mbuf_sched_set(struct rte_mbuf *m, uint32_t queue_id, | |
2539 | uint8_t traffic_class, | |
2540 | uint8_t color) | |
2541 | { | |
2542 | m->hash.sched = (struct rte_mbuf_sched){ | |
2543 | .queue_id = queue_id, | |
2544 | .traffic_class = traffic_class, | |
2545 | .color = color, | |
2546 | .reserved = 0, | |
2547 | }; | |
2548 | } | |
2549 | ||
7c673cae FG |
2550 | #ifdef __cplusplus |
2551 | } | |
2552 | #endif | |
2553 | ||
2554 | #endif /* _RTE_MBUF_H_ */ |