]>
Commit | Line | Data |
---|---|---|
91088554 | 1 | /* |
cf62fa4c | 2 | * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013 Nicira, Inc. |
91088554 DDP |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | * you may not use this file except in compliance with the License. | |
6 | * You may obtain a copy of the License at: | |
7 | * | |
8 | * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | * | |
10 | * Unless required by applicable law or agreed to in writing, software | |
11 | * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | * See the License for the specific language governing permissions and | |
14 | * limitations under the License. | |
15 | */ | |
16 | ||
cf62fa4c PS |
17 | #ifndef DPBUF_H |
18 | #define DPBUF_H 1 | |
91088554 | 19 | |
cf62fa4c PS |
20 | #include <stddef.h> |
21 | #include <stdint.h> | |
01961bbd DDP |
22 | |
23 | #ifdef DPDK_NETDEV | |
24 | #include <rte_config.h> | |
25 | #include <rte_mbuf.h> | |
26 | #endif | |
27 | ||
28 | #include "netdev-dpdk.h" | |
b19bab5b | 29 | #include "openvswitch/list.h" |
cf62fa4c PS |
30 | #include "packets.h" |
31 | #include "util.h" | |
91088554 DDP |
32 | |
33 | #ifdef __cplusplus | |
34 | extern "C" { | |
35 | #endif | |
36 | ||
cf62fa4c PS |
37 | enum OVS_PACKED_ENUM dp_packet_source { |
38 | DPBUF_MALLOC, /* Obtained via malloc(). */ | |
39 | DPBUF_STACK, /* Un-movable stack space or static buffer. */ | |
40 | DPBUF_STUB, /* Starts on stack, may expand into heap. */ | |
41 | DPBUF_DPDK, /* buffer data is from DPDK allocated memory. | |
88cbc2dd | 42 | * ref to dp_packet_init_dpdk() in dp-packet.c. */ |
cf62fa4c | 43 | }; |
91088554 | 44 | |
eb18b2a6 PS |
45 | #define DP_PACKET_CONTEXT_SIZE 64 |
46 | ||
82eb5b0a | 47 | /* Buffer for holding packet data. A dp_packet is automatically reallocated |
cf62fa4c | 48 | * as necessary if it grows too large for the available memory. |
cf62fa4c | 49 | */ |
e14deea0 | 50 | struct dp_packet { |
cf62fa4c PS |
51 | #ifdef DPDK_NETDEV |
52 | struct rte_mbuf mbuf; /* DPDK mbuf */ | |
53 | #else | |
b8e57534 | 54 | void *base_; /* First byte of allocated space. */ |
11a6fbd5 | 55 | uint16_t allocated_; /* Number of bytes allocated. */ |
b8e57534 MK |
56 | uint16_t data_ofs; /* First byte actually in use. */ |
57 | uint32_t size_; /* Number of bytes in use. */ | |
2bc1bbd2 | 58 | uint32_t rss_hash; /* Packet hash. */ |
f2f44f5d | 59 | bool rss_hash_valid; /* Is the 'rss_hash' valid? */ |
61a2647e | 60 | #endif |
cf62fa4c | 61 | enum dp_packet_source source; /* Source of memory allocated as 'base'. */ |
82eb5b0a DDP |
62 | uint8_t l2_pad_size; /* Detected l2 padding size. |
63 | * Padding is non-pullable. */ | |
64 | uint16_t l2_5_ofs; /* MPLS label stack offset, or UINT16_MAX */ | |
65 | uint16_t l3_ofs; /* Network-level header offset, | |
66 | * or UINT16_MAX. */ | |
67 | uint16_t l4_ofs; /* Transport-level header offset, | |
68 | or UINT16_MAX. */ | |
aaca4fe0 | 69 | uint32_t cutlen; /* length in bytes to cut from the end. */ |
eb18b2a6 PS |
70 | union { |
71 | struct pkt_metadata md; | |
72 | uint64_t data[DP_PACKET_CONTEXT_SIZE / 8]; | |
73 | }; | |
91088554 DDP |
74 | }; |
75 | ||
5a07c6e1 | 76 | static inline void *dp_packet_data(const struct dp_packet *); |
cf62fa4c | 77 | static inline void dp_packet_set_data(struct dp_packet *, void *); |
5a07c6e1 | 78 | static inline void *dp_packet_base(const struct dp_packet *); |
cf62fa4c PS |
79 | static inline void dp_packet_set_base(struct dp_packet *, void *); |
80 | ||
81 | static inline uint32_t dp_packet_size(const struct dp_packet *); | |
82 | static inline void dp_packet_set_size(struct dp_packet *, uint32_t); | |
83 | ||
11a6fbd5 DDP |
84 | static inline uint16_t dp_packet_get_allocated(const struct dp_packet *); |
85 | static inline void dp_packet_set_allocated(struct dp_packet *, uint16_t); | |
86 | ||
5a07c6e1 DDP |
87 | void *dp_packet_resize_l2(struct dp_packet *, int increment); |
88 | void *dp_packet_resize_l2_5(struct dp_packet *, int increment); | |
89 | static inline void *dp_packet_l2(const struct dp_packet *); | |
82eb5b0a | 90 | static inline void dp_packet_reset_offsets(struct dp_packet *); |
cf62fa4c PS |
91 | static inline uint8_t dp_packet_l2_pad_size(const struct dp_packet *); |
92 | static inline void dp_packet_set_l2_pad_size(struct dp_packet *, uint8_t); | |
5a07c6e1 | 93 | static inline void *dp_packet_l2_5(const struct dp_packet *); |
cf62fa4c | 94 | static inline void dp_packet_set_l2_5(struct dp_packet *, void *); |
5a07c6e1 | 95 | static inline void *dp_packet_l3(const struct dp_packet *); |
cf62fa4c | 96 | static inline void dp_packet_set_l3(struct dp_packet *, void *); |
5a07c6e1 | 97 | static inline void *dp_packet_l4(const struct dp_packet *); |
cf62fa4c PS |
98 | static inline void dp_packet_set_l4(struct dp_packet *, void *); |
99 | static inline size_t dp_packet_l4_size(const struct dp_packet *); | |
100 | static inline const void *dp_packet_get_tcp_payload(const struct dp_packet *); | |
101 | static inline const void *dp_packet_get_udp_payload(const struct dp_packet *); | |
102 | static inline const void *dp_packet_get_sctp_payload(const struct dp_packet *); | |
103 | static inline const void *dp_packet_get_icmp_payload(const struct dp_packet *); | |
104 | static inline const void *dp_packet_get_nd_payload(const struct dp_packet *); | |
105 | ||
106 | void dp_packet_use(struct dp_packet *, void *, size_t); | |
107 | void dp_packet_use_stub(struct dp_packet *, void *, size_t); | |
108 | void dp_packet_use_const(struct dp_packet *, const void *, size_t); | |
109 | ||
5a07c6e1 | 110 | void dp_packet_init_dpdk(struct dp_packet *, size_t allocated); |
cf62fa4c PS |
111 | |
112 | void dp_packet_init(struct dp_packet *, size_t); | |
113 | void dp_packet_uninit(struct dp_packet *); | |
cf62fa4c PS |
114 | |
115 | struct dp_packet *dp_packet_new(size_t); | |
116 | struct dp_packet *dp_packet_new_with_headroom(size_t, size_t headroom); | |
117 | struct dp_packet *dp_packet_clone(const struct dp_packet *); | |
118 | struct dp_packet *dp_packet_clone_with_headroom(const struct dp_packet *, | |
5a07c6e1 | 119 | size_t headroom); |
cf62fa4c PS |
120 | struct dp_packet *dp_packet_clone_data(const void *, size_t); |
121 | struct dp_packet *dp_packet_clone_data_with_headroom(const void *, size_t, | |
5a07c6e1 | 122 | size_t headroom); |
cf62fa4c PS |
123 | static inline void dp_packet_delete(struct dp_packet *); |
124 | ||
125 | static inline void *dp_packet_at(const struct dp_packet *, size_t offset, | |
5a07c6e1 DDP |
126 | size_t size); |
127 | static inline void *dp_packet_at_assert(const struct dp_packet *, | |
128 | size_t offset, size_t size); | |
cf62fa4c PS |
129 | static inline void *dp_packet_tail(const struct dp_packet *); |
130 | static inline void *dp_packet_end(const struct dp_packet *); | |
131 | ||
132 | void *dp_packet_put_uninit(struct dp_packet *, size_t); | |
133 | void *dp_packet_put_zeros(struct dp_packet *, size_t); | |
134 | void *dp_packet_put(struct dp_packet *, const void *, size_t); | |
135 | char *dp_packet_put_hex(struct dp_packet *, const char *s, size_t *n); | |
136 | void dp_packet_reserve(struct dp_packet *, size_t); | |
5a07c6e1 DDP |
137 | void dp_packet_reserve_with_tailroom(struct dp_packet *, size_t headroom, |
138 | size_t tailroom); | |
139 | void *dp_packet_push_uninit(struct dp_packet *, size_t); | |
cf62fa4c | 140 | void *dp_packet_push_zeros(struct dp_packet *, size_t); |
5a07c6e1 | 141 | void *dp_packet_push(struct dp_packet *, const void *, size_t); |
cf62fa4c PS |
142 | |
143 | static inline size_t dp_packet_headroom(const struct dp_packet *); | |
144 | static inline size_t dp_packet_tailroom(const struct dp_packet *); | |
145 | void dp_packet_prealloc_headroom(struct dp_packet *, size_t); | |
146 | void dp_packet_prealloc_tailroom(struct dp_packet *, size_t); | |
147 | void dp_packet_shift(struct dp_packet *, int); | |
91088554 | 148 | |
cf62fa4c PS |
149 | static inline void dp_packet_clear(struct dp_packet *); |
150 | static inline void *dp_packet_pull(struct dp_packet *, size_t); | |
151 | static inline void *dp_packet_try_pull(struct dp_packet *, size_t); | |
91088554 | 152 | |
cf62fa4c | 153 | void *dp_packet_steal_data(struct dp_packet *); |
91088554 | 154 | |
5a07c6e1 DDP |
155 | static inline bool dp_packet_equal(const struct dp_packet *, |
156 | const struct dp_packet *); | |
cf62fa4c PS |
157 | |
158 | \f | |
cf62fa4c | 159 | /* Frees memory that 'b' points to, as well as 'b' itself. */ |
5a07c6e1 DDP |
160 | static inline void |
161 | dp_packet_delete(struct dp_packet *b) | |
cf62fa4c PS |
162 | { |
163 | if (b) { | |
164 | if (b->source == DPBUF_DPDK) { | |
165 | /* If this dp_packet was allocated by DPDK it must have been | |
166 | * created as a dp_packet */ | |
167 | free_dpdk_buf((struct dp_packet*) b); | |
168 | return; | |
169 | } | |
170 | ||
171 | dp_packet_uninit(b); | |
172 | free(b); | |
173 | } | |
174 | } | |
175 | ||
176 | /* If 'b' contains at least 'offset + size' bytes of data, returns a pointer to | |
177 | * byte 'offset'. Otherwise, returns a null pointer. */ | |
5a07c6e1 DDP |
178 | static inline void * |
179 | dp_packet_at(const struct dp_packet *b, size_t offset, size_t size) | |
91088554 | 180 | { |
5a07c6e1 DDP |
181 | return offset + size <= dp_packet_size(b) |
182 | ? (char *) dp_packet_data(b) + offset | |
183 | : NULL; | |
cf62fa4c | 184 | } |
91088554 | 185 | |
cf62fa4c PS |
186 | /* Returns a pointer to byte 'offset' in 'b', which must contain at least |
187 | * 'offset + size' bytes of data. */ | |
5a07c6e1 DDP |
188 | static inline void * |
189 | dp_packet_at_assert(const struct dp_packet *b, size_t offset, size_t size) | |
cf62fa4c PS |
190 | { |
191 | ovs_assert(offset + size <= dp_packet_size(b)); | |
192 | return ((char *) dp_packet_data(b)) + offset; | |
193 | } | |
194 | ||
195 | /* Returns a pointer to byte following the last byte of data in use in 'b'. */ | |
5a07c6e1 DDP |
196 | static inline void * |
197 | dp_packet_tail(const struct dp_packet *b) | |
cf62fa4c PS |
198 | { |
199 | return (char *) dp_packet_data(b) + dp_packet_size(b); | |
200 | } | |
201 | ||
202 | /* Returns a pointer to byte following the last byte allocated for use (but | |
203 | * not necessarily in use) in 'b'. */ | |
5a07c6e1 DDP |
204 | static inline void * |
205 | dp_packet_end(const struct dp_packet *b) | |
cf62fa4c | 206 | { |
11a6fbd5 | 207 | return (char *) dp_packet_base(b) + dp_packet_get_allocated(b); |
cf62fa4c PS |
208 | } |
209 | ||
210 | /* Returns the number of bytes of headroom in 'b', that is, the number of bytes | |
211 | * of unused space in dp_packet 'b' before the data that is in use. (Most | |
5a07c6e1 DDP |
212 | * commonly, the data in a dp_packet is at its beginning, and thus the |
213 | * dp_packet's headroom is 0.) */ | |
214 | static inline size_t | |
215 | dp_packet_headroom(const struct dp_packet *b) | |
cf62fa4c | 216 | { |
5a07c6e1 | 217 | return (char *) dp_packet_data(b) - (char *) dp_packet_base(b); |
cf62fa4c PS |
218 | } |
219 | ||
5a07c6e1 DDP |
220 | /* Returns the number of bytes that may be appended to the tail end of |
221 | * dp_packet 'b' before the dp_packet must be reallocated. */ | |
222 | static inline size_t | |
223 | dp_packet_tailroom(const struct dp_packet *b) | |
cf62fa4c | 224 | { |
5a07c6e1 | 225 | return (char *) dp_packet_end(b) - (char *) dp_packet_tail(b); |
cf62fa4c PS |
226 | } |
227 | ||
228 | /* Clears any data from 'b'. */ | |
5a07c6e1 DDP |
229 | static inline void |
230 | dp_packet_clear(struct dp_packet *b) | |
cf62fa4c PS |
231 | { |
232 | dp_packet_set_data(b, dp_packet_base(b)); | |
233 | dp_packet_set_size(b, 0); | |
234 | } | |
235 | ||
236 | /* Removes 'size' bytes from the head end of 'b', which must contain at least | |
237 | * 'size' bytes of data. Returns the first byte of data removed. */ | |
5a07c6e1 DDP |
238 | static inline void * |
239 | dp_packet_pull(struct dp_packet *b, size_t size) | |
cf62fa4c PS |
240 | { |
241 | void *data = dp_packet_data(b); | |
242 | ovs_assert(dp_packet_size(b) - dp_packet_l2_pad_size(b) >= size); | |
5a07c6e1 | 243 | dp_packet_set_data(b, (char *) dp_packet_data(b) + size); |
cf62fa4c PS |
244 | dp_packet_set_size(b, dp_packet_size(b) - size); |
245 | return data; | |
246 | } | |
247 | ||
248 | /* If 'b' has at least 'size' bytes of data, removes that many bytes from the | |
249 | * head end of 'b' and returns the first byte removed. Otherwise, returns a | |
250 | * null pointer without modifying 'b'. */ | |
5a07c6e1 DDP |
251 | static inline void * |
252 | dp_packet_try_pull(struct dp_packet *b, size_t size) | |
cf62fa4c PS |
253 | { |
254 | return dp_packet_size(b) - dp_packet_l2_pad_size(b) >= size | |
255 | ? dp_packet_pull(b, size) : NULL; | |
256 | } | |
257 | ||
5a07c6e1 DDP |
258 | static inline bool |
259 | dp_packet_equal(const struct dp_packet *a, const struct dp_packet *b) | |
cf62fa4c PS |
260 | { |
261 | return dp_packet_size(a) == dp_packet_size(b) && | |
5a07c6e1 | 262 | !memcmp(dp_packet_data(a), dp_packet_data(b), dp_packet_size(a)); |
cf62fa4c PS |
263 | } |
264 | ||
82eb5b0a | 265 | /* Get the start of the Ethernet frame. 'l3_ofs' marks the end of the l2 |
cf62fa4c | 266 | * headers, so return NULL if it is not set. */ |
5a07c6e1 DDP |
267 | static inline void * |
268 | dp_packet_l2(const struct dp_packet *b) | |
cf62fa4c | 269 | { |
82eb5b0a | 270 | return (b->l3_ofs != UINT16_MAX) ? dp_packet_data(b) : NULL; |
cf62fa4c PS |
271 | } |
272 | ||
82eb5b0a DDP |
273 | /* Resets all layer offsets. 'l3' offset must be set before 'l2' can be |
274 | * retrieved. */ | |
5a07c6e1 DDP |
275 | static inline void |
276 | dp_packet_reset_offsets(struct dp_packet *b) | |
cf62fa4c | 277 | { |
cf62fa4c PS |
278 | b->l2_pad_size = 0; |
279 | b->l2_5_ofs = UINT16_MAX; | |
280 | b->l3_ofs = UINT16_MAX; | |
281 | b->l4_ofs = UINT16_MAX; | |
282 | } | |
283 | ||
5a07c6e1 DDP |
284 | static inline uint8_t |
285 | dp_packet_l2_pad_size(const struct dp_packet *b) | |
cf62fa4c PS |
286 | { |
287 | return b->l2_pad_size; | |
288 | } | |
289 | ||
5a07c6e1 DDP |
290 | static inline void |
291 | dp_packet_set_l2_pad_size(struct dp_packet *b, uint8_t pad_size) | |
cf62fa4c PS |
292 | { |
293 | ovs_assert(pad_size <= dp_packet_size(b)); | |
294 | b->l2_pad_size = pad_size; | |
295 | } | |
296 | ||
5a07c6e1 DDP |
297 | static inline void * |
298 | dp_packet_l2_5(const struct dp_packet *b) | |
cf62fa4c | 299 | { |
82eb5b0a DDP |
300 | return b->l2_5_ofs != UINT16_MAX |
301 | ? (char *) dp_packet_data(b) + b->l2_5_ofs | |
302 | : NULL; | |
cf62fa4c PS |
303 | } |
304 | ||
5a07c6e1 DDP |
305 | static inline void |
306 | dp_packet_set_l2_5(struct dp_packet *b, void *l2_5) | |
cf62fa4c | 307 | { |
82eb5b0a DDP |
308 | b->l2_5_ofs = l2_5 |
309 | ? (char *) l2_5 - (char *) dp_packet_data(b) | |
310 | : UINT16_MAX; | |
cf62fa4c PS |
311 | } |
312 | ||
5a07c6e1 DDP |
313 | static inline void * |
314 | dp_packet_l3(const struct dp_packet *b) | |
cf62fa4c | 315 | { |
82eb5b0a DDP |
316 | return b->l3_ofs != UINT16_MAX |
317 | ? (char *) dp_packet_data(b) + b->l3_ofs | |
318 | : NULL; | |
cf62fa4c PS |
319 | } |
320 | ||
5a07c6e1 DDP |
321 | static inline void |
322 | dp_packet_set_l3(struct dp_packet *b, void *l3) | |
cf62fa4c | 323 | { |
82eb5b0a | 324 | b->l3_ofs = l3 ? (char *) l3 - (char *) dp_packet_data(b) : UINT16_MAX; |
cf62fa4c PS |
325 | } |
326 | ||
5a07c6e1 DDP |
327 | static inline void * |
328 | dp_packet_l4(const struct dp_packet *b) | |
cf62fa4c | 329 | { |
82eb5b0a DDP |
330 | return b->l4_ofs != UINT16_MAX |
331 | ? (char *) dp_packet_data(b) + b->l4_ofs | |
332 | : NULL; | |
cf62fa4c PS |
333 | } |
334 | ||
5a07c6e1 DDP |
335 | static inline void |
336 | dp_packet_set_l4(struct dp_packet *b, void *l4) | |
cf62fa4c | 337 | { |
82eb5b0a | 338 | b->l4_ofs = l4 ? (char *) l4 - (char *) dp_packet_data(b) : UINT16_MAX; |
cf62fa4c PS |
339 | } |
340 | ||
5a07c6e1 DDP |
341 | static inline size_t |
342 | dp_packet_l4_size(const struct dp_packet *b) | |
cf62fa4c PS |
343 | { |
344 | return b->l4_ofs != UINT16_MAX | |
345 | ? (const char *)dp_packet_tail(b) - (const char *)dp_packet_l4(b) | |
346 | - dp_packet_l2_pad_size(b) | |
347 | : 0; | |
348 | } | |
349 | ||
5a07c6e1 DDP |
350 | static inline const void * |
351 | dp_packet_get_tcp_payload(const struct dp_packet *b) | |
cf62fa4c PS |
352 | { |
353 | size_t l4_size = dp_packet_l4_size(b); | |
354 | ||
355 | if (OVS_LIKELY(l4_size >= TCP_HEADER_LEN)) { | |
356 | struct tcp_header *tcp = dp_packet_l4(b); | |
357 | int tcp_len = TCP_OFFSET(tcp->tcp_ctl) * 4; | |
358 | ||
359 | if (OVS_LIKELY(tcp_len >= TCP_HEADER_LEN && tcp_len <= l4_size)) { | |
360 | return (const char *)tcp + tcp_len; | |
361 | } | |
362 | } | |
363 | return NULL; | |
364 | } | |
365 | ||
5a07c6e1 DDP |
366 | static inline const void * |
367 | dp_packet_get_udp_payload(const struct dp_packet *b) | |
cf62fa4c PS |
368 | { |
369 | return OVS_LIKELY(dp_packet_l4_size(b) >= UDP_HEADER_LEN) | |
370 | ? (const char *)dp_packet_l4(b) + UDP_HEADER_LEN : NULL; | |
371 | } | |
372 | ||
5a07c6e1 DDP |
373 | static inline const void * |
374 | dp_packet_get_sctp_payload(const struct dp_packet *b) | |
cf62fa4c PS |
375 | { |
376 | return OVS_LIKELY(dp_packet_l4_size(b) >= SCTP_HEADER_LEN) | |
377 | ? (const char *)dp_packet_l4(b) + SCTP_HEADER_LEN : NULL; | |
378 | } | |
379 | ||
5a07c6e1 DDP |
380 | static inline const void * |
381 | dp_packet_get_icmp_payload(const struct dp_packet *b) | |
cf62fa4c PS |
382 | { |
383 | return OVS_LIKELY(dp_packet_l4_size(b) >= ICMP_HEADER_LEN) | |
384 | ? (const char *)dp_packet_l4(b) + ICMP_HEADER_LEN : NULL; | |
385 | } | |
386 | ||
5a07c6e1 DDP |
387 | static inline const void * |
388 | dp_packet_get_nd_payload(const struct dp_packet *b) | |
cf62fa4c PS |
389 | { |
390 | return OVS_LIKELY(dp_packet_l4_size(b) >= ND_MSG_LEN) | |
391 | ? (const char *)dp_packet_l4(b) + ND_MSG_LEN : NULL; | |
392 | } | |
393 | ||
394 | #ifdef DPDK_NETDEV | |
395 | BUILD_ASSERT_DECL(offsetof(struct dp_packet, mbuf) == 0); | |
396 | ||
5a07c6e1 DDP |
397 | static inline void * |
398 | dp_packet_base(const struct dp_packet *b) | |
cf62fa4c PS |
399 | { |
400 | return b->mbuf.buf_addr; | |
401 | } | |
402 | ||
5a07c6e1 DDP |
403 | static inline void |
404 | dp_packet_set_base(struct dp_packet *b, void *d) | |
cf62fa4c PS |
405 | { |
406 | b->mbuf.buf_addr = d; | |
407 | } | |
408 | ||
5a07c6e1 DDP |
409 | static inline uint32_t |
410 | dp_packet_size(const struct dp_packet *b) | |
cf62fa4c | 411 | { |
b8e57534 | 412 | return b->mbuf.pkt_len; |
cf62fa4c PS |
413 | } |
414 | ||
5a07c6e1 DDP |
415 | static inline void |
416 | dp_packet_set_size(struct dp_packet *b, uint32_t v) | |
cf62fa4c | 417 | { |
b8e57534 MK |
418 | /* netdev-dpdk does not currently support segmentation; consequently, for |
419 | * all intents and purposes, 'data_len' (16 bit) and 'pkt_len' (32 bit) may | |
420 | * be used interchangably. | |
421 | * | |
422 | * On the datapath, it is expected that the size of packets | |
423 | * (and thus 'v') will always be <= UINT16_MAX; this means that there is no | |
424 | * loss of accuracy in assigning 'v' to 'data_len'. | |
b8e57534 MK |
425 | */ |
426 | b->mbuf.data_len = (uint16_t)v; /* Current seg length. */ | |
427 | b->mbuf.pkt_len = v; /* Total length of all segments linked to | |
428 | * this segment. */ | |
cf62fa4c PS |
429 | } |
430 | ||
5a07c6e1 DDP |
431 | static inline uint16_t |
432 | __packet_data(const struct dp_packet *b) | |
cf62fa4c | 433 | { |
b8e57534 | 434 | return b->mbuf.data_off; |
cf62fa4c PS |
435 | } |
436 | ||
5a07c6e1 DDP |
437 | static inline void |
438 | __packet_set_data(struct dp_packet *b, uint16_t v) | |
cf62fa4c | 439 | { |
b8e57534 | 440 | b->mbuf.data_off = v; |
cf62fa4c PS |
441 | } |
442 | ||
5a07c6e1 DDP |
443 | static inline uint16_t |
444 | dp_packet_get_allocated(const struct dp_packet *b) | |
11a6fbd5 DDP |
445 | { |
446 | return b->mbuf.buf_len; | |
447 | } | |
448 | ||
5a07c6e1 DDP |
449 | static inline void |
450 | dp_packet_set_allocated(struct dp_packet *b, uint16_t s) | |
11a6fbd5 DDP |
451 | { |
452 | b->mbuf.buf_len = s; | |
453 | } | |
b8e57534 | 454 | #else |
5a07c6e1 DDP |
455 | static inline void * |
456 | dp_packet_base(const struct dp_packet *b) | |
cf62fa4c PS |
457 | { |
458 | return b->base_; | |
459 | } | |
460 | ||
5a07c6e1 DDP |
461 | static inline void |
462 | dp_packet_set_base(struct dp_packet *b, void *d) | |
cf62fa4c PS |
463 | { |
464 | b->base_ = d; | |
465 | } | |
466 | ||
5a07c6e1 DDP |
467 | static inline uint32_t |
468 | dp_packet_size(const struct dp_packet *b) | |
cf62fa4c PS |
469 | { |
470 | return b->size_; | |
471 | } | |
472 | ||
5a07c6e1 DDP |
473 | static inline void |
474 | dp_packet_set_size(struct dp_packet *b, uint32_t v) | |
cf62fa4c PS |
475 | { |
476 | b->size_ = v; | |
477 | } | |
b8e57534 | 478 | |
5a07c6e1 DDP |
479 | static inline uint16_t |
480 | __packet_data(const struct dp_packet *b) | |
b8e57534 MK |
481 | { |
482 | return b->data_ofs; | |
483 | } | |
484 | ||
5a07c6e1 DDP |
485 | static inline void |
486 | __packet_set_data(struct dp_packet *b, uint16_t v) | |
b8e57534 MK |
487 | { |
488 | b->data_ofs = v; | |
489 | } | |
490 | ||
5a07c6e1 DDP |
491 | static inline uint16_t |
492 | dp_packet_get_allocated(const struct dp_packet *b) | |
11a6fbd5 DDP |
493 | { |
494 | return b->allocated_; | |
495 | } | |
496 | ||
5a07c6e1 DDP |
497 | static inline void |
498 | dp_packet_set_allocated(struct dp_packet *b, uint16_t s) | |
11a6fbd5 DDP |
499 | { |
500 | b->allocated_ = s; | |
501 | } | |
cf62fa4c PS |
502 | #endif |
503 | ||
aaca4fe0 WT |
504 | static inline void |
505 | dp_packet_reset_cutlen(struct dp_packet *b) | |
506 | { | |
507 | b->cutlen = 0; | |
508 | } | |
509 | ||
510 | static inline uint32_t | |
511 | dp_packet_set_cutlen(struct dp_packet *b, uint32_t max_len) | |
512 | { | |
513 | if (max_len < ETH_HEADER_LEN) { | |
514 | max_len = ETH_HEADER_LEN; | |
515 | } | |
516 | ||
517 | if (max_len >= dp_packet_size(b)) { | |
518 | b->cutlen = 0; | |
519 | } else { | |
520 | b->cutlen = dp_packet_size(b) - max_len; | |
521 | } | |
522 | return b->cutlen; | |
523 | } | |
524 | ||
525 | static inline uint32_t | |
526 | dp_packet_get_cutlen(struct dp_packet *b) | |
527 | { | |
528 | /* Always in valid range if user uses dp_packet_set_cutlen. */ | |
529 | return b->cutlen; | |
530 | } | |
531 | ||
5a07c6e1 DDP |
532 | static inline void * |
533 | dp_packet_data(const struct dp_packet *b) | |
b8e57534 | 534 | { |
5a07c6e1 DDP |
535 | return __packet_data(b) != UINT16_MAX |
536 | ? (char *) dp_packet_base(b) + __packet_data(b) : NULL; | |
b8e57534 MK |
537 | } |
538 | ||
5a07c6e1 DDP |
539 | static inline void |
540 | dp_packet_set_data(struct dp_packet *b, void *data) | |
b8e57534 MK |
541 | { |
542 | if (data) { | |
5a07c6e1 | 543 | __packet_set_data(b, (char *) data - (char *) dp_packet_base(b)); |
b8e57534 MK |
544 | } else { |
545 | __packet_set_data(b, UINT16_MAX); | |
546 | } | |
547 | } | |
548 | ||
5a07c6e1 DDP |
549 | static inline void |
550 | dp_packet_reset_packet(struct dp_packet *b, int off) | |
cf62fa4c PS |
551 | { |
552 | dp_packet_set_size(b, dp_packet_size(b) - off); | |
82eb5b0a | 553 | dp_packet_set_data(b, ((unsigned char *) dp_packet_data(b) + off)); |
b513ea34 | 554 | dp_packet_reset_offsets(b); |
91088554 DDP |
555 | } |
556 | ||
f2f44f5d DDP |
557 | /* Returns the RSS hash of the packet 'p'. Note that the returned value is |
558 | * correct only if 'dp_packet_rss_valid(p)' returns true */ | |
5a07c6e1 DDP |
559 | static inline uint32_t |
560 | dp_packet_get_rss_hash(struct dp_packet *p) | |
61a2647e DDP |
561 | { |
562 | #ifdef DPDK_NETDEV | |
b8e57534 | 563 | return p->mbuf.hash.rss; |
61a2647e | 564 | #else |
2bc1bbd2 | 565 | return p->rss_hash; |
61a2647e DDP |
566 | #endif |
567 | } | |
568 | ||
5a07c6e1 DDP |
569 | static inline void |
570 | dp_packet_set_rss_hash(struct dp_packet *p, uint32_t hash) | |
61a2647e DDP |
571 | { |
572 | #ifdef DPDK_NETDEV | |
b8e57534 | 573 | p->mbuf.hash.rss = hash; |
f2f44f5d | 574 | p->mbuf.ol_flags |= PKT_RX_RSS_HASH; |
61a2647e | 575 | #else |
2bc1bbd2 | 576 | p->rss_hash = hash; |
f2f44f5d DDP |
577 | p->rss_hash_valid = true; |
578 | #endif | |
579 | } | |
580 | ||
581 | static inline bool | |
582 | dp_packet_rss_valid(struct dp_packet *p) | |
583 | { | |
584 | #ifdef DPDK_NETDEV | |
585 | return p->mbuf.ol_flags & PKT_RX_RSS_HASH; | |
586 | #else | |
587 | return p->rss_hash_valid; | |
588 | #endif | |
589 | } | |
590 | ||
591 | static inline void | |
592 | dp_packet_rss_invalidate(struct dp_packet *p) | |
593 | { | |
594 | #ifdef DPDK_NETDEV | |
595 | p->mbuf.ol_flags &= ~PKT_RX_RSS_HASH; | |
596 | #else | |
597 | p->rss_hash_valid = false; | |
61a2647e DDP |
598 | #endif |
599 | } | |
600 | ||
1a2bb118 SC |
601 | static inline bool |
602 | dp_packet_ip_checksum_valid(struct dp_packet *p) | |
603 | { | |
604 | #ifdef DPDK_NETDEV | |
605 | return p->mbuf.ol_flags & PKT_RX_IP_CKSUM_GOOD; | |
606 | #else | |
607 | return 0 && p; | |
608 | #endif | |
609 | } | |
610 | ||
611 | static inline bool | |
612 | dp_packet_l4_checksum_valid(struct dp_packet *p) | |
613 | { | |
614 | #ifdef DPDK_NETDEV | |
615 | return p->mbuf.ol_flags & PKT_RX_L4_CKSUM_GOOD; | |
616 | #else | |
617 | return 0 && p; | |
618 | #endif | |
619 | } | |
620 | ||
621 | #ifdef DPDK_NETDEV | |
622 | static inline void | |
623 | reset_dp_packet_checksum_ol_flags(struct dp_packet *p) | |
624 | { | |
625 | p->mbuf.ol_flags &= ~(PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD | | |
626 | PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD); | |
627 | } | |
628 | #else | |
629 | #define reset_dp_packet_checksum_ol_flags(arg) | |
630 | #endif | |
631 | ||
1895cc8d PS |
632 | enum { NETDEV_MAX_BURST = 32 }; /* Maximum number packets in a batch. */ |
633 | ||
634 | struct dp_packet_batch { | |
72c84bc2 | 635 | size_t count; |
aaca4fe0 | 636 | bool trunc; /* true if the batch needs truncate. */ |
1895cc8d PS |
637 | struct dp_packet *packets[NETDEV_MAX_BURST]; |
638 | }; | |
639 | ||
72c84bc2 AZ |
640 | static inline void |
641 | dp_packet_batch_init(struct dp_packet_batch *batch) | |
1895cc8d | 642 | { |
72c84bc2 AZ |
643 | batch->count = 0; |
644 | batch->trunc = false; | |
1895cc8d PS |
645 | } |
646 | ||
647 | static inline void | |
72c84bc2 AZ |
648 | dp_packet_batch_add__(struct dp_packet_batch *batch, |
649 | struct dp_packet *packet, size_t limit) | |
1895cc8d | 650 | { |
72c84bc2 AZ |
651 | if (batch->count < limit) { |
652 | batch->packets[batch->count++] = packet; | |
653 | } else { | |
654 | dp_packet_delete(packet); | |
1895cc8d | 655 | } |
1895cc8d PS |
656 | } |
657 | ||
72c84bc2 AZ |
658 | /* When the batch is full, 'packet' will be dropped and freed. */ |
659 | static inline void | |
660 | dp_packet_batch_add(struct dp_packet_batch *batch, struct dp_packet *packet) | |
661 | { | |
662 | dp_packet_batch_add__(batch, packet, NETDEV_MAX_BURST); | |
663 | } | |
664 | ||
665 | static inline size_t | |
666 | dp_packet_batch_size(const struct dp_packet_batch *batch) | |
667 | { | |
668 | return batch->count; | |
669 | } | |
670 | ||
671 | /* | |
672 | * Clear 'batch' for refill. Use dp_packet_batch_refill() to add | |
673 | * packets back into the 'batch'. | |
674 | * | |
675 | * Return the original size of the 'batch'. */ | |
1895cc8d | 676 | static inline void |
72c84bc2 | 677 | dp_packet_batch_refill_init(struct dp_packet_batch *batch) |
1895cc8d | 678 | { |
72c84bc2 AZ |
679 | batch->count = 0; |
680 | }; | |
681 | ||
682 | static inline void | |
683 | dp_packet_batch_refill(struct dp_packet_batch *batch, | |
684 | struct dp_packet *packet, size_t idx) | |
685 | { | |
686 | dp_packet_batch_add__(batch, packet, MIN(NETDEV_MAX_BURST, idx + 1)); | |
687 | } | |
688 | ||
689 | static inline void | |
690 | dp_packet_batch_init_packet(struct dp_packet_batch *batch, struct dp_packet *p) | |
691 | { | |
692 | dp_packet_batch_init(batch); | |
693 | batch->count = 1; | |
694 | batch->packets[0] = p; | |
695 | } | |
696 | ||
697 | static inline bool | |
698 | dp_packet_batch_is_empty(const struct dp_packet_batch *batch) | |
699 | { | |
700 | return !dp_packet_batch_size(batch); | |
701 | } | |
702 | ||
703 | #define DP_PACKET_BATCH_FOR_EACH(PACKET, BATCH) \ | |
704 | for (size_t i = 0; i < dp_packet_batch_size(BATCH); i++) \ | |
705 | if (PACKET = BATCH->packets[i], true) | |
706 | ||
707 | /* Use this macro for cases where some packets in the 'BATCH' may be | |
708 | * dropped after going through each packet in the 'BATCH'. | |
709 | * | |
710 | * For packets to stay in the 'BATCH', they need to be refilled back | |
711 | * into the 'BATCH' by calling dp_packet_batch_refill(). Caller owns | |
712 | * the packets that are not refilled. | |
713 | * | |
714 | * Caller needs to supply 'SIZE', that stores the current number of | |
715 | * packets in 'BATCH'. It is best to declare this variable with | |
716 | * the 'const' modifier since it should not be modified by | |
717 | * the iterator. */ | |
718 | #define DP_PACKET_BATCH_REFILL_FOR_EACH(IDX, SIZE, PACKET, BATCH) \ | |
719 | for (dp_packet_batch_refill_init(BATCH), IDX=0; IDX < SIZE; IDX++) \ | |
720 | if (PACKET = BATCH->packets[IDX], true) | |
721 | ||
722 | static inline void | |
723 | dp_packet_batch_clone(struct dp_packet_batch *dst, | |
724 | struct dp_packet_batch *src) | |
725 | { | |
726 | struct dp_packet *packet; | |
727 | ||
728 | dp_packet_batch_init(dst); | |
729 | DP_PACKET_BATCH_FOR_EACH (packet, src) { | |
730 | dp_packet_batch_add(dst, dp_packet_clone(packet)); | |
731 | } | |
1895cc8d PS |
732 | } |
733 | ||
734 | static inline void | |
735 | dp_packet_delete_batch(struct dp_packet_batch *batch, bool may_steal) | |
736 | { | |
737 | if (may_steal) { | |
72c84bc2 | 738 | struct dp_packet *packet; |
1895cc8d | 739 | |
72c84bc2 AZ |
740 | DP_PACKET_BATCH_FOR_EACH (packet, batch) { |
741 | dp_packet_delete(packet); | |
1895cc8d | 742 | } |
72c84bc2 | 743 | dp_packet_batch_init(batch); |
1895cc8d PS |
744 | } |
745 | } | |
746 | ||
aaca4fe0 | 747 | static inline void |
72c84bc2 | 748 | dp_packet_batch_apply_cutlen(struct dp_packet_batch *batch) |
aaca4fe0 | 749 | { |
72c84bc2 AZ |
750 | if (batch->trunc) { |
751 | struct dp_packet *packet; | |
aaca4fe0 | 752 | |
72c84bc2 AZ |
753 | DP_PACKET_BATCH_FOR_EACH (packet, batch) { |
754 | uint32_t cutlen = dp_packet_get_cutlen(packet); | |
aaca4fe0 | 755 | |
72c84bc2 AZ |
756 | dp_packet_set_size(packet, dp_packet_size(packet) - cutlen); |
757 | dp_packet_reset_cutlen(packet); | |
758 | } | |
759 | batch->trunc = false; | |
aaca4fe0 | 760 | } |
aaca4fe0 WT |
761 | } |
762 | ||
763 | static inline void | |
72c84bc2 | 764 | dp_packet_batch_reset_cutlen(struct dp_packet_batch *batch) |
aaca4fe0 | 765 | { |
72c84bc2 AZ |
766 | if (batch->trunc) { |
767 | struct dp_packet *packet; | |
aaca4fe0 | 768 | |
72c84bc2 AZ |
769 | DP_PACKET_BATCH_FOR_EACH (packet, batch) { |
770 | dp_packet_reset_cutlen(packet); | |
771 | } | |
772 | batch->trunc = false; | |
aaca4fe0 WT |
773 | } |
774 | } | |
775 | ||
91088554 DDP |
776 | #ifdef __cplusplus |
777 | } | |
778 | #endif | |
779 | ||
aaaac302 | 780 | #endif /* dp-packet.h */ |