]>
Commit | Line | Data |
---|---|---|
91088554 | 1 | /* |
cf62fa4c | 2 | * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013 Nicira, Inc. |
91088554 DDP |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | * you may not use this file except in compliance with the License. | |
6 | * You may obtain a copy of the License at: | |
7 | * | |
8 | * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | * | |
10 | * Unless required by applicable law or agreed to in writing, software | |
11 | * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | * See the License for the specific language governing permissions and | |
14 | * limitations under the License. | |
15 | */ | |
16 | ||
cf62fa4c PS |
17 | #ifndef DPBUF_H |
18 | #define DPBUF_H 1 | |
91088554 | 19 | |
cf62fa4c PS |
20 | #include <stddef.h> |
21 | #include <stdint.h> | |
01961bbd DDP |
22 | |
23 | #ifdef DPDK_NETDEV | |
24 | #include <rte_config.h> | |
25 | #include <rte_mbuf.h> | |
26 | #endif | |
27 | ||
28 | #include "netdev-dpdk.h" | |
b19bab5b | 29 | #include "openvswitch/list.h" |
cf62fa4c PS |
30 | #include "packets.h" |
31 | #include "util.h" | |
2482b0b0 | 32 | #include "flow.h" |
91088554 DDP |
33 | |
34 | #ifdef __cplusplus | |
35 | extern "C" { | |
36 | #endif | |
37 | ||
cf62fa4c PS |
38 | enum OVS_PACKED_ENUM dp_packet_source { |
39 | DPBUF_MALLOC, /* Obtained via malloc(). */ | |
40 | DPBUF_STACK, /* Un-movable stack space or static buffer. */ | |
41 | DPBUF_STUB, /* Starts on stack, may expand into heap. */ | |
42 | DPBUF_DPDK, /* buffer data is from DPDK allocated memory. | |
84b70576 FA |
43 | * ref to dp_packet_init_dpdk() in dp-packet.c. |
44 | */ | |
cf62fa4c | 45 | }; |
91088554 | 46 | |
eb18b2a6 PS |
47 | #define DP_PACKET_CONTEXT_SIZE 64 |
48 | ||
82eb5b0a | 49 | /* Buffer for holding packet data. A dp_packet is automatically reallocated |
cf62fa4c | 50 | * as necessary if it grows too large for the available memory. |
2482b0b0 | 51 | * By default the packet type is set to Ethernet (PT_ETH). |
cf62fa4c | 52 | */ |
e14deea0 | 53 | struct dp_packet { |
cf62fa4c PS |
54 | #ifdef DPDK_NETDEV |
55 | struct rte_mbuf mbuf; /* DPDK mbuf */ | |
56 | #else | |
b8e57534 | 57 | void *base_; /* First byte of allocated space. */ |
11a6fbd5 | 58 | uint16_t allocated_; /* Number of bytes allocated. */ |
b8e57534 MK |
59 | uint16_t data_ofs; /* First byte actually in use. */ |
60 | uint32_t size_; /* Number of bytes in use. */ | |
2bc1bbd2 | 61 | uint32_t rss_hash; /* Packet hash. */ |
f2f44f5d | 62 | bool rss_hash_valid; /* Is the 'rss_hash' valid? */ |
61a2647e | 63 | #endif |
cf62fa4c | 64 | enum dp_packet_source source; /* Source of memory allocated as 'base'. */ |
84b70576 FA |
65 | |
66 | /* All the following elements of this struct are copied in a single call | |
67 | * of memcpy in dp_packet_clone_with_headroom. */ | |
82eb5b0a DDP |
68 | uint8_t l2_pad_size; /* Detected l2 padding size. |
69 | * Padding is non-pullable. */ | |
70 | uint16_t l2_5_ofs; /* MPLS label stack offset, or UINT16_MAX */ | |
71 | uint16_t l3_ofs; /* Network-level header offset, | |
72 | * or UINT16_MAX. */ | |
73 | uint16_t l4_ofs; /* Transport-level header offset, | |
74 | or UINT16_MAX. */ | |
aaca4fe0 | 75 | uint32_t cutlen; /* length in bytes to cut from the end. */ |
2482b0b0 | 76 | ovs_be32 packet_type; /* Packet type as defined in OpenFlow */ |
eb18b2a6 PS |
77 | union { |
78 | struct pkt_metadata md; | |
79 | uint64_t data[DP_PACKET_CONTEXT_SIZE / 8]; | |
80 | }; | |
91088554 DDP |
81 | }; |
82 | ||
5a07c6e1 | 83 | static inline void *dp_packet_data(const struct dp_packet *); |
cf62fa4c | 84 | static inline void dp_packet_set_data(struct dp_packet *, void *); |
5a07c6e1 | 85 | static inline void *dp_packet_base(const struct dp_packet *); |
cf62fa4c PS |
86 | static inline void dp_packet_set_base(struct dp_packet *, void *); |
87 | ||
88 | static inline uint32_t dp_packet_size(const struct dp_packet *); | |
89 | static inline void dp_packet_set_size(struct dp_packet *, uint32_t); | |
90 | ||
11a6fbd5 DDP |
91 | static inline uint16_t dp_packet_get_allocated(const struct dp_packet *); |
92 | static inline void dp_packet_set_allocated(struct dp_packet *, uint16_t); | |
93 | ||
5a07c6e1 DDP |
94 | void *dp_packet_resize_l2(struct dp_packet *, int increment); |
95 | void *dp_packet_resize_l2_5(struct dp_packet *, int increment); | |
2482b0b0 | 96 | static inline void *dp_packet_eth(const struct dp_packet *); |
82eb5b0a | 97 | static inline void dp_packet_reset_offsets(struct dp_packet *); |
cf62fa4c PS |
98 | static inline uint8_t dp_packet_l2_pad_size(const struct dp_packet *); |
99 | static inline void dp_packet_set_l2_pad_size(struct dp_packet *, uint8_t); | |
5a07c6e1 | 100 | static inline void *dp_packet_l2_5(const struct dp_packet *); |
cf62fa4c | 101 | static inline void dp_packet_set_l2_5(struct dp_packet *, void *); |
5a07c6e1 | 102 | static inline void *dp_packet_l3(const struct dp_packet *); |
cf62fa4c | 103 | static inline void dp_packet_set_l3(struct dp_packet *, void *); |
5a07c6e1 | 104 | static inline void *dp_packet_l4(const struct dp_packet *); |
cf62fa4c PS |
105 | static inline void dp_packet_set_l4(struct dp_packet *, void *); |
106 | static inline size_t dp_packet_l4_size(const struct dp_packet *); | |
107 | static inline const void *dp_packet_get_tcp_payload(const struct dp_packet *); | |
108 | static inline const void *dp_packet_get_udp_payload(const struct dp_packet *); | |
109 | static inline const void *dp_packet_get_sctp_payload(const struct dp_packet *); | |
110 | static inline const void *dp_packet_get_icmp_payload(const struct dp_packet *); | |
111 | static inline const void *dp_packet_get_nd_payload(const struct dp_packet *); | |
112 | ||
113 | void dp_packet_use(struct dp_packet *, void *, size_t); | |
114 | void dp_packet_use_stub(struct dp_packet *, void *, size_t); | |
115 | void dp_packet_use_const(struct dp_packet *, const void *, size_t); | |
116 | ||
5a07c6e1 | 117 | void dp_packet_init_dpdk(struct dp_packet *, size_t allocated); |
cf62fa4c PS |
118 | |
119 | void dp_packet_init(struct dp_packet *, size_t); | |
120 | void dp_packet_uninit(struct dp_packet *); | |
cf62fa4c PS |
121 | |
122 | struct dp_packet *dp_packet_new(size_t); | |
123 | struct dp_packet *dp_packet_new_with_headroom(size_t, size_t headroom); | |
124 | struct dp_packet *dp_packet_clone(const struct dp_packet *); | |
125 | struct dp_packet *dp_packet_clone_with_headroom(const struct dp_packet *, | |
5a07c6e1 | 126 | size_t headroom); |
cf62fa4c PS |
127 | struct dp_packet *dp_packet_clone_data(const void *, size_t); |
128 | struct dp_packet *dp_packet_clone_data_with_headroom(const void *, size_t, | |
5a07c6e1 | 129 | size_t headroom); |
cf62fa4c PS |
130 | static inline void dp_packet_delete(struct dp_packet *); |
131 | ||
132 | static inline void *dp_packet_at(const struct dp_packet *, size_t offset, | |
5a07c6e1 DDP |
133 | size_t size); |
134 | static inline void *dp_packet_at_assert(const struct dp_packet *, | |
135 | size_t offset, size_t size); | |
cf62fa4c PS |
136 | static inline void *dp_packet_tail(const struct dp_packet *); |
137 | static inline void *dp_packet_end(const struct dp_packet *); | |
138 | ||
139 | void *dp_packet_put_uninit(struct dp_packet *, size_t); | |
140 | void *dp_packet_put_zeros(struct dp_packet *, size_t); | |
141 | void *dp_packet_put(struct dp_packet *, const void *, size_t); | |
142 | char *dp_packet_put_hex(struct dp_packet *, const char *s, size_t *n); | |
143 | void dp_packet_reserve(struct dp_packet *, size_t); | |
5a07c6e1 DDP |
144 | void dp_packet_reserve_with_tailroom(struct dp_packet *, size_t headroom, |
145 | size_t tailroom); | |
146 | void *dp_packet_push_uninit(struct dp_packet *, size_t); | |
cf62fa4c | 147 | void *dp_packet_push_zeros(struct dp_packet *, size_t); |
5a07c6e1 | 148 | void *dp_packet_push(struct dp_packet *, const void *, size_t); |
cf62fa4c PS |
149 | |
150 | static inline size_t dp_packet_headroom(const struct dp_packet *); | |
151 | static inline size_t dp_packet_tailroom(const struct dp_packet *); | |
152 | void dp_packet_prealloc_headroom(struct dp_packet *, size_t); | |
153 | void dp_packet_prealloc_tailroom(struct dp_packet *, size_t); | |
154 | void dp_packet_shift(struct dp_packet *, int); | |
91088554 | 155 | |
cf62fa4c PS |
156 | static inline void dp_packet_clear(struct dp_packet *); |
157 | static inline void *dp_packet_pull(struct dp_packet *, size_t); | |
158 | static inline void *dp_packet_try_pull(struct dp_packet *, size_t); | |
91088554 | 159 | |
cf62fa4c | 160 | void *dp_packet_steal_data(struct dp_packet *); |
91088554 | 161 | |
5a07c6e1 DDP |
162 | static inline bool dp_packet_equal(const struct dp_packet *, |
163 | const struct dp_packet *); | |
cf62fa4c PS |
164 | |
165 | \f | |
cf62fa4c | 166 | /* Frees memory that 'b' points to, as well as 'b' itself. */ |
5a07c6e1 DDP |
167 | static inline void |
168 | dp_packet_delete(struct dp_packet *b) | |
cf62fa4c PS |
169 | { |
170 | if (b) { | |
171 | if (b->source == DPBUF_DPDK) { | |
172 | /* If this dp_packet was allocated by DPDK it must have been | |
173 | * created as a dp_packet */ | |
174 | free_dpdk_buf((struct dp_packet*) b); | |
175 | return; | |
176 | } | |
177 | ||
178 | dp_packet_uninit(b); | |
179 | free(b); | |
180 | } | |
181 | } | |
182 | ||
183 | /* If 'b' contains at least 'offset + size' bytes of data, returns a pointer to | |
184 | * byte 'offset'. Otherwise, returns a null pointer. */ | |
5a07c6e1 DDP |
185 | static inline void * |
186 | dp_packet_at(const struct dp_packet *b, size_t offset, size_t size) | |
91088554 | 187 | { |
5a07c6e1 DDP |
188 | return offset + size <= dp_packet_size(b) |
189 | ? (char *) dp_packet_data(b) + offset | |
190 | : NULL; | |
cf62fa4c | 191 | } |
91088554 | 192 | |
cf62fa4c PS |
193 | /* Returns a pointer to byte 'offset' in 'b', which must contain at least |
194 | * 'offset + size' bytes of data. */ | |
5a07c6e1 DDP |
195 | static inline void * |
196 | dp_packet_at_assert(const struct dp_packet *b, size_t offset, size_t size) | |
cf62fa4c PS |
197 | { |
198 | ovs_assert(offset + size <= dp_packet_size(b)); | |
199 | return ((char *) dp_packet_data(b)) + offset; | |
200 | } | |
201 | ||
202 | /* Returns a pointer to byte following the last byte of data in use in 'b'. */ | |
5a07c6e1 DDP |
203 | static inline void * |
204 | dp_packet_tail(const struct dp_packet *b) | |
cf62fa4c PS |
205 | { |
206 | return (char *) dp_packet_data(b) + dp_packet_size(b); | |
207 | } | |
208 | ||
209 | /* Returns a pointer to byte following the last byte allocated for use (but | |
210 | * not necessarily in use) in 'b'. */ | |
5a07c6e1 DDP |
211 | static inline void * |
212 | dp_packet_end(const struct dp_packet *b) | |
cf62fa4c | 213 | { |
11a6fbd5 | 214 | return (char *) dp_packet_base(b) + dp_packet_get_allocated(b); |
cf62fa4c PS |
215 | } |
216 | ||
217 | /* Returns the number of bytes of headroom in 'b', that is, the number of bytes | |
218 | * of unused space in dp_packet 'b' before the data that is in use. (Most | |
5a07c6e1 DDP |
219 | * commonly, the data in a dp_packet is at its beginning, and thus the |
220 | * dp_packet's headroom is 0.) */ | |
221 | static inline size_t | |
222 | dp_packet_headroom(const struct dp_packet *b) | |
cf62fa4c | 223 | { |
5a07c6e1 | 224 | return (char *) dp_packet_data(b) - (char *) dp_packet_base(b); |
cf62fa4c PS |
225 | } |
226 | ||
5a07c6e1 DDP |
227 | /* Returns the number of bytes that may be appended to the tail end of |
228 | * dp_packet 'b' before the dp_packet must be reallocated. */ | |
229 | static inline size_t | |
230 | dp_packet_tailroom(const struct dp_packet *b) | |
cf62fa4c | 231 | { |
5a07c6e1 | 232 | return (char *) dp_packet_end(b) - (char *) dp_packet_tail(b); |
cf62fa4c PS |
233 | } |
234 | ||
235 | /* Clears any data from 'b'. */ | |
5a07c6e1 DDP |
236 | static inline void |
237 | dp_packet_clear(struct dp_packet *b) | |
cf62fa4c PS |
238 | { |
239 | dp_packet_set_data(b, dp_packet_base(b)); | |
240 | dp_packet_set_size(b, 0); | |
241 | } | |
242 | ||
243 | /* Removes 'size' bytes from the head end of 'b', which must contain at least | |
244 | * 'size' bytes of data. Returns the first byte of data removed. */ | |
5a07c6e1 DDP |
245 | static inline void * |
246 | dp_packet_pull(struct dp_packet *b, size_t size) | |
cf62fa4c PS |
247 | { |
248 | void *data = dp_packet_data(b); | |
249 | ovs_assert(dp_packet_size(b) - dp_packet_l2_pad_size(b) >= size); | |
5a07c6e1 | 250 | dp_packet_set_data(b, (char *) dp_packet_data(b) + size); |
cf62fa4c PS |
251 | dp_packet_set_size(b, dp_packet_size(b) - size); |
252 | return data; | |
253 | } | |
254 | ||
255 | /* If 'b' has at least 'size' bytes of data, removes that many bytes from the | |
256 | * head end of 'b' and returns the first byte removed. Otherwise, returns a | |
257 | * null pointer without modifying 'b'. */ | |
5a07c6e1 DDP |
258 | static inline void * |
259 | dp_packet_try_pull(struct dp_packet *b, size_t size) | |
cf62fa4c PS |
260 | { |
261 | return dp_packet_size(b) - dp_packet_l2_pad_size(b) >= size | |
262 | ? dp_packet_pull(b, size) : NULL; | |
263 | } | |
264 | ||
5a07c6e1 DDP |
265 | static inline bool |
266 | dp_packet_equal(const struct dp_packet *a, const struct dp_packet *b) | |
cf62fa4c PS |
267 | { |
268 | return dp_packet_size(a) == dp_packet_size(b) && | |
5a07c6e1 | 269 | !memcmp(dp_packet_data(a), dp_packet_data(b), dp_packet_size(a)); |
cf62fa4c PS |
270 | } |
271 | ||
2482b0b0 JS |
272 | static inline bool |
273 | dp_packet_is_eth(const struct dp_packet *b) | |
274 | { | |
275 | return b->packet_type == htonl(PT_ETH); | |
276 | } | |
277 | ||
278 | /* Get the start of the Ethernet frame. 'l3_ofs' marks the end of the l2 | |
cf62fa4c | 279 | * headers, so return NULL if it is not set. */ |
5a07c6e1 | 280 | static inline void * |
2482b0b0 | 281 | dp_packet_eth(const struct dp_packet *b) |
cf62fa4c | 282 | { |
2482b0b0 JS |
283 | return (dp_packet_is_eth(b) && b->l3_ofs != UINT16_MAX) |
284 | ? dp_packet_data(b) : NULL; | |
cf62fa4c PS |
285 | } |
286 | ||
82eb5b0a DDP |
287 | /* Resets all layer offsets. 'l3' offset must be set before 'l2' can be |
288 | * retrieved. */ | |
5a07c6e1 DDP |
289 | static inline void |
290 | dp_packet_reset_offsets(struct dp_packet *b) | |
cf62fa4c | 291 | { |
cf62fa4c PS |
292 | b->l2_pad_size = 0; |
293 | b->l2_5_ofs = UINT16_MAX; | |
294 | b->l3_ofs = UINT16_MAX; | |
295 | b->l4_ofs = UINT16_MAX; | |
296 | } | |
297 | ||
5a07c6e1 DDP |
298 | static inline uint8_t |
299 | dp_packet_l2_pad_size(const struct dp_packet *b) | |
cf62fa4c PS |
300 | { |
301 | return b->l2_pad_size; | |
302 | } | |
303 | ||
5a07c6e1 DDP |
304 | static inline void |
305 | dp_packet_set_l2_pad_size(struct dp_packet *b, uint8_t pad_size) | |
cf62fa4c PS |
306 | { |
307 | ovs_assert(pad_size <= dp_packet_size(b)); | |
308 | b->l2_pad_size = pad_size; | |
309 | } | |
310 | ||
5a07c6e1 DDP |
311 | static inline void * |
312 | dp_packet_l2_5(const struct dp_packet *b) | |
cf62fa4c | 313 | { |
82eb5b0a DDP |
314 | return b->l2_5_ofs != UINT16_MAX |
315 | ? (char *) dp_packet_data(b) + b->l2_5_ofs | |
316 | : NULL; | |
cf62fa4c PS |
317 | } |
318 | ||
5a07c6e1 DDP |
319 | static inline void |
320 | dp_packet_set_l2_5(struct dp_packet *b, void *l2_5) | |
cf62fa4c | 321 | { |
82eb5b0a DDP |
322 | b->l2_5_ofs = l2_5 |
323 | ? (char *) l2_5 - (char *) dp_packet_data(b) | |
324 | : UINT16_MAX; | |
cf62fa4c PS |
325 | } |
326 | ||
5a07c6e1 DDP |
327 | static inline void * |
328 | dp_packet_l3(const struct dp_packet *b) | |
cf62fa4c | 329 | { |
82eb5b0a DDP |
330 | return b->l3_ofs != UINT16_MAX |
331 | ? (char *) dp_packet_data(b) + b->l3_ofs | |
332 | : NULL; | |
cf62fa4c PS |
333 | } |
334 | ||
5a07c6e1 DDP |
335 | static inline void |
336 | dp_packet_set_l3(struct dp_packet *b, void *l3) | |
cf62fa4c | 337 | { |
82eb5b0a | 338 | b->l3_ofs = l3 ? (char *) l3 - (char *) dp_packet_data(b) : UINT16_MAX; |
cf62fa4c PS |
339 | } |
340 | ||
5a07c6e1 DDP |
341 | static inline void * |
342 | dp_packet_l4(const struct dp_packet *b) | |
cf62fa4c | 343 | { |
82eb5b0a DDP |
344 | return b->l4_ofs != UINT16_MAX |
345 | ? (char *) dp_packet_data(b) + b->l4_ofs | |
346 | : NULL; | |
cf62fa4c PS |
347 | } |
348 | ||
5a07c6e1 DDP |
349 | static inline void |
350 | dp_packet_set_l4(struct dp_packet *b, void *l4) | |
cf62fa4c | 351 | { |
82eb5b0a | 352 | b->l4_ofs = l4 ? (char *) l4 - (char *) dp_packet_data(b) : UINT16_MAX; |
cf62fa4c PS |
353 | } |
354 | ||
5a07c6e1 DDP |
355 | static inline size_t |
356 | dp_packet_l4_size(const struct dp_packet *b) | |
cf62fa4c PS |
357 | { |
358 | return b->l4_ofs != UINT16_MAX | |
359 | ? (const char *)dp_packet_tail(b) - (const char *)dp_packet_l4(b) | |
360 | - dp_packet_l2_pad_size(b) | |
361 | : 0; | |
362 | } | |
363 | ||
5a07c6e1 DDP |
364 | static inline const void * |
365 | dp_packet_get_tcp_payload(const struct dp_packet *b) | |
cf62fa4c PS |
366 | { |
367 | size_t l4_size = dp_packet_l4_size(b); | |
368 | ||
369 | if (OVS_LIKELY(l4_size >= TCP_HEADER_LEN)) { | |
370 | struct tcp_header *tcp = dp_packet_l4(b); | |
371 | int tcp_len = TCP_OFFSET(tcp->tcp_ctl) * 4; | |
372 | ||
373 | if (OVS_LIKELY(tcp_len >= TCP_HEADER_LEN && tcp_len <= l4_size)) { | |
374 | return (const char *)tcp + tcp_len; | |
375 | } | |
376 | } | |
377 | return NULL; | |
378 | } | |
379 | ||
5a07c6e1 DDP |
380 | static inline const void * |
381 | dp_packet_get_udp_payload(const struct dp_packet *b) | |
cf62fa4c PS |
382 | { |
383 | return OVS_LIKELY(dp_packet_l4_size(b) >= UDP_HEADER_LEN) | |
384 | ? (const char *)dp_packet_l4(b) + UDP_HEADER_LEN : NULL; | |
385 | } | |
386 | ||
5a07c6e1 DDP |
387 | static inline const void * |
388 | dp_packet_get_sctp_payload(const struct dp_packet *b) | |
cf62fa4c PS |
389 | { |
390 | return OVS_LIKELY(dp_packet_l4_size(b) >= SCTP_HEADER_LEN) | |
391 | ? (const char *)dp_packet_l4(b) + SCTP_HEADER_LEN : NULL; | |
392 | } | |
393 | ||
5a07c6e1 DDP |
394 | static inline const void * |
395 | dp_packet_get_icmp_payload(const struct dp_packet *b) | |
cf62fa4c PS |
396 | { |
397 | return OVS_LIKELY(dp_packet_l4_size(b) >= ICMP_HEADER_LEN) | |
398 | ? (const char *)dp_packet_l4(b) + ICMP_HEADER_LEN : NULL; | |
399 | } | |
400 | ||
5a07c6e1 DDP |
401 | static inline const void * |
402 | dp_packet_get_nd_payload(const struct dp_packet *b) | |
cf62fa4c PS |
403 | { |
404 | return OVS_LIKELY(dp_packet_l4_size(b) >= ND_MSG_LEN) | |
405 | ? (const char *)dp_packet_l4(b) + ND_MSG_LEN : NULL; | |
406 | } | |
407 | ||
408 | #ifdef DPDK_NETDEV | |
409 | BUILD_ASSERT_DECL(offsetof(struct dp_packet, mbuf) == 0); | |
410 | ||
5a07c6e1 DDP |
411 | static inline void * |
412 | dp_packet_base(const struct dp_packet *b) | |
cf62fa4c PS |
413 | { |
414 | return b->mbuf.buf_addr; | |
415 | } | |
416 | ||
5a07c6e1 DDP |
417 | static inline void |
418 | dp_packet_set_base(struct dp_packet *b, void *d) | |
cf62fa4c PS |
419 | { |
420 | b->mbuf.buf_addr = d; | |
421 | } | |
422 | ||
5a07c6e1 DDP |
423 | static inline uint32_t |
424 | dp_packet_size(const struct dp_packet *b) | |
cf62fa4c | 425 | { |
b8e57534 | 426 | return b->mbuf.pkt_len; |
cf62fa4c PS |
427 | } |
428 | ||
5a07c6e1 DDP |
429 | static inline void |
430 | dp_packet_set_size(struct dp_packet *b, uint32_t v) | |
cf62fa4c | 431 | { |
b8e57534 MK |
432 | /* netdev-dpdk does not currently support segmentation; consequently, for |
433 | * all intents and purposes, 'data_len' (16 bit) and 'pkt_len' (32 bit) may | |
434 | * be used interchangably. | |
435 | * | |
436 | * On the datapath, it is expected that the size of packets | |
437 | * (and thus 'v') will always be <= UINT16_MAX; this means that there is no | |
438 | * loss of accuracy in assigning 'v' to 'data_len'. | |
b8e57534 MK |
439 | */ |
440 | b->mbuf.data_len = (uint16_t)v; /* Current seg length. */ | |
441 | b->mbuf.pkt_len = v; /* Total length of all segments linked to | |
442 | * this segment. */ | |
cf62fa4c PS |
443 | } |
444 | ||
5a07c6e1 DDP |
445 | static inline uint16_t |
446 | __packet_data(const struct dp_packet *b) | |
cf62fa4c | 447 | { |
b8e57534 | 448 | return b->mbuf.data_off; |
cf62fa4c PS |
449 | } |
450 | ||
5a07c6e1 DDP |
451 | static inline void |
452 | __packet_set_data(struct dp_packet *b, uint16_t v) | |
cf62fa4c | 453 | { |
b8e57534 | 454 | b->mbuf.data_off = v; |
cf62fa4c PS |
455 | } |
456 | ||
5a07c6e1 DDP |
457 | static inline uint16_t |
458 | dp_packet_get_allocated(const struct dp_packet *b) | |
11a6fbd5 DDP |
459 | { |
460 | return b->mbuf.buf_len; | |
461 | } | |
462 | ||
5a07c6e1 DDP |
463 | static inline void |
464 | dp_packet_set_allocated(struct dp_packet *b, uint16_t s) | |
11a6fbd5 DDP |
465 | { |
466 | b->mbuf.buf_len = s; | |
467 | } | |
e52ad721 FL |
468 | |
469 | /* Returns the RSS hash of the packet 'p'. Note that the returned value is | |
470 | * correct only if 'dp_packet_rss_valid(p)' returns true */ | |
471 | static inline uint32_t | |
472 | dp_packet_get_rss_hash(struct dp_packet *p) | |
cf62fa4c | 473 | { |
e52ad721 | 474 | return p->mbuf.hash.rss; |
cf62fa4c PS |
475 | } |
476 | ||
5a07c6e1 | 477 | static inline void |
e52ad721 | 478 | dp_packet_set_rss_hash(struct dp_packet *p, uint32_t hash) |
cf62fa4c | 479 | { |
e52ad721 FL |
480 | p->mbuf.hash.rss = hash; |
481 | p->mbuf.ol_flags |= PKT_RX_RSS_HASH; | |
cf62fa4c PS |
482 | } |
483 | ||
e52ad721 FL |
484 | static inline bool |
485 | dp_packet_rss_valid(struct dp_packet *p) | |
cf62fa4c | 486 | { |
e52ad721 | 487 | return p->mbuf.ol_flags & PKT_RX_RSS_HASH; |
cf62fa4c PS |
488 | } |
489 | ||
5a07c6e1 | 490 | static inline void |
e52ad721 | 491 | dp_packet_rss_invalidate(struct dp_packet *p OVS_UNUSED) |
cf62fa4c | 492 | { |
cf62fa4c | 493 | } |
b8e57534 | 494 | |
e52ad721 FL |
495 | static inline void |
496 | dp_packet_mbuf_rss_flag_reset(struct dp_packet *p) | |
b8e57534 | 497 | { |
e52ad721 | 498 | p->mbuf.ol_flags &= ~PKT_RX_RSS_HASH; |
b8e57534 MK |
499 | } |
500 | ||
4148de28 MK |
501 | /* This initialization is needed for packets that do not come from DPDK |
502 | * interfaces, when vswitchd is built with --with-dpdk. */ | |
5a07c6e1 | 503 | static inline void |
e52ad721 | 504 | dp_packet_mbuf_init(struct dp_packet *p) |
b8e57534 | 505 | { |
4148de28 MK |
506 | p->mbuf.ol_flags = p->mbuf.tx_offload = p->mbuf.packet_type = 0; |
507 | p->mbuf.nb_segs = 1; | |
508 | p->mbuf.next = NULL; | |
b8e57534 MK |
509 | } |
510 | ||
e52ad721 FL |
511 | static inline bool |
512 | dp_packet_ip_checksum_valid(struct dp_packet *p) | |
11a6fbd5 | 513 | { |
e52ad721 FL |
514 | return (p->mbuf.ol_flags & PKT_RX_IP_CKSUM_MASK) == |
515 | PKT_RX_IP_CKSUM_GOOD; | |
11a6fbd5 DDP |
516 | } |
517 | ||
e52ad721 FL |
518 | static inline bool |
519 | dp_packet_ip_checksum_bad(struct dp_packet *p) | |
11a6fbd5 | 520 | { |
e52ad721 FL |
521 | return (p->mbuf.ol_flags & PKT_RX_IP_CKSUM_MASK) == |
522 | PKT_RX_IP_CKSUM_BAD; | |
523 | } | |
524 | ||
525 | static inline bool | |
526 | dp_packet_l4_checksum_valid(struct dp_packet *p) | |
527 | { | |
528 | return (p->mbuf.ol_flags & PKT_RX_L4_CKSUM_MASK) == | |
529 | PKT_RX_L4_CKSUM_GOOD; | |
530 | } | |
531 | ||
532 | static inline bool | |
533 | dp_packet_l4_checksum_bad(struct dp_packet *p) | |
534 | { | |
535 | return (p->mbuf.ol_flags & PKT_RX_L4_CKSUM_MASK) == | |
536 | PKT_RX_L4_CKSUM_BAD; | |
11a6fbd5 | 537 | } |
cf62fa4c | 538 | |
aaca4fe0 | 539 | static inline void |
e52ad721 | 540 | reset_dp_packet_checksum_ol_flags(struct dp_packet *p) |
aaca4fe0 | 541 | { |
e52ad721 FL |
542 | p->mbuf.ol_flags &= ~(PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD | |
543 | PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD); | |
aaca4fe0 WT |
544 | } |
545 | ||
e52ad721 FL |
546 | static inline bool |
547 | dp_packet_has_flow_mark(struct dp_packet *p, uint32_t *mark) | |
aaca4fe0 | 548 | { |
e52ad721 FL |
549 | if (p->mbuf.ol_flags & PKT_RX_FDIR_ID) { |
550 | *mark = p->mbuf.hash.fdir.hi; | |
551 | return true; | |
aaca4fe0 WT |
552 | } |
553 | ||
e52ad721 | 554 | return false; |
aaca4fe0 WT |
555 | } |
556 | ||
e52ad721 FL |
557 | #else /* DPDK_NETDEV */ |
558 | static inline void * | |
559 | dp_packet_base(const struct dp_packet *b) | |
aaca4fe0 | 560 | { |
e52ad721 FL |
561 | return b->base_; |
562 | } | |
563 | ||
564 | static inline void | |
565 | dp_packet_set_base(struct dp_packet *b, void *d) | |
566 | { | |
567 | b->base_ = d; | |
aaca4fe0 WT |
568 | } |
569 | ||
a61a2891 | 570 | static inline uint32_t |
e52ad721 | 571 | dp_packet_size(const struct dp_packet *b) |
a61a2891 | 572 | { |
e52ad721 | 573 | return b->size_; |
a61a2891 BP |
574 | } |
575 | ||
e52ad721 FL |
576 | static inline void |
577 | dp_packet_set_size(struct dp_packet *b, uint32_t v) | |
b8e57534 | 578 | { |
e52ad721 FL |
579 | b->size_ = v; |
580 | } | |
581 | ||
582 | static inline uint16_t | |
583 | __packet_data(const struct dp_packet *b) | |
584 | { | |
585 | return b->data_ofs; | |
b8e57534 MK |
586 | } |
587 | ||
5a07c6e1 | 588 | static inline void |
e52ad721 | 589 | __packet_set_data(struct dp_packet *b, uint16_t v) |
b8e57534 | 590 | { |
e52ad721 FL |
591 | b->data_ofs = v; |
592 | } | |
593 | ||
594 | static inline uint16_t | |
595 | dp_packet_get_allocated(const struct dp_packet *b) | |
596 | { | |
597 | return b->allocated_; | |
b8e57534 MK |
598 | } |
599 | ||
5a07c6e1 | 600 | static inline void |
e52ad721 | 601 | dp_packet_set_allocated(struct dp_packet *b, uint16_t s) |
cf62fa4c | 602 | { |
e52ad721 | 603 | b->allocated_ = s; |
91088554 DDP |
604 | } |
605 | ||
f2f44f5d DDP |
606 | /* Returns the RSS hash of the packet 'p'. Note that the returned value is |
607 | * correct only if 'dp_packet_rss_valid(p)' returns true */ | |
5a07c6e1 DDP |
608 | static inline uint32_t |
609 | dp_packet_get_rss_hash(struct dp_packet *p) | |
61a2647e | 610 | { |
2bc1bbd2 | 611 | return p->rss_hash; |
61a2647e DDP |
612 | } |
613 | ||
5a07c6e1 DDP |
614 | static inline void |
615 | dp_packet_set_rss_hash(struct dp_packet *p, uint32_t hash) | |
61a2647e | 616 | { |
2bc1bbd2 | 617 | p->rss_hash = hash; |
f2f44f5d | 618 | p->rss_hash_valid = true; |
f2f44f5d DDP |
619 | } |
620 | ||
621 | static inline bool | |
622 | dp_packet_rss_valid(struct dp_packet *p) | |
623 | { | |
f2f44f5d | 624 | return p->rss_hash_valid; |
f2f44f5d DDP |
625 | } |
626 | ||
627 | static inline void | |
e52ad721 | 628 | dp_packet_rss_invalidate(struct dp_packet *p) |
f8121b39 | 629 | { |
f8121b39 | 630 | p->rss_hash_valid = false; |
f8121b39 DB |
631 | } |
632 | ||
633 | static inline void | |
634 | dp_packet_mbuf_rss_flag_reset(struct dp_packet *p OVS_UNUSED) | |
f2f44f5d | 635 | { |
f8121b39 DB |
636 | } |
637 | ||
f8121b39 DB |
638 | static inline void |
639 | dp_packet_mbuf_init(struct dp_packet *p OVS_UNUSED) | |
640 | { | |
61a2647e DDP |
641 | } |
642 | ||
1a2bb118 | 643 | static inline bool |
cfc19646 | 644 | dp_packet_ip_checksum_valid(struct dp_packet *p OVS_UNUSED) |
1a2bb118 | 645 | { |
cfc19646 | 646 | return false; |
7451af61 SC |
647 | } |
648 | ||
649 | static inline bool | |
cfc19646 | 650 | dp_packet_ip_checksum_bad(struct dp_packet *p OVS_UNUSED) |
7451af61 | 651 | { |
cfc19646 | 652 | return false; |
1a2bb118 SC |
653 | } |
654 | ||
655 | static inline bool | |
cfc19646 | 656 | dp_packet_l4_checksum_valid(struct dp_packet *p OVS_UNUSED) |
1a2bb118 | 657 | { |
cfc19646 | 658 | return false; |
7451af61 SC |
659 | } |
660 | ||
661 | static inline bool | |
cfc19646 | 662 | dp_packet_l4_checksum_bad(struct dp_packet *p OVS_UNUSED) |
7451af61 | 663 | { |
cfc19646 | 664 | return false; |
1a2bb118 SC |
665 | } |
666 | ||
1a2bb118 | 667 | static inline void |
e52ad721 | 668 | reset_dp_packet_checksum_ol_flags(struct dp_packet *p OVS_UNUSED) |
1a2bb118 | 669 | { |
1a2bb118 | 670 | } |
1a2bb118 | 671 | |
aab96ec4 YL |
672 | static inline bool |
673 | dp_packet_has_flow_mark(struct dp_packet *p OVS_UNUSED, | |
674 | uint32_t *mark OVS_UNUSED) | |
675 | { | |
aab96ec4 YL |
676 | return false; |
677 | } | |
e52ad721 FL |
678 | #endif /* DPDK_NETDEV */ |
679 | ||
680 | static inline void | |
681 | dp_packet_reset_cutlen(struct dp_packet *b) | |
682 | { | |
683 | b->cutlen = 0; | |
684 | } | |
685 | ||
686 | static inline uint32_t | |
687 | dp_packet_set_cutlen(struct dp_packet *b, uint32_t max_len) | |
688 | { | |
689 | if (max_len < ETH_HEADER_LEN) { | |
690 | max_len = ETH_HEADER_LEN; | |
691 | } | |
692 | ||
693 | if (max_len >= dp_packet_size(b)) { | |
694 | b->cutlen = 0; | |
695 | } else { | |
696 | b->cutlen = dp_packet_size(b) - max_len; | |
697 | } | |
698 | return b->cutlen; | |
699 | } | |
700 | ||
701 | static inline uint32_t | |
702 | dp_packet_get_cutlen(const struct dp_packet *b) | |
703 | { | |
704 | /* Always in valid range if user uses dp_packet_set_cutlen. */ | |
705 | return b->cutlen; | |
706 | } | |
707 | ||
708 | static inline uint32_t | |
709 | dp_packet_get_send_len(const struct dp_packet *b) | |
710 | { | |
711 | return dp_packet_size(b) - dp_packet_get_cutlen(b); | |
712 | } | |
713 | ||
714 | static inline void * | |
715 | dp_packet_data(const struct dp_packet *b) | |
716 | { | |
717 | return __packet_data(b) != UINT16_MAX | |
718 | ? (char *) dp_packet_base(b) + __packet_data(b) : NULL; | |
719 | } | |
720 | ||
721 | static inline void | |
722 | dp_packet_set_data(struct dp_packet *b, void *data) | |
723 | { | |
724 | if (data) { | |
725 | __packet_set_data(b, (char *) data - (char *) dp_packet_base(b)); | |
726 | } else { | |
727 | __packet_set_data(b, UINT16_MAX); | |
728 | } | |
729 | } | |
730 | ||
731 | static inline void | |
732 | dp_packet_reset_packet(struct dp_packet *b, int off) | |
733 | { | |
734 | dp_packet_set_size(b, dp_packet_size(b) - off); | |
735 | dp_packet_set_data(b, ((unsigned char *) dp_packet_data(b) + off)); | |
736 | dp_packet_reset_offsets(b); | |
737 | } | |
aab96ec4 | 738 | |
1895cc8d PS |
739 | enum { NETDEV_MAX_BURST = 32 }; /* Maximum number packets in a batch. */ |
740 | ||
741 | struct dp_packet_batch { | |
72c84bc2 | 742 | size_t count; |
aaca4fe0 | 743 | bool trunc; /* true if the batch needs truncate. */ |
1895cc8d PS |
744 | struct dp_packet *packets[NETDEV_MAX_BURST]; |
745 | }; | |
746 | ||
72c84bc2 AZ |
747 | static inline void |
748 | dp_packet_batch_init(struct dp_packet_batch *batch) | |
1895cc8d | 749 | { |
72c84bc2 AZ |
750 | batch->count = 0; |
751 | batch->trunc = false; | |
1895cc8d PS |
752 | } |
753 | ||
754 | static inline void | |
72c84bc2 AZ |
755 | dp_packet_batch_add__(struct dp_packet_batch *batch, |
756 | struct dp_packet *packet, size_t limit) | |
1895cc8d | 757 | { |
72c84bc2 AZ |
758 | if (batch->count < limit) { |
759 | batch->packets[batch->count++] = packet; | |
760 | } else { | |
761 | dp_packet_delete(packet); | |
1895cc8d | 762 | } |
1895cc8d PS |
763 | } |
764 | ||
72c84bc2 AZ |
765 | /* When the batch is full, 'packet' will be dropped and freed. */ |
766 | static inline void | |
767 | dp_packet_batch_add(struct dp_packet_batch *batch, struct dp_packet *packet) | |
768 | { | |
769 | dp_packet_batch_add__(batch, packet, NETDEV_MAX_BURST); | |
770 | } | |
771 | ||
772 | static inline size_t | |
773 | dp_packet_batch_size(const struct dp_packet_batch *batch) | |
774 | { | |
775 | return batch->count; | |
776 | } | |
777 | ||
feb86b41 IM |
778 | /* Clear 'batch' for refill. Use dp_packet_batch_refill() to add |
779 | * packets back into the 'batch'. */ | |
1895cc8d | 780 | static inline void |
72c84bc2 | 781 | dp_packet_batch_refill_init(struct dp_packet_batch *batch) |
1895cc8d | 782 | { |
72c84bc2 AZ |
783 | batch->count = 0; |
784 | }; | |
785 | ||
786 | static inline void | |
787 | dp_packet_batch_refill(struct dp_packet_batch *batch, | |
788 | struct dp_packet *packet, size_t idx) | |
789 | { | |
790 | dp_packet_batch_add__(batch, packet, MIN(NETDEV_MAX_BURST, idx + 1)); | |
791 | } | |
792 | ||
793 | static inline void | |
794 | dp_packet_batch_init_packet(struct dp_packet_batch *batch, struct dp_packet *p) | |
795 | { | |
796 | dp_packet_batch_init(batch); | |
797 | batch->count = 1; | |
798 | batch->packets[0] = p; | |
799 | } | |
800 | ||
801 | static inline bool | |
802 | dp_packet_batch_is_empty(const struct dp_packet_batch *batch) | |
803 | { | |
804 | return !dp_packet_batch_size(batch); | |
805 | } | |
806 | ||
e883448e JP |
807 | #define DP_PACKET_BATCH_FOR_EACH(IDX, PACKET, BATCH) \ |
808 | for (size_t IDX = 0; IDX < dp_packet_batch_size(BATCH); IDX++) \ | |
809 | if (PACKET = BATCH->packets[IDX], true) | |
72c84bc2 AZ |
810 | |
811 | /* Use this macro for cases where some packets in the 'BATCH' may be | |
812 | * dropped after going through each packet in the 'BATCH'. | |
813 | * | |
814 | * For packets to stay in the 'BATCH', they need to be refilled back | |
815 | * into the 'BATCH' by calling dp_packet_batch_refill(). Caller owns | |
816 | * the packets that are not refilled. | |
817 | * | |
818 | * Caller needs to supply 'SIZE', that stores the current number of | |
819 | * packets in 'BATCH'. It is best to declare this variable with | |
820 | * the 'const' modifier since it should not be modified by | |
821 | * the iterator. */ | |
822 | #define DP_PACKET_BATCH_REFILL_FOR_EACH(IDX, SIZE, PACKET, BATCH) \ | |
823 | for (dp_packet_batch_refill_init(BATCH), IDX=0; IDX < SIZE; IDX++) \ | |
824 | if (PACKET = BATCH->packets[IDX], true) | |
825 | ||
826 | static inline void | |
827 | dp_packet_batch_clone(struct dp_packet_batch *dst, | |
828 | struct dp_packet_batch *src) | |
829 | { | |
830 | struct dp_packet *packet; | |
831 | ||
832 | dp_packet_batch_init(dst); | |
e883448e | 833 | DP_PACKET_BATCH_FOR_EACH (i, packet, src) { |
72c84bc2 AZ |
834 | dp_packet_batch_add(dst, dp_packet_clone(packet)); |
835 | } | |
cf4cf916 | 836 | dst->trunc = src->trunc; |
1895cc8d PS |
837 | } |
838 | ||
839 | static inline void | |
7d7ded7a | 840 | dp_packet_delete_batch(struct dp_packet_batch *batch, bool should_steal) |
1895cc8d | 841 | { |
7d7ded7a | 842 | if (should_steal) { |
72c84bc2 | 843 | struct dp_packet *packet; |
1895cc8d | 844 | |
e883448e | 845 | DP_PACKET_BATCH_FOR_EACH (i, packet, batch) { |
72c84bc2 | 846 | dp_packet_delete(packet); |
1895cc8d | 847 | } |
72c84bc2 | 848 | dp_packet_batch_init(batch); |
1895cc8d PS |
849 | } |
850 | } | |
851 | ||
11d4c7a8 | 852 | static inline void |
75fb9148 | 853 | dp_packet_batch_init_packet_fields(struct dp_packet_batch *batch) |
11d4c7a8 DB |
854 | { |
855 | struct dp_packet *packet; | |
856 | ||
e883448e | 857 | DP_PACKET_BATCH_FOR_EACH (i, packet, batch) { |
11d4c7a8 | 858 | dp_packet_reset_cutlen(packet); |
75fb9148 | 859 | packet->packet_type = htonl(PT_ETH); |
11d4c7a8 DB |
860 | } |
861 | } | |
862 | ||
aaca4fe0 | 863 | static inline void |
72c84bc2 | 864 | dp_packet_batch_apply_cutlen(struct dp_packet_batch *batch) |
aaca4fe0 | 865 | { |
72c84bc2 AZ |
866 | if (batch->trunc) { |
867 | struct dp_packet *packet; | |
aaca4fe0 | 868 | |
e883448e | 869 | DP_PACKET_BATCH_FOR_EACH (i, packet, batch) { |
a61a2891 | 870 | dp_packet_set_size(packet, dp_packet_get_send_len(packet)); |
72c84bc2 AZ |
871 | dp_packet_reset_cutlen(packet); |
872 | } | |
873 | batch->trunc = false; | |
aaca4fe0 | 874 | } |
aaca4fe0 WT |
875 | } |
876 | ||
877 | static inline void | |
72c84bc2 | 878 | dp_packet_batch_reset_cutlen(struct dp_packet_batch *batch) |
aaca4fe0 | 879 | { |
72c84bc2 AZ |
880 | if (batch->trunc) { |
881 | struct dp_packet *packet; | |
aaca4fe0 | 882 | |
e883448e | 883 | DP_PACKET_BATCH_FOR_EACH (i, packet, batch) { |
72c84bc2 AZ |
884 | dp_packet_reset_cutlen(packet); |
885 | } | |
886 | batch->trunc = false; | |
aaca4fe0 WT |
887 | } |
888 | } | |
889 | ||
91088554 DDP |
890 | #ifdef __cplusplus |
891 | } | |
892 | #endif | |
893 | ||
aaaac302 | 894 | #endif /* dp-packet.h */ |