2 * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013 Nicira, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
25 #include "netdev-dpdk.h"
31 enum OVS_PACKED_ENUM dp_packet_source
{
32 DPBUF_MALLOC
, /* Obtained via malloc(). */
33 DPBUF_STACK
, /* Un-movable stack space or static buffer. */
34 DPBUF_STUB
, /* Starts on stack, may expand into heap. */
35 DPBUF_DPDK
, /* buffer data is from DPDK allocated memory.
36 * ref to build_dp_packet() in netdev-dpdk. */
39 /* Buffer for holding packet data. A dp_packet is automatically reallocated
40 * as necessary if it grows too large for the available memory.
44 struct rte_mbuf mbuf
; /* DPDK mbuf */
46 void *base_
; /* First byte of allocated space. */
47 uint16_t allocated_
; /* Number of bytes allocated. */
48 uint16_t data_ofs
; /* First byte actually in use. */
49 uint32_t size_
; /* Number of bytes in use. */
50 uint32_t rss_hash
; /* Packet hash. */
52 enum dp_packet_source source
; /* Source of memory allocated as 'base'. */
53 uint8_t l2_pad_size
; /* Detected l2 padding size.
54 * Padding is non-pullable. */
55 uint16_t l2_5_ofs
; /* MPLS label stack offset, or UINT16_MAX */
56 uint16_t l3_ofs
; /* Network-level header offset,
58 uint16_t l4_ofs
; /* Transport-level header offset,
60 struct pkt_metadata md
;
63 static inline void *dp_packet_data(const struct dp_packet
*);
64 static inline void dp_packet_set_data(struct dp_packet
*, void *);
65 static inline void *dp_packet_base(const struct dp_packet
*);
66 static inline void dp_packet_set_base(struct dp_packet
*, void *);
68 static inline uint32_t dp_packet_size(const struct dp_packet
*);
69 static inline void dp_packet_set_size(struct dp_packet
*, uint32_t);
71 static inline uint16_t dp_packet_get_allocated(const struct dp_packet
*);
72 static inline void dp_packet_set_allocated(struct dp_packet
*, uint16_t);
74 void *dp_packet_resize_l2(struct dp_packet
*, int increment
);
75 void *dp_packet_resize_l2_5(struct dp_packet
*, int increment
);
76 static inline void *dp_packet_l2(const struct dp_packet
*);
77 static inline void dp_packet_reset_offsets(struct dp_packet
*);
78 static inline uint8_t dp_packet_l2_pad_size(const struct dp_packet
*);
79 static inline void dp_packet_set_l2_pad_size(struct dp_packet
*, uint8_t);
80 static inline void *dp_packet_l2_5(const struct dp_packet
*);
81 static inline void dp_packet_set_l2_5(struct dp_packet
*, void *);
82 static inline void *dp_packet_l3(const struct dp_packet
*);
83 static inline void dp_packet_set_l3(struct dp_packet
*, void *);
84 static inline void *dp_packet_l4(const struct dp_packet
*);
85 static inline void dp_packet_set_l4(struct dp_packet
*, void *);
86 static inline size_t dp_packet_l4_size(const struct dp_packet
*);
87 static inline const void *dp_packet_get_tcp_payload(const struct dp_packet
*);
88 static inline const void *dp_packet_get_udp_payload(const struct dp_packet
*);
89 static inline const void *dp_packet_get_sctp_payload(const struct dp_packet
*);
90 static inline const void *dp_packet_get_icmp_payload(const struct dp_packet
*);
91 static inline const void *dp_packet_get_nd_payload(const struct dp_packet
*);
93 void dp_packet_use(struct dp_packet
*, void *, size_t);
94 void dp_packet_use_stub(struct dp_packet
*, void *, size_t);
95 void dp_packet_use_const(struct dp_packet
*, const void *, size_t);
97 void dp_packet_init_dpdk(struct dp_packet
*, size_t allocated
);
99 void dp_packet_init(struct dp_packet
*, size_t);
100 void dp_packet_uninit(struct dp_packet
*);
102 struct dp_packet
*dp_packet_new(size_t);
103 struct dp_packet
*dp_packet_new_with_headroom(size_t, size_t headroom
);
104 struct dp_packet
*dp_packet_clone(const struct dp_packet
*);
105 struct dp_packet
*dp_packet_clone_with_headroom(const struct dp_packet
*,
107 struct dp_packet
*dp_packet_clone_data(const void *, size_t);
108 struct dp_packet
*dp_packet_clone_data_with_headroom(const void *, size_t,
110 static inline void dp_packet_delete(struct dp_packet
*);
112 static inline void *dp_packet_at(const struct dp_packet
*, size_t offset
,
114 static inline void *dp_packet_at_assert(const struct dp_packet
*,
115 size_t offset
, size_t size
);
116 static inline void *dp_packet_tail(const struct dp_packet
*);
117 static inline void *dp_packet_end(const struct dp_packet
*);
119 void *dp_packet_put_uninit(struct dp_packet
*, size_t);
120 void *dp_packet_put_zeros(struct dp_packet
*, size_t);
121 void *dp_packet_put(struct dp_packet
*, const void *, size_t);
122 char *dp_packet_put_hex(struct dp_packet
*, const char *s
, size_t *n
);
123 void dp_packet_reserve(struct dp_packet
*, size_t);
124 void dp_packet_reserve_with_tailroom(struct dp_packet
*, size_t headroom
,
126 void *dp_packet_push_uninit(struct dp_packet
*, size_t);
127 void *dp_packet_push_zeros(struct dp_packet
*, size_t);
128 void *dp_packet_push(struct dp_packet
*, const void *, size_t);
130 static inline size_t dp_packet_headroom(const struct dp_packet
*);
131 static inline size_t dp_packet_tailroom(const struct dp_packet
*);
132 void dp_packet_prealloc_headroom(struct dp_packet
*, size_t);
133 void dp_packet_prealloc_tailroom(struct dp_packet
*, size_t);
134 void dp_packet_shift(struct dp_packet
*, int);
136 static inline void dp_packet_clear(struct dp_packet
*);
137 static inline void *dp_packet_pull(struct dp_packet
*, size_t);
138 static inline void *dp_packet_try_pull(struct dp_packet
*, size_t);
140 void *dp_packet_steal_data(struct dp_packet
*);
142 char *dp_packet_to_string(const struct dp_packet
*, size_t maxbytes
);
143 static inline bool dp_packet_equal(const struct dp_packet
*,
144 const struct dp_packet
*);
147 /* Frees memory that 'b' points to, as well as 'b' itself. */
149 dp_packet_delete(struct dp_packet
*b
)
152 if (b
->source
== DPBUF_DPDK
) {
153 /* If this dp_packet was allocated by DPDK it must have been
154 * created as a dp_packet */
155 free_dpdk_buf((struct dp_packet
*) b
);
164 /* If 'b' contains at least 'offset + size' bytes of data, returns a pointer to
165 * byte 'offset'. Otherwise, returns a null pointer. */
167 dp_packet_at(const struct dp_packet
*b
, size_t offset
, size_t size
)
169 return offset
+ size
<= dp_packet_size(b
)
170 ? (char *) dp_packet_data(b
) + offset
174 /* Returns a pointer to byte 'offset' in 'b', which must contain at least
175 * 'offset + size' bytes of data. */
177 dp_packet_at_assert(const struct dp_packet
*b
, size_t offset
, size_t size
)
179 ovs_assert(offset
+ size
<= dp_packet_size(b
));
180 return ((char *) dp_packet_data(b
)) + offset
;
183 /* Returns a pointer to byte following the last byte of data in use in 'b'. */
185 dp_packet_tail(const struct dp_packet
*b
)
187 return (char *) dp_packet_data(b
) + dp_packet_size(b
);
190 /* Returns a pointer to byte following the last byte allocated for use (but
191 * not necessarily in use) in 'b'. */
193 dp_packet_end(const struct dp_packet
*b
)
195 return (char *) dp_packet_base(b
) + dp_packet_get_allocated(b
);
198 /* Returns the number of bytes of headroom in 'b', that is, the number of bytes
199 * of unused space in dp_packet 'b' before the data that is in use. (Most
200 * commonly, the data in a dp_packet is at its beginning, and thus the
201 * dp_packet's headroom is 0.) */
203 dp_packet_headroom(const struct dp_packet
*b
)
205 return (char *) dp_packet_data(b
) - (char *) dp_packet_base(b
);
208 /* Returns the number of bytes that may be appended to the tail end of
209 * dp_packet 'b' before the dp_packet must be reallocated. */
211 dp_packet_tailroom(const struct dp_packet
*b
)
213 return (char *) dp_packet_end(b
) - (char *) dp_packet_tail(b
);
216 /* Clears any data from 'b'. */
218 dp_packet_clear(struct dp_packet
*b
)
220 dp_packet_set_data(b
, dp_packet_base(b
));
221 dp_packet_set_size(b
, 0);
224 /* Removes 'size' bytes from the head end of 'b', which must contain at least
225 * 'size' bytes of data. Returns the first byte of data removed. */
227 dp_packet_pull(struct dp_packet
*b
, size_t size
)
229 void *data
= dp_packet_data(b
);
230 ovs_assert(dp_packet_size(b
) - dp_packet_l2_pad_size(b
) >= size
);
231 dp_packet_set_data(b
, (char *) dp_packet_data(b
) + size
);
232 dp_packet_set_size(b
, dp_packet_size(b
) - size
);
236 /* If 'b' has at least 'size' bytes of data, removes that many bytes from the
237 * head end of 'b' and returns the first byte removed. Otherwise, returns a
238 * null pointer without modifying 'b'. */
240 dp_packet_try_pull(struct dp_packet
*b
, size_t size
)
242 return dp_packet_size(b
) - dp_packet_l2_pad_size(b
) >= size
243 ? dp_packet_pull(b
, size
) : NULL
;
247 dp_packet_equal(const struct dp_packet
*a
, const struct dp_packet
*b
)
249 return dp_packet_size(a
) == dp_packet_size(b
) &&
250 !memcmp(dp_packet_data(a
), dp_packet_data(b
), dp_packet_size(a
));
253 /* Get the start of the Ethernet frame. 'l3_ofs' marks the end of the l2
254 * headers, so return NULL if it is not set. */
256 dp_packet_l2(const struct dp_packet
*b
)
258 return (b
->l3_ofs
!= UINT16_MAX
) ? dp_packet_data(b
) : NULL
;
261 /* Resets all layer offsets. 'l3' offset must be set before 'l2' can be
264 dp_packet_reset_offsets(struct dp_packet
*b
)
267 b
->l2_5_ofs
= UINT16_MAX
;
268 b
->l3_ofs
= UINT16_MAX
;
269 b
->l4_ofs
= UINT16_MAX
;
272 static inline uint8_t
273 dp_packet_l2_pad_size(const struct dp_packet
*b
)
275 return b
->l2_pad_size
;
279 dp_packet_set_l2_pad_size(struct dp_packet
*b
, uint8_t pad_size
)
281 ovs_assert(pad_size
<= dp_packet_size(b
));
282 b
->l2_pad_size
= pad_size
;
286 dp_packet_l2_5(const struct dp_packet
*b
)
288 return b
->l2_5_ofs
!= UINT16_MAX
289 ? (char *) dp_packet_data(b
) + b
->l2_5_ofs
294 dp_packet_set_l2_5(struct dp_packet
*b
, void *l2_5
)
297 ? (char *) l2_5
- (char *) dp_packet_data(b
)
302 dp_packet_l3(const struct dp_packet
*b
)
304 return b
->l3_ofs
!= UINT16_MAX
305 ? (char *) dp_packet_data(b
) + b
->l3_ofs
310 dp_packet_set_l3(struct dp_packet
*b
, void *l3
)
312 b
->l3_ofs
= l3
? (char *) l3
- (char *) dp_packet_data(b
) : UINT16_MAX
;
316 dp_packet_l4(const struct dp_packet
*b
)
318 return b
->l4_ofs
!= UINT16_MAX
319 ? (char *) dp_packet_data(b
) + b
->l4_ofs
324 dp_packet_set_l4(struct dp_packet
*b
, void *l4
)
326 b
->l4_ofs
= l4
? (char *) l4
- (char *) dp_packet_data(b
) : UINT16_MAX
;
330 dp_packet_l4_size(const struct dp_packet
*b
)
332 return b
->l4_ofs
!= UINT16_MAX
333 ? (const char *)dp_packet_tail(b
) - (const char *)dp_packet_l4(b
)
334 - dp_packet_l2_pad_size(b
)
338 static inline const void *
339 dp_packet_get_tcp_payload(const struct dp_packet
*b
)
341 size_t l4_size
= dp_packet_l4_size(b
);
343 if (OVS_LIKELY(l4_size
>= TCP_HEADER_LEN
)) {
344 struct tcp_header
*tcp
= dp_packet_l4(b
);
345 int tcp_len
= TCP_OFFSET(tcp
->tcp_ctl
) * 4;
347 if (OVS_LIKELY(tcp_len
>= TCP_HEADER_LEN
&& tcp_len
<= l4_size
)) {
348 return (const char *)tcp
+ tcp_len
;
354 static inline const void *
355 dp_packet_get_udp_payload(const struct dp_packet
*b
)
357 return OVS_LIKELY(dp_packet_l4_size(b
) >= UDP_HEADER_LEN
)
358 ? (const char *)dp_packet_l4(b
) + UDP_HEADER_LEN
: NULL
;
361 static inline const void *
362 dp_packet_get_sctp_payload(const struct dp_packet
*b
)
364 return OVS_LIKELY(dp_packet_l4_size(b
) >= SCTP_HEADER_LEN
)
365 ? (const char *)dp_packet_l4(b
) + SCTP_HEADER_LEN
: NULL
;
368 static inline const void *
369 dp_packet_get_icmp_payload(const struct dp_packet
*b
)
371 return OVS_LIKELY(dp_packet_l4_size(b
) >= ICMP_HEADER_LEN
)
372 ? (const char *)dp_packet_l4(b
) + ICMP_HEADER_LEN
: NULL
;
375 static inline const void *
376 dp_packet_get_nd_payload(const struct dp_packet
*b
)
378 return OVS_LIKELY(dp_packet_l4_size(b
) >= ND_MSG_LEN
)
379 ? (const char *)dp_packet_l4(b
) + ND_MSG_LEN
: NULL
;
383 BUILD_ASSERT_DECL(offsetof(struct dp_packet
, mbuf
) == 0);
386 dp_packet_base(const struct dp_packet
*b
)
388 return b
->mbuf
.buf_addr
;
392 dp_packet_set_base(struct dp_packet
*b
, void *d
)
394 b
->mbuf
.buf_addr
= d
;
397 static inline uint32_t
398 dp_packet_size(const struct dp_packet
*b
)
400 return b
->mbuf
.pkt_len
;
404 dp_packet_set_size(struct dp_packet
*b
, uint32_t v
)
406 /* netdev-dpdk does not currently support segmentation; consequently, for
407 * all intents and purposes, 'data_len' (16 bit) and 'pkt_len' (32 bit) may
408 * be used interchangably.
410 * On the datapath, it is expected that the size of packets
411 * (and thus 'v') will always be <= UINT16_MAX; this means that there is no
412 * loss of accuracy in assigning 'v' to 'data_len'.
414 b
->mbuf
.data_len
= (uint16_t)v
; /* Current seg length. */
415 b
->mbuf
.pkt_len
= v
; /* Total length of all segments linked to
419 static inline uint16_t
420 __packet_data(const struct dp_packet
*b
)
422 return b
->mbuf
.data_off
;
426 __packet_set_data(struct dp_packet
*b
, uint16_t v
)
428 b
->mbuf
.data_off
= v
;
431 static inline uint16_t
432 dp_packet_get_allocated(const struct dp_packet
*b
)
434 return b
->mbuf
.buf_len
;
438 dp_packet_set_allocated(struct dp_packet
*b
, uint16_t s
)
444 dp_packet_base(const struct dp_packet
*b
)
450 dp_packet_set_base(struct dp_packet
*b
, void *d
)
455 static inline uint32_t
456 dp_packet_size(const struct dp_packet
*b
)
462 dp_packet_set_size(struct dp_packet
*b
, uint32_t v
)
467 static inline uint16_t
468 __packet_data(const struct dp_packet
*b
)
474 __packet_set_data(struct dp_packet
*b
, uint16_t v
)
479 static inline uint16_t
480 dp_packet_get_allocated(const struct dp_packet
*b
)
482 return b
->allocated_
;
486 dp_packet_set_allocated(struct dp_packet
*b
, uint16_t s
)
493 dp_packet_data(const struct dp_packet
*b
)
495 return __packet_data(b
) != UINT16_MAX
496 ? (char *) dp_packet_base(b
) + __packet_data(b
) : NULL
;
500 dp_packet_set_data(struct dp_packet
*b
, void *data
)
503 __packet_set_data(b
, (char *) data
- (char *) dp_packet_base(b
));
505 __packet_set_data(b
, UINT16_MAX
);
510 dp_packet_reset_packet(struct dp_packet
*b
, int off
)
512 dp_packet_set_size(b
, dp_packet_size(b
) - off
);
513 dp_packet_set_data(b
, ((unsigned char *) dp_packet_data(b
) + off
));
514 b
->l2_5_ofs
= b
->l3_ofs
= b
->l4_ofs
= UINT16_MAX
;
517 static inline uint32_t
518 dp_packet_get_rss_hash(struct dp_packet
*p
)
521 return p
->mbuf
.hash
.rss
;
528 dp_packet_set_rss_hash(struct dp_packet
*p
, uint32_t hash
)
531 p
->mbuf
.hash
.rss
= hash
;
541 #endif /* dp-packet.h */