2 * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013, 2016 Nicira, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
21 #include "dp-packet.h"
22 #include "netdev-afxdp.h"
23 #include "netdev-dpdk.h"
24 #include "openvswitch/dynamic-string.h"
28 dp_packet_init__(struct dp_packet
*b
, size_t allocated
, enum dp_packet_source source
)
30 dp_packet_set_allocated(b
, allocated
);
32 dp_packet_reset_offsets(b
);
33 pkt_metadata_init(&b
->md
, 0);
34 dp_packet_reset_cutlen(b
);
35 dp_packet_reset_offload(b
);
36 /* Initialize implementation-specific fields of dp_packet. */
37 dp_packet_init_specific(b
);
38 /* By default assume the packet type to be Ethernet. */
39 b
->packet_type
= htonl(PT_ETH
);
43 dp_packet_use__(struct dp_packet
*b
, void *base
, size_t allocated
,
44 enum dp_packet_source source
)
46 dp_packet_set_base(b
, base
);
47 dp_packet_set_data(b
, base
);
48 dp_packet_set_size(b
, 0);
50 dp_packet_init__(b
, allocated
, source
);
53 /* Initializes 'b' as an empty dp_packet that contains the 'allocated' bytes of
54 * memory starting at 'base'. 'base' should be the first byte of a region
55 * obtained from malloc(). It will be freed (with free()) if 'b' is resized or
58 dp_packet_use(struct dp_packet
*b
, void *base
, size_t allocated
)
60 dp_packet_use__(b
, base
, allocated
, DPBUF_MALLOC
);
64 /* Initialize 'b' as an empty dp_packet that contains
65 * memory starting at AF_XDP umem base.
68 dp_packet_use_afxdp(struct dp_packet
*b
, void *data
, size_t allocated
,
71 dp_packet_set_base(b
, (char *)data
- headroom
);
72 dp_packet_set_data(b
, data
);
73 dp_packet_set_size(b
, 0);
75 dp_packet_init__(b
, allocated
, DPBUF_AFXDP
);
79 /* Initializes 'b' as an empty dp_packet that contains the 'allocated' bytes of
80 * memory starting at 'base'. 'base' should point to a buffer on the stack.
81 * (Nothing actually relies on 'base' being allocated on the stack. It could
82 * be static or malloc()'d memory. But stack space is the most common use
85 * 'base' should be appropriately aligned. Using an array of uint32_t or
86 * uint64_t for the buffer is a reasonable way to ensure appropriate alignment
87 * for 32- or 64-bit data.
89 * An dp_packet operation that requires reallocating data will copy the provided
90 * buffer into a malloc()'d buffer. Thus, it is wise to call dp_packet_uninit()
91 * on an dp_packet initialized by this function, so that if it expanded into the
92 * heap, that memory is freed. */
94 dp_packet_use_stub(struct dp_packet
*b
, void *base
, size_t allocated
)
96 dp_packet_use__(b
, base
, allocated
, DPBUF_STUB
);
99 /* Initializes 'b' as an dp_packet whose data starts at 'data' and continues for
100 * 'size' bytes. This is appropriate for an dp_packet that will be used to
101 * inspect existing data, without moving it around or reallocating it, and
102 * generally without modifying it at all.
104 * An dp_packet operation that requires reallocating data will assert-fail if this
105 * function was used to initialize it. */
107 dp_packet_use_const(struct dp_packet
*b
, const void *data
, size_t size
)
109 dp_packet_use__(b
, CONST_CAST(void *, data
), size
, DPBUF_STACK
);
110 dp_packet_set_size(b
, size
);
113 /* Initializes 'b' as a DPDK dp-packet, which must have been allocated from a
114 * DPDK memory pool. */
116 dp_packet_init_dpdk(struct dp_packet
*b
)
118 b
->source
= DPBUF_DPDK
;
121 /* Initializes 'b' as an empty dp_packet with an initial capacity of 'size'
124 dp_packet_init(struct dp_packet
*b
, size_t size
)
126 dp_packet_use(b
, size
? xmalloc(size
) : NULL
, size
);
129 /* Frees memory that 'b' points to. */
131 dp_packet_uninit(struct dp_packet
*b
)
134 if (b
->source
== DPBUF_MALLOC
) {
135 free(dp_packet_base(b
));
136 } else if (b
->source
== DPBUF_DPDK
) {
138 /* If this dp_packet was allocated by DPDK it must have been
139 * created as a dp_packet */
140 free_dpdk_buf((struct dp_packet
*) b
);
142 } else if (b
->source
== DPBUF_AFXDP
) {
148 /* Creates and returns a new dp_packet with an initial capacity of 'size'
151 dp_packet_new(size_t size
)
153 struct dp_packet
*b
= xmalloc(sizeof *b
);
154 dp_packet_init(b
, size
);
158 /* Creates and returns a new dp_packet with an initial capacity of 'size +
159 * headroom' bytes, reserving the first 'headroom' bytes as headroom. */
161 dp_packet_new_with_headroom(size_t size
, size_t headroom
)
163 struct dp_packet
*b
= dp_packet_new(size
+ headroom
);
164 dp_packet_reserve(b
, headroom
);
168 /* Creates and returns a new dp_packet that initially contains a copy of the
169 * 'dp_packet_size(buffer)' bytes of data starting at 'buffer->data' with no headroom or
172 dp_packet_clone(const struct dp_packet
*buffer
)
174 return dp_packet_clone_with_headroom(buffer
, 0);
177 /* Creates and returns a new dp_packet whose data are copied from 'buffer'.
178 * The returned dp_packet will additionally have 'headroom' bytes of
181 dp_packet_clone_with_headroom(const struct dp_packet
*buffer
, size_t headroom
)
183 struct dp_packet
*new_buffer
;
186 new_buffer
= dp_packet_clone_data_with_headroom(dp_packet_data(buffer
),
187 dp_packet_size(buffer
),
189 /* Copy the following fields into the returned buffer: l2_pad_size,
190 * l2_5_ofs, l3_ofs, l4_ofs, cutlen, packet_type and md. */
191 memcpy(&new_buffer
->l2_pad_size
, &buffer
->l2_pad_size
,
192 sizeof(struct dp_packet
) -
193 offsetof(struct dp_packet
, l2_pad_size
));
195 *dp_packet_ol_flags_ptr(new_buffer
) = *dp_packet_ol_flags_ptr(buffer
);
196 *dp_packet_ol_flags_ptr(new_buffer
) &= DP_PACKET_OL_SUPPORTED_MASK
;
198 if (dp_packet_rss_valid(buffer
)) {
199 dp_packet_set_rss_hash(new_buffer
, dp_packet_get_rss_hash(buffer
));
201 if (dp_packet_has_flow_mark(buffer
, &mark
)) {
202 dp_packet_set_flow_mark(new_buffer
, mark
);
208 /* Creates and returns a new dp_packet that initially contains a copy of the
209 * 'size' bytes of data starting at 'data' with no headroom or tailroom. */
211 dp_packet_clone_data(const void *data
, size_t size
)
213 return dp_packet_clone_data_with_headroom(data
, size
, 0);
216 /* Creates and returns a new dp_packet that initially contains 'headroom' bytes of
217 * headroom followed by a copy of the 'size' bytes of data starting at
220 dp_packet_clone_data_with_headroom(const void *data
, size_t size
, size_t headroom
)
222 struct dp_packet
*b
= dp_packet_new_with_headroom(size
, headroom
);
223 dp_packet_put(b
, data
, size
);
228 dp_packet_copy__(struct dp_packet
*b
, uint8_t *new_base
,
229 size_t new_headroom
, size_t new_tailroom
)
231 const uint8_t *old_base
= dp_packet_base(b
);
232 size_t old_headroom
= dp_packet_headroom(b
);
233 size_t old_tailroom
= dp_packet_tailroom(b
);
234 size_t copy_headroom
= MIN(old_headroom
, new_headroom
);
235 size_t copy_tailroom
= MIN(old_tailroom
, new_tailroom
);
237 memcpy(&new_base
[new_headroom
- copy_headroom
],
238 &old_base
[old_headroom
- copy_headroom
],
239 copy_headroom
+ dp_packet_size(b
) + copy_tailroom
);
242 /* Reallocates 'b' so that it has exactly 'new_headroom' and 'new_tailroom'
243 * bytes of headroom and tailroom, respectively. */
245 dp_packet_resize(struct dp_packet
*b
, size_t new_headroom
, size_t new_tailroom
)
247 void *new_base
, *new_data
;
248 size_t new_allocated
;
250 new_allocated
= new_headroom
+ dp_packet_size(b
) + new_tailroom
;
257 if (new_headroom
== dp_packet_headroom(b
)) {
258 new_base
= xrealloc(dp_packet_base(b
), new_allocated
);
260 new_base
= xmalloc(new_allocated
);
261 dp_packet_copy__(b
, new_base
, new_headroom
, new_tailroom
);
262 free(dp_packet_base(b
));
273 b
->source
= DPBUF_MALLOC
;
274 new_base
= xmalloc(new_allocated
);
275 dp_packet_copy__(b
, new_base
, new_headroom
, new_tailroom
);
282 dp_packet_set_allocated(b
, new_allocated
);
283 dp_packet_set_base(b
, new_base
);
285 new_data
= (char *) new_base
+ new_headroom
;
286 if (dp_packet_data(b
) != new_data
) {
287 dp_packet_set_data(b
, new_data
);
291 /* Ensures that 'b' has room for at least 'size' bytes at its tail end,
292 * reallocating and copying its data if necessary. Its headroom, if any, is
295 dp_packet_prealloc_tailroom(struct dp_packet
*b
, size_t size
)
297 if (size
> dp_packet_tailroom(b
)) {
298 dp_packet_resize(b
, dp_packet_headroom(b
), MAX(size
, 64));
302 /* Ensures that 'b' has room for at least 'size' bytes at its head,
303 * reallocating and copying its data if necessary. Its tailroom, if any, is
306 dp_packet_prealloc_headroom(struct dp_packet
*b
, size_t size
)
308 if (size
> dp_packet_headroom(b
)) {
309 dp_packet_resize(b
, MAX(size
, 64), dp_packet_tailroom(b
));
313 /* Shifts all of the data within the allocated space in 'b' by 'delta' bytes.
314 * For example, a 'delta' of 1 would cause each byte of data to move one byte
315 * forward (from address 'p' to 'p+1'), and a 'delta' of -1 would cause each
316 * byte to move one byte backward (from 'p' to 'p-1'). */
318 dp_packet_shift(struct dp_packet
*b
, int delta
)
320 ovs_assert(delta
> 0 ? delta
<= dp_packet_tailroom(b
)
321 : delta
< 0 ? -delta
<= dp_packet_headroom(b
)
325 char *dst
= (char *) dp_packet_data(b
) + delta
;
326 memmove(dst
, dp_packet_data(b
), dp_packet_size(b
));
327 dp_packet_set_data(b
, dst
);
331 /* Appends 'size' bytes of data to the tail end of 'b', reallocating and
332 * copying its data if necessary. Returns a pointer to the first byte of the
333 * new data, which is left uninitialized. */
335 dp_packet_put_uninit(struct dp_packet
*b
, size_t size
)
338 dp_packet_prealloc_tailroom(b
, size
);
339 p
= dp_packet_tail(b
);
340 dp_packet_set_size(b
, dp_packet_size(b
) + size
);
344 /* Appends 'size' zeroed bytes to the tail end of 'b'. Data in 'b' is
345 * reallocated and copied if necessary. Returns a pointer to the first byte of
346 * the data's location in the dp_packet. */
348 dp_packet_put_zeros(struct dp_packet
*b
, size_t size
)
350 void *dst
= dp_packet_put_uninit(b
, size
);
351 memset(dst
, 0, size
);
355 /* Appends the 'size' bytes of data in 'p' to the tail end of 'b'. Data in 'b'
356 * is reallocated and copied if necessary. Returns a pointer to the first
357 * byte of the data's location in the dp_packet. */
359 dp_packet_put(struct dp_packet
*b
, const void *p
, size_t size
)
361 void *dst
= dp_packet_put_uninit(b
, size
);
362 memcpy(dst
, p
, size
);
366 /* Parses as many pairs of hex digits as possible (possibly separated by
367 * spaces) from the beginning of 's', appending bytes for their values to 'b'.
368 * Returns the first character of 's' that is not the first of a pair of hex
369 * digits. If 'n' is nonnull, stores the number of bytes added to 'b' in
372 dp_packet_put_hex(struct dp_packet
*b
, const char *s
, size_t *n
)
374 size_t initial_size
= dp_packet_size(b
);
379 s
+= strspn(s
, " \t\r\n");
380 byte
= hexits_value(s
, 2, &ok
);
383 *n
= dp_packet_size(b
) - initial_size
;
385 return CONST_CAST(char *, s
);
388 dp_packet_put(b
, &byte
, 1);
393 /* Reserves 'size' bytes of headroom so that they can be later allocated with
394 * dp_packet_push_uninit() without reallocating the dp_packet. */
396 dp_packet_reserve(struct dp_packet
*b
, size_t size
)
398 ovs_assert(!dp_packet_size(b
));
399 dp_packet_prealloc_tailroom(b
, size
);
400 dp_packet_set_data(b
, (char*)dp_packet_data(b
) + size
);
403 /* Reserves 'headroom' bytes at the head and 'tailroom' at the end so that
404 * they can be later allocated with dp_packet_push_uninit() or
405 * dp_packet_put_uninit() without reallocating the dp_packet. */
407 dp_packet_reserve_with_tailroom(struct dp_packet
*b
, size_t headroom
,
410 ovs_assert(!dp_packet_size(b
));
411 dp_packet_prealloc_tailroom(b
, headroom
+ tailroom
);
412 dp_packet_set_data(b
, (char*)dp_packet_data(b
) + headroom
);
415 /* Prefixes 'size' bytes to the head end of 'b', reallocating and copying its
416 * data if necessary. Returns a pointer to the first byte of the data's
417 * location in the dp_packet. The new data is left uninitialized. */
419 dp_packet_push_uninit(struct dp_packet
*b
, size_t size
)
421 dp_packet_prealloc_headroom(b
, size
);
422 dp_packet_set_data(b
, (char*)dp_packet_data(b
) - size
);
423 dp_packet_set_size(b
, dp_packet_size(b
) + size
);
424 return dp_packet_data(b
);
427 /* Prefixes 'size' zeroed bytes to the head end of 'b', reallocating and
428 * copying its data if necessary. Returns a pointer to the first byte of the
429 * data's location in the dp_packet. */
431 dp_packet_push_zeros(struct dp_packet
*b
, size_t size
)
433 void *dst
= dp_packet_push_uninit(b
, size
);
434 memset(dst
, 0, size
);
438 /* Copies the 'size' bytes starting at 'p' to the head end of 'b', reallocating
439 * and copying its data if necessary. Returns a pointer to the first byte of
440 * the data's location in the dp_packet. */
442 dp_packet_push(struct dp_packet
*b
, const void *p
, size_t size
)
444 void *dst
= dp_packet_push_uninit(b
, size
);
445 memcpy(dst
, p
, size
);
449 /* Returns the data in 'b' as a block of malloc()'d memory and frees the buffer
450 * within 'b'. (If 'b' itself was dynamically allocated, e.g. with
451 * dp_packet_new(), then it should still be freed with, e.g., dp_packet_delete().) */
453 dp_packet_steal_data(struct dp_packet
*b
)
456 ovs_assert(b
->source
!= DPBUF_DPDK
);
457 ovs_assert(b
->source
!= DPBUF_AFXDP
);
459 if (b
->source
== DPBUF_MALLOC
&& dp_packet_data(b
) == dp_packet_base(b
)) {
460 p
= dp_packet_data(b
);
462 p
= xmemdup(dp_packet_data(b
), dp_packet_size(b
));
463 if (b
->source
== DPBUF_MALLOC
) {
464 free(dp_packet_base(b
));
467 dp_packet_set_base(b
, NULL
);
468 dp_packet_set_data(b
, NULL
);
473 dp_packet_adjust_layer_offset(uint16_t *offset
, int increment
)
475 if (*offset
!= UINT16_MAX
) {
476 *offset
+= increment
;
480 /* Adjust the size of the l2_5 portion of the dp_packet, updating the l2
481 * pointer and the layer offsets. The caller is responsible for
482 * modifying the contents. */
484 dp_packet_resize_l2_5(struct dp_packet
*b
, int increment
)
486 if (increment
>= 0) {
487 dp_packet_push_uninit(b
, increment
);
489 dp_packet_pull(b
, -increment
);
492 /* Adjust layer offsets after l2_5. */
493 dp_packet_adjust_layer_offset(&b
->l3_ofs
, increment
);
494 dp_packet_adjust_layer_offset(&b
->l4_ofs
, increment
);
496 return dp_packet_data(b
);
499 /* Adjust the size of the l2 portion of the dp_packet, updating the l2
500 * pointer and the layer offsets. The caller is responsible for
501 * modifying the contents. */
503 dp_packet_resize_l2(struct dp_packet
*b
, int increment
)
505 dp_packet_resize_l2_5(b
, increment
);
506 dp_packet_adjust_layer_offset(&b
->l2_5_ofs
, increment
);
507 return dp_packet_data(b
);