1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation.
3 * Copyright 2014 6WIND S.A.
14 #include <sys/queue.h>
16 #include <rte_compat.h>
17 #include <rte_debug.h>
18 #include <rte_common.h>
20 #include <rte_memory.h>
21 #include <rte_launch.h>
23 #include <rte_per_lcore.h>
24 #include <rte_lcore.h>
25 #include <rte_atomic.h>
26 #include <rte_branch_prediction.h>
27 #include <rte_mempool.h>
29 #include <rte_mbuf_pool_ops.h>
30 #include <rte_string_fns.h>
31 #include <rte_hexdump.h>
32 #include <rte_errno.h>
33 #include <rte_memcpy.h>
36 * pktmbuf pool constructor, given as a callback function to
37 * rte_mempool_create(), or called directly if using
38 * rte_mempool_create_empty()/rte_mempool_populate()
41 rte_pktmbuf_pool_init(struct rte_mempool
*mp
, void *opaque_arg
)
43 struct rte_pktmbuf_pool_private
*user_mbp_priv
, *mbp_priv
;
44 struct rte_pktmbuf_pool_private default_mbp_priv
;
47 RTE_ASSERT(mp
->elt_size
>= sizeof(struct rte_mbuf
));
49 /* if no structure is provided, assume no mbuf private area */
50 user_mbp_priv
= opaque_arg
;
51 if (user_mbp_priv
== NULL
) {
52 memset(&default_mbp_priv
, 0, sizeof(default_mbp_priv
));
53 if (mp
->elt_size
> sizeof(struct rte_mbuf
))
54 roomsz
= mp
->elt_size
- sizeof(struct rte_mbuf
);
57 default_mbp_priv
.mbuf_data_room_size
= roomsz
;
58 user_mbp_priv
= &default_mbp_priv
;
61 RTE_ASSERT(mp
->elt_size
>= sizeof(struct rte_mbuf
) +
62 ((user_mbp_priv
->flags
& RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF
) ?
63 sizeof(struct rte_mbuf_ext_shared_info
) :
64 user_mbp_priv
->mbuf_data_room_size
) +
65 user_mbp_priv
->mbuf_priv_size
);
66 RTE_ASSERT((user_mbp_priv
->flags
&
67 ~RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF
) == 0);
69 mbp_priv
= rte_mempool_get_priv(mp
);
70 memcpy(mbp_priv
, user_mbp_priv
, sizeof(*mbp_priv
));
74 * pktmbuf constructor, given as a callback function to
75 * rte_mempool_obj_iter() or rte_mempool_create().
76 * Set the fields of a packet mbuf to their default values.
79 rte_pktmbuf_init(struct rte_mempool
*mp
,
80 __rte_unused
void *opaque_arg
,
82 __rte_unused
unsigned i
)
84 struct rte_mbuf
*m
= _m
;
85 uint32_t mbuf_size
, buf_len
, priv_size
;
87 priv_size
= rte_pktmbuf_priv_size(mp
);
88 mbuf_size
= sizeof(struct rte_mbuf
) + priv_size
;
89 buf_len
= rte_pktmbuf_data_room_size(mp
);
91 RTE_ASSERT(RTE_ALIGN(priv_size
, RTE_MBUF_PRIV_ALIGN
) == priv_size
);
92 RTE_ASSERT(mp
->elt_size
>= mbuf_size
);
93 RTE_ASSERT(buf_len
<= UINT16_MAX
);
95 memset(m
, 0, mbuf_size
);
96 /* start of buffer is after mbuf structure and priv data */
97 m
->priv_size
= priv_size
;
98 m
->buf_addr
= (char *)m
+ mbuf_size
;
99 m
->buf_iova
= rte_mempool_virt2iova(m
) + mbuf_size
;
100 m
->buf_len
= (uint16_t)buf_len
;
102 /* keep some headroom between start of buffer and data */
103 m
->data_off
= RTE_MIN(RTE_PKTMBUF_HEADROOM
, (uint16_t)m
->buf_len
);
105 /* init some constant fields */
108 m
->port
= MBUF_INVALID_PORT
;
109 rte_mbuf_refcnt_set(m
, 1);
114 * @internal The callback routine called when reference counter in shinfo
115 * for mbufs with pinned external buffer reaches zero. It means there is
116 * no more reference to buffer backing mbuf and this one should be freed.
117 * This routine is called for the regular (not with pinned external or
118 * indirect buffer) mbufs on detaching from the mbuf with pinned external
122 rte_pktmbuf_free_pinned_extmem(void *addr
, void *opaque
)
124 struct rte_mbuf
*m
= opaque
;
127 RTE_ASSERT(RTE_MBUF_HAS_EXTBUF(m
));
128 RTE_ASSERT(RTE_MBUF_HAS_PINNED_EXTBUF(m
));
129 RTE_ASSERT(m
->shinfo
->fcb_opaque
== m
);
131 rte_mbuf_ext_refcnt_set(m
->shinfo
, 1);
132 m
->ol_flags
= EXT_ATTACHED_MBUF
;
133 if (m
->next
!= NULL
) {
137 rte_mbuf_raw_free(m
);
140 /** The context to initialize the mbufs with pinned external buffers. */
141 struct rte_pktmbuf_extmem_init_ctx
{
142 const struct rte_pktmbuf_extmem
*ext_mem
; /* descriptor array. */
143 unsigned int ext_num
; /* number of descriptors in array. */
144 unsigned int ext
; /* loop descriptor index. */
145 size_t off
; /* loop buffer offset. */
149 * @internal Packet mbuf constructor for pools with pinned external memory.
151 * This function initializes some fields in the mbuf structure that are
152 * not modified by the user once created (origin pool, buffer start
153 * address, and so on). This function is given as a callback function to
154 * rte_mempool_obj_iter() called from rte_mempool_create_extmem().
157 * The mempool from which mbufs originate.
159 * A pointer to the rte_pktmbuf_extmem_init_ctx - initialization
162 * The mbuf to initialize.
164 * The index of the mbuf in the pool table.
167 __rte_pktmbuf_init_extmem(struct rte_mempool
*mp
,
170 __rte_unused
unsigned int i
)
172 struct rte_mbuf
*m
= _m
;
173 struct rte_pktmbuf_extmem_init_ctx
*ctx
= opaque_arg
;
174 const struct rte_pktmbuf_extmem
*ext_mem
;
175 uint32_t mbuf_size
, buf_len
, priv_size
;
176 struct rte_mbuf_ext_shared_info
*shinfo
;
178 priv_size
= rte_pktmbuf_priv_size(mp
);
179 mbuf_size
= sizeof(struct rte_mbuf
) + priv_size
;
180 buf_len
= rte_pktmbuf_data_room_size(mp
);
182 RTE_ASSERT(RTE_ALIGN(priv_size
, RTE_MBUF_PRIV_ALIGN
) == priv_size
);
183 RTE_ASSERT(mp
->elt_size
>= mbuf_size
);
184 RTE_ASSERT(buf_len
<= UINT16_MAX
);
186 memset(m
, 0, mbuf_size
);
187 m
->priv_size
= priv_size
;
188 m
->buf_len
= (uint16_t)buf_len
;
190 /* set the data buffer pointers to external memory */
191 ext_mem
= ctx
->ext_mem
+ ctx
->ext
;
193 RTE_ASSERT(ctx
->ext
< ctx
->ext_num
);
194 RTE_ASSERT(ctx
->off
< ext_mem
->buf_len
);
196 m
->buf_addr
= RTE_PTR_ADD(ext_mem
->buf_ptr
, ctx
->off
);
197 m
->buf_iova
= ext_mem
->buf_iova
== RTE_BAD_IOVA
?
198 RTE_BAD_IOVA
: (ext_mem
->buf_iova
+ ctx
->off
);
200 ctx
->off
+= ext_mem
->elt_size
;
201 if (ctx
->off
>= ext_mem
->buf_len
) {
205 /* keep some headroom between start of buffer and data */
206 m
->data_off
= RTE_MIN(RTE_PKTMBUF_HEADROOM
, (uint16_t)m
->buf_len
);
208 /* init some constant fields */
211 m
->port
= MBUF_INVALID_PORT
;
212 m
->ol_flags
= EXT_ATTACHED_MBUF
;
213 rte_mbuf_refcnt_set(m
, 1);
216 /* init external buffer shared info items */
217 shinfo
= RTE_PTR_ADD(m
, mbuf_size
);
219 shinfo
->free_cb
= rte_pktmbuf_free_pinned_extmem
;
220 shinfo
->fcb_opaque
= m
;
221 rte_mbuf_ext_refcnt_set(shinfo
, 1);
224 /* Helper to create a mbuf pool with given mempool ops name*/
226 rte_pktmbuf_pool_create_by_ops(const char *name
, unsigned int n
,
227 unsigned int cache_size
, uint16_t priv_size
, uint16_t data_room_size
,
228 int socket_id
, const char *ops_name
)
230 struct rte_mempool
*mp
;
231 struct rte_pktmbuf_pool_private mbp_priv
;
232 const char *mp_ops_name
= ops_name
;
236 if (RTE_ALIGN(priv_size
, RTE_MBUF_PRIV_ALIGN
) != priv_size
) {
237 RTE_LOG(ERR
, MBUF
, "mbuf priv_size=%u is not aligned\n",
242 elt_size
= sizeof(struct rte_mbuf
) + (unsigned)priv_size
+
243 (unsigned)data_room_size
;
244 memset(&mbp_priv
, 0, sizeof(mbp_priv
));
245 mbp_priv
.mbuf_data_room_size
= data_room_size
;
246 mbp_priv
.mbuf_priv_size
= priv_size
;
248 mp
= rte_mempool_create_empty(name
, n
, elt_size
, cache_size
,
249 sizeof(struct rte_pktmbuf_pool_private
), socket_id
, 0);
253 if (mp_ops_name
== NULL
)
254 mp_ops_name
= rte_mbuf_best_mempool_ops();
255 ret
= rte_mempool_set_ops_byname(mp
, mp_ops_name
, NULL
);
257 RTE_LOG(ERR
, MBUF
, "error setting mempool handler\n");
258 rte_mempool_free(mp
);
262 rte_pktmbuf_pool_init(mp
, &mbp_priv
);
264 ret
= rte_mempool_populate_default(mp
);
266 rte_mempool_free(mp
);
271 rte_mempool_obj_iter(mp
, rte_pktmbuf_init
, NULL
);
276 /* helper to create a mbuf pool */
278 rte_pktmbuf_pool_create(const char *name
, unsigned int n
,
279 unsigned int cache_size
, uint16_t priv_size
, uint16_t data_room_size
,
282 return rte_pktmbuf_pool_create_by_ops(name
, n
, cache_size
, priv_size
,
283 data_room_size
, socket_id
, NULL
);
286 /* Helper to create a mbuf pool with pinned external data buffers. */
288 rte_pktmbuf_pool_create_extbuf(const char *name
, unsigned int n
,
289 unsigned int cache_size
, uint16_t priv_size
,
290 uint16_t data_room_size
, int socket_id
,
291 const struct rte_pktmbuf_extmem
*ext_mem
,
292 unsigned int ext_num
)
294 struct rte_mempool
*mp
;
295 struct rte_pktmbuf_pool_private mbp_priv
;
296 struct rte_pktmbuf_extmem_init_ctx init_ctx
;
297 const char *mp_ops_name
;
298 unsigned int elt_size
;
299 unsigned int i
, n_elts
= 0;
302 if (RTE_ALIGN(priv_size
, RTE_MBUF_PRIV_ALIGN
) != priv_size
) {
303 RTE_LOG(ERR
, MBUF
, "mbuf priv_size=%u is not aligned\n",
308 /* Check the external memory descriptors. */
309 for (i
= 0; i
< ext_num
; i
++) {
310 const struct rte_pktmbuf_extmem
*extm
= ext_mem
+ i
;
312 if (!extm
->elt_size
|| !extm
->buf_len
|| !extm
->buf_ptr
) {
313 RTE_LOG(ERR
, MBUF
, "invalid extmem descriptor\n");
317 if (data_room_size
> extm
->elt_size
) {
318 RTE_LOG(ERR
, MBUF
, "ext elt_size=%u is too small\n",
323 n_elts
+= extm
->buf_len
/ extm
->elt_size
;
325 /* Check whether enough external memory provided. */
327 RTE_LOG(ERR
, MBUF
, "not enough extmem\n");
331 elt_size
= sizeof(struct rte_mbuf
) +
332 (unsigned int)priv_size
+
333 sizeof(struct rte_mbuf_ext_shared_info
);
335 memset(&mbp_priv
, 0, sizeof(mbp_priv
));
336 mbp_priv
.mbuf_data_room_size
= data_room_size
;
337 mbp_priv
.mbuf_priv_size
= priv_size
;
338 mbp_priv
.flags
= RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF
;
340 mp
= rte_mempool_create_empty(name
, n
, elt_size
, cache_size
,
341 sizeof(struct rte_pktmbuf_pool_private
), socket_id
, 0);
345 mp_ops_name
= rte_mbuf_best_mempool_ops();
346 ret
= rte_mempool_set_ops_byname(mp
, mp_ops_name
, NULL
);
348 RTE_LOG(ERR
, MBUF
, "error setting mempool handler\n");
349 rte_mempool_free(mp
);
353 rte_pktmbuf_pool_init(mp
, &mbp_priv
);
355 ret
= rte_mempool_populate_default(mp
);
357 rte_mempool_free(mp
);
362 init_ctx
= (struct rte_pktmbuf_extmem_init_ctx
){
368 rte_mempool_obj_iter(mp
, __rte_pktmbuf_init_extmem
, &init_ctx
);
373 /* do some sanity checks on a mbuf: panic if it fails */
375 rte_mbuf_sanity_check(const struct rte_mbuf
*m
, int is_header
)
379 if (rte_mbuf_check(m
, is_header
, &reason
))
380 rte_panic("%s\n", reason
);
383 int rte_mbuf_check(const struct rte_mbuf
*m
, int is_header
,
386 unsigned int nb_segs
, pkt_len
;
389 *reason
= "mbuf is NULL";
394 if (m
->pool
== NULL
) {
395 *reason
= "bad mbuf pool";
398 if (m
->buf_iova
== 0) {
399 *reason
= "bad IO addr";
402 if (m
->buf_addr
== NULL
) {
403 *reason
= "bad virt addr";
407 uint16_t cnt
= rte_mbuf_refcnt_read(m
);
408 if ((cnt
== 0) || (cnt
== UINT16_MAX
)) {
409 *reason
= "bad ref cnt";
413 /* nothing to check for sub-segments */
417 /* data_len is supposed to be not more than pkt_len */
418 if (m
->data_len
> m
->pkt_len
) {
419 *reason
= "bad data_len";
423 nb_segs
= m
->nb_segs
;
424 pkt_len
= m
->pkt_len
;
427 if (m
->data_off
> m
->buf_len
) {
428 *reason
= "data offset too big in mbuf segment";
431 if (m
->data_off
+ m
->data_len
> m
->buf_len
) {
432 *reason
= "data length too big in mbuf segment";
436 pkt_len
-= m
->data_len
;
437 } while ((m
= m
->next
) != NULL
);
440 *reason
= "bad nb_segs";
444 *reason
= "bad pkt_len";
452 * @internal helper function for freeing a bulk of packet mbuf segments
453 * via an array holding the packet mbuf segments from the same mempool
454 * pending to be freed.
457 * The packet mbuf segment to be freed.
459 * Pointer to the array of packet mbuf segments pending to be freed.
461 * Pointer to the number of elements held in the array.
463 * Number of elements the array can hold.
464 * Note: The compiler should optimize this parameter away when using a
465 * constant value, such as RTE_PKTMBUF_FREE_PENDING_SZ.
468 __rte_pktmbuf_free_seg_via_array(struct rte_mbuf
*m
,
469 struct rte_mbuf
** const pending
, unsigned int * const nb_pending
,
470 const unsigned int pending_sz
)
472 m
= rte_pktmbuf_prefree_seg(m
);
473 if (likely(m
!= NULL
)) {
474 if (*nb_pending
== pending_sz
||
475 (*nb_pending
> 0 && m
->pool
!= pending
[0]->pool
)) {
476 rte_mempool_put_bulk(pending
[0]->pool
,
477 (void **)pending
, *nb_pending
);
481 pending
[(*nb_pending
)++] = m
;
486 * Size of the array holding mbufs from the same mempool pending to be freed
489 #define RTE_PKTMBUF_FREE_PENDING_SZ 64
491 /* Free a bulk of packet mbufs back into their original mempools. */
492 void rte_pktmbuf_free_bulk(struct rte_mbuf
**mbufs
, unsigned int count
)
494 struct rte_mbuf
*m
, *m_next
, *pending
[RTE_PKTMBUF_FREE_PENDING_SZ
];
495 unsigned int idx
, nb_pending
= 0;
497 for (idx
= 0; idx
< count
; idx
++) {
499 if (unlikely(m
== NULL
))
502 __rte_mbuf_sanity_check(m
, 1);
506 __rte_pktmbuf_free_seg_via_array(m
,
507 pending
, &nb_pending
,
508 RTE_PKTMBUF_FREE_PENDING_SZ
);
514 rte_mempool_put_bulk(pending
[0]->pool
, (void **)pending
, nb_pending
);
517 /* Creates a shallow copy of mbuf */
519 rte_pktmbuf_clone(struct rte_mbuf
*md
, struct rte_mempool
*mp
)
521 struct rte_mbuf
*mc
, *mi
, **prev
;
525 mc
= rte_pktmbuf_alloc(mp
);
526 if (unlikely(mc
== NULL
))
531 pktlen
= md
->pkt_len
;
536 rte_pktmbuf_attach(mi
, md
);
539 } while ((md
= md
->next
) != NULL
&&
540 (mi
= rte_pktmbuf_alloc(mp
)) != NULL
);
544 mc
->pkt_len
= pktlen
;
546 /* Allocation of new indirect segment failed */
547 if (unlikely(mi
== NULL
)) {
548 rte_pktmbuf_free(mc
);
552 __rte_mbuf_sanity_check(mc
, 1);
556 /* convert multi-segment mbuf to single mbuf */
558 __rte_pktmbuf_linearize(struct rte_mbuf
*mbuf
)
560 size_t seg_len
, copy_len
;
562 struct rte_mbuf
*m_next
;
565 /* Extend first segment to the total packet length */
566 copy_len
= rte_pktmbuf_pkt_len(mbuf
) - rte_pktmbuf_data_len(mbuf
);
568 if (unlikely(copy_len
> rte_pktmbuf_tailroom(mbuf
)))
571 buffer
= rte_pktmbuf_mtod_offset(mbuf
, char *, mbuf
->data_len
);
572 mbuf
->data_len
= (uint16_t)(mbuf
->pkt_len
);
574 /* Append data from next segments to the first one */
579 seg_len
= rte_pktmbuf_data_len(m
);
580 rte_memcpy(buffer
, rte_pktmbuf_mtod(m
, char *), seg_len
);
583 rte_pktmbuf_free_seg(m
);
593 /* Create a deep copy of mbuf */
595 rte_pktmbuf_copy(const struct rte_mbuf
*m
, struct rte_mempool
*mp
,
596 uint32_t off
, uint32_t len
)
598 const struct rte_mbuf
*seg
= m
;
599 struct rte_mbuf
*mc
, *m_last
, **prev
;
601 /* garbage in check */
602 __rte_mbuf_sanity_check(m
, 1);
604 /* check for request to copy at offset past end of mbuf */
605 if (unlikely(off
>= m
->pkt_len
))
608 mc
= rte_pktmbuf_alloc(mp
);
609 if (unlikely(mc
== NULL
))
612 /* truncate requested length to available data */
613 if (len
> m
->pkt_len
- off
)
614 len
= m
->pkt_len
- off
;
616 __rte_pktmbuf_copy_hdr(mc
, m
);
618 /* copied mbuf is not indirect or external */
619 mc
->ol_flags
= m
->ol_flags
& ~(IND_ATTACHED_MBUF
|EXT_ATTACHED_MBUF
);
626 /* skip leading mbuf segments */
627 while (off
>= seg
->data_len
) {
628 off
-= seg
->data_len
;
632 /* current buffer is full, chain a new one */
633 if (rte_pktmbuf_tailroom(m_last
) == 0) {
634 m_last
= rte_pktmbuf_alloc(mp
);
635 if (unlikely(m_last
== NULL
)) {
636 rte_pktmbuf_free(mc
);
641 prev
= &m_last
->next
;
645 * copy the min of data in input segment (seg)
646 * vs space available in output (m_last)
648 copy_len
= RTE_MIN(seg
->data_len
- off
, len
);
649 if (copy_len
> rte_pktmbuf_tailroom(m_last
))
650 copy_len
= rte_pktmbuf_tailroom(m_last
);
652 /* append from seg to m_last */
653 rte_memcpy(rte_pktmbuf_mtod_offset(m_last
, char *,
655 rte_pktmbuf_mtod_offset(seg
, char *, off
),
658 /* update offsets and lengths */
659 m_last
->data_len
+= copy_len
;
660 mc
->pkt_len
+= copy_len
;
665 /* garbage out check */
666 __rte_mbuf_sanity_check(mc
, 1);
670 /* dump a mbuf on console */
672 rte_pktmbuf_dump(FILE *f
, const struct rte_mbuf
*m
, unsigned dump_len
)
675 unsigned int nb_segs
;
677 __rte_mbuf_sanity_check(m
, 1);
679 fprintf(f
, "dump mbuf at %p, iova=%#"PRIx64
", buf_len=%u\n",
680 m
, m
->buf_iova
, m
->buf_len
);
681 fprintf(f
, " pkt_len=%u, ol_flags=%#"PRIx64
", nb_segs=%u, port=%u",
682 m
->pkt_len
, m
->ol_flags
, m
->nb_segs
, m
->port
);
684 if (m
->ol_flags
& (PKT_RX_VLAN
| PKT_TX_VLAN
))
685 fprintf(f
, ", vlan_tci=%u", m
->vlan_tci
);
687 fprintf(f
, ", ptype=%#"PRIx32
"\n", m
->packet_type
);
689 nb_segs
= m
->nb_segs
;
691 while (m
&& nb_segs
!= 0) {
692 __rte_mbuf_sanity_check(m
, 0);
694 fprintf(f
, " segment at %p, data=%p, len=%u, off=%u, refcnt=%u\n",
695 m
, rte_pktmbuf_mtod(m
, void *),
696 m
->data_len
, m
->data_off
, rte_mbuf_refcnt_read(m
));
699 if (len
> m
->data_len
)
702 rte_hexdump(f
, NULL
, rte_pktmbuf_mtod(m
, void *), len
);
709 /* read len data bytes in a mbuf at specified offset (internal) */
710 const void *__rte_pktmbuf_read(const struct rte_mbuf
*m
, uint32_t off
,
711 uint32_t len
, void *buf
)
713 const struct rte_mbuf
*seg
= m
;
714 uint32_t buf_off
= 0, copy_len
;
716 if (off
+ len
> rte_pktmbuf_pkt_len(m
))
719 while (off
>= rte_pktmbuf_data_len(seg
)) {
720 off
-= rte_pktmbuf_data_len(seg
);
724 if (off
+ len
<= rte_pktmbuf_data_len(seg
))
725 return rte_pktmbuf_mtod_offset(seg
, char *, off
);
727 /* rare case: header is split among several segments */
729 copy_len
= rte_pktmbuf_data_len(seg
) - off
;
732 rte_memcpy((char *)buf
+ buf_off
,
733 rte_pktmbuf_mtod_offset(seg
, char *, off
), copy_len
);
744 * Get the name of a RX offload flag. Must be kept synchronized with flag
745 * definitions in rte_mbuf.h.
747 const char *rte_get_rx_ol_flag_name(uint64_t mask
)
750 case PKT_RX_VLAN
: return "PKT_RX_VLAN";
751 case PKT_RX_RSS_HASH
: return "PKT_RX_RSS_HASH";
752 case PKT_RX_FDIR
: return "PKT_RX_FDIR";
753 case PKT_RX_L4_CKSUM_BAD
: return "PKT_RX_L4_CKSUM_BAD";
754 case PKT_RX_L4_CKSUM_GOOD
: return "PKT_RX_L4_CKSUM_GOOD";
755 case PKT_RX_L4_CKSUM_NONE
: return "PKT_RX_L4_CKSUM_NONE";
756 case PKT_RX_IP_CKSUM_BAD
: return "PKT_RX_IP_CKSUM_BAD";
757 case PKT_RX_IP_CKSUM_GOOD
: return "PKT_RX_IP_CKSUM_GOOD";
758 case PKT_RX_IP_CKSUM_NONE
: return "PKT_RX_IP_CKSUM_NONE";
759 case PKT_RX_EIP_CKSUM_BAD
: return "PKT_RX_EIP_CKSUM_BAD";
760 case PKT_RX_VLAN_STRIPPED
: return "PKT_RX_VLAN_STRIPPED";
761 case PKT_RX_IEEE1588_PTP
: return "PKT_RX_IEEE1588_PTP";
762 case PKT_RX_IEEE1588_TMST
: return "PKT_RX_IEEE1588_TMST";
763 case PKT_RX_FDIR_ID
: return "PKT_RX_FDIR_ID";
764 case PKT_RX_FDIR_FLX
: return "PKT_RX_FDIR_FLX";
765 case PKT_RX_QINQ_STRIPPED
: return "PKT_RX_QINQ_STRIPPED";
766 case PKT_RX_QINQ
: return "PKT_RX_QINQ";
767 case PKT_RX_LRO
: return "PKT_RX_LRO";
768 case PKT_RX_TIMESTAMP
: return "PKT_RX_TIMESTAMP";
769 case PKT_RX_SEC_OFFLOAD
: return "PKT_RX_SEC_OFFLOAD";
770 case PKT_RX_SEC_OFFLOAD_FAILED
: return "PKT_RX_SEC_OFFLOAD_FAILED";
771 case PKT_RX_OUTER_L4_CKSUM_BAD
: return "PKT_RX_OUTER_L4_CKSUM_BAD";
772 case PKT_RX_OUTER_L4_CKSUM_GOOD
: return "PKT_RX_OUTER_L4_CKSUM_GOOD";
773 case PKT_RX_OUTER_L4_CKSUM_INVALID
:
774 return "PKT_RX_OUTER_L4_CKSUM_INVALID";
776 default: return NULL
;
783 const char *default_name
;
786 /* write the list of rx ol flags in buffer buf */
788 rte_get_rx_ol_flag_list(uint64_t mask
, char *buf
, size_t buflen
)
790 const struct flag_mask rx_flags
[] = {
791 { PKT_RX_VLAN
, PKT_RX_VLAN
, NULL
},
792 { PKT_RX_RSS_HASH
, PKT_RX_RSS_HASH
, NULL
},
793 { PKT_RX_FDIR
, PKT_RX_FDIR
, NULL
},
794 { PKT_RX_L4_CKSUM_BAD
, PKT_RX_L4_CKSUM_MASK
, NULL
},
795 { PKT_RX_L4_CKSUM_GOOD
, PKT_RX_L4_CKSUM_MASK
, NULL
},
796 { PKT_RX_L4_CKSUM_NONE
, PKT_RX_L4_CKSUM_MASK
, NULL
},
797 { PKT_RX_L4_CKSUM_UNKNOWN
, PKT_RX_L4_CKSUM_MASK
,
798 "PKT_RX_L4_CKSUM_UNKNOWN" },
799 { PKT_RX_IP_CKSUM_BAD
, PKT_RX_IP_CKSUM_MASK
, NULL
},
800 { PKT_RX_IP_CKSUM_GOOD
, PKT_RX_IP_CKSUM_MASK
, NULL
},
801 { PKT_RX_IP_CKSUM_NONE
, PKT_RX_IP_CKSUM_MASK
, NULL
},
802 { PKT_RX_IP_CKSUM_UNKNOWN
, PKT_RX_IP_CKSUM_MASK
,
803 "PKT_RX_IP_CKSUM_UNKNOWN" },
804 { PKT_RX_EIP_CKSUM_BAD
, PKT_RX_EIP_CKSUM_BAD
, NULL
},
805 { PKT_RX_VLAN_STRIPPED
, PKT_RX_VLAN_STRIPPED
, NULL
},
806 { PKT_RX_IEEE1588_PTP
, PKT_RX_IEEE1588_PTP
, NULL
},
807 { PKT_RX_IEEE1588_TMST
, PKT_RX_IEEE1588_TMST
, NULL
},
808 { PKT_RX_FDIR_ID
, PKT_RX_FDIR_ID
, NULL
},
809 { PKT_RX_FDIR_FLX
, PKT_RX_FDIR_FLX
, NULL
},
810 { PKT_RX_QINQ_STRIPPED
, PKT_RX_QINQ_STRIPPED
, NULL
},
811 { PKT_RX_LRO
, PKT_RX_LRO
, NULL
},
812 { PKT_RX_TIMESTAMP
, PKT_RX_TIMESTAMP
, NULL
},
813 { PKT_RX_SEC_OFFLOAD
, PKT_RX_SEC_OFFLOAD
, NULL
},
814 { PKT_RX_SEC_OFFLOAD_FAILED
, PKT_RX_SEC_OFFLOAD_FAILED
, NULL
},
815 { PKT_RX_QINQ
, PKT_RX_QINQ
, NULL
},
816 { PKT_RX_OUTER_L4_CKSUM_BAD
, PKT_RX_OUTER_L4_CKSUM_MASK
, NULL
},
817 { PKT_RX_OUTER_L4_CKSUM_GOOD
, PKT_RX_OUTER_L4_CKSUM_MASK
,
819 { PKT_RX_OUTER_L4_CKSUM_INVALID
, PKT_RX_OUTER_L4_CKSUM_MASK
,
821 { PKT_RX_OUTER_L4_CKSUM_UNKNOWN
, PKT_RX_OUTER_L4_CKSUM_MASK
,
822 "PKT_RX_OUTER_L4_CKSUM_UNKNOWN" },
832 for (i
= 0; i
< RTE_DIM(rx_flags
); i
++) {
833 if ((mask
& rx_flags
[i
].mask
) != rx_flags
[i
].flag
)
835 name
= rte_get_rx_ol_flag_name(rx_flags
[i
].flag
);
837 name
= rx_flags
[i
].default_name
;
838 ret
= snprintf(buf
, buflen
, "%s ", name
);
841 if ((size_t)ret
>= buflen
)
851 * Get the name of a TX offload flag. Must be kept synchronized with flag
852 * definitions in rte_mbuf.h.
854 const char *rte_get_tx_ol_flag_name(uint64_t mask
)
857 case PKT_TX_VLAN
: return "PKT_TX_VLAN";
858 case PKT_TX_IP_CKSUM
: return "PKT_TX_IP_CKSUM";
859 case PKT_TX_TCP_CKSUM
: return "PKT_TX_TCP_CKSUM";
860 case PKT_TX_SCTP_CKSUM
: return "PKT_TX_SCTP_CKSUM";
861 case PKT_TX_UDP_CKSUM
: return "PKT_TX_UDP_CKSUM";
862 case PKT_TX_IEEE1588_TMST
: return "PKT_TX_IEEE1588_TMST";
863 case PKT_TX_TCP_SEG
: return "PKT_TX_TCP_SEG";
864 case PKT_TX_IPV4
: return "PKT_TX_IPV4";
865 case PKT_TX_IPV6
: return "PKT_TX_IPV6";
866 case PKT_TX_OUTER_IP_CKSUM
: return "PKT_TX_OUTER_IP_CKSUM";
867 case PKT_TX_OUTER_IPV4
: return "PKT_TX_OUTER_IPV4";
868 case PKT_TX_OUTER_IPV6
: return "PKT_TX_OUTER_IPV6";
869 case PKT_TX_TUNNEL_VXLAN
: return "PKT_TX_TUNNEL_VXLAN";
870 case PKT_TX_TUNNEL_GTP
: return "PKT_TX_TUNNEL_GTP";
871 case PKT_TX_TUNNEL_GRE
: return "PKT_TX_TUNNEL_GRE";
872 case PKT_TX_TUNNEL_IPIP
: return "PKT_TX_TUNNEL_IPIP";
873 case PKT_TX_TUNNEL_GENEVE
: return "PKT_TX_TUNNEL_GENEVE";
874 case PKT_TX_TUNNEL_MPLSINUDP
: return "PKT_TX_TUNNEL_MPLSINUDP";
875 case PKT_TX_TUNNEL_VXLAN_GPE
: return "PKT_TX_TUNNEL_VXLAN_GPE";
876 case PKT_TX_TUNNEL_IP
: return "PKT_TX_TUNNEL_IP";
877 case PKT_TX_TUNNEL_UDP
: return "PKT_TX_TUNNEL_UDP";
878 case PKT_TX_QINQ
: return "PKT_TX_QINQ";
879 case PKT_TX_MACSEC
: return "PKT_TX_MACSEC";
880 case PKT_TX_SEC_OFFLOAD
: return "PKT_TX_SEC_OFFLOAD";
881 case PKT_TX_UDP_SEG
: return "PKT_TX_UDP_SEG";
882 case PKT_TX_OUTER_UDP_CKSUM
: return "PKT_TX_OUTER_UDP_CKSUM";
883 default: return NULL
;
887 /* write the list of tx ol flags in buffer buf */
889 rte_get_tx_ol_flag_list(uint64_t mask
, char *buf
, size_t buflen
)
891 const struct flag_mask tx_flags
[] = {
892 { PKT_TX_VLAN
, PKT_TX_VLAN
, NULL
},
893 { PKT_TX_IP_CKSUM
, PKT_TX_IP_CKSUM
, NULL
},
894 { PKT_TX_TCP_CKSUM
, PKT_TX_L4_MASK
, NULL
},
895 { PKT_TX_SCTP_CKSUM
, PKT_TX_L4_MASK
, NULL
},
896 { PKT_TX_UDP_CKSUM
, PKT_TX_L4_MASK
, NULL
},
897 { PKT_TX_L4_NO_CKSUM
, PKT_TX_L4_MASK
, "PKT_TX_L4_NO_CKSUM" },
898 { PKT_TX_IEEE1588_TMST
, PKT_TX_IEEE1588_TMST
, NULL
},
899 { PKT_TX_TCP_SEG
, PKT_TX_TCP_SEG
, NULL
},
900 { PKT_TX_IPV4
, PKT_TX_IPV4
, NULL
},
901 { PKT_TX_IPV6
, PKT_TX_IPV6
, NULL
},
902 { PKT_TX_OUTER_IP_CKSUM
, PKT_TX_OUTER_IP_CKSUM
, NULL
},
903 { PKT_TX_OUTER_IPV4
, PKT_TX_OUTER_IPV4
, NULL
},
904 { PKT_TX_OUTER_IPV6
, PKT_TX_OUTER_IPV6
, NULL
},
905 { PKT_TX_TUNNEL_VXLAN
, PKT_TX_TUNNEL_MASK
, NULL
},
906 { PKT_TX_TUNNEL_GTP
, PKT_TX_TUNNEL_MASK
, NULL
},
907 { PKT_TX_TUNNEL_GRE
, PKT_TX_TUNNEL_MASK
, NULL
},
908 { PKT_TX_TUNNEL_IPIP
, PKT_TX_TUNNEL_MASK
, NULL
},
909 { PKT_TX_TUNNEL_GENEVE
, PKT_TX_TUNNEL_MASK
, NULL
},
910 { PKT_TX_TUNNEL_MPLSINUDP
, PKT_TX_TUNNEL_MASK
, NULL
},
911 { PKT_TX_TUNNEL_VXLAN_GPE
, PKT_TX_TUNNEL_MASK
, NULL
},
912 { PKT_TX_TUNNEL_IP
, PKT_TX_TUNNEL_MASK
, NULL
},
913 { PKT_TX_TUNNEL_UDP
, PKT_TX_TUNNEL_MASK
, NULL
},
914 { PKT_TX_QINQ
, PKT_TX_QINQ
, NULL
},
915 { PKT_TX_MACSEC
, PKT_TX_MACSEC
, NULL
},
916 { PKT_TX_SEC_OFFLOAD
, PKT_TX_SEC_OFFLOAD
, NULL
},
917 { PKT_TX_UDP_SEG
, PKT_TX_UDP_SEG
, NULL
},
918 { PKT_TX_OUTER_UDP_CKSUM
, PKT_TX_OUTER_UDP_CKSUM
, NULL
},
928 for (i
= 0; i
< RTE_DIM(tx_flags
); i
++) {
929 if ((mask
& tx_flags
[i
].mask
) != tx_flags
[i
].flag
)
931 name
= rte_get_tx_ol_flag_name(tx_flags
[i
].flag
);
933 name
= tx_flags
[i
].default_name
;
934 ret
= snprintf(buf
, buflen
, "%s ", name
);
937 if ((size_t)ret
>= buflen
)