1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation.
3 * Copyright 2014 6WIND S.A.
13 #include <sys/queue.h>
16 #include <rte_common.h>
17 #include <rte_byteorder.h>
19 #include <rte_debug.h>
20 #include <rte_cycles.h>
21 #include <rte_memory.h>
22 #include <rte_memcpy.h>
23 #include <rte_launch.h>
25 #include <rte_per_lcore.h>
26 #include <rte_lcore.h>
27 #include <rte_atomic.h>
28 #include <rte_branch_prediction.h>
29 #include <rte_mempool.h>
31 #include <rte_interrupts.h>
33 #include <rte_ether.h>
34 #include <rte_ethdev.h>
39 #include <rte_prefetch.h>
40 #include <rte_string_fns.h>
47 #define IP_DEFTTL 64 /* from RFC 1340. */
48 #define IP_VERSION 0x40
49 #define IP_HDRLEN 0x05 /* default IP header length == five 32-bits words. */
50 #define IP_VHL_DEF (IP_VERSION | IP_HDRLEN)
52 #define GRE_CHECKSUM_PRESENT 0x8000
53 #define GRE_KEY_PRESENT 0x2000
54 #define GRE_SEQUENCE_PRESENT 0x1000
56 #define GRE_SUPPORTED_FIELDS (GRE_CHECKSUM_PRESENT | GRE_KEY_PRESENT |\
59 /* We cannot use rte_cpu_to_be_16() on a constant in a switch/case */
60 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
61 #define _htons(x) ((uint16_t)((((x) & 0x00ffU) << 8) | (((x) & 0xff00U) >> 8)))
66 uint16_t vxlan_gpe_udp_port
= 4790;
68 /* structure that caches offload info for the current packet */
69 struct testpmd_offload_info
{
77 uint16_t outer_ethertype
;
78 uint16_t outer_l2_len
;
79 uint16_t outer_l3_len
;
80 uint8_t outer_l4_proto
;
82 uint16_t tunnel_tso_segsz
;
86 /* simplified GRE header */
87 struct simple_gre_hdr
{
90 } __attribute__((__packed__
));
93 get_udptcp_checksum(void *l3_hdr
, void *l4_hdr
, uint16_t ethertype
)
95 if (ethertype
== _htons(ETHER_TYPE_IPv4
))
96 return rte_ipv4_udptcp_cksum(l3_hdr
, l4_hdr
);
97 else /* assume ethertype == ETHER_TYPE_IPv6 */
98 return rte_ipv6_udptcp_cksum(l3_hdr
, l4_hdr
);
101 /* Parse an IPv4 header to fill l3_len, l4_len, and l4_proto */
103 parse_ipv4(struct ipv4_hdr
*ipv4_hdr
, struct testpmd_offload_info
*info
)
105 struct tcp_hdr
*tcp_hdr
;
107 info
->l3_len
= (ipv4_hdr
->version_ihl
& 0x0f) * 4;
108 info
->l4_proto
= ipv4_hdr
->next_proto_id
;
110 /* only fill l4_len for TCP, it's useful for TSO */
111 if (info
->l4_proto
== IPPROTO_TCP
) {
112 tcp_hdr
= (struct tcp_hdr
*)((char *)ipv4_hdr
+ info
->l3_len
);
113 info
->l4_len
= (tcp_hdr
->data_off
& 0xf0) >> 2;
118 /* Parse an IPv6 header to fill l3_len, l4_len, and l4_proto */
120 parse_ipv6(struct ipv6_hdr
*ipv6_hdr
, struct testpmd_offload_info
*info
)
122 struct tcp_hdr
*tcp_hdr
;
124 info
->l3_len
= sizeof(struct ipv6_hdr
);
125 info
->l4_proto
= ipv6_hdr
->proto
;
127 /* only fill l4_len for TCP, it's useful for TSO */
128 if (info
->l4_proto
== IPPROTO_TCP
) {
129 tcp_hdr
= (struct tcp_hdr
*)((char *)ipv6_hdr
+ info
->l3_len
);
130 info
->l4_len
= (tcp_hdr
->data_off
& 0xf0) >> 2;
136 * Parse an ethernet header to fill the ethertype, l2_len, l3_len and
137 * ipproto. This function is able to recognize IPv4/IPv6 with one optional vlan
138 * header. The l4_len argument is only set in case of TCP (useful for TSO).
141 parse_ethernet(struct ether_hdr
*eth_hdr
, struct testpmd_offload_info
*info
)
143 struct ipv4_hdr
*ipv4_hdr
;
144 struct ipv6_hdr
*ipv6_hdr
;
146 info
->l2_len
= sizeof(struct ether_hdr
);
147 info
->ethertype
= eth_hdr
->ether_type
;
149 if (info
->ethertype
== _htons(ETHER_TYPE_VLAN
)) {
150 struct vlan_hdr
*vlan_hdr
= (struct vlan_hdr
*)(eth_hdr
+ 1);
152 info
->l2_len
+= sizeof(struct vlan_hdr
);
153 info
->ethertype
= vlan_hdr
->eth_proto
;
156 switch (info
->ethertype
) {
157 case _htons(ETHER_TYPE_IPv4
):
158 ipv4_hdr
= (struct ipv4_hdr
*) ((char *)eth_hdr
+ info
->l2_len
);
159 parse_ipv4(ipv4_hdr
, info
);
161 case _htons(ETHER_TYPE_IPv6
):
162 ipv6_hdr
= (struct ipv6_hdr
*) ((char *)eth_hdr
+ info
->l2_len
);
163 parse_ipv6(ipv6_hdr
, info
);
173 /* Parse a vxlan header */
175 parse_vxlan(struct udp_hdr
*udp_hdr
,
176 struct testpmd_offload_info
*info
,
179 struct ether_hdr
*eth_hdr
;
181 /* check udp destination port, 4789 is the default vxlan port
182 * (rfc7348) or that the rx offload flag is set (i40e only
184 if (udp_hdr
->dst_port
!= _htons(4789) &&
185 RTE_ETH_IS_TUNNEL_PKT(pkt_type
) == 0)
189 info
->outer_ethertype
= info
->ethertype
;
190 info
->outer_l2_len
= info
->l2_len
;
191 info
->outer_l3_len
= info
->l3_len
;
192 info
->outer_l4_proto
= info
->l4_proto
;
194 eth_hdr
= (struct ether_hdr
*)((char *)udp_hdr
+
195 sizeof(struct udp_hdr
) +
196 sizeof(struct vxlan_hdr
));
198 parse_ethernet(eth_hdr
, info
);
199 info
->l2_len
+= ETHER_VXLAN_HLEN
; /* add udp + vxlan */
202 /* Parse a vxlan-gpe header */
204 parse_vxlan_gpe(struct udp_hdr
*udp_hdr
,
205 struct testpmd_offload_info
*info
)
207 struct ether_hdr
*eth_hdr
;
208 struct ipv4_hdr
*ipv4_hdr
;
209 struct ipv6_hdr
*ipv6_hdr
;
210 struct vxlan_gpe_hdr
*vxlan_gpe_hdr
;
211 uint8_t vxlan_gpe_len
= sizeof(*vxlan_gpe_hdr
);
213 /* Check udp destination port. */
214 if (udp_hdr
->dst_port
!= _htons(vxlan_gpe_udp_port
))
217 vxlan_gpe_hdr
= (struct vxlan_gpe_hdr
*)((char *)udp_hdr
+
218 sizeof(struct udp_hdr
));
220 if (!vxlan_gpe_hdr
->proto
|| vxlan_gpe_hdr
->proto
==
221 VXLAN_GPE_TYPE_IPV4
) {
223 info
->outer_ethertype
= info
->ethertype
;
224 info
->outer_l2_len
= info
->l2_len
;
225 info
->outer_l3_len
= info
->l3_len
;
226 info
->outer_l4_proto
= info
->l4_proto
;
228 ipv4_hdr
= (struct ipv4_hdr
*)((char *)vxlan_gpe_hdr
+
231 parse_ipv4(ipv4_hdr
, info
);
232 info
->ethertype
= _htons(ETHER_TYPE_IPv4
);
235 } else if (vxlan_gpe_hdr
->proto
== VXLAN_GPE_TYPE_IPV6
) {
237 info
->outer_ethertype
= info
->ethertype
;
238 info
->outer_l2_len
= info
->l2_len
;
239 info
->outer_l3_len
= info
->l3_len
;
240 info
->outer_l4_proto
= info
->l4_proto
;
242 ipv6_hdr
= (struct ipv6_hdr
*)((char *)vxlan_gpe_hdr
+
245 info
->ethertype
= _htons(ETHER_TYPE_IPv6
);
246 parse_ipv6(ipv6_hdr
, info
);
249 } else if (vxlan_gpe_hdr
->proto
== VXLAN_GPE_TYPE_ETH
) {
251 info
->outer_ethertype
= info
->ethertype
;
252 info
->outer_l2_len
= info
->l2_len
;
253 info
->outer_l3_len
= info
->l3_len
;
254 info
->outer_l4_proto
= info
->l4_proto
;
256 eth_hdr
= (struct ether_hdr
*)((char *)vxlan_gpe_hdr
+
259 parse_ethernet(eth_hdr
, info
);
263 info
->l2_len
+= ETHER_VXLAN_GPE_HLEN
;
266 /* Parse a gre header */
268 parse_gre(struct simple_gre_hdr
*gre_hdr
, struct testpmd_offload_info
*info
)
270 struct ether_hdr
*eth_hdr
;
271 struct ipv4_hdr
*ipv4_hdr
;
272 struct ipv6_hdr
*ipv6_hdr
;
275 gre_len
+= sizeof(struct simple_gre_hdr
);
277 if (gre_hdr
->flags
& _htons(GRE_KEY_PRESENT
))
278 gre_len
+= GRE_EXT_LEN
;
279 if (gre_hdr
->flags
& _htons(GRE_SEQUENCE_PRESENT
))
280 gre_len
+= GRE_EXT_LEN
;
281 if (gre_hdr
->flags
& _htons(GRE_CHECKSUM_PRESENT
))
282 gre_len
+= GRE_EXT_LEN
;
284 if (gre_hdr
->proto
== _htons(ETHER_TYPE_IPv4
)) {
286 info
->outer_ethertype
= info
->ethertype
;
287 info
->outer_l2_len
= info
->l2_len
;
288 info
->outer_l3_len
= info
->l3_len
;
289 info
->outer_l4_proto
= info
->l4_proto
;
291 ipv4_hdr
= (struct ipv4_hdr
*)((char *)gre_hdr
+ gre_len
);
293 parse_ipv4(ipv4_hdr
, info
);
294 info
->ethertype
= _htons(ETHER_TYPE_IPv4
);
297 } else if (gre_hdr
->proto
== _htons(ETHER_TYPE_IPv6
)) {
299 info
->outer_ethertype
= info
->ethertype
;
300 info
->outer_l2_len
= info
->l2_len
;
301 info
->outer_l3_len
= info
->l3_len
;
302 info
->outer_l4_proto
= info
->l4_proto
;
304 ipv6_hdr
= (struct ipv6_hdr
*)((char *)gre_hdr
+ gre_len
);
306 info
->ethertype
= _htons(ETHER_TYPE_IPv6
);
307 parse_ipv6(ipv6_hdr
, info
);
310 } else if (gre_hdr
->proto
== _htons(ETHER_TYPE_TEB
)) {
312 info
->outer_ethertype
= info
->ethertype
;
313 info
->outer_l2_len
= info
->l2_len
;
314 info
->outer_l3_len
= info
->l3_len
;
315 info
->outer_l4_proto
= info
->l4_proto
;
317 eth_hdr
= (struct ether_hdr
*)((char *)gre_hdr
+ gre_len
);
319 parse_ethernet(eth_hdr
, info
);
323 info
->l2_len
+= gre_len
;
327 /* Parse an encapsulated ip or ipv6 header */
329 parse_encap_ip(void *encap_ip
, struct testpmd_offload_info
*info
)
331 struct ipv4_hdr
*ipv4_hdr
= encap_ip
;
332 struct ipv6_hdr
*ipv6_hdr
= encap_ip
;
335 ip_version
= (ipv4_hdr
->version_ihl
& 0xf0) >> 4;
337 if (ip_version
!= 4 && ip_version
!= 6)
341 info
->outer_ethertype
= info
->ethertype
;
342 info
->outer_l2_len
= info
->l2_len
;
343 info
->outer_l3_len
= info
->l3_len
;
345 if (ip_version
== 4) {
346 parse_ipv4(ipv4_hdr
, info
);
347 info
->ethertype
= _htons(ETHER_TYPE_IPv4
);
349 parse_ipv6(ipv6_hdr
, info
);
350 info
->ethertype
= _htons(ETHER_TYPE_IPv6
);
355 /* if possible, calculate the checksum of a packet in hw or sw,
356 * depending on the testpmd command line configuration */
358 process_inner_cksums(void *l3_hdr
, const struct testpmd_offload_info
*info
,
359 uint64_t tx_offloads
)
361 struct ipv4_hdr
*ipv4_hdr
= l3_hdr
;
362 struct udp_hdr
*udp_hdr
;
363 struct tcp_hdr
*tcp_hdr
;
364 struct sctp_hdr
*sctp_hdr
;
365 uint64_t ol_flags
= 0;
366 uint32_t max_pkt_len
, tso_segsz
= 0;
368 /* ensure packet is large enough to require tso */
369 if (!info
->is_tunnel
) {
370 max_pkt_len
= info
->l2_len
+ info
->l3_len
+ info
->l4_len
+
372 if (info
->tso_segsz
!= 0 && info
->pkt_len
> max_pkt_len
)
373 tso_segsz
= info
->tso_segsz
;
375 max_pkt_len
= info
->outer_l2_len
+ info
->outer_l3_len
+
376 info
->l2_len
+ info
->l3_len
+ info
->l4_len
+
377 info
->tunnel_tso_segsz
;
378 if (info
->tunnel_tso_segsz
!= 0 && info
->pkt_len
> max_pkt_len
)
379 tso_segsz
= info
->tunnel_tso_segsz
;
382 if (info
->ethertype
== _htons(ETHER_TYPE_IPv4
)) {
384 ipv4_hdr
->hdr_checksum
= 0;
386 ol_flags
|= PKT_TX_IPV4
;
387 if (info
->l4_proto
== IPPROTO_TCP
&& tso_segsz
) {
388 ol_flags
|= PKT_TX_IP_CKSUM
;
390 if (tx_offloads
& DEV_TX_OFFLOAD_IPV4_CKSUM
)
391 ol_flags
|= PKT_TX_IP_CKSUM
;
393 ipv4_hdr
->hdr_checksum
=
394 rte_ipv4_cksum(ipv4_hdr
);
396 } else if (info
->ethertype
== _htons(ETHER_TYPE_IPv6
))
397 ol_flags
|= PKT_TX_IPV6
;
399 return 0; /* packet type not supported, nothing to do */
401 if (info
->l4_proto
== IPPROTO_UDP
) {
402 udp_hdr
= (struct udp_hdr
*)((char *)l3_hdr
+ info
->l3_len
);
403 /* do not recalculate udp cksum if it was 0 */
404 if (udp_hdr
->dgram_cksum
!= 0) {
405 udp_hdr
->dgram_cksum
= 0;
406 if (tx_offloads
& DEV_TX_OFFLOAD_UDP_CKSUM
)
407 ol_flags
|= PKT_TX_UDP_CKSUM
;
409 udp_hdr
->dgram_cksum
=
410 get_udptcp_checksum(l3_hdr
, udp_hdr
,
414 if (info
->gso_enable
)
415 ol_flags
|= PKT_TX_UDP_SEG
;
416 } else if (info
->l4_proto
== IPPROTO_TCP
) {
417 tcp_hdr
= (struct tcp_hdr
*)((char *)l3_hdr
+ info
->l3_len
);
420 ol_flags
|= PKT_TX_TCP_SEG
;
421 else if (tx_offloads
& DEV_TX_OFFLOAD_TCP_CKSUM
)
422 ol_flags
|= PKT_TX_TCP_CKSUM
;
425 get_udptcp_checksum(l3_hdr
, tcp_hdr
,
428 if (info
->gso_enable
)
429 ol_flags
|= PKT_TX_TCP_SEG
;
430 } else if (info
->l4_proto
== IPPROTO_SCTP
) {
431 sctp_hdr
= (struct sctp_hdr
*)((char *)l3_hdr
+ info
->l3_len
);
433 /* sctp payload must be a multiple of 4 to be
435 if ((tx_offloads
& DEV_TX_OFFLOAD_SCTP_CKSUM
) &&
436 ((ipv4_hdr
->total_length
& 0x3) == 0)) {
437 ol_flags
|= PKT_TX_SCTP_CKSUM
;
439 /* XXX implement CRC32c, example available in
447 /* Calculate the checksum of outer header */
449 process_outer_cksums(void *outer_l3_hdr
, struct testpmd_offload_info
*info
,
450 uint64_t tx_offloads
, int tso_enabled
)
452 struct ipv4_hdr
*ipv4_hdr
= outer_l3_hdr
;
453 struct ipv6_hdr
*ipv6_hdr
= outer_l3_hdr
;
454 struct udp_hdr
*udp_hdr
;
455 uint64_t ol_flags
= 0;
457 if (info
->outer_ethertype
== _htons(ETHER_TYPE_IPv4
)) {
458 ipv4_hdr
->hdr_checksum
= 0;
459 ol_flags
|= PKT_TX_OUTER_IPV4
;
461 if (tx_offloads
& DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM
)
462 ol_flags
|= PKT_TX_OUTER_IP_CKSUM
;
464 ipv4_hdr
->hdr_checksum
= rte_ipv4_cksum(ipv4_hdr
);
466 ol_flags
|= PKT_TX_OUTER_IPV6
;
468 if (info
->outer_l4_proto
!= IPPROTO_UDP
)
471 udp_hdr
= (struct udp_hdr
*)((char *)outer_l3_hdr
+ info
->outer_l3_len
);
473 /* outer UDP checksum is done in software as we have no hardware
474 * supporting it today, and no API for it. In the other side, for
475 * UDP tunneling, like VXLAN or Geneve, outer UDP checksum can be
478 * If a packet will be TSOed into small packets by NIC, we cannot
479 * set/calculate a non-zero checksum, because it will be a wrong
480 * value after the packet be split into several small packets.
483 udp_hdr
->dgram_cksum
= 0;
485 /* do not recalculate udp cksum if it was 0 */
486 if (udp_hdr
->dgram_cksum
!= 0) {
487 udp_hdr
->dgram_cksum
= 0;
488 if (info
->outer_ethertype
== _htons(ETHER_TYPE_IPv4
))
489 udp_hdr
->dgram_cksum
=
490 rte_ipv4_udptcp_cksum(ipv4_hdr
, udp_hdr
);
492 udp_hdr
->dgram_cksum
=
493 rte_ipv6_udptcp_cksum(ipv6_hdr
, udp_hdr
);
501 * Performs actual copying.
502 * Returns number of segments in the destination mbuf on success,
503 * or negative error code on failure.
506 mbuf_copy_split(const struct rte_mbuf
*ms
, struct rte_mbuf
*md
[],
507 uint16_t seglen
[], uint8_t nb_seg
)
509 uint32_t dlen
, slen
, tlen
;
511 const struct rte_mbuf
*m
;
524 while (ms
!= NULL
&& i
!= nb_seg
) {
527 slen
= rte_pktmbuf_data_len(ms
);
528 src
= rte_pktmbuf_mtod(ms
, const uint8_t *);
532 dlen
= RTE_MIN(seglen
[i
], slen
);
533 md
[i
]->data_len
= dlen
;
534 md
[i
]->next
= (i
+ 1 == nb_seg
) ? NULL
: md
[i
+ 1];
535 dst
= rte_pktmbuf_mtod(md
[i
], uint8_t *);
538 len
= RTE_MIN(slen
, dlen
);
539 memcpy(dst
, src
, len
);
554 else if (tlen
!= m
->pkt_len
)
557 md
[0]->nb_segs
= nb_seg
;
558 md
[0]->pkt_len
= tlen
;
559 md
[0]->vlan_tci
= m
->vlan_tci
;
560 md
[0]->vlan_tci_outer
= m
->vlan_tci_outer
;
561 md
[0]->ol_flags
= m
->ol_flags
;
562 md
[0]->tx_offload
= m
->tx_offload
;
568 * Allocate a new mbuf with up to tx_pkt_nb_segs segments.
569 * Copy packet contents and offload information into then new segmented mbuf.
571 static struct rte_mbuf
*
572 pkt_copy_split(const struct rte_mbuf
*pkt
)
575 uint32_t i
, len
, nb_seg
;
576 struct rte_mempool
*mp
;
577 uint16_t seglen
[RTE_MAX_SEGS_PER_PKT
];
578 struct rte_mbuf
*p
, *md
[RTE_MAX_SEGS_PER_PKT
];
580 mp
= current_fwd_lcore()->mbp
;
582 if (tx_pkt_split
== TX_PKT_SPLIT_RND
)
583 nb_seg
= random() % tx_pkt_nb_segs
+ 1;
585 nb_seg
= tx_pkt_nb_segs
;
587 memcpy(seglen
, tx_pkt_seg_lengths
, nb_seg
* sizeof(seglen
[0]));
589 /* calculate number of segments to use and their length. */
591 for (i
= 0; i
!= nb_seg
&& len
< pkt
->pkt_len
; i
++) {
596 n
= pkt
->pkt_len
- len
;
598 /* update size of the last segment to fit rest of the packet */
606 p
= rte_pktmbuf_alloc(mp
);
609 "failed to allocate %u-th of %u mbuf "
610 "from mempool: %s\n",
611 nb_seg
- i
, nb_seg
, mp
->name
);
616 if (rte_pktmbuf_tailroom(md
[i
]) < seglen
[i
]) {
617 TESTPMD_LOG(ERR
, "mempool %s, %u-th segment: "
618 "expected seglen: %u, "
619 "actual mbuf tailroom: %u\n",
620 mp
->name
, i
, seglen
[i
],
621 rte_pktmbuf_tailroom(md
[i
]));
626 /* all mbufs successfully allocated, do copy */
628 rc
= mbuf_copy_split(pkt
, md
, seglen
, nb_seg
);
631 "mbuf_copy_split for %p(len=%u, nb_seg=%u) "
632 "into %u segments failed with error code: %d\n",
633 pkt
, pkt
->pkt_len
, pkt
->nb_segs
, nb_seg
, rc
);
635 /* figure out how many mbufs to free. */
639 /* free unused mbufs */
640 for (; i
!= nb_seg
; i
++) {
641 rte_pktmbuf_free_seg(md
[i
]);
649 * Receive a burst of packets, and for each packet:
650 * - parse packet, and try to recognize a supported packet type (1)
651 * - if it's not a supported packet type, don't touch the packet, else:
652 * - reprocess the checksum of all supported layers. This is done in SW
653 * or HW, depending on testpmd command line configuration
654 * - if TSO is enabled in testpmd command line, also flag the mbuf for TCP
655 * segmentation offload (this implies HW TCP checksum)
656 * Then transmit packets on the output port.
658 * (1) Supported packets are:
659 * Ether / (vlan) / IP|IP6 / UDP|TCP|SCTP .
660 * Ether / (vlan) / outer IP|IP6 / outer UDP / VxLAN / Ether / IP|IP6 /
662 * Ether / (vlan) / outer IP|IP6 / outer UDP / VXLAN-GPE / Ether / IP|IP6 /
664 * Ether / (vlan) / outer IP|IP6 / outer UDP / VXLAN-GPE / IP|IP6 /
666 * Ether / (vlan) / outer IP|IP6 / GRE / Ether / IP|IP6 / UDP|TCP|SCTP
667 * Ether / (vlan) / outer IP|IP6 / GRE / IP|IP6 / UDP|TCP|SCTP
668 * Ether / (vlan) / outer IP|IP6 / IP|IP6 / UDP|TCP|SCTP
670 * The testpmd command line for this forward engine sets the flags
671 * TESTPMD_TX_OFFLOAD_* in ports[tx_port].tx_ol_flags. They control
672 * wether a checksum must be calculated in software or in hardware. The
673 * IP, UDP, TCP and SCTP flags always concern the inner layer. The
674 * OUTER_IP is only useful for tunnel packets.
677 pkt_burst_checksum_forward(struct fwd_stream
*fs
)
679 struct rte_mbuf
*pkts_burst
[MAX_PKT_BURST
];
680 struct rte_mbuf
*gso_segments
[GSO_MAX_PKT_BURST
];
681 struct rte_gso_ctx
*gso_ctx
;
682 struct rte_mbuf
**tx_pkts_burst
;
683 struct rte_port
*txp
;
684 struct rte_mbuf
*m
, *p
;
685 struct ether_hdr
*eth_hdr
;
686 void *l3_hdr
= NULL
, *outer_l3_hdr
= NULL
; /* can be IPv4 or IPv6 */
688 uint16_t gro_pkts_num
;
694 uint64_t rx_ol_flags
, tx_ol_flags
;
695 uint64_t tx_offloads
;
697 uint32_t rx_bad_ip_csum
;
698 uint32_t rx_bad_l4_csum
;
699 struct testpmd_offload_info info
;
700 uint16_t nb_segments
= 0;
703 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
706 uint64_t core_cycles
;
709 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
710 start_tsc
= rte_rdtsc();
713 /* receive a burst of packet */
714 nb_rx
= rte_eth_rx_burst(fs
->rx_port
, fs
->rx_queue
, pkts_burst
,
716 if (unlikely(nb_rx
== 0))
718 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
719 fs
->rx_burst_stats
.pkt_burst_spread
[nb_rx
]++;
721 fs
->rx_packets
+= nb_rx
;
724 gro_enable
= gro_ports
[fs
->rx_port
].enable
;
726 txp
= &ports
[fs
->tx_port
];
727 tx_offloads
= txp
->dev_conf
.txmode
.offloads
;
728 memset(&info
, 0, sizeof(info
));
729 info
.tso_segsz
= txp
->tso_segsz
;
730 info
.tunnel_tso_segsz
= txp
->tunnel_tso_segsz
;
731 if (gso_ports
[fs
->tx_port
].enable
)
734 for (i
= 0; i
< nb_rx
; i
++) {
735 if (likely(i
< nb_rx
- 1))
736 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst
[i
+ 1],
741 info
.pkt_len
= rte_pktmbuf_pkt_len(m
);
742 tx_ol_flags
= m
->ol_flags
&
743 (IND_ATTACHED_MBUF
| EXT_ATTACHED_MBUF
);
744 rx_ol_flags
= m
->ol_flags
;
746 /* Update the L3/L4 checksum error packet statistics */
747 if ((rx_ol_flags
& PKT_RX_IP_CKSUM_MASK
) == PKT_RX_IP_CKSUM_BAD
)
749 if ((rx_ol_flags
& PKT_RX_L4_CKSUM_MASK
) == PKT_RX_L4_CKSUM_BAD
)
752 /* step 1: dissect packet, parsing optional vlan, ip4/ip6, vxlan
753 * and inner headers */
755 eth_hdr
= rte_pktmbuf_mtod(m
, struct ether_hdr
*);
756 ether_addr_copy(&peer_eth_addrs
[fs
->peer_addr
],
758 ether_addr_copy(&ports
[fs
->tx_port
].eth_addr
,
760 parse_ethernet(eth_hdr
, &info
);
761 l3_hdr
= (char *)eth_hdr
+ info
.l2_len
;
763 /* check if it's a supported tunnel */
764 if (txp
->parse_tunnel
) {
765 if (info
.l4_proto
== IPPROTO_UDP
) {
766 struct udp_hdr
*udp_hdr
;
768 udp_hdr
= (struct udp_hdr
*)((char *)l3_hdr
+
770 parse_vxlan_gpe(udp_hdr
, &info
);
771 if (info
.is_tunnel
) {
772 tx_ol_flags
|= PKT_TX_TUNNEL_VXLAN_GPE
;
774 parse_vxlan(udp_hdr
, &info
,
780 } else if (info
.l4_proto
== IPPROTO_GRE
) {
781 struct simple_gre_hdr
*gre_hdr
;
783 gre_hdr
= (struct simple_gre_hdr
*)
784 ((char *)l3_hdr
+ info
.l3_len
);
785 parse_gre(gre_hdr
, &info
);
787 tx_ol_flags
|= PKT_TX_TUNNEL_GRE
;
788 } else if (info
.l4_proto
== IPPROTO_IPIP
) {
791 encap_ip_hdr
= (char *)l3_hdr
+ info
.l3_len
;
792 parse_encap_ip(encap_ip_hdr
, &info
);
794 tx_ol_flags
|= PKT_TX_TUNNEL_IPIP
;
798 /* update l3_hdr and outer_l3_hdr if a tunnel was parsed */
799 if (info
.is_tunnel
) {
800 outer_l3_hdr
= l3_hdr
;
801 l3_hdr
= (char *)l3_hdr
+ info
.outer_l3_len
+ info
.l2_len
;
804 /* step 2: depending on user command line configuration,
805 * recompute checksum either in software or flag the
806 * mbuf to offload the calculation to the NIC. If TSO
807 * is configured, prepare the mbuf for TCP segmentation. */
809 /* process checksums of inner headers first */
810 tx_ol_flags
|= process_inner_cksums(l3_hdr
, &info
,
813 /* Then process outer headers if any. Note that the software
814 * checksum will be wrong if one of the inner checksums is
815 * processed in hardware. */
816 if (info
.is_tunnel
== 1) {
817 tx_ol_flags
|= process_outer_cksums(outer_l3_hdr
, &info
,
819 !!(tx_ol_flags
& PKT_TX_TCP_SEG
));
822 /* step 3: fill the mbuf meta data (flags and header lengths) */
825 if (info
.is_tunnel
== 1) {
826 if (info
.tunnel_tso_segsz
||
828 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM
) ||
829 (tx_ol_flags
& PKT_TX_OUTER_IPV6
)) {
830 m
->outer_l2_len
= info
.outer_l2_len
;
831 m
->outer_l3_len
= info
.outer_l3_len
;
832 m
->l2_len
= info
.l2_len
;
833 m
->l3_len
= info
.l3_len
;
834 m
->l4_len
= info
.l4_len
;
835 m
->tso_segsz
= info
.tunnel_tso_segsz
;
838 /* if there is a outer UDP cksum
839 processed in sw and the inner in hw,
840 the outer checksum will be wrong as
841 the payload will be modified by the
843 m
->l2_len
= info
.outer_l2_len
+
844 info
.outer_l3_len
+ info
.l2_len
;
845 m
->l3_len
= info
.l3_len
;
846 m
->l4_len
= info
.l4_len
;
849 /* this is only useful if an offload flag is
850 * set, but it does not hurt to fill it in any
852 m
->l2_len
= info
.l2_len
;
853 m
->l3_len
= info
.l3_len
;
854 m
->l4_len
= info
.l4_len
;
855 m
->tso_segsz
= info
.tso_segsz
;
857 m
->ol_flags
= tx_ol_flags
;
859 /* Do split & copy for the packet. */
860 if (tx_pkt_split
!= TX_PKT_SPLIT_OFF
) {
861 p
= pkt_copy_split(m
);
869 /* if verbose mode is enabled, dump debug info */
870 if (verbose_level
> 0) {
873 printf("-----------------\n");
874 printf("port=%u, mbuf=%p, pkt_len=%u, nb_segs=%u:\n",
875 fs
->rx_port
, m
, m
->pkt_len
, m
->nb_segs
);
876 /* dump rx parsed packet info */
877 rte_get_rx_ol_flag_list(rx_ol_flags
, buf
, sizeof(buf
));
878 printf("rx: l2_len=%d ethertype=%x l3_len=%d "
879 "l4_proto=%d l4_len=%d flags=%s\n",
880 info
.l2_len
, rte_be_to_cpu_16(info
.ethertype
),
881 info
.l3_len
, info
.l4_proto
, info
.l4_len
, buf
);
882 if (rx_ol_flags
& PKT_RX_LRO
)
883 printf("rx: m->lro_segsz=%u\n", m
->tso_segsz
);
884 if (info
.is_tunnel
== 1)
885 printf("rx: outer_l2_len=%d outer_ethertype=%x "
886 "outer_l3_len=%d\n", info
.outer_l2_len
,
887 rte_be_to_cpu_16(info
.outer_ethertype
),
889 /* dump tx packet info */
890 if ((tx_offloads
& (DEV_TX_OFFLOAD_IPV4_CKSUM
|
891 DEV_TX_OFFLOAD_UDP_CKSUM
|
892 DEV_TX_OFFLOAD_TCP_CKSUM
|
893 DEV_TX_OFFLOAD_SCTP_CKSUM
)) ||
895 printf("tx: m->l2_len=%d m->l3_len=%d "
897 m
->l2_len
, m
->l3_len
, m
->l4_len
);
898 if (info
.is_tunnel
== 1) {
900 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM
) ||
901 (tx_ol_flags
& PKT_TX_OUTER_IPV6
))
902 printf("tx: m->outer_l2_len=%d "
903 "m->outer_l3_len=%d\n",
906 if (info
.tunnel_tso_segsz
!= 0 &&
907 (m
->ol_flags
& PKT_TX_TCP_SEG
))
908 printf("tx: m->tso_segsz=%d\n",
910 } else if (info
.tso_segsz
!= 0 &&
911 (m
->ol_flags
& PKT_TX_TCP_SEG
))
912 printf("tx: m->tso_segsz=%d\n", m
->tso_segsz
);
913 rte_get_tx_ol_flag_list(m
->ol_flags
, buf
, sizeof(buf
));
914 printf("tx: flags=%s", buf
);
919 if (unlikely(gro_enable
)) {
920 if (gro_flush_cycles
== GRO_DEFAULT_FLUSH_CYCLES
) {
921 nb_rx
= rte_gro_reassemble_burst(pkts_burst
, nb_rx
,
922 &(gro_ports
[fs
->rx_port
].param
));
924 gro_ctx
= current_fwd_lcore()->gro_ctx
;
925 nb_rx
= rte_gro_reassemble(pkts_burst
, nb_rx
, gro_ctx
);
927 if (++fs
->gro_times
>= gro_flush_cycles
) {
928 gro_pkts_num
= rte_gro_get_pkt_count(gro_ctx
);
929 if (gro_pkts_num
> MAX_PKT_BURST
- nb_rx
)
930 gro_pkts_num
= MAX_PKT_BURST
- nb_rx
;
932 nb_rx
+= rte_gro_timeout_flush(gro_ctx
, 0,
941 if (gso_ports
[fs
->tx_port
].enable
== 0)
942 tx_pkts_burst
= pkts_burst
;
944 gso_ctx
= &(current_fwd_lcore()->gso_ctx
);
945 gso_ctx
->gso_size
= gso_max_segment_size
;
946 for (i
= 0; i
< nb_rx
; i
++) {
947 ret
= rte_gso_segment(pkts_burst
[i
], gso_ctx
,
948 &gso_segments
[nb_segments
],
949 GSO_MAX_PKT_BURST
- nb_segments
);
953 TESTPMD_LOG(DEBUG
, "Unable to segment packet");
954 rte_pktmbuf_free(pkts_burst
[i
]);
958 tx_pkts_burst
= gso_segments
;
962 nb_prep
= rte_eth_tx_prepare(fs
->tx_port
, fs
->tx_queue
,
963 tx_pkts_burst
, nb_rx
);
964 if (nb_prep
!= nb_rx
)
965 printf("Preparing packet burst to transmit failed: %s\n",
966 rte_strerror(rte_errno
));
968 nb_tx
= rte_eth_tx_burst(fs
->tx_port
, fs
->tx_queue
, tx_pkts_burst
,
974 if (unlikely(nb_tx
< nb_rx
) && fs
->retry_enabled
) {
976 while (nb_tx
< nb_rx
&& retry
++ < burst_tx_retry_num
) {
977 rte_delay_us(burst_tx_delay_time
);
978 nb_tx
+= rte_eth_tx_burst(fs
->tx_port
, fs
->tx_queue
,
979 &tx_pkts_burst
[nb_tx
], nb_rx
- nb_tx
);
982 fs
->tx_packets
+= nb_tx
;
983 fs
->rx_bad_ip_csum
+= rx_bad_ip_csum
;
984 fs
->rx_bad_l4_csum
+= rx_bad_l4_csum
;
986 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
987 fs
->tx_burst_stats
.pkt_burst_spread
[nb_tx
]++;
989 if (unlikely(nb_tx
< nb_rx
)) {
990 fs
->fwd_dropped
+= (nb_rx
- nb_tx
);
992 rte_pktmbuf_free(tx_pkts_burst
[nb_tx
]);
993 } while (++nb_tx
< nb_rx
);
996 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
997 end_tsc
= rte_rdtsc();
998 core_cycles
= (end_tsc
- start_tsc
);
999 fs
->core_cycles
= (uint64_t) (fs
->core_cycles
+ core_cycles
);
1003 struct fwd_engine csum_fwd_engine
= {
1004 .fwd_mode_name
= "csum",
1005 .port_fwd_begin
= NULL
,
1006 .port_fwd_end
= NULL
,
1007 .packet_fwd
= pkt_burst_checksum_forward
,