1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
13 #include <sys/queue.h>
16 #include <rte_common.h>
17 #include <rte_byteorder.h>
19 #include <rte_debug.h>
20 #include <rte_cycles.h>
21 #include <rte_memory.h>
22 #include <rte_memcpy.h>
23 #include <rte_launch.h>
25 #include <rte_per_lcore.h>
26 #include <rte_lcore.h>
27 #include <rte_atomic.h>
28 #include <rte_branch_prediction.h>
29 #include <rte_mempool.h>
31 #include <rte_interrupts.h>
33 #include <rte_ether.h>
34 #include <rte_ethdev.h>
38 #include <rte_string_fns.h>
43 #define UDP_SRC_PORT 1024
44 #define UDP_DST_PORT 1024
46 #define IP_SRC_ADDR ((192U << 24) | (168 << 16) | (0 << 8) | 1)
47 #define IP_DST_ADDR ((192U << 24) | (168 << 16) | (0 << 8) | 2)
49 #define IP_DEFTTL 64 /* from RFC 1340. */
50 #define IP_VERSION 0x40
51 #define IP_HDRLEN 0x05 /* default IP header length == five 32-bits words. */
52 #define IP_VHL_DEF (IP_VERSION | IP_HDRLEN)
54 static struct ipv4_hdr pkt_ip_hdr
; /**< IP header of transmitted packets. */
55 static struct udp_hdr pkt_udp_hdr
; /**< UDP header of transmitted packets. */
58 copy_buf_to_pkt_segs(void* buf
, unsigned len
, struct rte_mbuf
*pkt
,
66 while (offset
>= seg
->data_len
) {
67 offset
-= seg
->data_len
;
70 copy_len
= seg
->data_len
- offset
;
71 seg_buf
= rte_pktmbuf_mtod_offset(seg
, char *, offset
);
72 while (len
> copy_len
) {
73 rte_memcpy(seg_buf
, buf
, (size_t) copy_len
);
75 buf
= ((char*) buf
+ copy_len
);
77 seg_buf
= rte_pktmbuf_mtod(seg
, char *);
78 copy_len
= seg
->data_len
;
80 rte_memcpy(seg_buf
, buf
, (size_t) len
);
84 copy_buf_to_pkt(void* buf
, unsigned len
, struct rte_mbuf
*pkt
, unsigned offset
)
86 if (offset
+ len
<= pkt
->data_len
) {
87 rte_memcpy(rte_pktmbuf_mtod_offset(pkt
, char *, offset
),
91 copy_buf_to_pkt_segs(buf
, len
, pkt
, offset
);
95 setup_pkt_udp_ip_headers(struct ipv4_hdr
*ip_hdr
,
96 struct udp_hdr
*udp_hdr
,
97 uint16_t pkt_data_len
)
104 * Initialize UDP header.
106 pkt_len
= (uint16_t) (pkt_data_len
+ sizeof(struct udp_hdr
));
107 udp_hdr
->src_port
= rte_cpu_to_be_16(UDP_SRC_PORT
);
108 udp_hdr
->dst_port
= rte_cpu_to_be_16(UDP_DST_PORT
);
109 udp_hdr
->dgram_len
= RTE_CPU_TO_BE_16(pkt_len
);
110 udp_hdr
->dgram_cksum
= 0; /* No UDP checksum. */
113 * Initialize IP header.
115 pkt_len
= (uint16_t) (pkt_len
+ sizeof(struct ipv4_hdr
));
116 ip_hdr
->version_ihl
= IP_VHL_DEF
;
117 ip_hdr
->type_of_service
= 0;
118 ip_hdr
->fragment_offset
= 0;
119 ip_hdr
->time_to_live
= IP_DEFTTL
;
120 ip_hdr
->next_proto_id
= IPPROTO_UDP
;
121 ip_hdr
->packet_id
= 0;
122 ip_hdr
->total_length
= RTE_CPU_TO_BE_16(pkt_len
);
123 ip_hdr
->src_addr
= rte_cpu_to_be_32(IP_SRC_ADDR
);
124 ip_hdr
->dst_addr
= rte_cpu_to_be_32(IP_DST_ADDR
);
127 * Compute IP header checksum.
129 ptr16
= (unaligned_uint16_t
*) ip_hdr
;
131 ip_cksum
+= ptr16
[0]; ip_cksum
+= ptr16
[1];
132 ip_cksum
+= ptr16
[2]; ip_cksum
+= ptr16
[3];
133 ip_cksum
+= ptr16
[4];
134 ip_cksum
+= ptr16
[6]; ip_cksum
+= ptr16
[7];
135 ip_cksum
+= ptr16
[8]; ip_cksum
+= ptr16
[9];
138 * Reduce 32 bit checksum to 16 bits and complement it.
140 ip_cksum
= ((ip_cksum
& 0xFFFF0000) >> 16) +
141 (ip_cksum
& 0x0000FFFF);
142 if (ip_cksum
> 65535)
144 ip_cksum
= (~ip_cksum
) & 0x0000FFFF;
147 ip_hdr
->hdr_checksum
= (uint16_t) ip_cksum
;
151 * Transmit a burst of multi-segments packets.
154 pkt_burst_transmit(struct fwd_stream
*fs
)
156 struct rte_mbuf
*pkts_burst
[MAX_PKT_BURST
];
157 struct rte_port
*txp
;
158 struct rte_mbuf
*pkt
;
159 struct rte_mbuf
*pkt_seg
;
160 struct rte_mempool
*mbp
;
161 struct ether_hdr eth_hdr
;
164 uint16_t vlan_tci
, vlan_tci_outer
;
166 uint64_t ol_flags
= 0;
168 uint64_t tx_offloads
;
169 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
172 uint64_t core_cycles
;
174 uint32_t nb_segs
, pkt_len
;
176 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
177 start_tsc
= rte_rdtsc();
180 mbp
= current_fwd_lcore()->mbp
;
181 txp
= &ports
[fs
->tx_port
];
182 tx_offloads
= txp
->dev_conf
.txmode
.offloads
;
183 vlan_tci
= txp
->tx_vlan_id
;
184 vlan_tci_outer
= txp
->tx_vlan_id_outer
;
185 if (tx_offloads
& DEV_TX_OFFLOAD_VLAN_INSERT
)
186 ol_flags
= PKT_TX_VLAN_PKT
;
187 if (tx_offloads
& DEV_TX_OFFLOAD_QINQ_INSERT
)
188 ol_flags
|= PKT_TX_QINQ_PKT
;
189 if (tx_offloads
& DEV_TX_OFFLOAD_MACSEC_INSERT
)
190 ol_flags
|= PKT_TX_MACSEC
;
191 for (nb_pkt
= 0; nb_pkt
< nb_pkt_per_burst
; nb_pkt
++) {
192 pkt
= rte_mbuf_raw_alloc(mbp
);
201 * Using raw alloc is good to improve performance,
202 * but some consumers may use the headroom and so
203 * decrement data_off. We need to make sure it is
204 * reset to default value.
206 rte_pktmbuf_reset_headroom(pkt
);
207 pkt
->data_len
= tx_pkt_seg_lengths
[0];
209 if (tx_pkt_split
== TX_PKT_SPLIT_RND
)
210 nb_segs
= random() % tx_pkt_nb_segs
+ 1;
212 nb_segs
= tx_pkt_nb_segs
;
213 pkt_len
= pkt
->data_len
;
214 for (i
= 1; i
< nb_segs
; i
++) {
215 pkt_seg
->next
= rte_mbuf_raw_alloc(mbp
);
216 if (pkt_seg
->next
== NULL
) {
218 rte_pktmbuf_free(pkt
);
221 pkt_seg
= pkt_seg
->next
;
222 pkt_seg
->data_len
= tx_pkt_seg_lengths
[i
];
223 pkt_len
+= pkt_seg
->data_len
;
225 pkt_seg
->next
= NULL
; /* Last segment of packet. */
228 * Initialize Ethernet header.
230 ether_addr_copy(&peer_eth_addrs
[fs
->peer_addr
],ð_hdr
.d_addr
);
231 ether_addr_copy(&ports
[fs
->tx_port
].eth_addr
, ð_hdr
.s_addr
);
232 eth_hdr
.ether_type
= rte_cpu_to_be_16(ETHER_TYPE_IPv4
);
235 * Copy headers in first packet segment(s).
237 copy_buf_to_pkt(ð_hdr
, sizeof(eth_hdr
), pkt
, 0);
238 copy_buf_to_pkt(&pkt_ip_hdr
, sizeof(pkt_ip_hdr
), pkt
,
239 sizeof(struct ether_hdr
));
240 copy_buf_to_pkt(&pkt_udp_hdr
, sizeof(pkt_udp_hdr
), pkt
,
241 sizeof(struct ether_hdr
) +
242 sizeof(struct ipv4_hdr
));
245 * Complete first mbuf of packet and append it to the
246 * burst of packets to be transmitted.
248 pkt
->nb_segs
= nb_segs
;
249 pkt
->pkt_len
= pkt_len
;
250 pkt
->ol_flags
= ol_flags
;
251 pkt
->vlan_tci
= vlan_tci
;
252 pkt
->vlan_tci_outer
= vlan_tci_outer
;
253 pkt
->l2_len
= sizeof(struct ether_hdr
);
254 pkt
->l3_len
= sizeof(struct ipv4_hdr
);
255 pkts_burst
[nb_pkt
] = pkt
;
257 nb_tx
= rte_eth_tx_burst(fs
->tx_port
, fs
->tx_queue
, pkts_burst
, nb_pkt
);
261 if (unlikely(nb_tx
< nb_pkt
) && fs
->retry_enabled
) {
263 while (nb_tx
< nb_pkt
&& retry
++ < burst_tx_retry_num
) {
264 rte_delay_us(burst_tx_delay_time
);
265 nb_tx
+= rte_eth_tx_burst(fs
->tx_port
, fs
->tx_queue
,
266 &pkts_burst
[nb_tx
], nb_pkt
- nb_tx
);
269 fs
->tx_packets
+= nb_tx
;
271 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
272 fs
->tx_burst_stats
.pkt_burst_spread
[nb_tx
]++;
274 if (unlikely(nb_tx
< nb_pkt
)) {
275 if (verbose_level
> 0 && fs
->fwd_dropped
== 0)
276 printf("port %d tx_queue %d - drop "
277 "(nb_pkt:%u - nb_tx:%u)=%u packets\n",
278 fs
->tx_port
, fs
->tx_queue
,
279 (unsigned) nb_pkt
, (unsigned) nb_tx
,
280 (unsigned) (nb_pkt
- nb_tx
));
281 fs
->fwd_dropped
+= (nb_pkt
- nb_tx
);
283 rte_pktmbuf_free(pkts_burst
[nb_tx
]);
284 } while (++nb_tx
< nb_pkt
);
287 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
288 end_tsc
= rte_rdtsc();
289 core_cycles
= (end_tsc
- start_tsc
);
290 fs
->core_cycles
= (uint64_t) (fs
->core_cycles
+ core_cycles
);
295 tx_only_begin(__attribute__((unused
)) portid_t pi
)
297 uint16_t pkt_data_len
;
299 pkt_data_len
= (uint16_t) (tx_pkt_length
- (sizeof(struct ether_hdr
) +
300 sizeof(struct ipv4_hdr
) +
301 sizeof(struct udp_hdr
)));
302 setup_pkt_udp_ip_headers(&pkt_ip_hdr
, &pkt_udp_hdr
, pkt_data_len
);
305 struct fwd_engine tx_only_engine
= {
306 .fwd_mode_name
= "txonly",
307 .port_fwd_begin
= tx_only_begin
,
308 .port_fwd_end
= NULL
,
309 .packet_fwd
= pkt_burst_transmit
,