4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42 #include <sys/queue.h>
45 #include <rte_common.h>
46 #include <rte_byteorder.h>
48 #include <rte_debug.h>
49 #include <rte_cycles.h>
50 #include <rte_memory.h>
51 #include <rte_memcpy.h>
52 #include <rte_memzone.h>
53 #include <rte_launch.h>
55 #include <rte_per_lcore.h>
56 #include <rte_lcore.h>
57 #include <rte_atomic.h>
58 #include <rte_branch_prediction.h>
59 #include <rte_memory.h>
60 #include <rte_mempool.h>
62 #include <rte_memcpy.h>
63 #include <rte_interrupts.h>
65 #include <rte_ether.h>
66 #include <rte_ethdev.h>
70 #include <rte_string_fns.h>
74 #define UDP_SRC_PORT 1024
75 #define UDP_DST_PORT 1024
77 #define IP_SRC_ADDR ((192U << 24) | (168 << 16) | (0 << 8) | 1)
78 #define IP_DST_ADDR ((192U << 24) | (168 << 16) | (0 << 8) | 2)
80 #define IP_DEFTTL 64 /* from RFC 1340. */
81 #define IP_VERSION 0x40
82 #define IP_HDRLEN 0x05 /* default IP header length == five 32-bits words. */
83 #define IP_VHL_DEF (IP_VERSION | IP_HDRLEN)
85 static struct ipv4_hdr pkt_ip_hdr
; /**< IP header of transmitted packets. */
86 static struct udp_hdr pkt_udp_hdr
; /**< UDP header of transmitted packets. */
89 copy_buf_to_pkt_segs(void* buf
, unsigned len
, struct rte_mbuf
*pkt
,
97 while (offset
>= seg
->data_len
) {
98 offset
-= seg
->data_len
;
101 copy_len
= seg
->data_len
- offset
;
102 seg_buf
= rte_pktmbuf_mtod_offset(seg
, char *, offset
);
103 while (len
> copy_len
) {
104 rte_memcpy(seg_buf
, buf
, (size_t) copy_len
);
106 buf
= ((char*) buf
+ copy_len
);
108 seg_buf
= rte_pktmbuf_mtod(seg
, char *);
110 rte_memcpy(seg_buf
, buf
, (size_t) len
);
114 copy_buf_to_pkt(void* buf
, unsigned len
, struct rte_mbuf
*pkt
, unsigned offset
)
116 if (offset
+ len
<= pkt
->data_len
) {
117 rte_memcpy(rte_pktmbuf_mtod_offset(pkt
, char *, offset
),
121 copy_buf_to_pkt_segs(buf
, len
, pkt
, offset
);
125 setup_pkt_udp_ip_headers(struct ipv4_hdr
*ip_hdr
,
126 struct udp_hdr
*udp_hdr
,
127 uint16_t pkt_data_len
)
134 * Initialize UDP header.
136 pkt_len
= (uint16_t) (pkt_data_len
+ sizeof(struct udp_hdr
));
137 udp_hdr
->src_port
= rte_cpu_to_be_16(UDP_SRC_PORT
);
138 udp_hdr
->dst_port
= rte_cpu_to_be_16(UDP_DST_PORT
);
139 udp_hdr
->dgram_len
= RTE_CPU_TO_BE_16(pkt_len
);
140 udp_hdr
->dgram_cksum
= 0; /* No UDP checksum. */
143 * Initialize IP header.
145 pkt_len
= (uint16_t) (pkt_len
+ sizeof(struct ipv4_hdr
));
146 ip_hdr
->version_ihl
= IP_VHL_DEF
;
147 ip_hdr
->type_of_service
= 0;
148 ip_hdr
->fragment_offset
= 0;
149 ip_hdr
->time_to_live
= IP_DEFTTL
;
150 ip_hdr
->next_proto_id
= IPPROTO_UDP
;
151 ip_hdr
->packet_id
= 0;
152 ip_hdr
->total_length
= RTE_CPU_TO_BE_16(pkt_len
);
153 ip_hdr
->src_addr
= rte_cpu_to_be_32(IP_SRC_ADDR
);
154 ip_hdr
->dst_addr
= rte_cpu_to_be_32(IP_DST_ADDR
);
157 * Compute IP header checksum.
159 ptr16
= (unaligned_uint16_t
*) ip_hdr
;
161 ip_cksum
+= ptr16
[0]; ip_cksum
+= ptr16
[1];
162 ip_cksum
+= ptr16
[2]; ip_cksum
+= ptr16
[3];
163 ip_cksum
+= ptr16
[4];
164 ip_cksum
+= ptr16
[6]; ip_cksum
+= ptr16
[7];
165 ip_cksum
+= ptr16
[8]; ip_cksum
+= ptr16
[9];
168 * Reduce 32 bit checksum to 16 bits and complement it.
170 ip_cksum
= ((ip_cksum
& 0xFFFF0000) >> 16) +
171 (ip_cksum
& 0x0000FFFF);
172 if (ip_cksum
> 65535)
174 ip_cksum
= (~ip_cksum
) & 0x0000FFFF;
177 ip_hdr
->hdr_checksum
= (uint16_t) ip_cksum
;
181 * Transmit a burst of multi-segments packets.
184 pkt_burst_transmit(struct fwd_stream
*fs
)
186 struct rte_mbuf
*pkts_burst
[MAX_PKT_BURST
];
187 struct rte_port
*txp
;
188 struct rte_mbuf
*pkt
;
189 struct rte_mbuf
*pkt_seg
;
190 struct rte_mempool
*mbp
;
191 struct ether_hdr eth_hdr
;
194 uint16_t vlan_tci
, vlan_tci_outer
;
196 uint64_t ol_flags
= 0;
198 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
201 uint64_t core_cycles
;
203 uint32_t nb_segs
, pkt_len
;
205 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
206 start_tsc
= rte_rdtsc();
209 mbp
= current_fwd_lcore()->mbp
;
210 txp
= &ports
[fs
->tx_port
];
211 vlan_tci
= txp
->tx_vlan_id
;
212 vlan_tci_outer
= txp
->tx_vlan_id_outer
;
213 if (txp
->tx_ol_flags
& TESTPMD_TX_OFFLOAD_INSERT_VLAN
)
214 ol_flags
= PKT_TX_VLAN_PKT
;
215 if (txp
->tx_ol_flags
& TESTPMD_TX_OFFLOAD_INSERT_QINQ
)
216 ol_flags
|= PKT_TX_QINQ_PKT
;
217 for (nb_pkt
= 0; nb_pkt
< nb_pkt_per_burst
; nb_pkt
++) {
218 pkt
= rte_mbuf_raw_alloc(mbp
);
227 * Using raw alloc is good to improve performance,
228 * but some consumers may use the headroom and so
229 * decrement data_off. We need to make sure it is
230 * reset to default value.
232 rte_pktmbuf_reset_headroom(pkt
);
233 pkt
->data_len
= tx_pkt_seg_lengths
[0];
235 if (tx_pkt_split
== TX_PKT_SPLIT_RND
)
236 nb_segs
= random() % tx_pkt_nb_segs
+ 1;
238 nb_segs
= tx_pkt_nb_segs
;
239 pkt_len
= pkt
->data_len
;
240 for (i
= 1; i
< nb_segs
; i
++) {
241 pkt_seg
->next
= rte_mbuf_raw_alloc(mbp
);
242 if (pkt_seg
->next
== NULL
) {
244 rte_pktmbuf_free(pkt
);
247 pkt_seg
= pkt_seg
->next
;
248 pkt_seg
->data_len
= tx_pkt_seg_lengths
[i
];
249 pkt_len
+= pkt_seg
->data_len
;
251 pkt_seg
->next
= NULL
; /* Last segment of packet. */
254 * Initialize Ethernet header.
256 ether_addr_copy(&peer_eth_addrs
[fs
->peer_addr
],ð_hdr
.d_addr
);
257 ether_addr_copy(&ports
[fs
->tx_port
].eth_addr
, ð_hdr
.s_addr
);
258 eth_hdr
.ether_type
= rte_cpu_to_be_16(ETHER_TYPE_IPv4
);
261 * Copy headers in first packet segment(s).
263 copy_buf_to_pkt(ð_hdr
, sizeof(eth_hdr
), pkt
, 0);
264 copy_buf_to_pkt(&pkt_ip_hdr
, sizeof(pkt_ip_hdr
), pkt
,
265 sizeof(struct ether_hdr
));
266 copy_buf_to_pkt(&pkt_udp_hdr
, sizeof(pkt_udp_hdr
), pkt
,
267 sizeof(struct ether_hdr
) +
268 sizeof(struct ipv4_hdr
));
271 * Complete first mbuf of packet and append it to the
272 * burst of packets to be transmitted.
274 pkt
->nb_segs
= nb_segs
;
275 pkt
->pkt_len
= pkt_len
;
276 pkt
->ol_flags
= ol_flags
;
277 pkt
->vlan_tci
= vlan_tci
;
278 pkt
->vlan_tci_outer
= vlan_tci_outer
;
279 pkt
->l2_len
= sizeof(struct ether_hdr
);
280 pkt
->l3_len
= sizeof(struct ipv4_hdr
);
281 pkts_burst
[nb_pkt
] = pkt
;
283 nb_tx
= rte_eth_tx_burst(fs
->tx_port
, fs
->tx_queue
, pkts_burst
, nb_pkt
);
287 if (unlikely(nb_tx
< nb_pkt
) && fs
->retry_enabled
) {
289 while (nb_tx
< nb_pkt
&& retry
++ < burst_tx_retry_num
) {
290 rte_delay_us(burst_tx_delay_time
);
291 nb_tx
+= rte_eth_tx_burst(fs
->tx_port
, fs
->tx_queue
,
292 &pkts_burst
[nb_tx
], nb_pkt
- nb_tx
);
295 fs
->tx_packets
+= nb_tx
;
297 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
298 fs
->tx_burst_stats
.pkt_burst_spread
[nb_tx
]++;
300 if (unlikely(nb_tx
< nb_pkt
)) {
301 if (verbose_level
> 0 && fs
->fwd_dropped
== 0)
302 printf("port %d tx_queue %d - drop "
303 "(nb_pkt:%u - nb_tx:%u)=%u packets\n",
304 fs
->tx_port
, fs
->tx_queue
,
305 (unsigned) nb_pkt
, (unsigned) nb_tx
,
306 (unsigned) (nb_pkt
- nb_tx
));
307 fs
->fwd_dropped
+= (nb_pkt
- nb_tx
);
309 rte_pktmbuf_free(pkts_burst
[nb_tx
]);
310 } while (++nb_tx
< nb_pkt
);
313 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
314 end_tsc
= rte_rdtsc();
315 core_cycles
= (end_tsc
- start_tsc
);
316 fs
->core_cycles
= (uint64_t) (fs
->core_cycles
+ core_cycles
);
321 tx_only_begin(__attribute__((unused
)) portid_t pi
)
323 uint16_t pkt_data_len
;
325 pkt_data_len
= (uint16_t) (tx_pkt_length
- (sizeof(struct ether_hdr
) +
326 sizeof(struct ipv4_hdr
) +
327 sizeof(struct udp_hdr
)));
328 setup_pkt_udp_ip_headers(&pkt_ip_hdr
, &pkt_udp_hdr
, pkt_data_len
);
331 struct fwd_engine tx_only_engine
= {
332 .fwd_mode_name
= "txonly",
333 .port_fwd_begin
= tx_only_begin
,
334 .port_fwd_end
= NULL
,
335 .packet_fwd
= pkt_burst_transmit
,