1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2016 Intel Corporation
11 #include <sys/queue.h>
16 #include <netinet/in.h>
18 #include <rte_debug.h>
19 #include <rte_ether.h>
20 #include <rte_ethdev.h>
21 #include <rte_cycles.h>
30 #if defined(RTE_ARCH_X86) || defined(RTE_MACHINE_CPUFLAG_CRC32)
35 #include <rte_hash_crc.h>
36 #define DEFAULT_HASH_FUNC rte_hash_crc
38 #include <rte_jhash.h>
39 #define DEFAULT_HASH_FUNC rte_jhash
42 #define IPV6_ADDR_LEN 16
50 } __attribute__((__packed__
));
52 union ipv4_5tuple_host
{
65 #define XMM_NUM_IN_IPV6_5TUPLE 3
68 uint8_t ip_dst
[IPV6_ADDR_LEN
];
69 uint8_t ip_src
[IPV6_ADDR_LEN
];
73 } __attribute__((__packed__
));
75 union ipv6_5tuple_host
{
80 uint8_t ip_src
[IPV6_ADDR_LEN
];
81 uint8_t ip_dst
[IPV6_ADDR_LEN
];
86 xmm_t xmm
[XMM_NUM_IN_IPV6_5TUPLE
];
91 struct ipv4_l3fwd_em_route
{
92 struct ipv4_5tuple key
;
96 struct ipv6_l3fwd_em_route
{
97 struct ipv6_5tuple key
;
101 static struct ipv4_l3fwd_em_route ipv4_l3fwd_em_route_array
[] = {
102 {{IPv4(101, 0, 0, 0), IPv4(100, 10, 0, 1), 101, 11, IPPROTO_TCP
}, 0},
103 {{IPv4(201, 0, 0, 0), IPv4(200, 20, 0, 1), 102, 12, IPPROTO_TCP
}, 1},
104 {{IPv4(111, 0, 0, 0), IPv4(100, 30, 0, 1), 101, 11, IPPROTO_TCP
}, 2},
105 {{IPv4(211, 0, 0, 0), IPv4(200, 40, 0, 1), 102, 12, IPPROTO_TCP
}, 3},
108 static struct ipv6_l3fwd_em_route ipv6_l3fwd_em_route_array
[] = {
110 {0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
111 {0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
112 101, 11, IPPROTO_TCP
}, 0},
115 {0xfe, 0x90, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
116 {0xfe, 0x90, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
117 102, 12, IPPROTO_TCP
}, 1},
120 {0xfe, 0xa0, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
121 {0xfe, 0xa0, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
122 101, 11, IPPROTO_TCP
}, 2},
125 {0xfe, 0xb0, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
126 {0xfe, 0xb0, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
127 102, 12, IPPROTO_TCP
}, 3},
130 struct rte_hash
*ipv4_l3fwd_em_lookup_struct
[NB_SOCKETS
];
131 struct rte_hash
*ipv6_l3fwd_em_lookup_struct
[NB_SOCKETS
];
133 static inline uint32_t
134 ipv4_hash_crc(const void *data
, __rte_unused
uint32_t data_len
,
137 const union ipv4_5tuple_host
*k
;
143 p
= (const uint32_t *)&k
->port_src
;
146 init_val
= rte_hash_crc_4byte(t
, init_val
);
147 init_val
= rte_hash_crc_4byte(k
->ip_src
, init_val
);
148 init_val
= rte_hash_crc_4byte(k
->ip_dst
, init_val
);
149 init_val
= rte_hash_crc_4byte(*p
, init_val
);
151 init_val
= rte_jhash_1word(t
, init_val
);
152 init_val
= rte_jhash_1word(k
->ip_src
, init_val
);
153 init_val
= rte_jhash_1word(k
->ip_dst
, init_val
);
154 init_val
= rte_jhash_1word(*p
, init_val
);
160 static inline uint32_t
161 ipv6_hash_crc(const void *data
, __rte_unused
uint32_t data_len
,
164 const union ipv6_5tuple_host
*k
;
168 const uint32_t *ip_src0
, *ip_src1
, *ip_src2
, *ip_src3
;
169 const uint32_t *ip_dst0
, *ip_dst1
, *ip_dst2
, *ip_dst3
;
174 p
= (const uint32_t *)&k
->port_src
;
177 ip_src0
= (const uint32_t *) k
->ip_src
;
178 ip_src1
= (const uint32_t *)(k
->ip_src
+4);
179 ip_src2
= (const uint32_t *)(k
->ip_src
+8);
180 ip_src3
= (const uint32_t *)(k
->ip_src
+12);
181 ip_dst0
= (const uint32_t *) k
->ip_dst
;
182 ip_dst1
= (const uint32_t *)(k
->ip_dst
+4);
183 ip_dst2
= (const uint32_t *)(k
->ip_dst
+8);
184 ip_dst3
= (const uint32_t *)(k
->ip_dst
+12);
185 init_val
= rte_hash_crc_4byte(t
, init_val
);
186 init_val
= rte_hash_crc_4byte(*ip_src0
, init_val
);
187 init_val
= rte_hash_crc_4byte(*ip_src1
, init_val
);
188 init_val
= rte_hash_crc_4byte(*ip_src2
, init_val
);
189 init_val
= rte_hash_crc_4byte(*ip_src3
, init_val
);
190 init_val
= rte_hash_crc_4byte(*ip_dst0
, init_val
);
191 init_val
= rte_hash_crc_4byte(*ip_dst1
, init_val
);
192 init_val
= rte_hash_crc_4byte(*ip_dst2
, init_val
);
193 init_val
= rte_hash_crc_4byte(*ip_dst3
, init_val
);
194 init_val
= rte_hash_crc_4byte(*p
, init_val
);
196 init_val
= rte_jhash_1word(t
, init_val
);
197 init_val
= rte_jhash(k
->ip_src
,
198 sizeof(uint8_t) * IPV6_ADDR_LEN
, init_val
);
199 init_val
= rte_jhash(k
->ip_dst
,
200 sizeof(uint8_t) * IPV6_ADDR_LEN
, init_val
);
201 init_val
= rte_jhash_1word(*p
, init_val
);
206 #define IPV4_L3FWD_EM_NUM_ROUTES \
207 (sizeof(ipv4_l3fwd_em_route_array) / sizeof(ipv4_l3fwd_em_route_array[0]))
209 #define IPV6_L3FWD_EM_NUM_ROUTES \
210 (sizeof(ipv6_l3fwd_em_route_array) / sizeof(ipv6_l3fwd_em_route_array[0]))
212 static uint8_t ipv4_l3fwd_out_if
[L3FWD_HASH_ENTRIES
] __rte_cache_aligned
;
213 static uint8_t ipv6_l3fwd_out_if
[L3FWD_HASH_ENTRIES
] __rte_cache_aligned
;
215 static rte_xmm_t mask0
;
216 static rte_xmm_t mask1
;
217 static rte_xmm_t mask2
;
219 #if defined(RTE_MACHINE_CPUFLAG_SSE2)
221 em_mask_key(void *key
, xmm_t mask
)
223 __m128i data
= _mm_loadu_si128((__m128i
*)(key
));
225 return _mm_and_si128(data
, mask
);
227 #elif defined(RTE_MACHINE_CPUFLAG_NEON)
229 em_mask_key(void *key
, xmm_t mask
)
231 int32x4_t data
= vld1q_s32((int32_t *)key
);
233 return vandq_s32(data
, mask
);
235 #elif defined(RTE_MACHINE_CPUFLAG_ALTIVEC)
237 em_mask_key(void *key
, xmm_t mask
)
239 xmm_t data
= vec_ld(0, (xmm_t
*)(key
));
241 return vec_and(data
, mask
);
244 #error No vector engine (SSE, NEON, ALTIVEC) available, check your toolchain
247 static inline uint16_t
248 em_get_ipv4_dst_port(void *ipv4_hdr
, uint16_t portid
, void *lookup_struct
)
251 union ipv4_5tuple_host key
;
252 struct rte_hash
*ipv4_l3fwd_lookup_struct
=
253 (struct rte_hash
*)lookup_struct
;
255 ipv4_hdr
= (uint8_t *)ipv4_hdr
+ offsetof(struct ipv4_hdr
, time_to_live
);
258 * Get 5 tuple: dst port, src port, dst IP address,
259 * src IP address and protocol.
261 key
.xmm
= em_mask_key(ipv4_hdr
, mask0
.x
);
263 /* Find destination port */
264 ret
= rte_hash_lookup(ipv4_l3fwd_lookup_struct
, (const void *)&key
);
265 return (ret
< 0) ? portid
: ipv4_l3fwd_out_if
[ret
];
268 static inline uint16_t
269 em_get_ipv6_dst_port(void *ipv6_hdr
, uint16_t portid
, void *lookup_struct
)
272 union ipv6_5tuple_host key
;
273 struct rte_hash
*ipv6_l3fwd_lookup_struct
=
274 (struct rte_hash
*)lookup_struct
;
276 ipv6_hdr
= (uint8_t *)ipv6_hdr
+ offsetof(struct ipv6_hdr
, payload_len
);
277 void *data0
= ipv6_hdr
;
278 void *data1
= ((uint8_t *)ipv6_hdr
) + sizeof(xmm_t
);
279 void *data2
= ((uint8_t *)ipv6_hdr
) + sizeof(xmm_t
) + sizeof(xmm_t
);
281 /* Get part of 5 tuple: src IP address lower 96 bits and protocol */
282 key
.xmm
[0] = em_mask_key(data0
, mask1
.x
);
285 * Get part of 5 tuple: dst IP address lower 96 bits
286 * and src IP address higher 32 bits.
288 key
.xmm
[1] = *(xmm_t
*)data1
;
291 * Get part of 5 tuple: dst port and src port
292 * and dst IP address higher 32 bits.
294 key
.xmm
[2] = em_mask_key(data2
, mask2
.x
);
296 /* Find destination port */
297 ret
= rte_hash_lookup(ipv6_l3fwd_lookup_struct
, (const void *)&key
);
298 return (ret
< 0) ? portid
: ipv6_l3fwd_out_if
[ret
];
301 #if defined RTE_ARCH_X86 || defined RTE_MACHINE_CPUFLAG_NEON
302 #if defined(NO_HASH_MULTI_LOOKUP)
303 #include "l3fwd_em_sequential.h"
305 #include "l3fwd_em_hlm.h"
308 #include "l3fwd_em.h"
312 convert_ipv4_5tuple(struct ipv4_5tuple
*key1
,
313 union ipv4_5tuple_host
*key2
)
315 key2
->ip_dst
= rte_cpu_to_be_32(key1
->ip_dst
);
316 key2
->ip_src
= rte_cpu_to_be_32(key1
->ip_src
);
317 key2
->port_dst
= rte_cpu_to_be_16(key1
->port_dst
);
318 key2
->port_src
= rte_cpu_to_be_16(key1
->port_src
);
319 key2
->proto
= key1
->proto
;
325 convert_ipv6_5tuple(struct ipv6_5tuple
*key1
,
326 union ipv6_5tuple_host
*key2
)
330 for (i
= 0; i
< 16; i
++) {
331 key2
->ip_dst
[i
] = key1
->ip_dst
[i
];
332 key2
->ip_src
[i
] = key1
->ip_src
[i
];
334 key2
->port_dst
= rte_cpu_to_be_16(key1
->port_dst
);
335 key2
->port_src
= rte_cpu_to_be_16(key1
->port_src
);
336 key2
->proto
= key1
->proto
;
342 #define BYTE_VALUE_MAX 256
343 #define ALL_32_BITS 0xffffffff
344 #define BIT_8_TO_15 0x0000ff00
347 populate_ipv4_few_flow_into_table(const struct rte_hash
*h
)
352 mask0
= (rte_xmm_t
){.u32
= {BIT_8_TO_15
, ALL_32_BITS
,
353 ALL_32_BITS
, ALL_32_BITS
} };
355 for (i
= 0; i
< IPV4_L3FWD_EM_NUM_ROUTES
; i
++) {
356 struct ipv4_l3fwd_em_route entry
;
357 union ipv4_5tuple_host newkey
;
359 entry
= ipv4_l3fwd_em_route_array
[i
];
360 convert_ipv4_5tuple(&entry
.key
, &newkey
);
361 ret
= rte_hash_add_key(h
, (void *) &newkey
);
363 rte_exit(EXIT_FAILURE
, "Unable to add entry %" PRIu32
364 " to the l3fwd hash.\n", i
);
366 ipv4_l3fwd_out_if
[ret
] = entry
.if_out
;
368 printf("Hash: Adding 0x%" PRIx64
" keys\n",
369 (uint64_t)IPV4_L3FWD_EM_NUM_ROUTES
);
372 #define BIT_16_TO_23 0x00ff0000
374 populate_ipv6_few_flow_into_table(const struct rte_hash
*h
)
379 mask1
= (rte_xmm_t
){.u32
= {BIT_16_TO_23
, ALL_32_BITS
,
380 ALL_32_BITS
, ALL_32_BITS
} };
382 mask2
= (rte_xmm_t
){.u32
= {ALL_32_BITS
, ALL_32_BITS
, 0, 0} };
384 for (i
= 0; i
< IPV6_L3FWD_EM_NUM_ROUTES
; i
++) {
385 struct ipv6_l3fwd_em_route entry
;
386 union ipv6_5tuple_host newkey
;
388 entry
= ipv6_l3fwd_em_route_array
[i
];
389 convert_ipv6_5tuple(&entry
.key
, &newkey
);
390 ret
= rte_hash_add_key(h
, (void *) &newkey
);
392 rte_exit(EXIT_FAILURE
, "Unable to add entry %" PRIu32
393 " to the l3fwd hash.\n", i
);
395 ipv6_l3fwd_out_if
[ret
] = entry
.if_out
;
397 printf("Hash: Adding 0x%" PRIx64
"keys\n",
398 (uint64_t)IPV6_L3FWD_EM_NUM_ROUTES
);
401 #define NUMBER_PORT_USED 4
403 populate_ipv4_many_flow_into_table(const struct rte_hash
*h
,
404 unsigned int nr_flow
)
408 mask0
= (rte_xmm_t
){.u32
= {BIT_8_TO_15
, ALL_32_BITS
,
409 ALL_32_BITS
, ALL_32_BITS
} };
411 for (i
= 0; i
< nr_flow
; i
++) {
412 struct ipv4_l3fwd_em_route entry
;
413 union ipv4_5tuple_host newkey
;
415 uint8_t a
= (uint8_t)
416 ((i
/NUMBER_PORT_USED
)%BYTE_VALUE_MAX
);
417 uint8_t b
= (uint8_t)
418 (((i
/NUMBER_PORT_USED
)/BYTE_VALUE_MAX
)%BYTE_VALUE_MAX
);
419 uint8_t c
= (uint8_t)
420 ((i
/NUMBER_PORT_USED
)/(BYTE_VALUE_MAX
*BYTE_VALUE_MAX
));
422 /* Create the ipv4 exact match flow */
423 memset(&entry
, 0, sizeof(entry
));
424 switch (i
& (NUMBER_PORT_USED
- 1)) {
426 entry
= ipv4_l3fwd_em_route_array
[0];
427 entry
.key
.ip_dst
= IPv4(101, c
, b
, a
);
430 entry
= ipv4_l3fwd_em_route_array
[1];
431 entry
.key
.ip_dst
= IPv4(201, c
, b
, a
);
434 entry
= ipv4_l3fwd_em_route_array
[2];
435 entry
.key
.ip_dst
= IPv4(111, c
, b
, a
);
438 entry
= ipv4_l3fwd_em_route_array
[3];
439 entry
.key
.ip_dst
= IPv4(211, c
, b
, a
);
442 convert_ipv4_5tuple(&entry
.key
, &newkey
);
443 int32_t ret
= rte_hash_add_key(h
, (void *) &newkey
);
446 rte_exit(EXIT_FAILURE
, "Unable to add entry %u\n", i
);
448 ipv4_l3fwd_out_if
[ret
] = (uint8_t) entry
.if_out
;
451 printf("Hash: Adding 0x%x keys\n", nr_flow
);
455 populate_ipv6_many_flow_into_table(const struct rte_hash
*h
,
456 unsigned int nr_flow
)
460 mask1
= (rte_xmm_t
){.u32
= {BIT_16_TO_23
, ALL_32_BITS
,
461 ALL_32_BITS
, ALL_32_BITS
} };
462 mask2
= (rte_xmm_t
){.u32
= {ALL_32_BITS
, ALL_32_BITS
, 0, 0} };
464 for (i
= 0; i
< nr_flow
; i
++) {
465 struct ipv6_l3fwd_em_route entry
;
466 union ipv6_5tuple_host newkey
;
468 uint8_t a
= (uint8_t)
469 ((i
/NUMBER_PORT_USED
)%BYTE_VALUE_MAX
);
470 uint8_t b
= (uint8_t)
471 (((i
/NUMBER_PORT_USED
)/BYTE_VALUE_MAX
)%BYTE_VALUE_MAX
);
472 uint8_t c
= (uint8_t)
473 ((i
/NUMBER_PORT_USED
)/(BYTE_VALUE_MAX
*BYTE_VALUE_MAX
));
475 /* Create the ipv6 exact match flow */
476 memset(&entry
, 0, sizeof(entry
));
477 switch (i
& (NUMBER_PORT_USED
- 1)) {
479 entry
= ipv6_l3fwd_em_route_array
[0];
482 entry
= ipv6_l3fwd_em_route_array
[1];
485 entry
= ipv6_l3fwd_em_route_array
[2];
488 entry
= ipv6_l3fwd_em_route_array
[3];
491 entry
.key
.ip_dst
[13] = c
;
492 entry
.key
.ip_dst
[14] = b
;
493 entry
.key
.ip_dst
[15] = a
;
494 convert_ipv6_5tuple(&entry
.key
, &newkey
);
495 int32_t ret
= rte_hash_add_key(h
, (void *) &newkey
);
498 rte_exit(EXIT_FAILURE
, "Unable to add entry %u\n", i
);
500 ipv6_l3fwd_out_if
[ret
] = (uint8_t) entry
.if_out
;
503 printf("Hash: Adding 0x%x keys\n", nr_flow
);
507 * 1. IP packets without extension;
508 * 2. L4 payload should be either TCP or UDP.
511 em_check_ptype(int portid
)
514 int ptype_l3_ipv4_ext
= 0;
515 int ptype_l3_ipv6_ext
= 0;
516 int ptype_l4_tcp
= 0;
517 int ptype_l4_udp
= 0;
518 uint32_t ptype_mask
= RTE_PTYPE_L3_MASK
| RTE_PTYPE_L4_MASK
;
520 ret
= rte_eth_dev_get_supported_ptypes(portid
, ptype_mask
, NULL
, 0);
524 uint32_t ptypes
[ret
];
526 ret
= rte_eth_dev_get_supported_ptypes(portid
, ptype_mask
, ptypes
, ret
);
527 for (i
= 0; i
< ret
; ++i
) {
529 case RTE_PTYPE_L3_IPV4_EXT
:
530 ptype_l3_ipv4_ext
= 1;
532 case RTE_PTYPE_L3_IPV6_EXT
:
533 ptype_l3_ipv6_ext
= 1;
535 case RTE_PTYPE_L4_TCP
:
538 case RTE_PTYPE_L4_UDP
:
544 if (ptype_l3_ipv4_ext
== 0)
545 printf("port %d cannot parse RTE_PTYPE_L3_IPV4_EXT\n", portid
);
546 if (ptype_l3_ipv6_ext
== 0)
547 printf("port %d cannot parse RTE_PTYPE_L3_IPV6_EXT\n", portid
);
548 if (!ptype_l3_ipv4_ext
|| !ptype_l3_ipv6_ext
)
551 if (ptype_l4_tcp
== 0)
552 printf("port %d cannot parse RTE_PTYPE_L4_TCP\n", portid
);
553 if (ptype_l4_udp
== 0)
554 printf("port %d cannot parse RTE_PTYPE_L4_UDP\n", portid
);
555 if (ptype_l4_tcp
&& ptype_l4_udp
)
562 em_parse_ptype(struct rte_mbuf
*m
)
564 struct ether_hdr
*eth_hdr
;
565 uint32_t packet_type
= RTE_PTYPE_UNKNOWN
;
569 struct ipv4_hdr
*ipv4_hdr
;
570 struct ipv6_hdr
*ipv6_hdr
;
572 eth_hdr
= rte_pktmbuf_mtod(m
, struct ether_hdr
*);
573 ether_type
= eth_hdr
->ether_type
;
574 l3
= (uint8_t *)eth_hdr
+ sizeof(struct ether_hdr
);
575 if (ether_type
== rte_cpu_to_be_16(ETHER_TYPE_IPv4
)) {
576 ipv4_hdr
= (struct ipv4_hdr
*)l3
;
577 hdr_len
= (ipv4_hdr
->version_ihl
& IPV4_HDR_IHL_MASK
) *
579 if (hdr_len
== sizeof(struct ipv4_hdr
)) {
580 packet_type
|= RTE_PTYPE_L3_IPV4
;
581 if (ipv4_hdr
->next_proto_id
== IPPROTO_TCP
)
582 packet_type
|= RTE_PTYPE_L4_TCP
;
583 else if (ipv4_hdr
->next_proto_id
== IPPROTO_UDP
)
584 packet_type
|= RTE_PTYPE_L4_UDP
;
586 packet_type
|= RTE_PTYPE_L3_IPV4_EXT
;
587 } else if (ether_type
== rte_cpu_to_be_16(ETHER_TYPE_IPv6
)) {
588 ipv6_hdr
= (struct ipv6_hdr
*)l3
;
589 if (ipv6_hdr
->proto
== IPPROTO_TCP
)
590 packet_type
|= RTE_PTYPE_L3_IPV6
| RTE_PTYPE_L4_TCP
;
591 else if (ipv6_hdr
->proto
== IPPROTO_UDP
)
592 packet_type
|= RTE_PTYPE_L3_IPV6
| RTE_PTYPE_L4_UDP
;
594 packet_type
|= RTE_PTYPE_L3_IPV6_EXT_UNKNOWN
;
597 m
->packet_type
= packet_type
;
601 em_cb_parse_ptype(uint16_t port __rte_unused
, uint16_t queue __rte_unused
,
602 struct rte_mbuf
*pkts
[], uint16_t nb_pkts
,
603 uint16_t max_pkts __rte_unused
,
604 void *user_param __rte_unused
)
608 for (i
= 0; i
< nb_pkts
; ++i
)
609 em_parse_ptype(pkts
[i
]);
614 /* main processing loop */
616 em_main_loop(__attribute__((unused
)) void *dummy
)
618 struct rte_mbuf
*pkts_burst
[MAX_PKT_BURST
];
620 uint64_t prev_tsc
, diff_tsc
, cur_tsc
;
624 struct lcore_conf
*qconf
;
625 const uint64_t drain_tsc
= (rte_get_tsc_hz() + US_PER_S
- 1) /
626 US_PER_S
* BURST_TX_DRAIN_US
;
630 lcore_id
= rte_lcore_id();
631 qconf
= &lcore_conf
[lcore_id
];
633 if (qconf
->n_rx_queue
== 0) {
634 RTE_LOG(INFO
, L3FWD
, "lcore %u has nothing to do\n", lcore_id
);
638 RTE_LOG(INFO
, L3FWD
, "entering main loop on lcore %u\n", lcore_id
);
640 for (i
= 0; i
< qconf
->n_rx_queue
; i
++) {
642 portid
= qconf
->rx_queue_list
[i
].port_id
;
643 queueid
= qconf
->rx_queue_list
[i
].queue_id
;
645 " -- lcoreid=%u portid=%u rxqueueid=%hhu\n",
646 lcore_id
, portid
, queueid
);
649 while (!force_quit
) {
651 cur_tsc
= rte_rdtsc();
654 * TX burst queue drain
656 diff_tsc
= cur_tsc
- prev_tsc
;
657 if (unlikely(diff_tsc
> drain_tsc
)) {
659 for (i
= 0; i
< qconf
->n_tx_port
; ++i
) {
660 portid
= qconf
->tx_port_id
[i
];
661 if (qconf
->tx_mbufs
[portid
].len
== 0)
664 qconf
->tx_mbufs
[portid
].len
,
666 qconf
->tx_mbufs
[portid
].len
= 0;
673 * Read packet from RX queues
675 for (i
= 0; i
< qconf
->n_rx_queue
; ++i
) {
676 portid
= qconf
->rx_queue_list
[i
].port_id
;
677 queueid
= qconf
->rx_queue_list
[i
].queue_id
;
678 nb_rx
= rte_eth_rx_burst(portid
, queueid
, pkts_burst
,
683 #if defined RTE_ARCH_X86 || defined RTE_MACHINE_CPUFLAG_NEON
684 l3fwd_em_send_packets(nb_rx
, pkts_burst
,
687 l3fwd_em_no_opt_send_packets(nb_rx
, pkts_burst
,
697 * Initialize exact match (hash) parameters.
700 setup_hash(const int socketid
)
702 struct rte_hash_parameters ipv4_l3fwd_hash_params
= {
704 .entries
= L3FWD_HASH_ENTRIES
,
705 .key_len
= sizeof(union ipv4_5tuple_host
),
706 .hash_func
= ipv4_hash_crc
,
707 .hash_func_init_val
= 0,
710 struct rte_hash_parameters ipv6_l3fwd_hash_params
= {
712 .entries
= L3FWD_HASH_ENTRIES
,
713 .key_len
= sizeof(union ipv6_5tuple_host
),
714 .hash_func
= ipv6_hash_crc
,
715 .hash_func_init_val
= 0,
720 /* create ipv4 hash */
721 snprintf(s
, sizeof(s
), "ipv4_l3fwd_hash_%d", socketid
);
722 ipv4_l3fwd_hash_params
.name
= s
;
723 ipv4_l3fwd_hash_params
.socket_id
= socketid
;
724 ipv4_l3fwd_em_lookup_struct
[socketid
] =
725 rte_hash_create(&ipv4_l3fwd_hash_params
);
726 if (ipv4_l3fwd_em_lookup_struct
[socketid
] == NULL
)
727 rte_exit(EXIT_FAILURE
,
728 "Unable to create the l3fwd hash on socket %d\n",
731 /* create ipv6 hash */
732 snprintf(s
, sizeof(s
), "ipv6_l3fwd_hash_%d", socketid
);
733 ipv6_l3fwd_hash_params
.name
= s
;
734 ipv6_l3fwd_hash_params
.socket_id
= socketid
;
735 ipv6_l3fwd_em_lookup_struct
[socketid
] =
736 rte_hash_create(&ipv6_l3fwd_hash_params
);
737 if (ipv6_l3fwd_em_lookup_struct
[socketid
] == NULL
)
738 rte_exit(EXIT_FAILURE
,
739 "Unable to create the l3fwd hash on socket %d\n",
742 if (hash_entry_number
!= HASH_ENTRY_NUMBER_DEFAULT
) {
743 /* For testing hash matching with a large number of flows we
744 * generate millions of IP 5-tuples with an incremented dst
745 * address to initialize the hash table. */
747 /* populate the ipv4 hash */
748 populate_ipv4_many_flow_into_table(
749 ipv4_l3fwd_em_lookup_struct
[socketid
],
752 /* populate the ipv6 hash */
753 populate_ipv6_many_flow_into_table(
754 ipv6_l3fwd_em_lookup_struct
[socketid
],
759 * Use data in ipv4/ipv6 l3fwd lookup table
760 * directly to initialize the hash table.
763 /* populate the ipv4 hash */
764 populate_ipv4_few_flow_into_table(
765 ipv4_l3fwd_em_lookup_struct
[socketid
]);
767 /* populate the ipv6 hash */
768 populate_ipv6_few_flow_into_table(
769 ipv6_l3fwd_em_lookup_struct
[socketid
]);
774 /* Return ipv4/ipv6 em fwd lookup struct. */
776 em_get_ipv4_l3fwd_lookup_struct(const int socketid
)
778 return ipv4_l3fwd_em_lookup_struct
[socketid
];
782 em_get_ipv6_l3fwd_lookup_struct(const int socketid
)
784 return ipv6_l3fwd_em_lookup_struct
[socketid
];