]>
git.proxmox.com Git - ceph.git/blob - ceph/src/seastar/dpdk/examples/l3fwd/l3fwd_em_hlm_sse.h
4 * Copyright(c) 2016 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #ifndef __L3FWD_EM_HLM_SSE_H__
35 #define __L3FWD_EM_HLM_SSE_H__
37 #include "l3fwd_sse.h"
39 static inline __attribute__((always_inline
)) void
40 em_get_dst_port_ipv4x8(struct lcore_conf
*qconf
, struct rte_mbuf
*m
[8],
41 uint8_t portid
, uint16_t dst_port
[8])
44 union ipv4_5tuple_host key
[8];
47 data
[0] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m
[0], __m128i
*,
48 sizeof(struct ether_hdr
) +
49 offsetof(struct ipv4_hdr
, time_to_live
)));
50 data
[1] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m
[1], __m128i
*,
51 sizeof(struct ether_hdr
) +
52 offsetof(struct ipv4_hdr
, time_to_live
)));
53 data
[2] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m
[2], __m128i
*,
54 sizeof(struct ether_hdr
) +
55 offsetof(struct ipv4_hdr
, time_to_live
)));
56 data
[3] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m
[3], __m128i
*,
57 sizeof(struct ether_hdr
) +
58 offsetof(struct ipv4_hdr
, time_to_live
)));
59 data
[4] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m
[4], __m128i
*,
60 sizeof(struct ether_hdr
) +
61 offsetof(struct ipv4_hdr
, time_to_live
)));
62 data
[5] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m
[5], __m128i
*,
63 sizeof(struct ether_hdr
) +
64 offsetof(struct ipv4_hdr
, time_to_live
)));
65 data
[6] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m
[6], __m128i
*,
66 sizeof(struct ether_hdr
) +
67 offsetof(struct ipv4_hdr
, time_to_live
)));
68 data
[7] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m
[7], __m128i
*,
69 sizeof(struct ether_hdr
) +
70 offsetof(struct ipv4_hdr
, time_to_live
)));
72 key
[0].xmm
= _mm_and_si128(data
[0], mask0
.x
);
73 key
[1].xmm
= _mm_and_si128(data
[1], mask0
.x
);
74 key
[2].xmm
= _mm_and_si128(data
[2], mask0
.x
);
75 key
[3].xmm
= _mm_and_si128(data
[3], mask0
.x
);
76 key
[4].xmm
= _mm_and_si128(data
[4], mask0
.x
);
77 key
[5].xmm
= _mm_and_si128(data
[5], mask0
.x
);
78 key
[6].xmm
= _mm_and_si128(data
[6], mask0
.x
);
79 key
[7].xmm
= _mm_and_si128(data
[7], mask0
.x
);
81 const void *key_array
[8] = {&key
[0], &key
[1], &key
[2], &key
[3],
82 &key
[4], &key
[5], &key
[6], &key
[7]};
84 rte_hash_lookup_bulk(qconf
->ipv4_lookup_struct
, &key_array
[0], 8, ret
);
86 dst_port
[0] = (uint8_t) ((ret
[0] < 0) ?
87 portid
: ipv4_l3fwd_out_if
[ret
[0]]);
88 dst_port
[1] = (uint8_t) ((ret
[1] < 0) ?
89 portid
: ipv4_l3fwd_out_if
[ret
[1]]);
90 dst_port
[2] = (uint8_t) ((ret
[2] < 0) ?
91 portid
: ipv4_l3fwd_out_if
[ret
[2]]);
92 dst_port
[3] = (uint8_t) ((ret
[3] < 0) ?
93 portid
: ipv4_l3fwd_out_if
[ret
[3]]);
94 dst_port
[4] = (uint8_t) ((ret
[4] < 0) ?
95 portid
: ipv4_l3fwd_out_if
[ret
[4]]);
96 dst_port
[5] = (uint8_t) ((ret
[5] < 0) ?
97 portid
: ipv4_l3fwd_out_if
[ret
[5]]);
98 dst_port
[6] = (uint8_t) ((ret
[6] < 0) ?
99 portid
: ipv4_l3fwd_out_if
[ret
[6]]);
100 dst_port
[7] = (uint8_t) ((ret
[7] < 0) ?
101 portid
: ipv4_l3fwd_out_if
[ret
[7]]);
103 if (dst_port
[0] >= RTE_MAX_ETHPORTS
||
104 (enabled_port_mask
& 1 << dst_port
[0]) == 0)
105 dst_port
[0] = portid
;
107 if (dst_port
[1] >= RTE_MAX_ETHPORTS
||
108 (enabled_port_mask
& 1 << dst_port
[1]) == 0)
109 dst_port
[1] = portid
;
111 if (dst_port
[2] >= RTE_MAX_ETHPORTS
||
112 (enabled_port_mask
& 1 << dst_port
[2]) == 0)
113 dst_port
[2] = portid
;
115 if (dst_port
[3] >= RTE_MAX_ETHPORTS
||
116 (enabled_port_mask
& 1 << dst_port
[3]) == 0)
117 dst_port
[3] = portid
;
119 if (dst_port
[4] >= RTE_MAX_ETHPORTS
||
120 (enabled_port_mask
& 1 << dst_port
[4]) == 0)
121 dst_port
[4] = portid
;
123 if (dst_port
[5] >= RTE_MAX_ETHPORTS
||
124 (enabled_port_mask
& 1 << dst_port
[5]) == 0)
125 dst_port
[5] = portid
;
127 if (dst_port
[6] >= RTE_MAX_ETHPORTS
||
128 (enabled_port_mask
& 1 << dst_port
[6]) == 0)
129 dst_port
[6] = portid
;
131 if (dst_port
[7] >= RTE_MAX_ETHPORTS
||
132 (enabled_port_mask
& 1 << dst_port
[7]) == 0)
133 dst_port
[7] = portid
;
138 get_ipv6_5tuple(struct rte_mbuf
*m0
, __m128i mask0
,
139 __m128i mask1
, union ipv6_5tuple_host
*key
)
141 __m128i tmpdata0
= _mm_loadu_si128(
142 rte_pktmbuf_mtod_offset(m0
, __m128i
*,
143 sizeof(struct ether_hdr
) +
144 offsetof(struct ipv6_hdr
, payload_len
)));
146 __m128i tmpdata1
= _mm_loadu_si128(
147 rte_pktmbuf_mtod_offset(m0
, __m128i
*,
148 sizeof(struct ether_hdr
) +
149 offsetof(struct ipv6_hdr
, payload_len
) +
152 __m128i tmpdata2
= _mm_loadu_si128(
153 rte_pktmbuf_mtod_offset(m0
, __m128i
*,
154 sizeof(struct ether_hdr
) +
155 offsetof(struct ipv6_hdr
, payload_len
) +
156 sizeof(__m128i
) + sizeof(__m128i
)));
158 key
->xmm
[0] = _mm_and_si128(tmpdata0
, mask0
);
159 key
->xmm
[1] = tmpdata1
;
160 key
->xmm
[2] = _mm_and_si128(tmpdata2
, mask1
);
163 static inline __attribute__((always_inline
)) void
164 em_get_dst_port_ipv6x8(struct lcore_conf
*qconf
, struct rte_mbuf
*m
[8],
165 uint8_t portid
, uint16_t dst_port
[8])
168 union ipv6_5tuple_host key
[8];
170 get_ipv6_5tuple(m
[0], mask1
.x
, mask2
.x
, &key
[0]);
171 get_ipv6_5tuple(m
[1], mask1
.x
, mask2
.x
, &key
[1]);
172 get_ipv6_5tuple(m
[2], mask1
.x
, mask2
.x
, &key
[2]);
173 get_ipv6_5tuple(m
[3], mask1
.x
, mask2
.x
, &key
[3]);
174 get_ipv6_5tuple(m
[4], mask1
.x
, mask2
.x
, &key
[4]);
175 get_ipv6_5tuple(m
[5], mask1
.x
, mask2
.x
, &key
[5]);
176 get_ipv6_5tuple(m
[6], mask1
.x
, mask2
.x
, &key
[6]);
177 get_ipv6_5tuple(m
[7], mask1
.x
, mask2
.x
, &key
[7]);
179 const void *key_array
[8] = {&key
[0], &key
[1], &key
[2], &key
[3],
180 &key
[4], &key
[5], &key
[6], &key
[7]};
182 rte_hash_lookup_bulk(qconf
->ipv6_lookup_struct
, &key_array
[0], 8, ret
);
184 dst_port
[0] = (uint8_t) ((ret
[0] < 0) ?
185 portid
: ipv6_l3fwd_out_if
[ret
[0]]);
186 dst_port
[1] = (uint8_t) ((ret
[1] < 0) ?
187 portid
: ipv6_l3fwd_out_if
[ret
[1]]);
188 dst_port
[2] = (uint8_t) ((ret
[2] < 0) ?
189 portid
: ipv6_l3fwd_out_if
[ret
[2]]);
190 dst_port
[3] = (uint8_t) ((ret
[3] < 0) ?
191 portid
: ipv6_l3fwd_out_if
[ret
[3]]);
192 dst_port
[4] = (uint8_t) ((ret
[4] < 0) ?
193 portid
: ipv6_l3fwd_out_if
[ret
[4]]);
194 dst_port
[5] = (uint8_t) ((ret
[5] < 0) ?
195 portid
: ipv6_l3fwd_out_if
[ret
[5]]);
196 dst_port
[6] = (uint8_t) ((ret
[6] < 0) ?
197 portid
: ipv6_l3fwd_out_if
[ret
[6]]);
198 dst_port
[7] = (uint8_t) ((ret
[7] < 0) ?
199 portid
: ipv6_l3fwd_out_if
[ret
[7]]);
201 if (dst_port
[0] >= RTE_MAX_ETHPORTS
||
202 (enabled_port_mask
& 1 << dst_port
[0]) == 0)
203 dst_port
[0] = portid
;
205 if (dst_port
[1] >= RTE_MAX_ETHPORTS
||
206 (enabled_port_mask
& 1 << dst_port
[1]) == 0)
207 dst_port
[1] = portid
;
209 if (dst_port
[2] >= RTE_MAX_ETHPORTS
||
210 (enabled_port_mask
& 1 << dst_port
[2]) == 0)
211 dst_port
[2] = portid
;
213 if (dst_port
[3] >= RTE_MAX_ETHPORTS
||
214 (enabled_port_mask
& 1 << dst_port
[3]) == 0)
215 dst_port
[3] = portid
;
217 if (dst_port
[4] >= RTE_MAX_ETHPORTS
||
218 (enabled_port_mask
& 1 << dst_port
[4]) == 0)
219 dst_port
[4] = portid
;
221 if (dst_port
[5] >= RTE_MAX_ETHPORTS
||
222 (enabled_port_mask
& 1 << dst_port
[5]) == 0)
223 dst_port
[5] = portid
;
225 if (dst_port
[6] >= RTE_MAX_ETHPORTS
||
226 (enabled_port_mask
& 1 << dst_port
[6]) == 0)
227 dst_port
[6] = portid
;
229 if (dst_port
[7] >= RTE_MAX_ETHPORTS
||
230 (enabled_port_mask
& 1 << dst_port
[7]) == 0)
231 dst_port
[7] = portid
;
235 static inline __attribute__((always_inline
)) uint16_t
236 em_get_dst_port(const struct lcore_conf
*qconf
, struct rte_mbuf
*pkt
,
240 struct ipv4_hdr
*ipv4_hdr
;
241 struct ipv6_hdr
*ipv6_hdr
;
245 tcp_or_udp
= pkt
->packet_type
& (RTE_PTYPE_L4_TCP
| RTE_PTYPE_L4_UDP
);
246 l3_ptypes
= pkt
->packet_type
& RTE_PTYPE_L3_MASK
;
248 if (tcp_or_udp
&& (l3_ptypes
== RTE_PTYPE_L3_IPV4
)) {
250 /* Handle IPv4 headers.*/
251 ipv4_hdr
= rte_pktmbuf_mtod_offset(pkt
, struct ipv4_hdr
*,
252 sizeof(struct ether_hdr
));
254 next_hop
= em_get_ipv4_dst_port(ipv4_hdr
, portid
,
255 qconf
->ipv4_lookup_struct
);
257 if (next_hop
>= RTE_MAX_ETHPORTS
||
258 (enabled_port_mask
& 1 << next_hop
) == 0)
263 } else if (tcp_or_udp
&& (l3_ptypes
== RTE_PTYPE_L3_IPV6
)) {
265 /* Handle IPv6 headers.*/
266 ipv6_hdr
= rte_pktmbuf_mtod_offset(pkt
, struct ipv6_hdr
*,
267 sizeof(struct ether_hdr
));
269 next_hop
= em_get_ipv6_dst_port(ipv6_hdr
, portid
,
270 qconf
->ipv6_lookup_struct
);
272 if (next_hop
>= RTE_MAX_ETHPORTS
||
273 (enabled_port_mask
& 1 << next_hop
) == 0)
284 * Buffer optimized handling of packets, invoked
288 l3fwd_em_send_packets(int nb_rx
, struct rte_mbuf
**pkts_burst
,
289 uint8_t portid
, struct lcore_conf
*qconf
)
292 uint16_t dst_port
[MAX_PKT_BURST
];
295 * Send nb_rx - nb_rx%8 packets
298 int32_t n
= RTE_ALIGN_FLOOR(nb_rx
, 8);
300 for (j
= 0; j
< n
; j
+= 8) {
303 pkts_burst
[j
]->packet_type
&
304 pkts_burst
[j
+1]->packet_type
&
305 pkts_burst
[j
+2]->packet_type
&
306 pkts_burst
[j
+3]->packet_type
&
307 pkts_burst
[j
+4]->packet_type
&
308 pkts_burst
[j
+5]->packet_type
&
309 pkts_burst
[j
+6]->packet_type
&
310 pkts_burst
[j
+7]->packet_type
;
312 uint32_t l3_type
= pkt_type
& RTE_PTYPE_L3_MASK
;
313 uint32_t tcp_or_udp
= pkt_type
&
314 (RTE_PTYPE_L4_TCP
| RTE_PTYPE_L4_UDP
);
316 if (tcp_or_udp
&& (l3_type
== RTE_PTYPE_L3_IPV4
)) {
318 em_get_dst_port_ipv4x8(qconf
, &pkts_burst
[j
], portid
, &dst_port
[j
]);
320 } else if (tcp_or_udp
&& (l3_type
== RTE_PTYPE_L3_IPV6
)) {
322 em_get_dst_port_ipv6x8(qconf
, &pkts_burst
[j
], portid
, &dst_port
[j
]);
325 dst_port
[j
] = em_get_dst_port(qconf
, pkts_burst
[j
], portid
);
326 dst_port
[j
+1] = em_get_dst_port(qconf
, pkts_burst
[j
+1], portid
);
327 dst_port
[j
+2] = em_get_dst_port(qconf
, pkts_burst
[j
+2], portid
);
328 dst_port
[j
+3] = em_get_dst_port(qconf
, pkts_burst
[j
+3], portid
);
329 dst_port
[j
+4] = em_get_dst_port(qconf
, pkts_burst
[j
+4], portid
);
330 dst_port
[j
+5] = em_get_dst_port(qconf
, pkts_burst
[j
+5], portid
);
331 dst_port
[j
+6] = em_get_dst_port(qconf
, pkts_burst
[j
+6], portid
);
332 dst_port
[j
+7] = em_get_dst_port(qconf
, pkts_burst
[j
+7], portid
);
336 for (; j
< nb_rx
; j
++)
337 dst_port
[j
] = em_get_dst_port(qconf
, pkts_burst
[j
], portid
);
339 send_packets_multi(qconf
, pkts_burst
, dst_port
, nb_rx
);
342 #endif /* __L3FWD_EM_SSE_HLM_H__ */