]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | /*- |
2 | * BSD LICENSE | |
3 | * | |
4 | * Copyright(c) 2016 Intel Corporation. All rights reserved. | |
5 | * All rights reserved. | |
6 | * | |
7 | * Redistribution and use in source and binary forms, with or without | |
8 | * modification, are permitted provided that the following conditions | |
9 | * are met: | |
10 | * | |
11 | * * Redistributions of source code must retain the above copyright | |
12 | * notice, this list of conditions and the following disclaimer. | |
13 | * * Redistributions in binary form must reproduce the above copyright | |
14 | * notice, this list of conditions and the following disclaimer in | |
15 | * the documentation and/or other materials provided with the | |
16 | * distribution. | |
17 | * * Neither the name of Intel Corporation nor the names of its | |
18 | * contributors may be used to endorse or promote products derived | |
19 | * from this software without specific prior written permission. | |
20 | * | |
21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
24 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
25 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
26 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
27 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
28 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
29 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
30 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
31 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
32 | */ | |
33 | ||
34 | #ifndef __L3FWD_EM_HLM_SSE_H__ | |
35 | #define __L3FWD_EM_HLM_SSE_H__ | |
36 | ||
37 | #include "l3fwd_sse.h" | |
38 | ||
39 | static inline __attribute__((always_inline)) void | |
40 | em_get_dst_port_ipv4x8(struct lcore_conf *qconf, struct rte_mbuf *m[8], | |
41 | uint8_t portid, uint16_t dst_port[8]) | |
42 | { | |
43 | int32_t ret[8]; | |
44 | union ipv4_5tuple_host key[8]; | |
45 | __m128i data[8]; | |
46 | ||
47 | data[0] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m[0], __m128i *, | |
48 | sizeof(struct ether_hdr) + | |
49 | offsetof(struct ipv4_hdr, time_to_live))); | |
50 | data[1] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m[1], __m128i *, | |
51 | sizeof(struct ether_hdr) + | |
52 | offsetof(struct ipv4_hdr, time_to_live))); | |
53 | data[2] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m[2], __m128i *, | |
54 | sizeof(struct ether_hdr) + | |
55 | offsetof(struct ipv4_hdr, time_to_live))); | |
56 | data[3] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m[3], __m128i *, | |
57 | sizeof(struct ether_hdr) + | |
58 | offsetof(struct ipv4_hdr, time_to_live))); | |
59 | data[4] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m[4], __m128i *, | |
60 | sizeof(struct ether_hdr) + | |
61 | offsetof(struct ipv4_hdr, time_to_live))); | |
62 | data[5] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m[5], __m128i *, | |
63 | sizeof(struct ether_hdr) + | |
64 | offsetof(struct ipv4_hdr, time_to_live))); | |
65 | data[6] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m[6], __m128i *, | |
66 | sizeof(struct ether_hdr) + | |
67 | offsetof(struct ipv4_hdr, time_to_live))); | |
68 | data[7] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m[7], __m128i *, | |
69 | sizeof(struct ether_hdr) + | |
70 | offsetof(struct ipv4_hdr, time_to_live))); | |
71 | ||
72 | key[0].xmm = _mm_and_si128(data[0], mask0.x); | |
73 | key[1].xmm = _mm_and_si128(data[1], mask0.x); | |
74 | key[2].xmm = _mm_and_si128(data[2], mask0.x); | |
75 | key[3].xmm = _mm_and_si128(data[3], mask0.x); | |
76 | key[4].xmm = _mm_and_si128(data[4], mask0.x); | |
77 | key[5].xmm = _mm_and_si128(data[5], mask0.x); | |
78 | key[6].xmm = _mm_and_si128(data[6], mask0.x); | |
79 | key[7].xmm = _mm_and_si128(data[7], mask0.x); | |
80 | ||
81 | const void *key_array[8] = {&key[0], &key[1], &key[2], &key[3], | |
82 | &key[4], &key[5], &key[6], &key[7]}; | |
83 | ||
84 | rte_hash_lookup_bulk(qconf->ipv4_lookup_struct, &key_array[0], 8, ret); | |
85 | ||
86 | dst_port[0] = (uint8_t) ((ret[0] < 0) ? | |
87 | portid : ipv4_l3fwd_out_if[ret[0]]); | |
88 | dst_port[1] = (uint8_t) ((ret[1] < 0) ? | |
89 | portid : ipv4_l3fwd_out_if[ret[1]]); | |
90 | dst_port[2] = (uint8_t) ((ret[2] < 0) ? | |
91 | portid : ipv4_l3fwd_out_if[ret[2]]); | |
92 | dst_port[3] = (uint8_t) ((ret[3] < 0) ? | |
93 | portid : ipv4_l3fwd_out_if[ret[3]]); | |
94 | dst_port[4] = (uint8_t) ((ret[4] < 0) ? | |
95 | portid : ipv4_l3fwd_out_if[ret[4]]); | |
96 | dst_port[5] = (uint8_t) ((ret[5] < 0) ? | |
97 | portid : ipv4_l3fwd_out_if[ret[5]]); | |
98 | dst_port[6] = (uint8_t) ((ret[6] < 0) ? | |
99 | portid : ipv4_l3fwd_out_if[ret[6]]); | |
100 | dst_port[7] = (uint8_t) ((ret[7] < 0) ? | |
101 | portid : ipv4_l3fwd_out_if[ret[7]]); | |
102 | ||
103 | if (dst_port[0] >= RTE_MAX_ETHPORTS || | |
104 | (enabled_port_mask & 1 << dst_port[0]) == 0) | |
105 | dst_port[0] = portid; | |
106 | ||
107 | if (dst_port[1] >= RTE_MAX_ETHPORTS || | |
108 | (enabled_port_mask & 1 << dst_port[1]) == 0) | |
109 | dst_port[1] = portid; | |
110 | ||
111 | if (dst_port[2] >= RTE_MAX_ETHPORTS || | |
112 | (enabled_port_mask & 1 << dst_port[2]) == 0) | |
113 | dst_port[2] = portid; | |
114 | ||
115 | if (dst_port[3] >= RTE_MAX_ETHPORTS || | |
116 | (enabled_port_mask & 1 << dst_port[3]) == 0) | |
117 | dst_port[3] = portid; | |
118 | ||
119 | if (dst_port[4] >= RTE_MAX_ETHPORTS || | |
120 | (enabled_port_mask & 1 << dst_port[4]) == 0) | |
121 | dst_port[4] = portid; | |
122 | ||
123 | if (dst_port[5] >= RTE_MAX_ETHPORTS || | |
124 | (enabled_port_mask & 1 << dst_port[5]) == 0) | |
125 | dst_port[5] = portid; | |
126 | ||
127 | if (dst_port[6] >= RTE_MAX_ETHPORTS || | |
128 | (enabled_port_mask & 1 << dst_port[6]) == 0) | |
129 | dst_port[6] = portid; | |
130 | ||
131 | if (dst_port[7] >= RTE_MAX_ETHPORTS || | |
132 | (enabled_port_mask & 1 << dst_port[7]) == 0) | |
133 | dst_port[7] = portid; | |
134 | ||
135 | } | |
136 | ||
137 | static inline void | |
138 | get_ipv6_5tuple(struct rte_mbuf *m0, __m128i mask0, | |
139 | __m128i mask1, union ipv6_5tuple_host *key) | |
140 | { | |
141 | __m128i tmpdata0 = _mm_loadu_si128( | |
142 | rte_pktmbuf_mtod_offset(m0, __m128i *, | |
143 | sizeof(struct ether_hdr) + | |
144 | offsetof(struct ipv6_hdr, payload_len))); | |
145 | ||
146 | __m128i tmpdata1 = _mm_loadu_si128( | |
147 | rte_pktmbuf_mtod_offset(m0, __m128i *, | |
148 | sizeof(struct ether_hdr) + | |
149 | offsetof(struct ipv6_hdr, payload_len) + | |
150 | sizeof(__m128i))); | |
151 | ||
152 | __m128i tmpdata2 = _mm_loadu_si128( | |
153 | rte_pktmbuf_mtod_offset(m0, __m128i *, | |
154 | sizeof(struct ether_hdr) + | |
155 | offsetof(struct ipv6_hdr, payload_len) + | |
156 | sizeof(__m128i) + sizeof(__m128i))); | |
157 | ||
158 | key->xmm[0] = _mm_and_si128(tmpdata0, mask0); | |
159 | key->xmm[1] = tmpdata1; | |
160 | key->xmm[2] = _mm_and_si128(tmpdata2, mask1); | |
161 | } | |
162 | ||
163 | static inline __attribute__((always_inline)) void | |
164 | em_get_dst_port_ipv6x8(struct lcore_conf *qconf, struct rte_mbuf *m[8], | |
165 | uint8_t portid, uint16_t dst_port[8]) | |
166 | { | |
167 | int32_t ret[8]; | |
168 | union ipv6_5tuple_host key[8]; | |
169 | ||
170 | get_ipv6_5tuple(m[0], mask1.x, mask2.x, &key[0]); | |
171 | get_ipv6_5tuple(m[1], mask1.x, mask2.x, &key[1]); | |
172 | get_ipv6_5tuple(m[2], mask1.x, mask2.x, &key[2]); | |
173 | get_ipv6_5tuple(m[3], mask1.x, mask2.x, &key[3]); | |
174 | get_ipv6_5tuple(m[4], mask1.x, mask2.x, &key[4]); | |
175 | get_ipv6_5tuple(m[5], mask1.x, mask2.x, &key[5]); | |
176 | get_ipv6_5tuple(m[6], mask1.x, mask2.x, &key[6]); | |
177 | get_ipv6_5tuple(m[7], mask1.x, mask2.x, &key[7]); | |
178 | ||
179 | const void *key_array[8] = {&key[0], &key[1], &key[2], &key[3], | |
180 | &key[4], &key[5], &key[6], &key[7]}; | |
181 | ||
182 | rte_hash_lookup_bulk(qconf->ipv6_lookup_struct, &key_array[0], 8, ret); | |
183 | ||
184 | dst_port[0] = (uint8_t) ((ret[0] < 0) ? | |
185 | portid : ipv6_l3fwd_out_if[ret[0]]); | |
186 | dst_port[1] = (uint8_t) ((ret[1] < 0) ? | |
187 | portid : ipv6_l3fwd_out_if[ret[1]]); | |
188 | dst_port[2] = (uint8_t) ((ret[2] < 0) ? | |
189 | portid : ipv6_l3fwd_out_if[ret[2]]); | |
190 | dst_port[3] = (uint8_t) ((ret[3] < 0) ? | |
191 | portid : ipv6_l3fwd_out_if[ret[3]]); | |
192 | dst_port[4] = (uint8_t) ((ret[4] < 0) ? | |
193 | portid : ipv6_l3fwd_out_if[ret[4]]); | |
194 | dst_port[5] = (uint8_t) ((ret[5] < 0) ? | |
195 | portid : ipv6_l3fwd_out_if[ret[5]]); | |
196 | dst_port[6] = (uint8_t) ((ret[6] < 0) ? | |
197 | portid : ipv6_l3fwd_out_if[ret[6]]); | |
198 | dst_port[7] = (uint8_t) ((ret[7] < 0) ? | |
199 | portid : ipv6_l3fwd_out_if[ret[7]]); | |
200 | ||
201 | if (dst_port[0] >= RTE_MAX_ETHPORTS || | |
202 | (enabled_port_mask & 1 << dst_port[0]) == 0) | |
203 | dst_port[0] = portid; | |
204 | ||
205 | if (dst_port[1] >= RTE_MAX_ETHPORTS || | |
206 | (enabled_port_mask & 1 << dst_port[1]) == 0) | |
207 | dst_port[1] = portid; | |
208 | ||
209 | if (dst_port[2] >= RTE_MAX_ETHPORTS || | |
210 | (enabled_port_mask & 1 << dst_port[2]) == 0) | |
211 | dst_port[2] = portid; | |
212 | ||
213 | if (dst_port[3] >= RTE_MAX_ETHPORTS || | |
214 | (enabled_port_mask & 1 << dst_port[3]) == 0) | |
215 | dst_port[3] = portid; | |
216 | ||
217 | if (dst_port[4] >= RTE_MAX_ETHPORTS || | |
218 | (enabled_port_mask & 1 << dst_port[4]) == 0) | |
219 | dst_port[4] = portid; | |
220 | ||
221 | if (dst_port[5] >= RTE_MAX_ETHPORTS || | |
222 | (enabled_port_mask & 1 << dst_port[5]) == 0) | |
223 | dst_port[5] = portid; | |
224 | ||
225 | if (dst_port[6] >= RTE_MAX_ETHPORTS || | |
226 | (enabled_port_mask & 1 << dst_port[6]) == 0) | |
227 | dst_port[6] = portid; | |
228 | ||
229 | if (dst_port[7] >= RTE_MAX_ETHPORTS || | |
230 | (enabled_port_mask & 1 << dst_port[7]) == 0) | |
231 | dst_port[7] = portid; | |
232 | ||
233 | } | |
234 | ||
235 | static inline __attribute__((always_inline)) uint16_t | |
236 | em_get_dst_port(const struct lcore_conf *qconf, struct rte_mbuf *pkt, | |
237 | uint8_t portid) | |
238 | { | |
239 | uint8_t next_hop; | |
240 | struct ipv4_hdr *ipv4_hdr; | |
241 | struct ipv6_hdr *ipv6_hdr; | |
242 | uint32_t tcp_or_udp; | |
243 | uint32_t l3_ptypes; | |
244 | ||
245 | tcp_or_udp = pkt->packet_type & (RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP); | |
246 | l3_ptypes = pkt->packet_type & RTE_PTYPE_L3_MASK; | |
247 | ||
248 | if (tcp_or_udp && (l3_ptypes == RTE_PTYPE_L3_IPV4)) { | |
249 | ||
250 | /* Handle IPv4 headers.*/ | |
251 | ipv4_hdr = rte_pktmbuf_mtod_offset(pkt, struct ipv4_hdr *, | |
252 | sizeof(struct ether_hdr)); | |
253 | ||
254 | next_hop = em_get_ipv4_dst_port(ipv4_hdr, portid, | |
255 | qconf->ipv4_lookup_struct); | |
256 | ||
257 | if (next_hop >= RTE_MAX_ETHPORTS || | |
258 | (enabled_port_mask & 1 << next_hop) == 0) | |
259 | next_hop = portid; | |
260 | ||
261 | return next_hop; | |
262 | ||
263 | } else if (tcp_or_udp && (l3_ptypes == RTE_PTYPE_L3_IPV6)) { | |
264 | ||
265 | /* Handle IPv6 headers.*/ | |
266 | ipv6_hdr = rte_pktmbuf_mtod_offset(pkt, struct ipv6_hdr *, | |
267 | sizeof(struct ether_hdr)); | |
268 | ||
269 | next_hop = em_get_ipv6_dst_port(ipv6_hdr, portid, | |
270 | qconf->ipv6_lookup_struct); | |
271 | ||
272 | if (next_hop >= RTE_MAX_ETHPORTS || | |
273 | (enabled_port_mask & 1 << next_hop) == 0) | |
274 | next_hop = portid; | |
275 | ||
276 | return next_hop; | |
277 | ||
278 | } | |
279 | ||
280 | return portid; | |
281 | } | |
282 | ||
283 | /* | |
284 | * Buffer optimized handling of packets, invoked | |
285 | * from main_loop. | |
286 | */ | |
287 | static inline void | |
288 | l3fwd_em_send_packets(int nb_rx, struct rte_mbuf **pkts_burst, | |
289 | uint8_t portid, struct lcore_conf *qconf) | |
290 | { | |
291 | int32_t j; | |
292 | uint16_t dst_port[MAX_PKT_BURST]; | |
293 | ||
294 | /* | |
295 | * Send nb_rx - nb_rx%8 packets | |
296 | * in groups of 8. | |
297 | */ | |
298 | int32_t n = RTE_ALIGN_FLOOR(nb_rx, 8); | |
299 | ||
300 | for (j = 0; j < n; j += 8) { | |
301 | ||
302 | uint32_t pkt_type = | |
303 | pkts_burst[j]->packet_type & | |
304 | pkts_burst[j+1]->packet_type & | |
305 | pkts_burst[j+2]->packet_type & | |
306 | pkts_burst[j+3]->packet_type & | |
307 | pkts_burst[j+4]->packet_type & | |
308 | pkts_burst[j+5]->packet_type & | |
309 | pkts_burst[j+6]->packet_type & | |
310 | pkts_burst[j+7]->packet_type; | |
311 | ||
312 | uint32_t l3_type = pkt_type & RTE_PTYPE_L3_MASK; | |
313 | uint32_t tcp_or_udp = pkt_type & | |
314 | (RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP); | |
315 | ||
316 | if (tcp_or_udp && (l3_type == RTE_PTYPE_L3_IPV4)) { | |
317 | ||
318 | em_get_dst_port_ipv4x8(qconf, &pkts_burst[j], portid, &dst_port[j]); | |
319 | ||
320 | } else if (tcp_or_udp && (l3_type == RTE_PTYPE_L3_IPV6)) { | |
321 | ||
322 | em_get_dst_port_ipv6x8(qconf, &pkts_burst[j], portid, &dst_port[j]); | |
323 | ||
324 | } else { | |
325 | dst_port[j] = em_get_dst_port(qconf, pkts_burst[j], portid); | |
326 | dst_port[j+1] = em_get_dst_port(qconf, pkts_burst[j+1], portid); | |
327 | dst_port[j+2] = em_get_dst_port(qconf, pkts_burst[j+2], portid); | |
328 | dst_port[j+3] = em_get_dst_port(qconf, pkts_burst[j+3], portid); | |
329 | dst_port[j+4] = em_get_dst_port(qconf, pkts_burst[j+4], portid); | |
330 | dst_port[j+5] = em_get_dst_port(qconf, pkts_burst[j+5], portid); | |
331 | dst_port[j+6] = em_get_dst_port(qconf, pkts_burst[j+6], portid); | |
332 | dst_port[j+7] = em_get_dst_port(qconf, pkts_burst[j+7], portid); | |
333 | } | |
334 | } | |
335 | ||
336 | for (; j < nb_rx; j++) | |
337 | dst_port[j] = em_get_dst_port(qconf, pkts_burst[j], portid); | |
338 | ||
339 | send_packets_multi(qconf, pkts_burst, dst_port, nb_rx); | |
340 | ||
341 | } | |
342 | #endif /* __L3FWD_EM_SSE_HLM_H__ */ |