]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | /* SPDX-License-Identifier: BSD-3-Clause |
2 | * Copyright(c) 2016 Intel Corporation. | |
3 | * Copyright(c) 2017 IBM Corporation. | |
4 | * All rights reserved. | |
5 | */ | |
6 | ||
7 | #ifndef _L3FWD_ALTIVEC_H_ | |
8 | #define _L3FWD_ALTIVEC_H_ | |
9 | ||
10 | #include "l3fwd.h" | |
11 | #include "l3fwd_common.h" | |
12 | ||
13 | /* | |
14 | * Update source and destination MAC addresses in the ethernet header. | |
15 | * Perform RFC1812 checks and updates for IPV4 packets. | |
16 | */ | |
17 | static inline void | |
18 | processx4_step3(struct rte_mbuf *pkt[FWDSTEP], uint16_t dst_port[FWDSTEP]) | |
19 | { | |
20 | vector unsigned int te[FWDSTEP]; | |
21 | vector unsigned int ve[FWDSTEP]; | |
22 | vector unsigned int *p[FWDSTEP]; | |
23 | ||
24 | p[0] = rte_pktmbuf_mtod(pkt[0], vector unsigned int *); | |
25 | p[1] = rte_pktmbuf_mtod(pkt[1], vector unsigned int *); | |
26 | p[2] = rte_pktmbuf_mtod(pkt[2], vector unsigned int *); | |
27 | p[3] = rte_pktmbuf_mtod(pkt[3], vector unsigned int *); | |
28 | ||
29 | ve[0] = (vector unsigned int)val_eth[dst_port[0]]; | |
30 | te[0] = *p[0]; | |
31 | ||
32 | ve[1] = (vector unsigned int)val_eth[dst_port[1]]; | |
33 | te[1] = *p[1]; | |
34 | ||
35 | ve[2] = (vector unsigned int)val_eth[dst_port[2]]; | |
36 | te[2] = *p[2]; | |
37 | ||
38 | ve[3] = (vector unsigned int)val_eth[dst_port[3]]; | |
39 | te[3] = *p[3]; | |
40 | ||
41 | /* Update first 12 bytes, keep rest bytes intact. */ | |
42 | te[0] = (vector unsigned int)vec_sel( | |
43 | (vector unsigned short)ve[0], | |
44 | (vector unsigned short)te[0], | |
45 | (vector unsigned short) {0, 0, 0, 0, | |
46 | 0, 0, 0xffff, 0xffff}); | |
47 | ||
48 | te[1] = (vector unsigned int)vec_sel( | |
49 | (vector unsigned short)ve[1], | |
50 | (vector unsigned short)te[1], | |
51 | (vector unsigned short) {0, 0, 0, 0, | |
52 | 0, 0, 0xffff, 0xffff}); | |
53 | ||
54 | te[2] = (vector unsigned int)vec_sel( | |
55 | (vector unsigned short)ve[2], | |
56 | (vector unsigned short)te[2], | |
57 | (vector unsigned short) {0, 0, 0, 0, 0, | |
58 | 0, 0xffff, 0xffff}); | |
59 | ||
60 | te[3] = (vector unsigned int)vec_sel( | |
61 | (vector unsigned short)ve[3], | |
62 | (vector unsigned short)te[3], | |
63 | (vector unsigned short) {0, 0, 0, 0, | |
64 | 0, 0, 0xffff, 0xffff}); | |
65 | ||
66 | *p[0] = te[0]; | |
67 | *p[1] = te[1]; | |
68 | *p[2] = te[2]; | |
69 | *p[3] = te[3]; | |
70 | ||
f67539c2 TL |
71 | rfc1812_process((struct rte_ipv4_hdr *) |
72 | ((struct rte_ether_hdr *)p[0] + 1), | |
73 | &dst_port[0], pkt[0]->packet_type); | |
74 | rfc1812_process((struct rte_ipv4_hdr *) | |
75 | ((struct rte_ether_hdr *)p[1] + 1), | |
76 | &dst_port[1], pkt[1]->packet_type); | |
77 | rfc1812_process((struct rte_ipv4_hdr *) | |
78 | ((struct rte_ether_hdr *)p[2] + 1), | |
79 | &dst_port[2], pkt[2]->packet_type); | |
80 | rfc1812_process((struct rte_ipv4_hdr *) | |
81 | ((struct rte_ether_hdr *)p[3] + 1), | |
82 | &dst_port[3], pkt[3]->packet_type); | |
11fdf7f2 TL |
83 | } |
84 | ||
85 | /* | |
86 | * Group consecutive packets with the same destination port in bursts of 4. | |
87 | * Suppose we have array of destination ports: | |
88 | * dst_port[] = {a, b, c, d,, e, ... } | |
89 | * dp1 should contain: <a, b, c, d>, dp2: <b, c, d, e>. | |
90 | * We doing 4 comparisons at once and the result is 4 bit mask. | |
91 | * This mask is used as an index into prebuild array of pnum values. | |
92 | */ | |
93 | static inline uint16_t * | |
94 | port_groupx4(uint16_t pn[FWDSTEP + 1], uint16_t *lp, vector unsigned short dp1, | |
95 | vector unsigned short dp2) | |
96 | { | |
97 | union { | |
98 | uint16_t u16[FWDSTEP + 1]; | |
99 | uint64_t u64; | |
100 | } *pnum = (void *)pn; | |
101 | ||
102 | int32_t v; | |
103 | ||
104 | v = vec_any_eq(dp1, dp2); | |
105 | ||
106 | ||
107 | /* update last port counter. */ | |
108 | lp[0] += gptbl[v].lpv; | |
109 | ||
110 | /* if dest port value has changed. */ | |
111 | if (v != GRPMSK) { | |
112 | pnum->u64 = gptbl[v].pnum; | |
113 | pnum->u16[FWDSTEP] = 1; | |
114 | lp = pnum->u16 + gptbl[v].idx; | |
115 | } | |
116 | ||
117 | return lp; | |
118 | } | |
119 | ||
120 | /** | |
121 | * Process one packet: | |
122 | * Update source and destination MAC addresses in the ethernet header. | |
123 | * Perform RFC1812 checks and updates for IPV4 packets. | |
124 | */ | |
125 | static inline void | |
126 | process_packet(struct rte_mbuf *pkt, uint16_t *dst_port) | |
127 | { | |
f67539c2 | 128 | struct rte_ether_hdr *eth_hdr; |
11fdf7f2 TL |
129 | vector unsigned int te, ve; |
130 | ||
f67539c2 | 131 | eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *); |
11fdf7f2 TL |
132 | |
133 | te = *(vector unsigned int *)eth_hdr; | |
134 | ve = (vector unsigned int)val_eth[dst_port[0]]; | |
135 | ||
f67539c2 | 136 | rfc1812_process((struct rte_ipv4_hdr *)(eth_hdr + 1), dst_port, |
11fdf7f2 TL |
137 | pkt->packet_type); |
138 | ||
139 | /* dynamically vec_sel te and ve for MASK_ETH (0x3f) */ | |
140 | te = (vector unsigned int)vec_sel( | |
141 | (vector unsigned short)ve, | |
142 | (vector unsigned short)te, | |
143 | (vector unsigned short){0, 0, 0, 0, | |
144 | 0, 0, 0xffff, 0xffff}); | |
145 | ||
146 | *(vector unsigned int *)eth_hdr = te; | |
147 | } | |
148 | ||
149 | /** | |
150 | * Send packets burst from pkts_burst to the ports in dst_port array | |
151 | */ | |
152 | static __rte_always_inline void | |
153 | send_packets_multi(struct lcore_conf *qconf, struct rte_mbuf **pkts_burst, | |
154 | uint16_t dst_port[MAX_PKT_BURST], int nb_rx) | |
155 | { | |
156 | int32_t k; | |
157 | int j = 0; | |
158 | uint16_t dlp; | |
159 | uint16_t *lp; | |
160 | uint16_t pnum[MAX_PKT_BURST + 1]; | |
161 | ||
162 | /* | |
163 | * Finish packet processing and group consecutive | |
164 | * packets with the same destination port. | |
165 | */ | |
166 | k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP); | |
167 | if (k != 0) { | |
168 | vector unsigned short dp1, dp2; | |
169 | ||
170 | lp = pnum; | |
171 | lp[0] = 1; | |
172 | ||
173 | processx4_step3(pkts_burst, dst_port); | |
174 | ||
175 | /* dp1: <d[0], d[1], d[2], d[3], ... > */ | |
176 | dp1 = *(vector unsigned short *)dst_port; | |
177 | ||
178 | for (j = FWDSTEP; j != k; j += FWDSTEP) { | |
179 | processx4_step3(&pkts_burst[j], &dst_port[j]); | |
180 | ||
181 | /* | |
182 | * dp2: | |
183 | * <d[j-3], d[j-2], d[j-1], d[j], ... > | |
184 | */ | |
185 | dp2 = *((vector unsigned short *) | |
186 | &dst_port[j - FWDSTEP + 1]); | |
187 | lp = port_groupx4(&pnum[j - FWDSTEP], lp, dp1, dp2); | |
188 | ||
189 | /* | |
190 | * dp1: | |
191 | * <d[j], d[j+1], d[j+2], d[j+3], ... > | |
192 | */ | |
193 | dp1 = vec_sro(dp2, (vector unsigned char) { | |
194 | 0, 0, 0, 0, 0, 0, 0, 0, | |
195 | 0, 0, 0, (FWDSTEP - 1) * sizeof(dst_port[0])}); | |
196 | } | |
197 | ||
198 | /* | |
199 | * dp2: <d[j-3], d[j-2], d[j-1], d[j-1], ... > | |
200 | */ | |
201 | dp2 = vec_perm(dp1, (vector unsigned short){}, | |
202 | (vector unsigned char){0xf9}); | |
203 | lp = port_groupx4(&pnum[j - FWDSTEP], lp, dp1, dp2); | |
204 | ||
205 | /* | |
206 | * remove values added by the last repeated | |
207 | * dst port. | |
208 | */ | |
209 | lp[0]--; | |
210 | dlp = dst_port[j - 1]; | |
211 | } else { | |
212 | /* set dlp and lp to the never used values. */ | |
213 | dlp = BAD_PORT - 1; | |
214 | lp = pnum + MAX_PKT_BURST; | |
215 | } | |
216 | ||
217 | /* Process up to last 3 packets one by one. */ | |
218 | switch (nb_rx % FWDSTEP) { | |
219 | case 3: | |
220 | process_packet(pkts_burst[j], dst_port + j); | |
221 | GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); | |
222 | j++; | |
223 | /* fall-through */ | |
224 | case 2: | |
225 | process_packet(pkts_burst[j], dst_port + j); | |
226 | GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); | |
227 | j++; | |
228 | /* fall-through */ | |
229 | case 1: | |
230 | process_packet(pkts_burst[j], dst_port + j); | |
231 | GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); | |
232 | j++; | |
233 | } | |
234 | ||
235 | /* | |
236 | * Send packets out, through destination port. | |
237 | * Consecutive packets with the same destination port | |
238 | * are already grouped together. | |
239 | * If destination port for the packet equals BAD_PORT, | |
240 | * then free the packet without sending it out. | |
241 | */ | |
242 | for (j = 0; j < nb_rx; j += k) { | |
243 | ||
244 | int32_t m; | |
245 | uint16_t pn; | |
246 | ||
247 | pn = dst_port[j]; | |
248 | k = pnum[j]; | |
249 | ||
250 | if (likely(pn != BAD_PORT)) | |
251 | send_packetsx4(qconf, pn, pkts_burst + j, k); | |
252 | else | |
253 | for (m = j; m != j + k; m++) | |
254 | rte_pktmbuf_free(pkts_burst[m]); | |
255 | ||
256 | } | |
257 | } | |
258 | ||
259 | #endif /* _L3FWD_ALTIVEC_H_ */ |