]>
git.proxmox.com Git - ceph.git/blob - ceph/src/spdk/dpdk/examples/l3fwd/l3fwd_sse.h
1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016 Intel Corporation
10 #include "l3fwd_common.h"
13 * Update source and destination MAC addresses in the ethernet header.
14 * Perform RFC1812 checks and updates for IPV4 packets.
17 processx4_step3(struct rte_mbuf
*pkt
[FWDSTEP
], uint16_t dst_port
[FWDSTEP
])
23 p
[0] = rte_pktmbuf_mtod(pkt
[0], __m128i
*);
24 p
[1] = rte_pktmbuf_mtod(pkt
[1], __m128i
*);
25 p
[2] = rte_pktmbuf_mtod(pkt
[2], __m128i
*);
26 p
[3] = rte_pktmbuf_mtod(pkt
[3], __m128i
*);
28 ve
[0] = val_eth
[dst_port
[0]];
29 te
[0] = _mm_loadu_si128(p
[0]);
31 ve
[1] = val_eth
[dst_port
[1]];
32 te
[1] = _mm_loadu_si128(p
[1]);
34 ve
[2] = val_eth
[dst_port
[2]];
35 te
[2] = _mm_loadu_si128(p
[2]);
37 ve
[3] = val_eth
[dst_port
[3]];
38 te
[3] = _mm_loadu_si128(p
[3]);
40 /* Update first 12 bytes, keep rest bytes intact. */
41 te
[0] = _mm_blend_epi16(te
[0], ve
[0], MASK_ETH
);
42 te
[1] = _mm_blend_epi16(te
[1], ve
[1], MASK_ETH
);
43 te
[2] = _mm_blend_epi16(te
[2], ve
[2], MASK_ETH
);
44 te
[3] = _mm_blend_epi16(te
[3], ve
[3], MASK_ETH
);
46 _mm_storeu_si128(p
[0], te
[0]);
47 _mm_storeu_si128(p
[1], te
[1]);
48 _mm_storeu_si128(p
[2], te
[2]);
49 _mm_storeu_si128(p
[3], te
[3]);
51 rfc1812_process((struct ipv4_hdr
*)((struct ether_hdr
*)p
[0] + 1),
52 &dst_port
[0], pkt
[0]->packet_type
);
53 rfc1812_process((struct ipv4_hdr
*)((struct ether_hdr
*)p
[1] + 1),
54 &dst_port
[1], pkt
[1]->packet_type
);
55 rfc1812_process((struct ipv4_hdr
*)((struct ether_hdr
*)p
[2] + 1),
56 &dst_port
[2], pkt
[2]->packet_type
);
57 rfc1812_process((struct ipv4_hdr
*)((struct ether_hdr
*)p
[3] + 1),
58 &dst_port
[3], pkt
[3]->packet_type
);
62 * Group consecutive packets with the same destination port in bursts of 4.
63 * Suppose we have array of destionation ports:
64 * dst_port[] = {a, b, c, d,, e, ... }
65 * dp1 should contain: <a, b, c, d>, dp2: <b, c, d, e>.
66 * We doing 4 comparisons at once and the result is 4 bit mask.
67 * This mask is used as an index into prebuild array of pnum values.
69 static inline uint16_t *
70 port_groupx4(uint16_t pn
[FWDSTEP
+ 1], uint16_t *lp
, __m128i dp1
, __m128i dp2
)
73 uint16_t u16
[FWDSTEP
+ 1];
79 dp1
= _mm_cmpeq_epi16(dp1
, dp2
);
80 dp1
= _mm_unpacklo_epi16(dp1
, dp1
);
81 v
= _mm_movemask_ps((__m128
)dp1
);
83 /* update last port counter. */
84 lp
[0] += gptbl
[v
].lpv
;
86 /* if dest port value has changed. */
88 pnum
->u64
= gptbl
[v
].pnum
;
89 pnum
->u16
[FWDSTEP
] = 1;
90 lp
= pnum
->u16
+ gptbl
[v
].idx
;
98 * Update source and destination MAC addresses in the ethernet header.
99 * Perform RFC1812 checks and updates for IPV4 packets.
102 process_packet(struct rte_mbuf
*pkt
, uint16_t *dst_port
)
104 struct ether_hdr
*eth_hdr
;
107 eth_hdr
= rte_pktmbuf_mtod(pkt
, struct ether_hdr
*);
109 te
= _mm_loadu_si128((__m128i
*)eth_hdr
);
110 ve
= val_eth
[dst_port
[0]];
112 rfc1812_process((struct ipv4_hdr
*)(eth_hdr
+ 1), dst_port
,
115 te
= _mm_blend_epi16(te
, ve
, MASK_ETH
);
116 _mm_storeu_si128((__m128i
*)eth_hdr
, te
);
120 * Send packets burst from pkts_burst to the ports in dst_port array
122 static __rte_always_inline
void
123 send_packets_multi(struct lcore_conf
*qconf
, struct rte_mbuf
**pkts_burst
,
124 uint16_t dst_port
[MAX_PKT_BURST
], int nb_rx
)
130 uint16_t pnum
[MAX_PKT_BURST
+ 1];
133 * Finish packet processing and group consecutive
134 * packets with the same destination port.
136 k
= RTE_ALIGN_FLOOR(nb_rx
, FWDSTEP
);
143 processx4_step3(pkts_burst
, dst_port
);
145 /* dp1: <d[0], d[1], d[2], d[3], ... > */
146 dp1
= _mm_loadu_si128((__m128i
*)dst_port
);
148 for (j
= FWDSTEP
; j
!= k
; j
+= FWDSTEP
) {
149 processx4_step3(&pkts_burst
[j
], &dst_port
[j
]);
153 * <d[j-3], d[j-2], d[j-1], d[j], ... >
155 dp2
= _mm_loadu_si128((__m128i
*)
156 &dst_port
[j
- FWDSTEP
+ 1]);
157 lp
= port_groupx4(&pnum
[j
- FWDSTEP
], lp
, dp1
, dp2
);
161 * <d[j], d[j+1], d[j+2], d[j+3], ... >
163 dp1
= _mm_srli_si128(dp2
, (FWDSTEP
- 1) *
164 sizeof(dst_port
[0]));
168 * dp2: <d[j-3], d[j-2], d[j-1], d[j-1], ... >
170 dp2
= _mm_shufflelo_epi16(dp1
, 0xf9);
171 lp
= port_groupx4(&pnum
[j
- FWDSTEP
], lp
, dp1
, dp2
);
174 * remove values added by the last repeated
178 dlp
= dst_port
[j
- 1];
180 /* set dlp and lp to the never used values. */
182 lp
= pnum
+ MAX_PKT_BURST
;
185 /* Process up to last 3 packets one by one. */
186 switch (nb_rx
% FWDSTEP
) {
188 process_packet(pkts_burst
[j
], dst_port
+ j
);
189 GROUP_PORT_STEP(dlp
, dst_port
, lp
, pnum
, j
);
193 process_packet(pkts_burst
[j
], dst_port
+ j
);
194 GROUP_PORT_STEP(dlp
, dst_port
, lp
, pnum
, j
);
198 process_packet(pkts_burst
[j
], dst_port
+ j
);
199 GROUP_PORT_STEP(dlp
, dst_port
, lp
, pnum
, j
);
204 * Send packets out, through destination port.
205 * Consecutive packets with the same destination port
206 * are already grouped together.
207 * If destination port for the packet equals BAD_PORT,
208 * then free the packet without sending it out.
210 for (j
= 0; j
< nb_rx
; j
+= k
) {
218 if (likely(pn
!= BAD_PORT
))
219 send_packetsx4(qconf
, pn
, pkts_burst
+ j
, k
);
221 for (m
= j
; m
!= j
+ k
; m
++)
222 rte_pktmbuf_free(pkts_burst
[m
]);
227 #endif /* _L3FWD_SSE_H_ */