]>
Commit | Line | Data |
---|---|---|
f67539c2 TL |
1 | /* SPDX-License-Identifier: BSD-3-Clause |
2 | * Copyright(c) 2010-2015 Intel Corporation | |
3 | * Copyright(C) 2019 IBM Corporation | |
4 | */ | |
5 | ||
6 | #include <stdint.h> | |
7 | #include <stdio.h> | |
8 | #include <stdlib.h> | |
9 | #include <string.h> | |
10 | #include <errno.h> | |
11 | ||
12 | #include <rte_altivec.h> | |
13 | #include <rte_byteorder.h> | |
14 | #include <rte_branch_prediction.h> | |
15 | #include <rte_cycles.h> | |
16 | #include <rte_ether.h> | |
17 | #include <rte_ethdev_driver.h> | |
18 | #include <rte_errno.h> | |
19 | #include <rte_memory.h> | |
20 | #include <rte_mempool.h> | |
21 | #include <rte_malloc.h> | |
22 | #include <rte_mbuf.h> | |
23 | #include <rte_prefetch.h> | |
24 | #include <rte_string_fns.h> | |
25 | ||
26 | #include "virtio_rxtx_simple.h" | |
27 | ||
28 | #define RTE_VIRTIO_DESC_PER_LOOP 8 | |
29 | ||
30 | /* virtio vPMD receive routine, only accept(nb_pkts >= RTE_VIRTIO_DESC_PER_LOOP) | |
31 | * | |
32 | * This routine is for non-mergeable RX, one desc for each guest buffer. | |
33 | * This routine is based on the RX ring layout optimization. Each entry in the | |
34 | * avail ring points to the desc with the same index in the desc ring and this | |
35 | * will never be changed in the driver. | |
36 | * | |
37 | * - nb_pkts < RTE_VIRTIO_DESC_PER_LOOP, just return no packet | |
38 | */ | |
39 | uint16_t | |
40 | virtio_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, | |
41 | uint16_t nb_pkts) | |
42 | { | |
43 | struct virtnet_rx *rxvq = rx_queue; | |
44 | struct virtqueue *vq = rxvq->vq; | |
45 | struct virtio_hw *hw = vq->hw; | |
46 | uint16_t nb_used, nb_total; | |
47 | uint16_t desc_idx; | |
48 | struct vring_used_elem *rused; | |
49 | struct rte_mbuf **sw_ring; | |
50 | struct rte_mbuf **sw_ring_end; | |
51 | struct rte_mbuf **ref_rx_pkts; | |
52 | uint16_t nb_pkts_received = 0; | |
53 | const vector unsigned char zero = {0}; | |
54 | ||
55 | const vector unsigned char shuf_msk1 = { | |
56 | 0xFF, 0xFF, 0xFF, 0xFF, /* packet type */ | |
57 | 4, 5, 0xFF, 0xFF, /* vlan tci */ | |
58 | 4, 5, /* dat len */ | |
59 | 0xFF, 0xFF, /* vlan tci */ | |
60 | 0xFF, 0xFF, 0xFF, 0xFF | |
61 | }; | |
62 | ||
63 | const vector unsigned char shuf_msk2 = { | |
64 | 0xFF, 0xFF, 0xFF, 0xFF, /* packet type */ | |
65 | 12, 13, 0xFF, 0xFF, /* pkt len */ | |
66 | 12, 13, /* dat len */ | |
67 | 0xFF, 0xFF, /* vlan tci */ | |
68 | 0xFF, 0xFF, 0xFF, 0xFF | |
69 | }; | |
70 | ||
71 | /* | |
72 | * Subtract the header length. | |
73 | * In which case do we need the header length in used->len ? | |
74 | */ | |
75 | const vector unsigned short len_adjust = { | |
76 | 0, 0, | |
77 | (uint16_t)-vq->hw->vtnet_hdr_size, 0, | |
78 | (uint16_t)-vq->hw->vtnet_hdr_size, 0, | |
79 | 0, 0 | |
80 | }; | |
81 | ||
82 | if (unlikely(hw->started == 0)) | |
83 | return nb_pkts_received; | |
84 | ||
85 | if (unlikely(nb_pkts < RTE_VIRTIO_DESC_PER_LOOP)) | |
86 | return 0; | |
87 | ||
88 | nb_used = virtqueue_nused(vq); | |
89 | ||
90 | rte_compiler_barrier(); | |
91 | ||
92 | if (unlikely(nb_used == 0)) | |
93 | return 0; | |
94 | ||
95 | nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_VIRTIO_DESC_PER_LOOP); | |
96 | nb_used = RTE_MIN(nb_used, nb_pkts); | |
97 | ||
98 | desc_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); | |
99 | rused = &vq->vq_split.ring.used->ring[desc_idx]; | |
100 | sw_ring = &vq->sw_ring[desc_idx]; | |
101 | sw_ring_end = &vq->sw_ring[vq->vq_nentries]; | |
102 | ||
103 | rte_prefetch0(rused); | |
104 | ||
105 | if (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) { | |
106 | virtio_rxq_rearm_vec(rxvq); | |
107 | if (unlikely(virtqueue_kick_prepare(vq))) | |
108 | virtqueue_notify(vq); | |
109 | } | |
110 | ||
111 | nb_total = nb_used; | |
112 | ref_rx_pkts = rx_pkts; | |
113 | for (nb_pkts_received = 0; | |
114 | nb_pkts_received < nb_total;) { | |
115 | vector unsigned char desc[RTE_VIRTIO_DESC_PER_LOOP / 2]; | |
116 | vector unsigned char mbp[RTE_VIRTIO_DESC_PER_LOOP / 2]; | |
117 | vector unsigned char pkt_mb[RTE_VIRTIO_DESC_PER_LOOP]; | |
118 | ||
119 | mbp[0] = vec_vsx_ld(0, (unsigned char const *)(sw_ring + 0)); | |
120 | desc[0] = vec_vsx_ld(0, (unsigned char const *)(rused + 0)); | |
121 | *(vector unsigned char *)&rx_pkts[0] = mbp[0]; | |
122 | ||
123 | mbp[1] = vec_vsx_ld(0, (unsigned char const *)(sw_ring + 2)); | |
124 | desc[1] = vec_vsx_ld(0, (unsigned char const *)(rused + 2)); | |
125 | *(vector unsigned char *)&rx_pkts[2] = mbp[1]; | |
126 | ||
127 | mbp[2] = vec_vsx_ld(0, (unsigned char const *)(sw_ring + 4)); | |
128 | desc[2] = vec_vsx_ld(0, (unsigned char const *)(rused + 4)); | |
129 | *(vector unsigned char *)&rx_pkts[4] = mbp[2]; | |
130 | ||
131 | mbp[3] = vec_vsx_ld(0, (unsigned char const *)(sw_ring + 6)); | |
132 | desc[3] = vec_vsx_ld(0, (unsigned char const *)(rused + 6)); | |
133 | *(vector unsigned char *)&rx_pkts[6] = mbp[3]; | |
134 | ||
135 | pkt_mb[0] = vec_perm(desc[0], zero, shuf_msk1); | |
136 | pkt_mb[1] = vec_perm(desc[0], zero, shuf_msk2); | |
137 | pkt_mb[0] = (vector unsigned char) | |
138 | ((vector unsigned short)pkt_mb[0] + len_adjust); | |
139 | pkt_mb[1] = (vector unsigned char) | |
140 | ((vector unsigned short)pkt_mb[1] + len_adjust); | |
141 | *(vector unsigned char *)&rx_pkts[0]->rx_descriptor_fields1 = | |
142 | pkt_mb[0]; | |
143 | *(vector unsigned char *)&rx_pkts[1]->rx_descriptor_fields1 = | |
144 | pkt_mb[1]; | |
145 | ||
146 | pkt_mb[2] = vec_perm(desc[1], zero, shuf_msk1); | |
147 | pkt_mb[3] = vec_perm(desc[1], zero, shuf_msk2); | |
148 | pkt_mb[2] = (vector unsigned char) | |
149 | ((vector unsigned short)pkt_mb[2] + len_adjust); | |
150 | pkt_mb[3] = (vector unsigned char) | |
151 | ((vector unsigned short)pkt_mb[3] + len_adjust); | |
152 | *(vector unsigned char *)&rx_pkts[2]->rx_descriptor_fields1 = | |
153 | pkt_mb[2]; | |
154 | *(vector unsigned char *)&rx_pkts[3]->rx_descriptor_fields1 = | |
155 | pkt_mb[3]; | |
156 | ||
157 | pkt_mb[4] = vec_perm(desc[2], zero, shuf_msk1); | |
158 | pkt_mb[5] = vec_perm(desc[2], zero, shuf_msk2); | |
159 | pkt_mb[4] = (vector unsigned char) | |
160 | ((vector unsigned short)pkt_mb[4] + len_adjust); | |
161 | pkt_mb[5] = (vector unsigned char) | |
162 | ((vector unsigned short)pkt_mb[5] + len_adjust); | |
163 | *(vector unsigned char *)&rx_pkts[4]->rx_descriptor_fields1 = | |
164 | pkt_mb[4]; | |
165 | *(vector unsigned char *)&rx_pkts[5]->rx_descriptor_fields1 = | |
166 | pkt_mb[5]; | |
167 | ||
168 | pkt_mb[6] = vec_perm(desc[3], zero, shuf_msk1); | |
169 | pkt_mb[7] = vec_perm(desc[3], zero, shuf_msk2); | |
170 | pkt_mb[6] = (vector unsigned char) | |
171 | ((vector unsigned short)pkt_mb[6] + len_adjust); | |
172 | pkt_mb[7] = (vector unsigned char) | |
173 | ((vector unsigned short)pkt_mb[7] + len_adjust); | |
174 | *(vector unsigned char *)&rx_pkts[6]->rx_descriptor_fields1 = | |
175 | pkt_mb[6]; | |
176 | *(vector unsigned char *)&rx_pkts[7]->rx_descriptor_fields1 = | |
177 | pkt_mb[7]; | |
178 | ||
179 | if (unlikely(nb_used <= RTE_VIRTIO_DESC_PER_LOOP)) { | |
180 | if (sw_ring + nb_used <= sw_ring_end) | |
181 | nb_pkts_received += nb_used; | |
182 | else | |
183 | nb_pkts_received += sw_ring_end - sw_ring; | |
184 | break; | |
185 | } else { | |
186 | if (unlikely(sw_ring + RTE_VIRTIO_DESC_PER_LOOP >= | |
187 | sw_ring_end)) { | |
188 | nb_pkts_received += sw_ring_end - sw_ring; | |
189 | break; | |
190 | } else { | |
191 | nb_pkts_received += RTE_VIRTIO_DESC_PER_LOOP; | |
192 | ||
193 | rx_pkts += RTE_VIRTIO_DESC_PER_LOOP; | |
194 | sw_ring += RTE_VIRTIO_DESC_PER_LOOP; | |
195 | rused += RTE_VIRTIO_DESC_PER_LOOP; | |
196 | nb_used -= RTE_VIRTIO_DESC_PER_LOOP; | |
197 | } | |
198 | } | |
199 | } | |
200 | ||
201 | vq->vq_used_cons_idx += nb_pkts_received; | |
202 | vq->vq_free_cnt += nb_pkts_received; | |
203 | rxvq->stats.packets += nb_pkts_received; | |
204 | for (nb_used = 0; nb_used < nb_pkts_received; nb_used++) | |
205 | virtio_update_packet_stats(&rxvq->stats, ref_rx_pkts[nb_used]); | |
206 | ||
207 | return nb_pkts_received; | |
208 | } |