]>
Commit | Line | Data |
---|---|---|
9f95a23c TL |
1 | /* SPDX-License-Identifier: BSD-3-Clause |
2 | * Copyright(c) 2010-2015 Intel Corporation | |
7c673cae FG |
3 | */ |
4 | ||
5 | #ifndef _I40E_RXTX_VEC_COMMON_H_ | |
6 | #define _I40E_RXTX_VEC_COMMON_H_ | |
7 | #include <stdint.h> | |
9f95a23c | 8 | #include <rte_ethdev_driver.h> |
7c673cae FG |
9 | #include <rte_malloc.h> |
10 | ||
11 | #include "i40e_ethdev.h" | |
12 | #include "i40e_rxtx.h" | |
13 | ||
14 | static inline uint16_t | |
15 | reassemble_packets(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_bufs, | |
16 | uint16_t nb_bufs, uint8_t *split_flags) | |
17 | { | |
18 | struct rte_mbuf *pkts[RTE_I40E_VPMD_RX_BURST]; /*finished pkts*/ | |
19 | struct rte_mbuf *start = rxq->pkt_first_seg; | |
20 | struct rte_mbuf *end = rxq->pkt_last_seg; | |
21 | unsigned pkt_idx, buf_idx; | |
22 | ||
23 | for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) { | |
24 | if (end != NULL) { | |
25 | /* processing a split packet */ | |
26 | end->next = rx_bufs[buf_idx]; | |
27 | rx_bufs[buf_idx]->data_len += rxq->crc_len; | |
28 | ||
29 | start->nb_segs++; | |
30 | start->pkt_len += rx_bufs[buf_idx]->data_len; | |
31 | end = end->next; | |
32 | ||
33 | if (!split_flags[buf_idx]) { | |
34 | /* it's the last packet of the set */ | |
35 | start->hash = end->hash; | |
36 | start->ol_flags = end->ol_flags; | |
37 | /* we need to strip crc for the whole packet */ | |
38 | start->pkt_len -= rxq->crc_len; | |
11fdf7f2 | 39 | if (end->data_len > rxq->crc_len) |
7c673cae | 40 | end->data_len -= rxq->crc_len; |
11fdf7f2 | 41 | else { |
7c673cae FG |
42 | /* free up last mbuf */ |
43 | struct rte_mbuf *secondlast = start; | |
44 | ||
11fdf7f2 | 45 | start->nb_segs--; |
7c673cae FG |
46 | while (secondlast->next != end) |
47 | secondlast = secondlast->next; | |
48 | secondlast->data_len -= (rxq->crc_len - | |
49 | end->data_len); | |
50 | secondlast->next = NULL; | |
51 | rte_pktmbuf_free_seg(end); | |
7c673cae FG |
52 | } |
53 | pkts[pkt_idx++] = start; | |
54 | start = end = NULL; | |
55 | } | |
56 | } else { | |
57 | /* not processing a split packet */ | |
58 | if (!split_flags[buf_idx]) { | |
59 | /* not a split packet, save and skip */ | |
60 | pkts[pkt_idx++] = rx_bufs[buf_idx]; | |
61 | continue; | |
62 | } | |
63 | end = start = rx_bufs[buf_idx]; | |
64 | rx_bufs[buf_idx]->data_len += rxq->crc_len; | |
65 | rx_bufs[buf_idx]->pkt_len += rxq->crc_len; | |
66 | } | |
67 | } | |
68 | ||
69 | /* save the partial packet for next time */ | |
70 | rxq->pkt_first_seg = start; | |
71 | rxq->pkt_last_seg = end; | |
72 | memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts))); | |
73 | return pkt_idx; | |
74 | } | |
75 | ||
9f95a23c | 76 | static __rte_always_inline int |
7c673cae FG |
77 | i40e_tx_free_bufs(struct i40e_tx_queue *txq) |
78 | { | |
79 | struct i40e_tx_entry *txep; | |
80 | uint32_t n; | |
81 | uint32_t i; | |
82 | int nb_free = 0; | |
83 | struct rte_mbuf *m, *free[RTE_I40E_TX_MAX_FREE_BUF_SZ]; | |
84 | ||
85 | /* check DD bits on threshold descriptor */ | |
86 | if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & | |
87 | rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) != | |
88 | rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE)) | |
89 | return 0; | |
90 | ||
91 | n = txq->tx_rs_thresh; | |
92 | ||
93 | /* first buffer to free from S/W ring is at index | |
94 | * tx_next_dd - (tx_rs_thresh-1) | |
95 | */ | |
96 | txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)]; | |
11fdf7f2 | 97 | m = rte_pktmbuf_prefree_seg(txep[0].mbuf); |
7c673cae FG |
98 | if (likely(m != NULL)) { |
99 | free[0] = m; | |
100 | nb_free = 1; | |
101 | for (i = 1; i < n; i++) { | |
11fdf7f2 | 102 | m = rte_pktmbuf_prefree_seg(txep[i].mbuf); |
7c673cae FG |
103 | if (likely(m != NULL)) { |
104 | if (likely(m->pool == free[0]->pool)) { | |
105 | free[nb_free++] = m; | |
106 | } else { | |
107 | rte_mempool_put_bulk(free[0]->pool, | |
108 | (void *)free, | |
109 | nb_free); | |
110 | free[0] = m; | |
111 | nb_free = 1; | |
112 | } | |
113 | } | |
114 | } | |
115 | rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); | |
116 | } else { | |
117 | for (i = 1; i < n; i++) { | |
11fdf7f2 | 118 | m = rte_pktmbuf_prefree_seg(txep[i].mbuf); |
7c673cae FG |
119 | if (m != NULL) |
120 | rte_mempool_put(m->pool, m); | |
121 | } | |
122 | } | |
123 | ||
124 | /* buffers were freed, update counters */ | |
125 | txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh); | |
126 | txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh); | |
127 | if (txq->tx_next_dd >= txq->nb_tx_desc) | |
128 | txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); | |
129 | ||
130 | return txq->tx_rs_thresh; | |
131 | } | |
132 | ||
9f95a23c | 133 | static __rte_always_inline void |
7c673cae FG |
134 | tx_backlog_entry(struct i40e_tx_entry *txep, |
135 | struct rte_mbuf **tx_pkts, uint16_t nb_pkts) | |
136 | { | |
137 | int i; | |
138 | ||
139 | for (i = 0; i < (int)nb_pkts; ++i) | |
140 | txep[i].mbuf = tx_pkts[i]; | |
141 | } | |
142 | ||
143 | static inline void | |
144 | _i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq) | |
145 | { | |
146 | const unsigned mask = rxq->nb_rx_desc - 1; | |
147 | unsigned i; | |
148 | ||
149 | if (rxq->sw_ring == NULL || rxq->rxrearm_nb >= rxq->nb_rx_desc) | |
150 | return; | |
151 | ||
152 | /* free all mbufs that are valid in the ring */ | |
153 | if (rxq->rxrearm_nb == 0) { | |
154 | for (i = 0; i < rxq->nb_rx_desc; i++) { | |
155 | if (rxq->sw_ring[i].mbuf != NULL) | |
156 | rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); | |
157 | } | |
158 | } else { | |
159 | for (i = rxq->rx_tail; | |
160 | i != rxq->rxrearm_start; | |
161 | i = (i + 1) & mask) { | |
162 | if (rxq->sw_ring[i].mbuf != NULL) | |
163 | rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); | |
164 | } | |
165 | } | |
166 | ||
167 | rxq->rxrearm_nb = rxq->nb_rx_desc; | |
168 | ||
169 | /* set all entries to NULL */ | |
170 | memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc); | |
171 | } | |
172 | ||
173 | static inline int | |
174 | i40e_rxq_vec_setup_default(struct i40e_rx_queue *rxq) | |
175 | { | |
176 | uintptr_t p; | |
177 | struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */ | |
178 | ||
179 | mb_def.nb_segs = 1; | |
180 | mb_def.data_off = RTE_PKTMBUF_HEADROOM; | |
181 | mb_def.port = rxq->port_id; | |
182 | rte_mbuf_refcnt_set(&mb_def, 1); | |
183 | ||
184 | /* prevent compiler reordering: rearm_data covers previous fields */ | |
185 | rte_compiler_barrier(); | |
186 | p = (uintptr_t)&mb_def.rearm_data; | |
187 | rxq->mbuf_initializer = *(uint64_t *)p; | |
188 | return 0; | |
189 | } | |
190 | ||
191 | static inline int | |
192 | i40e_rx_vec_dev_conf_condition_check_default(struct rte_eth_dev *dev) | |
193 | { | |
194 | #ifndef RTE_LIBRTE_IEEE1588 | |
9f95a23c TL |
195 | struct i40e_adapter *ad = |
196 | I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); | |
7c673cae FG |
197 | struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; |
198 | struct rte_fdir_conf *fconf = &dev->data->dev_conf.fdir_conf; | |
9f95a23c TL |
199 | struct i40e_rx_queue *rxq; |
200 | uint16_t desc, i; | |
201 | bool first_queue; | |
7c673cae | 202 | |
7c673cae FG |
203 | /* no fdir support */ |
204 | if (fconf->mode != RTE_FDIR_MODE_NONE) | |
205 | return -1; | |
206 | ||
9f95a23c TL |
207 | /* no header split support */ |
208 | if (rxmode->offloads & DEV_RX_OFFLOAD_HEADER_SPLIT) | |
7c673cae FG |
209 | return -1; |
210 | ||
11fdf7f2 | 211 | /* no QinQ support */ |
9f95a23c | 212 | if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_EXTEND) |
11fdf7f2 TL |
213 | return -1; |
214 | ||
9f95a23c TL |
215 | /** |
216 | * Vector mode is allowed only when number of Rx queue | |
217 | * descriptor is power of 2. | |
218 | */ | |
219 | if (!dev->data->dev_started) { | |
220 | first_queue = true; | |
221 | for (i = 0; i < dev->data->nb_rx_queues; i++) { | |
222 | rxq = dev->data->rx_queues[i]; | |
223 | if (!rxq) | |
224 | continue; | |
225 | desc = rxq->nb_rx_desc; | |
226 | if (first_queue) | |
227 | ad->rx_vec_allowed = | |
228 | rte_is_power_of_2(desc); | |
229 | else | |
230 | ad->rx_vec_allowed = | |
231 | ad->rx_vec_allowed ? | |
232 | rte_is_power_of_2(desc) : | |
233 | ad->rx_vec_allowed; | |
234 | first_queue = false; | |
235 | } | |
236 | } else { | |
237 | /* Only check the first queue's descriptor number */ | |
238 | for (i = 0; i < dev->data->nb_rx_queues; i++) { | |
239 | rxq = dev->data->rx_queues[i]; | |
240 | if (!rxq) | |
241 | continue; | |
242 | desc = rxq->nb_rx_desc; | |
243 | ad->rx_vec_allowed = rte_is_power_of_2(desc); | |
244 | break; | |
245 | } | |
246 | } | |
247 | ||
7c673cae FG |
248 | return 0; |
249 | #else | |
250 | RTE_SET_USED(dev); | |
251 | return -1; | |
252 | #endif | |
253 | } | |
254 | #endif |