]>
Commit | Line | Data |
---|---|---|
9f95a23c | 1 | /* SPDX-License-Identifier: BSD-3-Clause |
11fdf7f2 | 2 | * |
9f95a23c | 3 | * Copyright (c) 2016-2018 Solarflare Communications Inc. |
11fdf7f2 TL |
4 | * All rights reserved. |
5 | * | |
6 | * This software was jointly developed between OKTET Labs (under contract | |
7 | * for Solarflare) and Solarflare Communications, Inc. | |
11fdf7f2 TL |
8 | */ |
9 | ||
10 | #include <rte_mempool.h> | |
11 | ||
12 | #include "efx.h" | |
13 | ||
14 | #include "sfc.h" | |
15 | #include "sfc_debug.h" | |
16 | #include "sfc_log.h" | |
17 | #include "sfc_ev.h" | |
18 | #include "sfc_rx.h" | |
19 | #include "sfc_kvargs.h" | |
20 | #include "sfc_tweak.h" | |
21 | ||
22 | /* | |
23 | * Maximum number of Rx queue flush attempt in the case of failure or | |
24 | * flush timeout | |
25 | */ | |
26 | #define SFC_RX_QFLUSH_ATTEMPTS (3) | |
27 | ||
28 | /* | |
29 | * Time to wait between event queue polling attempts when waiting for Rx | |
30 | * queue flush done or failed events. | |
31 | */ | |
32 | #define SFC_RX_QFLUSH_POLL_WAIT_MS (1) | |
33 | ||
34 | /* | |
35 | * Maximum number of event queue polling attempts when waiting for Rx queue | |
36 | * flush done or failed events. It defines Rx queue flush attempt timeout | |
37 | * together with SFC_RX_QFLUSH_POLL_WAIT_MS. | |
38 | */ | |
39 | #define SFC_RX_QFLUSH_POLL_ATTEMPTS (2000) | |
40 | ||
41 | void | |
9f95a23c | 42 | sfc_rx_qflush_done(struct sfc_rxq_info *rxq_info) |
11fdf7f2 | 43 | { |
9f95a23c TL |
44 | rxq_info->state |= SFC_RXQ_FLUSHED; |
45 | rxq_info->state &= ~SFC_RXQ_FLUSHING; | |
11fdf7f2 TL |
46 | } |
47 | ||
48 | void | |
9f95a23c | 49 | sfc_rx_qflush_failed(struct sfc_rxq_info *rxq_info) |
11fdf7f2 | 50 | { |
9f95a23c TL |
51 | rxq_info->state |= SFC_RXQ_FLUSH_FAILED; |
52 | rxq_info->state &= ~SFC_RXQ_FLUSHING; | |
11fdf7f2 TL |
53 | } |
54 | ||
55 | static void | |
56 | sfc_efx_rx_qrefill(struct sfc_efx_rxq *rxq) | |
57 | { | |
58 | unsigned int free_space; | |
59 | unsigned int bulks; | |
60 | void *objs[SFC_RX_REFILL_BULK]; | |
61 | efsys_dma_addr_t addr[RTE_DIM(objs)]; | |
62 | unsigned int added = rxq->added; | |
63 | unsigned int id; | |
64 | unsigned int i; | |
65 | struct sfc_efx_rx_sw_desc *rxd; | |
66 | struct rte_mbuf *m; | |
67 | uint16_t port_id = rxq->dp.dpq.port_id; | |
68 | ||
9f95a23c | 69 | free_space = rxq->max_fill_level - (added - rxq->completed); |
11fdf7f2 TL |
70 | |
71 | if (free_space < rxq->refill_threshold) | |
72 | return; | |
73 | ||
74 | bulks = free_space / RTE_DIM(objs); | |
75 | /* refill_threshold guarantees that bulks is positive */ | |
76 | SFC_ASSERT(bulks > 0); | |
77 | ||
78 | id = added & rxq->ptr_mask; | |
79 | do { | |
80 | if (unlikely(rte_mempool_get_bulk(rxq->refill_mb_pool, objs, | |
81 | RTE_DIM(objs)) < 0)) { | |
82 | /* | |
83 | * It is hardly a safe way to increment counter | |
84 | * from different contexts, but all PMDs do it. | |
85 | */ | |
86 | rxq->evq->sa->eth_dev->data->rx_mbuf_alloc_failed += | |
87 | RTE_DIM(objs); | |
88 | /* Return if we have posted nothing yet */ | |
89 | if (added == rxq->added) | |
90 | return; | |
91 | /* Push posted */ | |
92 | break; | |
93 | } | |
94 | ||
95 | for (i = 0; i < RTE_DIM(objs); | |
96 | ++i, id = (id + 1) & rxq->ptr_mask) { | |
97 | m = objs[i]; | |
98 | ||
9f95a23c TL |
99 | MBUF_RAW_ALLOC_CHECK(m); |
100 | ||
11fdf7f2 TL |
101 | rxd = &rxq->sw_desc[id]; |
102 | rxd->mbuf = m; | |
103 | ||
11fdf7f2 | 104 | m->data_off = RTE_PKTMBUF_HEADROOM; |
11fdf7f2 TL |
105 | m->port = port_id; |
106 | ||
9f95a23c | 107 | addr[i] = rte_pktmbuf_iova(m); |
11fdf7f2 TL |
108 | } |
109 | ||
110 | efx_rx_qpost(rxq->common, addr, rxq->buf_size, | |
111 | RTE_DIM(objs), rxq->completed, added); | |
112 | added += RTE_DIM(objs); | |
113 | } while (--bulks > 0); | |
114 | ||
115 | SFC_ASSERT(added != rxq->added); | |
116 | rxq->added = added; | |
117 | efx_rx_qpush(rxq->common, added, &rxq->pushed); | |
118 | } | |
119 | ||
120 | static uint64_t | |
121 | sfc_efx_rx_desc_flags_to_offload_flags(const unsigned int desc_flags) | |
122 | { | |
123 | uint64_t mbuf_flags = 0; | |
124 | ||
125 | switch (desc_flags & (EFX_PKT_IPV4 | EFX_CKSUM_IPV4)) { | |
126 | case (EFX_PKT_IPV4 | EFX_CKSUM_IPV4): | |
127 | mbuf_flags |= PKT_RX_IP_CKSUM_GOOD; | |
128 | break; | |
129 | case EFX_PKT_IPV4: | |
130 | mbuf_flags |= PKT_RX_IP_CKSUM_BAD; | |
131 | break; | |
132 | default: | |
133 | RTE_BUILD_BUG_ON(PKT_RX_IP_CKSUM_UNKNOWN != 0); | |
134 | SFC_ASSERT((mbuf_flags & PKT_RX_IP_CKSUM_MASK) == | |
135 | PKT_RX_IP_CKSUM_UNKNOWN); | |
136 | break; | |
137 | } | |
138 | ||
139 | switch ((desc_flags & | |
140 | (EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP))) { | |
141 | case (EFX_PKT_TCP | EFX_CKSUM_TCPUDP): | |
142 | case (EFX_PKT_UDP | EFX_CKSUM_TCPUDP): | |
143 | mbuf_flags |= PKT_RX_L4_CKSUM_GOOD; | |
144 | break; | |
145 | case EFX_PKT_TCP: | |
146 | case EFX_PKT_UDP: | |
147 | mbuf_flags |= PKT_RX_L4_CKSUM_BAD; | |
148 | break; | |
149 | default: | |
150 | RTE_BUILD_BUG_ON(PKT_RX_L4_CKSUM_UNKNOWN != 0); | |
151 | SFC_ASSERT((mbuf_flags & PKT_RX_L4_CKSUM_MASK) == | |
152 | PKT_RX_L4_CKSUM_UNKNOWN); | |
153 | break; | |
154 | } | |
155 | ||
156 | return mbuf_flags; | |
157 | } | |
158 | ||
159 | static uint32_t | |
160 | sfc_efx_rx_desc_flags_to_packet_type(const unsigned int desc_flags) | |
161 | { | |
162 | return RTE_PTYPE_L2_ETHER | | |
163 | ((desc_flags & EFX_PKT_IPV4) ? | |
164 | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN : 0) | | |
165 | ((desc_flags & EFX_PKT_IPV6) ? | |
166 | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN : 0) | | |
167 | ((desc_flags & EFX_PKT_TCP) ? RTE_PTYPE_L4_TCP : 0) | | |
168 | ((desc_flags & EFX_PKT_UDP) ? RTE_PTYPE_L4_UDP : 0); | |
169 | } | |
170 | ||
171 | static const uint32_t * | |
9f95a23c | 172 | sfc_efx_supported_ptypes_get(__rte_unused uint32_t tunnel_encaps) |
11fdf7f2 TL |
173 | { |
174 | static const uint32_t ptypes[] = { | |
175 | RTE_PTYPE_L2_ETHER, | |
176 | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, | |
177 | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, | |
178 | RTE_PTYPE_L4_TCP, | |
179 | RTE_PTYPE_L4_UDP, | |
180 | RTE_PTYPE_UNKNOWN | |
181 | }; | |
182 | ||
183 | return ptypes; | |
184 | } | |
185 | ||
186 | static void | |
187 | sfc_efx_rx_set_rss_hash(struct sfc_efx_rxq *rxq, unsigned int flags, | |
188 | struct rte_mbuf *m) | |
189 | { | |
11fdf7f2 TL |
190 | uint8_t *mbuf_data; |
191 | ||
192 | ||
193 | if ((rxq->flags & SFC_EFX_RXQ_FLAG_RSS_HASH) == 0) | |
194 | return; | |
195 | ||
196 | mbuf_data = rte_pktmbuf_mtod(m, uint8_t *); | |
197 | ||
198 | if (flags & (EFX_PKT_IPV4 | EFX_PKT_IPV6)) { | |
199 | m->hash.rss = efx_pseudo_hdr_hash_get(rxq->common, | |
200 | EFX_RX_HASHALG_TOEPLITZ, | |
201 | mbuf_data); | |
202 | ||
203 | m->ol_flags |= PKT_RX_RSS_HASH; | |
204 | } | |
11fdf7f2 TL |
205 | } |
206 | ||
207 | static uint16_t | |
208 | sfc_efx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) | |
209 | { | |
210 | struct sfc_dp_rxq *dp_rxq = rx_queue; | |
211 | struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq); | |
212 | unsigned int completed; | |
213 | unsigned int prefix_size = rxq->prefix_size; | |
214 | unsigned int done_pkts = 0; | |
215 | boolean_t discard_next = B_FALSE; | |
216 | struct rte_mbuf *scatter_pkt = NULL; | |
217 | ||
218 | if (unlikely((rxq->flags & SFC_EFX_RXQ_FLAG_RUNNING) == 0)) | |
219 | return 0; | |
220 | ||
221 | sfc_ev_qpoll(rxq->evq); | |
222 | ||
223 | completed = rxq->completed; | |
224 | while (completed != rxq->pending && done_pkts < nb_pkts) { | |
225 | unsigned int id; | |
226 | struct sfc_efx_rx_sw_desc *rxd; | |
227 | struct rte_mbuf *m; | |
228 | unsigned int seg_len; | |
229 | unsigned int desc_flags; | |
230 | ||
231 | id = completed++ & rxq->ptr_mask; | |
232 | rxd = &rxq->sw_desc[id]; | |
233 | m = rxd->mbuf; | |
234 | desc_flags = rxd->flags; | |
235 | ||
236 | if (discard_next) | |
237 | goto discard; | |
238 | ||
239 | if (desc_flags & (EFX_ADDR_MISMATCH | EFX_DISCARD)) | |
240 | goto discard; | |
241 | ||
242 | if (desc_flags & EFX_PKT_PREFIX_LEN) { | |
243 | uint16_t tmp_size; | |
244 | int rc __rte_unused; | |
245 | ||
246 | rc = efx_pseudo_hdr_pkt_length_get(rxq->common, | |
247 | rte_pktmbuf_mtod(m, uint8_t *), &tmp_size); | |
248 | SFC_ASSERT(rc == 0); | |
249 | seg_len = tmp_size; | |
250 | } else { | |
251 | seg_len = rxd->size - prefix_size; | |
252 | } | |
253 | ||
254 | rte_pktmbuf_data_len(m) = seg_len; | |
255 | rte_pktmbuf_pkt_len(m) = seg_len; | |
256 | ||
257 | if (scatter_pkt != NULL) { | |
258 | if (rte_pktmbuf_chain(scatter_pkt, m) != 0) { | |
259 | rte_pktmbuf_free(scatter_pkt); | |
260 | goto discard; | |
261 | } | |
262 | /* The packet to deliver */ | |
263 | m = scatter_pkt; | |
264 | } | |
265 | ||
266 | if (desc_flags & EFX_PKT_CONT) { | |
267 | /* The packet is scattered, more fragments to come */ | |
268 | scatter_pkt = m; | |
9f95a23c | 269 | /* Further fragments have no prefix */ |
11fdf7f2 TL |
270 | prefix_size = 0; |
271 | continue; | |
272 | } | |
273 | ||
274 | /* Scattered packet is done */ | |
275 | scatter_pkt = NULL; | |
276 | /* The first fragment of the packet has prefix */ | |
277 | prefix_size = rxq->prefix_size; | |
278 | ||
279 | m->ol_flags = | |
280 | sfc_efx_rx_desc_flags_to_offload_flags(desc_flags); | |
281 | m->packet_type = | |
282 | sfc_efx_rx_desc_flags_to_packet_type(desc_flags); | |
283 | ||
284 | /* | |
285 | * Extract RSS hash from the packet prefix and | |
286 | * set the corresponding field (if needed and possible) | |
287 | */ | |
288 | sfc_efx_rx_set_rss_hash(rxq, desc_flags, m); | |
289 | ||
290 | m->data_off += prefix_size; | |
291 | ||
292 | *rx_pkts++ = m; | |
293 | done_pkts++; | |
294 | continue; | |
295 | ||
296 | discard: | |
297 | discard_next = ((desc_flags & EFX_PKT_CONT) != 0); | |
9f95a23c | 298 | rte_mbuf_raw_free(m); |
11fdf7f2 TL |
299 | rxd->mbuf = NULL; |
300 | } | |
301 | ||
302 | /* pending is only moved when entire packet is received */ | |
303 | SFC_ASSERT(scatter_pkt == NULL); | |
304 | ||
305 | rxq->completed = completed; | |
306 | ||
307 | sfc_efx_rx_qrefill(rxq); | |
308 | ||
309 | return done_pkts; | |
310 | } | |
311 | ||
312 | static sfc_dp_rx_qdesc_npending_t sfc_efx_rx_qdesc_npending; | |
313 | static unsigned int | |
314 | sfc_efx_rx_qdesc_npending(struct sfc_dp_rxq *dp_rxq) | |
315 | { | |
316 | struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq); | |
317 | ||
318 | if ((rxq->flags & SFC_EFX_RXQ_FLAG_RUNNING) == 0) | |
319 | return 0; | |
320 | ||
321 | sfc_ev_qpoll(rxq->evq); | |
322 | ||
323 | return rxq->pending - rxq->completed; | |
324 | } | |
325 | ||
9f95a23c TL |
326 | static sfc_dp_rx_qdesc_status_t sfc_efx_rx_qdesc_status; |
327 | static int | |
328 | sfc_efx_rx_qdesc_status(struct sfc_dp_rxq *dp_rxq, uint16_t offset) | |
329 | { | |
330 | struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq); | |
331 | ||
332 | if (unlikely(offset > rxq->ptr_mask)) | |
333 | return -EINVAL; | |
334 | ||
335 | /* | |
336 | * Poll EvQ to derive up-to-date 'rxq->pending' figure; | |
337 | * it is required for the queue to be running, but the | |
338 | * check is omitted because API design assumes that it | |
339 | * is the duty of the caller to satisfy all conditions | |
340 | */ | |
341 | SFC_ASSERT((rxq->flags & SFC_EFX_RXQ_FLAG_RUNNING) == | |
342 | SFC_EFX_RXQ_FLAG_RUNNING); | |
343 | sfc_ev_qpoll(rxq->evq); | |
344 | ||
345 | /* | |
346 | * There is a handful of reserved entries in the ring, | |
347 | * but an explicit check whether the offset points to | |
348 | * a reserved entry is neglected since the two checks | |
349 | * below rely on the figures which take the HW limits | |
350 | * into account and thus if an entry is reserved, the | |
351 | * checks will fail and UNAVAIL code will be returned | |
352 | */ | |
353 | ||
354 | if (offset < (rxq->pending - rxq->completed)) | |
355 | return RTE_ETH_RX_DESC_DONE; | |
356 | ||
357 | if (offset < (rxq->added - rxq->completed)) | |
358 | return RTE_ETH_RX_DESC_AVAIL; | |
359 | ||
360 | return RTE_ETH_RX_DESC_UNAVAIL; | |
361 | } | |
362 | ||
363 | boolean_t | |
364 | sfc_rx_check_scatter(size_t pdu, size_t rx_buf_size, uint32_t rx_prefix_size, | |
365 | boolean_t rx_scatter_enabled, const char **error) | |
366 | { | |
367 | if ((rx_buf_size < pdu + rx_prefix_size) && !rx_scatter_enabled) { | |
368 | *error = "Rx scatter is disabled and RxQ mbuf pool object size is too small"; | |
369 | return B_FALSE; | |
370 | } | |
371 | ||
372 | return B_TRUE; | |
373 | } | |
374 | ||
375 | /** Get Rx datapath ops by the datapath RxQ handle */ | |
376 | const struct sfc_dp_rx * | |
377 | sfc_dp_rx_by_dp_rxq(const struct sfc_dp_rxq *dp_rxq) | |
378 | { | |
379 | const struct sfc_dp_queue *dpq = &dp_rxq->dpq; | |
380 | struct rte_eth_dev *eth_dev; | |
381 | struct sfc_adapter_priv *sap; | |
382 | ||
383 | SFC_ASSERT(rte_eth_dev_is_valid_port(dpq->port_id)); | |
384 | eth_dev = &rte_eth_devices[dpq->port_id]; | |
385 | ||
386 | sap = sfc_adapter_priv_by_eth_dev(eth_dev); | |
387 | ||
388 | return sap->dp_rx; | |
389 | } | |
390 | ||
391 | struct sfc_rxq_info * | |
392 | sfc_rxq_info_by_dp_rxq(const struct sfc_dp_rxq *dp_rxq) | |
393 | { | |
394 | const struct sfc_dp_queue *dpq = &dp_rxq->dpq; | |
395 | struct rte_eth_dev *eth_dev; | |
396 | struct sfc_adapter_shared *sas; | |
397 | ||
398 | SFC_ASSERT(rte_eth_dev_is_valid_port(dpq->port_id)); | |
399 | eth_dev = &rte_eth_devices[dpq->port_id]; | |
400 | ||
401 | sas = sfc_adapter_shared_by_eth_dev(eth_dev); | |
402 | ||
403 | SFC_ASSERT(dpq->queue_id < sas->rxq_count); | |
404 | return &sas->rxq_info[dpq->queue_id]; | |
405 | } | |
406 | ||
11fdf7f2 TL |
407 | struct sfc_rxq * |
408 | sfc_rxq_by_dp_rxq(const struct sfc_dp_rxq *dp_rxq) | |
409 | { | |
410 | const struct sfc_dp_queue *dpq = &dp_rxq->dpq; | |
411 | struct rte_eth_dev *eth_dev; | |
412 | struct sfc_adapter *sa; | |
11fdf7f2 TL |
413 | |
414 | SFC_ASSERT(rte_eth_dev_is_valid_port(dpq->port_id)); | |
415 | eth_dev = &rte_eth_devices[dpq->port_id]; | |
416 | ||
9f95a23c | 417 | sa = sfc_adapter_by_eth_dev(eth_dev); |
11fdf7f2 | 418 | |
9f95a23c TL |
419 | SFC_ASSERT(dpq->queue_id < sfc_sa2shared(sa)->rxq_count); |
420 | return &sa->rxq_ctrl[dpq->queue_id]; | |
421 | } | |
11fdf7f2 | 422 | |
9f95a23c TL |
423 | static sfc_dp_rx_qsize_up_rings_t sfc_efx_rx_qsize_up_rings; |
424 | static int | |
425 | sfc_efx_rx_qsize_up_rings(uint16_t nb_rx_desc, | |
426 | __rte_unused struct sfc_dp_rx_hw_limits *limits, | |
427 | __rte_unused struct rte_mempool *mb_pool, | |
428 | unsigned int *rxq_entries, | |
429 | unsigned int *evq_entries, | |
430 | unsigned int *rxq_max_fill_level) | |
431 | { | |
432 | *rxq_entries = nb_rx_desc; | |
433 | *evq_entries = nb_rx_desc; | |
434 | *rxq_max_fill_level = EFX_RXQ_LIMIT(*rxq_entries); | |
435 | return 0; | |
11fdf7f2 TL |
436 | } |
437 | ||
438 | static sfc_dp_rx_qcreate_t sfc_efx_rx_qcreate; | |
439 | static int | |
440 | sfc_efx_rx_qcreate(uint16_t port_id, uint16_t queue_id, | |
441 | const struct rte_pci_addr *pci_addr, int socket_id, | |
442 | const struct sfc_dp_rx_qcreate_info *info, | |
443 | struct sfc_dp_rxq **dp_rxqp) | |
444 | { | |
445 | struct sfc_efx_rxq *rxq; | |
446 | int rc; | |
447 | ||
448 | rc = ENOMEM; | |
449 | rxq = rte_zmalloc_socket("sfc-efx-rxq", sizeof(*rxq), | |
450 | RTE_CACHE_LINE_SIZE, socket_id); | |
451 | if (rxq == NULL) | |
452 | goto fail_rxq_alloc; | |
453 | ||
454 | sfc_dp_queue_init(&rxq->dp.dpq, port_id, queue_id, pci_addr); | |
455 | ||
456 | rc = ENOMEM; | |
457 | rxq->sw_desc = rte_calloc_socket("sfc-efx-rxq-sw_desc", | |
458 | info->rxq_entries, | |
459 | sizeof(*rxq->sw_desc), | |
460 | RTE_CACHE_LINE_SIZE, socket_id); | |
461 | if (rxq->sw_desc == NULL) | |
462 | goto fail_desc_alloc; | |
463 | ||
464 | /* efx datapath is bound to efx control path */ | |
465 | rxq->evq = sfc_rxq_by_dp_rxq(&rxq->dp)->evq; | |
466 | if (info->flags & SFC_RXQ_FLAG_RSS_HASH) | |
467 | rxq->flags |= SFC_EFX_RXQ_FLAG_RSS_HASH; | |
468 | rxq->ptr_mask = info->rxq_entries - 1; | |
469 | rxq->batch_max = info->batch_max; | |
470 | rxq->prefix_size = info->prefix_size; | |
9f95a23c | 471 | rxq->max_fill_level = info->max_fill_level; |
11fdf7f2 TL |
472 | rxq->refill_threshold = info->refill_threshold; |
473 | rxq->buf_size = info->buf_size; | |
474 | rxq->refill_mb_pool = info->refill_mb_pool; | |
475 | ||
476 | *dp_rxqp = &rxq->dp; | |
477 | return 0; | |
478 | ||
479 | fail_desc_alloc: | |
480 | rte_free(rxq); | |
481 | ||
482 | fail_rxq_alloc: | |
483 | return rc; | |
484 | } | |
485 | ||
486 | static sfc_dp_rx_qdestroy_t sfc_efx_rx_qdestroy; | |
487 | static void | |
488 | sfc_efx_rx_qdestroy(struct sfc_dp_rxq *dp_rxq) | |
489 | { | |
490 | struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq); | |
491 | ||
492 | rte_free(rxq->sw_desc); | |
493 | rte_free(rxq); | |
494 | } | |
495 | ||
496 | static sfc_dp_rx_qstart_t sfc_efx_rx_qstart; | |
497 | static int | |
498 | sfc_efx_rx_qstart(struct sfc_dp_rxq *dp_rxq, | |
499 | __rte_unused unsigned int evq_read_ptr) | |
500 | { | |
501 | /* libefx-based datapath is specific to libefx-based PMD */ | |
502 | struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq); | |
503 | struct sfc_rxq *crxq = sfc_rxq_by_dp_rxq(dp_rxq); | |
504 | ||
505 | rxq->common = crxq->common; | |
506 | ||
507 | rxq->pending = rxq->completed = rxq->added = rxq->pushed = 0; | |
508 | ||
509 | sfc_efx_rx_qrefill(rxq); | |
510 | ||
511 | rxq->flags |= (SFC_EFX_RXQ_FLAG_STARTED | SFC_EFX_RXQ_FLAG_RUNNING); | |
512 | ||
513 | return 0; | |
514 | } | |
515 | ||
516 | static sfc_dp_rx_qstop_t sfc_efx_rx_qstop; | |
517 | static void | |
518 | sfc_efx_rx_qstop(struct sfc_dp_rxq *dp_rxq, | |
519 | __rte_unused unsigned int *evq_read_ptr) | |
520 | { | |
521 | struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq); | |
522 | ||
523 | rxq->flags &= ~SFC_EFX_RXQ_FLAG_RUNNING; | |
524 | ||
525 | /* libefx-based datapath is bound to libefx-based PMD and uses | |
526 | * event queue structure directly. So, there is no necessity to | |
527 | * return EvQ read pointer. | |
528 | */ | |
529 | } | |
530 | ||
531 | static sfc_dp_rx_qpurge_t sfc_efx_rx_qpurge; | |
532 | static void | |
533 | sfc_efx_rx_qpurge(struct sfc_dp_rxq *dp_rxq) | |
534 | { | |
535 | struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq); | |
536 | unsigned int i; | |
537 | struct sfc_efx_rx_sw_desc *rxd; | |
538 | ||
539 | for (i = rxq->completed; i != rxq->added; ++i) { | |
540 | rxd = &rxq->sw_desc[i & rxq->ptr_mask]; | |
9f95a23c | 541 | rte_mbuf_raw_free(rxd->mbuf); |
11fdf7f2 TL |
542 | rxd->mbuf = NULL; |
543 | /* Packed stream relies on 0 in inactive SW desc. | |
544 | * Rx queue stop is not performance critical, so | |
545 | * there is no harm to do it always. | |
546 | */ | |
547 | rxd->flags = 0; | |
548 | rxd->size = 0; | |
549 | } | |
550 | ||
551 | rxq->flags &= ~SFC_EFX_RXQ_FLAG_STARTED; | |
552 | } | |
553 | ||
554 | struct sfc_dp_rx sfc_efx_rx = { | |
555 | .dp = { | |
556 | .name = SFC_KVARG_DATAPATH_EFX, | |
557 | .type = SFC_DP_RX, | |
558 | .hw_fw_caps = 0, | |
559 | }, | |
9f95a23c TL |
560 | .features = SFC_DP_RX_FEAT_SCATTER | |
561 | SFC_DP_RX_FEAT_CHECKSUM, | |
562 | .qsize_up_rings = sfc_efx_rx_qsize_up_rings, | |
11fdf7f2 TL |
563 | .qcreate = sfc_efx_rx_qcreate, |
564 | .qdestroy = sfc_efx_rx_qdestroy, | |
565 | .qstart = sfc_efx_rx_qstart, | |
566 | .qstop = sfc_efx_rx_qstop, | |
567 | .qpurge = sfc_efx_rx_qpurge, | |
568 | .supported_ptypes_get = sfc_efx_supported_ptypes_get, | |
569 | .qdesc_npending = sfc_efx_rx_qdesc_npending, | |
9f95a23c | 570 | .qdesc_status = sfc_efx_rx_qdesc_status, |
11fdf7f2 TL |
571 | .pkt_burst = sfc_efx_recv_pkts, |
572 | }; | |
573 | ||
11fdf7f2 TL |
574 | static void |
575 | sfc_rx_qflush(struct sfc_adapter *sa, unsigned int sw_index) | |
576 | { | |
9f95a23c | 577 | struct sfc_rxq_info *rxq_info; |
11fdf7f2 TL |
578 | struct sfc_rxq *rxq; |
579 | unsigned int retry_count; | |
580 | unsigned int wait_count; | |
9f95a23c TL |
581 | int rc; |
582 | ||
583 | rxq_info = &sfc_sa2shared(sa)->rxq_info[sw_index]; | |
584 | SFC_ASSERT(rxq_info->state & SFC_RXQ_STARTED); | |
11fdf7f2 | 585 | |
9f95a23c | 586 | rxq = &sa->rxq_ctrl[sw_index]; |
11fdf7f2 TL |
587 | |
588 | /* | |
589 | * Retry Rx queue flushing in the case of flush failed or | |
590 | * timeout. In the worst case it can delay for 6 seconds. | |
591 | */ | |
592 | for (retry_count = 0; | |
9f95a23c | 593 | ((rxq_info->state & SFC_RXQ_FLUSHED) == 0) && |
11fdf7f2 TL |
594 | (retry_count < SFC_RX_QFLUSH_ATTEMPTS); |
595 | ++retry_count) { | |
9f95a23c TL |
596 | rc = efx_rx_qflush(rxq->common); |
597 | if (rc != 0) { | |
598 | rxq_info->state |= (rc == EALREADY) ? | |
599 | SFC_RXQ_FLUSHED : SFC_RXQ_FLUSH_FAILED; | |
11fdf7f2 TL |
600 | break; |
601 | } | |
9f95a23c TL |
602 | rxq_info->state &= ~SFC_RXQ_FLUSH_FAILED; |
603 | rxq_info->state |= SFC_RXQ_FLUSHING; | |
11fdf7f2 TL |
604 | |
605 | /* | |
606 | * Wait for Rx queue flush done or failed event at least | |
607 | * SFC_RX_QFLUSH_POLL_WAIT_MS milliseconds and not more | |
608 | * than 2 seconds (SFC_RX_QFLUSH_POLL_WAIT_MS multiplied | |
609 | * by SFC_RX_QFLUSH_POLL_ATTEMPTS). | |
610 | */ | |
611 | wait_count = 0; | |
612 | do { | |
613 | rte_delay_ms(SFC_RX_QFLUSH_POLL_WAIT_MS); | |
614 | sfc_ev_qpoll(rxq->evq); | |
9f95a23c | 615 | } while ((rxq_info->state & SFC_RXQ_FLUSHING) && |
11fdf7f2 TL |
616 | (wait_count++ < SFC_RX_QFLUSH_POLL_ATTEMPTS)); |
617 | ||
9f95a23c | 618 | if (rxq_info->state & SFC_RXQ_FLUSHING) |
11fdf7f2 TL |
619 | sfc_err(sa, "RxQ %u flush timed out", sw_index); |
620 | ||
9f95a23c | 621 | if (rxq_info->state & SFC_RXQ_FLUSH_FAILED) |
11fdf7f2 TL |
622 | sfc_err(sa, "RxQ %u flush failed", sw_index); |
623 | ||
9f95a23c TL |
624 | if (rxq_info->state & SFC_RXQ_FLUSHED) |
625 | sfc_notice(sa, "RxQ %u flushed", sw_index); | |
11fdf7f2 TL |
626 | } |
627 | ||
9f95a23c | 628 | sa->priv.dp_rx->qpurge(rxq_info->dp); |
11fdf7f2 TL |
629 | } |
630 | ||
631 | static int | |
632 | sfc_rx_default_rxq_set_filter(struct sfc_adapter *sa, struct sfc_rxq *rxq) | |
633 | { | |
9f95a23c TL |
634 | struct sfc_rss *rss = &sfc_sa2shared(sa)->rss; |
635 | boolean_t need_rss = (rss->channels > 0) ? B_TRUE : B_FALSE; | |
11fdf7f2 TL |
636 | struct sfc_port *port = &sa->port; |
637 | int rc; | |
638 | ||
639 | /* | |
640 | * If promiscuous or all-multicast mode has been requested, setting | |
641 | * filter for the default Rx queue might fail, in particular, while | |
642 | * running over PCI function which is not a member of corresponding | |
643 | * privilege groups; if this occurs, few iterations will be made to | |
644 | * repeat this step without promiscuous and all-multicast flags set | |
645 | */ | |
646 | retry: | |
9f95a23c | 647 | rc = efx_mac_filter_default_rxq_set(sa->nic, rxq->common, need_rss); |
11fdf7f2 TL |
648 | if (rc == 0) |
649 | return 0; | |
650 | else if (rc != EOPNOTSUPP) | |
651 | return rc; | |
652 | ||
653 | if (port->promisc) { | |
654 | sfc_warn(sa, "promiscuous mode has been requested, " | |
655 | "but the HW rejects it"); | |
656 | sfc_warn(sa, "promiscuous mode will be disabled"); | |
657 | ||
658 | port->promisc = B_FALSE; | |
659 | rc = sfc_set_rx_mode(sa); | |
660 | if (rc != 0) | |
661 | return rc; | |
662 | ||
663 | goto retry; | |
664 | } | |
665 | ||
666 | if (port->allmulti) { | |
667 | sfc_warn(sa, "all-multicast mode has been requested, " | |
668 | "but the HW rejects it"); | |
669 | sfc_warn(sa, "all-multicast mode will be disabled"); | |
670 | ||
671 | port->allmulti = B_FALSE; | |
672 | rc = sfc_set_rx_mode(sa); | |
673 | if (rc != 0) | |
674 | return rc; | |
675 | ||
676 | goto retry; | |
677 | } | |
678 | ||
679 | return rc; | |
680 | } | |
681 | ||
682 | int | |
683 | sfc_rx_qstart(struct sfc_adapter *sa, unsigned int sw_index) | |
684 | { | |
685 | struct sfc_rxq_info *rxq_info; | |
686 | struct sfc_rxq *rxq; | |
687 | struct sfc_evq *evq; | |
688 | int rc; | |
689 | ||
690 | sfc_log_init(sa, "sw_index=%u", sw_index); | |
691 | ||
9f95a23c | 692 | SFC_ASSERT(sw_index < sfc_sa2shared(sa)->rxq_count); |
11fdf7f2 | 693 | |
9f95a23c TL |
694 | rxq_info = &sfc_sa2shared(sa)->rxq_info[sw_index]; |
695 | SFC_ASSERT(rxq_info->state == SFC_RXQ_INITIALIZED); | |
11fdf7f2 | 696 | |
9f95a23c | 697 | rxq = &sa->rxq_ctrl[sw_index]; |
11fdf7f2 TL |
698 | evq = rxq->evq; |
699 | ||
700 | rc = sfc_ev_qstart(evq, sfc_evq_index_by_rxq_sw_index(sa, sw_index)); | |
701 | if (rc != 0) | |
702 | goto fail_ev_qstart; | |
703 | ||
9f95a23c TL |
704 | switch (rxq_info->type) { |
705 | case EFX_RXQ_TYPE_DEFAULT: | |
706 | rc = efx_rx_qcreate(sa->nic, rxq->hw_index, 0, rxq_info->type, | |
707 | rxq->buf_size, | |
708 | &rxq->mem, rxq_info->entries, 0 /* not used on EF10 */, | |
709 | rxq_info->type_flags, evq->common, &rxq->common); | |
710 | break; | |
711 | case EFX_RXQ_TYPE_ES_SUPER_BUFFER: { | |
712 | struct rte_mempool *mp = rxq_info->refill_mb_pool; | |
713 | struct rte_mempool_info mp_info; | |
714 | ||
715 | rc = rte_mempool_ops_get_info(mp, &mp_info); | |
716 | if (rc != 0) { | |
717 | /* Positive errno is used in the driver */ | |
718 | rc = -rc; | |
719 | goto fail_mp_get_info; | |
720 | } | |
721 | if (mp_info.contig_block_size <= 0) { | |
722 | rc = EINVAL; | |
723 | goto fail_bad_contig_block_size; | |
724 | } | |
725 | rc = efx_rx_qcreate_es_super_buffer(sa->nic, rxq->hw_index, 0, | |
726 | mp_info.contig_block_size, rxq->buf_size, | |
727 | mp->header_size + mp->elt_size + mp->trailer_size, | |
728 | sa->rxd_wait_timeout_ns, | |
729 | &rxq->mem, rxq_info->entries, rxq_info->type_flags, | |
730 | evq->common, &rxq->common); | |
731 | break; | |
732 | } | |
733 | default: | |
734 | rc = ENOTSUP; | |
735 | } | |
11fdf7f2 TL |
736 | if (rc != 0) |
737 | goto fail_rx_qcreate; | |
738 | ||
739 | efx_rx_qenable(rxq->common); | |
740 | ||
9f95a23c | 741 | rc = sa->priv.dp_rx->qstart(rxq_info->dp, evq->read_ptr); |
11fdf7f2 TL |
742 | if (rc != 0) |
743 | goto fail_dp_qstart; | |
744 | ||
9f95a23c | 745 | rxq_info->state |= SFC_RXQ_STARTED; |
11fdf7f2 | 746 | |
9f95a23c | 747 | if (sw_index == 0 && !sfc_sa2shared(sa)->isolated) { |
11fdf7f2 TL |
748 | rc = sfc_rx_default_rxq_set_filter(sa, rxq); |
749 | if (rc != 0) | |
750 | goto fail_mac_filter_default_rxq_set; | |
751 | } | |
752 | ||
753 | /* It seems to be used by DPDK for debug purposes only ('rte_ether') */ | |
754 | sa->eth_dev->data->rx_queue_state[sw_index] = | |
755 | RTE_ETH_QUEUE_STATE_STARTED; | |
756 | ||
757 | return 0; | |
758 | ||
759 | fail_mac_filter_default_rxq_set: | |
9f95a23c | 760 | sa->priv.dp_rx->qstop(rxq_info->dp, &rxq->evq->read_ptr); |
11fdf7f2 TL |
761 | |
762 | fail_dp_qstart: | |
763 | sfc_rx_qflush(sa, sw_index); | |
764 | ||
765 | fail_rx_qcreate: | |
9f95a23c TL |
766 | fail_bad_contig_block_size: |
767 | fail_mp_get_info: | |
11fdf7f2 TL |
768 | sfc_ev_qstop(evq); |
769 | ||
770 | fail_ev_qstart: | |
771 | return rc; | |
772 | } | |
773 | ||
774 | void | |
775 | sfc_rx_qstop(struct sfc_adapter *sa, unsigned int sw_index) | |
776 | { | |
777 | struct sfc_rxq_info *rxq_info; | |
778 | struct sfc_rxq *rxq; | |
779 | ||
780 | sfc_log_init(sa, "sw_index=%u", sw_index); | |
781 | ||
9f95a23c | 782 | SFC_ASSERT(sw_index < sfc_sa2shared(sa)->rxq_count); |
11fdf7f2 | 783 | |
9f95a23c | 784 | rxq_info = &sfc_sa2shared(sa)->rxq_info[sw_index]; |
11fdf7f2 | 785 | |
9f95a23c | 786 | if (rxq_info->state == SFC_RXQ_INITIALIZED) |
11fdf7f2 | 787 | return; |
9f95a23c | 788 | SFC_ASSERT(rxq_info->state & SFC_RXQ_STARTED); |
11fdf7f2 TL |
789 | |
790 | /* It seems to be used by DPDK for debug purposes only ('rte_ether') */ | |
791 | sa->eth_dev->data->rx_queue_state[sw_index] = | |
792 | RTE_ETH_QUEUE_STATE_STOPPED; | |
793 | ||
9f95a23c TL |
794 | rxq = &sa->rxq_ctrl[sw_index]; |
795 | sa->priv.dp_rx->qstop(rxq_info->dp, &rxq->evq->read_ptr); | |
11fdf7f2 TL |
796 | |
797 | if (sw_index == 0) | |
798 | efx_mac_filter_default_rxq_clear(sa->nic); | |
799 | ||
800 | sfc_rx_qflush(sa, sw_index); | |
801 | ||
9f95a23c | 802 | rxq_info->state = SFC_RXQ_INITIALIZED; |
11fdf7f2 TL |
803 | |
804 | efx_rx_qdestroy(rxq->common); | |
805 | ||
806 | sfc_ev_qstop(rxq->evq); | |
807 | } | |
808 | ||
9f95a23c TL |
809 | uint64_t |
810 | sfc_rx_get_dev_offload_caps(struct sfc_adapter *sa) | |
811 | { | |
812 | const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); | |
813 | uint64_t caps = 0; | |
814 | ||
815 | caps |= DEV_RX_OFFLOAD_JUMBO_FRAME; | |
816 | ||
817 | if (sa->priv.dp_rx->features & SFC_DP_RX_FEAT_CHECKSUM) { | |
818 | caps |= DEV_RX_OFFLOAD_IPV4_CKSUM; | |
819 | caps |= DEV_RX_OFFLOAD_UDP_CKSUM; | |
820 | caps |= DEV_RX_OFFLOAD_TCP_CKSUM; | |
821 | } | |
822 | ||
823 | if (encp->enc_tunnel_encapsulations_supported && | |
824 | (sa->priv.dp_rx->features & SFC_DP_RX_FEAT_TUNNELS)) | |
825 | caps |= DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM; | |
826 | ||
827 | return caps; | |
828 | } | |
829 | ||
830 | uint64_t | |
831 | sfc_rx_get_queue_offload_caps(struct sfc_adapter *sa) | |
832 | { | |
833 | uint64_t caps = 0; | |
834 | ||
835 | if (sa->priv.dp_rx->features & SFC_DP_RX_FEAT_SCATTER) | |
836 | caps |= DEV_RX_OFFLOAD_SCATTER; | |
837 | ||
838 | return caps; | |
839 | } | |
840 | ||
11fdf7f2 | 841 | static int |
9f95a23c TL |
842 | sfc_rx_qcheck_conf(struct sfc_adapter *sa, unsigned int rxq_max_fill_level, |
843 | const struct rte_eth_rxconf *rx_conf, | |
844 | __rte_unused uint64_t offloads) | |
11fdf7f2 | 845 | { |
11fdf7f2 TL |
846 | int rc = 0; |
847 | ||
848 | if (rx_conf->rx_thresh.pthresh != 0 || | |
849 | rx_conf->rx_thresh.hthresh != 0 || | |
850 | rx_conf->rx_thresh.wthresh != 0) { | |
9f95a23c | 851 | sfc_warn(sa, |
11fdf7f2 | 852 | "RxQ prefetch/host/writeback thresholds are not supported"); |
11fdf7f2 TL |
853 | } |
854 | ||
9f95a23c | 855 | if (rx_conf->rx_free_thresh > rxq_max_fill_level) { |
11fdf7f2 TL |
856 | sfc_err(sa, |
857 | "RxQ free threshold too large: %u vs maximum %u", | |
9f95a23c | 858 | rx_conf->rx_free_thresh, rxq_max_fill_level); |
11fdf7f2 TL |
859 | rc = EINVAL; |
860 | } | |
861 | ||
862 | if (rx_conf->rx_drop_en == 0) { | |
863 | sfc_err(sa, "RxQ drop disable is not supported"); | |
864 | rc = EINVAL; | |
865 | } | |
866 | ||
867 | return rc; | |
868 | } | |
869 | ||
870 | static unsigned int | |
871 | sfc_rx_mbuf_data_alignment(struct rte_mempool *mb_pool) | |
872 | { | |
873 | uint32_t data_off; | |
874 | uint32_t order; | |
875 | ||
876 | /* The mbuf object itself is always cache line aligned */ | |
877 | order = rte_bsf32(RTE_CACHE_LINE_SIZE); | |
878 | ||
879 | /* Data offset from mbuf object start */ | |
880 | data_off = sizeof(struct rte_mbuf) + rte_pktmbuf_priv_size(mb_pool) + | |
881 | RTE_PKTMBUF_HEADROOM; | |
882 | ||
883 | order = MIN(order, rte_bsf32(data_off)); | |
884 | ||
9f95a23c | 885 | return 1u << order; |
11fdf7f2 TL |
886 | } |
887 | ||
888 | static uint16_t | |
889 | sfc_rx_mb_pool_buf_size(struct sfc_adapter *sa, struct rte_mempool *mb_pool) | |
890 | { | |
891 | const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); | |
892 | const uint32_t nic_align_start = MAX(1, encp->enc_rx_buf_align_start); | |
893 | const uint32_t nic_align_end = MAX(1, encp->enc_rx_buf_align_end); | |
894 | uint16_t buf_size; | |
895 | unsigned int buf_aligned; | |
896 | unsigned int start_alignment; | |
897 | unsigned int end_padding_alignment; | |
898 | ||
899 | /* Below it is assumed that both alignments are power of 2 */ | |
900 | SFC_ASSERT(rte_is_power_of_2(nic_align_start)); | |
901 | SFC_ASSERT(rte_is_power_of_2(nic_align_end)); | |
902 | ||
903 | /* | |
904 | * mbuf is always cache line aligned, double-check | |
905 | * that it meets rx buffer start alignment requirements. | |
906 | */ | |
907 | ||
908 | /* Start from mbuf pool data room size */ | |
909 | buf_size = rte_pktmbuf_data_room_size(mb_pool); | |
910 | ||
911 | /* Remove headroom */ | |
912 | if (buf_size <= RTE_PKTMBUF_HEADROOM) { | |
913 | sfc_err(sa, | |
914 | "RxQ mbuf pool %s object data room size %u is smaller than headroom %u", | |
915 | mb_pool->name, buf_size, RTE_PKTMBUF_HEADROOM); | |
916 | return 0; | |
917 | } | |
918 | buf_size -= RTE_PKTMBUF_HEADROOM; | |
919 | ||
920 | /* Calculate guaranteed data start alignment */ | |
921 | buf_aligned = sfc_rx_mbuf_data_alignment(mb_pool); | |
922 | ||
923 | /* Reserve space for start alignment */ | |
924 | if (buf_aligned < nic_align_start) { | |
925 | start_alignment = nic_align_start - buf_aligned; | |
926 | if (buf_size <= start_alignment) { | |
927 | sfc_err(sa, | |
928 | "RxQ mbuf pool %s object data room size %u is insufficient for headroom %u and buffer start alignment %u required by NIC", | |
929 | mb_pool->name, | |
930 | rte_pktmbuf_data_room_size(mb_pool), | |
931 | RTE_PKTMBUF_HEADROOM, start_alignment); | |
932 | return 0; | |
933 | } | |
934 | buf_aligned = nic_align_start; | |
935 | buf_size -= start_alignment; | |
936 | } else { | |
937 | start_alignment = 0; | |
938 | } | |
939 | ||
940 | /* Make sure that end padding does not write beyond the buffer */ | |
941 | if (buf_aligned < nic_align_end) { | |
942 | /* | |
943 | * Estimate space which can be lost. If guarnteed buffer | |
944 | * size is odd, lost space is (nic_align_end - 1). More | |
945 | * accurate formula is below. | |
946 | */ | |
947 | end_padding_alignment = nic_align_end - | |
948 | MIN(buf_aligned, 1u << (rte_bsf32(buf_size) - 1)); | |
949 | if (buf_size <= end_padding_alignment) { | |
950 | sfc_err(sa, | |
951 | "RxQ mbuf pool %s object data room size %u is insufficient for headroom %u, buffer start alignment %u and end padding alignment %u required by NIC", | |
952 | mb_pool->name, | |
953 | rte_pktmbuf_data_room_size(mb_pool), | |
954 | RTE_PKTMBUF_HEADROOM, start_alignment, | |
955 | end_padding_alignment); | |
956 | return 0; | |
957 | } | |
958 | buf_size -= end_padding_alignment; | |
959 | } else { | |
960 | /* | |
961 | * Start is aligned the same or better than end, | |
962 | * just align length. | |
963 | */ | |
964 | buf_size = P2ALIGN(buf_size, nic_align_end); | |
965 | } | |
966 | ||
967 | return buf_size; | |
968 | } | |
969 | ||
970 | int | |
971 | sfc_rx_qinit(struct sfc_adapter *sa, unsigned int sw_index, | |
972 | uint16_t nb_rx_desc, unsigned int socket_id, | |
973 | const struct rte_eth_rxconf *rx_conf, | |
974 | struct rte_mempool *mb_pool) | |
975 | { | |
976 | const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); | |
9f95a23c | 977 | struct sfc_rss *rss = &sfc_sa2shared(sa)->rss; |
11fdf7f2 | 978 | int rc; |
9f95a23c TL |
979 | unsigned int rxq_entries; |
980 | unsigned int evq_entries; | |
981 | unsigned int rxq_max_fill_level; | |
982 | uint64_t offloads; | |
11fdf7f2 TL |
983 | uint16_t buf_size; |
984 | struct sfc_rxq_info *rxq_info; | |
985 | struct sfc_evq *evq; | |
986 | struct sfc_rxq *rxq; | |
987 | struct sfc_dp_rx_qcreate_info info; | |
9f95a23c TL |
988 | struct sfc_dp_rx_hw_limits hw_limits; |
989 | uint16_t rx_free_thresh; | |
990 | const char *error; | |
991 | ||
992 | memset(&hw_limits, 0, sizeof(hw_limits)); | |
993 | hw_limits.rxq_max_entries = sa->rxq_max_entries; | |
994 | hw_limits.rxq_min_entries = sa->rxq_min_entries; | |
995 | hw_limits.evq_max_entries = sa->evq_max_entries; | |
996 | hw_limits.evq_min_entries = sa->evq_min_entries; | |
997 | ||
998 | rc = sa->priv.dp_rx->qsize_up_rings(nb_rx_desc, &hw_limits, mb_pool, | |
999 | &rxq_entries, &evq_entries, | |
1000 | &rxq_max_fill_level); | |
1001 | if (rc != 0) | |
1002 | goto fail_size_up_rings; | |
1003 | SFC_ASSERT(rxq_entries >= sa->rxq_min_entries); | |
1004 | SFC_ASSERT(rxq_entries <= sa->rxq_max_entries); | |
1005 | SFC_ASSERT(rxq_max_fill_level <= nb_rx_desc); | |
1006 | ||
1007 | offloads = rx_conf->offloads | | |
1008 | sa->eth_dev->data->dev_conf.rxmode.offloads; | |
1009 | rc = sfc_rx_qcheck_conf(sa, rxq_max_fill_level, rx_conf, offloads); | |
11fdf7f2 TL |
1010 | if (rc != 0) |
1011 | goto fail_bad_conf; | |
1012 | ||
1013 | buf_size = sfc_rx_mb_pool_buf_size(sa, mb_pool); | |
1014 | if (buf_size == 0) { | |
1015 | sfc_err(sa, "RxQ %u mbuf pool object size is too small", | |
1016 | sw_index); | |
1017 | rc = EINVAL; | |
1018 | goto fail_bad_conf; | |
1019 | } | |
1020 | ||
9f95a23c TL |
1021 | if (!sfc_rx_check_scatter(sa->port.pdu, buf_size, |
1022 | encp->enc_rx_prefix_size, | |
1023 | (offloads & DEV_RX_OFFLOAD_SCATTER), | |
1024 | &error)) { | |
1025 | sfc_err(sa, "RxQ %u MTU check failed: %s", sw_index, error); | |
11fdf7f2 TL |
1026 | sfc_err(sa, "RxQ %u calculated Rx buffer size is %u vs " |
1027 | "PDU size %u plus Rx prefix %u bytes", | |
1028 | sw_index, buf_size, (unsigned int)sa->port.pdu, | |
1029 | encp->enc_rx_prefix_size); | |
1030 | rc = EINVAL; | |
1031 | goto fail_bad_conf; | |
1032 | } | |
1033 | ||
9f95a23c TL |
1034 | SFC_ASSERT(sw_index < sfc_sa2shared(sa)->rxq_count); |
1035 | rxq_info = &sfc_sa2shared(sa)->rxq_info[sw_index]; | |
1036 | ||
1037 | SFC_ASSERT(rxq_entries <= rxq_info->max_entries); | |
1038 | rxq_info->entries = rxq_entries; | |
11fdf7f2 | 1039 | |
9f95a23c TL |
1040 | if (sa->priv.dp_rx->dp.hw_fw_caps & SFC_DP_HW_FW_CAP_RX_ES_SUPER_BUFFER) |
1041 | rxq_info->type = EFX_RXQ_TYPE_ES_SUPER_BUFFER; | |
1042 | else | |
1043 | rxq_info->type = EFX_RXQ_TYPE_DEFAULT; | |
1044 | ||
1045 | rxq_info->type_flags = | |
1046 | (offloads & DEV_RX_OFFLOAD_SCATTER) ? | |
1047 | EFX_RXQ_FLAG_SCATTER : EFX_RXQ_FLAG_NONE; | |
1048 | ||
1049 | if ((encp->enc_tunnel_encapsulations_supported != 0) && | |
1050 | (sa->priv.dp_rx->features & SFC_DP_RX_FEAT_TUNNELS)) | |
1051 | rxq_info->type_flags |= EFX_RXQ_FLAG_INNER_CLASSES; | |
11fdf7f2 TL |
1052 | |
1053 | rc = sfc_ev_qinit(sa, SFC_EVQ_TYPE_RX, sw_index, | |
9f95a23c | 1054 | evq_entries, socket_id, &evq); |
11fdf7f2 TL |
1055 | if (rc != 0) |
1056 | goto fail_ev_qinit; | |
1057 | ||
9f95a23c | 1058 | rxq = &sa->rxq_ctrl[sw_index]; |
11fdf7f2 TL |
1059 | rxq->evq = evq; |
1060 | rxq->hw_index = sw_index; | |
9f95a23c TL |
1061 | /* |
1062 | * If Rx refill threshold is specified (its value is non zero) in | |
1063 | * Rx configuration, use specified value. Otherwise use 1/8 of | |
1064 | * the Rx descriptors number as the default. It allows to keep | |
1065 | * Rx ring full-enough and does not refill too aggressive if | |
1066 | * packet rate is high. | |
1067 | * | |
1068 | * Since PMD refills in bulks waiting for full bulk may be | |
1069 | * refilled (basically round down), it is better to round up | |
1070 | * here to mitigate it a bit. | |
1071 | */ | |
1072 | rx_free_thresh = (rx_conf->rx_free_thresh != 0) ? | |
1073 | rx_conf->rx_free_thresh : EFX_DIV_ROUND_UP(nb_rx_desc, 8); | |
1074 | /* Rx refill threshold cannot be smaller than refill bulk */ | |
1075 | rxq_info->refill_threshold = | |
1076 | RTE_MAX(rx_free_thresh, SFC_RX_REFILL_BULK); | |
1077 | rxq_info->refill_mb_pool = mb_pool; | |
1078 | rxq->buf_size = buf_size; | |
1079 | ||
1080 | rc = sfc_dma_alloc(sa, "rxq", sw_index, | |
1081 | efx_rxq_size(sa->nic, rxq_info->entries), | |
11fdf7f2 TL |
1082 | socket_id, &rxq->mem); |
1083 | if (rc != 0) | |
1084 | goto fail_dma_alloc; | |
1085 | ||
1086 | memset(&info, 0, sizeof(info)); | |
9f95a23c TL |
1087 | info.refill_mb_pool = rxq_info->refill_mb_pool; |
1088 | info.max_fill_level = rxq_max_fill_level; | |
1089 | info.refill_threshold = rxq_info->refill_threshold; | |
11fdf7f2 TL |
1090 | info.buf_size = buf_size; |
1091 | info.batch_max = encp->enc_rx_batch_max; | |
1092 | info.prefix_size = encp->enc_rx_prefix_size; | |
1093 | ||
9f95a23c | 1094 | if (rss->hash_support == EFX_RX_HASH_AVAILABLE && rss->channels > 0) |
11fdf7f2 | 1095 | info.flags |= SFC_RXQ_FLAG_RSS_HASH; |
11fdf7f2 TL |
1096 | |
1097 | info.rxq_entries = rxq_info->entries; | |
1098 | info.rxq_hw_ring = rxq->mem.esm_base; | |
9f95a23c | 1099 | info.evq_entries = evq_entries; |
11fdf7f2 TL |
1100 | info.evq_hw_ring = evq->mem.esm_base; |
1101 | info.hw_index = rxq->hw_index; | |
1102 | info.mem_bar = sa->mem_bar.esb_base; | |
9f95a23c | 1103 | info.vi_window_shift = encp->enc_vi_window_shift; |
11fdf7f2 | 1104 | |
9f95a23c TL |
1105 | rc = sa->priv.dp_rx->qcreate(sa->eth_dev->data->port_id, sw_index, |
1106 | &RTE_ETH_DEV_TO_PCI(sa->eth_dev)->addr, | |
1107 | socket_id, &info, &rxq_info->dp); | |
11fdf7f2 TL |
1108 | if (rc != 0) |
1109 | goto fail_dp_rx_qcreate; | |
1110 | ||
9f95a23c | 1111 | evq->dp_rxq = rxq_info->dp; |
11fdf7f2 | 1112 | |
9f95a23c | 1113 | rxq_info->state = SFC_RXQ_INITIALIZED; |
11fdf7f2 TL |
1114 | |
1115 | rxq_info->deferred_start = (rx_conf->rx_deferred_start != 0); | |
1116 | ||
1117 | return 0; | |
1118 | ||
1119 | fail_dp_rx_qcreate: | |
1120 | sfc_dma_free(sa, &rxq->mem); | |
1121 | ||
1122 | fail_dma_alloc: | |
11fdf7f2 TL |
1123 | sfc_ev_qfini(evq); |
1124 | ||
1125 | fail_ev_qinit: | |
1126 | rxq_info->entries = 0; | |
1127 | ||
1128 | fail_bad_conf: | |
9f95a23c | 1129 | fail_size_up_rings: |
11fdf7f2 TL |
1130 | sfc_log_init(sa, "failed %d", rc); |
1131 | return rc; | |
1132 | } | |
1133 | ||
1134 | void | |
1135 | sfc_rx_qfini(struct sfc_adapter *sa, unsigned int sw_index) | |
1136 | { | |
1137 | struct sfc_rxq_info *rxq_info; | |
1138 | struct sfc_rxq *rxq; | |
1139 | ||
9f95a23c TL |
1140 | SFC_ASSERT(sw_index < sfc_sa2shared(sa)->rxq_count); |
1141 | sa->eth_dev->data->rx_queues[sw_index] = NULL; | |
11fdf7f2 | 1142 | |
9f95a23c | 1143 | rxq_info = &sfc_sa2shared(sa)->rxq_info[sw_index]; |
11fdf7f2 | 1144 | |
9f95a23c | 1145 | SFC_ASSERT(rxq_info->state == SFC_RXQ_INITIALIZED); |
11fdf7f2 | 1146 | |
9f95a23c TL |
1147 | sa->priv.dp_rx->qdestroy(rxq_info->dp); |
1148 | rxq_info->dp = NULL; | |
11fdf7f2 | 1149 | |
9f95a23c | 1150 | rxq_info->state &= ~SFC_RXQ_INITIALIZED; |
11fdf7f2 TL |
1151 | rxq_info->entries = 0; |
1152 | ||
9f95a23c TL |
1153 | rxq = &sa->rxq_ctrl[sw_index]; |
1154 | ||
11fdf7f2 TL |
1155 | sfc_dma_free(sa, &rxq->mem); |
1156 | ||
1157 | sfc_ev_qfini(rxq->evq); | |
1158 | rxq->evq = NULL; | |
9f95a23c | 1159 | } |
11fdf7f2 | 1160 | |
9f95a23c TL |
1161 | /* |
1162 | * Mapping between RTE RSS hash functions and their EFX counterparts. | |
1163 | */ | |
1164 | static const struct sfc_rss_hf_rte_to_efx sfc_rss_hf_map[] = { | |
1165 | { ETH_RSS_NONFRAG_IPV4_TCP, | |
1166 | EFX_RX_HASH(IPV4_TCP, 4TUPLE) }, | |
1167 | { ETH_RSS_NONFRAG_IPV4_UDP, | |
1168 | EFX_RX_HASH(IPV4_UDP, 4TUPLE) }, | |
1169 | { ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX, | |
1170 | EFX_RX_HASH(IPV6_TCP, 4TUPLE) }, | |
1171 | { ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_IPV6_UDP_EX, | |
1172 | EFX_RX_HASH(IPV6_UDP, 4TUPLE) }, | |
1173 | { ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | ETH_RSS_NONFRAG_IPV4_OTHER, | |
1174 | EFX_RX_HASH(IPV4_TCP, 2TUPLE) | EFX_RX_HASH(IPV4_UDP, 2TUPLE) | | |
1175 | EFX_RX_HASH(IPV4, 2TUPLE) }, | |
1176 | { ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER | | |
1177 | ETH_RSS_IPV6_EX, | |
1178 | EFX_RX_HASH(IPV6_TCP, 2TUPLE) | EFX_RX_HASH(IPV6_UDP, 2TUPLE) | | |
1179 | EFX_RX_HASH(IPV6, 2TUPLE) } | |
1180 | }; | |
1181 | ||
1182 | static efx_rx_hash_type_t | |
1183 | sfc_rx_hash_types_mask_supp(efx_rx_hash_type_t hash_type, | |
1184 | unsigned int *hash_type_flags_supported, | |
1185 | unsigned int nb_hash_type_flags_supported) | |
1186 | { | |
1187 | efx_rx_hash_type_t hash_type_masked = 0; | |
1188 | unsigned int i, j; | |
1189 | ||
1190 | for (i = 0; i < nb_hash_type_flags_supported; ++i) { | |
1191 | unsigned int class_tuple_lbn[] = { | |
1192 | EFX_RX_CLASS_IPV4_TCP_LBN, | |
1193 | EFX_RX_CLASS_IPV4_UDP_LBN, | |
1194 | EFX_RX_CLASS_IPV4_LBN, | |
1195 | EFX_RX_CLASS_IPV6_TCP_LBN, | |
1196 | EFX_RX_CLASS_IPV6_UDP_LBN, | |
1197 | EFX_RX_CLASS_IPV6_LBN | |
1198 | }; | |
1199 | ||
1200 | for (j = 0; j < RTE_DIM(class_tuple_lbn); ++j) { | |
1201 | unsigned int tuple_mask = EFX_RX_CLASS_HASH_4TUPLE; | |
1202 | unsigned int flag; | |
1203 | ||
1204 | tuple_mask <<= class_tuple_lbn[j]; | |
1205 | flag = hash_type & tuple_mask; | |
1206 | ||
1207 | if (flag == hash_type_flags_supported[i]) | |
1208 | hash_type_masked |= flag; | |
1209 | } | |
1210 | } | |
1211 | ||
1212 | return hash_type_masked; | |
1213 | } | |
1214 | ||
1215 | int | |
1216 | sfc_rx_hash_init(struct sfc_adapter *sa) | |
1217 | { | |
1218 | struct sfc_rss *rss = &sfc_sa2shared(sa)->rss; | |
1219 | const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); | |
1220 | uint32_t alg_mask = encp->enc_rx_scale_hash_alg_mask; | |
1221 | efx_rx_hash_alg_t alg; | |
1222 | unsigned int flags_supp[EFX_RX_HASH_NFLAGS]; | |
1223 | unsigned int nb_flags_supp; | |
1224 | struct sfc_rss_hf_rte_to_efx *hf_map; | |
1225 | struct sfc_rss_hf_rte_to_efx *entry; | |
1226 | efx_rx_hash_type_t efx_hash_types; | |
1227 | unsigned int i; | |
1228 | int rc; | |
1229 | ||
1230 | if (alg_mask & (1U << EFX_RX_HASHALG_TOEPLITZ)) | |
1231 | alg = EFX_RX_HASHALG_TOEPLITZ; | |
1232 | else if (alg_mask & (1U << EFX_RX_HASHALG_PACKED_STREAM)) | |
1233 | alg = EFX_RX_HASHALG_PACKED_STREAM; | |
1234 | else | |
1235 | return EINVAL; | |
1236 | ||
1237 | rc = efx_rx_scale_hash_flags_get(sa->nic, alg, flags_supp, | |
1238 | RTE_DIM(flags_supp), &nb_flags_supp); | |
1239 | if (rc != 0) | |
1240 | return rc; | |
1241 | ||
1242 | hf_map = rte_calloc_socket("sfc-rss-hf-map", | |
1243 | RTE_DIM(sfc_rss_hf_map), | |
1244 | sizeof(*hf_map), 0, sa->socket_id); | |
1245 | if (hf_map == NULL) | |
1246 | return ENOMEM; | |
1247 | ||
1248 | entry = hf_map; | |
1249 | efx_hash_types = 0; | |
1250 | for (i = 0; i < RTE_DIM(sfc_rss_hf_map); ++i) { | |
1251 | efx_rx_hash_type_t ht; | |
1252 | ||
1253 | ht = sfc_rx_hash_types_mask_supp(sfc_rss_hf_map[i].efx, | |
1254 | flags_supp, nb_flags_supp); | |
1255 | if (ht != 0) { | |
1256 | entry->rte = sfc_rss_hf_map[i].rte; | |
1257 | entry->efx = ht; | |
1258 | efx_hash_types |= ht; | |
1259 | ++entry; | |
1260 | } | |
1261 | } | |
1262 | ||
1263 | rss->hash_alg = alg; | |
1264 | rss->hf_map_nb_entries = (unsigned int)(entry - hf_map); | |
1265 | rss->hf_map = hf_map; | |
1266 | rss->hash_types = efx_hash_types; | |
1267 | ||
1268 | return 0; | |
11fdf7f2 TL |
1269 | } |
1270 | ||
9f95a23c TL |
1271 | void |
1272 | sfc_rx_hash_fini(struct sfc_adapter *sa) | |
1273 | { | |
1274 | struct sfc_rss *rss = &sfc_sa2shared(sa)->rss; | |
1275 | ||
1276 | rte_free(rss->hf_map); | |
1277 | } | |
1278 | ||
1279 | int | |
1280 | sfc_rx_hf_rte_to_efx(struct sfc_adapter *sa, uint64_t rte, | |
1281 | efx_rx_hash_type_t *efx) | |
11fdf7f2 | 1282 | { |
9f95a23c TL |
1283 | struct sfc_rss *rss = &sfc_sa2shared(sa)->rss; |
1284 | efx_rx_hash_type_t hash_types = 0; | |
1285 | unsigned int i; | |
11fdf7f2 | 1286 | |
9f95a23c TL |
1287 | for (i = 0; i < rss->hf_map_nb_entries; ++i) { |
1288 | uint64_t rte_mask = rss->hf_map[i].rte; | |
11fdf7f2 | 1289 | |
9f95a23c TL |
1290 | if ((rte & rte_mask) != 0) { |
1291 | rte &= ~rte_mask; | |
1292 | hash_types |= rss->hf_map[i].efx; | |
1293 | } | |
1294 | } | |
11fdf7f2 | 1295 | |
9f95a23c TL |
1296 | if (rte != 0) { |
1297 | sfc_err(sa, "unsupported hash functions requested"); | |
1298 | return EINVAL; | |
1299 | } | |
11fdf7f2 | 1300 | |
9f95a23c | 1301 | *efx = hash_types; |
11fdf7f2 | 1302 | |
9f95a23c | 1303 | return 0; |
11fdf7f2 TL |
1304 | } |
1305 | ||
1306 | uint64_t | |
9f95a23c | 1307 | sfc_rx_hf_efx_to_rte(struct sfc_rss *rss, efx_rx_hash_type_t efx) |
11fdf7f2 | 1308 | { |
9f95a23c TL |
1309 | uint64_t rte = 0; |
1310 | unsigned int i; | |
11fdf7f2 | 1311 | |
9f95a23c TL |
1312 | for (i = 0; i < rss->hf_map_nb_entries; ++i) { |
1313 | efx_rx_hash_type_t hash_type = rss->hf_map[i].efx; | |
11fdf7f2 | 1314 | |
9f95a23c TL |
1315 | if ((efx & hash_type) == hash_type) |
1316 | rte |= rss->hf_map[i].rte; | |
1317 | } | |
11fdf7f2 | 1318 | |
9f95a23c TL |
1319 | return rte; |
1320 | } | |
11fdf7f2 | 1321 | |
9f95a23c TL |
1322 | static int |
1323 | sfc_rx_process_adv_conf_rss(struct sfc_adapter *sa, | |
1324 | struct rte_eth_rss_conf *conf) | |
1325 | { | |
1326 | struct sfc_rss *rss = &sfc_sa2shared(sa)->rss; | |
1327 | efx_rx_hash_type_t efx_hash_types = rss->hash_types; | |
1328 | uint64_t rss_hf = sfc_rx_hf_efx_to_rte(rss, efx_hash_types); | |
1329 | int rc; | |
1330 | ||
1331 | if (rss->context_type != EFX_RX_SCALE_EXCLUSIVE) { | |
1332 | if ((conf->rss_hf != 0 && conf->rss_hf != rss_hf) || | |
1333 | conf->rss_key != NULL) | |
1334 | return EINVAL; | |
1335 | } | |
11fdf7f2 | 1336 | |
9f95a23c TL |
1337 | if (conf->rss_hf != 0) { |
1338 | rc = sfc_rx_hf_rte_to_efx(sa, conf->rss_hf, &efx_hash_types); | |
1339 | if (rc != 0) | |
1340 | return rc; | |
1341 | } | |
1342 | ||
1343 | if (conf->rss_key != NULL) { | |
1344 | if (conf->rss_key_len != sizeof(rss->key)) { | |
1345 | sfc_err(sa, "RSS key size is wrong (should be %lu)", | |
1346 | sizeof(rss->key)); | |
1347 | return EINVAL; | |
1348 | } | |
1349 | rte_memcpy(rss->key, conf->rss_key, sizeof(rss->key)); | |
1350 | } | |
1351 | ||
1352 | rss->hash_types = efx_hash_types; | |
1353 | ||
1354 | return 0; | |
11fdf7f2 | 1355 | } |
11fdf7f2 TL |
1356 | |
1357 | static int | |
1358 | sfc_rx_rss_config(struct sfc_adapter *sa) | |
1359 | { | |
9f95a23c | 1360 | struct sfc_rss *rss = &sfc_sa2shared(sa)->rss; |
11fdf7f2 TL |
1361 | int rc = 0; |
1362 | ||
9f95a23c TL |
1363 | if (rss->channels > 0) { |
1364 | rc = efx_rx_scale_mode_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT, | |
1365 | rss->hash_alg, rss->hash_types, | |
1366 | B_TRUE); | |
11fdf7f2 TL |
1367 | if (rc != 0) |
1368 | goto finish; | |
1369 | ||
9f95a23c TL |
1370 | rc = efx_rx_scale_key_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT, |
1371 | rss->key, sizeof(rss->key)); | |
11fdf7f2 TL |
1372 | if (rc != 0) |
1373 | goto finish; | |
1374 | ||
9f95a23c TL |
1375 | rc = efx_rx_scale_tbl_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT, |
1376 | rss->tbl, RTE_DIM(rss->tbl)); | |
11fdf7f2 TL |
1377 | } |
1378 | ||
1379 | finish: | |
11fdf7f2 TL |
1380 | return rc; |
1381 | } | |
1382 | ||
1383 | int | |
1384 | sfc_rx_start(struct sfc_adapter *sa) | |
1385 | { | |
9f95a23c | 1386 | struct sfc_adapter_shared * const sas = sfc_sa2shared(sa); |
11fdf7f2 TL |
1387 | unsigned int sw_index; |
1388 | int rc; | |
1389 | ||
9f95a23c | 1390 | sfc_log_init(sa, "rxq_count=%u", sas->rxq_count); |
11fdf7f2 TL |
1391 | |
1392 | rc = efx_rx_init(sa->nic); | |
1393 | if (rc != 0) | |
1394 | goto fail_rx_init; | |
1395 | ||
1396 | rc = sfc_rx_rss_config(sa); | |
1397 | if (rc != 0) | |
1398 | goto fail_rss_config; | |
1399 | ||
9f95a23c TL |
1400 | for (sw_index = 0; sw_index < sas->rxq_count; ++sw_index) { |
1401 | if (sas->rxq_info[sw_index].state == SFC_RXQ_INITIALIZED && | |
1402 | (!sas->rxq_info[sw_index].deferred_start || | |
1403 | sas->rxq_info[sw_index].deferred_started)) { | |
11fdf7f2 TL |
1404 | rc = sfc_rx_qstart(sa, sw_index); |
1405 | if (rc != 0) | |
1406 | goto fail_rx_qstart; | |
1407 | } | |
1408 | } | |
1409 | ||
1410 | return 0; | |
1411 | ||
1412 | fail_rx_qstart: | |
1413 | while (sw_index-- > 0) | |
1414 | sfc_rx_qstop(sa, sw_index); | |
1415 | ||
1416 | fail_rss_config: | |
1417 | efx_rx_fini(sa->nic); | |
1418 | ||
1419 | fail_rx_init: | |
1420 | sfc_log_init(sa, "failed %d", rc); | |
1421 | return rc; | |
1422 | } | |
1423 | ||
1424 | void | |
1425 | sfc_rx_stop(struct sfc_adapter *sa) | |
1426 | { | |
9f95a23c | 1427 | struct sfc_adapter_shared * const sas = sfc_sa2shared(sa); |
11fdf7f2 TL |
1428 | unsigned int sw_index; |
1429 | ||
9f95a23c | 1430 | sfc_log_init(sa, "rxq_count=%u", sas->rxq_count); |
11fdf7f2 | 1431 | |
9f95a23c | 1432 | sw_index = sas->rxq_count; |
11fdf7f2 | 1433 | while (sw_index-- > 0) { |
9f95a23c | 1434 | if (sas->rxq_info[sw_index].state & SFC_RXQ_STARTED) |
11fdf7f2 TL |
1435 | sfc_rx_qstop(sa, sw_index); |
1436 | } | |
1437 | ||
1438 | efx_rx_fini(sa->nic); | |
1439 | } | |
1440 | ||
1441 | static int | |
1442 | sfc_rx_qinit_info(struct sfc_adapter *sa, unsigned int sw_index) | |
1443 | { | |
9f95a23c TL |
1444 | struct sfc_adapter_shared * const sas = sfc_sa2shared(sa); |
1445 | struct sfc_rxq_info *rxq_info = &sas->rxq_info[sw_index]; | |
1446 | const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); | |
11fdf7f2 TL |
1447 | unsigned int max_entries; |
1448 | ||
9f95a23c | 1449 | max_entries = encp->enc_rxq_max_ndescs; |
11fdf7f2 TL |
1450 | SFC_ASSERT(rte_is_power_of_2(max_entries)); |
1451 | ||
1452 | rxq_info->max_entries = max_entries; | |
1453 | ||
1454 | return 0; | |
1455 | } | |
1456 | ||
1457 | static int | |
1458 | sfc_rx_check_mode(struct sfc_adapter *sa, struct rte_eth_rxmode *rxmode) | |
1459 | { | |
9f95a23c TL |
1460 | struct sfc_adapter_shared * const sas = sfc_sa2shared(sa); |
1461 | uint64_t offloads_supported = sfc_rx_get_dev_offload_caps(sa) | | |
1462 | sfc_rx_get_queue_offload_caps(sa); | |
1463 | struct sfc_rss *rss = &sas->rss; | |
11fdf7f2 TL |
1464 | int rc = 0; |
1465 | ||
1466 | switch (rxmode->mq_mode) { | |
1467 | case ETH_MQ_RX_NONE: | |
1468 | /* No special checks are required */ | |
1469 | break; | |
11fdf7f2 | 1470 | case ETH_MQ_RX_RSS: |
9f95a23c | 1471 | if (rss->context_type == EFX_RX_SCALE_UNAVAILABLE) { |
11fdf7f2 TL |
1472 | sfc_err(sa, "RSS is not available"); |
1473 | rc = EINVAL; | |
1474 | } | |
1475 | break; | |
11fdf7f2 TL |
1476 | default: |
1477 | sfc_err(sa, "Rx multi-queue mode %u not supported", | |
1478 | rxmode->mq_mode); | |
1479 | rc = EINVAL; | |
1480 | } | |
1481 | ||
9f95a23c TL |
1482 | /* |
1483 | * Requested offloads are validated against supported by ethdev, | |
1484 | * so unsupported offloads cannot be added as the result of | |
1485 | * below check. | |
1486 | */ | |
1487 | if ((rxmode->offloads & DEV_RX_OFFLOAD_CHECKSUM) != | |
1488 | (offloads_supported & DEV_RX_OFFLOAD_CHECKSUM)) { | |
1489 | sfc_warn(sa, "Rx checksum offloads cannot be disabled - always on (IPv4/TCP/UDP)"); | |
1490 | rxmode->offloads |= DEV_RX_OFFLOAD_CHECKSUM; | |
11fdf7f2 TL |
1491 | } |
1492 | ||
9f95a23c TL |
1493 | if ((offloads_supported & DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM) && |
1494 | (~rxmode->offloads & DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM)) { | |
1495 | sfc_warn(sa, "Rx outer IPv4 checksum offload cannot be disabled - always on"); | |
1496 | rxmode->offloads |= DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM; | |
11fdf7f2 TL |
1497 | } |
1498 | ||
1499 | return rc; | |
1500 | } | |
1501 | ||
1502 | /** | |
1503 | * Destroy excess queues that are no longer needed after reconfiguration | |
1504 | * or complete close. | |
1505 | */ | |
1506 | static void | |
1507 | sfc_rx_fini_queues(struct sfc_adapter *sa, unsigned int nb_rx_queues) | |
1508 | { | |
9f95a23c | 1509 | struct sfc_adapter_shared * const sas = sfc_sa2shared(sa); |
11fdf7f2 TL |
1510 | int sw_index; |
1511 | ||
9f95a23c | 1512 | SFC_ASSERT(nb_rx_queues <= sas->rxq_count); |
11fdf7f2 | 1513 | |
9f95a23c | 1514 | sw_index = sas->rxq_count; |
11fdf7f2 | 1515 | while (--sw_index >= (int)nb_rx_queues) { |
9f95a23c | 1516 | if (sas->rxq_info[sw_index].state & SFC_RXQ_INITIALIZED) |
11fdf7f2 TL |
1517 | sfc_rx_qfini(sa, sw_index); |
1518 | } | |
1519 | ||
9f95a23c | 1520 | sas->rxq_count = nb_rx_queues; |
11fdf7f2 TL |
1521 | } |
1522 | ||
1523 | /** | |
1524 | * Initialize Rx subsystem. | |
1525 | * | |
1526 | * Called at device (re)configuration stage when number of receive queues is | |
1527 | * specified together with other device level receive configuration. | |
1528 | * | |
1529 | * It should be used to allocate NUMA-unaware resources. | |
1530 | */ | |
1531 | int | |
1532 | sfc_rx_configure(struct sfc_adapter *sa) | |
1533 | { | |
9f95a23c TL |
1534 | struct sfc_adapter_shared * const sas = sfc_sa2shared(sa); |
1535 | struct sfc_rss *rss = &sas->rss; | |
11fdf7f2 TL |
1536 | struct rte_eth_conf *dev_conf = &sa->eth_dev->data->dev_conf; |
1537 | const unsigned int nb_rx_queues = sa->eth_dev->data->nb_rx_queues; | |
11fdf7f2 TL |
1538 | int rc; |
1539 | ||
1540 | sfc_log_init(sa, "nb_rx_queues=%u (old %u)", | |
9f95a23c | 1541 | nb_rx_queues, sas->rxq_count); |
11fdf7f2 TL |
1542 | |
1543 | rc = sfc_rx_check_mode(sa, &dev_conf->rxmode); | |
1544 | if (rc != 0) | |
1545 | goto fail_check_mode; | |
1546 | ||
9f95a23c TL |
1547 | if (nb_rx_queues == sas->rxq_count) |
1548 | goto configure_rss; | |
11fdf7f2 | 1549 | |
9f95a23c | 1550 | if (sas->rxq_info == NULL) { |
11fdf7f2 | 1551 | rc = ENOMEM; |
9f95a23c TL |
1552 | sas->rxq_info = rte_calloc_socket("sfc-rxqs", nb_rx_queues, |
1553 | sizeof(sas->rxq_info[0]), 0, | |
1554 | sa->socket_id); | |
1555 | if (sas->rxq_info == NULL) | |
11fdf7f2 | 1556 | goto fail_rxqs_alloc; |
9f95a23c TL |
1557 | |
1558 | /* | |
1559 | * Allocate primary process only RxQ control from heap | |
1560 | * since it should not be shared. | |
1561 | */ | |
1562 | rc = ENOMEM; | |
1563 | sa->rxq_ctrl = calloc(nb_rx_queues, sizeof(sa->rxq_ctrl[0])); | |
1564 | if (sa->rxq_ctrl == NULL) | |
1565 | goto fail_rxqs_ctrl_alloc; | |
11fdf7f2 TL |
1566 | } else { |
1567 | struct sfc_rxq_info *new_rxq_info; | |
9f95a23c | 1568 | struct sfc_rxq *new_rxq_ctrl; |
11fdf7f2 | 1569 | |
9f95a23c | 1570 | if (nb_rx_queues < sas->rxq_count) |
11fdf7f2 TL |
1571 | sfc_rx_fini_queues(sa, nb_rx_queues); |
1572 | ||
1573 | rc = ENOMEM; | |
1574 | new_rxq_info = | |
9f95a23c TL |
1575 | rte_realloc(sas->rxq_info, |
1576 | nb_rx_queues * sizeof(sas->rxq_info[0]), 0); | |
11fdf7f2 TL |
1577 | if (new_rxq_info == NULL && nb_rx_queues > 0) |
1578 | goto fail_rxqs_realloc; | |
1579 | ||
9f95a23c TL |
1580 | rc = ENOMEM; |
1581 | new_rxq_ctrl = realloc(sa->rxq_ctrl, | |
1582 | nb_rx_queues * sizeof(sa->rxq_ctrl[0])); | |
1583 | if (new_rxq_ctrl == NULL && nb_rx_queues > 0) | |
1584 | goto fail_rxqs_ctrl_realloc; | |
1585 | ||
1586 | sas->rxq_info = new_rxq_info; | |
1587 | sa->rxq_ctrl = new_rxq_ctrl; | |
1588 | if (nb_rx_queues > sas->rxq_count) { | |
1589 | memset(&sas->rxq_info[sas->rxq_count], 0, | |
1590 | (nb_rx_queues - sas->rxq_count) * | |
1591 | sizeof(sas->rxq_info[0])); | |
1592 | memset(&sa->rxq_ctrl[sas->rxq_count], 0, | |
1593 | (nb_rx_queues - sas->rxq_count) * | |
1594 | sizeof(sa->rxq_ctrl[0])); | |
1595 | } | |
11fdf7f2 TL |
1596 | } |
1597 | ||
9f95a23c TL |
1598 | while (sas->rxq_count < nb_rx_queues) { |
1599 | rc = sfc_rx_qinit_info(sa, sas->rxq_count); | |
11fdf7f2 TL |
1600 | if (rc != 0) |
1601 | goto fail_rx_qinit_info; | |
1602 | ||
9f95a23c | 1603 | sas->rxq_count++; |
11fdf7f2 TL |
1604 | } |
1605 | ||
9f95a23c TL |
1606 | configure_rss: |
1607 | rss->channels = (dev_conf->rxmode.mq_mode == ETH_MQ_RX_RSS) ? | |
1608 | MIN(sas->rxq_count, EFX_MAXRSS) : 0; | |
1609 | ||
1610 | if (rss->channels > 0) { | |
1611 | struct rte_eth_rss_conf *adv_conf_rss; | |
1612 | unsigned int sw_index; | |
11fdf7f2 | 1613 | |
11fdf7f2 | 1614 | for (sw_index = 0; sw_index < EFX_RSS_TBL_SIZE; ++sw_index) |
9f95a23c TL |
1615 | rss->tbl[sw_index] = sw_index % rss->channels; |
1616 | ||
1617 | adv_conf_rss = &dev_conf->rx_adv_conf.rss_conf; | |
1618 | rc = sfc_rx_process_adv_conf_rss(sa, adv_conf_rss); | |
1619 | if (rc != 0) | |
1620 | goto fail_rx_process_adv_conf_rss; | |
11fdf7f2 | 1621 | } |
11fdf7f2 | 1622 | |
11fdf7f2 TL |
1623 | return 0; |
1624 | ||
9f95a23c | 1625 | fail_rx_process_adv_conf_rss: |
11fdf7f2 | 1626 | fail_rx_qinit_info: |
9f95a23c | 1627 | fail_rxqs_ctrl_realloc: |
11fdf7f2 | 1628 | fail_rxqs_realloc: |
9f95a23c | 1629 | fail_rxqs_ctrl_alloc: |
11fdf7f2 TL |
1630 | fail_rxqs_alloc: |
1631 | sfc_rx_close(sa); | |
1632 | ||
1633 | fail_check_mode: | |
1634 | sfc_log_init(sa, "failed %d", rc); | |
1635 | return rc; | |
1636 | } | |
1637 | ||
1638 | /** | |
1639 | * Shutdown Rx subsystem. | |
1640 | * | |
1641 | * Called at device close stage, for example, before device shutdown. | |
1642 | */ | |
1643 | void | |
1644 | sfc_rx_close(struct sfc_adapter *sa) | |
1645 | { | |
9f95a23c TL |
1646 | struct sfc_rss *rss = &sfc_sa2shared(sa)->rss; |
1647 | ||
11fdf7f2 TL |
1648 | sfc_rx_fini_queues(sa, 0); |
1649 | ||
9f95a23c TL |
1650 | rss->channels = 0; |
1651 | ||
1652 | free(sa->rxq_ctrl); | |
1653 | sa->rxq_ctrl = NULL; | |
11fdf7f2 | 1654 | |
9f95a23c TL |
1655 | rte_free(sfc_sa2shared(sa)->rxq_info); |
1656 | sfc_sa2shared(sa)->rxq_info = NULL; | |
11fdf7f2 | 1657 | } |