]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | /* |
2 | * Copyright (c) 2016 QLogic Corporation. | |
3 | * All rights reserved. | |
4 | * www.qlogic.com | |
5 | * | |
6 | * See LICENSE.qede_pmd for copyright and licensing details. | |
7 | */ | |
8 | ||
9 | #include "qede_rxtx.h" | |
10 | ||
11 | static bool gro_disable = 1; /* mod_param */ | |
12 | ||
13 | static inline int qede_alloc_rx_buffer(struct qede_rx_queue *rxq) | |
14 | { | |
15 | struct rte_mbuf *new_mb = NULL; | |
16 | struct eth_rx_bd *rx_bd; | |
17 | dma_addr_t mapping; | |
18 | uint16_t idx = rxq->sw_rx_prod & NUM_RX_BDS(rxq); | |
19 | ||
20 | new_mb = rte_mbuf_raw_alloc(rxq->mb_pool); | |
21 | if (unlikely(!new_mb)) { | |
22 | PMD_RX_LOG(ERR, rxq, | |
23 | "Failed to allocate rx buffer " | |
24 | "sw_rx_prod %u sw_rx_cons %u mp entries %u free %u", | |
25 | idx, rxq->sw_rx_cons & NUM_RX_BDS(rxq), | |
26 | rte_mempool_avail_count(rxq->mb_pool), | |
27 | rte_mempool_in_use_count(rxq->mb_pool)); | |
28 | return -ENOMEM; | |
29 | } | |
30 | rxq->sw_rx_ring[idx].mbuf = new_mb; | |
31 | rxq->sw_rx_ring[idx].page_offset = 0; | |
32 | mapping = rte_mbuf_data_dma_addr_default(new_mb); | |
33 | /* Advance PROD and get BD pointer */ | |
34 | rx_bd = (struct eth_rx_bd *)ecore_chain_produce(&rxq->rx_bd_ring); | |
35 | rx_bd->addr.hi = rte_cpu_to_le_32(U64_HI(mapping)); | |
36 | rx_bd->addr.lo = rte_cpu_to_le_32(U64_LO(mapping)); | |
37 | rxq->sw_rx_prod++; | |
38 | return 0; | |
39 | } | |
40 | ||
41 | static void qede_rx_queue_release_mbufs(struct qede_rx_queue *rxq) | |
42 | { | |
43 | uint16_t i; | |
44 | ||
45 | if (rxq->sw_rx_ring != NULL) { | |
46 | for (i = 0; i < rxq->nb_rx_desc; i++) { | |
47 | if (rxq->sw_rx_ring[i].mbuf != NULL) { | |
48 | rte_pktmbuf_free(rxq->sw_rx_ring[i].mbuf); | |
49 | rxq->sw_rx_ring[i].mbuf = NULL; | |
50 | } | |
51 | } | |
52 | } | |
53 | } | |
54 | ||
55 | void qede_rx_queue_release(void *rx_queue) | |
56 | { | |
57 | struct qede_rx_queue *rxq = rx_queue; | |
58 | ||
59 | if (rxq != NULL) { | |
60 | qede_rx_queue_release_mbufs(rxq); | |
61 | rte_free(rxq->sw_rx_ring); | |
62 | rxq->sw_rx_ring = NULL; | |
63 | rte_free(rxq); | |
64 | rxq = NULL; | |
65 | } | |
66 | } | |
67 | ||
68 | static void qede_tx_queue_release_mbufs(struct qede_tx_queue *txq) | |
69 | { | |
70 | unsigned int i; | |
71 | ||
72 | PMD_TX_LOG(DEBUG, txq, "releasing %u mbufs\n", txq->nb_tx_desc); | |
73 | ||
74 | if (txq->sw_tx_ring) { | |
75 | for (i = 0; i < txq->nb_tx_desc; i++) { | |
76 | if (txq->sw_tx_ring[i].mbuf) { | |
77 | rte_pktmbuf_free(txq->sw_tx_ring[i].mbuf); | |
78 | txq->sw_tx_ring[i].mbuf = NULL; | |
79 | } | |
80 | } | |
81 | } | |
82 | } | |
83 | ||
84 | int | |
85 | qede_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, | |
86 | uint16_t nb_desc, unsigned int socket_id, | |
87 | const struct rte_eth_rxconf *rx_conf, | |
88 | struct rte_mempool *mp) | |
89 | { | |
90 | struct qede_dev *qdev = dev->data->dev_private; | |
91 | struct ecore_dev *edev = &qdev->edev; | |
92 | struct rte_eth_dev_data *eth_data = dev->data; | |
93 | struct qede_rx_queue *rxq; | |
94 | uint16_t pkt_len = (uint16_t)dev->data->dev_conf.rxmode.max_rx_pkt_len; | |
95 | size_t size; | |
96 | uint16_t data_size; | |
97 | int rc; | |
98 | int i; | |
99 | ||
100 | PMD_INIT_FUNC_TRACE(edev); | |
101 | ||
102 | /* Note: Ring size/align is controlled by struct rte_eth_desc_lim */ | |
103 | if (!rte_is_power_of_2(nb_desc)) { | |
104 | DP_ERR(edev, "Ring size %u is not power of 2\n", | |
105 | nb_desc); | |
106 | return -EINVAL; | |
107 | } | |
108 | ||
109 | /* Free memory prior to re-allocation if needed... */ | |
110 | if (dev->data->rx_queues[queue_idx] != NULL) { | |
111 | qede_rx_queue_release(dev->data->rx_queues[queue_idx]); | |
112 | dev->data->rx_queues[queue_idx] = NULL; | |
113 | } | |
114 | ||
115 | /* First allocate the rx queue data structure */ | |
116 | rxq = rte_zmalloc_socket("qede_rx_queue", sizeof(struct qede_rx_queue), | |
117 | RTE_CACHE_LINE_SIZE, socket_id); | |
118 | ||
119 | if (!rxq) { | |
120 | DP_ERR(edev, "Unable to allocate memory for rxq on socket %u", | |
121 | socket_id); | |
122 | return -ENOMEM; | |
123 | } | |
124 | ||
125 | rxq->qdev = qdev; | |
126 | rxq->mb_pool = mp; | |
127 | rxq->nb_rx_desc = nb_desc; | |
128 | rxq->queue_id = queue_idx; | |
129 | rxq->port_id = dev->data->port_id; | |
130 | ||
131 | /* Sanity check */ | |
132 | data_size = (uint16_t)rte_pktmbuf_data_room_size(mp) - | |
133 | RTE_PKTMBUF_HEADROOM; | |
134 | ||
135 | if (pkt_len > data_size && !dev->data->scattered_rx) { | |
136 | DP_ERR(edev, "MTU %u should not exceed dataroom %u\n", | |
137 | pkt_len, data_size); | |
138 | rte_free(rxq); | |
139 | return -EINVAL; | |
140 | } | |
141 | ||
142 | if (dev->data->scattered_rx) | |
143 | rxq->rx_buf_size = data_size; | |
144 | else | |
145 | rxq->rx_buf_size = pkt_len + QEDE_ETH_OVERHEAD; | |
146 | ||
147 | qdev->mtu = pkt_len; | |
148 | ||
149 | DP_INFO(edev, "MTU = %u ; RX buffer = %u\n", | |
150 | qdev->mtu, rxq->rx_buf_size); | |
151 | ||
152 | if (pkt_len > ETHER_MAX_LEN) { | |
153 | dev->data->dev_conf.rxmode.jumbo_frame = 1; | |
154 | DP_NOTICE(edev, false, "jumbo frame enabled\n"); | |
155 | } else { | |
156 | dev->data->dev_conf.rxmode.jumbo_frame = 0; | |
157 | } | |
158 | ||
159 | /* Allocate the parallel driver ring for Rx buffers */ | |
160 | size = sizeof(*rxq->sw_rx_ring) * rxq->nb_rx_desc; | |
161 | rxq->sw_rx_ring = rte_zmalloc_socket("sw_rx_ring", size, | |
162 | RTE_CACHE_LINE_SIZE, socket_id); | |
163 | if (!rxq->sw_rx_ring) { | |
164 | DP_NOTICE(edev, false, | |
165 | "Unable to alloc memory for sw_rx_ring on socket %u\n", | |
166 | socket_id); | |
167 | rte_free(rxq); | |
168 | rxq = NULL; | |
169 | return -ENOMEM; | |
170 | } | |
171 | ||
172 | /* Allocate FW Rx ring */ | |
173 | rc = qdev->ops->common->chain_alloc(edev, | |
174 | ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, | |
175 | ECORE_CHAIN_MODE_NEXT_PTR, | |
176 | ECORE_CHAIN_CNT_TYPE_U16, | |
177 | rxq->nb_rx_desc, | |
178 | sizeof(struct eth_rx_bd), | |
179 | &rxq->rx_bd_ring); | |
180 | ||
181 | if (rc != ECORE_SUCCESS) { | |
182 | DP_NOTICE(edev, false, | |
183 | "Unable to alloc memory for rxbd ring on socket %u\n", | |
184 | socket_id); | |
185 | rte_free(rxq->sw_rx_ring); | |
186 | rxq->sw_rx_ring = NULL; | |
187 | rte_free(rxq); | |
188 | rxq = NULL; | |
189 | return -ENOMEM; | |
190 | } | |
191 | ||
192 | /* Allocate FW completion ring */ | |
193 | rc = qdev->ops->common->chain_alloc(edev, | |
194 | ECORE_CHAIN_USE_TO_CONSUME, | |
195 | ECORE_CHAIN_MODE_PBL, | |
196 | ECORE_CHAIN_CNT_TYPE_U16, | |
197 | rxq->nb_rx_desc, | |
198 | sizeof(union eth_rx_cqe), | |
199 | &rxq->rx_comp_ring); | |
200 | ||
201 | if (rc != ECORE_SUCCESS) { | |
202 | DP_NOTICE(edev, false, | |
203 | "Unable to alloc memory for cqe ring on socket %u\n", | |
204 | socket_id); | |
205 | /* TBD: Freeing RX BD ring */ | |
206 | rte_free(rxq->sw_rx_ring); | |
207 | rxq->sw_rx_ring = NULL; | |
208 | rte_free(rxq); | |
209 | return -ENOMEM; | |
210 | } | |
211 | ||
212 | /* Allocate buffers for the Rx ring */ | |
213 | for (i = 0; i < rxq->nb_rx_desc; i++) { | |
214 | rc = qede_alloc_rx_buffer(rxq); | |
215 | if (rc) { | |
216 | DP_NOTICE(edev, false, | |
217 | "RX buffer allocation failed at idx=%d\n", i); | |
218 | goto err4; | |
219 | } | |
220 | } | |
221 | ||
222 | dev->data->rx_queues[queue_idx] = rxq; | |
223 | ||
224 | DP_INFO(edev, "rxq %d num_desc %u rx_buf_size=%u socket %u\n", | |
225 | queue_idx, nb_desc, qdev->mtu, socket_id); | |
226 | ||
227 | return 0; | |
228 | err4: | |
229 | qede_rx_queue_release(rxq); | |
230 | return -ENOMEM; | |
231 | } | |
232 | ||
233 | void qede_tx_queue_release(void *tx_queue) | |
234 | { | |
235 | struct qede_tx_queue *txq = tx_queue; | |
236 | ||
237 | if (txq != NULL) { | |
238 | qede_tx_queue_release_mbufs(txq); | |
239 | if (txq->sw_tx_ring) { | |
240 | rte_free(txq->sw_tx_ring); | |
241 | txq->sw_tx_ring = NULL; | |
242 | } | |
243 | rte_free(txq); | |
244 | } | |
245 | txq = NULL; | |
246 | } | |
247 | ||
248 | int | |
249 | qede_tx_queue_setup(struct rte_eth_dev *dev, | |
250 | uint16_t queue_idx, | |
251 | uint16_t nb_desc, | |
252 | unsigned int socket_id, | |
253 | const struct rte_eth_txconf *tx_conf) | |
254 | { | |
255 | struct qede_dev *qdev = dev->data->dev_private; | |
256 | struct ecore_dev *edev = &qdev->edev; | |
257 | struct qede_tx_queue *txq; | |
258 | int rc; | |
259 | ||
260 | PMD_INIT_FUNC_TRACE(edev); | |
261 | ||
262 | if (!rte_is_power_of_2(nb_desc)) { | |
263 | DP_ERR(edev, "Ring size %u is not power of 2\n", | |
264 | nb_desc); | |
265 | return -EINVAL; | |
266 | } | |
267 | ||
268 | /* Free memory prior to re-allocation if needed... */ | |
269 | if (dev->data->tx_queues[queue_idx] != NULL) { | |
270 | qede_tx_queue_release(dev->data->tx_queues[queue_idx]); | |
271 | dev->data->tx_queues[queue_idx] = NULL; | |
272 | } | |
273 | ||
274 | txq = rte_zmalloc_socket("qede_tx_queue", sizeof(struct qede_tx_queue), | |
275 | RTE_CACHE_LINE_SIZE, socket_id); | |
276 | ||
277 | if (txq == NULL) { | |
278 | DP_ERR(edev, | |
279 | "Unable to allocate memory for txq on socket %u", | |
280 | socket_id); | |
281 | return -ENOMEM; | |
282 | } | |
283 | ||
284 | txq->nb_tx_desc = nb_desc; | |
285 | txq->qdev = qdev; | |
286 | txq->port_id = dev->data->port_id; | |
287 | ||
288 | rc = qdev->ops->common->chain_alloc(edev, | |
289 | ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, | |
290 | ECORE_CHAIN_MODE_PBL, | |
291 | ECORE_CHAIN_CNT_TYPE_U16, | |
292 | txq->nb_tx_desc, | |
293 | sizeof(union eth_tx_bd_types), | |
294 | &txq->tx_pbl); | |
295 | if (rc != ECORE_SUCCESS) { | |
296 | DP_ERR(edev, | |
297 | "Unable to allocate memory for txbd ring on socket %u", | |
298 | socket_id); | |
299 | qede_tx_queue_release(txq); | |
300 | return -ENOMEM; | |
301 | } | |
302 | ||
303 | /* Allocate software ring */ | |
304 | txq->sw_tx_ring = rte_zmalloc_socket("txq->sw_tx_ring", | |
305 | (sizeof(struct qede_tx_entry) * | |
306 | txq->nb_tx_desc), | |
307 | RTE_CACHE_LINE_SIZE, socket_id); | |
308 | ||
309 | if (!txq->sw_tx_ring) { | |
310 | DP_ERR(edev, | |
311 | "Unable to allocate memory for txbd ring on socket %u", | |
312 | socket_id); | |
313 | qede_tx_queue_release(txq); | |
314 | return -ENOMEM; | |
315 | } | |
316 | ||
317 | txq->queue_id = queue_idx; | |
318 | ||
319 | txq->nb_tx_avail = txq->nb_tx_desc; | |
320 | ||
321 | txq->tx_free_thresh = | |
322 | tx_conf->tx_free_thresh ? tx_conf->tx_free_thresh : | |
323 | (txq->nb_tx_desc - QEDE_DEFAULT_TX_FREE_THRESH); | |
324 | ||
325 | dev->data->tx_queues[queue_idx] = txq; | |
326 | ||
327 | DP_INFO(edev, | |
328 | "txq %u num_desc %u tx_free_thresh %u socket %u\n", | |
329 | queue_idx, nb_desc, txq->tx_free_thresh, socket_id); | |
330 | ||
331 | return 0; | |
332 | } | |
333 | ||
334 | /* This function inits fp content and resets the SB, RXQ and TXQ arrays */ | |
335 | static void qede_init_fp(struct qede_dev *qdev) | |
336 | { | |
337 | struct qede_fastpath *fp; | |
338 | uint8_t i, rss_id, tc; | |
339 | int fp_rx = qdev->fp_num_rx, rxq = 0, txq = 0; | |
340 | ||
341 | memset((void *)qdev->fp_array, 0, (QEDE_QUEUE_CNT(qdev) * | |
342 | sizeof(*qdev->fp_array))); | |
343 | memset((void *)qdev->sb_array, 0, (QEDE_QUEUE_CNT(qdev) * | |
344 | sizeof(*qdev->sb_array))); | |
345 | for_each_queue(i) { | |
346 | fp = &qdev->fp_array[i]; | |
347 | if (fp_rx) { | |
348 | fp->type = QEDE_FASTPATH_RX; | |
349 | fp_rx--; | |
350 | } else{ | |
351 | fp->type = QEDE_FASTPATH_TX; | |
352 | } | |
353 | fp->qdev = qdev; | |
354 | fp->id = i; | |
355 | fp->sb_info = &qdev->sb_array[i]; | |
356 | snprintf(fp->name, sizeof(fp->name), "%s-fp-%d", "qdev", i); | |
357 | } | |
358 | ||
359 | qdev->gro_disable = gro_disable; | |
360 | } | |
361 | ||
362 | void qede_free_fp_arrays(struct qede_dev *qdev) | |
363 | { | |
364 | /* It asseumes qede_free_mem_load() is called before */ | |
365 | if (qdev->fp_array != NULL) { | |
366 | rte_free(qdev->fp_array); | |
367 | qdev->fp_array = NULL; | |
368 | } | |
369 | ||
370 | if (qdev->sb_array != NULL) { | |
371 | rte_free(qdev->sb_array); | |
372 | qdev->sb_array = NULL; | |
373 | } | |
374 | } | |
375 | ||
376 | int qede_alloc_fp_array(struct qede_dev *qdev) | |
377 | { | |
378 | struct qede_fastpath *fp; | |
379 | struct ecore_dev *edev = &qdev->edev; | |
380 | int i; | |
381 | ||
382 | qdev->fp_array = rte_calloc("fp", QEDE_QUEUE_CNT(qdev), | |
383 | sizeof(*qdev->fp_array), | |
384 | RTE_CACHE_LINE_SIZE); | |
385 | ||
386 | if (!qdev->fp_array) { | |
387 | DP_ERR(edev, "fp array allocation failed\n"); | |
388 | return -ENOMEM; | |
389 | } | |
390 | ||
391 | qdev->sb_array = rte_calloc("sb", QEDE_QUEUE_CNT(qdev), | |
392 | sizeof(*qdev->sb_array), | |
393 | RTE_CACHE_LINE_SIZE); | |
394 | ||
395 | if (!qdev->sb_array) { | |
396 | DP_ERR(edev, "sb array allocation failed\n"); | |
397 | rte_free(qdev->fp_array); | |
398 | return -ENOMEM; | |
399 | } | |
400 | ||
401 | return 0; | |
402 | } | |
403 | ||
404 | /* This function allocates fast-path status block memory */ | |
405 | static int | |
406 | qede_alloc_mem_sb(struct qede_dev *qdev, struct ecore_sb_info *sb_info, | |
407 | uint16_t sb_id) | |
408 | { | |
409 | struct ecore_dev *edev = &qdev->edev; | |
410 | struct status_block *sb_virt; | |
411 | dma_addr_t sb_phys; | |
412 | int rc; | |
413 | ||
414 | sb_virt = OSAL_DMA_ALLOC_COHERENT(edev, &sb_phys, sizeof(*sb_virt)); | |
415 | ||
416 | if (!sb_virt) { | |
417 | DP_ERR(edev, "Status block allocation failed\n"); | |
418 | return -ENOMEM; | |
419 | } | |
420 | ||
421 | rc = qdev->ops->common->sb_init(edev, sb_info, | |
422 | sb_virt, sb_phys, sb_id, | |
423 | QED_SB_TYPE_L2_QUEUE); | |
424 | if (rc) { | |
425 | DP_ERR(edev, "Status block initialization failed\n"); | |
426 | /* TBD: No dma_free_coherent possible */ | |
427 | return rc; | |
428 | } | |
429 | ||
430 | return 0; | |
431 | } | |
432 | ||
433 | int qede_alloc_fp_resc(struct qede_dev *qdev) | |
434 | { | |
435 | struct ecore_dev *edev = &qdev->edev; | |
436 | struct qede_fastpath *fp; | |
437 | uint32_t num_sbs; | |
438 | int rc, i; | |
439 | ||
440 | if (IS_VF(edev)) | |
441 | ecore_vf_get_num_sbs(ECORE_LEADING_HWFN(edev), &num_sbs); | |
442 | else | |
443 | num_sbs = (ecore_cxt_get_proto_cid_count | |
444 | (ECORE_LEADING_HWFN(edev), PROTOCOLID_ETH, NULL)) / 2; | |
445 | ||
446 | if (num_sbs == 0) { | |
447 | DP_ERR(edev, "No status blocks available\n"); | |
448 | return -EINVAL; | |
449 | } | |
450 | ||
451 | if (qdev->fp_array) | |
452 | qede_free_fp_arrays(qdev); | |
453 | ||
454 | rc = qede_alloc_fp_array(qdev); | |
455 | if (rc != 0) | |
456 | return rc; | |
457 | ||
458 | qede_init_fp(qdev); | |
459 | ||
460 | for (i = 0; i < QEDE_QUEUE_CNT(qdev); i++) { | |
461 | fp = &qdev->fp_array[i]; | |
462 | if (qede_alloc_mem_sb(qdev, fp->sb_info, i % num_sbs)) { | |
463 | qede_free_fp_arrays(qdev); | |
464 | return -ENOMEM; | |
465 | } | |
466 | } | |
467 | ||
468 | return 0; | |
469 | } | |
470 | ||
471 | void qede_dealloc_fp_resc(struct rte_eth_dev *eth_dev) | |
472 | { | |
473 | struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); | |
474 | ||
475 | qede_free_mem_load(eth_dev); | |
476 | qede_free_fp_arrays(qdev); | |
477 | } | |
478 | ||
479 | static inline void | |
480 | qede_update_rx_prod(struct qede_dev *edev, struct qede_rx_queue *rxq) | |
481 | { | |
482 | uint16_t bd_prod = ecore_chain_get_prod_idx(&rxq->rx_bd_ring); | |
483 | uint16_t cqe_prod = ecore_chain_get_prod_idx(&rxq->rx_comp_ring); | |
484 | struct eth_rx_prod_data rx_prods = { 0 }; | |
485 | ||
486 | /* Update producers */ | |
487 | rx_prods.bd_prod = rte_cpu_to_le_16(bd_prod); | |
488 | rx_prods.cqe_prod = rte_cpu_to_le_16(cqe_prod); | |
489 | ||
490 | /* Make sure that the BD and SGE data is updated before updating the | |
491 | * producers since FW might read the BD/SGE right after the producer | |
492 | * is updated. | |
493 | */ | |
494 | rte_wmb(); | |
495 | ||
496 | internal_ram_wr(rxq->hw_rxq_prod_addr, sizeof(rx_prods), | |
497 | (uint32_t *)&rx_prods); | |
498 | ||
499 | /* mmiowb is needed to synchronize doorbell writes from more than one | |
500 | * processor. It guarantees that the write arrives to the device before | |
501 | * the napi lock is released and another qede_poll is called (possibly | |
502 | * on another CPU). Without this barrier, the next doorbell can bypass | |
503 | * this doorbell. This is applicable to IA64/Altix systems. | |
504 | */ | |
505 | rte_wmb(); | |
506 | ||
507 | PMD_RX_LOG(DEBUG, rxq, "bd_prod %u cqe_prod %u\n", bd_prod, cqe_prod); | |
508 | } | |
509 | ||
510 | static inline uint32_t | |
511 | qede_rxfh_indir_default(uint32_t index, uint32_t n_rx_rings) | |
512 | { | |
513 | return index % n_rx_rings; | |
514 | } | |
515 | ||
516 | static void qede_prandom_bytes(uint32_t *buff, size_t bytes) | |
517 | { | |
518 | unsigned int i; | |
519 | ||
520 | srand((unsigned int)time(NULL)); | |
521 | ||
522 | for (i = 0; i < ECORE_RSS_KEY_SIZE; i++) | |
523 | buff[i] = rand(); | |
524 | } | |
525 | ||
526 | static bool | |
527 | qede_check_vport_rss_enable(struct rte_eth_dev *eth_dev, | |
528 | struct qed_update_vport_rss_params *rss_params) | |
529 | { | |
530 | struct rte_eth_rss_conf rss_conf; | |
531 | enum rte_eth_rx_mq_mode mode = eth_dev->data->dev_conf.rxmode.mq_mode; | |
532 | struct qede_dev *qdev = eth_dev->data->dev_private; | |
533 | struct ecore_dev *edev = &qdev->edev; | |
534 | uint8_t rss_caps; | |
535 | unsigned int i; | |
536 | uint64_t hf; | |
537 | uint32_t *key; | |
538 | ||
539 | PMD_INIT_FUNC_TRACE(edev); | |
540 | ||
541 | rss_conf = eth_dev->data->dev_conf.rx_adv_conf.rss_conf; | |
542 | key = (uint32_t *)rss_conf.rss_key; | |
543 | hf = rss_conf.rss_hf; | |
544 | ||
545 | /* Check if RSS conditions are met. | |
546 | * Note: Even though its meaningless to enable RSS with one queue, it | |
547 | * could be used to produce RSS Hash, so skipping that check. | |
548 | */ | |
549 | if (!(mode & ETH_MQ_RX_RSS)) { | |
550 | DP_INFO(edev, "RSS flag is not set\n"); | |
551 | return false; | |
552 | } | |
553 | ||
554 | if (hf == 0) { | |
555 | DP_INFO(edev, "Request to disable RSS\n"); | |
556 | return false; | |
557 | } | |
558 | ||
559 | memset(rss_params, 0, sizeof(*rss_params)); | |
560 | ||
561 | for (i = 0; i < ECORE_RSS_IND_TABLE_SIZE; i++) | |
562 | rss_params->rss_ind_table[i] = qede_rxfh_indir_default(i, | |
563 | QEDE_RSS_COUNT(qdev)); | |
564 | ||
565 | if (!key) | |
566 | qede_prandom_bytes(rss_params->rss_key, | |
567 | sizeof(rss_params->rss_key)); | |
568 | else | |
569 | memcpy(rss_params->rss_key, rss_conf.rss_key, | |
570 | rss_conf.rss_key_len); | |
571 | ||
572 | qede_init_rss_caps(&rss_caps, hf); | |
573 | ||
574 | rss_params->rss_caps = rss_caps; | |
575 | ||
576 | DP_INFO(edev, "RSS conditions are met\n"); | |
577 | ||
578 | return true; | |
579 | } | |
580 | ||
581 | static int qede_start_queues(struct rte_eth_dev *eth_dev, bool clear_stats) | |
582 | { | |
583 | struct qede_dev *qdev = eth_dev->data->dev_private; | |
584 | struct ecore_dev *edev = &qdev->edev; | |
585 | struct ecore_queue_start_common_params q_params; | |
586 | struct qed_update_vport_rss_params *rss_params = &qdev->rss_params; | |
587 | struct qed_dev_info *qed_info = &qdev->dev_info.common; | |
588 | struct qed_update_vport_params vport_update_params; | |
589 | struct qede_tx_queue *txq; | |
590 | struct qede_fastpath *fp; | |
591 | dma_addr_t p_phys_table; | |
592 | int txq_index; | |
593 | uint16_t page_cnt; | |
594 | int vlan_removal_en = 1; | |
595 | int rc, tc, i; | |
596 | ||
597 | for_each_queue(i) { | |
598 | fp = &qdev->fp_array[i]; | |
599 | if (fp->type & QEDE_FASTPATH_RX) { | |
600 | p_phys_table = ecore_chain_get_pbl_phys(&fp->rxq-> | |
601 | rx_comp_ring); | |
602 | page_cnt = ecore_chain_get_page_cnt(&fp->rxq-> | |
603 | rx_comp_ring); | |
604 | ||
605 | memset(&q_params, 0, sizeof(q_params)); | |
606 | q_params.queue_id = i; | |
607 | q_params.vport_id = 0; | |
608 | q_params.sb = fp->sb_info->igu_sb_id; | |
609 | q_params.sb_idx = RX_PI; | |
610 | ||
611 | ecore_sb_ack(fp->sb_info, IGU_INT_DISABLE, 0); | |
612 | ||
613 | rc = qdev->ops->q_rx_start(edev, i, &q_params, | |
614 | fp->rxq->rx_buf_size, | |
615 | fp->rxq->rx_bd_ring.p_phys_addr, | |
616 | p_phys_table, | |
617 | page_cnt, | |
618 | &fp->rxq->hw_rxq_prod_addr); | |
619 | if (rc) { | |
620 | DP_ERR(edev, "Start rxq #%d failed %d\n", | |
621 | fp->rxq->queue_id, rc); | |
622 | return rc; | |
623 | } | |
624 | ||
625 | fp->rxq->hw_cons_ptr = | |
626 | &fp->sb_info->sb_virt->pi_array[RX_PI]; | |
627 | ||
628 | qede_update_rx_prod(qdev, fp->rxq); | |
629 | } | |
630 | ||
631 | if (!(fp->type & QEDE_FASTPATH_TX)) | |
632 | continue; | |
633 | for (tc = 0; tc < qdev->num_tc; tc++) { | |
634 | txq = fp->txqs[tc]; | |
635 | txq_index = tc * QEDE_RSS_COUNT(qdev) + i; | |
636 | ||
637 | p_phys_table = ecore_chain_get_pbl_phys(&txq->tx_pbl); | |
638 | page_cnt = ecore_chain_get_page_cnt(&txq->tx_pbl); | |
639 | ||
640 | memset(&q_params, 0, sizeof(q_params)); | |
641 | q_params.queue_id = txq->queue_id; | |
642 | q_params.vport_id = 0; | |
643 | q_params.sb = fp->sb_info->igu_sb_id; | |
644 | q_params.sb_idx = TX_PI(tc); | |
645 | ||
646 | rc = qdev->ops->q_tx_start(edev, i, &q_params, | |
647 | p_phys_table, | |
648 | page_cnt, /* **pp_doorbell */ | |
649 | &txq->doorbell_addr); | |
650 | if (rc) { | |
651 | DP_ERR(edev, "Start txq %u failed %d\n", | |
652 | txq_index, rc); | |
653 | return rc; | |
654 | } | |
655 | ||
656 | txq->hw_cons_ptr = | |
657 | &fp->sb_info->sb_virt->pi_array[TX_PI(tc)]; | |
658 | SET_FIELD(txq->tx_db.data.params, | |
659 | ETH_DB_DATA_DEST, DB_DEST_XCM); | |
660 | SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_CMD, | |
661 | DB_AGG_CMD_SET); | |
662 | SET_FIELD(txq->tx_db.data.params, | |
663 | ETH_DB_DATA_AGG_VAL_SEL, | |
664 | DQ_XCM_ETH_TX_BD_PROD_CMD); | |
665 | ||
666 | txq->tx_db.data.agg_flags = DQ_XCM_ETH_DQ_CF_CMD; | |
667 | } | |
668 | } | |
669 | ||
670 | /* Prepare and send the vport enable */ | |
671 | memset(&vport_update_params, 0, sizeof(vport_update_params)); | |
672 | /* Update MTU via vport update */ | |
673 | vport_update_params.mtu = qdev->mtu; | |
674 | vport_update_params.vport_id = 0; | |
675 | vport_update_params.update_vport_active_flg = 1; | |
676 | vport_update_params.vport_active_flg = 1; | |
677 | ||
678 | /* @DPDK */ | |
679 | if (qed_info->mf_mode == MF_NPAR && qed_info->tx_switching) { | |
680 | /* TBD: Check SRIOV enabled for VF */ | |
681 | vport_update_params.update_tx_switching_flg = 1; | |
682 | vport_update_params.tx_switching_flg = 1; | |
683 | } | |
684 | ||
685 | if (qede_check_vport_rss_enable(eth_dev, rss_params)) { | |
686 | vport_update_params.update_rss_flg = 1; | |
687 | qdev->rss_enabled = 1; | |
688 | } else { | |
689 | qdev->rss_enabled = 0; | |
690 | } | |
691 | ||
692 | rte_memcpy(&vport_update_params.rss_params, rss_params, | |
693 | sizeof(*rss_params)); | |
694 | ||
695 | rc = qdev->ops->vport_update(edev, &vport_update_params); | |
696 | if (rc) { | |
697 | DP_ERR(edev, "Update V-PORT failed %d\n", rc); | |
698 | return rc; | |
699 | } | |
700 | ||
701 | return 0; | |
702 | } | |
703 | ||
704 | #ifdef ENC_SUPPORTED | |
705 | static bool qede_tunn_exist(uint16_t flag) | |
706 | { | |
707 | return !!((PARSING_AND_ERR_FLAGS_TUNNELEXIST_MASK << | |
708 | PARSING_AND_ERR_FLAGS_TUNNELEXIST_SHIFT) & flag); | |
709 | } | |
710 | ||
711 | static inline uint8_t qede_check_tunn_csum(uint16_t flag) | |
712 | { | |
713 | uint8_t tcsum = 0; | |
714 | uint16_t csum_flag = 0; | |
715 | ||
716 | if ((PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMWASCALCULATED_MASK << | |
717 | PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMWASCALCULATED_SHIFT) & flag) | |
718 | csum_flag |= PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMERROR_MASK << | |
719 | PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMERROR_SHIFT; | |
720 | ||
721 | if ((PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_MASK << | |
722 | PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_SHIFT) & flag) { | |
723 | csum_flag |= PARSING_AND_ERR_FLAGS_L4CHKSMERROR_MASK << | |
724 | PARSING_AND_ERR_FLAGS_L4CHKSMERROR_SHIFT; | |
725 | tcsum = QEDE_TUNN_CSUM_UNNECESSARY; | |
726 | } | |
727 | ||
728 | csum_flag |= PARSING_AND_ERR_FLAGS_TUNNELIPHDRERROR_MASK << | |
729 | PARSING_AND_ERR_FLAGS_TUNNELIPHDRERROR_SHIFT | | |
730 | PARSING_AND_ERR_FLAGS_IPHDRERROR_MASK << | |
731 | PARSING_AND_ERR_FLAGS_IPHDRERROR_SHIFT; | |
732 | ||
733 | if (csum_flag & flag) | |
734 | return QEDE_CSUM_ERROR; | |
735 | ||
736 | return QEDE_CSUM_UNNECESSARY | tcsum; | |
737 | } | |
738 | #else | |
739 | static inline uint8_t qede_tunn_exist(uint16_t flag) | |
740 | { | |
741 | return 0; | |
742 | } | |
743 | ||
744 | static inline uint8_t qede_check_tunn_csum(uint16_t flag) | |
745 | { | |
746 | return 0; | |
747 | } | |
748 | #endif | |
749 | ||
750 | static inline uint8_t qede_check_notunn_csum(uint16_t flag) | |
751 | { | |
752 | uint8_t csum = 0; | |
753 | uint16_t csum_flag = 0; | |
754 | ||
755 | if ((PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_MASK << | |
756 | PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_SHIFT) & flag) { | |
757 | csum_flag |= PARSING_AND_ERR_FLAGS_L4CHKSMERROR_MASK << | |
758 | PARSING_AND_ERR_FLAGS_L4CHKSMERROR_SHIFT; | |
759 | csum = QEDE_CSUM_UNNECESSARY; | |
760 | } | |
761 | ||
762 | csum_flag |= PARSING_AND_ERR_FLAGS_IPHDRERROR_MASK << | |
763 | PARSING_AND_ERR_FLAGS_IPHDRERROR_SHIFT; | |
764 | ||
765 | if (csum_flag & flag) | |
766 | return QEDE_CSUM_ERROR; | |
767 | ||
768 | return csum; | |
769 | } | |
770 | ||
771 | static inline uint8_t qede_check_csum(uint16_t flag) | |
772 | { | |
773 | if (likely(!qede_tunn_exist(flag))) | |
774 | return qede_check_notunn_csum(flag); | |
775 | else | |
776 | return qede_check_tunn_csum(flag); | |
777 | } | |
778 | ||
779 | static inline void qede_rx_bd_ring_consume(struct qede_rx_queue *rxq) | |
780 | { | |
781 | ecore_chain_consume(&rxq->rx_bd_ring); | |
782 | rxq->sw_rx_cons++; | |
783 | } | |
784 | ||
785 | static inline void | |
786 | qede_reuse_page(struct qede_dev *qdev, | |
787 | struct qede_rx_queue *rxq, struct qede_rx_entry *curr_cons) | |
788 | { | |
789 | struct eth_rx_bd *rx_bd_prod = ecore_chain_produce(&rxq->rx_bd_ring); | |
790 | uint16_t idx = rxq->sw_rx_cons & NUM_RX_BDS(rxq); | |
791 | struct qede_rx_entry *curr_prod; | |
792 | dma_addr_t new_mapping; | |
793 | ||
794 | curr_prod = &rxq->sw_rx_ring[idx]; | |
795 | *curr_prod = *curr_cons; | |
796 | ||
797 | new_mapping = rte_mbuf_data_dma_addr_default(curr_prod->mbuf) + | |
798 | curr_prod->page_offset; | |
799 | ||
800 | rx_bd_prod->addr.hi = rte_cpu_to_le_32(U64_HI(new_mapping)); | |
801 | rx_bd_prod->addr.lo = rte_cpu_to_le_32(U64_LO(new_mapping)); | |
802 | ||
803 | rxq->sw_rx_prod++; | |
804 | } | |
805 | ||
806 | static inline void | |
807 | qede_recycle_rx_bd_ring(struct qede_rx_queue *rxq, | |
808 | struct qede_dev *qdev, uint8_t count) | |
809 | { | |
810 | struct qede_rx_entry *curr_cons; | |
811 | ||
812 | for (; count > 0; count--) { | |
813 | curr_cons = &rxq->sw_rx_ring[rxq->sw_rx_cons & NUM_RX_BDS(rxq)]; | |
814 | qede_reuse_page(qdev, rxq, curr_cons); | |
815 | qede_rx_bd_ring_consume(rxq); | |
816 | } | |
817 | } | |
818 | ||
819 | static inline uint32_t qede_rx_cqe_to_pkt_type(uint16_t flags) | |
820 | { | |
821 | uint32_t p_type; | |
822 | /* TBD - L4 indications needed ? */ | |
823 | uint16_t protocol = ((PARSING_AND_ERR_FLAGS_L3TYPE_MASK << | |
824 | PARSING_AND_ERR_FLAGS_L3TYPE_SHIFT) & flags); | |
825 | ||
826 | /* protocol = 3 means LLC/SNAP over Ethernet */ | |
827 | if (unlikely(protocol == 0 || protocol == 3)) | |
828 | p_type = RTE_PTYPE_UNKNOWN; | |
829 | else if (protocol == 1) | |
830 | p_type = RTE_PTYPE_L3_IPV4; | |
831 | else if (protocol == 2) | |
832 | p_type = RTE_PTYPE_L3_IPV6; | |
833 | ||
834 | return RTE_PTYPE_L2_ETHER | p_type; | |
835 | } | |
836 | ||
837 | int qede_process_sg_pkts(void *p_rxq, struct rte_mbuf *rx_mb, | |
838 | int num_segs, uint16_t pkt_len) | |
839 | { | |
840 | struct qede_rx_queue *rxq = p_rxq; | |
841 | struct qede_dev *qdev = rxq->qdev; | |
842 | struct ecore_dev *edev = &qdev->edev; | |
843 | uint16_t sw_rx_index, cur_size; | |
844 | ||
845 | register struct rte_mbuf *seg1 = NULL; | |
846 | register struct rte_mbuf *seg2 = NULL; | |
847 | ||
848 | seg1 = rx_mb; | |
849 | while (num_segs) { | |
850 | cur_size = pkt_len > rxq->rx_buf_size ? | |
851 | rxq->rx_buf_size : pkt_len; | |
852 | if (!cur_size) { | |
853 | PMD_RX_LOG(DEBUG, rxq, | |
854 | "SG packet, len and num BD mismatch\n"); | |
855 | qede_recycle_rx_bd_ring(rxq, qdev, num_segs); | |
856 | return -EINVAL; | |
857 | } | |
858 | ||
859 | if (qede_alloc_rx_buffer(rxq)) { | |
860 | uint8_t index; | |
861 | ||
862 | PMD_RX_LOG(DEBUG, rxq, "Buffer allocation failed\n"); | |
863 | index = rxq->port_id; | |
864 | rte_eth_devices[index].data->rx_mbuf_alloc_failed++; | |
865 | rxq->rx_alloc_errors++; | |
866 | return -ENOMEM; | |
867 | } | |
868 | ||
869 | sw_rx_index = rxq->sw_rx_cons & NUM_RX_BDS(rxq); | |
870 | seg2 = rxq->sw_rx_ring[sw_rx_index].mbuf; | |
871 | qede_rx_bd_ring_consume(rxq); | |
872 | pkt_len -= cur_size; | |
873 | seg2->data_len = cur_size; | |
874 | seg1->next = seg2; | |
875 | seg1 = seg1->next; | |
876 | num_segs--; | |
877 | rxq->rx_segs++; | |
878 | continue; | |
879 | } | |
880 | seg1 = NULL; | |
881 | ||
882 | if (pkt_len) | |
883 | PMD_RX_LOG(DEBUG, rxq, | |
884 | "Mapped all BDs of jumbo, but still have %d bytes\n", | |
885 | pkt_len); | |
886 | ||
887 | return ECORE_SUCCESS; | |
888 | } | |
889 | ||
890 | uint16_t | |
891 | qede_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) | |
892 | { | |
893 | struct qede_rx_queue *rxq = p_rxq; | |
894 | struct qede_dev *qdev = rxq->qdev; | |
895 | struct ecore_dev *edev = &qdev->edev; | |
896 | struct qede_fastpath *fp = &qdev->fp_array[rxq->queue_id]; | |
897 | uint16_t hw_comp_cons, sw_comp_cons, sw_rx_index; | |
898 | uint16_t rx_pkt = 0; | |
899 | union eth_rx_cqe *cqe; | |
900 | struct eth_fast_path_rx_reg_cqe *fp_cqe; | |
901 | register struct rte_mbuf *rx_mb = NULL; | |
902 | register struct rte_mbuf *seg1 = NULL; | |
903 | enum eth_rx_cqe_type cqe_type; | |
904 | uint16_t len, pad, preload_idx, pkt_len, parse_flag; | |
905 | uint8_t csum_flag, num_segs; | |
906 | enum rss_hash_type htype; | |
907 | int ret; | |
908 | ||
909 | hw_comp_cons = rte_le_to_cpu_16(*rxq->hw_cons_ptr); | |
910 | sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring); | |
911 | ||
912 | rte_rmb(); | |
913 | ||
914 | if (hw_comp_cons == sw_comp_cons) | |
915 | return 0; | |
916 | ||
917 | while (sw_comp_cons != hw_comp_cons) { | |
918 | /* Get the CQE from the completion ring */ | |
919 | cqe = | |
920 | (union eth_rx_cqe *)ecore_chain_consume(&rxq->rx_comp_ring); | |
921 | cqe_type = cqe->fast_path_regular.type; | |
922 | ||
923 | if (unlikely(cqe_type == ETH_RX_CQE_TYPE_SLOW_PATH)) { | |
924 | PMD_RX_LOG(DEBUG, rxq, "Got a slowath CQE\n"); | |
925 | ||
926 | qdev->ops->eth_cqe_completion(edev, fp->id, | |
927 | (struct eth_slow_path_rx_cqe *)cqe); | |
928 | goto next_cqe; | |
929 | } | |
930 | ||
931 | /* Get the data from the SW ring */ | |
932 | sw_rx_index = rxq->sw_rx_cons & NUM_RX_BDS(rxq); | |
933 | rx_mb = rxq->sw_rx_ring[sw_rx_index].mbuf; | |
934 | assert(rx_mb != NULL); | |
935 | ||
936 | /* non GRO */ | |
937 | fp_cqe = &cqe->fast_path_regular; | |
938 | ||
939 | len = rte_le_to_cpu_16(fp_cqe->len_on_first_bd); | |
940 | pad = fp_cqe->placement_offset; | |
941 | assert((len + pad) <= rx_mb->buf_len); | |
942 | ||
943 | PMD_RX_LOG(DEBUG, rxq, | |
944 | "CQE type = 0x%x, flags = 0x%x, vlan = 0x%x" | |
945 | " len = %u, parsing_flags = %d\n", | |
946 | cqe_type, fp_cqe->bitfields, | |
947 | rte_le_to_cpu_16(fp_cqe->vlan_tag), | |
948 | len, rte_le_to_cpu_16(fp_cqe->pars_flags.flags)); | |
949 | ||
950 | /* If this is an error packet then drop it */ | |
951 | parse_flag = | |
952 | rte_le_to_cpu_16(cqe->fast_path_regular.pars_flags.flags); | |
953 | csum_flag = qede_check_csum(parse_flag); | |
954 | if (unlikely(csum_flag == QEDE_CSUM_ERROR)) { | |
955 | PMD_RX_LOG(ERR, rxq, | |
956 | "CQE in CONS = %u has error, flags = 0x%x " | |
957 | "dropping incoming packet\n", | |
958 | sw_comp_cons, parse_flag); | |
959 | rxq->rx_hw_errors++; | |
960 | qede_recycle_rx_bd_ring(rxq, qdev, fp_cqe->bd_num); | |
961 | goto next_cqe; | |
962 | } | |
963 | ||
964 | if (unlikely(qede_alloc_rx_buffer(rxq) != 0)) { | |
965 | PMD_RX_LOG(ERR, rxq, | |
966 | "New buffer allocation failed," | |
967 | "dropping incoming packet\n"); | |
968 | qede_recycle_rx_bd_ring(rxq, qdev, fp_cqe->bd_num); | |
969 | rte_eth_devices[rxq->port_id]. | |
970 | data->rx_mbuf_alloc_failed++; | |
971 | rxq->rx_alloc_errors++; | |
972 | break; | |
973 | } | |
974 | ||
975 | qede_rx_bd_ring_consume(rxq); | |
976 | ||
977 | if (fp_cqe->bd_num > 1) { | |
978 | pkt_len = rte_le_to_cpu_16(fp_cqe->pkt_len); | |
979 | num_segs = fp_cqe->bd_num - 1; | |
980 | ||
981 | rxq->rx_segs++; | |
982 | ||
983 | pkt_len -= len; | |
984 | seg1 = rx_mb; | |
985 | ret = qede_process_sg_pkts(p_rxq, seg1, num_segs, | |
986 | pkt_len); | |
987 | if (ret != ECORE_SUCCESS) { | |
988 | qede_recycle_rx_bd_ring(rxq, qdev, | |
989 | fp_cqe->bd_num); | |
990 | goto next_cqe; | |
991 | } | |
992 | } | |
993 | ||
994 | /* Prefetch next mbuf while processing current one. */ | |
995 | preload_idx = rxq->sw_rx_cons & NUM_RX_BDS(rxq); | |
996 | rte_prefetch0(rxq->sw_rx_ring[preload_idx].mbuf); | |
997 | ||
998 | /* Update MBUF fields */ | |
999 | rx_mb->ol_flags = 0; | |
1000 | rx_mb->data_off = pad + RTE_PKTMBUF_HEADROOM; | |
1001 | rx_mb->nb_segs = fp_cqe->bd_num; | |
1002 | rx_mb->data_len = len; | |
1003 | rx_mb->pkt_len = fp_cqe->pkt_len; | |
1004 | rx_mb->port = rxq->port_id; | |
1005 | rx_mb->packet_type = qede_rx_cqe_to_pkt_type(parse_flag); | |
1006 | ||
1007 | htype = (uint8_t)GET_FIELD(fp_cqe->bitfields, | |
1008 | ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE); | |
1009 | if (qdev->rss_enabled && htype) { | |
1010 | rx_mb->ol_flags |= PKT_RX_RSS_HASH; | |
1011 | rx_mb->hash.rss = rte_le_to_cpu_32(fp_cqe->rss_hash); | |
1012 | PMD_RX_LOG(DEBUG, rxq, "Hash result 0x%x\n", | |
1013 | rx_mb->hash.rss); | |
1014 | } | |
1015 | ||
1016 | rte_prefetch1(rte_pktmbuf_mtod(rx_mb, void *)); | |
1017 | ||
1018 | if (CQE_HAS_VLAN(parse_flag)) { | |
1019 | rx_mb->vlan_tci = rte_le_to_cpu_16(fp_cqe->vlan_tag); | |
1020 | rx_mb->ol_flags |= PKT_RX_VLAN_PKT; | |
1021 | } | |
1022 | ||
1023 | if (CQE_HAS_OUTER_VLAN(parse_flag)) { | |
1024 | /* FW does not provide indication of Outer VLAN tag, | |
1025 | * which is always stripped, so vlan_tci_outer is set | |
1026 | * to 0. Here vlan_tag represents inner VLAN tag. | |
1027 | */ | |
1028 | rx_mb->vlan_tci = rte_le_to_cpu_16(fp_cqe->vlan_tag); | |
1029 | rx_mb->ol_flags |= PKT_RX_QINQ_PKT; | |
1030 | rx_mb->vlan_tci_outer = 0; | |
1031 | } | |
1032 | ||
1033 | rx_pkts[rx_pkt] = rx_mb; | |
1034 | rx_pkt++; | |
1035 | next_cqe: | |
1036 | ecore_chain_recycle_consumed(&rxq->rx_comp_ring); | |
1037 | sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring); | |
1038 | if (rx_pkt == nb_pkts) { | |
1039 | PMD_RX_LOG(DEBUG, rxq, | |
1040 | "Budget reached nb_pkts=%u received=%u\n", | |
1041 | rx_pkt, nb_pkts); | |
1042 | break; | |
1043 | } | |
1044 | } | |
1045 | ||
1046 | qede_update_rx_prod(qdev, rxq); | |
1047 | ||
1048 | rxq->rcv_pkts += rx_pkt; | |
1049 | ||
1050 | PMD_RX_LOG(DEBUG, rxq, "rx_pkts=%u core=%d\n", rx_pkt, rte_lcore_id()); | |
1051 | ||
1052 | return rx_pkt; | |
1053 | } | |
1054 | ||
1055 | static inline int | |
1056 | qede_free_tx_pkt(struct ecore_dev *edev, struct qede_tx_queue *txq) | |
1057 | { | |
1058 | uint16_t nb_segs, idx = TX_CONS(txq); | |
1059 | struct eth_tx_bd *tx_data_bd; | |
1060 | struct rte_mbuf *mbuf = txq->sw_tx_ring[idx].mbuf; | |
1061 | ||
1062 | if (unlikely(!mbuf)) { | |
1063 | PMD_TX_LOG(ERR, txq, "null mbuf\n"); | |
1064 | PMD_TX_LOG(ERR, txq, | |
1065 | "tx_desc %u tx_avail %u tx_cons %u tx_prod %u\n", | |
1066 | txq->nb_tx_desc, txq->nb_tx_avail, idx, | |
1067 | TX_PROD(txq)); | |
1068 | return -1; | |
1069 | } | |
1070 | ||
1071 | nb_segs = mbuf->nb_segs; | |
1072 | while (nb_segs) { | |
1073 | /* It's like consuming rxbuf in recv() */ | |
1074 | ecore_chain_consume(&txq->tx_pbl); | |
1075 | txq->nb_tx_avail++; | |
1076 | nb_segs--; | |
1077 | } | |
1078 | rte_pktmbuf_free(mbuf); | |
1079 | txq->sw_tx_ring[idx].mbuf = NULL; | |
1080 | ||
1081 | return 0; | |
1082 | } | |
1083 | ||
1084 | static inline uint16_t | |
1085 | qede_process_tx_compl(struct ecore_dev *edev, struct qede_tx_queue *txq) | |
1086 | { | |
1087 | uint16_t tx_compl = 0; | |
1088 | uint16_t hw_bd_cons; | |
1089 | ||
1090 | hw_bd_cons = rte_le_to_cpu_16(*txq->hw_cons_ptr); | |
1091 | rte_compiler_barrier(); | |
1092 | ||
1093 | while (hw_bd_cons != ecore_chain_get_cons_idx(&txq->tx_pbl)) { | |
1094 | if (qede_free_tx_pkt(edev, txq)) { | |
1095 | PMD_TX_LOG(ERR, txq, | |
1096 | "hw_bd_cons = %u, chain_cons = %u\n", | |
1097 | hw_bd_cons, | |
1098 | ecore_chain_get_cons_idx(&txq->tx_pbl)); | |
1099 | break; | |
1100 | } | |
1101 | txq->sw_tx_cons++; /* Making TXD available */ | |
1102 | tx_compl++; | |
1103 | } | |
1104 | ||
1105 | PMD_TX_LOG(DEBUG, txq, "Tx compl %u sw_tx_cons %u avail %u\n", | |
1106 | tx_compl, txq->sw_tx_cons, txq->nb_tx_avail); | |
1107 | return tx_compl; | |
1108 | } | |
1109 | ||
1110 | /* Populate scatter gather buffer descriptor fields */ | |
1111 | static inline uint16_t qede_encode_sg_bd(struct qede_tx_queue *p_txq, | |
1112 | struct rte_mbuf *m_seg, | |
1113 | uint16_t count, | |
1114 | struct eth_tx_1st_bd *bd1) | |
1115 | { | |
1116 | struct qede_tx_queue *txq = p_txq; | |
1117 | struct eth_tx_2nd_bd *bd2 = NULL; | |
1118 | struct eth_tx_3rd_bd *bd3 = NULL; | |
1119 | struct eth_tx_bd *tx_bd = NULL; | |
1120 | uint16_t nb_segs = count; | |
1121 | dma_addr_t mapping; | |
1122 | ||
1123 | /* Check for scattered buffers */ | |
1124 | while (m_seg) { | |
1125 | if (nb_segs == 1) { | |
1126 | bd2 = (struct eth_tx_2nd_bd *) | |
1127 | ecore_chain_produce(&txq->tx_pbl); | |
1128 | memset(bd2, 0, sizeof(*bd2)); | |
1129 | mapping = rte_mbuf_data_dma_addr(m_seg); | |
1130 | bd2->addr.hi = rte_cpu_to_le_32(U64_HI(mapping)); | |
1131 | bd2->addr.lo = rte_cpu_to_le_32(U64_LO(mapping)); | |
1132 | bd2->nbytes = rte_cpu_to_le_16(m_seg->data_len); | |
1133 | } else if (nb_segs == 2) { | |
1134 | bd3 = (struct eth_tx_3rd_bd *) | |
1135 | ecore_chain_produce(&txq->tx_pbl); | |
1136 | memset(bd3, 0, sizeof(*bd3)); | |
1137 | mapping = rte_mbuf_data_dma_addr(m_seg); | |
1138 | bd3->addr.hi = rte_cpu_to_le_32(U64_HI(mapping)); | |
1139 | bd3->addr.lo = rte_cpu_to_le_32(U64_LO(mapping)); | |
1140 | bd3->nbytes = rte_cpu_to_le_16(m_seg->data_len); | |
1141 | } else { | |
1142 | tx_bd = (struct eth_tx_bd *) | |
1143 | ecore_chain_produce(&txq->tx_pbl); | |
1144 | memset(tx_bd, 0, sizeof(*tx_bd)); | |
1145 | mapping = rte_mbuf_data_dma_addr(m_seg); | |
1146 | tx_bd->addr.hi = rte_cpu_to_le_32(U64_HI(mapping)); | |
1147 | tx_bd->addr.lo = rte_cpu_to_le_32(U64_LO(mapping)); | |
1148 | tx_bd->nbytes = rte_cpu_to_le_16(m_seg->data_len); | |
1149 | } | |
1150 | nb_segs++; | |
1151 | bd1->data.nbds = nb_segs; | |
1152 | m_seg = m_seg->next; | |
1153 | } | |
1154 | ||
1155 | /* Return total scattered buffers */ | |
1156 | return nb_segs; | |
1157 | } | |
1158 | ||
1159 | uint16_t | |
1160 | qede_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) | |
1161 | { | |
1162 | struct qede_tx_queue *txq = p_txq; | |
1163 | struct qede_dev *qdev = txq->qdev; | |
1164 | struct ecore_dev *edev = &qdev->edev; | |
1165 | struct qede_fastpath *fp; | |
1166 | struct eth_tx_1st_bd *bd1; | |
1167 | struct rte_mbuf *m_seg = NULL; | |
1168 | uint16_t nb_tx_pkts; | |
1169 | uint16_t nb_pkt_sent = 0; | |
1170 | uint16_t bd_prod; | |
1171 | uint16_t idx; | |
1172 | uint16_t tx_count; | |
1173 | uint16_t nb_segs = 0; | |
1174 | ||
1175 | fp = &qdev->fp_array[QEDE_RSS_COUNT(qdev) + txq->queue_id]; | |
1176 | ||
1177 | if (unlikely(txq->nb_tx_avail < txq->tx_free_thresh)) { | |
1178 | PMD_TX_LOG(DEBUG, txq, "send=%u avail=%u free_thresh=%u\n", | |
1179 | nb_pkts, txq->nb_tx_avail, txq->tx_free_thresh); | |
1180 | (void)qede_process_tx_compl(edev, txq); | |
1181 | } | |
1182 | ||
1183 | nb_tx_pkts = RTE_MIN(nb_pkts, (txq->nb_tx_avail / | |
1184 | ETH_TX_MAX_BDS_PER_NON_LSO_PACKET)); | |
1185 | if (unlikely(nb_tx_pkts == 0)) { | |
1186 | PMD_TX_LOG(DEBUG, txq, "Out of BDs nb_pkts=%u avail=%u\n", | |
1187 | nb_pkts, txq->nb_tx_avail); | |
1188 | return 0; | |
1189 | } | |
1190 | ||
1191 | tx_count = nb_tx_pkts; | |
1192 | while (nb_tx_pkts--) { | |
1193 | /* Fill the entry in the SW ring and the BDs in the FW ring */ | |
1194 | idx = TX_PROD(txq); | |
1195 | struct rte_mbuf *mbuf = *tx_pkts++; | |
1196 | ||
1197 | txq->sw_tx_ring[idx].mbuf = mbuf; | |
1198 | bd1 = (struct eth_tx_1st_bd *)ecore_chain_produce(&txq->tx_pbl); | |
1199 | /* Zero init struct fields */ | |
1200 | bd1->data.bd_flags.bitfields = 0; | |
1201 | bd1->data.bitfields = 0; | |
1202 | ||
1203 | bd1->data.bd_flags.bitfields = | |
1204 | 1 << ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT; | |
1205 | /* Map MBUF linear data for DMA and set in the first BD */ | |
1206 | QEDE_BD_SET_ADDR_LEN(bd1, rte_mbuf_data_dma_addr(mbuf), | |
1207 | mbuf->pkt_len); | |
1208 | ||
1209 | /* Descriptor based VLAN insertion */ | |
1210 | if (mbuf->ol_flags & (PKT_TX_VLAN_PKT | PKT_TX_QINQ_PKT)) { | |
1211 | bd1->data.vlan = rte_cpu_to_le_16(mbuf->vlan_tci); | |
1212 | bd1->data.bd_flags.bitfields |= | |
1213 | 1 << ETH_TX_1ST_BD_FLAGS_VLAN_INSERTION_SHIFT; | |
1214 | } | |
1215 | ||
1216 | /* Offload the IP checksum in the hardware */ | |
1217 | if (mbuf->ol_flags & PKT_TX_IP_CKSUM) { | |
1218 | bd1->data.bd_flags.bitfields |= | |
1219 | 1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT; | |
1220 | } | |
1221 | ||
1222 | /* L4 checksum offload (tcp or udp) */ | |
1223 | if (mbuf->ol_flags & (PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) { | |
1224 | bd1->data.bd_flags.bitfields |= | |
1225 | 1 << ETH_TX_1ST_BD_FLAGS_L4_CSUM_SHIFT; | |
1226 | /* IPv6 + extn. -> later */ | |
1227 | } | |
1228 | ||
1229 | /* Handle fragmented MBUF */ | |
1230 | m_seg = mbuf->next; | |
1231 | nb_segs++; | |
1232 | bd1->data.nbds = nb_segs; | |
1233 | /* Encode scatter gather buffer descriptors if required */ | |
1234 | nb_segs = qede_encode_sg_bd(txq, m_seg, nb_segs, bd1); | |
1235 | txq->nb_tx_avail = txq->nb_tx_avail - nb_segs; | |
1236 | nb_segs = 0; | |
1237 | txq->sw_tx_prod++; | |
1238 | rte_prefetch0(txq->sw_tx_ring[TX_PROD(txq)].mbuf); | |
1239 | bd_prod = | |
1240 | rte_cpu_to_le_16(ecore_chain_get_prod_idx(&txq->tx_pbl)); | |
1241 | nb_pkt_sent++; | |
1242 | txq->xmit_pkts++; | |
1243 | } | |
1244 | ||
1245 | /* Write value of prod idx into bd_prod */ | |
1246 | txq->tx_db.data.bd_prod = bd_prod; | |
1247 | rte_wmb(); | |
1248 | rte_compiler_barrier(); | |
1249 | DIRECT_REG_WR(edev, txq->doorbell_addr, txq->tx_db.raw); | |
1250 | rte_wmb(); | |
1251 | ||
1252 | /* Check again for Tx completions */ | |
1253 | (void)qede_process_tx_compl(edev, txq); | |
1254 | ||
1255 | PMD_TX_LOG(DEBUG, txq, "to_send=%u can_send=%u sent=%u core=%d\n", | |
1256 | nb_pkts, tx_count, nb_pkt_sent, rte_lcore_id()); | |
1257 | ||
1258 | return nb_pkt_sent; | |
1259 | } | |
1260 | ||
1261 | static void qede_init_fp_queue(struct rte_eth_dev *eth_dev) | |
1262 | { | |
1263 | struct qede_dev *qdev = eth_dev->data->dev_private; | |
1264 | struct qede_fastpath *fp; | |
1265 | uint8_t i, rss_id, txq_index, tc; | |
1266 | int rxq = 0, txq = 0; | |
1267 | ||
1268 | for_each_queue(i) { | |
1269 | fp = &qdev->fp_array[i]; | |
1270 | if (fp->type & QEDE_FASTPATH_RX) { | |
1271 | fp->rxq = eth_dev->data->rx_queues[i]; | |
1272 | fp->rxq->queue_id = rxq++; | |
1273 | } | |
1274 | ||
1275 | if (fp->type & QEDE_FASTPATH_TX) { | |
1276 | for (tc = 0; tc < qdev->num_tc; tc++) { | |
1277 | txq_index = tc * QEDE_TSS_COUNT(qdev) + txq; | |
1278 | fp->txqs[tc] = | |
1279 | eth_dev->data->tx_queues[txq_index]; | |
1280 | fp->txqs[tc]->queue_id = txq_index; | |
1281 | } | |
1282 | txq++; | |
1283 | } | |
1284 | } | |
1285 | } | |
1286 | ||
1287 | int qede_dev_start(struct rte_eth_dev *eth_dev) | |
1288 | { | |
1289 | struct qede_dev *qdev = eth_dev->data->dev_private; | |
1290 | struct ecore_dev *edev = &qdev->edev; | |
1291 | struct qed_link_output link_output; | |
1292 | struct qede_fastpath *fp; | |
1293 | int rc, i; | |
1294 | ||
1295 | DP_INFO(edev, "Device state is %d\n", qdev->state); | |
1296 | ||
1297 | if (qdev->state == QEDE_DEV_START) { | |
1298 | DP_INFO(edev, "Port is already started\n"); | |
1299 | return 0; | |
1300 | } | |
1301 | ||
1302 | if (qdev->state == QEDE_DEV_CONFIG) | |
1303 | qede_init_fp_queue(eth_dev); | |
1304 | ||
1305 | rc = qede_start_queues(eth_dev, true); | |
1306 | if (rc) { | |
1307 | DP_ERR(edev, "Failed to start queues\n"); | |
1308 | /* TBD: free */ | |
1309 | return rc; | |
1310 | } | |
1311 | ||
1312 | /* Bring-up the link */ | |
1313 | qede_dev_set_link_state(eth_dev, true); | |
1314 | ||
1315 | /* Reset ring */ | |
1316 | if (qede_reset_fp_rings(qdev)) | |
1317 | return -ENOMEM; | |
1318 | ||
1319 | /* Start/resume traffic */ | |
1320 | qdev->ops->fastpath_start(edev); | |
1321 | ||
1322 | qdev->state = QEDE_DEV_START; | |
1323 | ||
1324 | DP_INFO(edev, "dev_state is QEDE_DEV_START\n"); | |
1325 | ||
1326 | return 0; | |
1327 | } | |
1328 | ||
1329 | static int qede_drain_txq(struct qede_dev *qdev, | |
1330 | struct qede_tx_queue *txq, bool allow_drain) | |
1331 | { | |
1332 | struct ecore_dev *edev = &qdev->edev; | |
1333 | int rc, cnt = 1000; | |
1334 | ||
1335 | while (txq->sw_tx_cons != txq->sw_tx_prod) { | |
1336 | qede_process_tx_compl(edev, txq); | |
1337 | if (!cnt) { | |
1338 | if (allow_drain) { | |
1339 | DP_NOTICE(edev, false, | |
1340 | "Tx queue[%u] is stuck," | |
1341 | "requesting MCP to drain\n", | |
1342 | txq->queue_id); | |
1343 | rc = qdev->ops->common->drain(edev); | |
1344 | if (rc) | |
1345 | return rc; | |
1346 | return qede_drain_txq(qdev, txq, false); | |
1347 | } | |
1348 | ||
1349 | DP_NOTICE(edev, false, | |
1350 | "Timeout waiting for tx queue[%d]:" | |
1351 | "PROD=%d, CONS=%d\n", | |
1352 | txq->queue_id, txq->sw_tx_prod, | |
1353 | txq->sw_tx_cons); | |
1354 | return -ENODEV; | |
1355 | } | |
1356 | cnt--; | |
1357 | DELAY(1000); | |
1358 | rte_compiler_barrier(); | |
1359 | } | |
1360 | ||
1361 | /* FW finished processing, wait for HW to transmit all tx packets */ | |
1362 | DELAY(2000); | |
1363 | ||
1364 | return 0; | |
1365 | } | |
1366 | ||
1367 | static int qede_stop_queues(struct qede_dev *qdev) | |
1368 | { | |
1369 | struct qed_update_vport_params vport_update_params; | |
1370 | struct ecore_dev *edev = &qdev->edev; | |
1371 | int rc, tc, i; | |
1372 | ||
1373 | /* Disable the vport */ | |
1374 | memset(&vport_update_params, 0, sizeof(vport_update_params)); | |
1375 | vport_update_params.vport_id = 0; | |
1376 | vport_update_params.update_vport_active_flg = 1; | |
1377 | vport_update_params.vport_active_flg = 0; | |
1378 | vport_update_params.update_rss_flg = 0; | |
1379 | ||
1380 | DP_INFO(edev, "Deactivate vport\n"); | |
1381 | ||
1382 | rc = qdev->ops->vport_update(edev, &vport_update_params); | |
1383 | if (rc) { | |
1384 | DP_ERR(edev, "Failed to update vport\n"); | |
1385 | return rc; | |
1386 | } | |
1387 | ||
1388 | DP_INFO(edev, "Flushing tx queues\n"); | |
1389 | ||
1390 | /* Flush Tx queues. If needed, request drain from MCP */ | |
1391 | for_each_queue(i) { | |
1392 | struct qede_fastpath *fp = &qdev->fp_array[i]; | |
1393 | ||
1394 | if (fp->type & QEDE_FASTPATH_TX) { | |
1395 | for (tc = 0; tc < qdev->num_tc; tc++) { | |
1396 | struct qede_tx_queue *txq = fp->txqs[tc]; | |
1397 | ||
1398 | rc = qede_drain_txq(qdev, txq, true); | |
1399 | if (rc) | |
1400 | return rc; | |
1401 | } | |
1402 | } | |
1403 | } | |
1404 | ||
1405 | /* Stop all Queues in reverse order */ | |
1406 | for (i = QEDE_QUEUE_CNT(qdev) - 1; i >= 0; i--) { | |
1407 | struct qed_stop_rxq_params rx_params; | |
1408 | ||
1409 | /* Stop the Tx Queue(s) */ | |
1410 | if (qdev->fp_array[i].type & QEDE_FASTPATH_TX) { | |
1411 | for (tc = 0; tc < qdev->num_tc; tc++) { | |
1412 | struct qed_stop_txq_params tx_params; | |
1413 | u8 val; | |
1414 | ||
1415 | tx_params.rss_id = i; | |
1416 | val = qdev->fp_array[i].txqs[tc]->queue_id; | |
1417 | tx_params.tx_queue_id = val; | |
1418 | ||
1419 | DP_INFO(edev, "Stopping tx queues\n"); | |
1420 | rc = qdev->ops->q_tx_stop(edev, &tx_params); | |
1421 | if (rc) { | |
1422 | DP_ERR(edev, "Failed to stop TXQ #%d\n", | |
1423 | tx_params.tx_queue_id); | |
1424 | return rc; | |
1425 | } | |
1426 | } | |
1427 | } | |
1428 | ||
1429 | /* Stop the Rx Queue */ | |
1430 | if (qdev->fp_array[i].type & QEDE_FASTPATH_RX) { | |
1431 | memset(&rx_params, 0, sizeof(rx_params)); | |
1432 | rx_params.rss_id = i; | |
1433 | rx_params.rx_queue_id = qdev->fp_array[i].rxq->queue_id; | |
1434 | rx_params.eq_completion_only = 1; | |
1435 | ||
1436 | DP_INFO(edev, "Stopping rx queues\n"); | |
1437 | ||
1438 | rc = qdev->ops->q_rx_stop(edev, &rx_params); | |
1439 | if (rc) { | |
1440 | DP_ERR(edev, "Failed to stop RXQ #%d\n", i); | |
1441 | return rc; | |
1442 | } | |
1443 | } | |
1444 | } | |
1445 | ||
1446 | return 0; | |
1447 | } | |
1448 | ||
1449 | int qede_reset_fp_rings(struct qede_dev *qdev) | |
1450 | { | |
1451 | struct qede_fastpath *fp; | |
1452 | struct qede_tx_queue *txq; | |
1453 | uint8_t tc; | |
1454 | uint16_t id, i; | |
1455 | ||
1456 | for_each_queue(id) { | |
1457 | fp = &qdev->fp_array[id]; | |
1458 | ||
1459 | if (fp->type & QEDE_FASTPATH_RX) { | |
1460 | DP_INFO(&qdev->edev, | |
1461 | "Reset FP chain for RSS %u\n", id); | |
1462 | qede_rx_queue_release_mbufs(fp->rxq); | |
1463 | ecore_chain_reset(&fp->rxq->rx_bd_ring); | |
1464 | ecore_chain_reset(&fp->rxq->rx_comp_ring); | |
1465 | fp->rxq->sw_rx_prod = 0; | |
1466 | fp->rxq->sw_rx_cons = 0; | |
1467 | *fp->rxq->hw_cons_ptr = 0; | |
1468 | for (i = 0; i < fp->rxq->nb_rx_desc; i++) { | |
1469 | if (qede_alloc_rx_buffer(fp->rxq)) { | |
1470 | DP_ERR(&qdev->edev, | |
1471 | "RX buffer allocation failed\n"); | |
1472 | return -ENOMEM; | |
1473 | } | |
1474 | } | |
1475 | } | |
1476 | if (fp->type & QEDE_FASTPATH_TX) { | |
1477 | for (tc = 0; tc < qdev->num_tc; tc++) { | |
1478 | txq = fp->txqs[tc]; | |
1479 | qede_tx_queue_release_mbufs(txq); | |
1480 | ecore_chain_reset(&txq->tx_pbl); | |
1481 | txq->sw_tx_cons = 0; | |
1482 | txq->sw_tx_prod = 0; | |
1483 | *txq->hw_cons_ptr = 0; | |
1484 | } | |
1485 | } | |
1486 | } | |
1487 | ||
1488 | return 0; | |
1489 | } | |
1490 | ||
1491 | /* This function frees all memory of a single fp */ | |
1492 | void qede_free_mem_load(struct rte_eth_dev *eth_dev) | |
1493 | { | |
1494 | struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); | |
1495 | struct qede_fastpath *fp; | |
1496 | uint16_t txq_idx; | |
1497 | uint8_t id; | |
1498 | uint8_t tc; | |
1499 | ||
1500 | for_each_queue(id) { | |
1501 | fp = &qdev->fp_array[id]; | |
1502 | if (fp->type & QEDE_FASTPATH_RX) { | |
1503 | qede_rx_queue_release(fp->rxq); | |
1504 | eth_dev->data->rx_queues[id] = NULL; | |
1505 | } else { | |
1506 | for (tc = 0; tc < qdev->num_tc; tc++) { | |
1507 | txq_idx = fp->txqs[tc]->queue_id; | |
1508 | qede_tx_queue_release(fp->txqs[tc]); | |
1509 | eth_dev->data->tx_queues[txq_idx] = NULL; | |
1510 | } | |
1511 | } | |
1512 | } | |
1513 | } | |
1514 | ||
1515 | void qede_dev_stop(struct rte_eth_dev *eth_dev) | |
1516 | { | |
1517 | struct qede_dev *qdev = eth_dev->data->dev_private; | |
1518 | struct ecore_dev *edev = &qdev->edev; | |
1519 | ||
1520 | DP_INFO(edev, "port %u\n", eth_dev->data->port_id); | |
1521 | ||
1522 | if (qdev->state != QEDE_DEV_START) { | |
1523 | DP_INFO(edev, "Device not yet started\n"); | |
1524 | return; | |
1525 | } | |
1526 | ||
1527 | if (qede_stop_queues(qdev)) | |
1528 | DP_ERR(edev, "Didn't succeed to close queues\n"); | |
1529 | ||
1530 | DP_INFO(edev, "Stopped queues\n"); | |
1531 | ||
1532 | qdev->ops->fastpath_stop(edev); | |
1533 | ||
1534 | /* Bring the link down */ | |
1535 | qede_dev_set_link_state(eth_dev, false); | |
1536 | ||
1537 | qdev->state = QEDE_DEV_STOP; | |
1538 | ||
1539 | DP_INFO(edev, "dev_state is QEDE_DEV_STOP\n"); | |
1540 | } |