1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2016-2018 Solarflare Communications Inc.
6 * This software was jointly developed between OKTET Labs (under contract
7 * for Solarflare) and Solarflare Communications, Inc.
10 #include <rte_debug.h>
11 #include <rte_cycles.h>
12 #include <rte_alarm.h>
13 #include <rte_branch_prediction.h>
18 #include "sfc_debug.h"
23 #include "sfc_kvargs.h"
26 /* Initial delay when waiting for event queue init complete event */
27 #define SFC_EVQ_INIT_BACKOFF_START_US (1)
28 /* Maximum delay between event queue polling attempts */
29 #define SFC_EVQ_INIT_BACKOFF_MAX_US (10 * 1000)
30 /* Event queue init approx timeout */
31 #define SFC_EVQ_INIT_TIMEOUT_US (2 * US_PER_S)
33 /* Management event queue polling period in microseconds */
34 #define SFC_MGMT_EV_QPOLL_PERIOD_US (US_PER_S)
37 sfc_evq_type2str(enum sfc_evq_type type
)
40 case SFC_EVQ_TYPE_MGMT
:
53 sfc_ev_initialized(void *arg
)
55 struct sfc_evq
*evq
= arg
;
57 /* Init done events may be duplicated on SFN7xxx (SFC bug 31631) */
58 SFC_ASSERT(evq
->init_state
== SFC_EVQ_STARTING
||
59 evq
->init_state
== SFC_EVQ_STARTED
);
61 evq
->init_state
= SFC_EVQ_STARTED
;
67 sfc_ev_nop_rx(void *arg
, uint32_t label
, uint32_t id
,
68 uint32_t size
, uint16_t flags
)
70 struct sfc_evq
*evq
= arg
;
73 "EVQ %u unexpected Rx event label=%u id=%#x size=%u flags=%#x",
74 evq
->evq_index
, label
, id
, size
, flags
);
79 sfc_ev_efx_rx(void *arg
, __rte_unused
uint32_t label
, uint32_t id
,
80 uint32_t size
, uint16_t flags
)
82 struct sfc_evq
*evq
= arg
;
83 struct sfc_efx_rxq
*rxq
;
85 unsigned int pending_id
;
88 struct sfc_efx_rx_sw_desc
*rxd
;
90 if (unlikely(evq
->exception
))
93 rxq
= sfc_efx_rxq_by_dp_rxq(evq
->dp_rxq
);
95 SFC_ASSERT(rxq
!= NULL
);
96 SFC_ASSERT(rxq
->evq
== evq
);
97 SFC_ASSERT(rxq
->flags
& SFC_EFX_RXQ_FLAG_STARTED
);
99 stop
= (id
+ 1) & rxq
->ptr_mask
;
100 pending_id
= rxq
->pending
& rxq
->ptr_mask
;
101 delta
= (stop
>= pending_id
) ? (stop
- pending_id
) :
102 (rxq
->ptr_mask
+ 1 - pending_id
+ stop
);
106 * Rx event with no new descriptors done and zero length
107 * is used to abort scattered packet when there is no room
110 if (unlikely(size
!= 0)) {
111 evq
->exception
= B_TRUE
;
113 "EVQ %u RxQ %u invalid RX abort "
114 "(id=%#x size=%u flags=%#x); needs restart",
115 evq
->evq_index
, rxq
->dp
.dpq
.queue_id
,
120 /* Add discard flag to the first fragment */
121 rxq
->sw_desc
[pending_id
].flags
|= EFX_DISCARD
;
122 /* Remove continue flag from the last fragment */
123 rxq
->sw_desc
[id
].flags
&= ~EFX_PKT_CONT
;
124 } else if (unlikely(delta
> rxq
->batch_max
)) {
125 evq
->exception
= B_TRUE
;
128 "EVQ %u RxQ %u completion out of order "
129 "(id=%#x delta=%u flags=%#x); needs restart",
130 evq
->evq_index
, rxq
->dp
.dpq
.queue_id
,
136 for (i
= pending_id
; i
!= stop
; i
= (i
+ 1) & rxq
->ptr_mask
) {
137 rxd
= &rxq
->sw_desc
[i
];
141 SFC_ASSERT(size
< (1 << 16));
142 rxd
->size
= (uint16_t)size
;
145 rxq
->pending
+= delta
;
152 sfc_ev_dp_rx(void *arg
, __rte_unused
uint32_t label
, uint32_t id
,
153 __rte_unused
uint32_t size
, __rte_unused
uint16_t flags
)
155 struct sfc_evq
*evq
= arg
;
156 struct sfc_dp_rxq
*dp_rxq
;
158 dp_rxq
= evq
->dp_rxq
;
159 SFC_ASSERT(dp_rxq
!= NULL
);
161 SFC_ASSERT(evq
->sa
->priv
.dp_rx
->qrx_ev
!= NULL
);
162 return evq
->sa
->priv
.dp_rx
->qrx_ev(dp_rxq
, id
);
166 sfc_ev_nop_rx_ps(void *arg
, uint32_t label
, uint32_t id
,
167 uint32_t pkt_count
, uint16_t flags
)
169 struct sfc_evq
*evq
= arg
;
172 "EVQ %u unexpected packed stream Rx event label=%u id=%#x pkt_count=%u flags=%#x",
173 evq
->evq_index
, label
, id
, pkt_count
, flags
);
177 /* It is not actually used on datapath, but required on RxQ flush */
179 sfc_ev_dp_rx_ps(void *arg
, __rte_unused
uint32_t label
, uint32_t id
,
180 __rte_unused
uint32_t pkt_count
, __rte_unused
uint16_t flags
)
182 struct sfc_evq
*evq
= arg
;
183 struct sfc_dp_rxq
*dp_rxq
;
185 dp_rxq
= evq
->dp_rxq
;
186 SFC_ASSERT(dp_rxq
!= NULL
);
188 if (evq
->sa
->priv
.dp_rx
->qrx_ps_ev
!= NULL
)
189 return evq
->sa
->priv
.dp_rx
->qrx_ps_ev(dp_rxq
, id
);
195 sfc_ev_nop_tx(void *arg
, uint32_t label
, uint32_t id
)
197 struct sfc_evq
*evq
= arg
;
199 sfc_err(evq
->sa
, "EVQ %u unexpected Tx event label=%u id=%#x",
200 evq
->evq_index
, label
, id
);
205 sfc_ev_tx(void *arg
, __rte_unused
uint32_t label
, uint32_t id
)
207 struct sfc_evq
*evq
= arg
;
208 struct sfc_dp_txq
*dp_txq
;
209 struct sfc_efx_txq
*txq
;
213 dp_txq
= evq
->dp_txq
;
214 SFC_ASSERT(dp_txq
!= NULL
);
216 txq
= sfc_efx_txq_by_dp_txq(dp_txq
);
217 SFC_ASSERT(txq
->evq
== evq
);
219 if (unlikely((txq
->flags
& SFC_EFX_TXQ_FLAG_STARTED
) == 0))
222 stop
= (id
+ 1) & txq
->ptr_mask
;
223 id
= txq
->pending
& txq
->ptr_mask
;
225 delta
= (stop
>= id
) ? (stop
- id
) : (txq
->ptr_mask
+ 1 - id
+ stop
);
227 txq
->pending
+= delta
;
234 sfc_ev_dp_tx(void *arg
, __rte_unused
uint32_t label
, uint32_t id
)
236 struct sfc_evq
*evq
= arg
;
237 struct sfc_dp_txq
*dp_txq
;
239 dp_txq
= evq
->dp_txq
;
240 SFC_ASSERT(dp_txq
!= NULL
);
242 SFC_ASSERT(evq
->sa
->priv
.dp_tx
->qtx_ev
!= NULL
);
243 return evq
->sa
->priv
.dp_tx
->qtx_ev(dp_txq
, id
);
247 sfc_ev_exception(void *arg
, uint32_t code
, __rte_unused
uint32_t data
)
249 struct sfc_evq
*evq
= arg
;
251 if (code
== EFX_EXCEPTION_UNKNOWN_SENSOREVT
)
254 evq
->exception
= B_TRUE
;
256 "hardware exception %s (code=%u, data=%#x) on EVQ %u;"
258 (code
== EFX_EXCEPTION_RX_RECOVERY
) ? "RX_RECOVERY" :
259 (code
== EFX_EXCEPTION_RX_DSC_ERROR
) ? "RX_DSC_ERROR" :
260 (code
== EFX_EXCEPTION_TX_DSC_ERROR
) ? "TX_DSC_ERROR" :
261 (code
== EFX_EXCEPTION_FWALERT_SRAM
) ? "FWALERT_SRAM" :
262 (code
== EFX_EXCEPTION_UNKNOWN_FWALERT
) ? "UNKNOWN_FWALERT" :
263 (code
== EFX_EXCEPTION_RX_ERROR
) ? "RX_ERROR" :
264 (code
== EFX_EXCEPTION_TX_ERROR
) ? "TX_ERROR" :
265 (code
== EFX_EXCEPTION_EV_ERROR
) ? "EV_ERROR" :
267 code
, data
, evq
->evq_index
);
273 sfc_ev_nop_rxq_flush_done(void *arg
, uint32_t rxq_hw_index
)
275 struct sfc_evq
*evq
= arg
;
277 sfc_err(evq
->sa
, "EVQ %u unexpected RxQ %u flush done",
278 evq
->evq_index
, rxq_hw_index
);
283 sfc_ev_rxq_flush_done(void *arg
, __rte_unused
uint32_t rxq_hw_index
)
285 struct sfc_evq
*evq
= arg
;
286 struct sfc_dp_rxq
*dp_rxq
;
289 dp_rxq
= evq
->dp_rxq
;
290 SFC_ASSERT(dp_rxq
!= NULL
);
292 rxq
= sfc_rxq_by_dp_rxq(dp_rxq
);
293 SFC_ASSERT(rxq
!= NULL
);
294 SFC_ASSERT(rxq
->hw_index
== rxq_hw_index
);
295 SFC_ASSERT(rxq
->evq
== evq
);
298 sfc_rx_qflush_done(sfc_rxq_info_by_dp_rxq(dp_rxq
));
304 sfc_ev_nop_rxq_flush_failed(void *arg
, uint32_t rxq_hw_index
)
306 struct sfc_evq
*evq
= arg
;
308 sfc_err(evq
->sa
, "EVQ %u unexpected RxQ %u flush failed",
309 evq
->evq_index
, rxq_hw_index
);
314 sfc_ev_rxq_flush_failed(void *arg
, __rte_unused
uint32_t rxq_hw_index
)
316 struct sfc_evq
*evq
= arg
;
317 struct sfc_dp_rxq
*dp_rxq
;
320 dp_rxq
= evq
->dp_rxq
;
321 SFC_ASSERT(dp_rxq
!= NULL
);
323 rxq
= sfc_rxq_by_dp_rxq(dp_rxq
);
324 SFC_ASSERT(rxq
!= NULL
);
325 SFC_ASSERT(rxq
->hw_index
== rxq_hw_index
);
326 SFC_ASSERT(rxq
->evq
== evq
);
329 sfc_rx_qflush_failed(sfc_rxq_info_by_dp_rxq(dp_rxq
));
335 sfc_ev_nop_txq_flush_done(void *arg
, uint32_t txq_hw_index
)
337 struct sfc_evq
*evq
= arg
;
339 sfc_err(evq
->sa
, "EVQ %u unexpected TxQ %u flush done",
340 evq
->evq_index
, txq_hw_index
);
345 sfc_ev_txq_flush_done(void *arg
, __rte_unused
uint32_t txq_hw_index
)
347 struct sfc_evq
*evq
= arg
;
348 struct sfc_dp_txq
*dp_txq
;
351 dp_txq
= evq
->dp_txq
;
352 SFC_ASSERT(dp_txq
!= NULL
);
354 txq
= sfc_txq_by_dp_txq(dp_txq
);
355 SFC_ASSERT(txq
!= NULL
);
356 SFC_ASSERT(txq
->hw_index
== txq_hw_index
);
357 SFC_ASSERT(txq
->evq
== evq
);
360 sfc_tx_qflush_done(sfc_txq_info_by_dp_txq(dp_txq
));
366 sfc_ev_software(void *arg
, uint16_t magic
)
368 struct sfc_evq
*evq
= arg
;
370 sfc_err(evq
->sa
, "EVQ %u unexpected software event magic=%#.4x",
371 evq
->evq_index
, magic
);
376 sfc_ev_sram(void *arg
, uint32_t code
)
378 struct sfc_evq
*evq
= arg
;
380 sfc_err(evq
->sa
, "EVQ %u unexpected SRAM event code=%u",
381 evq
->evq_index
, code
);
386 sfc_ev_wake_up(void *arg
, uint32_t index
)
388 struct sfc_evq
*evq
= arg
;
390 sfc_err(evq
->sa
, "EVQ %u unexpected wake up event index=%u",
391 evq
->evq_index
, index
);
396 sfc_ev_timer(void *arg
, uint32_t index
)
398 struct sfc_evq
*evq
= arg
;
400 sfc_err(evq
->sa
, "EVQ %u unexpected timer event index=%u",
401 evq
->evq_index
, index
);
406 sfc_ev_nop_link_change(void *arg
, __rte_unused efx_link_mode_t link_mode
)
408 struct sfc_evq
*evq
= arg
;
410 sfc_err(evq
->sa
, "EVQ %u unexpected link change event",
416 sfc_ev_link_change(void *arg
, efx_link_mode_t link_mode
)
418 struct sfc_evq
*evq
= arg
;
419 struct sfc_adapter
*sa
= evq
->sa
;
420 struct rte_eth_link new_link
;
422 sfc_port_link_mode_to_info(link_mode
, &new_link
);
423 if (rte_eth_linkstatus_set(sa
->eth_dev
, &new_link
))
424 evq
->sa
->port
.lsc_seq
++;
429 static const efx_ev_callbacks_t sfc_ev_callbacks
= {
430 .eec_initialized
= sfc_ev_initialized
,
431 .eec_rx
= sfc_ev_nop_rx
,
432 .eec_rx_ps
= sfc_ev_nop_rx_ps
,
433 .eec_tx
= sfc_ev_nop_tx
,
434 .eec_exception
= sfc_ev_exception
,
435 .eec_rxq_flush_done
= sfc_ev_nop_rxq_flush_done
,
436 .eec_rxq_flush_failed
= sfc_ev_nop_rxq_flush_failed
,
437 .eec_txq_flush_done
= sfc_ev_nop_txq_flush_done
,
438 .eec_software
= sfc_ev_software
,
439 .eec_sram
= sfc_ev_sram
,
440 .eec_wake_up
= sfc_ev_wake_up
,
441 .eec_timer
= sfc_ev_timer
,
442 .eec_link_change
= sfc_ev_link_change
,
445 static const efx_ev_callbacks_t sfc_ev_callbacks_efx_rx
= {
446 .eec_initialized
= sfc_ev_initialized
,
447 .eec_rx
= sfc_ev_efx_rx
,
448 .eec_rx_ps
= sfc_ev_nop_rx_ps
,
449 .eec_tx
= sfc_ev_nop_tx
,
450 .eec_exception
= sfc_ev_exception
,
451 .eec_rxq_flush_done
= sfc_ev_rxq_flush_done
,
452 .eec_rxq_flush_failed
= sfc_ev_rxq_flush_failed
,
453 .eec_txq_flush_done
= sfc_ev_nop_txq_flush_done
,
454 .eec_software
= sfc_ev_software
,
455 .eec_sram
= sfc_ev_sram
,
456 .eec_wake_up
= sfc_ev_wake_up
,
457 .eec_timer
= sfc_ev_timer
,
458 .eec_link_change
= sfc_ev_nop_link_change
,
461 static const efx_ev_callbacks_t sfc_ev_callbacks_dp_rx
= {
462 .eec_initialized
= sfc_ev_initialized
,
463 .eec_rx
= sfc_ev_dp_rx
,
464 .eec_rx_ps
= sfc_ev_dp_rx_ps
,
465 .eec_tx
= sfc_ev_nop_tx
,
466 .eec_exception
= sfc_ev_exception
,
467 .eec_rxq_flush_done
= sfc_ev_rxq_flush_done
,
468 .eec_rxq_flush_failed
= sfc_ev_rxq_flush_failed
,
469 .eec_txq_flush_done
= sfc_ev_nop_txq_flush_done
,
470 .eec_software
= sfc_ev_software
,
471 .eec_sram
= sfc_ev_sram
,
472 .eec_wake_up
= sfc_ev_wake_up
,
473 .eec_timer
= sfc_ev_timer
,
474 .eec_link_change
= sfc_ev_nop_link_change
,
477 static const efx_ev_callbacks_t sfc_ev_callbacks_efx_tx
= {
478 .eec_initialized
= sfc_ev_initialized
,
479 .eec_rx
= sfc_ev_nop_rx
,
480 .eec_rx_ps
= sfc_ev_nop_rx_ps
,
482 .eec_exception
= sfc_ev_exception
,
483 .eec_rxq_flush_done
= sfc_ev_nop_rxq_flush_done
,
484 .eec_rxq_flush_failed
= sfc_ev_nop_rxq_flush_failed
,
485 .eec_txq_flush_done
= sfc_ev_txq_flush_done
,
486 .eec_software
= sfc_ev_software
,
487 .eec_sram
= sfc_ev_sram
,
488 .eec_wake_up
= sfc_ev_wake_up
,
489 .eec_timer
= sfc_ev_timer
,
490 .eec_link_change
= sfc_ev_nop_link_change
,
493 static const efx_ev_callbacks_t sfc_ev_callbacks_dp_tx
= {
494 .eec_initialized
= sfc_ev_initialized
,
495 .eec_rx
= sfc_ev_nop_rx
,
496 .eec_rx_ps
= sfc_ev_nop_rx_ps
,
497 .eec_tx
= sfc_ev_dp_tx
,
498 .eec_exception
= sfc_ev_exception
,
499 .eec_rxq_flush_done
= sfc_ev_nop_rxq_flush_done
,
500 .eec_rxq_flush_failed
= sfc_ev_nop_rxq_flush_failed
,
501 .eec_txq_flush_done
= sfc_ev_txq_flush_done
,
502 .eec_software
= sfc_ev_software
,
503 .eec_sram
= sfc_ev_sram
,
504 .eec_wake_up
= sfc_ev_wake_up
,
505 .eec_timer
= sfc_ev_timer
,
506 .eec_link_change
= sfc_ev_nop_link_change
,
511 sfc_ev_qpoll(struct sfc_evq
*evq
)
513 SFC_ASSERT(evq
->init_state
== SFC_EVQ_STARTED
||
514 evq
->init_state
== SFC_EVQ_STARTING
);
516 /* Synchronize the DMA memory for reading not required */
518 efx_ev_qpoll(evq
->common
, &evq
->read_ptr
, evq
->callbacks
, evq
);
520 if (unlikely(evq
->exception
) && sfc_adapter_trylock(evq
->sa
)) {
521 struct sfc_adapter
*sa
= evq
->sa
;
524 if (evq
->dp_rxq
!= NULL
) {
525 unsigned int rxq_sw_index
;
527 rxq_sw_index
= evq
->dp_rxq
->dpq
.queue_id
;
530 "restart RxQ %u because of exception on its EvQ %u",
531 rxq_sw_index
, evq
->evq_index
);
533 sfc_rx_qstop(sa
, rxq_sw_index
);
534 rc
= sfc_rx_qstart(sa
, rxq_sw_index
);
536 sfc_err(sa
, "cannot restart RxQ %u",
540 if (evq
->dp_txq
!= NULL
) {
541 unsigned int txq_sw_index
;
543 txq_sw_index
= evq
->dp_txq
->dpq
.queue_id
;
546 "restart TxQ %u because of exception on its EvQ %u",
547 txq_sw_index
, evq
->evq_index
);
549 sfc_tx_qstop(sa
, txq_sw_index
);
550 rc
= sfc_tx_qstart(sa
, txq_sw_index
);
552 sfc_err(sa
, "cannot restart TxQ %u",
557 sfc_panic(sa
, "unrecoverable exception on EvQ %u",
560 sfc_adapter_unlock(sa
);
563 /* Poll-mode driver does not re-prime the event queue for interrupts */
567 sfc_ev_mgmt_qpoll(struct sfc_adapter
*sa
)
569 if (rte_spinlock_trylock(&sa
->mgmt_evq_lock
)) {
570 if (sa
->mgmt_evq_running
)
571 sfc_ev_qpoll(sa
->mgmt_evq
);
573 rte_spinlock_unlock(&sa
->mgmt_evq_lock
);
578 sfc_ev_qprime(struct sfc_evq
*evq
)
580 SFC_ASSERT(evq
->init_state
== SFC_EVQ_STARTED
);
581 return efx_ev_qprime(evq
->common
, evq
->read_ptr
);
584 /* Event queue HW index allocation scheme is described in sfc_ev.h. */
586 sfc_ev_qstart(struct sfc_evq
*evq
, unsigned int hw_index
)
588 struct sfc_adapter
*sa
= evq
->sa
;
590 uint32_t evq_flags
= sa
->evq_flags
;
591 unsigned int total_delay_us
;
592 unsigned int delay_us
;
595 sfc_log_init(sa
, "hw_index=%u", hw_index
);
599 evq
->evq_index
= hw_index
;
601 /* Clear all events */
602 (void)memset((void *)esmp
->esm_base
, 0xff,
603 efx_evq_size(sa
->nic
, evq
->entries
));
605 if (sa
->intr
.lsc_intr
&& hw_index
== sa
->mgmt_evq_index
)
606 evq_flags
|= EFX_EVQ_FLAGS_NOTIFY_INTERRUPT
;
608 evq_flags
|= EFX_EVQ_FLAGS_NOTIFY_DISABLED
;
610 /* Create the common code event queue */
611 rc
= efx_ev_qcreate(sa
->nic
, hw_index
, esmp
, evq
->entries
,
612 0 /* unused on EF10 */, 0, evq_flags
,
615 goto fail_ev_qcreate
;
617 SFC_ASSERT(evq
->dp_rxq
== NULL
|| evq
->dp_txq
== NULL
);
618 if (evq
->dp_rxq
!= 0) {
619 if (strcmp(sa
->priv
.dp_rx
->dp
.name
,
620 SFC_KVARG_DATAPATH_EFX
) == 0)
621 evq
->callbacks
= &sfc_ev_callbacks_efx_rx
;
623 evq
->callbacks
= &sfc_ev_callbacks_dp_rx
;
624 } else if (evq
->dp_txq
!= 0) {
625 if (strcmp(sa
->priv
.dp_tx
->dp
.name
,
626 SFC_KVARG_DATAPATH_EFX
) == 0)
627 evq
->callbacks
= &sfc_ev_callbacks_efx_tx
;
629 evq
->callbacks
= &sfc_ev_callbacks_dp_tx
;
631 evq
->callbacks
= &sfc_ev_callbacks
;
634 evq
->init_state
= SFC_EVQ_STARTING
;
636 /* Wait for the initialization event */
638 delay_us
= SFC_EVQ_INIT_BACKOFF_START_US
;
640 (void)sfc_ev_qpoll(evq
);
642 /* Check to see if the initialization complete indication
643 * posted by the hardware.
645 if (evq
->init_state
== SFC_EVQ_STARTED
)
648 /* Give event queue some time to init */
649 rte_delay_us(delay_us
);
651 total_delay_us
+= delay_us
;
653 /* Exponential backoff */
655 if (delay_us
> SFC_EVQ_INIT_BACKOFF_MAX_US
)
656 delay_us
= SFC_EVQ_INIT_BACKOFF_MAX_US
;
658 } while (total_delay_us
< SFC_EVQ_INIT_TIMEOUT_US
);
667 evq
->init_state
= SFC_EVQ_INITIALIZED
;
668 efx_ev_qdestroy(evq
->common
);
671 sfc_log_init(sa
, "failed %d", rc
);
676 sfc_ev_qstop(struct sfc_evq
*evq
)
681 sfc_log_init(evq
->sa
, "hw_index=%u", evq
->evq_index
);
683 if (evq
->init_state
!= SFC_EVQ_STARTED
)
686 evq
->init_state
= SFC_EVQ_INITIALIZED
;
687 evq
->callbacks
= NULL
;
689 evq
->exception
= B_FALSE
;
691 efx_ev_qdestroy(evq
->common
);
697 sfc_ev_mgmt_periodic_qpoll(void *arg
)
699 struct sfc_adapter
*sa
= arg
;
702 sfc_ev_mgmt_qpoll(sa
);
704 rc
= rte_eal_alarm_set(SFC_MGMT_EV_QPOLL_PERIOD_US
,
705 sfc_ev_mgmt_periodic_qpoll
, sa
);
706 if (rc
== -ENOTSUP
) {
707 sfc_warn(sa
, "alarms are not supported");
708 sfc_warn(sa
, "management EVQ must be polled indirectly using no-wait link status update");
709 } else if (rc
!= 0) {
711 "cannot rearm management EVQ polling alarm (rc=%d)",
717 sfc_ev_mgmt_periodic_qpoll_start(struct sfc_adapter
*sa
)
719 sfc_ev_mgmt_periodic_qpoll(sa
);
723 sfc_ev_mgmt_periodic_qpoll_stop(struct sfc_adapter
*sa
)
725 rte_eal_alarm_cancel(sfc_ev_mgmt_periodic_qpoll
, sa
);
729 sfc_ev_start(struct sfc_adapter
*sa
)
733 sfc_log_init(sa
, "entry");
735 rc
= efx_ev_init(sa
->nic
);
739 /* Start management EVQ used for global events */
742 * Management event queue start polls the queue, but it cannot
743 * interfere with other polling contexts since mgmt_evq_running
746 rc
= sfc_ev_qstart(sa
->mgmt_evq
, sa
->mgmt_evq_index
);
748 goto fail_mgmt_evq_start
;
750 rte_spinlock_lock(&sa
->mgmt_evq_lock
);
751 sa
->mgmt_evq_running
= true;
752 rte_spinlock_unlock(&sa
->mgmt_evq_lock
);
754 if (sa
->intr
.lsc_intr
) {
755 rc
= sfc_ev_qprime(sa
->mgmt_evq
);
757 goto fail_mgmt_evq_prime
;
761 * Start management EVQ polling. If interrupts are disabled
762 * (not used), it is required to process link status change
763 * and other device level events to avoid unrecoverable
764 * error because the event queue overflow.
766 sfc_ev_mgmt_periodic_qpoll_start(sa
);
769 * Rx/Tx event queues are started/stopped when corresponding
770 * Rx/Tx queue is started/stopped.
776 sfc_ev_qstop(sa
->mgmt_evq
);
779 efx_ev_fini(sa
->nic
);
782 sfc_log_init(sa
, "failed %d", rc
);
787 sfc_ev_stop(struct sfc_adapter
*sa
)
789 sfc_log_init(sa
, "entry");
791 sfc_ev_mgmt_periodic_qpoll_stop(sa
);
793 rte_spinlock_lock(&sa
->mgmt_evq_lock
);
794 sa
->mgmt_evq_running
= false;
795 rte_spinlock_unlock(&sa
->mgmt_evq_lock
);
797 sfc_ev_qstop(sa
->mgmt_evq
);
799 efx_ev_fini(sa
->nic
);
803 sfc_ev_qinit(struct sfc_adapter
*sa
,
804 enum sfc_evq_type type
, unsigned int type_index
,
805 unsigned int entries
, int socket_id
, struct sfc_evq
**evqp
)
810 sfc_log_init(sa
, "type=%s type_index=%u",
811 sfc_evq_type2str(type
), type_index
);
813 SFC_ASSERT(rte_is_power_of_2(entries
));
816 evq
= rte_zmalloc_socket("sfc-evq", sizeof(*evq
), RTE_CACHE_LINE_SIZE
,
823 evq
->entries
= entries
;
825 /* Allocate DMA space */
826 rc
= sfc_dma_alloc(sa
, sfc_evq_type2str(type
), type_index
,
827 efx_evq_size(sa
->nic
, evq
->entries
), socket_id
,
832 evq
->init_state
= SFC_EVQ_INITIALIZED
;
845 sfc_log_init(sa
, "failed %d", rc
);
850 sfc_ev_qfini(struct sfc_evq
*evq
)
852 struct sfc_adapter
*sa
= evq
->sa
;
854 SFC_ASSERT(evq
->init_state
== SFC_EVQ_INITIALIZED
);
856 sfc_dma_free(sa
, &evq
->mem
);
860 SFC_ASSERT(sa
->evq_count
> 0);
865 sfc_kvarg_perf_profile_handler(__rte_unused
const char *key
,
866 const char *value_str
, void *opaque
)
868 uint32_t *value
= opaque
;
870 if (strcasecmp(value_str
, SFC_KVARG_PERF_PROFILE_THROUGHPUT
) == 0)
871 *value
= EFX_EVQ_FLAGS_TYPE_THROUGHPUT
;
872 else if (strcasecmp(value_str
, SFC_KVARG_PERF_PROFILE_LOW_LATENCY
) == 0)
873 *value
= EFX_EVQ_FLAGS_TYPE_LOW_LATENCY
;
874 else if (strcasecmp(value_str
, SFC_KVARG_PERF_PROFILE_AUTO
) == 0)
875 *value
= EFX_EVQ_FLAGS_TYPE_AUTO
;
883 sfc_ev_attach(struct sfc_adapter
*sa
)
887 sfc_log_init(sa
, "entry");
889 sa
->evq_flags
= EFX_EVQ_FLAGS_TYPE_THROUGHPUT
;
890 rc
= sfc_kvargs_process(sa
, SFC_KVARG_PERF_PROFILE
,
891 sfc_kvarg_perf_profile_handler
,
894 sfc_err(sa
, "invalid %s parameter value",
895 SFC_KVARG_PERF_PROFILE
);
896 goto fail_kvarg_perf_profile
;
899 sa
->mgmt_evq_index
= 0;
900 rte_spinlock_init(&sa
->mgmt_evq_lock
);
902 rc
= sfc_ev_qinit(sa
, SFC_EVQ_TYPE_MGMT
, 0, sa
->evq_min_entries
,
903 sa
->socket_id
, &sa
->mgmt_evq
);
905 goto fail_mgmt_evq_init
;
908 * Rx/Tx event queues are created/destroyed when corresponding
909 * Rx/Tx queue is created/destroyed.
916 fail_kvarg_perf_profile
:
917 sfc_log_init(sa
, "failed %d", rc
);
922 sfc_ev_detach(struct sfc_adapter
*sa
)
924 sfc_log_init(sa
, "entry");
926 sfc_ev_qfini(sa
->mgmt_evq
);
928 if (sa
->evq_count
!= 0)
929 sfc_err(sa
, "%u EvQs are not destroyed before detach",