1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2007-2018 Solarflare Communications Inc.
11 #define EFX_TX_QSTAT_INCR(_etp, _stat) \
13 (_etp)->et_stat[_stat]++; \
14 _NOTE(CONSTANTCONDITION) \
17 #define EFX_TX_QSTAT_INCR(_etp, _stat)
22 static __checkReturn efx_rc_t
30 static __checkReturn efx_rc_t
33 __in
unsigned int index
,
34 __in
unsigned int label
,
35 __in efsys_mem_t
*esmp
,
41 __out
unsigned int *addedp
);
47 static __checkReturn efx_rc_t
50 __in_ecount(ndescs
) efx_buffer_t
*eb
,
51 __in
unsigned int ndescs
,
52 __in
unsigned int completed
,
53 __inout
unsigned int *addedp
);
58 __in
unsigned int added
,
59 __in
unsigned int pushed
);
61 static __checkReturn efx_rc_t
64 __in
unsigned int ns
);
66 static __checkReturn efx_rc_t
74 __checkReturn efx_rc_t
77 __in_ecount(ndescs
) efx_desc_t
*ed
,
78 __in
unsigned int ndescs
,
79 __in
unsigned int completed
,
80 __inout
unsigned int *addedp
);
83 siena_tx_qdesc_dma_create(
85 __in efsys_dma_addr_t addr
,
88 __out efx_desc_t
*edp
);
92 siena_tx_qstats_update(
94 __inout_ecount(TX_NQSTATS
) efsys_stat_t
*stat
);
97 #endif /* EFSYS_OPT_SIENA */
101 static const efx_tx_ops_t __efx_tx_siena_ops
= {
102 siena_tx_init
, /* etxo_init */
103 siena_tx_fini
, /* etxo_fini */
104 siena_tx_qcreate
, /* etxo_qcreate */
105 siena_tx_qdestroy
, /* etxo_qdestroy */
106 siena_tx_qpost
, /* etxo_qpost */
107 siena_tx_qpush
, /* etxo_qpush */
108 siena_tx_qpace
, /* etxo_qpace */
109 siena_tx_qflush
, /* etxo_qflush */
110 siena_tx_qenable
, /* etxo_qenable */
111 NULL
, /* etxo_qpio_enable */
112 NULL
, /* etxo_qpio_disable */
113 NULL
, /* etxo_qpio_write */
114 NULL
, /* etxo_qpio_post */
115 siena_tx_qdesc_post
, /* etxo_qdesc_post */
116 siena_tx_qdesc_dma_create
, /* etxo_qdesc_dma_create */
117 NULL
, /* etxo_qdesc_tso_create */
118 NULL
, /* etxo_qdesc_tso2_create */
119 NULL
, /* etxo_qdesc_vlantci_create */
120 NULL
, /* etxo_qdesc_checksum_create */
122 siena_tx_qstats_update
, /* etxo_qstats_update */
125 #endif /* EFSYS_OPT_SIENA */
127 #if EFSYS_OPT_HUNTINGTON
128 static const efx_tx_ops_t __efx_tx_hunt_ops
= {
129 ef10_tx_init
, /* etxo_init */
130 ef10_tx_fini
, /* etxo_fini */
131 ef10_tx_qcreate
, /* etxo_qcreate */
132 ef10_tx_qdestroy
, /* etxo_qdestroy */
133 ef10_tx_qpost
, /* etxo_qpost */
134 ef10_tx_qpush
, /* etxo_qpush */
135 ef10_tx_qpace
, /* etxo_qpace */
136 ef10_tx_qflush
, /* etxo_qflush */
137 ef10_tx_qenable
, /* etxo_qenable */
138 ef10_tx_qpio_enable
, /* etxo_qpio_enable */
139 ef10_tx_qpio_disable
, /* etxo_qpio_disable */
140 ef10_tx_qpio_write
, /* etxo_qpio_write */
141 ef10_tx_qpio_post
, /* etxo_qpio_post */
142 ef10_tx_qdesc_post
, /* etxo_qdesc_post */
143 ef10_tx_qdesc_dma_create
, /* etxo_qdesc_dma_create */
144 ef10_tx_qdesc_tso_create
, /* etxo_qdesc_tso_create */
145 ef10_tx_qdesc_tso2_create
, /* etxo_qdesc_tso2_create */
146 ef10_tx_qdesc_vlantci_create
, /* etxo_qdesc_vlantci_create */
147 ef10_tx_qdesc_checksum_create
, /* etxo_qdesc_checksum_create */
149 ef10_tx_qstats_update
, /* etxo_qstats_update */
152 #endif /* EFSYS_OPT_HUNTINGTON */
154 #if EFSYS_OPT_MEDFORD
155 static const efx_tx_ops_t __efx_tx_medford_ops
= {
156 ef10_tx_init
, /* etxo_init */
157 ef10_tx_fini
, /* etxo_fini */
158 ef10_tx_qcreate
, /* etxo_qcreate */
159 ef10_tx_qdestroy
, /* etxo_qdestroy */
160 ef10_tx_qpost
, /* etxo_qpost */
161 ef10_tx_qpush
, /* etxo_qpush */
162 ef10_tx_qpace
, /* etxo_qpace */
163 ef10_tx_qflush
, /* etxo_qflush */
164 ef10_tx_qenable
, /* etxo_qenable */
165 ef10_tx_qpio_enable
, /* etxo_qpio_enable */
166 ef10_tx_qpio_disable
, /* etxo_qpio_disable */
167 ef10_tx_qpio_write
, /* etxo_qpio_write */
168 ef10_tx_qpio_post
, /* etxo_qpio_post */
169 ef10_tx_qdesc_post
, /* etxo_qdesc_post */
170 ef10_tx_qdesc_dma_create
, /* etxo_qdesc_dma_create */
171 NULL
, /* etxo_qdesc_tso_create */
172 ef10_tx_qdesc_tso2_create
, /* etxo_qdesc_tso2_create */
173 ef10_tx_qdesc_vlantci_create
, /* etxo_qdesc_vlantci_create */
174 ef10_tx_qdesc_checksum_create
, /* etxo_qdesc_checksum_create */
176 ef10_tx_qstats_update
, /* etxo_qstats_update */
179 #endif /* EFSYS_OPT_MEDFORD */
181 #if EFSYS_OPT_MEDFORD2
182 static const efx_tx_ops_t __efx_tx_medford2_ops
= {
183 ef10_tx_init
, /* etxo_init */
184 ef10_tx_fini
, /* etxo_fini */
185 ef10_tx_qcreate
, /* etxo_qcreate */
186 ef10_tx_qdestroy
, /* etxo_qdestroy */
187 ef10_tx_qpost
, /* etxo_qpost */
188 ef10_tx_qpush
, /* etxo_qpush */
189 ef10_tx_qpace
, /* etxo_qpace */
190 ef10_tx_qflush
, /* etxo_qflush */
191 ef10_tx_qenable
, /* etxo_qenable */
192 ef10_tx_qpio_enable
, /* etxo_qpio_enable */
193 ef10_tx_qpio_disable
, /* etxo_qpio_disable */
194 ef10_tx_qpio_write
, /* etxo_qpio_write */
195 ef10_tx_qpio_post
, /* etxo_qpio_post */
196 ef10_tx_qdesc_post
, /* etxo_qdesc_post */
197 ef10_tx_qdesc_dma_create
, /* etxo_qdesc_dma_create */
198 NULL
, /* etxo_qdesc_tso_create */
199 ef10_tx_qdesc_tso2_create
, /* etxo_qdesc_tso2_create */
200 ef10_tx_qdesc_vlantci_create
, /* etxo_qdesc_vlantci_create */
201 ef10_tx_qdesc_checksum_create
, /* etxo_qdesc_checksum_create */
203 ef10_tx_qstats_update
, /* etxo_qstats_update */
206 #endif /* EFSYS_OPT_MEDFORD2 */
209 __checkReturn efx_rc_t
213 const efx_tx_ops_t
*etxop
;
216 EFSYS_ASSERT3U(enp
->en_magic
, ==, EFX_NIC_MAGIC
);
217 EFSYS_ASSERT3U(enp
->en_mod_flags
, &, EFX_MOD_NIC
);
219 if (!(enp
->en_mod_flags
& EFX_MOD_EV
)) {
224 if (enp
->en_mod_flags
& EFX_MOD_TX
) {
229 switch (enp
->en_family
) {
231 case EFX_FAMILY_SIENA
:
232 etxop
= &__efx_tx_siena_ops
;
234 #endif /* EFSYS_OPT_SIENA */
236 #if EFSYS_OPT_HUNTINGTON
237 case EFX_FAMILY_HUNTINGTON
:
238 etxop
= &__efx_tx_hunt_ops
;
240 #endif /* EFSYS_OPT_HUNTINGTON */
242 #if EFSYS_OPT_MEDFORD
243 case EFX_FAMILY_MEDFORD
:
244 etxop
= &__efx_tx_medford_ops
;
246 #endif /* EFSYS_OPT_MEDFORD */
248 #if EFSYS_OPT_MEDFORD2
249 case EFX_FAMILY_MEDFORD2
:
250 etxop
= &__efx_tx_medford2_ops
;
252 #endif /* EFSYS_OPT_MEDFORD2 */
260 EFSYS_ASSERT3U(enp
->en_tx_qcount
, ==, 0);
262 if ((rc
= etxop
->etxo_init(enp
)) != 0)
265 enp
->en_etxop
= etxop
;
266 enp
->en_mod_flags
|= EFX_MOD_TX
;
276 EFSYS_PROBE1(fail1
, efx_rc_t
, rc
);
278 enp
->en_etxop
= NULL
;
279 enp
->en_mod_flags
&= ~EFX_MOD_TX
;
287 const efx_tx_ops_t
*etxop
= enp
->en_etxop
;
289 EFSYS_ASSERT3U(enp
->en_magic
, ==, EFX_NIC_MAGIC
);
290 EFSYS_ASSERT3U(enp
->en_mod_flags
, &, EFX_MOD_NIC
);
291 EFSYS_ASSERT3U(enp
->en_mod_flags
, &, EFX_MOD_TX
);
292 EFSYS_ASSERT3U(enp
->en_tx_qcount
, ==, 0);
294 etxop
->etxo_fini(enp
);
296 enp
->en_etxop
= NULL
;
297 enp
->en_mod_flags
&= ~EFX_MOD_TX
;
302 __in
const efx_nic_t
*enp
,
303 __in
unsigned int ndescs
)
305 const efx_nic_cfg_t
*encp
= efx_nic_cfg_get(enp
);
307 return (ndescs
* encp
->enc_tx_desc_size
);
310 __checkReturn
unsigned int
312 __in
const efx_nic_t
*enp
,
313 __in
unsigned int ndescs
)
315 return (EFX_DIV_ROUND_UP(efx_txq_size(enp
, ndescs
), EFX_BUF_SIZE
));
318 __checkReturn efx_rc_t
321 __in
unsigned int index
,
322 __in
unsigned int label
,
323 __in efsys_mem_t
*esmp
,
328 __deref_out efx_txq_t
**etpp
,
329 __out
unsigned int *addedp
)
331 const efx_tx_ops_t
*etxop
= enp
->en_etxop
;
333 const efx_nic_cfg_t
*encp
= efx_nic_cfg_get(enp
);
336 EFSYS_ASSERT3U(enp
->en_magic
, ==, EFX_NIC_MAGIC
);
337 EFSYS_ASSERT3U(enp
->en_mod_flags
, &, EFX_MOD_TX
);
339 EFSYS_ASSERT3U(enp
->en_tx_qcount
+ 1, <,
340 enp
->en_nic_cfg
.enc_txq_limit
);
342 EFSYS_ASSERT(ISP2(encp
->enc_txq_max_ndescs
));
343 EFSYS_ASSERT(ISP2(encp
->enc_txq_min_ndescs
));
346 ndescs
< encp
->enc_txq_min_ndescs
||
347 ndescs
> encp
->enc_txq_max_ndescs
) {
352 /* Allocate an TXQ object */
353 EFSYS_KMEM_ALLOC(enp
->en_esip
, sizeof (efx_txq_t
), etp
);
360 etp
->et_magic
= EFX_TXQ_MAGIC
;
362 etp
->et_index
= index
;
363 etp
->et_mask
= ndescs
- 1;
366 /* Initial descriptor index may be modified by etxo_qcreate */
369 if ((rc
= etxop
->etxo_qcreate(enp
, index
, label
, esmp
,
370 ndescs
, id
, flags
, eep
, etp
, addedp
)) != 0)
380 EFSYS_KMEM_FREE(enp
->en_esip
, sizeof (efx_txq_t
), etp
);
384 EFSYS_PROBE1(fail1
, efx_rc_t
, rc
);
392 efx_nic_t
*enp
= etp
->et_enp
;
393 const efx_tx_ops_t
*etxop
= enp
->en_etxop
;
395 EFSYS_ASSERT3U(etp
->et_magic
, ==, EFX_TXQ_MAGIC
);
397 EFSYS_ASSERT(enp
->en_tx_qcount
!= 0);
400 etxop
->etxo_qdestroy(etp
);
402 /* Free the TXQ object */
403 EFSYS_KMEM_FREE(enp
->en_esip
, sizeof (efx_txq_t
), etp
);
406 __checkReturn efx_rc_t
409 __in_ecount(ndescs
) efx_buffer_t
*eb
,
410 __in
unsigned int ndescs
,
411 __in
unsigned int completed
,
412 __inout
unsigned int *addedp
)
414 efx_nic_t
*enp
= etp
->et_enp
;
415 const efx_tx_ops_t
*etxop
= enp
->en_etxop
;
418 EFSYS_ASSERT3U(etp
->et_magic
, ==, EFX_TXQ_MAGIC
);
420 if ((rc
= etxop
->etxo_qpost(etp
, eb
, ndescs
, completed
, addedp
)) != 0)
426 EFSYS_PROBE1(fail1
, efx_rc_t
, rc
);
433 __in
unsigned int added
,
434 __in
unsigned int pushed
)
436 efx_nic_t
*enp
= etp
->et_enp
;
437 const efx_tx_ops_t
*etxop
= enp
->en_etxop
;
439 EFSYS_ASSERT3U(etp
->et_magic
, ==, EFX_TXQ_MAGIC
);
441 etxop
->etxo_qpush(etp
, added
, pushed
);
444 __checkReturn efx_rc_t
447 __in
unsigned int ns
)
449 efx_nic_t
*enp
= etp
->et_enp
;
450 const efx_tx_ops_t
*etxop
= enp
->en_etxop
;
453 EFSYS_ASSERT3U(etp
->et_magic
, ==, EFX_TXQ_MAGIC
);
455 if ((rc
= etxop
->etxo_qpace(etp
, ns
)) != 0)
461 EFSYS_PROBE1(fail1
, efx_rc_t
, rc
);
465 __checkReturn efx_rc_t
469 efx_nic_t
*enp
= etp
->et_enp
;
470 const efx_tx_ops_t
*etxop
= enp
->en_etxop
;
473 EFSYS_ASSERT3U(etp
->et_magic
, ==, EFX_TXQ_MAGIC
);
475 if ((rc
= etxop
->etxo_qflush(etp
)) != 0)
481 EFSYS_PROBE1(fail1
, efx_rc_t
, rc
);
489 efx_nic_t
*enp
= etp
->et_enp
;
490 const efx_tx_ops_t
*etxop
= enp
->en_etxop
;
492 EFSYS_ASSERT3U(etp
->et_magic
, ==, EFX_TXQ_MAGIC
);
494 etxop
->etxo_qenable(etp
);
497 __checkReturn efx_rc_t
501 efx_nic_t
*enp
= etp
->et_enp
;
502 const efx_tx_ops_t
*etxop
= enp
->en_etxop
;
505 EFSYS_ASSERT3U(etp
->et_magic
, ==, EFX_TXQ_MAGIC
);
507 if (~enp
->en_features
& EFX_FEATURE_PIO_BUFFERS
) {
511 if (etxop
->etxo_qpio_enable
== NULL
) {
515 if ((rc
= etxop
->etxo_qpio_enable(etp
)) != 0)
525 EFSYS_PROBE1(fail1
, efx_rc_t
, rc
);
533 efx_nic_t
*enp
= etp
->et_enp
;
534 const efx_tx_ops_t
*etxop
= enp
->en_etxop
;
536 EFSYS_ASSERT3U(etp
->et_magic
, ==, EFX_TXQ_MAGIC
);
538 if (etxop
->etxo_qpio_disable
!= NULL
)
539 etxop
->etxo_qpio_disable(etp
);
542 __checkReturn efx_rc_t
545 __in_ecount(buf_length
) uint8_t *buffer
,
546 __in
size_t buf_length
,
547 __in
size_t pio_buf_offset
)
549 efx_nic_t
*enp
= etp
->et_enp
;
550 const efx_tx_ops_t
*etxop
= enp
->en_etxop
;
553 EFSYS_ASSERT3U(etp
->et_magic
, ==, EFX_TXQ_MAGIC
);
555 if (etxop
->etxo_qpio_write
!= NULL
) {
556 if ((rc
= etxop
->etxo_qpio_write(etp
, buffer
, buf_length
,
557 pio_buf_offset
)) != 0)
565 EFSYS_PROBE1(fail1
, efx_rc_t
, rc
);
569 __checkReturn efx_rc_t
572 __in
size_t pkt_length
,
573 __in
unsigned int completed
,
574 __inout
unsigned int *addedp
)
576 efx_nic_t
*enp
= etp
->et_enp
;
577 const efx_tx_ops_t
*etxop
= enp
->en_etxop
;
580 EFSYS_ASSERT3U(etp
->et_magic
, ==, EFX_TXQ_MAGIC
);
582 if (etxop
->etxo_qpio_post
!= NULL
) {
583 if ((rc
= etxop
->etxo_qpio_post(etp
, pkt_length
, completed
,
592 EFSYS_PROBE1(fail1
, efx_rc_t
, rc
);
596 __checkReturn efx_rc_t
599 __in_ecount(ndescs
) efx_desc_t
*ed
,
600 __in
unsigned int ndescs
,
601 __in
unsigned int completed
,
602 __inout
unsigned int *addedp
)
604 efx_nic_t
*enp
= etp
->et_enp
;
605 const efx_tx_ops_t
*etxop
= enp
->en_etxop
;
607 EFSYS_ASSERT3U(etp
->et_magic
, ==, EFX_TXQ_MAGIC
);
609 return (etxop
->etxo_qdesc_post(etp
, ed
, ndescs
, completed
, addedp
));
613 efx_tx_qdesc_dma_create(
615 __in efsys_dma_addr_t addr
,
618 __out efx_desc_t
*edp
)
620 efx_nic_t
*enp
= etp
->et_enp
;
621 const efx_tx_ops_t
*etxop
= enp
->en_etxop
;
623 EFSYS_ASSERT3U(etp
->et_magic
, ==, EFX_TXQ_MAGIC
);
624 EFSYS_ASSERT(etxop
->etxo_qdesc_dma_create
!= NULL
);
626 etxop
->etxo_qdesc_dma_create(etp
, addr
, size
, eop
, edp
);
630 efx_tx_qdesc_tso_create(
632 __in
uint16_t ipv4_id
,
633 __in
uint32_t tcp_seq
,
634 __in
uint8_t tcp_flags
,
635 __out efx_desc_t
*edp
)
637 efx_nic_t
*enp
= etp
->et_enp
;
638 const efx_tx_ops_t
*etxop
= enp
->en_etxop
;
640 EFSYS_ASSERT3U(etp
->et_magic
, ==, EFX_TXQ_MAGIC
);
641 EFSYS_ASSERT(etxop
->etxo_qdesc_tso_create
!= NULL
);
643 etxop
->etxo_qdesc_tso_create(etp
, ipv4_id
, tcp_seq
, tcp_flags
, edp
);
647 efx_tx_qdesc_tso2_create(
649 __in
uint16_t ipv4_id
,
650 __in
uint16_t outer_ipv4_id
,
651 __in
uint32_t tcp_seq
,
653 __out_ecount(count
) efx_desc_t
*edp
,
656 efx_nic_t
*enp
= etp
->et_enp
;
657 const efx_tx_ops_t
*etxop
= enp
->en_etxop
;
659 EFSYS_ASSERT3U(etp
->et_magic
, ==, EFX_TXQ_MAGIC
);
660 EFSYS_ASSERT(etxop
->etxo_qdesc_tso2_create
!= NULL
);
662 etxop
->etxo_qdesc_tso2_create(etp
, ipv4_id
, outer_ipv4_id
,
663 tcp_seq
, mss
, edp
, count
);
667 efx_tx_qdesc_vlantci_create(
670 __out efx_desc_t
*edp
)
672 efx_nic_t
*enp
= etp
->et_enp
;
673 const efx_tx_ops_t
*etxop
= enp
->en_etxop
;
675 EFSYS_ASSERT3U(etp
->et_magic
, ==, EFX_TXQ_MAGIC
);
676 EFSYS_ASSERT(etxop
->etxo_qdesc_vlantci_create
!= NULL
);
678 etxop
->etxo_qdesc_vlantci_create(etp
, tci
, edp
);
682 efx_tx_qdesc_checksum_create(
685 __out efx_desc_t
*edp
)
687 efx_nic_t
*enp
= etp
->et_enp
;
688 const efx_tx_ops_t
*etxop
= enp
->en_etxop
;
690 EFSYS_ASSERT3U(etp
->et_magic
, ==, EFX_TXQ_MAGIC
);
691 EFSYS_ASSERT(etxop
->etxo_qdesc_checksum_create
!= NULL
);
693 etxop
->etxo_qdesc_checksum_create(etp
, flags
, edp
);
699 efx_tx_qstats_update(
701 __inout_ecount(TX_NQSTATS
) efsys_stat_t
*stat
)
703 efx_nic_t
*enp
= etp
->et_enp
;
704 const efx_tx_ops_t
*etxop
= enp
->en_etxop
;
706 EFSYS_ASSERT3U(etp
->et_magic
, ==, EFX_TXQ_MAGIC
);
708 etxop
->etxo_qstats_update(etp
, stat
);
715 static __checkReturn efx_rc_t
722 * Disable the timer-based TX DMA backoff and allow TX DMA to be
723 * controlled by the RX FIFO fill level (although always allow a
726 EFX_BAR_READO(enp
, FR_AZ_TX_RESERVED_REG
, &oword
);
727 EFX_SET_OWORD_FIELD(oword
, FRF_AZ_TX_RX_SPACER
, 0xfe);
728 EFX_SET_OWORD_FIELD(oword
, FRF_AZ_TX_RX_SPACER_EN
, 1);
729 EFX_SET_OWORD_FIELD(oword
, FRF_AZ_TX_ONE_PKT_PER_Q
, 1);
730 EFX_SET_OWORD_FIELD(oword
, FRF_AZ_TX_PUSH_EN
, 0);
731 EFX_SET_OWORD_FIELD(oword
, FRF_AZ_TX_DIS_NON_IP_EV
, 1);
732 EFX_SET_OWORD_FIELD(oword
, FRF_AZ_TX_PREF_THRESHOLD
, 2);
733 EFX_SET_OWORD_FIELD(oword
, FRF_AZ_TX_PREF_WD_TMR
, 0x3fffff);
736 * Filter all packets less than 14 bytes to avoid parsing
739 EFX_SET_OWORD_FIELD(oword
, FRF_BZ_TX_FLUSH_MIN_LEN_EN
, 1);
740 EFX_BAR_WRITEO(enp
, FR_AZ_TX_RESERVED_REG
, &oword
);
743 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
744 * descriptors (which is bad).
746 EFX_BAR_READO(enp
, FR_AZ_TX_CFG_REG
, &oword
);
747 EFX_SET_OWORD_FIELD(oword
, FRF_AZ_TX_NO_EOP_DISC_EN
, 0);
748 EFX_BAR_WRITEO(enp
, FR_AZ_TX_CFG_REG
, &oword
);
753 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
759 id = (_added)++ & (_etp)->et_mask; \
760 offset = id * sizeof (efx_qword_t); \
762 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
763 unsigned int, id, efsys_dma_addr_t, (_addr), \
764 size_t, (_size), boolean_t, (_eop)); \
766 EFX_POPULATE_QWORD_4(qword, \
767 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
768 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
769 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
770 (uint32_t)((_addr) & 0xffffffff), \
771 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
772 (uint32_t)((_addr) >> 32)); \
773 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
775 _NOTE(CONSTANTCONDITION) \
778 static __checkReturn efx_rc_t
781 __in_ecount(ndescs
) efx_buffer_t
*eb
,
782 __in
unsigned int ndescs
,
783 __in
unsigned int completed
,
784 __inout
unsigned int *addedp
)
786 unsigned int added
= *addedp
;
789 if (added
- completed
+ ndescs
> EFX_TXQ_LIMIT(etp
->et_mask
+ 1))
792 for (i
= 0; i
< ndescs
; i
++) {
793 efx_buffer_t
*ebp
= &eb
[i
];
794 efsys_dma_addr_t start
= ebp
->eb_addr
;
795 size_t size
= ebp
->eb_size
;
796 efsys_dma_addr_t end
= start
+ size
;
799 * Fragments must not span 4k boundaries.
800 * Here it is a stricter requirement than the maximum length.
802 EFSYS_ASSERT(P2ROUNDUP(start
+ 1,
803 etp
->et_enp
->en_nic_cfg
.enc_tx_dma_desc_boundary
) >= end
);
805 EFX_TX_DESC(etp
, start
, size
, ebp
->eb_eop
, added
);
808 EFX_TX_QSTAT_INCR(etp
, TX_POST
);
817 __in
unsigned int added
,
818 __in
unsigned int pushed
)
820 efx_nic_t
*enp
= etp
->et_enp
;
825 /* Push the populated descriptors out */
826 wptr
= added
& etp
->et_mask
;
828 EFX_POPULATE_OWORD_1(oword
, FRF_AZ_TX_DESC_WPTR
, wptr
);
830 /* Only write the third DWORD */
831 EFX_POPULATE_DWORD_1(dword
,
832 EFX_DWORD_0
, EFX_OWORD_FIELD(oword
, EFX_DWORD_3
));
834 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
835 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp
->et_esmp
, etp
->et_mask
+ 1,
836 wptr
, pushed
& etp
->et_mask
);
837 EFSYS_PIO_WRITE_BARRIER();
838 EFX_BAR_TBL_WRITED3(enp
, FR_BZ_TX_DESC_UPD_REGP0
,
839 etp
->et_index
, &dword
, B_FALSE
);
842 #define EFX_MAX_PACE_VALUE 20
844 static __checkReturn efx_rc_t
847 __in
unsigned int ns
)
849 efx_nic_t
*enp
= etp
->et_enp
;
850 efx_nic_cfg_t
*encp
= &(enp
->en_nic_cfg
);
852 unsigned int pace_val
;
853 unsigned int timer_period
;
860 * The pace_val to write into the table is s.t
861 * ns <= timer_period * (2 ^ pace_val)
863 timer_period
= 104 / encp
->enc_clk_mult
;
864 for (pace_val
= 1; pace_val
<= EFX_MAX_PACE_VALUE
; pace_val
++) {
865 if ((timer_period
<< pace_val
) >= ns
)
869 if (pace_val
> EFX_MAX_PACE_VALUE
) {
874 /* Update the pacing table */
875 EFX_POPULATE_OWORD_1(oword
, FRF_AZ_TX_PACE
, pace_val
);
876 EFX_BAR_TBL_WRITEO(enp
, FR_AZ_TX_PACE_TBL
, etp
->et_index
,
882 EFSYS_PROBE1(fail1
, efx_rc_t
, rc
);
887 static __checkReturn efx_rc_t
891 efx_nic_t
*enp
= etp
->et_enp
;
895 efx_tx_qpace(etp
, 0);
897 label
= etp
->et_index
;
899 /* Flush the queue */
900 EFX_POPULATE_OWORD_2(oword
, FRF_AZ_TX_FLUSH_DESCQ_CMD
, 1,
901 FRF_AZ_TX_FLUSH_DESCQ
, label
);
902 EFX_BAR_WRITEO(enp
, FR_AZ_TX_FLUSH_DESCQ_REG
, &oword
);
911 efx_nic_t
*enp
= etp
->et_enp
;
914 EFX_BAR_TBL_READO(enp
, FR_AZ_TX_DESC_PTR_TBL
,
915 etp
->et_index
, &oword
, B_TRUE
);
917 EFSYS_PROBE5(tx_descq_ptr
, unsigned int, etp
->et_index
,
918 uint32_t, EFX_OWORD_FIELD(oword
, EFX_DWORD_3
),
919 uint32_t, EFX_OWORD_FIELD(oword
, EFX_DWORD_2
),
920 uint32_t, EFX_OWORD_FIELD(oword
, EFX_DWORD_1
),
921 uint32_t, EFX_OWORD_FIELD(oword
, EFX_DWORD_0
));
923 EFX_SET_OWORD_FIELD(oword
, FRF_AZ_TX_DC_HW_RPTR
, 0);
924 EFX_SET_OWORD_FIELD(oword
, FRF_AZ_TX_DESCQ_HW_RPTR
, 0);
925 EFX_SET_OWORD_FIELD(oword
, FRF_AZ_TX_DESCQ_EN
, 1);
927 EFX_BAR_TBL_WRITEO(enp
, FR_AZ_TX_DESC_PTR_TBL
,
928 etp
->et_index
, &oword
, B_TRUE
);
931 static __checkReturn efx_rc_t
934 __in
unsigned int index
,
935 __in
unsigned int label
,
936 __in efsys_mem_t
*esmp
,
942 __out
unsigned int *addedp
)
944 efx_nic_cfg_t
*encp
= &(enp
->en_nic_cfg
);
950 _NOTE(ARGUNUSED(esmp
))
952 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS
==
953 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH
));
954 EFSYS_ASSERT3U(label
, <, EFX_EV_TX_NLABELS
);
956 if (index
>= encp
->enc_txq_limit
) {
961 (1U << size
) <= encp
->enc_txq_max_ndescs
/ encp
->enc_txq_min_ndescs
;
963 if ((1U << size
) == (uint32_t)ndescs
/ encp
->enc_txq_min_ndescs
)
965 if (id
+ (1 << size
) >= encp
->enc_buftbl_limit
) {
970 inner_csum
= EFX_TXQ_CKSUM_INNER_IPV4
| EFX_TXQ_CKSUM_INNER_TCPUDP
;
971 if ((flags
& inner_csum
) != 0) {
976 /* Set up the new descriptor queue */
979 EFX_POPULATE_OWORD_6(oword
,
980 FRF_AZ_TX_DESCQ_BUF_BASE_ID
, id
,
981 FRF_AZ_TX_DESCQ_EVQ_ID
, eep
->ee_index
,
982 FRF_AZ_TX_DESCQ_OWNER_ID
, 0,
983 FRF_AZ_TX_DESCQ_LABEL
, label
,
984 FRF_AZ_TX_DESCQ_SIZE
, size
,
985 FRF_AZ_TX_DESCQ_TYPE
, 0);
987 EFX_SET_OWORD_FIELD(oword
, FRF_BZ_TX_NON_IP_DROP_DIS
, 1);
988 EFX_SET_OWORD_FIELD(oword
, FRF_BZ_TX_IP_CHKSM_DIS
,
989 (flags
& EFX_TXQ_CKSUM_IPV4
) ? 0 : 1);
990 EFX_SET_OWORD_FIELD(oword
, FRF_BZ_TX_TCP_CHKSM_DIS
,
991 (flags
& EFX_TXQ_CKSUM_TCPUDP
) ? 0 : 1);
993 EFX_BAR_TBL_WRITEO(enp
, FR_AZ_TX_DESC_PTR_TBL
,
994 etp
->et_index
, &oword
, B_TRUE
);
1003 EFSYS_PROBE1(fail1
, efx_rc_t
, rc
);
1008 __checkReturn efx_rc_t
1009 siena_tx_qdesc_post(
1010 __in efx_txq_t
*etp
,
1011 __in_ecount(ndescs
) efx_desc_t
*ed
,
1012 __in
unsigned int ndescs
,
1013 __in
unsigned int completed
,
1014 __inout
unsigned int *addedp
)
1016 unsigned int added
= *addedp
;
1020 if (added
- completed
+ ndescs
> EFX_TXQ_LIMIT(etp
->et_mask
+ 1)) {
1025 for (i
= 0; i
< ndescs
; i
++) {
1026 efx_desc_t
*edp
= &ed
[i
];
1030 id
= added
++ & etp
->et_mask
;
1031 offset
= id
* sizeof (efx_desc_t
);
1033 EFSYS_MEM_WRITEQ(etp
->et_esmp
, offset
, &edp
->ed_eq
);
1036 EFSYS_PROBE3(tx_desc_post
, unsigned int, etp
->et_index
,
1037 unsigned int, added
, unsigned int, ndescs
);
1039 EFX_TX_QSTAT_INCR(etp
, TX_POST
);
1045 EFSYS_PROBE1(fail1
, efx_rc_t
, rc
);
1050 siena_tx_qdesc_dma_create(
1051 __in efx_txq_t
*etp
,
1052 __in efsys_dma_addr_t addr
,
1055 __out efx_desc_t
*edp
)
1058 * Fragments must not span 4k boundaries.
1059 * Here it is a stricter requirement than the maximum length.
1061 EFSYS_ASSERT(P2ROUNDUP(addr
+ 1,
1062 etp
->et_enp
->en_nic_cfg
.enc_tx_dma_desc_boundary
) >= addr
+ size
);
1064 EFSYS_PROBE4(tx_desc_dma_create
, unsigned int, etp
->et_index
,
1065 efsys_dma_addr_t
, addr
,
1066 size_t, size
, boolean_t
, eop
);
1068 EFX_POPULATE_QWORD_4(edp
->ed_eq
,
1069 FSF_AZ_TX_KER_CONT
, eop
? 0 : 1,
1070 FSF_AZ_TX_KER_BYTE_COUNT
, (uint32_t)size
,
1071 FSF_AZ_TX_KER_BUF_ADDR_DW0
,
1072 (uint32_t)(addr
& 0xffffffff),
1073 FSF_AZ_TX_KER_BUF_ADDR_DW1
,
1074 (uint32_t)(addr
>> 32));
1077 #endif /* EFSYS_OPT_SIENA */
1079 #if EFSYS_OPT_QSTATS
1081 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */
1082 static const char * const __efx_tx_qstat_name
[] = {
1086 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1090 __in efx_nic_t
*enp
,
1091 __in
unsigned int id
)
1093 _NOTE(ARGUNUSED(enp
))
1094 EFSYS_ASSERT3U(enp
->en_magic
, ==, EFX_NIC_MAGIC
);
1095 EFSYS_ASSERT3U(id
, <, TX_NQSTATS
);
1097 return (__efx_tx_qstat_name
[id
]);
1099 #endif /* EFSYS_OPT_NAMES */
1100 #endif /* EFSYS_OPT_QSTATS */
1104 #if EFSYS_OPT_QSTATS
1106 siena_tx_qstats_update(
1107 __in efx_txq_t
*etp
,
1108 __inout_ecount(TX_NQSTATS
) efsys_stat_t
*stat
)
1112 for (id
= 0; id
< TX_NQSTATS
; id
++) {
1113 efsys_stat_t
*essp
= &stat
[id
];
1115 EFSYS_STAT_INCR(essp
, etp
->et_stat
[id
]);
1116 etp
->et_stat
[id
] = 0;
1119 #endif /* EFSYS_OPT_QSTATS */
1123 __in efx_txq_t
*etp
)
1125 efx_nic_t
*enp
= etp
->et_enp
;
1128 /* Purge descriptor queue */
1129 EFX_ZERO_OWORD(oword
);
1131 EFX_BAR_TBL_WRITEO(enp
, FR_AZ_TX_DESC_PTR_TBL
,
1132 etp
->et_index
, &oword
, B_TRUE
);
1137 __in efx_nic_t
*enp
)
1139 _NOTE(ARGUNUSED(enp
))
1142 #endif /* EFSYS_OPT_SIENA */