]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | /* SPDX-License-Identifier: BSD-3-Clause |
2 | * | |
3 | * Copyright (c) 2007-2018 Solarflare Communications Inc. | |
4 | * All rights reserved. | |
5 | */ | |
6 | ||
7 | #include "efx.h" | |
8 | #include "efx_impl.h" | |
9 | ||
10 | #if EFSYS_OPT_QSTATS | |
11 | #define EFX_TX_QSTAT_INCR(_etp, _stat) \ | |
12 | do { \ | |
13 | (_etp)->et_stat[_stat]++; \ | |
14 | _NOTE(CONSTANTCONDITION) \ | |
15 | } while (B_FALSE) | |
16 | #else | |
17 | #define EFX_TX_QSTAT_INCR(_etp, _stat) | |
18 | #endif | |
19 | ||
20 | #if EFSYS_OPT_SIENA | |
21 | ||
22 | static __checkReturn efx_rc_t | |
23 | siena_tx_init( | |
24 | __in efx_nic_t *enp); | |
25 | ||
26 | static void | |
27 | siena_tx_fini( | |
28 | __in efx_nic_t *enp); | |
29 | ||
30 | static __checkReturn efx_rc_t | |
31 | siena_tx_qcreate( | |
32 | __in efx_nic_t *enp, | |
33 | __in unsigned int index, | |
34 | __in unsigned int label, | |
35 | __in efsys_mem_t *esmp, | |
36 | __in size_t ndescs, | |
37 | __in uint32_t id, | |
38 | __in uint16_t flags, | |
39 | __in efx_evq_t *eep, | |
40 | __in efx_txq_t *etp, | |
41 | __out unsigned int *addedp); | |
42 | ||
43 | static void | |
44 | siena_tx_qdestroy( | |
45 | __in efx_txq_t *etp); | |
46 | ||
47 | static __checkReturn efx_rc_t | |
48 | siena_tx_qpost( | |
49 | __in efx_txq_t *etp, | |
50 | __in_ecount(ndescs) efx_buffer_t *eb, | |
51 | __in unsigned int ndescs, | |
52 | __in unsigned int completed, | |
53 | __inout unsigned int *addedp); | |
54 | ||
55 | static void | |
56 | siena_tx_qpush( | |
57 | __in efx_txq_t *etp, | |
58 | __in unsigned int added, | |
59 | __in unsigned int pushed); | |
60 | ||
61 | static __checkReturn efx_rc_t | |
62 | siena_tx_qpace( | |
63 | __in efx_txq_t *etp, | |
64 | __in unsigned int ns); | |
65 | ||
66 | static __checkReturn efx_rc_t | |
67 | siena_tx_qflush( | |
68 | __in efx_txq_t *etp); | |
69 | ||
70 | static void | |
71 | siena_tx_qenable( | |
72 | __in efx_txq_t *etp); | |
73 | ||
74 | __checkReturn efx_rc_t | |
75 | siena_tx_qdesc_post( | |
76 | __in efx_txq_t *etp, | |
77 | __in_ecount(ndescs) efx_desc_t *ed, | |
78 | __in unsigned int ndescs, | |
79 | __in unsigned int completed, | |
80 | __inout unsigned int *addedp); | |
81 | ||
82 | void | |
83 | siena_tx_qdesc_dma_create( | |
84 | __in efx_txq_t *etp, | |
85 | __in efsys_dma_addr_t addr, | |
86 | __in size_t size, | |
87 | __in boolean_t eop, | |
88 | __out efx_desc_t *edp); | |
89 | ||
90 | #if EFSYS_OPT_QSTATS | |
91 | static void | |
92 | siena_tx_qstats_update( | |
93 | __in efx_txq_t *etp, | |
94 | __inout_ecount(TX_NQSTATS) efsys_stat_t *stat); | |
95 | #endif | |
96 | ||
97 | #endif /* EFSYS_OPT_SIENA */ | |
98 | ||
99 | ||
100 | #if EFSYS_OPT_SIENA | |
101 | static const efx_tx_ops_t __efx_tx_siena_ops = { | |
102 | siena_tx_init, /* etxo_init */ | |
103 | siena_tx_fini, /* etxo_fini */ | |
104 | siena_tx_qcreate, /* etxo_qcreate */ | |
105 | siena_tx_qdestroy, /* etxo_qdestroy */ | |
106 | siena_tx_qpost, /* etxo_qpost */ | |
107 | siena_tx_qpush, /* etxo_qpush */ | |
108 | siena_tx_qpace, /* etxo_qpace */ | |
109 | siena_tx_qflush, /* etxo_qflush */ | |
110 | siena_tx_qenable, /* etxo_qenable */ | |
111 | NULL, /* etxo_qpio_enable */ | |
112 | NULL, /* etxo_qpio_disable */ | |
113 | NULL, /* etxo_qpio_write */ | |
114 | NULL, /* etxo_qpio_post */ | |
115 | siena_tx_qdesc_post, /* etxo_qdesc_post */ | |
116 | siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ | |
117 | NULL, /* etxo_qdesc_tso_create */ | |
118 | NULL, /* etxo_qdesc_tso2_create */ | |
119 | NULL, /* etxo_qdesc_vlantci_create */ | |
120 | NULL, /* etxo_qdesc_checksum_create */ | |
121 | #if EFSYS_OPT_QSTATS | |
122 | siena_tx_qstats_update, /* etxo_qstats_update */ | |
123 | #endif | |
124 | }; | |
125 | #endif /* EFSYS_OPT_SIENA */ | |
126 | ||
127 | #if EFSYS_OPT_HUNTINGTON | |
128 | static const efx_tx_ops_t __efx_tx_hunt_ops = { | |
129 | ef10_tx_init, /* etxo_init */ | |
130 | ef10_tx_fini, /* etxo_fini */ | |
131 | ef10_tx_qcreate, /* etxo_qcreate */ | |
132 | ef10_tx_qdestroy, /* etxo_qdestroy */ | |
133 | ef10_tx_qpost, /* etxo_qpost */ | |
134 | ef10_tx_qpush, /* etxo_qpush */ | |
135 | ef10_tx_qpace, /* etxo_qpace */ | |
136 | ef10_tx_qflush, /* etxo_qflush */ | |
137 | ef10_tx_qenable, /* etxo_qenable */ | |
138 | ef10_tx_qpio_enable, /* etxo_qpio_enable */ | |
139 | ef10_tx_qpio_disable, /* etxo_qpio_disable */ | |
140 | ef10_tx_qpio_write, /* etxo_qpio_write */ | |
141 | ef10_tx_qpio_post, /* etxo_qpio_post */ | |
142 | ef10_tx_qdesc_post, /* etxo_qdesc_post */ | |
143 | ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ | |
144 | ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */ | |
145 | ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ | |
146 | ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ | |
147 | ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */ | |
148 | #if EFSYS_OPT_QSTATS | |
149 | ef10_tx_qstats_update, /* etxo_qstats_update */ | |
150 | #endif | |
151 | }; | |
152 | #endif /* EFSYS_OPT_HUNTINGTON */ | |
153 | ||
154 | #if EFSYS_OPT_MEDFORD | |
155 | static const efx_tx_ops_t __efx_tx_medford_ops = { | |
156 | ef10_tx_init, /* etxo_init */ | |
157 | ef10_tx_fini, /* etxo_fini */ | |
158 | ef10_tx_qcreate, /* etxo_qcreate */ | |
159 | ef10_tx_qdestroy, /* etxo_qdestroy */ | |
160 | ef10_tx_qpost, /* etxo_qpost */ | |
161 | ef10_tx_qpush, /* etxo_qpush */ | |
162 | ef10_tx_qpace, /* etxo_qpace */ | |
163 | ef10_tx_qflush, /* etxo_qflush */ | |
164 | ef10_tx_qenable, /* etxo_qenable */ | |
165 | ef10_tx_qpio_enable, /* etxo_qpio_enable */ | |
166 | ef10_tx_qpio_disable, /* etxo_qpio_disable */ | |
167 | ef10_tx_qpio_write, /* etxo_qpio_write */ | |
168 | ef10_tx_qpio_post, /* etxo_qpio_post */ | |
169 | ef10_tx_qdesc_post, /* etxo_qdesc_post */ | |
170 | ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ | |
171 | NULL, /* etxo_qdesc_tso_create */ | |
172 | ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ | |
173 | ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ | |
174 | ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */ | |
175 | #if EFSYS_OPT_QSTATS | |
176 | ef10_tx_qstats_update, /* etxo_qstats_update */ | |
177 | #endif | |
178 | }; | |
179 | #endif /* EFSYS_OPT_MEDFORD */ | |
180 | ||
181 | #if EFSYS_OPT_MEDFORD2 | |
182 | static const efx_tx_ops_t __efx_tx_medford2_ops = { | |
183 | ef10_tx_init, /* etxo_init */ | |
184 | ef10_tx_fini, /* etxo_fini */ | |
185 | ef10_tx_qcreate, /* etxo_qcreate */ | |
186 | ef10_tx_qdestroy, /* etxo_qdestroy */ | |
187 | ef10_tx_qpost, /* etxo_qpost */ | |
188 | ef10_tx_qpush, /* etxo_qpush */ | |
189 | ef10_tx_qpace, /* etxo_qpace */ | |
190 | ef10_tx_qflush, /* etxo_qflush */ | |
191 | ef10_tx_qenable, /* etxo_qenable */ | |
192 | ef10_tx_qpio_enable, /* etxo_qpio_enable */ | |
193 | ef10_tx_qpio_disable, /* etxo_qpio_disable */ | |
194 | ef10_tx_qpio_write, /* etxo_qpio_write */ | |
195 | ef10_tx_qpio_post, /* etxo_qpio_post */ | |
196 | ef10_tx_qdesc_post, /* etxo_qdesc_post */ | |
197 | ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ | |
198 | NULL, /* etxo_qdesc_tso_create */ | |
199 | ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ | |
200 | ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ | |
201 | ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */ | |
202 | #if EFSYS_OPT_QSTATS | |
203 | ef10_tx_qstats_update, /* etxo_qstats_update */ | |
204 | #endif | |
205 | }; | |
206 | #endif /* EFSYS_OPT_MEDFORD2 */ | |
207 | ||
208 | ||
209 | __checkReturn efx_rc_t | |
210 | efx_tx_init( | |
211 | __in efx_nic_t *enp) | |
212 | { | |
213 | const efx_tx_ops_t *etxop; | |
214 | efx_rc_t rc; | |
215 | ||
216 | EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); | |
217 | EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); | |
218 | ||
219 | if (!(enp->en_mod_flags & EFX_MOD_EV)) { | |
220 | rc = EINVAL; | |
221 | goto fail1; | |
222 | } | |
223 | ||
224 | if (enp->en_mod_flags & EFX_MOD_TX) { | |
225 | rc = EINVAL; | |
226 | goto fail2; | |
227 | } | |
228 | ||
229 | switch (enp->en_family) { | |
230 | #if EFSYS_OPT_SIENA | |
231 | case EFX_FAMILY_SIENA: | |
232 | etxop = &__efx_tx_siena_ops; | |
233 | break; | |
234 | #endif /* EFSYS_OPT_SIENA */ | |
235 | ||
236 | #if EFSYS_OPT_HUNTINGTON | |
237 | case EFX_FAMILY_HUNTINGTON: | |
238 | etxop = &__efx_tx_hunt_ops; | |
239 | break; | |
240 | #endif /* EFSYS_OPT_HUNTINGTON */ | |
241 | ||
242 | #if EFSYS_OPT_MEDFORD | |
243 | case EFX_FAMILY_MEDFORD: | |
244 | etxop = &__efx_tx_medford_ops; | |
245 | break; | |
246 | #endif /* EFSYS_OPT_MEDFORD */ | |
247 | ||
248 | #if EFSYS_OPT_MEDFORD2 | |
249 | case EFX_FAMILY_MEDFORD2: | |
250 | etxop = &__efx_tx_medford2_ops; | |
251 | break; | |
252 | #endif /* EFSYS_OPT_MEDFORD2 */ | |
253 | ||
254 | default: | |
255 | EFSYS_ASSERT(0); | |
256 | rc = ENOTSUP; | |
257 | goto fail3; | |
258 | } | |
259 | ||
260 | EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); | |
261 | ||
262 | if ((rc = etxop->etxo_init(enp)) != 0) | |
263 | goto fail4; | |
264 | ||
265 | enp->en_etxop = etxop; | |
266 | enp->en_mod_flags |= EFX_MOD_TX; | |
267 | return (0); | |
268 | ||
269 | fail4: | |
270 | EFSYS_PROBE(fail4); | |
271 | fail3: | |
272 | EFSYS_PROBE(fail3); | |
273 | fail2: | |
274 | EFSYS_PROBE(fail2); | |
275 | fail1: | |
276 | EFSYS_PROBE1(fail1, efx_rc_t, rc); | |
277 | ||
278 | enp->en_etxop = NULL; | |
279 | enp->en_mod_flags &= ~EFX_MOD_TX; | |
280 | return (rc); | |
281 | } | |
282 | ||
283 | void | |
284 | efx_tx_fini( | |
285 | __in efx_nic_t *enp) | |
286 | { | |
287 | const efx_tx_ops_t *etxop = enp->en_etxop; | |
288 | ||
289 | EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); | |
290 | EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); | |
291 | EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); | |
292 | EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); | |
293 | ||
294 | etxop->etxo_fini(enp); | |
295 | ||
296 | enp->en_etxop = NULL; | |
297 | enp->en_mod_flags &= ~EFX_MOD_TX; | |
298 | } | |
299 | ||
9f95a23c TL |
300 | __checkReturn size_t |
301 | efx_txq_size( | |
302 | __in const efx_nic_t *enp, | |
303 | __in unsigned int ndescs) | |
304 | { | |
305 | const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp); | |
306 | ||
307 | return (ndescs * encp->enc_tx_desc_size); | |
308 | } | |
309 | ||
310 | __checkReturn unsigned int | |
311 | efx_txq_nbufs( | |
312 | __in const efx_nic_t *enp, | |
313 | __in unsigned int ndescs) | |
314 | { | |
315 | return (EFX_DIV_ROUND_UP(efx_txq_size(enp, ndescs), EFX_BUF_SIZE)); | |
316 | } | |
317 | ||
11fdf7f2 TL |
318 | __checkReturn efx_rc_t |
319 | efx_tx_qcreate( | |
320 | __in efx_nic_t *enp, | |
321 | __in unsigned int index, | |
322 | __in unsigned int label, | |
323 | __in efsys_mem_t *esmp, | |
324 | __in size_t ndescs, | |
325 | __in uint32_t id, | |
326 | __in uint16_t flags, | |
327 | __in efx_evq_t *eep, | |
328 | __deref_out efx_txq_t **etpp, | |
329 | __out unsigned int *addedp) | |
330 | { | |
331 | const efx_tx_ops_t *etxop = enp->en_etxop; | |
332 | efx_txq_t *etp; | |
9f95a23c | 333 | const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp); |
11fdf7f2 TL |
334 | efx_rc_t rc; |
335 | ||
336 | EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); | |
337 | EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); | |
338 | ||
339 | EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, | |
340 | enp->en_nic_cfg.enc_txq_limit); | |
341 | ||
9f95a23c TL |
342 | EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs)); |
343 | EFSYS_ASSERT(ISP2(encp->enc_txq_min_ndescs)); | |
344 | ||
345 | if (!ISP2(ndescs) || | |
346 | ndescs < encp->enc_txq_min_ndescs || | |
347 | ndescs > encp->enc_txq_max_ndescs) { | |
348 | rc = EINVAL; | |
349 | goto fail1; | |
350 | } | |
351 | ||
11fdf7f2 TL |
352 | /* Allocate an TXQ object */ |
353 | EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); | |
354 | ||
355 | if (etp == NULL) { | |
356 | rc = ENOMEM; | |
9f95a23c | 357 | goto fail2; |
11fdf7f2 TL |
358 | } |
359 | ||
360 | etp->et_magic = EFX_TXQ_MAGIC; | |
361 | etp->et_enp = enp; | |
362 | etp->et_index = index; | |
363 | etp->et_mask = ndescs - 1; | |
364 | etp->et_esmp = esmp; | |
365 | ||
366 | /* Initial descriptor index may be modified by etxo_qcreate */ | |
367 | *addedp = 0; | |
368 | ||
369 | if ((rc = etxop->etxo_qcreate(enp, index, label, esmp, | |
370 | ndescs, id, flags, eep, etp, addedp)) != 0) | |
9f95a23c | 371 | goto fail3; |
11fdf7f2 TL |
372 | |
373 | enp->en_tx_qcount++; | |
374 | *etpp = etp; | |
375 | ||
376 | return (0); | |
377 | ||
9f95a23c TL |
378 | fail3: |
379 | EFSYS_PROBE(fail3); | |
380 | EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); | |
11fdf7f2 TL |
381 | fail2: |
382 | EFSYS_PROBE(fail2); | |
11fdf7f2 TL |
383 | fail1: |
384 | EFSYS_PROBE1(fail1, efx_rc_t, rc); | |
385 | return (rc); | |
386 | } | |
387 | ||
388 | void | |
389 | efx_tx_qdestroy( | |
390 | __in efx_txq_t *etp) | |
391 | { | |
392 | efx_nic_t *enp = etp->et_enp; | |
393 | const efx_tx_ops_t *etxop = enp->en_etxop; | |
394 | ||
395 | EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); | |
396 | ||
397 | EFSYS_ASSERT(enp->en_tx_qcount != 0); | |
398 | --enp->en_tx_qcount; | |
399 | ||
400 | etxop->etxo_qdestroy(etp); | |
401 | ||
402 | /* Free the TXQ object */ | |
403 | EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); | |
404 | } | |
405 | ||
406 | __checkReturn efx_rc_t | |
407 | efx_tx_qpost( | |
408 | __in efx_txq_t *etp, | |
409 | __in_ecount(ndescs) efx_buffer_t *eb, | |
410 | __in unsigned int ndescs, | |
411 | __in unsigned int completed, | |
412 | __inout unsigned int *addedp) | |
413 | { | |
414 | efx_nic_t *enp = etp->et_enp; | |
415 | const efx_tx_ops_t *etxop = enp->en_etxop; | |
416 | efx_rc_t rc; | |
417 | ||
418 | EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); | |
419 | ||
420 | if ((rc = etxop->etxo_qpost(etp, eb, ndescs, completed, addedp)) != 0) | |
421 | goto fail1; | |
422 | ||
423 | return (0); | |
424 | ||
425 | fail1: | |
426 | EFSYS_PROBE1(fail1, efx_rc_t, rc); | |
427 | return (rc); | |
428 | } | |
429 | ||
430 | void | |
431 | efx_tx_qpush( | |
432 | __in efx_txq_t *etp, | |
433 | __in unsigned int added, | |
434 | __in unsigned int pushed) | |
435 | { | |
436 | efx_nic_t *enp = etp->et_enp; | |
437 | const efx_tx_ops_t *etxop = enp->en_etxop; | |
438 | ||
439 | EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); | |
440 | ||
441 | etxop->etxo_qpush(etp, added, pushed); | |
442 | } | |
443 | ||
444 | __checkReturn efx_rc_t | |
445 | efx_tx_qpace( | |
446 | __in efx_txq_t *etp, | |
447 | __in unsigned int ns) | |
448 | { | |
449 | efx_nic_t *enp = etp->et_enp; | |
450 | const efx_tx_ops_t *etxop = enp->en_etxop; | |
451 | efx_rc_t rc; | |
452 | ||
453 | EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); | |
454 | ||
455 | if ((rc = etxop->etxo_qpace(etp, ns)) != 0) | |
456 | goto fail1; | |
457 | ||
458 | return (0); | |
459 | ||
460 | fail1: | |
461 | EFSYS_PROBE1(fail1, efx_rc_t, rc); | |
462 | return (rc); | |
463 | } | |
464 | ||
465 | __checkReturn efx_rc_t | |
466 | efx_tx_qflush( | |
467 | __in efx_txq_t *etp) | |
468 | { | |
469 | efx_nic_t *enp = etp->et_enp; | |
470 | const efx_tx_ops_t *etxop = enp->en_etxop; | |
471 | efx_rc_t rc; | |
472 | ||
473 | EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); | |
474 | ||
475 | if ((rc = etxop->etxo_qflush(etp)) != 0) | |
476 | goto fail1; | |
477 | ||
478 | return (0); | |
479 | ||
480 | fail1: | |
481 | EFSYS_PROBE1(fail1, efx_rc_t, rc); | |
482 | return (rc); | |
483 | } | |
484 | ||
485 | void | |
486 | efx_tx_qenable( | |
487 | __in efx_txq_t *etp) | |
488 | { | |
489 | efx_nic_t *enp = etp->et_enp; | |
490 | const efx_tx_ops_t *etxop = enp->en_etxop; | |
491 | ||
492 | EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); | |
493 | ||
494 | etxop->etxo_qenable(etp); | |
495 | } | |
496 | ||
497 | __checkReturn efx_rc_t | |
498 | efx_tx_qpio_enable( | |
499 | __in efx_txq_t *etp) | |
500 | { | |
501 | efx_nic_t *enp = etp->et_enp; | |
502 | const efx_tx_ops_t *etxop = enp->en_etxop; | |
503 | efx_rc_t rc; | |
504 | ||
505 | EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); | |
506 | ||
507 | if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) { | |
508 | rc = ENOTSUP; | |
509 | goto fail1; | |
510 | } | |
511 | if (etxop->etxo_qpio_enable == NULL) { | |
512 | rc = ENOTSUP; | |
513 | goto fail2; | |
514 | } | |
515 | if ((rc = etxop->etxo_qpio_enable(etp)) != 0) | |
516 | goto fail3; | |
517 | ||
518 | return (0); | |
519 | ||
520 | fail3: | |
521 | EFSYS_PROBE(fail3); | |
522 | fail2: | |
523 | EFSYS_PROBE(fail2); | |
524 | fail1: | |
525 | EFSYS_PROBE1(fail1, efx_rc_t, rc); | |
526 | return (rc); | |
527 | } | |
528 | ||
529 | void | |
530 | efx_tx_qpio_disable( | |
531 | __in efx_txq_t *etp) | |
532 | { | |
533 | efx_nic_t *enp = etp->et_enp; | |
534 | const efx_tx_ops_t *etxop = enp->en_etxop; | |
535 | ||
536 | EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); | |
537 | ||
538 | if (etxop->etxo_qpio_disable != NULL) | |
539 | etxop->etxo_qpio_disable(etp); | |
540 | } | |
541 | ||
542 | __checkReturn efx_rc_t | |
543 | efx_tx_qpio_write( | |
544 | __in efx_txq_t *etp, | |
545 | __in_ecount(buf_length) uint8_t *buffer, | |
546 | __in size_t buf_length, | |
547 | __in size_t pio_buf_offset) | |
548 | { | |
549 | efx_nic_t *enp = etp->et_enp; | |
550 | const efx_tx_ops_t *etxop = enp->en_etxop; | |
551 | efx_rc_t rc; | |
552 | ||
553 | EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); | |
554 | ||
555 | if (etxop->etxo_qpio_write != NULL) { | |
556 | if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length, | |
557 | pio_buf_offset)) != 0) | |
558 | goto fail1; | |
559 | return (0); | |
560 | } | |
561 | ||
562 | return (ENOTSUP); | |
563 | ||
564 | fail1: | |
565 | EFSYS_PROBE1(fail1, efx_rc_t, rc); | |
566 | return (rc); | |
567 | } | |
568 | ||
569 | __checkReturn efx_rc_t | |
570 | efx_tx_qpio_post( | |
571 | __in efx_txq_t *etp, | |
572 | __in size_t pkt_length, | |
573 | __in unsigned int completed, | |
574 | __inout unsigned int *addedp) | |
575 | { | |
576 | efx_nic_t *enp = etp->et_enp; | |
577 | const efx_tx_ops_t *etxop = enp->en_etxop; | |
578 | efx_rc_t rc; | |
579 | ||
580 | EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); | |
581 | ||
582 | if (etxop->etxo_qpio_post != NULL) { | |
583 | if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed, | |
584 | addedp)) != 0) | |
585 | goto fail1; | |
586 | return (0); | |
587 | } | |
588 | ||
589 | return (ENOTSUP); | |
590 | ||
591 | fail1: | |
592 | EFSYS_PROBE1(fail1, efx_rc_t, rc); | |
593 | return (rc); | |
594 | } | |
595 | ||
596 | __checkReturn efx_rc_t | |
597 | efx_tx_qdesc_post( | |
598 | __in efx_txq_t *etp, | |
599 | __in_ecount(ndescs) efx_desc_t *ed, | |
600 | __in unsigned int ndescs, | |
601 | __in unsigned int completed, | |
602 | __inout unsigned int *addedp) | |
603 | { | |
604 | efx_nic_t *enp = etp->et_enp; | |
605 | const efx_tx_ops_t *etxop = enp->en_etxop; | |
11fdf7f2 TL |
606 | |
607 | EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); | |
608 | ||
9f95a23c | 609 | return (etxop->etxo_qdesc_post(etp, ed, ndescs, completed, addedp)); |
11fdf7f2 TL |
610 | } |
611 | ||
612 | void | |
613 | efx_tx_qdesc_dma_create( | |
614 | __in efx_txq_t *etp, | |
615 | __in efsys_dma_addr_t addr, | |
616 | __in size_t size, | |
617 | __in boolean_t eop, | |
618 | __out efx_desc_t *edp) | |
619 | { | |
620 | efx_nic_t *enp = etp->et_enp; | |
621 | const efx_tx_ops_t *etxop = enp->en_etxop; | |
622 | ||
623 | EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); | |
624 | EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL); | |
625 | ||
626 | etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp); | |
627 | } | |
628 | ||
629 | void | |
630 | efx_tx_qdesc_tso_create( | |
631 | __in efx_txq_t *etp, | |
632 | __in uint16_t ipv4_id, | |
633 | __in uint32_t tcp_seq, | |
634 | __in uint8_t tcp_flags, | |
635 | __out efx_desc_t *edp) | |
636 | { | |
637 | efx_nic_t *enp = etp->et_enp; | |
638 | const efx_tx_ops_t *etxop = enp->en_etxop; | |
639 | ||
640 | EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); | |
641 | EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL); | |
642 | ||
643 | etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp); | |
644 | } | |
645 | ||
646 | void | |
647 | efx_tx_qdesc_tso2_create( | |
648 | __in efx_txq_t *etp, | |
649 | __in uint16_t ipv4_id, | |
650 | __in uint16_t outer_ipv4_id, | |
651 | __in uint32_t tcp_seq, | |
652 | __in uint16_t mss, | |
653 | __out_ecount(count) efx_desc_t *edp, | |
654 | __in int count) | |
655 | { | |
656 | efx_nic_t *enp = etp->et_enp; | |
657 | const efx_tx_ops_t *etxop = enp->en_etxop; | |
658 | ||
659 | EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); | |
660 | EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL); | |
661 | ||
662 | etxop->etxo_qdesc_tso2_create(etp, ipv4_id, outer_ipv4_id, | |
663 | tcp_seq, mss, edp, count); | |
664 | } | |
665 | ||
666 | void | |
667 | efx_tx_qdesc_vlantci_create( | |
668 | __in efx_txq_t *etp, | |
669 | __in uint16_t tci, | |
670 | __out efx_desc_t *edp) | |
671 | { | |
672 | efx_nic_t *enp = etp->et_enp; | |
673 | const efx_tx_ops_t *etxop = enp->en_etxop; | |
674 | ||
675 | EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); | |
676 | EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL); | |
677 | ||
678 | etxop->etxo_qdesc_vlantci_create(etp, tci, edp); | |
679 | } | |
680 | ||
681 | void | |
682 | efx_tx_qdesc_checksum_create( | |
683 | __in efx_txq_t *etp, | |
684 | __in uint16_t flags, | |
685 | __out efx_desc_t *edp) | |
686 | { | |
687 | efx_nic_t *enp = etp->et_enp; | |
688 | const efx_tx_ops_t *etxop = enp->en_etxop; | |
689 | ||
690 | EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); | |
691 | EFSYS_ASSERT(etxop->etxo_qdesc_checksum_create != NULL); | |
692 | ||
693 | etxop->etxo_qdesc_checksum_create(etp, flags, edp); | |
694 | } | |
695 | ||
696 | ||
697 | #if EFSYS_OPT_QSTATS | |
698 | void | |
699 | efx_tx_qstats_update( | |
700 | __in efx_txq_t *etp, | |
701 | __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) | |
702 | { | |
703 | efx_nic_t *enp = etp->et_enp; | |
704 | const efx_tx_ops_t *etxop = enp->en_etxop; | |
705 | ||
706 | EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); | |
707 | ||
708 | etxop->etxo_qstats_update(etp, stat); | |
709 | } | |
710 | #endif | |
711 | ||
712 | ||
713 | #if EFSYS_OPT_SIENA | |
714 | ||
715 | static __checkReturn efx_rc_t | |
716 | siena_tx_init( | |
717 | __in efx_nic_t *enp) | |
718 | { | |
719 | efx_oword_t oword; | |
720 | ||
721 | /* | |
722 | * Disable the timer-based TX DMA backoff and allow TX DMA to be | |
723 | * controlled by the RX FIFO fill level (although always allow a | |
724 | * minimal trickle). | |
725 | */ | |
726 | EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); | |
727 | EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); | |
728 | EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); | |
729 | EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); | |
730 | EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); | |
731 | EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); | |
732 | EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); | |
733 | EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); | |
734 | ||
735 | /* | |
736 | * Filter all packets less than 14 bytes to avoid parsing | |
737 | * errors. | |
738 | */ | |
739 | EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); | |
740 | EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); | |
741 | ||
742 | /* | |
743 | * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 | |
744 | * descriptors (which is bad). | |
745 | */ | |
746 | EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); | |
747 | EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); | |
748 | EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); | |
749 | ||
750 | return (0); | |
751 | } | |
752 | ||
753 | #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ | |
754 | do { \ | |
755 | unsigned int id; \ | |
756 | size_t offset; \ | |
757 | efx_qword_t qword; \ | |
758 | \ | |
759 | id = (_added)++ & (_etp)->et_mask; \ | |
760 | offset = id * sizeof (efx_qword_t); \ | |
761 | \ | |
762 | EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ | |
763 | unsigned int, id, efsys_dma_addr_t, (_addr), \ | |
764 | size_t, (_size), boolean_t, (_eop)); \ | |
765 | \ | |
766 | EFX_POPULATE_QWORD_4(qword, \ | |
767 | FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ | |
768 | FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ | |
769 | FSF_AZ_TX_KER_BUF_ADDR_DW0, \ | |
770 | (uint32_t)((_addr) & 0xffffffff), \ | |
771 | FSF_AZ_TX_KER_BUF_ADDR_DW1, \ | |
772 | (uint32_t)((_addr) >> 32)); \ | |
773 | EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ | |
774 | \ | |
775 | _NOTE(CONSTANTCONDITION) \ | |
776 | } while (B_FALSE) | |
777 | ||
778 | static __checkReturn efx_rc_t | |
779 | siena_tx_qpost( | |
780 | __in efx_txq_t *etp, | |
781 | __in_ecount(ndescs) efx_buffer_t *eb, | |
782 | __in unsigned int ndescs, | |
783 | __in unsigned int completed, | |
784 | __inout unsigned int *addedp) | |
785 | { | |
786 | unsigned int added = *addedp; | |
787 | unsigned int i; | |
11fdf7f2 TL |
788 | |
789 | if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) | |
9f95a23c | 790 | return (ENOSPC); |
11fdf7f2 TL |
791 | |
792 | for (i = 0; i < ndescs; i++) { | |
793 | efx_buffer_t *ebp = &eb[i]; | |
794 | efsys_dma_addr_t start = ebp->eb_addr; | |
795 | size_t size = ebp->eb_size; | |
796 | efsys_dma_addr_t end = start + size; | |
797 | ||
798 | /* | |
799 | * Fragments must not span 4k boundaries. | |
800 | * Here it is a stricter requirement than the maximum length. | |
801 | */ | |
802 | EFSYS_ASSERT(P2ROUNDUP(start + 1, | |
803 | etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end); | |
804 | ||
805 | EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); | |
806 | } | |
807 | ||
808 | EFX_TX_QSTAT_INCR(etp, TX_POST); | |
809 | ||
810 | *addedp = added; | |
811 | return (0); | |
11fdf7f2 TL |
812 | } |
813 | ||
814 | static void | |
815 | siena_tx_qpush( | |
816 | __in efx_txq_t *etp, | |
817 | __in unsigned int added, | |
818 | __in unsigned int pushed) | |
819 | { | |
820 | efx_nic_t *enp = etp->et_enp; | |
821 | uint32_t wptr; | |
822 | efx_dword_t dword; | |
823 | efx_oword_t oword; | |
824 | ||
825 | /* Push the populated descriptors out */ | |
826 | wptr = added & etp->et_mask; | |
827 | ||
828 | EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); | |
829 | ||
830 | /* Only write the third DWORD */ | |
831 | EFX_POPULATE_DWORD_1(dword, | |
832 | EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); | |
833 | ||
834 | /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ | |
835 | EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, | |
836 | wptr, pushed & etp->et_mask); | |
837 | EFSYS_PIO_WRITE_BARRIER(); | |
838 | EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, | |
839 | etp->et_index, &dword, B_FALSE); | |
840 | } | |
841 | ||
842 | #define EFX_MAX_PACE_VALUE 20 | |
843 | ||
844 | static __checkReturn efx_rc_t | |
845 | siena_tx_qpace( | |
846 | __in efx_txq_t *etp, | |
847 | __in unsigned int ns) | |
848 | { | |
849 | efx_nic_t *enp = etp->et_enp; | |
850 | efx_nic_cfg_t *encp = &(enp->en_nic_cfg); | |
851 | efx_oword_t oword; | |
852 | unsigned int pace_val; | |
853 | unsigned int timer_period; | |
854 | efx_rc_t rc; | |
855 | ||
856 | if (ns == 0) { | |
857 | pace_val = 0; | |
858 | } else { | |
859 | /* | |
860 | * The pace_val to write into the table is s.t | |
861 | * ns <= timer_period * (2 ^ pace_val) | |
862 | */ | |
863 | timer_period = 104 / encp->enc_clk_mult; | |
864 | for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) { | |
865 | if ((timer_period << pace_val) >= ns) | |
866 | break; | |
867 | } | |
868 | } | |
869 | if (pace_val > EFX_MAX_PACE_VALUE) { | |
870 | rc = EINVAL; | |
871 | goto fail1; | |
872 | } | |
873 | ||
874 | /* Update the pacing table */ | |
875 | EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val); | |
876 | EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index, | |
877 | &oword, B_TRUE); | |
878 | ||
879 | return (0); | |
880 | ||
881 | fail1: | |
882 | EFSYS_PROBE1(fail1, efx_rc_t, rc); | |
883 | ||
884 | return (rc); | |
885 | } | |
886 | ||
887 | static __checkReturn efx_rc_t | |
888 | siena_tx_qflush( | |
889 | __in efx_txq_t *etp) | |
890 | { | |
891 | efx_nic_t *enp = etp->et_enp; | |
892 | efx_oword_t oword; | |
893 | uint32_t label; | |
894 | ||
895 | efx_tx_qpace(etp, 0); | |
896 | ||
897 | label = etp->et_index; | |
898 | ||
899 | /* Flush the queue */ | |
900 | EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, | |
901 | FRF_AZ_TX_FLUSH_DESCQ, label); | |
902 | EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); | |
903 | ||
904 | return (0); | |
905 | } | |
906 | ||
907 | static void | |
908 | siena_tx_qenable( | |
909 | __in efx_txq_t *etp) | |
910 | { | |
911 | efx_nic_t *enp = etp->et_enp; | |
912 | efx_oword_t oword; | |
913 | ||
914 | EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, | |
915 | etp->et_index, &oword, B_TRUE); | |
916 | ||
917 | EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, | |
918 | uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), | |
919 | uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), | |
920 | uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), | |
921 | uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); | |
922 | ||
923 | EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); | |
924 | EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); | |
925 | EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); | |
926 | ||
927 | EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, | |
928 | etp->et_index, &oword, B_TRUE); | |
929 | } | |
930 | ||
931 | static __checkReturn efx_rc_t | |
932 | siena_tx_qcreate( | |
933 | __in efx_nic_t *enp, | |
934 | __in unsigned int index, | |
935 | __in unsigned int label, | |
936 | __in efsys_mem_t *esmp, | |
937 | __in size_t ndescs, | |
938 | __in uint32_t id, | |
939 | __in uint16_t flags, | |
940 | __in efx_evq_t *eep, | |
941 | __in efx_txq_t *etp, | |
942 | __out unsigned int *addedp) | |
943 | { | |
944 | efx_nic_cfg_t *encp = &(enp->en_nic_cfg); | |
945 | efx_oword_t oword; | |
946 | uint32_t size; | |
947 | uint16_t inner_csum; | |
948 | efx_rc_t rc; | |
949 | ||
950 | _NOTE(ARGUNUSED(esmp)) | |
951 | ||
952 | EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == | |
953 | (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH)); | |
954 | EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS); | |
955 | ||
11fdf7f2 TL |
956 | if (index >= encp->enc_txq_limit) { |
957 | rc = EINVAL; | |
9f95a23c | 958 | goto fail1; |
11fdf7f2 TL |
959 | } |
960 | for (size = 0; | |
9f95a23c | 961 | (1U << size) <= encp->enc_txq_max_ndescs / encp->enc_txq_min_ndescs; |
11fdf7f2 | 962 | size++) |
9f95a23c | 963 | if ((1U << size) == (uint32_t)ndescs / encp->enc_txq_min_ndescs) |
11fdf7f2 TL |
964 | break; |
965 | if (id + (1 << size) >= encp->enc_buftbl_limit) { | |
966 | rc = EINVAL; | |
9f95a23c | 967 | goto fail2; |
11fdf7f2 TL |
968 | } |
969 | ||
970 | inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP; | |
971 | if ((flags & inner_csum) != 0) { | |
972 | rc = EINVAL; | |
9f95a23c | 973 | goto fail3; |
11fdf7f2 TL |
974 | } |
975 | ||
976 | /* Set up the new descriptor queue */ | |
977 | *addedp = 0; | |
978 | ||
979 | EFX_POPULATE_OWORD_6(oword, | |
980 | FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, | |
981 | FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, | |
982 | FRF_AZ_TX_DESCQ_OWNER_ID, 0, | |
983 | FRF_AZ_TX_DESCQ_LABEL, label, | |
984 | FRF_AZ_TX_DESCQ_SIZE, size, | |
985 | FRF_AZ_TX_DESCQ_TYPE, 0); | |
986 | ||
987 | EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); | |
988 | EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, | |
989 | (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1); | |
990 | EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, | |
991 | (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1); | |
992 | ||
993 | EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, | |
994 | etp->et_index, &oword, B_TRUE); | |
995 | ||
996 | return (0); | |
997 | ||
11fdf7f2 TL |
998 | fail3: |
999 | EFSYS_PROBE(fail3); | |
1000 | fail2: | |
1001 | EFSYS_PROBE(fail2); | |
1002 | fail1: | |
1003 | EFSYS_PROBE1(fail1, efx_rc_t, rc); | |
1004 | ||
1005 | return (rc); | |
1006 | } | |
1007 | ||
1008 | __checkReturn efx_rc_t | |
1009 | siena_tx_qdesc_post( | |
1010 | __in efx_txq_t *etp, | |
1011 | __in_ecount(ndescs) efx_desc_t *ed, | |
1012 | __in unsigned int ndescs, | |
1013 | __in unsigned int completed, | |
1014 | __inout unsigned int *addedp) | |
1015 | { | |
1016 | unsigned int added = *addedp; | |
1017 | unsigned int i; | |
1018 | efx_rc_t rc; | |
1019 | ||
1020 | if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) { | |
1021 | rc = ENOSPC; | |
1022 | goto fail1; | |
1023 | } | |
1024 | ||
1025 | for (i = 0; i < ndescs; i++) { | |
1026 | efx_desc_t *edp = &ed[i]; | |
1027 | unsigned int id; | |
1028 | size_t offset; | |
1029 | ||
1030 | id = added++ & etp->et_mask; | |
1031 | offset = id * sizeof (efx_desc_t); | |
1032 | ||
1033 | EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq); | |
1034 | } | |
1035 | ||
1036 | EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index, | |
1037 | unsigned int, added, unsigned int, ndescs); | |
1038 | ||
1039 | EFX_TX_QSTAT_INCR(etp, TX_POST); | |
1040 | ||
1041 | *addedp = added; | |
1042 | return (0); | |
1043 | ||
1044 | fail1: | |
1045 | EFSYS_PROBE1(fail1, efx_rc_t, rc); | |
1046 | return (rc); | |
1047 | } | |
1048 | ||
1049 | void | |
1050 | siena_tx_qdesc_dma_create( | |
1051 | __in efx_txq_t *etp, | |
1052 | __in efsys_dma_addr_t addr, | |
1053 | __in size_t size, | |
1054 | __in boolean_t eop, | |
1055 | __out efx_desc_t *edp) | |
1056 | { | |
1057 | /* | |
1058 | * Fragments must not span 4k boundaries. | |
1059 | * Here it is a stricter requirement than the maximum length. | |
1060 | */ | |
1061 | EFSYS_ASSERT(P2ROUNDUP(addr + 1, | |
1062 | etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size); | |
1063 | ||
1064 | EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index, | |
1065 | efsys_dma_addr_t, addr, | |
1066 | size_t, size, boolean_t, eop); | |
1067 | ||
1068 | EFX_POPULATE_QWORD_4(edp->ed_eq, | |
1069 | FSF_AZ_TX_KER_CONT, eop ? 0 : 1, | |
1070 | FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size, | |
1071 | FSF_AZ_TX_KER_BUF_ADDR_DW0, | |
1072 | (uint32_t)(addr & 0xffffffff), | |
1073 | FSF_AZ_TX_KER_BUF_ADDR_DW1, | |
1074 | (uint32_t)(addr >> 32)); | |
1075 | } | |
1076 | ||
1077 | #endif /* EFSYS_OPT_SIENA */ | |
1078 | ||
1079 | #if EFSYS_OPT_QSTATS | |
1080 | #if EFSYS_OPT_NAMES | |
1081 | /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */ | |
1082 | static const char * const __efx_tx_qstat_name[] = { | |
1083 | "post", | |
1084 | "post_pio", | |
1085 | }; | |
1086 | /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ | |
1087 | ||
1088 | const char * | |
1089 | efx_tx_qstat_name( | |
1090 | __in efx_nic_t *enp, | |
1091 | __in unsigned int id) | |
1092 | { | |
1093 | _NOTE(ARGUNUSED(enp)) | |
1094 | EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); | |
1095 | EFSYS_ASSERT3U(id, <, TX_NQSTATS); | |
1096 | ||
1097 | return (__efx_tx_qstat_name[id]); | |
1098 | } | |
1099 | #endif /* EFSYS_OPT_NAMES */ | |
1100 | #endif /* EFSYS_OPT_QSTATS */ | |
1101 | ||
1102 | #if EFSYS_OPT_SIENA | |
1103 | ||
1104 | #if EFSYS_OPT_QSTATS | |
1105 | static void | |
1106 | siena_tx_qstats_update( | |
1107 | __in efx_txq_t *etp, | |
1108 | __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) | |
1109 | { | |
1110 | unsigned int id; | |
1111 | ||
1112 | for (id = 0; id < TX_NQSTATS; id++) { | |
1113 | efsys_stat_t *essp = &stat[id]; | |
1114 | ||
1115 | EFSYS_STAT_INCR(essp, etp->et_stat[id]); | |
1116 | etp->et_stat[id] = 0; | |
1117 | } | |
1118 | } | |
1119 | #endif /* EFSYS_OPT_QSTATS */ | |
1120 | ||
1121 | static void | |
1122 | siena_tx_qdestroy( | |
1123 | __in efx_txq_t *etp) | |
1124 | { | |
1125 | efx_nic_t *enp = etp->et_enp; | |
1126 | efx_oword_t oword; | |
1127 | ||
1128 | /* Purge descriptor queue */ | |
1129 | EFX_ZERO_OWORD(oword); | |
1130 | ||
1131 | EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, | |
1132 | etp->et_index, &oword, B_TRUE); | |
1133 | } | |
1134 | ||
1135 | static void | |
1136 | siena_tx_fini( | |
1137 | __in efx_nic_t *enp) | |
1138 | { | |
1139 | _NOTE(ARGUNUSED(enp)) | |
1140 | } | |
1141 | ||
1142 | #endif /* EFSYS_OPT_SIENA */ |