]> git.proxmox.com Git - ceph.git/blame - ceph/src/spdk/dpdk/drivers/net/sfc/base/efx_tx.c
projects / ceph.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
update source to Ceph Pacific 16.2.2
[ceph.git] / ceph / src / spdk / dpdk / drivers / net / sfc / base / efx_tx.c
CommitLineData
11fdf7f2
TL		 1/* SPDX-License-Identifier: BSD-3-Clause
2 *
f67539c2
TL		 3 * Copyright(c) 2019-2020 Xilinx, Inc.
4 * Copyright(c) 2007-2019 Solarflare Communications Inc.
11fdf7f2
TL		 5 */
6
7#include "efx.h"
8#include "efx_impl.h"
9
10#if EFSYS_OPT_QSTATS
11#define EFX_TX_QSTAT_INCR(_etp, _stat) \
12 do { \
13 (_etp)->et_stat[_stat]++; \
14 _NOTE(CONSTANTCONDITION) \
15 } while (B_FALSE)
16#else
17#define EFX_TX_QSTAT_INCR(_etp, _stat)
18#endif
19
20#if EFSYS_OPT_SIENA
21
22static __checkReturn efx_rc_t
23siena_tx_init(
24 __in efx_nic_t *enp);
25
26static void
27siena_tx_fini(
28 __in efx_nic_t *enp);
29
30static __checkReturn efx_rc_t
31siena_tx_qcreate(
32 __in efx_nic_t *enp,
33 __in unsigned int index,
34 __in unsigned int label,
35 __in efsys_mem_t *esmp,
36 __in size_t ndescs,
37 __in uint32_t id,
38 __in uint16_t flags,
39 __in efx_evq_t *eep,
40 __in efx_txq_t *etp,
41 __out unsigned int *addedp);
42
43static void
44siena_tx_qdestroy(
45 __in efx_txq_t *etp);
46
47static __checkReturn efx_rc_t
48siena_tx_qpost(
49 __in efx_txq_t *etp,
50 __in_ecount(ndescs) efx_buffer_t *eb,
51 __in unsigned int ndescs,
52 __in unsigned int completed,
53 __inout unsigned int *addedp);
54
55static void
56siena_tx_qpush(
57 __in efx_txq_t *etp,
58 __in unsigned int added,
59 __in unsigned int pushed);
60
61static __checkReturn efx_rc_t
62siena_tx_qpace(
63 __in efx_txq_t *etp,
64 __in unsigned int ns);
65
66static __checkReturn efx_rc_t
67siena_tx_qflush(
68 __in efx_txq_t *etp);
69
70static void
71siena_tx_qenable(
72 __in efx_txq_t *etp);
73
74 __checkReturn efx_rc_t
75siena_tx_qdesc_post(
76 __in efx_txq_t *etp,
77 __in_ecount(ndescs) efx_desc_t *ed,
78 __in unsigned int ndescs,
79 __in unsigned int completed,
80 __inout unsigned int *addedp);
81
82 void
83siena_tx_qdesc_dma_create(
84 __in efx_txq_t *etp,
85 __in efsys_dma_addr_t addr,
86 __in size_t size,
87 __in boolean_t eop,
88 __out efx_desc_t *edp);
89
90#if EFSYS_OPT_QSTATS
91static void
92siena_tx_qstats_update(
93 __in efx_txq_t *etp,
94 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
95#endif
96
97#endif /* EFSYS_OPT_SIENA */
98
99
100#if EFSYS_OPT_SIENA
101static const efx_tx_ops_t __efx_tx_siena_ops = {
102 siena_tx_init, /* etxo_init */
103 siena_tx_fini, /* etxo_fini */
104 siena_tx_qcreate, /* etxo_qcreate */
105 siena_tx_qdestroy, /* etxo_qdestroy */
106 siena_tx_qpost, /* etxo_qpost */
107 siena_tx_qpush, /* etxo_qpush */
108 siena_tx_qpace, /* etxo_qpace */
109 siena_tx_qflush, /* etxo_qflush */
110 siena_tx_qenable, /* etxo_qenable */
111 NULL, /* etxo_qpio_enable */
112 NULL, /* etxo_qpio_disable */
113 NULL, /* etxo_qpio_write */
114 NULL, /* etxo_qpio_post */
115 siena_tx_qdesc_post, /* etxo_qdesc_post */
116 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
117 NULL, /* etxo_qdesc_tso_create */
118 NULL, /* etxo_qdesc_tso2_create */
119 NULL, /* etxo_qdesc_vlantci_create */
120 NULL, /* etxo_qdesc_checksum_create */
121#if EFSYS_OPT_QSTATS
122 siena_tx_qstats_update, /* etxo_qstats_update */
123#endif
124};
125#endif /* EFSYS_OPT_SIENA */
126
127#if EFSYS_OPT_HUNTINGTON
128static const efx_tx_ops_t __efx_tx_hunt_ops = {
129 ef10_tx_init, /* etxo_init */
130 ef10_tx_fini, /* etxo_fini */
131 ef10_tx_qcreate, /* etxo_qcreate */
132 ef10_tx_qdestroy, /* etxo_qdestroy */
133 ef10_tx_qpost, /* etxo_qpost */
134 ef10_tx_qpush, /* etxo_qpush */
135 ef10_tx_qpace, /* etxo_qpace */
136 ef10_tx_qflush, /* etxo_qflush */
137 ef10_tx_qenable, /* etxo_qenable */
138 ef10_tx_qpio_enable, /* etxo_qpio_enable */
139 ef10_tx_qpio_disable, /* etxo_qpio_disable */
140 ef10_tx_qpio_write, /* etxo_qpio_write */
141 ef10_tx_qpio_post, /* etxo_qpio_post */
142 ef10_tx_qdesc_post, /* etxo_qdesc_post */
143 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
144 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
145 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
146 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
147 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
148#if EFSYS_OPT_QSTATS
149 ef10_tx_qstats_update, /* etxo_qstats_update */
150#endif
151};
152#endif /* EFSYS_OPT_HUNTINGTON */
153
154#if EFSYS_OPT_MEDFORD
155static const efx_tx_ops_t __efx_tx_medford_ops = {
156 ef10_tx_init, /* etxo_init */
157 ef10_tx_fini, /* etxo_fini */
158 ef10_tx_qcreate, /* etxo_qcreate */
159 ef10_tx_qdestroy, /* etxo_qdestroy */
160 ef10_tx_qpost, /* etxo_qpost */
161 ef10_tx_qpush, /* etxo_qpush */
162 ef10_tx_qpace, /* etxo_qpace */
163 ef10_tx_qflush, /* etxo_qflush */
164 ef10_tx_qenable, /* etxo_qenable */
165 ef10_tx_qpio_enable, /* etxo_qpio_enable */
166 ef10_tx_qpio_disable, /* etxo_qpio_disable */
167 ef10_tx_qpio_write, /* etxo_qpio_write */
168 ef10_tx_qpio_post, /* etxo_qpio_post */
169 ef10_tx_qdesc_post, /* etxo_qdesc_post */
170 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
171 NULL, /* etxo_qdesc_tso_create */
172 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
173 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
174 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
175#if EFSYS_OPT_QSTATS
176 ef10_tx_qstats_update, /* etxo_qstats_update */
177#endif
178};
179#endif /* EFSYS_OPT_MEDFORD */
180
181#if EFSYS_OPT_MEDFORD2
182static const efx_tx_ops_t __efx_tx_medford2_ops = {
183 ef10_tx_init, /* etxo_init */
184 ef10_tx_fini, /* etxo_fini */
185 ef10_tx_qcreate, /* etxo_qcreate */
186 ef10_tx_qdestroy, /* etxo_qdestroy */
187 ef10_tx_qpost, /* etxo_qpost */
188 ef10_tx_qpush, /* etxo_qpush */
189 ef10_tx_qpace, /* etxo_qpace */
190 ef10_tx_qflush, /* etxo_qflush */
191 ef10_tx_qenable, /* etxo_qenable */
192 ef10_tx_qpio_enable, /* etxo_qpio_enable */
193 ef10_tx_qpio_disable, /* etxo_qpio_disable */
194 ef10_tx_qpio_write, /* etxo_qpio_write */
195 ef10_tx_qpio_post, /* etxo_qpio_post */
196 ef10_tx_qdesc_post, /* etxo_qdesc_post */
197 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
198 NULL, /* etxo_qdesc_tso_create */
199 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
200 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
201 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
202#if EFSYS_OPT_QSTATS
203 ef10_tx_qstats_update, /* etxo_qstats_update */
204#endif
205};
206#endif /* EFSYS_OPT_MEDFORD2 */
207
208
209 __checkReturn efx_rc_t
210efx_tx_init(
211 __in efx_nic_t *enp)
212{
213 const efx_tx_ops_t *etxop;
214 efx_rc_t rc;
215
216 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
217 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
218
219 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
220 rc = EINVAL;
221 goto fail1;
222 }
223
224 if (enp->en_mod_flags & EFX_MOD_TX) {
225 rc = EINVAL;
226 goto fail2;
227 }
228
229 switch (enp->en_family) {
230#if EFSYS_OPT_SIENA
231 case EFX_FAMILY_SIENA:
232 etxop = &__efx_tx_siena_ops;
233 break;
234#endif /* EFSYS_OPT_SIENA */
235
236#if EFSYS_OPT_HUNTINGTON
237 case EFX_FAMILY_HUNTINGTON:
238 etxop = &__efx_tx_hunt_ops;
239 break;
240#endif /* EFSYS_OPT_HUNTINGTON */
241
242#if EFSYS_OPT_MEDFORD
243 case EFX_FAMILY_MEDFORD:
244 etxop = &__efx_tx_medford_ops;
245 break;
246#endif /* EFSYS_OPT_MEDFORD */
247
248#if EFSYS_OPT_MEDFORD2
249 case EFX_FAMILY_MEDFORD2:
250 etxop = &__efx_tx_medford2_ops;
251 break;
252#endif /* EFSYS_OPT_MEDFORD2 */
253
254 default:
255 EFSYS_ASSERT(0);
256 rc = ENOTSUP;
257 goto fail3;
258 }
259
260 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
261
262 if ((rc = etxop->etxo_init(enp)) != 0)
263 goto fail4;
264
265 enp->en_etxop = etxop;
266 enp->en_mod_flags |= EFX_MOD_TX;
267 return (0);
268
269fail4:
270 EFSYS_PROBE(fail4);
271fail3:
272 EFSYS_PROBE(fail3);
273fail2:
274 EFSYS_PROBE(fail2);
275fail1:
276 EFSYS_PROBE1(fail1, efx_rc_t, rc);
277
278 enp->en_etxop = NULL;
279 enp->en_mod_flags &= ~EFX_MOD_TX;
280 return (rc);
281}
282
283 void
284efx_tx_fini(
285 __in efx_nic_t *enp)
286{
287 const efx_tx_ops_t *etxop = enp->en_etxop;
288
289 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
290 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
291 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
292 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
293
294 etxop->etxo_fini(enp);
295
296 enp->en_etxop = NULL;
297 enp->en_mod_flags &= ~EFX_MOD_TX;
298}
299
9f95a23c
TL		 300 __checkReturn size_t
301efx_txq_size(
302 __in const efx_nic_t *enp,
303 __in unsigned int ndescs)
304{
305 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
306
307 return (ndescs * encp->enc_tx_desc_size);
308}
309
310 __checkReturn unsigned int
311efx_txq_nbufs(
312 __in const efx_nic_t *enp,
313 __in unsigned int ndescs)
314{
315 return (EFX_DIV_ROUND_UP(efx_txq_size(enp, ndescs), EFX_BUF_SIZE));
316}
317
11fdf7f2
TL		 318 __checkReturn efx_rc_t
319efx_tx_qcreate(
320 __in efx_nic_t *enp,
321 __in unsigned int index,
322 __in unsigned int label,
323 __in efsys_mem_t *esmp,
324 __in size_t ndescs,
325 __in uint32_t id,
326 __in uint16_t flags,
327 __in efx_evq_t *eep,
328 __deref_out efx_txq_t **etpp,
329 __out unsigned int *addedp)
330{
331 const efx_tx_ops_t *etxop = enp->en_etxop;
332 efx_txq_t *etp;
9f95a23c 333 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
11fdf7f2
TL		 334 efx_rc_t rc;
335
336 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
337 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
338
339 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <,
340 enp->en_nic_cfg.enc_txq_limit);
341
9f95a23c
TL		 342 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs));
343 EFSYS_ASSERT(ISP2(encp->enc_txq_min_ndescs));
344
345 if (!ISP2(ndescs) ||
346 ndescs < encp->enc_txq_min_ndescs ||
347 ndescs > encp->enc_txq_max_ndescs) {
348 rc = EINVAL;
349 goto fail1;
350 }
351
11fdf7f2
TL		 352 /* Allocate an TXQ object */
353 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
354
355 if (etp == NULL) {
356 rc = ENOMEM;
9f95a23c 357 goto fail2;
11fdf7f2
TL		 358 }
359
360 etp->et_magic = EFX_TXQ_MAGIC;
361 etp->et_enp = enp;
362 etp->et_index = index;
363 etp->et_mask = ndescs - 1;
364 etp->et_esmp = esmp;
365
366 /* Initial descriptor index may be modified by etxo_qcreate */
367 *addedp = 0;
368
369 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
370 ndescs, id, flags, eep, etp, addedp)) != 0)
9f95a23c 371 goto fail3;
11fdf7f2
TL		 372
373 enp->en_tx_qcount++;
374 *etpp = etp;
375
376 return (0);
377
9f95a23c
TL		 378fail3:
379 EFSYS_PROBE(fail3);
380 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
11fdf7f2
TL		 381fail2:
382 EFSYS_PROBE(fail2);
11fdf7f2
TL		 383fail1:
384 EFSYS_PROBE1(fail1, efx_rc_t, rc);
385 return (rc);
386}
387
388 void
389efx_tx_qdestroy(
390 __in efx_txq_t *etp)
391{
392 efx_nic_t *enp = etp->et_enp;
393 const efx_tx_ops_t *etxop = enp->en_etxop;
394
395 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
396
397 EFSYS_ASSERT(enp->en_tx_qcount != 0);
398 --enp->en_tx_qcount;
399
400 etxop->etxo_qdestroy(etp);
401
402 /* Free the TXQ object */
403 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
404}
405
406 __checkReturn efx_rc_t
407efx_tx_qpost(
408 __in efx_txq_t *etp,
409 __in_ecount(ndescs) efx_buffer_t *eb,
410 __in unsigned int ndescs,
411 __in unsigned int completed,
412 __inout unsigned int *addedp)
413{
414 efx_nic_t *enp = etp->et_enp;
415 const efx_tx_ops_t *etxop = enp->en_etxop;
416 efx_rc_t rc;
417
418 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
419
420 if ((rc = etxop->etxo_qpost(etp, eb, ndescs, completed, addedp)) != 0)
421 goto fail1;
422
423 return (0);
424
425fail1:
426 EFSYS_PROBE1(fail1, efx_rc_t, rc);
427 return (rc);
428}
429
430 void
431efx_tx_qpush(
432 __in efx_txq_t *etp,
433 __in unsigned int added,
434 __in unsigned int pushed)
435{
436 efx_nic_t *enp = etp->et_enp;
437 const efx_tx_ops_t *etxop = enp->en_etxop;
438
439 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
440
441 etxop->etxo_qpush(etp, added, pushed);
442}
443
444 __checkReturn efx_rc_t
445efx_tx_qpace(
446 __in efx_txq_t *etp,
447 __in unsigned int ns)
448{
449 efx_nic_t *enp = etp->et_enp;
450 const efx_tx_ops_t *etxop = enp->en_etxop;
451 efx_rc_t rc;
452
453 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
454
455 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
456 goto fail1;
457
458 return (0);
459
460fail1:
461 EFSYS_PROBE1(fail1, efx_rc_t, rc);
462 return (rc);
463}
464
465 __checkReturn efx_rc_t
466efx_tx_qflush(
467 __in efx_txq_t *etp)
468{
469 efx_nic_t *enp = etp->et_enp;
470 const efx_tx_ops_t *etxop = enp->en_etxop;
471 efx_rc_t rc;
472
473 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
474
475 if ((rc = etxop->etxo_qflush(etp)) != 0)
476 goto fail1;
477
478 return (0);
479
480fail1:
481 EFSYS_PROBE1(fail1, efx_rc_t, rc);
482 return (rc);
483}
484
485 void
486efx_tx_qenable(
487 __in efx_txq_t *etp)
488{
489 efx_nic_t *enp = etp->et_enp;
490 const efx_tx_ops_t *etxop = enp->en_etxop;
491
492 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
493
494 etxop->etxo_qenable(etp);
495}
496
497 __checkReturn efx_rc_t
498efx_tx_qpio_enable(
499 __in efx_txq_t *etp)
500{
501 efx_nic_t *enp = etp->et_enp;
502 const efx_tx_ops_t *etxop = enp->en_etxop;
503 efx_rc_t rc;
504
505 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
506
507 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
508 rc = ENOTSUP;
509 goto fail1;
510 }
511 if (etxop->etxo_qpio_enable == NULL) {
512 rc = ENOTSUP;
513 goto fail2;
514 }
515 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
516 goto fail3;
517
518 return (0);
519
520fail3:
521 EFSYS_PROBE(fail3);
522fail2:
523 EFSYS_PROBE(fail2);
524fail1:
525 EFSYS_PROBE1(fail1, efx_rc_t, rc);
526 return (rc);
527}
528
529 void
530efx_tx_qpio_disable(
531 __in efx_txq_t *etp)
532{
533 efx_nic_t *enp = etp->et_enp;
534 const efx_tx_ops_t *etxop = enp->en_etxop;
535
536 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
537
538 if (etxop->etxo_qpio_disable != NULL)
539 etxop->etxo_qpio_disable(etp);
540}
541
542 __checkReturn efx_rc_t
543efx_tx_qpio_write(
544 __in efx_txq_t *etp,
545 __in_ecount(buf_length) uint8_t *buffer,
546 __in size_t buf_length,
547 __in size_t pio_buf_offset)
548{
549 efx_nic_t *enp = etp->et_enp;
550 const efx_tx_ops_t *etxop = enp->en_etxop;
551 efx_rc_t rc;
552
553 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
554
555 if (etxop->etxo_qpio_write != NULL) {
556 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
557 pio_buf_offset)) != 0)
558 goto fail1;
559 return (0);
560 }
561
562 return (ENOTSUP);
563
564fail1:
565 EFSYS_PROBE1(fail1, efx_rc_t, rc);
566 return (rc);
567}
568
569 __checkReturn efx_rc_t
570efx_tx_qpio_post(
571 __in efx_txq_t *etp,
572 __in size_t pkt_length,
573 __in unsigned int completed,
574 __inout unsigned int *addedp)
575{
576 efx_nic_t *enp = etp->et_enp;
577 const efx_tx_ops_t *etxop = enp->en_etxop;
578 efx_rc_t rc;
579
580 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
581
582 if (etxop->etxo_qpio_post != NULL) {
583 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
584 addedp)) != 0)
585 goto fail1;
586 return (0);
587 }
588
589 return (ENOTSUP);
590
591fail1:
592 EFSYS_PROBE1(fail1, efx_rc_t, rc);
593 return (rc);
594}
595
596 __checkReturn efx_rc_t
597efx_tx_qdesc_post(
598 __in efx_txq_t *etp,
599 __in_ecount(ndescs) efx_desc_t *ed,
600 __in unsigned int ndescs,
601 __in unsigned int completed,
602 __inout unsigned int *addedp)
603{
604 efx_nic_t *enp = etp->et_enp;
605 const efx_tx_ops_t *etxop = enp->en_etxop;
11fdf7f2
TL		 606
607 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
608
9f95a23c 609 return (etxop->etxo_qdesc_post(etp, ed, ndescs, completed, addedp));
11fdf7f2
TL		 610}
611
612 void
613efx_tx_qdesc_dma_create(
614 __in efx_txq_t *etp,
615 __in efsys_dma_addr_t addr,
616 __in size_t size,
617 __in boolean_t eop,
618 __out efx_desc_t *edp)
619{
620 efx_nic_t *enp = etp->et_enp;
621 const efx_tx_ops_t *etxop = enp->en_etxop;
622
623 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
624 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
625
626 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
627}
628
629 void
630efx_tx_qdesc_tso_create(
631 __in efx_txq_t *etp,
632 __in uint16_t ipv4_id,
633 __in uint32_t tcp_seq,
634 __in uint8_t tcp_flags,
635 __out efx_desc_t *edp)
636{
637 efx_nic_t *enp = etp->et_enp;
638 const efx_tx_ops_t *etxop = enp->en_etxop;
639
640 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
641 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
642
643 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
644}
645
646 void
647efx_tx_qdesc_tso2_create(
648 __in efx_txq_t *etp,
649 __in uint16_t ipv4_id,
650 __in uint16_t outer_ipv4_id,
651 __in uint32_t tcp_seq,
652 __in uint16_t mss,
653 __out_ecount(count) efx_desc_t *edp,
654 __in int count)
655{
656 efx_nic_t *enp = etp->et_enp;
657 const efx_tx_ops_t *etxop = enp->en_etxop;
658
659 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
660 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
661
662 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, outer_ipv4_id,
663 tcp_seq, mss, edp, count);
664}
665
666 void
667efx_tx_qdesc_vlantci_create(
668 __in efx_txq_t *etp,
669 __in uint16_t tci,
670 __out efx_desc_t *edp)
671{
672 efx_nic_t *enp = etp->et_enp;
673 const efx_tx_ops_t *etxop = enp->en_etxop;
674
675 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
676 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
677
678 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
679}
680
681 void
682efx_tx_qdesc_checksum_create(
683 __in efx_txq_t *etp,
684 __in uint16_t flags,
685 __out efx_desc_t *edp)
686{
687 efx_nic_t *enp = etp->et_enp;
688 const efx_tx_ops_t *etxop = enp->en_etxop;
689
690 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
691 EFSYS_ASSERT(etxop->etxo_qdesc_checksum_create != NULL);
692
693 etxop->etxo_qdesc_checksum_create(etp, flags, edp);
694}
695
696
697#if EFSYS_OPT_QSTATS
698 void
699efx_tx_qstats_update(
700 __in efx_txq_t *etp,
701 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
702{
703 efx_nic_t *enp = etp->et_enp;
704 const efx_tx_ops_t *etxop = enp->en_etxop;
705
706 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
707
708 etxop->etxo_qstats_update(etp, stat);
709}
710#endif
711
712
713#if EFSYS_OPT_SIENA
714
715static __checkReturn efx_rc_t
716siena_tx_init(
717 __in efx_nic_t *enp)
718{
719 efx_oword_t oword;
720
721 /*
722 * Disable the timer-based TX DMA backoff and allow TX DMA to be
723 * controlled by the RX FIFO fill level (although always allow a
724 * minimal trickle).
725 */
726 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
727 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
728 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
729 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
730 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
731 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
732 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
733 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
734
735 /*
736 * Filter all packets less than 14 bytes to avoid parsing
737 * errors.
738 */
739 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
740 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
741
742 /*
743 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
744 * descriptors (which is bad).
745 */
746 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
747 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
748 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
749
750 return (0);
751}
752
753#define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
754 do { \
755 unsigned int id; \
756 size_t offset; \
757 efx_qword_t qword; \
758 \
759 id = (_added)++ & (_etp)->et_mask; \
760 offset = id * sizeof (efx_qword_t); \
761 \
762 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
763 unsigned int, id, efsys_dma_addr_t, (_addr), \
764 size_t, (_size), boolean_t, (_eop)); \
765 \
766 EFX_POPULATE_QWORD_4(qword, \
767 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
768 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
769 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
770 (uint32_t)((_addr) & 0xffffffff), \
771 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
772 (uint32_t)((_addr) >> 32)); \
773 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
774 \
775 _NOTE(CONSTANTCONDITION) \
776 } while (B_FALSE)
777
778static __checkReturn efx_rc_t
779siena_tx_qpost(
780 __in efx_txq_t *etp,
781 __in_ecount(ndescs) efx_buffer_t *eb,
782 __in unsigned int ndescs,
783 __in unsigned int completed,
784 __inout unsigned int *addedp)
785{
786 unsigned int added = *addedp;
787 unsigned int i;
11fdf7f2
TL		 788
789 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1))
9f95a23c 790 return (ENOSPC);
11fdf7f2
TL		 791
792 for (i = 0; i < ndescs; i++) {
793 efx_buffer_t *ebp = &eb[i];
794 efsys_dma_addr_t start = ebp->eb_addr;
795 size_t size = ebp->eb_size;
796 efsys_dma_addr_t end = start + size;
797
798 /*
799 * Fragments must not span 4k boundaries.
800 * Here it is a stricter requirement than the maximum length.
801 */
f67539c2 802 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, start + 1,
11fdf7f2
TL		 803 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end);
804
805 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
806 }
807
808 EFX_TX_QSTAT_INCR(etp, TX_POST);
809
810 *addedp = added;
811 return (0);
11fdf7f2
TL		 812}
813
814static void
815siena_tx_qpush(
816 __in efx_txq_t *etp,
817 __in unsigned int added,
818 __in unsigned int pushed)
819{
820 efx_nic_t *enp = etp->et_enp;
821 uint32_t wptr;
822 efx_dword_t dword;
823 efx_oword_t oword;
824
825 /* Push the populated descriptors out */
826 wptr = added & etp->et_mask;
827
828 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
829
830 /* Only write the third DWORD */
831 EFX_POPULATE_DWORD_1(dword,
832 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
833
834 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
835 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
836 wptr, pushed & etp->et_mask);
837 EFSYS_PIO_WRITE_BARRIER();
838 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
839 etp->et_index, &dword, B_FALSE);
840}
841
842#define EFX_MAX_PACE_VALUE 20
843
844static __checkReturn efx_rc_t
845siena_tx_qpace(
846 __in efx_txq_t *etp,
847 __in unsigned int ns)
848{
849 efx_nic_t *enp = etp->et_enp;
850 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
851 efx_oword_t oword;
852 unsigned int pace_val;
853 unsigned int timer_period;
854 efx_rc_t rc;
855
856 if (ns == 0) {
857 pace_val = 0;
858 } else {
859 /*
860 * The pace_val to write into the table is s.t
861 * ns <= timer_period * (2 ^ pace_val)
862 */
863 timer_period = 104 / encp->enc_clk_mult;
864 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
865 if ((timer_period << pace_val) >= ns)
866 break;
867 }
868 }
869 if (pace_val > EFX_MAX_PACE_VALUE) {
870 rc = EINVAL;
871 goto fail1;
872 }
873
874 /* Update the pacing table */
875 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
876 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
877 &oword, B_TRUE);
878
879 return (0);
880
881fail1:
882 EFSYS_PROBE1(fail1, efx_rc_t, rc);
883
884 return (rc);
885}
886
887static __checkReturn efx_rc_t
888siena_tx_qflush(
889 __in efx_txq_t *etp)
890{
891 efx_nic_t *enp = etp->et_enp;
892 efx_oword_t oword;
893 uint32_t label;
894
895 efx_tx_qpace(etp, 0);
896
897 label = etp->et_index;
898
899 /* Flush the queue */
900 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
901 FRF_AZ_TX_FLUSH_DESCQ, label);
902 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
903
904 return (0);
905}
906
907static void
908siena_tx_qenable(
909 __in efx_txq_t *etp)
910{
911 efx_nic_t *enp = etp->et_enp;
912 efx_oword_t oword;
913
914 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
915 etp->et_index, &oword, B_TRUE);
916
917 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
918 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
919 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
920 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
921 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
922
923 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
924 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
925 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
926
927 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
928 etp->et_index, &oword, B_TRUE);
929}
930
931static __checkReturn efx_rc_t
932siena_tx_qcreate(
933 __in efx_nic_t *enp,
934 __in unsigned int index,
935 __in unsigned int label,
936 __in efsys_mem_t *esmp,
937 __in size_t ndescs,
938 __in uint32_t id,
939 __in uint16_t flags,
940 __in efx_evq_t *eep,
941 __in efx_txq_t *etp,
942 __out unsigned int *addedp)
943{
944 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
945 efx_oword_t oword;
946 uint32_t size;
947 uint16_t inner_csum;
948 efx_rc_t rc;
949
950 _NOTE(ARGUNUSED(esmp))
951
952 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
953 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
954 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
955
11fdf7f2
TL		 956 if (index >= encp->enc_txq_limit) {
957 rc = EINVAL;
9f95a23c 958 goto fail1;
11fdf7f2
TL		 959 }
960 for (size = 0;
9f95a23c 961 (1U << size) <= encp->enc_txq_max_ndescs / encp->enc_txq_min_ndescs;
11fdf7f2 962 size++)
9f95a23c 963 if ((1U << size) == (uint32_t)ndescs / encp->enc_txq_min_ndescs)
11fdf7f2
TL		 964 break;
965 if (id + (1 << size) >= encp->enc_buftbl_limit) {
966 rc = EINVAL;
9f95a23c 967 goto fail2;
11fdf7f2
TL		 968 }
969
970 inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP;
971 if ((flags & inner_csum) != 0) {
972 rc = EINVAL;
9f95a23c 973 goto fail3;
11fdf7f2
TL		 974 }
975
976 /* Set up the new descriptor queue */
977 *addedp = 0;
978
979 EFX_POPULATE_OWORD_6(oword,
980 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
981 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
982 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
983 FRF_AZ_TX_DESCQ_LABEL, label,
984 FRF_AZ_TX_DESCQ_SIZE, size,
985 FRF_AZ_TX_DESCQ_TYPE, 0);
986
987 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
988 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
989 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
990 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
991 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
992
993 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
994 etp->et_index, &oword, B_TRUE);
995
996 return (0);
997
11fdf7f2
TL		 998fail3:
999 EFSYS_PROBE(fail3);
1000fail2:
1001 EFSYS_PROBE(fail2);
1002fail1:
1003 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1004
1005 return (rc);
1006}
1007
1008 __checkReturn efx_rc_t
1009siena_tx_qdesc_post(
1010 __in efx_txq_t *etp,
1011 __in_ecount(ndescs) efx_desc_t *ed,
1012 __in unsigned int ndescs,
1013 __in unsigned int completed,
1014 __inout unsigned int *addedp)
1015{
1016 unsigned int added = *addedp;
1017 unsigned int i;
1018 efx_rc_t rc;
1019
1020 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
1021 rc = ENOSPC;
1022 goto fail1;
1023 }
1024
1025 for (i = 0; i < ndescs; i++) {
1026 efx_desc_t *edp = &ed[i];
1027 unsigned int id;
1028 size_t offset;
1029
1030 id = added++ & etp->et_mask;
1031 offset = id * sizeof (efx_desc_t);
1032
1033 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
1034 }
1035
1036 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
1037 unsigned int, added, unsigned int, ndescs);
1038
1039 EFX_TX_QSTAT_INCR(etp, TX_POST);
1040
1041 *addedp = added;
1042 return (0);
1043
1044fail1:
1045 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1046 return (rc);
1047}
1048
1049 void
1050siena_tx_qdesc_dma_create(
1051 __in efx_txq_t *etp,
1052 __in efsys_dma_addr_t addr,
1053 __in size_t size,
1054 __in boolean_t eop,
1055 __out efx_desc_t *edp)
1056{
1057 /*
1058 * Fragments must not span 4k boundaries.
1059 * Here it is a stricter requirement than the maximum length.
1060 */
f67539c2 1061 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, addr + 1,
11fdf7f2
TL		 1062 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size);
1063
1064 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1065 efsys_dma_addr_t, addr,
1066 size_t, size, boolean_t, eop);
1067
1068 EFX_POPULATE_QWORD_4(edp->ed_eq,
1069 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1070 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1071 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1072 (uint32_t)(addr & 0xffffffff),
1073 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1074 (uint32_t)(addr >> 32));
1075}
1076
1077#endif /* EFSYS_OPT_SIENA */
1078
1079#if EFSYS_OPT_QSTATS
1080#if EFSYS_OPT_NAMES
1081/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */
1082static const char * const __efx_tx_qstat_name[] = {
1083 "post",
1084 "post_pio",
1085};
1086/* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1087
1088 const char *
1089efx_tx_qstat_name(
1090 __in efx_nic_t *enp,
1091 __in unsigned int id)
1092{
1093 _NOTE(ARGUNUSED(enp))
1094 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1095 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1096
1097 return (__efx_tx_qstat_name[id]);
1098}
1099#endif /* EFSYS_OPT_NAMES */
1100#endif /* EFSYS_OPT_QSTATS */
1101
1102#if EFSYS_OPT_SIENA
1103
1104#if EFSYS_OPT_QSTATS
1105static void
1106siena_tx_qstats_update(
1107 __in efx_txq_t *etp,
1108 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1109{
1110 unsigned int id;
1111
1112 for (id = 0; id < TX_NQSTATS; id++) {
1113 efsys_stat_t *essp = &stat[id];
1114
1115 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1116 etp->et_stat[id] = 0;
1117 }
1118}
1119#endif /* EFSYS_OPT_QSTATS */
1120
1121static void
1122siena_tx_qdestroy(
1123 __in efx_txq_t *etp)
1124{
1125 efx_nic_t *enp = etp->et_enp;
1126 efx_oword_t oword;
1127
1128 /* Purge descriptor queue */
1129 EFX_ZERO_OWORD(oword);
1130
1131 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1132 etp->et_index, &oword, B_TRUE);
1133}
1134
1135static void
1136siena_tx_fini(
1137 __in efx_nic_t *enp)
1138{
1139 _NOTE(ARGUNUSED(enp))
1140}
1141
1142#endif /* EFSYS_OPT_SIENA */
Ceph