]> git.proxmox.com Git - ceph.git/blob - ceph/src/spdk/dpdk/drivers/net/sfc/base/efx_tx.c
projects / ceph.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
update sources to ceph Nautilus 14.2.1
[ceph.git] / ceph / src / spdk / dpdk / drivers / net / sfc / base / efx_tx.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2 *
3 * Copyright (c) 2007-2018 Solarflare Communications Inc.
4 * All rights reserved.
5 */
6
7 #include "efx.h"
8 #include "efx_impl.h"
9
10 #if EFSYS_OPT_QSTATS
11 #define EFX_TX_QSTAT_INCR(_etp, _stat) \
12 do { \
13 (_etp)->et_stat[_stat]++; \
14 _NOTE(CONSTANTCONDITION) \
15 } while (B_FALSE)
16 #else
17 #define EFX_TX_QSTAT_INCR(_etp, _stat)
18 #endif
19
20 #if EFSYS_OPT_SIENA
21
22 static __checkReturn efx_rc_t
23 siena_tx_init(
24 __in efx_nic_t *enp);
25
26 static void
27 siena_tx_fini(
28 __in efx_nic_t *enp);
29
30 static __checkReturn efx_rc_t
31 siena_tx_qcreate(
32 __in efx_nic_t *enp,
33 __in unsigned int index,
34 __in unsigned int label,
35 __in efsys_mem_t *esmp,
36 __in size_t ndescs,
37 __in uint32_t id,
38 __in uint16_t flags,
39 __in efx_evq_t *eep,
40 __in efx_txq_t *etp,
41 __out unsigned int *addedp);
42
43 static void
44 siena_tx_qdestroy(
45 __in efx_txq_t *etp);
46
47 static __checkReturn efx_rc_t
48 siena_tx_qpost(
49 __in efx_txq_t *etp,
50 __in_ecount(ndescs) efx_buffer_t *eb,
51 __in unsigned int ndescs,
52 __in unsigned int completed,
53 __inout unsigned int *addedp);
54
55 static void
56 siena_tx_qpush(
57 __in efx_txq_t *etp,
58 __in unsigned int added,
59 __in unsigned int pushed);
60
61 static __checkReturn efx_rc_t
62 siena_tx_qpace(
63 __in efx_txq_t *etp,
64 __in unsigned int ns);
65
66 static __checkReturn efx_rc_t
67 siena_tx_qflush(
68 __in efx_txq_t *etp);
69
70 static void
71 siena_tx_qenable(
72 __in efx_txq_t *etp);
73
74 __checkReturn efx_rc_t
75 siena_tx_qdesc_post(
76 __in efx_txq_t *etp,
77 __in_ecount(ndescs) efx_desc_t *ed,
78 __in unsigned int ndescs,
79 __in unsigned int completed,
80 __inout unsigned int *addedp);
81
82 void
83 siena_tx_qdesc_dma_create(
84 __in efx_txq_t *etp,
85 __in efsys_dma_addr_t addr,
86 __in size_t size,
87 __in boolean_t eop,
88 __out efx_desc_t *edp);
89
90 #if EFSYS_OPT_QSTATS
91 static void
92 siena_tx_qstats_update(
93 __in efx_txq_t *etp,
94 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
95 #endif
96
97 #endif /* EFSYS_OPT_SIENA */
98
99
100 #if EFSYS_OPT_SIENA
101 static const efx_tx_ops_t __efx_tx_siena_ops = {
102 siena_tx_init, /* etxo_init */
103 siena_tx_fini, /* etxo_fini */
104 siena_tx_qcreate, /* etxo_qcreate */
105 siena_tx_qdestroy, /* etxo_qdestroy */
106 siena_tx_qpost, /* etxo_qpost */
107 siena_tx_qpush, /* etxo_qpush */
108 siena_tx_qpace, /* etxo_qpace */
109 siena_tx_qflush, /* etxo_qflush */
110 siena_tx_qenable, /* etxo_qenable */
111 NULL, /* etxo_qpio_enable */
112 NULL, /* etxo_qpio_disable */
113 NULL, /* etxo_qpio_write */
114 NULL, /* etxo_qpio_post */
115 siena_tx_qdesc_post, /* etxo_qdesc_post */
116 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
117 NULL, /* etxo_qdesc_tso_create */
118 NULL, /* etxo_qdesc_tso2_create */
119 NULL, /* etxo_qdesc_vlantci_create */
120 NULL, /* etxo_qdesc_checksum_create */
121 #if EFSYS_OPT_QSTATS
122 siena_tx_qstats_update, /* etxo_qstats_update */
123 #endif
124 };
125 #endif /* EFSYS_OPT_SIENA */
126
127 #if EFSYS_OPT_HUNTINGTON
128 static const efx_tx_ops_t __efx_tx_hunt_ops = {
129 ef10_tx_init, /* etxo_init */
130 ef10_tx_fini, /* etxo_fini */
131 ef10_tx_qcreate, /* etxo_qcreate */
132 ef10_tx_qdestroy, /* etxo_qdestroy */
133 ef10_tx_qpost, /* etxo_qpost */
134 ef10_tx_qpush, /* etxo_qpush */
135 ef10_tx_qpace, /* etxo_qpace */
136 ef10_tx_qflush, /* etxo_qflush */
137 ef10_tx_qenable, /* etxo_qenable */
138 ef10_tx_qpio_enable, /* etxo_qpio_enable */
139 ef10_tx_qpio_disable, /* etxo_qpio_disable */
140 ef10_tx_qpio_write, /* etxo_qpio_write */
141 ef10_tx_qpio_post, /* etxo_qpio_post */
142 ef10_tx_qdesc_post, /* etxo_qdesc_post */
143 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
144 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
145 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
146 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
147 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
148 #if EFSYS_OPT_QSTATS
149 ef10_tx_qstats_update, /* etxo_qstats_update */
150 #endif
151 };
152 #endif /* EFSYS_OPT_HUNTINGTON */
153
154 #if EFSYS_OPT_MEDFORD
155 static const efx_tx_ops_t __efx_tx_medford_ops = {
156 ef10_tx_init, /* etxo_init */
157 ef10_tx_fini, /* etxo_fini */
158 ef10_tx_qcreate, /* etxo_qcreate */
159 ef10_tx_qdestroy, /* etxo_qdestroy */
160 ef10_tx_qpost, /* etxo_qpost */
161 ef10_tx_qpush, /* etxo_qpush */
162 ef10_tx_qpace, /* etxo_qpace */
163 ef10_tx_qflush, /* etxo_qflush */
164 ef10_tx_qenable, /* etxo_qenable */
165 ef10_tx_qpio_enable, /* etxo_qpio_enable */
166 ef10_tx_qpio_disable, /* etxo_qpio_disable */
167 ef10_tx_qpio_write, /* etxo_qpio_write */
168 ef10_tx_qpio_post, /* etxo_qpio_post */
169 ef10_tx_qdesc_post, /* etxo_qdesc_post */
170 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
171 NULL, /* etxo_qdesc_tso_create */
172 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
173 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
174 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
175 #if EFSYS_OPT_QSTATS
176 ef10_tx_qstats_update, /* etxo_qstats_update */
177 #endif
178 };
179 #endif /* EFSYS_OPT_MEDFORD */
180
181 #if EFSYS_OPT_MEDFORD2
182 static const efx_tx_ops_t __efx_tx_medford2_ops = {
183 ef10_tx_init, /* etxo_init */
184 ef10_tx_fini, /* etxo_fini */
185 ef10_tx_qcreate, /* etxo_qcreate */
186 ef10_tx_qdestroy, /* etxo_qdestroy */
187 ef10_tx_qpost, /* etxo_qpost */
188 ef10_tx_qpush, /* etxo_qpush */
189 ef10_tx_qpace, /* etxo_qpace */
190 ef10_tx_qflush, /* etxo_qflush */
191 ef10_tx_qenable, /* etxo_qenable */
192 ef10_tx_qpio_enable, /* etxo_qpio_enable */
193 ef10_tx_qpio_disable, /* etxo_qpio_disable */
194 ef10_tx_qpio_write, /* etxo_qpio_write */
195 ef10_tx_qpio_post, /* etxo_qpio_post */
196 ef10_tx_qdesc_post, /* etxo_qdesc_post */
197 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
198 NULL, /* etxo_qdesc_tso_create */
199 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
200 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
201 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
202 #if EFSYS_OPT_QSTATS
203 ef10_tx_qstats_update, /* etxo_qstats_update */
204 #endif
205 };
206 #endif /* EFSYS_OPT_MEDFORD2 */
207
208
209 __checkReturn efx_rc_t
210 efx_tx_init(
211 __in efx_nic_t *enp)
212 {
213 const efx_tx_ops_t *etxop;
214 efx_rc_t rc;
215
216 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
217 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
218
219 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
220 rc = EINVAL;
221 goto fail1;
222 }
223
224 if (enp->en_mod_flags & EFX_MOD_TX) {
225 rc = EINVAL;
226 goto fail2;
227 }
228
229 switch (enp->en_family) {
230 #if EFSYS_OPT_SIENA
231 case EFX_FAMILY_SIENA:
232 etxop = &__efx_tx_siena_ops;
233 break;
234 #endif /* EFSYS_OPT_SIENA */
235
236 #if EFSYS_OPT_HUNTINGTON
237 case EFX_FAMILY_HUNTINGTON:
238 etxop = &__efx_tx_hunt_ops;
239 break;
240 #endif /* EFSYS_OPT_HUNTINGTON */
241
242 #if EFSYS_OPT_MEDFORD
243 case EFX_FAMILY_MEDFORD:
244 etxop = &__efx_tx_medford_ops;
245 break;
246 #endif /* EFSYS_OPT_MEDFORD */
247
248 #if EFSYS_OPT_MEDFORD2
249 case EFX_FAMILY_MEDFORD2:
250 etxop = &__efx_tx_medford2_ops;
251 break;
252 #endif /* EFSYS_OPT_MEDFORD2 */
253
254 default:
255 EFSYS_ASSERT(0);
256 rc = ENOTSUP;
257 goto fail3;
258 }
259
260 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
261
262 if ((rc = etxop->etxo_init(enp)) != 0)
263 goto fail4;
264
265 enp->en_etxop = etxop;
266 enp->en_mod_flags |= EFX_MOD_TX;
267 return (0);
268
269 fail4:
270 EFSYS_PROBE(fail4);
271 fail3:
272 EFSYS_PROBE(fail3);
273 fail2:
274 EFSYS_PROBE(fail2);
275 fail1:
276 EFSYS_PROBE1(fail1, efx_rc_t, rc);
277
278 enp->en_etxop = NULL;
279 enp->en_mod_flags &= ~EFX_MOD_TX;
280 return (rc);
281 }
282
283 void
284 efx_tx_fini(
285 __in efx_nic_t *enp)
286 {
287 const efx_tx_ops_t *etxop = enp->en_etxop;
288
289 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
290 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
291 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
292 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
293
294 etxop->etxo_fini(enp);
295
296 enp->en_etxop = NULL;
297 enp->en_mod_flags &= ~EFX_MOD_TX;
298 }
299
300 __checkReturn efx_rc_t
301 efx_tx_qcreate(
302 __in efx_nic_t *enp,
303 __in unsigned int index,
304 __in unsigned int label,
305 __in efsys_mem_t *esmp,
306 __in size_t ndescs,
307 __in uint32_t id,
308 __in uint16_t flags,
309 __in efx_evq_t *eep,
310 __deref_out efx_txq_t **etpp,
311 __out unsigned int *addedp)
312 {
313 const efx_tx_ops_t *etxop = enp->en_etxop;
314 efx_txq_t *etp;
315 efx_rc_t rc;
316
317 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
318 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
319
320 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <,
321 enp->en_nic_cfg.enc_txq_limit);
322
323 /* Allocate an TXQ object */
324 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
325
326 if (etp == NULL) {
327 rc = ENOMEM;
328 goto fail1;
329 }
330
331 etp->et_magic = EFX_TXQ_MAGIC;
332 etp->et_enp = enp;
333 etp->et_index = index;
334 etp->et_mask = ndescs - 1;
335 etp->et_esmp = esmp;
336
337 /* Initial descriptor index may be modified by etxo_qcreate */
338 *addedp = 0;
339
340 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
341 ndescs, id, flags, eep, etp, addedp)) != 0)
342 goto fail2;
343
344 enp->en_tx_qcount++;
345 *etpp = etp;
346
347 return (0);
348
349 fail2:
350 EFSYS_PROBE(fail2);
351 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
352 fail1:
353 EFSYS_PROBE1(fail1, efx_rc_t, rc);
354 return (rc);
355 }
356
357 void
358 efx_tx_qdestroy(
359 __in efx_txq_t *etp)
360 {
361 efx_nic_t *enp = etp->et_enp;
362 const efx_tx_ops_t *etxop = enp->en_etxop;
363
364 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
365
366 EFSYS_ASSERT(enp->en_tx_qcount != 0);
367 --enp->en_tx_qcount;
368
369 etxop->etxo_qdestroy(etp);
370
371 /* Free the TXQ object */
372 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
373 }
374
375 __checkReturn efx_rc_t
376 efx_tx_qpost(
377 __in efx_txq_t *etp,
378 __in_ecount(ndescs) efx_buffer_t *eb,
379 __in unsigned int ndescs,
380 __in unsigned int completed,
381 __inout unsigned int *addedp)
382 {
383 efx_nic_t *enp = etp->et_enp;
384 const efx_tx_ops_t *etxop = enp->en_etxop;
385 efx_rc_t rc;
386
387 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
388
389 if ((rc = etxop->etxo_qpost(etp, eb, ndescs, completed, addedp)) != 0)
390 goto fail1;
391
392 return (0);
393
394 fail1:
395 EFSYS_PROBE1(fail1, efx_rc_t, rc);
396 return (rc);
397 }
398
399 void
400 efx_tx_qpush(
401 __in efx_txq_t *etp,
402 __in unsigned int added,
403 __in unsigned int pushed)
404 {
405 efx_nic_t *enp = etp->et_enp;
406 const efx_tx_ops_t *etxop = enp->en_etxop;
407
408 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
409
410 etxop->etxo_qpush(etp, added, pushed);
411 }
412
413 __checkReturn efx_rc_t
414 efx_tx_qpace(
415 __in efx_txq_t *etp,
416 __in unsigned int ns)
417 {
418 efx_nic_t *enp = etp->et_enp;
419 const efx_tx_ops_t *etxop = enp->en_etxop;
420 efx_rc_t rc;
421
422 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
423
424 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
425 goto fail1;
426
427 return (0);
428
429 fail1:
430 EFSYS_PROBE1(fail1, efx_rc_t, rc);
431 return (rc);
432 }
433
434 __checkReturn efx_rc_t
435 efx_tx_qflush(
436 __in efx_txq_t *etp)
437 {
438 efx_nic_t *enp = etp->et_enp;
439 const efx_tx_ops_t *etxop = enp->en_etxop;
440 efx_rc_t rc;
441
442 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
443
444 if ((rc = etxop->etxo_qflush(etp)) != 0)
445 goto fail1;
446
447 return (0);
448
449 fail1:
450 EFSYS_PROBE1(fail1, efx_rc_t, rc);
451 return (rc);
452 }
453
454 void
455 efx_tx_qenable(
456 __in efx_txq_t *etp)
457 {
458 efx_nic_t *enp = etp->et_enp;
459 const efx_tx_ops_t *etxop = enp->en_etxop;
460
461 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
462
463 etxop->etxo_qenable(etp);
464 }
465
466 __checkReturn efx_rc_t
467 efx_tx_qpio_enable(
468 __in efx_txq_t *etp)
469 {
470 efx_nic_t *enp = etp->et_enp;
471 const efx_tx_ops_t *etxop = enp->en_etxop;
472 efx_rc_t rc;
473
474 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
475
476 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
477 rc = ENOTSUP;
478 goto fail1;
479 }
480 if (etxop->etxo_qpio_enable == NULL) {
481 rc = ENOTSUP;
482 goto fail2;
483 }
484 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
485 goto fail3;
486
487 return (0);
488
489 fail3:
490 EFSYS_PROBE(fail3);
491 fail2:
492 EFSYS_PROBE(fail2);
493 fail1:
494 EFSYS_PROBE1(fail1, efx_rc_t, rc);
495 return (rc);
496 }
497
498 void
499 efx_tx_qpio_disable(
500 __in efx_txq_t *etp)
501 {
502 efx_nic_t *enp = etp->et_enp;
503 const efx_tx_ops_t *etxop = enp->en_etxop;
504
505 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
506
507 if (etxop->etxo_qpio_disable != NULL)
508 etxop->etxo_qpio_disable(etp);
509 }
510
511 __checkReturn efx_rc_t
512 efx_tx_qpio_write(
513 __in efx_txq_t *etp,
514 __in_ecount(buf_length) uint8_t *buffer,
515 __in size_t buf_length,
516 __in size_t pio_buf_offset)
517 {
518 efx_nic_t *enp = etp->et_enp;
519 const efx_tx_ops_t *etxop = enp->en_etxop;
520 efx_rc_t rc;
521
522 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
523
524 if (etxop->etxo_qpio_write != NULL) {
525 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
526 pio_buf_offset)) != 0)
527 goto fail1;
528 return (0);
529 }
530
531 return (ENOTSUP);
532
533 fail1:
534 EFSYS_PROBE1(fail1, efx_rc_t, rc);
535 return (rc);
536 }
537
538 __checkReturn efx_rc_t
539 efx_tx_qpio_post(
540 __in efx_txq_t *etp,
541 __in size_t pkt_length,
542 __in unsigned int completed,
543 __inout unsigned int *addedp)
544 {
545 efx_nic_t *enp = etp->et_enp;
546 const efx_tx_ops_t *etxop = enp->en_etxop;
547 efx_rc_t rc;
548
549 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
550
551 if (etxop->etxo_qpio_post != NULL) {
552 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
553 addedp)) != 0)
554 goto fail1;
555 return (0);
556 }
557
558 return (ENOTSUP);
559
560 fail1:
561 EFSYS_PROBE1(fail1, efx_rc_t, rc);
562 return (rc);
563 }
564
565 __checkReturn efx_rc_t
566 efx_tx_qdesc_post(
567 __in efx_txq_t *etp,
568 __in_ecount(ndescs) efx_desc_t *ed,
569 __in unsigned int ndescs,
570 __in unsigned int completed,
571 __inout unsigned int *addedp)
572 {
573 efx_nic_t *enp = etp->et_enp;
574 const efx_tx_ops_t *etxop = enp->en_etxop;
575 efx_rc_t rc;
576
577 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
578
579 if ((rc = etxop->etxo_qdesc_post(etp, ed,
580 ndescs, completed, addedp)) != 0)
581 goto fail1;
582
583 return (0);
584
585 fail1:
586 EFSYS_PROBE1(fail1, efx_rc_t, rc);
587 return (rc);
588 }
589
590 void
591 efx_tx_qdesc_dma_create(
592 __in efx_txq_t *etp,
593 __in efsys_dma_addr_t addr,
594 __in size_t size,
595 __in boolean_t eop,
596 __out efx_desc_t *edp)
597 {
598 efx_nic_t *enp = etp->et_enp;
599 const efx_tx_ops_t *etxop = enp->en_etxop;
600
601 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
602 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
603
604 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
605 }
606
607 void
608 efx_tx_qdesc_tso_create(
609 __in efx_txq_t *etp,
610 __in uint16_t ipv4_id,
611 __in uint32_t tcp_seq,
612 __in uint8_t tcp_flags,
613 __out efx_desc_t *edp)
614 {
615 efx_nic_t *enp = etp->et_enp;
616 const efx_tx_ops_t *etxop = enp->en_etxop;
617
618 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
619 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
620
621 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
622 }
623
624 void
625 efx_tx_qdesc_tso2_create(
626 __in efx_txq_t *etp,
627 __in uint16_t ipv4_id,
628 __in uint16_t outer_ipv4_id,
629 __in uint32_t tcp_seq,
630 __in uint16_t mss,
631 __out_ecount(count) efx_desc_t *edp,
632 __in int count)
633 {
634 efx_nic_t *enp = etp->et_enp;
635 const efx_tx_ops_t *etxop = enp->en_etxop;
636
637 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
638 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
639
640 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, outer_ipv4_id,
641 tcp_seq, mss, edp, count);
642 }
643
644 void
645 efx_tx_qdesc_vlantci_create(
646 __in efx_txq_t *etp,
647 __in uint16_t tci,
648 __out efx_desc_t *edp)
649 {
650 efx_nic_t *enp = etp->et_enp;
651 const efx_tx_ops_t *etxop = enp->en_etxop;
652
653 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
654 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
655
656 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
657 }
658
659 void
660 efx_tx_qdesc_checksum_create(
661 __in efx_txq_t *etp,
662 __in uint16_t flags,
663 __out efx_desc_t *edp)
664 {
665 efx_nic_t *enp = etp->et_enp;
666 const efx_tx_ops_t *etxop = enp->en_etxop;
667
668 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
669 EFSYS_ASSERT(etxop->etxo_qdesc_checksum_create != NULL);
670
671 etxop->etxo_qdesc_checksum_create(etp, flags, edp);
672 }
673
674
675 #if EFSYS_OPT_QSTATS
676 void
677 efx_tx_qstats_update(
678 __in efx_txq_t *etp,
679 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
680 {
681 efx_nic_t *enp = etp->et_enp;
682 const efx_tx_ops_t *etxop = enp->en_etxop;
683
684 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
685
686 etxop->etxo_qstats_update(etp, stat);
687 }
688 #endif
689
690
691 #if EFSYS_OPT_SIENA
692
693 static __checkReturn efx_rc_t
694 siena_tx_init(
695 __in efx_nic_t *enp)
696 {
697 efx_oword_t oword;
698
699 /*
700 * Disable the timer-based TX DMA backoff and allow TX DMA to be
701 * controlled by the RX FIFO fill level (although always allow a
702 * minimal trickle).
703 */
704 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
705 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
706 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
707 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
708 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
709 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
710 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
711 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
712
713 /*
714 * Filter all packets less than 14 bytes to avoid parsing
715 * errors.
716 */
717 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
718 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
719
720 /*
721 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
722 * descriptors (which is bad).
723 */
724 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
725 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
726 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
727
728 return (0);
729 }
730
731 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
732 do { \
733 unsigned int id; \
734 size_t offset; \
735 efx_qword_t qword; \
736 \
737 id = (_added)++ & (_etp)->et_mask; \
738 offset = id * sizeof (efx_qword_t); \
739 \
740 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
741 unsigned int, id, efsys_dma_addr_t, (_addr), \
742 size_t, (_size), boolean_t, (_eop)); \
743 \
744 EFX_POPULATE_QWORD_4(qword, \
745 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
746 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
747 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
748 (uint32_t)((_addr) & 0xffffffff), \
749 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
750 (uint32_t)((_addr) >> 32)); \
751 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
752 \
753 _NOTE(CONSTANTCONDITION) \
754 } while (B_FALSE)
755
756 static __checkReturn efx_rc_t
757 siena_tx_qpost(
758 __in efx_txq_t *etp,
759 __in_ecount(ndescs) efx_buffer_t *eb,
760 __in unsigned int ndescs,
761 __in unsigned int completed,
762 __inout unsigned int *addedp)
763 {
764 unsigned int added = *addedp;
765 unsigned int i;
766 int rc = ENOSPC;
767
768 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1))
769 goto fail1;
770
771 for (i = 0; i < ndescs; i++) {
772 efx_buffer_t *ebp = &eb[i];
773 efsys_dma_addr_t start = ebp->eb_addr;
774 size_t size = ebp->eb_size;
775 efsys_dma_addr_t end = start + size;
776
777 /*
778 * Fragments must not span 4k boundaries.
779 * Here it is a stricter requirement than the maximum length.
780 */
781 EFSYS_ASSERT(P2ROUNDUP(start + 1,
782 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end);
783
784 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
785 }
786
787 EFX_TX_QSTAT_INCR(etp, TX_POST);
788
789 *addedp = added;
790 return (0);
791
792 fail1:
793 EFSYS_PROBE1(fail1, efx_rc_t, rc);
794
795 return (rc);
796 }
797
798 static void
799 siena_tx_qpush(
800 __in efx_txq_t *etp,
801 __in unsigned int added,
802 __in unsigned int pushed)
803 {
804 efx_nic_t *enp = etp->et_enp;
805 uint32_t wptr;
806 efx_dword_t dword;
807 efx_oword_t oword;
808
809 /* Push the populated descriptors out */
810 wptr = added & etp->et_mask;
811
812 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
813
814 /* Only write the third DWORD */
815 EFX_POPULATE_DWORD_1(dword,
816 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
817
818 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
819 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
820 wptr, pushed & etp->et_mask);
821 EFSYS_PIO_WRITE_BARRIER();
822 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
823 etp->et_index, &dword, B_FALSE);
824 }
825
826 #define EFX_MAX_PACE_VALUE 20
827
828 static __checkReturn efx_rc_t
829 siena_tx_qpace(
830 __in efx_txq_t *etp,
831 __in unsigned int ns)
832 {
833 efx_nic_t *enp = etp->et_enp;
834 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
835 efx_oword_t oword;
836 unsigned int pace_val;
837 unsigned int timer_period;
838 efx_rc_t rc;
839
840 if (ns == 0) {
841 pace_val = 0;
842 } else {
843 /*
844 * The pace_val to write into the table is s.t
845 * ns <= timer_period * (2 ^ pace_val)
846 */
847 timer_period = 104 / encp->enc_clk_mult;
848 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
849 if ((timer_period << pace_val) >= ns)
850 break;
851 }
852 }
853 if (pace_val > EFX_MAX_PACE_VALUE) {
854 rc = EINVAL;
855 goto fail1;
856 }
857
858 /* Update the pacing table */
859 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
860 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
861 &oword, B_TRUE);
862
863 return (0);
864
865 fail1:
866 EFSYS_PROBE1(fail1, efx_rc_t, rc);
867
868 return (rc);
869 }
870
871 static __checkReturn efx_rc_t
872 siena_tx_qflush(
873 __in efx_txq_t *etp)
874 {
875 efx_nic_t *enp = etp->et_enp;
876 efx_oword_t oword;
877 uint32_t label;
878
879 efx_tx_qpace(etp, 0);
880
881 label = etp->et_index;
882
883 /* Flush the queue */
884 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
885 FRF_AZ_TX_FLUSH_DESCQ, label);
886 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
887
888 return (0);
889 }
890
891 static void
892 siena_tx_qenable(
893 __in efx_txq_t *etp)
894 {
895 efx_nic_t *enp = etp->et_enp;
896 efx_oword_t oword;
897
898 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
899 etp->et_index, &oword, B_TRUE);
900
901 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
902 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
903 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
904 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
905 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
906
907 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
908 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
909 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
910
911 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
912 etp->et_index, &oword, B_TRUE);
913 }
914
915 static __checkReturn efx_rc_t
916 siena_tx_qcreate(
917 __in efx_nic_t *enp,
918 __in unsigned int index,
919 __in unsigned int label,
920 __in efsys_mem_t *esmp,
921 __in size_t ndescs,
922 __in uint32_t id,
923 __in uint16_t flags,
924 __in efx_evq_t *eep,
925 __in efx_txq_t *etp,
926 __out unsigned int *addedp)
927 {
928 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
929 efx_oword_t oword;
930 uint32_t size;
931 uint16_t inner_csum;
932 efx_rc_t rc;
933
934 _NOTE(ARGUNUSED(esmp))
935
936 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
937 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
938 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
939
940 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs));
941 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));
942
943 if (!ISP2(ndescs) ||
944 (ndescs < EFX_TXQ_MINNDESCS) || (ndescs > EFX_EVQ_MAXNEVS)) {
945 rc = EINVAL;
946 goto fail1;
947 }
948 if (index >= encp->enc_txq_limit) {
949 rc = EINVAL;
950 goto fail2;
951 }
952 for (size = 0;
953 (1 << size) <= (int)(encp->enc_txq_max_ndescs / EFX_TXQ_MINNDESCS);
954 size++)
955 if ((1 << size) == (int)(ndescs / EFX_TXQ_MINNDESCS))
956 break;
957 if (id + (1 << size) >= encp->enc_buftbl_limit) {
958 rc = EINVAL;
959 goto fail3;
960 }
961
962 inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP;
963 if ((flags & inner_csum) != 0) {
964 rc = EINVAL;
965 goto fail4;
966 }
967
968 /* Set up the new descriptor queue */
969 *addedp = 0;
970
971 EFX_POPULATE_OWORD_6(oword,
972 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
973 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
974 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
975 FRF_AZ_TX_DESCQ_LABEL, label,
976 FRF_AZ_TX_DESCQ_SIZE, size,
977 FRF_AZ_TX_DESCQ_TYPE, 0);
978
979 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
980 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
981 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
982 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
983 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
984
985 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
986 etp->et_index, &oword, B_TRUE);
987
988 return (0);
989
990 fail4:
991 EFSYS_PROBE(fail4);
992 fail3:
993 EFSYS_PROBE(fail3);
994 fail2:
995 EFSYS_PROBE(fail2);
996 fail1:
997 EFSYS_PROBE1(fail1, efx_rc_t, rc);
998
999 return (rc);
1000 }
1001
1002 __checkReturn efx_rc_t
1003 siena_tx_qdesc_post(
1004 __in efx_txq_t *etp,
1005 __in_ecount(ndescs) efx_desc_t *ed,
1006 __in unsigned int ndescs,
1007 __in unsigned int completed,
1008 __inout unsigned int *addedp)
1009 {
1010 unsigned int added = *addedp;
1011 unsigned int i;
1012 efx_rc_t rc;
1013
1014 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
1015 rc = ENOSPC;
1016 goto fail1;
1017 }
1018
1019 for (i = 0; i < ndescs; i++) {
1020 efx_desc_t *edp = &ed[i];
1021 unsigned int id;
1022 size_t offset;
1023
1024 id = added++ & etp->et_mask;
1025 offset = id * sizeof (efx_desc_t);
1026
1027 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
1028 }
1029
1030 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
1031 unsigned int, added, unsigned int, ndescs);
1032
1033 EFX_TX_QSTAT_INCR(etp, TX_POST);
1034
1035 *addedp = added;
1036 return (0);
1037
1038 fail1:
1039 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1040 return (rc);
1041 }
1042
1043 void
1044 siena_tx_qdesc_dma_create(
1045 __in efx_txq_t *etp,
1046 __in efsys_dma_addr_t addr,
1047 __in size_t size,
1048 __in boolean_t eop,
1049 __out efx_desc_t *edp)
1050 {
1051 /*
1052 * Fragments must not span 4k boundaries.
1053 * Here it is a stricter requirement than the maximum length.
1054 */
1055 EFSYS_ASSERT(P2ROUNDUP(addr + 1,
1056 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size);
1057
1058 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1059 efsys_dma_addr_t, addr,
1060 size_t, size, boolean_t, eop);
1061
1062 EFX_POPULATE_QWORD_4(edp->ed_eq,
1063 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1064 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1065 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1066 (uint32_t)(addr & 0xffffffff),
1067 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1068 (uint32_t)(addr >> 32));
1069 }
1070
1071 #endif /* EFSYS_OPT_SIENA */
1072
1073 #if EFSYS_OPT_QSTATS
1074 #if EFSYS_OPT_NAMES
1075 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */
1076 static const char * const __efx_tx_qstat_name[] = {
1077 "post",
1078 "post_pio",
1079 };
1080 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1081
1082 const char *
1083 efx_tx_qstat_name(
1084 __in efx_nic_t *enp,
1085 __in unsigned int id)
1086 {
1087 _NOTE(ARGUNUSED(enp))
1088 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1089 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1090
1091 return (__efx_tx_qstat_name[id]);
1092 }
1093 #endif /* EFSYS_OPT_NAMES */
1094 #endif /* EFSYS_OPT_QSTATS */
1095
1096 #if EFSYS_OPT_SIENA
1097
1098 #if EFSYS_OPT_QSTATS
1099 static void
1100 siena_tx_qstats_update(
1101 __in efx_txq_t *etp,
1102 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1103 {
1104 unsigned int id;
1105
1106 for (id = 0; id < TX_NQSTATS; id++) {
1107 efsys_stat_t *essp = &stat[id];
1108
1109 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1110 etp->et_stat[id] = 0;
1111 }
1112 }
1113 #endif /* EFSYS_OPT_QSTATS */
1114
1115 static void
1116 siena_tx_qdestroy(
1117 __in efx_txq_t *etp)
1118 {
1119 efx_nic_t *enp = etp->et_enp;
1120 efx_oword_t oword;
1121
1122 /* Purge descriptor queue */
1123 EFX_ZERO_OWORD(oword);
1124
1125 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1126 etp->et_index, &oword, B_TRUE);
1127 }
1128
1129 static void
1130 siena_tx_fini(
1131 __in efx_nic_t *enp)
1132 {
1133 _NOTE(ARGUNUSED(enp))
1134 }
1135
1136 #endif /* EFSYS_OPT_SIENA */
Ceph