]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | /* SPDX-License-Identifier: BSD-3-Clause |
2 | * | |
f67539c2 TL |
3 | * Copyright(c) 2019-2020 Xilinx, Inc. |
4 | * Copyright(c) 2012-2019 Solarflare Communications Inc. | |
11fdf7f2 TL |
5 | */ |
6 | ||
7 | #include "efx.h" | |
8 | #include "efx_impl.h" | |
9 | ||
10 | ||
9f95a23c | 11 | #if EFX_OPTS_EF10() |
11fdf7f2 TL |
12 | |
13 | #if EFSYS_OPT_QSTATS | |
14 | #define EFX_TX_QSTAT_INCR(_etp, _stat) \ | |
15 | do { \ | |
16 | (_etp)->et_stat[_stat]++; \ | |
17 | _NOTE(CONSTANTCONDITION) \ | |
18 | } while (B_FALSE) | |
19 | #else | |
20 | #define EFX_TX_QSTAT_INCR(_etp, _stat) | |
21 | #endif | |
22 | ||
23 | static __checkReturn efx_rc_t | |
24 | efx_mcdi_init_txq( | |
25 | __in efx_nic_t *enp, | |
26 | __in uint32_t ndescs, | |
27 | __in uint32_t target_evq, | |
28 | __in uint32_t label, | |
29 | __in uint32_t instance, | |
30 | __in uint16_t flags, | |
31 | __in efsys_mem_t *esmp) | |
32 | { | |
33 | efx_mcdi_req_t req; | |
9f95a23c TL |
34 | EFX_MCDI_DECLARE_BUF(payload, |
35 | MC_CMD_INIT_TXQ_IN_LEN(EF10_TXQ_MAXNBUFS), | |
36 | MC_CMD_INIT_TXQ_OUT_LEN); | |
11fdf7f2 TL |
37 | efx_qword_t *dma_addr; |
38 | uint64_t addr; | |
39 | int npages; | |
40 | int i; | |
41 | efx_rc_t rc; | |
42 | ||
9f95a23c TL |
43 | EFSYS_ASSERT(EF10_TXQ_MAXNBUFS >= |
44 | efx_txq_nbufs(enp, enp->en_nic_cfg.enc_txq_max_ndescs)); | |
11fdf7f2 | 45 | |
9f95a23c TL |
46 | if ((esmp == NULL) || |
47 | (EFSYS_MEM_SIZE(esmp) < efx_txq_size(enp, ndescs))) { | |
11fdf7f2 TL |
48 | rc = EINVAL; |
49 | goto fail1; | |
50 | } | |
51 | ||
9f95a23c | 52 | npages = efx_txq_nbufs(enp, ndescs); |
11fdf7f2 TL |
53 | if (MC_CMD_INIT_TXQ_IN_LEN(npages) > sizeof (payload)) { |
54 | rc = EINVAL; | |
55 | goto fail2; | |
56 | } | |
57 | ||
11fdf7f2 TL |
58 | req.emr_cmd = MC_CMD_INIT_TXQ; |
59 | req.emr_in_buf = payload; | |
60 | req.emr_in_length = MC_CMD_INIT_TXQ_IN_LEN(npages); | |
61 | req.emr_out_buf = payload; | |
62 | req.emr_out_length = MC_CMD_INIT_TXQ_OUT_LEN; | |
63 | ||
64 | MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_SIZE, ndescs); | |
65 | MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_TARGET_EVQ, target_evq); | |
66 | MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_LABEL, label); | |
67 | MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_INSTANCE, instance); | |
68 | ||
69 | MCDI_IN_POPULATE_DWORD_9(req, INIT_TXQ_IN_FLAGS, | |
70 | INIT_TXQ_IN_FLAG_BUFF_MODE, 0, | |
71 | INIT_TXQ_IN_FLAG_IP_CSUM_DIS, | |
72 | (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1, | |
73 | INIT_TXQ_IN_FLAG_TCP_CSUM_DIS, | |
74 | (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1, | |
75 | INIT_TXQ_EXT_IN_FLAG_INNER_IP_CSUM_EN, | |
76 | (flags & EFX_TXQ_CKSUM_INNER_IPV4) ? 1 : 0, | |
77 | INIT_TXQ_EXT_IN_FLAG_INNER_TCP_CSUM_EN, | |
78 | (flags & EFX_TXQ_CKSUM_INNER_TCPUDP) ? 1 : 0, | |
79 | INIT_TXQ_EXT_IN_FLAG_TSOV2_EN, (flags & EFX_TXQ_FATSOV2) ? 1 : 0, | |
80 | INIT_TXQ_IN_FLAG_TCP_UDP_ONLY, 0, | |
81 | INIT_TXQ_IN_CRC_MODE, 0, | |
82 | INIT_TXQ_IN_FLAG_TIMESTAMP, 0); | |
83 | ||
84 | MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_OWNER_ID, 0); | |
f67539c2 | 85 | MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_PORT_ID, enp->en_vport_id); |
11fdf7f2 TL |
86 | |
87 | dma_addr = MCDI_IN2(req, efx_qword_t, INIT_TXQ_IN_DMA_ADDR); | |
88 | addr = EFSYS_MEM_ADDR(esmp); | |
89 | ||
90 | for (i = 0; i < npages; i++) { | |
91 | EFX_POPULATE_QWORD_2(*dma_addr, | |
92 | EFX_DWORD_1, (uint32_t)(addr >> 32), | |
93 | EFX_DWORD_0, (uint32_t)(addr & 0xffffffff)); | |
94 | ||
95 | dma_addr++; | |
96 | addr += EFX_BUF_SIZE; | |
97 | } | |
98 | ||
99 | efx_mcdi_execute(enp, &req); | |
100 | ||
101 | if (req.emr_rc != 0) { | |
102 | rc = req.emr_rc; | |
103 | goto fail3; | |
104 | } | |
105 | ||
106 | return (0); | |
107 | ||
108 | fail3: | |
109 | EFSYS_PROBE(fail3); | |
110 | fail2: | |
111 | EFSYS_PROBE(fail2); | |
112 | fail1: | |
113 | EFSYS_PROBE1(fail1, efx_rc_t, rc); | |
114 | ||
115 | return (rc); | |
116 | } | |
117 | ||
118 | static __checkReturn efx_rc_t | |
119 | efx_mcdi_fini_txq( | |
120 | __in efx_nic_t *enp, | |
121 | __in uint32_t instance) | |
122 | { | |
123 | efx_mcdi_req_t req; | |
9f95a23c TL |
124 | EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FINI_TXQ_IN_LEN, |
125 | MC_CMD_FINI_TXQ_OUT_LEN); | |
11fdf7f2 TL |
126 | efx_rc_t rc; |
127 | ||
11fdf7f2 TL |
128 | req.emr_cmd = MC_CMD_FINI_TXQ; |
129 | req.emr_in_buf = payload; | |
130 | req.emr_in_length = MC_CMD_FINI_TXQ_IN_LEN; | |
131 | req.emr_out_buf = payload; | |
132 | req.emr_out_length = MC_CMD_FINI_TXQ_OUT_LEN; | |
133 | ||
134 | MCDI_IN_SET_DWORD(req, FINI_TXQ_IN_INSTANCE, instance); | |
135 | ||
136 | efx_mcdi_execute_quiet(enp, &req); | |
137 | ||
138 | if (req.emr_rc != 0) { | |
139 | rc = req.emr_rc; | |
140 | goto fail1; | |
141 | } | |
142 | ||
143 | return (0); | |
144 | ||
145 | fail1: | |
146 | /* | |
147 | * EALREADY is not an error, but indicates that the MC has rebooted and | |
148 | * that the TXQ has already been destroyed. | |
149 | */ | |
150 | if (rc != EALREADY) | |
151 | EFSYS_PROBE1(fail1, efx_rc_t, rc); | |
152 | ||
153 | return (rc); | |
154 | } | |
155 | ||
156 | __checkReturn efx_rc_t | |
157 | ef10_tx_init( | |
158 | __in efx_nic_t *enp) | |
159 | { | |
160 | _NOTE(ARGUNUSED(enp)) | |
161 | return (0); | |
162 | } | |
163 | ||
164 | void | |
165 | ef10_tx_fini( | |
166 | __in efx_nic_t *enp) | |
167 | { | |
168 | _NOTE(ARGUNUSED(enp)) | |
169 | } | |
170 | ||
171 | __checkReturn efx_rc_t | |
172 | ef10_tx_qcreate( | |
173 | __in efx_nic_t *enp, | |
174 | __in unsigned int index, | |
175 | __in unsigned int label, | |
176 | __in efsys_mem_t *esmp, | |
177 | __in size_t ndescs, | |
178 | __in uint32_t id, | |
179 | __in uint16_t flags, | |
180 | __in efx_evq_t *eep, | |
181 | __in efx_txq_t *etp, | |
182 | __out unsigned int *addedp) | |
183 | { | |
184 | efx_nic_cfg_t *encp = &enp->en_nic_cfg; | |
185 | uint16_t inner_csum; | |
186 | efx_desc_t desc; | |
187 | efx_rc_t rc; | |
188 | ||
189 | _NOTE(ARGUNUSED(id)) | |
190 | ||
191 | inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP; | |
192 | if (((flags & inner_csum) != 0) && | |
193 | (encp->enc_tunnel_encapsulations_supported == 0)) { | |
194 | rc = EINVAL; | |
195 | goto fail1; | |
196 | } | |
197 | ||
198 | if ((rc = efx_mcdi_init_txq(enp, ndescs, eep->ee_index, label, index, | |
199 | flags, esmp)) != 0) | |
200 | goto fail2; | |
201 | ||
202 | /* | |
203 | * A previous user of this TX queue may have written a descriptor to the | |
204 | * TX push collector, but not pushed the doorbell (e.g. after a crash). | |
205 | * The next doorbell write would then push the stale descriptor. | |
206 | * | |
207 | * Ensure the (per network port) TX push collector is cleared by writing | |
208 | * a no-op TX option descriptor. See bug29981 for details. | |
209 | */ | |
210 | *addedp = 1; | |
211 | ef10_tx_qdesc_checksum_create(etp, flags, &desc); | |
212 | ||
213 | EFSYS_MEM_WRITEQ(etp->et_esmp, 0, &desc.ed_eq); | |
214 | ef10_tx_qpush(etp, *addedp, 0); | |
215 | ||
216 | return (0); | |
217 | ||
218 | fail2: | |
219 | EFSYS_PROBE(fail2); | |
220 | fail1: | |
221 | EFSYS_PROBE1(fail1, efx_rc_t, rc); | |
222 | ||
223 | return (rc); | |
224 | } | |
225 | ||
226 | void | |
227 | ef10_tx_qdestroy( | |
228 | __in efx_txq_t *etp) | |
229 | { | |
230 | /* FIXME */ | |
231 | _NOTE(ARGUNUSED(etp)) | |
232 | /* FIXME */ | |
233 | } | |
234 | ||
235 | __checkReturn efx_rc_t | |
236 | ef10_tx_qpio_enable( | |
237 | __in efx_txq_t *etp) | |
238 | { | |
239 | efx_nic_t *enp = etp->et_enp; | |
240 | efx_piobuf_handle_t handle; | |
241 | efx_rc_t rc; | |
242 | ||
243 | if (etp->et_pio_size != 0) { | |
244 | rc = EALREADY; | |
245 | goto fail1; | |
246 | } | |
247 | ||
248 | /* Sub-allocate a PIO block from a piobuf */ | |
249 | if ((rc = ef10_nic_pio_alloc(enp, | |
250 | &etp->et_pio_bufnum, | |
251 | &handle, | |
252 | &etp->et_pio_blknum, | |
253 | &etp->et_pio_offset, | |
254 | &etp->et_pio_size)) != 0) { | |
255 | goto fail2; | |
256 | } | |
257 | EFSYS_ASSERT3U(etp->et_pio_size, !=, 0); | |
258 | ||
259 | /* Link the piobuf to this TXQ */ | |
260 | if ((rc = ef10_nic_pio_link(enp, etp->et_index, handle)) != 0) { | |
261 | goto fail3; | |
262 | } | |
263 | ||
264 | /* | |
265 | * et_pio_offset is the offset of the sub-allocated block within the | |
266 | * hardware PIO buffer. It is used as the buffer address in the PIO | |
267 | * option descriptor. | |
268 | * | |
269 | * et_pio_write_offset is the offset of the sub-allocated block from the | |
270 | * start of the write-combined memory mapping, and is used for writing | |
271 | * data into the PIO buffer. | |
272 | */ | |
273 | etp->et_pio_write_offset = | |
274 | (etp->et_pio_bufnum * ER_DZ_TX_PIOBUF_STEP) + | |
275 | ER_DZ_TX_PIOBUF_OFST + etp->et_pio_offset; | |
276 | ||
277 | return (0); | |
278 | ||
279 | fail3: | |
280 | EFSYS_PROBE(fail3); | |
9f95a23c | 281 | (void) ef10_nic_pio_free(enp, etp->et_pio_bufnum, etp->et_pio_blknum); |
11fdf7f2 TL |
282 | fail2: |
283 | EFSYS_PROBE(fail2); | |
284 | etp->et_pio_size = 0; | |
285 | fail1: | |
286 | EFSYS_PROBE1(fail1, efx_rc_t, rc); | |
287 | ||
288 | return (rc); | |
289 | } | |
290 | ||
291 | void | |
292 | ef10_tx_qpio_disable( | |
293 | __in efx_txq_t *etp) | |
294 | { | |
295 | efx_nic_t *enp = etp->et_enp; | |
296 | ||
297 | if (etp->et_pio_size != 0) { | |
298 | /* Unlink the piobuf from this TXQ */ | |
9f95a23c TL |
299 | if (ef10_nic_pio_unlink(enp, etp->et_index) != 0) |
300 | return; | |
11fdf7f2 TL |
301 | |
302 | /* Free the sub-allocated PIO block */ | |
9f95a23c TL |
303 | (void) ef10_nic_pio_free(enp, etp->et_pio_bufnum, |
304 | etp->et_pio_blknum); | |
11fdf7f2 TL |
305 | etp->et_pio_size = 0; |
306 | etp->et_pio_write_offset = 0; | |
307 | } | |
308 | } | |
309 | ||
310 | __checkReturn efx_rc_t | |
311 | ef10_tx_qpio_write( | |
312 | __in efx_txq_t *etp, | |
313 | __in_ecount(length) uint8_t *buffer, | |
314 | __in size_t length, | |
315 | __in size_t offset) | |
316 | { | |
317 | efx_nic_t *enp = etp->et_enp; | |
318 | efsys_bar_t *esbp = enp->en_esbp; | |
319 | uint32_t write_offset; | |
320 | uint32_t write_offset_limit; | |
321 | efx_qword_t *eqp; | |
322 | efx_rc_t rc; | |
323 | ||
324 | EFSYS_ASSERT(length % sizeof (efx_qword_t) == 0); | |
325 | ||
326 | if (etp->et_pio_size == 0) { | |
327 | rc = ENOENT; | |
328 | goto fail1; | |
329 | } | |
330 | if (offset + length > etp->et_pio_size) { | |
331 | rc = ENOSPC; | |
332 | goto fail2; | |
333 | } | |
334 | ||
335 | /* | |
336 | * Writes to PIO buffers must be 64 bit aligned, and multiples of | |
337 | * 64 bits. | |
338 | */ | |
339 | write_offset = etp->et_pio_write_offset + offset; | |
340 | write_offset_limit = write_offset + length; | |
341 | eqp = (efx_qword_t *)buffer; | |
342 | while (write_offset < write_offset_limit) { | |
343 | EFSYS_BAR_WC_WRITEQ(esbp, write_offset, eqp); | |
344 | eqp++; | |
345 | write_offset += sizeof (efx_qword_t); | |
346 | } | |
347 | ||
348 | return (0); | |
349 | ||
350 | fail2: | |
351 | EFSYS_PROBE(fail2); | |
352 | fail1: | |
353 | EFSYS_PROBE1(fail1, efx_rc_t, rc); | |
354 | ||
355 | return (rc); | |
356 | } | |
357 | ||
358 | __checkReturn efx_rc_t | |
359 | ef10_tx_qpio_post( | |
360 | __in efx_txq_t *etp, | |
361 | __in size_t pkt_length, | |
362 | __in unsigned int completed, | |
363 | __inout unsigned int *addedp) | |
364 | { | |
365 | efx_qword_t pio_desc; | |
366 | unsigned int id; | |
367 | size_t offset; | |
368 | unsigned int added = *addedp; | |
369 | efx_rc_t rc; | |
370 | ||
371 | ||
372 | if (added - completed + 1 > EFX_TXQ_LIMIT(etp->et_mask + 1)) { | |
373 | rc = ENOSPC; | |
374 | goto fail1; | |
375 | } | |
376 | ||
377 | if (etp->et_pio_size == 0) { | |
378 | rc = ENOENT; | |
379 | goto fail2; | |
380 | } | |
381 | ||
382 | id = added++ & etp->et_mask; | |
383 | offset = id * sizeof (efx_qword_t); | |
384 | ||
385 | EFSYS_PROBE4(tx_pio_post, unsigned int, etp->et_index, | |
386 | unsigned int, id, uint32_t, etp->et_pio_offset, | |
387 | size_t, pkt_length); | |
388 | ||
389 | EFX_POPULATE_QWORD_5(pio_desc, | |
390 | ESF_DZ_TX_DESC_IS_OPT, 1, | |
391 | ESF_DZ_TX_OPTION_TYPE, 1, | |
392 | ESF_DZ_TX_PIO_CONT, 0, | |
393 | ESF_DZ_TX_PIO_BYTE_CNT, pkt_length, | |
394 | ESF_DZ_TX_PIO_BUF_ADDR, etp->et_pio_offset); | |
395 | ||
396 | EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &pio_desc); | |
397 | ||
398 | EFX_TX_QSTAT_INCR(etp, TX_POST_PIO); | |
399 | ||
400 | *addedp = added; | |
401 | return (0); | |
402 | ||
403 | fail2: | |
404 | EFSYS_PROBE(fail2); | |
405 | fail1: | |
406 | EFSYS_PROBE1(fail1, efx_rc_t, rc); | |
407 | ||
408 | return (rc); | |
409 | } | |
410 | ||
411 | __checkReturn efx_rc_t | |
412 | ef10_tx_qpost( | |
413 | __in efx_txq_t *etp, | |
414 | __in_ecount(ndescs) efx_buffer_t *eb, | |
415 | __in unsigned int ndescs, | |
416 | __in unsigned int completed, | |
417 | __inout unsigned int *addedp) | |
418 | { | |
419 | unsigned int added = *addedp; | |
420 | unsigned int i; | |
421 | efx_rc_t rc; | |
422 | ||
423 | if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) { | |
424 | rc = ENOSPC; | |
425 | goto fail1; | |
426 | } | |
427 | ||
428 | for (i = 0; i < ndescs; i++) { | |
429 | efx_buffer_t *ebp = &eb[i]; | |
430 | efsys_dma_addr_t addr = ebp->eb_addr; | |
431 | size_t size = ebp->eb_size; | |
432 | boolean_t eop = ebp->eb_eop; | |
433 | unsigned int id; | |
434 | size_t offset; | |
435 | efx_qword_t qword; | |
436 | ||
437 | /* No limitations on boundary crossing */ | |
438 | EFSYS_ASSERT(size <= | |
439 | etp->et_enp->en_nic_cfg.enc_tx_dma_desc_size_max); | |
440 | ||
441 | id = added++ & etp->et_mask; | |
442 | offset = id * sizeof (efx_qword_t); | |
443 | ||
444 | EFSYS_PROBE5(tx_post, unsigned int, etp->et_index, | |
445 | unsigned int, id, efsys_dma_addr_t, addr, | |
446 | size_t, size, boolean_t, eop); | |
447 | ||
448 | EFX_POPULATE_QWORD_5(qword, | |
449 | ESF_DZ_TX_KER_TYPE, 0, | |
450 | ESF_DZ_TX_KER_CONT, (eop) ? 0 : 1, | |
451 | ESF_DZ_TX_KER_BYTE_CNT, (uint32_t)(size), | |
452 | ESF_DZ_TX_KER_BUF_ADDR_DW0, (uint32_t)(addr & 0xffffffff), | |
453 | ESF_DZ_TX_KER_BUF_ADDR_DW1, (uint32_t)(addr >> 32)); | |
454 | ||
455 | EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &qword); | |
456 | } | |
457 | ||
458 | EFX_TX_QSTAT_INCR(etp, TX_POST); | |
459 | ||
460 | *addedp = added; | |
461 | return (0); | |
462 | ||
463 | fail1: | |
464 | EFSYS_PROBE1(fail1, efx_rc_t, rc); | |
465 | ||
466 | return (rc); | |
467 | } | |
468 | ||
469 | /* | |
470 | * This improves performance by, when possible, pushing a TX descriptor at the | |
471 | * same time as the doorbell. The descriptor must be added to the TXQ, so that | |
472 | * can be used if the hardware decides not to use the pushed descriptor. | |
473 | */ | |
474 | void | |
475 | ef10_tx_qpush( | |
476 | __in efx_txq_t *etp, | |
477 | __in unsigned int added, | |
478 | __in unsigned int pushed) | |
479 | { | |
480 | efx_nic_t *enp = etp->et_enp; | |
481 | unsigned int wptr; | |
482 | unsigned int id; | |
483 | size_t offset; | |
484 | efx_qword_t desc; | |
485 | efx_oword_t oword; | |
486 | ||
487 | wptr = added & etp->et_mask; | |
488 | id = pushed & etp->et_mask; | |
489 | offset = id * sizeof (efx_qword_t); | |
490 | ||
491 | EFSYS_MEM_READQ(etp->et_esmp, offset, &desc); | |
492 | ||
493 | /* | |
494 | * Bug 65776: TSO option descriptors cannot be pushed if pacer bypass is | |
495 | * enabled on the event queue this transmit queue is attached to. | |
496 | * | |
497 | * To ensure the code is safe, it is easiest to simply test the type of | |
498 | * the descriptor to push, and only push it is if it not a TSO option | |
499 | * descriptor. | |
500 | */ | |
501 | if ((EFX_QWORD_FIELD(desc, ESF_DZ_TX_DESC_IS_OPT) != 1) || | |
502 | (EFX_QWORD_FIELD(desc, ESF_DZ_TX_OPTION_TYPE) != | |
503 | ESE_DZ_TX_OPTION_DESC_TSO)) { | |
504 | /* Push the descriptor and update the wptr. */ | |
505 | EFX_POPULATE_OWORD_3(oword, ERF_DZ_TX_DESC_WPTR, wptr, | |
506 | ERF_DZ_TX_DESC_HWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_1), | |
507 | ERF_DZ_TX_DESC_LWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_0)); | |
508 | ||
509 | /* Ensure ordering of memory (descriptors) and PIO (doorbell) */ | |
510 | EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, | |
511 | wptr, id); | |
512 | EFSYS_PIO_WRITE_BARRIER(); | |
513 | EFX_BAR_VI_DOORBELL_WRITEO(enp, ER_DZ_TX_DESC_UPD_REG, | |
514 | etp->et_index, &oword); | |
515 | } else { | |
516 | efx_dword_t dword; | |
517 | ||
518 | /* | |
519 | * Only update the wptr. This is signalled to the hardware by | |
520 | * only writing one DWORD of the doorbell register. | |
521 | */ | |
522 | EFX_POPULATE_OWORD_1(oword, ERF_DZ_TX_DESC_WPTR, wptr); | |
523 | dword = oword.eo_dword[2]; | |
524 | ||
525 | /* Ensure ordering of memory (descriptors) and PIO (doorbell) */ | |
526 | EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, | |
527 | wptr, id); | |
528 | EFSYS_PIO_WRITE_BARRIER(); | |
529 | EFX_BAR_VI_WRITED2(enp, ER_DZ_TX_DESC_UPD_REG, | |
530 | etp->et_index, &dword, B_FALSE); | |
531 | } | |
532 | } | |
533 | ||
534 | __checkReturn efx_rc_t | |
535 | ef10_tx_qdesc_post( | |
536 | __in efx_txq_t *etp, | |
537 | __in_ecount(ndescs) efx_desc_t *ed, | |
538 | __in unsigned int ndescs, | |
539 | __in unsigned int completed, | |
540 | __inout unsigned int *addedp) | |
541 | { | |
542 | unsigned int added = *addedp; | |
543 | unsigned int i; | |
11fdf7f2 | 544 | |
9f95a23c TL |
545 | if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) |
546 | return (ENOSPC); | |
11fdf7f2 TL |
547 | |
548 | for (i = 0; i < ndescs; i++) { | |
549 | efx_desc_t *edp = &ed[i]; | |
550 | unsigned int id; | |
551 | size_t offset; | |
552 | ||
553 | id = added++ & etp->et_mask; | |
554 | offset = id * sizeof (efx_desc_t); | |
555 | ||
556 | EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq); | |
557 | } | |
558 | ||
559 | EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index, | |
560 | unsigned int, added, unsigned int, ndescs); | |
561 | ||
562 | EFX_TX_QSTAT_INCR(etp, TX_POST); | |
563 | ||
564 | *addedp = added; | |
565 | return (0); | |
11fdf7f2 TL |
566 | } |
567 | ||
568 | void | |
569 | ef10_tx_qdesc_dma_create( | |
570 | __in efx_txq_t *etp, | |
571 | __in efsys_dma_addr_t addr, | |
572 | __in size_t size, | |
573 | __in boolean_t eop, | |
574 | __out efx_desc_t *edp) | |
575 | { | |
576 | _NOTE(ARGUNUSED(etp)) | |
577 | ||
578 | /* No limitations on boundary crossing */ | |
579 | EFSYS_ASSERT(size <= etp->et_enp->en_nic_cfg.enc_tx_dma_desc_size_max); | |
580 | ||
581 | EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index, | |
582 | efsys_dma_addr_t, addr, | |
583 | size_t, size, boolean_t, eop); | |
584 | ||
585 | EFX_POPULATE_QWORD_5(edp->ed_eq, | |
586 | ESF_DZ_TX_KER_TYPE, 0, | |
587 | ESF_DZ_TX_KER_CONT, (eop) ? 0 : 1, | |
588 | ESF_DZ_TX_KER_BYTE_CNT, (uint32_t)(size), | |
589 | ESF_DZ_TX_KER_BUF_ADDR_DW0, (uint32_t)(addr & 0xffffffff), | |
590 | ESF_DZ_TX_KER_BUF_ADDR_DW1, (uint32_t)(addr >> 32)); | |
591 | } | |
592 | ||
593 | void | |
594 | ef10_tx_qdesc_tso_create( | |
595 | __in efx_txq_t *etp, | |
596 | __in uint16_t ipv4_id, | |
597 | __in uint32_t tcp_seq, | |
598 | __in uint8_t tcp_flags, | |
599 | __out efx_desc_t *edp) | |
600 | { | |
601 | _NOTE(ARGUNUSED(etp)) | |
602 | ||
603 | EFSYS_PROBE4(tx_desc_tso_create, unsigned int, etp->et_index, | |
604 | uint16_t, ipv4_id, uint32_t, tcp_seq, | |
605 | uint8_t, tcp_flags); | |
606 | ||
607 | EFX_POPULATE_QWORD_5(edp->ed_eq, | |
608 | ESF_DZ_TX_DESC_IS_OPT, 1, | |
609 | ESF_DZ_TX_OPTION_TYPE, | |
610 | ESE_DZ_TX_OPTION_DESC_TSO, | |
611 | ESF_DZ_TX_TSO_TCP_FLAGS, tcp_flags, | |
612 | ESF_DZ_TX_TSO_IP_ID, ipv4_id, | |
613 | ESF_DZ_TX_TSO_TCP_SEQNO, tcp_seq); | |
614 | } | |
615 | ||
616 | void | |
617 | ef10_tx_qdesc_tso2_create( | |
618 | __in efx_txq_t *etp, | |
619 | __in uint16_t ipv4_id, | |
620 | __in uint16_t outer_ipv4_id, | |
621 | __in uint32_t tcp_seq, | |
622 | __in uint16_t tcp_mss, | |
623 | __out_ecount(count) efx_desc_t *edp, | |
624 | __in int count) | |
625 | { | |
626 | _NOTE(ARGUNUSED(etp, count)) | |
627 | ||
628 | EFSYS_PROBE4(tx_desc_tso2_create, unsigned int, etp->et_index, | |
629 | uint16_t, ipv4_id, uint32_t, tcp_seq, | |
630 | uint16_t, tcp_mss); | |
631 | ||
632 | EFSYS_ASSERT(count >= EFX_TX_FATSOV2_OPT_NDESCS); | |
633 | ||
9f95a23c | 634 | EFX_POPULATE_QWORD_5(edp[0].ed_eq, |
11fdf7f2 TL |
635 | ESF_DZ_TX_DESC_IS_OPT, 1, |
636 | ESF_DZ_TX_OPTION_TYPE, | |
637 | ESE_DZ_TX_OPTION_DESC_TSO, | |
638 | ESF_DZ_TX_TSO_OPTION_TYPE, | |
639 | ESE_DZ_TX_TSO_OPTION_DESC_FATSO2A, | |
640 | ESF_DZ_TX_TSO_IP_ID, ipv4_id, | |
11fdf7f2 | 641 | ESF_DZ_TX_TSO_TCP_SEQNO, tcp_seq); |
9f95a23c | 642 | EFX_POPULATE_QWORD_5(edp[1].ed_eq, |
11fdf7f2 TL |
643 | ESF_DZ_TX_DESC_IS_OPT, 1, |
644 | ESF_DZ_TX_OPTION_TYPE, | |
645 | ESE_DZ_TX_OPTION_DESC_TSO, | |
646 | ESF_DZ_TX_TSO_OPTION_TYPE, | |
647 | ESE_DZ_TX_TSO_OPTION_DESC_FATSO2B, | |
9f95a23c TL |
648 | ESF_DZ_TX_TSO_TCP_MSS, tcp_mss, |
649 | ESF_DZ_TX_TSO_OUTER_IPID, outer_ipv4_id); | |
11fdf7f2 TL |
650 | } |
651 | ||
652 | void | |
653 | ef10_tx_qdesc_vlantci_create( | |
654 | __in efx_txq_t *etp, | |
655 | __in uint16_t tci, | |
656 | __out efx_desc_t *edp) | |
657 | { | |
658 | _NOTE(ARGUNUSED(etp)) | |
659 | ||
660 | EFSYS_PROBE2(tx_desc_vlantci_create, unsigned int, etp->et_index, | |
661 | uint16_t, tci); | |
662 | ||
663 | EFX_POPULATE_QWORD_4(edp->ed_eq, | |
664 | ESF_DZ_TX_DESC_IS_OPT, 1, | |
665 | ESF_DZ_TX_OPTION_TYPE, | |
666 | ESE_DZ_TX_OPTION_DESC_VLAN, | |
667 | ESF_DZ_TX_VLAN_OP, tci ? 1 : 0, | |
668 | ESF_DZ_TX_VLAN_TAG1, tci); | |
669 | } | |
670 | ||
671 | void | |
672 | ef10_tx_qdesc_checksum_create( | |
673 | __in efx_txq_t *etp, | |
674 | __in uint16_t flags, | |
675 | __out efx_desc_t *edp) | |
676 | { | |
677 | _NOTE(ARGUNUSED(etp)); | |
678 | ||
679 | EFSYS_PROBE2(tx_desc_checksum_create, unsigned int, etp->et_index, | |
680 | uint32_t, flags); | |
681 | ||
682 | EFX_POPULATE_QWORD_6(edp->ed_eq, | |
683 | ESF_DZ_TX_DESC_IS_OPT, 1, | |
684 | ESF_DZ_TX_OPTION_TYPE, ESE_DZ_TX_OPTION_DESC_CRC_CSUM, | |
685 | ESF_DZ_TX_OPTION_UDP_TCP_CSUM, | |
686 | (flags & EFX_TXQ_CKSUM_TCPUDP) ? 1 : 0, | |
687 | ESF_DZ_TX_OPTION_IP_CSUM, | |
688 | (flags & EFX_TXQ_CKSUM_IPV4) ? 1 : 0, | |
689 | ESF_DZ_TX_OPTION_INNER_UDP_TCP_CSUM, | |
690 | (flags & EFX_TXQ_CKSUM_INNER_TCPUDP) ? 1 : 0, | |
691 | ESF_DZ_TX_OPTION_INNER_IP_CSUM, | |
692 | (flags & EFX_TXQ_CKSUM_INNER_IPV4) ? 1 : 0); | |
693 | } | |
694 | ||
695 | ||
696 | __checkReturn efx_rc_t | |
697 | ef10_tx_qpace( | |
698 | __in efx_txq_t *etp, | |
699 | __in unsigned int ns) | |
700 | { | |
701 | efx_rc_t rc; | |
702 | ||
703 | /* FIXME */ | |
704 | _NOTE(ARGUNUSED(etp, ns)) | |
705 | _NOTE(CONSTANTCONDITION) | |
706 | if (B_FALSE) { | |
707 | rc = ENOTSUP; | |
708 | goto fail1; | |
709 | } | |
710 | /* FIXME */ | |
711 | ||
712 | return (0); | |
713 | ||
714 | fail1: | |
715 | EFSYS_PROBE1(fail1, efx_rc_t, rc); | |
716 | ||
717 | return (rc); | |
718 | } | |
719 | ||
720 | __checkReturn efx_rc_t | |
721 | ef10_tx_qflush( | |
722 | __in efx_txq_t *etp) | |
723 | { | |
724 | efx_nic_t *enp = etp->et_enp; | |
725 | efx_rc_t rc; | |
726 | ||
727 | if ((rc = efx_mcdi_fini_txq(enp, etp->et_index)) != 0) | |
728 | goto fail1; | |
729 | ||
730 | return (0); | |
731 | ||
732 | fail1: | |
733 | /* | |
734 | * EALREADY is not an error, but indicates that the MC has rebooted and | |
735 | * that the TXQ has already been destroyed. Callers need to know that | |
736 | * the TXQ flush has completed to avoid waiting until timeout for a | |
737 | * flush done event that will not be delivered. | |
738 | */ | |
739 | if (rc != EALREADY) | |
740 | EFSYS_PROBE1(fail1, efx_rc_t, rc); | |
741 | ||
742 | return (rc); | |
743 | } | |
744 | ||
745 | void | |
746 | ef10_tx_qenable( | |
747 | __in efx_txq_t *etp) | |
748 | { | |
749 | /* FIXME */ | |
750 | _NOTE(ARGUNUSED(etp)) | |
751 | /* FIXME */ | |
752 | } | |
753 | ||
754 | #if EFSYS_OPT_QSTATS | |
755 | void | |
756 | ef10_tx_qstats_update( | |
757 | __in efx_txq_t *etp, | |
758 | __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) | |
759 | { | |
760 | unsigned int id; | |
761 | ||
762 | for (id = 0; id < TX_NQSTATS; id++) { | |
763 | efsys_stat_t *essp = &stat[id]; | |
764 | ||
765 | EFSYS_STAT_INCR(essp, etp->et_stat[id]); | |
766 | etp->et_stat[id] = 0; | |
767 | } | |
768 | } | |
769 | ||
770 | #endif /* EFSYS_OPT_QSTATS */ | |
771 | ||
9f95a23c | 772 | #endif /* EFX_OPTS_EF10() */ |