2 * Copyright (c) 2012-2019 The Linux Foundation. All rights reserved.
4 * Permission to use, copy, modify, and/or distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 #include <linux/etherdevice.h>
18 #include <linux/moduleparam.h>
19 #include <linux/prefetch.h>
20 #include <linux/types.h>
21 #include <linux/list.h>
23 #include <linux/ipv6.h>
25 #include "txrx_edma.h"
29 /* Max number of entries (packets to complete) to update the hwtail of tx
30 * status ring. Should be power of 2
32 #define WIL_EDMA_TX_SRING_UPDATE_HW_TAIL 128
33 #define WIL_EDMA_MAX_DATA_OFFSET (2)
34 /* RX buffer size must be aligned to 4 bytes */
35 #define WIL_EDMA_RX_BUF_LEN_DEFAULT (2048)
36 #define MAX_INVALID_BUFF_ID_RETRY (3)
38 static void wil_tx_desc_unmap_edma(struct device
*dev
,
39 union wil_tx_desc
*desc
,
42 struct wil_tx_enhanced_desc
*d
= (struct wil_tx_enhanced_desc
*)desc
;
43 dma_addr_t pa
= wil_tx_desc_get_addr_edma(&d
->dma
);
44 u16 dmalen
= le16_to_cpu(d
->dma
.length
);
46 switch (ctx
->mapped_as
) {
47 case wil_mapped_as_single
:
48 dma_unmap_single(dev
, pa
, dmalen
, DMA_TO_DEVICE
);
50 case wil_mapped_as_page
:
51 dma_unmap_page(dev
, pa
, dmalen
, DMA_TO_DEVICE
);
58 static int wil_find_free_sring(struct wil6210_priv
*wil
)
62 for (i
= 0; i
< WIL6210_MAX_STATUS_RINGS
; i
++) {
63 if (!wil
->srings
[i
].va
)
70 static void wil_sring_free(struct wil6210_priv
*wil
,
71 struct wil_status_ring
*sring
)
73 struct device
*dev
= wil_to_dev(wil
);
76 if (!sring
|| !sring
->va
)
79 sz
= sring
->elem_size
* sring
->size
;
81 wil_dbg_misc(wil
, "status_ring_free, size(bytes)=%zu, 0x%p:%pad\n",
82 sz
, sring
->va
, &sring
->pa
);
84 dma_free_coherent(dev
, sz
, (void *)sring
->va
, sring
->pa
);
89 static int wil_sring_alloc(struct wil6210_priv
*wil
,
90 struct wil_status_ring
*sring
)
92 struct device
*dev
= wil_to_dev(wil
);
93 size_t sz
= sring
->elem_size
* sring
->size
;
95 wil_dbg_misc(wil
, "status_ring_alloc: size=%zu\n", sz
);
98 wil_err(wil
, "Cannot allocate a zero size status ring\n");
104 /* Status messages are allocated and initialized to 0. This is necessary
105 * since DR bit should be initialized to 0.
107 sring
->va
= dma_alloc_coherent(dev
, sz
, &sring
->pa
, GFP_KERNEL
);
111 wil_dbg_misc(wil
, "status_ring[%d] 0x%p:%pad\n", sring
->size
, sring
->va
,
117 static int wil_tx_init_edma(struct wil6210_priv
*wil
)
119 int ring_id
= wil_find_free_sring(wil
);
120 struct wil_status_ring
*sring
;
122 u16 status_ring_size
;
124 if (wil
->tx_status_ring_order
< WIL_SRING_SIZE_ORDER_MIN
||
125 wil
->tx_status_ring_order
> WIL_SRING_SIZE_ORDER_MAX
)
126 wil
->tx_status_ring_order
= WIL_TX_SRING_SIZE_ORDER_DEFAULT
;
128 status_ring_size
= 1 << wil
->tx_status_ring_order
;
130 wil_dbg_misc(wil
, "init TX sring: size=%u, ring_id=%u\n",
131 status_ring_size
, ring_id
);
136 /* Allocate Tx status ring. Tx descriptor rings will be
137 * allocated on WMI connect event
139 sring
= &wil
->srings
[ring_id
];
141 sring
->is_rx
= false;
142 sring
->size
= status_ring_size
;
143 sring
->elem_size
= sizeof(struct wil_ring_tx_status
);
144 rc
= wil_sring_alloc(wil
, sring
);
148 rc
= wil_wmi_tx_sring_cfg(wil
, ring_id
);
152 sring
->desc_rdy_pol
= 1;
153 wil
->tx_sring_idx
= ring_id
;
157 wil_sring_free(wil
, sring
);
162 * Allocate one skb for Rx descriptor RING
164 static int wil_ring_alloc_skb_edma(struct wil6210_priv
*wil
,
165 struct wil_ring
*ring
, u32 i
)
167 struct device
*dev
= wil_to_dev(wil
);
168 unsigned int sz
= wil
->rx_buf_len
;
171 struct list_head
*active
= &wil
->rx_buff_mgmt
.active
;
172 struct list_head
*free
= &wil
->rx_buff_mgmt
.free
;
173 struct wil_rx_buff
*rx_buff
;
174 struct wil_rx_buff
*buff_arr
= wil
->rx_buff_mgmt
.buff_arr
;
176 struct wil_rx_enhanced_desc dd
, *d
= &dd
;
177 struct wil_rx_enhanced_desc
*_d
= (struct wil_rx_enhanced_desc
*)
178 &ring
->va
[i
].rx
.enhanced
;
180 if (unlikely(list_empty(free
))) {
181 wil
->rx_buff_mgmt
.free_list_empty_cnt
++;
185 skb
= dev_alloc_skb(sz
);
192 * Make sure that the network stack calculates checksum for packets
193 * which failed the HW checksum calculation
195 skb
->ip_summed
= CHECKSUM_NONE
;
197 pa
= dma_map_single(dev
, skb
->data
, skb
->len
, DMA_FROM_DEVICE
);
198 if (unlikely(dma_mapping_error(dev
, pa
))) {
203 /* Get the buffer ID - the index of the rx buffer in the buff_arr */
204 rx_buff
= list_first_entry(free
, struct wil_rx_buff
, list
);
205 buff_id
= rx_buff
->id
;
207 /* Move a buffer from the free list to the active list */
208 list_move(&rx_buff
->list
, active
);
210 buff_arr
[buff_id
].skb
= skb
;
212 wil_desc_set_addr_edma(&d
->dma
.addr
, &d
->dma
.addr_high_high
, pa
);
213 d
->dma
.length
= cpu_to_le16(sz
);
214 d
->mac
.buff_id
= cpu_to_le16(buff_id
);
217 /* Save the physical address in skb->cb for later use in dma_unmap */
218 memcpy(skb
->cb
, &pa
, sizeof(pa
));
224 void wil_get_next_rx_status_msg(struct wil_status_ring
*sring
, u8
*dr_bit
,
227 struct wil_rx_status_compressed
*_msg
;
229 _msg
= (struct wil_rx_status_compressed
*)
230 (sring
->va
+ (sring
->elem_size
* sring
->swhead
));
231 *dr_bit
= WIL_GET_BITS(_msg
->d0
, 31, 31);
232 /* make sure dr_bit is read before the rest of status msg */
234 memcpy(msg
, (void *)_msg
, sring
->elem_size
);
237 static inline void wil_sring_advance_swhead(struct wil_status_ring
*sring
)
239 sring
->swhead
= (sring
->swhead
+ 1) % sring
->size
;
240 if (sring
->swhead
== 0)
241 sring
->desc_rdy_pol
= 1 - sring
->desc_rdy_pol
;
244 static int wil_rx_refill_edma(struct wil6210_priv
*wil
)
246 struct wil_ring
*ring
= &wil
->ring_rx
;
249 ring
->swtail
= *ring
->edma_rx_swtail
.va
;
251 for (; next_head
= wil_ring_next_head(ring
),
252 (next_head
!= ring
->swtail
);
253 ring
->swhead
= next_head
) {
254 rc
= wil_ring_alloc_skb_edma(wil
, ring
, ring
->swhead
);
257 wil_dbg_txrx(wil
, "No free buffer ID found\n");
259 wil_err_ratelimited(wil
,
260 "Error %d in refill desc[%d]\n",
266 /* make sure all writes to descriptors (shared memory) are done before
267 * committing them to HW
271 wil_w(wil
, ring
->hwtail
, ring
->swhead
);
276 static void wil_move_all_rx_buff_to_free_list(struct wil6210_priv
*wil
,
277 struct wil_ring
*ring
)
279 struct device
*dev
= wil_to_dev(wil
);
280 struct list_head
*active
= &wil
->rx_buff_mgmt
.active
;
283 if (!wil
->rx_buff_mgmt
.buff_arr
)
286 while (!list_empty(active
)) {
287 struct wil_rx_buff
*rx_buff
=
288 list_first_entry(active
, struct wil_rx_buff
, list
);
289 struct sk_buff
*skb
= rx_buff
->skb
;
291 if (unlikely(!skb
)) {
292 wil_err(wil
, "No Rx skb at buff_id %d\n", rx_buff
->id
);
295 memcpy(&pa
, skb
->cb
, sizeof(pa
));
296 dma_unmap_single(dev
, pa
, wil
->rx_buf_len
,
301 /* Move the buffer from the active to the free list */
302 list_move(&rx_buff
->list
, &wil
->rx_buff_mgmt
.free
);
306 static void wil_free_rx_buff_arr(struct wil6210_priv
*wil
)
308 struct wil_ring
*ring
= &wil
->ring_rx
;
310 if (!wil
->rx_buff_mgmt
.buff_arr
)
313 /* Move all the buffers to the free list in case active list is
314 * not empty in order to release all SKBs before deleting the array
316 wil_move_all_rx_buff_to_free_list(wil
, ring
);
318 kfree(wil
->rx_buff_mgmt
.buff_arr
);
319 wil
->rx_buff_mgmt
.buff_arr
= NULL
;
322 static int wil_init_rx_buff_arr(struct wil6210_priv
*wil
,
325 struct wil_rx_buff
*buff_arr
;
326 struct list_head
*active
= &wil
->rx_buff_mgmt
.active
;
327 struct list_head
*free
= &wil
->rx_buff_mgmt
.free
;
330 wil
->rx_buff_mgmt
.buff_arr
= kcalloc(size
+ 1,
331 sizeof(struct wil_rx_buff
),
333 if (!wil
->rx_buff_mgmt
.buff_arr
)
337 INIT_LIST_HEAD(active
);
338 INIT_LIST_HEAD(free
);
341 * buffer id 0 should not be used (marks invalid id).
343 buff_arr
= wil
->rx_buff_mgmt
.buff_arr
;
344 for (i
= 1; i
<= size
; i
++) {
345 list_add(&buff_arr
[i
].list
, free
);
349 wil
->rx_buff_mgmt
.size
= size
+ 1;
354 static int wil_init_rx_sring(struct wil6210_priv
*wil
,
355 u16 status_ring_size
,
359 struct wil_status_ring
*sring
= &wil
->srings
[ring_id
];
362 wil_dbg_misc(wil
, "init RX sring: size=%u, ring_id=%u\n",
363 status_ring_size
, ring_id
);
365 memset(&sring
->rx_data
, 0, sizeof(sring
->rx_data
));
368 sring
->size
= status_ring_size
;
369 sring
->elem_size
= elem_size
;
370 rc
= wil_sring_alloc(wil
, sring
);
374 rc
= wil_wmi_rx_sring_add(wil
, ring_id
);
378 sring
->desc_rdy_pol
= 1;
382 wil_sring_free(wil
, sring
);
386 static int wil_ring_alloc_desc_ring(struct wil6210_priv
*wil
,
387 struct wil_ring
*ring
)
389 struct device
*dev
= wil_to_dev(wil
);
390 size_t sz
= ring
->size
* sizeof(ring
->va
[0]);
392 wil_dbg_misc(wil
, "alloc_desc_ring:\n");
394 BUILD_BUG_ON(sizeof(ring
->va
[0]) != 32);
398 ring
->ctx
= kcalloc(ring
->size
, sizeof(ring
->ctx
[0]), GFP_KERNEL
);
402 ring
->va
= dma_alloc_coherent(dev
, sz
, &ring
->pa
, GFP_KERNEL
);
407 sz
= sizeof(*ring
->edma_rx_swtail
.va
);
408 ring
->edma_rx_swtail
.va
=
409 dma_alloc_coherent(dev
, sz
, &ring
->edma_rx_swtail
.pa
,
411 if (!ring
->edma_rx_swtail
.va
)
415 wil_dbg_misc(wil
, "%s ring[%d] 0x%p:%pad 0x%p\n",
416 ring
->is_rx
? "RX" : "TX",
417 ring
->size
, ring
->va
, &ring
->pa
, ring
->ctx
);
421 dma_free_coherent(dev
, ring
->size
* sizeof(ring
->va
[0]),
422 (void *)ring
->va
, ring
->pa
);
431 static void wil_ring_free_edma(struct wil6210_priv
*wil
, struct wil_ring
*ring
)
433 struct device
*dev
= wil_to_dev(wil
);
440 sz
= ring
->size
* sizeof(ring
->va
[0]);
442 lockdep_assert_held(&wil
->mutex
);
444 wil_dbg_misc(wil
, "free Rx ring [%d] 0x%p:%pad 0x%p\n",
445 ring
->size
, ring
->va
,
446 &ring
->pa
, ring
->ctx
);
448 wil_move_all_rx_buff_to_free_list(wil
, ring
);
449 dma_free_coherent(dev
, sizeof(*ring
->edma_rx_swtail
.va
),
450 ring
->edma_rx_swtail
.va
,
451 ring
->edma_rx_swtail
.pa
);
456 ring_index
= ring
- wil
->ring_tx
;
458 wil_dbg_misc(wil
, "free Tx ring %d [%d] 0x%p:%pad 0x%p\n",
459 ring_index
, ring
->size
, ring
->va
,
460 &ring
->pa
, ring
->ctx
);
462 while (!wil_ring_is_empty(ring
)) {
465 struct wil_tx_enhanced_desc dd
, *d
= &dd
;
466 struct wil_tx_enhanced_desc
*_d
=
467 (struct wil_tx_enhanced_desc
*)
468 &ring
->va
[ring
->swtail
].tx
.enhanced
;
470 ctx
= &ring
->ctx
[ring
->swtail
];
473 "ctx(%d) was already completed\n",
475 ring
->swtail
= wil_ring_next_tail(ring
);
479 wil_tx_desc_unmap_edma(dev
, (union wil_tx_desc
*)d
, ctx
);
481 dev_kfree_skb_any(ctx
->skb
);
482 ring
->swtail
= wil_ring_next_tail(ring
);
486 dma_free_coherent(dev
, sz
, (void *)ring
->va
, ring
->pa
);
493 static int wil_init_rx_desc_ring(struct wil6210_priv
*wil
, u16 desc_ring_size
,
496 struct wil_ring
*ring
= &wil
->ring_rx
;
499 wil_dbg_misc(wil
, "init RX desc ring\n");
501 ring
->size
= desc_ring_size
;
503 rc
= wil_ring_alloc_desc_ring(wil
, ring
);
507 rc
= wil_wmi_rx_desc_ring_add(wil
, status_ring_id
);
513 wil_ring_free_edma(wil
, ring
);
517 static void wil_get_reorder_params_edma(struct wil6210_priv
*wil
,
518 struct sk_buff
*skb
, int *tid
,
519 int *cid
, int *mid
, u16
*seq
,
520 int *mcast
, int *retry
)
522 struct wil_rx_status_extended
*s
= wil_skb_rxstatus(skb
);
524 *tid
= wil_rx_status_get_tid(s
);
525 *cid
= wil_rx_status_get_cid(s
);
526 *mid
= wil_rx_status_get_mid(s
);
527 *seq
= le16_to_cpu(wil_rx_status_get_seq(wil
, s
));
528 *mcast
= wil_rx_status_get_mcast(s
);
529 *retry
= wil_rx_status_get_retry(s
);
532 static void wil_get_netif_rx_params_edma(struct sk_buff
*skb
, int *cid
,
535 struct wil_rx_status_extended
*s
= wil_skb_rxstatus(skb
);
537 *cid
= wil_rx_status_get_cid(s
);
538 *security
= wil_rx_status_get_security(s
);
541 static int wil_rx_crypto_check_edma(struct wil6210_priv
*wil
,
544 struct wil_rx_status_extended
*st
;
545 int cid
, tid
, key_id
, mc
;
546 struct wil_sta_info
*s
;
547 struct wil_tid_crypto_rx
*c
;
548 struct wil_tid_crypto_rx_single
*cc
;
551 /* In HW reorder, HW is responsible for crypto check */
552 if (wil
->use_rx_hw_reordering
)
555 st
= wil_skb_rxstatus(skb
);
557 cid
= wil_rx_status_get_cid(st
);
558 tid
= wil_rx_status_get_tid(st
);
559 key_id
= wil_rx_status_get_key_id(st
);
560 mc
= wil_rx_status_get_mcast(st
);
562 c
= mc
? &s
->group_crypto_rx
: &s
->tid_crypto_rx
[tid
];
563 cc
= &c
->key_id
[key_id
];
564 pn
= (u8
*)&st
->ext
.pn_15_0
;
567 wil_err_ratelimited(wil
,
568 "Key missing. CID %d TID %d MCast %d KEY_ID %d\n",
569 cid
, tid
, mc
, key_id
);
573 if (reverse_memcmp(pn
, cc
->pn
, IEEE80211_GCMP_PN_LEN
) <= 0) {
574 wil_err_ratelimited(wil
,
575 "Replay attack. CID %d TID %d MCast %d KEY_ID %d PN %6phN last %6phN\n",
576 cid
, tid
, mc
, key_id
, pn
, cc
->pn
);
579 memcpy(cc
->pn
, pn
, IEEE80211_GCMP_PN_LEN
);
584 static bool wil_is_rx_idle_edma(struct wil6210_priv
*wil
)
586 struct wil_status_ring
*sring
;
587 struct wil_rx_status_extended msg1
;
592 for (i
= 0; i
< wil
->num_rx_status_rings
; i
++) {
593 sring
= &wil
->srings
[i
];
597 wil_get_next_rx_status_msg(sring
, &dr_bit
, msg
);
599 /* Check if there are unhandled RX status messages */
600 if (dr_bit
== sring
->desc_rdy_pol
)
607 static void wil_rx_buf_len_init_edma(struct wil6210_priv
*wil
)
609 /* RX buffer size must be aligned to 4 bytes */
610 wil
->rx_buf_len
= rx_large_buf
?
611 WIL_MAX_ETH_MTU
: WIL_EDMA_RX_BUF_LEN_DEFAULT
;
614 static int wil_rx_init_edma(struct wil6210_priv
*wil
, uint desc_ring_order
)
616 u16 status_ring_size
, desc_ring_size
= 1 << desc_ring_order
;
617 struct wil_ring
*ring
= &wil
->ring_rx
;
619 size_t elem_size
= wil
->use_compressed_rx_status
?
620 sizeof(struct wil_rx_status_compressed
) :
621 sizeof(struct wil_rx_status_extended
);
624 /* In SW reorder one must use extended status messages */
625 if (wil
->use_compressed_rx_status
&& !wil
->use_rx_hw_reordering
) {
627 "compressed RX status cannot be used with SW reorder\n");
630 if (wil
->rx_status_ring_order
<= desc_ring_order
)
631 /* make sure sring is larger than desc ring */
632 wil
->rx_status_ring_order
= desc_ring_order
+ 1;
633 if (wil
->rx_buff_id_count
<= desc_ring_size
)
634 /* make sure we will not run out of buff_ids */
635 wil
->rx_buff_id_count
= desc_ring_size
+ 512;
636 if (wil
->rx_status_ring_order
< WIL_SRING_SIZE_ORDER_MIN
||
637 wil
->rx_status_ring_order
> WIL_SRING_SIZE_ORDER_MAX
)
638 wil
->rx_status_ring_order
= WIL_RX_SRING_SIZE_ORDER_DEFAULT
;
640 status_ring_size
= 1 << wil
->rx_status_ring_order
;
643 "rx_init, desc_ring_size=%u, status_ring_size=%u, elem_size=%zu\n",
644 desc_ring_size
, status_ring_size
, elem_size
);
646 wil_rx_buf_len_init_edma(wil
);
648 /* Use debugfs dbg_num_rx_srings if set, reserve one sring for TX */
649 if (wil
->num_rx_status_rings
> WIL6210_MAX_STATUS_RINGS
- 1)
650 wil
->num_rx_status_rings
= WIL6210_MAX_STATUS_RINGS
- 1;
652 wil_dbg_misc(wil
, "rx_init: allocate %d status rings\n",
653 wil
->num_rx_status_rings
);
655 rc
= wil_wmi_cfg_def_rx_offload(wil
, wil
->rx_buf_len
);
659 /* Allocate status ring */
660 for (i
= 0; i
< wil
->num_rx_status_rings
; i
++) {
661 int sring_id
= wil_find_free_sring(wil
);
665 goto err_free_status
;
667 rc
= wil_init_rx_sring(wil
, status_ring_size
, elem_size
,
670 goto err_free_status
;
673 /* Allocate descriptor ring */
674 rc
= wil_init_rx_desc_ring(wil
, desc_ring_size
,
675 WIL_DEFAULT_RX_STATUS_RING_ID
);
677 goto err_free_status
;
679 if (wil
->rx_buff_id_count
>= status_ring_size
) {
681 "rx_buff_id_count %d exceeds sring_size %d. set it to %d\n",
682 wil
->rx_buff_id_count
, status_ring_size
,
683 status_ring_size
- 1);
684 wil
->rx_buff_id_count
= status_ring_size
- 1;
687 /* Allocate Rx buffer array */
688 rc
= wil_init_rx_buff_arr(wil
, wil
->rx_buff_id_count
);
692 /* Fill descriptor ring with credits */
693 rc
= wil_rx_refill_edma(wil
);
695 goto err_free_rx_buff_arr
;
698 err_free_rx_buff_arr
:
699 wil_free_rx_buff_arr(wil
);
701 wil_ring_free_edma(wil
, ring
);
703 for (i
= 0; i
< wil
->num_rx_status_rings
; i
++)
704 wil_sring_free(wil
, &wil
->srings
[i
]);
709 static int wil_ring_init_tx_edma(struct wil6210_vif
*vif
, int ring_id
,
710 int size
, int cid
, int tid
)
712 struct wil6210_priv
*wil
= vif_to_wil(vif
);
714 struct wil_ring
*ring
= &wil
->ring_tx
[ring_id
];
715 struct wil_ring_tx_data
*txdata
= &wil
->ring_tx_data
[ring_id
];
717 lockdep_assert_held(&wil
->mutex
);
720 "init TX ring: ring_id=%u, cid=%u, tid=%u, sring_id=%u\n",
721 ring_id
, cid
, tid
, wil
->tx_sring_idx
);
723 wil_tx_data_init(txdata
);
725 rc
= wil_ring_alloc_desc_ring(wil
, ring
);
729 wil
->ring2cid_tid
[ring_id
][0] = cid
;
730 wil
->ring2cid_tid
[ring_id
][1] = tid
;
732 txdata
->dot1x_open
= true;
734 rc
= wil_wmi_tx_desc_ring_add(vif
, ring_id
, cid
, tid
);
736 wil_err(wil
, "WMI_TX_DESC_RING_ADD_CMD failed\n");
740 if (txdata
->dot1x_open
&& agg_wsize
>= 0)
741 wil_addba_tx_request(wil
, ring_id
, agg_wsize
);
745 spin_lock_bh(&txdata
->lock
);
746 txdata
->dot1x_open
= false;
748 spin_unlock_bh(&txdata
->lock
);
749 wil_ring_free_edma(wil
, ring
);
750 wil
->ring2cid_tid
[ring_id
][0] = wil
->max_assoc_sta
;
751 wil
->ring2cid_tid
[ring_id
][1] = 0;
757 static int wil_tx_ring_modify_edma(struct wil6210_vif
*vif
, int ring_id
,
760 struct wil6210_priv
*wil
= vif_to_wil(vif
);
762 wil_err(wil
, "ring modify is not supported for EDMA\n");
767 /* This function is used only for RX SW reorder */
768 static int wil_check_bar(struct wil6210_priv
*wil
, void *msg
, int cid
,
769 struct sk_buff
*skb
, struct wil_net_stats
*stats
)
776 struct wil6210_vif
*vif
;
778 ftype
= wil_rx_status_get_frame_type(wil
, msg
);
779 if (ftype
== IEEE80211_FTYPE_DATA
)
782 fc1
= wil_rx_status_get_fc1(wil
, msg
);
783 mid
= wil_rx_status_get_mid(msg
);
784 tid
= wil_rx_status_get_tid(msg
);
785 seq
= le16_to_cpu(wil_rx_status_get_seq(wil
, msg
));
786 vif
= wil
->vifs
[mid
];
788 if (unlikely(!vif
)) {
789 wil_dbg_txrx(wil
, "RX descriptor with invalid mid %d", mid
);
794 "Non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n",
795 fc1
, mid
, cid
, tid
, seq
);
797 stats
->rx_non_data_frame
++;
798 if (wil_is_back_req(fc1
)) {
800 "BAR: MID %d CID %d TID %d Seq 0x%03x\n",
802 wil_rx_bar(wil
, vif
, cid
, tid
, seq
);
804 u32 sz
= wil
->use_compressed_rx_status
?
805 sizeof(struct wil_rx_status_compressed
) :
806 sizeof(struct wil_rx_status_extended
);
808 /* print again all info. One can enable only this
809 * without overhead for printing every Rx frame
812 "Unhandled non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n",
813 fc1
, mid
, cid
, tid
, seq
);
814 wil_hex_dump_txrx("RxS ", DUMP_PREFIX_NONE
, 32, 4,
815 (const void *)msg
, sz
, false);
816 wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET
, 16, 1,
817 skb
->data
, skb_headlen(skb
), false);
823 static int wil_rx_error_check_edma(struct wil6210_priv
*wil
,
825 struct wil_net_stats
*stats
)
828 void *msg
= wil_skb_rxstatus(skb
);
830 l2_rx_status
= wil_rx_status_get_l2_rx_status(msg
);
831 if (l2_rx_status
!= 0) {
832 wil_dbg_txrx(wil
, "L2 RX error, l2_rx_status=0x%x\n",
834 /* Due to HW issue, KEY error will trigger a MIC error */
835 if (l2_rx_status
== WIL_RX_EDMA_ERROR_MIC
) {
836 wil_err_ratelimited(wil
,
837 "L2 MIC/KEY error, dropping packet\n");
838 stats
->rx_mic_error
++;
840 if (l2_rx_status
== WIL_RX_EDMA_ERROR_KEY
) {
841 wil_err_ratelimited(wil
,
842 "L2 KEY error, dropping packet\n");
843 stats
->rx_key_error
++;
845 if (l2_rx_status
== WIL_RX_EDMA_ERROR_REPLAY
) {
846 wil_err_ratelimited(wil
,
847 "L2 REPLAY error, dropping packet\n");
850 if (l2_rx_status
== WIL_RX_EDMA_ERROR_AMSDU
) {
851 wil_err_ratelimited(wil
,
852 "L2 AMSDU error, dropping packet\n");
853 stats
->rx_amsdu_error
++;
858 skb
->ip_summed
= wil_rx_status_get_checksum(msg
, stats
);
863 static struct sk_buff
*wil_sring_reap_rx_edma(struct wil6210_priv
*wil
,
864 struct wil_status_ring
*sring
)
866 struct device
*dev
= wil_to_dev(wil
);
867 struct wil_rx_status_extended msg1
;
872 struct wil_ring_rx_data
*rxdata
= &sring
->rx_data
;
873 unsigned int sz
= wil
->rx_buf_len
;
874 struct wil_net_stats
*stats
= NULL
;
877 bool eop
, headstolen
;
881 struct wil_rx_status_extended
*s
;
882 u16 sring_idx
= sring
- wil
->srings
;
884 BUILD_BUG_ON(sizeof(struct wil_rx_status_extended
) > sizeof(skb
->cb
));
887 wil_get_next_rx_status_msg(sring
, &dr_bit
, msg
);
889 /* Completed handling all the ready status messages */
890 if (dr_bit
!= sring
->desc_rdy_pol
)
893 /* Extract the buffer ID from the status message */
894 buff_id
= le16_to_cpu(wil_rx_status_get_buff_id(msg
));
897 struct wil_rx_status_extended
*s
;
898 int invalid_buff_id_retry
= 0;
901 "buff_id is not updated yet by HW, (swhead 0x%x)\n",
903 if (++invalid_buff_id_retry
> MAX_INVALID_BUFF_ID_RETRY
)
906 /* Read the status message again */
907 s
= (struct wil_rx_status_extended
*)
908 (sring
->va
+ (sring
->elem_size
* sring
->swhead
));
909 *(struct wil_rx_status_extended
*)msg
= *s
;
910 buff_id
= le16_to_cpu(wil_rx_status_get_buff_id(msg
));
913 if (unlikely(!wil_val_in_range(buff_id
, 1, wil
->rx_buff_mgmt
.size
))) {
914 wil_err(wil
, "Corrupt buff_id=%d, sring->swhead=%d\n",
915 buff_id
, sring
->swhead
);
916 wil_rx_status_reset_buff_id(sring
);
917 wil_sring_advance_swhead(sring
);
918 sring
->invalid_buff_id_cnt
++;
922 /* Extract the SKB from the rx_buff management array */
923 skb
= wil
->rx_buff_mgmt
.buff_arr
[buff_id
].skb
;
924 wil
->rx_buff_mgmt
.buff_arr
[buff_id
].skb
= NULL
;
926 wil_err(wil
, "No Rx skb at buff_id %d\n", buff_id
);
927 wil_rx_status_reset_buff_id(sring
);
928 /* Move the buffer from the active list to the free list */
929 list_move_tail(&wil
->rx_buff_mgmt
.buff_arr
[buff_id
].list
,
930 &wil
->rx_buff_mgmt
.free
);
931 wil_sring_advance_swhead(sring
);
932 sring
->invalid_buff_id_cnt
++;
936 wil_rx_status_reset_buff_id(sring
);
937 wil_sring_advance_swhead(sring
);
939 memcpy(&pa
, skb
->cb
, sizeof(pa
));
940 dma_unmap_single(dev
, pa
, sz
, DMA_FROM_DEVICE
);
941 dmalen
= le16_to_cpu(wil_rx_status_get_length(msg
));
943 trace_wil6210_rx_status(wil
, wil
->use_compressed_rx_status
, buff_id
,
945 wil_dbg_txrx(wil
, "Rx, buff_id=%u, sring_idx=%u, dmalen=%u bytes\n",
946 buff_id
, sring_idx
, dmalen
);
947 wil_hex_dump_txrx("RxS ", DUMP_PREFIX_NONE
, 32, 4,
948 (const void *)msg
, wil
->use_compressed_rx_status
?
949 sizeof(struct wil_rx_status_compressed
) :
950 sizeof(struct wil_rx_status_extended
), false);
952 /* Move the buffer from the active list to the free list */
953 list_move_tail(&wil
->rx_buff_mgmt
.buff_arr
[buff_id
].list
,
954 &wil
->rx_buff_mgmt
.free
);
956 eop
= wil_rx_status_get_eop(msg
);
958 cid
= wil_rx_status_get_cid(msg
);
959 if (unlikely(!wil_val_in_range(cid
, 0, wil
->max_assoc_sta
))) {
960 wil_err(wil
, "Corrupt cid=%d, sring->swhead=%d\n",
962 rxdata
->skipping
= true;
965 stats
= &wil
->sta
[cid
].stats
;
967 if (unlikely(dmalen
< ETH_HLEN
)) {
968 wil_dbg_txrx(wil
, "Short frame, len = %d\n", dmalen
);
969 stats
->rx_short_frame
++;
970 rxdata
->skipping
= true;
974 if (unlikely(dmalen
> sz
)) {
975 wil_err(wil
, "Rx size too large: %d bytes!\n", dmalen
);
976 stats
->rx_large_frame
++;
977 rxdata
->skipping
= true;
981 /* skipping indicates if a certain SKB should be dropped.
982 * It is set in case there is an error on the current SKB or in case
983 * of RX chaining: as long as we manage to merge the SKBs it will
984 * be false. once we have a bad SKB or we don't manage to merge SKBs
985 * it will be set to the !EOP value of the current SKB.
986 * This guarantees that all the following SKBs until EOP will also
989 if (unlikely(rxdata
->skipping
)) {
992 kfree_skb(rxdata
->skb
);
995 rxdata
->skipping
= !eop
;
999 skb_trim(skb
, dmalen
);
1001 prefetch(skb
->data
);
1006 if (likely(skb_try_coalesce(rxdata
->skb
, skb
, &headstolen
,
1008 kfree_skb_partial(skb
, headstolen
);
1010 wil_err(wil
, "failed to merge skbs!\n");
1012 kfree_skb(rxdata
->skb
);
1014 rxdata
->skipping
= !eop
;
1022 /* reaching here rxdata->skb always contains a full packet */
1025 rxdata
->skipping
= false;
1028 stats
->last_mcs_rx
= wil_rx_status_get_mcs(msg
);
1029 if (stats
->last_mcs_rx
< ARRAY_SIZE(stats
->rx_per_mcs
))
1030 stats
->rx_per_mcs
[stats
->last_mcs_rx
]++;
1032 stats
->last_cb_mode_rx
= wil_rx_status_get_cb_mode(msg
);
1035 if (!wil
->use_rx_hw_reordering
&& !wil
->use_compressed_rx_status
&&
1036 wil_check_bar(wil
, msg
, cid
, skb
, stats
) == -EAGAIN
) {
1041 /* Compensate for the HW data alignment according to the status
1044 data_offset
= wil_rx_status_get_data_offset(msg
);
1045 if (data_offset
== 0xFF ||
1046 data_offset
> WIL_EDMA_MAX_DATA_OFFSET
) {
1047 wil_err(wil
, "Unexpected data offset %d\n", data_offset
);
1052 skb_pull(skb
, data_offset
);
1054 wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET
, 16, 1,
1055 skb
->data
, skb_headlen(skb
), false);
1057 /* Has to be done after dma_unmap_single as skb->cb is also
1058 * used for holding the pa
1060 s
= wil_skb_rxstatus(skb
);
1061 memcpy(s
, msg
, sring
->elem_size
);
1066 void wil_rx_handle_edma(struct wil6210_priv
*wil
, int *quota
)
1068 struct net_device
*ndev
;
1069 struct wil_ring
*ring
= &wil
->ring_rx
;
1070 struct wil_status_ring
*sring
;
1071 struct sk_buff
*skb
;
1074 if (unlikely(!ring
->va
)) {
1075 wil_err(wil
, "Rx IRQ while Rx not yet initialized\n");
1078 wil_dbg_txrx(wil
, "rx_handle\n");
1080 for (i
= 0; i
< wil
->num_rx_status_rings
; i
++) {
1081 sring
= &wil
->srings
[i
];
1082 if (unlikely(!sring
->va
)) {
1084 "Rx IRQ while Rx status ring %d not yet initialized\n",
1089 while ((*quota
> 0) &&
1091 wil_sring_reap_rx_edma(wil
, sring
)))) {
1093 if (wil
->use_rx_hw_reordering
) {
1094 void *msg
= wil_skb_rxstatus(skb
);
1095 int mid
= wil_rx_status_get_mid(msg
);
1096 struct wil6210_vif
*vif
= wil
->vifs
[mid
];
1098 if (unlikely(!vif
)) {
1100 "RX desc invalid mid %d",
1105 ndev
= vif_to_ndev(vif
);
1106 wil_netif_rx_any(skb
, ndev
);
1108 wil_rx_reorder(wil
, skb
);
1112 wil_w(wil
, sring
->hwtail
, (sring
->swhead
- 1) % sring
->size
);
1115 wil_rx_refill_edma(wil
);
1118 static int wil_tx_desc_map_edma(union wil_tx_desc
*desc
,
1123 struct wil_tx_enhanced_desc
*d
=
1124 (struct wil_tx_enhanced_desc
*)&desc
->enhanced
;
1126 memset(d
, 0, sizeof(struct wil_tx_enhanced_desc
));
1128 wil_desc_set_addr_edma(&d
->dma
.addr
, &d
->dma
.addr_high_high
, pa
);
1130 /* 0..6: mac_length; 7:ip_version 0-IP6 1-IP4*/
1131 d
->dma
.length
= cpu_to_le16((u16
)len
);
1132 d
->mac
.d
[0] = (ring_index
<< WIL_EDMA_DESC_TX_MAC_CFG_0_QID_POS
);
1133 /* translation type: 0 - bypass; 1 - 802.3; 2 - native wifi;
1136 d
->mac
.d
[2] = BIT(MAC_CFG_DESC_TX_2_SNAP_HDR_INSERTION_EN_POS
) |
1137 (0x3 << MAC_CFG_DESC_TX_2_L2_TRANSLATION_TYPE_POS
);
1143 wil_get_next_tx_status_msg(struct wil_status_ring
*sring
, u8
*dr_bit
,
1144 struct wil_ring_tx_status
*msg
)
1146 struct wil_ring_tx_status
*_msg
= (struct wil_ring_tx_status
*)
1147 (sring
->va
+ (sring
->elem_size
* sring
->swhead
));
1149 *dr_bit
= _msg
->desc_ready
>> TX_STATUS_DESC_READY_POS
;
1150 /* make sure dr_bit is read before the rest of status msg */
1156 * Clean up transmitted skb's from the Tx descriptor RING.
1157 * Return number of descriptors cleared.
1159 int wil_tx_sring_handler(struct wil6210_priv
*wil
,
1160 struct wil_status_ring
*sring
)
1162 struct net_device
*ndev
;
1163 struct device
*dev
= wil_to_dev(wil
);
1164 struct wil_ring
*ring
= NULL
;
1165 struct wil_ring_tx_data
*txdata
;
1166 /* Total number of completed descriptors in all descriptor rings */
1169 struct wil_net_stats
*stats
;
1170 struct wil_tx_enhanced_desc
*_d
;
1171 unsigned int ring_id
;
1172 unsigned int num_descs
, num_statuses
= 0;
1174 u8 dr_bit
; /* Descriptor Ready bit */
1175 struct wil_ring_tx_status msg
;
1176 struct wil6210_vif
*vif
;
1177 int used_before_complete
;
1180 wil_get_next_tx_status_msg(sring
, &dr_bit
, &msg
);
1182 /* Process completion messages while DR bit has the expected polarity */
1183 while (dr_bit
== sring
->desc_rdy_pol
) {
1184 num_descs
= msg
.num_descriptors
;
1186 wil_err(wil
, "invalid num_descs 0\n");
1190 /* Find the corresponding descriptor ring */
1191 ring_id
= msg
.ring_id
;
1193 if (unlikely(ring_id
>= WIL6210_MAX_TX_RINGS
)) {
1194 wil_err(wil
, "invalid ring id %d\n", ring_id
);
1197 ring
= &wil
->ring_tx
[ring_id
];
1198 if (unlikely(!ring
->va
)) {
1199 wil_err(wil
, "Tx irq[%d]: ring not initialized\n",
1203 txdata
= &wil
->ring_tx_data
[ring_id
];
1204 if (unlikely(!txdata
->enabled
)) {
1205 wil_info(wil
, "Tx irq[%d]: ring disabled\n", ring_id
);
1208 vif
= wil
->vifs
[txdata
->mid
];
1209 if (unlikely(!vif
)) {
1210 wil_dbg_txrx(wil
, "invalid MID %d for ring %d\n",
1211 txdata
->mid
, ring_id
);
1215 ndev
= vif_to_ndev(vif
);
1217 cid
= wil
->ring2cid_tid
[ring_id
][0];
1218 stats
= (cid
< wil
->max_assoc_sta
) ? &wil
->sta
[cid
].stats
:
1222 "tx_status: completed desc_ring (%d), num_descs (%d)\n",
1223 ring_id
, num_descs
);
1225 used_before_complete
= wil_ring_used_tx(ring
);
1227 for (i
= 0 ; i
< num_descs
; ++i
) {
1228 struct wil_ctx
*ctx
= &ring
->ctx
[ring
->swtail
];
1229 struct wil_tx_enhanced_desc dd
, *d
= &dd
;
1231 struct sk_buff
*skb
= ctx
->skb
;
1233 _d
= (struct wil_tx_enhanced_desc
*)
1234 &ring
->va
[ring
->swtail
].tx
.enhanced
;
1237 dmalen
= le16_to_cpu(d
->dma
.length
);
1238 trace_wil6210_tx_status(&msg
, ring
->swtail
, dmalen
);
1240 "TxC[%2d][%3d] : %d bytes, status 0x%02x\n",
1241 ring_id
, ring
->swtail
, dmalen
,
1243 wil_hex_dump_txrx("TxS ", DUMP_PREFIX_NONE
, 32, 4,
1244 (const void *)&msg
, sizeof(msg
),
1247 wil_tx_desc_unmap_edma(dev
,
1248 (union wil_tx_desc
*)d
,
1252 if (likely(msg
.status
== 0)) {
1253 ndev
->stats
.tx_packets
++;
1254 ndev
->stats
.tx_bytes
+= skb
->len
;
1256 stats
->tx_packets
++;
1257 stats
->tx_bytes
+= skb
->len
;
1259 wil_tx_latency_calc(wil
, skb
,
1263 ndev
->stats
.tx_errors
++;
1268 if (skb
->protocol
== cpu_to_be16(ETH_P_PAE
))
1269 wil_tx_complete_handle_eapol(vif
, skb
);
1271 wil_consume_skb(skb
, msg
.status
== 0);
1273 memset(ctx
, 0, sizeof(*ctx
));
1274 /* Make sure the ctx is zeroed before updating the tail
1275 * to prevent a case where wil_tx_ring will see
1276 * this descriptor as used and handle it before ctx zero
1281 ring
->swtail
= wil_ring_next_tail(ring
);
1286 /* performance monitoring */
1287 used_new
= wil_ring_used_tx(ring
);
1288 if (wil_val_in_range(wil
->ring_idle_trsh
,
1289 used_new
, used_before_complete
)) {
1290 wil_dbg_txrx(wil
, "Ring[%2d] idle %d -> %d\n",
1291 ring_id
, used_before_complete
, used_new
);
1292 txdata
->last_idle
= get_cycles();
1297 if (num_statuses
% WIL_EDMA_TX_SRING_UPDATE_HW_TAIL
== 0)
1298 /* update HW tail to allow HW to push new statuses */
1299 wil_w(wil
, sring
->hwtail
, sring
->swhead
);
1301 wil_sring_advance_swhead(sring
);
1303 wil_get_next_tx_status_msg(sring
, &dr_bit
, &msg
);
1306 /* shall we wake net queues? */
1308 wil_update_net_queues(wil
, vif
, NULL
, false);
1310 if (num_statuses
% WIL_EDMA_TX_SRING_UPDATE_HW_TAIL
!= 0)
1311 /* Update the HW tail ptr (RD ptr) */
1312 wil_w(wil
, sring
->hwtail
, (sring
->swhead
- 1) % sring
->size
);
1318 * Sets the descriptor @d up for csum and/or TSO offloading. The corresponding
1319 * @skb is used to obtain the protocol and headers length.
1320 * @tso_desc_type is a descriptor type for TSO: 0 - a header, 1 - first data,
1321 * 2 - middle, 3 - last descriptor.
1323 static void wil_tx_desc_offload_setup_tso_edma(struct wil_tx_enhanced_desc
*d
,
1324 int tso_desc_type
, bool is_ipv4
,
1326 int skb_net_hdr_len
,
1329 /* Number of descriptors */
1331 /* Maximum Segment Size */
1332 d
->mac
.tso_mss
|= cpu_to_le16(mss
>> 2);
1333 /* L4 header len: TCP header length */
1334 d
->dma
.l4_hdr_len
|= tcp_hdr_len
& DMA_CFG_DESC_TX_0_L4_LENGTH_MSK
;
1335 /* EOP, TSO desc type, Segmentation enable,
1336 * Insert IPv4 and TCP / UDP Checksum
1338 d
->dma
.cmd
|= BIT(WIL_EDMA_DESC_TX_CFG_EOP_POS
) |
1339 tso_desc_type
<< WIL_EDMA_DESC_TX_CFG_TSO_DESC_TYPE_POS
|
1340 BIT(WIL_EDMA_DESC_TX_CFG_SEG_EN_POS
) |
1341 BIT(WIL_EDMA_DESC_TX_CFG_INSERT_IP_CHKSUM_POS
) |
1342 BIT(WIL_EDMA_DESC_TX_CFG_INSERT_TCP_CHKSUM_POS
);
1343 /* Calculate pseudo-header */
1344 d
->dma
.w1
|= BIT(WIL_EDMA_DESC_TX_CFG_PSEUDO_HEADER_CALC_EN_POS
) |
1345 BIT(WIL_EDMA_DESC_TX_CFG_L4_TYPE_POS
);
1346 /* IP Header Length */
1347 d
->dma
.ip_length
|= skb_net_hdr_len
;
1348 /* MAC header length and IP address family*/
1349 d
->dma
.b11
|= ETH_HLEN
|
1350 is_ipv4
<< DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS
;
1353 static int wil_tx_tso_gen_desc(struct wil6210_priv
*wil
, void *buff_addr
,
1354 int len
, uint i
, int tso_desc_type
,
1355 skb_frag_t
*frag
, struct wil_ring
*ring
,
1356 struct sk_buff
*skb
, bool is_ipv4
,
1357 int tcp_hdr_len
, int skb_net_hdr_len
,
1358 int mss
, int *descs_used
)
1360 struct device
*dev
= wil_to_dev(wil
);
1361 struct wil_tx_enhanced_desc
*_desc
= (struct wil_tx_enhanced_desc
*)
1362 &ring
->va
[i
].tx
.enhanced
;
1363 struct wil_tx_enhanced_desc desc_mem
, *d
= &desc_mem
;
1364 int ring_index
= ring
- wil
->ring_tx
;
1371 pa
= dma_map_single(dev
, buff_addr
, len
, DMA_TO_DEVICE
);
1372 ring
->ctx
[i
].mapped_as
= wil_mapped_as_single
;
1374 pa
= skb_frag_dma_map(dev
, frag
, 0, len
, DMA_TO_DEVICE
);
1375 ring
->ctx
[i
].mapped_as
= wil_mapped_as_page
;
1377 if (unlikely(dma_mapping_error(dev
, pa
))) {
1378 wil_err(wil
, "TSO: Skb DMA map error\n");
1382 wil
->txrx_ops
.tx_desc_map((union wil_tx_desc
*)d
, pa
,
1384 wil_tx_desc_offload_setup_tso_edma(d
, tso_desc_type
, is_ipv4
,
1386 skb_net_hdr_len
, mss
);
1388 /* hold reference to skb
1389 * to prevent skb release before accounting
1390 * in case of immediate "tx done"
1392 if (tso_desc_type
== wil_tso_type_lst
)
1393 ring
->ctx
[i
].skb
= skb_get(skb
);
1395 wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE
, 32, 4,
1396 (const void *)d
, sizeof(*d
), false);
1404 static int __wil_tx_ring_tso_edma(struct wil6210_priv
*wil
,
1405 struct wil6210_vif
*vif
,
1406 struct wil_ring
*ring
,
1407 struct sk_buff
*skb
)
1409 int ring_index
= ring
- wil
->ring_tx
;
1410 struct wil_ring_tx_data
*txdata
= &wil
->ring_tx_data
[ring_index
];
1411 int nr_frags
= skb_shinfo(skb
)->nr_frags
;
1412 int min_desc_required
= nr_frags
+ 2; /* Headers, Head, Fragments */
1413 int used
, avail
= wil_ring_avail_tx(ring
);
1414 int f
, hdrlen
, headlen
;
1417 u32 swhead
= ring
->swhead
;
1418 int descs_used
= 0; /* total number of used descriptors */
1421 int skb_net_hdr_len
;
1422 int mss
= skb_shinfo(skb
)->gso_size
;
1424 wil_dbg_txrx(wil
, "tx_ring_tso: %d bytes to ring %d\n", skb
->len
,
1427 if (unlikely(!txdata
->enabled
))
1430 if (unlikely(avail
< min_desc_required
)) {
1431 wil_err_ratelimited(wil
,
1432 "TSO: Tx ring[%2d] full. No space for %d fragments\n",
1433 ring_index
, min_desc_required
);
1437 gso_type
= skb_shinfo(skb
)->gso_type
& (SKB_GSO_TCPV6
| SKB_GSO_TCPV4
);
1449 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1452 /* tcp header length and skb network header length are fixed for all
1453 * packet's descriptors - read them once here
1455 tcp_hdr_len
= tcp_hdrlen(skb
);
1456 skb_net_hdr_len
= skb_network_header_len(skb
);
1458 /* First descriptor must contain the header only
1459 * Header Length = MAC header len + IP header len + TCP header len
1461 hdrlen
= ETH_HLEN
+ tcp_hdr_len
+ skb_net_hdr_len
;
1462 wil_dbg_txrx(wil
, "TSO: process header descriptor, hdrlen %u\n",
1464 rc
= wil_tx_tso_gen_desc(wil
, skb
->data
, hdrlen
, swhead
,
1465 wil_tso_type_hdr
, NULL
, ring
, skb
,
1466 is_ipv4
, tcp_hdr_len
, skb_net_hdr_len
,
1471 /* Second descriptor contains the head */
1472 headlen
= skb_headlen(skb
) - hdrlen
;
1473 wil_dbg_txrx(wil
, "TSO: process skb head, headlen %u\n", headlen
);
1474 rc
= wil_tx_tso_gen_desc(wil
, skb
->data
+ hdrlen
, headlen
,
1475 (swhead
+ descs_used
) % ring
->size
,
1476 (nr_frags
!= 0) ? wil_tso_type_first
:
1477 wil_tso_type_lst
, NULL
, ring
, skb
,
1478 is_ipv4
, tcp_hdr_len
, skb_net_hdr_len
,
1483 /* Rest of the descriptors are from the SKB fragments */
1484 for (f
= 0; f
< nr_frags
; f
++) {
1485 skb_frag_t
*frag
= &skb_shinfo(skb
)->frags
[f
];
1486 int len
= skb_frag_size(frag
);
1488 wil_dbg_txrx(wil
, "TSO: frag[%d]: len %u, descs_used %d\n", f
,
1491 rc
= wil_tx_tso_gen_desc(wil
, NULL
, len
,
1492 (swhead
+ descs_used
) % ring
->size
,
1493 (f
!= nr_frags
- 1) ?
1494 wil_tso_type_mid
: wil_tso_type_lst
,
1495 frag
, ring
, skb
, is_ipv4
,
1496 tcp_hdr_len
, skb_net_hdr_len
,
1502 /* performance monitoring */
1503 used
= wil_ring_used_tx(ring
);
1504 if (wil_val_in_range(wil
->ring_idle_trsh
,
1505 used
, used
+ descs_used
)) {
1506 txdata
->idle
+= get_cycles() - txdata
->last_idle
;
1507 wil_dbg_txrx(wil
, "Ring[%2d] not idle %d -> %d\n",
1508 ring_index
, used
, used
+ descs_used
);
1511 /* advance swhead */
1512 wil_ring_advance_head(ring
, descs_used
);
1513 wil_dbg_txrx(wil
, "TSO: Tx swhead %d -> %d\n", swhead
, ring
->swhead
);
1515 /* make sure all writes to descriptors (shared memory) are done before
1516 * committing them to HW
1520 if (wil
->tx_latency
)
1521 *(ktime_t
*)&skb
->cb
= ktime_get();
1523 memset(skb
->cb
, 0, sizeof(ktime_t
));
1525 wil_w(wil
, ring
->hwtail
, ring
->swhead
);
1530 while (descs_used
> 0) {
1531 struct device
*dev
= wil_to_dev(wil
);
1532 struct wil_ctx
*ctx
;
1533 int i
= (swhead
+ descs_used
- 1) % ring
->size
;
1534 struct wil_tx_enhanced_desc dd
, *d
= &dd
;
1535 struct wil_tx_enhanced_desc
*_desc
=
1536 (struct wil_tx_enhanced_desc
*)
1537 &ring
->va
[i
].tx
.enhanced
;
1540 ctx
= &ring
->ctx
[i
];
1541 wil_tx_desc_unmap_edma(dev
, (union wil_tx_desc
*)d
, ctx
);
1542 memset(ctx
, 0, sizeof(*ctx
));
1548 static int wil_ring_init_bcast_edma(struct wil6210_vif
*vif
, int ring_id
,
1551 struct wil6210_priv
*wil
= vif_to_wil(vif
);
1552 struct wil_ring
*ring
= &wil
->ring_tx
[ring_id
];
1554 struct wil_ring_tx_data
*txdata
= &wil
->ring_tx_data
[ring_id
];
1556 wil_dbg_misc(wil
, "init bcast: ring_id=%d, sring_id=%d\n",
1557 ring_id
, wil
->tx_sring_idx
);
1559 lockdep_assert_held(&wil
->mutex
);
1561 wil_tx_data_init(txdata
);
1563 ring
->is_rx
= false;
1564 rc
= wil_ring_alloc_desc_ring(wil
, ring
);
1568 wil
->ring2cid_tid
[ring_id
][0] = WIL6210_MAX_CID
; /* CID */
1569 wil
->ring2cid_tid
[ring_id
][1] = 0; /* TID */
1571 txdata
->dot1x_open
= true;
1573 rc
= wil_wmi_bcast_desc_ring_add(vif
, ring_id
);
1580 spin_lock_bh(&txdata
->lock
);
1581 txdata
->enabled
= 0;
1582 txdata
->dot1x_open
= false;
1583 spin_unlock_bh(&txdata
->lock
);
1584 wil_ring_free_edma(wil
, ring
);
1590 static void wil_tx_fini_edma(struct wil6210_priv
*wil
)
1592 struct wil_status_ring
*sring
= &wil
->srings
[wil
->tx_sring_idx
];
1594 wil_dbg_misc(wil
, "free TX sring\n");
1596 wil_sring_free(wil
, sring
);
1599 static void wil_rx_data_free(struct wil_status_ring
*sring
)
1604 kfree_skb(sring
->rx_data
.skb
);
1605 sring
->rx_data
.skb
= NULL
;
1608 static void wil_rx_fini_edma(struct wil6210_priv
*wil
)
1610 struct wil_ring
*ring
= &wil
->ring_rx
;
1613 wil_dbg_misc(wil
, "rx_fini_edma\n");
1615 wil_ring_free_edma(wil
, ring
);
1617 for (i
= 0; i
< wil
->num_rx_status_rings
; i
++) {
1618 wil_rx_data_free(&wil
->srings
[i
]);
1619 wil_sring_free(wil
, &wil
->srings
[i
]);
1622 wil_free_rx_buff_arr(wil
);
1625 void wil_init_txrx_ops_edma(struct wil6210_priv
*wil
)
1627 wil
->txrx_ops
.configure_interrupt_moderation
=
1628 wil_configure_interrupt_moderation_edma
;
1630 wil
->txrx_ops
.ring_init_tx
= wil_ring_init_tx_edma
;
1631 wil
->txrx_ops
.ring_fini_tx
= wil_ring_free_edma
;
1632 wil
->txrx_ops
.ring_init_bcast
= wil_ring_init_bcast_edma
;
1633 wil
->txrx_ops
.tx_init
= wil_tx_init_edma
;
1634 wil
->txrx_ops
.tx_fini
= wil_tx_fini_edma
;
1635 wil
->txrx_ops
.tx_desc_map
= wil_tx_desc_map_edma
;
1636 wil
->txrx_ops
.tx_desc_unmap
= wil_tx_desc_unmap_edma
;
1637 wil
->txrx_ops
.tx_ring_tso
= __wil_tx_ring_tso_edma
;
1638 wil
->txrx_ops
.tx_ring_modify
= wil_tx_ring_modify_edma
;
1640 wil
->txrx_ops
.rx_init
= wil_rx_init_edma
;
1641 wil
->txrx_ops
.wmi_addba_rx_resp
= wmi_addba_rx_resp_edma
;
1642 wil
->txrx_ops
.get_reorder_params
= wil_get_reorder_params_edma
;
1643 wil
->txrx_ops
.get_netif_rx_params
= wil_get_netif_rx_params_edma
;
1644 wil
->txrx_ops
.rx_crypto_check
= wil_rx_crypto_check_edma
;
1645 wil
->txrx_ops
.rx_error_check
= wil_rx_error_check_edma
;
1646 wil
->txrx_ops
.is_rx_idle
= wil_is_rx_idle_edma
;
1647 wil
->txrx_ops
.rx_fini
= wil_rx_fini_edma
;