2 * Copyright (c) 2012-2017 Qualcomm Atheros, Inc.
4 * Permission to use, copy, modify, and/or distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 #include <linux/etherdevice.h>
18 #include <net/ieee80211_radiotap.h>
19 #include <linux/if_arp.h>
20 #include <linux/moduleparam.h>
22 #include <linux/ipv6.h>
24 #include <linux/prefetch.h>
31 static bool rtap_include_phy_info
;
32 module_param(rtap_include_phy_info
, bool, 0444);
33 MODULE_PARM_DESC(rtap_include_phy_info
,
34 " Include PHY info in the radiotap header, default - no");
37 module_param(rx_align_2
, bool, 0444);
38 MODULE_PARM_DESC(rx_align_2
, " align Rx buffers on 4*n+2, default - no");
41 module_param(rx_large_buf
, bool, 0444);
42 MODULE_PARM_DESC(rx_large_buf
, " allocate 8KB RX buffers, default - no");
44 static inline uint
wil_rx_snaplen(void)
46 return rx_align_2
? 6 : 0;
49 static inline int wil_vring_is_empty(struct vring
*vring
)
51 return vring
->swhead
== vring
->swtail
;
54 static inline u32
wil_vring_next_tail(struct vring
*vring
)
56 return (vring
->swtail
+ 1) % vring
->size
;
59 static inline void wil_vring_advance_head(struct vring
*vring
, int n
)
61 vring
->swhead
= (vring
->swhead
+ n
) % vring
->size
;
64 static inline int wil_vring_is_full(struct vring
*vring
)
66 return wil_vring_next_tail(vring
) == vring
->swhead
;
69 /* Used space in Tx Vring */
70 static inline int wil_vring_used_tx(struct vring
*vring
)
72 u32 swhead
= vring
->swhead
;
73 u32 swtail
= vring
->swtail
;
74 return (vring
->size
+ swhead
- swtail
) % vring
->size
;
77 /* Available space in Tx Vring */
78 static inline int wil_vring_avail_tx(struct vring
*vring
)
80 return vring
->size
- wil_vring_used_tx(vring
) - 1;
83 /* wil_vring_wmark_low - low watermark for available descriptor space */
84 static inline int wil_vring_wmark_low(struct vring
*vring
)
89 /* wil_vring_wmark_high - high watermark for available descriptor space */
90 static inline int wil_vring_wmark_high(struct vring
*vring
)
95 /* returns true if num avail descriptors is lower than wmark_low */
96 static inline int wil_vring_avail_low(struct vring
*vring
)
98 return wil_vring_avail_tx(vring
) < wil_vring_wmark_low(vring
);
101 /* returns true if num avail descriptors is higher than wmark_high */
102 static inline int wil_vring_avail_high(struct vring
*vring
)
104 return wil_vring_avail_tx(vring
) > wil_vring_wmark_high(vring
);
107 /* wil_val_in_range - check if value in [min,max) */
108 static inline bool wil_val_in_range(int val
, int min
, int max
)
110 return val
>= min
&& val
< max
;
113 static int wil_vring_alloc(struct wil6210_priv
*wil
, struct vring
*vring
)
115 struct device
*dev
= wil_to_dev(wil
);
116 size_t sz
= vring
->size
* sizeof(vring
->va
[0]);
119 wil_dbg_misc(wil
, "vring_alloc:\n");
121 BUILD_BUG_ON(sizeof(vring
->va
[0]) != 32);
125 vring
->ctx
= kcalloc(vring
->size
, sizeof(vring
->ctx
[0]), GFP_KERNEL
);
131 /* vring->va should be aligned on its size rounded up to power of 2
132 * This is granted by the dma_alloc_coherent.
134 * HW has limitation that all vrings addresses must share the same
135 * upper 16 msb bits part of 48 bits address. To workaround that,
136 * if we are using 48 bit addresses switch to 32 bit allocation
137 * before allocating vring memory.
139 * There's no check for the return value of dma_set_mask_and_coherent,
140 * since we assume if we were able to set the mask during
141 * initialization in this system it will not fail if we set it again
143 if (wil
->use_extended_dma_addr
)
144 dma_set_mask_and_coherent(dev
, DMA_BIT_MASK(32));
146 vring
->va
= dma_alloc_coherent(dev
, sz
, &vring
->pa
, GFP_KERNEL
);
153 if (wil
->use_extended_dma_addr
)
154 dma_set_mask_and_coherent(dev
, DMA_BIT_MASK(48));
156 /* initially, all descriptors are SW owned
157 * For Tx and Rx, ownership bit is at the same location, thus
160 for (i
= 0; i
< vring
->size
; i
++) {
161 volatile struct vring_tx_desc
*_d
= &vring
->va
[i
].tx
;
163 _d
->dma
.status
= TX_DMA_STATUS_DU
;
166 wil_dbg_misc(wil
, "vring[%d] 0x%p:%pad 0x%p\n", vring
->size
,
167 vring
->va
, &vring
->pa
, vring
->ctx
);
172 static void wil_txdesc_unmap(struct device
*dev
, struct vring_tx_desc
*d
,
175 dma_addr_t pa
= wil_desc_addr(&d
->dma
.addr
);
176 u16 dmalen
= le16_to_cpu(d
->dma
.length
);
178 switch (ctx
->mapped_as
) {
179 case wil_mapped_as_single
:
180 dma_unmap_single(dev
, pa
, dmalen
, DMA_TO_DEVICE
);
182 case wil_mapped_as_page
:
183 dma_unmap_page(dev
, pa
, dmalen
, DMA_TO_DEVICE
);
190 static void wil_vring_free(struct wil6210_priv
*wil
, struct vring
*vring
,
193 struct device
*dev
= wil_to_dev(wil
);
194 size_t sz
= vring
->size
* sizeof(vring
->va
[0]);
196 lockdep_assert_held(&wil
->mutex
);
198 int vring_index
= vring
- wil
->vring_tx
;
200 wil_dbg_misc(wil
, "free Tx vring %d [%d] 0x%p:%pad 0x%p\n",
201 vring_index
, vring
->size
, vring
->va
,
202 &vring
->pa
, vring
->ctx
);
204 wil_dbg_misc(wil
, "free Rx vring [%d] 0x%p:%pad 0x%p\n",
205 vring
->size
, vring
->va
,
206 &vring
->pa
, vring
->ctx
);
209 while (!wil_vring_is_empty(vring
)) {
215 struct vring_tx_desc dd
, *d
= &dd
;
216 volatile struct vring_tx_desc
*_d
=
217 &vring
->va
[vring
->swtail
].tx
;
219 ctx
= &vring
->ctx
[vring
->swtail
];
222 "ctx(%d) was already completed\n",
224 vring
->swtail
= wil_vring_next_tail(vring
);
228 wil_txdesc_unmap(dev
, d
, ctx
);
230 dev_kfree_skb_any(ctx
->skb
);
231 vring
->swtail
= wil_vring_next_tail(vring
);
233 struct vring_rx_desc dd
, *d
= &dd
;
234 volatile struct vring_rx_desc
*_d
=
235 &vring
->va
[vring
->swhead
].rx
;
237 ctx
= &vring
->ctx
[vring
->swhead
];
239 pa
= wil_desc_addr(&d
->dma
.addr
);
240 dmalen
= le16_to_cpu(d
->dma
.length
);
241 dma_unmap_single(dev
, pa
, dmalen
, DMA_FROM_DEVICE
);
243 wil_vring_advance_head(vring
, 1);
246 dma_free_coherent(dev
, sz
, (void *)vring
->va
, vring
->pa
);
254 * Allocate one skb for Rx VRING
256 * Safe to call from IRQ
258 static int wil_vring_alloc_skb(struct wil6210_priv
*wil
, struct vring
*vring
,
261 struct device
*dev
= wil_to_dev(wil
);
262 unsigned int sz
= wil
->rx_buf_len
+ ETH_HLEN
+ wil_rx_snaplen();
263 struct vring_rx_desc dd
, *d
= &dd
;
264 volatile struct vring_rx_desc
*_d
= &vring
->va
[i
].rx
;
266 struct sk_buff
*skb
= dev_alloc_skb(sz
+ headroom
);
271 skb_reserve(skb
, headroom
);
274 pa
= dma_map_single(dev
, skb
->data
, skb
->len
, DMA_FROM_DEVICE
);
275 if (unlikely(dma_mapping_error(dev
, pa
))) {
280 d
->dma
.d0
= RX_DMA_D0_CMD_DMA_RT
| RX_DMA_D0_CMD_DMA_IT
;
281 wil_desc_addr_set(&d
->dma
.addr
, pa
);
282 /* ip_length don't care */
284 /* error don't care */
285 d
->dma
.status
= 0; /* BIT(0) should be 0 for HW_OWNED */
286 d
->dma
.length
= cpu_to_le16(sz
);
288 vring
->ctx
[i
].skb
= skb
;
294 * Adds radiotap header
296 * Any error indicated as "Bad FCS"
298 * Vendor data for 04:ce:14-1 (Wilocity-1) consists of:
299 * - Rx descriptor: 32 bytes
302 static void wil_rx_add_radiotap_header(struct wil6210_priv
*wil
,
305 struct wireless_dev
*wdev
= wil
->wdev
;
306 struct wil6210_rtap
{
307 struct ieee80211_radiotap_header rthdr
;
308 /* fields should be in the order of bits in rthdr.it_present */
312 __le16 chnl_freq
__aligned(2);
319 struct wil6210_rtap_vendor
{
320 struct wil6210_rtap rtap
;
322 u8 vendor_oui
[3] __aligned(2);
327 struct vring_rx_desc
*d
= wil_skb_rxdesc(skb
);
328 struct wil6210_rtap_vendor
*rtap_vendor
;
329 int rtap_len
= sizeof(struct wil6210_rtap
);
330 int phy_length
= 0; /* phy info header size, bytes */
331 static char phy_data
[128];
332 struct ieee80211_channel
*ch
= wdev
->preset_chandef
.chan
;
334 if (rtap_include_phy_info
) {
335 rtap_len
= sizeof(*rtap_vendor
) + sizeof(*d
);
336 /* calculate additional length */
337 if (d
->dma
.status
& RX_DMA_STATUS_PHY_INFO
) {
339 * PHY info starts from 8-byte boundary
340 * there are 8-byte lines, last line may be partially
341 * written (HW bug), thus FW configures for last line
342 * to be excessive. Driver skips this last line.
344 int len
= min_t(int, 8 + sizeof(phy_data
),
345 wil_rxdesc_phy_length(d
));
348 void *p
= skb_tail_pointer(skb
);
349 void *pa
= PTR_ALIGN(p
, 8);
351 if (skb_tailroom(skb
) >= len
+ (pa
- p
)) {
352 phy_length
= len
- 8;
353 memcpy(phy_data
, pa
, phy_length
);
357 rtap_len
+= phy_length
;
360 if (skb_headroom(skb
) < rtap_len
&&
361 pskb_expand_head(skb
, rtap_len
, 0, GFP_ATOMIC
)) {
362 wil_err(wil
, "Unable to expand headroom to %d\n", rtap_len
);
366 rtap_vendor
= skb_push(skb
, rtap_len
);
367 memset(rtap_vendor
, 0, rtap_len
);
369 rtap_vendor
->rtap
.rthdr
.it_version
= PKTHDR_RADIOTAP_VERSION
;
370 rtap_vendor
->rtap
.rthdr
.it_len
= cpu_to_le16(rtap_len
);
371 rtap_vendor
->rtap
.rthdr
.it_present
= cpu_to_le32(
372 (1 << IEEE80211_RADIOTAP_FLAGS
) |
373 (1 << IEEE80211_RADIOTAP_CHANNEL
) |
374 (1 << IEEE80211_RADIOTAP_MCS
));
375 if (d
->dma
.status
& RX_DMA_STATUS_ERROR
)
376 rtap_vendor
->rtap
.flags
|= IEEE80211_RADIOTAP_F_BADFCS
;
378 rtap_vendor
->rtap
.chnl_freq
= cpu_to_le16(ch
? ch
->center_freq
: 58320);
379 rtap_vendor
->rtap
.chnl_flags
= cpu_to_le16(0);
381 rtap_vendor
->rtap
.mcs_present
= IEEE80211_RADIOTAP_MCS_HAVE_MCS
;
382 rtap_vendor
->rtap
.mcs_flags
= 0;
383 rtap_vendor
->rtap
.mcs_index
= wil_rxdesc_mcs(d
);
385 if (rtap_include_phy_info
) {
386 rtap_vendor
->rtap
.rthdr
.it_present
|= cpu_to_le32(1 <<
387 IEEE80211_RADIOTAP_VENDOR_NAMESPACE
);
388 /* OUI for Wilocity 04:ce:14 */
389 rtap_vendor
->vendor_oui
[0] = 0x04;
390 rtap_vendor
->vendor_oui
[1] = 0xce;
391 rtap_vendor
->vendor_oui
[2] = 0x14;
392 rtap_vendor
->vendor_ns
= 1;
393 /* Rx descriptor + PHY data */
394 rtap_vendor
->vendor_skip
= cpu_to_le16(sizeof(*d
) +
396 memcpy(rtap_vendor
->vendor_data
, (void *)d
, sizeof(*d
));
397 memcpy(rtap_vendor
->vendor_data
+ sizeof(*d
), phy_data
,
402 /* similar to ieee80211_ version, but FC contain only 1-st byte */
403 static inline int wil_is_back_req(u8 fc
)
405 return (fc
& (IEEE80211_FCTL_FTYPE
| IEEE80211_FCTL_STYPE
)) ==
406 (IEEE80211_FTYPE_CTL
| IEEE80211_STYPE_BACK_REQ
);
410 * reap 1 frame from @swhead
412 * Rx descriptor copied to skb->cb
414 * Safe to call from IRQ
416 static struct sk_buff
*wil_vring_reap_rx(struct wil6210_priv
*wil
,
419 struct device
*dev
= wil_to_dev(wil
);
420 struct net_device
*ndev
= wil_to_ndev(wil
);
421 volatile struct vring_rx_desc
*_d
;
422 struct vring_rx_desc
*d
;
425 unsigned int snaplen
= wil_rx_snaplen();
426 unsigned int sz
= wil
->rx_buf_len
+ ETH_HLEN
+ snaplen
;
431 struct wil_net_stats
*stats
;
433 BUILD_BUG_ON(sizeof(struct vring_rx_desc
) > sizeof(skb
->cb
));
436 if (unlikely(wil_vring_is_empty(vring
)))
439 i
= (int)vring
->swhead
;
440 _d
= &vring
->va
[i
].rx
;
441 if (unlikely(!(_d
->dma
.status
& RX_DMA_STATUS_DU
))) {
442 /* it is not error, we just reached end of Rx done area */
446 skb
= vring
->ctx
[i
].skb
;
447 vring
->ctx
[i
].skb
= NULL
;
448 wil_vring_advance_head(vring
, 1);
450 wil_err(wil
, "No Rx skb at [%d]\n", i
);
453 d
= wil_skb_rxdesc(skb
);
455 pa
= wil_desc_addr(&d
->dma
.addr
);
457 dma_unmap_single(dev
, pa
, sz
, DMA_FROM_DEVICE
);
458 dmalen
= le16_to_cpu(d
->dma
.length
);
460 trace_wil6210_rx(i
, d
);
461 wil_dbg_txrx(wil
, "Rx[%3d] : %d bytes\n", i
, dmalen
);
462 wil_hex_dump_txrx("RxD ", DUMP_PREFIX_NONE
, 32, 4,
463 (const void *)d
, sizeof(*d
), false);
465 cid
= wil_rxdesc_cid(d
);
466 stats
= &wil
->sta
[cid
].stats
;
468 if (unlikely(dmalen
> sz
)) {
469 wil_err(wil
, "Rx size too large: %d bytes!\n", dmalen
);
470 stats
->rx_large_frame
++;
474 skb_trim(skb
, dmalen
);
478 wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET
, 16, 1,
479 skb
->data
, skb_headlen(skb
), false);
481 stats
->last_mcs_rx
= wil_rxdesc_mcs(d
);
482 if (stats
->last_mcs_rx
< ARRAY_SIZE(stats
->rx_per_mcs
))
483 stats
->rx_per_mcs
[stats
->last_mcs_rx
]++;
485 /* use radiotap header only if required */
486 if (ndev
->type
== ARPHRD_IEEE80211_RADIOTAP
)
487 wil_rx_add_radiotap_header(wil
, skb
);
489 /* no extra checks if in sniffer mode */
490 if (ndev
->type
!= ARPHRD_ETHER
)
492 /* Non-data frames may be delivered through Rx DMA channel (ex: BAR)
493 * Driver should recognize it by frame type, that is found
494 * in Rx descriptor. If type is not data, it is 802.11 frame as is
496 ftype
= wil_rxdesc_ftype(d
) << 2;
497 if (unlikely(ftype
!= IEEE80211_FTYPE_DATA
)) {
498 u8 fc1
= wil_rxdesc_fc1(d
);
499 int mid
= wil_rxdesc_mid(d
);
500 int tid
= wil_rxdesc_tid(d
);
501 u16 seq
= wil_rxdesc_seq(d
);
504 "Non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n",
505 fc1
, mid
, cid
, tid
, seq
);
506 stats
->rx_non_data_frame
++;
507 if (wil_is_back_req(fc1
)) {
509 "BAR: MID %d CID %d TID %d Seq 0x%03x\n",
511 wil_rx_bar(wil
, cid
, tid
, seq
);
513 /* print again all info. One can enable only this
514 * without overhead for printing every Rx frame
517 "Unhandled non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n",
518 fc1
, mid
, cid
, tid
, seq
);
519 wil_hex_dump_txrx("RxD ", DUMP_PREFIX_NONE
, 32, 4,
520 (const void *)d
, sizeof(*d
), false);
521 wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET
, 16, 1,
522 skb
->data
, skb_headlen(skb
), false);
528 if (unlikely(skb
->len
< ETH_HLEN
+ snaplen
)) {
529 wil_err(wil
, "Short frame, len = %d\n", skb
->len
);
530 stats
->rx_short_frame
++;
535 /* L4 IDENT is on when HW calculated checksum, check status
536 * and in case of error drop the packet
537 * higher stack layers will handle retransmission (if required)
539 if (likely(d
->dma
.status
& RX_DMA_STATUS_L4I
)) {
540 /* L4 protocol identified, csum calculated */
541 if (likely((d
->dma
.error
& RX_DMA_ERROR_L4_ERR
) == 0))
542 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
543 /* If HW reports bad checksum, let IP stack re-check it
544 * For example, HW don't understand Microsoft IP stack that
545 * mis-calculates TCP checksum - if it should be 0x0,
546 * it writes 0xffff in violation of RFC 1624
552 * +-------+-------+---------+------------+------+
553 * | SA(6) | DA(6) | SNAP(6) | ETHTYPE(2) | DATA |
554 * +-------+-------+---------+------------+------+
555 * Need to remove SNAP, shifting SA and DA forward
557 memmove(skb
->data
+ snaplen
, skb
->data
, 2 * ETH_ALEN
);
558 skb_pull(skb
, snaplen
);
565 * allocate and fill up to @count buffers in rx ring
566 * buffers posted at @swtail
568 static int wil_rx_refill(struct wil6210_priv
*wil
, int count
)
570 struct net_device
*ndev
= wil_to_ndev(wil
);
571 struct vring
*v
= &wil
->vring_rx
;
574 int headroom
= ndev
->type
== ARPHRD_IEEE80211_RADIOTAP
?
575 WIL6210_RTAP_SIZE
: 0;
577 for (; next_tail
= wil_vring_next_tail(v
),
578 (next_tail
!= v
->swhead
) && (count
-- > 0);
579 v
->swtail
= next_tail
) {
580 rc
= wil_vring_alloc_skb(wil
, v
, v
->swtail
, headroom
);
582 wil_err(wil
, "Error %d in wil_rx_refill[%d]\n",
588 /* make sure all writes to descriptors (shared memory) are done before
589 * committing them to HW
593 wil_w(wil
, v
->hwtail
, v
->swtail
);
599 * reverse_memcmp - Compare two areas of memory, in reverse order
600 * @cs: One area of memory
601 * @ct: Another area of memory
602 * @count: The size of the area.
604 * Cut'n'paste from original memcmp (see lib/string.c)
605 * with minimal modifications
607 static int reverse_memcmp(const void *cs
, const void *ct
, size_t count
)
609 const unsigned char *su1
, *su2
;
612 for (su1
= cs
+ count
- 1, su2
= ct
+ count
- 1; count
> 0;
613 --su1
, --su2
, count
--) {
621 static int wil_rx_crypto_check(struct wil6210_priv
*wil
, struct sk_buff
*skb
)
623 struct vring_rx_desc
*d
= wil_skb_rxdesc(skb
);
624 int cid
= wil_rxdesc_cid(d
);
625 int tid
= wil_rxdesc_tid(d
);
626 int key_id
= wil_rxdesc_key_id(d
);
627 int mc
= wil_rxdesc_mcast(d
);
628 struct wil_sta_info
*s
= &wil
->sta
[cid
];
629 struct wil_tid_crypto_rx
*c
= mc
? &s
->group_crypto_rx
:
630 &s
->tid_crypto_rx
[tid
];
631 struct wil_tid_crypto_rx_single
*cc
= &c
->key_id
[key_id
];
632 const u8
*pn
= (u8
*)&d
->mac
.pn_15_0
;
635 wil_err_ratelimited(wil
,
636 "Key missing. CID %d TID %d MCast %d KEY_ID %d\n",
637 cid
, tid
, mc
, key_id
);
641 if (reverse_memcmp(pn
, cc
->pn
, IEEE80211_GCMP_PN_LEN
) <= 0) {
642 wil_err_ratelimited(wil
,
643 "Replay attack. CID %d TID %d MCast %d KEY_ID %d PN %6phN last %6phN\n",
644 cid
, tid
, mc
, key_id
, pn
, cc
->pn
);
647 memcpy(cc
->pn
, pn
, IEEE80211_GCMP_PN_LEN
);
653 * Pass Rx packet to the netif. Update statistics.
654 * Called in softirq context (NAPI poll).
656 void wil_netif_rx_any(struct sk_buff
*skb
, struct net_device
*ndev
)
658 gro_result_t rc
= GRO_NORMAL
;
659 struct wil6210_priv
*wil
= ndev_to_wil(ndev
);
660 struct wireless_dev
*wdev
= wil_to_wdev(wil
);
661 unsigned int len
= skb
->len
;
662 struct vring_rx_desc
*d
= wil_skb_rxdesc(skb
);
663 int cid
= wil_rxdesc_cid(d
); /* always 0..7, no need to check */
664 int security
= wil_rxdesc_security(d
);
665 struct ethhdr
*eth
= (void *)skb
->data
;
666 /* here looking for DA, not A1, thus Rxdesc's 'mcast' indication
667 * is not suitable, need to look at data
669 int mcast
= is_multicast_ether_addr(eth
->h_dest
);
670 struct wil_net_stats
*stats
= &wil
->sta
[cid
].stats
;
671 struct sk_buff
*xmit_skb
= NULL
;
672 static const char * const gro_res_str
[] = {
673 [GRO_MERGED
] = "GRO_MERGED",
674 [GRO_MERGED_FREE
] = "GRO_MERGED_FREE",
675 [GRO_HELD
] = "GRO_HELD",
676 [GRO_NORMAL
] = "GRO_NORMAL",
677 [GRO_DROP
] = "GRO_DROP",
680 if (ndev
->features
& NETIF_F_RXHASH
)
681 /* fake L4 to ensure it won't be re-calculated later
682 * set hash to any non-zero value to activate rps
683 * mechanism, core will be chosen according
684 * to user-level rps configuration.
686 skb_set_hash(skb
, 1, PKT_HASH_TYPE_L4
);
690 if (security
&& (wil_rx_crypto_check(wil
, skb
) != 0)) {
697 if (wdev
->iftype
== NL80211_IFTYPE_AP
&& !wil
->ap_isolate
) {
699 /* send multicast frames both to higher layers in
700 * local net stack and back to the wireless medium
702 xmit_skb
= skb_copy(skb
, GFP_ATOMIC
);
704 int xmit_cid
= wil_find_cid(wil
, eth
->h_dest
);
707 /* The destination station is associated to
708 * this AP (in this VLAN), so send the frame
709 * directly to it and do not pass it to local
718 /* Send to wireless media and increase priority by 256 to
719 * keep the received priority instead of reclassifying
720 * the frame (see cfg80211_classify8021d).
722 xmit_skb
->dev
= ndev
;
723 xmit_skb
->priority
+= 256;
724 xmit_skb
->protocol
= htons(ETH_P_802_3
);
725 skb_reset_network_header(xmit_skb
);
726 skb_reset_mac_header(xmit_skb
);
727 wil_dbg_txrx(wil
, "Rx -> Tx %d bytes\n", len
);
728 dev_queue_xmit(xmit_skb
);
731 if (skb
) { /* deliver to local stack */
733 skb
->protocol
= eth_type_trans(skb
, ndev
);
734 rc
= napi_gro_receive(&wil
->napi_rx
, skb
);
735 wil_dbg_txrx(wil
, "Rx complete %d bytes => %s\n",
736 len
, gro_res_str
[rc
]);
739 /* statistics. rc set to GRO_NORMAL for AP bridging */
740 if (unlikely(rc
== GRO_DROP
)) {
741 ndev
->stats
.rx_dropped
++;
743 wil_dbg_txrx(wil
, "Rx drop %d bytes\n", len
);
745 ndev
->stats
.rx_packets
++;
747 ndev
->stats
.rx_bytes
+= len
;
748 stats
->rx_bytes
+= len
;
750 ndev
->stats
.multicast
++;
755 * Proceed all completed skb's from Rx VRING
757 * Safe to call from NAPI poll, i.e. softirq with interrupts enabled
759 void wil_rx_handle(struct wil6210_priv
*wil
, int *quota
)
761 struct net_device
*ndev
= wil_to_ndev(wil
);
762 struct vring
*v
= &wil
->vring_rx
;
765 if (unlikely(!v
->va
)) {
766 wil_err(wil
, "Rx IRQ while Rx not yet initialized\n");
769 wil_dbg_txrx(wil
, "rx_handle\n");
770 while ((*quota
> 0) && (NULL
!= (skb
= wil_vring_reap_rx(wil
, v
)))) {
773 if (wil
->wdev
->iftype
== NL80211_IFTYPE_MONITOR
) {
775 skb_reset_mac_header(skb
);
776 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
777 skb
->pkt_type
= PACKET_OTHERHOST
;
778 skb
->protocol
= htons(ETH_P_802_2
);
779 wil_netif_rx_any(skb
, ndev
);
781 wil_rx_reorder(wil
, skb
);
784 wil_rx_refill(wil
, v
->size
);
787 static void wil_rx_buf_len_init(struct wil6210_priv
*wil
)
789 wil
->rx_buf_len
= rx_large_buf
?
790 WIL_MAX_ETH_MTU
: TXRX_BUF_LEN_DEFAULT
- WIL_MAX_MPDU_OVERHEAD
;
791 if (mtu_max
> wil
->rx_buf_len
) {
792 /* do not allow RX buffers to be smaller than mtu_max, for
793 * backward compatibility (mtu_max parameter was also used
794 * to support receiving large packets)
796 wil_info(wil
, "Override RX buffer to mtu_max(%d)\n", mtu_max
);
797 wil
->rx_buf_len
= mtu_max
;
801 int wil_rx_init(struct wil6210_priv
*wil
, u16 size
)
803 struct vring
*vring
= &wil
->vring_rx
;
806 wil_dbg_misc(wil
, "rx_init\n");
809 wil_err(wil
, "Rx ring already allocated\n");
813 wil_rx_buf_len_init(wil
);
816 rc
= wil_vring_alloc(wil
, vring
);
820 rc
= wmi_rx_chain_add(wil
, vring
);
824 rc
= wil_rx_refill(wil
, vring
->size
);
830 wil_vring_free(wil
, vring
, 0);
835 void wil_rx_fini(struct wil6210_priv
*wil
)
837 struct vring
*vring
= &wil
->vring_rx
;
839 wil_dbg_misc(wil
, "rx_fini\n");
842 wil_vring_free(wil
, vring
, 0);
845 static inline void wil_tx_data_init(struct vring_tx_data
*txdata
)
847 spin_lock_bh(&txdata
->lock
);
848 txdata
->dot1x_open
= 0;
851 txdata
->last_idle
= 0;
853 txdata
->agg_wsize
= 0;
854 txdata
->agg_timeout
= 0;
855 txdata
->agg_amsdu
= 0;
856 txdata
->addba_in_progress
= false;
857 spin_unlock_bh(&txdata
->lock
);
860 int wil_vring_init_tx(struct wil6210_priv
*wil
, int id
, int size
,
864 struct wmi_vring_cfg_cmd cmd
= {
865 .action
= cpu_to_le32(WMI_VRING_CMD_ADD
),
869 cpu_to_le16(wil_mtu2macbuf(mtu_max
)),
870 .ring_size
= cpu_to_le16(size
),
873 .cidxtid
= mk_cidxtid(cid
, tid
),
874 .encap_trans_type
= WMI_VRING_ENC_TYPE_802_3
,
879 .priority
= cpu_to_le16(0),
880 .timeslot_us
= cpu_to_le16(0xfff),
885 struct wmi_cmd_hdr wmi
;
886 struct wmi_vring_cfg_done_event cmd
;
888 struct vring
*vring
= &wil
->vring_tx
[id
];
889 struct vring_tx_data
*txdata
= &wil
->vring_tx_data
[id
];
891 wil_dbg_misc(wil
, "vring_init_tx: max_mpdu_size %d\n",
892 cmd
.vring_cfg
.tx_sw_ring
.max_mpdu_size
);
893 lockdep_assert_held(&wil
->mutex
);
896 wil_err(wil
, "Tx ring [%d] already allocated\n", id
);
901 wil_tx_data_init(txdata
);
903 rc
= wil_vring_alloc(wil
, vring
);
907 wil
->vring2cid_tid
[id
][0] = cid
;
908 wil
->vring2cid_tid
[id
][1] = tid
;
910 cmd
.vring_cfg
.tx_sw_ring
.ring_mem_base
= cpu_to_le64(vring
->pa
);
913 txdata
->dot1x_open
= true;
914 rc
= wmi_call(wil
, WMI_VRING_CFG_CMDID
, &cmd
, sizeof(cmd
),
915 WMI_VRING_CFG_DONE_EVENTID
, &reply
, sizeof(reply
), 100);
919 if (reply
.cmd
.status
!= WMI_FW_STATUS_SUCCESS
) {
920 wil_err(wil
, "Tx config failed, status 0x%02x\n",
926 spin_lock_bh(&txdata
->lock
);
927 vring
->hwtail
= le32_to_cpu(reply
.cmd
.tx_vring_tail_ptr
);
929 spin_unlock_bh(&txdata
->lock
);
931 if (txdata
->dot1x_open
&& (agg_wsize
>= 0))
932 wil_addba_tx_request(wil
, id
, agg_wsize
);
936 spin_lock_bh(&txdata
->lock
);
937 txdata
->dot1x_open
= false;
939 spin_unlock_bh(&txdata
->lock
);
940 wil_vring_free(wil
, vring
, 1);
941 wil
->vring2cid_tid
[id
][0] = WIL6210_MAX_CID
;
942 wil
->vring2cid_tid
[id
][1] = 0;
949 int wil_vring_init_bcast(struct wil6210_priv
*wil
, int id
, int size
)
952 struct wmi_bcast_vring_cfg_cmd cmd
= {
953 .action
= cpu_to_le32(WMI_VRING_CMD_ADD
),
957 cpu_to_le16(wil_mtu2macbuf(mtu_max
)),
958 .ring_size
= cpu_to_le16(size
),
961 .encap_trans_type
= WMI_VRING_ENC_TYPE_802_3
,
965 struct wmi_cmd_hdr wmi
;
966 struct wmi_vring_cfg_done_event cmd
;
968 struct vring
*vring
= &wil
->vring_tx
[id
];
969 struct vring_tx_data
*txdata
= &wil
->vring_tx_data
[id
];
971 wil_dbg_misc(wil
, "vring_init_bcast: max_mpdu_size %d\n",
972 cmd
.vring_cfg
.tx_sw_ring
.max_mpdu_size
);
973 lockdep_assert_held(&wil
->mutex
);
976 wil_err(wil
, "Tx ring [%d] already allocated\n", id
);
981 wil_tx_data_init(txdata
);
983 rc
= wil_vring_alloc(wil
, vring
);
987 wil
->vring2cid_tid
[id
][0] = WIL6210_MAX_CID
; /* CID */
988 wil
->vring2cid_tid
[id
][1] = 0; /* TID */
990 cmd
.vring_cfg
.tx_sw_ring
.ring_mem_base
= cpu_to_le64(vring
->pa
);
993 txdata
->dot1x_open
= true;
994 rc
= wmi_call(wil
, WMI_BCAST_VRING_CFG_CMDID
, &cmd
, sizeof(cmd
),
995 WMI_VRING_CFG_DONE_EVENTID
, &reply
, sizeof(reply
), 100);
999 if (reply
.cmd
.status
!= WMI_FW_STATUS_SUCCESS
) {
1000 wil_err(wil
, "Tx config failed, status 0x%02x\n",
1006 spin_lock_bh(&txdata
->lock
);
1007 vring
->hwtail
= le32_to_cpu(reply
.cmd
.tx_vring_tail_ptr
);
1008 txdata
->enabled
= 1;
1009 spin_unlock_bh(&txdata
->lock
);
1013 spin_lock_bh(&txdata
->lock
);
1014 txdata
->enabled
= 0;
1015 txdata
->dot1x_open
= false;
1016 spin_unlock_bh(&txdata
->lock
);
1017 wil_vring_free(wil
, vring
, 1);
1023 void wil_vring_fini_tx(struct wil6210_priv
*wil
, int id
)
1025 struct vring
*vring
= &wil
->vring_tx
[id
];
1026 struct vring_tx_data
*txdata
= &wil
->vring_tx_data
[id
];
1028 lockdep_assert_held(&wil
->mutex
);
1033 wil_dbg_misc(wil
, "vring_fini_tx: id=%d\n", id
);
1035 spin_lock_bh(&txdata
->lock
);
1036 txdata
->dot1x_open
= false;
1037 txdata
->enabled
= 0; /* no Tx can be in progress or start anew */
1038 spin_unlock_bh(&txdata
->lock
);
1039 /* napi_synchronize waits for completion of the current NAPI but will
1040 * not prevent the next NAPI run.
1041 * Add a memory barrier to guarantee that txdata->enabled is zeroed
1042 * before napi_synchronize so that the next scheduled NAPI will not
1046 /* make sure NAPI won't touch this vring */
1047 if (test_bit(wil_status_napi_en
, wil
->status
))
1048 napi_synchronize(&wil
->napi_tx
);
1050 wil_vring_free(wil
, vring
, 1);
1053 static struct vring
*wil_find_tx_ucast(struct wil6210_priv
*wil
,
1054 struct sk_buff
*skb
)
1057 struct ethhdr
*eth
= (void *)skb
->data
;
1058 int cid
= wil_find_cid(wil
, eth
->h_dest
);
1063 /* TODO: fix for multiple TID */
1064 for (i
= 0; i
< ARRAY_SIZE(wil
->vring2cid_tid
); i
++) {
1065 if (!wil
->vring_tx_data
[i
].dot1x_open
&&
1066 (skb
->protocol
!= cpu_to_be16(ETH_P_PAE
)))
1068 if (wil
->vring2cid_tid
[i
][0] == cid
) {
1069 struct vring
*v
= &wil
->vring_tx
[i
];
1070 struct vring_tx_data
*txdata
= &wil
->vring_tx_data
[i
];
1072 wil_dbg_txrx(wil
, "find_tx_ucast: (%pM) -> [%d]\n",
1074 if (v
->va
&& txdata
->enabled
) {
1078 "find_tx_ucast: vring[%d] not valid\n",
1088 static int wil_tx_vring(struct wil6210_priv
*wil
, struct vring
*vring
,
1089 struct sk_buff
*skb
);
1091 static struct vring
*wil_find_tx_vring_sta(struct wil6210_priv
*wil
,
1092 struct sk_buff
*skb
)
1097 struct vring_tx_data
*txdata
;
1099 /* In the STA mode, it is expected to have only 1 VRING
1100 * for the AP we connected to.
1101 * find 1-st vring eligible for this skb and use it.
1103 for (i
= 0; i
< WIL6210_MAX_TX_RINGS
; i
++) {
1104 v
= &wil
->vring_tx
[i
];
1105 txdata
= &wil
->vring_tx_data
[i
];
1106 if (!v
->va
|| !txdata
->enabled
)
1109 cid
= wil
->vring2cid_tid
[i
][0];
1110 if (cid
>= WIL6210_MAX_CID
) /* skip BCAST */
1113 if (!wil
->vring_tx_data
[i
].dot1x_open
&&
1114 (skb
->protocol
!= cpu_to_be16(ETH_P_PAE
)))
1117 wil_dbg_txrx(wil
, "Tx -> ring %d\n", i
);
1122 wil_dbg_txrx(wil
, "Tx while no vrings active?\n");
1127 /* Use one of 2 strategies:
1129 * 1. New (real broadcast):
1130 * use dedicated broadcast vring
1131 * 2. Old (pseudo-DMS):
1132 * Find 1-st vring and return it;
1133 * duplicate skb and send it to other active vrings;
1134 * in all cases override dest address to unicast peer's address
1135 * Use old strategy when new is not supported yet:
1138 static struct vring
*wil_find_tx_bcast_1(struct wil6210_priv
*wil
,
1139 struct sk_buff
*skb
)
1142 struct vring_tx_data
*txdata
;
1143 int i
= wil
->bcast_vring
;
1147 v
= &wil
->vring_tx
[i
];
1148 txdata
= &wil
->vring_tx_data
[i
];
1149 if (!v
->va
|| !txdata
->enabled
)
1151 if (!wil
->vring_tx_data
[i
].dot1x_open
&&
1152 (skb
->protocol
!= cpu_to_be16(ETH_P_PAE
)))
1158 static void wil_set_da_for_vring(struct wil6210_priv
*wil
,
1159 struct sk_buff
*skb
, int vring_index
)
1161 struct ethhdr
*eth
= (void *)skb
->data
;
1162 int cid
= wil
->vring2cid_tid
[vring_index
][0];
1164 ether_addr_copy(eth
->h_dest
, wil
->sta
[cid
].addr
);
1167 static struct vring
*wil_find_tx_bcast_2(struct wil6210_priv
*wil
,
1168 struct sk_buff
*skb
)
1170 struct vring
*v
, *v2
;
1171 struct sk_buff
*skb2
;
1174 struct ethhdr
*eth
= (void *)skb
->data
;
1175 char *src
= eth
->h_source
;
1176 struct vring_tx_data
*txdata
;
1178 /* find 1-st vring eligible for data */
1179 for (i
= 0; i
< WIL6210_MAX_TX_RINGS
; i
++) {
1180 v
= &wil
->vring_tx
[i
];
1181 txdata
= &wil
->vring_tx_data
[i
];
1182 if (!v
->va
|| !txdata
->enabled
)
1185 cid
= wil
->vring2cid_tid
[i
][0];
1186 if (cid
>= WIL6210_MAX_CID
) /* skip BCAST */
1188 if (!wil
->vring_tx_data
[i
].dot1x_open
&&
1189 (skb
->protocol
!= cpu_to_be16(ETH_P_PAE
)))
1192 /* don't Tx back to source when re-routing Rx->Tx at the AP */
1193 if (0 == memcmp(wil
->sta
[cid
].addr
, src
, ETH_ALEN
))
1199 wil_dbg_txrx(wil
, "Tx while no vrings active?\n");
1204 wil_dbg_txrx(wil
, "BCAST -> ring %d\n", i
);
1205 wil_set_da_for_vring(wil
, skb
, i
);
1207 /* find other active vrings and duplicate skb for each */
1208 for (i
++; i
< WIL6210_MAX_TX_RINGS
; i
++) {
1209 v2
= &wil
->vring_tx
[i
];
1212 cid
= wil
->vring2cid_tid
[i
][0];
1213 if (cid
>= WIL6210_MAX_CID
) /* skip BCAST */
1215 if (!wil
->vring_tx_data
[i
].dot1x_open
&&
1216 (skb
->protocol
!= cpu_to_be16(ETH_P_PAE
)))
1219 if (0 == memcmp(wil
->sta
[cid
].addr
, src
, ETH_ALEN
))
1222 skb2
= skb_copy(skb
, GFP_ATOMIC
);
1224 wil_dbg_txrx(wil
, "BCAST DUP -> ring %d\n", i
);
1225 wil_set_da_for_vring(wil
, skb2
, i
);
1226 wil_tx_vring(wil
, v2
, skb2
);
1228 wil_err(wil
, "skb_copy failed\n");
1235 static int wil_tx_desc_map(struct vring_tx_desc
*d
, dma_addr_t pa
, u32 len
,
1238 wil_desc_addr_set(&d
->dma
.addr
, pa
);
1239 d
->dma
.ip_length
= 0;
1240 /* 0..6: mac_length; 7:ip_version 0-IP6 1-IP4*/
1241 d
->dma
.b11
= 0/*14 | BIT(7)*/;
1243 d
->dma
.status
= 0; /* BIT(0) should be 0 for HW_OWNED */
1244 d
->dma
.length
= cpu_to_le16((u16
)len
);
1245 d
->dma
.d0
= (vring_index
<< DMA_CFG_DESC_TX_0_QID_POS
);
1249 d
->mac
.ucode_cmd
= 0;
1250 /* translation type: 0 - bypass; 1 - 802.3; 2 - native wifi */
1251 d
->mac
.d
[2] = BIT(MAC_CFG_DESC_TX_2_SNAP_HDR_INSERTION_EN_POS
) |
1252 (1 << MAC_CFG_DESC_TX_2_L2_TRANSLATION_TYPE_POS
);
1258 void wil_tx_desc_set_nr_frags(struct vring_tx_desc
*d
, int nr_frags
)
1260 d
->mac
.d
[2] |= (nr_frags
<< MAC_CFG_DESC_TX_2_NUM_OF_DESCRIPTORS_POS
);
1264 * Sets the descriptor @d up for csum and/or TSO offloading. The corresponding
1265 * @skb is used to obtain the protocol and headers length.
1266 * @tso_desc_type is a descriptor type for TSO: 0 - a header, 1 - first data,
1267 * 2 - middle, 3 - last descriptor.
1270 static void wil_tx_desc_offload_setup_tso(struct vring_tx_desc
*d
,
1271 struct sk_buff
*skb
,
1272 int tso_desc_type
, bool is_ipv4
,
1273 int tcp_hdr_len
, int skb_net_hdr_len
)
1275 d
->dma
.b11
= ETH_HLEN
; /* MAC header length */
1276 d
->dma
.b11
|= is_ipv4
<< DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS
;
1278 d
->dma
.d0
|= (2 << DMA_CFG_DESC_TX_0_L4_TYPE_POS
);
1279 /* L4 header len: TCP header length */
1280 d
->dma
.d0
|= (tcp_hdr_len
& DMA_CFG_DESC_TX_0_L4_LENGTH_MSK
);
1282 /* Setup TSO: bit and desc type */
1283 d
->dma
.d0
|= (BIT(DMA_CFG_DESC_TX_0_TCP_SEG_EN_POS
)) |
1284 (tso_desc_type
<< DMA_CFG_DESC_TX_0_SEGMENT_BUF_DETAILS_POS
);
1285 d
->dma
.d0
|= (is_ipv4
<< DMA_CFG_DESC_TX_0_IPV4_CHECKSUM_EN_POS
);
1287 d
->dma
.ip_length
= skb_net_hdr_len
;
1288 /* Enable TCP/UDP checksum */
1289 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_TCP_UDP_CHECKSUM_EN_POS
);
1290 /* Calculate pseudo-header */
1291 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_PSEUDO_HEADER_CALC_EN_POS
);
1295 * Sets the descriptor @d up for csum. The corresponding
1296 * @skb is used to obtain the protocol and headers length.
1297 * Returns the protocol: 0 - not TCP, 1 - TCPv4, 2 - TCPv6.
1298 * Note, if d==NULL, the function only returns the protocol result.
1300 * It is very similar to previous wil_tx_desc_offload_setup_tso. This
1301 * is "if unrolling" to optimize the critical path.
1304 static int wil_tx_desc_offload_setup(struct vring_tx_desc
*d
,
1305 struct sk_buff
*skb
){
1308 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1311 d
->dma
.b11
= ETH_HLEN
; /* MAC header length */
1313 switch (skb
->protocol
) {
1314 case cpu_to_be16(ETH_P_IP
):
1315 protocol
= ip_hdr(skb
)->protocol
;
1316 d
->dma
.b11
|= BIT(DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS
);
1318 case cpu_to_be16(ETH_P_IPV6
):
1319 protocol
= ipv6_hdr(skb
)->nexthdr
;
1327 d
->dma
.d0
|= (2 << DMA_CFG_DESC_TX_0_L4_TYPE_POS
);
1328 /* L4 header len: TCP header length */
1330 (tcp_hdrlen(skb
) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK
);
1333 /* L4 header len: UDP header length */
1335 (sizeof(struct udphdr
) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK
);
1341 d
->dma
.ip_length
= skb_network_header_len(skb
);
1342 /* Enable TCP/UDP checksum */
1343 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_TCP_UDP_CHECKSUM_EN_POS
);
1344 /* Calculate pseudo-header */
1345 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_PSEUDO_HEADER_CALC_EN_POS
);
1350 static inline void wil_tx_last_desc(struct vring_tx_desc
*d
)
1352 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_CMD_EOP_POS
) |
1353 BIT(DMA_CFG_DESC_TX_0_CMD_MARK_WB_POS
) |
1354 BIT(DMA_CFG_DESC_TX_0_CMD_DMA_IT_POS
);
1357 static inline void wil_set_tx_desc_last_tso(volatile struct vring_tx_desc
*d
)
1359 d
->dma
.d0
|= wil_tso_type_lst
<<
1360 DMA_CFG_DESC_TX_0_SEGMENT_BUF_DETAILS_POS
;
1363 static int __wil_tx_vring_tso(struct wil6210_priv
*wil
, struct vring
*vring
,
1364 struct sk_buff
*skb
)
1366 struct device
*dev
= wil_to_dev(wil
);
1368 /* point to descriptors in shared memory */
1369 volatile struct vring_tx_desc
*_desc
= NULL
, *_hdr_desc
,
1370 *_first_desc
= NULL
;
1372 /* pointers to shadow descriptors */
1373 struct vring_tx_desc desc_mem
, hdr_desc_mem
, first_desc_mem
,
1374 *d
= &hdr_desc_mem
, *hdr_desc
= &hdr_desc_mem
,
1375 *first_desc
= &first_desc_mem
;
1377 /* pointer to shadow descriptors' context */
1378 struct wil_ctx
*hdr_ctx
, *first_ctx
= NULL
;
1380 int descs_used
= 0; /* total number of used descriptors */
1381 int sg_desc_cnt
= 0; /* number of descriptors for current mss*/
1383 u32 swhead
= vring
->swhead
;
1384 int used
, avail
= wil_vring_avail_tx(vring
);
1385 int nr_frags
= skb_shinfo(skb
)->nr_frags
;
1386 int min_desc_required
= nr_frags
+ 1;
1387 int mss
= skb_shinfo(skb
)->gso_size
; /* payload size w/o headers */
1388 int f
, len
, hdrlen
, headlen
;
1389 int vring_index
= vring
- wil
->vring_tx
;
1390 struct vring_tx_data
*txdata
= &wil
->vring_tx_data
[vring_index
];
1393 const skb_frag_t
*frag
= NULL
;
1396 int hdr_compensation_need
= true;
1397 int desc_tso_type
= wil_tso_type_first
;
1400 int skb_net_hdr_len
;
1404 wil_dbg_txrx(wil
, "tx_vring_tso: %d bytes to vring %d\n", skb
->len
,
1407 if (unlikely(!txdata
->enabled
))
1410 /* A typical page 4K is 3-4 payloads, we assume each fragment
1411 * is a full payload, that's how min_desc_required has been
1412 * calculated. In real we might need more or less descriptors,
1413 * this is the initial check only.
1415 if (unlikely(avail
< min_desc_required
)) {
1416 wil_err_ratelimited(wil
,
1417 "TSO: Tx ring[%2d] full. No space for %d fragments\n",
1418 vring_index
, min_desc_required
);
1422 /* Header Length = MAC header len + IP header len + TCP header len*/
1424 (int)skb_network_header_len(skb
) +
1427 gso_type
= skb_shinfo(skb
)->gso_type
& (SKB_GSO_TCPV6
| SKB_GSO_TCPV4
);
1430 /* TCP v4, zero out the IP length and IPv4 checksum fields
1431 * as required by the offloading doc
1433 ip_hdr(skb
)->tot_len
= 0;
1434 ip_hdr(skb
)->check
= 0;
1438 /* TCP v6, zero out the payload length */
1439 ipv6_hdr(skb
)->payload_len
= 0;
1443 /* other than TCPv4 or TCPv6 types are not supported for TSO.
1444 * It is also illegal for both to be set simultaneously
1449 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1452 /* tcp header length and skb network header length are fixed for all
1453 * packet's descriptors - read then once here
1455 tcp_hdr_len
= tcp_hdrlen(skb
);
1456 skb_net_hdr_len
= skb_network_header_len(skb
);
1458 _hdr_desc
= &vring
->va
[i
].tx
;
1460 pa
= dma_map_single(dev
, skb
->data
, hdrlen
, DMA_TO_DEVICE
);
1461 if (unlikely(dma_mapping_error(dev
, pa
))) {
1462 wil_err(wil
, "TSO: Skb head DMA map error\n");
1466 wil_tx_desc_map(hdr_desc
, pa
, hdrlen
, vring_index
);
1467 wil_tx_desc_offload_setup_tso(hdr_desc
, skb
, wil_tso_type_hdr
, is_ipv4
,
1468 tcp_hdr_len
, skb_net_hdr_len
);
1469 wil_tx_last_desc(hdr_desc
);
1471 vring
->ctx
[i
].mapped_as
= wil_mapped_as_single
;
1472 hdr_ctx
= &vring
->ctx
[i
];
1475 headlen
= skb_headlen(skb
) - hdrlen
;
1477 for (f
= headlen
? -1 : 0; f
< nr_frags
; f
++) {
1480 wil_dbg_txrx(wil
, "TSO: process skb head, len %u\n",
1483 frag
= &skb_shinfo(skb
)->frags
[f
];
1485 wil_dbg_txrx(wil
, "TSO: frag[%d]: len %u\n", f
, len
);
1490 "TSO: len %d, rem_data %d, descs_used %d\n",
1491 len
, rem_data
, descs_used
);
1493 if (descs_used
== avail
) {
1494 wil_err_ratelimited(wil
, "TSO: ring overflow\n");
1499 lenmss
= min_t(int, rem_data
, len
);
1500 i
= (swhead
+ descs_used
) % vring
->size
;
1501 wil_dbg_txrx(wil
, "TSO: lenmss %d, i %d\n", lenmss
, i
);
1504 pa
= skb_frag_dma_map(dev
, frag
,
1505 frag
->size
- len
, lenmss
,
1507 vring
->ctx
[i
].mapped_as
= wil_mapped_as_page
;
1509 pa
= dma_map_single(dev
,
1511 skb_headlen(skb
) - headlen
,
1514 vring
->ctx
[i
].mapped_as
= wil_mapped_as_single
;
1518 if (unlikely(dma_mapping_error(dev
, pa
))) {
1519 wil_err(wil
, "TSO: DMA map page error\n");
1523 _desc
= &vring
->va
[i
].tx
;
1526 _first_desc
= _desc
;
1527 first_ctx
= &vring
->ctx
[i
];
1533 wil_tx_desc_map(d
, pa
, lenmss
, vring_index
);
1534 wil_tx_desc_offload_setup_tso(d
, skb
, desc_tso_type
,
1535 is_ipv4
, tcp_hdr_len
,
1538 /* use tso_type_first only once */
1539 desc_tso_type
= wil_tso_type_mid
;
1541 descs_used
++; /* desc used so far */
1542 sg_desc_cnt
++; /* desc used for this segment */
1547 "TSO: len %d, rem_data %d, descs_used %d, sg_desc_cnt %d,\n",
1548 len
, rem_data
, descs_used
, sg_desc_cnt
);
1550 /* Close the segment if reached mss size or last frag*/
1551 if (rem_data
== 0 || (f
== nr_frags
- 1 && len
== 0)) {
1552 if (hdr_compensation_need
) {
1553 /* first segment include hdr desc for
1556 hdr_ctx
->nr_frags
= sg_desc_cnt
;
1557 wil_tx_desc_set_nr_frags(first_desc
,
1560 hdr_compensation_need
= false;
1562 wil_tx_desc_set_nr_frags(first_desc
,
1565 first_ctx
->nr_frags
= sg_desc_cnt
- 1;
1567 wil_tx_last_desc(d
);
1569 /* first descriptor may also be the last
1570 * for this mss - make sure not to copy
1573 if (first_desc
!= d
)
1574 *_first_desc
= *first_desc
;
1576 /*last descriptor will be copied at the end
1577 * of this TS processing
1579 if (f
< nr_frags
- 1 || len
> 0)
1585 } else if (first_desc
!= d
) /* update mid descriptor */
1590 /* first descriptor may also be the last.
1591 * in this case d pointer is invalid
1593 if (_first_desc
== _desc
)
1596 /* Last data descriptor */
1597 wil_set_tx_desc_last_tso(d
);
1600 /* Fill the total number of descriptors in first desc (hdr)*/
1601 wil_tx_desc_set_nr_frags(hdr_desc
, descs_used
);
1602 *_hdr_desc
= *hdr_desc
;
1604 /* hold reference to skb
1605 * to prevent skb release before accounting
1606 * in case of immediate "tx done"
1608 vring
->ctx
[i
].skb
= skb_get(skb
);
1610 /* performance monitoring */
1611 used
= wil_vring_used_tx(vring
);
1612 if (wil_val_in_range(vring_idle_trsh
,
1613 used
, used
+ descs_used
)) {
1614 txdata
->idle
+= get_cycles() - txdata
->last_idle
;
1615 wil_dbg_txrx(wil
, "Ring[%2d] not idle %d -> %d\n",
1616 vring_index
, used
, used
+ descs_used
);
1619 /* Make sure to advance the head only after descriptor update is done.
1620 * This will prevent a race condition where the completion thread
1621 * will see the DU bit set from previous run and will handle the
1622 * skb before it was completed.
1626 /* advance swhead */
1627 wil_vring_advance_head(vring
, descs_used
);
1628 wil_dbg_txrx(wil
, "TSO: Tx swhead %d -> %d\n", swhead
, vring
->swhead
);
1630 /* make sure all writes to descriptors (shared memory) are done before
1631 * committing them to HW
1635 wil_w(wil
, vring
->hwtail
, vring
->swhead
);
1639 while (descs_used
> 0) {
1640 struct wil_ctx
*ctx
;
1642 i
= (swhead
+ descs_used
- 1) % vring
->size
;
1643 d
= (struct vring_tx_desc
*)&vring
->va
[i
].tx
;
1644 _desc
= &vring
->va
[i
].tx
;
1646 _desc
->dma
.status
= TX_DMA_STATUS_DU
;
1647 ctx
= &vring
->ctx
[i
];
1648 wil_txdesc_unmap(dev
, d
, ctx
);
1649 memset(ctx
, 0, sizeof(*ctx
));
1656 static int __wil_tx_vring(struct wil6210_priv
*wil
, struct vring
*vring
,
1657 struct sk_buff
*skb
)
1659 struct device
*dev
= wil_to_dev(wil
);
1660 struct vring_tx_desc dd
, *d
= &dd
;
1661 volatile struct vring_tx_desc
*_d
;
1662 u32 swhead
= vring
->swhead
;
1663 int avail
= wil_vring_avail_tx(vring
);
1664 int nr_frags
= skb_shinfo(skb
)->nr_frags
;
1666 int vring_index
= vring
- wil
->vring_tx
;
1667 struct vring_tx_data
*txdata
= &wil
->vring_tx_data
[vring_index
];
1671 bool mcast
= (vring_index
== wil
->bcast_vring
);
1672 uint len
= skb_headlen(skb
);
1674 wil_dbg_txrx(wil
, "tx_vring: %d bytes to vring %d\n", skb
->len
,
1677 if (unlikely(!txdata
->enabled
))
1680 if (unlikely(avail
< 1 + nr_frags
)) {
1681 wil_err_ratelimited(wil
,
1682 "Tx ring[%2d] full. No space for %d fragments\n",
1683 vring_index
, 1 + nr_frags
);
1686 _d
= &vring
->va
[i
].tx
;
1688 pa
= dma_map_single(dev
, skb
->data
, skb_headlen(skb
), DMA_TO_DEVICE
);
1690 wil_dbg_txrx(wil
, "Tx[%2d] skb %d bytes 0x%p -> %pad\n", vring_index
,
1691 skb_headlen(skb
), skb
->data
, &pa
);
1692 wil_hex_dump_txrx("Tx ", DUMP_PREFIX_OFFSET
, 16, 1,
1693 skb
->data
, skb_headlen(skb
), false);
1695 if (unlikely(dma_mapping_error(dev
, pa
)))
1697 vring
->ctx
[i
].mapped_as
= wil_mapped_as_single
;
1699 wil_tx_desc_map(d
, pa
, len
, vring_index
);
1700 if (unlikely(mcast
)) {
1701 d
->mac
.d
[0] |= BIT(MAC_CFG_DESC_TX_0_MCS_EN_POS
); /* MCS 0 */
1702 if (unlikely(len
> WIL_BCAST_MCS0_LIMIT
)) /* set MCS 1 */
1703 d
->mac
.d
[0] |= (1 << MAC_CFG_DESC_TX_0_MCS_INDEX_POS
);
1705 /* Process TCP/UDP checksum offloading */
1706 if (unlikely(wil_tx_desc_offload_setup(d
, skb
))) {
1707 wil_err(wil
, "Tx[%2d] Failed to set cksum, drop packet\n",
1712 vring
->ctx
[i
].nr_frags
= nr_frags
;
1713 wil_tx_desc_set_nr_frags(d
, nr_frags
+ 1);
1715 /* middle segments */
1716 for (; f
< nr_frags
; f
++) {
1717 const struct skb_frag_struct
*frag
=
1718 &skb_shinfo(skb
)->frags
[f
];
1719 int len
= skb_frag_size(frag
);
1722 wil_dbg_txrx(wil
, "Tx[%2d] desc[%4d]\n", vring_index
, i
);
1723 wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE
, 32, 4,
1724 (const void *)d
, sizeof(*d
), false);
1725 i
= (swhead
+ f
+ 1) % vring
->size
;
1726 _d
= &vring
->va
[i
].tx
;
1727 pa
= skb_frag_dma_map(dev
, frag
, 0, skb_frag_size(frag
),
1729 if (unlikely(dma_mapping_error(dev
, pa
))) {
1730 wil_err(wil
, "Tx[%2d] failed to map fragment\n",
1734 vring
->ctx
[i
].mapped_as
= wil_mapped_as_page
;
1735 wil_tx_desc_map(d
, pa
, len
, vring_index
);
1736 /* no need to check return code -
1737 * if it succeeded for 1-st descriptor,
1738 * it will succeed here too
1740 wil_tx_desc_offload_setup(d
, skb
);
1742 /* for the last seg only */
1743 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_CMD_EOP_POS
);
1744 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_CMD_MARK_WB_POS
);
1745 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_CMD_DMA_IT_POS
);
1747 wil_dbg_txrx(wil
, "Tx[%2d] desc[%4d]\n", vring_index
, i
);
1748 wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE
, 32, 4,
1749 (const void *)d
, sizeof(*d
), false);
1751 /* hold reference to skb
1752 * to prevent skb release before accounting
1753 * in case of immediate "tx done"
1755 vring
->ctx
[i
].skb
= skb_get(skb
);
1757 /* performance monitoring */
1758 used
= wil_vring_used_tx(vring
);
1759 if (wil_val_in_range(vring_idle_trsh
,
1760 used
, used
+ nr_frags
+ 1)) {
1761 txdata
->idle
+= get_cycles() - txdata
->last_idle
;
1762 wil_dbg_txrx(wil
, "Ring[%2d] not idle %d -> %d\n",
1763 vring_index
, used
, used
+ nr_frags
+ 1);
1766 /* Make sure to advance the head only after descriptor update is done.
1767 * This will prevent a race condition where the completion thread
1768 * will see the DU bit set from previous run and will handle the
1769 * skb before it was completed.
1773 /* advance swhead */
1774 wil_vring_advance_head(vring
, nr_frags
+ 1);
1775 wil_dbg_txrx(wil
, "Tx[%2d] swhead %d -> %d\n", vring_index
, swhead
,
1777 trace_wil6210_tx(vring_index
, swhead
, skb
->len
, nr_frags
);
1779 /* make sure all writes to descriptors (shared memory) are done before
1780 * committing them to HW
1784 wil_w(wil
, vring
->hwtail
, vring
->swhead
);
1788 /* unmap what we have mapped */
1789 nr_frags
= f
+ 1; /* frags mapped + one for skb head */
1790 for (f
= 0; f
< nr_frags
; f
++) {
1791 struct wil_ctx
*ctx
;
1793 i
= (swhead
+ f
) % vring
->size
;
1794 ctx
= &vring
->ctx
[i
];
1795 _d
= &vring
->va
[i
].tx
;
1797 _d
->dma
.status
= TX_DMA_STATUS_DU
;
1798 wil_txdesc_unmap(dev
, d
, ctx
);
1800 memset(ctx
, 0, sizeof(*ctx
));
1806 static int wil_tx_vring(struct wil6210_priv
*wil
, struct vring
*vring
,
1807 struct sk_buff
*skb
)
1809 int vring_index
= vring
- wil
->vring_tx
;
1810 struct vring_tx_data
*txdata
= &wil
->vring_tx_data
[vring_index
];
1813 spin_lock(&txdata
->lock
);
1815 rc
= (skb_is_gso(skb
) ? __wil_tx_vring_tso
: __wil_tx_vring
)
1818 spin_unlock(&txdata
->lock
);
1824 * Check status of tx vrings and stop/wake net queues if needed
1826 * This function does one of two checks:
1827 * In case check_stop is true, will check if net queues need to be stopped. If
1828 * the conditions for stopping are met, netif_tx_stop_all_queues() is called.
1829 * In case check_stop is false, will check if net queues need to be waked. If
1830 * the conditions for waking are met, netif_tx_wake_all_queues() is called.
1831 * vring is the vring which is currently being modified by either adding
1832 * descriptors (tx) into it or removing descriptors (tx complete) from it. Can
1833 * be null when irrelevant (e.g. connect/disconnect events).
1835 * The implementation is to stop net queues if modified vring has low
1836 * descriptor availability. Wake if all vrings are not in low descriptor
1837 * availability and modified vring has high descriptor availability.
1839 static inline void __wil_update_net_queues(struct wil6210_priv
*wil
,
1840 struct vring
*vring
,
1846 wil_dbg_txrx(wil
, "vring %d, check_stop=%d, stopped=%d",
1847 (int)(vring
- wil
->vring_tx
), check_stop
,
1848 wil
->net_queue_stopped
);
1850 wil_dbg_txrx(wil
, "check_stop=%d, stopped=%d",
1851 check_stop
, wil
->net_queue_stopped
);
1853 if (check_stop
== wil
->net_queue_stopped
)
1854 /* net queues already in desired state */
1858 if (!vring
|| unlikely(wil_vring_avail_low(vring
))) {
1859 /* not enough room in the vring */
1860 netif_tx_stop_all_queues(wil_to_ndev(wil
));
1861 wil
->net_queue_stopped
= true;
1862 wil_dbg_txrx(wil
, "netif_tx_stop called\n");
1868 for (i
= 0; i
< WIL6210_MAX_TX_RINGS
; i
++) {
1869 struct vring
*cur_vring
= &wil
->vring_tx
[i
];
1870 struct vring_tx_data
*txdata
= &wil
->vring_tx_data
[i
];
1872 if (!cur_vring
->va
|| !txdata
->enabled
|| cur_vring
== vring
)
1875 if (wil_vring_avail_low(cur_vring
)) {
1876 wil_dbg_txrx(wil
, "vring %d full, can't wake\n",
1877 (int)(cur_vring
- wil
->vring_tx
));
1882 if (!vring
|| wil_vring_avail_high(vring
)) {
1883 /* enough room in the vring */
1884 wil_dbg_txrx(wil
, "calling netif_tx_wake\n");
1885 netif_tx_wake_all_queues(wil_to_ndev(wil
));
1886 wil
->net_queue_stopped
= false;
1890 void wil_update_net_queues(struct wil6210_priv
*wil
, struct vring
*vring
,
1893 spin_lock(&wil
->net_queue_lock
);
1894 __wil_update_net_queues(wil
, vring
, check_stop
);
1895 spin_unlock(&wil
->net_queue_lock
);
1898 void wil_update_net_queues_bh(struct wil6210_priv
*wil
, struct vring
*vring
,
1901 spin_lock_bh(&wil
->net_queue_lock
);
1902 __wil_update_net_queues(wil
, vring
, check_stop
);
1903 spin_unlock_bh(&wil
->net_queue_lock
);
1906 netdev_tx_t
wil_start_xmit(struct sk_buff
*skb
, struct net_device
*ndev
)
1908 struct wil6210_priv
*wil
= ndev_to_wil(ndev
);
1909 struct ethhdr
*eth
= (void *)skb
->data
;
1910 bool bcast
= is_multicast_ether_addr(eth
->h_dest
);
1911 struct vring
*vring
;
1912 static bool pr_once_fw
;
1915 wil_dbg_txrx(wil
, "start_xmit\n");
1916 if (unlikely(!test_bit(wil_status_fwready
, wil
->status
))) {
1918 wil_err(wil
, "FW not ready\n");
1923 if (unlikely(!test_bit(wil_status_fwconnected
, wil
->status
))) {
1924 wil_dbg_ratelimited(wil
, "FW not connected, packet dropped\n");
1927 if (unlikely(wil
->wdev
->iftype
== NL80211_IFTYPE_MONITOR
)) {
1928 wil_err(wil
, "Xmit in monitor mode not supported\n");
1934 if (wil
->wdev
->iftype
== NL80211_IFTYPE_STATION
&& !wil
->pbss
) {
1935 /* in STA mode (ESS), all to same VRING (to AP) */
1936 vring
= wil_find_tx_vring_sta(wil
, skb
);
1939 /* in pbss, no bcast VRING - duplicate skb in
1940 * all stations VRINGs
1942 vring
= wil_find_tx_bcast_2(wil
, skb
);
1943 else if (wil
->wdev
->iftype
== NL80211_IFTYPE_AP
)
1944 /* AP has a dedicated bcast VRING */
1945 vring
= wil_find_tx_bcast_1(wil
, skb
);
1947 /* unexpected combination, fallback to duplicating
1948 * the skb in all stations VRINGs
1950 vring
= wil_find_tx_bcast_2(wil
, skb
);
1952 /* unicast, find specific VRING by dest. address */
1953 vring
= wil_find_tx_ucast(wil
, skb
);
1955 if (unlikely(!vring
)) {
1956 wil_dbg_txrx(wil
, "No Tx VRING found for %pM\n", eth
->h_dest
);
1959 /* set up vring entry */
1960 rc
= wil_tx_vring(wil
, vring
, skb
);
1964 /* shall we stop net queues? */
1965 wil_update_net_queues_bh(wil
, vring
, true);
1966 /* statistics will be updated on the tx_complete */
1967 dev_kfree_skb_any(skb
);
1968 return NETDEV_TX_OK
;
1970 return NETDEV_TX_BUSY
;
1972 break; /* goto drop; */
1975 ndev
->stats
.tx_dropped
++;
1976 dev_kfree_skb_any(skb
);
1978 return NET_XMIT_DROP
;
1981 static inline bool wil_need_txstat(struct sk_buff
*skb
)
1983 struct ethhdr
*eth
= (void *)skb
->data
;
1985 return is_unicast_ether_addr(eth
->h_dest
) && skb
->sk
&&
1986 (skb_shinfo(skb
)->tx_flags
& SKBTX_WIFI_STATUS
);
1989 static inline void wil_consume_skb(struct sk_buff
*skb
, bool acked
)
1991 if (unlikely(wil_need_txstat(skb
)))
1992 skb_complete_wifi_ack(skb
, acked
);
1994 acked
? dev_consume_skb_any(skb
) : dev_kfree_skb_any(skb
);
1998 * Clean up transmitted skb's from the Tx VRING
2000 * Return number of descriptors cleared
2002 * Safe to call from IRQ
2004 int wil_tx_complete(struct wil6210_priv
*wil
, int ringid
)
2006 struct net_device
*ndev
= wil_to_ndev(wil
);
2007 struct device
*dev
= wil_to_dev(wil
);
2008 struct vring
*vring
= &wil
->vring_tx
[ringid
];
2009 struct vring_tx_data
*txdata
= &wil
->vring_tx_data
[ringid
];
2011 int cid
= wil
->vring2cid_tid
[ringid
][0];
2012 struct wil_net_stats
*stats
= NULL
;
2013 volatile struct vring_tx_desc
*_d
;
2014 int used_before_complete
;
2017 if (unlikely(!vring
->va
)) {
2018 wil_err(wil
, "Tx irq[%d]: vring not initialized\n", ringid
);
2022 if (unlikely(!txdata
->enabled
)) {
2023 wil_info(wil
, "Tx irq[%d]: vring disabled\n", ringid
);
2027 wil_dbg_txrx(wil
, "tx_complete: (%d)\n", ringid
);
2029 used_before_complete
= wil_vring_used_tx(vring
);
2031 if (cid
< WIL6210_MAX_CID
)
2032 stats
= &wil
->sta
[cid
].stats
;
2034 while (!wil_vring_is_empty(vring
)) {
2036 struct wil_ctx
*ctx
= &vring
->ctx
[vring
->swtail
];
2038 * For the fragmented skb, HW will set DU bit only for the
2039 * last fragment. look for it.
2040 * In TSO the first DU will include hdr desc
2042 int lf
= (vring
->swtail
+ ctx
->nr_frags
) % vring
->size
;
2043 /* TODO: check we are not past head */
2045 _d
= &vring
->va
[lf
].tx
;
2046 if (unlikely(!(_d
->dma
.status
& TX_DMA_STATUS_DU
)))
2049 new_swtail
= (lf
+ 1) % vring
->size
;
2050 while (vring
->swtail
!= new_swtail
) {
2051 struct vring_tx_desc dd
, *d
= &dd
;
2053 struct sk_buff
*skb
;
2055 ctx
= &vring
->ctx
[vring
->swtail
];
2057 _d
= &vring
->va
[vring
->swtail
].tx
;
2061 dmalen
= le16_to_cpu(d
->dma
.length
);
2062 trace_wil6210_tx_done(ringid
, vring
->swtail
, dmalen
,
2065 "TxC[%2d][%3d] : %d bytes, status 0x%02x err 0x%02x\n",
2066 ringid
, vring
->swtail
, dmalen
,
2067 d
->dma
.status
, d
->dma
.error
);
2068 wil_hex_dump_txrx("TxCD ", DUMP_PREFIX_NONE
, 32, 4,
2069 (const void *)d
, sizeof(*d
), false);
2071 wil_txdesc_unmap(dev
, d
, ctx
);
2074 if (likely(d
->dma
.error
== 0)) {
2075 ndev
->stats
.tx_packets
++;
2076 ndev
->stats
.tx_bytes
+= skb
->len
;
2078 stats
->tx_packets
++;
2079 stats
->tx_bytes
+= skb
->len
;
2082 ndev
->stats
.tx_errors
++;
2086 wil_consume_skb(skb
, d
->dma
.error
== 0);
2088 memset(ctx
, 0, sizeof(*ctx
));
2089 /* Make sure the ctx is zeroed before updating the tail
2090 * to prevent a case where wil_tx_vring will see
2091 * this descriptor as used and handle it before ctx zero
2095 /* There is no need to touch HW descriptor:
2096 * - ststus bit TX_DMA_STATUS_DU is set by design,
2097 * so hardware will not try to process this desc.,
2098 * - rest of descriptor will be initialized on Tx.
2100 vring
->swtail
= wil_vring_next_tail(vring
);
2105 /* performance monitoring */
2106 used_new
= wil_vring_used_tx(vring
);
2107 if (wil_val_in_range(vring_idle_trsh
,
2108 used_new
, used_before_complete
)) {
2109 wil_dbg_txrx(wil
, "Ring[%2d] idle %d -> %d\n",
2110 ringid
, used_before_complete
, used_new
);
2111 txdata
->last_idle
= get_cycles();
2114 /* shall we wake net queues? */
2116 wil_update_net_queues(wil
, vring
, false);