2 * aQuantia Corporation Network Driver
3 * Copyright (C) 2014-2017 aQuantia Corporation. All rights reserved
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
10 /* File aq_ring.c: Definition of functions for Rx/Tx rings. */
16 #include <linux/netdevice.h>
17 #include <linux/etherdevice.h>
19 static struct aq_ring_s
*aq_ring_alloc(struct aq_ring_s
*self
,
20 struct aq_nic_s
*aq_nic
)
25 kcalloc(self
->size
, sizeof(struct aq_ring_buff_s
), GFP_KERNEL
);
27 if (!self
->buff_ring
) {
31 self
->dx_ring
= dma_alloc_coherent(aq_nic_get_dev(aq_nic
),
32 self
->size
* self
->dx_size
,
33 &self
->dx_ring_pa
, GFP_KERNEL
);
47 struct aq_ring_s
*aq_ring_tx_alloc(struct aq_ring_s
*self
,
48 struct aq_nic_s
*aq_nic
,
50 struct aq_nic_cfg_s
*aq_nic_cfg
)
54 self
->aq_nic
= aq_nic
;
56 self
->size
= aq_nic_cfg
->txds
;
57 self
->dx_size
= aq_nic_cfg
->aq_hw_caps
->txd_size
;
59 self
= aq_ring_alloc(self
, aq_nic
);
73 struct aq_ring_s
*aq_ring_rx_alloc(struct aq_ring_s
*self
,
74 struct aq_nic_s
*aq_nic
,
76 struct aq_nic_cfg_s
*aq_nic_cfg
)
80 self
->aq_nic
= aq_nic
;
82 self
->size
= aq_nic_cfg
->rxds
;
83 self
->dx_size
= aq_nic_cfg
->aq_hw_caps
->rxd_size
;
85 self
= aq_ring_alloc(self
, aq_nic
);
99 int aq_ring_init(struct aq_ring_s
*self
)
107 static inline bool aq_ring_dx_in_range(unsigned int h
, unsigned int i
,
110 return (h
< t
) ? ((h
< i
) && (i
< t
)) : ((h
< i
) || (i
< t
));
113 void aq_ring_update_queue_state(struct aq_ring_s
*ring
)
115 if (aq_ring_avail_dx(ring
) <= AQ_CFG_SKB_FRAGS_MAX
)
116 aq_ring_queue_stop(ring
);
117 else if (aq_ring_avail_dx(ring
) > AQ_CFG_RESTART_DESC_THRES
)
118 aq_ring_queue_wake(ring
);
121 void aq_ring_queue_wake(struct aq_ring_s
*ring
)
123 struct net_device
*ndev
= aq_nic_get_ndev(ring
->aq_nic
);
125 if (__netif_subqueue_stopped(ndev
, ring
->idx
)) {
126 netif_wake_subqueue(ndev
, ring
->idx
);
127 ring
->stats
.tx
.queue_restarts
++;
131 void aq_ring_queue_stop(struct aq_ring_s
*ring
)
133 struct net_device
*ndev
= aq_nic_get_ndev(ring
->aq_nic
);
135 if (!__netif_subqueue_stopped(ndev
, ring
->idx
))
136 netif_stop_subqueue(ndev
, ring
->idx
);
139 bool aq_ring_tx_clean(struct aq_ring_s
*self
)
141 struct device
*dev
= aq_nic_get_dev(self
->aq_nic
);
142 unsigned int budget
= AQ_CFG_TX_CLEAN_BUDGET
;
144 for (; self
->sw_head
!= self
->hw_head
&& budget
--;
145 self
->sw_head
= aq_ring_next_dx(self
, self
->sw_head
)) {
146 struct aq_ring_buff_s
*buff
= &self
->buff_ring
[self
->sw_head
];
148 if (likely(buff
->is_mapped
)) {
149 if (unlikely(buff
->is_sop
)) {
151 buff
->eop_index
!= 0xffffU
&&
152 (!aq_ring_dx_in_range(self
->sw_head
,
157 dma_unmap_single(dev
, buff
->pa
, buff
->len
,
160 dma_unmap_page(dev
, buff
->pa
, buff
->len
,
165 if (unlikely(buff
->is_eop
))
166 dev_kfree_skb_any(buff
->skb
);
169 buff
->eop_index
= 0xffffU
;
175 static void aq_rx_checksum(struct aq_ring_s
*self
,
176 struct aq_ring_buff_s
*buff
,
179 if (!(self
->aq_nic
->ndev
->features
& NETIF_F_RXCSUM
))
182 if (unlikely(buff
->is_cso_err
)) {
183 ++self
->stats
.rx
.errors
;
184 skb
->ip_summed
= CHECKSUM_NONE
;
187 if (buff
->is_ip_cso
) {
188 __skb_incr_checksum_unnecessary(skb
);
190 skb
->ip_summed
= CHECKSUM_NONE
;
193 if (buff
->is_udp_cso
|| buff
->is_tcp_cso
)
194 __skb_incr_checksum_unnecessary(skb
);
197 #define AQ_SKB_ALIGN SKB_DATA_ALIGN(sizeof(struct skb_shared_info))
198 int aq_ring_rx_clean(struct aq_ring_s
*self
,
199 struct napi_struct
*napi
,
203 struct net_device
*ndev
= aq_nic_get_ndev(self
->aq_nic
);
205 bool is_rsc_completed
= true;
207 for (; (self
->sw_head
!= self
->hw_head
) && budget
;
208 self
->sw_head
= aq_ring_next_dx(self
, self
->sw_head
),
209 --budget
, ++(*work_done
)) {
210 struct aq_ring_buff_s
*buff
= &self
->buff_ring
[self
->sw_head
];
211 struct sk_buff
*skb
= NULL
;
212 unsigned int next_
= 0U;
214 struct aq_ring_buff_s
*buff_
= NULL
;
216 if (buff
->is_error
) {
217 __free_pages(buff
->page
, 0);
221 if (buff
->is_cleaned
)
225 for (next_
= buff
->next
,
226 buff_
= &self
->buff_ring
[next_
]; true;
228 buff_
= &self
->buff_ring
[next_
]) {
230 aq_ring_dx_in_range(self
->sw_head
,
234 if (unlikely(!is_rsc_completed
)) {
235 is_rsc_completed
= false;
243 if (!is_rsc_completed
) {
249 /* for single fragment packets use build_skb() */
251 buff
->len
<= AQ_CFG_RX_FRAME_MAX
- AQ_SKB_ALIGN
) {
252 skb
= build_skb(page_address(buff
->page
),
253 AQ_CFG_RX_FRAME_MAX
);
254 if (unlikely(!skb
)) {
259 skb_put(skb
, buff
->len
);
261 skb
= netdev_alloc_skb(ndev
, ETH_HLEN
);
262 if (unlikely(!skb
)) {
266 skb_put(skb
, ETH_HLEN
);
267 memcpy(skb
->data
, page_address(buff
->page
), ETH_HLEN
);
269 skb_add_rx_frag(skb
, 0, buff
->page
, ETH_HLEN
,
270 buff
->len
- ETH_HLEN
,
271 SKB_TRUESIZE(buff
->len
- ETH_HLEN
));
274 for (i
= 1U, next_
= buff
->next
,
275 buff_
= &self
->buff_ring
[next_
];
276 true; next_
= buff_
->next
,
277 buff_
= &self
->buff_ring
[next_
], ++i
) {
278 skb_add_rx_frag(skb
, i
,
281 SKB_TRUESIZE(buff
->len
-
283 buff_
->is_cleaned
= 1;
291 skb
->protocol
= eth_type_trans(skb
, ndev
);
293 aq_rx_checksum(self
, buff
, skb
);
295 skb_set_hash(skb
, buff
->rss_hash
,
296 buff
->is_hash_l4
? PKT_HASH_TYPE_L4
:
299 skb_record_rx_queue(skb
, self
->idx
);
301 ++self
->stats
.rx
.packets
;
302 self
->stats
.rx
.bytes
+= skb
->len
;
304 napi_gro_receive(napi
, skb
);
311 int aq_ring_rx_fill(struct aq_ring_s
*self
)
313 unsigned int pages_order
= fls(AQ_CFG_RX_FRAME_MAX
/ PAGE_SIZE
+
314 (AQ_CFG_RX_FRAME_MAX
% PAGE_SIZE
? 1 : 0)) - 1;
315 struct aq_ring_buff_s
*buff
= NULL
;
319 for (i
= aq_ring_avail_dx(self
); i
--;
320 self
->sw_tail
= aq_ring_next_dx(self
, self
->sw_tail
)) {
321 buff
= &self
->buff_ring
[self
->sw_tail
];
324 buff
->len
= AQ_CFG_RX_FRAME_MAX
;
326 buff
->page
= alloc_pages(GFP_ATOMIC
| __GFP_COMP
, pages_order
);
332 buff
->pa
= dma_map_page(aq_nic_get_dev(self
->aq_nic
),
334 AQ_CFG_RX_FRAME_MAX
, DMA_FROM_DEVICE
);
336 if (dma_mapping_error(aq_nic_get_dev(self
->aq_nic
), buff
->pa
)) {
346 if (buff
&& buff
->page
)
347 __free_pages(buff
->page
, 0);
353 void aq_ring_rx_deinit(struct aq_ring_s
*self
)
358 for (; self
->sw_head
!= self
->sw_tail
;
359 self
->sw_head
= aq_ring_next_dx(self
, self
->sw_head
)) {
360 struct aq_ring_buff_s
*buff
= &self
->buff_ring
[self
->sw_head
];
362 dma_unmap_page(aq_nic_get_dev(self
->aq_nic
), buff
->pa
,
363 AQ_CFG_RX_FRAME_MAX
, DMA_FROM_DEVICE
);
365 __free_pages(buff
->page
, 0);
371 void aq_ring_free(struct aq_ring_s
*self
)
376 kfree(self
->buff_ring
);
379 dma_free_coherent(aq_nic_get_dev(self
->aq_nic
),
380 self
->size
* self
->dx_size
, self
->dx_ring
,