]>
Commit | Line | Data |
---|---|---|
018423e9 DV |
1 | /* |
2 | * aQuantia Corporation Network Driver | |
3 | * Copyright (C) 2014-2017 aQuantia Corporation. All rights reserved | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify it | |
6 | * under the terms and conditions of the GNU General Public License, | |
7 | * version 2, as published by the Free Software Foundation. | |
8 | */ | |
9 | ||
10 | /* File aq_ring.c: Definition of functions for Rx/Tx rings. */ | |
11 | ||
12 | #include "aq_ring.h" | |
13 | #include "aq_nic.h" | |
14 | #include "aq_hw.h" | |
15 | ||
16 | #include <linux/netdevice.h> | |
17 | #include <linux/etherdevice.h> | |
18 | ||
19 | static struct aq_ring_s *aq_ring_alloc(struct aq_ring_s *self, | |
20 | struct aq_nic_s *aq_nic) | |
21 | { | |
22 | int err = 0; | |
23 | ||
24 | self->buff_ring = | |
25 | kcalloc(self->size, sizeof(struct aq_ring_buff_s), GFP_KERNEL); | |
26 | ||
27 | if (!self->buff_ring) { | |
28 | err = -ENOMEM; | |
29 | goto err_exit; | |
30 | } | |
31 | self->dx_ring = dma_alloc_coherent(aq_nic_get_dev(aq_nic), | |
32 | self->size * self->dx_size, | |
33 | &self->dx_ring_pa, GFP_KERNEL); | |
34 | if (!self->dx_ring) { | |
35 | err = -ENOMEM; | |
36 | goto err_exit; | |
37 | } | |
38 | ||
39 | err_exit: | |
40 | if (err < 0) { | |
41 | aq_ring_free(self); | |
42 | self = NULL; | |
43 | } | |
44 | return self; | |
45 | } | |
46 | ||
47 | struct aq_ring_s *aq_ring_tx_alloc(struct aq_ring_s *self, | |
48 | struct aq_nic_s *aq_nic, | |
49 | unsigned int idx, | |
50 | struct aq_nic_cfg_s *aq_nic_cfg) | |
51 | { | |
52 | int err = 0; | |
53 | ||
54 | self->aq_nic = aq_nic; | |
55 | self->idx = idx; | |
56 | self->size = aq_nic_cfg->txds; | |
57 | self->dx_size = aq_nic_cfg->aq_hw_caps->txd_size; | |
58 | ||
59 | self = aq_ring_alloc(self, aq_nic); | |
60 | if (!self) { | |
61 | err = -ENOMEM; | |
62 | goto err_exit; | |
63 | } | |
64 | ||
65 | err_exit: | |
66 | if (err < 0) { | |
67 | aq_ring_free(self); | |
68 | self = NULL; | |
69 | } | |
70 | return self; | |
71 | } | |
72 | ||
73 | struct aq_ring_s *aq_ring_rx_alloc(struct aq_ring_s *self, | |
74 | struct aq_nic_s *aq_nic, | |
75 | unsigned int idx, | |
76 | struct aq_nic_cfg_s *aq_nic_cfg) | |
77 | { | |
78 | int err = 0; | |
79 | ||
80 | self->aq_nic = aq_nic; | |
81 | self->idx = idx; | |
82 | self->size = aq_nic_cfg->rxds; | |
83 | self->dx_size = aq_nic_cfg->aq_hw_caps->rxd_size; | |
84 | ||
85 | self = aq_ring_alloc(self, aq_nic); | |
86 | if (!self) { | |
87 | err = -ENOMEM; | |
88 | goto err_exit; | |
89 | } | |
90 | ||
91 | err_exit: | |
92 | if (err < 0) { | |
93 | aq_ring_free(self); | |
94 | self = NULL; | |
95 | } | |
96 | return self; | |
97 | } | |
98 | ||
99 | int aq_ring_init(struct aq_ring_s *self) | |
100 | { | |
101 | self->hw_head = 0; | |
102 | self->sw_head = 0; | |
103 | self->sw_tail = 0; | |
104 | return 0; | |
105 | } | |
106 | ||
c7545689 PB |
107 | static inline bool aq_ring_dx_in_range(unsigned int h, unsigned int i, |
108 | unsigned int t) | |
109 | { | |
110 | return (h < t) ? ((h < i) && (i < t)) : ((h < i) || (i < t)); | |
111 | } | |
112 | ||
3aec6412 IR |
113 | void aq_ring_update_queue_state(struct aq_ring_s *ring) |
114 | { | |
115 | if (aq_ring_avail_dx(ring) <= AQ_CFG_SKB_FRAGS_MAX) | |
116 | aq_ring_queue_stop(ring); | |
117 | else if (aq_ring_avail_dx(ring) > AQ_CFG_RESTART_DESC_THRES) | |
118 | aq_ring_queue_wake(ring); | |
119 | } | |
120 | ||
121 | void aq_ring_queue_wake(struct aq_ring_s *ring) | |
122 | { | |
123 | struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic); | |
124 | ||
125 | if (__netif_subqueue_stopped(ndev, ring->idx)) { | |
126 | netif_wake_subqueue(ndev, ring->idx); | |
127 | ring->stats.tx.queue_restarts++; | |
128 | } | |
129 | } | |
130 | ||
131 | void aq_ring_queue_stop(struct aq_ring_s *ring) | |
132 | { | |
133 | struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic); | |
134 | ||
135 | if (!__netif_subqueue_stopped(ndev, ring->idx)) | |
136 | netif_stop_subqueue(ndev, ring->idx); | |
137 | } | |
138 | ||
eb36bedf | 139 | void aq_ring_tx_clean(struct aq_ring_s *self) |
018423e9 DV |
140 | { |
141 | struct device *dev = aq_nic_get_dev(self->aq_nic); | |
142 | ||
143 | for (; self->sw_head != self->hw_head; | |
144 | self->sw_head = aq_ring_next_dx(self, self->sw_head)) { | |
145 | struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; | |
146 | ||
147 | if (likely(buff->is_mapped)) { | |
c7545689 PB |
148 | if (unlikely(buff->is_sop)) { |
149 | if (!buff->is_eop && | |
150 | buff->eop_index != 0xffffU && | |
151 | (!aq_ring_dx_in_range(self->sw_head, | |
152 | buff->eop_index, | |
153 | self->hw_head))) | |
154 | break; | |
155 | ||
018423e9 DV |
156 | dma_unmap_single(dev, buff->pa, buff->len, |
157 | DMA_TO_DEVICE); | |
c7545689 | 158 | } else { |
018423e9 DV |
159 | dma_unmap_page(dev, buff->pa, buff->len, |
160 | DMA_TO_DEVICE); | |
c7545689 | 161 | } |
018423e9 DV |
162 | } |
163 | ||
164 | if (unlikely(buff->is_eop)) | |
165 | dev_kfree_skb_any(buff->skb); | |
018423e9 | 166 | |
c7545689 PB |
167 | buff->pa = 0U; |
168 | buff->eop_index = 0xffffU; | |
169 | } | |
018423e9 DV |
170 | } |
171 | ||
172 | #define AQ_SKB_ALIGN SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) | |
a54df682 PB |
173 | int aq_ring_rx_clean(struct aq_ring_s *self, |
174 | struct napi_struct *napi, | |
175 | int *work_done, | |
176 | int budget) | |
018423e9 DV |
177 | { |
178 | struct net_device *ndev = aq_nic_get_ndev(self->aq_nic); | |
179 | int err = 0; | |
180 | bool is_rsc_completed = true; | |
181 | ||
182 | for (; (self->sw_head != self->hw_head) && budget; | |
183 | self->sw_head = aq_ring_next_dx(self, self->sw_head), | |
184 | --budget, ++(*work_done)) { | |
185 | struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; | |
186 | struct sk_buff *skb = NULL; | |
187 | unsigned int next_ = 0U; | |
188 | unsigned int i = 0U; | |
189 | struct aq_ring_buff_s *buff_ = NULL; | |
190 | ||
191 | if (buff->is_error) { | |
192 | __free_pages(buff->page, 0); | |
193 | continue; | |
194 | } | |
195 | ||
196 | if (buff->is_cleaned) | |
197 | continue; | |
198 | ||
199 | if (!buff->is_eop) { | |
200 | for (next_ = buff->next, | |
201 | buff_ = &self->buff_ring[next_]; true; | |
202 | next_ = buff_->next, | |
203 | buff_ = &self->buff_ring[next_]) { | |
204 | is_rsc_completed = | |
205 | aq_ring_dx_in_range(self->sw_head, | |
206 | next_, | |
207 | self->hw_head); | |
208 | ||
209 | if (unlikely(!is_rsc_completed)) { | |
210 | is_rsc_completed = false; | |
211 | break; | |
212 | } | |
213 | ||
214 | if (buff_->is_eop) | |
215 | break; | |
216 | } | |
217 | ||
218 | if (!is_rsc_completed) { | |
219 | err = 0; | |
220 | goto err_exit; | |
221 | } | |
222 | } | |
223 | ||
224 | /* for single fragment packets use build_skb() */ | |
225 | if (buff->is_eop) { | |
226 | skb = build_skb(page_address(buff->page), | |
227 | buff->len + AQ_SKB_ALIGN); | |
228 | if (unlikely(!skb)) { | |
229 | err = -ENOMEM; | |
230 | goto err_exit; | |
231 | } | |
232 | ||
018423e9 DV |
233 | skb_put(skb, buff->len); |
234 | } else { | |
235 | skb = netdev_alloc_skb(ndev, ETH_HLEN); | |
236 | if (unlikely(!skb)) { | |
237 | err = -ENOMEM; | |
238 | goto err_exit; | |
239 | } | |
240 | skb_put(skb, ETH_HLEN); | |
241 | memcpy(skb->data, page_address(buff->page), ETH_HLEN); | |
242 | ||
243 | skb_add_rx_frag(skb, 0, buff->page, ETH_HLEN, | |
244 | buff->len - ETH_HLEN, | |
245 | SKB_TRUESIZE(buff->len - ETH_HLEN)); | |
246 | ||
247 | for (i = 1U, next_ = buff->next, | |
248 | buff_ = &self->buff_ring[next_]; true; | |
249 | next_ = buff_->next, | |
250 | buff_ = &self->buff_ring[next_], ++i) { | |
251 | skb_add_rx_frag(skb, i, buff_->page, 0, | |
252 | buff_->len, | |
253 | SKB_TRUESIZE(buff->len - | |
254 | ETH_HLEN)); | |
255 | buff_->is_cleaned = 1; | |
256 | ||
257 | if (buff_->is_eop) | |
258 | break; | |
259 | } | |
260 | } | |
261 | ||
262 | skb->protocol = eth_type_trans(skb, ndev); | |
263 | if (unlikely(buff->is_cso_err)) { | |
264 | ++self->stats.rx.errors; | |
219f1d79 | 265 | skb->ip_summed = CHECKSUM_NONE; |
018423e9 DV |
266 | } else { |
267 | if (buff->is_ip_cso) { | |
268 | __skb_incr_checksum_unnecessary(skb); | |
269 | if (buff->is_udp_cso || buff->is_tcp_cso) | |
270 | __skb_incr_checksum_unnecessary(skb); | |
271 | } else { | |
272 | skb->ip_summed = CHECKSUM_NONE; | |
273 | } | |
274 | } | |
275 | ||
276 | skb_set_hash(skb, buff->rss_hash, | |
277 | buff->is_hash_l4 ? PKT_HASH_TYPE_L4 : | |
278 | PKT_HASH_TYPE_NONE); | |
279 | ||
280 | skb_record_rx_queue(skb, self->idx); | |
281 | ||
a54df682 | 282 | napi_gro_receive(napi, skb); |
018423e9 DV |
283 | |
284 | ++self->stats.rx.packets; | |
285 | self->stats.rx.bytes += skb->len; | |
286 | } | |
287 | ||
288 | err_exit: | |
289 | return err; | |
290 | } | |
291 | ||
292 | int aq_ring_rx_fill(struct aq_ring_s *self) | |
293 | { | |
89b64388 PB |
294 | unsigned int pages_order = fls(AQ_CFG_RX_FRAME_MAX / PAGE_SIZE + |
295 | (AQ_CFG_RX_FRAME_MAX % PAGE_SIZE ? 1 : 0)) - 1; | |
018423e9 DV |
296 | struct aq_ring_buff_s *buff = NULL; |
297 | int err = 0; | |
298 | int i = 0; | |
299 | ||
300 | for (i = aq_ring_avail_dx(self); i--; | |
301 | self->sw_tail = aq_ring_next_dx(self, self->sw_tail)) { | |
302 | buff = &self->buff_ring[self->sw_tail]; | |
303 | ||
304 | buff->flags = 0U; | |
305 | buff->len = AQ_CFG_RX_FRAME_MAX; | |
306 | ||
453f85d4 | 307 | buff->page = alloc_pages(GFP_ATOMIC | __GFP_COMP, pages_order); |
018423e9 DV |
308 | if (!buff->page) { |
309 | err = -ENOMEM; | |
310 | goto err_exit; | |
311 | } | |
312 | ||
313 | buff->pa = dma_map_page(aq_nic_get_dev(self->aq_nic), | |
314 | buff->page, 0, | |
315 | AQ_CFG_RX_FRAME_MAX, DMA_FROM_DEVICE); | |
316 | ||
ff1176f6 DC |
317 | if (dma_mapping_error(aq_nic_get_dev(self->aq_nic), buff->pa)) { |
318 | err = -ENOMEM; | |
018423e9 | 319 | goto err_exit; |
ff1176f6 | 320 | } |
018423e9 DV |
321 | |
322 | buff = NULL; | |
323 | } | |
018423e9 DV |
324 | |
325 | err_exit: | |
326 | if (err < 0) { | |
327 | if (buff && buff->page) | |
328 | __free_pages(buff->page, 0); | |
329 | } | |
330 | ||
331 | return err; | |
332 | } | |
333 | ||
334 | void aq_ring_rx_deinit(struct aq_ring_s *self) | |
335 | { | |
336 | if (!self) | |
337 | goto err_exit; | |
338 | ||
339 | for (; self->sw_head != self->sw_tail; | |
340 | self->sw_head = aq_ring_next_dx(self, self->sw_head)) { | |
341 | struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; | |
342 | ||
343 | dma_unmap_page(aq_nic_get_dev(self->aq_nic), buff->pa, | |
344 | AQ_CFG_RX_FRAME_MAX, DMA_FROM_DEVICE); | |
345 | ||
346 | __free_pages(buff->page, 0); | |
347 | } | |
348 | ||
349 | err_exit:; | |
350 | } | |
351 | ||
018423e9 DV |
352 | void aq_ring_free(struct aq_ring_s *self) |
353 | { | |
354 | if (!self) | |
355 | goto err_exit; | |
356 | ||
357 | kfree(self->buff_ring); | |
358 | ||
359 | if (self->dx_ring) | |
360 | dma_free_coherent(aq_nic_get_dev(self->aq_nic), | |
361 | self->size * self->dx_size, self->dx_ring, | |
362 | self->dx_ring_pa); | |
363 | ||
364 | err_exit:; | |
365 | } |