]>
Commit | Line | Data |
---|---|---|
018423e9 DV |
1 | /* |
2 | * aQuantia Corporation Network Driver | |
3 | * Copyright (C) 2014-2017 aQuantia Corporation. All rights reserved | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify it | |
6 | * under the terms and conditions of the GNU General Public License, | |
7 | * version 2, as published by the Free Software Foundation. | |
8 | */ | |
9 | ||
10 | /* File aq_ring.c: Definition of functions for Rx/Tx rings. */ | |
11 | ||
12 | #include "aq_ring.h" | |
13 | #include "aq_nic.h" | |
14 | #include "aq_hw.h" | |
15 | ||
16 | #include <linux/netdevice.h> | |
17 | #include <linux/etherdevice.h> | |
18 | ||
19 | static struct aq_ring_s *aq_ring_alloc(struct aq_ring_s *self, | |
20 | struct aq_nic_s *aq_nic) | |
21 | { | |
22 | int err = 0; | |
23 | ||
24 | self->buff_ring = | |
25 | kcalloc(self->size, sizeof(struct aq_ring_buff_s), GFP_KERNEL); | |
26 | ||
27 | if (!self->buff_ring) { | |
28 | err = -ENOMEM; | |
29 | goto err_exit; | |
30 | } | |
31 | self->dx_ring = dma_alloc_coherent(aq_nic_get_dev(aq_nic), | |
32 | self->size * self->dx_size, | |
33 | &self->dx_ring_pa, GFP_KERNEL); | |
34 | if (!self->dx_ring) { | |
35 | err = -ENOMEM; | |
36 | goto err_exit; | |
37 | } | |
38 | ||
39 | err_exit: | |
40 | if (err < 0) { | |
41 | aq_ring_free(self); | |
42 | self = NULL; | |
43 | } | |
44 | return self; | |
45 | } | |
46 | ||
47 | struct aq_ring_s *aq_ring_tx_alloc(struct aq_ring_s *self, | |
48 | struct aq_nic_s *aq_nic, | |
49 | unsigned int idx, | |
50 | struct aq_nic_cfg_s *aq_nic_cfg) | |
51 | { | |
52 | int err = 0; | |
53 | ||
54 | self->aq_nic = aq_nic; | |
55 | self->idx = idx; | |
56 | self->size = aq_nic_cfg->txds; | |
57 | self->dx_size = aq_nic_cfg->aq_hw_caps->txd_size; | |
58 | ||
59 | self = aq_ring_alloc(self, aq_nic); | |
60 | if (!self) { | |
61 | err = -ENOMEM; | |
62 | goto err_exit; | |
63 | } | |
64 | ||
65 | err_exit: | |
66 | if (err < 0) { | |
67 | aq_ring_free(self); | |
68 | self = NULL; | |
69 | } | |
70 | return self; | |
71 | } | |
72 | ||
73 | struct aq_ring_s *aq_ring_rx_alloc(struct aq_ring_s *self, | |
74 | struct aq_nic_s *aq_nic, | |
75 | unsigned int idx, | |
76 | struct aq_nic_cfg_s *aq_nic_cfg) | |
77 | { | |
78 | int err = 0; | |
79 | ||
80 | self->aq_nic = aq_nic; | |
81 | self->idx = idx; | |
82 | self->size = aq_nic_cfg->rxds; | |
83 | self->dx_size = aq_nic_cfg->aq_hw_caps->rxd_size; | |
84 | ||
85 | self = aq_ring_alloc(self, aq_nic); | |
86 | if (!self) { | |
87 | err = -ENOMEM; | |
88 | goto err_exit; | |
89 | } | |
90 | ||
91 | err_exit: | |
92 | if (err < 0) { | |
93 | aq_ring_free(self); | |
94 | self = NULL; | |
95 | } | |
96 | return self; | |
97 | } | |
98 | ||
99 | int aq_ring_init(struct aq_ring_s *self) | |
100 | { | |
101 | self->hw_head = 0; | |
102 | self->sw_head = 0; | |
103 | self->sw_tail = 0; | |
9f0dd8c3 | 104 | spin_lock_init(&self->header.lock); |
018423e9 DV |
105 | return 0; |
106 | } | |
107 | ||
eb36bedf | 108 | void aq_ring_tx_clean(struct aq_ring_s *self) |
018423e9 DV |
109 | { |
110 | struct device *dev = aq_nic_get_dev(self->aq_nic); | |
111 | ||
112 | for (; self->sw_head != self->hw_head; | |
113 | self->sw_head = aq_ring_next_dx(self, self->sw_head)) { | |
114 | struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; | |
115 | ||
116 | if (likely(buff->is_mapped)) { | |
117 | if (unlikely(buff->is_sop)) | |
118 | dma_unmap_single(dev, buff->pa, buff->len, | |
119 | DMA_TO_DEVICE); | |
120 | else | |
121 | dma_unmap_page(dev, buff->pa, buff->len, | |
122 | DMA_TO_DEVICE); | |
123 | } | |
124 | ||
125 | if (unlikely(buff->is_eop)) | |
126 | dev_kfree_skb_any(buff->skb); | |
127 | } | |
018423e9 DV |
128 | } |
129 | ||
130 | static inline unsigned int aq_ring_dx_in_range(unsigned int h, unsigned int i, | |
131 | unsigned int t) | |
132 | { | |
133 | return (h < t) ? ((h < i) && (i < t)) : ((h < i) || (i < t)); | |
134 | } | |
135 | ||
136 | #define AQ_SKB_ALIGN SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) | |
137 | int aq_ring_rx_clean(struct aq_ring_s *self, int *work_done, int budget) | |
138 | { | |
139 | struct net_device *ndev = aq_nic_get_ndev(self->aq_nic); | |
140 | int err = 0; | |
141 | bool is_rsc_completed = true; | |
142 | ||
143 | for (; (self->sw_head != self->hw_head) && budget; | |
144 | self->sw_head = aq_ring_next_dx(self, self->sw_head), | |
145 | --budget, ++(*work_done)) { | |
146 | struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; | |
147 | struct sk_buff *skb = NULL; | |
148 | unsigned int next_ = 0U; | |
149 | unsigned int i = 0U; | |
150 | struct aq_ring_buff_s *buff_ = NULL; | |
151 | ||
152 | if (buff->is_error) { | |
153 | __free_pages(buff->page, 0); | |
154 | continue; | |
155 | } | |
156 | ||
157 | if (buff->is_cleaned) | |
158 | continue; | |
159 | ||
160 | if (!buff->is_eop) { | |
161 | for (next_ = buff->next, | |
162 | buff_ = &self->buff_ring[next_]; true; | |
163 | next_ = buff_->next, | |
164 | buff_ = &self->buff_ring[next_]) { | |
165 | is_rsc_completed = | |
166 | aq_ring_dx_in_range(self->sw_head, | |
167 | next_, | |
168 | self->hw_head); | |
169 | ||
170 | if (unlikely(!is_rsc_completed)) { | |
171 | is_rsc_completed = false; | |
172 | break; | |
173 | } | |
174 | ||
175 | if (buff_->is_eop) | |
176 | break; | |
177 | } | |
178 | ||
179 | if (!is_rsc_completed) { | |
180 | err = 0; | |
181 | goto err_exit; | |
182 | } | |
183 | } | |
184 | ||
185 | /* for single fragment packets use build_skb() */ | |
186 | if (buff->is_eop) { | |
187 | skb = build_skb(page_address(buff->page), | |
188 | buff->len + AQ_SKB_ALIGN); | |
189 | if (unlikely(!skb)) { | |
190 | err = -ENOMEM; | |
191 | goto err_exit; | |
192 | } | |
193 | ||
018423e9 DV |
194 | skb_put(skb, buff->len); |
195 | } else { | |
196 | skb = netdev_alloc_skb(ndev, ETH_HLEN); | |
197 | if (unlikely(!skb)) { | |
198 | err = -ENOMEM; | |
199 | goto err_exit; | |
200 | } | |
201 | skb_put(skb, ETH_HLEN); | |
202 | memcpy(skb->data, page_address(buff->page), ETH_HLEN); | |
203 | ||
204 | skb_add_rx_frag(skb, 0, buff->page, ETH_HLEN, | |
205 | buff->len - ETH_HLEN, | |
206 | SKB_TRUESIZE(buff->len - ETH_HLEN)); | |
207 | ||
208 | for (i = 1U, next_ = buff->next, | |
209 | buff_ = &self->buff_ring[next_]; true; | |
210 | next_ = buff_->next, | |
211 | buff_ = &self->buff_ring[next_], ++i) { | |
212 | skb_add_rx_frag(skb, i, buff_->page, 0, | |
213 | buff_->len, | |
214 | SKB_TRUESIZE(buff->len - | |
215 | ETH_HLEN)); | |
216 | buff_->is_cleaned = 1; | |
217 | ||
218 | if (buff_->is_eop) | |
219 | break; | |
220 | } | |
221 | } | |
222 | ||
223 | skb->protocol = eth_type_trans(skb, ndev); | |
224 | if (unlikely(buff->is_cso_err)) { | |
225 | ++self->stats.rx.errors; | |
226 | __skb_mark_checksum_bad(skb); | |
227 | } else { | |
228 | if (buff->is_ip_cso) { | |
229 | __skb_incr_checksum_unnecessary(skb); | |
230 | if (buff->is_udp_cso || buff->is_tcp_cso) | |
231 | __skb_incr_checksum_unnecessary(skb); | |
232 | } else { | |
233 | skb->ip_summed = CHECKSUM_NONE; | |
234 | } | |
235 | } | |
236 | ||
237 | skb_set_hash(skb, buff->rss_hash, | |
238 | buff->is_hash_l4 ? PKT_HASH_TYPE_L4 : | |
239 | PKT_HASH_TYPE_NONE); | |
240 | ||
241 | skb_record_rx_queue(skb, self->idx); | |
242 | ||
243 | netif_receive_skb(skb); | |
244 | ||
245 | ++self->stats.rx.packets; | |
246 | self->stats.rx.bytes += skb->len; | |
247 | } | |
248 | ||
249 | err_exit: | |
250 | return err; | |
251 | } | |
252 | ||
253 | int aq_ring_rx_fill(struct aq_ring_s *self) | |
254 | { | |
89b64388 PB |
255 | unsigned int pages_order = fls(AQ_CFG_RX_FRAME_MAX / PAGE_SIZE + |
256 | (AQ_CFG_RX_FRAME_MAX % PAGE_SIZE ? 1 : 0)) - 1; | |
018423e9 DV |
257 | struct aq_ring_buff_s *buff = NULL; |
258 | int err = 0; | |
259 | int i = 0; | |
260 | ||
261 | for (i = aq_ring_avail_dx(self); i--; | |
262 | self->sw_tail = aq_ring_next_dx(self, self->sw_tail)) { | |
263 | buff = &self->buff_ring[self->sw_tail]; | |
264 | ||
265 | buff->flags = 0U; | |
266 | buff->len = AQ_CFG_RX_FRAME_MAX; | |
267 | ||
268 | buff->page = alloc_pages(GFP_ATOMIC | __GFP_COLD | | |
89b64388 | 269 | __GFP_COMP, pages_order); |
018423e9 DV |
270 | if (!buff->page) { |
271 | err = -ENOMEM; | |
272 | goto err_exit; | |
273 | } | |
274 | ||
275 | buff->pa = dma_map_page(aq_nic_get_dev(self->aq_nic), | |
276 | buff->page, 0, | |
277 | AQ_CFG_RX_FRAME_MAX, DMA_FROM_DEVICE); | |
278 | ||
ff1176f6 DC |
279 | if (dma_mapping_error(aq_nic_get_dev(self->aq_nic), buff->pa)) { |
280 | err = -ENOMEM; | |
018423e9 | 281 | goto err_exit; |
ff1176f6 | 282 | } |
018423e9 DV |
283 | |
284 | buff = NULL; | |
285 | } | |
018423e9 DV |
286 | |
287 | err_exit: | |
288 | if (err < 0) { | |
289 | if (buff && buff->page) | |
290 | __free_pages(buff->page, 0); | |
291 | } | |
292 | ||
293 | return err; | |
294 | } | |
295 | ||
296 | void aq_ring_rx_deinit(struct aq_ring_s *self) | |
297 | { | |
298 | if (!self) | |
299 | goto err_exit; | |
300 | ||
301 | for (; self->sw_head != self->sw_tail; | |
302 | self->sw_head = aq_ring_next_dx(self, self->sw_head)) { | |
303 | struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; | |
304 | ||
305 | dma_unmap_page(aq_nic_get_dev(self->aq_nic), buff->pa, | |
306 | AQ_CFG_RX_FRAME_MAX, DMA_FROM_DEVICE); | |
307 | ||
308 | __free_pages(buff->page, 0); | |
309 | } | |
310 | ||
311 | err_exit:; | |
312 | } | |
313 | ||
018423e9 DV |
314 | void aq_ring_free(struct aq_ring_s *self) |
315 | { | |
316 | if (!self) | |
317 | goto err_exit; | |
318 | ||
319 | kfree(self->buff_ring); | |
320 | ||
321 | if (self->dx_ring) | |
322 | dma_free_coherent(aq_nic_get_dev(self->aq_nic), | |
323 | self->size * self->dx_size, self->dx_ring, | |
324 | self->dx_ring_pa); | |
325 | ||
326 | err_exit:; | |
327 | } |