]>
Commit | Line | Data |
---|---|---|
5a6681e2 EC |
1 | /**************************************************************************** |
2 | * Driver for Solarflare network controllers and boards | |
3 | * Copyright 2005-2006 Fen Systems Ltd. | |
4 | * Copyright 2005-2013 Solarflare Communications Inc. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify it | |
7 | * under the terms of the GNU General Public License version 2 as published | |
8 | * by the Free Software Foundation, incorporated herein by reference. | |
9 | */ | |
10 | ||
11 | #include <linux/pci.h> | |
12 | #include <linux/tcp.h> | |
13 | #include <linux/ip.h> | |
14 | #include <linux/in.h> | |
15 | #include <linux/ipv6.h> | |
16 | #include <linux/slab.h> | |
17 | #include <net/ipv6.h> | |
18 | #include <linux/if_ether.h> | |
19 | #include <linux/highmem.h> | |
20 | #include <linux/cache.h> | |
21 | #include "net_driver.h" | |
22 | #include "efx.h" | |
23 | #include "io.h" | |
24 | #include "nic.h" | |
25 | #include "tx.h" | |
26 | #include "workarounds.h" | |
27 | ||
28 | static inline u8 *ef4_tx_get_copy_buffer(struct ef4_tx_queue *tx_queue, | |
29 | struct ef4_tx_buffer *buffer) | |
30 | { | |
31 | unsigned int index = ef4_tx_queue_get_insert_index(tx_queue); | |
32 | struct ef4_buffer *page_buf = | |
33 | &tx_queue->cb_page[index >> (PAGE_SHIFT - EF4_TX_CB_ORDER)]; | |
34 | unsigned int offset = | |
35 | ((index << EF4_TX_CB_ORDER) + NET_IP_ALIGN) & (PAGE_SIZE - 1); | |
36 | ||
37 | if (unlikely(!page_buf->addr) && | |
38 | ef4_nic_alloc_buffer(tx_queue->efx, page_buf, PAGE_SIZE, | |
39 | GFP_ATOMIC)) | |
40 | return NULL; | |
41 | buffer->dma_addr = page_buf->dma_addr + offset; | |
42 | buffer->unmap_len = 0; | |
43 | return (u8 *)page_buf->addr + offset; | |
44 | } | |
45 | ||
46 | u8 *ef4_tx_get_copy_buffer_limited(struct ef4_tx_queue *tx_queue, | |
47 | struct ef4_tx_buffer *buffer, size_t len) | |
48 | { | |
49 | if (len > EF4_TX_CB_SIZE) | |
50 | return NULL; | |
51 | return ef4_tx_get_copy_buffer(tx_queue, buffer); | |
52 | } | |
53 | ||
54 | static void ef4_dequeue_buffer(struct ef4_tx_queue *tx_queue, | |
55 | struct ef4_tx_buffer *buffer, | |
56 | unsigned int *pkts_compl, | |
57 | unsigned int *bytes_compl) | |
58 | { | |
59 | if (buffer->unmap_len) { | |
60 | struct device *dma_dev = &tx_queue->efx->pci_dev->dev; | |
61 | dma_addr_t unmap_addr = buffer->dma_addr - buffer->dma_offset; | |
62 | if (buffer->flags & EF4_TX_BUF_MAP_SINGLE) | |
63 | dma_unmap_single(dma_dev, unmap_addr, buffer->unmap_len, | |
64 | DMA_TO_DEVICE); | |
65 | else | |
66 | dma_unmap_page(dma_dev, unmap_addr, buffer->unmap_len, | |
67 | DMA_TO_DEVICE); | |
68 | buffer->unmap_len = 0; | |
69 | } | |
70 | ||
71 | if (buffer->flags & EF4_TX_BUF_SKB) { | |
72 | (*pkts_compl)++; | |
73 | (*bytes_compl) += buffer->skb->len; | |
74 | dev_consume_skb_any((struct sk_buff *)buffer->skb); | |
75 | netif_vdbg(tx_queue->efx, tx_done, tx_queue->efx->net_dev, | |
76 | "TX queue %d transmission id %x complete\n", | |
77 | tx_queue->queue, tx_queue->read_count); | |
78 | } | |
79 | ||
80 | buffer->len = 0; | |
81 | buffer->flags = 0; | |
82 | } | |
83 | ||
84 | unsigned int ef4_tx_max_skb_descs(struct ef4_nic *efx) | |
85 | { | |
86 | /* This is probably too much since we don't have any TSO support; | |
87 | * it's a left-over from when we had Software TSO. But it's safer | |
88 | * to leave it as-is than try to determine a new bound. | |
89 | */ | |
90 | /* Header and payload descriptor for each output segment, plus | |
91 | * one for every input fragment boundary within a segment | |
92 | */ | |
93 | unsigned int max_descs = EF4_TSO_MAX_SEGS * 2 + MAX_SKB_FRAGS; | |
94 | ||
95 | /* Possibly one more per segment for the alignment workaround, | |
96 | * or for option descriptors | |
97 | */ | |
98 | if (EF4_WORKAROUND_5391(efx)) | |
99 | max_descs += EF4_TSO_MAX_SEGS; | |
100 | ||
101 | /* Possibly more for PCIe page boundaries within input fragments */ | |
102 | if (PAGE_SIZE > EF4_PAGE_SIZE) | |
103 | max_descs += max_t(unsigned int, MAX_SKB_FRAGS, | |
104 | DIV_ROUND_UP(GSO_MAX_SIZE, EF4_PAGE_SIZE)); | |
105 | ||
106 | return max_descs; | |
107 | } | |
108 | ||
109 | static void ef4_tx_maybe_stop_queue(struct ef4_tx_queue *txq1) | |
110 | { | |
111 | /* We need to consider both queues that the net core sees as one */ | |
112 | struct ef4_tx_queue *txq2 = ef4_tx_queue_partner(txq1); | |
113 | struct ef4_nic *efx = txq1->efx; | |
114 | unsigned int fill_level; | |
115 | ||
116 | fill_level = max(txq1->insert_count - txq1->old_read_count, | |
117 | txq2->insert_count - txq2->old_read_count); | |
118 | if (likely(fill_level < efx->txq_stop_thresh)) | |
119 | return; | |
120 | ||
121 | /* We used the stale old_read_count above, which gives us a | |
122 | * pessimistic estimate of the fill level (which may even | |
123 | * validly be >= efx->txq_entries). Now try again using | |
124 | * read_count (more likely to be a cache miss). | |
125 | * | |
126 | * If we read read_count and then conditionally stop the | |
127 | * queue, it is possible for the completion path to race with | |
128 | * us and complete all outstanding descriptors in the middle, | |
129 | * after which there will be no more completions to wake it. | |
130 | * Therefore we stop the queue first, then read read_count | |
131 | * (with a memory barrier to ensure the ordering), then | |
132 | * restart the queue if the fill level turns out to be low | |
133 | * enough. | |
134 | */ | |
135 | netif_tx_stop_queue(txq1->core_txq); | |
136 | smp_mb(); | |
137 | txq1->old_read_count = ACCESS_ONCE(txq1->read_count); | |
138 | txq2->old_read_count = ACCESS_ONCE(txq2->read_count); | |
139 | ||
140 | fill_level = max(txq1->insert_count - txq1->old_read_count, | |
141 | txq2->insert_count - txq2->old_read_count); | |
142 | EF4_BUG_ON_PARANOID(fill_level >= efx->txq_entries); | |
143 | if (likely(fill_level < efx->txq_stop_thresh)) { | |
144 | smp_mb(); | |
145 | if (likely(!efx->loopback_selftest)) | |
146 | netif_tx_start_queue(txq1->core_txq); | |
147 | } | |
148 | } | |
149 | ||
150 | static int ef4_enqueue_skb_copy(struct ef4_tx_queue *tx_queue, | |
151 | struct sk_buff *skb) | |
152 | { | |
153 | unsigned int min_len = tx_queue->tx_min_size; | |
154 | unsigned int copy_len = skb->len; | |
155 | struct ef4_tx_buffer *buffer; | |
156 | u8 *copy_buffer; | |
157 | int rc; | |
158 | ||
159 | EF4_BUG_ON_PARANOID(copy_len > EF4_TX_CB_SIZE); | |
160 | ||
161 | buffer = ef4_tx_queue_get_insert_buffer(tx_queue); | |
162 | ||
163 | copy_buffer = ef4_tx_get_copy_buffer(tx_queue, buffer); | |
164 | if (unlikely(!copy_buffer)) | |
165 | return -ENOMEM; | |
166 | ||
167 | rc = skb_copy_bits(skb, 0, copy_buffer, copy_len); | |
168 | EF4_WARN_ON_PARANOID(rc); | |
169 | if (unlikely(copy_len < min_len)) { | |
170 | memset(copy_buffer + copy_len, 0, min_len - copy_len); | |
171 | buffer->len = min_len; | |
172 | } else { | |
173 | buffer->len = copy_len; | |
174 | } | |
175 | ||
176 | buffer->skb = skb; | |
177 | buffer->flags = EF4_TX_BUF_SKB; | |
178 | ||
179 | ++tx_queue->insert_count; | |
180 | return rc; | |
181 | } | |
182 | ||
183 | static struct ef4_tx_buffer *ef4_tx_map_chunk(struct ef4_tx_queue *tx_queue, | |
184 | dma_addr_t dma_addr, | |
185 | size_t len) | |
186 | { | |
187 | const struct ef4_nic_type *nic_type = tx_queue->efx->type; | |
188 | struct ef4_tx_buffer *buffer; | |
189 | unsigned int dma_len; | |
190 | ||
191 | /* Map the fragment taking account of NIC-dependent DMA limits. */ | |
192 | do { | |
193 | buffer = ef4_tx_queue_get_insert_buffer(tx_queue); | |
194 | dma_len = nic_type->tx_limit_len(tx_queue, dma_addr, len); | |
195 | ||
196 | buffer->len = dma_len; | |
197 | buffer->dma_addr = dma_addr; | |
198 | buffer->flags = EF4_TX_BUF_CONT; | |
199 | len -= dma_len; | |
200 | dma_addr += dma_len; | |
201 | ++tx_queue->insert_count; | |
202 | } while (len); | |
203 | ||
204 | return buffer; | |
205 | } | |
206 | ||
207 | /* Map all data from an SKB for DMA and create descriptors on the queue. | |
208 | */ | |
209 | static int ef4_tx_map_data(struct ef4_tx_queue *tx_queue, struct sk_buff *skb) | |
210 | { | |
211 | struct ef4_nic *efx = tx_queue->efx; | |
212 | struct device *dma_dev = &efx->pci_dev->dev; | |
213 | unsigned int frag_index, nr_frags; | |
214 | dma_addr_t dma_addr, unmap_addr; | |
215 | unsigned short dma_flags; | |
216 | size_t len, unmap_len; | |
217 | ||
218 | nr_frags = skb_shinfo(skb)->nr_frags; | |
219 | frag_index = 0; | |
220 | ||
221 | /* Map header data. */ | |
222 | len = skb_headlen(skb); | |
223 | dma_addr = dma_map_single(dma_dev, skb->data, len, DMA_TO_DEVICE); | |
224 | dma_flags = EF4_TX_BUF_MAP_SINGLE; | |
225 | unmap_len = len; | |
226 | unmap_addr = dma_addr; | |
227 | ||
228 | if (unlikely(dma_mapping_error(dma_dev, dma_addr))) | |
229 | return -EIO; | |
230 | ||
231 | /* Add descriptors for each fragment. */ | |
232 | do { | |
233 | struct ef4_tx_buffer *buffer; | |
234 | skb_frag_t *fragment; | |
235 | ||
236 | buffer = ef4_tx_map_chunk(tx_queue, dma_addr, len); | |
237 | ||
238 | /* The final descriptor for a fragment is responsible for | |
239 | * unmapping the whole fragment. | |
240 | */ | |
241 | buffer->flags = EF4_TX_BUF_CONT | dma_flags; | |
242 | buffer->unmap_len = unmap_len; | |
243 | buffer->dma_offset = buffer->dma_addr - unmap_addr; | |
244 | ||
245 | if (frag_index >= nr_frags) { | |
246 | /* Store SKB details with the final buffer for | |
247 | * the completion. | |
248 | */ | |
249 | buffer->skb = skb; | |
250 | buffer->flags = EF4_TX_BUF_SKB | dma_flags; | |
251 | return 0; | |
252 | } | |
253 | ||
254 | /* Move on to the next fragment. */ | |
255 | fragment = &skb_shinfo(skb)->frags[frag_index++]; | |
256 | len = skb_frag_size(fragment); | |
257 | dma_addr = skb_frag_dma_map(dma_dev, fragment, | |
258 | 0, len, DMA_TO_DEVICE); | |
259 | dma_flags = 0; | |
260 | unmap_len = len; | |
261 | unmap_addr = dma_addr; | |
262 | ||
263 | if (unlikely(dma_mapping_error(dma_dev, dma_addr))) | |
264 | return -EIO; | |
265 | } while (1); | |
266 | } | |
267 | ||
268 | /* Remove buffers put into a tx_queue. None of the buffers must have | |
269 | * an skb attached. | |
270 | */ | |
271 | static void ef4_enqueue_unwind(struct ef4_tx_queue *tx_queue) | |
272 | { | |
273 | struct ef4_tx_buffer *buffer; | |
274 | ||
275 | /* Work backwards until we hit the original insert pointer value */ | |
276 | while (tx_queue->insert_count != tx_queue->write_count) { | |
277 | --tx_queue->insert_count; | |
278 | buffer = __ef4_tx_queue_get_insert_buffer(tx_queue); | |
279 | ef4_dequeue_buffer(tx_queue, buffer, NULL, NULL); | |
280 | } | |
281 | } | |
282 | ||
283 | /* | |
284 | * Add a socket buffer to a TX queue | |
285 | * | |
286 | * This maps all fragments of a socket buffer for DMA and adds them to | |
287 | * the TX queue. The queue's insert pointer will be incremented by | |
288 | * the number of fragments in the socket buffer. | |
289 | * | |
290 | * If any DMA mapping fails, any mapped fragments will be unmapped, | |
291 | * the queue's insert pointer will be restored to its original value. | |
292 | * | |
293 | * This function is split out from ef4_hard_start_xmit to allow the | |
294 | * loopback test to direct packets via specific TX queues. | |
295 | * | |
296 | * Returns NETDEV_TX_OK. | |
297 | * You must hold netif_tx_lock() to call this function. | |
298 | */ | |
299 | netdev_tx_t ef4_enqueue_skb(struct ef4_tx_queue *tx_queue, struct sk_buff *skb) | |
300 | { | |
301 | bool data_mapped = false; | |
302 | unsigned int skb_len; | |
303 | ||
304 | skb_len = skb->len; | |
305 | EF4_WARN_ON_PARANOID(skb_is_gso(skb)); | |
306 | ||
307 | if (skb_len < tx_queue->tx_min_size || | |
308 | (skb->data_len && skb_len <= EF4_TX_CB_SIZE)) { | |
309 | /* Pad short packets or coalesce short fragmented packets. */ | |
310 | if (ef4_enqueue_skb_copy(tx_queue, skb)) | |
311 | goto err; | |
312 | tx_queue->cb_packets++; | |
313 | data_mapped = true; | |
314 | } | |
315 | ||
316 | /* Map for DMA and create descriptors if we haven't done so already. */ | |
317 | if (!data_mapped && (ef4_tx_map_data(tx_queue, skb))) | |
318 | goto err; | |
319 | ||
320 | /* Update BQL */ | |
321 | netdev_tx_sent_queue(tx_queue->core_txq, skb_len); | |
322 | ||
323 | /* Pass off to hardware */ | |
324 | if (!skb->xmit_more || netif_xmit_stopped(tx_queue->core_txq)) { | |
325 | struct ef4_tx_queue *txq2 = ef4_tx_queue_partner(tx_queue); | |
326 | ||
327 | /* There could be packets left on the partner queue if those | |
328 | * SKBs had skb->xmit_more set. If we do not push those they | |
329 | * could be left for a long time and cause a netdev watchdog. | |
330 | */ | |
331 | if (txq2->xmit_more_available) | |
332 | ef4_nic_push_buffers(txq2); | |
333 | ||
334 | ef4_nic_push_buffers(tx_queue); | |
335 | } else { | |
336 | tx_queue->xmit_more_available = skb->xmit_more; | |
337 | } | |
338 | ||
339 | tx_queue->tx_packets++; | |
340 | ||
341 | ef4_tx_maybe_stop_queue(tx_queue); | |
342 | ||
343 | return NETDEV_TX_OK; | |
344 | ||
345 | ||
346 | err: | |
347 | ef4_enqueue_unwind(tx_queue); | |
348 | dev_kfree_skb_any(skb); | |
349 | return NETDEV_TX_OK; | |
350 | } | |
351 | ||
352 | /* Remove packets from the TX queue | |
353 | * | |
354 | * This removes packets from the TX queue, up to and including the | |
355 | * specified index. | |
356 | */ | |
357 | static void ef4_dequeue_buffers(struct ef4_tx_queue *tx_queue, | |
358 | unsigned int index, | |
359 | unsigned int *pkts_compl, | |
360 | unsigned int *bytes_compl) | |
361 | { | |
362 | struct ef4_nic *efx = tx_queue->efx; | |
363 | unsigned int stop_index, read_ptr; | |
364 | ||
365 | stop_index = (index + 1) & tx_queue->ptr_mask; | |
366 | read_ptr = tx_queue->read_count & tx_queue->ptr_mask; | |
367 | ||
368 | while (read_ptr != stop_index) { | |
369 | struct ef4_tx_buffer *buffer = &tx_queue->buffer[read_ptr]; | |
370 | ||
371 | if (!(buffer->flags & EF4_TX_BUF_OPTION) && | |
372 | unlikely(buffer->len == 0)) { | |
373 | netif_err(efx, tx_err, efx->net_dev, | |
374 | "TX queue %d spurious TX completion id %x\n", | |
375 | tx_queue->queue, read_ptr); | |
376 | ef4_schedule_reset(efx, RESET_TYPE_TX_SKIP); | |
377 | return; | |
378 | } | |
379 | ||
380 | ef4_dequeue_buffer(tx_queue, buffer, pkts_compl, bytes_compl); | |
381 | ||
382 | ++tx_queue->read_count; | |
383 | read_ptr = tx_queue->read_count & tx_queue->ptr_mask; | |
384 | } | |
385 | } | |
386 | ||
387 | /* Initiate a packet transmission. We use one channel per CPU | |
388 | * (sharing when we have more CPUs than channels). On Falcon, the TX | |
389 | * completion events will be directed back to the CPU that transmitted | |
390 | * the packet, which should be cache-efficient. | |
391 | * | |
392 | * Context: non-blocking. | |
393 | * Note that returning anything other than NETDEV_TX_OK will cause the | |
394 | * OS to free the skb. | |
395 | */ | |
396 | netdev_tx_t ef4_hard_start_xmit(struct sk_buff *skb, | |
397 | struct net_device *net_dev) | |
398 | { | |
399 | struct ef4_nic *efx = netdev_priv(net_dev); | |
400 | struct ef4_tx_queue *tx_queue; | |
401 | unsigned index, type; | |
402 | ||
403 | EF4_WARN_ON_PARANOID(!netif_device_present(net_dev)); | |
404 | ||
405 | index = skb_get_queue_mapping(skb); | |
406 | type = skb->ip_summed == CHECKSUM_PARTIAL ? EF4_TXQ_TYPE_OFFLOAD : 0; | |
407 | if (index >= efx->n_tx_channels) { | |
408 | index -= efx->n_tx_channels; | |
409 | type |= EF4_TXQ_TYPE_HIGHPRI; | |
410 | } | |
411 | tx_queue = ef4_get_tx_queue(efx, index, type); | |
412 | ||
413 | return ef4_enqueue_skb(tx_queue, skb); | |
414 | } | |
415 | ||
416 | void ef4_init_tx_queue_core_txq(struct ef4_tx_queue *tx_queue) | |
417 | { | |
418 | struct ef4_nic *efx = tx_queue->efx; | |
419 | ||
420 | /* Must be inverse of queue lookup in ef4_hard_start_xmit() */ | |
421 | tx_queue->core_txq = | |
422 | netdev_get_tx_queue(efx->net_dev, | |
423 | tx_queue->queue / EF4_TXQ_TYPES + | |
424 | ((tx_queue->queue & EF4_TXQ_TYPE_HIGHPRI) ? | |
425 | efx->n_tx_channels : 0)); | |
426 | } | |
427 | ||
2572ac53 | 428 | int ef4_setup_tc(struct net_device *net_dev, enum tc_setup_type type, |
de4784ca | 429 | void *type_data) |
5a6681e2 EC |
430 | { |
431 | struct ef4_nic *efx = netdev_priv(net_dev); | |
de4784ca | 432 | struct tc_mqprio_qopt *mqprio = type_data; |
5a6681e2 EC |
433 | struct ef4_channel *channel; |
434 | struct ef4_tx_queue *tx_queue; | |
435 | unsigned tc, num_tc; | |
436 | int rc; | |
437 | ||
2572ac53 | 438 | if (type != TC_SETUP_MQPRIO) |
38cf0426 | 439 | return -EOPNOTSUPP; |
5a6681e2 | 440 | |
de4784ca | 441 | num_tc = mqprio->num_tc; |
5a6681e2 EC |
442 | |
443 | if (ef4_nic_rev(efx) < EF4_REV_FALCON_B0 || num_tc > EF4_MAX_TX_TC) | |
444 | return -EINVAL; | |
445 | ||
de4784ca | 446 | mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; |
56f36acd | 447 | |
5a6681e2 EC |
448 | if (num_tc == net_dev->num_tc) |
449 | return 0; | |
450 | ||
451 | for (tc = 0; tc < num_tc; tc++) { | |
452 | net_dev->tc_to_txq[tc].offset = tc * efx->n_tx_channels; | |
453 | net_dev->tc_to_txq[tc].count = efx->n_tx_channels; | |
454 | } | |
455 | ||
456 | if (num_tc > net_dev->num_tc) { | |
457 | /* Initialise high-priority queues as necessary */ | |
458 | ef4_for_each_channel(channel, efx) { | |
459 | ef4_for_each_possible_channel_tx_queue(tx_queue, | |
460 | channel) { | |
461 | if (!(tx_queue->queue & EF4_TXQ_TYPE_HIGHPRI)) | |
462 | continue; | |
463 | if (!tx_queue->buffer) { | |
464 | rc = ef4_probe_tx_queue(tx_queue); | |
465 | if (rc) | |
466 | return rc; | |
467 | } | |
468 | if (!tx_queue->initialised) | |
469 | ef4_init_tx_queue(tx_queue); | |
470 | ef4_init_tx_queue_core_txq(tx_queue); | |
471 | } | |
472 | } | |
473 | } else { | |
474 | /* Reduce number of classes before number of queues */ | |
475 | net_dev->num_tc = num_tc; | |
476 | } | |
477 | ||
478 | rc = netif_set_real_num_tx_queues(net_dev, | |
479 | max_t(int, num_tc, 1) * | |
480 | efx->n_tx_channels); | |
481 | if (rc) | |
482 | return rc; | |
483 | ||
484 | /* Do not destroy high-priority queues when they become | |
485 | * unused. We would have to flush them first, and it is | |
486 | * fairly difficult to flush a subset of TX queues. Leave | |
487 | * it to ef4_fini_channels(). | |
488 | */ | |
489 | ||
490 | net_dev->num_tc = num_tc; | |
491 | return 0; | |
492 | } | |
493 | ||
494 | void ef4_xmit_done(struct ef4_tx_queue *tx_queue, unsigned int index) | |
495 | { | |
496 | unsigned fill_level; | |
497 | struct ef4_nic *efx = tx_queue->efx; | |
498 | struct ef4_tx_queue *txq2; | |
499 | unsigned int pkts_compl = 0, bytes_compl = 0; | |
500 | ||
501 | EF4_BUG_ON_PARANOID(index > tx_queue->ptr_mask); | |
502 | ||
503 | ef4_dequeue_buffers(tx_queue, index, &pkts_compl, &bytes_compl); | |
504 | tx_queue->pkts_compl += pkts_compl; | |
505 | tx_queue->bytes_compl += bytes_compl; | |
506 | ||
507 | if (pkts_compl > 1) | |
508 | ++tx_queue->merge_events; | |
509 | ||
510 | /* See if we need to restart the netif queue. This memory | |
511 | * barrier ensures that we write read_count (inside | |
512 | * ef4_dequeue_buffers()) before reading the queue status. | |
513 | */ | |
514 | smp_mb(); | |
515 | if (unlikely(netif_tx_queue_stopped(tx_queue->core_txq)) && | |
516 | likely(efx->port_enabled) && | |
517 | likely(netif_device_present(efx->net_dev))) { | |
518 | txq2 = ef4_tx_queue_partner(tx_queue); | |
519 | fill_level = max(tx_queue->insert_count - tx_queue->read_count, | |
520 | txq2->insert_count - txq2->read_count); | |
521 | if (fill_level <= efx->txq_wake_thresh) | |
522 | netif_tx_wake_queue(tx_queue->core_txq); | |
523 | } | |
524 | ||
525 | /* Check whether the hardware queue is now empty */ | |
526 | if ((int)(tx_queue->read_count - tx_queue->old_write_count) >= 0) { | |
527 | tx_queue->old_write_count = ACCESS_ONCE(tx_queue->write_count); | |
528 | if (tx_queue->read_count == tx_queue->old_write_count) { | |
529 | smp_mb(); | |
530 | tx_queue->empty_read_count = | |
531 | tx_queue->read_count | EF4_EMPTY_COUNT_VALID; | |
532 | } | |
533 | } | |
534 | } | |
535 | ||
536 | static unsigned int ef4_tx_cb_page_count(struct ef4_tx_queue *tx_queue) | |
537 | { | |
538 | return DIV_ROUND_UP(tx_queue->ptr_mask + 1, PAGE_SIZE >> EF4_TX_CB_ORDER); | |
539 | } | |
540 | ||
541 | int ef4_probe_tx_queue(struct ef4_tx_queue *tx_queue) | |
542 | { | |
543 | struct ef4_nic *efx = tx_queue->efx; | |
544 | unsigned int entries; | |
545 | int rc; | |
546 | ||
547 | /* Create the smallest power-of-two aligned ring */ | |
548 | entries = max(roundup_pow_of_two(efx->txq_entries), EF4_MIN_DMAQ_SIZE); | |
549 | EF4_BUG_ON_PARANOID(entries > EF4_MAX_DMAQ_SIZE); | |
550 | tx_queue->ptr_mask = entries - 1; | |
551 | ||
552 | netif_dbg(efx, probe, efx->net_dev, | |
553 | "creating TX queue %d size %#x mask %#x\n", | |
554 | tx_queue->queue, efx->txq_entries, tx_queue->ptr_mask); | |
555 | ||
556 | /* Allocate software ring */ | |
557 | tx_queue->buffer = kcalloc(entries, sizeof(*tx_queue->buffer), | |
558 | GFP_KERNEL); | |
559 | if (!tx_queue->buffer) | |
560 | return -ENOMEM; | |
561 | ||
562 | tx_queue->cb_page = kcalloc(ef4_tx_cb_page_count(tx_queue), | |
563 | sizeof(tx_queue->cb_page[0]), GFP_KERNEL); | |
564 | if (!tx_queue->cb_page) { | |
565 | rc = -ENOMEM; | |
566 | goto fail1; | |
567 | } | |
568 | ||
569 | /* Allocate hardware ring */ | |
570 | rc = ef4_nic_probe_tx(tx_queue); | |
571 | if (rc) | |
572 | goto fail2; | |
573 | ||
574 | return 0; | |
575 | ||
576 | fail2: | |
577 | kfree(tx_queue->cb_page); | |
578 | tx_queue->cb_page = NULL; | |
579 | fail1: | |
580 | kfree(tx_queue->buffer); | |
581 | tx_queue->buffer = NULL; | |
582 | return rc; | |
583 | } | |
584 | ||
585 | void ef4_init_tx_queue(struct ef4_tx_queue *tx_queue) | |
586 | { | |
587 | struct ef4_nic *efx = tx_queue->efx; | |
588 | ||
589 | netif_dbg(efx, drv, efx->net_dev, | |
590 | "initialising TX queue %d\n", tx_queue->queue); | |
591 | ||
592 | tx_queue->insert_count = 0; | |
593 | tx_queue->write_count = 0; | |
594 | tx_queue->old_write_count = 0; | |
595 | tx_queue->read_count = 0; | |
596 | tx_queue->old_read_count = 0; | |
597 | tx_queue->empty_read_count = 0 | EF4_EMPTY_COUNT_VALID; | |
598 | tx_queue->xmit_more_available = false; | |
599 | ||
600 | /* Some older hardware requires Tx writes larger than 32. */ | |
601 | tx_queue->tx_min_size = EF4_WORKAROUND_15592(efx) ? 33 : 0; | |
602 | ||
603 | /* Set up TX descriptor ring */ | |
604 | ef4_nic_init_tx(tx_queue); | |
605 | ||
606 | tx_queue->initialised = true; | |
607 | } | |
608 | ||
609 | void ef4_fini_tx_queue(struct ef4_tx_queue *tx_queue) | |
610 | { | |
611 | struct ef4_tx_buffer *buffer; | |
612 | ||
613 | netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev, | |
614 | "shutting down TX queue %d\n", tx_queue->queue); | |
615 | ||
616 | if (!tx_queue->buffer) | |
617 | return; | |
618 | ||
619 | /* Free any buffers left in the ring */ | |
620 | while (tx_queue->read_count != tx_queue->write_count) { | |
621 | unsigned int pkts_compl = 0, bytes_compl = 0; | |
622 | buffer = &tx_queue->buffer[tx_queue->read_count & tx_queue->ptr_mask]; | |
623 | ef4_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl); | |
624 | ||
625 | ++tx_queue->read_count; | |
626 | } | |
627 | tx_queue->xmit_more_available = false; | |
628 | netdev_tx_reset_queue(tx_queue->core_txq); | |
629 | } | |
630 | ||
631 | void ef4_remove_tx_queue(struct ef4_tx_queue *tx_queue) | |
632 | { | |
633 | int i; | |
634 | ||
635 | if (!tx_queue->buffer) | |
636 | return; | |
637 | ||
638 | netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev, | |
639 | "destroying TX queue %d\n", tx_queue->queue); | |
640 | ef4_nic_remove_tx(tx_queue); | |
641 | ||
642 | if (tx_queue->cb_page) { | |
643 | for (i = 0; i < ef4_tx_cb_page_count(tx_queue); i++) | |
644 | ef4_nic_free_buffer(tx_queue->efx, | |
645 | &tx_queue->cb_page[i]); | |
646 | kfree(tx_queue->cb_page); | |
647 | tx_queue->cb_page = NULL; | |
648 | } | |
649 | ||
650 | kfree(tx_queue->buffer); | |
651 | tx_queue->buffer = NULL; | |
652 | } |