2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
4 * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 #include <linux/skbuff.h>
36 #include <linux/netdevice.h>
37 #include <linux/etherdevice.h>
38 #include <linux/if_vlan.h>
40 #include <linux/dma-mapping.h>
41 #include <linux/jiffies.h>
42 #include <linux/prefetch.h>
43 #include <linux/export.h>
47 #include <net/busy_poll.h>
48 #ifdef CONFIG_CHELSIO_T4_FCOE
49 #include <scsi/fc/fc_fcoe.h>
50 #endif /* CONFIG_CHELSIO_T4_FCOE */
53 #include "t4_values.h"
56 #include "cxgb4_ptp.h"
57 #include "cxgb4_uld.h"
60 * Rx buffer size. We use largish buffers if possible but settle for single
61 * pages under memory shortage.
64 # define FL_PG_ORDER 0
66 # define FL_PG_ORDER (16 - PAGE_SHIFT)
69 /* RX_PULL_LEN should be <= RX_COPY_THRES */
70 #define RX_COPY_THRES 256
71 #define RX_PULL_LEN 128
74 * Main body length for sk_buffs used for Rx Ethernet packets with fragments.
75 * Should be >= RX_PULL_LEN but possibly bigger to give pskb_may_pull some room.
77 #define RX_PKT_SKB_LEN 512
80 * Max number of Tx descriptors we clean up at a time. Should be modest as
81 * freeing skbs isn't cheap and it happens while holding locks. We just need
82 * to free packets faster than they arrive, we eventually catch up and keep
83 * the amortized cost reasonable. Must be >= 2 * TXQ_STOP_THRES. It should
84 * also match the CIDX Flush Threshold.
86 #define MAX_TX_RECLAIM 32
89 * Max number of Rx buffers we replenish at a time. Again keep this modest,
90 * allocating buffers isn't cheap either.
92 #define MAX_RX_REFILL 16U
95 * Period of the Rx queue check timer. This timer is infrequent as it has
96 * something to do only when the system experiences severe memory shortage.
98 #define RX_QCHECK_PERIOD (HZ / 2)
101 * Period of the Tx queue check timer.
103 #define TX_QCHECK_PERIOD (HZ / 2)
106 * Max number of Tx descriptors to be reclaimed by the Tx timer.
108 #define MAX_TIMER_TX_RECLAIM 100
111 * Timer index used when backing off due to memory shortage.
113 #define NOMEM_TMR_IDX (SGE_NTIMERS - 1)
116 * Suspension threshold for non-Ethernet Tx queues. We require enough room
117 * for a full sized WR.
119 #define TXQ_STOP_THRES (SGE_MAX_WR_LEN / sizeof(struct tx_desc))
122 * Max Tx descriptor space we allow for an Ethernet packet to be inlined
125 #define MAX_IMM_TX_PKT_LEN 256
128 * Max size of a WR sent through a control Tx queue.
130 #define MAX_CTRL_WR_LEN SGE_MAX_WR_LEN
132 struct rx_sw_desc
{ /* SW state per Rx descriptor */
138 * Rx buffer sizes for "useskbs" Free List buffers (one ingress packet pe skb
139 * buffer). We currently only support two sizes for 1500- and 9000-byte MTUs.
140 * We could easily support more but there doesn't seem to be much need for
143 #define FL_MTU_SMALL 1500
144 #define FL_MTU_LARGE 9000
146 static inline unsigned int fl_mtu_bufsize(struct adapter
*adapter
,
149 struct sge
*s
= &adapter
->sge
;
151 return ALIGN(s
->pktshift
+ ETH_HLEN
+ VLAN_HLEN
+ mtu
, s
->fl_align
);
154 #define FL_MTU_SMALL_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_SMALL)
155 #define FL_MTU_LARGE_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_LARGE)
158 * Bits 0..3 of rx_sw_desc.dma_addr have special meaning. The hardware uses
159 * these to specify the buffer size as an index into the SGE Free List Buffer
160 * Size register array. We also use bit 4, when the buffer has been unmapped
161 * for DMA, but this is of course never sent to the hardware and is only used
162 * to prevent double unmappings. All of the above requires that the Free List
163 * Buffers which we allocate have the bottom 5 bits free (0) -- i.e. are
164 * 32-byte or or a power of 2 greater in alignment. Since the SGE's minimal
165 * Free List Buffer alignment is 32 bytes, this works out for us ...
168 RX_BUF_FLAGS
= 0x1f, /* bottom five bits are special */
169 RX_BUF_SIZE
= 0x0f, /* bottom three bits are for buf sizes */
170 RX_UNMAPPED_BUF
= 0x10, /* buffer is not mapped */
173 * XXX We shouldn't depend on being able to use these indices.
174 * XXX Especially when some other Master PF has initialized the
175 * XXX adapter or we use the Firmware Configuration File. We
176 * XXX should really search through the Host Buffer Size register
177 * XXX array for the appropriately sized buffer indices.
179 RX_SMALL_PG_BUF
= 0x0, /* small (PAGE_SIZE) page buffer */
180 RX_LARGE_PG_BUF
= 0x1, /* buffer large (FL_PG_ORDER) page buffer */
182 RX_SMALL_MTU_BUF
= 0x2, /* small MTU buffer */
183 RX_LARGE_MTU_BUF
= 0x3, /* large MTU buffer */
186 static int timer_pkt_quota
[] = {1, 1, 2, 3, 4, 5};
187 #define MIN_NAPI_WORK 1
189 static inline dma_addr_t
get_buf_addr(const struct rx_sw_desc
*d
)
191 return d
->dma_addr
& ~(dma_addr_t
)RX_BUF_FLAGS
;
194 static inline bool is_buf_mapped(const struct rx_sw_desc
*d
)
196 return !(d
->dma_addr
& RX_UNMAPPED_BUF
);
200 * txq_avail - return the number of available slots in a Tx queue
203 * Returns the number of descriptors in a Tx queue available to write new
206 static inline unsigned int txq_avail(const struct sge_txq
*q
)
208 return q
->size
- 1 - q
->in_use
;
212 * fl_cap - return the capacity of a free-buffer list
215 * Returns the capacity of a free-buffer list. The capacity is less than
216 * the size because one descriptor needs to be left unpopulated, otherwise
217 * HW will think the FL is empty.
219 static inline unsigned int fl_cap(const struct sge_fl
*fl
)
221 return fl
->size
- 8; /* 1 descriptor = 8 buffers */
225 * fl_starving - return whether a Free List is starving.
226 * @adapter: pointer to the adapter
229 * Tests specified Free List to see whether the number of buffers
230 * available to the hardware has falled below our "starvation"
233 static inline bool fl_starving(const struct adapter
*adapter
,
234 const struct sge_fl
*fl
)
236 const struct sge
*s
= &adapter
->sge
;
238 return fl
->avail
- fl
->pend_cred
<= s
->fl_starve_thres
;
241 int cxgb4_map_skb(struct device
*dev
, const struct sk_buff
*skb
,
244 const skb_frag_t
*fp
, *end
;
245 const struct skb_shared_info
*si
;
247 *addr
= dma_map_single(dev
, skb
->data
, skb_headlen(skb
), DMA_TO_DEVICE
);
248 if (dma_mapping_error(dev
, *addr
))
251 si
= skb_shinfo(skb
);
252 end
= &si
->frags
[si
->nr_frags
];
254 for (fp
= si
->frags
; fp
< end
; fp
++) {
255 *++addr
= skb_frag_dma_map(dev
, fp
, 0, skb_frag_size(fp
),
257 if (dma_mapping_error(dev
, *addr
))
263 while (fp
-- > si
->frags
)
264 dma_unmap_page(dev
, *--addr
, skb_frag_size(fp
), DMA_TO_DEVICE
);
266 dma_unmap_single(dev
, addr
[-1], skb_headlen(skb
), DMA_TO_DEVICE
);
270 EXPORT_SYMBOL(cxgb4_map_skb
);
272 #ifdef CONFIG_NEED_DMA_MAP_STATE
273 static void unmap_skb(struct device
*dev
, const struct sk_buff
*skb
,
274 const dma_addr_t
*addr
)
276 const skb_frag_t
*fp
, *end
;
277 const struct skb_shared_info
*si
;
279 dma_unmap_single(dev
, *addr
++, skb_headlen(skb
), DMA_TO_DEVICE
);
281 si
= skb_shinfo(skb
);
282 end
= &si
->frags
[si
->nr_frags
];
283 for (fp
= si
->frags
; fp
< end
; fp
++)
284 dma_unmap_page(dev
, *addr
++, skb_frag_size(fp
), DMA_TO_DEVICE
);
288 * deferred_unmap_destructor - unmap a packet when it is freed
291 * This is the packet destructor used for Tx packets that need to remain
292 * mapped until they are freed rather than until their Tx descriptors are
295 static void deferred_unmap_destructor(struct sk_buff
*skb
)
297 unmap_skb(skb
->dev
->dev
.parent
, skb
, (dma_addr_t
*)skb
->head
);
301 static void unmap_sgl(struct device
*dev
, const struct sk_buff
*skb
,
302 const struct ulptx_sgl
*sgl
, const struct sge_txq
*q
)
304 const struct ulptx_sge_pair
*p
;
305 unsigned int nfrags
= skb_shinfo(skb
)->nr_frags
;
307 if (likely(skb_headlen(skb
)))
308 dma_unmap_single(dev
, be64_to_cpu(sgl
->addr0
), ntohl(sgl
->len0
),
311 dma_unmap_page(dev
, be64_to_cpu(sgl
->addr0
), ntohl(sgl
->len0
),
317 * the complexity below is because of the possibility of a wrap-around
318 * in the middle of an SGL
320 for (p
= sgl
->sge
; nfrags
>= 2; nfrags
-= 2) {
321 if (likely((u8
*)(p
+ 1) <= (u8
*)q
->stat
)) {
322 unmap
: dma_unmap_page(dev
, be64_to_cpu(p
->addr
[0]),
323 ntohl(p
->len
[0]), DMA_TO_DEVICE
);
324 dma_unmap_page(dev
, be64_to_cpu(p
->addr
[1]),
325 ntohl(p
->len
[1]), DMA_TO_DEVICE
);
327 } else if ((u8
*)p
== (u8
*)q
->stat
) {
328 p
= (const struct ulptx_sge_pair
*)q
->desc
;
330 } else if ((u8
*)p
+ 8 == (u8
*)q
->stat
) {
331 const __be64
*addr
= (const __be64
*)q
->desc
;
333 dma_unmap_page(dev
, be64_to_cpu(addr
[0]),
334 ntohl(p
->len
[0]), DMA_TO_DEVICE
);
335 dma_unmap_page(dev
, be64_to_cpu(addr
[1]),
336 ntohl(p
->len
[1]), DMA_TO_DEVICE
);
337 p
= (const struct ulptx_sge_pair
*)&addr
[2];
339 const __be64
*addr
= (const __be64
*)q
->desc
;
341 dma_unmap_page(dev
, be64_to_cpu(p
->addr
[0]),
342 ntohl(p
->len
[0]), DMA_TO_DEVICE
);
343 dma_unmap_page(dev
, be64_to_cpu(addr
[0]),
344 ntohl(p
->len
[1]), DMA_TO_DEVICE
);
345 p
= (const struct ulptx_sge_pair
*)&addr
[1];
351 if ((u8
*)p
== (u8
*)q
->stat
)
352 p
= (const struct ulptx_sge_pair
*)q
->desc
;
353 addr
= (u8
*)p
+ 16 <= (u8
*)q
->stat
? p
->addr
[0] :
354 *(const __be64
*)q
->desc
;
355 dma_unmap_page(dev
, be64_to_cpu(addr
), ntohl(p
->len
[0]),
361 * free_tx_desc - reclaims Tx descriptors and their buffers
362 * @adapter: the adapter
363 * @q: the Tx queue to reclaim descriptors from
364 * @n: the number of descriptors to reclaim
365 * @unmap: whether the buffers should be unmapped for DMA
367 * Reclaims Tx descriptors from an SGE Tx queue and frees the associated
368 * Tx buffers. Called with the Tx queue lock held.
370 void free_tx_desc(struct adapter
*adap
, struct sge_txq
*q
,
371 unsigned int n
, bool unmap
)
373 struct tx_sw_desc
*d
;
374 unsigned int cidx
= q
->cidx
;
375 struct device
*dev
= adap
->pdev_dev
;
379 if (d
->skb
) { /* an SGL is present */
381 unmap_sgl(dev
, d
->skb
, d
->sgl
, q
);
382 dev_consume_skb_any(d
->skb
);
386 if (++cidx
== q
->size
) {
395 * Return the number of reclaimable descriptors in a Tx queue.
397 static inline int reclaimable(const struct sge_txq
*q
)
399 int hw_cidx
= ntohs(READ_ONCE(q
->stat
->cidx
));
401 return hw_cidx
< 0 ? hw_cidx
+ q
->size
: hw_cidx
;
405 * reclaim_completed_tx - reclaims completed TX Descriptors
407 * @q: the Tx queue to reclaim completed descriptors from
408 * @maxreclaim: the maximum number of TX Descriptors to reclaim or -1
409 * @unmap: whether the buffers should be unmapped for DMA
411 * Reclaims Tx Descriptors that the SGE has indicated it has processed,
412 * and frees the associated buffers if possible. If @max == -1, then
413 * we'll use a defaiult maximum. Called with the TX Queue locked.
415 static inline int reclaim_completed_tx(struct adapter
*adap
, struct sge_txq
*q
,
416 int maxreclaim
, bool unmap
)
418 int reclaim
= reclaimable(q
);
422 * Limit the amount of clean up work we do at a time to keep
423 * the Tx lock hold time O(1).
426 maxreclaim
= MAX_TX_RECLAIM
;
427 if (reclaim
> maxreclaim
)
428 reclaim
= maxreclaim
;
430 free_tx_desc(adap
, q
, reclaim
, unmap
);
431 q
->in_use
-= reclaim
;
438 * cxgb4_reclaim_completed_tx - reclaims completed Tx descriptors
440 * @q: the Tx queue to reclaim completed descriptors from
441 * @unmap: whether the buffers should be unmapped for DMA
443 * Reclaims Tx descriptors that the SGE has indicated it has processed,
444 * and frees the associated buffers if possible. Called with the Tx
447 void cxgb4_reclaim_completed_tx(struct adapter
*adap
, struct sge_txq
*q
,
450 (void)reclaim_completed_tx(adap
, q
, -1, unmap
);
452 EXPORT_SYMBOL(cxgb4_reclaim_completed_tx
);
454 static inline int get_buf_size(struct adapter
*adapter
,
455 const struct rx_sw_desc
*d
)
457 struct sge
*s
= &adapter
->sge
;
458 unsigned int rx_buf_size_idx
= d
->dma_addr
& RX_BUF_SIZE
;
461 switch (rx_buf_size_idx
) {
462 case RX_SMALL_PG_BUF
:
463 buf_size
= PAGE_SIZE
;
466 case RX_LARGE_PG_BUF
:
467 buf_size
= PAGE_SIZE
<< s
->fl_pg_order
;
470 case RX_SMALL_MTU_BUF
:
471 buf_size
= FL_MTU_SMALL_BUFSIZE(adapter
);
474 case RX_LARGE_MTU_BUF
:
475 buf_size
= FL_MTU_LARGE_BUFSIZE(adapter
);
486 * free_rx_bufs - free the Rx buffers on an SGE free list
488 * @q: the SGE free list to free buffers from
489 * @n: how many buffers to free
491 * Release the next @n buffers on an SGE free-buffer Rx queue. The
492 * buffers must be made inaccessible to HW before calling this function.
494 static void free_rx_bufs(struct adapter
*adap
, struct sge_fl
*q
, int n
)
497 struct rx_sw_desc
*d
= &q
->sdesc
[q
->cidx
];
499 if (is_buf_mapped(d
))
500 dma_unmap_page(adap
->pdev_dev
, get_buf_addr(d
),
501 get_buf_size(adap
, d
),
505 if (++q
->cidx
== q
->size
)
512 * unmap_rx_buf - unmap the current Rx buffer on an SGE free list
514 * @q: the SGE free list
516 * Unmap the current buffer on an SGE free-buffer Rx queue. The
517 * buffer must be made inaccessible to HW before calling this function.
519 * This is similar to @free_rx_bufs above but does not free the buffer.
520 * Do note that the FL still loses any further access to the buffer.
522 static void unmap_rx_buf(struct adapter
*adap
, struct sge_fl
*q
)
524 struct rx_sw_desc
*d
= &q
->sdesc
[q
->cidx
];
526 if (is_buf_mapped(d
))
527 dma_unmap_page(adap
->pdev_dev
, get_buf_addr(d
),
528 get_buf_size(adap
, d
), PCI_DMA_FROMDEVICE
);
530 if (++q
->cidx
== q
->size
)
535 static inline void ring_fl_db(struct adapter
*adap
, struct sge_fl
*q
)
537 if (q
->pend_cred
>= 8) {
538 u32 val
= adap
->params
.arch
.sge_fl_db
;
540 if (is_t4(adap
->params
.chip
))
541 val
|= PIDX_V(q
->pend_cred
/ 8);
543 val
|= PIDX_T5_V(q
->pend_cred
/ 8);
545 /* Make sure all memory writes to the Free List queue are
546 * committed before we tell the hardware about them.
550 /* If we don't have access to the new User Doorbell (T5+), use
551 * the old doorbell mechanism; otherwise use the new BAR2
554 if (unlikely(q
->bar2_addr
== NULL
)) {
555 t4_write_reg(adap
, MYPF_REG(SGE_PF_KDOORBELL_A
),
556 val
| QID_V(q
->cntxt_id
));
558 writel(val
| QID_V(q
->bar2_qid
),
559 q
->bar2_addr
+ SGE_UDB_KDOORBELL
);
561 /* This Write memory Barrier will force the write to
562 * the User Doorbell area to be flushed.
570 static inline void set_rx_sw_desc(struct rx_sw_desc
*sd
, struct page
*pg
,
574 sd
->dma_addr
= mapping
; /* includes size low bits */
578 * refill_fl - refill an SGE Rx buffer ring
580 * @q: the ring to refill
581 * @n: the number of new buffers to allocate
582 * @gfp: the gfp flags for the allocations
584 * (Re)populate an SGE free-buffer queue with up to @n new packet buffers,
585 * allocated with the supplied gfp flags. The caller must assure that
586 * @n does not exceed the queue's capacity. If afterwards the queue is
587 * found critically low mark it as starving in the bitmap of starving FLs.
589 * Returns the number of buffers allocated.
591 static unsigned int refill_fl(struct adapter
*adap
, struct sge_fl
*q
, int n
,
594 struct sge
*s
= &adap
->sge
;
597 unsigned int cred
= q
->avail
;
598 __be64
*d
= &q
->desc
[q
->pidx
];
599 struct rx_sw_desc
*sd
= &q
->sdesc
[q
->pidx
];
602 #ifdef CONFIG_DEBUG_FS
603 if (test_bit(q
->cntxt_id
- adap
->sge
.egr_start
, adap
->sge
.blocked_fl
))
608 node
= dev_to_node(adap
->pdev_dev
);
610 if (s
->fl_pg_order
== 0)
611 goto alloc_small_pages
;
614 * Prefer large buffers
617 pg
= alloc_pages_node(node
, gfp
| __GFP_COMP
, s
->fl_pg_order
);
619 q
->large_alloc_failed
++;
620 break; /* fall back to single pages */
623 mapping
= dma_map_page(adap
->pdev_dev
, pg
, 0,
624 PAGE_SIZE
<< s
->fl_pg_order
,
626 if (unlikely(dma_mapping_error(adap
->pdev_dev
, mapping
))) {
627 __free_pages(pg
, s
->fl_pg_order
);
629 goto out
; /* do not try small pages for this error */
631 mapping
|= RX_LARGE_PG_BUF
;
632 *d
++ = cpu_to_be64(mapping
);
634 set_rx_sw_desc(sd
, pg
, mapping
);
638 if (++q
->pidx
== q
->size
) {
648 pg
= alloc_pages_node(node
, gfp
, 0);
654 mapping
= dma_map_page(adap
->pdev_dev
, pg
, 0, PAGE_SIZE
,
656 if (unlikely(dma_mapping_error(adap
->pdev_dev
, mapping
))) {
661 *d
++ = cpu_to_be64(mapping
);
663 set_rx_sw_desc(sd
, pg
, mapping
);
667 if (++q
->pidx
== q
->size
) {
674 out
: cred
= q
->avail
- cred
;
675 q
->pend_cred
+= cred
;
678 if (unlikely(fl_starving(adap
, q
))) {
681 set_bit(q
->cntxt_id
- adap
->sge
.egr_start
,
682 adap
->sge
.starving_fl
);
688 static inline void __refill_fl(struct adapter
*adap
, struct sge_fl
*fl
)
690 refill_fl(adap
, fl
, min(MAX_RX_REFILL
, fl_cap(fl
) - fl
->avail
),
695 * alloc_ring - allocate resources for an SGE descriptor ring
696 * @dev: the PCI device's core device
697 * @nelem: the number of descriptors
698 * @elem_size: the size of each descriptor
699 * @sw_size: the size of the SW state associated with each ring element
700 * @phys: the physical address of the allocated ring
701 * @metadata: address of the array holding the SW state for the ring
702 * @stat_size: extra space in HW ring for status information
703 * @node: preferred node for memory allocations
705 * Allocates resources for an SGE descriptor ring, such as Tx queues,
706 * free buffer lists, or response queues. Each SGE ring requires
707 * space for its HW descriptors plus, optionally, space for the SW state
708 * associated with each HW entry (the metadata). The function returns
709 * three values: the virtual address for the HW ring (the return value
710 * of the function), the bus address of the HW ring, and the address
713 static void *alloc_ring(struct device
*dev
, size_t nelem
, size_t elem_size
,
714 size_t sw_size
, dma_addr_t
*phys
, void *metadata
,
715 size_t stat_size
, int node
)
717 size_t len
= nelem
* elem_size
+ stat_size
;
719 void *p
= dma_alloc_coherent(dev
, len
, phys
, GFP_KERNEL
);
724 s
= kcalloc_node(sw_size
, nelem
, GFP_KERNEL
, node
);
727 dma_free_coherent(dev
, len
, p
, *phys
);
732 *(void **)metadata
= s
;
737 * sgl_len - calculates the size of an SGL of the given capacity
738 * @n: the number of SGL entries
740 * Calculates the number of flits needed for a scatter/gather list that
741 * can hold the given number of entries.
743 static inline unsigned int sgl_len(unsigned int n
)
745 /* A Direct Scatter Gather List uses 32-bit lengths and 64-bit PCI DMA
746 * addresses. The DSGL Work Request starts off with a 32-bit DSGL
747 * ULPTX header, then Length0, then Address0, then, for 1 <= i <= N,
748 * repeated sequences of { Length[i], Length[i+1], Address[i],
749 * Address[i+1] } (this ensures that all addresses are on 64-bit
750 * boundaries). If N is even, then Length[N+1] should be set to 0 and
751 * Address[N+1] is omitted.
753 * The following calculation incorporates all of the above. It's
754 * somewhat hard to follow but, briefly: the "+2" accounts for the
755 * first two flits which include the DSGL header, Length0 and
756 * Address0; the "(3*(n-1))/2" covers the main body of list entries (3
757 * flits for every pair of the remaining N) +1 if (n-1) is odd; and
758 * finally the "+((n-1)&1)" adds the one remaining flit needed if
762 return (3 * n
) / 2 + (n
& 1) + 2;
766 * flits_to_desc - returns the num of Tx descriptors for the given flits
767 * @n: the number of flits
769 * Returns the number of Tx descriptors needed for the supplied number
772 static inline unsigned int flits_to_desc(unsigned int n
)
774 BUG_ON(n
> SGE_MAX_WR_LEN
/ 8);
775 return DIV_ROUND_UP(n
, 8);
779 * is_eth_imm - can an Ethernet packet be sent as immediate data?
782 * Returns whether an Ethernet packet is small enough to fit as
783 * immediate data. Return value corresponds to headroom required.
785 static inline int is_eth_imm(const struct sk_buff
*skb
, unsigned int chip_ver
)
789 if (skb
->encapsulation
&& skb_shinfo(skb
)->gso_size
&&
790 chip_ver
> CHELSIO_T5
) {
791 hdrlen
= sizeof(struct cpl_tx_tnl_lso
);
792 hdrlen
+= sizeof(struct cpl_tx_pkt_core
);
794 hdrlen
= skb_shinfo(skb
)->gso_size
?
795 sizeof(struct cpl_tx_pkt_lso_core
) : 0;
796 hdrlen
+= sizeof(struct cpl_tx_pkt
);
798 if (skb
->len
<= MAX_IMM_TX_PKT_LEN
- hdrlen
)
804 * calc_tx_flits - calculate the number of flits for a packet Tx WR
807 * Returns the number of flits needed for a Tx WR for the given Ethernet
808 * packet, including the needed WR and CPL headers.
810 static inline unsigned int calc_tx_flits(const struct sk_buff
*skb
,
811 unsigned int chip_ver
)
814 int hdrlen
= is_eth_imm(skb
, chip_ver
);
816 /* If the skb is small enough, we can pump it out as a work request
817 * with only immediate data. In that case we just have to have the
818 * TX Packet header plus the skb data in the Work Request.
822 return DIV_ROUND_UP(skb
->len
+ hdrlen
, sizeof(__be64
));
824 /* Otherwise, we're going to have to construct a Scatter gather list
825 * of the skb body and fragments. We also include the flits necessary
826 * for the TX Packet Work Request and CPL. We always have a firmware
827 * Write Header (incorporated as part of the cpl_tx_pkt_lso and
828 * cpl_tx_pkt structures), followed by either a TX Packet Write CPL
829 * message or, if we're doing a Large Send Offload, an LSO CPL message
830 * with an embedded TX Packet Write CPL message.
832 flits
= sgl_len(skb_shinfo(skb
)->nr_frags
+ 1);
833 if (skb_shinfo(skb
)->gso_size
) {
834 if (skb
->encapsulation
&& chip_ver
> CHELSIO_T5
)
835 hdrlen
= sizeof(struct fw_eth_tx_pkt_wr
) +
836 sizeof(struct cpl_tx_tnl_lso
);
838 hdrlen
= sizeof(struct fw_eth_tx_pkt_wr
) +
839 sizeof(struct cpl_tx_pkt_lso_core
);
841 hdrlen
+= sizeof(struct cpl_tx_pkt_core
);
842 flits
+= (hdrlen
/ sizeof(__be64
));
844 flits
+= (sizeof(struct fw_eth_tx_pkt_wr
) +
845 sizeof(struct cpl_tx_pkt_core
)) / sizeof(__be64
);
851 * calc_tx_descs - calculate the number of Tx descriptors for a packet
854 * Returns the number of Tx descriptors needed for the given Ethernet
855 * packet, including the needed WR and CPL headers.
857 static inline unsigned int calc_tx_descs(const struct sk_buff
*skb
,
858 unsigned int chip_ver
)
860 return flits_to_desc(calc_tx_flits(skb
, chip_ver
));
864 * cxgb4_write_sgl - populate a scatter/gather list for a packet
866 * @q: the Tx queue we are writing into
867 * @sgl: starting location for writing the SGL
868 * @end: points right after the end of the SGL
869 * @start: start offset into skb main-body data to include in the SGL
870 * @addr: the list of bus addresses for the SGL elements
872 * Generates a gather list for the buffers that make up a packet.
873 * The caller must provide adequate space for the SGL that will be written.
874 * The SGL includes all of the packet's page fragments and the data in its
875 * main body except for the first @start bytes. @sgl must be 16-byte
876 * aligned and within a Tx descriptor with available space. @end points
877 * right after the end of the SGL but does not account for any potential
878 * wrap around, i.e., @end > @sgl.
880 void cxgb4_write_sgl(const struct sk_buff
*skb
, struct sge_txq
*q
,
881 struct ulptx_sgl
*sgl
, u64
*end
, unsigned int start
,
882 const dma_addr_t
*addr
)
885 struct ulptx_sge_pair
*to
;
886 const struct skb_shared_info
*si
= skb_shinfo(skb
);
887 unsigned int nfrags
= si
->nr_frags
;
888 struct ulptx_sge_pair buf
[MAX_SKB_FRAGS
/ 2 + 1];
890 len
= skb_headlen(skb
) - start
;
892 sgl
->len0
= htonl(len
);
893 sgl
->addr0
= cpu_to_be64(addr
[0] + start
);
896 sgl
->len0
= htonl(skb_frag_size(&si
->frags
[0]));
897 sgl
->addr0
= cpu_to_be64(addr
[1]);
900 sgl
->cmd_nsge
= htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL
) |
901 ULPTX_NSGE_V(nfrags
));
902 if (likely(--nfrags
== 0))
905 * Most of the complexity below deals with the possibility we hit the
906 * end of the queue in the middle of writing the SGL. For this case
907 * only we create the SGL in a temporary buffer and then copy it.
909 to
= (u8
*)end
> (u8
*)q
->stat
? buf
: sgl
->sge
;
911 for (i
= (nfrags
!= si
->nr_frags
); nfrags
>= 2; nfrags
-= 2, to
++) {
912 to
->len
[0] = cpu_to_be32(skb_frag_size(&si
->frags
[i
]));
913 to
->len
[1] = cpu_to_be32(skb_frag_size(&si
->frags
[++i
]));
914 to
->addr
[0] = cpu_to_be64(addr
[i
]);
915 to
->addr
[1] = cpu_to_be64(addr
[++i
]);
918 to
->len
[0] = cpu_to_be32(skb_frag_size(&si
->frags
[i
]));
919 to
->len
[1] = cpu_to_be32(0);
920 to
->addr
[0] = cpu_to_be64(addr
[i
+ 1]);
922 if (unlikely((u8
*)end
> (u8
*)q
->stat
)) {
923 unsigned int part0
= (u8
*)q
->stat
- (u8
*)sgl
->sge
, part1
;
926 memcpy(sgl
->sge
, buf
, part0
);
927 part1
= (u8
*)end
- (u8
*)q
->stat
;
928 memcpy(q
->desc
, (u8
*)buf
+ part0
, part1
);
929 end
= (void *)q
->desc
+ part1
;
931 if ((uintptr_t)end
& 8) /* 0-pad to multiple of 16 */
934 EXPORT_SYMBOL(cxgb4_write_sgl
);
936 /* This function copies 64 byte coalesced work request to
937 * memory mapped BAR2 space. For coalesced WR SGE fetches
938 * data from the FIFO instead of from Host.
940 static void cxgb_pio_copy(u64 __iomem
*dst
, u64
*src
)
953 * cxgb4_ring_tx_db - check and potentially ring a Tx queue's doorbell
956 * @n: number of new descriptors to give to HW
958 * Ring the doorbel for a Tx queue.
960 inline void cxgb4_ring_tx_db(struct adapter
*adap
, struct sge_txq
*q
, int n
)
962 /* Make sure that all writes to the TX Descriptors are committed
963 * before we tell the hardware about them.
967 /* If we don't have access to the new User Doorbell (T5+), use the old
968 * doorbell mechanism; otherwise use the new BAR2 mechanism.
970 if (unlikely(q
->bar2_addr
== NULL
)) {
974 /* For T4 we need to participate in the Doorbell Recovery
977 spin_lock_irqsave(&q
->db_lock
, flags
);
979 t4_write_reg(adap
, MYPF_REG(SGE_PF_KDOORBELL_A
),
980 QID_V(q
->cntxt_id
) | val
);
983 q
->db_pidx
= q
->pidx
;
984 spin_unlock_irqrestore(&q
->db_lock
, flags
);
986 u32 val
= PIDX_T5_V(n
);
988 /* T4 and later chips share the same PIDX field offset within
989 * the doorbell, but T5 and later shrank the field in order to
990 * gain a bit for Doorbell Priority. The field was absurdly
991 * large in the first place (14 bits) so we just use the T5
992 * and later limits and warn if a Queue ID is too large.
994 WARN_ON(val
& DBPRIO_F
);
996 /* If we're only writing a single TX Descriptor and we can use
997 * Inferred QID registers, we can use the Write Combining
998 * Gather Buffer; otherwise we use the simple doorbell.
1000 if (n
== 1 && q
->bar2_qid
== 0) {
1001 int index
= (q
->pidx
1004 u64
*wr
= (u64
*)&q
->desc
[index
];
1006 cxgb_pio_copy((u64 __iomem
*)
1007 (q
->bar2_addr
+ SGE_UDB_WCDOORBELL
),
1010 writel(val
| QID_V(q
->bar2_qid
),
1011 q
->bar2_addr
+ SGE_UDB_KDOORBELL
);
1014 /* This Write Memory Barrier will force the write to the User
1015 * Doorbell area to be flushed. This is needed to prevent
1016 * writes on different CPUs for the same queue from hitting
1017 * the adapter out of order. This is required when some Work
1018 * Requests take the Write Combine Gather Buffer path (user
1019 * doorbell area offset [SGE_UDB_WCDOORBELL..+63]) and some
1020 * take the traditional path where we simply increment the
1021 * PIDX (User Doorbell area SGE_UDB_KDOORBELL) and have the
1022 * hardware DMA read the actual Work Request.
1027 EXPORT_SYMBOL(cxgb4_ring_tx_db
);
1030 * cxgb4_inline_tx_skb - inline a packet's data into Tx descriptors
1032 * @q: the Tx queue where the packet will be inlined
1033 * @pos: starting position in the Tx queue where to inline the packet
1035 * Inline a packet's contents directly into Tx descriptors, starting at
1036 * the given position within the Tx DMA ring.
1037 * Most of the complexity of this operation is dealing with wrap arounds
1038 * in the middle of the packet we want to inline.
1040 void cxgb4_inline_tx_skb(const struct sk_buff
*skb
,
1041 const struct sge_txq
*q
, void *pos
)
1043 int left
= (void *)q
->stat
- pos
;
1046 if (likely(skb
->len
<= left
)) {
1047 if (likely(!skb
->data_len
))
1048 skb_copy_from_linear_data(skb
, pos
, skb
->len
);
1050 skb_copy_bits(skb
, 0, pos
, skb
->len
);
1053 skb_copy_bits(skb
, 0, pos
, left
);
1054 skb_copy_bits(skb
, left
, q
->desc
, skb
->len
- left
);
1055 pos
= (void *)q
->desc
+ (skb
->len
- left
);
1058 /* 0-pad to multiple of 16 */
1059 p
= PTR_ALIGN(pos
, 8);
1060 if ((uintptr_t)p
& 8)
1063 EXPORT_SYMBOL(cxgb4_inline_tx_skb
);
1065 static void *inline_tx_skb_header(const struct sk_buff
*skb
,
1066 const struct sge_txq
*q
, void *pos
,
1070 int left
= (void *)q
->stat
- pos
;
1072 if (likely(length
<= left
)) {
1073 memcpy(pos
, skb
->data
, length
);
1076 memcpy(pos
, skb
->data
, left
);
1077 memcpy(q
->desc
, skb
->data
+ left
, length
- left
);
1078 pos
= (void *)q
->desc
+ (length
- left
);
1080 /* 0-pad to multiple of 16 */
1081 p
= PTR_ALIGN(pos
, 8);
1082 if ((uintptr_t)p
& 8) {
1090 * Figure out what HW csum a packet wants and return the appropriate control
1093 static u64
hwcsum(enum chip_type chip
, const struct sk_buff
*skb
)
1096 bool inner_hdr_csum
= false;
1099 if (skb
->encapsulation
&&
1100 (CHELSIO_CHIP_VERSION(chip
) > CHELSIO_T5
))
1101 inner_hdr_csum
= true;
1103 if (inner_hdr_csum
) {
1104 ver
= inner_ip_hdr(skb
)->version
;
1105 proto
= (ver
== 4) ? inner_ip_hdr(skb
)->protocol
:
1106 inner_ipv6_hdr(skb
)->nexthdr
;
1108 ver
= ip_hdr(skb
)->version
;
1109 proto
= (ver
== 4) ? ip_hdr(skb
)->protocol
:
1110 ipv6_hdr(skb
)->nexthdr
;
1114 if (proto
== IPPROTO_TCP
)
1115 csum_type
= TX_CSUM_TCPIP
;
1116 else if (proto
== IPPROTO_UDP
)
1117 csum_type
= TX_CSUM_UDPIP
;
1120 * unknown protocol, disable HW csum
1121 * and hope a bad packet is detected
1123 return TXPKT_L4CSUM_DIS_F
;
1127 * this doesn't work with extension headers
1129 if (proto
== IPPROTO_TCP
)
1130 csum_type
= TX_CSUM_TCPIP6
;
1131 else if (proto
== IPPROTO_UDP
)
1132 csum_type
= TX_CSUM_UDPIP6
;
1137 if (likely(csum_type
>= TX_CSUM_TCPIP
)) {
1138 int eth_hdr_len
, l4_len
;
1141 if (inner_hdr_csum
) {
1142 /* This allows checksum offload for all encapsulated
1143 * packets like GRE etc..
1145 l4_len
= skb_inner_network_header_len(skb
);
1146 eth_hdr_len
= skb_inner_network_offset(skb
) - ETH_HLEN
;
1148 l4_len
= skb_network_header_len(skb
);
1149 eth_hdr_len
= skb_network_offset(skb
) - ETH_HLEN
;
1151 hdr_len
= TXPKT_IPHDR_LEN_V(l4_len
);
1153 if (CHELSIO_CHIP_VERSION(chip
) <= CHELSIO_T5
)
1154 hdr_len
|= TXPKT_ETHHDR_LEN_V(eth_hdr_len
);
1156 hdr_len
|= T6_TXPKT_ETHHDR_LEN_V(eth_hdr_len
);
1157 return TXPKT_CSUM_TYPE_V(csum_type
) | hdr_len
;
1159 int start
= skb_transport_offset(skb
);
1161 return TXPKT_CSUM_TYPE_V(csum_type
) |
1162 TXPKT_CSUM_START_V(start
) |
1163 TXPKT_CSUM_LOC_V(start
+ skb
->csum_offset
);
1167 static void eth_txq_stop(struct sge_eth_txq
*q
)
1169 netif_tx_stop_queue(q
->txq
);
1173 static inline void txq_advance(struct sge_txq
*q
, unsigned int n
)
1177 if (q
->pidx
>= q
->size
)
1181 #ifdef CONFIG_CHELSIO_T4_FCOE
1183 cxgb_fcoe_offload(struct sk_buff
*skb
, struct adapter
*adap
,
1184 const struct port_info
*pi
, u64
*cntrl
)
1186 const struct cxgb_fcoe
*fcoe
= &pi
->fcoe
;
1188 if (!(fcoe
->flags
& CXGB_FCOE_ENABLED
))
1191 if (skb
->protocol
!= htons(ETH_P_FCOE
))
1194 skb_reset_mac_header(skb
);
1195 skb
->mac_len
= sizeof(struct ethhdr
);
1197 skb_set_network_header(skb
, skb
->mac_len
);
1198 skb_set_transport_header(skb
, skb
->mac_len
+ sizeof(struct fcoe_hdr
));
1200 if (!cxgb_fcoe_sof_eof_supported(adap
, skb
))
1203 /* FC CRC offload */
1204 *cntrl
= TXPKT_CSUM_TYPE_V(TX_CSUM_FCOE
) |
1205 TXPKT_L4CSUM_DIS_F
| TXPKT_IPCSUM_DIS_F
|
1206 TXPKT_CSUM_START_V(CXGB_FCOE_TXPKT_CSUM_START
) |
1207 TXPKT_CSUM_END_V(CXGB_FCOE_TXPKT_CSUM_END
) |
1208 TXPKT_CSUM_LOC_V(CXGB_FCOE_TXPKT_CSUM_END
);
1211 #endif /* CONFIG_CHELSIO_T4_FCOE */
1213 /* Returns tunnel type if hardware supports offloading of the same.
1214 * It is called only for T5 and onwards.
1216 enum cpl_tx_tnl_lso_type
cxgb_encap_offload_supported(struct sk_buff
*skb
)
1219 enum cpl_tx_tnl_lso_type tnl_type
= TX_TNL_TYPE_OPAQUE
;
1220 struct port_info
*pi
= netdev_priv(skb
->dev
);
1221 struct adapter
*adapter
= pi
->adapter
;
1223 if (skb
->inner_protocol_type
!= ENCAP_TYPE_ETHER
||
1224 skb
->inner_protocol
!= htons(ETH_P_TEB
))
1227 switch (vlan_get_protocol(skb
)) {
1228 case htons(ETH_P_IP
):
1229 l4_hdr
= ip_hdr(skb
)->protocol
;
1231 case htons(ETH_P_IPV6
):
1232 l4_hdr
= ipv6_hdr(skb
)->nexthdr
;
1240 if (adapter
->vxlan_port
== udp_hdr(skb
)->dest
)
1241 tnl_type
= TX_TNL_TYPE_VXLAN
;
1242 else if (adapter
->geneve_port
== udp_hdr(skb
)->dest
)
1243 tnl_type
= TX_TNL_TYPE_GENEVE
;
1252 static inline void t6_fill_tnl_lso(struct sk_buff
*skb
,
1253 struct cpl_tx_tnl_lso
*tnl_lso
,
1254 enum cpl_tx_tnl_lso_type tnl_type
)
1257 int in_eth_xtra_len
;
1258 int l3hdr_len
= skb_network_header_len(skb
);
1259 int eth_xtra_len
= skb_network_offset(skb
) - ETH_HLEN
;
1260 const struct skb_shared_info
*ssi
= skb_shinfo(skb
);
1261 bool v6
= (ip_hdr(skb
)->version
== 6);
1263 val
= CPL_TX_TNL_LSO_OPCODE_V(CPL_TX_TNL_LSO
) |
1264 CPL_TX_TNL_LSO_FIRST_F
|
1265 CPL_TX_TNL_LSO_LAST_F
|
1266 (v6
? CPL_TX_TNL_LSO_IPV6OUT_F
: 0) |
1267 CPL_TX_TNL_LSO_ETHHDRLENOUT_V(eth_xtra_len
/ 4) |
1268 CPL_TX_TNL_LSO_IPHDRLENOUT_V(l3hdr_len
/ 4) |
1269 (v6
? 0 : CPL_TX_TNL_LSO_IPHDRCHKOUT_F
) |
1270 CPL_TX_TNL_LSO_IPLENSETOUT_F
|
1271 (v6
? 0 : CPL_TX_TNL_LSO_IPIDINCOUT_F
);
1272 tnl_lso
->op_to_IpIdSplitOut
= htonl(val
);
1274 tnl_lso
->IpIdOffsetOut
= 0;
1276 /* Get the tunnel header length */
1277 val
= skb_inner_mac_header(skb
) - skb_mac_header(skb
);
1278 in_eth_xtra_len
= skb_inner_network_header(skb
) -
1279 skb_inner_mac_header(skb
) - ETH_HLEN
;
1282 case TX_TNL_TYPE_VXLAN
:
1283 case TX_TNL_TYPE_GENEVE
:
1284 tnl_lso
->UdpLenSetOut_to_TnlHdrLen
=
1285 htons(CPL_TX_TNL_LSO_UDPCHKCLROUT_F
|
1286 CPL_TX_TNL_LSO_UDPLENSETOUT_F
);
1289 tnl_lso
->UdpLenSetOut_to_TnlHdrLen
= 0;
1293 tnl_lso
->UdpLenSetOut_to_TnlHdrLen
|=
1294 htons(CPL_TX_TNL_LSO_TNLHDRLEN_V(val
) |
1295 CPL_TX_TNL_LSO_TNLTYPE_V(tnl_type
));
1299 val
= CPL_TX_TNL_LSO_ETHHDRLEN_V(in_eth_xtra_len
/ 4) |
1300 CPL_TX_TNL_LSO_IPV6_V(inner_ip_hdr(skb
)->version
== 6) |
1301 CPL_TX_TNL_LSO_IPHDRLEN_V(skb_inner_network_header_len(skb
) / 4) |
1302 CPL_TX_TNL_LSO_TCPHDRLEN_V(inner_tcp_hdrlen(skb
) / 4);
1303 tnl_lso
->Flow_to_TcpHdrLen
= htonl(val
);
1305 tnl_lso
->IpIdOffset
= htons(0);
1307 tnl_lso
->IpIdSplit_to_Mss
= htons(CPL_TX_TNL_LSO_MSS_V(ssi
->gso_size
));
1308 tnl_lso
->TCPSeqOffset
= htonl(0);
1309 tnl_lso
->EthLenOffset_Size
= htonl(CPL_TX_TNL_LSO_SIZE_V(skb
->len
));
1313 * t4_sge_eth_txq_egress_update - handle Ethernet TX Queue update
1314 * @adap: the adapter
1315 * @eq: the Ethernet TX Queue
1316 * @maxreclaim: the maximum number of TX Descriptors to reclaim or -1
1318 * We're typically called here to update the state of an Ethernet TX
1319 * Queue with respect to the hardware's progress in consuming the TX
1320 * Work Requests that we've put on that Egress Queue. This happens
1321 * when we get Egress Queue Update messages and also prophylactically
1322 * in regular timer-based Ethernet TX Queue maintenance.
1324 int t4_sge_eth_txq_egress_update(struct adapter
*adap
, struct sge_eth_txq
*eq
,
1327 struct sge_txq
*q
= &eq
->q
;
1328 unsigned int reclaimed
;
1330 if (!q
->in_use
|| !__netif_tx_trylock(eq
->txq
))
1333 /* Reclaim pending completed TX Descriptors. */
1334 reclaimed
= reclaim_completed_tx(adap
, &eq
->q
, maxreclaim
, true);
1336 /* If the TX Queue is currently stopped and there's now more than half
1337 * the queue available, restart it. Otherwise bail out since the rest
1338 * of what we want do here is with the possibility of shipping any
1339 * currently buffered Coalesced TX Work Request.
1341 if (netif_tx_queue_stopped(eq
->txq
) && txq_avail(q
) > (q
->size
/ 2)) {
1342 netif_tx_wake_queue(eq
->txq
);
1346 __netif_tx_unlock(eq
->txq
);
1351 * cxgb4_eth_xmit - add a packet to an Ethernet Tx queue
1353 * @dev: the egress net device
1355 * Add a packet to an SGE Ethernet Tx queue. Runs with softirqs disabled.
1357 static netdev_tx_t
cxgb4_eth_xmit(struct sk_buff
*skb
, struct net_device
*dev
)
1359 u32 wr_mid
, ctrl0
, op
;
1360 u64 cntrl
, *end
, *sgl
;
1362 unsigned int flits
, ndesc
;
1363 struct adapter
*adap
;
1364 struct sge_eth_txq
*q
;
1365 const struct port_info
*pi
;
1366 struct fw_eth_tx_pkt_wr
*wr
;
1367 struct cpl_tx_pkt_core
*cpl
;
1368 const struct skb_shared_info
*ssi
;
1369 dma_addr_t addr
[MAX_SKB_FRAGS
+ 1];
1370 bool immediate
= false;
1371 int len
, max_pkt_len
;
1372 bool ptp_enabled
= is_ptp_enabled(skb
, dev
);
1373 unsigned int chip_ver
;
1374 enum cpl_tx_tnl_lso_type tnl_type
= TX_TNL_TYPE_OPAQUE
;
1376 #ifdef CONFIG_CHELSIO_T4_FCOE
1378 #endif /* CONFIG_CHELSIO_T4_FCOE */
1381 * The chip min packet length is 10 octets but play safe and reject
1382 * anything shorter than an Ethernet header.
1384 if (unlikely(skb
->len
< ETH_HLEN
)) {
1385 out_free
: dev_kfree_skb_any(skb
);
1386 return NETDEV_TX_OK
;
1389 /* Discard the packet if the length is greater than mtu */
1390 max_pkt_len
= ETH_HLEN
+ dev
->mtu
;
1391 if (skb_vlan_tagged(skb
))
1392 max_pkt_len
+= VLAN_HLEN
;
1393 if (!skb_shinfo(skb
)->gso_size
&& (unlikely(skb
->len
> max_pkt_len
)))
1396 pi
= netdev_priv(dev
);
1398 ssi
= skb_shinfo(skb
);
1399 #ifdef CONFIG_CHELSIO_IPSEC_INLINE
1400 if (xfrm_offload(skb
) && !ssi
->gso_size
)
1401 return adap
->uld
[CXGB4_ULD_CRYPTO
].tx_handler(skb
, dev
);
1402 #endif /* CHELSIO_IPSEC_INLINE */
1404 qidx
= skb_get_queue_mapping(skb
);
1406 spin_lock(&adap
->ptp_lock
);
1407 if (!(adap
->ptp_tx_skb
)) {
1408 skb_shinfo(skb
)->tx_flags
|= SKBTX_IN_PROGRESS
;
1409 adap
->ptp_tx_skb
= skb_get(skb
);
1411 spin_unlock(&adap
->ptp_lock
);
1414 q
= &adap
->sge
.ptptxq
;
1416 q
= &adap
->sge
.ethtxq
[qidx
+ pi
->first_qset
];
1418 skb_tx_timestamp(skb
);
1420 reclaim_completed_tx(adap
, &q
->q
, -1, true);
1421 cntrl
= TXPKT_L4CSUM_DIS_F
| TXPKT_IPCSUM_DIS_F
;
1423 #ifdef CONFIG_CHELSIO_T4_FCOE
1424 err
= cxgb_fcoe_offload(skb
, adap
, pi
, &cntrl
);
1425 if (unlikely(err
== -ENOTSUPP
)) {
1427 spin_unlock(&adap
->ptp_lock
);
1430 #endif /* CONFIG_CHELSIO_T4_FCOE */
1432 chip_ver
= CHELSIO_CHIP_VERSION(adap
->params
.chip
);
1433 flits
= calc_tx_flits(skb
, chip_ver
);
1434 ndesc
= flits_to_desc(flits
);
1435 credits
= txq_avail(&q
->q
) - ndesc
;
1437 if (unlikely(credits
< 0)) {
1439 dev_err(adap
->pdev_dev
,
1440 "%s: Tx ring %u full while queue awake!\n",
1443 spin_unlock(&adap
->ptp_lock
);
1444 return NETDEV_TX_BUSY
;
1447 if (is_eth_imm(skb
, chip_ver
))
1450 if (skb
->encapsulation
&& chip_ver
> CHELSIO_T5
)
1451 tnl_type
= cxgb_encap_offload_supported(skb
);
1454 unlikely(cxgb4_map_skb(adap
->pdev_dev
, skb
, addr
) < 0)) {
1457 spin_unlock(&adap
->ptp_lock
);
1461 wr_mid
= FW_WR_LEN16_V(DIV_ROUND_UP(flits
, 2));
1462 if (unlikely(credits
< ETHTXQ_STOP_THRES
)) {
1463 /* After we're done injecting the Work Request for this
1464 * packet, we'll be below our "stop threshold" so stop the TX
1465 * Queue now and schedule a request for an SGE Egress Queue
1466 * Update message. The queue will get started later on when
1467 * the firmware processes this Work Request and sends us an
1468 * Egress Queue Status Update message indicating that space
1473 /* If we're using the SGE Doorbell Queue Timer facility, we
1474 * don't need to ask the Firmware to send us Egress Queue CIDX
1475 * Updates: the Hardware will do this automatically. And
1476 * since we send the Ingress Queue CIDX Updates to the
1477 * corresponding Ethernet Response Queue, we'll get them very
1481 wr_mid
|= FW_WR_EQUEQ_F
| FW_WR_EQUIQ_F
;
1484 wr
= (void *)&q
->q
.desc
[q
->q
.pidx
];
1485 wr
->equiq_to_len16
= htonl(wr_mid
);
1486 wr
->r3
= cpu_to_be64(0);
1487 end
= (u64
*)wr
+ flits
;
1489 len
= immediate
? skb
->len
: 0;
1490 len
+= sizeof(*cpl
);
1491 if (ssi
->gso_size
) {
1492 struct cpl_tx_pkt_lso_core
*lso
= (void *)(wr
+ 1);
1493 bool v6
= (ssi
->gso_type
& SKB_GSO_TCPV6
) != 0;
1494 int l3hdr_len
= skb_network_header_len(skb
);
1495 int eth_xtra_len
= skb_network_offset(skb
) - ETH_HLEN
;
1496 struct cpl_tx_tnl_lso
*tnl_lso
= (void *)(wr
+ 1);
1499 len
+= sizeof(*tnl_lso
);
1501 len
+= sizeof(*lso
);
1503 wr
->op_immdlen
= htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR
) |
1504 FW_WR_IMMDLEN_V(len
));
1506 struct iphdr
*iph
= ip_hdr(skb
);
1508 t6_fill_tnl_lso(skb
, tnl_lso
, tnl_type
);
1509 cpl
= (void *)(tnl_lso
+ 1);
1510 /* Driver is expected to compute partial checksum that
1511 * does not include the IP Total Length.
1513 if (iph
->version
== 4) {
1516 iph
->check
= (u16
)(~ip_fast_csum((u8
*)iph
,
1519 if (skb
->ip_summed
== CHECKSUM_PARTIAL
)
1520 cntrl
= hwcsum(adap
->params
.chip
, skb
);
1522 lso
->lso_ctrl
= htonl(LSO_OPCODE_V(CPL_TX_PKT_LSO
) |
1523 LSO_FIRST_SLICE_F
| LSO_LAST_SLICE_F
|
1525 LSO_ETHHDR_LEN_V(eth_xtra_len
/ 4) |
1526 LSO_IPHDR_LEN_V(l3hdr_len
/ 4) |
1527 LSO_TCPHDR_LEN_V(tcp_hdr(skb
)->doff
));
1528 lso
->ipid_ofst
= htons(0);
1529 lso
->mss
= htons(ssi
->gso_size
);
1530 lso
->seqno_offset
= htonl(0);
1531 if (is_t4(adap
->params
.chip
))
1532 lso
->len
= htonl(skb
->len
);
1534 lso
->len
= htonl(LSO_T5_XFER_SIZE_V(skb
->len
));
1535 cpl
= (void *)(lso
+ 1);
1537 if (CHELSIO_CHIP_VERSION(adap
->params
.chip
)
1539 cntrl
= TXPKT_ETHHDR_LEN_V(eth_xtra_len
);
1541 cntrl
= T6_TXPKT_ETHHDR_LEN_V(eth_xtra_len
);
1543 cntrl
|= TXPKT_CSUM_TYPE_V(v6
?
1544 TX_CSUM_TCPIP6
: TX_CSUM_TCPIP
) |
1545 TXPKT_IPHDR_LEN_V(l3hdr_len
);
1547 sgl
= (u64
*)(cpl
+ 1); /* sgl start here */
1548 if (unlikely((u8
*)sgl
>= (u8
*)q
->q
.stat
)) {
1549 /* If current position is already at the end of the
1550 * txq, reset the current to point to start of the queue
1551 * and update the end ptr as well.
1553 if (sgl
== (u64
*)q
->q
.stat
) {
1554 int left
= (u8
*)end
- (u8
*)q
->q
.stat
;
1556 end
= (void *)q
->q
.desc
+ left
;
1557 sgl
= (void *)q
->q
.desc
;
1561 q
->tx_cso
+= ssi
->gso_segs
;
1564 op
= FW_PTP_TX_PKT_WR
;
1566 op
= FW_ETH_TX_PKT_WR
;
1567 wr
->op_immdlen
= htonl(FW_WR_OP_V(op
) |
1568 FW_WR_IMMDLEN_V(len
));
1569 cpl
= (void *)(wr
+ 1);
1570 sgl
= (u64
*)(cpl
+ 1);
1571 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
1572 cntrl
= hwcsum(adap
->params
.chip
, skb
) |
1578 if (skb_vlan_tag_present(skb
)) {
1580 cntrl
|= TXPKT_VLAN_VLD_F
| TXPKT_VLAN_V(skb_vlan_tag_get(skb
));
1581 #ifdef CONFIG_CHELSIO_T4_FCOE
1582 if (skb
->protocol
== htons(ETH_P_FCOE
))
1583 cntrl
|= TXPKT_VLAN_V(
1584 ((skb
->priority
& 0x7) << VLAN_PRIO_SHIFT
));
1585 #endif /* CONFIG_CHELSIO_T4_FCOE */
1588 ctrl0
= TXPKT_OPCODE_V(CPL_TX_PKT_XT
) | TXPKT_INTF_V(pi
->tx_chan
) |
1589 TXPKT_PF_V(adap
->pf
);
1591 ctrl0
|= TXPKT_TSTAMP_F
;
1592 #ifdef CONFIG_CHELSIO_T4_DCB
1593 if (is_t4(adap
->params
.chip
))
1594 ctrl0
|= TXPKT_OVLAN_IDX_V(q
->dcb_prio
);
1596 ctrl0
|= TXPKT_T5_OVLAN_IDX_V(q
->dcb_prio
);
1598 cpl
->ctrl0
= htonl(ctrl0
);
1599 cpl
->pack
= htons(0);
1600 cpl
->len
= htons(skb
->len
);
1601 cpl
->ctrl1
= cpu_to_be64(cntrl
);
1604 cxgb4_inline_tx_skb(skb
, &q
->q
, sgl
);
1605 dev_consume_skb_any(skb
);
1609 cxgb4_write_sgl(skb
, &q
->q
, (void *)sgl
, end
, 0, addr
);
1612 last_desc
= q
->q
.pidx
+ ndesc
- 1;
1613 if (last_desc
>= q
->q
.size
)
1614 last_desc
-= q
->q
.size
;
1615 q
->q
.sdesc
[last_desc
].skb
= skb
;
1616 q
->q
.sdesc
[last_desc
].sgl
= (struct ulptx_sgl
*)sgl
;
1619 txq_advance(&q
->q
, ndesc
);
1621 cxgb4_ring_tx_db(adap
, &q
->q
, ndesc
);
1623 spin_unlock(&adap
->ptp_lock
);
1624 return NETDEV_TX_OK
;
1629 /* Egress Queue sizes, producer and consumer indices are all in units
1630 * of Egress Context Units bytes. Note that as far as the hardware is
1631 * concerned, the free list is an Egress Queue (the host produces free
1632 * buffers which the hardware consumes) and free list entries are
1633 * 64-bit PCI DMA addresses.
1635 EQ_UNIT
= SGE_EQ_IDXSIZE
,
1636 FL_PER_EQ_UNIT
= EQ_UNIT
/ sizeof(__be64
),
1637 TXD_PER_EQ_UNIT
= EQ_UNIT
/ sizeof(__be64
),
1639 T4VF_ETHTXQ_MAX_HDR
= (sizeof(struct fw_eth_tx_pkt_vm_wr
) +
1640 sizeof(struct cpl_tx_pkt_lso_core
) +
1641 sizeof(struct cpl_tx_pkt_core
)) / sizeof(__be64
),
1645 * t4vf_is_eth_imm - can an Ethernet packet be sent as immediate data?
1648 * Returns whether an Ethernet packet is small enough to fit completely as
1651 static inline int t4vf_is_eth_imm(const struct sk_buff
*skb
)
1653 /* The VF Driver uses the FW_ETH_TX_PKT_VM_WR firmware Work Request
1654 * which does not accommodate immediate data. We could dike out all
1655 * of the support code for immediate data but that would tie our hands
1656 * too much if we ever want to enhace the firmware. It would also
1657 * create more differences between the PF and VF Drivers.
1663 * t4vf_calc_tx_flits - calculate the number of flits for a packet TX WR
1666 * Returns the number of flits needed for a TX Work Request for the
1667 * given Ethernet packet, including the needed WR and CPL headers.
1669 static inline unsigned int t4vf_calc_tx_flits(const struct sk_buff
*skb
)
1673 /* If the skb is small enough, we can pump it out as a work request
1674 * with only immediate data. In that case we just have to have the
1675 * TX Packet header plus the skb data in the Work Request.
1677 if (t4vf_is_eth_imm(skb
))
1678 return DIV_ROUND_UP(skb
->len
+ sizeof(struct cpl_tx_pkt
),
1681 /* Otherwise, we're going to have to construct a Scatter gather list
1682 * of the skb body and fragments. We also include the flits necessary
1683 * for the TX Packet Work Request and CPL. We always have a firmware
1684 * Write Header (incorporated as part of the cpl_tx_pkt_lso and
1685 * cpl_tx_pkt structures), followed by either a TX Packet Write CPL
1686 * message or, if we're doing a Large Send Offload, an LSO CPL message
1687 * with an embedded TX Packet Write CPL message.
1689 flits
= sgl_len(skb_shinfo(skb
)->nr_frags
+ 1);
1690 if (skb_shinfo(skb
)->gso_size
)
1691 flits
+= (sizeof(struct fw_eth_tx_pkt_vm_wr
) +
1692 sizeof(struct cpl_tx_pkt_lso_core
) +
1693 sizeof(struct cpl_tx_pkt_core
)) / sizeof(__be64
);
1695 flits
+= (sizeof(struct fw_eth_tx_pkt_vm_wr
) +
1696 sizeof(struct cpl_tx_pkt_core
)) / sizeof(__be64
);
1701 * cxgb4_vf_eth_xmit - add a packet to an Ethernet TX queue
1703 * @dev: the egress net device
1705 * Add a packet to an SGE Ethernet TX queue. Runs with softirqs disabled.
1707 static netdev_tx_t
cxgb4_vf_eth_xmit(struct sk_buff
*skb
,
1708 struct net_device
*dev
)
1710 dma_addr_t addr
[MAX_SKB_FRAGS
+ 1];
1711 const struct skb_shared_info
*ssi
;
1712 struct fw_eth_tx_pkt_vm_wr
*wr
;
1713 int qidx
, credits
, max_pkt_len
;
1714 struct cpl_tx_pkt_core
*cpl
;
1715 const struct port_info
*pi
;
1716 unsigned int flits
, ndesc
;
1717 struct sge_eth_txq
*txq
;
1718 struct adapter
*adapter
;
1721 const size_t fw_hdr_copy_len
= sizeof(wr
->ethmacdst
) +
1722 sizeof(wr
->ethmacsrc
) +
1723 sizeof(wr
->ethtype
) +
1724 sizeof(wr
->vlantci
);
1726 /* The chip minimum packet length is 10 octets but the firmware
1727 * command that we are using requires that we copy the Ethernet header
1728 * (including the VLAN tag) into the header so we reject anything
1729 * smaller than that ...
1731 if (unlikely(skb
->len
< fw_hdr_copy_len
))
1734 /* Discard the packet if the length is greater than mtu */
1735 max_pkt_len
= ETH_HLEN
+ dev
->mtu
;
1736 if (skb_vlan_tag_present(skb
))
1737 max_pkt_len
+= VLAN_HLEN
;
1738 if (!skb_shinfo(skb
)->gso_size
&& (unlikely(skb
->len
> max_pkt_len
)))
1741 /* Figure out which TX Queue we're going to use. */
1742 pi
= netdev_priv(dev
);
1743 adapter
= pi
->adapter
;
1744 qidx
= skb_get_queue_mapping(skb
);
1745 WARN_ON(qidx
>= pi
->nqsets
);
1746 txq
= &adapter
->sge
.ethtxq
[pi
->first_qset
+ qidx
];
1748 /* Take this opportunity to reclaim any TX Descriptors whose DMA
1749 * transfers have completed.
1751 reclaim_completed_tx(adapter
, &txq
->q
, -1, true);
1753 /* Calculate the number of flits and TX Descriptors we're going to
1754 * need along with how many TX Descriptors will be left over after
1755 * we inject our Work Request.
1757 flits
= t4vf_calc_tx_flits(skb
);
1758 ndesc
= flits_to_desc(flits
);
1759 credits
= txq_avail(&txq
->q
) - ndesc
;
1761 if (unlikely(credits
< 0)) {
1762 /* Not enough room for this packet's Work Request. Stop the
1763 * TX Queue and return a "busy" condition. The queue will get
1764 * started later on when the firmware informs us that space
1768 dev_err(adapter
->pdev_dev
,
1769 "%s: TX ring %u full while queue awake!\n",
1771 return NETDEV_TX_BUSY
;
1774 if (!t4vf_is_eth_imm(skb
) &&
1775 unlikely(cxgb4_map_skb(adapter
->pdev_dev
, skb
, addr
) < 0)) {
1776 /* We need to map the skb into PCI DMA space (because it can't
1777 * be in-lined directly into the Work Request) and the mapping
1778 * operation failed. Record the error and drop the packet.
1784 wr_mid
= FW_WR_LEN16_V(DIV_ROUND_UP(flits
, 2));
1785 if (unlikely(credits
< ETHTXQ_STOP_THRES
)) {
1786 /* After we're done injecting the Work Request for this
1787 * packet, we'll be below our "stop threshold" so stop the TX
1788 * Queue now and schedule a request for an SGE Egress Queue
1789 * Update message. The queue will get started later on when
1790 * the firmware processes this Work Request and sends us an
1791 * Egress Queue Status Update message indicating that space
1796 /* If we're using the SGE Doorbell Queue Timer facility, we
1797 * don't need to ask the Firmware to send us Egress Queue CIDX
1798 * Updates: the Hardware will do this automatically. And
1799 * since we send the Ingress Queue CIDX Updates to the
1800 * corresponding Ethernet Response Queue, we'll get them very
1804 wr_mid
|= FW_WR_EQUEQ_F
| FW_WR_EQUIQ_F
;
1807 /* Start filling in our Work Request. Note that we do _not_ handle
1808 * the WR Header wrapping around the TX Descriptor Ring. If our
1809 * maximum header size ever exceeds one TX Descriptor, we'll need to
1810 * do something else here.
1812 WARN_ON(DIV_ROUND_UP(T4VF_ETHTXQ_MAX_HDR
, TXD_PER_EQ_UNIT
) > 1);
1813 wr
= (void *)&txq
->q
.desc
[txq
->q
.pidx
];
1814 wr
->equiq_to_len16
= cpu_to_be32(wr_mid
);
1815 wr
->r3
[0] = cpu_to_be32(0);
1816 wr
->r3
[1] = cpu_to_be32(0);
1817 skb_copy_from_linear_data(skb
, (void *)wr
->ethmacdst
, fw_hdr_copy_len
);
1818 end
= (u64
*)wr
+ flits
;
1820 /* If this is a Large Send Offload packet we'll put in an LSO CPL
1821 * message with an encapsulated TX Packet CPL message. Otherwise we
1822 * just use a TX Packet CPL message.
1824 ssi
= skb_shinfo(skb
);
1825 if (ssi
->gso_size
) {
1826 struct cpl_tx_pkt_lso_core
*lso
= (void *)(wr
+ 1);
1827 bool v6
= (ssi
->gso_type
& SKB_GSO_TCPV6
) != 0;
1828 int l3hdr_len
= skb_network_header_len(skb
);
1829 int eth_xtra_len
= skb_network_offset(skb
) - ETH_HLEN
;
1832 cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_PKT_VM_WR
) |
1833 FW_WR_IMMDLEN_V(sizeof(*lso
) +
1835 /* Fill in the LSO CPL message. */
1837 cpu_to_be32(LSO_OPCODE_V(CPL_TX_PKT_LSO
) |
1841 LSO_ETHHDR_LEN_V(eth_xtra_len
/ 4) |
1842 LSO_IPHDR_LEN_V(l3hdr_len
/ 4) |
1843 LSO_TCPHDR_LEN_V(tcp_hdr(skb
)->doff
));
1844 lso
->ipid_ofst
= cpu_to_be16(0);
1845 lso
->mss
= cpu_to_be16(ssi
->gso_size
);
1846 lso
->seqno_offset
= cpu_to_be32(0);
1847 if (is_t4(adapter
->params
.chip
))
1848 lso
->len
= cpu_to_be32(skb
->len
);
1850 lso
->len
= cpu_to_be32(LSO_T5_XFER_SIZE_V(skb
->len
));
1852 /* Set up TX Packet CPL pointer, control word and perform
1855 cpl
= (void *)(lso
+ 1);
1857 if (CHELSIO_CHIP_VERSION(adapter
->params
.chip
) <= CHELSIO_T5
)
1858 cntrl
= TXPKT_ETHHDR_LEN_V(eth_xtra_len
);
1860 cntrl
= T6_TXPKT_ETHHDR_LEN_V(eth_xtra_len
);
1862 cntrl
|= TXPKT_CSUM_TYPE_V(v6
?
1863 TX_CSUM_TCPIP6
: TX_CSUM_TCPIP
) |
1864 TXPKT_IPHDR_LEN_V(l3hdr_len
);
1866 txq
->tx_cso
+= ssi
->gso_segs
;
1870 len
= (t4vf_is_eth_imm(skb
)
1871 ? skb
->len
+ sizeof(*cpl
)
1874 cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_PKT_VM_WR
) |
1875 FW_WR_IMMDLEN_V(len
));
1877 /* Set up TX Packet CPL pointer, control word and perform
1880 cpl
= (void *)(wr
+ 1);
1881 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
1882 cntrl
= hwcsum(adapter
->params
.chip
, skb
) |
1886 cntrl
= TXPKT_L4CSUM_DIS_F
| TXPKT_IPCSUM_DIS_F
;
1890 /* If there's a VLAN tag present, add that to the list of things to
1891 * do in this Work Request.
1893 if (skb_vlan_tag_present(skb
)) {
1895 cntrl
|= TXPKT_VLAN_VLD_F
| TXPKT_VLAN_V(skb_vlan_tag_get(skb
));
1898 /* Fill in the TX Packet CPL message header. */
1899 cpl
->ctrl0
= cpu_to_be32(TXPKT_OPCODE_V(CPL_TX_PKT_XT
) |
1900 TXPKT_INTF_V(pi
->port_id
) |
1902 cpl
->pack
= cpu_to_be16(0);
1903 cpl
->len
= cpu_to_be16(skb
->len
);
1904 cpl
->ctrl1
= cpu_to_be64(cntrl
);
1906 /* Fill in the body of the TX Packet CPL message with either in-lined
1907 * data or a Scatter/Gather List.
1909 if (t4vf_is_eth_imm(skb
)) {
1910 /* In-line the packet's data and free the skb since we don't
1911 * need it any longer.
1913 cxgb4_inline_tx_skb(skb
, &txq
->q
, cpl
+ 1);
1914 dev_consume_skb_any(skb
);
1916 /* Write the skb's Scatter/Gather list into the TX Packet CPL
1917 * message and retain a pointer to the skb so we can free it
1918 * later when its DMA completes. (We store the skb pointer
1919 * in the Software Descriptor corresponding to the last TX
1920 * Descriptor used by the Work Request.)
1922 * The retained skb will be freed when the corresponding TX
1923 * Descriptors are reclaimed after their DMAs complete.
1924 * However, this could take quite a while since, in general,
1925 * the hardware is set up to be lazy about sending DMA
1926 * completion notifications to us and we mostly perform TX
1927 * reclaims in the transmit routine.
1929 * This is good for performamce but means that we rely on new
1930 * TX packets arriving to run the destructors of completed
1931 * packets, which open up space in their sockets' send queues.
1932 * Sometimes we do not get such new packets causing TX to
1933 * stall. A single UDP transmitter is a good example of this
1934 * situation. We have a clean up timer that periodically
1935 * reclaims completed packets but it doesn't run often enough
1936 * (nor do we want it to) to prevent lengthy stalls. A
1937 * solution to this problem is to run the destructor early,
1938 * after the packet is queued but before it's DMAd. A con is
1939 * that we lie to socket memory accounting, but the amount of
1940 * extra memory is reasonable (limited by the number of TX
1941 * descriptors), the packets do actually get freed quickly by
1942 * new packets almost always, and for protocols like TCP that
1943 * wait for acks to really free up the data the extra memory
1944 * is even less. On the positive side we run the destructors
1945 * on the sending CPU rather than on a potentially different
1946 * completing CPU, usually a good thing.
1948 * Run the destructor before telling the DMA engine about the
1949 * packet to make sure it doesn't complete and get freed
1952 struct ulptx_sgl
*sgl
= (struct ulptx_sgl
*)(cpl
+ 1);
1953 struct sge_txq
*tq
= &txq
->q
;
1956 /* If the Work Request header was an exact multiple of our TX
1957 * Descriptor length, then it's possible that the starting SGL
1958 * pointer lines up exactly with the end of our TX Descriptor
1959 * ring. If that's the case, wrap around to the beginning
1962 if (unlikely((void *)sgl
== (void *)tq
->stat
)) {
1963 sgl
= (void *)tq
->desc
;
1964 end
= (void *)((void *)tq
->desc
+
1965 ((void *)end
- (void *)tq
->stat
));
1968 cxgb4_write_sgl(skb
, tq
, sgl
, end
, 0, addr
);
1971 last_desc
= tq
->pidx
+ ndesc
- 1;
1972 if (last_desc
>= tq
->size
)
1973 last_desc
-= tq
->size
;
1974 tq
->sdesc
[last_desc
].skb
= skb
;
1975 tq
->sdesc
[last_desc
].sgl
= sgl
;
1978 /* Advance our internal TX Queue state, tell the hardware about
1979 * the new TX descriptors and return success.
1981 txq_advance(&txq
->q
, ndesc
);
1983 cxgb4_ring_tx_db(adapter
, &txq
->q
, ndesc
);
1984 return NETDEV_TX_OK
;
1987 /* An error of some sort happened. Free the TX skb and tell the
1988 * OS that we've "dealt" with the packet ...
1990 dev_kfree_skb_any(skb
);
1991 return NETDEV_TX_OK
;
1994 netdev_tx_t
t4_start_xmit(struct sk_buff
*skb
, struct net_device
*dev
)
1996 struct port_info
*pi
= netdev_priv(dev
);
1998 if (unlikely(pi
->eth_flags
& PRIV_FLAG_PORT_TX_VM
))
1999 return cxgb4_vf_eth_xmit(skb
, dev
);
2001 return cxgb4_eth_xmit(skb
, dev
);
2005 * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs
2006 * @q: the SGE control Tx queue
2008 * This is a variant of cxgb4_reclaim_completed_tx() that is used
2009 * for Tx queues that send only immediate data (presently just
2010 * the control queues) and thus do not have any sk_buffs to release.
2012 static inline void reclaim_completed_tx_imm(struct sge_txq
*q
)
2014 int hw_cidx
= ntohs(READ_ONCE(q
->stat
->cidx
));
2015 int reclaim
= hw_cidx
- q
->cidx
;
2020 q
->in_use
-= reclaim
;
2025 * is_imm - check whether a packet can be sent as immediate data
2028 * Returns true if a packet can be sent as a WR with immediate data.
2030 static inline int is_imm(const struct sk_buff
*skb
)
2032 return skb
->len
<= MAX_CTRL_WR_LEN
;
2036 * ctrlq_check_stop - check if a control queue is full and should stop
2038 * @wr: most recent WR written to the queue
2040 * Check if a control queue has become full and should be stopped.
2041 * We clean up control queue descriptors very lazily, only when we are out.
2042 * If the queue is still full after reclaiming any completed descriptors
2043 * we suspend it and have the last WR wake it up.
2045 static void ctrlq_check_stop(struct sge_ctrl_txq
*q
, struct fw_wr_hdr
*wr
)
2047 reclaim_completed_tx_imm(&q
->q
);
2048 if (unlikely(txq_avail(&q
->q
) < TXQ_STOP_THRES
)) {
2049 wr
->lo
|= htonl(FW_WR_EQUEQ_F
| FW_WR_EQUIQ_F
);
2056 * ctrl_xmit - send a packet through an SGE control Tx queue
2057 * @q: the control queue
2060 * Send a packet through an SGE control Tx queue. Packets sent through
2061 * a control queue must fit entirely as immediate data.
2063 static int ctrl_xmit(struct sge_ctrl_txq
*q
, struct sk_buff
*skb
)
2066 struct fw_wr_hdr
*wr
;
2068 if (unlikely(!is_imm(skb
))) {
2071 return NET_XMIT_DROP
;
2074 ndesc
= DIV_ROUND_UP(skb
->len
, sizeof(struct tx_desc
));
2075 spin_lock(&q
->sendq
.lock
);
2077 if (unlikely(q
->full
)) {
2078 skb
->priority
= ndesc
; /* save for restart */
2079 __skb_queue_tail(&q
->sendq
, skb
);
2080 spin_unlock(&q
->sendq
.lock
);
2084 wr
= (struct fw_wr_hdr
*)&q
->q
.desc
[q
->q
.pidx
];
2085 cxgb4_inline_tx_skb(skb
, &q
->q
, wr
);
2087 txq_advance(&q
->q
, ndesc
);
2088 if (unlikely(txq_avail(&q
->q
) < TXQ_STOP_THRES
))
2089 ctrlq_check_stop(q
, wr
);
2091 cxgb4_ring_tx_db(q
->adap
, &q
->q
, ndesc
);
2092 spin_unlock(&q
->sendq
.lock
);
2095 return NET_XMIT_SUCCESS
;
2099 * restart_ctrlq - restart a suspended control queue
2100 * @data: the control queue to restart
2102 * Resumes transmission on a suspended Tx control queue.
2104 static void restart_ctrlq(unsigned long data
)
2106 struct sk_buff
*skb
;
2107 unsigned int written
= 0;
2108 struct sge_ctrl_txq
*q
= (struct sge_ctrl_txq
*)data
;
2110 spin_lock(&q
->sendq
.lock
);
2111 reclaim_completed_tx_imm(&q
->q
);
2112 BUG_ON(txq_avail(&q
->q
) < TXQ_STOP_THRES
); /* q should be empty */
2114 while ((skb
= __skb_dequeue(&q
->sendq
)) != NULL
) {
2115 struct fw_wr_hdr
*wr
;
2116 unsigned int ndesc
= skb
->priority
; /* previously saved */
2119 /* Write descriptors and free skbs outside the lock to limit
2120 * wait times. q->full is still set so new skbs will be queued.
2122 wr
= (struct fw_wr_hdr
*)&q
->q
.desc
[q
->q
.pidx
];
2123 txq_advance(&q
->q
, ndesc
);
2124 spin_unlock(&q
->sendq
.lock
);
2126 cxgb4_inline_tx_skb(skb
, &q
->q
, wr
);
2129 if (unlikely(txq_avail(&q
->q
) < TXQ_STOP_THRES
)) {
2130 unsigned long old
= q
->q
.stops
;
2132 ctrlq_check_stop(q
, wr
);
2133 if (q
->q
.stops
!= old
) { /* suspended anew */
2134 spin_lock(&q
->sendq
.lock
);
2139 cxgb4_ring_tx_db(q
->adap
, &q
->q
, written
);
2142 spin_lock(&q
->sendq
.lock
);
2147 cxgb4_ring_tx_db(q
->adap
, &q
->q
, written
);
2148 spin_unlock(&q
->sendq
.lock
);
2152 * t4_mgmt_tx - send a management message
2153 * @adap: the adapter
2154 * @skb: the packet containing the management message
2156 * Send a management message through control queue 0.
2158 int t4_mgmt_tx(struct adapter
*adap
, struct sk_buff
*skb
)
2163 ret
= ctrl_xmit(&adap
->sge
.ctrlq
[0], skb
);
2169 * is_ofld_imm - check whether a packet can be sent as immediate data
2172 * Returns true if a packet can be sent as an offload WR with immediate
2173 * data. We currently use the same limit as for Ethernet packets.
2175 static inline int is_ofld_imm(const struct sk_buff
*skb
)
2177 struct work_request_hdr
*req
= (struct work_request_hdr
*)skb
->data
;
2178 unsigned long opcode
= FW_WR_OP_G(ntohl(req
->wr_hi
));
2180 if (opcode
== FW_CRYPTO_LOOKASIDE_WR
)
2181 return skb
->len
<= SGE_MAX_WR_LEN
;
2183 return skb
->len
<= MAX_IMM_TX_PKT_LEN
;
2187 * calc_tx_flits_ofld - calculate # of flits for an offload packet
2190 * Returns the number of flits needed for the given offload packet.
2191 * These packets are already fully constructed and no additional headers
2194 static inline unsigned int calc_tx_flits_ofld(const struct sk_buff
*skb
)
2196 unsigned int flits
, cnt
;
2198 if (is_ofld_imm(skb
))
2199 return DIV_ROUND_UP(skb
->len
, 8);
2201 flits
= skb_transport_offset(skb
) / 8U; /* headers */
2202 cnt
= skb_shinfo(skb
)->nr_frags
;
2203 if (skb_tail_pointer(skb
) != skb_transport_header(skb
))
2205 return flits
+ sgl_len(cnt
);
2209 * txq_stop_maperr - stop a Tx queue due to I/O MMU exhaustion
2210 * @adap: the adapter
2211 * @q: the queue to stop
2213 * Mark a Tx queue stopped due to I/O MMU exhaustion and resulting
2214 * inability to map packets. A periodic timer attempts to restart
2217 static void txq_stop_maperr(struct sge_uld_txq
*q
)
2221 set_bit(q
->q
.cntxt_id
- q
->adap
->sge
.egr_start
,
2222 q
->adap
->sge
.txq_maperr
);
2226 * ofldtxq_stop - stop an offload Tx queue that has become full
2227 * @q: the queue to stop
2228 * @wr: the Work Request causing the queue to become full
2230 * Stops an offload Tx queue that has become full and modifies the packet
2231 * being written to request a wakeup.
2233 static void ofldtxq_stop(struct sge_uld_txq
*q
, struct fw_wr_hdr
*wr
)
2235 wr
->lo
|= htonl(FW_WR_EQUEQ_F
| FW_WR_EQUIQ_F
);
2241 * service_ofldq - service/restart a suspended offload queue
2242 * @q: the offload queue
2244 * Services an offload Tx queue by moving packets from its Pending Send
2245 * Queue to the Hardware TX ring. The function starts and ends with the
2246 * Send Queue locked, but drops the lock while putting the skb at the
2247 * head of the Send Queue onto the Hardware TX Ring. Dropping the lock
2248 * allows more skbs to be added to the Send Queue by other threads.
2249 * The packet being processed at the head of the Pending Send Queue is
2250 * left on the queue in case we experience DMA Mapping errors, etc.
2251 * and need to give up and restart later.
2253 * service_ofldq() can be thought of as a task which opportunistically
2254 * uses other threads execution contexts. We use the Offload Queue
2255 * boolean "service_ofldq_running" to make sure that only one instance
2256 * is ever running at a time ...
2258 static void service_ofldq(struct sge_uld_txq
*q
)
2260 u64
*pos
, *before
, *end
;
2262 struct sk_buff
*skb
;
2263 struct sge_txq
*txq
;
2265 unsigned int written
= 0;
2266 unsigned int flits
, ndesc
;
2268 /* If another thread is currently in service_ofldq() processing the
2269 * Pending Send Queue then there's nothing to do. Otherwise, flag
2270 * that we're doing the work and continue. Examining/modifying
2271 * the Offload Queue boolean "service_ofldq_running" must be done
2272 * while holding the Pending Send Queue Lock.
2274 if (q
->service_ofldq_running
)
2276 q
->service_ofldq_running
= true;
2278 while ((skb
= skb_peek(&q
->sendq
)) != NULL
&& !q
->full
) {
2279 /* We drop the lock while we're working with the skb at the
2280 * head of the Pending Send Queue. This allows more skbs to
2281 * be added to the Pending Send Queue while we're working on
2282 * this one. We don't need to lock to guard the TX Ring
2283 * updates because only one thread of execution is ever
2284 * allowed into service_ofldq() at a time.
2286 spin_unlock(&q
->sendq
.lock
);
2288 cxgb4_reclaim_completed_tx(q
->adap
, &q
->q
, false);
2290 flits
= skb
->priority
; /* previously saved */
2291 ndesc
= flits_to_desc(flits
);
2292 credits
= txq_avail(&q
->q
) - ndesc
;
2293 BUG_ON(credits
< 0);
2294 if (unlikely(credits
< TXQ_STOP_THRES
))
2295 ofldtxq_stop(q
, (struct fw_wr_hdr
*)skb
->data
);
2297 pos
= (u64
*)&q
->q
.desc
[q
->q
.pidx
];
2298 if (is_ofld_imm(skb
))
2299 cxgb4_inline_tx_skb(skb
, &q
->q
, pos
);
2300 else if (cxgb4_map_skb(q
->adap
->pdev_dev
, skb
,
2301 (dma_addr_t
*)skb
->head
)) {
2303 spin_lock(&q
->sendq
.lock
);
2306 int last_desc
, hdr_len
= skb_transport_offset(skb
);
2308 /* The WR headers may not fit within one descriptor.
2309 * So we need to deal with wrap-around here.
2311 before
= (u64
*)pos
;
2312 end
= (u64
*)pos
+ flits
;
2314 pos
= (void *)inline_tx_skb_header(skb
, &q
->q
,
2317 if (before
> (u64
*)pos
) {
2318 left
= (u8
*)end
- (u8
*)txq
->stat
;
2319 end
= (void *)txq
->desc
+ left
;
2322 /* If current position is already at the end of the
2323 * ofld queue, reset the current to point to
2324 * start of the queue and update the end ptr as well.
2326 if (pos
== (u64
*)txq
->stat
) {
2327 left
= (u8
*)end
- (u8
*)txq
->stat
;
2328 end
= (void *)txq
->desc
+ left
;
2329 pos
= (void *)txq
->desc
;
2332 cxgb4_write_sgl(skb
, &q
->q
, (void *)pos
,
2334 (dma_addr_t
*)skb
->head
);
2335 #ifdef CONFIG_NEED_DMA_MAP_STATE
2336 skb
->dev
= q
->adap
->port
[0];
2337 skb
->destructor
= deferred_unmap_destructor
;
2339 last_desc
= q
->q
.pidx
+ ndesc
- 1;
2340 if (last_desc
>= q
->q
.size
)
2341 last_desc
-= q
->q
.size
;
2342 q
->q
.sdesc
[last_desc
].skb
= skb
;
2345 txq_advance(&q
->q
, ndesc
);
2347 if (unlikely(written
> 32)) {
2348 cxgb4_ring_tx_db(q
->adap
, &q
->q
, written
);
2352 /* Reacquire the Pending Send Queue Lock so we can unlink the
2353 * skb we've just successfully transferred to the TX Ring and
2354 * loop for the next skb which may be at the head of the
2355 * Pending Send Queue.
2357 spin_lock(&q
->sendq
.lock
);
2358 __skb_unlink(skb
, &q
->sendq
);
2359 if (is_ofld_imm(skb
))
2362 if (likely(written
))
2363 cxgb4_ring_tx_db(q
->adap
, &q
->q
, written
);
2365 /*Indicate that no thread is processing the Pending Send Queue
2368 q
->service_ofldq_running
= false;
2372 * ofld_xmit - send a packet through an offload queue
2373 * @q: the Tx offload queue
2376 * Send an offload packet through an SGE offload queue.
2378 static int ofld_xmit(struct sge_uld_txq
*q
, struct sk_buff
*skb
)
2380 skb
->priority
= calc_tx_flits_ofld(skb
); /* save for restart */
2381 spin_lock(&q
->sendq
.lock
);
2383 /* Queue the new skb onto the Offload Queue's Pending Send Queue. If
2384 * that results in this new skb being the only one on the queue, start
2385 * servicing it. If there are other skbs already on the list, then
2386 * either the queue is currently being processed or it's been stopped
2387 * for some reason and it'll be restarted at a later time. Restart
2388 * paths are triggered by events like experiencing a DMA Mapping Error
2389 * or filling the Hardware TX Ring.
2391 __skb_queue_tail(&q
->sendq
, skb
);
2392 if (q
->sendq
.qlen
== 1)
2395 spin_unlock(&q
->sendq
.lock
);
2396 return NET_XMIT_SUCCESS
;
2400 * restart_ofldq - restart a suspended offload queue
2401 * @data: the offload queue to restart
2403 * Resumes transmission on a suspended Tx offload queue.
2405 static void restart_ofldq(unsigned long data
)
2407 struct sge_uld_txq
*q
= (struct sge_uld_txq
*)data
;
2409 spin_lock(&q
->sendq
.lock
);
2410 q
->full
= 0; /* the queue actually is completely empty now */
2412 spin_unlock(&q
->sendq
.lock
);
2416 * skb_txq - return the Tx queue an offload packet should use
2419 * Returns the Tx queue an offload packet should use as indicated by bits
2420 * 1-15 in the packet's queue_mapping.
2422 static inline unsigned int skb_txq(const struct sk_buff
*skb
)
2424 return skb
->queue_mapping
>> 1;
2428 * is_ctrl_pkt - return whether an offload packet is a control packet
2431 * Returns whether an offload packet should use an OFLD or a CTRL
2432 * Tx queue as indicated by bit 0 in the packet's queue_mapping.
2434 static inline unsigned int is_ctrl_pkt(const struct sk_buff
*skb
)
2436 return skb
->queue_mapping
& 1;
2439 static inline int uld_send(struct adapter
*adap
, struct sk_buff
*skb
,
2440 unsigned int tx_uld_type
)
2442 struct sge_uld_txq_info
*txq_info
;
2443 struct sge_uld_txq
*txq
;
2444 unsigned int idx
= skb_txq(skb
);
2446 if (unlikely(is_ctrl_pkt(skb
))) {
2447 /* Single ctrl queue is a requirement for LE workaround path */
2448 if (adap
->tids
.nsftids
)
2450 return ctrl_xmit(&adap
->sge
.ctrlq
[idx
], skb
);
2453 txq_info
= adap
->sge
.uld_txq_info
[tx_uld_type
];
2454 if (unlikely(!txq_info
)) {
2456 return NET_XMIT_DROP
;
2459 txq
= &txq_info
->uldtxq
[idx
];
2460 return ofld_xmit(txq
, skb
);
2464 * t4_ofld_send - send an offload packet
2465 * @adap: the adapter
2468 * Sends an offload packet. We use the packet queue_mapping to select the
2469 * appropriate Tx queue as follows: bit 0 indicates whether the packet
2470 * should be sent as regular or control, bits 1-15 select the queue.
2472 int t4_ofld_send(struct adapter
*adap
, struct sk_buff
*skb
)
2477 ret
= uld_send(adap
, skb
, CXGB4_TX_OFLD
);
2483 * cxgb4_ofld_send - send an offload packet
2484 * @dev: the net device
2487 * Sends an offload packet. This is an exported version of @t4_ofld_send,
2488 * intended for ULDs.
2490 int cxgb4_ofld_send(struct net_device
*dev
, struct sk_buff
*skb
)
2492 return t4_ofld_send(netdev2adap(dev
), skb
);
2494 EXPORT_SYMBOL(cxgb4_ofld_send
);
2496 static void *inline_tx_header(const void *src
,
2497 const struct sge_txq
*q
,
2498 void *pos
, int length
)
2500 int left
= (void *)q
->stat
- pos
;
2503 if (likely(length
<= left
)) {
2504 memcpy(pos
, src
, length
);
2507 memcpy(pos
, src
, left
);
2508 memcpy(q
->desc
, src
+ left
, length
- left
);
2509 pos
= (void *)q
->desc
+ (length
- left
);
2511 /* 0-pad to multiple of 16 */
2512 p
= PTR_ALIGN(pos
, 8);
2513 if ((uintptr_t)p
& 8) {
2521 * ofld_xmit_direct - copy a WR into offload queue
2522 * @q: the Tx offload queue
2523 * @src: location of WR
2526 * Copy an immediate WR into an uncontended SGE offload queue.
2528 static int ofld_xmit_direct(struct sge_uld_txq
*q
, const void *src
,
2535 /* Use the lower limit as the cut-off */
2536 if (len
> MAX_IMM_OFLD_TX_DATA_WR_LEN
) {
2538 return NET_XMIT_DROP
;
2541 /* Don't return NET_XMIT_CN here as the current
2542 * implementation doesn't queue the request
2543 * using an skb when the following conditions not met
2545 if (!spin_trylock(&q
->sendq
.lock
))
2546 return NET_XMIT_DROP
;
2548 if (q
->full
|| !skb_queue_empty(&q
->sendq
) ||
2549 q
->service_ofldq_running
) {
2550 spin_unlock(&q
->sendq
.lock
);
2551 return NET_XMIT_DROP
;
2553 ndesc
= flits_to_desc(DIV_ROUND_UP(len
, 8));
2554 credits
= txq_avail(&q
->q
) - ndesc
;
2555 pos
= (u64
*)&q
->q
.desc
[q
->q
.pidx
];
2557 /* ofldtxq_stop modifies WR header in-situ */
2558 inline_tx_header(src
, &q
->q
, pos
, len
);
2559 if (unlikely(credits
< TXQ_STOP_THRES
))
2560 ofldtxq_stop(q
, (struct fw_wr_hdr
*)pos
);
2561 txq_advance(&q
->q
, ndesc
);
2562 cxgb4_ring_tx_db(q
->adap
, &q
->q
, ndesc
);
2564 spin_unlock(&q
->sendq
.lock
);
2565 return NET_XMIT_SUCCESS
;
2568 int cxgb4_immdata_send(struct net_device
*dev
, unsigned int idx
,
2569 const void *src
, unsigned int len
)
2571 struct sge_uld_txq_info
*txq_info
;
2572 struct sge_uld_txq
*txq
;
2573 struct adapter
*adap
;
2576 adap
= netdev2adap(dev
);
2579 txq_info
= adap
->sge
.uld_txq_info
[CXGB4_TX_OFLD
];
2580 if (unlikely(!txq_info
)) {
2583 return NET_XMIT_DROP
;
2585 txq
= &txq_info
->uldtxq
[idx
];
2587 ret
= ofld_xmit_direct(txq
, src
, len
);
2589 return net_xmit_eval(ret
);
2591 EXPORT_SYMBOL(cxgb4_immdata_send
);
2594 * t4_crypto_send - send crypto packet
2595 * @adap: the adapter
2598 * Sends crypto packet. We use the packet queue_mapping to select the
2599 * appropriate Tx queue as follows: bit 0 indicates whether the packet
2600 * should be sent as regular or control, bits 1-15 select the queue.
2602 static int t4_crypto_send(struct adapter
*adap
, struct sk_buff
*skb
)
2607 ret
= uld_send(adap
, skb
, CXGB4_TX_CRYPTO
);
2613 * cxgb4_crypto_send - send crypto packet
2614 * @dev: the net device
2617 * Sends crypto packet. This is an exported version of @t4_crypto_send,
2618 * intended for ULDs.
2620 int cxgb4_crypto_send(struct net_device
*dev
, struct sk_buff
*skb
)
2622 return t4_crypto_send(netdev2adap(dev
), skb
);
2624 EXPORT_SYMBOL(cxgb4_crypto_send
);
2626 static inline void copy_frags(struct sk_buff
*skb
,
2627 const struct pkt_gl
*gl
, unsigned int offset
)
2631 /* usually there's just one frag */
2632 __skb_fill_page_desc(skb
, 0, gl
->frags
[0].page
,
2633 gl
->frags
[0].offset
+ offset
,
2634 gl
->frags
[0].size
- offset
);
2635 skb_shinfo(skb
)->nr_frags
= gl
->nfrags
;
2636 for (i
= 1; i
< gl
->nfrags
; i
++)
2637 __skb_fill_page_desc(skb
, i
, gl
->frags
[i
].page
,
2638 gl
->frags
[i
].offset
,
2641 /* get a reference to the last page, we don't own it */
2642 get_page(gl
->frags
[gl
->nfrags
- 1].page
);
2646 * cxgb4_pktgl_to_skb - build an sk_buff from a packet gather list
2647 * @gl: the gather list
2648 * @skb_len: size of sk_buff main body if it carries fragments
2649 * @pull_len: amount of data to move to the sk_buff's main body
2651 * Builds an sk_buff from the given packet gather list. Returns the
2652 * sk_buff or %NULL if sk_buff allocation failed.
2654 struct sk_buff
*cxgb4_pktgl_to_skb(const struct pkt_gl
*gl
,
2655 unsigned int skb_len
, unsigned int pull_len
)
2657 struct sk_buff
*skb
;
2660 * Below we rely on RX_COPY_THRES being less than the smallest Rx buffer
2661 * size, which is expected since buffers are at least PAGE_SIZEd.
2662 * In this case packets up to RX_COPY_THRES have only one fragment.
2664 if (gl
->tot_len
<= RX_COPY_THRES
) {
2665 skb
= dev_alloc_skb(gl
->tot_len
);
2668 __skb_put(skb
, gl
->tot_len
);
2669 skb_copy_to_linear_data(skb
, gl
->va
, gl
->tot_len
);
2671 skb
= dev_alloc_skb(skb_len
);
2674 __skb_put(skb
, pull_len
);
2675 skb_copy_to_linear_data(skb
, gl
->va
, pull_len
);
2677 copy_frags(skb
, gl
, pull_len
);
2678 skb
->len
= gl
->tot_len
;
2679 skb
->data_len
= skb
->len
- pull_len
;
2680 skb
->truesize
+= skb
->data_len
;
2684 EXPORT_SYMBOL(cxgb4_pktgl_to_skb
);
2687 * t4_pktgl_free - free a packet gather list
2688 * @gl: the gather list
2690 * Releases the pages of a packet gather list. We do not own the last
2691 * page on the list and do not free it.
2693 static void t4_pktgl_free(const struct pkt_gl
*gl
)
2696 const struct page_frag
*p
;
2698 for (p
= gl
->frags
, n
= gl
->nfrags
- 1; n
--; p
++)
2703 * Process an MPS trace packet. Give it an unused protocol number so it won't
2704 * be delivered to anyone and send it to the stack for capture.
2706 static noinline
int handle_trace_pkt(struct adapter
*adap
,
2707 const struct pkt_gl
*gl
)
2709 struct sk_buff
*skb
;
2711 skb
= cxgb4_pktgl_to_skb(gl
, RX_PULL_LEN
, RX_PULL_LEN
);
2712 if (unlikely(!skb
)) {
2717 if (is_t4(adap
->params
.chip
))
2718 __skb_pull(skb
, sizeof(struct cpl_trace_pkt
));
2720 __skb_pull(skb
, sizeof(struct cpl_t5_trace_pkt
));
2722 skb_reset_mac_header(skb
);
2723 skb
->protocol
= htons(0xffff);
2724 skb
->dev
= adap
->port
[0];
2725 netif_receive_skb(skb
);
2730 * cxgb4_sgetim_to_hwtstamp - convert sge time stamp to hw time stamp
2731 * @adap: the adapter
2732 * @hwtstamps: time stamp structure to update
2733 * @sgetstamp: 60bit iqe timestamp
2735 * Every ingress queue entry has the 60-bit timestamp, convert that timestamp
2736 * which is in Core Clock ticks into ktime_t and assign it
2738 static void cxgb4_sgetim_to_hwtstamp(struct adapter
*adap
,
2739 struct skb_shared_hwtstamps
*hwtstamps
,
2743 u64 tmp
= (sgetstamp
* 1000 * 1000 + adap
->params
.vpd
.cclk
/ 2);
2745 ns
= div_u64(tmp
, adap
->params
.vpd
.cclk
);
2747 memset(hwtstamps
, 0, sizeof(*hwtstamps
));
2748 hwtstamps
->hwtstamp
= ns_to_ktime(ns
);
2751 static void do_gro(struct sge_eth_rxq
*rxq
, const struct pkt_gl
*gl
,
2752 const struct cpl_rx_pkt
*pkt
, unsigned long tnl_hdr_len
)
2754 struct adapter
*adapter
= rxq
->rspq
.adap
;
2755 struct sge
*s
= &adapter
->sge
;
2756 struct port_info
*pi
;
2758 struct sk_buff
*skb
;
2760 skb
= napi_get_frags(&rxq
->rspq
.napi
);
2761 if (unlikely(!skb
)) {
2763 rxq
->stats
.rx_drops
++;
2767 copy_frags(skb
, gl
, s
->pktshift
);
2769 skb
->csum_level
= 1;
2770 skb
->len
= gl
->tot_len
- s
->pktshift
;
2771 skb
->data_len
= skb
->len
;
2772 skb
->truesize
+= skb
->data_len
;
2773 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
2774 skb_record_rx_queue(skb
, rxq
->rspq
.idx
);
2775 pi
= netdev_priv(skb
->dev
);
2777 cxgb4_sgetim_to_hwtstamp(adapter
, skb_hwtstamps(skb
),
2779 if (rxq
->rspq
.netdev
->features
& NETIF_F_RXHASH
)
2780 skb_set_hash(skb
, (__force u32
)pkt
->rsshdr
.hash_val
,
2783 if (unlikely(pkt
->vlan_ex
)) {
2784 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
), ntohs(pkt
->vlan
));
2785 rxq
->stats
.vlan_ex
++;
2787 ret
= napi_gro_frags(&rxq
->rspq
.napi
);
2788 if (ret
== GRO_HELD
)
2789 rxq
->stats
.lro_pkts
++;
2790 else if (ret
== GRO_MERGED
|| ret
== GRO_MERGED_FREE
)
2791 rxq
->stats
.lro_merged
++;
2793 rxq
->stats
.rx_cso
++;
2803 * t4_systim_to_hwstamp - read hardware time stamp
2804 * @adap: the adapter
2807 * Read Time Stamp from MPS packet and insert in skb which
2808 * is forwarded to PTP application
2810 static noinline
int t4_systim_to_hwstamp(struct adapter
*adapter
,
2811 struct sk_buff
*skb
)
2813 struct skb_shared_hwtstamps
*hwtstamps
;
2814 struct cpl_rx_mps_pkt
*cpl
= NULL
;
2815 unsigned char *data
;
2818 cpl
= (struct cpl_rx_mps_pkt
*)skb
->data
;
2819 if (!(CPL_RX_MPS_PKT_TYPE_G(ntohl(cpl
->op_to_r1_hi
)) &
2820 X_CPL_RX_MPS_PKT_TYPE_PTP
))
2821 return RX_PTP_PKT_ERR
;
2823 data
= skb
->data
+ sizeof(*cpl
);
2824 skb_pull(skb
, 2 * sizeof(u64
) + sizeof(struct cpl_rx_mps_pkt
));
2825 offset
= ETH_HLEN
+ IPV4_HLEN(skb
->data
) + UDP_HLEN
;
2826 if (skb
->len
< offset
+ OFF_PTP_SEQUENCE_ID
+ sizeof(short))
2827 return RX_PTP_PKT_ERR
;
2829 hwtstamps
= skb_hwtstamps(skb
);
2830 memset(hwtstamps
, 0, sizeof(*hwtstamps
));
2831 hwtstamps
->hwtstamp
= ns_to_ktime(be64_to_cpu(*((u64
*)data
)));
2833 return RX_PTP_PKT_SUC
;
2837 * t4_rx_hststamp - Recv PTP Event Message
2838 * @adap: the adapter
2839 * @rsp: the response queue descriptor holding the RX_PKT message
2842 * PTP enabled and MPS packet, read HW timestamp
2844 static int t4_rx_hststamp(struct adapter
*adapter
, const __be64
*rsp
,
2845 struct sge_eth_rxq
*rxq
, struct sk_buff
*skb
)
2849 if (unlikely((*(u8
*)rsp
== CPL_RX_MPS_PKT
) &&
2850 !is_t4(adapter
->params
.chip
))) {
2851 ret
= t4_systim_to_hwstamp(adapter
, skb
);
2852 if (ret
== RX_PTP_PKT_ERR
) {
2854 rxq
->stats
.rx_drops
++;
2858 return RX_NON_PTP_PKT
;
2862 * t4_tx_hststamp - Loopback PTP Transmit Event Message
2863 * @adap: the adapter
2865 * @dev: the ingress net device
2867 * Read hardware timestamp for the loopback PTP Tx event message
2869 static int t4_tx_hststamp(struct adapter
*adapter
, struct sk_buff
*skb
,
2870 struct net_device
*dev
)
2872 struct port_info
*pi
= netdev_priv(dev
);
2874 if (!is_t4(adapter
->params
.chip
) && adapter
->ptp_tx_skb
) {
2875 cxgb4_ptp_read_hwstamp(adapter
, pi
);
2883 * t4_tx_completion_handler - handle CPL_SGE_EGR_UPDATE messages
2884 * @rspq: Ethernet RX Response Queue associated with Ethernet TX Queue
2885 * @rsp: Response Entry pointer into Response Queue
2886 * @gl: Gather List pointer
2888 * For adapters which support the SGE Doorbell Queue Timer facility,
2889 * we configure the Ethernet TX Queues to send CIDX Updates to the
2890 * Associated Ethernet RX Response Queue with CPL_SGE_EGR_UPDATE
2891 * messages. This adds a small load to PCIe Link RX bandwidth and,
2892 * potentially, higher CPU Interrupt load, but allows us to respond
2893 * much more quickly to the CIDX Updates. This is important for
2894 * Upper Layer Software which isn't willing to have a large amount
2895 * of TX Data outstanding before receiving DMA Completions.
2897 static void t4_tx_completion_handler(struct sge_rspq
*rspq
,
2899 const struct pkt_gl
*gl
)
2901 u8 opcode
= ((const struct rss_header
*)rsp
)->opcode
;
2902 struct port_info
*pi
= netdev_priv(rspq
->netdev
);
2903 struct adapter
*adapter
= rspq
->adap
;
2904 struct sge
*s
= &adapter
->sge
;
2905 struct sge_eth_txq
*txq
;
2907 /* skip RSS header */
2910 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
2912 if (unlikely(opcode
== CPL_FW4_MSG
&&
2913 ((const struct cpl_fw4_msg
*)rsp
)->type
==
2916 opcode
= ((const struct rss_header
*)rsp
)->opcode
;
2920 if (unlikely(opcode
!= CPL_SGE_EGR_UPDATE
)) {
2921 pr_info("%s: unexpected FW4/CPL %#x on Rx queue\n",
2926 txq
= &s
->ethtxq
[pi
->first_qset
+ rspq
->idx
];
2928 /* We've got the Hardware Consumer Index Update in the Egress Update
2929 * message. If we're using the SGE Doorbell Queue Timer mechanism,
2930 * these Egress Update messages will be our sole CIDX Updates we get
2931 * since we don't want to chew up PCIe bandwidth for both Ingress
2932 * Messages and Status Page writes. However, The code which manages
2933 * reclaiming successfully DMA'ed TX Work Requests uses the CIDX value
2934 * stored in the Status Page at the end of the TX Queue. It's easiest
2935 * to simply copy the CIDX Update value from the Egress Update message
2936 * to the Status Page. Also note that no Endian issues need to be
2937 * considered here since both are Big Endian and we're just copying
2938 * bytes consistently ...
2941 struct cpl_sge_egr_update
*egr
;
2943 egr
= (struct cpl_sge_egr_update
*)rsp
;
2944 WRITE_ONCE(txq
->q
.stat
->cidx
, egr
->cidx
);
2947 t4_sge_eth_txq_egress_update(adapter
, txq
, -1);
2951 * t4_ethrx_handler - process an ingress ethernet packet
2952 * @q: the response queue that received the packet
2953 * @rsp: the response queue descriptor holding the RX_PKT message
2954 * @si: the gather list of packet fragments
2956 * Process an ingress ethernet packet and deliver it to the stack.
2958 int t4_ethrx_handler(struct sge_rspq
*q
, const __be64
*rsp
,
2959 const struct pkt_gl
*si
)
2962 struct sk_buff
*skb
;
2963 const struct cpl_rx_pkt
*pkt
;
2964 struct sge_eth_rxq
*rxq
= container_of(q
, struct sge_eth_rxq
, rspq
);
2965 struct adapter
*adapter
= q
->adap
;
2966 struct sge
*s
= &q
->adap
->sge
;
2967 int cpl_trace_pkt
= is_t4(q
->adap
->params
.chip
) ?
2968 CPL_TRACE_PKT
: CPL_TRACE_PKT_T5
;
2969 u16 err_vec
, tnl_hdr_len
= 0;
2970 struct port_info
*pi
;
2973 /* If we're looking at TX Queue CIDX Update, handle that separately
2976 if (unlikely((*(u8
*)rsp
== CPL_FW4_MSG
) ||
2977 (*(u8
*)rsp
== CPL_SGE_EGR_UPDATE
))) {
2978 t4_tx_completion_handler(q
, rsp
, si
);
2982 if (unlikely(*(u8
*)rsp
== cpl_trace_pkt
))
2983 return handle_trace_pkt(q
->adap
, si
);
2985 pkt
= (const struct cpl_rx_pkt
*)rsp
;
2986 /* Compressed error vector is enabled for T6 only */
2987 if (q
->adap
->params
.tp
.rx_pkt_encap
) {
2988 err_vec
= T6_COMPR_RXERR_VEC_G(be16_to_cpu(pkt
->err_vec
));
2989 tnl_hdr_len
= T6_RX_TNLHDR_LEN_G(ntohs(pkt
->err_vec
));
2991 err_vec
= be16_to_cpu(pkt
->err_vec
);
2994 csum_ok
= pkt
->csum_calc
&& !err_vec
&&
2995 (q
->netdev
->features
& NETIF_F_RXCSUM
);
2998 rxq
->stats
.bad_rx_pkts
++;
3000 if (((pkt
->l2info
& htonl(RXF_TCP_F
)) ||
3002 (q
->netdev
->features
& NETIF_F_GRO
) && csum_ok
&& !pkt
->ip_frag
) {
3003 do_gro(rxq
, si
, pkt
, tnl_hdr_len
);
3007 skb
= cxgb4_pktgl_to_skb(si
, RX_PKT_SKB_LEN
, RX_PULL_LEN
);
3008 if (unlikely(!skb
)) {
3010 rxq
->stats
.rx_drops
++;
3013 pi
= netdev_priv(q
->netdev
);
3015 /* Handle PTP Event Rx packet */
3016 if (unlikely(pi
->ptp_enable
)) {
3017 ret
= t4_rx_hststamp(adapter
, rsp
, rxq
, skb
);
3018 if (ret
== RX_PTP_PKT_ERR
)
3022 __skb_pull(skb
, s
->pktshift
); /* remove ethernet header pad */
3024 /* Handle the PTP Event Tx Loopback packet */
3025 if (unlikely(pi
->ptp_enable
&& !ret
&&
3026 (pkt
->l2info
& htonl(RXF_UDP_F
)) &&
3027 cxgb4_ptp_is_ptp_rx(skb
))) {
3028 if (!t4_tx_hststamp(adapter
, skb
, q
->netdev
))
3032 skb
->protocol
= eth_type_trans(skb
, q
->netdev
);
3033 skb_record_rx_queue(skb
, q
->idx
);
3034 if (skb
->dev
->features
& NETIF_F_RXHASH
)
3035 skb_set_hash(skb
, (__force u32
)pkt
->rsshdr
.hash_val
,
3041 cxgb4_sgetim_to_hwtstamp(q
->adap
, skb_hwtstamps(skb
),
3043 if (csum_ok
&& (pkt
->l2info
& htonl(RXF_UDP_F
| RXF_TCP_F
))) {
3044 if (!pkt
->ip_frag
) {
3045 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
3046 rxq
->stats
.rx_cso
++;
3047 } else if (pkt
->l2info
& htonl(RXF_IP_F
)) {
3048 __sum16 c
= (__force __sum16
)pkt
->csum
;
3049 skb
->csum
= csum_unfold(c
);
3052 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
3053 skb
->csum_level
= 1;
3055 skb
->ip_summed
= CHECKSUM_COMPLETE
;
3057 rxq
->stats
.rx_cso
++;
3060 skb_checksum_none_assert(skb
);
3061 #ifdef CONFIG_CHELSIO_T4_FCOE
3062 #define CPL_RX_PKT_FLAGS (RXF_PSH_F | RXF_SYN_F | RXF_UDP_F | \
3063 RXF_TCP_F | RXF_IP_F | RXF_IP6_F | RXF_LRO_F)
3065 if (!(pkt
->l2info
& cpu_to_be32(CPL_RX_PKT_FLAGS
))) {
3066 if ((pkt
->l2info
& cpu_to_be32(RXF_FCOE_F
)) &&
3067 (pi
->fcoe
.flags
& CXGB_FCOE_ENABLED
)) {
3068 if (q
->adap
->params
.tp
.rx_pkt_encap
)
3070 T6_COMPR_RXERR_SUM_F
;
3072 csum_ok
= err_vec
& RXERR_CSUM_F
;
3074 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
3078 #undef CPL_RX_PKT_FLAGS
3079 #endif /* CONFIG_CHELSIO_T4_FCOE */
3082 if (unlikely(pkt
->vlan_ex
)) {
3083 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
), ntohs(pkt
->vlan
));
3084 rxq
->stats
.vlan_ex
++;
3086 skb_mark_napi_id(skb
, &q
->napi
);
3087 netif_receive_skb(skb
);
3092 * restore_rx_bufs - put back a packet's Rx buffers
3093 * @si: the packet gather list
3094 * @q: the SGE free list
3095 * @frags: number of FL buffers to restore
3097 * Puts back on an FL the Rx buffers associated with @si. The buffers
3098 * have already been unmapped and are left unmapped, we mark them so to
3099 * prevent further unmapping attempts.
3101 * This function undoes a series of @unmap_rx_buf calls when we find out
3102 * that the current packet can't be processed right away afterall and we
3103 * need to come back to it later. This is a very rare event and there's
3104 * no effort to make this particularly efficient.
3106 static void restore_rx_bufs(const struct pkt_gl
*si
, struct sge_fl
*q
,
3109 struct rx_sw_desc
*d
;
3113 q
->cidx
= q
->size
- 1;
3116 d
= &q
->sdesc
[q
->cidx
];
3117 d
->page
= si
->frags
[frags
].page
;
3118 d
->dma_addr
|= RX_UNMAPPED_BUF
;
3124 * is_new_response - check if a response is newly written
3125 * @r: the response descriptor
3126 * @q: the response queue
3128 * Returns true if a response descriptor contains a yet unprocessed
3131 static inline bool is_new_response(const struct rsp_ctrl
*r
,
3132 const struct sge_rspq
*q
)
3134 return (r
->type_gen
>> RSPD_GEN_S
) == q
->gen
;
3138 * rspq_next - advance to the next entry in a response queue
3141 * Updates the state of a response queue to advance it to the next entry.
3143 static inline void rspq_next(struct sge_rspq
*q
)
3145 q
->cur_desc
= (void *)q
->cur_desc
+ q
->iqe_len
;
3146 if (unlikely(++q
->cidx
== q
->size
)) {
3149 q
->cur_desc
= q
->desc
;
3154 * process_responses - process responses from an SGE response queue
3155 * @q: the ingress queue to process
3156 * @budget: how many responses can be processed in this round
3158 * Process responses from an SGE response queue up to the supplied budget.
3159 * Responses include received packets as well as control messages from FW
3162 * Additionally choose the interrupt holdoff time for the next interrupt
3163 * on this queue. If the system is under memory shortage use a fairly
3164 * long delay to help recovery.
3166 static int process_responses(struct sge_rspq
*q
, int budget
)
3169 int budget_left
= budget
;
3170 const struct rsp_ctrl
*rc
;
3171 struct sge_eth_rxq
*rxq
= container_of(q
, struct sge_eth_rxq
, rspq
);
3172 struct adapter
*adapter
= q
->adap
;
3173 struct sge
*s
= &adapter
->sge
;
3175 while (likely(budget_left
)) {
3176 rc
= (void *)q
->cur_desc
+ (q
->iqe_len
- sizeof(*rc
));
3177 if (!is_new_response(rc
, q
)) {
3178 if (q
->flush_handler
)
3179 q
->flush_handler(q
);
3184 rsp_type
= RSPD_TYPE_G(rc
->type_gen
);
3185 if (likely(rsp_type
== RSPD_TYPE_FLBUF_X
)) {
3186 struct page_frag
*fp
;
3188 const struct rx_sw_desc
*rsd
;
3189 u32 len
= ntohl(rc
->pldbuflen_qid
), bufsz
, frags
;
3191 if (len
& RSPD_NEWBUF_F
) {
3192 if (likely(q
->offset
> 0)) {
3193 free_rx_bufs(q
->adap
, &rxq
->fl
, 1);
3196 len
= RSPD_LEN_G(len
);
3200 /* gather packet fragments */
3201 for (frags
= 0, fp
= si
.frags
; ; frags
++, fp
++) {
3202 rsd
= &rxq
->fl
.sdesc
[rxq
->fl
.cidx
];
3203 bufsz
= get_buf_size(adapter
, rsd
);
3204 fp
->page
= rsd
->page
;
3205 fp
->offset
= q
->offset
;
3206 fp
->size
= min(bufsz
, len
);
3210 unmap_rx_buf(q
->adap
, &rxq
->fl
);
3213 si
.sgetstamp
= SGE_TIMESTAMP_G(
3214 be64_to_cpu(rc
->last_flit
));
3216 * Last buffer remains mapped so explicitly make it
3217 * coherent for CPU access.
3219 dma_sync_single_for_cpu(q
->adap
->pdev_dev
,
3221 fp
->size
, DMA_FROM_DEVICE
);
3223 si
.va
= page_address(si
.frags
[0].page
) +
3227 si
.nfrags
= frags
+ 1;
3228 ret
= q
->handler(q
, q
->cur_desc
, &si
);
3229 if (likely(ret
== 0))
3230 q
->offset
+= ALIGN(fp
->size
, s
->fl_align
);
3232 restore_rx_bufs(&si
, &rxq
->fl
, frags
);
3233 } else if (likely(rsp_type
== RSPD_TYPE_CPL_X
)) {
3234 ret
= q
->handler(q
, q
->cur_desc
, NULL
);
3236 ret
= q
->handler(q
, (const __be64
*)rc
, CXGB4_MSG_AN
);
3239 if (unlikely(ret
)) {
3240 /* couldn't process descriptor, back off for recovery */
3241 q
->next_intr_params
= QINTR_TIMER_IDX_V(NOMEM_TMR_IDX
);
3249 if (q
->offset
>= 0 && fl_cap(&rxq
->fl
) - rxq
->fl
.avail
>= 16)
3250 __refill_fl(q
->adap
, &rxq
->fl
);
3251 return budget
- budget_left
;
3255 * napi_rx_handler - the NAPI handler for Rx processing
3256 * @napi: the napi instance
3257 * @budget: how many packets we can process in this round
3259 * Handler for new data events when using NAPI. This does not need any
3260 * locking or protection from interrupts as data interrupts are off at
3261 * this point and other adapter interrupts do not interfere (the latter
3262 * in not a concern at all with MSI-X as non-data interrupts then have
3263 * a separate handler).
3265 static int napi_rx_handler(struct napi_struct
*napi
, int budget
)
3267 unsigned int params
;
3268 struct sge_rspq
*q
= container_of(napi
, struct sge_rspq
, napi
);
3272 work_done
= process_responses(q
, budget
);
3273 if (likely(work_done
< budget
)) {
3276 napi_complete_done(napi
, work_done
);
3277 timer_index
= QINTR_TIMER_IDX_G(q
->next_intr_params
);
3279 if (q
->adaptive_rx
) {
3280 if (work_done
> max(timer_pkt_quota
[timer_index
],
3282 timer_index
= (timer_index
+ 1);
3284 timer_index
= timer_index
- 1;
3286 timer_index
= clamp(timer_index
, 0, SGE_TIMERREGS
- 1);
3287 q
->next_intr_params
=
3288 QINTR_TIMER_IDX_V(timer_index
) |
3290 params
= q
->next_intr_params
;
3292 params
= q
->next_intr_params
;
3293 q
->next_intr_params
= q
->intr_params
;
3296 params
= QINTR_TIMER_IDX_V(7);
3298 val
= CIDXINC_V(work_done
) | SEINTARM_V(params
);
3300 /* If we don't have access to the new User GTS (T5+), use the old
3301 * doorbell mechanism; otherwise use the new BAR2 mechanism.
3303 if (unlikely(q
->bar2_addr
== NULL
)) {
3304 t4_write_reg(q
->adap
, MYPF_REG(SGE_PF_GTS_A
),
3305 val
| INGRESSQID_V((u32
)q
->cntxt_id
));
3307 writel(val
| INGRESSQID_V(q
->bar2_qid
),
3308 q
->bar2_addr
+ SGE_UDB_GTS
);
3315 * The MSI-X interrupt handler for an SGE response queue.
3317 irqreturn_t
t4_sge_intr_msix(int irq
, void *cookie
)
3319 struct sge_rspq
*q
= cookie
;
3321 napi_schedule(&q
->napi
);
3326 * Process the indirect interrupt entries in the interrupt queue and kick off
3327 * NAPI for each queue that has generated an entry.
3329 static unsigned int process_intrq(struct adapter
*adap
)
3331 unsigned int credits
;
3332 const struct rsp_ctrl
*rc
;
3333 struct sge_rspq
*q
= &adap
->sge
.intrq
;
3336 spin_lock(&adap
->sge
.intrq_lock
);
3337 for (credits
= 0; ; credits
++) {
3338 rc
= (void *)q
->cur_desc
+ (q
->iqe_len
- sizeof(*rc
));
3339 if (!is_new_response(rc
, q
))
3343 if (RSPD_TYPE_G(rc
->type_gen
) == RSPD_TYPE_INTR_X
) {
3344 unsigned int qid
= ntohl(rc
->pldbuflen_qid
);
3346 qid
-= adap
->sge
.ingr_start
;
3347 napi_schedule(&adap
->sge
.ingr_map
[qid
]->napi
);
3353 val
= CIDXINC_V(credits
) | SEINTARM_V(q
->intr_params
);
3355 /* If we don't have access to the new User GTS (T5+), use the old
3356 * doorbell mechanism; otherwise use the new BAR2 mechanism.
3358 if (unlikely(q
->bar2_addr
== NULL
)) {
3359 t4_write_reg(adap
, MYPF_REG(SGE_PF_GTS_A
),
3360 val
| INGRESSQID_V(q
->cntxt_id
));
3362 writel(val
| INGRESSQID_V(q
->bar2_qid
),
3363 q
->bar2_addr
+ SGE_UDB_GTS
);
3366 spin_unlock(&adap
->sge
.intrq_lock
);
3371 * The MSI interrupt handler, which handles data events from SGE response queues
3372 * as well as error and other async events as they all use the same MSI vector.
3374 static irqreturn_t
t4_intr_msi(int irq
, void *cookie
)
3376 struct adapter
*adap
= cookie
;
3378 if (adap
->flags
& CXGB4_MASTER_PF
)
3379 t4_slow_intr_handler(adap
);
3380 process_intrq(adap
);
3385 * Interrupt handler for legacy INTx interrupts.
3386 * Handles data events from SGE response queues as well as error and other
3387 * async events as they all use the same interrupt line.
3389 static irqreturn_t
t4_intr_intx(int irq
, void *cookie
)
3391 struct adapter
*adap
= cookie
;
3393 t4_write_reg(adap
, MYPF_REG(PCIE_PF_CLI_A
), 0);
3394 if (((adap
->flags
& CXGB4_MASTER_PF
) && t4_slow_intr_handler(adap
)) |
3395 process_intrq(adap
))
3397 return IRQ_NONE
; /* probably shared interrupt */
3401 * t4_intr_handler - select the top-level interrupt handler
3402 * @adap: the adapter
3404 * Selects the top-level interrupt handler based on the type of interrupts
3405 * (MSI-X, MSI, or INTx).
3407 irq_handler_t
t4_intr_handler(struct adapter
*adap
)
3409 if (adap
->flags
& CXGB4_USING_MSIX
)
3410 return t4_sge_intr_msix
;
3411 if (adap
->flags
& CXGB4_USING_MSI
)
3413 return t4_intr_intx
;
3416 static void sge_rx_timer_cb(struct timer_list
*t
)
3420 struct adapter
*adap
= from_timer(adap
, t
, sge
.rx_timer
);
3421 struct sge
*s
= &adap
->sge
;
3423 for (i
= 0; i
< BITS_TO_LONGS(s
->egr_sz
); i
++)
3424 for (m
= s
->starving_fl
[i
]; m
; m
&= m
- 1) {
3425 struct sge_eth_rxq
*rxq
;
3426 unsigned int id
= __ffs(m
) + i
* BITS_PER_LONG
;
3427 struct sge_fl
*fl
= s
->egr_map
[id
];
3429 clear_bit(id
, s
->starving_fl
);
3430 smp_mb__after_atomic();
3432 if (fl_starving(adap
, fl
)) {
3433 rxq
= container_of(fl
, struct sge_eth_rxq
, fl
);
3434 if (napi_reschedule(&rxq
->rspq
.napi
))
3437 set_bit(id
, s
->starving_fl
);
3440 /* The remainder of the SGE RX Timer Callback routine is dedicated to
3441 * global Master PF activities like checking for chip ingress stalls,
3444 if (!(adap
->flags
& CXGB4_MASTER_PF
))
3447 t4_idma_monitor(adap
, &s
->idma_monitor
, HZ
, RX_QCHECK_PERIOD
);
3450 mod_timer(&s
->rx_timer
, jiffies
+ RX_QCHECK_PERIOD
);
3453 static void sge_tx_timer_cb(struct timer_list
*t
)
3455 struct adapter
*adap
= from_timer(adap
, t
, sge
.tx_timer
);
3456 struct sge
*s
= &adap
->sge
;
3457 unsigned long m
, period
;
3458 unsigned int i
, budget
;
3460 for (i
= 0; i
< BITS_TO_LONGS(s
->egr_sz
); i
++)
3461 for (m
= s
->txq_maperr
[i
]; m
; m
&= m
- 1) {
3462 unsigned long id
= __ffs(m
) + i
* BITS_PER_LONG
;
3463 struct sge_uld_txq
*txq
= s
->egr_map
[id
];
3465 clear_bit(id
, s
->txq_maperr
);
3466 tasklet_schedule(&txq
->qresume_tsk
);
3469 if (!is_t4(adap
->params
.chip
)) {
3470 struct sge_eth_txq
*q
= &s
->ptptxq
;
3473 spin_lock(&adap
->ptp_lock
);
3474 avail
= reclaimable(&q
->q
);
3477 free_tx_desc(adap
, &q
->q
, avail
, false);
3478 q
->q
.in_use
-= avail
;
3480 spin_unlock(&adap
->ptp_lock
);
3483 budget
= MAX_TIMER_TX_RECLAIM
;
3484 i
= s
->ethtxq_rover
;
3486 budget
-= t4_sge_eth_txq_egress_update(adap
, &s
->ethtxq
[i
],
3491 if (++i
>= s
->ethqsets
)
3493 } while (i
!= s
->ethtxq_rover
);
3494 s
->ethtxq_rover
= i
;
3497 /* If we found too many reclaimable packets schedule a timer
3498 * in the near future to continue where we left off.
3502 /* We reclaimed all reclaimable TX Descriptors, so reschedule
3503 * at the normal period.
3505 period
= TX_QCHECK_PERIOD
;
3508 mod_timer(&s
->tx_timer
, jiffies
+ period
);
3512 * bar2_address - return the BAR2 address for an SGE Queue's Registers
3513 * @adapter: the adapter
3514 * @qid: the SGE Queue ID
3515 * @qtype: the SGE Queue Type (Egress or Ingress)
3516 * @pbar2_qid: BAR2 Queue ID or 0 for Queue ID inferred SGE Queues
3518 * Returns the BAR2 address for the SGE Queue Registers associated with
3519 * @qid. If BAR2 SGE Registers aren't available, returns NULL. Also
3520 * returns the BAR2 Queue ID to be used with writes to the BAR2 SGE
3521 * Queue Registers. If the BAR2 Queue ID is 0, then "Inferred Queue ID"
3522 * Registers are supported (e.g. the Write Combining Doorbell Buffer).
3524 static void __iomem
*bar2_address(struct adapter
*adapter
,
3526 enum t4_bar2_qtype qtype
,
3527 unsigned int *pbar2_qid
)
3532 ret
= t4_bar2_sge_qregs(adapter
, qid
, qtype
, 0,
3533 &bar2_qoffset
, pbar2_qid
);
3537 return adapter
->bar2
+ bar2_qoffset
;
3540 /* @intr_idx: MSI/MSI-X vector if >=0, -(absolute qid + 1) if < 0
3541 * @cong: < 0 -> no congestion feedback, >= 0 -> congestion channel map
3543 int t4_sge_alloc_rxq(struct adapter
*adap
, struct sge_rspq
*iq
, bool fwevtq
,
3544 struct net_device
*dev
, int intr_idx
,
3545 struct sge_fl
*fl
, rspq_handler_t hnd
,
3546 rspq_flush_handler_t flush_hnd
, int cong
)
3550 struct sge
*s
= &adap
->sge
;
3551 struct port_info
*pi
= netdev_priv(dev
);
3552 int relaxed
= !(adap
->flags
& CXGB4_ROOT_NO_RELAXED_ORDERING
);
3554 /* Size needs to be multiple of 16, including status entry. */
3555 iq
->size
= roundup(iq
->size
, 16);
3557 iq
->desc
= alloc_ring(adap
->pdev_dev
, iq
->size
, iq
->iqe_len
, 0,
3558 &iq
->phys_addr
, NULL
, 0,
3559 dev_to_node(adap
->pdev_dev
));
3563 memset(&c
, 0, sizeof(c
));
3564 c
.op_to_vfn
= htonl(FW_CMD_OP_V(FW_IQ_CMD
) | FW_CMD_REQUEST_F
|
3565 FW_CMD_WRITE_F
| FW_CMD_EXEC_F
|
3566 FW_IQ_CMD_PFN_V(adap
->pf
) | FW_IQ_CMD_VFN_V(0));
3567 c
.alloc_to_len16
= htonl(FW_IQ_CMD_ALLOC_F
| FW_IQ_CMD_IQSTART_F
|
3569 c
.type_to_iqandstindex
= htonl(FW_IQ_CMD_TYPE_V(FW_IQ_TYPE_FL_INT_CAP
) |
3570 FW_IQ_CMD_IQASYNCH_V(fwevtq
) | FW_IQ_CMD_VIID_V(pi
->viid
) |
3571 FW_IQ_CMD_IQANDST_V(intr_idx
< 0) |
3572 FW_IQ_CMD_IQANUD_V(UPDATEDELIVERY_INTERRUPT_X
) |
3573 FW_IQ_CMD_IQANDSTINDEX_V(intr_idx
>= 0 ? intr_idx
:
3575 c
.iqdroprss_to_iqesize
= htons(FW_IQ_CMD_IQPCIECH_V(pi
->tx_chan
) |
3576 FW_IQ_CMD_IQGTSMODE_F
|
3577 FW_IQ_CMD_IQINTCNTTHRESH_V(iq
->pktcnt_idx
) |
3578 FW_IQ_CMD_IQESIZE_V(ilog2(iq
->iqe_len
) - 4));
3579 c
.iqsize
= htons(iq
->size
);
3580 c
.iqaddr
= cpu_to_be64(iq
->phys_addr
);
3582 c
.iqns_to_fl0congen
= htonl(FW_IQ_CMD_IQFLINTCONGEN_F
|
3583 FW_IQ_CMD_IQTYPE_V(cong
? FW_IQ_IQTYPE_NIC
3584 : FW_IQ_IQTYPE_OFLD
));
3587 unsigned int chip_ver
=
3588 CHELSIO_CHIP_VERSION(adap
->params
.chip
);
3590 /* Allocate the ring for the hardware free list (with space
3591 * for its status page) along with the associated software
3592 * descriptor ring. The free list size needs to be a multiple
3593 * of the Egress Queue Unit and at least 2 Egress Units larger
3594 * than the SGE's Egress Congrestion Threshold
3595 * (fl_starve_thres - 1).
3597 if (fl
->size
< s
->fl_starve_thres
- 1 + 2 * 8)
3598 fl
->size
= s
->fl_starve_thres
- 1 + 2 * 8;
3599 fl
->size
= roundup(fl
->size
, 8);
3600 fl
->desc
= alloc_ring(adap
->pdev_dev
, fl
->size
, sizeof(__be64
),
3601 sizeof(struct rx_sw_desc
), &fl
->addr
,
3602 &fl
->sdesc
, s
->stat_len
,
3603 dev_to_node(adap
->pdev_dev
));
3607 flsz
= fl
->size
/ 8 + s
->stat_len
/ sizeof(struct tx_desc
);
3608 c
.iqns_to_fl0congen
|= htonl(FW_IQ_CMD_FL0PACKEN_F
|
3609 FW_IQ_CMD_FL0FETCHRO_V(relaxed
) |
3610 FW_IQ_CMD_FL0DATARO_V(relaxed
) |
3611 FW_IQ_CMD_FL0PADEN_F
);
3613 c
.iqns_to_fl0congen
|=
3614 htonl(FW_IQ_CMD_FL0CNGCHMAP_V(cong
) |
3615 FW_IQ_CMD_FL0CONGCIF_F
|
3616 FW_IQ_CMD_FL0CONGEN_F
);
3617 /* In T6, for egress queue type FL there is internal overhead
3618 * of 16B for header going into FLM module. Hence the maximum
3619 * allowed burst size is 448 bytes. For T4/T5, the hardware
3620 * doesn't coalesce fetch requests if more than 64 bytes of
3621 * Free List pointers are provided, so we use a 128-byte Fetch
3622 * Burst Minimum there (T6 implements coalescing so we can use
3623 * the smaller 64-byte value there).
3625 c
.fl0dcaen_to_fl0cidxfthresh
=
3626 htons(FW_IQ_CMD_FL0FBMIN_V(chip_ver
<= CHELSIO_T5
?
3627 FETCHBURSTMIN_128B_X
:
3628 FETCHBURSTMIN_64B_T6_X
) |
3629 FW_IQ_CMD_FL0FBMAX_V((chip_ver
<= CHELSIO_T5
) ?
3630 FETCHBURSTMAX_512B_X
:
3631 FETCHBURSTMAX_256B_X
));
3632 c
.fl0size
= htons(flsz
);
3633 c
.fl0addr
= cpu_to_be64(fl
->addr
);
3636 ret
= t4_wr_mbox(adap
, adap
->mbox
, &c
, sizeof(c
), &c
);
3640 netif_napi_add(dev
, &iq
->napi
, napi_rx_handler
, 64);
3641 iq
->cur_desc
= iq
->desc
;
3644 iq
->next_intr_params
= iq
->intr_params
;
3645 iq
->cntxt_id
= ntohs(c
.iqid
);
3646 iq
->abs_id
= ntohs(c
.physiqid
);
3647 iq
->bar2_addr
= bar2_address(adap
,
3649 T4_BAR2_QTYPE_INGRESS
,
3651 iq
->size
--; /* subtract status entry */
3654 iq
->flush_handler
= flush_hnd
;
3656 memset(&iq
->lro_mgr
, 0, sizeof(struct t4_lro_mgr
));
3657 skb_queue_head_init(&iq
->lro_mgr
.lroq
);
3659 /* set offset to -1 to distinguish ingress queues without FL */
3660 iq
->offset
= fl
? 0 : -1;
3662 adap
->sge
.ingr_map
[iq
->cntxt_id
- adap
->sge
.ingr_start
] = iq
;
3665 fl
->cntxt_id
= ntohs(c
.fl0id
);
3666 fl
->avail
= fl
->pend_cred
= 0;
3667 fl
->pidx
= fl
->cidx
= 0;
3668 fl
->alloc_failed
= fl
->large_alloc_failed
= fl
->starving
= 0;
3669 adap
->sge
.egr_map
[fl
->cntxt_id
- adap
->sge
.egr_start
] = fl
;
3671 /* Note, we must initialize the BAR2 Free List User Doorbell
3672 * information before refilling the Free List!
3674 fl
->bar2_addr
= bar2_address(adap
,
3676 T4_BAR2_QTYPE_EGRESS
,
3678 refill_fl(adap
, fl
, fl_cap(fl
), GFP_KERNEL
);
3681 /* For T5 and later we attempt to set up the Congestion Manager values
3682 * of the new RX Ethernet Queue. This should really be handled by
3683 * firmware because it's more complex than any host driver wants to
3684 * get involved with and it's different per chip and this is almost
3685 * certainly wrong. Firmware would be wrong as well, but it would be
3686 * a lot easier to fix in one place ... For now we do something very
3687 * simple (and hopefully less wrong).
3689 if (!is_t4(adap
->params
.chip
) && cong
>= 0) {
3690 u32 param
, val
, ch_map
= 0;
3692 u16 cng_ch_bits_log
= adap
->params
.arch
.cng_ch_bits_log
;
3694 param
= (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ
) |
3695 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DMAQ_CONM_CTXT
) |
3696 FW_PARAMS_PARAM_YZ_V(iq
->cntxt_id
));
3698 val
= CONMCTXT_CNGTPMODE_V(CONMCTXT_CNGTPMODE_QUEUE_X
);
3701 CONMCTXT_CNGTPMODE_V(CONMCTXT_CNGTPMODE_CHANNEL_X
);
3702 for (i
= 0; i
< 4; i
++) {
3703 if (cong
& (1 << i
))
3704 ch_map
|= 1 << (i
<< cng_ch_bits_log
);
3706 val
|= CONMCTXT_CNGCHMAP_V(ch_map
);
3708 ret
= t4_set_params(adap
, adap
->mbox
, adap
->pf
, 0, 1,
3711 dev_warn(adap
->pdev_dev
, "Failed to set Congestion"
3712 " Manager Context for Ingress Queue %d: %d\n",
3713 iq
->cntxt_id
, -ret
);
3722 dma_free_coherent(adap
->pdev_dev
, iq
->size
* iq
->iqe_len
,
3723 iq
->desc
, iq
->phys_addr
);
3726 if (fl
&& fl
->desc
) {
3729 dma_free_coherent(adap
->pdev_dev
, flsz
* sizeof(struct tx_desc
),
3730 fl
->desc
, fl
->addr
);
3736 static void init_txq(struct adapter
*adap
, struct sge_txq
*q
, unsigned int id
)
3739 q
->bar2_addr
= bar2_address(adap
,
3741 T4_BAR2_QTYPE_EGRESS
,
3744 q
->cidx
= q
->pidx
= 0;
3745 q
->stops
= q
->restarts
= 0;
3746 q
->stat
= (void *)&q
->desc
[q
->size
];
3747 spin_lock_init(&q
->db_lock
);
3748 adap
->sge
.egr_map
[id
- adap
->sge
.egr_start
] = q
;
3752 * t4_sge_alloc_eth_txq - allocate an Ethernet TX Queue
3753 * @adap: the adapter
3754 * @txq: the SGE Ethernet TX Queue to initialize
3755 * @dev: the Linux Network Device
3756 * @netdevq: the corresponding Linux TX Queue
3757 * @iqid: the Ingress Queue to which to deliver CIDX Update messages
3758 * @dbqt: whether this TX Queue will use the SGE Doorbell Queue Timers
3760 int t4_sge_alloc_eth_txq(struct adapter
*adap
, struct sge_eth_txq
*txq
,
3761 struct net_device
*dev
, struct netdev_queue
*netdevq
,
3762 unsigned int iqid
, u8 dbqt
)
3764 unsigned int chip_ver
= CHELSIO_CHIP_VERSION(adap
->params
.chip
);
3765 struct port_info
*pi
= netdev_priv(dev
);
3766 struct sge
*s
= &adap
->sge
;
3767 struct fw_eq_eth_cmd c
;
3770 /* Add status entries */
3771 nentries
= txq
->q
.size
+ s
->stat_len
/ sizeof(struct tx_desc
);
3773 txq
->q
.desc
= alloc_ring(adap
->pdev_dev
, txq
->q
.size
,
3774 sizeof(struct tx_desc
), sizeof(struct tx_sw_desc
),
3775 &txq
->q
.phys_addr
, &txq
->q
.sdesc
, s
->stat_len
,
3776 netdev_queue_numa_node_read(netdevq
));
3780 memset(&c
, 0, sizeof(c
));
3781 c
.op_to_vfn
= htonl(FW_CMD_OP_V(FW_EQ_ETH_CMD
) | FW_CMD_REQUEST_F
|
3782 FW_CMD_WRITE_F
| FW_CMD_EXEC_F
|
3783 FW_EQ_ETH_CMD_PFN_V(adap
->pf
) |
3784 FW_EQ_ETH_CMD_VFN_V(0));
3785 c
.alloc_to_len16
= htonl(FW_EQ_ETH_CMD_ALLOC_F
|
3786 FW_EQ_ETH_CMD_EQSTART_F
| FW_LEN16(c
));
3788 /* For TX Ethernet Queues using the SGE Doorbell Queue Timer
3789 * mechanism, we use Ingress Queue messages for Hardware Consumer
3790 * Index Updates on the TX Queue. Otherwise we have the Hardware
3791 * write the CIDX Updates into the Status Page at the end of the
3794 c
.autoequiqe_to_viid
= htonl(FW_EQ_ETH_CMD_AUTOEQUEQE_F
|
3795 FW_EQ_ETH_CMD_VIID_V(pi
->viid
));
3797 c
.fetchszm_to_iqid
=
3798 htonl(FW_EQ_ETH_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X
) |
3799 FW_EQ_ETH_CMD_PCIECHN_V(pi
->tx_chan
) |
3800 FW_EQ_ETH_CMD_FETCHRO_F
| FW_EQ_ETH_CMD_IQID_V(iqid
));
3802 /* Note that the CIDX Flush Threshold should match MAX_TX_RECLAIM. */
3804 htonl(FW_EQ_ETH_CMD_FBMIN_V(chip_ver
<= CHELSIO_T5
3805 ? FETCHBURSTMIN_64B_X
3806 : FETCHBURSTMIN_64B_T6_X
) |
3807 FW_EQ_ETH_CMD_FBMAX_V(FETCHBURSTMAX_512B_X
) |
3808 FW_EQ_ETH_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X
) |
3809 FW_EQ_ETH_CMD_EQSIZE_V(nentries
));
3811 c
.eqaddr
= cpu_to_be64(txq
->q
.phys_addr
);
3813 /* If we're using the SGE Doorbell Queue Timer mechanism, pass in the
3814 * currently configured Timer Index. THis can be changed later via an
3815 * ethtool -C tx-usecs {Timer Val} command. Note that the SGE
3816 * Doorbell Queue mode is currently automatically enabled in the
3817 * Firmware by setting either AUTOEQUEQE or AUTOEQUIQE ...
3821 cpu_to_be32(FW_EQ_ETH_CMD_TIMEREN_F
|
3822 FW_EQ_ETH_CMD_TIMERIX_V(txq
->dbqtimerix
));
3824 ret
= t4_wr_mbox(adap
, adap
->mbox
, &c
, sizeof(c
), &c
);
3826 kfree(txq
->q
.sdesc
);
3827 txq
->q
.sdesc
= NULL
;
3828 dma_free_coherent(adap
->pdev_dev
,
3829 nentries
* sizeof(struct tx_desc
),
3830 txq
->q
.desc
, txq
->q
.phys_addr
);
3835 txq
->q
.q_type
= CXGB4_TXQ_ETH
;
3836 init_txq(adap
, &txq
->q
, FW_EQ_ETH_CMD_EQID_G(ntohl(c
.eqid_pkd
)));
3838 txq
->tso
= txq
->tx_cso
= txq
->vlan_ins
= 0;
3839 txq
->mapping_err
= 0;
3845 int t4_sge_alloc_ctrl_txq(struct adapter
*adap
, struct sge_ctrl_txq
*txq
,
3846 struct net_device
*dev
, unsigned int iqid
,
3847 unsigned int cmplqid
)
3849 unsigned int chip_ver
= CHELSIO_CHIP_VERSION(adap
->params
.chip
);
3850 struct port_info
*pi
= netdev_priv(dev
);
3851 struct sge
*s
= &adap
->sge
;
3852 struct fw_eq_ctrl_cmd c
;
3855 /* Add status entries */
3856 nentries
= txq
->q
.size
+ s
->stat_len
/ sizeof(struct tx_desc
);
3858 txq
->q
.desc
= alloc_ring(adap
->pdev_dev
, nentries
,
3859 sizeof(struct tx_desc
), 0, &txq
->q
.phys_addr
,
3860 NULL
, 0, dev_to_node(adap
->pdev_dev
));
3864 c
.op_to_vfn
= htonl(FW_CMD_OP_V(FW_EQ_CTRL_CMD
) | FW_CMD_REQUEST_F
|
3865 FW_CMD_WRITE_F
| FW_CMD_EXEC_F
|
3866 FW_EQ_CTRL_CMD_PFN_V(adap
->pf
) |
3867 FW_EQ_CTRL_CMD_VFN_V(0));
3868 c
.alloc_to_len16
= htonl(FW_EQ_CTRL_CMD_ALLOC_F
|
3869 FW_EQ_CTRL_CMD_EQSTART_F
| FW_LEN16(c
));
3870 c
.cmpliqid_eqid
= htonl(FW_EQ_CTRL_CMD_CMPLIQID_V(cmplqid
));
3871 c
.physeqid_pkd
= htonl(0);
3872 c
.fetchszm_to_iqid
=
3873 htonl(FW_EQ_CTRL_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X
) |
3874 FW_EQ_CTRL_CMD_PCIECHN_V(pi
->tx_chan
) |
3875 FW_EQ_CTRL_CMD_FETCHRO_F
| FW_EQ_CTRL_CMD_IQID_V(iqid
));
3877 htonl(FW_EQ_CTRL_CMD_FBMIN_V(chip_ver
<= CHELSIO_T5
3878 ? FETCHBURSTMIN_64B_X
3879 : FETCHBURSTMIN_64B_T6_X
) |
3880 FW_EQ_CTRL_CMD_FBMAX_V(FETCHBURSTMAX_512B_X
) |
3881 FW_EQ_CTRL_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X
) |
3882 FW_EQ_CTRL_CMD_EQSIZE_V(nentries
));
3883 c
.eqaddr
= cpu_to_be64(txq
->q
.phys_addr
);
3885 ret
= t4_wr_mbox(adap
, adap
->mbox
, &c
, sizeof(c
), &c
);
3887 dma_free_coherent(adap
->pdev_dev
,
3888 nentries
* sizeof(struct tx_desc
),
3889 txq
->q
.desc
, txq
->q
.phys_addr
);
3894 txq
->q
.q_type
= CXGB4_TXQ_CTRL
;
3895 init_txq(adap
, &txq
->q
, FW_EQ_CTRL_CMD_EQID_G(ntohl(c
.cmpliqid_eqid
)));
3897 skb_queue_head_init(&txq
->sendq
);
3898 tasklet_init(&txq
->qresume_tsk
, restart_ctrlq
, (unsigned long)txq
);
3903 int t4_sge_mod_ctrl_txq(struct adapter
*adap
, unsigned int eqid
,
3904 unsigned int cmplqid
)
3908 param
= (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ
) |
3909 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DMAQ_EQ_CMPLIQID_CTRL
) |
3910 FW_PARAMS_PARAM_YZ_V(eqid
));
3912 return t4_set_params(adap
, adap
->mbox
, adap
->pf
, 0, 1, ¶m
, &val
);
3915 int t4_sge_alloc_uld_txq(struct adapter
*adap
, struct sge_uld_txq
*txq
,
3916 struct net_device
*dev
, unsigned int iqid
,
3917 unsigned int uld_type
)
3919 unsigned int chip_ver
= CHELSIO_CHIP_VERSION(adap
->params
.chip
);
3921 struct fw_eq_ofld_cmd c
;
3922 struct sge
*s
= &adap
->sge
;
3923 struct port_info
*pi
= netdev_priv(dev
);
3924 int cmd
= FW_EQ_OFLD_CMD
;
3926 /* Add status entries */
3927 nentries
= txq
->q
.size
+ s
->stat_len
/ sizeof(struct tx_desc
);
3929 txq
->q
.desc
= alloc_ring(adap
->pdev_dev
, txq
->q
.size
,
3930 sizeof(struct tx_desc
), sizeof(struct tx_sw_desc
),
3931 &txq
->q
.phys_addr
, &txq
->q
.sdesc
, s
->stat_len
,
3936 memset(&c
, 0, sizeof(c
));
3937 if (unlikely(uld_type
== CXGB4_TX_CRYPTO
))
3938 cmd
= FW_EQ_CTRL_CMD
;
3939 c
.op_to_vfn
= htonl(FW_CMD_OP_V(cmd
) | FW_CMD_REQUEST_F
|
3940 FW_CMD_WRITE_F
| FW_CMD_EXEC_F
|
3941 FW_EQ_OFLD_CMD_PFN_V(adap
->pf
) |
3942 FW_EQ_OFLD_CMD_VFN_V(0));
3943 c
.alloc_to_len16
= htonl(FW_EQ_OFLD_CMD_ALLOC_F
|
3944 FW_EQ_OFLD_CMD_EQSTART_F
| FW_LEN16(c
));
3945 c
.fetchszm_to_iqid
=
3946 htonl(FW_EQ_OFLD_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X
) |
3947 FW_EQ_OFLD_CMD_PCIECHN_V(pi
->tx_chan
) |
3948 FW_EQ_OFLD_CMD_FETCHRO_F
| FW_EQ_OFLD_CMD_IQID_V(iqid
));
3950 htonl(FW_EQ_OFLD_CMD_FBMIN_V(chip_ver
<= CHELSIO_T5
3951 ? FETCHBURSTMIN_64B_X
3952 : FETCHBURSTMIN_64B_T6_X
) |
3953 FW_EQ_OFLD_CMD_FBMAX_V(FETCHBURSTMAX_512B_X
) |
3954 FW_EQ_OFLD_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X
) |
3955 FW_EQ_OFLD_CMD_EQSIZE_V(nentries
));
3956 c
.eqaddr
= cpu_to_be64(txq
->q
.phys_addr
);
3958 ret
= t4_wr_mbox(adap
, adap
->mbox
, &c
, sizeof(c
), &c
);
3960 kfree(txq
->q
.sdesc
);
3961 txq
->q
.sdesc
= NULL
;
3962 dma_free_coherent(adap
->pdev_dev
,
3963 nentries
* sizeof(struct tx_desc
),
3964 txq
->q
.desc
, txq
->q
.phys_addr
);
3969 txq
->q
.q_type
= CXGB4_TXQ_ULD
;
3970 init_txq(adap
, &txq
->q
, FW_EQ_OFLD_CMD_EQID_G(ntohl(c
.eqid_pkd
)));
3972 skb_queue_head_init(&txq
->sendq
);
3973 tasklet_init(&txq
->qresume_tsk
, restart_ofldq
, (unsigned long)txq
);
3975 txq
->mapping_err
= 0;
3979 void free_txq(struct adapter
*adap
, struct sge_txq
*q
)
3981 struct sge
*s
= &adap
->sge
;
3983 dma_free_coherent(adap
->pdev_dev
,
3984 q
->size
* sizeof(struct tx_desc
) + s
->stat_len
,
3985 q
->desc
, q
->phys_addr
);
3991 void free_rspq_fl(struct adapter
*adap
, struct sge_rspq
*rq
,
3994 struct sge
*s
= &adap
->sge
;
3995 unsigned int fl_id
= fl
? fl
->cntxt_id
: 0xffff;
3997 adap
->sge
.ingr_map
[rq
->cntxt_id
- adap
->sge
.ingr_start
] = NULL
;
3998 t4_iq_free(adap
, adap
->mbox
, adap
->pf
, 0, FW_IQ_TYPE_FL_INT_CAP
,
3999 rq
->cntxt_id
, fl_id
, 0xffff);
4000 dma_free_coherent(adap
->pdev_dev
, (rq
->size
+ 1) * rq
->iqe_len
,
4001 rq
->desc
, rq
->phys_addr
);
4002 netif_napi_del(&rq
->napi
);
4004 rq
->cntxt_id
= rq
->abs_id
= 0;
4008 free_rx_bufs(adap
, fl
, fl
->avail
);
4009 dma_free_coherent(adap
->pdev_dev
, fl
->size
* 8 + s
->stat_len
,
4010 fl
->desc
, fl
->addr
);
4019 * t4_free_ofld_rxqs - free a block of consecutive Rx queues
4020 * @adap: the adapter
4021 * @n: number of queues
4022 * @q: pointer to first queue
4024 * Release the resources of a consecutive block of offload Rx queues.
4026 void t4_free_ofld_rxqs(struct adapter
*adap
, int n
, struct sge_ofld_rxq
*q
)
4028 for ( ; n
; n
--, q
++)
4030 free_rspq_fl(adap
, &q
->rspq
,
4031 q
->fl
.size
? &q
->fl
: NULL
);
4035 * t4_free_sge_resources - free SGE resources
4036 * @adap: the adapter
4038 * Frees resources used by the SGE queue sets.
4040 void t4_free_sge_resources(struct adapter
*adap
)
4043 struct sge_eth_rxq
*eq
;
4044 struct sge_eth_txq
*etq
;
4046 /* stop all Rx queues in order to start them draining */
4047 for (i
= 0; i
< adap
->sge
.ethqsets
; i
++) {
4048 eq
= &adap
->sge
.ethrxq
[i
];
4050 t4_iq_stop(adap
, adap
->mbox
, adap
->pf
, 0,
4051 FW_IQ_TYPE_FL_INT_CAP
,
4053 eq
->fl
.size
? eq
->fl
.cntxt_id
: 0xffff,
4057 /* clean up Ethernet Tx/Rx queues */
4058 for (i
= 0; i
< adap
->sge
.ethqsets
; i
++) {
4059 eq
= &adap
->sge
.ethrxq
[i
];
4061 free_rspq_fl(adap
, &eq
->rspq
,
4062 eq
->fl
.size
? &eq
->fl
: NULL
);
4064 etq
= &adap
->sge
.ethtxq
[i
];
4066 t4_eth_eq_free(adap
, adap
->mbox
, adap
->pf
, 0,
4068 __netif_tx_lock_bh(etq
->txq
);
4069 free_tx_desc(adap
, &etq
->q
, etq
->q
.in_use
, true);
4070 __netif_tx_unlock_bh(etq
->txq
);
4071 kfree(etq
->q
.sdesc
);
4072 free_txq(adap
, &etq
->q
);
4076 /* clean up control Tx queues */
4077 for (i
= 0; i
< ARRAY_SIZE(adap
->sge
.ctrlq
); i
++) {
4078 struct sge_ctrl_txq
*cq
= &adap
->sge
.ctrlq
[i
];
4081 tasklet_kill(&cq
->qresume_tsk
);
4082 t4_ctrl_eq_free(adap
, adap
->mbox
, adap
->pf
, 0,
4084 __skb_queue_purge(&cq
->sendq
);
4085 free_txq(adap
, &cq
->q
);
4089 if (adap
->sge
.fw_evtq
.desc
)
4090 free_rspq_fl(adap
, &adap
->sge
.fw_evtq
, NULL
);
4092 if (adap
->sge
.intrq
.desc
)
4093 free_rspq_fl(adap
, &adap
->sge
.intrq
, NULL
);
4095 if (!is_t4(adap
->params
.chip
)) {
4096 etq
= &adap
->sge
.ptptxq
;
4098 t4_eth_eq_free(adap
, adap
->mbox
, adap
->pf
, 0,
4100 spin_lock_bh(&adap
->ptp_lock
);
4101 free_tx_desc(adap
, &etq
->q
, etq
->q
.in_use
, true);
4102 spin_unlock_bh(&adap
->ptp_lock
);
4103 kfree(etq
->q
.sdesc
);
4104 free_txq(adap
, &etq
->q
);
4108 /* clear the reverse egress queue map */
4109 memset(adap
->sge
.egr_map
, 0,
4110 adap
->sge
.egr_sz
* sizeof(*adap
->sge
.egr_map
));
4113 void t4_sge_start(struct adapter
*adap
)
4115 adap
->sge
.ethtxq_rover
= 0;
4116 mod_timer(&adap
->sge
.rx_timer
, jiffies
+ RX_QCHECK_PERIOD
);
4117 mod_timer(&adap
->sge
.tx_timer
, jiffies
+ TX_QCHECK_PERIOD
);
4121 * t4_sge_stop - disable SGE operation
4122 * @adap: the adapter
4124 * Stop tasklets and timers associated with the DMA engine. Note that
4125 * this is effective only if measures have been taken to disable any HW
4126 * events that may restart them.
4128 void t4_sge_stop(struct adapter
*adap
)
4131 struct sge
*s
= &adap
->sge
;
4133 if (in_interrupt()) /* actions below require waiting */
4136 if (s
->rx_timer
.function
)
4137 del_timer_sync(&s
->rx_timer
);
4138 if (s
->tx_timer
.function
)
4139 del_timer_sync(&s
->tx_timer
);
4141 if (is_offload(adap
)) {
4142 struct sge_uld_txq_info
*txq_info
;
4144 txq_info
= adap
->sge
.uld_txq_info
[CXGB4_TX_OFLD
];
4146 struct sge_uld_txq
*txq
= txq_info
->uldtxq
;
4148 for_each_ofldtxq(&adap
->sge
, i
) {
4150 tasklet_kill(&txq
->qresume_tsk
);
4155 if (is_pci_uld(adap
)) {
4156 struct sge_uld_txq_info
*txq_info
;
4158 txq_info
= adap
->sge
.uld_txq_info
[CXGB4_TX_CRYPTO
];
4160 struct sge_uld_txq
*txq
= txq_info
->uldtxq
;
4162 for_each_ofldtxq(&adap
->sge
, i
) {
4164 tasklet_kill(&txq
->qresume_tsk
);
4169 for (i
= 0; i
< ARRAY_SIZE(s
->ctrlq
); i
++) {
4170 struct sge_ctrl_txq
*cq
= &s
->ctrlq
[i
];
4173 tasklet_kill(&cq
->qresume_tsk
);
4178 * t4_sge_init_soft - grab core SGE values needed by SGE code
4179 * @adap: the adapter
4181 * We need to grab the SGE operating parameters that we need to have
4182 * in order to do our job and make sure we can live with them.
4185 static int t4_sge_init_soft(struct adapter
*adap
)
4187 struct sge
*s
= &adap
->sge
;
4188 u32 fl_small_pg
, fl_large_pg
, fl_small_mtu
, fl_large_mtu
;
4189 u32 timer_value_0_and_1
, timer_value_2_and_3
, timer_value_4_and_5
;
4190 u32 ingress_rx_threshold
;
4193 * Verify that CPL messages are going to the Ingress Queue for
4194 * process_responses() and that only packet data is going to the
4197 if ((t4_read_reg(adap
, SGE_CONTROL_A
) & RXPKTCPLMODE_F
) !=
4198 RXPKTCPLMODE_V(RXPKTCPLMODE_SPLIT_X
)) {
4199 dev_err(adap
->pdev_dev
, "bad SGE CPL MODE\n");
4204 * Validate the Host Buffer Register Array indices that we want to
4207 * XXX Note that we should really read through the Host Buffer Size
4208 * XXX register array and find the indices of the Buffer Sizes which
4209 * XXX meet our needs!
4211 #define READ_FL_BUF(x) \
4212 t4_read_reg(adap, SGE_FL_BUFFER_SIZE0_A+(x)*sizeof(u32))
4214 fl_small_pg
= READ_FL_BUF(RX_SMALL_PG_BUF
);
4215 fl_large_pg
= READ_FL_BUF(RX_LARGE_PG_BUF
);
4216 fl_small_mtu
= READ_FL_BUF(RX_SMALL_MTU_BUF
);
4217 fl_large_mtu
= READ_FL_BUF(RX_LARGE_MTU_BUF
);
4219 /* We only bother using the Large Page logic if the Large Page Buffer
4220 * is larger than our Page Size Buffer.
4222 if (fl_large_pg
<= fl_small_pg
)
4227 /* The Page Size Buffer must be exactly equal to our Page Size and the
4228 * Large Page Size Buffer should be 0 (per above) or a power of 2.
4230 if (fl_small_pg
!= PAGE_SIZE
||
4231 (fl_large_pg
& (fl_large_pg
-1)) != 0) {
4232 dev_err(adap
->pdev_dev
, "bad SGE FL page buffer sizes [%d, %d]\n",
4233 fl_small_pg
, fl_large_pg
);
4237 s
->fl_pg_order
= ilog2(fl_large_pg
) - PAGE_SHIFT
;
4239 if (fl_small_mtu
< FL_MTU_SMALL_BUFSIZE(adap
) ||
4240 fl_large_mtu
< FL_MTU_LARGE_BUFSIZE(adap
)) {
4241 dev_err(adap
->pdev_dev
, "bad SGE FL MTU sizes [%d, %d]\n",
4242 fl_small_mtu
, fl_large_mtu
);
4247 * Retrieve our RX interrupt holdoff timer values and counter
4248 * threshold values from the SGE parameters.
4250 timer_value_0_and_1
= t4_read_reg(adap
, SGE_TIMER_VALUE_0_AND_1_A
);
4251 timer_value_2_and_3
= t4_read_reg(adap
, SGE_TIMER_VALUE_2_AND_3_A
);
4252 timer_value_4_and_5
= t4_read_reg(adap
, SGE_TIMER_VALUE_4_AND_5_A
);
4253 s
->timer_val
[0] = core_ticks_to_us(adap
,
4254 TIMERVALUE0_G(timer_value_0_and_1
));
4255 s
->timer_val
[1] = core_ticks_to_us(adap
,
4256 TIMERVALUE1_G(timer_value_0_and_1
));
4257 s
->timer_val
[2] = core_ticks_to_us(adap
,
4258 TIMERVALUE2_G(timer_value_2_and_3
));
4259 s
->timer_val
[3] = core_ticks_to_us(adap
,
4260 TIMERVALUE3_G(timer_value_2_and_3
));
4261 s
->timer_val
[4] = core_ticks_to_us(adap
,
4262 TIMERVALUE4_G(timer_value_4_and_5
));
4263 s
->timer_val
[5] = core_ticks_to_us(adap
,
4264 TIMERVALUE5_G(timer_value_4_and_5
));
4266 ingress_rx_threshold
= t4_read_reg(adap
, SGE_INGRESS_RX_THRESHOLD_A
);
4267 s
->counter_val
[0] = THRESHOLD_0_G(ingress_rx_threshold
);
4268 s
->counter_val
[1] = THRESHOLD_1_G(ingress_rx_threshold
);
4269 s
->counter_val
[2] = THRESHOLD_2_G(ingress_rx_threshold
);
4270 s
->counter_val
[3] = THRESHOLD_3_G(ingress_rx_threshold
);
4276 * t4_sge_init - initialize SGE
4277 * @adap: the adapter
4279 * Perform low-level SGE code initialization needed every time after a
4282 int t4_sge_init(struct adapter
*adap
)
4284 struct sge
*s
= &adap
->sge
;
4285 u32 sge_control
, sge_conm_ctrl
;
4286 int ret
, egress_threshold
;
4289 * Ingress Padding Boundary and Egress Status Page Size are set up by
4290 * t4_fixup_host_params().
4292 sge_control
= t4_read_reg(adap
, SGE_CONTROL_A
);
4293 s
->pktshift
= PKTSHIFT_G(sge_control
);
4294 s
->stat_len
= (sge_control
& EGRSTATUSPAGESIZE_F
) ? 128 : 64;
4296 s
->fl_align
= t4_fl_pkt_align(adap
);
4297 ret
= t4_sge_init_soft(adap
);
4302 * A FL with <= fl_starve_thres buffers is starving and a periodic
4303 * timer will attempt to refill it. This needs to be larger than the
4304 * SGE's Egress Congestion Threshold. If it isn't, then we can get
4305 * stuck waiting for new packets while the SGE is waiting for us to
4306 * give it more Free List entries. (Note that the SGE's Egress
4307 * Congestion Threshold is in units of 2 Free List pointers.) For T4,
4308 * there was only a single field to control this. For T5 there's the
4309 * original field which now only applies to Unpacked Mode Free List
4310 * buffers and a new field which only applies to Packed Mode Free List
4313 sge_conm_ctrl
= t4_read_reg(adap
, SGE_CONM_CTRL_A
);
4314 switch (CHELSIO_CHIP_VERSION(adap
->params
.chip
)) {
4316 egress_threshold
= EGRTHRESHOLD_G(sge_conm_ctrl
);
4319 egress_threshold
= EGRTHRESHOLDPACKING_G(sge_conm_ctrl
);
4322 egress_threshold
= T6_EGRTHRESHOLDPACKING_G(sge_conm_ctrl
);
4325 dev_err(adap
->pdev_dev
, "Unsupported Chip version %d\n",
4326 CHELSIO_CHIP_VERSION(adap
->params
.chip
));
4329 s
->fl_starve_thres
= 2*egress_threshold
+ 1;
4331 t4_idma_monitor_init(adap
, &s
->idma_monitor
);
4333 /* Set up timers used for recuring callbacks to process RX and TX
4334 * administrative tasks.
4336 timer_setup(&s
->rx_timer
, sge_rx_timer_cb
, 0);
4337 timer_setup(&s
->tx_timer
, sge_tx_timer_cb
, 0);
4339 spin_lock_init(&s
->intrq_lock
);