1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright (c) 2018 Intel Corporation */
7 #include <linux/kobject.h>
9 #include <linux/netdevice.h>
10 #include <linux/vmalloc.h>
11 #include <linux/ethtool.h>
12 #include <linux/sctp.h>
16 /* forward declaration */
17 void igc_set_ethtool_ops(struct net_device
*);
22 void igc_up(struct igc_adapter
*adapter
);
23 void igc_down(struct igc_adapter
*adapter
);
24 int igc_setup_tx_resources(struct igc_ring
*ring
);
25 int igc_setup_rx_resources(struct igc_ring
*ring
);
26 void igc_free_tx_resources(struct igc_ring
*ring
);
27 void igc_free_rx_resources(struct igc_ring
*ring
);
28 unsigned int igc_get_max_rss_queues(struct igc_adapter
*adapter
);
29 void igc_set_flag_queue_pairs(struct igc_adapter
*adapter
,
30 const u32 max_rss_queues
);
31 int igc_reinit_queues(struct igc_adapter
*adapter
);
32 void igc_write_rss_indir_tbl(struct igc_adapter
*adapter
);
33 bool igc_has_link(struct igc_adapter
*adapter
);
34 void igc_reset(struct igc_adapter
*adapter
);
35 int igc_set_spd_dplx(struct igc_adapter
*adapter
, u32 spd
, u8 dplx
);
36 int igc_add_mac_steering_filter(struct igc_adapter
*adapter
,
37 const u8
*addr
, u8 queue
, u8 flags
);
38 int igc_del_mac_steering_filter(struct igc_adapter
*adapter
,
39 const u8
*addr
, u8 queue
, u8 flags
);
40 void igc_update_stats(struct igc_adapter
*adapter
);
42 extern char igc_driver_name
[];
43 extern char igc_driver_version
[];
45 #define IGC_REGS_LEN 740
46 #define IGC_RETA_SIZE 128
48 /* Interrupt defines */
49 #define IGC_START_ITR 648 /* ~6000 ints/sec */
50 #define IGC_FLAG_HAS_MSI BIT(0)
51 #define IGC_FLAG_QUEUE_PAIRS BIT(3)
52 #define IGC_FLAG_DMAC BIT(4)
53 #define IGC_FLAG_NEED_LINK_UPDATE BIT(9)
54 #define IGC_FLAG_MEDIA_RESET BIT(10)
55 #define IGC_FLAG_MAS_ENABLE BIT(12)
56 #define IGC_FLAG_HAS_MSIX BIT(13)
57 #define IGC_FLAG_VLAN_PROMISC BIT(15)
58 #define IGC_FLAG_RX_LEGACY BIT(16)
60 #define IGC_FLAG_RSS_FIELD_IPV4_UDP BIT(6)
61 #define IGC_FLAG_RSS_FIELD_IPV6_UDP BIT(7)
63 #define IGC_MRQC_ENABLE_RSS_MQ 0x00000002
64 #define IGC_MRQC_RSS_FIELD_IPV4_UDP 0x00400000
65 #define IGC_MRQC_RSS_FIELD_IPV6_UDP 0x00800000
67 #define IGC_START_ITR 648 /* ~6000 ints/sec */
68 #define IGC_4K_ITR 980
69 #define IGC_20K_ITR 196
70 #define IGC_70K_ITR 56
72 #define IGC_DEFAULT_ITR 3 /* dynamic */
73 #define IGC_MAX_ITR_USECS 10000
74 #define IGC_MIN_ITR_USECS 10
75 #define NON_Q_VECTORS 1
76 #define MAX_MSIX_ENTRIES 10
78 /* TX/RX descriptor defines */
79 #define IGC_DEFAULT_TXD 256
80 #define IGC_DEFAULT_TX_WORK 128
81 #define IGC_MIN_TXD 80
82 #define IGC_MAX_TXD 4096
84 #define IGC_DEFAULT_RXD 256
85 #define IGC_MIN_RXD 80
86 #define IGC_MAX_RXD 4096
88 /* Transmit and receive queues */
89 #define IGC_MAX_RX_QUEUES 4
90 #define IGC_MAX_TX_QUEUES 4
92 #define MAX_Q_VECTORS 8
93 #define MAX_STD_JUMBO_FRAME_SIZE 9216
95 /* Supported Rx Buffer Sizes */
96 #define IGC_RXBUFFER_256 256
97 #define IGC_RXBUFFER_2048 2048
98 #define IGC_RXBUFFER_3072 3072
100 #define AUTO_ALL_MODES 0
101 #define IGC_RX_HDR_LEN IGC_RXBUFFER_256
103 /* RX and TX descriptor control thresholds.
104 * PTHRESH - MAC will consider prefetch if it has fewer than this number of
105 * descriptors available in its onboard memory.
106 * Setting this to 0 disables RX descriptor prefetch.
107 * HTHRESH - MAC will only prefetch if there are at least this many descriptors
108 * available in host memory.
109 * If PTHRESH is 0, this should also be 0.
110 * WTHRESH - RX descriptor writeback threshold - MAC will delay writing back
111 * descriptors until either it has this many to write back, or the
114 #define IGC_RX_PTHRESH 8
115 #define IGC_RX_HTHRESH 8
116 #define IGC_TX_PTHRESH 8
117 #define IGC_TX_HTHRESH 1
118 #define IGC_RX_WTHRESH 4
119 #define IGC_TX_WTHRESH 16
121 #define IGC_RX_DMA_ATTR \
122 (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
124 #define IGC_TS_HDR_LEN 16
126 #define IGC_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN)
128 #if (PAGE_SIZE < 8192)
129 #define IGC_MAX_FRAME_BUILD_SKB \
130 (SKB_WITH_OVERHEAD(IGC_RXBUFFER_2048) - IGC_SKB_PAD - IGC_TS_HDR_LEN)
132 #define IGC_MAX_FRAME_BUILD_SKB (IGC_RXBUFFER_2048 - IGC_TS_HDR_LEN)
135 /* How many Rx Buffers do we bundle into one write to the hardware ? */
136 #define IGC_RX_BUFFER_WRITE 16 /* Must be power of 2 */
139 #define IGC_TX_FLAGS_VLAN_MASK 0xffff0000
141 /* igc_test_staterr - tests bits within Rx descriptor status and error fields */
142 static inline __le32
igc_test_staterr(union igc_adv_rx_desc
*rx_desc
,
143 const u32 stat_err_bits
)
145 return rx_desc
->wb
.upper
.status_error
& cpu_to_le32(stat_err_bits
);
152 __IGC_PTP_TX_IN_PROGRESS
,
157 IGC_TX_FLAGS_VLAN
= 0x01,
158 IGC_TX_FLAGS_TSO
= 0x02,
159 IGC_TX_FLAGS_TSTAMP
= 0x04,
162 IGC_TX_FLAGS_IPV4
= 0x10,
163 IGC_TX_FLAGS_CSUM
= 0x20,
170 /* The largest size we can write to the descriptor is 65535. In order to
171 * maintain a power of two alignment we have to limit ourselves to 32K.
173 #define IGC_MAX_TXD_PWR 15
174 #define IGC_MAX_DATA_PER_TXD BIT(IGC_MAX_TXD_PWR)
176 /* Tx Descriptors needed, worst case */
177 #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IGC_MAX_DATA_PER_TXD)
178 #define DESC_NEEDED (MAX_SKB_FRAGS + 4)
180 /* wrapper around a pointer to a socket buffer,
181 * so a DMA handle can be stored along with the buffer
183 struct igc_tx_buffer
{
184 union igc_adv_tx_desc
*next_to_watch
;
185 unsigned long time_stamp
;
187 unsigned int bytecount
;
191 DEFINE_DMA_UNMAP_ADDR(dma
);
192 DEFINE_DMA_UNMAP_LEN(len
);
196 struct igc_rx_buffer
{
199 #if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536)
207 struct igc_tx_queue_stats
{
214 struct igc_rx_queue_stats
{
222 struct igc_rx_packet_stats
{
223 u64 ipv4_packets
; /* IPv4 headers processed */
224 u64 ipv4e_packets
; /* IPv4E headers with extensions processed */
225 u64 ipv6_packets
; /* IPv6 headers processed */
226 u64 ipv6e_packets
; /* IPv6E headers with extensions processed */
227 u64 tcp_packets
; /* TCP headers processed */
228 u64 udp_packets
; /* UDP headers processed */
229 u64 sctp_packets
; /* SCTP headers processed */
230 u64 nfs_packets
; /* NFS headers processe */
234 struct igc_ring_container
{
235 struct igc_ring
*ring
; /* pointer to linked list of rings */
236 unsigned int total_bytes
; /* total bytes processed this int */
237 unsigned int total_packets
; /* total packets processed this int */
238 u16 work_limit
; /* total work allowed per interrupt */
239 u8 count
; /* total number of rings in vector */
240 u8 itr
; /* current ITR setting for ring */
244 struct igc_q_vector
*q_vector
; /* backlink to q_vector */
245 struct net_device
*netdev
; /* back pointer to net_device */
246 struct device
*dev
; /* device for dma mapping */
247 union { /* array of buffer info structs */
248 struct igc_tx_buffer
*tx_buffer_info
;
249 struct igc_rx_buffer
*rx_buffer_info
;
251 void *desc
; /* descriptor ring memory */
252 unsigned long flags
; /* ring specific flags */
253 void __iomem
*tail
; /* pointer to ring tail register */
254 dma_addr_t dma
; /* phys address of the ring */
255 unsigned int size
; /* length of desc. ring in bytes */
257 u16 count
; /* number of desc. in the ring */
258 u8 queue_index
; /* logical index of the ring*/
259 u8 reg_idx
; /* physical index of the ring */
260 bool launchtime_enable
; /* true if LaunchTime is enabled */
262 /* everything past this point are written often */
270 struct igc_tx_queue_stats tx_stats
;
271 struct u64_stats_sync tx_syncp
;
272 struct u64_stats_sync tx_syncp2
;
276 struct igc_rx_queue_stats rx_stats
;
277 struct igc_rx_packet_stats pkt_stats
;
278 struct u64_stats_sync rx_syncp
;
282 } ____cacheline_internodealigned_in_smp
;
284 struct igc_q_vector
{
285 struct igc_adapter
*adapter
; /* backlink */
286 void __iomem
*itr_register
;
287 u32 eims_value
; /* EIMS mask value */
292 struct igc_ring_container rx
, tx
;
294 struct napi_struct napi
;
296 struct rcu_head rcu
; /* to avoid race with update stats on free */
297 char name
[IFNAMSIZ
+ 9];
298 struct net_device poll_dev
;
300 /* for dynamic allocation of rings associated with this q_vector */
301 struct igc_ring ring
[0] ____cacheline_internodealigned_in_smp
;
304 #define MAX_ETYPE_FILTER (4 - 1)
306 enum igc_filter_match_flags
{
307 IGC_FILTER_FLAG_ETHER_TYPE
= 0x1,
308 IGC_FILTER_FLAG_VLAN_TCI
= 0x2,
309 IGC_FILTER_FLAG_SRC_MAC_ADDR
= 0x4,
310 IGC_FILTER_FLAG_DST_MAC_ADDR
= 0x8,
313 /* RX network flow classification data structure */
314 struct igc_nfc_input
{
315 /* Byte layout in order, all values with MSB first:
316 * match_flags - 1 byte
323 u8 src_addr
[ETH_ALEN
];
324 u8 dst_addr
[ETH_ALEN
];
327 struct igc_nfc_filter
{
328 struct hlist_node nfc_node
;
329 struct igc_nfc_input filter
;
330 unsigned long cookie
;
336 struct igc_mac_addr
{
339 u8 state
; /* bitmask */
342 #define IGC_MAC_STATE_DEFAULT 0x1
343 #define IGC_MAC_STATE_IN_USE 0x2
344 #define IGC_MAC_STATE_SRC_ADDR 0x4
345 #define IGC_MAC_STATE_QUEUE_STEERING 0x8
347 #define IGC_MAX_RXNFC_FILTERS 16
349 /* Board specific private data structure */
351 struct net_device
*netdev
;
355 unsigned int num_q_vectors
;
357 struct msix_entry
*msix_entries
;
361 u32 tx_timeout_count
;
363 struct igc_ring
*tx_ring
[IGC_MAX_TX_QUEUES
];
367 struct igc_ring
*rx_ring
[IGC_MAX_RX_QUEUES
];
369 struct timer_list watchdog_timer
;
370 struct timer_list dma_err_timer
;
371 struct timer_list phy_info_timer
;
379 /* Interrupt Throttle Rate */
383 struct work_struct reset_task
;
384 struct work_struct watchdog_task
;
385 struct work_struct dma_err_task
;
388 u8 tx_timeout_factor
;
394 /* OS defined structs */
395 struct pci_dev
*pdev
;
396 /* lock for statistics */
397 spinlock_t stats64_lock
;
398 struct rtnl_link_stats64 stats64
;
400 /* structs defined in igc_hw.h */
402 struct igc_hw_stats stats
;
404 struct igc_q_vector
*q_vector
[MAX_Q_VECTORS
];
405 u32 eims_enable_mask
;
411 u32 tx_hwtstamp_timeouts
;
412 u32 tx_hwtstamp_skipped
;
413 u32 rx_hwtstamp_cleared
;
416 u32 rss_indir_tbl_init
;
418 /* RX network flow classification support */
419 struct hlist_head nfc_filter_list
;
420 struct hlist_head cls_flower_list
;
421 unsigned int nfc_filter_count
;
423 /* lock for RX network flow classification filter */
425 bool etype_bitmap
[MAX_ETYPE_FILTER
];
427 struct igc_mac_addr
*mac_table
;
429 u8 rss_indir_tbl
[IGC_RETA_SIZE
];
431 unsigned long link_check_timeout
;
435 /* igc_desc_unused - calculate if we have unused descriptors */
436 static inline u16
igc_desc_unused(const struct igc_ring
*ring
)
438 u16 ntc
= ring
->next_to_clean
;
439 u16 ntu
= ring
->next_to_use
;
441 return ((ntc
> ntu
) ? 0 : ring
->count
) + ntc
- ntu
- 1;
444 static inline s32
igc_get_phy_info(struct igc_hw
*hw
)
446 if (hw
->phy
.ops
.get_phy_info
)
447 return hw
->phy
.ops
.get_phy_info(hw
);
452 static inline s32
igc_reset_phy(struct igc_hw
*hw
)
454 if (hw
->phy
.ops
.reset
)
455 return hw
->phy
.ops
.reset(hw
);
460 static inline struct netdev_queue
*txring_txq(const struct igc_ring
*tx_ring
)
462 return netdev_get_tx_queue(tx_ring
->netdev
, tx_ring
->queue_index
);
465 enum igc_ring_flags_t
{
466 IGC_RING_FLAG_RX_3K_BUFFER
,
467 IGC_RING_FLAG_RX_BUILD_SKB_ENABLED
,
468 IGC_RING_FLAG_RX_SCTP_CSUM
,
469 IGC_RING_FLAG_RX_LB_VLAN_BSWAP
,
470 IGC_RING_FLAG_TX_CTX_IDX
,
471 IGC_RING_FLAG_TX_DETECT_HANG
474 #define ring_uses_large_buffer(ring) \
475 test_bit(IGC_RING_FLAG_RX_3K_BUFFER, &(ring)->flags)
477 #define ring_uses_build_skb(ring) \
478 test_bit(IGC_RING_FLAG_RX_BUILD_SKB_ENABLED, &(ring)->flags)
480 static inline unsigned int igc_rx_bufsz(struct igc_ring
*ring
)
482 #if (PAGE_SIZE < 8192)
483 if (ring_uses_large_buffer(ring
))
484 return IGC_RXBUFFER_3072
;
486 if (ring_uses_build_skb(ring
))
487 return IGC_MAX_FRAME_BUILD_SKB
+ IGC_TS_HDR_LEN
;
489 return IGC_RXBUFFER_2048
;
492 static inline unsigned int igc_rx_pg_order(struct igc_ring
*ring
)
494 #if (PAGE_SIZE < 8192)
495 if (ring_uses_large_buffer(ring
))
501 static inline s32
igc_read_phy_reg(struct igc_hw
*hw
, u32 offset
, u16
*data
)
503 if (hw
->phy
.ops
.read_reg
)
504 return hw
->phy
.ops
.read_reg(hw
, offset
, data
);
509 /* forward declaration */
510 void igc_reinit_locked(struct igc_adapter
*);
511 int igc_add_filter(struct igc_adapter
*adapter
,
512 struct igc_nfc_filter
*input
);
513 int igc_erase_filter(struct igc_adapter
*adapter
,
514 struct igc_nfc_filter
*input
);
516 #define igc_rx_pg_size(_ring) (PAGE_SIZE << igc_rx_pg_order(_ring))
518 #define IGC_TXD_DCMD (IGC_ADVTXD_DCMD_EOP | IGC_ADVTXD_DCMD_RS)
520 #define IGC_RX_DESC(R, i) \
521 (&(((union igc_adv_rx_desc *)((R)->desc))[i]))
522 #define IGC_TX_DESC(R, i) \
523 (&(((union igc_adv_tx_desc *)((R)->desc))[i]))
524 #define IGC_TX_CTXTDESC(R, i) \
525 (&(((struct igc_adv_tx_context_desc *)((R)->desc))[i]))