1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2018 Intel Corporation */
4 #include <linux/module.h>
5 #include <linux/types.h>
6 #include <linux/if_vlan.h>
17 #define DRV_VERSION "0.0.1-k"
18 #define DRV_SUMMARY "Intel(R) 2.5G Ethernet Linux Driver"
20 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
22 static int debug
= -1;
24 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
25 MODULE_DESCRIPTION(DRV_SUMMARY
);
26 MODULE_LICENSE("GPL v2");
27 MODULE_VERSION(DRV_VERSION
);
28 module_param(debug
, int, 0);
29 MODULE_PARM_DESC(debug
, "Debug level (0=none,...,16=all)");
31 char igc_driver_name
[] = "igc";
32 char igc_driver_version
[] = DRV_VERSION
;
33 static const char igc_driver_string
[] = DRV_SUMMARY
;
34 static const char igc_copyright
[] =
35 "Copyright(c) 2018 Intel Corporation.";
37 static const struct igc_info
*igc_info_tbl
[] = {
38 [board_base
] = &igc_base_info
,
41 static const struct pci_device_id igc_pci_tbl
[] = {
42 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I225_LM
), board_base
},
43 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I225_V
), board_base
},
44 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I225_I
), board_base
},
45 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I220_V
), board_base
},
46 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I225_K
), board_base
},
47 /* required last entry */
51 MODULE_DEVICE_TABLE(pci
, igc_pci_tbl
);
53 /* forward declaration */
54 static void igc_clean_tx_ring(struct igc_ring
*tx_ring
);
55 static int igc_sw_init(struct igc_adapter
*);
56 static void igc_configure(struct igc_adapter
*adapter
);
57 static void igc_power_down_link(struct igc_adapter
*adapter
);
58 static void igc_set_default_mac_filter(struct igc_adapter
*adapter
);
59 static void igc_set_rx_mode(struct net_device
*netdev
);
60 static void igc_write_itr(struct igc_q_vector
*q_vector
);
61 static void igc_assign_vector(struct igc_q_vector
*q_vector
, int msix_vector
);
62 static void igc_free_q_vector(struct igc_adapter
*adapter
, int v_idx
);
63 static void igc_set_interrupt_capability(struct igc_adapter
*adapter
,
65 static void igc_free_q_vectors(struct igc_adapter
*adapter
);
66 static void igc_irq_disable(struct igc_adapter
*adapter
);
67 static void igc_irq_enable(struct igc_adapter
*adapter
);
68 static void igc_configure_msix(struct igc_adapter
*adapter
);
69 static bool igc_alloc_mapped_page(struct igc_ring
*rx_ring
,
70 struct igc_rx_buffer
*bi
);
79 void igc_reset(struct igc_adapter
*adapter
)
81 struct pci_dev
*pdev
= adapter
->pdev
;
82 struct igc_hw
*hw
= &adapter
->hw
;
83 struct igc_fc_info
*fc
= &hw
->fc
;
86 /* Repartition PBA for greater than 9k MTU if required */
89 /* flow control settings
90 * The high water mark must be low enough to fit one full frame
91 * after transmitting the pause frame. As such we must have enough
92 * space to allow for us to complete our current transmit and then
93 * receive the frame that is in progress from the link partner.
95 * - the full Rx FIFO size minus one full Tx plus one full Rx frame
97 hwm
= (pba
<< 10) - (adapter
->max_frame_size
+ MAX_JUMBO_FRAME_SIZE
);
99 fc
->high_water
= hwm
& 0xFFFFFFF0; /* 16-byte granularity */
100 fc
->low_water
= fc
->high_water
- 16;
101 fc
->pause_time
= 0xFFFF;
103 fc
->current_mode
= fc
->requested_mode
;
105 hw
->mac
.ops
.reset_hw(hw
);
107 if (hw
->mac
.ops
.init_hw(hw
))
108 dev_err(&pdev
->dev
, "Hardware Error\n");
110 if (!netif_running(adapter
->netdev
))
111 igc_power_down_link(adapter
);
113 igc_get_phy_info(hw
);
117 * igc_power_up_link - Power up the phy/serdes link
118 * @adapter: address of board private structure
120 static void igc_power_up_link(struct igc_adapter
*adapter
)
122 igc_reset_phy(&adapter
->hw
);
124 if (adapter
->hw
.phy
.media_type
== igc_media_type_copper
)
125 igc_power_up_phy_copper(&adapter
->hw
);
127 igc_setup_link(&adapter
->hw
);
131 * igc_power_down_link - Power down the phy/serdes link
132 * @adapter: address of board private structure
134 static void igc_power_down_link(struct igc_adapter
*adapter
)
136 if (adapter
->hw
.phy
.media_type
== igc_media_type_copper
)
137 igc_power_down_phy_copper_base(&adapter
->hw
);
141 * igc_release_hw_control - release control of the h/w to f/w
142 * @adapter: address of board private structure
144 * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
145 * For ASF and Pass Through versions of f/w this means that the
146 * driver is no longer loaded.
148 static void igc_release_hw_control(struct igc_adapter
*adapter
)
150 struct igc_hw
*hw
= &adapter
->hw
;
153 /* Let firmware take over control of h/w */
154 ctrl_ext
= rd32(IGC_CTRL_EXT
);
156 ctrl_ext
& ~IGC_CTRL_EXT_DRV_LOAD
);
160 * igc_get_hw_control - get control of the h/w from f/w
161 * @adapter: address of board private structure
163 * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
164 * For ASF and Pass Through versions of f/w this means that
165 * the driver is loaded.
167 static void igc_get_hw_control(struct igc_adapter
*adapter
)
169 struct igc_hw
*hw
= &adapter
->hw
;
172 /* Let firmware know the driver has taken over */
173 ctrl_ext
= rd32(IGC_CTRL_EXT
);
175 ctrl_ext
| IGC_CTRL_EXT_DRV_LOAD
);
179 * igc_free_tx_resources - Free Tx Resources per Queue
180 * @tx_ring: Tx descriptor ring for a specific queue
182 * Free all transmit software resources
184 void igc_free_tx_resources(struct igc_ring
*tx_ring
)
186 igc_clean_tx_ring(tx_ring
);
188 vfree(tx_ring
->tx_buffer_info
);
189 tx_ring
->tx_buffer_info
= NULL
;
191 /* if not set, then don't free */
195 dma_free_coherent(tx_ring
->dev
, tx_ring
->size
,
196 tx_ring
->desc
, tx_ring
->dma
);
198 tx_ring
->desc
= NULL
;
202 * igc_free_all_tx_resources - Free Tx Resources for All Queues
203 * @adapter: board private structure
205 * Free all transmit software resources
207 static void igc_free_all_tx_resources(struct igc_adapter
*adapter
)
211 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
212 igc_free_tx_resources(adapter
->tx_ring
[i
]);
216 * igc_clean_tx_ring - Free Tx Buffers
217 * @tx_ring: ring to be cleaned
219 static void igc_clean_tx_ring(struct igc_ring
*tx_ring
)
221 u16 i
= tx_ring
->next_to_clean
;
222 struct igc_tx_buffer
*tx_buffer
= &tx_ring
->tx_buffer_info
[i
];
224 while (i
!= tx_ring
->next_to_use
) {
225 union igc_adv_tx_desc
*eop_desc
, *tx_desc
;
227 /* Free all the Tx ring sk_buffs */
228 dev_kfree_skb_any(tx_buffer
->skb
);
230 /* unmap skb header data */
231 dma_unmap_single(tx_ring
->dev
,
232 dma_unmap_addr(tx_buffer
, dma
),
233 dma_unmap_len(tx_buffer
, len
),
236 /* check for eop_desc to determine the end of the packet */
237 eop_desc
= tx_buffer
->next_to_watch
;
238 tx_desc
= IGC_TX_DESC(tx_ring
, i
);
240 /* unmap remaining buffers */
241 while (tx_desc
!= eop_desc
) {
245 if (unlikely(i
== tx_ring
->count
)) {
247 tx_buffer
= tx_ring
->tx_buffer_info
;
248 tx_desc
= IGC_TX_DESC(tx_ring
, 0);
251 /* unmap any remaining paged data */
252 if (dma_unmap_len(tx_buffer
, len
))
253 dma_unmap_page(tx_ring
->dev
,
254 dma_unmap_addr(tx_buffer
, dma
),
255 dma_unmap_len(tx_buffer
, len
),
259 /* move us one more past the eop_desc for start of next pkt */
262 if (unlikely(i
== tx_ring
->count
)) {
264 tx_buffer
= tx_ring
->tx_buffer_info
;
268 /* reset BQL for queue */
269 netdev_tx_reset_queue(txring_txq(tx_ring
));
271 /* reset next_to_use and next_to_clean */
272 tx_ring
->next_to_use
= 0;
273 tx_ring
->next_to_clean
= 0;
277 * igc_clean_all_tx_rings - Free Tx Buffers for all queues
278 * @adapter: board private structure
280 static void igc_clean_all_tx_rings(struct igc_adapter
*adapter
)
284 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
285 if (adapter
->tx_ring
[i
])
286 igc_clean_tx_ring(adapter
->tx_ring
[i
]);
290 * igc_setup_tx_resources - allocate Tx resources (Descriptors)
291 * @tx_ring: tx descriptor ring (for a specific queue) to setup
293 * Return 0 on success, negative on failure
295 int igc_setup_tx_resources(struct igc_ring
*tx_ring
)
297 struct device
*dev
= tx_ring
->dev
;
300 size
= sizeof(struct igc_tx_buffer
) * tx_ring
->count
;
301 tx_ring
->tx_buffer_info
= vzalloc(size
);
302 if (!tx_ring
->tx_buffer_info
)
305 /* round up to nearest 4K */
306 tx_ring
->size
= tx_ring
->count
* sizeof(union igc_adv_tx_desc
);
307 tx_ring
->size
= ALIGN(tx_ring
->size
, 4096);
309 tx_ring
->desc
= dma_alloc_coherent(dev
, tx_ring
->size
,
310 &tx_ring
->dma
, GFP_KERNEL
);
315 tx_ring
->next_to_use
= 0;
316 tx_ring
->next_to_clean
= 0;
321 vfree(tx_ring
->tx_buffer_info
);
323 "Unable to allocate memory for the transmit descriptor ring\n");
328 * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues
329 * @adapter: board private structure
331 * Return 0 on success, negative on failure
333 static int igc_setup_all_tx_resources(struct igc_adapter
*adapter
)
335 struct pci_dev
*pdev
= adapter
->pdev
;
338 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
339 err
= igc_setup_tx_resources(adapter
->tx_ring
[i
]);
342 "Allocation for Tx Queue %u failed\n", i
);
343 for (i
--; i
>= 0; i
--)
344 igc_free_tx_resources(adapter
->tx_ring
[i
]);
353 * igc_clean_rx_ring - Free Rx Buffers per Queue
354 * @rx_ring: ring to free buffers from
356 static void igc_clean_rx_ring(struct igc_ring
*rx_ring
)
358 u16 i
= rx_ring
->next_to_clean
;
360 dev_kfree_skb(rx_ring
->skb
);
363 /* Free all the Rx ring sk_buffs */
364 while (i
!= rx_ring
->next_to_alloc
) {
365 struct igc_rx_buffer
*buffer_info
= &rx_ring
->rx_buffer_info
[i
];
367 /* Invalidate cache lines that may have been written to by
368 * device so that we avoid corrupting memory.
370 dma_sync_single_range_for_cpu(rx_ring
->dev
,
372 buffer_info
->page_offset
,
373 igc_rx_bufsz(rx_ring
),
376 /* free resources associated with mapping */
377 dma_unmap_page_attrs(rx_ring
->dev
,
379 igc_rx_pg_size(rx_ring
),
382 __page_frag_cache_drain(buffer_info
->page
,
383 buffer_info
->pagecnt_bias
);
386 if (i
== rx_ring
->count
)
390 rx_ring
->next_to_alloc
= 0;
391 rx_ring
->next_to_clean
= 0;
392 rx_ring
->next_to_use
= 0;
396 * igc_clean_all_rx_rings - Free Rx Buffers for all queues
397 * @adapter: board private structure
399 static void igc_clean_all_rx_rings(struct igc_adapter
*adapter
)
403 for (i
= 0; i
< adapter
->num_rx_queues
; i
++)
404 if (adapter
->rx_ring
[i
])
405 igc_clean_rx_ring(adapter
->rx_ring
[i
]);
409 * igc_free_rx_resources - Free Rx Resources
410 * @rx_ring: ring to clean the resources from
412 * Free all receive software resources
414 void igc_free_rx_resources(struct igc_ring
*rx_ring
)
416 igc_clean_rx_ring(rx_ring
);
418 vfree(rx_ring
->rx_buffer_info
);
419 rx_ring
->rx_buffer_info
= NULL
;
421 /* if not set, then don't free */
425 dma_free_coherent(rx_ring
->dev
, rx_ring
->size
,
426 rx_ring
->desc
, rx_ring
->dma
);
428 rx_ring
->desc
= NULL
;
432 * igc_free_all_rx_resources - Free Rx Resources for All Queues
433 * @adapter: board private structure
435 * Free all receive software resources
437 static void igc_free_all_rx_resources(struct igc_adapter
*adapter
)
441 for (i
= 0; i
< adapter
->num_rx_queues
; i
++)
442 igc_free_rx_resources(adapter
->rx_ring
[i
]);
446 * igc_setup_rx_resources - allocate Rx resources (Descriptors)
447 * @rx_ring: rx descriptor ring (for a specific queue) to setup
449 * Returns 0 on success, negative on failure
451 int igc_setup_rx_resources(struct igc_ring
*rx_ring
)
453 struct device
*dev
= rx_ring
->dev
;
456 size
= sizeof(struct igc_rx_buffer
) * rx_ring
->count
;
457 rx_ring
->rx_buffer_info
= vzalloc(size
);
458 if (!rx_ring
->rx_buffer_info
)
461 desc_len
= sizeof(union igc_adv_rx_desc
);
463 /* Round up to nearest 4K */
464 rx_ring
->size
= rx_ring
->count
* desc_len
;
465 rx_ring
->size
= ALIGN(rx_ring
->size
, 4096);
467 rx_ring
->desc
= dma_alloc_coherent(dev
, rx_ring
->size
,
468 &rx_ring
->dma
, GFP_KERNEL
);
473 rx_ring
->next_to_alloc
= 0;
474 rx_ring
->next_to_clean
= 0;
475 rx_ring
->next_to_use
= 0;
480 vfree(rx_ring
->rx_buffer_info
);
481 rx_ring
->rx_buffer_info
= NULL
;
483 "Unable to allocate memory for the receive descriptor ring\n");
488 * igc_setup_all_rx_resources - wrapper to allocate Rx resources
489 * (Descriptors) for all queues
490 * @adapter: board private structure
492 * Return 0 on success, negative on failure
494 static int igc_setup_all_rx_resources(struct igc_adapter
*adapter
)
496 struct pci_dev
*pdev
= adapter
->pdev
;
499 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
500 err
= igc_setup_rx_resources(adapter
->rx_ring
[i
]);
503 "Allocation for Rx Queue %u failed\n", i
);
504 for (i
--; i
>= 0; i
--)
505 igc_free_rx_resources(adapter
->rx_ring
[i
]);
514 * igc_configure_rx_ring - Configure a receive ring after Reset
515 * @adapter: board private structure
516 * @ring: receive ring to be configured
518 * Configure the Rx unit of the MAC after a reset.
520 static void igc_configure_rx_ring(struct igc_adapter
*adapter
,
521 struct igc_ring
*ring
)
523 struct igc_hw
*hw
= &adapter
->hw
;
524 union igc_adv_rx_desc
*rx_desc
;
525 int reg_idx
= ring
->reg_idx
;
526 u32 srrctl
= 0, rxdctl
= 0;
527 u64 rdba
= ring
->dma
;
529 /* disable the queue */
530 wr32(IGC_RXDCTL(reg_idx
), 0);
532 /* Set DMA base address registers */
533 wr32(IGC_RDBAL(reg_idx
),
534 rdba
& 0x00000000ffffffffULL
);
535 wr32(IGC_RDBAH(reg_idx
), rdba
>> 32);
536 wr32(IGC_RDLEN(reg_idx
),
537 ring
->count
* sizeof(union igc_adv_rx_desc
));
539 /* initialize head and tail */
540 ring
->tail
= adapter
->io_addr
+ IGC_RDT(reg_idx
);
541 wr32(IGC_RDH(reg_idx
), 0);
542 writel(0, ring
->tail
);
544 /* reset next-to- use/clean to place SW in sync with hardware */
545 ring
->next_to_clean
= 0;
546 ring
->next_to_use
= 0;
548 /* set descriptor configuration */
549 srrctl
= IGC_RX_HDR_LEN
<< IGC_SRRCTL_BSIZEHDRSIZE_SHIFT
;
550 if (ring_uses_large_buffer(ring
))
551 srrctl
|= IGC_RXBUFFER_3072
>> IGC_SRRCTL_BSIZEPKT_SHIFT
;
553 srrctl
|= IGC_RXBUFFER_2048
>> IGC_SRRCTL_BSIZEPKT_SHIFT
;
554 srrctl
|= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF
;
556 wr32(IGC_SRRCTL(reg_idx
), srrctl
);
558 rxdctl
|= IGC_RX_PTHRESH
;
559 rxdctl
|= IGC_RX_HTHRESH
<< 8;
560 rxdctl
|= IGC_RX_WTHRESH
<< 16;
562 /* initialize rx_buffer_info */
563 memset(ring
->rx_buffer_info
, 0,
564 sizeof(struct igc_rx_buffer
) * ring
->count
);
566 /* initialize Rx descriptor 0 */
567 rx_desc
= IGC_RX_DESC(ring
, 0);
568 rx_desc
->wb
.upper
.length
= 0;
570 /* enable receive descriptor fetching */
571 rxdctl
|= IGC_RXDCTL_QUEUE_ENABLE
;
573 wr32(IGC_RXDCTL(reg_idx
), rxdctl
);
577 * igc_configure_rx - Configure receive Unit after Reset
578 * @adapter: board private structure
580 * Configure the Rx unit of the MAC after a reset.
582 static void igc_configure_rx(struct igc_adapter
*adapter
)
586 /* Setup the HW Rx Head and Tail Descriptor Pointers and
587 * the Base and Length of the Rx Descriptor Ring
589 for (i
= 0; i
< adapter
->num_rx_queues
; i
++)
590 igc_configure_rx_ring(adapter
, adapter
->rx_ring
[i
]);
594 * igc_configure_tx_ring - Configure transmit ring after Reset
595 * @adapter: board private structure
596 * @ring: tx ring to configure
598 * Configure a transmit ring after a reset.
600 static void igc_configure_tx_ring(struct igc_adapter
*adapter
,
601 struct igc_ring
*ring
)
603 struct igc_hw
*hw
= &adapter
->hw
;
604 int reg_idx
= ring
->reg_idx
;
605 u64 tdba
= ring
->dma
;
608 /* disable the queue */
609 wr32(IGC_TXDCTL(reg_idx
), 0);
613 wr32(IGC_TDLEN(reg_idx
),
614 ring
->count
* sizeof(union igc_adv_tx_desc
));
615 wr32(IGC_TDBAL(reg_idx
),
616 tdba
& 0x00000000ffffffffULL
);
617 wr32(IGC_TDBAH(reg_idx
), tdba
>> 32);
619 ring
->tail
= adapter
->io_addr
+ IGC_TDT(reg_idx
);
620 wr32(IGC_TDH(reg_idx
), 0);
621 writel(0, ring
->tail
);
623 txdctl
|= IGC_TX_PTHRESH
;
624 txdctl
|= IGC_TX_HTHRESH
<< 8;
625 txdctl
|= IGC_TX_WTHRESH
<< 16;
627 txdctl
|= IGC_TXDCTL_QUEUE_ENABLE
;
628 wr32(IGC_TXDCTL(reg_idx
), txdctl
);
632 * igc_configure_tx - Configure transmit Unit after Reset
633 * @adapter: board private structure
635 * Configure the Tx unit of the MAC after a reset.
637 static void igc_configure_tx(struct igc_adapter
*adapter
)
641 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
642 igc_configure_tx_ring(adapter
, adapter
->tx_ring
[i
]);
646 * igc_setup_mrqc - configure the multiple receive queue control registers
647 * @adapter: Board private structure
649 static void igc_setup_mrqc(struct igc_adapter
*adapter
)
651 struct igc_hw
*hw
= &adapter
->hw
;
652 u32 j
, num_rx_queues
;
656 netdev_rss_key_fill(rss_key
, sizeof(rss_key
));
657 for (j
= 0; j
< 10; j
++)
658 wr32(IGC_RSSRK(j
), rss_key
[j
]);
660 num_rx_queues
= adapter
->rss_queues
;
662 if (adapter
->rss_indir_tbl_init
!= num_rx_queues
) {
663 for (j
= 0; j
< IGC_RETA_SIZE
; j
++)
664 adapter
->rss_indir_tbl
[j
] =
665 (j
* num_rx_queues
) / IGC_RETA_SIZE
;
666 adapter
->rss_indir_tbl_init
= num_rx_queues
;
668 igc_write_rss_indir_tbl(adapter
);
670 /* Disable raw packet checksumming so that RSS hash is placed in
671 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
672 * offloads as they are enabled by default
674 rxcsum
= rd32(IGC_RXCSUM
);
675 rxcsum
|= IGC_RXCSUM_PCSD
;
677 /* Enable Receive Checksum Offload for SCTP */
678 rxcsum
|= IGC_RXCSUM_CRCOFL
;
680 /* Don't need to set TUOFL or IPOFL, they default to 1 */
681 wr32(IGC_RXCSUM
, rxcsum
);
683 /* Generate RSS hash based on packet types, TCP/UDP
684 * port numbers and/or IPv4/v6 src and dst addresses
686 mrqc
= IGC_MRQC_RSS_FIELD_IPV4
|
687 IGC_MRQC_RSS_FIELD_IPV4_TCP
|
688 IGC_MRQC_RSS_FIELD_IPV6
|
689 IGC_MRQC_RSS_FIELD_IPV6_TCP
|
690 IGC_MRQC_RSS_FIELD_IPV6_TCP_EX
;
692 if (adapter
->flags
& IGC_FLAG_RSS_FIELD_IPV4_UDP
)
693 mrqc
|= IGC_MRQC_RSS_FIELD_IPV4_UDP
;
694 if (adapter
->flags
& IGC_FLAG_RSS_FIELD_IPV6_UDP
)
695 mrqc
|= IGC_MRQC_RSS_FIELD_IPV6_UDP
;
697 mrqc
|= IGC_MRQC_ENABLE_RSS_MQ
;
699 wr32(IGC_MRQC
, mrqc
);
703 * igc_setup_rctl - configure the receive control registers
704 * @adapter: Board private structure
706 static void igc_setup_rctl(struct igc_adapter
*adapter
)
708 struct igc_hw
*hw
= &adapter
->hw
;
711 rctl
= rd32(IGC_RCTL
);
713 rctl
&= ~(3 << IGC_RCTL_MO_SHIFT
);
714 rctl
&= ~(IGC_RCTL_LBM_TCVR
| IGC_RCTL_LBM_MAC
);
716 rctl
|= IGC_RCTL_EN
| IGC_RCTL_BAM
| IGC_RCTL_RDMTS_HALF
|
717 (hw
->mac
.mc_filter_type
<< IGC_RCTL_MO_SHIFT
);
719 /* enable stripping of CRC. Newer features require
720 * that the HW strips the CRC.
722 rctl
|= IGC_RCTL_SECRC
;
724 /* disable store bad packets and clear size bits. */
725 rctl
&= ~(IGC_RCTL_SBP
| IGC_RCTL_SZ_256
);
727 /* enable LPE to allow for reception of jumbo frames */
728 rctl
|= IGC_RCTL_LPE
;
730 /* disable queue 0 to prevent tail write w/o re-config */
731 wr32(IGC_RXDCTL(0), 0);
733 /* This is useful for sniffing bad packets. */
734 if (adapter
->netdev
->features
& NETIF_F_RXALL
) {
735 /* UPE and MPE will be handled by normal PROMISC logic
738 rctl
|= (IGC_RCTL_SBP
| /* Receive bad packets */
739 IGC_RCTL_BAM
| /* RX All Bcast Pkts */
740 IGC_RCTL_PMCF
); /* RX All MAC Ctrl Pkts */
742 rctl
&= ~(IGC_RCTL_DPF
| /* Allow filtered pause */
743 IGC_RCTL_CFIEN
); /* Disable VLAN CFIEN Filter */
746 wr32(IGC_RCTL
, rctl
);
750 * igc_setup_tctl - configure the transmit control registers
751 * @adapter: Board private structure
753 static void igc_setup_tctl(struct igc_adapter
*adapter
)
755 struct igc_hw
*hw
= &adapter
->hw
;
758 /* disable queue 0 which icould be enabled by default */
759 wr32(IGC_TXDCTL(0), 0);
761 /* Program the Transmit Control Register */
762 tctl
= rd32(IGC_TCTL
);
763 tctl
&= ~IGC_TCTL_CT
;
764 tctl
|= IGC_TCTL_PSP
| IGC_TCTL_RTLC
|
765 (IGC_COLLISION_THRESHOLD
<< IGC_CT_SHIFT
);
767 /* Enable transmits */
770 wr32(IGC_TCTL
, tctl
);
774 * igc_set_mac - Change the Ethernet Address of the NIC
775 * @netdev: network interface device structure
776 * @p: pointer to an address structure
778 * Returns 0 on success, negative on failure
780 static int igc_set_mac(struct net_device
*netdev
, void *p
)
782 struct igc_adapter
*adapter
= netdev_priv(netdev
);
783 struct igc_hw
*hw
= &adapter
->hw
;
784 struct sockaddr
*addr
= p
;
786 if (!is_valid_ether_addr(addr
->sa_data
))
787 return -EADDRNOTAVAIL
;
789 memcpy(netdev
->dev_addr
, addr
->sa_data
, netdev
->addr_len
);
790 memcpy(hw
->mac
.addr
, addr
->sa_data
, netdev
->addr_len
);
792 /* set the correct pool for the new PF MAC address in entry 0 */
793 igc_set_default_mac_filter(adapter
);
799 * igc_write_mc_addr_list - write multicast addresses to MTA
800 * @netdev: network interface device structure
802 * Writes multicast address list to the MTA hash table.
803 * Returns: -ENOMEM on failure
804 * 0 on no addresses written
805 * X on writing X addresses to MTA
807 static int igc_write_mc_addr_list(struct net_device
*netdev
)
809 struct igc_adapter
*adapter
= netdev_priv(netdev
);
810 struct igc_hw
*hw
= &adapter
->hw
;
811 struct netdev_hw_addr
*ha
;
815 if (netdev_mc_empty(netdev
)) {
816 /* nothing to program, so clear mc list */
817 igc_update_mc_addr_list(hw
, NULL
, 0);
821 mta_list
= kcalloc(netdev_mc_count(netdev
), 6, GFP_ATOMIC
);
825 /* The shared function expects a packed array of only addresses. */
827 netdev_for_each_mc_addr(ha
, netdev
)
828 memcpy(mta_list
+ (i
++ * ETH_ALEN
), ha
->addr
, ETH_ALEN
);
830 igc_update_mc_addr_list(hw
, mta_list
, i
);
833 return netdev_mc_count(netdev
);
836 static void igc_tx_ctxtdesc(struct igc_ring
*tx_ring
,
837 struct igc_tx_buffer
*first
,
838 u32 vlan_macip_lens
, u32 type_tucmd
,
841 struct igc_adv_tx_context_desc
*context_desc
;
842 u16 i
= tx_ring
->next_to_use
;
843 struct timespec64 ts
;
845 context_desc
= IGC_TX_CTXTDESC(tx_ring
, i
);
848 tx_ring
->next_to_use
= (i
< tx_ring
->count
) ? i
: 0;
850 /* set bits to identify this as an advanced context descriptor */
851 type_tucmd
|= IGC_TXD_CMD_DEXT
| IGC_ADVTXD_DTYP_CTXT
;
853 /* For 82575, context index must be unique per ring. */
854 if (test_bit(IGC_RING_FLAG_TX_CTX_IDX
, &tx_ring
->flags
))
855 mss_l4len_idx
|= tx_ring
->reg_idx
<< 4;
857 context_desc
->vlan_macip_lens
= cpu_to_le32(vlan_macip_lens
);
858 context_desc
->type_tucmd_mlhl
= cpu_to_le32(type_tucmd
);
859 context_desc
->mss_l4len_idx
= cpu_to_le32(mss_l4len_idx
);
861 /* We assume there is always a valid Tx time available. Invalid times
862 * should have been handled by the upper layers.
864 if (tx_ring
->launchtime_enable
) {
865 ts
= ktime_to_timespec64(first
->skb
->tstamp
);
866 first
->skb
->tstamp
= ktime_set(0, 0);
867 context_desc
->launch_time
= cpu_to_le32(ts
.tv_nsec
/ 32);
869 context_desc
->launch_time
= 0;
873 static inline bool igc_ipv6_csum_is_sctp(struct sk_buff
*skb
)
875 unsigned int offset
= 0;
877 ipv6_find_hdr(skb
, &offset
, IPPROTO_SCTP
, NULL
, NULL
);
879 return offset
== skb_checksum_start_offset(skb
);
882 static void igc_tx_csum(struct igc_ring
*tx_ring
, struct igc_tx_buffer
*first
)
884 struct sk_buff
*skb
= first
->skb
;
885 u32 vlan_macip_lens
= 0;
888 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
) {
890 if (!(first
->tx_flags
& IGC_TX_FLAGS_VLAN
) &&
891 !tx_ring
->launchtime_enable
)
896 switch (skb
->csum_offset
) {
897 case offsetof(struct tcphdr
, check
):
898 type_tucmd
= IGC_ADVTXD_TUCMD_L4T_TCP
;
900 case offsetof(struct udphdr
, check
):
902 case offsetof(struct sctphdr
, checksum
):
903 /* validate that this is actually an SCTP request */
904 if ((first
->protocol
== htons(ETH_P_IP
) &&
905 (ip_hdr(skb
)->protocol
== IPPROTO_SCTP
)) ||
906 (first
->protocol
== htons(ETH_P_IPV6
) &&
907 igc_ipv6_csum_is_sctp(skb
))) {
908 type_tucmd
= IGC_ADVTXD_TUCMD_L4T_SCTP
;
913 skb_checksum_help(skb
);
917 /* update TX checksum flag */
918 first
->tx_flags
|= IGC_TX_FLAGS_CSUM
;
919 vlan_macip_lens
= skb_checksum_start_offset(skb
) -
920 skb_network_offset(skb
);
922 vlan_macip_lens
|= skb_network_offset(skb
) << IGC_ADVTXD_MACLEN_SHIFT
;
923 vlan_macip_lens
|= first
->tx_flags
& IGC_TX_FLAGS_VLAN_MASK
;
925 igc_tx_ctxtdesc(tx_ring
, first
, vlan_macip_lens
, type_tucmd
, 0);
928 static int __igc_maybe_stop_tx(struct igc_ring
*tx_ring
, const u16 size
)
930 struct net_device
*netdev
= tx_ring
->netdev
;
932 netif_stop_subqueue(netdev
, tx_ring
->queue_index
);
934 /* memory barriier comment */
937 /* We need to check again in a case another CPU has just
938 * made room available.
940 if (igc_desc_unused(tx_ring
) < size
)
944 netif_wake_subqueue(netdev
, tx_ring
->queue_index
);
946 u64_stats_update_begin(&tx_ring
->tx_syncp2
);
947 tx_ring
->tx_stats
.restart_queue2
++;
948 u64_stats_update_end(&tx_ring
->tx_syncp2
);
953 static inline int igc_maybe_stop_tx(struct igc_ring
*tx_ring
, const u16 size
)
955 if (igc_desc_unused(tx_ring
) >= size
)
957 return __igc_maybe_stop_tx(tx_ring
, size
);
960 static u32
igc_tx_cmd_type(struct sk_buff
*skb
, u32 tx_flags
)
962 /* set type for advanced descriptor with frame checksum insertion */
963 u32 cmd_type
= IGC_ADVTXD_DTYP_DATA
|
964 IGC_ADVTXD_DCMD_DEXT
|
965 IGC_ADVTXD_DCMD_IFCS
;
970 static void igc_tx_olinfo_status(struct igc_ring
*tx_ring
,
971 union igc_adv_tx_desc
*tx_desc
,
972 u32 tx_flags
, unsigned int paylen
)
974 u32 olinfo_status
= paylen
<< IGC_ADVTXD_PAYLEN_SHIFT
;
976 /* insert L4 checksum */
977 olinfo_status
|= (tx_flags
& IGC_TX_FLAGS_CSUM
) *
978 ((IGC_TXD_POPTS_TXSM
<< 8) /
981 /* insert IPv4 checksum */
982 olinfo_status
|= (tx_flags
& IGC_TX_FLAGS_IPV4
) *
983 (((IGC_TXD_POPTS_IXSM
<< 8)) /
986 tx_desc
->read
.olinfo_status
= cpu_to_le32(olinfo_status
);
989 static int igc_tx_map(struct igc_ring
*tx_ring
,
990 struct igc_tx_buffer
*first
,
993 struct sk_buff
*skb
= first
->skb
;
994 struct igc_tx_buffer
*tx_buffer
;
995 union igc_adv_tx_desc
*tx_desc
;
996 u32 tx_flags
= first
->tx_flags
;
998 u16 i
= tx_ring
->next_to_use
;
999 unsigned int data_len
, size
;
1001 u32 cmd_type
= igc_tx_cmd_type(skb
, tx_flags
);
1003 tx_desc
= IGC_TX_DESC(tx_ring
, i
);
1005 igc_tx_olinfo_status(tx_ring
, tx_desc
, tx_flags
, skb
->len
- hdr_len
);
1007 size
= skb_headlen(skb
);
1008 data_len
= skb
->data_len
;
1010 dma
= dma_map_single(tx_ring
->dev
, skb
->data
, size
, DMA_TO_DEVICE
);
1014 for (frag
= &skb_shinfo(skb
)->frags
[0];; frag
++) {
1015 if (dma_mapping_error(tx_ring
->dev
, dma
))
1018 /* record length, and DMA address */
1019 dma_unmap_len_set(tx_buffer
, len
, size
);
1020 dma_unmap_addr_set(tx_buffer
, dma
, dma
);
1022 tx_desc
->read
.buffer_addr
= cpu_to_le64(dma
);
1024 while (unlikely(size
> IGC_MAX_DATA_PER_TXD
)) {
1025 tx_desc
->read
.cmd_type_len
=
1026 cpu_to_le32(cmd_type
^ IGC_MAX_DATA_PER_TXD
);
1030 if (i
== tx_ring
->count
) {
1031 tx_desc
= IGC_TX_DESC(tx_ring
, 0);
1034 tx_desc
->read
.olinfo_status
= 0;
1036 dma
+= IGC_MAX_DATA_PER_TXD
;
1037 size
-= IGC_MAX_DATA_PER_TXD
;
1039 tx_desc
->read
.buffer_addr
= cpu_to_le64(dma
);
1042 if (likely(!data_len
))
1045 tx_desc
->read
.cmd_type_len
= cpu_to_le32(cmd_type
^ size
);
1049 if (i
== tx_ring
->count
) {
1050 tx_desc
= IGC_TX_DESC(tx_ring
, 0);
1053 tx_desc
->read
.olinfo_status
= 0;
1055 size
= skb_frag_size(frag
);
1058 dma
= skb_frag_dma_map(tx_ring
->dev
, frag
, 0,
1059 size
, DMA_TO_DEVICE
);
1061 tx_buffer
= &tx_ring
->tx_buffer_info
[i
];
1064 /* write last descriptor with RS and EOP bits */
1065 cmd_type
|= size
| IGC_TXD_DCMD
;
1066 tx_desc
->read
.cmd_type_len
= cpu_to_le32(cmd_type
);
1068 netdev_tx_sent_queue(txring_txq(tx_ring
), first
->bytecount
);
1070 /* set the timestamp */
1071 first
->time_stamp
= jiffies
;
1073 skb_tx_timestamp(skb
);
1075 /* Force memory writes to complete before letting h/w know there
1076 * are new descriptors to fetch. (Only applicable for weak-ordered
1077 * memory model archs, such as IA-64).
1079 * We also need this memory barrier to make certain all of the
1080 * status bits have been updated before next_to_watch is written.
1084 /* set next_to_watch value indicating a packet is present */
1085 first
->next_to_watch
= tx_desc
;
1088 if (i
== tx_ring
->count
)
1091 tx_ring
->next_to_use
= i
;
1093 /* Make sure there is space in the ring for the next send. */
1094 igc_maybe_stop_tx(tx_ring
, DESC_NEEDED
);
1096 if (netif_xmit_stopped(txring_txq(tx_ring
)) || !netdev_xmit_more()) {
1097 writel(i
, tx_ring
->tail
);
1102 dev_err(tx_ring
->dev
, "TX DMA map failed\n");
1103 tx_buffer
= &tx_ring
->tx_buffer_info
[i
];
1105 /* clear dma mappings for failed tx_buffer_info map */
1106 while (tx_buffer
!= first
) {
1107 if (dma_unmap_len(tx_buffer
, len
))
1108 dma_unmap_page(tx_ring
->dev
,
1109 dma_unmap_addr(tx_buffer
, dma
),
1110 dma_unmap_len(tx_buffer
, len
),
1112 dma_unmap_len_set(tx_buffer
, len
, 0);
1115 i
+= tx_ring
->count
;
1116 tx_buffer
= &tx_ring
->tx_buffer_info
[i
];
1119 if (dma_unmap_len(tx_buffer
, len
))
1120 dma_unmap_single(tx_ring
->dev
,
1121 dma_unmap_addr(tx_buffer
, dma
),
1122 dma_unmap_len(tx_buffer
, len
),
1124 dma_unmap_len_set(tx_buffer
, len
, 0);
1126 dev_kfree_skb_any(tx_buffer
->skb
);
1127 tx_buffer
->skb
= NULL
;
1129 tx_ring
->next_to_use
= i
;
1134 static netdev_tx_t
igc_xmit_frame_ring(struct sk_buff
*skb
,
1135 struct igc_ring
*tx_ring
)
1137 u16 count
= TXD_USE_COUNT(skb_headlen(skb
));
1138 __be16 protocol
= vlan_get_protocol(skb
);
1139 struct igc_tx_buffer
*first
;
1144 /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD,
1145 * + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD,
1146 * + 2 desc gap to keep tail from touching head,
1147 * + 1 desc for context descriptor,
1148 * otherwise try next time
1150 for (f
= 0; f
< skb_shinfo(skb
)->nr_frags
; f
++)
1151 count
+= TXD_USE_COUNT(skb_frag_size(
1152 &skb_shinfo(skb
)->frags
[f
]));
1154 if (igc_maybe_stop_tx(tx_ring
, count
+ 3)) {
1155 /* this is a hard error */
1156 return NETDEV_TX_BUSY
;
1159 /* record the location of the first descriptor for this packet */
1160 first
= &tx_ring
->tx_buffer_info
[tx_ring
->next_to_use
];
1162 first
->bytecount
= skb
->len
;
1163 first
->gso_segs
= 1;
1165 /* record initial flags and protocol */
1166 first
->tx_flags
= tx_flags
;
1167 first
->protocol
= protocol
;
1169 igc_tx_csum(tx_ring
, first
);
1171 igc_tx_map(tx_ring
, first
, hdr_len
);
1173 return NETDEV_TX_OK
;
1176 static inline struct igc_ring
*igc_tx_queue_mapping(struct igc_adapter
*adapter
,
1177 struct sk_buff
*skb
)
1179 unsigned int r_idx
= skb
->queue_mapping
;
1181 if (r_idx
>= adapter
->num_tx_queues
)
1182 r_idx
= r_idx
% adapter
->num_tx_queues
;
1184 return adapter
->tx_ring
[r_idx
];
1187 static netdev_tx_t
igc_xmit_frame(struct sk_buff
*skb
,
1188 struct net_device
*netdev
)
1190 struct igc_adapter
*adapter
= netdev_priv(netdev
);
1192 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb
1193 * in order to meet this minimum size requirement.
1195 if (skb
->len
< 17) {
1196 if (skb_padto(skb
, 17))
1197 return NETDEV_TX_OK
;
1201 return igc_xmit_frame_ring(skb
, igc_tx_queue_mapping(adapter
, skb
));
1204 static void igc_rx_checksum(struct igc_ring
*ring
,
1205 union igc_adv_rx_desc
*rx_desc
,
1206 struct sk_buff
*skb
)
1208 skb_checksum_none_assert(skb
);
1210 /* Ignore Checksum bit is set */
1211 if (igc_test_staterr(rx_desc
, IGC_RXD_STAT_IXSM
))
1214 /* Rx checksum disabled via ethtool */
1215 if (!(ring
->netdev
->features
& NETIF_F_RXCSUM
))
1218 /* TCP/UDP checksum error bit is set */
1219 if (igc_test_staterr(rx_desc
,
1220 IGC_RXDEXT_STATERR_TCPE
|
1221 IGC_RXDEXT_STATERR_IPE
)) {
1222 /* work around errata with sctp packets where the TCPE aka
1223 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
1224 * packets (aka let the stack check the crc32c)
1226 if (!(skb
->len
== 60 &&
1227 test_bit(IGC_RING_FLAG_RX_SCTP_CSUM
, &ring
->flags
))) {
1228 u64_stats_update_begin(&ring
->rx_syncp
);
1229 ring
->rx_stats
.csum_err
++;
1230 u64_stats_update_end(&ring
->rx_syncp
);
1232 /* let the stack verify checksum errors */
1235 /* It must be a TCP or UDP packet with a valid checksum */
1236 if (igc_test_staterr(rx_desc
, IGC_RXD_STAT_TCPCS
|
1237 IGC_RXD_STAT_UDPCS
))
1238 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
1240 dev_dbg(ring
->dev
, "cksum success: bits %08X\n",
1241 le32_to_cpu(rx_desc
->wb
.upper
.status_error
));
1244 static inline void igc_rx_hash(struct igc_ring
*ring
,
1245 union igc_adv_rx_desc
*rx_desc
,
1246 struct sk_buff
*skb
)
1248 if (ring
->netdev
->features
& NETIF_F_RXHASH
)
1250 le32_to_cpu(rx_desc
->wb
.lower
.hi_dword
.rss
),
1255 * igc_process_skb_fields - Populate skb header fields from Rx descriptor
1256 * @rx_ring: rx descriptor ring packet is being transacted on
1257 * @rx_desc: pointer to the EOP Rx descriptor
1258 * @skb: pointer to current skb being populated
1260 * This function checks the ring, descriptor, and packet information in
1261 * order to populate the hash, checksum, VLAN, timestamp, protocol, and
1262 * other fields within the skb.
1264 static void igc_process_skb_fields(struct igc_ring
*rx_ring
,
1265 union igc_adv_rx_desc
*rx_desc
,
1266 struct sk_buff
*skb
)
1268 igc_rx_hash(rx_ring
, rx_desc
, skb
);
1270 igc_rx_checksum(rx_ring
, rx_desc
, skb
);
1272 skb_record_rx_queue(skb
, rx_ring
->queue_index
);
1274 skb
->protocol
= eth_type_trans(skb
, rx_ring
->netdev
);
1277 static struct igc_rx_buffer
*igc_get_rx_buffer(struct igc_ring
*rx_ring
,
1278 const unsigned int size
)
1280 struct igc_rx_buffer
*rx_buffer
;
1282 rx_buffer
= &rx_ring
->rx_buffer_info
[rx_ring
->next_to_clean
];
1283 prefetchw(rx_buffer
->page
);
1285 /* we are reusing so sync this buffer for CPU use */
1286 dma_sync_single_range_for_cpu(rx_ring
->dev
,
1288 rx_buffer
->page_offset
,
1292 rx_buffer
->pagecnt_bias
--;
1298 * igc_add_rx_frag - Add contents of Rx buffer to sk_buff
1299 * @rx_ring: rx descriptor ring to transact packets on
1300 * @rx_buffer: buffer containing page to add
1301 * @skb: sk_buff to place the data into
1302 * @size: size of buffer to be added
1304 * This function will add the data contained in rx_buffer->page to the skb.
1306 static void igc_add_rx_frag(struct igc_ring
*rx_ring
,
1307 struct igc_rx_buffer
*rx_buffer
,
1308 struct sk_buff
*skb
,
1311 #if (PAGE_SIZE < 8192)
1312 unsigned int truesize
= igc_rx_pg_size(rx_ring
) / 2;
1314 skb_add_rx_frag(skb
, skb_shinfo(skb
)->nr_frags
, rx_buffer
->page
,
1315 rx_buffer
->page_offset
, size
, truesize
);
1316 rx_buffer
->page_offset
^= truesize
;
1318 unsigned int truesize
= ring_uses_build_skb(rx_ring
) ?
1319 SKB_DATA_ALIGN(IGC_SKB_PAD
+ size
) :
1320 SKB_DATA_ALIGN(size
);
1321 skb_add_rx_frag(skb
, skb_shinfo(skb
)->nr_frags
, rx_buffer
->page
,
1322 rx_buffer
->page_offset
, size
, truesize
);
1323 rx_buffer
->page_offset
+= truesize
;
1327 static struct sk_buff
*igc_build_skb(struct igc_ring
*rx_ring
,
1328 struct igc_rx_buffer
*rx_buffer
,
1329 union igc_adv_rx_desc
*rx_desc
,
1332 void *va
= page_address(rx_buffer
->page
) + rx_buffer
->page_offset
;
1333 #if (PAGE_SIZE < 8192)
1334 unsigned int truesize
= igc_rx_pg_size(rx_ring
) / 2;
1336 unsigned int truesize
= SKB_DATA_ALIGN(sizeof(struct skb_shared_info
)) +
1337 SKB_DATA_ALIGN(IGC_SKB_PAD
+ size
);
1339 struct sk_buff
*skb
;
1341 /* prefetch first cache line of first page */
1343 #if L1_CACHE_BYTES < 128
1344 prefetch(va
+ L1_CACHE_BYTES
);
1347 /* build an skb around the page buffer */
1348 skb
= build_skb(va
- IGC_SKB_PAD
, truesize
);
1352 /* update pointers within the skb to store the data */
1353 skb_reserve(skb
, IGC_SKB_PAD
);
1354 __skb_put(skb
, size
);
1356 /* update buffer offset */
1357 #if (PAGE_SIZE < 8192)
1358 rx_buffer
->page_offset
^= truesize
;
1360 rx_buffer
->page_offset
+= truesize
;
1366 static struct sk_buff
*igc_construct_skb(struct igc_ring
*rx_ring
,
1367 struct igc_rx_buffer
*rx_buffer
,
1368 union igc_adv_rx_desc
*rx_desc
,
1371 void *va
= page_address(rx_buffer
->page
) + rx_buffer
->page_offset
;
1372 #if (PAGE_SIZE < 8192)
1373 unsigned int truesize
= igc_rx_pg_size(rx_ring
) / 2;
1375 unsigned int truesize
= SKB_DATA_ALIGN(size
);
1377 unsigned int headlen
;
1378 struct sk_buff
*skb
;
1380 /* prefetch first cache line of first page */
1382 #if L1_CACHE_BYTES < 128
1383 prefetch(va
+ L1_CACHE_BYTES
);
1386 /* allocate a skb to store the frags */
1387 skb
= napi_alloc_skb(&rx_ring
->q_vector
->napi
, IGC_RX_HDR_LEN
);
1391 /* Determine available headroom for copy */
1393 if (headlen
> IGC_RX_HDR_LEN
)
1394 headlen
= eth_get_headlen(skb
->dev
, va
, IGC_RX_HDR_LEN
);
1396 /* align pull length to size of long to optimize memcpy performance */
1397 memcpy(__skb_put(skb
, headlen
), va
, ALIGN(headlen
, sizeof(long)));
1399 /* update all of the pointers */
1402 skb_add_rx_frag(skb
, 0, rx_buffer
->page
,
1403 (va
+ headlen
) - page_address(rx_buffer
->page
),
1405 #if (PAGE_SIZE < 8192)
1406 rx_buffer
->page_offset
^= truesize
;
1408 rx_buffer
->page_offset
+= truesize
;
1411 rx_buffer
->pagecnt_bias
++;
1418 * igc_reuse_rx_page - page flip buffer and store it back on the ring
1419 * @rx_ring: rx descriptor ring to store buffers on
1420 * @old_buff: donor buffer to have page reused
1422 * Synchronizes page for reuse by the adapter
1424 static void igc_reuse_rx_page(struct igc_ring
*rx_ring
,
1425 struct igc_rx_buffer
*old_buff
)
1427 u16 nta
= rx_ring
->next_to_alloc
;
1428 struct igc_rx_buffer
*new_buff
;
1430 new_buff
= &rx_ring
->rx_buffer_info
[nta
];
1432 /* update, and store next to alloc */
1434 rx_ring
->next_to_alloc
= (nta
< rx_ring
->count
) ? nta
: 0;
1436 /* Transfer page from old buffer to new buffer.
1437 * Move each member individually to avoid possible store
1438 * forwarding stalls.
1440 new_buff
->dma
= old_buff
->dma
;
1441 new_buff
->page
= old_buff
->page
;
1442 new_buff
->page_offset
= old_buff
->page_offset
;
1443 new_buff
->pagecnt_bias
= old_buff
->pagecnt_bias
;
1446 static inline bool igc_page_is_reserved(struct page
*page
)
1448 return (page_to_nid(page
) != numa_mem_id()) || page_is_pfmemalloc(page
);
1451 static bool igc_can_reuse_rx_page(struct igc_rx_buffer
*rx_buffer
)
1453 unsigned int pagecnt_bias
= rx_buffer
->pagecnt_bias
;
1454 struct page
*page
= rx_buffer
->page
;
1456 /* avoid re-using remote pages */
1457 if (unlikely(igc_page_is_reserved(page
)))
1460 #if (PAGE_SIZE < 8192)
1461 /* if we are only owner of page we can reuse it */
1462 if (unlikely((page_ref_count(page
) - pagecnt_bias
) > 1))
1465 #define IGC_LAST_OFFSET \
1466 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048)
1468 if (rx_buffer
->page_offset
> IGC_LAST_OFFSET
)
1472 /* If we have drained the page fragment pool we need to update
1473 * the pagecnt_bias and page count so that we fully restock the
1474 * number of references the driver holds.
1476 if (unlikely(!pagecnt_bias
)) {
1477 page_ref_add(page
, USHRT_MAX
);
1478 rx_buffer
->pagecnt_bias
= USHRT_MAX
;
1485 * igc_is_non_eop - process handling of non-EOP buffers
1486 * @rx_ring: Rx ring being processed
1487 * @rx_desc: Rx descriptor for current buffer
1488 * @skb: current socket buffer containing buffer in progress
1490 * This function updates next to clean. If the buffer is an EOP buffer
1491 * this function exits returning false, otherwise it will place the
1492 * sk_buff in the next buffer to be chained and return true indicating
1493 * that this is in fact a non-EOP buffer.
1495 static bool igc_is_non_eop(struct igc_ring
*rx_ring
,
1496 union igc_adv_rx_desc
*rx_desc
)
1498 u32 ntc
= rx_ring
->next_to_clean
+ 1;
1500 /* fetch, update, and store next to clean */
1501 ntc
= (ntc
< rx_ring
->count
) ? ntc
: 0;
1502 rx_ring
->next_to_clean
= ntc
;
1504 prefetch(IGC_RX_DESC(rx_ring
, ntc
));
1506 if (likely(igc_test_staterr(rx_desc
, IGC_RXD_STAT_EOP
)))
1513 * igc_cleanup_headers - Correct corrupted or empty headers
1514 * @rx_ring: rx descriptor ring packet is being transacted on
1515 * @rx_desc: pointer to the EOP Rx descriptor
1516 * @skb: pointer to current skb being fixed
1518 * Address the case where we are pulling data in on pages only
1519 * and as such no data is present in the skb header.
1521 * In addition if skb is not at least 60 bytes we need to pad it so that
1522 * it is large enough to qualify as a valid Ethernet frame.
1524 * Returns true if an error was encountered and skb was freed.
1526 static bool igc_cleanup_headers(struct igc_ring
*rx_ring
,
1527 union igc_adv_rx_desc
*rx_desc
,
1528 struct sk_buff
*skb
)
1530 if (unlikely((igc_test_staterr(rx_desc
,
1531 IGC_RXDEXT_ERR_FRAME_ERR_MASK
)))) {
1532 struct net_device
*netdev
= rx_ring
->netdev
;
1534 if (!(netdev
->features
& NETIF_F_RXALL
)) {
1535 dev_kfree_skb_any(skb
);
1540 /* if eth_skb_pad returns an error the skb was freed */
1541 if (eth_skb_pad(skb
))
1547 static void igc_put_rx_buffer(struct igc_ring
*rx_ring
,
1548 struct igc_rx_buffer
*rx_buffer
)
1550 if (igc_can_reuse_rx_page(rx_buffer
)) {
1551 /* hand second half of page back to the ring */
1552 igc_reuse_rx_page(rx_ring
, rx_buffer
);
1554 /* We are not reusing the buffer so unmap it and free
1555 * any references we are holding to it
1557 dma_unmap_page_attrs(rx_ring
->dev
, rx_buffer
->dma
,
1558 igc_rx_pg_size(rx_ring
), DMA_FROM_DEVICE
,
1560 __page_frag_cache_drain(rx_buffer
->page
,
1561 rx_buffer
->pagecnt_bias
);
1564 /* clear contents of rx_buffer */
1565 rx_buffer
->page
= NULL
;
1569 * igc_alloc_rx_buffers - Replace used receive buffers; packet split
1570 * @adapter: address of board private structure
1572 static void igc_alloc_rx_buffers(struct igc_ring
*rx_ring
, u16 cleaned_count
)
1574 union igc_adv_rx_desc
*rx_desc
;
1575 u16 i
= rx_ring
->next_to_use
;
1576 struct igc_rx_buffer
*bi
;
1583 rx_desc
= IGC_RX_DESC(rx_ring
, i
);
1584 bi
= &rx_ring
->rx_buffer_info
[i
];
1585 i
-= rx_ring
->count
;
1587 bufsz
= igc_rx_bufsz(rx_ring
);
1590 if (!igc_alloc_mapped_page(rx_ring
, bi
))
1593 /* sync the buffer for use by the device */
1594 dma_sync_single_range_for_device(rx_ring
->dev
, bi
->dma
,
1595 bi
->page_offset
, bufsz
,
1598 /* Refresh the desc even if buffer_addrs didn't change
1599 * because each write-back erases this info.
1601 rx_desc
->read
.pkt_addr
= cpu_to_le64(bi
->dma
+ bi
->page_offset
);
1607 rx_desc
= IGC_RX_DESC(rx_ring
, 0);
1608 bi
= rx_ring
->rx_buffer_info
;
1609 i
-= rx_ring
->count
;
1612 /* clear the length for the next_to_use descriptor */
1613 rx_desc
->wb
.upper
.length
= 0;
1616 } while (cleaned_count
);
1618 i
+= rx_ring
->count
;
1620 if (rx_ring
->next_to_use
!= i
) {
1621 /* record the next descriptor to use */
1622 rx_ring
->next_to_use
= i
;
1624 /* update next to alloc since we have filled the ring */
1625 rx_ring
->next_to_alloc
= i
;
1627 /* Force memory writes to complete before letting h/w
1628 * know there are new descriptors to fetch. (Only
1629 * applicable for weak-ordered memory model archs,
1633 writel(i
, rx_ring
->tail
);
1637 static int igc_clean_rx_irq(struct igc_q_vector
*q_vector
, const int budget
)
1639 unsigned int total_bytes
= 0, total_packets
= 0;
1640 struct igc_ring
*rx_ring
= q_vector
->rx
.ring
;
1641 struct sk_buff
*skb
= rx_ring
->skb
;
1642 u16 cleaned_count
= igc_desc_unused(rx_ring
);
1644 while (likely(total_packets
< budget
)) {
1645 union igc_adv_rx_desc
*rx_desc
;
1646 struct igc_rx_buffer
*rx_buffer
;
1649 /* return some buffers to hardware, one at a time is too slow */
1650 if (cleaned_count
>= IGC_RX_BUFFER_WRITE
) {
1651 igc_alloc_rx_buffers(rx_ring
, cleaned_count
);
1655 rx_desc
= IGC_RX_DESC(rx_ring
, rx_ring
->next_to_clean
);
1656 size
= le16_to_cpu(rx_desc
->wb
.upper
.length
);
1660 /* This memory barrier is needed to keep us from reading
1661 * any other fields out of the rx_desc until we know the
1662 * descriptor has been written back
1666 rx_buffer
= igc_get_rx_buffer(rx_ring
, size
);
1668 /* retrieve a buffer from the ring */
1670 igc_add_rx_frag(rx_ring
, rx_buffer
, skb
, size
);
1671 else if (ring_uses_build_skb(rx_ring
))
1672 skb
= igc_build_skb(rx_ring
, rx_buffer
, rx_desc
, size
);
1674 skb
= igc_construct_skb(rx_ring
, rx_buffer
,
1677 /* exit if we failed to retrieve a buffer */
1679 rx_ring
->rx_stats
.alloc_failed
++;
1680 rx_buffer
->pagecnt_bias
++;
1684 igc_put_rx_buffer(rx_ring
, rx_buffer
);
1687 /* fetch next buffer in frame if non-eop */
1688 if (igc_is_non_eop(rx_ring
, rx_desc
))
1691 /* verify the packet layout is correct */
1692 if (igc_cleanup_headers(rx_ring
, rx_desc
, skb
)) {
1697 /* probably a little skewed due to removing CRC */
1698 total_bytes
+= skb
->len
;
1700 /* populate checksum, timestamp, VLAN, and protocol */
1701 igc_process_skb_fields(rx_ring
, rx_desc
, skb
);
1703 napi_gro_receive(&q_vector
->napi
, skb
);
1705 /* reset skb pointer */
1708 /* update budget accounting */
1712 /* place incomplete frames back on ring for completion */
1715 u64_stats_update_begin(&rx_ring
->rx_syncp
);
1716 rx_ring
->rx_stats
.packets
+= total_packets
;
1717 rx_ring
->rx_stats
.bytes
+= total_bytes
;
1718 u64_stats_update_end(&rx_ring
->rx_syncp
);
1719 q_vector
->rx
.total_packets
+= total_packets
;
1720 q_vector
->rx
.total_bytes
+= total_bytes
;
1723 igc_alloc_rx_buffers(rx_ring
, cleaned_count
);
1725 return total_packets
;
1728 static inline unsigned int igc_rx_offset(struct igc_ring
*rx_ring
)
1730 return ring_uses_build_skb(rx_ring
) ? IGC_SKB_PAD
: 0;
1733 static bool igc_alloc_mapped_page(struct igc_ring
*rx_ring
,
1734 struct igc_rx_buffer
*bi
)
1736 struct page
*page
= bi
->page
;
1739 /* since we are recycling buffers we should seldom need to alloc */
1743 /* alloc new page for storage */
1744 page
= dev_alloc_pages(igc_rx_pg_order(rx_ring
));
1745 if (unlikely(!page
)) {
1746 rx_ring
->rx_stats
.alloc_failed
++;
1750 /* map page for use */
1751 dma
= dma_map_page_attrs(rx_ring
->dev
, page
, 0,
1752 igc_rx_pg_size(rx_ring
),
1756 /* if mapping failed free memory back to system since
1757 * there isn't much point in holding memory we can't use
1759 if (dma_mapping_error(rx_ring
->dev
, dma
)) {
1762 rx_ring
->rx_stats
.alloc_failed
++;
1768 bi
->page_offset
= igc_rx_offset(rx_ring
);
1769 bi
->pagecnt_bias
= 1;
1775 * igc_clean_tx_irq - Reclaim resources after transmit completes
1776 * @q_vector: pointer to q_vector containing needed info
1777 * @napi_budget: Used to determine if we are in netpoll
1779 * returns true if ring is completely cleaned
1781 static bool igc_clean_tx_irq(struct igc_q_vector
*q_vector
, int napi_budget
)
1783 struct igc_adapter
*adapter
= q_vector
->adapter
;
1784 unsigned int total_bytes
= 0, total_packets
= 0;
1785 unsigned int budget
= q_vector
->tx
.work_limit
;
1786 struct igc_ring
*tx_ring
= q_vector
->tx
.ring
;
1787 unsigned int i
= tx_ring
->next_to_clean
;
1788 struct igc_tx_buffer
*tx_buffer
;
1789 union igc_adv_tx_desc
*tx_desc
;
1791 if (test_bit(__IGC_DOWN
, &adapter
->state
))
1794 tx_buffer
= &tx_ring
->tx_buffer_info
[i
];
1795 tx_desc
= IGC_TX_DESC(tx_ring
, i
);
1796 i
-= tx_ring
->count
;
1799 union igc_adv_tx_desc
*eop_desc
= tx_buffer
->next_to_watch
;
1801 /* if next_to_watch is not set then there is no work pending */
1805 /* prevent any other reads prior to eop_desc */
1808 /* if DD is not set pending work has not been completed */
1809 if (!(eop_desc
->wb
.status
& cpu_to_le32(IGC_TXD_STAT_DD
)))
1812 /* clear next_to_watch to prevent false hangs */
1813 tx_buffer
->next_to_watch
= NULL
;
1815 /* update the statistics for this packet */
1816 total_bytes
+= tx_buffer
->bytecount
;
1817 total_packets
+= tx_buffer
->gso_segs
;
1820 napi_consume_skb(tx_buffer
->skb
, napi_budget
);
1822 /* unmap skb header data */
1823 dma_unmap_single(tx_ring
->dev
,
1824 dma_unmap_addr(tx_buffer
, dma
),
1825 dma_unmap_len(tx_buffer
, len
),
1828 /* clear tx_buffer data */
1829 dma_unmap_len_set(tx_buffer
, len
, 0);
1831 /* clear last DMA location and unmap remaining buffers */
1832 while (tx_desc
!= eop_desc
) {
1837 i
-= tx_ring
->count
;
1838 tx_buffer
= tx_ring
->tx_buffer_info
;
1839 tx_desc
= IGC_TX_DESC(tx_ring
, 0);
1842 /* unmap any remaining paged data */
1843 if (dma_unmap_len(tx_buffer
, len
)) {
1844 dma_unmap_page(tx_ring
->dev
,
1845 dma_unmap_addr(tx_buffer
, dma
),
1846 dma_unmap_len(tx_buffer
, len
),
1848 dma_unmap_len_set(tx_buffer
, len
, 0);
1852 /* move us one more past the eop_desc for start of next pkt */
1857 i
-= tx_ring
->count
;
1858 tx_buffer
= tx_ring
->tx_buffer_info
;
1859 tx_desc
= IGC_TX_DESC(tx_ring
, 0);
1862 /* issue prefetch for next Tx descriptor */
1865 /* update budget accounting */
1867 } while (likely(budget
));
1869 netdev_tx_completed_queue(txring_txq(tx_ring
),
1870 total_packets
, total_bytes
);
1872 i
+= tx_ring
->count
;
1873 tx_ring
->next_to_clean
= i
;
1874 u64_stats_update_begin(&tx_ring
->tx_syncp
);
1875 tx_ring
->tx_stats
.bytes
+= total_bytes
;
1876 tx_ring
->tx_stats
.packets
+= total_packets
;
1877 u64_stats_update_end(&tx_ring
->tx_syncp
);
1878 q_vector
->tx
.total_bytes
+= total_bytes
;
1879 q_vector
->tx
.total_packets
+= total_packets
;
1881 if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG
, &tx_ring
->flags
)) {
1882 struct igc_hw
*hw
= &adapter
->hw
;
1884 /* Detect a transmit hang in hardware, this serializes the
1885 * check with the clearing of time_stamp and movement of i
1887 clear_bit(IGC_RING_FLAG_TX_DETECT_HANG
, &tx_ring
->flags
);
1888 if (tx_buffer
->next_to_watch
&&
1889 time_after(jiffies
, tx_buffer
->time_stamp
+
1890 (adapter
->tx_timeout_factor
* HZ
)) &&
1891 !(rd32(IGC_STATUS
) & IGC_STATUS_TXOFF
)) {
1892 /* detected Tx unit hang */
1893 dev_err(tx_ring
->dev
,
1894 "Detected Tx Unit Hang\n"
1898 " next_to_use <%x>\n"
1899 " next_to_clean <%x>\n"
1900 "buffer_info[next_to_clean]\n"
1901 " time_stamp <%lx>\n"
1902 " next_to_watch <%p>\n"
1904 " desc.status <%x>\n",
1905 tx_ring
->queue_index
,
1906 rd32(IGC_TDH(tx_ring
->reg_idx
)),
1907 readl(tx_ring
->tail
),
1908 tx_ring
->next_to_use
,
1909 tx_ring
->next_to_clean
,
1910 tx_buffer
->time_stamp
,
1911 tx_buffer
->next_to_watch
,
1913 tx_buffer
->next_to_watch
->wb
.status
);
1914 netif_stop_subqueue(tx_ring
->netdev
,
1915 tx_ring
->queue_index
);
1917 /* we are about to reset, no point in enabling stuff */
1922 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
1923 if (unlikely(total_packets
&&
1924 netif_carrier_ok(tx_ring
->netdev
) &&
1925 igc_desc_unused(tx_ring
) >= TX_WAKE_THRESHOLD
)) {
1926 /* Make sure that anybody stopping the queue after this
1927 * sees the new next_to_clean.
1930 if (__netif_subqueue_stopped(tx_ring
->netdev
,
1931 tx_ring
->queue_index
) &&
1932 !(test_bit(__IGC_DOWN
, &adapter
->state
))) {
1933 netif_wake_subqueue(tx_ring
->netdev
,
1934 tx_ring
->queue_index
);
1936 u64_stats_update_begin(&tx_ring
->tx_syncp
);
1937 tx_ring
->tx_stats
.restart_queue
++;
1938 u64_stats_update_end(&tx_ring
->tx_syncp
);
1946 * igc_up - Open the interface and prepare it to handle traffic
1947 * @adapter: board private structure
1949 void igc_up(struct igc_adapter
*adapter
)
1951 struct igc_hw
*hw
= &adapter
->hw
;
1954 /* hardware has been reset, we need to reload some things */
1955 igc_configure(adapter
);
1957 clear_bit(__IGC_DOWN
, &adapter
->state
);
1959 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
1960 napi_enable(&adapter
->q_vector
[i
]->napi
);
1962 if (adapter
->msix_entries
)
1963 igc_configure_msix(adapter
);
1965 igc_assign_vector(adapter
->q_vector
[0], 0);
1967 /* Clear any pending interrupts. */
1969 igc_irq_enable(adapter
);
1971 netif_tx_start_all_queues(adapter
->netdev
);
1973 /* start the watchdog. */
1974 hw
->mac
.get_link_status
= 1;
1975 schedule_work(&adapter
->watchdog_task
);
1979 * igc_update_stats - Update the board statistics counters
1980 * @adapter: board private structure
1982 void igc_update_stats(struct igc_adapter
*adapter
)
1984 struct rtnl_link_stats64
*net_stats
= &adapter
->stats64
;
1985 struct pci_dev
*pdev
= adapter
->pdev
;
1986 struct igc_hw
*hw
= &adapter
->hw
;
1987 u64 _bytes
, _packets
;
1993 /* Prevent stats update while adapter is being reset, or if the pci
1994 * connection is down.
1996 if (adapter
->link_speed
== 0)
1998 if (pci_channel_offline(pdev
))
2005 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
2006 struct igc_ring
*ring
= adapter
->rx_ring
[i
];
2007 u32 rqdpc
= rd32(IGC_RQDPC(i
));
2009 if (hw
->mac
.type
>= igc_i225
)
2010 wr32(IGC_RQDPC(i
), 0);
2013 ring
->rx_stats
.drops
+= rqdpc
;
2014 net_stats
->rx_fifo_errors
+= rqdpc
;
2018 start
= u64_stats_fetch_begin_irq(&ring
->rx_syncp
);
2019 _bytes
= ring
->rx_stats
.bytes
;
2020 _packets
= ring
->rx_stats
.packets
;
2021 } while (u64_stats_fetch_retry_irq(&ring
->rx_syncp
, start
));
2023 packets
+= _packets
;
2026 net_stats
->rx_bytes
= bytes
;
2027 net_stats
->rx_packets
= packets
;
2031 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
2032 struct igc_ring
*ring
= adapter
->tx_ring
[i
];
2035 start
= u64_stats_fetch_begin_irq(&ring
->tx_syncp
);
2036 _bytes
= ring
->tx_stats
.bytes
;
2037 _packets
= ring
->tx_stats
.packets
;
2038 } while (u64_stats_fetch_retry_irq(&ring
->tx_syncp
, start
));
2040 packets
+= _packets
;
2042 net_stats
->tx_bytes
= bytes
;
2043 net_stats
->tx_packets
= packets
;
2046 /* read stats registers */
2047 adapter
->stats
.crcerrs
+= rd32(IGC_CRCERRS
);
2048 adapter
->stats
.gprc
+= rd32(IGC_GPRC
);
2049 adapter
->stats
.gorc
+= rd32(IGC_GORCL
);
2050 rd32(IGC_GORCH
); /* clear GORCL */
2051 adapter
->stats
.bprc
+= rd32(IGC_BPRC
);
2052 adapter
->stats
.mprc
+= rd32(IGC_MPRC
);
2053 adapter
->stats
.roc
+= rd32(IGC_ROC
);
2055 adapter
->stats
.prc64
+= rd32(IGC_PRC64
);
2056 adapter
->stats
.prc127
+= rd32(IGC_PRC127
);
2057 adapter
->stats
.prc255
+= rd32(IGC_PRC255
);
2058 adapter
->stats
.prc511
+= rd32(IGC_PRC511
);
2059 adapter
->stats
.prc1023
+= rd32(IGC_PRC1023
);
2060 adapter
->stats
.prc1522
+= rd32(IGC_PRC1522
);
2061 adapter
->stats
.symerrs
+= rd32(IGC_SYMERRS
);
2062 adapter
->stats
.sec
+= rd32(IGC_SEC
);
2064 mpc
= rd32(IGC_MPC
);
2065 adapter
->stats
.mpc
+= mpc
;
2066 net_stats
->rx_fifo_errors
+= mpc
;
2067 adapter
->stats
.scc
+= rd32(IGC_SCC
);
2068 adapter
->stats
.ecol
+= rd32(IGC_ECOL
);
2069 adapter
->stats
.mcc
+= rd32(IGC_MCC
);
2070 adapter
->stats
.latecol
+= rd32(IGC_LATECOL
);
2071 adapter
->stats
.dc
+= rd32(IGC_DC
);
2072 adapter
->stats
.rlec
+= rd32(IGC_RLEC
);
2073 adapter
->stats
.xonrxc
+= rd32(IGC_XONRXC
);
2074 adapter
->stats
.xontxc
+= rd32(IGC_XONTXC
);
2075 adapter
->stats
.xoffrxc
+= rd32(IGC_XOFFRXC
);
2076 adapter
->stats
.xofftxc
+= rd32(IGC_XOFFTXC
);
2077 adapter
->stats
.fcruc
+= rd32(IGC_FCRUC
);
2078 adapter
->stats
.gptc
+= rd32(IGC_GPTC
);
2079 adapter
->stats
.gotc
+= rd32(IGC_GOTCL
);
2080 rd32(IGC_GOTCH
); /* clear GOTCL */
2081 adapter
->stats
.rnbc
+= rd32(IGC_RNBC
);
2082 adapter
->stats
.ruc
+= rd32(IGC_RUC
);
2083 adapter
->stats
.rfc
+= rd32(IGC_RFC
);
2084 adapter
->stats
.rjc
+= rd32(IGC_RJC
);
2085 adapter
->stats
.tor
+= rd32(IGC_TORH
);
2086 adapter
->stats
.tot
+= rd32(IGC_TOTH
);
2087 adapter
->stats
.tpr
+= rd32(IGC_TPR
);
2089 adapter
->stats
.ptc64
+= rd32(IGC_PTC64
);
2090 adapter
->stats
.ptc127
+= rd32(IGC_PTC127
);
2091 adapter
->stats
.ptc255
+= rd32(IGC_PTC255
);
2092 adapter
->stats
.ptc511
+= rd32(IGC_PTC511
);
2093 adapter
->stats
.ptc1023
+= rd32(IGC_PTC1023
);
2094 adapter
->stats
.ptc1522
+= rd32(IGC_PTC1522
);
2096 adapter
->stats
.mptc
+= rd32(IGC_MPTC
);
2097 adapter
->stats
.bptc
+= rd32(IGC_BPTC
);
2099 adapter
->stats
.tpt
+= rd32(IGC_TPT
);
2100 adapter
->stats
.colc
+= rd32(IGC_COLC
);
2102 adapter
->stats
.algnerrc
+= rd32(IGC_ALGNERRC
);
2104 adapter
->stats
.tsctc
+= rd32(IGC_TSCTC
);
2105 adapter
->stats
.tsctfc
+= rd32(IGC_TSCTFC
);
2107 adapter
->stats
.iac
+= rd32(IGC_IAC
);
2108 adapter
->stats
.icrxoc
+= rd32(IGC_ICRXOC
);
2109 adapter
->stats
.icrxptc
+= rd32(IGC_ICRXPTC
);
2110 adapter
->stats
.icrxatc
+= rd32(IGC_ICRXATC
);
2111 adapter
->stats
.ictxptc
+= rd32(IGC_ICTXPTC
);
2112 adapter
->stats
.ictxatc
+= rd32(IGC_ICTXATC
);
2113 adapter
->stats
.ictxqec
+= rd32(IGC_ICTXQEC
);
2114 adapter
->stats
.ictxqmtc
+= rd32(IGC_ICTXQMTC
);
2115 adapter
->stats
.icrxdmtc
+= rd32(IGC_ICRXDMTC
);
2117 /* Fill out the OS statistics structure */
2118 net_stats
->multicast
= adapter
->stats
.mprc
;
2119 net_stats
->collisions
= adapter
->stats
.colc
;
2123 /* RLEC on some newer hardware can be incorrect so build
2124 * our own version based on RUC and ROC
2126 net_stats
->rx_errors
= adapter
->stats
.rxerrc
+
2127 adapter
->stats
.crcerrs
+ adapter
->stats
.algnerrc
+
2128 adapter
->stats
.ruc
+ adapter
->stats
.roc
+
2129 adapter
->stats
.cexterr
;
2130 net_stats
->rx_length_errors
= adapter
->stats
.ruc
+
2132 net_stats
->rx_crc_errors
= adapter
->stats
.crcerrs
;
2133 net_stats
->rx_frame_errors
= adapter
->stats
.algnerrc
;
2134 net_stats
->rx_missed_errors
= adapter
->stats
.mpc
;
2137 net_stats
->tx_errors
= adapter
->stats
.ecol
+
2138 adapter
->stats
.latecol
;
2139 net_stats
->tx_aborted_errors
= adapter
->stats
.ecol
;
2140 net_stats
->tx_window_errors
= adapter
->stats
.latecol
;
2141 net_stats
->tx_carrier_errors
= adapter
->stats
.tncrs
;
2143 /* Tx Dropped needs to be maintained elsewhere */
2145 /* Management Stats */
2146 adapter
->stats
.mgptc
+= rd32(IGC_MGTPTC
);
2147 adapter
->stats
.mgprc
+= rd32(IGC_MGTPRC
);
2148 adapter
->stats
.mgpdc
+= rd32(IGC_MGTPDC
);
2151 static void igc_nfc_filter_exit(struct igc_adapter
*adapter
)
2153 struct igc_nfc_filter
*rule
;
2155 spin_lock(&adapter
->nfc_lock
);
2157 hlist_for_each_entry(rule
, &adapter
->nfc_filter_list
, nfc_node
)
2158 igc_erase_filter(adapter
, rule
);
2160 hlist_for_each_entry(rule
, &adapter
->cls_flower_list
, nfc_node
)
2161 igc_erase_filter(adapter
, rule
);
2163 spin_unlock(&adapter
->nfc_lock
);
2166 static void igc_nfc_filter_restore(struct igc_adapter
*adapter
)
2168 struct igc_nfc_filter
*rule
;
2170 spin_lock(&adapter
->nfc_lock
);
2172 hlist_for_each_entry(rule
, &adapter
->nfc_filter_list
, nfc_node
)
2173 igc_add_filter(adapter
, rule
);
2175 spin_unlock(&adapter
->nfc_lock
);
2179 * igc_down - Close the interface
2180 * @adapter: board private structure
2182 void igc_down(struct igc_adapter
*adapter
)
2184 struct net_device
*netdev
= adapter
->netdev
;
2185 struct igc_hw
*hw
= &adapter
->hw
;
2189 set_bit(__IGC_DOWN
, &adapter
->state
);
2191 /* disable receives in the hardware */
2192 rctl
= rd32(IGC_RCTL
);
2193 wr32(IGC_RCTL
, rctl
& ~IGC_RCTL_EN
);
2194 /* flush and sleep below */
2196 igc_nfc_filter_exit(adapter
);
2198 /* set trans_start so we don't get spurious watchdogs during reset */
2199 netif_trans_update(netdev
);
2201 netif_carrier_off(netdev
);
2202 netif_tx_stop_all_queues(netdev
);
2204 /* disable transmits in the hardware */
2205 tctl
= rd32(IGC_TCTL
);
2206 tctl
&= ~IGC_TCTL_EN
;
2207 wr32(IGC_TCTL
, tctl
);
2208 /* flush both disables and wait for them to finish */
2210 usleep_range(10000, 20000);
2212 igc_irq_disable(adapter
);
2214 adapter
->flags
&= ~IGC_FLAG_NEED_LINK_UPDATE
;
2216 for (i
= 0; i
< adapter
->num_q_vectors
; i
++) {
2217 if (adapter
->q_vector
[i
]) {
2218 napi_synchronize(&adapter
->q_vector
[i
]->napi
);
2219 napi_disable(&adapter
->q_vector
[i
]->napi
);
2223 del_timer_sync(&adapter
->watchdog_timer
);
2224 del_timer_sync(&adapter
->phy_info_timer
);
2226 /* record the stats before reset*/
2227 spin_lock(&adapter
->stats64_lock
);
2228 igc_update_stats(adapter
);
2229 spin_unlock(&adapter
->stats64_lock
);
2231 adapter
->link_speed
= 0;
2232 adapter
->link_duplex
= 0;
2234 if (!pci_channel_offline(adapter
->pdev
))
2237 /* clear VLAN promisc flag so VFTA will be updated if necessary */
2238 adapter
->flags
&= ~IGC_FLAG_VLAN_PROMISC
;
2240 igc_clean_all_tx_rings(adapter
);
2241 igc_clean_all_rx_rings(adapter
);
2244 void igc_reinit_locked(struct igc_adapter
*adapter
)
2246 WARN_ON(in_interrupt());
2247 while (test_and_set_bit(__IGC_RESETTING
, &adapter
->state
))
2248 usleep_range(1000, 2000);
2251 clear_bit(__IGC_RESETTING
, &adapter
->state
);
2254 static void igc_reset_task(struct work_struct
*work
)
2256 struct igc_adapter
*adapter
;
2258 adapter
= container_of(work
, struct igc_adapter
, reset_task
);
2260 netdev_err(adapter
->netdev
, "Reset adapter\n");
2261 igc_reinit_locked(adapter
);
2265 * igc_change_mtu - Change the Maximum Transfer Unit
2266 * @netdev: network interface device structure
2267 * @new_mtu: new value for maximum frame size
2269 * Returns 0 on success, negative on failure
2271 static int igc_change_mtu(struct net_device
*netdev
, int new_mtu
)
2273 int max_frame
= new_mtu
+ ETH_HLEN
+ ETH_FCS_LEN
+ VLAN_HLEN
;
2274 struct igc_adapter
*adapter
= netdev_priv(netdev
);
2276 /* adjust max frame to be at least the size of a standard frame */
2277 if (max_frame
< (ETH_FRAME_LEN
+ ETH_FCS_LEN
))
2278 max_frame
= ETH_FRAME_LEN
+ ETH_FCS_LEN
;
2280 while (test_and_set_bit(__IGC_RESETTING
, &adapter
->state
))
2281 usleep_range(1000, 2000);
2283 /* igc_down has a dependency on max_frame_size */
2284 adapter
->max_frame_size
= max_frame
;
2286 if (netif_running(netdev
))
2289 netdev_dbg(netdev
, "changing MTU from %d to %d\n",
2290 netdev
->mtu
, new_mtu
);
2291 netdev
->mtu
= new_mtu
;
2293 if (netif_running(netdev
))
2298 clear_bit(__IGC_RESETTING
, &adapter
->state
);
2304 * igc_get_stats - Get System Network Statistics
2305 * @netdev: network interface device structure
2307 * Returns the address of the device statistics structure.
2308 * The statistics are updated here and also from the timer callback.
2310 static struct net_device_stats
*igc_get_stats(struct net_device
*netdev
)
2312 struct igc_adapter
*adapter
= netdev_priv(netdev
);
2314 if (!test_bit(__IGC_RESETTING
, &adapter
->state
))
2315 igc_update_stats(adapter
);
2317 /* only return the current stats */
2318 return &netdev
->stats
;
2321 static netdev_features_t
igc_fix_features(struct net_device
*netdev
,
2322 netdev_features_t features
)
2324 /* Since there is no support for separate Rx/Tx vlan accel
2325 * enable/disable make sure Tx flag is always in same state as Rx.
2327 if (features
& NETIF_F_HW_VLAN_CTAG_RX
)
2328 features
|= NETIF_F_HW_VLAN_CTAG_TX
;
2330 features
&= ~NETIF_F_HW_VLAN_CTAG_TX
;
2335 static int igc_set_features(struct net_device
*netdev
,
2336 netdev_features_t features
)
2338 netdev_features_t changed
= netdev
->features
^ features
;
2339 struct igc_adapter
*adapter
= netdev_priv(netdev
);
2341 /* Add VLAN support */
2342 if (!(changed
& (NETIF_F_RXALL
| NETIF_F_NTUPLE
)))
2345 if (!(features
& NETIF_F_NTUPLE
)) {
2346 struct hlist_node
*node2
;
2347 struct igc_nfc_filter
*rule
;
2349 spin_lock(&adapter
->nfc_lock
);
2350 hlist_for_each_entry_safe(rule
, node2
,
2351 &adapter
->nfc_filter_list
, nfc_node
) {
2352 igc_erase_filter(adapter
, rule
);
2353 hlist_del(&rule
->nfc_node
);
2356 spin_unlock(&adapter
->nfc_lock
);
2357 adapter
->nfc_filter_count
= 0;
2360 netdev
->features
= features
;
2362 if (netif_running(netdev
))
2363 igc_reinit_locked(adapter
);
2370 static netdev_features_t
2371 igc_features_check(struct sk_buff
*skb
, struct net_device
*dev
,
2372 netdev_features_t features
)
2374 unsigned int network_hdr_len
, mac_hdr_len
;
2376 /* Make certain the headers can be described by a context descriptor */
2377 mac_hdr_len
= skb_network_header(skb
) - skb
->data
;
2378 if (unlikely(mac_hdr_len
> IGC_MAX_MAC_HDR_LEN
))
2379 return features
& ~(NETIF_F_HW_CSUM
|
2381 NETIF_F_HW_VLAN_CTAG_TX
|
2385 network_hdr_len
= skb_checksum_start(skb
) - skb_network_header(skb
);
2386 if (unlikely(network_hdr_len
> IGC_MAX_NETWORK_HDR_LEN
))
2387 return features
& ~(NETIF_F_HW_CSUM
|
2392 /* We can only support IPv4 TSO in tunnels if we can mangle the
2393 * inner IP ID field, so strip TSO if MANGLEID is not supported.
2395 if (skb
->encapsulation
&& !(features
& NETIF_F_TSO_MANGLEID
))
2396 features
&= ~NETIF_F_TSO
;
2402 * igc_configure - configure the hardware for RX and TX
2403 * @adapter: private board structure
2405 static void igc_configure(struct igc_adapter
*adapter
)
2407 struct net_device
*netdev
= adapter
->netdev
;
2410 igc_get_hw_control(adapter
);
2411 igc_set_rx_mode(netdev
);
2413 igc_setup_tctl(adapter
);
2414 igc_setup_mrqc(adapter
);
2415 igc_setup_rctl(adapter
);
2417 igc_nfc_filter_restore(adapter
);
2418 igc_configure_tx(adapter
);
2419 igc_configure_rx(adapter
);
2421 igc_rx_fifo_flush_base(&adapter
->hw
);
2423 /* call igc_desc_unused which always leaves
2424 * at least 1 descriptor unused to make sure
2425 * next_to_use != next_to_clean
2427 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
2428 struct igc_ring
*ring
= adapter
->rx_ring
[i
];
2430 igc_alloc_rx_buffers(ring
, igc_desc_unused(ring
));
2435 * igc_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table
2436 * @adapter: address of board private structure
2437 * @index: Index of the RAR entry which need to be synced with MAC table
2439 static void igc_rar_set_index(struct igc_adapter
*adapter
, u32 index
)
2441 u8
*addr
= adapter
->mac_table
[index
].addr
;
2442 struct igc_hw
*hw
= &adapter
->hw
;
2443 u32 rar_low
, rar_high
;
2445 /* HW expects these to be in network order when they are plugged
2446 * into the registers which are little endian. In order to guarantee
2447 * that ordering we need to do an leXX_to_cpup here in order to be
2448 * ready for the byteswap that occurs with writel
2450 rar_low
= le32_to_cpup((__le32
*)(addr
));
2451 rar_high
= le16_to_cpup((__le16
*)(addr
+ 4));
2453 /* Indicate to hardware the Address is Valid. */
2454 if (adapter
->mac_table
[index
].state
& IGC_MAC_STATE_IN_USE
) {
2455 if (is_valid_ether_addr(addr
))
2456 rar_high
|= IGC_RAH_AV
;
2458 rar_high
|= IGC_RAH_POOL_1
<<
2459 adapter
->mac_table
[index
].queue
;
2462 wr32(IGC_RAL(index
), rar_low
);
2464 wr32(IGC_RAH(index
), rar_high
);
2468 /* Set default MAC address for the PF in the first RAR entry */
2469 static void igc_set_default_mac_filter(struct igc_adapter
*adapter
)
2471 struct igc_mac_addr
*mac_table
= &adapter
->mac_table
[0];
2473 ether_addr_copy(mac_table
->addr
, adapter
->hw
.mac
.addr
);
2474 mac_table
->state
= IGC_MAC_STATE_DEFAULT
| IGC_MAC_STATE_IN_USE
;
2476 igc_rar_set_index(adapter
, 0);
2479 /* If the filter to be added and an already existing filter express
2480 * the same address and address type, it should be possible to only
2481 * override the other configurations, for example the queue to steer
2484 static bool igc_mac_entry_can_be_used(const struct igc_mac_addr
*entry
,
2485 const u8
*addr
, const u8 flags
)
2487 if (!(entry
->state
& IGC_MAC_STATE_IN_USE
))
2490 if ((entry
->state
& IGC_MAC_STATE_SRC_ADDR
) !=
2491 (flags
& IGC_MAC_STATE_SRC_ADDR
))
2494 if (!ether_addr_equal(addr
, entry
->addr
))
2500 /* Add a MAC filter for 'addr' directing matching traffic to 'queue',
2501 * 'flags' is used to indicate what kind of match is made, match is by
2502 * default for the destination address, if matching by source address
2503 * is desired the flag IGC_MAC_STATE_SRC_ADDR can be used.
2505 static int igc_add_mac_filter_flags(struct igc_adapter
*adapter
,
2506 const u8
*addr
, const u8 queue
,
2509 struct igc_hw
*hw
= &adapter
->hw
;
2510 int rar_entries
= hw
->mac
.rar_entry_count
;
2513 if (is_zero_ether_addr(addr
))
2516 /* Search for the first empty entry in the MAC table.
2517 * Do not touch entries at the end of the table reserved for the VF MAC
2520 for (i
= 0; i
< rar_entries
; i
++) {
2521 if (!igc_mac_entry_can_be_used(&adapter
->mac_table
[i
],
2525 ether_addr_copy(adapter
->mac_table
[i
].addr
, addr
);
2526 adapter
->mac_table
[i
].queue
= queue
;
2527 adapter
->mac_table
[i
].state
|= IGC_MAC_STATE_IN_USE
| flags
;
2529 igc_rar_set_index(adapter
, i
);
2536 int igc_add_mac_steering_filter(struct igc_adapter
*adapter
,
2537 const u8
*addr
, u8 queue
, u8 flags
)
2539 return igc_add_mac_filter_flags(adapter
, addr
, queue
,
2540 IGC_MAC_STATE_QUEUE_STEERING
| flags
);
2543 /* Remove a MAC filter for 'addr' directing matching traffic to
2544 * 'queue', 'flags' is used to indicate what kind of match need to be
2545 * removed, match is by default for the destination address, if
2546 * matching by source address is to be removed the flag
2547 * IGC_MAC_STATE_SRC_ADDR can be used.
2549 static int igc_del_mac_filter_flags(struct igc_adapter
*adapter
,
2550 const u8
*addr
, const u8 queue
,
2553 struct igc_hw
*hw
= &adapter
->hw
;
2554 int rar_entries
= hw
->mac
.rar_entry_count
;
2557 if (is_zero_ether_addr(addr
))
2560 /* Search for matching entry in the MAC table based on given address
2561 * and queue. Do not touch entries at the end of the table reserved
2562 * for the VF MAC addresses.
2564 for (i
= 0; i
< rar_entries
; i
++) {
2565 if (!(adapter
->mac_table
[i
].state
& IGC_MAC_STATE_IN_USE
))
2567 if ((adapter
->mac_table
[i
].state
& flags
) != flags
)
2569 if (adapter
->mac_table
[i
].queue
!= queue
)
2571 if (!ether_addr_equal(adapter
->mac_table
[i
].addr
, addr
))
2574 /* When a filter for the default address is "deleted",
2575 * we return it to its initial configuration
2577 if (adapter
->mac_table
[i
].state
& IGC_MAC_STATE_DEFAULT
) {
2578 adapter
->mac_table
[i
].state
=
2579 IGC_MAC_STATE_DEFAULT
| IGC_MAC_STATE_IN_USE
;
2581 adapter
->mac_table
[i
].state
= 0;
2582 adapter
->mac_table
[i
].queue
= 0;
2583 memset(adapter
->mac_table
[i
].addr
, 0, ETH_ALEN
);
2586 igc_rar_set_index(adapter
, i
);
2593 int igc_del_mac_steering_filter(struct igc_adapter
*adapter
,
2594 const u8
*addr
, u8 queue
, u8 flags
)
2596 return igc_del_mac_filter_flags(adapter
, addr
, queue
,
2597 IGC_MAC_STATE_QUEUE_STEERING
| flags
);
2600 /* Add a MAC filter for 'addr' directing matching traffic to 'queue',
2601 * 'flags' is used to indicate what kind of match is made, match is by
2602 * default for the destination address, if matching by source address
2603 * is desired the flag IGC_MAC_STATE_SRC_ADDR can be used.
2605 static int igc_add_mac_filter(struct igc_adapter
*adapter
,
2606 const u8
*addr
, const u8 queue
)
2608 struct igc_hw
*hw
= &adapter
->hw
;
2609 int rar_entries
= hw
->mac
.rar_entry_count
;
2612 if (is_zero_ether_addr(addr
))
2615 /* Search for the first empty entry in the MAC table.
2616 * Do not touch entries at the end of the table reserved for the VF MAC
2619 for (i
= 0; i
< rar_entries
; i
++) {
2620 if (!igc_mac_entry_can_be_used(&adapter
->mac_table
[i
],
2624 ether_addr_copy(adapter
->mac_table
[i
].addr
, addr
);
2625 adapter
->mac_table
[i
].queue
= queue
;
2626 adapter
->mac_table
[i
].state
|= IGC_MAC_STATE_IN_USE
;
2628 igc_rar_set_index(adapter
, i
);
2635 /* Remove a MAC filter for 'addr' directing matching traffic to
2636 * 'queue', 'flags' is used to indicate what kind of match need to be
2637 * removed, match is by default for the destination address, if
2638 * matching by source address is to be removed the flag
2639 * IGC_MAC_STATE_SRC_ADDR can be used.
2641 static int igc_del_mac_filter(struct igc_adapter
*adapter
,
2642 const u8
*addr
, const u8 queue
)
2644 struct igc_hw
*hw
= &adapter
->hw
;
2645 int rar_entries
= hw
->mac
.rar_entry_count
;
2648 if (is_zero_ether_addr(addr
))
2651 /* Search for matching entry in the MAC table based on given address
2652 * and queue. Do not touch entries at the end of the table reserved
2653 * for the VF MAC addresses.
2655 for (i
= 0; i
< rar_entries
; i
++) {
2656 if (!(adapter
->mac_table
[i
].state
& IGC_MAC_STATE_IN_USE
))
2658 if (adapter
->mac_table
[i
].state
!= 0)
2660 if (adapter
->mac_table
[i
].queue
!= queue
)
2662 if (!ether_addr_equal(adapter
->mac_table
[i
].addr
, addr
))
2665 /* When a filter for the default address is "deleted",
2666 * we return it to its initial configuration
2668 if (adapter
->mac_table
[i
].state
& IGC_MAC_STATE_DEFAULT
) {
2669 adapter
->mac_table
[i
].state
=
2670 IGC_MAC_STATE_DEFAULT
| IGC_MAC_STATE_IN_USE
;
2671 adapter
->mac_table
[i
].queue
= 0;
2673 adapter
->mac_table
[i
].state
= 0;
2674 adapter
->mac_table
[i
].queue
= 0;
2675 memset(adapter
->mac_table
[i
].addr
, 0, ETH_ALEN
);
2678 igc_rar_set_index(adapter
, i
);
2685 static int igc_uc_sync(struct net_device
*netdev
, const unsigned char *addr
)
2687 struct igc_adapter
*adapter
= netdev_priv(netdev
);
2690 ret
= igc_add_mac_filter(adapter
, addr
, adapter
->num_rx_queues
);
2692 return min_t(int, ret
, 0);
2695 static int igc_uc_unsync(struct net_device
*netdev
, const unsigned char *addr
)
2697 struct igc_adapter
*adapter
= netdev_priv(netdev
);
2699 igc_del_mac_filter(adapter
, addr
, adapter
->num_rx_queues
);
2705 * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2706 * @netdev: network interface device structure
2708 * The set_rx_mode entry point is called whenever the unicast or multicast
2709 * address lists or the network interface flags are updated. This routine is
2710 * responsible for configuring the hardware for proper unicast, multicast,
2711 * promiscuous mode, and all-multi behavior.
2713 static void igc_set_rx_mode(struct net_device
*netdev
)
2715 struct igc_adapter
*adapter
= netdev_priv(netdev
);
2716 struct igc_hw
*hw
= &adapter
->hw
;
2717 u32 rctl
= 0, rlpml
= MAX_JUMBO_FRAME_SIZE
;
2720 /* Check for Promiscuous and All Multicast modes */
2721 if (netdev
->flags
& IFF_PROMISC
) {
2722 rctl
|= IGC_RCTL_UPE
| IGC_RCTL_MPE
;
2724 if (netdev
->flags
& IFF_ALLMULTI
) {
2725 rctl
|= IGC_RCTL_MPE
;
2727 /* Write addresses to the MTA, if the attempt fails
2728 * then we should just turn on promiscuous mode so
2729 * that we can at least receive multicast traffic
2731 count
= igc_write_mc_addr_list(netdev
);
2733 rctl
|= IGC_RCTL_MPE
;
2737 /* Write addresses to available RAR registers, if there is not
2738 * sufficient space to store all the addresses then enable
2739 * unicast promiscuous mode
2741 if (__dev_uc_sync(netdev
, igc_uc_sync
, igc_uc_unsync
))
2742 rctl
|= IGC_RCTL_UPE
;
2744 /* update state of unicast and multicast */
2745 rctl
|= rd32(IGC_RCTL
) & ~(IGC_RCTL_UPE
| IGC_RCTL_MPE
);
2746 wr32(IGC_RCTL
, rctl
);
2748 #if (PAGE_SIZE < 8192)
2749 if (adapter
->max_frame_size
<= IGC_MAX_FRAME_BUILD_SKB
)
2750 rlpml
= IGC_MAX_FRAME_BUILD_SKB
;
2752 wr32(IGC_RLPML
, rlpml
);
2756 * igc_msix_other - msix other interrupt handler
2757 * @irq: interrupt number
2758 * @data: pointer to a q_vector
2760 static irqreturn_t
igc_msix_other(int irq
, void *data
)
2762 struct igc_adapter
*adapter
= data
;
2763 struct igc_hw
*hw
= &adapter
->hw
;
2764 u32 icr
= rd32(IGC_ICR
);
2766 /* reading ICR causes bit 31 of EICR to be cleared */
2767 if (icr
& IGC_ICR_DRSTA
)
2768 schedule_work(&adapter
->reset_task
);
2770 if (icr
& IGC_ICR_DOUTSYNC
) {
2771 /* HW is reporting DMA is out of sync */
2772 adapter
->stats
.doosync
++;
2775 if (icr
& IGC_ICR_LSC
) {
2776 hw
->mac
.get_link_status
= 1;
2777 /* guard against interrupt when we're going down */
2778 if (!test_bit(__IGC_DOWN
, &adapter
->state
))
2779 mod_timer(&adapter
->watchdog_timer
, jiffies
+ 1);
2782 wr32(IGC_EIMS
, adapter
->eims_other
);
2788 * igc_write_ivar - configure ivar for given MSI-X vector
2789 * @hw: pointer to the HW structure
2790 * @msix_vector: vector number we are allocating to a given ring
2791 * @index: row index of IVAR register to write within IVAR table
2792 * @offset: column offset of in IVAR, should be multiple of 8
2794 * The IVAR table consists of 2 columns,
2795 * each containing an cause allocation for an Rx and Tx ring, and a
2796 * variable number of rows depending on the number of queues supported.
2798 static void igc_write_ivar(struct igc_hw
*hw
, int msix_vector
,
2799 int index
, int offset
)
2801 u32 ivar
= array_rd32(IGC_IVAR0
, index
);
2803 /* clear any bits that are currently set */
2804 ivar
&= ~((u32
)0xFF << offset
);
2806 /* write vector and valid bit */
2807 ivar
|= (msix_vector
| IGC_IVAR_VALID
) << offset
;
2809 array_wr32(IGC_IVAR0
, index
, ivar
);
2812 static void igc_assign_vector(struct igc_q_vector
*q_vector
, int msix_vector
)
2814 struct igc_adapter
*adapter
= q_vector
->adapter
;
2815 struct igc_hw
*hw
= &adapter
->hw
;
2816 int rx_queue
= IGC_N0_QUEUE
;
2817 int tx_queue
= IGC_N0_QUEUE
;
2819 if (q_vector
->rx
.ring
)
2820 rx_queue
= q_vector
->rx
.ring
->reg_idx
;
2821 if (q_vector
->tx
.ring
)
2822 tx_queue
= q_vector
->tx
.ring
->reg_idx
;
2824 switch (hw
->mac
.type
) {
2826 if (rx_queue
> IGC_N0_QUEUE
)
2827 igc_write_ivar(hw
, msix_vector
,
2829 (rx_queue
& 0x1) << 4);
2830 if (tx_queue
> IGC_N0_QUEUE
)
2831 igc_write_ivar(hw
, msix_vector
,
2833 ((tx_queue
& 0x1) << 4) + 8);
2834 q_vector
->eims_value
= BIT(msix_vector
);
2837 WARN_ONCE(hw
->mac
.type
!= igc_i225
, "Wrong MAC type\n");
2841 /* add q_vector eims value to global eims_enable_mask */
2842 adapter
->eims_enable_mask
|= q_vector
->eims_value
;
2844 /* configure q_vector to set itr on first interrupt */
2845 q_vector
->set_itr
= 1;
2849 * igc_configure_msix - Configure MSI-X hardware
2850 * @adapter: Pointer to adapter structure
2852 * igc_configure_msix sets up the hardware to properly
2853 * generate MSI-X interrupts.
2855 static void igc_configure_msix(struct igc_adapter
*adapter
)
2857 struct igc_hw
*hw
= &adapter
->hw
;
2861 adapter
->eims_enable_mask
= 0;
2863 /* set vector for other causes, i.e. link changes */
2864 switch (hw
->mac
.type
) {
2866 /* Turn on MSI-X capability first, or our settings
2867 * won't stick. And it will take days to debug.
2869 wr32(IGC_GPIE
, IGC_GPIE_MSIX_MODE
|
2870 IGC_GPIE_PBA
| IGC_GPIE_EIAME
|
2873 /* enable msix_other interrupt */
2874 adapter
->eims_other
= BIT(vector
);
2875 tmp
= (vector
++ | IGC_IVAR_VALID
) << 8;
2877 wr32(IGC_IVAR_MISC
, tmp
);
2880 /* do nothing, since nothing else supports MSI-X */
2882 } /* switch (hw->mac.type) */
2884 adapter
->eims_enable_mask
|= adapter
->eims_other
;
2886 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
2887 igc_assign_vector(adapter
->q_vector
[i
], vector
++);
2892 static irqreturn_t
igc_msix_ring(int irq
, void *data
)
2894 struct igc_q_vector
*q_vector
= data
;
2896 /* Write the ITR value calculated from the previous interrupt. */
2897 igc_write_itr(q_vector
);
2899 napi_schedule(&q_vector
->napi
);
2905 * igc_request_msix - Initialize MSI-X interrupts
2906 * @adapter: Pointer to adapter structure
2908 * igc_request_msix allocates MSI-X vectors and requests interrupts from the
2911 static int igc_request_msix(struct igc_adapter
*adapter
)
2913 int i
= 0, err
= 0, vector
= 0, free_vector
= 0;
2914 struct net_device
*netdev
= adapter
->netdev
;
2916 err
= request_irq(adapter
->msix_entries
[vector
].vector
,
2917 &igc_msix_other
, 0, netdev
->name
, adapter
);
2921 for (i
= 0; i
< adapter
->num_q_vectors
; i
++) {
2922 struct igc_q_vector
*q_vector
= adapter
->q_vector
[i
];
2926 q_vector
->itr_register
= adapter
->io_addr
+ IGC_EITR(vector
);
2928 if (q_vector
->rx
.ring
&& q_vector
->tx
.ring
)
2929 sprintf(q_vector
->name
, "%s-TxRx-%u", netdev
->name
,
2930 q_vector
->rx
.ring
->queue_index
);
2931 else if (q_vector
->tx
.ring
)
2932 sprintf(q_vector
->name
, "%s-tx-%u", netdev
->name
,
2933 q_vector
->tx
.ring
->queue_index
);
2934 else if (q_vector
->rx
.ring
)
2935 sprintf(q_vector
->name
, "%s-rx-%u", netdev
->name
,
2936 q_vector
->rx
.ring
->queue_index
);
2938 sprintf(q_vector
->name
, "%s-unused", netdev
->name
);
2940 err
= request_irq(adapter
->msix_entries
[vector
].vector
,
2941 igc_msix_ring
, 0, q_vector
->name
,
2947 igc_configure_msix(adapter
);
2951 /* free already assigned IRQs */
2952 free_irq(adapter
->msix_entries
[free_vector
++].vector
, adapter
);
2955 for (i
= 0; i
< vector
; i
++) {
2956 free_irq(adapter
->msix_entries
[free_vector
++].vector
,
2957 adapter
->q_vector
[i
]);
2964 * igc_reset_q_vector - Reset config for interrupt vector
2965 * @adapter: board private structure to initialize
2966 * @v_idx: Index of vector to be reset
2968 * If NAPI is enabled it will delete any references to the
2969 * NAPI struct. This is preparation for igc_free_q_vector.
2971 static void igc_reset_q_vector(struct igc_adapter
*adapter
, int v_idx
)
2973 struct igc_q_vector
*q_vector
= adapter
->q_vector
[v_idx
];
2975 /* if we're coming from igc_set_interrupt_capability, the vectors are
2981 if (q_vector
->tx
.ring
)
2982 adapter
->tx_ring
[q_vector
->tx
.ring
->queue_index
] = NULL
;
2984 if (q_vector
->rx
.ring
)
2985 adapter
->rx_ring
[q_vector
->rx
.ring
->queue_index
] = NULL
;
2987 netif_napi_del(&q_vector
->napi
);
2990 static void igc_reset_interrupt_capability(struct igc_adapter
*adapter
)
2992 int v_idx
= adapter
->num_q_vectors
;
2994 if (adapter
->msix_entries
) {
2995 pci_disable_msix(adapter
->pdev
);
2996 kfree(adapter
->msix_entries
);
2997 adapter
->msix_entries
= NULL
;
2998 } else if (adapter
->flags
& IGC_FLAG_HAS_MSI
) {
2999 pci_disable_msi(adapter
->pdev
);
3003 igc_reset_q_vector(adapter
, v_idx
);
3007 * igc_clear_interrupt_scheme - reset the device to a state of no interrupts
3008 * @adapter: Pointer to adapter structure
3010 * This function resets the device so that it has 0 rx queues, tx queues, and
3011 * MSI-X interrupts allocated.
3013 static void igc_clear_interrupt_scheme(struct igc_adapter
*adapter
)
3015 igc_free_q_vectors(adapter
);
3016 igc_reset_interrupt_capability(adapter
);
3020 * igc_free_q_vectors - Free memory allocated for interrupt vectors
3021 * @adapter: board private structure to initialize
3023 * This function frees the memory allocated to the q_vectors. In addition if
3024 * NAPI is enabled it will delete any references to the NAPI struct prior
3025 * to freeing the q_vector.
3027 static void igc_free_q_vectors(struct igc_adapter
*adapter
)
3029 int v_idx
= adapter
->num_q_vectors
;
3031 adapter
->num_tx_queues
= 0;
3032 adapter
->num_rx_queues
= 0;
3033 adapter
->num_q_vectors
= 0;
3036 igc_reset_q_vector(adapter
, v_idx
);
3037 igc_free_q_vector(adapter
, v_idx
);
3042 * igc_free_q_vector - Free memory allocated for specific interrupt vector
3043 * @adapter: board private structure to initialize
3044 * @v_idx: Index of vector to be freed
3046 * This function frees the memory allocated to the q_vector.
3048 static void igc_free_q_vector(struct igc_adapter
*adapter
, int v_idx
)
3050 struct igc_q_vector
*q_vector
= adapter
->q_vector
[v_idx
];
3052 adapter
->q_vector
[v_idx
] = NULL
;
3054 /* igc_get_stats64() might access the rings on this vector,
3055 * we must wait a grace period before freeing it.
3058 kfree_rcu(q_vector
, rcu
);
3061 /* Need to wait a few seconds after link up to get diagnostic information from
3064 static void igc_update_phy_info(struct timer_list
*t
)
3066 struct igc_adapter
*adapter
= from_timer(adapter
, t
, phy_info_timer
);
3068 igc_get_phy_info(&adapter
->hw
);
3072 * igc_has_link - check shared code for link and determine up/down
3073 * @adapter: pointer to driver private info
3075 bool igc_has_link(struct igc_adapter
*adapter
)
3077 struct igc_hw
*hw
= &adapter
->hw
;
3078 bool link_active
= false;
3080 /* get_link_status is set on LSC (link status) interrupt or
3081 * rx sequence error interrupt. get_link_status will stay
3082 * false until the igc_check_for_link establishes link
3083 * for copper adapters ONLY
3085 switch (hw
->phy
.media_type
) {
3086 case igc_media_type_copper
:
3087 if (!hw
->mac
.get_link_status
)
3089 hw
->mac
.ops
.check_for_link(hw
);
3090 link_active
= !hw
->mac
.get_link_status
;
3093 case igc_media_type_unknown
:
3097 if (hw
->mac
.type
== igc_i225
&&
3098 hw
->phy
.id
== I225_I_PHY_ID
) {
3099 if (!netif_carrier_ok(adapter
->netdev
)) {
3100 adapter
->flags
&= ~IGC_FLAG_NEED_LINK_UPDATE
;
3101 } else if (!(adapter
->flags
& IGC_FLAG_NEED_LINK_UPDATE
)) {
3102 adapter
->flags
|= IGC_FLAG_NEED_LINK_UPDATE
;
3103 adapter
->link_check_timeout
= jiffies
;
3111 * igc_watchdog - Timer Call-back
3112 * @data: pointer to adapter cast into an unsigned long
3114 static void igc_watchdog(struct timer_list
*t
)
3116 struct igc_adapter
*adapter
= from_timer(adapter
, t
, watchdog_timer
);
3117 /* Do the rest outside of interrupt context */
3118 schedule_work(&adapter
->watchdog_task
);
3121 static void igc_watchdog_task(struct work_struct
*work
)
3123 struct igc_adapter
*adapter
= container_of(work
,
3126 struct net_device
*netdev
= adapter
->netdev
;
3127 struct igc_hw
*hw
= &adapter
->hw
;
3128 struct igc_phy_info
*phy
= &hw
->phy
;
3129 u16 phy_data
, retry_count
= 20;
3134 link
= igc_has_link(adapter
);
3136 if (adapter
->flags
& IGC_FLAG_NEED_LINK_UPDATE
) {
3137 if (time_after(jiffies
, (adapter
->link_check_timeout
+ HZ
)))
3138 adapter
->flags
&= ~IGC_FLAG_NEED_LINK_UPDATE
;
3143 /* Force link down if we have fiber to swap to */
3144 if (adapter
->flags
& IGC_FLAG_MAS_ENABLE
) {
3145 if (hw
->phy
.media_type
== igc_media_type_copper
) {
3146 connsw
= rd32(IGC_CONNSW
);
3147 if (!(connsw
& IGC_CONNSW_AUTOSENSE_EN
))
3152 if (!netif_carrier_ok(netdev
)) {
3155 hw
->mac
.ops
.get_speed_and_duplex(hw
,
3156 &adapter
->link_speed
,
3157 &adapter
->link_duplex
);
3159 ctrl
= rd32(IGC_CTRL
);
3160 /* Link status message must follow this format */
3162 "igc: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
3164 adapter
->link_speed
,
3165 adapter
->link_duplex
== FULL_DUPLEX
?
3167 (ctrl
& IGC_CTRL_TFCE
) &&
3168 (ctrl
& IGC_CTRL_RFCE
) ? "RX/TX" :
3169 (ctrl
& IGC_CTRL_RFCE
) ? "RX" :
3170 (ctrl
& IGC_CTRL_TFCE
) ? "TX" : "None");
3172 /* check if SmartSpeed worked */
3173 igc_check_downshift(hw
);
3174 if (phy
->speed_downgraded
)
3175 netdev_warn(netdev
, "Link Speed was downgraded by SmartSpeed\n");
3177 /* adjust timeout factor according to speed/duplex */
3178 adapter
->tx_timeout_factor
= 1;
3179 switch (adapter
->link_speed
) {
3181 adapter
->tx_timeout_factor
= 14;
3184 /* maybe add some timeout factor ? */
3188 if (adapter
->link_speed
!= SPEED_1000
)
3191 /* wait for Remote receiver status OK */
3193 if (!igc_read_phy_reg(hw
, PHY_1000T_STATUS
,
3195 if (!(phy_data
& SR_1000T_REMOTE_RX_STATUS
) &&
3199 goto retry_read_status
;
3200 } else if (!retry_count
) {
3201 dev_err(&adapter
->pdev
->dev
, "exceed max 2 second\n");
3204 dev_err(&adapter
->pdev
->dev
, "read 1000Base-T Status Reg\n");
3207 netif_carrier_on(netdev
);
3209 /* link state has changed, schedule phy info update */
3210 if (!test_bit(__IGC_DOWN
, &adapter
->state
))
3211 mod_timer(&adapter
->phy_info_timer
,
3212 round_jiffies(jiffies
+ 2 * HZ
));
3215 if (netif_carrier_ok(netdev
)) {
3216 adapter
->link_speed
= 0;
3217 adapter
->link_duplex
= 0;
3219 /* Links status message must follow this format */
3220 netdev_info(netdev
, "igc: %s NIC Link is Down\n",
3222 netif_carrier_off(netdev
);
3224 /* link state has changed, schedule phy info update */
3225 if (!test_bit(__IGC_DOWN
, &adapter
->state
))
3226 mod_timer(&adapter
->phy_info_timer
,
3227 round_jiffies(jiffies
+ 2 * HZ
));
3229 /* link is down, time to check for alternate media */
3230 if (adapter
->flags
& IGC_FLAG_MAS_ENABLE
) {
3231 if (adapter
->flags
& IGC_FLAG_MEDIA_RESET
) {
3232 schedule_work(&adapter
->reset_task
);
3233 /* return immediately */
3238 /* also check for alternate media here */
3239 } else if (!netif_carrier_ok(netdev
) &&
3240 (adapter
->flags
& IGC_FLAG_MAS_ENABLE
)) {
3241 if (adapter
->flags
& IGC_FLAG_MEDIA_RESET
) {
3242 schedule_work(&adapter
->reset_task
);
3243 /* return immediately */
3249 spin_lock(&adapter
->stats64_lock
);
3250 igc_update_stats(adapter
);
3251 spin_unlock(&adapter
->stats64_lock
);
3253 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
3254 struct igc_ring
*tx_ring
= adapter
->tx_ring
[i
];
3256 if (!netif_carrier_ok(netdev
)) {
3257 /* We've lost link, so the controller stops DMA,
3258 * but we've got queued Tx work that's never going
3259 * to get done, so reset controller to flush Tx.
3260 * (Do the reset outside of interrupt context).
3262 if (igc_desc_unused(tx_ring
) + 1 < tx_ring
->count
) {
3263 adapter
->tx_timeout_count
++;
3264 schedule_work(&adapter
->reset_task
);
3265 /* return immediately since reset is imminent */
3270 /* Force detection of hung controller every watchdog period */
3271 set_bit(IGC_RING_FLAG_TX_DETECT_HANG
, &tx_ring
->flags
);
3274 /* Cause software interrupt to ensure Rx ring is cleaned */
3275 if (adapter
->flags
& IGC_FLAG_HAS_MSIX
) {
3278 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
3279 eics
|= adapter
->q_vector
[i
]->eims_value
;
3280 wr32(IGC_EICS
, eics
);
3282 wr32(IGC_ICS
, IGC_ICS_RXDMT0
);
3285 /* Reset the timer */
3286 if (!test_bit(__IGC_DOWN
, &adapter
->state
)) {
3287 if (adapter
->flags
& IGC_FLAG_NEED_LINK_UPDATE
)
3288 mod_timer(&adapter
->watchdog_timer
,
3289 round_jiffies(jiffies
+ HZ
));
3291 mod_timer(&adapter
->watchdog_timer
,
3292 round_jiffies(jiffies
+ 2 * HZ
));
3297 * igc_update_ring_itr - update the dynamic ITR value based on packet size
3298 * @q_vector: pointer to q_vector
3300 * Stores a new ITR value based on strictly on packet size. This
3301 * algorithm is less sophisticated than that used in igc_update_itr,
3302 * due to the difficulty of synchronizing statistics across multiple
3303 * receive rings. The divisors and thresholds used by this function
3304 * were determined based on theoretical maximum wire speed and testing
3305 * data, in order to minimize response time while increasing bulk
3307 * NOTE: This function is called only when operating in a multiqueue
3308 * receive environment.
3310 static void igc_update_ring_itr(struct igc_q_vector
*q_vector
)
3312 struct igc_adapter
*adapter
= q_vector
->adapter
;
3313 int new_val
= q_vector
->itr_val
;
3314 int avg_wire_size
= 0;
3315 unsigned int packets
;
3317 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3318 * ints/sec - ITR timer value of 120 ticks.
3320 switch (adapter
->link_speed
) {
3323 new_val
= IGC_4K_ITR
;
3329 packets
= q_vector
->rx
.total_packets
;
3331 avg_wire_size
= q_vector
->rx
.total_bytes
/ packets
;
3333 packets
= q_vector
->tx
.total_packets
;
3335 avg_wire_size
= max_t(u32
, avg_wire_size
,
3336 q_vector
->tx
.total_bytes
/ packets
);
3338 /* if avg_wire_size isn't set no work was done */
3342 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3343 avg_wire_size
+= 24;
3345 /* Don't starve jumbo frames */
3346 avg_wire_size
= min(avg_wire_size
, 3000);
3348 /* Give a little boost to mid-size frames */
3349 if (avg_wire_size
> 300 && avg_wire_size
< 1200)
3350 new_val
= avg_wire_size
/ 3;
3352 new_val
= avg_wire_size
/ 2;
3354 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3355 if (new_val
< IGC_20K_ITR
&&
3356 ((q_vector
->rx
.ring
&& adapter
->rx_itr_setting
== 3) ||
3357 (!q_vector
->rx
.ring
&& adapter
->tx_itr_setting
== 3)))
3358 new_val
= IGC_20K_ITR
;
3361 if (new_val
!= q_vector
->itr_val
) {
3362 q_vector
->itr_val
= new_val
;
3363 q_vector
->set_itr
= 1;
3366 q_vector
->rx
.total_bytes
= 0;
3367 q_vector
->rx
.total_packets
= 0;
3368 q_vector
->tx
.total_bytes
= 0;
3369 q_vector
->tx
.total_packets
= 0;
3373 * igc_update_itr - update the dynamic ITR value based on statistics
3374 * @q_vector: pointer to q_vector
3375 * @ring_container: ring info to update the itr for
3377 * Stores a new ITR value based on packets and byte
3378 * counts during the last interrupt. The advantage of per interrupt
3379 * computation is faster updates and more accurate ITR for the current
3380 * traffic pattern. Constants in this function were computed
3381 * based on theoretical maximum wire speed and thresholds were set based
3382 * on testing data as well as attempting to minimize response time
3383 * while increasing bulk throughput.
3384 * NOTE: These calculations are only valid when operating in a single-
3385 * queue environment.
3387 static void igc_update_itr(struct igc_q_vector
*q_vector
,
3388 struct igc_ring_container
*ring_container
)
3390 unsigned int packets
= ring_container
->total_packets
;
3391 unsigned int bytes
= ring_container
->total_bytes
;
3392 u8 itrval
= ring_container
->itr
;
3394 /* no packets, exit with status unchanged */
3399 case lowest_latency
:
3400 /* handle TSO and jumbo frames */
3401 if (bytes
/ packets
> 8000)
3402 itrval
= bulk_latency
;
3403 else if ((packets
< 5) && (bytes
> 512))
3404 itrval
= low_latency
;
3406 case low_latency
: /* 50 usec aka 20000 ints/s */
3407 if (bytes
> 10000) {
3408 /* this if handles the TSO accounting */
3409 if (bytes
/ packets
> 8000)
3410 itrval
= bulk_latency
;
3411 else if ((packets
< 10) || ((bytes
/ packets
) > 1200))
3412 itrval
= bulk_latency
;
3413 else if ((packets
> 35))
3414 itrval
= lowest_latency
;
3415 } else if (bytes
/ packets
> 2000) {
3416 itrval
= bulk_latency
;
3417 } else if (packets
<= 2 && bytes
< 512) {
3418 itrval
= lowest_latency
;
3421 case bulk_latency
: /* 250 usec aka 4000 ints/s */
3422 if (bytes
> 25000) {
3424 itrval
= low_latency
;
3425 } else if (bytes
< 1500) {
3426 itrval
= low_latency
;
3431 /* clear work counters since we have the values we need */
3432 ring_container
->total_bytes
= 0;
3433 ring_container
->total_packets
= 0;
3435 /* write updated itr to ring container */
3436 ring_container
->itr
= itrval
;
3440 * igc_intr_msi - Interrupt Handler
3441 * @irq: interrupt number
3442 * @data: pointer to a network interface device structure
3444 static irqreturn_t
igc_intr_msi(int irq
, void *data
)
3446 struct igc_adapter
*adapter
= data
;
3447 struct igc_q_vector
*q_vector
= adapter
->q_vector
[0];
3448 struct igc_hw
*hw
= &adapter
->hw
;
3449 /* read ICR disables interrupts using IAM */
3450 u32 icr
= rd32(IGC_ICR
);
3452 igc_write_itr(q_vector
);
3454 if (icr
& IGC_ICR_DRSTA
)
3455 schedule_work(&adapter
->reset_task
);
3457 if (icr
& IGC_ICR_DOUTSYNC
) {
3458 /* HW is reporting DMA is out of sync */
3459 adapter
->stats
.doosync
++;
3462 if (icr
& (IGC_ICR_RXSEQ
| IGC_ICR_LSC
)) {
3463 hw
->mac
.get_link_status
= 1;
3464 if (!test_bit(__IGC_DOWN
, &adapter
->state
))
3465 mod_timer(&adapter
->watchdog_timer
, jiffies
+ 1);
3468 napi_schedule(&q_vector
->napi
);
3474 * igc_intr - Legacy Interrupt Handler
3475 * @irq: interrupt number
3476 * @data: pointer to a network interface device structure
3478 static irqreturn_t
igc_intr(int irq
, void *data
)
3480 struct igc_adapter
*adapter
= data
;
3481 struct igc_q_vector
*q_vector
= adapter
->q_vector
[0];
3482 struct igc_hw
*hw
= &adapter
->hw
;
3483 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
3484 * need for the IMC write
3486 u32 icr
= rd32(IGC_ICR
);
3488 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
3489 * not set, then the adapter didn't send an interrupt
3491 if (!(icr
& IGC_ICR_INT_ASSERTED
))
3494 igc_write_itr(q_vector
);
3496 if (icr
& IGC_ICR_DRSTA
)
3497 schedule_work(&adapter
->reset_task
);
3499 if (icr
& IGC_ICR_DOUTSYNC
) {
3500 /* HW is reporting DMA is out of sync */
3501 adapter
->stats
.doosync
++;
3504 if (icr
& (IGC_ICR_RXSEQ
| IGC_ICR_LSC
)) {
3505 hw
->mac
.get_link_status
= 1;
3506 /* guard against interrupt when we're going down */
3507 if (!test_bit(__IGC_DOWN
, &adapter
->state
))
3508 mod_timer(&adapter
->watchdog_timer
, jiffies
+ 1);
3511 napi_schedule(&q_vector
->napi
);
3516 static void igc_set_itr(struct igc_q_vector
*q_vector
)
3518 struct igc_adapter
*adapter
= q_vector
->adapter
;
3519 u32 new_itr
= q_vector
->itr_val
;
3522 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3523 switch (adapter
->link_speed
) {
3527 new_itr
= IGC_4K_ITR
;
3533 igc_update_itr(q_vector
, &q_vector
->tx
);
3534 igc_update_itr(q_vector
, &q_vector
->rx
);
3536 current_itr
= max(q_vector
->rx
.itr
, q_vector
->tx
.itr
);
3538 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3539 if (current_itr
== lowest_latency
&&
3540 ((q_vector
->rx
.ring
&& adapter
->rx_itr_setting
== 3) ||
3541 (!q_vector
->rx
.ring
&& adapter
->tx_itr_setting
== 3)))
3542 current_itr
= low_latency
;
3544 switch (current_itr
) {
3545 /* counts and packets in update_itr are dependent on these numbers */
3546 case lowest_latency
:
3547 new_itr
= IGC_70K_ITR
; /* 70,000 ints/sec */
3550 new_itr
= IGC_20K_ITR
; /* 20,000 ints/sec */
3553 new_itr
= IGC_4K_ITR
; /* 4,000 ints/sec */
3560 if (new_itr
!= q_vector
->itr_val
) {
3561 /* this attempts to bias the interrupt rate towards Bulk
3562 * by adding intermediate steps when interrupt rate is
3565 new_itr
= new_itr
> q_vector
->itr_val
?
3566 max((new_itr
* q_vector
->itr_val
) /
3567 (new_itr
+ (q_vector
->itr_val
>> 2)),
3569 /* Don't write the value here; it resets the adapter's
3570 * internal timer, and causes us to delay far longer than
3571 * we should between interrupts. Instead, we write the ITR
3572 * value at the beginning of the next interrupt so the timing
3573 * ends up being correct.
3575 q_vector
->itr_val
= new_itr
;
3576 q_vector
->set_itr
= 1;
3580 static void igc_ring_irq_enable(struct igc_q_vector
*q_vector
)
3582 struct igc_adapter
*adapter
= q_vector
->adapter
;
3583 struct igc_hw
*hw
= &adapter
->hw
;
3585 if ((q_vector
->rx
.ring
&& (adapter
->rx_itr_setting
& 3)) ||
3586 (!q_vector
->rx
.ring
&& (adapter
->tx_itr_setting
& 3))) {
3587 if (adapter
->num_q_vectors
== 1)
3588 igc_set_itr(q_vector
);
3590 igc_update_ring_itr(q_vector
);
3593 if (!test_bit(__IGC_DOWN
, &adapter
->state
)) {
3594 if (adapter
->msix_entries
)
3595 wr32(IGC_EIMS
, q_vector
->eims_value
);
3597 igc_irq_enable(adapter
);
3602 * igc_poll - NAPI Rx polling callback
3603 * @napi: napi polling structure
3604 * @budget: count of how many packets we should handle
3606 static int igc_poll(struct napi_struct
*napi
, int budget
)
3608 struct igc_q_vector
*q_vector
= container_of(napi
,
3609 struct igc_q_vector
,
3611 bool clean_complete
= true;
3614 if (q_vector
->tx
.ring
)
3615 clean_complete
= igc_clean_tx_irq(q_vector
, budget
);
3617 if (q_vector
->rx
.ring
) {
3618 int cleaned
= igc_clean_rx_irq(q_vector
, budget
);
3620 work_done
+= cleaned
;
3621 if (cleaned
>= budget
)
3622 clean_complete
= false;
3625 /* If all work not completed, return budget and keep polling */
3626 if (!clean_complete
)
3629 /* Exit the polling mode, but don't re-enable interrupts if stack might
3630 * poll us due to busy-polling
3632 if (likely(napi_complete_done(napi
, work_done
)))
3633 igc_ring_irq_enable(q_vector
);
3635 return min(work_done
, budget
- 1);
3639 * igc_set_interrupt_capability - set MSI or MSI-X if supported
3640 * @adapter: Pointer to adapter structure
3642 * Attempt to configure interrupts using the best available
3643 * capabilities of the hardware and kernel.
3645 static void igc_set_interrupt_capability(struct igc_adapter
*adapter
,
3653 adapter
->flags
|= IGC_FLAG_HAS_MSIX
;
3655 /* Number of supported queues. */
3656 adapter
->num_rx_queues
= adapter
->rss_queues
;
3658 adapter
->num_tx_queues
= adapter
->rss_queues
;
3660 /* start with one vector for every Rx queue */
3661 numvecs
= adapter
->num_rx_queues
;
3663 /* if Tx handler is separate add 1 for every Tx queue */
3664 if (!(adapter
->flags
& IGC_FLAG_QUEUE_PAIRS
))
3665 numvecs
+= adapter
->num_tx_queues
;
3667 /* store the number of vectors reserved for queues */
3668 adapter
->num_q_vectors
= numvecs
;
3670 /* add 1 vector for link status interrupts */
3673 adapter
->msix_entries
= kcalloc(numvecs
, sizeof(struct msix_entry
),
3676 if (!adapter
->msix_entries
)
3679 /* populate entry values */
3680 for (i
= 0; i
< numvecs
; i
++)
3681 adapter
->msix_entries
[i
].entry
= i
;
3683 err
= pci_enable_msix_range(adapter
->pdev
,
3684 adapter
->msix_entries
,
3690 kfree(adapter
->msix_entries
);
3691 adapter
->msix_entries
= NULL
;
3693 igc_reset_interrupt_capability(adapter
);
3696 adapter
->flags
&= ~IGC_FLAG_HAS_MSIX
;
3698 adapter
->rss_queues
= 1;
3699 adapter
->flags
|= IGC_FLAG_QUEUE_PAIRS
;
3700 adapter
->num_rx_queues
= 1;
3701 adapter
->num_tx_queues
= 1;
3702 adapter
->num_q_vectors
= 1;
3703 if (!pci_enable_msi(adapter
->pdev
))
3704 adapter
->flags
|= IGC_FLAG_HAS_MSI
;
3707 static void igc_add_ring(struct igc_ring
*ring
,
3708 struct igc_ring_container
*head
)
3715 * igc_alloc_q_vector - Allocate memory for a single interrupt vector
3716 * @adapter: board private structure to initialize
3717 * @v_count: q_vectors allocated on adapter, used for ring interleaving
3718 * @v_idx: index of vector in adapter struct
3719 * @txr_count: total number of Tx rings to allocate
3720 * @txr_idx: index of first Tx ring to allocate
3721 * @rxr_count: total number of Rx rings to allocate
3722 * @rxr_idx: index of first Rx ring to allocate
3724 * We allocate one q_vector. If allocation fails we return -ENOMEM.
3726 static int igc_alloc_q_vector(struct igc_adapter
*adapter
,
3727 unsigned int v_count
, unsigned int v_idx
,
3728 unsigned int txr_count
, unsigned int txr_idx
,
3729 unsigned int rxr_count
, unsigned int rxr_idx
)
3731 struct igc_q_vector
*q_vector
;
3732 struct igc_ring
*ring
;
3735 /* igc only supports 1 Tx and/or 1 Rx queue per vector */
3736 if (txr_count
> 1 || rxr_count
> 1)
3739 ring_count
= txr_count
+ rxr_count
;
3741 /* allocate q_vector and rings */
3742 q_vector
= adapter
->q_vector
[v_idx
];
3744 q_vector
= kzalloc(struct_size(q_vector
, ring
, ring_count
),
3747 memset(q_vector
, 0, struct_size(q_vector
, ring
, ring_count
));
3751 /* initialize NAPI */
3752 netif_napi_add(adapter
->netdev
, &q_vector
->napi
,
3755 /* tie q_vector and adapter together */
3756 adapter
->q_vector
[v_idx
] = q_vector
;
3757 q_vector
->adapter
= adapter
;
3759 /* initialize work limits */
3760 q_vector
->tx
.work_limit
= adapter
->tx_work_limit
;
3762 /* initialize ITR configuration */
3763 q_vector
->itr_register
= adapter
->io_addr
+ IGC_EITR(0);
3764 q_vector
->itr_val
= IGC_START_ITR
;
3766 /* initialize pointer to rings */
3767 ring
= q_vector
->ring
;
3769 /* initialize ITR */
3771 /* rx or rx/tx vector */
3772 if (!adapter
->rx_itr_setting
|| adapter
->rx_itr_setting
> 3)
3773 q_vector
->itr_val
= adapter
->rx_itr_setting
;
3775 /* tx only vector */
3776 if (!adapter
->tx_itr_setting
|| adapter
->tx_itr_setting
> 3)
3777 q_vector
->itr_val
= adapter
->tx_itr_setting
;
3781 /* assign generic ring traits */
3782 ring
->dev
= &adapter
->pdev
->dev
;
3783 ring
->netdev
= adapter
->netdev
;
3785 /* configure backlink on ring */
3786 ring
->q_vector
= q_vector
;
3788 /* update q_vector Tx values */
3789 igc_add_ring(ring
, &q_vector
->tx
);
3791 /* apply Tx specific ring traits */
3792 ring
->count
= adapter
->tx_ring_count
;
3793 ring
->queue_index
= txr_idx
;
3795 /* assign ring to adapter */
3796 adapter
->tx_ring
[txr_idx
] = ring
;
3798 /* push pointer to next ring */
3803 /* assign generic ring traits */
3804 ring
->dev
= &adapter
->pdev
->dev
;
3805 ring
->netdev
= adapter
->netdev
;
3807 /* configure backlink on ring */
3808 ring
->q_vector
= q_vector
;
3810 /* update q_vector Rx values */
3811 igc_add_ring(ring
, &q_vector
->rx
);
3813 /* apply Rx specific ring traits */
3814 ring
->count
= adapter
->rx_ring_count
;
3815 ring
->queue_index
= rxr_idx
;
3817 /* assign ring to adapter */
3818 adapter
->rx_ring
[rxr_idx
] = ring
;
3825 * igc_alloc_q_vectors - Allocate memory for interrupt vectors
3826 * @adapter: board private structure to initialize
3828 * We allocate one q_vector per queue interrupt. If allocation fails we
3831 static int igc_alloc_q_vectors(struct igc_adapter
*adapter
)
3833 int rxr_remaining
= adapter
->num_rx_queues
;
3834 int txr_remaining
= adapter
->num_tx_queues
;
3835 int rxr_idx
= 0, txr_idx
= 0, v_idx
= 0;
3836 int q_vectors
= adapter
->num_q_vectors
;
3839 if (q_vectors
>= (rxr_remaining
+ txr_remaining
)) {
3840 for (; rxr_remaining
; v_idx
++) {
3841 err
= igc_alloc_q_vector(adapter
, q_vectors
, v_idx
,
3847 /* update counts and index */
3853 for (; v_idx
< q_vectors
; v_idx
++) {
3854 int rqpv
= DIV_ROUND_UP(rxr_remaining
, q_vectors
- v_idx
);
3855 int tqpv
= DIV_ROUND_UP(txr_remaining
, q_vectors
- v_idx
);
3857 err
= igc_alloc_q_vector(adapter
, q_vectors
, v_idx
,
3858 tqpv
, txr_idx
, rqpv
, rxr_idx
);
3863 /* update counts and index */
3864 rxr_remaining
-= rqpv
;
3865 txr_remaining
-= tqpv
;
3873 adapter
->num_tx_queues
= 0;
3874 adapter
->num_rx_queues
= 0;
3875 adapter
->num_q_vectors
= 0;
3878 igc_free_q_vector(adapter
, v_idx
);
3884 * igc_cache_ring_register - Descriptor ring to register mapping
3885 * @adapter: board private structure to initialize
3887 * Once we know the feature-set enabled for the device, we'll cache
3888 * the register offset the descriptor ring is assigned to.
3890 static void igc_cache_ring_register(struct igc_adapter
*adapter
)
3894 switch (adapter
->hw
.mac
.type
) {
3898 for (; i
< adapter
->num_rx_queues
; i
++)
3899 adapter
->rx_ring
[i
]->reg_idx
= i
;
3900 for (; j
< adapter
->num_tx_queues
; j
++)
3901 adapter
->tx_ring
[j
]->reg_idx
= j
;
3907 * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
3908 * @adapter: Pointer to adapter structure
3910 * This function initializes the interrupts and allocates all of the queues.
3912 static int igc_init_interrupt_scheme(struct igc_adapter
*adapter
, bool msix
)
3914 struct pci_dev
*pdev
= adapter
->pdev
;
3917 igc_set_interrupt_capability(adapter
, msix
);
3919 err
= igc_alloc_q_vectors(adapter
);
3921 dev_err(&pdev
->dev
, "Unable to allocate memory for vectors\n");
3922 goto err_alloc_q_vectors
;
3925 igc_cache_ring_register(adapter
);
3929 err_alloc_q_vectors
:
3930 igc_reset_interrupt_capability(adapter
);
3934 static void igc_free_irq(struct igc_adapter
*adapter
)
3936 if (adapter
->msix_entries
) {
3939 free_irq(adapter
->msix_entries
[vector
++].vector
, adapter
);
3941 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
3942 free_irq(adapter
->msix_entries
[vector
++].vector
,
3943 adapter
->q_vector
[i
]);
3945 free_irq(adapter
->pdev
->irq
, adapter
);
3950 * igc_irq_disable - Mask off interrupt generation on the NIC
3951 * @adapter: board private structure
3953 static void igc_irq_disable(struct igc_adapter
*adapter
)
3955 struct igc_hw
*hw
= &adapter
->hw
;
3957 if (adapter
->msix_entries
) {
3958 u32 regval
= rd32(IGC_EIAM
);
3960 wr32(IGC_EIAM
, regval
& ~adapter
->eims_enable_mask
);
3961 wr32(IGC_EIMC
, adapter
->eims_enable_mask
);
3962 regval
= rd32(IGC_EIAC
);
3963 wr32(IGC_EIAC
, regval
& ~adapter
->eims_enable_mask
);
3970 if (adapter
->msix_entries
) {
3973 synchronize_irq(adapter
->msix_entries
[vector
++].vector
);
3975 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
3976 synchronize_irq(adapter
->msix_entries
[vector
++].vector
);
3978 synchronize_irq(adapter
->pdev
->irq
);
3983 * igc_irq_enable - Enable default interrupt generation settings
3984 * @adapter: board private structure
3986 static void igc_irq_enable(struct igc_adapter
*adapter
)
3988 struct igc_hw
*hw
= &adapter
->hw
;
3990 if (adapter
->msix_entries
) {
3991 u32 ims
= IGC_IMS_LSC
| IGC_IMS_DOUTSYNC
| IGC_IMS_DRSTA
;
3992 u32 regval
= rd32(IGC_EIAC
);
3994 wr32(IGC_EIAC
, regval
| adapter
->eims_enable_mask
);
3995 regval
= rd32(IGC_EIAM
);
3996 wr32(IGC_EIAM
, regval
| adapter
->eims_enable_mask
);
3997 wr32(IGC_EIMS
, adapter
->eims_enable_mask
);
4000 wr32(IGC_IMS
, IMS_ENABLE_MASK
| IGC_IMS_DRSTA
);
4001 wr32(IGC_IAM
, IMS_ENABLE_MASK
| IGC_IMS_DRSTA
);
4006 * igc_request_irq - initialize interrupts
4007 * @adapter: Pointer to adapter structure
4009 * Attempts to configure interrupts using the best available
4010 * capabilities of the hardware and kernel.
4012 static int igc_request_irq(struct igc_adapter
*adapter
)
4014 struct net_device
*netdev
= adapter
->netdev
;
4015 struct pci_dev
*pdev
= adapter
->pdev
;
4018 if (adapter
->flags
& IGC_FLAG_HAS_MSIX
) {
4019 err
= igc_request_msix(adapter
);
4022 /* fall back to MSI */
4023 igc_free_all_tx_resources(adapter
);
4024 igc_free_all_rx_resources(adapter
);
4026 igc_clear_interrupt_scheme(adapter
);
4027 err
= igc_init_interrupt_scheme(adapter
, false);
4030 igc_setup_all_tx_resources(adapter
);
4031 igc_setup_all_rx_resources(adapter
);
4032 igc_configure(adapter
);
4035 igc_assign_vector(adapter
->q_vector
[0], 0);
4037 if (adapter
->flags
& IGC_FLAG_HAS_MSI
) {
4038 err
= request_irq(pdev
->irq
, &igc_intr_msi
, 0,
4039 netdev
->name
, adapter
);
4043 /* fall back to legacy interrupts */
4044 igc_reset_interrupt_capability(adapter
);
4045 adapter
->flags
&= ~IGC_FLAG_HAS_MSI
;
4048 err
= request_irq(pdev
->irq
, &igc_intr
, IRQF_SHARED
,
4049 netdev
->name
, adapter
);
4052 dev_err(&pdev
->dev
, "Error %d getting interrupt\n",
4059 static void igc_write_itr(struct igc_q_vector
*q_vector
)
4061 u32 itr_val
= q_vector
->itr_val
& IGC_QVECTOR_MASK
;
4063 if (!q_vector
->set_itr
)
4067 itr_val
= IGC_ITR_VAL_MASK
;
4069 itr_val
|= IGC_EITR_CNT_IGNR
;
4071 writel(itr_val
, q_vector
->itr_register
);
4072 q_vector
->set_itr
= 0;
4076 * igc_open - Called when a network interface is made active
4077 * @netdev: network interface device structure
4079 * Returns 0 on success, negative value on failure
4081 * The open entry point is called when a network interface is made
4082 * active by the system (IFF_UP). At this point all resources needed
4083 * for transmit and receive operations are allocated, the interrupt
4084 * handler is registered with the OS, the watchdog timer is started,
4085 * and the stack is notified that the interface is ready.
4087 static int __igc_open(struct net_device
*netdev
, bool resuming
)
4089 struct igc_adapter
*adapter
= netdev_priv(netdev
);
4090 struct igc_hw
*hw
= &adapter
->hw
;
4094 /* disallow open during test */
4096 if (test_bit(__IGC_TESTING
, &adapter
->state
)) {
4101 netif_carrier_off(netdev
);
4103 /* allocate transmit descriptors */
4104 err
= igc_setup_all_tx_resources(adapter
);
4108 /* allocate receive descriptors */
4109 err
= igc_setup_all_rx_resources(adapter
);
4113 igc_power_up_link(adapter
);
4115 igc_configure(adapter
);
4117 err
= igc_request_irq(adapter
);
4121 /* Notify the stack of the actual queue counts. */
4122 err
= netif_set_real_num_tx_queues(netdev
, adapter
->num_tx_queues
);
4124 goto err_set_queues
;
4126 err
= netif_set_real_num_rx_queues(netdev
, adapter
->num_rx_queues
);
4128 goto err_set_queues
;
4130 clear_bit(__IGC_DOWN
, &adapter
->state
);
4132 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
4133 napi_enable(&adapter
->q_vector
[i
]->napi
);
4135 /* Clear any pending interrupts. */
4137 igc_irq_enable(adapter
);
4139 netif_tx_start_all_queues(netdev
);
4141 /* start the watchdog. */
4142 hw
->mac
.get_link_status
= 1;
4143 schedule_work(&adapter
->watchdog_task
);
4148 igc_free_irq(adapter
);
4150 igc_release_hw_control(adapter
);
4151 igc_power_down_link(adapter
);
4152 igc_free_all_rx_resources(adapter
);
4154 igc_free_all_tx_resources(adapter
);
4161 static int igc_open(struct net_device
*netdev
)
4163 return __igc_open(netdev
, false);
4167 * igc_close - Disables a network interface
4168 * @netdev: network interface device structure
4170 * Returns 0, this is not allowed to fail
4172 * The close entry point is called when an interface is de-activated
4173 * by the OS. The hardware is still under the driver's control, but
4174 * needs to be disabled. A global MAC reset is issued to stop the
4175 * hardware, and all transmit and receive resources are freed.
4177 static int __igc_close(struct net_device
*netdev
, bool suspending
)
4179 struct igc_adapter
*adapter
= netdev_priv(netdev
);
4181 WARN_ON(test_bit(__IGC_RESETTING
, &adapter
->state
));
4185 igc_release_hw_control(adapter
);
4187 igc_free_irq(adapter
);
4189 igc_free_all_tx_resources(adapter
);
4190 igc_free_all_rx_resources(adapter
);
4195 static int igc_close(struct net_device
*netdev
)
4197 if (netif_device_present(netdev
) || netdev
->dismantle
)
4198 return __igc_close(netdev
, false);
4202 static const struct net_device_ops igc_netdev_ops
= {
4203 .ndo_open
= igc_open
,
4204 .ndo_stop
= igc_close
,
4205 .ndo_start_xmit
= igc_xmit_frame
,
4206 .ndo_set_rx_mode
= igc_set_rx_mode
,
4207 .ndo_set_mac_address
= igc_set_mac
,
4208 .ndo_change_mtu
= igc_change_mtu
,
4209 .ndo_get_stats
= igc_get_stats
,
4210 .ndo_fix_features
= igc_fix_features
,
4211 .ndo_set_features
= igc_set_features
,
4212 .ndo_features_check
= igc_features_check
,
4215 /* PCIe configuration access */
4216 void igc_read_pci_cfg(struct igc_hw
*hw
, u32 reg
, u16
*value
)
4218 struct igc_adapter
*adapter
= hw
->back
;
4220 pci_read_config_word(adapter
->pdev
, reg
, value
);
4223 void igc_write_pci_cfg(struct igc_hw
*hw
, u32 reg
, u16
*value
)
4225 struct igc_adapter
*adapter
= hw
->back
;
4227 pci_write_config_word(adapter
->pdev
, reg
, *value
);
4230 s32
igc_read_pcie_cap_reg(struct igc_hw
*hw
, u32 reg
, u16
*value
)
4232 struct igc_adapter
*adapter
= hw
->back
;
4234 if (!pci_is_pcie(adapter
->pdev
))
4235 return -IGC_ERR_CONFIG
;
4237 pcie_capability_read_word(adapter
->pdev
, reg
, value
);
4242 s32
igc_write_pcie_cap_reg(struct igc_hw
*hw
, u32 reg
, u16
*value
)
4244 struct igc_adapter
*adapter
= hw
->back
;
4246 if (!pci_is_pcie(adapter
->pdev
))
4247 return -IGC_ERR_CONFIG
;
4249 pcie_capability_write_word(adapter
->pdev
, reg
, *value
);
4254 u32
igc_rd32(struct igc_hw
*hw
, u32 reg
)
4256 struct igc_adapter
*igc
= container_of(hw
, struct igc_adapter
, hw
);
4257 u8 __iomem
*hw_addr
= READ_ONCE(hw
->hw_addr
);
4260 if (IGC_REMOVED(hw_addr
))
4263 value
= readl(&hw_addr
[reg
]);
4265 /* reads should not return all F's */
4266 if (!(~value
) && (!reg
|| !(~readl(hw_addr
)))) {
4267 struct net_device
*netdev
= igc
->netdev
;
4270 netif_device_detach(netdev
);
4271 netdev_err(netdev
, "PCIe link lost, device now detached\n");
4272 WARN(pci_device_is_present(igc
->pdev
),
4273 "igc: Failed to read reg 0x%x!\n", reg
);
4279 int igc_set_spd_dplx(struct igc_adapter
*adapter
, u32 spd
, u8 dplx
)
4281 struct pci_dev
*pdev
= adapter
->pdev
;
4282 struct igc_mac_info
*mac
= &adapter
->hw
.mac
;
4286 /* Make sure dplx is at most 1 bit and lsb of speed is not set
4287 * for the switch() below to work
4289 if ((spd
& 1) || (dplx
& ~1))
4292 switch (spd
+ dplx
) {
4293 case SPEED_10
+ DUPLEX_HALF
:
4294 mac
->forced_speed_duplex
= ADVERTISE_10_HALF
;
4296 case SPEED_10
+ DUPLEX_FULL
:
4297 mac
->forced_speed_duplex
= ADVERTISE_10_FULL
;
4299 case SPEED_100
+ DUPLEX_HALF
:
4300 mac
->forced_speed_duplex
= ADVERTISE_100_HALF
;
4302 case SPEED_100
+ DUPLEX_FULL
:
4303 mac
->forced_speed_duplex
= ADVERTISE_100_FULL
;
4305 case SPEED_1000
+ DUPLEX_FULL
:
4307 adapter
->hw
.phy
.autoneg_advertised
= ADVERTISE_1000_FULL
;
4309 case SPEED_1000
+ DUPLEX_HALF
: /* not supported */
4311 case SPEED_2500
+ DUPLEX_FULL
:
4313 adapter
->hw
.phy
.autoneg_advertised
= ADVERTISE_2500_FULL
;
4315 case SPEED_2500
+ DUPLEX_HALF
: /* not supported */
4320 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
4321 adapter
->hw
.phy
.mdix
= AUTO_ALL_MODES
;
4326 dev_err(&pdev
->dev
, "Unsupported Speed/Duplex configuration\n");
4331 * igc_probe - Device Initialization Routine
4332 * @pdev: PCI device information struct
4333 * @ent: entry in igc_pci_tbl
4335 * Returns 0 on success, negative on failure
4337 * igc_probe initializes an adapter identified by a pci_dev structure.
4338 * The OS initialization, configuring the adapter private structure,
4339 * and a hardware reset occur.
4341 static int igc_probe(struct pci_dev
*pdev
,
4342 const struct pci_device_id
*ent
)
4344 struct igc_adapter
*adapter
;
4345 struct net_device
*netdev
;
4347 const struct igc_info
*ei
= igc_info_tbl
[ent
->driver_data
];
4350 err
= pci_enable_device_mem(pdev
);
4354 err
= dma_set_mask(&pdev
->dev
, DMA_BIT_MASK(64));
4356 err
= dma_set_coherent_mask(&pdev
->dev
,
4359 err
= dma_set_mask(&pdev
->dev
, DMA_BIT_MASK(32));
4361 err
= dma_set_coherent_mask(&pdev
->dev
,
4364 dev_err(&pdev
->dev
, "igc: Wrong DMA config\n");
4370 err
= pci_request_selected_regions(pdev
,
4371 pci_select_bars(pdev
,
4377 pci_enable_pcie_error_reporting(pdev
);
4379 pci_set_master(pdev
);
4382 netdev
= alloc_etherdev_mq(sizeof(struct igc_adapter
),
4386 goto err_alloc_etherdev
;
4388 SET_NETDEV_DEV(netdev
, &pdev
->dev
);
4390 pci_set_drvdata(pdev
, netdev
);
4391 adapter
= netdev_priv(netdev
);
4392 adapter
->netdev
= netdev
;
4393 adapter
->pdev
= pdev
;
4396 adapter
->port_num
= hw
->bus
.func
;
4397 adapter
->msg_enable
= netif_msg_init(debug
, DEFAULT_MSG_ENABLE
);
4399 err
= pci_save_state(pdev
);
4404 adapter
->io_addr
= ioremap(pci_resource_start(pdev
, 0),
4405 pci_resource_len(pdev
, 0));
4406 if (!adapter
->io_addr
)
4409 /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */
4410 hw
->hw_addr
= adapter
->io_addr
;
4412 netdev
->netdev_ops
= &igc_netdev_ops
;
4413 igc_set_ethtool_ops(netdev
);
4414 netdev
->watchdog_timeo
= 5 * HZ
;
4416 netdev
->mem_start
= pci_resource_start(pdev
, 0);
4417 netdev
->mem_end
= pci_resource_end(pdev
, 0);
4419 /* PCI config space info */
4420 hw
->vendor_id
= pdev
->vendor
;
4421 hw
->device_id
= pdev
->device
;
4422 hw
->revision_id
= pdev
->revision
;
4423 hw
->subsystem_vendor_id
= pdev
->subsystem_vendor
;
4424 hw
->subsystem_device_id
= pdev
->subsystem_device
;
4426 /* Copy the default MAC and PHY function pointers */
4427 memcpy(&hw
->mac
.ops
, ei
->mac_ops
, sizeof(hw
->mac
.ops
));
4428 memcpy(&hw
->phy
.ops
, ei
->phy_ops
, sizeof(hw
->phy
.ops
));
4430 /* Initialize skew-specific constants */
4431 err
= ei
->get_invariants(hw
);
4435 /* Add supported features to the features list*/
4436 netdev
->features
|= NETIF_F_RXCSUM
;
4437 netdev
->features
|= NETIF_F_HW_CSUM
;
4438 netdev
->features
|= NETIF_F_SCTP_CRC
;
4440 /* setup the private structure */
4441 err
= igc_sw_init(adapter
);
4445 /* copy netdev features into list of user selectable features */
4446 netdev
->hw_features
|= NETIF_F_NTUPLE
;
4447 netdev
->hw_features
|= netdev
->features
;
4449 /* MTU range: 68 - 9216 */
4450 netdev
->min_mtu
= ETH_MIN_MTU
;
4451 netdev
->max_mtu
= MAX_STD_JUMBO_FRAME_SIZE
;
4453 /* before reading the NVM, reset the controller to put the device in a
4454 * known good starting state
4456 hw
->mac
.ops
.reset_hw(hw
);
4458 if (igc_get_flash_presence_i225(hw
)) {
4459 if (hw
->nvm
.ops
.validate(hw
) < 0) {
4461 "The NVM Checksum Is Not Valid\n");
4467 if (eth_platform_get_mac_address(&pdev
->dev
, hw
->mac
.addr
)) {
4468 /* copy the MAC address out of the NVM */
4469 if (hw
->mac
.ops
.read_mac_addr(hw
))
4470 dev_err(&pdev
->dev
, "NVM Read Error\n");
4473 memcpy(netdev
->dev_addr
, hw
->mac
.addr
, netdev
->addr_len
);
4475 if (!is_valid_ether_addr(netdev
->dev_addr
)) {
4476 dev_err(&pdev
->dev
, "Invalid MAC Address\n");
4481 /* configure RXPBSIZE and TXPBSIZE */
4482 wr32(IGC_RXPBS
, I225_RXPBSIZE_DEFAULT
);
4483 wr32(IGC_TXPBS
, I225_TXPBSIZE_DEFAULT
);
4485 timer_setup(&adapter
->watchdog_timer
, igc_watchdog
, 0);
4486 timer_setup(&adapter
->phy_info_timer
, igc_update_phy_info
, 0);
4488 INIT_WORK(&adapter
->reset_task
, igc_reset_task
);
4489 INIT_WORK(&adapter
->watchdog_task
, igc_watchdog_task
);
4491 /* Initialize link properties that are user-changeable */
4492 adapter
->fc_autoneg
= true;
4493 hw
->mac
.autoneg
= true;
4494 hw
->phy
.autoneg_advertised
= 0xaf;
4496 hw
->fc
.requested_mode
= igc_fc_default
;
4497 hw
->fc
.current_mode
= igc_fc_default
;
4499 /* reset the hardware with the new settings */
4502 /* let the f/w know that the h/w is now under the control of the
4505 igc_get_hw_control(adapter
);
4507 strncpy(netdev
->name
, "eth%d", IFNAMSIZ
);
4508 err
= register_netdev(netdev
);
4512 /* carrier off reporting is important to ethtool even BEFORE open */
4513 netif_carrier_off(netdev
);
4515 /* Check if Media Autosense is enabled */
4518 /* print pcie link status and MAC address */
4519 pcie_print_link_status(pdev
);
4520 netdev_info(netdev
, "MAC: %pM\n", netdev
->dev_addr
);
4525 igc_release_hw_control(adapter
);
4527 if (!igc_check_reset_block(hw
))
4530 igc_clear_interrupt_scheme(adapter
);
4531 iounmap(adapter
->io_addr
);
4533 free_netdev(netdev
);
4535 pci_release_selected_regions(pdev
,
4536 pci_select_bars(pdev
, IORESOURCE_MEM
));
4539 pci_disable_device(pdev
);
4544 * igc_remove - Device Removal Routine
4545 * @pdev: PCI device information struct
4547 * igc_remove is called by the PCI subsystem to alert the driver
4548 * that it should release a PCI device. This could be caused by a
4549 * Hot-Plug event, or because the driver is going to be removed from
4552 static void igc_remove(struct pci_dev
*pdev
)
4554 struct net_device
*netdev
= pci_get_drvdata(pdev
);
4555 struct igc_adapter
*adapter
= netdev_priv(netdev
);
4557 set_bit(__IGC_DOWN
, &adapter
->state
);
4559 del_timer_sync(&adapter
->watchdog_timer
);
4560 del_timer_sync(&adapter
->phy_info_timer
);
4562 cancel_work_sync(&adapter
->reset_task
);
4563 cancel_work_sync(&adapter
->watchdog_task
);
4565 /* Release control of h/w to f/w. If f/w is AMT enabled, this
4566 * would have already happened in close and is redundant.
4568 igc_release_hw_control(adapter
);
4569 unregister_netdev(netdev
);
4571 igc_clear_interrupt_scheme(adapter
);
4572 pci_iounmap(pdev
, adapter
->io_addr
);
4573 pci_release_mem_regions(pdev
);
4575 kfree(adapter
->mac_table
);
4576 free_netdev(netdev
);
4578 pci_disable_pcie_error_reporting(pdev
);
4580 pci_disable_device(pdev
);
4583 static struct pci_driver igc_driver
= {
4584 .name
= igc_driver_name
,
4585 .id_table
= igc_pci_tbl
,
4587 .remove
= igc_remove
,
4590 void igc_set_flag_queue_pairs(struct igc_adapter
*adapter
,
4591 const u32 max_rss_queues
)
4593 /* Determine if we need to pair queues. */
4594 /* If rss_queues > half of max_rss_queues, pair the queues in
4595 * order to conserve interrupts due to limited supply.
4597 if (adapter
->rss_queues
> (max_rss_queues
/ 2))
4598 adapter
->flags
|= IGC_FLAG_QUEUE_PAIRS
;
4600 adapter
->flags
&= ~IGC_FLAG_QUEUE_PAIRS
;
4603 unsigned int igc_get_max_rss_queues(struct igc_adapter
*adapter
)
4605 unsigned int max_rss_queues
;
4607 /* Determine the maximum number of RSS queues supported. */
4608 max_rss_queues
= IGC_MAX_RX_QUEUES
;
4610 return max_rss_queues
;
4613 static void igc_init_queue_configuration(struct igc_adapter
*adapter
)
4617 max_rss_queues
= igc_get_max_rss_queues(adapter
);
4618 adapter
->rss_queues
= min_t(u32
, max_rss_queues
, num_online_cpus());
4620 igc_set_flag_queue_pairs(adapter
, max_rss_queues
);
4624 * igc_sw_init - Initialize general software structures (struct igc_adapter)
4625 * @adapter: board private structure to initialize
4627 * igc_sw_init initializes the Adapter private data structure.
4628 * Fields are initialized based on PCI device information and
4629 * OS network device settings (MTU size).
4631 static int igc_sw_init(struct igc_adapter
*adapter
)
4633 struct net_device
*netdev
= adapter
->netdev
;
4634 struct pci_dev
*pdev
= adapter
->pdev
;
4635 struct igc_hw
*hw
= &adapter
->hw
;
4637 int size
= sizeof(struct igc_mac_addr
) * hw
->mac
.rar_entry_count
;
4639 pci_read_config_word(pdev
, PCI_COMMAND
, &hw
->bus
.pci_cmd_word
);
4641 /* set default ring sizes */
4642 adapter
->tx_ring_count
= IGC_DEFAULT_TXD
;
4643 adapter
->rx_ring_count
= IGC_DEFAULT_RXD
;
4645 /* set default ITR values */
4646 adapter
->rx_itr_setting
= IGC_DEFAULT_ITR
;
4647 adapter
->tx_itr_setting
= IGC_DEFAULT_ITR
;
4649 /* set default work limits */
4650 adapter
->tx_work_limit
= IGC_DEFAULT_TX_WORK
;
4652 /* adjust max frame to be at least the size of a standard frame */
4653 adapter
->max_frame_size
= netdev
->mtu
+ ETH_HLEN
+ ETH_FCS_LEN
+
4655 adapter
->min_frame_size
= ETH_ZLEN
+ ETH_FCS_LEN
;
4657 spin_lock_init(&adapter
->nfc_lock
);
4658 spin_lock_init(&adapter
->stats64_lock
);
4659 /* Assume MSI-X interrupts, will be checked during IRQ allocation */
4660 adapter
->flags
|= IGC_FLAG_HAS_MSIX
;
4662 adapter
->mac_table
= kzalloc(size
, GFP_ATOMIC
);
4663 if (!adapter
->mac_table
)
4666 igc_init_queue_configuration(adapter
);
4668 /* This call may decrease the number of queues */
4669 if (igc_init_interrupt_scheme(adapter
, true)) {
4670 dev_err(&pdev
->dev
, "Unable to allocate memory for queues\n");
4674 /* Explicitly disable IRQ since the NIC can be in any state. */
4675 igc_irq_disable(adapter
);
4677 set_bit(__IGC_DOWN
, &adapter
->state
);
4683 * igc_reinit_queues - return error
4684 * @adapter: pointer to adapter structure
4686 int igc_reinit_queues(struct igc_adapter
*adapter
)
4688 struct net_device
*netdev
= adapter
->netdev
;
4689 struct pci_dev
*pdev
= adapter
->pdev
;
4692 if (netif_running(netdev
))
4695 igc_reset_interrupt_capability(adapter
);
4697 if (igc_init_interrupt_scheme(adapter
, true)) {
4698 dev_err(&pdev
->dev
, "Unable to allocate memory for queues\n");
4702 if (netif_running(netdev
))
4703 err
= igc_open(netdev
);
4709 * igc_get_hw_dev - return device
4710 * @hw: pointer to hardware structure
4712 * used by hardware layer to print debugging information
4714 struct net_device
*igc_get_hw_dev(struct igc_hw
*hw
)
4716 struct igc_adapter
*adapter
= hw
->back
;
4718 return adapter
->netdev
;
4722 * igc_init_module - Driver Registration Routine
4724 * igc_init_module is the first routine called when the driver is
4725 * loaded. All it does is register with the PCI subsystem.
4727 static int __init
igc_init_module(void)
4731 pr_info("%s - version %s\n",
4732 igc_driver_string
, igc_driver_version
);
4734 pr_info("%s\n", igc_copyright
);
4736 ret
= pci_register_driver(&igc_driver
);
4740 module_init(igc_init_module
);
4743 * igc_exit_module - Driver Exit Cleanup Routine
4745 * igc_exit_module is called just before the driver is removed
4748 static void __exit
igc_exit_module(void)
4750 pci_unregister_driver(&igc_driver
);
4753 module_exit(igc_exit_module
);