1 /* Intel(R) Gigabit Ethernet Linux driver
2 * Copyright(c) 2007-2014 Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, see <http://www.gnu.org/licenses/>.
16 * The full GNU General Public License is included in this distribution in
17 * the file called "COPYING".
19 * Contact Information:
20 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
21 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
24 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
26 #include <linux/module.h>
27 #include <linux/types.h>
28 #include <linux/init.h>
29 #include <linux/bitops.h>
30 #include <linux/vmalloc.h>
31 #include <linux/pagemap.h>
32 #include <linux/netdevice.h>
33 #include <linux/ipv6.h>
34 #include <linux/slab.h>
35 #include <net/checksum.h>
36 #include <net/ip6_checksum.h>
37 #include <net/pkt_sched.h>
38 #include <linux/net_tstamp.h>
39 #include <linux/mii.h>
40 #include <linux/ethtool.h>
42 #include <linux/if_vlan.h>
43 #include <linux/pci.h>
44 #include <linux/pci-aspm.h>
45 #include <linux/delay.h>
46 #include <linux/interrupt.h>
48 #include <linux/tcp.h>
49 #include <linux/sctp.h>
50 #include <linux/if_ether.h>
51 #include <linux/aer.h>
52 #include <linux/prefetch.h>
53 #include <linux/pm_runtime.h>
54 #include <linux/etherdevice.h>
56 #include <linux/dca.h>
58 #include <linux/i2c.h>
64 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
65 __stringify(BUILD) "-k"
68 QUEUE_MODE_STRICT_PRIORITY
,
69 QUEUE_MODE_STREAM_RESERVATION
,
77 char igb_driver_name
[] = "igb";
78 char igb_driver_version
[] = DRV_VERSION
;
79 static const char igb_driver_string
[] =
80 "Intel(R) Gigabit Ethernet Network Driver";
81 static const char igb_copyright
[] =
82 "Copyright (c) 2007-2014 Intel Corporation.";
84 static const struct e1000_info
*igb_info_tbl
[] = {
85 [board_82575
] = &e1000_82575_info
,
88 static const struct pci_device_id igb_pci_tbl
[] = {
89 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I354_BACKPLANE_1GBPS
) },
90 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I354_SGMII
) },
91 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS
) },
92 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I211_COPPER
), board_82575
},
93 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I210_COPPER
), board_82575
},
94 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I210_FIBER
), board_82575
},
95 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I210_SERDES
), board_82575
},
96 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I210_SGMII
), board_82575
},
97 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I210_COPPER_FLASHLESS
), board_82575
},
98 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I210_SERDES_FLASHLESS
), board_82575
},
99 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I350_COPPER
), board_82575
},
100 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I350_FIBER
), board_82575
},
101 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I350_SERDES
), board_82575
},
102 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I350_SGMII
), board_82575
},
103 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82580_COPPER
), board_82575
},
104 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82580_FIBER
), board_82575
},
105 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82580_QUAD_FIBER
), board_82575
},
106 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82580_SERDES
), board_82575
},
107 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82580_SGMII
), board_82575
},
108 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82580_COPPER_DUAL
), board_82575
},
109 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_DH89XXCC_SGMII
), board_82575
},
110 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_DH89XXCC_SERDES
), board_82575
},
111 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_DH89XXCC_BACKPLANE
), board_82575
},
112 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_DH89XXCC_SFP
), board_82575
},
113 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82576
), board_82575
},
114 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82576_NS
), board_82575
},
115 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82576_NS_SERDES
), board_82575
},
116 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82576_FIBER
), board_82575
},
117 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82576_SERDES
), board_82575
},
118 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82576_SERDES_QUAD
), board_82575
},
119 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82576_QUAD_COPPER_ET2
), board_82575
},
120 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82576_QUAD_COPPER
), board_82575
},
121 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82575EB_COPPER
), board_82575
},
122 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82575EB_FIBER_SERDES
), board_82575
},
123 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82575GB_QUAD_COPPER
), board_82575
},
124 /* required last entry */
128 MODULE_DEVICE_TABLE(pci
, igb_pci_tbl
);
130 static int igb_setup_all_tx_resources(struct igb_adapter
*);
131 static int igb_setup_all_rx_resources(struct igb_adapter
*);
132 static void igb_free_all_tx_resources(struct igb_adapter
*);
133 static void igb_free_all_rx_resources(struct igb_adapter
*);
134 static void igb_setup_mrqc(struct igb_adapter
*);
135 static int igb_probe(struct pci_dev
*, const struct pci_device_id
*);
136 static void igb_remove(struct pci_dev
*pdev
);
137 static int igb_sw_init(struct igb_adapter
*);
138 int igb_open(struct net_device
*);
139 int igb_close(struct net_device
*);
140 static void igb_configure(struct igb_adapter
*);
141 static void igb_configure_tx(struct igb_adapter
*);
142 static void igb_configure_rx(struct igb_adapter
*);
143 static void igb_clean_all_tx_rings(struct igb_adapter
*);
144 static void igb_clean_all_rx_rings(struct igb_adapter
*);
145 static void igb_clean_tx_ring(struct igb_ring
*);
146 static void igb_clean_rx_ring(struct igb_ring
*);
147 static void igb_set_rx_mode(struct net_device
*);
148 static void igb_update_phy_info(struct timer_list
*);
149 static void igb_watchdog(struct timer_list
*);
150 static void igb_watchdog_task(struct work_struct
*);
151 static netdev_tx_t
igb_xmit_frame(struct sk_buff
*skb
, struct net_device
*);
152 static void igb_get_stats64(struct net_device
*dev
,
153 struct rtnl_link_stats64
*stats
);
154 static int igb_change_mtu(struct net_device
*, int);
155 static int igb_set_mac(struct net_device
*, void *);
156 static void igb_set_uta(struct igb_adapter
*adapter
, bool set
);
157 static irqreturn_t
igb_intr(int irq
, void *);
158 static irqreturn_t
igb_intr_msi(int irq
, void *);
159 static irqreturn_t
igb_msix_other(int irq
, void *);
160 static irqreturn_t
igb_msix_ring(int irq
, void *);
161 #ifdef CONFIG_IGB_DCA
162 static void igb_update_dca(struct igb_q_vector
*);
163 static void igb_setup_dca(struct igb_adapter
*);
164 #endif /* CONFIG_IGB_DCA */
165 static int igb_poll(struct napi_struct
*, int);
166 static bool igb_clean_tx_irq(struct igb_q_vector
*, int);
167 static int igb_clean_rx_irq(struct igb_q_vector
*, int);
168 static int igb_ioctl(struct net_device
*, struct ifreq
*, int cmd
);
169 static void igb_tx_timeout(struct net_device
*);
170 static void igb_reset_task(struct work_struct
*);
171 static void igb_vlan_mode(struct net_device
*netdev
,
172 netdev_features_t features
);
173 static int igb_vlan_rx_add_vid(struct net_device
*, __be16
, u16
);
174 static int igb_vlan_rx_kill_vid(struct net_device
*, __be16
, u16
);
175 static void igb_restore_vlan(struct igb_adapter
*);
176 static void igb_rar_set_index(struct igb_adapter
*, u32
);
177 static void igb_ping_all_vfs(struct igb_adapter
*);
178 static void igb_msg_task(struct igb_adapter
*);
179 static void igb_vmm_control(struct igb_adapter
*);
180 static int igb_set_vf_mac(struct igb_adapter
*, int, unsigned char *);
181 static void igb_flush_mac_table(struct igb_adapter
*);
182 static int igb_available_rars(struct igb_adapter
*, u8
);
183 static void igb_set_default_mac_filter(struct igb_adapter
*);
184 static int igb_uc_sync(struct net_device
*, const unsigned char *);
185 static int igb_uc_unsync(struct net_device
*, const unsigned char *);
186 static void igb_restore_vf_multicasts(struct igb_adapter
*adapter
);
187 static int igb_ndo_set_vf_mac(struct net_device
*netdev
, int vf
, u8
*mac
);
188 static int igb_ndo_set_vf_vlan(struct net_device
*netdev
,
189 int vf
, u16 vlan
, u8 qos
, __be16 vlan_proto
);
190 static int igb_ndo_set_vf_bw(struct net_device
*, int, int, int);
191 static int igb_ndo_set_vf_spoofchk(struct net_device
*netdev
, int vf
,
193 static int igb_ndo_get_vf_config(struct net_device
*netdev
, int vf
,
194 struct ifla_vf_info
*ivi
);
195 static void igb_check_vf_rate_limit(struct igb_adapter
*);
196 static void igb_nfc_filter_exit(struct igb_adapter
*adapter
);
197 static void igb_nfc_filter_restore(struct igb_adapter
*adapter
);
199 #ifdef CONFIG_PCI_IOV
200 static int igb_vf_configure(struct igb_adapter
*adapter
, int vf
);
201 static int igb_pci_enable_sriov(struct pci_dev
*dev
, int num_vfs
);
202 static int igb_disable_sriov(struct pci_dev
*dev
);
203 static int igb_pci_disable_sriov(struct pci_dev
*dev
);
206 static int igb_suspend(struct device
*);
207 static int igb_resume(struct device
*);
208 static int igb_runtime_suspend(struct device
*dev
);
209 static int igb_runtime_resume(struct device
*dev
);
210 static int igb_runtime_idle(struct device
*dev
);
211 static const struct dev_pm_ops igb_pm_ops
= {
212 SET_SYSTEM_SLEEP_PM_OPS(igb_suspend
, igb_resume
)
213 SET_RUNTIME_PM_OPS(igb_runtime_suspend
, igb_runtime_resume
,
216 static void igb_shutdown(struct pci_dev
*);
217 static int igb_pci_sriov_configure(struct pci_dev
*dev
, int num_vfs
);
218 #ifdef CONFIG_IGB_DCA
219 static int igb_notify_dca(struct notifier_block
*, unsigned long, void *);
220 static struct notifier_block dca_notifier
= {
221 .notifier_call
= igb_notify_dca
,
226 #ifdef CONFIG_NET_POLL_CONTROLLER
227 /* for netdump / net console */
228 static void igb_netpoll(struct net_device
*);
230 #ifdef CONFIG_PCI_IOV
231 static unsigned int max_vfs
;
232 module_param(max_vfs
, uint
, 0);
233 MODULE_PARM_DESC(max_vfs
, "Maximum number of virtual functions to allocate per physical function");
234 #endif /* CONFIG_PCI_IOV */
236 static pci_ers_result_t
igb_io_error_detected(struct pci_dev
*,
237 pci_channel_state_t
);
238 static pci_ers_result_t
igb_io_slot_reset(struct pci_dev
*);
239 static void igb_io_resume(struct pci_dev
*);
241 static const struct pci_error_handlers igb_err_handler
= {
242 .error_detected
= igb_io_error_detected
,
243 .slot_reset
= igb_io_slot_reset
,
244 .resume
= igb_io_resume
,
247 static void igb_init_dmac(struct igb_adapter
*adapter
, u32 pba
);
249 static struct pci_driver igb_driver
= {
250 .name
= igb_driver_name
,
251 .id_table
= igb_pci_tbl
,
253 .remove
= igb_remove
,
255 .driver
.pm
= &igb_pm_ops
,
257 .shutdown
= igb_shutdown
,
258 .sriov_configure
= igb_pci_sriov_configure
,
259 .err_handler
= &igb_err_handler
262 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
263 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
264 MODULE_LICENSE("GPL");
265 MODULE_VERSION(DRV_VERSION
);
267 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
268 static int debug
= -1;
269 module_param(debug
, int, 0);
270 MODULE_PARM_DESC(debug
, "Debug level (0=none,...,16=all)");
272 struct igb_reg_info
{
277 static const struct igb_reg_info igb_reg_info_tbl
[] = {
279 /* General Registers */
280 {E1000_CTRL
, "CTRL"},
281 {E1000_STATUS
, "STATUS"},
282 {E1000_CTRL_EXT
, "CTRL_EXT"},
284 /* Interrupt Registers */
288 {E1000_RCTL
, "RCTL"},
289 {E1000_RDLEN(0), "RDLEN"},
290 {E1000_RDH(0), "RDH"},
291 {E1000_RDT(0), "RDT"},
292 {E1000_RXDCTL(0), "RXDCTL"},
293 {E1000_RDBAL(0), "RDBAL"},
294 {E1000_RDBAH(0), "RDBAH"},
297 {E1000_TCTL
, "TCTL"},
298 {E1000_TDBAL(0), "TDBAL"},
299 {E1000_TDBAH(0), "TDBAH"},
300 {E1000_TDLEN(0), "TDLEN"},
301 {E1000_TDH(0), "TDH"},
302 {E1000_TDT(0), "TDT"},
303 {E1000_TXDCTL(0), "TXDCTL"},
304 {E1000_TDFH
, "TDFH"},
305 {E1000_TDFT
, "TDFT"},
306 {E1000_TDFHS
, "TDFHS"},
307 {E1000_TDFPC
, "TDFPC"},
309 /* List Terminator */
313 /* igb_regdump - register printout routine */
314 static void igb_regdump(struct e1000_hw
*hw
, struct igb_reg_info
*reginfo
)
320 switch (reginfo
->ofs
) {
322 for (n
= 0; n
< 4; n
++)
323 regs
[n
] = rd32(E1000_RDLEN(n
));
326 for (n
= 0; n
< 4; n
++)
327 regs
[n
] = rd32(E1000_RDH(n
));
330 for (n
= 0; n
< 4; n
++)
331 regs
[n
] = rd32(E1000_RDT(n
));
333 case E1000_RXDCTL(0):
334 for (n
= 0; n
< 4; n
++)
335 regs
[n
] = rd32(E1000_RXDCTL(n
));
338 for (n
= 0; n
< 4; n
++)
339 regs
[n
] = rd32(E1000_RDBAL(n
));
342 for (n
= 0; n
< 4; n
++)
343 regs
[n
] = rd32(E1000_RDBAH(n
));
346 for (n
= 0; n
< 4; n
++)
347 regs
[n
] = rd32(E1000_RDBAL(n
));
350 for (n
= 0; n
< 4; n
++)
351 regs
[n
] = rd32(E1000_TDBAH(n
));
354 for (n
= 0; n
< 4; n
++)
355 regs
[n
] = rd32(E1000_TDLEN(n
));
358 for (n
= 0; n
< 4; n
++)
359 regs
[n
] = rd32(E1000_TDH(n
));
362 for (n
= 0; n
< 4; n
++)
363 regs
[n
] = rd32(E1000_TDT(n
));
365 case E1000_TXDCTL(0):
366 for (n
= 0; n
< 4; n
++)
367 regs
[n
] = rd32(E1000_TXDCTL(n
));
370 pr_info("%-15s %08x\n", reginfo
->name
, rd32(reginfo
->ofs
));
374 snprintf(rname
, 16, "%s%s", reginfo
->name
, "[0-3]");
375 pr_info("%-15s %08x %08x %08x %08x\n", rname
, regs
[0], regs
[1],
379 /* igb_dump - Print registers, Tx-rings and Rx-rings */
380 static void igb_dump(struct igb_adapter
*adapter
)
382 struct net_device
*netdev
= adapter
->netdev
;
383 struct e1000_hw
*hw
= &adapter
->hw
;
384 struct igb_reg_info
*reginfo
;
385 struct igb_ring
*tx_ring
;
386 union e1000_adv_tx_desc
*tx_desc
;
387 struct my_u0
{ u64 a
; u64 b
; } *u0
;
388 struct igb_ring
*rx_ring
;
389 union e1000_adv_rx_desc
*rx_desc
;
393 if (!netif_msg_hw(adapter
))
396 /* Print netdevice Info */
398 dev_info(&adapter
->pdev
->dev
, "Net device Info\n");
399 pr_info("Device Name state trans_start\n");
400 pr_info("%-15s %016lX %016lX\n", netdev
->name
,
401 netdev
->state
, dev_trans_start(netdev
));
404 /* Print Registers */
405 dev_info(&adapter
->pdev
->dev
, "Register Dump\n");
406 pr_info(" Register Name Value\n");
407 for (reginfo
= (struct igb_reg_info
*)igb_reg_info_tbl
;
408 reginfo
->name
; reginfo
++) {
409 igb_regdump(hw
, reginfo
);
412 /* Print TX Ring Summary */
413 if (!netdev
|| !netif_running(netdev
))
416 dev_info(&adapter
->pdev
->dev
, "TX Rings Summary\n");
417 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
418 for (n
= 0; n
< adapter
->num_tx_queues
; n
++) {
419 struct igb_tx_buffer
*buffer_info
;
420 tx_ring
= adapter
->tx_ring
[n
];
421 buffer_info
= &tx_ring
->tx_buffer_info
[tx_ring
->next_to_clean
];
422 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
423 n
, tx_ring
->next_to_use
, tx_ring
->next_to_clean
,
424 (u64
)dma_unmap_addr(buffer_info
, dma
),
425 dma_unmap_len(buffer_info
, len
),
426 buffer_info
->next_to_watch
,
427 (u64
)buffer_info
->time_stamp
);
431 if (!netif_msg_tx_done(adapter
))
432 goto rx_ring_summary
;
434 dev_info(&adapter
->pdev
->dev
, "TX Rings Dump\n");
436 /* Transmit Descriptor Formats
438 * Advanced Transmit Descriptor
439 * +--------------------------------------------------------------+
440 * 0 | Buffer Address [63:0] |
441 * +--------------------------------------------------------------+
442 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
443 * +--------------------------------------------------------------+
444 * 63 46 45 40 39 38 36 35 32 31 24 15 0
447 for (n
= 0; n
< adapter
->num_tx_queues
; n
++) {
448 tx_ring
= adapter
->tx_ring
[n
];
449 pr_info("------------------------------------\n");
450 pr_info("TX QUEUE INDEX = %d\n", tx_ring
->queue_index
);
451 pr_info("------------------------------------\n");
452 pr_info("T [desc] [address 63:0 ] [PlPOCIStDDM Ln] [bi->dma ] leng ntw timestamp bi->skb\n");
454 for (i
= 0; tx_ring
->desc
&& (i
< tx_ring
->count
); i
++) {
455 const char *next_desc
;
456 struct igb_tx_buffer
*buffer_info
;
457 tx_desc
= IGB_TX_DESC(tx_ring
, i
);
458 buffer_info
= &tx_ring
->tx_buffer_info
[i
];
459 u0
= (struct my_u0
*)tx_desc
;
460 if (i
== tx_ring
->next_to_use
&&
461 i
== tx_ring
->next_to_clean
)
462 next_desc
= " NTC/U";
463 else if (i
== tx_ring
->next_to_use
)
465 else if (i
== tx_ring
->next_to_clean
)
470 pr_info("T [0x%03X] %016llX %016llX %016llX %04X %p %016llX %p%s\n",
471 i
, le64_to_cpu(u0
->a
),
473 (u64
)dma_unmap_addr(buffer_info
, dma
),
474 dma_unmap_len(buffer_info
, len
),
475 buffer_info
->next_to_watch
,
476 (u64
)buffer_info
->time_stamp
,
477 buffer_info
->skb
, next_desc
);
479 if (netif_msg_pktdata(adapter
) && buffer_info
->skb
)
480 print_hex_dump(KERN_INFO
, "",
482 16, 1, buffer_info
->skb
->data
,
483 dma_unmap_len(buffer_info
, len
),
488 /* Print RX Rings Summary */
490 dev_info(&adapter
->pdev
->dev
, "RX Rings Summary\n");
491 pr_info("Queue [NTU] [NTC]\n");
492 for (n
= 0; n
< adapter
->num_rx_queues
; n
++) {
493 rx_ring
= adapter
->rx_ring
[n
];
494 pr_info(" %5d %5X %5X\n",
495 n
, rx_ring
->next_to_use
, rx_ring
->next_to_clean
);
499 if (!netif_msg_rx_status(adapter
))
502 dev_info(&adapter
->pdev
->dev
, "RX Rings Dump\n");
504 /* Advanced Receive Descriptor (Read) Format
506 * +-----------------------------------------------------+
507 * 0 | Packet Buffer Address [63:1] |A0/NSE|
508 * +----------------------------------------------+------+
509 * 8 | Header Buffer Address [63:1] | DD |
510 * +-----------------------------------------------------+
513 * Advanced Receive Descriptor (Write-Back) Format
515 * 63 48 47 32 31 30 21 20 17 16 4 3 0
516 * +------------------------------------------------------+
517 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
518 * | Checksum Ident | | | | Type | Type |
519 * +------------------------------------------------------+
520 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
521 * +------------------------------------------------------+
522 * 63 48 47 32 31 20 19 0
525 for (n
= 0; n
< adapter
->num_rx_queues
; n
++) {
526 rx_ring
= adapter
->rx_ring
[n
];
527 pr_info("------------------------------------\n");
528 pr_info("RX QUEUE INDEX = %d\n", rx_ring
->queue_index
);
529 pr_info("------------------------------------\n");
530 pr_info("R [desc] [ PktBuf A0] [ HeadBuf DD] [bi->dma ] [bi->skb] <-- Adv Rx Read format\n");
531 pr_info("RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] ---------------- [bi->skb] <-- Adv Rx Write-Back format\n");
533 for (i
= 0; i
< rx_ring
->count
; i
++) {
534 const char *next_desc
;
535 struct igb_rx_buffer
*buffer_info
;
536 buffer_info
= &rx_ring
->rx_buffer_info
[i
];
537 rx_desc
= IGB_RX_DESC(rx_ring
, i
);
538 u0
= (struct my_u0
*)rx_desc
;
539 staterr
= le32_to_cpu(rx_desc
->wb
.upper
.status_error
);
541 if (i
== rx_ring
->next_to_use
)
543 else if (i
== rx_ring
->next_to_clean
)
548 if (staterr
& E1000_RXD_STAT_DD
) {
549 /* Descriptor Done */
550 pr_info("%s[0x%03X] %016llX %016llX ---------------- %s\n",
556 pr_info("%s[0x%03X] %016llX %016llX %016llX %s\n",
560 (u64
)buffer_info
->dma
,
563 if (netif_msg_pktdata(adapter
) &&
564 buffer_info
->dma
&& buffer_info
->page
) {
565 print_hex_dump(KERN_INFO
, "",
568 page_address(buffer_info
->page
) +
569 buffer_info
->page_offset
,
570 igb_rx_bufsz(rx_ring
), true);
581 * igb_get_i2c_data - Reads the I2C SDA data bit
582 * @hw: pointer to hardware structure
583 * @i2cctl: Current value of I2CCTL register
585 * Returns the I2C data bit value
587 static int igb_get_i2c_data(void *data
)
589 struct igb_adapter
*adapter
= (struct igb_adapter
*)data
;
590 struct e1000_hw
*hw
= &adapter
->hw
;
591 s32 i2cctl
= rd32(E1000_I2CPARAMS
);
593 return !!(i2cctl
& E1000_I2C_DATA_IN
);
597 * igb_set_i2c_data - Sets the I2C data bit
598 * @data: pointer to hardware structure
599 * @state: I2C data value (0 or 1) to set
601 * Sets the I2C data bit
603 static void igb_set_i2c_data(void *data
, int state
)
605 struct igb_adapter
*adapter
= (struct igb_adapter
*)data
;
606 struct e1000_hw
*hw
= &adapter
->hw
;
607 s32 i2cctl
= rd32(E1000_I2CPARAMS
);
610 i2cctl
|= E1000_I2C_DATA_OUT
;
612 i2cctl
&= ~E1000_I2C_DATA_OUT
;
614 i2cctl
&= ~E1000_I2C_DATA_OE_N
;
615 i2cctl
|= E1000_I2C_CLK_OE_N
;
616 wr32(E1000_I2CPARAMS
, i2cctl
);
622 * igb_set_i2c_clk - Sets the I2C SCL clock
623 * @data: pointer to hardware structure
624 * @state: state to set clock
626 * Sets the I2C clock line to state
628 static void igb_set_i2c_clk(void *data
, int state
)
630 struct igb_adapter
*adapter
= (struct igb_adapter
*)data
;
631 struct e1000_hw
*hw
= &adapter
->hw
;
632 s32 i2cctl
= rd32(E1000_I2CPARAMS
);
635 i2cctl
|= E1000_I2C_CLK_OUT
;
636 i2cctl
&= ~E1000_I2C_CLK_OE_N
;
638 i2cctl
&= ~E1000_I2C_CLK_OUT
;
639 i2cctl
&= ~E1000_I2C_CLK_OE_N
;
641 wr32(E1000_I2CPARAMS
, i2cctl
);
646 * igb_get_i2c_clk - Gets the I2C SCL clock state
647 * @data: pointer to hardware structure
649 * Gets the I2C clock state
651 static int igb_get_i2c_clk(void *data
)
653 struct igb_adapter
*adapter
= (struct igb_adapter
*)data
;
654 struct e1000_hw
*hw
= &adapter
->hw
;
655 s32 i2cctl
= rd32(E1000_I2CPARAMS
);
657 return !!(i2cctl
& E1000_I2C_CLK_IN
);
660 static const struct i2c_algo_bit_data igb_i2c_algo
= {
661 .setsda
= igb_set_i2c_data
,
662 .setscl
= igb_set_i2c_clk
,
663 .getsda
= igb_get_i2c_data
,
664 .getscl
= igb_get_i2c_clk
,
670 * igb_get_hw_dev - return device
671 * @hw: pointer to hardware structure
673 * used by hardware layer to print debugging information
675 struct net_device
*igb_get_hw_dev(struct e1000_hw
*hw
)
677 struct igb_adapter
*adapter
= hw
->back
;
678 return adapter
->netdev
;
682 * igb_init_module - Driver Registration Routine
684 * igb_init_module is the first routine called when the driver is
685 * loaded. All it does is register with the PCI subsystem.
687 static int __init
igb_init_module(void)
691 pr_info("%s - version %s\n",
692 igb_driver_string
, igb_driver_version
);
693 pr_info("%s\n", igb_copyright
);
695 #ifdef CONFIG_IGB_DCA
696 dca_register_notify(&dca_notifier
);
698 ret
= pci_register_driver(&igb_driver
);
702 module_init(igb_init_module
);
705 * igb_exit_module - Driver Exit Cleanup Routine
707 * igb_exit_module is called just before the driver is removed
710 static void __exit
igb_exit_module(void)
712 #ifdef CONFIG_IGB_DCA
713 dca_unregister_notify(&dca_notifier
);
715 pci_unregister_driver(&igb_driver
);
718 module_exit(igb_exit_module
);
720 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
722 * igb_cache_ring_register - Descriptor ring to register mapping
723 * @adapter: board private structure to initialize
725 * Once we know the feature-set enabled for the device, we'll cache
726 * the register offset the descriptor ring is assigned to.
728 static void igb_cache_ring_register(struct igb_adapter
*adapter
)
731 u32 rbase_offset
= adapter
->vfs_allocated_count
;
733 switch (adapter
->hw
.mac
.type
) {
735 /* The queues are allocated for virtualization such that VF 0
736 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
737 * In order to avoid collision we start at the first free queue
738 * and continue consuming queues in the same sequence
740 if (adapter
->vfs_allocated_count
) {
741 for (; i
< adapter
->rss_queues
; i
++)
742 adapter
->rx_ring
[i
]->reg_idx
= rbase_offset
+
754 for (; i
< adapter
->num_rx_queues
; i
++)
755 adapter
->rx_ring
[i
]->reg_idx
= rbase_offset
+ i
;
756 for (; j
< adapter
->num_tx_queues
; j
++)
757 adapter
->tx_ring
[j
]->reg_idx
= rbase_offset
+ j
;
762 u32
igb_rd32(struct e1000_hw
*hw
, u32 reg
)
764 struct igb_adapter
*igb
= container_of(hw
, struct igb_adapter
, hw
);
765 u8 __iomem
*hw_addr
= ACCESS_ONCE(hw
->hw_addr
);
768 if (E1000_REMOVED(hw_addr
))
771 value
= readl(&hw_addr
[reg
]);
773 /* reads should not return all F's */
774 if (!(~value
) && (!reg
|| !(~readl(hw_addr
)))) {
775 struct net_device
*netdev
= igb
->netdev
;
777 netif_device_detach(netdev
);
778 netdev_err(netdev
, "PCIe link lost, device now detached\n");
785 * igb_write_ivar - configure ivar for given MSI-X vector
786 * @hw: pointer to the HW structure
787 * @msix_vector: vector number we are allocating to a given ring
788 * @index: row index of IVAR register to write within IVAR table
789 * @offset: column offset of in IVAR, should be multiple of 8
791 * This function is intended to handle the writing of the IVAR register
792 * for adapters 82576 and newer. The IVAR table consists of 2 columns,
793 * each containing an cause allocation for an Rx and Tx ring, and a
794 * variable number of rows depending on the number of queues supported.
796 static void igb_write_ivar(struct e1000_hw
*hw
, int msix_vector
,
797 int index
, int offset
)
799 u32 ivar
= array_rd32(E1000_IVAR0
, index
);
801 /* clear any bits that are currently set */
802 ivar
&= ~((u32
)0xFF << offset
);
804 /* write vector and valid bit */
805 ivar
|= (msix_vector
| E1000_IVAR_VALID
) << offset
;
807 array_wr32(E1000_IVAR0
, index
, ivar
);
810 #define IGB_N0_QUEUE -1
811 static void igb_assign_vector(struct igb_q_vector
*q_vector
, int msix_vector
)
813 struct igb_adapter
*adapter
= q_vector
->adapter
;
814 struct e1000_hw
*hw
= &adapter
->hw
;
815 int rx_queue
= IGB_N0_QUEUE
;
816 int tx_queue
= IGB_N0_QUEUE
;
819 if (q_vector
->rx
.ring
)
820 rx_queue
= q_vector
->rx
.ring
->reg_idx
;
821 if (q_vector
->tx
.ring
)
822 tx_queue
= q_vector
->tx
.ring
->reg_idx
;
824 switch (hw
->mac
.type
) {
826 /* The 82575 assigns vectors using a bitmask, which matches the
827 * bitmask for the EICR/EIMS/EIMC registers. To assign one
828 * or more queues to a vector, we write the appropriate bits
829 * into the MSIXBM register for that vector.
831 if (rx_queue
> IGB_N0_QUEUE
)
832 msixbm
= E1000_EICR_RX_QUEUE0
<< rx_queue
;
833 if (tx_queue
> IGB_N0_QUEUE
)
834 msixbm
|= E1000_EICR_TX_QUEUE0
<< tx_queue
;
835 if (!(adapter
->flags
& IGB_FLAG_HAS_MSIX
) && msix_vector
== 0)
836 msixbm
|= E1000_EIMS_OTHER
;
837 array_wr32(E1000_MSIXBM(0), msix_vector
, msixbm
);
838 q_vector
->eims_value
= msixbm
;
841 /* 82576 uses a table that essentially consists of 2 columns
842 * with 8 rows. The ordering is column-major so we use the
843 * lower 3 bits as the row index, and the 4th bit as the
846 if (rx_queue
> IGB_N0_QUEUE
)
847 igb_write_ivar(hw
, msix_vector
,
849 (rx_queue
& 0x8) << 1);
850 if (tx_queue
> IGB_N0_QUEUE
)
851 igb_write_ivar(hw
, msix_vector
,
853 ((tx_queue
& 0x8) << 1) + 8);
854 q_vector
->eims_value
= BIT(msix_vector
);
861 /* On 82580 and newer adapters the scheme is similar to 82576
862 * however instead of ordering column-major we have things
863 * ordered row-major. So we traverse the table by using
864 * bit 0 as the column offset, and the remaining bits as the
867 if (rx_queue
> IGB_N0_QUEUE
)
868 igb_write_ivar(hw
, msix_vector
,
870 (rx_queue
& 0x1) << 4);
871 if (tx_queue
> IGB_N0_QUEUE
)
872 igb_write_ivar(hw
, msix_vector
,
874 ((tx_queue
& 0x1) << 4) + 8);
875 q_vector
->eims_value
= BIT(msix_vector
);
882 /* add q_vector eims value to global eims_enable_mask */
883 adapter
->eims_enable_mask
|= q_vector
->eims_value
;
885 /* configure q_vector to set itr on first interrupt */
886 q_vector
->set_itr
= 1;
890 * igb_configure_msix - Configure MSI-X hardware
891 * @adapter: board private structure to initialize
893 * igb_configure_msix sets up the hardware to properly
894 * generate MSI-X interrupts.
896 static void igb_configure_msix(struct igb_adapter
*adapter
)
900 struct e1000_hw
*hw
= &adapter
->hw
;
902 adapter
->eims_enable_mask
= 0;
904 /* set vector for other causes, i.e. link changes */
905 switch (hw
->mac
.type
) {
907 tmp
= rd32(E1000_CTRL_EXT
);
908 /* enable MSI-X PBA support*/
909 tmp
|= E1000_CTRL_EXT_PBA_CLR
;
911 /* Auto-Mask interrupts upon ICR read. */
912 tmp
|= E1000_CTRL_EXT_EIAME
;
913 tmp
|= E1000_CTRL_EXT_IRCA
;
915 wr32(E1000_CTRL_EXT
, tmp
);
917 /* enable msix_other interrupt */
918 array_wr32(E1000_MSIXBM(0), vector
++, E1000_EIMS_OTHER
);
919 adapter
->eims_other
= E1000_EIMS_OTHER
;
929 /* Turn on MSI-X capability first, or our settings
930 * won't stick. And it will take days to debug.
932 wr32(E1000_GPIE
, E1000_GPIE_MSIX_MODE
|
933 E1000_GPIE_PBA
| E1000_GPIE_EIAME
|
936 /* enable msix_other interrupt */
937 adapter
->eims_other
= BIT(vector
);
938 tmp
= (vector
++ | E1000_IVAR_VALID
) << 8;
940 wr32(E1000_IVAR_MISC
, tmp
);
943 /* do nothing, since nothing else supports MSI-X */
945 } /* switch (hw->mac.type) */
947 adapter
->eims_enable_mask
|= adapter
->eims_other
;
949 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
950 igb_assign_vector(adapter
->q_vector
[i
], vector
++);
956 * igb_request_msix - Initialize MSI-X interrupts
957 * @adapter: board private structure to initialize
959 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
962 static int igb_request_msix(struct igb_adapter
*adapter
)
964 struct net_device
*netdev
= adapter
->netdev
;
965 int i
, err
= 0, vector
= 0, free_vector
= 0;
967 err
= request_irq(adapter
->msix_entries
[vector
].vector
,
968 igb_msix_other
, 0, netdev
->name
, adapter
);
972 for (i
= 0; i
< adapter
->num_q_vectors
; i
++) {
973 struct igb_q_vector
*q_vector
= adapter
->q_vector
[i
];
977 q_vector
->itr_register
= adapter
->io_addr
+ E1000_EITR(vector
);
979 if (q_vector
->rx
.ring
&& q_vector
->tx
.ring
)
980 sprintf(q_vector
->name
, "%s-TxRx-%u", netdev
->name
,
981 q_vector
->rx
.ring
->queue_index
);
982 else if (q_vector
->tx
.ring
)
983 sprintf(q_vector
->name
, "%s-tx-%u", netdev
->name
,
984 q_vector
->tx
.ring
->queue_index
);
985 else if (q_vector
->rx
.ring
)
986 sprintf(q_vector
->name
, "%s-rx-%u", netdev
->name
,
987 q_vector
->rx
.ring
->queue_index
);
989 sprintf(q_vector
->name
, "%s-unused", netdev
->name
);
991 err
= request_irq(adapter
->msix_entries
[vector
].vector
,
992 igb_msix_ring
, 0, q_vector
->name
,
998 igb_configure_msix(adapter
);
1002 /* free already assigned IRQs */
1003 free_irq(adapter
->msix_entries
[free_vector
++].vector
, adapter
);
1006 for (i
= 0; i
< vector
; i
++) {
1007 free_irq(adapter
->msix_entries
[free_vector
++].vector
,
1008 adapter
->q_vector
[i
]);
1015 * igb_free_q_vector - Free memory allocated for specific interrupt vector
1016 * @adapter: board private structure to initialize
1017 * @v_idx: Index of vector to be freed
1019 * This function frees the memory allocated to the q_vector.
1021 static void igb_free_q_vector(struct igb_adapter
*adapter
, int v_idx
)
1023 struct igb_q_vector
*q_vector
= adapter
->q_vector
[v_idx
];
1025 adapter
->q_vector
[v_idx
] = NULL
;
1027 /* igb_get_stats64() might access the rings on this vector,
1028 * we must wait a grace period before freeing it.
1031 kfree_rcu(q_vector
, rcu
);
1035 * igb_reset_q_vector - Reset config for interrupt vector
1036 * @adapter: board private structure to initialize
1037 * @v_idx: Index of vector to be reset
1039 * If NAPI is enabled it will delete any references to the
1040 * NAPI struct. This is preparation for igb_free_q_vector.
1042 static void igb_reset_q_vector(struct igb_adapter
*adapter
, int v_idx
)
1044 struct igb_q_vector
*q_vector
= adapter
->q_vector
[v_idx
];
1046 /* Coming from igb_set_interrupt_capability, the vectors are not yet
1047 * allocated. So, q_vector is NULL so we should stop here.
1052 if (q_vector
->tx
.ring
)
1053 adapter
->tx_ring
[q_vector
->tx
.ring
->queue_index
] = NULL
;
1055 if (q_vector
->rx
.ring
)
1056 adapter
->rx_ring
[q_vector
->rx
.ring
->queue_index
] = NULL
;
1058 netif_napi_del(&q_vector
->napi
);
1062 static void igb_reset_interrupt_capability(struct igb_adapter
*adapter
)
1064 int v_idx
= adapter
->num_q_vectors
;
1066 if (adapter
->flags
& IGB_FLAG_HAS_MSIX
)
1067 pci_disable_msix(adapter
->pdev
);
1068 else if (adapter
->flags
& IGB_FLAG_HAS_MSI
)
1069 pci_disable_msi(adapter
->pdev
);
1072 igb_reset_q_vector(adapter
, v_idx
);
1076 * igb_free_q_vectors - Free memory allocated for interrupt vectors
1077 * @adapter: board private structure to initialize
1079 * This function frees the memory allocated to the q_vectors. In addition if
1080 * NAPI is enabled it will delete any references to the NAPI struct prior
1081 * to freeing the q_vector.
1083 static void igb_free_q_vectors(struct igb_adapter
*adapter
)
1085 int v_idx
= adapter
->num_q_vectors
;
1087 adapter
->num_tx_queues
= 0;
1088 adapter
->num_rx_queues
= 0;
1089 adapter
->num_q_vectors
= 0;
1092 igb_reset_q_vector(adapter
, v_idx
);
1093 igb_free_q_vector(adapter
, v_idx
);
1098 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
1099 * @adapter: board private structure to initialize
1101 * This function resets the device so that it has 0 Rx queues, Tx queues, and
1102 * MSI-X interrupts allocated.
1104 static void igb_clear_interrupt_scheme(struct igb_adapter
*adapter
)
1106 igb_free_q_vectors(adapter
);
1107 igb_reset_interrupt_capability(adapter
);
1111 * igb_set_interrupt_capability - set MSI or MSI-X if supported
1112 * @adapter: board private structure to initialize
1113 * @msix: boolean value of MSIX capability
1115 * Attempt to configure interrupts using the best available
1116 * capabilities of the hardware and kernel.
1118 static void igb_set_interrupt_capability(struct igb_adapter
*adapter
, bool msix
)
1125 adapter
->flags
|= IGB_FLAG_HAS_MSIX
;
1127 /* Number of supported queues. */
1128 adapter
->num_rx_queues
= adapter
->rss_queues
;
1129 if (adapter
->vfs_allocated_count
)
1130 adapter
->num_tx_queues
= 1;
1132 adapter
->num_tx_queues
= adapter
->rss_queues
;
1134 /* start with one vector for every Rx queue */
1135 numvecs
= adapter
->num_rx_queues
;
1137 /* if Tx handler is separate add 1 for every Tx queue */
1138 if (!(adapter
->flags
& IGB_FLAG_QUEUE_PAIRS
))
1139 numvecs
+= adapter
->num_tx_queues
;
1141 /* store the number of vectors reserved for queues */
1142 adapter
->num_q_vectors
= numvecs
;
1144 /* add 1 vector for link status interrupts */
1146 for (i
= 0; i
< numvecs
; i
++)
1147 adapter
->msix_entries
[i
].entry
= i
;
1149 err
= pci_enable_msix_range(adapter
->pdev
,
1150 adapter
->msix_entries
,
1156 igb_reset_interrupt_capability(adapter
);
1158 /* If we can't do MSI-X, try MSI */
1160 adapter
->flags
&= ~IGB_FLAG_HAS_MSIX
;
1161 #ifdef CONFIG_PCI_IOV
1162 /* disable SR-IOV for non MSI-X configurations */
1163 if (adapter
->vf_data
) {
1164 struct e1000_hw
*hw
= &adapter
->hw
;
1165 /* disable iov and allow time for transactions to clear */
1166 pci_disable_sriov(adapter
->pdev
);
1169 kfree(adapter
->vf_mac_list
);
1170 adapter
->vf_mac_list
= NULL
;
1171 kfree(adapter
->vf_data
);
1172 adapter
->vf_data
= NULL
;
1173 wr32(E1000_IOVCTL
, E1000_IOVCTL_REUSE_VFQ
);
1176 dev_info(&adapter
->pdev
->dev
, "IOV Disabled\n");
1179 adapter
->vfs_allocated_count
= 0;
1180 adapter
->rss_queues
= 1;
1181 adapter
->flags
|= IGB_FLAG_QUEUE_PAIRS
;
1182 adapter
->num_rx_queues
= 1;
1183 adapter
->num_tx_queues
= 1;
1184 adapter
->num_q_vectors
= 1;
1185 if (!pci_enable_msi(adapter
->pdev
))
1186 adapter
->flags
|= IGB_FLAG_HAS_MSI
;
1189 static void igb_add_ring(struct igb_ring
*ring
,
1190 struct igb_ring_container
*head
)
1197 * igb_alloc_q_vector - Allocate memory for a single interrupt vector
1198 * @adapter: board private structure to initialize
1199 * @v_count: q_vectors allocated on adapter, used for ring interleaving
1200 * @v_idx: index of vector in adapter struct
1201 * @txr_count: total number of Tx rings to allocate
1202 * @txr_idx: index of first Tx ring to allocate
1203 * @rxr_count: total number of Rx rings to allocate
1204 * @rxr_idx: index of first Rx ring to allocate
1206 * We allocate one q_vector. If allocation fails we return -ENOMEM.
1208 static int igb_alloc_q_vector(struct igb_adapter
*adapter
,
1209 int v_count
, int v_idx
,
1210 int txr_count
, int txr_idx
,
1211 int rxr_count
, int rxr_idx
)
1213 struct igb_q_vector
*q_vector
;
1214 struct igb_ring
*ring
;
1215 int ring_count
, size
;
1217 /* igb only supports 1 Tx and/or 1 Rx queue per vector */
1218 if (txr_count
> 1 || rxr_count
> 1)
1221 ring_count
= txr_count
+ rxr_count
;
1222 size
= sizeof(struct igb_q_vector
) +
1223 (sizeof(struct igb_ring
) * ring_count
);
1225 /* allocate q_vector and rings */
1226 q_vector
= adapter
->q_vector
[v_idx
];
1228 q_vector
= kzalloc(size
, GFP_KERNEL
);
1229 } else if (size
> ksize(q_vector
)) {
1230 kfree_rcu(q_vector
, rcu
);
1231 q_vector
= kzalloc(size
, GFP_KERNEL
);
1233 memset(q_vector
, 0, size
);
1238 /* initialize NAPI */
1239 netif_napi_add(adapter
->netdev
, &q_vector
->napi
,
1242 /* tie q_vector and adapter together */
1243 adapter
->q_vector
[v_idx
] = q_vector
;
1244 q_vector
->adapter
= adapter
;
1246 /* initialize work limits */
1247 q_vector
->tx
.work_limit
= adapter
->tx_work_limit
;
1249 /* initialize ITR configuration */
1250 q_vector
->itr_register
= adapter
->io_addr
+ E1000_EITR(0);
1251 q_vector
->itr_val
= IGB_START_ITR
;
1253 /* initialize pointer to rings */
1254 ring
= q_vector
->ring
;
1258 /* rx or rx/tx vector */
1259 if (!adapter
->rx_itr_setting
|| adapter
->rx_itr_setting
> 3)
1260 q_vector
->itr_val
= adapter
->rx_itr_setting
;
1262 /* tx only vector */
1263 if (!adapter
->tx_itr_setting
|| adapter
->tx_itr_setting
> 3)
1264 q_vector
->itr_val
= adapter
->tx_itr_setting
;
1268 /* assign generic ring traits */
1269 ring
->dev
= &adapter
->pdev
->dev
;
1270 ring
->netdev
= adapter
->netdev
;
1272 /* configure backlink on ring */
1273 ring
->q_vector
= q_vector
;
1275 /* update q_vector Tx values */
1276 igb_add_ring(ring
, &q_vector
->tx
);
1278 /* For 82575, context index must be unique per ring. */
1279 if (adapter
->hw
.mac
.type
== e1000_82575
)
1280 set_bit(IGB_RING_FLAG_TX_CTX_IDX
, &ring
->flags
);
1282 /* apply Tx specific ring traits */
1283 ring
->count
= adapter
->tx_ring_count
;
1284 ring
->queue_index
= txr_idx
;
1286 ring
->cbs_enable
= false;
1287 ring
->idleslope
= 0;
1288 ring
->sendslope
= 0;
1292 u64_stats_init(&ring
->tx_syncp
);
1293 u64_stats_init(&ring
->tx_syncp2
);
1295 /* assign ring to adapter */
1296 adapter
->tx_ring
[txr_idx
] = ring
;
1298 /* push pointer to next ring */
1303 /* assign generic ring traits */
1304 ring
->dev
= &adapter
->pdev
->dev
;
1305 ring
->netdev
= adapter
->netdev
;
1307 /* configure backlink on ring */
1308 ring
->q_vector
= q_vector
;
1310 /* update q_vector Rx values */
1311 igb_add_ring(ring
, &q_vector
->rx
);
1313 /* set flag indicating ring supports SCTP checksum offload */
1314 if (adapter
->hw
.mac
.type
>= e1000_82576
)
1315 set_bit(IGB_RING_FLAG_RX_SCTP_CSUM
, &ring
->flags
);
1317 /* On i350, i354, i210, and i211, loopback VLAN packets
1318 * have the tag byte-swapped.
1320 if (adapter
->hw
.mac
.type
>= e1000_i350
)
1321 set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP
, &ring
->flags
);
1323 /* apply Rx specific ring traits */
1324 ring
->count
= adapter
->rx_ring_count
;
1325 ring
->queue_index
= rxr_idx
;
1327 u64_stats_init(&ring
->rx_syncp
);
1329 /* assign ring to adapter */
1330 adapter
->rx_ring
[rxr_idx
] = ring
;
1338 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1339 * @adapter: board private structure to initialize
1341 * We allocate one q_vector per queue interrupt. If allocation fails we
1344 static int igb_alloc_q_vectors(struct igb_adapter
*adapter
)
1346 int q_vectors
= adapter
->num_q_vectors
;
1347 int rxr_remaining
= adapter
->num_rx_queues
;
1348 int txr_remaining
= adapter
->num_tx_queues
;
1349 int rxr_idx
= 0, txr_idx
= 0, v_idx
= 0;
1352 if (q_vectors
>= (rxr_remaining
+ txr_remaining
)) {
1353 for (; rxr_remaining
; v_idx
++) {
1354 err
= igb_alloc_q_vector(adapter
, q_vectors
, v_idx
,
1360 /* update counts and index */
1366 for (; v_idx
< q_vectors
; v_idx
++) {
1367 int rqpv
= DIV_ROUND_UP(rxr_remaining
, q_vectors
- v_idx
);
1368 int tqpv
= DIV_ROUND_UP(txr_remaining
, q_vectors
- v_idx
);
1370 err
= igb_alloc_q_vector(adapter
, q_vectors
, v_idx
,
1371 tqpv
, txr_idx
, rqpv
, rxr_idx
);
1376 /* update counts and index */
1377 rxr_remaining
-= rqpv
;
1378 txr_remaining
-= tqpv
;
1386 adapter
->num_tx_queues
= 0;
1387 adapter
->num_rx_queues
= 0;
1388 adapter
->num_q_vectors
= 0;
1391 igb_free_q_vector(adapter
, v_idx
);
1397 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1398 * @adapter: board private structure to initialize
1399 * @msix: boolean value of MSIX capability
1401 * This function initializes the interrupts and allocates all of the queues.
1403 static int igb_init_interrupt_scheme(struct igb_adapter
*adapter
, bool msix
)
1405 struct pci_dev
*pdev
= adapter
->pdev
;
1408 igb_set_interrupt_capability(adapter
, msix
);
1410 err
= igb_alloc_q_vectors(adapter
);
1412 dev_err(&pdev
->dev
, "Unable to allocate memory for vectors\n");
1413 goto err_alloc_q_vectors
;
1416 igb_cache_ring_register(adapter
);
1420 err_alloc_q_vectors
:
1421 igb_reset_interrupt_capability(adapter
);
1426 * igb_request_irq - initialize interrupts
1427 * @adapter: board private structure to initialize
1429 * Attempts to configure interrupts using the best available
1430 * capabilities of the hardware and kernel.
1432 static int igb_request_irq(struct igb_adapter
*adapter
)
1434 struct net_device
*netdev
= adapter
->netdev
;
1435 struct pci_dev
*pdev
= adapter
->pdev
;
1438 if (adapter
->flags
& IGB_FLAG_HAS_MSIX
) {
1439 err
= igb_request_msix(adapter
);
1442 /* fall back to MSI */
1443 igb_free_all_tx_resources(adapter
);
1444 igb_free_all_rx_resources(adapter
);
1446 igb_clear_interrupt_scheme(adapter
);
1447 err
= igb_init_interrupt_scheme(adapter
, false);
1451 igb_setup_all_tx_resources(adapter
);
1452 igb_setup_all_rx_resources(adapter
);
1453 igb_configure(adapter
);
1456 igb_assign_vector(adapter
->q_vector
[0], 0);
1458 if (adapter
->flags
& IGB_FLAG_HAS_MSI
) {
1459 err
= request_irq(pdev
->irq
, igb_intr_msi
, 0,
1460 netdev
->name
, adapter
);
1464 /* fall back to legacy interrupts */
1465 igb_reset_interrupt_capability(adapter
);
1466 adapter
->flags
&= ~IGB_FLAG_HAS_MSI
;
1469 err
= request_irq(pdev
->irq
, igb_intr
, IRQF_SHARED
,
1470 netdev
->name
, adapter
);
1473 dev_err(&pdev
->dev
, "Error %d getting interrupt\n",
1480 static void igb_free_irq(struct igb_adapter
*adapter
)
1482 if (adapter
->flags
& IGB_FLAG_HAS_MSIX
) {
1485 free_irq(adapter
->msix_entries
[vector
++].vector
, adapter
);
1487 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
1488 free_irq(adapter
->msix_entries
[vector
++].vector
,
1489 adapter
->q_vector
[i
]);
1491 free_irq(adapter
->pdev
->irq
, adapter
);
1496 * igb_irq_disable - Mask off interrupt generation on the NIC
1497 * @adapter: board private structure
1499 static void igb_irq_disable(struct igb_adapter
*adapter
)
1501 struct e1000_hw
*hw
= &adapter
->hw
;
1503 /* we need to be careful when disabling interrupts. The VFs are also
1504 * mapped into these registers and so clearing the bits can cause
1505 * issues on the VF drivers so we only need to clear what we set
1507 if (adapter
->flags
& IGB_FLAG_HAS_MSIX
) {
1508 u32 regval
= rd32(E1000_EIAM
);
1510 wr32(E1000_EIAM
, regval
& ~adapter
->eims_enable_mask
);
1511 wr32(E1000_EIMC
, adapter
->eims_enable_mask
);
1512 regval
= rd32(E1000_EIAC
);
1513 wr32(E1000_EIAC
, regval
& ~adapter
->eims_enable_mask
);
1517 wr32(E1000_IMC
, ~0);
1519 if (adapter
->flags
& IGB_FLAG_HAS_MSIX
) {
1522 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
1523 synchronize_irq(adapter
->msix_entries
[i
].vector
);
1525 synchronize_irq(adapter
->pdev
->irq
);
1530 * igb_irq_enable - Enable default interrupt generation settings
1531 * @adapter: board private structure
1533 static void igb_irq_enable(struct igb_adapter
*adapter
)
1535 struct e1000_hw
*hw
= &adapter
->hw
;
1537 if (adapter
->flags
& IGB_FLAG_HAS_MSIX
) {
1538 u32 ims
= E1000_IMS_LSC
| E1000_IMS_DOUTSYNC
| E1000_IMS_DRSTA
;
1539 u32 regval
= rd32(E1000_EIAC
);
1541 wr32(E1000_EIAC
, regval
| adapter
->eims_enable_mask
);
1542 regval
= rd32(E1000_EIAM
);
1543 wr32(E1000_EIAM
, regval
| adapter
->eims_enable_mask
);
1544 wr32(E1000_EIMS
, adapter
->eims_enable_mask
);
1545 if (adapter
->vfs_allocated_count
) {
1546 wr32(E1000_MBVFIMR
, 0xFF);
1547 ims
|= E1000_IMS_VMMB
;
1549 wr32(E1000_IMS
, ims
);
1551 wr32(E1000_IMS
, IMS_ENABLE_MASK
|
1553 wr32(E1000_IAM
, IMS_ENABLE_MASK
|
1558 static void igb_update_mng_vlan(struct igb_adapter
*adapter
)
1560 struct e1000_hw
*hw
= &adapter
->hw
;
1561 u16 pf_id
= adapter
->vfs_allocated_count
;
1562 u16 vid
= adapter
->hw
.mng_cookie
.vlan_id
;
1563 u16 old_vid
= adapter
->mng_vlan_id
;
1565 if (hw
->mng_cookie
.status
& E1000_MNG_DHCP_COOKIE_STATUS_VLAN
) {
1566 /* add VID to filter table */
1567 igb_vfta_set(hw
, vid
, pf_id
, true, true);
1568 adapter
->mng_vlan_id
= vid
;
1570 adapter
->mng_vlan_id
= IGB_MNG_VLAN_NONE
;
1573 if ((old_vid
!= (u16
)IGB_MNG_VLAN_NONE
) &&
1575 !test_bit(old_vid
, adapter
->active_vlans
)) {
1576 /* remove VID from filter table */
1577 igb_vfta_set(hw
, vid
, pf_id
, false, true);
1582 * igb_release_hw_control - release control of the h/w to f/w
1583 * @adapter: address of board private structure
1585 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1586 * For ASF and Pass Through versions of f/w this means that the
1587 * driver is no longer loaded.
1589 static void igb_release_hw_control(struct igb_adapter
*adapter
)
1591 struct e1000_hw
*hw
= &adapter
->hw
;
1594 /* Let firmware take over control of h/w */
1595 ctrl_ext
= rd32(E1000_CTRL_EXT
);
1596 wr32(E1000_CTRL_EXT
,
1597 ctrl_ext
& ~E1000_CTRL_EXT_DRV_LOAD
);
1601 * igb_get_hw_control - get control of the h/w from f/w
1602 * @adapter: address of board private structure
1604 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1605 * For ASF and Pass Through versions of f/w this means that
1606 * the driver is loaded.
1608 static void igb_get_hw_control(struct igb_adapter
*adapter
)
1610 struct e1000_hw
*hw
= &adapter
->hw
;
1613 /* Let firmware know the driver has taken over */
1614 ctrl_ext
= rd32(E1000_CTRL_EXT
);
1615 wr32(E1000_CTRL_EXT
,
1616 ctrl_ext
| E1000_CTRL_EXT_DRV_LOAD
);
1619 static void enable_fqtss(struct igb_adapter
*adapter
, bool enable
)
1621 struct net_device
*netdev
= adapter
->netdev
;
1622 struct e1000_hw
*hw
= &adapter
->hw
;
1624 WARN_ON(hw
->mac
.type
!= e1000_i210
);
1627 adapter
->flags
|= IGB_FLAG_FQTSS
;
1629 adapter
->flags
&= ~IGB_FLAG_FQTSS
;
1631 if (netif_running(netdev
))
1632 schedule_work(&adapter
->reset_task
);
1635 static bool is_fqtss_enabled(struct igb_adapter
*adapter
)
1637 return (adapter
->flags
& IGB_FLAG_FQTSS
) ? true : false;
1640 static void set_tx_desc_fetch_prio(struct e1000_hw
*hw
, int queue
,
1641 enum tx_queue_prio prio
)
1645 WARN_ON(hw
->mac
.type
!= e1000_i210
);
1646 WARN_ON(queue
< 0 || queue
> 4);
1648 val
= rd32(E1000_I210_TXDCTL(queue
));
1650 if (prio
== TX_QUEUE_PRIO_HIGH
)
1651 val
|= E1000_TXDCTL_PRIORITY
;
1653 val
&= ~E1000_TXDCTL_PRIORITY
;
1655 wr32(E1000_I210_TXDCTL(queue
), val
);
1658 static void set_queue_mode(struct e1000_hw
*hw
, int queue
, enum queue_mode mode
)
1662 WARN_ON(hw
->mac
.type
!= e1000_i210
);
1663 WARN_ON(queue
< 0 || queue
> 1);
1665 val
= rd32(E1000_I210_TQAVCC(queue
));
1667 if (mode
== QUEUE_MODE_STREAM_RESERVATION
)
1668 val
|= E1000_TQAVCC_QUEUEMODE
;
1670 val
&= ~E1000_TQAVCC_QUEUEMODE
;
1672 wr32(E1000_I210_TQAVCC(queue
), val
);
1676 * igb_configure_cbs - Configure Credit-Based Shaper (CBS)
1677 * @adapter: pointer to adapter struct
1678 * @queue: queue number
1679 * @enable: true = enable CBS, false = disable CBS
1680 * @idleslope: idleSlope in kbps
1681 * @sendslope: sendSlope in kbps
1682 * @hicredit: hiCredit in bytes
1683 * @locredit: loCredit in bytes
1685 * Configure CBS for a given hardware queue. When disabling, idleslope,
1686 * sendslope, hicredit, locredit arguments are ignored. Returns 0 if
1687 * success. Negative otherwise.
1689 static void igb_configure_cbs(struct igb_adapter
*adapter
, int queue
,
1690 bool enable
, int idleslope
, int sendslope
,
1691 int hicredit
, int locredit
)
1693 struct net_device
*netdev
= adapter
->netdev
;
1694 struct e1000_hw
*hw
= &adapter
->hw
;
1698 WARN_ON(hw
->mac
.type
!= e1000_i210
);
1699 WARN_ON(queue
< 0 || queue
> 1);
1702 set_tx_desc_fetch_prio(hw
, queue
, TX_QUEUE_PRIO_HIGH
);
1703 set_queue_mode(hw
, queue
, QUEUE_MODE_STREAM_RESERVATION
);
1705 /* According to i210 datasheet section 7.2.7.7, we should set
1706 * the 'idleSlope' field from TQAVCC register following the
1709 * For 100 Mbps link speed:
1711 * value = BW * 0x7735 * 0.2 (E1)
1713 * For 1000Mbps link speed:
1715 * value = BW * 0x7735 * 2 (E2)
1717 * E1 and E2 can be merged into one equation as shown below.
1718 * Note that 'link-speed' is in Mbps.
1720 * value = BW * 0x7735 * 2 * link-speed
1721 * -------------- (E3)
1724 * 'BW' is the percentage bandwidth out of full link speed
1725 * which can be found with the following equation. Note that
1726 * idleSlope here is the parameter from this function which
1730 * ----------------- (E4)
1733 * That said, we can come up with a generic equation to
1734 * calculate the value we should set it TQAVCC register by
1735 * replacing 'BW' in E3 by E4. The resulting equation is:
1737 * value = idleSlope * 0x7735 * 2 * link-speed
1738 * ----------------- -------------- (E5)
1739 * link-speed * 1000 1000
1741 * 'link-speed' is present in both sides of the fraction so
1742 * it is canceled out. The final equation is the following:
1744 * value = idleSlope * 61034
1745 * ----------------- (E6)
1748 value
= DIV_ROUND_UP_ULL(idleslope
* 61034ULL, 1000000);
1750 tqavcc
= rd32(E1000_I210_TQAVCC(queue
));
1751 tqavcc
&= ~E1000_TQAVCC_IDLESLOPE_MASK
;
1753 wr32(E1000_I210_TQAVCC(queue
), tqavcc
);
1755 wr32(E1000_I210_TQAVHC(queue
), 0x80000000 + hicredit
* 0x7735);
1757 set_tx_desc_fetch_prio(hw
, queue
, TX_QUEUE_PRIO_LOW
);
1758 set_queue_mode(hw
, queue
, QUEUE_MODE_STRICT_PRIORITY
);
1760 /* Set idleSlope to zero. */
1761 tqavcc
= rd32(E1000_I210_TQAVCC(queue
));
1762 tqavcc
&= ~E1000_TQAVCC_IDLESLOPE_MASK
;
1763 wr32(E1000_I210_TQAVCC(queue
), tqavcc
);
1765 /* Set hiCredit to zero. */
1766 wr32(E1000_I210_TQAVHC(queue
), 0);
1769 /* XXX: In i210 controller the sendSlope and loCredit parameters from
1770 * CBS are not configurable by software so we don't do any 'controller
1771 * configuration' in respect to these parameters.
1774 netdev_dbg(netdev
, "CBS %s: queue %d idleslope %d sendslope %d hiCredit %d locredit %d\n",
1775 (enable
) ? "enabled" : "disabled", queue
,
1776 idleslope
, sendslope
, hicredit
, locredit
);
1779 static int igb_save_cbs_params(struct igb_adapter
*adapter
, int queue
,
1780 bool enable
, int idleslope
, int sendslope
,
1781 int hicredit
, int locredit
)
1783 struct igb_ring
*ring
;
1785 if (queue
< 0 || queue
> adapter
->num_tx_queues
)
1788 ring
= adapter
->tx_ring
[queue
];
1790 ring
->cbs_enable
= enable
;
1791 ring
->idleslope
= idleslope
;
1792 ring
->sendslope
= sendslope
;
1793 ring
->hicredit
= hicredit
;
1794 ring
->locredit
= locredit
;
1799 static bool is_any_cbs_enabled(struct igb_adapter
*adapter
)
1801 struct igb_ring
*ring
;
1804 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
1805 ring
= adapter
->tx_ring
[i
];
1807 if (ring
->cbs_enable
)
1814 static void igb_setup_tx_mode(struct igb_adapter
*adapter
)
1816 struct net_device
*netdev
= adapter
->netdev
;
1817 struct e1000_hw
*hw
= &adapter
->hw
;
1820 /* Only i210 controller supports changing the transmission mode. */
1821 if (hw
->mac
.type
!= e1000_i210
)
1824 if (is_fqtss_enabled(adapter
)) {
1827 /* Configure TQAVCTRL register: set transmit mode to 'Qav',
1828 * set data fetch arbitration to 'round robin' and set data
1829 * transfer arbitration to 'credit shaper algorithm.
1831 val
= rd32(E1000_I210_TQAVCTRL
);
1832 val
|= E1000_TQAVCTRL_XMIT_MODE
| E1000_TQAVCTRL_DATATRANARB
;
1833 val
&= ~E1000_TQAVCTRL_DATAFETCHARB
;
1834 wr32(E1000_I210_TQAVCTRL
, val
);
1836 /* Configure Tx and Rx packet buffers sizes as described in
1837 * i210 datasheet section 7.2.7.7.
1839 val
= rd32(E1000_TXPBS
);
1840 val
&= ~I210_TXPBSIZE_MASK
;
1841 val
|= I210_TXPBSIZE_PB0_8KB
| I210_TXPBSIZE_PB1_8KB
|
1842 I210_TXPBSIZE_PB2_4KB
| I210_TXPBSIZE_PB3_4KB
;
1843 wr32(E1000_TXPBS
, val
);
1845 val
= rd32(E1000_RXPBS
);
1846 val
&= ~I210_RXPBSIZE_MASK
;
1847 val
|= I210_RXPBSIZE_PB_32KB
;
1848 wr32(E1000_RXPBS
, val
);
1850 /* Section 8.12.9 states that MAX_TPKT_SIZE from DTXMXPKTSZ
1851 * register should not exceed the buffer size programmed in
1852 * TXPBS. The smallest buffer size programmed in TXPBS is 4kB
1853 * so according to the datasheet we should set MAX_TPKT_SIZE to
1856 * However, when we do so, no frame from queue 2 and 3 are
1857 * transmitted. It seems the MAX_TPKT_SIZE should not be great
1858 * or _equal_ to the buffer size programmed in TXPBS. For this
1859 * reason, we set set MAX_ TPKT_SIZE to (4kB - 1) / 64.
1861 val
= (4096 - 1) / 64;
1862 wr32(E1000_I210_DTXMXPKTSZ
, val
);
1864 /* Since FQTSS mode is enabled, apply any CBS configuration
1865 * previously set. If no previous CBS configuration has been
1866 * done, then the initial configuration is applied, which means
1869 max_queue
= (adapter
->num_tx_queues
< I210_SR_QUEUES_NUM
) ?
1870 adapter
->num_tx_queues
: I210_SR_QUEUES_NUM
;
1872 for (i
= 0; i
< max_queue
; i
++) {
1873 struct igb_ring
*ring
= adapter
->tx_ring
[i
];
1875 igb_configure_cbs(adapter
, i
, ring
->cbs_enable
,
1876 ring
->idleslope
, ring
->sendslope
,
1877 ring
->hicredit
, ring
->locredit
);
1880 wr32(E1000_RXPBS
, I210_RXPBSIZE_DEFAULT
);
1881 wr32(E1000_TXPBS
, I210_TXPBSIZE_DEFAULT
);
1882 wr32(E1000_I210_DTXMXPKTSZ
, I210_DTXMXPKTSZ_DEFAULT
);
1884 val
= rd32(E1000_I210_TQAVCTRL
);
1885 /* According to Section 8.12.21, the other flags we've set when
1886 * enabling FQTSS are not relevant when disabling FQTSS so we
1887 * don't set they here.
1889 val
&= ~E1000_TQAVCTRL_XMIT_MODE
;
1890 wr32(E1000_I210_TQAVCTRL
, val
);
1893 netdev_dbg(netdev
, "FQTSS %s\n", (is_fqtss_enabled(adapter
)) ?
1894 "enabled" : "disabled");
1898 * igb_configure - configure the hardware for RX and TX
1899 * @adapter: private board structure
1901 static void igb_configure(struct igb_adapter
*adapter
)
1903 struct net_device
*netdev
= adapter
->netdev
;
1906 igb_get_hw_control(adapter
);
1907 igb_set_rx_mode(netdev
);
1908 igb_setup_tx_mode(adapter
);
1910 igb_restore_vlan(adapter
);
1912 igb_setup_tctl(adapter
);
1913 igb_setup_mrqc(adapter
);
1914 igb_setup_rctl(adapter
);
1916 igb_nfc_filter_restore(adapter
);
1917 igb_configure_tx(adapter
);
1918 igb_configure_rx(adapter
);
1920 igb_rx_fifo_flush_82575(&adapter
->hw
);
1922 /* call igb_desc_unused which always leaves
1923 * at least 1 descriptor unused to make sure
1924 * next_to_use != next_to_clean
1926 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
1927 struct igb_ring
*ring
= adapter
->rx_ring
[i
];
1928 igb_alloc_rx_buffers(ring
, igb_desc_unused(ring
));
1933 * igb_power_up_link - Power up the phy/serdes link
1934 * @adapter: address of board private structure
1936 void igb_power_up_link(struct igb_adapter
*adapter
)
1938 igb_reset_phy(&adapter
->hw
);
1940 if (adapter
->hw
.phy
.media_type
== e1000_media_type_copper
)
1941 igb_power_up_phy_copper(&adapter
->hw
);
1943 igb_power_up_serdes_link_82575(&adapter
->hw
);
1945 igb_setup_link(&adapter
->hw
);
1949 * igb_power_down_link - Power down the phy/serdes link
1950 * @adapter: address of board private structure
1952 static void igb_power_down_link(struct igb_adapter
*adapter
)
1954 if (adapter
->hw
.phy
.media_type
== e1000_media_type_copper
)
1955 igb_power_down_phy_copper_82575(&adapter
->hw
);
1957 igb_shutdown_serdes_link_82575(&adapter
->hw
);
1961 * Detect and switch function for Media Auto Sense
1962 * @adapter: address of the board private structure
1964 static void igb_check_swap_media(struct igb_adapter
*adapter
)
1966 struct e1000_hw
*hw
= &adapter
->hw
;
1967 u32 ctrl_ext
, connsw
;
1968 bool swap_now
= false;
1970 ctrl_ext
= rd32(E1000_CTRL_EXT
);
1971 connsw
= rd32(E1000_CONNSW
);
1973 /* need to live swap if current media is copper and we have fiber/serdes
1977 if ((hw
->phy
.media_type
== e1000_media_type_copper
) &&
1978 (!(connsw
& E1000_CONNSW_AUTOSENSE_EN
))) {
1980 } else if (!(connsw
& E1000_CONNSW_SERDESD
)) {
1981 /* copper signal takes time to appear */
1982 if (adapter
->copper_tries
< 4) {
1983 adapter
->copper_tries
++;
1984 connsw
|= E1000_CONNSW_AUTOSENSE_CONF
;
1985 wr32(E1000_CONNSW
, connsw
);
1988 adapter
->copper_tries
= 0;
1989 if ((connsw
& E1000_CONNSW_PHYSD
) &&
1990 (!(connsw
& E1000_CONNSW_PHY_PDN
))) {
1992 connsw
&= ~E1000_CONNSW_AUTOSENSE_CONF
;
1993 wr32(E1000_CONNSW
, connsw
);
2001 switch (hw
->phy
.media_type
) {
2002 case e1000_media_type_copper
:
2003 netdev_info(adapter
->netdev
,
2004 "MAS: changing media to fiber/serdes\n");
2006 E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES
;
2007 adapter
->flags
|= IGB_FLAG_MEDIA_RESET
;
2008 adapter
->copper_tries
= 0;
2010 case e1000_media_type_internal_serdes
:
2011 case e1000_media_type_fiber
:
2012 netdev_info(adapter
->netdev
,
2013 "MAS: changing media to copper\n");
2015 ~E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES
;
2016 adapter
->flags
|= IGB_FLAG_MEDIA_RESET
;
2019 /* shouldn't get here during regular operation */
2020 netdev_err(adapter
->netdev
,
2021 "AMS: Invalid media type found, returning\n");
2024 wr32(E1000_CTRL_EXT
, ctrl_ext
);
2028 * igb_up - Open the interface and prepare it to handle traffic
2029 * @adapter: board private structure
2031 int igb_up(struct igb_adapter
*adapter
)
2033 struct e1000_hw
*hw
= &adapter
->hw
;
2036 /* hardware has been reset, we need to reload some things */
2037 igb_configure(adapter
);
2039 clear_bit(__IGB_DOWN
, &adapter
->state
);
2041 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
2042 napi_enable(&(adapter
->q_vector
[i
]->napi
));
2044 if (adapter
->flags
& IGB_FLAG_HAS_MSIX
)
2045 igb_configure_msix(adapter
);
2047 igb_assign_vector(adapter
->q_vector
[0], 0);
2049 /* Clear any pending interrupts. */
2051 igb_irq_enable(adapter
);
2053 /* notify VFs that reset has been completed */
2054 if (adapter
->vfs_allocated_count
) {
2055 u32 reg_data
= rd32(E1000_CTRL_EXT
);
2057 reg_data
|= E1000_CTRL_EXT_PFRSTD
;
2058 wr32(E1000_CTRL_EXT
, reg_data
);
2061 netif_tx_start_all_queues(adapter
->netdev
);
2063 /* start the watchdog. */
2064 hw
->mac
.get_link_status
= 1;
2065 schedule_work(&adapter
->watchdog_task
);
2067 if ((adapter
->flags
& IGB_FLAG_EEE
) &&
2068 (!hw
->dev_spec
._82575
.eee_disable
))
2069 adapter
->eee_advert
= MDIO_EEE_100TX
| MDIO_EEE_1000T
;
2074 void igb_down(struct igb_adapter
*adapter
)
2076 struct net_device
*netdev
= adapter
->netdev
;
2077 struct e1000_hw
*hw
= &adapter
->hw
;
2081 /* signal that we're down so the interrupt handler does not
2082 * reschedule our watchdog timer
2084 set_bit(__IGB_DOWN
, &adapter
->state
);
2086 /* disable receives in the hardware */
2087 rctl
= rd32(E1000_RCTL
);
2088 wr32(E1000_RCTL
, rctl
& ~E1000_RCTL_EN
);
2089 /* flush and sleep below */
2091 igb_nfc_filter_exit(adapter
);
2093 netif_carrier_off(netdev
);
2094 netif_tx_stop_all_queues(netdev
);
2096 /* disable transmits in the hardware */
2097 tctl
= rd32(E1000_TCTL
);
2098 tctl
&= ~E1000_TCTL_EN
;
2099 wr32(E1000_TCTL
, tctl
);
2100 /* flush both disables and wait for them to finish */
2102 usleep_range(10000, 11000);
2104 igb_irq_disable(adapter
);
2106 adapter
->flags
&= ~IGB_FLAG_NEED_LINK_UPDATE
;
2108 for (i
= 0; i
< adapter
->num_q_vectors
; i
++) {
2109 if (adapter
->q_vector
[i
]) {
2110 napi_synchronize(&adapter
->q_vector
[i
]->napi
);
2111 napi_disable(&adapter
->q_vector
[i
]->napi
);
2115 del_timer_sync(&adapter
->watchdog_timer
);
2116 del_timer_sync(&adapter
->phy_info_timer
);
2118 /* record the stats before reset*/
2119 spin_lock(&adapter
->stats64_lock
);
2120 igb_update_stats(adapter
);
2121 spin_unlock(&adapter
->stats64_lock
);
2123 adapter
->link_speed
= 0;
2124 adapter
->link_duplex
= 0;
2126 if (!pci_channel_offline(adapter
->pdev
))
2129 /* clear VLAN promisc flag so VFTA will be updated if necessary */
2130 adapter
->flags
&= ~IGB_FLAG_VLAN_PROMISC
;
2132 igb_clean_all_tx_rings(adapter
);
2133 igb_clean_all_rx_rings(adapter
);
2134 #ifdef CONFIG_IGB_DCA
2136 /* since we reset the hardware DCA settings were cleared */
2137 igb_setup_dca(adapter
);
2141 void igb_reinit_locked(struct igb_adapter
*adapter
)
2143 WARN_ON(in_interrupt());
2144 while (test_and_set_bit(__IGB_RESETTING
, &adapter
->state
))
2145 usleep_range(1000, 2000);
2148 clear_bit(__IGB_RESETTING
, &adapter
->state
);
2151 /** igb_enable_mas - Media Autosense re-enable after swap
2153 * @adapter: adapter struct
2155 static void igb_enable_mas(struct igb_adapter
*adapter
)
2157 struct e1000_hw
*hw
= &adapter
->hw
;
2158 u32 connsw
= rd32(E1000_CONNSW
);
2160 /* configure for SerDes media detect */
2161 if ((hw
->phy
.media_type
== e1000_media_type_copper
) &&
2162 (!(connsw
& E1000_CONNSW_SERDESD
))) {
2163 connsw
|= E1000_CONNSW_ENRGSRC
;
2164 connsw
|= E1000_CONNSW_AUTOSENSE_EN
;
2165 wr32(E1000_CONNSW
, connsw
);
2170 void igb_reset(struct igb_adapter
*adapter
)
2172 struct pci_dev
*pdev
= adapter
->pdev
;
2173 struct e1000_hw
*hw
= &adapter
->hw
;
2174 struct e1000_mac_info
*mac
= &hw
->mac
;
2175 struct e1000_fc_info
*fc
= &hw
->fc
;
2178 /* Repartition Pba for greater than 9k mtu
2179 * To take effect CTRL.RST is required.
2181 switch (mac
->type
) {
2185 pba
= rd32(E1000_RXPBS
);
2186 pba
= igb_rxpbs_adjust_82580(pba
);
2189 pba
= rd32(E1000_RXPBS
);
2190 pba
&= E1000_RXPBS_SIZE_MASK_82576
;
2196 pba
= E1000_PBA_34K
;
2200 if (mac
->type
== e1000_82575
) {
2201 u32 min_rx_space
, min_tx_space
, needed_tx_space
;
2203 /* write Rx PBA so that hardware can report correct Tx PBA */
2204 wr32(E1000_PBA
, pba
);
2206 /* To maintain wire speed transmits, the Tx FIFO should be
2207 * large enough to accommodate two full transmit packets,
2208 * rounded up to the next 1KB and expressed in KB. Likewise,
2209 * the Rx FIFO should be large enough to accommodate at least
2210 * one full receive packet and is similarly rounded up and
2213 min_rx_space
= DIV_ROUND_UP(MAX_JUMBO_FRAME_SIZE
, 1024);
2215 /* The Tx FIFO also stores 16 bytes of information about the Tx
2216 * but don't include Ethernet FCS because hardware appends it.
2217 * We only need to round down to the nearest 512 byte block
2218 * count since the value we care about is 2 frames, not 1.
2220 min_tx_space
= adapter
->max_frame_size
;
2221 min_tx_space
+= sizeof(union e1000_adv_tx_desc
) - ETH_FCS_LEN
;
2222 min_tx_space
= DIV_ROUND_UP(min_tx_space
, 512);
2224 /* upper 16 bits has Tx packet buffer allocation size in KB */
2225 needed_tx_space
= min_tx_space
- (rd32(E1000_PBA
) >> 16);
2227 /* If current Tx allocation is less than the min Tx FIFO size,
2228 * and the min Tx FIFO size is less than the current Rx FIFO
2229 * allocation, take space away from current Rx allocation.
2231 if (needed_tx_space
< pba
) {
2232 pba
-= needed_tx_space
;
2234 /* if short on Rx space, Rx wins and must trump Tx
2237 if (pba
< min_rx_space
)
2241 /* adjust PBA for jumbo frames */
2242 wr32(E1000_PBA
, pba
);
2245 /* flow control settings
2246 * The high water mark must be low enough to fit one full frame
2247 * after transmitting the pause frame. As such we must have enough
2248 * space to allow for us to complete our current transmit and then
2249 * receive the frame that is in progress from the link partner.
2251 * - the full Rx FIFO size minus one full Tx plus one full Rx frame
2253 hwm
= (pba
<< 10) - (adapter
->max_frame_size
+ MAX_JUMBO_FRAME_SIZE
);
2255 fc
->high_water
= hwm
& 0xFFFFFFF0; /* 16-byte granularity */
2256 fc
->low_water
= fc
->high_water
- 16;
2257 fc
->pause_time
= 0xFFFF;
2259 fc
->current_mode
= fc
->requested_mode
;
2261 /* disable receive for all VFs and wait one second */
2262 if (adapter
->vfs_allocated_count
) {
2265 for (i
= 0 ; i
< adapter
->vfs_allocated_count
; i
++)
2266 adapter
->vf_data
[i
].flags
&= IGB_VF_FLAG_PF_SET_MAC
;
2268 /* ping all the active vfs to let them know we are going down */
2269 igb_ping_all_vfs(adapter
);
2271 /* disable transmits and receives */
2272 wr32(E1000_VFRE
, 0);
2273 wr32(E1000_VFTE
, 0);
2276 /* Allow time for pending master requests to run */
2277 hw
->mac
.ops
.reset_hw(hw
);
2280 if (adapter
->flags
& IGB_FLAG_MEDIA_RESET
) {
2281 /* need to resetup here after media swap */
2282 adapter
->ei
.get_invariants(hw
);
2283 adapter
->flags
&= ~IGB_FLAG_MEDIA_RESET
;
2285 if ((mac
->type
== e1000_82575
) &&
2286 (adapter
->flags
& IGB_FLAG_MAS_ENABLE
)) {
2287 igb_enable_mas(adapter
);
2289 if (hw
->mac
.ops
.init_hw(hw
))
2290 dev_err(&pdev
->dev
, "Hardware Error\n");
2292 /* RAR registers were cleared during init_hw, clear mac table */
2293 igb_flush_mac_table(adapter
);
2294 __dev_uc_unsync(adapter
->netdev
, NULL
);
2296 /* Recover default RAR entry */
2297 igb_set_default_mac_filter(adapter
);
2299 /* Flow control settings reset on hardware reset, so guarantee flow
2300 * control is off when forcing speed.
2302 if (!hw
->mac
.autoneg
)
2303 igb_force_mac_fc(hw
);
2305 igb_init_dmac(adapter
, pba
);
2306 #ifdef CONFIG_IGB_HWMON
2307 /* Re-initialize the thermal sensor on i350 devices. */
2308 if (!test_bit(__IGB_DOWN
, &adapter
->state
)) {
2309 if (mac
->type
== e1000_i350
&& hw
->bus
.func
== 0) {
2310 /* If present, re-initialize the external thermal sensor
2314 mac
->ops
.init_thermal_sensor_thresh(hw
);
2318 /* Re-establish EEE setting */
2319 if (hw
->phy
.media_type
== e1000_media_type_copper
) {
2320 switch (mac
->type
) {
2324 igb_set_eee_i350(hw
, true, true);
2327 igb_set_eee_i354(hw
, true, true);
2333 if (!netif_running(adapter
->netdev
))
2334 igb_power_down_link(adapter
);
2336 igb_update_mng_vlan(adapter
);
2338 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
2339 wr32(E1000_VET
, ETHERNET_IEEE_VLAN_TYPE
);
2341 /* Re-enable PTP, where applicable. */
2342 if (adapter
->ptp_flags
& IGB_PTP_ENABLED
)
2343 igb_ptp_reset(adapter
);
2345 igb_get_phy_info(hw
);
2348 static netdev_features_t
igb_fix_features(struct net_device
*netdev
,
2349 netdev_features_t features
)
2351 /* Since there is no support for separate Rx/Tx vlan accel
2352 * enable/disable make sure Tx flag is always in same state as Rx.
2354 if (features
& NETIF_F_HW_VLAN_CTAG_RX
)
2355 features
|= NETIF_F_HW_VLAN_CTAG_TX
;
2357 features
&= ~NETIF_F_HW_VLAN_CTAG_TX
;
2362 static int igb_set_features(struct net_device
*netdev
,
2363 netdev_features_t features
)
2365 netdev_features_t changed
= netdev
->features
^ features
;
2366 struct igb_adapter
*adapter
= netdev_priv(netdev
);
2368 if (changed
& NETIF_F_HW_VLAN_CTAG_RX
)
2369 igb_vlan_mode(netdev
, features
);
2371 if (!(changed
& (NETIF_F_RXALL
| NETIF_F_NTUPLE
)))
2374 if (!(features
& NETIF_F_NTUPLE
)) {
2375 struct hlist_node
*node2
;
2376 struct igb_nfc_filter
*rule
;
2378 spin_lock(&adapter
->nfc_lock
);
2379 hlist_for_each_entry_safe(rule
, node2
,
2380 &adapter
->nfc_filter_list
, nfc_node
) {
2381 igb_erase_filter(adapter
, rule
);
2382 hlist_del(&rule
->nfc_node
);
2385 spin_unlock(&adapter
->nfc_lock
);
2386 adapter
->nfc_filter_count
= 0;
2389 netdev
->features
= features
;
2391 if (netif_running(netdev
))
2392 igb_reinit_locked(adapter
);
2399 static int igb_ndo_fdb_add(struct ndmsg
*ndm
, struct nlattr
*tb
[],
2400 struct net_device
*dev
,
2401 const unsigned char *addr
, u16 vid
,
2404 /* guarantee we can provide a unique filter for the unicast address */
2405 if (is_unicast_ether_addr(addr
) || is_link_local_ether_addr(addr
)) {
2406 struct igb_adapter
*adapter
= netdev_priv(dev
);
2407 int vfn
= adapter
->vfs_allocated_count
;
2409 if (netdev_uc_count(dev
) >= igb_available_rars(adapter
, vfn
))
2413 return ndo_dflt_fdb_add(ndm
, tb
, dev
, addr
, vid
, flags
);
2416 #define IGB_MAX_MAC_HDR_LEN 127
2417 #define IGB_MAX_NETWORK_HDR_LEN 511
2419 static netdev_features_t
2420 igb_features_check(struct sk_buff
*skb
, struct net_device
*dev
,
2421 netdev_features_t features
)
2423 unsigned int network_hdr_len
, mac_hdr_len
;
2425 /* Make certain the headers can be described by a context descriptor */
2426 mac_hdr_len
= skb_network_header(skb
) - skb
->data
;
2427 if (unlikely(mac_hdr_len
> IGB_MAX_MAC_HDR_LEN
))
2428 return features
& ~(NETIF_F_HW_CSUM
|
2430 NETIF_F_HW_VLAN_CTAG_TX
|
2434 network_hdr_len
= skb_checksum_start(skb
) - skb_network_header(skb
);
2435 if (unlikely(network_hdr_len
> IGB_MAX_NETWORK_HDR_LEN
))
2436 return features
& ~(NETIF_F_HW_CSUM
|
2441 /* We can only support IPV4 TSO in tunnels if we can mangle the
2442 * inner IP ID field, so strip TSO if MANGLEID is not supported.
2444 if (skb
->encapsulation
&& !(features
& NETIF_F_TSO_MANGLEID
))
2445 features
&= ~NETIF_F_TSO
;
2450 static int igb_offload_cbs(struct igb_adapter
*adapter
,
2451 struct tc_cbs_qopt_offload
*qopt
)
2453 struct e1000_hw
*hw
= &adapter
->hw
;
2456 /* CBS offloading is only supported by i210 controller. */
2457 if (hw
->mac
.type
!= e1000_i210
)
2460 /* CBS offloading is only supported by queue 0 and queue 1. */
2461 if (qopt
->queue
< 0 || qopt
->queue
> 1)
2464 err
= igb_save_cbs_params(adapter
, qopt
->queue
, qopt
->enable
,
2465 qopt
->idleslope
, qopt
->sendslope
,
2466 qopt
->hicredit
, qopt
->locredit
);
2470 if (is_fqtss_enabled(adapter
)) {
2471 igb_configure_cbs(adapter
, qopt
->queue
, qopt
->enable
,
2472 qopt
->idleslope
, qopt
->sendslope
,
2473 qopt
->hicredit
, qopt
->locredit
);
2475 if (!is_any_cbs_enabled(adapter
))
2476 enable_fqtss(adapter
, false);
2479 enable_fqtss(adapter
, true);
2485 static int igb_setup_tc(struct net_device
*dev
, enum tc_setup_type type
,
2488 struct igb_adapter
*adapter
= netdev_priv(dev
);
2492 return igb_offload_cbs(adapter
, type_data
);
2499 static const struct net_device_ops igb_netdev_ops
= {
2500 .ndo_open
= igb_open
,
2501 .ndo_stop
= igb_close
,
2502 .ndo_start_xmit
= igb_xmit_frame
,
2503 .ndo_get_stats64
= igb_get_stats64
,
2504 .ndo_set_rx_mode
= igb_set_rx_mode
,
2505 .ndo_set_mac_address
= igb_set_mac
,
2506 .ndo_change_mtu
= igb_change_mtu
,
2507 .ndo_do_ioctl
= igb_ioctl
,
2508 .ndo_tx_timeout
= igb_tx_timeout
,
2509 .ndo_validate_addr
= eth_validate_addr
,
2510 .ndo_vlan_rx_add_vid
= igb_vlan_rx_add_vid
,
2511 .ndo_vlan_rx_kill_vid
= igb_vlan_rx_kill_vid
,
2512 .ndo_set_vf_mac
= igb_ndo_set_vf_mac
,
2513 .ndo_set_vf_vlan
= igb_ndo_set_vf_vlan
,
2514 .ndo_set_vf_rate
= igb_ndo_set_vf_bw
,
2515 .ndo_set_vf_spoofchk
= igb_ndo_set_vf_spoofchk
,
2516 .ndo_get_vf_config
= igb_ndo_get_vf_config
,
2517 #ifdef CONFIG_NET_POLL_CONTROLLER
2518 .ndo_poll_controller
= igb_netpoll
,
2520 .ndo_fix_features
= igb_fix_features
,
2521 .ndo_set_features
= igb_set_features
,
2522 .ndo_fdb_add
= igb_ndo_fdb_add
,
2523 .ndo_features_check
= igb_features_check
,
2524 .ndo_setup_tc
= igb_setup_tc
,
2528 * igb_set_fw_version - Configure version string for ethtool
2529 * @adapter: adapter struct
2531 void igb_set_fw_version(struct igb_adapter
*adapter
)
2533 struct e1000_hw
*hw
= &adapter
->hw
;
2534 struct e1000_fw_version fw
;
2536 igb_get_fw_version(hw
, &fw
);
2538 switch (hw
->mac
.type
) {
2541 if (!(igb_get_flash_presence_i210(hw
))) {
2542 snprintf(adapter
->fw_version
,
2543 sizeof(adapter
->fw_version
),
2545 fw
.invm_major
, fw
.invm_minor
,
2551 /* if option is rom valid, display its version too */
2553 snprintf(adapter
->fw_version
,
2554 sizeof(adapter
->fw_version
),
2555 "%d.%d, 0x%08x, %d.%d.%d",
2556 fw
.eep_major
, fw
.eep_minor
, fw
.etrack_id
,
2557 fw
.or_major
, fw
.or_build
, fw
.or_patch
);
2559 } else if (fw
.etrack_id
!= 0X0000) {
2560 snprintf(adapter
->fw_version
,
2561 sizeof(adapter
->fw_version
),
2563 fw
.eep_major
, fw
.eep_minor
, fw
.etrack_id
);
2565 snprintf(adapter
->fw_version
,
2566 sizeof(adapter
->fw_version
),
2568 fw
.eep_major
, fw
.eep_minor
, fw
.eep_build
);
2575 * igb_init_mas - init Media Autosense feature if enabled in the NVM
2577 * @adapter: adapter struct
2579 static void igb_init_mas(struct igb_adapter
*adapter
)
2581 struct e1000_hw
*hw
= &adapter
->hw
;
2584 hw
->nvm
.ops
.read(hw
, NVM_COMPAT
, 1, &eeprom_data
);
2585 switch (hw
->bus
.func
) {
2587 if (eeprom_data
& IGB_MAS_ENABLE_0
) {
2588 adapter
->flags
|= IGB_FLAG_MAS_ENABLE
;
2589 netdev_info(adapter
->netdev
,
2590 "MAS: Enabling Media Autosense for port %d\n",
2595 if (eeprom_data
& IGB_MAS_ENABLE_1
) {
2596 adapter
->flags
|= IGB_FLAG_MAS_ENABLE
;
2597 netdev_info(adapter
->netdev
,
2598 "MAS: Enabling Media Autosense for port %d\n",
2603 if (eeprom_data
& IGB_MAS_ENABLE_2
) {
2604 adapter
->flags
|= IGB_FLAG_MAS_ENABLE
;
2605 netdev_info(adapter
->netdev
,
2606 "MAS: Enabling Media Autosense for port %d\n",
2611 if (eeprom_data
& IGB_MAS_ENABLE_3
) {
2612 adapter
->flags
|= IGB_FLAG_MAS_ENABLE
;
2613 netdev_info(adapter
->netdev
,
2614 "MAS: Enabling Media Autosense for port %d\n",
2619 /* Shouldn't get here */
2620 netdev_err(adapter
->netdev
,
2621 "MAS: Invalid port configuration, returning\n");
2627 * igb_init_i2c - Init I2C interface
2628 * @adapter: pointer to adapter structure
2630 static s32
igb_init_i2c(struct igb_adapter
*adapter
)
2634 /* I2C interface supported on i350 devices */
2635 if (adapter
->hw
.mac
.type
!= e1000_i350
)
2638 /* Initialize the i2c bus which is controlled by the registers.
2639 * This bus will use the i2c_algo_bit structue that implements
2640 * the protocol through toggling of the 4 bits in the register.
2642 adapter
->i2c_adap
.owner
= THIS_MODULE
;
2643 adapter
->i2c_algo
= igb_i2c_algo
;
2644 adapter
->i2c_algo
.data
= adapter
;
2645 adapter
->i2c_adap
.algo_data
= &adapter
->i2c_algo
;
2646 adapter
->i2c_adap
.dev
.parent
= &adapter
->pdev
->dev
;
2647 strlcpy(adapter
->i2c_adap
.name
, "igb BB",
2648 sizeof(adapter
->i2c_adap
.name
));
2649 status
= i2c_bit_add_bus(&adapter
->i2c_adap
);
2654 * igb_probe - Device Initialization Routine
2655 * @pdev: PCI device information struct
2656 * @ent: entry in igb_pci_tbl
2658 * Returns 0 on success, negative on failure
2660 * igb_probe initializes an adapter identified by a pci_dev structure.
2661 * The OS initialization, configuring of the adapter private structure,
2662 * and a hardware reset occur.
2664 static int igb_probe(struct pci_dev
*pdev
, const struct pci_device_id
*ent
)
2666 struct net_device
*netdev
;
2667 struct igb_adapter
*adapter
;
2668 struct e1000_hw
*hw
;
2669 u16 eeprom_data
= 0;
2671 static int global_quad_port_a
; /* global quad port a indication */
2672 const struct e1000_info
*ei
= igb_info_tbl
[ent
->driver_data
];
2673 int err
, pci_using_dac
;
2674 u8 part_str
[E1000_PBANUM_LENGTH
];
2676 /* Catch broken hardware that put the wrong VF device ID in
2677 * the PCIe SR-IOV capability.
2679 if (pdev
->is_virtfn
) {
2680 WARN(1, KERN_ERR
"%s (%hx:%hx) should not be a VF!\n",
2681 pci_name(pdev
), pdev
->vendor
, pdev
->device
);
2685 err
= pci_enable_device_mem(pdev
);
2690 err
= dma_set_mask_and_coherent(&pdev
->dev
, DMA_BIT_MASK(64));
2694 err
= dma_set_mask_and_coherent(&pdev
->dev
, DMA_BIT_MASK(32));
2697 "No usable DMA configuration, aborting\n");
2702 err
= pci_request_mem_regions(pdev
, igb_driver_name
);
2706 pci_enable_pcie_error_reporting(pdev
);
2708 pci_set_master(pdev
);
2709 pci_save_state(pdev
);
2712 netdev
= alloc_etherdev_mq(sizeof(struct igb_adapter
),
2715 goto err_alloc_etherdev
;
2717 SET_NETDEV_DEV(netdev
, &pdev
->dev
);
2719 pci_set_drvdata(pdev
, netdev
);
2720 adapter
= netdev_priv(netdev
);
2721 adapter
->netdev
= netdev
;
2722 adapter
->pdev
= pdev
;
2725 adapter
->msg_enable
= netif_msg_init(debug
, DEFAULT_MSG_ENABLE
);
2728 adapter
->io_addr
= pci_iomap(pdev
, 0, 0);
2729 if (!adapter
->io_addr
)
2731 /* hw->hw_addr can be altered, we'll use adapter->io_addr for unmap */
2732 hw
->hw_addr
= adapter
->io_addr
;
2734 netdev
->netdev_ops
= &igb_netdev_ops
;
2735 igb_set_ethtool_ops(netdev
);
2736 netdev
->watchdog_timeo
= 5 * HZ
;
2738 strncpy(netdev
->name
, pci_name(pdev
), sizeof(netdev
->name
) - 1);
2740 netdev
->mem_start
= pci_resource_start(pdev
, 0);
2741 netdev
->mem_end
= pci_resource_end(pdev
, 0);
2743 /* PCI config space info */
2744 hw
->vendor_id
= pdev
->vendor
;
2745 hw
->device_id
= pdev
->device
;
2746 hw
->revision_id
= pdev
->revision
;
2747 hw
->subsystem_vendor_id
= pdev
->subsystem_vendor
;
2748 hw
->subsystem_device_id
= pdev
->subsystem_device
;
2750 /* Copy the default MAC, PHY and NVM function pointers */
2751 memcpy(&hw
->mac
.ops
, ei
->mac_ops
, sizeof(hw
->mac
.ops
));
2752 memcpy(&hw
->phy
.ops
, ei
->phy_ops
, sizeof(hw
->phy
.ops
));
2753 memcpy(&hw
->nvm
.ops
, ei
->nvm_ops
, sizeof(hw
->nvm
.ops
));
2754 /* Initialize skew-specific constants */
2755 err
= ei
->get_invariants(hw
);
2759 /* setup the private structure */
2760 err
= igb_sw_init(adapter
);
2764 igb_get_bus_info_pcie(hw
);
2766 hw
->phy
.autoneg_wait_to_complete
= false;
2768 /* Copper options */
2769 if (hw
->phy
.media_type
== e1000_media_type_copper
) {
2770 hw
->phy
.mdix
= AUTO_ALL_MODES
;
2771 hw
->phy
.disable_polarity_correction
= false;
2772 hw
->phy
.ms_type
= e1000_ms_hw_default
;
2775 if (igb_check_reset_block(hw
))
2776 dev_info(&pdev
->dev
,
2777 "PHY reset is blocked due to SOL/IDER session.\n");
2779 /* features is initialized to 0 in allocation, it might have bits
2780 * set by igb_sw_init so we should use an or instead of an
2783 netdev
->features
|= NETIF_F_SG
|
2790 if (hw
->mac
.type
>= e1000_82576
)
2791 netdev
->features
|= NETIF_F_SCTP_CRC
;
2793 #define IGB_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
2794 NETIF_F_GSO_GRE_CSUM | \
2795 NETIF_F_GSO_IPXIP4 | \
2796 NETIF_F_GSO_IPXIP6 | \
2797 NETIF_F_GSO_UDP_TUNNEL | \
2798 NETIF_F_GSO_UDP_TUNNEL_CSUM)
2800 netdev
->gso_partial_features
= IGB_GSO_PARTIAL_FEATURES
;
2801 netdev
->features
|= NETIF_F_GSO_PARTIAL
| IGB_GSO_PARTIAL_FEATURES
;
2803 /* copy netdev features into list of user selectable features */
2804 netdev
->hw_features
|= netdev
->features
|
2805 NETIF_F_HW_VLAN_CTAG_RX
|
2806 NETIF_F_HW_VLAN_CTAG_TX
|
2809 if (hw
->mac
.type
>= e1000_i350
)
2810 netdev
->hw_features
|= NETIF_F_NTUPLE
;
2813 netdev
->features
|= NETIF_F_HIGHDMA
;
2815 netdev
->vlan_features
|= netdev
->features
| NETIF_F_TSO_MANGLEID
;
2816 netdev
->mpls_features
|= NETIF_F_HW_CSUM
;
2817 netdev
->hw_enc_features
|= netdev
->vlan_features
;
2819 /* set this bit last since it cannot be part of vlan_features */
2820 netdev
->features
|= NETIF_F_HW_VLAN_CTAG_FILTER
|
2821 NETIF_F_HW_VLAN_CTAG_RX
|
2822 NETIF_F_HW_VLAN_CTAG_TX
;
2824 netdev
->priv_flags
|= IFF_SUPP_NOFCS
;
2826 netdev
->priv_flags
|= IFF_UNICAST_FLT
;
2828 /* MTU range: 68 - 9216 */
2829 netdev
->min_mtu
= ETH_MIN_MTU
;
2830 netdev
->max_mtu
= MAX_STD_JUMBO_FRAME_SIZE
;
2832 adapter
->en_mng_pt
= igb_enable_mng_pass_thru(hw
);
2834 /* before reading the NVM, reset the controller to put the device in a
2835 * known good starting state
2837 hw
->mac
.ops
.reset_hw(hw
);
2839 /* make sure the NVM is good , i211/i210 parts can have special NVM
2840 * that doesn't contain a checksum
2842 switch (hw
->mac
.type
) {
2845 if (igb_get_flash_presence_i210(hw
)) {
2846 if (hw
->nvm
.ops
.validate(hw
) < 0) {
2848 "The NVM Checksum Is Not Valid\n");
2855 if (hw
->nvm
.ops
.validate(hw
) < 0) {
2856 dev_err(&pdev
->dev
, "The NVM Checksum Is Not Valid\n");
2863 if (eth_platform_get_mac_address(&pdev
->dev
, hw
->mac
.addr
)) {
2864 /* copy the MAC address out of the NVM */
2865 if (hw
->mac
.ops
.read_mac_addr(hw
))
2866 dev_err(&pdev
->dev
, "NVM Read Error\n");
2869 memcpy(netdev
->dev_addr
, hw
->mac
.addr
, netdev
->addr_len
);
2871 if (!is_valid_ether_addr(netdev
->dev_addr
)) {
2872 dev_err(&pdev
->dev
, "Invalid MAC Address\n");
2877 igb_set_default_mac_filter(adapter
);
2879 /* get firmware version for ethtool -i */
2880 igb_set_fw_version(adapter
);
2882 /* configure RXPBSIZE and TXPBSIZE */
2883 if (hw
->mac
.type
== e1000_i210
) {
2884 wr32(E1000_RXPBS
, I210_RXPBSIZE_DEFAULT
);
2885 wr32(E1000_TXPBS
, I210_TXPBSIZE_DEFAULT
);
2888 timer_setup(&adapter
->watchdog_timer
, igb_watchdog
, 0);
2889 timer_setup(&adapter
->phy_info_timer
, igb_update_phy_info
, 0);
2891 INIT_WORK(&adapter
->reset_task
, igb_reset_task
);
2892 INIT_WORK(&adapter
->watchdog_task
, igb_watchdog_task
);
2894 /* Initialize link properties that are user-changeable */
2895 adapter
->fc_autoneg
= true;
2896 hw
->mac
.autoneg
= true;
2897 hw
->phy
.autoneg_advertised
= 0x2f;
2899 hw
->fc
.requested_mode
= e1000_fc_default
;
2900 hw
->fc
.current_mode
= e1000_fc_default
;
2902 igb_validate_mdi_setting(hw
);
2904 /* By default, support wake on port A */
2905 if (hw
->bus
.func
== 0)
2906 adapter
->flags
|= IGB_FLAG_WOL_SUPPORTED
;
2908 /* Check the NVM for wake support on non-port A ports */
2909 if (hw
->mac
.type
>= e1000_82580
)
2910 hw
->nvm
.ops
.read(hw
, NVM_INIT_CONTROL3_PORT_A
+
2911 NVM_82580_LAN_FUNC_OFFSET(hw
->bus
.func
), 1,
2913 else if (hw
->bus
.func
== 1)
2914 hw
->nvm
.ops
.read(hw
, NVM_INIT_CONTROL3_PORT_B
, 1, &eeprom_data
);
2916 if (eeprom_data
& IGB_EEPROM_APME
)
2917 adapter
->flags
|= IGB_FLAG_WOL_SUPPORTED
;
2919 /* now that we have the eeprom settings, apply the special cases where
2920 * the eeprom may be wrong or the board simply won't support wake on
2921 * lan on a particular port
2923 switch (pdev
->device
) {
2924 case E1000_DEV_ID_82575GB_QUAD_COPPER
:
2925 adapter
->flags
&= ~IGB_FLAG_WOL_SUPPORTED
;
2927 case E1000_DEV_ID_82575EB_FIBER_SERDES
:
2928 case E1000_DEV_ID_82576_FIBER
:
2929 case E1000_DEV_ID_82576_SERDES
:
2930 /* Wake events only supported on port A for dual fiber
2931 * regardless of eeprom setting
2933 if (rd32(E1000_STATUS
) & E1000_STATUS_FUNC_1
)
2934 adapter
->flags
&= ~IGB_FLAG_WOL_SUPPORTED
;
2936 case E1000_DEV_ID_82576_QUAD_COPPER
:
2937 case E1000_DEV_ID_82576_QUAD_COPPER_ET2
:
2938 /* if quad port adapter, disable WoL on all but port A */
2939 if (global_quad_port_a
!= 0)
2940 adapter
->flags
&= ~IGB_FLAG_WOL_SUPPORTED
;
2942 adapter
->flags
|= IGB_FLAG_QUAD_PORT_A
;
2943 /* Reset for multiple quad port adapters */
2944 if (++global_quad_port_a
== 4)
2945 global_quad_port_a
= 0;
2948 /* If the device can't wake, don't set software support */
2949 if (!device_can_wakeup(&adapter
->pdev
->dev
))
2950 adapter
->flags
&= ~IGB_FLAG_WOL_SUPPORTED
;
2953 /* initialize the wol settings based on the eeprom settings */
2954 if (adapter
->flags
& IGB_FLAG_WOL_SUPPORTED
)
2955 adapter
->wol
|= E1000_WUFC_MAG
;
2957 /* Some vendors want WoL disabled by default, but still supported */
2958 if ((hw
->mac
.type
== e1000_i350
) &&
2959 (pdev
->subsystem_vendor
== PCI_VENDOR_ID_HP
)) {
2960 adapter
->flags
|= IGB_FLAG_WOL_SUPPORTED
;
2964 /* Some vendors want the ability to Use the EEPROM setting as
2965 * enable/disable only, and not for capability
2967 if (((hw
->mac
.type
== e1000_i350
) ||
2968 (hw
->mac
.type
== e1000_i354
)) &&
2969 (pdev
->subsystem_vendor
== PCI_VENDOR_ID_DELL
)) {
2970 adapter
->flags
|= IGB_FLAG_WOL_SUPPORTED
;
2973 if (hw
->mac
.type
== e1000_i350
) {
2974 if (((pdev
->subsystem_device
== 0x5001) ||
2975 (pdev
->subsystem_device
== 0x5002)) &&
2976 (hw
->bus
.func
== 0)) {
2977 adapter
->flags
|= IGB_FLAG_WOL_SUPPORTED
;
2980 if (pdev
->subsystem_device
== 0x1F52)
2981 adapter
->flags
|= IGB_FLAG_WOL_SUPPORTED
;
2984 device_set_wakeup_enable(&adapter
->pdev
->dev
,
2985 adapter
->flags
& IGB_FLAG_WOL_SUPPORTED
);
2987 /* reset the hardware with the new settings */
2990 /* Init the I2C interface */
2991 err
= igb_init_i2c(adapter
);
2993 dev_err(&pdev
->dev
, "failed to init i2c interface\n");
2997 /* let the f/w know that the h/w is now under the control of the
3000 igb_get_hw_control(adapter
);
3002 strcpy(netdev
->name
, "eth%d");
3003 err
= register_netdev(netdev
);
3007 /* carrier off reporting is important to ethtool even BEFORE open */
3008 netif_carrier_off(netdev
);
3010 #ifdef CONFIG_IGB_DCA
3011 if (dca_add_requester(&pdev
->dev
) == 0) {
3012 adapter
->flags
|= IGB_FLAG_DCA_ENABLED
;
3013 dev_info(&pdev
->dev
, "DCA enabled\n");
3014 igb_setup_dca(adapter
);
3018 #ifdef CONFIG_IGB_HWMON
3019 /* Initialize the thermal sensor on i350 devices. */
3020 if (hw
->mac
.type
== e1000_i350
&& hw
->bus
.func
== 0) {
3023 /* Read the NVM to determine if this i350 device supports an
3024 * external thermal sensor.
3026 hw
->nvm
.ops
.read(hw
, NVM_ETS_CFG
, 1, &ets_word
);
3027 if (ets_word
!= 0x0000 && ets_word
!= 0xFFFF)
3028 adapter
->ets
= true;
3030 adapter
->ets
= false;
3031 if (igb_sysfs_init(adapter
))
3033 "failed to allocate sysfs resources\n");
3035 adapter
->ets
= false;
3038 /* Check if Media Autosense is enabled */
3040 if (hw
->dev_spec
._82575
.mas_capable
)
3041 igb_init_mas(adapter
);
3043 /* do hw tstamp init after resetting */
3044 igb_ptp_init(adapter
);
3046 dev_info(&pdev
->dev
, "Intel(R) Gigabit Ethernet Network Connection\n");
3047 /* print bus type/speed/width info, not applicable to i354 */
3048 if (hw
->mac
.type
!= e1000_i354
) {
3049 dev_info(&pdev
->dev
, "%s: (PCIe:%s:%s) %pM\n",
3051 ((hw
->bus
.speed
== e1000_bus_speed_2500
) ? "2.5Gb/s" :
3052 (hw
->bus
.speed
== e1000_bus_speed_5000
) ? "5.0Gb/s" :
3054 ((hw
->bus
.width
== e1000_bus_width_pcie_x4
) ?
3056 (hw
->bus
.width
== e1000_bus_width_pcie_x2
) ?
3058 (hw
->bus
.width
== e1000_bus_width_pcie_x1
) ?
3059 "Width x1" : "unknown"), netdev
->dev_addr
);
3062 if ((hw
->mac
.type
>= e1000_i210
||
3063 igb_get_flash_presence_i210(hw
))) {
3064 ret_val
= igb_read_part_string(hw
, part_str
,
3065 E1000_PBANUM_LENGTH
);
3067 ret_val
= -E1000_ERR_INVM_VALUE_NOT_FOUND
;
3071 strcpy(part_str
, "Unknown");
3072 dev_info(&pdev
->dev
, "%s: PBA No: %s\n", netdev
->name
, part_str
);
3073 dev_info(&pdev
->dev
,
3074 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
3075 (adapter
->flags
& IGB_FLAG_HAS_MSIX
) ? "MSI-X" :
3076 (adapter
->flags
& IGB_FLAG_HAS_MSI
) ? "MSI" : "legacy",
3077 adapter
->num_rx_queues
, adapter
->num_tx_queues
);
3078 if (hw
->phy
.media_type
== e1000_media_type_copper
) {
3079 switch (hw
->mac
.type
) {
3083 /* Enable EEE for internal copper PHY devices */
3084 err
= igb_set_eee_i350(hw
, true, true);
3086 (!hw
->dev_spec
._82575
.eee_disable
)) {
3087 adapter
->eee_advert
=
3088 MDIO_EEE_100TX
| MDIO_EEE_1000T
;
3089 adapter
->flags
|= IGB_FLAG_EEE
;
3093 if ((rd32(E1000_CTRL_EXT
) &
3094 E1000_CTRL_EXT_LINK_MODE_SGMII
)) {
3095 err
= igb_set_eee_i354(hw
, true, true);
3097 (!hw
->dev_spec
._82575
.eee_disable
)) {
3098 adapter
->eee_advert
=
3099 MDIO_EEE_100TX
| MDIO_EEE_1000T
;
3100 adapter
->flags
|= IGB_FLAG_EEE
;
3108 pm_runtime_put_noidle(&pdev
->dev
);
3112 igb_release_hw_control(adapter
);
3113 memset(&adapter
->i2c_adap
, 0, sizeof(adapter
->i2c_adap
));
3115 if (!igb_check_reset_block(hw
))
3118 if (hw
->flash_address
)
3119 iounmap(hw
->flash_address
);
3121 kfree(adapter
->mac_table
);
3122 kfree(adapter
->shadow_vfta
);
3123 igb_clear_interrupt_scheme(adapter
);
3124 #ifdef CONFIG_PCI_IOV
3125 igb_disable_sriov(pdev
);
3127 pci_iounmap(pdev
, adapter
->io_addr
);
3129 free_netdev(netdev
);
3131 pci_release_mem_regions(pdev
);
3134 pci_disable_device(pdev
);
3138 #ifdef CONFIG_PCI_IOV
3139 static int igb_disable_sriov(struct pci_dev
*pdev
)
3141 struct net_device
*netdev
= pci_get_drvdata(pdev
);
3142 struct igb_adapter
*adapter
= netdev_priv(netdev
);
3143 struct e1000_hw
*hw
= &adapter
->hw
;
3145 /* reclaim resources allocated to VFs */
3146 if (adapter
->vf_data
) {
3147 /* disable iov and allow time for transactions to clear */
3148 if (pci_vfs_assigned(pdev
)) {
3149 dev_warn(&pdev
->dev
,
3150 "Cannot deallocate SR-IOV virtual functions while they are assigned - VFs will not be deallocated\n");
3153 pci_disable_sriov(pdev
);
3157 kfree(adapter
->vf_mac_list
);
3158 adapter
->vf_mac_list
= NULL
;
3159 kfree(adapter
->vf_data
);
3160 adapter
->vf_data
= NULL
;
3161 adapter
->vfs_allocated_count
= 0;
3162 wr32(E1000_IOVCTL
, E1000_IOVCTL_REUSE_VFQ
);
3165 dev_info(&pdev
->dev
, "IOV Disabled\n");
3167 /* Re-enable DMA Coalescing flag since IOV is turned off */
3168 adapter
->flags
|= IGB_FLAG_DMAC
;
3174 static int igb_enable_sriov(struct pci_dev
*pdev
, int num_vfs
)
3176 struct net_device
*netdev
= pci_get_drvdata(pdev
);
3177 struct igb_adapter
*adapter
= netdev_priv(netdev
);
3178 int old_vfs
= pci_num_vf(pdev
);
3179 struct vf_mac_filter
*mac_list
;
3181 int num_vf_mac_filters
, i
;
3183 if (!(adapter
->flags
& IGB_FLAG_HAS_MSIX
) || num_vfs
> 7) {
3191 dev_info(&pdev
->dev
, "%d pre-allocated VFs found - override max_vfs setting of %d\n",
3193 adapter
->vfs_allocated_count
= old_vfs
;
3195 adapter
->vfs_allocated_count
= num_vfs
;
3197 adapter
->vf_data
= kcalloc(adapter
->vfs_allocated_count
,
3198 sizeof(struct vf_data_storage
), GFP_KERNEL
);
3200 /* if allocation failed then we do not support SR-IOV */
3201 if (!adapter
->vf_data
) {
3202 adapter
->vfs_allocated_count
= 0;
3204 "Unable to allocate memory for VF Data Storage\n");
3209 /* Due to the limited number of RAR entries calculate potential
3210 * number of MAC filters available for the VFs. Reserve entries
3211 * for PF default MAC, PF MAC filters and at least one RAR entry
3212 * for each VF for VF MAC.
3214 num_vf_mac_filters
= adapter
->hw
.mac
.rar_entry_count
-
3215 (1 + IGB_PF_MAC_FILTERS_RESERVED
+
3216 adapter
->vfs_allocated_count
);
3218 adapter
->vf_mac_list
= kcalloc(num_vf_mac_filters
,
3219 sizeof(struct vf_mac_filter
),
3222 mac_list
= adapter
->vf_mac_list
;
3223 INIT_LIST_HEAD(&adapter
->vf_macs
.l
);
3225 if (adapter
->vf_mac_list
) {
3226 /* Initialize list of VF MAC filters */
3227 for (i
= 0; i
< num_vf_mac_filters
; i
++) {
3229 mac_list
->free
= true;
3230 list_add(&mac_list
->l
, &adapter
->vf_macs
.l
);
3234 /* If we could not allocate memory for the VF MAC filters
3235 * we can continue without this feature but warn user.
3238 "Unable to allocate memory for VF MAC filter list\n");
3241 /* only call pci_enable_sriov() if no VFs are allocated already */
3243 err
= pci_enable_sriov(pdev
, adapter
->vfs_allocated_count
);
3247 dev_info(&pdev
->dev
, "%d VFs allocated\n",
3248 adapter
->vfs_allocated_count
);
3249 for (i
= 0; i
< adapter
->vfs_allocated_count
; i
++)
3250 igb_vf_configure(adapter
, i
);
3252 /* DMA Coalescing is not supported in IOV mode. */
3253 adapter
->flags
&= ~IGB_FLAG_DMAC
;
3257 kfree(adapter
->vf_mac_list
);
3258 adapter
->vf_mac_list
= NULL
;
3259 kfree(adapter
->vf_data
);
3260 adapter
->vf_data
= NULL
;
3261 adapter
->vfs_allocated_count
= 0;
3268 * igb_remove_i2c - Cleanup I2C interface
3269 * @adapter: pointer to adapter structure
3271 static void igb_remove_i2c(struct igb_adapter
*adapter
)
3273 /* free the adapter bus structure */
3274 i2c_del_adapter(&adapter
->i2c_adap
);
3278 * igb_remove - Device Removal Routine
3279 * @pdev: PCI device information struct
3281 * igb_remove is called by the PCI subsystem to alert the driver
3282 * that it should release a PCI device. The could be caused by a
3283 * Hot-Plug event, or because the driver is going to be removed from
3286 static void igb_remove(struct pci_dev
*pdev
)
3288 struct net_device
*netdev
= pci_get_drvdata(pdev
);
3289 struct igb_adapter
*adapter
= netdev_priv(netdev
);
3290 struct e1000_hw
*hw
= &adapter
->hw
;
3292 pm_runtime_get_noresume(&pdev
->dev
);
3293 #ifdef CONFIG_IGB_HWMON
3294 igb_sysfs_exit(adapter
);
3296 igb_remove_i2c(adapter
);
3297 igb_ptp_stop(adapter
);
3298 /* The watchdog timer may be rescheduled, so explicitly
3299 * disable watchdog from being rescheduled.
3301 set_bit(__IGB_DOWN
, &adapter
->state
);
3302 del_timer_sync(&adapter
->watchdog_timer
);
3303 del_timer_sync(&adapter
->phy_info_timer
);
3305 cancel_work_sync(&adapter
->reset_task
);
3306 cancel_work_sync(&adapter
->watchdog_task
);
3308 #ifdef CONFIG_IGB_DCA
3309 if (adapter
->flags
& IGB_FLAG_DCA_ENABLED
) {
3310 dev_info(&pdev
->dev
, "DCA disabled\n");
3311 dca_remove_requester(&pdev
->dev
);
3312 adapter
->flags
&= ~IGB_FLAG_DCA_ENABLED
;
3313 wr32(E1000_DCA_CTRL
, E1000_DCA_CTRL_DCA_MODE_DISABLE
);
3317 /* Release control of h/w to f/w. If f/w is AMT enabled, this
3318 * would have already happened in close and is redundant.
3320 igb_release_hw_control(adapter
);
3322 #ifdef CONFIG_PCI_IOV
3323 igb_disable_sriov(pdev
);
3326 unregister_netdev(netdev
);
3328 igb_clear_interrupt_scheme(adapter
);
3330 pci_iounmap(pdev
, adapter
->io_addr
);
3331 if (hw
->flash_address
)
3332 iounmap(hw
->flash_address
);
3333 pci_release_mem_regions(pdev
);
3335 kfree(adapter
->mac_table
);
3336 kfree(adapter
->shadow_vfta
);
3337 free_netdev(netdev
);
3339 pci_disable_pcie_error_reporting(pdev
);
3341 pci_disable_device(pdev
);
3345 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
3346 * @adapter: board private structure to initialize
3348 * This function initializes the vf specific data storage and then attempts to
3349 * allocate the VFs. The reason for ordering it this way is because it is much
3350 * mor expensive time wise to disable SR-IOV than it is to allocate and free
3351 * the memory for the VFs.
3353 static void igb_probe_vfs(struct igb_adapter
*adapter
)
3355 #ifdef CONFIG_PCI_IOV
3356 struct pci_dev
*pdev
= adapter
->pdev
;
3357 struct e1000_hw
*hw
= &adapter
->hw
;
3359 /* Virtualization features not supported on i210 family. */
3360 if ((hw
->mac
.type
== e1000_i210
) || (hw
->mac
.type
== e1000_i211
))
3363 /* Of the below we really only want the effect of getting
3364 * IGB_FLAG_HAS_MSIX set (if available), without which
3365 * igb_enable_sriov() has no effect.
3367 igb_set_interrupt_capability(adapter
, true);
3368 igb_reset_interrupt_capability(adapter
);
3370 pci_sriov_set_totalvfs(pdev
, 7);
3371 igb_enable_sriov(pdev
, max_vfs
);
3373 #endif /* CONFIG_PCI_IOV */
3376 static void igb_init_queue_configuration(struct igb_adapter
*adapter
)
3378 struct e1000_hw
*hw
= &adapter
->hw
;
3381 /* Determine the maximum number of RSS queues supported. */
3382 switch (hw
->mac
.type
) {
3384 max_rss_queues
= IGB_MAX_RX_QUEUES_I211
;
3388 max_rss_queues
= IGB_MAX_RX_QUEUES_82575
;
3391 /* I350 cannot do RSS and SR-IOV at the same time */
3392 if (!!adapter
->vfs_allocated_count
) {
3398 if (!!adapter
->vfs_allocated_count
) {
3406 max_rss_queues
= IGB_MAX_RX_QUEUES
;
3410 adapter
->rss_queues
= min_t(u32
, max_rss_queues
, num_online_cpus());
3412 igb_set_flag_queue_pairs(adapter
, max_rss_queues
);
3415 void igb_set_flag_queue_pairs(struct igb_adapter
*adapter
,
3416 const u32 max_rss_queues
)
3418 struct e1000_hw
*hw
= &adapter
->hw
;
3420 /* Determine if we need to pair queues. */
3421 switch (hw
->mac
.type
) {
3424 /* Device supports enough interrupts without queue pairing. */
3432 /* If rss_queues > half of max_rss_queues, pair the queues in
3433 * order to conserve interrupts due to limited supply.
3435 if (adapter
->rss_queues
> (max_rss_queues
/ 2))
3436 adapter
->flags
|= IGB_FLAG_QUEUE_PAIRS
;
3438 adapter
->flags
&= ~IGB_FLAG_QUEUE_PAIRS
;
3444 * igb_sw_init - Initialize general software structures (struct igb_adapter)
3445 * @adapter: board private structure to initialize
3447 * igb_sw_init initializes the Adapter private data structure.
3448 * Fields are initialized based on PCI device information and
3449 * OS network device settings (MTU size).
3451 static int igb_sw_init(struct igb_adapter
*adapter
)
3453 struct e1000_hw
*hw
= &adapter
->hw
;
3454 struct net_device
*netdev
= adapter
->netdev
;
3455 struct pci_dev
*pdev
= adapter
->pdev
;
3457 pci_read_config_word(pdev
, PCI_COMMAND
, &hw
->bus
.pci_cmd_word
);
3459 /* set default ring sizes */
3460 adapter
->tx_ring_count
= IGB_DEFAULT_TXD
;
3461 adapter
->rx_ring_count
= IGB_DEFAULT_RXD
;
3463 /* set default ITR values */
3464 adapter
->rx_itr_setting
= IGB_DEFAULT_ITR
;
3465 adapter
->tx_itr_setting
= IGB_DEFAULT_ITR
;
3467 /* set default work limits */
3468 adapter
->tx_work_limit
= IGB_DEFAULT_TX_WORK
;
3470 adapter
->max_frame_size
= netdev
->mtu
+ ETH_HLEN
+ ETH_FCS_LEN
+
3472 adapter
->min_frame_size
= ETH_ZLEN
+ ETH_FCS_LEN
;
3474 spin_lock_init(&adapter
->nfc_lock
);
3475 spin_lock_init(&adapter
->stats64_lock
);
3476 #ifdef CONFIG_PCI_IOV
3477 switch (hw
->mac
.type
) {
3481 dev_warn(&pdev
->dev
,
3482 "Maximum of 7 VFs per PF, using max\n");
3483 max_vfs
= adapter
->vfs_allocated_count
= 7;
3485 adapter
->vfs_allocated_count
= max_vfs
;
3486 if (adapter
->vfs_allocated_count
)
3487 dev_warn(&pdev
->dev
,
3488 "Enabling SR-IOV VFs using the module parameter is deprecated - please use the pci sysfs interface.\n");
3493 #endif /* CONFIG_PCI_IOV */
3495 /* Assume MSI-X interrupts, will be checked during IRQ allocation */
3496 adapter
->flags
|= IGB_FLAG_HAS_MSIX
;
3498 adapter
->mac_table
= kzalloc(sizeof(struct igb_mac_addr
) *
3499 hw
->mac
.rar_entry_count
, GFP_ATOMIC
);
3500 if (!adapter
->mac_table
)
3503 igb_probe_vfs(adapter
);
3505 igb_init_queue_configuration(adapter
);
3507 /* Setup and initialize a copy of the hw vlan table array */
3508 adapter
->shadow_vfta
= kcalloc(E1000_VLAN_FILTER_TBL_SIZE
, sizeof(u32
),
3510 if (!adapter
->shadow_vfta
)
3513 /* This call may decrease the number of queues */
3514 if (igb_init_interrupt_scheme(adapter
, true)) {
3515 dev_err(&pdev
->dev
, "Unable to allocate memory for queues\n");
3519 /* Explicitly disable IRQ since the NIC can be in any state. */
3520 igb_irq_disable(adapter
);
3522 if (hw
->mac
.type
>= e1000_i350
)
3523 adapter
->flags
&= ~IGB_FLAG_DMAC
;
3525 set_bit(__IGB_DOWN
, &adapter
->state
);
3530 * igb_open - Called when a network interface is made active
3531 * @netdev: network interface device structure
3533 * Returns 0 on success, negative value on failure
3535 * The open entry point is called when a network interface is made
3536 * active by the system (IFF_UP). At this point all resources needed
3537 * for transmit and receive operations are allocated, the interrupt
3538 * handler is registered with the OS, the watchdog timer is started,
3539 * and the stack is notified that the interface is ready.
3541 static int __igb_open(struct net_device
*netdev
, bool resuming
)
3543 struct igb_adapter
*adapter
= netdev_priv(netdev
);
3544 struct e1000_hw
*hw
= &adapter
->hw
;
3545 struct pci_dev
*pdev
= adapter
->pdev
;
3549 /* disallow open during test */
3550 if (test_bit(__IGB_TESTING
, &adapter
->state
)) {
3556 pm_runtime_get_sync(&pdev
->dev
);
3558 netif_carrier_off(netdev
);
3560 /* allocate transmit descriptors */
3561 err
= igb_setup_all_tx_resources(adapter
);
3565 /* allocate receive descriptors */
3566 err
= igb_setup_all_rx_resources(adapter
);
3570 igb_power_up_link(adapter
);
3572 /* before we allocate an interrupt, we must be ready to handle it.
3573 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
3574 * as soon as we call pci_request_irq, so we have to setup our
3575 * clean_rx handler before we do so.
3577 igb_configure(adapter
);
3579 err
= igb_request_irq(adapter
);
3583 /* Notify the stack of the actual queue counts. */
3584 err
= netif_set_real_num_tx_queues(adapter
->netdev
,
3585 adapter
->num_tx_queues
);
3587 goto err_set_queues
;
3589 err
= netif_set_real_num_rx_queues(adapter
->netdev
,
3590 adapter
->num_rx_queues
);
3592 goto err_set_queues
;
3594 /* From here on the code is the same as igb_up() */
3595 clear_bit(__IGB_DOWN
, &adapter
->state
);
3597 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
3598 napi_enable(&(adapter
->q_vector
[i
]->napi
));
3600 /* Clear any pending interrupts. */
3603 igb_irq_enable(adapter
);
3605 /* notify VFs that reset has been completed */
3606 if (adapter
->vfs_allocated_count
) {
3607 u32 reg_data
= rd32(E1000_CTRL_EXT
);
3609 reg_data
|= E1000_CTRL_EXT_PFRSTD
;
3610 wr32(E1000_CTRL_EXT
, reg_data
);
3613 netif_tx_start_all_queues(netdev
);
3616 pm_runtime_put(&pdev
->dev
);
3618 /* start the watchdog. */
3619 hw
->mac
.get_link_status
= 1;
3620 schedule_work(&adapter
->watchdog_task
);
3625 igb_free_irq(adapter
);
3627 igb_release_hw_control(adapter
);
3628 igb_power_down_link(adapter
);
3629 igb_free_all_rx_resources(adapter
);
3631 igb_free_all_tx_resources(adapter
);
3635 pm_runtime_put(&pdev
->dev
);
3640 int igb_open(struct net_device
*netdev
)
3642 return __igb_open(netdev
, false);
3646 * igb_close - Disables a network interface
3647 * @netdev: network interface device structure
3649 * Returns 0, this is not allowed to fail
3651 * The close entry point is called when an interface is de-activated
3652 * by the OS. The hardware is still under the driver's control, but
3653 * needs to be disabled. A global MAC reset is issued to stop the
3654 * hardware, and all transmit and receive resources are freed.
3656 static int __igb_close(struct net_device
*netdev
, bool suspending
)
3658 struct igb_adapter
*adapter
= netdev_priv(netdev
);
3659 struct pci_dev
*pdev
= adapter
->pdev
;
3661 WARN_ON(test_bit(__IGB_RESETTING
, &adapter
->state
));
3664 pm_runtime_get_sync(&pdev
->dev
);
3667 igb_free_irq(adapter
);
3669 igb_free_all_tx_resources(adapter
);
3670 igb_free_all_rx_resources(adapter
);
3673 pm_runtime_put_sync(&pdev
->dev
);
3677 int igb_close(struct net_device
*netdev
)
3679 if (netif_device_present(netdev
))
3680 return __igb_close(netdev
, false);
3685 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
3686 * @tx_ring: tx descriptor ring (for a specific queue) to setup
3688 * Return 0 on success, negative on failure
3690 int igb_setup_tx_resources(struct igb_ring
*tx_ring
)
3692 struct device
*dev
= tx_ring
->dev
;
3695 size
= sizeof(struct igb_tx_buffer
) * tx_ring
->count
;
3697 tx_ring
->tx_buffer_info
= vmalloc(size
);
3698 if (!tx_ring
->tx_buffer_info
)
3701 /* round up to nearest 4K */
3702 tx_ring
->size
= tx_ring
->count
* sizeof(union e1000_adv_tx_desc
);
3703 tx_ring
->size
= ALIGN(tx_ring
->size
, 4096);
3705 tx_ring
->desc
= dma_alloc_coherent(dev
, tx_ring
->size
,
3706 &tx_ring
->dma
, GFP_KERNEL
);
3710 tx_ring
->next_to_use
= 0;
3711 tx_ring
->next_to_clean
= 0;
3716 vfree(tx_ring
->tx_buffer_info
);
3717 tx_ring
->tx_buffer_info
= NULL
;
3718 dev_err(dev
, "Unable to allocate memory for the Tx descriptor ring\n");
3723 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
3724 * (Descriptors) for all queues
3725 * @adapter: board private structure
3727 * Return 0 on success, negative on failure
3729 static int igb_setup_all_tx_resources(struct igb_adapter
*adapter
)
3731 struct pci_dev
*pdev
= adapter
->pdev
;
3734 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
3735 err
= igb_setup_tx_resources(adapter
->tx_ring
[i
]);
3738 "Allocation for Tx Queue %u failed\n", i
);
3739 for (i
--; i
>= 0; i
--)
3740 igb_free_tx_resources(adapter
->tx_ring
[i
]);
3749 * igb_setup_tctl - configure the transmit control registers
3750 * @adapter: Board private structure
3752 void igb_setup_tctl(struct igb_adapter
*adapter
)
3754 struct e1000_hw
*hw
= &adapter
->hw
;
3757 /* disable queue 0 which is enabled by default on 82575 and 82576 */
3758 wr32(E1000_TXDCTL(0), 0);
3760 /* Program the Transmit Control Register */
3761 tctl
= rd32(E1000_TCTL
);
3762 tctl
&= ~E1000_TCTL_CT
;
3763 tctl
|= E1000_TCTL_PSP
| E1000_TCTL_RTLC
|
3764 (E1000_COLLISION_THRESHOLD
<< E1000_CT_SHIFT
);
3766 igb_config_collision_dist(hw
);
3768 /* Enable transmits */
3769 tctl
|= E1000_TCTL_EN
;
3771 wr32(E1000_TCTL
, tctl
);
3775 * igb_configure_tx_ring - Configure transmit ring after Reset
3776 * @adapter: board private structure
3777 * @ring: tx ring to configure
3779 * Configure a transmit ring after a reset.
3781 void igb_configure_tx_ring(struct igb_adapter
*adapter
,
3782 struct igb_ring
*ring
)
3784 struct e1000_hw
*hw
= &adapter
->hw
;
3786 u64 tdba
= ring
->dma
;
3787 int reg_idx
= ring
->reg_idx
;
3789 /* disable the queue */
3790 wr32(E1000_TXDCTL(reg_idx
), 0);
3794 wr32(E1000_TDLEN(reg_idx
),
3795 ring
->count
* sizeof(union e1000_adv_tx_desc
));
3796 wr32(E1000_TDBAL(reg_idx
),
3797 tdba
& 0x00000000ffffffffULL
);
3798 wr32(E1000_TDBAH(reg_idx
), tdba
>> 32);
3800 ring
->tail
= adapter
->io_addr
+ E1000_TDT(reg_idx
);
3801 wr32(E1000_TDH(reg_idx
), 0);
3802 writel(0, ring
->tail
);
3804 txdctl
|= IGB_TX_PTHRESH
;
3805 txdctl
|= IGB_TX_HTHRESH
<< 8;
3806 txdctl
|= IGB_TX_WTHRESH
<< 16;
3808 /* reinitialize tx_buffer_info */
3809 memset(ring
->tx_buffer_info
, 0,
3810 sizeof(struct igb_tx_buffer
) * ring
->count
);
3812 txdctl
|= E1000_TXDCTL_QUEUE_ENABLE
;
3813 wr32(E1000_TXDCTL(reg_idx
), txdctl
);
3817 * igb_configure_tx - Configure transmit Unit after Reset
3818 * @adapter: board private structure
3820 * Configure the Tx unit of the MAC after a reset.
3822 static void igb_configure_tx(struct igb_adapter
*adapter
)
3826 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
3827 igb_configure_tx_ring(adapter
, adapter
->tx_ring
[i
]);
3831 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
3832 * @rx_ring: Rx descriptor ring (for a specific queue) to setup
3834 * Returns 0 on success, negative on failure
3836 int igb_setup_rx_resources(struct igb_ring
*rx_ring
)
3838 struct device
*dev
= rx_ring
->dev
;
3841 size
= sizeof(struct igb_rx_buffer
) * rx_ring
->count
;
3843 rx_ring
->rx_buffer_info
= vmalloc(size
);
3844 if (!rx_ring
->rx_buffer_info
)
3847 /* Round up to nearest 4K */
3848 rx_ring
->size
= rx_ring
->count
* sizeof(union e1000_adv_rx_desc
);
3849 rx_ring
->size
= ALIGN(rx_ring
->size
, 4096);
3851 rx_ring
->desc
= dma_alloc_coherent(dev
, rx_ring
->size
,
3852 &rx_ring
->dma
, GFP_KERNEL
);
3856 rx_ring
->next_to_alloc
= 0;
3857 rx_ring
->next_to_clean
= 0;
3858 rx_ring
->next_to_use
= 0;
3863 vfree(rx_ring
->rx_buffer_info
);
3864 rx_ring
->rx_buffer_info
= NULL
;
3865 dev_err(dev
, "Unable to allocate memory for the Rx descriptor ring\n");
3870 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
3871 * (Descriptors) for all queues
3872 * @adapter: board private structure
3874 * Return 0 on success, negative on failure
3876 static int igb_setup_all_rx_resources(struct igb_adapter
*adapter
)
3878 struct pci_dev
*pdev
= adapter
->pdev
;
3881 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
3882 err
= igb_setup_rx_resources(adapter
->rx_ring
[i
]);
3885 "Allocation for Rx Queue %u failed\n", i
);
3886 for (i
--; i
>= 0; i
--)
3887 igb_free_rx_resources(adapter
->rx_ring
[i
]);
3896 * igb_setup_mrqc - configure the multiple receive queue control registers
3897 * @adapter: Board private structure
3899 static void igb_setup_mrqc(struct igb_adapter
*adapter
)
3901 struct e1000_hw
*hw
= &adapter
->hw
;
3903 u32 j
, num_rx_queues
;
3906 netdev_rss_key_fill(rss_key
, sizeof(rss_key
));
3907 for (j
= 0; j
< 10; j
++)
3908 wr32(E1000_RSSRK(j
), rss_key
[j
]);
3910 num_rx_queues
= adapter
->rss_queues
;
3912 switch (hw
->mac
.type
) {
3914 /* 82576 supports 2 RSS queues for SR-IOV */
3915 if (adapter
->vfs_allocated_count
)
3922 if (adapter
->rss_indir_tbl_init
!= num_rx_queues
) {
3923 for (j
= 0; j
< IGB_RETA_SIZE
; j
++)
3924 adapter
->rss_indir_tbl
[j
] =
3925 (j
* num_rx_queues
) / IGB_RETA_SIZE
;
3926 adapter
->rss_indir_tbl_init
= num_rx_queues
;
3928 igb_write_rss_indir_tbl(adapter
);
3930 /* Disable raw packet checksumming so that RSS hash is placed in
3931 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
3932 * offloads as they are enabled by default
3934 rxcsum
= rd32(E1000_RXCSUM
);
3935 rxcsum
|= E1000_RXCSUM_PCSD
;
3937 if (adapter
->hw
.mac
.type
>= e1000_82576
)
3938 /* Enable Receive Checksum Offload for SCTP */
3939 rxcsum
|= E1000_RXCSUM_CRCOFL
;
3941 /* Don't need to set TUOFL or IPOFL, they default to 1 */
3942 wr32(E1000_RXCSUM
, rxcsum
);
3944 /* Generate RSS hash based on packet types, TCP/UDP
3945 * port numbers and/or IPv4/v6 src and dst addresses
3947 mrqc
= E1000_MRQC_RSS_FIELD_IPV4
|
3948 E1000_MRQC_RSS_FIELD_IPV4_TCP
|
3949 E1000_MRQC_RSS_FIELD_IPV6
|
3950 E1000_MRQC_RSS_FIELD_IPV6_TCP
|
3951 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX
;
3953 if (adapter
->flags
& IGB_FLAG_RSS_FIELD_IPV4_UDP
)
3954 mrqc
|= E1000_MRQC_RSS_FIELD_IPV4_UDP
;
3955 if (adapter
->flags
& IGB_FLAG_RSS_FIELD_IPV6_UDP
)
3956 mrqc
|= E1000_MRQC_RSS_FIELD_IPV6_UDP
;
3958 /* If VMDq is enabled then we set the appropriate mode for that, else
3959 * we default to RSS so that an RSS hash is calculated per packet even
3960 * if we are only using one queue
3962 if (adapter
->vfs_allocated_count
) {
3963 if (hw
->mac
.type
> e1000_82575
) {
3964 /* Set the default pool for the PF's first queue */
3965 u32 vtctl
= rd32(E1000_VT_CTL
);
3967 vtctl
&= ~(E1000_VT_CTL_DEFAULT_POOL_MASK
|
3968 E1000_VT_CTL_DISABLE_DEF_POOL
);
3969 vtctl
|= adapter
->vfs_allocated_count
<<
3970 E1000_VT_CTL_DEFAULT_POOL_SHIFT
;
3971 wr32(E1000_VT_CTL
, vtctl
);
3973 if (adapter
->rss_queues
> 1)
3974 mrqc
|= E1000_MRQC_ENABLE_VMDQ_RSS_MQ
;
3976 mrqc
|= E1000_MRQC_ENABLE_VMDQ
;
3978 if (hw
->mac
.type
!= e1000_i211
)
3979 mrqc
|= E1000_MRQC_ENABLE_RSS_MQ
;
3981 igb_vmm_control(adapter
);
3983 wr32(E1000_MRQC
, mrqc
);
3987 * igb_setup_rctl - configure the receive control registers
3988 * @adapter: Board private structure
3990 void igb_setup_rctl(struct igb_adapter
*adapter
)
3992 struct e1000_hw
*hw
= &adapter
->hw
;
3995 rctl
= rd32(E1000_RCTL
);
3997 rctl
&= ~(3 << E1000_RCTL_MO_SHIFT
);
3998 rctl
&= ~(E1000_RCTL_LBM_TCVR
| E1000_RCTL_LBM_MAC
);
4000 rctl
|= E1000_RCTL_EN
| E1000_RCTL_BAM
| E1000_RCTL_RDMTS_HALF
|
4001 (hw
->mac
.mc_filter_type
<< E1000_RCTL_MO_SHIFT
);
4003 /* enable stripping of CRC. It's unlikely this will break BMC
4004 * redirection as it did with e1000. Newer features require
4005 * that the HW strips the CRC.
4007 rctl
|= E1000_RCTL_SECRC
;
4009 /* disable store bad packets and clear size bits. */
4010 rctl
&= ~(E1000_RCTL_SBP
| E1000_RCTL_SZ_256
);
4012 /* enable LPE to allow for reception of jumbo frames */
4013 rctl
|= E1000_RCTL_LPE
;
4015 /* disable queue 0 to prevent tail write w/o re-config */
4016 wr32(E1000_RXDCTL(0), 0);
4018 /* Attention!!! For SR-IOV PF driver operations you must enable
4019 * queue drop for all VF and PF queues to prevent head of line blocking
4020 * if an un-trusted VF does not provide descriptors to hardware.
4022 if (adapter
->vfs_allocated_count
) {
4023 /* set all queue drop enable bits */
4024 wr32(E1000_QDE
, ALL_QUEUES
);
4027 /* This is useful for sniffing bad packets. */
4028 if (adapter
->netdev
->features
& NETIF_F_RXALL
) {
4029 /* UPE and MPE will be handled by normal PROMISC logic
4030 * in e1000e_set_rx_mode
4032 rctl
|= (E1000_RCTL_SBP
| /* Receive bad packets */
4033 E1000_RCTL_BAM
| /* RX All Bcast Pkts */
4034 E1000_RCTL_PMCF
); /* RX All MAC Ctrl Pkts */
4036 rctl
&= ~(E1000_RCTL_DPF
| /* Allow filtered pause */
4037 E1000_RCTL_CFIEN
); /* Dis VLAN CFIEN Filter */
4038 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
4039 * and that breaks VLANs.
4043 wr32(E1000_RCTL
, rctl
);
4046 static inline int igb_set_vf_rlpml(struct igb_adapter
*adapter
, int size
,
4049 struct e1000_hw
*hw
= &adapter
->hw
;
4052 if (size
> MAX_JUMBO_FRAME_SIZE
)
4053 size
= MAX_JUMBO_FRAME_SIZE
;
4055 vmolr
= rd32(E1000_VMOLR(vfn
));
4056 vmolr
&= ~E1000_VMOLR_RLPML_MASK
;
4057 vmolr
|= size
| E1000_VMOLR_LPE
;
4058 wr32(E1000_VMOLR(vfn
), vmolr
);
4063 static inline void igb_set_vf_vlan_strip(struct igb_adapter
*adapter
,
4064 int vfn
, bool enable
)
4066 struct e1000_hw
*hw
= &adapter
->hw
;
4069 if (hw
->mac
.type
< e1000_82576
)
4072 if (hw
->mac
.type
== e1000_i350
)
4073 reg
= E1000_DVMOLR(vfn
);
4075 reg
= E1000_VMOLR(vfn
);
4079 val
|= E1000_VMOLR_STRVLAN
;
4081 val
&= ~(E1000_VMOLR_STRVLAN
);
4085 static inline void igb_set_vmolr(struct igb_adapter
*adapter
,
4088 struct e1000_hw
*hw
= &adapter
->hw
;
4091 /* This register exists only on 82576 and newer so if we are older then
4092 * we should exit and do nothing
4094 if (hw
->mac
.type
< e1000_82576
)
4097 vmolr
= rd32(E1000_VMOLR(vfn
));
4099 vmolr
|= E1000_VMOLR_AUPE
; /* Accept untagged packets */
4101 vmolr
&= ~(E1000_VMOLR_AUPE
); /* Tagged packets ONLY */
4103 /* clear all bits that might not be set */
4104 vmolr
&= ~(E1000_VMOLR_BAM
| E1000_VMOLR_RSSE
);
4106 if (adapter
->rss_queues
> 1 && vfn
== adapter
->vfs_allocated_count
)
4107 vmolr
|= E1000_VMOLR_RSSE
; /* enable RSS */
4108 /* for VMDq only allow the VFs and pool 0 to accept broadcast and
4111 if (vfn
<= adapter
->vfs_allocated_count
)
4112 vmolr
|= E1000_VMOLR_BAM
; /* Accept broadcast */
4114 wr32(E1000_VMOLR(vfn
), vmolr
);
4118 * igb_configure_rx_ring - Configure a receive ring after Reset
4119 * @adapter: board private structure
4120 * @ring: receive ring to be configured
4122 * Configure the Rx unit of the MAC after a reset.
4124 void igb_configure_rx_ring(struct igb_adapter
*adapter
,
4125 struct igb_ring
*ring
)
4127 struct e1000_hw
*hw
= &adapter
->hw
;
4128 union e1000_adv_rx_desc
*rx_desc
;
4129 u64 rdba
= ring
->dma
;
4130 int reg_idx
= ring
->reg_idx
;
4131 u32 srrctl
= 0, rxdctl
= 0;
4133 /* disable the queue */
4134 wr32(E1000_RXDCTL(reg_idx
), 0);
4136 /* Set DMA base address registers */
4137 wr32(E1000_RDBAL(reg_idx
),
4138 rdba
& 0x00000000ffffffffULL
);
4139 wr32(E1000_RDBAH(reg_idx
), rdba
>> 32);
4140 wr32(E1000_RDLEN(reg_idx
),
4141 ring
->count
* sizeof(union e1000_adv_rx_desc
));
4143 /* initialize head and tail */
4144 ring
->tail
= adapter
->io_addr
+ E1000_RDT(reg_idx
);
4145 wr32(E1000_RDH(reg_idx
), 0);
4146 writel(0, ring
->tail
);
4148 /* set descriptor configuration */
4149 srrctl
= IGB_RX_HDR_LEN
<< E1000_SRRCTL_BSIZEHDRSIZE_SHIFT
;
4150 if (ring_uses_large_buffer(ring
))
4151 srrctl
|= IGB_RXBUFFER_3072
>> E1000_SRRCTL_BSIZEPKT_SHIFT
;
4153 srrctl
|= IGB_RXBUFFER_2048
>> E1000_SRRCTL_BSIZEPKT_SHIFT
;
4154 srrctl
|= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF
;
4155 if (hw
->mac
.type
>= e1000_82580
)
4156 srrctl
|= E1000_SRRCTL_TIMESTAMP
;
4157 /* Only set Drop Enable if we are supporting multiple queues */
4158 if (adapter
->vfs_allocated_count
|| adapter
->num_rx_queues
> 1)
4159 srrctl
|= E1000_SRRCTL_DROP_EN
;
4161 wr32(E1000_SRRCTL(reg_idx
), srrctl
);
4163 /* set filtering for VMDQ pools */
4164 igb_set_vmolr(adapter
, reg_idx
& 0x7, true);
4166 rxdctl
|= IGB_RX_PTHRESH
;
4167 rxdctl
|= IGB_RX_HTHRESH
<< 8;
4168 rxdctl
|= IGB_RX_WTHRESH
<< 16;
4170 /* initialize rx_buffer_info */
4171 memset(ring
->rx_buffer_info
, 0,
4172 sizeof(struct igb_rx_buffer
) * ring
->count
);
4174 /* initialize Rx descriptor 0 */
4175 rx_desc
= IGB_RX_DESC(ring
, 0);
4176 rx_desc
->wb
.upper
.length
= 0;
4178 /* enable receive descriptor fetching */
4179 rxdctl
|= E1000_RXDCTL_QUEUE_ENABLE
;
4180 wr32(E1000_RXDCTL(reg_idx
), rxdctl
);
4183 static void igb_set_rx_buffer_len(struct igb_adapter
*adapter
,
4184 struct igb_ring
*rx_ring
)
4186 /* set build_skb and buffer size flags */
4187 clear_ring_build_skb_enabled(rx_ring
);
4188 clear_ring_uses_large_buffer(rx_ring
);
4190 if (adapter
->flags
& IGB_FLAG_RX_LEGACY
)
4193 set_ring_build_skb_enabled(rx_ring
);
4195 #if (PAGE_SIZE < 8192)
4196 if (adapter
->max_frame_size
<= IGB_MAX_FRAME_BUILD_SKB
)
4199 set_ring_uses_large_buffer(rx_ring
);
4204 * igb_configure_rx - Configure receive Unit after Reset
4205 * @adapter: board private structure
4207 * Configure the Rx unit of the MAC after a reset.
4209 static void igb_configure_rx(struct igb_adapter
*adapter
)
4213 /* set the correct pool for the PF default MAC address in entry 0 */
4214 igb_set_default_mac_filter(adapter
);
4216 /* Setup the HW Rx Head and Tail Descriptor Pointers and
4217 * the Base and Length of the Rx Descriptor Ring
4219 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
4220 struct igb_ring
*rx_ring
= adapter
->rx_ring
[i
];
4222 igb_set_rx_buffer_len(adapter
, rx_ring
);
4223 igb_configure_rx_ring(adapter
, rx_ring
);
4228 * igb_free_tx_resources - Free Tx Resources per Queue
4229 * @tx_ring: Tx descriptor ring for a specific queue
4231 * Free all transmit software resources
4233 void igb_free_tx_resources(struct igb_ring
*tx_ring
)
4235 igb_clean_tx_ring(tx_ring
);
4237 vfree(tx_ring
->tx_buffer_info
);
4238 tx_ring
->tx_buffer_info
= NULL
;
4240 /* if not set, then don't free */
4244 dma_free_coherent(tx_ring
->dev
, tx_ring
->size
,
4245 tx_ring
->desc
, tx_ring
->dma
);
4247 tx_ring
->desc
= NULL
;
4251 * igb_free_all_tx_resources - Free Tx Resources for All Queues
4252 * @adapter: board private structure
4254 * Free all transmit software resources
4256 static void igb_free_all_tx_resources(struct igb_adapter
*adapter
)
4260 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
4261 if (adapter
->tx_ring
[i
])
4262 igb_free_tx_resources(adapter
->tx_ring
[i
]);
4266 * igb_clean_tx_ring - Free Tx Buffers
4267 * @tx_ring: ring to be cleaned
4269 static void igb_clean_tx_ring(struct igb_ring
*tx_ring
)
4271 u16 i
= tx_ring
->next_to_clean
;
4272 struct igb_tx_buffer
*tx_buffer
= &tx_ring
->tx_buffer_info
[i
];
4274 while (i
!= tx_ring
->next_to_use
) {
4275 union e1000_adv_tx_desc
*eop_desc
, *tx_desc
;
4277 /* Free all the Tx ring sk_buffs */
4278 dev_kfree_skb_any(tx_buffer
->skb
);
4280 /* unmap skb header data */
4281 dma_unmap_single(tx_ring
->dev
,
4282 dma_unmap_addr(tx_buffer
, dma
),
4283 dma_unmap_len(tx_buffer
, len
),
4286 /* check for eop_desc to determine the end of the packet */
4287 eop_desc
= tx_buffer
->next_to_watch
;
4288 tx_desc
= IGB_TX_DESC(tx_ring
, i
);
4290 /* unmap remaining buffers */
4291 while (tx_desc
!= eop_desc
) {
4295 if (unlikely(i
== tx_ring
->count
)) {
4297 tx_buffer
= tx_ring
->tx_buffer_info
;
4298 tx_desc
= IGB_TX_DESC(tx_ring
, 0);
4301 /* unmap any remaining paged data */
4302 if (dma_unmap_len(tx_buffer
, len
))
4303 dma_unmap_page(tx_ring
->dev
,
4304 dma_unmap_addr(tx_buffer
, dma
),
4305 dma_unmap_len(tx_buffer
, len
),
4309 /* move us one more past the eop_desc for start of next pkt */
4312 if (unlikely(i
== tx_ring
->count
)) {
4314 tx_buffer
= tx_ring
->tx_buffer_info
;
4318 /* reset BQL for queue */
4319 netdev_tx_reset_queue(txring_txq(tx_ring
));
4321 /* reset next_to_use and next_to_clean */
4322 tx_ring
->next_to_use
= 0;
4323 tx_ring
->next_to_clean
= 0;
4327 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
4328 * @adapter: board private structure
4330 static void igb_clean_all_tx_rings(struct igb_adapter
*adapter
)
4334 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
4335 if (adapter
->tx_ring
[i
])
4336 igb_clean_tx_ring(adapter
->tx_ring
[i
]);
4340 * igb_free_rx_resources - Free Rx Resources
4341 * @rx_ring: ring to clean the resources from
4343 * Free all receive software resources
4345 void igb_free_rx_resources(struct igb_ring
*rx_ring
)
4347 igb_clean_rx_ring(rx_ring
);
4349 vfree(rx_ring
->rx_buffer_info
);
4350 rx_ring
->rx_buffer_info
= NULL
;
4352 /* if not set, then don't free */
4356 dma_free_coherent(rx_ring
->dev
, rx_ring
->size
,
4357 rx_ring
->desc
, rx_ring
->dma
);
4359 rx_ring
->desc
= NULL
;
4363 * igb_free_all_rx_resources - Free Rx Resources for All Queues
4364 * @adapter: board private structure
4366 * Free all receive software resources
4368 static void igb_free_all_rx_resources(struct igb_adapter
*adapter
)
4372 for (i
= 0; i
< adapter
->num_rx_queues
; i
++)
4373 if (adapter
->rx_ring
[i
])
4374 igb_free_rx_resources(adapter
->rx_ring
[i
]);
4378 * igb_clean_rx_ring - Free Rx Buffers per Queue
4379 * @rx_ring: ring to free buffers from
4381 static void igb_clean_rx_ring(struct igb_ring
*rx_ring
)
4383 u16 i
= rx_ring
->next_to_clean
;
4386 dev_kfree_skb(rx_ring
->skb
);
4387 rx_ring
->skb
= NULL
;
4389 /* Free all the Rx ring sk_buffs */
4390 while (i
!= rx_ring
->next_to_alloc
) {
4391 struct igb_rx_buffer
*buffer_info
= &rx_ring
->rx_buffer_info
[i
];
4393 /* Invalidate cache lines that may have been written to by
4394 * device so that we avoid corrupting memory.
4396 dma_sync_single_range_for_cpu(rx_ring
->dev
,
4398 buffer_info
->page_offset
,
4399 igb_rx_bufsz(rx_ring
),
4402 /* free resources associated with mapping */
4403 dma_unmap_page_attrs(rx_ring
->dev
,
4405 igb_rx_pg_size(rx_ring
),
4408 __page_frag_cache_drain(buffer_info
->page
,
4409 buffer_info
->pagecnt_bias
);
4412 if (i
== rx_ring
->count
)
4416 rx_ring
->next_to_alloc
= 0;
4417 rx_ring
->next_to_clean
= 0;
4418 rx_ring
->next_to_use
= 0;
4422 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
4423 * @adapter: board private structure
4425 static void igb_clean_all_rx_rings(struct igb_adapter
*adapter
)
4429 for (i
= 0; i
< adapter
->num_rx_queues
; i
++)
4430 if (adapter
->rx_ring
[i
])
4431 igb_clean_rx_ring(adapter
->rx_ring
[i
]);
4435 * igb_set_mac - Change the Ethernet Address of the NIC
4436 * @netdev: network interface device structure
4437 * @p: pointer to an address structure
4439 * Returns 0 on success, negative on failure
4441 static int igb_set_mac(struct net_device
*netdev
, void *p
)
4443 struct igb_adapter
*adapter
= netdev_priv(netdev
);
4444 struct e1000_hw
*hw
= &adapter
->hw
;
4445 struct sockaddr
*addr
= p
;
4447 if (!is_valid_ether_addr(addr
->sa_data
))
4448 return -EADDRNOTAVAIL
;
4450 memcpy(netdev
->dev_addr
, addr
->sa_data
, netdev
->addr_len
);
4451 memcpy(hw
->mac
.addr
, addr
->sa_data
, netdev
->addr_len
);
4453 /* set the correct pool for the new PF MAC address in entry 0 */
4454 igb_set_default_mac_filter(adapter
);
4460 * igb_write_mc_addr_list - write multicast addresses to MTA
4461 * @netdev: network interface device structure
4463 * Writes multicast address list to the MTA hash table.
4464 * Returns: -ENOMEM on failure
4465 * 0 on no addresses written
4466 * X on writing X addresses to MTA
4468 static int igb_write_mc_addr_list(struct net_device
*netdev
)
4470 struct igb_adapter
*adapter
= netdev_priv(netdev
);
4471 struct e1000_hw
*hw
= &adapter
->hw
;
4472 struct netdev_hw_addr
*ha
;
4476 if (netdev_mc_empty(netdev
)) {
4477 /* nothing to program, so clear mc list */
4478 igb_update_mc_addr_list(hw
, NULL
, 0);
4479 igb_restore_vf_multicasts(adapter
);
4483 mta_list
= kzalloc(netdev_mc_count(netdev
) * 6, GFP_ATOMIC
);
4487 /* The shared function expects a packed array of only addresses. */
4489 netdev_for_each_mc_addr(ha
, netdev
)
4490 memcpy(mta_list
+ (i
++ * ETH_ALEN
), ha
->addr
, ETH_ALEN
);
4492 igb_update_mc_addr_list(hw
, mta_list
, i
);
4495 return netdev_mc_count(netdev
);
4498 static int igb_vlan_promisc_enable(struct igb_adapter
*adapter
)
4500 struct e1000_hw
*hw
= &adapter
->hw
;
4503 switch (hw
->mac
.type
) {
4507 /* VLAN filtering needed for VLAN prio filter */
4508 if (adapter
->netdev
->features
& NETIF_F_NTUPLE
)
4514 /* VLAN filtering needed for pool filtering */
4515 if (adapter
->vfs_allocated_count
)
4522 /* We are already in VLAN promisc, nothing to do */
4523 if (adapter
->flags
& IGB_FLAG_VLAN_PROMISC
)
4526 if (!adapter
->vfs_allocated_count
)
4529 /* Add PF to all active pools */
4530 pf_id
= adapter
->vfs_allocated_count
+ E1000_VLVF_POOLSEL_SHIFT
;
4532 for (i
= E1000_VLVF_ARRAY_SIZE
; --i
;) {
4533 u32 vlvf
= rd32(E1000_VLVF(i
));
4536 wr32(E1000_VLVF(i
), vlvf
);
4540 /* Set all bits in the VLAN filter table array */
4541 for (i
= E1000_VLAN_FILTER_TBL_SIZE
; i
--;)
4542 hw
->mac
.ops
.write_vfta(hw
, i
, ~0U);
4544 /* Set flag so we don't redo unnecessary work */
4545 adapter
->flags
|= IGB_FLAG_VLAN_PROMISC
;
4550 #define VFTA_BLOCK_SIZE 8
4551 static void igb_scrub_vfta(struct igb_adapter
*adapter
, u32 vfta_offset
)
4553 struct e1000_hw
*hw
= &adapter
->hw
;
4554 u32 vfta
[VFTA_BLOCK_SIZE
] = { 0 };
4555 u32 vid_start
= vfta_offset
* 32;
4556 u32 vid_end
= vid_start
+ (VFTA_BLOCK_SIZE
* 32);
4557 u32 i
, vid
, word
, bits
, pf_id
;
4559 /* guarantee that we don't scrub out management VLAN */
4560 vid
= adapter
->mng_vlan_id
;
4561 if (vid
>= vid_start
&& vid
< vid_end
)
4562 vfta
[(vid
- vid_start
) / 32] |= BIT(vid
% 32);
4564 if (!adapter
->vfs_allocated_count
)
4567 pf_id
= adapter
->vfs_allocated_count
+ E1000_VLVF_POOLSEL_SHIFT
;
4569 for (i
= E1000_VLVF_ARRAY_SIZE
; --i
;) {
4570 u32 vlvf
= rd32(E1000_VLVF(i
));
4572 /* pull VLAN ID from VLVF */
4573 vid
= vlvf
& VLAN_VID_MASK
;
4575 /* only concern ourselves with a certain range */
4576 if (vid
< vid_start
|| vid
>= vid_end
)
4579 if (vlvf
& E1000_VLVF_VLANID_ENABLE
) {
4580 /* record VLAN ID in VFTA */
4581 vfta
[(vid
- vid_start
) / 32] |= BIT(vid
% 32);
4583 /* if PF is part of this then continue */
4584 if (test_bit(vid
, adapter
->active_vlans
))
4588 /* remove PF from the pool */
4590 bits
&= rd32(E1000_VLVF(i
));
4591 wr32(E1000_VLVF(i
), bits
);
4595 /* extract values from active_vlans and write back to VFTA */
4596 for (i
= VFTA_BLOCK_SIZE
; i
--;) {
4597 vid
= (vfta_offset
+ i
) * 32;
4598 word
= vid
/ BITS_PER_LONG
;
4599 bits
= vid
% BITS_PER_LONG
;
4601 vfta
[i
] |= adapter
->active_vlans
[word
] >> bits
;
4603 hw
->mac
.ops
.write_vfta(hw
, vfta_offset
+ i
, vfta
[i
]);
4607 static void igb_vlan_promisc_disable(struct igb_adapter
*adapter
)
4611 /* We are not in VLAN promisc, nothing to do */
4612 if (!(adapter
->flags
& IGB_FLAG_VLAN_PROMISC
))
4615 /* Set flag so we don't redo unnecessary work */
4616 adapter
->flags
&= ~IGB_FLAG_VLAN_PROMISC
;
4618 for (i
= 0; i
< E1000_VLAN_FILTER_TBL_SIZE
; i
+= VFTA_BLOCK_SIZE
)
4619 igb_scrub_vfta(adapter
, i
);
4623 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
4624 * @netdev: network interface device structure
4626 * The set_rx_mode entry point is called whenever the unicast or multicast
4627 * address lists or the network interface flags are updated. This routine is
4628 * responsible for configuring the hardware for proper unicast, multicast,
4629 * promiscuous mode, and all-multi behavior.
4631 static void igb_set_rx_mode(struct net_device
*netdev
)
4633 struct igb_adapter
*adapter
= netdev_priv(netdev
);
4634 struct e1000_hw
*hw
= &adapter
->hw
;
4635 unsigned int vfn
= adapter
->vfs_allocated_count
;
4636 u32 rctl
= 0, vmolr
= 0, rlpml
= MAX_JUMBO_FRAME_SIZE
;
4639 /* Check for Promiscuous and All Multicast modes */
4640 if (netdev
->flags
& IFF_PROMISC
) {
4641 rctl
|= E1000_RCTL_UPE
| E1000_RCTL_MPE
;
4642 vmolr
|= E1000_VMOLR_MPME
;
4644 /* enable use of UTA filter to force packets to default pool */
4645 if (hw
->mac
.type
== e1000_82576
)
4646 vmolr
|= E1000_VMOLR_ROPE
;
4648 if (netdev
->flags
& IFF_ALLMULTI
) {
4649 rctl
|= E1000_RCTL_MPE
;
4650 vmolr
|= E1000_VMOLR_MPME
;
4652 /* Write addresses to the MTA, if the attempt fails
4653 * then we should just turn on promiscuous mode so
4654 * that we can at least receive multicast traffic
4656 count
= igb_write_mc_addr_list(netdev
);
4658 rctl
|= E1000_RCTL_MPE
;
4659 vmolr
|= E1000_VMOLR_MPME
;
4661 vmolr
|= E1000_VMOLR_ROMPE
;
4666 /* Write addresses to available RAR registers, if there is not
4667 * sufficient space to store all the addresses then enable
4668 * unicast promiscuous mode
4670 if (__dev_uc_sync(netdev
, igb_uc_sync
, igb_uc_unsync
)) {
4671 rctl
|= E1000_RCTL_UPE
;
4672 vmolr
|= E1000_VMOLR_ROPE
;
4675 /* enable VLAN filtering by default */
4676 rctl
|= E1000_RCTL_VFE
;
4678 /* disable VLAN filtering for modes that require it */
4679 if ((netdev
->flags
& IFF_PROMISC
) ||
4680 (netdev
->features
& NETIF_F_RXALL
)) {
4681 /* if we fail to set all rules then just clear VFE */
4682 if (igb_vlan_promisc_enable(adapter
))
4683 rctl
&= ~E1000_RCTL_VFE
;
4685 igb_vlan_promisc_disable(adapter
);
4688 /* update state of unicast, multicast, and VLAN filtering modes */
4689 rctl
|= rd32(E1000_RCTL
) & ~(E1000_RCTL_UPE
| E1000_RCTL_MPE
|
4691 wr32(E1000_RCTL
, rctl
);
4693 #if (PAGE_SIZE < 8192)
4694 if (!adapter
->vfs_allocated_count
) {
4695 if (adapter
->max_frame_size
<= IGB_MAX_FRAME_BUILD_SKB
)
4696 rlpml
= IGB_MAX_FRAME_BUILD_SKB
;
4699 wr32(E1000_RLPML
, rlpml
);
4701 /* In order to support SR-IOV and eventually VMDq it is necessary to set
4702 * the VMOLR to enable the appropriate modes. Without this workaround
4703 * we will have issues with VLAN tag stripping not being done for frames
4704 * that are only arriving because we are the default pool
4706 if ((hw
->mac
.type
< e1000_82576
) || (hw
->mac
.type
> e1000_i350
))
4709 /* set UTA to appropriate mode */
4710 igb_set_uta(adapter
, !!(vmolr
& E1000_VMOLR_ROPE
));
4712 vmolr
|= rd32(E1000_VMOLR(vfn
)) &
4713 ~(E1000_VMOLR_ROPE
| E1000_VMOLR_MPME
| E1000_VMOLR_ROMPE
);
4715 /* enable Rx jumbo frames, restrict as needed to support build_skb */
4716 vmolr
&= ~E1000_VMOLR_RLPML_MASK
;
4717 #if (PAGE_SIZE < 8192)
4718 if (adapter
->max_frame_size
<= IGB_MAX_FRAME_BUILD_SKB
)
4719 vmolr
|= IGB_MAX_FRAME_BUILD_SKB
;
4722 vmolr
|= MAX_JUMBO_FRAME_SIZE
;
4723 vmolr
|= E1000_VMOLR_LPE
;
4725 wr32(E1000_VMOLR(vfn
), vmolr
);
4727 igb_restore_vf_multicasts(adapter
);
4730 static void igb_check_wvbr(struct igb_adapter
*adapter
)
4732 struct e1000_hw
*hw
= &adapter
->hw
;
4735 switch (hw
->mac
.type
) {
4738 wvbr
= rd32(E1000_WVBR
);
4746 adapter
->wvbr
|= wvbr
;
4749 #define IGB_STAGGERED_QUEUE_OFFSET 8
4751 static void igb_spoof_check(struct igb_adapter
*adapter
)
4758 for (j
= 0; j
< adapter
->vfs_allocated_count
; j
++) {
4759 if (adapter
->wvbr
& BIT(j
) ||
4760 adapter
->wvbr
& BIT(j
+ IGB_STAGGERED_QUEUE_OFFSET
)) {
4761 dev_warn(&adapter
->pdev
->dev
,
4762 "Spoof event(s) detected on VF %d\n", j
);
4765 BIT(j
+ IGB_STAGGERED_QUEUE_OFFSET
));
4770 /* Need to wait a few seconds after link up to get diagnostic information from
4773 static void igb_update_phy_info(struct timer_list
*t
)
4775 struct igb_adapter
*adapter
= from_timer(adapter
, t
, phy_info_timer
);
4776 igb_get_phy_info(&adapter
->hw
);
4780 * igb_has_link - check shared code for link and determine up/down
4781 * @adapter: pointer to driver private info
4783 bool igb_has_link(struct igb_adapter
*adapter
)
4785 struct e1000_hw
*hw
= &adapter
->hw
;
4786 bool link_active
= false;
4788 /* get_link_status is set on LSC (link status) interrupt or
4789 * rx sequence error interrupt. get_link_status will stay
4790 * false until the e1000_check_for_link establishes link
4791 * for copper adapters ONLY
4793 switch (hw
->phy
.media_type
) {
4794 case e1000_media_type_copper
:
4795 if (!hw
->mac
.get_link_status
)
4797 case e1000_media_type_internal_serdes
:
4798 hw
->mac
.ops
.check_for_link(hw
);
4799 link_active
= !hw
->mac
.get_link_status
;
4802 case e1000_media_type_unknown
:
4806 if (((hw
->mac
.type
== e1000_i210
) ||
4807 (hw
->mac
.type
== e1000_i211
)) &&
4808 (hw
->phy
.id
== I210_I_PHY_ID
)) {
4809 if (!netif_carrier_ok(adapter
->netdev
)) {
4810 adapter
->flags
&= ~IGB_FLAG_NEED_LINK_UPDATE
;
4811 } else if (!(adapter
->flags
& IGB_FLAG_NEED_LINK_UPDATE
)) {
4812 adapter
->flags
|= IGB_FLAG_NEED_LINK_UPDATE
;
4813 adapter
->link_check_timeout
= jiffies
;
4820 static bool igb_thermal_sensor_event(struct e1000_hw
*hw
, u32 event
)
4823 u32 ctrl_ext
, thstat
;
4825 /* check for thermal sensor event on i350 copper only */
4826 if (hw
->mac
.type
== e1000_i350
) {
4827 thstat
= rd32(E1000_THSTAT
);
4828 ctrl_ext
= rd32(E1000_CTRL_EXT
);
4830 if ((hw
->phy
.media_type
== e1000_media_type_copper
) &&
4831 !(ctrl_ext
& E1000_CTRL_EXT_LINK_MODE_SGMII
))
4832 ret
= !!(thstat
& event
);
4839 * igb_check_lvmmc - check for malformed packets received
4840 * and indicated in LVMMC register
4841 * @adapter: pointer to adapter
4843 static void igb_check_lvmmc(struct igb_adapter
*adapter
)
4845 struct e1000_hw
*hw
= &adapter
->hw
;
4848 lvmmc
= rd32(E1000_LVMMC
);
4850 if (unlikely(net_ratelimit())) {
4851 netdev_warn(adapter
->netdev
,
4852 "malformed Tx packet detected and dropped, LVMMC:0x%08x\n",
4859 * igb_watchdog - Timer Call-back
4860 * @data: pointer to adapter cast into an unsigned long
4862 static void igb_watchdog(struct timer_list
*t
)
4864 struct igb_adapter
*adapter
= from_timer(adapter
, t
, watchdog_timer
);
4865 /* Do the rest outside of interrupt context */
4866 schedule_work(&adapter
->watchdog_task
);
4869 static void igb_watchdog_task(struct work_struct
*work
)
4871 struct igb_adapter
*adapter
= container_of(work
,
4874 struct e1000_hw
*hw
= &adapter
->hw
;
4875 struct e1000_phy_info
*phy
= &hw
->phy
;
4876 struct net_device
*netdev
= adapter
->netdev
;
4880 u16 phy_data
, retry_count
= 20;
4882 link
= igb_has_link(adapter
);
4884 if (adapter
->flags
& IGB_FLAG_NEED_LINK_UPDATE
) {
4885 if (time_after(jiffies
, (adapter
->link_check_timeout
+ HZ
)))
4886 adapter
->flags
&= ~IGB_FLAG_NEED_LINK_UPDATE
;
4891 /* Force link down if we have fiber to swap to */
4892 if (adapter
->flags
& IGB_FLAG_MAS_ENABLE
) {
4893 if (hw
->phy
.media_type
== e1000_media_type_copper
) {
4894 connsw
= rd32(E1000_CONNSW
);
4895 if (!(connsw
& E1000_CONNSW_AUTOSENSE_EN
))
4900 /* Perform a reset if the media type changed. */
4901 if (hw
->dev_spec
._82575
.media_changed
) {
4902 hw
->dev_spec
._82575
.media_changed
= false;
4903 adapter
->flags
|= IGB_FLAG_MEDIA_RESET
;
4906 /* Cancel scheduled suspend requests. */
4907 pm_runtime_resume(netdev
->dev
.parent
);
4909 if (!netif_carrier_ok(netdev
)) {
4912 hw
->mac
.ops
.get_speed_and_duplex(hw
,
4913 &adapter
->link_speed
,
4914 &adapter
->link_duplex
);
4916 ctrl
= rd32(E1000_CTRL
);
4917 /* Links status message must follow this format */
4919 "igb: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
4921 adapter
->link_speed
,
4922 adapter
->link_duplex
== FULL_DUPLEX
?
4924 (ctrl
& E1000_CTRL_TFCE
) &&
4925 (ctrl
& E1000_CTRL_RFCE
) ? "RX/TX" :
4926 (ctrl
& E1000_CTRL_RFCE
) ? "RX" :
4927 (ctrl
& E1000_CTRL_TFCE
) ? "TX" : "None");
4929 /* disable EEE if enabled */
4930 if ((adapter
->flags
& IGB_FLAG_EEE
) &&
4931 (adapter
->link_duplex
== HALF_DUPLEX
)) {
4932 dev_info(&adapter
->pdev
->dev
,
4933 "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex.\n");
4934 adapter
->hw
.dev_spec
._82575
.eee_disable
= true;
4935 adapter
->flags
&= ~IGB_FLAG_EEE
;
4938 /* check if SmartSpeed worked */
4939 igb_check_downshift(hw
);
4940 if (phy
->speed_downgraded
)
4941 netdev_warn(netdev
, "Link Speed was downgraded by SmartSpeed\n");
4943 /* check for thermal sensor event */
4944 if (igb_thermal_sensor_event(hw
,
4945 E1000_THSTAT_LINK_THROTTLE
))
4946 netdev_info(netdev
, "The network adapter link speed was downshifted because it overheated\n");
4948 /* adjust timeout factor according to speed/duplex */
4949 adapter
->tx_timeout_factor
= 1;
4950 switch (adapter
->link_speed
) {
4952 adapter
->tx_timeout_factor
= 14;
4955 /* maybe add some timeout factor ? */
4959 if (adapter
->link_speed
!= SPEED_1000
)
4962 /* wait for Remote receiver status OK */
4964 if (!igb_read_phy_reg(hw
, PHY_1000T_STATUS
,
4966 if (!(phy_data
& SR_1000T_REMOTE_RX_STATUS
) &&
4970 goto retry_read_status
;
4971 } else if (!retry_count
) {
4972 dev_err(&adapter
->pdev
->dev
, "exceed max 2 second\n");
4975 dev_err(&adapter
->pdev
->dev
, "read 1000Base-T Status Reg\n");
4978 netif_carrier_on(netdev
);
4980 igb_ping_all_vfs(adapter
);
4981 igb_check_vf_rate_limit(adapter
);
4983 /* link state has changed, schedule phy info update */
4984 if (!test_bit(__IGB_DOWN
, &adapter
->state
))
4985 mod_timer(&adapter
->phy_info_timer
,
4986 round_jiffies(jiffies
+ 2 * HZ
));
4989 if (netif_carrier_ok(netdev
)) {
4990 adapter
->link_speed
= 0;
4991 adapter
->link_duplex
= 0;
4993 /* check for thermal sensor event */
4994 if (igb_thermal_sensor_event(hw
,
4995 E1000_THSTAT_PWR_DOWN
)) {
4996 netdev_err(netdev
, "The network adapter was stopped because it overheated\n");
4999 /* Links status message must follow this format */
5000 netdev_info(netdev
, "igb: %s NIC Link is Down\n",
5002 netif_carrier_off(netdev
);
5004 igb_ping_all_vfs(adapter
);
5006 /* link state has changed, schedule phy info update */
5007 if (!test_bit(__IGB_DOWN
, &adapter
->state
))
5008 mod_timer(&adapter
->phy_info_timer
,
5009 round_jiffies(jiffies
+ 2 * HZ
));
5011 /* link is down, time to check for alternate media */
5012 if (adapter
->flags
& IGB_FLAG_MAS_ENABLE
) {
5013 igb_check_swap_media(adapter
);
5014 if (adapter
->flags
& IGB_FLAG_MEDIA_RESET
) {
5015 schedule_work(&adapter
->reset_task
);
5016 /* return immediately */
5020 pm_schedule_suspend(netdev
->dev
.parent
,
5023 /* also check for alternate media here */
5024 } else if (!netif_carrier_ok(netdev
) &&
5025 (adapter
->flags
& IGB_FLAG_MAS_ENABLE
)) {
5026 igb_check_swap_media(adapter
);
5027 if (adapter
->flags
& IGB_FLAG_MEDIA_RESET
) {
5028 schedule_work(&adapter
->reset_task
);
5029 /* return immediately */
5035 spin_lock(&adapter
->stats64_lock
);
5036 igb_update_stats(adapter
);
5037 spin_unlock(&adapter
->stats64_lock
);
5039 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
5040 struct igb_ring
*tx_ring
= adapter
->tx_ring
[i
];
5041 if (!netif_carrier_ok(netdev
)) {
5042 /* We've lost link, so the controller stops DMA,
5043 * but we've got queued Tx work that's never going
5044 * to get done, so reset controller to flush Tx.
5045 * (Do the reset outside of interrupt context).
5047 if (igb_desc_unused(tx_ring
) + 1 < tx_ring
->count
) {
5048 adapter
->tx_timeout_count
++;
5049 schedule_work(&adapter
->reset_task
);
5050 /* return immediately since reset is imminent */
5055 /* Force detection of hung controller every watchdog period */
5056 set_bit(IGB_RING_FLAG_TX_DETECT_HANG
, &tx_ring
->flags
);
5059 /* Cause software interrupt to ensure Rx ring is cleaned */
5060 if (adapter
->flags
& IGB_FLAG_HAS_MSIX
) {
5063 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
5064 eics
|= adapter
->q_vector
[i
]->eims_value
;
5065 wr32(E1000_EICS
, eics
);
5067 wr32(E1000_ICS
, E1000_ICS_RXDMT0
);
5070 igb_spoof_check(adapter
);
5071 igb_ptp_rx_hang(adapter
);
5072 igb_ptp_tx_hang(adapter
);
5074 /* Check LVMMC register on i350/i354 only */
5075 if ((adapter
->hw
.mac
.type
== e1000_i350
) ||
5076 (adapter
->hw
.mac
.type
== e1000_i354
))
5077 igb_check_lvmmc(adapter
);
5079 /* Reset the timer */
5080 if (!test_bit(__IGB_DOWN
, &adapter
->state
)) {
5081 if (adapter
->flags
& IGB_FLAG_NEED_LINK_UPDATE
)
5082 mod_timer(&adapter
->watchdog_timer
,
5083 round_jiffies(jiffies
+ HZ
));
5085 mod_timer(&adapter
->watchdog_timer
,
5086 round_jiffies(jiffies
+ 2 * HZ
));
5090 enum latency_range
{
5094 latency_invalid
= 255
5098 * igb_update_ring_itr - update the dynamic ITR value based on packet size
5099 * @q_vector: pointer to q_vector
5101 * Stores a new ITR value based on strictly on packet size. This
5102 * algorithm is less sophisticated than that used in igb_update_itr,
5103 * due to the difficulty of synchronizing statistics across multiple
5104 * receive rings. The divisors and thresholds used by this function
5105 * were determined based on theoretical maximum wire speed and testing
5106 * data, in order to minimize response time while increasing bulk
5108 * This functionality is controlled by ethtool's coalescing settings.
5109 * NOTE: This function is called only when operating in a multiqueue
5110 * receive environment.
5112 static void igb_update_ring_itr(struct igb_q_vector
*q_vector
)
5114 int new_val
= q_vector
->itr_val
;
5115 int avg_wire_size
= 0;
5116 struct igb_adapter
*adapter
= q_vector
->adapter
;
5117 unsigned int packets
;
5119 /* For non-gigabit speeds, just fix the interrupt rate at 4000
5120 * ints/sec - ITR timer value of 120 ticks.
5122 if (adapter
->link_speed
!= SPEED_1000
) {
5123 new_val
= IGB_4K_ITR
;
5127 packets
= q_vector
->rx
.total_packets
;
5129 avg_wire_size
= q_vector
->rx
.total_bytes
/ packets
;
5131 packets
= q_vector
->tx
.total_packets
;
5133 avg_wire_size
= max_t(u32
, avg_wire_size
,
5134 q_vector
->tx
.total_bytes
/ packets
);
5136 /* if avg_wire_size isn't set no work was done */
5140 /* Add 24 bytes to size to account for CRC, preamble, and gap */
5141 avg_wire_size
+= 24;
5143 /* Don't starve jumbo frames */
5144 avg_wire_size
= min(avg_wire_size
, 3000);
5146 /* Give a little boost to mid-size frames */
5147 if ((avg_wire_size
> 300) && (avg_wire_size
< 1200))
5148 new_val
= avg_wire_size
/ 3;
5150 new_val
= avg_wire_size
/ 2;
5152 /* conservative mode (itr 3) eliminates the lowest_latency setting */
5153 if (new_val
< IGB_20K_ITR
&&
5154 ((q_vector
->rx
.ring
&& adapter
->rx_itr_setting
== 3) ||
5155 (!q_vector
->rx
.ring
&& adapter
->tx_itr_setting
== 3)))
5156 new_val
= IGB_20K_ITR
;
5159 if (new_val
!= q_vector
->itr_val
) {
5160 q_vector
->itr_val
= new_val
;
5161 q_vector
->set_itr
= 1;
5164 q_vector
->rx
.total_bytes
= 0;
5165 q_vector
->rx
.total_packets
= 0;
5166 q_vector
->tx
.total_bytes
= 0;
5167 q_vector
->tx
.total_packets
= 0;
5171 * igb_update_itr - update the dynamic ITR value based on statistics
5172 * @q_vector: pointer to q_vector
5173 * @ring_container: ring info to update the itr for
5175 * Stores a new ITR value based on packets and byte
5176 * counts during the last interrupt. The advantage of per interrupt
5177 * computation is faster updates and more accurate ITR for the current
5178 * traffic pattern. Constants in this function were computed
5179 * based on theoretical maximum wire speed and thresholds were set based
5180 * on testing data as well as attempting to minimize response time
5181 * while increasing bulk throughput.
5182 * This functionality is controlled by ethtool's coalescing settings.
5183 * NOTE: These calculations are only valid when operating in a single-
5184 * queue environment.
5186 static void igb_update_itr(struct igb_q_vector
*q_vector
,
5187 struct igb_ring_container
*ring_container
)
5189 unsigned int packets
= ring_container
->total_packets
;
5190 unsigned int bytes
= ring_container
->total_bytes
;
5191 u8 itrval
= ring_container
->itr
;
5193 /* no packets, exit with status unchanged */
5198 case lowest_latency
:
5199 /* handle TSO and jumbo frames */
5200 if (bytes
/packets
> 8000)
5201 itrval
= bulk_latency
;
5202 else if ((packets
< 5) && (bytes
> 512))
5203 itrval
= low_latency
;
5205 case low_latency
: /* 50 usec aka 20000 ints/s */
5206 if (bytes
> 10000) {
5207 /* this if handles the TSO accounting */
5208 if (bytes
/packets
> 8000)
5209 itrval
= bulk_latency
;
5210 else if ((packets
< 10) || ((bytes
/packets
) > 1200))
5211 itrval
= bulk_latency
;
5212 else if ((packets
> 35))
5213 itrval
= lowest_latency
;
5214 } else if (bytes
/packets
> 2000) {
5215 itrval
= bulk_latency
;
5216 } else if (packets
<= 2 && bytes
< 512) {
5217 itrval
= lowest_latency
;
5220 case bulk_latency
: /* 250 usec aka 4000 ints/s */
5221 if (bytes
> 25000) {
5223 itrval
= low_latency
;
5224 } else if (bytes
< 1500) {
5225 itrval
= low_latency
;
5230 /* clear work counters since we have the values we need */
5231 ring_container
->total_bytes
= 0;
5232 ring_container
->total_packets
= 0;
5234 /* write updated itr to ring container */
5235 ring_container
->itr
= itrval
;
5238 static void igb_set_itr(struct igb_q_vector
*q_vector
)
5240 struct igb_adapter
*adapter
= q_vector
->adapter
;
5241 u32 new_itr
= q_vector
->itr_val
;
5244 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
5245 if (adapter
->link_speed
!= SPEED_1000
) {
5247 new_itr
= IGB_4K_ITR
;
5251 igb_update_itr(q_vector
, &q_vector
->tx
);
5252 igb_update_itr(q_vector
, &q_vector
->rx
);
5254 current_itr
= max(q_vector
->rx
.itr
, q_vector
->tx
.itr
);
5256 /* conservative mode (itr 3) eliminates the lowest_latency setting */
5257 if (current_itr
== lowest_latency
&&
5258 ((q_vector
->rx
.ring
&& adapter
->rx_itr_setting
== 3) ||
5259 (!q_vector
->rx
.ring
&& adapter
->tx_itr_setting
== 3)))
5260 current_itr
= low_latency
;
5262 switch (current_itr
) {
5263 /* counts and packets in update_itr are dependent on these numbers */
5264 case lowest_latency
:
5265 new_itr
= IGB_70K_ITR
; /* 70,000 ints/sec */
5268 new_itr
= IGB_20K_ITR
; /* 20,000 ints/sec */
5271 new_itr
= IGB_4K_ITR
; /* 4,000 ints/sec */
5278 if (new_itr
!= q_vector
->itr_val
) {
5279 /* this attempts to bias the interrupt rate towards Bulk
5280 * by adding intermediate steps when interrupt rate is
5283 new_itr
= new_itr
> q_vector
->itr_val
?
5284 max((new_itr
* q_vector
->itr_val
) /
5285 (new_itr
+ (q_vector
->itr_val
>> 2)),
5287 /* Don't write the value here; it resets the adapter's
5288 * internal timer, and causes us to delay far longer than
5289 * we should between interrupts. Instead, we write the ITR
5290 * value at the beginning of the next interrupt so the timing
5291 * ends up being correct.
5293 q_vector
->itr_val
= new_itr
;
5294 q_vector
->set_itr
= 1;
5298 static void igb_tx_ctxtdesc(struct igb_ring
*tx_ring
, u32 vlan_macip_lens
,
5299 u32 type_tucmd
, u32 mss_l4len_idx
)
5301 struct e1000_adv_tx_context_desc
*context_desc
;
5302 u16 i
= tx_ring
->next_to_use
;
5304 context_desc
= IGB_TX_CTXTDESC(tx_ring
, i
);
5307 tx_ring
->next_to_use
= (i
< tx_ring
->count
) ? i
: 0;
5309 /* set bits to identify this as an advanced context descriptor */
5310 type_tucmd
|= E1000_TXD_CMD_DEXT
| E1000_ADVTXD_DTYP_CTXT
;
5312 /* For 82575, context index must be unique per ring. */
5313 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX
, &tx_ring
->flags
))
5314 mss_l4len_idx
|= tx_ring
->reg_idx
<< 4;
5316 context_desc
->vlan_macip_lens
= cpu_to_le32(vlan_macip_lens
);
5317 context_desc
->seqnum_seed
= 0;
5318 context_desc
->type_tucmd_mlhl
= cpu_to_le32(type_tucmd
);
5319 context_desc
->mss_l4len_idx
= cpu_to_le32(mss_l4len_idx
);
5322 static int igb_tso(struct igb_ring
*tx_ring
,
5323 struct igb_tx_buffer
*first
,
5326 u32 vlan_macip_lens
, type_tucmd
, mss_l4len_idx
;
5327 struct sk_buff
*skb
= first
->skb
;
5337 u32 paylen
, l4_offset
;
5340 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
5343 if (!skb_is_gso(skb
))
5346 err
= skb_cow_head(skb
, 0);
5350 ip
.hdr
= skb_network_header(skb
);
5351 l4
.hdr
= skb_checksum_start(skb
);
5353 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
5354 type_tucmd
= E1000_ADVTXD_TUCMD_L4T_TCP
;
5356 /* initialize outer IP header fields */
5357 if (ip
.v4
->version
== 4) {
5358 unsigned char *csum_start
= skb_checksum_start(skb
);
5359 unsigned char *trans_start
= ip
.hdr
+ (ip
.v4
->ihl
* 4);
5361 /* IP header will have to cancel out any data that
5362 * is not a part of the outer IP header
5364 ip
.v4
->check
= csum_fold(csum_partial(trans_start
,
5365 csum_start
- trans_start
,
5367 type_tucmd
|= E1000_ADVTXD_TUCMD_IPV4
;
5370 first
->tx_flags
|= IGB_TX_FLAGS_TSO
|
5374 ip
.v6
->payload_len
= 0;
5375 first
->tx_flags
|= IGB_TX_FLAGS_TSO
|
5379 /* determine offset of inner transport header */
5380 l4_offset
= l4
.hdr
- skb
->data
;
5382 /* compute length of segmentation header */
5383 *hdr_len
= (l4
.tcp
->doff
* 4) + l4_offset
;
5385 /* remove payload length from inner checksum */
5386 paylen
= skb
->len
- l4_offset
;
5387 csum_replace_by_diff(&l4
.tcp
->check
, htonl(paylen
));
5389 /* update gso size and bytecount with header size */
5390 first
->gso_segs
= skb_shinfo(skb
)->gso_segs
;
5391 first
->bytecount
+= (first
->gso_segs
- 1) * *hdr_len
;
5394 mss_l4len_idx
= (*hdr_len
- l4_offset
) << E1000_ADVTXD_L4LEN_SHIFT
;
5395 mss_l4len_idx
|= skb_shinfo(skb
)->gso_size
<< E1000_ADVTXD_MSS_SHIFT
;
5397 /* VLAN MACLEN IPLEN */
5398 vlan_macip_lens
= l4
.hdr
- ip
.hdr
;
5399 vlan_macip_lens
|= (ip
.hdr
- skb
->data
) << E1000_ADVTXD_MACLEN_SHIFT
;
5400 vlan_macip_lens
|= first
->tx_flags
& IGB_TX_FLAGS_VLAN_MASK
;
5402 igb_tx_ctxtdesc(tx_ring
, vlan_macip_lens
, type_tucmd
, mss_l4len_idx
);
5407 static inline bool igb_ipv6_csum_is_sctp(struct sk_buff
*skb
)
5409 unsigned int offset
= 0;
5411 ipv6_find_hdr(skb
, &offset
, IPPROTO_SCTP
, NULL
, NULL
);
5413 return offset
== skb_checksum_start_offset(skb
);
5416 static void igb_tx_csum(struct igb_ring
*tx_ring
, struct igb_tx_buffer
*first
)
5418 struct sk_buff
*skb
= first
->skb
;
5419 u32 vlan_macip_lens
= 0;
5422 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
) {
5424 if (!(first
->tx_flags
& IGB_TX_FLAGS_VLAN
))
5429 switch (skb
->csum_offset
) {
5430 case offsetof(struct tcphdr
, check
):
5431 type_tucmd
= E1000_ADVTXD_TUCMD_L4T_TCP
;
5433 case offsetof(struct udphdr
, check
):
5435 case offsetof(struct sctphdr
, checksum
):
5436 /* validate that this is actually an SCTP request */
5437 if (((first
->protocol
== htons(ETH_P_IP
)) &&
5438 (ip_hdr(skb
)->protocol
== IPPROTO_SCTP
)) ||
5439 ((first
->protocol
== htons(ETH_P_IPV6
)) &&
5440 igb_ipv6_csum_is_sctp(skb
))) {
5441 type_tucmd
= E1000_ADVTXD_TUCMD_L4T_SCTP
;
5445 skb_checksum_help(skb
);
5449 /* update TX checksum flag */
5450 first
->tx_flags
|= IGB_TX_FLAGS_CSUM
;
5451 vlan_macip_lens
= skb_checksum_start_offset(skb
) -
5452 skb_network_offset(skb
);
5454 vlan_macip_lens
|= skb_network_offset(skb
) << E1000_ADVTXD_MACLEN_SHIFT
;
5455 vlan_macip_lens
|= first
->tx_flags
& IGB_TX_FLAGS_VLAN_MASK
;
5457 igb_tx_ctxtdesc(tx_ring
, vlan_macip_lens
, type_tucmd
, 0);
5460 #define IGB_SET_FLAG(_input, _flag, _result) \
5461 ((_flag <= _result) ? \
5462 ((u32)(_input & _flag) * (_result / _flag)) : \
5463 ((u32)(_input & _flag) / (_flag / _result)))
5465 static u32
igb_tx_cmd_type(struct sk_buff
*skb
, u32 tx_flags
)
5467 /* set type for advanced descriptor with frame checksum insertion */
5468 u32 cmd_type
= E1000_ADVTXD_DTYP_DATA
|
5469 E1000_ADVTXD_DCMD_DEXT
|
5470 E1000_ADVTXD_DCMD_IFCS
;
5472 /* set HW vlan bit if vlan is present */
5473 cmd_type
|= IGB_SET_FLAG(tx_flags
, IGB_TX_FLAGS_VLAN
,
5474 (E1000_ADVTXD_DCMD_VLE
));
5476 /* set segmentation bits for TSO */
5477 cmd_type
|= IGB_SET_FLAG(tx_flags
, IGB_TX_FLAGS_TSO
,
5478 (E1000_ADVTXD_DCMD_TSE
));
5480 /* set timestamp bit if present */
5481 cmd_type
|= IGB_SET_FLAG(tx_flags
, IGB_TX_FLAGS_TSTAMP
,
5482 (E1000_ADVTXD_MAC_TSTAMP
));
5484 /* insert frame checksum */
5485 cmd_type
^= IGB_SET_FLAG(skb
->no_fcs
, 1, E1000_ADVTXD_DCMD_IFCS
);
5490 static void igb_tx_olinfo_status(struct igb_ring
*tx_ring
,
5491 union e1000_adv_tx_desc
*tx_desc
,
5492 u32 tx_flags
, unsigned int paylen
)
5494 u32 olinfo_status
= paylen
<< E1000_ADVTXD_PAYLEN_SHIFT
;
5496 /* 82575 requires a unique index per ring */
5497 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX
, &tx_ring
->flags
))
5498 olinfo_status
|= tx_ring
->reg_idx
<< 4;
5500 /* insert L4 checksum */
5501 olinfo_status
|= IGB_SET_FLAG(tx_flags
,
5503 (E1000_TXD_POPTS_TXSM
<< 8));
5505 /* insert IPv4 checksum */
5506 olinfo_status
|= IGB_SET_FLAG(tx_flags
,
5508 (E1000_TXD_POPTS_IXSM
<< 8));
5510 tx_desc
->read
.olinfo_status
= cpu_to_le32(olinfo_status
);
5513 static int __igb_maybe_stop_tx(struct igb_ring
*tx_ring
, const u16 size
)
5515 struct net_device
*netdev
= tx_ring
->netdev
;
5517 netif_stop_subqueue(netdev
, tx_ring
->queue_index
);
5519 /* Herbert's original patch had:
5520 * smp_mb__after_netif_stop_queue();
5521 * but since that doesn't exist yet, just open code it.
5525 /* We need to check again in a case another CPU has just
5526 * made room available.
5528 if (igb_desc_unused(tx_ring
) < size
)
5532 netif_wake_subqueue(netdev
, tx_ring
->queue_index
);
5534 u64_stats_update_begin(&tx_ring
->tx_syncp2
);
5535 tx_ring
->tx_stats
.restart_queue2
++;
5536 u64_stats_update_end(&tx_ring
->tx_syncp2
);
5541 static inline int igb_maybe_stop_tx(struct igb_ring
*tx_ring
, const u16 size
)
5543 if (igb_desc_unused(tx_ring
) >= size
)
5545 return __igb_maybe_stop_tx(tx_ring
, size
);
5548 static int igb_tx_map(struct igb_ring
*tx_ring
,
5549 struct igb_tx_buffer
*first
,
5552 struct sk_buff
*skb
= first
->skb
;
5553 struct igb_tx_buffer
*tx_buffer
;
5554 union e1000_adv_tx_desc
*tx_desc
;
5555 struct skb_frag_struct
*frag
;
5557 unsigned int data_len
, size
;
5558 u32 tx_flags
= first
->tx_flags
;
5559 u32 cmd_type
= igb_tx_cmd_type(skb
, tx_flags
);
5560 u16 i
= tx_ring
->next_to_use
;
5562 tx_desc
= IGB_TX_DESC(tx_ring
, i
);
5564 igb_tx_olinfo_status(tx_ring
, tx_desc
, tx_flags
, skb
->len
- hdr_len
);
5566 size
= skb_headlen(skb
);
5567 data_len
= skb
->data_len
;
5569 dma
= dma_map_single(tx_ring
->dev
, skb
->data
, size
, DMA_TO_DEVICE
);
5573 for (frag
= &skb_shinfo(skb
)->frags
[0];; frag
++) {
5574 if (dma_mapping_error(tx_ring
->dev
, dma
))
5577 /* record length, and DMA address */
5578 dma_unmap_len_set(tx_buffer
, len
, size
);
5579 dma_unmap_addr_set(tx_buffer
, dma
, dma
);
5581 tx_desc
->read
.buffer_addr
= cpu_to_le64(dma
);
5583 while (unlikely(size
> IGB_MAX_DATA_PER_TXD
)) {
5584 tx_desc
->read
.cmd_type_len
=
5585 cpu_to_le32(cmd_type
^ IGB_MAX_DATA_PER_TXD
);
5589 if (i
== tx_ring
->count
) {
5590 tx_desc
= IGB_TX_DESC(tx_ring
, 0);
5593 tx_desc
->read
.olinfo_status
= 0;
5595 dma
+= IGB_MAX_DATA_PER_TXD
;
5596 size
-= IGB_MAX_DATA_PER_TXD
;
5598 tx_desc
->read
.buffer_addr
= cpu_to_le64(dma
);
5601 if (likely(!data_len
))
5604 tx_desc
->read
.cmd_type_len
= cpu_to_le32(cmd_type
^ size
);
5608 if (i
== tx_ring
->count
) {
5609 tx_desc
= IGB_TX_DESC(tx_ring
, 0);
5612 tx_desc
->read
.olinfo_status
= 0;
5614 size
= skb_frag_size(frag
);
5617 dma
= skb_frag_dma_map(tx_ring
->dev
, frag
, 0,
5618 size
, DMA_TO_DEVICE
);
5620 tx_buffer
= &tx_ring
->tx_buffer_info
[i
];
5623 /* write last descriptor with RS and EOP bits */
5624 cmd_type
|= size
| IGB_TXD_DCMD
;
5625 tx_desc
->read
.cmd_type_len
= cpu_to_le32(cmd_type
);
5627 netdev_tx_sent_queue(txring_txq(tx_ring
), first
->bytecount
);
5629 /* set the timestamp */
5630 first
->time_stamp
= jiffies
;
5632 /* Force memory writes to complete before letting h/w know there
5633 * are new descriptors to fetch. (Only applicable for weak-ordered
5634 * memory model archs, such as IA-64).
5636 * We also need this memory barrier to make certain all of the
5637 * status bits have been updated before next_to_watch is written.
5641 /* set next_to_watch value indicating a packet is present */
5642 first
->next_to_watch
= tx_desc
;
5645 if (i
== tx_ring
->count
)
5648 tx_ring
->next_to_use
= i
;
5650 /* Make sure there is space in the ring for the next send. */
5651 igb_maybe_stop_tx(tx_ring
, DESC_NEEDED
);
5653 if (netif_xmit_stopped(txring_txq(tx_ring
)) || !skb
->xmit_more
) {
5654 writel(i
, tx_ring
->tail
);
5656 /* we need this if more than one processor can write to our tail
5657 * at a time, it synchronizes IO on IA64/Altix systems
5664 dev_err(tx_ring
->dev
, "TX DMA map failed\n");
5665 tx_buffer
= &tx_ring
->tx_buffer_info
[i
];
5667 /* clear dma mappings for failed tx_buffer_info map */
5668 while (tx_buffer
!= first
) {
5669 if (dma_unmap_len(tx_buffer
, len
))
5670 dma_unmap_page(tx_ring
->dev
,
5671 dma_unmap_addr(tx_buffer
, dma
),
5672 dma_unmap_len(tx_buffer
, len
),
5674 dma_unmap_len_set(tx_buffer
, len
, 0);
5677 i
+= tx_ring
->count
;
5678 tx_buffer
= &tx_ring
->tx_buffer_info
[i
];
5681 if (dma_unmap_len(tx_buffer
, len
))
5682 dma_unmap_single(tx_ring
->dev
,
5683 dma_unmap_addr(tx_buffer
, dma
),
5684 dma_unmap_len(tx_buffer
, len
),
5686 dma_unmap_len_set(tx_buffer
, len
, 0);
5688 dev_kfree_skb_any(tx_buffer
->skb
);
5689 tx_buffer
->skb
= NULL
;
5691 tx_ring
->next_to_use
= i
;
5696 netdev_tx_t
igb_xmit_frame_ring(struct sk_buff
*skb
,
5697 struct igb_ring
*tx_ring
)
5699 struct igb_tx_buffer
*first
;
5703 u16 count
= TXD_USE_COUNT(skb_headlen(skb
));
5704 __be16 protocol
= vlan_get_protocol(skb
);
5707 /* need: 1 descriptor per page * PAGE_SIZE/IGB_MAX_DATA_PER_TXD,
5708 * + 1 desc for skb_headlen/IGB_MAX_DATA_PER_TXD,
5709 * + 2 desc gap to keep tail from touching head,
5710 * + 1 desc for context descriptor,
5711 * otherwise try next time
5713 for (f
= 0; f
< skb_shinfo(skb
)->nr_frags
; f
++)
5714 count
+= TXD_USE_COUNT(skb_shinfo(skb
)->frags
[f
].size
);
5716 if (igb_maybe_stop_tx(tx_ring
, count
+ 3)) {
5717 /* this is a hard error */
5718 return NETDEV_TX_BUSY
;
5721 /* record the location of the first descriptor for this packet */
5722 first
= &tx_ring
->tx_buffer_info
[tx_ring
->next_to_use
];
5724 first
->bytecount
= skb
->len
;
5725 first
->gso_segs
= 1;
5727 if (unlikely(skb_shinfo(skb
)->tx_flags
& SKBTX_HW_TSTAMP
)) {
5728 struct igb_adapter
*adapter
= netdev_priv(tx_ring
->netdev
);
5730 if (adapter
->tstamp_config
.tx_type
& HWTSTAMP_TX_ON
&&
5731 !test_and_set_bit_lock(__IGB_PTP_TX_IN_PROGRESS
,
5733 skb_shinfo(skb
)->tx_flags
|= SKBTX_IN_PROGRESS
;
5734 tx_flags
|= IGB_TX_FLAGS_TSTAMP
;
5736 adapter
->ptp_tx_skb
= skb_get(skb
);
5737 adapter
->ptp_tx_start
= jiffies
;
5738 if (adapter
->hw
.mac
.type
== e1000_82576
)
5739 schedule_work(&adapter
->ptp_tx_work
);
5741 adapter
->tx_hwtstamp_skipped
++;
5745 skb_tx_timestamp(skb
);
5747 if (skb_vlan_tag_present(skb
)) {
5748 tx_flags
|= IGB_TX_FLAGS_VLAN
;
5749 tx_flags
|= (skb_vlan_tag_get(skb
) << IGB_TX_FLAGS_VLAN_SHIFT
);
5752 /* record initial flags and protocol */
5753 first
->tx_flags
= tx_flags
;
5754 first
->protocol
= protocol
;
5756 tso
= igb_tso(tx_ring
, first
, &hdr_len
);
5760 igb_tx_csum(tx_ring
, first
);
5762 if (igb_tx_map(tx_ring
, first
, hdr_len
))
5763 goto cleanup_tx_tstamp
;
5765 return NETDEV_TX_OK
;
5768 dev_kfree_skb_any(first
->skb
);
5771 if (unlikely(tx_flags
& IGB_TX_FLAGS_TSTAMP
)) {
5772 struct igb_adapter
*adapter
= netdev_priv(tx_ring
->netdev
);
5774 dev_kfree_skb_any(adapter
->ptp_tx_skb
);
5775 adapter
->ptp_tx_skb
= NULL
;
5776 if (adapter
->hw
.mac
.type
== e1000_82576
)
5777 cancel_work_sync(&adapter
->ptp_tx_work
);
5778 clear_bit_unlock(__IGB_PTP_TX_IN_PROGRESS
, &adapter
->state
);
5781 return NETDEV_TX_OK
;
5784 static inline struct igb_ring
*igb_tx_queue_mapping(struct igb_adapter
*adapter
,
5785 struct sk_buff
*skb
)
5787 unsigned int r_idx
= skb
->queue_mapping
;
5789 if (r_idx
>= adapter
->num_tx_queues
)
5790 r_idx
= r_idx
% adapter
->num_tx_queues
;
5792 return adapter
->tx_ring
[r_idx
];
5795 static netdev_tx_t
igb_xmit_frame(struct sk_buff
*skb
,
5796 struct net_device
*netdev
)
5798 struct igb_adapter
*adapter
= netdev_priv(netdev
);
5800 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb
5801 * in order to meet this minimum size requirement.
5803 if (skb_put_padto(skb
, 17))
5804 return NETDEV_TX_OK
;
5806 return igb_xmit_frame_ring(skb
, igb_tx_queue_mapping(adapter
, skb
));
5810 * igb_tx_timeout - Respond to a Tx Hang
5811 * @netdev: network interface device structure
5813 static void igb_tx_timeout(struct net_device
*netdev
)
5815 struct igb_adapter
*adapter
= netdev_priv(netdev
);
5816 struct e1000_hw
*hw
= &adapter
->hw
;
5818 /* Do the reset outside of interrupt context */
5819 adapter
->tx_timeout_count
++;
5821 if (hw
->mac
.type
>= e1000_82580
)
5822 hw
->dev_spec
._82575
.global_device_reset
= true;
5824 schedule_work(&adapter
->reset_task
);
5826 (adapter
->eims_enable_mask
& ~adapter
->eims_other
));
5829 static void igb_reset_task(struct work_struct
*work
)
5831 struct igb_adapter
*adapter
;
5832 adapter
= container_of(work
, struct igb_adapter
, reset_task
);
5835 netdev_err(adapter
->netdev
, "Reset adapter\n");
5836 igb_reinit_locked(adapter
);
5840 * igb_get_stats64 - Get System Network Statistics
5841 * @netdev: network interface device structure
5842 * @stats: rtnl_link_stats64 pointer
5844 static void igb_get_stats64(struct net_device
*netdev
,
5845 struct rtnl_link_stats64
*stats
)
5847 struct igb_adapter
*adapter
= netdev_priv(netdev
);
5849 spin_lock(&adapter
->stats64_lock
);
5850 igb_update_stats(adapter
);
5851 memcpy(stats
, &adapter
->stats64
, sizeof(*stats
));
5852 spin_unlock(&adapter
->stats64_lock
);
5856 * igb_change_mtu - Change the Maximum Transfer Unit
5857 * @netdev: network interface device structure
5858 * @new_mtu: new value for maximum frame size
5860 * Returns 0 on success, negative on failure
5862 static int igb_change_mtu(struct net_device
*netdev
, int new_mtu
)
5864 struct igb_adapter
*adapter
= netdev_priv(netdev
);
5865 struct pci_dev
*pdev
= adapter
->pdev
;
5866 int max_frame
= new_mtu
+ ETH_HLEN
+ ETH_FCS_LEN
+ VLAN_HLEN
;
5868 /* adjust max frame to be at least the size of a standard frame */
5869 if (max_frame
< (ETH_FRAME_LEN
+ ETH_FCS_LEN
))
5870 max_frame
= ETH_FRAME_LEN
+ ETH_FCS_LEN
;
5872 while (test_and_set_bit(__IGB_RESETTING
, &adapter
->state
))
5873 usleep_range(1000, 2000);
5875 /* igb_down has a dependency on max_frame_size */
5876 adapter
->max_frame_size
= max_frame
;
5878 if (netif_running(netdev
))
5881 dev_info(&pdev
->dev
, "changing MTU from %d to %d\n",
5882 netdev
->mtu
, new_mtu
);
5883 netdev
->mtu
= new_mtu
;
5885 if (netif_running(netdev
))
5890 clear_bit(__IGB_RESETTING
, &adapter
->state
);
5896 * igb_update_stats - Update the board statistics counters
5897 * @adapter: board private structure
5899 void igb_update_stats(struct igb_adapter
*adapter
)
5901 struct rtnl_link_stats64
*net_stats
= &adapter
->stats64
;
5902 struct e1000_hw
*hw
= &adapter
->hw
;
5903 struct pci_dev
*pdev
= adapter
->pdev
;
5908 u64 _bytes
, _packets
;
5910 /* Prevent stats update while adapter is being reset, or if the pci
5911 * connection is down.
5913 if (adapter
->link_speed
== 0)
5915 if (pci_channel_offline(pdev
))
5922 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
5923 struct igb_ring
*ring
= adapter
->rx_ring
[i
];
5924 u32 rqdpc
= rd32(E1000_RQDPC(i
));
5925 if (hw
->mac
.type
>= e1000_i210
)
5926 wr32(E1000_RQDPC(i
), 0);
5929 ring
->rx_stats
.drops
+= rqdpc
;
5930 net_stats
->rx_fifo_errors
+= rqdpc
;
5934 start
= u64_stats_fetch_begin_irq(&ring
->rx_syncp
);
5935 _bytes
= ring
->rx_stats
.bytes
;
5936 _packets
= ring
->rx_stats
.packets
;
5937 } while (u64_stats_fetch_retry_irq(&ring
->rx_syncp
, start
));
5939 packets
+= _packets
;
5942 net_stats
->rx_bytes
= bytes
;
5943 net_stats
->rx_packets
= packets
;
5947 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
5948 struct igb_ring
*ring
= adapter
->tx_ring
[i
];
5950 start
= u64_stats_fetch_begin_irq(&ring
->tx_syncp
);
5951 _bytes
= ring
->tx_stats
.bytes
;
5952 _packets
= ring
->tx_stats
.packets
;
5953 } while (u64_stats_fetch_retry_irq(&ring
->tx_syncp
, start
));
5955 packets
+= _packets
;
5957 net_stats
->tx_bytes
= bytes
;
5958 net_stats
->tx_packets
= packets
;
5961 /* read stats registers */
5962 adapter
->stats
.crcerrs
+= rd32(E1000_CRCERRS
);
5963 adapter
->stats
.gprc
+= rd32(E1000_GPRC
);
5964 adapter
->stats
.gorc
+= rd32(E1000_GORCL
);
5965 rd32(E1000_GORCH
); /* clear GORCL */
5966 adapter
->stats
.bprc
+= rd32(E1000_BPRC
);
5967 adapter
->stats
.mprc
+= rd32(E1000_MPRC
);
5968 adapter
->stats
.roc
+= rd32(E1000_ROC
);
5970 adapter
->stats
.prc64
+= rd32(E1000_PRC64
);
5971 adapter
->stats
.prc127
+= rd32(E1000_PRC127
);
5972 adapter
->stats
.prc255
+= rd32(E1000_PRC255
);
5973 adapter
->stats
.prc511
+= rd32(E1000_PRC511
);
5974 adapter
->stats
.prc1023
+= rd32(E1000_PRC1023
);
5975 adapter
->stats
.prc1522
+= rd32(E1000_PRC1522
);
5976 adapter
->stats
.symerrs
+= rd32(E1000_SYMERRS
);
5977 adapter
->stats
.sec
+= rd32(E1000_SEC
);
5979 mpc
= rd32(E1000_MPC
);
5980 adapter
->stats
.mpc
+= mpc
;
5981 net_stats
->rx_fifo_errors
+= mpc
;
5982 adapter
->stats
.scc
+= rd32(E1000_SCC
);
5983 adapter
->stats
.ecol
+= rd32(E1000_ECOL
);
5984 adapter
->stats
.mcc
+= rd32(E1000_MCC
);
5985 adapter
->stats
.latecol
+= rd32(E1000_LATECOL
);
5986 adapter
->stats
.dc
+= rd32(E1000_DC
);
5987 adapter
->stats
.rlec
+= rd32(E1000_RLEC
);
5988 adapter
->stats
.xonrxc
+= rd32(E1000_XONRXC
);
5989 adapter
->stats
.xontxc
+= rd32(E1000_XONTXC
);
5990 adapter
->stats
.xoffrxc
+= rd32(E1000_XOFFRXC
);
5991 adapter
->stats
.xofftxc
+= rd32(E1000_XOFFTXC
);
5992 adapter
->stats
.fcruc
+= rd32(E1000_FCRUC
);
5993 adapter
->stats
.gptc
+= rd32(E1000_GPTC
);
5994 adapter
->stats
.gotc
+= rd32(E1000_GOTCL
);
5995 rd32(E1000_GOTCH
); /* clear GOTCL */
5996 adapter
->stats
.rnbc
+= rd32(E1000_RNBC
);
5997 adapter
->stats
.ruc
+= rd32(E1000_RUC
);
5998 adapter
->stats
.rfc
+= rd32(E1000_RFC
);
5999 adapter
->stats
.rjc
+= rd32(E1000_RJC
);
6000 adapter
->stats
.tor
+= rd32(E1000_TORH
);
6001 adapter
->stats
.tot
+= rd32(E1000_TOTH
);
6002 adapter
->stats
.tpr
+= rd32(E1000_TPR
);
6004 adapter
->stats
.ptc64
+= rd32(E1000_PTC64
);
6005 adapter
->stats
.ptc127
+= rd32(E1000_PTC127
);
6006 adapter
->stats
.ptc255
+= rd32(E1000_PTC255
);
6007 adapter
->stats
.ptc511
+= rd32(E1000_PTC511
);
6008 adapter
->stats
.ptc1023
+= rd32(E1000_PTC1023
);
6009 adapter
->stats
.ptc1522
+= rd32(E1000_PTC1522
);
6011 adapter
->stats
.mptc
+= rd32(E1000_MPTC
);
6012 adapter
->stats
.bptc
+= rd32(E1000_BPTC
);
6014 adapter
->stats
.tpt
+= rd32(E1000_TPT
);
6015 adapter
->stats
.colc
+= rd32(E1000_COLC
);
6017 adapter
->stats
.algnerrc
+= rd32(E1000_ALGNERRC
);
6018 /* read internal phy specific stats */
6019 reg
= rd32(E1000_CTRL_EXT
);
6020 if (!(reg
& E1000_CTRL_EXT_LINK_MODE_MASK
)) {
6021 adapter
->stats
.rxerrc
+= rd32(E1000_RXERRC
);
6023 /* this stat has invalid values on i210/i211 */
6024 if ((hw
->mac
.type
!= e1000_i210
) &&
6025 (hw
->mac
.type
!= e1000_i211
))
6026 adapter
->stats
.tncrs
+= rd32(E1000_TNCRS
);
6029 adapter
->stats
.tsctc
+= rd32(E1000_TSCTC
);
6030 adapter
->stats
.tsctfc
+= rd32(E1000_TSCTFC
);
6032 adapter
->stats
.iac
+= rd32(E1000_IAC
);
6033 adapter
->stats
.icrxoc
+= rd32(E1000_ICRXOC
);
6034 adapter
->stats
.icrxptc
+= rd32(E1000_ICRXPTC
);
6035 adapter
->stats
.icrxatc
+= rd32(E1000_ICRXATC
);
6036 adapter
->stats
.ictxptc
+= rd32(E1000_ICTXPTC
);
6037 adapter
->stats
.ictxatc
+= rd32(E1000_ICTXATC
);
6038 adapter
->stats
.ictxqec
+= rd32(E1000_ICTXQEC
);
6039 adapter
->stats
.ictxqmtc
+= rd32(E1000_ICTXQMTC
);
6040 adapter
->stats
.icrxdmtc
+= rd32(E1000_ICRXDMTC
);
6042 /* Fill out the OS statistics structure */
6043 net_stats
->multicast
= adapter
->stats
.mprc
;
6044 net_stats
->collisions
= adapter
->stats
.colc
;
6048 /* RLEC on some newer hardware can be incorrect so build
6049 * our own version based on RUC and ROC
6051 net_stats
->rx_errors
= adapter
->stats
.rxerrc
+
6052 adapter
->stats
.crcerrs
+ adapter
->stats
.algnerrc
+
6053 adapter
->stats
.ruc
+ adapter
->stats
.roc
+
6054 adapter
->stats
.cexterr
;
6055 net_stats
->rx_length_errors
= adapter
->stats
.ruc
+
6057 net_stats
->rx_crc_errors
= adapter
->stats
.crcerrs
;
6058 net_stats
->rx_frame_errors
= adapter
->stats
.algnerrc
;
6059 net_stats
->rx_missed_errors
= adapter
->stats
.mpc
;
6062 net_stats
->tx_errors
= adapter
->stats
.ecol
+
6063 adapter
->stats
.latecol
;
6064 net_stats
->tx_aborted_errors
= adapter
->stats
.ecol
;
6065 net_stats
->tx_window_errors
= adapter
->stats
.latecol
;
6066 net_stats
->tx_carrier_errors
= adapter
->stats
.tncrs
;
6068 /* Tx Dropped needs to be maintained elsewhere */
6070 /* Management Stats */
6071 adapter
->stats
.mgptc
+= rd32(E1000_MGTPTC
);
6072 adapter
->stats
.mgprc
+= rd32(E1000_MGTPRC
);
6073 adapter
->stats
.mgpdc
+= rd32(E1000_MGTPDC
);
6076 reg
= rd32(E1000_MANC
);
6077 if (reg
& E1000_MANC_EN_BMC2OS
) {
6078 adapter
->stats
.o2bgptc
+= rd32(E1000_O2BGPTC
);
6079 adapter
->stats
.o2bspc
+= rd32(E1000_O2BSPC
);
6080 adapter
->stats
.b2ospc
+= rd32(E1000_B2OSPC
);
6081 adapter
->stats
.b2ogprc
+= rd32(E1000_B2OGPRC
);
6085 static void igb_tsync_interrupt(struct igb_adapter
*adapter
)
6087 struct e1000_hw
*hw
= &adapter
->hw
;
6088 struct ptp_clock_event event
;
6089 struct timespec64 ts
;
6090 u32 ack
= 0, tsauxc
, sec
, nsec
, tsicr
= rd32(E1000_TSICR
);
6092 if (tsicr
& TSINTR_SYS_WRAP
) {
6093 event
.type
= PTP_CLOCK_PPS
;
6094 if (adapter
->ptp_caps
.pps
)
6095 ptp_clock_event(adapter
->ptp_clock
, &event
);
6096 ack
|= TSINTR_SYS_WRAP
;
6099 if (tsicr
& E1000_TSICR_TXTS
) {
6100 /* retrieve hardware timestamp */
6101 schedule_work(&adapter
->ptp_tx_work
);
6102 ack
|= E1000_TSICR_TXTS
;
6105 if (tsicr
& TSINTR_TT0
) {
6106 spin_lock(&adapter
->tmreg_lock
);
6107 ts
= timespec64_add(adapter
->perout
[0].start
,
6108 adapter
->perout
[0].period
);
6109 /* u32 conversion of tv_sec is safe until y2106 */
6110 wr32(E1000_TRGTTIML0
, ts
.tv_nsec
);
6111 wr32(E1000_TRGTTIMH0
, (u32
)ts
.tv_sec
);
6112 tsauxc
= rd32(E1000_TSAUXC
);
6113 tsauxc
|= TSAUXC_EN_TT0
;
6114 wr32(E1000_TSAUXC
, tsauxc
);
6115 adapter
->perout
[0].start
= ts
;
6116 spin_unlock(&adapter
->tmreg_lock
);
6120 if (tsicr
& TSINTR_TT1
) {
6121 spin_lock(&adapter
->tmreg_lock
);
6122 ts
= timespec64_add(adapter
->perout
[1].start
,
6123 adapter
->perout
[1].period
);
6124 wr32(E1000_TRGTTIML1
, ts
.tv_nsec
);
6125 wr32(E1000_TRGTTIMH1
, (u32
)ts
.tv_sec
);
6126 tsauxc
= rd32(E1000_TSAUXC
);
6127 tsauxc
|= TSAUXC_EN_TT1
;
6128 wr32(E1000_TSAUXC
, tsauxc
);
6129 adapter
->perout
[1].start
= ts
;
6130 spin_unlock(&adapter
->tmreg_lock
);
6134 if (tsicr
& TSINTR_AUTT0
) {
6135 nsec
= rd32(E1000_AUXSTMPL0
);
6136 sec
= rd32(E1000_AUXSTMPH0
);
6137 event
.type
= PTP_CLOCK_EXTTS
;
6139 event
.timestamp
= sec
* 1000000000ULL + nsec
;
6140 ptp_clock_event(adapter
->ptp_clock
, &event
);
6141 ack
|= TSINTR_AUTT0
;
6144 if (tsicr
& TSINTR_AUTT1
) {
6145 nsec
= rd32(E1000_AUXSTMPL1
);
6146 sec
= rd32(E1000_AUXSTMPH1
);
6147 event
.type
= PTP_CLOCK_EXTTS
;
6149 event
.timestamp
= sec
* 1000000000ULL + nsec
;
6150 ptp_clock_event(adapter
->ptp_clock
, &event
);
6151 ack
|= TSINTR_AUTT1
;
6154 /* acknowledge the interrupts */
6155 wr32(E1000_TSICR
, ack
);
6158 static irqreturn_t
igb_msix_other(int irq
, void *data
)
6160 struct igb_adapter
*adapter
= data
;
6161 struct e1000_hw
*hw
= &adapter
->hw
;
6162 u32 icr
= rd32(E1000_ICR
);
6163 /* reading ICR causes bit 31 of EICR to be cleared */
6165 if (icr
& E1000_ICR_DRSTA
)
6166 schedule_work(&adapter
->reset_task
);
6168 if (icr
& E1000_ICR_DOUTSYNC
) {
6169 /* HW is reporting DMA is out of sync */
6170 adapter
->stats
.doosync
++;
6171 /* The DMA Out of Sync is also indication of a spoof event
6172 * in IOV mode. Check the Wrong VM Behavior register to
6173 * see if it is really a spoof event.
6175 igb_check_wvbr(adapter
);
6178 /* Check for a mailbox event */
6179 if (icr
& E1000_ICR_VMMB
)
6180 igb_msg_task(adapter
);
6182 if (icr
& E1000_ICR_LSC
) {
6183 hw
->mac
.get_link_status
= 1;
6184 /* guard against interrupt when we're going down */
6185 if (!test_bit(__IGB_DOWN
, &adapter
->state
))
6186 mod_timer(&adapter
->watchdog_timer
, jiffies
+ 1);
6189 if (icr
& E1000_ICR_TS
)
6190 igb_tsync_interrupt(adapter
);
6192 wr32(E1000_EIMS
, adapter
->eims_other
);
6197 static void igb_write_itr(struct igb_q_vector
*q_vector
)
6199 struct igb_adapter
*adapter
= q_vector
->adapter
;
6200 u32 itr_val
= q_vector
->itr_val
& 0x7FFC;
6202 if (!q_vector
->set_itr
)
6208 if (adapter
->hw
.mac
.type
== e1000_82575
)
6209 itr_val
|= itr_val
<< 16;
6211 itr_val
|= E1000_EITR_CNT_IGNR
;
6213 writel(itr_val
, q_vector
->itr_register
);
6214 q_vector
->set_itr
= 0;
6217 static irqreturn_t
igb_msix_ring(int irq
, void *data
)
6219 struct igb_q_vector
*q_vector
= data
;
6221 /* Write the ITR value calculated from the previous interrupt. */
6222 igb_write_itr(q_vector
);
6224 napi_schedule(&q_vector
->napi
);
6229 #ifdef CONFIG_IGB_DCA
6230 static void igb_update_tx_dca(struct igb_adapter
*adapter
,
6231 struct igb_ring
*tx_ring
,
6234 struct e1000_hw
*hw
= &adapter
->hw
;
6235 u32 txctrl
= dca3_get_tag(tx_ring
->dev
, cpu
);
6237 if (hw
->mac
.type
!= e1000_82575
)
6238 txctrl
<<= E1000_DCA_TXCTRL_CPUID_SHIFT
;
6240 /* We can enable relaxed ordering for reads, but not writes when
6241 * DCA is enabled. This is due to a known issue in some chipsets
6242 * which will cause the DCA tag to be cleared.
6244 txctrl
|= E1000_DCA_TXCTRL_DESC_RRO_EN
|
6245 E1000_DCA_TXCTRL_DATA_RRO_EN
|
6246 E1000_DCA_TXCTRL_DESC_DCA_EN
;
6248 wr32(E1000_DCA_TXCTRL(tx_ring
->reg_idx
), txctrl
);
6251 static void igb_update_rx_dca(struct igb_adapter
*adapter
,
6252 struct igb_ring
*rx_ring
,
6255 struct e1000_hw
*hw
= &adapter
->hw
;
6256 u32 rxctrl
= dca3_get_tag(&adapter
->pdev
->dev
, cpu
);
6258 if (hw
->mac
.type
!= e1000_82575
)
6259 rxctrl
<<= E1000_DCA_RXCTRL_CPUID_SHIFT
;
6261 /* We can enable relaxed ordering for reads, but not writes when
6262 * DCA is enabled. This is due to a known issue in some chipsets
6263 * which will cause the DCA tag to be cleared.
6265 rxctrl
|= E1000_DCA_RXCTRL_DESC_RRO_EN
|
6266 E1000_DCA_RXCTRL_DESC_DCA_EN
;
6268 wr32(E1000_DCA_RXCTRL(rx_ring
->reg_idx
), rxctrl
);
6271 static void igb_update_dca(struct igb_q_vector
*q_vector
)
6273 struct igb_adapter
*adapter
= q_vector
->adapter
;
6274 int cpu
= get_cpu();
6276 if (q_vector
->cpu
== cpu
)
6279 if (q_vector
->tx
.ring
)
6280 igb_update_tx_dca(adapter
, q_vector
->tx
.ring
, cpu
);
6282 if (q_vector
->rx
.ring
)
6283 igb_update_rx_dca(adapter
, q_vector
->rx
.ring
, cpu
);
6285 q_vector
->cpu
= cpu
;
6290 static void igb_setup_dca(struct igb_adapter
*adapter
)
6292 struct e1000_hw
*hw
= &adapter
->hw
;
6295 if (!(adapter
->flags
& IGB_FLAG_DCA_ENABLED
))
6298 /* Always use CB2 mode, difference is masked in the CB driver. */
6299 wr32(E1000_DCA_CTRL
, E1000_DCA_CTRL_DCA_MODE_CB2
);
6301 for (i
= 0; i
< adapter
->num_q_vectors
; i
++) {
6302 adapter
->q_vector
[i
]->cpu
= -1;
6303 igb_update_dca(adapter
->q_vector
[i
]);
6307 static int __igb_notify_dca(struct device
*dev
, void *data
)
6309 struct net_device
*netdev
= dev_get_drvdata(dev
);
6310 struct igb_adapter
*adapter
= netdev_priv(netdev
);
6311 struct pci_dev
*pdev
= adapter
->pdev
;
6312 struct e1000_hw
*hw
= &adapter
->hw
;
6313 unsigned long event
= *(unsigned long *)data
;
6316 case DCA_PROVIDER_ADD
:
6317 /* if already enabled, don't do it again */
6318 if (adapter
->flags
& IGB_FLAG_DCA_ENABLED
)
6320 if (dca_add_requester(dev
) == 0) {
6321 adapter
->flags
|= IGB_FLAG_DCA_ENABLED
;
6322 dev_info(&pdev
->dev
, "DCA enabled\n");
6323 igb_setup_dca(adapter
);
6326 /* Fall Through since DCA is disabled. */
6327 case DCA_PROVIDER_REMOVE
:
6328 if (adapter
->flags
& IGB_FLAG_DCA_ENABLED
) {
6329 /* without this a class_device is left
6330 * hanging around in the sysfs model
6332 dca_remove_requester(dev
);
6333 dev_info(&pdev
->dev
, "DCA disabled\n");
6334 adapter
->flags
&= ~IGB_FLAG_DCA_ENABLED
;
6335 wr32(E1000_DCA_CTRL
, E1000_DCA_CTRL_DCA_MODE_DISABLE
);
6343 static int igb_notify_dca(struct notifier_block
*nb
, unsigned long event
,
6348 ret_val
= driver_for_each_device(&igb_driver
.driver
, NULL
, &event
,
6351 return ret_val
? NOTIFY_BAD
: NOTIFY_DONE
;
6353 #endif /* CONFIG_IGB_DCA */
6355 #ifdef CONFIG_PCI_IOV
6356 static int igb_vf_configure(struct igb_adapter
*adapter
, int vf
)
6358 unsigned char mac_addr
[ETH_ALEN
];
6360 eth_zero_addr(mac_addr
);
6361 igb_set_vf_mac(adapter
, vf
, mac_addr
);
6363 /* By default spoof check is enabled for all VFs */
6364 adapter
->vf_data
[vf
].spoofchk_enabled
= true;
6370 static void igb_ping_all_vfs(struct igb_adapter
*adapter
)
6372 struct e1000_hw
*hw
= &adapter
->hw
;
6376 for (i
= 0 ; i
< adapter
->vfs_allocated_count
; i
++) {
6377 ping
= E1000_PF_CONTROL_MSG
;
6378 if (adapter
->vf_data
[i
].flags
& IGB_VF_FLAG_CTS
)
6379 ping
|= E1000_VT_MSGTYPE_CTS
;
6380 igb_write_mbx(hw
, &ping
, 1, i
);
6384 static int igb_set_vf_promisc(struct igb_adapter
*adapter
, u32
*msgbuf
, u32 vf
)
6386 struct e1000_hw
*hw
= &adapter
->hw
;
6387 u32 vmolr
= rd32(E1000_VMOLR(vf
));
6388 struct vf_data_storage
*vf_data
= &adapter
->vf_data
[vf
];
6390 vf_data
->flags
&= ~(IGB_VF_FLAG_UNI_PROMISC
|
6391 IGB_VF_FLAG_MULTI_PROMISC
);
6392 vmolr
&= ~(E1000_VMOLR_ROPE
| E1000_VMOLR_ROMPE
| E1000_VMOLR_MPME
);
6394 if (*msgbuf
& E1000_VF_SET_PROMISC_MULTICAST
) {
6395 vmolr
|= E1000_VMOLR_MPME
;
6396 vf_data
->flags
|= IGB_VF_FLAG_MULTI_PROMISC
;
6397 *msgbuf
&= ~E1000_VF_SET_PROMISC_MULTICAST
;
6399 /* if we have hashes and we are clearing a multicast promisc
6400 * flag we need to write the hashes to the MTA as this step
6401 * was previously skipped
6403 if (vf_data
->num_vf_mc_hashes
> 30) {
6404 vmolr
|= E1000_VMOLR_MPME
;
6405 } else if (vf_data
->num_vf_mc_hashes
) {
6408 vmolr
|= E1000_VMOLR_ROMPE
;
6409 for (j
= 0; j
< vf_data
->num_vf_mc_hashes
; j
++)
6410 igb_mta_set(hw
, vf_data
->vf_mc_hashes
[j
]);
6414 wr32(E1000_VMOLR(vf
), vmolr
);
6416 /* there are flags left unprocessed, likely not supported */
6417 if (*msgbuf
& E1000_VT_MSGINFO_MASK
)
6423 static int igb_set_vf_multicasts(struct igb_adapter
*adapter
,
6424 u32
*msgbuf
, u32 vf
)
6426 int n
= (msgbuf
[0] & E1000_VT_MSGINFO_MASK
) >> E1000_VT_MSGINFO_SHIFT
;
6427 u16
*hash_list
= (u16
*)&msgbuf
[1];
6428 struct vf_data_storage
*vf_data
= &adapter
->vf_data
[vf
];
6431 /* salt away the number of multicast addresses assigned
6432 * to this VF for later use to restore when the PF multi cast
6435 vf_data
->num_vf_mc_hashes
= n
;
6437 /* only up to 30 hash values supported */
6441 /* store the hashes for later use */
6442 for (i
= 0; i
< n
; i
++)
6443 vf_data
->vf_mc_hashes
[i
] = hash_list
[i
];
6445 /* Flush and reset the mta with the new values */
6446 igb_set_rx_mode(adapter
->netdev
);
6451 static void igb_restore_vf_multicasts(struct igb_adapter
*adapter
)
6453 struct e1000_hw
*hw
= &adapter
->hw
;
6454 struct vf_data_storage
*vf_data
;
6457 for (i
= 0; i
< adapter
->vfs_allocated_count
; i
++) {
6458 u32 vmolr
= rd32(E1000_VMOLR(i
));
6460 vmolr
&= ~(E1000_VMOLR_ROMPE
| E1000_VMOLR_MPME
);
6462 vf_data
= &adapter
->vf_data
[i
];
6464 if ((vf_data
->num_vf_mc_hashes
> 30) ||
6465 (vf_data
->flags
& IGB_VF_FLAG_MULTI_PROMISC
)) {
6466 vmolr
|= E1000_VMOLR_MPME
;
6467 } else if (vf_data
->num_vf_mc_hashes
) {
6468 vmolr
|= E1000_VMOLR_ROMPE
;
6469 for (j
= 0; j
< vf_data
->num_vf_mc_hashes
; j
++)
6470 igb_mta_set(hw
, vf_data
->vf_mc_hashes
[j
]);
6472 wr32(E1000_VMOLR(i
), vmolr
);
6476 static void igb_clear_vf_vfta(struct igb_adapter
*adapter
, u32 vf
)
6478 struct e1000_hw
*hw
= &adapter
->hw
;
6479 u32 pool_mask
, vlvf_mask
, i
;
6481 /* create mask for VF and other pools */
6482 pool_mask
= E1000_VLVF_POOLSEL_MASK
;
6483 vlvf_mask
= BIT(E1000_VLVF_POOLSEL_SHIFT
+ vf
);
6485 /* drop PF from pool bits */
6486 pool_mask
&= ~BIT(E1000_VLVF_POOLSEL_SHIFT
+
6487 adapter
->vfs_allocated_count
);
6489 /* Find the vlan filter for this id */
6490 for (i
= E1000_VLVF_ARRAY_SIZE
; i
--;) {
6491 u32 vlvf
= rd32(E1000_VLVF(i
));
6492 u32 vfta_mask
, vid
, vfta
;
6494 /* remove the vf from the pool */
6495 if (!(vlvf
& vlvf_mask
))
6498 /* clear out bit from VLVF */
6501 /* if other pools are present, just remove ourselves */
6502 if (vlvf
& pool_mask
)
6505 /* if PF is present, leave VFTA */
6506 if (vlvf
& E1000_VLVF_POOLSEL_MASK
)
6509 vid
= vlvf
& E1000_VLVF_VLANID_MASK
;
6510 vfta_mask
= BIT(vid
% 32);
6512 /* clear bit from VFTA */
6513 vfta
= adapter
->shadow_vfta
[vid
/ 32];
6514 if (vfta
& vfta_mask
)
6515 hw
->mac
.ops
.write_vfta(hw
, vid
/ 32, vfta
^ vfta_mask
);
6517 /* clear pool selection enable */
6518 if (adapter
->flags
& IGB_FLAG_VLAN_PROMISC
)
6519 vlvf
&= E1000_VLVF_POOLSEL_MASK
;
6523 /* clear pool bits */
6524 wr32(E1000_VLVF(i
), vlvf
);
6528 static int igb_find_vlvf_entry(struct e1000_hw
*hw
, u32 vlan
)
6533 /* short cut the special case */
6537 /* Search for the VLAN id in the VLVF entries */
6538 for (idx
= E1000_VLVF_ARRAY_SIZE
; --idx
;) {
6539 vlvf
= rd32(E1000_VLVF(idx
));
6540 if ((vlvf
& VLAN_VID_MASK
) == vlan
)
6547 static void igb_update_pf_vlvf(struct igb_adapter
*adapter
, u32 vid
)
6549 struct e1000_hw
*hw
= &adapter
->hw
;
6553 idx
= igb_find_vlvf_entry(hw
, vid
);
6557 /* See if any other pools are set for this VLAN filter
6558 * entry other than the PF.
6560 pf_id
= adapter
->vfs_allocated_count
+ E1000_VLVF_POOLSEL_SHIFT
;
6561 bits
= ~BIT(pf_id
) & E1000_VLVF_POOLSEL_MASK
;
6562 bits
&= rd32(E1000_VLVF(idx
));
6564 /* Disable the filter so this falls into the default pool. */
6566 if (adapter
->flags
& IGB_FLAG_VLAN_PROMISC
)
6567 wr32(E1000_VLVF(idx
), BIT(pf_id
));
6569 wr32(E1000_VLVF(idx
), 0);
6573 static s32
igb_set_vf_vlan(struct igb_adapter
*adapter
, u32 vid
,
6576 int pf_id
= adapter
->vfs_allocated_count
;
6577 struct e1000_hw
*hw
= &adapter
->hw
;
6580 /* If VLAN overlaps with one the PF is currently monitoring make
6581 * sure that we are able to allocate a VLVF entry. This may be
6582 * redundant but it guarantees PF will maintain visibility to
6585 if (add
&& test_bit(vid
, adapter
->active_vlans
)) {
6586 err
= igb_vfta_set(hw
, vid
, pf_id
, true, false);
6591 err
= igb_vfta_set(hw
, vid
, vf
, add
, false);
6596 /* If we failed to add the VF VLAN or we are removing the VF VLAN
6597 * we may need to drop the PF pool bit in order to allow us to free
6598 * up the VLVF resources.
6600 if (test_bit(vid
, adapter
->active_vlans
) ||
6601 (adapter
->flags
& IGB_FLAG_VLAN_PROMISC
))
6602 igb_update_pf_vlvf(adapter
, vid
);
6607 static void igb_set_vmvir(struct igb_adapter
*adapter
, u32 vid
, u32 vf
)
6609 struct e1000_hw
*hw
= &adapter
->hw
;
6612 wr32(E1000_VMVIR(vf
), (vid
| E1000_VMVIR_VLANA_DEFAULT
));
6614 wr32(E1000_VMVIR(vf
), 0);
6617 static int igb_enable_port_vlan(struct igb_adapter
*adapter
, int vf
,
6622 err
= igb_set_vf_vlan(adapter
, vlan
, true, vf
);
6626 igb_set_vmvir(adapter
, vlan
| (qos
<< VLAN_PRIO_SHIFT
), vf
);
6627 igb_set_vmolr(adapter
, vf
, !vlan
);
6629 /* revoke access to previous VLAN */
6630 if (vlan
!= adapter
->vf_data
[vf
].pf_vlan
)
6631 igb_set_vf_vlan(adapter
, adapter
->vf_data
[vf
].pf_vlan
,
6634 adapter
->vf_data
[vf
].pf_vlan
= vlan
;
6635 adapter
->vf_data
[vf
].pf_qos
= qos
;
6636 igb_set_vf_vlan_strip(adapter
, vf
, true);
6637 dev_info(&adapter
->pdev
->dev
,
6638 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan
, qos
, vf
);
6639 if (test_bit(__IGB_DOWN
, &adapter
->state
)) {
6640 dev_warn(&adapter
->pdev
->dev
,
6641 "The VF VLAN has been set, but the PF device is not up.\n");
6642 dev_warn(&adapter
->pdev
->dev
,
6643 "Bring the PF device up before attempting to use the VF device.\n");
6649 static int igb_disable_port_vlan(struct igb_adapter
*adapter
, int vf
)
6651 /* Restore tagless access via VLAN 0 */
6652 igb_set_vf_vlan(adapter
, 0, true, vf
);
6654 igb_set_vmvir(adapter
, 0, vf
);
6655 igb_set_vmolr(adapter
, vf
, true);
6657 /* Remove any PF assigned VLAN */
6658 if (adapter
->vf_data
[vf
].pf_vlan
)
6659 igb_set_vf_vlan(adapter
, adapter
->vf_data
[vf
].pf_vlan
,
6662 adapter
->vf_data
[vf
].pf_vlan
= 0;
6663 adapter
->vf_data
[vf
].pf_qos
= 0;
6664 igb_set_vf_vlan_strip(adapter
, vf
, false);
6669 static int igb_ndo_set_vf_vlan(struct net_device
*netdev
, int vf
,
6670 u16 vlan
, u8 qos
, __be16 vlan_proto
)
6672 struct igb_adapter
*adapter
= netdev_priv(netdev
);
6674 if ((vf
>= adapter
->vfs_allocated_count
) || (vlan
> 4095) || (qos
> 7))
6677 if (vlan_proto
!= htons(ETH_P_8021Q
))
6678 return -EPROTONOSUPPORT
;
6680 return (vlan
|| qos
) ? igb_enable_port_vlan(adapter
, vf
, vlan
, qos
) :
6681 igb_disable_port_vlan(adapter
, vf
);
6684 static int igb_set_vf_vlan_msg(struct igb_adapter
*adapter
, u32
*msgbuf
, u32 vf
)
6686 int add
= (msgbuf
[0] & E1000_VT_MSGINFO_MASK
) >> E1000_VT_MSGINFO_SHIFT
;
6687 int vid
= (msgbuf
[1] & E1000_VLVF_VLANID_MASK
);
6690 if (adapter
->vf_data
[vf
].pf_vlan
)
6693 /* VLAN 0 is a special case, don't allow it to be removed */
6697 ret
= igb_set_vf_vlan(adapter
, vid
, !!add
, vf
);
6699 igb_set_vf_vlan_strip(adapter
, vf
, !!vid
);
6703 static inline void igb_vf_reset(struct igb_adapter
*adapter
, u32 vf
)
6705 struct vf_data_storage
*vf_data
= &adapter
->vf_data
[vf
];
6707 /* clear flags - except flag that indicates PF has set the MAC */
6708 vf_data
->flags
&= IGB_VF_FLAG_PF_SET_MAC
;
6709 vf_data
->last_nack
= jiffies
;
6711 /* reset vlans for device */
6712 igb_clear_vf_vfta(adapter
, vf
);
6713 igb_set_vf_vlan(adapter
, vf_data
->pf_vlan
, true, vf
);
6714 igb_set_vmvir(adapter
, vf_data
->pf_vlan
|
6715 (vf_data
->pf_qos
<< VLAN_PRIO_SHIFT
), vf
);
6716 igb_set_vmolr(adapter
, vf
, !vf_data
->pf_vlan
);
6717 igb_set_vf_vlan_strip(adapter
, vf
, !!(vf_data
->pf_vlan
));
6719 /* reset multicast table array for vf */
6720 adapter
->vf_data
[vf
].num_vf_mc_hashes
= 0;
6722 /* Flush and reset the mta with the new values */
6723 igb_set_rx_mode(adapter
->netdev
);
6726 static void igb_vf_reset_event(struct igb_adapter
*adapter
, u32 vf
)
6728 unsigned char *vf_mac
= adapter
->vf_data
[vf
].vf_mac_addresses
;
6730 /* clear mac address as we were hotplug removed/added */
6731 if (!(adapter
->vf_data
[vf
].flags
& IGB_VF_FLAG_PF_SET_MAC
))
6732 eth_zero_addr(vf_mac
);
6734 /* process remaining reset events */
6735 igb_vf_reset(adapter
, vf
);
6738 static void igb_vf_reset_msg(struct igb_adapter
*adapter
, u32 vf
)
6740 struct e1000_hw
*hw
= &adapter
->hw
;
6741 unsigned char *vf_mac
= adapter
->vf_data
[vf
].vf_mac_addresses
;
6743 u8
*addr
= (u8
*)(&msgbuf
[1]);
6745 /* process all the same items cleared in a function level reset */
6746 igb_vf_reset(adapter
, vf
);
6748 /* set vf mac address */
6749 igb_set_vf_mac(adapter
, vf
, vf_mac
);
6751 /* enable transmit and receive for vf */
6752 reg
= rd32(E1000_VFTE
);
6753 wr32(E1000_VFTE
, reg
| BIT(vf
));
6754 reg
= rd32(E1000_VFRE
);
6755 wr32(E1000_VFRE
, reg
| BIT(vf
));
6757 adapter
->vf_data
[vf
].flags
|= IGB_VF_FLAG_CTS
;
6759 /* reply to reset with ack and vf mac address */
6760 if (!is_zero_ether_addr(vf_mac
)) {
6761 msgbuf
[0] = E1000_VF_RESET
| E1000_VT_MSGTYPE_ACK
;
6762 memcpy(addr
, vf_mac
, ETH_ALEN
);
6764 msgbuf
[0] = E1000_VF_RESET
| E1000_VT_MSGTYPE_NACK
;
6766 igb_write_mbx(hw
, msgbuf
, 3, vf
);
6769 static void igb_flush_mac_table(struct igb_adapter
*adapter
)
6771 struct e1000_hw
*hw
= &adapter
->hw
;
6774 for (i
= 0; i
< hw
->mac
.rar_entry_count
; i
++) {
6775 adapter
->mac_table
[i
].state
&= ~IGB_MAC_STATE_IN_USE
;
6776 memset(adapter
->mac_table
[i
].addr
, 0, ETH_ALEN
);
6777 adapter
->mac_table
[i
].queue
= 0;
6778 igb_rar_set_index(adapter
, i
);
6782 static int igb_available_rars(struct igb_adapter
*adapter
, u8 queue
)
6784 struct e1000_hw
*hw
= &adapter
->hw
;
6785 /* do not count rar entries reserved for VFs MAC addresses */
6786 int rar_entries
= hw
->mac
.rar_entry_count
-
6787 adapter
->vfs_allocated_count
;
6790 for (i
= 0; i
< rar_entries
; i
++) {
6791 /* do not count default entries */
6792 if (adapter
->mac_table
[i
].state
& IGB_MAC_STATE_DEFAULT
)
6795 /* do not count "in use" entries for different queues */
6796 if ((adapter
->mac_table
[i
].state
& IGB_MAC_STATE_IN_USE
) &&
6797 (adapter
->mac_table
[i
].queue
!= queue
))
6806 /* Set default MAC address for the PF in the first RAR entry */
6807 static void igb_set_default_mac_filter(struct igb_adapter
*adapter
)
6809 struct igb_mac_addr
*mac_table
= &adapter
->mac_table
[0];
6811 ether_addr_copy(mac_table
->addr
, adapter
->hw
.mac
.addr
);
6812 mac_table
->queue
= adapter
->vfs_allocated_count
;
6813 mac_table
->state
= IGB_MAC_STATE_DEFAULT
| IGB_MAC_STATE_IN_USE
;
6815 igb_rar_set_index(adapter
, 0);
6818 static int igb_add_mac_filter(struct igb_adapter
*adapter
, const u8
*addr
,
6821 struct e1000_hw
*hw
= &adapter
->hw
;
6822 int rar_entries
= hw
->mac
.rar_entry_count
-
6823 adapter
->vfs_allocated_count
;
6826 if (is_zero_ether_addr(addr
))
6829 /* Search for the first empty entry in the MAC table.
6830 * Do not touch entries at the end of the table reserved for the VF MAC
6833 for (i
= 0; i
< rar_entries
; i
++) {
6834 if (adapter
->mac_table
[i
].state
& IGB_MAC_STATE_IN_USE
)
6837 ether_addr_copy(adapter
->mac_table
[i
].addr
, addr
);
6838 adapter
->mac_table
[i
].queue
= queue
;
6839 adapter
->mac_table
[i
].state
|= IGB_MAC_STATE_IN_USE
;
6841 igb_rar_set_index(adapter
, i
);
6848 static int igb_del_mac_filter(struct igb_adapter
*adapter
, const u8
*addr
,
6851 struct e1000_hw
*hw
= &adapter
->hw
;
6852 int rar_entries
= hw
->mac
.rar_entry_count
-
6853 adapter
->vfs_allocated_count
;
6856 if (is_zero_ether_addr(addr
))
6859 /* Search for matching entry in the MAC table based on given address
6860 * and queue. Do not touch entries at the end of the table reserved
6861 * for the VF MAC addresses.
6863 for (i
= 0; i
< rar_entries
; i
++) {
6864 if (!(adapter
->mac_table
[i
].state
& IGB_MAC_STATE_IN_USE
))
6866 if (adapter
->mac_table
[i
].queue
!= queue
)
6868 if (!ether_addr_equal(adapter
->mac_table
[i
].addr
, addr
))
6871 adapter
->mac_table
[i
].state
&= ~IGB_MAC_STATE_IN_USE
;
6872 memset(adapter
->mac_table
[i
].addr
, 0, ETH_ALEN
);
6873 adapter
->mac_table
[i
].queue
= 0;
6875 igb_rar_set_index(adapter
, i
);
6882 static int igb_uc_sync(struct net_device
*netdev
, const unsigned char *addr
)
6884 struct igb_adapter
*adapter
= netdev_priv(netdev
);
6887 ret
= igb_add_mac_filter(adapter
, addr
, adapter
->vfs_allocated_count
);
6889 return min_t(int, ret
, 0);
6892 static int igb_uc_unsync(struct net_device
*netdev
, const unsigned char *addr
)
6894 struct igb_adapter
*adapter
= netdev_priv(netdev
);
6896 igb_del_mac_filter(adapter
, addr
, adapter
->vfs_allocated_count
);
6901 static int igb_set_vf_mac_filter(struct igb_adapter
*adapter
, const int vf
,
6902 const u32 info
, const u8
*addr
)
6904 struct pci_dev
*pdev
= adapter
->pdev
;
6905 struct vf_data_storage
*vf_data
= &adapter
->vf_data
[vf
];
6906 struct list_head
*pos
;
6907 struct vf_mac_filter
*entry
= NULL
;
6911 case E1000_VF_MAC_FILTER_CLR
:
6912 /* remove all unicast MAC filters related to the current VF */
6913 list_for_each(pos
, &adapter
->vf_macs
.l
) {
6914 entry
= list_entry(pos
, struct vf_mac_filter
, l
);
6915 if (entry
->vf
== vf
) {
6918 igb_del_mac_filter(adapter
, entry
->vf_mac
, vf
);
6922 case E1000_VF_MAC_FILTER_ADD
:
6923 if (vf_data
->flags
& IGB_VF_FLAG_PF_SET_MAC
) {
6924 dev_warn(&pdev
->dev
,
6925 "VF %d requested MAC filter but is administratively denied\n",
6930 if (!is_valid_ether_addr(addr
)) {
6931 dev_warn(&pdev
->dev
,
6932 "VF %d attempted to set invalid MAC filter\n",
6937 /* try to find empty slot in the list */
6938 list_for_each(pos
, &adapter
->vf_macs
.l
) {
6939 entry
= list_entry(pos
, struct vf_mac_filter
, l
);
6944 if (entry
&& entry
->free
) {
6945 entry
->free
= false;
6947 ether_addr_copy(entry
->vf_mac
, addr
);
6949 ret
= igb_add_mac_filter(adapter
, addr
, vf
);
6950 ret
= min_t(int, ret
, 0);
6956 dev_warn(&pdev
->dev
,
6957 "VF %d has requested MAC filter but there is no space for it\n",
6968 static int igb_set_vf_mac_addr(struct igb_adapter
*adapter
, u32
*msg
, int vf
)
6970 struct pci_dev
*pdev
= adapter
->pdev
;
6971 struct vf_data_storage
*vf_data
= &adapter
->vf_data
[vf
];
6972 u32 info
= msg
[0] & E1000_VT_MSGINFO_MASK
;
6974 /* The VF MAC Address is stored in a packed array of bytes
6975 * starting at the second 32 bit word of the msg array
6977 unsigned char *addr
= (unsigned char *)&msg
[1];
6981 if (vf_data
->flags
& IGB_VF_FLAG_PF_SET_MAC
) {
6982 dev_warn(&pdev
->dev
,
6983 "VF %d attempted to override administratively set MAC address\nReload the VF driver to resume operations\n",
6988 if (!is_valid_ether_addr(addr
)) {
6989 dev_warn(&pdev
->dev
,
6990 "VF %d attempted to set invalid MAC\n",
6995 ret
= igb_set_vf_mac(adapter
, vf
, addr
);
6997 ret
= igb_set_vf_mac_filter(adapter
, vf
, info
, addr
);
7003 static void igb_rcv_ack_from_vf(struct igb_adapter
*adapter
, u32 vf
)
7005 struct e1000_hw
*hw
= &adapter
->hw
;
7006 struct vf_data_storage
*vf_data
= &adapter
->vf_data
[vf
];
7007 u32 msg
= E1000_VT_MSGTYPE_NACK
;
7009 /* if device isn't clear to send it shouldn't be reading either */
7010 if (!(vf_data
->flags
& IGB_VF_FLAG_CTS
) &&
7011 time_after(jiffies
, vf_data
->last_nack
+ (2 * HZ
))) {
7012 igb_write_mbx(hw
, &msg
, 1, vf
);
7013 vf_data
->last_nack
= jiffies
;
7017 static void igb_rcv_msg_from_vf(struct igb_adapter
*adapter
, u32 vf
)
7019 struct pci_dev
*pdev
= adapter
->pdev
;
7020 u32 msgbuf
[E1000_VFMAILBOX_SIZE
];
7021 struct e1000_hw
*hw
= &adapter
->hw
;
7022 struct vf_data_storage
*vf_data
= &adapter
->vf_data
[vf
];
7025 retval
= igb_read_mbx(hw
, msgbuf
, E1000_VFMAILBOX_SIZE
, vf
, false);
7028 /* if receive failed revoke VF CTS stats and restart init */
7029 dev_err(&pdev
->dev
, "Error receiving message from VF\n");
7030 vf_data
->flags
&= ~IGB_VF_FLAG_CTS
;
7031 if (!time_after(jiffies
, vf_data
->last_nack
+ (2 * HZ
)))
7036 /* this is a message we already processed, do nothing */
7037 if (msgbuf
[0] & (E1000_VT_MSGTYPE_ACK
| E1000_VT_MSGTYPE_NACK
))
7040 /* until the vf completes a reset it should not be
7041 * allowed to start any configuration.
7043 if (msgbuf
[0] == E1000_VF_RESET
) {
7044 /* unlocks mailbox */
7045 igb_vf_reset_msg(adapter
, vf
);
7049 if (!(vf_data
->flags
& IGB_VF_FLAG_CTS
)) {
7050 if (!time_after(jiffies
, vf_data
->last_nack
+ (2 * HZ
)))
7056 switch ((msgbuf
[0] & 0xFFFF)) {
7057 case E1000_VF_SET_MAC_ADDR
:
7058 retval
= igb_set_vf_mac_addr(adapter
, msgbuf
, vf
);
7060 case E1000_VF_SET_PROMISC
:
7061 retval
= igb_set_vf_promisc(adapter
, msgbuf
, vf
);
7063 case E1000_VF_SET_MULTICAST
:
7064 retval
= igb_set_vf_multicasts(adapter
, msgbuf
, vf
);
7066 case E1000_VF_SET_LPE
:
7067 retval
= igb_set_vf_rlpml(adapter
, msgbuf
[1], vf
);
7069 case E1000_VF_SET_VLAN
:
7071 if (vf_data
->pf_vlan
)
7072 dev_warn(&pdev
->dev
,
7073 "VF %d attempted to override administratively set VLAN tag\nReload the VF driver to resume operations\n",
7076 retval
= igb_set_vf_vlan_msg(adapter
, msgbuf
, vf
);
7079 dev_err(&pdev
->dev
, "Unhandled Msg %08x\n", msgbuf
[0]);
7084 msgbuf
[0] |= E1000_VT_MSGTYPE_CTS
;
7086 /* notify the VF of the results of what it sent us */
7088 msgbuf
[0] |= E1000_VT_MSGTYPE_NACK
;
7090 msgbuf
[0] |= E1000_VT_MSGTYPE_ACK
;
7092 /* unlocks mailbox */
7093 igb_write_mbx(hw
, msgbuf
, 1, vf
);
7097 igb_unlock_mbx(hw
, vf
);
7100 static void igb_msg_task(struct igb_adapter
*adapter
)
7102 struct e1000_hw
*hw
= &adapter
->hw
;
7105 for (vf
= 0; vf
< adapter
->vfs_allocated_count
; vf
++) {
7106 /* process any reset requests */
7107 if (!igb_check_for_rst(hw
, vf
))
7108 igb_vf_reset_event(adapter
, vf
);
7110 /* process any messages pending */
7111 if (!igb_check_for_msg(hw
, vf
))
7112 igb_rcv_msg_from_vf(adapter
, vf
);
7114 /* process any acks */
7115 if (!igb_check_for_ack(hw
, vf
))
7116 igb_rcv_ack_from_vf(adapter
, vf
);
7121 * igb_set_uta - Set unicast filter table address
7122 * @adapter: board private structure
7123 * @set: boolean indicating if we are setting or clearing bits
7125 * The unicast table address is a register array of 32-bit registers.
7126 * The table is meant to be used in a way similar to how the MTA is used
7127 * however due to certain limitations in the hardware it is necessary to
7128 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
7129 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
7131 static void igb_set_uta(struct igb_adapter
*adapter
, bool set
)
7133 struct e1000_hw
*hw
= &adapter
->hw
;
7134 u32 uta
= set
? ~0 : 0;
7137 /* we only need to do this if VMDq is enabled */
7138 if (!adapter
->vfs_allocated_count
)
7141 for (i
= hw
->mac
.uta_reg_count
; i
--;)
7142 array_wr32(E1000_UTA
, i
, uta
);
7146 * igb_intr_msi - Interrupt Handler
7147 * @irq: interrupt number
7148 * @data: pointer to a network interface device structure
7150 static irqreturn_t
igb_intr_msi(int irq
, void *data
)
7152 struct igb_adapter
*adapter
= data
;
7153 struct igb_q_vector
*q_vector
= adapter
->q_vector
[0];
7154 struct e1000_hw
*hw
= &adapter
->hw
;
7155 /* read ICR disables interrupts using IAM */
7156 u32 icr
= rd32(E1000_ICR
);
7158 igb_write_itr(q_vector
);
7160 if (icr
& E1000_ICR_DRSTA
)
7161 schedule_work(&adapter
->reset_task
);
7163 if (icr
& E1000_ICR_DOUTSYNC
) {
7164 /* HW is reporting DMA is out of sync */
7165 adapter
->stats
.doosync
++;
7168 if (icr
& (E1000_ICR_RXSEQ
| E1000_ICR_LSC
)) {
7169 hw
->mac
.get_link_status
= 1;
7170 if (!test_bit(__IGB_DOWN
, &adapter
->state
))
7171 mod_timer(&adapter
->watchdog_timer
, jiffies
+ 1);
7174 if (icr
& E1000_ICR_TS
)
7175 igb_tsync_interrupt(adapter
);
7177 napi_schedule(&q_vector
->napi
);
7183 * igb_intr - Legacy Interrupt Handler
7184 * @irq: interrupt number
7185 * @data: pointer to a network interface device structure
7187 static irqreturn_t
igb_intr(int irq
, void *data
)
7189 struct igb_adapter
*adapter
= data
;
7190 struct igb_q_vector
*q_vector
= adapter
->q_vector
[0];
7191 struct e1000_hw
*hw
= &adapter
->hw
;
7192 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
7193 * need for the IMC write
7195 u32 icr
= rd32(E1000_ICR
);
7197 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
7198 * not set, then the adapter didn't send an interrupt
7200 if (!(icr
& E1000_ICR_INT_ASSERTED
))
7203 igb_write_itr(q_vector
);
7205 if (icr
& E1000_ICR_DRSTA
)
7206 schedule_work(&adapter
->reset_task
);
7208 if (icr
& E1000_ICR_DOUTSYNC
) {
7209 /* HW is reporting DMA is out of sync */
7210 adapter
->stats
.doosync
++;
7213 if (icr
& (E1000_ICR_RXSEQ
| E1000_ICR_LSC
)) {
7214 hw
->mac
.get_link_status
= 1;
7215 /* guard against interrupt when we're going down */
7216 if (!test_bit(__IGB_DOWN
, &adapter
->state
))
7217 mod_timer(&adapter
->watchdog_timer
, jiffies
+ 1);
7220 if (icr
& E1000_ICR_TS
)
7221 igb_tsync_interrupt(adapter
);
7223 napi_schedule(&q_vector
->napi
);
7228 static void igb_ring_irq_enable(struct igb_q_vector
*q_vector
)
7230 struct igb_adapter
*adapter
= q_vector
->adapter
;
7231 struct e1000_hw
*hw
= &adapter
->hw
;
7233 if ((q_vector
->rx
.ring
&& (adapter
->rx_itr_setting
& 3)) ||
7234 (!q_vector
->rx
.ring
&& (adapter
->tx_itr_setting
& 3))) {
7235 if ((adapter
->num_q_vectors
== 1) && !adapter
->vf_data
)
7236 igb_set_itr(q_vector
);
7238 igb_update_ring_itr(q_vector
);
7241 if (!test_bit(__IGB_DOWN
, &adapter
->state
)) {
7242 if (adapter
->flags
& IGB_FLAG_HAS_MSIX
)
7243 wr32(E1000_EIMS
, q_vector
->eims_value
);
7245 igb_irq_enable(adapter
);
7250 * igb_poll - NAPI Rx polling callback
7251 * @napi: napi polling structure
7252 * @budget: count of how many packets we should handle
7254 static int igb_poll(struct napi_struct
*napi
, int budget
)
7256 struct igb_q_vector
*q_vector
= container_of(napi
,
7257 struct igb_q_vector
,
7259 bool clean_complete
= true;
7262 #ifdef CONFIG_IGB_DCA
7263 if (q_vector
->adapter
->flags
& IGB_FLAG_DCA_ENABLED
)
7264 igb_update_dca(q_vector
);
7266 if (q_vector
->tx
.ring
)
7267 clean_complete
= igb_clean_tx_irq(q_vector
, budget
);
7269 if (q_vector
->rx
.ring
) {
7270 int cleaned
= igb_clean_rx_irq(q_vector
, budget
);
7272 work_done
+= cleaned
;
7273 if (cleaned
>= budget
)
7274 clean_complete
= false;
7277 /* If all work not completed, return budget and keep polling */
7278 if (!clean_complete
)
7281 /* If not enough Rx work done, exit the polling mode */
7282 napi_complete_done(napi
, work_done
);
7283 igb_ring_irq_enable(q_vector
);
7289 * igb_clean_tx_irq - Reclaim resources after transmit completes
7290 * @q_vector: pointer to q_vector containing needed info
7291 * @napi_budget: Used to determine if we are in netpoll
7293 * returns true if ring is completely cleaned
7295 static bool igb_clean_tx_irq(struct igb_q_vector
*q_vector
, int napi_budget
)
7297 struct igb_adapter
*adapter
= q_vector
->adapter
;
7298 struct igb_ring
*tx_ring
= q_vector
->tx
.ring
;
7299 struct igb_tx_buffer
*tx_buffer
;
7300 union e1000_adv_tx_desc
*tx_desc
;
7301 unsigned int total_bytes
= 0, total_packets
= 0;
7302 unsigned int budget
= q_vector
->tx
.work_limit
;
7303 unsigned int i
= tx_ring
->next_to_clean
;
7305 if (test_bit(__IGB_DOWN
, &adapter
->state
))
7308 tx_buffer
= &tx_ring
->tx_buffer_info
[i
];
7309 tx_desc
= IGB_TX_DESC(tx_ring
, i
);
7310 i
-= tx_ring
->count
;
7313 union e1000_adv_tx_desc
*eop_desc
= tx_buffer
->next_to_watch
;
7315 /* if next_to_watch is not set then there is no work pending */
7319 /* prevent any other reads prior to eop_desc */
7320 read_barrier_depends();
7322 /* if DD is not set pending work has not been completed */
7323 if (!(eop_desc
->wb
.status
& cpu_to_le32(E1000_TXD_STAT_DD
)))
7326 /* clear next_to_watch to prevent false hangs */
7327 tx_buffer
->next_to_watch
= NULL
;
7329 /* update the statistics for this packet */
7330 total_bytes
+= tx_buffer
->bytecount
;
7331 total_packets
+= tx_buffer
->gso_segs
;
7334 napi_consume_skb(tx_buffer
->skb
, napi_budget
);
7336 /* unmap skb header data */
7337 dma_unmap_single(tx_ring
->dev
,
7338 dma_unmap_addr(tx_buffer
, dma
),
7339 dma_unmap_len(tx_buffer
, len
),
7342 /* clear tx_buffer data */
7343 dma_unmap_len_set(tx_buffer
, len
, 0);
7345 /* clear last DMA location and unmap remaining buffers */
7346 while (tx_desc
!= eop_desc
) {
7351 i
-= tx_ring
->count
;
7352 tx_buffer
= tx_ring
->tx_buffer_info
;
7353 tx_desc
= IGB_TX_DESC(tx_ring
, 0);
7356 /* unmap any remaining paged data */
7357 if (dma_unmap_len(tx_buffer
, len
)) {
7358 dma_unmap_page(tx_ring
->dev
,
7359 dma_unmap_addr(tx_buffer
, dma
),
7360 dma_unmap_len(tx_buffer
, len
),
7362 dma_unmap_len_set(tx_buffer
, len
, 0);
7366 /* move us one more past the eop_desc for start of next pkt */
7371 i
-= tx_ring
->count
;
7372 tx_buffer
= tx_ring
->tx_buffer_info
;
7373 tx_desc
= IGB_TX_DESC(tx_ring
, 0);
7376 /* issue prefetch for next Tx descriptor */
7379 /* update budget accounting */
7381 } while (likely(budget
));
7383 netdev_tx_completed_queue(txring_txq(tx_ring
),
7384 total_packets
, total_bytes
);
7385 i
+= tx_ring
->count
;
7386 tx_ring
->next_to_clean
= i
;
7387 u64_stats_update_begin(&tx_ring
->tx_syncp
);
7388 tx_ring
->tx_stats
.bytes
+= total_bytes
;
7389 tx_ring
->tx_stats
.packets
+= total_packets
;
7390 u64_stats_update_end(&tx_ring
->tx_syncp
);
7391 q_vector
->tx
.total_bytes
+= total_bytes
;
7392 q_vector
->tx
.total_packets
+= total_packets
;
7394 if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG
, &tx_ring
->flags
)) {
7395 struct e1000_hw
*hw
= &adapter
->hw
;
7397 /* Detect a transmit hang in hardware, this serializes the
7398 * check with the clearing of time_stamp and movement of i
7400 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG
, &tx_ring
->flags
);
7401 if (tx_buffer
->next_to_watch
&&
7402 time_after(jiffies
, tx_buffer
->time_stamp
+
7403 (adapter
->tx_timeout_factor
* HZ
)) &&
7404 !(rd32(E1000_STATUS
) & E1000_STATUS_TXOFF
)) {
7406 /* detected Tx unit hang */
7407 dev_err(tx_ring
->dev
,
7408 "Detected Tx Unit Hang\n"
7412 " next_to_use <%x>\n"
7413 " next_to_clean <%x>\n"
7414 "buffer_info[next_to_clean]\n"
7415 " time_stamp <%lx>\n"
7416 " next_to_watch <%p>\n"
7418 " desc.status <%x>\n",
7419 tx_ring
->queue_index
,
7420 rd32(E1000_TDH(tx_ring
->reg_idx
)),
7421 readl(tx_ring
->tail
),
7422 tx_ring
->next_to_use
,
7423 tx_ring
->next_to_clean
,
7424 tx_buffer
->time_stamp
,
7425 tx_buffer
->next_to_watch
,
7427 tx_buffer
->next_to_watch
->wb
.status
);
7428 netif_stop_subqueue(tx_ring
->netdev
,
7429 tx_ring
->queue_index
);
7431 /* we are about to reset, no point in enabling stuff */
7436 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
7437 if (unlikely(total_packets
&&
7438 netif_carrier_ok(tx_ring
->netdev
) &&
7439 igb_desc_unused(tx_ring
) >= TX_WAKE_THRESHOLD
)) {
7440 /* Make sure that anybody stopping the queue after this
7441 * sees the new next_to_clean.
7444 if (__netif_subqueue_stopped(tx_ring
->netdev
,
7445 tx_ring
->queue_index
) &&
7446 !(test_bit(__IGB_DOWN
, &adapter
->state
))) {
7447 netif_wake_subqueue(tx_ring
->netdev
,
7448 tx_ring
->queue_index
);
7450 u64_stats_update_begin(&tx_ring
->tx_syncp
);
7451 tx_ring
->tx_stats
.restart_queue
++;
7452 u64_stats_update_end(&tx_ring
->tx_syncp
);
7460 * igb_reuse_rx_page - page flip buffer and store it back on the ring
7461 * @rx_ring: rx descriptor ring to store buffers on
7462 * @old_buff: donor buffer to have page reused
7464 * Synchronizes page for reuse by the adapter
7466 static void igb_reuse_rx_page(struct igb_ring
*rx_ring
,
7467 struct igb_rx_buffer
*old_buff
)
7469 struct igb_rx_buffer
*new_buff
;
7470 u16 nta
= rx_ring
->next_to_alloc
;
7472 new_buff
= &rx_ring
->rx_buffer_info
[nta
];
7474 /* update, and store next to alloc */
7476 rx_ring
->next_to_alloc
= (nta
< rx_ring
->count
) ? nta
: 0;
7478 /* Transfer page from old buffer to new buffer.
7479 * Move each member individually to avoid possible store
7480 * forwarding stalls.
7482 new_buff
->dma
= old_buff
->dma
;
7483 new_buff
->page
= old_buff
->page
;
7484 new_buff
->page_offset
= old_buff
->page_offset
;
7485 new_buff
->pagecnt_bias
= old_buff
->pagecnt_bias
;
7488 static inline bool igb_page_is_reserved(struct page
*page
)
7490 return (page_to_nid(page
) != numa_mem_id()) || page_is_pfmemalloc(page
);
7493 static bool igb_can_reuse_rx_page(struct igb_rx_buffer
*rx_buffer
)
7495 unsigned int pagecnt_bias
= rx_buffer
->pagecnt_bias
;
7496 struct page
*page
= rx_buffer
->page
;
7498 /* avoid re-using remote pages */
7499 if (unlikely(igb_page_is_reserved(page
)))
7502 #if (PAGE_SIZE < 8192)
7503 /* if we are only owner of page we can reuse it */
7504 if (unlikely((page_ref_count(page
) - pagecnt_bias
) > 1))
7507 #define IGB_LAST_OFFSET \
7508 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGB_RXBUFFER_2048)
7510 if (rx_buffer
->page_offset
> IGB_LAST_OFFSET
)
7514 /* If we have drained the page fragment pool we need to update
7515 * the pagecnt_bias and page count so that we fully restock the
7516 * number of references the driver holds.
7518 if (unlikely(!pagecnt_bias
)) {
7519 page_ref_add(page
, USHRT_MAX
);
7520 rx_buffer
->pagecnt_bias
= USHRT_MAX
;
7527 * igb_add_rx_frag - Add contents of Rx buffer to sk_buff
7528 * @rx_ring: rx descriptor ring to transact packets on
7529 * @rx_buffer: buffer containing page to add
7530 * @skb: sk_buff to place the data into
7531 * @size: size of buffer to be added
7533 * This function will add the data contained in rx_buffer->page to the skb.
7535 static void igb_add_rx_frag(struct igb_ring
*rx_ring
,
7536 struct igb_rx_buffer
*rx_buffer
,
7537 struct sk_buff
*skb
,
7540 #if (PAGE_SIZE < 8192)
7541 unsigned int truesize
= igb_rx_pg_size(rx_ring
) / 2;
7543 unsigned int truesize
= ring_uses_build_skb(rx_ring
) ?
7544 SKB_DATA_ALIGN(IGB_SKB_PAD
+ size
) :
7545 SKB_DATA_ALIGN(size
);
7547 skb_add_rx_frag(skb
, skb_shinfo(skb
)->nr_frags
, rx_buffer
->page
,
7548 rx_buffer
->page_offset
, size
, truesize
);
7549 #if (PAGE_SIZE < 8192)
7550 rx_buffer
->page_offset
^= truesize
;
7552 rx_buffer
->page_offset
+= truesize
;
7556 static struct sk_buff
*igb_construct_skb(struct igb_ring
*rx_ring
,
7557 struct igb_rx_buffer
*rx_buffer
,
7558 union e1000_adv_rx_desc
*rx_desc
,
7561 void *va
= page_address(rx_buffer
->page
) + rx_buffer
->page_offset
;
7562 #if (PAGE_SIZE < 8192)
7563 unsigned int truesize
= igb_rx_pg_size(rx_ring
) / 2;
7565 unsigned int truesize
= SKB_DATA_ALIGN(size
);
7567 unsigned int headlen
;
7568 struct sk_buff
*skb
;
7570 /* prefetch first cache line of first page */
7572 #if L1_CACHE_BYTES < 128
7573 prefetch(va
+ L1_CACHE_BYTES
);
7576 /* allocate a skb to store the frags */
7577 skb
= napi_alloc_skb(&rx_ring
->q_vector
->napi
, IGB_RX_HDR_LEN
);
7581 if (unlikely(igb_test_staterr(rx_desc
, E1000_RXDADV_STAT_TSIP
))) {
7582 igb_ptp_rx_pktstamp(rx_ring
->q_vector
, va
, skb
);
7583 va
+= IGB_TS_HDR_LEN
;
7584 size
-= IGB_TS_HDR_LEN
;
7587 /* Determine available headroom for copy */
7589 if (headlen
> IGB_RX_HDR_LEN
)
7590 headlen
= eth_get_headlen(va
, IGB_RX_HDR_LEN
);
7592 /* align pull length to size of long to optimize memcpy performance */
7593 memcpy(__skb_put(skb
, headlen
), va
, ALIGN(headlen
, sizeof(long)));
7595 /* update all of the pointers */
7598 skb_add_rx_frag(skb
, 0, rx_buffer
->page
,
7599 (va
+ headlen
) - page_address(rx_buffer
->page
),
7601 #if (PAGE_SIZE < 8192)
7602 rx_buffer
->page_offset
^= truesize
;
7604 rx_buffer
->page_offset
+= truesize
;
7607 rx_buffer
->pagecnt_bias
++;
7613 static struct sk_buff
*igb_build_skb(struct igb_ring
*rx_ring
,
7614 struct igb_rx_buffer
*rx_buffer
,
7615 union e1000_adv_rx_desc
*rx_desc
,
7618 void *va
= page_address(rx_buffer
->page
) + rx_buffer
->page_offset
;
7619 #if (PAGE_SIZE < 8192)
7620 unsigned int truesize
= igb_rx_pg_size(rx_ring
) / 2;
7622 unsigned int truesize
= SKB_DATA_ALIGN(sizeof(struct skb_shared_info
)) +
7623 SKB_DATA_ALIGN(IGB_SKB_PAD
+ size
);
7625 struct sk_buff
*skb
;
7627 /* prefetch first cache line of first page */
7629 #if L1_CACHE_BYTES < 128
7630 prefetch(va
+ L1_CACHE_BYTES
);
7633 /* build an skb around the page buffer */
7634 skb
= build_skb(va
- IGB_SKB_PAD
, truesize
);
7638 /* update pointers within the skb to store the data */
7639 skb_reserve(skb
, IGB_SKB_PAD
);
7640 __skb_put(skb
, size
);
7642 /* pull timestamp out of packet data */
7643 if (igb_test_staterr(rx_desc
, E1000_RXDADV_STAT_TSIP
)) {
7644 igb_ptp_rx_pktstamp(rx_ring
->q_vector
, skb
->data
, skb
);
7645 __skb_pull(skb
, IGB_TS_HDR_LEN
);
7648 /* update buffer offset */
7649 #if (PAGE_SIZE < 8192)
7650 rx_buffer
->page_offset
^= truesize
;
7652 rx_buffer
->page_offset
+= truesize
;
7658 static inline void igb_rx_checksum(struct igb_ring
*ring
,
7659 union e1000_adv_rx_desc
*rx_desc
,
7660 struct sk_buff
*skb
)
7662 skb_checksum_none_assert(skb
);
7664 /* Ignore Checksum bit is set */
7665 if (igb_test_staterr(rx_desc
, E1000_RXD_STAT_IXSM
))
7668 /* Rx checksum disabled via ethtool */
7669 if (!(ring
->netdev
->features
& NETIF_F_RXCSUM
))
7672 /* TCP/UDP checksum error bit is set */
7673 if (igb_test_staterr(rx_desc
,
7674 E1000_RXDEXT_STATERR_TCPE
|
7675 E1000_RXDEXT_STATERR_IPE
)) {
7676 /* work around errata with sctp packets where the TCPE aka
7677 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
7678 * packets, (aka let the stack check the crc32c)
7680 if (!((skb
->len
== 60) &&
7681 test_bit(IGB_RING_FLAG_RX_SCTP_CSUM
, &ring
->flags
))) {
7682 u64_stats_update_begin(&ring
->rx_syncp
);
7683 ring
->rx_stats
.csum_err
++;
7684 u64_stats_update_end(&ring
->rx_syncp
);
7686 /* let the stack verify checksum errors */
7689 /* It must be a TCP or UDP packet with a valid checksum */
7690 if (igb_test_staterr(rx_desc
, E1000_RXD_STAT_TCPCS
|
7691 E1000_RXD_STAT_UDPCS
))
7692 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
7694 dev_dbg(ring
->dev
, "cksum success: bits %08X\n",
7695 le32_to_cpu(rx_desc
->wb
.upper
.status_error
));
7698 static inline void igb_rx_hash(struct igb_ring
*ring
,
7699 union e1000_adv_rx_desc
*rx_desc
,
7700 struct sk_buff
*skb
)
7702 if (ring
->netdev
->features
& NETIF_F_RXHASH
)
7704 le32_to_cpu(rx_desc
->wb
.lower
.hi_dword
.rss
),
7709 * igb_is_non_eop - process handling of non-EOP buffers
7710 * @rx_ring: Rx ring being processed
7711 * @rx_desc: Rx descriptor for current buffer
7712 * @skb: current socket buffer containing buffer in progress
7714 * This function updates next to clean. If the buffer is an EOP buffer
7715 * this function exits returning false, otherwise it will place the
7716 * sk_buff in the next buffer to be chained and return true indicating
7717 * that this is in fact a non-EOP buffer.
7719 static bool igb_is_non_eop(struct igb_ring
*rx_ring
,
7720 union e1000_adv_rx_desc
*rx_desc
)
7722 u32 ntc
= rx_ring
->next_to_clean
+ 1;
7724 /* fetch, update, and store next to clean */
7725 ntc
= (ntc
< rx_ring
->count
) ? ntc
: 0;
7726 rx_ring
->next_to_clean
= ntc
;
7728 prefetch(IGB_RX_DESC(rx_ring
, ntc
));
7730 if (likely(igb_test_staterr(rx_desc
, E1000_RXD_STAT_EOP
)))
7737 * igb_cleanup_headers - Correct corrupted or empty headers
7738 * @rx_ring: rx descriptor ring packet is being transacted on
7739 * @rx_desc: pointer to the EOP Rx descriptor
7740 * @skb: pointer to current skb being fixed
7742 * Address the case where we are pulling data in on pages only
7743 * and as such no data is present in the skb header.
7745 * In addition if skb is not at least 60 bytes we need to pad it so that
7746 * it is large enough to qualify as a valid Ethernet frame.
7748 * Returns true if an error was encountered and skb was freed.
7750 static bool igb_cleanup_headers(struct igb_ring
*rx_ring
,
7751 union e1000_adv_rx_desc
*rx_desc
,
7752 struct sk_buff
*skb
)
7754 if (unlikely((igb_test_staterr(rx_desc
,
7755 E1000_RXDEXT_ERR_FRAME_ERR_MASK
)))) {
7756 struct net_device
*netdev
= rx_ring
->netdev
;
7757 if (!(netdev
->features
& NETIF_F_RXALL
)) {
7758 dev_kfree_skb_any(skb
);
7763 /* if eth_skb_pad returns an error the skb was freed */
7764 if (eth_skb_pad(skb
))
7771 * igb_process_skb_fields - Populate skb header fields from Rx descriptor
7772 * @rx_ring: rx descriptor ring packet is being transacted on
7773 * @rx_desc: pointer to the EOP Rx descriptor
7774 * @skb: pointer to current skb being populated
7776 * This function checks the ring, descriptor, and packet information in
7777 * order to populate the hash, checksum, VLAN, timestamp, protocol, and
7778 * other fields within the skb.
7780 static void igb_process_skb_fields(struct igb_ring
*rx_ring
,
7781 union e1000_adv_rx_desc
*rx_desc
,
7782 struct sk_buff
*skb
)
7784 struct net_device
*dev
= rx_ring
->netdev
;
7786 igb_rx_hash(rx_ring
, rx_desc
, skb
);
7788 igb_rx_checksum(rx_ring
, rx_desc
, skb
);
7790 if (igb_test_staterr(rx_desc
, E1000_RXDADV_STAT_TS
) &&
7791 !igb_test_staterr(rx_desc
, E1000_RXDADV_STAT_TSIP
))
7792 igb_ptp_rx_rgtstamp(rx_ring
->q_vector
, skb
);
7794 if ((dev
->features
& NETIF_F_HW_VLAN_CTAG_RX
) &&
7795 igb_test_staterr(rx_desc
, E1000_RXD_STAT_VP
)) {
7798 if (igb_test_staterr(rx_desc
, E1000_RXDEXT_STATERR_LB
) &&
7799 test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP
, &rx_ring
->flags
))
7800 vid
= be16_to_cpu(rx_desc
->wb
.upper
.vlan
);
7802 vid
= le16_to_cpu(rx_desc
->wb
.upper
.vlan
);
7804 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
), vid
);
7807 skb_record_rx_queue(skb
, rx_ring
->queue_index
);
7809 skb
->protocol
= eth_type_trans(skb
, rx_ring
->netdev
);
7812 static struct igb_rx_buffer
*igb_get_rx_buffer(struct igb_ring
*rx_ring
,
7813 const unsigned int size
)
7815 struct igb_rx_buffer
*rx_buffer
;
7817 rx_buffer
= &rx_ring
->rx_buffer_info
[rx_ring
->next_to_clean
];
7818 prefetchw(rx_buffer
->page
);
7820 /* we are reusing so sync this buffer for CPU use */
7821 dma_sync_single_range_for_cpu(rx_ring
->dev
,
7823 rx_buffer
->page_offset
,
7827 rx_buffer
->pagecnt_bias
--;
7832 static void igb_put_rx_buffer(struct igb_ring
*rx_ring
,
7833 struct igb_rx_buffer
*rx_buffer
)
7835 if (igb_can_reuse_rx_page(rx_buffer
)) {
7836 /* hand second half of page back to the ring */
7837 igb_reuse_rx_page(rx_ring
, rx_buffer
);
7839 /* We are not reusing the buffer so unmap it and free
7840 * any references we are holding to it
7842 dma_unmap_page_attrs(rx_ring
->dev
, rx_buffer
->dma
,
7843 igb_rx_pg_size(rx_ring
), DMA_FROM_DEVICE
,
7845 __page_frag_cache_drain(rx_buffer
->page
,
7846 rx_buffer
->pagecnt_bias
);
7849 /* clear contents of rx_buffer */
7850 rx_buffer
->page
= NULL
;
7853 static int igb_clean_rx_irq(struct igb_q_vector
*q_vector
, const int budget
)
7855 struct igb_ring
*rx_ring
= q_vector
->rx
.ring
;
7856 struct sk_buff
*skb
= rx_ring
->skb
;
7857 unsigned int total_bytes
= 0, total_packets
= 0;
7858 u16 cleaned_count
= igb_desc_unused(rx_ring
);
7860 while (likely(total_packets
< budget
)) {
7861 union e1000_adv_rx_desc
*rx_desc
;
7862 struct igb_rx_buffer
*rx_buffer
;
7865 /* return some buffers to hardware, one at a time is too slow */
7866 if (cleaned_count
>= IGB_RX_BUFFER_WRITE
) {
7867 igb_alloc_rx_buffers(rx_ring
, cleaned_count
);
7871 rx_desc
= IGB_RX_DESC(rx_ring
, rx_ring
->next_to_clean
);
7872 size
= le16_to_cpu(rx_desc
->wb
.upper
.length
);
7876 /* This memory barrier is needed to keep us from reading
7877 * any other fields out of the rx_desc until we know the
7878 * descriptor has been written back
7882 rx_buffer
= igb_get_rx_buffer(rx_ring
, size
);
7884 /* retrieve a buffer from the ring */
7886 igb_add_rx_frag(rx_ring
, rx_buffer
, skb
, size
);
7887 else if (ring_uses_build_skb(rx_ring
))
7888 skb
= igb_build_skb(rx_ring
, rx_buffer
, rx_desc
, size
);
7890 skb
= igb_construct_skb(rx_ring
, rx_buffer
,
7893 /* exit if we failed to retrieve a buffer */
7895 rx_ring
->rx_stats
.alloc_failed
++;
7896 rx_buffer
->pagecnt_bias
++;
7900 igb_put_rx_buffer(rx_ring
, rx_buffer
);
7903 /* fetch next buffer in frame if non-eop */
7904 if (igb_is_non_eop(rx_ring
, rx_desc
))
7907 /* verify the packet layout is correct */
7908 if (igb_cleanup_headers(rx_ring
, rx_desc
, skb
)) {
7913 /* probably a little skewed due to removing CRC */
7914 total_bytes
+= skb
->len
;
7916 /* populate checksum, timestamp, VLAN, and protocol */
7917 igb_process_skb_fields(rx_ring
, rx_desc
, skb
);
7919 napi_gro_receive(&q_vector
->napi
, skb
);
7921 /* reset skb pointer */
7924 /* update budget accounting */
7928 /* place incomplete frames back on ring for completion */
7931 u64_stats_update_begin(&rx_ring
->rx_syncp
);
7932 rx_ring
->rx_stats
.packets
+= total_packets
;
7933 rx_ring
->rx_stats
.bytes
+= total_bytes
;
7934 u64_stats_update_end(&rx_ring
->rx_syncp
);
7935 q_vector
->rx
.total_packets
+= total_packets
;
7936 q_vector
->rx
.total_bytes
+= total_bytes
;
7939 igb_alloc_rx_buffers(rx_ring
, cleaned_count
);
7941 return total_packets
;
7944 static inline unsigned int igb_rx_offset(struct igb_ring
*rx_ring
)
7946 return ring_uses_build_skb(rx_ring
) ? IGB_SKB_PAD
: 0;
7949 static bool igb_alloc_mapped_page(struct igb_ring
*rx_ring
,
7950 struct igb_rx_buffer
*bi
)
7952 struct page
*page
= bi
->page
;
7955 /* since we are recycling buffers we should seldom need to alloc */
7959 /* alloc new page for storage */
7960 page
= dev_alloc_pages(igb_rx_pg_order(rx_ring
));
7961 if (unlikely(!page
)) {
7962 rx_ring
->rx_stats
.alloc_failed
++;
7966 /* map page for use */
7967 dma
= dma_map_page_attrs(rx_ring
->dev
, page
, 0,
7968 igb_rx_pg_size(rx_ring
),
7972 /* if mapping failed free memory back to system since
7973 * there isn't much point in holding memory we can't use
7975 if (dma_mapping_error(rx_ring
->dev
, dma
)) {
7976 __free_pages(page
, igb_rx_pg_order(rx_ring
));
7978 rx_ring
->rx_stats
.alloc_failed
++;
7984 bi
->page_offset
= igb_rx_offset(rx_ring
);
7985 bi
->pagecnt_bias
= 1;
7991 * igb_alloc_rx_buffers - Replace used receive buffers; packet split
7992 * @adapter: address of board private structure
7994 void igb_alloc_rx_buffers(struct igb_ring
*rx_ring
, u16 cleaned_count
)
7996 union e1000_adv_rx_desc
*rx_desc
;
7997 struct igb_rx_buffer
*bi
;
7998 u16 i
= rx_ring
->next_to_use
;
8005 rx_desc
= IGB_RX_DESC(rx_ring
, i
);
8006 bi
= &rx_ring
->rx_buffer_info
[i
];
8007 i
-= rx_ring
->count
;
8009 bufsz
= igb_rx_bufsz(rx_ring
);
8012 if (!igb_alloc_mapped_page(rx_ring
, bi
))
8015 /* sync the buffer for use by the device */
8016 dma_sync_single_range_for_device(rx_ring
->dev
, bi
->dma
,
8017 bi
->page_offset
, bufsz
,
8020 /* Refresh the desc even if buffer_addrs didn't change
8021 * because each write-back erases this info.
8023 rx_desc
->read
.pkt_addr
= cpu_to_le64(bi
->dma
+ bi
->page_offset
);
8029 rx_desc
= IGB_RX_DESC(rx_ring
, 0);
8030 bi
= rx_ring
->rx_buffer_info
;
8031 i
-= rx_ring
->count
;
8034 /* clear the length for the next_to_use descriptor */
8035 rx_desc
->wb
.upper
.length
= 0;
8038 } while (cleaned_count
);
8040 i
+= rx_ring
->count
;
8042 if (rx_ring
->next_to_use
!= i
) {
8043 /* record the next descriptor to use */
8044 rx_ring
->next_to_use
= i
;
8046 /* update next to alloc since we have filled the ring */
8047 rx_ring
->next_to_alloc
= i
;
8049 /* Force memory writes to complete before letting h/w
8050 * know there are new descriptors to fetch. (Only
8051 * applicable for weak-ordered memory model archs,
8055 writel(i
, rx_ring
->tail
);
8065 static int igb_mii_ioctl(struct net_device
*netdev
, struct ifreq
*ifr
, int cmd
)
8067 struct igb_adapter
*adapter
= netdev_priv(netdev
);
8068 struct mii_ioctl_data
*data
= if_mii(ifr
);
8070 if (adapter
->hw
.phy
.media_type
!= e1000_media_type_copper
)
8075 data
->phy_id
= adapter
->hw
.phy
.addr
;
8078 if (igb_read_phy_reg(&adapter
->hw
, data
->reg_num
& 0x1F,
8095 static int igb_ioctl(struct net_device
*netdev
, struct ifreq
*ifr
, int cmd
)
8101 return igb_mii_ioctl(netdev
, ifr
, cmd
);
8103 return igb_ptp_get_ts_config(netdev
, ifr
);
8105 return igb_ptp_set_ts_config(netdev
, ifr
);
8111 void igb_read_pci_cfg(struct e1000_hw
*hw
, u32 reg
, u16
*value
)
8113 struct igb_adapter
*adapter
= hw
->back
;
8115 pci_read_config_word(adapter
->pdev
, reg
, value
);
8118 void igb_write_pci_cfg(struct e1000_hw
*hw
, u32 reg
, u16
*value
)
8120 struct igb_adapter
*adapter
= hw
->back
;
8122 pci_write_config_word(adapter
->pdev
, reg
, *value
);
8125 s32
igb_read_pcie_cap_reg(struct e1000_hw
*hw
, u32 reg
, u16
*value
)
8127 struct igb_adapter
*adapter
= hw
->back
;
8129 if (pcie_capability_read_word(adapter
->pdev
, reg
, value
))
8130 return -E1000_ERR_CONFIG
;
8135 s32
igb_write_pcie_cap_reg(struct e1000_hw
*hw
, u32 reg
, u16
*value
)
8137 struct igb_adapter
*adapter
= hw
->back
;
8139 if (pcie_capability_write_word(adapter
->pdev
, reg
, *value
))
8140 return -E1000_ERR_CONFIG
;
8145 static void igb_vlan_mode(struct net_device
*netdev
, netdev_features_t features
)
8147 struct igb_adapter
*adapter
= netdev_priv(netdev
);
8148 struct e1000_hw
*hw
= &adapter
->hw
;
8150 bool enable
= !!(features
& NETIF_F_HW_VLAN_CTAG_RX
);
8153 /* enable VLAN tag insert/strip */
8154 ctrl
= rd32(E1000_CTRL
);
8155 ctrl
|= E1000_CTRL_VME
;
8156 wr32(E1000_CTRL
, ctrl
);
8158 /* Disable CFI check */
8159 rctl
= rd32(E1000_RCTL
);
8160 rctl
&= ~E1000_RCTL_CFIEN
;
8161 wr32(E1000_RCTL
, rctl
);
8163 /* disable VLAN tag insert/strip */
8164 ctrl
= rd32(E1000_CTRL
);
8165 ctrl
&= ~E1000_CTRL_VME
;
8166 wr32(E1000_CTRL
, ctrl
);
8169 igb_set_vf_vlan_strip(adapter
, adapter
->vfs_allocated_count
, enable
);
8172 static int igb_vlan_rx_add_vid(struct net_device
*netdev
,
8173 __be16 proto
, u16 vid
)
8175 struct igb_adapter
*adapter
= netdev_priv(netdev
);
8176 struct e1000_hw
*hw
= &adapter
->hw
;
8177 int pf_id
= adapter
->vfs_allocated_count
;
8179 /* add the filter since PF can receive vlans w/o entry in vlvf */
8180 if (!vid
|| !(adapter
->flags
& IGB_FLAG_VLAN_PROMISC
))
8181 igb_vfta_set(hw
, vid
, pf_id
, true, !!vid
);
8183 set_bit(vid
, adapter
->active_vlans
);
8188 static int igb_vlan_rx_kill_vid(struct net_device
*netdev
,
8189 __be16 proto
, u16 vid
)
8191 struct igb_adapter
*adapter
= netdev_priv(netdev
);
8192 int pf_id
= adapter
->vfs_allocated_count
;
8193 struct e1000_hw
*hw
= &adapter
->hw
;
8195 /* remove VID from filter table */
8196 if (vid
&& !(adapter
->flags
& IGB_FLAG_VLAN_PROMISC
))
8197 igb_vfta_set(hw
, vid
, pf_id
, false, true);
8199 clear_bit(vid
, adapter
->active_vlans
);
8204 static void igb_restore_vlan(struct igb_adapter
*adapter
)
8208 igb_vlan_mode(adapter
->netdev
, adapter
->netdev
->features
);
8209 igb_vlan_rx_add_vid(adapter
->netdev
, htons(ETH_P_8021Q
), 0);
8211 for_each_set_bit_from(vid
, adapter
->active_vlans
, VLAN_N_VID
)
8212 igb_vlan_rx_add_vid(adapter
->netdev
, htons(ETH_P_8021Q
), vid
);
8215 int igb_set_spd_dplx(struct igb_adapter
*adapter
, u32 spd
, u8 dplx
)
8217 struct pci_dev
*pdev
= adapter
->pdev
;
8218 struct e1000_mac_info
*mac
= &adapter
->hw
.mac
;
8222 /* Make sure dplx is at most 1 bit and lsb of speed is not set
8223 * for the switch() below to work
8225 if ((spd
& 1) || (dplx
& ~1))
8228 /* Fiber NIC's only allow 1000 gbps Full duplex
8229 * and 100Mbps Full duplex for 100baseFx sfp
8231 if (adapter
->hw
.phy
.media_type
== e1000_media_type_internal_serdes
) {
8232 switch (spd
+ dplx
) {
8233 case SPEED_10
+ DUPLEX_HALF
:
8234 case SPEED_10
+ DUPLEX_FULL
:
8235 case SPEED_100
+ DUPLEX_HALF
:
8242 switch (spd
+ dplx
) {
8243 case SPEED_10
+ DUPLEX_HALF
:
8244 mac
->forced_speed_duplex
= ADVERTISE_10_HALF
;
8246 case SPEED_10
+ DUPLEX_FULL
:
8247 mac
->forced_speed_duplex
= ADVERTISE_10_FULL
;
8249 case SPEED_100
+ DUPLEX_HALF
:
8250 mac
->forced_speed_duplex
= ADVERTISE_100_HALF
;
8252 case SPEED_100
+ DUPLEX_FULL
:
8253 mac
->forced_speed_duplex
= ADVERTISE_100_FULL
;
8255 case SPEED_1000
+ DUPLEX_FULL
:
8257 adapter
->hw
.phy
.autoneg_advertised
= ADVERTISE_1000_FULL
;
8259 case SPEED_1000
+ DUPLEX_HALF
: /* not supported */
8264 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
8265 adapter
->hw
.phy
.mdix
= AUTO_ALL_MODES
;
8270 dev_err(&pdev
->dev
, "Unsupported Speed/Duplex configuration\n");
8274 static int __igb_shutdown(struct pci_dev
*pdev
, bool *enable_wake
,
8277 struct net_device
*netdev
= pci_get_drvdata(pdev
);
8278 struct igb_adapter
*adapter
= netdev_priv(netdev
);
8279 struct e1000_hw
*hw
= &adapter
->hw
;
8280 u32 ctrl
, rctl
, status
;
8281 u32 wufc
= runtime
? E1000_WUFC_LNKC
: adapter
->wol
;
8287 netif_device_detach(netdev
);
8289 if (netif_running(netdev
))
8290 __igb_close(netdev
, true);
8292 igb_ptp_suspend(adapter
);
8294 igb_clear_interrupt_scheme(adapter
);
8298 retval
= pci_save_state(pdev
);
8303 status
= rd32(E1000_STATUS
);
8304 if (status
& E1000_STATUS_LU
)
8305 wufc
&= ~E1000_WUFC_LNKC
;
8308 igb_setup_rctl(adapter
);
8309 igb_set_rx_mode(netdev
);
8311 /* turn on all-multi mode if wake on multicast is enabled */
8312 if (wufc
& E1000_WUFC_MC
) {
8313 rctl
= rd32(E1000_RCTL
);
8314 rctl
|= E1000_RCTL_MPE
;
8315 wr32(E1000_RCTL
, rctl
);
8318 ctrl
= rd32(E1000_CTRL
);
8319 /* advertise wake from D3Cold */
8320 #define E1000_CTRL_ADVD3WUC 0x00100000
8321 /* phy power management enable */
8322 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
8323 ctrl
|= E1000_CTRL_ADVD3WUC
;
8324 wr32(E1000_CTRL
, ctrl
);
8326 /* Allow time for pending master requests to run */
8327 igb_disable_pcie_master(hw
);
8329 wr32(E1000_WUC
, E1000_WUC_PME_EN
);
8330 wr32(E1000_WUFC
, wufc
);
8333 wr32(E1000_WUFC
, 0);
8336 *enable_wake
= wufc
|| adapter
->en_mng_pt
;
8338 igb_power_down_link(adapter
);
8340 igb_power_up_link(adapter
);
8342 /* Release control of h/w to f/w. If f/w is AMT enabled, this
8343 * would have already happened in close and is redundant.
8345 igb_release_hw_control(adapter
);
8347 pci_disable_device(pdev
);
8352 static void igb_deliver_wake_packet(struct net_device
*netdev
)
8354 struct igb_adapter
*adapter
= netdev_priv(netdev
);
8355 struct e1000_hw
*hw
= &adapter
->hw
;
8356 struct sk_buff
*skb
;
8359 wupl
= rd32(E1000_WUPL
) & E1000_WUPL_MASK
;
8361 /* WUPM stores only the first 128 bytes of the wake packet.
8362 * Read the packet only if we have the whole thing.
8364 if ((wupl
== 0) || (wupl
> E1000_WUPM_BYTES
))
8367 skb
= netdev_alloc_skb_ip_align(netdev
, E1000_WUPM_BYTES
);
8373 /* Ensure reads are 32-bit aligned */
8374 wupl
= roundup(wupl
, 4);
8376 memcpy_fromio(skb
->data
, hw
->hw_addr
+ E1000_WUPM_REG(0), wupl
);
8378 skb
->protocol
= eth_type_trans(skb
, netdev
);
8382 static int __maybe_unused
igb_suspend(struct device
*dev
)
8386 struct pci_dev
*pdev
= to_pci_dev(dev
);
8388 retval
= __igb_shutdown(pdev
, &wake
, 0);
8393 pci_prepare_to_sleep(pdev
);
8395 pci_wake_from_d3(pdev
, false);
8396 pci_set_power_state(pdev
, PCI_D3hot
);
8402 static int __maybe_unused
igb_resume(struct device
*dev
)
8404 struct pci_dev
*pdev
= to_pci_dev(dev
);
8405 struct net_device
*netdev
= pci_get_drvdata(pdev
);
8406 struct igb_adapter
*adapter
= netdev_priv(netdev
);
8407 struct e1000_hw
*hw
= &adapter
->hw
;
8410 pci_set_power_state(pdev
, PCI_D0
);
8411 pci_restore_state(pdev
);
8412 pci_save_state(pdev
);
8414 if (!pci_device_is_present(pdev
))
8416 err
= pci_enable_device_mem(pdev
);
8419 "igb: Cannot enable PCI device from suspend\n");
8422 pci_set_master(pdev
);
8424 pci_enable_wake(pdev
, PCI_D3hot
, 0);
8425 pci_enable_wake(pdev
, PCI_D3cold
, 0);
8427 if (igb_init_interrupt_scheme(adapter
, true)) {
8428 dev_err(&pdev
->dev
, "Unable to allocate memory for queues\n");
8434 /* let the f/w know that the h/w is now under the control of the
8437 igb_get_hw_control(adapter
);
8439 val
= rd32(E1000_WUS
);
8440 if (val
& WAKE_PKT_WUS
)
8441 igb_deliver_wake_packet(netdev
);
8443 wr32(E1000_WUS
, ~0);
8446 if (!err
&& netif_running(netdev
))
8447 err
= __igb_open(netdev
, true);
8450 netif_device_attach(netdev
);
8456 static int __maybe_unused
igb_runtime_idle(struct device
*dev
)
8458 struct pci_dev
*pdev
= to_pci_dev(dev
);
8459 struct net_device
*netdev
= pci_get_drvdata(pdev
);
8460 struct igb_adapter
*adapter
= netdev_priv(netdev
);
8462 if (!igb_has_link(adapter
))
8463 pm_schedule_suspend(dev
, MSEC_PER_SEC
* 5);
8468 static int __maybe_unused
igb_runtime_suspend(struct device
*dev
)
8470 struct pci_dev
*pdev
= to_pci_dev(dev
);
8474 retval
= __igb_shutdown(pdev
, &wake
, 1);
8479 pci_prepare_to_sleep(pdev
);
8481 pci_wake_from_d3(pdev
, false);
8482 pci_set_power_state(pdev
, PCI_D3hot
);
8488 static int __maybe_unused
igb_runtime_resume(struct device
*dev
)
8490 return igb_resume(dev
);
8493 static void igb_shutdown(struct pci_dev
*pdev
)
8497 __igb_shutdown(pdev
, &wake
, 0);
8499 if (system_state
== SYSTEM_POWER_OFF
) {
8500 pci_wake_from_d3(pdev
, wake
);
8501 pci_set_power_state(pdev
, PCI_D3hot
);
8505 #ifdef CONFIG_PCI_IOV
8506 static int igb_sriov_reinit(struct pci_dev
*dev
)
8508 struct net_device
*netdev
= pci_get_drvdata(dev
);
8509 struct igb_adapter
*adapter
= netdev_priv(netdev
);
8510 struct pci_dev
*pdev
= adapter
->pdev
;
8514 if (netif_running(netdev
))
8519 igb_clear_interrupt_scheme(adapter
);
8521 igb_init_queue_configuration(adapter
);
8523 if (igb_init_interrupt_scheme(adapter
, true)) {
8525 dev_err(&pdev
->dev
, "Unable to allocate memory for queues\n");
8529 if (netif_running(netdev
))
8537 static int igb_pci_disable_sriov(struct pci_dev
*dev
)
8539 int err
= igb_disable_sriov(dev
);
8542 err
= igb_sriov_reinit(dev
);
8547 static int igb_pci_enable_sriov(struct pci_dev
*dev
, int num_vfs
)
8549 int err
= igb_enable_sriov(dev
, num_vfs
);
8554 err
= igb_sriov_reinit(dev
);
8563 static int igb_pci_sriov_configure(struct pci_dev
*dev
, int num_vfs
)
8565 #ifdef CONFIG_PCI_IOV
8567 return igb_pci_disable_sriov(dev
);
8569 return igb_pci_enable_sriov(dev
, num_vfs
);
8574 #ifdef CONFIG_NET_POLL_CONTROLLER
8575 /* Polling 'interrupt' - used by things like netconsole to send skbs
8576 * without having to re-enable interrupts. It's not called while
8577 * the interrupt routine is executing.
8579 static void igb_netpoll(struct net_device
*netdev
)
8581 struct igb_adapter
*adapter
= netdev_priv(netdev
);
8582 struct e1000_hw
*hw
= &adapter
->hw
;
8583 struct igb_q_vector
*q_vector
;
8586 for (i
= 0; i
< adapter
->num_q_vectors
; i
++) {
8587 q_vector
= adapter
->q_vector
[i
];
8588 if (adapter
->flags
& IGB_FLAG_HAS_MSIX
)
8589 wr32(E1000_EIMC
, q_vector
->eims_value
);
8591 igb_irq_disable(adapter
);
8592 napi_schedule(&q_vector
->napi
);
8595 #endif /* CONFIG_NET_POLL_CONTROLLER */
8598 * igb_io_error_detected - called when PCI error is detected
8599 * @pdev: Pointer to PCI device
8600 * @state: The current pci connection state
8602 * This function is called after a PCI bus error affecting
8603 * this device has been detected.
8605 static pci_ers_result_t
igb_io_error_detected(struct pci_dev
*pdev
,
8606 pci_channel_state_t state
)
8608 struct net_device
*netdev
= pci_get_drvdata(pdev
);
8609 struct igb_adapter
*adapter
= netdev_priv(netdev
);
8611 netif_device_detach(netdev
);
8613 if (state
== pci_channel_io_perm_failure
)
8614 return PCI_ERS_RESULT_DISCONNECT
;
8616 if (netif_running(netdev
))
8618 pci_disable_device(pdev
);
8620 /* Request a slot slot reset. */
8621 return PCI_ERS_RESULT_NEED_RESET
;
8625 * igb_io_slot_reset - called after the pci bus has been reset.
8626 * @pdev: Pointer to PCI device
8628 * Restart the card from scratch, as if from a cold-boot. Implementation
8629 * resembles the first-half of the igb_resume routine.
8631 static pci_ers_result_t
igb_io_slot_reset(struct pci_dev
*pdev
)
8633 struct net_device
*netdev
= pci_get_drvdata(pdev
);
8634 struct igb_adapter
*adapter
= netdev_priv(netdev
);
8635 struct e1000_hw
*hw
= &adapter
->hw
;
8636 pci_ers_result_t result
;
8639 if (pci_enable_device_mem(pdev
)) {
8641 "Cannot re-enable PCI device after reset.\n");
8642 result
= PCI_ERS_RESULT_DISCONNECT
;
8644 pci_set_master(pdev
);
8645 pci_restore_state(pdev
);
8646 pci_save_state(pdev
);
8648 pci_enable_wake(pdev
, PCI_D3hot
, 0);
8649 pci_enable_wake(pdev
, PCI_D3cold
, 0);
8651 /* In case of PCI error, adapter lose its HW address
8652 * so we should re-assign it here.
8654 hw
->hw_addr
= adapter
->io_addr
;
8657 wr32(E1000_WUS
, ~0);
8658 result
= PCI_ERS_RESULT_RECOVERED
;
8661 err
= pci_cleanup_aer_uncorrect_error_status(pdev
);
8664 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
8666 /* non-fatal, continue */
8673 * igb_io_resume - called when traffic can start flowing again.
8674 * @pdev: Pointer to PCI device
8676 * This callback is called when the error recovery driver tells us that
8677 * its OK to resume normal operation. Implementation resembles the
8678 * second-half of the igb_resume routine.
8680 static void igb_io_resume(struct pci_dev
*pdev
)
8682 struct net_device
*netdev
= pci_get_drvdata(pdev
);
8683 struct igb_adapter
*adapter
= netdev_priv(netdev
);
8685 if (netif_running(netdev
)) {
8686 if (igb_up(adapter
)) {
8687 dev_err(&pdev
->dev
, "igb_up failed after reset\n");
8692 netif_device_attach(netdev
);
8694 /* let the f/w know that the h/w is now under the control of the
8697 igb_get_hw_control(adapter
);
8701 * igb_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table
8702 * @adapter: Pointer to adapter structure
8703 * @index: Index of the RAR entry which need to be synced with MAC table
8705 static void igb_rar_set_index(struct igb_adapter
*adapter
, u32 index
)
8707 struct e1000_hw
*hw
= &adapter
->hw
;
8708 u32 rar_low
, rar_high
;
8709 u8
*addr
= adapter
->mac_table
[index
].addr
;
8711 /* HW expects these to be in network order when they are plugged
8712 * into the registers which are little endian. In order to guarantee
8713 * that ordering we need to do an leXX_to_cpup here in order to be
8714 * ready for the byteswap that occurs with writel
8716 rar_low
= le32_to_cpup((__le32
*)(addr
));
8717 rar_high
= le16_to_cpup((__le16
*)(addr
+ 4));
8719 /* Indicate to hardware the Address is Valid. */
8720 if (adapter
->mac_table
[index
].state
& IGB_MAC_STATE_IN_USE
) {
8721 rar_high
|= E1000_RAH_AV
;
8723 if (hw
->mac
.type
== e1000_82575
)
8724 rar_high
|= E1000_RAH_POOL_1
*
8725 adapter
->mac_table
[index
].queue
;
8727 rar_high
|= E1000_RAH_POOL_1
<<
8728 adapter
->mac_table
[index
].queue
;
8731 wr32(E1000_RAL(index
), rar_low
);
8733 wr32(E1000_RAH(index
), rar_high
);
8737 static int igb_set_vf_mac(struct igb_adapter
*adapter
,
8738 int vf
, unsigned char *mac_addr
)
8740 struct e1000_hw
*hw
= &adapter
->hw
;
8741 /* VF MAC addresses start at end of receive addresses and moves
8742 * towards the first, as a result a collision should not be possible
8744 int rar_entry
= hw
->mac
.rar_entry_count
- (vf
+ 1);
8745 unsigned char *vf_mac_addr
= adapter
->vf_data
[vf
].vf_mac_addresses
;
8747 ether_addr_copy(vf_mac_addr
, mac_addr
);
8748 ether_addr_copy(adapter
->mac_table
[rar_entry
].addr
, mac_addr
);
8749 adapter
->mac_table
[rar_entry
].queue
= vf
;
8750 adapter
->mac_table
[rar_entry
].state
|= IGB_MAC_STATE_IN_USE
;
8751 igb_rar_set_index(adapter
, rar_entry
);
8756 static int igb_ndo_set_vf_mac(struct net_device
*netdev
, int vf
, u8
*mac
)
8758 struct igb_adapter
*adapter
= netdev_priv(netdev
);
8759 if (!is_valid_ether_addr(mac
) || (vf
>= adapter
->vfs_allocated_count
))
8761 adapter
->vf_data
[vf
].flags
|= IGB_VF_FLAG_PF_SET_MAC
;
8762 dev_info(&adapter
->pdev
->dev
, "setting MAC %pM on VF %d\n", mac
, vf
);
8763 dev_info(&adapter
->pdev
->dev
,
8764 "Reload the VF driver to make this change effective.");
8765 if (test_bit(__IGB_DOWN
, &adapter
->state
)) {
8766 dev_warn(&adapter
->pdev
->dev
,
8767 "The VF MAC address has been set, but the PF device is not up.\n");
8768 dev_warn(&adapter
->pdev
->dev
,
8769 "Bring the PF device up before attempting to use the VF device.\n");
8771 return igb_set_vf_mac(adapter
, vf
, mac
);
8774 static int igb_link_mbps(int internal_link_speed
)
8776 switch (internal_link_speed
) {
8786 static void igb_set_vf_rate_limit(struct e1000_hw
*hw
, int vf
, int tx_rate
,
8793 /* Calculate the rate factor values to set */
8794 rf_int
= link_speed
/ tx_rate
;
8795 rf_dec
= (link_speed
- (rf_int
* tx_rate
));
8796 rf_dec
= (rf_dec
* BIT(E1000_RTTBCNRC_RF_INT_SHIFT
)) /
8799 bcnrc_val
= E1000_RTTBCNRC_RS_ENA
;
8800 bcnrc_val
|= ((rf_int
<< E1000_RTTBCNRC_RF_INT_SHIFT
) &
8801 E1000_RTTBCNRC_RF_INT_MASK
);
8802 bcnrc_val
|= (rf_dec
& E1000_RTTBCNRC_RF_DEC_MASK
);
8807 wr32(E1000_RTTDQSEL
, vf
); /* vf X uses queue X */
8808 /* Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
8809 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
8811 wr32(E1000_RTTBCNRM
, 0x14);
8812 wr32(E1000_RTTBCNRC
, bcnrc_val
);
8815 static void igb_check_vf_rate_limit(struct igb_adapter
*adapter
)
8817 int actual_link_speed
, i
;
8818 bool reset_rate
= false;
8820 /* VF TX rate limit was not set or not supported */
8821 if ((adapter
->vf_rate_link_speed
== 0) ||
8822 (adapter
->hw
.mac
.type
!= e1000_82576
))
8825 actual_link_speed
= igb_link_mbps(adapter
->link_speed
);
8826 if (actual_link_speed
!= adapter
->vf_rate_link_speed
) {
8828 adapter
->vf_rate_link_speed
= 0;
8829 dev_info(&adapter
->pdev
->dev
,
8830 "Link speed has been changed. VF Transmit rate is disabled\n");
8833 for (i
= 0; i
< adapter
->vfs_allocated_count
; i
++) {
8835 adapter
->vf_data
[i
].tx_rate
= 0;
8837 igb_set_vf_rate_limit(&adapter
->hw
, i
,
8838 adapter
->vf_data
[i
].tx_rate
,
8843 static int igb_ndo_set_vf_bw(struct net_device
*netdev
, int vf
,
8844 int min_tx_rate
, int max_tx_rate
)
8846 struct igb_adapter
*adapter
= netdev_priv(netdev
);
8847 struct e1000_hw
*hw
= &adapter
->hw
;
8848 int actual_link_speed
;
8850 if (hw
->mac
.type
!= e1000_82576
)
8856 actual_link_speed
= igb_link_mbps(adapter
->link_speed
);
8857 if ((vf
>= adapter
->vfs_allocated_count
) ||
8858 (!(rd32(E1000_STATUS
) & E1000_STATUS_LU
)) ||
8859 (max_tx_rate
< 0) ||
8860 (max_tx_rate
> actual_link_speed
))
8863 adapter
->vf_rate_link_speed
= actual_link_speed
;
8864 adapter
->vf_data
[vf
].tx_rate
= (u16
)max_tx_rate
;
8865 igb_set_vf_rate_limit(hw
, vf
, max_tx_rate
, actual_link_speed
);
8870 static int igb_ndo_set_vf_spoofchk(struct net_device
*netdev
, int vf
,
8873 struct igb_adapter
*adapter
= netdev_priv(netdev
);
8874 struct e1000_hw
*hw
= &adapter
->hw
;
8875 u32 reg_val
, reg_offset
;
8877 if (!adapter
->vfs_allocated_count
)
8880 if (vf
>= adapter
->vfs_allocated_count
)
8883 reg_offset
= (hw
->mac
.type
== e1000_82576
) ? E1000_DTXSWC
: E1000_TXSWC
;
8884 reg_val
= rd32(reg_offset
);
8886 reg_val
|= (BIT(vf
) |
8887 BIT(vf
+ E1000_DTXSWC_VLAN_SPOOF_SHIFT
));
8889 reg_val
&= ~(BIT(vf
) |
8890 BIT(vf
+ E1000_DTXSWC_VLAN_SPOOF_SHIFT
));
8891 wr32(reg_offset
, reg_val
);
8893 adapter
->vf_data
[vf
].spoofchk_enabled
= setting
;
8897 static int igb_ndo_get_vf_config(struct net_device
*netdev
,
8898 int vf
, struct ifla_vf_info
*ivi
)
8900 struct igb_adapter
*adapter
= netdev_priv(netdev
);
8901 if (vf
>= adapter
->vfs_allocated_count
)
8904 memcpy(&ivi
->mac
, adapter
->vf_data
[vf
].vf_mac_addresses
, ETH_ALEN
);
8905 ivi
->max_tx_rate
= adapter
->vf_data
[vf
].tx_rate
;
8906 ivi
->min_tx_rate
= 0;
8907 ivi
->vlan
= adapter
->vf_data
[vf
].pf_vlan
;
8908 ivi
->qos
= adapter
->vf_data
[vf
].pf_qos
;
8909 ivi
->spoofchk
= adapter
->vf_data
[vf
].spoofchk_enabled
;
8913 static void igb_vmm_control(struct igb_adapter
*adapter
)
8915 struct e1000_hw
*hw
= &adapter
->hw
;
8918 switch (hw
->mac
.type
) {
8924 /* replication is not supported for 82575 */
8927 /* notify HW that the MAC is adding vlan tags */
8928 reg
= rd32(E1000_DTXCTL
);
8929 reg
|= E1000_DTXCTL_VLAN_ADDED
;
8930 wr32(E1000_DTXCTL
, reg
);
8933 /* enable replication vlan tag stripping */
8934 reg
= rd32(E1000_RPLOLR
);
8935 reg
|= E1000_RPLOLR_STRVLAN
;
8936 wr32(E1000_RPLOLR
, reg
);
8939 /* none of the above registers are supported by i350 */
8943 if (adapter
->vfs_allocated_count
) {
8944 igb_vmdq_set_loopback_pf(hw
, true);
8945 igb_vmdq_set_replication_pf(hw
, true);
8946 igb_vmdq_set_anti_spoofing_pf(hw
, true,
8947 adapter
->vfs_allocated_count
);
8949 igb_vmdq_set_loopback_pf(hw
, false);
8950 igb_vmdq_set_replication_pf(hw
, false);
8954 static void igb_init_dmac(struct igb_adapter
*adapter
, u32 pba
)
8956 struct e1000_hw
*hw
= &adapter
->hw
;
8960 if (hw
->mac
.type
> e1000_82580
) {
8961 if (adapter
->flags
& IGB_FLAG_DMAC
) {
8964 /* force threshold to 0. */
8965 wr32(E1000_DMCTXTH
, 0);
8967 /* DMA Coalescing high water mark needs to be greater
8968 * than the Rx threshold. Set hwm to PBA - max frame
8969 * size in 16B units, capping it at PBA - 6KB.
8971 hwm
= 64 * (pba
- 6);
8972 reg
= rd32(E1000_FCRTC
);
8973 reg
&= ~E1000_FCRTC_RTH_COAL_MASK
;
8974 reg
|= ((hwm
<< E1000_FCRTC_RTH_COAL_SHIFT
)
8975 & E1000_FCRTC_RTH_COAL_MASK
);
8976 wr32(E1000_FCRTC
, reg
);
8978 /* Set the DMA Coalescing Rx threshold to PBA - 2 * max
8979 * frame size, capping it at PBA - 10KB.
8981 dmac_thr
= pba
- 10;
8982 reg
= rd32(E1000_DMACR
);
8983 reg
&= ~E1000_DMACR_DMACTHR_MASK
;
8984 reg
|= ((dmac_thr
<< E1000_DMACR_DMACTHR_SHIFT
)
8985 & E1000_DMACR_DMACTHR_MASK
);
8987 /* transition to L0x or L1 if available..*/
8988 reg
|= (E1000_DMACR_DMAC_EN
| E1000_DMACR_DMAC_LX_MASK
);
8990 /* watchdog timer= +-1000 usec in 32usec intervals */
8993 /* Disable BMC-to-OS Watchdog Enable */
8994 if (hw
->mac
.type
!= e1000_i354
)
8995 reg
&= ~E1000_DMACR_DC_BMC2OSW_EN
;
8997 wr32(E1000_DMACR
, reg
);
8999 /* no lower threshold to disable
9000 * coalescing(smart fifb)-UTRESH=0
9002 wr32(E1000_DMCRTRH
, 0);
9004 reg
= (IGB_DMCTLX_DCFLUSH_DIS
| 0x4);
9006 wr32(E1000_DMCTLX
, reg
);
9008 /* free space in tx packet buffer to wake from
9011 wr32(E1000_DMCTXTH
, (IGB_MIN_TXPBSIZE
-
9012 (IGB_TX_BUF_4096
+ adapter
->max_frame_size
)) >> 6);
9014 /* make low power state decision controlled
9017 reg
= rd32(E1000_PCIEMISC
);
9018 reg
&= ~E1000_PCIEMISC_LX_DECISION
;
9019 wr32(E1000_PCIEMISC
, reg
);
9020 } /* endif adapter->dmac is not disabled */
9021 } else if (hw
->mac
.type
== e1000_82580
) {
9022 u32 reg
= rd32(E1000_PCIEMISC
);
9024 wr32(E1000_PCIEMISC
, reg
& ~E1000_PCIEMISC_LX_DECISION
);
9025 wr32(E1000_DMACR
, 0);
9030 * igb_read_i2c_byte - Reads 8 bit word over I2C
9031 * @hw: pointer to hardware structure
9032 * @byte_offset: byte offset to read
9033 * @dev_addr: device address
9036 * Performs byte read operation over I2C interface at
9037 * a specified device address.
9039 s32
igb_read_i2c_byte(struct e1000_hw
*hw
, u8 byte_offset
,
9040 u8 dev_addr
, u8
*data
)
9042 struct igb_adapter
*adapter
= container_of(hw
, struct igb_adapter
, hw
);
9043 struct i2c_client
*this_client
= adapter
->i2c_client
;
9048 return E1000_ERR_I2C
;
9050 swfw_mask
= E1000_SWFW_PHY0_SM
;
9052 if (hw
->mac
.ops
.acquire_swfw_sync(hw
, swfw_mask
))
9053 return E1000_ERR_SWFW_SYNC
;
9055 status
= i2c_smbus_read_byte_data(this_client
, byte_offset
);
9056 hw
->mac
.ops
.release_swfw_sync(hw
, swfw_mask
);
9059 return E1000_ERR_I2C
;
9067 * igb_write_i2c_byte - Writes 8 bit word over I2C
9068 * @hw: pointer to hardware structure
9069 * @byte_offset: byte offset to write
9070 * @dev_addr: device address
9071 * @data: value to write
9073 * Performs byte write operation over I2C interface at
9074 * a specified device address.
9076 s32
igb_write_i2c_byte(struct e1000_hw
*hw
, u8 byte_offset
,
9077 u8 dev_addr
, u8 data
)
9079 struct igb_adapter
*adapter
= container_of(hw
, struct igb_adapter
, hw
);
9080 struct i2c_client
*this_client
= adapter
->i2c_client
;
9082 u16 swfw_mask
= E1000_SWFW_PHY0_SM
;
9085 return E1000_ERR_I2C
;
9087 if (hw
->mac
.ops
.acquire_swfw_sync(hw
, swfw_mask
))
9088 return E1000_ERR_SWFW_SYNC
;
9089 status
= i2c_smbus_write_byte_data(this_client
, byte_offset
, data
);
9090 hw
->mac
.ops
.release_swfw_sync(hw
, swfw_mask
);
9093 return E1000_ERR_I2C
;
9099 int igb_reinit_queues(struct igb_adapter
*adapter
)
9101 struct net_device
*netdev
= adapter
->netdev
;
9102 struct pci_dev
*pdev
= adapter
->pdev
;
9105 if (netif_running(netdev
))
9108 igb_reset_interrupt_capability(adapter
);
9110 if (igb_init_interrupt_scheme(adapter
, true)) {
9111 dev_err(&pdev
->dev
, "Unable to allocate memory for queues\n");
9115 if (netif_running(netdev
))
9116 err
= igb_open(netdev
);
9121 static void igb_nfc_filter_exit(struct igb_adapter
*adapter
)
9123 struct igb_nfc_filter
*rule
;
9125 spin_lock(&adapter
->nfc_lock
);
9127 hlist_for_each_entry(rule
, &adapter
->nfc_filter_list
, nfc_node
)
9128 igb_erase_filter(adapter
, rule
);
9130 spin_unlock(&adapter
->nfc_lock
);
9133 static void igb_nfc_filter_restore(struct igb_adapter
*adapter
)
9135 struct igb_nfc_filter
*rule
;
9137 spin_lock(&adapter
->nfc_lock
);
9139 hlist_for_each_entry(rule
, &adapter
->nfc_filter_list
, nfc_node
)
9140 igb_add_filter(adapter
, rule
);
9142 spin_unlock(&adapter
->nfc_lock
);