]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - drivers/net/ethernet/intel/igb/igb_main.c
projects / mirror_ubuntu-zesty-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
igb: Fix stats output on i210/i211 parts.
[mirror_ubuntu-zesty-kernel.git] / drivers / net / ethernet / intel / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
6e861326 4 Copyright(c) 2007-2012 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
876d2d6f
JK		 28#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
29
9d5c8243
AK		 30#include <linux/module.h>
31#include <linux/types.h>
32#include <linux/init.h>
b2cb09b1 33#include <linux/bitops.h>
9d5c8243
AK		 34#include <linux/vmalloc.h>
35#include <linux/pagemap.h>
36#include <linux/netdevice.h>
9d5c8243 37#include <linux/ipv6.h>
5a0e3ad6 38#include <linux/slab.h>
9d5c8243
AK		 39#include <net/checksum.h>
40#include <net/ip6_checksum.h>
c6cb090b 41#include <linux/net_tstamp.h>
9d5c8243
AK		 42#include <linux/mii.h>
43#include <linux/ethtool.h>
01789349 44#include <linux/if.h>
9d5c8243
AK		 45#include <linux/if_vlan.h>
46#include <linux/pci.h>
c54106bb 47#include <linux/pci-aspm.h>
9d5c8243
AK		 48#include <linux/delay.h>
49#include <linux/interrupt.h>
7d13a7d0
AD		 50#include <linux/ip.h>
51#include <linux/tcp.h>
52#include <linux/sctp.h>
9d5c8243 53#include <linux/if_ether.h>
40a914fa 54#include <linux/aer.h>
70c71606 55#include <linux/prefetch.h>
749ab2cd 56#include <linux/pm_runtime.h>
421e02f0 57#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 58#include <linux/dca.h>
59#endif
9d5c8243
AK		 60#include "igb.h"
61
200e5fd5
CW		 62#define MAJ 4
63#define MIN 0
64#define BUILD 1
0d1fe82d 65#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
929dd047 66__stringify(BUILD) "-k"
9d5c8243
AK		 67char igb_driver_name[] = "igb";
68char igb_driver_version[] = DRV_VERSION;
69static const char igb_driver_string[] =
70 "Intel(R) Gigabit Ethernet Network Driver";
6e861326 71static const char igb_copyright[] = "Copyright (c) 2007-2012 Intel Corporation.";
9d5c8243 72
9d5c8243
AK		 73static const struct e1000_info *igb_info_tbl[] = {
74 [board_82575] = &e1000_82575_info,
75};
76
a3aa1884 77static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
f96a8a0b
CW		 78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 },
79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 },
81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 },
82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 },
d2ba2ed8
AD		 83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
55cac248
AD		 87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
6493d24f 89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
55cac248
AD		 90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
308fb39a
JG		 93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
1b5dda33
GJ		 95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
2d064c06 97 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 98 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 99 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 100 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
101 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 102 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
b894fa26 103 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
c8ea5ea9 104 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 105 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
106 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
107 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
108 /* required last entry */
109 {0, }
110};
111
112MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
113
114void igb_reset(struct igb_adapter *);
115static int igb_setup_all_tx_resources(struct igb_adapter *);
116static int igb_setup_all_rx_resources(struct igb_adapter *);
117static void igb_free_all_tx_resources(struct igb_adapter *);
118static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 119static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 120static int igb_probe(struct pci_dev *, const struct pci_device_id *);
121static void __devexit igb_remove(struct pci_dev *pdev);
122static int igb_sw_init(struct igb_adapter *);
123static int igb_open(struct net_device *);
124static int igb_close(struct net_device *);
125static void igb_configure_tx(struct igb_adapter *);
126static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 127static void igb_clean_all_tx_rings(struct igb_adapter *);
128static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 129static void igb_clean_tx_ring(struct igb_ring *);
130static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 131static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 132static void igb_update_phy_info(unsigned long);
133static void igb_watchdog(unsigned long);
134static void igb_watchdog_task(struct work_struct *);
cd392f5c 135static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
12dcd86b
ED		 136static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
137 struct rtnl_link_stats64 *stats);
9d5c8243
AK		 138static int igb_change_mtu(struct net_device *, int);
139static int igb_set_mac(struct net_device *, void *);
68d480c4 140static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 141static irqreturn_t igb_intr(int irq, void *);
142static irqreturn_t igb_intr_msi(int irq, void *);
143static irqreturn_t igb_msix_other(int irq, void *);
047e0030 144static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 145#ifdef CONFIG_IGB_DCA
047e0030 146static void igb_update_dca(struct igb_q_vector *);
fe4506b6 147static void igb_setup_dca(struct igb_adapter *);
421e02f0 148#endif /* CONFIG_IGB_DCA */
661086df 149static int igb_poll(struct napi_struct *, int);
13fde97a 150static bool igb_clean_tx_irq(struct igb_q_vector *);
cd392f5c 151static bool igb_clean_rx_irq(struct igb_q_vector *, int);
9d5c8243
AK		 152static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
153static void igb_tx_timeout(struct net_device *);
154static void igb_reset_task(struct work_struct *);
c8f44aff 155static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features);
8e586137
JP		 156static int igb_vlan_rx_add_vid(struct net_device *, u16);
157static int igb_vlan_rx_kill_vid(struct net_device *, u16);
9d5c8243 158static void igb_restore_vlan(struct igb_adapter *);
26ad9178 159static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 160static void igb_ping_all_vfs(struct igb_adapter *);
161static void igb_msg_task(struct igb_adapter *);
4ae196df 162static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 163static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 164static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 165static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
166static int igb_ndo_set_vf_vlan(struct net_device *netdev,
167 int vf, u16 vlan, u8 qos);
168static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
169static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
170 struct ifla_vf_info *ivi);
17dc566c 171static void igb_check_vf_rate_limit(struct igb_adapter *);
46a01698
RL		 172
173#ifdef CONFIG_PCI_IOV
0224d663 174static int igb_vf_configure(struct igb_adapter *adapter, int vf);
f557147c 175static bool igb_vfs_are_assigned(struct igb_adapter *adapter);
46a01698 176#endif
9d5c8243 177
9d5c8243 178#ifdef CONFIG_PM
d9dd966d 179#ifdef CONFIG_PM_SLEEP
749ab2cd 180static int igb_suspend(struct device *);
d9dd966d 181#endif
749ab2cd
YZ		 182static int igb_resume(struct device *);
183#ifdef CONFIG_PM_RUNTIME
184static int igb_runtime_suspend(struct device *dev);
185static int igb_runtime_resume(struct device *dev);
186static int igb_runtime_idle(struct device *dev);
187#endif
188static const struct dev_pm_ops igb_pm_ops = {
189 SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
190 SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
191 igb_runtime_idle)
192};
9d5c8243
AK		 193#endif
194static void igb_shutdown(struct pci_dev *);
421e02f0 195#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 196static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
197static struct notifier_block dca_notifier = {
198 .notifier_call = igb_notify_dca,
199 .next = NULL,
200 .priority = 0
201};
202#endif
9d5c8243
AK		 203#ifdef CONFIG_NET_POLL_CONTROLLER
204/* for netdump / net console */
205static void igb_netpoll(struct net_device *);
206#endif
37680117 207#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 208static unsigned int max_vfs = 0;
209module_param(max_vfs, uint, 0);
210MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
211 "per physical function");
212#endif /* CONFIG_PCI_IOV */
213
9d5c8243
AK		 214static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
215 pci_channel_state_t);
216static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
217static void igb_io_resume(struct pci_dev *);
218
219static struct pci_error_handlers igb_err_handler = {
220 .error_detected = igb_io_error_detected,
221 .slot_reset = igb_io_slot_reset,
222 .resume = igb_io_resume,
223};
224
b6e0c419 225static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
9d5c8243
AK		 226
227static struct pci_driver igb_driver = {
228 .name = igb_driver_name,
229 .id_table = igb_pci_tbl,
230 .probe = igb_probe,
231 .remove = __devexit_p(igb_remove),
232#ifdef CONFIG_PM
749ab2cd 233 .driver.pm = &igb_pm_ops,
9d5c8243
AK		 234#endif
235 .shutdown = igb_shutdown,
236 .err_handler = &igb_err_handler
237};
238
239MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
240MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
241MODULE_LICENSE("GPL");
242MODULE_VERSION(DRV_VERSION);
243
b3f4d599 244#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
245static int debug = -1;
246module_param(debug, int, 0);
247MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
248
c97ec42a
TI		 249struct igb_reg_info {
250 u32 ofs;
251 char *name;
252};
253
254static const struct igb_reg_info igb_reg_info_tbl[] = {
255
256 /* General Registers */
257 {E1000_CTRL, "CTRL"},
258 {E1000_STATUS, "STATUS"},
259 {E1000_CTRL_EXT, "CTRL_EXT"},
260
261 /* Interrupt Registers */
262 {E1000_ICR, "ICR"},
263
264 /* RX Registers */
265 {E1000_RCTL, "RCTL"},
266 {E1000_RDLEN(0), "RDLEN"},
267 {E1000_RDH(0), "RDH"},
268 {E1000_RDT(0), "RDT"},
269 {E1000_RXDCTL(0), "RXDCTL"},
270 {E1000_RDBAL(0), "RDBAL"},
271 {E1000_RDBAH(0), "RDBAH"},
272
273 /* TX Registers */
274 {E1000_TCTL, "TCTL"},
275 {E1000_TDBAL(0), "TDBAL"},
276 {E1000_TDBAH(0), "TDBAH"},
277 {E1000_TDLEN(0), "TDLEN"},
278 {E1000_TDH(0), "TDH"},
279 {E1000_TDT(0), "TDT"},
280 {E1000_TXDCTL(0), "TXDCTL"},
281 {E1000_TDFH, "TDFH"},
282 {E1000_TDFT, "TDFT"},
283 {E1000_TDFHS, "TDFHS"},
284 {E1000_TDFPC, "TDFPC"},
285
286 /* List Terminator */
287 {}
288};
289
290/*
291 * igb_regdump - register printout routine
292 */
293static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
294{
295 int n = 0;
296 char rname[16];
297 u32 regs[8];
298
299 switch (reginfo->ofs) {
300 case E1000_RDLEN(0):
301 for (n = 0; n < 4; n++)
302 regs[n] = rd32(E1000_RDLEN(n));
303 break;
304 case E1000_RDH(0):
305 for (n = 0; n < 4; n++)
306 regs[n] = rd32(E1000_RDH(n));
307 break;
308 case E1000_RDT(0):
309 for (n = 0; n < 4; n++)
310 regs[n] = rd32(E1000_RDT(n));
311 break;
312 case E1000_RXDCTL(0):
313 for (n = 0; n < 4; n++)
314 regs[n] = rd32(E1000_RXDCTL(n));
315 break;
316 case E1000_RDBAL(0):
317 for (n = 0; n < 4; n++)
318 regs[n] = rd32(E1000_RDBAL(n));
319 break;
320 case E1000_RDBAH(0):
321 for (n = 0; n < 4; n++)
322 regs[n] = rd32(E1000_RDBAH(n));
323 break;
324 case E1000_TDBAL(0):
325 for (n = 0; n < 4; n++)
326 regs[n] = rd32(E1000_RDBAL(n));
327 break;
328 case E1000_TDBAH(0):
329 for (n = 0; n < 4; n++)
330 regs[n] = rd32(E1000_TDBAH(n));
331 break;
332 case E1000_TDLEN(0):
333 for (n = 0; n < 4; n++)
334 regs[n] = rd32(E1000_TDLEN(n));
335 break;
336 case E1000_TDH(0):
337 for (n = 0; n < 4; n++)
338 regs[n] = rd32(E1000_TDH(n));
339 break;
340 case E1000_TDT(0):
341 for (n = 0; n < 4; n++)
342 regs[n] = rd32(E1000_TDT(n));
343 break;
344 case E1000_TXDCTL(0):
345 for (n = 0; n < 4; n++)
346 regs[n] = rd32(E1000_TXDCTL(n));
347 break;
348 default:
876d2d6f 349 pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs));
c97ec42a
TI		 350 return;
351 }
352
353 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
876d2d6f
JK		 354 pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1],
355 regs[2], regs[3]);
c97ec42a
TI		 356}
357
358/*
359 * igb_dump - Print registers, tx-rings and rx-rings
360 */
361static void igb_dump(struct igb_adapter *adapter)
362{
363 struct net_device *netdev = adapter->netdev;
364 struct e1000_hw *hw = &adapter->hw;
365 struct igb_reg_info *reginfo;
c97ec42a
TI		 366 struct igb_ring *tx_ring;
367 union e1000_adv_tx_desc *tx_desc;
368 struct my_u0 { u64 a; u64 b; } *u0;
c97ec42a
TI		 369 struct igb_ring *rx_ring;
370 union e1000_adv_rx_desc *rx_desc;
371 u32 staterr;
6ad4edfc 372 u16 i, n;
c97ec42a
TI		 373
374 if (!netif_msg_hw(adapter))
375 return;
376
377 /* Print netdevice Info */
378 if (netdev) {
379 dev_info(&adapter->pdev->dev, "Net device Info\n");
876d2d6f
JK		 380 pr_info("Device Name state trans_start "
381 "last_rx\n");
382 pr_info("%-15s %016lX %016lX %016lX\n", netdev->name,
383 netdev->state, netdev->trans_start, netdev->last_rx);
c97ec42a
TI		 384 }
385
386 /* Print Registers */
387 dev_info(&adapter->pdev->dev, "Register Dump\n");
876d2d6f 388 pr_info(" Register Name Value\n");
c97ec42a
TI		 389 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
390 reginfo->name; reginfo++) {
391 igb_regdump(hw, reginfo);
392 }
393
394 /* Print TX Ring Summary */
395 if (!netdev || !netif_running(netdev))
396 goto exit;
397
398 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
876d2d6f 399 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
c97ec42a 400 for (n = 0; n < adapter->num_tx_queues; n++) {
06034649 401 struct igb_tx_buffer *buffer_info;
c97ec42a 402 tx_ring = adapter->tx_ring[n];
06034649 403 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
876d2d6f
JK		 404 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
405 n, tx_ring->next_to_use, tx_ring->next_to_clean,
406 (u64)buffer_info->dma,
407 buffer_info->length,
408 buffer_info->next_to_watch,
409 (u64)buffer_info->time_stamp);
c97ec42a
TI		 410 }
411
412 /* Print TX Rings */
413 if (!netif_msg_tx_done(adapter))
414 goto rx_ring_summary;
415
416 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
417
418 /* Transmit Descriptor Formats
419 *
420 * Advanced Transmit Descriptor
421 * +--------------------------------------------------------------+
422 * 0 | Buffer Address [63:0] |
423 * +--------------------------------------------------------------+
424 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
425 * +--------------------------------------------------------------+
426 * 63 46 45 40 39 38 36 35 32 31 24 15 0
427 */
428
429 for (n = 0; n < adapter->num_tx_queues; n++) {
430 tx_ring = adapter->tx_ring[n];
876d2d6f
JK		 431 pr_info("------------------------------------\n");
432 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
433 pr_info("------------------------------------\n");
434 pr_info("T [desc] [address 63:0 ] [PlPOCIStDDM Ln] "
435 "[bi->dma ] leng ntw timestamp "
436 "bi->skb\n");
c97ec42a
TI		 437
438 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
876d2d6f 439 const char *next_desc;
06034649 440 struct igb_tx_buffer *buffer_info;
60136906 441 tx_desc = IGB_TX_DESC(tx_ring, i);
06034649 442 buffer_info = &tx_ring->tx_buffer_info[i];
c97ec42a 443 u0 = (struct my_u0 *)tx_desc;
876d2d6f
JK		 444 if (i == tx_ring->next_to_use &&
445 i == tx_ring->next_to_clean)
446 next_desc = " NTC/U";
447 else if (i == tx_ring->next_to_use)
448 next_desc = " NTU";
449 else if (i == tx_ring->next_to_clean)
450 next_desc = " NTC";
451 else
452 next_desc = "";
453
454 pr_info("T [0x%03X] %016llX %016llX %016llX"
455 " %04X %p %016llX %p%s\n", i,
c97ec42a
TI		 456 le64_to_cpu(u0->a),
457 le64_to_cpu(u0->b),
458 (u64)buffer_info->dma,
459 buffer_info->length,
460 buffer_info->next_to_watch,
461 (u64)buffer_info->time_stamp,
876d2d6f 462 buffer_info->skb, next_desc);
c97ec42a 463
b669588a 464 if (netif_msg_pktdata(adapter) && buffer_info->skb)
c97ec42a
TI		 465 print_hex_dump(KERN_INFO, "",
466 DUMP_PREFIX_ADDRESS,
b669588a 467 16, 1, buffer_info->skb->data,
c97ec42a
TI		 468 buffer_info->length, true);
469 }
470 }
471
472 /* Print RX Rings Summary */
473rx_ring_summary:
474 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
876d2d6f 475 pr_info("Queue [NTU] [NTC]\n");
c97ec42a
TI		 476 for (n = 0; n < adapter->num_rx_queues; n++) {
477 rx_ring = adapter->rx_ring[n];
876d2d6f
JK		 478 pr_info(" %5d %5X %5X\n",
479 n, rx_ring->next_to_use, rx_ring->next_to_clean);
c97ec42a
TI		 480 }
481
482 /* Print RX Rings */
483 if (!netif_msg_rx_status(adapter))
484 goto exit;
485
486 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
487
488 /* Advanced Receive Descriptor (Read) Format
489 * 63 1 0
490 * +-----------------------------------------------------+
491 * 0 | Packet Buffer Address [63:1] |A0/NSE|
492 * +----------------------------------------------+------+
493 * 8 | Header Buffer Address [63:1] | DD |
494 * +-----------------------------------------------------+
495 *
496 *
497 * Advanced Receive Descriptor (Write-Back) Format
498 *
499 * 63 48 47 32 31 30 21 20 17 16 4 3 0
500 * +------------------------------------------------------+
501 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
502 * | Checksum Ident | | | | Type | Type |
503 * +------------------------------------------------------+
504 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
505 * +------------------------------------------------------+
506 * 63 48 47 32 31 20 19 0
507 */
508
509 for (n = 0; n < adapter->num_rx_queues; n++) {
510 rx_ring = adapter->rx_ring[n];
876d2d6f
JK		 511 pr_info("------------------------------------\n");
512 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
513 pr_info("------------------------------------\n");
514 pr_info("R [desc] [ PktBuf A0] [ HeadBuf DD] "
515 "[bi->dma ] [bi->skb] <-- Adv Rx Read format\n");
516 pr_info("RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] -----"
517 "----------- [bi->skb] <-- Adv Rx Write-Back format\n");
c97ec42a
TI		 518
519 for (i = 0; i < rx_ring->count; i++) {
876d2d6f 520 const char *next_desc;
06034649
AD		 521 struct igb_rx_buffer *buffer_info;
522 buffer_info = &rx_ring->rx_buffer_info[i];
60136906 523 rx_desc = IGB_RX_DESC(rx_ring, i);
c97ec42a
TI		 524 u0 = (struct my_u0 *)rx_desc;
525 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
876d2d6f
JK		 526
527 if (i == rx_ring->next_to_use)
528 next_desc = " NTU";
529 else if (i == rx_ring->next_to_clean)
530 next_desc = " NTC";
531 else
532 next_desc = "";
533
c97ec42a
TI		 534 if (staterr & E1000_RXD_STAT_DD) {
535 /* Descriptor Done */
876d2d6f
JK		 536 pr_info("%s[0x%03X] %016llX %016llX -------"
537 "--------- %p%s\n", "RWB", i,
c97ec42a
TI		 538 le64_to_cpu(u0->a),
539 le64_to_cpu(u0->b),
876d2d6f 540 buffer_info->skb, next_desc);
c97ec42a 541 } else {
876d2d6f
JK		 542 pr_info("%s[0x%03X] %016llX %016llX %016llX"
543 " %p%s\n", "R ", i,
c97ec42a
TI		 544 le64_to_cpu(u0->a),
545 le64_to_cpu(u0->b),
546 (u64)buffer_info->dma,
876d2d6f 547 buffer_info->skb, next_desc);
c97ec42a 548
b669588a
ET		 549 if (netif_msg_pktdata(adapter) &&
550 buffer_info->dma && buffer_info->skb) {
c97ec42a 551 print_hex_dump(KERN_INFO, "",
b669588a
ET		 552 DUMP_PREFIX_ADDRESS,
553 16, 1, buffer_info->skb->data,
554 IGB_RX_HDR_LEN, true);
44390ca6
AD		 555 print_hex_dump(KERN_INFO, "",
556 DUMP_PREFIX_ADDRESS,
557 16, 1,
b669588a
ET		 558 page_address(buffer_info->page) +
559 buffer_info->page_offset,
44390ca6 560 PAGE_SIZE/2, true);
c97ec42a
TI		 561 }
562 }
c97ec42a
TI		 563 }
564 }
565
566exit:
567 return;
568}
569
9d5c8243 570/**
c041076a 571 * igb_get_hw_dev - return device
9d5c8243
AK		 572 * used by hardware layer to print debugging information
573 **/
c041076a 574struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
9d5c8243
AK		 575{
576 struct igb_adapter *adapter = hw->back;
c041076a 577 return adapter->netdev;
9d5c8243 578}
38c845c7 579
9d5c8243
AK		 580/**
581 * igb_init_module - Driver Registration Routine
582 *
583 * igb_init_module is the first routine called when the driver is
584 * loaded. All it does is register with the PCI subsystem.
585 **/
586static int __init igb_init_module(void)
587{
588 int ret;
876d2d6f 589 pr_info("%s - version %s\n",
9d5c8243
AK		 590 igb_driver_string, igb_driver_version);
591
876d2d6f 592 pr_info("%s\n", igb_copyright);
9d5c8243 593
421e02f0 594#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 595 dca_register_notify(&dca_notifier);
596#endif
bbd98fe4 597 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 598 return ret;
599}
600
601module_init(igb_init_module);
602
603/**
604 * igb_exit_module - Driver Exit Cleanup Routine
605 *
606 * igb_exit_module is called just before the driver is removed
607 * from memory.
608 **/
609static void __exit igb_exit_module(void)
610{
421e02f0 611#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 612 dca_unregister_notify(&dca_notifier);
613#endif
9d5c8243
AK		 614 pci_unregister_driver(&igb_driver);
615}
616
617module_exit(igb_exit_module);
618
26bc19ec
AD		 619#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
620/**
621 * igb_cache_ring_register - Descriptor ring to register mapping
622 * @adapter: board private structure to initialize
623 *
624 * Once we know the feature-set enabled for the device, we'll cache
625 * the register offset the descriptor ring is assigned to.
626 **/
627static void igb_cache_ring_register(struct igb_adapter *adapter)
628{
ee1b9f06 629 int i = 0, j = 0;
047e0030 630 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 631
632 switch (adapter->hw.mac.type) {
633 case e1000_82576:
634 /* The queues are allocated for virtualization such that VF 0
635 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
636 * In order to avoid collision we start at the first free queue
637 * and continue consuming queues in the same sequence
638 */
ee1b9f06 639 if (adapter->vfs_allocated_count) {
a99955fc 640 for (; i < adapter->rss_queues; i++)
3025a446
AD		 641 adapter->rx_ring[i]->reg_idx = rbase_offset +
642 Q_IDX_82576(i);
ee1b9f06 643 }
26bc19ec 644 case e1000_82575:
55cac248 645 case e1000_82580:
d2ba2ed8 646 case e1000_i350:
f96a8a0b
CW		 647 case e1000_i210:
648 case e1000_i211:
26bc19ec 649 default:
ee1b9f06 650 for (; i < adapter->num_rx_queues; i++)
3025a446 651 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 652 for (; j < adapter->num_tx_queues; j++)
3025a446 653 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 654 break;
655 }
656}
657
047e0030
AD		 658static void igb_free_queues(struct igb_adapter *adapter)
659{
3025a446 660 int i;
047e0030 661
3025a446
AD		 662 for (i = 0; i < adapter->num_tx_queues; i++) {
663 kfree(adapter->tx_ring[i]);
664 adapter->tx_ring[i] = NULL;
665 }
666 for (i = 0; i < adapter->num_rx_queues; i++) {
667 kfree(adapter->rx_ring[i]);
668 adapter->rx_ring[i] = NULL;
669 }
047e0030
AD		 670 adapter->num_rx_queues = 0;
671 adapter->num_tx_queues = 0;
672}
673
9d5c8243
AK		 674/**
675 * igb_alloc_queues - Allocate memory for all rings
676 * @adapter: board private structure to initialize
677 *
678 * We allocate one ring per queue at run-time since we don't know the
679 * number of queues at compile-time.
680 **/
681static int igb_alloc_queues(struct igb_adapter *adapter)
682{
3025a446 683 struct igb_ring *ring;
9d5c8243 684 int i;
81c2fc22 685 int orig_node = adapter->node;
9d5c8243 686
661086df 687 for (i = 0; i < adapter->num_tx_queues; i++) {
81c2fc22
AD		 688 if (orig_node == -1) {
689 int cur_node = next_online_node(adapter->node);
690 if (cur_node == MAX_NUMNODES)
691 cur_node = first_online_node;
692 adapter->node = cur_node;
693 }
694 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
695 adapter->node);
696 if (!ring)
697 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
3025a446
AD		 698 if (!ring)
699 goto err;
68fd9910 700 ring->count = adapter->tx_ring_count;
661086df 701 ring->queue_index = i;
59d71989 702 ring->dev = &adapter->pdev->dev;
e694e964 703 ring->netdev = adapter->netdev;
81c2fc22 704 ring->numa_node = adapter->node;
85ad76b2
AD		 705 /* For 82575, context index must be unique per ring. */
706 if (adapter->hw.mac.type == e1000_82575)
866cff06 707 set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
3025a446 708 adapter->tx_ring[i] = ring;
661086df 709 }
81c2fc22
AD		 710 /* Restore the adapter's original node */
711 adapter->node = orig_node;
85ad76b2 712
9d5c8243 713 for (i = 0; i < adapter->num_rx_queues; i++) {
81c2fc22
AD		 714 if (orig_node == -1) {
715 int cur_node = next_online_node(adapter->node);
716 if (cur_node == MAX_NUMNODES)
717 cur_node = first_online_node;
718 adapter->node = cur_node;
719 }
720 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
721 adapter->node);
722 if (!ring)
723 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
3025a446
AD		 724 if (!ring)
725 goto err;
68fd9910 726 ring->count = adapter->rx_ring_count;
844290e5 727 ring->queue_index = i;
59d71989 728 ring->dev = &adapter->pdev->dev;
e694e964 729 ring->netdev = adapter->netdev;
81c2fc22 730 ring->numa_node = adapter->node;
85ad76b2
AD		 731 /* set flag indicating ring supports SCTP checksum offload */
732 if (adapter->hw.mac.type >= e1000_82576)
866cff06 733 set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
8be10e91 734
f96a8a0b
CW		 735 /*
736 * On i350, i210, and i211, loopback VLAN packets
737 * have the tag byte-swapped.
738 * */
739 if (adapter->hw.mac.type >= e1000_i350)
8be10e91
AD		 740 set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
741
3025a446 742 adapter->rx_ring[i] = ring;
9d5c8243 743 }
81c2fc22
AD		 744 /* Restore the adapter's original node */
745 adapter->node = orig_node;
26bc19ec
AD		 746
747 igb_cache_ring_register(adapter);
9d5c8243 748
047e0030 749 return 0;
a88f10ec 750
047e0030 751err:
81c2fc22
AD		 752 /* Restore the adapter's original node */
753 adapter->node = orig_node;
047e0030 754 igb_free_queues(adapter);
d1a8c9e1 755
047e0030 756 return -ENOMEM;
a88f10ec
AD		 757}
758
4be000c8
AD		 759/**
760 * igb_write_ivar - configure ivar for given MSI-X vector
761 * @hw: pointer to the HW structure
762 * @msix_vector: vector number we are allocating to a given ring
763 * @index: row index of IVAR register to write within IVAR table
764 * @offset: column offset of in IVAR, should be multiple of 8
765 *
766 * This function is intended to handle the writing of the IVAR register
767 * for adapters 82576 and newer. The IVAR table consists of 2 columns,
768 * each containing an cause allocation for an Rx and Tx ring, and a
769 * variable number of rows depending on the number of queues supported.
770 **/
771static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
772 int index, int offset)
773{
774 u32 ivar = array_rd32(E1000_IVAR0, index);
775
776 /* clear any bits that are currently set */
777 ivar &= ~((u32)0xFF << offset);
778
779 /* write vector and valid bit */
780 ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
781
782 array_wr32(E1000_IVAR0, index, ivar);
783}
784
9d5c8243 785#define IGB_N0_QUEUE -1
047e0030 786static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243 787{
047e0030 788 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 789 struct e1000_hw *hw = &adapter->hw;
047e0030
AD		 790 int rx_queue = IGB_N0_QUEUE;
791 int tx_queue = IGB_N0_QUEUE;
4be000c8 792 u32 msixbm = 0;
047e0030 793
0ba82994
AD		 794 if (q_vector->rx.ring)
795 rx_queue = q_vector->rx.ring->reg_idx;
796 if (q_vector->tx.ring)
797 tx_queue = q_vector->tx.ring->reg_idx;
2d064c06
AD		 798
799 switch (hw->mac.type) {
800 case e1000_82575:
9d5c8243
AK		 801 /* The 82575 assigns vectors using a bitmask, which matches the
802 bitmask for the EICR/EIMS/EIMC registers. To assign one
803 or more queues to a vector, we write the appropriate bits
804 into the MSIXBM register for that vector. */
047e0030 805 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 806 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 807 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 808 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 809 if (!adapter->msix_entries && msix_vector == 0)
810 msixbm |= E1000_EIMS_OTHER;
9d5c8243 811 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 812 q_vector->eims_value = msixbm;
2d064c06
AD		 813 break;
814 case e1000_82576:
4be000c8
AD		 815 /*
816 * 82576 uses a table that essentially consists of 2 columns
817 * with 8 rows. The ordering is column-major so we use the
818 * lower 3 bits as the row index, and the 4th bit as the
819 * column offset.
820 */
821 if (rx_queue > IGB_N0_QUEUE)
822 igb_write_ivar(hw, msix_vector,
823 rx_queue & 0x7,
824 (rx_queue & 0x8) << 1);
825 if (tx_queue > IGB_N0_QUEUE)
826 igb_write_ivar(hw, msix_vector,
827 tx_queue & 0x7,
828 ((tx_queue & 0x8) << 1) + 8);
047e0030 829 q_vector->eims_value = 1 << msix_vector;
2d064c06 830 break;
55cac248 831 case e1000_82580:
d2ba2ed8 832 case e1000_i350:
f96a8a0b
CW		 833 case e1000_i210:
834 case e1000_i211:
4be000c8
AD		 835 /*
836 * On 82580 and newer adapters the scheme is similar to 82576
837 * however instead of ordering column-major we have things
838 * ordered row-major. So we traverse the table by using
839 * bit 0 as the column offset, and the remaining bits as the
840 * row index.
841 */
842 if (rx_queue > IGB_N0_QUEUE)
843 igb_write_ivar(hw, msix_vector,
844 rx_queue >> 1,
845 (rx_queue & 0x1) << 4);
846 if (tx_queue > IGB_N0_QUEUE)
847 igb_write_ivar(hw, msix_vector,
848 tx_queue >> 1,
849 ((tx_queue & 0x1) << 4) + 8);
55cac248
AD		 850 q_vector->eims_value = 1 << msix_vector;
851 break;
2d064c06
AD		 852 default:
853 BUG();
854 break;
855 }
26b39276
AD		 856
857 /* add q_vector eims value to global eims_enable_mask */
858 adapter->eims_enable_mask |= q_vector->eims_value;
859
860 /* configure q_vector to set itr on first interrupt */
861 q_vector->set_itr = 1;
9d5c8243
AK		 862}
863
864/**
865 * igb_configure_msix - Configure MSI-X hardware
866 *
867 * igb_configure_msix sets up the hardware to properly
868 * generate MSI-X interrupts.
869 **/
870static void igb_configure_msix(struct igb_adapter *adapter)
871{
872 u32 tmp;
873 int i, vector = 0;
874 struct e1000_hw *hw = &adapter->hw;
875
876 adapter->eims_enable_mask = 0;
9d5c8243
AK		 877
878 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 879 switch (hw->mac.type) {
880 case e1000_82575:
9d5c8243
AK		 881 tmp = rd32(E1000_CTRL_EXT);
882 /* enable MSI-X PBA support*/
883 tmp |= E1000_CTRL_EXT_PBA_CLR;
884
885 /* Auto-Mask interrupts upon ICR read. */
886 tmp |= E1000_CTRL_EXT_EIAME;
887 tmp |= E1000_CTRL_EXT_IRCA;
888
889 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 890
891 /* enable msix_other interrupt */
892 array_wr32(E1000_MSIXBM(0), vector++,
893 E1000_EIMS_OTHER);
844290e5 894 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 895
2d064c06
AD		 896 break;
897
898 case e1000_82576:
55cac248 899 case e1000_82580:
d2ba2ed8 900 case e1000_i350:
f96a8a0b
CW		 901 case e1000_i210:
902 case e1000_i211:
047e0030
AD		 903 /* Turn on MSI-X capability first, or our settings
904 * won't stick. And it will take days to debug. */
905 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
906 E1000_GPIE_PBA | E1000_GPIE_EIAME |
907 E1000_GPIE_NSICR);
908
909 /* enable msix_other interrupt */
910 adapter->eims_other = 1 << vector;
2d064c06 911 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 912
047e0030 913 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 914 break;
915 default:
916 /* do nothing, since nothing else supports MSI-X */
917 break;
918 } /* switch (hw->mac.type) */
047e0030
AD		 919
920 adapter->eims_enable_mask |= adapter->eims_other;
921
26b39276
AD		 922 for (i = 0; i < adapter->num_q_vectors; i++)
923 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 924
9d5c8243
AK		 925 wrfl();
926}
927
928/**
929 * igb_request_msix - Initialize MSI-X interrupts
930 *
931 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
932 * kernel.
933 **/
934static int igb_request_msix(struct igb_adapter *adapter)
935{
936 struct net_device *netdev = adapter->netdev;
047e0030 937 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 938 int i, err = 0, vector = 0;
939
047e0030 940 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 941 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 942 if (err)
943 goto out;
944 vector++;
945
946 for (i = 0; i < adapter->num_q_vectors; i++) {
947 struct igb_q_vector *q_vector = adapter->q_vector[i];
948
949 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
950
0ba82994 951 if (q_vector->rx.ring && q_vector->tx.ring)
047e0030 952 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
0ba82994
AD		 953 q_vector->rx.ring->queue_index);
954 else if (q_vector->tx.ring)
047e0030 955 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
0ba82994
AD		 956 q_vector->tx.ring->queue_index);
957 else if (q_vector->rx.ring)
047e0030 958 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
0ba82994 959 q_vector->rx.ring->queue_index);
9d5c8243 960 else
047e0030
AD		 961 sprintf(q_vector->name, "%s-unused", netdev->name);
962
9d5c8243 963 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 964 igb_msix_ring, 0, q_vector->name,
047e0030 965 q_vector);
9d5c8243
AK		 966 if (err)
967 goto out;
9d5c8243
AK		 968 vector++;
969 }
970
9d5c8243
AK		 971 igb_configure_msix(adapter);
972 return 0;
973out:
974 return err;
975}
976
977static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
978{
979 if (adapter->msix_entries) {
980 pci_disable_msix(adapter->pdev);
981 kfree(adapter->msix_entries);
982 adapter->msix_entries = NULL;
047e0030 983 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 984 pci_disable_msi(adapter->pdev);
047e0030 985 }
9d5c8243
AK		 986}
987
047e0030
AD		 988/**
989 * igb_free_q_vectors - Free memory allocated for interrupt vectors
990 * @adapter: board private structure to initialize
991 *
992 * This function frees the memory allocated to the q_vectors. In addition if
993 * NAPI is enabled it will delete any references to the NAPI struct prior
994 * to freeing the q_vector.
995 **/
996static void igb_free_q_vectors(struct igb_adapter *adapter)
997{
998 int v_idx;
999
1000 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1001 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1002 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 1003 if (!q_vector)
1004 continue;
047e0030
AD		 1005 netif_napi_del(&q_vector->napi);
1006 kfree(q_vector);
1007 }
1008 adapter->num_q_vectors = 0;
1009}
1010
1011/**
1012 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
1013 *
1014 * This function resets the device so that it has 0 rx queues, tx queues, and
1015 * MSI-X interrupts allocated.
1016 */
1017static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
1018{
1019 igb_free_queues(adapter);
1020 igb_free_q_vectors(adapter);
1021 igb_reset_interrupt_capability(adapter);
1022}
9d5c8243
AK		 1023
1024/**
1025 * igb_set_interrupt_capability - set MSI or MSI-X if supported
1026 *
1027 * Attempt to configure interrupts using the best available
1028 * capabilities of the hardware and kernel.
1029 **/
21adef3e 1030static int igb_set_interrupt_capability(struct igb_adapter *adapter)
9d5c8243
AK		 1031{
1032 int err;
1033 int numvecs, i;
1034
83b7180d 1035 /* Number of supported queues. */
a99955fc 1036 adapter->num_rx_queues = adapter->rss_queues;
5fa8517f
GR		 1037 if (adapter->vfs_allocated_count)
1038 adapter->num_tx_queues = 1;
1039 else
1040 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 1041
047e0030
AD		 1042 /* start with one vector for every rx queue */
1043 numvecs = adapter->num_rx_queues;
1044
3ad2f3fb 1045 /* if tx handler is separate add 1 for every tx queue */
a99955fc
AD		 1046 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1047 numvecs += adapter->num_tx_queues;
047e0030
AD		 1048
1049 /* store the number of vectors reserved for queues */
1050 adapter->num_q_vectors = numvecs;
1051
1052 /* add 1 vector for link status interrupts */
1053 numvecs++;
9d5c8243
AK		 1054 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1055 GFP_KERNEL);
f96a8a0b 1056
9d5c8243
AK		 1057 if (!adapter->msix_entries)
1058 goto msi_only;
1059
1060 for (i = 0; i < numvecs; i++)
1061 adapter->msix_entries[i].entry = i;
1062
1063 err = pci_enable_msix(adapter->pdev,
1064 adapter->msix_entries,
1065 numvecs);
1066 if (err == 0)
34a20e89 1067 goto out;
9d5c8243
AK		 1068
1069 igb_reset_interrupt_capability(adapter);
1070
1071 /* If we can't do MSI-X, try MSI */
1072msi_only:
2a3abf6d
AD		 1073#ifdef CONFIG_PCI_IOV
1074 /* disable SR-IOV for non MSI-X configurations */
1075 if (adapter->vf_data) {
1076 struct e1000_hw *hw = &adapter->hw;
1077 /* disable iov and allow time for transactions to clear */
1078 pci_disable_sriov(adapter->pdev);
1079 msleep(500);
1080
1081 kfree(adapter->vf_data);
1082 adapter->vf_data = NULL;
1083 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 1084 wrfl();
2a3abf6d
AD		 1085 msleep(100);
1086 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1087 }
1088#endif
4fc82adf 1089 adapter->vfs_allocated_count = 0;
a99955fc 1090 adapter->rss_queues = 1;
4fc82adf 1091 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 1092 adapter->num_rx_queues = 1;
661086df 1093 adapter->num_tx_queues = 1;
047e0030 1094 adapter->num_q_vectors = 1;
9d5c8243 1095 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1096 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 1097out:
21adef3e 1098 /* Notify the stack of the (possibly) reduced queue counts. */
cfb8c3aa 1099 rtnl_lock();
21adef3e 1100 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
cfb8c3aa
BP		 1101 err = netif_set_real_num_rx_queues(adapter->netdev,
1102 adapter->num_rx_queues);
1103 rtnl_unlock();
1104 return err;
9d5c8243
AK		 1105}
1106
047e0030
AD		 1107/**
1108 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1109 * @adapter: board private structure to initialize
1110 *
1111 * We allocate one q_vector per queue interrupt. If allocation fails we
1112 * return -ENOMEM.
1113 **/
1114static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1115{
1116 struct igb_q_vector *q_vector;
1117 struct e1000_hw *hw = &adapter->hw;
1118 int v_idx;
81c2fc22 1119 int orig_node = adapter->node;
047e0030
AD		 1120
1121 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
81c2fc22
AD		 1122 if ((adapter->num_q_vectors == (adapter->num_rx_queues +
1123 adapter->num_tx_queues)) &&
1124 (adapter->num_rx_queues == v_idx))
1125 adapter->node = orig_node;
1126 if (orig_node == -1) {
1127 int cur_node = next_online_node(adapter->node);
1128 if (cur_node == MAX_NUMNODES)
1129 cur_node = first_online_node;
1130 adapter->node = cur_node;
1131 }
1132 q_vector = kzalloc_node(sizeof(struct igb_q_vector), GFP_KERNEL,
1133 adapter->node);
1134 if (!q_vector)
1135 q_vector = kzalloc(sizeof(struct igb_q_vector),
1136 GFP_KERNEL);
047e0030
AD		 1137 if (!q_vector)
1138 goto err_out;
1139 q_vector->adapter = adapter;
047e0030
AD		 1140 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1141 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1142 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1143 adapter->q_vector[v_idx] = q_vector;
1144 }
81c2fc22
AD		 1145 /* Restore the adapter's original node */
1146 adapter->node = orig_node;
1147
047e0030
AD		 1148 return 0;
1149
1150err_out:
81c2fc22
AD		 1151 /* Restore the adapter's original node */
1152 adapter->node = orig_node;
fe0592b4 1153 igb_free_q_vectors(adapter);
047e0030
AD		 1154 return -ENOMEM;
1155}
1156
1157static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1158 int ring_idx, int v_idx)
1159{
3025a446 1160 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1161
0ba82994
AD		 1162 q_vector->rx.ring = adapter->rx_ring[ring_idx];
1163 q_vector->rx.ring->q_vector = q_vector;
1164 q_vector->rx.count++;
4fc82adf
AD		 1165 q_vector->itr_val = adapter->rx_itr_setting;
1166 if (q_vector->itr_val && q_vector->itr_val <= 3)
1167 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1168}
1169
1170static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1171 int ring_idx, int v_idx)
1172{
3025a446 1173 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1174
0ba82994
AD		 1175 q_vector->tx.ring = adapter->tx_ring[ring_idx];
1176 q_vector->tx.ring->q_vector = q_vector;
1177 q_vector->tx.count++;
4fc82adf 1178 q_vector->itr_val = adapter->tx_itr_setting;
0ba82994 1179 q_vector->tx.work_limit = adapter->tx_work_limit;
4fc82adf
AD		 1180 if (q_vector->itr_val && q_vector->itr_val <= 3)
1181 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1182}
1183
1184/**
1185 * igb_map_ring_to_vector - maps allocated queues to vectors
1186 *
1187 * This function maps the recently allocated queues to vectors.
1188 **/
1189static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1190{
1191 int i;
1192 int v_idx = 0;
1193
1194 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1195 (adapter->num_q_vectors < adapter->num_tx_queues))
1196 return -ENOMEM;
1197
1198 if (adapter->num_q_vectors >=
1199 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1200 for (i = 0; i < adapter->num_rx_queues; i++)
1201 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1202 for (i = 0; i < adapter->num_tx_queues; i++)
1203 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1204 } else {
1205 for (i = 0; i < adapter->num_rx_queues; i++) {
1206 if (i < adapter->num_tx_queues)
1207 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1208 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1209 }
1210 for (; i < adapter->num_tx_queues; i++)
1211 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1212 }
1213 return 0;
1214}
1215
1216/**
1217 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1218 *
1219 * This function initializes the interrupts and allocates all of the queues.
1220 **/
1221static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1222{
1223 struct pci_dev *pdev = adapter->pdev;
1224 int err;
1225
21adef3e
BH		 1226 err = igb_set_interrupt_capability(adapter);
1227 if (err)
1228 return err;
047e0030
AD		 1229
1230 err = igb_alloc_q_vectors(adapter);
1231 if (err) {
1232 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1233 goto err_alloc_q_vectors;
1234 }
1235
1236 err = igb_alloc_queues(adapter);
1237 if (err) {
1238 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1239 goto err_alloc_queues;
1240 }
1241
1242 err = igb_map_ring_to_vector(adapter);
1243 if (err) {
1244 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1245 goto err_map_queues;
1246 }
1247
1248
1249 return 0;
1250err_map_queues:
1251 igb_free_queues(adapter);
1252err_alloc_queues:
1253 igb_free_q_vectors(adapter);
1254err_alloc_q_vectors:
1255 igb_reset_interrupt_capability(adapter);
1256 return err;
1257}
1258
9d5c8243
AK		 1259/**
1260 * igb_request_irq - initialize interrupts
1261 *
1262 * Attempts to configure interrupts using the best available
1263 * capabilities of the hardware and kernel.
1264 **/
1265static int igb_request_irq(struct igb_adapter *adapter)
1266{
1267 struct net_device *netdev = adapter->netdev;
047e0030 1268 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1269 int err = 0;
1270
1271 if (adapter->msix_entries) {
1272 err = igb_request_msix(adapter);
844290e5 1273 if (!err)
9d5c8243 1274 goto request_done;
9d5c8243 1275 /* fall back to MSI */
047e0030 1276 igb_clear_interrupt_scheme(adapter);
c74d588e 1277 if (!pci_enable_msi(pdev))
7dfc16fa 1278 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1279 igb_free_all_tx_resources(adapter);
1280 igb_free_all_rx_resources(adapter);
047e0030 1281 adapter->num_tx_queues = 1;
9d5c8243 1282 adapter->num_rx_queues = 1;
047e0030
AD		 1283 adapter->num_q_vectors = 1;
1284 err = igb_alloc_q_vectors(adapter);
1285 if (err) {
1286 dev_err(&pdev->dev,
1287 "Unable to allocate memory for vectors\n");
1288 goto request_done;
1289 }
1290 err = igb_alloc_queues(adapter);
1291 if (err) {
1292 dev_err(&pdev->dev,
1293 "Unable to allocate memory for queues\n");
1294 igb_free_q_vectors(adapter);
1295 goto request_done;
1296 }
1297 igb_setup_all_tx_resources(adapter);
1298 igb_setup_all_rx_resources(adapter);
9d5c8243 1299 }
844290e5 1300
c74d588e
AD		 1301 igb_assign_vector(adapter->q_vector[0], 0);
1302
7dfc16fa 1303 if (adapter->flags & IGB_FLAG_HAS_MSI) {
c74d588e 1304 err = request_irq(pdev->irq, igb_intr_msi, 0,
047e0030 1305 netdev->name, adapter);
9d5c8243
AK		 1306 if (!err)
1307 goto request_done;
047e0030 1308
9d5c8243
AK		 1309 /* fall back to legacy interrupts */
1310 igb_reset_interrupt_capability(adapter);
7dfc16fa 1311 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1312 }
1313
c74d588e 1314 err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
047e0030 1315 netdev->name, adapter);
9d5c8243 1316
6cb5e577 1317 if (err)
c74d588e 1318 dev_err(&pdev->dev, "Error %d getting interrupt\n",
9d5c8243 1319 err);
9d5c8243
AK		 1320
1321request_done:
1322 return err;
1323}
1324
1325static void igb_free_irq(struct igb_adapter *adapter)
1326{
9d5c8243
AK		 1327 if (adapter->msix_entries) {
1328 int vector = 0, i;
1329
047e0030 1330 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 1331
0d1ae7f4 1332 for (i = 0; i < adapter->num_q_vectors; i++)
047e0030 1333 free_irq(adapter->msix_entries[vector++].vector,
0d1ae7f4 1334 adapter->q_vector[i]);
047e0030
AD		 1335 } else {
1336 free_irq(adapter->pdev->irq, adapter);
9d5c8243 1337 }
9d5c8243
AK		 1338}
1339
1340/**
1341 * igb_irq_disable - Mask off interrupt generation on the NIC
1342 * @adapter: board private structure
1343 **/
1344static void igb_irq_disable(struct igb_adapter *adapter)
1345{
1346 struct e1000_hw *hw = &adapter->hw;
1347
25568a53
AD		 1348 /*
1349 * we need to be careful when disabling interrupts. The VFs are also
1350 * mapped into these registers and so clearing the bits can cause
1351 * issues on the VF drivers so we only need to clear what we set
1352 */
9d5c8243 1353 if (adapter->msix_entries) {
2dfd1212
AD		 1354 u32 regval = rd32(E1000_EIAM);
1355 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1356 wr32(E1000_EIMC, adapter->eims_enable_mask);
1357 regval = rd32(E1000_EIAC);
1358 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 1359 }
844290e5
PW		 1360
1361 wr32(E1000_IAM, 0);
9d5c8243
AK		 1362 wr32(E1000_IMC, ~0);
1363 wrfl();
81a61859
ET		 1364 if (adapter->msix_entries) {
1365 int i;
1366 for (i = 0; i < adapter->num_q_vectors; i++)
1367 synchronize_irq(adapter->msix_entries[i].vector);
1368 } else {
1369 synchronize_irq(adapter->pdev->irq);
1370 }
9d5c8243
AK		 1371}
1372
1373/**
1374 * igb_irq_enable - Enable default interrupt generation settings
1375 * @adapter: board private structure
1376 **/
1377static void igb_irq_enable(struct igb_adapter *adapter)
1378{
1379 struct e1000_hw *hw = &adapter->hw;
1380
1381 if (adapter->msix_entries) {
06218a8d 1382 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
2dfd1212
AD		 1383 u32 regval = rd32(E1000_EIAC);
1384 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1385 regval = rd32(E1000_EIAM);
1386 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 1387 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 1388 if (adapter->vfs_allocated_count) {
4ae196df 1389 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1390 ims |= E1000_IMS_VMMB;
1391 }
1392 wr32(E1000_IMS, ims);
844290e5 1393 } else {
55cac248
AD		 1394 wr32(E1000_IMS, IMS_ENABLE_MASK |
1395 E1000_IMS_DRSTA);
1396 wr32(E1000_IAM, IMS_ENABLE_MASK |
1397 E1000_IMS_DRSTA);
844290e5 1398 }
9d5c8243
AK		 1399}
1400
1401static void igb_update_mng_vlan(struct igb_adapter *adapter)
1402{
51466239 1403 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1404 u16 vid = adapter->hw.mng_cookie.vlan_id;
1405 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1406
1407 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1408 /* add VID to filter table */
1409 igb_vfta_set(hw, vid, true);
1410 adapter->mng_vlan_id = vid;
1411 } else {
1412 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1413 }
1414
1415 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1416 (vid != old_vid) &&
b2cb09b1 1417 !test_bit(old_vid, adapter->active_vlans)) {
51466239
AD		 1418 /* remove VID from filter table */
1419 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1420 }
1421}
1422
1423/**
1424 * igb_release_hw_control - release control of the h/w to f/w
1425 * @adapter: address of board private structure
1426 *
1427 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1428 * For ASF and Pass Through versions of f/w this means that the
1429 * driver is no longer loaded.
1430 *
1431 **/
1432static void igb_release_hw_control(struct igb_adapter *adapter)
1433{
1434 struct e1000_hw *hw = &adapter->hw;
1435 u32 ctrl_ext;
1436
1437 /* Let firmware take over control of h/w */
1438 ctrl_ext = rd32(E1000_CTRL_EXT);
1439 wr32(E1000_CTRL_EXT,
1440 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1441}
1442
9d5c8243
AK		 1443/**
1444 * igb_get_hw_control - get control of the h/w from f/w
1445 * @adapter: address of board private structure
1446 *
1447 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1448 * For ASF and Pass Through versions of f/w this means that
1449 * the driver is loaded.
1450 *
1451 **/
1452static void igb_get_hw_control(struct igb_adapter *adapter)
1453{
1454 struct e1000_hw *hw = &adapter->hw;
1455 u32 ctrl_ext;
1456
1457 /* Let firmware know the driver has taken over */
1458 ctrl_ext = rd32(E1000_CTRL_EXT);
1459 wr32(E1000_CTRL_EXT,
1460 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1461}
1462
9d5c8243
AK		 1463/**
1464 * igb_configure - configure the hardware for RX and TX
1465 * @adapter: private board structure
1466 **/
1467static void igb_configure(struct igb_adapter *adapter)
1468{
1469 struct net_device *netdev = adapter->netdev;
1470 int i;
1471
1472 igb_get_hw_control(adapter);
ff41f8dc 1473 igb_set_rx_mode(netdev);
9d5c8243
AK		 1474
1475 igb_restore_vlan(adapter);
9d5c8243 1476
85b430b4 1477 igb_setup_tctl(adapter);
06cf2666 1478 igb_setup_mrqc(adapter);
9d5c8243 1479 igb_setup_rctl(adapter);
85b430b4
AD		 1480
1481 igb_configure_tx(adapter);
9d5c8243 1482 igb_configure_rx(adapter);
662d7205
AD		 1483
1484 igb_rx_fifo_flush_82575(&adapter->hw);
1485
c493ea45 1486 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1487 * at least 1 descriptor unused to make sure
1488 * next_to_use != next_to_clean */
1489 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1490 struct igb_ring *ring = adapter->rx_ring[i];
cd392f5c 1491 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
9d5c8243 1492 }
9d5c8243
AK		 1493}
1494
88a268c1
NN		 1495/**
1496 * igb_power_up_link - Power up the phy/serdes link
1497 * @adapter: address of board private structure
1498 **/
1499void igb_power_up_link(struct igb_adapter *adapter)
1500{
76886596
AA		 1501 igb_reset_phy(&adapter->hw);
1502
88a268c1
NN		 1503 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1504 igb_power_up_phy_copper(&adapter->hw);
1505 else
1506 igb_power_up_serdes_link_82575(&adapter->hw);
1507}
1508
1509/**
1510 * igb_power_down_link - Power down the phy/serdes link
1511 * @adapter: address of board private structure
1512 */
1513static void igb_power_down_link(struct igb_adapter *adapter)
1514{
1515 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1516 igb_power_down_phy_copper_82575(&adapter->hw);
1517 else
1518 igb_shutdown_serdes_link_82575(&adapter->hw);
1519}
9d5c8243
AK		 1520
1521/**
1522 * igb_up - Open the interface and prepare it to handle traffic
1523 * @adapter: board private structure
1524 **/
9d5c8243
AK		 1525int igb_up(struct igb_adapter *adapter)
1526{
1527 struct e1000_hw *hw = &adapter->hw;
1528 int i;
1529
1530 /* hardware has been reset, we need to reload some things */
1531 igb_configure(adapter);
1532
1533 clear_bit(__IGB_DOWN, &adapter->state);
1534
0d1ae7f4
AD		 1535 for (i = 0; i < adapter->num_q_vectors; i++)
1536 napi_enable(&(adapter->q_vector[i]->napi));
1537
844290e5 1538 if (adapter->msix_entries)
9d5c8243 1539 igb_configure_msix(adapter);
feeb2721
AD		 1540 else
1541 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1542
1543 /* Clear any pending interrupts. */
1544 rd32(E1000_ICR);
1545 igb_irq_enable(adapter);
1546
d4960307
AD		 1547 /* notify VFs that reset has been completed */
1548 if (adapter->vfs_allocated_count) {
1549 u32 reg_data = rd32(E1000_CTRL_EXT);
1550 reg_data |= E1000_CTRL_EXT_PFRSTD;
1551 wr32(E1000_CTRL_EXT, reg_data);
1552 }
1553
4cb9be7a
JB		 1554 netif_tx_start_all_queues(adapter->netdev);
1555
25568a53
AD		 1556 /* start the watchdog. */
1557 hw->mac.get_link_status = 1;
1558 schedule_work(&adapter->watchdog_task);
1559
9d5c8243
AK		 1560 return 0;
1561}
1562
1563void igb_down(struct igb_adapter *adapter)
1564{
9d5c8243 1565 struct net_device *netdev = adapter->netdev;
330a6d6a 1566 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1567 u32 tctl, rctl;
1568 int i;
1569
1570 /* signal that we're down so the interrupt handler does not
1571 * reschedule our watchdog timer */
1572 set_bit(__IGB_DOWN, &adapter->state);
1573
1574 /* disable receives in the hardware */
1575 rctl = rd32(E1000_RCTL);
1576 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1577 /* flush and sleep below */
1578
fd2ea0a7 1579 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1580
1581 /* disable transmits in the hardware */
1582 tctl = rd32(E1000_TCTL);
1583 tctl &= ~E1000_TCTL_EN;
1584 wr32(E1000_TCTL, tctl);
1585 /* flush both disables and wait for them to finish */
1586 wrfl();
1587 msleep(10);
1588
0d1ae7f4
AD		 1589 for (i = 0; i < adapter->num_q_vectors; i++)
1590 napi_disable(&(adapter->q_vector[i]->napi));
9d5c8243 1591
9d5c8243
AK		 1592 igb_irq_disable(adapter);
1593
1594 del_timer_sync(&adapter->watchdog_timer);
1595 del_timer_sync(&adapter->phy_info_timer);
1596
9d5c8243 1597 netif_carrier_off(netdev);
04fe6358
AD		 1598
1599 /* record the stats before reset*/
12dcd86b
ED		 1600 spin_lock(&adapter->stats64_lock);
1601 igb_update_stats(adapter, &adapter->stats64);
1602 spin_unlock(&adapter->stats64_lock);
04fe6358 1603
9d5c8243
AK		 1604 adapter->link_speed = 0;
1605 adapter->link_duplex = 0;
1606
3023682e
JK		 1607 if (!pci_channel_offline(adapter->pdev))
1608 igb_reset(adapter);
9d5c8243
AK		 1609 igb_clean_all_tx_rings(adapter);
1610 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1611#ifdef CONFIG_IGB_DCA
1612
1613 /* since we reset the hardware DCA settings were cleared */
1614 igb_setup_dca(adapter);
1615#endif
9d5c8243
AK		 1616}
1617
1618void igb_reinit_locked(struct igb_adapter *adapter)
1619{
1620 WARN_ON(in_interrupt());
1621 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1622 msleep(1);
1623 igb_down(adapter);
1624 igb_up(adapter);
1625 clear_bit(__IGB_RESETTING, &adapter->state);
1626}
1627
1628void igb_reset(struct igb_adapter *adapter)
1629{
090b1795 1630 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1631 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1632 struct e1000_mac_info *mac = &hw->mac;
1633 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1634 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1635 u16 hwm;
1636
1637 /* Repartition Pba for greater than 9k mtu
1638 * To take effect CTRL.RST is required.
1639 */
fa4dfae0 1640 switch (mac->type) {
d2ba2ed8 1641 case e1000_i350:
55cac248
AD		 1642 case e1000_82580:
1643 pba = rd32(E1000_RXPBS);
1644 pba = igb_rxpbs_adjust_82580(pba);
1645 break;
fa4dfae0 1646 case e1000_82576:
d249be54
AD		 1647 pba = rd32(E1000_RXPBS);
1648 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1649 break;
1650 case e1000_82575:
f96a8a0b
CW		 1651 case e1000_i210:
1652 case e1000_i211:
fa4dfae0
AD		 1653 default:
1654 pba = E1000_PBA_34K;
1655 break;
2d064c06 1656 }
9d5c8243 1657
2d064c06
AD		 1658 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1659 (mac->type < e1000_82576)) {
9d5c8243
AK		 1660 /* adjust PBA for jumbo frames */
1661 wr32(E1000_PBA, pba);
1662
1663 /* To maintain wire speed transmits, the Tx FIFO should be
1664 * large enough to accommodate two full transmit packets,
1665 * rounded up to the next 1KB and expressed in KB. Likewise,
1666 * the Rx FIFO should be large enough to accommodate at least
1667 * one full receive packet and is similarly rounded up and
1668 * expressed in KB. */
1669 pba = rd32(E1000_PBA);
1670 /* upper 16 bits has Tx packet buffer allocation size in KB */
1671 tx_space = pba >> 16;
1672 /* lower 16 bits has Rx packet buffer allocation size in KB */
1673 pba &= 0xffff;
1674 /* the tx fifo also stores 16 bytes of information about the tx
1675 * but don't include ethernet FCS because hardware appends it */
1676 min_tx_space = (adapter->max_frame_size +
85e8d004 1677 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1678 ETH_FCS_LEN) * 2;
1679 min_tx_space = ALIGN(min_tx_space, 1024);
1680 min_tx_space >>= 10;
1681 /* software strips receive CRC, so leave room for it */
1682 min_rx_space = adapter->max_frame_size;
1683 min_rx_space = ALIGN(min_rx_space, 1024);
1684 min_rx_space >>= 10;
1685
1686 /* If current Tx allocation is less than the min Tx FIFO size,
1687 * and the min Tx FIFO size is less than the current Rx FIFO
1688 * allocation, take space away from current Rx allocation */
1689 if (tx_space < min_tx_space &&
1690 ((min_tx_space - tx_space) < pba)) {
1691 pba = pba - (min_tx_space - tx_space);
1692
1693 /* if short on rx space, rx wins and must trump tx
1694 * adjustment */
1695 if (pba < min_rx_space)
1696 pba = min_rx_space;
1697 }
2d064c06 1698 wr32(E1000_PBA, pba);
9d5c8243 1699 }
9d5c8243
AK		 1700
1701 /* flow control settings */
1702 /* The high water mark must be low enough to fit one full frame
1703 * (or the size used for early receive) above it in the Rx FIFO.
1704 * Set it to the lower of:
1705 * - 90% of the Rx FIFO size, or
1706 * - the full Rx FIFO size minus one full frame */
1707 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1708 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1709
d405ea3e
AD		 1710 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1711 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1712 fc->pause_time = 0xFFFF;
1713 fc->send_xon = 1;
0cce119a 1714 fc->current_mode = fc->requested_mode;
9d5c8243 1715
4ae196df
AD		 1716 /* disable receive for all VFs and wait one second */
1717 if (adapter->vfs_allocated_count) {
1718 int i;
1719 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
8fa7e0f7 1720 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
4ae196df
AD		 1721
1722 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1723 igb_ping_all_vfs(adapter);
4ae196df
AD		 1724
1725 /* disable transmits and receives */
1726 wr32(E1000_VFRE, 0);
1727 wr32(E1000_VFTE, 0);
1728 }
1729
9d5c8243 1730 /* Allow time for pending master requests to run */
330a6d6a 1731 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1732 wr32(E1000_WUC, 0);
1733
330a6d6a 1734 if (hw->mac.ops.init_hw(hw))
090b1795 1735 dev_err(&pdev->dev, "Hardware Error\n");
831ec0b4 1736
a27416bb
MV		 1737 /*
1738 * Flow control settings reset on hardware reset, so guarantee flow
1739 * control is off when forcing speed.
1740 */
1741 if (!hw->mac.autoneg)
1742 igb_force_mac_fc(hw);
1743
b6e0c419 1744 igb_init_dmac(adapter, pba);
88a268c1
NN		 1745 if (!netif_running(adapter->netdev))
1746 igb_power_down_link(adapter);
1747
9d5c8243
AK		 1748 igb_update_mng_vlan(adapter);
1749
1750 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1751 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1752
1f6e8178
MV		 1753#ifdef CONFIG_IGB_PTP
1754 /* Re-enable PTP, where applicable. */
1755 igb_ptp_reset(adapter);
1756#endif /* CONFIG_IGB_PTP */
1757
330a6d6a 1758 igb_get_phy_info(hw);
9d5c8243
AK		 1759}
1760
c8f44aff
MM		 1761static netdev_features_t igb_fix_features(struct net_device *netdev,
1762 netdev_features_t features)
b2cb09b1
JP		 1763{
1764 /*
1765 * Since there is no support for separate rx/tx vlan accel
1766 * enable/disable make sure tx flag is always in same state as rx.
1767 */
1768 if (features & NETIF_F_HW_VLAN_RX)
1769 features |= NETIF_F_HW_VLAN_TX;
1770 else
1771 features &= ~NETIF_F_HW_VLAN_TX;
1772
1773 return features;
1774}
1775
c8f44aff
MM		 1776static int igb_set_features(struct net_device *netdev,
1777 netdev_features_t features)
ac52caa3 1778{
c8f44aff 1779 netdev_features_t changed = netdev->features ^ features;
89eaefb6 1780 struct igb_adapter *adapter = netdev_priv(netdev);
ac52caa3 1781
b2cb09b1
JP		 1782 if (changed & NETIF_F_HW_VLAN_RX)
1783 igb_vlan_mode(netdev, features);
1784
89eaefb6
BG		 1785 if (!(changed & NETIF_F_RXALL))
1786 return 0;
1787
1788 netdev->features = features;
1789
1790 if (netif_running(netdev))
1791 igb_reinit_locked(adapter);
1792 else
1793 igb_reset(adapter);
1794
ac52caa3
MM		 1795 return 0;
1796}
1797
2e5c6922 1798static const struct net_device_ops igb_netdev_ops = {
559e9c49 1799 .ndo_open = igb_open,
2e5c6922 1800 .ndo_stop = igb_close,
cd392f5c 1801 .ndo_start_xmit = igb_xmit_frame,
12dcd86b 1802 .ndo_get_stats64 = igb_get_stats64,
ff41f8dc 1803 .ndo_set_rx_mode = igb_set_rx_mode,
2e5c6922
SH		 1804 .ndo_set_mac_address = igb_set_mac,
1805 .ndo_change_mtu = igb_change_mtu,
1806 .ndo_do_ioctl = igb_ioctl,
1807 .ndo_tx_timeout = igb_tx_timeout,
1808 .ndo_validate_addr = eth_validate_addr,
2e5c6922
SH		 1809 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1810 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1811 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1812 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1813 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1814 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1815#ifdef CONFIG_NET_POLL_CONTROLLER
1816 .ndo_poll_controller = igb_netpoll,
1817#endif
b2cb09b1
JP		 1818 .ndo_fix_features = igb_fix_features,
1819 .ndo_set_features = igb_set_features,
2e5c6922
SH		 1820};
1821
d67974f0
CW		 1822/**
1823 * igb_set_fw_version - Configure version string for ethtool
1824 * @adapter: adapter struct
1825 *
1826 **/
1827void igb_set_fw_version(struct igb_adapter *adapter)
1828{
1829 struct e1000_hw *hw = &adapter->hw;
1830 u16 eeprom_verh, eeprom_verl, comb_verh, comb_verl, comb_offset;
1831 u16 major, build, patch, fw_version;
1832 u32 etrack_id;
1833
1834 hw->nvm.ops.read(hw, 5, 1, &fw_version);
1835 if (adapter->hw.mac.type != e1000_i211) {
1836 hw->nvm.ops.read(hw, NVM_ETRACK_WORD, 1, &eeprom_verh);
1837 hw->nvm.ops.read(hw, (NVM_ETRACK_WORD + 1), 1, &eeprom_verl);
1838 etrack_id = (eeprom_verh << IGB_ETRACK_SHIFT) | eeprom_verl;
1839
1840 /* combo image version needs to be found */
1841 hw->nvm.ops.read(hw, NVM_COMB_VER_PTR, 1, &comb_offset);
1842 if ((comb_offset != 0x0) &&
1843 (comb_offset != IGB_NVM_VER_INVALID)) {
1844 hw->nvm.ops.read(hw, (NVM_COMB_VER_OFF + comb_offset
1845 + 1), 1, &comb_verh);
1846 hw->nvm.ops.read(hw, (NVM_COMB_VER_OFF + comb_offset),
1847 1, &comb_verl);
1848
1849 /* Only display Option Rom if it exists and is valid */
1850 if ((comb_verh && comb_verl) &&
1851 ((comb_verh != IGB_NVM_VER_INVALID) &&
1852 (comb_verl != IGB_NVM_VER_INVALID))) {
1853 major = comb_verl >> IGB_COMB_VER_SHFT;
1854 build = (comb_verl << IGB_COMB_VER_SHFT) |
1855 (comb_verh >> IGB_COMB_VER_SHFT);
1856 patch = comb_verh & IGB_COMB_VER_MASK;
1857 snprintf(adapter->fw_version,
1858 sizeof(adapter->fw_version),
1859 "%d.%d%d, 0x%08x, %d.%d.%d",
1860 (fw_version & IGB_MAJOR_MASK) >>
1861 IGB_MAJOR_SHIFT,
1862 (fw_version & IGB_MINOR_MASK) >>
1863 IGB_MINOR_SHIFT,
1864 (fw_version & IGB_BUILD_MASK),
1865 etrack_id, major, build, patch);
1866 goto out;
1867 }
1868 }
1869 snprintf(adapter->fw_version, sizeof(adapter->fw_version),
1870 "%d.%d%d, 0x%08x",
1871 (fw_version & IGB_MAJOR_MASK) >> IGB_MAJOR_SHIFT,
1872 (fw_version & IGB_MINOR_MASK) >> IGB_MINOR_SHIFT,
1873 (fw_version & IGB_BUILD_MASK), etrack_id);
1874 } else {
1875 snprintf(adapter->fw_version, sizeof(adapter->fw_version),
1876 "%d.%d%d",
1877 (fw_version & IGB_MAJOR_MASK) >> IGB_MAJOR_SHIFT,
1878 (fw_version & IGB_MINOR_MASK) >> IGB_MINOR_SHIFT,
1879 (fw_version & IGB_BUILD_MASK));
1880 }
1881out:
1882 return;
1883}
1884
9d5c8243
AK		 1885/**
1886 * igb_probe - Device Initialization Routine
1887 * @pdev: PCI device information struct
1888 * @ent: entry in igb_pci_tbl
1889 *
1890 * Returns 0 on success, negative on failure
1891 *
1892 * igb_probe initializes an adapter identified by a pci_dev structure.
1893 * The OS initialization, configuring of the adapter private structure,
1894 * and a hardware reset occur.
1895 **/
1896static int __devinit igb_probe(struct pci_dev *pdev,
1897 const struct pci_device_id *ent)
1898{
1899 struct net_device *netdev;
1900 struct igb_adapter *adapter;
1901 struct e1000_hw *hw;
4337e993 1902 u16 eeprom_data = 0;
9835fd73 1903 s32 ret_val;
4337e993 1904 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1905 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1906 unsigned long mmio_start, mmio_len;
2d6a5e95 1907 int err, pci_using_dac;
9d5c8243 1908 u16 eeprom_apme_mask = IGB_EEPROM_APME;
9835fd73 1909 u8 part_str[E1000_PBANUM_LENGTH];
9d5c8243 1910
bded64a7
AG		 1911 /* Catch broken hardware that put the wrong VF device ID in
1912 * the PCIe SR-IOV capability.
1913 */
1914 if (pdev->is_virtfn) {
1915 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
f96a8a0b 1916 pci_name(pdev), pdev->vendor, pdev->device);
bded64a7
AG		 1917 return -EINVAL;
1918 }
1919
aed5dec3 1920 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1921 if (err)
1922 return err;
1923
1924 pci_using_dac = 0;
59d71989 1925 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243 1926 if (!err) {
59d71989 1927 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243
AK		 1928 if (!err)
1929 pci_using_dac = 1;
1930 } else {
59d71989 1931 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243 1932 if (err) {
59d71989 1933 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243
AK		 1934 if (err) {
1935 dev_err(&pdev->dev, "No usable DMA "
1936 "configuration, aborting\n");
1937 goto err_dma;
1938 }
1939 }
1940 }
1941
aed5dec3
AD		 1942 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1943 IORESOURCE_MEM),
1944 igb_driver_name);
9d5c8243
AK		 1945 if (err)
1946 goto err_pci_reg;
1947
19d5afd4 1948 pci_enable_pcie_error_reporting(pdev);
40a914fa 1949
9d5c8243 1950 pci_set_master(pdev);
c682fc23 1951 pci_save_state(pdev);
9d5c8243
AK		 1952
1953 err = -ENOMEM;
1bfaf07b 1954 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1cc3bd87 1955 IGB_MAX_TX_QUEUES);
9d5c8243
AK		 1956 if (!netdev)
1957 goto err_alloc_etherdev;
1958
1959 SET_NETDEV_DEV(netdev, &pdev->dev);
1960
1961 pci_set_drvdata(pdev, netdev);
1962 adapter = netdev_priv(netdev);
1963 adapter->netdev = netdev;
1964 adapter->pdev = pdev;
1965 hw = &adapter->hw;
1966 hw->back = adapter;
b3f4d599 1967 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
9d5c8243
AK		 1968
1969 mmio_start = pci_resource_start(pdev, 0);
1970 mmio_len = pci_resource_len(pdev, 0);
1971
1972 err = -EIO;
28b0759c
AD		 1973 hw->hw_addr = ioremap(mmio_start, mmio_len);
1974 if (!hw->hw_addr)
9d5c8243
AK		 1975 goto err_ioremap;
1976
2e5c6922 1977 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1978 igb_set_ethtool_ops(netdev);
9d5c8243 1979 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1980
1981 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1982
1983 netdev->mem_start = mmio_start;
1984 netdev->mem_end = mmio_start + mmio_len;
1985
9d5c8243
AK		 1986 /* PCI config space info */
1987 hw->vendor_id = pdev->vendor;
1988 hw->device_id = pdev->device;
1989 hw->revision_id = pdev->revision;
1990 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1991 hw->subsystem_device_id = pdev->subsystem_device;
1992
9d5c8243
AK		 1993 /* Copy the default MAC, PHY and NVM function pointers */
1994 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1995 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1996 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1997 /* Initialize skew-specific constants */
1998 err = ei->get_invariants(hw);
1999 if (err)
450c87c8 2000 goto err_sw_init;
9d5c8243 2001
450c87c8 2002 /* setup the private structure */
9d5c8243
AK		 2003 err = igb_sw_init(adapter);
2004 if (err)
2005 goto err_sw_init;
2006
2007 igb_get_bus_info_pcie(hw);
2008
2009 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 2010
2011 /* Copper options */
2012 if (hw->phy.media_type == e1000_media_type_copper) {
2013 hw->phy.mdix = AUTO_ALL_MODES;
2014 hw->phy.disable_polarity_correction = false;
2015 hw->phy.ms_type = e1000_ms_hw_default;
2016 }
2017
2018 if (igb_check_reset_block(hw))
2019 dev_info(&pdev->dev,
2020 "PHY reset is blocked due to SOL/IDER session.\n");
2021
077887c3
AD		 2022 /*
2023 * features is initialized to 0 in allocation, it might have bits
2024 * set by igb_sw_init so we should use an or instead of an
2025 * assignment.
2026 */
2027 netdev->features |= NETIF_F_SG |
2028 NETIF_F_IP_CSUM |
2029 NETIF_F_IPV6_CSUM |
2030 NETIF_F_TSO |
2031 NETIF_F_TSO6 |
2032 NETIF_F_RXHASH |
2033 NETIF_F_RXCSUM |
2034 NETIF_F_HW_VLAN_RX |
2035 NETIF_F_HW_VLAN_TX;
2036
2037 /* copy netdev features into list of user selectable features */
2038 netdev->hw_features |= netdev->features;
89eaefb6 2039 netdev->hw_features |= NETIF_F_RXALL;
077887c3
AD		 2040
2041 /* set this bit last since it cannot be part of hw_features */
2042 netdev->features |= NETIF_F_HW_VLAN_FILTER;
2043
2044 netdev->vlan_features |= NETIF_F_TSO |
2045 NETIF_F_TSO6 |
2046 NETIF_F_IP_CSUM |
2047 NETIF_F_IPV6_CSUM |
2048 NETIF_F_SG;
48f29ffc 2049
6b8f0922
BG		 2050 netdev->priv_flags |= IFF_SUPP_NOFCS;
2051
7b872a55 2052 if (pci_using_dac) {
9d5c8243 2053 netdev->features |= NETIF_F_HIGHDMA;
7b872a55
YZ		 2054 netdev->vlan_features |= NETIF_F_HIGHDMA;
2055 }
9d5c8243 2056
ac52caa3
MM		 2057 if (hw->mac.type >= e1000_82576) {
2058 netdev->hw_features |= NETIF_F_SCTP_CSUM;
b9473560 2059 netdev->features |= NETIF_F_SCTP_CSUM;
ac52caa3 2060 }
b9473560 2061
01789349
JP		 2062 netdev->priv_flags |= IFF_UNICAST_FLT;
2063
330a6d6a 2064 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 2065
2066 /* before reading the NVM, reset the controller to put the device in a
2067 * known good starting state */
2068 hw->mac.ops.reset_hw(hw);
2069
f96a8a0b
CW		 2070 /*
2071 * make sure the NVM is good , i211 parts have special NVM that
2072 * doesn't contain a checksum
2073 */
2074 if (hw->mac.type != e1000_i211) {
2075 if (hw->nvm.ops.validate(hw) < 0) {
2076 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
2077 err = -EIO;
2078 goto err_eeprom;
2079 }
9d5c8243
AK		 2080 }
2081
2082 /* copy the MAC address out of the NVM */
2083 if (hw->mac.ops.read_mac_addr(hw))
2084 dev_err(&pdev->dev, "NVM Read Error\n");
2085
2086 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2087 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
2088
2089 if (!is_valid_ether_addr(netdev->perm_addr)) {
2090 dev_err(&pdev->dev, "Invalid MAC Address\n");
2091 err = -EIO;
2092 goto err_eeprom;
2093 }
2094
d67974f0
CW		 2095 /* get firmware version for ethtool -i */
2096 igb_set_fw_version(adapter);
2097
c061b18d 2098 setup_timer(&adapter->watchdog_timer, igb_watchdog,
0e340485 2099 (unsigned long) adapter);
c061b18d 2100 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
0e340485 2101 (unsigned long) adapter);
9d5c8243
AK		 2102
2103 INIT_WORK(&adapter->reset_task, igb_reset_task);
2104 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2105
450c87c8 2106 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 2107 adapter->fc_autoneg = true;
2108 hw->mac.autoneg = true;
2109 hw->phy.autoneg_advertised = 0x2f;
2110
0cce119a
AD		 2111 hw->fc.requested_mode = e1000_fc_default;
2112 hw->fc.current_mode = e1000_fc_default;
9d5c8243 2113
9d5c8243
AK		 2114 igb_validate_mdi_setting(hw);
2115
9d5c8243
AK		 2116 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2117 * enable the ACPI Magic Packet filter
2118 */
2119
a2cf8b6c 2120 if (hw->bus.func == 0)
312c75ae 2121 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
6d337dce 2122 else if (hw->mac.type >= e1000_82580)
55cac248
AD		 2123 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2124 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2125 &eeprom_data);
a2cf8b6c
AD		 2126 else if (hw->bus.func == 1)
2127 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 2128
2129 if (eeprom_data & eeprom_apme_mask)
2130 adapter->eeprom_wol |= E1000_WUFC_MAG;
2131
2132 /* now that we have the eeprom settings, apply the special cases where
2133 * the eeprom may be wrong or the board simply won't support wake on
2134 * lan on a particular port */
2135 switch (pdev->device) {
2136 case E1000_DEV_ID_82575GB_QUAD_COPPER:
2137 adapter->eeprom_wol = 0;
2138 break;
2139 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 2140 case E1000_DEV_ID_82576_FIBER:
2141 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 2142 /* Wake events only supported on port A for dual fiber
2143 * regardless of eeprom setting */
2144 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
2145 adapter->eeprom_wol = 0;
2146 break;
c8ea5ea9 2147 case E1000_DEV_ID_82576_QUAD_COPPER:
d5aa2252 2148 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
c8ea5ea9
AD		 2149 /* if quad port adapter, disable WoL on all but port A */
2150 if (global_quad_port_a != 0)
2151 adapter->eeprom_wol = 0;
2152 else
2153 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2154 /* Reset for multiple quad port adapters */
2155 if (++global_quad_port_a == 4)
2156 global_quad_port_a = 0;
2157 break;
9d5c8243
AK		 2158 }
2159
2160 /* initialize the wol settings based on the eeprom settings */
2161 adapter->wol = adapter->eeprom_wol;
e1b86d84 2162 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 2163
2164 /* reset the hardware with the new settings */
2165 igb_reset(adapter);
2166
2167 /* let the f/w know that the h/w is now under the control of the
2168 * driver. */
2169 igb_get_hw_control(adapter);
2170
9d5c8243
AK		 2171 strcpy(netdev->name, "eth%d");
2172 err = register_netdev(netdev);
2173 if (err)
2174 goto err_register;
2175
b168dfc5
JB		 2176 /* carrier off reporting is important to ethtool even BEFORE open */
2177 netif_carrier_off(netdev);
2178
421e02f0 2179#ifdef CONFIG_IGB_DCA
bbd98fe4 2180 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 2181 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 2182 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 2183 igb_setup_dca(adapter);
2184 }
fe4506b6 2185
38c845c7 2186#endif
3c89f6d0 2187
7ebae817 2188#ifdef CONFIG_IGB_PTP
673b8b70 2189 /* do hw tstamp init after resetting */
7ebae817 2190 igb_ptp_init(adapter);
3c89f6d0 2191#endif /* CONFIG_IGB_PTP */
673b8b70 2192
9d5c8243
AK		 2193 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
2194 /* print bus type/speed/width info */
7c510e4b 2195 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 2196 netdev->name,
559e9c49 2197 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
ff846f52 2198 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
559e9c49 2199 "unknown"),
59c3de89
AD		 2200 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
2201 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
2202 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2203 "unknown"),
7c510e4b 2204 netdev->dev_addr);
9d5c8243 2205
9835fd73
CW		 2206 ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH);
2207 if (ret_val)
2208 strcpy(part_str, "Unknown");
2209 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
9d5c8243
AK		 2210 dev_info(&pdev->dev,
2211 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2212 adapter->msix_entries ? "MSI-X" :
7dfc16fa 2213 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243 2214 adapter->num_rx_queues, adapter->num_tx_queues);
09b068d4
CW		 2215 switch (hw->mac.type) {
2216 case e1000_i350:
f96a8a0b
CW		 2217 case e1000_i210:
2218 case e1000_i211:
09b068d4
CW		 2219 igb_set_eee_i350(hw);
2220 break;
2221 default:
2222 break;
2223 }
749ab2cd
YZ		 2224
2225 pm_runtime_put_noidle(&pdev->dev);
9d5c8243
AK		 2226 return 0;
2227
2228err_register:
2229 igb_release_hw_control(adapter);
2230err_eeprom:
2231 if (!igb_check_reset_block(hw))
f5f4cf08 2232 igb_reset_phy(hw);
9d5c8243
AK		 2233
2234 if (hw->flash_address)
2235 iounmap(hw->flash_address);
9d5c8243 2236err_sw_init:
047e0030 2237 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 2238 iounmap(hw->hw_addr);
2239err_ioremap:
2240 free_netdev(netdev);
2241err_alloc_etherdev:
559e9c49
AD		 2242 pci_release_selected_regions(pdev,
2243 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2244err_pci_reg:
2245err_dma:
2246 pci_disable_device(pdev);
2247 return err;
2248}
2249
2250/**
2251 * igb_remove - Device Removal Routine
2252 * @pdev: PCI device information struct
2253 *
2254 * igb_remove is called by the PCI subsystem to alert the driver
2255 * that it should release a PCI device. The could be caused by a
2256 * Hot-Plug event, or because the driver is going to be removed from
2257 * memory.
2258 **/
2259static void __devexit igb_remove(struct pci_dev *pdev)
2260{
2261 struct net_device *netdev = pci_get_drvdata(pdev);
2262 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 2263 struct e1000_hw *hw = &adapter->hw;
9d5c8243 2264
749ab2cd 2265 pm_runtime_get_noresume(&pdev->dev);
7ebae817 2266#ifdef CONFIG_IGB_PTP
a79f4f88 2267 igb_ptp_stop(adapter);
3c89f6d0 2268#endif /* CONFIG_IGB_PTP */
749ab2cd 2269
760141a5
TH		 2270 /*
2271 * The watchdog timer may be rescheduled, so explicitly
2272 * disable watchdog from being rescheduled.
2273 */
9d5c8243
AK		 2274 set_bit(__IGB_DOWN, &adapter->state);
2275 del_timer_sync(&adapter->watchdog_timer);
2276 del_timer_sync(&adapter->phy_info_timer);
2277
760141a5
TH		 2278 cancel_work_sync(&adapter->reset_task);
2279 cancel_work_sync(&adapter->watchdog_task);
9d5c8243 2280
421e02f0 2281#ifdef CONFIG_IGB_DCA
7dfc16fa 2282 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 2283 dev_info(&pdev->dev, "DCA disabled\n");
2284 dca_remove_requester(&pdev->dev);
7dfc16fa 2285 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 2286 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 2287 }
2288#endif
2289
9d5c8243
AK		 2290 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2291 * would have already happened in close and is redundant. */
2292 igb_release_hw_control(adapter);
2293
2294 unregister_netdev(netdev);
2295
047e0030 2296 igb_clear_interrupt_scheme(adapter);
9d5c8243 2297
37680117
AD		 2298#ifdef CONFIG_PCI_IOV
2299 /* reclaim resources allocated to VFs */
2300 if (adapter->vf_data) {
2301 /* disable iov and allow time for transactions to clear */
f557147c
SA		 2302 if (igb_vfs_are_assigned(adapter)) {
2303 dev_info(&pdev->dev, "Unloading driver while VFs are assigned - VFs will not be deallocated\n");
2304 } else {
0224d663
GR		 2305 pci_disable_sriov(pdev);
2306 msleep(500);
0224d663 2307 }
37680117
AD		 2308
2309 kfree(adapter->vf_data);
2310 adapter->vf_data = NULL;
2311 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 2312 wrfl();
37680117
AD		 2313 msleep(100);
2314 dev_info(&pdev->dev, "IOV Disabled\n");
2315 }
2316#endif
559e9c49 2317
28b0759c
AD		 2318 iounmap(hw->hw_addr);
2319 if (hw->flash_address)
2320 iounmap(hw->flash_address);
559e9c49
AD		 2321 pci_release_selected_regions(pdev,
2322 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243 2323
1128c756 2324 kfree(adapter->shadow_vfta);
9d5c8243
AK		 2325 free_netdev(netdev);
2326
19d5afd4 2327 pci_disable_pcie_error_reporting(pdev);
40a914fa 2328
9d5c8243
AK		 2329 pci_disable_device(pdev);
2330}
2331
a6b623e0
AD		 2332/**
2333 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2334 * @adapter: board private structure to initialize
2335 *
2336 * This function initializes the vf specific data storage and then attempts to
2337 * allocate the VFs. The reason for ordering it this way is because it is much
2338 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2339 * the memory for the VFs.
2340 **/
2341static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2342{
2343#ifdef CONFIG_PCI_IOV
2344 struct pci_dev *pdev = adapter->pdev;
f96a8a0b 2345 struct e1000_hw *hw = &adapter->hw;
f557147c 2346 int old_vfs = pci_num_vf(adapter->pdev);
0224d663 2347 int i;
a6b623e0 2348
f96a8a0b
CW		 2349 /* Virtualization features not supported on i210 family. */
2350 if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211))
2351 return;
2352
0224d663
GR		 2353 if (old_vfs) {
2354 dev_info(&pdev->dev, "%d pre-allocated VFs found - override "
2355 "max_vfs setting of %d\n", old_vfs, max_vfs);
2356 adapter->vfs_allocated_count = old_vfs;
a6b623e0
AD		 2357 }
2358
0224d663
GR		 2359 if (!adapter->vfs_allocated_count)
2360 return;
2361
2362 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2363 sizeof(struct vf_data_storage), GFP_KERNEL);
f96a8a0b 2364
0224d663
GR		 2365 /* if allocation failed then we do not support SR-IOV */
2366 if (!adapter->vf_data) {
a6b623e0 2367 adapter->vfs_allocated_count = 0;
0224d663
GR		 2368 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2369 "Data Storage\n");
2370 goto out;
a6b623e0 2371 }
0224d663
GR		 2372
2373 if (!old_vfs) {
2374 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count))
2375 goto err_out;
2376 }
2377 dev_info(&pdev->dev, "%d VFs allocated\n",
2378 adapter->vfs_allocated_count);
2379 for (i = 0; i < adapter->vfs_allocated_count; i++)
2380 igb_vf_configure(adapter, i);
2381
2382 /* DMA Coalescing is not supported in IOV mode. */
2383 adapter->flags &= ~IGB_FLAG_DMAC;
2384 goto out;
2385err_out:
2386 kfree(adapter->vf_data);
2387 adapter->vf_data = NULL;
2388 adapter->vfs_allocated_count = 0;
2389out:
2390 return;
a6b623e0
AD		 2391#endif /* CONFIG_PCI_IOV */
2392}
2393
9d5c8243
AK		 2394/**
2395 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2396 * @adapter: board private structure to initialize
2397 *
2398 * igb_sw_init initializes the Adapter private data structure.
2399 * Fields are initialized based on PCI device information and
2400 * OS network device settings (MTU size).
2401 **/
2402static int __devinit igb_sw_init(struct igb_adapter *adapter)
2403{
2404 struct e1000_hw *hw = &adapter->hw;
2405 struct net_device *netdev = adapter->netdev;
2406 struct pci_dev *pdev = adapter->pdev;
374a542d 2407 u32 max_rss_queues;
9d5c8243
AK		 2408
2409 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2410
13fde97a 2411 /* set default ring sizes */
68fd9910
AD		 2412 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2413 adapter->rx_ring_count = IGB_DEFAULT_RXD;
13fde97a
AD		 2414
2415 /* set default ITR values */
4fc82adf
AD		 2416 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2417 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2418
13fde97a
AD		 2419 /* set default work limits */
2420 adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
2421
153285f9
AD		 2422 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
2423 VLAN_HLEN;
9d5c8243
AK		 2424 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2425
81c2fc22
AD		 2426 adapter->node = -1;
2427
12dcd86b 2428 spin_lock_init(&adapter->stats64_lock);
a6b623e0 2429#ifdef CONFIG_PCI_IOV
6b78bb1d
CW		 2430 switch (hw->mac.type) {
2431 case e1000_82576:
2432 case e1000_i350:
9b082d73
SA		 2433 if (max_vfs > 7) {
2434 dev_warn(&pdev->dev,
2435 "Maximum of 7 VFs per PF, using max\n");
2436 adapter->vfs_allocated_count = 7;
2437 } else
2438 adapter->vfs_allocated_count = max_vfs;
6b78bb1d
CW		 2439 break;
2440 default:
2441 break;
2442 }
a6b623e0 2443#endif /* CONFIG_PCI_IOV */
374a542d
MV		 2444
2445 /* Determine the maximum number of RSS queues supported. */
f96a8a0b 2446 switch (hw->mac.type) {
374a542d
MV		 2447 case e1000_i211:
2448 max_rss_queues = IGB_MAX_RX_QUEUES_I211;
2449 break;
2450 case e1000_82575:
f96a8a0b 2451 case e1000_i210:
374a542d
MV		 2452 max_rss_queues = IGB_MAX_RX_QUEUES_82575;
2453 break;
2454 case e1000_i350:
2455 /* I350 cannot do RSS and SR-IOV at the same time */
2456 if (!!adapter->vfs_allocated_count) {
2457 max_rss_queues = 1;
2458 break;
2459 }
2460 /* fall through */
2461 case e1000_82576:
2462 if (!!adapter->vfs_allocated_count) {
2463 max_rss_queues = 2;
2464 break;
2465 }
2466 /* fall through */
2467 case e1000_82580:
2468 default:
2469 max_rss_queues = IGB_MAX_RX_QUEUES;
f96a8a0b 2470 break;
374a542d
MV		 2471 }
2472
2473 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
2474
2475 /* Determine if we need to pair queues. */
2476 switch (hw->mac.type) {
2477 case e1000_82575:
f96a8a0b 2478 case e1000_i211:
374a542d 2479 /* Device supports enough interrupts without queue pairing. */
f96a8a0b 2480 break;
374a542d
MV		 2481 case e1000_82576:
2482 /*
2483 * If VFs are going to be allocated with RSS queues then we
2484 * should pair the queues in order to conserve interrupts due
2485 * to limited supply.
2486 */
2487 if ((adapter->rss_queues > 1) &&
2488 (adapter->vfs_allocated_count > 6))
2489 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2490 /* fall through */
2491 case e1000_82580:
2492 case e1000_i350:
2493 case e1000_i210:
f96a8a0b 2494 default:
374a542d
MV		 2495 /*
2496 * If rss_queues > half of max_rss_queues, pair the queues in
2497 * order to conserve interrupts due to limited supply.
2498 */
2499 if (adapter->rss_queues > (max_rss_queues / 2))
2500 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
f96a8a0b
CW		 2501 break;
2502 }
a99955fc 2503
1128c756
CW		 2504 /* Setup and initialize a copy of the hw vlan table array */
2505 adapter->shadow_vfta = kzalloc(sizeof(u32) *
2506 E1000_VLAN_FILTER_TBL_SIZE,
2507 GFP_ATOMIC);
2508
a6b623e0 2509 /* This call may decrease the number of queues */
047e0030 2510 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 2511 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2512 return -ENOMEM;
2513 }
2514
a6b623e0
AD		 2515 igb_probe_vfs(adapter);
2516
9d5c8243
AK		 2517 /* Explicitly disable IRQ since the NIC can be in any state. */
2518 igb_irq_disable(adapter);
2519
f96a8a0b 2520 if (hw->mac.type >= e1000_i350)
831ec0b4
CW		 2521 adapter->flags &= ~IGB_FLAG_DMAC;
2522
9d5c8243
AK		 2523 set_bit(__IGB_DOWN, &adapter->state);
2524 return 0;
2525}
2526
2527/**
2528 * igb_open - Called when a network interface is made active
2529 * @netdev: network interface device structure
2530 *
2531 * Returns 0 on success, negative value on failure
2532 *
2533 * The open entry point is called when a network interface is made
2534 * active by the system (IFF_UP). At this point all resources needed
2535 * for transmit and receive operations are allocated, the interrupt
2536 * handler is registered with the OS, the watchdog timer is started,
2537 * and the stack is notified that the interface is ready.
2538 **/
749ab2cd 2539static int __igb_open(struct net_device *netdev, bool resuming)
9d5c8243
AK		 2540{
2541 struct igb_adapter *adapter = netdev_priv(netdev);
2542 struct e1000_hw *hw = &adapter->hw;
749ab2cd 2543 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2544 int err;
2545 int i;
2546
2547 /* disallow open during test */
749ab2cd
YZ		 2548 if (test_bit(__IGB_TESTING, &adapter->state)) {
2549 WARN_ON(resuming);
9d5c8243 2550 return -EBUSY;
749ab2cd
YZ		 2551 }
2552
2553 if (!resuming)
2554 pm_runtime_get_sync(&pdev->dev);
9d5c8243 2555
b168dfc5
JB		 2556 netif_carrier_off(netdev);
2557
9d5c8243
AK		 2558 /* allocate transmit descriptors */
2559 err = igb_setup_all_tx_resources(adapter);
2560 if (err)
2561 goto err_setup_tx;
2562
2563 /* allocate receive descriptors */
2564 err = igb_setup_all_rx_resources(adapter);
2565 if (err)
2566 goto err_setup_rx;
2567
88a268c1 2568 igb_power_up_link(adapter);
9d5c8243 2569
9d5c8243
AK		 2570 /* before we allocate an interrupt, we must be ready to handle it.
2571 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2572 * as soon as we call pci_request_irq, so we have to setup our
2573 * clean_rx handler before we do so. */
2574 igb_configure(adapter);
2575
2576 err = igb_request_irq(adapter);
2577 if (err)
2578 goto err_req_irq;
2579
2580 /* From here on the code is the same as igb_up() */
2581 clear_bit(__IGB_DOWN, &adapter->state);
2582
0d1ae7f4
AD		 2583 for (i = 0; i < adapter->num_q_vectors; i++)
2584 napi_enable(&(adapter->q_vector[i]->napi));
9d5c8243
AK		 2585
2586 /* Clear any pending interrupts. */
2587 rd32(E1000_ICR);
844290e5
PW		 2588
2589 igb_irq_enable(adapter);
2590
d4960307
AD		 2591 /* notify VFs that reset has been completed */
2592 if (adapter->vfs_allocated_count) {
2593 u32 reg_data = rd32(E1000_CTRL_EXT);
2594 reg_data |= E1000_CTRL_EXT_PFRSTD;
2595 wr32(E1000_CTRL_EXT, reg_data);
2596 }
2597
d55b53ff
JK		 2598 netif_tx_start_all_queues(netdev);
2599
749ab2cd
YZ		 2600 if (!resuming)
2601 pm_runtime_put(&pdev->dev);
2602
25568a53
AD		 2603 /* start the watchdog. */
2604 hw->mac.get_link_status = 1;
2605 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2606
2607 return 0;
2608
2609err_req_irq:
2610 igb_release_hw_control(adapter);
88a268c1 2611 igb_power_down_link(adapter);
9d5c8243
AK		 2612 igb_free_all_rx_resources(adapter);
2613err_setup_rx:
2614 igb_free_all_tx_resources(adapter);
2615err_setup_tx:
2616 igb_reset(adapter);
749ab2cd
YZ		 2617 if (!resuming)
2618 pm_runtime_put(&pdev->dev);
9d5c8243
AK		 2619
2620 return err;
2621}
2622
749ab2cd
YZ		 2623static int igb_open(struct net_device *netdev)
2624{
2625 return __igb_open(netdev, false);
2626}
2627
9d5c8243
AK		 2628/**
2629 * igb_close - Disables a network interface
2630 * @netdev: network interface device structure
2631 *
2632 * Returns 0, this is not allowed to fail
2633 *
2634 * The close entry point is called when an interface is de-activated
2635 * by the OS. The hardware is still under the driver's control, but
2636 * needs to be disabled. A global MAC reset is issued to stop the
2637 * hardware, and all transmit and receive resources are freed.
2638 **/
749ab2cd 2639static int __igb_close(struct net_device *netdev, bool suspending)
9d5c8243
AK		 2640{
2641 struct igb_adapter *adapter = netdev_priv(netdev);
749ab2cd 2642 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2643
2644 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
9d5c8243 2645
749ab2cd
YZ		 2646 if (!suspending)
2647 pm_runtime_get_sync(&pdev->dev);
2648
2649 igb_down(adapter);
9d5c8243
AK		 2650 igb_free_irq(adapter);
2651
2652 igb_free_all_tx_resources(adapter);
2653 igb_free_all_rx_resources(adapter);
2654
749ab2cd
YZ		 2655 if (!suspending)
2656 pm_runtime_put_sync(&pdev->dev);
9d5c8243
AK		 2657 return 0;
2658}
2659
749ab2cd
YZ		 2660static int igb_close(struct net_device *netdev)
2661{
2662 return __igb_close(netdev, false);
2663}
2664
9d5c8243
AK		 2665/**
2666 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2667 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2668 *
2669 * Return 0 on success, negative on failure
2670 **/
80785298 2671int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2672{
59d71989 2673 struct device *dev = tx_ring->dev;
81c2fc22 2674 int orig_node = dev_to_node(dev);
9d5c8243
AK		 2675 int size;
2676
06034649 2677 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
81c2fc22
AD		 2678 tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node);
2679 if (!tx_ring->tx_buffer_info)
2680 tx_ring->tx_buffer_info = vzalloc(size);
06034649 2681 if (!tx_ring->tx_buffer_info)
9d5c8243 2682 goto err;
9d5c8243
AK		 2683
2684 /* round up to nearest 4K */
85e8d004 2685 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2686 tx_ring->size = ALIGN(tx_ring->size, 4096);
2687
81c2fc22 2688 set_dev_node(dev, tx_ring->numa_node);
59d71989
AD		 2689 tx_ring->desc = dma_alloc_coherent(dev,
2690 tx_ring->size,
2691 &tx_ring->dma,
2692 GFP_KERNEL);
81c2fc22
AD		 2693 set_dev_node(dev, orig_node);
2694 if (!tx_ring->desc)
2695 tx_ring->desc = dma_alloc_coherent(dev,
2696 tx_ring->size,
2697 &tx_ring->dma,
2698 GFP_KERNEL);
9d5c8243
AK		 2699
2700 if (!tx_ring->desc)
2701 goto err;
2702
9d5c8243
AK		 2703 tx_ring->next_to_use = 0;
2704 tx_ring->next_to_clean = 0;
81c2fc22 2705
9d5c8243
AK		 2706 return 0;
2707
2708err:
06034649 2709 vfree(tx_ring->tx_buffer_info);
59d71989 2710 dev_err(dev,
9d5c8243
AK		 2711 "Unable to allocate memory for the transmit descriptor ring\n");
2712 return -ENOMEM;
2713}
2714
2715/**
2716 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2717 * (Descriptors) for all queues
2718 * @adapter: board private structure
2719 *
2720 * Return 0 on success, negative on failure
2721 **/
2722static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2723{
439705e1 2724 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2725 int i, err = 0;
2726
2727 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2728 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2729 if (err) {
439705e1 2730 dev_err(&pdev->dev,
9d5c8243
AK		 2731 "Allocation for Tx Queue %u failed\n", i);
2732 for (i--; i >= 0; i--)
3025a446 2733 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2734 break;
2735 }
2736 }
2737
2738 return err;
2739}
2740
2741/**
85b430b4
AD		 2742 * igb_setup_tctl - configure the transmit control registers
2743 * @adapter: Board private structure
9d5c8243 2744 **/
d7ee5b3a 2745void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2746{
9d5c8243
AK		 2747 struct e1000_hw *hw = &adapter->hw;
2748 u32 tctl;
9d5c8243 2749
85b430b4
AD		 2750 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2751 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2752
2753 /* Program the Transmit Control Register */
9d5c8243
AK		 2754 tctl = rd32(E1000_TCTL);
2755 tctl &= ~E1000_TCTL_CT;
2756 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2757 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2758
2759 igb_config_collision_dist(hw);
2760
9d5c8243
AK		 2761 /* Enable transmits */
2762 tctl |= E1000_TCTL_EN;
2763
2764 wr32(E1000_TCTL, tctl);
2765}
2766
85b430b4
AD		 2767/**
2768 * igb_configure_tx_ring - Configure transmit ring after Reset
2769 * @adapter: board private structure
2770 * @ring: tx ring to configure
2771 *
2772 * Configure a transmit ring after a reset.
2773 **/
d7ee5b3a
AD		 2774void igb_configure_tx_ring(struct igb_adapter *adapter,
2775 struct igb_ring *ring)
85b430b4
AD		 2776{
2777 struct e1000_hw *hw = &adapter->hw;
a74420e0 2778 u32 txdctl = 0;
85b430b4
AD		 2779 u64 tdba = ring->dma;
2780 int reg_idx = ring->reg_idx;
2781
2782 /* disable the queue */
a74420e0 2783 wr32(E1000_TXDCTL(reg_idx), 0);
85b430b4
AD		 2784 wrfl();
2785 mdelay(10);
2786
2787 wr32(E1000_TDLEN(reg_idx),
2788 ring->count * sizeof(union e1000_adv_tx_desc));
2789 wr32(E1000_TDBAL(reg_idx),
2790 tdba & 0x00000000ffffffffULL);
2791 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2792
fce99e34 2793 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
a74420e0 2794 wr32(E1000_TDH(reg_idx), 0);
fce99e34 2795 writel(0, ring->tail);
85b430b4
AD		 2796
2797 txdctl |= IGB_TX_PTHRESH;
2798 txdctl |= IGB_TX_HTHRESH << 8;
2799 txdctl |= IGB_TX_WTHRESH << 16;
2800
2801 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2802 wr32(E1000_TXDCTL(reg_idx), txdctl);
2803}
2804
2805/**
2806 * igb_configure_tx - Configure transmit Unit after Reset
2807 * @adapter: board private structure
2808 *
2809 * Configure the Tx unit of the MAC after a reset.
2810 **/
2811static void igb_configure_tx(struct igb_adapter *adapter)
2812{
2813 int i;
2814
2815 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2816 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2817}
2818
9d5c8243
AK		 2819/**
2820 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2821 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2822 *
2823 * Returns 0 on success, negative on failure
2824 **/
80785298 2825int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2826{
59d71989 2827 struct device *dev = rx_ring->dev;
81c2fc22 2828 int orig_node = dev_to_node(dev);
9d5c8243
AK		 2829 int size, desc_len;
2830
06034649 2831 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
81c2fc22
AD		 2832 rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node);
2833 if (!rx_ring->rx_buffer_info)
2834 rx_ring->rx_buffer_info = vzalloc(size);
06034649 2835 if (!rx_ring->rx_buffer_info)
9d5c8243 2836 goto err;
9d5c8243
AK		 2837
2838 desc_len = sizeof(union e1000_adv_rx_desc);
2839
2840 /* Round up to nearest 4K */
2841 rx_ring->size = rx_ring->count * desc_len;
2842 rx_ring->size = ALIGN(rx_ring->size, 4096);
2843
81c2fc22 2844 set_dev_node(dev, rx_ring->numa_node);
59d71989
AD		 2845 rx_ring->desc = dma_alloc_coherent(dev,
2846 rx_ring->size,
2847 &rx_ring->dma,
2848 GFP_KERNEL);
81c2fc22
AD		 2849 set_dev_node(dev, orig_node);
2850 if (!rx_ring->desc)
2851 rx_ring->desc = dma_alloc_coherent(dev,
2852 rx_ring->size,
2853 &rx_ring->dma,
2854 GFP_KERNEL);
9d5c8243
AK		 2855
2856 if (!rx_ring->desc)
2857 goto err;
2858
2859 rx_ring->next_to_clean = 0;
2860 rx_ring->next_to_use = 0;
9d5c8243 2861
9d5c8243
AK		 2862 return 0;
2863
2864err:
06034649
AD		 2865 vfree(rx_ring->rx_buffer_info);
2866 rx_ring->rx_buffer_info = NULL;
59d71989
AD		 2867 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2868 " ring\n");
9d5c8243
AK		 2869 return -ENOMEM;
2870}
2871
2872/**
2873 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2874 * (Descriptors) for all queues
2875 * @adapter: board private structure
2876 *
2877 * Return 0 on success, negative on failure
2878 **/
2879static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2880{
439705e1 2881 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2882 int i, err = 0;
2883
2884 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2885 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2886 if (err) {
439705e1 2887 dev_err(&pdev->dev,
9d5c8243
AK		 2888 "Allocation for Rx Queue %u failed\n", i);
2889 for (i--; i >= 0; i--)
3025a446 2890 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2891 break;
2892 }
2893 }
2894
2895 return err;
2896}
2897
06cf2666
AD		 2898/**
2899 * igb_setup_mrqc - configure the multiple receive queue control registers
2900 * @adapter: Board private structure
2901 **/
2902static void igb_setup_mrqc(struct igb_adapter *adapter)
2903{
2904 struct e1000_hw *hw = &adapter->hw;
2905 u32 mrqc, rxcsum;
2906 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2907 union e1000_reta {
2908 u32 dword;
2909 u8 bytes[4];
2910 } reta;
2911 static const u8 rsshash[40] = {
2912 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2913 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2914 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2915 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2916
2917 /* Fill out hash function seeds */
2918 for (j = 0; j < 10; j++) {
2919 u32 rsskey = rsshash[(j * 4)];
2920 rsskey |= rsshash[(j * 4) + 1] << 8;
2921 rsskey |= rsshash[(j * 4) + 2] << 16;
2922 rsskey |= rsshash[(j * 4) + 3] << 24;
2923 array_wr32(E1000_RSSRK(0), j, rsskey);
2924 }
2925
a99955fc 2926 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2927
2928 if (adapter->vfs_allocated_count) {
2929 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2930 switch (hw->mac.type) {
d2ba2ed8 2931 case e1000_i350:
55cac248
AD		 2932 case e1000_82580:
2933 num_rx_queues = 1;
2934 shift = 0;
2935 break;
06cf2666
AD		 2936 case e1000_82576:
2937 shift = 3;
2938 num_rx_queues = 2;
2939 break;
2940 case e1000_82575:
2941 shift = 2;
2942 shift2 = 6;
2943 default:
2944 break;
2945 }
2946 } else {
2947 if (hw->mac.type == e1000_82575)
2948 shift = 6;
2949 }
2950
2951 for (j = 0; j < (32 * 4); j++) {
2952 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2953 if (shift2)
2954 reta.bytes[j & 3] |= num_rx_queues << shift2;
2955 if ((j & 3) == 3)
2956 wr32(E1000_RETA(j >> 2), reta.dword);
2957 }
2958
2959 /*
2960 * Disable raw packet checksumming so that RSS hash is placed in
2961 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2962 * offloads as they are enabled by default
2963 */
2964 rxcsum = rd32(E1000_RXCSUM);
2965 rxcsum |= E1000_RXCSUM_PCSD;
2966
2967 if (adapter->hw.mac.type >= e1000_82576)
2968 /* Enable Receive Checksum Offload for SCTP */
2969 rxcsum |= E1000_RXCSUM_CRCOFL;
2970
2971 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2972 wr32(E1000_RXCSUM, rxcsum);
f96a8a0b
CW		 2973 /*
2974 * Generate RSS hash based on TCP port numbers and/or
2975 * IPv4/v6 src and dst addresses since UDP cannot be
2976 * hashed reliably due to IP fragmentation
2977 */
2978
2979 mrqc = E1000_MRQC_RSS_FIELD_IPV4 |
2980 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2981 E1000_MRQC_RSS_FIELD_IPV6 |
2982 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2983 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
06cf2666
AD		 2984
2985 /* If VMDq is enabled then we set the appropriate mode for that, else
2986 * we default to RSS so that an RSS hash is calculated per packet even
2987 * if we are only using one queue */
2988 if (adapter->vfs_allocated_count) {
2989 if (hw->mac.type > e1000_82575) {
2990 /* Set the default pool for the PF's first queue */
2991 u32 vtctl = rd32(E1000_VT_CTL);
2992 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2993 E1000_VT_CTL_DISABLE_DEF_POOL);
2994 vtctl |= adapter->vfs_allocated_count <<
2995 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2996 wr32(E1000_VT_CTL, vtctl);
2997 }
a99955fc 2998 if (adapter->rss_queues > 1)
f96a8a0b 2999 mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
06cf2666 3000 else
f96a8a0b 3001 mrqc |= E1000_MRQC_ENABLE_VMDQ;
06cf2666 3002 } else {
f96a8a0b
CW		 3003 if (hw->mac.type != e1000_i211)
3004 mrqc |= E1000_MRQC_ENABLE_RSS_4Q;
06cf2666
AD		 3005 }
3006 igb_vmm_control(adapter);
3007
06cf2666
AD		 3008 wr32(E1000_MRQC, mrqc);
3009}
3010
9d5c8243
AK		 3011/**
3012 * igb_setup_rctl - configure the receive control registers
3013 * @adapter: Board private structure
3014 **/
d7ee5b3a 3015void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 3016{
3017 struct e1000_hw *hw = &adapter->hw;
3018 u32 rctl;
9d5c8243
AK		 3019
3020 rctl = rd32(E1000_RCTL);
3021
3022 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 3023 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 3024
69d728ba 3025 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 3026 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 3027
87cb7e8c
AK		 3028 /*
3029 * enable stripping of CRC. It's unlikely this will break BMC
3030 * redirection as it did with e1000. Newer features require
3031 * that the HW strips the CRC.
73cd78f1 3032 */
87cb7e8c 3033 rctl |= E1000_RCTL_SECRC;
9d5c8243 3034
559e9c49 3035 /* disable store bad packets and clear size bits. */
ec54d7d6 3036 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 3037
6ec43fe6
AD		 3038 /* enable LPE to prevent packets larger than max_frame_size */
3039 rctl |= E1000_RCTL_LPE;
9d5c8243 3040
952f72a8
AD		 3041 /* disable queue 0 to prevent tail write w/o re-config */
3042 wr32(E1000_RXDCTL(0), 0);
9d5c8243 3043
e1739522
AD		 3044 /* Attention!!! For SR-IOV PF driver operations you must enable
3045 * queue drop for all VF and PF queues to prevent head of line blocking
3046 * if an un-trusted VF does not provide descriptors to hardware.
3047 */
3048 if (adapter->vfs_allocated_count) {
e1739522
AD		 3049 /* set all queue drop enable bits */
3050 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 3051 }
3052
89eaefb6
BG		 3053 /* This is useful for sniffing bad packets. */
3054 if (adapter->netdev->features & NETIF_F_RXALL) {
3055 /* UPE and MPE will be handled by normal PROMISC logic
3056 * in e1000e_set_rx_mode */
3057 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
3058 E1000_RCTL_BAM | /* RX All Bcast Pkts */
3059 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
3060
3061 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
3062 E1000_RCTL_DPF | /* Allow filtered pause */
3063 E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
3064 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
3065 * and that breaks VLANs.
3066 */
3067 }
3068
9d5c8243
AK		 3069 wr32(E1000_RCTL, rctl);
3070}
3071
7d5753f0
AD		 3072static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
3073 int vfn)
3074{
3075 struct e1000_hw *hw = &adapter->hw;
3076 u32 vmolr;
3077
3078 /* if it isn't the PF check to see if VFs are enabled and
3079 * increase the size to support vlan tags */
3080 if (vfn < adapter->vfs_allocated_count &&
3081 adapter->vf_data[vfn].vlans_enabled)
3082 size += VLAN_TAG_SIZE;
3083
3084 vmolr = rd32(E1000_VMOLR(vfn));
3085 vmolr &= ~E1000_VMOLR_RLPML_MASK;
3086 vmolr |= size | E1000_VMOLR_LPE;
3087 wr32(E1000_VMOLR(vfn), vmolr);
3088
3089 return 0;
3090}
3091
e1739522
AD		 3092/**
3093 * igb_rlpml_set - set maximum receive packet size
3094 * @adapter: board private structure
3095 *
3096 * Configure maximum receivable packet size.
3097 **/
3098static void igb_rlpml_set(struct igb_adapter *adapter)
3099{
153285f9 3100 u32 max_frame_size = adapter->max_frame_size;
e1739522
AD		 3101 struct e1000_hw *hw = &adapter->hw;
3102 u16 pf_id = adapter->vfs_allocated_count;
3103
e1739522
AD		 3104 if (pf_id) {
3105 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
153285f9
AD		 3106 /*
3107 * If we're in VMDQ or SR-IOV mode, then set global RLPML
3108 * to our max jumbo frame size, in case we need to enable
3109 * jumbo frames on one of the rings later.
3110 * This will not pass over-length frames into the default
3111 * queue because it's gated by the VMOLR.RLPML.
3112 */
7d5753f0 3113 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 3114 }
3115
3116 wr32(E1000_RLPML, max_frame_size);
3117}
3118
8151d294
WM		 3119static inline void igb_set_vmolr(struct igb_adapter *adapter,
3120 int vfn, bool aupe)
7d5753f0
AD		 3121{
3122 struct e1000_hw *hw = &adapter->hw;
3123 u32 vmolr;
3124
3125 /*
3126 * This register exists only on 82576 and newer so if we are older then
3127 * we should exit and do nothing
3128 */
3129 if (hw->mac.type < e1000_82576)
3130 return;
3131
3132 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 3133 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
3134 if (aupe)
3135 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
3136 else
3137 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 3138
3139 /* clear all bits that might not be set */
3140 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
3141
a99955fc 3142 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 3143 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3144 /*
3145 * for VMDq only allow the VFs and pool 0 to accept broadcast and
3146 * multicast packets
3147 */
3148 if (vfn <= adapter->vfs_allocated_count)
3149 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
3150
3151 wr32(E1000_VMOLR(vfn), vmolr);
3152}
3153
85b430b4
AD		 3154/**
3155 * igb_configure_rx_ring - Configure a receive ring after Reset
3156 * @adapter: board private structure
3157 * @ring: receive ring to be configured
3158 *
3159 * Configure the Rx unit of the MAC after a reset.
3160 **/
d7ee5b3a
AD		 3161void igb_configure_rx_ring(struct igb_adapter *adapter,
3162 struct igb_ring *ring)
85b430b4
AD		 3163{
3164 struct e1000_hw *hw = &adapter->hw;
3165 u64 rdba = ring->dma;
3166 int reg_idx = ring->reg_idx;
a74420e0 3167 u32 srrctl = 0, rxdctl = 0;
85b430b4
AD		 3168
3169 /* disable the queue */
a74420e0 3170 wr32(E1000_RXDCTL(reg_idx), 0);
85b430b4
AD		 3171
3172 /* Set DMA base address registers */
3173 wr32(E1000_RDBAL(reg_idx),
3174 rdba & 0x00000000ffffffffULL);
3175 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
3176 wr32(E1000_RDLEN(reg_idx),
3177 ring->count * sizeof(union e1000_adv_rx_desc));
3178
3179 /* initialize head and tail */
fce99e34 3180 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
a74420e0 3181 wr32(E1000_RDH(reg_idx), 0);
fce99e34 3182 writel(0, ring->tail);
85b430b4 3183
952f72a8 3184 /* set descriptor configuration */
44390ca6 3185 srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
952f72a8 3186#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
44390ca6 3187 srrctl |= IGB_RXBUFFER_16384 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
952f72a8 3188#else
44390ca6 3189 srrctl |= (PAGE_SIZE / 2) >> E1000_SRRCTL_BSIZEPKT_SHIFT;
952f72a8 3190#endif
44390ca6 3191 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3c89f6d0 3192#ifdef CONFIG_IGB_PTP
06218a8d 3193 if (hw->mac.type >= e1000_82580)
757b77e2 3194 srrctl |= E1000_SRRCTL_TIMESTAMP;
3c89f6d0 3195#endif /* CONFIG_IGB_PTP */
e6bdb6fe
NN		 3196 /* Only set Drop Enable if we are supporting multiple queues */
3197 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
3198 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 3199
3200 wr32(E1000_SRRCTL(reg_idx), srrctl);
3201
7d5753f0 3202 /* set filtering for VMDQ pools */
8151d294 3203 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 3204
85b430b4
AD		 3205 rxdctl |= IGB_RX_PTHRESH;
3206 rxdctl |= IGB_RX_HTHRESH << 8;
3207 rxdctl |= IGB_RX_WTHRESH << 16;
a74420e0
AD		 3208
3209 /* enable receive descriptor fetching */
3210 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
85b430b4
AD		 3211 wr32(E1000_RXDCTL(reg_idx), rxdctl);
3212}
3213
9d5c8243
AK		 3214/**
3215 * igb_configure_rx - Configure receive Unit after Reset
3216 * @adapter: board private structure
3217 *
3218 * Configure the Rx unit of the MAC after a reset.
3219 **/
3220static void igb_configure_rx(struct igb_adapter *adapter)
3221{
9107584e 3222 int i;
9d5c8243 3223
68d480c4
AD		 3224 /* set UTA to appropriate mode */
3225 igb_set_uta(adapter);
3226
26ad9178
AD		 3227 /* set the correct pool for the PF default MAC address in entry 0 */
3228 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
3229 adapter->vfs_allocated_count);
3230
06cf2666
AD		 3231 /* Setup the HW Rx Head and Tail Descriptor Pointers and
3232 * the Base and Length of the Rx Descriptor Ring */
3233 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3234 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 3235}
3236
3237/**
3238 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 3239 * @tx_ring: Tx descriptor ring for a specific queue
3240 *
3241 * Free all transmit software resources
3242 **/
68fd9910 3243void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 3244{
3b644cf6 3245 igb_clean_tx_ring(tx_ring);
9d5c8243 3246
06034649
AD		 3247 vfree(tx_ring->tx_buffer_info);
3248 tx_ring->tx_buffer_info = NULL;
9d5c8243 3249
439705e1
AD		 3250 /* if not set, then don't free */
3251 if (!tx_ring->desc)
3252 return;
3253
59d71989
AD		 3254 dma_free_coherent(tx_ring->dev, tx_ring->size,
3255 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 3256
3257 tx_ring->desc = NULL;
3258}
3259
3260/**
3261 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3262 * @adapter: board private structure
3263 *
3264 * Free all transmit software resources
3265 **/
3266static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3267{
3268 int i;
3269
3270 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3271 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 3272}
3273
ebe42d16
AD		 3274void igb_unmap_and_free_tx_resource(struct igb_ring *ring,
3275 struct igb_tx_buffer *tx_buffer)
3276{
3277 if (tx_buffer->skb) {
3278 dev_kfree_skb_any(tx_buffer->skb);
3279 if (tx_buffer->dma)
3280 dma_unmap_single(ring->dev,
3281 tx_buffer->dma,
3282 tx_buffer->length,
3283 DMA_TO_DEVICE);
3284 } else if (tx_buffer->dma) {
3285 dma_unmap_page(ring->dev,
3286 tx_buffer->dma,
3287 tx_buffer->length,
3288 DMA_TO_DEVICE);
3289 }
3290 tx_buffer->next_to_watch = NULL;
3291 tx_buffer->skb = NULL;
3292 tx_buffer->dma = 0;
3293 /* buffer_info must be completely set up in the transmit path */
9d5c8243
AK		 3294}
3295
3296/**
3297 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 3298 * @tx_ring: ring to be cleaned
3299 **/
3b644cf6 3300static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243 3301{
06034649 3302 struct igb_tx_buffer *buffer_info;
9d5c8243 3303 unsigned long size;
6ad4edfc 3304 u16 i;
9d5c8243 3305
06034649 3306 if (!tx_ring->tx_buffer_info)
9d5c8243
AK		 3307 return;
3308 /* Free all the Tx ring sk_buffs */
3309
3310 for (i = 0; i < tx_ring->count; i++) {
06034649 3311 buffer_info = &tx_ring->tx_buffer_info[i];
80785298 3312 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 3313 }
3314
dad8a3b3
JF		 3315 netdev_tx_reset_queue(txring_txq(tx_ring));
3316
06034649
AD		 3317 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
3318 memset(tx_ring->tx_buffer_info, 0, size);
9d5c8243
AK		 3319
3320 /* Zero out the descriptor ring */
9d5c8243
AK		 3321 memset(tx_ring->desc, 0, tx_ring->size);
3322
3323 tx_ring->next_to_use = 0;
3324 tx_ring->next_to_clean = 0;
9d5c8243
AK		 3325}
3326
3327/**
3328 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3329 * @adapter: board private structure
3330 **/
3331static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3332{
3333 int i;
3334
3335 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3336 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 3337}
3338
3339/**
3340 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 3341 * @rx_ring: ring to clean the resources from
3342 *
3343 * Free all receive software resources
3344 **/
68fd9910 3345void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 3346{
3b644cf6 3347 igb_clean_rx_ring(rx_ring);
9d5c8243 3348
06034649
AD		 3349 vfree(rx_ring->rx_buffer_info);
3350 rx_ring->rx_buffer_info = NULL;
9d5c8243 3351
439705e1
AD		 3352 /* if not set, then don't free */
3353 if (!rx_ring->desc)
3354 return;
3355
59d71989
AD		 3356 dma_free_coherent(rx_ring->dev, rx_ring->size,
3357 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 3358
3359 rx_ring->desc = NULL;
3360}
3361
3362/**
3363 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3364 * @adapter: board private structure
3365 *
3366 * Free all receive software resources
3367 **/
3368static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3369{
3370 int i;
3371
3372 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3373 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 3374}
3375
3376/**
3377 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 3378 * @rx_ring: ring to free buffers from
3379 **/
3b644cf6 3380static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243 3381{
9d5c8243 3382 unsigned long size;
c023cd88 3383 u16 i;
9d5c8243 3384
06034649 3385 if (!rx_ring->rx_buffer_info)
9d5c8243 3386 return;
439705e1 3387
9d5c8243
AK		 3388 /* Free all the Rx ring sk_buffs */
3389 for (i = 0; i < rx_ring->count; i++) {
06034649 3390 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
9d5c8243 3391 if (buffer_info->dma) {
59d71989 3392 dma_unmap_single(rx_ring->dev,
80785298 3393 buffer_info->dma,
44390ca6 3394 IGB_RX_HDR_LEN,
59d71989 3395 DMA_FROM_DEVICE);
9d5c8243
AK		 3396 buffer_info->dma = 0;
3397 }
3398
3399 if (buffer_info->skb) {
3400 dev_kfree_skb(buffer_info->skb);
3401 buffer_info->skb = NULL;
3402 }
6ec43fe6 3403 if (buffer_info->page_dma) {
59d71989 3404 dma_unmap_page(rx_ring->dev,
80785298 3405 buffer_info->page_dma,
6ec43fe6 3406 PAGE_SIZE / 2,
59d71989 3407 DMA_FROM_DEVICE);
6ec43fe6
AD		 3408 buffer_info->page_dma = 0;
3409 }
9d5c8243 3410 if (buffer_info->page) {
9d5c8243
AK		 3411 put_page(buffer_info->page);
3412 buffer_info->page = NULL;
bf36c1a0 3413 buffer_info->page_offset = 0;
9d5c8243
AK		 3414 }
3415 }
3416
06034649
AD		 3417 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3418 memset(rx_ring->rx_buffer_info, 0, size);
9d5c8243
AK		 3419
3420 /* Zero out the descriptor ring */
3421 memset(rx_ring->desc, 0, rx_ring->size);
3422
3423 rx_ring->next_to_clean = 0;
3424 rx_ring->next_to_use = 0;
9d5c8243
AK		 3425}
3426
3427/**
3428 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3429 * @adapter: board private structure
3430 **/
3431static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3432{
3433 int i;
3434
3435 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3436 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 3437}
3438
3439/**
3440 * igb_set_mac - Change the Ethernet Address of the NIC
3441 * @netdev: network interface device structure
3442 * @p: pointer to an address structure
3443 *
3444 * Returns 0 on success, negative on failure
3445 **/
3446static int igb_set_mac(struct net_device *netdev, void *p)
3447{
3448 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 3449 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 3450 struct sockaddr *addr = p;
3451
3452 if (!is_valid_ether_addr(addr->sa_data))
3453 return -EADDRNOTAVAIL;
3454
3455 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 3456 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 3457
26ad9178
AD		 3458 /* set the correct pool for the new PF MAC address in entry 0 */
3459 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3460 adapter->vfs_allocated_count);
e1739522 3461
9d5c8243
AK		 3462 return 0;
3463}
3464
3465/**
68d480c4 3466 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 3467 * @netdev: network interface device structure
3468 *
68d480c4
AD		 3469 * Writes multicast address list to the MTA hash table.
3470 * Returns: -ENOMEM on failure
3471 * 0 on no addresses written
3472 * X on writing X addresses to MTA
9d5c8243 3473 **/
68d480c4 3474static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 3475{
3476 struct igb_adapter *adapter = netdev_priv(netdev);
3477 struct e1000_hw *hw = &adapter->hw;
22bedad3 3478 struct netdev_hw_addr *ha;
68d480c4 3479 u8 *mta_list;
9d5c8243
AK		 3480 int i;
3481
4cd24eaf 3482 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 3483 /* nothing to program, so clear mc list */
3484 igb_update_mc_addr_list(hw, NULL, 0);
3485 igb_restore_vf_multicasts(adapter);
3486 return 0;
3487 }
9d5c8243 3488
4cd24eaf 3489 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 3490 if (!mta_list)
3491 return -ENOMEM;
ff41f8dc 3492
68d480c4 3493 /* The shared function expects a packed array of only addresses. */
48e2f183 3494 i = 0;
22bedad3
JP		 3495 netdev_for_each_mc_addr(ha, netdev)
3496 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
68d480c4 3497
68d480c4
AD		 3498 igb_update_mc_addr_list(hw, mta_list, i);
3499 kfree(mta_list);
3500
4cd24eaf 3501 return netdev_mc_count(netdev);
68d480c4
AD		 3502}
3503
3504/**
3505 * igb_write_uc_addr_list - write unicast addresses to RAR table
3506 * @netdev: network interface device structure
3507 *
3508 * Writes unicast address list to the RAR table.
3509 * Returns: -ENOMEM on failure/insufficient address space
3510 * 0 on no addresses written
3511 * X on writing X addresses to the RAR table
3512 **/
3513static int igb_write_uc_addr_list(struct net_device *netdev)
3514{
3515 struct igb_adapter *adapter = netdev_priv(netdev);
3516 struct e1000_hw *hw = &adapter->hw;
3517 unsigned int vfn = adapter->vfs_allocated_count;
3518 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3519 int count = 0;
3520
3521 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 3522 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 3523 return -ENOMEM;
9d5c8243 3524
32e7bfc4 3525 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 3526 struct netdev_hw_addr *ha;
32e7bfc4
JP		 3527
3528 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 3529 if (!rar_entries)
3530 break;
26ad9178
AD		 3531 igb_rar_set_qsel(adapter, ha->addr,
3532 rar_entries--,
68d480c4
AD		 3533 vfn);
3534 count++;
ff41f8dc
AD		 3535 }
3536 }
3537 /* write the addresses in reverse order to avoid write combining */
3538 for (; rar_entries > 0 ; rar_entries--) {
3539 wr32(E1000_RAH(rar_entries), 0);
3540 wr32(E1000_RAL(rar_entries), 0);
3541 }
3542 wrfl();
3543
68d480c4
AD		 3544 return count;
3545}
3546
3547/**
3548 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3549 * @netdev: network interface device structure
3550 *
3551 * The set_rx_mode entry point is called whenever the unicast or multicast
3552 * address lists or the network interface flags are updated. This routine is
3553 * responsible for configuring the hardware for proper unicast, multicast,
3554 * promiscuous mode, and all-multi behavior.
3555 **/
3556static void igb_set_rx_mode(struct net_device *netdev)
3557{
3558 struct igb_adapter *adapter = netdev_priv(netdev);
3559 struct e1000_hw *hw = &adapter->hw;
3560 unsigned int vfn = adapter->vfs_allocated_count;
3561 u32 rctl, vmolr = 0;
3562 int count;
3563
3564 /* Check for Promiscuous and All Multicast modes */
3565 rctl = rd32(E1000_RCTL);
3566
3567 /* clear the effected bits */
3568 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3569
3570 if (netdev->flags & IFF_PROMISC) {
3571 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3572 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3573 } else {
3574 if (netdev->flags & IFF_ALLMULTI) {
3575 rctl |= E1000_RCTL_MPE;
3576 vmolr |= E1000_VMOLR_MPME;
3577 } else {
3578 /*
3579 * Write addresses to the MTA, if the attempt fails
25985edc 3580 * then we should just turn on promiscuous mode so
68d480c4
AD		 3581 * that we can at least receive multicast traffic
3582 */
3583 count = igb_write_mc_addr_list(netdev);
3584 if (count < 0) {
3585 rctl |= E1000_RCTL_MPE;
3586 vmolr |= E1000_VMOLR_MPME;
3587 } else if (count) {
3588 vmolr |= E1000_VMOLR_ROMPE;
3589 }
3590 }
3591 /*
3592 * Write addresses to available RAR registers, if there is not
3593 * sufficient space to store all the addresses then enable
25985edc 3594 * unicast promiscuous mode
68d480c4
AD		 3595 */
3596 count = igb_write_uc_addr_list(netdev);
3597 if (count < 0) {
3598 rctl |= E1000_RCTL_UPE;
3599 vmolr |= E1000_VMOLR_ROPE;
3600 }
3601 rctl |= E1000_RCTL_VFE;
28fc06f5 3602 }
68d480c4 3603 wr32(E1000_RCTL, rctl);
28fc06f5 3604
68d480c4
AD		 3605 /*
3606 * In order to support SR-IOV and eventually VMDq it is necessary to set
3607 * the VMOLR to enable the appropriate modes. Without this workaround
3608 * we will have issues with VLAN tag stripping not being done for frames
3609 * that are only arriving because we are the default pool
3610 */
f96a8a0b 3611 if ((hw->mac.type < e1000_82576) || (hw->mac.type > e1000_i350))
28fc06f5 3612 return;
9d5c8243 3613
68d480c4
AD		 3614 vmolr |= rd32(E1000_VMOLR(vfn)) &
3615 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3616 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3617 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3618}
3619
13800469
GR		 3620static void igb_check_wvbr(struct igb_adapter *adapter)
3621{
3622 struct e1000_hw *hw = &adapter->hw;
3623 u32 wvbr = 0;
3624
3625 switch (hw->mac.type) {
3626 case e1000_82576:
3627 case e1000_i350:
3628 if (!(wvbr = rd32(E1000_WVBR)))
3629 return;
3630 break;
3631 default:
3632 break;
3633 }
3634
3635 adapter->wvbr |= wvbr;
3636}
3637
3638#define IGB_STAGGERED_QUEUE_OFFSET 8
3639
3640static void igb_spoof_check(struct igb_adapter *adapter)
3641{
3642 int j;
3643
3644 if (!adapter->wvbr)
3645 return;
3646
3647 for(j = 0; j < adapter->vfs_allocated_count; j++) {
3648 if (adapter->wvbr & (1 << j) ||
3649 adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3650 dev_warn(&adapter->pdev->dev,
3651 "Spoof event(s) detected on VF %d\n", j);
3652 adapter->wvbr &=
3653 ~((1 << j) |
3654 (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3655 }
3656 }
3657}
3658
9d5c8243
AK		 3659/* Need to wait a few seconds after link up to get diagnostic information from
3660 * the phy */
3661static void igb_update_phy_info(unsigned long data)
3662{
3663 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3664 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3665}
3666
4d6b725e
AD		 3667/**
3668 * igb_has_link - check shared code for link and determine up/down
3669 * @adapter: pointer to driver private info
3670 **/
3145535a 3671bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3672{
3673 struct e1000_hw *hw = &adapter->hw;
3674 bool link_active = false;
3675 s32 ret_val = 0;
3676
3677 /* get_link_status is set on LSC (link status) interrupt or
3678 * rx sequence error interrupt. get_link_status will stay
3679 * false until the e1000_check_for_link establishes link
3680 * for copper adapters ONLY
3681 */
3682 switch (hw->phy.media_type) {
3683 case e1000_media_type_copper:
3684 if (hw->mac.get_link_status) {
3685 ret_val = hw->mac.ops.check_for_link(hw);
3686 link_active = !hw->mac.get_link_status;
3687 } else {
3688 link_active = true;
3689 }
3690 break;
4d6b725e
AD		 3691 case e1000_media_type_internal_serdes:
3692 ret_val = hw->mac.ops.check_for_link(hw);
3693 link_active = hw->mac.serdes_has_link;
3694 break;
3695 default:
3696 case e1000_media_type_unknown:
3697 break;
3698 }
3699
3700 return link_active;
3701}
3702
563988dc
SA		 3703static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
3704{
3705 bool ret = false;
3706 u32 ctrl_ext, thstat;
3707
f96a8a0b 3708 /* check for thermal sensor event on i350 copper only */
563988dc
SA		 3709 if (hw->mac.type == e1000_i350) {
3710 thstat = rd32(E1000_THSTAT);
3711 ctrl_ext = rd32(E1000_CTRL_EXT);
3712
3713 if ((hw->phy.media_type == e1000_media_type_copper) &&
3714 !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3715 ret = !!(thstat & event);
3716 }
3717 }
3718
3719 return ret;
3720}
3721
9d5c8243
AK		 3722/**
3723 * igb_watchdog - Timer Call-back
3724 * @data: pointer to adapter cast into an unsigned long
3725 **/
3726static void igb_watchdog(unsigned long data)
3727{
3728 struct igb_adapter *adapter = (struct igb_adapter *)data;
3729 /* Do the rest outside of interrupt context */
3730 schedule_work(&adapter->watchdog_task);
3731}
3732
3733static void igb_watchdog_task(struct work_struct *work)
3734{
3735 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3736 struct igb_adapter,
3737 watchdog_task);
9d5c8243 3738 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3739 struct net_device *netdev = adapter->netdev;
563988dc 3740 u32 link;
7a6ea550 3741 int i;
9d5c8243 3742
4d6b725e 3743 link = igb_has_link(adapter);
9d5c8243 3744 if (link) {
749ab2cd
YZ		 3745 /* Cancel scheduled suspend requests. */
3746 pm_runtime_resume(netdev->dev.parent);
3747
9d5c8243
AK		 3748 if (!netif_carrier_ok(netdev)) {
3749 u32 ctrl;
330a6d6a
AD		 3750 hw->mac.ops.get_speed_and_duplex(hw,
3751 &adapter->link_speed,
3752 &adapter->link_duplex);
9d5c8243
AK		 3753
3754 ctrl = rd32(E1000_CTRL);
527d47c1 3755 /* Links status message must follow this format */
876d2d6f
JK		 3756 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s "
3757 "Duplex, Flow Control: %s\n",
559e9c49
AD		 3758 netdev->name,
3759 adapter->link_speed,
3760 adapter->link_duplex == FULL_DUPLEX ?
876d2d6f
JK		 3761 "Full" : "Half",
3762 (ctrl & E1000_CTRL_TFCE) &&
3763 (ctrl & E1000_CTRL_RFCE) ? "RX/TX" :
3764 (ctrl & E1000_CTRL_RFCE) ? "RX" :
3765 (ctrl & E1000_CTRL_TFCE) ? "TX" : "None");
9d5c8243 3766
563988dc 3767 /* check for thermal sensor event */
876d2d6f
JK		 3768 if (igb_thermal_sensor_event(hw,
3769 E1000_THSTAT_LINK_THROTTLE)) {
3770 netdev_info(netdev, "The network adapter link "
3771 "speed was downshifted because it "
3772 "overheated\n");
7ef5ed1c 3773 }
563988dc 3774
d07f3e37 3775 /* adjust timeout factor according to speed/duplex */
9d5c8243
AK		 3776 adapter->tx_timeout_factor = 1;
3777 switch (adapter->link_speed) {
3778 case SPEED_10:
9d5c8243
AK		 3779 adapter->tx_timeout_factor = 14;
3780 break;
3781 case SPEED_100:
9d5c8243
AK		 3782 /* maybe add some timeout factor ? */
3783 break;
3784 }
3785
3786 netif_carrier_on(netdev);
9d5c8243 3787
4ae196df 3788 igb_ping_all_vfs(adapter);
17dc566c 3789 igb_check_vf_rate_limit(adapter);
4ae196df 3790
4b1a9877 3791 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3792 if (!test_bit(__IGB_DOWN, &adapter->state))
3793 mod_timer(&adapter->phy_info_timer,
3794 round_jiffies(jiffies + 2 * HZ));
3795 }
3796 } else {
3797 if (netif_carrier_ok(netdev)) {
3798 adapter->link_speed = 0;
3799 adapter->link_duplex = 0;
563988dc
SA		 3800
3801 /* check for thermal sensor event */
876d2d6f
JK		 3802 if (igb_thermal_sensor_event(hw,
3803 E1000_THSTAT_PWR_DOWN)) {
3804 netdev_err(netdev, "The network adapter was "
3805 "stopped because it overheated\n");
7ef5ed1c 3806 }
563988dc 3807
527d47c1
AD		 3808 /* Links status message must follow this format */
3809 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3810 netdev->name);
9d5c8243 3811 netif_carrier_off(netdev);
4b1a9877 3812
4ae196df
AD		 3813 igb_ping_all_vfs(adapter);
3814
4b1a9877 3815 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3816 if (!test_bit(__IGB_DOWN, &adapter->state))
3817 mod_timer(&adapter->phy_info_timer,
3818 round_jiffies(jiffies + 2 * HZ));
749ab2cd
YZ		 3819
3820 pm_schedule_suspend(netdev->dev.parent,
3821 MSEC_PER_SEC * 5);
9d5c8243
AK		 3822 }
3823 }
3824
12dcd86b
ED		 3825 spin_lock(&adapter->stats64_lock);
3826 igb_update_stats(adapter, &adapter->stats64);
3827 spin_unlock(&adapter->stats64_lock);
9d5c8243 3828
dbabb065 3829 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3830 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3831 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3832 /* We've lost link, so the controller stops DMA,
3833 * but we've got queued Tx work that's never going
3834 * to get done, so reset controller to flush Tx.
3835 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3836 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3837 adapter->tx_timeout_count++;
3838 schedule_work(&adapter->reset_task);
3839 /* return immediately since reset is imminent */
3840 return;
3841 }
9d5c8243 3842 }
9d5c8243 3843
dbabb065 3844 /* Force detection of hung controller every watchdog period */
6d095fa8 3845 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
dbabb065 3846 }
f7ba205e 3847
9d5c8243 3848 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3849 if (adapter->msix_entries) {
047e0030 3850 u32 eics = 0;
0d1ae7f4
AD		 3851 for (i = 0; i < adapter->num_q_vectors; i++)
3852 eics |= adapter->q_vector[i]->eims_value;
7a6ea550
AD		 3853 wr32(E1000_EICS, eics);
3854 } else {
3855 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3856 }
9d5c8243 3857
13800469
GR		 3858 igb_spoof_check(adapter);
3859
9d5c8243
AK		 3860 /* Reset the timer */
3861 if (!test_bit(__IGB_DOWN, &adapter->state))
3862 mod_timer(&adapter->watchdog_timer,
3863 round_jiffies(jiffies + 2 * HZ));
3864}
3865
3866enum latency_range {
3867 lowest_latency = 0,
3868 low_latency = 1,
3869 bulk_latency = 2,
3870 latency_invalid = 255
3871};
3872
6eb5a7f1
AD		 3873/**
3874 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3875 *
3876 * Stores a new ITR value based on strictly on packet size. This
3877 * algorithm is less sophisticated than that used in igb_update_itr,
3878 * due to the difficulty of synchronizing statistics across multiple
eef35c2d 3879 * receive rings. The divisors and thresholds used by this function
6eb5a7f1
AD		 3880 * were determined based on theoretical maximum wire speed and testing
3881 * data, in order to minimize response time while increasing bulk
3882 * throughput.
3883 * This functionality is controlled by the InterruptThrottleRate module
3884 * parameter (see igb_param.c)
3885 * NOTE: This function is called only when operating in a multiqueue
3886 * receive environment.
047e0030 3887 * @q_vector: pointer to q_vector
6eb5a7f1 3888 **/
047e0030 3889static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3890{
047e0030 3891 int new_val = q_vector->itr_val;
6eb5a7f1 3892 int avg_wire_size = 0;
047e0030 3893 struct igb_adapter *adapter = q_vector->adapter;
12dcd86b 3894 unsigned int packets;
9d5c8243 3895
6eb5a7f1
AD		 3896 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3897 * ints/sec - ITR timer value of 120 ticks.
3898 */
3899 if (adapter->link_speed != SPEED_1000) {
0ba82994 3900 new_val = IGB_4K_ITR;
6eb5a7f1 3901 goto set_itr_val;
9d5c8243 3902 }
047e0030 3903
0ba82994
AD		 3904 packets = q_vector->rx.total_packets;
3905 if (packets)
3906 avg_wire_size = q_vector->rx.total_bytes / packets;
047e0030 3907
0ba82994
AD		 3908 packets = q_vector->tx.total_packets;
3909 if (packets)
3910 avg_wire_size = max_t(u32, avg_wire_size,
3911 q_vector->tx.total_bytes / packets);
047e0030
AD		 3912
3913 /* if avg_wire_size isn't set no work was done */
3914 if (!avg_wire_size)
3915 goto clear_counts;
9d5c8243 3916
6eb5a7f1
AD		 3917 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3918 avg_wire_size += 24;
3919
3920 /* Don't starve jumbo frames */
3921 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3922
6eb5a7f1
AD		 3923 /* Give a little boost to mid-size frames */
3924 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3925 new_val = avg_wire_size / 3;
3926 else
3927 new_val = avg_wire_size / 2;
9d5c8243 3928
0ba82994
AD		 3929 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3930 if (new_val < IGB_20K_ITR &&
3931 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3932 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3933 new_val = IGB_20K_ITR;
abe1c363 3934
6eb5a7f1 3935set_itr_val:
047e0030
AD		 3936 if (new_val != q_vector->itr_val) {
3937 q_vector->itr_val = new_val;
3938 q_vector->set_itr = 1;
9d5c8243 3939 }
6eb5a7f1 3940clear_counts:
0ba82994
AD		 3941 q_vector->rx.total_bytes = 0;
3942 q_vector->rx.total_packets = 0;
3943 q_vector->tx.total_bytes = 0;
3944 q_vector->tx.total_packets = 0;
9d5c8243
AK		 3945}
3946
3947/**
3948 * igb_update_itr - update the dynamic ITR value based on statistics
3949 * Stores a new ITR value based on packets and byte
3950 * counts during the last interrupt. The advantage of per interrupt
3951 * computation is faster updates and more accurate ITR for the current
3952 * traffic pattern. Constants in this function were computed
3953 * based on theoretical maximum wire speed and thresholds were set based
3954 * on testing data as well as attempting to minimize response time
3955 * while increasing bulk throughput.
3956 * this functionality is controlled by the InterruptThrottleRate module
3957 * parameter (see igb_param.c)
3958 * NOTE: These calculations are only valid when operating in a single-
3959 * queue environment.
0ba82994
AD		 3960 * @q_vector: pointer to q_vector
3961 * @ring_container: ring info to update the itr for
9d5c8243 3962 **/
0ba82994
AD		 3963static void igb_update_itr(struct igb_q_vector *q_vector,
3964 struct igb_ring_container *ring_container)
9d5c8243 3965{
0ba82994
AD		 3966 unsigned int packets = ring_container->total_packets;
3967 unsigned int bytes = ring_container->total_bytes;
3968 u8 itrval = ring_container->itr;
9d5c8243 3969
0ba82994 3970 /* no packets, exit with status unchanged */
9d5c8243 3971 if (packets == 0)
0ba82994 3972 return;
9d5c8243 3973
0ba82994 3974 switch (itrval) {
9d5c8243
AK		 3975 case lowest_latency:
3976 /* handle TSO and jumbo frames */
3977 if (bytes/packets > 8000)
0ba82994 3978 itrval = bulk_latency;
9d5c8243 3979 else if ((packets < 5) && (bytes > 512))
0ba82994 3980 itrval = low_latency;
9d5c8243
AK		 3981 break;
3982 case low_latency: /* 50 usec aka 20000 ints/s */
3983 if (bytes > 10000) {
3984 /* this if handles the TSO accounting */
3985 if (bytes/packets > 8000) {
0ba82994 3986 itrval = bulk_latency;
9d5c8243 3987 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
0ba82994 3988 itrval = bulk_latency;
9d5c8243 3989 } else if ((packets > 35)) {
0ba82994 3990 itrval = lowest_latency;
9d5c8243
AK		 3991 }
3992 } else if (bytes/packets > 2000) {
0ba82994 3993 itrval = bulk_latency;
9d5c8243 3994 } else if (packets <= 2 && bytes < 512) {
0ba82994 3995 itrval = lowest_latency;
9d5c8243
AK		 3996 }
3997 break;
3998 case bulk_latency: /* 250 usec aka 4000 ints/s */
3999 if (bytes > 25000) {
4000 if (packets > 35)
0ba82994 4001 itrval = low_latency;
1e5c3d21 4002 } else if (bytes < 1500) {
0ba82994 4003 itrval = low_latency;
9d5c8243
AK		 4004 }
4005 break;
4006 }
4007
0ba82994
AD		 4008 /* clear work counters since we have the values we need */
4009 ring_container->total_bytes = 0;
4010 ring_container->total_packets = 0;
4011
4012 /* write updated itr to ring container */
4013 ring_container->itr = itrval;
9d5c8243
AK		 4014}
4015
0ba82994 4016static void igb_set_itr(struct igb_q_vector *q_vector)
9d5c8243 4017{
0ba82994 4018 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4019 u32 new_itr = q_vector->itr_val;
0ba82994 4020 u8 current_itr = 0;
9d5c8243
AK		 4021
4022 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
4023 if (adapter->link_speed != SPEED_1000) {
4024 current_itr = 0;
0ba82994 4025 new_itr = IGB_4K_ITR;
9d5c8243
AK		 4026 goto set_itr_now;
4027 }
4028
0ba82994
AD		 4029 igb_update_itr(q_vector, &q_vector->tx);
4030 igb_update_itr(q_vector, &q_vector->rx);
9d5c8243 4031
0ba82994 4032 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
9d5c8243 4033
6eb5a7f1 4034 /* conservative mode (itr 3) eliminates the lowest_latency setting */
0ba82994
AD		 4035 if (current_itr == lowest_latency &&
4036 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4037 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
6eb5a7f1
AD		 4038 current_itr = low_latency;
4039
9d5c8243
AK		 4040 switch (current_itr) {
4041 /* counts and packets in update_itr are dependent on these numbers */
4042 case lowest_latency:
0ba82994 4043 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
9d5c8243
AK		 4044 break;
4045 case low_latency:
0ba82994 4046 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
9d5c8243
AK		 4047 break;
4048 case bulk_latency:
0ba82994 4049 new_itr = IGB_4K_ITR; /* 4,000 ints/sec */
9d5c8243
AK		 4050 break;
4051 default:
4052 break;
4053 }
4054
4055set_itr_now:
047e0030 4056 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 4057 /* this attempts to bias the interrupt rate towards Bulk
4058 * by adding intermediate steps when interrupt rate is
4059 * increasing */
047e0030
AD		 4060 new_itr = new_itr > q_vector->itr_val ?
4061 max((new_itr * q_vector->itr_val) /
4062 (new_itr + (q_vector->itr_val >> 2)),
0ba82994 4063 new_itr) :
9d5c8243
AK		 4064 new_itr;
4065 /* Don't write the value here; it resets the adapter's
4066 * internal timer, and causes us to delay far longer than
4067 * we should between interrupts. Instead, we write the ITR
4068 * value at the beginning of the next interrupt so the timing
4069 * ends up being correct.
4070 */
047e0030
AD		 4071 q_vector->itr_val = new_itr;
4072 q_vector->set_itr = 1;
9d5c8243 4073 }
9d5c8243
AK		 4074}
4075
c50b52a0
SH		 4076static void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
4077 u32 type_tucmd, u32 mss_l4len_idx)
7d13a7d0
AD		 4078{
4079 struct e1000_adv_tx_context_desc *context_desc;
4080 u16 i = tx_ring->next_to_use;
4081
4082 context_desc = IGB_TX_CTXTDESC(tx_ring, i);
4083
4084 i++;
4085 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
4086
4087 /* set bits to identify this as an advanced context descriptor */
4088 type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
4089
4090 /* For 82575, context index must be unique per ring. */
866cff06 4091 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
7d13a7d0
AD		 4092 mss_l4len_idx |= tx_ring->reg_idx << 4;
4093
4094 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
4095 context_desc->seqnum_seed = 0;
4096 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
4097 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
4098}
4099
7af40ad9
AD		 4100static int igb_tso(struct igb_ring *tx_ring,
4101 struct igb_tx_buffer *first,
4102 u8 *hdr_len)
9d5c8243 4103{
7af40ad9 4104 struct sk_buff *skb = first->skb;
7d13a7d0
AD		 4105 u32 vlan_macip_lens, type_tucmd;
4106 u32 mss_l4len_idx, l4len;
4107
4108 if (!skb_is_gso(skb))
4109 return 0;
9d5c8243
AK		 4110
4111 if (skb_header_cloned(skb)) {
7af40ad9 4112 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
9d5c8243
AK		 4113 if (err)
4114 return err;
4115 }
4116
7d13a7d0
AD		 4117 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4118 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
9d5c8243 4119
7af40ad9 4120 if (first->protocol == __constant_htons(ETH_P_IP)) {
9d5c8243
AK		 4121 struct iphdr *iph = ip_hdr(skb);
4122 iph->tot_len = 0;
4123 iph->check = 0;
4124 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
4125 iph->daddr, 0,
4126 IPPROTO_TCP,
4127 0);
7d13a7d0 4128 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
7af40ad9
AD		 4129 first->tx_flags |= IGB_TX_FLAGS_TSO |
4130 IGB_TX_FLAGS_CSUM |
4131 IGB_TX_FLAGS_IPV4;
8e1e8a47 4132 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 4133 ipv6_hdr(skb)->payload_len = 0;
4134 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4135 &ipv6_hdr(skb)->daddr,
4136 0, IPPROTO_TCP, 0);
7af40ad9
AD		 4137 first->tx_flags |= IGB_TX_FLAGS_TSO |
4138 IGB_TX_FLAGS_CSUM;
9d5c8243
AK		 4139 }
4140
7af40ad9 4141 /* compute header lengths */
7d13a7d0
AD		 4142 l4len = tcp_hdrlen(skb);
4143 *hdr_len = skb_transport_offset(skb) + l4len;
9d5c8243 4144
7af40ad9
AD		 4145 /* update gso size and bytecount with header size */
4146 first->gso_segs = skb_shinfo(skb)->gso_segs;
4147 first->bytecount += (first->gso_segs - 1) * *hdr_len;
4148
9d5c8243 4149 /* MSS L4LEN IDX */
7d13a7d0
AD		 4150 mss_l4len_idx = l4len << E1000_ADVTXD_L4LEN_SHIFT;
4151 mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
9d5c8243 4152
7d13a7d0
AD		 4153 /* VLAN MACLEN IPLEN */
4154 vlan_macip_lens = skb_network_header_len(skb);
4155 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
7af40ad9 4156 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
9d5c8243 4157
7d13a7d0 4158 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
9d5c8243 4159
7d13a7d0 4160 return 1;
9d5c8243
AK		 4161}
4162
7af40ad9 4163static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
9d5c8243 4164{
7af40ad9 4165 struct sk_buff *skb = first->skb;
7d13a7d0
AD		 4166 u32 vlan_macip_lens = 0;
4167 u32 mss_l4len_idx = 0;
4168 u32 type_tucmd = 0;
9d5c8243 4169
7d13a7d0 4170 if (skb->ip_summed != CHECKSUM_PARTIAL) {
7af40ad9
AD		 4171 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN))
4172 return;
7d13a7d0
AD		 4173 } else {
4174 u8 l4_hdr = 0;
7af40ad9 4175 switch (first->protocol) {
7d13a7d0
AD		 4176 case __constant_htons(ETH_P_IP):
4177 vlan_macip_lens |= skb_network_header_len(skb);
4178 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4179 l4_hdr = ip_hdr(skb)->protocol;
4180 break;
4181 case __constant_htons(ETH_P_IPV6):
4182 vlan_macip_lens |= skb_network_header_len(skb);
4183 l4_hdr = ipv6_hdr(skb)->nexthdr;
4184 break;
4185 default:
4186 if (unlikely(net_ratelimit())) {
4187 dev_warn(tx_ring->dev,
4188 "partial checksum but proto=%x!\n",
7af40ad9 4189 first->protocol);
fa4a7ef3 4190 }
7d13a7d0
AD		 4191 break;
4192 }
fa4a7ef3 4193
7d13a7d0
AD		 4194 switch (l4_hdr) {
4195 case IPPROTO_TCP:
4196 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4197 mss_l4len_idx = tcp_hdrlen(skb) <<
4198 E1000_ADVTXD_L4LEN_SHIFT;
4199 break;
4200 case IPPROTO_SCTP:
4201 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
4202 mss_l4len_idx = sizeof(struct sctphdr) <<
4203 E1000_ADVTXD_L4LEN_SHIFT;
4204 break;
4205 case IPPROTO_UDP:
4206 mss_l4len_idx = sizeof(struct udphdr) <<
4207 E1000_ADVTXD_L4LEN_SHIFT;
4208 break;
4209 default:
4210 if (unlikely(net_ratelimit())) {
4211 dev_warn(tx_ring->dev,
4212 "partial checksum but l4 proto=%x!\n",
4213 l4_hdr);
44b0cda3 4214 }
7d13a7d0 4215 break;
9d5c8243 4216 }
7af40ad9
AD		 4217
4218 /* update TX checksum flag */
4219 first->tx_flags |= IGB_TX_FLAGS_CSUM;
7d13a7d0 4220 }
9d5c8243 4221
7d13a7d0 4222 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
7af40ad9 4223 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
9d5c8243 4224
7d13a7d0 4225 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
9d5c8243
AK		 4226}
4227
e032afc8
AD		 4228static __le32 igb_tx_cmd_type(u32 tx_flags)
4229{
4230 /* set type for advanced descriptor with frame checksum insertion */
4231 __le32 cmd_type = cpu_to_le32(E1000_ADVTXD_DTYP_DATA |
4232 E1000_ADVTXD_DCMD_IFCS |
4233 E1000_ADVTXD_DCMD_DEXT);
4234
4235 /* set HW vlan bit if vlan is present */
4236 if (tx_flags & IGB_TX_FLAGS_VLAN)
4237 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_VLE);
4238
3c89f6d0 4239#ifdef CONFIG_IGB_PTP
e032afc8 4240 /* set timestamp bit if present */
1f6e8178 4241 if (unlikely(tx_flags & IGB_TX_FLAGS_TSTAMP))
e032afc8 4242 cmd_type |= cpu_to_le32(E1000_ADVTXD_MAC_TSTAMP);
3c89f6d0 4243#endif /* CONFIG_IGB_PTP */
e032afc8
AD		 4244
4245 /* set segmentation bits for TSO */
4246 if (tx_flags & IGB_TX_FLAGS_TSO)
4247 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_TSE);
4248
4249 return cmd_type;
4250}
4251
7af40ad9
AD		 4252static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
4253 union e1000_adv_tx_desc *tx_desc,
4254 u32 tx_flags, unsigned int paylen)
e032afc8
AD		 4255{
4256 u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
4257
4258 /* 82575 requires a unique index per ring if any offload is enabled */
4259 if ((tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_VLAN)) &&
866cff06 4260 test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
e032afc8
AD		 4261 olinfo_status |= tx_ring->reg_idx << 4;
4262
4263 /* insert L4 checksum */
4264 if (tx_flags & IGB_TX_FLAGS_CSUM) {
4265 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4266
4267 /* insert IPv4 checksum */
4268 if (tx_flags & IGB_TX_FLAGS_IPV4)
4269 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4270 }
4271
7af40ad9 4272 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
e032afc8
AD		 4273}
4274
ebe42d16
AD		 4275/*
4276 * The largest size we can write to the descriptor is 65535. In order to
4277 * maintain a power of two alignment we have to limit ourselves to 32K.
4278 */
4279#define IGB_MAX_TXD_PWR 15
7af40ad9 4280#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
9d5c8243 4281
7af40ad9
AD		 4282static void igb_tx_map(struct igb_ring *tx_ring,
4283 struct igb_tx_buffer *first,
ebe42d16 4284 const u8 hdr_len)
9d5c8243 4285{
7af40ad9 4286 struct sk_buff *skb = first->skb;
ebe42d16
AD		 4287 struct igb_tx_buffer *tx_buffer_info;
4288 union e1000_adv_tx_desc *tx_desc;
4289 dma_addr_t dma;
4290 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
4291 unsigned int data_len = skb->data_len;
4292 unsigned int size = skb_headlen(skb);
4293 unsigned int paylen = skb->len - hdr_len;
4294 __le32 cmd_type;
7af40ad9 4295 u32 tx_flags = first->tx_flags;
ebe42d16 4296 u16 i = tx_ring->next_to_use;
ebe42d16
AD		 4297
4298 tx_desc = IGB_TX_DESC(tx_ring, i);
4299
7af40ad9 4300 igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, paylen);
ebe42d16
AD		 4301 cmd_type = igb_tx_cmd_type(tx_flags);
4302
4303 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
4304 if (dma_mapping_error(tx_ring->dev, dma))
6366ad33 4305 goto dma_error;
9d5c8243 4306
ebe42d16
AD		 4307 /* record length, and DMA address */
4308 first->length = size;
4309 first->dma = dma;
ebe42d16
AD		 4310 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4311
4312 for (;;) {
4313 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
4314 tx_desc->read.cmd_type_len =
4315 cmd_type | cpu_to_le32(IGB_MAX_DATA_PER_TXD);
4316
4317 i++;
4318 tx_desc++;
4319 if (i == tx_ring->count) {
4320 tx_desc = IGB_TX_DESC(tx_ring, 0);
4321 i = 0;
4322 }
4323
4324 dma += IGB_MAX_DATA_PER_TXD;
4325 size -= IGB_MAX_DATA_PER_TXD;
4326
4327 tx_desc->read.olinfo_status = 0;
4328 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4329 }
4330
4331 if (likely(!data_len))
4332 break;
2bbfebe2 4333
ebe42d16 4334 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
9d5c8243 4335
65689fef 4336 i++;
ebe42d16
AD		 4337 tx_desc++;
4338 if (i == tx_ring->count) {
4339 tx_desc = IGB_TX_DESC(tx_ring, 0);
65689fef 4340 i = 0;
ebe42d16 4341 }
65689fef 4342
9e903e08 4343 size = skb_frag_size(frag);
ebe42d16
AD		 4344 data_len -= size;
4345
4346 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
4347 size, DMA_TO_DEVICE);
4348 if (dma_mapping_error(tx_ring->dev, dma))
6366ad33
AD		 4349 goto dma_error;
4350
ebe42d16
AD		 4351 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4352 tx_buffer_info->length = size;
4353 tx_buffer_info->dma = dma;
4354
4355 tx_desc->read.olinfo_status = 0;
4356 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4357
4358 frag++;
9d5c8243
AK		 4359 }
4360
bdbc0631
ED		 4361 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
4362
ebe42d16
AD		 4363 /* write last descriptor with RS and EOP bits */
4364 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IGB_TXD_DCMD);
6b8f0922
BG		 4365 if (unlikely(skb->no_fcs))
4366 cmd_type &= ~(cpu_to_le32(E1000_ADVTXD_DCMD_IFCS));
ebe42d16 4367 tx_desc->read.cmd_type_len = cmd_type;
8542db05
AD		 4368
4369 /* set the timestamp */
4370 first->time_stamp = jiffies;
4371
ebe42d16
AD		 4372 /*
4373 * Force memory writes to complete before letting h/w know there
4374 * are new descriptors to fetch. (Only applicable for weak-ordered
4375 * memory model archs, such as IA-64).
4376 *
4377 * We also need this memory barrier to make certain all of the
4378 * status bits have been updated before next_to_watch is written.
4379 */
4380 wmb();
4381
8542db05 4382 /* set next_to_watch value indicating a packet is present */
ebe42d16 4383 first->next_to_watch = tx_desc;
9d5c8243 4384
ebe42d16
AD		 4385 i++;
4386 if (i == tx_ring->count)
4387 i = 0;
6366ad33 4388
ebe42d16 4389 tx_ring->next_to_use = i;
6366ad33 4390
ebe42d16 4391 writel(i, tx_ring->tail);
6366ad33 4392
ebe42d16
AD		 4393 /* we need this if more than one processor can write to our tail
4394 * at a time, it syncronizes IO on IA64/Altix systems */
4395 mmiowb();
4396
4397 return;
4398
4399dma_error:
4400 dev_err(tx_ring->dev, "TX DMA map failed\n");
4401
4402 /* clear dma mappings for failed tx_buffer_info map */
4403 for (;;) {
4404 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4405 igb_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
4406 if (tx_buffer_info == first)
4407 break;
a77ff709
NN		 4408 if (i == 0)
4409 i = tx_ring->count;
6366ad33 4410 i--;
6366ad33
AD		 4411 }
4412
9d5c8243 4413 tx_ring->next_to_use = i;
9d5c8243
AK		 4414}
4415
6ad4edfc 4416static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
9d5c8243 4417{
e694e964
AD		 4418 struct net_device *netdev = tx_ring->netdev;
4419
661086df 4420 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 4421
9d5c8243
AK		 4422 /* Herbert's original patch had:
4423 * smp_mb__after_netif_stop_queue();
4424 * but since that doesn't exist yet, just open code it. */
4425 smp_mb();
4426
4427 /* We need to check again in a case another CPU has just
4428 * made room available. */
c493ea45 4429 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 4430 return -EBUSY;
4431
4432 /* A reprieve! */
661086df 4433 netif_wake_subqueue(netdev, tx_ring->queue_index);
12dcd86b
ED		 4434
4435 u64_stats_update_begin(&tx_ring->tx_syncp2);
4436 tx_ring->tx_stats.restart_queue2++;
4437 u64_stats_update_end(&tx_ring->tx_syncp2);
4438
9d5c8243
AK		 4439 return 0;
4440}
4441
6ad4edfc 4442static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
9d5c8243 4443{
c493ea45 4444 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 4445 return 0;
e694e964 4446 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 4447}
4448
cd392f5c
AD		 4449netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
4450 struct igb_ring *tx_ring)
9d5c8243 4451{
1f6e8178
MV		 4452#ifdef CONFIG_IGB_PTP
4453 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
4454#endif /* CONFIG_IGB_PTP */
8542db05 4455 struct igb_tx_buffer *first;
ebe42d16 4456 int tso;
91d4ee33 4457 u32 tx_flags = 0;
31f6adbb 4458 __be16 protocol = vlan_get_protocol(skb);
91d4ee33 4459 u8 hdr_len = 0;
9d5c8243 4460
9d5c8243
AK		 4461 /* need: 1 descriptor per page,
4462 * + 2 desc gap to keep tail from touching head,
4463 * + 1 desc for skb->data,
4464 * + 1 desc for context descriptor,
4465 * otherwise try next time */
e694e964 4466 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 4467 /* this is a hard error */
9d5c8243
AK		 4468 return NETDEV_TX_BUSY;
4469 }
33af6bcc 4470
7af40ad9
AD		 4471 /* record the location of the first descriptor for this packet */
4472 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
4473 first->skb = skb;
4474 first->bytecount = skb->len;
4475 first->gso_segs = 1;
4476
3c89f6d0 4477#ifdef CONFIG_IGB_PTP
1f6e8178
MV		 4478 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
4479 !(adapter->ptp_tx_skb))) {
2244d07b 4480 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
33af6bcc 4481 tx_flags |= IGB_TX_FLAGS_TSTAMP;
1f6e8178
MV		 4482
4483 adapter->ptp_tx_skb = skb_get(skb);
4484 if (adapter->hw.mac.type == e1000_82576)
4485 schedule_work(&adapter->ptp_tx_work);
33af6bcc 4486 }
3c89f6d0 4487#endif /* CONFIG_IGB_PTP */
9d5c8243 4488
eab6d18d 4489 if (vlan_tx_tag_present(skb)) {
9d5c8243
AK		 4490 tx_flags |= IGB_TX_FLAGS_VLAN;
4491 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4492 }
4493
7af40ad9
AD		 4494 /* record initial flags and protocol */
4495 first->tx_flags = tx_flags;
4496 first->protocol = protocol;
cdfd01fc 4497
7af40ad9
AD		 4498 tso = igb_tso(tx_ring, first, &hdr_len);
4499 if (tso < 0)
7d13a7d0 4500 goto out_drop;
7af40ad9
AD		 4501 else if (!tso)
4502 igb_tx_csum(tx_ring, first);
9d5c8243 4503
7af40ad9 4504 igb_tx_map(tx_ring, first, hdr_len);
85ad76b2
AD		 4505
4506 /* Make sure there is space in the ring for the next send. */
e694e964 4507 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 4508
9d5c8243 4509 return NETDEV_TX_OK;
7d13a7d0
AD		 4510
4511out_drop:
7af40ad9
AD		 4512 igb_unmap_and_free_tx_resource(tx_ring, first);
4513
7d13a7d0 4514 return NETDEV_TX_OK;
9d5c8243
AK		 4515}
4516
1cc3bd87
AD		 4517static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
4518 struct sk_buff *skb)
4519{
4520 unsigned int r_idx = skb->queue_mapping;
4521
4522 if (r_idx >= adapter->num_tx_queues)
4523 r_idx = r_idx % adapter->num_tx_queues;
4524
4525 return adapter->tx_ring[r_idx];
4526}
4527
cd392f5c
AD		 4528static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
4529 struct net_device *netdev)
9d5c8243
AK		 4530{
4531 struct igb_adapter *adapter = netdev_priv(netdev);
b1a436c3
AD		 4532
4533 if (test_bit(__IGB_DOWN, &adapter->state)) {
4534 dev_kfree_skb_any(skb);
4535 return NETDEV_TX_OK;
4536 }
4537
4538 if (skb->len <= 0) {
4539 dev_kfree_skb_any(skb);
4540 return NETDEV_TX_OK;
4541 }
4542
1cc3bd87
AD		 4543 /*
4544 * The minimum packet size with TCTL.PSP set is 17 so pad the skb
4545 * in order to meet this minimum size requirement.
4546 */
4547 if (skb->len < 17) {
4548 if (skb_padto(skb, 17))
4549 return NETDEV_TX_OK;
4550 skb->len = 17;
4551 }
9d5c8243 4552
1cc3bd87 4553 return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
9d5c8243
AK		 4554}
4555
4556/**
4557 * igb_tx_timeout - Respond to a Tx Hang
4558 * @netdev: network interface device structure
4559 **/
4560static void igb_tx_timeout(struct net_device *netdev)
4561{
4562 struct igb_adapter *adapter = netdev_priv(netdev);
4563 struct e1000_hw *hw = &adapter->hw;
4564
4565 /* Do the reset outside of interrupt context */
4566 adapter->tx_timeout_count++;
f7ba205e 4567
06218a8d 4568 if (hw->mac.type >= e1000_82580)
55cac248
AD		 4569 hw->dev_spec._82575.global_device_reset = true;
4570
9d5c8243 4571 schedule_work(&adapter->reset_task);
265de409
AD		 4572 wr32(E1000_EICS,
4573 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 4574}
4575
4576static void igb_reset_task(struct work_struct *work)
4577{
4578 struct igb_adapter *adapter;
4579 adapter = container_of(work, struct igb_adapter, reset_task);
4580
c97ec42a
TI		 4581 igb_dump(adapter);
4582 netdev_err(adapter->netdev, "Reset adapter\n");
9d5c8243
AK		 4583 igb_reinit_locked(adapter);
4584}
4585
4586/**
12dcd86b 4587 * igb_get_stats64 - Get System Network Statistics
9d5c8243 4588 * @netdev: network interface device structure
12dcd86b 4589 * @stats: rtnl_link_stats64 pointer
9d5c8243 4590 *
9d5c8243 4591 **/
12dcd86b
ED		 4592static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4593 struct rtnl_link_stats64 *stats)
9d5c8243 4594{
12dcd86b
ED		 4595 struct igb_adapter *adapter = netdev_priv(netdev);
4596
4597 spin_lock(&adapter->stats64_lock);
4598 igb_update_stats(adapter, &adapter->stats64);
4599 memcpy(stats, &adapter->stats64, sizeof(*stats));
4600 spin_unlock(&adapter->stats64_lock);
4601
4602 return stats;
9d5c8243
AK		 4603}
4604
4605/**
4606 * igb_change_mtu - Change the Maximum Transfer Unit
4607 * @netdev: network interface device structure
4608 * @new_mtu: new value for maximum frame size
4609 *
4610 * Returns 0 on success, negative on failure
4611 **/
4612static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4613{
4614 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4615 struct pci_dev *pdev = adapter->pdev;
153285f9 4616 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
9d5c8243 4617
c809d227 4618 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 4619 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 4620 return -EINVAL;
4621 }
4622
153285f9 4623#define MAX_STD_JUMBO_FRAME_SIZE 9238
9d5c8243 4624 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 4625 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 4626 return -EINVAL;
4627 }
4628
4629 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4630 msleep(1);
73cd78f1 4631
9d5c8243
AK		 4632 /* igb_down has a dependency on max_frame_size */
4633 adapter->max_frame_size = max_frame;
559e9c49 4634
4c844851
AD		 4635 if (netif_running(netdev))
4636 igb_down(adapter);
9d5c8243 4637
090b1795 4638 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 4639 netdev->mtu, new_mtu);
4640 netdev->mtu = new_mtu;
4641
4642 if (netif_running(netdev))
4643 igb_up(adapter);
4644 else
4645 igb_reset(adapter);
4646
4647 clear_bit(__IGB_RESETTING, &adapter->state);
4648
4649 return 0;
4650}
4651
4652/**
4653 * igb_update_stats - Update the board statistics counters
4654 * @adapter: board private structure
4655 **/
4656
12dcd86b
ED		 4657void igb_update_stats(struct igb_adapter *adapter,
4658 struct rtnl_link_stats64 *net_stats)
9d5c8243
AK		 4659{
4660 struct e1000_hw *hw = &adapter->hw;
4661 struct pci_dev *pdev = adapter->pdev;
fa3d9a6d 4662 u32 reg, mpc;
9d5c8243 4663 u16 phy_tmp;
3f9c0164
AD		 4664 int i;
4665 u64 bytes, packets;
12dcd86b
ED		 4666 unsigned int start;
4667 u64 _bytes, _packets;
9d5c8243
AK		 4668
4669#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4670
4671 /*
4672 * Prevent stats update while adapter is being reset, or if the pci
4673 * connection is down.
4674 */
4675 if (adapter->link_speed == 0)
4676 return;
4677 if (pci_channel_offline(pdev))
4678 return;
4679
3f9c0164
AD		 4680 bytes = 0;
4681 packets = 0;
4682 for (i = 0; i < adapter->num_rx_queues; i++) {
ae1c07a6 4683 u32 rqdpc = rd32(E1000_RQDPC(i));
3025a446 4684 struct igb_ring *ring = adapter->rx_ring[i];
12dcd86b 4685
ae1c07a6
AD		 4686 if (rqdpc) {
4687 ring->rx_stats.drops += rqdpc;
4688 net_stats->rx_fifo_errors += rqdpc;
4689 }
12dcd86b
ED		 4690
4691 do {
4692 start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4693 _bytes = ring->rx_stats.bytes;
4694 _packets = ring->rx_stats.packets;
4695 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4696 bytes += _bytes;
4697 packets += _packets;
3f9c0164
AD		 4698 }
4699
128e45eb
AD		 4700 net_stats->rx_bytes = bytes;
4701 net_stats->rx_packets = packets;
3f9c0164
AD		 4702
4703 bytes = 0;
4704 packets = 0;
4705 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 4706 struct igb_ring *ring = adapter->tx_ring[i];
12dcd86b
ED		 4707 do {
4708 start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4709 _bytes = ring->tx_stats.bytes;
4710 _packets = ring->tx_stats.packets;
4711 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4712 bytes += _bytes;
4713 packets += _packets;
3f9c0164 4714 }
128e45eb
AD		 4715 net_stats->tx_bytes = bytes;
4716 net_stats->tx_packets = packets;
3f9c0164
AD		 4717
4718 /* read stats registers */
9d5c8243
AK		 4719 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4720 adapter->stats.gprc += rd32(E1000_GPRC);
4721 adapter->stats.gorc += rd32(E1000_GORCL);
4722 rd32(E1000_GORCH); /* clear GORCL */
4723 adapter->stats.bprc += rd32(E1000_BPRC);
4724 adapter->stats.mprc += rd32(E1000_MPRC);
4725 adapter->stats.roc += rd32(E1000_ROC);
4726
4727 adapter->stats.prc64 += rd32(E1000_PRC64);
4728 adapter->stats.prc127 += rd32(E1000_PRC127);
4729 adapter->stats.prc255 += rd32(E1000_PRC255);
4730 adapter->stats.prc511 += rd32(E1000_PRC511);
4731 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4732 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4733 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4734 adapter->stats.sec += rd32(E1000_SEC);
4735
fa3d9a6d
MW		 4736 mpc = rd32(E1000_MPC);
4737 adapter->stats.mpc += mpc;
4738 net_stats->rx_fifo_errors += mpc;
9d5c8243
AK		 4739 adapter->stats.scc += rd32(E1000_SCC);
4740 adapter->stats.ecol += rd32(E1000_ECOL);
4741 adapter->stats.mcc += rd32(E1000_MCC);
4742 adapter->stats.latecol += rd32(E1000_LATECOL);
4743 adapter->stats.dc += rd32(E1000_DC);
4744 adapter->stats.rlec += rd32(E1000_RLEC);
4745 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4746 adapter->stats.xontxc += rd32(E1000_XONTXC);
4747 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4748 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4749 adapter->stats.fcruc += rd32(E1000_FCRUC);
4750 adapter->stats.gptc += rd32(E1000_GPTC);
4751 adapter->stats.gotc += rd32(E1000_GOTCL);
4752 rd32(E1000_GOTCH); /* clear GOTCL */
fa3d9a6d 4753 adapter->stats.rnbc += rd32(E1000_RNBC);
9d5c8243
AK		 4754 adapter->stats.ruc += rd32(E1000_RUC);
4755 adapter->stats.rfc += rd32(E1000_RFC);
4756 adapter->stats.rjc += rd32(E1000_RJC);
4757 adapter->stats.tor += rd32(E1000_TORH);
4758 adapter->stats.tot += rd32(E1000_TOTH);
4759 adapter->stats.tpr += rd32(E1000_TPR);
4760
4761 adapter->stats.ptc64 += rd32(E1000_PTC64);
4762 adapter->stats.ptc127 += rd32(E1000_PTC127);
4763 adapter->stats.ptc255 += rd32(E1000_PTC255);
4764 adapter->stats.ptc511 += rd32(E1000_PTC511);
4765 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4766 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4767
4768 adapter->stats.mptc += rd32(E1000_MPTC);
4769 adapter->stats.bptc += rd32(E1000_BPTC);
4770
2d0b0f69
NN		 4771 adapter->stats.tpt += rd32(E1000_TPT);
4772 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4773
4774 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4775 /* read internal phy specific stats */
4776 reg = rd32(E1000_CTRL_EXT);
4777 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4778 adapter->stats.rxerrc += rd32(E1000_RXERRC);
3dbdf969
CW		 4779
4780 /* this stat has invalid values on i210/i211 */
4781 if ((hw->mac.type != e1000_i210) &&
4782 (hw->mac.type != e1000_i211))
4783 adapter->stats.tncrs += rd32(E1000_TNCRS);
43915c7c
NN		 4784 }
4785
9d5c8243
AK		 4786 adapter->stats.tsctc += rd32(E1000_TSCTC);
4787 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4788
4789 adapter->stats.iac += rd32(E1000_IAC);
4790 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4791 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4792 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4793 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4794 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4795 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4796 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4797 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4798
4799 /* Fill out the OS statistics structure */
128e45eb
AD		 4800 net_stats->multicast = adapter->stats.mprc;
4801 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4802
4803 /* Rx Errors */
4804
4805 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4806 * our own version based on RUC and ROC */
128e45eb 4807 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4808 adapter->stats.crcerrs + adapter->stats.algnerrc +
4809 adapter->stats.ruc + adapter->stats.roc +
4810 adapter->stats.cexterr;
128e45eb
AD		 4811 net_stats->rx_length_errors = adapter->stats.ruc +
4812 adapter->stats.roc;
4813 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4814 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4815 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4816
4817 /* Tx Errors */
128e45eb
AD		 4818 net_stats->tx_errors = adapter->stats.ecol +
4819 adapter->stats.latecol;
4820 net_stats->tx_aborted_errors = adapter->stats.ecol;
4821 net_stats->tx_window_errors = adapter->stats.latecol;
4822 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4823
4824 /* Tx Dropped needs to be maintained elsewhere */
4825
4826 /* Phy Stats */
4827 if (hw->phy.media_type == e1000_media_type_copper) {
4828 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4829 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4830 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4831 adapter->phy_stats.idle_errors += phy_tmp;
4832 }
4833 }
4834
4835 /* Management Stats */
4836 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4837 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4838 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
0a915b95
CW		 4839
4840 /* OS2BMC Stats */
4841 reg = rd32(E1000_MANC);
4842 if (reg & E1000_MANC_EN_BMC2OS) {
4843 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
4844 adapter->stats.o2bspc += rd32(E1000_O2BSPC);
4845 adapter->stats.b2ospc += rd32(E1000_B2OSPC);
4846 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
4847 }
9d5c8243
AK		 4848}
4849
9d5c8243
AK		 4850static irqreturn_t igb_msix_other(int irq, void *data)
4851{
047e0030 4852 struct igb_adapter *adapter = data;
9d5c8243 4853 struct e1000_hw *hw = &adapter->hw;
844290e5 4854 u32 icr = rd32(E1000_ICR);
844290e5 4855 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4856
7f081d40
AD		 4857 if (icr & E1000_ICR_DRSTA)
4858 schedule_work(&adapter->reset_task);
4859
047e0030 4860 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4861 /* HW is reporting DMA is out of sync */
4862 adapter->stats.doosync++;
13800469
GR		 4863 /* The DMA Out of Sync is also indication of a spoof event
4864 * in IOV mode. Check the Wrong VM Behavior register to
4865 * see if it is really a spoof event. */
4866 igb_check_wvbr(adapter);
dda0e083 4867 }
eebbbdba 4868
4ae196df
AD		 4869 /* Check for a mailbox event */
4870 if (icr & E1000_ICR_VMMB)
4871 igb_msg_task(adapter);
4872
4873 if (icr & E1000_ICR_LSC) {
4874 hw->mac.get_link_status = 1;
4875 /* guard against interrupt when we're going down */
4876 if (!test_bit(__IGB_DOWN, &adapter->state))
4877 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4878 }
4879
1f6e8178
MV		 4880#ifdef CONFIG_IGB_PTP
4881 if (icr & E1000_ICR_TS) {
4882 u32 tsicr = rd32(E1000_TSICR);
4883
4884 if (tsicr & E1000_TSICR_TXTS) {
4885 /* acknowledge the interrupt */
4886 wr32(E1000_TSICR, E1000_TSICR_TXTS);
4887 /* retrieve hardware timestamp */
4888 schedule_work(&adapter->ptp_tx_work);
4889 }
4890 }
4891#endif /* CONFIG_IGB_PTP */
4892
844290e5 4893 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4894
4895 return IRQ_HANDLED;
4896}
4897
047e0030 4898static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4899{
26b39276 4900 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4901 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4902
047e0030
AD		 4903 if (!q_vector->set_itr)
4904 return;
73cd78f1 4905
047e0030
AD		 4906 if (!itr_val)
4907 itr_val = 0x4;
661086df 4908
26b39276
AD		 4909 if (adapter->hw.mac.type == e1000_82575)
4910 itr_val |= itr_val << 16;
661086df 4911 else
0ba82994 4912 itr_val |= E1000_EITR_CNT_IGNR;
661086df 4913
047e0030
AD		 4914 writel(itr_val, q_vector->itr_register);
4915 q_vector->set_itr = 0;
6eb5a7f1
AD		 4916}
4917
047e0030 4918static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4919{
047e0030 4920 struct igb_q_vector *q_vector = data;
9d5c8243 4921
047e0030
AD		 4922 /* Write the ITR value calculated from the previous interrupt. */
4923 igb_write_itr(q_vector);
9d5c8243 4924
047e0030 4925 napi_schedule(&q_vector->napi);
844290e5 4926
047e0030 4927 return IRQ_HANDLED;
fe4506b6
JC		 4928}
4929
421e02f0 4930#ifdef CONFIG_IGB_DCA
047e0030 4931static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4932{
047e0030 4933 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4934 struct e1000_hw *hw = &adapter->hw;
4935 int cpu = get_cpu();
fe4506b6 4936
047e0030
AD		 4937 if (q_vector->cpu == cpu)
4938 goto out_no_update;
4939
0ba82994
AD		 4940 if (q_vector->tx.ring) {
4941 int q = q_vector->tx.ring->reg_idx;
047e0030
AD		 4942 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4943 if (hw->mac.type == e1000_82575) {
4944 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4945 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4946 } else {
047e0030
AD		 4947 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4948 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4949 E1000_DCA_TXCTRL_CPUID_SHIFT;
4950 }
4951 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4952 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4953 }
0ba82994
AD		 4954 if (q_vector->rx.ring) {
4955 int q = q_vector->rx.ring->reg_idx;
047e0030
AD		 4956 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4957 if (hw->mac.type == e1000_82575) {
2d064c06 4958 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4959 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4960 } else {
4961 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4962 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4963 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4964 }
fe4506b6
JC		 4965 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4966 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4967 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4968 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4969 }
047e0030
AD		 4970 q_vector->cpu = cpu;
4971out_no_update:
fe4506b6
JC		 4972 put_cpu();
4973}
4974
4975static void igb_setup_dca(struct igb_adapter *adapter)
4976{
7e0e99ef 4977 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4978 int i;
4979
7dfc16fa 4980 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4981 return;
4982
7e0e99ef
AD		 4983 /* Always use CB2 mode, difference is masked in the CB driver. */
4984 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4985
047e0030 4986 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4987 adapter->q_vector[i]->cpu = -1;
4988 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4989 }
4990}
4991
4992static int __igb_notify_dca(struct device *dev, void *data)
4993{
4994 struct net_device *netdev = dev_get_drvdata(dev);
4995 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4996 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4997 struct e1000_hw *hw = &adapter->hw;
4998 unsigned long event = *(unsigned long *)data;
4999
5000 switch (event) {
5001 case DCA_PROVIDER_ADD:
5002 /* if already enabled, don't do it again */
7dfc16fa 5003 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 5004 break;
fe4506b6 5005 if (dca_add_requester(dev) == 0) {
bbd98fe4 5006 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 5007 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 5008 igb_setup_dca(adapter);
5009 break;
5010 }
5011 /* Fall Through since DCA is disabled. */
5012 case DCA_PROVIDER_REMOVE:
7dfc16fa 5013 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 5014 /* without this a class_device is left
047e0030 5015 * hanging around in the sysfs model */
fe4506b6 5016 dca_remove_requester(dev);
090b1795 5017 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 5018 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 5019 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 5020 }
5021 break;
5022 }
bbd98fe4 5023
fe4506b6 5024 return 0;
9d5c8243
AK		 5025}
5026
fe4506b6
JC		 5027static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
5028 void *p)
5029{
5030 int ret_val;
5031
5032 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
5033 __igb_notify_dca);
5034
5035 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
5036}
421e02f0 5037#endif /* CONFIG_IGB_DCA */
9d5c8243 5038
0224d663
GR		 5039#ifdef CONFIG_PCI_IOV
5040static int igb_vf_configure(struct igb_adapter *adapter, int vf)
5041{
5042 unsigned char mac_addr[ETH_ALEN];
0224d663 5043
7efd26d0 5044 eth_random_addr(mac_addr);
0224d663
GR		 5045 igb_set_vf_mac(adapter, vf, mac_addr);
5046
f557147c 5047 return 0;
0224d663
GR		 5048}
5049
f557147c 5050static bool igb_vfs_are_assigned(struct igb_adapter *adapter)
0224d663 5051{
0224d663 5052 struct pci_dev *pdev = adapter->pdev;
f557147c
SA		 5053 struct pci_dev *vfdev;
5054 int dev_id;
0224d663
GR		 5055
5056 switch (adapter->hw.mac.type) {
5057 case e1000_82576:
f557147c 5058 dev_id = IGB_82576_VF_DEV_ID;
0224d663
GR		 5059 break;
5060 case e1000_i350:
f557147c 5061 dev_id = IGB_I350_VF_DEV_ID;
0224d663
GR		 5062 break;
5063 default:
f557147c 5064 return false;
0224d663
GR		 5065 }
5066
f557147c
SA		 5067 /* loop through all the VFs to see if we own any that are assigned */
5068 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL, dev_id, NULL);
5069 while (vfdev) {
5070 /* if we don't own it we don't care */
5071 if (vfdev->is_virtfn && vfdev->physfn == pdev) {
5072 /* if it is assigned we cannot release it */
5073 if (vfdev->dev_flags & PCI_DEV_FLAGS_ASSIGNED)
0224d663
GR		 5074 return true;
5075 }
f557147c
SA		 5076
5077 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL, dev_id, vfdev);
0224d663 5078 }
f557147c 5079
0224d663
GR		 5080 return false;
5081}
5082
5083#endif
4ae196df
AD		 5084static void igb_ping_all_vfs(struct igb_adapter *adapter)
5085{
5086 struct e1000_hw *hw = &adapter->hw;
5087 u32 ping;
5088 int i;
5089
5090 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
5091 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 5092 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 5093 ping |= E1000_VT_MSGTYPE_CTS;
5094 igb_write_mbx(hw, &ping, 1, i);
5095 }
5096}
5097
7d5753f0
AD		 5098static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5099{
5100 struct e1000_hw *hw = &adapter->hw;
5101 u32 vmolr = rd32(E1000_VMOLR(vf));
5102 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5103
d85b9004 5104 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
7d5753f0
AD		 5105 IGB_VF_FLAG_MULTI_PROMISC);
5106 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5107
5108 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
5109 vmolr |= E1000_VMOLR_MPME;
d85b9004 5110 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
7d5753f0
AD		 5111 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
5112 } else {
5113 /*
5114 * if we have hashes and we are clearing a multicast promisc
5115 * flag we need to write the hashes to the MTA as this step
5116 * was previously skipped
5117 */
5118 if (vf_data->num_vf_mc_hashes > 30) {
5119 vmolr |= E1000_VMOLR_MPME;
5120 } else if (vf_data->num_vf_mc_hashes) {
5121 int j;
5122 vmolr |= E1000_VMOLR_ROMPE;
5123 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5124 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5125 }
5126 }
5127
5128 wr32(E1000_VMOLR(vf), vmolr);
5129
5130 /* there are flags left unprocessed, likely not supported */
5131 if (*msgbuf & E1000_VT_MSGINFO_MASK)
5132 return -EINVAL;
5133
5134 return 0;
5135
5136}
5137
4ae196df
AD		 5138static int igb_set_vf_multicasts(struct igb_adapter *adapter,
5139 u32 *msgbuf, u32 vf)
5140{
5141 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5142 u16 *hash_list = (u16 *)&msgbuf[1];
5143 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5144 int i;
5145
7d5753f0 5146 /* salt away the number of multicast addresses assigned
4ae196df
AD		 5147 * to this VF for later use to restore when the PF multi cast
5148 * list changes
5149 */
5150 vf_data->num_vf_mc_hashes = n;
5151
7d5753f0
AD		 5152 /* only up to 30 hash values supported */
5153 if (n > 30)
5154 n = 30;
5155
5156 /* store the hashes for later use */
4ae196df 5157 for (i = 0; i < n; i++)
a419aef8 5158 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 5159
5160 /* Flush and reset the mta with the new values */
ff41f8dc 5161 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 5162
5163 return 0;
5164}
5165
5166static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
5167{
5168 struct e1000_hw *hw = &adapter->hw;
5169 struct vf_data_storage *vf_data;
5170 int i, j;
5171
5172 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 5173 u32 vmolr = rd32(E1000_VMOLR(i));
5174 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5175
4ae196df 5176 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 5177
5178 if ((vf_data->num_vf_mc_hashes > 30) ||
5179 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
5180 vmolr |= E1000_VMOLR_MPME;
5181 } else if (vf_data->num_vf_mc_hashes) {
5182 vmolr |= E1000_VMOLR_ROMPE;
5183 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5184 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5185 }
5186 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 5187 }
5188}
5189
5190static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
5191{
5192 struct e1000_hw *hw = &adapter->hw;
5193 u32 pool_mask, reg, vid;
5194 int i;
5195
5196 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5197
5198 /* Find the vlan filter for this id */
5199 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5200 reg = rd32(E1000_VLVF(i));
5201
5202 /* remove the vf from the pool */
5203 reg &= ~pool_mask;
5204
5205 /* if pool is empty then remove entry from vfta */
5206 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
5207 (reg & E1000_VLVF_VLANID_ENABLE)) {
5208 reg = 0;
5209 vid = reg & E1000_VLVF_VLANID_MASK;
5210 igb_vfta_set(hw, vid, false);
5211 }
5212
5213 wr32(E1000_VLVF(i), reg);
5214 }
ae641bdc
AD		 5215
5216 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 5217}
5218
5219static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
5220{
5221 struct e1000_hw *hw = &adapter->hw;
5222 u32 reg, i;
5223
51466239
AD		 5224 /* The vlvf table only exists on 82576 hardware and newer */
5225 if (hw->mac.type < e1000_82576)
5226 return -1;
5227
5228 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 5229 if (!adapter->vfs_allocated_count)
5230 return -1;
5231
5232 /* Find the vlan filter for this id */
5233 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5234 reg = rd32(E1000_VLVF(i));
5235 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
5236 vid == (reg & E1000_VLVF_VLANID_MASK))
5237 break;
5238 }
5239
5240 if (add) {
5241 if (i == E1000_VLVF_ARRAY_SIZE) {
5242 /* Did not find a matching VLAN ID entry that was
5243 * enabled. Search for a free filter entry, i.e.
5244 * one without the enable bit set
5245 */
5246 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5247 reg = rd32(E1000_VLVF(i));
5248 if (!(reg & E1000_VLVF_VLANID_ENABLE))
5249 break;
5250 }
5251 }
5252 if (i < E1000_VLVF_ARRAY_SIZE) {
5253 /* Found an enabled/available entry */
5254 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5255
5256 /* if !enabled we need to set this up in vfta */
5257 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 5258 /* add VID to filter table */
5259 igb_vfta_set(hw, vid, true);
4ae196df
AD		 5260 reg |= E1000_VLVF_VLANID_ENABLE;
5261 }
cad6d05f
AD		 5262 reg &= ~E1000_VLVF_VLANID_MASK;
5263 reg |= vid;
4ae196df 5264 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5265
5266 /* do not modify RLPML for PF devices */
5267 if (vf >= adapter->vfs_allocated_count)
5268 return 0;
5269
5270 if (!adapter->vf_data[vf].vlans_enabled) {
5271 u32 size;
5272 reg = rd32(E1000_VMOLR(vf));
5273 size = reg & E1000_VMOLR_RLPML_MASK;
5274 size += 4;
5275 reg &= ~E1000_VMOLR_RLPML_MASK;
5276 reg |= size;
5277 wr32(E1000_VMOLR(vf), reg);
5278 }
ae641bdc 5279
51466239 5280 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 5281 }
5282 } else {
5283 if (i < E1000_VLVF_ARRAY_SIZE) {
5284 /* remove vf from the pool */
5285 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
5286 /* if pool is empty then remove entry from vfta */
5287 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
5288 reg = 0;
5289 igb_vfta_set(hw, vid, false);
5290 }
5291 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5292
5293 /* do not modify RLPML for PF devices */
5294 if (vf >= adapter->vfs_allocated_count)
5295 return 0;
5296
5297 adapter->vf_data[vf].vlans_enabled--;
5298 if (!adapter->vf_data[vf].vlans_enabled) {
5299 u32 size;
5300 reg = rd32(E1000_VMOLR(vf));
5301 size = reg & E1000_VMOLR_RLPML_MASK;
5302 size -= 4;
5303 reg &= ~E1000_VMOLR_RLPML_MASK;
5304 reg |= size;
5305 wr32(E1000_VMOLR(vf), reg);
5306 }
4ae196df
AD		 5307 }
5308 }
8151d294
WM		 5309 return 0;
5310}
5311
5312static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
5313{
5314 struct e1000_hw *hw = &adapter->hw;
5315
5316 if (vid)
5317 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
5318 else
5319 wr32(E1000_VMVIR(vf), 0);
5320}
5321
5322static int igb_ndo_set_vf_vlan(struct net_device *netdev,
5323 int vf, u16 vlan, u8 qos)
5324{
5325 int err = 0;
5326 struct igb_adapter *adapter = netdev_priv(netdev);
5327
5328 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
5329 return -EINVAL;
5330 if (vlan || qos) {
5331 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
5332 if (err)
5333 goto out;
5334 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
5335 igb_set_vmolr(adapter, vf, !vlan);
5336 adapter->vf_data[vf].pf_vlan = vlan;
5337 adapter->vf_data[vf].pf_qos = qos;
5338 dev_info(&adapter->pdev->dev,
5339 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5340 if (test_bit(__IGB_DOWN, &adapter->state)) {
5341 dev_warn(&adapter->pdev->dev,
5342 "The VF VLAN has been set,"
5343 " but the PF device is not up.\n");
5344 dev_warn(&adapter->pdev->dev,
5345 "Bring the PF device up before"
5346 " attempting to use the VF device.\n");
5347 }
5348 } else {
5349 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5350 false, vf);
5351 igb_set_vmvir(adapter, vlan, vf);
5352 igb_set_vmolr(adapter, vf, true);
5353 adapter->vf_data[vf].pf_vlan = 0;
5354 adapter->vf_data[vf].pf_qos = 0;
5355 }
5356out:
5357 return err;
4ae196df
AD		 5358}
5359
5360static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5361{
5362 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5363 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5364
5365 return igb_vlvf_set(adapter, vid, add, vf);
5366}
5367
f2ca0dbe 5368static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 5369{
8fa7e0f7
GR		 5370 /* clear flags - except flag that indicates PF has set the MAC */
5371 adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
f2ca0dbe 5372 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 5373
5374 /* reset offloads to defaults */
8151d294 5375 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 5376
5377 /* reset vlans for device */
5378 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 5379 if (adapter->vf_data[vf].pf_vlan)
5380 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5381 adapter->vf_data[vf].pf_vlan,
5382 adapter->vf_data[vf].pf_qos);
5383 else
5384 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 5385
5386 /* reset multicast table array for vf */
5387 adapter->vf_data[vf].num_vf_mc_hashes = 0;
5388
5389 /* Flush and reset the mta with the new values */
ff41f8dc 5390 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 5391}
5392
f2ca0dbe
AD		 5393static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5394{
5395 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5396
5397 /* generate a new mac address as we were hotplug removed/added */
8151d294 5398 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
7efd26d0 5399 eth_random_addr(vf_mac);
f2ca0dbe
AD		 5400
5401 /* process remaining reset events */
5402 igb_vf_reset(adapter, vf);
5403}
5404
5405static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 5406{
5407 struct e1000_hw *hw = &adapter->hw;
5408 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 5409 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 5410 u32 reg, msgbuf[3];
5411 u8 *addr = (u8 *)(&msgbuf[1]);
5412
5413 /* process all the same items cleared in a function level reset */
f2ca0dbe 5414 igb_vf_reset(adapter, vf);
4ae196df
AD		 5415
5416 /* set vf mac address */
26ad9178 5417 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 5418
5419 /* enable transmit and receive for vf */
5420 reg = rd32(E1000_VFTE);
5421 wr32(E1000_VFTE, reg | (1 << vf));
5422 reg = rd32(E1000_VFRE);
5423 wr32(E1000_VFRE, reg | (1 << vf));
5424
8fa7e0f7 5425 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
4ae196df
AD		 5426
5427 /* reply to reset with ack and vf mac address */
5428 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5429 memcpy(addr, vf_mac, 6);
5430 igb_write_mbx(hw, msgbuf, 3, vf);
5431}
5432
5433static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5434{
de42edde
GR		 5435 /*
5436 * The VF MAC Address is stored in a packed array of bytes
5437 * starting at the second 32 bit word of the msg array
5438 */
f2ca0dbe
AD		 5439 unsigned char *addr = (char *)&msg[1];
5440 int err = -1;
4ae196df 5441
f2ca0dbe
AD		 5442 if (is_valid_ether_addr(addr))
5443 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 5444
f2ca0dbe 5445 return err;
4ae196df
AD		 5446}
5447
5448static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5449{
5450 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5451 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5452 u32 msg = E1000_VT_MSGTYPE_NACK;
5453
5454 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 5455 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5456 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 5457 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 5458 vf_data->last_nack = jiffies;
4ae196df
AD		 5459 }
5460}
5461
f2ca0dbe 5462static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 5463{
f2ca0dbe
AD		 5464 struct pci_dev *pdev = adapter->pdev;
5465 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 5466 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5467 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5468 s32 retval;
5469
f2ca0dbe 5470 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 5471
fef45f4c
AD		 5472 if (retval) {
5473 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 5474 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 5475 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5476 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5477 return;
5478 goto out;
5479 }
4ae196df
AD		 5480
5481 /* this is a message we already processed, do nothing */
5482 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 5483 return;
4ae196df
AD		 5484
5485 /*
5486 * until the vf completes a reset it should not be
5487 * allowed to start any configuration.
5488 */
5489
5490 if (msgbuf[0] == E1000_VF_RESET) {
5491 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 5492 return;
4ae196df
AD		 5493 }
5494
f2ca0dbe 5495 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 5496 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5497 return;
5498 retval = -1;
5499 goto out;
4ae196df
AD		 5500 }
5501
5502 switch ((msgbuf[0] & 0xFFFF)) {
5503 case E1000_VF_SET_MAC_ADDR:
a6b5ea35
GR		 5504 retval = -EINVAL;
5505 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
5506 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5507 else
5508 dev_warn(&pdev->dev,
5509 "VF %d attempted to override administratively "
5510 "set MAC address\nReload the VF driver to "
5511 "resume operations\n", vf);
4ae196df 5512 break;
7d5753f0
AD		 5513 case E1000_VF_SET_PROMISC:
5514 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5515 break;
4ae196df
AD		 5516 case E1000_VF_SET_MULTICAST:
5517 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5518 break;
5519 case E1000_VF_SET_LPE:
5520 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5521 break;
5522 case E1000_VF_SET_VLAN:
a6b5ea35
GR		 5523 retval = -1;
5524 if (vf_data->pf_vlan)
5525 dev_warn(&pdev->dev,
5526 "VF %d attempted to override administratively "
5527 "set VLAN tag\nReload the VF driver to "
5528 "resume operations\n", vf);
8151d294
WM		 5529 else
5530 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 5531 break;
5532 default:
090b1795 5533 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 5534 retval = -1;
5535 break;
5536 }
5537
fef45f4c
AD		 5538 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5539out:
4ae196df
AD		 5540 /* notify the VF of the results of what it sent us */
5541 if (retval)
5542 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5543 else
5544 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5545
4ae196df 5546 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 5547}
4ae196df 5548
f2ca0dbe
AD		 5549static void igb_msg_task(struct igb_adapter *adapter)
5550{
5551 struct e1000_hw *hw = &adapter->hw;
5552 u32 vf;
5553
5554 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5555 /* process any reset requests */
5556 if (!igb_check_for_rst(hw, vf))
5557 igb_vf_reset_event(adapter, vf);
5558
5559 /* process any messages pending */
5560 if (!igb_check_for_msg(hw, vf))
5561 igb_rcv_msg_from_vf(adapter, vf);
5562
5563 /* process any acks */
5564 if (!igb_check_for_ack(hw, vf))
5565 igb_rcv_ack_from_vf(adapter, vf);
5566 }
4ae196df
AD		 5567}
5568
68d480c4
AD		 5569/**
5570 * igb_set_uta - Set unicast filter table address
5571 * @adapter: board private structure
5572 *
5573 * The unicast table address is a register array of 32-bit registers.
5574 * The table is meant to be used in a way similar to how the MTA is used
5575 * however due to certain limitations in the hardware it is necessary to
25985edc
LDM		 5576 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
5577 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
68d480c4
AD		 5578 **/
5579static void igb_set_uta(struct igb_adapter *adapter)
5580{
5581 struct e1000_hw *hw = &adapter->hw;
5582 int i;
5583
5584 /* The UTA table only exists on 82576 hardware and newer */
5585 if (hw->mac.type < e1000_82576)
5586 return;
5587
5588 /* we only need to do this if VMDq is enabled */
5589 if (!adapter->vfs_allocated_count)
5590 return;
5591
5592 for (i = 0; i < hw->mac.uta_reg_count; i++)
5593 array_wr32(E1000_UTA, i, ~0);
5594}
5595
9d5c8243
AK		 5596/**
5597 * igb_intr_msi - Interrupt Handler
5598 * @irq: interrupt number
5599 * @data: pointer to a network interface device structure
5600 **/
5601static irqreturn_t igb_intr_msi(int irq, void *data)
5602{
047e0030
AD		 5603 struct igb_adapter *adapter = data;
5604 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5605 struct e1000_hw *hw = &adapter->hw;
5606 /* read ICR disables interrupts using IAM */
5607 u32 icr = rd32(E1000_ICR);
5608
047e0030 5609 igb_write_itr(q_vector);
9d5c8243 5610
7f081d40
AD		 5611 if (icr & E1000_ICR_DRSTA)
5612 schedule_work(&adapter->reset_task);
5613
047e0030 5614 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5615 /* HW is reporting DMA is out of sync */
5616 adapter->stats.doosync++;
5617 }
5618
9d5c8243
AK		 5619 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5620 hw->mac.get_link_status = 1;
5621 if (!test_bit(__IGB_DOWN, &adapter->state))
5622 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5623 }
5624
1f6e8178
MV		 5625#ifdef CONFIG_IGB_PTP
5626 if (icr & E1000_ICR_TS) {
5627 u32 tsicr = rd32(E1000_TSICR);
5628
5629 if (tsicr & E1000_TSICR_TXTS) {
5630 /* acknowledge the interrupt */
5631 wr32(E1000_TSICR, E1000_TSICR_TXTS);
5632 /* retrieve hardware timestamp */
5633 schedule_work(&adapter->ptp_tx_work);
5634 }
5635 }
5636#endif /* CONFIG_IGB_PTP */
5637
047e0030 5638 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5639
5640 return IRQ_HANDLED;
5641}
5642
5643/**
4a3c6433 5644 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 5645 * @irq: interrupt number
5646 * @data: pointer to a network interface device structure
5647 **/
5648static irqreturn_t igb_intr(int irq, void *data)
5649{
047e0030
AD		 5650 struct igb_adapter *adapter = data;
5651 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5652 struct e1000_hw *hw = &adapter->hw;
5653 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5654 * need for the IMC write */
5655 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 5656
5657 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5658 * not set, then the adapter didn't send an interrupt */
5659 if (!(icr & E1000_ICR_INT_ASSERTED))
5660 return IRQ_NONE;
5661
0ba82994
AD		 5662 igb_write_itr(q_vector);
5663
7f081d40
AD		 5664 if (icr & E1000_ICR_DRSTA)
5665 schedule_work(&adapter->reset_task);
5666
047e0030 5667 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5668 /* HW is reporting DMA is out of sync */
5669 adapter->stats.doosync++;
5670 }
5671
9d5c8243
AK		 5672 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5673 hw->mac.get_link_status = 1;
5674 /* guard against interrupt when we're going down */
5675 if (!test_bit(__IGB_DOWN, &adapter->state))
5676 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5677 }
5678
1f6e8178
MV		 5679#ifdef CONFIG_IGB_PTP
5680 if (icr & E1000_ICR_TS) {
5681 u32 tsicr = rd32(E1000_TSICR);
5682
5683 if (tsicr & E1000_TSICR_TXTS) {
5684 /* acknowledge the interrupt */
5685 wr32(E1000_TSICR, E1000_TSICR_TXTS);
5686 /* retrieve hardware timestamp */
5687 schedule_work(&adapter->ptp_tx_work);
5688 }
5689 }
5690#endif /* CONFIG_IGB_PTP */
5691
047e0030 5692 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5693
5694 return IRQ_HANDLED;
5695}
5696
c50b52a0 5697static void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 5698{
047e0030 5699 struct igb_adapter *adapter = q_vector->adapter;
46544258 5700 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5701
0ba82994
AD		 5702 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
5703 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
5704 if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
5705 igb_set_itr(q_vector);
46544258 5706 else
047e0030 5707 igb_update_ring_itr(q_vector);
9d5c8243
AK		 5708 }
5709
46544258
AD		 5710 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5711 if (adapter->msix_entries)
047e0030 5712 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 5713 else
5714 igb_irq_enable(adapter);
5715 }
9d5c8243
AK		 5716}
5717
46544258
AD		 5718/**
5719 * igb_poll - NAPI Rx polling callback
5720 * @napi: napi polling structure
5721 * @budget: count of how many packets we should handle
5722 **/
5723static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 5724{
047e0030
AD		 5725 struct igb_q_vector *q_vector = container_of(napi,
5726 struct igb_q_vector,
5727 napi);
16eb8815 5728 bool clean_complete = true;
9d5c8243 5729
421e02f0 5730#ifdef CONFIG_IGB_DCA
047e0030
AD		 5731 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5732 igb_update_dca(q_vector);
fe4506b6 5733#endif
0ba82994 5734 if (q_vector->tx.ring)
13fde97a 5735 clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 5736
0ba82994 5737 if (q_vector->rx.ring)
cd392f5c 5738 clean_complete &= igb_clean_rx_irq(q_vector, budget);
047e0030 5739
16eb8815
AD		 5740 /* If all work not completed, return budget and keep polling */
5741 if (!clean_complete)
5742 return budget;
46544258 5743
9d5c8243 5744 /* If not enough Rx work done, exit the polling mode */
16eb8815
AD		 5745 napi_complete(napi);
5746 igb_ring_irq_enable(q_vector);
9d5c8243 5747
16eb8815 5748 return 0;
9d5c8243 5749}
6d8126f9 5750
9d5c8243
AK		 5751/**
5752 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 5753 * @q_vector: pointer to q_vector containing needed info
49ce9c2c 5754 *
9d5c8243
AK		 5755 * returns true if ring is completely cleaned
5756 **/
047e0030 5757static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 5758{
047e0030 5759 struct igb_adapter *adapter = q_vector->adapter;
0ba82994 5760 struct igb_ring *tx_ring = q_vector->tx.ring;
06034649 5761 struct igb_tx_buffer *tx_buffer;
8542db05 5762 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 5763 unsigned int total_bytes = 0, total_packets = 0;
0ba82994 5764 unsigned int budget = q_vector->tx.work_limit;
8542db05 5765 unsigned int i = tx_ring->next_to_clean;
9d5c8243 5766
13fde97a
AD		 5767 if (test_bit(__IGB_DOWN, &adapter->state))
5768 return true;
0e014cb1 5769
06034649 5770 tx_buffer = &tx_ring->tx_buffer_info[i];
13fde97a 5771 tx_desc = IGB_TX_DESC(tx_ring, i);
8542db05 5772 i -= tx_ring->count;
9d5c8243 5773
13fde97a 5774 for (; budget; budget--) {
8542db05 5775 eop_desc = tx_buffer->next_to_watch;
13fde97a 5776
8542db05
AD		 5777 /* prevent any other reads prior to eop_desc */
5778 rmb();
5779
5780 /* if next_to_watch is not set then there is no work pending */
5781 if (!eop_desc)
5782 break;
13fde97a
AD		 5783
5784 /* if DD is not set pending work has not been completed */
5785 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
5786 break;
5787
8542db05
AD		 5788 /* clear next_to_watch to prevent false hangs */
5789 tx_buffer->next_to_watch = NULL;
9d5c8243 5790
ebe42d16
AD		 5791 /* update the statistics for this packet */
5792 total_bytes += tx_buffer->bytecount;
5793 total_packets += tx_buffer->gso_segs;
13fde97a 5794
ebe42d16
AD		 5795 /* free the skb */
5796 dev_kfree_skb_any(tx_buffer->skb);
5797 tx_buffer->skb = NULL;
13fde97a 5798
ebe42d16
AD		 5799 /* unmap skb header data */
5800 dma_unmap_single(tx_ring->dev,
5801 tx_buffer->dma,
5802 tx_buffer->length,
5803 DMA_TO_DEVICE);
5804
5805 /* clear last DMA location and unmap remaining buffers */
5806 while (tx_desc != eop_desc) {
5807 tx_buffer->dma = 0;
9d5c8243 5808
13fde97a
AD		 5809 tx_buffer++;
5810 tx_desc++;
9d5c8243 5811 i++;
8542db05
AD		 5812 if (unlikely(!i)) {
5813 i -= tx_ring->count;
06034649 5814 tx_buffer = tx_ring->tx_buffer_info;
13fde97a
AD		 5815 tx_desc = IGB_TX_DESC(tx_ring, 0);
5816 }
ebe42d16
AD		 5817
5818 /* unmap any remaining paged data */
5819 if (tx_buffer->dma) {
5820 dma_unmap_page(tx_ring->dev,
5821 tx_buffer->dma,
5822 tx_buffer->length,
5823 DMA_TO_DEVICE);
5824 }
5825 }
5826
5827 /* clear last DMA location */
5828 tx_buffer->dma = 0;
5829
5830 /* move us one more past the eop_desc for start of next pkt */
5831 tx_buffer++;
5832 tx_desc++;
5833 i++;
5834 if (unlikely(!i)) {
5835 i -= tx_ring->count;
5836 tx_buffer = tx_ring->tx_buffer_info;
5837 tx_desc = IGB_TX_DESC(tx_ring, 0);
5838 }
0e014cb1
AD		 5839 }
5840
bdbc0631
ED		 5841 netdev_tx_completed_queue(txring_txq(tx_ring),
5842 total_packets, total_bytes);
8542db05 5843 i += tx_ring->count;
9d5c8243 5844 tx_ring->next_to_clean = i;
13fde97a
AD		 5845 u64_stats_update_begin(&tx_ring->tx_syncp);
5846 tx_ring->tx_stats.bytes += total_bytes;
5847 tx_ring->tx_stats.packets += total_packets;
5848 u64_stats_update_end(&tx_ring->tx_syncp);
0ba82994
AD		 5849 q_vector->tx.total_bytes += total_bytes;
5850 q_vector->tx.total_packets += total_packets;
9d5c8243 5851
6d095fa8 5852 if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
13fde97a 5853 struct e1000_hw *hw = &adapter->hw;
12dcd86b 5854
8542db05 5855 eop_desc = tx_buffer->next_to_watch;
9d5c8243 5856
9d5c8243
AK		 5857 /* Detect a transmit hang in hardware, this serializes the
5858 * check with the clearing of time_stamp and movement of i */
6d095fa8 5859 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
8542db05
AD		 5860 if (eop_desc &&
5861 time_after(jiffies, tx_buffer->time_stamp +
8e95a202
JP		 5862 (adapter->tx_timeout_factor * HZ)) &&
5863 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5864
9d5c8243 5865 /* detected Tx unit hang */
59d71989 5866 dev_err(tx_ring->dev,
9d5c8243 5867 "Detected Tx Unit Hang\n"
2d064c06 5868 " Tx Queue <%d>\n"
9d5c8243
AK		 5869 " TDH <%x>\n"
5870 " TDT <%x>\n"
5871 " next_to_use <%x>\n"
5872 " next_to_clean <%x>\n"
9d5c8243
AK		 5873 "buffer_info[next_to_clean]\n"
5874 " time_stamp <%lx>\n"
8542db05 5875 " next_to_watch <%p>\n"
9d5c8243
AK		 5876 " jiffies <%lx>\n"
5877 " desc.status <%x>\n",
2d064c06 5878 tx_ring->queue_index,
238ac817 5879 rd32(E1000_TDH(tx_ring->reg_idx)),
fce99e34 5880 readl(tx_ring->tail),
9d5c8243
AK		 5881 tx_ring->next_to_use,
5882 tx_ring->next_to_clean,
8542db05
AD		 5883 tx_buffer->time_stamp,
5884 eop_desc,
9d5c8243 5885 jiffies,
0e014cb1 5886 eop_desc->wb.status);
13fde97a
AD		 5887 netif_stop_subqueue(tx_ring->netdev,
5888 tx_ring->queue_index);
5889
5890 /* we are about to reset, no point in enabling stuff */
5891 return true;
9d5c8243
AK		 5892 }
5893 }
13fde97a
AD		 5894
5895 if (unlikely(total_packets &&
5896 netif_carrier_ok(tx_ring->netdev) &&
5897 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5898 /* Make sure that anybody stopping the queue after this
5899 * sees the new next_to_clean.
5900 */
5901 smp_mb();
5902 if (__netif_subqueue_stopped(tx_ring->netdev,
5903 tx_ring->queue_index) &&
5904 !(test_bit(__IGB_DOWN, &adapter->state))) {
5905 netif_wake_subqueue(tx_ring->netdev,
5906 tx_ring->queue_index);
5907
5908 u64_stats_update_begin(&tx_ring->tx_syncp);
5909 tx_ring->tx_stats.restart_queue++;
5910 u64_stats_update_end(&tx_ring->tx_syncp);
5911 }
5912 }
5913
5914 return !!budget;
9d5c8243
AK		 5915}
5916
cd392f5c 5917static inline void igb_rx_checksum(struct igb_ring *ring,
3ceb90fd
AD		 5918 union e1000_adv_rx_desc *rx_desc,
5919 struct sk_buff *skb)
9d5c8243 5920{
bc8acf2c 5921 skb_checksum_none_assert(skb);
9d5c8243 5922
294e7d78 5923 /* Ignore Checksum bit is set */
3ceb90fd 5924 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
294e7d78
AD		 5925 return;
5926
5927 /* Rx checksum disabled via ethtool */
5928 if (!(ring->netdev->features & NETIF_F_RXCSUM))
9d5c8243 5929 return;
85ad76b2 5930
9d5c8243 5931 /* TCP/UDP checksum error bit is set */
3ceb90fd
AD		 5932 if (igb_test_staterr(rx_desc,
5933 E1000_RXDEXT_STATERR_TCPE |
5934 E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5935 /*
5936 * work around errata with sctp packets where the TCPE aka
5937 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5938 * packets, (aka let the stack check the crc32c)
5939 */
866cff06
AD		 5940 if (!((skb->len == 60) &&
5941 test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
12dcd86b 5942 u64_stats_update_begin(&ring->rx_syncp);
04a5fcaa 5943 ring->rx_stats.csum_err++;
12dcd86b
ED		 5944 u64_stats_update_end(&ring->rx_syncp);
5945 }
9d5c8243 5946 /* let the stack verify checksum errors */
9d5c8243
AK		 5947 return;
5948 }
5949 /* It must be a TCP or UDP packet with a valid checksum */
3ceb90fd
AD		 5950 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
5951 E1000_RXD_STAT_UDPCS))
9d5c8243
AK		 5952 skb->ip_summed = CHECKSUM_UNNECESSARY;
5953
3ceb90fd
AD		 5954 dev_dbg(ring->dev, "cksum success: bits %08X\n",
5955 le32_to_cpu(rx_desc->wb.upper.status_error));
9d5c8243
AK		 5956}
5957
077887c3
AD		 5958static inline void igb_rx_hash(struct igb_ring *ring,
5959 union e1000_adv_rx_desc *rx_desc,
5960 struct sk_buff *skb)
5961{
5962 if (ring->netdev->features & NETIF_F_RXHASH)
5963 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
5964}
5965
8be10e91
AD		 5966static void igb_rx_vlan(struct igb_ring *ring,
5967 union e1000_adv_rx_desc *rx_desc,
5968 struct sk_buff *skb)
5969{
5970 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
5971 u16 vid;
5972 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
5973 test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags))
5974 vid = be16_to_cpu(rx_desc->wb.upper.vlan);
5975 else
5976 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
5977
5978 __vlan_hwaccel_put_tag(skb, vid);
5979 }
5980}
5981
44390ca6 5982static inline u16 igb_get_hlen(union e1000_adv_rx_desc *rx_desc)
2d94d8ab
AD		 5983{
5984 /* HW will not DMA in data larger than the given buffer, even if it
5985 * parses the (NFS, of course) header to be larger. In that case, it
5986 * fills the header buffer and spills the rest into the page.
5987 */
5988 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5989 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
44390ca6
AD		 5990 if (hlen > IGB_RX_HDR_LEN)
5991 hlen = IGB_RX_HDR_LEN;
2d94d8ab
AD		 5992 return hlen;
5993}
5994
cd392f5c 5995static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, int budget)
9d5c8243 5996{
0ba82994 5997 struct igb_ring *rx_ring = q_vector->rx.ring;
16eb8815
AD		 5998 union e1000_adv_rx_desc *rx_desc;
5999 const int current_node = numa_node_id();
9d5c8243 6000 unsigned int total_bytes = 0, total_packets = 0;
16eb8815
AD		 6001 u16 cleaned_count = igb_desc_unused(rx_ring);
6002 u16 i = rx_ring->next_to_clean;
9d5c8243 6003
60136906 6004 rx_desc = IGB_RX_DESC(rx_ring, i);
9d5c8243 6005
3ceb90fd 6006 while (igb_test_staterr(rx_desc, E1000_RXD_STAT_DD)) {
06034649 6007 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
16eb8815
AD		 6008 struct sk_buff *skb = buffer_info->skb;
6009 union e1000_adv_rx_desc *next_rxd;
9d5c8243 6010
69d3ca53 6011 buffer_info->skb = NULL;
16eb8815 6012 prefetch(skb->data);
69d3ca53
AD		 6013
6014 i++;
6015 if (i == rx_ring->count)
6016 i = 0;
42d0781a 6017
60136906 6018 next_rxd = IGB_RX_DESC(rx_ring, i);
69d3ca53 6019 prefetch(next_rxd);
9d5c8243 6020
16eb8815
AD		 6021 /*
6022 * This memory barrier is needed to keep us from reading
6023 * any other fields out of the rx_desc until we know the
6024 * RXD_STAT_DD bit is set
6025 */
6026 rmb();
9d5c8243 6027
16eb8815
AD		 6028 if (!skb_is_nonlinear(skb)) {
6029 __skb_put(skb, igb_get_hlen(rx_desc));
6030 dma_unmap_single(rx_ring->dev, buffer_info->dma,
44390ca6 6031 IGB_RX_HDR_LEN,
59d71989 6032 DMA_FROM_DEVICE);
91615f76 6033 buffer_info->dma = 0;
bf36c1a0
AD		 6034 }
6035
16eb8815
AD		 6036 if (rx_desc->wb.upper.length) {
6037 u16 length = le16_to_cpu(rx_desc->wb.upper.length);
bf36c1a0 6038
aa913403 6039 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
bf36c1a0
AD		 6040 buffer_info->page,
6041 buffer_info->page_offset,
6042 length);
6043
16eb8815
AD		 6044 skb->len += length;
6045 skb->data_len += length;
95b9c1df 6046 skb->truesize += PAGE_SIZE / 2;
16eb8815 6047
d1eff350
AD		 6048 if ((page_count(buffer_info->page) != 1) ||
6049 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 6050 buffer_info->page = NULL;
6051 else
6052 get_page(buffer_info->page);
9d5c8243 6053
16eb8815
AD		 6054 dma_unmap_page(rx_ring->dev, buffer_info->page_dma,
6055 PAGE_SIZE / 2, DMA_FROM_DEVICE);
6056 buffer_info->page_dma = 0;
9d5c8243 6057 }
9d5c8243 6058
3ceb90fd 6059 if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)) {
06034649
AD		 6060 struct igb_rx_buffer *next_buffer;
6061 next_buffer = &rx_ring->rx_buffer_info[i];
b2d56536
AD		 6062 buffer_info->skb = next_buffer->skb;
6063 buffer_info->dma = next_buffer->dma;
6064 next_buffer->skb = skb;
6065 next_buffer->dma = 0;
bf36c1a0
AD		 6066 goto next_desc;
6067 }
44390ca6 6068
89eaefb6
BG		 6069 if (unlikely((igb_test_staterr(rx_desc,
6070 E1000_RXDEXT_ERR_FRAME_ERR_MASK))
6071 && !(rx_ring->netdev->features & NETIF_F_RXALL))) {
16eb8815 6072 dev_kfree_skb_any(skb);
9d5c8243
AK		 6073 goto next_desc;
6074 }
9d5c8243 6075
7ebae817 6076#ifdef CONFIG_IGB_PTP
a79f4f88 6077 igb_ptp_rx_hwtstamp(q_vector, rx_desc, skb);
3c89f6d0 6078#endif /* CONFIG_IGB_PTP */
077887c3 6079 igb_rx_hash(rx_ring, rx_desc, skb);
3ceb90fd 6080 igb_rx_checksum(rx_ring, rx_desc, skb);
8be10e91 6081 igb_rx_vlan(rx_ring, rx_desc, skb);
3ceb90fd
AD		 6082
6083 total_bytes += skb->len;
6084 total_packets++;
6085
6086 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
6087
b2cb09b1 6088 napi_gro_receive(&q_vector->napi, skb);
9d5c8243 6089
16eb8815 6090 budget--;
9d5c8243 6091next_desc:
16eb8815
AD		 6092 if (!budget)
6093 break;
6094
6095 cleaned_count++;
9d5c8243
AK		 6096 /* return some buffers to hardware, one at a time is too slow */
6097 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
cd392f5c 6098 igb_alloc_rx_buffers(rx_ring, cleaned_count);
9d5c8243
AK		 6099 cleaned_count = 0;
6100 }
6101
6102 /* use prefetched values */
6103 rx_desc = next_rxd;
9d5c8243 6104 }
bf36c1a0 6105
9d5c8243 6106 rx_ring->next_to_clean = i;
12dcd86b 6107 u64_stats_update_begin(&rx_ring->rx_syncp);
9d5c8243
AK		 6108 rx_ring->rx_stats.packets += total_packets;
6109 rx_ring->rx_stats.bytes += total_bytes;
12dcd86b 6110 u64_stats_update_end(&rx_ring->rx_syncp);
0ba82994
AD		 6111 q_vector->rx.total_packets += total_packets;
6112 q_vector->rx.total_bytes += total_bytes;
c023cd88
AD		 6113
6114 if (cleaned_count)
cd392f5c 6115 igb_alloc_rx_buffers(rx_ring, cleaned_count);
c023cd88 6116
16eb8815 6117 return !!budget;
9d5c8243
AK		 6118}
6119
c023cd88 6120static bool igb_alloc_mapped_skb(struct igb_ring *rx_ring,
06034649 6121 struct igb_rx_buffer *bi)
c023cd88
AD		 6122{
6123 struct sk_buff *skb = bi->skb;
6124 dma_addr_t dma = bi->dma;
6125
6126 if (dma)
6127 return true;
6128
6129 if (likely(!skb)) {
6130 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
6131 IGB_RX_HDR_LEN);
6132 bi->skb = skb;
6133 if (!skb) {
6134 rx_ring->rx_stats.alloc_failed++;
6135 return false;
6136 }
6137
6138 /* initialize skb for ring */
6139 skb_record_rx_queue(skb, rx_ring->queue_index);
6140 }
6141
6142 dma = dma_map_single(rx_ring->dev, skb->data,
6143 IGB_RX_HDR_LEN, DMA_FROM_DEVICE);
6144
6145 if (dma_mapping_error(rx_ring->dev, dma)) {
6146 rx_ring->rx_stats.alloc_failed++;
6147 return false;
6148 }
6149
6150 bi->dma = dma;
6151 return true;
6152}
6153
6154static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
06034649 6155 struct igb_rx_buffer *bi)
c023cd88
AD		 6156{
6157 struct page *page = bi->page;
6158 dma_addr_t page_dma = bi->page_dma;
6159 unsigned int page_offset = bi->page_offset ^ (PAGE_SIZE / 2);
6160
6161 if (page_dma)
6162 return true;
6163
6164 if (!page) {
0614002b 6165 page = __skb_alloc_page(GFP_ATOMIC, bi->skb);
c023cd88
AD		 6166 bi->page = page;
6167 if (unlikely(!page)) {
6168 rx_ring->rx_stats.alloc_failed++;
6169 return false;
6170 }
6171 }
6172
6173 page_dma = dma_map_page(rx_ring->dev, page,
6174 page_offset, PAGE_SIZE / 2,
6175 DMA_FROM_DEVICE);
6176
6177 if (dma_mapping_error(rx_ring->dev, page_dma)) {
6178 rx_ring->rx_stats.alloc_failed++;
6179 return false;
6180 }
6181
6182 bi->page_dma = page_dma;
6183 bi->page_offset = page_offset;
6184 return true;
6185}
6186
9d5c8243 6187/**
cd392f5c 6188 * igb_alloc_rx_buffers - Replace used receive buffers; packet split
9d5c8243
AK		 6189 * @adapter: address of board private structure
6190 **/
cd392f5c 6191void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
9d5c8243 6192{
9d5c8243 6193 union e1000_adv_rx_desc *rx_desc;
06034649 6194 struct igb_rx_buffer *bi;
c023cd88 6195 u16 i = rx_ring->next_to_use;
9d5c8243 6196
60136906 6197 rx_desc = IGB_RX_DESC(rx_ring, i);
06034649 6198 bi = &rx_ring->rx_buffer_info[i];
c023cd88 6199 i -= rx_ring->count;
9d5c8243
AK		 6200
6201 while (cleaned_count--) {
c023cd88
AD		 6202 if (!igb_alloc_mapped_skb(rx_ring, bi))
6203 break;
9d5c8243 6204
c023cd88
AD		 6205 /* Refresh the desc even if buffer_addrs didn't change
6206 * because each write-back erases this info. */
6207 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
9d5c8243 6208
c023cd88
AD		 6209 if (!igb_alloc_mapped_page(rx_ring, bi))
6210 break;
6211
6212 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
9d5c8243 6213
c023cd88
AD		 6214 rx_desc++;
6215 bi++;
9d5c8243 6216 i++;
c023cd88 6217 if (unlikely(!i)) {
60136906 6218 rx_desc = IGB_RX_DESC(rx_ring, 0);
06034649 6219 bi = rx_ring->rx_buffer_info;
c023cd88
AD		 6220 i -= rx_ring->count;
6221 }
6222
6223 /* clear the hdr_addr for the next_to_use descriptor */
6224 rx_desc->read.hdr_addr = 0;
9d5c8243
AK		 6225 }
6226
c023cd88
AD		 6227 i += rx_ring->count;
6228
9d5c8243
AK		 6229 if (rx_ring->next_to_use != i) {
6230 rx_ring->next_to_use = i;
9d5c8243
AK		 6231
6232 /* Force memory writes to complete before letting h/w
6233 * know there are new descriptors to fetch. (Only
6234 * applicable for weak-ordered memory model archs,
6235 * such as IA-64). */
6236 wmb();
fce99e34 6237 writel(i, rx_ring->tail);
9d5c8243
AK		 6238 }
6239}
6240
6241/**
6242 * igb_mii_ioctl -
6243 * @netdev:
6244 * @ifreq:
6245 * @cmd:
6246 **/
6247static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6248{
6249 struct igb_adapter *adapter = netdev_priv(netdev);
6250 struct mii_ioctl_data *data = if_mii(ifr);
6251
6252 if (adapter->hw.phy.media_type != e1000_media_type_copper)
6253 return -EOPNOTSUPP;
6254
6255 switch (cmd) {
6256 case SIOCGMIIPHY:
6257 data->phy_id = adapter->hw.phy.addr;
6258 break;
6259 case SIOCGMIIREG:
f5f4cf08
AD		 6260 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
6261 &data->val_out))
9d5c8243
AK		 6262 return -EIO;
6263 break;
6264 case SIOCSMIIREG:
6265 default:
6266 return -EOPNOTSUPP;
6267 }
6268 return 0;
6269}
6270
6271/**
6272 * igb_ioctl -
6273 * @netdev:
6274 * @ifreq:
6275 * @cmd:
6276 **/
6277static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6278{
6279 switch (cmd) {
6280 case SIOCGMIIPHY:
6281 case SIOCGMIIREG:
6282 case SIOCSMIIREG:
6283 return igb_mii_ioctl(netdev, ifr, cmd);
3c89f6d0 6284#ifdef CONFIG_IGB_PTP
c6cb090b 6285 case SIOCSHWTSTAMP:
a79f4f88 6286 return igb_ptp_hwtstamp_ioctl(netdev, ifr, cmd);
3c89f6d0 6287#endif /* CONFIG_IGB_PTP */
9d5c8243
AK		 6288 default:
6289 return -EOPNOTSUPP;
6290 }
6291}
6292
009bc06e
AD		 6293s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6294{
6295 struct igb_adapter *adapter = hw->back;
6296 u16 cap_offset;
6297
bdaae04c 6298 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6299 if (!cap_offset)
6300 return -E1000_ERR_CONFIG;
6301
6302 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6303
6304 return 0;
6305}
6306
6307s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6308{
6309 struct igb_adapter *adapter = hw->back;
6310 u16 cap_offset;
6311
bdaae04c 6312 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6313 if (!cap_offset)
6314 return -E1000_ERR_CONFIG;
6315
6316 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6317
6318 return 0;
6319}
6320
c8f44aff 6321static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features)
9d5c8243
AK		 6322{
6323 struct igb_adapter *adapter = netdev_priv(netdev);
6324 struct e1000_hw *hw = &adapter->hw;
6325 u32 ctrl, rctl;
5faf030c 6326 bool enable = !!(features & NETIF_F_HW_VLAN_RX);
9d5c8243 6327
5faf030c 6328 if (enable) {
9d5c8243
AK		 6329 /* enable VLAN tag insert/strip */
6330 ctrl = rd32(E1000_CTRL);
6331 ctrl |= E1000_CTRL_VME;
6332 wr32(E1000_CTRL, ctrl);
6333
51466239 6334 /* Disable CFI check */
9d5c8243 6335 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 6336 rctl &= ~E1000_RCTL_CFIEN;
6337 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 6338 } else {
6339 /* disable VLAN tag insert/strip */
6340 ctrl = rd32(E1000_CTRL);
6341 ctrl &= ~E1000_CTRL_VME;
6342 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 6343 }
6344
e1739522 6345 igb_rlpml_set(adapter);
9d5c8243
AK		 6346}
6347
8e586137 6348static int igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
9d5c8243
AK		 6349{
6350 struct igb_adapter *adapter = netdev_priv(netdev);
6351 struct e1000_hw *hw = &adapter->hw;
4ae196df 6352 int pf_id = adapter->vfs_allocated_count;
9d5c8243 6353
51466239
AD		 6354 /* attempt to add filter to vlvf array */
6355 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 6356
51466239
AD		 6357 /* add the filter since PF can receive vlans w/o entry in vlvf */
6358 igb_vfta_set(hw, vid, true);
b2cb09b1
JP		 6359
6360 set_bit(vid, adapter->active_vlans);
8e586137
JP		 6361
6362 return 0;
9d5c8243
AK		 6363}
6364
8e586137 6365static int igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
9d5c8243
AK		 6366{
6367 struct igb_adapter *adapter = netdev_priv(netdev);
6368 struct e1000_hw *hw = &adapter->hw;
4ae196df 6369 int pf_id = adapter->vfs_allocated_count;
51466239 6370 s32 err;
9d5c8243 6371
51466239
AD		 6372 /* remove vlan from VLVF table array */
6373 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 6374
51466239
AD		 6375 /* if vid was not present in VLVF just remove it from table */
6376 if (err)
4ae196df 6377 igb_vfta_set(hw, vid, false);
b2cb09b1
JP		 6378
6379 clear_bit(vid, adapter->active_vlans);
8e586137
JP		 6380
6381 return 0;
9d5c8243
AK		 6382}
6383
6384static void igb_restore_vlan(struct igb_adapter *adapter)
6385{
b2cb09b1 6386 u16 vid;
9d5c8243 6387
5faf030c
AD		 6388 igb_vlan_mode(adapter->netdev, adapter->netdev->features);
6389
b2cb09b1
JP		 6390 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
6391 igb_vlan_rx_add_vid(adapter->netdev, vid);
9d5c8243
AK		 6392}
6393
14ad2513 6394int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
9d5c8243 6395{
090b1795 6396 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 6397 struct e1000_mac_info *mac = &adapter->hw.mac;
6398
6399 mac->autoneg = 0;
6400
14ad2513
DD		 6401 /* Make sure dplx is at most 1 bit and lsb of speed is not set
6402 * for the switch() below to work */
6403 if ((spd & 1) || (dplx & ~1))
6404 goto err_inval;
6405
cd2638a8
CW		 6406 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6407 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
14ad2513
DD		 6408 spd != SPEED_1000 &&
6409 dplx != DUPLEX_FULL)
6410 goto err_inval;
cd2638a8 6411
14ad2513 6412 switch (spd + dplx) {
9d5c8243
AK		 6413 case SPEED_10 + DUPLEX_HALF:
6414 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6415 break;
6416 case SPEED_10 + DUPLEX_FULL:
6417 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6418 break;
6419 case SPEED_100 + DUPLEX_HALF:
6420 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6421 break;
6422 case SPEED_100 + DUPLEX_FULL:
6423 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6424 break;
6425 case SPEED_1000 + DUPLEX_FULL:
6426 mac->autoneg = 1;
6427 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6428 break;
6429 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6430 default:
14ad2513 6431 goto err_inval;
9d5c8243 6432 }
8376dad0
JB		 6433
6434 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
6435 adapter->hw.phy.mdix = AUTO_ALL_MODES;
6436
9d5c8243 6437 return 0;
14ad2513
DD		 6438
6439err_inval:
6440 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6441 return -EINVAL;
9d5c8243
AK		 6442}
6443
749ab2cd
YZ		 6444static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
6445 bool runtime)
9d5c8243
AK		 6446{
6447 struct net_device *netdev = pci_get_drvdata(pdev);
6448 struct igb_adapter *adapter = netdev_priv(netdev);
6449 struct e1000_hw *hw = &adapter->hw;
2d064c06 6450 u32 ctrl, rctl, status;
749ab2cd 6451 u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
9d5c8243
AK		 6452#ifdef CONFIG_PM
6453 int retval = 0;
6454#endif
6455
6456 netif_device_detach(netdev);
6457
a88f10ec 6458 if (netif_running(netdev))
749ab2cd 6459 __igb_close(netdev, true);
a88f10ec 6460
047e0030 6461 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 6462
6463#ifdef CONFIG_PM
6464 retval = pci_save_state(pdev);
6465 if (retval)
6466 return retval;
6467#endif
6468
6469 status = rd32(E1000_STATUS);
6470 if (status & E1000_STATUS_LU)
6471 wufc &= ~E1000_WUFC_LNKC;
6472
6473 if (wufc) {
6474 igb_setup_rctl(adapter);
ff41f8dc 6475 igb_set_rx_mode(netdev);
9d5c8243
AK		 6476
6477 /* turn on all-multi mode if wake on multicast is enabled */
6478 if (wufc & E1000_WUFC_MC) {
6479 rctl = rd32(E1000_RCTL);
6480 rctl |= E1000_RCTL_MPE;
6481 wr32(E1000_RCTL, rctl);
6482 }
6483
6484 ctrl = rd32(E1000_CTRL);
6485 /* advertise wake from D3Cold */
6486 #define E1000_CTRL_ADVD3WUC 0x00100000
6487 /* phy power management enable */
6488 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6489 ctrl |= E1000_CTRL_ADVD3WUC;
6490 wr32(E1000_CTRL, ctrl);
6491
9d5c8243 6492 /* Allow time for pending master requests to run */
330a6d6a 6493 igb_disable_pcie_master(hw);
9d5c8243
AK		 6494
6495 wr32(E1000_WUC, E1000_WUC_PME_EN);
6496 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 6497 } else {
6498 wr32(E1000_WUC, 0);
6499 wr32(E1000_WUFC, 0);
9d5c8243
AK		 6500 }
6501
3fe7c4c9
RW		 6502 *enable_wake = wufc || adapter->en_mng_pt;
6503 if (!*enable_wake)
88a268c1
NN		 6504 igb_power_down_link(adapter);
6505 else
6506 igb_power_up_link(adapter);
9d5c8243
AK		 6507
6508 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6509 * would have already happened in close and is redundant. */
6510 igb_release_hw_control(adapter);
6511
6512 pci_disable_device(pdev);
6513
9d5c8243
AK		 6514 return 0;
6515}
6516
6517#ifdef CONFIG_PM
d9dd966d 6518#ifdef CONFIG_PM_SLEEP
749ab2cd 6519static int igb_suspend(struct device *dev)
3fe7c4c9
RW		 6520{
6521 int retval;
6522 bool wake;
749ab2cd 6523 struct pci_dev *pdev = to_pci_dev(dev);
3fe7c4c9 6524
749ab2cd 6525 retval = __igb_shutdown(pdev, &wake, 0);
3fe7c4c9
RW		 6526 if (retval)
6527 return retval;
6528
6529 if (wake) {
6530 pci_prepare_to_sleep(pdev);
6531 } else {
6532 pci_wake_from_d3(pdev, false);
6533 pci_set_power_state(pdev, PCI_D3hot);
6534 }
6535
6536 return 0;
6537}
d9dd966d 6538#endif /* CONFIG_PM_SLEEP */
3fe7c4c9 6539
749ab2cd 6540static int igb_resume(struct device *dev)
9d5c8243 6541{
749ab2cd 6542 struct pci_dev *pdev = to_pci_dev(dev);
9d5c8243
AK		 6543 struct net_device *netdev = pci_get_drvdata(pdev);
6544 struct igb_adapter *adapter = netdev_priv(netdev);
6545 struct e1000_hw *hw = &adapter->hw;
6546 u32 err;
6547
6548 pci_set_power_state(pdev, PCI_D0);
6549 pci_restore_state(pdev);
b94f2d77 6550 pci_save_state(pdev);
42bfd33a 6551
aed5dec3 6552 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 6553 if (err) {
6554 dev_err(&pdev->dev,
6555 "igb: Cannot enable PCI device from suspend\n");
6556 return err;
6557 }
6558 pci_set_master(pdev);
6559
6560 pci_enable_wake(pdev, PCI_D3hot, 0);
6561 pci_enable_wake(pdev, PCI_D3cold, 0);
6562
cfb8c3aa 6563 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 6564 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6565 return -ENOMEM;
9d5c8243
AK		 6566 }
6567
9d5c8243 6568 igb_reset(adapter);
a8564f03
AD		 6569
6570 /* let the f/w know that the h/w is now under the control of the
6571 * driver. */
6572 igb_get_hw_control(adapter);
6573
9d5c8243
AK		 6574 wr32(E1000_WUS, ~0);
6575
749ab2cd
YZ		 6576 if (netdev->flags & IFF_UP) {
6577 err = __igb_open(netdev, true);
a88f10ec
AD		 6578 if (err)
6579 return err;
6580 }
9d5c8243
AK		 6581
6582 netif_device_attach(netdev);
749ab2cd
YZ		 6583 return 0;
6584}
6585
6586#ifdef CONFIG_PM_RUNTIME
6587static int igb_runtime_idle(struct device *dev)
6588{
6589 struct pci_dev *pdev = to_pci_dev(dev);
6590 struct net_device *netdev = pci_get_drvdata(pdev);
6591 struct igb_adapter *adapter = netdev_priv(netdev);
6592
6593 if (!igb_has_link(adapter))
6594 pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
6595
6596 return -EBUSY;
6597}
6598
6599static int igb_runtime_suspend(struct device *dev)
6600{
6601 struct pci_dev *pdev = to_pci_dev(dev);
6602 int retval;
6603 bool wake;
6604
6605 retval = __igb_shutdown(pdev, &wake, 1);
6606 if (retval)
6607 return retval;
6608
6609 if (wake) {
6610 pci_prepare_to_sleep(pdev);
6611 } else {
6612 pci_wake_from_d3(pdev, false);
6613 pci_set_power_state(pdev, PCI_D3hot);
6614 }
9d5c8243 6615
9d5c8243
AK		 6616 return 0;
6617}
749ab2cd
YZ		 6618
6619static int igb_runtime_resume(struct device *dev)
6620{
6621 return igb_resume(dev);
6622}
6623#endif /* CONFIG_PM_RUNTIME */
9d5c8243
AK		 6624#endif
6625
6626static void igb_shutdown(struct pci_dev *pdev)
6627{
3fe7c4c9
RW		 6628 bool wake;
6629
749ab2cd 6630 __igb_shutdown(pdev, &wake, 0);
3fe7c4c9
RW		 6631
6632 if (system_state == SYSTEM_POWER_OFF) {
6633 pci_wake_from_d3(pdev, wake);
6634 pci_set_power_state(pdev, PCI_D3hot);
6635 }
9d5c8243
AK		 6636}
6637
6638#ifdef CONFIG_NET_POLL_CONTROLLER
6639/*
6640 * Polling 'interrupt' - used by things like netconsole to send skbs
6641 * without having to re-enable interrupts. It's not called while
6642 * the interrupt routine is executing.
6643 */
6644static void igb_netpoll(struct net_device *netdev)
6645{
6646 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 6647 struct e1000_hw *hw = &adapter->hw;
0d1ae7f4 6648 struct igb_q_vector *q_vector;
9d5c8243 6649 int i;
9d5c8243 6650
047e0030 6651 for (i = 0; i < adapter->num_q_vectors; i++) {
0d1ae7f4
AD		 6652 q_vector = adapter->q_vector[i];
6653 if (adapter->msix_entries)
6654 wr32(E1000_EIMC, q_vector->eims_value);
6655 else
6656 igb_irq_disable(adapter);
047e0030 6657 napi_schedule(&q_vector->napi);
eebbbdba 6658 }
9d5c8243
AK		 6659}
6660#endif /* CONFIG_NET_POLL_CONTROLLER */
6661
6662/**
6663 * igb_io_error_detected - called when PCI error is detected
6664 * @pdev: Pointer to PCI device
6665 * @state: The current pci connection state
6666 *
6667 * This function is called after a PCI bus error affecting
6668 * this device has been detected.
6669 */
6670static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6671 pci_channel_state_t state)
6672{
6673 struct net_device *netdev = pci_get_drvdata(pdev);
6674 struct igb_adapter *adapter = netdev_priv(netdev);
6675
6676 netif_device_detach(netdev);
6677
59ed6eec
AD		 6678 if (state == pci_channel_io_perm_failure)
6679 return PCI_ERS_RESULT_DISCONNECT;
6680
9d5c8243
AK		 6681 if (netif_running(netdev))
6682 igb_down(adapter);
6683 pci_disable_device(pdev);
6684
6685 /* Request a slot slot reset. */
6686 return PCI_ERS_RESULT_NEED_RESET;
6687}
6688
6689/**
6690 * igb_io_slot_reset - called after the pci bus has been reset.
6691 * @pdev: Pointer to PCI device
6692 *
6693 * Restart the card from scratch, as if from a cold-boot. Implementation
6694 * resembles the first-half of the igb_resume routine.
6695 */
6696static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6697{
6698 struct net_device *netdev = pci_get_drvdata(pdev);
6699 struct igb_adapter *adapter = netdev_priv(netdev);
6700 struct e1000_hw *hw = &adapter->hw;
40a914fa 6701 pci_ers_result_t result;
42bfd33a 6702 int err;
9d5c8243 6703
aed5dec3 6704 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6705 dev_err(&pdev->dev,
6706 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6707 result = PCI_ERS_RESULT_DISCONNECT;
6708 } else {
6709 pci_set_master(pdev);
6710 pci_restore_state(pdev);
b94f2d77 6711 pci_save_state(pdev);
9d5c8243 6712
40a914fa
AD		 6713 pci_enable_wake(pdev, PCI_D3hot, 0);
6714 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6715
40a914fa
AD		 6716 igb_reset(adapter);
6717 wr32(E1000_WUS, ~0);
6718 result = PCI_ERS_RESULT_RECOVERED;
6719 }
9d5c8243 6720
ea943d41
JK		 6721 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6722 if (err) {
6723 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6724 "failed 0x%0x\n", err);
6725 /* non-fatal, continue */
6726 }
40a914fa
AD		 6727
6728 return result;
9d5c8243
AK		 6729}
6730
6731/**
6732 * igb_io_resume - called when traffic can start flowing again.
6733 * @pdev: Pointer to PCI device
6734 *
6735 * This callback is called when the error recovery driver tells us that
6736 * its OK to resume normal operation. Implementation resembles the
6737 * second-half of the igb_resume routine.
6738 */
6739static void igb_io_resume(struct pci_dev *pdev)
6740{
6741 struct net_device *netdev = pci_get_drvdata(pdev);
6742 struct igb_adapter *adapter = netdev_priv(netdev);
6743
9d5c8243
AK		 6744 if (netif_running(netdev)) {
6745 if (igb_up(adapter)) {
6746 dev_err(&pdev->dev, "igb_up failed after reset\n");
6747 return;
6748 }
6749 }
6750
6751 netif_device_attach(netdev);
6752
6753 /* let the f/w know that the h/w is now under the control of the
6754 * driver. */
6755 igb_get_hw_control(adapter);
9d5c8243
AK		 6756}
6757
26ad9178
AD		 6758static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6759 u8 qsel)
6760{
6761 u32 rar_low, rar_high;
6762 struct e1000_hw *hw = &adapter->hw;
6763
6764 /* HW expects these in little endian so we reverse the byte order
6765 * from network order (big endian) to little endian
6766 */
6767 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6768 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6769 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6770
6771 /* Indicate to hardware the Address is Valid. */
6772 rar_high |= E1000_RAH_AV;
6773
6774 if (hw->mac.type == e1000_82575)
6775 rar_high |= E1000_RAH_POOL_1 * qsel;
6776 else
6777 rar_high |= E1000_RAH_POOL_1 << qsel;
6778
6779 wr32(E1000_RAL(index), rar_low);
6780 wrfl();
6781 wr32(E1000_RAH(index), rar_high);
6782 wrfl();
6783}
6784
4ae196df
AD		 6785static int igb_set_vf_mac(struct igb_adapter *adapter,
6786 int vf, unsigned char *mac_addr)
6787{
6788 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 6789 /* VF MAC addresses start at end of receive addresses and moves
6790 * torwards the first, as a result a collision should not be possible */
6791 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 6792
37680117 6793 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 6794
26ad9178 6795 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 6796
6797 return 0;
6798}
6799
8151d294
WM		 6800static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6801{
6802 struct igb_adapter *adapter = netdev_priv(netdev);
6803 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6804 return -EINVAL;
6805 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6806 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6807 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6808 " change effective.");
6809 if (test_bit(__IGB_DOWN, &adapter->state)) {
6810 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6811 " but the PF device is not up.\n");
6812 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6813 " attempting to use the VF device.\n");
6814 }
6815 return igb_set_vf_mac(adapter, vf, mac);
6816}
6817
17dc566c
LL		 6818static int igb_link_mbps(int internal_link_speed)
6819{
6820 switch (internal_link_speed) {
6821 case SPEED_100:
6822 return 100;
6823 case SPEED_1000:
6824 return 1000;
6825 default:
6826 return 0;
6827 }
6828}
6829
6830static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
6831 int link_speed)
6832{
6833 int rf_dec, rf_int;
6834 u32 bcnrc_val;
6835
6836 if (tx_rate != 0) {
6837 /* Calculate the rate factor values to set */
6838 rf_int = link_speed / tx_rate;
6839 rf_dec = (link_speed - (rf_int * tx_rate));
6840 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
6841
6842 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
6843 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
6844 E1000_RTTBCNRC_RF_INT_MASK);
6845 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
6846 } else {
6847 bcnrc_val = 0;
6848 }
6849
6850 wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
f00b0da7
LL		 6851 /*
6852 * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
6853 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
6854 */
6855 wr32(E1000_RTTBCNRM, 0x14);
17dc566c
LL		 6856 wr32(E1000_RTTBCNRC, bcnrc_val);
6857}
6858
6859static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
6860{
6861 int actual_link_speed, i;
6862 bool reset_rate = false;
6863
6864 /* VF TX rate limit was not set or not supported */
6865 if ((adapter->vf_rate_link_speed == 0) ||
6866 (adapter->hw.mac.type != e1000_82576))
6867 return;
6868
6869 actual_link_speed = igb_link_mbps(adapter->link_speed);
6870 if (actual_link_speed != adapter->vf_rate_link_speed) {
6871 reset_rate = true;
6872 adapter->vf_rate_link_speed = 0;
6873 dev_info(&adapter->pdev->dev,
6874 "Link speed has been changed. VF Transmit "
6875 "rate is disabled\n");
6876 }
6877
6878 for (i = 0; i < adapter->vfs_allocated_count; i++) {
6879 if (reset_rate)
6880 adapter->vf_data[i].tx_rate = 0;
6881
6882 igb_set_vf_rate_limit(&adapter->hw, i,
6883 adapter->vf_data[i].tx_rate,
6884 actual_link_speed);
6885 }
6886}
6887
8151d294
WM		 6888static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6889{
17dc566c
LL		 6890 struct igb_adapter *adapter = netdev_priv(netdev);
6891 struct e1000_hw *hw = &adapter->hw;
6892 int actual_link_speed;
6893
6894 if (hw->mac.type != e1000_82576)
6895 return -EOPNOTSUPP;
6896
6897 actual_link_speed = igb_link_mbps(adapter->link_speed);
6898 if ((vf >= adapter->vfs_allocated_count) ||
6899 (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
6900 (tx_rate < 0) || (tx_rate > actual_link_speed))
6901 return -EINVAL;
6902
6903 adapter->vf_rate_link_speed = actual_link_speed;
6904 adapter->vf_data[vf].tx_rate = (u16)tx_rate;
6905 igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
6906
6907 return 0;
8151d294
WM		 6908}
6909
6910static int igb_ndo_get_vf_config(struct net_device *netdev,
6911 int vf, struct ifla_vf_info *ivi)
6912{
6913 struct igb_adapter *adapter = netdev_priv(netdev);
6914 if (vf >= adapter->vfs_allocated_count)
6915 return -EINVAL;
6916 ivi->vf = vf;
6917 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
17dc566c 6918 ivi->tx_rate = adapter->vf_data[vf].tx_rate;
8151d294
WM		 6919 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6920 ivi->qos = adapter->vf_data[vf].pf_qos;
6921 return 0;
6922}
6923
4ae196df
AD		 6924static void igb_vmm_control(struct igb_adapter *adapter)
6925{
6926 struct e1000_hw *hw = &adapter->hw;
10d8e907 6927 u32 reg;
4ae196df 6928
52a1dd4d
AD		 6929 switch (hw->mac.type) {
6930 case e1000_82575:
f96a8a0b
CW		 6931 case e1000_i210:
6932 case e1000_i211:
52a1dd4d
AD		 6933 default:
6934 /* replication is not supported for 82575 */
4ae196df 6935 return;
52a1dd4d
AD		 6936 case e1000_82576:
6937 /* notify HW that the MAC is adding vlan tags */
6938 reg = rd32(E1000_DTXCTL);
6939 reg |= E1000_DTXCTL_VLAN_ADDED;
6940 wr32(E1000_DTXCTL, reg);
6941 case e1000_82580:
6942 /* enable replication vlan tag stripping */
6943 reg = rd32(E1000_RPLOLR);
6944 reg |= E1000_RPLOLR_STRVLAN;
6945 wr32(E1000_RPLOLR, reg);
d2ba2ed8
AD		 6946 case e1000_i350:
6947 /* none of the above registers are supported by i350 */
52a1dd4d
AD		 6948 break;
6949 }
10d8e907 6950
d4960307
AD		 6951 if (adapter->vfs_allocated_count) {
6952 igb_vmdq_set_loopback_pf(hw, true);
6953 igb_vmdq_set_replication_pf(hw, true);
13800469
GR		 6954 igb_vmdq_set_anti_spoofing_pf(hw, true,
6955 adapter->vfs_allocated_count);
d4960307
AD		 6956 } else {
6957 igb_vmdq_set_loopback_pf(hw, false);
6958 igb_vmdq_set_replication_pf(hw, false);
6959 }
4ae196df
AD		 6960}
6961
b6e0c419
CW		 6962static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
6963{
6964 struct e1000_hw *hw = &adapter->hw;
6965 u32 dmac_thr;
6966 u16 hwm;
6967
6968 if (hw->mac.type > e1000_82580) {
6969 if (adapter->flags & IGB_FLAG_DMAC) {
6970 u32 reg;
6971
6972 /* force threshold to 0. */
6973 wr32(E1000_DMCTXTH, 0);
6974
6975 /*
e8c626e9
MV		 6976 * DMA Coalescing high water mark needs to be greater
6977 * than the Rx threshold. Set hwm to PBA - max frame
6978 * size in 16B units, capping it at PBA - 6KB.
b6e0c419 6979 */
e8c626e9
MV		 6980 hwm = 64 * pba - adapter->max_frame_size / 16;
6981 if (hwm < 64 * (pba - 6))
6982 hwm = 64 * (pba - 6);
6983 reg = rd32(E1000_FCRTC);
6984 reg &= ~E1000_FCRTC_RTH_COAL_MASK;
6985 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
6986 & E1000_FCRTC_RTH_COAL_MASK);
6987 wr32(E1000_FCRTC, reg);
6988
6989 /*
6990 * Set the DMA Coalescing Rx threshold to PBA - 2 * max
6991 * frame size, capping it at PBA - 10KB.
6992 */
6993 dmac_thr = pba - adapter->max_frame_size / 512;
6994 if (dmac_thr < pba - 10)
6995 dmac_thr = pba - 10;
b6e0c419
CW		 6996 reg = rd32(E1000_DMACR);
6997 reg &= ~E1000_DMACR_DMACTHR_MASK;
b6e0c419
CW		 6998 reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
6999 & E1000_DMACR_DMACTHR_MASK);
7000
7001 /* transition to L0x or L1 if available..*/
7002 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
7003
7004 /* watchdog timer= +-1000 usec in 32usec intervals */
7005 reg |= (1000 >> 5);
0c02dd98
MV		 7006
7007 /* Disable BMC-to-OS Watchdog Enable */
7008 reg &= ~E1000_DMACR_DC_BMC2OSW_EN;
b6e0c419
CW		 7009 wr32(E1000_DMACR, reg);
7010
7011 /*
7012 * no lower threshold to disable
7013 * coalescing(smart fifb)-UTRESH=0
7014 */
7015 wr32(E1000_DMCRTRH, 0);
b6e0c419
CW		 7016
7017 reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
7018
7019 wr32(E1000_DMCTLX, reg);
7020
7021 /*
7022 * free space in tx packet buffer to wake from
7023 * DMA coal
7024 */
7025 wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
7026 (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
7027
7028 /*
7029 * make low power state decision controlled
7030 * by DMA coal
7031 */
7032 reg = rd32(E1000_PCIEMISC);
7033 reg &= ~E1000_PCIEMISC_LX_DECISION;
7034 wr32(E1000_PCIEMISC, reg);
7035 } /* endif adapter->dmac is not disabled */
7036 } else if (hw->mac.type == e1000_82580) {
7037 u32 reg = rd32(E1000_PCIEMISC);
7038 wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION);
7039 wr32(E1000_DMACR, 0);
7040 }
7041}
7042
9d5c8243 7043/* igb_main.c */
ubuntu-zesty-kernel mirror