]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - drivers/net/ethernet/intel/igb/igb_main.c
projects / mirror_ubuntu-zesty-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net: Fix non-kernel-doc comments with kernel-doc start marker
[mirror_ubuntu-zesty-kernel.git] / drivers / net / ethernet / intel / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
6e861326 4 Copyright(c) 2007-2012 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
876d2d6f
JK		 28#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
29
9d5c8243
AK		 30#include <linux/module.h>
31#include <linux/types.h>
32#include <linux/init.h>
b2cb09b1 33#include <linux/bitops.h>
9d5c8243
AK		 34#include <linux/vmalloc.h>
35#include <linux/pagemap.h>
36#include <linux/netdevice.h>
9d5c8243 37#include <linux/ipv6.h>
5a0e3ad6 38#include <linux/slab.h>
9d5c8243
AK		 39#include <net/checksum.h>
40#include <net/ip6_checksum.h>
c6cb090b 41#include <linux/net_tstamp.h>
9d5c8243
AK		 42#include <linux/mii.h>
43#include <linux/ethtool.h>
01789349 44#include <linux/if.h>
9d5c8243
AK		 45#include <linux/if_vlan.h>
46#include <linux/pci.h>
c54106bb 47#include <linux/pci-aspm.h>
9d5c8243
AK		 48#include <linux/delay.h>
49#include <linux/interrupt.h>
7d13a7d0
AD		 50#include <linux/ip.h>
51#include <linux/tcp.h>
52#include <linux/sctp.h>
9d5c8243 53#include <linux/if_ether.h>
40a914fa 54#include <linux/aer.h>
70c71606 55#include <linux/prefetch.h>
749ab2cd 56#include <linux/pm_runtime.h>
421e02f0 57#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 58#include <linux/dca.h>
59#endif
9d5c8243
AK		 60#include "igb.h"
61
200e5fd5
CW		 62#define MAJ 4
63#define MIN 0
64#define BUILD 1
0d1fe82d 65#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
929dd047 66__stringify(BUILD) "-k"
9d5c8243
AK		 67char igb_driver_name[] = "igb";
68char igb_driver_version[] = DRV_VERSION;
69static const char igb_driver_string[] =
70 "Intel(R) Gigabit Ethernet Network Driver";
6e861326 71static const char igb_copyright[] = "Copyright (c) 2007-2012 Intel Corporation.";
9d5c8243 72
9d5c8243
AK		 73static const struct e1000_info *igb_info_tbl[] = {
74 [board_82575] = &e1000_82575_info,
75};
76
a3aa1884 77static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
f96a8a0b
CW		 78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 },
79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 },
81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 },
82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 },
d2ba2ed8
AD		 83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
55cac248
AD		 87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
6493d24f 89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
55cac248
AD		 90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
308fb39a
JG		 93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
1b5dda33
GJ		 95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
2d064c06 97 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 98 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 99 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 100 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
101 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 102 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
b894fa26 103 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
c8ea5ea9 104 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 105 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
106 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
107 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
108 /* required last entry */
109 {0, }
110};
111
112MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
113
114void igb_reset(struct igb_adapter *);
115static int igb_setup_all_tx_resources(struct igb_adapter *);
116static int igb_setup_all_rx_resources(struct igb_adapter *);
117static void igb_free_all_tx_resources(struct igb_adapter *);
118static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 119static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 120static int igb_probe(struct pci_dev *, const struct pci_device_id *);
121static void __devexit igb_remove(struct pci_dev *pdev);
122static int igb_sw_init(struct igb_adapter *);
123static int igb_open(struct net_device *);
124static int igb_close(struct net_device *);
125static void igb_configure_tx(struct igb_adapter *);
126static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 127static void igb_clean_all_tx_rings(struct igb_adapter *);
128static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 129static void igb_clean_tx_ring(struct igb_ring *);
130static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 131static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 132static void igb_update_phy_info(unsigned long);
133static void igb_watchdog(unsigned long);
134static void igb_watchdog_task(struct work_struct *);
cd392f5c 135static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
12dcd86b
ED		 136static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
137 struct rtnl_link_stats64 *stats);
9d5c8243
AK		 138static int igb_change_mtu(struct net_device *, int);
139static int igb_set_mac(struct net_device *, void *);
68d480c4 140static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 141static irqreturn_t igb_intr(int irq, void *);
142static irqreturn_t igb_intr_msi(int irq, void *);
143static irqreturn_t igb_msix_other(int irq, void *);
047e0030 144static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 145#ifdef CONFIG_IGB_DCA
047e0030 146static void igb_update_dca(struct igb_q_vector *);
fe4506b6 147static void igb_setup_dca(struct igb_adapter *);
421e02f0 148#endif /* CONFIG_IGB_DCA */
661086df 149static int igb_poll(struct napi_struct *, int);
13fde97a 150static bool igb_clean_tx_irq(struct igb_q_vector *);
cd392f5c 151static bool igb_clean_rx_irq(struct igb_q_vector *, int);
9d5c8243
AK		 152static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
153static void igb_tx_timeout(struct net_device *);
154static void igb_reset_task(struct work_struct *);
c8f44aff 155static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features);
8e586137
JP		 156static int igb_vlan_rx_add_vid(struct net_device *, u16);
157static int igb_vlan_rx_kill_vid(struct net_device *, u16);
9d5c8243 158static void igb_restore_vlan(struct igb_adapter *);
26ad9178 159static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 160static void igb_ping_all_vfs(struct igb_adapter *);
161static void igb_msg_task(struct igb_adapter *);
4ae196df 162static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 163static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 164static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 165static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
166static int igb_ndo_set_vf_vlan(struct net_device *netdev,
167 int vf, u16 vlan, u8 qos);
168static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
169static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
170 struct ifla_vf_info *ivi);
17dc566c 171static void igb_check_vf_rate_limit(struct igb_adapter *);
46a01698
RL		 172
173#ifdef CONFIG_PCI_IOV
0224d663
GR		 174static int igb_vf_configure(struct igb_adapter *adapter, int vf);
175static int igb_find_enabled_vfs(struct igb_adapter *adapter);
176static int igb_check_vf_assignment(struct igb_adapter *adapter);
46a01698 177#endif
9d5c8243 178
9d5c8243 179#ifdef CONFIG_PM
d9dd966d 180#ifdef CONFIG_PM_SLEEP
749ab2cd 181static int igb_suspend(struct device *);
d9dd966d 182#endif
749ab2cd
YZ		 183static int igb_resume(struct device *);
184#ifdef CONFIG_PM_RUNTIME
185static int igb_runtime_suspend(struct device *dev);
186static int igb_runtime_resume(struct device *dev);
187static int igb_runtime_idle(struct device *dev);
188#endif
189static const struct dev_pm_ops igb_pm_ops = {
190 SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
191 SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
192 igb_runtime_idle)
193};
9d5c8243
AK		 194#endif
195static void igb_shutdown(struct pci_dev *);
421e02f0 196#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 197static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
198static struct notifier_block dca_notifier = {
199 .notifier_call = igb_notify_dca,
200 .next = NULL,
201 .priority = 0
202};
203#endif
9d5c8243
AK		 204#ifdef CONFIG_NET_POLL_CONTROLLER
205/* for netdump / net console */
206static void igb_netpoll(struct net_device *);
207#endif
37680117 208#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 209static unsigned int max_vfs = 0;
210module_param(max_vfs, uint, 0);
211MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
212 "per physical function");
213#endif /* CONFIG_PCI_IOV */
214
9d5c8243
AK		 215static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
216 pci_channel_state_t);
217static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
218static void igb_io_resume(struct pci_dev *);
219
220static struct pci_error_handlers igb_err_handler = {
221 .error_detected = igb_io_error_detected,
222 .slot_reset = igb_io_slot_reset,
223 .resume = igb_io_resume,
224};
225
b6e0c419 226static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
9d5c8243
AK		 227
228static struct pci_driver igb_driver = {
229 .name = igb_driver_name,
230 .id_table = igb_pci_tbl,
231 .probe = igb_probe,
232 .remove = __devexit_p(igb_remove),
233#ifdef CONFIG_PM
749ab2cd 234 .driver.pm = &igb_pm_ops,
9d5c8243
AK		 235#endif
236 .shutdown = igb_shutdown,
237 .err_handler = &igb_err_handler
238};
239
240MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
241MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
242MODULE_LICENSE("GPL");
243MODULE_VERSION(DRV_VERSION);
244
b3f4d599 245#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
246static int debug = -1;
247module_param(debug, int, 0);
248MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
249
c97ec42a
TI		 250struct igb_reg_info {
251 u32 ofs;
252 char *name;
253};
254
255static const struct igb_reg_info igb_reg_info_tbl[] = {
256
257 /* General Registers */
258 {E1000_CTRL, "CTRL"},
259 {E1000_STATUS, "STATUS"},
260 {E1000_CTRL_EXT, "CTRL_EXT"},
261
262 /* Interrupt Registers */
263 {E1000_ICR, "ICR"},
264
265 /* RX Registers */
266 {E1000_RCTL, "RCTL"},
267 {E1000_RDLEN(0), "RDLEN"},
268 {E1000_RDH(0), "RDH"},
269 {E1000_RDT(0), "RDT"},
270 {E1000_RXDCTL(0), "RXDCTL"},
271 {E1000_RDBAL(0), "RDBAL"},
272 {E1000_RDBAH(0), "RDBAH"},
273
274 /* TX Registers */
275 {E1000_TCTL, "TCTL"},
276 {E1000_TDBAL(0), "TDBAL"},
277 {E1000_TDBAH(0), "TDBAH"},
278 {E1000_TDLEN(0), "TDLEN"},
279 {E1000_TDH(0), "TDH"},
280 {E1000_TDT(0), "TDT"},
281 {E1000_TXDCTL(0), "TXDCTL"},
282 {E1000_TDFH, "TDFH"},
283 {E1000_TDFT, "TDFT"},
284 {E1000_TDFHS, "TDFHS"},
285 {E1000_TDFPC, "TDFPC"},
286
287 /* List Terminator */
288 {}
289};
290
291/*
292 * igb_regdump - register printout routine
293 */
294static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
295{
296 int n = 0;
297 char rname[16];
298 u32 regs[8];
299
300 switch (reginfo->ofs) {
301 case E1000_RDLEN(0):
302 for (n = 0; n < 4; n++)
303 regs[n] = rd32(E1000_RDLEN(n));
304 break;
305 case E1000_RDH(0):
306 for (n = 0; n < 4; n++)
307 regs[n] = rd32(E1000_RDH(n));
308 break;
309 case E1000_RDT(0):
310 for (n = 0; n < 4; n++)
311 regs[n] = rd32(E1000_RDT(n));
312 break;
313 case E1000_RXDCTL(0):
314 for (n = 0; n < 4; n++)
315 regs[n] = rd32(E1000_RXDCTL(n));
316 break;
317 case E1000_RDBAL(0):
318 for (n = 0; n < 4; n++)
319 regs[n] = rd32(E1000_RDBAL(n));
320 break;
321 case E1000_RDBAH(0):
322 for (n = 0; n < 4; n++)
323 regs[n] = rd32(E1000_RDBAH(n));
324 break;
325 case E1000_TDBAL(0):
326 for (n = 0; n < 4; n++)
327 regs[n] = rd32(E1000_RDBAL(n));
328 break;
329 case E1000_TDBAH(0):
330 for (n = 0; n < 4; n++)
331 regs[n] = rd32(E1000_TDBAH(n));
332 break;
333 case E1000_TDLEN(0):
334 for (n = 0; n < 4; n++)
335 regs[n] = rd32(E1000_TDLEN(n));
336 break;
337 case E1000_TDH(0):
338 for (n = 0; n < 4; n++)
339 regs[n] = rd32(E1000_TDH(n));
340 break;
341 case E1000_TDT(0):
342 for (n = 0; n < 4; n++)
343 regs[n] = rd32(E1000_TDT(n));
344 break;
345 case E1000_TXDCTL(0):
346 for (n = 0; n < 4; n++)
347 regs[n] = rd32(E1000_TXDCTL(n));
348 break;
349 default:
876d2d6f 350 pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs));
c97ec42a
TI		 351 return;
352 }
353
354 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
876d2d6f
JK		 355 pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1],
356 regs[2], regs[3]);
c97ec42a
TI		 357}
358
359/*
360 * igb_dump - Print registers, tx-rings and rx-rings
361 */
362static void igb_dump(struct igb_adapter *adapter)
363{
364 struct net_device *netdev = adapter->netdev;
365 struct e1000_hw *hw = &adapter->hw;
366 struct igb_reg_info *reginfo;
c97ec42a
TI		 367 struct igb_ring *tx_ring;
368 union e1000_adv_tx_desc *tx_desc;
369 struct my_u0 { u64 a; u64 b; } *u0;
c97ec42a
TI		 370 struct igb_ring *rx_ring;
371 union e1000_adv_rx_desc *rx_desc;
372 u32 staterr;
6ad4edfc 373 u16 i, n;
c97ec42a
TI		 374
375 if (!netif_msg_hw(adapter))
376 return;
377
378 /* Print netdevice Info */
379 if (netdev) {
380 dev_info(&adapter->pdev->dev, "Net device Info\n");
876d2d6f
JK		 381 pr_info("Device Name state trans_start "
382 "last_rx\n");
383 pr_info("%-15s %016lX %016lX %016lX\n", netdev->name,
384 netdev->state, netdev->trans_start, netdev->last_rx);
c97ec42a
TI		 385 }
386
387 /* Print Registers */
388 dev_info(&adapter->pdev->dev, "Register Dump\n");
876d2d6f 389 pr_info(" Register Name Value\n");
c97ec42a
TI		 390 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
391 reginfo->name; reginfo++) {
392 igb_regdump(hw, reginfo);
393 }
394
395 /* Print TX Ring Summary */
396 if (!netdev || !netif_running(netdev))
397 goto exit;
398
399 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
876d2d6f 400 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
c97ec42a 401 for (n = 0; n < adapter->num_tx_queues; n++) {
06034649 402 struct igb_tx_buffer *buffer_info;
c97ec42a 403 tx_ring = adapter->tx_ring[n];
06034649 404 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
876d2d6f
JK		 405 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
406 n, tx_ring->next_to_use, tx_ring->next_to_clean,
407 (u64)buffer_info->dma,
408 buffer_info->length,
409 buffer_info->next_to_watch,
410 (u64)buffer_info->time_stamp);
c97ec42a
TI		 411 }
412
413 /* Print TX Rings */
414 if (!netif_msg_tx_done(adapter))
415 goto rx_ring_summary;
416
417 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
418
419 /* Transmit Descriptor Formats
420 *
421 * Advanced Transmit Descriptor
422 * +--------------------------------------------------------------+
423 * 0 | Buffer Address [63:0] |
424 * +--------------------------------------------------------------+
425 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
426 * +--------------------------------------------------------------+
427 * 63 46 45 40 39 38 36 35 32 31 24 15 0
428 */
429
430 for (n = 0; n < adapter->num_tx_queues; n++) {
431 tx_ring = adapter->tx_ring[n];
876d2d6f
JK		 432 pr_info("------------------------------------\n");
433 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
434 pr_info("------------------------------------\n");
435 pr_info("T [desc] [address 63:0 ] [PlPOCIStDDM Ln] "
436 "[bi->dma ] leng ntw timestamp "
437 "bi->skb\n");
c97ec42a
TI		 438
439 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
876d2d6f 440 const char *next_desc;
06034649 441 struct igb_tx_buffer *buffer_info;
60136906 442 tx_desc = IGB_TX_DESC(tx_ring, i);
06034649 443 buffer_info = &tx_ring->tx_buffer_info[i];
c97ec42a 444 u0 = (struct my_u0 *)tx_desc;
876d2d6f
JK		 445 if (i == tx_ring->next_to_use &&
446 i == tx_ring->next_to_clean)
447 next_desc = " NTC/U";
448 else if (i == tx_ring->next_to_use)
449 next_desc = " NTU";
450 else if (i == tx_ring->next_to_clean)
451 next_desc = " NTC";
452 else
453 next_desc = "";
454
455 pr_info("T [0x%03X] %016llX %016llX %016llX"
456 " %04X %p %016llX %p%s\n", i,
c97ec42a
TI		 457 le64_to_cpu(u0->a),
458 le64_to_cpu(u0->b),
459 (u64)buffer_info->dma,
460 buffer_info->length,
461 buffer_info->next_to_watch,
462 (u64)buffer_info->time_stamp,
876d2d6f 463 buffer_info->skb, next_desc);
c97ec42a
TI		 464
465 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
466 print_hex_dump(KERN_INFO, "",
467 DUMP_PREFIX_ADDRESS,
468 16, 1, phys_to_virt(buffer_info->dma),
469 buffer_info->length, true);
470 }
471 }
472
473 /* Print RX Rings Summary */
474rx_ring_summary:
475 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
876d2d6f 476 pr_info("Queue [NTU] [NTC]\n");
c97ec42a
TI		 477 for (n = 0; n < adapter->num_rx_queues; n++) {
478 rx_ring = adapter->rx_ring[n];
876d2d6f
JK		 479 pr_info(" %5d %5X %5X\n",
480 n, rx_ring->next_to_use, rx_ring->next_to_clean);
c97ec42a
TI		 481 }
482
483 /* Print RX Rings */
484 if (!netif_msg_rx_status(adapter))
485 goto exit;
486
487 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
488
489 /* Advanced Receive Descriptor (Read) Format
490 * 63 1 0
491 * +-----------------------------------------------------+
492 * 0 | Packet Buffer Address [63:1] |A0/NSE|
493 * +----------------------------------------------+------+
494 * 8 | Header Buffer Address [63:1] | DD |
495 * +-----------------------------------------------------+
496 *
497 *
498 * Advanced Receive Descriptor (Write-Back) Format
499 *
500 * 63 48 47 32 31 30 21 20 17 16 4 3 0
501 * +------------------------------------------------------+
502 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
503 * | Checksum Ident | | | | Type | Type |
504 * +------------------------------------------------------+
505 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
506 * +------------------------------------------------------+
507 * 63 48 47 32 31 20 19 0
508 */
509
510 for (n = 0; n < adapter->num_rx_queues; n++) {
511 rx_ring = adapter->rx_ring[n];
876d2d6f
JK		 512 pr_info("------------------------------------\n");
513 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
514 pr_info("------------------------------------\n");
515 pr_info("R [desc] [ PktBuf A0] [ HeadBuf DD] "
516 "[bi->dma ] [bi->skb] <-- Adv Rx Read format\n");
517 pr_info("RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] -----"
518 "----------- [bi->skb] <-- Adv Rx Write-Back format\n");
c97ec42a
TI		 519
520 for (i = 0; i < rx_ring->count; i++) {
876d2d6f 521 const char *next_desc;
06034649
AD		 522 struct igb_rx_buffer *buffer_info;
523 buffer_info = &rx_ring->rx_buffer_info[i];
60136906 524 rx_desc = IGB_RX_DESC(rx_ring, i);
c97ec42a
TI		 525 u0 = (struct my_u0 *)rx_desc;
526 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
876d2d6f
JK		 527
528 if (i == rx_ring->next_to_use)
529 next_desc = " NTU";
530 else if (i == rx_ring->next_to_clean)
531 next_desc = " NTC";
532 else
533 next_desc = "";
534
c97ec42a
TI		 535 if (staterr & E1000_RXD_STAT_DD) {
536 /* Descriptor Done */
876d2d6f
JK		 537 pr_info("%s[0x%03X] %016llX %016llX -------"
538 "--------- %p%s\n", "RWB", i,
c97ec42a
TI		 539 le64_to_cpu(u0->a),
540 le64_to_cpu(u0->b),
876d2d6f 541 buffer_info->skb, next_desc);
c97ec42a 542 } else {
876d2d6f
JK		 543 pr_info("%s[0x%03X] %016llX %016llX %016llX"
544 " %p%s\n", "R ", i,
c97ec42a
TI		 545 le64_to_cpu(u0->a),
546 le64_to_cpu(u0->b),
547 (u64)buffer_info->dma,
876d2d6f 548 buffer_info->skb, next_desc);
c97ec42a
TI		 549
550 if (netif_msg_pktdata(adapter)) {
551 print_hex_dump(KERN_INFO, "",
552 DUMP_PREFIX_ADDRESS,
553 16, 1,
554 phys_to_virt(buffer_info->dma),
44390ca6
AD		 555 IGB_RX_HDR_LEN, true);
556 print_hex_dump(KERN_INFO, "",
557 DUMP_PREFIX_ADDRESS,
558 16, 1,
559 phys_to_virt(
560 buffer_info->page_dma +
561 buffer_info->page_offset),
562 PAGE_SIZE/2, true);
c97ec42a
TI		 563 }
564 }
c97ec42a
TI		 565 }
566 }
567
568exit:
569 return;
570}
571
9d5c8243 572/**
c041076a 573 * igb_get_hw_dev - return device
9d5c8243
AK		 574 * used by hardware layer to print debugging information
575 **/
c041076a 576struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
9d5c8243
AK		 577{
578 struct igb_adapter *adapter = hw->back;
c041076a 579 return adapter->netdev;
9d5c8243 580}
38c845c7 581
9d5c8243
AK		 582/**
583 * igb_init_module - Driver Registration Routine
584 *
585 * igb_init_module is the first routine called when the driver is
586 * loaded. All it does is register with the PCI subsystem.
587 **/
588static int __init igb_init_module(void)
589{
590 int ret;
876d2d6f 591 pr_info("%s - version %s\n",
9d5c8243
AK		 592 igb_driver_string, igb_driver_version);
593
876d2d6f 594 pr_info("%s\n", igb_copyright);
9d5c8243 595
421e02f0 596#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 597 dca_register_notify(&dca_notifier);
598#endif
bbd98fe4 599 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 600 return ret;
601}
602
603module_init(igb_init_module);
604
605/**
606 * igb_exit_module - Driver Exit Cleanup Routine
607 *
608 * igb_exit_module is called just before the driver is removed
609 * from memory.
610 **/
611static void __exit igb_exit_module(void)
612{
421e02f0 613#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 614 dca_unregister_notify(&dca_notifier);
615#endif
9d5c8243
AK		 616 pci_unregister_driver(&igb_driver);
617}
618
619module_exit(igb_exit_module);
620
26bc19ec
AD		 621#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
622/**
623 * igb_cache_ring_register - Descriptor ring to register mapping
624 * @adapter: board private structure to initialize
625 *
626 * Once we know the feature-set enabled for the device, we'll cache
627 * the register offset the descriptor ring is assigned to.
628 **/
629static void igb_cache_ring_register(struct igb_adapter *adapter)
630{
ee1b9f06 631 int i = 0, j = 0;
047e0030 632 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 633
634 switch (adapter->hw.mac.type) {
635 case e1000_82576:
636 /* The queues are allocated for virtualization such that VF 0
637 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
638 * In order to avoid collision we start at the first free queue
639 * and continue consuming queues in the same sequence
640 */
ee1b9f06 641 if (adapter->vfs_allocated_count) {
a99955fc 642 for (; i < adapter->rss_queues; i++)
3025a446
AD		 643 adapter->rx_ring[i]->reg_idx = rbase_offset +
644 Q_IDX_82576(i);
ee1b9f06 645 }
26bc19ec 646 case e1000_82575:
55cac248 647 case e1000_82580:
d2ba2ed8 648 case e1000_i350:
f96a8a0b
CW		 649 case e1000_i210:
650 case e1000_i211:
26bc19ec 651 default:
ee1b9f06 652 for (; i < adapter->num_rx_queues; i++)
3025a446 653 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 654 for (; j < adapter->num_tx_queues; j++)
3025a446 655 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 656 break;
657 }
658}
659
047e0030
AD		 660static void igb_free_queues(struct igb_adapter *adapter)
661{
3025a446 662 int i;
047e0030 663
3025a446
AD		 664 for (i = 0; i < adapter->num_tx_queues; i++) {
665 kfree(adapter->tx_ring[i]);
666 adapter->tx_ring[i] = NULL;
667 }
668 for (i = 0; i < adapter->num_rx_queues; i++) {
669 kfree(adapter->rx_ring[i]);
670 adapter->rx_ring[i] = NULL;
671 }
047e0030
AD		 672 adapter->num_rx_queues = 0;
673 adapter->num_tx_queues = 0;
674}
675
9d5c8243
AK		 676/**
677 * igb_alloc_queues - Allocate memory for all rings
678 * @adapter: board private structure to initialize
679 *
680 * We allocate one ring per queue at run-time since we don't know the
681 * number of queues at compile-time.
682 **/
683static int igb_alloc_queues(struct igb_adapter *adapter)
684{
3025a446 685 struct igb_ring *ring;
9d5c8243 686 int i;
81c2fc22 687 int orig_node = adapter->node;
9d5c8243 688
661086df 689 for (i = 0; i < adapter->num_tx_queues; i++) {
81c2fc22
AD		 690 if (orig_node == -1) {
691 int cur_node = next_online_node(adapter->node);
692 if (cur_node == MAX_NUMNODES)
693 cur_node = first_online_node;
694 adapter->node = cur_node;
695 }
696 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
697 adapter->node);
698 if (!ring)
699 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
3025a446
AD		 700 if (!ring)
701 goto err;
68fd9910 702 ring->count = adapter->tx_ring_count;
661086df 703 ring->queue_index = i;
59d71989 704 ring->dev = &adapter->pdev->dev;
e694e964 705 ring->netdev = adapter->netdev;
81c2fc22 706 ring->numa_node = adapter->node;
85ad76b2
AD		 707 /* For 82575, context index must be unique per ring. */
708 if (adapter->hw.mac.type == e1000_82575)
866cff06 709 set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
3025a446 710 adapter->tx_ring[i] = ring;
661086df 711 }
81c2fc22
AD		 712 /* Restore the adapter's original node */
713 adapter->node = orig_node;
85ad76b2 714
9d5c8243 715 for (i = 0; i < adapter->num_rx_queues; i++) {
81c2fc22
AD		 716 if (orig_node == -1) {
717 int cur_node = next_online_node(adapter->node);
718 if (cur_node == MAX_NUMNODES)
719 cur_node = first_online_node;
720 adapter->node = cur_node;
721 }
722 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
723 adapter->node);
724 if (!ring)
725 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
3025a446
AD		 726 if (!ring)
727 goto err;
68fd9910 728 ring->count = adapter->rx_ring_count;
844290e5 729 ring->queue_index = i;
59d71989 730 ring->dev = &adapter->pdev->dev;
e694e964 731 ring->netdev = adapter->netdev;
81c2fc22 732 ring->numa_node = adapter->node;
85ad76b2
AD		 733 /* set flag indicating ring supports SCTP checksum offload */
734 if (adapter->hw.mac.type >= e1000_82576)
866cff06 735 set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
8be10e91 736
f96a8a0b
CW		 737 /*
738 * On i350, i210, and i211, loopback VLAN packets
739 * have the tag byte-swapped.
740 * */
741 if (adapter->hw.mac.type >= e1000_i350)
8be10e91
AD		 742 set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
743
3025a446 744 adapter->rx_ring[i] = ring;
9d5c8243 745 }
81c2fc22
AD		 746 /* Restore the adapter's original node */
747 adapter->node = orig_node;
26bc19ec
AD		 748
749 igb_cache_ring_register(adapter);
9d5c8243 750
047e0030 751 return 0;
a88f10ec 752
047e0030 753err:
81c2fc22
AD		 754 /* Restore the adapter's original node */
755 adapter->node = orig_node;
047e0030 756 igb_free_queues(adapter);
d1a8c9e1 757
047e0030 758 return -ENOMEM;
a88f10ec
AD		 759}
760
4be000c8
AD		 761/**
762 * igb_write_ivar - configure ivar for given MSI-X vector
763 * @hw: pointer to the HW structure
764 * @msix_vector: vector number we are allocating to a given ring
765 * @index: row index of IVAR register to write within IVAR table
766 * @offset: column offset of in IVAR, should be multiple of 8
767 *
768 * This function is intended to handle the writing of the IVAR register
769 * for adapters 82576 and newer. The IVAR table consists of 2 columns,
770 * each containing an cause allocation for an Rx and Tx ring, and a
771 * variable number of rows depending on the number of queues supported.
772 **/
773static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
774 int index, int offset)
775{
776 u32 ivar = array_rd32(E1000_IVAR0, index);
777
778 /* clear any bits that are currently set */
779 ivar &= ~((u32)0xFF << offset);
780
781 /* write vector and valid bit */
782 ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
783
784 array_wr32(E1000_IVAR0, index, ivar);
785}
786
9d5c8243 787#define IGB_N0_QUEUE -1
047e0030 788static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243 789{
047e0030 790 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 791 struct e1000_hw *hw = &adapter->hw;
047e0030
AD		 792 int rx_queue = IGB_N0_QUEUE;
793 int tx_queue = IGB_N0_QUEUE;
4be000c8 794 u32 msixbm = 0;
047e0030 795
0ba82994
AD		 796 if (q_vector->rx.ring)
797 rx_queue = q_vector->rx.ring->reg_idx;
798 if (q_vector->tx.ring)
799 tx_queue = q_vector->tx.ring->reg_idx;
2d064c06
AD		 800
801 switch (hw->mac.type) {
802 case e1000_82575:
9d5c8243
AK		 803 /* The 82575 assigns vectors using a bitmask, which matches the
804 bitmask for the EICR/EIMS/EIMC registers. To assign one
805 or more queues to a vector, we write the appropriate bits
806 into the MSIXBM register for that vector. */
047e0030 807 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 808 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 809 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 810 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 811 if (!adapter->msix_entries && msix_vector == 0)
812 msixbm |= E1000_EIMS_OTHER;
9d5c8243 813 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 814 q_vector->eims_value = msixbm;
2d064c06
AD		 815 break;
816 case e1000_82576:
4be000c8
AD		 817 /*
818 * 82576 uses a table that essentially consists of 2 columns
819 * with 8 rows. The ordering is column-major so we use the
820 * lower 3 bits as the row index, and the 4th bit as the
821 * column offset.
822 */
823 if (rx_queue > IGB_N0_QUEUE)
824 igb_write_ivar(hw, msix_vector,
825 rx_queue & 0x7,
826 (rx_queue & 0x8) << 1);
827 if (tx_queue > IGB_N0_QUEUE)
828 igb_write_ivar(hw, msix_vector,
829 tx_queue & 0x7,
830 ((tx_queue & 0x8) << 1) + 8);
047e0030 831 q_vector->eims_value = 1 << msix_vector;
2d064c06 832 break;
55cac248 833 case e1000_82580:
d2ba2ed8 834 case e1000_i350:
f96a8a0b
CW		 835 case e1000_i210:
836 case e1000_i211:
4be000c8
AD		 837 /*
838 * On 82580 and newer adapters the scheme is similar to 82576
839 * however instead of ordering column-major we have things
840 * ordered row-major. So we traverse the table by using
841 * bit 0 as the column offset, and the remaining bits as the
842 * row index.
843 */
844 if (rx_queue > IGB_N0_QUEUE)
845 igb_write_ivar(hw, msix_vector,
846 rx_queue >> 1,
847 (rx_queue & 0x1) << 4);
848 if (tx_queue > IGB_N0_QUEUE)
849 igb_write_ivar(hw, msix_vector,
850 tx_queue >> 1,
851 ((tx_queue & 0x1) << 4) + 8);
55cac248
AD		 852 q_vector->eims_value = 1 << msix_vector;
853 break;
2d064c06
AD		 854 default:
855 BUG();
856 break;
857 }
26b39276
AD		 858
859 /* add q_vector eims value to global eims_enable_mask */
860 adapter->eims_enable_mask |= q_vector->eims_value;
861
862 /* configure q_vector to set itr on first interrupt */
863 q_vector->set_itr = 1;
9d5c8243
AK		 864}
865
866/**
867 * igb_configure_msix - Configure MSI-X hardware
868 *
869 * igb_configure_msix sets up the hardware to properly
870 * generate MSI-X interrupts.
871 **/
872static void igb_configure_msix(struct igb_adapter *adapter)
873{
874 u32 tmp;
875 int i, vector = 0;
876 struct e1000_hw *hw = &adapter->hw;
877
878 adapter->eims_enable_mask = 0;
9d5c8243
AK		 879
880 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 881 switch (hw->mac.type) {
882 case e1000_82575:
9d5c8243
AK		 883 tmp = rd32(E1000_CTRL_EXT);
884 /* enable MSI-X PBA support*/
885 tmp |= E1000_CTRL_EXT_PBA_CLR;
886
887 /* Auto-Mask interrupts upon ICR read. */
888 tmp |= E1000_CTRL_EXT_EIAME;
889 tmp |= E1000_CTRL_EXT_IRCA;
890
891 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 892
893 /* enable msix_other interrupt */
894 array_wr32(E1000_MSIXBM(0), vector++,
895 E1000_EIMS_OTHER);
844290e5 896 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 897
2d064c06
AD		 898 break;
899
900 case e1000_82576:
55cac248 901 case e1000_82580:
d2ba2ed8 902 case e1000_i350:
f96a8a0b
CW		 903 case e1000_i210:
904 case e1000_i211:
047e0030
AD		 905 /* Turn on MSI-X capability first, or our settings
906 * won't stick. And it will take days to debug. */
907 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
908 E1000_GPIE_PBA | E1000_GPIE_EIAME |
909 E1000_GPIE_NSICR);
910
911 /* enable msix_other interrupt */
912 adapter->eims_other = 1 << vector;
2d064c06 913 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 914
047e0030 915 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 916 break;
917 default:
918 /* do nothing, since nothing else supports MSI-X */
919 break;
920 } /* switch (hw->mac.type) */
047e0030
AD		 921
922 adapter->eims_enable_mask |= adapter->eims_other;
923
26b39276
AD		 924 for (i = 0; i < adapter->num_q_vectors; i++)
925 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 926
9d5c8243
AK		 927 wrfl();
928}
929
930/**
931 * igb_request_msix - Initialize MSI-X interrupts
932 *
933 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
934 * kernel.
935 **/
936static int igb_request_msix(struct igb_adapter *adapter)
937{
938 struct net_device *netdev = adapter->netdev;
047e0030 939 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 940 int i, err = 0, vector = 0;
941
047e0030 942 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 943 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 944 if (err)
945 goto out;
946 vector++;
947
948 for (i = 0; i < adapter->num_q_vectors; i++) {
949 struct igb_q_vector *q_vector = adapter->q_vector[i];
950
951 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
952
0ba82994 953 if (q_vector->rx.ring && q_vector->tx.ring)
047e0030 954 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
0ba82994
AD		 955 q_vector->rx.ring->queue_index);
956 else if (q_vector->tx.ring)
047e0030 957 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
0ba82994
AD		 958 q_vector->tx.ring->queue_index);
959 else if (q_vector->rx.ring)
047e0030 960 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
0ba82994 961 q_vector->rx.ring->queue_index);
9d5c8243 962 else
047e0030
AD		 963 sprintf(q_vector->name, "%s-unused", netdev->name);
964
9d5c8243 965 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 966 igb_msix_ring, 0, q_vector->name,
047e0030 967 q_vector);
9d5c8243
AK		 968 if (err)
969 goto out;
9d5c8243
AK		 970 vector++;
971 }
972
9d5c8243
AK		 973 igb_configure_msix(adapter);
974 return 0;
975out:
976 return err;
977}
978
979static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
980{
981 if (adapter->msix_entries) {
982 pci_disable_msix(adapter->pdev);
983 kfree(adapter->msix_entries);
984 adapter->msix_entries = NULL;
047e0030 985 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 986 pci_disable_msi(adapter->pdev);
047e0030 987 }
9d5c8243
AK		 988}
989
047e0030
AD		 990/**
991 * igb_free_q_vectors - Free memory allocated for interrupt vectors
992 * @adapter: board private structure to initialize
993 *
994 * This function frees the memory allocated to the q_vectors. In addition if
995 * NAPI is enabled it will delete any references to the NAPI struct prior
996 * to freeing the q_vector.
997 **/
998static void igb_free_q_vectors(struct igb_adapter *adapter)
999{
1000 int v_idx;
1001
1002 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1003 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1004 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 1005 if (!q_vector)
1006 continue;
047e0030
AD		 1007 netif_napi_del(&q_vector->napi);
1008 kfree(q_vector);
1009 }
1010 adapter->num_q_vectors = 0;
1011}
1012
1013/**
1014 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
1015 *
1016 * This function resets the device so that it has 0 rx queues, tx queues, and
1017 * MSI-X interrupts allocated.
1018 */
1019static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
1020{
1021 igb_free_queues(adapter);
1022 igb_free_q_vectors(adapter);
1023 igb_reset_interrupt_capability(adapter);
1024}
9d5c8243
AK		 1025
1026/**
1027 * igb_set_interrupt_capability - set MSI or MSI-X if supported
1028 *
1029 * Attempt to configure interrupts using the best available
1030 * capabilities of the hardware and kernel.
1031 **/
21adef3e 1032static int igb_set_interrupt_capability(struct igb_adapter *adapter)
9d5c8243
AK		 1033{
1034 int err;
1035 int numvecs, i;
1036
83b7180d 1037 /* Number of supported queues. */
a99955fc 1038 adapter->num_rx_queues = adapter->rss_queues;
5fa8517f
GR		 1039 if (adapter->vfs_allocated_count)
1040 adapter->num_tx_queues = 1;
1041 else
1042 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 1043
047e0030
AD		 1044 /* start with one vector for every rx queue */
1045 numvecs = adapter->num_rx_queues;
1046
3ad2f3fb 1047 /* if tx handler is separate add 1 for every tx queue */
a99955fc
AD		 1048 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1049 numvecs += adapter->num_tx_queues;
047e0030
AD		 1050
1051 /* store the number of vectors reserved for queues */
1052 adapter->num_q_vectors = numvecs;
1053
1054 /* add 1 vector for link status interrupts */
1055 numvecs++;
9d5c8243
AK		 1056 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1057 GFP_KERNEL);
f96a8a0b 1058
9d5c8243
AK		 1059 if (!adapter->msix_entries)
1060 goto msi_only;
1061
1062 for (i = 0; i < numvecs; i++)
1063 adapter->msix_entries[i].entry = i;
1064
1065 err = pci_enable_msix(adapter->pdev,
1066 adapter->msix_entries,
1067 numvecs);
1068 if (err == 0)
34a20e89 1069 goto out;
9d5c8243
AK		 1070
1071 igb_reset_interrupt_capability(adapter);
1072
1073 /* If we can't do MSI-X, try MSI */
1074msi_only:
2a3abf6d
AD		 1075#ifdef CONFIG_PCI_IOV
1076 /* disable SR-IOV for non MSI-X configurations */
1077 if (adapter->vf_data) {
1078 struct e1000_hw *hw = &adapter->hw;
1079 /* disable iov and allow time for transactions to clear */
1080 pci_disable_sriov(adapter->pdev);
1081 msleep(500);
1082
1083 kfree(adapter->vf_data);
1084 adapter->vf_data = NULL;
1085 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 1086 wrfl();
2a3abf6d
AD		 1087 msleep(100);
1088 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1089 }
1090#endif
4fc82adf 1091 adapter->vfs_allocated_count = 0;
a99955fc 1092 adapter->rss_queues = 1;
4fc82adf 1093 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 1094 adapter->num_rx_queues = 1;
661086df 1095 adapter->num_tx_queues = 1;
047e0030 1096 adapter->num_q_vectors = 1;
9d5c8243 1097 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1098 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 1099out:
21adef3e 1100 /* Notify the stack of the (possibly) reduced queue counts. */
cfb8c3aa 1101 rtnl_lock();
21adef3e 1102 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
cfb8c3aa
BP		 1103 err = netif_set_real_num_rx_queues(adapter->netdev,
1104 adapter->num_rx_queues);
1105 rtnl_unlock();
1106 return err;
9d5c8243
AK		 1107}
1108
047e0030
AD		 1109/**
1110 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1111 * @adapter: board private structure to initialize
1112 *
1113 * We allocate one q_vector per queue interrupt. If allocation fails we
1114 * return -ENOMEM.
1115 **/
1116static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1117{
1118 struct igb_q_vector *q_vector;
1119 struct e1000_hw *hw = &adapter->hw;
1120 int v_idx;
81c2fc22 1121 int orig_node = adapter->node;
047e0030
AD		 1122
1123 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
81c2fc22
AD		 1124 if ((adapter->num_q_vectors == (adapter->num_rx_queues +
1125 adapter->num_tx_queues)) &&
1126 (adapter->num_rx_queues == v_idx))
1127 adapter->node = orig_node;
1128 if (orig_node == -1) {
1129 int cur_node = next_online_node(adapter->node);
1130 if (cur_node == MAX_NUMNODES)
1131 cur_node = first_online_node;
1132 adapter->node = cur_node;
1133 }
1134 q_vector = kzalloc_node(sizeof(struct igb_q_vector), GFP_KERNEL,
1135 adapter->node);
1136 if (!q_vector)
1137 q_vector = kzalloc(sizeof(struct igb_q_vector),
1138 GFP_KERNEL);
047e0030
AD		 1139 if (!q_vector)
1140 goto err_out;
1141 q_vector->adapter = adapter;
047e0030
AD		 1142 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1143 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1144 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1145 adapter->q_vector[v_idx] = q_vector;
1146 }
81c2fc22
AD		 1147 /* Restore the adapter's original node */
1148 adapter->node = orig_node;
1149
047e0030
AD		 1150 return 0;
1151
1152err_out:
81c2fc22
AD		 1153 /* Restore the adapter's original node */
1154 adapter->node = orig_node;
fe0592b4 1155 igb_free_q_vectors(adapter);
047e0030
AD		 1156 return -ENOMEM;
1157}
1158
1159static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1160 int ring_idx, int v_idx)
1161{
3025a446 1162 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1163
0ba82994
AD		 1164 q_vector->rx.ring = adapter->rx_ring[ring_idx];
1165 q_vector->rx.ring->q_vector = q_vector;
1166 q_vector->rx.count++;
4fc82adf
AD		 1167 q_vector->itr_val = adapter->rx_itr_setting;
1168 if (q_vector->itr_val && q_vector->itr_val <= 3)
1169 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1170}
1171
1172static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1173 int ring_idx, int v_idx)
1174{
3025a446 1175 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1176
0ba82994
AD		 1177 q_vector->tx.ring = adapter->tx_ring[ring_idx];
1178 q_vector->tx.ring->q_vector = q_vector;
1179 q_vector->tx.count++;
4fc82adf 1180 q_vector->itr_val = adapter->tx_itr_setting;
0ba82994 1181 q_vector->tx.work_limit = adapter->tx_work_limit;
4fc82adf
AD		 1182 if (q_vector->itr_val && q_vector->itr_val <= 3)
1183 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1184}
1185
1186/**
1187 * igb_map_ring_to_vector - maps allocated queues to vectors
1188 *
1189 * This function maps the recently allocated queues to vectors.
1190 **/
1191static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1192{
1193 int i;
1194 int v_idx = 0;
1195
1196 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1197 (adapter->num_q_vectors < adapter->num_tx_queues))
1198 return -ENOMEM;
1199
1200 if (adapter->num_q_vectors >=
1201 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1202 for (i = 0; i < adapter->num_rx_queues; i++)
1203 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1204 for (i = 0; i < adapter->num_tx_queues; i++)
1205 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1206 } else {
1207 for (i = 0; i < adapter->num_rx_queues; i++) {
1208 if (i < adapter->num_tx_queues)
1209 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1210 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1211 }
1212 for (; i < adapter->num_tx_queues; i++)
1213 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1214 }
1215 return 0;
1216}
1217
1218/**
1219 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1220 *
1221 * This function initializes the interrupts and allocates all of the queues.
1222 **/
1223static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1224{
1225 struct pci_dev *pdev = adapter->pdev;
1226 int err;
1227
21adef3e
BH		 1228 err = igb_set_interrupt_capability(adapter);
1229 if (err)
1230 return err;
047e0030
AD		 1231
1232 err = igb_alloc_q_vectors(adapter);
1233 if (err) {
1234 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1235 goto err_alloc_q_vectors;
1236 }
1237
1238 err = igb_alloc_queues(adapter);
1239 if (err) {
1240 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1241 goto err_alloc_queues;
1242 }
1243
1244 err = igb_map_ring_to_vector(adapter);
1245 if (err) {
1246 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1247 goto err_map_queues;
1248 }
1249
1250
1251 return 0;
1252err_map_queues:
1253 igb_free_queues(adapter);
1254err_alloc_queues:
1255 igb_free_q_vectors(adapter);
1256err_alloc_q_vectors:
1257 igb_reset_interrupt_capability(adapter);
1258 return err;
1259}
1260
9d5c8243
AK		 1261/**
1262 * igb_request_irq - initialize interrupts
1263 *
1264 * Attempts to configure interrupts using the best available
1265 * capabilities of the hardware and kernel.
1266 **/
1267static int igb_request_irq(struct igb_adapter *adapter)
1268{
1269 struct net_device *netdev = adapter->netdev;
047e0030 1270 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1271 int err = 0;
1272
1273 if (adapter->msix_entries) {
1274 err = igb_request_msix(adapter);
844290e5 1275 if (!err)
9d5c8243 1276 goto request_done;
9d5c8243 1277 /* fall back to MSI */
047e0030 1278 igb_clear_interrupt_scheme(adapter);
c74d588e 1279 if (!pci_enable_msi(pdev))
7dfc16fa 1280 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1281 igb_free_all_tx_resources(adapter);
1282 igb_free_all_rx_resources(adapter);
047e0030 1283 adapter->num_tx_queues = 1;
9d5c8243 1284 adapter->num_rx_queues = 1;
047e0030
AD		 1285 adapter->num_q_vectors = 1;
1286 err = igb_alloc_q_vectors(adapter);
1287 if (err) {
1288 dev_err(&pdev->dev,
1289 "Unable to allocate memory for vectors\n");
1290 goto request_done;
1291 }
1292 err = igb_alloc_queues(adapter);
1293 if (err) {
1294 dev_err(&pdev->dev,
1295 "Unable to allocate memory for queues\n");
1296 igb_free_q_vectors(adapter);
1297 goto request_done;
1298 }
1299 igb_setup_all_tx_resources(adapter);
1300 igb_setup_all_rx_resources(adapter);
9d5c8243 1301 }
844290e5 1302
c74d588e
AD		 1303 igb_assign_vector(adapter->q_vector[0], 0);
1304
7dfc16fa 1305 if (adapter->flags & IGB_FLAG_HAS_MSI) {
c74d588e 1306 err = request_irq(pdev->irq, igb_intr_msi, 0,
047e0030 1307 netdev->name, adapter);
9d5c8243
AK		 1308 if (!err)
1309 goto request_done;
047e0030 1310
9d5c8243
AK		 1311 /* fall back to legacy interrupts */
1312 igb_reset_interrupt_capability(adapter);
7dfc16fa 1313 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1314 }
1315
c74d588e 1316 err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
047e0030 1317 netdev->name, adapter);
9d5c8243 1318
6cb5e577 1319 if (err)
c74d588e 1320 dev_err(&pdev->dev, "Error %d getting interrupt\n",
9d5c8243 1321 err);
9d5c8243
AK		 1322
1323request_done:
1324 return err;
1325}
1326
1327static void igb_free_irq(struct igb_adapter *adapter)
1328{
9d5c8243
AK		 1329 if (adapter->msix_entries) {
1330 int vector = 0, i;
1331
047e0030 1332 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 1333
0d1ae7f4 1334 for (i = 0; i < adapter->num_q_vectors; i++)
047e0030 1335 free_irq(adapter->msix_entries[vector++].vector,
0d1ae7f4 1336 adapter->q_vector[i]);
047e0030
AD		 1337 } else {
1338 free_irq(adapter->pdev->irq, adapter);
9d5c8243 1339 }
9d5c8243
AK		 1340}
1341
1342/**
1343 * igb_irq_disable - Mask off interrupt generation on the NIC
1344 * @adapter: board private structure
1345 **/
1346static void igb_irq_disable(struct igb_adapter *adapter)
1347{
1348 struct e1000_hw *hw = &adapter->hw;
1349
25568a53
AD		 1350 /*
1351 * we need to be careful when disabling interrupts. The VFs are also
1352 * mapped into these registers and so clearing the bits can cause
1353 * issues on the VF drivers so we only need to clear what we set
1354 */
9d5c8243 1355 if (adapter->msix_entries) {
2dfd1212
AD		 1356 u32 regval = rd32(E1000_EIAM);
1357 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1358 wr32(E1000_EIMC, adapter->eims_enable_mask);
1359 regval = rd32(E1000_EIAC);
1360 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 1361 }
844290e5
PW		 1362
1363 wr32(E1000_IAM, 0);
9d5c8243
AK		 1364 wr32(E1000_IMC, ~0);
1365 wrfl();
81a61859
ET		 1366 if (adapter->msix_entries) {
1367 int i;
1368 for (i = 0; i < adapter->num_q_vectors; i++)
1369 synchronize_irq(adapter->msix_entries[i].vector);
1370 } else {
1371 synchronize_irq(adapter->pdev->irq);
1372 }
9d5c8243
AK		 1373}
1374
1375/**
1376 * igb_irq_enable - Enable default interrupt generation settings
1377 * @adapter: board private structure
1378 **/
1379static void igb_irq_enable(struct igb_adapter *adapter)
1380{
1381 struct e1000_hw *hw = &adapter->hw;
1382
1383 if (adapter->msix_entries) {
06218a8d 1384 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
2dfd1212
AD		 1385 u32 regval = rd32(E1000_EIAC);
1386 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1387 regval = rd32(E1000_EIAM);
1388 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 1389 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 1390 if (adapter->vfs_allocated_count) {
4ae196df 1391 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1392 ims |= E1000_IMS_VMMB;
1393 }
1394 wr32(E1000_IMS, ims);
844290e5 1395 } else {
55cac248
AD		 1396 wr32(E1000_IMS, IMS_ENABLE_MASK |
1397 E1000_IMS_DRSTA);
1398 wr32(E1000_IAM, IMS_ENABLE_MASK |
1399 E1000_IMS_DRSTA);
844290e5 1400 }
9d5c8243
AK		 1401}
1402
1403static void igb_update_mng_vlan(struct igb_adapter *adapter)
1404{
51466239 1405 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1406 u16 vid = adapter->hw.mng_cookie.vlan_id;
1407 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1408
1409 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1410 /* add VID to filter table */
1411 igb_vfta_set(hw, vid, true);
1412 adapter->mng_vlan_id = vid;
1413 } else {
1414 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1415 }
1416
1417 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1418 (vid != old_vid) &&
b2cb09b1 1419 !test_bit(old_vid, adapter->active_vlans)) {
51466239
AD		 1420 /* remove VID from filter table */
1421 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1422 }
1423}
1424
1425/**
1426 * igb_release_hw_control - release control of the h/w to f/w
1427 * @adapter: address of board private structure
1428 *
1429 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1430 * For ASF and Pass Through versions of f/w this means that the
1431 * driver is no longer loaded.
1432 *
1433 **/
1434static void igb_release_hw_control(struct igb_adapter *adapter)
1435{
1436 struct e1000_hw *hw = &adapter->hw;
1437 u32 ctrl_ext;
1438
1439 /* Let firmware take over control of h/w */
1440 ctrl_ext = rd32(E1000_CTRL_EXT);
1441 wr32(E1000_CTRL_EXT,
1442 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1443}
1444
9d5c8243
AK		 1445/**
1446 * igb_get_hw_control - get control of the h/w from f/w
1447 * @adapter: address of board private structure
1448 *
1449 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1450 * For ASF and Pass Through versions of f/w this means that
1451 * the driver is loaded.
1452 *
1453 **/
1454static void igb_get_hw_control(struct igb_adapter *adapter)
1455{
1456 struct e1000_hw *hw = &adapter->hw;
1457 u32 ctrl_ext;
1458
1459 /* Let firmware know the driver has taken over */
1460 ctrl_ext = rd32(E1000_CTRL_EXT);
1461 wr32(E1000_CTRL_EXT,
1462 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1463}
1464
9d5c8243
AK		 1465/**
1466 * igb_configure - configure the hardware for RX and TX
1467 * @adapter: private board structure
1468 **/
1469static void igb_configure(struct igb_adapter *adapter)
1470{
1471 struct net_device *netdev = adapter->netdev;
1472 int i;
1473
1474 igb_get_hw_control(adapter);
ff41f8dc 1475 igb_set_rx_mode(netdev);
9d5c8243
AK		 1476
1477 igb_restore_vlan(adapter);
9d5c8243 1478
85b430b4 1479 igb_setup_tctl(adapter);
06cf2666 1480 igb_setup_mrqc(adapter);
9d5c8243 1481 igb_setup_rctl(adapter);
85b430b4
AD		 1482
1483 igb_configure_tx(adapter);
9d5c8243 1484 igb_configure_rx(adapter);
662d7205
AD		 1485
1486 igb_rx_fifo_flush_82575(&adapter->hw);
1487
c493ea45 1488 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1489 * at least 1 descriptor unused to make sure
1490 * next_to_use != next_to_clean */
1491 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1492 struct igb_ring *ring = adapter->rx_ring[i];
cd392f5c 1493 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
9d5c8243 1494 }
9d5c8243
AK		 1495}
1496
88a268c1
NN		 1497/**
1498 * igb_power_up_link - Power up the phy/serdes link
1499 * @adapter: address of board private structure
1500 **/
1501void igb_power_up_link(struct igb_adapter *adapter)
1502{
1503 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1504 igb_power_up_phy_copper(&adapter->hw);
1505 else
1506 igb_power_up_serdes_link_82575(&adapter->hw);
a95a0744 1507 igb_reset_phy(&adapter->hw);
88a268c1
NN		 1508}
1509
1510/**
1511 * igb_power_down_link - Power down the phy/serdes link
1512 * @adapter: address of board private structure
1513 */
1514static void igb_power_down_link(struct igb_adapter *adapter)
1515{
1516 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1517 igb_power_down_phy_copper_82575(&adapter->hw);
1518 else
1519 igb_shutdown_serdes_link_82575(&adapter->hw);
1520}
9d5c8243
AK		 1521
1522/**
1523 * igb_up - Open the interface and prepare it to handle traffic
1524 * @adapter: board private structure
1525 **/
9d5c8243
AK		 1526int igb_up(struct igb_adapter *adapter)
1527{
1528 struct e1000_hw *hw = &adapter->hw;
1529 int i;
1530
1531 /* hardware has been reset, we need to reload some things */
1532 igb_configure(adapter);
1533
1534 clear_bit(__IGB_DOWN, &adapter->state);
1535
0d1ae7f4
AD		 1536 for (i = 0; i < adapter->num_q_vectors; i++)
1537 napi_enable(&(adapter->q_vector[i]->napi));
1538
844290e5 1539 if (adapter->msix_entries)
9d5c8243 1540 igb_configure_msix(adapter);
feeb2721
AD		 1541 else
1542 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1543
1544 /* Clear any pending interrupts. */
1545 rd32(E1000_ICR);
1546 igb_irq_enable(adapter);
1547
d4960307
AD		 1548 /* notify VFs that reset has been completed */
1549 if (adapter->vfs_allocated_count) {
1550 u32 reg_data = rd32(E1000_CTRL_EXT);
1551 reg_data |= E1000_CTRL_EXT_PFRSTD;
1552 wr32(E1000_CTRL_EXT, reg_data);
1553 }
1554
4cb9be7a
JB		 1555 netif_tx_start_all_queues(adapter->netdev);
1556
25568a53
AD		 1557 /* start the watchdog. */
1558 hw->mac.get_link_status = 1;
1559 schedule_work(&adapter->watchdog_task);
1560
9d5c8243
AK		 1561 return 0;
1562}
1563
1564void igb_down(struct igb_adapter *adapter)
1565{
9d5c8243 1566 struct net_device *netdev = adapter->netdev;
330a6d6a 1567 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1568 u32 tctl, rctl;
1569 int i;
1570
1571 /* signal that we're down so the interrupt handler does not
1572 * reschedule our watchdog timer */
1573 set_bit(__IGB_DOWN, &adapter->state);
1574
1575 /* disable receives in the hardware */
1576 rctl = rd32(E1000_RCTL);
1577 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1578 /* flush and sleep below */
1579
fd2ea0a7 1580 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1581
1582 /* disable transmits in the hardware */
1583 tctl = rd32(E1000_TCTL);
1584 tctl &= ~E1000_TCTL_EN;
1585 wr32(E1000_TCTL, tctl);
1586 /* flush both disables and wait for them to finish */
1587 wrfl();
1588 msleep(10);
1589
0d1ae7f4
AD		 1590 for (i = 0; i < adapter->num_q_vectors; i++)
1591 napi_disable(&(adapter->q_vector[i]->napi));
9d5c8243 1592
9d5c8243
AK		 1593 igb_irq_disable(adapter);
1594
1595 del_timer_sync(&adapter->watchdog_timer);
1596 del_timer_sync(&adapter->phy_info_timer);
1597
9d5c8243 1598 netif_carrier_off(netdev);
04fe6358
AD		 1599
1600 /* record the stats before reset*/
12dcd86b
ED		 1601 spin_lock(&adapter->stats64_lock);
1602 igb_update_stats(adapter, &adapter->stats64);
1603 spin_unlock(&adapter->stats64_lock);
04fe6358 1604
9d5c8243
AK		 1605 adapter->link_speed = 0;
1606 adapter->link_duplex = 0;
1607
3023682e
JK		 1608 if (!pci_channel_offline(adapter->pdev))
1609 igb_reset(adapter);
9d5c8243
AK		 1610 igb_clean_all_tx_rings(adapter);
1611 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1612#ifdef CONFIG_IGB_DCA
1613
1614 /* since we reset the hardware DCA settings were cleared */
1615 igb_setup_dca(adapter);
1616#endif
9d5c8243
AK		 1617}
1618
1619void igb_reinit_locked(struct igb_adapter *adapter)
1620{
1621 WARN_ON(in_interrupt());
1622 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1623 msleep(1);
1624 igb_down(adapter);
1625 igb_up(adapter);
1626 clear_bit(__IGB_RESETTING, &adapter->state);
1627}
1628
1629void igb_reset(struct igb_adapter *adapter)
1630{
090b1795 1631 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1632 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1633 struct e1000_mac_info *mac = &hw->mac;
1634 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1635 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1636 u16 hwm;
1637
1638 /* Repartition Pba for greater than 9k mtu
1639 * To take effect CTRL.RST is required.
1640 */
fa4dfae0 1641 switch (mac->type) {
d2ba2ed8 1642 case e1000_i350:
55cac248
AD		 1643 case e1000_82580:
1644 pba = rd32(E1000_RXPBS);
1645 pba = igb_rxpbs_adjust_82580(pba);
1646 break;
fa4dfae0 1647 case e1000_82576:
d249be54
AD		 1648 pba = rd32(E1000_RXPBS);
1649 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1650 break;
1651 case e1000_82575:
f96a8a0b
CW		 1652 case e1000_i210:
1653 case e1000_i211:
fa4dfae0
AD		 1654 default:
1655 pba = E1000_PBA_34K;
1656 break;
2d064c06 1657 }
9d5c8243 1658
2d064c06
AD		 1659 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1660 (mac->type < e1000_82576)) {
9d5c8243
AK		 1661 /* adjust PBA for jumbo frames */
1662 wr32(E1000_PBA, pba);
1663
1664 /* To maintain wire speed transmits, the Tx FIFO should be
1665 * large enough to accommodate two full transmit packets,
1666 * rounded up to the next 1KB and expressed in KB. Likewise,
1667 * the Rx FIFO should be large enough to accommodate at least
1668 * one full receive packet and is similarly rounded up and
1669 * expressed in KB. */
1670 pba = rd32(E1000_PBA);
1671 /* upper 16 bits has Tx packet buffer allocation size in KB */
1672 tx_space = pba >> 16;
1673 /* lower 16 bits has Rx packet buffer allocation size in KB */
1674 pba &= 0xffff;
1675 /* the tx fifo also stores 16 bytes of information about the tx
1676 * but don't include ethernet FCS because hardware appends it */
1677 min_tx_space = (adapter->max_frame_size +
85e8d004 1678 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1679 ETH_FCS_LEN) * 2;
1680 min_tx_space = ALIGN(min_tx_space, 1024);
1681 min_tx_space >>= 10;
1682 /* software strips receive CRC, so leave room for it */
1683 min_rx_space = adapter->max_frame_size;
1684 min_rx_space = ALIGN(min_rx_space, 1024);
1685 min_rx_space >>= 10;
1686
1687 /* If current Tx allocation is less than the min Tx FIFO size,
1688 * and the min Tx FIFO size is less than the current Rx FIFO
1689 * allocation, take space away from current Rx allocation */
1690 if (tx_space < min_tx_space &&
1691 ((min_tx_space - tx_space) < pba)) {
1692 pba = pba - (min_tx_space - tx_space);
1693
1694 /* if short on rx space, rx wins and must trump tx
1695 * adjustment */
1696 if (pba < min_rx_space)
1697 pba = min_rx_space;
1698 }
2d064c06 1699 wr32(E1000_PBA, pba);
9d5c8243 1700 }
9d5c8243
AK		 1701
1702 /* flow control settings */
1703 /* The high water mark must be low enough to fit one full frame
1704 * (or the size used for early receive) above it in the Rx FIFO.
1705 * Set it to the lower of:
1706 * - 90% of the Rx FIFO size, or
1707 * - the full Rx FIFO size minus one full frame */
1708 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1709 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1710
d405ea3e
AD		 1711 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1712 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1713 fc->pause_time = 0xFFFF;
1714 fc->send_xon = 1;
0cce119a 1715 fc->current_mode = fc->requested_mode;
9d5c8243 1716
4ae196df
AD		 1717 /* disable receive for all VFs and wait one second */
1718 if (adapter->vfs_allocated_count) {
1719 int i;
1720 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
8fa7e0f7 1721 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
4ae196df
AD		 1722
1723 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1724 igb_ping_all_vfs(adapter);
4ae196df
AD		 1725
1726 /* disable transmits and receives */
1727 wr32(E1000_VFRE, 0);
1728 wr32(E1000_VFTE, 0);
1729 }
1730
9d5c8243 1731 /* Allow time for pending master requests to run */
330a6d6a 1732 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1733 wr32(E1000_WUC, 0);
1734
330a6d6a 1735 if (hw->mac.ops.init_hw(hw))
090b1795 1736 dev_err(&pdev->dev, "Hardware Error\n");
831ec0b4 1737
a27416bb
MV		 1738 /*
1739 * Flow control settings reset on hardware reset, so guarantee flow
1740 * control is off when forcing speed.
1741 */
1742 if (!hw->mac.autoneg)
1743 igb_force_mac_fc(hw);
1744
b6e0c419 1745 igb_init_dmac(adapter, pba);
88a268c1
NN		 1746 if (!netif_running(adapter->netdev))
1747 igb_power_down_link(adapter);
1748
9d5c8243
AK		 1749 igb_update_mng_vlan(adapter);
1750
1751 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1752 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1753
330a6d6a 1754 igb_get_phy_info(hw);
9d5c8243
AK		 1755}
1756
c8f44aff
MM		 1757static netdev_features_t igb_fix_features(struct net_device *netdev,
1758 netdev_features_t features)
b2cb09b1
JP		 1759{
1760 /*
1761 * Since there is no support for separate rx/tx vlan accel
1762 * enable/disable make sure tx flag is always in same state as rx.
1763 */
1764 if (features & NETIF_F_HW_VLAN_RX)
1765 features |= NETIF_F_HW_VLAN_TX;
1766 else
1767 features &= ~NETIF_F_HW_VLAN_TX;
1768
1769 return features;
1770}
1771
c8f44aff
MM		 1772static int igb_set_features(struct net_device *netdev,
1773 netdev_features_t features)
ac52caa3 1774{
c8f44aff 1775 netdev_features_t changed = netdev->features ^ features;
89eaefb6 1776 struct igb_adapter *adapter = netdev_priv(netdev);
ac52caa3 1777
b2cb09b1
JP		 1778 if (changed & NETIF_F_HW_VLAN_RX)
1779 igb_vlan_mode(netdev, features);
1780
89eaefb6
BG		 1781 if (!(changed & NETIF_F_RXALL))
1782 return 0;
1783
1784 netdev->features = features;
1785
1786 if (netif_running(netdev))
1787 igb_reinit_locked(adapter);
1788 else
1789 igb_reset(adapter);
1790
ac52caa3
MM		 1791 return 0;
1792}
1793
2e5c6922 1794static const struct net_device_ops igb_netdev_ops = {
559e9c49 1795 .ndo_open = igb_open,
2e5c6922 1796 .ndo_stop = igb_close,
cd392f5c 1797 .ndo_start_xmit = igb_xmit_frame,
12dcd86b 1798 .ndo_get_stats64 = igb_get_stats64,
ff41f8dc 1799 .ndo_set_rx_mode = igb_set_rx_mode,
2e5c6922
SH		 1800 .ndo_set_mac_address = igb_set_mac,
1801 .ndo_change_mtu = igb_change_mtu,
1802 .ndo_do_ioctl = igb_ioctl,
1803 .ndo_tx_timeout = igb_tx_timeout,
1804 .ndo_validate_addr = eth_validate_addr,
2e5c6922
SH		 1805 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1806 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1807 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1808 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1809 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1810 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1811#ifdef CONFIG_NET_POLL_CONTROLLER
1812 .ndo_poll_controller = igb_netpoll,
1813#endif
b2cb09b1
JP		 1814 .ndo_fix_features = igb_fix_features,
1815 .ndo_set_features = igb_set_features,
2e5c6922
SH		 1816};
1817
d67974f0
CW		 1818/**
1819 * igb_set_fw_version - Configure version string for ethtool
1820 * @adapter: adapter struct
1821 *
1822 **/
1823void igb_set_fw_version(struct igb_adapter *adapter)
1824{
1825 struct e1000_hw *hw = &adapter->hw;
1826 u16 eeprom_verh, eeprom_verl, comb_verh, comb_verl, comb_offset;
1827 u16 major, build, patch, fw_version;
1828 u32 etrack_id;
1829
1830 hw->nvm.ops.read(hw, 5, 1, &fw_version);
1831 if (adapter->hw.mac.type != e1000_i211) {
1832 hw->nvm.ops.read(hw, NVM_ETRACK_WORD, 1, &eeprom_verh);
1833 hw->nvm.ops.read(hw, (NVM_ETRACK_WORD + 1), 1, &eeprom_verl);
1834 etrack_id = (eeprom_verh << IGB_ETRACK_SHIFT) | eeprom_verl;
1835
1836 /* combo image version needs to be found */
1837 hw->nvm.ops.read(hw, NVM_COMB_VER_PTR, 1, &comb_offset);
1838 if ((comb_offset != 0x0) &&
1839 (comb_offset != IGB_NVM_VER_INVALID)) {
1840 hw->nvm.ops.read(hw, (NVM_COMB_VER_OFF + comb_offset
1841 + 1), 1, &comb_verh);
1842 hw->nvm.ops.read(hw, (NVM_COMB_VER_OFF + comb_offset),
1843 1, &comb_verl);
1844
1845 /* Only display Option Rom if it exists and is valid */
1846 if ((comb_verh && comb_verl) &&
1847 ((comb_verh != IGB_NVM_VER_INVALID) &&
1848 (comb_verl != IGB_NVM_VER_INVALID))) {
1849 major = comb_verl >> IGB_COMB_VER_SHFT;
1850 build = (comb_verl << IGB_COMB_VER_SHFT) |
1851 (comb_verh >> IGB_COMB_VER_SHFT);
1852 patch = comb_verh & IGB_COMB_VER_MASK;
1853 snprintf(adapter->fw_version,
1854 sizeof(adapter->fw_version),
1855 "%d.%d%d, 0x%08x, %d.%d.%d",
1856 (fw_version & IGB_MAJOR_MASK) >>
1857 IGB_MAJOR_SHIFT,
1858 (fw_version & IGB_MINOR_MASK) >>
1859 IGB_MINOR_SHIFT,
1860 (fw_version & IGB_BUILD_MASK),
1861 etrack_id, major, build, patch);
1862 goto out;
1863 }
1864 }
1865 snprintf(adapter->fw_version, sizeof(adapter->fw_version),
1866 "%d.%d%d, 0x%08x",
1867 (fw_version & IGB_MAJOR_MASK) >> IGB_MAJOR_SHIFT,
1868 (fw_version & IGB_MINOR_MASK) >> IGB_MINOR_SHIFT,
1869 (fw_version & IGB_BUILD_MASK), etrack_id);
1870 } else {
1871 snprintf(adapter->fw_version, sizeof(adapter->fw_version),
1872 "%d.%d%d",
1873 (fw_version & IGB_MAJOR_MASK) >> IGB_MAJOR_SHIFT,
1874 (fw_version & IGB_MINOR_MASK) >> IGB_MINOR_SHIFT,
1875 (fw_version & IGB_BUILD_MASK));
1876 }
1877out:
1878 return;
1879}
1880
9d5c8243
AK		 1881/**
1882 * igb_probe - Device Initialization Routine
1883 * @pdev: PCI device information struct
1884 * @ent: entry in igb_pci_tbl
1885 *
1886 * Returns 0 on success, negative on failure
1887 *
1888 * igb_probe initializes an adapter identified by a pci_dev structure.
1889 * The OS initialization, configuring of the adapter private structure,
1890 * and a hardware reset occur.
1891 **/
1892static int __devinit igb_probe(struct pci_dev *pdev,
1893 const struct pci_device_id *ent)
1894{
1895 struct net_device *netdev;
1896 struct igb_adapter *adapter;
1897 struct e1000_hw *hw;
4337e993 1898 u16 eeprom_data = 0;
9835fd73 1899 s32 ret_val;
4337e993 1900 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1901 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1902 unsigned long mmio_start, mmio_len;
2d6a5e95 1903 int err, pci_using_dac;
9d5c8243 1904 u16 eeprom_apme_mask = IGB_EEPROM_APME;
9835fd73 1905 u8 part_str[E1000_PBANUM_LENGTH];
9d5c8243 1906
bded64a7
AG		 1907 /* Catch broken hardware that put the wrong VF device ID in
1908 * the PCIe SR-IOV capability.
1909 */
1910 if (pdev->is_virtfn) {
1911 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
f96a8a0b 1912 pci_name(pdev), pdev->vendor, pdev->device);
bded64a7
AG		 1913 return -EINVAL;
1914 }
1915
aed5dec3 1916 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1917 if (err)
1918 return err;
1919
1920 pci_using_dac = 0;
59d71989 1921 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243 1922 if (!err) {
59d71989 1923 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243
AK		 1924 if (!err)
1925 pci_using_dac = 1;
1926 } else {
59d71989 1927 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243 1928 if (err) {
59d71989 1929 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243
AK		 1930 if (err) {
1931 dev_err(&pdev->dev, "No usable DMA "
1932 "configuration, aborting\n");
1933 goto err_dma;
1934 }
1935 }
1936 }
1937
aed5dec3
AD		 1938 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1939 IORESOURCE_MEM),
1940 igb_driver_name);
9d5c8243
AK		 1941 if (err)
1942 goto err_pci_reg;
1943
19d5afd4 1944 pci_enable_pcie_error_reporting(pdev);
40a914fa 1945
9d5c8243 1946 pci_set_master(pdev);
c682fc23 1947 pci_save_state(pdev);
9d5c8243
AK		 1948
1949 err = -ENOMEM;
1bfaf07b 1950 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1cc3bd87 1951 IGB_MAX_TX_QUEUES);
9d5c8243
AK		 1952 if (!netdev)
1953 goto err_alloc_etherdev;
1954
1955 SET_NETDEV_DEV(netdev, &pdev->dev);
1956
1957 pci_set_drvdata(pdev, netdev);
1958 adapter = netdev_priv(netdev);
1959 adapter->netdev = netdev;
1960 adapter->pdev = pdev;
1961 hw = &adapter->hw;
1962 hw->back = adapter;
b3f4d599 1963 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
9d5c8243
AK		 1964
1965 mmio_start = pci_resource_start(pdev, 0);
1966 mmio_len = pci_resource_len(pdev, 0);
1967
1968 err = -EIO;
28b0759c
AD		 1969 hw->hw_addr = ioremap(mmio_start, mmio_len);
1970 if (!hw->hw_addr)
9d5c8243
AK		 1971 goto err_ioremap;
1972
2e5c6922 1973 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1974 igb_set_ethtool_ops(netdev);
9d5c8243 1975 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1976
1977 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1978
1979 netdev->mem_start = mmio_start;
1980 netdev->mem_end = mmio_start + mmio_len;
1981
9d5c8243
AK		 1982 /* PCI config space info */
1983 hw->vendor_id = pdev->vendor;
1984 hw->device_id = pdev->device;
1985 hw->revision_id = pdev->revision;
1986 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1987 hw->subsystem_device_id = pdev->subsystem_device;
1988
9d5c8243
AK		 1989 /* Copy the default MAC, PHY and NVM function pointers */
1990 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1991 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1992 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1993 /* Initialize skew-specific constants */
1994 err = ei->get_invariants(hw);
1995 if (err)
450c87c8 1996 goto err_sw_init;
9d5c8243 1997
450c87c8 1998 /* setup the private structure */
9d5c8243
AK		 1999 err = igb_sw_init(adapter);
2000 if (err)
2001 goto err_sw_init;
2002
2003 igb_get_bus_info_pcie(hw);
2004
2005 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 2006
2007 /* Copper options */
2008 if (hw->phy.media_type == e1000_media_type_copper) {
2009 hw->phy.mdix = AUTO_ALL_MODES;
2010 hw->phy.disable_polarity_correction = false;
2011 hw->phy.ms_type = e1000_ms_hw_default;
2012 }
2013
2014 if (igb_check_reset_block(hw))
2015 dev_info(&pdev->dev,
2016 "PHY reset is blocked due to SOL/IDER session.\n");
2017
077887c3
AD		 2018 /*
2019 * features is initialized to 0 in allocation, it might have bits
2020 * set by igb_sw_init so we should use an or instead of an
2021 * assignment.
2022 */
2023 netdev->features |= NETIF_F_SG |
2024 NETIF_F_IP_CSUM |
2025 NETIF_F_IPV6_CSUM |
2026 NETIF_F_TSO |
2027 NETIF_F_TSO6 |
2028 NETIF_F_RXHASH |
2029 NETIF_F_RXCSUM |
2030 NETIF_F_HW_VLAN_RX |
2031 NETIF_F_HW_VLAN_TX;
2032
2033 /* copy netdev features into list of user selectable features */
2034 netdev->hw_features |= netdev->features;
89eaefb6 2035 netdev->hw_features |= NETIF_F_RXALL;
077887c3
AD		 2036
2037 /* set this bit last since it cannot be part of hw_features */
2038 netdev->features |= NETIF_F_HW_VLAN_FILTER;
2039
2040 netdev->vlan_features |= NETIF_F_TSO |
2041 NETIF_F_TSO6 |
2042 NETIF_F_IP_CSUM |
2043 NETIF_F_IPV6_CSUM |
2044 NETIF_F_SG;
48f29ffc 2045
6b8f0922
BG		 2046 netdev->priv_flags |= IFF_SUPP_NOFCS;
2047
7b872a55 2048 if (pci_using_dac) {
9d5c8243 2049 netdev->features |= NETIF_F_HIGHDMA;
7b872a55
YZ		 2050 netdev->vlan_features |= NETIF_F_HIGHDMA;
2051 }
9d5c8243 2052
ac52caa3
MM		 2053 if (hw->mac.type >= e1000_82576) {
2054 netdev->hw_features |= NETIF_F_SCTP_CSUM;
b9473560 2055 netdev->features |= NETIF_F_SCTP_CSUM;
ac52caa3 2056 }
b9473560 2057
01789349
JP		 2058 netdev->priv_flags |= IFF_UNICAST_FLT;
2059
330a6d6a 2060 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 2061
2062 /* before reading the NVM, reset the controller to put the device in a
2063 * known good starting state */
2064 hw->mac.ops.reset_hw(hw);
2065
f96a8a0b
CW		 2066 /*
2067 * make sure the NVM is good , i211 parts have special NVM that
2068 * doesn't contain a checksum
2069 */
2070 if (hw->mac.type != e1000_i211) {
2071 if (hw->nvm.ops.validate(hw) < 0) {
2072 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
2073 err = -EIO;
2074 goto err_eeprom;
2075 }
9d5c8243
AK		 2076 }
2077
2078 /* copy the MAC address out of the NVM */
2079 if (hw->mac.ops.read_mac_addr(hw))
2080 dev_err(&pdev->dev, "NVM Read Error\n");
2081
2082 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2083 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
2084
2085 if (!is_valid_ether_addr(netdev->perm_addr)) {
2086 dev_err(&pdev->dev, "Invalid MAC Address\n");
2087 err = -EIO;
2088 goto err_eeprom;
2089 }
2090
d67974f0
CW		 2091 /* get firmware version for ethtool -i */
2092 igb_set_fw_version(adapter);
2093
c061b18d 2094 setup_timer(&adapter->watchdog_timer, igb_watchdog,
0e340485 2095 (unsigned long) adapter);
c061b18d 2096 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
0e340485 2097 (unsigned long) adapter);
9d5c8243
AK		 2098
2099 INIT_WORK(&adapter->reset_task, igb_reset_task);
2100 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2101
450c87c8 2102 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 2103 adapter->fc_autoneg = true;
2104 hw->mac.autoneg = true;
2105 hw->phy.autoneg_advertised = 0x2f;
2106
0cce119a
AD		 2107 hw->fc.requested_mode = e1000_fc_default;
2108 hw->fc.current_mode = e1000_fc_default;
9d5c8243 2109
9d5c8243
AK		 2110 igb_validate_mdi_setting(hw);
2111
9d5c8243
AK		 2112 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2113 * enable the ACPI Magic Packet filter
2114 */
2115
a2cf8b6c 2116 if (hw->bus.func == 0)
312c75ae 2117 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
6d337dce 2118 else if (hw->mac.type >= e1000_82580)
55cac248
AD		 2119 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2120 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2121 &eeprom_data);
a2cf8b6c
AD		 2122 else if (hw->bus.func == 1)
2123 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 2124
2125 if (eeprom_data & eeprom_apme_mask)
2126 adapter->eeprom_wol |= E1000_WUFC_MAG;
2127
2128 /* now that we have the eeprom settings, apply the special cases where
2129 * the eeprom may be wrong or the board simply won't support wake on
2130 * lan on a particular port */
2131 switch (pdev->device) {
2132 case E1000_DEV_ID_82575GB_QUAD_COPPER:
2133 adapter->eeprom_wol = 0;
2134 break;
2135 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 2136 case E1000_DEV_ID_82576_FIBER:
2137 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 2138 /* Wake events only supported on port A for dual fiber
2139 * regardless of eeprom setting */
2140 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
2141 adapter->eeprom_wol = 0;
2142 break;
c8ea5ea9 2143 case E1000_DEV_ID_82576_QUAD_COPPER:
d5aa2252 2144 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
c8ea5ea9
AD		 2145 /* if quad port adapter, disable WoL on all but port A */
2146 if (global_quad_port_a != 0)
2147 adapter->eeprom_wol = 0;
2148 else
2149 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2150 /* Reset for multiple quad port adapters */
2151 if (++global_quad_port_a == 4)
2152 global_quad_port_a = 0;
2153 break;
9d5c8243
AK		 2154 }
2155
2156 /* initialize the wol settings based on the eeprom settings */
2157 adapter->wol = adapter->eeprom_wol;
e1b86d84 2158 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 2159
2160 /* reset the hardware with the new settings */
2161 igb_reset(adapter);
2162
2163 /* let the f/w know that the h/w is now under the control of the
2164 * driver. */
2165 igb_get_hw_control(adapter);
2166
9d5c8243
AK		 2167 strcpy(netdev->name, "eth%d");
2168 err = register_netdev(netdev);
2169 if (err)
2170 goto err_register;
2171
b168dfc5
JB		 2172 /* carrier off reporting is important to ethtool even BEFORE open */
2173 netif_carrier_off(netdev);
2174
421e02f0 2175#ifdef CONFIG_IGB_DCA
bbd98fe4 2176 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 2177 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 2178 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 2179 igb_setup_dca(adapter);
2180 }
fe4506b6 2181
38c845c7 2182#endif
7ebae817 2183#ifdef CONFIG_IGB_PTP
673b8b70 2184 /* do hw tstamp init after resetting */
7ebae817 2185 igb_ptp_init(adapter);
673b8b70 2186
7ebae817 2187#endif
9d5c8243
AK		 2188 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
2189 /* print bus type/speed/width info */
7c510e4b 2190 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 2191 netdev->name,
559e9c49 2192 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
ff846f52 2193 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
559e9c49 2194 "unknown"),
59c3de89
AD		 2195 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
2196 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
2197 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2198 "unknown"),
7c510e4b 2199 netdev->dev_addr);
9d5c8243 2200
9835fd73
CW		 2201 ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH);
2202 if (ret_val)
2203 strcpy(part_str, "Unknown");
2204 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
9d5c8243
AK		 2205 dev_info(&pdev->dev,
2206 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2207 adapter->msix_entries ? "MSI-X" :
7dfc16fa 2208 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243 2209 adapter->num_rx_queues, adapter->num_tx_queues);
09b068d4
CW		 2210 switch (hw->mac.type) {
2211 case e1000_i350:
f96a8a0b
CW		 2212 case e1000_i210:
2213 case e1000_i211:
09b068d4
CW		 2214 igb_set_eee_i350(hw);
2215 break;
2216 default:
2217 break;
2218 }
749ab2cd
YZ		 2219
2220 pm_runtime_put_noidle(&pdev->dev);
9d5c8243
AK		 2221 return 0;
2222
2223err_register:
2224 igb_release_hw_control(adapter);
2225err_eeprom:
2226 if (!igb_check_reset_block(hw))
f5f4cf08 2227 igb_reset_phy(hw);
9d5c8243
AK		 2228
2229 if (hw->flash_address)
2230 iounmap(hw->flash_address);
9d5c8243 2231err_sw_init:
047e0030 2232 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 2233 iounmap(hw->hw_addr);
2234err_ioremap:
2235 free_netdev(netdev);
2236err_alloc_etherdev:
559e9c49
AD		 2237 pci_release_selected_regions(pdev,
2238 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2239err_pci_reg:
2240err_dma:
2241 pci_disable_device(pdev);
2242 return err;
2243}
2244
2245/**
2246 * igb_remove - Device Removal Routine
2247 * @pdev: PCI device information struct
2248 *
2249 * igb_remove is called by the PCI subsystem to alert the driver
2250 * that it should release a PCI device. The could be caused by a
2251 * Hot-Plug event, or because the driver is going to be removed from
2252 * memory.
2253 **/
2254static void __devexit igb_remove(struct pci_dev *pdev)
2255{
2256 struct net_device *netdev = pci_get_drvdata(pdev);
2257 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 2258 struct e1000_hw *hw = &adapter->hw;
9d5c8243 2259
749ab2cd 2260 pm_runtime_get_noresume(&pdev->dev);
7ebae817
RC		 2261#ifdef CONFIG_IGB_PTP
2262 igb_ptp_remove(adapter);
749ab2cd 2263
7ebae817 2264#endif
760141a5
TH		 2265 /*
2266 * The watchdog timer may be rescheduled, so explicitly
2267 * disable watchdog from being rescheduled.
2268 */
9d5c8243
AK		 2269 set_bit(__IGB_DOWN, &adapter->state);
2270 del_timer_sync(&adapter->watchdog_timer);
2271 del_timer_sync(&adapter->phy_info_timer);
2272
760141a5
TH		 2273 cancel_work_sync(&adapter->reset_task);
2274 cancel_work_sync(&adapter->watchdog_task);
9d5c8243 2275
421e02f0 2276#ifdef CONFIG_IGB_DCA
7dfc16fa 2277 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 2278 dev_info(&pdev->dev, "DCA disabled\n");
2279 dca_remove_requester(&pdev->dev);
7dfc16fa 2280 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 2281 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 2282 }
2283#endif
2284
9d5c8243
AK		 2285 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2286 * would have already happened in close and is redundant. */
2287 igb_release_hw_control(adapter);
2288
2289 unregister_netdev(netdev);
2290
047e0030 2291 igb_clear_interrupt_scheme(adapter);
9d5c8243 2292
37680117
AD		 2293#ifdef CONFIG_PCI_IOV
2294 /* reclaim resources allocated to VFs */
2295 if (adapter->vf_data) {
2296 /* disable iov and allow time for transactions to clear */
0224d663
GR		 2297 if (!igb_check_vf_assignment(adapter)) {
2298 pci_disable_sriov(pdev);
2299 msleep(500);
2300 } else {
2301 dev_info(&pdev->dev, "VF(s) assigned to guests!\n");
2302 }
37680117
AD		 2303
2304 kfree(adapter->vf_data);
2305 adapter->vf_data = NULL;
2306 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 2307 wrfl();
37680117
AD		 2308 msleep(100);
2309 dev_info(&pdev->dev, "IOV Disabled\n");
2310 }
2311#endif
559e9c49 2312
28b0759c
AD		 2313 iounmap(hw->hw_addr);
2314 if (hw->flash_address)
2315 iounmap(hw->flash_address);
559e9c49
AD		 2316 pci_release_selected_regions(pdev,
2317 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243 2318
1128c756 2319 kfree(adapter->shadow_vfta);
9d5c8243
AK		 2320 free_netdev(netdev);
2321
19d5afd4 2322 pci_disable_pcie_error_reporting(pdev);
40a914fa 2323
9d5c8243
AK		 2324 pci_disable_device(pdev);
2325}
2326
a6b623e0
AD		 2327/**
2328 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2329 * @adapter: board private structure to initialize
2330 *
2331 * This function initializes the vf specific data storage and then attempts to
2332 * allocate the VFs. The reason for ordering it this way is because it is much
2333 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2334 * the memory for the VFs.
2335 **/
2336static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2337{
2338#ifdef CONFIG_PCI_IOV
2339 struct pci_dev *pdev = adapter->pdev;
f96a8a0b 2340 struct e1000_hw *hw = &adapter->hw;
0224d663
GR		 2341 int old_vfs = igb_find_enabled_vfs(adapter);
2342 int i;
a6b623e0 2343
f96a8a0b
CW		 2344 /* Virtualization features not supported on i210 family. */
2345 if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211))
2346 return;
2347
0224d663
GR		 2348 if (old_vfs) {
2349 dev_info(&pdev->dev, "%d pre-allocated VFs found - override "
2350 "max_vfs setting of %d\n", old_vfs, max_vfs);
2351 adapter->vfs_allocated_count = old_vfs;
a6b623e0
AD		 2352 }
2353
0224d663
GR		 2354 if (!adapter->vfs_allocated_count)
2355 return;
2356
2357 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2358 sizeof(struct vf_data_storage), GFP_KERNEL);
f96a8a0b 2359
0224d663
GR		 2360 /* if allocation failed then we do not support SR-IOV */
2361 if (!adapter->vf_data) {
a6b623e0 2362 adapter->vfs_allocated_count = 0;
0224d663
GR		 2363 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2364 "Data Storage\n");
2365 goto out;
a6b623e0 2366 }
0224d663
GR		 2367
2368 if (!old_vfs) {
2369 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count))
2370 goto err_out;
2371 }
2372 dev_info(&pdev->dev, "%d VFs allocated\n",
2373 adapter->vfs_allocated_count);
2374 for (i = 0; i < adapter->vfs_allocated_count; i++)
2375 igb_vf_configure(adapter, i);
2376
2377 /* DMA Coalescing is not supported in IOV mode. */
2378 adapter->flags &= ~IGB_FLAG_DMAC;
2379 goto out;
2380err_out:
2381 kfree(adapter->vf_data);
2382 adapter->vf_data = NULL;
2383 adapter->vfs_allocated_count = 0;
2384out:
2385 return;
a6b623e0
AD		 2386#endif /* CONFIG_PCI_IOV */
2387}
2388
9d5c8243
AK		 2389/**
2390 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2391 * @adapter: board private structure to initialize
2392 *
2393 * igb_sw_init initializes the Adapter private data structure.
2394 * Fields are initialized based on PCI device information and
2395 * OS network device settings (MTU size).
2396 **/
2397static int __devinit igb_sw_init(struct igb_adapter *adapter)
2398{
2399 struct e1000_hw *hw = &adapter->hw;
2400 struct net_device *netdev = adapter->netdev;
2401 struct pci_dev *pdev = adapter->pdev;
374a542d 2402 u32 max_rss_queues;
9d5c8243
AK		 2403
2404 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2405
13fde97a 2406 /* set default ring sizes */
68fd9910
AD		 2407 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2408 adapter->rx_ring_count = IGB_DEFAULT_RXD;
13fde97a
AD		 2409
2410 /* set default ITR values */
4fc82adf
AD		 2411 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2412 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2413
13fde97a
AD		 2414 /* set default work limits */
2415 adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
2416
153285f9
AD		 2417 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
2418 VLAN_HLEN;
9d5c8243
AK		 2419 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2420
81c2fc22
AD		 2421 adapter->node = -1;
2422
12dcd86b 2423 spin_lock_init(&adapter->stats64_lock);
a6b623e0 2424#ifdef CONFIG_PCI_IOV
6b78bb1d
CW		 2425 switch (hw->mac.type) {
2426 case e1000_82576:
2427 case e1000_i350:
9b082d73
SA		 2428 if (max_vfs > 7) {
2429 dev_warn(&pdev->dev,
2430 "Maximum of 7 VFs per PF, using max\n");
2431 adapter->vfs_allocated_count = 7;
2432 } else
2433 adapter->vfs_allocated_count = max_vfs;
6b78bb1d
CW		 2434 break;
2435 default:
2436 break;
2437 }
a6b623e0 2438#endif /* CONFIG_PCI_IOV */
374a542d
MV		 2439
2440 /* Determine the maximum number of RSS queues supported. */
f96a8a0b 2441 switch (hw->mac.type) {
374a542d
MV		 2442 case e1000_i211:
2443 max_rss_queues = IGB_MAX_RX_QUEUES_I211;
2444 break;
2445 case e1000_82575:
f96a8a0b 2446 case e1000_i210:
374a542d
MV		 2447 max_rss_queues = IGB_MAX_RX_QUEUES_82575;
2448 break;
2449 case e1000_i350:
2450 /* I350 cannot do RSS and SR-IOV at the same time */
2451 if (!!adapter->vfs_allocated_count) {
2452 max_rss_queues = 1;
2453 break;
2454 }
2455 /* fall through */
2456 case e1000_82576:
2457 if (!!adapter->vfs_allocated_count) {
2458 max_rss_queues = 2;
2459 break;
2460 }
2461 /* fall through */
2462 case e1000_82580:
2463 default:
2464 max_rss_queues = IGB_MAX_RX_QUEUES;
f96a8a0b 2465 break;
374a542d
MV		 2466 }
2467
2468 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
2469
2470 /* Determine if we need to pair queues. */
2471 switch (hw->mac.type) {
2472 case e1000_82575:
f96a8a0b 2473 case e1000_i211:
374a542d 2474 /* Device supports enough interrupts without queue pairing. */
f96a8a0b 2475 break;
374a542d
MV		 2476 case e1000_82576:
2477 /*
2478 * If VFs are going to be allocated with RSS queues then we
2479 * should pair the queues in order to conserve interrupts due
2480 * to limited supply.
2481 */
2482 if ((adapter->rss_queues > 1) &&
2483 (adapter->vfs_allocated_count > 6))
2484 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2485 /* fall through */
2486 case e1000_82580:
2487 case e1000_i350:
2488 case e1000_i210:
f96a8a0b 2489 default:
374a542d
MV		 2490 /*
2491 * If rss_queues > half of max_rss_queues, pair the queues in
2492 * order to conserve interrupts due to limited supply.
2493 */
2494 if (adapter->rss_queues > (max_rss_queues / 2))
2495 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
f96a8a0b
CW		 2496 break;
2497 }
a99955fc 2498
1128c756
CW		 2499 /* Setup and initialize a copy of the hw vlan table array */
2500 adapter->shadow_vfta = kzalloc(sizeof(u32) *
2501 E1000_VLAN_FILTER_TBL_SIZE,
2502 GFP_ATOMIC);
2503
a6b623e0 2504 /* This call may decrease the number of queues */
047e0030 2505 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 2506 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2507 return -ENOMEM;
2508 }
2509
a6b623e0
AD		 2510 igb_probe_vfs(adapter);
2511
9d5c8243
AK		 2512 /* Explicitly disable IRQ since the NIC can be in any state. */
2513 igb_irq_disable(adapter);
2514
f96a8a0b 2515 if (hw->mac.type >= e1000_i350)
831ec0b4
CW		 2516 adapter->flags &= ~IGB_FLAG_DMAC;
2517
9d5c8243
AK		 2518 set_bit(__IGB_DOWN, &adapter->state);
2519 return 0;
2520}
2521
2522/**
2523 * igb_open - Called when a network interface is made active
2524 * @netdev: network interface device structure
2525 *
2526 * Returns 0 on success, negative value on failure
2527 *
2528 * The open entry point is called when a network interface is made
2529 * active by the system (IFF_UP). At this point all resources needed
2530 * for transmit and receive operations are allocated, the interrupt
2531 * handler is registered with the OS, the watchdog timer is started,
2532 * and the stack is notified that the interface is ready.
2533 **/
749ab2cd 2534static int __igb_open(struct net_device *netdev, bool resuming)
9d5c8243
AK		 2535{
2536 struct igb_adapter *adapter = netdev_priv(netdev);
2537 struct e1000_hw *hw = &adapter->hw;
749ab2cd 2538 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2539 int err;
2540 int i;
2541
2542 /* disallow open during test */
749ab2cd
YZ		 2543 if (test_bit(__IGB_TESTING, &adapter->state)) {
2544 WARN_ON(resuming);
9d5c8243 2545 return -EBUSY;
749ab2cd
YZ		 2546 }
2547
2548 if (!resuming)
2549 pm_runtime_get_sync(&pdev->dev);
9d5c8243 2550
b168dfc5
JB		 2551 netif_carrier_off(netdev);
2552
9d5c8243
AK		 2553 /* allocate transmit descriptors */
2554 err = igb_setup_all_tx_resources(adapter);
2555 if (err)
2556 goto err_setup_tx;
2557
2558 /* allocate receive descriptors */
2559 err = igb_setup_all_rx_resources(adapter);
2560 if (err)
2561 goto err_setup_rx;
2562
88a268c1 2563 igb_power_up_link(adapter);
9d5c8243 2564
9d5c8243
AK		 2565 /* before we allocate an interrupt, we must be ready to handle it.
2566 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2567 * as soon as we call pci_request_irq, so we have to setup our
2568 * clean_rx handler before we do so. */
2569 igb_configure(adapter);
2570
2571 err = igb_request_irq(adapter);
2572 if (err)
2573 goto err_req_irq;
2574
2575 /* From here on the code is the same as igb_up() */
2576 clear_bit(__IGB_DOWN, &adapter->state);
2577
0d1ae7f4
AD		 2578 for (i = 0; i < adapter->num_q_vectors; i++)
2579 napi_enable(&(adapter->q_vector[i]->napi));
9d5c8243
AK		 2580
2581 /* Clear any pending interrupts. */
2582 rd32(E1000_ICR);
844290e5
PW		 2583
2584 igb_irq_enable(adapter);
2585
d4960307
AD		 2586 /* notify VFs that reset has been completed */
2587 if (adapter->vfs_allocated_count) {
2588 u32 reg_data = rd32(E1000_CTRL_EXT);
2589 reg_data |= E1000_CTRL_EXT_PFRSTD;
2590 wr32(E1000_CTRL_EXT, reg_data);
2591 }
2592
d55b53ff
JK		 2593 netif_tx_start_all_queues(netdev);
2594
749ab2cd
YZ		 2595 if (!resuming)
2596 pm_runtime_put(&pdev->dev);
2597
25568a53
AD		 2598 /* start the watchdog. */
2599 hw->mac.get_link_status = 1;
2600 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2601
2602 return 0;
2603
2604err_req_irq:
2605 igb_release_hw_control(adapter);
88a268c1 2606 igb_power_down_link(adapter);
9d5c8243
AK		 2607 igb_free_all_rx_resources(adapter);
2608err_setup_rx:
2609 igb_free_all_tx_resources(adapter);
2610err_setup_tx:
2611 igb_reset(adapter);
749ab2cd
YZ		 2612 if (!resuming)
2613 pm_runtime_put(&pdev->dev);
9d5c8243
AK		 2614
2615 return err;
2616}
2617
749ab2cd
YZ		 2618static int igb_open(struct net_device *netdev)
2619{
2620 return __igb_open(netdev, false);
2621}
2622
9d5c8243
AK		 2623/**
2624 * igb_close - Disables a network interface
2625 * @netdev: network interface device structure
2626 *
2627 * Returns 0, this is not allowed to fail
2628 *
2629 * The close entry point is called when an interface is de-activated
2630 * by the OS. The hardware is still under the driver's control, but
2631 * needs to be disabled. A global MAC reset is issued to stop the
2632 * hardware, and all transmit and receive resources are freed.
2633 **/
749ab2cd 2634static int __igb_close(struct net_device *netdev, bool suspending)
9d5c8243
AK		 2635{
2636 struct igb_adapter *adapter = netdev_priv(netdev);
749ab2cd 2637 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2638
2639 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
9d5c8243 2640
749ab2cd
YZ		 2641 if (!suspending)
2642 pm_runtime_get_sync(&pdev->dev);
2643
2644 igb_down(adapter);
9d5c8243
AK		 2645 igb_free_irq(adapter);
2646
2647 igb_free_all_tx_resources(adapter);
2648 igb_free_all_rx_resources(adapter);
2649
749ab2cd
YZ		 2650 if (!suspending)
2651 pm_runtime_put_sync(&pdev->dev);
9d5c8243
AK		 2652 return 0;
2653}
2654
749ab2cd
YZ		 2655static int igb_close(struct net_device *netdev)
2656{
2657 return __igb_close(netdev, false);
2658}
2659
9d5c8243
AK		 2660/**
2661 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2662 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2663 *
2664 * Return 0 on success, negative on failure
2665 **/
80785298 2666int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2667{
59d71989 2668 struct device *dev = tx_ring->dev;
81c2fc22 2669 int orig_node = dev_to_node(dev);
9d5c8243
AK		 2670 int size;
2671
06034649 2672 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
81c2fc22
AD		 2673 tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node);
2674 if (!tx_ring->tx_buffer_info)
2675 tx_ring->tx_buffer_info = vzalloc(size);
06034649 2676 if (!tx_ring->tx_buffer_info)
9d5c8243 2677 goto err;
9d5c8243
AK		 2678
2679 /* round up to nearest 4K */
85e8d004 2680 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2681 tx_ring->size = ALIGN(tx_ring->size, 4096);
2682
81c2fc22 2683 set_dev_node(dev, tx_ring->numa_node);
59d71989
AD		 2684 tx_ring->desc = dma_alloc_coherent(dev,
2685 tx_ring->size,
2686 &tx_ring->dma,
2687 GFP_KERNEL);
81c2fc22
AD		 2688 set_dev_node(dev, orig_node);
2689 if (!tx_ring->desc)
2690 tx_ring->desc = dma_alloc_coherent(dev,
2691 tx_ring->size,
2692 &tx_ring->dma,
2693 GFP_KERNEL);
9d5c8243
AK		 2694
2695 if (!tx_ring->desc)
2696 goto err;
2697
9d5c8243
AK		 2698 tx_ring->next_to_use = 0;
2699 tx_ring->next_to_clean = 0;
81c2fc22 2700
9d5c8243
AK		 2701 return 0;
2702
2703err:
06034649 2704 vfree(tx_ring->tx_buffer_info);
59d71989 2705 dev_err(dev,
9d5c8243
AK		 2706 "Unable to allocate memory for the transmit descriptor ring\n");
2707 return -ENOMEM;
2708}
2709
2710/**
2711 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2712 * (Descriptors) for all queues
2713 * @adapter: board private structure
2714 *
2715 * Return 0 on success, negative on failure
2716 **/
2717static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2718{
439705e1 2719 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2720 int i, err = 0;
2721
2722 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2723 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2724 if (err) {
439705e1 2725 dev_err(&pdev->dev,
9d5c8243
AK		 2726 "Allocation for Tx Queue %u failed\n", i);
2727 for (i--; i >= 0; i--)
3025a446 2728 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2729 break;
2730 }
2731 }
2732
2733 return err;
2734}
2735
2736/**
85b430b4
AD		 2737 * igb_setup_tctl - configure the transmit control registers
2738 * @adapter: Board private structure
9d5c8243 2739 **/
d7ee5b3a 2740void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2741{
9d5c8243
AK		 2742 struct e1000_hw *hw = &adapter->hw;
2743 u32 tctl;
9d5c8243 2744
85b430b4
AD		 2745 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2746 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2747
2748 /* Program the Transmit Control Register */
9d5c8243
AK		 2749 tctl = rd32(E1000_TCTL);
2750 tctl &= ~E1000_TCTL_CT;
2751 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2752 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2753
2754 igb_config_collision_dist(hw);
2755
9d5c8243
AK		 2756 /* Enable transmits */
2757 tctl |= E1000_TCTL_EN;
2758
2759 wr32(E1000_TCTL, tctl);
2760}
2761
85b430b4
AD		 2762/**
2763 * igb_configure_tx_ring - Configure transmit ring after Reset
2764 * @adapter: board private structure
2765 * @ring: tx ring to configure
2766 *
2767 * Configure a transmit ring after a reset.
2768 **/
d7ee5b3a
AD		 2769void igb_configure_tx_ring(struct igb_adapter *adapter,
2770 struct igb_ring *ring)
85b430b4
AD		 2771{
2772 struct e1000_hw *hw = &adapter->hw;
a74420e0 2773 u32 txdctl = 0;
85b430b4
AD		 2774 u64 tdba = ring->dma;
2775 int reg_idx = ring->reg_idx;
2776
2777 /* disable the queue */
a74420e0 2778 wr32(E1000_TXDCTL(reg_idx), 0);
85b430b4
AD		 2779 wrfl();
2780 mdelay(10);
2781
2782 wr32(E1000_TDLEN(reg_idx),
2783 ring->count * sizeof(union e1000_adv_tx_desc));
2784 wr32(E1000_TDBAL(reg_idx),
2785 tdba & 0x00000000ffffffffULL);
2786 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2787
fce99e34 2788 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
a74420e0 2789 wr32(E1000_TDH(reg_idx), 0);
fce99e34 2790 writel(0, ring->tail);
85b430b4
AD		 2791
2792 txdctl |= IGB_TX_PTHRESH;
2793 txdctl |= IGB_TX_HTHRESH << 8;
2794 txdctl |= IGB_TX_WTHRESH << 16;
2795
2796 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2797 wr32(E1000_TXDCTL(reg_idx), txdctl);
2798}
2799
2800/**
2801 * igb_configure_tx - Configure transmit Unit after Reset
2802 * @adapter: board private structure
2803 *
2804 * Configure the Tx unit of the MAC after a reset.
2805 **/
2806static void igb_configure_tx(struct igb_adapter *adapter)
2807{
2808 int i;
2809
2810 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2811 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2812}
2813
9d5c8243
AK		 2814/**
2815 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2816 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2817 *
2818 * Returns 0 on success, negative on failure
2819 **/
80785298 2820int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2821{
59d71989 2822 struct device *dev = rx_ring->dev;
81c2fc22 2823 int orig_node = dev_to_node(dev);
9d5c8243
AK		 2824 int size, desc_len;
2825
06034649 2826 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
81c2fc22
AD		 2827 rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node);
2828 if (!rx_ring->rx_buffer_info)
2829 rx_ring->rx_buffer_info = vzalloc(size);
06034649 2830 if (!rx_ring->rx_buffer_info)
9d5c8243 2831 goto err;
9d5c8243
AK		 2832
2833 desc_len = sizeof(union e1000_adv_rx_desc);
2834
2835 /* Round up to nearest 4K */
2836 rx_ring->size = rx_ring->count * desc_len;
2837 rx_ring->size = ALIGN(rx_ring->size, 4096);
2838
81c2fc22 2839 set_dev_node(dev, rx_ring->numa_node);
59d71989
AD		 2840 rx_ring->desc = dma_alloc_coherent(dev,
2841 rx_ring->size,
2842 &rx_ring->dma,
2843 GFP_KERNEL);
81c2fc22
AD		 2844 set_dev_node(dev, orig_node);
2845 if (!rx_ring->desc)
2846 rx_ring->desc = dma_alloc_coherent(dev,
2847 rx_ring->size,
2848 &rx_ring->dma,
2849 GFP_KERNEL);
9d5c8243
AK		 2850
2851 if (!rx_ring->desc)
2852 goto err;
2853
2854 rx_ring->next_to_clean = 0;
2855 rx_ring->next_to_use = 0;
9d5c8243 2856
9d5c8243
AK		 2857 return 0;
2858
2859err:
06034649
AD		 2860 vfree(rx_ring->rx_buffer_info);
2861 rx_ring->rx_buffer_info = NULL;
59d71989
AD		 2862 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2863 " ring\n");
9d5c8243
AK		 2864 return -ENOMEM;
2865}
2866
2867/**
2868 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2869 * (Descriptors) for all queues
2870 * @adapter: board private structure
2871 *
2872 * Return 0 on success, negative on failure
2873 **/
2874static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2875{
439705e1 2876 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2877 int i, err = 0;
2878
2879 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2880 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2881 if (err) {
439705e1 2882 dev_err(&pdev->dev,
9d5c8243
AK		 2883 "Allocation for Rx Queue %u failed\n", i);
2884 for (i--; i >= 0; i--)
3025a446 2885 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2886 break;
2887 }
2888 }
2889
2890 return err;
2891}
2892
06cf2666
AD		 2893/**
2894 * igb_setup_mrqc - configure the multiple receive queue control registers
2895 * @adapter: Board private structure
2896 **/
2897static void igb_setup_mrqc(struct igb_adapter *adapter)
2898{
2899 struct e1000_hw *hw = &adapter->hw;
2900 u32 mrqc, rxcsum;
2901 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2902 union e1000_reta {
2903 u32 dword;
2904 u8 bytes[4];
2905 } reta;
2906 static const u8 rsshash[40] = {
2907 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2908 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2909 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2910 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2911
2912 /* Fill out hash function seeds */
2913 for (j = 0; j < 10; j++) {
2914 u32 rsskey = rsshash[(j * 4)];
2915 rsskey |= rsshash[(j * 4) + 1] << 8;
2916 rsskey |= rsshash[(j * 4) + 2] << 16;
2917 rsskey |= rsshash[(j * 4) + 3] << 24;
2918 array_wr32(E1000_RSSRK(0), j, rsskey);
2919 }
2920
a99955fc 2921 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2922
2923 if (adapter->vfs_allocated_count) {
2924 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2925 switch (hw->mac.type) {
d2ba2ed8 2926 case e1000_i350:
55cac248
AD		 2927 case e1000_82580:
2928 num_rx_queues = 1;
2929 shift = 0;
2930 break;
06cf2666
AD		 2931 case e1000_82576:
2932 shift = 3;
2933 num_rx_queues = 2;
2934 break;
2935 case e1000_82575:
2936 shift = 2;
2937 shift2 = 6;
2938 default:
2939 break;
2940 }
2941 } else {
2942 if (hw->mac.type == e1000_82575)
2943 shift = 6;
2944 }
2945
2946 for (j = 0; j < (32 * 4); j++) {
2947 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2948 if (shift2)
2949 reta.bytes[j & 3] |= num_rx_queues << shift2;
2950 if ((j & 3) == 3)
2951 wr32(E1000_RETA(j >> 2), reta.dword);
2952 }
2953
2954 /*
2955 * Disable raw packet checksumming so that RSS hash is placed in
2956 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2957 * offloads as they are enabled by default
2958 */
2959 rxcsum = rd32(E1000_RXCSUM);
2960 rxcsum |= E1000_RXCSUM_PCSD;
2961
2962 if (adapter->hw.mac.type >= e1000_82576)
2963 /* Enable Receive Checksum Offload for SCTP */
2964 rxcsum |= E1000_RXCSUM_CRCOFL;
2965
2966 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2967 wr32(E1000_RXCSUM, rxcsum);
f96a8a0b
CW		 2968 /*
2969 * Generate RSS hash based on TCP port numbers and/or
2970 * IPv4/v6 src and dst addresses since UDP cannot be
2971 * hashed reliably due to IP fragmentation
2972 */
2973
2974 mrqc = E1000_MRQC_RSS_FIELD_IPV4 |
2975 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2976 E1000_MRQC_RSS_FIELD_IPV6 |
2977 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2978 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
06cf2666
AD		 2979
2980 /* If VMDq is enabled then we set the appropriate mode for that, else
2981 * we default to RSS so that an RSS hash is calculated per packet even
2982 * if we are only using one queue */
2983 if (adapter->vfs_allocated_count) {
2984 if (hw->mac.type > e1000_82575) {
2985 /* Set the default pool for the PF's first queue */
2986 u32 vtctl = rd32(E1000_VT_CTL);
2987 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2988 E1000_VT_CTL_DISABLE_DEF_POOL);
2989 vtctl |= adapter->vfs_allocated_count <<
2990 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2991 wr32(E1000_VT_CTL, vtctl);
2992 }
a99955fc 2993 if (adapter->rss_queues > 1)
f96a8a0b 2994 mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
06cf2666 2995 else
f96a8a0b 2996 mrqc |= E1000_MRQC_ENABLE_VMDQ;
06cf2666 2997 } else {
f96a8a0b
CW		 2998 if (hw->mac.type != e1000_i211)
2999 mrqc |= E1000_MRQC_ENABLE_RSS_4Q;
06cf2666
AD		 3000 }
3001 igb_vmm_control(adapter);
3002
06cf2666
AD		 3003 wr32(E1000_MRQC, mrqc);
3004}
3005
9d5c8243
AK		 3006/**
3007 * igb_setup_rctl - configure the receive control registers
3008 * @adapter: Board private structure
3009 **/
d7ee5b3a 3010void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 3011{
3012 struct e1000_hw *hw = &adapter->hw;
3013 u32 rctl;
9d5c8243
AK		 3014
3015 rctl = rd32(E1000_RCTL);
3016
3017 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 3018 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 3019
69d728ba 3020 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 3021 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 3022
87cb7e8c
AK		 3023 /*
3024 * enable stripping of CRC. It's unlikely this will break BMC
3025 * redirection as it did with e1000. Newer features require
3026 * that the HW strips the CRC.
73cd78f1 3027 */
87cb7e8c 3028 rctl |= E1000_RCTL_SECRC;
9d5c8243 3029
559e9c49 3030 /* disable store bad packets and clear size bits. */
ec54d7d6 3031 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 3032
6ec43fe6
AD		 3033 /* enable LPE to prevent packets larger than max_frame_size */
3034 rctl |= E1000_RCTL_LPE;
9d5c8243 3035
952f72a8
AD		 3036 /* disable queue 0 to prevent tail write w/o re-config */
3037 wr32(E1000_RXDCTL(0), 0);
9d5c8243 3038
e1739522
AD		 3039 /* Attention!!! For SR-IOV PF driver operations you must enable
3040 * queue drop for all VF and PF queues to prevent head of line blocking
3041 * if an un-trusted VF does not provide descriptors to hardware.
3042 */
3043 if (adapter->vfs_allocated_count) {
e1739522
AD		 3044 /* set all queue drop enable bits */
3045 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 3046 }
3047
89eaefb6
BG		 3048 /* This is useful for sniffing bad packets. */
3049 if (adapter->netdev->features & NETIF_F_RXALL) {
3050 /* UPE and MPE will be handled by normal PROMISC logic
3051 * in e1000e_set_rx_mode */
3052 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
3053 E1000_RCTL_BAM | /* RX All Bcast Pkts */
3054 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
3055
3056 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
3057 E1000_RCTL_DPF | /* Allow filtered pause */
3058 E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
3059 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
3060 * and that breaks VLANs.
3061 */
3062 }
3063
9d5c8243
AK		 3064 wr32(E1000_RCTL, rctl);
3065}
3066
7d5753f0
AD		 3067static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
3068 int vfn)
3069{
3070 struct e1000_hw *hw = &adapter->hw;
3071 u32 vmolr;
3072
3073 /* if it isn't the PF check to see if VFs are enabled and
3074 * increase the size to support vlan tags */
3075 if (vfn < adapter->vfs_allocated_count &&
3076 adapter->vf_data[vfn].vlans_enabled)
3077 size += VLAN_TAG_SIZE;
3078
3079 vmolr = rd32(E1000_VMOLR(vfn));
3080 vmolr &= ~E1000_VMOLR_RLPML_MASK;
3081 vmolr |= size | E1000_VMOLR_LPE;
3082 wr32(E1000_VMOLR(vfn), vmolr);
3083
3084 return 0;
3085}
3086
e1739522
AD		 3087/**
3088 * igb_rlpml_set - set maximum receive packet size
3089 * @adapter: board private structure
3090 *
3091 * Configure maximum receivable packet size.
3092 **/
3093static void igb_rlpml_set(struct igb_adapter *adapter)
3094{
153285f9 3095 u32 max_frame_size = adapter->max_frame_size;
e1739522
AD		 3096 struct e1000_hw *hw = &adapter->hw;
3097 u16 pf_id = adapter->vfs_allocated_count;
3098
e1739522
AD		 3099 if (pf_id) {
3100 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
153285f9
AD		 3101 /*
3102 * If we're in VMDQ or SR-IOV mode, then set global RLPML
3103 * to our max jumbo frame size, in case we need to enable
3104 * jumbo frames on one of the rings later.
3105 * This will not pass over-length frames into the default
3106 * queue because it's gated by the VMOLR.RLPML.
3107 */
7d5753f0 3108 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 3109 }
3110
3111 wr32(E1000_RLPML, max_frame_size);
3112}
3113
8151d294
WM		 3114static inline void igb_set_vmolr(struct igb_adapter *adapter,
3115 int vfn, bool aupe)
7d5753f0
AD		 3116{
3117 struct e1000_hw *hw = &adapter->hw;
3118 u32 vmolr;
3119
3120 /*
3121 * This register exists only on 82576 and newer so if we are older then
3122 * we should exit and do nothing
3123 */
3124 if (hw->mac.type < e1000_82576)
3125 return;
3126
3127 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 3128 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
3129 if (aupe)
3130 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
3131 else
3132 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 3133
3134 /* clear all bits that might not be set */
3135 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
3136
a99955fc 3137 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 3138 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3139 /*
3140 * for VMDq only allow the VFs and pool 0 to accept broadcast and
3141 * multicast packets
3142 */
3143 if (vfn <= adapter->vfs_allocated_count)
3144 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
3145
3146 wr32(E1000_VMOLR(vfn), vmolr);
3147}
3148
85b430b4
AD		 3149/**
3150 * igb_configure_rx_ring - Configure a receive ring after Reset
3151 * @adapter: board private structure
3152 * @ring: receive ring to be configured
3153 *
3154 * Configure the Rx unit of the MAC after a reset.
3155 **/
d7ee5b3a
AD		 3156void igb_configure_rx_ring(struct igb_adapter *adapter,
3157 struct igb_ring *ring)
85b430b4
AD		 3158{
3159 struct e1000_hw *hw = &adapter->hw;
3160 u64 rdba = ring->dma;
3161 int reg_idx = ring->reg_idx;
a74420e0 3162 u32 srrctl = 0, rxdctl = 0;
85b430b4
AD		 3163
3164 /* disable the queue */
a74420e0 3165 wr32(E1000_RXDCTL(reg_idx), 0);
85b430b4
AD		 3166
3167 /* Set DMA base address registers */
3168 wr32(E1000_RDBAL(reg_idx),
3169 rdba & 0x00000000ffffffffULL);
3170 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
3171 wr32(E1000_RDLEN(reg_idx),
3172 ring->count * sizeof(union e1000_adv_rx_desc));
3173
3174 /* initialize head and tail */
fce99e34 3175 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
a74420e0 3176 wr32(E1000_RDH(reg_idx), 0);
fce99e34 3177 writel(0, ring->tail);
85b430b4 3178
952f72a8 3179 /* set descriptor configuration */
44390ca6 3180 srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
952f72a8 3181#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
44390ca6 3182 srrctl |= IGB_RXBUFFER_16384 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
952f72a8 3183#else
44390ca6 3184 srrctl |= (PAGE_SIZE / 2) >> E1000_SRRCTL_BSIZEPKT_SHIFT;
952f72a8 3185#endif
44390ca6 3186 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
06218a8d 3187 if (hw->mac.type >= e1000_82580)
757b77e2 3188 srrctl |= E1000_SRRCTL_TIMESTAMP;
e6bdb6fe
NN		 3189 /* Only set Drop Enable if we are supporting multiple queues */
3190 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
3191 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 3192
3193 wr32(E1000_SRRCTL(reg_idx), srrctl);
3194
7d5753f0 3195 /* set filtering for VMDQ pools */
8151d294 3196 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 3197
85b430b4
AD		 3198 rxdctl |= IGB_RX_PTHRESH;
3199 rxdctl |= IGB_RX_HTHRESH << 8;
3200 rxdctl |= IGB_RX_WTHRESH << 16;
a74420e0
AD		 3201
3202 /* enable receive descriptor fetching */
3203 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
85b430b4
AD		 3204 wr32(E1000_RXDCTL(reg_idx), rxdctl);
3205}
3206
9d5c8243
AK		 3207/**
3208 * igb_configure_rx - Configure receive Unit after Reset
3209 * @adapter: board private structure
3210 *
3211 * Configure the Rx unit of the MAC after a reset.
3212 **/
3213static void igb_configure_rx(struct igb_adapter *adapter)
3214{
9107584e 3215 int i;
9d5c8243 3216
68d480c4
AD		 3217 /* set UTA to appropriate mode */
3218 igb_set_uta(adapter);
3219
26ad9178
AD		 3220 /* set the correct pool for the PF default MAC address in entry 0 */
3221 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
3222 adapter->vfs_allocated_count);
3223
06cf2666
AD		 3224 /* Setup the HW Rx Head and Tail Descriptor Pointers and
3225 * the Base and Length of the Rx Descriptor Ring */
3226 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3227 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 3228}
3229
3230/**
3231 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 3232 * @tx_ring: Tx descriptor ring for a specific queue
3233 *
3234 * Free all transmit software resources
3235 **/
68fd9910 3236void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 3237{
3b644cf6 3238 igb_clean_tx_ring(tx_ring);
9d5c8243 3239
06034649
AD		 3240 vfree(tx_ring->tx_buffer_info);
3241 tx_ring->tx_buffer_info = NULL;
9d5c8243 3242
439705e1
AD		 3243 /* if not set, then don't free */
3244 if (!tx_ring->desc)
3245 return;
3246
59d71989
AD		 3247 dma_free_coherent(tx_ring->dev, tx_ring->size,
3248 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 3249
3250 tx_ring->desc = NULL;
3251}
3252
3253/**
3254 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3255 * @adapter: board private structure
3256 *
3257 * Free all transmit software resources
3258 **/
3259static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3260{
3261 int i;
3262
3263 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3264 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 3265}
3266
ebe42d16
AD		 3267void igb_unmap_and_free_tx_resource(struct igb_ring *ring,
3268 struct igb_tx_buffer *tx_buffer)
3269{
3270 if (tx_buffer->skb) {
3271 dev_kfree_skb_any(tx_buffer->skb);
3272 if (tx_buffer->dma)
3273 dma_unmap_single(ring->dev,
3274 tx_buffer->dma,
3275 tx_buffer->length,
3276 DMA_TO_DEVICE);
3277 } else if (tx_buffer->dma) {
3278 dma_unmap_page(ring->dev,
3279 tx_buffer->dma,
3280 tx_buffer->length,
3281 DMA_TO_DEVICE);
3282 }
3283 tx_buffer->next_to_watch = NULL;
3284 tx_buffer->skb = NULL;
3285 tx_buffer->dma = 0;
3286 /* buffer_info must be completely set up in the transmit path */
9d5c8243
AK		 3287}
3288
3289/**
3290 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 3291 * @tx_ring: ring to be cleaned
3292 **/
3b644cf6 3293static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243 3294{
06034649 3295 struct igb_tx_buffer *buffer_info;
9d5c8243 3296 unsigned long size;
6ad4edfc 3297 u16 i;
9d5c8243 3298
06034649 3299 if (!tx_ring->tx_buffer_info)
9d5c8243
AK		 3300 return;
3301 /* Free all the Tx ring sk_buffs */
3302
3303 for (i = 0; i < tx_ring->count; i++) {
06034649 3304 buffer_info = &tx_ring->tx_buffer_info[i];
80785298 3305 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 3306 }
3307
dad8a3b3
JF		 3308 netdev_tx_reset_queue(txring_txq(tx_ring));
3309
06034649
AD		 3310 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
3311 memset(tx_ring->tx_buffer_info, 0, size);
9d5c8243
AK		 3312
3313 /* Zero out the descriptor ring */
9d5c8243
AK		 3314 memset(tx_ring->desc, 0, tx_ring->size);
3315
3316 tx_ring->next_to_use = 0;
3317 tx_ring->next_to_clean = 0;
9d5c8243
AK		 3318}
3319
3320/**
3321 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3322 * @adapter: board private structure
3323 **/
3324static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3325{
3326 int i;
3327
3328 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3329 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 3330}
3331
3332/**
3333 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 3334 * @rx_ring: ring to clean the resources from
3335 *
3336 * Free all receive software resources
3337 **/
68fd9910 3338void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 3339{
3b644cf6 3340 igb_clean_rx_ring(rx_ring);
9d5c8243 3341
06034649
AD		 3342 vfree(rx_ring->rx_buffer_info);
3343 rx_ring->rx_buffer_info = NULL;
9d5c8243 3344
439705e1
AD		 3345 /* if not set, then don't free */
3346 if (!rx_ring->desc)
3347 return;
3348
59d71989
AD		 3349 dma_free_coherent(rx_ring->dev, rx_ring->size,
3350 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 3351
3352 rx_ring->desc = NULL;
3353}
3354
3355/**
3356 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3357 * @adapter: board private structure
3358 *
3359 * Free all receive software resources
3360 **/
3361static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3362{
3363 int i;
3364
3365 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3366 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 3367}
3368
3369/**
3370 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 3371 * @rx_ring: ring to free buffers from
3372 **/
3b644cf6 3373static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243 3374{
9d5c8243 3375 unsigned long size;
c023cd88 3376 u16 i;
9d5c8243 3377
06034649 3378 if (!rx_ring->rx_buffer_info)
9d5c8243 3379 return;
439705e1 3380
9d5c8243
AK		 3381 /* Free all the Rx ring sk_buffs */
3382 for (i = 0; i < rx_ring->count; i++) {
06034649 3383 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
9d5c8243 3384 if (buffer_info->dma) {
59d71989 3385 dma_unmap_single(rx_ring->dev,
80785298 3386 buffer_info->dma,
44390ca6 3387 IGB_RX_HDR_LEN,
59d71989 3388 DMA_FROM_DEVICE);
9d5c8243
AK		 3389 buffer_info->dma = 0;
3390 }
3391
3392 if (buffer_info->skb) {
3393 dev_kfree_skb(buffer_info->skb);
3394 buffer_info->skb = NULL;
3395 }
6ec43fe6 3396 if (buffer_info->page_dma) {
59d71989 3397 dma_unmap_page(rx_ring->dev,
80785298 3398 buffer_info->page_dma,
6ec43fe6 3399 PAGE_SIZE / 2,
59d71989 3400 DMA_FROM_DEVICE);
6ec43fe6
AD		 3401 buffer_info->page_dma = 0;
3402 }
9d5c8243 3403 if (buffer_info->page) {
9d5c8243
AK		 3404 put_page(buffer_info->page);
3405 buffer_info->page = NULL;
bf36c1a0 3406 buffer_info->page_offset = 0;
9d5c8243
AK		 3407 }
3408 }
3409
06034649
AD		 3410 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3411 memset(rx_ring->rx_buffer_info, 0, size);
9d5c8243
AK		 3412
3413 /* Zero out the descriptor ring */
3414 memset(rx_ring->desc, 0, rx_ring->size);
3415
3416 rx_ring->next_to_clean = 0;
3417 rx_ring->next_to_use = 0;
9d5c8243
AK		 3418}
3419
3420/**
3421 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3422 * @adapter: board private structure
3423 **/
3424static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3425{
3426 int i;
3427
3428 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3429 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 3430}
3431
3432/**
3433 * igb_set_mac - Change the Ethernet Address of the NIC
3434 * @netdev: network interface device structure
3435 * @p: pointer to an address structure
3436 *
3437 * Returns 0 on success, negative on failure
3438 **/
3439static int igb_set_mac(struct net_device *netdev, void *p)
3440{
3441 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 3442 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 3443 struct sockaddr *addr = p;
3444
3445 if (!is_valid_ether_addr(addr->sa_data))
3446 return -EADDRNOTAVAIL;
3447
3448 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 3449 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 3450
26ad9178
AD		 3451 /* set the correct pool for the new PF MAC address in entry 0 */
3452 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3453 adapter->vfs_allocated_count);
e1739522 3454
9d5c8243
AK		 3455 return 0;
3456}
3457
3458/**
68d480c4 3459 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 3460 * @netdev: network interface device structure
3461 *
68d480c4
AD		 3462 * Writes multicast address list to the MTA hash table.
3463 * Returns: -ENOMEM on failure
3464 * 0 on no addresses written
3465 * X on writing X addresses to MTA
9d5c8243 3466 **/
68d480c4 3467static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 3468{
3469 struct igb_adapter *adapter = netdev_priv(netdev);
3470 struct e1000_hw *hw = &adapter->hw;
22bedad3 3471 struct netdev_hw_addr *ha;
68d480c4 3472 u8 *mta_list;
9d5c8243
AK		 3473 int i;
3474
4cd24eaf 3475 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 3476 /* nothing to program, so clear mc list */
3477 igb_update_mc_addr_list(hw, NULL, 0);
3478 igb_restore_vf_multicasts(adapter);
3479 return 0;
3480 }
9d5c8243 3481
4cd24eaf 3482 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 3483 if (!mta_list)
3484 return -ENOMEM;
ff41f8dc 3485
68d480c4 3486 /* The shared function expects a packed array of only addresses. */
48e2f183 3487 i = 0;
22bedad3
JP		 3488 netdev_for_each_mc_addr(ha, netdev)
3489 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
68d480c4 3490
68d480c4
AD		 3491 igb_update_mc_addr_list(hw, mta_list, i);
3492 kfree(mta_list);
3493
4cd24eaf 3494 return netdev_mc_count(netdev);
68d480c4
AD		 3495}
3496
3497/**
3498 * igb_write_uc_addr_list - write unicast addresses to RAR table
3499 * @netdev: network interface device structure
3500 *
3501 * Writes unicast address list to the RAR table.
3502 * Returns: -ENOMEM on failure/insufficient address space
3503 * 0 on no addresses written
3504 * X on writing X addresses to the RAR table
3505 **/
3506static int igb_write_uc_addr_list(struct net_device *netdev)
3507{
3508 struct igb_adapter *adapter = netdev_priv(netdev);
3509 struct e1000_hw *hw = &adapter->hw;
3510 unsigned int vfn = adapter->vfs_allocated_count;
3511 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3512 int count = 0;
3513
3514 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 3515 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 3516 return -ENOMEM;
9d5c8243 3517
32e7bfc4 3518 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 3519 struct netdev_hw_addr *ha;
32e7bfc4
JP		 3520
3521 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 3522 if (!rar_entries)
3523 break;
26ad9178
AD		 3524 igb_rar_set_qsel(adapter, ha->addr,
3525 rar_entries--,
68d480c4
AD		 3526 vfn);
3527 count++;
ff41f8dc
AD		 3528 }
3529 }
3530 /* write the addresses in reverse order to avoid write combining */
3531 for (; rar_entries > 0 ; rar_entries--) {
3532 wr32(E1000_RAH(rar_entries), 0);
3533 wr32(E1000_RAL(rar_entries), 0);
3534 }
3535 wrfl();
3536
68d480c4
AD		 3537 return count;
3538}
3539
3540/**
3541 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3542 * @netdev: network interface device structure
3543 *
3544 * The set_rx_mode entry point is called whenever the unicast or multicast
3545 * address lists or the network interface flags are updated. This routine is
3546 * responsible for configuring the hardware for proper unicast, multicast,
3547 * promiscuous mode, and all-multi behavior.
3548 **/
3549static void igb_set_rx_mode(struct net_device *netdev)
3550{
3551 struct igb_adapter *adapter = netdev_priv(netdev);
3552 struct e1000_hw *hw = &adapter->hw;
3553 unsigned int vfn = adapter->vfs_allocated_count;
3554 u32 rctl, vmolr = 0;
3555 int count;
3556
3557 /* Check for Promiscuous and All Multicast modes */
3558 rctl = rd32(E1000_RCTL);
3559
3560 /* clear the effected bits */
3561 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3562
3563 if (netdev->flags & IFF_PROMISC) {
3564 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3565 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3566 } else {
3567 if (netdev->flags & IFF_ALLMULTI) {
3568 rctl |= E1000_RCTL_MPE;
3569 vmolr |= E1000_VMOLR_MPME;
3570 } else {
3571 /*
3572 * Write addresses to the MTA, if the attempt fails
25985edc 3573 * then we should just turn on promiscuous mode so
68d480c4
AD		 3574 * that we can at least receive multicast traffic
3575 */
3576 count = igb_write_mc_addr_list(netdev);
3577 if (count < 0) {
3578 rctl |= E1000_RCTL_MPE;
3579 vmolr |= E1000_VMOLR_MPME;
3580 } else if (count) {
3581 vmolr |= E1000_VMOLR_ROMPE;
3582 }
3583 }
3584 /*
3585 * Write addresses to available RAR registers, if there is not
3586 * sufficient space to store all the addresses then enable
25985edc 3587 * unicast promiscuous mode
68d480c4
AD		 3588 */
3589 count = igb_write_uc_addr_list(netdev);
3590 if (count < 0) {
3591 rctl |= E1000_RCTL_UPE;
3592 vmolr |= E1000_VMOLR_ROPE;
3593 }
3594 rctl |= E1000_RCTL_VFE;
28fc06f5 3595 }
68d480c4 3596 wr32(E1000_RCTL, rctl);
28fc06f5 3597
68d480c4
AD		 3598 /*
3599 * In order to support SR-IOV and eventually VMDq it is necessary to set
3600 * the VMOLR to enable the appropriate modes. Without this workaround
3601 * we will have issues with VLAN tag stripping not being done for frames
3602 * that are only arriving because we are the default pool
3603 */
f96a8a0b 3604 if ((hw->mac.type < e1000_82576) || (hw->mac.type > e1000_i350))
28fc06f5 3605 return;
9d5c8243 3606
68d480c4
AD		 3607 vmolr |= rd32(E1000_VMOLR(vfn)) &
3608 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3609 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3610 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3611}
3612
13800469
GR		 3613static void igb_check_wvbr(struct igb_adapter *adapter)
3614{
3615 struct e1000_hw *hw = &adapter->hw;
3616 u32 wvbr = 0;
3617
3618 switch (hw->mac.type) {
3619 case e1000_82576:
3620 case e1000_i350:
3621 if (!(wvbr = rd32(E1000_WVBR)))
3622 return;
3623 break;
3624 default:
3625 break;
3626 }
3627
3628 adapter->wvbr |= wvbr;
3629}
3630
3631#define IGB_STAGGERED_QUEUE_OFFSET 8
3632
3633static void igb_spoof_check(struct igb_adapter *adapter)
3634{
3635 int j;
3636
3637 if (!adapter->wvbr)
3638 return;
3639
3640 for(j = 0; j < adapter->vfs_allocated_count; j++) {
3641 if (adapter->wvbr & (1 << j) ||
3642 adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3643 dev_warn(&adapter->pdev->dev,
3644 "Spoof event(s) detected on VF %d\n", j);
3645 adapter->wvbr &=
3646 ~((1 << j) |
3647 (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3648 }
3649 }
3650}
3651
9d5c8243
AK		 3652/* Need to wait a few seconds after link up to get diagnostic information from
3653 * the phy */
3654static void igb_update_phy_info(unsigned long data)
3655{
3656 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3657 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3658}
3659
4d6b725e
AD		 3660/**
3661 * igb_has_link - check shared code for link and determine up/down
3662 * @adapter: pointer to driver private info
3663 **/
3145535a 3664bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3665{
3666 struct e1000_hw *hw = &adapter->hw;
3667 bool link_active = false;
3668 s32 ret_val = 0;
3669
3670 /* get_link_status is set on LSC (link status) interrupt or
3671 * rx sequence error interrupt. get_link_status will stay
3672 * false until the e1000_check_for_link establishes link
3673 * for copper adapters ONLY
3674 */
3675 switch (hw->phy.media_type) {
3676 case e1000_media_type_copper:
3677 if (hw->mac.get_link_status) {
3678 ret_val = hw->mac.ops.check_for_link(hw);
3679 link_active = !hw->mac.get_link_status;
3680 } else {
3681 link_active = true;
3682 }
3683 break;
4d6b725e
AD		 3684 case e1000_media_type_internal_serdes:
3685 ret_val = hw->mac.ops.check_for_link(hw);
3686 link_active = hw->mac.serdes_has_link;
3687 break;
3688 default:
3689 case e1000_media_type_unknown:
3690 break;
3691 }
3692
3693 return link_active;
3694}
3695
563988dc
SA		 3696static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
3697{
3698 bool ret = false;
3699 u32 ctrl_ext, thstat;
3700
f96a8a0b 3701 /* check for thermal sensor event on i350 copper only */
563988dc
SA		 3702 if (hw->mac.type == e1000_i350) {
3703 thstat = rd32(E1000_THSTAT);
3704 ctrl_ext = rd32(E1000_CTRL_EXT);
3705
3706 if ((hw->phy.media_type == e1000_media_type_copper) &&
3707 !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3708 ret = !!(thstat & event);
3709 }
3710 }
3711
3712 return ret;
3713}
3714
9d5c8243
AK		 3715/**
3716 * igb_watchdog - Timer Call-back
3717 * @data: pointer to adapter cast into an unsigned long
3718 **/
3719static void igb_watchdog(unsigned long data)
3720{
3721 struct igb_adapter *adapter = (struct igb_adapter *)data;
3722 /* Do the rest outside of interrupt context */
3723 schedule_work(&adapter->watchdog_task);
3724}
3725
3726static void igb_watchdog_task(struct work_struct *work)
3727{
3728 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3729 struct igb_adapter,
3730 watchdog_task);
9d5c8243 3731 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3732 struct net_device *netdev = adapter->netdev;
563988dc 3733 u32 link;
7a6ea550 3734 int i;
9d5c8243 3735
4d6b725e 3736 link = igb_has_link(adapter);
9d5c8243 3737 if (link) {
749ab2cd
YZ		 3738 /* Cancel scheduled suspend requests. */
3739 pm_runtime_resume(netdev->dev.parent);
3740
9d5c8243
AK		 3741 if (!netif_carrier_ok(netdev)) {
3742 u32 ctrl;
330a6d6a
AD		 3743 hw->mac.ops.get_speed_and_duplex(hw,
3744 &adapter->link_speed,
3745 &adapter->link_duplex);
9d5c8243
AK		 3746
3747 ctrl = rd32(E1000_CTRL);
527d47c1 3748 /* Links status message must follow this format */
876d2d6f
JK		 3749 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s "
3750 "Duplex, Flow Control: %s\n",
559e9c49
AD		 3751 netdev->name,
3752 adapter->link_speed,
3753 adapter->link_duplex == FULL_DUPLEX ?
876d2d6f
JK		 3754 "Full" : "Half",
3755 (ctrl & E1000_CTRL_TFCE) &&
3756 (ctrl & E1000_CTRL_RFCE) ? "RX/TX" :
3757 (ctrl & E1000_CTRL_RFCE) ? "RX" :
3758 (ctrl & E1000_CTRL_TFCE) ? "TX" : "None");
9d5c8243 3759
563988dc 3760 /* check for thermal sensor event */
876d2d6f
JK		 3761 if (igb_thermal_sensor_event(hw,
3762 E1000_THSTAT_LINK_THROTTLE)) {
3763 netdev_info(netdev, "The network adapter link "
3764 "speed was downshifted because it "
3765 "overheated\n");
7ef5ed1c 3766 }
563988dc 3767
d07f3e37 3768 /* adjust timeout factor according to speed/duplex */
9d5c8243
AK		 3769 adapter->tx_timeout_factor = 1;
3770 switch (adapter->link_speed) {
3771 case SPEED_10:
9d5c8243
AK		 3772 adapter->tx_timeout_factor = 14;
3773 break;
3774 case SPEED_100:
9d5c8243
AK		 3775 /* maybe add some timeout factor ? */
3776 break;
3777 }
3778
3779 netif_carrier_on(netdev);
9d5c8243 3780
4ae196df 3781 igb_ping_all_vfs(adapter);
17dc566c 3782 igb_check_vf_rate_limit(adapter);
4ae196df 3783
4b1a9877 3784 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3785 if (!test_bit(__IGB_DOWN, &adapter->state))
3786 mod_timer(&adapter->phy_info_timer,
3787 round_jiffies(jiffies + 2 * HZ));
3788 }
3789 } else {
3790 if (netif_carrier_ok(netdev)) {
3791 adapter->link_speed = 0;
3792 adapter->link_duplex = 0;
563988dc
SA		 3793
3794 /* check for thermal sensor event */
876d2d6f
JK		 3795 if (igb_thermal_sensor_event(hw,
3796 E1000_THSTAT_PWR_DOWN)) {
3797 netdev_err(netdev, "The network adapter was "
3798 "stopped because it overheated\n");
7ef5ed1c 3799 }
563988dc 3800
527d47c1
AD		 3801 /* Links status message must follow this format */
3802 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3803 netdev->name);
9d5c8243 3804 netif_carrier_off(netdev);
4b1a9877 3805
4ae196df
AD		 3806 igb_ping_all_vfs(adapter);
3807
4b1a9877 3808 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3809 if (!test_bit(__IGB_DOWN, &adapter->state))
3810 mod_timer(&adapter->phy_info_timer,
3811 round_jiffies(jiffies + 2 * HZ));
749ab2cd
YZ		 3812
3813 pm_schedule_suspend(netdev->dev.parent,
3814 MSEC_PER_SEC * 5);
9d5c8243
AK		 3815 }
3816 }
3817
12dcd86b
ED		 3818 spin_lock(&adapter->stats64_lock);
3819 igb_update_stats(adapter, &adapter->stats64);
3820 spin_unlock(&adapter->stats64_lock);
9d5c8243 3821
dbabb065 3822 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3823 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3824 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3825 /* We've lost link, so the controller stops DMA,
3826 * but we've got queued Tx work that's never going
3827 * to get done, so reset controller to flush Tx.
3828 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3829 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3830 adapter->tx_timeout_count++;
3831 schedule_work(&adapter->reset_task);
3832 /* return immediately since reset is imminent */
3833 return;
3834 }
9d5c8243 3835 }
9d5c8243 3836
dbabb065 3837 /* Force detection of hung controller every watchdog period */
6d095fa8 3838 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
dbabb065 3839 }
f7ba205e 3840
9d5c8243 3841 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3842 if (adapter->msix_entries) {
047e0030 3843 u32 eics = 0;
0d1ae7f4
AD		 3844 for (i = 0; i < adapter->num_q_vectors; i++)
3845 eics |= adapter->q_vector[i]->eims_value;
7a6ea550
AD		 3846 wr32(E1000_EICS, eics);
3847 } else {
3848 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3849 }
9d5c8243 3850
13800469
GR		 3851 igb_spoof_check(adapter);
3852
9d5c8243
AK		 3853 /* Reset the timer */
3854 if (!test_bit(__IGB_DOWN, &adapter->state))
3855 mod_timer(&adapter->watchdog_timer,
3856 round_jiffies(jiffies + 2 * HZ));
3857}
3858
3859enum latency_range {
3860 lowest_latency = 0,
3861 low_latency = 1,
3862 bulk_latency = 2,
3863 latency_invalid = 255
3864};
3865
6eb5a7f1
AD		 3866/**
3867 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3868 *
3869 * Stores a new ITR value based on strictly on packet size. This
3870 * algorithm is less sophisticated than that used in igb_update_itr,
3871 * due to the difficulty of synchronizing statistics across multiple
eef35c2d 3872 * receive rings. The divisors and thresholds used by this function
6eb5a7f1
AD		 3873 * were determined based on theoretical maximum wire speed and testing
3874 * data, in order to minimize response time while increasing bulk
3875 * throughput.
3876 * This functionality is controlled by the InterruptThrottleRate module
3877 * parameter (see igb_param.c)
3878 * NOTE: This function is called only when operating in a multiqueue
3879 * receive environment.
047e0030 3880 * @q_vector: pointer to q_vector
6eb5a7f1 3881 **/
047e0030 3882static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3883{
047e0030 3884 int new_val = q_vector->itr_val;
6eb5a7f1 3885 int avg_wire_size = 0;
047e0030 3886 struct igb_adapter *adapter = q_vector->adapter;
12dcd86b 3887 unsigned int packets;
9d5c8243 3888
6eb5a7f1
AD		 3889 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3890 * ints/sec - ITR timer value of 120 ticks.
3891 */
3892 if (adapter->link_speed != SPEED_1000) {
0ba82994 3893 new_val = IGB_4K_ITR;
6eb5a7f1 3894 goto set_itr_val;
9d5c8243 3895 }
047e0030 3896
0ba82994
AD		 3897 packets = q_vector->rx.total_packets;
3898 if (packets)
3899 avg_wire_size = q_vector->rx.total_bytes / packets;
047e0030 3900
0ba82994
AD		 3901 packets = q_vector->tx.total_packets;
3902 if (packets)
3903 avg_wire_size = max_t(u32, avg_wire_size,
3904 q_vector->tx.total_bytes / packets);
047e0030
AD		 3905
3906 /* if avg_wire_size isn't set no work was done */
3907 if (!avg_wire_size)
3908 goto clear_counts;
9d5c8243 3909
6eb5a7f1
AD		 3910 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3911 avg_wire_size += 24;
3912
3913 /* Don't starve jumbo frames */
3914 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3915
6eb5a7f1
AD		 3916 /* Give a little boost to mid-size frames */
3917 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3918 new_val = avg_wire_size / 3;
3919 else
3920 new_val = avg_wire_size / 2;
9d5c8243 3921
0ba82994
AD		 3922 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3923 if (new_val < IGB_20K_ITR &&
3924 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3925 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3926 new_val = IGB_20K_ITR;
abe1c363 3927
6eb5a7f1 3928set_itr_val:
047e0030
AD		 3929 if (new_val != q_vector->itr_val) {
3930 q_vector->itr_val = new_val;
3931 q_vector->set_itr = 1;
9d5c8243 3932 }
6eb5a7f1 3933clear_counts:
0ba82994
AD		 3934 q_vector->rx.total_bytes = 0;
3935 q_vector->rx.total_packets = 0;
3936 q_vector->tx.total_bytes = 0;
3937 q_vector->tx.total_packets = 0;
9d5c8243
AK		 3938}
3939
3940/**
3941 * igb_update_itr - update the dynamic ITR value based on statistics
3942 * Stores a new ITR value based on packets and byte
3943 * counts during the last interrupt. The advantage of per interrupt
3944 * computation is faster updates and more accurate ITR for the current
3945 * traffic pattern. Constants in this function were computed
3946 * based on theoretical maximum wire speed and thresholds were set based
3947 * on testing data as well as attempting to minimize response time
3948 * while increasing bulk throughput.
3949 * this functionality is controlled by the InterruptThrottleRate module
3950 * parameter (see igb_param.c)
3951 * NOTE: These calculations are only valid when operating in a single-
3952 * queue environment.
0ba82994
AD		 3953 * @q_vector: pointer to q_vector
3954 * @ring_container: ring info to update the itr for
9d5c8243 3955 **/
0ba82994
AD		 3956static void igb_update_itr(struct igb_q_vector *q_vector,
3957 struct igb_ring_container *ring_container)
9d5c8243 3958{
0ba82994
AD		 3959 unsigned int packets = ring_container->total_packets;
3960 unsigned int bytes = ring_container->total_bytes;
3961 u8 itrval = ring_container->itr;
9d5c8243 3962
0ba82994 3963 /* no packets, exit with status unchanged */
9d5c8243 3964 if (packets == 0)
0ba82994 3965 return;
9d5c8243 3966
0ba82994 3967 switch (itrval) {
9d5c8243
AK		 3968 case lowest_latency:
3969 /* handle TSO and jumbo frames */
3970 if (bytes/packets > 8000)
0ba82994 3971 itrval = bulk_latency;
9d5c8243 3972 else if ((packets < 5) && (bytes > 512))
0ba82994 3973 itrval = low_latency;
9d5c8243
AK		 3974 break;
3975 case low_latency: /* 50 usec aka 20000 ints/s */
3976 if (bytes > 10000) {
3977 /* this if handles the TSO accounting */
3978 if (bytes/packets > 8000) {
0ba82994 3979 itrval = bulk_latency;
9d5c8243 3980 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
0ba82994 3981 itrval = bulk_latency;
9d5c8243 3982 } else if ((packets > 35)) {
0ba82994 3983 itrval = lowest_latency;
9d5c8243
AK		 3984 }
3985 } else if (bytes/packets > 2000) {
0ba82994 3986 itrval = bulk_latency;
9d5c8243 3987 } else if (packets <= 2 && bytes < 512) {
0ba82994 3988 itrval = lowest_latency;
9d5c8243
AK		 3989 }
3990 break;
3991 case bulk_latency: /* 250 usec aka 4000 ints/s */
3992 if (bytes > 25000) {
3993 if (packets > 35)
0ba82994 3994 itrval = low_latency;
1e5c3d21 3995 } else if (bytes < 1500) {
0ba82994 3996 itrval = low_latency;
9d5c8243
AK		 3997 }
3998 break;
3999 }
4000
0ba82994
AD		 4001 /* clear work counters since we have the values we need */
4002 ring_container->total_bytes = 0;
4003 ring_container->total_packets = 0;
4004
4005 /* write updated itr to ring container */
4006 ring_container->itr = itrval;
9d5c8243
AK		 4007}
4008
0ba82994 4009static void igb_set_itr(struct igb_q_vector *q_vector)
9d5c8243 4010{
0ba82994 4011 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4012 u32 new_itr = q_vector->itr_val;
0ba82994 4013 u8 current_itr = 0;
9d5c8243
AK		 4014
4015 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
4016 if (adapter->link_speed != SPEED_1000) {
4017 current_itr = 0;
0ba82994 4018 new_itr = IGB_4K_ITR;
9d5c8243
AK		 4019 goto set_itr_now;
4020 }
4021
0ba82994
AD		 4022 igb_update_itr(q_vector, &q_vector->tx);
4023 igb_update_itr(q_vector, &q_vector->rx);
9d5c8243 4024
0ba82994 4025 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
9d5c8243 4026
6eb5a7f1 4027 /* conservative mode (itr 3) eliminates the lowest_latency setting */
0ba82994
AD		 4028 if (current_itr == lowest_latency &&
4029 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4030 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
6eb5a7f1
AD		 4031 current_itr = low_latency;
4032
9d5c8243
AK		 4033 switch (current_itr) {
4034 /* counts and packets in update_itr are dependent on these numbers */
4035 case lowest_latency:
0ba82994 4036 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
9d5c8243
AK		 4037 break;
4038 case low_latency:
0ba82994 4039 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
9d5c8243
AK		 4040 break;
4041 case bulk_latency:
0ba82994 4042 new_itr = IGB_4K_ITR; /* 4,000 ints/sec */
9d5c8243
AK		 4043 break;
4044 default:
4045 break;
4046 }
4047
4048set_itr_now:
047e0030 4049 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 4050 /* this attempts to bias the interrupt rate towards Bulk
4051 * by adding intermediate steps when interrupt rate is
4052 * increasing */
047e0030
AD		 4053 new_itr = new_itr > q_vector->itr_val ?
4054 max((new_itr * q_vector->itr_val) /
4055 (new_itr + (q_vector->itr_val >> 2)),
0ba82994 4056 new_itr) :
9d5c8243
AK		 4057 new_itr;
4058 /* Don't write the value here; it resets the adapter's
4059 * internal timer, and causes us to delay far longer than
4060 * we should between interrupts. Instead, we write the ITR
4061 * value at the beginning of the next interrupt so the timing
4062 * ends up being correct.
4063 */
047e0030
AD		 4064 q_vector->itr_val = new_itr;
4065 q_vector->set_itr = 1;
9d5c8243 4066 }
9d5c8243
AK		 4067}
4068
c50b52a0
SH		 4069static void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
4070 u32 type_tucmd, u32 mss_l4len_idx)
7d13a7d0
AD		 4071{
4072 struct e1000_adv_tx_context_desc *context_desc;
4073 u16 i = tx_ring->next_to_use;
4074
4075 context_desc = IGB_TX_CTXTDESC(tx_ring, i);
4076
4077 i++;
4078 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
4079
4080 /* set bits to identify this as an advanced context descriptor */
4081 type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
4082
4083 /* For 82575, context index must be unique per ring. */
866cff06 4084 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
7d13a7d0
AD		 4085 mss_l4len_idx |= tx_ring->reg_idx << 4;
4086
4087 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
4088 context_desc->seqnum_seed = 0;
4089 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
4090 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
4091}
4092
7af40ad9
AD		 4093static int igb_tso(struct igb_ring *tx_ring,
4094 struct igb_tx_buffer *first,
4095 u8 *hdr_len)
9d5c8243 4096{
7af40ad9 4097 struct sk_buff *skb = first->skb;
7d13a7d0
AD		 4098 u32 vlan_macip_lens, type_tucmd;
4099 u32 mss_l4len_idx, l4len;
4100
4101 if (!skb_is_gso(skb))
4102 return 0;
9d5c8243
AK		 4103
4104 if (skb_header_cloned(skb)) {
7af40ad9 4105 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
9d5c8243
AK		 4106 if (err)
4107 return err;
4108 }
4109
7d13a7d0
AD		 4110 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4111 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
9d5c8243 4112
7af40ad9 4113 if (first->protocol == __constant_htons(ETH_P_IP)) {
9d5c8243
AK		 4114 struct iphdr *iph = ip_hdr(skb);
4115 iph->tot_len = 0;
4116 iph->check = 0;
4117 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
4118 iph->daddr, 0,
4119 IPPROTO_TCP,
4120 0);
7d13a7d0 4121 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
7af40ad9
AD		 4122 first->tx_flags |= IGB_TX_FLAGS_TSO |
4123 IGB_TX_FLAGS_CSUM |
4124 IGB_TX_FLAGS_IPV4;
8e1e8a47 4125 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 4126 ipv6_hdr(skb)->payload_len = 0;
4127 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4128 &ipv6_hdr(skb)->daddr,
4129 0, IPPROTO_TCP, 0);
7af40ad9
AD		 4130 first->tx_flags |= IGB_TX_FLAGS_TSO |
4131 IGB_TX_FLAGS_CSUM;
9d5c8243
AK		 4132 }
4133
7af40ad9 4134 /* compute header lengths */
7d13a7d0
AD		 4135 l4len = tcp_hdrlen(skb);
4136 *hdr_len = skb_transport_offset(skb) + l4len;
9d5c8243 4137
7af40ad9
AD		 4138 /* update gso size and bytecount with header size */
4139 first->gso_segs = skb_shinfo(skb)->gso_segs;
4140 first->bytecount += (first->gso_segs - 1) * *hdr_len;
4141
9d5c8243 4142 /* MSS L4LEN IDX */
7d13a7d0
AD		 4143 mss_l4len_idx = l4len << E1000_ADVTXD_L4LEN_SHIFT;
4144 mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
9d5c8243 4145
7d13a7d0
AD		 4146 /* VLAN MACLEN IPLEN */
4147 vlan_macip_lens = skb_network_header_len(skb);
4148 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
7af40ad9 4149 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
9d5c8243 4150
7d13a7d0 4151 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
9d5c8243 4152
7d13a7d0 4153 return 1;
9d5c8243
AK		 4154}
4155
7af40ad9 4156static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
9d5c8243 4157{
7af40ad9 4158 struct sk_buff *skb = first->skb;
7d13a7d0
AD		 4159 u32 vlan_macip_lens = 0;
4160 u32 mss_l4len_idx = 0;
4161 u32 type_tucmd = 0;
9d5c8243 4162
7d13a7d0 4163 if (skb->ip_summed != CHECKSUM_PARTIAL) {
7af40ad9
AD		 4164 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN))
4165 return;
7d13a7d0
AD		 4166 } else {
4167 u8 l4_hdr = 0;
7af40ad9 4168 switch (first->protocol) {
7d13a7d0
AD		 4169 case __constant_htons(ETH_P_IP):
4170 vlan_macip_lens |= skb_network_header_len(skb);
4171 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4172 l4_hdr = ip_hdr(skb)->protocol;
4173 break;
4174 case __constant_htons(ETH_P_IPV6):
4175 vlan_macip_lens |= skb_network_header_len(skb);
4176 l4_hdr = ipv6_hdr(skb)->nexthdr;
4177 break;
4178 default:
4179 if (unlikely(net_ratelimit())) {
4180 dev_warn(tx_ring->dev,
4181 "partial checksum but proto=%x!\n",
7af40ad9 4182 first->protocol);
fa4a7ef3 4183 }
7d13a7d0
AD		 4184 break;
4185 }
fa4a7ef3 4186
7d13a7d0
AD		 4187 switch (l4_hdr) {
4188 case IPPROTO_TCP:
4189 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4190 mss_l4len_idx = tcp_hdrlen(skb) <<
4191 E1000_ADVTXD_L4LEN_SHIFT;
4192 break;
4193 case IPPROTO_SCTP:
4194 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
4195 mss_l4len_idx = sizeof(struct sctphdr) <<
4196 E1000_ADVTXD_L4LEN_SHIFT;
4197 break;
4198 case IPPROTO_UDP:
4199 mss_l4len_idx = sizeof(struct udphdr) <<
4200 E1000_ADVTXD_L4LEN_SHIFT;
4201 break;
4202 default:
4203 if (unlikely(net_ratelimit())) {
4204 dev_warn(tx_ring->dev,
4205 "partial checksum but l4 proto=%x!\n",
4206 l4_hdr);
44b0cda3 4207 }
7d13a7d0 4208 break;
9d5c8243 4209 }
7af40ad9
AD		 4210
4211 /* update TX checksum flag */
4212 first->tx_flags |= IGB_TX_FLAGS_CSUM;
7d13a7d0 4213 }
9d5c8243 4214
7d13a7d0 4215 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
7af40ad9 4216 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
9d5c8243 4217
7d13a7d0 4218 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
9d5c8243
AK		 4219}
4220
e032afc8
AD		 4221static __le32 igb_tx_cmd_type(u32 tx_flags)
4222{
4223 /* set type for advanced descriptor with frame checksum insertion */
4224 __le32 cmd_type = cpu_to_le32(E1000_ADVTXD_DTYP_DATA |
4225 E1000_ADVTXD_DCMD_IFCS |
4226 E1000_ADVTXD_DCMD_DEXT);
4227
4228 /* set HW vlan bit if vlan is present */
4229 if (tx_flags & IGB_TX_FLAGS_VLAN)
4230 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_VLE);
4231
4232 /* set timestamp bit if present */
4233 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4234 cmd_type |= cpu_to_le32(E1000_ADVTXD_MAC_TSTAMP);
4235
4236 /* set segmentation bits for TSO */
4237 if (tx_flags & IGB_TX_FLAGS_TSO)
4238 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_TSE);
4239
4240 return cmd_type;
4241}
4242
7af40ad9
AD		 4243static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
4244 union e1000_adv_tx_desc *tx_desc,
4245 u32 tx_flags, unsigned int paylen)
e032afc8
AD		 4246{
4247 u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
4248
4249 /* 82575 requires a unique index per ring if any offload is enabled */
4250 if ((tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_VLAN)) &&
866cff06 4251 test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
e032afc8
AD		 4252 olinfo_status |= tx_ring->reg_idx << 4;
4253
4254 /* insert L4 checksum */
4255 if (tx_flags & IGB_TX_FLAGS_CSUM) {
4256 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4257
4258 /* insert IPv4 checksum */
4259 if (tx_flags & IGB_TX_FLAGS_IPV4)
4260 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4261 }
4262
7af40ad9 4263 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
e032afc8
AD		 4264}
4265
ebe42d16
AD		 4266/*
4267 * The largest size we can write to the descriptor is 65535. In order to
4268 * maintain a power of two alignment we have to limit ourselves to 32K.
4269 */
4270#define IGB_MAX_TXD_PWR 15
7af40ad9 4271#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
9d5c8243 4272
7af40ad9
AD		 4273static void igb_tx_map(struct igb_ring *tx_ring,
4274 struct igb_tx_buffer *first,
ebe42d16 4275 const u8 hdr_len)
9d5c8243 4276{
7af40ad9 4277 struct sk_buff *skb = first->skb;
ebe42d16
AD		 4278 struct igb_tx_buffer *tx_buffer_info;
4279 union e1000_adv_tx_desc *tx_desc;
4280 dma_addr_t dma;
4281 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
4282 unsigned int data_len = skb->data_len;
4283 unsigned int size = skb_headlen(skb);
4284 unsigned int paylen = skb->len - hdr_len;
4285 __le32 cmd_type;
7af40ad9 4286 u32 tx_flags = first->tx_flags;
ebe42d16 4287 u16 i = tx_ring->next_to_use;
ebe42d16
AD		 4288
4289 tx_desc = IGB_TX_DESC(tx_ring, i);
4290
7af40ad9 4291 igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, paylen);
ebe42d16
AD		 4292 cmd_type = igb_tx_cmd_type(tx_flags);
4293
4294 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
4295 if (dma_mapping_error(tx_ring->dev, dma))
6366ad33 4296 goto dma_error;
9d5c8243 4297
ebe42d16
AD		 4298 /* record length, and DMA address */
4299 first->length = size;
4300 first->dma = dma;
ebe42d16
AD		 4301 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4302
4303 for (;;) {
4304 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
4305 tx_desc->read.cmd_type_len =
4306 cmd_type | cpu_to_le32(IGB_MAX_DATA_PER_TXD);
4307
4308 i++;
4309 tx_desc++;
4310 if (i == tx_ring->count) {
4311 tx_desc = IGB_TX_DESC(tx_ring, 0);
4312 i = 0;
4313 }
4314
4315 dma += IGB_MAX_DATA_PER_TXD;
4316 size -= IGB_MAX_DATA_PER_TXD;
4317
4318 tx_desc->read.olinfo_status = 0;
4319 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4320 }
4321
4322 if (likely(!data_len))
4323 break;
2bbfebe2 4324
ebe42d16 4325 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
9d5c8243 4326
65689fef 4327 i++;
ebe42d16
AD		 4328 tx_desc++;
4329 if (i == tx_ring->count) {
4330 tx_desc = IGB_TX_DESC(tx_ring, 0);
65689fef 4331 i = 0;
ebe42d16 4332 }
65689fef 4333
9e903e08 4334 size = skb_frag_size(frag);
ebe42d16
AD		 4335 data_len -= size;
4336
4337 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
4338 size, DMA_TO_DEVICE);
4339 if (dma_mapping_error(tx_ring->dev, dma))
6366ad33
AD		 4340 goto dma_error;
4341
ebe42d16
AD		 4342 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4343 tx_buffer_info->length = size;
4344 tx_buffer_info->dma = dma;
4345
4346 tx_desc->read.olinfo_status = 0;
4347 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4348
4349 frag++;
9d5c8243
AK		 4350 }
4351
bdbc0631
ED		 4352 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
4353
ebe42d16
AD		 4354 /* write last descriptor with RS and EOP bits */
4355 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IGB_TXD_DCMD);
6b8f0922
BG		 4356 if (unlikely(skb->no_fcs))
4357 cmd_type &= ~(cpu_to_le32(E1000_ADVTXD_DCMD_IFCS));
ebe42d16 4358 tx_desc->read.cmd_type_len = cmd_type;
8542db05
AD		 4359
4360 /* set the timestamp */
4361 first->time_stamp = jiffies;
4362
ebe42d16
AD		 4363 /*
4364 * Force memory writes to complete before letting h/w know there
4365 * are new descriptors to fetch. (Only applicable for weak-ordered
4366 * memory model archs, such as IA-64).
4367 *
4368 * We also need this memory barrier to make certain all of the
4369 * status bits have been updated before next_to_watch is written.
4370 */
4371 wmb();
4372
8542db05 4373 /* set next_to_watch value indicating a packet is present */
ebe42d16 4374 first->next_to_watch = tx_desc;
9d5c8243 4375
ebe42d16
AD		 4376 i++;
4377 if (i == tx_ring->count)
4378 i = 0;
6366ad33 4379
ebe42d16 4380 tx_ring->next_to_use = i;
6366ad33 4381
ebe42d16 4382 writel(i, tx_ring->tail);
6366ad33 4383
ebe42d16
AD		 4384 /* we need this if more than one processor can write to our tail
4385 * at a time, it syncronizes IO on IA64/Altix systems */
4386 mmiowb();
4387
4388 return;
4389
4390dma_error:
4391 dev_err(tx_ring->dev, "TX DMA map failed\n");
4392
4393 /* clear dma mappings for failed tx_buffer_info map */
4394 for (;;) {
4395 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4396 igb_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
4397 if (tx_buffer_info == first)
4398 break;
a77ff709
NN		 4399 if (i == 0)
4400 i = tx_ring->count;
6366ad33 4401 i--;
6366ad33
AD		 4402 }
4403
9d5c8243 4404 tx_ring->next_to_use = i;
9d5c8243
AK		 4405}
4406
6ad4edfc 4407static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
9d5c8243 4408{
e694e964
AD		 4409 struct net_device *netdev = tx_ring->netdev;
4410
661086df 4411 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 4412
9d5c8243
AK		 4413 /* Herbert's original patch had:
4414 * smp_mb__after_netif_stop_queue();
4415 * but since that doesn't exist yet, just open code it. */
4416 smp_mb();
4417
4418 /* We need to check again in a case another CPU has just
4419 * made room available. */
c493ea45 4420 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 4421 return -EBUSY;
4422
4423 /* A reprieve! */
661086df 4424 netif_wake_subqueue(netdev, tx_ring->queue_index);
12dcd86b
ED		 4425
4426 u64_stats_update_begin(&tx_ring->tx_syncp2);
4427 tx_ring->tx_stats.restart_queue2++;
4428 u64_stats_update_end(&tx_ring->tx_syncp2);
4429
9d5c8243
AK		 4430 return 0;
4431}
4432
6ad4edfc 4433static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
9d5c8243 4434{
c493ea45 4435 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 4436 return 0;
e694e964 4437 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 4438}
4439
cd392f5c
AD		 4440netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
4441 struct igb_ring *tx_ring)
9d5c8243 4442{
8542db05 4443 struct igb_tx_buffer *first;
ebe42d16 4444 int tso;
91d4ee33 4445 u32 tx_flags = 0;
31f6adbb 4446 __be16 protocol = vlan_get_protocol(skb);
91d4ee33 4447 u8 hdr_len = 0;
9d5c8243 4448
9d5c8243
AK		 4449 /* need: 1 descriptor per page,
4450 * + 2 desc gap to keep tail from touching head,
4451 * + 1 desc for skb->data,
4452 * + 1 desc for context descriptor,
4453 * otherwise try next time */
e694e964 4454 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 4455 /* this is a hard error */
9d5c8243
AK		 4456 return NETDEV_TX_BUSY;
4457 }
33af6bcc 4458
7af40ad9
AD		 4459 /* record the location of the first descriptor for this packet */
4460 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
4461 first->skb = skb;
4462 first->bytecount = skb->len;
4463 first->gso_segs = 1;
4464
2244d07b
OH		 4465 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4466 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
33af6bcc 4467 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 4468 }
9d5c8243 4469
eab6d18d 4470 if (vlan_tx_tag_present(skb)) {
9d5c8243
AK		 4471 tx_flags |= IGB_TX_FLAGS_VLAN;
4472 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4473 }
4474
7af40ad9
AD		 4475 /* record initial flags and protocol */
4476 first->tx_flags = tx_flags;
4477 first->protocol = protocol;
cdfd01fc 4478
7af40ad9
AD		 4479 tso = igb_tso(tx_ring, first, &hdr_len);
4480 if (tso < 0)
7d13a7d0 4481 goto out_drop;
7af40ad9
AD		 4482 else if (!tso)
4483 igb_tx_csum(tx_ring, first);
9d5c8243 4484
7af40ad9 4485 igb_tx_map(tx_ring, first, hdr_len);
85ad76b2
AD		 4486
4487 /* Make sure there is space in the ring for the next send. */
e694e964 4488 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 4489
9d5c8243 4490 return NETDEV_TX_OK;
7d13a7d0
AD		 4491
4492out_drop:
7af40ad9
AD		 4493 igb_unmap_and_free_tx_resource(tx_ring, first);
4494
7d13a7d0 4495 return NETDEV_TX_OK;
9d5c8243
AK		 4496}
4497
1cc3bd87
AD		 4498static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
4499 struct sk_buff *skb)
4500{
4501 unsigned int r_idx = skb->queue_mapping;
4502
4503 if (r_idx >= adapter->num_tx_queues)
4504 r_idx = r_idx % adapter->num_tx_queues;
4505
4506 return adapter->tx_ring[r_idx];
4507}
4508
cd392f5c
AD		 4509static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
4510 struct net_device *netdev)
9d5c8243
AK		 4511{
4512 struct igb_adapter *adapter = netdev_priv(netdev);
b1a436c3
AD		 4513
4514 if (test_bit(__IGB_DOWN, &adapter->state)) {
4515 dev_kfree_skb_any(skb);
4516 return NETDEV_TX_OK;
4517 }
4518
4519 if (skb->len <= 0) {
4520 dev_kfree_skb_any(skb);
4521 return NETDEV_TX_OK;
4522 }
4523
1cc3bd87
AD		 4524 /*
4525 * The minimum packet size with TCTL.PSP set is 17 so pad the skb
4526 * in order to meet this minimum size requirement.
4527 */
4528 if (skb->len < 17) {
4529 if (skb_padto(skb, 17))
4530 return NETDEV_TX_OK;
4531 skb->len = 17;
4532 }
9d5c8243 4533
1cc3bd87 4534 return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
9d5c8243
AK		 4535}
4536
4537/**
4538 * igb_tx_timeout - Respond to a Tx Hang
4539 * @netdev: network interface device structure
4540 **/
4541static void igb_tx_timeout(struct net_device *netdev)
4542{
4543 struct igb_adapter *adapter = netdev_priv(netdev);
4544 struct e1000_hw *hw = &adapter->hw;
4545
4546 /* Do the reset outside of interrupt context */
4547 adapter->tx_timeout_count++;
f7ba205e 4548
06218a8d 4549 if (hw->mac.type >= e1000_82580)
55cac248
AD		 4550 hw->dev_spec._82575.global_device_reset = true;
4551
9d5c8243 4552 schedule_work(&adapter->reset_task);
265de409
AD		 4553 wr32(E1000_EICS,
4554 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 4555}
4556
4557static void igb_reset_task(struct work_struct *work)
4558{
4559 struct igb_adapter *adapter;
4560 adapter = container_of(work, struct igb_adapter, reset_task);
4561
c97ec42a
TI		 4562 igb_dump(adapter);
4563 netdev_err(adapter->netdev, "Reset adapter\n");
9d5c8243
AK		 4564 igb_reinit_locked(adapter);
4565}
4566
4567/**
12dcd86b 4568 * igb_get_stats64 - Get System Network Statistics
9d5c8243 4569 * @netdev: network interface device structure
12dcd86b 4570 * @stats: rtnl_link_stats64 pointer
9d5c8243 4571 *
9d5c8243 4572 **/
12dcd86b
ED		 4573static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4574 struct rtnl_link_stats64 *stats)
9d5c8243 4575{
12dcd86b
ED		 4576 struct igb_adapter *adapter = netdev_priv(netdev);
4577
4578 spin_lock(&adapter->stats64_lock);
4579 igb_update_stats(adapter, &adapter->stats64);
4580 memcpy(stats, &adapter->stats64, sizeof(*stats));
4581 spin_unlock(&adapter->stats64_lock);
4582
4583 return stats;
9d5c8243
AK		 4584}
4585
4586/**
4587 * igb_change_mtu - Change the Maximum Transfer Unit
4588 * @netdev: network interface device structure
4589 * @new_mtu: new value for maximum frame size
4590 *
4591 * Returns 0 on success, negative on failure
4592 **/
4593static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4594{
4595 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4596 struct pci_dev *pdev = adapter->pdev;
153285f9 4597 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
9d5c8243 4598
c809d227 4599 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 4600 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 4601 return -EINVAL;
4602 }
4603
153285f9 4604#define MAX_STD_JUMBO_FRAME_SIZE 9238
9d5c8243 4605 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 4606 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 4607 return -EINVAL;
4608 }
4609
4610 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4611 msleep(1);
73cd78f1 4612
9d5c8243
AK		 4613 /* igb_down has a dependency on max_frame_size */
4614 adapter->max_frame_size = max_frame;
559e9c49 4615
4c844851
AD		 4616 if (netif_running(netdev))
4617 igb_down(adapter);
9d5c8243 4618
090b1795 4619 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 4620 netdev->mtu, new_mtu);
4621 netdev->mtu = new_mtu;
4622
4623 if (netif_running(netdev))
4624 igb_up(adapter);
4625 else
4626 igb_reset(adapter);
4627
4628 clear_bit(__IGB_RESETTING, &adapter->state);
4629
4630 return 0;
4631}
4632
4633/**
4634 * igb_update_stats - Update the board statistics counters
4635 * @adapter: board private structure
4636 **/
4637
12dcd86b
ED		 4638void igb_update_stats(struct igb_adapter *adapter,
4639 struct rtnl_link_stats64 *net_stats)
9d5c8243
AK		 4640{
4641 struct e1000_hw *hw = &adapter->hw;
4642 struct pci_dev *pdev = adapter->pdev;
fa3d9a6d 4643 u32 reg, mpc;
9d5c8243 4644 u16 phy_tmp;
3f9c0164
AD		 4645 int i;
4646 u64 bytes, packets;
12dcd86b
ED		 4647 unsigned int start;
4648 u64 _bytes, _packets;
9d5c8243
AK		 4649
4650#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4651
4652 /*
4653 * Prevent stats update while adapter is being reset, or if the pci
4654 * connection is down.
4655 */
4656 if (adapter->link_speed == 0)
4657 return;
4658 if (pci_channel_offline(pdev))
4659 return;
4660
3f9c0164
AD		 4661 bytes = 0;
4662 packets = 0;
4663 for (i = 0; i < adapter->num_rx_queues; i++) {
4664 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446 4665 struct igb_ring *ring = adapter->rx_ring[i];
12dcd86b 4666
3025a446 4667 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 4668 net_stats->rx_fifo_errors += rqdpc_tmp;
12dcd86b
ED		 4669
4670 do {
4671 start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4672 _bytes = ring->rx_stats.bytes;
4673 _packets = ring->rx_stats.packets;
4674 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4675 bytes += _bytes;
4676 packets += _packets;
3f9c0164
AD		 4677 }
4678
128e45eb
AD		 4679 net_stats->rx_bytes = bytes;
4680 net_stats->rx_packets = packets;
3f9c0164
AD		 4681
4682 bytes = 0;
4683 packets = 0;
4684 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 4685 struct igb_ring *ring = adapter->tx_ring[i];
12dcd86b
ED		 4686 do {
4687 start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4688 _bytes = ring->tx_stats.bytes;
4689 _packets = ring->tx_stats.packets;
4690 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4691 bytes += _bytes;
4692 packets += _packets;
3f9c0164 4693 }
128e45eb
AD		 4694 net_stats->tx_bytes = bytes;
4695 net_stats->tx_packets = packets;
3f9c0164
AD		 4696
4697 /* read stats registers */
9d5c8243
AK		 4698 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4699 adapter->stats.gprc += rd32(E1000_GPRC);
4700 adapter->stats.gorc += rd32(E1000_GORCL);
4701 rd32(E1000_GORCH); /* clear GORCL */
4702 adapter->stats.bprc += rd32(E1000_BPRC);
4703 adapter->stats.mprc += rd32(E1000_MPRC);
4704 adapter->stats.roc += rd32(E1000_ROC);
4705
4706 adapter->stats.prc64 += rd32(E1000_PRC64);
4707 adapter->stats.prc127 += rd32(E1000_PRC127);
4708 adapter->stats.prc255 += rd32(E1000_PRC255);
4709 adapter->stats.prc511 += rd32(E1000_PRC511);
4710 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4711 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4712 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4713 adapter->stats.sec += rd32(E1000_SEC);
4714
fa3d9a6d
MW		 4715 mpc = rd32(E1000_MPC);
4716 adapter->stats.mpc += mpc;
4717 net_stats->rx_fifo_errors += mpc;
9d5c8243
AK		 4718 adapter->stats.scc += rd32(E1000_SCC);
4719 adapter->stats.ecol += rd32(E1000_ECOL);
4720 adapter->stats.mcc += rd32(E1000_MCC);
4721 adapter->stats.latecol += rd32(E1000_LATECOL);
4722 adapter->stats.dc += rd32(E1000_DC);
4723 adapter->stats.rlec += rd32(E1000_RLEC);
4724 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4725 adapter->stats.xontxc += rd32(E1000_XONTXC);
4726 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4727 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4728 adapter->stats.fcruc += rd32(E1000_FCRUC);
4729 adapter->stats.gptc += rd32(E1000_GPTC);
4730 adapter->stats.gotc += rd32(E1000_GOTCL);
4731 rd32(E1000_GOTCH); /* clear GOTCL */
fa3d9a6d 4732 adapter->stats.rnbc += rd32(E1000_RNBC);
9d5c8243
AK		 4733 adapter->stats.ruc += rd32(E1000_RUC);
4734 adapter->stats.rfc += rd32(E1000_RFC);
4735 adapter->stats.rjc += rd32(E1000_RJC);
4736 adapter->stats.tor += rd32(E1000_TORH);
4737 adapter->stats.tot += rd32(E1000_TOTH);
4738 adapter->stats.tpr += rd32(E1000_TPR);
4739
4740 adapter->stats.ptc64 += rd32(E1000_PTC64);
4741 adapter->stats.ptc127 += rd32(E1000_PTC127);
4742 adapter->stats.ptc255 += rd32(E1000_PTC255);
4743 adapter->stats.ptc511 += rd32(E1000_PTC511);
4744 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4745 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4746
4747 adapter->stats.mptc += rd32(E1000_MPTC);
4748 adapter->stats.bptc += rd32(E1000_BPTC);
4749
2d0b0f69
NN		 4750 adapter->stats.tpt += rd32(E1000_TPT);
4751 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4752
4753 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4754 /* read internal phy specific stats */
4755 reg = rd32(E1000_CTRL_EXT);
4756 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4757 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4758 adapter->stats.tncrs += rd32(E1000_TNCRS);
4759 }
4760
9d5c8243
AK		 4761 adapter->stats.tsctc += rd32(E1000_TSCTC);
4762 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4763
4764 adapter->stats.iac += rd32(E1000_IAC);
4765 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4766 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4767 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4768 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4769 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4770 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4771 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4772 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4773
4774 /* Fill out the OS statistics structure */
128e45eb
AD		 4775 net_stats->multicast = adapter->stats.mprc;
4776 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4777
4778 /* Rx Errors */
4779
4780 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4781 * our own version based on RUC and ROC */
128e45eb 4782 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4783 adapter->stats.crcerrs + adapter->stats.algnerrc +
4784 adapter->stats.ruc + adapter->stats.roc +
4785 adapter->stats.cexterr;
128e45eb
AD		 4786 net_stats->rx_length_errors = adapter->stats.ruc +
4787 adapter->stats.roc;
4788 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4789 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4790 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4791
4792 /* Tx Errors */
128e45eb
AD		 4793 net_stats->tx_errors = adapter->stats.ecol +
4794 adapter->stats.latecol;
4795 net_stats->tx_aborted_errors = adapter->stats.ecol;
4796 net_stats->tx_window_errors = adapter->stats.latecol;
4797 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4798
4799 /* Tx Dropped needs to be maintained elsewhere */
4800
4801 /* Phy Stats */
4802 if (hw->phy.media_type == e1000_media_type_copper) {
4803 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4804 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4805 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4806 adapter->phy_stats.idle_errors += phy_tmp;
4807 }
4808 }
4809
4810 /* Management Stats */
4811 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4812 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4813 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
0a915b95
CW		 4814
4815 /* OS2BMC Stats */
4816 reg = rd32(E1000_MANC);
4817 if (reg & E1000_MANC_EN_BMC2OS) {
4818 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
4819 adapter->stats.o2bspc += rd32(E1000_O2BSPC);
4820 adapter->stats.b2ospc += rd32(E1000_B2OSPC);
4821 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
4822 }
9d5c8243
AK		 4823}
4824
9d5c8243
AK		 4825static irqreturn_t igb_msix_other(int irq, void *data)
4826{
047e0030 4827 struct igb_adapter *adapter = data;
9d5c8243 4828 struct e1000_hw *hw = &adapter->hw;
844290e5 4829 u32 icr = rd32(E1000_ICR);
844290e5 4830 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4831
7f081d40
AD		 4832 if (icr & E1000_ICR_DRSTA)
4833 schedule_work(&adapter->reset_task);
4834
047e0030 4835 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4836 /* HW is reporting DMA is out of sync */
4837 adapter->stats.doosync++;
13800469
GR		 4838 /* The DMA Out of Sync is also indication of a spoof event
4839 * in IOV mode. Check the Wrong VM Behavior register to
4840 * see if it is really a spoof event. */
4841 igb_check_wvbr(adapter);
dda0e083 4842 }
eebbbdba 4843
4ae196df
AD		 4844 /* Check for a mailbox event */
4845 if (icr & E1000_ICR_VMMB)
4846 igb_msg_task(adapter);
4847
4848 if (icr & E1000_ICR_LSC) {
4849 hw->mac.get_link_status = 1;
4850 /* guard against interrupt when we're going down */
4851 if (!test_bit(__IGB_DOWN, &adapter->state))
4852 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4853 }
4854
844290e5 4855 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4856
4857 return IRQ_HANDLED;
4858}
4859
047e0030 4860static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4861{
26b39276 4862 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4863 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4864
047e0030
AD		 4865 if (!q_vector->set_itr)
4866 return;
73cd78f1 4867
047e0030
AD		 4868 if (!itr_val)
4869 itr_val = 0x4;
661086df 4870
26b39276
AD		 4871 if (adapter->hw.mac.type == e1000_82575)
4872 itr_val |= itr_val << 16;
661086df 4873 else
0ba82994 4874 itr_val |= E1000_EITR_CNT_IGNR;
661086df 4875
047e0030
AD		 4876 writel(itr_val, q_vector->itr_register);
4877 q_vector->set_itr = 0;
6eb5a7f1
AD		 4878}
4879
047e0030 4880static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4881{
047e0030 4882 struct igb_q_vector *q_vector = data;
9d5c8243 4883
047e0030
AD		 4884 /* Write the ITR value calculated from the previous interrupt. */
4885 igb_write_itr(q_vector);
9d5c8243 4886
047e0030 4887 napi_schedule(&q_vector->napi);
844290e5 4888
047e0030 4889 return IRQ_HANDLED;
fe4506b6
JC		 4890}
4891
421e02f0 4892#ifdef CONFIG_IGB_DCA
047e0030 4893static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4894{
047e0030 4895 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4896 struct e1000_hw *hw = &adapter->hw;
4897 int cpu = get_cpu();
fe4506b6 4898
047e0030
AD		 4899 if (q_vector->cpu == cpu)
4900 goto out_no_update;
4901
0ba82994
AD		 4902 if (q_vector->tx.ring) {
4903 int q = q_vector->tx.ring->reg_idx;
047e0030
AD		 4904 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4905 if (hw->mac.type == e1000_82575) {
4906 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4907 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4908 } else {
047e0030
AD		 4909 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4910 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4911 E1000_DCA_TXCTRL_CPUID_SHIFT;
4912 }
4913 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4914 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4915 }
0ba82994
AD		 4916 if (q_vector->rx.ring) {
4917 int q = q_vector->rx.ring->reg_idx;
047e0030
AD		 4918 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4919 if (hw->mac.type == e1000_82575) {
2d064c06 4920 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4921 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4922 } else {
4923 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4924 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4925 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4926 }
fe4506b6
JC		 4927 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4928 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4929 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4930 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4931 }
047e0030
AD		 4932 q_vector->cpu = cpu;
4933out_no_update:
fe4506b6
JC		 4934 put_cpu();
4935}
4936
4937static void igb_setup_dca(struct igb_adapter *adapter)
4938{
7e0e99ef 4939 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4940 int i;
4941
7dfc16fa 4942 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4943 return;
4944
7e0e99ef
AD		 4945 /* Always use CB2 mode, difference is masked in the CB driver. */
4946 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4947
047e0030 4948 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4949 adapter->q_vector[i]->cpu = -1;
4950 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4951 }
4952}
4953
4954static int __igb_notify_dca(struct device *dev, void *data)
4955{
4956 struct net_device *netdev = dev_get_drvdata(dev);
4957 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4958 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4959 struct e1000_hw *hw = &adapter->hw;
4960 unsigned long event = *(unsigned long *)data;
4961
4962 switch (event) {
4963 case DCA_PROVIDER_ADD:
4964 /* if already enabled, don't do it again */
7dfc16fa 4965 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4966 break;
fe4506b6 4967 if (dca_add_requester(dev) == 0) {
bbd98fe4 4968 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4969 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4970 igb_setup_dca(adapter);
4971 break;
4972 }
4973 /* Fall Through since DCA is disabled. */
4974 case DCA_PROVIDER_REMOVE:
7dfc16fa 4975 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4976 /* without this a class_device is left
047e0030 4977 * hanging around in the sysfs model */
fe4506b6 4978 dca_remove_requester(dev);
090b1795 4979 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4980 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4981 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4982 }
4983 break;
4984 }
bbd98fe4 4985
fe4506b6 4986 return 0;
9d5c8243
AK		 4987}
4988
fe4506b6
JC		 4989static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4990 void *p)
4991{
4992 int ret_val;
4993
4994 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4995 __igb_notify_dca);
4996
4997 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4998}
421e02f0 4999#endif /* CONFIG_IGB_DCA */
9d5c8243 5000
0224d663
GR		 5001#ifdef CONFIG_PCI_IOV
5002static int igb_vf_configure(struct igb_adapter *adapter, int vf)
5003{
5004 unsigned char mac_addr[ETH_ALEN];
5005 struct pci_dev *pdev = adapter->pdev;
5006 struct e1000_hw *hw = &adapter->hw;
5007 struct pci_dev *pvfdev;
5008 unsigned int device_id;
5009 u16 thisvf_devfn;
5010
5011 random_ether_addr(mac_addr);
5012 igb_set_vf_mac(adapter, vf, mac_addr);
5013
5014 switch (adapter->hw.mac.type) {
5015 case e1000_82576:
5016 device_id = IGB_82576_VF_DEV_ID;
5017 /* VF Stride for 82576 is 2 */
5018 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 1)) |
5019 (pdev->devfn & 1);
5020 break;
5021 case e1000_i350:
5022 device_id = IGB_I350_VF_DEV_ID;
5023 /* VF Stride for I350 is 4 */
5024 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 2)) |
5025 (pdev->devfn & 3);
5026 break;
5027 default:
5028 device_id = 0;
5029 thisvf_devfn = 0;
5030 break;
5031 }
5032
5033 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
5034 while (pvfdev) {
5035 if (pvfdev->devfn == thisvf_devfn)
5036 break;
5037 pvfdev = pci_get_device(hw->vendor_id,
5038 device_id, pvfdev);
5039 }
5040
5041 if (pvfdev)
5042 adapter->vf_data[vf].vfdev = pvfdev;
5043 else
5044 dev_err(&pdev->dev,
5045 "Couldn't find pci dev ptr for VF %4.4x\n",
5046 thisvf_devfn);
5047 return pvfdev != NULL;
5048}
5049
5050static int igb_find_enabled_vfs(struct igb_adapter *adapter)
5051{
5052 struct e1000_hw *hw = &adapter->hw;
5053 struct pci_dev *pdev = adapter->pdev;
5054 struct pci_dev *pvfdev;
5055 u16 vf_devfn = 0;
5056 u16 vf_stride;
5057 unsigned int device_id;
5058 int vfs_found = 0;
5059
5060 switch (adapter->hw.mac.type) {
5061 case e1000_82576:
5062 device_id = IGB_82576_VF_DEV_ID;
5063 /* VF Stride for 82576 is 2 */
5064 vf_stride = 2;
5065 break;
5066 case e1000_i350:
5067 device_id = IGB_I350_VF_DEV_ID;
5068 /* VF Stride for I350 is 4 */
5069 vf_stride = 4;
5070 break;
5071 default:
5072 device_id = 0;
5073 vf_stride = 0;
5074 break;
5075 }
5076
5077 vf_devfn = pdev->devfn + 0x80;
5078 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
5079 while (pvfdev) {
06292921
GR		 5080 if (pvfdev->devfn == vf_devfn &&
5081 (pvfdev->bus->number >= pdev->bus->number))
0224d663
GR		 5082 vfs_found++;
5083 vf_devfn += vf_stride;
5084 pvfdev = pci_get_device(hw->vendor_id,
5085 device_id, pvfdev);
5086 }
5087
5088 return vfs_found;
5089}
5090
5091static int igb_check_vf_assignment(struct igb_adapter *adapter)
5092{
5093 int i;
5094 for (i = 0; i < adapter->vfs_allocated_count; i++) {
5095 if (adapter->vf_data[i].vfdev) {
5096 if (adapter->vf_data[i].vfdev->dev_flags &
5097 PCI_DEV_FLAGS_ASSIGNED)
5098 return true;
5099 }
5100 }
5101 return false;
5102}
5103
5104#endif
4ae196df
AD		 5105static void igb_ping_all_vfs(struct igb_adapter *adapter)
5106{
5107 struct e1000_hw *hw = &adapter->hw;
5108 u32 ping;
5109 int i;
5110
5111 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
5112 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 5113 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 5114 ping |= E1000_VT_MSGTYPE_CTS;
5115 igb_write_mbx(hw, &ping, 1, i);
5116 }
5117}
5118
7d5753f0
AD		 5119static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5120{
5121 struct e1000_hw *hw = &adapter->hw;
5122 u32 vmolr = rd32(E1000_VMOLR(vf));
5123 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5124
d85b9004 5125 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
7d5753f0
AD		 5126 IGB_VF_FLAG_MULTI_PROMISC);
5127 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5128
5129 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
5130 vmolr |= E1000_VMOLR_MPME;
d85b9004 5131 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
7d5753f0
AD		 5132 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
5133 } else {
5134 /*
5135 * if we have hashes and we are clearing a multicast promisc
5136 * flag we need to write the hashes to the MTA as this step
5137 * was previously skipped
5138 */
5139 if (vf_data->num_vf_mc_hashes > 30) {
5140 vmolr |= E1000_VMOLR_MPME;
5141 } else if (vf_data->num_vf_mc_hashes) {
5142 int j;
5143 vmolr |= E1000_VMOLR_ROMPE;
5144 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5145 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5146 }
5147 }
5148
5149 wr32(E1000_VMOLR(vf), vmolr);
5150
5151 /* there are flags left unprocessed, likely not supported */
5152 if (*msgbuf & E1000_VT_MSGINFO_MASK)
5153 return -EINVAL;
5154
5155 return 0;
5156
5157}
5158
4ae196df
AD		 5159static int igb_set_vf_multicasts(struct igb_adapter *adapter,
5160 u32 *msgbuf, u32 vf)
5161{
5162 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5163 u16 *hash_list = (u16 *)&msgbuf[1];
5164 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5165 int i;
5166
7d5753f0 5167 /* salt away the number of multicast addresses assigned
4ae196df
AD		 5168 * to this VF for later use to restore when the PF multi cast
5169 * list changes
5170 */
5171 vf_data->num_vf_mc_hashes = n;
5172
7d5753f0
AD		 5173 /* only up to 30 hash values supported */
5174 if (n > 30)
5175 n = 30;
5176
5177 /* store the hashes for later use */
4ae196df 5178 for (i = 0; i < n; i++)
a419aef8 5179 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 5180
5181 /* Flush and reset the mta with the new values */
ff41f8dc 5182 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 5183
5184 return 0;
5185}
5186
5187static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
5188{
5189 struct e1000_hw *hw = &adapter->hw;
5190 struct vf_data_storage *vf_data;
5191 int i, j;
5192
5193 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 5194 u32 vmolr = rd32(E1000_VMOLR(i));
5195 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5196
4ae196df 5197 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 5198
5199 if ((vf_data->num_vf_mc_hashes > 30) ||
5200 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
5201 vmolr |= E1000_VMOLR_MPME;
5202 } else if (vf_data->num_vf_mc_hashes) {
5203 vmolr |= E1000_VMOLR_ROMPE;
5204 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5205 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5206 }
5207 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 5208 }
5209}
5210
5211static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
5212{
5213 struct e1000_hw *hw = &adapter->hw;
5214 u32 pool_mask, reg, vid;
5215 int i;
5216
5217 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5218
5219 /* Find the vlan filter for this id */
5220 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5221 reg = rd32(E1000_VLVF(i));
5222
5223 /* remove the vf from the pool */
5224 reg &= ~pool_mask;
5225
5226 /* if pool is empty then remove entry from vfta */
5227 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
5228 (reg & E1000_VLVF_VLANID_ENABLE)) {
5229 reg = 0;
5230 vid = reg & E1000_VLVF_VLANID_MASK;
5231 igb_vfta_set(hw, vid, false);
5232 }
5233
5234 wr32(E1000_VLVF(i), reg);
5235 }
ae641bdc
AD		 5236
5237 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 5238}
5239
5240static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
5241{
5242 struct e1000_hw *hw = &adapter->hw;
5243 u32 reg, i;
5244
51466239
AD		 5245 /* The vlvf table only exists on 82576 hardware and newer */
5246 if (hw->mac.type < e1000_82576)
5247 return -1;
5248
5249 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 5250 if (!adapter->vfs_allocated_count)
5251 return -1;
5252
5253 /* Find the vlan filter for this id */
5254 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5255 reg = rd32(E1000_VLVF(i));
5256 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
5257 vid == (reg & E1000_VLVF_VLANID_MASK))
5258 break;
5259 }
5260
5261 if (add) {
5262 if (i == E1000_VLVF_ARRAY_SIZE) {
5263 /* Did not find a matching VLAN ID entry that was
5264 * enabled. Search for a free filter entry, i.e.
5265 * one without the enable bit set
5266 */
5267 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5268 reg = rd32(E1000_VLVF(i));
5269 if (!(reg & E1000_VLVF_VLANID_ENABLE))
5270 break;
5271 }
5272 }
5273 if (i < E1000_VLVF_ARRAY_SIZE) {
5274 /* Found an enabled/available entry */
5275 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5276
5277 /* if !enabled we need to set this up in vfta */
5278 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 5279 /* add VID to filter table */
5280 igb_vfta_set(hw, vid, true);
4ae196df
AD		 5281 reg |= E1000_VLVF_VLANID_ENABLE;
5282 }
cad6d05f
AD		 5283 reg &= ~E1000_VLVF_VLANID_MASK;
5284 reg |= vid;
4ae196df 5285 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5286
5287 /* do not modify RLPML for PF devices */
5288 if (vf >= adapter->vfs_allocated_count)
5289 return 0;
5290
5291 if (!adapter->vf_data[vf].vlans_enabled) {
5292 u32 size;
5293 reg = rd32(E1000_VMOLR(vf));
5294 size = reg & E1000_VMOLR_RLPML_MASK;
5295 size += 4;
5296 reg &= ~E1000_VMOLR_RLPML_MASK;
5297 reg |= size;
5298 wr32(E1000_VMOLR(vf), reg);
5299 }
ae641bdc 5300
51466239 5301 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 5302 }
5303 } else {
5304 if (i < E1000_VLVF_ARRAY_SIZE) {
5305 /* remove vf from the pool */
5306 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
5307 /* if pool is empty then remove entry from vfta */
5308 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
5309 reg = 0;
5310 igb_vfta_set(hw, vid, false);
5311 }
5312 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5313
5314 /* do not modify RLPML for PF devices */
5315 if (vf >= adapter->vfs_allocated_count)
5316 return 0;
5317
5318 adapter->vf_data[vf].vlans_enabled--;
5319 if (!adapter->vf_data[vf].vlans_enabled) {
5320 u32 size;
5321 reg = rd32(E1000_VMOLR(vf));
5322 size = reg & E1000_VMOLR_RLPML_MASK;
5323 size -= 4;
5324 reg &= ~E1000_VMOLR_RLPML_MASK;
5325 reg |= size;
5326 wr32(E1000_VMOLR(vf), reg);
5327 }
4ae196df
AD		 5328 }
5329 }
8151d294
WM		 5330 return 0;
5331}
5332
5333static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
5334{
5335 struct e1000_hw *hw = &adapter->hw;
5336
5337 if (vid)
5338 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
5339 else
5340 wr32(E1000_VMVIR(vf), 0);
5341}
5342
5343static int igb_ndo_set_vf_vlan(struct net_device *netdev,
5344 int vf, u16 vlan, u8 qos)
5345{
5346 int err = 0;
5347 struct igb_adapter *adapter = netdev_priv(netdev);
5348
5349 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
5350 return -EINVAL;
5351 if (vlan || qos) {
5352 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
5353 if (err)
5354 goto out;
5355 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
5356 igb_set_vmolr(adapter, vf, !vlan);
5357 adapter->vf_data[vf].pf_vlan = vlan;
5358 adapter->vf_data[vf].pf_qos = qos;
5359 dev_info(&adapter->pdev->dev,
5360 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5361 if (test_bit(__IGB_DOWN, &adapter->state)) {
5362 dev_warn(&adapter->pdev->dev,
5363 "The VF VLAN has been set,"
5364 " but the PF device is not up.\n");
5365 dev_warn(&adapter->pdev->dev,
5366 "Bring the PF device up before"
5367 " attempting to use the VF device.\n");
5368 }
5369 } else {
5370 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5371 false, vf);
5372 igb_set_vmvir(adapter, vlan, vf);
5373 igb_set_vmolr(adapter, vf, true);
5374 adapter->vf_data[vf].pf_vlan = 0;
5375 adapter->vf_data[vf].pf_qos = 0;
5376 }
5377out:
5378 return err;
4ae196df
AD		 5379}
5380
5381static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5382{
5383 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5384 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5385
5386 return igb_vlvf_set(adapter, vid, add, vf);
5387}
5388
f2ca0dbe 5389static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 5390{
8fa7e0f7
GR		 5391 /* clear flags - except flag that indicates PF has set the MAC */
5392 adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
f2ca0dbe 5393 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 5394
5395 /* reset offloads to defaults */
8151d294 5396 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 5397
5398 /* reset vlans for device */
5399 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 5400 if (adapter->vf_data[vf].pf_vlan)
5401 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5402 adapter->vf_data[vf].pf_vlan,
5403 adapter->vf_data[vf].pf_qos);
5404 else
5405 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 5406
5407 /* reset multicast table array for vf */
5408 adapter->vf_data[vf].num_vf_mc_hashes = 0;
5409
5410 /* Flush and reset the mta with the new values */
ff41f8dc 5411 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 5412}
5413
f2ca0dbe
AD		 5414static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5415{
5416 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5417
5418 /* generate a new mac address as we were hotplug removed/added */
8151d294
WM		 5419 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5420 random_ether_addr(vf_mac);
f2ca0dbe
AD		 5421
5422 /* process remaining reset events */
5423 igb_vf_reset(adapter, vf);
5424}
5425
5426static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 5427{
5428 struct e1000_hw *hw = &adapter->hw;
5429 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 5430 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 5431 u32 reg, msgbuf[3];
5432 u8 *addr = (u8 *)(&msgbuf[1]);
5433
5434 /* process all the same items cleared in a function level reset */
f2ca0dbe 5435 igb_vf_reset(adapter, vf);
4ae196df
AD		 5436
5437 /* set vf mac address */
26ad9178 5438 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 5439
5440 /* enable transmit and receive for vf */
5441 reg = rd32(E1000_VFTE);
5442 wr32(E1000_VFTE, reg | (1 << vf));
5443 reg = rd32(E1000_VFRE);
5444 wr32(E1000_VFRE, reg | (1 << vf));
5445
8fa7e0f7 5446 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
4ae196df
AD		 5447
5448 /* reply to reset with ack and vf mac address */
5449 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5450 memcpy(addr, vf_mac, 6);
5451 igb_write_mbx(hw, msgbuf, 3, vf);
5452}
5453
5454static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5455{
de42edde
GR		 5456 /*
5457 * The VF MAC Address is stored in a packed array of bytes
5458 * starting at the second 32 bit word of the msg array
5459 */
f2ca0dbe
AD		 5460 unsigned char *addr = (char *)&msg[1];
5461 int err = -1;
4ae196df 5462
f2ca0dbe
AD		 5463 if (is_valid_ether_addr(addr))
5464 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 5465
f2ca0dbe 5466 return err;
4ae196df
AD		 5467}
5468
5469static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5470{
5471 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5472 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5473 u32 msg = E1000_VT_MSGTYPE_NACK;
5474
5475 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 5476 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5477 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 5478 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 5479 vf_data->last_nack = jiffies;
4ae196df
AD		 5480 }
5481}
5482
f2ca0dbe 5483static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 5484{
f2ca0dbe
AD		 5485 struct pci_dev *pdev = adapter->pdev;
5486 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 5487 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5488 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5489 s32 retval;
5490
f2ca0dbe 5491 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 5492
fef45f4c
AD		 5493 if (retval) {
5494 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 5495 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 5496 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5497 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5498 return;
5499 goto out;
5500 }
4ae196df
AD		 5501
5502 /* this is a message we already processed, do nothing */
5503 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 5504 return;
4ae196df
AD		 5505
5506 /*
5507 * until the vf completes a reset it should not be
5508 * allowed to start any configuration.
5509 */
5510
5511 if (msgbuf[0] == E1000_VF_RESET) {
5512 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 5513 return;
4ae196df
AD		 5514 }
5515
f2ca0dbe 5516 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 5517 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5518 return;
5519 retval = -1;
5520 goto out;
4ae196df
AD		 5521 }
5522
5523 switch ((msgbuf[0] & 0xFFFF)) {
5524 case E1000_VF_SET_MAC_ADDR:
a6b5ea35
GR		 5525 retval = -EINVAL;
5526 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
5527 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5528 else
5529 dev_warn(&pdev->dev,
5530 "VF %d attempted to override administratively "
5531 "set MAC address\nReload the VF driver to "
5532 "resume operations\n", vf);
4ae196df 5533 break;
7d5753f0
AD		 5534 case E1000_VF_SET_PROMISC:
5535 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5536 break;
4ae196df
AD		 5537 case E1000_VF_SET_MULTICAST:
5538 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5539 break;
5540 case E1000_VF_SET_LPE:
5541 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5542 break;
5543 case E1000_VF_SET_VLAN:
a6b5ea35
GR		 5544 retval = -1;
5545 if (vf_data->pf_vlan)
5546 dev_warn(&pdev->dev,
5547 "VF %d attempted to override administratively "
5548 "set VLAN tag\nReload the VF driver to "
5549 "resume operations\n", vf);
8151d294
WM		 5550 else
5551 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 5552 break;
5553 default:
090b1795 5554 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 5555 retval = -1;
5556 break;
5557 }
5558
fef45f4c
AD		 5559 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5560out:
4ae196df
AD		 5561 /* notify the VF of the results of what it sent us */
5562 if (retval)
5563 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5564 else
5565 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5566
4ae196df 5567 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 5568}
4ae196df 5569
f2ca0dbe
AD		 5570static void igb_msg_task(struct igb_adapter *adapter)
5571{
5572 struct e1000_hw *hw = &adapter->hw;
5573 u32 vf;
5574
5575 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5576 /* process any reset requests */
5577 if (!igb_check_for_rst(hw, vf))
5578 igb_vf_reset_event(adapter, vf);
5579
5580 /* process any messages pending */
5581 if (!igb_check_for_msg(hw, vf))
5582 igb_rcv_msg_from_vf(adapter, vf);
5583
5584 /* process any acks */
5585 if (!igb_check_for_ack(hw, vf))
5586 igb_rcv_ack_from_vf(adapter, vf);
5587 }
4ae196df
AD		 5588}
5589
68d480c4
AD		 5590/**
5591 * igb_set_uta - Set unicast filter table address
5592 * @adapter: board private structure
5593 *
5594 * The unicast table address is a register array of 32-bit registers.
5595 * The table is meant to be used in a way similar to how the MTA is used
5596 * however due to certain limitations in the hardware it is necessary to
25985edc
LDM		 5597 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
5598 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
68d480c4
AD		 5599 **/
5600static void igb_set_uta(struct igb_adapter *adapter)
5601{
5602 struct e1000_hw *hw = &adapter->hw;
5603 int i;
5604
5605 /* The UTA table only exists on 82576 hardware and newer */
5606 if (hw->mac.type < e1000_82576)
5607 return;
5608
5609 /* we only need to do this if VMDq is enabled */
5610 if (!adapter->vfs_allocated_count)
5611 return;
5612
5613 for (i = 0; i < hw->mac.uta_reg_count; i++)
5614 array_wr32(E1000_UTA, i, ~0);
5615}
5616
9d5c8243
AK		 5617/**
5618 * igb_intr_msi - Interrupt Handler
5619 * @irq: interrupt number
5620 * @data: pointer to a network interface device structure
5621 **/
5622static irqreturn_t igb_intr_msi(int irq, void *data)
5623{
047e0030
AD		 5624 struct igb_adapter *adapter = data;
5625 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5626 struct e1000_hw *hw = &adapter->hw;
5627 /* read ICR disables interrupts using IAM */
5628 u32 icr = rd32(E1000_ICR);
5629
047e0030 5630 igb_write_itr(q_vector);
9d5c8243 5631
7f081d40
AD		 5632 if (icr & E1000_ICR_DRSTA)
5633 schedule_work(&adapter->reset_task);
5634
047e0030 5635 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5636 /* HW is reporting DMA is out of sync */
5637 adapter->stats.doosync++;
5638 }
5639
9d5c8243
AK		 5640 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5641 hw->mac.get_link_status = 1;
5642 if (!test_bit(__IGB_DOWN, &adapter->state))
5643 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5644 }
5645
047e0030 5646 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5647
5648 return IRQ_HANDLED;
5649}
5650
5651/**
4a3c6433 5652 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 5653 * @irq: interrupt number
5654 * @data: pointer to a network interface device structure
5655 **/
5656static irqreturn_t igb_intr(int irq, void *data)
5657{
047e0030
AD		 5658 struct igb_adapter *adapter = data;
5659 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5660 struct e1000_hw *hw = &adapter->hw;
5661 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5662 * need for the IMC write */
5663 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 5664
5665 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5666 * not set, then the adapter didn't send an interrupt */
5667 if (!(icr & E1000_ICR_INT_ASSERTED))
5668 return IRQ_NONE;
5669
0ba82994
AD		 5670 igb_write_itr(q_vector);
5671
7f081d40
AD		 5672 if (icr & E1000_ICR_DRSTA)
5673 schedule_work(&adapter->reset_task);
5674
047e0030 5675 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5676 /* HW is reporting DMA is out of sync */
5677 adapter->stats.doosync++;
5678 }
5679
9d5c8243
AK		 5680 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5681 hw->mac.get_link_status = 1;
5682 /* guard against interrupt when we're going down */
5683 if (!test_bit(__IGB_DOWN, &adapter->state))
5684 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5685 }
5686
047e0030 5687 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5688
5689 return IRQ_HANDLED;
5690}
5691
c50b52a0 5692static void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 5693{
047e0030 5694 struct igb_adapter *adapter = q_vector->adapter;
46544258 5695 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5696
0ba82994
AD		 5697 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
5698 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
5699 if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
5700 igb_set_itr(q_vector);
46544258 5701 else
047e0030 5702 igb_update_ring_itr(q_vector);
9d5c8243
AK		 5703 }
5704
46544258
AD		 5705 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5706 if (adapter->msix_entries)
047e0030 5707 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 5708 else
5709 igb_irq_enable(adapter);
5710 }
9d5c8243
AK		 5711}
5712
46544258
AD		 5713/**
5714 * igb_poll - NAPI Rx polling callback
5715 * @napi: napi polling structure
5716 * @budget: count of how many packets we should handle
5717 **/
5718static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 5719{
047e0030
AD		 5720 struct igb_q_vector *q_vector = container_of(napi,
5721 struct igb_q_vector,
5722 napi);
16eb8815 5723 bool clean_complete = true;
9d5c8243 5724
421e02f0 5725#ifdef CONFIG_IGB_DCA
047e0030
AD		 5726 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5727 igb_update_dca(q_vector);
fe4506b6 5728#endif
0ba82994 5729 if (q_vector->tx.ring)
13fde97a 5730 clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 5731
0ba82994 5732 if (q_vector->rx.ring)
cd392f5c 5733 clean_complete &= igb_clean_rx_irq(q_vector, budget);
047e0030 5734
16eb8815
AD		 5735 /* If all work not completed, return budget and keep polling */
5736 if (!clean_complete)
5737 return budget;
46544258 5738
9d5c8243 5739 /* If not enough Rx work done, exit the polling mode */
16eb8815
AD		 5740 napi_complete(napi);
5741 igb_ring_irq_enable(q_vector);
9d5c8243 5742
16eb8815 5743 return 0;
9d5c8243 5744}
6d8126f9 5745
7ebae817 5746#ifdef CONFIG_IGB_PTP
c5b9bd5e
AD		 5747/**
5748 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5749 * @q_vector: pointer to q_vector containing needed info
06034649 5750 * @buffer: pointer to igb_tx_buffer structure
33af6bcc
PO		 5751 *
5752 * If we were asked to do hardware stamping and such a time stamp is
5753 * available, then it must have been for this skb here because we only
5754 * allow only one such packet into the queue.
5755 */
06034649
AD		 5756static void igb_tx_hwtstamp(struct igb_q_vector *q_vector,
5757 struct igb_tx_buffer *buffer_info)
33af6bcc 5758{
c5b9bd5e 5759 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc 5760 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 5761 struct skb_shared_hwtstamps shhwtstamps;
5762 u64 regval;
33af6bcc 5763
c5b9bd5e 5764 /* if skb does not support hw timestamp or TX stamp not valid exit */
2bbfebe2 5765 if (likely(!(buffer_info->tx_flags & IGB_TX_FLAGS_TSTAMP)) ||
c5b9bd5e
AD		 5766 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5767 return;
5768
5769 regval = rd32(E1000_TXSTMPL);
5770 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5771
5772 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
2873957d 5773 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
33af6bcc
PO		 5774}
5775
7ebae817 5776#endif
9d5c8243
AK		 5777/**
5778 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 5779 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 5780 * returns true if ring is completely cleaned
5781 **/
047e0030 5782static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 5783{
047e0030 5784 struct igb_adapter *adapter = q_vector->adapter;
0ba82994 5785 struct igb_ring *tx_ring = q_vector->tx.ring;
06034649 5786 struct igb_tx_buffer *tx_buffer;
8542db05 5787 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 5788 unsigned int total_bytes = 0, total_packets = 0;
0ba82994 5789 unsigned int budget = q_vector->tx.work_limit;
8542db05 5790 unsigned int i = tx_ring->next_to_clean;
9d5c8243 5791
13fde97a
AD		 5792 if (test_bit(__IGB_DOWN, &adapter->state))
5793 return true;
0e014cb1 5794
06034649 5795 tx_buffer = &tx_ring->tx_buffer_info[i];
13fde97a 5796 tx_desc = IGB_TX_DESC(tx_ring, i);
8542db05 5797 i -= tx_ring->count;
9d5c8243 5798
13fde97a 5799 for (; budget; budget--) {
8542db05 5800 eop_desc = tx_buffer->next_to_watch;
13fde97a 5801
8542db05
AD		 5802 /* prevent any other reads prior to eop_desc */
5803 rmb();
5804
5805 /* if next_to_watch is not set then there is no work pending */
5806 if (!eop_desc)
5807 break;
13fde97a
AD		 5808
5809 /* if DD is not set pending work has not been completed */
5810 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
5811 break;
5812
8542db05
AD		 5813 /* clear next_to_watch to prevent false hangs */
5814 tx_buffer->next_to_watch = NULL;
9d5c8243 5815
ebe42d16
AD		 5816 /* update the statistics for this packet */
5817 total_bytes += tx_buffer->bytecount;
5818 total_packets += tx_buffer->gso_segs;
13fde97a 5819
7ebae817 5820#ifdef CONFIG_IGB_PTP
ebe42d16
AD		 5821 /* retrieve hardware timestamp */
5822 igb_tx_hwtstamp(q_vector, tx_buffer);
5823
7ebae817 5824#endif
ebe42d16
AD		 5825 /* free the skb */
5826 dev_kfree_skb_any(tx_buffer->skb);
5827 tx_buffer->skb = NULL;
13fde97a 5828
ebe42d16
AD		 5829 /* unmap skb header data */
5830 dma_unmap_single(tx_ring->dev,
5831 tx_buffer->dma,
5832 tx_buffer->length,
5833 DMA_TO_DEVICE);
5834
5835 /* clear last DMA location and unmap remaining buffers */
5836 while (tx_desc != eop_desc) {
5837 tx_buffer->dma = 0;
9d5c8243 5838
13fde97a
AD		 5839 tx_buffer++;
5840 tx_desc++;
9d5c8243 5841 i++;
8542db05
AD		 5842 if (unlikely(!i)) {
5843 i -= tx_ring->count;
06034649 5844 tx_buffer = tx_ring->tx_buffer_info;
13fde97a
AD		 5845 tx_desc = IGB_TX_DESC(tx_ring, 0);
5846 }
ebe42d16
AD		 5847
5848 /* unmap any remaining paged data */
5849 if (tx_buffer->dma) {
5850 dma_unmap_page(tx_ring->dev,
5851 tx_buffer->dma,
5852 tx_buffer->length,
5853 DMA_TO_DEVICE);
5854 }
5855 }
5856
5857 /* clear last DMA location */
5858 tx_buffer->dma = 0;
5859
5860 /* move us one more past the eop_desc for start of next pkt */
5861 tx_buffer++;
5862 tx_desc++;
5863 i++;
5864 if (unlikely(!i)) {
5865 i -= tx_ring->count;
5866 tx_buffer = tx_ring->tx_buffer_info;
5867 tx_desc = IGB_TX_DESC(tx_ring, 0);
5868 }
0e014cb1
AD		 5869 }
5870
bdbc0631
ED		 5871 netdev_tx_completed_queue(txring_txq(tx_ring),
5872 total_packets, total_bytes);
8542db05 5873 i += tx_ring->count;
9d5c8243 5874 tx_ring->next_to_clean = i;
13fde97a
AD		 5875 u64_stats_update_begin(&tx_ring->tx_syncp);
5876 tx_ring->tx_stats.bytes += total_bytes;
5877 tx_ring->tx_stats.packets += total_packets;
5878 u64_stats_update_end(&tx_ring->tx_syncp);
0ba82994
AD		 5879 q_vector->tx.total_bytes += total_bytes;
5880 q_vector->tx.total_packets += total_packets;
9d5c8243 5881
6d095fa8 5882 if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
13fde97a 5883 struct e1000_hw *hw = &adapter->hw;
12dcd86b 5884
8542db05 5885 eop_desc = tx_buffer->next_to_watch;
9d5c8243 5886
9d5c8243
AK		 5887 /* Detect a transmit hang in hardware, this serializes the
5888 * check with the clearing of time_stamp and movement of i */
6d095fa8 5889 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
8542db05
AD		 5890 if (eop_desc &&
5891 time_after(jiffies, tx_buffer->time_stamp +
8e95a202
JP		 5892 (adapter->tx_timeout_factor * HZ)) &&
5893 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5894
9d5c8243 5895 /* detected Tx unit hang */
59d71989 5896 dev_err(tx_ring->dev,
9d5c8243 5897 "Detected Tx Unit Hang\n"
2d064c06 5898 " Tx Queue <%d>\n"
9d5c8243
AK		 5899 " TDH <%x>\n"
5900 " TDT <%x>\n"
5901 " next_to_use <%x>\n"
5902 " next_to_clean <%x>\n"
9d5c8243
AK		 5903 "buffer_info[next_to_clean]\n"
5904 " time_stamp <%lx>\n"
8542db05 5905 " next_to_watch <%p>\n"
9d5c8243
AK		 5906 " jiffies <%lx>\n"
5907 " desc.status <%x>\n",
2d064c06 5908 tx_ring->queue_index,
238ac817 5909 rd32(E1000_TDH(tx_ring->reg_idx)),
fce99e34 5910 readl(tx_ring->tail),
9d5c8243
AK		 5911 tx_ring->next_to_use,
5912 tx_ring->next_to_clean,
8542db05
AD		 5913 tx_buffer->time_stamp,
5914 eop_desc,
9d5c8243 5915 jiffies,
0e014cb1 5916 eop_desc->wb.status);
13fde97a
AD		 5917 netif_stop_subqueue(tx_ring->netdev,
5918 tx_ring->queue_index);
5919
5920 /* we are about to reset, no point in enabling stuff */
5921 return true;
9d5c8243
AK		 5922 }
5923 }
13fde97a
AD		 5924
5925 if (unlikely(total_packets &&
5926 netif_carrier_ok(tx_ring->netdev) &&
5927 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5928 /* Make sure that anybody stopping the queue after this
5929 * sees the new next_to_clean.
5930 */
5931 smp_mb();
5932 if (__netif_subqueue_stopped(tx_ring->netdev,
5933 tx_ring->queue_index) &&
5934 !(test_bit(__IGB_DOWN, &adapter->state))) {
5935 netif_wake_subqueue(tx_ring->netdev,
5936 tx_ring->queue_index);
5937
5938 u64_stats_update_begin(&tx_ring->tx_syncp);
5939 tx_ring->tx_stats.restart_queue++;
5940 u64_stats_update_end(&tx_ring->tx_syncp);
5941 }
5942 }
5943
5944 return !!budget;
9d5c8243
AK		 5945}
5946
cd392f5c 5947static inline void igb_rx_checksum(struct igb_ring *ring,
3ceb90fd
AD		 5948 union e1000_adv_rx_desc *rx_desc,
5949 struct sk_buff *skb)
9d5c8243 5950{
bc8acf2c 5951 skb_checksum_none_assert(skb);
9d5c8243 5952
294e7d78 5953 /* Ignore Checksum bit is set */
3ceb90fd 5954 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
294e7d78
AD		 5955 return;
5956
5957 /* Rx checksum disabled via ethtool */
5958 if (!(ring->netdev->features & NETIF_F_RXCSUM))
9d5c8243 5959 return;
85ad76b2 5960
9d5c8243 5961 /* TCP/UDP checksum error bit is set */
3ceb90fd
AD		 5962 if (igb_test_staterr(rx_desc,
5963 E1000_RXDEXT_STATERR_TCPE |
5964 E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5965 /*
5966 * work around errata with sctp packets where the TCPE aka
5967 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5968 * packets, (aka let the stack check the crc32c)
5969 */
866cff06
AD		 5970 if (!((skb->len == 60) &&
5971 test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
12dcd86b 5972 u64_stats_update_begin(&ring->rx_syncp);
04a5fcaa 5973 ring->rx_stats.csum_err++;
12dcd86b
ED		 5974 u64_stats_update_end(&ring->rx_syncp);
5975 }
9d5c8243 5976 /* let the stack verify checksum errors */
9d5c8243
AK		 5977 return;
5978 }
5979 /* It must be a TCP or UDP packet with a valid checksum */
3ceb90fd
AD		 5980 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
5981 E1000_RXD_STAT_UDPCS))
9d5c8243
AK		 5982 skb->ip_summed = CHECKSUM_UNNECESSARY;
5983
3ceb90fd
AD		 5984 dev_dbg(ring->dev, "cksum success: bits %08X\n",
5985 le32_to_cpu(rx_desc->wb.upper.status_error));
9d5c8243
AK		 5986}
5987
077887c3
AD		 5988static inline void igb_rx_hash(struct igb_ring *ring,
5989 union e1000_adv_rx_desc *rx_desc,
5990 struct sk_buff *skb)
5991{
5992 if (ring->netdev->features & NETIF_F_RXHASH)
5993 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
5994}
5995
7ebae817 5996#ifdef CONFIG_IGB_PTP
3ceb90fd
AD		 5997static void igb_rx_hwtstamp(struct igb_q_vector *q_vector,
5998 union e1000_adv_rx_desc *rx_desc,
5999 struct sk_buff *skb)
c5b9bd5e
AD		 6000{
6001 struct igb_adapter *adapter = q_vector->adapter;
6002 struct e1000_hw *hw = &adapter->hw;
6003 u64 regval;
6004
3ceb90fd
AD		 6005 if (!igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP |
6006 E1000_RXDADV_STAT_TS))
6007 return;
6008
c5b9bd5e
AD		 6009 /*
6010 * If this bit is set, then the RX registers contain the time stamp. No
6011 * other packet will be time stamped until we read these registers, so
6012 * read the registers to make them available again. Because only one
6013 * packet can be time stamped at a time, we know that the register
6014 * values must belong to this one here and therefore we don't need to
6015 * compare any of the additional attributes stored for it.
6016 *
2244d07b 6017 * If nothing went wrong, then it should have a shared tx_flags that we
c5b9bd5e
AD		 6018 * can turn into a skb_shared_hwtstamps.
6019 */
3ceb90fd 6020 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
757b77e2
NN		 6021 u32 *stamp = (u32 *)skb->data;
6022 regval = le32_to_cpu(*(stamp + 2));
6023 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
6024 skb_pull(skb, IGB_TS_HDR_LEN);
6025 } else {
6026 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
6027 return;
c5b9bd5e 6028
757b77e2
NN		 6029 regval = rd32(E1000_RXSTMPL);
6030 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
6031 }
c5b9bd5e
AD		 6032
6033 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
6034}
8be10e91 6035
7ebae817 6036#endif
8be10e91
AD		 6037static void igb_rx_vlan(struct igb_ring *ring,
6038 union e1000_adv_rx_desc *rx_desc,
6039 struct sk_buff *skb)
6040{
6041 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
6042 u16 vid;
6043 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
6044 test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags))
6045 vid = be16_to_cpu(rx_desc->wb.upper.vlan);
6046 else
6047 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
6048
6049 __vlan_hwaccel_put_tag(skb, vid);
6050 }
6051}
6052
44390ca6 6053static inline u16 igb_get_hlen(union e1000_adv_rx_desc *rx_desc)
2d94d8ab
AD		 6054{
6055 /* HW will not DMA in data larger than the given buffer, even if it
6056 * parses the (NFS, of course) header to be larger. In that case, it
6057 * fills the header buffer and spills the rest into the page.
6058 */
6059 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
6060 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
44390ca6
AD		 6061 if (hlen > IGB_RX_HDR_LEN)
6062 hlen = IGB_RX_HDR_LEN;
2d94d8ab
AD		 6063 return hlen;
6064}
6065
cd392f5c 6066static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, int budget)
9d5c8243 6067{
0ba82994 6068 struct igb_ring *rx_ring = q_vector->rx.ring;
16eb8815
AD		 6069 union e1000_adv_rx_desc *rx_desc;
6070 const int current_node = numa_node_id();
9d5c8243 6071 unsigned int total_bytes = 0, total_packets = 0;
16eb8815
AD		 6072 u16 cleaned_count = igb_desc_unused(rx_ring);
6073 u16 i = rx_ring->next_to_clean;
9d5c8243 6074
60136906 6075 rx_desc = IGB_RX_DESC(rx_ring, i);
9d5c8243 6076
3ceb90fd 6077 while (igb_test_staterr(rx_desc, E1000_RXD_STAT_DD)) {
06034649 6078 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
16eb8815
AD		 6079 struct sk_buff *skb = buffer_info->skb;
6080 union e1000_adv_rx_desc *next_rxd;
9d5c8243 6081
69d3ca53 6082 buffer_info->skb = NULL;
16eb8815 6083 prefetch(skb->data);
69d3ca53
AD		 6084
6085 i++;
6086 if (i == rx_ring->count)
6087 i = 0;
42d0781a 6088
60136906 6089 next_rxd = IGB_RX_DESC(rx_ring, i);
69d3ca53 6090 prefetch(next_rxd);
9d5c8243 6091
16eb8815
AD		 6092 /*
6093 * This memory barrier is needed to keep us from reading
6094 * any other fields out of the rx_desc until we know the
6095 * RXD_STAT_DD bit is set
6096 */
6097 rmb();
9d5c8243 6098
16eb8815
AD		 6099 if (!skb_is_nonlinear(skb)) {
6100 __skb_put(skb, igb_get_hlen(rx_desc));
6101 dma_unmap_single(rx_ring->dev, buffer_info->dma,
44390ca6 6102 IGB_RX_HDR_LEN,
59d71989 6103 DMA_FROM_DEVICE);
91615f76 6104 buffer_info->dma = 0;
bf36c1a0
AD		 6105 }
6106
16eb8815
AD		 6107 if (rx_desc->wb.upper.length) {
6108 u16 length = le16_to_cpu(rx_desc->wb.upper.length);
bf36c1a0 6109
aa913403 6110 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
bf36c1a0
AD		 6111 buffer_info->page,
6112 buffer_info->page_offset,
6113 length);
6114
16eb8815
AD		 6115 skb->len += length;
6116 skb->data_len += length;
95b9c1df 6117 skb->truesize += PAGE_SIZE / 2;
16eb8815 6118
d1eff350
AD		 6119 if ((page_count(buffer_info->page) != 1) ||
6120 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 6121 buffer_info->page = NULL;
6122 else
6123 get_page(buffer_info->page);
9d5c8243 6124
16eb8815
AD		 6125 dma_unmap_page(rx_ring->dev, buffer_info->page_dma,
6126 PAGE_SIZE / 2, DMA_FROM_DEVICE);
6127 buffer_info->page_dma = 0;
9d5c8243 6128 }
9d5c8243 6129
3ceb90fd 6130 if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)) {
06034649
AD		 6131 struct igb_rx_buffer *next_buffer;
6132 next_buffer = &rx_ring->rx_buffer_info[i];
b2d56536
AD		 6133 buffer_info->skb = next_buffer->skb;
6134 buffer_info->dma = next_buffer->dma;
6135 next_buffer->skb = skb;
6136 next_buffer->dma = 0;
bf36c1a0
AD		 6137 goto next_desc;
6138 }
44390ca6 6139
89eaefb6
BG		 6140 if (unlikely((igb_test_staterr(rx_desc,
6141 E1000_RXDEXT_ERR_FRAME_ERR_MASK))
6142 && !(rx_ring->netdev->features & NETIF_F_RXALL))) {
16eb8815 6143 dev_kfree_skb_any(skb);
9d5c8243
AK		 6144 goto next_desc;
6145 }
9d5c8243 6146
7ebae817 6147#ifdef CONFIG_IGB_PTP
3ceb90fd 6148 igb_rx_hwtstamp(q_vector, rx_desc, skb);
7ebae817 6149#endif
077887c3 6150 igb_rx_hash(rx_ring, rx_desc, skb);
3ceb90fd 6151 igb_rx_checksum(rx_ring, rx_desc, skb);
8be10e91 6152 igb_rx_vlan(rx_ring, rx_desc, skb);
3ceb90fd
AD		 6153
6154 total_bytes += skb->len;
6155 total_packets++;
6156
6157 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
6158
b2cb09b1 6159 napi_gro_receive(&q_vector->napi, skb);
9d5c8243 6160
16eb8815 6161 budget--;
9d5c8243 6162next_desc:
16eb8815
AD		 6163 if (!budget)
6164 break;
6165
6166 cleaned_count++;
9d5c8243
AK		 6167 /* return some buffers to hardware, one at a time is too slow */
6168 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
cd392f5c 6169 igb_alloc_rx_buffers(rx_ring, cleaned_count);
9d5c8243
AK		 6170 cleaned_count = 0;
6171 }
6172
6173 /* use prefetched values */
6174 rx_desc = next_rxd;
9d5c8243 6175 }
bf36c1a0 6176
9d5c8243 6177 rx_ring->next_to_clean = i;
12dcd86b 6178 u64_stats_update_begin(&rx_ring->rx_syncp);
9d5c8243
AK		 6179 rx_ring->rx_stats.packets += total_packets;
6180 rx_ring->rx_stats.bytes += total_bytes;
12dcd86b 6181 u64_stats_update_end(&rx_ring->rx_syncp);
0ba82994
AD		 6182 q_vector->rx.total_packets += total_packets;
6183 q_vector->rx.total_bytes += total_bytes;
c023cd88
AD		 6184
6185 if (cleaned_count)
cd392f5c 6186 igb_alloc_rx_buffers(rx_ring, cleaned_count);
c023cd88 6187
16eb8815 6188 return !!budget;
9d5c8243
AK		 6189}
6190
c023cd88 6191static bool igb_alloc_mapped_skb(struct igb_ring *rx_ring,
06034649 6192 struct igb_rx_buffer *bi)
c023cd88
AD		 6193{
6194 struct sk_buff *skb = bi->skb;
6195 dma_addr_t dma = bi->dma;
6196
6197 if (dma)
6198 return true;
6199
6200 if (likely(!skb)) {
6201 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
6202 IGB_RX_HDR_LEN);
6203 bi->skb = skb;
6204 if (!skb) {
6205 rx_ring->rx_stats.alloc_failed++;
6206 return false;
6207 }
6208
6209 /* initialize skb for ring */
6210 skb_record_rx_queue(skb, rx_ring->queue_index);
6211 }
6212
6213 dma = dma_map_single(rx_ring->dev, skb->data,
6214 IGB_RX_HDR_LEN, DMA_FROM_DEVICE);
6215
6216 if (dma_mapping_error(rx_ring->dev, dma)) {
6217 rx_ring->rx_stats.alloc_failed++;
6218 return false;
6219 }
6220
6221 bi->dma = dma;
6222 return true;
6223}
6224
6225static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
06034649 6226 struct igb_rx_buffer *bi)
c023cd88
AD		 6227{
6228 struct page *page = bi->page;
6229 dma_addr_t page_dma = bi->page_dma;
6230 unsigned int page_offset = bi->page_offset ^ (PAGE_SIZE / 2);
6231
6232 if (page_dma)
6233 return true;
6234
6235 if (!page) {
1f2149c1 6236 page = alloc_page(GFP_ATOMIC | __GFP_COLD);
c023cd88
AD		 6237 bi->page = page;
6238 if (unlikely(!page)) {
6239 rx_ring->rx_stats.alloc_failed++;
6240 return false;
6241 }
6242 }
6243
6244 page_dma = dma_map_page(rx_ring->dev, page,
6245 page_offset, PAGE_SIZE / 2,
6246 DMA_FROM_DEVICE);
6247
6248 if (dma_mapping_error(rx_ring->dev, page_dma)) {
6249 rx_ring->rx_stats.alloc_failed++;
6250 return false;
6251 }
6252
6253 bi->page_dma = page_dma;
6254 bi->page_offset = page_offset;
6255 return true;
6256}
6257
9d5c8243 6258/**
cd392f5c 6259 * igb_alloc_rx_buffers - Replace used receive buffers; packet split
9d5c8243
AK		 6260 * @adapter: address of board private structure
6261 **/
cd392f5c 6262void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
9d5c8243 6263{
9d5c8243 6264 union e1000_adv_rx_desc *rx_desc;
06034649 6265 struct igb_rx_buffer *bi;
c023cd88 6266 u16 i = rx_ring->next_to_use;
9d5c8243 6267
60136906 6268 rx_desc = IGB_RX_DESC(rx_ring, i);
06034649 6269 bi = &rx_ring->rx_buffer_info[i];
c023cd88 6270 i -= rx_ring->count;
9d5c8243
AK		 6271
6272 while (cleaned_count--) {
c023cd88
AD		 6273 if (!igb_alloc_mapped_skb(rx_ring, bi))
6274 break;
9d5c8243 6275
c023cd88
AD		 6276 /* Refresh the desc even if buffer_addrs didn't change
6277 * because each write-back erases this info. */
6278 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
9d5c8243 6279
c023cd88
AD		 6280 if (!igb_alloc_mapped_page(rx_ring, bi))
6281 break;
6282
6283 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
9d5c8243 6284
c023cd88
AD		 6285 rx_desc++;
6286 bi++;
9d5c8243 6287 i++;
c023cd88 6288 if (unlikely(!i)) {
60136906 6289 rx_desc = IGB_RX_DESC(rx_ring, 0);
06034649 6290 bi = rx_ring->rx_buffer_info;
c023cd88
AD		 6291 i -= rx_ring->count;
6292 }
6293
6294 /* clear the hdr_addr for the next_to_use descriptor */
6295 rx_desc->read.hdr_addr = 0;
9d5c8243
AK		 6296 }
6297
c023cd88
AD		 6298 i += rx_ring->count;
6299
9d5c8243
AK		 6300 if (rx_ring->next_to_use != i) {
6301 rx_ring->next_to_use = i;
9d5c8243
AK		 6302
6303 /* Force memory writes to complete before letting h/w
6304 * know there are new descriptors to fetch. (Only
6305 * applicable for weak-ordered memory model archs,
6306 * such as IA-64). */
6307 wmb();
fce99e34 6308 writel(i, rx_ring->tail);
9d5c8243
AK		 6309 }
6310}
6311
6312/**
6313 * igb_mii_ioctl -
6314 * @netdev:
6315 * @ifreq:
6316 * @cmd:
6317 **/
6318static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6319{
6320 struct igb_adapter *adapter = netdev_priv(netdev);
6321 struct mii_ioctl_data *data = if_mii(ifr);
6322
6323 if (adapter->hw.phy.media_type != e1000_media_type_copper)
6324 return -EOPNOTSUPP;
6325
6326 switch (cmd) {
6327 case SIOCGMIIPHY:
6328 data->phy_id = adapter->hw.phy.addr;
6329 break;
6330 case SIOCGMIIREG:
f5f4cf08
AD		 6331 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
6332 &data->val_out))
9d5c8243
AK		 6333 return -EIO;
6334 break;
6335 case SIOCSMIIREG:
6336 default:
6337 return -EOPNOTSUPP;
6338 }
6339 return 0;
6340}
6341
c6cb090b
PO		 6342/**
6343 * igb_hwtstamp_ioctl - control hardware time stamping
6344 * @netdev:
6345 * @ifreq:
6346 * @cmd:
6347 *
33af6bcc
PO		 6348 * Outgoing time stamping can be enabled and disabled. Play nice and
6349 * disable it when requested, although it shouldn't case any overhead
6350 * when no packet needs it. At most one packet in the queue may be
6351 * marked for time stamping, otherwise it would be impossible to tell
6352 * for sure to which packet the hardware time stamp belongs.
6353 *
6354 * Incoming time stamping has to be configured via the hardware
6355 * filters. Not all combinations are supported, in particular event
6356 * type has to be specified. Matching the kind of event packet is
6357 * not supported, with the exception of "all V2 events regardless of
6358 * level 2 or 4".
6359 *
c6cb090b
PO		 6360 **/
6361static int igb_hwtstamp_ioctl(struct net_device *netdev,
6362 struct ifreq *ifr, int cmd)
6363{
33af6bcc
PO		 6364 struct igb_adapter *adapter = netdev_priv(netdev);
6365 struct e1000_hw *hw = &adapter->hw;
c6cb090b 6366 struct hwtstamp_config config;
c5b9bd5e
AD		 6367 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
6368 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 6369 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 6370 bool is_l4 = false;
6371 bool is_l2 = false;
33af6bcc 6372 u32 regval;
c6cb090b
PO		 6373
6374 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
6375 return -EFAULT;
6376
6377 /* reserved for future extensions */
6378 if (config.flags)
6379 return -EINVAL;
6380
33af6bcc
PO		 6381 switch (config.tx_type) {
6382 case HWTSTAMP_TX_OFF:
c5b9bd5e 6383 tsync_tx_ctl = 0;
33af6bcc 6384 case HWTSTAMP_TX_ON:
33af6bcc
PO		 6385 break;
6386 default:
6387 return -ERANGE;
6388 }
6389
6390 switch (config.rx_filter) {
6391 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 6392 tsync_rx_ctl = 0;
33af6bcc
PO		 6393 break;
6394 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
6395 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
6396 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
6397 case HWTSTAMP_FILTER_ALL:
6398 /*
6399 * register TSYNCRXCFG must be set, therefore it is not
6400 * possible to time stamp both Sync and Delay_Req messages
6401 * => fall back to time stamping all packets
6402 */
c5b9bd5e 6403 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 6404 config.rx_filter = HWTSTAMP_FILTER_ALL;
6405 break;
6406 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 6407 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 6408 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 6409 is_l4 = true;
33af6bcc
PO		 6410 break;
6411 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 6412 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 6413 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 6414 is_l4 = true;
33af6bcc
PO		 6415 break;
6416 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
6417 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 6418 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 6419 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 6420 is_l2 = true;
6421 is_l4 = true;
33af6bcc
PO		 6422 config.rx_filter = HWTSTAMP_FILTER_SOME;
6423 break;
6424 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
6425 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 6426 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 6427 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 6428 is_l2 = true;
6429 is_l4 = true;
33af6bcc
PO		 6430 config.rx_filter = HWTSTAMP_FILTER_SOME;
6431 break;
6432 case HWTSTAMP_FILTER_PTP_V2_EVENT:
6433 case HWTSTAMP_FILTER_PTP_V2_SYNC:
6434 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 6435 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 6436 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 6437 is_l2 = true;
11ba69e8 6438 is_l4 = true;
33af6bcc
PO		 6439 break;
6440 default:
6441 return -ERANGE;
6442 }
6443
c5b9bd5e
AD		 6444 if (hw->mac.type == e1000_82575) {
6445 if (tsync_rx_ctl | tsync_tx_ctl)
6446 return -EINVAL;
6447 return 0;
6448 }
6449
757b77e2
NN		 6450 /*
6451 * Per-packet timestamping only works if all packets are
6452 * timestamped, so enable timestamping in all packets as
6453 * long as one rx filter was configured.
6454 */
06218a8d 6455 if ((hw->mac.type >= e1000_82580) && tsync_rx_ctl) {
757b77e2
NN		 6456 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
6457 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
6458 }
6459
33af6bcc
PO		 6460 /* enable/disable TX */
6461 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 6462 regval &= ~E1000_TSYNCTXCTL_ENABLED;
6463 regval |= tsync_tx_ctl;
33af6bcc
PO		 6464 wr32(E1000_TSYNCTXCTL, regval);
6465
c5b9bd5e 6466 /* enable/disable RX */
33af6bcc 6467 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 6468 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
6469 regval |= tsync_rx_ctl;
33af6bcc 6470 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 6471
c5b9bd5e
AD		 6472 /* define which PTP packets are time stamped */
6473 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 6474
c5b9bd5e
AD		 6475 /* define ethertype filter for timestamped packets */
6476 if (is_l2)
6477 wr32(E1000_ETQF(3),
6478 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
6479 E1000_ETQF_1588 | /* enable timestamping */
6480 ETH_P_1588)); /* 1588 eth protocol type */
6481 else
6482 wr32(E1000_ETQF(3), 0);
6483
6484#define PTP_PORT 319
6485 /* L4 Queue Filter[3]: filter by destination port and protocol */
6486 if (is_l4) {
6487 u32 ftqf = (IPPROTO_UDP /* UDP */
6488 | E1000_FTQF_VF_BP /* VF not compared */
6489 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
6490 | E1000_FTQF_MASK); /* mask all inputs */
6491 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
6492
6493 wr32(E1000_IMIR(3), htons(PTP_PORT));
6494 wr32(E1000_IMIREXT(3),
6495 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
6496 if (hw->mac.type == e1000_82576) {
6497 /* enable source port check */
6498 wr32(E1000_SPQF(3), htons(PTP_PORT));
6499 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
6500 }
6501 wr32(E1000_FTQF(3), ftqf);
6502 } else {
6503 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
6504 }
33af6bcc
PO		 6505 wrfl();
6506
6507 adapter->hwtstamp_config = config;
6508
6509 /* clear TX/RX time stamp registers, just to be sure */
6510 regval = rd32(E1000_TXSTMPH);
6511 regval = rd32(E1000_RXSTMPH);
c6cb090b 6512
33af6bcc
PO		 6513 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
6514 -EFAULT : 0;
c6cb090b
PO		 6515}
6516
9d5c8243
AK		 6517/**
6518 * igb_ioctl -
6519 * @netdev:
6520 * @ifreq:
6521 * @cmd:
6522 **/
6523static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6524{
6525 switch (cmd) {
6526 case SIOCGMIIPHY:
6527 case SIOCGMIIREG:
6528 case SIOCSMIIREG:
6529 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 6530 case SIOCSHWTSTAMP:
6531 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 6532 default:
6533 return -EOPNOTSUPP;
6534 }
6535}
6536
009bc06e
AD		 6537s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6538{
6539 struct igb_adapter *adapter = hw->back;
6540 u16 cap_offset;
6541
bdaae04c 6542 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6543 if (!cap_offset)
6544 return -E1000_ERR_CONFIG;
6545
6546 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6547
6548 return 0;
6549}
6550
6551s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6552{
6553 struct igb_adapter *adapter = hw->back;
6554 u16 cap_offset;
6555
bdaae04c 6556 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6557 if (!cap_offset)
6558 return -E1000_ERR_CONFIG;
6559
6560 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6561
6562 return 0;
6563}
6564
c8f44aff 6565static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features)
9d5c8243
AK		 6566{
6567 struct igb_adapter *adapter = netdev_priv(netdev);
6568 struct e1000_hw *hw = &adapter->hw;
6569 u32 ctrl, rctl;
5faf030c 6570 bool enable = !!(features & NETIF_F_HW_VLAN_RX);
9d5c8243 6571
5faf030c 6572 if (enable) {
9d5c8243
AK		 6573 /* enable VLAN tag insert/strip */
6574 ctrl = rd32(E1000_CTRL);
6575 ctrl |= E1000_CTRL_VME;
6576 wr32(E1000_CTRL, ctrl);
6577
51466239 6578 /* Disable CFI check */
9d5c8243 6579 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 6580 rctl &= ~E1000_RCTL_CFIEN;
6581 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 6582 } else {
6583 /* disable VLAN tag insert/strip */
6584 ctrl = rd32(E1000_CTRL);
6585 ctrl &= ~E1000_CTRL_VME;
6586 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 6587 }
6588
e1739522 6589 igb_rlpml_set(adapter);
9d5c8243
AK		 6590}
6591
8e586137 6592static int igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
9d5c8243
AK		 6593{
6594 struct igb_adapter *adapter = netdev_priv(netdev);
6595 struct e1000_hw *hw = &adapter->hw;
4ae196df 6596 int pf_id = adapter->vfs_allocated_count;
9d5c8243 6597
51466239
AD		 6598 /* attempt to add filter to vlvf array */
6599 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 6600
51466239
AD		 6601 /* add the filter since PF can receive vlans w/o entry in vlvf */
6602 igb_vfta_set(hw, vid, true);
b2cb09b1
JP		 6603
6604 set_bit(vid, adapter->active_vlans);
8e586137
JP		 6605
6606 return 0;
9d5c8243
AK		 6607}
6608
8e586137 6609static int igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
9d5c8243
AK		 6610{
6611 struct igb_adapter *adapter = netdev_priv(netdev);
6612 struct e1000_hw *hw = &adapter->hw;
4ae196df 6613 int pf_id = adapter->vfs_allocated_count;
51466239 6614 s32 err;
9d5c8243 6615
51466239
AD		 6616 /* remove vlan from VLVF table array */
6617 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 6618
51466239
AD		 6619 /* if vid was not present in VLVF just remove it from table */
6620 if (err)
4ae196df 6621 igb_vfta_set(hw, vid, false);
b2cb09b1
JP		 6622
6623 clear_bit(vid, adapter->active_vlans);
8e586137
JP		 6624
6625 return 0;
9d5c8243
AK		 6626}
6627
6628static void igb_restore_vlan(struct igb_adapter *adapter)
6629{
b2cb09b1 6630 u16 vid;
9d5c8243 6631
5faf030c
AD		 6632 igb_vlan_mode(adapter->netdev, adapter->netdev->features);
6633
b2cb09b1
JP		 6634 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
6635 igb_vlan_rx_add_vid(adapter->netdev, vid);
9d5c8243
AK		 6636}
6637
14ad2513 6638int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
9d5c8243 6639{
090b1795 6640 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 6641 struct e1000_mac_info *mac = &adapter->hw.mac;
6642
6643 mac->autoneg = 0;
6644
14ad2513
DD		 6645 /* Make sure dplx is at most 1 bit and lsb of speed is not set
6646 * for the switch() below to work */
6647 if ((spd & 1) || (dplx & ~1))
6648 goto err_inval;
6649
cd2638a8
CW		 6650 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6651 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
14ad2513
DD		 6652 spd != SPEED_1000 &&
6653 dplx != DUPLEX_FULL)
6654 goto err_inval;
cd2638a8 6655
14ad2513 6656 switch (spd + dplx) {
9d5c8243
AK		 6657 case SPEED_10 + DUPLEX_HALF:
6658 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6659 break;
6660 case SPEED_10 + DUPLEX_FULL:
6661 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6662 break;
6663 case SPEED_100 + DUPLEX_HALF:
6664 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6665 break;
6666 case SPEED_100 + DUPLEX_FULL:
6667 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6668 break;
6669 case SPEED_1000 + DUPLEX_FULL:
6670 mac->autoneg = 1;
6671 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6672 break;
6673 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6674 default:
14ad2513 6675 goto err_inval;
9d5c8243
AK		 6676 }
6677 return 0;
14ad2513
DD		 6678
6679err_inval:
6680 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6681 return -EINVAL;
9d5c8243
AK		 6682}
6683
749ab2cd
YZ		 6684static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
6685 bool runtime)
9d5c8243
AK		 6686{
6687 struct net_device *netdev = pci_get_drvdata(pdev);
6688 struct igb_adapter *adapter = netdev_priv(netdev);
6689 struct e1000_hw *hw = &adapter->hw;
2d064c06 6690 u32 ctrl, rctl, status;
749ab2cd 6691 u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
9d5c8243
AK		 6692#ifdef CONFIG_PM
6693 int retval = 0;
6694#endif
6695
6696 netif_device_detach(netdev);
6697
a88f10ec 6698 if (netif_running(netdev))
749ab2cd 6699 __igb_close(netdev, true);
a88f10ec 6700
047e0030 6701 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 6702
6703#ifdef CONFIG_PM
6704 retval = pci_save_state(pdev);
6705 if (retval)
6706 return retval;
6707#endif
6708
6709 status = rd32(E1000_STATUS);
6710 if (status & E1000_STATUS_LU)
6711 wufc &= ~E1000_WUFC_LNKC;
6712
6713 if (wufc) {
6714 igb_setup_rctl(adapter);
ff41f8dc 6715 igb_set_rx_mode(netdev);
9d5c8243
AK		 6716
6717 /* turn on all-multi mode if wake on multicast is enabled */
6718 if (wufc & E1000_WUFC_MC) {
6719 rctl = rd32(E1000_RCTL);
6720 rctl |= E1000_RCTL_MPE;
6721 wr32(E1000_RCTL, rctl);
6722 }
6723
6724 ctrl = rd32(E1000_CTRL);
6725 /* advertise wake from D3Cold */
6726 #define E1000_CTRL_ADVD3WUC 0x00100000
6727 /* phy power management enable */
6728 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6729 ctrl |= E1000_CTRL_ADVD3WUC;
6730 wr32(E1000_CTRL, ctrl);
6731
9d5c8243 6732 /* Allow time for pending master requests to run */
330a6d6a 6733 igb_disable_pcie_master(hw);
9d5c8243
AK		 6734
6735 wr32(E1000_WUC, E1000_WUC_PME_EN);
6736 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 6737 } else {
6738 wr32(E1000_WUC, 0);
6739 wr32(E1000_WUFC, 0);
9d5c8243
AK		 6740 }
6741
3fe7c4c9
RW		 6742 *enable_wake = wufc || adapter->en_mng_pt;
6743 if (!*enable_wake)
88a268c1
NN		 6744 igb_power_down_link(adapter);
6745 else
6746 igb_power_up_link(adapter);
9d5c8243
AK		 6747
6748 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6749 * would have already happened in close and is redundant. */
6750 igb_release_hw_control(adapter);
6751
6752 pci_disable_device(pdev);
6753
9d5c8243
AK		 6754 return 0;
6755}
6756
6757#ifdef CONFIG_PM
d9dd966d 6758#ifdef CONFIG_PM_SLEEP
749ab2cd 6759static int igb_suspend(struct device *dev)
3fe7c4c9
RW		 6760{
6761 int retval;
6762 bool wake;
749ab2cd 6763 struct pci_dev *pdev = to_pci_dev(dev);
3fe7c4c9 6764
749ab2cd 6765 retval = __igb_shutdown(pdev, &wake, 0);
3fe7c4c9
RW		 6766 if (retval)
6767 return retval;
6768
6769 if (wake) {
6770 pci_prepare_to_sleep(pdev);
6771 } else {
6772 pci_wake_from_d3(pdev, false);
6773 pci_set_power_state(pdev, PCI_D3hot);
6774 }
6775
6776 return 0;
6777}
d9dd966d 6778#endif /* CONFIG_PM_SLEEP */
3fe7c4c9 6779
749ab2cd 6780static int igb_resume(struct device *dev)
9d5c8243 6781{
749ab2cd 6782 struct pci_dev *pdev = to_pci_dev(dev);
9d5c8243
AK		 6783 struct net_device *netdev = pci_get_drvdata(pdev);
6784 struct igb_adapter *adapter = netdev_priv(netdev);
6785 struct e1000_hw *hw = &adapter->hw;
6786 u32 err;
6787
6788 pci_set_power_state(pdev, PCI_D0);
6789 pci_restore_state(pdev);
b94f2d77 6790 pci_save_state(pdev);
42bfd33a 6791
aed5dec3 6792 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 6793 if (err) {
6794 dev_err(&pdev->dev,
6795 "igb: Cannot enable PCI device from suspend\n");
6796 return err;
6797 }
6798 pci_set_master(pdev);
6799
6800 pci_enable_wake(pdev, PCI_D3hot, 0);
6801 pci_enable_wake(pdev, PCI_D3cold, 0);
6802
cfb8c3aa 6803 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 6804 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6805 return -ENOMEM;
9d5c8243
AK		 6806 }
6807
9d5c8243 6808 igb_reset(adapter);
a8564f03
AD		 6809
6810 /* let the f/w know that the h/w is now under the control of the
6811 * driver. */
6812 igb_get_hw_control(adapter);
6813
9d5c8243
AK		 6814 wr32(E1000_WUS, ~0);
6815
749ab2cd
YZ		 6816 if (netdev->flags & IFF_UP) {
6817 err = __igb_open(netdev, true);
a88f10ec
AD		 6818 if (err)
6819 return err;
6820 }
9d5c8243
AK		 6821
6822 netif_device_attach(netdev);
749ab2cd
YZ		 6823 return 0;
6824}
6825
6826#ifdef CONFIG_PM_RUNTIME
6827static int igb_runtime_idle(struct device *dev)
6828{
6829 struct pci_dev *pdev = to_pci_dev(dev);
6830 struct net_device *netdev = pci_get_drvdata(pdev);
6831 struct igb_adapter *adapter = netdev_priv(netdev);
6832
6833 if (!igb_has_link(adapter))
6834 pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
6835
6836 return -EBUSY;
6837}
6838
6839static int igb_runtime_suspend(struct device *dev)
6840{
6841 struct pci_dev *pdev = to_pci_dev(dev);
6842 int retval;
6843 bool wake;
6844
6845 retval = __igb_shutdown(pdev, &wake, 1);
6846 if (retval)
6847 return retval;
6848
6849 if (wake) {
6850 pci_prepare_to_sleep(pdev);
6851 } else {
6852 pci_wake_from_d3(pdev, false);
6853 pci_set_power_state(pdev, PCI_D3hot);
6854 }
9d5c8243 6855
9d5c8243
AK		 6856 return 0;
6857}
749ab2cd
YZ		 6858
6859static int igb_runtime_resume(struct device *dev)
6860{
6861 return igb_resume(dev);
6862}
6863#endif /* CONFIG_PM_RUNTIME */
9d5c8243
AK		 6864#endif
6865
6866static void igb_shutdown(struct pci_dev *pdev)
6867{
3fe7c4c9
RW		 6868 bool wake;
6869
749ab2cd 6870 __igb_shutdown(pdev, &wake, 0);
3fe7c4c9
RW		 6871
6872 if (system_state == SYSTEM_POWER_OFF) {
6873 pci_wake_from_d3(pdev, wake);
6874 pci_set_power_state(pdev, PCI_D3hot);
6875 }
9d5c8243
AK		 6876}
6877
6878#ifdef CONFIG_NET_POLL_CONTROLLER
6879/*
6880 * Polling 'interrupt' - used by things like netconsole to send skbs
6881 * without having to re-enable interrupts. It's not called while
6882 * the interrupt routine is executing.
6883 */
6884static void igb_netpoll(struct net_device *netdev)
6885{
6886 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 6887 struct e1000_hw *hw = &adapter->hw;
0d1ae7f4 6888 struct igb_q_vector *q_vector;
9d5c8243 6889 int i;
9d5c8243 6890
047e0030 6891 for (i = 0; i < adapter->num_q_vectors; i++) {
0d1ae7f4
AD		 6892 q_vector = adapter->q_vector[i];
6893 if (adapter->msix_entries)
6894 wr32(E1000_EIMC, q_vector->eims_value);
6895 else
6896 igb_irq_disable(adapter);
047e0030 6897 napi_schedule(&q_vector->napi);
eebbbdba 6898 }
9d5c8243
AK		 6899}
6900#endif /* CONFIG_NET_POLL_CONTROLLER */
6901
6902/**
6903 * igb_io_error_detected - called when PCI error is detected
6904 * @pdev: Pointer to PCI device
6905 * @state: The current pci connection state
6906 *
6907 * This function is called after a PCI bus error affecting
6908 * this device has been detected.
6909 */
6910static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6911 pci_channel_state_t state)
6912{
6913 struct net_device *netdev = pci_get_drvdata(pdev);
6914 struct igb_adapter *adapter = netdev_priv(netdev);
6915
6916 netif_device_detach(netdev);
6917
59ed6eec
AD		 6918 if (state == pci_channel_io_perm_failure)
6919 return PCI_ERS_RESULT_DISCONNECT;
6920
9d5c8243
AK		 6921 if (netif_running(netdev))
6922 igb_down(adapter);
6923 pci_disable_device(pdev);
6924
6925 /* Request a slot slot reset. */
6926 return PCI_ERS_RESULT_NEED_RESET;
6927}
6928
6929/**
6930 * igb_io_slot_reset - called after the pci bus has been reset.
6931 * @pdev: Pointer to PCI device
6932 *
6933 * Restart the card from scratch, as if from a cold-boot. Implementation
6934 * resembles the first-half of the igb_resume routine.
6935 */
6936static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6937{
6938 struct net_device *netdev = pci_get_drvdata(pdev);
6939 struct igb_adapter *adapter = netdev_priv(netdev);
6940 struct e1000_hw *hw = &adapter->hw;
40a914fa 6941 pci_ers_result_t result;
42bfd33a 6942 int err;
9d5c8243 6943
aed5dec3 6944 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6945 dev_err(&pdev->dev,
6946 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6947 result = PCI_ERS_RESULT_DISCONNECT;
6948 } else {
6949 pci_set_master(pdev);
6950 pci_restore_state(pdev);
b94f2d77 6951 pci_save_state(pdev);
9d5c8243 6952
40a914fa
AD		 6953 pci_enable_wake(pdev, PCI_D3hot, 0);
6954 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6955
40a914fa
AD		 6956 igb_reset(adapter);
6957 wr32(E1000_WUS, ~0);
6958 result = PCI_ERS_RESULT_RECOVERED;
6959 }
9d5c8243 6960
ea943d41
JK		 6961 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6962 if (err) {
6963 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6964 "failed 0x%0x\n", err);
6965 /* non-fatal, continue */
6966 }
40a914fa
AD		 6967
6968 return result;
9d5c8243
AK		 6969}
6970
6971/**
6972 * igb_io_resume - called when traffic can start flowing again.
6973 * @pdev: Pointer to PCI device
6974 *
6975 * This callback is called when the error recovery driver tells us that
6976 * its OK to resume normal operation. Implementation resembles the
6977 * second-half of the igb_resume routine.
6978 */
6979static void igb_io_resume(struct pci_dev *pdev)
6980{
6981 struct net_device *netdev = pci_get_drvdata(pdev);
6982 struct igb_adapter *adapter = netdev_priv(netdev);
6983
9d5c8243
AK		 6984 if (netif_running(netdev)) {
6985 if (igb_up(adapter)) {
6986 dev_err(&pdev->dev, "igb_up failed after reset\n");
6987 return;
6988 }
6989 }
6990
6991 netif_device_attach(netdev);
6992
6993 /* let the f/w know that the h/w is now under the control of the
6994 * driver. */
6995 igb_get_hw_control(adapter);
9d5c8243
AK		 6996}
6997
26ad9178
AD		 6998static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6999 u8 qsel)
7000{
7001 u32 rar_low, rar_high;
7002 struct e1000_hw *hw = &adapter->hw;
7003
7004 /* HW expects these in little endian so we reverse the byte order
7005 * from network order (big endian) to little endian
7006 */
7007 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
7008 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
7009 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
7010
7011 /* Indicate to hardware the Address is Valid. */
7012 rar_high |= E1000_RAH_AV;
7013
7014 if (hw->mac.type == e1000_82575)
7015 rar_high |= E1000_RAH_POOL_1 * qsel;
7016 else
7017 rar_high |= E1000_RAH_POOL_1 << qsel;
7018
7019 wr32(E1000_RAL(index), rar_low);
7020 wrfl();
7021 wr32(E1000_RAH(index), rar_high);
7022 wrfl();
7023}
7024
4ae196df
AD		 7025static int igb_set_vf_mac(struct igb_adapter *adapter,
7026 int vf, unsigned char *mac_addr)
7027{
7028 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 7029 /* VF MAC addresses start at end of receive addresses and moves
7030 * torwards the first, as a result a collision should not be possible */
7031 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 7032
37680117 7033 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 7034
26ad9178 7035 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 7036
7037 return 0;
7038}
7039
8151d294
WM		 7040static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
7041{
7042 struct igb_adapter *adapter = netdev_priv(netdev);
7043 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
7044 return -EINVAL;
7045 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
7046 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
7047 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
7048 " change effective.");
7049 if (test_bit(__IGB_DOWN, &adapter->state)) {
7050 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
7051 " but the PF device is not up.\n");
7052 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
7053 " attempting to use the VF device.\n");
7054 }
7055 return igb_set_vf_mac(adapter, vf, mac);
7056}
7057
17dc566c
LL		 7058static int igb_link_mbps(int internal_link_speed)
7059{
7060 switch (internal_link_speed) {
7061 case SPEED_100:
7062 return 100;
7063 case SPEED_1000:
7064 return 1000;
7065 default:
7066 return 0;
7067 }
7068}
7069
7070static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
7071 int link_speed)
7072{
7073 int rf_dec, rf_int;
7074 u32 bcnrc_val;
7075
7076 if (tx_rate != 0) {
7077 /* Calculate the rate factor values to set */
7078 rf_int = link_speed / tx_rate;
7079 rf_dec = (link_speed - (rf_int * tx_rate));
7080 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
7081
7082 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
7083 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
7084 E1000_RTTBCNRC_RF_INT_MASK);
7085 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
7086 } else {
7087 bcnrc_val = 0;
7088 }
7089
7090 wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
f00b0da7
LL		 7091 /*
7092 * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
7093 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
7094 */
7095 wr32(E1000_RTTBCNRM, 0x14);
17dc566c
LL		 7096 wr32(E1000_RTTBCNRC, bcnrc_val);
7097}
7098
7099static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
7100{
7101 int actual_link_speed, i;
7102 bool reset_rate = false;
7103
7104 /* VF TX rate limit was not set or not supported */
7105 if ((adapter->vf_rate_link_speed == 0) ||
7106 (adapter->hw.mac.type != e1000_82576))
7107 return;
7108
7109 actual_link_speed = igb_link_mbps(adapter->link_speed);
7110 if (actual_link_speed != adapter->vf_rate_link_speed) {
7111 reset_rate = true;
7112 adapter->vf_rate_link_speed = 0;
7113 dev_info(&adapter->pdev->dev,
7114 "Link speed has been changed. VF Transmit "
7115 "rate is disabled\n");
7116 }
7117
7118 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7119 if (reset_rate)
7120 adapter->vf_data[i].tx_rate = 0;
7121
7122 igb_set_vf_rate_limit(&adapter->hw, i,
7123 adapter->vf_data[i].tx_rate,
7124 actual_link_speed);
7125 }
7126}
7127
8151d294
WM		 7128static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
7129{
17dc566c
LL		 7130 struct igb_adapter *adapter = netdev_priv(netdev);
7131 struct e1000_hw *hw = &adapter->hw;
7132 int actual_link_speed;
7133
7134 if (hw->mac.type != e1000_82576)
7135 return -EOPNOTSUPP;
7136
7137 actual_link_speed = igb_link_mbps(adapter->link_speed);
7138 if ((vf >= adapter->vfs_allocated_count) ||
7139 (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
7140 (tx_rate < 0) || (tx_rate > actual_link_speed))
7141 return -EINVAL;
7142
7143 adapter->vf_rate_link_speed = actual_link_speed;
7144 adapter->vf_data[vf].tx_rate = (u16)tx_rate;
7145 igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
7146
7147 return 0;
8151d294
WM		 7148}
7149
7150static int igb_ndo_get_vf_config(struct net_device *netdev,
7151 int vf, struct ifla_vf_info *ivi)
7152{
7153 struct igb_adapter *adapter = netdev_priv(netdev);
7154 if (vf >= adapter->vfs_allocated_count)
7155 return -EINVAL;
7156 ivi->vf = vf;
7157 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
17dc566c 7158 ivi->tx_rate = adapter->vf_data[vf].tx_rate;
8151d294
WM		 7159 ivi->vlan = adapter->vf_data[vf].pf_vlan;
7160 ivi->qos = adapter->vf_data[vf].pf_qos;
7161 return 0;
7162}
7163
4ae196df
AD		 7164static void igb_vmm_control(struct igb_adapter *adapter)
7165{
7166 struct e1000_hw *hw = &adapter->hw;
10d8e907 7167 u32 reg;
4ae196df 7168
52a1dd4d
AD		 7169 switch (hw->mac.type) {
7170 case e1000_82575:
f96a8a0b
CW		 7171 case e1000_i210:
7172 case e1000_i211:
52a1dd4d
AD		 7173 default:
7174 /* replication is not supported for 82575 */
4ae196df 7175 return;
52a1dd4d
AD		 7176 case e1000_82576:
7177 /* notify HW that the MAC is adding vlan tags */
7178 reg = rd32(E1000_DTXCTL);
7179 reg |= E1000_DTXCTL_VLAN_ADDED;
7180 wr32(E1000_DTXCTL, reg);
7181 case e1000_82580:
7182 /* enable replication vlan tag stripping */
7183 reg = rd32(E1000_RPLOLR);
7184 reg |= E1000_RPLOLR_STRVLAN;
7185 wr32(E1000_RPLOLR, reg);
d2ba2ed8
AD		 7186 case e1000_i350:
7187 /* none of the above registers are supported by i350 */
52a1dd4d
AD		 7188 break;
7189 }
10d8e907 7190
d4960307
AD		 7191 if (adapter->vfs_allocated_count) {
7192 igb_vmdq_set_loopback_pf(hw, true);
7193 igb_vmdq_set_replication_pf(hw, true);
13800469
GR		 7194 igb_vmdq_set_anti_spoofing_pf(hw, true,
7195 adapter->vfs_allocated_count);
d4960307
AD		 7196 } else {
7197 igb_vmdq_set_loopback_pf(hw, false);
7198 igb_vmdq_set_replication_pf(hw, false);
7199 }
4ae196df
AD		 7200}
7201
b6e0c419
CW		 7202static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
7203{
7204 struct e1000_hw *hw = &adapter->hw;
7205 u32 dmac_thr;
7206 u16 hwm;
7207
7208 if (hw->mac.type > e1000_82580) {
7209 if (adapter->flags & IGB_FLAG_DMAC) {
7210 u32 reg;
7211
7212 /* force threshold to 0. */
7213 wr32(E1000_DMCTXTH, 0);
7214
7215 /*
e8c626e9
MV		 7216 * DMA Coalescing high water mark needs to be greater
7217 * than the Rx threshold. Set hwm to PBA - max frame
7218 * size in 16B units, capping it at PBA - 6KB.
b6e0c419 7219 */
e8c626e9
MV		 7220 hwm = 64 * pba - adapter->max_frame_size / 16;
7221 if (hwm < 64 * (pba - 6))
7222 hwm = 64 * (pba - 6);
7223 reg = rd32(E1000_FCRTC);
7224 reg &= ~E1000_FCRTC_RTH_COAL_MASK;
7225 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
7226 & E1000_FCRTC_RTH_COAL_MASK);
7227 wr32(E1000_FCRTC, reg);
7228
7229 /*
7230 * Set the DMA Coalescing Rx threshold to PBA - 2 * max
7231 * frame size, capping it at PBA - 10KB.
7232 */
7233 dmac_thr = pba - adapter->max_frame_size / 512;
7234 if (dmac_thr < pba - 10)
7235 dmac_thr = pba - 10;
b6e0c419
CW		 7236 reg = rd32(E1000_DMACR);
7237 reg &= ~E1000_DMACR_DMACTHR_MASK;
b6e0c419
CW		 7238 reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
7239 & E1000_DMACR_DMACTHR_MASK);
7240
7241 /* transition to L0x or L1 if available..*/
7242 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
7243
7244 /* watchdog timer= +-1000 usec in 32usec intervals */
7245 reg |= (1000 >> 5);
0c02dd98
MV		 7246
7247 /* Disable BMC-to-OS Watchdog Enable */
7248 reg &= ~E1000_DMACR_DC_BMC2OSW_EN;
b6e0c419
CW		 7249 wr32(E1000_DMACR, reg);
7250
7251 /*
7252 * no lower threshold to disable
7253 * coalescing(smart fifb)-UTRESH=0
7254 */
7255 wr32(E1000_DMCRTRH, 0);
b6e0c419
CW		 7256
7257 reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
7258
7259 wr32(E1000_DMCTLX, reg);
7260
7261 /*
7262 * free space in tx packet buffer to wake from
7263 * DMA coal
7264 */
7265 wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
7266 (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
7267
7268 /*
7269 * make low power state decision controlled
7270 * by DMA coal
7271 */
7272 reg = rd32(E1000_PCIEMISC);
7273 reg &= ~E1000_PCIEMISC_LX_DECISION;
7274 wr32(E1000_PCIEMISC, reg);
7275 } /* endif adapter->dmac is not disabled */
7276 } else if (hw->mac.type == e1000_82580) {
7277 u32 reg = rd32(E1000_PCIEMISC);
7278 wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION);
7279 wr32(E1000_DMACR, 0);
7280 }
7281}
7282
9d5c8243 7283/* igb_main.c */
ubuntu-zesty-kernel mirror