]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - drivers/net/ethernet/intel/igb/igb_main.c
projects / mirror_ubuntu-zesty-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tcp: must free metrics at net dismantle
[mirror_ubuntu-zesty-kernel.git] / drivers / net / ethernet / intel / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
6e861326 4 Copyright(c) 2007-2012 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
876d2d6f
JK		 28#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
29
9d5c8243
AK		 30#include <linux/module.h>
31#include <linux/types.h>
32#include <linux/init.h>
b2cb09b1 33#include <linux/bitops.h>
9d5c8243
AK		 34#include <linux/vmalloc.h>
35#include <linux/pagemap.h>
36#include <linux/netdevice.h>
9d5c8243 37#include <linux/ipv6.h>
5a0e3ad6 38#include <linux/slab.h>
9d5c8243
AK		 39#include <net/checksum.h>
40#include <net/ip6_checksum.h>
c6cb090b 41#include <linux/net_tstamp.h>
9d5c8243
AK		 42#include <linux/mii.h>
43#include <linux/ethtool.h>
01789349 44#include <linux/if.h>
9d5c8243
AK		 45#include <linux/if_vlan.h>
46#include <linux/pci.h>
c54106bb 47#include <linux/pci-aspm.h>
9d5c8243
AK		 48#include <linux/delay.h>
49#include <linux/interrupt.h>
7d13a7d0
AD		 50#include <linux/ip.h>
51#include <linux/tcp.h>
52#include <linux/sctp.h>
9d5c8243 53#include <linux/if_ether.h>
40a914fa 54#include <linux/aer.h>
70c71606 55#include <linux/prefetch.h>
749ab2cd 56#include <linux/pm_runtime.h>
421e02f0 57#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 58#include <linux/dca.h>
59#endif
9d5c8243
AK		 60#include "igb.h"
61
200e5fd5
CW		 62#define MAJ 4
63#define MIN 0
64#define BUILD 1
0d1fe82d 65#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
929dd047 66__stringify(BUILD) "-k"
9d5c8243
AK		 67char igb_driver_name[] = "igb";
68char igb_driver_version[] = DRV_VERSION;
69static const char igb_driver_string[] =
70 "Intel(R) Gigabit Ethernet Network Driver";
6e861326 71static const char igb_copyright[] = "Copyright (c) 2007-2012 Intel Corporation.";
9d5c8243 72
9d5c8243
AK		 73static const struct e1000_info *igb_info_tbl[] = {
74 [board_82575] = &e1000_82575_info,
75};
76
a3aa1884 77static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
f96a8a0b
CW		 78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 },
79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 },
81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 },
82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 },
d2ba2ed8
AD		 83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
55cac248
AD		 87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
6493d24f 89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
55cac248
AD		 90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
308fb39a
JG		 93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
1b5dda33
GJ		 95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
2d064c06 97 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 98 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 99 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 100 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
101 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 102 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
b894fa26 103 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
c8ea5ea9 104 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 105 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
106 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
107 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
108 /* required last entry */
109 {0, }
110};
111
112MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
113
114void igb_reset(struct igb_adapter *);
115static int igb_setup_all_tx_resources(struct igb_adapter *);
116static int igb_setup_all_rx_resources(struct igb_adapter *);
117static void igb_free_all_tx_resources(struct igb_adapter *);
118static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 119static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 120static int igb_probe(struct pci_dev *, const struct pci_device_id *);
121static void __devexit igb_remove(struct pci_dev *pdev);
122static int igb_sw_init(struct igb_adapter *);
123static int igb_open(struct net_device *);
124static int igb_close(struct net_device *);
125static void igb_configure_tx(struct igb_adapter *);
126static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 127static void igb_clean_all_tx_rings(struct igb_adapter *);
128static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 129static void igb_clean_tx_ring(struct igb_ring *);
130static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 131static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 132static void igb_update_phy_info(unsigned long);
133static void igb_watchdog(unsigned long);
134static void igb_watchdog_task(struct work_struct *);
cd392f5c 135static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
12dcd86b
ED		 136static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
137 struct rtnl_link_stats64 *stats);
9d5c8243
AK		 138static int igb_change_mtu(struct net_device *, int);
139static int igb_set_mac(struct net_device *, void *);
68d480c4 140static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 141static irqreturn_t igb_intr(int irq, void *);
142static irqreturn_t igb_intr_msi(int irq, void *);
143static irqreturn_t igb_msix_other(int irq, void *);
047e0030 144static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 145#ifdef CONFIG_IGB_DCA
047e0030 146static void igb_update_dca(struct igb_q_vector *);
fe4506b6 147static void igb_setup_dca(struct igb_adapter *);
421e02f0 148#endif /* CONFIG_IGB_DCA */
661086df 149static int igb_poll(struct napi_struct *, int);
13fde97a 150static bool igb_clean_tx_irq(struct igb_q_vector *);
cd392f5c 151static bool igb_clean_rx_irq(struct igb_q_vector *, int);
9d5c8243
AK		 152static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
153static void igb_tx_timeout(struct net_device *);
154static void igb_reset_task(struct work_struct *);
c8f44aff 155static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features);
8e586137
JP		 156static int igb_vlan_rx_add_vid(struct net_device *, u16);
157static int igb_vlan_rx_kill_vid(struct net_device *, u16);
9d5c8243 158static void igb_restore_vlan(struct igb_adapter *);
26ad9178 159static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 160static void igb_ping_all_vfs(struct igb_adapter *);
161static void igb_msg_task(struct igb_adapter *);
4ae196df 162static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 163static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 164static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 165static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
166static int igb_ndo_set_vf_vlan(struct net_device *netdev,
167 int vf, u16 vlan, u8 qos);
168static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
169static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
170 struct ifla_vf_info *ivi);
17dc566c 171static void igb_check_vf_rate_limit(struct igb_adapter *);
46a01698
RL		 172
173#ifdef CONFIG_PCI_IOV
0224d663
GR		 174static int igb_vf_configure(struct igb_adapter *adapter, int vf);
175static int igb_find_enabled_vfs(struct igb_adapter *adapter);
176static int igb_check_vf_assignment(struct igb_adapter *adapter);
46a01698 177#endif
9d5c8243 178
9d5c8243 179#ifdef CONFIG_PM
d9dd966d 180#ifdef CONFIG_PM_SLEEP
749ab2cd 181static int igb_suspend(struct device *);
d9dd966d 182#endif
749ab2cd
YZ		 183static int igb_resume(struct device *);
184#ifdef CONFIG_PM_RUNTIME
185static int igb_runtime_suspend(struct device *dev);
186static int igb_runtime_resume(struct device *dev);
187static int igb_runtime_idle(struct device *dev);
188#endif
189static const struct dev_pm_ops igb_pm_ops = {
190 SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
191 SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
192 igb_runtime_idle)
193};
9d5c8243
AK		 194#endif
195static void igb_shutdown(struct pci_dev *);
421e02f0 196#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 197static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
198static struct notifier_block dca_notifier = {
199 .notifier_call = igb_notify_dca,
200 .next = NULL,
201 .priority = 0
202};
203#endif
9d5c8243
AK		 204#ifdef CONFIG_NET_POLL_CONTROLLER
205/* for netdump / net console */
206static void igb_netpoll(struct net_device *);
207#endif
37680117 208#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 209static unsigned int max_vfs = 0;
210module_param(max_vfs, uint, 0);
211MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
212 "per physical function");
213#endif /* CONFIG_PCI_IOV */
214
9d5c8243
AK		 215static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
216 pci_channel_state_t);
217static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
218static void igb_io_resume(struct pci_dev *);
219
220static struct pci_error_handlers igb_err_handler = {
221 .error_detected = igb_io_error_detected,
222 .slot_reset = igb_io_slot_reset,
223 .resume = igb_io_resume,
224};
225
b6e0c419 226static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
9d5c8243
AK		 227
228static struct pci_driver igb_driver = {
229 .name = igb_driver_name,
230 .id_table = igb_pci_tbl,
231 .probe = igb_probe,
232 .remove = __devexit_p(igb_remove),
233#ifdef CONFIG_PM
749ab2cd 234 .driver.pm = &igb_pm_ops,
9d5c8243
AK		 235#endif
236 .shutdown = igb_shutdown,
237 .err_handler = &igb_err_handler
238};
239
240MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
241MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
242MODULE_LICENSE("GPL");
243MODULE_VERSION(DRV_VERSION);
244
b3f4d599 245#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
246static int debug = -1;
247module_param(debug, int, 0);
248MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
249
c97ec42a
TI		 250struct igb_reg_info {
251 u32 ofs;
252 char *name;
253};
254
255static const struct igb_reg_info igb_reg_info_tbl[] = {
256
257 /* General Registers */
258 {E1000_CTRL, "CTRL"},
259 {E1000_STATUS, "STATUS"},
260 {E1000_CTRL_EXT, "CTRL_EXT"},
261
262 /* Interrupt Registers */
263 {E1000_ICR, "ICR"},
264
265 /* RX Registers */
266 {E1000_RCTL, "RCTL"},
267 {E1000_RDLEN(0), "RDLEN"},
268 {E1000_RDH(0), "RDH"},
269 {E1000_RDT(0), "RDT"},
270 {E1000_RXDCTL(0), "RXDCTL"},
271 {E1000_RDBAL(0), "RDBAL"},
272 {E1000_RDBAH(0), "RDBAH"},
273
274 /* TX Registers */
275 {E1000_TCTL, "TCTL"},
276 {E1000_TDBAL(0), "TDBAL"},
277 {E1000_TDBAH(0), "TDBAH"},
278 {E1000_TDLEN(0), "TDLEN"},
279 {E1000_TDH(0), "TDH"},
280 {E1000_TDT(0), "TDT"},
281 {E1000_TXDCTL(0), "TXDCTL"},
282 {E1000_TDFH, "TDFH"},
283 {E1000_TDFT, "TDFT"},
284 {E1000_TDFHS, "TDFHS"},
285 {E1000_TDFPC, "TDFPC"},
286
287 /* List Terminator */
288 {}
289};
290
291/*
292 * igb_regdump - register printout routine
293 */
294static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
295{
296 int n = 0;
297 char rname[16];
298 u32 regs[8];
299
300 switch (reginfo->ofs) {
301 case E1000_RDLEN(0):
302 for (n = 0; n < 4; n++)
303 regs[n] = rd32(E1000_RDLEN(n));
304 break;
305 case E1000_RDH(0):
306 for (n = 0; n < 4; n++)
307 regs[n] = rd32(E1000_RDH(n));
308 break;
309 case E1000_RDT(0):
310 for (n = 0; n < 4; n++)
311 regs[n] = rd32(E1000_RDT(n));
312 break;
313 case E1000_RXDCTL(0):
314 for (n = 0; n < 4; n++)
315 regs[n] = rd32(E1000_RXDCTL(n));
316 break;
317 case E1000_RDBAL(0):
318 for (n = 0; n < 4; n++)
319 regs[n] = rd32(E1000_RDBAL(n));
320 break;
321 case E1000_RDBAH(0):
322 for (n = 0; n < 4; n++)
323 regs[n] = rd32(E1000_RDBAH(n));
324 break;
325 case E1000_TDBAL(0):
326 for (n = 0; n < 4; n++)
327 regs[n] = rd32(E1000_RDBAL(n));
328 break;
329 case E1000_TDBAH(0):
330 for (n = 0; n < 4; n++)
331 regs[n] = rd32(E1000_TDBAH(n));
332 break;
333 case E1000_TDLEN(0):
334 for (n = 0; n < 4; n++)
335 regs[n] = rd32(E1000_TDLEN(n));
336 break;
337 case E1000_TDH(0):
338 for (n = 0; n < 4; n++)
339 regs[n] = rd32(E1000_TDH(n));
340 break;
341 case E1000_TDT(0):
342 for (n = 0; n < 4; n++)
343 regs[n] = rd32(E1000_TDT(n));
344 break;
345 case E1000_TXDCTL(0):
346 for (n = 0; n < 4; n++)
347 regs[n] = rd32(E1000_TXDCTL(n));
348 break;
349 default:
876d2d6f 350 pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs));
c97ec42a
TI		 351 return;
352 }
353
354 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
876d2d6f
JK		 355 pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1],
356 regs[2], regs[3]);
c97ec42a
TI		 357}
358
359/*
360 * igb_dump - Print registers, tx-rings and rx-rings
361 */
362static void igb_dump(struct igb_adapter *adapter)
363{
364 struct net_device *netdev = adapter->netdev;
365 struct e1000_hw *hw = &adapter->hw;
366 struct igb_reg_info *reginfo;
c97ec42a
TI		 367 struct igb_ring *tx_ring;
368 union e1000_adv_tx_desc *tx_desc;
369 struct my_u0 { u64 a; u64 b; } *u0;
c97ec42a
TI		 370 struct igb_ring *rx_ring;
371 union e1000_adv_rx_desc *rx_desc;
372 u32 staterr;
6ad4edfc 373 u16 i, n;
c97ec42a
TI		 374
375 if (!netif_msg_hw(adapter))
376 return;
377
378 /* Print netdevice Info */
379 if (netdev) {
380 dev_info(&adapter->pdev->dev, "Net device Info\n");
876d2d6f
JK		 381 pr_info("Device Name state trans_start "
382 "last_rx\n");
383 pr_info("%-15s %016lX %016lX %016lX\n", netdev->name,
384 netdev->state, netdev->trans_start, netdev->last_rx);
c97ec42a
TI		 385 }
386
387 /* Print Registers */
388 dev_info(&adapter->pdev->dev, "Register Dump\n");
876d2d6f 389 pr_info(" Register Name Value\n");
c97ec42a
TI		 390 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
391 reginfo->name; reginfo++) {
392 igb_regdump(hw, reginfo);
393 }
394
395 /* Print TX Ring Summary */
396 if (!netdev || !netif_running(netdev))
397 goto exit;
398
399 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
876d2d6f 400 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
c97ec42a 401 for (n = 0; n < adapter->num_tx_queues; n++) {
06034649 402 struct igb_tx_buffer *buffer_info;
c97ec42a 403 tx_ring = adapter->tx_ring[n];
06034649 404 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
876d2d6f
JK		 405 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
406 n, tx_ring->next_to_use, tx_ring->next_to_clean,
407 (u64)buffer_info->dma,
408 buffer_info->length,
409 buffer_info->next_to_watch,
410 (u64)buffer_info->time_stamp);
c97ec42a
TI		 411 }
412
413 /* Print TX Rings */
414 if (!netif_msg_tx_done(adapter))
415 goto rx_ring_summary;
416
417 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
418
419 /* Transmit Descriptor Formats
420 *
421 * Advanced Transmit Descriptor
422 * +--------------------------------------------------------------+
423 * 0 | Buffer Address [63:0] |
424 * +--------------------------------------------------------------+
425 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
426 * +--------------------------------------------------------------+
427 * 63 46 45 40 39 38 36 35 32 31 24 15 0
428 */
429
430 for (n = 0; n < adapter->num_tx_queues; n++) {
431 tx_ring = adapter->tx_ring[n];
876d2d6f
JK		 432 pr_info("------------------------------------\n");
433 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
434 pr_info("------------------------------------\n");
435 pr_info("T [desc] [address 63:0 ] [PlPOCIStDDM Ln] "
436 "[bi->dma ] leng ntw timestamp "
437 "bi->skb\n");
c97ec42a
TI		 438
439 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
876d2d6f 440 const char *next_desc;
06034649 441 struct igb_tx_buffer *buffer_info;
60136906 442 tx_desc = IGB_TX_DESC(tx_ring, i);
06034649 443 buffer_info = &tx_ring->tx_buffer_info[i];
c97ec42a 444 u0 = (struct my_u0 *)tx_desc;
876d2d6f
JK		 445 if (i == tx_ring->next_to_use &&
446 i == tx_ring->next_to_clean)
447 next_desc = " NTC/U";
448 else if (i == tx_ring->next_to_use)
449 next_desc = " NTU";
450 else if (i == tx_ring->next_to_clean)
451 next_desc = " NTC";
452 else
453 next_desc = "";
454
455 pr_info("T [0x%03X] %016llX %016llX %016llX"
456 " %04X %p %016llX %p%s\n", i,
c97ec42a
TI		 457 le64_to_cpu(u0->a),
458 le64_to_cpu(u0->b),
459 (u64)buffer_info->dma,
460 buffer_info->length,
461 buffer_info->next_to_watch,
462 (u64)buffer_info->time_stamp,
876d2d6f 463 buffer_info->skb, next_desc);
c97ec42a
TI		 464
465 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
466 print_hex_dump(KERN_INFO, "",
467 DUMP_PREFIX_ADDRESS,
468 16, 1, phys_to_virt(buffer_info->dma),
469 buffer_info->length, true);
470 }
471 }
472
473 /* Print RX Rings Summary */
474rx_ring_summary:
475 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
876d2d6f 476 pr_info("Queue [NTU] [NTC]\n");
c97ec42a
TI		 477 for (n = 0; n < adapter->num_rx_queues; n++) {
478 rx_ring = adapter->rx_ring[n];
876d2d6f
JK		 479 pr_info(" %5d %5X %5X\n",
480 n, rx_ring->next_to_use, rx_ring->next_to_clean);
c97ec42a
TI		 481 }
482
483 /* Print RX Rings */
484 if (!netif_msg_rx_status(adapter))
485 goto exit;
486
487 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
488
489 /* Advanced Receive Descriptor (Read) Format
490 * 63 1 0
491 * +-----------------------------------------------------+
492 * 0 | Packet Buffer Address [63:1] |A0/NSE|
493 * +----------------------------------------------+------+
494 * 8 | Header Buffer Address [63:1] | DD |
495 * +-----------------------------------------------------+
496 *
497 *
498 * Advanced Receive Descriptor (Write-Back) Format
499 *
500 * 63 48 47 32 31 30 21 20 17 16 4 3 0
501 * +------------------------------------------------------+
502 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
503 * | Checksum Ident | | | | Type | Type |
504 * +------------------------------------------------------+
505 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
506 * +------------------------------------------------------+
507 * 63 48 47 32 31 20 19 0
508 */
509
510 for (n = 0; n < adapter->num_rx_queues; n++) {
511 rx_ring = adapter->rx_ring[n];
876d2d6f
JK		 512 pr_info("------------------------------------\n");
513 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
514 pr_info("------------------------------------\n");
515 pr_info("R [desc] [ PktBuf A0] [ HeadBuf DD] "
516 "[bi->dma ] [bi->skb] <-- Adv Rx Read format\n");
517 pr_info("RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] -----"
518 "----------- [bi->skb] <-- Adv Rx Write-Back format\n");
c97ec42a
TI		 519
520 for (i = 0; i < rx_ring->count; i++) {
876d2d6f 521 const char *next_desc;
06034649
AD		 522 struct igb_rx_buffer *buffer_info;
523 buffer_info = &rx_ring->rx_buffer_info[i];
60136906 524 rx_desc = IGB_RX_DESC(rx_ring, i);
c97ec42a
TI		 525 u0 = (struct my_u0 *)rx_desc;
526 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
876d2d6f
JK		 527
528 if (i == rx_ring->next_to_use)
529 next_desc = " NTU";
530 else if (i == rx_ring->next_to_clean)
531 next_desc = " NTC";
532 else
533 next_desc = "";
534
c97ec42a
TI		 535 if (staterr & E1000_RXD_STAT_DD) {
536 /* Descriptor Done */
876d2d6f
JK		 537 pr_info("%s[0x%03X] %016llX %016llX -------"
538 "--------- %p%s\n", "RWB", i,
c97ec42a
TI		 539 le64_to_cpu(u0->a),
540 le64_to_cpu(u0->b),
876d2d6f 541 buffer_info->skb, next_desc);
c97ec42a 542 } else {
876d2d6f
JK		 543 pr_info("%s[0x%03X] %016llX %016llX %016llX"
544 " %p%s\n", "R ", i,
c97ec42a
TI		 545 le64_to_cpu(u0->a),
546 le64_to_cpu(u0->b),
547 (u64)buffer_info->dma,
876d2d6f 548 buffer_info->skb, next_desc);
c97ec42a
TI		 549
550 if (netif_msg_pktdata(adapter)) {
551 print_hex_dump(KERN_INFO, "",
552 DUMP_PREFIX_ADDRESS,
553 16, 1,
554 phys_to_virt(buffer_info->dma),
44390ca6
AD		 555 IGB_RX_HDR_LEN, true);
556 print_hex_dump(KERN_INFO, "",
557 DUMP_PREFIX_ADDRESS,
558 16, 1,
559 phys_to_virt(
560 buffer_info->page_dma +
561 buffer_info->page_offset),
562 PAGE_SIZE/2, true);
c97ec42a
TI		 563 }
564 }
c97ec42a
TI		 565 }
566 }
567
568exit:
569 return;
570}
571
9d5c8243 572/**
c041076a 573 * igb_get_hw_dev - return device
9d5c8243
AK		 574 * used by hardware layer to print debugging information
575 **/
c041076a 576struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
9d5c8243
AK		 577{
578 struct igb_adapter *adapter = hw->back;
c041076a 579 return adapter->netdev;
9d5c8243 580}
38c845c7 581
9d5c8243
AK		 582/**
583 * igb_init_module - Driver Registration Routine
584 *
585 * igb_init_module is the first routine called when the driver is
586 * loaded. All it does is register with the PCI subsystem.
587 **/
588static int __init igb_init_module(void)
589{
590 int ret;
876d2d6f 591 pr_info("%s - version %s\n",
9d5c8243
AK		 592 igb_driver_string, igb_driver_version);
593
876d2d6f 594 pr_info("%s\n", igb_copyright);
9d5c8243 595
421e02f0 596#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 597 dca_register_notify(&dca_notifier);
598#endif
bbd98fe4 599 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 600 return ret;
601}
602
603module_init(igb_init_module);
604
605/**
606 * igb_exit_module - Driver Exit Cleanup Routine
607 *
608 * igb_exit_module is called just before the driver is removed
609 * from memory.
610 **/
611static void __exit igb_exit_module(void)
612{
421e02f0 613#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 614 dca_unregister_notify(&dca_notifier);
615#endif
9d5c8243
AK		 616 pci_unregister_driver(&igb_driver);
617}
618
619module_exit(igb_exit_module);
620
26bc19ec
AD		 621#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
622/**
623 * igb_cache_ring_register - Descriptor ring to register mapping
624 * @adapter: board private structure to initialize
625 *
626 * Once we know the feature-set enabled for the device, we'll cache
627 * the register offset the descriptor ring is assigned to.
628 **/
629static void igb_cache_ring_register(struct igb_adapter *adapter)
630{
ee1b9f06 631 int i = 0, j = 0;
047e0030 632 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 633
634 switch (adapter->hw.mac.type) {
635 case e1000_82576:
636 /* The queues are allocated for virtualization such that VF 0
637 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
638 * In order to avoid collision we start at the first free queue
639 * and continue consuming queues in the same sequence
640 */
ee1b9f06 641 if (adapter->vfs_allocated_count) {
a99955fc 642 for (; i < adapter->rss_queues; i++)
3025a446
AD		 643 adapter->rx_ring[i]->reg_idx = rbase_offset +
644 Q_IDX_82576(i);
ee1b9f06 645 }
26bc19ec 646 case e1000_82575:
55cac248 647 case e1000_82580:
d2ba2ed8 648 case e1000_i350:
f96a8a0b
CW		 649 case e1000_i210:
650 case e1000_i211:
26bc19ec 651 default:
ee1b9f06 652 for (; i < adapter->num_rx_queues; i++)
3025a446 653 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 654 for (; j < adapter->num_tx_queues; j++)
3025a446 655 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 656 break;
657 }
658}
659
047e0030
AD		 660static void igb_free_queues(struct igb_adapter *adapter)
661{
3025a446 662 int i;
047e0030 663
3025a446
AD		 664 for (i = 0; i < adapter->num_tx_queues; i++) {
665 kfree(adapter->tx_ring[i]);
666 adapter->tx_ring[i] = NULL;
667 }
668 for (i = 0; i < adapter->num_rx_queues; i++) {
669 kfree(adapter->rx_ring[i]);
670 adapter->rx_ring[i] = NULL;
671 }
047e0030
AD		 672 adapter->num_rx_queues = 0;
673 adapter->num_tx_queues = 0;
674}
675
9d5c8243
AK		 676/**
677 * igb_alloc_queues - Allocate memory for all rings
678 * @adapter: board private structure to initialize
679 *
680 * We allocate one ring per queue at run-time since we don't know the
681 * number of queues at compile-time.
682 **/
683static int igb_alloc_queues(struct igb_adapter *adapter)
684{
3025a446 685 struct igb_ring *ring;
9d5c8243 686 int i;
81c2fc22 687 int orig_node = adapter->node;
9d5c8243 688
661086df 689 for (i = 0; i < adapter->num_tx_queues; i++) {
81c2fc22
AD		 690 if (orig_node == -1) {
691 int cur_node = next_online_node(adapter->node);
692 if (cur_node == MAX_NUMNODES)
693 cur_node = first_online_node;
694 adapter->node = cur_node;
695 }
696 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
697 adapter->node);
698 if (!ring)
699 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
3025a446
AD		 700 if (!ring)
701 goto err;
68fd9910 702 ring->count = adapter->tx_ring_count;
661086df 703 ring->queue_index = i;
59d71989 704 ring->dev = &adapter->pdev->dev;
e694e964 705 ring->netdev = adapter->netdev;
81c2fc22 706 ring->numa_node = adapter->node;
85ad76b2
AD		 707 /* For 82575, context index must be unique per ring. */
708 if (adapter->hw.mac.type == e1000_82575)
866cff06 709 set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
3025a446 710 adapter->tx_ring[i] = ring;
661086df 711 }
81c2fc22
AD		 712 /* Restore the adapter's original node */
713 adapter->node = orig_node;
85ad76b2 714
9d5c8243 715 for (i = 0; i < adapter->num_rx_queues; i++) {
81c2fc22
AD		 716 if (orig_node == -1) {
717 int cur_node = next_online_node(adapter->node);
718 if (cur_node == MAX_NUMNODES)
719 cur_node = first_online_node;
720 adapter->node = cur_node;
721 }
722 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
723 adapter->node);
724 if (!ring)
725 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
3025a446
AD		 726 if (!ring)
727 goto err;
68fd9910 728 ring->count = adapter->rx_ring_count;
844290e5 729 ring->queue_index = i;
59d71989 730 ring->dev = &adapter->pdev->dev;
e694e964 731 ring->netdev = adapter->netdev;
81c2fc22 732 ring->numa_node = adapter->node;
85ad76b2
AD		 733 /* set flag indicating ring supports SCTP checksum offload */
734 if (adapter->hw.mac.type >= e1000_82576)
866cff06 735 set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
8be10e91 736
f96a8a0b
CW		 737 /*
738 * On i350, i210, and i211, loopback VLAN packets
739 * have the tag byte-swapped.
740 * */
741 if (adapter->hw.mac.type >= e1000_i350)
8be10e91
AD		 742 set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
743
3025a446 744 adapter->rx_ring[i] = ring;
9d5c8243 745 }
81c2fc22
AD		 746 /* Restore the adapter's original node */
747 adapter->node = orig_node;
26bc19ec
AD		 748
749 igb_cache_ring_register(adapter);
9d5c8243 750
047e0030 751 return 0;
a88f10ec 752
047e0030 753err:
81c2fc22
AD		 754 /* Restore the adapter's original node */
755 adapter->node = orig_node;
047e0030 756 igb_free_queues(adapter);
d1a8c9e1 757
047e0030 758 return -ENOMEM;
a88f10ec
AD		 759}
760
4be000c8
AD		 761/**
762 * igb_write_ivar - configure ivar for given MSI-X vector
763 * @hw: pointer to the HW structure
764 * @msix_vector: vector number we are allocating to a given ring
765 * @index: row index of IVAR register to write within IVAR table
766 * @offset: column offset of in IVAR, should be multiple of 8
767 *
768 * This function is intended to handle the writing of the IVAR register
769 * for adapters 82576 and newer. The IVAR table consists of 2 columns,
770 * each containing an cause allocation for an Rx and Tx ring, and a
771 * variable number of rows depending on the number of queues supported.
772 **/
773static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
774 int index, int offset)
775{
776 u32 ivar = array_rd32(E1000_IVAR0, index);
777
778 /* clear any bits that are currently set */
779 ivar &= ~((u32)0xFF << offset);
780
781 /* write vector and valid bit */
782 ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
783
784 array_wr32(E1000_IVAR0, index, ivar);
785}
786
9d5c8243 787#define IGB_N0_QUEUE -1
047e0030 788static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243 789{
047e0030 790 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 791 struct e1000_hw *hw = &adapter->hw;
047e0030
AD		 792 int rx_queue = IGB_N0_QUEUE;
793 int tx_queue = IGB_N0_QUEUE;
4be000c8 794 u32 msixbm = 0;
047e0030 795
0ba82994
AD		 796 if (q_vector->rx.ring)
797 rx_queue = q_vector->rx.ring->reg_idx;
798 if (q_vector->tx.ring)
799 tx_queue = q_vector->tx.ring->reg_idx;
2d064c06
AD		 800
801 switch (hw->mac.type) {
802 case e1000_82575:
9d5c8243
AK		 803 /* The 82575 assigns vectors using a bitmask, which matches the
804 bitmask for the EICR/EIMS/EIMC registers. To assign one
805 or more queues to a vector, we write the appropriate bits
806 into the MSIXBM register for that vector. */
047e0030 807 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 808 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 809 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 810 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 811 if (!adapter->msix_entries && msix_vector == 0)
812 msixbm |= E1000_EIMS_OTHER;
9d5c8243 813 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 814 q_vector->eims_value = msixbm;
2d064c06
AD		 815 break;
816 case e1000_82576:
4be000c8
AD		 817 /*
818 * 82576 uses a table that essentially consists of 2 columns
819 * with 8 rows. The ordering is column-major so we use the
820 * lower 3 bits as the row index, and the 4th bit as the
821 * column offset.
822 */
823 if (rx_queue > IGB_N0_QUEUE)
824 igb_write_ivar(hw, msix_vector,
825 rx_queue & 0x7,
826 (rx_queue & 0x8) << 1);
827 if (tx_queue > IGB_N0_QUEUE)
828 igb_write_ivar(hw, msix_vector,
829 tx_queue & 0x7,
830 ((tx_queue & 0x8) << 1) + 8);
047e0030 831 q_vector->eims_value = 1 << msix_vector;
2d064c06 832 break;
55cac248 833 case e1000_82580:
d2ba2ed8 834 case e1000_i350:
f96a8a0b
CW		 835 case e1000_i210:
836 case e1000_i211:
4be000c8
AD		 837 /*
838 * On 82580 and newer adapters the scheme is similar to 82576
839 * however instead of ordering column-major we have things
840 * ordered row-major. So we traverse the table by using
841 * bit 0 as the column offset, and the remaining bits as the
842 * row index.
843 */
844 if (rx_queue > IGB_N0_QUEUE)
845 igb_write_ivar(hw, msix_vector,
846 rx_queue >> 1,
847 (rx_queue & 0x1) << 4);
848 if (tx_queue > IGB_N0_QUEUE)
849 igb_write_ivar(hw, msix_vector,
850 tx_queue >> 1,
851 ((tx_queue & 0x1) << 4) + 8);
55cac248
AD		 852 q_vector->eims_value = 1 << msix_vector;
853 break;
2d064c06
AD		 854 default:
855 BUG();
856 break;
857 }
26b39276
AD		 858
859 /* add q_vector eims value to global eims_enable_mask */
860 adapter->eims_enable_mask |= q_vector->eims_value;
861
862 /* configure q_vector to set itr on first interrupt */
863 q_vector->set_itr = 1;
9d5c8243
AK		 864}
865
866/**
867 * igb_configure_msix - Configure MSI-X hardware
868 *
869 * igb_configure_msix sets up the hardware to properly
870 * generate MSI-X interrupts.
871 **/
872static void igb_configure_msix(struct igb_adapter *adapter)
873{
874 u32 tmp;
875 int i, vector = 0;
876 struct e1000_hw *hw = &adapter->hw;
877
878 adapter->eims_enable_mask = 0;
9d5c8243
AK		 879
880 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 881 switch (hw->mac.type) {
882 case e1000_82575:
9d5c8243
AK		 883 tmp = rd32(E1000_CTRL_EXT);
884 /* enable MSI-X PBA support*/
885 tmp |= E1000_CTRL_EXT_PBA_CLR;
886
887 /* Auto-Mask interrupts upon ICR read. */
888 tmp |= E1000_CTRL_EXT_EIAME;
889 tmp |= E1000_CTRL_EXT_IRCA;
890
891 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 892
893 /* enable msix_other interrupt */
894 array_wr32(E1000_MSIXBM(0), vector++,
895 E1000_EIMS_OTHER);
844290e5 896 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 897
2d064c06
AD		 898 break;
899
900 case e1000_82576:
55cac248 901 case e1000_82580:
d2ba2ed8 902 case e1000_i350:
f96a8a0b
CW		 903 case e1000_i210:
904 case e1000_i211:
047e0030
AD		 905 /* Turn on MSI-X capability first, or our settings
906 * won't stick. And it will take days to debug. */
907 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
908 E1000_GPIE_PBA | E1000_GPIE_EIAME |
909 E1000_GPIE_NSICR);
910
911 /* enable msix_other interrupt */
912 adapter->eims_other = 1 << vector;
2d064c06 913 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 914
047e0030 915 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 916 break;
917 default:
918 /* do nothing, since nothing else supports MSI-X */
919 break;
920 } /* switch (hw->mac.type) */
047e0030
AD		 921
922 adapter->eims_enable_mask |= adapter->eims_other;
923
26b39276
AD		 924 for (i = 0; i < adapter->num_q_vectors; i++)
925 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 926
9d5c8243
AK		 927 wrfl();
928}
929
930/**
931 * igb_request_msix - Initialize MSI-X interrupts
932 *
933 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
934 * kernel.
935 **/
936static int igb_request_msix(struct igb_adapter *adapter)
937{
938 struct net_device *netdev = adapter->netdev;
047e0030 939 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 940 int i, err = 0, vector = 0;
941
047e0030 942 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 943 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 944 if (err)
945 goto out;
946 vector++;
947
948 for (i = 0; i < adapter->num_q_vectors; i++) {
949 struct igb_q_vector *q_vector = adapter->q_vector[i];
950
951 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
952
0ba82994 953 if (q_vector->rx.ring && q_vector->tx.ring)
047e0030 954 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
0ba82994
AD		 955 q_vector->rx.ring->queue_index);
956 else if (q_vector->tx.ring)
047e0030 957 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
0ba82994
AD		 958 q_vector->tx.ring->queue_index);
959 else if (q_vector->rx.ring)
047e0030 960 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
0ba82994 961 q_vector->rx.ring->queue_index);
9d5c8243 962 else
047e0030
AD		 963 sprintf(q_vector->name, "%s-unused", netdev->name);
964
9d5c8243 965 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 966 igb_msix_ring, 0, q_vector->name,
047e0030 967 q_vector);
9d5c8243
AK		 968 if (err)
969 goto out;
9d5c8243
AK		 970 vector++;
971 }
972
9d5c8243
AK		 973 igb_configure_msix(adapter);
974 return 0;
975out:
976 return err;
977}
978
979static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
980{
981 if (adapter->msix_entries) {
982 pci_disable_msix(adapter->pdev);
983 kfree(adapter->msix_entries);
984 adapter->msix_entries = NULL;
047e0030 985 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 986 pci_disable_msi(adapter->pdev);
047e0030 987 }
9d5c8243
AK		 988}
989
047e0030
AD		 990/**
991 * igb_free_q_vectors - Free memory allocated for interrupt vectors
992 * @adapter: board private structure to initialize
993 *
994 * This function frees the memory allocated to the q_vectors. In addition if
995 * NAPI is enabled it will delete any references to the NAPI struct prior
996 * to freeing the q_vector.
997 **/
998static void igb_free_q_vectors(struct igb_adapter *adapter)
999{
1000 int v_idx;
1001
1002 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1003 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1004 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 1005 if (!q_vector)
1006 continue;
047e0030
AD		 1007 netif_napi_del(&q_vector->napi);
1008 kfree(q_vector);
1009 }
1010 adapter->num_q_vectors = 0;
1011}
1012
1013/**
1014 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
1015 *
1016 * This function resets the device so that it has 0 rx queues, tx queues, and
1017 * MSI-X interrupts allocated.
1018 */
1019static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
1020{
1021 igb_free_queues(adapter);
1022 igb_free_q_vectors(adapter);
1023 igb_reset_interrupt_capability(adapter);
1024}
9d5c8243
AK		 1025
1026/**
1027 * igb_set_interrupt_capability - set MSI or MSI-X if supported
1028 *
1029 * Attempt to configure interrupts using the best available
1030 * capabilities of the hardware and kernel.
1031 **/
21adef3e 1032static int igb_set_interrupt_capability(struct igb_adapter *adapter)
9d5c8243
AK		 1033{
1034 int err;
1035 int numvecs, i;
1036
83b7180d 1037 /* Number of supported queues. */
a99955fc 1038 adapter->num_rx_queues = adapter->rss_queues;
5fa8517f
GR		 1039 if (adapter->vfs_allocated_count)
1040 adapter->num_tx_queues = 1;
1041 else
1042 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 1043
047e0030
AD		 1044 /* start with one vector for every rx queue */
1045 numvecs = adapter->num_rx_queues;
1046
3ad2f3fb 1047 /* if tx handler is separate add 1 for every tx queue */
a99955fc
AD		 1048 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1049 numvecs += adapter->num_tx_queues;
047e0030
AD		 1050
1051 /* store the number of vectors reserved for queues */
1052 adapter->num_q_vectors = numvecs;
1053
1054 /* add 1 vector for link status interrupts */
1055 numvecs++;
9d5c8243
AK		 1056 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1057 GFP_KERNEL);
f96a8a0b 1058
9d5c8243
AK		 1059 if (!adapter->msix_entries)
1060 goto msi_only;
1061
1062 for (i = 0; i < numvecs; i++)
1063 adapter->msix_entries[i].entry = i;
1064
1065 err = pci_enable_msix(adapter->pdev,
1066 adapter->msix_entries,
1067 numvecs);
1068 if (err == 0)
34a20e89 1069 goto out;
9d5c8243
AK		 1070
1071 igb_reset_interrupt_capability(adapter);
1072
1073 /* If we can't do MSI-X, try MSI */
1074msi_only:
2a3abf6d
AD		 1075#ifdef CONFIG_PCI_IOV
1076 /* disable SR-IOV for non MSI-X configurations */
1077 if (adapter->vf_data) {
1078 struct e1000_hw *hw = &adapter->hw;
1079 /* disable iov and allow time for transactions to clear */
1080 pci_disable_sriov(adapter->pdev);
1081 msleep(500);
1082
1083 kfree(adapter->vf_data);
1084 adapter->vf_data = NULL;
1085 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 1086 wrfl();
2a3abf6d
AD		 1087 msleep(100);
1088 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1089 }
1090#endif
4fc82adf 1091 adapter->vfs_allocated_count = 0;
a99955fc 1092 adapter->rss_queues = 1;
4fc82adf 1093 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 1094 adapter->num_rx_queues = 1;
661086df 1095 adapter->num_tx_queues = 1;
047e0030 1096 adapter->num_q_vectors = 1;
9d5c8243 1097 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1098 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 1099out:
21adef3e 1100 /* Notify the stack of the (possibly) reduced queue counts. */
cfb8c3aa 1101 rtnl_lock();
21adef3e 1102 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
cfb8c3aa
BP		 1103 err = netif_set_real_num_rx_queues(adapter->netdev,
1104 adapter->num_rx_queues);
1105 rtnl_unlock();
1106 return err;
9d5c8243
AK		 1107}
1108
047e0030
AD		 1109/**
1110 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1111 * @adapter: board private structure to initialize
1112 *
1113 * We allocate one q_vector per queue interrupt. If allocation fails we
1114 * return -ENOMEM.
1115 **/
1116static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1117{
1118 struct igb_q_vector *q_vector;
1119 struct e1000_hw *hw = &adapter->hw;
1120 int v_idx;
81c2fc22 1121 int orig_node = adapter->node;
047e0030
AD		 1122
1123 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
81c2fc22
AD		 1124 if ((adapter->num_q_vectors == (adapter->num_rx_queues +
1125 adapter->num_tx_queues)) &&
1126 (adapter->num_rx_queues == v_idx))
1127 adapter->node = orig_node;
1128 if (orig_node == -1) {
1129 int cur_node = next_online_node(adapter->node);
1130 if (cur_node == MAX_NUMNODES)
1131 cur_node = first_online_node;
1132 adapter->node = cur_node;
1133 }
1134 q_vector = kzalloc_node(sizeof(struct igb_q_vector), GFP_KERNEL,
1135 adapter->node);
1136 if (!q_vector)
1137 q_vector = kzalloc(sizeof(struct igb_q_vector),
1138 GFP_KERNEL);
047e0030
AD		 1139 if (!q_vector)
1140 goto err_out;
1141 q_vector->adapter = adapter;
047e0030
AD		 1142 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1143 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1144 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1145 adapter->q_vector[v_idx] = q_vector;
1146 }
81c2fc22
AD		 1147 /* Restore the adapter's original node */
1148 adapter->node = orig_node;
1149
047e0030
AD		 1150 return 0;
1151
1152err_out:
81c2fc22
AD		 1153 /* Restore the adapter's original node */
1154 adapter->node = orig_node;
fe0592b4 1155 igb_free_q_vectors(adapter);
047e0030
AD		 1156 return -ENOMEM;
1157}
1158
1159static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1160 int ring_idx, int v_idx)
1161{
3025a446 1162 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1163
0ba82994
AD		 1164 q_vector->rx.ring = adapter->rx_ring[ring_idx];
1165 q_vector->rx.ring->q_vector = q_vector;
1166 q_vector->rx.count++;
4fc82adf
AD		 1167 q_vector->itr_val = adapter->rx_itr_setting;
1168 if (q_vector->itr_val && q_vector->itr_val <= 3)
1169 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1170}
1171
1172static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1173 int ring_idx, int v_idx)
1174{
3025a446 1175 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1176
0ba82994
AD		 1177 q_vector->tx.ring = adapter->tx_ring[ring_idx];
1178 q_vector->tx.ring->q_vector = q_vector;
1179 q_vector->tx.count++;
4fc82adf 1180 q_vector->itr_val = adapter->tx_itr_setting;
0ba82994 1181 q_vector->tx.work_limit = adapter->tx_work_limit;
4fc82adf
AD		 1182 if (q_vector->itr_val && q_vector->itr_val <= 3)
1183 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1184}
1185
1186/**
1187 * igb_map_ring_to_vector - maps allocated queues to vectors
1188 *
1189 * This function maps the recently allocated queues to vectors.
1190 **/
1191static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1192{
1193 int i;
1194 int v_idx = 0;
1195
1196 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1197 (adapter->num_q_vectors < adapter->num_tx_queues))
1198 return -ENOMEM;
1199
1200 if (adapter->num_q_vectors >=
1201 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1202 for (i = 0; i < adapter->num_rx_queues; i++)
1203 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1204 for (i = 0; i < adapter->num_tx_queues; i++)
1205 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1206 } else {
1207 for (i = 0; i < adapter->num_rx_queues; i++) {
1208 if (i < adapter->num_tx_queues)
1209 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1210 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1211 }
1212 for (; i < adapter->num_tx_queues; i++)
1213 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1214 }
1215 return 0;
1216}
1217
1218/**
1219 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1220 *
1221 * This function initializes the interrupts and allocates all of the queues.
1222 **/
1223static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1224{
1225 struct pci_dev *pdev = adapter->pdev;
1226 int err;
1227
21adef3e
BH		 1228 err = igb_set_interrupt_capability(adapter);
1229 if (err)
1230 return err;
047e0030
AD		 1231
1232 err = igb_alloc_q_vectors(adapter);
1233 if (err) {
1234 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1235 goto err_alloc_q_vectors;
1236 }
1237
1238 err = igb_alloc_queues(adapter);
1239 if (err) {
1240 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1241 goto err_alloc_queues;
1242 }
1243
1244 err = igb_map_ring_to_vector(adapter);
1245 if (err) {
1246 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1247 goto err_map_queues;
1248 }
1249
1250
1251 return 0;
1252err_map_queues:
1253 igb_free_queues(adapter);
1254err_alloc_queues:
1255 igb_free_q_vectors(adapter);
1256err_alloc_q_vectors:
1257 igb_reset_interrupt_capability(adapter);
1258 return err;
1259}
1260
9d5c8243
AK		 1261/**
1262 * igb_request_irq - initialize interrupts
1263 *
1264 * Attempts to configure interrupts using the best available
1265 * capabilities of the hardware and kernel.
1266 **/
1267static int igb_request_irq(struct igb_adapter *adapter)
1268{
1269 struct net_device *netdev = adapter->netdev;
047e0030 1270 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1271 int err = 0;
1272
1273 if (adapter->msix_entries) {
1274 err = igb_request_msix(adapter);
844290e5 1275 if (!err)
9d5c8243 1276 goto request_done;
9d5c8243 1277 /* fall back to MSI */
047e0030 1278 igb_clear_interrupt_scheme(adapter);
c74d588e 1279 if (!pci_enable_msi(pdev))
7dfc16fa 1280 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1281 igb_free_all_tx_resources(adapter);
1282 igb_free_all_rx_resources(adapter);
047e0030 1283 adapter->num_tx_queues = 1;
9d5c8243 1284 adapter->num_rx_queues = 1;
047e0030
AD		 1285 adapter->num_q_vectors = 1;
1286 err = igb_alloc_q_vectors(adapter);
1287 if (err) {
1288 dev_err(&pdev->dev,
1289 "Unable to allocate memory for vectors\n");
1290 goto request_done;
1291 }
1292 err = igb_alloc_queues(adapter);
1293 if (err) {
1294 dev_err(&pdev->dev,
1295 "Unable to allocate memory for queues\n");
1296 igb_free_q_vectors(adapter);
1297 goto request_done;
1298 }
1299 igb_setup_all_tx_resources(adapter);
1300 igb_setup_all_rx_resources(adapter);
9d5c8243 1301 }
844290e5 1302
c74d588e
AD		 1303 igb_assign_vector(adapter->q_vector[0], 0);
1304
7dfc16fa 1305 if (adapter->flags & IGB_FLAG_HAS_MSI) {
c74d588e 1306 err = request_irq(pdev->irq, igb_intr_msi, 0,
047e0030 1307 netdev->name, adapter);
9d5c8243
AK		 1308 if (!err)
1309 goto request_done;
047e0030 1310
9d5c8243
AK		 1311 /* fall back to legacy interrupts */
1312 igb_reset_interrupt_capability(adapter);
7dfc16fa 1313 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1314 }
1315
c74d588e 1316 err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
047e0030 1317 netdev->name, adapter);
9d5c8243 1318
6cb5e577 1319 if (err)
c74d588e 1320 dev_err(&pdev->dev, "Error %d getting interrupt\n",
9d5c8243 1321 err);
9d5c8243
AK		 1322
1323request_done:
1324 return err;
1325}
1326
1327static void igb_free_irq(struct igb_adapter *adapter)
1328{
9d5c8243
AK		 1329 if (adapter->msix_entries) {
1330 int vector = 0, i;
1331
047e0030 1332 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 1333
0d1ae7f4 1334 for (i = 0; i < adapter->num_q_vectors; i++)
047e0030 1335 free_irq(adapter->msix_entries[vector++].vector,
0d1ae7f4 1336 adapter->q_vector[i]);
047e0030
AD		 1337 } else {
1338 free_irq(adapter->pdev->irq, adapter);
9d5c8243 1339 }
9d5c8243
AK		 1340}
1341
1342/**
1343 * igb_irq_disable - Mask off interrupt generation on the NIC
1344 * @adapter: board private structure
1345 **/
1346static void igb_irq_disable(struct igb_adapter *adapter)
1347{
1348 struct e1000_hw *hw = &adapter->hw;
1349
25568a53
AD		 1350 /*
1351 * we need to be careful when disabling interrupts. The VFs are also
1352 * mapped into these registers and so clearing the bits can cause
1353 * issues on the VF drivers so we only need to clear what we set
1354 */
9d5c8243 1355 if (adapter->msix_entries) {
2dfd1212
AD		 1356 u32 regval = rd32(E1000_EIAM);
1357 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1358 wr32(E1000_EIMC, adapter->eims_enable_mask);
1359 regval = rd32(E1000_EIAC);
1360 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 1361 }
844290e5
PW		 1362
1363 wr32(E1000_IAM, 0);
9d5c8243
AK		 1364 wr32(E1000_IMC, ~0);
1365 wrfl();
81a61859
ET		 1366 if (adapter->msix_entries) {
1367 int i;
1368 for (i = 0; i < adapter->num_q_vectors; i++)
1369 synchronize_irq(adapter->msix_entries[i].vector);
1370 } else {
1371 synchronize_irq(adapter->pdev->irq);
1372 }
9d5c8243
AK		 1373}
1374
1375/**
1376 * igb_irq_enable - Enable default interrupt generation settings
1377 * @adapter: board private structure
1378 **/
1379static void igb_irq_enable(struct igb_adapter *adapter)
1380{
1381 struct e1000_hw *hw = &adapter->hw;
1382
1383 if (adapter->msix_entries) {
06218a8d 1384 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
2dfd1212
AD		 1385 u32 regval = rd32(E1000_EIAC);
1386 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1387 regval = rd32(E1000_EIAM);
1388 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 1389 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 1390 if (adapter->vfs_allocated_count) {
4ae196df 1391 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1392 ims |= E1000_IMS_VMMB;
1393 }
1394 wr32(E1000_IMS, ims);
844290e5 1395 } else {
55cac248
AD		 1396 wr32(E1000_IMS, IMS_ENABLE_MASK |
1397 E1000_IMS_DRSTA);
1398 wr32(E1000_IAM, IMS_ENABLE_MASK |
1399 E1000_IMS_DRSTA);
844290e5 1400 }
9d5c8243
AK		 1401}
1402
1403static void igb_update_mng_vlan(struct igb_adapter *adapter)
1404{
51466239 1405 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1406 u16 vid = adapter->hw.mng_cookie.vlan_id;
1407 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1408
1409 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1410 /* add VID to filter table */
1411 igb_vfta_set(hw, vid, true);
1412 adapter->mng_vlan_id = vid;
1413 } else {
1414 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1415 }
1416
1417 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1418 (vid != old_vid) &&
b2cb09b1 1419 !test_bit(old_vid, adapter->active_vlans)) {
51466239
AD		 1420 /* remove VID from filter table */
1421 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1422 }
1423}
1424
1425/**
1426 * igb_release_hw_control - release control of the h/w to f/w
1427 * @adapter: address of board private structure
1428 *
1429 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1430 * For ASF and Pass Through versions of f/w this means that the
1431 * driver is no longer loaded.
1432 *
1433 **/
1434static void igb_release_hw_control(struct igb_adapter *adapter)
1435{
1436 struct e1000_hw *hw = &adapter->hw;
1437 u32 ctrl_ext;
1438
1439 /* Let firmware take over control of h/w */
1440 ctrl_ext = rd32(E1000_CTRL_EXT);
1441 wr32(E1000_CTRL_EXT,
1442 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1443}
1444
9d5c8243
AK		 1445/**
1446 * igb_get_hw_control - get control of the h/w from f/w
1447 * @adapter: address of board private structure
1448 *
1449 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1450 * For ASF and Pass Through versions of f/w this means that
1451 * the driver is loaded.
1452 *
1453 **/
1454static void igb_get_hw_control(struct igb_adapter *adapter)
1455{
1456 struct e1000_hw *hw = &adapter->hw;
1457 u32 ctrl_ext;
1458
1459 /* Let firmware know the driver has taken over */
1460 ctrl_ext = rd32(E1000_CTRL_EXT);
1461 wr32(E1000_CTRL_EXT,
1462 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1463}
1464
9d5c8243
AK		 1465/**
1466 * igb_configure - configure the hardware for RX and TX
1467 * @adapter: private board structure
1468 **/
1469static void igb_configure(struct igb_adapter *adapter)
1470{
1471 struct net_device *netdev = adapter->netdev;
1472 int i;
1473
1474 igb_get_hw_control(adapter);
ff41f8dc 1475 igb_set_rx_mode(netdev);
9d5c8243
AK		 1476
1477 igb_restore_vlan(adapter);
9d5c8243 1478
85b430b4 1479 igb_setup_tctl(adapter);
06cf2666 1480 igb_setup_mrqc(adapter);
9d5c8243 1481 igb_setup_rctl(adapter);
85b430b4
AD		 1482
1483 igb_configure_tx(adapter);
9d5c8243 1484 igb_configure_rx(adapter);
662d7205
AD		 1485
1486 igb_rx_fifo_flush_82575(&adapter->hw);
1487
c493ea45 1488 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1489 * at least 1 descriptor unused to make sure
1490 * next_to_use != next_to_clean */
1491 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1492 struct igb_ring *ring = adapter->rx_ring[i];
cd392f5c 1493 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
9d5c8243 1494 }
9d5c8243
AK		 1495}
1496
88a268c1
NN		 1497/**
1498 * igb_power_up_link - Power up the phy/serdes link
1499 * @adapter: address of board private structure
1500 **/
1501void igb_power_up_link(struct igb_adapter *adapter)
1502{
76886596
AA		 1503 igb_reset_phy(&adapter->hw);
1504
88a268c1
NN		 1505 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1506 igb_power_up_phy_copper(&adapter->hw);
1507 else
1508 igb_power_up_serdes_link_82575(&adapter->hw);
1509}
1510
1511/**
1512 * igb_power_down_link - Power down the phy/serdes link
1513 * @adapter: address of board private structure
1514 */
1515static void igb_power_down_link(struct igb_adapter *adapter)
1516{
1517 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1518 igb_power_down_phy_copper_82575(&adapter->hw);
1519 else
1520 igb_shutdown_serdes_link_82575(&adapter->hw);
1521}
9d5c8243
AK		 1522
1523/**
1524 * igb_up - Open the interface and prepare it to handle traffic
1525 * @adapter: board private structure
1526 **/
9d5c8243
AK		 1527int igb_up(struct igb_adapter *adapter)
1528{
1529 struct e1000_hw *hw = &adapter->hw;
1530 int i;
1531
1532 /* hardware has been reset, we need to reload some things */
1533 igb_configure(adapter);
1534
1535 clear_bit(__IGB_DOWN, &adapter->state);
1536
0d1ae7f4
AD		 1537 for (i = 0; i < adapter->num_q_vectors; i++)
1538 napi_enable(&(adapter->q_vector[i]->napi));
1539
844290e5 1540 if (adapter->msix_entries)
9d5c8243 1541 igb_configure_msix(adapter);
feeb2721
AD		 1542 else
1543 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1544
1545 /* Clear any pending interrupts. */
1546 rd32(E1000_ICR);
1547 igb_irq_enable(adapter);
1548
d4960307
AD		 1549 /* notify VFs that reset has been completed */
1550 if (adapter->vfs_allocated_count) {
1551 u32 reg_data = rd32(E1000_CTRL_EXT);
1552 reg_data |= E1000_CTRL_EXT_PFRSTD;
1553 wr32(E1000_CTRL_EXT, reg_data);
1554 }
1555
4cb9be7a
JB		 1556 netif_tx_start_all_queues(adapter->netdev);
1557
25568a53
AD		 1558 /* start the watchdog. */
1559 hw->mac.get_link_status = 1;
1560 schedule_work(&adapter->watchdog_task);
1561
9d5c8243
AK		 1562 return 0;
1563}
1564
1565void igb_down(struct igb_adapter *adapter)
1566{
9d5c8243 1567 struct net_device *netdev = adapter->netdev;
330a6d6a 1568 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1569 u32 tctl, rctl;
1570 int i;
1571
1572 /* signal that we're down so the interrupt handler does not
1573 * reschedule our watchdog timer */
1574 set_bit(__IGB_DOWN, &adapter->state);
1575
1576 /* disable receives in the hardware */
1577 rctl = rd32(E1000_RCTL);
1578 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1579 /* flush and sleep below */
1580
fd2ea0a7 1581 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1582
1583 /* disable transmits in the hardware */
1584 tctl = rd32(E1000_TCTL);
1585 tctl &= ~E1000_TCTL_EN;
1586 wr32(E1000_TCTL, tctl);
1587 /* flush both disables and wait for them to finish */
1588 wrfl();
1589 msleep(10);
1590
0d1ae7f4
AD		 1591 for (i = 0; i < adapter->num_q_vectors; i++)
1592 napi_disable(&(adapter->q_vector[i]->napi));
9d5c8243 1593
9d5c8243
AK		 1594 igb_irq_disable(adapter);
1595
1596 del_timer_sync(&adapter->watchdog_timer);
1597 del_timer_sync(&adapter->phy_info_timer);
1598
9d5c8243 1599 netif_carrier_off(netdev);
04fe6358
AD		 1600
1601 /* record the stats before reset*/
12dcd86b
ED		 1602 spin_lock(&adapter->stats64_lock);
1603 igb_update_stats(adapter, &adapter->stats64);
1604 spin_unlock(&adapter->stats64_lock);
04fe6358 1605
9d5c8243
AK		 1606 adapter->link_speed = 0;
1607 adapter->link_duplex = 0;
1608
3023682e
JK		 1609 if (!pci_channel_offline(adapter->pdev))
1610 igb_reset(adapter);
9d5c8243
AK		 1611 igb_clean_all_tx_rings(adapter);
1612 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1613#ifdef CONFIG_IGB_DCA
1614
1615 /* since we reset the hardware DCA settings were cleared */
1616 igb_setup_dca(adapter);
1617#endif
9d5c8243
AK		 1618}
1619
1620void igb_reinit_locked(struct igb_adapter *adapter)
1621{
1622 WARN_ON(in_interrupt());
1623 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1624 msleep(1);
1625 igb_down(adapter);
1626 igb_up(adapter);
1627 clear_bit(__IGB_RESETTING, &adapter->state);
1628}
1629
1630void igb_reset(struct igb_adapter *adapter)
1631{
090b1795 1632 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1633 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1634 struct e1000_mac_info *mac = &hw->mac;
1635 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1636 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1637 u16 hwm;
1638
1639 /* Repartition Pba for greater than 9k mtu
1640 * To take effect CTRL.RST is required.
1641 */
fa4dfae0 1642 switch (mac->type) {
d2ba2ed8 1643 case e1000_i350:
55cac248
AD		 1644 case e1000_82580:
1645 pba = rd32(E1000_RXPBS);
1646 pba = igb_rxpbs_adjust_82580(pba);
1647 break;
fa4dfae0 1648 case e1000_82576:
d249be54
AD		 1649 pba = rd32(E1000_RXPBS);
1650 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1651 break;
1652 case e1000_82575:
f96a8a0b
CW		 1653 case e1000_i210:
1654 case e1000_i211:
fa4dfae0
AD		 1655 default:
1656 pba = E1000_PBA_34K;
1657 break;
2d064c06 1658 }
9d5c8243 1659
2d064c06
AD		 1660 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1661 (mac->type < e1000_82576)) {
9d5c8243
AK		 1662 /* adjust PBA for jumbo frames */
1663 wr32(E1000_PBA, pba);
1664
1665 /* To maintain wire speed transmits, the Tx FIFO should be
1666 * large enough to accommodate two full transmit packets,
1667 * rounded up to the next 1KB and expressed in KB. Likewise,
1668 * the Rx FIFO should be large enough to accommodate at least
1669 * one full receive packet and is similarly rounded up and
1670 * expressed in KB. */
1671 pba = rd32(E1000_PBA);
1672 /* upper 16 bits has Tx packet buffer allocation size in KB */
1673 tx_space = pba >> 16;
1674 /* lower 16 bits has Rx packet buffer allocation size in KB */
1675 pba &= 0xffff;
1676 /* the tx fifo also stores 16 bytes of information about the tx
1677 * but don't include ethernet FCS because hardware appends it */
1678 min_tx_space = (adapter->max_frame_size +
85e8d004 1679 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1680 ETH_FCS_LEN) * 2;
1681 min_tx_space = ALIGN(min_tx_space, 1024);
1682 min_tx_space >>= 10;
1683 /* software strips receive CRC, so leave room for it */
1684 min_rx_space = adapter->max_frame_size;
1685 min_rx_space = ALIGN(min_rx_space, 1024);
1686 min_rx_space >>= 10;
1687
1688 /* If current Tx allocation is less than the min Tx FIFO size,
1689 * and the min Tx FIFO size is less than the current Rx FIFO
1690 * allocation, take space away from current Rx allocation */
1691 if (tx_space < min_tx_space &&
1692 ((min_tx_space - tx_space) < pba)) {
1693 pba = pba - (min_tx_space - tx_space);
1694
1695 /* if short on rx space, rx wins and must trump tx
1696 * adjustment */
1697 if (pba < min_rx_space)
1698 pba = min_rx_space;
1699 }
2d064c06 1700 wr32(E1000_PBA, pba);
9d5c8243 1701 }
9d5c8243
AK		 1702
1703 /* flow control settings */
1704 /* The high water mark must be low enough to fit one full frame
1705 * (or the size used for early receive) above it in the Rx FIFO.
1706 * Set it to the lower of:
1707 * - 90% of the Rx FIFO size, or
1708 * - the full Rx FIFO size minus one full frame */
1709 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1710 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1711
d405ea3e
AD		 1712 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1713 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1714 fc->pause_time = 0xFFFF;
1715 fc->send_xon = 1;
0cce119a 1716 fc->current_mode = fc->requested_mode;
9d5c8243 1717
4ae196df
AD		 1718 /* disable receive for all VFs and wait one second */
1719 if (adapter->vfs_allocated_count) {
1720 int i;
1721 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
8fa7e0f7 1722 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
4ae196df
AD		 1723
1724 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1725 igb_ping_all_vfs(adapter);
4ae196df
AD		 1726
1727 /* disable transmits and receives */
1728 wr32(E1000_VFRE, 0);
1729 wr32(E1000_VFTE, 0);
1730 }
1731
9d5c8243 1732 /* Allow time for pending master requests to run */
330a6d6a 1733 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1734 wr32(E1000_WUC, 0);
1735
330a6d6a 1736 if (hw->mac.ops.init_hw(hw))
090b1795 1737 dev_err(&pdev->dev, "Hardware Error\n");
831ec0b4 1738
a27416bb
MV		 1739 /*
1740 * Flow control settings reset on hardware reset, so guarantee flow
1741 * control is off when forcing speed.
1742 */
1743 if (!hw->mac.autoneg)
1744 igb_force_mac_fc(hw);
1745
b6e0c419 1746 igb_init_dmac(adapter, pba);
88a268c1
NN		 1747 if (!netif_running(adapter->netdev))
1748 igb_power_down_link(adapter);
1749
9d5c8243
AK		 1750 igb_update_mng_vlan(adapter);
1751
1752 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1753 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1754
330a6d6a 1755 igb_get_phy_info(hw);
9d5c8243
AK		 1756}
1757
c8f44aff
MM		 1758static netdev_features_t igb_fix_features(struct net_device *netdev,
1759 netdev_features_t features)
b2cb09b1
JP		 1760{
1761 /*
1762 * Since there is no support for separate rx/tx vlan accel
1763 * enable/disable make sure tx flag is always in same state as rx.
1764 */
1765 if (features & NETIF_F_HW_VLAN_RX)
1766 features |= NETIF_F_HW_VLAN_TX;
1767 else
1768 features &= ~NETIF_F_HW_VLAN_TX;
1769
1770 return features;
1771}
1772
c8f44aff
MM		 1773static int igb_set_features(struct net_device *netdev,
1774 netdev_features_t features)
ac52caa3 1775{
c8f44aff 1776 netdev_features_t changed = netdev->features ^ features;
89eaefb6 1777 struct igb_adapter *adapter = netdev_priv(netdev);
ac52caa3 1778
b2cb09b1
JP		 1779 if (changed & NETIF_F_HW_VLAN_RX)
1780 igb_vlan_mode(netdev, features);
1781
89eaefb6
BG		 1782 if (!(changed & NETIF_F_RXALL))
1783 return 0;
1784
1785 netdev->features = features;
1786
1787 if (netif_running(netdev))
1788 igb_reinit_locked(adapter);
1789 else
1790 igb_reset(adapter);
1791
ac52caa3
MM		 1792 return 0;
1793}
1794
2e5c6922 1795static const struct net_device_ops igb_netdev_ops = {
559e9c49 1796 .ndo_open = igb_open,
2e5c6922 1797 .ndo_stop = igb_close,
cd392f5c 1798 .ndo_start_xmit = igb_xmit_frame,
12dcd86b 1799 .ndo_get_stats64 = igb_get_stats64,
ff41f8dc 1800 .ndo_set_rx_mode = igb_set_rx_mode,
2e5c6922
SH		 1801 .ndo_set_mac_address = igb_set_mac,
1802 .ndo_change_mtu = igb_change_mtu,
1803 .ndo_do_ioctl = igb_ioctl,
1804 .ndo_tx_timeout = igb_tx_timeout,
1805 .ndo_validate_addr = eth_validate_addr,
2e5c6922
SH		 1806 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1807 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1808 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1809 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1810 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1811 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1812#ifdef CONFIG_NET_POLL_CONTROLLER
1813 .ndo_poll_controller = igb_netpoll,
1814#endif
b2cb09b1
JP		 1815 .ndo_fix_features = igb_fix_features,
1816 .ndo_set_features = igb_set_features,
2e5c6922
SH		 1817};
1818
d67974f0
CW		 1819/**
1820 * igb_set_fw_version - Configure version string for ethtool
1821 * @adapter: adapter struct
1822 *
1823 **/
1824void igb_set_fw_version(struct igb_adapter *adapter)
1825{
1826 struct e1000_hw *hw = &adapter->hw;
1827 u16 eeprom_verh, eeprom_verl, comb_verh, comb_verl, comb_offset;
1828 u16 major, build, patch, fw_version;
1829 u32 etrack_id;
1830
1831 hw->nvm.ops.read(hw, 5, 1, &fw_version);
1832 if (adapter->hw.mac.type != e1000_i211) {
1833 hw->nvm.ops.read(hw, NVM_ETRACK_WORD, 1, &eeprom_verh);
1834 hw->nvm.ops.read(hw, (NVM_ETRACK_WORD + 1), 1, &eeprom_verl);
1835 etrack_id = (eeprom_verh << IGB_ETRACK_SHIFT) | eeprom_verl;
1836
1837 /* combo image version needs to be found */
1838 hw->nvm.ops.read(hw, NVM_COMB_VER_PTR, 1, &comb_offset);
1839 if ((comb_offset != 0x0) &&
1840 (comb_offset != IGB_NVM_VER_INVALID)) {
1841 hw->nvm.ops.read(hw, (NVM_COMB_VER_OFF + comb_offset
1842 + 1), 1, &comb_verh);
1843 hw->nvm.ops.read(hw, (NVM_COMB_VER_OFF + comb_offset),
1844 1, &comb_verl);
1845
1846 /* Only display Option Rom if it exists and is valid */
1847 if ((comb_verh && comb_verl) &&
1848 ((comb_verh != IGB_NVM_VER_INVALID) &&
1849 (comb_verl != IGB_NVM_VER_INVALID))) {
1850 major = comb_verl >> IGB_COMB_VER_SHFT;
1851 build = (comb_verl << IGB_COMB_VER_SHFT) |
1852 (comb_verh >> IGB_COMB_VER_SHFT);
1853 patch = comb_verh & IGB_COMB_VER_MASK;
1854 snprintf(adapter->fw_version,
1855 sizeof(adapter->fw_version),
1856 "%d.%d%d, 0x%08x, %d.%d.%d",
1857 (fw_version & IGB_MAJOR_MASK) >>
1858 IGB_MAJOR_SHIFT,
1859 (fw_version & IGB_MINOR_MASK) >>
1860 IGB_MINOR_SHIFT,
1861 (fw_version & IGB_BUILD_MASK),
1862 etrack_id, major, build, patch);
1863 goto out;
1864 }
1865 }
1866 snprintf(adapter->fw_version, sizeof(adapter->fw_version),
1867 "%d.%d%d, 0x%08x",
1868 (fw_version & IGB_MAJOR_MASK) >> IGB_MAJOR_SHIFT,
1869 (fw_version & IGB_MINOR_MASK) >> IGB_MINOR_SHIFT,
1870 (fw_version & IGB_BUILD_MASK), etrack_id);
1871 } else {
1872 snprintf(adapter->fw_version, sizeof(adapter->fw_version),
1873 "%d.%d%d",
1874 (fw_version & IGB_MAJOR_MASK) >> IGB_MAJOR_SHIFT,
1875 (fw_version & IGB_MINOR_MASK) >> IGB_MINOR_SHIFT,
1876 (fw_version & IGB_BUILD_MASK));
1877 }
1878out:
1879 return;
1880}
1881
9d5c8243
AK		 1882/**
1883 * igb_probe - Device Initialization Routine
1884 * @pdev: PCI device information struct
1885 * @ent: entry in igb_pci_tbl
1886 *
1887 * Returns 0 on success, negative on failure
1888 *
1889 * igb_probe initializes an adapter identified by a pci_dev structure.
1890 * The OS initialization, configuring of the adapter private structure,
1891 * and a hardware reset occur.
1892 **/
1893static int __devinit igb_probe(struct pci_dev *pdev,
1894 const struct pci_device_id *ent)
1895{
1896 struct net_device *netdev;
1897 struct igb_adapter *adapter;
1898 struct e1000_hw *hw;
4337e993 1899 u16 eeprom_data = 0;
9835fd73 1900 s32 ret_val;
4337e993 1901 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1902 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1903 unsigned long mmio_start, mmio_len;
2d6a5e95 1904 int err, pci_using_dac;
9d5c8243 1905 u16 eeprom_apme_mask = IGB_EEPROM_APME;
9835fd73 1906 u8 part_str[E1000_PBANUM_LENGTH];
9d5c8243 1907
bded64a7
AG		 1908 /* Catch broken hardware that put the wrong VF device ID in
1909 * the PCIe SR-IOV capability.
1910 */
1911 if (pdev->is_virtfn) {
1912 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
f96a8a0b 1913 pci_name(pdev), pdev->vendor, pdev->device);
bded64a7
AG		 1914 return -EINVAL;
1915 }
1916
aed5dec3 1917 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1918 if (err)
1919 return err;
1920
1921 pci_using_dac = 0;
59d71989 1922 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243 1923 if (!err) {
59d71989 1924 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243
AK		 1925 if (!err)
1926 pci_using_dac = 1;
1927 } else {
59d71989 1928 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243 1929 if (err) {
59d71989 1930 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243
AK		 1931 if (err) {
1932 dev_err(&pdev->dev, "No usable DMA "
1933 "configuration, aborting\n");
1934 goto err_dma;
1935 }
1936 }
1937 }
1938
aed5dec3
AD		 1939 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1940 IORESOURCE_MEM),
1941 igb_driver_name);
9d5c8243
AK		 1942 if (err)
1943 goto err_pci_reg;
1944
19d5afd4 1945 pci_enable_pcie_error_reporting(pdev);
40a914fa 1946
9d5c8243 1947 pci_set_master(pdev);
c682fc23 1948 pci_save_state(pdev);
9d5c8243
AK		 1949
1950 err = -ENOMEM;
1bfaf07b 1951 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1cc3bd87 1952 IGB_MAX_TX_QUEUES);
9d5c8243
AK		 1953 if (!netdev)
1954 goto err_alloc_etherdev;
1955
1956 SET_NETDEV_DEV(netdev, &pdev->dev);
1957
1958 pci_set_drvdata(pdev, netdev);
1959 adapter = netdev_priv(netdev);
1960 adapter->netdev = netdev;
1961 adapter->pdev = pdev;
1962 hw = &adapter->hw;
1963 hw->back = adapter;
b3f4d599 1964 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
9d5c8243
AK		 1965
1966 mmio_start = pci_resource_start(pdev, 0);
1967 mmio_len = pci_resource_len(pdev, 0);
1968
1969 err = -EIO;
28b0759c
AD		 1970 hw->hw_addr = ioremap(mmio_start, mmio_len);
1971 if (!hw->hw_addr)
9d5c8243
AK		 1972 goto err_ioremap;
1973
2e5c6922 1974 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1975 igb_set_ethtool_ops(netdev);
9d5c8243 1976 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1977
1978 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1979
1980 netdev->mem_start = mmio_start;
1981 netdev->mem_end = mmio_start + mmio_len;
1982
9d5c8243
AK		 1983 /* PCI config space info */
1984 hw->vendor_id = pdev->vendor;
1985 hw->device_id = pdev->device;
1986 hw->revision_id = pdev->revision;
1987 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1988 hw->subsystem_device_id = pdev->subsystem_device;
1989
9d5c8243
AK		 1990 /* Copy the default MAC, PHY and NVM function pointers */
1991 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1992 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1993 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1994 /* Initialize skew-specific constants */
1995 err = ei->get_invariants(hw);
1996 if (err)
450c87c8 1997 goto err_sw_init;
9d5c8243 1998
450c87c8 1999 /* setup the private structure */
9d5c8243
AK		 2000 err = igb_sw_init(adapter);
2001 if (err)
2002 goto err_sw_init;
2003
2004 igb_get_bus_info_pcie(hw);
2005
2006 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 2007
2008 /* Copper options */
2009 if (hw->phy.media_type == e1000_media_type_copper) {
2010 hw->phy.mdix = AUTO_ALL_MODES;
2011 hw->phy.disable_polarity_correction = false;
2012 hw->phy.ms_type = e1000_ms_hw_default;
2013 }
2014
2015 if (igb_check_reset_block(hw))
2016 dev_info(&pdev->dev,
2017 "PHY reset is blocked due to SOL/IDER session.\n");
2018
077887c3
AD		 2019 /*
2020 * features is initialized to 0 in allocation, it might have bits
2021 * set by igb_sw_init so we should use an or instead of an
2022 * assignment.
2023 */
2024 netdev->features |= NETIF_F_SG |
2025 NETIF_F_IP_CSUM |
2026 NETIF_F_IPV6_CSUM |
2027 NETIF_F_TSO |
2028 NETIF_F_TSO6 |
2029 NETIF_F_RXHASH |
2030 NETIF_F_RXCSUM |
2031 NETIF_F_HW_VLAN_RX |
2032 NETIF_F_HW_VLAN_TX;
2033
2034 /* copy netdev features into list of user selectable features */
2035 netdev->hw_features |= netdev->features;
89eaefb6 2036 netdev->hw_features |= NETIF_F_RXALL;
077887c3
AD		 2037
2038 /* set this bit last since it cannot be part of hw_features */
2039 netdev->features |= NETIF_F_HW_VLAN_FILTER;
2040
2041 netdev->vlan_features |= NETIF_F_TSO |
2042 NETIF_F_TSO6 |
2043 NETIF_F_IP_CSUM |
2044 NETIF_F_IPV6_CSUM |
2045 NETIF_F_SG;
48f29ffc 2046
6b8f0922
BG		 2047 netdev->priv_flags |= IFF_SUPP_NOFCS;
2048
7b872a55 2049 if (pci_using_dac) {
9d5c8243 2050 netdev->features |= NETIF_F_HIGHDMA;
7b872a55
YZ		 2051 netdev->vlan_features |= NETIF_F_HIGHDMA;
2052 }
9d5c8243 2053
ac52caa3
MM		 2054 if (hw->mac.type >= e1000_82576) {
2055 netdev->hw_features |= NETIF_F_SCTP_CSUM;
b9473560 2056 netdev->features |= NETIF_F_SCTP_CSUM;
ac52caa3 2057 }
b9473560 2058
01789349
JP		 2059 netdev->priv_flags |= IFF_UNICAST_FLT;
2060
330a6d6a 2061 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 2062
2063 /* before reading the NVM, reset the controller to put the device in a
2064 * known good starting state */
2065 hw->mac.ops.reset_hw(hw);
2066
f96a8a0b
CW		 2067 /*
2068 * make sure the NVM is good , i211 parts have special NVM that
2069 * doesn't contain a checksum
2070 */
2071 if (hw->mac.type != e1000_i211) {
2072 if (hw->nvm.ops.validate(hw) < 0) {
2073 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
2074 err = -EIO;
2075 goto err_eeprom;
2076 }
9d5c8243
AK		 2077 }
2078
2079 /* copy the MAC address out of the NVM */
2080 if (hw->mac.ops.read_mac_addr(hw))
2081 dev_err(&pdev->dev, "NVM Read Error\n");
2082
2083 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2084 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
2085
2086 if (!is_valid_ether_addr(netdev->perm_addr)) {
2087 dev_err(&pdev->dev, "Invalid MAC Address\n");
2088 err = -EIO;
2089 goto err_eeprom;
2090 }
2091
d67974f0
CW		 2092 /* get firmware version for ethtool -i */
2093 igb_set_fw_version(adapter);
2094
c061b18d 2095 setup_timer(&adapter->watchdog_timer, igb_watchdog,
0e340485 2096 (unsigned long) adapter);
c061b18d 2097 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
0e340485 2098 (unsigned long) adapter);
9d5c8243
AK		 2099
2100 INIT_WORK(&adapter->reset_task, igb_reset_task);
2101 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2102
450c87c8 2103 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 2104 adapter->fc_autoneg = true;
2105 hw->mac.autoneg = true;
2106 hw->phy.autoneg_advertised = 0x2f;
2107
0cce119a
AD		 2108 hw->fc.requested_mode = e1000_fc_default;
2109 hw->fc.current_mode = e1000_fc_default;
9d5c8243 2110
9d5c8243
AK		 2111 igb_validate_mdi_setting(hw);
2112
9d5c8243
AK		 2113 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2114 * enable the ACPI Magic Packet filter
2115 */
2116
a2cf8b6c 2117 if (hw->bus.func == 0)
312c75ae 2118 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
6d337dce 2119 else if (hw->mac.type >= e1000_82580)
55cac248
AD		 2120 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2121 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2122 &eeprom_data);
a2cf8b6c
AD		 2123 else if (hw->bus.func == 1)
2124 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 2125
2126 if (eeprom_data & eeprom_apme_mask)
2127 adapter->eeprom_wol |= E1000_WUFC_MAG;
2128
2129 /* now that we have the eeprom settings, apply the special cases where
2130 * the eeprom may be wrong or the board simply won't support wake on
2131 * lan on a particular port */
2132 switch (pdev->device) {
2133 case E1000_DEV_ID_82575GB_QUAD_COPPER:
2134 adapter->eeprom_wol = 0;
2135 break;
2136 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 2137 case E1000_DEV_ID_82576_FIBER:
2138 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 2139 /* Wake events only supported on port A for dual fiber
2140 * regardless of eeprom setting */
2141 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
2142 adapter->eeprom_wol = 0;
2143 break;
c8ea5ea9 2144 case E1000_DEV_ID_82576_QUAD_COPPER:
d5aa2252 2145 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
c8ea5ea9
AD		 2146 /* if quad port adapter, disable WoL on all but port A */
2147 if (global_quad_port_a != 0)
2148 adapter->eeprom_wol = 0;
2149 else
2150 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2151 /* Reset for multiple quad port adapters */
2152 if (++global_quad_port_a == 4)
2153 global_quad_port_a = 0;
2154 break;
9d5c8243
AK		 2155 }
2156
2157 /* initialize the wol settings based on the eeprom settings */
2158 adapter->wol = adapter->eeprom_wol;
e1b86d84 2159 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 2160
2161 /* reset the hardware with the new settings */
2162 igb_reset(adapter);
2163
2164 /* let the f/w know that the h/w is now under the control of the
2165 * driver. */
2166 igb_get_hw_control(adapter);
2167
9d5c8243
AK		 2168 strcpy(netdev->name, "eth%d");
2169 err = register_netdev(netdev);
2170 if (err)
2171 goto err_register;
2172
b168dfc5
JB		 2173 /* carrier off reporting is important to ethtool even BEFORE open */
2174 netif_carrier_off(netdev);
2175
421e02f0 2176#ifdef CONFIG_IGB_DCA
bbd98fe4 2177 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 2178 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 2179 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 2180 igb_setup_dca(adapter);
2181 }
fe4506b6 2182
38c845c7 2183#endif
7ebae817 2184#ifdef CONFIG_IGB_PTP
673b8b70 2185 /* do hw tstamp init after resetting */
7ebae817 2186 igb_ptp_init(adapter);
673b8b70 2187
7ebae817 2188#endif
9d5c8243
AK		 2189 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
2190 /* print bus type/speed/width info */
7c510e4b 2191 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 2192 netdev->name,
559e9c49 2193 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
ff846f52 2194 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
559e9c49 2195 "unknown"),
59c3de89
AD		 2196 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
2197 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
2198 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2199 "unknown"),
7c510e4b 2200 netdev->dev_addr);
9d5c8243 2201
9835fd73
CW		 2202 ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH);
2203 if (ret_val)
2204 strcpy(part_str, "Unknown");
2205 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
9d5c8243
AK		 2206 dev_info(&pdev->dev,
2207 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2208 adapter->msix_entries ? "MSI-X" :
7dfc16fa 2209 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243 2210 adapter->num_rx_queues, adapter->num_tx_queues);
09b068d4
CW		 2211 switch (hw->mac.type) {
2212 case e1000_i350:
f96a8a0b
CW		 2213 case e1000_i210:
2214 case e1000_i211:
09b068d4
CW		 2215 igb_set_eee_i350(hw);
2216 break;
2217 default:
2218 break;
2219 }
749ab2cd
YZ		 2220
2221 pm_runtime_put_noidle(&pdev->dev);
9d5c8243
AK		 2222 return 0;
2223
2224err_register:
2225 igb_release_hw_control(adapter);
2226err_eeprom:
2227 if (!igb_check_reset_block(hw))
f5f4cf08 2228 igb_reset_phy(hw);
9d5c8243
AK		 2229
2230 if (hw->flash_address)
2231 iounmap(hw->flash_address);
9d5c8243 2232err_sw_init:
047e0030 2233 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 2234 iounmap(hw->hw_addr);
2235err_ioremap:
2236 free_netdev(netdev);
2237err_alloc_etherdev:
559e9c49
AD		 2238 pci_release_selected_regions(pdev,
2239 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2240err_pci_reg:
2241err_dma:
2242 pci_disable_device(pdev);
2243 return err;
2244}
2245
2246/**
2247 * igb_remove - Device Removal Routine
2248 * @pdev: PCI device information struct
2249 *
2250 * igb_remove is called by the PCI subsystem to alert the driver
2251 * that it should release a PCI device. The could be caused by a
2252 * Hot-Plug event, or because the driver is going to be removed from
2253 * memory.
2254 **/
2255static void __devexit igb_remove(struct pci_dev *pdev)
2256{
2257 struct net_device *netdev = pci_get_drvdata(pdev);
2258 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 2259 struct e1000_hw *hw = &adapter->hw;
9d5c8243 2260
749ab2cd 2261 pm_runtime_get_noresume(&pdev->dev);
7ebae817
RC		 2262#ifdef CONFIG_IGB_PTP
2263 igb_ptp_remove(adapter);
749ab2cd 2264
7ebae817 2265#endif
760141a5
TH		 2266 /*
2267 * The watchdog timer may be rescheduled, so explicitly
2268 * disable watchdog from being rescheduled.
2269 */
9d5c8243
AK		 2270 set_bit(__IGB_DOWN, &adapter->state);
2271 del_timer_sync(&adapter->watchdog_timer);
2272 del_timer_sync(&adapter->phy_info_timer);
2273
760141a5
TH		 2274 cancel_work_sync(&adapter->reset_task);
2275 cancel_work_sync(&adapter->watchdog_task);
9d5c8243 2276
421e02f0 2277#ifdef CONFIG_IGB_DCA
7dfc16fa 2278 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 2279 dev_info(&pdev->dev, "DCA disabled\n");
2280 dca_remove_requester(&pdev->dev);
7dfc16fa 2281 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 2282 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 2283 }
2284#endif
2285
9d5c8243
AK		 2286 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2287 * would have already happened in close and is redundant. */
2288 igb_release_hw_control(adapter);
2289
2290 unregister_netdev(netdev);
2291
047e0030 2292 igb_clear_interrupt_scheme(adapter);
9d5c8243 2293
37680117
AD		 2294#ifdef CONFIG_PCI_IOV
2295 /* reclaim resources allocated to VFs */
2296 if (adapter->vf_data) {
2297 /* disable iov and allow time for transactions to clear */
0224d663
GR		 2298 if (!igb_check_vf_assignment(adapter)) {
2299 pci_disable_sriov(pdev);
2300 msleep(500);
2301 } else {
2302 dev_info(&pdev->dev, "VF(s) assigned to guests!\n");
2303 }
37680117
AD		 2304
2305 kfree(adapter->vf_data);
2306 adapter->vf_data = NULL;
2307 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 2308 wrfl();
37680117
AD		 2309 msleep(100);
2310 dev_info(&pdev->dev, "IOV Disabled\n");
2311 }
2312#endif
559e9c49 2313
28b0759c
AD		 2314 iounmap(hw->hw_addr);
2315 if (hw->flash_address)
2316 iounmap(hw->flash_address);
559e9c49
AD		 2317 pci_release_selected_regions(pdev,
2318 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243 2319
1128c756 2320 kfree(adapter->shadow_vfta);
9d5c8243
AK		 2321 free_netdev(netdev);
2322
19d5afd4 2323 pci_disable_pcie_error_reporting(pdev);
40a914fa 2324
9d5c8243
AK		 2325 pci_disable_device(pdev);
2326}
2327
a6b623e0
AD		 2328/**
2329 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2330 * @adapter: board private structure to initialize
2331 *
2332 * This function initializes the vf specific data storage and then attempts to
2333 * allocate the VFs. The reason for ordering it this way is because it is much
2334 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2335 * the memory for the VFs.
2336 **/
2337static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2338{
2339#ifdef CONFIG_PCI_IOV
2340 struct pci_dev *pdev = adapter->pdev;
f96a8a0b 2341 struct e1000_hw *hw = &adapter->hw;
0224d663
GR		 2342 int old_vfs = igb_find_enabled_vfs(adapter);
2343 int i;
a6b623e0 2344
f96a8a0b
CW		 2345 /* Virtualization features not supported on i210 family. */
2346 if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211))
2347 return;
2348
0224d663
GR		 2349 if (old_vfs) {
2350 dev_info(&pdev->dev, "%d pre-allocated VFs found - override "
2351 "max_vfs setting of %d\n", old_vfs, max_vfs);
2352 adapter->vfs_allocated_count = old_vfs;
a6b623e0
AD		 2353 }
2354
0224d663
GR		 2355 if (!adapter->vfs_allocated_count)
2356 return;
2357
2358 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2359 sizeof(struct vf_data_storage), GFP_KERNEL);
f96a8a0b 2360
0224d663
GR		 2361 /* if allocation failed then we do not support SR-IOV */
2362 if (!adapter->vf_data) {
a6b623e0 2363 adapter->vfs_allocated_count = 0;
0224d663
GR		 2364 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2365 "Data Storage\n");
2366 goto out;
a6b623e0 2367 }
0224d663
GR		 2368
2369 if (!old_vfs) {
2370 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count))
2371 goto err_out;
2372 }
2373 dev_info(&pdev->dev, "%d VFs allocated\n",
2374 adapter->vfs_allocated_count);
2375 for (i = 0; i < adapter->vfs_allocated_count; i++)
2376 igb_vf_configure(adapter, i);
2377
2378 /* DMA Coalescing is not supported in IOV mode. */
2379 adapter->flags &= ~IGB_FLAG_DMAC;
2380 goto out;
2381err_out:
2382 kfree(adapter->vf_data);
2383 adapter->vf_data = NULL;
2384 adapter->vfs_allocated_count = 0;
2385out:
2386 return;
a6b623e0
AD		 2387#endif /* CONFIG_PCI_IOV */
2388}
2389
9d5c8243
AK		 2390/**
2391 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2392 * @adapter: board private structure to initialize
2393 *
2394 * igb_sw_init initializes the Adapter private data structure.
2395 * Fields are initialized based on PCI device information and
2396 * OS network device settings (MTU size).
2397 **/
2398static int __devinit igb_sw_init(struct igb_adapter *adapter)
2399{
2400 struct e1000_hw *hw = &adapter->hw;
2401 struct net_device *netdev = adapter->netdev;
2402 struct pci_dev *pdev = adapter->pdev;
374a542d 2403 u32 max_rss_queues;
9d5c8243
AK		 2404
2405 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2406
13fde97a 2407 /* set default ring sizes */
68fd9910
AD		 2408 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2409 adapter->rx_ring_count = IGB_DEFAULT_RXD;
13fde97a
AD		 2410
2411 /* set default ITR values */
4fc82adf
AD		 2412 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2413 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2414
13fde97a
AD		 2415 /* set default work limits */
2416 adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
2417
153285f9
AD		 2418 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
2419 VLAN_HLEN;
9d5c8243
AK		 2420 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2421
81c2fc22
AD		 2422 adapter->node = -1;
2423
12dcd86b 2424 spin_lock_init(&adapter->stats64_lock);
a6b623e0 2425#ifdef CONFIG_PCI_IOV
6b78bb1d
CW		 2426 switch (hw->mac.type) {
2427 case e1000_82576:
2428 case e1000_i350:
9b082d73
SA		 2429 if (max_vfs > 7) {
2430 dev_warn(&pdev->dev,
2431 "Maximum of 7 VFs per PF, using max\n");
2432 adapter->vfs_allocated_count = 7;
2433 } else
2434 adapter->vfs_allocated_count = max_vfs;
6b78bb1d
CW		 2435 break;
2436 default:
2437 break;
2438 }
a6b623e0 2439#endif /* CONFIG_PCI_IOV */
374a542d
MV		 2440
2441 /* Determine the maximum number of RSS queues supported. */
f96a8a0b 2442 switch (hw->mac.type) {
374a542d
MV		 2443 case e1000_i211:
2444 max_rss_queues = IGB_MAX_RX_QUEUES_I211;
2445 break;
2446 case e1000_82575:
f96a8a0b 2447 case e1000_i210:
374a542d
MV		 2448 max_rss_queues = IGB_MAX_RX_QUEUES_82575;
2449 break;
2450 case e1000_i350:
2451 /* I350 cannot do RSS and SR-IOV at the same time */
2452 if (!!adapter->vfs_allocated_count) {
2453 max_rss_queues = 1;
2454 break;
2455 }
2456 /* fall through */
2457 case e1000_82576:
2458 if (!!adapter->vfs_allocated_count) {
2459 max_rss_queues = 2;
2460 break;
2461 }
2462 /* fall through */
2463 case e1000_82580:
2464 default:
2465 max_rss_queues = IGB_MAX_RX_QUEUES;
f96a8a0b 2466 break;
374a542d
MV		 2467 }
2468
2469 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
2470
2471 /* Determine if we need to pair queues. */
2472 switch (hw->mac.type) {
2473 case e1000_82575:
f96a8a0b 2474 case e1000_i211:
374a542d 2475 /* Device supports enough interrupts without queue pairing. */
f96a8a0b 2476 break;
374a542d
MV		 2477 case e1000_82576:
2478 /*
2479 * If VFs are going to be allocated with RSS queues then we
2480 * should pair the queues in order to conserve interrupts due
2481 * to limited supply.
2482 */
2483 if ((adapter->rss_queues > 1) &&
2484 (adapter->vfs_allocated_count > 6))
2485 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2486 /* fall through */
2487 case e1000_82580:
2488 case e1000_i350:
2489 case e1000_i210:
f96a8a0b 2490 default:
374a542d
MV		 2491 /*
2492 * If rss_queues > half of max_rss_queues, pair the queues in
2493 * order to conserve interrupts due to limited supply.
2494 */
2495 if (adapter->rss_queues > (max_rss_queues / 2))
2496 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
f96a8a0b
CW		 2497 break;
2498 }
a99955fc 2499
1128c756
CW		 2500 /* Setup and initialize a copy of the hw vlan table array */
2501 adapter->shadow_vfta = kzalloc(sizeof(u32) *
2502 E1000_VLAN_FILTER_TBL_SIZE,
2503 GFP_ATOMIC);
2504
a6b623e0 2505 /* This call may decrease the number of queues */
047e0030 2506 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 2507 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2508 return -ENOMEM;
2509 }
2510
a6b623e0
AD		 2511 igb_probe_vfs(adapter);
2512
9d5c8243
AK		 2513 /* Explicitly disable IRQ since the NIC can be in any state. */
2514 igb_irq_disable(adapter);
2515
f96a8a0b 2516 if (hw->mac.type >= e1000_i350)
831ec0b4
CW		 2517 adapter->flags &= ~IGB_FLAG_DMAC;
2518
9d5c8243
AK		 2519 set_bit(__IGB_DOWN, &adapter->state);
2520 return 0;
2521}
2522
2523/**
2524 * igb_open - Called when a network interface is made active
2525 * @netdev: network interface device structure
2526 *
2527 * Returns 0 on success, negative value on failure
2528 *
2529 * The open entry point is called when a network interface is made
2530 * active by the system (IFF_UP). At this point all resources needed
2531 * for transmit and receive operations are allocated, the interrupt
2532 * handler is registered with the OS, the watchdog timer is started,
2533 * and the stack is notified that the interface is ready.
2534 **/
749ab2cd 2535static int __igb_open(struct net_device *netdev, bool resuming)
9d5c8243
AK		 2536{
2537 struct igb_adapter *adapter = netdev_priv(netdev);
2538 struct e1000_hw *hw = &adapter->hw;
749ab2cd 2539 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2540 int err;
2541 int i;
2542
2543 /* disallow open during test */
749ab2cd
YZ		 2544 if (test_bit(__IGB_TESTING, &adapter->state)) {
2545 WARN_ON(resuming);
9d5c8243 2546 return -EBUSY;
749ab2cd
YZ		 2547 }
2548
2549 if (!resuming)
2550 pm_runtime_get_sync(&pdev->dev);
9d5c8243 2551
b168dfc5
JB		 2552 netif_carrier_off(netdev);
2553
9d5c8243
AK		 2554 /* allocate transmit descriptors */
2555 err = igb_setup_all_tx_resources(adapter);
2556 if (err)
2557 goto err_setup_tx;
2558
2559 /* allocate receive descriptors */
2560 err = igb_setup_all_rx_resources(adapter);
2561 if (err)
2562 goto err_setup_rx;
2563
88a268c1 2564 igb_power_up_link(adapter);
9d5c8243 2565
9d5c8243
AK		 2566 /* before we allocate an interrupt, we must be ready to handle it.
2567 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2568 * as soon as we call pci_request_irq, so we have to setup our
2569 * clean_rx handler before we do so. */
2570 igb_configure(adapter);
2571
2572 err = igb_request_irq(adapter);
2573 if (err)
2574 goto err_req_irq;
2575
2576 /* From here on the code is the same as igb_up() */
2577 clear_bit(__IGB_DOWN, &adapter->state);
2578
0d1ae7f4
AD		 2579 for (i = 0; i < adapter->num_q_vectors; i++)
2580 napi_enable(&(adapter->q_vector[i]->napi));
9d5c8243
AK		 2581
2582 /* Clear any pending interrupts. */
2583 rd32(E1000_ICR);
844290e5
PW		 2584
2585 igb_irq_enable(adapter);
2586
d4960307
AD		 2587 /* notify VFs that reset has been completed */
2588 if (adapter->vfs_allocated_count) {
2589 u32 reg_data = rd32(E1000_CTRL_EXT);
2590 reg_data |= E1000_CTRL_EXT_PFRSTD;
2591 wr32(E1000_CTRL_EXT, reg_data);
2592 }
2593
d55b53ff
JK		 2594 netif_tx_start_all_queues(netdev);
2595
749ab2cd
YZ		 2596 if (!resuming)
2597 pm_runtime_put(&pdev->dev);
2598
25568a53
AD		 2599 /* start the watchdog. */
2600 hw->mac.get_link_status = 1;
2601 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2602
2603 return 0;
2604
2605err_req_irq:
2606 igb_release_hw_control(adapter);
88a268c1 2607 igb_power_down_link(adapter);
9d5c8243
AK		 2608 igb_free_all_rx_resources(adapter);
2609err_setup_rx:
2610 igb_free_all_tx_resources(adapter);
2611err_setup_tx:
2612 igb_reset(adapter);
749ab2cd
YZ		 2613 if (!resuming)
2614 pm_runtime_put(&pdev->dev);
9d5c8243
AK		 2615
2616 return err;
2617}
2618
749ab2cd
YZ		 2619static int igb_open(struct net_device *netdev)
2620{
2621 return __igb_open(netdev, false);
2622}
2623
9d5c8243
AK		 2624/**
2625 * igb_close - Disables a network interface
2626 * @netdev: network interface device structure
2627 *
2628 * Returns 0, this is not allowed to fail
2629 *
2630 * The close entry point is called when an interface is de-activated
2631 * by the OS. The hardware is still under the driver's control, but
2632 * needs to be disabled. A global MAC reset is issued to stop the
2633 * hardware, and all transmit and receive resources are freed.
2634 **/
749ab2cd 2635static int __igb_close(struct net_device *netdev, bool suspending)
9d5c8243
AK		 2636{
2637 struct igb_adapter *adapter = netdev_priv(netdev);
749ab2cd 2638 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2639
2640 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
9d5c8243 2641
749ab2cd
YZ		 2642 if (!suspending)
2643 pm_runtime_get_sync(&pdev->dev);
2644
2645 igb_down(adapter);
9d5c8243
AK		 2646 igb_free_irq(adapter);
2647
2648 igb_free_all_tx_resources(adapter);
2649 igb_free_all_rx_resources(adapter);
2650
749ab2cd
YZ		 2651 if (!suspending)
2652 pm_runtime_put_sync(&pdev->dev);
9d5c8243
AK		 2653 return 0;
2654}
2655
749ab2cd
YZ		 2656static int igb_close(struct net_device *netdev)
2657{
2658 return __igb_close(netdev, false);
2659}
2660
9d5c8243
AK		 2661/**
2662 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2663 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2664 *
2665 * Return 0 on success, negative on failure
2666 **/
80785298 2667int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2668{
59d71989 2669 struct device *dev = tx_ring->dev;
81c2fc22 2670 int orig_node = dev_to_node(dev);
9d5c8243
AK		 2671 int size;
2672
06034649 2673 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
81c2fc22
AD		 2674 tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node);
2675 if (!tx_ring->tx_buffer_info)
2676 tx_ring->tx_buffer_info = vzalloc(size);
06034649 2677 if (!tx_ring->tx_buffer_info)
9d5c8243 2678 goto err;
9d5c8243
AK		 2679
2680 /* round up to nearest 4K */
85e8d004 2681 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2682 tx_ring->size = ALIGN(tx_ring->size, 4096);
2683
81c2fc22 2684 set_dev_node(dev, tx_ring->numa_node);
59d71989
AD		 2685 tx_ring->desc = dma_alloc_coherent(dev,
2686 tx_ring->size,
2687 &tx_ring->dma,
2688 GFP_KERNEL);
81c2fc22
AD		 2689 set_dev_node(dev, orig_node);
2690 if (!tx_ring->desc)
2691 tx_ring->desc = dma_alloc_coherent(dev,
2692 tx_ring->size,
2693 &tx_ring->dma,
2694 GFP_KERNEL);
9d5c8243
AK		 2695
2696 if (!tx_ring->desc)
2697 goto err;
2698
9d5c8243
AK		 2699 tx_ring->next_to_use = 0;
2700 tx_ring->next_to_clean = 0;
81c2fc22 2701
9d5c8243
AK		 2702 return 0;
2703
2704err:
06034649 2705 vfree(tx_ring->tx_buffer_info);
59d71989 2706 dev_err(dev,
9d5c8243
AK		 2707 "Unable to allocate memory for the transmit descriptor ring\n");
2708 return -ENOMEM;
2709}
2710
2711/**
2712 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2713 * (Descriptors) for all queues
2714 * @adapter: board private structure
2715 *
2716 * Return 0 on success, negative on failure
2717 **/
2718static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2719{
439705e1 2720 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2721 int i, err = 0;
2722
2723 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2724 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2725 if (err) {
439705e1 2726 dev_err(&pdev->dev,
9d5c8243
AK		 2727 "Allocation for Tx Queue %u failed\n", i);
2728 for (i--; i >= 0; i--)
3025a446 2729 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2730 break;
2731 }
2732 }
2733
2734 return err;
2735}
2736
2737/**
85b430b4
AD		 2738 * igb_setup_tctl - configure the transmit control registers
2739 * @adapter: Board private structure
9d5c8243 2740 **/
d7ee5b3a 2741void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2742{
9d5c8243
AK		 2743 struct e1000_hw *hw = &adapter->hw;
2744 u32 tctl;
9d5c8243 2745
85b430b4
AD		 2746 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2747 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2748
2749 /* Program the Transmit Control Register */
9d5c8243
AK		 2750 tctl = rd32(E1000_TCTL);
2751 tctl &= ~E1000_TCTL_CT;
2752 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2753 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2754
2755 igb_config_collision_dist(hw);
2756
9d5c8243
AK		 2757 /* Enable transmits */
2758 tctl |= E1000_TCTL_EN;
2759
2760 wr32(E1000_TCTL, tctl);
2761}
2762
85b430b4
AD		 2763/**
2764 * igb_configure_tx_ring - Configure transmit ring after Reset
2765 * @adapter: board private structure
2766 * @ring: tx ring to configure
2767 *
2768 * Configure a transmit ring after a reset.
2769 **/
d7ee5b3a
AD		 2770void igb_configure_tx_ring(struct igb_adapter *adapter,
2771 struct igb_ring *ring)
85b430b4
AD		 2772{
2773 struct e1000_hw *hw = &adapter->hw;
a74420e0 2774 u32 txdctl = 0;
85b430b4
AD		 2775 u64 tdba = ring->dma;
2776 int reg_idx = ring->reg_idx;
2777
2778 /* disable the queue */
a74420e0 2779 wr32(E1000_TXDCTL(reg_idx), 0);
85b430b4
AD		 2780 wrfl();
2781 mdelay(10);
2782
2783 wr32(E1000_TDLEN(reg_idx),
2784 ring->count * sizeof(union e1000_adv_tx_desc));
2785 wr32(E1000_TDBAL(reg_idx),
2786 tdba & 0x00000000ffffffffULL);
2787 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2788
fce99e34 2789 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
a74420e0 2790 wr32(E1000_TDH(reg_idx), 0);
fce99e34 2791 writel(0, ring->tail);
85b430b4
AD		 2792
2793 txdctl |= IGB_TX_PTHRESH;
2794 txdctl |= IGB_TX_HTHRESH << 8;
2795 txdctl |= IGB_TX_WTHRESH << 16;
2796
2797 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2798 wr32(E1000_TXDCTL(reg_idx), txdctl);
2799}
2800
2801/**
2802 * igb_configure_tx - Configure transmit Unit after Reset
2803 * @adapter: board private structure
2804 *
2805 * Configure the Tx unit of the MAC after a reset.
2806 **/
2807static void igb_configure_tx(struct igb_adapter *adapter)
2808{
2809 int i;
2810
2811 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2812 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2813}
2814
9d5c8243
AK		 2815/**
2816 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2817 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2818 *
2819 * Returns 0 on success, negative on failure
2820 **/
80785298 2821int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2822{
59d71989 2823 struct device *dev = rx_ring->dev;
81c2fc22 2824 int orig_node = dev_to_node(dev);
9d5c8243
AK		 2825 int size, desc_len;
2826
06034649 2827 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
81c2fc22
AD		 2828 rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node);
2829 if (!rx_ring->rx_buffer_info)
2830 rx_ring->rx_buffer_info = vzalloc(size);
06034649 2831 if (!rx_ring->rx_buffer_info)
9d5c8243 2832 goto err;
9d5c8243
AK		 2833
2834 desc_len = sizeof(union e1000_adv_rx_desc);
2835
2836 /* Round up to nearest 4K */
2837 rx_ring->size = rx_ring->count * desc_len;
2838 rx_ring->size = ALIGN(rx_ring->size, 4096);
2839
81c2fc22 2840 set_dev_node(dev, rx_ring->numa_node);
59d71989
AD		 2841 rx_ring->desc = dma_alloc_coherent(dev,
2842 rx_ring->size,
2843 &rx_ring->dma,
2844 GFP_KERNEL);
81c2fc22
AD		 2845 set_dev_node(dev, orig_node);
2846 if (!rx_ring->desc)
2847 rx_ring->desc = dma_alloc_coherent(dev,
2848 rx_ring->size,
2849 &rx_ring->dma,
2850 GFP_KERNEL);
9d5c8243
AK		 2851
2852 if (!rx_ring->desc)
2853 goto err;
2854
2855 rx_ring->next_to_clean = 0;
2856 rx_ring->next_to_use = 0;
9d5c8243 2857
9d5c8243
AK		 2858 return 0;
2859
2860err:
06034649
AD		 2861 vfree(rx_ring->rx_buffer_info);
2862 rx_ring->rx_buffer_info = NULL;
59d71989
AD		 2863 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2864 " ring\n");
9d5c8243
AK		 2865 return -ENOMEM;
2866}
2867
2868/**
2869 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2870 * (Descriptors) for all queues
2871 * @adapter: board private structure
2872 *
2873 * Return 0 on success, negative on failure
2874 **/
2875static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2876{
439705e1 2877 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2878 int i, err = 0;
2879
2880 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2881 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2882 if (err) {
439705e1 2883 dev_err(&pdev->dev,
9d5c8243
AK		 2884 "Allocation for Rx Queue %u failed\n", i);
2885 for (i--; i >= 0; i--)
3025a446 2886 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2887 break;
2888 }
2889 }
2890
2891 return err;
2892}
2893
06cf2666
AD		 2894/**
2895 * igb_setup_mrqc - configure the multiple receive queue control registers
2896 * @adapter: Board private structure
2897 **/
2898static void igb_setup_mrqc(struct igb_adapter *adapter)
2899{
2900 struct e1000_hw *hw = &adapter->hw;
2901 u32 mrqc, rxcsum;
2902 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2903 union e1000_reta {
2904 u32 dword;
2905 u8 bytes[4];
2906 } reta;
2907 static const u8 rsshash[40] = {
2908 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2909 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2910 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2911 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2912
2913 /* Fill out hash function seeds */
2914 for (j = 0; j < 10; j++) {
2915 u32 rsskey = rsshash[(j * 4)];
2916 rsskey |= rsshash[(j * 4) + 1] << 8;
2917 rsskey |= rsshash[(j * 4) + 2] << 16;
2918 rsskey |= rsshash[(j * 4) + 3] << 24;
2919 array_wr32(E1000_RSSRK(0), j, rsskey);
2920 }
2921
a99955fc 2922 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2923
2924 if (adapter->vfs_allocated_count) {
2925 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2926 switch (hw->mac.type) {
d2ba2ed8 2927 case e1000_i350:
55cac248
AD		 2928 case e1000_82580:
2929 num_rx_queues = 1;
2930 shift = 0;
2931 break;
06cf2666
AD		 2932 case e1000_82576:
2933 shift = 3;
2934 num_rx_queues = 2;
2935 break;
2936 case e1000_82575:
2937 shift = 2;
2938 shift2 = 6;
2939 default:
2940 break;
2941 }
2942 } else {
2943 if (hw->mac.type == e1000_82575)
2944 shift = 6;
2945 }
2946
2947 for (j = 0; j < (32 * 4); j++) {
2948 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2949 if (shift2)
2950 reta.bytes[j & 3] |= num_rx_queues << shift2;
2951 if ((j & 3) == 3)
2952 wr32(E1000_RETA(j >> 2), reta.dword);
2953 }
2954
2955 /*
2956 * Disable raw packet checksumming so that RSS hash is placed in
2957 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2958 * offloads as they are enabled by default
2959 */
2960 rxcsum = rd32(E1000_RXCSUM);
2961 rxcsum |= E1000_RXCSUM_PCSD;
2962
2963 if (adapter->hw.mac.type >= e1000_82576)
2964 /* Enable Receive Checksum Offload for SCTP */
2965 rxcsum |= E1000_RXCSUM_CRCOFL;
2966
2967 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2968 wr32(E1000_RXCSUM, rxcsum);
f96a8a0b
CW		 2969 /*
2970 * Generate RSS hash based on TCP port numbers and/or
2971 * IPv4/v6 src and dst addresses since UDP cannot be
2972 * hashed reliably due to IP fragmentation
2973 */
2974
2975 mrqc = E1000_MRQC_RSS_FIELD_IPV4 |
2976 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2977 E1000_MRQC_RSS_FIELD_IPV6 |
2978 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2979 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
06cf2666
AD		 2980
2981 /* If VMDq is enabled then we set the appropriate mode for that, else
2982 * we default to RSS so that an RSS hash is calculated per packet even
2983 * if we are only using one queue */
2984 if (adapter->vfs_allocated_count) {
2985 if (hw->mac.type > e1000_82575) {
2986 /* Set the default pool for the PF's first queue */
2987 u32 vtctl = rd32(E1000_VT_CTL);
2988 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2989 E1000_VT_CTL_DISABLE_DEF_POOL);
2990 vtctl |= adapter->vfs_allocated_count <<
2991 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2992 wr32(E1000_VT_CTL, vtctl);
2993 }
a99955fc 2994 if (adapter->rss_queues > 1)
f96a8a0b 2995 mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
06cf2666 2996 else
f96a8a0b 2997 mrqc |= E1000_MRQC_ENABLE_VMDQ;
06cf2666 2998 } else {
f96a8a0b
CW		 2999 if (hw->mac.type != e1000_i211)
3000 mrqc |= E1000_MRQC_ENABLE_RSS_4Q;
06cf2666
AD		 3001 }
3002 igb_vmm_control(adapter);
3003
06cf2666
AD		 3004 wr32(E1000_MRQC, mrqc);
3005}
3006
9d5c8243
AK		 3007/**
3008 * igb_setup_rctl - configure the receive control registers
3009 * @adapter: Board private structure
3010 **/
d7ee5b3a 3011void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 3012{
3013 struct e1000_hw *hw = &adapter->hw;
3014 u32 rctl;
9d5c8243
AK		 3015
3016 rctl = rd32(E1000_RCTL);
3017
3018 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 3019 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 3020
69d728ba 3021 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 3022 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 3023
87cb7e8c
AK		 3024 /*
3025 * enable stripping of CRC. It's unlikely this will break BMC
3026 * redirection as it did with e1000. Newer features require
3027 * that the HW strips the CRC.
73cd78f1 3028 */
87cb7e8c 3029 rctl |= E1000_RCTL_SECRC;
9d5c8243 3030
559e9c49 3031 /* disable store bad packets and clear size bits. */
ec54d7d6 3032 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 3033
6ec43fe6
AD		 3034 /* enable LPE to prevent packets larger than max_frame_size */
3035 rctl |= E1000_RCTL_LPE;
9d5c8243 3036
952f72a8
AD		 3037 /* disable queue 0 to prevent tail write w/o re-config */
3038 wr32(E1000_RXDCTL(0), 0);
9d5c8243 3039
e1739522
AD		 3040 /* Attention!!! For SR-IOV PF driver operations you must enable
3041 * queue drop for all VF and PF queues to prevent head of line blocking
3042 * if an un-trusted VF does not provide descriptors to hardware.
3043 */
3044 if (adapter->vfs_allocated_count) {
e1739522
AD		 3045 /* set all queue drop enable bits */
3046 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 3047 }
3048
89eaefb6
BG		 3049 /* This is useful for sniffing bad packets. */
3050 if (adapter->netdev->features & NETIF_F_RXALL) {
3051 /* UPE and MPE will be handled by normal PROMISC logic
3052 * in e1000e_set_rx_mode */
3053 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
3054 E1000_RCTL_BAM | /* RX All Bcast Pkts */
3055 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
3056
3057 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
3058 E1000_RCTL_DPF | /* Allow filtered pause */
3059 E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
3060 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
3061 * and that breaks VLANs.
3062 */
3063 }
3064
9d5c8243
AK		 3065 wr32(E1000_RCTL, rctl);
3066}
3067
7d5753f0
AD		 3068static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
3069 int vfn)
3070{
3071 struct e1000_hw *hw = &adapter->hw;
3072 u32 vmolr;
3073
3074 /* if it isn't the PF check to see if VFs are enabled and
3075 * increase the size to support vlan tags */
3076 if (vfn < adapter->vfs_allocated_count &&
3077 adapter->vf_data[vfn].vlans_enabled)
3078 size += VLAN_TAG_SIZE;
3079
3080 vmolr = rd32(E1000_VMOLR(vfn));
3081 vmolr &= ~E1000_VMOLR_RLPML_MASK;
3082 vmolr |= size | E1000_VMOLR_LPE;
3083 wr32(E1000_VMOLR(vfn), vmolr);
3084
3085 return 0;
3086}
3087
e1739522
AD		 3088/**
3089 * igb_rlpml_set - set maximum receive packet size
3090 * @adapter: board private structure
3091 *
3092 * Configure maximum receivable packet size.
3093 **/
3094static void igb_rlpml_set(struct igb_adapter *adapter)
3095{
153285f9 3096 u32 max_frame_size = adapter->max_frame_size;
e1739522
AD		 3097 struct e1000_hw *hw = &adapter->hw;
3098 u16 pf_id = adapter->vfs_allocated_count;
3099
e1739522
AD		 3100 if (pf_id) {
3101 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
153285f9
AD		 3102 /*
3103 * If we're in VMDQ or SR-IOV mode, then set global RLPML
3104 * to our max jumbo frame size, in case we need to enable
3105 * jumbo frames on one of the rings later.
3106 * This will not pass over-length frames into the default
3107 * queue because it's gated by the VMOLR.RLPML.
3108 */
7d5753f0 3109 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 3110 }
3111
3112 wr32(E1000_RLPML, max_frame_size);
3113}
3114
8151d294
WM		 3115static inline void igb_set_vmolr(struct igb_adapter *adapter,
3116 int vfn, bool aupe)
7d5753f0
AD		 3117{
3118 struct e1000_hw *hw = &adapter->hw;
3119 u32 vmolr;
3120
3121 /*
3122 * This register exists only on 82576 and newer so if we are older then
3123 * we should exit and do nothing
3124 */
3125 if (hw->mac.type < e1000_82576)
3126 return;
3127
3128 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 3129 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
3130 if (aupe)
3131 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
3132 else
3133 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 3134
3135 /* clear all bits that might not be set */
3136 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
3137
a99955fc 3138 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 3139 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3140 /*
3141 * for VMDq only allow the VFs and pool 0 to accept broadcast and
3142 * multicast packets
3143 */
3144 if (vfn <= adapter->vfs_allocated_count)
3145 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
3146
3147 wr32(E1000_VMOLR(vfn), vmolr);
3148}
3149
85b430b4
AD		 3150/**
3151 * igb_configure_rx_ring - Configure a receive ring after Reset
3152 * @adapter: board private structure
3153 * @ring: receive ring to be configured
3154 *
3155 * Configure the Rx unit of the MAC after a reset.
3156 **/
d7ee5b3a
AD		 3157void igb_configure_rx_ring(struct igb_adapter *adapter,
3158 struct igb_ring *ring)
85b430b4
AD		 3159{
3160 struct e1000_hw *hw = &adapter->hw;
3161 u64 rdba = ring->dma;
3162 int reg_idx = ring->reg_idx;
a74420e0 3163 u32 srrctl = 0, rxdctl = 0;
85b430b4
AD		 3164
3165 /* disable the queue */
a74420e0 3166 wr32(E1000_RXDCTL(reg_idx), 0);
85b430b4
AD		 3167
3168 /* Set DMA base address registers */
3169 wr32(E1000_RDBAL(reg_idx),
3170 rdba & 0x00000000ffffffffULL);
3171 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
3172 wr32(E1000_RDLEN(reg_idx),
3173 ring->count * sizeof(union e1000_adv_rx_desc));
3174
3175 /* initialize head and tail */
fce99e34 3176 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
a74420e0 3177 wr32(E1000_RDH(reg_idx), 0);
fce99e34 3178 writel(0, ring->tail);
85b430b4 3179
952f72a8 3180 /* set descriptor configuration */
44390ca6 3181 srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
952f72a8 3182#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
44390ca6 3183 srrctl |= IGB_RXBUFFER_16384 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
952f72a8 3184#else
44390ca6 3185 srrctl |= (PAGE_SIZE / 2) >> E1000_SRRCTL_BSIZEPKT_SHIFT;
952f72a8 3186#endif
44390ca6 3187 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
06218a8d 3188 if (hw->mac.type >= e1000_82580)
757b77e2 3189 srrctl |= E1000_SRRCTL_TIMESTAMP;
e6bdb6fe
NN		 3190 /* Only set Drop Enable if we are supporting multiple queues */
3191 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
3192 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 3193
3194 wr32(E1000_SRRCTL(reg_idx), srrctl);
3195
7d5753f0 3196 /* set filtering for VMDQ pools */
8151d294 3197 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 3198
85b430b4
AD		 3199 rxdctl |= IGB_RX_PTHRESH;
3200 rxdctl |= IGB_RX_HTHRESH << 8;
3201 rxdctl |= IGB_RX_WTHRESH << 16;
a74420e0
AD		 3202
3203 /* enable receive descriptor fetching */
3204 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
85b430b4
AD		 3205 wr32(E1000_RXDCTL(reg_idx), rxdctl);
3206}
3207
9d5c8243
AK		 3208/**
3209 * igb_configure_rx - Configure receive Unit after Reset
3210 * @adapter: board private structure
3211 *
3212 * Configure the Rx unit of the MAC after a reset.
3213 **/
3214static void igb_configure_rx(struct igb_adapter *adapter)
3215{
9107584e 3216 int i;
9d5c8243 3217
68d480c4
AD		 3218 /* set UTA to appropriate mode */
3219 igb_set_uta(adapter);
3220
26ad9178
AD		 3221 /* set the correct pool for the PF default MAC address in entry 0 */
3222 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
3223 adapter->vfs_allocated_count);
3224
06cf2666
AD		 3225 /* Setup the HW Rx Head and Tail Descriptor Pointers and
3226 * the Base and Length of the Rx Descriptor Ring */
3227 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3228 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 3229}
3230
3231/**
3232 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 3233 * @tx_ring: Tx descriptor ring for a specific queue
3234 *
3235 * Free all transmit software resources
3236 **/
68fd9910 3237void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 3238{
3b644cf6 3239 igb_clean_tx_ring(tx_ring);
9d5c8243 3240
06034649
AD		 3241 vfree(tx_ring->tx_buffer_info);
3242 tx_ring->tx_buffer_info = NULL;
9d5c8243 3243
439705e1
AD		 3244 /* if not set, then don't free */
3245 if (!tx_ring->desc)
3246 return;
3247
59d71989
AD		 3248 dma_free_coherent(tx_ring->dev, tx_ring->size,
3249 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 3250
3251 tx_ring->desc = NULL;
3252}
3253
3254/**
3255 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3256 * @adapter: board private structure
3257 *
3258 * Free all transmit software resources
3259 **/
3260static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3261{
3262 int i;
3263
3264 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3265 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 3266}
3267
ebe42d16
AD		 3268void igb_unmap_and_free_tx_resource(struct igb_ring *ring,
3269 struct igb_tx_buffer *tx_buffer)
3270{
3271 if (tx_buffer->skb) {
3272 dev_kfree_skb_any(tx_buffer->skb);
3273 if (tx_buffer->dma)
3274 dma_unmap_single(ring->dev,
3275 tx_buffer->dma,
3276 tx_buffer->length,
3277 DMA_TO_DEVICE);
3278 } else if (tx_buffer->dma) {
3279 dma_unmap_page(ring->dev,
3280 tx_buffer->dma,
3281 tx_buffer->length,
3282 DMA_TO_DEVICE);
3283 }
3284 tx_buffer->next_to_watch = NULL;
3285 tx_buffer->skb = NULL;
3286 tx_buffer->dma = 0;
3287 /* buffer_info must be completely set up in the transmit path */
9d5c8243
AK		 3288}
3289
3290/**
3291 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 3292 * @tx_ring: ring to be cleaned
3293 **/
3b644cf6 3294static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243 3295{
06034649 3296 struct igb_tx_buffer *buffer_info;
9d5c8243 3297 unsigned long size;
6ad4edfc 3298 u16 i;
9d5c8243 3299
06034649 3300 if (!tx_ring->tx_buffer_info)
9d5c8243
AK		 3301 return;
3302 /* Free all the Tx ring sk_buffs */
3303
3304 for (i = 0; i < tx_ring->count; i++) {
06034649 3305 buffer_info = &tx_ring->tx_buffer_info[i];
80785298 3306 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 3307 }
3308
dad8a3b3
JF		 3309 netdev_tx_reset_queue(txring_txq(tx_ring));
3310
06034649
AD		 3311 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
3312 memset(tx_ring->tx_buffer_info, 0, size);
9d5c8243
AK		 3313
3314 /* Zero out the descriptor ring */
9d5c8243
AK		 3315 memset(tx_ring->desc, 0, tx_ring->size);
3316
3317 tx_ring->next_to_use = 0;
3318 tx_ring->next_to_clean = 0;
9d5c8243
AK		 3319}
3320
3321/**
3322 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3323 * @adapter: board private structure
3324 **/
3325static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3326{
3327 int i;
3328
3329 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3330 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 3331}
3332
3333/**
3334 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 3335 * @rx_ring: ring to clean the resources from
3336 *
3337 * Free all receive software resources
3338 **/
68fd9910 3339void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 3340{
3b644cf6 3341 igb_clean_rx_ring(rx_ring);
9d5c8243 3342
06034649
AD		 3343 vfree(rx_ring->rx_buffer_info);
3344 rx_ring->rx_buffer_info = NULL;
9d5c8243 3345
439705e1
AD		 3346 /* if not set, then don't free */
3347 if (!rx_ring->desc)
3348 return;
3349
59d71989
AD		 3350 dma_free_coherent(rx_ring->dev, rx_ring->size,
3351 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 3352
3353 rx_ring->desc = NULL;
3354}
3355
3356/**
3357 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3358 * @adapter: board private structure
3359 *
3360 * Free all receive software resources
3361 **/
3362static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3363{
3364 int i;
3365
3366 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3367 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 3368}
3369
3370/**
3371 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 3372 * @rx_ring: ring to free buffers from
3373 **/
3b644cf6 3374static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243 3375{
9d5c8243 3376 unsigned long size;
c023cd88 3377 u16 i;
9d5c8243 3378
06034649 3379 if (!rx_ring->rx_buffer_info)
9d5c8243 3380 return;
439705e1 3381
9d5c8243
AK		 3382 /* Free all the Rx ring sk_buffs */
3383 for (i = 0; i < rx_ring->count; i++) {
06034649 3384 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
9d5c8243 3385 if (buffer_info->dma) {
59d71989 3386 dma_unmap_single(rx_ring->dev,
80785298 3387 buffer_info->dma,
44390ca6 3388 IGB_RX_HDR_LEN,
59d71989 3389 DMA_FROM_DEVICE);
9d5c8243
AK		 3390 buffer_info->dma = 0;
3391 }
3392
3393 if (buffer_info->skb) {
3394 dev_kfree_skb(buffer_info->skb);
3395 buffer_info->skb = NULL;
3396 }
6ec43fe6 3397 if (buffer_info->page_dma) {
59d71989 3398 dma_unmap_page(rx_ring->dev,
80785298 3399 buffer_info->page_dma,
6ec43fe6 3400 PAGE_SIZE / 2,
59d71989 3401 DMA_FROM_DEVICE);
6ec43fe6
AD		 3402 buffer_info->page_dma = 0;
3403 }
9d5c8243 3404 if (buffer_info->page) {
9d5c8243
AK		 3405 put_page(buffer_info->page);
3406 buffer_info->page = NULL;
bf36c1a0 3407 buffer_info->page_offset = 0;
9d5c8243
AK		 3408 }
3409 }
3410
06034649
AD		 3411 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3412 memset(rx_ring->rx_buffer_info, 0, size);
9d5c8243
AK		 3413
3414 /* Zero out the descriptor ring */
3415 memset(rx_ring->desc, 0, rx_ring->size);
3416
3417 rx_ring->next_to_clean = 0;
3418 rx_ring->next_to_use = 0;
9d5c8243
AK		 3419}
3420
3421/**
3422 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3423 * @adapter: board private structure
3424 **/
3425static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3426{
3427 int i;
3428
3429 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3430 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 3431}
3432
3433/**
3434 * igb_set_mac - Change the Ethernet Address of the NIC
3435 * @netdev: network interface device structure
3436 * @p: pointer to an address structure
3437 *
3438 * Returns 0 on success, negative on failure
3439 **/
3440static int igb_set_mac(struct net_device *netdev, void *p)
3441{
3442 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 3443 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 3444 struct sockaddr *addr = p;
3445
3446 if (!is_valid_ether_addr(addr->sa_data))
3447 return -EADDRNOTAVAIL;
3448
3449 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 3450 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 3451
26ad9178
AD		 3452 /* set the correct pool for the new PF MAC address in entry 0 */
3453 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3454 adapter->vfs_allocated_count);
e1739522 3455
9d5c8243
AK		 3456 return 0;
3457}
3458
3459/**
68d480c4 3460 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 3461 * @netdev: network interface device structure
3462 *
68d480c4
AD		 3463 * Writes multicast address list to the MTA hash table.
3464 * Returns: -ENOMEM on failure
3465 * 0 on no addresses written
3466 * X on writing X addresses to MTA
9d5c8243 3467 **/
68d480c4 3468static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 3469{
3470 struct igb_adapter *adapter = netdev_priv(netdev);
3471 struct e1000_hw *hw = &adapter->hw;
22bedad3 3472 struct netdev_hw_addr *ha;
68d480c4 3473 u8 *mta_list;
9d5c8243
AK		 3474 int i;
3475
4cd24eaf 3476 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 3477 /* nothing to program, so clear mc list */
3478 igb_update_mc_addr_list(hw, NULL, 0);
3479 igb_restore_vf_multicasts(adapter);
3480 return 0;
3481 }
9d5c8243 3482
4cd24eaf 3483 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 3484 if (!mta_list)
3485 return -ENOMEM;
ff41f8dc 3486
68d480c4 3487 /* The shared function expects a packed array of only addresses. */
48e2f183 3488 i = 0;
22bedad3
JP		 3489 netdev_for_each_mc_addr(ha, netdev)
3490 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
68d480c4 3491
68d480c4
AD		 3492 igb_update_mc_addr_list(hw, mta_list, i);
3493 kfree(mta_list);
3494
4cd24eaf 3495 return netdev_mc_count(netdev);
68d480c4
AD		 3496}
3497
3498/**
3499 * igb_write_uc_addr_list - write unicast addresses to RAR table
3500 * @netdev: network interface device structure
3501 *
3502 * Writes unicast address list to the RAR table.
3503 * Returns: -ENOMEM on failure/insufficient address space
3504 * 0 on no addresses written
3505 * X on writing X addresses to the RAR table
3506 **/
3507static int igb_write_uc_addr_list(struct net_device *netdev)
3508{
3509 struct igb_adapter *adapter = netdev_priv(netdev);
3510 struct e1000_hw *hw = &adapter->hw;
3511 unsigned int vfn = adapter->vfs_allocated_count;
3512 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3513 int count = 0;
3514
3515 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 3516 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 3517 return -ENOMEM;
9d5c8243 3518
32e7bfc4 3519 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 3520 struct netdev_hw_addr *ha;
32e7bfc4
JP		 3521
3522 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 3523 if (!rar_entries)
3524 break;
26ad9178
AD		 3525 igb_rar_set_qsel(adapter, ha->addr,
3526 rar_entries--,
68d480c4
AD		 3527 vfn);
3528 count++;
ff41f8dc
AD		 3529 }
3530 }
3531 /* write the addresses in reverse order to avoid write combining */
3532 for (; rar_entries > 0 ; rar_entries--) {
3533 wr32(E1000_RAH(rar_entries), 0);
3534 wr32(E1000_RAL(rar_entries), 0);
3535 }
3536 wrfl();
3537
68d480c4
AD		 3538 return count;
3539}
3540
3541/**
3542 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3543 * @netdev: network interface device structure
3544 *
3545 * The set_rx_mode entry point is called whenever the unicast or multicast
3546 * address lists or the network interface flags are updated. This routine is
3547 * responsible for configuring the hardware for proper unicast, multicast,
3548 * promiscuous mode, and all-multi behavior.
3549 **/
3550static void igb_set_rx_mode(struct net_device *netdev)
3551{
3552 struct igb_adapter *adapter = netdev_priv(netdev);
3553 struct e1000_hw *hw = &adapter->hw;
3554 unsigned int vfn = adapter->vfs_allocated_count;
3555 u32 rctl, vmolr = 0;
3556 int count;
3557
3558 /* Check for Promiscuous and All Multicast modes */
3559 rctl = rd32(E1000_RCTL);
3560
3561 /* clear the effected bits */
3562 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3563
3564 if (netdev->flags & IFF_PROMISC) {
3565 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3566 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3567 } else {
3568 if (netdev->flags & IFF_ALLMULTI) {
3569 rctl |= E1000_RCTL_MPE;
3570 vmolr |= E1000_VMOLR_MPME;
3571 } else {
3572 /*
3573 * Write addresses to the MTA, if the attempt fails
25985edc 3574 * then we should just turn on promiscuous mode so
68d480c4
AD		 3575 * that we can at least receive multicast traffic
3576 */
3577 count = igb_write_mc_addr_list(netdev);
3578 if (count < 0) {
3579 rctl |= E1000_RCTL_MPE;
3580 vmolr |= E1000_VMOLR_MPME;
3581 } else if (count) {
3582 vmolr |= E1000_VMOLR_ROMPE;
3583 }
3584 }
3585 /*
3586 * Write addresses to available RAR registers, if there is not
3587 * sufficient space to store all the addresses then enable
25985edc 3588 * unicast promiscuous mode
68d480c4
AD		 3589 */
3590 count = igb_write_uc_addr_list(netdev);
3591 if (count < 0) {
3592 rctl |= E1000_RCTL_UPE;
3593 vmolr |= E1000_VMOLR_ROPE;
3594 }
3595 rctl |= E1000_RCTL_VFE;
28fc06f5 3596 }
68d480c4 3597 wr32(E1000_RCTL, rctl);
28fc06f5 3598
68d480c4
AD		 3599 /*
3600 * In order to support SR-IOV and eventually VMDq it is necessary to set
3601 * the VMOLR to enable the appropriate modes. Without this workaround
3602 * we will have issues with VLAN tag stripping not being done for frames
3603 * that are only arriving because we are the default pool
3604 */
f96a8a0b 3605 if ((hw->mac.type < e1000_82576) || (hw->mac.type > e1000_i350))
28fc06f5 3606 return;
9d5c8243 3607
68d480c4
AD		 3608 vmolr |= rd32(E1000_VMOLR(vfn)) &
3609 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3610 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3611 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3612}
3613
13800469
GR		 3614static void igb_check_wvbr(struct igb_adapter *adapter)
3615{
3616 struct e1000_hw *hw = &adapter->hw;
3617 u32 wvbr = 0;
3618
3619 switch (hw->mac.type) {
3620 case e1000_82576:
3621 case e1000_i350:
3622 if (!(wvbr = rd32(E1000_WVBR)))
3623 return;
3624 break;
3625 default:
3626 break;
3627 }
3628
3629 adapter->wvbr |= wvbr;
3630}
3631
3632#define IGB_STAGGERED_QUEUE_OFFSET 8
3633
3634static void igb_spoof_check(struct igb_adapter *adapter)
3635{
3636 int j;
3637
3638 if (!adapter->wvbr)
3639 return;
3640
3641 for(j = 0; j < adapter->vfs_allocated_count; j++) {
3642 if (adapter->wvbr & (1 << j) ||
3643 adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3644 dev_warn(&adapter->pdev->dev,
3645 "Spoof event(s) detected on VF %d\n", j);
3646 adapter->wvbr &=
3647 ~((1 << j) |
3648 (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3649 }
3650 }
3651}
3652
9d5c8243
AK		 3653/* Need to wait a few seconds after link up to get diagnostic information from
3654 * the phy */
3655static void igb_update_phy_info(unsigned long data)
3656{
3657 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3658 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3659}
3660
4d6b725e
AD		 3661/**
3662 * igb_has_link - check shared code for link and determine up/down
3663 * @adapter: pointer to driver private info
3664 **/
3145535a 3665bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3666{
3667 struct e1000_hw *hw = &adapter->hw;
3668 bool link_active = false;
3669 s32 ret_val = 0;
3670
3671 /* get_link_status is set on LSC (link status) interrupt or
3672 * rx sequence error interrupt. get_link_status will stay
3673 * false until the e1000_check_for_link establishes link
3674 * for copper adapters ONLY
3675 */
3676 switch (hw->phy.media_type) {
3677 case e1000_media_type_copper:
3678 if (hw->mac.get_link_status) {
3679 ret_val = hw->mac.ops.check_for_link(hw);
3680 link_active = !hw->mac.get_link_status;
3681 } else {
3682 link_active = true;
3683 }
3684 break;
4d6b725e
AD		 3685 case e1000_media_type_internal_serdes:
3686 ret_val = hw->mac.ops.check_for_link(hw);
3687 link_active = hw->mac.serdes_has_link;
3688 break;
3689 default:
3690 case e1000_media_type_unknown:
3691 break;
3692 }
3693
3694 return link_active;
3695}
3696
563988dc
SA		 3697static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
3698{
3699 bool ret = false;
3700 u32 ctrl_ext, thstat;
3701
f96a8a0b 3702 /* check for thermal sensor event on i350 copper only */
563988dc
SA		 3703 if (hw->mac.type == e1000_i350) {
3704 thstat = rd32(E1000_THSTAT);
3705 ctrl_ext = rd32(E1000_CTRL_EXT);
3706
3707 if ((hw->phy.media_type == e1000_media_type_copper) &&
3708 !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3709 ret = !!(thstat & event);
3710 }
3711 }
3712
3713 return ret;
3714}
3715
9d5c8243
AK		 3716/**
3717 * igb_watchdog - Timer Call-back
3718 * @data: pointer to adapter cast into an unsigned long
3719 **/
3720static void igb_watchdog(unsigned long data)
3721{
3722 struct igb_adapter *adapter = (struct igb_adapter *)data;
3723 /* Do the rest outside of interrupt context */
3724 schedule_work(&adapter->watchdog_task);
3725}
3726
3727static void igb_watchdog_task(struct work_struct *work)
3728{
3729 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3730 struct igb_adapter,
3731 watchdog_task);
9d5c8243 3732 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3733 struct net_device *netdev = adapter->netdev;
563988dc 3734 u32 link;
7a6ea550 3735 int i;
9d5c8243 3736
4d6b725e 3737 link = igb_has_link(adapter);
9d5c8243 3738 if (link) {
749ab2cd
YZ		 3739 /* Cancel scheduled suspend requests. */
3740 pm_runtime_resume(netdev->dev.parent);
3741
9d5c8243
AK		 3742 if (!netif_carrier_ok(netdev)) {
3743 u32 ctrl;
330a6d6a
AD		 3744 hw->mac.ops.get_speed_and_duplex(hw,
3745 &adapter->link_speed,
3746 &adapter->link_duplex);
9d5c8243
AK		 3747
3748 ctrl = rd32(E1000_CTRL);
527d47c1 3749 /* Links status message must follow this format */
876d2d6f
JK		 3750 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s "
3751 "Duplex, Flow Control: %s\n",
559e9c49
AD		 3752 netdev->name,
3753 adapter->link_speed,
3754 adapter->link_duplex == FULL_DUPLEX ?
876d2d6f
JK		 3755 "Full" : "Half",
3756 (ctrl & E1000_CTRL_TFCE) &&
3757 (ctrl & E1000_CTRL_RFCE) ? "RX/TX" :
3758 (ctrl & E1000_CTRL_RFCE) ? "RX" :
3759 (ctrl & E1000_CTRL_TFCE) ? "TX" : "None");
9d5c8243 3760
563988dc 3761 /* check for thermal sensor event */
876d2d6f
JK		 3762 if (igb_thermal_sensor_event(hw,
3763 E1000_THSTAT_LINK_THROTTLE)) {
3764 netdev_info(netdev, "The network adapter link "
3765 "speed was downshifted because it "
3766 "overheated\n");
7ef5ed1c 3767 }
563988dc 3768
d07f3e37 3769 /* adjust timeout factor according to speed/duplex */
9d5c8243
AK		 3770 adapter->tx_timeout_factor = 1;
3771 switch (adapter->link_speed) {
3772 case SPEED_10:
9d5c8243
AK		 3773 adapter->tx_timeout_factor = 14;
3774 break;
3775 case SPEED_100:
9d5c8243
AK		 3776 /* maybe add some timeout factor ? */
3777 break;
3778 }
3779
3780 netif_carrier_on(netdev);
9d5c8243 3781
4ae196df 3782 igb_ping_all_vfs(adapter);
17dc566c 3783 igb_check_vf_rate_limit(adapter);
4ae196df 3784
4b1a9877 3785 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3786 if (!test_bit(__IGB_DOWN, &adapter->state))
3787 mod_timer(&adapter->phy_info_timer,
3788 round_jiffies(jiffies + 2 * HZ));
3789 }
3790 } else {
3791 if (netif_carrier_ok(netdev)) {
3792 adapter->link_speed = 0;
3793 adapter->link_duplex = 0;
563988dc
SA		 3794
3795 /* check for thermal sensor event */
876d2d6f
JK		 3796 if (igb_thermal_sensor_event(hw,
3797 E1000_THSTAT_PWR_DOWN)) {
3798 netdev_err(netdev, "The network adapter was "
3799 "stopped because it overheated\n");
7ef5ed1c 3800 }
563988dc 3801
527d47c1
AD		 3802 /* Links status message must follow this format */
3803 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3804 netdev->name);
9d5c8243 3805 netif_carrier_off(netdev);
4b1a9877 3806
4ae196df
AD		 3807 igb_ping_all_vfs(adapter);
3808
4b1a9877 3809 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3810 if (!test_bit(__IGB_DOWN, &adapter->state))
3811 mod_timer(&adapter->phy_info_timer,
3812 round_jiffies(jiffies + 2 * HZ));
749ab2cd
YZ		 3813
3814 pm_schedule_suspend(netdev->dev.parent,
3815 MSEC_PER_SEC * 5);
9d5c8243
AK		 3816 }
3817 }
3818
12dcd86b
ED		 3819 spin_lock(&adapter->stats64_lock);
3820 igb_update_stats(adapter, &adapter->stats64);
3821 spin_unlock(&adapter->stats64_lock);
9d5c8243 3822
dbabb065 3823 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3824 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3825 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3826 /* We've lost link, so the controller stops DMA,
3827 * but we've got queued Tx work that's never going
3828 * to get done, so reset controller to flush Tx.
3829 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3830 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3831 adapter->tx_timeout_count++;
3832 schedule_work(&adapter->reset_task);
3833 /* return immediately since reset is imminent */
3834 return;
3835 }
9d5c8243 3836 }
9d5c8243 3837
dbabb065 3838 /* Force detection of hung controller every watchdog period */
6d095fa8 3839 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
dbabb065 3840 }
f7ba205e 3841
9d5c8243 3842 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3843 if (adapter->msix_entries) {
047e0030 3844 u32 eics = 0;
0d1ae7f4
AD		 3845 for (i = 0; i < adapter->num_q_vectors; i++)
3846 eics |= adapter->q_vector[i]->eims_value;
7a6ea550
AD		 3847 wr32(E1000_EICS, eics);
3848 } else {
3849 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3850 }
9d5c8243 3851
13800469
GR		 3852 igb_spoof_check(adapter);
3853
9d5c8243
AK		 3854 /* Reset the timer */
3855 if (!test_bit(__IGB_DOWN, &adapter->state))
3856 mod_timer(&adapter->watchdog_timer,
3857 round_jiffies(jiffies + 2 * HZ));
3858}
3859
3860enum latency_range {
3861 lowest_latency = 0,
3862 low_latency = 1,
3863 bulk_latency = 2,
3864 latency_invalid = 255
3865};
3866
6eb5a7f1
AD		 3867/**
3868 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3869 *
3870 * Stores a new ITR value based on strictly on packet size. This
3871 * algorithm is less sophisticated than that used in igb_update_itr,
3872 * due to the difficulty of synchronizing statistics across multiple
eef35c2d 3873 * receive rings. The divisors and thresholds used by this function
6eb5a7f1
AD		 3874 * were determined based on theoretical maximum wire speed and testing
3875 * data, in order to minimize response time while increasing bulk
3876 * throughput.
3877 * This functionality is controlled by the InterruptThrottleRate module
3878 * parameter (see igb_param.c)
3879 * NOTE: This function is called only when operating in a multiqueue
3880 * receive environment.
047e0030 3881 * @q_vector: pointer to q_vector
6eb5a7f1 3882 **/
047e0030 3883static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3884{
047e0030 3885 int new_val = q_vector->itr_val;
6eb5a7f1 3886 int avg_wire_size = 0;
047e0030 3887 struct igb_adapter *adapter = q_vector->adapter;
12dcd86b 3888 unsigned int packets;
9d5c8243 3889
6eb5a7f1
AD		 3890 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3891 * ints/sec - ITR timer value of 120 ticks.
3892 */
3893 if (adapter->link_speed != SPEED_1000) {
0ba82994 3894 new_val = IGB_4K_ITR;
6eb5a7f1 3895 goto set_itr_val;
9d5c8243 3896 }
047e0030 3897
0ba82994
AD		 3898 packets = q_vector->rx.total_packets;
3899 if (packets)
3900 avg_wire_size = q_vector->rx.total_bytes / packets;
047e0030 3901
0ba82994
AD		 3902 packets = q_vector->tx.total_packets;
3903 if (packets)
3904 avg_wire_size = max_t(u32, avg_wire_size,
3905 q_vector->tx.total_bytes / packets);
047e0030
AD		 3906
3907 /* if avg_wire_size isn't set no work was done */
3908 if (!avg_wire_size)
3909 goto clear_counts;
9d5c8243 3910
6eb5a7f1
AD		 3911 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3912 avg_wire_size += 24;
3913
3914 /* Don't starve jumbo frames */
3915 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3916
6eb5a7f1
AD		 3917 /* Give a little boost to mid-size frames */
3918 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3919 new_val = avg_wire_size / 3;
3920 else
3921 new_val = avg_wire_size / 2;
9d5c8243 3922
0ba82994
AD		 3923 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3924 if (new_val < IGB_20K_ITR &&
3925 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3926 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3927 new_val = IGB_20K_ITR;
abe1c363 3928
6eb5a7f1 3929set_itr_val:
047e0030
AD		 3930 if (new_val != q_vector->itr_val) {
3931 q_vector->itr_val = new_val;
3932 q_vector->set_itr = 1;
9d5c8243 3933 }
6eb5a7f1 3934clear_counts:
0ba82994
AD		 3935 q_vector->rx.total_bytes = 0;
3936 q_vector->rx.total_packets = 0;
3937 q_vector->tx.total_bytes = 0;
3938 q_vector->tx.total_packets = 0;
9d5c8243
AK		 3939}
3940
3941/**
3942 * igb_update_itr - update the dynamic ITR value based on statistics
3943 * Stores a new ITR value based on packets and byte
3944 * counts during the last interrupt. The advantage of per interrupt
3945 * computation is faster updates and more accurate ITR for the current
3946 * traffic pattern. Constants in this function were computed
3947 * based on theoretical maximum wire speed and thresholds were set based
3948 * on testing data as well as attempting to minimize response time
3949 * while increasing bulk throughput.
3950 * this functionality is controlled by the InterruptThrottleRate module
3951 * parameter (see igb_param.c)
3952 * NOTE: These calculations are only valid when operating in a single-
3953 * queue environment.
0ba82994
AD		 3954 * @q_vector: pointer to q_vector
3955 * @ring_container: ring info to update the itr for
9d5c8243 3956 **/
0ba82994
AD		 3957static void igb_update_itr(struct igb_q_vector *q_vector,
3958 struct igb_ring_container *ring_container)
9d5c8243 3959{
0ba82994
AD		 3960 unsigned int packets = ring_container->total_packets;
3961 unsigned int bytes = ring_container->total_bytes;
3962 u8 itrval = ring_container->itr;
9d5c8243 3963
0ba82994 3964 /* no packets, exit with status unchanged */
9d5c8243 3965 if (packets == 0)
0ba82994 3966 return;
9d5c8243 3967
0ba82994 3968 switch (itrval) {
9d5c8243
AK		 3969 case lowest_latency:
3970 /* handle TSO and jumbo frames */
3971 if (bytes/packets > 8000)
0ba82994 3972 itrval = bulk_latency;
9d5c8243 3973 else if ((packets < 5) && (bytes > 512))
0ba82994 3974 itrval = low_latency;
9d5c8243
AK		 3975 break;
3976 case low_latency: /* 50 usec aka 20000 ints/s */
3977 if (bytes > 10000) {
3978 /* this if handles the TSO accounting */
3979 if (bytes/packets > 8000) {
0ba82994 3980 itrval = bulk_latency;
9d5c8243 3981 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
0ba82994 3982 itrval = bulk_latency;
9d5c8243 3983 } else if ((packets > 35)) {
0ba82994 3984 itrval = lowest_latency;
9d5c8243
AK		 3985 }
3986 } else if (bytes/packets > 2000) {
0ba82994 3987 itrval = bulk_latency;
9d5c8243 3988 } else if (packets <= 2 && bytes < 512) {
0ba82994 3989 itrval = lowest_latency;
9d5c8243
AK		 3990 }
3991 break;
3992 case bulk_latency: /* 250 usec aka 4000 ints/s */
3993 if (bytes > 25000) {
3994 if (packets > 35)
0ba82994 3995 itrval = low_latency;
1e5c3d21 3996 } else if (bytes < 1500) {
0ba82994 3997 itrval = low_latency;
9d5c8243
AK		 3998 }
3999 break;
4000 }
4001
0ba82994
AD		 4002 /* clear work counters since we have the values we need */
4003 ring_container->total_bytes = 0;
4004 ring_container->total_packets = 0;
4005
4006 /* write updated itr to ring container */
4007 ring_container->itr = itrval;
9d5c8243
AK		 4008}
4009
0ba82994 4010static void igb_set_itr(struct igb_q_vector *q_vector)
9d5c8243 4011{
0ba82994 4012 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4013 u32 new_itr = q_vector->itr_val;
0ba82994 4014 u8 current_itr = 0;
9d5c8243
AK		 4015
4016 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
4017 if (adapter->link_speed != SPEED_1000) {
4018 current_itr = 0;
0ba82994 4019 new_itr = IGB_4K_ITR;
9d5c8243
AK		 4020 goto set_itr_now;
4021 }
4022
0ba82994
AD		 4023 igb_update_itr(q_vector, &q_vector->tx);
4024 igb_update_itr(q_vector, &q_vector->rx);
9d5c8243 4025
0ba82994 4026 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
9d5c8243 4027
6eb5a7f1 4028 /* conservative mode (itr 3) eliminates the lowest_latency setting */
0ba82994
AD		 4029 if (current_itr == lowest_latency &&
4030 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4031 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
6eb5a7f1
AD		 4032 current_itr = low_latency;
4033
9d5c8243
AK		 4034 switch (current_itr) {
4035 /* counts and packets in update_itr are dependent on these numbers */
4036 case lowest_latency:
0ba82994 4037 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
9d5c8243
AK		 4038 break;
4039 case low_latency:
0ba82994 4040 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
9d5c8243
AK		 4041 break;
4042 case bulk_latency:
0ba82994 4043 new_itr = IGB_4K_ITR; /* 4,000 ints/sec */
9d5c8243
AK		 4044 break;
4045 default:
4046 break;
4047 }
4048
4049set_itr_now:
047e0030 4050 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 4051 /* this attempts to bias the interrupt rate towards Bulk
4052 * by adding intermediate steps when interrupt rate is
4053 * increasing */
047e0030
AD		 4054 new_itr = new_itr > q_vector->itr_val ?
4055 max((new_itr * q_vector->itr_val) /
4056 (new_itr + (q_vector->itr_val >> 2)),
0ba82994 4057 new_itr) :
9d5c8243
AK		 4058 new_itr;
4059 /* Don't write the value here; it resets the adapter's
4060 * internal timer, and causes us to delay far longer than
4061 * we should between interrupts. Instead, we write the ITR
4062 * value at the beginning of the next interrupt so the timing
4063 * ends up being correct.
4064 */
047e0030
AD		 4065 q_vector->itr_val = new_itr;
4066 q_vector->set_itr = 1;
9d5c8243 4067 }
9d5c8243
AK		 4068}
4069
c50b52a0
SH		 4070static void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
4071 u32 type_tucmd, u32 mss_l4len_idx)
7d13a7d0
AD		 4072{
4073 struct e1000_adv_tx_context_desc *context_desc;
4074 u16 i = tx_ring->next_to_use;
4075
4076 context_desc = IGB_TX_CTXTDESC(tx_ring, i);
4077
4078 i++;
4079 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
4080
4081 /* set bits to identify this as an advanced context descriptor */
4082 type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
4083
4084 /* For 82575, context index must be unique per ring. */
866cff06 4085 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
7d13a7d0
AD		 4086 mss_l4len_idx |= tx_ring->reg_idx << 4;
4087
4088 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
4089 context_desc->seqnum_seed = 0;
4090 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
4091 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
4092}
4093
7af40ad9
AD		 4094static int igb_tso(struct igb_ring *tx_ring,
4095 struct igb_tx_buffer *first,
4096 u8 *hdr_len)
9d5c8243 4097{
7af40ad9 4098 struct sk_buff *skb = first->skb;
7d13a7d0
AD		 4099 u32 vlan_macip_lens, type_tucmd;
4100 u32 mss_l4len_idx, l4len;
4101
4102 if (!skb_is_gso(skb))
4103 return 0;
9d5c8243
AK		 4104
4105 if (skb_header_cloned(skb)) {
7af40ad9 4106 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
9d5c8243
AK		 4107 if (err)
4108 return err;
4109 }
4110
7d13a7d0
AD		 4111 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4112 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
9d5c8243 4113
7af40ad9 4114 if (first->protocol == __constant_htons(ETH_P_IP)) {
9d5c8243
AK		 4115 struct iphdr *iph = ip_hdr(skb);
4116 iph->tot_len = 0;
4117 iph->check = 0;
4118 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
4119 iph->daddr, 0,
4120 IPPROTO_TCP,
4121 0);
7d13a7d0 4122 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
7af40ad9
AD		 4123 first->tx_flags |= IGB_TX_FLAGS_TSO |
4124 IGB_TX_FLAGS_CSUM |
4125 IGB_TX_FLAGS_IPV4;
8e1e8a47 4126 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 4127 ipv6_hdr(skb)->payload_len = 0;
4128 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4129 &ipv6_hdr(skb)->daddr,
4130 0, IPPROTO_TCP, 0);
7af40ad9
AD		 4131 first->tx_flags |= IGB_TX_FLAGS_TSO |
4132 IGB_TX_FLAGS_CSUM;
9d5c8243
AK		 4133 }
4134
7af40ad9 4135 /* compute header lengths */
7d13a7d0
AD		 4136 l4len = tcp_hdrlen(skb);
4137 *hdr_len = skb_transport_offset(skb) + l4len;
9d5c8243 4138
7af40ad9
AD		 4139 /* update gso size and bytecount with header size */
4140 first->gso_segs = skb_shinfo(skb)->gso_segs;
4141 first->bytecount += (first->gso_segs - 1) * *hdr_len;
4142
9d5c8243 4143 /* MSS L4LEN IDX */
7d13a7d0
AD		 4144 mss_l4len_idx = l4len << E1000_ADVTXD_L4LEN_SHIFT;
4145 mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
9d5c8243 4146
7d13a7d0
AD		 4147 /* VLAN MACLEN IPLEN */
4148 vlan_macip_lens = skb_network_header_len(skb);
4149 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
7af40ad9 4150 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
9d5c8243 4151
7d13a7d0 4152 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
9d5c8243 4153
7d13a7d0 4154 return 1;
9d5c8243
AK		 4155}
4156
7af40ad9 4157static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
9d5c8243 4158{
7af40ad9 4159 struct sk_buff *skb = first->skb;
7d13a7d0
AD		 4160 u32 vlan_macip_lens = 0;
4161 u32 mss_l4len_idx = 0;
4162 u32 type_tucmd = 0;
9d5c8243 4163
7d13a7d0 4164 if (skb->ip_summed != CHECKSUM_PARTIAL) {
7af40ad9
AD		 4165 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN))
4166 return;
7d13a7d0
AD		 4167 } else {
4168 u8 l4_hdr = 0;
7af40ad9 4169 switch (first->protocol) {
7d13a7d0
AD		 4170 case __constant_htons(ETH_P_IP):
4171 vlan_macip_lens |= skb_network_header_len(skb);
4172 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4173 l4_hdr = ip_hdr(skb)->protocol;
4174 break;
4175 case __constant_htons(ETH_P_IPV6):
4176 vlan_macip_lens |= skb_network_header_len(skb);
4177 l4_hdr = ipv6_hdr(skb)->nexthdr;
4178 break;
4179 default:
4180 if (unlikely(net_ratelimit())) {
4181 dev_warn(tx_ring->dev,
4182 "partial checksum but proto=%x!\n",
7af40ad9 4183 first->protocol);
fa4a7ef3 4184 }
7d13a7d0
AD		 4185 break;
4186 }
fa4a7ef3 4187
7d13a7d0
AD		 4188 switch (l4_hdr) {
4189 case IPPROTO_TCP:
4190 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4191 mss_l4len_idx = tcp_hdrlen(skb) <<
4192 E1000_ADVTXD_L4LEN_SHIFT;
4193 break;
4194 case IPPROTO_SCTP:
4195 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
4196 mss_l4len_idx = sizeof(struct sctphdr) <<
4197 E1000_ADVTXD_L4LEN_SHIFT;
4198 break;
4199 case IPPROTO_UDP:
4200 mss_l4len_idx = sizeof(struct udphdr) <<
4201 E1000_ADVTXD_L4LEN_SHIFT;
4202 break;
4203 default:
4204 if (unlikely(net_ratelimit())) {
4205 dev_warn(tx_ring->dev,
4206 "partial checksum but l4 proto=%x!\n",
4207 l4_hdr);
44b0cda3 4208 }
7d13a7d0 4209 break;
9d5c8243 4210 }
7af40ad9
AD		 4211
4212 /* update TX checksum flag */
4213 first->tx_flags |= IGB_TX_FLAGS_CSUM;
7d13a7d0 4214 }
9d5c8243 4215
7d13a7d0 4216 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
7af40ad9 4217 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
9d5c8243 4218
7d13a7d0 4219 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
9d5c8243
AK		 4220}
4221
e032afc8
AD		 4222static __le32 igb_tx_cmd_type(u32 tx_flags)
4223{
4224 /* set type for advanced descriptor with frame checksum insertion */
4225 __le32 cmd_type = cpu_to_le32(E1000_ADVTXD_DTYP_DATA |
4226 E1000_ADVTXD_DCMD_IFCS |
4227 E1000_ADVTXD_DCMD_DEXT);
4228
4229 /* set HW vlan bit if vlan is present */
4230 if (tx_flags & IGB_TX_FLAGS_VLAN)
4231 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_VLE);
4232
4233 /* set timestamp bit if present */
4234 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4235 cmd_type |= cpu_to_le32(E1000_ADVTXD_MAC_TSTAMP);
4236
4237 /* set segmentation bits for TSO */
4238 if (tx_flags & IGB_TX_FLAGS_TSO)
4239 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_TSE);
4240
4241 return cmd_type;
4242}
4243
7af40ad9
AD		 4244static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
4245 union e1000_adv_tx_desc *tx_desc,
4246 u32 tx_flags, unsigned int paylen)
e032afc8
AD		 4247{
4248 u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
4249
4250 /* 82575 requires a unique index per ring if any offload is enabled */
4251 if ((tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_VLAN)) &&
866cff06 4252 test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
e032afc8
AD		 4253 olinfo_status |= tx_ring->reg_idx << 4;
4254
4255 /* insert L4 checksum */
4256 if (tx_flags & IGB_TX_FLAGS_CSUM) {
4257 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4258
4259 /* insert IPv4 checksum */
4260 if (tx_flags & IGB_TX_FLAGS_IPV4)
4261 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4262 }
4263
7af40ad9 4264 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
e032afc8
AD		 4265}
4266
ebe42d16
AD		 4267/*
4268 * The largest size we can write to the descriptor is 65535. In order to
4269 * maintain a power of two alignment we have to limit ourselves to 32K.
4270 */
4271#define IGB_MAX_TXD_PWR 15
7af40ad9 4272#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
9d5c8243 4273
7af40ad9
AD		 4274static void igb_tx_map(struct igb_ring *tx_ring,
4275 struct igb_tx_buffer *first,
ebe42d16 4276 const u8 hdr_len)
9d5c8243 4277{
7af40ad9 4278 struct sk_buff *skb = first->skb;
ebe42d16
AD		 4279 struct igb_tx_buffer *tx_buffer_info;
4280 union e1000_adv_tx_desc *tx_desc;
4281 dma_addr_t dma;
4282 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
4283 unsigned int data_len = skb->data_len;
4284 unsigned int size = skb_headlen(skb);
4285 unsigned int paylen = skb->len - hdr_len;
4286 __le32 cmd_type;
7af40ad9 4287 u32 tx_flags = first->tx_flags;
ebe42d16 4288 u16 i = tx_ring->next_to_use;
ebe42d16
AD		 4289
4290 tx_desc = IGB_TX_DESC(tx_ring, i);
4291
7af40ad9 4292 igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, paylen);
ebe42d16
AD		 4293 cmd_type = igb_tx_cmd_type(tx_flags);
4294
4295 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
4296 if (dma_mapping_error(tx_ring->dev, dma))
6366ad33 4297 goto dma_error;
9d5c8243 4298
ebe42d16
AD		 4299 /* record length, and DMA address */
4300 first->length = size;
4301 first->dma = dma;
ebe42d16
AD		 4302 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4303
4304 for (;;) {
4305 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
4306 tx_desc->read.cmd_type_len =
4307 cmd_type | cpu_to_le32(IGB_MAX_DATA_PER_TXD);
4308
4309 i++;
4310 tx_desc++;
4311 if (i == tx_ring->count) {
4312 tx_desc = IGB_TX_DESC(tx_ring, 0);
4313 i = 0;
4314 }
4315
4316 dma += IGB_MAX_DATA_PER_TXD;
4317 size -= IGB_MAX_DATA_PER_TXD;
4318
4319 tx_desc->read.olinfo_status = 0;
4320 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4321 }
4322
4323 if (likely(!data_len))
4324 break;
2bbfebe2 4325
ebe42d16 4326 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
9d5c8243 4327
65689fef 4328 i++;
ebe42d16
AD		 4329 tx_desc++;
4330 if (i == tx_ring->count) {
4331 tx_desc = IGB_TX_DESC(tx_ring, 0);
65689fef 4332 i = 0;
ebe42d16 4333 }
65689fef 4334
9e903e08 4335 size = skb_frag_size(frag);
ebe42d16
AD		 4336 data_len -= size;
4337
4338 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
4339 size, DMA_TO_DEVICE);
4340 if (dma_mapping_error(tx_ring->dev, dma))
6366ad33
AD		 4341 goto dma_error;
4342
ebe42d16
AD		 4343 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4344 tx_buffer_info->length = size;
4345 tx_buffer_info->dma = dma;
4346
4347 tx_desc->read.olinfo_status = 0;
4348 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4349
4350 frag++;
9d5c8243
AK		 4351 }
4352
bdbc0631
ED		 4353 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
4354
ebe42d16
AD		 4355 /* write last descriptor with RS and EOP bits */
4356 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IGB_TXD_DCMD);
6b8f0922
BG		 4357 if (unlikely(skb->no_fcs))
4358 cmd_type &= ~(cpu_to_le32(E1000_ADVTXD_DCMD_IFCS));
ebe42d16 4359 tx_desc->read.cmd_type_len = cmd_type;
8542db05
AD		 4360
4361 /* set the timestamp */
4362 first->time_stamp = jiffies;
4363
ebe42d16
AD		 4364 /*
4365 * Force memory writes to complete before letting h/w know there
4366 * are new descriptors to fetch. (Only applicable for weak-ordered
4367 * memory model archs, such as IA-64).
4368 *
4369 * We also need this memory barrier to make certain all of the
4370 * status bits have been updated before next_to_watch is written.
4371 */
4372 wmb();
4373
8542db05 4374 /* set next_to_watch value indicating a packet is present */
ebe42d16 4375 first->next_to_watch = tx_desc;
9d5c8243 4376
ebe42d16
AD		 4377 i++;
4378 if (i == tx_ring->count)
4379 i = 0;
6366ad33 4380
ebe42d16 4381 tx_ring->next_to_use = i;
6366ad33 4382
ebe42d16 4383 writel(i, tx_ring->tail);
6366ad33 4384
ebe42d16
AD		 4385 /* we need this if more than one processor can write to our tail
4386 * at a time, it syncronizes IO on IA64/Altix systems */
4387 mmiowb();
4388
4389 return;
4390
4391dma_error:
4392 dev_err(tx_ring->dev, "TX DMA map failed\n");
4393
4394 /* clear dma mappings for failed tx_buffer_info map */
4395 for (;;) {
4396 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4397 igb_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
4398 if (tx_buffer_info == first)
4399 break;
a77ff709
NN		 4400 if (i == 0)
4401 i = tx_ring->count;
6366ad33 4402 i--;
6366ad33
AD		 4403 }
4404
9d5c8243 4405 tx_ring->next_to_use = i;
9d5c8243
AK		 4406}
4407
6ad4edfc 4408static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
9d5c8243 4409{
e694e964
AD		 4410 struct net_device *netdev = tx_ring->netdev;
4411
661086df 4412 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 4413
9d5c8243
AK		 4414 /* Herbert's original patch had:
4415 * smp_mb__after_netif_stop_queue();
4416 * but since that doesn't exist yet, just open code it. */
4417 smp_mb();
4418
4419 /* We need to check again in a case another CPU has just
4420 * made room available. */
c493ea45 4421 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 4422 return -EBUSY;
4423
4424 /* A reprieve! */
661086df 4425 netif_wake_subqueue(netdev, tx_ring->queue_index);
12dcd86b
ED		 4426
4427 u64_stats_update_begin(&tx_ring->tx_syncp2);
4428 tx_ring->tx_stats.restart_queue2++;
4429 u64_stats_update_end(&tx_ring->tx_syncp2);
4430
9d5c8243
AK		 4431 return 0;
4432}
4433
6ad4edfc 4434static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
9d5c8243 4435{
c493ea45 4436 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 4437 return 0;
e694e964 4438 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 4439}
4440
cd392f5c
AD		 4441netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
4442 struct igb_ring *tx_ring)
9d5c8243 4443{
8542db05 4444 struct igb_tx_buffer *first;
ebe42d16 4445 int tso;
91d4ee33 4446 u32 tx_flags = 0;
31f6adbb 4447 __be16 protocol = vlan_get_protocol(skb);
91d4ee33 4448 u8 hdr_len = 0;
9d5c8243 4449
9d5c8243
AK		 4450 /* need: 1 descriptor per page,
4451 * + 2 desc gap to keep tail from touching head,
4452 * + 1 desc for skb->data,
4453 * + 1 desc for context descriptor,
4454 * otherwise try next time */
e694e964 4455 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 4456 /* this is a hard error */
9d5c8243
AK		 4457 return NETDEV_TX_BUSY;
4458 }
33af6bcc 4459
7af40ad9
AD		 4460 /* record the location of the first descriptor for this packet */
4461 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
4462 first->skb = skb;
4463 first->bytecount = skb->len;
4464 first->gso_segs = 1;
4465
2244d07b
OH		 4466 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4467 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
33af6bcc 4468 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 4469 }
9d5c8243 4470
eab6d18d 4471 if (vlan_tx_tag_present(skb)) {
9d5c8243
AK		 4472 tx_flags |= IGB_TX_FLAGS_VLAN;
4473 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4474 }
4475
7af40ad9
AD		 4476 /* record initial flags and protocol */
4477 first->tx_flags = tx_flags;
4478 first->protocol = protocol;
cdfd01fc 4479
7af40ad9
AD		 4480 tso = igb_tso(tx_ring, first, &hdr_len);
4481 if (tso < 0)
7d13a7d0 4482 goto out_drop;
7af40ad9
AD		 4483 else if (!tso)
4484 igb_tx_csum(tx_ring, first);
9d5c8243 4485
7af40ad9 4486 igb_tx_map(tx_ring, first, hdr_len);
85ad76b2
AD		 4487
4488 /* Make sure there is space in the ring for the next send. */
e694e964 4489 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 4490
9d5c8243 4491 return NETDEV_TX_OK;
7d13a7d0
AD		 4492
4493out_drop:
7af40ad9
AD		 4494 igb_unmap_and_free_tx_resource(tx_ring, first);
4495
7d13a7d0 4496 return NETDEV_TX_OK;
9d5c8243
AK		 4497}
4498
1cc3bd87
AD		 4499static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
4500 struct sk_buff *skb)
4501{
4502 unsigned int r_idx = skb->queue_mapping;
4503
4504 if (r_idx >= adapter->num_tx_queues)
4505 r_idx = r_idx % adapter->num_tx_queues;
4506
4507 return adapter->tx_ring[r_idx];
4508}
4509
cd392f5c
AD		 4510static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
4511 struct net_device *netdev)
9d5c8243
AK		 4512{
4513 struct igb_adapter *adapter = netdev_priv(netdev);
b1a436c3
AD		 4514
4515 if (test_bit(__IGB_DOWN, &adapter->state)) {
4516 dev_kfree_skb_any(skb);
4517 return NETDEV_TX_OK;
4518 }
4519
4520 if (skb->len <= 0) {
4521 dev_kfree_skb_any(skb);
4522 return NETDEV_TX_OK;
4523 }
4524
1cc3bd87
AD		 4525 /*
4526 * The minimum packet size with TCTL.PSP set is 17 so pad the skb
4527 * in order to meet this minimum size requirement.
4528 */
4529 if (skb->len < 17) {
4530 if (skb_padto(skb, 17))
4531 return NETDEV_TX_OK;
4532 skb->len = 17;
4533 }
9d5c8243 4534
1cc3bd87 4535 return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
9d5c8243
AK		 4536}
4537
4538/**
4539 * igb_tx_timeout - Respond to a Tx Hang
4540 * @netdev: network interface device structure
4541 **/
4542static void igb_tx_timeout(struct net_device *netdev)
4543{
4544 struct igb_adapter *adapter = netdev_priv(netdev);
4545 struct e1000_hw *hw = &adapter->hw;
4546
4547 /* Do the reset outside of interrupt context */
4548 adapter->tx_timeout_count++;
f7ba205e 4549
06218a8d 4550 if (hw->mac.type >= e1000_82580)
55cac248
AD		 4551 hw->dev_spec._82575.global_device_reset = true;
4552
9d5c8243 4553 schedule_work(&adapter->reset_task);
265de409
AD		 4554 wr32(E1000_EICS,
4555 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 4556}
4557
4558static void igb_reset_task(struct work_struct *work)
4559{
4560 struct igb_adapter *adapter;
4561 adapter = container_of(work, struct igb_adapter, reset_task);
4562
c97ec42a
TI		 4563 igb_dump(adapter);
4564 netdev_err(adapter->netdev, "Reset adapter\n");
9d5c8243
AK		 4565 igb_reinit_locked(adapter);
4566}
4567
4568/**
12dcd86b 4569 * igb_get_stats64 - Get System Network Statistics
9d5c8243 4570 * @netdev: network interface device structure
12dcd86b 4571 * @stats: rtnl_link_stats64 pointer
9d5c8243 4572 *
9d5c8243 4573 **/
12dcd86b
ED		 4574static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4575 struct rtnl_link_stats64 *stats)
9d5c8243 4576{
12dcd86b
ED		 4577 struct igb_adapter *adapter = netdev_priv(netdev);
4578
4579 spin_lock(&adapter->stats64_lock);
4580 igb_update_stats(adapter, &adapter->stats64);
4581 memcpy(stats, &adapter->stats64, sizeof(*stats));
4582 spin_unlock(&adapter->stats64_lock);
4583
4584 return stats;
9d5c8243
AK		 4585}
4586
4587/**
4588 * igb_change_mtu - Change the Maximum Transfer Unit
4589 * @netdev: network interface device structure
4590 * @new_mtu: new value for maximum frame size
4591 *
4592 * Returns 0 on success, negative on failure
4593 **/
4594static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4595{
4596 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4597 struct pci_dev *pdev = adapter->pdev;
153285f9 4598 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
9d5c8243 4599
c809d227 4600 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 4601 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 4602 return -EINVAL;
4603 }
4604
153285f9 4605#define MAX_STD_JUMBO_FRAME_SIZE 9238
9d5c8243 4606 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 4607 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 4608 return -EINVAL;
4609 }
4610
4611 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4612 msleep(1);
73cd78f1 4613
9d5c8243
AK		 4614 /* igb_down has a dependency on max_frame_size */
4615 adapter->max_frame_size = max_frame;
559e9c49 4616
4c844851
AD		 4617 if (netif_running(netdev))
4618 igb_down(adapter);
9d5c8243 4619
090b1795 4620 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 4621 netdev->mtu, new_mtu);
4622 netdev->mtu = new_mtu;
4623
4624 if (netif_running(netdev))
4625 igb_up(adapter);
4626 else
4627 igb_reset(adapter);
4628
4629 clear_bit(__IGB_RESETTING, &adapter->state);
4630
4631 return 0;
4632}
4633
4634/**
4635 * igb_update_stats - Update the board statistics counters
4636 * @adapter: board private structure
4637 **/
4638
12dcd86b
ED		 4639void igb_update_stats(struct igb_adapter *adapter,
4640 struct rtnl_link_stats64 *net_stats)
9d5c8243
AK		 4641{
4642 struct e1000_hw *hw = &adapter->hw;
4643 struct pci_dev *pdev = adapter->pdev;
fa3d9a6d 4644 u32 reg, mpc;
9d5c8243 4645 u16 phy_tmp;
3f9c0164
AD		 4646 int i;
4647 u64 bytes, packets;
12dcd86b
ED		 4648 unsigned int start;
4649 u64 _bytes, _packets;
9d5c8243
AK		 4650
4651#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4652
4653 /*
4654 * Prevent stats update while adapter is being reset, or if the pci
4655 * connection is down.
4656 */
4657 if (adapter->link_speed == 0)
4658 return;
4659 if (pci_channel_offline(pdev))
4660 return;
4661
3f9c0164
AD		 4662 bytes = 0;
4663 packets = 0;
4664 for (i = 0; i < adapter->num_rx_queues; i++) {
4665 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446 4666 struct igb_ring *ring = adapter->rx_ring[i];
12dcd86b 4667
3025a446 4668 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 4669 net_stats->rx_fifo_errors += rqdpc_tmp;
12dcd86b
ED		 4670
4671 do {
4672 start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4673 _bytes = ring->rx_stats.bytes;
4674 _packets = ring->rx_stats.packets;
4675 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4676 bytes += _bytes;
4677 packets += _packets;
3f9c0164
AD		 4678 }
4679
128e45eb
AD		 4680 net_stats->rx_bytes = bytes;
4681 net_stats->rx_packets = packets;
3f9c0164
AD		 4682
4683 bytes = 0;
4684 packets = 0;
4685 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 4686 struct igb_ring *ring = adapter->tx_ring[i];
12dcd86b
ED		 4687 do {
4688 start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4689 _bytes = ring->tx_stats.bytes;
4690 _packets = ring->tx_stats.packets;
4691 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4692 bytes += _bytes;
4693 packets += _packets;
3f9c0164 4694 }
128e45eb
AD		 4695 net_stats->tx_bytes = bytes;
4696 net_stats->tx_packets = packets;
3f9c0164
AD		 4697
4698 /* read stats registers */
9d5c8243
AK		 4699 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4700 adapter->stats.gprc += rd32(E1000_GPRC);
4701 adapter->stats.gorc += rd32(E1000_GORCL);
4702 rd32(E1000_GORCH); /* clear GORCL */
4703 adapter->stats.bprc += rd32(E1000_BPRC);
4704 adapter->stats.mprc += rd32(E1000_MPRC);
4705 adapter->stats.roc += rd32(E1000_ROC);
4706
4707 adapter->stats.prc64 += rd32(E1000_PRC64);
4708 adapter->stats.prc127 += rd32(E1000_PRC127);
4709 adapter->stats.prc255 += rd32(E1000_PRC255);
4710 adapter->stats.prc511 += rd32(E1000_PRC511);
4711 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4712 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4713 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4714 adapter->stats.sec += rd32(E1000_SEC);
4715
fa3d9a6d
MW		 4716 mpc = rd32(E1000_MPC);
4717 adapter->stats.mpc += mpc;
4718 net_stats->rx_fifo_errors += mpc;
9d5c8243
AK		 4719 adapter->stats.scc += rd32(E1000_SCC);
4720 adapter->stats.ecol += rd32(E1000_ECOL);
4721 adapter->stats.mcc += rd32(E1000_MCC);
4722 adapter->stats.latecol += rd32(E1000_LATECOL);
4723 adapter->stats.dc += rd32(E1000_DC);
4724 adapter->stats.rlec += rd32(E1000_RLEC);
4725 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4726 adapter->stats.xontxc += rd32(E1000_XONTXC);
4727 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4728 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4729 adapter->stats.fcruc += rd32(E1000_FCRUC);
4730 adapter->stats.gptc += rd32(E1000_GPTC);
4731 adapter->stats.gotc += rd32(E1000_GOTCL);
4732 rd32(E1000_GOTCH); /* clear GOTCL */
fa3d9a6d 4733 adapter->stats.rnbc += rd32(E1000_RNBC);
9d5c8243
AK		 4734 adapter->stats.ruc += rd32(E1000_RUC);
4735 adapter->stats.rfc += rd32(E1000_RFC);
4736 adapter->stats.rjc += rd32(E1000_RJC);
4737 adapter->stats.tor += rd32(E1000_TORH);
4738 adapter->stats.tot += rd32(E1000_TOTH);
4739 adapter->stats.tpr += rd32(E1000_TPR);
4740
4741 adapter->stats.ptc64 += rd32(E1000_PTC64);
4742 adapter->stats.ptc127 += rd32(E1000_PTC127);
4743 adapter->stats.ptc255 += rd32(E1000_PTC255);
4744 adapter->stats.ptc511 += rd32(E1000_PTC511);
4745 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4746 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4747
4748 adapter->stats.mptc += rd32(E1000_MPTC);
4749 adapter->stats.bptc += rd32(E1000_BPTC);
4750
2d0b0f69
NN		 4751 adapter->stats.tpt += rd32(E1000_TPT);
4752 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4753
4754 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4755 /* read internal phy specific stats */
4756 reg = rd32(E1000_CTRL_EXT);
4757 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4758 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4759 adapter->stats.tncrs += rd32(E1000_TNCRS);
4760 }
4761
9d5c8243
AK		 4762 adapter->stats.tsctc += rd32(E1000_TSCTC);
4763 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4764
4765 adapter->stats.iac += rd32(E1000_IAC);
4766 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4767 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4768 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4769 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4770 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4771 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4772 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4773 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4774
4775 /* Fill out the OS statistics structure */
128e45eb
AD		 4776 net_stats->multicast = adapter->stats.mprc;
4777 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4778
4779 /* Rx Errors */
4780
4781 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4782 * our own version based on RUC and ROC */
128e45eb 4783 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4784 adapter->stats.crcerrs + adapter->stats.algnerrc +
4785 adapter->stats.ruc + adapter->stats.roc +
4786 adapter->stats.cexterr;
128e45eb
AD		 4787 net_stats->rx_length_errors = adapter->stats.ruc +
4788 adapter->stats.roc;
4789 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4790 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4791 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4792
4793 /* Tx Errors */
128e45eb
AD		 4794 net_stats->tx_errors = adapter->stats.ecol +
4795 adapter->stats.latecol;
4796 net_stats->tx_aborted_errors = adapter->stats.ecol;
4797 net_stats->tx_window_errors = adapter->stats.latecol;
4798 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4799
4800 /* Tx Dropped needs to be maintained elsewhere */
4801
4802 /* Phy Stats */
4803 if (hw->phy.media_type == e1000_media_type_copper) {
4804 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4805 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4806 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4807 adapter->phy_stats.idle_errors += phy_tmp;
4808 }
4809 }
4810
4811 /* Management Stats */
4812 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4813 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4814 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
0a915b95
CW		 4815
4816 /* OS2BMC Stats */
4817 reg = rd32(E1000_MANC);
4818 if (reg & E1000_MANC_EN_BMC2OS) {
4819 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
4820 adapter->stats.o2bspc += rd32(E1000_O2BSPC);
4821 adapter->stats.b2ospc += rd32(E1000_B2OSPC);
4822 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
4823 }
9d5c8243
AK		 4824}
4825
9d5c8243
AK		 4826static irqreturn_t igb_msix_other(int irq, void *data)
4827{
047e0030 4828 struct igb_adapter *adapter = data;
9d5c8243 4829 struct e1000_hw *hw = &adapter->hw;
844290e5 4830 u32 icr = rd32(E1000_ICR);
844290e5 4831 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4832
7f081d40
AD		 4833 if (icr & E1000_ICR_DRSTA)
4834 schedule_work(&adapter->reset_task);
4835
047e0030 4836 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4837 /* HW is reporting DMA is out of sync */
4838 adapter->stats.doosync++;
13800469
GR		 4839 /* The DMA Out of Sync is also indication of a spoof event
4840 * in IOV mode. Check the Wrong VM Behavior register to
4841 * see if it is really a spoof event. */
4842 igb_check_wvbr(adapter);
dda0e083 4843 }
eebbbdba 4844
4ae196df
AD		 4845 /* Check for a mailbox event */
4846 if (icr & E1000_ICR_VMMB)
4847 igb_msg_task(adapter);
4848
4849 if (icr & E1000_ICR_LSC) {
4850 hw->mac.get_link_status = 1;
4851 /* guard against interrupt when we're going down */
4852 if (!test_bit(__IGB_DOWN, &adapter->state))
4853 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4854 }
4855
844290e5 4856 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4857
4858 return IRQ_HANDLED;
4859}
4860
047e0030 4861static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4862{
26b39276 4863 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4864 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4865
047e0030
AD		 4866 if (!q_vector->set_itr)
4867 return;
73cd78f1 4868
047e0030
AD		 4869 if (!itr_val)
4870 itr_val = 0x4;
661086df 4871
26b39276
AD		 4872 if (adapter->hw.mac.type == e1000_82575)
4873 itr_val |= itr_val << 16;
661086df 4874 else
0ba82994 4875 itr_val |= E1000_EITR_CNT_IGNR;
661086df 4876
047e0030
AD		 4877 writel(itr_val, q_vector->itr_register);
4878 q_vector->set_itr = 0;
6eb5a7f1
AD		 4879}
4880
047e0030 4881static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4882{
047e0030 4883 struct igb_q_vector *q_vector = data;
9d5c8243 4884
047e0030
AD		 4885 /* Write the ITR value calculated from the previous interrupt. */
4886 igb_write_itr(q_vector);
9d5c8243 4887
047e0030 4888 napi_schedule(&q_vector->napi);
844290e5 4889
047e0030 4890 return IRQ_HANDLED;
fe4506b6
JC		 4891}
4892
421e02f0 4893#ifdef CONFIG_IGB_DCA
047e0030 4894static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4895{
047e0030 4896 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4897 struct e1000_hw *hw = &adapter->hw;
4898 int cpu = get_cpu();
fe4506b6 4899
047e0030
AD		 4900 if (q_vector->cpu == cpu)
4901 goto out_no_update;
4902
0ba82994
AD		 4903 if (q_vector->tx.ring) {
4904 int q = q_vector->tx.ring->reg_idx;
047e0030
AD		 4905 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4906 if (hw->mac.type == e1000_82575) {
4907 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4908 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4909 } else {
047e0030
AD		 4910 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4911 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4912 E1000_DCA_TXCTRL_CPUID_SHIFT;
4913 }
4914 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4915 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4916 }
0ba82994
AD		 4917 if (q_vector->rx.ring) {
4918 int q = q_vector->rx.ring->reg_idx;
047e0030
AD		 4919 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4920 if (hw->mac.type == e1000_82575) {
2d064c06 4921 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4922 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4923 } else {
4924 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4925 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4926 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4927 }
fe4506b6
JC		 4928 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4929 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4930 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4931 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4932 }
047e0030
AD		 4933 q_vector->cpu = cpu;
4934out_no_update:
fe4506b6
JC		 4935 put_cpu();
4936}
4937
4938static void igb_setup_dca(struct igb_adapter *adapter)
4939{
7e0e99ef 4940 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4941 int i;
4942
7dfc16fa 4943 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4944 return;
4945
7e0e99ef
AD		 4946 /* Always use CB2 mode, difference is masked in the CB driver. */
4947 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4948
047e0030 4949 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4950 adapter->q_vector[i]->cpu = -1;
4951 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4952 }
4953}
4954
4955static int __igb_notify_dca(struct device *dev, void *data)
4956{
4957 struct net_device *netdev = dev_get_drvdata(dev);
4958 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4959 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4960 struct e1000_hw *hw = &adapter->hw;
4961 unsigned long event = *(unsigned long *)data;
4962
4963 switch (event) {
4964 case DCA_PROVIDER_ADD:
4965 /* if already enabled, don't do it again */
7dfc16fa 4966 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4967 break;
fe4506b6 4968 if (dca_add_requester(dev) == 0) {
bbd98fe4 4969 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4970 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4971 igb_setup_dca(adapter);
4972 break;
4973 }
4974 /* Fall Through since DCA is disabled. */
4975 case DCA_PROVIDER_REMOVE:
7dfc16fa 4976 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4977 /* without this a class_device is left
047e0030 4978 * hanging around in the sysfs model */
fe4506b6 4979 dca_remove_requester(dev);
090b1795 4980 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4981 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4982 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4983 }
4984 break;
4985 }
bbd98fe4 4986
fe4506b6 4987 return 0;
9d5c8243
AK		 4988}
4989
fe4506b6
JC		 4990static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4991 void *p)
4992{
4993 int ret_val;
4994
4995 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4996 __igb_notify_dca);
4997
4998 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4999}
421e02f0 5000#endif /* CONFIG_IGB_DCA */
9d5c8243 5001
0224d663
GR		 5002#ifdef CONFIG_PCI_IOV
5003static int igb_vf_configure(struct igb_adapter *adapter, int vf)
5004{
5005 unsigned char mac_addr[ETH_ALEN];
5006 struct pci_dev *pdev = adapter->pdev;
5007 struct e1000_hw *hw = &adapter->hw;
5008 struct pci_dev *pvfdev;
5009 unsigned int device_id;
5010 u16 thisvf_devfn;
5011
7efd26d0 5012 eth_random_addr(mac_addr);
0224d663
GR		 5013 igb_set_vf_mac(adapter, vf, mac_addr);
5014
5015 switch (adapter->hw.mac.type) {
5016 case e1000_82576:
5017 device_id = IGB_82576_VF_DEV_ID;
5018 /* VF Stride for 82576 is 2 */
5019 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 1)) |
5020 (pdev->devfn & 1);
5021 break;
5022 case e1000_i350:
5023 device_id = IGB_I350_VF_DEV_ID;
5024 /* VF Stride for I350 is 4 */
5025 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 2)) |
5026 (pdev->devfn & 3);
5027 break;
5028 default:
5029 device_id = 0;
5030 thisvf_devfn = 0;
5031 break;
5032 }
5033
5034 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
5035 while (pvfdev) {
5036 if (pvfdev->devfn == thisvf_devfn)
5037 break;
5038 pvfdev = pci_get_device(hw->vendor_id,
5039 device_id, pvfdev);
5040 }
5041
5042 if (pvfdev)
5043 adapter->vf_data[vf].vfdev = pvfdev;
5044 else
5045 dev_err(&pdev->dev,
5046 "Couldn't find pci dev ptr for VF %4.4x\n",
5047 thisvf_devfn);
5048 return pvfdev != NULL;
5049}
5050
5051static int igb_find_enabled_vfs(struct igb_adapter *adapter)
5052{
5053 struct e1000_hw *hw = &adapter->hw;
5054 struct pci_dev *pdev = adapter->pdev;
5055 struct pci_dev *pvfdev;
5056 u16 vf_devfn = 0;
5057 u16 vf_stride;
5058 unsigned int device_id;
5059 int vfs_found = 0;
5060
5061 switch (adapter->hw.mac.type) {
5062 case e1000_82576:
5063 device_id = IGB_82576_VF_DEV_ID;
5064 /* VF Stride for 82576 is 2 */
5065 vf_stride = 2;
5066 break;
5067 case e1000_i350:
5068 device_id = IGB_I350_VF_DEV_ID;
5069 /* VF Stride for I350 is 4 */
5070 vf_stride = 4;
5071 break;
5072 default:
5073 device_id = 0;
5074 vf_stride = 0;
5075 break;
5076 }
5077
5078 vf_devfn = pdev->devfn + 0x80;
5079 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
5080 while (pvfdev) {
06292921
GR		 5081 if (pvfdev->devfn == vf_devfn &&
5082 (pvfdev->bus->number >= pdev->bus->number))
0224d663
GR		 5083 vfs_found++;
5084 vf_devfn += vf_stride;
5085 pvfdev = pci_get_device(hw->vendor_id,
5086 device_id, pvfdev);
5087 }
5088
5089 return vfs_found;
5090}
5091
5092static int igb_check_vf_assignment(struct igb_adapter *adapter)
5093{
5094 int i;
5095 for (i = 0; i < adapter->vfs_allocated_count; i++) {
5096 if (adapter->vf_data[i].vfdev) {
5097 if (adapter->vf_data[i].vfdev->dev_flags &
5098 PCI_DEV_FLAGS_ASSIGNED)
5099 return true;
5100 }
5101 }
5102 return false;
5103}
5104
5105#endif
4ae196df
AD		 5106static void igb_ping_all_vfs(struct igb_adapter *adapter)
5107{
5108 struct e1000_hw *hw = &adapter->hw;
5109 u32 ping;
5110 int i;
5111
5112 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
5113 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 5114 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 5115 ping |= E1000_VT_MSGTYPE_CTS;
5116 igb_write_mbx(hw, &ping, 1, i);
5117 }
5118}
5119
7d5753f0
AD		 5120static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5121{
5122 struct e1000_hw *hw = &adapter->hw;
5123 u32 vmolr = rd32(E1000_VMOLR(vf));
5124 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5125
d85b9004 5126 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
7d5753f0
AD		 5127 IGB_VF_FLAG_MULTI_PROMISC);
5128 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5129
5130 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
5131 vmolr |= E1000_VMOLR_MPME;
d85b9004 5132 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
7d5753f0
AD		 5133 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
5134 } else {
5135 /*
5136 * if we have hashes and we are clearing a multicast promisc
5137 * flag we need to write the hashes to the MTA as this step
5138 * was previously skipped
5139 */
5140 if (vf_data->num_vf_mc_hashes > 30) {
5141 vmolr |= E1000_VMOLR_MPME;
5142 } else if (vf_data->num_vf_mc_hashes) {
5143 int j;
5144 vmolr |= E1000_VMOLR_ROMPE;
5145 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5146 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5147 }
5148 }
5149
5150 wr32(E1000_VMOLR(vf), vmolr);
5151
5152 /* there are flags left unprocessed, likely not supported */
5153 if (*msgbuf & E1000_VT_MSGINFO_MASK)
5154 return -EINVAL;
5155
5156 return 0;
5157
5158}
5159
4ae196df
AD		 5160static int igb_set_vf_multicasts(struct igb_adapter *adapter,
5161 u32 *msgbuf, u32 vf)
5162{
5163 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5164 u16 *hash_list = (u16 *)&msgbuf[1];
5165 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5166 int i;
5167
7d5753f0 5168 /* salt away the number of multicast addresses assigned
4ae196df
AD		 5169 * to this VF for later use to restore when the PF multi cast
5170 * list changes
5171 */
5172 vf_data->num_vf_mc_hashes = n;
5173
7d5753f0
AD		 5174 /* only up to 30 hash values supported */
5175 if (n > 30)
5176 n = 30;
5177
5178 /* store the hashes for later use */
4ae196df 5179 for (i = 0; i < n; i++)
a419aef8 5180 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 5181
5182 /* Flush and reset the mta with the new values */
ff41f8dc 5183 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 5184
5185 return 0;
5186}
5187
5188static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
5189{
5190 struct e1000_hw *hw = &adapter->hw;
5191 struct vf_data_storage *vf_data;
5192 int i, j;
5193
5194 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 5195 u32 vmolr = rd32(E1000_VMOLR(i));
5196 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5197
4ae196df 5198 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 5199
5200 if ((vf_data->num_vf_mc_hashes > 30) ||
5201 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
5202 vmolr |= E1000_VMOLR_MPME;
5203 } else if (vf_data->num_vf_mc_hashes) {
5204 vmolr |= E1000_VMOLR_ROMPE;
5205 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5206 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5207 }
5208 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 5209 }
5210}
5211
5212static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
5213{
5214 struct e1000_hw *hw = &adapter->hw;
5215 u32 pool_mask, reg, vid;
5216 int i;
5217
5218 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5219
5220 /* Find the vlan filter for this id */
5221 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5222 reg = rd32(E1000_VLVF(i));
5223
5224 /* remove the vf from the pool */
5225 reg &= ~pool_mask;
5226
5227 /* if pool is empty then remove entry from vfta */
5228 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
5229 (reg & E1000_VLVF_VLANID_ENABLE)) {
5230 reg = 0;
5231 vid = reg & E1000_VLVF_VLANID_MASK;
5232 igb_vfta_set(hw, vid, false);
5233 }
5234
5235 wr32(E1000_VLVF(i), reg);
5236 }
ae641bdc
AD		 5237
5238 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 5239}
5240
5241static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
5242{
5243 struct e1000_hw *hw = &adapter->hw;
5244 u32 reg, i;
5245
51466239
AD		 5246 /* The vlvf table only exists on 82576 hardware and newer */
5247 if (hw->mac.type < e1000_82576)
5248 return -1;
5249
5250 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 5251 if (!adapter->vfs_allocated_count)
5252 return -1;
5253
5254 /* Find the vlan filter for this id */
5255 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5256 reg = rd32(E1000_VLVF(i));
5257 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
5258 vid == (reg & E1000_VLVF_VLANID_MASK))
5259 break;
5260 }
5261
5262 if (add) {
5263 if (i == E1000_VLVF_ARRAY_SIZE) {
5264 /* Did not find a matching VLAN ID entry that was
5265 * enabled. Search for a free filter entry, i.e.
5266 * one without the enable bit set
5267 */
5268 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5269 reg = rd32(E1000_VLVF(i));
5270 if (!(reg & E1000_VLVF_VLANID_ENABLE))
5271 break;
5272 }
5273 }
5274 if (i < E1000_VLVF_ARRAY_SIZE) {
5275 /* Found an enabled/available entry */
5276 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5277
5278 /* if !enabled we need to set this up in vfta */
5279 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 5280 /* add VID to filter table */
5281 igb_vfta_set(hw, vid, true);
4ae196df
AD		 5282 reg |= E1000_VLVF_VLANID_ENABLE;
5283 }
cad6d05f
AD		 5284 reg &= ~E1000_VLVF_VLANID_MASK;
5285 reg |= vid;
4ae196df 5286 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5287
5288 /* do not modify RLPML for PF devices */
5289 if (vf >= adapter->vfs_allocated_count)
5290 return 0;
5291
5292 if (!adapter->vf_data[vf].vlans_enabled) {
5293 u32 size;
5294 reg = rd32(E1000_VMOLR(vf));
5295 size = reg & E1000_VMOLR_RLPML_MASK;
5296 size += 4;
5297 reg &= ~E1000_VMOLR_RLPML_MASK;
5298 reg |= size;
5299 wr32(E1000_VMOLR(vf), reg);
5300 }
ae641bdc 5301
51466239 5302 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 5303 }
5304 } else {
5305 if (i < E1000_VLVF_ARRAY_SIZE) {
5306 /* remove vf from the pool */
5307 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
5308 /* if pool is empty then remove entry from vfta */
5309 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
5310 reg = 0;
5311 igb_vfta_set(hw, vid, false);
5312 }
5313 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5314
5315 /* do not modify RLPML for PF devices */
5316 if (vf >= adapter->vfs_allocated_count)
5317 return 0;
5318
5319 adapter->vf_data[vf].vlans_enabled--;
5320 if (!adapter->vf_data[vf].vlans_enabled) {
5321 u32 size;
5322 reg = rd32(E1000_VMOLR(vf));
5323 size = reg & E1000_VMOLR_RLPML_MASK;
5324 size -= 4;
5325 reg &= ~E1000_VMOLR_RLPML_MASK;
5326 reg |= size;
5327 wr32(E1000_VMOLR(vf), reg);
5328 }
4ae196df
AD		 5329 }
5330 }
8151d294
WM		 5331 return 0;
5332}
5333
5334static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
5335{
5336 struct e1000_hw *hw = &adapter->hw;
5337
5338 if (vid)
5339 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
5340 else
5341 wr32(E1000_VMVIR(vf), 0);
5342}
5343
5344static int igb_ndo_set_vf_vlan(struct net_device *netdev,
5345 int vf, u16 vlan, u8 qos)
5346{
5347 int err = 0;
5348 struct igb_adapter *adapter = netdev_priv(netdev);
5349
5350 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
5351 return -EINVAL;
5352 if (vlan || qos) {
5353 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
5354 if (err)
5355 goto out;
5356 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
5357 igb_set_vmolr(adapter, vf, !vlan);
5358 adapter->vf_data[vf].pf_vlan = vlan;
5359 adapter->vf_data[vf].pf_qos = qos;
5360 dev_info(&adapter->pdev->dev,
5361 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5362 if (test_bit(__IGB_DOWN, &adapter->state)) {
5363 dev_warn(&adapter->pdev->dev,
5364 "The VF VLAN has been set,"
5365 " but the PF device is not up.\n");
5366 dev_warn(&adapter->pdev->dev,
5367 "Bring the PF device up before"
5368 " attempting to use the VF device.\n");
5369 }
5370 } else {
5371 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5372 false, vf);
5373 igb_set_vmvir(adapter, vlan, vf);
5374 igb_set_vmolr(adapter, vf, true);
5375 adapter->vf_data[vf].pf_vlan = 0;
5376 adapter->vf_data[vf].pf_qos = 0;
5377 }
5378out:
5379 return err;
4ae196df
AD		 5380}
5381
5382static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5383{
5384 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5385 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5386
5387 return igb_vlvf_set(adapter, vid, add, vf);
5388}
5389
f2ca0dbe 5390static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 5391{
8fa7e0f7
GR		 5392 /* clear flags - except flag that indicates PF has set the MAC */
5393 adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
f2ca0dbe 5394 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 5395
5396 /* reset offloads to defaults */
8151d294 5397 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 5398
5399 /* reset vlans for device */
5400 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 5401 if (adapter->vf_data[vf].pf_vlan)
5402 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5403 adapter->vf_data[vf].pf_vlan,
5404 adapter->vf_data[vf].pf_qos);
5405 else
5406 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 5407
5408 /* reset multicast table array for vf */
5409 adapter->vf_data[vf].num_vf_mc_hashes = 0;
5410
5411 /* Flush and reset the mta with the new values */
ff41f8dc 5412 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 5413}
5414
f2ca0dbe
AD		 5415static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5416{
5417 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5418
5419 /* generate a new mac address as we were hotplug removed/added */
8151d294 5420 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
7efd26d0 5421 eth_random_addr(vf_mac);
f2ca0dbe
AD		 5422
5423 /* process remaining reset events */
5424 igb_vf_reset(adapter, vf);
5425}
5426
5427static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 5428{
5429 struct e1000_hw *hw = &adapter->hw;
5430 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 5431 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 5432 u32 reg, msgbuf[3];
5433 u8 *addr = (u8 *)(&msgbuf[1]);
5434
5435 /* process all the same items cleared in a function level reset */
f2ca0dbe 5436 igb_vf_reset(adapter, vf);
4ae196df
AD		 5437
5438 /* set vf mac address */
26ad9178 5439 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 5440
5441 /* enable transmit and receive for vf */
5442 reg = rd32(E1000_VFTE);
5443 wr32(E1000_VFTE, reg | (1 << vf));
5444 reg = rd32(E1000_VFRE);
5445 wr32(E1000_VFRE, reg | (1 << vf));
5446
8fa7e0f7 5447 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
4ae196df
AD		 5448
5449 /* reply to reset with ack and vf mac address */
5450 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5451 memcpy(addr, vf_mac, 6);
5452 igb_write_mbx(hw, msgbuf, 3, vf);
5453}
5454
5455static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5456{
de42edde
GR		 5457 /*
5458 * The VF MAC Address is stored in a packed array of bytes
5459 * starting at the second 32 bit word of the msg array
5460 */
f2ca0dbe
AD		 5461 unsigned char *addr = (char *)&msg[1];
5462 int err = -1;
4ae196df 5463
f2ca0dbe
AD		 5464 if (is_valid_ether_addr(addr))
5465 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 5466
f2ca0dbe 5467 return err;
4ae196df
AD		 5468}
5469
5470static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5471{
5472 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5473 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5474 u32 msg = E1000_VT_MSGTYPE_NACK;
5475
5476 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 5477 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5478 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 5479 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 5480 vf_data->last_nack = jiffies;
4ae196df
AD		 5481 }
5482}
5483
f2ca0dbe 5484static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 5485{
f2ca0dbe
AD		 5486 struct pci_dev *pdev = adapter->pdev;
5487 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 5488 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5489 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5490 s32 retval;
5491
f2ca0dbe 5492 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 5493
fef45f4c
AD		 5494 if (retval) {
5495 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 5496 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 5497 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5498 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5499 return;
5500 goto out;
5501 }
4ae196df
AD		 5502
5503 /* this is a message we already processed, do nothing */
5504 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 5505 return;
4ae196df
AD		 5506
5507 /*
5508 * until the vf completes a reset it should not be
5509 * allowed to start any configuration.
5510 */
5511
5512 if (msgbuf[0] == E1000_VF_RESET) {
5513 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 5514 return;
4ae196df
AD		 5515 }
5516
f2ca0dbe 5517 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 5518 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5519 return;
5520 retval = -1;
5521 goto out;
4ae196df
AD		 5522 }
5523
5524 switch ((msgbuf[0] & 0xFFFF)) {
5525 case E1000_VF_SET_MAC_ADDR:
a6b5ea35
GR		 5526 retval = -EINVAL;
5527 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
5528 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5529 else
5530 dev_warn(&pdev->dev,
5531 "VF %d attempted to override administratively "
5532 "set MAC address\nReload the VF driver to "
5533 "resume operations\n", vf);
4ae196df 5534 break;
7d5753f0
AD		 5535 case E1000_VF_SET_PROMISC:
5536 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5537 break;
4ae196df
AD		 5538 case E1000_VF_SET_MULTICAST:
5539 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5540 break;
5541 case E1000_VF_SET_LPE:
5542 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5543 break;
5544 case E1000_VF_SET_VLAN:
a6b5ea35
GR		 5545 retval = -1;
5546 if (vf_data->pf_vlan)
5547 dev_warn(&pdev->dev,
5548 "VF %d attempted to override administratively "
5549 "set VLAN tag\nReload the VF driver to "
5550 "resume operations\n", vf);
8151d294
WM		 5551 else
5552 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 5553 break;
5554 default:
090b1795 5555 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 5556 retval = -1;
5557 break;
5558 }
5559
fef45f4c
AD		 5560 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5561out:
4ae196df
AD		 5562 /* notify the VF of the results of what it sent us */
5563 if (retval)
5564 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5565 else
5566 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5567
4ae196df 5568 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 5569}
4ae196df 5570
f2ca0dbe
AD		 5571static void igb_msg_task(struct igb_adapter *adapter)
5572{
5573 struct e1000_hw *hw = &adapter->hw;
5574 u32 vf;
5575
5576 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5577 /* process any reset requests */
5578 if (!igb_check_for_rst(hw, vf))
5579 igb_vf_reset_event(adapter, vf);
5580
5581 /* process any messages pending */
5582 if (!igb_check_for_msg(hw, vf))
5583 igb_rcv_msg_from_vf(adapter, vf);
5584
5585 /* process any acks */
5586 if (!igb_check_for_ack(hw, vf))
5587 igb_rcv_ack_from_vf(adapter, vf);
5588 }
4ae196df
AD		 5589}
5590
68d480c4
AD		 5591/**
5592 * igb_set_uta - Set unicast filter table address
5593 * @adapter: board private structure
5594 *
5595 * The unicast table address is a register array of 32-bit registers.
5596 * The table is meant to be used in a way similar to how the MTA is used
5597 * however due to certain limitations in the hardware it is necessary to
25985edc
LDM		 5598 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
5599 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
68d480c4
AD		 5600 **/
5601static void igb_set_uta(struct igb_adapter *adapter)
5602{
5603 struct e1000_hw *hw = &adapter->hw;
5604 int i;
5605
5606 /* The UTA table only exists on 82576 hardware and newer */
5607 if (hw->mac.type < e1000_82576)
5608 return;
5609
5610 /* we only need to do this if VMDq is enabled */
5611 if (!adapter->vfs_allocated_count)
5612 return;
5613
5614 for (i = 0; i < hw->mac.uta_reg_count; i++)
5615 array_wr32(E1000_UTA, i, ~0);
5616}
5617
9d5c8243
AK		 5618/**
5619 * igb_intr_msi - Interrupt Handler
5620 * @irq: interrupt number
5621 * @data: pointer to a network interface device structure
5622 **/
5623static irqreturn_t igb_intr_msi(int irq, void *data)
5624{
047e0030
AD		 5625 struct igb_adapter *adapter = data;
5626 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5627 struct e1000_hw *hw = &adapter->hw;
5628 /* read ICR disables interrupts using IAM */
5629 u32 icr = rd32(E1000_ICR);
5630
047e0030 5631 igb_write_itr(q_vector);
9d5c8243 5632
7f081d40
AD		 5633 if (icr & E1000_ICR_DRSTA)
5634 schedule_work(&adapter->reset_task);
5635
047e0030 5636 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5637 /* HW is reporting DMA is out of sync */
5638 adapter->stats.doosync++;
5639 }
5640
9d5c8243
AK		 5641 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5642 hw->mac.get_link_status = 1;
5643 if (!test_bit(__IGB_DOWN, &adapter->state))
5644 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5645 }
5646
047e0030 5647 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5648
5649 return IRQ_HANDLED;
5650}
5651
5652/**
4a3c6433 5653 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 5654 * @irq: interrupt number
5655 * @data: pointer to a network interface device structure
5656 **/
5657static irqreturn_t igb_intr(int irq, void *data)
5658{
047e0030
AD		 5659 struct igb_adapter *adapter = data;
5660 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5661 struct e1000_hw *hw = &adapter->hw;
5662 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5663 * need for the IMC write */
5664 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 5665
5666 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5667 * not set, then the adapter didn't send an interrupt */
5668 if (!(icr & E1000_ICR_INT_ASSERTED))
5669 return IRQ_NONE;
5670
0ba82994
AD		 5671 igb_write_itr(q_vector);
5672
7f081d40
AD		 5673 if (icr & E1000_ICR_DRSTA)
5674 schedule_work(&adapter->reset_task);
5675
047e0030 5676 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5677 /* HW is reporting DMA is out of sync */
5678 adapter->stats.doosync++;
5679 }
5680
9d5c8243
AK		 5681 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5682 hw->mac.get_link_status = 1;
5683 /* guard against interrupt when we're going down */
5684 if (!test_bit(__IGB_DOWN, &adapter->state))
5685 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5686 }
5687
047e0030 5688 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5689
5690 return IRQ_HANDLED;
5691}
5692
c50b52a0 5693static void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 5694{
047e0030 5695 struct igb_adapter *adapter = q_vector->adapter;
46544258 5696 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5697
0ba82994
AD		 5698 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
5699 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
5700 if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
5701 igb_set_itr(q_vector);
46544258 5702 else
047e0030 5703 igb_update_ring_itr(q_vector);
9d5c8243
AK		 5704 }
5705
46544258
AD		 5706 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5707 if (adapter->msix_entries)
047e0030 5708 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 5709 else
5710 igb_irq_enable(adapter);
5711 }
9d5c8243
AK		 5712}
5713
46544258
AD		 5714/**
5715 * igb_poll - NAPI Rx polling callback
5716 * @napi: napi polling structure
5717 * @budget: count of how many packets we should handle
5718 **/
5719static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 5720{
047e0030
AD		 5721 struct igb_q_vector *q_vector = container_of(napi,
5722 struct igb_q_vector,
5723 napi);
16eb8815 5724 bool clean_complete = true;
9d5c8243 5725
421e02f0 5726#ifdef CONFIG_IGB_DCA
047e0030
AD		 5727 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5728 igb_update_dca(q_vector);
fe4506b6 5729#endif
0ba82994 5730 if (q_vector->tx.ring)
13fde97a 5731 clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 5732
0ba82994 5733 if (q_vector->rx.ring)
cd392f5c 5734 clean_complete &= igb_clean_rx_irq(q_vector, budget);
047e0030 5735
16eb8815
AD		 5736 /* If all work not completed, return budget and keep polling */
5737 if (!clean_complete)
5738 return budget;
46544258 5739
9d5c8243 5740 /* If not enough Rx work done, exit the polling mode */
16eb8815
AD		 5741 napi_complete(napi);
5742 igb_ring_irq_enable(q_vector);
9d5c8243 5743
16eb8815 5744 return 0;
9d5c8243 5745}
6d8126f9 5746
7ebae817 5747#ifdef CONFIG_IGB_PTP
c5b9bd5e
AD		 5748/**
5749 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5750 * @q_vector: pointer to q_vector containing needed info
06034649 5751 * @buffer: pointer to igb_tx_buffer structure
33af6bcc
PO		 5752 *
5753 * If we were asked to do hardware stamping and such a time stamp is
5754 * available, then it must have been for this skb here because we only
5755 * allow only one such packet into the queue.
5756 */
06034649
AD		 5757static void igb_tx_hwtstamp(struct igb_q_vector *q_vector,
5758 struct igb_tx_buffer *buffer_info)
33af6bcc 5759{
c5b9bd5e 5760 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc 5761 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 5762 struct skb_shared_hwtstamps shhwtstamps;
5763 u64 regval;
33af6bcc 5764
c5b9bd5e 5765 /* if skb does not support hw timestamp or TX stamp not valid exit */
2bbfebe2 5766 if (likely(!(buffer_info->tx_flags & IGB_TX_FLAGS_TSTAMP)) ||
c5b9bd5e
AD		 5767 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5768 return;
5769
5770 regval = rd32(E1000_TXSTMPL);
5771 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5772
5773 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
2873957d 5774 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
33af6bcc
PO		 5775}
5776
7ebae817 5777#endif
9d5c8243
AK		 5778/**
5779 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 5780 * @q_vector: pointer to q_vector containing needed info
49ce9c2c 5781 *
9d5c8243
AK		 5782 * returns true if ring is completely cleaned
5783 **/
047e0030 5784static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 5785{
047e0030 5786 struct igb_adapter *adapter = q_vector->adapter;
0ba82994 5787 struct igb_ring *tx_ring = q_vector->tx.ring;
06034649 5788 struct igb_tx_buffer *tx_buffer;
8542db05 5789 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 5790 unsigned int total_bytes = 0, total_packets = 0;
0ba82994 5791 unsigned int budget = q_vector->tx.work_limit;
8542db05 5792 unsigned int i = tx_ring->next_to_clean;
9d5c8243 5793
13fde97a
AD		 5794 if (test_bit(__IGB_DOWN, &adapter->state))
5795 return true;
0e014cb1 5796
06034649 5797 tx_buffer = &tx_ring->tx_buffer_info[i];
13fde97a 5798 tx_desc = IGB_TX_DESC(tx_ring, i);
8542db05 5799 i -= tx_ring->count;
9d5c8243 5800
13fde97a 5801 for (; budget; budget--) {
8542db05 5802 eop_desc = tx_buffer->next_to_watch;
13fde97a 5803
8542db05
AD		 5804 /* prevent any other reads prior to eop_desc */
5805 rmb();
5806
5807 /* if next_to_watch is not set then there is no work pending */
5808 if (!eop_desc)
5809 break;
13fde97a
AD		 5810
5811 /* if DD is not set pending work has not been completed */
5812 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
5813 break;
5814
8542db05
AD		 5815 /* clear next_to_watch to prevent false hangs */
5816 tx_buffer->next_to_watch = NULL;
9d5c8243 5817
ebe42d16
AD		 5818 /* update the statistics for this packet */
5819 total_bytes += tx_buffer->bytecount;
5820 total_packets += tx_buffer->gso_segs;
13fde97a 5821
7ebae817 5822#ifdef CONFIG_IGB_PTP
ebe42d16
AD		 5823 /* retrieve hardware timestamp */
5824 igb_tx_hwtstamp(q_vector, tx_buffer);
5825
7ebae817 5826#endif
ebe42d16
AD		 5827 /* free the skb */
5828 dev_kfree_skb_any(tx_buffer->skb);
5829 tx_buffer->skb = NULL;
13fde97a 5830
ebe42d16
AD		 5831 /* unmap skb header data */
5832 dma_unmap_single(tx_ring->dev,
5833 tx_buffer->dma,
5834 tx_buffer->length,
5835 DMA_TO_DEVICE);
5836
5837 /* clear last DMA location and unmap remaining buffers */
5838 while (tx_desc != eop_desc) {
5839 tx_buffer->dma = 0;
9d5c8243 5840
13fde97a
AD		 5841 tx_buffer++;
5842 tx_desc++;
9d5c8243 5843 i++;
8542db05
AD		 5844 if (unlikely(!i)) {
5845 i -= tx_ring->count;
06034649 5846 tx_buffer = tx_ring->tx_buffer_info;
13fde97a
AD		 5847 tx_desc = IGB_TX_DESC(tx_ring, 0);
5848 }
ebe42d16
AD		 5849
5850 /* unmap any remaining paged data */
5851 if (tx_buffer->dma) {
5852 dma_unmap_page(tx_ring->dev,
5853 tx_buffer->dma,
5854 tx_buffer->length,
5855 DMA_TO_DEVICE);
5856 }
5857 }
5858
5859 /* clear last DMA location */
5860 tx_buffer->dma = 0;
5861
5862 /* move us one more past the eop_desc for start of next pkt */
5863 tx_buffer++;
5864 tx_desc++;
5865 i++;
5866 if (unlikely(!i)) {
5867 i -= tx_ring->count;
5868 tx_buffer = tx_ring->tx_buffer_info;
5869 tx_desc = IGB_TX_DESC(tx_ring, 0);
5870 }
0e014cb1
AD		 5871 }
5872
bdbc0631
ED		 5873 netdev_tx_completed_queue(txring_txq(tx_ring),
5874 total_packets, total_bytes);
8542db05 5875 i += tx_ring->count;
9d5c8243 5876 tx_ring->next_to_clean = i;
13fde97a
AD		 5877 u64_stats_update_begin(&tx_ring->tx_syncp);
5878 tx_ring->tx_stats.bytes += total_bytes;
5879 tx_ring->tx_stats.packets += total_packets;
5880 u64_stats_update_end(&tx_ring->tx_syncp);
0ba82994
AD		 5881 q_vector->tx.total_bytes += total_bytes;
5882 q_vector->tx.total_packets += total_packets;
9d5c8243 5883
6d095fa8 5884 if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
13fde97a 5885 struct e1000_hw *hw = &adapter->hw;
12dcd86b 5886
8542db05 5887 eop_desc = tx_buffer->next_to_watch;
9d5c8243 5888
9d5c8243
AK		 5889 /* Detect a transmit hang in hardware, this serializes the
5890 * check with the clearing of time_stamp and movement of i */
6d095fa8 5891 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
8542db05
AD		 5892 if (eop_desc &&
5893 time_after(jiffies, tx_buffer->time_stamp +
8e95a202
JP		 5894 (adapter->tx_timeout_factor * HZ)) &&
5895 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5896
9d5c8243 5897 /* detected Tx unit hang */
59d71989 5898 dev_err(tx_ring->dev,
9d5c8243 5899 "Detected Tx Unit Hang\n"
2d064c06 5900 " Tx Queue <%d>\n"
9d5c8243
AK		 5901 " TDH <%x>\n"
5902 " TDT <%x>\n"
5903 " next_to_use <%x>\n"
5904 " next_to_clean <%x>\n"
9d5c8243
AK		 5905 "buffer_info[next_to_clean]\n"
5906 " time_stamp <%lx>\n"
8542db05 5907 " next_to_watch <%p>\n"
9d5c8243
AK		 5908 " jiffies <%lx>\n"
5909 " desc.status <%x>\n",
2d064c06 5910 tx_ring->queue_index,
238ac817 5911 rd32(E1000_TDH(tx_ring->reg_idx)),
fce99e34 5912 readl(tx_ring->tail),
9d5c8243
AK		 5913 tx_ring->next_to_use,
5914 tx_ring->next_to_clean,
8542db05
AD		 5915 tx_buffer->time_stamp,
5916 eop_desc,
9d5c8243 5917 jiffies,
0e014cb1 5918 eop_desc->wb.status);
13fde97a
AD		 5919 netif_stop_subqueue(tx_ring->netdev,
5920 tx_ring->queue_index);
5921
5922 /* we are about to reset, no point in enabling stuff */
5923 return true;
9d5c8243
AK		 5924 }
5925 }
13fde97a
AD		 5926
5927 if (unlikely(total_packets &&
5928 netif_carrier_ok(tx_ring->netdev) &&
5929 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5930 /* Make sure that anybody stopping the queue after this
5931 * sees the new next_to_clean.
5932 */
5933 smp_mb();
5934 if (__netif_subqueue_stopped(tx_ring->netdev,
5935 tx_ring->queue_index) &&
5936 !(test_bit(__IGB_DOWN, &adapter->state))) {
5937 netif_wake_subqueue(tx_ring->netdev,
5938 tx_ring->queue_index);
5939
5940 u64_stats_update_begin(&tx_ring->tx_syncp);
5941 tx_ring->tx_stats.restart_queue++;
5942 u64_stats_update_end(&tx_ring->tx_syncp);
5943 }
5944 }
5945
5946 return !!budget;
9d5c8243
AK		 5947}
5948
cd392f5c 5949static inline void igb_rx_checksum(struct igb_ring *ring,
3ceb90fd
AD		 5950 union e1000_adv_rx_desc *rx_desc,
5951 struct sk_buff *skb)
9d5c8243 5952{
bc8acf2c 5953 skb_checksum_none_assert(skb);
9d5c8243 5954
294e7d78 5955 /* Ignore Checksum bit is set */
3ceb90fd 5956 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
294e7d78
AD		 5957 return;
5958
5959 /* Rx checksum disabled via ethtool */
5960 if (!(ring->netdev->features & NETIF_F_RXCSUM))
9d5c8243 5961 return;
85ad76b2 5962
9d5c8243 5963 /* TCP/UDP checksum error bit is set */
3ceb90fd
AD		 5964 if (igb_test_staterr(rx_desc,
5965 E1000_RXDEXT_STATERR_TCPE |
5966 E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5967 /*
5968 * work around errata with sctp packets where the TCPE aka
5969 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5970 * packets, (aka let the stack check the crc32c)
5971 */
866cff06
AD		 5972 if (!((skb->len == 60) &&
5973 test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
12dcd86b 5974 u64_stats_update_begin(&ring->rx_syncp);
04a5fcaa 5975 ring->rx_stats.csum_err++;
12dcd86b
ED		 5976 u64_stats_update_end(&ring->rx_syncp);
5977 }
9d5c8243 5978 /* let the stack verify checksum errors */
9d5c8243
AK		 5979 return;
5980 }
5981 /* It must be a TCP or UDP packet with a valid checksum */
3ceb90fd
AD		 5982 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
5983 E1000_RXD_STAT_UDPCS))
9d5c8243
AK		 5984 skb->ip_summed = CHECKSUM_UNNECESSARY;
5985
3ceb90fd
AD		 5986 dev_dbg(ring->dev, "cksum success: bits %08X\n",
5987 le32_to_cpu(rx_desc->wb.upper.status_error));
9d5c8243
AK		 5988}
5989
077887c3
AD		 5990static inline void igb_rx_hash(struct igb_ring *ring,
5991 union e1000_adv_rx_desc *rx_desc,
5992 struct sk_buff *skb)
5993{
5994 if (ring->netdev->features & NETIF_F_RXHASH)
5995 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
5996}
5997
7ebae817 5998#ifdef CONFIG_IGB_PTP
3ceb90fd
AD		 5999static void igb_rx_hwtstamp(struct igb_q_vector *q_vector,
6000 union e1000_adv_rx_desc *rx_desc,
6001 struct sk_buff *skb)
c5b9bd5e
AD		 6002{
6003 struct igb_adapter *adapter = q_vector->adapter;
6004 struct e1000_hw *hw = &adapter->hw;
6005 u64 regval;
6006
3ceb90fd
AD		 6007 if (!igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP |
6008 E1000_RXDADV_STAT_TS))
6009 return;
6010
c5b9bd5e
AD		 6011 /*
6012 * If this bit is set, then the RX registers contain the time stamp. No
6013 * other packet will be time stamped until we read these registers, so
6014 * read the registers to make them available again. Because only one
6015 * packet can be time stamped at a time, we know that the register
6016 * values must belong to this one here and therefore we don't need to
6017 * compare any of the additional attributes stored for it.
6018 *
2244d07b 6019 * If nothing went wrong, then it should have a shared tx_flags that we
c5b9bd5e
AD		 6020 * can turn into a skb_shared_hwtstamps.
6021 */
3ceb90fd 6022 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
757b77e2
NN		 6023 u32 *stamp = (u32 *)skb->data;
6024 regval = le32_to_cpu(*(stamp + 2));
6025 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
6026 skb_pull(skb, IGB_TS_HDR_LEN);
6027 } else {
6028 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
6029 return;
c5b9bd5e 6030
757b77e2
NN		 6031 regval = rd32(E1000_RXSTMPL);
6032 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
6033 }
c5b9bd5e
AD		 6034
6035 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
6036}
8be10e91 6037
7ebae817 6038#endif
8be10e91
AD		 6039static void igb_rx_vlan(struct igb_ring *ring,
6040 union e1000_adv_rx_desc *rx_desc,
6041 struct sk_buff *skb)
6042{
6043 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
6044 u16 vid;
6045 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
6046 test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags))
6047 vid = be16_to_cpu(rx_desc->wb.upper.vlan);
6048 else
6049 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
6050
6051 __vlan_hwaccel_put_tag(skb, vid);
6052 }
6053}
6054
44390ca6 6055static inline u16 igb_get_hlen(union e1000_adv_rx_desc *rx_desc)
2d94d8ab
AD		 6056{
6057 /* HW will not DMA in data larger than the given buffer, even if it
6058 * parses the (NFS, of course) header to be larger. In that case, it
6059 * fills the header buffer and spills the rest into the page.
6060 */
6061 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
6062 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
44390ca6
AD		 6063 if (hlen > IGB_RX_HDR_LEN)
6064 hlen = IGB_RX_HDR_LEN;
2d94d8ab
AD		 6065 return hlen;
6066}
6067
cd392f5c 6068static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, int budget)
9d5c8243 6069{
0ba82994 6070 struct igb_ring *rx_ring = q_vector->rx.ring;
16eb8815
AD		 6071 union e1000_adv_rx_desc *rx_desc;
6072 const int current_node = numa_node_id();
9d5c8243 6073 unsigned int total_bytes = 0, total_packets = 0;
16eb8815
AD		 6074 u16 cleaned_count = igb_desc_unused(rx_ring);
6075 u16 i = rx_ring->next_to_clean;
9d5c8243 6076
60136906 6077 rx_desc = IGB_RX_DESC(rx_ring, i);
9d5c8243 6078
3ceb90fd 6079 while (igb_test_staterr(rx_desc, E1000_RXD_STAT_DD)) {
06034649 6080 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
16eb8815
AD		 6081 struct sk_buff *skb = buffer_info->skb;
6082 union e1000_adv_rx_desc *next_rxd;
9d5c8243 6083
69d3ca53 6084 buffer_info->skb = NULL;
16eb8815 6085 prefetch(skb->data);
69d3ca53
AD		 6086
6087 i++;
6088 if (i == rx_ring->count)
6089 i = 0;
42d0781a 6090
60136906 6091 next_rxd = IGB_RX_DESC(rx_ring, i);
69d3ca53 6092 prefetch(next_rxd);
9d5c8243 6093
16eb8815
AD		 6094 /*
6095 * This memory barrier is needed to keep us from reading
6096 * any other fields out of the rx_desc until we know the
6097 * RXD_STAT_DD bit is set
6098 */
6099 rmb();
9d5c8243 6100
16eb8815
AD		 6101 if (!skb_is_nonlinear(skb)) {
6102 __skb_put(skb, igb_get_hlen(rx_desc));
6103 dma_unmap_single(rx_ring->dev, buffer_info->dma,
44390ca6 6104 IGB_RX_HDR_LEN,
59d71989 6105 DMA_FROM_DEVICE);
91615f76 6106 buffer_info->dma = 0;
bf36c1a0
AD		 6107 }
6108
16eb8815
AD		 6109 if (rx_desc->wb.upper.length) {
6110 u16 length = le16_to_cpu(rx_desc->wb.upper.length);
bf36c1a0 6111
aa913403 6112 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
bf36c1a0
AD		 6113 buffer_info->page,
6114 buffer_info->page_offset,
6115 length);
6116
16eb8815
AD		 6117 skb->len += length;
6118 skb->data_len += length;
95b9c1df 6119 skb->truesize += PAGE_SIZE / 2;
16eb8815 6120
d1eff350
AD		 6121 if ((page_count(buffer_info->page) != 1) ||
6122 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 6123 buffer_info->page = NULL;
6124 else
6125 get_page(buffer_info->page);
9d5c8243 6126
16eb8815
AD		 6127 dma_unmap_page(rx_ring->dev, buffer_info->page_dma,
6128 PAGE_SIZE / 2, DMA_FROM_DEVICE);
6129 buffer_info->page_dma = 0;
9d5c8243 6130 }
9d5c8243 6131
3ceb90fd 6132 if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)) {
06034649
AD		 6133 struct igb_rx_buffer *next_buffer;
6134 next_buffer = &rx_ring->rx_buffer_info[i];
b2d56536
AD		 6135 buffer_info->skb = next_buffer->skb;
6136 buffer_info->dma = next_buffer->dma;
6137 next_buffer->skb = skb;
6138 next_buffer->dma = 0;
bf36c1a0
AD		 6139 goto next_desc;
6140 }
44390ca6 6141
89eaefb6
BG		 6142 if (unlikely((igb_test_staterr(rx_desc,
6143 E1000_RXDEXT_ERR_FRAME_ERR_MASK))
6144 && !(rx_ring->netdev->features & NETIF_F_RXALL))) {
16eb8815 6145 dev_kfree_skb_any(skb);
9d5c8243
AK		 6146 goto next_desc;
6147 }
9d5c8243 6148
7ebae817 6149#ifdef CONFIG_IGB_PTP
3ceb90fd 6150 igb_rx_hwtstamp(q_vector, rx_desc, skb);
7ebae817 6151#endif
077887c3 6152 igb_rx_hash(rx_ring, rx_desc, skb);
3ceb90fd 6153 igb_rx_checksum(rx_ring, rx_desc, skb);
8be10e91 6154 igb_rx_vlan(rx_ring, rx_desc, skb);
3ceb90fd
AD		 6155
6156 total_bytes += skb->len;
6157 total_packets++;
6158
6159 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
6160
b2cb09b1 6161 napi_gro_receive(&q_vector->napi, skb);
9d5c8243 6162
16eb8815 6163 budget--;
9d5c8243 6164next_desc:
16eb8815
AD		 6165 if (!budget)
6166 break;
6167
6168 cleaned_count++;
9d5c8243
AK		 6169 /* return some buffers to hardware, one at a time is too slow */
6170 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
cd392f5c 6171 igb_alloc_rx_buffers(rx_ring, cleaned_count);
9d5c8243
AK		 6172 cleaned_count = 0;
6173 }
6174
6175 /* use prefetched values */
6176 rx_desc = next_rxd;
9d5c8243 6177 }
bf36c1a0 6178
9d5c8243 6179 rx_ring->next_to_clean = i;
12dcd86b 6180 u64_stats_update_begin(&rx_ring->rx_syncp);
9d5c8243
AK		 6181 rx_ring->rx_stats.packets += total_packets;
6182 rx_ring->rx_stats.bytes += total_bytes;
12dcd86b 6183 u64_stats_update_end(&rx_ring->rx_syncp);
0ba82994
AD		 6184 q_vector->rx.total_packets += total_packets;
6185 q_vector->rx.total_bytes += total_bytes;
c023cd88
AD		 6186
6187 if (cleaned_count)
cd392f5c 6188 igb_alloc_rx_buffers(rx_ring, cleaned_count);
c023cd88 6189
16eb8815 6190 return !!budget;
9d5c8243
AK		 6191}
6192
c023cd88 6193static bool igb_alloc_mapped_skb(struct igb_ring *rx_ring,
06034649 6194 struct igb_rx_buffer *bi)
c023cd88
AD		 6195{
6196 struct sk_buff *skb = bi->skb;
6197 dma_addr_t dma = bi->dma;
6198
6199 if (dma)
6200 return true;
6201
6202 if (likely(!skb)) {
6203 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
6204 IGB_RX_HDR_LEN);
6205 bi->skb = skb;
6206 if (!skb) {
6207 rx_ring->rx_stats.alloc_failed++;
6208 return false;
6209 }
6210
6211 /* initialize skb for ring */
6212 skb_record_rx_queue(skb, rx_ring->queue_index);
6213 }
6214
6215 dma = dma_map_single(rx_ring->dev, skb->data,
6216 IGB_RX_HDR_LEN, DMA_FROM_DEVICE);
6217
6218 if (dma_mapping_error(rx_ring->dev, dma)) {
6219 rx_ring->rx_stats.alloc_failed++;
6220 return false;
6221 }
6222
6223 bi->dma = dma;
6224 return true;
6225}
6226
6227static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
06034649 6228 struct igb_rx_buffer *bi)
c023cd88
AD		 6229{
6230 struct page *page = bi->page;
6231 dma_addr_t page_dma = bi->page_dma;
6232 unsigned int page_offset = bi->page_offset ^ (PAGE_SIZE / 2);
6233
6234 if (page_dma)
6235 return true;
6236
6237 if (!page) {
0614002b 6238 page = __skb_alloc_page(GFP_ATOMIC, bi->skb);
c023cd88
AD		 6239 bi->page = page;
6240 if (unlikely(!page)) {
6241 rx_ring->rx_stats.alloc_failed++;
6242 return false;
6243 }
6244 }
6245
6246 page_dma = dma_map_page(rx_ring->dev, page,
6247 page_offset, PAGE_SIZE / 2,
6248 DMA_FROM_DEVICE);
6249
6250 if (dma_mapping_error(rx_ring->dev, page_dma)) {
6251 rx_ring->rx_stats.alloc_failed++;
6252 return false;
6253 }
6254
6255 bi->page_dma = page_dma;
6256 bi->page_offset = page_offset;
6257 return true;
6258}
6259
9d5c8243 6260/**
cd392f5c 6261 * igb_alloc_rx_buffers - Replace used receive buffers; packet split
9d5c8243
AK		 6262 * @adapter: address of board private structure
6263 **/
cd392f5c 6264void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
9d5c8243 6265{
9d5c8243 6266 union e1000_adv_rx_desc *rx_desc;
06034649 6267 struct igb_rx_buffer *bi;
c023cd88 6268 u16 i = rx_ring->next_to_use;
9d5c8243 6269
60136906 6270 rx_desc = IGB_RX_DESC(rx_ring, i);
06034649 6271 bi = &rx_ring->rx_buffer_info[i];
c023cd88 6272 i -= rx_ring->count;
9d5c8243
AK		 6273
6274 while (cleaned_count--) {
c023cd88
AD		 6275 if (!igb_alloc_mapped_skb(rx_ring, bi))
6276 break;
9d5c8243 6277
c023cd88
AD		 6278 /* Refresh the desc even if buffer_addrs didn't change
6279 * because each write-back erases this info. */
6280 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
9d5c8243 6281
c023cd88
AD		 6282 if (!igb_alloc_mapped_page(rx_ring, bi))
6283 break;
6284
6285 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
9d5c8243 6286
c023cd88
AD		 6287 rx_desc++;
6288 bi++;
9d5c8243 6289 i++;
c023cd88 6290 if (unlikely(!i)) {
60136906 6291 rx_desc = IGB_RX_DESC(rx_ring, 0);
06034649 6292 bi = rx_ring->rx_buffer_info;
c023cd88
AD		 6293 i -= rx_ring->count;
6294 }
6295
6296 /* clear the hdr_addr for the next_to_use descriptor */
6297 rx_desc->read.hdr_addr = 0;
9d5c8243
AK		 6298 }
6299
c023cd88
AD		 6300 i += rx_ring->count;
6301
9d5c8243
AK		 6302 if (rx_ring->next_to_use != i) {
6303 rx_ring->next_to_use = i;
9d5c8243
AK		 6304
6305 /* Force memory writes to complete before letting h/w
6306 * know there are new descriptors to fetch. (Only
6307 * applicable for weak-ordered memory model archs,
6308 * such as IA-64). */
6309 wmb();
fce99e34 6310 writel(i, rx_ring->tail);
9d5c8243
AK		 6311 }
6312}
6313
6314/**
6315 * igb_mii_ioctl -
6316 * @netdev:
6317 * @ifreq:
6318 * @cmd:
6319 **/
6320static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6321{
6322 struct igb_adapter *adapter = netdev_priv(netdev);
6323 struct mii_ioctl_data *data = if_mii(ifr);
6324
6325 if (adapter->hw.phy.media_type != e1000_media_type_copper)
6326 return -EOPNOTSUPP;
6327
6328 switch (cmd) {
6329 case SIOCGMIIPHY:
6330 data->phy_id = adapter->hw.phy.addr;
6331 break;
6332 case SIOCGMIIREG:
f5f4cf08
AD		 6333 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
6334 &data->val_out))
9d5c8243
AK		 6335 return -EIO;
6336 break;
6337 case SIOCSMIIREG:
6338 default:
6339 return -EOPNOTSUPP;
6340 }
6341 return 0;
6342}
6343
c6cb090b
PO		 6344/**
6345 * igb_hwtstamp_ioctl - control hardware time stamping
6346 * @netdev:
6347 * @ifreq:
6348 * @cmd:
6349 *
33af6bcc
PO		 6350 * Outgoing time stamping can be enabled and disabled. Play nice and
6351 * disable it when requested, although it shouldn't case any overhead
6352 * when no packet needs it. At most one packet in the queue may be
6353 * marked for time stamping, otherwise it would be impossible to tell
6354 * for sure to which packet the hardware time stamp belongs.
6355 *
6356 * Incoming time stamping has to be configured via the hardware
6357 * filters. Not all combinations are supported, in particular event
6358 * type has to be specified. Matching the kind of event packet is
6359 * not supported, with the exception of "all V2 events regardless of
6360 * level 2 or 4".
6361 *
c6cb090b
PO		 6362 **/
6363static int igb_hwtstamp_ioctl(struct net_device *netdev,
6364 struct ifreq *ifr, int cmd)
6365{
33af6bcc
PO		 6366 struct igb_adapter *adapter = netdev_priv(netdev);
6367 struct e1000_hw *hw = &adapter->hw;
c6cb090b 6368 struct hwtstamp_config config;
c5b9bd5e
AD		 6369 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
6370 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 6371 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 6372 bool is_l4 = false;
6373 bool is_l2 = false;
33af6bcc 6374 u32 regval;
c6cb090b
PO		 6375
6376 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
6377 return -EFAULT;
6378
6379 /* reserved for future extensions */
6380 if (config.flags)
6381 return -EINVAL;
6382
33af6bcc
PO		 6383 switch (config.tx_type) {
6384 case HWTSTAMP_TX_OFF:
c5b9bd5e 6385 tsync_tx_ctl = 0;
33af6bcc 6386 case HWTSTAMP_TX_ON:
33af6bcc
PO		 6387 break;
6388 default:
6389 return -ERANGE;
6390 }
6391
6392 switch (config.rx_filter) {
6393 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 6394 tsync_rx_ctl = 0;
33af6bcc
PO		 6395 break;
6396 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
6397 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
6398 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
6399 case HWTSTAMP_FILTER_ALL:
6400 /*
6401 * register TSYNCRXCFG must be set, therefore it is not
6402 * possible to time stamp both Sync and Delay_Req messages
6403 * => fall back to time stamping all packets
6404 */
c5b9bd5e 6405 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 6406 config.rx_filter = HWTSTAMP_FILTER_ALL;
6407 break;
6408 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 6409 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 6410 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 6411 is_l4 = true;
33af6bcc
PO		 6412 break;
6413 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 6414 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 6415 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 6416 is_l4 = true;
33af6bcc
PO		 6417 break;
6418 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
6419 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 6420 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 6421 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 6422 is_l2 = true;
6423 is_l4 = true;
33af6bcc
PO		 6424 config.rx_filter = HWTSTAMP_FILTER_SOME;
6425 break;
6426 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
6427 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 6428 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 6429 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 6430 is_l2 = true;
6431 is_l4 = true;
33af6bcc
PO		 6432 config.rx_filter = HWTSTAMP_FILTER_SOME;
6433 break;
6434 case HWTSTAMP_FILTER_PTP_V2_EVENT:
6435 case HWTSTAMP_FILTER_PTP_V2_SYNC:
6436 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 6437 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 6438 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 6439 is_l2 = true;
11ba69e8 6440 is_l4 = true;
33af6bcc
PO		 6441 break;
6442 default:
6443 return -ERANGE;
6444 }
6445
c5b9bd5e
AD		 6446 if (hw->mac.type == e1000_82575) {
6447 if (tsync_rx_ctl | tsync_tx_ctl)
6448 return -EINVAL;
6449 return 0;
6450 }
6451
757b77e2
NN		 6452 /*
6453 * Per-packet timestamping only works if all packets are
6454 * timestamped, so enable timestamping in all packets as
6455 * long as one rx filter was configured.
6456 */
06218a8d 6457 if ((hw->mac.type >= e1000_82580) && tsync_rx_ctl) {
757b77e2
NN		 6458 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
6459 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
6460 }
6461
33af6bcc
PO		 6462 /* enable/disable TX */
6463 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 6464 regval &= ~E1000_TSYNCTXCTL_ENABLED;
6465 regval |= tsync_tx_ctl;
33af6bcc
PO		 6466 wr32(E1000_TSYNCTXCTL, regval);
6467
c5b9bd5e 6468 /* enable/disable RX */
33af6bcc 6469 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 6470 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
6471 regval |= tsync_rx_ctl;
33af6bcc 6472 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 6473
c5b9bd5e
AD		 6474 /* define which PTP packets are time stamped */
6475 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 6476
c5b9bd5e
AD		 6477 /* define ethertype filter for timestamped packets */
6478 if (is_l2)
6479 wr32(E1000_ETQF(3),
6480 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
6481 E1000_ETQF_1588 | /* enable timestamping */
6482 ETH_P_1588)); /* 1588 eth protocol type */
6483 else
6484 wr32(E1000_ETQF(3), 0);
6485
6486#define PTP_PORT 319
6487 /* L4 Queue Filter[3]: filter by destination port and protocol */
6488 if (is_l4) {
6489 u32 ftqf = (IPPROTO_UDP /* UDP */
6490 | E1000_FTQF_VF_BP /* VF not compared */
6491 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
6492 | E1000_FTQF_MASK); /* mask all inputs */
6493 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
6494
6495 wr32(E1000_IMIR(3), htons(PTP_PORT));
6496 wr32(E1000_IMIREXT(3),
6497 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
6498 if (hw->mac.type == e1000_82576) {
6499 /* enable source port check */
6500 wr32(E1000_SPQF(3), htons(PTP_PORT));
6501 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
6502 }
6503 wr32(E1000_FTQF(3), ftqf);
6504 } else {
6505 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
6506 }
33af6bcc
PO		 6507 wrfl();
6508
6509 adapter->hwtstamp_config = config;
6510
6511 /* clear TX/RX time stamp registers, just to be sure */
6512 regval = rd32(E1000_TXSTMPH);
6513 regval = rd32(E1000_RXSTMPH);
c6cb090b 6514
33af6bcc
PO		 6515 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
6516 -EFAULT : 0;
c6cb090b
PO		 6517}
6518
9d5c8243
AK		 6519/**
6520 * igb_ioctl -
6521 * @netdev:
6522 * @ifreq:
6523 * @cmd:
6524 **/
6525static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6526{
6527 switch (cmd) {
6528 case SIOCGMIIPHY:
6529 case SIOCGMIIREG:
6530 case SIOCSMIIREG:
6531 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 6532 case SIOCSHWTSTAMP:
6533 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 6534 default:
6535 return -EOPNOTSUPP;
6536 }
6537}
6538
009bc06e
AD		 6539s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6540{
6541 struct igb_adapter *adapter = hw->back;
6542 u16 cap_offset;
6543
bdaae04c 6544 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6545 if (!cap_offset)
6546 return -E1000_ERR_CONFIG;
6547
6548 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6549
6550 return 0;
6551}
6552
6553s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6554{
6555 struct igb_adapter *adapter = hw->back;
6556 u16 cap_offset;
6557
bdaae04c 6558 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6559 if (!cap_offset)
6560 return -E1000_ERR_CONFIG;
6561
6562 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6563
6564 return 0;
6565}
6566
c8f44aff 6567static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features)
9d5c8243
AK		 6568{
6569 struct igb_adapter *adapter = netdev_priv(netdev);
6570 struct e1000_hw *hw = &adapter->hw;
6571 u32 ctrl, rctl;
5faf030c 6572 bool enable = !!(features & NETIF_F_HW_VLAN_RX);
9d5c8243 6573
5faf030c 6574 if (enable) {
9d5c8243
AK		 6575 /* enable VLAN tag insert/strip */
6576 ctrl = rd32(E1000_CTRL);
6577 ctrl |= E1000_CTRL_VME;
6578 wr32(E1000_CTRL, ctrl);
6579
51466239 6580 /* Disable CFI check */
9d5c8243 6581 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 6582 rctl &= ~E1000_RCTL_CFIEN;
6583 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 6584 } else {
6585 /* disable VLAN tag insert/strip */
6586 ctrl = rd32(E1000_CTRL);
6587 ctrl &= ~E1000_CTRL_VME;
6588 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 6589 }
6590
e1739522 6591 igb_rlpml_set(adapter);
9d5c8243
AK		 6592}
6593
8e586137 6594static int igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
9d5c8243
AK		 6595{
6596 struct igb_adapter *adapter = netdev_priv(netdev);
6597 struct e1000_hw *hw = &adapter->hw;
4ae196df 6598 int pf_id = adapter->vfs_allocated_count;
9d5c8243 6599
51466239
AD		 6600 /* attempt to add filter to vlvf array */
6601 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 6602
51466239
AD		 6603 /* add the filter since PF can receive vlans w/o entry in vlvf */
6604 igb_vfta_set(hw, vid, true);
b2cb09b1
JP		 6605
6606 set_bit(vid, adapter->active_vlans);
8e586137
JP		 6607
6608 return 0;
9d5c8243
AK		 6609}
6610
8e586137 6611static int igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
9d5c8243
AK		 6612{
6613 struct igb_adapter *adapter = netdev_priv(netdev);
6614 struct e1000_hw *hw = &adapter->hw;
4ae196df 6615 int pf_id = adapter->vfs_allocated_count;
51466239 6616 s32 err;
9d5c8243 6617
51466239
AD		 6618 /* remove vlan from VLVF table array */
6619 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 6620
51466239
AD		 6621 /* if vid was not present in VLVF just remove it from table */
6622 if (err)
4ae196df 6623 igb_vfta_set(hw, vid, false);
b2cb09b1
JP		 6624
6625 clear_bit(vid, adapter->active_vlans);
8e586137
JP		 6626
6627 return 0;
9d5c8243
AK		 6628}
6629
6630static void igb_restore_vlan(struct igb_adapter *adapter)
6631{
b2cb09b1 6632 u16 vid;
9d5c8243 6633
5faf030c
AD		 6634 igb_vlan_mode(adapter->netdev, adapter->netdev->features);
6635
b2cb09b1
JP		 6636 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
6637 igb_vlan_rx_add_vid(adapter->netdev, vid);
9d5c8243
AK		 6638}
6639
14ad2513 6640int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
9d5c8243 6641{
090b1795 6642 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 6643 struct e1000_mac_info *mac = &adapter->hw.mac;
6644
6645 mac->autoneg = 0;
6646
14ad2513
DD		 6647 /* Make sure dplx is at most 1 bit and lsb of speed is not set
6648 * for the switch() below to work */
6649 if ((spd & 1) || (dplx & ~1))
6650 goto err_inval;
6651
cd2638a8
CW		 6652 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6653 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
14ad2513
DD		 6654 spd != SPEED_1000 &&
6655 dplx != DUPLEX_FULL)
6656 goto err_inval;
cd2638a8 6657
14ad2513 6658 switch (spd + dplx) {
9d5c8243
AK		 6659 case SPEED_10 + DUPLEX_HALF:
6660 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6661 break;
6662 case SPEED_10 + DUPLEX_FULL:
6663 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6664 break;
6665 case SPEED_100 + DUPLEX_HALF:
6666 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6667 break;
6668 case SPEED_100 + DUPLEX_FULL:
6669 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6670 break;
6671 case SPEED_1000 + DUPLEX_FULL:
6672 mac->autoneg = 1;
6673 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6674 break;
6675 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6676 default:
14ad2513 6677 goto err_inval;
9d5c8243
AK		 6678 }
6679 return 0;
14ad2513
DD		 6680
6681err_inval:
6682 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6683 return -EINVAL;
9d5c8243
AK		 6684}
6685
749ab2cd
YZ		 6686static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
6687 bool runtime)
9d5c8243
AK		 6688{
6689 struct net_device *netdev = pci_get_drvdata(pdev);
6690 struct igb_adapter *adapter = netdev_priv(netdev);
6691 struct e1000_hw *hw = &adapter->hw;
2d064c06 6692 u32 ctrl, rctl, status;
749ab2cd 6693 u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
9d5c8243
AK		 6694#ifdef CONFIG_PM
6695 int retval = 0;
6696#endif
6697
6698 netif_device_detach(netdev);
6699
a88f10ec 6700 if (netif_running(netdev))
749ab2cd 6701 __igb_close(netdev, true);
a88f10ec 6702
047e0030 6703 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 6704
6705#ifdef CONFIG_PM
6706 retval = pci_save_state(pdev);
6707 if (retval)
6708 return retval;
6709#endif
6710
6711 status = rd32(E1000_STATUS);
6712 if (status & E1000_STATUS_LU)
6713 wufc &= ~E1000_WUFC_LNKC;
6714
6715 if (wufc) {
6716 igb_setup_rctl(adapter);
ff41f8dc 6717 igb_set_rx_mode(netdev);
9d5c8243
AK		 6718
6719 /* turn on all-multi mode if wake on multicast is enabled */
6720 if (wufc & E1000_WUFC_MC) {
6721 rctl = rd32(E1000_RCTL);
6722 rctl |= E1000_RCTL_MPE;
6723 wr32(E1000_RCTL, rctl);
6724 }
6725
6726 ctrl = rd32(E1000_CTRL);
6727 /* advertise wake from D3Cold */
6728 #define E1000_CTRL_ADVD3WUC 0x00100000
6729 /* phy power management enable */
6730 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6731 ctrl |= E1000_CTRL_ADVD3WUC;
6732 wr32(E1000_CTRL, ctrl);
6733
9d5c8243 6734 /* Allow time for pending master requests to run */
330a6d6a 6735 igb_disable_pcie_master(hw);
9d5c8243
AK		 6736
6737 wr32(E1000_WUC, E1000_WUC_PME_EN);
6738 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 6739 } else {
6740 wr32(E1000_WUC, 0);
6741 wr32(E1000_WUFC, 0);
9d5c8243
AK		 6742 }
6743
3fe7c4c9
RW		 6744 *enable_wake = wufc || adapter->en_mng_pt;
6745 if (!*enable_wake)
88a268c1
NN		 6746 igb_power_down_link(adapter);
6747 else
6748 igb_power_up_link(adapter);
9d5c8243
AK		 6749
6750 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6751 * would have already happened in close and is redundant. */
6752 igb_release_hw_control(adapter);
6753
6754 pci_disable_device(pdev);
6755
9d5c8243
AK		 6756 return 0;
6757}
6758
6759#ifdef CONFIG_PM
d9dd966d 6760#ifdef CONFIG_PM_SLEEP
749ab2cd 6761static int igb_suspend(struct device *dev)
3fe7c4c9
RW		 6762{
6763 int retval;
6764 bool wake;
749ab2cd 6765 struct pci_dev *pdev = to_pci_dev(dev);
3fe7c4c9 6766
749ab2cd 6767 retval = __igb_shutdown(pdev, &wake, 0);
3fe7c4c9
RW		 6768 if (retval)
6769 return retval;
6770
6771 if (wake) {
6772 pci_prepare_to_sleep(pdev);
6773 } else {
6774 pci_wake_from_d3(pdev, false);
6775 pci_set_power_state(pdev, PCI_D3hot);
6776 }
6777
6778 return 0;
6779}
d9dd966d 6780#endif /* CONFIG_PM_SLEEP */
3fe7c4c9 6781
749ab2cd 6782static int igb_resume(struct device *dev)
9d5c8243 6783{
749ab2cd 6784 struct pci_dev *pdev = to_pci_dev(dev);
9d5c8243
AK		 6785 struct net_device *netdev = pci_get_drvdata(pdev);
6786 struct igb_adapter *adapter = netdev_priv(netdev);
6787 struct e1000_hw *hw = &adapter->hw;
6788 u32 err;
6789
6790 pci_set_power_state(pdev, PCI_D0);
6791 pci_restore_state(pdev);
b94f2d77 6792 pci_save_state(pdev);
42bfd33a 6793
aed5dec3 6794 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 6795 if (err) {
6796 dev_err(&pdev->dev,
6797 "igb: Cannot enable PCI device from suspend\n");
6798 return err;
6799 }
6800 pci_set_master(pdev);
6801
6802 pci_enable_wake(pdev, PCI_D3hot, 0);
6803 pci_enable_wake(pdev, PCI_D3cold, 0);
6804
cfb8c3aa 6805 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 6806 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6807 return -ENOMEM;
9d5c8243
AK		 6808 }
6809
9d5c8243 6810 igb_reset(adapter);
a8564f03
AD		 6811
6812 /* let the f/w know that the h/w is now under the control of the
6813 * driver. */
6814 igb_get_hw_control(adapter);
6815
9d5c8243
AK		 6816 wr32(E1000_WUS, ~0);
6817
749ab2cd
YZ		 6818 if (netdev->flags & IFF_UP) {
6819 err = __igb_open(netdev, true);
a88f10ec
AD		 6820 if (err)
6821 return err;
6822 }
9d5c8243
AK		 6823
6824 netif_device_attach(netdev);
749ab2cd
YZ		 6825 return 0;
6826}
6827
6828#ifdef CONFIG_PM_RUNTIME
6829static int igb_runtime_idle(struct device *dev)
6830{
6831 struct pci_dev *pdev = to_pci_dev(dev);
6832 struct net_device *netdev = pci_get_drvdata(pdev);
6833 struct igb_adapter *adapter = netdev_priv(netdev);
6834
6835 if (!igb_has_link(adapter))
6836 pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
6837
6838 return -EBUSY;
6839}
6840
6841static int igb_runtime_suspend(struct device *dev)
6842{
6843 struct pci_dev *pdev = to_pci_dev(dev);
6844 int retval;
6845 bool wake;
6846
6847 retval = __igb_shutdown(pdev, &wake, 1);
6848 if (retval)
6849 return retval;
6850
6851 if (wake) {
6852 pci_prepare_to_sleep(pdev);
6853 } else {
6854 pci_wake_from_d3(pdev, false);
6855 pci_set_power_state(pdev, PCI_D3hot);
6856 }
9d5c8243 6857
9d5c8243
AK		 6858 return 0;
6859}
749ab2cd
YZ		 6860
6861static int igb_runtime_resume(struct device *dev)
6862{
6863 return igb_resume(dev);
6864}
6865#endif /* CONFIG_PM_RUNTIME */
9d5c8243
AK		 6866#endif
6867
6868static void igb_shutdown(struct pci_dev *pdev)
6869{
3fe7c4c9
RW		 6870 bool wake;
6871
749ab2cd 6872 __igb_shutdown(pdev, &wake, 0);
3fe7c4c9
RW		 6873
6874 if (system_state == SYSTEM_POWER_OFF) {
6875 pci_wake_from_d3(pdev, wake);
6876 pci_set_power_state(pdev, PCI_D3hot);
6877 }
9d5c8243
AK		 6878}
6879
6880#ifdef CONFIG_NET_POLL_CONTROLLER
6881/*
6882 * Polling 'interrupt' - used by things like netconsole to send skbs
6883 * without having to re-enable interrupts. It's not called while
6884 * the interrupt routine is executing.
6885 */
6886static void igb_netpoll(struct net_device *netdev)
6887{
6888 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 6889 struct e1000_hw *hw = &adapter->hw;
0d1ae7f4 6890 struct igb_q_vector *q_vector;
9d5c8243 6891 int i;
9d5c8243 6892
047e0030 6893 for (i = 0; i < adapter->num_q_vectors; i++) {
0d1ae7f4
AD		 6894 q_vector = adapter->q_vector[i];
6895 if (adapter->msix_entries)
6896 wr32(E1000_EIMC, q_vector->eims_value);
6897 else
6898 igb_irq_disable(adapter);
047e0030 6899 napi_schedule(&q_vector->napi);
eebbbdba 6900 }
9d5c8243
AK		 6901}
6902#endif /* CONFIG_NET_POLL_CONTROLLER */
6903
6904/**
6905 * igb_io_error_detected - called when PCI error is detected
6906 * @pdev: Pointer to PCI device
6907 * @state: The current pci connection state
6908 *
6909 * This function is called after a PCI bus error affecting
6910 * this device has been detected.
6911 */
6912static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6913 pci_channel_state_t state)
6914{
6915 struct net_device *netdev = pci_get_drvdata(pdev);
6916 struct igb_adapter *adapter = netdev_priv(netdev);
6917
6918 netif_device_detach(netdev);
6919
59ed6eec
AD		 6920 if (state == pci_channel_io_perm_failure)
6921 return PCI_ERS_RESULT_DISCONNECT;
6922
9d5c8243
AK		 6923 if (netif_running(netdev))
6924 igb_down(adapter);
6925 pci_disable_device(pdev);
6926
6927 /* Request a slot slot reset. */
6928 return PCI_ERS_RESULT_NEED_RESET;
6929}
6930
6931/**
6932 * igb_io_slot_reset - called after the pci bus has been reset.
6933 * @pdev: Pointer to PCI device
6934 *
6935 * Restart the card from scratch, as if from a cold-boot. Implementation
6936 * resembles the first-half of the igb_resume routine.
6937 */
6938static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6939{
6940 struct net_device *netdev = pci_get_drvdata(pdev);
6941 struct igb_adapter *adapter = netdev_priv(netdev);
6942 struct e1000_hw *hw = &adapter->hw;
40a914fa 6943 pci_ers_result_t result;
42bfd33a 6944 int err;
9d5c8243 6945
aed5dec3 6946 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6947 dev_err(&pdev->dev,
6948 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6949 result = PCI_ERS_RESULT_DISCONNECT;
6950 } else {
6951 pci_set_master(pdev);
6952 pci_restore_state(pdev);
b94f2d77 6953 pci_save_state(pdev);
9d5c8243 6954
40a914fa
AD		 6955 pci_enable_wake(pdev, PCI_D3hot, 0);
6956 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6957
40a914fa
AD		 6958 igb_reset(adapter);
6959 wr32(E1000_WUS, ~0);
6960 result = PCI_ERS_RESULT_RECOVERED;
6961 }
9d5c8243 6962
ea943d41
JK		 6963 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6964 if (err) {
6965 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6966 "failed 0x%0x\n", err);
6967 /* non-fatal, continue */
6968 }
40a914fa
AD		 6969
6970 return result;
9d5c8243
AK		 6971}
6972
6973/**
6974 * igb_io_resume - called when traffic can start flowing again.
6975 * @pdev: Pointer to PCI device
6976 *
6977 * This callback is called when the error recovery driver tells us that
6978 * its OK to resume normal operation. Implementation resembles the
6979 * second-half of the igb_resume routine.
6980 */
6981static void igb_io_resume(struct pci_dev *pdev)
6982{
6983 struct net_device *netdev = pci_get_drvdata(pdev);
6984 struct igb_adapter *adapter = netdev_priv(netdev);
6985
9d5c8243
AK		 6986 if (netif_running(netdev)) {
6987 if (igb_up(adapter)) {
6988 dev_err(&pdev->dev, "igb_up failed after reset\n");
6989 return;
6990 }
6991 }
6992
6993 netif_device_attach(netdev);
6994
6995 /* let the f/w know that the h/w is now under the control of the
6996 * driver. */
6997 igb_get_hw_control(adapter);
9d5c8243
AK		 6998}
6999
26ad9178
AD		 7000static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
7001 u8 qsel)
7002{
7003 u32 rar_low, rar_high;
7004 struct e1000_hw *hw = &adapter->hw;
7005
7006 /* HW expects these in little endian so we reverse the byte order
7007 * from network order (big endian) to little endian
7008 */
7009 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
7010 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
7011 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
7012
7013 /* Indicate to hardware the Address is Valid. */
7014 rar_high |= E1000_RAH_AV;
7015
7016 if (hw->mac.type == e1000_82575)
7017 rar_high |= E1000_RAH_POOL_1 * qsel;
7018 else
7019 rar_high |= E1000_RAH_POOL_1 << qsel;
7020
7021 wr32(E1000_RAL(index), rar_low);
7022 wrfl();
7023 wr32(E1000_RAH(index), rar_high);
7024 wrfl();
7025}
7026
4ae196df
AD		 7027static int igb_set_vf_mac(struct igb_adapter *adapter,
7028 int vf, unsigned char *mac_addr)
7029{
7030 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 7031 /* VF MAC addresses start at end of receive addresses and moves
7032 * torwards the first, as a result a collision should not be possible */
7033 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 7034
37680117 7035 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 7036
26ad9178 7037 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 7038
7039 return 0;
7040}
7041
8151d294
WM		 7042static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
7043{
7044 struct igb_adapter *adapter = netdev_priv(netdev);
7045 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
7046 return -EINVAL;
7047 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
7048 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
7049 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
7050 " change effective.");
7051 if (test_bit(__IGB_DOWN, &adapter->state)) {
7052 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
7053 " but the PF device is not up.\n");
7054 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
7055 " attempting to use the VF device.\n");
7056 }
7057 return igb_set_vf_mac(adapter, vf, mac);
7058}
7059
17dc566c
LL		 7060static int igb_link_mbps(int internal_link_speed)
7061{
7062 switch (internal_link_speed) {
7063 case SPEED_100:
7064 return 100;
7065 case SPEED_1000:
7066 return 1000;
7067 default:
7068 return 0;
7069 }
7070}
7071
7072static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
7073 int link_speed)
7074{
7075 int rf_dec, rf_int;
7076 u32 bcnrc_val;
7077
7078 if (tx_rate != 0) {
7079 /* Calculate the rate factor values to set */
7080 rf_int = link_speed / tx_rate;
7081 rf_dec = (link_speed - (rf_int * tx_rate));
7082 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
7083
7084 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
7085 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
7086 E1000_RTTBCNRC_RF_INT_MASK);
7087 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
7088 } else {
7089 bcnrc_val = 0;
7090 }
7091
7092 wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
f00b0da7
LL		 7093 /*
7094 * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
7095 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
7096 */
7097 wr32(E1000_RTTBCNRM, 0x14);
17dc566c
LL		 7098 wr32(E1000_RTTBCNRC, bcnrc_val);
7099}
7100
7101static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
7102{
7103 int actual_link_speed, i;
7104 bool reset_rate = false;
7105
7106 /* VF TX rate limit was not set or not supported */
7107 if ((adapter->vf_rate_link_speed == 0) ||
7108 (adapter->hw.mac.type != e1000_82576))
7109 return;
7110
7111 actual_link_speed = igb_link_mbps(adapter->link_speed);
7112 if (actual_link_speed != adapter->vf_rate_link_speed) {
7113 reset_rate = true;
7114 adapter->vf_rate_link_speed = 0;
7115 dev_info(&adapter->pdev->dev,
7116 "Link speed has been changed. VF Transmit "
7117 "rate is disabled\n");
7118 }
7119
7120 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7121 if (reset_rate)
7122 adapter->vf_data[i].tx_rate = 0;
7123
7124 igb_set_vf_rate_limit(&adapter->hw, i,
7125 adapter->vf_data[i].tx_rate,
7126 actual_link_speed);
7127 }
7128}
7129
8151d294
WM		 7130static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
7131{
17dc566c
LL		 7132 struct igb_adapter *adapter = netdev_priv(netdev);
7133 struct e1000_hw *hw = &adapter->hw;
7134 int actual_link_speed;
7135
7136 if (hw->mac.type != e1000_82576)
7137 return -EOPNOTSUPP;
7138
7139 actual_link_speed = igb_link_mbps(adapter->link_speed);
7140 if ((vf >= adapter->vfs_allocated_count) ||
7141 (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
7142 (tx_rate < 0) || (tx_rate > actual_link_speed))
7143 return -EINVAL;
7144
7145 adapter->vf_rate_link_speed = actual_link_speed;
7146 adapter->vf_data[vf].tx_rate = (u16)tx_rate;
7147 igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
7148
7149 return 0;
8151d294
WM		 7150}
7151
7152static int igb_ndo_get_vf_config(struct net_device *netdev,
7153 int vf, struct ifla_vf_info *ivi)
7154{
7155 struct igb_adapter *adapter = netdev_priv(netdev);
7156 if (vf >= adapter->vfs_allocated_count)
7157 return -EINVAL;
7158 ivi->vf = vf;
7159 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
17dc566c 7160 ivi->tx_rate = adapter->vf_data[vf].tx_rate;
8151d294
WM		 7161 ivi->vlan = adapter->vf_data[vf].pf_vlan;
7162 ivi->qos = adapter->vf_data[vf].pf_qos;
7163 return 0;
7164}
7165
4ae196df
AD		 7166static void igb_vmm_control(struct igb_adapter *adapter)
7167{
7168 struct e1000_hw *hw = &adapter->hw;
10d8e907 7169 u32 reg;
4ae196df 7170
52a1dd4d
AD		 7171 switch (hw->mac.type) {
7172 case e1000_82575:
f96a8a0b
CW		 7173 case e1000_i210:
7174 case e1000_i211:
52a1dd4d
AD		 7175 default:
7176 /* replication is not supported for 82575 */
4ae196df 7177 return;
52a1dd4d
AD		 7178 case e1000_82576:
7179 /* notify HW that the MAC is adding vlan tags */
7180 reg = rd32(E1000_DTXCTL);
7181 reg |= E1000_DTXCTL_VLAN_ADDED;
7182 wr32(E1000_DTXCTL, reg);
7183 case e1000_82580:
7184 /* enable replication vlan tag stripping */
7185 reg = rd32(E1000_RPLOLR);
7186 reg |= E1000_RPLOLR_STRVLAN;
7187 wr32(E1000_RPLOLR, reg);
d2ba2ed8
AD		 7188 case e1000_i350:
7189 /* none of the above registers are supported by i350 */
52a1dd4d
AD		 7190 break;
7191 }
10d8e907 7192
d4960307
AD		 7193 if (adapter->vfs_allocated_count) {
7194 igb_vmdq_set_loopback_pf(hw, true);
7195 igb_vmdq_set_replication_pf(hw, true);
13800469
GR		 7196 igb_vmdq_set_anti_spoofing_pf(hw, true,
7197 adapter->vfs_allocated_count);
d4960307
AD		 7198 } else {
7199 igb_vmdq_set_loopback_pf(hw, false);
7200 igb_vmdq_set_replication_pf(hw, false);
7201 }
4ae196df
AD		 7202}
7203
b6e0c419
CW		 7204static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
7205{
7206 struct e1000_hw *hw = &adapter->hw;
7207 u32 dmac_thr;
7208 u16 hwm;
7209
7210 if (hw->mac.type > e1000_82580) {
7211 if (adapter->flags & IGB_FLAG_DMAC) {
7212 u32 reg;
7213
7214 /* force threshold to 0. */
7215 wr32(E1000_DMCTXTH, 0);
7216
7217 /*
e8c626e9
MV		 7218 * DMA Coalescing high water mark needs to be greater
7219 * than the Rx threshold. Set hwm to PBA - max frame
7220 * size in 16B units, capping it at PBA - 6KB.
b6e0c419 7221 */
e8c626e9
MV		 7222 hwm = 64 * pba - adapter->max_frame_size / 16;
7223 if (hwm < 64 * (pba - 6))
7224 hwm = 64 * (pba - 6);
7225 reg = rd32(E1000_FCRTC);
7226 reg &= ~E1000_FCRTC_RTH_COAL_MASK;
7227 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
7228 & E1000_FCRTC_RTH_COAL_MASK);
7229 wr32(E1000_FCRTC, reg);
7230
7231 /*
7232 * Set the DMA Coalescing Rx threshold to PBA - 2 * max
7233 * frame size, capping it at PBA - 10KB.
7234 */
7235 dmac_thr = pba - adapter->max_frame_size / 512;
7236 if (dmac_thr < pba - 10)
7237 dmac_thr = pba - 10;
b6e0c419
CW		 7238 reg = rd32(E1000_DMACR);
7239 reg &= ~E1000_DMACR_DMACTHR_MASK;
b6e0c419
CW		 7240 reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
7241 & E1000_DMACR_DMACTHR_MASK);
7242
7243 /* transition to L0x or L1 if available..*/
7244 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
7245
7246 /* watchdog timer= +-1000 usec in 32usec intervals */
7247 reg |= (1000 >> 5);
0c02dd98
MV		 7248
7249 /* Disable BMC-to-OS Watchdog Enable */
7250 reg &= ~E1000_DMACR_DC_BMC2OSW_EN;
b6e0c419
CW		 7251 wr32(E1000_DMACR, reg);
7252
7253 /*
7254 * no lower threshold to disable
7255 * coalescing(smart fifb)-UTRESH=0
7256 */
7257 wr32(E1000_DMCRTRH, 0);
b6e0c419
CW		 7258
7259 reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
7260
7261 wr32(E1000_DMCTLX, reg);
7262
7263 /*
7264 * free space in tx packet buffer to wake from
7265 * DMA coal
7266 */
7267 wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
7268 (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
7269
7270 /*
7271 * make low power state decision controlled
7272 * by DMA coal
7273 */
7274 reg = rd32(E1000_PCIEMISC);
7275 reg &= ~E1000_PCIEMISC_LX_DECISION;
7276 wr32(E1000_PCIEMISC, reg);
7277 } /* endif adapter->dmac is not disabled */
7278 } else if (hw->mac.type == e1000_82580) {
7279 u32 reg = rd32(E1000_PCIEMISC);
7280 wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION);
7281 wr32(E1000_DMACR, 0);
7282 }
7283}
7284
9d5c8243 7285/* igb_main.c */
ubuntu-zesty-kernel mirror