]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - drivers/net/ethernet/intel/igb/igb_main.c
projects / mirror_ubuntu-zesty-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
igb: split buffer_info into tx_buffer_info and rx_buffer_info
[mirror_ubuntu-zesty-kernel.git] / drivers / net / ethernet / intel / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
4297f99b 4 Copyright(c) 2007-2011 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
b2cb09b1 31#include <linux/bitops.h>
9d5c8243
AK		 32#include <linux/vmalloc.h>
33#include <linux/pagemap.h>
34#include <linux/netdevice.h>
9d5c8243 35#include <linux/ipv6.h>
5a0e3ad6 36#include <linux/slab.h>
9d5c8243
AK		 37#include <net/checksum.h>
38#include <net/ip6_checksum.h>
c6cb090b 39#include <linux/net_tstamp.h>
9d5c8243
AK		 40#include <linux/mii.h>
41#include <linux/ethtool.h>
01789349 42#include <linux/if.h>
9d5c8243
AK		 43#include <linux/if_vlan.h>
44#include <linux/pci.h>
c54106bb 45#include <linux/pci-aspm.h>
9d5c8243
AK		 46#include <linux/delay.h>
47#include <linux/interrupt.h>
48#include <linux/if_ether.h>
40a914fa 49#include <linux/aer.h>
70c71606 50#include <linux/prefetch.h>
421e02f0 51#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 52#include <linux/dca.h>
53#endif
9d5c8243
AK		 54#include "igb.h"
55
0d1fe82d
CW		 56#define MAJ 3
57#define MIN 0
58#define BUILD 6
0d1fe82d 59#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
929dd047 60__stringify(BUILD) "-k"
9d5c8243
AK		 61char igb_driver_name[] = "igb";
62char igb_driver_version[] = DRV_VERSION;
63static const char igb_driver_string[] =
64 "Intel(R) Gigabit Ethernet Network Driver";
4c4b42cb 65static const char igb_copyright[] = "Copyright (c) 2007-2011 Intel Corporation.";
9d5c8243 66
9d5c8243
AK		 67static const struct e1000_info *igb_info_tbl[] = {
68 [board_82575] = &e1000_82575_info,
69};
70
a3aa1884 71static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
d2ba2ed8
AD		 72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
55cac248
AD		 76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
6493d24f 78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
55cac248
AD		 79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
308fb39a
JG		 82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
1b5dda33
GJ		 84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
2d064c06 86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
b894fa26 92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
c8ea5ea9 93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
97 /* required last entry */
98 {0, }
99};
100
101MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
102
103void igb_reset(struct igb_adapter *);
104static int igb_setup_all_tx_resources(struct igb_adapter *);
105static int igb_setup_all_rx_resources(struct igb_adapter *);
106static void igb_free_all_tx_resources(struct igb_adapter *);
107static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 108static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 109static int igb_probe(struct pci_dev *, const struct pci_device_id *);
110static void __devexit igb_remove(struct pci_dev *pdev);
673b8b70 111static void igb_init_hw_timer(struct igb_adapter *adapter);
9d5c8243
AK		 112static int igb_sw_init(struct igb_adapter *);
113static int igb_open(struct net_device *);
114static int igb_close(struct net_device *);
115static void igb_configure_tx(struct igb_adapter *);
116static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 117static void igb_clean_all_tx_rings(struct igb_adapter *);
118static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 119static void igb_clean_tx_ring(struct igb_ring *);
120static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 121static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 122static void igb_update_phy_info(unsigned long);
123static void igb_watchdog(unsigned long);
124static void igb_watchdog_task(struct work_struct *);
cd392f5c 125static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
12dcd86b
ED		 126static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
127 struct rtnl_link_stats64 *stats);
9d5c8243
AK		 128static int igb_change_mtu(struct net_device *, int);
129static int igb_set_mac(struct net_device *, void *);
68d480c4 130static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 131static irqreturn_t igb_intr(int irq, void *);
132static irqreturn_t igb_intr_msi(int irq, void *);
133static irqreturn_t igb_msix_other(int irq, void *);
047e0030 134static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 135#ifdef CONFIG_IGB_DCA
047e0030 136static void igb_update_dca(struct igb_q_vector *);
fe4506b6 137static void igb_setup_dca(struct igb_adapter *);
421e02f0 138#endif /* CONFIG_IGB_DCA */
661086df 139static int igb_poll(struct napi_struct *, int);
13fde97a 140static bool igb_clean_tx_irq(struct igb_q_vector *);
cd392f5c 141static bool igb_clean_rx_irq(struct igb_q_vector *, int);
9d5c8243
AK		 142static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
143static void igb_tx_timeout(struct net_device *);
144static void igb_reset_task(struct work_struct *);
b2cb09b1 145static void igb_vlan_mode(struct net_device *netdev, u32 features);
9d5c8243
AK		 146static void igb_vlan_rx_add_vid(struct net_device *, u16);
147static void igb_vlan_rx_kill_vid(struct net_device *, u16);
148static void igb_restore_vlan(struct igb_adapter *);
26ad9178 149static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 150static void igb_ping_all_vfs(struct igb_adapter *);
151static void igb_msg_task(struct igb_adapter *);
4ae196df 152static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 153static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 154static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 155static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
156static int igb_ndo_set_vf_vlan(struct net_device *netdev,
157 int vf, u16 vlan, u8 qos);
158static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
159static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
160 struct ifla_vf_info *ivi);
17dc566c 161static void igb_check_vf_rate_limit(struct igb_adapter *);
9d5c8243 162
9d5c8243 163#ifdef CONFIG_PM
3fe7c4c9 164static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 165static int igb_resume(struct pci_dev *);
166#endif
167static void igb_shutdown(struct pci_dev *);
421e02f0 168#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 169static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
170static struct notifier_block dca_notifier = {
171 .notifier_call = igb_notify_dca,
172 .next = NULL,
173 .priority = 0
174};
175#endif
9d5c8243
AK		 176#ifdef CONFIG_NET_POLL_CONTROLLER
177/* for netdump / net console */
178static void igb_netpoll(struct net_device *);
179#endif
37680117 180#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 181static unsigned int max_vfs = 0;
182module_param(max_vfs, uint, 0);
183MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
184 "per physical function");
185#endif /* CONFIG_PCI_IOV */
186
9d5c8243
AK		 187static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
188 pci_channel_state_t);
189static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
190static void igb_io_resume(struct pci_dev *);
191
192static struct pci_error_handlers igb_err_handler = {
193 .error_detected = igb_io_error_detected,
194 .slot_reset = igb_io_slot_reset,
195 .resume = igb_io_resume,
196};
197
198
199static struct pci_driver igb_driver = {
200 .name = igb_driver_name,
201 .id_table = igb_pci_tbl,
202 .probe = igb_probe,
203 .remove = __devexit_p(igb_remove),
204#ifdef CONFIG_PM
25985edc 205 /* Power Management Hooks */
9d5c8243
AK		 206 .suspend = igb_suspend,
207 .resume = igb_resume,
208#endif
209 .shutdown = igb_shutdown,
210 .err_handler = &igb_err_handler
211};
212
213MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
214MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
215MODULE_LICENSE("GPL");
216MODULE_VERSION(DRV_VERSION);
217
c97ec42a
TI		 218struct igb_reg_info {
219 u32 ofs;
220 char *name;
221};
222
223static const struct igb_reg_info igb_reg_info_tbl[] = {
224
225 /* General Registers */
226 {E1000_CTRL, "CTRL"},
227 {E1000_STATUS, "STATUS"},
228 {E1000_CTRL_EXT, "CTRL_EXT"},
229
230 /* Interrupt Registers */
231 {E1000_ICR, "ICR"},
232
233 /* RX Registers */
234 {E1000_RCTL, "RCTL"},
235 {E1000_RDLEN(0), "RDLEN"},
236 {E1000_RDH(0), "RDH"},
237 {E1000_RDT(0), "RDT"},
238 {E1000_RXDCTL(0), "RXDCTL"},
239 {E1000_RDBAL(0), "RDBAL"},
240 {E1000_RDBAH(0), "RDBAH"},
241
242 /* TX Registers */
243 {E1000_TCTL, "TCTL"},
244 {E1000_TDBAL(0), "TDBAL"},
245 {E1000_TDBAH(0), "TDBAH"},
246 {E1000_TDLEN(0), "TDLEN"},
247 {E1000_TDH(0), "TDH"},
248 {E1000_TDT(0), "TDT"},
249 {E1000_TXDCTL(0), "TXDCTL"},
250 {E1000_TDFH, "TDFH"},
251 {E1000_TDFT, "TDFT"},
252 {E1000_TDFHS, "TDFHS"},
253 {E1000_TDFPC, "TDFPC"},
254
255 /* List Terminator */
256 {}
257};
258
259/*
260 * igb_regdump - register printout routine
261 */
262static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
263{
264 int n = 0;
265 char rname[16];
266 u32 regs[8];
267
268 switch (reginfo->ofs) {
269 case E1000_RDLEN(0):
270 for (n = 0; n < 4; n++)
271 regs[n] = rd32(E1000_RDLEN(n));
272 break;
273 case E1000_RDH(0):
274 for (n = 0; n < 4; n++)
275 regs[n] = rd32(E1000_RDH(n));
276 break;
277 case E1000_RDT(0):
278 for (n = 0; n < 4; n++)
279 regs[n] = rd32(E1000_RDT(n));
280 break;
281 case E1000_RXDCTL(0):
282 for (n = 0; n < 4; n++)
283 regs[n] = rd32(E1000_RXDCTL(n));
284 break;
285 case E1000_RDBAL(0):
286 for (n = 0; n < 4; n++)
287 regs[n] = rd32(E1000_RDBAL(n));
288 break;
289 case E1000_RDBAH(0):
290 for (n = 0; n < 4; n++)
291 regs[n] = rd32(E1000_RDBAH(n));
292 break;
293 case E1000_TDBAL(0):
294 for (n = 0; n < 4; n++)
295 regs[n] = rd32(E1000_RDBAL(n));
296 break;
297 case E1000_TDBAH(0):
298 for (n = 0; n < 4; n++)
299 regs[n] = rd32(E1000_TDBAH(n));
300 break;
301 case E1000_TDLEN(0):
302 for (n = 0; n < 4; n++)
303 regs[n] = rd32(E1000_TDLEN(n));
304 break;
305 case E1000_TDH(0):
306 for (n = 0; n < 4; n++)
307 regs[n] = rd32(E1000_TDH(n));
308 break;
309 case E1000_TDT(0):
310 for (n = 0; n < 4; n++)
311 regs[n] = rd32(E1000_TDT(n));
312 break;
313 case E1000_TXDCTL(0):
314 for (n = 0; n < 4; n++)
315 regs[n] = rd32(E1000_TXDCTL(n));
316 break;
317 default:
318 printk(KERN_INFO "%-15s %08x\n",
319 reginfo->name, rd32(reginfo->ofs));
320 return;
321 }
322
323 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
324 printk(KERN_INFO "%-15s ", rname);
325 for (n = 0; n < 4; n++)
326 printk(KERN_CONT "%08x ", regs[n]);
327 printk(KERN_CONT "\n");
328}
329
330/*
331 * igb_dump - Print registers, tx-rings and rx-rings
332 */
333static void igb_dump(struct igb_adapter *adapter)
334{
335 struct net_device *netdev = adapter->netdev;
336 struct e1000_hw *hw = &adapter->hw;
337 struct igb_reg_info *reginfo;
338 int n = 0;
339 struct igb_ring *tx_ring;
340 union e1000_adv_tx_desc *tx_desc;
341 struct my_u0 { u64 a; u64 b; } *u0;
c97ec42a
TI		 342 struct igb_ring *rx_ring;
343 union e1000_adv_rx_desc *rx_desc;
344 u32 staterr;
345 int i = 0;
346
347 if (!netif_msg_hw(adapter))
348 return;
349
350 /* Print netdevice Info */
351 if (netdev) {
352 dev_info(&adapter->pdev->dev, "Net device Info\n");
353 printk(KERN_INFO "Device Name state "
354 "trans_start last_rx\n");
355 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
356 netdev->name,
357 netdev->state,
358 netdev->trans_start,
359 netdev->last_rx);
360 }
361
362 /* Print Registers */
363 dev_info(&adapter->pdev->dev, "Register Dump\n");
364 printk(KERN_INFO " Register Name Value\n");
365 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
366 reginfo->name; reginfo++) {
367 igb_regdump(hw, reginfo);
368 }
369
370 /* Print TX Ring Summary */
371 if (!netdev || !netif_running(netdev))
372 goto exit;
373
374 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
375 printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma ]"
376 " leng ntw timestamp\n");
377 for (n = 0; n < adapter->num_tx_queues; n++) {
06034649 378 struct igb_tx_buffer *buffer_info;
c97ec42a 379 tx_ring = adapter->tx_ring[n];
06034649 380 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
c97ec42a
TI		 381 printk(KERN_INFO " %5d %5X %5X %016llX %04X %3X %016llX\n",
382 n, tx_ring->next_to_use, tx_ring->next_to_clean,
383 (u64)buffer_info->dma,
384 buffer_info->length,
385 buffer_info->next_to_watch,
386 (u64)buffer_info->time_stamp);
387 }
388
389 /* Print TX Rings */
390 if (!netif_msg_tx_done(adapter))
391 goto rx_ring_summary;
392
393 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
394
395 /* Transmit Descriptor Formats
396 *
397 * Advanced Transmit Descriptor
398 * +--------------------------------------------------------------+
399 * 0 | Buffer Address [63:0] |
400 * +--------------------------------------------------------------+
401 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
402 * +--------------------------------------------------------------+
403 * 63 46 45 40 39 38 36 35 32 31 24 15 0
404 */
405
406 for (n = 0; n < adapter->num_tx_queues; n++) {
407 tx_ring = adapter->tx_ring[n];
408 printk(KERN_INFO "------------------------------------\n");
409 printk(KERN_INFO "TX QUEUE INDEX = %d\n", tx_ring->queue_index);
410 printk(KERN_INFO "------------------------------------\n");
411 printk(KERN_INFO "T [desc] [address 63:0 ] "
412 "[PlPOCIStDDM Ln] [bi->dma ] "
413 "leng ntw timestamp bi->skb\n");
414
415 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
06034649 416 struct igb_tx_buffer *buffer_info;
60136906 417 tx_desc = IGB_TX_DESC(tx_ring, i);
06034649 418 buffer_info = &tx_ring->tx_buffer_info[i];
c97ec42a
TI		 419 u0 = (struct my_u0 *)tx_desc;
420 printk(KERN_INFO "T [0x%03X] %016llX %016llX %016llX"
421 " %04X %3X %016llX %p", i,
422 le64_to_cpu(u0->a),
423 le64_to_cpu(u0->b),
424 (u64)buffer_info->dma,
425 buffer_info->length,
426 buffer_info->next_to_watch,
427 (u64)buffer_info->time_stamp,
428 buffer_info->skb);
429 if (i == tx_ring->next_to_use &&
430 i == tx_ring->next_to_clean)
431 printk(KERN_CONT " NTC/U\n");
432 else if (i == tx_ring->next_to_use)
433 printk(KERN_CONT " NTU\n");
434 else if (i == tx_ring->next_to_clean)
435 printk(KERN_CONT " NTC\n");
436 else
437 printk(KERN_CONT "\n");
438
439 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
440 print_hex_dump(KERN_INFO, "",
441 DUMP_PREFIX_ADDRESS,
442 16, 1, phys_to_virt(buffer_info->dma),
443 buffer_info->length, true);
444 }
445 }
446
447 /* Print RX Rings Summary */
448rx_ring_summary:
449 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
450 printk(KERN_INFO "Queue [NTU] [NTC]\n");
451 for (n = 0; n < adapter->num_rx_queues; n++) {
452 rx_ring = adapter->rx_ring[n];
453 printk(KERN_INFO " %5d %5X %5X\n", n,
454 rx_ring->next_to_use, rx_ring->next_to_clean);
455 }
456
457 /* Print RX Rings */
458 if (!netif_msg_rx_status(adapter))
459 goto exit;
460
461 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
462
463 /* Advanced Receive Descriptor (Read) Format
464 * 63 1 0
465 * +-----------------------------------------------------+
466 * 0 | Packet Buffer Address [63:1] |A0/NSE|
467 * +----------------------------------------------+------+
468 * 8 | Header Buffer Address [63:1] | DD |
469 * +-----------------------------------------------------+
470 *
471 *
472 * Advanced Receive Descriptor (Write-Back) Format
473 *
474 * 63 48 47 32 31 30 21 20 17 16 4 3 0
475 * +------------------------------------------------------+
476 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
477 * | Checksum Ident | | | | Type | Type |
478 * +------------------------------------------------------+
479 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
480 * +------------------------------------------------------+
481 * 63 48 47 32 31 20 19 0
482 */
483
484 for (n = 0; n < adapter->num_rx_queues; n++) {
485 rx_ring = adapter->rx_ring[n];
486 printk(KERN_INFO "------------------------------------\n");
487 printk(KERN_INFO "RX QUEUE INDEX = %d\n", rx_ring->queue_index);
488 printk(KERN_INFO "------------------------------------\n");
489 printk(KERN_INFO "R [desc] [ PktBuf A0] "
490 "[ HeadBuf DD] [bi->dma ] [bi->skb] "
491 "<-- Adv Rx Read format\n");
492 printk(KERN_INFO "RWB[desc] [PcsmIpSHl PtRs] "
493 "[vl er S cks ln] ---------------- [bi->skb] "
494 "<-- Adv Rx Write-Back format\n");
495
496 for (i = 0; i < rx_ring->count; i++) {
06034649
AD		 497 struct igb_rx_buffer *buffer_info;
498 buffer_info = &rx_ring->rx_buffer_info[i];
60136906 499 rx_desc = IGB_RX_DESC(rx_ring, i);
c97ec42a
TI		 500 u0 = (struct my_u0 *)rx_desc;
501 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
502 if (staterr & E1000_RXD_STAT_DD) {
503 /* Descriptor Done */
504 printk(KERN_INFO "RWB[0x%03X] %016llX "
505 "%016llX ---------------- %p", i,
506 le64_to_cpu(u0->a),
507 le64_to_cpu(u0->b),
508 buffer_info->skb);
509 } else {
510 printk(KERN_INFO "R [0x%03X] %016llX "
511 "%016llX %016llX %p", i,
512 le64_to_cpu(u0->a),
513 le64_to_cpu(u0->b),
514 (u64)buffer_info->dma,
515 buffer_info->skb);
516
517 if (netif_msg_pktdata(adapter)) {
518 print_hex_dump(KERN_INFO, "",
519 DUMP_PREFIX_ADDRESS,
520 16, 1,
521 phys_to_virt(buffer_info->dma),
44390ca6
AD		 522 IGB_RX_HDR_LEN, true);
523 print_hex_dump(KERN_INFO, "",
524 DUMP_PREFIX_ADDRESS,
525 16, 1,
526 phys_to_virt(
527 buffer_info->page_dma +
528 buffer_info->page_offset),
529 PAGE_SIZE/2, true);
c97ec42a
TI		 530 }
531 }
532
533 if (i == rx_ring->next_to_use)
534 printk(KERN_CONT " NTU\n");
535 else if (i == rx_ring->next_to_clean)
536 printk(KERN_CONT " NTC\n");
537 else
538 printk(KERN_CONT "\n");
539
540 }
541 }
542
543exit:
544 return;
545}
546
547
38c845c7
PO		 548/**
549 * igb_read_clock - read raw cycle counter (to be used by time counter)
550 */
551static cycle_t igb_read_clock(const struct cyclecounter *tc)
552{
553 struct igb_adapter *adapter =
554 container_of(tc, struct igb_adapter, cycles);
555 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 556 u64 stamp = 0;
557 int shift = 0;
38c845c7 558
55cac248
AD		 559 /*
560 * The timestamp latches on lowest register read. For the 82580
561 * the lowest register is SYSTIMR instead of SYSTIML. However we never
562 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
563 */
564 if (hw->mac.type == e1000_82580) {
565 stamp = rd32(E1000_SYSTIMR) >> 8;
566 shift = IGB_82580_TSYNC_SHIFT;
567 }
568
c5b9bd5e
AD		 569 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
570 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 571 return stamp;
572}
573
9d5c8243 574/**
c041076a 575 * igb_get_hw_dev - return device
9d5c8243
AK		 576 * used by hardware layer to print debugging information
577 **/
c041076a 578struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
9d5c8243
AK		 579{
580 struct igb_adapter *adapter = hw->back;
c041076a 581 return adapter->netdev;
9d5c8243 582}
38c845c7 583
9d5c8243
AK		 584/**
585 * igb_init_module - Driver Registration Routine
586 *
587 * igb_init_module is the first routine called when the driver is
588 * loaded. All it does is register with the PCI subsystem.
589 **/
590static int __init igb_init_module(void)
591{
592 int ret;
593 printk(KERN_INFO "%s - version %s\n",
594 igb_driver_string, igb_driver_version);
595
596 printk(KERN_INFO "%s\n", igb_copyright);
597
421e02f0 598#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 599 dca_register_notify(&dca_notifier);
600#endif
bbd98fe4 601 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 602 return ret;
603}
604
605module_init(igb_init_module);
606
607/**
608 * igb_exit_module - Driver Exit Cleanup Routine
609 *
610 * igb_exit_module is called just before the driver is removed
611 * from memory.
612 **/
613static void __exit igb_exit_module(void)
614{
421e02f0 615#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 616 dca_unregister_notify(&dca_notifier);
617#endif
9d5c8243
AK		 618 pci_unregister_driver(&igb_driver);
619}
620
621module_exit(igb_exit_module);
622
26bc19ec
AD		 623#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
624/**
625 * igb_cache_ring_register - Descriptor ring to register mapping
626 * @adapter: board private structure to initialize
627 *
628 * Once we know the feature-set enabled for the device, we'll cache
629 * the register offset the descriptor ring is assigned to.
630 **/
631static void igb_cache_ring_register(struct igb_adapter *adapter)
632{
ee1b9f06 633 int i = 0, j = 0;
047e0030 634 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 635
636 switch (adapter->hw.mac.type) {
637 case e1000_82576:
638 /* The queues are allocated for virtualization such that VF 0
639 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
640 * In order to avoid collision we start at the first free queue
641 * and continue consuming queues in the same sequence
642 */
ee1b9f06 643 if (adapter->vfs_allocated_count) {
a99955fc 644 for (; i < adapter->rss_queues; i++)
3025a446
AD		 645 adapter->rx_ring[i]->reg_idx = rbase_offset +
646 Q_IDX_82576(i);
ee1b9f06 647 }
26bc19ec 648 case e1000_82575:
55cac248 649 case e1000_82580:
d2ba2ed8 650 case e1000_i350:
26bc19ec 651 default:
ee1b9f06 652 for (; i < adapter->num_rx_queues; i++)
3025a446 653 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 654 for (; j < adapter->num_tx_queues; j++)
3025a446 655 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 656 break;
657 }
658}
659
047e0030
AD		 660static void igb_free_queues(struct igb_adapter *adapter)
661{
3025a446 662 int i;
047e0030 663
3025a446
AD		 664 for (i = 0; i < adapter->num_tx_queues; i++) {
665 kfree(adapter->tx_ring[i]);
666 adapter->tx_ring[i] = NULL;
667 }
668 for (i = 0; i < adapter->num_rx_queues; i++) {
669 kfree(adapter->rx_ring[i]);
670 adapter->rx_ring[i] = NULL;
671 }
047e0030
AD		 672 adapter->num_rx_queues = 0;
673 adapter->num_tx_queues = 0;
674}
675
9d5c8243
AK		 676/**
677 * igb_alloc_queues - Allocate memory for all rings
678 * @adapter: board private structure to initialize
679 *
680 * We allocate one ring per queue at run-time since we don't know the
681 * number of queues at compile-time.
682 **/
683static int igb_alloc_queues(struct igb_adapter *adapter)
684{
3025a446 685 struct igb_ring *ring;
9d5c8243
AK		 686 int i;
687
661086df 688 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 689 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
690 if (!ring)
691 goto err;
68fd9910 692 ring->count = adapter->tx_ring_count;
661086df 693 ring->queue_index = i;
59d71989 694 ring->dev = &adapter->pdev->dev;
e694e964 695 ring->netdev = adapter->netdev;
85ad76b2
AD		 696 /* For 82575, context index must be unique per ring. */
697 if (adapter->hw.mac.type == e1000_82575)
698 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
3025a446 699 adapter->tx_ring[i] = ring;
661086df 700 }
85ad76b2 701
9d5c8243 702 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446
AD		 703 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
704 if (!ring)
705 goto err;
68fd9910 706 ring->count = adapter->rx_ring_count;
844290e5 707 ring->queue_index = i;
59d71989 708 ring->dev = &adapter->pdev->dev;
e694e964 709 ring->netdev = adapter->netdev;
85ad76b2
AD		 710 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
711 /* set flag indicating ring supports SCTP checksum offload */
712 if (adapter->hw.mac.type >= e1000_82576)
713 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
3025a446 714 adapter->rx_ring[i] = ring;
9d5c8243 715 }
26bc19ec
AD		 716
717 igb_cache_ring_register(adapter);
9d5c8243 718
047e0030 719 return 0;
a88f10ec 720
047e0030
AD		 721err:
722 igb_free_queues(adapter);
d1a8c9e1 723
047e0030 724 return -ENOMEM;
a88f10ec
AD		 725}
726
9d5c8243 727#define IGB_N0_QUEUE -1
047e0030 728static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 729{
730 u32 msixbm = 0;
047e0030 731 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 732 struct e1000_hw *hw = &adapter->hw;
2d064c06 733 u32 ivar, index;
047e0030
AD		 734 int rx_queue = IGB_N0_QUEUE;
735 int tx_queue = IGB_N0_QUEUE;
736
737 if (q_vector->rx_ring)
738 rx_queue = q_vector->rx_ring->reg_idx;
739 if (q_vector->tx_ring)
740 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 741
742 switch (hw->mac.type) {
743 case e1000_82575:
9d5c8243
AK		 744 /* The 82575 assigns vectors using a bitmask, which matches the
745 bitmask for the EICR/EIMS/EIMC registers. To assign one
746 or more queues to a vector, we write the appropriate bits
747 into the MSIXBM register for that vector. */
047e0030 748 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 749 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 750 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 751 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 752 if (!adapter->msix_entries && msix_vector == 0)
753 msixbm |= E1000_EIMS_OTHER;
9d5c8243 754 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 755 q_vector->eims_value = msixbm;
2d064c06
AD		 756 break;
757 case e1000_82576:
26bc19ec 758 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 759 Each queue has a single entry in the table to which we write
760 a vector number along with a "valid" bit. Sadly, the layout
761 of the table is somewhat counterintuitive. */
762 if (rx_queue > IGB_N0_QUEUE) {
047e0030 763 index = (rx_queue & 0x7);
2d064c06 764 ivar = array_rd32(E1000_IVAR0, index);
047e0030 765 if (rx_queue < 8) {
26bc19ec
AD		 766 /* vector goes into low byte of register */
767 ivar = ivar & 0xFFFFFF00;
768 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 769 } else {
770 /* vector goes into third byte of register */
771 ivar = ivar & 0xFF00FFFF;
772 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 773 }
2d064c06
AD		 774 array_wr32(E1000_IVAR0, index, ivar);
775 }
776 if (tx_queue > IGB_N0_QUEUE) {
047e0030 777 index = (tx_queue & 0x7);
2d064c06 778 ivar = array_rd32(E1000_IVAR0, index);
047e0030 779 if (tx_queue < 8) {
26bc19ec
AD		 780 /* vector goes into second byte of register */
781 ivar = ivar & 0xFFFF00FF;
782 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 783 } else {
784 /* vector goes into high byte of register */
785 ivar = ivar & 0x00FFFFFF;
786 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 787 }
2d064c06
AD		 788 array_wr32(E1000_IVAR0, index, ivar);
789 }
047e0030 790 q_vector->eims_value = 1 << msix_vector;
2d064c06 791 break;
55cac248 792 case e1000_82580:
d2ba2ed8 793 case e1000_i350:
55cac248
AD		 794 /* 82580 uses the same table-based approach as 82576 but has fewer
795 entries as a result we carry over for queues greater than 4. */
796 if (rx_queue > IGB_N0_QUEUE) {
797 index = (rx_queue >> 1);
798 ivar = array_rd32(E1000_IVAR0, index);
799 if (rx_queue & 0x1) {
800 /* vector goes into third byte of register */
801 ivar = ivar & 0xFF00FFFF;
802 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
803 } else {
804 /* vector goes into low byte of register */
805 ivar = ivar & 0xFFFFFF00;
806 ivar |= msix_vector | E1000_IVAR_VALID;
807 }
808 array_wr32(E1000_IVAR0, index, ivar);
809 }
810 if (tx_queue > IGB_N0_QUEUE) {
811 index = (tx_queue >> 1);
812 ivar = array_rd32(E1000_IVAR0, index);
813 if (tx_queue & 0x1) {
814 /* vector goes into high byte of register */
815 ivar = ivar & 0x00FFFFFF;
816 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
817 } else {
818 /* vector goes into second byte of register */
819 ivar = ivar & 0xFFFF00FF;
820 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
821 }
822 array_wr32(E1000_IVAR0, index, ivar);
823 }
824 q_vector->eims_value = 1 << msix_vector;
825 break;
2d064c06
AD		 826 default:
827 BUG();
828 break;
829 }
26b39276
AD		 830
831 /* add q_vector eims value to global eims_enable_mask */
832 adapter->eims_enable_mask |= q_vector->eims_value;
833
834 /* configure q_vector to set itr on first interrupt */
835 q_vector->set_itr = 1;
9d5c8243
AK		 836}
837
838/**
839 * igb_configure_msix - Configure MSI-X hardware
840 *
841 * igb_configure_msix sets up the hardware to properly
842 * generate MSI-X interrupts.
843 **/
844static void igb_configure_msix(struct igb_adapter *adapter)
845{
846 u32 tmp;
847 int i, vector = 0;
848 struct e1000_hw *hw = &adapter->hw;
849
850 adapter->eims_enable_mask = 0;
9d5c8243
AK		 851
852 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 853 switch (hw->mac.type) {
854 case e1000_82575:
9d5c8243
AK		 855 tmp = rd32(E1000_CTRL_EXT);
856 /* enable MSI-X PBA support*/
857 tmp |= E1000_CTRL_EXT_PBA_CLR;
858
859 /* Auto-Mask interrupts upon ICR read. */
860 tmp |= E1000_CTRL_EXT_EIAME;
861 tmp |= E1000_CTRL_EXT_IRCA;
862
863 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 864
865 /* enable msix_other interrupt */
866 array_wr32(E1000_MSIXBM(0), vector++,
867 E1000_EIMS_OTHER);
844290e5 868 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 869
2d064c06
AD		 870 break;
871
872 case e1000_82576:
55cac248 873 case e1000_82580:
d2ba2ed8 874 case e1000_i350:
047e0030
AD		 875 /* Turn on MSI-X capability first, or our settings
876 * won't stick. And it will take days to debug. */
877 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
878 E1000_GPIE_PBA | E1000_GPIE_EIAME |
879 E1000_GPIE_NSICR);
880
881 /* enable msix_other interrupt */
882 adapter->eims_other = 1 << vector;
2d064c06 883 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 884
047e0030 885 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 886 break;
887 default:
888 /* do nothing, since nothing else supports MSI-X */
889 break;
890 } /* switch (hw->mac.type) */
047e0030
AD		 891
892 adapter->eims_enable_mask |= adapter->eims_other;
893
26b39276
AD		 894 for (i = 0; i < adapter->num_q_vectors; i++)
895 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 896
9d5c8243
AK		 897 wrfl();
898}
899
900/**
901 * igb_request_msix - Initialize MSI-X interrupts
902 *
903 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
904 * kernel.
905 **/
906static int igb_request_msix(struct igb_adapter *adapter)
907{
908 struct net_device *netdev = adapter->netdev;
047e0030 909 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 910 int i, err = 0, vector = 0;
911
047e0030 912 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 913 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 914 if (err)
915 goto out;
916 vector++;
917
918 for (i = 0; i < adapter->num_q_vectors; i++) {
919 struct igb_q_vector *q_vector = adapter->q_vector[i];
920
921 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
922
923 if (q_vector->rx_ring && q_vector->tx_ring)
924 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
925 q_vector->rx_ring->queue_index);
926 else if (q_vector->tx_ring)
927 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
928 q_vector->tx_ring->queue_index);
929 else if (q_vector->rx_ring)
930 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
931 q_vector->rx_ring->queue_index);
9d5c8243 932 else
047e0030
AD		 933 sprintf(q_vector->name, "%s-unused", netdev->name);
934
9d5c8243 935 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 936 igb_msix_ring, 0, q_vector->name,
047e0030 937 q_vector);
9d5c8243
AK		 938 if (err)
939 goto out;
9d5c8243
AK		 940 vector++;
941 }
942
9d5c8243
AK		 943 igb_configure_msix(adapter);
944 return 0;
945out:
946 return err;
947}
948
949static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
950{
951 if (adapter->msix_entries) {
952 pci_disable_msix(adapter->pdev);
953 kfree(adapter->msix_entries);
954 adapter->msix_entries = NULL;
047e0030 955 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 956 pci_disable_msi(adapter->pdev);
047e0030 957 }
9d5c8243
AK		 958}
959
047e0030
AD		 960/**
961 * igb_free_q_vectors - Free memory allocated for interrupt vectors
962 * @adapter: board private structure to initialize
963 *
964 * This function frees the memory allocated to the q_vectors. In addition if
965 * NAPI is enabled it will delete any references to the NAPI struct prior
966 * to freeing the q_vector.
967 **/
968static void igb_free_q_vectors(struct igb_adapter *adapter)
969{
970 int v_idx;
971
972 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
973 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
974 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 975 if (!q_vector)
976 continue;
047e0030
AD		 977 netif_napi_del(&q_vector->napi);
978 kfree(q_vector);
979 }
980 adapter->num_q_vectors = 0;
981}
982
983/**
984 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
985 *
986 * This function resets the device so that it has 0 rx queues, tx queues, and
987 * MSI-X interrupts allocated.
988 */
989static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
990{
991 igb_free_queues(adapter);
992 igb_free_q_vectors(adapter);
993 igb_reset_interrupt_capability(adapter);
994}
9d5c8243
AK		 995
996/**
997 * igb_set_interrupt_capability - set MSI or MSI-X if supported
998 *
999 * Attempt to configure interrupts using the best available
1000 * capabilities of the hardware and kernel.
1001 **/
21adef3e 1002static int igb_set_interrupt_capability(struct igb_adapter *adapter)
9d5c8243
AK		 1003{
1004 int err;
1005 int numvecs, i;
1006
83b7180d 1007 /* Number of supported queues. */
a99955fc 1008 adapter->num_rx_queues = adapter->rss_queues;
5fa8517f
GR		 1009 if (adapter->vfs_allocated_count)
1010 adapter->num_tx_queues = 1;
1011 else
1012 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 1013
047e0030
AD		 1014 /* start with one vector for every rx queue */
1015 numvecs = adapter->num_rx_queues;
1016
3ad2f3fb 1017 /* if tx handler is separate add 1 for every tx queue */
a99955fc
AD		 1018 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1019 numvecs += adapter->num_tx_queues;
047e0030
AD		 1020
1021 /* store the number of vectors reserved for queues */
1022 adapter->num_q_vectors = numvecs;
1023
1024 /* add 1 vector for link status interrupts */
1025 numvecs++;
9d5c8243
AK		 1026 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1027 GFP_KERNEL);
1028 if (!adapter->msix_entries)
1029 goto msi_only;
1030
1031 for (i = 0; i < numvecs; i++)
1032 adapter->msix_entries[i].entry = i;
1033
1034 err = pci_enable_msix(adapter->pdev,
1035 adapter->msix_entries,
1036 numvecs);
1037 if (err == 0)
34a20e89 1038 goto out;
9d5c8243
AK		 1039
1040 igb_reset_interrupt_capability(adapter);
1041
1042 /* If we can't do MSI-X, try MSI */
1043msi_only:
2a3abf6d
AD		 1044#ifdef CONFIG_PCI_IOV
1045 /* disable SR-IOV for non MSI-X configurations */
1046 if (adapter->vf_data) {
1047 struct e1000_hw *hw = &adapter->hw;
1048 /* disable iov and allow time for transactions to clear */
1049 pci_disable_sriov(adapter->pdev);
1050 msleep(500);
1051
1052 kfree(adapter->vf_data);
1053 adapter->vf_data = NULL;
1054 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 1055 wrfl();
2a3abf6d
AD		 1056 msleep(100);
1057 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1058 }
1059#endif
4fc82adf 1060 adapter->vfs_allocated_count = 0;
a99955fc 1061 adapter->rss_queues = 1;
4fc82adf 1062 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 1063 adapter->num_rx_queues = 1;
661086df 1064 adapter->num_tx_queues = 1;
047e0030 1065 adapter->num_q_vectors = 1;
9d5c8243 1066 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1067 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 1068out:
21adef3e
BH		 1069 /* Notify the stack of the (possibly) reduced queue counts. */
1070 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
1071 return netif_set_real_num_rx_queues(adapter->netdev,
1072 adapter->num_rx_queues);
9d5c8243
AK		 1073}
1074
047e0030
AD		 1075/**
1076 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1077 * @adapter: board private structure to initialize
1078 *
1079 * We allocate one q_vector per queue interrupt. If allocation fails we
1080 * return -ENOMEM.
1081 **/
1082static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1083{
1084 struct igb_q_vector *q_vector;
1085 struct e1000_hw *hw = &adapter->hw;
1086 int v_idx;
1087
1088 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1089 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
1090 if (!q_vector)
1091 goto err_out;
1092 q_vector->adapter = adapter;
047e0030
AD		 1093 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1094 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1095 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1096 adapter->q_vector[v_idx] = q_vector;
1097 }
1098 return 0;
1099
1100err_out:
fe0592b4 1101 igb_free_q_vectors(adapter);
047e0030
AD		 1102 return -ENOMEM;
1103}
1104
1105static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1106 int ring_idx, int v_idx)
1107{
3025a446 1108 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1109
3025a446 1110 q_vector->rx_ring = adapter->rx_ring[ring_idx];
047e0030 1111 q_vector->rx_ring->q_vector = q_vector;
4fc82adf
AD		 1112 q_vector->itr_val = adapter->rx_itr_setting;
1113 if (q_vector->itr_val && q_vector->itr_val <= 3)
1114 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1115}
1116
1117static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1118 int ring_idx, int v_idx)
1119{
3025a446 1120 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1121
3025a446 1122 q_vector->tx_ring = adapter->tx_ring[ring_idx];
047e0030 1123 q_vector->tx_ring->q_vector = q_vector;
4fc82adf 1124 q_vector->itr_val = adapter->tx_itr_setting;
13fde97a 1125 q_vector->tx_work_limit = adapter->tx_work_limit;
4fc82adf
AD		 1126 if (q_vector->itr_val && q_vector->itr_val <= 3)
1127 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1128}
1129
1130/**
1131 * igb_map_ring_to_vector - maps allocated queues to vectors
1132 *
1133 * This function maps the recently allocated queues to vectors.
1134 **/
1135static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1136{
1137 int i;
1138 int v_idx = 0;
1139
1140 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1141 (adapter->num_q_vectors < adapter->num_tx_queues))
1142 return -ENOMEM;
1143
1144 if (adapter->num_q_vectors >=
1145 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1146 for (i = 0; i < adapter->num_rx_queues; i++)
1147 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1148 for (i = 0; i < adapter->num_tx_queues; i++)
1149 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1150 } else {
1151 for (i = 0; i < adapter->num_rx_queues; i++) {
1152 if (i < adapter->num_tx_queues)
1153 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1154 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1155 }
1156 for (; i < adapter->num_tx_queues; i++)
1157 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1158 }
1159 return 0;
1160}
1161
1162/**
1163 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1164 *
1165 * This function initializes the interrupts and allocates all of the queues.
1166 **/
1167static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1168{
1169 struct pci_dev *pdev = adapter->pdev;
1170 int err;
1171
21adef3e
BH		 1172 err = igb_set_interrupt_capability(adapter);
1173 if (err)
1174 return err;
047e0030
AD		 1175
1176 err = igb_alloc_q_vectors(adapter);
1177 if (err) {
1178 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1179 goto err_alloc_q_vectors;
1180 }
1181
1182 err = igb_alloc_queues(adapter);
1183 if (err) {
1184 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1185 goto err_alloc_queues;
1186 }
1187
1188 err = igb_map_ring_to_vector(adapter);
1189 if (err) {
1190 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1191 goto err_map_queues;
1192 }
1193
1194
1195 return 0;
1196err_map_queues:
1197 igb_free_queues(adapter);
1198err_alloc_queues:
1199 igb_free_q_vectors(adapter);
1200err_alloc_q_vectors:
1201 igb_reset_interrupt_capability(adapter);
1202 return err;
1203}
1204
9d5c8243
AK		 1205/**
1206 * igb_request_irq - initialize interrupts
1207 *
1208 * Attempts to configure interrupts using the best available
1209 * capabilities of the hardware and kernel.
1210 **/
1211static int igb_request_irq(struct igb_adapter *adapter)
1212{
1213 struct net_device *netdev = adapter->netdev;
047e0030 1214 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1215 int err = 0;
1216
1217 if (adapter->msix_entries) {
1218 err = igb_request_msix(adapter);
844290e5 1219 if (!err)
9d5c8243 1220 goto request_done;
9d5c8243 1221 /* fall back to MSI */
047e0030 1222 igb_clear_interrupt_scheme(adapter);
9d5c8243 1223 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1224 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1225 igb_free_all_tx_resources(adapter);
1226 igb_free_all_rx_resources(adapter);
047e0030 1227 adapter->num_tx_queues = 1;
9d5c8243 1228 adapter->num_rx_queues = 1;
047e0030
AD		 1229 adapter->num_q_vectors = 1;
1230 err = igb_alloc_q_vectors(adapter);
1231 if (err) {
1232 dev_err(&pdev->dev,
1233 "Unable to allocate memory for vectors\n");
1234 goto request_done;
1235 }
1236 err = igb_alloc_queues(adapter);
1237 if (err) {
1238 dev_err(&pdev->dev,
1239 "Unable to allocate memory for queues\n");
1240 igb_free_q_vectors(adapter);
1241 goto request_done;
1242 }
1243 igb_setup_all_tx_resources(adapter);
1244 igb_setup_all_rx_resources(adapter);
844290e5 1245 } else {
feeb2721 1246 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243 1247 }
844290e5 1248
7dfc16fa 1249 if (adapter->flags & IGB_FLAG_HAS_MSI) {
a0607fd3 1250 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
047e0030 1251 netdev->name, adapter);
9d5c8243
AK		 1252 if (!err)
1253 goto request_done;
047e0030 1254
9d5c8243
AK		 1255 /* fall back to legacy interrupts */
1256 igb_reset_interrupt_capability(adapter);
7dfc16fa 1257 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1258 }
1259
a0607fd3 1260 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
047e0030 1261 netdev->name, adapter);
9d5c8243 1262
6cb5e577 1263 if (err)
9d5c8243
AK		 1264 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
1265 err);
9d5c8243
AK		 1266
1267request_done:
1268 return err;
1269}
1270
1271static void igb_free_irq(struct igb_adapter *adapter)
1272{
9d5c8243
AK		 1273 if (adapter->msix_entries) {
1274 int vector = 0, i;
1275
047e0030 1276 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 1277
047e0030
AD		 1278 for (i = 0; i < adapter->num_q_vectors; i++) {
1279 struct igb_q_vector *q_vector = adapter->q_vector[i];
1280 free_irq(adapter->msix_entries[vector++].vector,
1281 q_vector);
1282 }
1283 } else {
1284 free_irq(adapter->pdev->irq, adapter);
9d5c8243 1285 }
9d5c8243
AK		 1286}
1287
1288/**
1289 * igb_irq_disable - Mask off interrupt generation on the NIC
1290 * @adapter: board private structure
1291 **/
1292static void igb_irq_disable(struct igb_adapter *adapter)
1293{
1294 struct e1000_hw *hw = &adapter->hw;
1295
25568a53
AD		 1296 /*
1297 * we need to be careful when disabling interrupts. The VFs are also
1298 * mapped into these registers and so clearing the bits can cause
1299 * issues on the VF drivers so we only need to clear what we set
1300 */
9d5c8243 1301 if (adapter->msix_entries) {
2dfd1212
AD		 1302 u32 regval = rd32(E1000_EIAM);
1303 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1304 wr32(E1000_EIMC, adapter->eims_enable_mask);
1305 regval = rd32(E1000_EIAC);
1306 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 1307 }
844290e5
PW		 1308
1309 wr32(E1000_IAM, 0);
9d5c8243
AK		 1310 wr32(E1000_IMC, ~0);
1311 wrfl();
81a61859
ET		 1312 if (adapter->msix_entries) {
1313 int i;
1314 for (i = 0; i < adapter->num_q_vectors; i++)
1315 synchronize_irq(adapter->msix_entries[i].vector);
1316 } else {
1317 synchronize_irq(adapter->pdev->irq);
1318 }
9d5c8243
AK		 1319}
1320
1321/**
1322 * igb_irq_enable - Enable default interrupt generation settings
1323 * @adapter: board private structure
1324 **/
1325static void igb_irq_enable(struct igb_adapter *adapter)
1326{
1327 struct e1000_hw *hw = &adapter->hw;
1328
1329 if (adapter->msix_entries) {
25568a53 1330 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
2dfd1212
AD		 1331 u32 regval = rd32(E1000_EIAC);
1332 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1333 regval = rd32(E1000_EIAM);
1334 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 1335 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 1336 if (adapter->vfs_allocated_count) {
4ae196df 1337 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1338 ims |= E1000_IMS_VMMB;
1339 }
55cac248
AD		 1340 if (adapter->hw.mac.type == e1000_82580)
1341 ims |= E1000_IMS_DRSTA;
1342
25568a53 1343 wr32(E1000_IMS, ims);
844290e5 1344 } else {
55cac248
AD		 1345 wr32(E1000_IMS, IMS_ENABLE_MASK |
1346 E1000_IMS_DRSTA);
1347 wr32(E1000_IAM, IMS_ENABLE_MASK |
1348 E1000_IMS_DRSTA);
844290e5 1349 }
9d5c8243
AK		 1350}
1351
1352static void igb_update_mng_vlan(struct igb_adapter *adapter)
1353{
51466239 1354 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1355 u16 vid = adapter->hw.mng_cookie.vlan_id;
1356 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1357
1358 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1359 /* add VID to filter table */
1360 igb_vfta_set(hw, vid, true);
1361 adapter->mng_vlan_id = vid;
1362 } else {
1363 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1364 }
1365
1366 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1367 (vid != old_vid) &&
b2cb09b1 1368 !test_bit(old_vid, adapter->active_vlans)) {
51466239
AD		 1369 /* remove VID from filter table */
1370 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1371 }
1372}
1373
1374/**
1375 * igb_release_hw_control - release control of the h/w to f/w
1376 * @adapter: address of board private structure
1377 *
1378 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1379 * For ASF and Pass Through versions of f/w this means that the
1380 * driver is no longer loaded.
1381 *
1382 **/
1383static void igb_release_hw_control(struct igb_adapter *adapter)
1384{
1385 struct e1000_hw *hw = &adapter->hw;
1386 u32 ctrl_ext;
1387
1388 /* Let firmware take over control of h/w */
1389 ctrl_ext = rd32(E1000_CTRL_EXT);
1390 wr32(E1000_CTRL_EXT,
1391 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1392}
1393
9d5c8243
AK		 1394/**
1395 * igb_get_hw_control - get control of the h/w from f/w
1396 * @adapter: address of board private structure
1397 *
1398 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1399 * For ASF and Pass Through versions of f/w this means that
1400 * the driver is loaded.
1401 *
1402 **/
1403static void igb_get_hw_control(struct igb_adapter *adapter)
1404{
1405 struct e1000_hw *hw = &adapter->hw;
1406 u32 ctrl_ext;
1407
1408 /* Let firmware know the driver has taken over */
1409 ctrl_ext = rd32(E1000_CTRL_EXT);
1410 wr32(E1000_CTRL_EXT,
1411 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1412}
1413
9d5c8243
AK		 1414/**
1415 * igb_configure - configure the hardware for RX and TX
1416 * @adapter: private board structure
1417 **/
1418static void igb_configure(struct igb_adapter *adapter)
1419{
1420 struct net_device *netdev = adapter->netdev;
1421 int i;
1422
1423 igb_get_hw_control(adapter);
ff41f8dc 1424 igb_set_rx_mode(netdev);
9d5c8243
AK		 1425
1426 igb_restore_vlan(adapter);
9d5c8243 1427
85b430b4 1428 igb_setup_tctl(adapter);
06cf2666 1429 igb_setup_mrqc(adapter);
9d5c8243 1430 igb_setup_rctl(adapter);
85b430b4
AD		 1431
1432 igb_configure_tx(adapter);
9d5c8243 1433 igb_configure_rx(adapter);
662d7205
AD		 1434
1435 igb_rx_fifo_flush_82575(&adapter->hw);
1436
c493ea45 1437 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1438 * at least 1 descriptor unused to make sure
1439 * next_to_use != next_to_clean */
1440 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1441 struct igb_ring *ring = adapter->rx_ring[i];
cd392f5c 1442 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
9d5c8243 1443 }
9d5c8243
AK		 1444}
1445
88a268c1
NN		 1446/**
1447 * igb_power_up_link - Power up the phy/serdes link
1448 * @adapter: address of board private structure
1449 **/
1450void igb_power_up_link(struct igb_adapter *adapter)
1451{
1452 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1453 igb_power_up_phy_copper(&adapter->hw);
1454 else
1455 igb_power_up_serdes_link_82575(&adapter->hw);
1456}
1457
1458/**
1459 * igb_power_down_link - Power down the phy/serdes link
1460 * @adapter: address of board private structure
1461 */
1462static void igb_power_down_link(struct igb_adapter *adapter)
1463{
1464 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1465 igb_power_down_phy_copper_82575(&adapter->hw);
1466 else
1467 igb_shutdown_serdes_link_82575(&adapter->hw);
1468}
9d5c8243
AK		 1469
1470/**
1471 * igb_up - Open the interface and prepare it to handle traffic
1472 * @adapter: board private structure
1473 **/
9d5c8243
AK		 1474int igb_up(struct igb_adapter *adapter)
1475{
1476 struct e1000_hw *hw = &adapter->hw;
1477 int i;
1478
1479 /* hardware has been reset, we need to reload some things */
1480 igb_configure(adapter);
1481
1482 clear_bit(__IGB_DOWN, &adapter->state);
1483
047e0030
AD		 1484 for (i = 0; i < adapter->num_q_vectors; i++) {
1485 struct igb_q_vector *q_vector = adapter->q_vector[i];
1486 napi_enable(&q_vector->napi);
1487 }
844290e5 1488 if (adapter->msix_entries)
9d5c8243 1489 igb_configure_msix(adapter);
feeb2721
AD		 1490 else
1491 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1492
1493 /* Clear any pending interrupts. */
1494 rd32(E1000_ICR);
1495 igb_irq_enable(adapter);
1496
d4960307
AD		 1497 /* notify VFs that reset has been completed */
1498 if (adapter->vfs_allocated_count) {
1499 u32 reg_data = rd32(E1000_CTRL_EXT);
1500 reg_data |= E1000_CTRL_EXT_PFRSTD;
1501 wr32(E1000_CTRL_EXT, reg_data);
1502 }
1503
4cb9be7a
JB		 1504 netif_tx_start_all_queues(adapter->netdev);
1505
25568a53
AD		 1506 /* start the watchdog. */
1507 hw->mac.get_link_status = 1;
1508 schedule_work(&adapter->watchdog_task);
1509
9d5c8243
AK		 1510 return 0;
1511}
1512
1513void igb_down(struct igb_adapter *adapter)
1514{
9d5c8243 1515 struct net_device *netdev = adapter->netdev;
330a6d6a 1516 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1517 u32 tctl, rctl;
1518 int i;
1519
1520 /* signal that we're down so the interrupt handler does not
1521 * reschedule our watchdog timer */
1522 set_bit(__IGB_DOWN, &adapter->state);
1523
1524 /* disable receives in the hardware */
1525 rctl = rd32(E1000_RCTL);
1526 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1527 /* flush and sleep below */
1528
fd2ea0a7 1529 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1530
1531 /* disable transmits in the hardware */
1532 tctl = rd32(E1000_TCTL);
1533 tctl &= ~E1000_TCTL_EN;
1534 wr32(E1000_TCTL, tctl);
1535 /* flush both disables and wait for them to finish */
1536 wrfl();
1537 msleep(10);
1538
047e0030
AD		 1539 for (i = 0; i < adapter->num_q_vectors; i++) {
1540 struct igb_q_vector *q_vector = adapter->q_vector[i];
1541 napi_disable(&q_vector->napi);
1542 }
9d5c8243 1543
9d5c8243
AK		 1544 igb_irq_disable(adapter);
1545
1546 del_timer_sync(&adapter->watchdog_timer);
1547 del_timer_sync(&adapter->phy_info_timer);
1548
9d5c8243 1549 netif_carrier_off(netdev);
04fe6358
AD		 1550
1551 /* record the stats before reset*/
12dcd86b
ED		 1552 spin_lock(&adapter->stats64_lock);
1553 igb_update_stats(adapter, &adapter->stats64);
1554 spin_unlock(&adapter->stats64_lock);
04fe6358 1555
9d5c8243
AK		 1556 adapter->link_speed = 0;
1557 adapter->link_duplex = 0;
1558
3023682e
JK		 1559 if (!pci_channel_offline(adapter->pdev))
1560 igb_reset(adapter);
9d5c8243
AK		 1561 igb_clean_all_tx_rings(adapter);
1562 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1563#ifdef CONFIG_IGB_DCA
1564
1565 /* since we reset the hardware DCA settings were cleared */
1566 igb_setup_dca(adapter);
1567#endif
9d5c8243
AK		 1568}
1569
1570void igb_reinit_locked(struct igb_adapter *adapter)
1571{
1572 WARN_ON(in_interrupt());
1573 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1574 msleep(1);
1575 igb_down(adapter);
1576 igb_up(adapter);
1577 clear_bit(__IGB_RESETTING, &adapter->state);
1578}
1579
1580void igb_reset(struct igb_adapter *adapter)
1581{
090b1795 1582 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1583 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1584 struct e1000_mac_info *mac = &hw->mac;
1585 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1586 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1587 u16 hwm;
1588
1589 /* Repartition Pba for greater than 9k mtu
1590 * To take effect CTRL.RST is required.
1591 */
fa4dfae0 1592 switch (mac->type) {
d2ba2ed8 1593 case e1000_i350:
55cac248
AD		 1594 case e1000_82580:
1595 pba = rd32(E1000_RXPBS);
1596 pba = igb_rxpbs_adjust_82580(pba);
1597 break;
fa4dfae0 1598 case e1000_82576:
d249be54
AD		 1599 pba = rd32(E1000_RXPBS);
1600 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1601 break;
1602 case e1000_82575:
1603 default:
1604 pba = E1000_PBA_34K;
1605 break;
2d064c06 1606 }
9d5c8243 1607
2d064c06
AD		 1608 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1609 (mac->type < e1000_82576)) {
9d5c8243
AK		 1610 /* adjust PBA for jumbo frames */
1611 wr32(E1000_PBA, pba);
1612
1613 /* To maintain wire speed transmits, the Tx FIFO should be
1614 * large enough to accommodate two full transmit packets,
1615 * rounded up to the next 1KB and expressed in KB. Likewise,
1616 * the Rx FIFO should be large enough to accommodate at least
1617 * one full receive packet and is similarly rounded up and
1618 * expressed in KB. */
1619 pba = rd32(E1000_PBA);
1620 /* upper 16 bits has Tx packet buffer allocation size in KB */
1621 tx_space = pba >> 16;
1622 /* lower 16 bits has Rx packet buffer allocation size in KB */
1623 pba &= 0xffff;
1624 /* the tx fifo also stores 16 bytes of information about the tx
1625 * but don't include ethernet FCS because hardware appends it */
1626 min_tx_space = (adapter->max_frame_size +
85e8d004 1627 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1628 ETH_FCS_LEN) * 2;
1629 min_tx_space = ALIGN(min_tx_space, 1024);
1630 min_tx_space >>= 10;
1631 /* software strips receive CRC, so leave room for it */
1632 min_rx_space = adapter->max_frame_size;
1633 min_rx_space = ALIGN(min_rx_space, 1024);
1634 min_rx_space >>= 10;
1635
1636 /* If current Tx allocation is less than the min Tx FIFO size,
1637 * and the min Tx FIFO size is less than the current Rx FIFO
1638 * allocation, take space away from current Rx allocation */
1639 if (tx_space < min_tx_space &&
1640 ((min_tx_space - tx_space) < pba)) {
1641 pba = pba - (min_tx_space - tx_space);
1642
1643 /* if short on rx space, rx wins and must trump tx
1644 * adjustment */
1645 if (pba < min_rx_space)
1646 pba = min_rx_space;
1647 }
2d064c06 1648 wr32(E1000_PBA, pba);
9d5c8243 1649 }
9d5c8243
AK		 1650
1651 /* flow control settings */
1652 /* The high water mark must be low enough to fit one full frame
1653 * (or the size used for early receive) above it in the Rx FIFO.
1654 * Set it to the lower of:
1655 * - 90% of the Rx FIFO size, or
1656 * - the full Rx FIFO size minus one full frame */
1657 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1658 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1659
d405ea3e
AD		 1660 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1661 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1662 fc->pause_time = 0xFFFF;
1663 fc->send_xon = 1;
0cce119a 1664 fc->current_mode = fc->requested_mode;
9d5c8243 1665
4ae196df
AD		 1666 /* disable receive for all VFs and wait one second */
1667 if (adapter->vfs_allocated_count) {
1668 int i;
1669 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
8fa7e0f7 1670 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
4ae196df
AD		 1671
1672 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1673 igb_ping_all_vfs(adapter);
4ae196df
AD		 1674
1675 /* disable transmits and receives */
1676 wr32(E1000_VFRE, 0);
1677 wr32(E1000_VFTE, 0);
1678 }
1679
9d5c8243 1680 /* Allow time for pending master requests to run */
330a6d6a 1681 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1682 wr32(E1000_WUC, 0);
1683
330a6d6a 1684 if (hw->mac.ops.init_hw(hw))
090b1795 1685 dev_err(&pdev->dev, "Hardware Error\n");
831ec0b4
CW		 1686 if (hw->mac.type > e1000_82580) {
1687 if (adapter->flags & IGB_FLAG_DMAC) {
1688 u32 reg;
1689
1690 /*
1691 * DMA Coalescing high water mark needs to be higher
1692 * than * the * Rx threshold. The Rx threshold is
1693 * currently * pba - 6, so we * should use a high water
1694 * mark of pba * - 4. */
1695 hwm = (pba - 4) << 10;
1696
1697 reg = (((pba-6) << E1000_DMACR_DMACTHR_SHIFT)
1698 & E1000_DMACR_DMACTHR_MASK);
1699
1700 /* transition to L0x or L1 if available..*/
1701 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
1702
1703 /* watchdog timer= +-1000 usec in 32usec intervals */
1704 reg |= (1000 >> 5);
1705 wr32(E1000_DMACR, reg);
1706
1707 /* no lower threshold to disable coalescing(smart fifb)
1708 * -UTRESH=0*/
1709 wr32(E1000_DMCRTRH, 0);
1710
1711 /* set hwm to PBA - 2 * max frame size */
1712 wr32(E1000_FCRTC, hwm);
1713
1714 /*
1715 * This sets the time to wait before requesting tran-
1716 * sition to * low power state to number of usecs needed
1717 * to receive 1 512 * byte frame at gigabit line rate
1718 */
1719 reg = rd32(E1000_DMCTLX);
1720 reg |= IGB_DMCTLX_DCFLUSH_DIS;
1721
1722 /* Delay 255 usec before entering Lx state. */
1723 reg |= 0xFF;
1724 wr32(E1000_DMCTLX, reg);
1725
1726 /* free space in Tx packet buffer to wake from DMAC */
1727 wr32(E1000_DMCTXTH,
1728 (IGB_MIN_TXPBSIZE -
1729 (IGB_TX_BUF_4096 + adapter->max_frame_size))
1730 >> 6);
1731
1732 /* make low power state decision controlled by DMAC */
1733 reg = rd32(E1000_PCIEMISC);
1734 reg |= E1000_PCIEMISC_LX_DECISION;
1735 wr32(E1000_PCIEMISC, reg);
1736 } /* end if IGB_FLAG_DMAC set */
1737 }
55cac248
AD		 1738 if (hw->mac.type == e1000_82580) {
1739 u32 reg = rd32(E1000_PCIEMISC);
1740 wr32(E1000_PCIEMISC,
1741 reg & ~E1000_PCIEMISC_LX_DECISION);
1742 }
88a268c1
NN		 1743 if (!netif_running(adapter->netdev))
1744 igb_power_down_link(adapter);
1745
9d5c8243
AK		 1746 igb_update_mng_vlan(adapter);
1747
1748 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1749 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1750
330a6d6a 1751 igb_get_phy_info(hw);
9d5c8243
AK		 1752}
1753
b2cb09b1
JP		 1754static u32 igb_fix_features(struct net_device *netdev, u32 features)
1755{
1756 /*
1757 * Since there is no support for separate rx/tx vlan accel
1758 * enable/disable make sure tx flag is always in same state as rx.
1759 */
1760 if (features & NETIF_F_HW_VLAN_RX)
1761 features |= NETIF_F_HW_VLAN_TX;
1762 else
1763 features &= ~NETIF_F_HW_VLAN_TX;
1764
1765 return features;
1766}
1767
ac52caa3
MM		 1768static int igb_set_features(struct net_device *netdev, u32 features)
1769{
1770 struct igb_adapter *adapter = netdev_priv(netdev);
1771 int i;
b2cb09b1 1772 u32 changed = netdev->features ^ features;
ac52caa3
MM		 1773
1774 for (i = 0; i < adapter->num_rx_queues; i++) {
1775 if (features & NETIF_F_RXCSUM)
1776 adapter->rx_ring[i]->flags |= IGB_RING_FLAG_RX_CSUM;
1777 else
1778 adapter->rx_ring[i]->flags &= ~IGB_RING_FLAG_RX_CSUM;
1779 }
1780
b2cb09b1
JP		 1781 if (changed & NETIF_F_HW_VLAN_RX)
1782 igb_vlan_mode(netdev, features);
1783
ac52caa3
MM		 1784 return 0;
1785}
1786
2e5c6922 1787static const struct net_device_ops igb_netdev_ops = {
559e9c49 1788 .ndo_open = igb_open,
2e5c6922 1789 .ndo_stop = igb_close,
cd392f5c 1790 .ndo_start_xmit = igb_xmit_frame,
12dcd86b 1791 .ndo_get_stats64 = igb_get_stats64,
ff41f8dc 1792 .ndo_set_rx_mode = igb_set_rx_mode,
2e5c6922
SH		 1793 .ndo_set_mac_address = igb_set_mac,
1794 .ndo_change_mtu = igb_change_mtu,
1795 .ndo_do_ioctl = igb_ioctl,
1796 .ndo_tx_timeout = igb_tx_timeout,
1797 .ndo_validate_addr = eth_validate_addr,
2e5c6922
SH		 1798 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1799 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1800 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1801 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1802 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1803 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1804#ifdef CONFIG_NET_POLL_CONTROLLER
1805 .ndo_poll_controller = igb_netpoll,
1806#endif
b2cb09b1
JP		 1807 .ndo_fix_features = igb_fix_features,
1808 .ndo_set_features = igb_set_features,
2e5c6922
SH		 1809};
1810
9d5c8243
AK		 1811/**
1812 * igb_probe - Device Initialization Routine
1813 * @pdev: PCI device information struct
1814 * @ent: entry in igb_pci_tbl
1815 *
1816 * Returns 0 on success, negative on failure
1817 *
1818 * igb_probe initializes an adapter identified by a pci_dev structure.
1819 * The OS initialization, configuring of the adapter private structure,
1820 * and a hardware reset occur.
1821 **/
1822static int __devinit igb_probe(struct pci_dev *pdev,
1823 const struct pci_device_id *ent)
1824{
1825 struct net_device *netdev;
1826 struct igb_adapter *adapter;
1827 struct e1000_hw *hw;
4337e993 1828 u16 eeprom_data = 0;
9835fd73 1829 s32 ret_val;
4337e993 1830 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1831 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1832 unsigned long mmio_start, mmio_len;
2d6a5e95 1833 int err, pci_using_dac;
9d5c8243 1834 u16 eeprom_apme_mask = IGB_EEPROM_APME;
9835fd73 1835 u8 part_str[E1000_PBANUM_LENGTH];
9d5c8243 1836
bded64a7
AG		 1837 /* Catch broken hardware that put the wrong VF device ID in
1838 * the PCIe SR-IOV capability.
1839 */
1840 if (pdev->is_virtfn) {
1841 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1842 pci_name(pdev), pdev->vendor, pdev->device);
1843 return -EINVAL;
1844 }
1845
aed5dec3 1846 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1847 if (err)
1848 return err;
1849
1850 pci_using_dac = 0;
59d71989 1851 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243 1852 if (!err) {
59d71989 1853 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243
AK		 1854 if (!err)
1855 pci_using_dac = 1;
1856 } else {
59d71989 1857 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243 1858 if (err) {
59d71989 1859 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243
AK		 1860 if (err) {
1861 dev_err(&pdev->dev, "No usable DMA "
1862 "configuration, aborting\n");
1863 goto err_dma;
1864 }
1865 }
1866 }
1867
aed5dec3
AD		 1868 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1869 IORESOURCE_MEM),
1870 igb_driver_name);
9d5c8243
AK		 1871 if (err)
1872 goto err_pci_reg;
1873
19d5afd4 1874 pci_enable_pcie_error_reporting(pdev);
40a914fa 1875
9d5c8243 1876 pci_set_master(pdev);
c682fc23 1877 pci_save_state(pdev);
9d5c8243
AK		 1878
1879 err = -ENOMEM;
1bfaf07b 1880 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1cc3bd87 1881 IGB_MAX_TX_QUEUES);
9d5c8243
AK		 1882 if (!netdev)
1883 goto err_alloc_etherdev;
1884
1885 SET_NETDEV_DEV(netdev, &pdev->dev);
1886
1887 pci_set_drvdata(pdev, netdev);
1888 adapter = netdev_priv(netdev);
1889 adapter->netdev = netdev;
1890 adapter->pdev = pdev;
1891 hw = &adapter->hw;
1892 hw->back = adapter;
1893 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1894
1895 mmio_start = pci_resource_start(pdev, 0);
1896 mmio_len = pci_resource_len(pdev, 0);
1897
1898 err = -EIO;
28b0759c
AD		 1899 hw->hw_addr = ioremap(mmio_start, mmio_len);
1900 if (!hw->hw_addr)
9d5c8243
AK		 1901 goto err_ioremap;
1902
2e5c6922 1903 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1904 igb_set_ethtool_ops(netdev);
9d5c8243 1905 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1906
1907 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1908
1909 netdev->mem_start = mmio_start;
1910 netdev->mem_end = mmio_start + mmio_len;
1911
9d5c8243
AK		 1912 /* PCI config space info */
1913 hw->vendor_id = pdev->vendor;
1914 hw->device_id = pdev->device;
1915 hw->revision_id = pdev->revision;
1916 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1917 hw->subsystem_device_id = pdev->subsystem_device;
1918
9d5c8243
AK		 1919 /* Copy the default MAC, PHY and NVM function pointers */
1920 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1921 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1922 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1923 /* Initialize skew-specific constants */
1924 err = ei->get_invariants(hw);
1925 if (err)
450c87c8 1926 goto err_sw_init;
9d5c8243 1927
450c87c8 1928 /* setup the private structure */
9d5c8243
AK		 1929 err = igb_sw_init(adapter);
1930 if (err)
1931 goto err_sw_init;
1932
1933 igb_get_bus_info_pcie(hw);
1934
1935 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 1936
1937 /* Copper options */
1938 if (hw->phy.media_type == e1000_media_type_copper) {
1939 hw->phy.mdix = AUTO_ALL_MODES;
1940 hw->phy.disable_polarity_correction = false;
1941 hw->phy.ms_type = e1000_ms_hw_default;
1942 }
1943
1944 if (igb_check_reset_block(hw))
1945 dev_info(&pdev->dev,
1946 "PHY reset is blocked due to SOL/IDER session.\n");
1947
ac52caa3 1948 netdev->hw_features = NETIF_F_SG |
7d8eb29e 1949 NETIF_F_IP_CSUM |
ac52caa3
MM		 1950 NETIF_F_IPV6_CSUM |
1951 NETIF_F_TSO |
1952 NETIF_F_TSO6 |
b2cb09b1
JP		 1953 NETIF_F_RXCSUM |
1954 NETIF_F_HW_VLAN_RX;
ac52caa3
MM		 1955
1956 netdev->features = netdev->hw_features |
9d5c8243 1957 NETIF_F_HW_VLAN_TX |
9d5c8243
AK		 1958 NETIF_F_HW_VLAN_FILTER;
1959
48f29ffc
JK		 1960 netdev->vlan_features |= NETIF_F_TSO;
1961 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 1962 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 1963 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 1964 netdev->vlan_features |= NETIF_F_SG;
1965
7b872a55 1966 if (pci_using_dac) {
9d5c8243 1967 netdev->features |= NETIF_F_HIGHDMA;
7b872a55
YZ		 1968 netdev->vlan_features |= NETIF_F_HIGHDMA;
1969 }
9d5c8243 1970
ac52caa3
MM		 1971 if (hw->mac.type >= e1000_82576) {
1972 netdev->hw_features |= NETIF_F_SCTP_CSUM;
b9473560 1973 netdev->features |= NETIF_F_SCTP_CSUM;
ac52caa3 1974 }
b9473560 1975
01789349
JP		 1976 netdev->priv_flags |= IFF_UNICAST_FLT;
1977
330a6d6a 1978 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 1979
1980 /* before reading the NVM, reset the controller to put the device in a
1981 * known good starting state */
1982 hw->mac.ops.reset_hw(hw);
1983
1984 /* make sure the NVM is good */
4322e561 1985 if (hw->nvm.ops.validate(hw) < 0) {
9d5c8243
AK		 1986 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1987 err = -EIO;
1988 goto err_eeprom;
1989 }
1990
1991 /* copy the MAC address out of the NVM */
1992 if (hw->mac.ops.read_mac_addr(hw))
1993 dev_err(&pdev->dev, "NVM Read Error\n");
1994
1995 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1996 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1997
1998 if (!is_valid_ether_addr(netdev->perm_addr)) {
1999 dev_err(&pdev->dev, "Invalid MAC Address\n");
2000 err = -EIO;
2001 goto err_eeprom;
2002 }
2003
c061b18d 2004 setup_timer(&adapter->watchdog_timer, igb_watchdog,
0e340485 2005 (unsigned long) adapter);
c061b18d 2006 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
0e340485 2007 (unsigned long) adapter);
9d5c8243
AK		 2008
2009 INIT_WORK(&adapter->reset_task, igb_reset_task);
2010 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2011
450c87c8 2012 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 2013 adapter->fc_autoneg = true;
2014 hw->mac.autoneg = true;
2015 hw->phy.autoneg_advertised = 0x2f;
2016
0cce119a
AD		 2017 hw->fc.requested_mode = e1000_fc_default;
2018 hw->fc.current_mode = e1000_fc_default;
9d5c8243 2019
9d5c8243
AK		 2020 igb_validate_mdi_setting(hw);
2021
9d5c8243
AK		 2022 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2023 * enable the ACPI Magic Packet filter
2024 */
2025
a2cf8b6c 2026 if (hw->bus.func == 0)
312c75ae 2027 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
6d337dce 2028 else if (hw->mac.type >= e1000_82580)
55cac248
AD		 2029 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2030 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2031 &eeprom_data);
a2cf8b6c
AD		 2032 else if (hw->bus.func == 1)
2033 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 2034
2035 if (eeprom_data & eeprom_apme_mask)
2036 adapter->eeprom_wol |= E1000_WUFC_MAG;
2037
2038 /* now that we have the eeprom settings, apply the special cases where
2039 * the eeprom may be wrong or the board simply won't support wake on
2040 * lan on a particular port */
2041 switch (pdev->device) {
2042 case E1000_DEV_ID_82575GB_QUAD_COPPER:
2043 adapter->eeprom_wol = 0;
2044 break;
2045 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 2046 case E1000_DEV_ID_82576_FIBER:
2047 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 2048 /* Wake events only supported on port A for dual fiber
2049 * regardless of eeprom setting */
2050 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
2051 adapter->eeprom_wol = 0;
2052 break;
c8ea5ea9 2053 case E1000_DEV_ID_82576_QUAD_COPPER:
d5aa2252 2054 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
c8ea5ea9
AD		 2055 /* if quad port adapter, disable WoL on all but port A */
2056 if (global_quad_port_a != 0)
2057 adapter->eeprom_wol = 0;
2058 else
2059 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2060 /* Reset for multiple quad port adapters */
2061 if (++global_quad_port_a == 4)
2062 global_quad_port_a = 0;
2063 break;
9d5c8243
AK		 2064 }
2065
2066 /* initialize the wol settings based on the eeprom settings */
2067 adapter->wol = adapter->eeprom_wol;
e1b86d84 2068 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 2069
2070 /* reset the hardware with the new settings */
2071 igb_reset(adapter);
2072
2073 /* let the f/w know that the h/w is now under the control of the
2074 * driver. */
2075 igb_get_hw_control(adapter);
2076
9d5c8243
AK		 2077 strcpy(netdev->name, "eth%d");
2078 err = register_netdev(netdev);
2079 if (err)
2080 goto err_register;
2081
b2cb09b1
JP		 2082 igb_vlan_mode(netdev, netdev->features);
2083
b168dfc5
JB		 2084 /* carrier off reporting is important to ethtool even BEFORE open */
2085 netif_carrier_off(netdev);
2086
421e02f0 2087#ifdef CONFIG_IGB_DCA
bbd98fe4 2088 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 2089 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 2090 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 2091 igb_setup_dca(adapter);
2092 }
fe4506b6 2093
38c845c7 2094#endif
673b8b70
AB		 2095 /* do hw tstamp init after resetting */
2096 igb_init_hw_timer(adapter);
2097
9d5c8243
AK		 2098 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
2099 /* print bus type/speed/width info */
7c510e4b 2100 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 2101 netdev->name,
559e9c49 2102 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
ff846f52 2103 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
559e9c49 2104 "unknown"),
59c3de89
AD		 2105 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
2106 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
2107 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2108 "unknown"),
7c510e4b 2109 netdev->dev_addr);
9d5c8243 2110
9835fd73
CW		 2111 ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH);
2112 if (ret_val)
2113 strcpy(part_str, "Unknown");
2114 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
9d5c8243
AK		 2115 dev_info(&pdev->dev,
2116 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2117 adapter->msix_entries ? "MSI-X" :
7dfc16fa 2118 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243 2119 adapter->num_rx_queues, adapter->num_tx_queues);
09b068d4
CW		 2120 switch (hw->mac.type) {
2121 case e1000_i350:
2122 igb_set_eee_i350(hw);
2123 break;
2124 default:
2125 break;
2126 }
9d5c8243
AK		 2127 return 0;
2128
2129err_register:
2130 igb_release_hw_control(adapter);
2131err_eeprom:
2132 if (!igb_check_reset_block(hw))
f5f4cf08 2133 igb_reset_phy(hw);
9d5c8243
AK		 2134
2135 if (hw->flash_address)
2136 iounmap(hw->flash_address);
9d5c8243 2137err_sw_init:
047e0030 2138 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 2139 iounmap(hw->hw_addr);
2140err_ioremap:
2141 free_netdev(netdev);
2142err_alloc_etherdev:
559e9c49
AD		 2143 pci_release_selected_regions(pdev,
2144 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2145err_pci_reg:
2146err_dma:
2147 pci_disable_device(pdev);
2148 return err;
2149}
2150
2151/**
2152 * igb_remove - Device Removal Routine
2153 * @pdev: PCI device information struct
2154 *
2155 * igb_remove is called by the PCI subsystem to alert the driver
2156 * that it should release a PCI device. The could be caused by a
2157 * Hot-Plug event, or because the driver is going to be removed from
2158 * memory.
2159 **/
2160static void __devexit igb_remove(struct pci_dev *pdev)
2161{
2162 struct net_device *netdev = pci_get_drvdata(pdev);
2163 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 2164 struct e1000_hw *hw = &adapter->hw;
9d5c8243 2165
760141a5
TH		 2166 /*
2167 * The watchdog timer may be rescheduled, so explicitly
2168 * disable watchdog from being rescheduled.
2169 */
9d5c8243
AK		 2170 set_bit(__IGB_DOWN, &adapter->state);
2171 del_timer_sync(&adapter->watchdog_timer);
2172 del_timer_sync(&adapter->phy_info_timer);
2173
760141a5
TH		 2174 cancel_work_sync(&adapter->reset_task);
2175 cancel_work_sync(&adapter->watchdog_task);
9d5c8243 2176
421e02f0 2177#ifdef CONFIG_IGB_DCA
7dfc16fa 2178 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 2179 dev_info(&pdev->dev, "DCA disabled\n");
2180 dca_remove_requester(&pdev->dev);
7dfc16fa 2181 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 2182 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 2183 }
2184#endif
2185
9d5c8243
AK		 2186 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2187 * would have already happened in close and is redundant. */
2188 igb_release_hw_control(adapter);
2189
2190 unregister_netdev(netdev);
2191
047e0030 2192 igb_clear_interrupt_scheme(adapter);
9d5c8243 2193
37680117
AD		 2194#ifdef CONFIG_PCI_IOV
2195 /* reclaim resources allocated to VFs */
2196 if (adapter->vf_data) {
2197 /* disable iov and allow time for transactions to clear */
2198 pci_disable_sriov(pdev);
2199 msleep(500);
2200
2201 kfree(adapter->vf_data);
2202 adapter->vf_data = NULL;
2203 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 2204 wrfl();
37680117
AD		 2205 msleep(100);
2206 dev_info(&pdev->dev, "IOV Disabled\n");
2207 }
2208#endif
559e9c49 2209
28b0759c
AD		 2210 iounmap(hw->hw_addr);
2211 if (hw->flash_address)
2212 iounmap(hw->flash_address);
559e9c49
AD		 2213 pci_release_selected_regions(pdev,
2214 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2215
2216 free_netdev(netdev);
2217
19d5afd4 2218 pci_disable_pcie_error_reporting(pdev);
40a914fa 2219
9d5c8243
AK		 2220 pci_disable_device(pdev);
2221}
2222
a6b623e0
AD		 2223/**
2224 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2225 * @adapter: board private structure to initialize
2226 *
2227 * This function initializes the vf specific data storage and then attempts to
2228 * allocate the VFs. The reason for ordering it this way is because it is much
2229 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2230 * the memory for the VFs.
2231 **/
2232static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2233{
2234#ifdef CONFIG_PCI_IOV
2235 struct pci_dev *pdev = adapter->pdev;
2236
a6b623e0
AD		 2237 if (adapter->vfs_allocated_count) {
2238 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2239 sizeof(struct vf_data_storage),
2240 GFP_KERNEL);
2241 /* if allocation failed then we do not support SR-IOV */
2242 if (!adapter->vf_data) {
2243 adapter->vfs_allocated_count = 0;
2244 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2245 "Data Storage\n");
2246 }
2247 }
2248
2249 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
2250 kfree(adapter->vf_data);
2251 adapter->vf_data = NULL;
2252#endif /* CONFIG_PCI_IOV */
2253 adapter->vfs_allocated_count = 0;
2254#ifdef CONFIG_PCI_IOV
2255 } else {
2256 unsigned char mac_addr[ETH_ALEN];
2257 int i;
2258 dev_info(&pdev->dev, "%d vfs allocated\n",
2259 adapter->vfs_allocated_count);
2260 for (i = 0; i < adapter->vfs_allocated_count; i++) {
2261 random_ether_addr(mac_addr);
2262 igb_set_vf_mac(adapter, i, mac_addr);
2263 }
831ec0b4
CW		 2264 /* DMA Coalescing is not supported in IOV mode. */
2265 if (adapter->flags & IGB_FLAG_DMAC)
2266 adapter->flags &= ~IGB_FLAG_DMAC;
a6b623e0
AD		 2267 }
2268#endif /* CONFIG_PCI_IOV */
2269}
2270
115f459a
AD		 2271
2272/**
2273 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2274 * @adapter: board private structure to initialize
2275 *
2276 * igb_init_hw_timer initializes the function pointer and values for the hw
2277 * timer found in hardware.
2278 **/
2279static void igb_init_hw_timer(struct igb_adapter *adapter)
2280{
2281 struct e1000_hw *hw = &adapter->hw;
2282
2283 switch (hw->mac.type) {
d2ba2ed8 2284 case e1000_i350:
55cac248
AD		 2285 case e1000_82580:
2286 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2287 adapter->cycles.read = igb_read_clock;
2288 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2289 adapter->cycles.mult = 1;
2290 /*
2291 * The 82580 timesync updates the system timer every 8ns by 8ns
2292 * and the value cannot be shifted. Instead we need to shift
2293 * the registers to generate a 64bit timer value. As a result
2294 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2295 * 24 in order to generate a larger value for synchronization.
2296 */
2297 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2298 /* disable system timer temporarily by setting bit 31 */
2299 wr32(E1000_TSAUXC, 0x80000000);
2300 wrfl();
2301
2302 /* Set registers so that rollover occurs soon to test this. */
2303 wr32(E1000_SYSTIMR, 0x00000000);
2304 wr32(E1000_SYSTIML, 0x80000000);
2305 wr32(E1000_SYSTIMH, 0x000000FF);
2306 wrfl();
2307
2308 /* enable system timer by clearing bit 31 */
2309 wr32(E1000_TSAUXC, 0x0);
2310 wrfl();
2311
2312 timecounter_init(&adapter->clock,
2313 &adapter->cycles,
2314 ktime_to_ns(ktime_get_real()));
2315 /*
2316 * Synchronize our NIC clock against system wall clock. NIC
2317 * time stamp reading requires ~3us per sample, each sample
2318 * was pretty stable even under load => only require 10
2319 * samples for each offset comparison.
2320 */
2321 memset(&adapter->compare, 0, sizeof(adapter->compare));
2322 adapter->compare.source = &adapter->clock;
2323 adapter->compare.target = ktime_get_real;
2324 adapter->compare.num_samples = 10;
2325 timecompare_update(&adapter->compare, 0);
2326 break;
115f459a
AD		 2327 case e1000_82576:
2328 /*
2329 * Initialize hardware timer: we keep it running just in case
2330 * that some program needs it later on.
2331 */
2332 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2333 adapter->cycles.read = igb_read_clock;
2334 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2335 adapter->cycles.mult = 1;
2336 /**
2337 * Scale the NIC clock cycle by a large factor so that
2338 * relatively small clock corrections can be added or
25985edc 2339 * subtracted at each clock tick. The drawbacks of a large
115f459a
AD		 2340 * factor are a) that the clock register overflows more quickly
2341 * (not such a big deal) and b) that the increment per tick has
2342 * to fit into 24 bits. As a result we need to use a shift of
2343 * 19 so we can fit a value of 16 into the TIMINCA register.
2344 */
2345 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2346 wr32(E1000_TIMINCA,
2347 (1 << E1000_TIMINCA_16NS_SHIFT) |
2348 (16 << IGB_82576_TSYNC_SHIFT));
2349
2350 /* Set registers so that rollover occurs soon to test this. */
2351 wr32(E1000_SYSTIML, 0x00000000);
2352 wr32(E1000_SYSTIMH, 0xFF800000);
2353 wrfl();
2354
2355 timecounter_init(&adapter->clock,
2356 &adapter->cycles,
2357 ktime_to_ns(ktime_get_real()));
2358 /*
2359 * Synchronize our NIC clock against system wall clock. NIC
2360 * time stamp reading requires ~3us per sample, each sample
2361 * was pretty stable even under load => only require 10
2362 * samples for each offset comparison.
2363 */
2364 memset(&adapter->compare, 0, sizeof(adapter->compare));
2365 adapter->compare.source = &adapter->clock;
2366 adapter->compare.target = ktime_get_real;
2367 adapter->compare.num_samples = 10;
2368 timecompare_update(&adapter->compare, 0);
2369 break;
2370 case e1000_82575:
2371 /* 82575 does not support timesync */
2372 default:
2373 break;
2374 }
2375
2376}
2377
9d5c8243
AK		 2378/**
2379 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2380 * @adapter: board private structure to initialize
2381 *
2382 * igb_sw_init initializes the Adapter private data structure.
2383 * Fields are initialized based on PCI device information and
2384 * OS network device settings (MTU size).
2385 **/
2386static int __devinit igb_sw_init(struct igb_adapter *adapter)
2387{
2388 struct e1000_hw *hw = &adapter->hw;
2389 struct net_device *netdev = adapter->netdev;
2390 struct pci_dev *pdev = adapter->pdev;
2391
2392 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2393
13fde97a 2394 /* set default ring sizes */
68fd9910
AD		 2395 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2396 adapter->rx_ring_count = IGB_DEFAULT_RXD;
13fde97a
AD		 2397
2398 /* set default ITR values */
4fc82adf
AD		 2399 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2400 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2401
13fde97a
AD		 2402 /* set default work limits */
2403 adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
2404
153285f9
AD		 2405 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
2406 VLAN_HLEN;
9d5c8243
AK		 2407 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2408
12dcd86b 2409 spin_lock_init(&adapter->stats64_lock);
a6b623e0 2410#ifdef CONFIG_PCI_IOV
6b78bb1d
CW		 2411 switch (hw->mac.type) {
2412 case e1000_82576:
2413 case e1000_i350:
9b082d73
SA		 2414 if (max_vfs > 7) {
2415 dev_warn(&pdev->dev,
2416 "Maximum of 7 VFs per PF, using max\n");
2417 adapter->vfs_allocated_count = 7;
2418 } else
2419 adapter->vfs_allocated_count = max_vfs;
6b78bb1d
CW		 2420 break;
2421 default:
2422 break;
2423 }
a6b623e0 2424#endif /* CONFIG_PCI_IOV */
a99955fc 2425 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
665c8c8e
WM		 2426 /* i350 cannot do RSS and SR-IOV at the same time */
2427 if (hw->mac.type == e1000_i350 && adapter->vfs_allocated_count)
2428 adapter->rss_queues = 1;
a99955fc
AD		 2429
2430 /*
2431 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2432 * then we should combine the queues into a queue pair in order to
2433 * conserve interrupts due to limited supply
2434 */
2435 if ((adapter->rss_queues > 4) ||
2436 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2437 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2438
a6b623e0 2439 /* This call may decrease the number of queues */
047e0030 2440 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 2441 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2442 return -ENOMEM;
2443 }
2444
a6b623e0
AD		 2445 igb_probe_vfs(adapter);
2446
9d5c8243
AK		 2447 /* Explicitly disable IRQ since the NIC can be in any state. */
2448 igb_irq_disable(adapter);
2449
831ec0b4
CW		 2450 if (hw->mac.type == e1000_i350)
2451 adapter->flags &= ~IGB_FLAG_DMAC;
2452
9d5c8243
AK		 2453 set_bit(__IGB_DOWN, &adapter->state);
2454 return 0;
2455}
2456
2457/**
2458 * igb_open - Called when a network interface is made active
2459 * @netdev: network interface device structure
2460 *
2461 * Returns 0 on success, negative value on failure
2462 *
2463 * The open entry point is called when a network interface is made
2464 * active by the system (IFF_UP). At this point all resources needed
2465 * for transmit and receive operations are allocated, the interrupt
2466 * handler is registered with the OS, the watchdog timer is started,
2467 * and the stack is notified that the interface is ready.
2468 **/
2469static int igb_open(struct net_device *netdev)
2470{
2471 struct igb_adapter *adapter = netdev_priv(netdev);
2472 struct e1000_hw *hw = &adapter->hw;
2473 int err;
2474 int i;
2475
2476 /* disallow open during test */
2477 if (test_bit(__IGB_TESTING, &adapter->state))
2478 return -EBUSY;
2479
b168dfc5
JB		 2480 netif_carrier_off(netdev);
2481
9d5c8243
AK		 2482 /* allocate transmit descriptors */
2483 err = igb_setup_all_tx_resources(adapter);
2484 if (err)
2485 goto err_setup_tx;
2486
2487 /* allocate receive descriptors */
2488 err = igb_setup_all_rx_resources(adapter);
2489 if (err)
2490 goto err_setup_rx;
2491
88a268c1 2492 igb_power_up_link(adapter);
9d5c8243 2493
9d5c8243
AK		 2494 /* before we allocate an interrupt, we must be ready to handle it.
2495 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2496 * as soon as we call pci_request_irq, so we have to setup our
2497 * clean_rx handler before we do so. */
2498 igb_configure(adapter);
2499
2500 err = igb_request_irq(adapter);
2501 if (err)
2502 goto err_req_irq;
2503
2504 /* From here on the code is the same as igb_up() */
2505 clear_bit(__IGB_DOWN, &adapter->state);
2506
047e0030
AD		 2507 for (i = 0; i < adapter->num_q_vectors; i++) {
2508 struct igb_q_vector *q_vector = adapter->q_vector[i];
2509 napi_enable(&q_vector->napi);
2510 }
9d5c8243
AK		 2511
2512 /* Clear any pending interrupts. */
2513 rd32(E1000_ICR);
844290e5
PW		 2514
2515 igb_irq_enable(adapter);
2516
d4960307
AD		 2517 /* notify VFs that reset has been completed */
2518 if (adapter->vfs_allocated_count) {
2519 u32 reg_data = rd32(E1000_CTRL_EXT);
2520 reg_data |= E1000_CTRL_EXT_PFRSTD;
2521 wr32(E1000_CTRL_EXT, reg_data);
2522 }
2523
d55b53ff
JK		 2524 netif_tx_start_all_queues(netdev);
2525
25568a53
AD		 2526 /* start the watchdog. */
2527 hw->mac.get_link_status = 1;
2528 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2529
2530 return 0;
2531
2532err_req_irq:
2533 igb_release_hw_control(adapter);
88a268c1 2534 igb_power_down_link(adapter);
9d5c8243
AK		 2535 igb_free_all_rx_resources(adapter);
2536err_setup_rx:
2537 igb_free_all_tx_resources(adapter);
2538err_setup_tx:
2539 igb_reset(adapter);
2540
2541 return err;
2542}
2543
2544/**
2545 * igb_close - Disables a network interface
2546 * @netdev: network interface device structure
2547 *
2548 * Returns 0, this is not allowed to fail
2549 *
2550 * The close entry point is called when an interface is de-activated
2551 * by the OS. The hardware is still under the driver's control, but
2552 * needs to be disabled. A global MAC reset is issued to stop the
2553 * hardware, and all transmit and receive resources are freed.
2554 **/
2555static int igb_close(struct net_device *netdev)
2556{
2557 struct igb_adapter *adapter = netdev_priv(netdev);
2558
2559 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2560 igb_down(adapter);
2561
2562 igb_free_irq(adapter);
2563
2564 igb_free_all_tx_resources(adapter);
2565 igb_free_all_rx_resources(adapter);
2566
9d5c8243
AK		 2567 return 0;
2568}
2569
2570/**
2571 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2572 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2573 *
2574 * Return 0 on success, negative on failure
2575 **/
80785298 2576int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2577{
59d71989 2578 struct device *dev = tx_ring->dev;
9d5c8243
AK		 2579 int size;
2580
06034649
AD		 2581 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
2582 tx_ring->tx_buffer_info = vzalloc(size);
2583 if (!tx_ring->tx_buffer_info)
9d5c8243 2584 goto err;
9d5c8243
AK		 2585
2586 /* round up to nearest 4K */
85e8d004 2587 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2588 tx_ring->size = ALIGN(tx_ring->size, 4096);
2589
59d71989
AD		 2590 tx_ring->desc = dma_alloc_coherent(dev,
2591 tx_ring->size,
2592 &tx_ring->dma,
2593 GFP_KERNEL);
9d5c8243
AK		 2594
2595 if (!tx_ring->desc)
2596 goto err;
2597
9d5c8243
AK		 2598 tx_ring->next_to_use = 0;
2599 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2600 return 0;
2601
2602err:
06034649 2603 vfree(tx_ring->tx_buffer_info);
59d71989 2604 dev_err(dev,
9d5c8243
AK		 2605 "Unable to allocate memory for the transmit descriptor ring\n");
2606 return -ENOMEM;
2607}
2608
2609/**
2610 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2611 * (Descriptors) for all queues
2612 * @adapter: board private structure
2613 *
2614 * Return 0 on success, negative on failure
2615 **/
2616static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2617{
439705e1 2618 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2619 int i, err = 0;
2620
2621 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2622 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2623 if (err) {
439705e1 2624 dev_err(&pdev->dev,
9d5c8243
AK		 2625 "Allocation for Tx Queue %u failed\n", i);
2626 for (i--; i >= 0; i--)
3025a446 2627 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2628 break;
2629 }
2630 }
2631
2632 return err;
2633}
2634
2635/**
85b430b4
AD		 2636 * igb_setup_tctl - configure the transmit control registers
2637 * @adapter: Board private structure
9d5c8243 2638 **/
d7ee5b3a 2639void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2640{
9d5c8243
AK		 2641 struct e1000_hw *hw = &adapter->hw;
2642 u32 tctl;
9d5c8243 2643
85b430b4
AD		 2644 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2645 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2646
2647 /* Program the Transmit Control Register */
9d5c8243
AK		 2648 tctl = rd32(E1000_TCTL);
2649 tctl &= ~E1000_TCTL_CT;
2650 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2651 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2652
2653 igb_config_collision_dist(hw);
2654
9d5c8243
AK		 2655 /* Enable transmits */
2656 tctl |= E1000_TCTL_EN;
2657
2658 wr32(E1000_TCTL, tctl);
2659}
2660
85b430b4
AD		 2661/**
2662 * igb_configure_tx_ring - Configure transmit ring after Reset
2663 * @adapter: board private structure
2664 * @ring: tx ring to configure
2665 *
2666 * Configure a transmit ring after a reset.
2667 **/
d7ee5b3a
AD		 2668void igb_configure_tx_ring(struct igb_adapter *adapter,
2669 struct igb_ring *ring)
85b430b4
AD		 2670{
2671 struct e1000_hw *hw = &adapter->hw;
a74420e0 2672 u32 txdctl = 0;
85b430b4
AD		 2673 u64 tdba = ring->dma;
2674 int reg_idx = ring->reg_idx;
2675
2676 /* disable the queue */
a74420e0 2677 wr32(E1000_TXDCTL(reg_idx), 0);
85b430b4
AD		 2678 wrfl();
2679 mdelay(10);
2680
2681 wr32(E1000_TDLEN(reg_idx),
2682 ring->count * sizeof(union e1000_adv_tx_desc));
2683 wr32(E1000_TDBAL(reg_idx),
2684 tdba & 0x00000000ffffffffULL);
2685 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2686
fce99e34 2687 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
a74420e0 2688 wr32(E1000_TDH(reg_idx), 0);
fce99e34 2689 writel(0, ring->tail);
85b430b4
AD		 2690
2691 txdctl |= IGB_TX_PTHRESH;
2692 txdctl |= IGB_TX_HTHRESH << 8;
2693 txdctl |= IGB_TX_WTHRESH << 16;
2694
2695 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2696 wr32(E1000_TXDCTL(reg_idx), txdctl);
2697}
2698
2699/**
2700 * igb_configure_tx - Configure transmit Unit after Reset
2701 * @adapter: board private structure
2702 *
2703 * Configure the Tx unit of the MAC after a reset.
2704 **/
2705static void igb_configure_tx(struct igb_adapter *adapter)
2706{
2707 int i;
2708
2709 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2710 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2711}
2712
9d5c8243
AK		 2713/**
2714 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2715 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2716 *
2717 * Returns 0 on success, negative on failure
2718 **/
80785298 2719int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2720{
59d71989 2721 struct device *dev = rx_ring->dev;
9d5c8243
AK		 2722 int size, desc_len;
2723
06034649
AD		 2724 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
2725 rx_ring->rx_buffer_info = vzalloc(size);
2726 if (!rx_ring->rx_buffer_info)
9d5c8243 2727 goto err;
9d5c8243
AK		 2728
2729 desc_len = sizeof(union e1000_adv_rx_desc);
2730
2731 /* Round up to nearest 4K */
2732 rx_ring->size = rx_ring->count * desc_len;
2733 rx_ring->size = ALIGN(rx_ring->size, 4096);
2734
59d71989
AD		 2735 rx_ring->desc = dma_alloc_coherent(dev,
2736 rx_ring->size,
2737 &rx_ring->dma,
2738 GFP_KERNEL);
9d5c8243
AK		 2739
2740 if (!rx_ring->desc)
2741 goto err;
2742
2743 rx_ring->next_to_clean = 0;
2744 rx_ring->next_to_use = 0;
9d5c8243 2745
9d5c8243
AK		 2746 return 0;
2747
2748err:
06034649
AD		 2749 vfree(rx_ring->rx_buffer_info);
2750 rx_ring->rx_buffer_info = NULL;
59d71989
AD		 2751 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2752 " ring\n");
9d5c8243
AK		 2753 return -ENOMEM;
2754}
2755
2756/**
2757 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2758 * (Descriptors) for all queues
2759 * @adapter: board private structure
2760 *
2761 * Return 0 on success, negative on failure
2762 **/
2763static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2764{
439705e1 2765 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2766 int i, err = 0;
2767
2768 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2769 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2770 if (err) {
439705e1 2771 dev_err(&pdev->dev,
9d5c8243
AK		 2772 "Allocation for Rx Queue %u failed\n", i);
2773 for (i--; i >= 0; i--)
3025a446 2774 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2775 break;
2776 }
2777 }
2778
2779 return err;
2780}
2781
06cf2666
AD		 2782/**
2783 * igb_setup_mrqc - configure the multiple receive queue control registers
2784 * @adapter: Board private structure
2785 **/
2786static void igb_setup_mrqc(struct igb_adapter *adapter)
2787{
2788 struct e1000_hw *hw = &adapter->hw;
2789 u32 mrqc, rxcsum;
2790 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2791 union e1000_reta {
2792 u32 dword;
2793 u8 bytes[4];
2794 } reta;
2795 static const u8 rsshash[40] = {
2796 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2797 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2798 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2799 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2800
2801 /* Fill out hash function seeds */
2802 for (j = 0; j < 10; j++) {
2803 u32 rsskey = rsshash[(j * 4)];
2804 rsskey |= rsshash[(j * 4) + 1] << 8;
2805 rsskey |= rsshash[(j * 4) + 2] << 16;
2806 rsskey |= rsshash[(j * 4) + 3] << 24;
2807 array_wr32(E1000_RSSRK(0), j, rsskey);
2808 }
2809
a99955fc 2810 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2811
2812 if (adapter->vfs_allocated_count) {
2813 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2814 switch (hw->mac.type) {
d2ba2ed8 2815 case e1000_i350:
55cac248
AD		 2816 case e1000_82580:
2817 num_rx_queues = 1;
2818 shift = 0;
2819 break;
06cf2666
AD		 2820 case e1000_82576:
2821 shift = 3;
2822 num_rx_queues = 2;
2823 break;
2824 case e1000_82575:
2825 shift = 2;
2826 shift2 = 6;
2827 default:
2828 break;
2829 }
2830 } else {
2831 if (hw->mac.type == e1000_82575)
2832 shift = 6;
2833 }
2834
2835 for (j = 0; j < (32 * 4); j++) {
2836 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2837 if (shift2)
2838 reta.bytes[j & 3] |= num_rx_queues << shift2;
2839 if ((j & 3) == 3)
2840 wr32(E1000_RETA(j >> 2), reta.dword);
2841 }
2842
2843 /*
2844 * Disable raw packet checksumming so that RSS hash is placed in
2845 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2846 * offloads as they are enabled by default
2847 */
2848 rxcsum = rd32(E1000_RXCSUM);
2849 rxcsum |= E1000_RXCSUM_PCSD;
2850
2851 if (adapter->hw.mac.type >= e1000_82576)
2852 /* Enable Receive Checksum Offload for SCTP */
2853 rxcsum |= E1000_RXCSUM_CRCOFL;
2854
2855 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2856 wr32(E1000_RXCSUM, rxcsum);
2857
2858 /* If VMDq is enabled then we set the appropriate mode for that, else
2859 * we default to RSS so that an RSS hash is calculated per packet even
2860 * if we are only using one queue */
2861 if (adapter->vfs_allocated_count) {
2862 if (hw->mac.type > e1000_82575) {
2863 /* Set the default pool for the PF's first queue */
2864 u32 vtctl = rd32(E1000_VT_CTL);
2865 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2866 E1000_VT_CTL_DISABLE_DEF_POOL);
2867 vtctl |= adapter->vfs_allocated_count <<
2868 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2869 wr32(E1000_VT_CTL, vtctl);
2870 }
a99955fc 2871 if (adapter->rss_queues > 1)
06cf2666
AD		 2872 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2873 else
2874 mrqc = E1000_MRQC_ENABLE_VMDQ;
2875 } else {
2876 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2877 }
2878 igb_vmm_control(adapter);
2879
4478a9cd
AD		 2880 /*
2881 * Generate RSS hash based on TCP port numbers and/or
2882 * IPv4/v6 src and dst addresses since UDP cannot be
2883 * hashed reliably due to IP fragmentation
2884 */
2885 mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
2886 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2887 E1000_MRQC_RSS_FIELD_IPV6 |
2888 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2889 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
06cf2666
AD		 2890
2891 wr32(E1000_MRQC, mrqc);
2892}
2893
9d5c8243
AK		 2894/**
2895 * igb_setup_rctl - configure the receive control registers
2896 * @adapter: Board private structure
2897 **/
d7ee5b3a 2898void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2899{
2900 struct e1000_hw *hw = &adapter->hw;
2901 u32 rctl;
9d5c8243
AK		 2902
2903 rctl = rd32(E1000_RCTL);
2904
2905 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2906 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2907
69d728ba 2908 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2909 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2910
87cb7e8c
AK		 2911 /*
2912 * enable stripping of CRC. It's unlikely this will break BMC
2913 * redirection as it did with e1000. Newer features require
2914 * that the HW strips the CRC.
73cd78f1 2915 */
87cb7e8c 2916 rctl |= E1000_RCTL_SECRC;
9d5c8243 2917
559e9c49 2918 /* disable store bad packets and clear size bits. */
ec54d7d6 2919 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2920
6ec43fe6
AD		 2921 /* enable LPE to prevent packets larger than max_frame_size */
2922 rctl |= E1000_RCTL_LPE;
9d5c8243 2923
952f72a8
AD		 2924 /* disable queue 0 to prevent tail write w/o re-config */
2925 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2926
e1739522
AD		 2927 /* Attention!!! For SR-IOV PF driver operations you must enable
2928 * queue drop for all VF and PF queues to prevent head of line blocking
2929 * if an un-trusted VF does not provide descriptors to hardware.
2930 */
2931 if (adapter->vfs_allocated_count) {
e1739522
AD		 2932 /* set all queue drop enable bits */
2933 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 2934 }
2935
9d5c8243
AK		 2936 wr32(E1000_RCTL, rctl);
2937}
2938
7d5753f0
AD		 2939static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2940 int vfn)
2941{
2942 struct e1000_hw *hw = &adapter->hw;
2943 u32 vmolr;
2944
2945 /* if it isn't the PF check to see if VFs are enabled and
2946 * increase the size to support vlan tags */
2947 if (vfn < adapter->vfs_allocated_count &&
2948 adapter->vf_data[vfn].vlans_enabled)
2949 size += VLAN_TAG_SIZE;
2950
2951 vmolr = rd32(E1000_VMOLR(vfn));
2952 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2953 vmolr |= size | E1000_VMOLR_LPE;
2954 wr32(E1000_VMOLR(vfn), vmolr);
2955
2956 return 0;
2957}
2958
e1739522
AD		 2959/**
2960 * igb_rlpml_set - set maximum receive packet size
2961 * @adapter: board private structure
2962 *
2963 * Configure maximum receivable packet size.
2964 **/
2965static void igb_rlpml_set(struct igb_adapter *adapter)
2966{
153285f9 2967 u32 max_frame_size = adapter->max_frame_size;
e1739522
AD		 2968 struct e1000_hw *hw = &adapter->hw;
2969 u16 pf_id = adapter->vfs_allocated_count;
2970
e1739522
AD		 2971 if (pf_id) {
2972 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
153285f9
AD		 2973 /*
2974 * If we're in VMDQ or SR-IOV mode, then set global RLPML
2975 * to our max jumbo frame size, in case we need to enable
2976 * jumbo frames on one of the rings later.
2977 * This will not pass over-length frames into the default
2978 * queue because it's gated by the VMOLR.RLPML.
2979 */
7d5753f0 2980 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 2981 }
2982
2983 wr32(E1000_RLPML, max_frame_size);
2984}
2985
8151d294
WM		 2986static inline void igb_set_vmolr(struct igb_adapter *adapter,
2987 int vfn, bool aupe)
7d5753f0
AD		 2988{
2989 struct e1000_hw *hw = &adapter->hw;
2990 u32 vmolr;
2991
2992 /*
2993 * This register exists only on 82576 and newer so if we are older then
2994 * we should exit and do nothing
2995 */
2996 if (hw->mac.type < e1000_82576)
2997 return;
2998
2999 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 3000 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
3001 if (aupe)
3002 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
3003 else
3004 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 3005
3006 /* clear all bits that might not be set */
3007 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
3008
a99955fc 3009 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 3010 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3011 /*
3012 * for VMDq only allow the VFs and pool 0 to accept broadcast and
3013 * multicast packets
3014 */
3015 if (vfn <= adapter->vfs_allocated_count)
3016 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
3017
3018 wr32(E1000_VMOLR(vfn), vmolr);
3019}
3020
85b430b4
AD		 3021/**
3022 * igb_configure_rx_ring - Configure a receive ring after Reset
3023 * @adapter: board private structure
3024 * @ring: receive ring to be configured
3025 *
3026 * Configure the Rx unit of the MAC after a reset.
3027 **/
d7ee5b3a
AD		 3028void igb_configure_rx_ring(struct igb_adapter *adapter,
3029 struct igb_ring *ring)
85b430b4
AD		 3030{
3031 struct e1000_hw *hw = &adapter->hw;
3032 u64 rdba = ring->dma;
3033 int reg_idx = ring->reg_idx;
a74420e0 3034 u32 srrctl = 0, rxdctl = 0;
85b430b4
AD		 3035
3036 /* disable the queue */
a74420e0 3037 wr32(E1000_RXDCTL(reg_idx), 0);
85b430b4
AD		 3038
3039 /* Set DMA base address registers */
3040 wr32(E1000_RDBAL(reg_idx),
3041 rdba & 0x00000000ffffffffULL);
3042 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
3043 wr32(E1000_RDLEN(reg_idx),
3044 ring->count * sizeof(union e1000_adv_rx_desc));
3045
3046 /* initialize head and tail */
fce99e34 3047 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
a74420e0 3048 wr32(E1000_RDH(reg_idx), 0);
fce99e34 3049 writel(0, ring->tail);
85b430b4 3050
952f72a8 3051 /* set descriptor configuration */
44390ca6 3052 srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
952f72a8 3053#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
44390ca6 3054 srrctl |= IGB_RXBUFFER_16384 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
952f72a8 3055#else
44390ca6 3056 srrctl |= (PAGE_SIZE / 2) >> E1000_SRRCTL_BSIZEPKT_SHIFT;
952f72a8 3057#endif
44390ca6 3058 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
757b77e2
NN		 3059 if (hw->mac.type == e1000_82580)
3060 srrctl |= E1000_SRRCTL_TIMESTAMP;
e6bdb6fe
NN		 3061 /* Only set Drop Enable if we are supporting multiple queues */
3062 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
3063 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 3064
3065 wr32(E1000_SRRCTL(reg_idx), srrctl);
3066
7d5753f0 3067 /* set filtering for VMDQ pools */
8151d294 3068 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 3069
85b430b4
AD		 3070 rxdctl |= IGB_RX_PTHRESH;
3071 rxdctl |= IGB_RX_HTHRESH << 8;
3072 rxdctl |= IGB_RX_WTHRESH << 16;
a74420e0
AD		 3073
3074 /* enable receive descriptor fetching */
3075 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
85b430b4
AD		 3076 wr32(E1000_RXDCTL(reg_idx), rxdctl);
3077}
3078
9d5c8243
AK		 3079/**
3080 * igb_configure_rx - Configure receive Unit after Reset
3081 * @adapter: board private structure
3082 *
3083 * Configure the Rx unit of the MAC after a reset.
3084 **/
3085static void igb_configure_rx(struct igb_adapter *adapter)
3086{
9107584e 3087 int i;
9d5c8243 3088
68d480c4
AD		 3089 /* set UTA to appropriate mode */
3090 igb_set_uta(adapter);
3091
26ad9178
AD		 3092 /* set the correct pool for the PF default MAC address in entry 0 */
3093 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
3094 adapter->vfs_allocated_count);
3095
06cf2666
AD		 3096 /* Setup the HW Rx Head and Tail Descriptor Pointers and
3097 * the Base and Length of the Rx Descriptor Ring */
3098 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3099 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 3100}
3101
3102/**
3103 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 3104 * @tx_ring: Tx descriptor ring for a specific queue
3105 *
3106 * Free all transmit software resources
3107 **/
68fd9910 3108void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 3109{
3b644cf6 3110 igb_clean_tx_ring(tx_ring);
9d5c8243 3111
06034649
AD		 3112 vfree(tx_ring->tx_buffer_info);
3113 tx_ring->tx_buffer_info = NULL;
9d5c8243 3114
439705e1
AD		 3115 /* if not set, then don't free */
3116 if (!tx_ring->desc)
3117 return;
3118
59d71989
AD		 3119 dma_free_coherent(tx_ring->dev, tx_ring->size,
3120 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 3121
3122 tx_ring->desc = NULL;
3123}
3124
3125/**
3126 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3127 * @adapter: board private structure
3128 *
3129 * Free all transmit software resources
3130 **/
3131static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3132{
3133 int i;
3134
3135 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3136 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 3137}
3138
b1a436c3 3139void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
06034649 3140 struct igb_tx_buffer *buffer_info)
9d5c8243 3141{
6366ad33
AD		 3142 if (buffer_info->dma) {
3143 if (buffer_info->mapped_as_page)
59d71989 3144 dma_unmap_page(tx_ring->dev,
6366ad33
AD		 3145 buffer_info->dma,
3146 buffer_info->length,
59d71989 3147 DMA_TO_DEVICE);
6366ad33 3148 else
59d71989 3149 dma_unmap_single(tx_ring->dev,
6366ad33
AD		 3150 buffer_info->dma,
3151 buffer_info->length,
59d71989 3152 DMA_TO_DEVICE);
6366ad33
AD		 3153 buffer_info->dma = 0;
3154 }
9d5c8243
AK		 3155 if (buffer_info->skb) {
3156 dev_kfree_skb_any(buffer_info->skb);
3157 buffer_info->skb = NULL;
3158 }
3159 buffer_info->time_stamp = 0;
6366ad33
AD		 3160 buffer_info->length = 0;
3161 buffer_info->next_to_watch = 0;
3162 buffer_info->mapped_as_page = false;
9d5c8243
AK		 3163}
3164
3165/**
3166 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 3167 * @tx_ring: ring to be cleaned
3168 **/
3b644cf6 3169static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243 3170{
06034649 3171 struct igb_tx_buffer *buffer_info;
9d5c8243
AK		 3172 unsigned long size;
3173 unsigned int i;
3174
06034649 3175 if (!tx_ring->tx_buffer_info)
9d5c8243
AK		 3176 return;
3177 /* Free all the Tx ring sk_buffs */
3178
3179 for (i = 0; i < tx_ring->count; i++) {
06034649 3180 buffer_info = &tx_ring->tx_buffer_info[i];
80785298 3181 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 3182 }
3183
06034649
AD		 3184 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
3185 memset(tx_ring->tx_buffer_info, 0, size);
9d5c8243
AK		 3186
3187 /* Zero out the descriptor ring */
9d5c8243
AK		 3188 memset(tx_ring->desc, 0, tx_ring->size);
3189
3190 tx_ring->next_to_use = 0;
3191 tx_ring->next_to_clean = 0;
9d5c8243
AK		 3192}
3193
3194/**
3195 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3196 * @adapter: board private structure
3197 **/
3198static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3199{
3200 int i;
3201
3202 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3203 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 3204}
3205
3206/**
3207 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 3208 * @rx_ring: ring to clean the resources from
3209 *
3210 * Free all receive software resources
3211 **/
68fd9910 3212void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 3213{
3b644cf6 3214 igb_clean_rx_ring(rx_ring);
9d5c8243 3215
06034649
AD		 3216 vfree(rx_ring->rx_buffer_info);
3217 rx_ring->rx_buffer_info = NULL;
9d5c8243 3218
439705e1
AD		 3219 /* if not set, then don't free */
3220 if (!rx_ring->desc)
3221 return;
3222
59d71989
AD		 3223 dma_free_coherent(rx_ring->dev, rx_ring->size,
3224 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 3225
3226 rx_ring->desc = NULL;
3227}
3228
3229/**
3230 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3231 * @adapter: board private structure
3232 *
3233 * Free all receive software resources
3234 **/
3235static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3236{
3237 int i;
3238
3239 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3240 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 3241}
3242
3243/**
3244 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 3245 * @rx_ring: ring to free buffers from
3246 **/
3b644cf6 3247static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243 3248{
9d5c8243 3249 unsigned long size;
c023cd88 3250 u16 i;
9d5c8243 3251
06034649 3252 if (!rx_ring->rx_buffer_info)
9d5c8243 3253 return;
439705e1 3254
9d5c8243
AK		 3255 /* Free all the Rx ring sk_buffs */
3256 for (i = 0; i < rx_ring->count; i++) {
06034649 3257 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
9d5c8243 3258 if (buffer_info->dma) {
59d71989 3259 dma_unmap_single(rx_ring->dev,
80785298 3260 buffer_info->dma,
44390ca6 3261 IGB_RX_HDR_LEN,
59d71989 3262 DMA_FROM_DEVICE);
9d5c8243
AK		 3263 buffer_info->dma = 0;
3264 }
3265
3266 if (buffer_info->skb) {
3267 dev_kfree_skb(buffer_info->skb);
3268 buffer_info->skb = NULL;
3269 }
6ec43fe6 3270 if (buffer_info->page_dma) {
59d71989 3271 dma_unmap_page(rx_ring->dev,
80785298 3272 buffer_info->page_dma,
6ec43fe6 3273 PAGE_SIZE / 2,
59d71989 3274 DMA_FROM_DEVICE);
6ec43fe6
AD		 3275 buffer_info->page_dma = 0;
3276 }
9d5c8243 3277 if (buffer_info->page) {
9d5c8243
AK		 3278 put_page(buffer_info->page);
3279 buffer_info->page = NULL;
bf36c1a0 3280 buffer_info->page_offset = 0;
9d5c8243
AK		 3281 }
3282 }
3283
06034649
AD		 3284 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3285 memset(rx_ring->rx_buffer_info, 0, size);
9d5c8243
AK		 3286
3287 /* Zero out the descriptor ring */
3288 memset(rx_ring->desc, 0, rx_ring->size);
3289
3290 rx_ring->next_to_clean = 0;
3291 rx_ring->next_to_use = 0;
9d5c8243
AK		 3292}
3293
3294/**
3295 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3296 * @adapter: board private structure
3297 **/
3298static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3299{
3300 int i;
3301
3302 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3303 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 3304}
3305
3306/**
3307 * igb_set_mac - Change the Ethernet Address of the NIC
3308 * @netdev: network interface device structure
3309 * @p: pointer to an address structure
3310 *
3311 * Returns 0 on success, negative on failure
3312 **/
3313static int igb_set_mac(struct net_device *netdev, void *p)
3314{
3315 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 3316 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 3317 struct sockaddr *addr = p;
3318
3319 if (!is_valid_ether_addr(addr->sa_data))
3320 return -EADDRNOTAVAIL;
3321
3322 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 3323 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 3324
26ad9178
AD		 3325 /* set the correct pool for the new PF MAC address in entry 0 */
3326 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3327 adapter->vfs_allocated_count);
e1739522 3328
9d5c8243
AK		 3329 return 0;
3330}
3331
3332/**
68d480c4 3333 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 3334 * @netdev: network interface device structure
3335 *
68d480c4
AD		 3336 * Writes multicast address list to the MTA hash table.
3337 * Returns: -ENOMEM on failure
3338 * 0 on no addresses written
3339 * X on writing X addresses to MTA
9d5c8243 3340 **/
68d480c4 3341static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 3342{
3343 struct igb_adapter *adapter = netdev_priv(netdev);
3344 struct e1000_hw *hw = &adapter->hw;
22bedad3 3345 struct netdev_hw_addr *ha;
68d480c4 3346 u8 *mta_list;
9d5c8243
AK		 3347 int i;
3348
4cd24eaf 3349 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 3350 /* nothing to program, so clear mc list */
3351 igb_update_mc_addr_list(hw, NULL, 0);
3352 igb_restore_vf_multicasts(adapter);
3353 return 0;
3354 }
9d5c8243 3355
4cd24eaf 3356 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 3357 if (!mta_list)
3358 return -ENOMEM;
ff41f8dc 3359
68d480c4 3360 /* The shared function expects a packed array of only addresses. */
48e2f183 3361 i = 0;
22bedad3
JP		 3362 netdev_for_each_mc_addr(ha, netdev)
3363 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
68d480c4 3364
68d480c4
AD		 3365 igb_update_mc_addr_list(hw, mta_list, i);
3366 kfree(mta_list);
3367
4cd24eaf 3368 return netdev_mc_count(netdev);
68d480c4
AD		 3369}
3370
3371/**
3372 * igb_write_uc_addr_list - write unicast addresses to RAR table
3373 * @netdev: network interface device structure
3374 *
3375 * Writes unicast address list to the RAR table.
3376 * Returns: -ENOMEM on failure/insufficient address space
3377 * 0 on no addresses written
3378 * X on writing X addresses to the RAR table
3379 **/
3380static int igb_write_uc_addr_list(struct net_device *netdev)
3381{
3382 struct igb_adapter *adapter = netdev_priv(netdev);
3383 struct e1000_hw *hw = &adapter->hw;
3384 unsigned int vfn = adapter->vfs_allocated_count;
3385 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3386 int count = 0;
3387
3388 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 3389 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 3390 return -ENOMEM;
9d5c8243 3391
32e7bfc4 3392 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 3393 struct netdev_hw_addr *ha;
32e7bfc4
JP		 3394
3395 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 3396 if (!rar_entries)
3397 break;
26ad9178
AD		 3398 igb_rar_set_qsel(adapter, ha->addr,
3399 rar_entries--,
68d480c4
AD		 3400 vfn);
3401 count++;
ff41f8dc
AD		 3402 }
3403 }
3404 /* write the addresses in reverse order to avoid write combining */
3405 for (; rar_entries > 0 ; rar_entries--) {
3406 wr32(E1000_RAH(rar_entries), 0);
3407 wr32(E1000_RAL(rar_entries), 0);
3408 }
3409 wrfl();
3410
68d480c4
AD		 3411 return count;
3412}
3413
3414/**
3415 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3416 * @netdev: network interface device structure
3417 *
3418 * The set_rx_mode entry point is called whenever the unicast or multicast
3419 * address lists or the network interface flags are updated. This routine is
3420 * responsible for configuring the hardware for proper unicast, multicast,
3421 * promiscuous mode, and all-multi behavior.
3422 **/
3423static void igb_set_rx_mode(struct net_device *netdev)
3424{
3425 struct igb_adapter *adapter = netdev_priv(netdev);
3426 struct e1000_hw *hw = &adapter->hw;
3427 unsigned int vfn = adapter->vfs_allocated_count;
3428 u32 rctl, vmolr = 0;
3429 int count;
3430
3431 /* Check for Promiscuous and All Multicast modes */
3432 rctl = rd32(E1000_RCTL);
3433
3434 /* clear the effected bits */
3435 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3436
3437 if (netdev->flags & IFF_PROMISC) {
3438 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3439 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3440 } else {
3441 if (netdev->flags & IFF_ALLMULTI) {
3442 rctl |= E1000_RCTL_MPE;
3443 vmolr |= E1000_VMOLR_MPME;
3444 } else {
3445 /*
3446 * Write addresses to the MTA, if the attempt fails
25985edc 3447 * then we should just turn on promiscuous mode so
68d480c4
AD		 3448 * that we can at least receive multicast traffic
3449 */
3450 count = igb_write_mc_addr_list(netdev);
3451 if (count < 0) {
3452 rctl |= E1000_RCTL_MPE;
3453 vmolr |= E1000_VMOLR_MPME;
3454 } else if (count) {
3455 vmolr |= E1000_VMOLR_ROMPE;
3456 }
3457 }
3458 /*
3459 * Write addresses to available RAR registers, if there is not
3460 * sufficient space to store all the addresses then enable
25985edc 3461 * unicast promiscuous mode
68d480c4
AD		 3462 */
3463 count = igb_write_uc_addr_list(netdev);
3464 if (count < 0) {
3465 rctl |= E1000_RCTL_UPE;
3466 vmolr |= E1000_VMOLR_ROPE;
3467 }
3468 rctl |= E1000_RCTL_VFE;
28fc06f5 3469 }
68d480c4 3470 wr32(E1000_RCTL, rctl);
28fc06f5 3471
68d480c4
AD		 3472 /*
3473 * In order to support SR-IOV and eventually VMDq it is necessary to set
3474 * the VMOLR to enable the appropriate modes. Without this workaround
3475 * we will have issues with VLAN tag stripping not being done for frames
3476 * that are only arriving because we are the default pool
3477 */
3478 if (hw->mac.type < e1000_82576)
28fc06f5 3479 return;
9d5c8243 3480
68d480c4
AD		 3481 vmolr |= rd32(E1000_VMOLR(vfn)) &
3482 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3483 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3484 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3485}
3486
13800469
GR		 3487static void igb_check_wvbr(struct igb_adapter *adapter)
3488{
3489 struct e1000_hw *hw = &adapter->hw;
3490 u32 wvbr = 0;
3491
3492 switch (hw->mac.type) {
3493 case e1000_82576:
3494 case e1000_i350:
3495 if (!(wvbr = rd32(E1000_WVBR)))
3496 return;
3497 break;
3498 default:
3499 break;
3500 }
3501
3502 adapter->wvbr |= wvbr;
3503}
3504
3505#define IGB_STAGGERED_QUEUE_OFFSET 8
3506
3507static void igb_spoof_check(struct igb_adapter *adapter)
3508{
3509 int j;
3510
3511 if (!adapter->wvbr)
3512 return;
3513
3514 for(j = 0; j < adapter->vfs_allocated_count; j++) {
3515 if (adapter->wvbr & (1 << j) ||
3516 adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3517 dev_warn(&adapter->pdev->dev,
3518 "Spoof event(s) detected on VF %d\n", j);
3519 adapter->wvbr &=
3520 ~((1 << j) |
3521 (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3522 }
3523 }
3524}
3525
9d5c8243
AK		 3526/* Need to wait a few seconds after link up to get diagnostic information from
3527 * the phy */
3528static void igb_update_phy_info(unsigned long data)
3529{
3530 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3531 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3532}
3533
4d6b725e
AD		 3534/**
3535 * igb_has_link - check shared code for link and determine up/down
3536 * @adapter: pointer to driver private info
3537 **/
3145535a 3538bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3539{
3540 struct e1000_hw *hw = &adapter->hw;
3541 bool link_active = false;
3542 s32 ret_val = 0;
3543
3544 /* get_link_status is set on LSC (link status) interrupt or
3545 * rx sequence error interrupt. get_link_status will stay
3546 * false until the e1000_check_for_link establishes link
3547 * for copper adapters ONLY
3548 */
3549 switch (hw->phy.media_type) {
3550 case e1000_media_type_copper:
3551 if (hw->mac.get_link_status) {
3552 ret_val = hw->mac.ops.check_for_link(hw);
3553 link_active = !hw->mac.get_link_status;
3554 } else {
3555 link_active = true;
3556 }
3557 break;
4d6b725e
AD		 3558 case e1000_media_type_internal_serdes:
3559 ret_val = hw->mac.ops.check_for_link(hw);
3560 link_active = hw->mac.serdes_has_link;
3561 break;
3562 default:
3563 case e1000_media_type_unknown:
3564 break;
3565 }
3566
3567 return link_active;
3568}
3569
563988dc
SA		 3570static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
3571{
3572 bool ret = false;
3573 u32 ctrl_ext, thstat;
3574
3575 /* check for thermal sensor event on i350, copper only */
3576 if (hw->mac.type == e1000_i350) {
3577 thstat = rd32(E1000_THSTAT);
3578 ctrl_ext = rd32(E1000_CTRL_EXT);
3579
3580 if ((hw->phy.media_type == e1000_media_type_copper) &&
3581 !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3582 ret = !!(thstat & event);
3583 }
3584 }
3585
3586 return ret;
3587}
3588
9d5c8243
AK		 3589/**
3590 * igb_watchdog - Timer Call-back
3591 * @data: pointer to adapter cast into an unsigned long
3592 **/
3593static void igb_watchdog(unsigned long data)
3594{
3595 struct igb_adapter *adapter = (struct igb_adapter *)data;
3596 /* Do the rest outside of interrupt context */
3597 schedule_work(&adapter->watchdog_task);
3598}
3599
3600static void igb_watchdog_task(struct work_struct *work)
3601{
3602 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3603 struct igb_adapter,
3604 watchdog_task);
9d5c8243 3605 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3606 struct net_device *netdev = adapter->netdev;
563988dc 3607 u32 link;
7a6ea550 3608 int i;
9d5c8243 3609
4d6b725e 3610 link = igb_has_link(adapter);
9d5c8243
AK		 3611 if (link) {
3612 if (!netif_carrier_ok(netdev)) {
3613 u32 ctrl;
330a6d6a
AD		 3614 hw->mac.ops.get_speed_and_duplex(hw,
3615 &adapter->link_speed,
3616 &adapter->link_duplex);
9d5c8243
AK		 3617
3618 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 3619 /* Links status message must follow this format */
3620 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 3621 "Flow Control: %s\n",
559e9c49
AD		 3622 netdev->name,
3623 adapter->link_speed,
3624 adapter->link_duplex == FULL_DUPLEX ?
9d5c8243 3625 "Full Duplex" : "Half Duplex",
559e9c49
AD		 3626 ((ctrl & E1000_CTRL_TFCE) &&
3627 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3628 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3629 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
9d5c8243 3630
563988dc
SA		 3631 /* check for thermal sensor event */
3632 if (igb_thermal_sensor_event(hw, E1000_THSTAT_LINK_THROTTLE)) {
3633 printk(KERN_INFO "igb: %s The network adapter "
3634 "link speed was downshifted "
3635 "because it overheated.\n",
3636 netdev->name);
7ef5ed1c 3637 }
563988dc 3638
d07f3e37 3639 /* adjust timeout factor according to speed/duplex */
9d5c8243
AK		 3640 adapter->tx_timeout_factor = 1;
3641 switch (adapter->link_speed) {
3642 case SPEED_10:
9d5c8243
AK		 3643 adapter->tx_timeout_factor = 14;
3644 break;
3645 case SPEED_100:
9d5c8243
AK		 3646 /* maybe add some timeout factor ? */
3647 break;
3648 }
3649
3650 netif_carrier_on(netdev);
9d5c8243 3651
4ae196df 3652 igb_ping_all_vfs(adapter);
17dc566c 3653 igb_check_vf_rate_limit(adapter);
4ae196df 3654
4b1a9877 3655 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3656 if (!test_bit(__IGB_DOWN, &adapter->state))
3657 mod_timer(&adapter->phy_info_timer,
3658 round_jiffies(jiffies + 2 * HZ));
3659 }
3660 } else {
3661 if (netif_carrier_ok(netdev)) {
3662 adapter->link_speed = 0;
3663 adapter->link_duplex = 0;
563988dc
SA		 3664
3665 /* check for thermal sensor event */
3666 if (igb_thermal_sensor_event(hw, E1000_THSTAT_PWR_DOWN)) {
3667 printk(KERN_ERR "igb: %s The network adapter "
3668 "was stopped because it "
3669 "overheated.\n",
7ef5ed1c 3670 netdev->name);
7ef5ed1c 3671 }
563988dc 3672
527d47c1
AD		 3673 /* Links status message must follow this format */
3674 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3675 netdev->name);
9d5c8243 3676 netif_carrier_off(netdev);
4b1a9877 3677
4ae196df
AD		 3678 igb_ping_all_vfs(adapter);
3679
4b1a9877 3680 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3681 if (!test_bit(__IGB_DOWN, &adapter->state))
3682 mod_timer(&adapter->phy_info_timer,
3683 round_jiffies(jiffies + 2 * HZ));
3684 }
3685 }
3686
12dcd86b
ED		 3687 spin_lock(&adapter->stats64_lock);
3688 igb_update_stats(adapter, &adapter->stats64);
3689 spin_unlock(&adapter->stats64_lock);
9d5c8243 3690
dbabb065 3691 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3692 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3693 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3694 /* We've lost link, so the controller stops DMA,
3695 * but we've got queued Tx work that's never going
3696 * to get done, so reset controller to flush Tx.
3697 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3698 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3699 adapter->tx_timeout_count++;
3700 schedule_work(&adapter->reset_task);
3701 /* return immediately since reset is imminent */
3702 return;
3703 }
9d5c8243 3704 }
9d5c8243 3705
dbabb065
AD		 3706 /* Force detection of hung controller every watchdog period */
3707 tx_ring->detect_tx_hung = true;
3708 }
f7ba205e 3709
9d5c8243 3710 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3711 if (adapter->msix_entries) {
047e0030
AD		 3712 u32 eics = 0;
3713 for (i = 0; i < adapter->num_q_vectors; i++) {
3714 struct igb_q_vector *q_vector = adapter->q_vector[i];
3715 eics |= q_vector->eims_value;
3716 }
7a6ea550
AD		 3717 wr32(E1000_EICS, eics);
3718 } else {
3719 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3720 }
9d5c8243 3721
13800469
GR		 3722 igb_spoof_check(adapter);
3723
9d5c8243
AK		 3724 /* Reset the timer */
3725 if (!test_bit(__IGB_DOWN, &adapter->state))
3726 mod_timer(&adapter->watchdog_timer,
3727 round_jiffies(jiffies + 2 * HZ));
3728}
3729
3730enum latency_range {
3731 lowest_latency = 0,
3732 low_latency = 1,
3733 bulk_latency = 2,
3734 latency_invalid = 255
3735};
3736
6eb5a7f1
AD		 3737/**
3738 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3739 *
3740 * Stores a new ITR value based on strictly on packet size. This
3741 * algorithm is less sophisticated than that used in igb_update_itr,
3742 * due to the difficulty of synchronizing statistics across multiple
eef35c2d 3743 * receive rings. The divisors and thresholds used by this function
6eb5a7f1
AD		 3744 * were determined based on theoretical maximum wire speed and testing
3745 * data, in order to minimize response time while increasing bulk
3746 * throughput.
3747 * This functionality is controlled by the InterruptThrottleRate module
3748 * parameter (see igb_param.c)
3749 * NOTE: This function is called only when operating in a multiqueue
3750 * receive environment.
047e0030 3751 * @q_vector: pointer to q_vector
6eb5a7f1 3752 **/
047e0030 3753static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3754{
047e0030 3755 int new_val = q_vector->itr_val;
6eb5a7f1 3756 int avg_wire_size = 0;
047e0030 3757 struct igb_adapter *adapter = q_vector->adapter;
12dcd86b
ED		 3758 struct igb_ring *ring;
3759 unsigned int packets;
9d5c8243 3760
6eb5a7f1
AD		 3761 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3762 * ints/sec - ITR timer value of 120 ticks.
3763 */
3764 if (adapter->link_speed != SPEED_1000) {
047e0030 3765 new_val = 976;
6eb5a7f1 3766 goto set_itr_val;
9d5c8243 3767 }
047e0030 3768
12dcd86b
ED		 3769 ring = q_vector->rx_ring;
3770 if (ring) {
3771 packets = ACCESS_ONCE(ring->total_packets);
3772
3773 if (packets)
3774 avg_wire_size = ring->total_bytes / packets;
047e0030
AD		 3775 }
3776
12dcd86b
ED		 3777 ring = q_vector->tx_ring;
3778 if (ring) {
3779 packets = ACCESS_ONCE(ring->total_packets);
3780
3781 if (packets)
3782 avg_wire_size = max_t(u32, avg_wire_size,
3783 ring->total_bytes / packets);
047e0030
AD		 3784 }
3785
3786 /* if avg_wire_size isn't set no work was done */
3787 if (!avg_wire_size)
3788 goto clear_counts;
9d5c8243 3789
6eb5a7f1
AD		 3790 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3791 avg_wire_size += 24;
3792
3793 /* Don't starve jumbo frames */
3794 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3795
6eb5a7f1
AD		 3796 /* Give a little boost to mid-size frames */
3797 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3798 new_val = avg_wire_size / 3;
3799 else
3800 new_val = avg_wire_size / 2;
9d5c8243 3801
abe1c363
NN		 3802 /* when in itr mode 3 do not exceed 20K ints/sec */
3803 if (adapter->rx_itr_setting == 3 && new_val < 196)
3804 new_val = 196;
3805
6eb5a7f1 3806set_itr_val:
047e0030
AD		 3807 if (new_val != q_vector->itr_val) {
3808 q_vector->itr_val = new_val;
3809 q_vector->set_itr = 1;
9d5c8243 3810 }
6eb5a7f1 3811clear_counts:
047e0030
AD		 3812 if (q_vector->rx_ring) {
3813 q_vector->rx_ring->total_bytes = 0;
3814 q_vector->rx_ring->total_packets = 0;
3815 }
3816 if (q_vector->tx_ring) {
3817 q_vector->tx_ring->total_bytes = 0;
3818 q_vector->tx_ring->total_packets = 0;
3819 }
9d5c8243
AK		 3820}
3821
3822/**
3823 * igb_update_itr - update the dynamic ITR value based on statistics
3824 * Stores a new ITR value based on packets and byte
3825 * counts during the last interrupt. The advantage of per interrupt
3826 * computation is faster updates and more accurate ITR for the current
3827 * traffic pattern. Constants in this function were computed
3828 * based on theoretical maximum wire speed and thresholds were set based
3829 * on testing data as well as attempting to minimize response time
3830 * while increasing bulk throughput.
3831 * this functionality is controlled by the InterruptThrottleRate module
3832 * parameter (see igb_param.c)
3833 * NOTE: These calculations are only valid when operating in a single-
3834 * queue environment.
3835 * @adapter: pointer to adapter
047e0030 3836 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3837 * @packets: the number of packets during this measurement interval
3838 * @bytes: the number of bytes during this measurement interval
3839 **/
3840static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3841 int packets, int bytes)
3842{
3843 unsigned int retval = itr_setting;
3844
3845 if (packets == 0)
3846 goto update_itr_done;
3847
3848 switch (itr_setting) {
3849 case lowest_latency:
3850 /* handle TSO and jumbo frames */
3851 if (bytes/packets > 8000)
3852 retval = bulk_latency;
3853 else if ((packets < 5) && (bytes > 512))
3854 retval = low_latency;
3855 break;
3856 case low_latency: /* 50 usec aka 20000 ints/s */
3857 if (bytes > 10000) {
3858 /* this if handles the TSO accounting */
3859 if (bytes/packets > 8000) {
3860 retval = bulk_latency;
3861 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3862 retval = bulk_latency;
3863 } else if ((packets > 35)) {
3864 retval = lowest_latency;
3865 }
3866 } else if (bytes/packets > 2000) {
3867 retval = bulk_latency;
3868 } else if (packets <= 2 && bytes < 512) {
3869 retval = lowest_latency;
3870 }
3871 break;
3872 case bulk_latency: /* 250 usec aka 4000 ints/s */
3873 if (bytes > 25000) {
3874 if (packets > 35)
3875 retval = low_latency;
1e5c3d21 3876 } else if (bytes < 1500) {
9d5c8243
AK		 3877 retval = low_latency;
3878 }
3879 break;
3880 }
3881
3882update_itr_done:
3883 return retval;
3884}
3885
6eb5a7f1 3886static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3887{
047e0030 3888 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3889 u16 current_itr;
047e0030 3890 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3891
3892 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3893 if (adapter->link_speed != SPEED_1000) {
3894 current_itr = 0;
3895 new_itr = 4000;
3896 goto set_itr_now;
3897 }
3898
3899 adapter->rx_itr = igb_update_itr(adapter,
3900 adapter->rx_itr,
3025a446
AD		 3901 q_vector->rx_ring->total_packets,
3902 q_vector->rx_ring->total_bytes);
9d5c8243 3903
047e0030
AD		 3904 adapter->tx_itr = igb_update_itr(adapter,
3905 adapter->tx_itr,
3025a446
AD		 3906 q_vector->tx_ring->total_packets,
3907 q_vector->tx_ring->total_bytes);
047e0030 3908 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3909
6eb5a7f1 3910 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4fc82adf 3911 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3912 current_itr = low_latency;
3913
9d5c8243
AK		 3914 switch (current_itr) {
3915 /* counts and packets in update_itr are dependent on these numbers */
3916 case lowest_latency:
78b1f607 3917 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3918 break;
3919 case low_latency:
78b1f607 3920 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3921 break;
3922 case bulk_latency:
78b1f607 3923 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3924 break;
3925 default:
3926 break;
3927 }
3928
3929set_itr_now:
3025a446
AD		 3930 q_vector->rx_ring->total_bytes = 0;
3931 q_vector->rx_ring->total_packets = 0;
3932 q_vector->tx_ring->total_bytes = 0;
3933 q_vector->tx_ring->total_packets = 0;
6eb5a7f1 3934
047e0030 3935 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3936 /* this attempts to bias the interrupt rate towards Bulk
3937 * by adding intermediate steps when interrupt rate is
3938 * increasing */
047e0030
AD		 3939 new_itr = new_itr > q_vector->itr_val ?
3940 max((new_itr * q_vector->itr_val) /
3941 (new_itr + (q_vector->itr_val >> 2)),
3942 new_itr) :
9d5c8243
AK		 3943 new_itr;
3944 /* Don't write the value here; it resets the adapter's
3945 * internal timer, and causes us to delay far longer than
3946 * we should between interrupts. Instead, we write the ITR
3947 * value at the beginning of the next interrupt so the timing
3948 * ends up being correct.
3949 */
047e0030
AD		 3950 q_vector->itr_val = new_itr;
3951 q_vector->set_itr = 1;
9d5c8243 3952 }
9d5c8243
AK		 3953}
3954
9d5c8243
AK		 3955#define IGB_TX_FLAGS_CSUM 0x00000001
3956#define IGB_TX_FLAGS_VLAN 0x00000002
3957#define IGB_TX_FLAGS_TSO 0x00000004
3958#define IGB_TX_FLAGS_IPV4 0x00000008
cdfd01fc
AD		 3959#define IGB_TX_FLAGS_TSTAMP 0x00000010
3960#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3961#define IGB_TX_FLAGS_VLAN_SHIFT 16
9d5c8243 3962
cd392f5c
AD		 3963static inline int igb_tso(struct igb_ring *tx_ring,
3964 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
9d5c8243
AK		 3965{
3966 struct e1000_adv_tx_context_desc *context_desc;
3967 unsigned int i;
3968 int err;
06034649 3969 struct igb_tx_buffer *buffer_info;
9d5c8243 3970 u32 info = 0, tu_cmd = 0;
91d4ee33
NN		 3971 u32 mss_l4len_idx;
3972 u8 l4len;
9d5c8243
AK		 3973
3974 if (skb_header_cloned(skb)) {
3975 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3976 if (err)
3977 return err;
3978 }
3979
3980 l4len = tcp_hdrlen(skb);
3981 *hdr_len += l4len;
3982
3983 if (skb->protocol == htons(ETH_P_IP)) {
3984 struct iphdr *iph = ip_hdr(skb);
3985 iph->tot_len = 0;
3986 iph->check = 0;
3987 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3988 iph->daddr, 0,
3989 IPPROTO_TCP,
3990 0);
8e1e8a47 3991 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 3992 ipv6_hdr(skb)->payload_len = 0;
3993 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3994 &ipv6_hdr(skb)->daddr,
3995 0, IPPROTO_TCP, 0);
3996 }
3997
3998 i = tx_ring->next_to_use;
3999
06034649 4000 buffer_info = &tx_ring->tx_buffer_info[i];
60136906 4001 context_desc = IGB_TX_CTXTDESC(tx_ring, i);
9d5c8243
AK		 4002 /* VLAN MACLEN IPLEN */
4003 if (tx_flags & IGB_TX_FLAGS_VLAN)
4004 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
4005 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
4006 *hdr_len += skb_network_offset(skb);
4007 info |= skb_network_header_len(skb);
4008 *hdr_len += skb_network_header_len(skb);
4009 context_desc->vlan_macip_lens = cpu_to_le32(info);
4010
4011 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4012 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
4013
4014 if (skb->protocol == htons(ETH_P_IP))
4015 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
4016 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4017
4018 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
4019
4020 /* MSS L4LEN IDX */
4021 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
4022 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
4023
73cd78f1 4024 /* For 82575, context index must be unique per ring. */
85ad76b2
AD		 4025 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
4026 mss_l4len_idx |= tx_ring->reg_idx << 4;
9d5c8243
AK		 4027
4028 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
4029 context_desc->seqnum_seed = 0;
4030
4031 buffer_info->time_stamp = jiffies;
0e014cb1 4032 buffer_info->next_to_watch = i;
9d5c8243
AK		 4033 buffer_info->dma = 0;
4034 i++;
4035 if (i == tx_ring->count)
4036 i = 0;
4037
4038 tx_ring->next_to_use = i;
4039
4040 return true;
4041}
4042
cd392f5c
AD		 4043static inline bool igb_tx_csum(struct igb_ring *tx_ring,
4044 struct sk_buff *skb, u32 tx_flags)
9d5c8243
AK		 4045{
4046 struct e1000_adv_tx_context_desc *context_desc;
59d71989 4047 struct device *dev = tx_ring->dev;
06034649 4048 struct igb_tx_buffer *buffer_info;
9d5c8243 4049 u32 info = 0, tu_cmd = 0;
80785298 4050 unsigned int i;
9d5c8243
AK		 4051
4052 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
4053 (tx_flags & IGB_TX_FLAGS_VLAN)) {
4054 i = tx_ring->next_to_use;
06034649 4055 buffer_info = &tx_ring->tx_buffer_info[i];
60136906 4056 context_desc = IGB_TX_CTXTDESC(tx_ring, i);
9d5c8243
AK		 4057
4058 if (tx_flags & IGB_TX_FLAGS_VLAN)
4059 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
cdfd01fc 4060
9d5c8243
AK		 4061 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
4062 if (skb->ip_summed == CHECKSUM_PARTIAL)
4063 info |= skb_network_header_len(skb);
4064
4065 context_desc->vlan_macip_lens = cpu_to_le32(info);
4066
4067 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
4068
4069 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fa4a7ef3
AJ		 4070 __be16 protocol;
4071
4072 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
4073 const struct vlan_ethhdr *vhdr =
4074 (const struct vlan_ethhdr*)skb->data;
4075
4076 protocol = vhdr->h_vlan_encapsulated_proto;
4077 } else {
4078 protocol = skb->protocol;
4079 }
4080
4081 switch (protocol) {
09640e63 4082 case cpu_to_be16(ETH_P_IP):
9d5c8243 4083 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
44b0cda3
MW		 4084 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
4085 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 4086 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
4087 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3 4088 break;
09640e63 4089 case cpu_to_be16(ETH_P_IPV6):
44b0cda3
MW		 4090 /* XXX what about other V6 headers?? */
4091 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
4092 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 4093 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
4094 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3
MW		 4095 break;
4096 default:
4097 if (unlikely(net_ratelimit()))
59d71989 4098 dev_warn(dev,
44b0cda3
MW		 4099 "partial checksum but proto=%x!\n",
4100 skb->protocol);
4101 break;
4102 }
9d5c8243
AK		 4103 }
4104
4105 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
4106 context_desc->seqnum_seed = 0;
85ad76b2 4107 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
7dfc16fa 4108 context_desc->mss_l4len_idx =
85ad76b2 4109 cpu_to_le32(tx_ring->reg_idx << 4);
9d5c8243
AK		 4110
4111 buffer_info->time_stamp = jiffies;
0e014cb1 4112 buffer_info->next_to_watch = i;
9d5c8243
AK		 4113 buffer_info->dma = 0;
4114
4115 i++;
4116 if (i == tx_ring->count)
4117 i = 0;
4118 tx_ring->next_to_use = i;
4119
4120 return true;
4121 }
9d5c8243
AK		 4122 return false;
4123}
4124
4125#define IGB_MAX_TXD_PWR 16
4126#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
4127
cd392f5c
AD		 4128static inline int igb_tx_map(struct igb_ring *tx_ring, struct sk_buff *skb,
4129 unsigned int first)
9d5c8243 4130{
06034649 4131 struct igb_tx_buffer *buffer_info;
59d71989 4132 struct device *dev = tx_ring->dev;
2873957d 4133 unsigned int hlen = skb_headlen(skb);
9d5c8243
AK		 4134 unsigned int count = 0, i;
4135 unsigned int f;
2873957d 4136 u16 gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
9d5c8243
AK		 4137
4138 i = tx_ring->next_to_use;
4139
06034649 4140 buffer_info = &tx_ring->tx_buffer_info[i];
2873957d
NN		 4141 BUG_ON(hlen >= IGB_MAX_DATA_PER_TXD);
4142 buffer_info->length = hlen;
9d5c8243
AK		 4143 /* set time_stamp *before* dma to help avoid a possible race */
4144 buffer_info->time_stamp = jiffies;
0e014cb1 4145 buffer_info->next_to_watch = i;
2873957d 4146 buffer_info->dma = dma_map_single(dev, skb->data, hlen,
59d71989
AD		 4147 DMA_TO_DEVICE);
4148 if (dma_mapping_error(dev, buffer_info->dma))
6366ad33 4149 goto dma_error;
9d5c8243
AK		 4150
4151 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
2873957d
NN		 4152 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[f];
4153 unsigned int len = frag->size;
9d5c8243 4154
8581145f 4155 count++;
65689fef
AD		 4156 i++;
4157 if (i == tx_ring->count)
4158 i = 0;
4159
06034649 4160 buffer_info = &tx_ring->tx_buffer_info[i];
9d5c8243
AK		 4161 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
4162 buffer_info->length = len;
4163 buffer_info->time_stamp = jiffies;
0e014cb1 4164 buffer_info->next_to_watch = i;
6366ad33 4165 buffer_info->mapped_as_page = true;
877749bf 4166 buffer_info->dma = skb_frag_dma_map(dev, frag, 0, len,
59d71989
AD		 4167 DMA_TO_DEVICE);
4168 if (dma_mapping_error(dev, buffer_info->dma))
6366ad33
AD		 4169 goto dma_error;
4170
9d5c8243
AK		 4171 }
4172
06034649
AD		 4173 buffer_info->skb = skb;
4174 buffer_info->tx_flags = skb_shinfo(skb)->tx_flags;
2873957d 4175 /* multiply data chunks by size of headers */
06034649
AD		 4176 buffer_info->bytecount = ((gso_segs - 1) * hlen) + skb->len;
4177 buffer_info->gso_segs = gso_segs;
4178 tx_ring->tx_buffer_info[first].next_to_watch = i;
9d5c8243 4179
cdfd01fc 4180 return ++count;
6366ad33
AD		 4181
4182dma_error:
59d71989 4183 dev_err(dev, "TX DMA map failed\n");
6366ad33
AD		 4184
4185 /* clear timestamp and dma mappings for failed buffer_info mapping */
4186 buffer_info->dma = 0;
4187 buffer_info->time_stamp = 0;
4188 buffer_info->length = 0;
4189 buffer_info->next_to_watch = 0;
4190 buffer_info->mapped_as_page = false;
6366ad33
AD		 4191
4192 /* clear timestamp and dma mappings for remaining portion of packet */
a77ff709
NN		 4193 while (count--) {
4194 if (i == 0)
4195 i = tx_ring->count;
6366ad33 4196 i--;
06034649 4197 buffer_info = &tx_ring->tx_buffer_info[i];
6366ad33
AD		 4198 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
4199 }
4200
4201 return 0;
9d5c8243
AK		 4202}
4203
cd392f5c
AD		 4204static inline void igb_tx_queue(struct igb_ring *tx_ring,
4205 u32 tx_flags, int count, u32 paylen,
4206 u8 hdr_len)
9d5c8243 4207{
cdfd01fc 4208 union e1000_adv_tx_desc *tx_desc;
06034649 4209 struct igb_tx_buffer *buffer_info;
9d5c8243 4210 u32 olinfo_status = 0, cmd_type_len;
cdfd01fc 4211 unsigned int i = tx_ring->next_to_use;
9d5c8243
AK		 4212
4213 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
4214 E1000_ADVTXD_DCMD_DEXT);
4215
4216 if (tx_flags & IGB_TX_FLAGS_VLAN)
4217 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
4218
33af6bcc
PO		 4219 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4220 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
4221
9d5c8243
AK		 4222 if (tx_flags & IGB_TX_FLAGS_TSO) {
4223 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
4224
4225 /* insert tcp checksum */
4226 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4227
4228 /* insert ip checksum */
4229 if (tx_flags & IGB_TX_FLAGS_IPV4)
4230 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4231
4232 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
4233 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4234 }
4235
85ad76b2
AD		 4236 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
4237 (tx_flags & (IGB_TX_FLAGS_CSUM |
4238 IGB_TX_FLAGS_TSO |
7dfc16fa 4239 IGB_TX_FLAGS_VLAN)))
85ad76b2 4240 olinfo_status |= tx_ring->reg_idx << 4;
9d5c8243
AK		 4241
4242 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
4243
cdfd01fc 4244 do {
06034649 4245 buffer_info = &tx_ring->tx_buffer_info[i];
60136906 4246 tx_desc = IGB_TX_DESC(tx_ring, i);
9d5c8243
AK		 4247 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
4248 tx_desc->read.cmd_type_len =
4249 cpu_to_le32(cmd_type_len | buffer_info->length);
4250 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
cdfd01fc 4251 count--;
9d5c8243
AK		 4252 i++;
4253 if (i == tx_ring->count)
4254 i = 0;
cdfd01fc 4255 } while (count > 0);
9d5c8243 4256
85ad76b2 4257 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
9d5c8243
AK		 4258 /* Force memory writes to complete before letting h/w
4259 * know there are new descriptors to fetch. (Only
4260 * applicable for weak-ordered memory model archs,
4261 * such as IA-64). */
4262 wmb();
4263
4264 tx_ring->next_to_use = i;
fce99e34 4265 writel(i, tx_ring->tail);
9d5c8243
AK		 4266 /* we need this if more than one processor can write to our tail
4267 * at a time, it syncronizes IO on IA64/Altix systems */
4268 mmiowb();
4269}
4270
e694e964 4271static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4272{
e694e964
AD		 4273 struct net_device *netdev = tx_ring->netdev;
4274
661086df 4275 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 4276
9d5c8243
AK		 4277 /* Herbert's original patch had:
4278 * smp_mb__after_netif_stop_queue();
4279 * but since that doesn't exist yet, just open code it. */
4280 smp_mb();
4281
4282 /* We need to check again in a case another CPU has just
4283 * made room available. */
c493ea45 4284 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 4285 return -EBUSY;
4286
4287 /* A reprieve! */
661086df 4288 netif_wake_subqueue(netdev, tx_ring->queue_index);
12dcd86b
ED		 4289
4290 u64_stats_update_begin(&tx_ring->tx_syncp2);
4291 tx_ring->tx_stats.restart_queue2++;
4292 u64_stats_update_end(&tx_ring->tx_syncp2);
4293
9d5c8243
AK		 4294 return 0;
4295}
4296
717ba089 4297static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4298{
c493ea45 4299 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 4300 return 0;
e694e964 4301 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 4302}
4303
cd392f5c
AD		 4304netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
4305 struct igb_ring *tx_ring)
9d5c8243 4306{
cdfd01fc 4307 int tso = 0, count;
91d4ee33
NN		 4308 u32 tx_flags = 0;
4309 u16 first;
4310 u8 hdr_len = 0;
9d5c8243 4311
9d5c8243
AK		 4312 /* need: 1 descriptor per page,
4313 * + 2 desc gap to keep tail from touching head,
4314 * + 1 desc for skb->data,
4315 * + 1 desc for context descriptor,
4316 * otherwise try next time */
e694e964 4317 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 4318 /* this is a hard error */
9d5c8243
AK		 4319 return NETDEV_TX_BUSY;
4320 }
33af6bcc 4321
2244d07b
OH		 4322 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4323 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
33af6bcc 4324 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 4325 }
9d5c8243 4326
eab6d18d 4327 if (vlan_tx_tag_present(skb)) {
9d5c8243
AK		 4328 tx_flags |= IGB_TX_FLAGS_VLAN;
4329 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4330 }
4331
661086df
PWJ		 4332 if (skb->protocol == htons(ETH_P_IP))
4333 tx_flags |= IGB_TX_FLAGS_IPV4;
4334
0e014cb1 4335 first = tx_ring->next_to_use;
85ad76b2 4336 if (skb_is_gso(skb)) {
cd392f5c 4337 tso = igb_tso(tx_ring, skb, tx_flags, &hdr_len);
cdfd01fc 4338
85ad76b2
AD		 4339 if (tso < 0) {
4340 dev_kfree_skb_any(skb);
4341 return NETDEV_TX_OK;
4342 }
9d5c8243
AK		 4343 }
4344
4345 if (tso)
4346 tx_flags |= IGB_TX_FLAGS_TSO;
cd392f5c 4347 else if (igb_tx_csum(tx_ring, skb, tx_flags) &&
bc1cbd34
AD		 4348 (skb->ip_summed == CHECKSUM_PARTIAL))
4349 tx_flags |= IGB_TX_FLAGS_CSUM;
9d5c8243 4350
65689fef 4351 /*
cdfd01fc 4352 * count reflects descriptors mapped, if 0 or less then mapping error
25985edc 4353 * has occurred and we need to rewind the descriptor queue
65689fef 4354 */
cd392f5c 4355 count = igb_tx_map(tx_ring, skb, first);
6366ad33 4356 if (!count) {
65689fef 4357 dev_kfree_skb_any(skb);
06034649 4358 tx_ring->tx_buffer_info[first].time_stamp = 0;
65689fef 4359 tx_ring->next_to_use = first;
85ad76b2 4360 return NETDEV_TX_OK;
65689fef 4361 }
9d5c8243 4362
cd392f5c 4363 igb_tx_queue(tx_ring, tx_flags, count, skb->len, hdr_len);
85ad76b2
AD		 4364
4365 /* Make sure there is space in the ring for the next send. */
e694e964 4366 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 4367
9d5c8243
AK		 4368 return NETDEV_TX_OK;
4369}
4370
1cc3bd87
AD		 4371static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
4372 struct sk_buff *skb)
4373{
4374 unsigned int r_idx = skb->queue_mapping;
4375
4376 if (r_idx >= adapter->num_tx_queues)
4377 r_idx = r_idx % adapter->num_tx_queues;
4378
4379 return adapter->tx_ring[r_idx];
4380}
4381
cd392f5c
AD		 4382static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
4383 struct net_device *netdev)
9d5c8243
AK		 4384{
4385 struct igb_adapter *adapter = netdev_priv(netdev);
b1a436c3
AD		 4386
4387 if (test_bit(__IGB_DOWN, &adapter->state)) {
4388 dev_kfree_skb_any(skb);
4389 return NETDEV_TX_OK;
4390 }
4391
4392 if (skb->len <= 0) {
4393 dev_kfree_skb_any(skb);
4394 return NETDEV_TX_OK;
4395 }
4396
1cc3bd87
AD		 4397 /*
4398 * The minimum packet size with TCTL.PSP set is 17 so pad the skb
4399 * in order to meet this minimum size requirement.
4400 */
4401 if (skb->len < 17) {
4402 if (skb_padto(skb, 17))
4403 return NETDEV_TX_OK;
4404 skb->len = 17;
4405 }
9d5c8243 4406
1cc3bd87 4407 return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
9d5c8243
AK		 4408}
4409
4410/**
4411 * igb_tx_timeout - Respond to a Tx Hang
4412 * @netdev: network interface device structure
4413 **/
4414static void igb_tx_timeout(struct net_device *netdev)
4415{
4416 struct igb_adapter *adapter = netdev_priv(netdev);
4417 struct e1000_hw *hw = &adapter->hw;
4418
4419 /* Do the reset outside of interrupt context */
4420 adapter->tx_timeout_count++;
f7ba205e 4421
55cac248
AD		 4422 if (hw->mac.type == e1000_82580)
4423 hw->dev_spec._82575.global_device_reset = true;
4424
9d5c8243 4425 schedule_work(&adapter->reset_task);
265de409
AD		 4426 wr32(E1000_EICS,
4427 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 4428}
4429
4430static void igb_reset_task(struct work_struct *work)
4431{
4432 struct igb_adapter *adapter;
4433 adapter = container_of(work, struct igb_adapter, reset_task);
4434
c97ec42a
TI		 4435 igb_dump(adapter);
4436 netdev_err(adapter->netdev, "Reset adapter\n");
9d5c8243
AK		 4437 igb_reinit_locked(adapter);
4438}
4439
4440/**
12dcd86b 4441 * igb_get_stats64 - Get System Network Statistics
9d5c8243 4442 * @netdev: network interface device structure
12dcd86b 4443 * @stats: rtnl_link_stats64 pointer
9d5c8243 4444 *
9d5c8243 4445 **/
12dcd86b
ED		 4446static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4447 struct rtnl_link_stats64 *stats)
9d5c8243 4448{
12dcd86b
ED		 4449 struct igb_adapter *adapter = netdev_priv(netdev);
4450
4451 spin_lock(&adapter->stats64_lock);
4452 igb_update_stats(adapter, &adapter->stats64);
4453 memcpy(stats, &adapter->stats64, sizeof(*stats));
4454 spin_unlock(&adapter->stats64_lock);
4455
4456 return stats;
9d5c8243
AK		 4457}
4458
4459/**
4460 * igb_change_mtu - Change the Maximum Transfer Unit
4461 * @netdev: network interface device structure
4462 * @new_mtu: new value for maximum frame size
4463 *
4464 * Returns 0 on success, negative on failure
4465 **/
4466static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4467{
4468 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4469 struct pci_dev *pdev = adapter->pdev;
153285f9 4470 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
9d5c8243 4471
c809d227 4472 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 4473 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 4474 return -EINVAL;
4475 }
4476
153285f9 4477#define MAX_STD_JUMBO_FRAME_SIZE 9238
9d5c8243 4478 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 4479 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 4480 return -EINVAL;
4481 }
4482
4483 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4484 msleep(1);
73cd78f1 4485
9d5c8243
AK		 4486 /* igb_down has a dependency on max_frame_size */
4487 adapter->max_frame_size = max_frame;
559e9c49 4488
4c844851
AD		 4489 if (netif_running(netdev))
4490 igb_down(adapter);
9d5c8243 4491
090b1795 4492 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 4493 netdev->mtu, new_mtu);
4494 netdev->mtu = new_mtu;
4495
4496 if (netif_running(netdev))
4497 igb_up(adapter);
4498 else
4499 igb_reset(adapter);
4500
4501 clear_bit(__IGB_RESETTING, &adapter->state);
4502
4503 return 0;
4504}
4505
4506/**
4507 * igb_update_stats - Update the board statistics counters
4508 * @adapter: board private structure
4509 **/
4510
12dcd86b
ED		 4511void igb_update_stats(struct igb_adapter *adapter,
4512 struct rtnl_link_stats64 *net_stats)
9d5c8243
AK		 4513{
4514 struct e1000_hw *hw = &adapter->hw;
4515 struct pci_dev *pdev = adapter->pdev;
fa3d9a6d 4516 u32 reg, mpc;
9d5c8243 4517 u16 phy_tmp;
3f9c0164
AD		 4518 int i;
4519 u64 bytes, packets;
12dcd86b
ED		 4520 unsigned int start;
4521 u64 _bytes, _packets;
9d5c8243
AK		 4522
4523#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4524
4525 /*
4526 * Prevent stats update while adapter is being reset, or if the pci
4527 * connection is down.
4528 */
4529 if (adapter->link_speed == 0)
4530 return;
4531 if (pci_channel_offline(pdev))
4532 return;
4533
3f9c0164
AD		 4534 bytes = 0;
4535 packets = 0;
4536 for (i = 0; i < adapter->num_rx_queues; i++) {
4537 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446 4538 struct igb_ring *ring = adapter->rx_ring[i];
12dcd86b 4539
3025a446 4540 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 4541 net_stats->rx_fifo_errors += rqdpc_tmp;
12dcd86b
ED		 4542
4543 do {
4544 start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4545 _bytes = ring->rx_stats.bytes;
4546 _packets = ring->rx_stats.packets;
4547 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4548 bytes += _bytes;
4549 packets += _packets;
3f9c0164
AD		 4550 }
4551
128e45eb
AD		 4552 net_stats->rx_bytes = bytes;
4553 net_stats->rx_packets = packets;
3f9c0164
AD		 4554
4555 bytes = 0;
4556 packets = 0;
4557 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 4558 struct igb_ring *ring = adapter->tx_ring[i];
12dcd86b
ED		 4559 do {
4560 start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4561 _bytes = ring->tx_stats.bytes;
4562 _packets = ring->tx_stats.packets;
4563 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4564 bytes += _bytes;
4565 packets += _packets;
3f9c0164 4566 }
128e45eb
AD		 4567 net_stats->tx_bytes = bytes;
4568 net_stats->tx_packets = packets;
3f9c0164
AD		 4569
4570 /* read stats registers */
9d5c8243
AK		 4571 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4572 adapter->stats.gprc += rd32(E1000_GPRC);
4573 adapter->stats.gorc += rd32(E1000_GORCL);
4574 rd32(E1000_GORCH); /* clear GORCL */
4575 adapter->stats.bprc += rd32(E1000_BPRC);
4576 adapter->stats.mprc += rd32(E1000_MPRC);
4577 adapter->stats.roc += rd32(E1000_ROC);
4578
4579 adapter->stats.prc64 += rd32(E1000_PRC64);
4580 adapter->stats.prc127 += rd32(E1000_PRC127);
4581 adapter->stats.prc255 += rd32(E1000_PRC255);
4582 adapter->stats.prc511 += rd32(E1000_PRC511);
4583 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4584 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4585 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4586 adapter->stats.sec += rd32(E1000_SEC);
4587
fa3d9a6d
MW		 4588 mpc = rd32(E1000_MPC);
4589 adapter->stats.mpc += mpc;
4590 net_stats->rx_fifo_errors += mpc;
9d5c8243
AK		 4591 adapter->stats.scc += rd32(E1000_SCC);
4592 adapter->stats.ecol += rd32(E1000_ECOL);
4593 adapter->stats.mcc += rd32(E1000_MCC);
4594 adapter->stats.latecol += rd32(E1000_LATECOL);
4595 adapter->stats.dc += rd32(E1000_DC);
4596 adapter->stats.rlec += rd32(E1000_RLEC);
4597 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4598 adapter->stats.xontxc += rd32(E1000_XONTXC);
4599 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4600 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4601 adapter->stats.fcruc += rd32(E1000_FCRUC);
4602 adapter->stats.gptc += rd32(E1000_GPTC);
4603 adapter->stats.gotc += rd32(E1000_GOTCL);
4604 rd32(E1000_GOTCH); /* clear GOTCL */
fa3d9a6d 4605 adapter->stats.rnbc += rd32(E1000_RNBC);
9d5c8243
AK		 4606 adapter->stats.ruc += rd32(E1000_RUC);
4607 adapter->stats.rfc += rd32(E1000_RFC);
4608 adapter->stats.rjc += rd32(E1000_RJC);
4609 adapter->stats.tor += rd32(E1000_TORH);
4610 adapter->stats.tot += rd32(E1000_TOTH);
4611 adapter->stats.tpr += rd32(E1000_TPR);
4612
4613 adapter->stats.ptc64 += rd32(E1000_PTC64);
4614 adapter->stats.ptc127 += rd32(E1000_PTC127);
4615 adapter->stats.ptc255 += rd32(E1000_PTC255);
4616 adapter->stats.ptc511 += rd32(E1000_PTC511);
4617 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4618 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4619
4620 adapter->stats.mptc += rd32(E1000_MPTC);
4621 adapter->stats.bptc += rd32(E1000_BPTC);
4622
2d0b0f69
NN		 4623 adapter->stats.tpt += rd32(E1000_TPT);
4624 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4625
4626 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4627 /* read internal phy specific stats */
4628 reg = rd32(E1000_CTRL_EXT);
4629 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4630 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4631 adapter->stats.tncrs += rd32(E1000_TNCRS);
4632 }
4633
9d5c8243
AK		 4634 adapter->stats.tsctc += rd32(E1000_TSCTC);
4635 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4636
4637 adapter->stats.iac += rd32(E1000_IAC);
4638 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4639 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4640 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4641 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4642 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4643 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4644 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4645 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4646
4647 /* Fill out the OS statistics structure */
128e45eb
AD		 4648 net_stats->multicast = adapter->stats.mprc;
4649 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4650
4651 /* Rx Errors */
4652
4653 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4654 * our own version based on RUC and ROC */
128e45eb 4655 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4656 adapter->stats.crcerrs + adapter->stats.algnerrc +
4657 adapter->stats.ruc + adapter->stats.roc +
4658 adapter->stats.cexterr;
128e45eb
AD		 4659 net_stats->rx_length_errors = adapter->stats.ruc +
4660 adapter->stats.roc;
4661 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4662 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4663 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4664
4665 /* Tx Errors */
128e45eb
AD		 4666 net_stats->tx_errors = adapter->stats.ecol +
4667 adapter->stats.latecol;
4668 net_stats->tx_aborted_errors = adapter->stats.ecol;
4669 net_stats->tx_window_errors = adapter->stats.latecol;
4670 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4671
4672 /* Tx Dropped needs to be maintained elsewhere */
4673
4674 /* Phy Stats */
4675 if (hw->phy.media_type == e1000_media_type_copper) {
4676 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4677 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4678 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4679 adapter->phy_stats.idle_errors += phy_tmp;
4680 }
4681 }
4682
4683 /* Management Stats */
4684 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4685 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4686 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
0a915b95
CW		 4687
4688 /* OS2BMC Stats */
4689 reg = rd32(E1000_MANC);
4690 if (reg & E1000_MANC_EN_BMC2OS) {
4691 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
4692 adapter->stats.o2bspc += rd32(E1000_O2BSPC);
4693 adapter->stats.b2ospc += rd32(E1000_B2OSPC);
4694 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
4695 }
9d5c8243
AK		 4696}
4697
9d5c8243
AK		 4698static irqreturn_t igb_msix_other(int irq, void *data)
4699{
047e0030 4700 struct igb_adapter *adapter = data;
9d5c8243 4701 struct e1000_hw *hw = &adapter->hw;
844290e5 4702 u32 icr = rd32(E1000_ICR);
844290e5 4703 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4704
7f081d40
AD		 4705 if (icr & E1000_ICR_DRSTA)
4706 schedule_work(&adapter->reset_task);
4707
047e0030 4708 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4709 /* HW is reporting DMA is out of sync */
4710 adapter->stats.doosync++;
13800469
GR		 4711 /* The DMA Out of Sync is also indication of a spoof event
4712 * in IOV mode. Check the Wrong VM Behavior register to
4713 * see if it is really a spoof event. */
4714 igb_check_wvbr(adapter);
dda0e083 4715 }
eebbbdba 4716
4ae196df
AD		 4717 /* Check for a mailbox event */
4718 if (icr & E1000_ICR_VMMB)
4719 igb_msg_task(adapter);
4720
4721 if (icr & E1000_ICR_LSC) {
4722 hw->mac.get_link_status = 1;
4723 /* guard against interrupt when we're going down */
4724 if (!test_bit(__IGB_DOWN, &adapter->state))
4725 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4726 }
4727
25568a53
AD		 4728 if (adapter->vfs_allocated_count)
4729 wr32(E1000_IMS, E1000_IMS_LSC |
4730 E1000_IMS_VMMB |
4731 E1000_IMS_DOUTSYNC);
4732 else
4733 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
844290e5 4734 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4735
4736 return IRQ_HANDLED;
4737}
4738
047e0030 4739static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4740{
26b39276 4741 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4742 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4743
047e0030
AD		 4744 if (!q_vector->set_itr)
4745 return;
73cd78f1 4746
047e0030
AD		 4747 if (!itr_val)
4748 itr_val = 0x4;
661086df 4749
26b39276
AD		 4750 if (adapter->hw.mac.type == e1000_82575)
4751 itr_val |= itr_val << 16;
661086df 4752 else
047e0030 4753 itr_val |= 0x8000000;
661086df 4754
047e0030
AD		 4755 writel(itr_val, q_vector->itr_register);
4756 q_vector->set_itr = 0;
6eb5a7f1
AD		 4757}
4758
047e0030 4759static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4760{
047e0030 4761 struct igb_q_vector *q_vector = data;
9d5c8243 4762
047e0030
AD		 4763 /* Write the ITR value calculated from the previous interrupt. */
4764 igb_write_itr(q_vector);
9d5c8243 4765
047e0030 4766 napi_schedule(&q_vector->napi);
844290e5 4767
047e0030 4768 return IRQ_HANDLED;
fe4506b6
JC		 4769}
4770
421e02f0 4771#ifdef CONFIG_IGB_DCA
047e0030 4772static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4773{
047e0030 4774 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4775 struct e1000_hw *hw = &adapter->hw;
4776 int cpu = get_cpu();
fe4506b6 4777
047e0030
AD		 4778 if (q_vector->cpu == cpu)
4779 goto out_no_update;
4780
4781 if (q_vector->tx_ring) {
4782 int q = q_vector->tx_ring->reg_idx;
4783 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4784 if (hw->mac.type == e1000_82575) {
4785 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4786 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4787 } else {
047e0030
AD		 4788 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4789 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4790 E1000_DCA_TXCTRL_CPUID_SHIFT;
4791 }
4792 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4793 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4794 }
4795 if (q_vector->rx_ring) {
4796 int q = q_vector->rx_ring->reg_idx;
4797 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4798 if (hw->mac.type == e1000_82575) {
2d064c06 4799 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4800 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4801 } else {
4802 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4803 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4804 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4805 }
fe4506b6
JC		 4806 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4807 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4808 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4809 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4810 }
047e0030
AD		 4811 q_vector->cpu = cpu;
4812out_no_update:
fe4506b6
JC		 4813 put_cpu();
4814}
4815
4816static void igb_setup_dca(struct igb_adapter *adapter)
4817{
7e0e99ef 4818 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4819 int i;
4820
7dfc16fa 4821 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4822 return;
4823
7e0e99ef
AD		 4824 /* Always use CB2 mode, difference is masked in the CB driver. */
4825 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4826
047e0030 4827 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4828 adapter->q_vector[i]->cpu = -1;
4829 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4830 }
4831}
4832
4833static int __igb_notify_dca(struct device *dev, void *data)
4834{
4835 struct net_device *netdev = dev_get_drvdata(dev);
4836 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4837 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4838 struct e1000_hw *hw = &adapter->hw;
4839 unsigned long event = *(unsigned long *)data;
4840
4841 switch (event) {
4842 case DCA_PROVIDER_ADD:
4843 /* if already enabled, don't do it again */
7dfc16fa 4844 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4845 break;
fe4506b6 4846 if (dca_add_requester(dev) == 0) {
bbd98fe4 4847 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4848 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4849 igb_setup_dca(adapter);
4850 break;
4851 }
4852 /* Fall Through since DCA is disabled. */
4853 case DCA_PROVIDER_REMOVE:
7dfc16fa 4854 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4855 /* without this a class_device is left
047e0030 4856 * hanging around in the sysfs model */
fe4506b6 4857 dca_remove_requester(dev);
090b1795 4858 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4859 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4860 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4861 }
4862 break;
4863 }
bbd98fe4 4864
fe4506b6 4865 return 0;
9d5c8243
AK		 4866}
4867
fe4506b6
JC		 4868static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4869 void *p)
4870{
4871 int ret_val;
4872
4873 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4874 __igb_notify_dca);
4875
4876 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4877}
421e02f0 4878#endif /* CONFIG_IGB_DCA */
9d5c8243 4879
4ae196df
AD		 4880static void igb_ping_all_vfs(struct igb_adapter *adapter)
4881{
4882 struct e1000_hw *hw = &adapter->hw;
4883 u32 ping;
4884 int i;
4885
4886 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4887 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 4888 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 4889 ping |= E1000_VT_MSGTYPE_CTS;
4890 igb_write_mbx(hw, &ping, 1, i);
4891 }
4892}
4893
7d5753f0
AD		 4894static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4895{
4896 struct e1000_hw *hw = &adapter->hw;
4897 u32 vmolr = rd32(E1000_VMOLR(vf));
4898 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4899
d85b9004 4900 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
7d5753f0
AD		 4901 IGB_VF_FLAG_MULTI_PROMISC);
4902 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4903
4904 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4905 vmolr |= E1000_VMOLR_MPME;
d85b9004 4906 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
7d5753f0
AD		 4907 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4908 } else {
4909 /*
4910 * if we have hashes and we are clearing a multicast promisc
4911 * flag we need to write the hashes to the MTA as this step
4912 * was previously skipped
4913 */
4914 if (vf_data->num_vf_mc_hashes > 30) {
4915 vmolr |= E1000_VMOLR_MPME;
4916 } else if (vf_data->num_vf_mc_hashes) {
4917 int j;
4918 vmolr |= E1000_VMOLR_ROMPE;
4919 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4920 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4921 }
4922 }
4923
4924 wr32(E1000_VMOLR(vf), vmolr);
4925
4926 /* there are flags left unprocessed, likely not supported */
4927 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4928 return -EINVAL;
4929
4930 return 0;
4931
4932}
4933
4ae196df
AD		 4934static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4935 u32 *msgbuf, u32 vf)
4936{
4937 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4938 u16 *hash_list = (u16 *)&msgbuf[1];
4939 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4940 int i;
4941
7d5753f0 4942 /* salt away the number of multicast addresses assigned
4ae196df
AD		 4943 * to this VF for later use to restore when the PF multi cast
4944 * list changes
4945 */
4946 vf_data->num_vf_mc_hashes = n;
4947
7d5753f0
AD		 4948 /* only up to 30 hash values supported */
4949 if (n > 30)
4950 n = 30;
4951
4952 /* store the hashes for later use */
4ae196df 4953 for (i = 0; i < n; i++)
a419aef8 4954 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 4955
4956 /* Flush and reset the mta with the new values */
ff41f8dc 4957 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4958
4959 return 0;
4960}
4961
4962static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4963{
4964 struct e1000_hw *hw = &adapter->hw;
4965 struct vf_data_storage *vf_data;
4966 int i, j;
4967
4968 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 4969 u32 vmolr = rd32(E1000_VMOLR(i));
4970 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4971
4ae196df 4972 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 4973
4974 if ((vf_data->num_vf_mc_hashes > 30) ||
4975 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4976 vmolr |= E1000_VMOLR_MPME;
4977 } else if (vf_data->num_vf_mc_hashes) {
4978 vmolr |= E1000_VMOLR_ROMPE;
4979 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4980 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4981 }
4982 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 4983 }
4984}
4985
4986static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4987{
4988 struct e1000_hw *hw = &adapter->hw;
4989 u32 pool_mask, reg, vid;
4990 int i;
4991
4992 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4993
4994 /* Find the vlan filter for this id */
4995 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4996 reg = rd32(E1000_VLVF(i));
4997
4998 /* remove the vf from the pool */
4999 reg &= ~pool_mask;
5000
5001 /* if pool is empty then remove entry from vfta */
5002 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
5003 (reg & E1000_VLVF_VLANID_ENABLE)) {
5004 reg = 0;
5005 vid = reg & E1000_VLVF_VLANID_MASK;
5006 igb_vfta_set(hw, vid, false);
5007 }
5008
5009 wr32(E1000_VLVF(i), reg);
5010 }
ae641bdc
AD		 5011
5012 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 5013}
5014
5015static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
5016{
5017 struct e1000_hw *hw = &adapter->hw;
5018 u32 reg, i;
5019
51466239
AD		 5020 /* The vlvf table only exists on 82576 hardware and newer */
5021 if (hw->mac.type < e1000_82576)
5022 return -1;
5023
5024 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 5025 if (!adapter->vfs_allocated_count)
5026 return -1;
5027
5028 /* Find the vlan filter for this id */
5029 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5030 reg = rd32(E1000_VLVF(i));
5031 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
5032 vid == (reg & E1000_VLVF_VLANID_MASK))
5033 break;
5034 }
5035
5036 if (add) {
5037 if (i == E1000_VLVF_ARRAY_SIZE) {
5038 /* Did not find a matching VLAN ID entry that was
5039 * enabled. Search for a free filter entry, i.e.
5040 * one without the enable bit set
5041 */
5042 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5043 reg = rd32(E1000_VLVF(i));
5044 if (!(reg & E1000_VLVF_VLANID_ENABLE))
5045 break;
5046 }
5047 }
5048 if (i < E1000_VLVF_ARRAY_SIZE) {
5049 /* Found an enabled/available entry */
5050 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5051
5052 /* if !enabled we need to set this up in vfta */
5053 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 5054 /* add VID to filter table */
5055 igb_vfta_set(hw, vid, true);
4ae196df
AD		 5056 reg |= E1000_VLVF_VLANID_ENABLE;
5057 }
cad6d05f
AD		 5058 reg &= ~E1000_VLVF_VLANID_MASK;
5059 reg |= vid;
4ae196df 5060 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5061
5062 /* do not modify RLPML for PF devices */
5063 if (vf >= adapter->vfs_allocated_count)
5064 return 0;
5065
5066 if (!adapter->vf_data[vf].vlans_enabled) {
5067 u32 size;
5068 reg = rd32(E1000_VMOLR(vf));
5069 size = reg & E1000_VMOLR_RLPML_MASK;
5070 size += 4;
5071 reg &= ~E1000_VMOLR_RLPML_MASK;
5072 reg |= size;
5073 wr32(E1000_VMOLR(vf), reg);
5074 }
ae641bdc 5075
51466239 5076 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 5077 return 0;
5078 }
5079 } else {
5080 if (i < E1000_VLVF_ARRAY_SIZE) {
5081 /* remove vf from the pool */
5082 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
5083 /* if pool is empty then remove entry from vfta */
5084 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
5085 reg = 0;
5086 igb_vfta_set(hw, vid, false);
5087 }
5088 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5089
5090 /* do not modify RLPML for PF devices */
5091 if (vf >= adapter->vfs_allocated_count)
5092 return 0;
5093
5094 adapter->vf_data[vf].vlans_enabled--;
5095 if (!adapter->vf_data[vf].vlans_enabled) {
5096 u32 size;
5097 reg = rd32(E1000_VMOLR(vf));
5098 size = reg & E1000_VMOLR_RLPML_MASK;
5099 size -= 4;
5100 reg &= ~E1000_VMOLR_RLPML_MASK;
5101 reg |= size;
5102 wr32(E1000_VMOLR(vf), reg);
5103 }
4ae196df
AD		 5104 }
5105 }
8151d294
WM		 5106 return 0;
5107}
5108
5109static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
5110{
5111 struct e1000_hw *hw = &adapter->hw;
5112
5113 if (vid)
5114 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
5115 else
5116 wr32(E1000_VMVIR(vf), 0);
5117}
5118
5119static int igb_ndo_set_vf_vlan(struct net_device *netdev,
5120 int vf, u16 vlan, u8 qos)
5121{
5122 int err = 0;
5123 struct igb_adapter *adapter = netdev_priv(netdev);
5124
5125 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
5126 return -EINVAL;
5127 if (vlan || qos) {
5128 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
5129 if (err)
5130 goto out;
5131 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
5132 igb_set_vmolr(adapter, vf, !vlan);
5133 adapter->vf_data[vf].pf_vlan = vlan;
5134 adapter->vf_data[vf].pf_qos = qos;
5135 dev_info(&adapter->pdev->dev,
5136 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5137 if (test_bit(__IGB_DOWN, &adapter->state)) {
5138 dev_warn(&adapter->pdev->dev,
5139 "The VF VLAN has been set,"
5140 " but the PF device is not up.\n");
5141 dev_warn(&adapter->pdev->dev,
5142 "Bring the PF device up before"
5143 " attempting to use the VF device.\n");
5144 }
5145 } else {
5146 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5147 false, vf);
5148 igb_set_vmvir(adapter, vlan, vf);
5149 igb_set_vmolr(adapter, vf, true);
5150 adapter->vf_data[vf].pf_vlan = 0;
5151 adapter->vf_data[vf].pf_qos = 0;
5152 }
5153out:
5154 return err;
4ae196df
AD		 5155}
5156
5157static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5158{
5159 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5160 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5161
5162 return igb_vlvf_set(adapter, vid, add, vf);
5163}
5164
f2ca0dbe 5165static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 5166{
8fa7e0f7
GR		 5167 /* clear flags - except flag that indicates PF has set the MAC */
5168 adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
f2ca0dbe 5169 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 5170
5171 /* reset offloads to defaults */
8151d294 5172 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 5173
5174 /* reset vlans for device */
5175 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 5176 if (adapter->vf_data[vf].pf_vlan)
5177 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5178 adapter->vf_data[vf].pf_vlan,
5179 adapter->vf_data[vf].pf_qos);
5180 else
5181 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 5182
5183 /* reset multicast table array for vf */
5184 adapter->vf_data[vf].num_vf_mc_hashes = 0;
5185
5186 /* Flush and reset the mta with the new values */
ff41f8dc 5187 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 5188}
5189
f2ca0dbe
AD		 5190static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5191{
5192 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5193
5194 /* generate a new mac address as we were hotplug removed/added */
8151d294
WM		 5195 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5196 random_ether_addr(vf_mac);
f2ca0dbe
AD		 5197
5198 /* process remaining reset events */
5199 igb_vf_reset(adapter, vf);
5200}
5201
5202static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 5203{
5204 struct e1000_hw *hw = &adapter->hw;
5205 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 5206 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 5207 u32 reg, msgbuf[3];
5208 u8 *addr = (u8 *)(&msgbuf[1]);
5209
5210 /* process all the same items cleared in a function level reset */
f2ca0dbe 5211 igb_vf_reset(adapter, vf);
4ae196df
AD		 5212
5213 /* set vf mac address */
26ad9178 5214 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 5215
5216 /* enable transmit and receive for vf */
5217 reg = rd32(E1000_VFTE);
5218 wr32(E1000_VFTE, reg | (1 << vf));
5219 reg = rd32(E1000_VFRE);
5220 wr32(E1000_VFRE, reg | (1 << vf));
5221
8fa7e0f7 5222 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
4ae196df
AD		 5223
5224 /* reply to reset with ack and vf mac address */
5225 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5226 memcpy(addr, vf_mac, 6);
5227 igb_write_mbx(hw, msgbuf, 3, vf);
5228}
5229
5230static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5231{
de42edde
GR		 5232 /*
5233 * The VF MAC Address is stored in a packed array of bytes
5234 * starting at the second 32 bit word of the msg array
5235 */
f2ca0dbe
AD		 5236 unsigned char *addr = (char *)&msg[1];
5237 int err = -1;
4ae196df 5238
f2ca0dbe
AD		 5239 if (is_valid_ether_addr(addr))
5240 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 5241
f2ca0dbe 5242 return err;
4ae196df
AD		 5243}
5244
5245static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5246{
5247 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5248 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5249 u32 msg = E1000_VT_MSGTYPE_NACK;
5250
5251 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 5252 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5253 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 5254 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 5255 vf_data->last_nack = jiffies;
4ae196df
AD		 5256 }
5257}
5258
f2ca0dbe 5259static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 5260{
f2ca0dbe
AD		 5261 struct pci_dev *pdev = adapter->pdev;
5262 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 5263 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5264 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5265 s32 retval;
5266
f2ca0dbe 5267 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 5268
fef45f4c
AD		 5269 if (retval) {
5270 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 5271 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 5272 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5273 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5274 return;
5275 goto out;
5276 }
4ae196df
AD		 5277
5278 /* this is a message we already processed, do nothing */
5279 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 5280 return;
4ae196df
AD		 5281
5282 /*
5283 * until the vf completes a reset it should not be
5284 * allowed to start any configuration.
5285 */
5286
5287 if (msgbuf[0] == E1000_VF_RESET) {
5288 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 5289 return;
4ae196df
AD		 5290 }
5291
f2ca0dbe 5292 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 5293 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5294 return;
5295 retval = -1;
5296 goto out;
4ae196df
AD		 5297 }
5298
5299 switch ((msgbuf[0] & 0xFFFF)) {
5300 case E1000_VF_SET_MAC_ADDR:
a6b5ea35
GR		 5301 retval = -EINVAL;
5302 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
5303 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5304 else
5305 dev_warn(&pdev->dev,
5306 "VF %d attempted to override administratively "
5307 "set MAC address\nReload the VF driver to "
5308 "resume operations\n", vf);
4ae196df 5309 break;
7d5753f0
AD		 5310 case E1000_VF_SET_PROMISC:
5311 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5312 break;
4ae196df
AD		 5313 case E1000_VF_SET_MULTICAST:
5314 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5315 break;
5316 case E1000_VF_SET_LPE:
5317 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5318 break;
5319 case E1000_VF_SET_VLAN:
a6b5ea35
GR		 5320 retval = -1;
5321 if (vf_data->pf_vlan)
5322 dev_warn(&pdev->dev,
5323 "VF %d attempted to override administratively "
5324 "set VLAN tag\nReload the VF driver to "
5325 "resume operations\n", vf);
8151d294
WM		 5326 else
5327 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 5328 break;
5329 default:
090b1795 5330 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 5331 retval = -1;
5332 break;
5333 }
5334
fef45f4c
AD		 5335 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5336out:
4ae196df
AD		 5337 /* notify the VF of the results of what it sent us */
5338 if (retval)
5339 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5340 else
5341 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5342
4ae196df 5343 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 5344}
4ae196df 5345
f2ca0dbe
AD		 5346static void igb_msg_task(struct igb_adapter *adapter)
5347{
5348 struct e1000_hw *hw = &adapter->hw;
5349 u32 vf;
5350
5351 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5352 /* process any reset requests */
5353 if (!igb_check_for_rst(hw, vf))
5354 igb_vf_reset_event(adapter, vf);
5355
5356 /* process any messages pending */
5357 if (!igb_check_for_msg(hw, vf))
5358 igb_rcv_msg_from_vf(adapter, vf);
5359
5360 /* process any acks */
5361 if (!igb_check_for_ack(hw, vf))
5362 igb_rcv_ack_from_vf(adapter, vf);
5363 }
4ae196df
AD		 5364}
5365
68d480c4
AD		 5366/**
5367 * igb_set_uta - Set unicast filter table address
5368 * @adapter: board private structure
5369 *
5370 * The unicast table address is a register array of 32-bit registers.
5371 * The table is meant to be used in a way similar to how the MTA is used
5372 * however due to certain limitations in the hardware it is necessary to
25985edc
LDM		 5373 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
5374 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
68d480c4
AD		 5375 **/
5376static void igb_set_uta(struct igb_adapter *adapter)
5377{
5378 struct e1000_hw *hw = &adapter->hw;
5379 int i;
5380
5381 /* The UTA table only exists on 82576 hardware and newer */
5382 if (hw->mac.type < e1000_82576)
5383 return;
5384
5385 /* we only need to do this if VMDq is enabled */
5386 if (!adapter->vfs_allocated_count)
5387 return;
5388
5389 for (i = 0; i < hw->mac.uta_reg_count; i++)
5390 array_wr32(E1000_UTA, i, ~0);
5391}
5392
9d5c8243
AK		 5393/**
5394 * igb_intr_msi - Interrupt Handler
5395 * @irq: interrupt number
5396 * @data: pointer to a network interface device structure
5397 **/
5398static irqreturn_t igb_intr_msi(int irq, void *data)
5399{
047e0030
AD		 5400 struct igb_adapter *adapter = data;
5401 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5402 struct e1000_hw *hw = &adapter->hw;
5403 /* read ICR disables interrupts using IAM */
5404 u32 icr = rd32(E1000_ICR);
5405
047e0030 5406 igb_write_itr(q_vector);
9d5c8243 5407
7f081d40
AD		 5408 if (icr & E1000_ICR_DRSTA)
5409 schedule_work(&adapter->reset_task);
5410
047e0030 5411 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5412 /* HW is reporting DMA is out of sync */
5413 adapter->stats.doosync++;
5414 }
5415
9d5c8243
AK		 5416 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5417 hw->mac.get_link_status = 1;
5418 if (!test_bit(__IGB_DOWN, &adapter->state))
5419 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5420 }
5421
047e0030 5422 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5423
5424 return IRQ_HANDLED;
5425}
5426
5427/**
4a3c6433 5428 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 5429 * @irq: interrupt number
5430 * @data: pointer to a network interface device structure
5431 **/
5432static irqreturn_t igb_intr(int irq, void *data)
5433{
047e0030
AD		 5434 struct igb_adapter *adapter = data;
5435 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5436 struct e1000_hw *hw = &adapter->hw;
5437 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5438 * need for the IMC write */
5439 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 5440 if (!icr)
5441 return IRQ_NONE; /* Not our interrupt */
5442
047e0030 5443 igb_write_itr(q_vector);
9d5c8243
AK		 5444
5445 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5446 * not set, then the adapter didn't send an interrupt */
5447 if (!(icr & E1000_ICR_INT_ASSERTED))
5448 return IRQ_NONE;
5449
7f081d40
AD		 5450 if (icr & E1000_ICR_DRSTA)
5451 schedule_work(&adapter->reset_task);
5452
047e0030 5453 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5454 /* HW is reporting DMA is out of sync */
5455 adapter->stats.doosync++;
5456 }
5457
9d5c8243
AK		 5458 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5459 hw->mac.get_link_status = 1;
5460 /* guard against interrupt when we're going down */
5461 if (!test_bit(__IGB_DOWN, &adapter->state))
5462 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5463 }
5464
047e0030 5465 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5466
5467 return IRQ_HANDLED;
5468}
5469
047e0030 5470static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 5471{
047e0030 5472 struct igb_adapter *adapter = q_vector->adapter;
46544258 5473 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5474
4fc82adf
AD		 5475 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
5476 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
047e0030 5477 if (!adapter->msix_entries)
6eb5a7f1 5478 igb_set_itr(adapter);
46544258 5479 else
047e0030 5480 igb_update_ring_itr(q_vector);
9d5c8243
AK		 5481 }
5482
46544258
AD		 5483 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5484 if (adapter->msix_entries)
047e0030 5485 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 5486 else
5487 igb_irq_enable(adapter);
5488 }
9d5c8243
AK		 5489}
5490
46544258
AD		 5491/**
5492 * igb_poll - NAPI Rx polling callback
5493 * @napi: napi polling structure
5494 * @budget: count of how many packets we should handle
5495 **/
5496static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 5497{
047e0030
AD		 5498 struct igb_q_vector *q_vector = container_of(napi,
5499 struct igb_q_vector,
5500 napi);
16eb8815 5501 bool clean_complete = true;
9d5c8243 5502
421e02f0 5503#ifdef CONFIG_IGB_DCA
047e0030
AD		 5504 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5505 igb_update_dca(q_vector);
fe4506b6 5506#endif
047e0030 5507 if (q_vector->tx_ring)
13fde97a 5508 clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 5509
047e0030 5510 if (q_vector->rx_ring)
cd392f5c 5511 clean_complete &= igb_clean_rx_irq(q_vector, budget);
047e0030 5512
16eb8815
AD		 5513 /* If all work not completed, return budget and keep polling */
5514 if (!clean_complete)
5515 return budget;
46544258 5516
9d5c8243 5517 /* If not enough Rx work done, exit the polling mode */
16eb8815
AD		 5518 napi_complete(napi);
5519 igb_ring_irq_enable(q_vector);
9d5c8243 5520
16eb8815 5521 return 0;
9d5c8243 5522}
6d8126f9 5523
33af6bcc 5524/**
c5b9bd5e 5525 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 5526 * @adapter: board private structure
c5b9bd5e
AD		 5527 * @shhwtstamps: timestamp structure to update
5528 * @regval: unsigned 64bit system time value.
5529 *
5530 * We need to convert the system time value stored in the RX/TXSTMP registers
5531 * into a hwtstamp which can be used by the upper level timestamping functions
5532 */
5533static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5534 struct skb_shared_hwtstamps *shhwtstamps,
5535 u64 regval)
5536{
5537 u64 ns;
5538
55cac248
AD		 5539 /*
5540 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5541 * 24 to match clock shift we setup earlier.
5542 */
5543 if (adapter->hw.mac.type == e1000_82580)
5544 regval <<= IGB_82580_TSYNC_SHIFT;
5545
c5b9bd5e
AD		 5546 ns = timecounter_cyc2time(&adapter->clock, regval);
5547 timecompare_update(&adapter->compare, ns);
5548 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5549 shhwtstamps->hwtstamp = ns_to_ktime(ns);
5550 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5551}
5552
5553/**
5554 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5555 * @q_vector: pointer to q_vector containing needed info
06034649 5556 * @buffer: pointer to igb_tx_buffer structure
33af6bcc
PO		 5557 *
5558 * If we were asked to do hardware stamping and such a time stamp is
5559 * available, then it must have been for this skb here because we only
5560 * allow only one such packet into the queue.
5561 */
06034649
AD		 5562static void igb_tx_hwtstamp(struct igb_q_vector *q_vector,
5563 struct igb_tx_buffer *buffer_info)
33af6bcc 5564{
c5b9bd5e 5565 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc 5566 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 5567 struct skb_shared_hwtstamps shhwtstamps;
5568 u64 regval;
33af6bcc 5569
c5b9bd5e 5570 /* if skb does not support hw timestamp or TX stamp not valid exit */
2244d07b 5571 if (likely(!(buffer_info->tx_flags & SKBTX_HW_TSTAMP)) ||
c5b9bd5e
AD		 5572 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5573 return;
5574
5575 regval = rd32(E1000_TXSTMPL);
5576 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5577
5578 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
2873957d 5579 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
33af6bcc
PO		 5580}
5581
9d5c8243
AK		 5582/**
5583 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 5584 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 5585 * returns true if ring is completely cleaned
5586 **/
047e0030 5587static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 5588{
047e0030
AD		 5589 struct igb_adapter *adapter = q_vector->adapter;
5590 struct igb_ring *tx_ring = q_vector->tx_ring;
06034649 5591 struct igb_tx_buffer *tx_buffer;
13fde97a 5592 union e1000_adv_tx_desc *tx_desc;
9d5c8243 5593 unsigned int total_bytes = 0, total_packets = 0;
13fde97a
AD		 5594 unsigned int budget = q_vector->tx_work_limit;
5595 u16 i = tx_ring->next_to_clean;
9d5c8243 5596
13fde97a
AD		 5597 if (test_bit(__IGB_DOWN, &adapter->state))
5598 return true;
0e014cb1 5599
06034649 5600 tx_buffer = &tx_ring->tx_buffer_info[i];
13fde97a 5601 tx_desc = IGB_TX_DESC(tx_ring, i);
9d5c8243 5602
13fde97a
AD		 5603 for (; budget; budget--) {
5604 u16 eop = tx_buffer->next_to_watch;
5605 union e1000_adv_tx_desc *eop_desc;
5606
5607 eop_desc = IGB_TX_DESC(tx_ring, eop);
5608
5609 /* if DD is not set pending work has not been completed */
5610 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
5611 break;
5612
5613 /* prevent any other reads prior to eop_desc being verified */
5614 rmb();
9d5c8243 5615
13fde97a 5616 do {
0e014cb1 5617 tx_desc->wb.status = 0;
13fde97a
AD		 5618 if (likely(tx_desc == eop_desc)) {
5619 eop_desc = NULL;
5620
5621 total_bytes += tx_buffer->bytecount;
5622 total_packets += tx_buffer->gso_segs;
5623 igb_tx_hwtstamp(q_vector, tx_buffer);
5624 }
5625
5626 igb_unmap_and_free_tx_resource(tx_ring, tx_buffer);
9d5c8243 5627
13fde97a
AD		 5628 tx_buffer++;
5629 tx_desc++;
9d5c8243 5630 i++;
13fde97a 5631 if (unlikely(i == tx_ring->count)) {
9d5c8243 5632 i = 0;
06034649 5633 tx_buffer = tx_ring->tx_buffer_info;
13fde97a
AD		 5634 tx_desc = IGB_TX_DESC(tx_ring, 0);
5635 }
5636 } while (eop_desc);
0e014cb1
AD		 5637 }
5638
9d5c8243 5639 tx_ring->next_to_clean = i;
13fde97a
AD		 5640 u64_stats_update_begin(&tx_ring->tx_syncp);
5641 tx_ring->tx_stats.bytes += total_bytes;
5642 tx_ring->tx_stats.packets += total_packets;
5643 u64_stats_update_end(&tx_ring->tx_syncp);
5644 tx_ring->total_bytes += total_bytes;
5645 tx_ring->total_packets += total_packets;
9d5c8243 5646
13fde97a
AD		 5647 if (tx_ring->detect_tx_hung) {
5648 struct e1000_hw *hw = &adapter->hw;
06034649 5649 u16 eop = tx_ring->tx_buffer_info[i].next_to_watch;
13fde97a 5650 union e1000_adv_tx_desc *eop_desc;
12dcd86b 5651
13fde97a 5652 eop_desc = IGB_TX_DESC(tx_ring, eop);
9d5c8243 5653
9d5c8243
AK		 5654 /* Detect a transmit hang in hardware, this serializes the
5655 * check with the clearing of time_stamp and movement of i */
5656 tx_ring->detect_tx_hung = false;
06034649
AD		 5657 if (tx_ring->tx_buffer_info[i].time_stamp &&
5658 time_after(jiffies, tx_ring->tx_buffer_info[i].time_stamp +
8e95a202
JP		 5659 (adapter->tx_timeout_factor * HZ)) &&
5660 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5661
9d5c8243 5662 /* detected Tx unit hang */
59d71989 5663 dev_err(tx_ring->dev,
9d5c8243 5664 "Detected Tx Unit Hang\n"
2d064c06 5665 " Tx Queue <%d>\n"
9d5c8243
AK		 5666 " TDH <%x>\n"
5667 " TDT <%x>\n"
5668 " next_to_use <%x>\n"
5669 " next_to_clean <%x>\n"
9d5c8243
AK		 5670 "buffer_info[next_to_clean]\n"
5671 " time_stamp <%lx>\n"
0e014cb1 5672 " next_to_watch <%x>\n"
9d5c8243
AK		 5673 " jiffies <%lx>\n"
5674 " desc.status <%x>\n",
2d064c06 5675 tx_ring->queue_index,
238ac817 5676 rd32(E1000_TDH(tx_ring->reg_idx)),
fce99e34 5677 readl(tx_ring->tail),
9d5c8243
AK		 5678 tx_ring->next_to_use,
5679 tx_ring->next_to_clean,
06034649 5680 tx_ring->tx_buffer_info[eop].time_stamp,
0e014cb1 5681 eop,
9d5c8243 5682 jiffies,
0e014cb1 5683 eop_desc->wb.status);
13fde97a
AD		 5684 netif_stop_subqueue(tx_ring->netdev,
5685 tx_ring->queue_index);
5686
5687 /* we are about to reset, no point in enabling stuff */
5688 return true;
9d5c8243
AK		 5689 }
5690 }
13fde97a
AD		 5691
5692 if (unlikely(total_packets &&
5693 netif_carrier_ok(tx_ring->netdev) &&
5694 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5695 /* Make sure that anybody stopping the queue after this
5696 * sees the new next_to_clean.
5697 */
5698 smp_mb();
5699 if (__netif_subqueue_stopped(tx_ring->netdev,
5700 tx_ring->queue_index) &&
5701 !(test_bit(__IGB_DOWN, &adapter->state))) {
5702 netif_wake_subqueue(tx_ring->netdev,
5703 tx_ring->queue_index);
5704
5705 u64_stats_update_begin(&tx_ring->tx_syncp);
5706 tx_ring->tx_stats.restart_queue++;
5707 u64_stats_update_end(&tx_ring->tx_syncp);
5708 }
5709 }
5710
5711 return !!budget;
9d5c8243
AK		 5712}
5713
cd392f5c
AD		 5714static inline void igb_rx_checksum(struct igb_ring *ring,
5715 u32 status_err, struct sk_buff *skb)
9d5c8243 5716{
bc8acf2c 5717 skb_checksum_none_assert(skb);
9d5c8243
AK		 5718
5719 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 5720 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5721 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 5722 return;
85ad76b2 5723
9d5c8243
AK		 5724 /* TCP/UDP checksum error bit is set */
5725 if (status_err &
5726 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5727 /*
5728 * work around errata with sctp packets where the TCPE aka
5729 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5730 * packets, (aka let the stack check the crc32c)
5731 */
85ad76b2 5732 if ((skb->len == 60) &&
12dcd86b
ED		 5733 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM)) {
5734 u64_stats_update_begin(&ring->rx_syncp);
04a5fcaa 5735 ring->rx_stats.csum_err++;
12dcd86b
ED		 5736 u64_stats_update_end(&ring->rx_syncp);
5737 }
9d5c8243 5738 /* let the stack verify checksum errors */
9d5c8243
AK		 5739 return;
5740 }
5741 /* It must be a TCP or UDP packet with a valid checksum */
5742 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5743 skb->ip_summed = CHECKSUM_UNNECESSARY;
5744
59d71989 5745 dev_dbg(ring->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 5746}
5747
757b77e2 5748static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
c5b9bd5e
AD		 5749 struct sk_buff *skb)
5750{
5751 struct igb_adapter *adapter = q_vector->adapter;
5752 struct e1000_hw *hw = &adapter->hw;
5753 u64 regval;
5754
5755 /*
5756 * If this bit is set, then the RX registers contain the time stamp. No
5757 * other packet will be time stamped until we read these registers, so
5758 * read the registers to make them available again. Because only one
5759 * packet can be time stamped at a time, we know that the register
5760 * values must belong to this one here and therefore we don't need to
5761 * compare any of the additional attributes stored for it.
5762 *
2244d07b 5763 * If nothing went wrong, then it should have a shared tx_flags that we
c5b9bd5e
AD		 5764 * can turn into a skb_shared_hwtstamps.
5765 */
757b77e2
NN		 5766 if (staterr & E1000_RXDADV_STAT_TSIP) {
5767 u32 *stamp = (u32 *)skb->data;
5768 regval = le32_to_cpu(*(stamp + 2));
5769 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5770 skb_pull(skb, IGB_TS_HDR_LEN);
5771 } else {
5772 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5773 return;
c5b9bd5e 5774
757b77e2
NN		 5775 regval = rd32(E1000_RXSTMPL);
5776 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5777 }
c5b9bd5e
AD		 5778
5779 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5780}
44390ca6 5781static inline u16 igb_get_hlen(union e1000_adv_rx_desc *rx_desc)
2d94d8ab
AD		 5782{
5783 /* HW will not DMA in data larger than the given buffer, even if it
5784 * parses the (NFS, of course) header to be larger. In that case, it
5785 * fills the header buffer and spills the rest into the page.
5786 */
5787 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5788 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
44390ca6
AD		 5789 if (hlen > IGB_RX_HDR_LEN)
5790 hlen = IGB_RX_HDR_LEN;
2d94d8ab
AD		 5791 return hlen;
5792}
5793
cd392f5c 5794static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, int budget)
9d5c8243 5795{
047e0030 5796 struct igb_ring *rx_ring = q_vector->rx_ring;
16eb8815
AD		 5797 union e1000_adv_rx_desc *rx_desc;
5798 const int current_node = numa_node_id();
9d5c8243 5799 unsigned int total_bytes = 0, total_packets = 0;
2d94d8ab 5800 u32 staterr;
16eb8815
AD		 5801 u16 cleaned_count = igb_desc_unused(rx_ring);
5802 u16 i = rx_ring->next_to_clean;
9d5c8243 5803
60136906 5804 rx_desc = IGB_RX_DESC(rx_ring, i);
9d5c8243
AK		 5805 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5806
5807 while (staterr & E1000_RXD_STAT_DD) {
06034649 5808 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
16eb8815
AD		 5809 struct sk_buff *skb = buffer_info->skb;
5810 union e1000_adv_rx_desc *next_rxd;
9d5c8243 5811
69d3ca53 5812 buffer_info->skb = NULL;
16eb8815 5813 prefetch(skb->data);
69d3ca53
AD		 5814
5815 i++;
5816 if (i == rx_ring->count)
5817 i = 0;
42d0781a 5818
60136906 5819 next_rxd = IGB_RX_DESC(rx_ring, i);
69d3ca53 5820 prefetch(next_rxd);
9d5c8243 5821
16eb8815
AD		 5822 /*
5823 * This memory barrier is needed to keep us from reading
5824 * any other fields out of the rx_desc until we know the
5825 * RXD_STAT_DD bit is set
5826 */
5827 rmb();
9d5c8243 5828
16eb8815
AD		 5829 if (!skb_is_nonlinear(skb)) {
5830 __skb_put(skb, igb_get_hlen(rx_desc));
5831 dma_unmap_single(rx_ring->dev, buffer_info->dma,
44390ca6 5832 IGB_RX_HDR_LEN,
59d71989 5833 DMA_FROM_DEVICE);
91615f76 5834 buffer_info->dma = 0;
bf36c1a0
AD		 5835 }
5836
16eb8815
AD		 5837 if (rx_desc->wb.upper.length) {
5838 u16 length = le16_to_cpu(rx_desc->wb.upper.length);
bf36c1a0 5839
aa913403 5840 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
bf36c1a0
AD		 5841 buffer_info->page,
5842 buffer_info->page_offset,
5843 length);
5844
16eb8815
AD		 5845 skb->len += length;
5846 skb->data_len += length;
5847 skb->truesize += length;
5848
d1eff350
AD		 5849 if ((page_count(buffer_info->page) != 1) ||
5850 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 5851 buffer_info->page = NULL;
5852 else
5853 get_page(buffer_info->page);
9d5c8243 5854
16eb8815
AD		 5855 dma_unmap_page(rx_ring->dev, buffer_info->page_dma,
5856 PAGE_SIZE / 2, DMA_FROM_DEVICE);
5857 buffer_info->page_dma = 0;
9d5c8243 5858 }
9d5c8243 5859
bf36c1a0 5860 if (!(staterr & E1000_RXD_STAT_EOP)) {
06034649
AD		 5861 struct igb_rx_buffer *next_buffer;
5862 next_buffer = &rx_ring->rx_buffer_info[i];
b2d56536
AD		 5863 buffer_info->skb = next_buffer->skb;
5864 buffer_info->dma = next_buffer->dma;
5865 next_buffer->skb = skb;
5866 next_buffer->dma = 0;
bf36c1a0
AD		 5867 goto next_desc;
5868 }
44390ca6 5869
9d5c8243 5870 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
16eb8815 5871 dev_kfree_skb_any(skb);
9d5c8243
AK		 5872 goto next_desc;
5873 }
9d5c8243 5874
757b77e2
NN		 5875 if (staterr & (E1000_RXDADV_STAT_TSIP | E1000_RXDADV_STAT_TS))
5876 igb_rx_hwtstamp(q_vector, staterr, skb);
9d5c8243
AK		 5877 total_bytes += skb->len;
5878 total_packets++;
5879
cd392f5c 5880 igb_rx_checksum(rx_ring, staterr, skb);
9d5c8243 5881
16eb8815 5882 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
047e0030 5883
b2cb09b1
JP		 5884 if (staterr & E1000_RXD_STAT_VP) {
5885 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
9d5c8243 5886
b2cb09b1
JP		 5887 __vlan_hwaccel_put_tag(skb, vid);
5888 }
5889 napi_gro_receive(&q_vector->napi, skb);
9d5c8243 5890
16eb8815 5891 budget--;
9d5c8243 5892next_desc:
16eb8815
AD		 5893 if (!budget)
5894 break;
5895
5896 cleaned_count++;
9d5c8243
AK		 5897 /* return some buffers to hardware, one at a time is too slow */
5898 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
cd392f5c 5899 igb_alloc_rx_buffers(rx_ring, cleaned_count);
9d5c8243
AK		 5900 cleaned_count = 0;
5901 }
5902
5903 /* use prefetched values */
5904 rx_desc = next_rxd;
9d5c8243
AK		 5905 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5906 }
bf36c1a0 5907
9d5c8243 5908 rx_ring->next_to_clean = i;
12dcd86b 5909 u64_stats_update_begin(&rx_ring->rx_syncp);
9d5c8243
AK		 5910 rx_ring->rx_stats.packets += total_packets;
5911 rx_ring->rx_stats.bytes += total_bytes;
12dcd86b 5912 u64_stats_update_end(&rx_ring->rx_syncp);
c023cd88
AD		 5913 rx_ring->total_packets += total_packets;
5914 rx_ring->total_bytes += total_bytes;
5915
5916 if (cleaned_count)
cd392f5c 5917 igb_alloc_rx_buffers(rx_ring, cleaned_count);
c023cd88 5918
16eb8815 5919 return !!budget;
9d5c8243
AK		 5920}
5921
c023cd88 5922static bool igb_alloc_mapped_skb(struct igb_ring *rx_ring,
06034649 5923 struct igb_rx_buffer *bi)
c023cd88
AD		 5924{
5925 struct sk_buff *skb = bi->skb;
5926 dma_addr_t dma = bi->dma;
5927
5928 if (dma)
5929 return true;
5930
5931 if (likely(!skb)) {
5932 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
5933 IGB_RX_HDR_LEN);
5934 bi->skb = skb;
5935 if (!skb) {
5936 rx_ring->rx_stats.alloc_failed++;
5937 return false;
5938 }
5939
5940 /* initialize skb for ring */
5941 skb_record_rx_queue(skb, rx_ring->queue_index);
5942 }
5943
5944 dma = dma_map_single(rx_ring->dev, skb->data,
5945 IGB_RX_HDR_LEN, DMA_FROM_DEVICE);
5946
5947 if (dma_mapping_error(rx_ring->dev, dma)) {
5948 rx_ring->rx_stats.alloc_failed++;
5949 return false;
5950 }
5951
5952 bi->dma = dma;
5953 return true;
5954}
5955
5956static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
06034649 5957 struct igb_rx_buffer *bi)
c023cd88
AD		 5958{
5959 struct page *page = bi->page;
5960 dma_addr_t page_dma = bi->page_dma;
5961 unsigned int page_offset = bi->page_offset ^ (PAGE_SIZE / 2);
5962
5963 if (page_dma)
5964 return true;
5965
5966 if (!page) {
5967 page = netdev_alloc_page(rx_ring->netdev);
5968 bi->page = page;
5969 if (unlikely(!page)) {
5970 rx_ring->rx_stats.alloc_failed++;
5971 return false;
5972 }
5973 }
5974
5975 page_dma = dma_map_page(rx_ring->dev, page,
5976 page_offset, PAGE_SIZE / 2,
5977 DMA_FROM_DEVICE);
5978
5979 if (dma_mapping_error(rx_ring->dev, page_dma)) {
5980 rx_ring->rx_stats.alloc_failed++;
5981 return false;
5982 }
5983
5984 bi->page_dma = page_dma;
5985 bi->page_offset = page_offset;
5986 return true;
5987}
5988
9d5c8243 5989/**
cd392f5c 5990 * igb_alloc_rx_buffers - Replace used receive buffers; packet split
9d5c8243
AK		 5991 * @adapter: address of board private structure
5992 **/
cd392f5c 5993void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
9d5c8243 5994{
9d5c8243 5995 union e1000_adv_rx_desc *rx_desc;
06034649 5996 struct igb_rx_buffer *bi;
c023cd88 5997 u16 i = rx_ring->next_to_use;
9d5c8243 5998
60136906 5999 rx_desc = IGB_RX_DESC(rx_ring, i);
06034649 6000 bi = &rx_ring->rx_buffer_info[i];
c023cd88 6001 i -= rx_ring->count;
9d5c8243
AK		 6002
6003 while (cleaned_count--) {
c023cd88
AD		 6004 if (!igb_alloc_mapped_skb(rx_ring, bi))
6005 break;
9d5c8243 6006
c023cd88
AD		 6007 /* Refresh the desc even if buffer_addrs didn't change
6008 * because each write-back erases this info. */
6009 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
9d5c8243 6010
c023cd88
AD		 6011 if (!igb_alloc_mapped_page(rx_ring, bi))
6012 break;
6013
6014 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
9d5c8243 6015
c023cd88
AD		 6016 rx_desc++;
6017 bi++;
9d5c8243 6018 i++;
c023cd88 6019 if (unlikely(!i)) {
60136906 6020 rx_desc = IGB_RX_DESC(rx_ring, 0);
06034649 6021 bi = rx_ring->rx_buffer_info;
c023cd88
AD		 6022 i -= rx_ring->count;
6023 }
6024
6025 /* clear the hdr_addr for the next_to_use descriptor */
6026 rx_desc->read.hdr_addr = 0;
9d5c8243
AK		 6027 }
6028
c023cd88
AD		 6029 i += rx_ring->count;
6030
9d5c8243
AK		 6031 if (rx_ring->next_to_use != i) {
6032 rx_ring->next_to_use = i;
9d5c8243
AK		 6033
6034 /* Force memory writes to complete before letting h/w
6035 * know there are new descriptors to fetch. (Only
6036 * applicable for weak-ordered memory model archs,
6037 * such as IA-64). */
6038 wmb();
fce99e34 6039 writel(i, rx_ring->tail);
9d5c8243
AK		 6040 }
6041}
6042
6043/**
6044 * igb_mii_ioctl -
6045 * @netdev:
6046 * @ifreq:
6047 * @cmd:
6048 **/
6049static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6050{
6051 struct igb_adapter *adapter = netdev_priv(netdev);
6052 struct mii_ioctl_data *data = if_mii(ifr);
6053
6054 if (adapter->hw.phy.media_type != e1000_media_type_copper)
6055 return -EOPNOTSUPP;
6056
6057 switch (cmd) {
6058 case SIOCGMIIPHY:
6059 data->phy_id = adapter->hw.phy.addr;
6060 break;
6061 case SIOCGMIIREG:
f5f4cf08
AD		 6062 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
6063 &data->val_out))
9d5c8243
AK		 6064 return -EIO;
6065 break;
6066 case SIOCSMIIREG:
6067 default:
6068 return -EOPNOTSUPP;
6069 }
6070 return 0;
6071}
6072
c6cb090b
PO		 6073/**
6074 * igb_hwtstamp_ioctl - control hardware time stamping
6075 * @netdev:
6076 * @ifreq:
6077 * @cmd:
6078 *
33af6bcc
PO		 6079 * Outgoing time stamping can be enabled and disabled. Play nice and
6080 * disable it when requested, although it shouldn't case any overhead
6081 * when no packet needs it. At most one packet in the queue may be
6082 * marked for time stamping, otherwise it would be impossible to tell
6083 * for sure to which packet the hardware time stamp belongs.
6084 *
6085 * Incoming time stamping has to be configured via the hardware
6086 * filters. Not all combinations are supported, in particular event
6087 * type has to be specified. Matching the kind of event packet is
6088 * not supported, with the exception of "all V2 events regardless of
6089 * level 2 or 4".
6090 *
c6cb090b
PO		 6091 **/
6092static int igb_hwtstamp_ioctl(struct net_device *netdev,
6093 struct ifreq *ifr, int cmd)
6094{
33af6bcc
PO		 6095 struct igb_adapter *adapter = netdev_priv(netdev);
6096 struct e1000_hw *hw = &adapter->hw;
c6cb090b 6097 struct hwtstamp_config config;
c5b9bd5e
AD		 6098 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
6099 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 6100 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 6101 bool is_l4 = false;
6102 bool is_l2 = false;
33af6bcc 6103 u32 regval;
c6cb090b
PO		 6104
6105 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
6106 return -EFAULT;
6107
6108 /* reserved for future extensions */
6109 if (config.flags)
6110 return -EINVAL;
6111
33af6bcc
PO		 6112 switch (config.tx_type) {
6113 case HWTSTAMP_TX_OFF:
c5b9bd5e 6114 tsync_tx_ctl = 0;
33af6bcc 6115 case HWTSTAMP_TX_ON:
33af6bcc
PO		 6116 break;
6117 default:
6118 return -ERANGE;
6119 }
6120
6121 switch (config.rx_filter) {
6122 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 6123 tsync_rx_ctl = 0;
33af6bcc
PO		 6124 break;
6125 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
6126 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
6127 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
6128 case HWTSTAMP_FILTER_ALL:
6129 /*
6130 * register TSYNCRXCFG must be set, therefore it is not
6131 * possible to time stamp both Sync and Delay_Req messages
6132 * => fall back to time stamping all packets
6133 */
c5b9bd5e 6134 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 6135 config.rx_filter = HWTSTAMP_FILTER_ALL;
6136 break;
6137 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 6138 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 6139 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 6140 is_l4 = true;
33af6bcc
PO		 6141 break;
6142 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 6143 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 6144 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 6145 is_l4 = true;
33af6bcc
PO		 6146 break;
6147 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
6148 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 6149 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 6150 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 6151 is_l2 = true;
6152 is_l4 = true;
33af6bcc
PO		 6153 config.rx_filter = HWTSTAMP_FILTER_SOME;
6154 break;
6155 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
6156 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 6157 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 6158 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 6159 is_l2 = true;
6160 is_l4 = true;
33af6bcc
PO		 6161 config.rx_filter = HWTSTAMP_FILTER_SOME;
6162 break;
6163 case HWTSTAMP_FILTER_PTP_V2_EVENT:
6164 case HWTSTAMP_FILTER_PTP_V2_SYNC:
6165 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 6166 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 6167 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 6168 is_l2 = true;
33af6bcc
PO		 6169 break;
6170 default:
6171 return -ERANGE;
6172 }
6173
c5b9bd5e
AD		 6174 if (hw->mac.type == e1000_82575) {
6175 if (tsync_rx_ctl | tsync_tx_ctl)
6176 return -EINVAL;
6177 return 0;
6178 }
6179
757b77e2
NN		 6180 /*
6181 * Per-packet timestamping only works if all packets are
6182 * timestamped, so enable timestamping in all packets as
6183 * long as one rx filter was configured.
6184 */
6185 if ((hw->mac.type == e1000_82580) && tsync_rx_ctl) {
6186 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
6187 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
6188 }
6189
33af6bcc
PO		 6190 /* enable/disable TX */
6191 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 6192 regval &= ~E1000_TSYNCTXCTL_ENABLED;
6193 regval |= tsync_tx_ctl;
33af6bcc
PO		 6194 wr32(E1000_TSYNCTXCTL, regval);
6195
c5b9bd5e 6196 /* enable/disable RX */
33af6bcc 6197 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 6198 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
6199 regval |= tsync_rx_ctl;
33af6bcc 6200 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 6201
c5b9bd5e
AD		 6202 /* define which PTP packets are time stamped */
6203 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 6204
c5b9bd5e
AD		 6205 /* define ethertype filter for timestamped packets */
6206 if (is_l2)
6207 wr32(E1000_ETQF(3),
6208 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
6209 E1000_ETQF_1588 | /* enable timestamping */
6210 ETH_P_1588)); /* 1588 eth protocol type */
6211 else
6212 wr32(E1000_ETQF(3), 0);
6213
6214#define PTP_PORT 319
6215 /* L4 Queue Filter[3]: filter by destination port and protocol */
6216 if (is_l4) {
6217 u32 ftqf = (IPPROTO_UDP /* UDP */
6218 | E1000_FTQF_VF_BP /* VF not compared */
6219 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
6220 | E1000_FTQF_MASK); /* mask all inputs */
6221 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
6222
6223 wr32(E1000_IMIR(3), htons(PTP_PORT));
6224 wr32(E1000_IMIREXT(3),
6225 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
6226 if (hw->mac.type == e1000_82576) {
6227 /* enable source port check */
6228 wr32(E1000_SPQF(3), htons(PTP_PORT));
6229 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
6230 }
6231 wr32(E1000_FTQF(3), ftqf);
6232 } else {
6233 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
6234 }
33af6bcc
PO		 6235 wrfl();
6236
6237 adapter->hwtstamp_config = config;
6238
6239 /* clear TX/RX time stamp registers, just to be sure */
6240 regval = rd32(E1000_TXSTMPH);
6241 regval = rd32(E1000_RXSTMPH);
c6cb090b 6242
33af6bcc
PO		 6243 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
6244 -EFAULT : 0;
c6cb090b
PO		 6245}
6246
9d5c8243
AK		 6247/**
6248 * igb_ioctl -
6249 * @netdev:
6250 * @ifreq:
6251 * @cmd:
6252 **/
6253static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6254{
6255 switch (cmd) {
6256 case SIOCGMIIPHY:
6257 case SIOCGMIIREG:
6258 case SIOCSMIIREG:
6259 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 6260 case SIOCSHWTSTAMP:
6261 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 6262 default:
6263 return -EOPNOTSUPP;
6264 }
6265}
6266
009bc06e
AD		 6267s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6268{
6269 struct igb_adapter *adapter = hw->back;
6270 u16 cap_offset;
6271
bdaae04c 6272 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6273 if (!cap_offset)
6274 return -E1000_ERR_CONFIG;
6275
6276 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6277
6278 return 0;
6279}
6280
6281s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6282{
6283 struct igb_adapter *adapter = hw->back;
6284 u16 cap_offset;
6285
bdaae04c 6286 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6287 if (!cap_offset)
6288 return -E1000_ERR_CONFIG;
6289
6290 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6291
6292 return 0;
6293}
6294
b2cb09b1 6295static void igb_vlan_mode(struct net_device *netdev, u32 features)
9d5c8243
AK		 6296{
6297 struct igb_adapter *adapter = netdev_priv(netdev);
6298 struct e1000_hw *hw = &adapter->hw;
6299 u32 ctrl, rctl;
6300
6301 igb_irq_disable(adapter);
9d5c8243 6302
b2cb09b1 6303 if (features & NETIF_F_HW_VLAN_RX) {
9d5c8243
AK		 6304 /* enable VLAN tag insert/strip */
6305 ctrl = rd32(E1000_CTRL);
6306 ctrl |= E1000_CTRL_VME;
6307 wr32(E1000_CTRL, ctrl);
6308
51466239 6309 /* Disable CFI check */
9d5c8243 6310 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 6311 rctl &= ~E1000_RCTL_CFIEN;
6312 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 6313 } else {
6314 /* disable VLAN tag insert/strip */
6315 ctrl = rd32(E1000_CTRL);
6316 ctrl &= ~E1000_CTRL_VME;
6317 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 6318 }
6319
e1739522
AD		 6320 igb_rlpml_set(adapter);
6321
9d5c8243
AK		 6322 if (!test_bit(__IGB_DOWN, &adapter->state))
6323 igb_irq_enable(adapter);
6324}
6325
6326static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6327{
6328 struct igb_adapter *adapter = netdev_priv(netdev);
6329 struct e1000_hw *hw = &adapter->hw;
4ae196df 6330 int pf_id = adapter->vfs_allocated_count;
9d5c8243 6331
51466239
AD		 6332 /* attempt to add filter to vlvf array */
6333 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 6334
51466239
AD		 6335 /* add the filter since PF can receive vlans w/o entry in vlvf */
6336 igb_vfta_set(hw, vid, true);
b2cb09b1
JP		 6337
6338 set_bit(vid, adapter->active_vlans);
9d5c8243
AK		 6339}
6340
6341static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6342{
6343 struct igb_adapter *adapter = netdev_priv(netdev);
6344 struct e1000_hw *hw = &adapter->hw;
4ae196df 6345 int pf_id = adapter->vfs_allocated_count;
51466239 6346 s32 err;
9d5c8243
AK		 6347
6348 igb_irq_disable(adapter);
9d5c8243
AK		 6349
6350 if (!test_bit(__IGB_DOWN, &adapter->state))
6351 igb_irq_enable(adapter);
6352
51466239
AD		 6353 /* remove vlan from VLVF table array */
6354 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 6355
51466239
AD		 6356 /* if vid was not present in VLVF just remove it from table */
6357 if (err)
4ae196df 6358 igb_vfta_set(hw, vid, false);
b2cb09b1
JP		 6359
6360 clear_bit(vid, adapter->active_vlans);
9d5c8243
AK		 6361}
6362
6363static void igb_restore_vlan(struct igb_adapter *adapter)
6364{
b2cb09b1 6365 u16 vid;
9d5c8243 6366
b2cb09b1
JP		 6367 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
6368 igb_vlan_rx_add_vid(adapter->netdev, vid);
9d5c8243
AK		 6369}
6370
14ad2513 6371int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
9d5c8243 6372{
090b1795 6373 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 6374 struct e1000_mac_info *mac = &adapter->hw.mac;
6375
6376 mac->autoneg = 0;
6377
14ad2513
DD		 6378 /* Make sure dplx is at most 1 bit and lsb of speed is not set
6379 * for the switch() below to work */
6380 if ((spd & 1) || (dplx & ~1))
6381 goto err_inval;
6382
cd2638a8
CW		 6383 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6384 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
14ad2513
DD		 6385 spd != SPEED_1000 &&
6386 dplx != DUPLEX_FULL)
6387 goto err_inval;
cd2638a8 6388
14ad2513 6389 switch (spd + dplx) {
9d5c8243
AK		 6390 case SPEED_10 + DUPLEX_HALF:
6391 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6392 break;
6393 case SPEED_10 + DUPLEX_FULL:
6394 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6395 break;
6396 case SPEED_100 + DUPLEX_HALF:
6397 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6398 break;
6399 case SPEED_100 + DUPLEX_FULL:
6400 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6401 break;
6402 case SPEED_1000 + DUPLEX_FULL:
6403 mac->autoneg = 1;
6404 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6405 break;
6406 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6407 default:
14ad2513 6408 goto err_inval;
9d5c8243
AK		 6409 }
6410 return 0;
14ad2513
DD		 6411
6412err_inval:
6413 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6414 return -EINVAL;
9d5c8243
AK		 6415}
6416
3fe7c4c9 6417static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 6418{
6419 struct net_device *netdev = pci_get_drvdata(pdev);
6420 struct igb_adapter *adapter = netdev_priv(netdev);
6421 struct e1000_hw *hw = &adapter->hw;
2d064c06 6422 u32 ctrl, rctl, status;
9d5c8243
AK		 6423 u32 wufc = adapter->wol;
6424#ifdef CONFIG_PM
6425 int retval = 0;
6426#endif
6427
6428 netif_device_detach(netdev);
6429
a88f10ec
AD		 6430 if (netif_running(netdev))
6431 igb_close(netdev);
6432
047e0030 6433 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 6434
6435#ifdef CONFIG_PM
6436 retval = pci_save_state(pdev);
6437 if (retval)
6438 return retval;
6439#endif
6440
6441 status = rd32(E1000_STATUS);
6442 if (status & E1000_STATUS_LU)
6443 wufc &= ~E1000_WUFC_LNKC;
6444
6445 if (wufc) {
6446 igb_setup_rctl(adapter);
ff41f8dc 6447 igb_set_rx_mode(netdev);
9d5c8243
AK		 6448
6449 /* turn on all-multi mode if wake on multicast is enabled */
6450 if (wufc & E1000_WUFC_MC) {
6451 rctl = rd32(E1000_RCTL);
6452 rctl |= E1000_RCTL_MPE;
6453 wr32(E1000_RCTL, rctl);
6454 }
6455
6456 ctrl = rd32(E1000_CTRL);
6457 /* advertise wake from D3Cold */
6458 #define E1000_CTRL_ADVD3WUC 0x00100000
6459 /* phy power management enable */
6460 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6461 ctrl |= E1000_CTRL_ADVD3WUC;
6462 wr32(E1000_CTRL, ctrl);
6463
9d5c8243 6464 /* Allow time for pending master requests to run */
330a6d6a 6465 igb_disable_pcie_master(hw);
9d5c8243
AK		 6466
6467 wr32(E1000_WUC, E1000_WUC_PME_EN);
6468 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 6469 } else {
6470 wr32(E1000_WUC, 0);
6471 wr32(E1000_WUFC, 0);
9d5c8243
AK		 6472 }
6473
3fe7c4c9
RW		 6474 *enable_wake = wufc || adapter->en_mng_pt;
6475 if (!*enable_wake)
88a268c1
NN		 6476 igb_power_down_link(adapter);
6477 else
6478 igb_power_up_link(adapter);
9d5c8243
AK		 6479
6480 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6481 * would have already happened in close and is redundant. */
6482 igb_release_hw_control(adapter);
6483
6484 pci_disable_device(pdev);
6485
9d5c8243
AK		 6486 return 0;
6487}
6488
6489#ifdef CONFIG_PM
3fe7c4c9
RW		 6490static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
6491{
6492 int retval;
6493 bool wake;
6494
6495 retval = __igb_shutdown(pdev, &wake);
6496 if (retval)
6497 return retval;
6498
6499 if (wake) {
6500 pci_prepare_to_sleep(pdev);
6501 } else {
6502 pci_wake_from_d3(pdev, false);
6503 pci_set_power_state(pdev, PCI_D3hot);
6504 }
6505
6506 return 0;
6507}
6508
9d5c8243
AK		 6509static int igb_resume(struct pci_dev *pdev)
6510{
6511 struct net_device *netdev = pci_get_drvdata(pdev);
6512 struct igb_adapter *adapter = netdev_priv(netdev);
6513 struct e1000_hw *hw = &adapter->hw;
6514 u32 err;
6515
6516 pci_set_power_state(pdev, PCI_D0);
6517 pci_restore_state(pdev);
b94f2d77 6518 pci_save_state(pdev);
42bfd33a 6519
aed5dec3 6520 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 6521 if (err) {
6522 dev_err(&pdev->dev,
6523 "igb: Cannot enable PCI device from suspend\n");
6524 return err;
6525 }
6526 pci_set_master(pdev);
6527
6528 pci_enable_wake(pdev, PCI_D3hot, 0);
6529 pci_enable_wake(pdev, PCI_D3cold, 0);
6530
047e0030 6531 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 6532 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6533 return -ENOMEM;
9d5c8243
AK		 6534 }
6535
9d5c8243 6536 igb_reset(adapter);
a8564f03
AD		 6537
6538 /* let the f/w know that the h/w is now under the control of the
6539 * driver. */
6540 igb_get_hw_control(adapter);
6541
9d5c8243
AK		 6542 wr32(E1000_WUS, ~0);
6543
a88f10ec
AD		 6544 if (netif_running(netdev)) {
6545 err = igb_open(netdev);
6546 if (err)
6547 return err;
6548 }
9d5c8243
AK		 6549
6550 netif_device_attach(netdev);
6551
9d5c8243
AK		 6552 return 0;
6553}
6554#endif
6555
6556static void igb_shutdown(struct pci_dev *pdev)
6557{
3fe7c4c9
RW		 6558 bool wake;
6559
6560 __igb_shutdown(pdev, &wake);
6561
6562 if (system_state == SYSTEM_POWER_OFF) {
6563 pci_wake_from_d3(pdev, wake);
6564 pci_set_power_state(pdev, PCI_D3hot);
6565 }
9d5c8243
AK		 6566}
6567
6568#ifdef CONFIG_NET_POLL_CONTROLLER
6569/*
6570 * Polling 'interrupt' - used by things like netconsole to send skbs
6571 * without having to re-enable interrupts. It's not called while
6572 * the interrupt routine is executing.
6573 */
6574static void igb_netpoll(struct net_device *netdev)
6575{
6576 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 6577 struct e1000_hw *hw = &adapter->hw;
9d5c8243 6578 int i;
9d5c8243 6579
eebbbdba 6580 if (!adapter->msix_entries) {
047e0030 6581 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 6582 igb_irq_disable(adapter);
047e0030 6583 napi_schedule(&q_vector->napi);
eebbbdba
AD		 6584 return;
6585 }
9d5c8243 6586
047e0030
AD		 6587 for (i = 0; i < adapter->num_q_vectors; i++) {
6588 struct igb_q_vector *q_vector = adapter->q_vector[i];
6589 wr32(E1000_EIMC, q_vector->eims_value);
6590 napi_schedule(&q_vector->napi);
eebbbdba 6591 }
9d5c8243
AK		 6592}
6593#endif /* CONFIG_NET_POLL_CONTROLLER */
6594
6595/**
6596 * igb_io_error_detected - called when PCI error is detected
6597 * @pdev: Pointer to PCI device
6598 * @state: The current pci connection state
6599 *
6600 * This function is called after a PCI bus error affecting
6601 * this device has been detected.
6602 */
6603static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6604 pci_channel_state_t state)
6605{
6606 struct net_device *netdev = pci_get_drvdata(pdev);
6607 struct igb_adapter *adapter = netdev_priv(netdev);
6608
6609 netif_device_detach(netdev);
6610
59ed6eec
AD		 6611 if (state == pci_channel_io_perm_failure)
6612 return PCI_ERS_RESULT_DISCONNECT;
6613
9d5c8243
AK		 6614 if (netif_running(netdev))
6615 igb_down(adapter);
6616 pci_disable_device(pdev);
6617
6618 /* Request a slot slot reset. */
6619 return PCI_ERS_RESULT_NEED_RESET;
6620}
6621
6622/**
6623 * igb_io_slot_reset - called after the pci bus has been reset.
6624 * @pdev: Pointer to PCI device
6625 *
6626 * Restart the card from scratch, as if from a cold-boot. Implementation
6627 * resembles the first-half of the igb_resume routine.
6628 */
6629static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6630{
6631 struct net_device *netdev = pci_get_drvdata(pdev);
6632 struct igb_adapter *adapter = netdev_priv(netdev);
6633 struct e1000_hw *hw = &adapter->hw;
40a914fa 6634 pci_ers_result_t result;
42bfd33a 6635 int err;
9d5c8243 6636
aed5dec3 6637 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6638 dev_err(&pdev->dev,
6639 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6640 result = PCI_ERS_RESULT_DISCONNECT;
6641 } else {
6642 pci_set_master(pdev);
6643 pci_restore_state(pdev);
b94f2d77 6644 pci_save_state(pdev);
9d5c8243 6645
40a914fa
AD		 6646 pci_enable_wake(pdev, PCI_D3hot, 0);
6647 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6648
40a914fa
AD		 6649 igb_reset(adapter);
6650 wr32(E1000_WUS, ~0);
6651 result = PCI_ERS_RESULT_RECOVERED;
6652 }
9d5c8243 6653
ea943d41
JK		 6654 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6655 if (err) {
6656 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6657 "failed 0x%0x\n", err);
6658 /* non-fatal, continue */
6659 }
40a914fa
AD		 6660
6661 return result;
9d5c8243
AK		 6662}
6663
6664/**
6665 * igb_io_resume - called when traffic can start flowing again.
6666 * @pdev: Pointer to PCI device
6667 *
6668 * This callback is called when the error recovery driver tells us that
6669 * its OK to resume normal operation. Implementation resembles the
6670 * second-half of the igb_resume routine.
6671 */
6672static void igb_io_resume(struct pci_dev *pdev)
6673{
6674 struct net_device *netdev = pci_get_drvdata(pdev);
6675 struct igb_adapter *adapter = netdev_priv(netdev);
6676
9d5c8243
AK		 6677 if (netif_running(netdev)) {
6678 if (igb_up(adapter)) {
6679 dev_err(&pdev->dev, "igb_up failed after reset\n");
6680 return;
6681 }
6682 }
6683
6684 netif_device_attach(netdev);
6685
6686 /* let the f/w know that the h/w is now under the control of the
6687 * driver. */
6688 igb_get_hw_control(adapter);
9d5c8243
AK		 6689}
6690
26ad9178
AD		 6691static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6692 u8 qsel)
6693{
6694 u32 rar_low, rar_high;
6695 struct e1000_hw *hw = &adapter->hw;
6696
6697 /* HW expects these in little endian so we reverse the byte order
6698 * from network order (big endian) to little endian
6699 */
6700 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6701 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6702 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6703
6704 /* Indicate to hardware the Address is Valid. */
6705 rar_high |= E1000_RAH_AV;
6706
6707 if (hw->mac.type == e1000_82575)
6708 rar_high |= E1000_RAH_POOL_1 * qsel;
6709 else
6710 rar_high |= E1000_RAH_POOL_1 << qsel;
6711
6712 wr32(E1000_RAL(index), rar_low);
6713 wrfl();
6714 wr32(E1000_RAH(index), rar_high);
6715 wrfl();
6716}
6717
4ae196df
AD		 6718static int igb_set_vf_mac(struct igb_adapter *adapter,
6719 int vf, unsigned char *mac_addr)
6720{
6721 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 6722 /* VF MAC addresses start at end of receive addresses and moves
6723 * torwards the first, as a result a collision should not be possible */
6724 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 6725
37680117 6726 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 6727
26ad9178 6728 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 6729
6730 return 0;
6731}
6732
8151d294
WM		 6733static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6734{
6735 struct igb_adapter *adapter = netdev_priv(netdev);
6736 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6737 return -EINVAL;
6738 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6739 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6740 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6741 " change effective.");
6742 if (test_bit(__IGB_DOWN, &adapter->state)) {
6743 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6744 " but the PF device is not up.\n");
6745 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6746 " attempting to use the VF device.\n");
6747 }
6748 return igb_set_vf_mac(adapter, vf, mac);
6749}
6750
17dc566c
LL		 6751static int igb_link_mbps(int internal_link_speed)
6752{
6753 switch (internal_link_speed) {
6754 case SPEED_100:
6755 return 100;
6756 case SPEED_1000:
6757 return 1000;
6758 default:
6759 return 0;
6760 }
6761}
6762
6763static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
6764 int link_speed)
6765{
6766 int rf_dec, rf_int;
6767 u32 bcnrc_val;
6768
6769 if (tx_rate != 0) {
6770 /* Calculate the rate factor values to set */
6771 rf_int = link_speed / tx_rate;
6772 rf_dec = (link_speed - (rf_int * tx_rate));
6773 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
6774
6775 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
6776 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
6777 E1000_RTTBCNRC_RF_INT_MASK);
6778 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
6779 } else {
6780 bcnrc_val = 0;
6781 }
6782
6783 wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
6784 wr32(E1000_RTTBCNRC, bcnrc_val);
6785}
6786
6787static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
6788{
6789 int actual_link_speed, i;
6790 bool reset_rate = false;
6791
6792 /* VF TX rate limit was not set or not supported */
6793 if ((adapter->vf_rate_link_speed == 0) ||
6794 (adapter->hw.mac.type != e1000_82576))
6795 return;
6796
6797 actual_link_speed = igb_link_mbps(adapter->link_speed);
6798 if (actual_link_speed != adapter->vf_rate_link_speed) {
6799 reset_rate = true;
6800 adapter->vf_rate_link_speed = 0;
6801 dev_info(&adapter->pdev->dev,
6802 "Link speed has been changed. VF Transmit "
6803 "rate is disabled\n");
6804 }
6805
6806 for (i = 0; i < adapter->vfs_allocated_count; i++) {
6807 if (reset_rate)
6808 adapter->vf_data[i].tx_rate = 0;
6809
6810 igb_set_vf_rate_limit(&adapter->hw, i,
6811 adapter->vf_data[i].tx_rate,
6812 actual_link_speed);
6813 }
6814}
6815
8151d294
WM		 6816static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6817{
17dc566c
LL		 6818 struct igb_adapter *adapter = netdev_priv(netdev);
6819 struct e1000_hw *hw = &adapter->hw;
6820 int actual_link_speed;
6821
6822 if (hw->mac.type != e1000_82576)
6823 return -EOPNOTSUPP;
6824
6825 actual_link_speed = igb_link_mbps(adapter->link_speed);
6826 if ((vf >= adapter->vfs_allocated_count) ||
6827 (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
6828 (tx_rate < 0) || (tx_rate > actual_link_speed))
6829 return -EINVAL;
6830
6831 adapter->vf_rate_link_speed = actual_link_speed;
6832 adapter->vf_data[vf].tx_rate = (u16)tx_rate;
6833 igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
6834
6835 return 0;
8151d294
WM		 6836}
6837
6838static int igb_ndo_get_vf_config(struct net_device *netdev,
6839 int vf, struct ifla_vf_info *ivi)
6840{
6841 struct igb_adapter *adapter = netdev_priv(netdev);
6842 if (vf >= adapter->vfs_allocated_count)
6843 return -EINVAL;
6844 ivi->vf = vf;
6845 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
17dc566c 6846 ivi->tx_rate = adapter->vf_data[vf].tx_rate;
8151d294
WM		 6847 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6848 ivi->qos = adapter->vf_data[vf].pf_qos;
6849 return 0;
6850}
6851
4ae196df
AD		 6852static void igb_vmm_control(struct igb_adapter *adapter)
6853{
6854 struct e1000_hw *hw = &adapter->hw;
10d8e907 6855 u32 reg;
4ae196df 6856
52a1dd4d
AD		 6857 switch (hw->mac.type) {
6858 case e1000_82575:
6859 default:
6860 /* replication is not supported for 82575 */
4ae196df 6861 return;
52a1dd4d
AD		 6862 case e1000_82576:
6863 /* notify HW that the MAC is adding vlan tags */
6864 reg = rd32(E1000_DTXCTL);
6865 reg |= E1000_DTXCTL_VLAN_ADDED;
6866 wr32(E1000_DTXCTL, reg);
6867 case e1000_82580:
6868 /* enable replication vlan tag stripping */
6869 reg = rd32(E1000_RPLOLR);
6870 reg |= E1000_RPLOLR_STRVLAN;
6871 wr32(E1000_RPLOLR, reg);
d2ba2ed8
AD		 6872 case e1000_i350:
6873 /* none of the above registers are supported by i350 */
52a1dd4d
AD		 6874 break;
6875 }
10d8e907 6876
d4960307
AD		 6877 if (adapter->vfs_allocated_count) {
6878 igb_vmdq_set_loopback_pf(hw, true);
6879 igb_vmdq_set_replication_pf(hw, true);
13800469
GR		 6880 igb_vmdq_set_anti_spoofing_pf(hw, true,
6881 adapter->vfs_allocated_count);
d4960307
AD		 6882 } else {
6883 igb_vmdq_set_loopback_pf(hw, false);
6884 igb_vmdq_set_replication_pf(hw, false);
6885 }
4ae196df
AD		 6886}
6887
9d5c8243 6888/* igb_main.c */
ubuntu-zesty-kernel mirror