]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - drivers/net/ethernet/intel/igb/igb_main.c
projects / mirror_ubuntu-zesty-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
bonding: use ndo_change_rx_flags callback
[mirror_ubuntu-zesty-kernel.git] / drivers / net / ethernet / intel / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
4297f99b 4 Copyright(c) 2007-2011 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
b2cb09b1 31#include <linux/bitops.h>
9d5c8243
AK		 32#include <linux/vmalloc.h>
33#include <linux/pagemap.h>
34#include <linux/netdevice.h>
9d5c8243 35#include <linux/ipv6.h>
5a0e3ad6 36#include <linux/slab.h>
9d5c8243
AK		 37#include <net/checksum.h>
38#include <net/ip6_checksum.h>
c6cb090b 39#include <linux/net_tstamp.h>
9d5c8243
AK		 40#include <linux/mii.h>
41#include <linux/ethtool.h>
42#include <linux/if_vlan.h>
43#include <linux/pci.h>
c54106bb 44#include <linux/pci-aspm.h>
9d5c8243
AK		 45#include <linux/delay.h>
46#include <linux/interrupt.h>
47#include <linux/if_ether.h>
40a914fa 48#include <linux/aer.h>
70c71606 49#include <linux/prefetch.h>
421e02f0 50#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 51#include <linux/dca.h>
52#endif
9d5c8243
AK		 53#include "igb.h"
54
0d1fe82d
CW		 55#define MAJ 3
56#define MIN 0
57#define BUILD 6
0d1fe82d 58#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
929dd047 59__stringify(BUILD) "-k"
9d5c8243
AK		 60char igb_driver_name[] = "igb";
61char igb_driver_version[] = DRV_VERSION;
62static const char igb_driver_string[] =
63 "Intel(R) Gigabit Ethernet Network Driver";
4c4b42cb 64static const char igb_copyright[] = "Copyright (c) 2007-2011 Intel Corporation.";
9d5c8243 65
9d5c8243
AK		 66static const struct e1000_info *igb_info_tbl[] = {
67 [board_82575] = &e1000_82575_info,
68};
69
a3aa1884 70static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
d2ba2ed8
AD		 71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
55cac248
AD		 75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
6493d24f 77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
55cac248
AD		 78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
308fb39a
JG		 81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
1b5dda33
GJ		 83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
2d064c06 85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
b894fa26 91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
c8ea5ea9 92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
96 /* required last entry */
97 {0, }
98};
99
100MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
101
102void igb_reset(struct igb_adapter *);
103static int igb_setup_all_tx_resources(struct igb_adapter *);
104static int igb_setup_all_rx_resources(struct igb_adapter *);
105static void igb_free_all_tx_resources(struct igb_adapter *);
106static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 107static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 108static int igb_probe(struct pci_dev *, const struct pci_device_id *);
109static void __devexit igb_remove(struct pci_dev *pdev);
673b8b70 110static void igb_init_hw_timer(struct igb_adapter *adapter);
9d5c8243
AK		 111static int igb_sw_init(struct igb_adapter *);
112static int igb_open(struct net_device *);
113static int igb_close(struct net_device *);
114static void igb_configure_tx(struct igb_adapter *);
115static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 116static void igb_clean_all_tx_rings(struct igb_adapter *);
117static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 118static void igb_clean_tx_ring(struct igb_ring *);
119static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 120static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 121static void igb_update_phy_info(unsigned long);
122static void igb_watchdog(unsigned long);
123static void igb_watchdog_task(struct work_struct *);
b1a436c3 124static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
12dcd86b
ED		 125static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
126 struct rtnl_link_stats64 *stats);
9d5c8243
AK		 127static int igb_change_mtu(struct net_device *, int);
128static int igb_set_mac(struct net_device *, void *);
68d480c4 129static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 130static irqreturn_t igb_intr(int irq, void *);
131static irqreturn_t igb_intr_msi(int irq, void *);
132static irqreturn_t igb_msix_other(int irq, void *);
047e0030 133static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 134#ifdef CONFIG_IGB_DCA
047e0030 135static void igb_update_dca(struct igb_q_vector *);
fe4506b6 136static void igb_setup_dca(struct igb_adapter *);
421e02f0 137#endif /* CONFIG_IGB_DCA */
047e0030 138static bool igb_clean_tx_irq(struct igb_q_vector *);
661086df 139static int igb_poll(struct napi_struct *, int);
047e0030 140static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
9d5c8243
AK		 141static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
142static void igb_tx_timeout(struct net_device *);
143static void igb_reset_task(struct work_struct *);
b2cb09b1 144static void igb_vlan_mode(struct net_device *netdev, u32 features);
9d5c8243
AK		 145static void igb_vlan_rx_add_vid(struct net_device *, u16);
146static void igb_vlan_rx_kill_vid(struct net_device *, u16);
147static void igb_restore_vlan(struct igb_adapter *);
26ad9178 148static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 149static void igb_ping_all_vfs(struct igb_adapter *);
150static void igb_msg_task(struct igb_adapter *);
4ae196df 151static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 152static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 153static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 154static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
155static int igb_ndo_set_vf_vlan(struct net_device *netdev,
156 int vf, u16 vlan, u8 qos);
157static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
158static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
159 struct ifla_vf_info *ivi);
17dc566c 160static void igb_check_vf_rate_limit(struct igb_adapter *);
9d5c8243 161
9d5c8243 162#ifdef CONFIG_PM
3fe7c4c9 163static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 164static int igb_resume(struct pci_dev *);
165#endif
166static void igb_shutdown(struct pci_dev *);
421e02f0 167#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 168static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
169static struct notifier_block dca_notifier = {
170 .notifier_call = igb_notify_dca,
171 .next = NULL,
172 .priority = 0
173};
174#endif
9d5c8243
AK		 175#ifdef CONFIG_NET_POLL_CONTROLLER
176/* for netdump / net console */
177static void igb_netpoll(struct net_device *);
178#endif
37680117 179#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 180static unsigned int max_vfs = 0;
181module_param(max_vfs, uint, 0);
182MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
183 "per physical function");
184#endif /* CONFIG_PCI_IOV */
185
9d5c8243
AK		 186static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
187 pci_channel_state_t);
188static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
189static void igb_io_resume(struct pci_dev *);
190
191static struct pci_error_handlers igb_err_handler = {
192 .error_detected = igb_io_error_detected,
193 .slot_reset = igb_io_slot_reset,
194 .resume = igb_io_resume,
195};
196
197
198static struct pci_driver igb_driver = {
199 .name = igb_driver_name,
200 .id_table = igb_pci_tbl,
201 .probe = igb_probe,
202 .remove = __devexit_p(igb_remove),
203#ifdef CONFIG_PM
25985edc 204 /* Power Management Hooks */
9d5c8243
AK		 205 .suspend = igb_suspend,
206 .resume = igb_resume,
207#endif
208 .shutdown = igb_shutdown,
209 .err_handler = &igb_err_handler
210};
211
212MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
213MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
214MODULE_LICENSE("GPL");
215MODULE_VERSION(DRV_VERSION);
216
c97ec42a
TI		 217struct igb_reg_info {
218 u32 ofs;
219 char *name;
220};
221
222static const struct igb_reg_info igb_reg_info_tbl[] = {
223
224 /* General Registers */
225 {E1000_CTRL, "CTRL"},
226 {E1000_STATUS, "STATUS"},
227 {E1000_CTRL_EXT, "CTRL_EXT"},
228
229 /* Interrupt Registers */
230 {E1000_ICR, "ICR"},
231
232 /* RX Registers */
233 {E1000_RCTL, "RCTL"},
234 {E1000_RDLEN(0), "RDLEN"},
235 {E1000_RDH(0), "RDH"},
236 {E1000_RDT(0), "RDT"},
237 {E1000_RXDCTL(0), "RXDCTL"},
238 {E1000_RDBAL(0), "RDBAL"},
239 {E1000_RDBAH(0), "RDBAH"},
240
241 /* TX Registers */
242 {E1000_TCTL, "TCTL"},
243 {E1000_TDBAL(0), "TDBAL"},
244 {E1000_TDBAH(0), "TDBAH"},
245 {E1000_TDLEN(0), "TDLEN"},
246 {E1000_TDH(0), "TDH"},
247 {E1000_TDT(0), "TDT"},
248 {E1000_TXDCTL(0), "TXDCTL"},
249 {E1000_TDFH, "TDFH"},
250 {E1000_TDFT, "TDFT"},
251 {E1000_TDFHS, "TDFHS"},
252 {E1000_TDFPC, "TDFPC"},
253
254 /* List Terminator */
255 {}
256};
257
258/*
259 * igb_regdump - register printout routine
260 */
261static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
262{
263 int n = 0;
264 char rname[16];
265 u32 regs[8];
266
267 switch (reginfo->ofs) {
268 case E1000_RDLEN(0):
269 for (n = 0; n < 4; n++)
270 regs[n] = rd32(E1000_RDLEN(n));
271 break;
272 case E1000_RDH(0):
273 for (n = 0; n < 4; n++)
274 regs[n] = rd32(E1000_RDH(n));
275 break;
276 case E1000_RDT(0):
277 for (n = 0; n < 4; n++)
278 regs[n] = rd32(E1000_RDT(n));
279 break;
280 case E1000_RXDCTL(0):
281 for (n = 0; n < 4; n++)
282 regs[n] = rd32(E1000_RXDCTL(n));
283 break;
284 case E1000_RDBAL(0):
285 for (n = 0; n < 4; n++)
286 regs[n] = rd32(E1000_RDBAL(n));
287 break;
288 case E1000_RDBAH(0):
289 for (n = 0; n < 4; n++)
290 regs[n] = rd32(E1000_RDBAH(n));
291 break;
292 case E1000_TDBAL(0):
293 for (n = 0; n < 4; n++)
294 regs[n] = rd32(E1000_RDBAL(n));
295 break;
296 case E1000_TDBAH(0):
297 for (n = 0; n < 4; n++)
298 regs[n] = rd32(E1000_TDBAH(n));
299 break;
300 case E1000_TDLEN(0):
301 for (n = 0; n < 4; n++)
302 regs[n] = rd32(E1000_TDLEN(n));
303 break;
304 case E1000_TDH(0):
305 for (n = 0; n < 4; n++)
306 regs[n] = rd32(E1000_TDH(n));
307 break;
308 case E1000_TDT(0):
309 for (n = 0; n < 4; n++)
310 regs[n] = rd32(E1000_TDT(n));
311 break;
312 case E1000_TXDCTL(0):
313 for (n = 0; n < 4; n++)
314 regs[n] = rd32(E1000_TXDCTL(n));
315 break;
316 default:
317 printk(KERN_INFO "%-15s %08x\n",
318 reginfo->name, rd32(reginfo->ofs));
319 return;
320 }
321
322 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
323 printk(KERN_INFO "%-15s ", rname);
324 for (n = 0; n < 4; n++)
325 printk(KERN_CONT "%08x ", regs[n]);
326 printk(KERN_CONT "\n");
327}
328
329/*
330 * igb_dump - Print registers, tx-rings and rx-rings
331 */
332static void igb_dump(struct igb_adapter *adapter)
333{
334 struct net_device *netdev = adapter->netdev;
335 struct e1000_hw *hw = &adapter->hw;
336 struct igb_reg_info *reginfo;
337 int n = 0;
338 struct igb_ring *tx_ring;
339 union e1000_adv_tx_desc *tx_desc;
340 struct my_u0 { u64 a; u64 b; } *u0;
341 struct igb_buffer *buffer_info;
342 struct igb_ring *rx_ring;
343 union e1000_adv_rx_desc *rx_desc;
344 u32 staterr;
345 int i = 0;
346
347 if (!netif_msg_hw(adapter))
348 return;
349
350 /* Print netdevice Info */
351 if (netdev) {
352 dev_info(&adapter->pdev->dev, "Net device Info\n");
353 printk(KERN_INFO "Device Name state "
354 "trans_start last_rx\n");
355 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
356 netdev->name,
357 netdev->state,
358 netdev->trans_start,
359 netdev->last_rx);
360 }
361
362 /* Print Registers */
363 dev_info(&adapter->pdev->dev, "Register Dump\n");
364 printk(KERN_INFO " Register Name Value\n");
365 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
366 reginfo->name; reginfo++) {
367 igb_regdump(hw, reginfo);
368 }
369
370 /* Print TX Ring Summary */
371 if (!netdev || !netif_running(netdev))
372 goto exit;
373
374 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
375 printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma ]"
376 " leng ntw timestamp\n");
377 for (n = 0; n < adapter->num_tx_queues; n++) {
378 tx_ring = adapter->tx_ring[n];
379 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
380 printk(KERN_INFO " %5d %5X %5X %016llX %04X %3X %016llX\n",
381 n, tx_ring->next_to_use, tx_ring->next_to_clean,
382 (u64)buffer_info->dma,
383 buffer_info->length,
384 buffer_info->next_to_watch,
385 (u64)buffer_info->time_stamp);
386 }
387
388 /* Print TX Rings */
389 if (!netif_msg_tx_done(adapter))
390 goto rx_ring_summary;
391
392 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
393
394 /* Transmit Descriptor Formats
395 *
396 * Advanced Transmit Descriptor
397 * +--------------------------------------------------------------+
398 * 0 | Buffer Address [63:0] |
399 * +--------------------------------------------------------------+
400 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
401 * +--------------------------------------------------------------+
402 * 63 46 45 40 39 38 36 35 32 31 24 15 0
403 */
404
405 for (n = 0; n < adapter->num_tx_queues; n++) {
406 tx_ring = adapter->tx_ring[n];
407 printk(KERN_INFO "------------------------------------\n");
408 printk(KERN_INFO "TX QUEUE INDEX = %d\n", tx_ring->queue_index);
409 printk(KERN_INFO "------------------------------------\n");
410 printk(KERN_INFO "T [desc] [address 63:0 ] "
411 "[PlPOCIStDDM Ln] [bi->dma ] "
412 "leng ntw timestamp bi->skb\n");
413
414 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
415 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
416 buffer_info = &tx_ring->buffer_info[i];
417 u0 = (struct my_u0 *)tx_desc;
418 printk(KERN_INFO "T [0x%03X] %016llX %016llX %016llX"
419 " %04X %3X %016llX %p", i,
420 le64_to_cpu(u0->a),
421 le64_to_cpu(u0->b),
422 (u64)buffer_info->dma,
423 buffer_info->length,
424 buffer_info->next_to_watch,
425 (u64)buffer_info->time_stamp,
426 buffer_info->skb);
427 if (i == tx_ring->next_to_use &&
428 i == tx_ring->next_to_clean)
429 printk(KERN_CONT " NTC/U\n");
430 else if (i == tx_ring->next_to_use)
431 printk(KERN_CONT " NTU\n");
432 else if (i == tx_ring->next_to_clean)
433 printk(KERN_CONT " NTC\n");
434 else
435 printk(KERN_CONT "\n");
436
437 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
438 print_hex_dump(KERN_INFO, "",
439 DUMP_PREFIX_ADDRESS,
440 16, 1, phys_to_virt(buffer_info->dma),
441 buffer_info->length, true);
442 }
443 }
444
445 /* Print RX Rings Summary */
446rx_ring_summary:
447 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
448 printk(KERN_INFO "Queue [NTU] [NTC]\n");
449 for (n = 0; n < adapter->num_rx_queues; n++) {
450 rx_ring = adapter->rx_ring[n];
451 printk(KERN_INFO " %5d %5X %5X\n", n,
452 rx_ring->next_to_use, rx_ring->next_to_clean);
453 }
454
455 /* Print RX Rings */
456 if (!netif_msg_rx_status(adapter))
457 goto exit;
458
459 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
460
461 /* Advanced Receive Descriptor (Read) Format
462 * 63 1 0
463 * +-----------------------------------------------------+
464 * 0 | Packet Buffer Address [63:1] |A0/NSE|
465 * +----------------------------------------------+------+
466 * 8 | Header Buffer Address [63:1] | DD |
467 * +-----------------------------------------------------+
468 *
469 *
470 * Advanced Receive Descriptor (Write-Back) Format
471 *
472 * 63 48 47 32 31 30 21 20 17 16 4 3 0
473 * +------------------------------------------------------+
474 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
475 * | Checksum Ident | | | | Type | Type |
476 * +------------------------------------------------------+
477 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
478 * +------------------------------------------------------+
479 * 63 48 47 32 31 20 19 0
480 */
481
482 for (n = 0; n < adapter->num_rx_queues; n++) {
483 rx_ring = adapter->rx_ring[n];
484 printk(KERN_INFO "------------------------------------\n");
485 printk(KERN_INFO "RX QUEUE INDEX = %d\n", rx_ring->queue_index);
486 printk(KERN_INFO "------------------------------------\n");
487 printk(KERN_INFO "R [desc] [ PktBuf A0] "
488 "[ HeadBuf DD] [bi->dma ] [bi->skb] "
489 "<-- Adv Rx Read format\n");
490 printk(KERN_INFO "RWB[desc] [PcsmIpSHl PtRs] "
491 "[vl er S cks ln] ---------------- [bi->skb] "
492 "<-- Adv Rx Write-Back format\n");
493
494 for (i = 0; i < rx_ring->count; i++) {
495 buffer_info = &rx_ring->buffer_info[i];
496 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
497 u0 = (struct my_u0 *)rx_desc;
498 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
499 if (staterr & E1000_RXD_STAT_DD) {
500 /* Descriptor Done */
501 printk(KERN_INFO "RWB[0x%03X] %016llX "
502 "%016llX ---------------- %p", i,
503 le64_to_cpu(u0->a),
504 le64_to_cpu(u0->b),
505 buffer_info->skb);
506 } else {
507 printk(KERN_INFO "R [0x%03X] %016llX "
508 "%016llX %016llX %p", i,
509 le64_to_cpu(u0->a),
510 le64_to_cpu(u0->b),
511 (u64)buffer_info->dma,
512 buffer_info->skb);
513
514 if (netif_msg_pktdata(adapter)) {
515 print_hex_dump(KERN_INFO, "",
516 DUMP_PREFIX_ADDRESS,
517 16, 1,
518 phys_to_virt(buffer_info->dma),
519 rx_ring->rx_buffer_len, true);
520 if (rx_ring->rx_buffer_len
521 < IGB_RXBUFFER_1024)
522 print_hex_dump(KERN_INFO, "",
523 DUMP_PREFIX_ADDRESS,
524 16, 1,
525 phys_to_virt(
526 buffer_info->page_dma +
527 buffer_info->page_offset),
528 PAGE_SIZE/2, true);
529 }
530 }
531
532 if (i == rx_ring->next_to_use)
533 printk(KERN_CONT " NTU\n");
534 else if (i == rx_ring->next_to_clean)
535 printk(KERN_CONT " NTC\n");
536 else
537 printk(KERN_CONT "\n");
538
539 }
540 }
541
542exit:
543 return;
544}
545
546
38c845c7
PO		 547/**
548 * igb_read_clock - read raw cycle counter (to be used by time counter)
549 */
550static cycle_t igb_read_clock(const struct cyclecounter *tc)
551{
552 struct igb_adapter *adapter =
553 container_of(tc, struct igb_adapter, cycles);
554 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 555 u64 stamp = 0;
556 int shift = 0;
38c845c7 557
55cac248
AD		 558 /*
559 * The timestamp latches on lowest register read. For the 82580
560 * the lowest register is SYSTIMR instead of SYSTIML. However we never
561 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
562 */
563 if (hw->mac.type == e1000_82580) {
564 stamp = rd32(E1000_SYSTIMR) >> 8;
565 shift = IGB_82580_TSYNC_SHIFT;
566 }
567
c5b9bd5e
AD		 568 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
569 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 570 return stamp;
571}
572
9d5c8243 573/**
c041076a 574 * igb_get_hw_dev - return device
9d5c8243
AK		 575 * used by hardware layer to print debugging information
576 **/
c041076a 577struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
9d5c8243
AK		 578{
579 struct igb_adapter *adapter = hw->back;
c041076a 580 return adapter->netdev;
9d5c8243 581}
38c845c7 582
9d5c8243
AK		 583/**
584 * igb_init_module - Driver Registration Routine
585 *
586 * igb_init_module is the first routine called when the driver is
587 * loaded. All it does is register with the PCI subsystem.
588 **/
589static int __init igb_init_module(void)
590{
591 int ret;
592 printk(KERN_INFO "%s - version %s\n",
593 igb_driver_string, igb_driver_version);
594
595 printk(KERN_INFO "%s\n", igb_copyright);
596
421e02f0 597#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 598 dca_register_notify(&dca_notifier);
599#endif
bbd98fe4 600 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 601 return ret;
602}
603
604module_init(igb_init_module);
605
606/**
607 * igb_exit_module - Driver Exit Cleanup Routine
608 *
609 * igb_exit_module is called just before the driver is removed
610 * from memory.
611 **/
612static void __exit igb_exit_module(void)
613{
421e02f0 614#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 615 dca_unregister_notify(&dca_notifier);
616#endif
9d5c8243
AK		 617 pci_unregister_driver(&igb_driver);
618}
619
620module_exit(igb_exit_module);
621
26bc19ec
AD		 622#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
623/**
624 * igb_cache_ring_register - Descriptor ring to register mapping
625 * @adapter: board private structure to initialize
626 *
627 * Once we know the feature-set enabled for the device, we'll cache
628 * the register offset the descriptor ring is assigned to.
629 **/
630static void igb_cache_ring_register(struct igb_adapter *adapter)
631{
ee1b9f06 632 int i = 0, j = 0;
047e0030 633 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 634
635 switch (adapter->hw.mac.type) {
636 case e1000_82576:
637 /* The queues are allocated for virtualization such that VF 0
638 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
639 * In order to avoid collision we start at the first free queue
640 * and continue consuming queues in the same sequence
641 */
ee1b9f06 642 if (adapter->vfs_allocated_count) {
a99955fc 643 for (; i < adapter->rss_queues; i++)
3025a446
AD		 644 adapter->rx_ring[i]->reg_idx = rbase_offset +
645 Q_IDX_82576(i);
ee1b9f06 646 }
26bc19ec 647 case e1000_82575:
55cac248 648 case e1000_82580:
d2ba2ed8 649 case e1000_i350:
26bc19ec 650 default:
ee1b9f06 651 for (; i < adapter->num_rx_queues; i++)
3025a446 652 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 653 for (; j < adapter->num_tx_queues; j++)
3025a446 654 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 655 break;
656 }
657}
658
047e0030
AD		 659static void igb_free_queues(struct igb_adapter *adapter)
660{
3025a446 661 int i;
047e0030 662
3025a446
AD		 663 for (i = 0; i < adapter->num_tx_queues; i++) {
664 kfree(adapter->tx_ring[i]);
665 adapter->tx_ring[i] = NULL;
666 }
667 for (i = 0; i < adapter->num_rx_queues; i++) {
668 kfree(adapter->rx_ring[i]);
669 adapter->rx_ring[i] = NULL;
670 }
047e0030
AD		 671 adapter->num_rx_queues = 0;
672 adapter->num_tx_queues = 0;
673}
674
9d5c8243
AK		 675/**
676 * igb_alloc_queues - Allocate memory for all rings
677 * @adapter: board private structure to initialize
678 *
679 * We allocate one ring per queue at run-time since we don't know the
680 * number of queues at compile-time.
681 **/
682static int igb_alloc_queues(struct igb_adapter *adapter)
683{
3025a446 684 struct igb_ring *ring;
9d5c8243
AK		 685 int i;
686
661086df 687 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 688 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
689 if (!ring)
690 goto err;
68fd9910 691 ring->count = adapter->tx_ring_count;
661086df 692 ring->queue_index = i;
59d71989 693 ring->dev = &adapter->pdev->dev;
e694e964 694 ring->netdev = adapter->netdev;
85ad76b2
AD		 695 /* For 82575, context index must be unique per ring. */
696 if (adapter->hw.mac.type == e1000_82575)
697 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
3025a446 698 adapter->tx_ring[i] = ring;
661086df 699 }
85ad76b2 700
9d5c8243 701 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446
AD		 702 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
703 if (!ring)
704 goto err;
68fd9910 705 ring->count = adapter->rx_ring_count;
844290e5 706 ring->queue_index = i;
59d71989 707 ring->dev = &adapter->pdev->dev;
e694e964 708 ring->netdev = adapter->netdev;
4c844851 709 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85ad76b2
AD		 710 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
711 /* set flag indicating ring supports SCTP checksum offload */
712 if (adapter->hw.mac.type >= e1000_82576)
713 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
3025a446 714 adapter->rx_ring[i] = ring;
9d5c8243 715 }
26bc19ec
AD		 716
717 igb_cache_ring_register(adapter);
9d5c8243 718
047e0030 719 return 0;
a88f10ec 720
047e0030
AD		 721err:
722 igb_free_queues(adapter);
d1a8c9e1 723
047e0030 724 return -ENOMEM;
a88f10ec
AD		 725}
726
9d5c8243 727#define IGB_N0_QUEUE -1
047e0030 728static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 729{
730 u32 msixbm = 0;
047e0030 731 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 732 struct e1000_hw *hw = &adapter->hw;
2d064c06 733 u32 ivar, index;
047e0030
AD		 734 int rx_queue = IGB_N0_QUEUE;
735 int tx_queue = IGB_N0_QUEUE;
736
737 if (q_vector->rx_ring)
738 rx_queue = q_vector->rx_ring->reg_idx;
739 if (q_vector->tx_ring)
740 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 741
742 switch (hw->mac.type) {
743 case e1000_82575:
9d5c8243
AK		 744 /* The 82575 assigns vectors using a bitmask, which matches the
745 bitmask for the EICR/EIMS/EIMC registers. To assign one
746 or more queues to a vector, we write the appropriate bits
747 into the MSIXBM register for that vector. */
047e0030 748 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 749 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 750 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 751 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 752 if (!adapter->msix_entries && msix_vector == 0)
753 msixbm |= E1000_EIMS_OTHER;
9d5c8243 754 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 755 q_vector->eims_value = msixbm;
2d064c06
AD		 756 break;
757 case e1000_82576:
26bc19ec 758 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 759 Each queue has a single entry in the table to which we write
760 a vector number along with a "valid" bit. Sadly, the layout
761 of the table is somewhat counterintuitive. */
762 if (rx_queue > IGB_N0_QUEUE) {
047e0030 763 index = (rx_queue & 0x7);
2d064c06 764 ivar = array_rd32(E1000_IVAR0, index);
047e0030 765 if (rx_queue < 8) {
26bc19ec
AD		 766 /* vector goes into low byte of register */
767 ivar = ivar & 0xFFFFFF00;
768 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 769 } else {
770 /* vector goes into third byte of register */
771 ivar = ivar & 0xFF00FFFF;
772 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 773 }
2d064c06
AD		 774 array_wr32(E1000_IVAR0, index, ivar);
775 }
776 if (tx_queue > IGB_N0_QUEUE) {
047e0030 777 index = (tx_queue & 0x7);
2d064c06 778 ivar = array_rd32(E1000_IVAR0, index);
047e0030 779 if (tx_queue < 8) {
26bc19ec
AD		 780 /* vector goes into second byte of register */
781 ivar = ivar & 0xFFFF00FF;
782 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 783 } else {
784 /* vector goes into high byte of register */
785 ivar = ivar & 0x00FFFFFF;
786 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 787 }
2d064c06
AD		 788 array_wr32(E1000_IVAR0, index, ivar);
789 }
047e0030 790 q_vector->eims_value = 1 << msix_vector;
2d064c06 791 break;
55cac248 792 case e1000_82580:
d2ba2ed8 793 case e1000_i350:
55cac248
AD		 794 /* 82580 uses the same table-based approach as 82576 but has fewer
795 entries as a result we carry over for queues greater than 4. */
796 if (rx_queue > IGB_N0_QUEUE) {
797 index = (rx_queue >> 1);
798 ivar = array_rd32(E1000_IVAR0, index);
799 if (rx_queue & 0x1) {
800 /* vector goes into third byte of register */
801 ivar = ivar & 0xFF00FFFF;
802 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
803 } else {
804 /* vector goes into low byte of register */
805 ivar = ivar & 0xFFFFFF00;
806 ivar |= msix_vector | E1000_IVAR_VALID;
807 }
808 array_wr32(E1000_IVAR0, index, ivar);
809 }
810 if (tx_queue > IGB_N0_QUEUE) {
811 index = (tx_queue >> 1);
812 ivar = array_rd32(E1000_IVAR0, index);
813 if (tx_queue & 0x1) {
814 /* vector goes into high byte of register */
815 ivar = ivar & 0x00FFFFFF;
816 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
817 } else {
818 /* vector goes into second byte of register */
819 ivar = ivar & 0xFFFF00FF;
820 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
821 }
822 array_wr32(E1000_IVAR0, index, ivar);
823 }
824 q_vector->eims_value = 1 << msix_vector;
825 break;
2d064c06
AD		 826 default:
827 BUG();
828 break;
829 }
26b39276
AD		 830
831 /* add q_vector eims value to global eims_enable_mask */
832 adapter->eims_enable_mask |= q_vector->eims_value;
833
834 /* configure q_vector to set itr on first interrupt */
835 q_vector->set_itr = 1;
9d5c8243
AK		 836}
837
838/**
839 * igb_configure_msix - Configure MSI-X hardware
840 *
841 * igb_configure_msix sets up the hardware to properly
842 * generate MSI-X interrupts.
843 **/
844static void igb_configure_msix(struct igb_adapter *adapter)
845{
846 u32 tmp;
847 int i, vector = 0;
848 struct e1000_hw *hw = &adapter->hw;
849
850 adapter->eims_enable_mask = 0;
9d5c8243
AK		 851
852 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 853 switch (hw->mac.type) {
854 case e1000_82575:
9d5c8243
AK		 855 tmp = rd32(E1000_CTRL_EXT);
856 /* enable MSI-X PBA support*/
857 tmp |= E1000_CTRL_EXT_PBA_CLR;
858
859 /* Auto-Mask interrupts upon ICR read. */
860 tmp |= E1000_CTRL_EXT_EIAME;
861 tmp |= E1000_CTRL_EXT_IRCA;
862
863 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 864
865 /* enable msix_other interrupt */
866 array_wr32(E1000_MSIXBM(0), vector++,
867 E1000_EIMS_OTHER);
844290e5 868 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 869
2d064c06
AD		 870 break;
871
872 case e1000_82576:
55cac248 873 case e1000_82580:
d2ba2ed8 874 case e1000_i350:
047e0030
AD		 875 /* Turn on MSI-X capability first, or our settings
876 * won't stick. And it will take days to debug. */
877 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
878 E1000_GPIE_PBA | E1000_GPIE_EIAME |
879 E1000_GPIE_NSICR);
880
881 /* enable msix_other interrupt */
882 adapter->eims_other = 1 << vector;
2d064c06 883 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 884
047e0030 885 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 886 break;
887 default:
888 /* do nothing, since nothing else supports MSI-X */
889 break;
890 } /* switch (hw->mac.type) */
047e0030
AD		 891
892 adapter->eims_enable_mask |= adapter->eims_other;
893
26b39276
AD		 894 for (i = 0; i < adapter->num_q_vectors; i++)
895 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 896
9d5c8243
AK		 897 wrfl();
898}
899
900/**
901 * igb_request_msix - Initialize MSI-X interrupts
902 *
903 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
904 * kernel.
905 **/
906static int igb_request_msix(struct igb_adapter *adapter)
907{
908 struct net_device *netdev = adapter->netdev;
047e0030 909 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 910 int i, err = 0, vector = 0;
911
047e0030 912 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 913 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 914 if (err)
915 goto out;
916 vector++;
917
918 for (i = 0; i < adapter->num_q_vectors; i++) {
919 struct igb_q_vector *q_vector = adapter->q_vector[i];
920
921 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
922
923 if (q_vector->rx_ring && q_vector->tx_ring)
924 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
925 q_vector->rx_ring->queue_index);
926 else if (q_vector->tx_ring)
927 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
928 q_vector->tx_ring->queue_index);
929 else if (q_vector->rx_ring)
930 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
931 q_vector->rx_ring->queue_index);
9d5c8243 932 else
047e0030
AD		 933 sprintf(q_vector->name, "%s-unused", netdev->name);
934
9d5c8243 935 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 936 igb_msix_ring, 0, q_vector->name,
047e0030 937 q_vector);
9d5c8243
AK		 938 if (err)
939 goto out;
9d5c8243
AK		 940 vector++;
941 }
942
9d5c8243
AK		 943 igb_configure_msix(adapter);
944 return 0;
945out:
946 return err;
947}
948
949static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
950{
951 if (adapter->msix_entries) {
952 pci_disable_msix(adapter->pdev);
953 kfree(adapter->msix_entries);
954 adapter->msix_entries = NULL;
047e0030 955 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 956 pci_disable_msi(adapter->pdev);
047e0030 957 }
9d5c8243
AK		 958}
959
047e0030
AD		 960/**
961 * igb_free_q_vectors - Free memory allocated for interrupt vectors
962 * @adapter: board private structure to initialize
963 *
964 * This function frees the memory allocated to the q_vectors. In addition if
965 * NAPI is enabled it will delete any references to the NAPI struct prior
966 * to freeing the q_vector.
967 **/
968static void igb_free_q_vectors(struct igb_adapter *adapter)
969{
970 int v_idx;
971
972 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
973 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
974 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 975 if (!q_vector)
976 continue;
047e0030
AD		 977 netif_napi_del(&q_vector->napi);
978 kfree(q_vector);
979 }
980 adapter->num_q_vectors = 0;
981}
982
983/**
984 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
985 *
986 * This function resets the device so that it has 0 rx queues, tx queues, and
987 * MSI-X interrupts allocated.
988 */
989static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
990{
991 igb_free_queues(adapter);
992 igb_free_q_vectors(adapter);
993 igb_reset_interrupt_capability(adapter);
994}
9d5c8243
AK		 995
996/**
997 * igb_set_interrupt_capability - set MSI or MSI-X if supported
998 *
999 * Attempt to configure interrupts using the best available
1000 * capabilities of the hardware and kernel.
1001 **/
21adef3e 1002static int igb_set_interrupt_capability(struct igb_adapter *adapter)
9d5c8243
AK		 1003{
1004 int err;
1005 int numvecs, i;
1006
83b7180d 1007 /* Number of supported queues. */
a99955fc 1008 adapter->num_rx_queues = adapter->rss_queues;
5fa8517f
GR		 1009 if (adapter->vfs_allocated_count)
1010 adapter->num_tx_queues = 1;
1011 else
1012 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 1013
047e0030
AD		 1014 /* start with one vector for every rx queue */
1015 numvecs = adapter->num_rx_queues;
1016
3ad2f3fb 1017 /* if tx handler is separate add 1 for every tx queue */
a99955fc
AD		 1018 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1019 numvecs += adapter->num_tx_queues;
047e0030
AD		 1020
1021 /* store the number of vectors reserved for queues */
1022 adapter->num_q_vectors = numvecs;
1023
1024 /* add 1 vector for link status interrupts */
1025 numvecs++;
9d5c8243
AK		 1026 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1027 GFP_KERNEL);
1028 if (!adapter->msix_entries)
1029 goto msi_only;
1030
1031 for (i = 0; i < numvecs; i++)
1032 adapter->msix_entries[i].entry = i;
1033
1034 err = pci_enable_msix(adapter->pdev,
1035 adapter->msix_entries,
1036 numvecs);
1037 if (err == 0)
34a20e89 1038 goto out;
9d5c8243
AK		 1039
1040 igb_reset_interrupt_capability(adapter);
1041
1042 /* If we can't do MSI-X, try MSI */
1043msi_only:
2a3abf6d
AD		 1044#ifdef CONFIG_PCI_IOV
1045 /* disable SR-IOV for non MSI-X configurations */
1046 if (adapter->vf_data) {
1047 struct e1000_hw *hw = &adapter->hw;
1048 /* disable iov and allow time for transactions to clear */
1049 pci_disable_sriov(adapter->pdev);
1050 msleep(500);
1051
1052 kfree(adapter->vf_data);
1053 adapter->vf_data = NULL;
1054 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 1055 wrfl();
2a3abf6d
AD		 1056 msleep(100);
1057 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1058 }
1059#endif
4fc82adf 1060 adapter->vfs_allocated_count = 0;
a99955fc 1061 adapter->rss_queues = 1;
4fc82adf 1062 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 1063 adapter->num_rx_queues = 1;
661086df 1064 adapter->num_tx_queues = 1;
047e0030 1065 adapter->num_q_vectors = 1;
9d5c8243 1066 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1067 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 1068out:
21adef3e
BH		 1069 /* Notify the stack of the (possibly) reduced queue counts. */
1070 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
1071 return netif_set_real_num_rx_queues(adapter->netdev,
1072 adapter->num_rx_queues);
9d5c8243
AK		 1073}
1074
047e0030
AD		 1075/**
1076 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1077 * @adapter: board private structure to initialize
1078 *
1079 * We allocate one q_vector per queue interrupt. If allocation fails we
1080 * return -ENOMEM.
1081 **/
1082static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1083{
1084 struct igb_q_vector *q_vector;
1085 struct e1000_hw *hw = &adapter->hw;
1086 int v_idx;
1087
1088 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1089 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
1090 if (!q_vector)
1091 goto err_out;
1092 q_vector->adapter = adapter;
047e0030
AD		 1093 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1094 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1095 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1096 adapter->q_vector[v_idx] = q_vector;
1097 }
1098 return 0;
1099
1100err_out:
fe0592b4 1101 igb_free_q_vectors(adapter);
047e0030
AD		 1102 return -ENOMEM;
1103}
1104
1105static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1106 int ring_idx, int v_idx)
1107{
3025a446 1108 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1109
3025a446 1110 q_vector->rx_ring = adapter->rx_ring[ring_idx];
047e0030 1111 q_vector->rx_ring->q_vector = q_vector;
4fc82adf
AD		 1112 q_vector->itr_val = adapter->rx_itr_setting;
1113 if (q_vector->itr_val && q_vector->itr_val <= 3)
1114 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1115}
1116
1117static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1118 int ring_idx, int v_idx)
1119{
3025a446 1120 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1121
3025a446 1122 q_vector->tx_ring = adapter->tx_ring[ring_idx];
047e0030 1123 q_vector->tx_ring->q_vector = q_vector;
4fc82adf
AD		 1124 q_vector->itr_val = adapter->tx_itr_setting;
1125 if (q_vector->itr_val && q_vector->itr_val <= 3)
1126 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1127}
1128
1129/**
1130 * igb_map_ring_to_vector - maps allocated queues to vectors
1131 *
1132 * This function maps the recently allocated queues to vectors.
1133 **/
1134static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1135{
1136 int i;
1137 int v_idx = 0;
1138
1139 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1140 (adapter->num_q_vectors < adapter->num_tx_queues))
1141 return -ENOMEM;
1142
1143 if (adapter->num_q_vectors >=
1144 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1145 for (i = 0; i < adapter->num_rx_queues; i++)
1146 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1147 for (i = 0; i < adapter->num_tx_queues; i++)
1148 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1149 } else {
1150 for (i = 0; i < adapter->num_rx_queues; i++) {
1151 if (i < adapter->num_tx_queues)
1152 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1153 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1154 }
1155 for (; i < adapter->num_tx_queues; i++)
1156 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1157 }
1158 return 0;
1159}
1160
1161/**
1162 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1163 *
1164 * This function initializes the interrupts and allocates all of the queues.
1165 **/
1166static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1167{
1168 struct pci_dev *pdev = adapter->pdev;
1169 int err;
1170
21adef3e
BH		 1171 err = igb_set_interrupt_capability(adapter);
1172 if (err)
1173 return err;
047e0030
AD		 1174
1175 err = igb_alloc_q_vectors(adapter);
1176 if (err) {
1177 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1178 goto err_alloc_q_vectors;
1179 }
1180
1181 err = igb_alloc_queues(adapter);
1182 if (err) {
1183 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1184 goto err_alloc_queues;
1185 }
1186
1187 err = igb_map_ring_to_vector(adapter);
1188 if (err) {
1189 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1190 goto err_map_queues;
1191 }
1192
1193
1194 return 0;
1195err_map_queues:
1196 igb_free_queues(adapter);
1197err_alloc_queues:
1198 igb_free_q_vectors(adapter);
1199err_alloc_q_vectors:
1200 igb_reset_interrupt_capability(adapter);
1201 return err;
1202}
1203
9d5c8243
AK		 1204/**
1205 * igb_request_irq - initialize interrupts
1206 *
1207 * Attempts to configure interrupts using the best available
1208 * capabilities of the hardware and kernel.
1209 **/
1210static int igb_request_irq(struct igb_adapter *adapter)
1211{
1212 struct net_device *netdev = adapter->netdev;
047e0030 1213 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1214 int err = 0;
1215
1216 if (adapter->msix_entries) {
1217 err = igb_request_msix(adapter);
844290e5 1218 if (!err)
9d5c8243 1219 goto request_done;
9d5c8243 1220 /* fall back to MSI */
047e0030 1221 igb_clear_interrupt_scheme(adapter);
9d5c8243 1222 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1223 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1224 igb_free_all_tx_resources(adapter);
1225 igb_free_all_rx_resources(adapter);
047e0030 1226 adapter->num_tx_queues = 1;
9d5c8243 1227 adapter->num_rx_queues = 1;
047e0030
AD		 1228 adapter->num_q_vectors = 1;
1229 err = igb_alloc_q_vectors(adapter);
1230 if (err) {
1231 dev_err(&pdev->dev,
1232 "Unable to allocate memory for vectors\n");
1233 goto request_done;
1234 }
1235 err = igb_alloc_queues(adapter);
1236 if (err) {
1237 dev_err(&pdev->dev,
1238 "Unable to allocate memory for queues\n");
1239 igb_free_q_vectors(adapter);
1240 goto request_done;
1241 }
1242 igb_setup_all_tx_resources(adapter);
1243 igb_setup_all_rx_resources(adapter);
844290e5 1244 } else {
feeb2721 1245 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243 1246 }
844290e5 1247
7dfc16fa 1248 if (adapter->flags & IGB_FLAG_HAS_MSI) {
a0607fd3 1249 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
047e0030 1250 netdev->name, adapter);
9d5c8243
AK		 1251 if (!err)
1252 goto request_done;
047e0030 1253
9d5c8243
AK		 1254 /* fall back to legacy interrupts */
1255 igb_reset_interrupt_capability(adapter);
7dfc16fa 1256 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1257 }
1258
a0607fd3 1259 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
047e0030 1260 netdev->name, adapter);
9d5c8243 1261
6cb5e577 1262 if (err)
9d5c8243
AK		 1263 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
1264 err);
9d5c8243
AK		 1265
1266request_done:
1267 return err;
1268}
1269
1270static void igb_free_irq(struct igb_adapter *adapter)
1271{
9d5c8243
AK		 1272 if (adapter->msix_entries) {
1273 int vector = 0, i;
1274
047e0030 1275 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 1276
047e0030
AD		 1277 for (i = 0; i < adapter->num_q_vectors; i++) {
1278 struct igb_q_vector *q_vector = adapter->q_vector[i];
1279 free_irq(adapter->msix_entries[vector++].vector,
1280 q_vector);
1281 }
1282 } else {
1283 free_irq(adapter->pdev->irq, adapter);
9d5c8243 1284 }
9d5c8243
AK		 1285}
1286
1287/**
1288 * igb_irq_disable - Mask off interrupt generation on the NIC
1289 * @adapter: board private structure
1290 **/
1291static void igb_irq_disable(struct igb_adapter *adapter)
1292{
1293 struct e1000_hw *hw = &adapter->hw;
1294
25568a53
AD		 1295 /*
1296 * we need to be careful when disabling interrupts. The VFs are also
1297 * mapped into these registers and so clearing the bits can cause
1298 * issues on the VF drivers so we only need to clear what we set
1299 */
9d5c8243 1300 if (adapter->msix_entries) {
2dfd1212
AD		 1301 u32 regval = rd32(E1000_EIAM);
1302 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1303 wr32(E1000_EIMC, adapter->eims_enable_mask);
1304 regval = rd32(E1000_EIAC);
1305 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 1306 }
844290e5
PW		 1307
1308 wr32(E1000_IAM, 0);
9d5c8243
AK		 1309 wr32(E1000_IMC, ~0);
1310 wrfl();
81a61859
ET		 1311 if (adapter->msix_entries) {
1312 int i;
1313 for (i = 0; i < adapter->num_q_vectors; i++)
1314 synchronize_irq(adapter->msix_entries[i].vector);
1315 } else {
1316 synchronize_irq(adapter->pdev->irq);
1317 }
9d5c8243
AK		 1318}
1319
1320/**
1321 * igb_irq_enable - Enable default interrupt generation settings
1322 * @adapter: board private structure
1323 **/
1324static void igb_irq_enable(struct igb_adapter *adapter)
1325{
1326 struct e1000_hw *hw = &adapter->hw;
1327
1328 if (adapter->msix_entries) {
25568a53 1329 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
2dfd1212
AD		 1330 u32 regval = rd32(E1000_EIAC);
1331 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1332 regval = rd32(E1000_EIAM);
1333 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 1334 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 1335 if (adapter->vfs_allocated_count) {
4ae196df 1336 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1337 ims |= E1000_IMS_VMMB;
1338 }
55cac248
AD		 1339 if (adapter->hw.mac.type == e1000_82580)
1340 ims |= E1000_IMS_DRSTA;
1341
25568a53 1342 wr32(E1000_IMS, ims);
844290e5 1343 } else {
55cac248
AD		 1344 wr32(E1000_IMS, IMS_ENABLE_MASK |
1345 E1000_IMS_DRSTA);
1346 wr32(E1000_IAM, IMS_ENABLE_MASK |
1347 E1000_IMS_DRSTA);
844290e5 1348 }
9d5c8243
AK		 1349}
1350
1351static void igb_update_mng_vlan(struct igb_adapter *adapter)
1352{
51466239 1353 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1354 u16 vid = adapter->hw.mng_cookie.vlan_id;
1355 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1356
1357 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1358 /* add VID to filter table */
1359 igb_vfta_set(hw, vid, true);
1360 adapter->mng_vlan_id = vid;
1361 } else {
1362 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1363 }
1364
1365 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1366 (vid != old_vid) &&
b2cb09b1 1367 !test_bit(old_vid, adapter->active_vlans)) {
51466239
AD		 1368 /* remove VID from filter table */
1369 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1370 }
1371}
1372
1373/**
1374 * igb_release_hw_control - release control of the h/w to f/w
1375 * @adapter: address of board private structure
1376 *
1377 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1378 * For ASF and Pass Through versions of f/w this means that the
1379 * driver is no longer loaded.
1380 *
1381 **/
1382static void igb_release_hw_control(struct igb_adapter *adapter)
1383{
1384 struct e1000_hw *hw = &adapter->hw;
1385 u32 ctrl_ext;
1386
1387 /* Let firmware take over control of h/w */
1388 ctrl_ext = rd32(E1000_CTRL_EXT);
1389 wr32(E1000_CTRL_EXT,
1390 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1391}
1392
9d5c8243
AK		 1393/**
1394 * igb_get_hw_control - get control of the h/w from f/w
1395 * @adapter: address of board private structure
1396 *
1397 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1398 * For ASF and Pass Through versions of f/w this means that
1399 * the driver is loaded.
1400 *
1401 **/
1402static void igb_get_hw_control(struct igb_adapter *adapter)
1403{
1404 struct e1000_hw *hw = &adapter->hw;
1405 u32 ctrl_ext;
1406
1407 /* Let firmware know the driver has taken over */
1408 ctrl_ext = rd32(E1000_CTRL_EXT);
1409 wr32(E1000_CTRL_EXT,
1410 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1411}
1412
9d5c8243
AK		 1413/**
1414 * igb_configure - configure the hardware for RX and TX
1415 * @adapter: private board structure
1416 **/
1417static void igb_configure(struct igb_adapter *adapter)
1418{
1419 struct net_device *netdev = adapter->netdev;
1420 int i;
1421
1422 igb_get_hw_control(adapter);
ff41f8dc 1423 igb_set_rx_mode(netdev);
9d5c8243
AK		 1424
1425 igb_restore_vlan(adapter);
9d5c8243 1426
85b430b4 1427 igb_setup_tctl(adapter);
06cf2666 1428 igb_setup_mrqc(adapter);
9d5c8243 1429 igb_setup_rctl(adapter);
85b430b4
AD		 1430
1431 igb_configure_tx(adapter);
9d5c8243 1432 igb_configure_rx(adapter);
662d7205
AD		 1433
1434 igb_rx_fifo_flush_82575(&adapter->hw);
1435
c493ea45 1436 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1437 * at least 1 descriptor unused to make sure
1438 * next_to_use != next_to_clean */
1439 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1440 struct igb_ring *ring = adapter->rx_ring[i];
c493ea45 1441 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
9d5c8243 1442 }
9d5c8243
AK		 1443}
1444
88a268c1
NN		 1445/**
1446 * igb_power_up_link - Power up the phy/serdes link
1447 * @adapter: address of board private structure
1448 **/
1449void igb_power_up_link(struct igb_adapter *adapter)
1450{
1451 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1452 igb_power_up_phy_copper(&adapter->hw);
1453 else
1454 igb_power_up_serdes_link_82575(&adapter->hw);
1455}
1456
1457/**
1458 * igb_power_down_link - Power down the phy/serdes link
1459 * @adapter: address of board private structure
1460 */
1461static void igb_power_down_link(struct igb_adapter *adapter)
1462{
1463 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1464 igb_power_down_phy_copper_82575(&adapter->hw);
1465 else
1466 igb_shutdown_serdes_link_82575(&adapter->hw);
1467}
9d5c8243
AK		 1468
1469/**
1470 * igb_up - Open the interface and prepare it to handle traffic
1471 * @adapter: board private structure
1472 **/
9d5c8243
AK		 1473int igb_up(struct igb_adapter *adapter)
1474{
1475 struct e1000_hw *hw = &adapter->hw;
1476 int i;
1477
1478 /* hardware has been reset, we need to reload some things */
1479 igb_configure(adapter);
1480
1481 clear_bit(__IGB_DOWN, &adapter->state);
1482
047e0030
AD		 1483 for (i = 0; i < adapter->num_q_vectors; i++) {
1484 struct igb_q_vector *q_vector = adapter->q_vector[i];
1485 napi_enable(&q_vector->napi);
1486 }
844290e5 1487 if (adapter->msix_entries)
9d5c8243 1488 igb_configure_msix(adapter);
feeb2721
AD		 1489 else
1490 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1491
1492 /* Clear any pending interrupts. */
1493 rd32(E1000_ICR);
1494 igb_irq_enable(adapter);
1495
d4960307
AD		 1496 /* notify VFs that reset has been completed */
1497 if (adapter->vfs_allocated_count) {
1498 u32 reg_data = rd32(E1000_CTRL_EXT);
1499 reg_data |= E1000_CTRL_EXT_PFRSTD;
1500 wr32(E1000_CTRL_EXT, reg_data);
1501 }
1502
4cb9be7a
JB		 1503 netif_tx_start_all_queues(adapter->netdev);
1504
25568a53
AD		 1505 /* start the watchdog. */
1506 hw->mac.get_link_status = 1;
1507 schedule_work(&adapter->watchdog_task);
1508
9d5c8243
AK		 1509 return 0;
1510}
1511
1512void igb_down(struct igb_adapter *adapter)
1513{
9d5c8243 1514 struct net_device *netdev = adapter->netdev;
330a6d6a 1515 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1516 u32 tctl, rctl;
1517 int i;
1518
1519 /* signal that we're down so the interrupt handler does not
1520 * reschedule our watchdog timer */
1521 set_bit(__IGB_DOWN, &adapter->state);
1522
1523 /* disable receives in the hardware */
1524 rctl = rd32(E1000_RCTL);
1525 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1526 /* flush and sleep below */
1527
fd2ea0a7 1528 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1529
1530 /* disable transmits in the hardware */
1531 tctl = rd32(E1000_TCTL);
1532 tctl &= ~E1000_TCTL_EN;
1533 wr32(E1000_TCTL, tctl);
1534 /* flush both disables and wait for them to finish */
1535 wrfl();
1536 msleep(10);
1537
047e0030
AD		 1538 for (i = 0; i < adapter->num_q_vectors; i++) {
1539 struct igb_q_vector *q_vector = adapter->q_vector[i];
1540 napi_disable(&q_vector->napi);
1541 }
9d5c8243 1542
9d5c8243
AK		 1543 igb_irq_disable(adapter);
1544
1545 del_timer_sync(&adapter->watchdog_timer);
1546 del_timer_sync(&adapter->phy_info_timer);
1547
9d5c8243 1548 netif_carrier_off(netdev);
04fe6358
AD		 1549
1550 /* record the stats before reset*/
12dcd86b
ED		 1551 spin_lock(&adapter->stats64_lock);
1552 igb_update_stats(adapter, &adapter->stats64);
1553 spin_unlock(&adapter->stats64_lock);
04fe6358 1554
9d5c8243
AK		 1555 adapter->link_speed = 0;
1556 adapter->link_duplex = 0;
1557
3023682e
JK		 1558 if (!pci_channel_offline(adapter->pdev))
1559 igb_reset(adapter);
9d5c8243
AK		 1560 igb_clean_all_tx_rings(adapter);
1561 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1562#ifdef CONFIG_IGB_DCA
1563
1564 /* since we reset the hardware DCA settings were cleared */
1565 igb_setup_dca(adapter);
1566#endif
9d5c8243
AK		 1567}
1568
1569void igb_reinit_locked(struct igb_adapter *adapter)
1570{
1571 WARN_ON(in_interrupt());
1572 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1573 msleep(1);
1574 igb_down(adapter);
1575 igb_up(adapter);
1576 clear_bit(__IGB_RESETTING, &adapter->state);
1577}
1578
1579void igb_reset(struct igb_adapter *adapter)
1580{
090b1795 1581 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1582 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1583 struct e1000_mac_info *mac = &hw->mac;
1584 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1585 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1586 u16 hwm;
1587
1588 /* Repartition Pba for greater than 9k mtu
1589 * To take effect CTRL.RST is required.
1590 */
fa4dfae0 1591 switch (mac->type) {
d2ba2ed8 1592 case e1000_i350:
55cac248
AD		 1593 case e1000_82580:
1594 pba = rd32(E1000_RXPBS);
1595 pba = igb_rxpbs_adjust_82580(pba);
1596 break;
fa4dfae0 1597 case e1000_82576:
d249be54
AD		 1598 pba = rd32(E1000_RXPBS);
1599 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1600 break;
1601 case e1000_82575:
1602 default:
1603 pba = E1000_PBA_34K;
1604 break;
2d064c06 1605 }
9d5c8243 1606
2d064c06
AD		 1607 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1608 (mac->type < e1000_82576)) {
9d5c8243
AK		 1609 /* adjust PBA for jumbo frames */
1610 wr32(E1000_PBA, pba);
1611
1612 /* To maintain wire speed transmits, the Tx FIFO should be
1613 * large enough to accommodate two full transmit packets,
1614 * rounded up to the next 1KB and expressed in KB. Likewise,
1615 * the Rx FIFO should be large enough to accommodate at least
1616 * one full receive packet and is similarly rounded up and
1617 * expressed in KB. */
1618 pba = rd32(E1000_PBA);
1619 /* upper 16 bits has Tx packet buffer allocation size in KB */
1620 tx_space = pba >> 16;
1621 /* lower 16 bits has Rx packet buffer allocation size in KB */
1622 pba &= 0xffff;
1623 /* the tx fifo also stores 16 bytes of information about the tx
1624 * but don't include ethernet FCS because hardware appends it */
1625 min_tx_space = (adapter->max_frame_size +
85e8d004 1626 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1627 ETH_FCS_LEN) * 2;
1628 min_tx_space = ALIGN(min_tx_space, 1024);
1629 min_tx_space >>= 10;
1630 /* software strips receive CRC, so leave room for it */
1631 min_rx_space = adapter->max_frame_size;
1632 min_rx_space = ALIGN(min_rx_space, 1024);
1633 min_rx_space >>= 10;
1634
1635 /* If current Tx allocation is less than the min Tx FIFO size,
1636 * and the min Tx FIFO size is less than the current Rx FIFO
1637 * allocation, take space away from current Rx allocation */
1638 if (tx_space < min_tx_space &&
1639 ((min_tx_space - tx_space) < pba)) {
1640 pba = pba - (min_tx_space - tx_space);
1641
1642 /* if short on rx space, rx wins and must trump tx
1643 * adjustment */
1644 if (pba < min_rx_space)
1645 pba = min_rx_space;
1646 }
2d064c06 1647 wr32(E1000_PBA, pba);
9d5c8243 1648 }
9d5c8243
AK		 1649
1650 /* flow control settings */
1651 /* The high water mark must be low enough to fit one full frame
1652 * (or the size used for early receive) above it in the Rx FIFO.
1653 * Set it to the lower of:
1654 * - 90% of the Rx FIFO size, or
1655 * - the full Rx FIFO size minus one full frame */
1656 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1657 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1658
d405ea3e
AD		 1659 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1660 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1661 fc->pause_time = 0xFFFF;
1662 fc->send_xon = 1;
0cce119a 1663 fc->current_mode = fc->requested_mode;
9d5c8243 1664
4ae196df
AD		 1665 /* disable receive for all VFs and wait one second */
1666 if (adapter->vfs_allocated_count) {
1667 int i;
1668 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
8fa7e0f7 1669 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
4ae196df
AD		 1670
1671 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1672 igb_ping_all_vfs(adapter);
4ae196df
AD		 1673
1674 /* disable transmits and receives */
1675 wr32(E1000_VFRE, 0);
1676 wr32(E1000_VFTE, 0);
1677 }
1678
9d5c8243 1679 /* Allow time for pending master requests to run */
330a6d6a 1680 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1681 wr32(E1000_WUC, 0);
1682
330a6d6a 1683 if (hw->mac.ops.init_hw(hw))
090b1795 1684 dev_err(&pdev->dev, "Hardware Error\n");
831ec0b4
CW		 1685 if (hw->mac.type > e1000_82580) {
1686 if (adapter->flags & IGB_FLAG_DMAC) {
1687 u32 reg;
1688
1689 /*
1690 * DMA Coalescing high water mark needs to be higher
1691 * than * the * Rx threshold. The Rx threshold is
1692 * currently * pba - 6, so we * should use a high water
1693 * mark of pba * - 4. */
1694 hwm = (pba - 4) << 10;
1695
1696 reg = (((pba-6) << E1000_DMACR_DMACTHR_SHIFT)
1697 & E1000_DMACR_DMACTHR_MASK);
1698
1699 /* transition to L0x or L1 if available..*/
1700 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
1701
1702 /* watchdog timer= +-1000 usec in 32usec intervals */
1703 reg |= (1000 >> 5);
1704 wr32(E1000_DMACR, reg);
1705
1706 /* no lower threshold to disable coalescing(smart fifb)
1707 * -UTRESH=0*/
1708 wr32(E1000_DMCRTRH, 0);
1709
1710 /* set hwm to PBA - 2 * max frame size */
1711 wr32(E1000_FCRTC, hwm);
1712
1713 /*
1714 * This sets the time to wait before requesting tran-
1715 * sition to * low power state to number of usecs needed
1716 * to receive 1 512 * byte frame at gigabit line rate
1717 */
1718 reg = rd32(E1000_DMCTLX);
1719 reg |= IGB_DMCTLX_DCFLUSH_DIS;
1720
1721 /* Delay 255 usec before entering Lx state. */
1722 reg |= 0xFF;
1723 wr32(E1000_DMCTLX, reg);
1724
1725 /* free space in Tx packet buffer to wake from DMAC */
1726 wr32(E1000_DMCTXTH,
1727 (IGB_MIN_TXPBSIZE -
1728 (IGB_TX_BUF_4096 + adapter->max_frame_size))
1729 >> 6);
1730
1731 /* make low power state decision controlled by DMAC */
1732 reg = rd32(E1000_PCIEMISC);
1733 reg |= E1000_PCIEMISC_LX_DECISION;
1734 wr32(E1000_PCIEMISC, reg);
1735 } /* end if IGB_FLAG_DMAC set */
1736 }
55cac248
AD		 1737 if (hw->mac.type == e1000_82580) {
1738 u32 reg = rd32(E1000_PCIEMISC);
1739 wr32(E1000_PCIEMISC,
1740 reg & ~E1000_PCIEMISC_LX_DECISION);
1741 }
88a268c1
NN		 1742 if (!netif_running(adapter->netdev))
1743 igb_power_down_link(adapter);
1744
9d5c8243
AK		 1745 igb_update_mng_vlan(adapter);
1746
1747 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1748 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1749
330a6d6a 1750 igb_get_phy_info(hw);
9d5c8243
AK		 1751}
1752
b2cb09b1
JP		 1753static u32 igb_fix_features(struct net_device *netdev, u32 features)
1754{
1755 /*
1756 * Since there is no support for separate rx/tx vlan accel
1757 * enable/disable make sure tx flag is always in same state as rx.
1758 */
1759 if (features & NETIF_F_HW_VLAN_RX)
1760 features |= NETIF_F_HW_VLAN_TX;
1761 else
1762 features &= ~NETIF_F_HW_VLAN_TX;
1763
1764 return features;
1765}
1766
ac52caa3
MM		 1767static int igb_set_features(struct net_device *netdev, u32 features)
1768{
1769 struct igb_adapter *adapter = netdev_priv(netdev);
1770 int i;
b2cb09b1 1771 u32 changed = netdev->features ^ features;
ac52caa3
MM		 1772
1773 for (i = 0; i < adapter->num_rx_queues; i++) {
1774 if (features & NETIF_F_RXCSUM)
1775 adapter->rx_ring[i]->flags |= IGB_RING_FLAG_RX_CSUM;
1776 else
1777 adapter->rx_ring[i]->flags &= ~IGB_RING_FLAG_RX_CSUM;
1778 }
1779
b2cb09b1
JP		 1780 if (changed & NETIF_F_HW_VLAN_RX)
1781 igb_vlan_mode(netdev, features);
1782
ac52caa3
MM		 1783 return 0;
1784}
1785
2e5c6922 1786static const struct net_device_ops igb_netdev_ops = {
559e9c49 1787 .ndo_open = igb_open,
2e5c6922 1788 .ndo_stop = igb_close,
00829823 1789 .ndo_start_xmit = igb_xmit_frame_adv,
12dcd86b 1790 .ndo_get_stats64 = igb_get_stats64,
ff41f8dc
AD		 1791 .ndo_set_rx_mode = igb_set_rx_mode,
1792 .ndo_set_multicast_list = igb_set_rx_mode,
2e5c6922
SH		 1793 .ndo_set_mac_address = igb_set_mac,
1794 .ndo_change_mtu = igb_change_mtu,
1795 .ndo_do_ioctl = igb_ioctl,
1796 .ndo_tx_timeout = igb_tx_timeout,
1797 .ndo_validate_addr = eth_validate_addr,
2e5c6922
SH		 1798 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1799 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1800 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1801 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1802 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1803 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1804#ifdef CONFIG_NET_POLL_CONTROLLER
1805 .ndo_poll_controller = igb_netpoll,
1806#endif
b2cb09b1
JP		 1807 .ndo_fix_features = igb_fix_features,
1808 .ndo_set_features = igb_set_features,
2e5c6922
SH		 1809};
1810
9d5c8243
AK		 1811/**
1812 * igb_probe - Device Initialization Routine
1813 * @pdev: PCI device information struct
1814 * @ent: entry in igb_pci_tbl
1815 *
1816 * Returns 0 on success, negative on failure
1817 *
1818 * igb_probe initializes an adapter identified by a pci_dev structure.
1819 * The OS initialization, configuring of the adapter private structure,
1820 * and a hardware reset occur.
1821 **/
1822static int __devinit igb_probe(struct pci_dev *pdev,
1823 const struct pci_device_id *ent)
1824{
1825 struct net_device *netdev;
1826 struct igb_adapter *adapter;
1827 struct e1000_hw *hw;
4337e993 1828 u16 eeprom_data = 0;
9835fd73 1829 s32 ret_val;
4337e993 1830 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1831 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1832 unsigned long mmio_start, mmio_len;
2d6a5e95 1833 int err, pci_using_dac;
9d5c8243 1834 u16 eeprom_apme_mask = IGB_EEPROM_APME;
9835fd73 1835 u8 part_str[E1000_PBANUM_LENGTH];
9d5c8243 1836
bded64a7
AG		 1837 /* Catch broken hardware that put the wrong VF device ID in
1838 * the PCIe SR-IOV capability.
1839 */
1840 if (pdev->is_virtfn) {
1841 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1842 pci_name(pdev), pdev->vendor, pdev->device);
1843 return -EINVAL;
1844 }
1845
aed5dec3 1846 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1847 if (err)
1848 return err;
1849
1850 pci_using_dac = 0;
59d71989 1851 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243 1852 if (!err) {
59d71989 1853 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243
AK		 1854 if (!err)
1855 pci_using_dac = 1;
1856 } else {
59d71989 1857 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243 1858 if (err) {
59d71989 1859 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243
AK		 1860 if (err) {
1861 dev_err(&pdev->dev, "No usable DMA "
1862 "configuration, aborting\n");
1863 goto err_dma;
1864 }
1865 }
1866 }
1867
aed5dec3
AD		 1868 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1869 IORESOURCE_MEM),
1870 igb_driver_name);
9d5c8243
AK		 1871 if (err)
1872 goto err_pci_reg;
1873
19d5afd4 1874 pci_enable_pcie_error_reporting(pdev);
40a914fa 1875
9d5c8243 1876 pci_set_master(pdev);
c682fc23 1877 pci_save_state(pdev);
9d5c8243
AK		 1878
1879 err = -ENOMEM;
1bfaf07b
AD		 1880 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1881 IGB_ABS_MAX_TX_QUEUES);
9d5c8243
AK		 1882 if (!netdev)
1883 goto err_alloc_etherdev;
1884
1885 SET_NETDEV_DEV(netdev, &pdev->dev);
1886
1887 pci_set_drvdata(pdev, netdev);
1888 adapter = netdev_priv(netdev);
1889 adapter->netdev = netdev;
1890 adapter->pdev = pdev;
1891 hw = &adapter->hw;
1892 hw->back = adapter;
1893 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1894
1895 mmio_start = pci_resource_start(pdev, 0);
1896 mmio_len = pci_resource_len(pdev, 0);
1897
1898 err = -EIO;
28b0759c
AD		 1899 hw->hw_addr = ioremap(mmio_start, mmio_len);
1900 if (!hw->hw_addr)
9d5c8243
AK		 1901 goto err_ioremap;
1902
2e5c6922 1903 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1904 igb_set_ethtool_ops(netdev);
9d5c8243 1905 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1906
1907 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1908
1909 netdev->mem_start = mmio_start;
1910 netdev->mem_end = mmio_start + mmio_len;
1911
9d5c8243
AK		 1912 /* PCI config space info */
1913 hw->vendor_id = pdev->vendor;
1914 hw->device_id = pdev->device;
1915 hw->revision_id = pdev->revision;
1916 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1917 hw->subsystem_device_id = pdev->subsystem_device;
1918
9d5c8243
AK		 1919 /* Copy the default MAC, PHY and NVM function pointers */
1920 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1921 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1922 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1923 /* Initialize skew-specific constants */
1924 err = ei->get_invariants(hw);
1925 if (err)
450c87c8 1926 goto err_sw_init;
9d5c8243 1927
450c87c8 1928 /* setup the private structure */
9d5c8243
AK		 1929 err = igb_sw_init(adapter);
1930 if (err)
1931 goto err_sw_init;
1932
1933 igb_get_bus_info_pcie(hw);
1934
1935 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 1936
1937 /* Copper options */
1938 if (hw->phy.media_type == e1000_media_type_copper) {
1939 hw->phy.mdix = AUTO_ALL_MODES;
1940 hw->phy.disable_polarity_correction = false;
1941 hw->phy.ms_type = e1000_ms_hw_default;
1942 }
1943
1944 if (igb_check_reset_block(hw))
1945 dev_info(&pdev->dev,
1946 "PHY reset is blocked due to SOL/IDER session.\n");
1947
ac52caa3 1948 netdev->hw_features = NETIF_F_SG |
7d8eb29e 1949 NETIF_F_IP_CSUM |
ac52caa3
MM		 1950 NETIF_F_IPV6_CSUM |
1951 NETIF_F_TSO |
1952 NETIF_F_TSO6 |
b2cb09b1
JP		 1953 NETIF_F_RXCSUM |
1954 NETIF_F_HW_VLAN_RX;
ac52caa3
MM		 1955
1956 netdev->features = netdev->hw_features |
9d5c8243 1957 NETIF_F_HW_VLAN_TX |
9d5c8243
AK		 1958 NETIF_F_HW_VLAN_FILTER;
1959
48f29ffc
JK		 1960 netdev->vlan_features |= NETIF_F_TSO;
1961 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 1962 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 1963 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 1964 netdev->vlan_features |= NETIF_F_SG;
1965
7b872a55 1966 if (pci_using_dac) {
9d5c8243 1967 netdev->features |= NETIF_F_HIGHDMA;
7b872a55
YZ		 1968 netdev->vlan_features |= NETIF_F_HIGHDMA;
1969 }
9d5c8243 1970
ac52caa3
MM		 1971 if (hw->mac.type >= e1000_82576) {
1972 netdev->hw_features |= NETIF_F_SCTP_CSUM;
b9473560 1973 netdev->features |= NETIF_F_SCTP_CSUM;
ac52caa3 1974 }
b9473560 1975
330a6d6a 1976 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 1977
1978 /* before reading the NVM, reset the controller to put the device in a
1979 * known good starting state */
1980 hw->mac.ops.reset_hw(hw);
1981
1982 /* make sure the NVM is good */
4322e561 1983 if (hw->nvm.ops.validate(hw) < 0) {
9d5c8243
AK		 1984 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1985 err = -EIO;
1986 goto err_eeprom;
1987 }
1988
1989 /* copy the MAC address out of the NVM */
1990 if (hw->mac.ops.read_mac_addr(hw))
1991 dev_err(&pdev->dev, "NVM Read Error\n");
1992
1993 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1994 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1995
1996 if (!is_valid_ether_addr(netdev->perm_addr)) {
1997 dev_err(&pdev->dev, "Invalid MAC Address\n");
1998 err = -EIO;
1999 goto err_eeprom;
2000 }
2001
c061b18d 2002 setup_timer(&adapter->watchdog_timer, igb_watchdog,
0e340485 2003 (unsigned long) adapter);
c061b18d 2004 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
0e340485 2005 (unsigned long) adapter);
9d5c8243
AK		 2006
2007 INIT_WORK(&adapter->reset_task, igb_reset_task);
2008 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2009
450c87c8 2010 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 2011 adapter->fc_autoneg = true;
2012 hw->mac.autoneg = true;
2013 hw->phy.autoneg_advertised = 0x2f;
2014
0cce119a
AD		 2015 hw->fc.requested_mode = e1000_fc_default;
2016 hw->fc.current_mode = e1000_fc_default;
9d5c8243 2017
9d5c8243
AK		 2018 igb_validate_mdi_setting(hw);
2019
9d5c8243
AK		 2020 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2021 * enable the ACPI Magic Packet filter
2022 */
2023
a2cf8b6c 2024 if (hw->bus.func == 0)
312c75ae 2025 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
6d337dce 2026 else if (hw->mac.type >= e1000_82580)
55cac248
AD		 2027 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2028 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2029 &eeprom_data);
a2cf8b6c
AD		 2030 else if (hw->bus.func == 1)
2031 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 2032
2033 if (eeprom_data & eeprom_apme_mask)
2034 adapter->eeprom_wol |= E1000_WUFC_MAG;
2035
2036 /* now that we have the eeprom settings, apply the special cases where
2037 * the eeprom may be wrong or the board simply won't support wake on
2038 * lan on a particular port */
2039 switch (pdev->device) {
2040 case E1000_DEV_ID_82575GB_QUAD_COPPER:
2041 adapter->eeprom_wol = 0;
2042 break;
2043 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 2044 case E1000_DEV_ID_82576_FIBER:
2045 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 2046 /* Wake events only supported on port A for dual fiber
2047 * regardless of eeprom setting */
2048 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
2049 adapter->eeprom_wol = 0;
2050 break;
c8ea5ea9 2051 case E1000_DEV_ID_82576_QUAD_COPPER:
d5aa2252 2052 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
c8ea5ea9
AD		 2053 /* if quad port adapter, disable WoL on all but port A */
2054 if (global_quad_port_a != 0)
2055 adapter->eeprom_wol = 0;
2056 else
2057 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2058 /* Reset for multiple quad port adapters */
2059 if (++global_quad_port_a == 4)
2060 global_quad_port_a = 0;
2061 break;
9d5c8243
AK		 2062 }
2063
2064 /* initialize the wol settings based on the eeprom settings */
2065 adapter->wol = adapter->eeprom_wol;
e1b86d84 2066 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 2067
2068 /* reset the hardware with the new settings */
2069 igb_reset(adapter);
2070
2071 /* let the f/w know that the h/w is now under the control of the
2072 * driver. */
2073 igb_get_hw_control(adapter);
2074
9d5c8243
AK		 2075 strcpy(netdev->name, "eth%d");
2076 err = register_netdev(netdev);
2077 if (err)
2078 goto err_register;
2079
b2cb09b1
JP		 2080 igb_vlan_mode(netdev, netdev->features);
2081
b168dfc5
JB		 2082 /* carrier off reporting is important to ethtool even BEFORE open */
2083 netif_carrier_off(netdev);
2084
421e02f0 2085#ifdef CONFIG_IGB_DCA
bbd98fe4 2086 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 2087 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 2088 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 2089 igb_setup_dca(adapter);
2090 }
fe4506b6 2091
38c845c7 2092#endif
673b8b70
AB		 2093 /* do hw tstamp init after resetting */
2094 igb_init_hw_timer(adapter);
2095
9d5c8243
AK		 2096 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
2097 /* print bus type/speed/width info */
7c510e4b 2098 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 2099 netdev->name,
559e9c49 2100 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
ff846f52 2101 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
559e9c49 2102 "unknown"),
59c3de89
AD		 2103 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
2104 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
2105 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2106 "unknown"),
7c510e4b 2107 netdev->dev_addr);
9d5c8243 2108
9835fd73
CW		 2109 ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH);
2110 if (ret_val)
2111 strcpy(part_str, "Unknown");
2112 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
9d5c8243
AK		 2113 dev_info(&pdev->dev,
2114 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2115 adapter->msix_entries ? "MSI-X" :
7dfc16fa 2116 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243 2117 adapter->num_rx_queues, adapter->num_tx_queues);
09b068d4
CW		 2118 switch (hw->mac.type) {
2119 case e1000_i350:
2120 igb_set_eee_i350(hw);
2121 break;
2122 default:
2123 break;
2124 }
9d5c8243
AK		 2125 return 0;
2126
2127err_register:
2128 igb_release_hw_control(adapter);
2129err_eeprom:
2130 if (!igb_check_reset_block(hw))
f5f4cf08 2131 igb_reset_phy(hw);
9d5c8243
AK		 2132
2133 if (hw->flash_address)
2134 iounmap(hw->flash_address);
9d5c8243 2135err_sw_init:
047e0030 2136 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 2137 iounmap(hw->hw_addr);
2138err_ioremap:
2139 free_netdev(netdev);
2140err_alloc_etherdev:
559e9c49
AD		 2141 pci_release_selected_regions(pdev,
2142 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2143err_pci_reg:
2144err_dma:
2145 pci_disable_device(pdev);
2146 return err;
2147}
2148
2149/**
2150 * igb_remove - Device Removal Routine
2151 * @pdev: PCI device information struct
2152 *
2153 * igb_remove is called by the PCI subsystem to alert the driver
2154 * that it should release a PCI device. The could be caused by a
2155 * Hot-Plug event, or because the driver is going to be removed from
2156 * memory.
2157 **/
2158static void __devexit igb_remove(struct pci_dev *pdev)
2159{
2160 struct net_device *netdev = pci_get_drvdata(pdev);
2161 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 2162 struct e1000_hw *hw = &adapter->hw;
9d5c8243 2163
760141a5
TH		 2164 /*
2165 * The watchdog timer may be rescheduled, so explicitly
2166 * disable watchdog from being rescheduled.
2167 */
9d5c8243
AK		 2168 set_bit(__IGB_DOWN, &adapter->state);
2169 del_timer_sync(&adapter->watchdog_timer);
2170 del_timer_sync(&adapter->phy_info_timer);
2171
760141a5
TH		 2172 cancel_work_sync(&adapter->reset_task);
2173 cancel_work_sync(&adapter->watchdog_task);
9d5c8243 2174
421e02f0 2175#ifdef CONFIG_IGB_DCA
7dfc16fa 2176 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 2177 dev_info(&pdev->dev, "DCA disabled\n");
2178 dca_remove_requester(&pdev->dev);
7dfc16fa 2179 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 2180 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 2181 }
2182#endif
2183
9d5c8243
AK		 2184 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2185 * would have already happened in close and is redundant. */
2186 igb_release_hw_control(adapter);
2187
2188 unregister_netdev(netdev);
2189
047e0030 2190 igb_clear_interrupt_scheme(adapter);
9d5c8243 2191
37680117
AD		 2192#ifdef CONFIG_PCI_IOV
2193 /* reclaim resources allocated to VFs */
2194 if (adapter->vf_data) {
2195 /* disable iov and allow time for transactions to clear */
2196 pci_disable_sriov(pdev);
2197 msleep(500);
2198
2199 kfree(adapter->vf_data);
2200 adapter->vf_data = NULL;
2201 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 2202 wrfl();
37680117
AD		 2203 msleep(100);
2204 dev_info(&pdev->dev, "IOV Disabled\n");
2205 }
2206#endif
559e9c49 2207
28b0759c
AD		 2208 iounmap(hw->hw_addr);
2209 if (hw->flash_address)
2210 iounmap(hw->flash_address);
559e9c49
AD		 2211 pci_release_selected_regions(pdev,
2212 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2213
2214 free_netdev(netdev);
2215
19d5afd4 2216 pci_disable_pcie_error_reporting(pdev);
40a914fa 2217
9d5c8243
AK		 2218 pci_disable_device(pdev);
2219}
2220
a6b623e0
AD		 2221/**
2222 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2223 * @adapter: board private structure to initialize
2224 *
2225 * This function initializes the vf specific data storage and then attempts to
2226 * allocate the VFs. The reason for ordering it this way is because it is much
2227 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2228 * the memory for the VFs.
2229 **/
2230static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2231{
2232#ifdef CONFIG_PCI_IOV
2233 struct pci_dev *pdev = adapter->pdev;
2234
a6b623e0
AD		 2235 if (adapter->vfs_allocated_count) {
2236 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2237 sizeof(struct vf_data_storage),
2238 GFP_KERNEL);
2239 /* if allocation failed then we do not support SR-IOV */
2240 if (!adapter->vf_data) {
2241 adapter->vfs_allocated_count = 0;
2242 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2243 "Data Storage\n");
2244 }
2245 }
2246
2247 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
2248 kfree(adapter->vf_data);
2249 adapter->vf_data = NULL;
2250#endif /* CONFIG_PCI_IOV */
2251 adapter->vfs_allocated_count = 0;
2252#ifdef CONFIG_PCI_IOV
2253 } else {
2254 unsigned char mac_addr[ETH_ALEN];
2255 int i;
2256 dev_info(&pdev->dev, "%d vfs allocated\n",
2257 adapter->vfs_allocated_count);
2258 for (i = 0; i < adapter->vfs_allocated_count; i++) {
2259 random_ether_addr(mac_addr);
2260 igb_set_vf_mac(adapter, i, mac_addr);
2261 }
831ec0b4
CW		 2262 /* DMA Coalescing is not supported in IOV mode. */
2263 if (adapter->flags & IGB_FLAG_DMAC)
2264 adapter->flags &= ~IGB_FLAG_DMAC;
a6b623e0
AD		 2265 }
2266#endif /* CONFIG_PCI_IOV */
2267}
2268
115f459a
AD		 2269
2270/**
2271 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2272 * @adapter: board private structure to initialize
2273 *
2274 * igb_init_hw_timer initializes the function pointer and values for the hw
2275 * timer found in hardware.
2276 **/
2277static void igb_init_hw_timer(struct igb_adapter *adapter)
2278{
2279 struct e1000_hw *hw = &adapter->hw;
2280
2281 switch (hw->mac.type) {
d2ba2ed8 2282 case e1000_i350:
55cac248
AD		 2283 case e1000_82580:
2284 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2285 adapter->cycles.read = igb_read_clock;
2286 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2287 adapter->cycles.mult = 1;
2288 /*
2289 * The 82580 timesync updates the system timer every 8ns by 8ns
2290 * and the value cannot be shifted. Instead we need to shift
2291 * the registers to generate a 64bit timer value. As a result
2292 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2293 * 24 in order to generate a larger value for synchronization.
2294 */
2295 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2296 /* disable system timer temporarily by setting bit 31 */
2297 wr32(E1000_TSAUXC, 0x80000000);
2298 wrfl();
2299
2300 /* Set registers so that rollover occurs soon to test this. */
2301 wr32(E1000_SYSTIMR, 0x00000000);
2302 wr32(E1000_SYSTIML, 0x80000000);
2303 wr32(E1000_SYSTIMH, 0x000000FF);
2304 wrfl();
2305
2306 /* enable system timer by clearing bit 31 */
2307 wr32(E1000_TSAUXC, 0x0);
2308 wrfl();
2309
2310 timecounter_init(&adapter->clock,
2311 &adapter->cycles,
2312 ktime_to_ns(ktime_get_real()));
2313 /*
2314 * Synchronize our NIC clock against system wall clock. NIC
2315 * time stamp reading requires ~3us per sample, each sample
2316 * was pretty stable even under load => only require 10
2317 * samples for each offset comparison.
2318 */
2319 memset(&adapter->compare, 0, sizeof(adapter->compare));
2320 adapter->compare.source = &adapter->clock;
2321 adapter->compare.target = ktime_get_real;
2322 adapter->compare.num_samples = 10;
2323 timecompare_update(&adapter->compare, 0);
2324 break;
115f459a
AD		 2325 case e1000_82576:
2326 /*
2327 * Initialize hardware timer: we keep it running just in case
2328 * that some program needs it later on.
2329 */
2330 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2331 adapter->cycles.read = igb_read_clock;
2332 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2333 adapter->cycles.mult = 1;
2334 /**
2335 * Scale the NIC clock cycle by a large factor so that
2336 * relatively small clock corrections can be added or
25985edc 2337 * subtracted at each clock tick. The drawbacks of a large
115f459a
AD		 2338 * factor are a) that the clock register overflows more quickly
2339 * (not such a big deal) and b) that the increment per tick has
2340 * to fit into 24 bits. As a result we need to use a shift of
2341 * 19 so we can fit a value of 16 into the TIMINCA register.
2342 */
2343 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2344 wr32(E1000_TIMINCA,
2345 (1 << E1000_TIMINCA_16NS_SHIFT) |
2346 (16 << IGB_82576_TSYNC_SHIFT));
2347
2348 /* Set registers so that rollover occurs soon to test this. */
2349 wr32(E1000_SYSTIML, 0x00000000);
2350 wr32(E1000_SYSTIMH, 0xFF800000);
2351 wrfl();
2352
2353 timecounter_init(&adapter->clock,
2354 &adapter->cycles,
2355 ktime_to_ns(ktime_get_real()));
2356 /*
2357 * Synchronize our NIC clock against system wall clock. NIC
2358 * time stamp reading requires ~3us per sample, each sample
2359 * was pretty stable even under load => only require 10
2360 * samples for each offset comparison.
2361 */
2362 memset(&adapter->compare, 0, sizeof(adapter->compare));
2363 adapter->compare.source = &adapter->clock;
2364 adapter->compare.target = ktime_get_real;
2365 adapter->compare.num_samples = 10;
2366 timecompare_update(&adapter->compare, 0);
2367 break;
2368 case e1000_82575:
2369 /* 82575 does not support timesync */
2370 default:
2371 break;
2372 }
2373
2374}
2375
9d5c8243
AK		 2376/**
2377 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2378 * @adapter: board private structure to initialize
2379 *
2380 * igb_sw_init initializes the Adapter private data structure.
2381 * Fields are initialized based on PCI device information and
2382 * OS network device settings (MTU size).
2383 **/
2384static int __devinit igb_sw_init(struct igb_adapter *adapter)
2385{
2386 struct e1000_hw *hw = &adapter->hw;
2387 struct net_device *netdev = adapter->netdev;
2388 struct pci_dev *pdev = adapter->pdev;
2389
2390 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2391
68fd9910
AD		 2392 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2393 adapter->rx_ring_count = IGB_DEFAULT_RXD;
4fc82adf
AD		 2394 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2395 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2396
9d5c8243
AK		 2397 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
2398 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2399
12dcd86b 2400 spin_lock_init(&adapter->stats64_lock);
a6b623e0 2401#ifdef CONFIG_PCI_IOV
6b78bb1d
CW		 2402 switch (hw->mac.type) {
2403 case e1000_82576:
2404 case e1000_i350:
9b082d73
SA		 2405 if (max_vfs > 7) {
2406 dev_warn(&pdev->dev,
2407 "Maximum of 7 VFs per PF, using max\n");
2408 adapter->vfs_allocated_count = 7;
2409 } else
2410 adapter->vfs_allocated_count = max_vfs;
6b78bb1d
CW		 2411 break;
2412 default:
2413 break;
2414 }
a6b623e0 2415#endif /* CONFIG_PCI_IOV */
a99955fc 2416 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
665c8c8e
WM		 2417 /* i350 cannot do RSS and SR-IOV at the same time */
2418 if (hw->mac.type == e1000_i350 && adapter->vfs_allocated_count)
2419 adapter->rss_queues = 1;
a99955fc
AD		 2420
2421 /*
2422 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2423 * then we should combine the queues into a queue pair in order to
2424 * conserve interrupts due to limited supply
2425 */
2426 if ((adapter->rss_queues > 4) ||
2427 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2428 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2429
a6b623e0 2430 /* This call may decrease the number of queues */
047e0030 2431 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 2432 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2433 return -ENOMEM;
2434 }
2435
a6b623e0
AD		 2436 igb_probe_vfs(adapter);
2437
9d5c8243
AK		 2438 /* Explicitly disable IRQ since the NIC can be in any state. */
2439 igb_irq_disable(adapter);
2440
831ec0b4
CW		 2441 if (hw->mac.type == e1000_i350)
2442 adapter->flags &= ~IGB_FLAG_DMAC;
2443
9d5c8243
AK		 2444 set_bit(__IGB_DOWN, &adapter->state);
2445 return 0;
2446}
2447
2448/**
2449 * igb_open - Called when a network interface is made active
2450 * @netdev: network interface device structure
2451 *
2452 * Returns 0 on success, negative value on failure
2453 *
2454 * The open entry point is called when a network interface is made
2455 * active by the system (IFF_UP). At this point all resources needed
2456 * for transmit and receive operations are allocated, the interrupt
2457 * handler is registered with the OS, the watchdog timer is started,
2458 * and the stack is notified that the interface is ready.
2459 **/
2460static int igb_open(struct net_device *netdev)
2461{
2462 struct igb_adapter *adapter = netdev_priv(netdev);
2463 struct e1000_hw *hw = &adapter->hw;
2464 int err;
2465 int i;
2466
2467 /* disallow open during test */
2468 if (test_bit(__IGB_TESTING, &adapter->state))
2469 return -EBUSY;
2470
b168dfc5
JB		 2471 netif_carrier_off(netdev);
2472
9d5c8243
AK		 2473 /* allocate transmit descriptors */
2474 err = igb_setup_all_tx_resources(adapter);
2475 if (err)
2476 goto err_setup_tx;
2477
2478 /* allocate receive descriptors */
2479 err = igb_setup_all_rx_resources(adapter);
2480 if (err)
2481 goto err_setup_rx;
2482
88a268c1 2483 igb_power_up_link(adapter);
9d5c8243 2484
9d5c8243
AK		 2485 /* before we allocate an interrupt, we must be ready to handle it.
2486 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2487 * as soon as we call pci_request_irq, so we have to setup our
2488 * clean_rx handler before we do so. */
2489 igb_configure(adapter);
2490
2491 err = igb_request_irq(adapter);
2492 if (err)
2493 goto err_req_irq;
2494
2495 /* From here on the code is the same as igb_up() */
2496 clear_bit(__IGB_DOWN, &adapter->state);
2497
047e0030
AD		 2498 for (i = 0; i < adapter->num_q_vectors; i++) {
2499 struct igb_q_vector *q_vector = adapter->q_vector[i];
2500 napi_enable(&q_vector->napi);
2501 }
9d5c8243
AK		 2502
2503 /* Clear any pending interrupts. */
2504 rd32(E1000_ICR);
844290e5
PW		 2505
2506 igb_irq_enable(adapter);
2507
d4960307
AD		 2508 /* notify VFs that reset has been completed */
2509 if (adapter->vfs_allocated_count) {
2510 u32 reg_data = rd32(E1000_CTRL_EXT);
2511 reg_data |= E1000_CTRL_EXT_PFRSTD;
2512 wr32(E1000_CTRL_EXT, reg_data);
2513 }
2514
d55b53ff
JK		 2515 netif_tx_start_all_queues(netdev);
2516
25568a53
AD		 2517 /* start the watchdog. */
2518 hw->mac.get_link_status = 1;
2519 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2520
2521 return 0;
2522
2523err_req_irq:
2524 igb_release_hw_control(adapter);
88a268c1 2525 igb_power_down_link(adapter);
9d5c8243
AK		 2526 igb_free_all_rx_resources(adapter);
2527err_setup_rx:
2528 igb_free_all_tx_resources(adapter);
2529err_setup_tx:
2530 igb_reset(adapter);
2531
2532 return err;
2533}
2534
2535/**
2536 * igb_close - Disables a network interface
2537 * @netdev: network interface device structure
2538 *
2539 * Returns 0, this is not allowed to fail
2540 *
2541 * The close entry point is called when an interface is de-activated
2542 * by the OS. The hardware is still under the driver's control, but
2543 * needs to be disabled. A global MAC reset is issued to stop the
2544 * hardware, and all transmit and receive resources are freed.
2545 **/
2546static int igb_close(struct net_device *netdev)
2547{
2548 struct igb_adapter *adapter = netdev_priv(netdev);
2549
2550 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2551 igb_down(adapter);
2552
2553 igb_free_irq(adapter);
2554
2555 igb_free_all_tx_resources(adapter);
2556 igb_free_all_rx_resources(adapter);
2557
9d5c8243
AK		 2558 return 0;
2559}
2560
2561/**
2562 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2563 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2564 *
2565 * Return 0 on success, negative on failure
2566 **/
80785298 2567int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2568{
59d71989 2569 struct device *dev = tx_ring->dev;
9d5c8243
AK		 2570 int size;
2571
2572 size = sizeof(struct igb_buffer) * tx_ring->count;
89bf67f1 2573 tx_ring->buffer_info = vzalloc(size);
9d5c8243
AK		 2574 if (!tx_ring->buffer_info)
2575 goto err;
9d5c8243
AK		 2576
2577 /* round up to nearest 4K */
85e8d004 2578 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2579 tx_ring->size = ALIGN(tx_ring->size, 4096);
2580
59d71989
AD		 2581 tx_ring->desc = dma_alloc_coherent(dev,
2582 tx_ring->size,
2583 &tx_ring->dma,
2584 GFP_KERNEL);
9d5c8243
AK		 2585
2586 if (!tx_ring->desc)
2587 goto err;
2588
9d5c8243
AK		 2589 tx_ring->next_to_use = 0;
2590 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2591 return 0;
2592
2593err:
2594 vfree(tx_ring->buffer_info);
59d71989 2595 dev_err(dev,
9d5c8243
AK		 2596 "Unable to allocate memory for the transmit descriptor ring\n");
2597 return -ENOMEM;
2598}
2599
2600/**
2601 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2602 * (Descriptors) for all queues
2603 * @adapter: board private structure
2604 *
2605 * Return 0 on success, negative on failure
2606 **/
2607static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2608{
439705e1 2609 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2610 int i, err = 0;
2611
2612 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2613 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2614 if (err) {
439705e1 2615 dev_err(&pdev->dev,
9d5c8243
AK		 2616 "Allocation for Tx Queue %u failed\n", i);
2617 for (i--; i >= 0; i--)
3025a446 2618 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2619 break;
2620 }
2621 }
2622
a99955fc 2623 for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
439705e1 2624 int r_idx = i % adapter->num_tx_queues;
3025a446 2625 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
eebbbdba 2626 }
9d5c8243
AK		 2627 return err;
2628}
2629
2630/**
85b430b4
AD		 2631 * igb_setup_tctl - configure the transmit control registers
2632 * @adapter: Board private structure
9d5c8243 2633 **/
d7ee5b3a 2634void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2635{
9d5c8243
AK		 2636 struct e1000_hw *hw = &adapter->hw;
2637 u32 tctl;
9d5c8243 2638
85b430b4
AD		 2639 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2640 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2641
2642 /* Program the Transmit Control Register */
9d5c8243
AK		 2643 tctl = rd32(E1000_TCTL);
2644 tctl &= ~E1000_TCTL_CT;
2645 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2646 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2647
2648 igb_config_collision_dist(hw);
2649
9d5c8243
AK		 2650 /* Enable transmits */
2651 tctl |= E1000_TCTL_EN;
2652
2653 wr32(E1000_TCTL, tctl);
2654}
2655
85b430b4
AD		 2656/**
2657 * igb_configure_tx_ring - Configure transmit ring after Reset
2658 * @adapter: board private structure
2659 * @ring: tx ring to configure
2660 *
2661 * Configure a transmit ring after a reset.
2662 **/
d7ee5b3a
AD		 2663void igb_configure_tx_ring(struct igb_adapter *adapter,
2664 struct igb_ring *ring)
85b430b4
AD		 2665{
2666 struct e1000_hw *hw = &adapter->hw;
2667 u32 txdctl;
2668 u64 tdba = ring->dma;
2669 int reg_idx = ring->reg_idx;
2670
2671 /* disable the queue */
2672 txdctl = rd32(E1000_TXDCTL(reg_idx));
2673 wr32(E1000_TXDCTL(reg_idx),
2674 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2675 wrfl();
2676 mdelay(10);
2677
2678 wr32(E1000_TDLEN(reg_idx),
2679 ring->count * sizeof(union e1000_adv_tx_desc));
2680 wr32(E1000_TDBAL(reg_idx),
2681 tdba & 0x00000000ffffffffULL);
2682 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2683
fce99e34
AD		 2684 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2685 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2686 writel(0, ring->head);
2687 writel(0, ring->tail);
85b430b4
AD		 2688
2689 txdctl |= IGB_TX_PTHRESH;
2690 txdctl |= IGB_TX_HTHRESH << 8;
2691 txdctl |= IGB_TX_WTHRESH << 16;
2692
2693 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2694 wr32(E1000_TXDCTL(reg_idx), txdctl);
2695}
2696
2697/**
2698 * igb_configure_tx - Configure transmit Unit after Reset
2699 * @adapter: board private structure
2700 *
2701 * Configure the Tx unit of the MAC after a reset.
2702 **/
2703static void igb_configure_tx(struct igb_adapter *adapter)
2704{
2705 int i;
2706
2707 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2708 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2709}
2710
9d5c8243
AK		 2711/**
2712 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2713 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2714 *
2715 * Returns 0 on success, negative on failure
2716 **/
80785298 2717int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2718{
59d71989 2719 struct device *dev = rx_ring->dev;
9d5c8243
AK		 2720 int size, desc_len;
2721
2722 size = sizeof(struct igb_buffer) * rx_ring->count;
89bf67f1 2723 rx_ring->buffer_info = vzalloc(size);
9d5c8243
AK		 2724 if (!rx_ring->buffer_info)
2725 goto err;
9d5c8243
AK		 2726
2727 desc_len = sizeof(union e1000_adv_rx_desc);
2728
2729 /* Round up to nearest 4K */
2730 rx_ring->size = rx_ring->count * desc_len;
2731 rx_ring->size = ALIGN(rx_ring->size, 4096);
2732
59d71989
AD		 2733 rx_ring->desc = dma_alloc_coherent(dev,
2734 rx_ring->size,
2735 &rx_ring->dma,
2736 GFP_KERNEL);
9d5c8243
AK		 2737
2738 if (!rx_ring->desc)
2739 goto err;
2740
2741 rx_ring->next_to_clean = 0;
2742 rx_ring->next_to_use = 0;
9d5c8243 2743
9d5c8243
AK		 2744 return 0;
2745
2746err:
2747 vfree(rx_ring->buffer_info);
439705e1 2748 rx_ring->buffer_info = NULL;
59d71989
AD		 2749 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2750 " ring\n");
9d5c8243
AK		 2751 return -ENOMEM;
2752}
2753
2754/**
2755 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2756 * (Descriptors) for all queues
2757 * @adapter: board private structure
2758 *
2759 * Return 0 on success, negative on failure
2760 **/
2761static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2762{
439705e1 2763 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2764 int i, err = 0;
2765
2766 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2767 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2768 if (err) {
439705e1 2769 dev_err(&pdev->dev,
9d5c8243
AK		 2770 "Allocation for Rx Queue %u failed\n", i);
2771 for (i--; i >= 0; i--)
3025a446 2772 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2773 break;
2774 }
2775 }
2776
2777 return err;
2778}
2779
06cf2666
AD		 2780/**
2781 * igb_setup_mrqc - configure the multiple receive queue control registers
2782 * @adapter: Board private structure
2783 **/
2784static void igb_setup_mrqc(struct igb_adapter *adapter)
2785{
2786 struct e1000_hw *hw = &adapter->hw;
2787 u32 mrqc, rxcsum;
2788 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2789 union e1000_reta {
2790 u32 dword;
2791 u8 bytes[4];
2792 } reta;
2793 static const u8 rsshash[40] = {
2794 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2795 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2796 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2797 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2798
2799 /* Fill out hash function seeds */
2800 for (j = 0; j < 10; j++) {
2801 u32 rsskey = rsshash[(j * 4)];
2802 rsskey |= rsshash[(j * 4) + 1] << 8;
2803 rsskey |= rsshash[(j * 4) + 2] << 16;
2804 rsskey |= rsshash[(j * 4) + 3] << 24;
2805 array_wr32(E1000_RSSRK(0), j, rsskey);
2806 }
2807
a99955fc 2808 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2809
2810 if (adapter->vfs_allocated_count) {
2811 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2812 switch (hw->mac.type) {
d2ba2ed8 2813 case e1000_i350:
55cac248
AD		 2814 case e1000_82580:
2815 num_rx_queues = 1;
2816 shift = 0;
2817 break;
06cf2666
AD		 2818 case e1000_82576:
2819 shift = 3;
2820 num_rx_queues = 2;
2821 break;
2822 case e1000_82575:
2823 shift = 2;
2824 shift2 = 6;
2825 default:
2826 break;
2827 }
2828 } else {
2829 if (hw->mac.type == e1000_82575)
2830 shift = 6;
2831 }
2832
2833 for (j = 0; j < (32 * 4); j++) {
2834 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2835 if (shift2)
2836 reta.bytes[j & 3] |= num_rx_queues << shift2;
2837 if ((j & 3) == 3)
2838 wr32(E1000_RETA(j >> 2), reta.dword);
2839 }
2840
2841 /*
2842 * Disable raw packet checksumming so that RSS hash is placed in
2843 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2844 * offloads as they are enabled by default
2845 */
2846 rxcsum = rd32(E1000_RXCSUM);
2847 rxcsum |= E1000_RXCSUM_PCSD;
2848
2849 if (adapter->hw.mac.type >= e1000_82576)
2850 /* Enable Receive Checksum Offload for SCTP */
2851 rxcsum |= E1000_RXCSUM_CRCOFL;
2852
2853 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2854 wr32(E1000_RXCSUM, rxcsum);
2855
2856 /* If VMDq is enabled then we set the appropriate mode for that, else
2857 * we default to RSS so that an RSS hash is calculated per packet even
2858 * if we are only using one queue */
2859 if (adapter->vfs_allocated_count) {
2860 if (hw->mac.type > e1000_82575) {
2861 /* Set the default pool for the PF's first queue */
2862 u32 vtctl = rd32(E1000_VT_CTL);
2863 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2864 E1000_VT_CTL_DISABLE_DEF_POOL);
2865 vtctl |= adapter->vfs_allocated_count <<
2866 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2867 wr32(E1000_VT_CTL, vtctl);
2868 }
a99955fc 2869 if (adapter->rss_queues > 1)
06cf2666
AD		 2870 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2871 else
2872 mrqc = E1000_MRQC_ENABLE_VMDQ;
2873 } else {
2874 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2875 }
2876 igb_vmm_control(adapter);
2877
4478a9cd
AD		 2878 /*
2879 * Generate RSS hash based on TCP port numbers and/or
2880 * IPv4/v6 src and dst addresses since UDP cannot be
2881 * hashed reliably due to IP fragmentation
2882 */
2883 mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
2884 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2885 E1000_MRQC_RSS_FIELD_IPV6 |
2886 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2887 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
06cf2666
AD		 2888
2889 wr32(E1000_MRQC, mrqc);
2890}
2891
9d5c8243
AK		 2892/**
2893 * igb_setup_rctl - configure the receive control registers
2894 * @adapter: Board private structure
2895 **/
d7ee5b3a 2896void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2897{
2898 struct e1000_hw *hw = &adapter->hw;
2899 u32 rctl;
9d5c8243
AK		 2900
2901 rctl = rd32(E1000_RCTL);
2902
2903 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2904 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2905
69d728ba 2906 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2907 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2908
87cb7e8c
AK		 2909 /*
2910 * enable stripping of CRC. It's unlikely this will break BMC
2911 * redirection as it did with e1000. Newer features require
2912 * that the HW strips the CRC.
73cd78f1 2913 */
87cb7e8c 2914 rctl |= E1000_RCTL_SECRC;
9d5c8243 2915
559e9c49 2916 /* disable store bad packets and clear size bits. */
ec54d7d6 2917 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2918
6ec43fe6
AD		 2919 /* enable LPE to prevent packets larger than max_frame_size */
2920 rctl |= E1000_RCTL_LPE;
9d5c8243 2921
952f72a8
AD		 2922 /* disable queue 0 to prevent tail write w/o re-config */
2923 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2924
e1739522
AD		 2925 /* Attention!!! For SR-IOV PF driver operations you must enable
2926 * queue drop for all VF and PF queues to prevent head of line blocking
2927 * if an un-trusted VF does not provide descriptors to hardware.
2928 */
2929 if (adapter->vfs_allocated_count) {
e1739522
AD		 2930 /* set all queue drop enable bits */
2931 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 2932 }
2933
9d5c8243
AK		 2934 wr32(E1000_RCTL, rctl);
2935}
2936
7d5753f0
AD		 2937static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2938 int vfn)
2939{
2940 struct e1000_hw *hw = &adapter->hw;
2941 u32 vmolr;
2942
2943 /* if it isn't the PF check to see if VFs are enabled and
2944 * increase the size to support vlan tags */
2945 if (vfn < adapter->vfs_allocated_count &&
2946 adapter->vf_data[vfn].vlans_enabled)
2947 size += VLAN_TAG_SIZE;
2948
2949 vmolr = rd32(E1000_VMOLR(vfn));
2950 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2951 vmolr |= size | E1000_VMOLR_LPE;
2952 wr32(E1000_VMOLR(vfn), vmolr);
2953
2954 return 0;
2955}
2956
e1739522
AD		 2957/**
2958 * igb_rlpml_set - set maximum receive packet size
2959 * @adapter: board private structure
2960 *
2961 * Configure maximum receivable packet size.
2962 **/
2963static void igb_rlpml_set(struct igb_adapter *adapter)
2964{
b2cb09b1 2965 u32 max_frame_size;
e1739522
AD		 2966 struct e1000_hw *hw = &adapter->hw;
2967 u16 pf_id = adapter->vfs_allocated_count;
2968
b2cb09b1 2969 max_frame_size = adapter->max_frame_size + VLAN_TAG_SIZE;
e1739522
AD		 2970
2971 /* if vfs are enabled we set RLPML to the largest possible request
2972 * size and set the VMOLR RLPML to the size we need */
2973 if (pf_id) {
2974 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
7d5753f0 2975 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 2976 }
2977
2978 wr32(E1000_RLPML, max_frame_size);
2979}
2980
8151d294
WM		 2981static inline void igb_set_vmolr(struct igb_adapter *adapter,
2982 int vfn, bool aupe)
7d5753f0
AD		 2983{
2984 struct e1000_hw *hw = &adapter->hw;
2985 u32 vmolr;
2986
2987 /*
2988 * This register exists only on 82576 and newer so if we are older then
2989 * we should exit and do nothing
2990 */
2991 if (hw->mac.type < e1000_82576)
2992 return;
2993
2994 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 2995 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2996 if (aupe)
2997 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
2998 else
2999 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 3000
3001 /* clear all bits that might not be set */
3002 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
3003
a99955fc 3004 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 3005 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3006 /*
3007 * for VMDq only allow the VFs and pool 0 to accept broadcast and
3008 * multicast packets
3009 */
3010 if (vfn <= adapter->vfs_allocated_count)
3011 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
3012
3013 wr32(E1000_VMOLR(vfn), vmolr);
3014}
3015
85b430b4
AD		 3016/**
3017 * igb_configure_rx_ring - Configure a receive ring after Reset
3018 * @adapter: board private structure
3019 * @ring: receive ring to be configured
3020 *
3021 * Configure the Rx unit of the MAC after a reset.
3022 **/
d7ee5b3a
AD		 3023void igb_configure_rx_ring(struct igb_adapter *adapter,
3024 struct igb_ring *ring)
85b430b4
AD		 3025{
3026 struct e1000_hw *hw = &adapter->hw;
3027 u64 rdba = ring->dma;
3028 int reg_idx = ring->reg_idx;
952f72a8 3029 u32 srrctl, rxdctl;
85b430b4
AD		 3030
3031 /* disable the queue */
3032 rxdctl = rd32(E1000_RXDCTL(reg_idx));
3033 wr32(E1000_RXDCTL(reg_idx),
3034 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
3035
3036 /* Set DMA base address registers */
3037 wr32(E1000_RDBAL(reg_idx),
3038 rdba & 0x00000000ffffffffULL);
3039 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
3040 wr32(E1000_RDLEN(reg_idx),
3041 ring->count * sizeof(union e1000_adv_rx_desc));
3042
3043 /* initialize head and tail */
fce99e34
AD		 3044 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
3045 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
3046 writel(0, ring->head);
3047 writel(0, ring->tail);
85b430b4 3048
952f72a8 3049 /* set descriptor configuration */
4c844851
AD		 3050 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
3051 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
952f72a8
AD		 3052 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
3053#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
3054 srrctl |= IGB_RXBUFFER_16384 >>
3055 E1000_SRRCTL_BSIZEPKT_SHIFT;
3056#else
3057 srrctl |= (PAGE_SIZE / 2) >>
3058 E1000_SRRCTL_BSIZEPKT_SHIFT;
3059#endif
3060 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3061 } else {
4c844851 3062 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
952f72a8
AD		 3063 E1000_SRRCTL_BSIZEPKT_SHIFT;
3064 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
3065 }
757b77e2
NN		 3066 if (hw->mac.type == e1000_82580)
3067 srrctl |= E1000_SRRCTL_TIMESTAMP;
e6bdb6fe
NN		 3068 /* Only set Drop Enable if we are supporting multiple queues */
3069 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
3070 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 3071
3072 wr32(E1000_SRRCTL(reg_idx), srrctl);
3073
7d5753f0 3074 /* set filtering for VMDQ pools */
8151d294 3075 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 3076
85b430b4
AD		 3077 /* enable receive descriptor fetching */
3078 rxdctl = rd32(E1000_RXDCTL(reg_idx));
3079 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
3080 rxdctl &= 0xFFF00000;
3081 rxdctl |= IGB_RX_PTHRESH;
3082 rxdctl |= IGB_RX_HTHRESH << 8;
3083 rxdctl |= IGB_RX_WTHRESH << 16;
3084 wr32(E1000_RXDCTL(reg_idx), rxdctl);
3085}
3086
9d5c8243
AK		 3087/**
3088 * igb_configure_rx - Configure receive Unit after Reset
3089 * @adapter: board private structure
3090 *
3091 * Configure the Rx unit of the MAC after a reset.
3092 **/
3093static void igb_configure_rx(struct igb_adapter *adapter)
3094{
9107584e 3095 int i;
9d5c8243 3096
68d480c4
AD		 3097 /* set UTA to appropriate mode */
3098 igb_set_uta(adapter);
3099
26ad9178
AD		 3100 /* set the correct pool for the PF default MAC address in entry 0 */
3101 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
3102 adapter->vfs_allocated_count);
3103
06cf2666
AD		 3104 /* Setup the HW Rx Head and Tail Descriptor Pointers and
3105 * the Base and Length of the Rx Descriptor Ring */
3106 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3107 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 3108}
3109
3110/**
3111 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 3112 * @tx_ring: Tx descriptor ring for a specific queue
3113 *
3114 * Free all transmit software resources
3115 **/
68fd9910 3116void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 3117{
3b644cf6 3118 igb_clean_tx_ring(tx_ring);
9d5c8243
AK		 3119
3120 vfree(tx_ring->buffer_info);
3121 tx_ring->buffer_info = NULL;
3122
439705e1
AD		 3123 /* if not set, then don't free */
3124 if (!tx_ring->desc)
3125 return;
3126
59d71989
AD		 3127 dma_free_coherent(tx_ring->dev, tx_ring->size,
3128 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 3129
3130 tx_ring->desc = NULL;
3131}
3132
3133/**
3134 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3135 * @adapter: board private structure
3136 *
3137 * Free all transmit software resources
3138 **/
3139static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3140{
3141 int i;
3142
3143 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3144 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 3145}
3146
b1a436c3
AD		 3147void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
3148 struct igb_buffer *buffer_info)
9d5c8243 3149{
6366ad33
AD		 3150 if (buffer_info->dma) {
3151 if (buffer_info->mapped_as_page)
59d71989 3152 dma_unmap_page(tx_ring->dev,
6366ad33
AD		 3153 buffer_info->dma,
3154 buffer_info->length,
59d71989 3155 DMA_TO_DEVICE);
6366ad33 3156 else
59d71989 3157 dma_unmap_single(tx_ring->dev,
6366ad33
AD		 3158 buffer_info->dma,
3159 buffer_info->length,
59d71989 3160 DMA_TO_DEVICE);
6366ad33
AD		 3161 buffer_info->dma = 0;
3162 }
9d5c8243
AK		 3163 if (buffer_info->skb) {
3164 dev_kfree_skb_any(buffer_info->skb);
3165 buffer_info->skb = NULL;
3166 }
3167 buffer_info->time_stamp = 0;
6366ad33
AD		 3168 buffer_info->length = 0;
3169 buffer_info->next_to_watch = 0;
3170 buffer_info->mapped_as_page = false;
9d5c8243
AK		 3171}
3172
3173/**
3174 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 3175 * @tx_ring: ring to be cleaned
3176 **/
3b644cf6 3177static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243
AK		 3178{
3179 struct igb_buffer *buffer_info;
3180 unsigned long size;
3181 unsigned int i;
3182
3183 if (!tx_ring->buffer_info)
3184 return;
3185 /* Free all the Tx ring sk_buffs */
3186
3187 for (i = 0; i < tx_ring->count; i++) {
3188 buffer_info = &tx_ring->buffer_info[i];
80785298 3189 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 3190 }
3191
3192 size = sizeof(struct igb_buffer) * tx_ring->count;
3193 memset(tx_ring->buffer_info, 0, size);
3194
3195 /* Zero out the descriptor ring */
9d5c8243
AK		 3196 memset(tx_ring->desc, 0, tx_ring->size);
3197
3198 tx_ring->next_to_use = 0;
3199 tx_ring->next_to_clean = 0;
9d5c8243
AK		 3200}
3201
3202/**
3203 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3204 * @adapter: board private structure
3205 **/
3206static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3207{
3208 int i;
3209
3210 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3211 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 3212}
3213
3214/**
3215 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 3216 * @rx_ring: ring to clean the resources from
3217 *
3218 * Free all receive software resources
3219 **/
68fd9910 3220void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 3221{
3b644cf6 3222 igb_clean_rx_ring(rx_ring);
9d5c8243
AK		 3223
3224 vfree(rx_ring->buffer_info);
3225 rx_ring->buffer_info = NULL;
3226
439705e1
AD		 3227 /* if not set, then don't free */
3228 if (!rx_ring->desc)
3229 return;
3230
59d71989
AD		 3231 dma_free_coherent(rx_ring->dev, rx_ring->size,
3232 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 3233
3234 rx_ring->desc = NULL;
3235}
3236
3237/**
3238 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3239 * @adapter: board private structure
3240 *
3241 * Free all receive software resources
3242 **/
3243static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3244{
3245 int i;
3246
3247 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3248 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 3249}
3250
3251/**
3252 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 3253 * @rx_ring: ring to free buffers from
3254 **/
3b644cf6 3255static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243
AK		 3256{
3257 struct igb_buffer *buffer_info;
9d5c8243
AK		 3258 unsigned long size;
3259 unsigned int i;
3260
3261 if (!rx_ring->buffer_info)
3262 return;
439705e1 3263
9d5c8243
AK		 3264 /* Free all the Rx ring sk_buffs */
3265 for (i = 0; i < rx_ring->count; i++) {
3266 buffer_info = &rx_ring->buffer_info[i];
3267 if (buffer_info->dma) {
59d71989 3268 dma_unmap_single(rx_ring->dev,
80785298 3269 buffer_info->dma,
4c844851 3270 rx_ring->rx_buffer_len,
59d71989 3271 DMA_FROM_DEVICE);
9d5c8243
AK		 3272 buffer_info->dma = 0;
3273 }
3274
3275 if (buffer_info->skb) {
3276 dev_kfree_skb(buffer_info->skb);
3277 buffer_info->skb = NULL;
3278 }
6ec43fe6 3279 if (buffer_info->page_dma) {
59d71989 3280 dma_unmap_page(rx_ring->dev,
80785298 3281 buffer_info->page_dma,
6ec43fe6 3282 PAGE_SIZE / 2,
59d71989 3283 DMA_FROM_DEVICE);
6ec43fe6
AD		 3284 buffer_info->page_dma = 0;
3285 }
9d5c8243 3286 if (buffer_info->page) {
9d5c8243
AK		 3287 put_page(buffer_info->page);
3288 buffer_info->page = NULL;
bf36c1a0 3289 buffer_info->page_offset = 0;
9d5c8243
AK		 3290 }
3291 }
3292
9d5c8243
AK		 3293 size = sizeof(struct igb_buffer) * rx_ring->count;
3294 memset(rx_ring->buffer_info, 0, size);
3295
3296 /* Zero out the descriptor ring */
3297 memset(rx_ring->desc, 0, rx_ring->size);
3298
3299 rx_ring->next_to_clean = 0;
3300 rx_ring->next_to_use = 0;
9d5c8243
AK		 3301}
3302
3303/**
3304 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3305 * @adapter: board private structure
3306 **/
3307static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3308{
3309 int i;
3310
3311 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3312 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 3313}
3314
3315/**
3316 * igb_set_mac - Change the Ethernet Address of the NIC
3317 * @netdev: network interface device structure
3318 * @p: pointer to an address structure
3319 *
3320 * Returns 0 on success, negative on failure
3321 **/
3322static int igb_set_mac(struct net_device *netdev, void *p)
3323{
3324 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 3325 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 3326 struct sockaddr *addr = p;
3327
3328 if (!is_valid_ether_addr(addr->sa_data))
3329 return -EADDRNOTAVAIL;
3330
3331 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 3332 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 3333
26ad9178
AD		 3334 /* set the correct pool for the new PF MAC address in entry 0 */
3335 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3336 adapter->vfs_allocated_count);
e1739522 3337
9d5c8243
AK		 3338 return 0;
3339}
3340
3341/**
68d480c4 3342 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 3343 * @netdev: network interface device structure
3344 *
68d480c4
AD		 3345 * Writes multicast address list to the MTA hash table.
3346 * Returns: -ENOMEM on failure
3347 * 0 on no addresses written
3348 * X on writing X addresses to MTA
9d5c8243 3349 **/
68d480c4 3350static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 3351{
3352 struct igb_adapter *adapter = netdev_priv(netdev);
3353 struct e1000_hw *hw = &adapter->hw;
22bedad3 3354 struct netdev_hw_addr *ha;
68d480c4 3355 u8 *mta_list;
9d5c8243
AK		 3356 int i;
3357
4cd24eaf 3358 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 3359 /* nothing to program, so clear mc list */
3360 igb_update_mc_addr_list(hw, NULL, 0);
3361 igb_restore_vf_multicasts(adapter);
3362 return 0;
3363 }
9d5c8243 3364
4cd24eaf 3365 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 3366 if (!mta_list)
3367 return -ENOMEM;
ff41f8dc 3368
68d480c4 3369 /* The shared function expects a packed array of only addresses. */
48e2f183 3370 i = 0;
22bedad3
JP		 3371 netdev_for_each_mc_addr(ha, netdev)
3372 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
68d480c4 3373
68d480c4
AD		 3374 igb_update_mc_addr_list(hw, mta_list, i);
3375 kfree(mta_list);
3376
4cd24eaf 3377 return netdev_mc_count(netdev);
68d480c4
AD		 3378}
3379
3380/**
3381 * igb_write_uc_addr_list - write unicast addresses to RAR table
3382 * @netdev: network interface device structure
3383 *
3384 * Writes unicast address list to the RAR table.
3385 * Returns: -ENOMEM on failure/insufficient address space
3386 * 0 on no addresses written
3387 * X on writing X addresses to the RAR table
3388 **/
3389static int igb_write_uc_addr_list(struct net_device *netdev)
3390{
3391 struct igb_adapter *adapter = netdev_priv(netdev);
3392 struct e1000_hw *hw = &adapter->hw;
3393 unsigned int vfn = adapter->vfs_allocated_count;
3394 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3395 int count = 0;
3396
3397 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 3398 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 3399 return -ENOMEM;
9d5c8243 3400
32e7bfc4 3401 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 3402 struct netdev_hw_addr *ha;
32e7bfc4
JP		 3403
3404 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 3405 if (!rar_entries)
3406 break;
26ad9178
AD		 3407 igb_rar_set_qsel(adapter, ha->addr,
3408 rar_entries--,
68d480c4
AD		 3409 vfn);
3410 count++;
ff41f8dc
AD		 3411 }
3412 }
3413 /* write the addresses in reverse order to avoid write combining */
3414 for (; rar_entries > 0 ; rar_entries--) {
3415 wr32(E1000_RAH(rar_entries), 0);
3416 wr32(E1000_RAL(rar_entries), 0);
3417 }
3418 wrfl();
3419
68d480c4
AD		 3420 return count;
3421}
3422
3423/**
3424 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3425 * @netdev: network interface device structure
3426 *
3427 * The set_rx_mode entry point is called whenever the unicast or multicast
3428 * address lists or the network interface flags are updated. This routine is
3429 * responsible for configuring the hardware for proper unicast, multicast,
3430 * promiscuous mode, and all-multi behavior.
3431 **/
3432static void igb_set_rx_mode(struct net_device *netdev)
3433{
3434 struct igb_adapter *adapter = netdev_priv(netdev);
3435 struct e1000_hw *hw = &adapter->hw;
3436 unsigned int vfn = adapter->vfs_allocated_count;
3437 u32 rctl, vmolr = 0;
3438 int count;
3439
3440 /* Check for Promiscuous and All Multicast modes */
3441 rctl = rd32(E1000_RCTL);
3442
3443 /* clear the effected bits */
3444 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3445
3446 if (netdev->flags & IFF_PROMISC) {
3447 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3448 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3449 } else {
3450 if (netdev->flags & IFF_ALLMULTI) {
3451 rctl |= E1000_RCTL_MPE;
3452 vmolr |= E1000_VMOLR_MPME;
3453 } else {
3454 /*
3455 * Write addresses to the MTA, if the attempt fails
25985edc 3456 * then we should just turn on promiscuous mode so
68d480c4
AD		 3457 * that we can at least receive multicast traffic
3458 */
3459 count = igb_write_mc_addr_list(netdev);
3460 if (count < 0) {
3461 rctl |= E1000_RCTL_MPE;
3462 vmolr |= E1000_VMOLR_MPME;
3463 } else if (count) {
3464 vmolr |= E1000_VMOLR_ROMPE;
3465 }
3466 }
3467 /*
3468 * Write addresses to available RAR registers, if there is not
3469 * sufficient space to store all the addresses then enable
25985edc 3470 * unicast promiscuous mode
68d480c4
AD		 3471 */
3472 count = igb_write_uc_addr_list(netdev);
3473 if (count < 0) {
3474 rctl |= E1000_RCTL_UPE;
3475 vmolr |= E1000_VMOLR_ROPE;
3476 }
3477 rctl |= E1000_RCTL_VFE;
28fc06f5 3478 }
68d480c4 3479 wr32(E1000_RCTL, rctl);
28fc06f5 3480
68d480c4
AD		 3481 /*
3482 * In order to support SR-IOV and eventually VMDq it is necessary to set
3483 * the VMOLR to enable the appropriate modes. Without this workaround
3484 * we will have issues with VLAN tag stripping not being done for frames
3485 * that are only arriving because we are the default pool
3486 */
3487 if (hw->mac.type < e1000_82576)
28fc06f5 3488 return;
9d5c8243 3489
68d480c4
AD		 3490 vmolr |= rd32(E1000_VMOLR(vfn)) &
3491 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3492 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3493 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3494}
3495
13800469
GR		 3496static void igb_check_wvbr(struct igb_adapter *adapter)
3497{
3498 struct e1000_hw *hw = &adapter->hw;
3499 u32 wvbr = 0;
3500
3501 switch (hw->mac.type) {
3502 case e1000_82576:
3503 case e1000_i350:
3504 if (!(wvbr = rd32(E1000_WVBR)))
3505 return;
3506 break;
3507 default:
3508 break;
3509 }
3510
3511 adapter->wvbr |= wvbr;
3512}
3513
3514#define IGB_STAGGERED_QUEUE_OFFSET 8
3515
3516static void igb_spoof_check(struct igb_adapter *adapter)
3517{
3518 int j;
3519
3520 if (!adapter->wvbr)
3521 return;
3522
3523 for(j = 0; j < adapter->vfs_allocated_count; j++) {
3524 if (adapter->wvbr & (1 << j) ||
3525 adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3526 dev_warn(&adapter->pdev->dev,
3527 "Spoof event(s) detected on VF %d\n", j);
3528 adapter->wvbr &=
3529 ~((1 << j) |
3530 (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3531 }
3532 }
3533}
3534
9d5c8243
AK		 3535/* Need to wait a few seconds after link up to get diagnostic information from
3536 * the phy */
3537static void igb_update_phy_info(unsigned long data)
3538{
3539 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3540 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3541}
3542
4d6b725e
AD		 3543/**
3544 * igb_has_link - check shared code for link and determine up/down
3545 * @adapter: pointer to driver private info
3546 **/
3145535a 3547bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3548{
3549 struct e1000_hw *hw = &adapter->hw;
3550 bool link_active = false;
3551 s32 ret_val = 0;
3552
3553 /* get_link_status is set on LSC (link status) interrupt or
3554 * rx sequence error interrupt. get_link_status will stay
3555 * false until the e1000_check_for_link establishes link
3556 * for copper adapters ONLY
3557 */
3558 switch (hw->phy.media_type) {
3559 case e1000_media_type_copper:
3560 if (hw->mac.get_link_status) {
3561 ret_val = hw->mac.ops.check_for_link(hw);
3562 link_active = !hw->mac.get_link_status;
3563 } else {
3564 link_active = true;
3565 }
3566 break;
4d6b725e
AD		 3567 case e1000_media_type_internal_serdes:
3568 ret_val = hw->mac.ops.check_for_link(hw);
3569 link_active = hw->mac.serdes_has_link;
3570 break;
3571 default:
3572 case e1000_media_type_unknown:
3573 break;
3574 }
3575
3576 return link_active;
3577}
3578
563988dc
SA		 3579static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
3580{
3581 bool ret = false;
3582 u32 ctrl_ext, thstat;
3583
3584 /* check for thermal sensor event on i350, copper only */
3585 if (hw->mac.type == e1000_i350) {
3586 thstat = rd32(E1000_THSTAT);
3587 ctrl_ext = rd32(E1000_CTRL_EXT);
3588
3589 if ((hw->phy.media_type == e1000_media_type_copper) &&
3590 !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3591 ret = !!(thstat & event);
3592 }
3593 }
3594
3595 return ret;
3596}
3597
9d5c8243
AK		 3598/**
3599 * igb_watchdog - Timer Call-back
3600 * @data: pointer to adapter cast into an unsigned long
3601 **/
3602static void igb_watchdog(unsigned long data)
3603{
3604 struct igb_adapter *adapter = (struct igb_adapter *)data;
3605 /* Do the rest outside of interrupt context */
3606 schedule_work(&adapter->watchdog_task);
3607}
3608
3609static void igb_watchdog_task(struct work_struct *work)
3610{
3611 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3612 struct igb_adapter,
3613 watchdog_task);
9d5c8243 3614 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3615 struct net_device *netdev = adapter->netdev;
563988dc 3616 u32 link;
7a6ea550 3617 int i;
9d5c8243 3618
4d6b725e 3619 link = igb_has_link(adapter);
9d5c8243
AK		 3620 if (link) {
3621 if (!netif_carrier_ok(netdev)) {
3622 u32 ctrl;
330a6d6a
AD		 3623 hw->mac.ops.get_speed_and_duplex(hw,
3624 &adapter->link_speed,
3625 &adapter->link_duplex);
9d5c8243
AK		 3626
3627 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 3628 /* Links status message must follow this format */
3629 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 3630 "Flow Control: %s\n",
559e9c49
AD		 3631 netdev->name,
3632 adapter->link_speed,
3633 adapter->link_duplex == FULL_DUPLEX ?
9d5c8243 3634 "Full Duplex" : "Half Duplex",
559e9c49
AD		 3635 ((ctrl & E1000_CTRL_TFCE) &&
3636 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3637 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3638 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
9d5c8243 3639
563988dc
SA		 3640 /* check for thermal sensor event */
3641 if (igb_thermal_sensor_event(hw, E1000_THSTAT_LINK_THROTTLE)) {
3642 printk(KERN_INFO "igb: %s The network adapter "
3643 "link speed was downshifted "
3644 "because it overheated.\n",
3645 netdev->name);
7ef5ed1c 3646 }
563988dc 3647
d07f3e37 3648 /* adjust timeout factor according to speed/duplex */
9d5c8243
AK		 3649 adapter->tx_timeout_factor = 1;
3650 switch (adapter->link_speed) {
3651 case SPEED_10:
9d5c8243
AK		 3652 adapter->tx_timeout_factor = 14;
3653 break;
3654 case SPEED_100:
9d5c8243
AK		 3655 /* maybe add some timeout factor ? */
3656 break;
3657 }
3658
3659 netif_carrier_on(netdev);
9d5c8243 3660
4ae196df 3661 igb_ping_all_vfs(adapter);
17dc566c 3662 igb_check_vf_rate_limit(adapter);
4ae196df 3663
4b1a9877 3664 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3665 if (!test_bit(__IGB_DOWN, &adapter->state))
3666 mod_timer(&adapter->phy_info_timer,
3667 round_jiffies(jiffies + 2 * HZ));
3668 }
3669 } else {
3670 if (netif_carrier_ok(netdev)) {
3671 adapter->link_speed = 0;
3672 adapter->link_duplex = 0;
563988dc
SA		 3673
3674 /* check for thermal sensor event */
3675 if (igb_thermal_sensor_event(hw, E1000_THSTAT_PWR_DOWN)) {
3676 printk(KERN_ERR "igb: %s The network adapter "
3677 "was stopped because it "
3678 "overheated.\n",
7ef5ed1c 3679 netdev->name);
7ef5ed1c 3680 }
563988dc 3681
527d47c1
AD		 3682 /* Links status message must follow this format */
3683 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3684 netdev->name);
9d5c8243 3685 netif_carrier_off(netdev);
4b1a9877 3686
4ae196df
AD		 3687 igb_ping_all_vfs(adapter);
3688
4b1a9877 3689 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3690 if (!test_bit(__IGB_DOWN, &adapter->state))
3691 mod_timer(&adapter->phy_info_timer,
3692 round_jiffies(jiffies + 2 * HZ));
3693 }
3694 }
3695
12dcd86b
ED		 3696 spin_lock(&adapter->stats64_lock);
3697 igb_update_stats(adapter, &adapter->stats64);
3698 spin_unlock(&adapter->stats64_lock);
9d5c8243 3699
dbabb065 3700 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3701 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3702 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3703 /* We've lost link, so the controller stops DMA,
3704 * but we've got queued Tx work that's never going
3705 * to get done, so reset controller to flush Tx.
3706 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3707 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3708 adapter->tx_timeout_count++;
3709 schedule_work(&adapter->reset_task);
3710 /* return immediately since reset is imminent */
3711 return;
3712 }
9d5c8243 3713 }
9d5c8243 3714
dbabb065
AD		 3715 /* Force detection of hung controller every watchdog period */
3716 tx_ring->detect_tx_hung = true;
3717 }
f7ba205e 3718
9d5c8243 3719 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3720 if (adapter->msix_entries) {
047e0030
AD		 3721 u32 eics = 0;
3722 for (i = 0; i < adapter->num_q_vectors; i++) {
3723 struct igb_q_vector *q_vector = adapter->q_vector[i];
3724 eics |= q_vector->eims_value;
3725 }
7a6ea550
AD		 3726 wr32(E1000_EICS, eics);
3727 } else {
3728 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3729 }
9d5c8243 3730
13800469
GR		 3731 igb_spoof_check(adapter);
3732
9d5c8243
AK		 3733 /* Reset the timer */
3734 if (!test_bit(__IGB_DOWN, &adapter->state))
3735 mod_timer(&adapter->watchdog_timer,
3736 round_jiffies(jiffies + 2 * HZ));
3737}
3738
3739enum latency_range {
3740 lowest_latency = 0,
3741 low_latency = 1,
3742 bulk_latency = 2,
3743 latency_invalid = 255
3744};
3745
6eb5a7f1
AD		 3746/**
3747 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3748 *
3749 * Stores a new ITR value based on strictly on packet size. This
3750 * algorithm is less sophisticated than that used in igb_update_itr,
3751 * due to the difficulty of synchronizing statistics across multiple
eef35c2d 3752 * receive rings. The divisors and thresholds used by this function
6eb5a7f1
AD		 3753 * were determined based on theoretical maximum wire speed and testing
3754 * data, in order to minimize response time while increasing bulk
3755 * throughput.
3756 * This functionality is controlled by the InterruptThrottleRate module
3757 * parameter (see igb_param.c)
3758 * NOTE: This function is called only when operating in a multiqueue
3759 * receive environment.
047e0030 3760 * @q_vector: pointer to q_vector
6eb5a7f1 3761 **/
047e0030 3762static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3763{
047e0030 3764 int new_val = q_vector->itr_val;
6eb5a7f1 3765 int avg_wire_size = 0;
047e0030 3766 struct igb_adapter *adapter = q_vector->adapter;
12dcd86b
ED		 3767 struct igb_ring *ring;
3768 unsigned int packets;
9d5c8243 3769
6eb5a7f1
AD		 3770 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3771 * ints/sec - ITR timer value of 120 ticks.
3772 */
3773 if (adapter->link_speed != SPEED_1000) {
047e0030 3774 new_val = 976;
6eb5a7f1 3775 goto set_itr_val;
9d5c8243 3776 }
047e0030 3777
12dcd86b
ED		 3778 ring = q_vector->rx_ring;
3779 if (ring) {
3780 packets = ACCESS_ONCE(ring->total_packets);
3781
3782 if (packets)
3783 avg_wire_size = ring->total_bytes / packets;
047e0030
AD		 3784 }
3785
12dcd86b
ED		 3786 ring = q_vector->tx_ring;
3787 if (ring) {
3788 packets = ACCESS_ONCE(ring->total_packets);
3789
3790 if (packets)
3791 avg_wire_size = max_t(u32, avg_wire_size,
3792 ring->total_bytes / packets);
047e0030
AD		 3793 }
3794
3795 /* if avg_wire_size isn't set no work was done */
3796 if (!avg_wire_size)
3797 goto clear_counts;
9d5c8243 3798
6eb5a7f1
AD		 3799 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3800 avg_wire_size += 24;
3801
3802 /* Don't starve jumbo frames */
3803 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3804
6eb5a7f1
AD		 3805 /* Give a little boost to mid-size frames */
3806 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3807 new_val = avg_wire_size / 3;
3808 else
3809 new_val = avg_wire_size / 2;
9d5c8243 3810
abe1c363
NN		 3811 /* when in itr mode 3 do not exceed 20K ints/sec */
3812 if (adapter->rx_itr_setting == 3 && new_val < 196)
3813 new_val = 196;
3814
6eb5a7f1 3815set_itr_val:
047e0030
AD		 3816 if (new_val != q_vector->itr_val) {
3817 q_vector->itr_val = new_val;
3818 q_vector->set_itr = 1;
9d5c8243 3819 }
6eb5a7f1 3820clear_counts:
047e0030
AD		 3821 if (q_vector->rx_ring) {
3822 q_vector->rx_ring->total_bytes = 0;
3823 q_vector->rx_ring->total_packets = 0;
3824 }
3825 if (q_vector->tx_ring) {
3826 q_vector->tx_ring->total_bytes = 0;
3827 q_vector->tx_ring->total_packets = 0;
3828 }
9d5c8243
AK		 3829}
3830
3831/**
3832 * igb_update_itr - update the dynamic ITR value based on statistics
3833 * Stores a new ITR value based on packets and byte
3834 * counts during the last interrupt. The advantage of per interrupt
3835 * computation is faster updates and more accurate ITR for the current
3836 * traffic pattern. Constants in this function were computed
3837 * based on theoretical maximum wire speed and thresholds were set based
3838 * on testing data as well as attempting to minimize response time
3839 * while increasing bulk throughput.
3840 * this functionality is controlled by the InterruptThrottleRate module
3841 * parameter (see igb_param.c)
3842 * NOTE: These calculations are only valid when operating in a single-
3843 * queue environment.
3844 * @adapter: pointer to adapter
047e0030 3845 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3846 * @packets: the number of packets during this measurement interval
3847 * @bytes: the number of bytes during this measurement interval
3848 **/
3849static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3850 int packets, int bytes)
3851{
3852 unsigned int retval = itr_setting;
3853
3854 if (packets == 0)
3855 goto update_itr_done;
3856
3857 switch (itr_setting) {
3858 case lowest_latency:
3859 /* handle TSO and jumbo frames */
3860 if (bytes/packets > 8000)
3861 retval = bulk_latency;
3862 else if ((packets < 5) && (bytes > 512))
3863 retval = low_latency;
3864 break;
3865 case low_latency: /* 50 usec aka 20000 ints/s */
3866 if (bytes > 10000) {
3867 /* this if handles the TSO accounting */
3868 if (bytes/packets > 8000) {
3869 retval = bulk_latency;
3870 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3871 retval = bulk_latency;
3872 } else if ((packets > 35)) {
3873 retval = lowest_latency;
3874 }
3875 } else if (bytes/packets > 2000) {
3876 retval = bulk_latency;
3877 } else if (packets <= 2 && bytes < 512) {
3878 retval = lowest_latency;
3879 }
3880 break;
3881 case bulk_latency: /* 250 usec aka 4000 ints/s */
3882 if (bytes > 25000) {
3883 if (packets > 35)
3884 retval = low_latency;
1e5c3d21 3885 } else if (bytes < 1500) {
9d5c8243
AK		 3886 retval = low_latency;
3887 }
3888 break;
3889 }
3890
3891update_itr_done:
3892 return retval;
3893}
3894
6eb5a7f1 3895static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3896{
047e0030 3897 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3898 u16 current_itr;
047e0030 3899 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3900
3901 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3902 if (adapter->link_speed != SPEED_1000) {
3903 current_itr = 0;
3904 new_itr = 4000;
3905 goto set_itr_now;
3906 }
3907
3908 adapter->rx_itr = igb_update_itr(adapter,
3909 adapter->rx_itr,
3025a446
AD		 3910 q_vector->rx_ring->total_packets,
3911 q_vector->rx_ring->total_bytes);
9d5c8243 3912
047e0030
AD		 3913 adapter->tx_itr = igb_update_itr(adapter,
3914 adapter->tx_itr,
3025a446
AD		 3915 q_vector->tx_ring->total_packets,
3916 q_vector->tx_ring->total_bytes);
047e0030 3917 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3918
6eb5a7f1 3919 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4fc82adf 3920 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3921 current_itr = low_latency;
3922
9d5c8243
AK		 3923 switch (current_itr) {
3924 /* counts and packets in update_itr are dependent on these numbers */
3925 case lowest_latency:
78b1f607 3926 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3927 break;
3928 case low_latency:
78b1f607 3929 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3930 break;
3931 case bulk_latency:
78b1f607 3932 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3933 break;
3934 default:
3935 break;
3936 }
3937
3938set_itr_now:
3025a446
AD		 3939 q_vector->rx_ring->total_bytes = 0;
3940 q_vector->rx_ring->total_packets = 0;
3941 q_vector->tx_ring->total_bytes = 0;
3942 q_vector->tx_ring->total_packets = 0;
6eb5a7f1 3943
047e0030 3944 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3945 /* this attempts to bias the interrupt rate towards Bulk
3946 * by adding intermediate steps when interrupt rate is
3947 * increasing */
047e0030
AD		 3948 new_itr = new_itr > q_vector->itr_val ?
3949 max((new_itr * q_vector->itr_val) /
3950 (new_itr + (q_vector->itr_val >> 2)),
3951 new_itr) :
9d5c8243
AK		 3952 new_itr;
3953 /* Don't write the value here; it resets the adapter's
3954 * internal timer, and causes us to delay far longer than
3955 * we should between interrupts. Instead, we write the ITR
3956 * value at the beginning of the next interrupt so the timing
3957 * ends up being correct.
3958 */
047e0030
AD		 3959 q_vector->itr_val = new_itr;
3960 q_vector->set_itr = 1;
9d5c8243 3961 }
9d5c8243
AK		 3962}
3963
9d5c8243
AK		 3964#define IGB_TX_FLAGS_CSUM 0x00000001
3965#define IGB_TX_FLAGS_VLAN 0x00000002
3966#define IGB_TX_FLAGS_TSO 0x00000004
3967#define IGB_TX_FLAGS_IPV4 0x00000008
cdfd01fc
AD		 3968#define IGB_TX_FLAGS_TSTAMP 0x00000010
3969#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3970#define IGB_TX_FLAGS_VLAN_SHIFT 16
9d5c8243 3971
85ad76b2 3972static inline int igb_tso_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3973 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3974{
3975 struct e1000_adv_tx_context_desc *context_desc;
3976 unsigned int i;
3977 int err;
3978 struct igb_buffer *buffer_info;
3979 u32 info = 0, tu_cmd = 0;
91d4ee33
NN		 3980 u32 mss_l4len_idx;
3981 u8 l4len;
9d5c8243
AK		 3982
3983 if (skb_header_cloned(skb)) {
3984 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3985 if (err)
3986 return err;
3987 }
3988
3989 l4len = tcp_hdrlen(skb);
3990 *hdr_len += l4len;
3991
3992 if (skb->protocol == htons(ETH_P_IP)) {
3993 struct iphdr *iph = ip_hdr(skb);
3994 iph->tot_len = 0;
3995 iph->check = 0;
3996 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3997 iph->daddr, 0,
3998 IPPROTO_TCP,
3999 0);
8e1e8a47 4000 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 4001 ipv6_hdr(skb)->payload_len = 0;
4002 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4003 &ipv6_hdr(skb)->daddr,
4004 0, IPPROTO_TCP, 0);
4005 }
4006
4007 i = tx_ring->next_to_use;
4008
4009 buffer_info = &tx_ring->buffer_info[i];
4010 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
4011 /* VLAN MACLEN IPLEN */
4012 if (tx_flags & IGB_TX_FLAGS_VLAN)
4013 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
4014 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
4015 *hdr_len += skb_network_offset(skb);
4016 info |= skb_network_header_len(skb);
4017 *hdr_len += skb_network_header_len(skb);
4018 context_desc->vlan_macip_lens = cpu_to_le32(info);
4019
4020 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4021 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
4022
4023 if (skb->protocol == htons(ETH_P_IP))
4024 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
4025 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4026
4027 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
4028
4029 /* MSS L4LEN IDX */
4030 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
4031 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
4032
73cd78f1 4033 /* For 82575, context index must be unique per ring. */
85ad76b2
AD		 4034 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
4035 mss_l4len_idx |= tx_ring->reg_idx << 4;
9d5c8243
AK		 4036
4037 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
4038 context_desc->seqnum_seed = 0;
4039
4040 buffer_info->time_stamp = jiffies;
0e014cb1 4041 buffer_info->next_to_watch = i;
9d5c8243
AK		 4042 buffer_info->dma = 0;
4043 i++;
4044 if (i == tx_ring->count)
4045 i = 0;
4046
4047 tx_ring->next_to_use = i;
4048
4049 return true;
4050}
4051
85ad76b2
AD		 4052static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
4053 struct sk_buff *skb, u32 tx_flags)
9d5c8243
AK		 4054{
4055 struct e1000_adv_tx_context_desc *context_desc;
59d71989 4056 struct device *dev = tx_ring->dev;
9d5c8243
AK		 4057 struct igb_buffer *buffer_info;
4058 u32 info = 0, tu_cmd = 0;
80785298 4059 unsigned int i;
9d5c8243
AK		 4060
4061 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
4062 (tx_flags & IGB_TX_FLAGS_VLAN)) {
4063 i = tx_ring->next_to_use;
4064 buffer_info = &tx_ring->buffer_info[i];
4065 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
4066
4067 if (tx_flags & IGB_TX_FLAGS_VLAN)
4068 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
cdfd01fc 4069
9d5c8243
AK		 4070 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
4071 if (skb->ip_summed == CHECKSUM_PARTIAL)
4072 info |= skb_network_header_len(skb);
4073
4074 context_desc->vlan_macip_lens = cpu_to_le32(info);
4075
4076 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
4077
4078 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fa4a7ef3
AJ		 4079 __be16 protocol;
4080
4081 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
4082 const struct vlan_ethhdr *vhdr =
4083 (const struct vlan_ethhdr*)skb->data;
4084
4085 protocol = vhdr->h_vlan_encapsulated_proto;
4086 } else {
4087 protocol = skb->protocol;
4088 }
4089
4090 switch (protocol) {
09640e63 4091 case cpu_to_be16(ETH_P_IP):
9d5c8243 4092 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
44b0cda3
MW		 4093 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
4094 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 4095 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
4096 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3 4097 break;
09640e63 4098 case cpu_to_be16(ETH_P_IPV6):
44b0cda3
MW		 4099 /* XXX what about other V6 headers?? */
4100 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
4101 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 4102 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
4103 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3
MW		 4104 break;
4105 default:
4106 if (unlikely(net_ratelimit()))
59d71989 4107 dev_warn(dev,
44b0cda3
MW		 4108 "partial checksum but proto=%x!\n",
4109 skb->protocol);
4110 break;
4111 }
9d5c8243
AK		 4112 }
4113
4114 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
4115 context_desc->seqnum_seed = 0;
85ad76b2 4116 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
7dfc16fa 4117 context_desc->mss_l4len_idx =
85ad76b2 4118 cpu_to_le32(tx_ring->reg_idx << 4);
9d5c8243
AK		 4119
4120 buffer_info->time_stamp = jiffies;
0e014cb1 4121 buffer_info->next_to_watch = i;
9d5c8243
AK		 4122 buffer_info->dma = 0;
4123
4124 i++;
4125 if (i == tx_ring->count)
4126 i = 0;
4127 tx_ring->next_to_use = i;
4128
4129 return true;
4130 }
9d5c8243
AK		 4131 return false;
4132}
4133
4134#define IGB_MAX_TXD_PWR 16
4135#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
4136
80785298 4137static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
0e014cb1 4138 unsigned int first)
9d5c8243
AK		 4139{
4140 struct igb_buffer *buffer_info;
59d71989 4141 struct device *dev = tx_ring->dev;
2873957d 4142 unsigned int hlen = skb_headlen(skb);
9d5c8243
AK		 4143 unsigned int count = 0, i;
4144 unsigned int f;
2873957d 4145 u16 gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
9d5c8243
AK		 4146
4147 i = tx_ring->next_to_use;
4148
4149 buffer_info = &tx_ring->buffer_info[i];
2873957d
NN		 4150 BUG_ON(hlen >= IGB_MAX_DATA_PER_TXD);
4151 buffer_info->length = hlen;
9d5c8243
AK		 4152 /* set time_stamp *before* dma to help avoid a possible race */
4153 buffer_info->time_stamp = jiffies;
0e014cb1 4154 buffer_info->next_to_watch = i;
2873957d 4155 buffer_info->dma = dma_map_single(dev, skb->data, hlen,
59d71989
AD		 4156 DMA_TO_DEVICE);
4157 if (dma_mapping_error(dev, buffer_info->dma))
6366ad33 4158 goto dma_error;
9d5c8243
AK		 4159
4160 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
2873957d
NN		 4161 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[f];
4162 unsigned int len = frag->size;
9d5c8243 4163
8581145f 4164 count++;
65689fef
AD		 4165 i++;
4166 if (i == tx_ring->count)
4167 i = 0;
4168
9d5c8243
AK		 4169 buffer_info = &tx_ring->buffer_info[i];
4170 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
4171 buffer_info->length = len;
4172 buffer_info->time_stamp = jiffies;
0e014cb1 4173 buffer_info->next_to_watch = i;
6366ad33 4174 buffer_info->mapped_as_page = true;
59d71989 4175 buffer_info->dma = dma_map_page(dev,
6366ad33
AD		 4176 frag->page,
4177 frag->page_offset,
4178 len,
59d71989
AD		 4179 DMA_TO_DEVICE);
4180 if (dma_mapping_error(dev, buffer_info->dma))
6366ad33
AD		 4181 goto dma_error;
4182
9d5c8243
AK		 4183 }
4184
9d5c8243 4185 tx_ring->buffer_info[i].skb = skb;
2244d07b 4186 tx_ring->buffer_info[i].tx_flags = skb_shinfo(skb)->tx_flags;
2873957d
NN		 4187 /* multiply data chunks by size of headers */
4188 tx_ring->buffer_info[i].bytecount = ((gso_segs - 1) * hlen) + skb->len;
4189 tx_ring->buffer_info[i].gso_segs = gso_segs;
0e014cb1 4190 tx_ring->buffer_info[first].next_to_watch = i;
9d5c8243 4191
cdfd01fc 4192 return ++count;
6366ad33
AD		 4193
4194dma_error:
59d71989 4195 dev_err(dev, "TX DMA map failed\n");
6366ad33
AD		 4196
4197 /* clear timestamp and dma mappings for failed buffer_info mapping */
4198 buffer_info->dma = 0;
4199 buffer_info->time_stamp = 0;
4200 buffer_info->length = 0;
4201 buffer_info->next_to_watch = 0;
4202 buffer_info->mapped_as_page = false;
6366ad33
AD		 4203
4204 /* clear timestamp and dma mappings for remaining portion of packet */
a77ff709
NN		 4205 while (count--) {
4206 if (i == 0)
4207 i = tx_ring->count;
6366ad33 4208 i--;
6366ad33
AD		 4209 buffer_info = &tx_ring->buffer_info[i];
4210 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
4211 }
4212
4213 return 0;
9d5c8243
AK		 4214}
4215
85ad76b2 4216static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
91d4ee33 4217 u32 tx_flags, int count, u32 paylen,
9d5c8243
AK		 4218 u8 hdr_len)
4219{
cdfd01fc 4220 union e1000_adv_tx_desc *tx_desc;
9d5c8243
AK		 4221 struct igb_buffer *buffer_info;
4222 u32 olinfo_status = 0, cmd_type_len;
cdfd01fc 4223 unsigned int i = tx_ring->next_to_use;
9d5c8243
AK		 4224
4225 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
4226 E1000_ADVTXD_DCMD_DEXT);
4227
4228 if (tx_flags & IGB_TX_FLAGS_VLAN)
4229 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
4230
33af6bcc
PO		 4231 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4232 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
4233
9d5c8243
AK		 4234 if (tx_flags & IGB_TX_FLAGS_TSO) {
4235 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
4236
4237 /* insert tcp checksum */
4238 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4239
4240 /* insert ip checksum */
4241 if (tx_flags & IGB_TX_FLAGS_IPV4)
4242 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4243
4244 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
4245 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4246 }
4247
85ad76b2
AD		 4248 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
4249 (tx_flags & (IGB_TX_FLAGS_CSUM |
4250 IGB_TX_FLAGS_TSO |
7dfc16fa 4251 IGB_TX_FLAGS_VLAN)))
85ad76b2 4252 olinfo_status |= tx_ring->reg_idx << 4;
9d5c8243
AK		 4253
4254 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
4255
cdfd01fc 4256 do {
9d5c8243
AK		 4257 buffer_info = &tx_ring->buffer_info[i];
4258 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
4259 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
4260 tx_desc->read.cmd_type_len =
4261 cpu_to_le32(cmd_type_len | buffer_info->length);
4262 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
cdfd01fc 4263 count--;
9d5c8243
AK		 4264 i++;
4265 if (i == tx_ring->count)
4266 i = 0;
cdfd01fc 4267 } while (count > 0);
9d5c8243 4268
85ad76b2 4269 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
9d5c8243
AK		 4270 /* Force memory writes to complete before letting h/w
4271 * know there are new descriptors to fetch. (Only
4272 * applicable for weak-ordered memory model archs,
4273 * such as IA-64). */
4274 wmb();
4275
4276 tx_ring->next_to_use = i;
fce99e34 4277 writel(i, tx_ring->tail);
9d5c8243
AK		 4278 /* we need this if more than one processor can write to our tail
4279 * at a time, it syncronizes IO on IA64/Altix systems */
4280 mmiowb();
4281}
4282
e694e964 4283static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4284{
e694e964
AD		 4285 struct net_device *netdev = tx_ring->netdev;
4286
661086df 4287 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 4288
9d5c8243
AK		 4289 /* Herbert's original patch had:
4290 * smp_mb__after_netif_stop_queue();
4291 * but since that doesn't exist yet, just open code it. */
4292 smp_mb();
4293
4294 /* We need to check again in a case another CPU has just
4295 * made room available. */
c493ea45 4296 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 4297 return -EBUSY;
4298
4299 /* A reprieve! */
661086df 4300 netif_wake_subqueue(netdev, tx_ring->queue_index);
12dcd86b
ED		 4301
4302 u64_stats_update_begin(&tx_ring->tx_syncp2);
4303 tx_ring->tx_stats.restart_queue2++;
4304 u64_stats_update_end(&tx_ring->tx_syncp2);
4305
9d5c8243
AK		 4306 return 0;
4307}
4308
717ba089 4309static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4310{
c493ea45 4311 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 4312 return 0;
e694e964 4313 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 4314}
4315
b1a436c3
AD		 4316netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
4317 struct igb_ring *tx_ring)
9d5c8243 4318{
cdfd01fc 4319 int tso = 0, count;
91d4ee33
NN		 4320 u32 tx_flags = 0;
4321 u16 first;
4322 u8 hdr_len = 0;
9d5c8243 4323
9d5c8243
AK		 4324 /* need: 1 descriptor per page,
4325 * + 2 desc gap to keep tail from touching head,
4326 * + 1 desc for skb->data,
4327 * + 1 desc for context descriptor,
4328 * otherwise try next time */
e694e964 4329 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 4330 /* this is a hard error */
9d5c8243
AK		 4331 return NETDEV_TX_BUSY;
4332 }
33af6bcc 4333
2244d07b
OH		 4334 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4335 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
33af6bcc 4336 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 4337 }
9d5c8243 4338
eab6d18d 4339 if (vlan_tx_tag_present(skb)) {
9d5c8243
AK		 4340 tx_flags |= IGB_TX_FLAGS_VLAN;
4341 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4342 }
4343
661086df
PWJ		 4344 if (skb->protocol == htons(ETH_P_IP))
4345 tx_flags |= IGB_TX_FLAGS_IPV4;
4346
0e014cb1 4347 first = tx_ring->next_to_use;
85ad76b2
AD		 4348 if (skb_is_gso(skb)) {
4349 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
cdfd01fc 4350
85ad76b2
AD		 4351 if (tso < 0) {
4352 dev_kfree_skb_any(skb);
4353 return NETDEV_TX_OK;
4354 }
9d5c8243
AK		 4355 }
4356
4357 if (tso)
4358 tx_flags |= IGB_TX_FLAGS_TSO;
85ad76b2 4359 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
bc1cbd34
AD		 4360 (skb->ip_summed == CHECKSUM_PARTIAL))
4361 tx_flags |= IGB_TX_FLAGS_CSUM;
9d5c8243 4362
65689fef 4363 /*
cdfd01fc 4364 * count reflects descriptors mapped, if 0 or less then mapping error
25985edc 4365 * has occurred and we need to rewind the descriptor queue
65689fef 4366 */
80785298 4367 count = igb_tx_map_adv(tx_ring, skb, first);
6366ad33 4368 if (!count) {
65689fef
AD		 4369 dev_kfree_skb_any(skb);
4370 tx_ring->buffer_info[first].time_stamp = 0;
4371 tx_ring->next_to_use = first;
85ad76b2 4372 return NETDEV_TX_OK;
65689fef 4373 }
9d5c8243 4374
85ad76b2
AD		 4375 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
4376
4377 /* Make sure there is space in the ring for the next send. */
e694e964 4378 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 4379
9d5c8243
AK		 4380 return NETDEV_TX_OK;
4381}
4382
3b29a56d
SH		 4383static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
4384 struct net_device *netdev)
9d5c8243
AK		 4385{
4386 struct igb_adapter *adapter = netdev_priv(netdev);
661086df 4387 struct igb_ring *tx_ring;
661086df 4388 int r_idx = 0;
b1a436c3
AD		 4389
4390 if (test_bit(__IGB_DOWN, &adapter->state)) {
4391 dev_kfree_skb_any(skb);
4392 return NETDEV_TX_OK;
4393 }
4394
4395 if (skb->len <= 0) {
4396 dev_kfree_skb_any(skb);
4397 return NETDEV_TX_OK;
4398 }
4399
1bfaf07b 4400 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
661086df 4401 tx_ring = adapter->multi_tx_table[r_idx];
9d5c8243
AK		 4402
4403 /* This goes back to the question of how to logically map a tx queue
4404 * to a flow. Right now, performance is impacted slightly negatively
4405 * if using multiple tx queues. If the stack breaks away from a
4406 * single qdisc implementation, we can look at this again. */
e694e964 4407 return igb_xmit_frame_ring_adv(skb, tx_ring);
9d5c8243
AK		 4408}
4409
4410/**
4411 * igb_tx_timeout - Respond to a Tx Hang
4412 * @netdev: network interface device structure
4413 **/
4414static void igb_tx_timeout(struct net_device *netdev)
4415{
4416 struct igb_adapter *adapter = netdev_priv(netdev);
4417 struct e1000_hw *hw = &adapter->hw;
4418
4419 /* Do the reset outside of interrupt context */
4420 adapter->tx_timeout_count++;
f7ba205e 4421
55cac248
AD		 4422 if (hw->mac.type == e1000_82580)
4423 hw->dev_spec._82575.global_device_reset = true;
4424
9d5c8243 4425 schedule_work(&adapter->reset_task);
265de409
AD		 4426 wr32(E1000_EICS,
4427 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 4428}
4429
4430static void igb_reset_task(struct work_struct *work)
4431{
4432 struct igb_adapter *adapter;
4433 adapter = container_of(work, struct igb_adapter, reset_task);
4434
c97ec42a
TI		 4435 igb_dump(adapter);
4436 netdev_err(adapter->netdev, "Reset adapter\n");
9d5c8243
AK		 4437 igb_reinit_locked(adapter);
4438}
4439
4440/**
12dcd86b 4441 * igb_get_stats64 - Get System Network Statistics
9d5c8243 4442 * @netdev: network interface device structure
12dcd86b 4443 * @stats: rtnl_link_stats64 pointer
9d5c8243 4444 *
9d5c8243 4445 **/
12dcd86b
ED		 4446static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4447 struct rtnl_link_stats64 *stats)
9d5c8243 4448{
12dcd86b
ED		 4449 struct igb_adapter *adapter = netdev_priv(netdev);
4450
4451 spin_lock(&adapter->stats64_lock);
4452 igb_update_stats(adapter, &adapter->stats64);
4453 memcpy(stats, &adapter->stats64, sizeof(*stats));
4454 spin_unlock(&adapter->stats64_lock);
4455
4456 return stats;
9d5c8243
AK		 4457}
4458
4459/**
4460 * igb_change_mtu - Change the Maximum Transfer Unit
4461 * @netdev: network interface device structure
4462 * @new_mtu: new value for maximum frame size
4463 *
4464 * Returns 0 on success, negative on failure
4465 **/
4466static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4467{
4468 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4469 struct pci_dev *pdev = adapter->pdev;
9d5c8243 4470 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4c844851 4471 u32 rx_buffer_len, i;
9d5c8243 4472
c809d227 4473 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 4474 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 4475 return -EINVAL;
4476 }
4477
9d5c8243 4478 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 4479 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 4480 return -EINVAL;
4481 }
4482
4483 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4484 msleep(1);
73cd78f1 4485
9d5c8243
AK		 4486 /* igb_down has a dependency on max_frame_size */
4487 adapter->max_frame_size = max_frame;
559e9c49 4488
9d5c8243
AK		 4489 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
4490 * means we reserve 2 more, this pushes us to allocate from the next
4491 * larger slab size.
4492 * i.e. RXBUFFER_2048 --> size-4096 slab
4493 */
4494
757b77e2
NN		 4495 if (adapter->hw.mac.type == e1000_82580)
4496 max_frame += IGB_TS_HDR_LEN;
4497
7d95b717 4498 if (max_frame <= IGB_RXBUFFER_1024)
4c844851 4499 rx_buffer_len = IGB_RXBUFFER_1024;
6ec43fe6 4500 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4c844851 4501 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
6ec43fe6 4502 else
4c844851
AD		 4503 rx_buffer_len = IGB_RXBUFFER_128;
4504
757b77e2
NN		 4505 if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN + IGB_TS_HDR_LEN) ||
4506 (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN))
4507 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN;
4508
4509 if ((adapter->hw.mac.type == e1000_82580) &&
4510 (rx_buffer_len == IGB_RXBUFFER_128))
4511 rx_buffer_len += IGB_RXBUFFER_64;
4512
4c844851
AD		 4513 if (netif_running(netdev))
4514 igb_down(adapter);
9d5c8243 4515
090b1795 4516 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 4517 netdev->mtu, new_mtu);
4518 netdev->mtu = new_mtu;
4519
4c844851 4520 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 4521 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4c844851 4522
9d5c8243
AK		 4523 if (netif_running(netdev))
4524 igb_up(adapter);
4525 else
4526 igb_reset(adapter);
4527
4528 clear_bit(__IGB_RESETTING, &adapter->state);
4529
4530 return 0;
4531}
4532
4533/**
4534 * igb_update_stats - Update the board statistics counters
4535 * @adapter: board private structure
4536 **/
4537
12dcd86b
ED		 4538void igb_update_stats(struct igb_adapter *adapter,
4539 struct rtnl_link_stats64 *net_stats)
9d5c8243
AK		 4540{
4541 struct e1000_hw *hw = &adapter->hw;
4542 struct pci_dev *pdev = adapter->pdev;
fa3d9a6d 4543 u32 reg, mpc;
9d5c8243 4544 u16 phy_tmp;
3f9c0164
AD		 4545 int i;
4546 u64 bytes, packets;
12dcd86b
ED		 4547 unsigned int start;
4548 u64 _bytes, _packets;
9d5c8243
AK		 4549
4550#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4551
4552 /*
4553 * Prevent stats update while adapter is being reset, or if the pci
4554 * connection is down.
4555 */
4556 if (adapter->link_speed == 0)
4557 return;
4558 if (pci_channel_offline(pdev))
4559 return;
4560
3f9c0164
AD		 4561 bytes = 0;
4562 packets = 0;
4563 for (i = 0; i < adapter->num_rx_queues; i++) {
4564 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446 4565 struct igb_ring *ring = adapter->rx_ring[i];
12dcd86b 4566
3025a446 4567 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 4568 net_stats->rx_fifo_errors += rqdpc_tmp;
12dcd86b
ED		 4569
4570 do {
4571 start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4572 _bytes = ring->rx_stats.bytes;
4573 _packets = ring->rx_stats.packets;
4574 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4575 bytes += _bytes;
4576 packets += _packets;
3f9c0164
AD		 4577 }
4578
128e45eb
AD		 4579 net_stats->rx_bytes = bytes;
4580 net_stats->rx_packets = packets;
3f9c0164
AD		 4581
4582 bytes = 0;
4583 packets = 0;
4584 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 4585 struct igb_ring *ring = adapter->tx_ring[i];
12dcd86b
ED		 4586 do {
4587 start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4588 _bytes = ring->tx_stats.bytes;
4589 _packets = ring->tx_stats.packets;
4590 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4591 bytes += _bytes;
4592 packets += _packets;
3f9c0164 4593 }
128e45eb
AD		 4594 net_stats->tx_bytes = bytes;
4595 net_stats->tx_packets = packets;
3f9c0164
AD		 4596
4597 /* read stats registers */
9d5c8243
AK		 4598 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4599 adapter->stats.gprc += rd32(E1000_GPRC);
4600 adapter->stats.gorc += rd32(E1000_GORCL);
4601 rd32(E1000_GORCH); /* clear GORCL */
4602 adapter->stats.bprc += rd32(E1000_BPRC);
4603 adapter->stats.mprc += rd32(E1000_MPRC);
4604 adapter->stats.roc += rd32(E1000_ROC);
4605
4606 adapter->stats.prc64 += rd32(E1000_PRC64);
4607 adapter->stats.prc127 += rd32(E1000_PRC127);
4608 adapter->stats.prc255 += rd32(E1000_PRC255);
4609 adapter->stats.prc511 += rd32(E1000_PRC511);
4610 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4611 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4612 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4613 adapter->stats.sec += rd32(E1000_SEC);
4614
fa3d9a6d
MW		 4615 mpc = rd32(E1000_MPC);
4616 adapter->stats.mpc += mpc;
4617 net_stats->rx_fifo_errors += mpc;
9d5c8243
AK		 4618 adapter->stats.scc += rd32(E1000_SCC);
4619 adapter->stats.ecol += rd32(E1000_ECOL);
4620 adapter->stats.mcc += rd32(E1000_MCC);
4621 adapter->stats.latecol += rd32(E1000_LATECOL);
4622 adapter->stats.dc += rd32(E1000_DC);
4623 adapter->stats.rlec += rd32(E1000_RLEC);
4624 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4625 adapter->stats.xontxc += rd32(E1000_XONTXC);
4626 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4627 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4628 adapter->stats.fcruc += rd32(E1000_FCRUC);
4629 adapter->stats.gptc += rd32(E1000_GPTC);
4630 adapter->stats.gotc += rd32(E1000_GOTCL);
4631 rd32(E1000_GOTCH); /* clear GOTCL */
fa3d9a6d 4632 adapter->stats.rnbc += rd32(E1000_RNBC);
9d5c8243
AK		 4633 adapter->stats.ruc += rd32(E1000_RUC);
4634 adapter->stats.rfc += rd32(E1000_RFC);
4635 adapter->stats.rjc += rd32(E1000_RJC);
4636 adapter->stats.tor += rd32(E1000_TORH);
4637 adapter->stats.tot += rd32(E1000_TOTH);
4638 adapter->stats.tpr += rd32(E1000_TPR);
4639
4640 adapter->stats.ptc64 += rd32(E1000_PTC64);
4641 adapter->stats.ptc127 += rd32(E1000_PTC127);
4642 adapter->stats.ptc255 += rd32(E1000_PTC255);
4643 adapter->stats.ptc511 += rd32(E1000_PTC511);
4644 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4645 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4646
4647 adapter->stats.mptc += rd32(E1000_MPTC);
4648 adapter->stats.bptc += rd32(E1000_BPTC);
4649
2d0b0f69
NN		 4650 adapter->stats.tpt += rd32(E1000_TPT);
4651 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4652
4653 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4654 /* read internal phy specific stats */
4655 reg = rd32(E1000_CTRL_EXT);
4656 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4657 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4658 adapter->stats.tncrs += rd32(E1000_TNCRS);
4659 }
4660
9d5c8243
AK		 4661 adapter->stats.tsctc += rd32(E1000_TSCTC);
4662 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4663
4664 adapter->stats.iac += rd32(E1000_IAC);
4665 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4666 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4667 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4668 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4669 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4670 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4671 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4672 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4673
4674 /* Fill out the OS statistics structure */
128e45eb
AD		 4675 net_stats->multicast = adapter->stats.mprc;
4676 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4677
4678 /* Rx Errors */
4679
4680 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4681 * our own version based on RUC and ROC */
128e45eb 4682 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4683 adapter->stats.crcerrs + adapter->stats.algnerrc +
4684 adapter->stats.ruc + adapter->stats.roc +
4685 adapter->stats.cexterr;
128e45eb
AD		 4686 net_stats->rx_length_errors = adapter->stats.ruc +
4687 adapter->stats.roc;
4688 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4689 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4690 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4691
4692 /* Tx Errors */
128e45eb
AD		 4693 net_stats->tx_errors = adapter->stats.ecol +
4694 adapter->stats.latecol;
4695 net_stats->tx_aborted_errors = adapter->stats.ecol;
4696 net_stats->tx_window_errors = adapter->stats.latecol;
4697 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4698
4699 /* Tx Dropped needs to be maintained elsewhere */
4700
4701 /* Phy Stats */
4702 if (hw->phy.media_type == e1000_media_type_copper) {
4703 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4704 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4705 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4706 adapter->phy_stats.idle_errors += phy_tmp;
4707 }
4708 }
4709
4710 /* Management Stats */
4711 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4712 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4713 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
0a915b95
CW		 4714
4715 /* OS2BMC Stats */
4716 reg = rd32(E1000_MANC);
4717 if (reg & E1000_MANC_EN_BMC2OS) {
4718 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
4719 adapter->stats.o2bspc += rd32(E1000_O2BSPC);
4720 adapter->stats.b2ospc += rd32(E1000_B2OSPC);
4721 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
4722 }
9d5c8243
AK		 4723}
4724
9d5c8243
AK		 4725static irqreturn_t igb_msix_other(int irq, void *data)
4726{
047e0030 4727 struct igb_adapter *adapter = data;
9d5c8243 4728 struct e1000_hw *hw = &adapter->hw;
844290e5 4729 u32 icr = rd32(E1000_ICR);
844290e5 4730 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4731
7f081d40
AD		 4732 if (icr & E1000_ICR_DRSTA)
4733 schedule_work(&adapter->reset_task);
4734
047e0030 4735 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4736 /* HW is reporting DMA is out of sync */
4737 adapter->stats.doosync++;
13800469
GR		 4738 /* The DMA Out of Sync is also indication of a spoof event
4739 * in IOV mode. Check the Wrong VM Behavior register to
4740 * see if it is really a spoof event. */
4741 igb_check_wvbr(adapter);
dda0e083 4742 }
eebbbdba 4743
4ae196df
AD		 4744 /* Check for a mailbox event */
4745 if (icr & E1000_ICR_VMMB)
4746 igb_msg_task(adapter);
4747
4748 if (icr & E1000_ICR_LSC) {
4749 hw->mac.get_link_status = 1;
4750 /* guard against interrupt when we're going down */
4751 if (!test_bit(__IGB_DOWN, &adapter->state))
4752 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4753 }
4754
25568a53
AD		 4755 if (adapter->vfs_allocated_count)
4756 wr32(E1000_IMS, E1000_IMS_LSC |
4757 E1000_IMS_VMMB |
4758 E1000_IMS_DOUTSYNC);
4759 else
4760 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
844290e5 4761 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4762
4763 return IRQ_HANDLED;
4764}
4765
047e0030 4766static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4767{
26b39276 4768 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4769 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4770
047e0030
AD		 4771 if (!q_vector->set_itr)
4772 return;
73cd78f1 4773
047e0030
AD		 4774 if (!itr_val)
4775 itr_val = 0x4;
661086df 4776
26b39276
AD		 4777 if (adapter->hw.mac.type == e1000_82575)
4778 itr_val |= itr_val << 16;
661086df 4779 else
047e0030 4780 itr_val |= 0x8000000;
661086df 4781
047e0030
AD		 4782 writel(itr_val, q_vector->itr_register);
4783 q_vector->set_itr = 0;
6eb5a7f1
AD		 4784}
4785
047e0030 4786static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4787{
047e0030 4788 struct igb_q_vector *q_vector = data;
9d5c8243 4789
047e0030
AD		 4790 /* Write the ITR value calculated from the previous interrupt. */
4791 igb_write_itr(q_vector);
9d5c8243 4792
047e0030 4793 napi_schedule(&q_vector->napi);
844290e5 4794
047e0030 4795 return IRQ_HANDLED;
fe4506b6
JC		 4796}
4797
421e02f0 4798#ifdef CONFIG_IGB_DCA
047e0030 4799static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4800{
047e0030 4801 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4802 struct e1000_hw *hw = &adapter->hw;
4803 int cpu = get_cpu();
fe4506b6 4804
047e0030
AD		 4805 if (q_vector->cpu == cpu)
4806 goto out_no_update;
4807
4808 if (q_vector->tx_ring) {
4809 int q = q_vector->tx_ring->reg_idx;
4810 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4811 if (hw->mac.type == e1000_82575) {
4812 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4813 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4814 } else {
047e0030
AD		 4815 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4816 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4817 E1000_DCA_TXCTRL_CPUID_SHIFT;
4818 }
4819 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4820 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4821 }
4822 if (q_vector->rx_ring) {
4823 int q = q_vector->rx_ring->reg_idx;
4824 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4825 if (hw->mac.type == e1000_82575) {
2d064c06 4826 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4827 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4828 } else {
4829 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4830 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4831 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4832 }
fe4506b6
JC		 4833 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4834 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4835 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4836 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4837 }
047e0030
AD		 4838 q_vector->cpu = cpu;
4839out_no_update:
fe4506b6
JC		 4840 put_cpu();
4841}
4842
4843static void igb_setup_dca(struct igb_adapter *adapter)
4844{
7e0e99ef 4845 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4846 int i;
4847
7dfc16fa 4848 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4849 return;
4850
7e0e99ef
AD		 4851 /* Always use CB2 mode, difference is masked in the CB driver. */
4852 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4853
047e0030 4854 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4855 adapter->q_vector[i]->cpu = -1;
4856 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4857 }
4858}
4859
4860static int __igb_notify_dca(struct device *dev, void *data)
4861{
4862 struct net_device *netdev = dev_get_drvdata(dev);
4863 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4864 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4865 struct e1000_hw *hw = &adapter->hw;
4866 unsigned long event = *(unsigned long *)data;
4867
4868 switch (event) {
4869 case DCA_PROVIDER_ADD:
4870 /* if already enabled, don't do it again */
7dfc16fa 4871 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4872 break;
fe4506b6 4873 if (dca_add_requester(dev) == 0) {
bbd98fe4 4874 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4875 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4876 igb_setup_dca(adapter);
4877 break;
4878 }
4879 /* Fall Through since DCA is disabled. */
4880 case DCA_PROVIDER_REMOVE:
7dfc16fa 4881 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4882 /* without this a class_device is left
047e0030 4883 * hanging around in the sysfs model */
fe4506b6 4884 dca_remove_requester(dev);
090b1795 4885 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4886 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4887 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4888 }
4889 break;
4890 }
bbd98fe4 4891
fe4506b6 4892 return 0;
9d5c8243
AK		 4893}
4894
fe4506b6
JC		 4895static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4896 void *p)
4897{
4898 int ret_val;
4899
4900 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4901 __igb_notify_dca);
4902
4903 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4904}
421e02f0 4905#endif /* CONFIG_IGB_DCA */
9d5c8243 4906
4ae196df
AD		 4907static void igb_ping_all_vfs(struct igb_adapter *adapter)
4908{
4909 struct e1000_hw *hw = &adapter->hw;
4910 u32 ping;
4911 int i;
4912
4913 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4914 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 4915 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 4916 ping |= E1000_VT_MSGTYPE_CTS;
4917 igb_write_mbx(hw, &ping, 1, i);
4918 }
4919}
4920
7d5753f0
AD		 4921static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4922{
4923 struct e1000_hw *hw = &adapter->hw;
4924 u32 vmolr = rd32(E1000_VMOLR(vf));
4925 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4926
d85b9004 4927 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
7d5753f0
AD		 4928 IGB_VF_FLAG_MULTI_PROMISC);
4929 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4930
4931 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4932 vmolr |= E1000_VMOLR_MPME;
d85b9004 4933 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
7d5753f0
AD		 4934 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4935 } else {
4936 /*
4937 * if we have hashes and we are clearing a multicast promisc
4938 * flag we need to write the hashes to the MTA as this step
4939 * was previously skipped
4940 */
4941 if (vf_data->num_vf_mc_hashes > 30) {
4942 vmolr |= E1000_VMOLR_MPME;
4943 } else if (vf_data->num_vf_mc_hashes) {
4944 int j;
4945 vmolr |= E1000_VMOLR_ROMPE;
4946 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4947 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4948 }
4949 }
4950
4951 wr32(E1000_VMOLR(vf), vmolr);
4952
4953 /* there are flags left unprocessed, likely not supported */
4954 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4955 return -EINVAL;
4956
4957 return 0;
4958
4959}
4960
4ae196df
AD		 4961static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4962 u32 *msgbuf, u32 vf)
4963{
4964 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4965 u16 *hash_list = (u16 *)&msgbuf[1];
4966 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4967 int i;
4968
7d5753f0 4969 /* salt away the number of multicast addresses assigned
4ae196df
AD		 4970 * to this VF for later use to restore when the PF multi cast
4971 * list changes
4972 */
4973 vf_data->num_vf_mc_hashes = n;
4974
7d5753f0
AD		 4975 /* only up to 30 hash values supported */
4976 if (n > 30)
4977 n = 30;
4978
4979 /* store the hashes for later use */
4ae196df 4980 for (i = 0; i < n; i++)
a419aef8 4981 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 4982
4983 /* Flush and reset the mta with the new values */
ff41f8dc 4984 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4985
4986 return 0;
4987}
4988
4989static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4990{
4991 struct e1000_hw *hw = &adapter->hw;
4992 struct vf_data_storage *vf_data;
4993 int i, j;
4994
4995 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 4996 u32 vmolr = rd32(E1000_VMOLR(i));
4997 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4998
4ae196df 4999 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 5000
5001 if ((vf_data->num_vf_mc_hashes > 30) ||
5002 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
5003 vmolr |= E1000_VMOLR_MPME;
5004 } else if (vf_data->num_vf_mc_hashes) {
5005 vmolr |= E1000_VMOLR_ROMPE;
5006 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5007 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5008 }
5009 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 5010 }
5011}
5012
5013static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
5014{
5015 struct e1000_hw *hw = &adapter->hw;
5016 u32 pool_mask, reg, vid;
5017 int i;
5018
5019 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5020
5021 /* Find the vlan filter for this id */
5022 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5023 reg = rd32(E1000_VLVF(i));
5024
5025 /* remove the vf from the pool */
5026 reg &= ~pool_mask;
5027
5028 /* if pool is empty then remove entry from vfta */
5029 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
5030 (reg & E1000_VLVF_VLANID_ENABLE)) {
5031 reg = 0;
5032 vid = reg & E1000_VLVF_VLANID_MASK;
5033 igb_vfta_set(hw, vid, false);
5034 }
5035
5036 wr32(E1000_VLVF(i), reg);
5037 }
ae641bdc
AD		 5038
5039 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 5040}
5041
5042static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
5043{
5044 struct e1000_hw *hw = &adapter->hw;
5045 u32 reg, i;
5046
51466239
AD		 5047 /* The vlvf table only exists on 82576 hardware and newer */
5048 if (hw->mac.type < e1000_82576)
5049 return -1;
5050
5051 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 5052 if (!adapter->vfs_allocated_count)
5053 return -1;
5054
5055 /* Find the vlan filter for this id */
5056 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5057 reg = rd32(E1000_VLVF(i));
5058 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
5059 vid == (reg & E1000_VLVF_VLANID_MASK))
5060 break;
5061 }
5062
5063 if (add) {
5064 if (i == E1000_VLVF_ARRAY_SIZE) {
5065 /* Did not find a matching VLAN ID entry that was
5066 * enabled. Search for a free filter entry, i.e.
5067 * one without the enable bit set
5068 */
5069 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5070 reg = rd32(E1000_VLVF(i));
5071 if (!(reg & E1000_VLVF_VLANID_ENABLE))
5072 break;
5073 }
5074 }
5075 if (i < E1000_VLVF_ARRAY_SIZE) {
5076 /* Found an enabled/available entry */
5077 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5078
5079 /* if !enabled we need to set this up in vfta */
5080 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 5081 /* add VID to filter table */
5082 igb_vfta_set(hw, vid, true);
4ae196df
AD		 5083 reg |= E1000_VLVF_VLANID_ENABLE;
5084 }
cad6d05f
AD		 5085 reg &= ~E1000_VLVF_VLANID_MASK;
5086 reg |= vid;
4ae196df 5087 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5088
5089 /* do not modify RLPML for PF devices */
5090 if (vf >= adapter->vfs_allocated_count)
5091 return 0;
5092
5093 if (!adapter->vf_data[vf].vlans_enabled) {
5094 u32 size;
5095 reg = rd32(E1000_VMOLR(vf));
5096 size = reg & E1000_VMOLR_RLPML_MASK;
5097 size += 4;
5098 reg &= ~E1000_VMOLR_RLPML_MASK;
5099 reg |= size;
5100 wr32(E1000_VMOLR(vf), reg);
5101 }
ae641bdc 5102
51466239 5103 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 5104 return 0;
5105 }
5106 } else {
5107 if (i < E1000_VLVF_ARRAY_SIZE) {
5108 /* remove vf from the pool */
5109 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
5110 /* if pool is empty then remove entry from vfta */
5111 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
5112 reg = 0;
5113 igb_vfta_set(hw, vid, false);
5114 }
5115 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5116
5117 /* do not modify RLPML for PF devices */
5118 if (vf >= adapter->vfs_allocated_count)
5119 return 0;
5120
5121 adapter->vf_data[vf].vlans_enabled--;
5122 if (!adapter->vf_data[vf].vlans_enabled) {
5123 u32 size;
5124 reg = rd32(E1000_VMOLR(vf));
5125 size = reg & E1000_VMOLR_RLPML_MASK;
5126 size -= 4;
5127 reg &= ~E1000_VMOLR_RLPML_MASK;
5128 reg |= size;
5129 wr32(E1000_VMOLR(vf), reg);
5130 }
4ae196df
AD		 5131 }
5132 }
8151d294
WM		 5133 return 0;
5134}
5135
5136static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
5137{
5138 struct e1000_hw *hw = &adapter->hw;
5139
5140 if (vid)
5141 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
5142 else
5143 wr32(E1000_VMVIR(vf), 0);
5144}
5145
5146static int igb_ndo_set_vf_vlan(struct net_device *netdev,
5147 int vf, u16 vlan, u8 qos)
5148{
5149 int err = 0;
5150 struct igb_adapter *adapter = netdev_priv(netdev);
5151
5152 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
5153 return -EINVAL;
5154 if (vlan || qos) {
5155 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
5156 if (err)
5157 goto out;
5158 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
5159 igb_set_vmolr(adapter, vf, !vlan);
5160 adapter->vf_data[vf].pf_vlan = vlan;
5161 adapter->vf_data[vf].pf_qos = qos;
5162 dev_info(&adapter->pdev->dev,
5163 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5164 if (test_bit(__IGB_DOWN, &adapter->state)) {
5165 dev_warn(&adapter->pdev->dev,
5166 "The VF VLAN has been set,"
5167 " but the PF device is not up.\n");
5168 dev_warn(&adapter->pdev->dev,
5169 "Bring the PF device up before"
5170 " attempting to use the VF device.\n");
5171 }
5172 } else {
5173 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5174 false, vf);
5175 igb_set_vmvir(adapter, vlan, vf);
5176 igb_set_vmolr(adapter, vf, true);
5177 adapter->vf_data[vf].pf_vlan = 0;
5178 adapter->vf_data[vf].pf_qos = 0;
5179 }
5180out:
5181 return err;
4ae196df
AD		 5182}
5183
5184static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5185{
5186 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5187 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5188
5189 return igb_vlvf_set(adapter, vid, add, vf);
5190}
5191
f2ca0dbe 5192static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 5193{
8fa7e0f7
GR		 5194 /* clear flags - except flag that indicates PF has set the MAC */
5195 adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
f2ca0dbe 5196 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 5197
5198 /* reset offloads to defaults */
8151d294 5199 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 5200
5201 /* reset vlans for device */
5202 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 5203 if (adapter->vf_data[vf].pf_vlan)
5204 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5205 adapter->vf_data[vf].pf_vlan,
5206 adapter->vf_data[vf].pf_qos);
5207 else
5208 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 5209
5210 /* reset multicast table array for vf */
5211 adapter->vf_data[vf].num_vf_mc_hashes = 0;
5212
5213 /* Flush and reset the mta with the new values */
ff41f8dc 5214 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 5215}
5216
f2ca0dbe
AD		 5217static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5218{
5219 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5220
5221 /* generate a new mac address as we were hotplug removed/added */
8151d294
WM		 5222 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5223 random_ether_addr(vf_mac);
f2ca0dbe
AD		 5224
5225 /* process remaining reset events */
5226 igb_vf_reset(adapter, vf);
5227}
5228
5229static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 5230{
5231 struct e1000_hw *hw = &adapter->hw;
5232 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 5233 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 5234 u32 reg, msgbuf[3];
5235 u8 *addr = (u8 *)(&msgbuf[1]);
5236
5237 /* process all the same items cleared in a function level reset */
f2ca0dbe 5238 igb_vf_reset(adapter, vf);
4ae196df
AD		 5239
5240 /* set vf mac address */
26ad9178 5241 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 5242
5243 /* enable transmit and receive for vf */
5244 reg = rd32(E1000_VFTE);
5245 wr32(E1000_VFTE, reg | (1 << vf));
5246 reg = rd32(E1000_VFRE);
5247 wr32(E1000_VFRE, reg | (1 << vf));
5248
8fa7e0f7 5249 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
4ae196df
AD		 5250
5251 /* reply to reset with ack and vf mac address */
5252 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5253 memcpy(addr, vf_mac, 6);
5254 igb_write_mbx(hw, msgbuf, 3, vf);
5255}
5256
5257static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5258{
de42edde
GR		 5259 /*
5260 * The VF MAC Address is stored in a packed array of bytes
5261 * starting at the second 32 bit word of the msg array
5262 */
f2ca0dbe
AD		 5263 unsigned char *addr = (char *)&msg[1];
5264 int err = -1;
4ae196df 5265
f2ca0dbe
AD		 5266 if (is_valid_ether_addr(addr))
5267 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 5268
f2ca0dbe 5269 return err;
4ae196df
AD		 5270}
5271
5272static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5273{
5274 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5275 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5276 u32 msg = E1000_VT_MSGTYPE_NACK;
5277
5278 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 5279 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5280 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 5281 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 5282 vf_data->last_nack = jiffies;
4ae196df
AD		 5283 }
5284}
5285
f2ca0dbe 5286static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 5287{
f2ca0dbe
AD		 5288 struct pci_dev *pdev = adapter->pdev;
5289 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 5290 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5291 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5292 s32 retval;
5293
f2ca0dbe 5294 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 5295
fef45f4c
AD		 5296 if (retval) {
5297 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 5298 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 5299 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5300 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5301 return;
5302 goto out;
5303 }
4ae196df
AD		 5304
5305 /* this is a message we already processed, do nothing */
5306 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 5307 return;
4ae196df
AD		 5308
5309 /*
5310 * until the vf completes a reset it should not be
5311 * allowed to start any configuration.
5312 */
5313
5314 if (msgbuf[0] == E1000_VF_RESET) {
5315 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 5316 return;
4ae196df
AD		 5317 }
5318
f2ca0dbe 5319 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 5320 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5321 return;
5322 retval = -1;
5323 goto out;
4ae196df
AD		 5324 }
5325
5326 switch ((msgbuf[0] & 0xFFFF)) {
5327 case E1000_VF_SET_MAC_ADDR:
a6b5ea35
GR		 5328 retval = -EINVAL;
5329 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
5330 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5331 else
5332 dev_warn(&pdev->dev,
5333 "VF %d attempted to override administratively "
5334 "set MAC address\nReload the VF driver to "
5335 "resume operations\n", vf);
4ae196df 5336 break;
7d5753f0
AD		 5337 case E1000_VF_SET_PROMISC:
5338 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5339 break;
4ae196df
AD		 5340 case E1000_VF_SET_MULTICAST:
5341 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5342 break;
5343 case E1000_VF_SET_LPE:
5344 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5345 break;
5346 case E1000_VF_SET_VLAN:
a6b5ea35
GR		 5347 retval = -1;
5348 if (vf_data->pf_vlan)
5349 dev_warn(&pdev->dev,
5350 "VF %d attempted to override administratively "
5351 "set VLAN tag\nReload the VF driver to "
5352 "resume operations\n", vf);
8151d294
WM		 5353 else
5354 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 5355 break;
5356 default:
090b1795 5357 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 5358 retval = -1;
5359 break;
5360 }
5361
fef45f4c
AD		 5362 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5363out:
4ae196df
AD		 5364 /* notify the VF of the results of what it sent us */
5365 if (retval)
5366 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5367 else
5368 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5369
4ae196df 5370 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 5371}
4ae196df 5372
f2ca0dbe
AD		 5373static void igb_msg_task(struct igb_adapter *adapter)
5374{
5375 struct e1000_hw *hw = &adapter->hw;
5376 u32 vf;
5377
5378 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5379 /* process any reset requests */
5380 if (!igb_check_for_rst(hw, vf))
5381 igb_vf_reset_event(adapter, vf);
5382
5383 /* process any messages pending */
5384 if (!igb_check_for_msg(hw, vf))
5385 igb_rcv_msg_from_vf(adapter, vf);
5386
5387 /* process any acks */
5388 if (!igb_check_for_ack(hw, vf))
5389 igb_rcv_ack_from_vf(adapter, vf);
5390 }
4ae196df
AD		 5391}
5392
68d480c4
AD		 5393/**
5394 * igb_set_uta - Set unicast filter table address
5395 * @adapter: board private structure
5396 *
5397 * The unicast table address is a register array of 32-bit registers.
5398 * The table is meant to be used in a way similar to how the MTA is used
5399 * however due to certain limitations in the hardware it is necessary to
25985edc
LDM		 5400 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
5401 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
68d480c4
AD		 5402 **/
5403static void igb_set_uta(struct igb_adapter *adapter)
5404{
5405 struct e1000_hw *hw = &adapter->hw;
5406 int i;
5407
5408 /* The UTA table only exists on 82576 hardware and newer */
5409 if (hw->mac.type < e1000_82576)
5410 return;
5411
5412 /* we only need to do this if VMDq is enabled */
5413 if (!adapter->vfs_allocated_count)
5414 return;
5415
5416 for (i = 0; i < hw->mac.uta_reg_count; i++)
5417 array_wr32(E1000_UTA, i, ~0);
5418}
5419
9d5c8243
AK		 5420/**
5421 * igb_intr_msi - Interrupt Handler
5422 * @irq: interrupt number
5423 * @data: pointer to a network interface device structure
5424 **/
5425static irqreturn_t igb_intr_msi(int irq, void *data)
5426{
047e0030
AD		 5427 struct igb_adapter *adapter = data;
5428 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5429 struct e1000_hw *hw = &adapter->hw;
5430 /* read ICR disables interrupts using IAM */
5431 u32 icr = rd32(E1000_ICR);
5432
047e0030 5433 igb_write_itr(q_vector);
9d5c8243 5434
7f081d40
AD		 5435 if (icr & E1000_ICR_DRSTA)
5436 schedule_work(&adapter->reset_task);
5437
047e0030 5438 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5439 /* HW is reporting DMA is out of sync */
5440 adapter->stats.doosync++;
5441 }
5442
9d5c8243
AK		 5443 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5444 hw->mac.get_link_status = 1;
5445 if (!test_bit(__IGB_DOWN, &adapter->state))
5446 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5447 }
5448
047e0030 5449 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5450
5451 return IRQ_HANDLED;
5452}
5453
5454/**
4a3c6433 5455 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 5456 * @irq: interrupt number
5457 * @data: pointer to a network interface device structure
5458 **/
5459static irqreturn_t igb_intr(int irq, void *data)
5460{
047e0030
AD		 5461 struct igb_adapter *adapter = data;
5462 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5463 struct e1000_hw *hw = &adapter->hw;
5464 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5465 * need for the IMC write */
5466 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 5467 if (!icr)
5468 return IRQ_NONE; /* Not our interrupt */
5469
047e0030 5470 igb_write_itr(q_vector);
9d5c8243
AK		 5471
5472 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5473 * not set, then the adapter didn't send an interrupt */
5474 if (!(icr & E1000_ICR_INT_ASSERTED))
5475 return IRQ_NONE;
5476
7f081d40
AD		 5477 if (icr & E1000_ICR_DRSTA)
5478 schedule_work(&adapter->reset_task);
5479
047e0030 5480 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5481 /* HW is reporting DMA is out of sync */
5482 adapter->stats.doosync++;
5483 }
5484
9d5c8243
AK		 5485 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5486 hw->mac.get_link_status = 1;
5487 /* guard against interrupt when we're going down */
5488 if (!test_bit(__IGB_DOWN, &adapter->state))
5489 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5490 }
5491
047e0030 5492 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5493
5494 return IRQ_HANDLED;
5495}
5496
047e0030 5497static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 5498{
047e0030 5499 struct igb_adapter *adapter = q_vector->adapter;
46544258 5500 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5501
4fc82adf
AD		 5502 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
5503 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
047e0030 5504 if (!adapter->msix_entries)
6eb5a7f1 5505 igb_set_itr(adapter);
46544258 5506 else
047e0030 5507 igb_update_ring_itr(q_vector);
9d5c8243
AK		 5508 }
5509
46544258
AD		 5510 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5511 if (adapter->msix_entries)
047e0030 5512 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 5513 else
5514 igb_irq_enable(adapter);
5515 }
9d5c8243
AK		 5516}
5517
46544258
AD		 5518/**
5519 * igb_poll - NAPI Rx polling callback
5520 * @napi: napi polling structure
5521 * @budget: count of how many packets we should handle
5522 **/
5523static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 5524{
047e0030
AD		 5525 struct igb_q_vector *q_vector = container_of(napi,
5526 struct igb_q_vector,
5527 napi);
5528 int tx_clean_complete = 1, work_done = 0;
9d5c8243 5529
421e02f0 5530#ifdef CONFIG_IGB_DCA
047e0030
AD		 5531 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5532 igb_update_dca(q_vector);
fe4506b6 5533#endif
047e0030
AD		 5534 if (q_vector->tx_ring)
5535 tx_clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 5536
047e0030
AD		 5537 if (q_vector->rx_ring)
5538 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
5539
5540 if (!tx_clean_complete)
5541 work_done = budget;
46544258 5542
9d5c8243 5543 /* If not enough Rx work done, exit the polling mode */
5e6d5b17 5544 if (work_done < budget) {
288379f0 5545 napi_complete(napi);
047e0030 5546 igb_ring_irq_enable(q_vector);
9d5c8243
AK		 5547 }
5548
46544258 5549 return work_done;
9d5c8243 5550}
6d8126f9 5551
33af6bcc 5552/**
c5b9bd5e 5553 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 5554 * @adapter: board private structure
c5b9bd5e
AD		 5555 * @shhwtstamps: timestamp structure to update
5556 * @regval: unsigned 64bit system time value.
5557 *
5558 * We need to convert the system time value stored in the RX/TXSTMP registers
5559 * into a hwtstamp which can be used by the upper level timestamping functions
5560 */
5561static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5562 struct skb_shared_hwtstamps *shhwtstamps,
5563 u64 regval)
5564{
5565 u64 ns;
5566
55cac248
AD		 5567 /*
5568 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5569 * 24 to match clock shift we setup earlier.
5570 */
5571 if (adapter->hw.mac.type == e1000_82580)
5572 regval <<= IGB_82580_TSYNC_SHIFT;
5573
c5b9bd5e
AD		 5574 ns = timecounter_cyc2time(&adapter->clock, regval);
5575 timecompare_update(&adapter->compare, ns);
5576 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5577 shhwtstamps->hwtstamp = ns_to_ktime(ns);
5578 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5579}
5580
5581/**
5582 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5583 * @q_vector: pointer to q_vector containing needed info
2873957d 5584 * @buffer: pointer to igb_buffer structure
33af6bcc
PO		 5585 *
5586 * If we were asked to do hardware stamping and such a time stamp is
5587 * available, then it must have been for this skb here because we only
5588 * allow only one such packet into the queue.
5589 */
2873957d 5590static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct igb_buffer *buffer_info)
33af6bcc 5591{
c5b9bd5e 5592 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc 5593 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 5594 struct skb_shared_hwtstamps shhwtstamps;
5595 u64 regval;
33af6bcc 5596
c5b9bd5e 5597 /* if skb does not support hw timestamp or TX stamp not valid exit */
2244d07b 5598 if (likely(!(buffer_info->tx_flags & SKBTX_HW_TSTAMP)) ||
c5b9bd5e
AD		 5599 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5600 return;
5601
5602 regval = rd32(E1000_TXSTMPL);
5603 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5604
5605 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
2873957d 5606 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
33af6bcc
PO		 5607}
5608
9d5c8243
AK		 5609/**
5610 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 5611 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 5612 * returns true if ring is completely cleaned
5613 **/
047e0030 5614static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 5615{
047e0030
AD		 5616 struct igb_adapter *adapter = q_vector->adapter;
5617 struct igb_ring *tx_ring = q_vector->tx_ring;
e694e964 5618 struct net_device *netdev = tx_ring->netdev;
0e014cb1 5619 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5620 struct igb_buffer *buffer_info;
0e014cb1 5621 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 5622 unsigned int total_bytes = 0, total_packets = 0;
0e014cb1
AD		 5623 unsigned int i, eop, count = 0;
5624 bool cleaned = false;
9d5c8243 5625
9d5c8243 5626 i = tx_ring->next_to_clean;
0e014cb1
AD		 5627 eop = tx_ring->buffer_info[i].next_to_watch;
5628 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5629
5630 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5631 (count < tx_ring->count)) {
2d0bb1c1 5632 rmb(); /* read buffer_info after eop_desc status */
0e014cb1
AD		 5633 for (cleaned = false; !cleaned; count++) {
5634 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
9d5c8243 5635 buffer_info = &tx_ring->buffer_info[i];
0e014cb1 5636 cleaned = (i == eop);
9d5c8243 5637
2873957d
NN		 5638 if (buffer_info->skb) {
5639 total_bytes += buffer_info->bytecount;
9d5c8243 5640 /* gso_segs is currently only valid for tcp */
2873957d
NN		 5641 total_packets += buffer_info->gso_segs;
5642 igb_tx_hwtstamp(q_vector, buffer_info);
9d5c8243
AK		 5643 }
5644
80785298 5645 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
0e014cb1 5646 tx_desc->wb.status = 0;
9d5c8243
AK		 5647
5648 i++;
5649 if (i == tx_ring->count)
5650 i = 0;
9d5c8243 5651 }
0e014cb1
AD		 5652 eop = tx_ring->buffer_info[i].next_to_watch;
5653 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5654 }
5655
9d5c8243
AK		 5656 tx_ring->next_to_clean = i;
5657
fc7d345d 5658 if (unlikely(count &&
9d5c8243 5659 netif_carrier_ok(netdev) &&
c493ea45 5660 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
9d5c8243
AK		 5661 /* Make sure that anybody stopping the queue after this
5662 * sees the new next_to_clean.
5663 */
5664 smp_mb();
661086df
PWJ		 5665 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5666 !(test_bit(__IGB_DOWN, &adapter->state))) {
5667 netif_wake_subqueue(netdev, tx_ring->queue_index);
12dcd86b
ED		 5668
5669 u64_stats_update_begin(&tx_ring->tx_syncp);
04a5fcaa 5670 tx_ring->tx_stats.restart_queue++;
12dcd86b 5671 u64_stats_update_end(&tx_ring->tx_syncp);
661086df 5672 }
9d5c8243
AK		 5673 }
5674
5675 if (tx_ring->detect_tx_hung) {
5676 /* Detect a transmit hang in hardware, this serializes the
5677 * check with the clearing of time_stamp and movement of i */
5678 tx_ring->detect_tx_hung = false;
5679 if (tx_ring->buffer_info[i].time_stamp &&
5680 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
8e95a202
JP		 5681 (adapter->tx_timeout_factor * HZ)) &&
5682 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5683
9d5c8243 5684 /* detected Tx unit hang */
59d71989 5685 dev_err(tx_ring->dev,
9d5c8243 5686 "Detected Tx Unit Hang\n"
2d064c06 5687 " Tx Queue <%d>\n"
9d5c8243
AK		 5688 " TDH <%x>\n"
5689 " TDT <%x>\n"
5690 " next_to_use <%x>\n"
5691 " next_to_clean <%x>\n"
9d5c8243
AK		 5692 "buffer_info[next_to_clean]\n"
5693 " time_stamp <%lx>\n"
0e014cb1 5694 " next_to_watch <%x>\n"
9d5c8243
AK		 5695 " jiffies <%lx>\n"
5696 " desc.status <%x>\n",
2d064c06 5697 tx_ring->queue_index,
fce99e34
AD		 5698 readl(tx_ring->head),
5699 readl(tx_ring->tail),
9d5c8243
AK		 5700 tx_ring->next_to_use,
5701 tx_ring->next_to_clean,
f7ba205e 5702 tx_ring->buffer_info[eop].time_stamp,
0e014cb1 5703 eop,
9d5c8243 5704 jiffies,
0e014cb1 5705 eop_desc->wb.status);
661086df 5706 netif_stop_subqueue(netdev, tx_ring->queue_index);
9d5c8243
AK		 5707 }
5708 }
5709 tx_ring->total_bytes += total_bytes;
5710 tx_ring->total_packets += total_packets;
12dcd86b 5711 u64_stats_update_begin(&tx_ring->tx_syncp);
e21ed353
AD		 5712 tx_ring->tx_stats.bytes += total_bytes;
5713 tx_ring->tx_stats.packets += total_packets;
12dcd86b 5714 u64_stats_update_end(&tx_ring->tx_syncp);
807540ba 5715 return count < tx_ring->count;
9d5c8243
AK		 5716}
5717
04a5fcaa 5718static inline void igb_rx_checksum_adv(struct igb_ring *ring,
9d5c8243
AK		 5719 u32 status_err, struct sk_buff *skb)
5720{
bc8acf2c 5721 skb_checksum_none_assert(skb);
9d5c8243
AK		 5722
5723 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 5724 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5725 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 5726 return;
85ad76b2 5727
9d5c8243
AK		 5728 /* TCP/UDP checksum error bit is set */
5729 if (status_err &
5730 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5731 /*
5732 * work around errata with sctp packets where the TCPE aka
5733 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5734 * packets, (aka let the stack check the crc32c)
5735 */
85ad76b2 5736 if ((skb->len == 60) &&
12dcd86b
ED		 5737 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM)) {
5738 u64_stats_update_begin(&ring->rx_syncp);
04a5fcaa 5739 ring->rx_stats.csum_err++;
12dcd86b
ED		 5740 u64_stats_update_end(&ring->rx_syncp);
5741 }
9d5c8243 5742 /* let the stack verify checksum errors */
9d5c8243
AK		 5743 return;
5744 }
5745 /* It must be a TCP or UDP packet with a valid checksum */
5746 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5747 skb->ip_summed = CHECKSUM_UNNECESSARY;
5748
59d71989 5749 dev_dbg(ring->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 5750}
5751
757b77e2 5752static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
c5b9bd5e
AD		 5753 struct sk_buff *skb)
5754{
5755 struct igb_adapter *adapter = q_vector->adapter;
5756 struct e1000_hw *hw = &adapter->hw;
5757 u64 regval;
5758
5759 /*
5760 * If this bit is set, then the RX registers contain the time stamp. No
5761 * other packet will be time stamped until we read these registers, so
5762 * read the registers to make them available again. Because only one
5763 * packet can be time stamped at a time, we know that the register
5764 * values must belong to this one here and therefore we don't need to
5765 * compare any of the additional attributes stored for it.
5766 *
2244d07b 5767 * If nothing went wrong, then it should have a shared tx_flags that we
c5b9bd5e
AD		 5768 * can turn into a skb_shared_hwtstamps.
5769 */
757b77e2
NN		 5770 if (staterr & E1000_RXDADV_STAT_TSIP) {
5771 u32 *stamp = (u32 *)skb->data;
5772 regval = le32_to_cpu(*(stamp + 2));
5773 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5774 skb_pull(skb, IGB_TS_HDR_LEN);
5775 } else {
5776 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5777 return;
c5b9bd5e 5778
757b77e2
NN		 5779 regval = rd32(E1000_RXSTMPL);
5780 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5781 }
c5b9bd5e
AD		 5782
5783 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5784}
4c844851 5785static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
2d94d8ab
AD		 5786 union e1000_adv_rx_desc *rx_desc)
5787{
5788 /* HW will not DMA in data larger than the given buffer, even if it
5789 * parses the (NFS, of course) header to be larger. In that case, it
5790 * fills the header buffer and spills the rest into the page.
5791 */
5792 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5793 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4c844851
AD		 5794 if (hlen > rx_ring->rx_buffer_len)
5795 hlen = rx_ring->rx_buffer_len;
2d94d8ab
AD		 5796 return hlen;
5797}
5798
047e0030
AD		 5799static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5800 int *work_done, int budget)
9d5c8243 5801{
047e0030 5802 struct igb_ring *rx_ring = q_vector->rx_ring;
e694e964 5803 struct net_device *netdev = rx_ring->netdev;
59d71989 5804 struct device *dev = rx_ring->dev;
9d5c8243
AK		 5805 union e1000_adv_rx_desc *rx_desc , *next_rxd;
5806 struct igb_buffer *buffer_info , *next_buffer;
5807 struct sk_buff *skb;
9d5c8243
AK		 5808 bool cleaned = false;
5809 int cleaned_count = 0;
d1eff350 5810 int current_node = numa_node_id();
9d5c8243 5811 unsigned int total_bytes = 0, total_packets = 0;
73cd78f1 5812 unsigned int i;
2d94d8ab
AD		 5813 u32 staterr;
5814 u16 length;
9d5c8243
AK		 5815
5816 i = rx_ring->next_to_clean;
69d3ca53 5817 buffer_info = &rx_ring->buffer_info[i];
9d5c8243
AK		 5818 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5819 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5820
5821 while (staterr & E1000_RXD_STAT_DD) {
5822 if (*work_done >= budget)
5823 break;
5824 (*work_done)++;
2d0bb1c1 5825 rmb(); /* read descriptor and rx_buffer_info after status DD */
9d5c8243 5826
69d3ca53
AD		 5827 skb = buffer_info->skb;
5828 prefetch(skb->data - NET_IP_ALIGN);
5829 buffer_info->skb = NULL;
5830
5831 i++;
5832 if (i == rx_ring->count)
5833 i = 0;
42d0781a 5834
69d3ca53
AD		 5835 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5836 prefetch(next_rxd);
5837 next_buffer = &rx_ring->buffer_info[i];
9d5c8243
AK		 5838
5839 length = le16_to_cpu(rx_desc->wb.upper.length);
5840 cleaned = true;
5841 cleaned_count++;
5842
2d94d8ab 5843 if (buffer_info->dma) {
59d71989 5844 dma_unmap_single(dev, buffer_info->dma,
4c844851 5845 rx_ring->rx_buffer_len,
59d71989 5846 DMA_FROM_DEVICE);
91615f76 5847 buffer_info->dma = 0;
4c844851 5848 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
6ec43fe6
AD		 5849 skb_put(skb, length);
5850 goto send_up;
5851 }
4c844851 5852 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
bf36c1a0
AD		 5853 }
5854
5855 if (length) {
59d71989
AD		 5856 dma_unmap_page(dev, buffer_info->page_dma,
5857 PAGE_SIZE / 2, DMA_FROM_DEVICE);
9d5c8243 5858 buffer_info->page_dma = 0;
bf36c1a0 5859
aa913403 5860 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
bf36c1a0
AD		 5861 buffer_info->page,
5862 buffer_info->page_offset,
5863 length);
5864
d1eff350
AD		 5865 if ((page_count(buffer_info->page) != 1) ||
5866 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 5867 buffer_info->page = NULL;
5868 else
5869 get_page(buffer_info->page);
9d5c8243
AK		 5870
5871 skb->len += length;
5872 skb->data_len += length;
bf36c1a0 5873 skb->truesize += length;
9d5c8243 5874 }
9d5c8243 5875
bf36c1a0 5876 if (!(staterr & E1000_RXD_STAT_EOP)) {
b2d56536
AD		 5877 buffer_info->skb = next_buffer->skb;
5878 buffer_info->dma = next_buffer->dma;
5879 next_buffer->skb = skb;
5880 next_buffer->dma = 0;
bf36c1a0
AD		 5881 goto next_desc;
5882 }
69d3ca53 5883send_up:
9d5c8243
AK		 5884 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5885 dev_kfree_skb_irq(skb);
5886 goto next_desc;
5887 }
9d5c8243 5888
757b77e2
NN		 5889 if (staterr & (E1000_RXDADV_STAT_TSIP | E1000_RXDADV_STAT_TS))
5890 igb_rx_hwtstamp(q_vector, staterr, skb);
9d5c8243
AK		 5891 total_bytes += skb->len;
5892 total_packets++;
5893
85ad76b2 5894 igb_rx_checksum_adv(rx_ring, staterr, skb);
9d5c8243
AK		 5895
5896 skb->protocol = eth_type_trans(skb, netdev);
047e0030
AD		 5897 skb_record_rx_queue(skb, rx_ring->queue_index);
5898
b2cb09b1
JP		 5899 if (staterr & E1000_RXD_STAT_VP) {
5900 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
9d5c8243 5901
b2cb09b1
JP		 5902 __vlan_hwaccel_put_tag(skb, vid);
5903 }
5904 napi_gro_receive(&q_vector->napi, skb);
9d5c8243 5905
9d5c8243
AK		 5906next_desc:
5907 rx_desc->wb.upper.status_error = 0;
5908
5909 /* return some buffers to hardware, one at a time is too slow */
5910 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
3b644cf6 5911 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5912 cleaned_count = 0;
5913 }
5914
5915 /* use prefetched values */
5916 rx_desc = next_rxd;
5917 buffer_info = next_buffer;
9d5c8243
AK		 5918 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5919 }
bf36c1a0 5920
9d5c8243 5921 rx_ring->next_to_clean = i;
c493ea45 5922 cleaned_count = igb_desc_unused(rx_ring);
9d5c8243
AK		 5923
5924 if (cleaned_count)
3b644cf6 5925 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5926
5927 rx_ring->total_packets += total_packets;
5928 rx_ring->total_bytes += total_bytes;
12dcd86b 5929 u64_stats_update_begin(&rx_ring->rx_syncp);
9d5c8243
AK		 5930 rx_ring->rx_stats.packets += total_packets;
5931 rx_ring->rx_stats.bytes += total_bytes;
12dcd86b 5932 u64_stats_update_end(&rx_ring->rx_syncp);
9d5c8243
AK		 5933 return cleaned;
5934}
5935
9d5c8243
AK		 5936/**
5937 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5938 * @adapter: address of board private structure
5939 **/
d7ee5b3a 5940void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
9d5c8243 5941{
e694e964 5942 struct net_device *netdev = rx_ring->netdev;
9d5c8243
AK		 5943 union e1000_adv_rx_desc *rx_desc;
5944 struct igb_buffer *buffer_info;
5945 struct sk_buff *skb;
5946 unsigned int i;
db761762 5947 int bufsz;
9d5c8243
AK		 5948
5949 i = rx_ring->next_to_use;
5950 buffer_info = &rx_ring->buffer_info[i];
5951
4c844851 5952 bufsz = rx_ring->rx_buffer_len;
db761762 5953
9d5c8243
AK		 5954 while (cleaned_count--) {
5955 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5956
6ec43fe6 5957 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
9d5c8243 5958 if (!buffer_info->page) {
42d0781a 5959 buffer_info->page = netdev_alloc_page(netdev);
12dcd86b
ED		 5960 if (unlikely(!buffer_info->page)) {
5961 u64_stats_update_begin(&rx_ring->rx_syncp);
04a5fcaa 5962 rx_ring->rx_stats.alloc_failed++;
12dcd86b 5963 u64_stats_update_end(&rx_ring->rx_syncp);
bf36c1a0
AD		 5964 goto no_buffers;
5965 }
5966 buffer_info->page_offset = 0;
5967 } else {
5968 buffer_info->page_offset ^= PAGE_SIZE / 2;
9d5c8243
AK		 5969 }
5970 buffer_info->page_dma =
59d71989 5971 dma_map_page(rx_ring->dev, buffer_info->page,
bf36c1a0
AD		 5972 buffer_info->page_offset,
5973 PAGE_SIZE / 2,
59d71989
AD		 5974 DMA_FROM_DEVICE);
5975 if (dma_mapping_error(rx_ring->dev,
5976 buffer_info->page_dma)) {
42d0781a 5977 buffer_info->page_dma = 0;
12dcd86b 5978 u64_stats_update_begin(&rx_ring->rx_syncp);
42d0781a 5979 rx_ring->rx_stats.alloc_failed++;
12dcd86b 5980 u64_stats_update_end(&rx_ring->rx_syncp);
42d0781a
AD		 5981 goto no_buffers;
5982 }
9d5c8243
AK		 5983 }
5984
42d0781a
AD		 5985 skb = buffer_info->skb;
5986 if (!skb) {
89d71a66 5987 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
12dcd86b
ED		 5988 if (unlikely(!skb)) {
5989 u64_stats_update_begin(&rx_ring->rx_syncp);
04a5fcaa 5990 rx_ring->rx_stats.alloc_failed++;
12dcd86b 5991 u64_stats_update_end(&rx_ring->rx_syncp);
9d5c8243
AK		 5992 goto no_buffers;
5993 }
5994
9d5c8243 5995 buffer_info->skb = skb;
42d0781a
AD		 5996 }
5997 if (!buffer_info->dma) {
59d71989 5998 buffer_info->dma = dma_map_single(rx_ring->dev,
80785298 5999 skb->data,
9d5c8243 6000 bufsz,
59d71989
AD		 6001 DMA_FROM_DEVICE);
6002 if (dma_mapping_error(rx_ring->dev,
6003 buffer_info->dma)) {
42d0781a 6004 buffer_info->dma = 0;
12dcd86b 6005 u64_stats_update_begin(&rx_ring->rx_syncp);
42d0781a 6006 rx_ring->rx_stats.alloc_failed++;
12dcd86b 6007 u64_stats_update_end(&rx_ring->rx_syncp);
42d0781a
AD		 6008 goto no_buffers;
6009 }
9d5c8243
AK		 6010 }
6011 /* Refresh the desc even if buffer_addrs didn't change because
6012 * each write-back erases this info. */
6ec43fe6 6013 if (bufsz < IGB_RXBUFFER_1024) {
9d5c8243
AK		 6014 rx_desc->read.pkt_addr =
6015 cpu_to_le64(buffer_info->page_dma);
6016 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
6017 } else {
42d0781a 6018 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
9d5c8243
AK		 6019 rx_desc->read.hdr_addr = 0;
6020 }
6021
6022 i++;
6023 if (i == rx_ring->count)
6024 i = 0;
6025 buffer_info = &rx_ring->buffer_info[i];
6026 }
6027
6028no_buffers:
6029 if (rx_ring->next_to_use != i) {
6030 rx_ring->next_to_use = i;
6031 if (i == 0)
6032 i = (rx_ring->count - 1);
6033 else
6034 i--;
6035
6036 /* Force memory writes to complete before letting h/w
6037 * know there are new descriptors to fetch. (Only
6038 * applicable for weak-ordered memory model archs,
6039 * such as IA-64). */
6040 wmb();
fce99e34 6041 writel(i, rx_ring->tail);
9d5c8243
AK		 6042 }
6043}
6044
6045/**
6046 * igb_mii_ioctl -
6047 * @netdev:
6048 * @ifreq:
6049 * @cmd:
6050 **/
6051static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6052{
6053 struct igb_adapter *adapter = netdev_priv(netdev);
6054 struct mii_ioctl_data *data = if_mii(ifr);
6055
6056 if (adapter->hw.phy.media_type != e1000_media_type_copper)
6057 return -EOPNOTSUPP;
6058
6059 switch (cmd) {
6060 case SIOCGMIIPHY:
6061 data->phy_id = adapter->hw.phy.addr;
6062 break;
6063 case SIOCGMIIREG:
f5f4cf08
AD		 6064 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
6065 &data->val_out))
9d5c8243
AK		 6066 return -EIO;
6067 break;
6068 case SIOCSMIIREG:
6069 default:
6070 return -EOPNOTSUPP;
6071 }
6072 return 0;
6073}
6074
c6cb090b
PO		 6075/**
6076 * igb_hwtstamp_ioctl - control hardware time stamping
6077 * @netdev:
6078 * @ifreq:
6079 * @cmd:
6080 *
33af6bcc
PO		 6081 * Outgoing time stamping can be enabled and disabled. Play nice and
6082 * disable it when requested, although it shouldn't case any overhead
6083 * when no packet needs it. At most one packet in the queue may be
6084 * marked for time stamping, otherwise it would be impossible to tell
6085 * for sure to which packet the hardware time stamp belongs.
6086 *
6087 * Incoming time stamping has to be configured via the hardware
6088 * filters. Not all combinations are supported, in particular event
6089 * type has to be specified. Matching the kind of event packet is
6090 * not supported, with the exception of "all V2 events regardless of
6091 * level 2 or 4".
6092 *
c6cb090b
PO		 6093 **/
6094static int igb_hwtstamp_ioctl(struct net_device *netdev,
6095 struct ifreq *ifr, int cmd)
6096{
33af6bcc
PO		 6097 struct igb_adapter *adapter = netdev_priv(netdev);
6098 struct e1000_hw *hw = &adapter->hw;
c6cb090b 6099 struct hwtstamp_config config;
c5b9bd5e
AD		 6100 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
6101 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 6102 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 6103 bool is_l4 = false;
6104 bool is_l2 = false;
33af6bcc 6105 u32 regval;
c6cb090b
PO		 6106
6107 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
6108 return -EFAULT;
6109
6110 /* reserved for future extensions */
6111 if (config.flags)
6112 return -EINVAL;
6113
33af6bcc
PO		 6114 switch (config.tx_type) {
6115 case HWTSTAMP_TX_OFF:
c5b9bd5e 6116 tsync_tx_ctl = 0;
33af6bcc 6117 case HWTSTAMP_TX_ON:
33af6bcc
PO		 6118 break;
6119 default:
6120 return -ERANGE;
6121 }
6122
6123 switch (config.rx_filter) {
6124 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 6125 tsync_rx_ctl = 0;
33af6bcc
PO		 6126 break;
6127 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
6128 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
6129 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
6130 case HWTSTAMP_FILTER_ALL:
6131 /*
6132 * register TSYNCRXCFG must be set, therefore it is not
6133 * possible to time stamp both Sync and Delay_Req messages
6134 * => fall back to time stamping all packets
6135 */
c5b9bd5e 6136 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 6137 config.rx_filter = HWTSTAMP_FILTER_ALL;
6138 break;
6139 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 6140 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 6141 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 6142 is_l4 = true;
33af6bcc
PO		 6143 break;
6144 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 6145 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 6146 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 6147 is_l4 = true;
33af6bcc
PO		 6148 break;
6149 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
6150 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 6151 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 6152 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 6153 is_l2 = true;
6154 is_l4 = true;
33af6bcc
PO		 6155 config.rx_filter = HWTSTAMP_FILTER_SOME;
6156 break;
6157 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
6158 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 6159 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 6160 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 6161 is_l2 = true;
6162 is_l4 = true;
33af6bcc
PO		 6163 config.rx_filter = HWTSTAMP_FILTER_SOME;
6164 break;
6165 case HWTSTAMP_FILTER_PTP_V2_EVENT:
6166 case HWTSTAMP_FILTER_PTP_V2_SYNC:
6167 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 6168 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 6169 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 6170 is_l2 = true;
33af6bcc
PO		 6171 break;
6172 default:
6173 return -ERANGE;
6174 }
6175
c5b9bd5e
AD		 6176 if (hw->mac.type == e1000_82575) {
6177 if (tsync_rx_ctl | tsync_tx_ctl)
6178 return -EINVAL;
6179 return 0;
6180 }
6181
757b77e2
NN		 6182 /*
6183 * Per-packet timestamping only works if all packets are
6184 * timestamped, so enable timestamping in all packets as
6185 * long as one rx filter was configured.
6186 */
6187 if ((hw->mac.type == e1000_82580) && tsync_rx_ctl) {
6188 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
6189 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
6190 }
6191
33af6bcc
PO		 6192 /* enable/disable TX */
6193 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 6194 regval &= ~E1000_TSYNCTXCTL_ENABLED;
6195 regval |= tsync_tx_ctl;
33af6bcc
PO		 6196 wr32(E1000_TSYNCTXCTL, regval);
6197
c5b9bd5e 6198 /* enable/disable RX */
33af6bcc 6199 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 6200 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
6201 regval |= tsync_rx_ctl;
33af6bcc 6202 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 6203
c5b9bd5e
AD		 6204 /* define which PTP packets are time stamped */
6205 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 6206
c5b9bd5e
AD		 6207 /* define ethertype filter for timestamped packets */
6208 if (is_l2)
6209 wr32(E1000_ETQF(3),
6210 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
6211 E1000_ETQF_1588 | /* enable timestamping */
6212 ETH_P_1588)); /* 1588 eth protocol type */
6213 else
6214 wr32(E1000_ETQF(3), 0);
6215
6216#define PTP_PORT 319
6217 /* L4 Queue Filter[3]: filter by destination port and protocol */
6218 if (is_l4) {
6219 u32 ftqf = (IPPROTO_UDP /* UDP */
6220 | E1000_FTQF_VF_BP /* VF not compared */
6221 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
6222 | E1000_FTQF_MASK); /* mask all inputs */
6223 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
6224
6225 wr32(E1000_IMIR(3), htons(PTP_PORT));
6226 wr32(E1000_IMIREXT(3),
6227 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
6228 if (hw->mac.type == e1000_82576) {
6229 /* enable source port check */
6230 wr32(E1000_SPQF(3), htons(PTP_PORT));
6231 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
6232 }
6233 wr32(E1000_FTQF(3), ftqf);
6234 } else {
6235 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
6236 }
33af6bcc
PO		 6237 wrfl();
6238
6239 adapter->hwtstamp_config = config;
6240
6241 /* clear TX/RX time stamp registers, just to be sure */
6242 regval = rd32(E1000_TXSTMPH);
6243 regval = rd32(E1000_RXSTMPH);
c6cb090b 6244
33af6bcc
PO		 6245 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
6246 -EFAULT : 0;
c6cb090b
PO		 6247}
6248
9d5c8243
AK		 6249/**
6250 * igb_ioctl -
6251 * @netdev:
6252 * @ifreq:
6253 * @cmd:
6254 **/
6255static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6256{
6257 switch (cmd) {
6258 case SIOCGMIIPHY:
6259 case SIOCGMIIREG:
6260 case SIOCSMIIREG:
6261 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 6262 case SIOCSHWTSTAMP:
6263 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 6264 default:
6265 return -EOPNOTSUPP;
6266 }
6267}
6268
009bc06e
AD		 6269s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6270{
6271 struct igb_adapter *adapter = hw->back;
6272 u16 cap_offset;
6273
bdaae04c 6274 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6275 if (!cap_offset)
6276 return -E1000_ERR_CONFIG;
6277
6278 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6279
6280 return 0;
6281}
6282
6283s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6284{
6285 struct igb_adapter *adapter = hw->back;
6286 u16 cap_offset;
6287
bdaae04c 6288 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6289 if (!cap_offset)
6290 return -E1000_ERR_CONFIG;
6291
6292 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6293
6294 return 0;
6295}
6296
b2cb09b1 6297static void igb_vlan_mode(struct net_device *netdev, u32 features)
9d5c8243
AK		 6298{
6299 struct igb_adapter *adapter = netdev_priv(netdev);
6300 struct e1000_hw *hw = &adapter->hw;
6301 u32 ctrl, rctl;
6302
6303 igb_irq_disable(adapter);
9d5c8243 6304
b2cb09b1 6305 if (features & NETIF_F_HW_VLAN_RX) {
9d5c8243
AK		 6306 /* enable VLAN tag insert/strip */
6307 ctrl = rd32(E1000_CTRL);
6308 ctrl |= E1000_CTRL_VME;
6309 wr32(E1000_CTRL, ctrl);
6310
51466239 6311 /* Disable CFI check */
9d5c8243 6312 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 6313 rctl &= ~E1000_RCTL_CFIEN;
6314 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 6315 } else {
6316 /* disable VLAN tag insert/strip */
6317 ctrl = rd32(E1000_CTRL);
6318 ctrl &= ~E1000_CTRL_VME;
6319 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 6320 }
6321
e1739522
AD		 6322 igb_rlpml_set(adapter);
6323
9d5c8243
AK		 6324 if (!test_bit(__IGB_DOWN, &adapter->state))
6325 igb_irq_enable(adapter);
6326}
6327
6328static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6329{
6330 struct igb_adapter *adapter = netdev_priv(netdev);
6331 struct e1000_hw *hw = &adapter->hw;
4ae196df 6332 int pf_id = adapter->vfs_allocated_count;
9d5c8243 6333
51466239
AD		 6334 /* attempt to add filter to vlvf array */
6335 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 6336
51466239
AD		 6337 /* add the filter since PF can receive vlans w/o entry in vlvf */
6338 igb_vfta_set(hw, vid, true);
b2cb09b1
JP		 6339
6340 set_bit(vid, adapter->active_vlans);
9d5c8243
AK		 6341}
6342
6343static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6344{
6345 struct igb_adapter *adapter = netdev_priv(netdev);
6346 struct e1000_hw *hw = &adapter->hw;
4ae196df 6347 int pf_id = adapter->vfs_allocated_count;
51466239 6348 s32 err;
9d5c8243
AK		 6349
6350 igb_irq_disable(adapter);
9d5c8243
AK		 6351
6352 if (!test_bit(__IGB_DOWN, &adapter->state))
6353 igb_irq_enable(adapter);
6354
51466239
AD		 6355 /* remove vlan from VLVF table array */
6356 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 6357
51466239
AD		 6358 /* if vid was not present in VLVF just remove it from table */
6359 if (err)
4ae196df 6360 igb_vfta_set(hw, vid, false);
b2cb09b1
JP		 6361
6362 clear_bit(vid, adapter->active_vlans);
9d5c8243
AK		 6363}
6364
6365static void igb_restore_vlan(struct igb_adapter *adapter)
6366{
b2cb09b1 6367 u16 vid;
9d5c8243 6368
b2cb09b1
JP		 6369 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
6370 igb_vlan_rx_add_vid(adapter->netdev, vid);
9d5c8243
AK		 6371}
6372
14ad2513 6373int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
9d5c8243 6374{
090b1795 6375 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 6376 struct e1000_mac_info *mac = &adapter->hw.mac;
6377
6378 mac->autoneg = 0;
6379
14ad2513
DD		 6380 /* Make sure dplx is at most 1 bit and lsb of speed is not set
6381 * for the switch() below to work */
6382 if ((spd & 1) || (dplx & ~1))
6383 goto err_inval;
6384
cd2638a8
CW		 6385 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6386 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
14ad2513
DD		 6387 spd != SPEED_1000 &&
6388 dplx != DUPLEX_FULL)
6389 goto err_inval;
cd2638a8 6390
14ad2513 6391 switch (spd + dplx) {
9d5c8243
AK		 6392 case SPEED_10 + DUPLEX_HALF:
6393 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6394 break;
6395 case SPEED_10 + DUPLEX_FULL:
6396 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6397 break;
6398 case SPEED_100 + DUPLEX_HALF:
6399 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6400 break;
6401 case SPEED_100 + DUPLEX_FULL:
6402 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6403 break;
6404 case SPEED_1000 + DUPLEX_FULL:
6405 mac->autoneg = 1;
6406 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6407 break;
6408 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6409 default:
14ad2513 6410 goto err_inval;
9d5c8243
AK		 6411 }
6412 return 0;
14ad2513
DD		 6413
6414err_inval:
6415 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6416 return -EINVAL;
9d5c8243
AK		 6417}
6418
3fe7c4c9 6419static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 6420{
6421 struct net_device *netdev = pci_get_drvdata(pdev);
6422 struct igb_adapter *adapter = netdev_priv(netdev);
6423 struct e1000_hw *hw = &adapter->hw;
2d064c06 6424 u32 ctrl, rctl, status;
9d5c8243
AK		 6425 u32 wufc = adapter->wol;
6426#ifdef CONFIG_PM
6427 int retval = 0;
6428#endif
6429
6430 netif_device_detach(netdev);
6431
a88f10ec
AD		 6432 if (netif_running(netdev))
6433 igb_close(netdev);
6434
047e0030 6435 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 6436
6437#ifdef CONFIG_PM
6438 retval = pci_save_state(pdev);
6439 if (retval)
6440 return retval;
6441#endif
6442
6443 status = rd32(E1000_STATUS);
6444 if (status & E1000_STATUS_LU)
6445 wufc &= ~E1000_WUFC_LNKC;
6446
6447 if (wufc) {
6448 igb_setup_rctl(adapter);
ff41f8dc 6449 igb_set_rx_mode(netdev);
9d5c8243
AK		 6450
6451 /* turn on all-multi mode if wake on multicast is enabled */
6452 if (wufc & E1000_WUFC_MC) {
6453 rctl = rd32(E1000_RCTL);
6454 rctl |= E1000_RCTL_MPE;
6455 wr32(E1000_RCTL, rctl);
6456 }
6457
6458 ctrl = rd32(E1000_CTRL);
6459 /* advertise wake from D3Cold */
6460 #define E1000_CTRL_ADVD3WUC 0x00100000
6461 /* phy power management enable */
6462 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6463 ctrl |= E1000_CTRL_ADVD3WUC;
6464 wr32(E1000_CTRL, ctrl);
6465
9d5c8243 6466 /* Allow time for pending master requests to run */
330a6d6a 6467 igb_disable_pcie_master(hw);
9d5c8243
AK		 6468
6469 wr32(E1000_WUC, E1000_WUC_PME_EN);
6470 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 6471 } else {
6472 wr32(E1000_WUC, 0);
6473 wr32(E1000_WUFC, 0);
9d5c8243
AK		 6474 }
6475
3fe7c4c9
RW		 6476 *enable_wake = wufc || adapter->en_mng_pt;
6477 if (!*enable_wake)
88a268c1
NN		 6478 igb_power_down_link(adapter);
6479 else
6480 igb_power_up_link(adapter);
9d5c8243
AK		 6481
6482 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6483 * would have already happened in close and is redundant. */
6484 igb_release_hw_control(adapter);
6485
6486 pci_disable_device(pdev);
6487
9d5c8243
AK		 6488 return 0;
6489}
6490
6491#ifdef CONFIG_PM
3fe7c4c9
RW		 6492static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
6493{
6494 int retval;
6495 bool wake;
6496
6497 retval = __igb_shutdown(pdev, &wake);
6498 if (retval)
6499 return retval;
6500
6501 if (wake) {
6502 pci_prepare_to_sleep(pdev);
6503 } else {
6504 pci_wake_from_d3(pdev, false);
6505 pci_set_power_state(pdev, PCI_D3hot);
6506 }
6507
6508 return 0;
6509}
6510
9d5c8243
AK		 6511static int igb_resume(struct pci_dev *pdev)
6512{
6513 struct net_device *netdev = pci_get_drvdata(pdev);
6514 struct igb_adapter *adapter = netdev_priv(netdev);
6515 struct e1000_hw *hw = &adapter->hw;
6516 u32 err;
6517
6518 pci_set_power_state(pdev, PCI_D0);
6519 pci_restore_state(pdev);
b94f2d77 6520 pci_save_state(pdev);
42bfd33a 6521
aed5dec3 6522 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 6523 if (err) {
6524 dev_err(&pdev->dev,
6525 "igb: Cannot enable PCI device from suspend\n");
6526 return err;
6527 }
6528 pci_set_master(pdev);
6529
6530 pci_enable_wake(pdev, PCI_D3hot, 0);
6531 pci_enable_wake(pdev, PCI_D3cold, 0);
6532
047e0030 6533 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 6534 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6535 return -ENOMEM;
9d5c8243
AK		 6536 }
6537
9d5c8243 6538 igb_reset(adapter);
a8564f03
AD		 6539
6540 /* let the f/w know that the h/w is now under the control of the
6541 * driver. */
6542 igb_get_hw_control(adapter);
6543
9d5c8243
AK		 6544 wr32(E1000_WUS, ~0);
6545
a88f10ec
AD		 6546 if (netif_running(netdev)) {
6547 err = igb_open(netdev);
6548 if (err)
6549 return err;
6550 }
9d5c8243
AK		 6551
6552 netif_device_attach(netdev);
6553
9d5c8243
AK		 6554 return 0;
6555}
6556#endif
6557
6558static void igb_shutdown(struct pci_dev *pdev)
6559{
3fe7c4c9
RW		 6560 bool wake;
6561
6562 __igb_shutdown(pdev, &wake);
6563
6564 if (system_state == SYSTEM_POWER_OFF) {
6565 pci_wake_from_d3(pdev, wake);
6566 pci_set_power_state(pdev, PCI_D3hot);
6567 }
9d5c8243
AK		 6568}
6569
6570#ifdef CONFIG_NET_POLL_CONTROLLER
6571/*
6572 * Polling 'interrupt' - used by things like netconsole to send skbs
6573 * without having to re-enable interrupts. It's not called while
6574 * the interrupt routine is executing.
6575 */
6576static void igb_netpoll(struct net_device *netdev)
6577{
6578 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 6579 struct e1000_hw *hw = &adapter->hw;
9d5c8243 6580 int i;
9d5c8243 6581
eebbbdba 6582 if (!adapter->msix_entries) {
047e0030 6583 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 6584 igb_irq_disable(adapter);
047e0030 6585 napi_schedule(&q_vector->napi);
eebbbdba
AD		 6586 return;
6587 }
9d5c8243 6588
047e0030
AD		 6589 for (i = 0; i < adapter->num_q_vectors; i++) {
6590 struct igb_q_vector *q_vector = adapter->q_vector[i];
6591 wr32(E1000_EIMC, q_vector->eims_value);
6592 napi_schedule(&q_vector->napi);
eebbbdba 6593 }
9d5c8243
AK		 6594}
6595#endif /* CONFIG_NET_POLL_CONTROLLER */
6596
6597/**
6598 * igb_io_error_detected - called when PCI error is detected
6599 * @pdev: Pointer to PCI device
6600 * @state: The current pci connection state
6601 *
6602 * This function is called after a PCI bus error affecting
6603 * this device has been detected.
6604 */
6605static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6606 pci_channel_state_t state)
6607{
6608 struct net_device *netdev = pci_get_drvdata(pdev);
6609 struct igb_adapter *adapter = netdev_priv(netdev);
6610
6611 netif_device_detach(netdev);
6612
59ed6eec
AD		 6613 if (state == pci_channel_io_perm_failure)
6614 return PCI_ERS_RESULT_DISCONNECT;
6615
9d5c8243
AK		 6616 if (netif_running(netdev))
6617 igb_down(adapter);
6618 pci_disable_device(pdev);
6619
6620 /* Request a slot slot reset. */
6621 return PCI_ERS_RESULT_NEED_RESET;
6622}
6623
6624/**
6625 * igb_io_slot_reset - called after the pci bus has been reset.
6626 * @pdev: Pointer to PCI device
6627 *
6628 * Restart the card from scratch, as if from a cold-boot. Implementation
6629 * resembles the first-half of the igb_resume routine.
6630 */
6631static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6632{
6633 struct net_device *netdev = pci_get_drvdata(pdev);
6634 struct igb_adapter *adapter = netdev_priv(netdev);
6635 struct e1000_hw *hw = &adapter->hw;
40a914fa 6636 pci_ers_result_t result;
42bfd33a 6637 int err;
9d5c8243 6638
aed5dec3 6639 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6640 dev_err(&pdev->dev,
6641 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6642 result = PCI_ERS_RESULT_DISCONNECT;
6643 } else {
6644 pci_set_master(pdev);
6645 pci_restore_state(pdev);
b94f2d77 6646 pci_save_state(pdev);
9d5c8243 6647
40a914fa
AD		 6648 pci_enable_wake(pdev, PCI_D3hot, 0);
6649 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6650
40a914fa
AD		 6651 igb_reset(adapter);
6652 wr32(E1000_WUS, ~0);
6653 result = PCI_ERS_RESULT_RECOVERED;
6654 }
9d5c8243 6655
ea943d41
JK		 6656 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6657 if (err) {
6658 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6659 "failed 0x%0x\n", err);
6660 /* non-fatal, continue */
6661 }
40a914fa
AD		 6662
6663 return result;
9d5c8243
AK		 6664}
6665
6666/**
6667 * igb_io_resume - called when traffic can start flowing again.
6668 * @pdev: Pointer to PCI device
6669 *
6670 * This callback is called when the error recovery driver tells us that
6671 * its OK to resume normal operation. Implementation resembles the
6672 * second-half of the igb_resume routine.
6673 */
6674static void igb_io_resume(struct pci_dev *pdev)
6675{
6676 struct net_device *netdev = pci_get_drvdata(pdev);
6677 struct igb_adapter *adapter = netdev_priv(netdev);
6678
9d5c8243
AK		 6679 if (netif_running(netdev)) {
6680 if (igb_up(adapter)) {
6681 dev_err(&pdev->dev, "igb_up failed after reset\n");
6682 return;
6683 }
6684 }
6685
6686 netif_device_attach(netdev);
6687
6688 /* let the f/w know that the h/w is now under the control of the
6689 * driver. */
6690 igb_get_hw_control(adapter);
9d5c8243
AK		 6691}
6692
26ad9178
AD		 6693static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6694 u8 qsel)
6695{
6696 u32 rar_low, rar_high;
6697 struct e1000_hw *hw = &adapter->hw;
6698
6699 /* HW expects these in little endian so we reverse the byte order
6700 * from network order (big endian) to little endian
6701 */
6702 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6703 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6704 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6705
6706 /* Indicate to hardware the Address is Valid. */
6707 rar_high |= E1000_RAH_AV;
6708
6709 if (hw->mac.type == e1000_82575)
6710 rar_high |= E1000_RAH_POOL_1 * qsel;
6711 else
6712 rar_high |= E1000_RAH_POOL_1 << qsel;
6713
6714 wr32(E1000_RAL(index), rar_low);
6715 wrfl();
6716 wr32(E1000_RAH(index), rar_high);
6717 wrfl();
6718}
6719
4ae196df
AD		 6720static int igb_set_vf_mac(struct igb_adapter *adapter,
6721 int vf, unsigned char *mac_addr)
6722{
6723 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 6724 /* VF MAC addresses start at end of receive addresses and moves
6725 * torwards the first, as a result a collision should not be possible */
6726 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 6727
37680117 6728 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 6729
26ad9178 6730 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 6731
6732 return 0;
6733}
6734
8151d294
WM		 6735static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6736{
6737 struct igb_adapter *adapter = netdev_priv(netdev);
6738 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6739 return -EINVAL;
6740 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6741 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6742 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6743 " change effective.");
6744 if (test_bit(__IGB_DOWN, &adapter->state)) {
6745 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6746 " but the PF device is not up.\n");
6747 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6748 " attempting to use the VF device.\n");
6749 }
6750 return igb_set_vf_mac(adapter, vf, mac);
6751}
6752
17dc566c
LL		 6753static int igb_link_mbps(int internal_link_speed)
6754{
6755 switch (internal_link_speed) {
6756 case SPEED_100:
6757 return 100;
6758 case SPEED_1000:
6759 return 1000;
6760 default:
6761 return 0;
6762 }
6763}
6764
6765static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
6766 int link_speed)
6767{
6768 int rf_dec, rf_int;
6769 u32 bcnrc_val;
6770
6771 if (tx_rate != 0) {
6772 /* Calculate the rate factor values to set */
6773 rf_int = link_speed / tx_rate;
6774 rf_dec = (link_speed - (rf_int * tx_rate));
6775 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
6776
6777 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
6778 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
6779 E1000_RTTBCNRC_RF_INT_MASK);
6780 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
6781 } else {
6782 bcnrc_val = 0;
6783 }
6784
6785 wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
6786 wr32(E1000_RTTBCNRC, bcnrc_val);
6787}
6788
6789static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
6790{
6791 int actual_link_speed, i;
6792 bool reset_rate = false;
6793
6794 /* VF TX rate limit was not set or not supported */
6795 if ((adapter->vf_rate_link_speed == 0) ||
6796 (adapter->hw.mac.type != e1000_82576))
6797 return;
6798
6799 actual_link_speed = igb_link_mbps(adapter->link_speed);
6800 if (actual_link_speed != adapter->vf_rate_link_speed) {
6801 reset_rate = true;
6802 adapter->vf_rate_link_speed = 0;
6803 dev_info(&adapter->pdev->dev,
6804 "Link speed has been changed. VF Transmit "
6805 "rate is disabled\n");
6806 }
6807
6808 for (i = 0; i < adapter->vfs_allocated_count; i++) {
6809 if (reset_rate)
6810 adapter->vf_data[i].tx_rate = 0;
6811
6812 igb_set_vf_rate_limit(&adapter->hw, i,
6813 adapter->vf_data[i].tx_rate,
6814 actual_link_speed);
6815 }
6816}
6817
8151d294
WM		 6818static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6819{
17dc566c
LL		 6820 struct igb_adapter *adapter = netdev_priv(netdev);
6821 struct e1000_hw *hw = &adapter->hw;
6822 int actual_link_speed;
6823
6824 if (hw->mac.type != e1000_82576)
6825 return -EOPNOTSUPP;
6826
6827 actual_link_speed = igb_link_mbps(adapter->link_speed);
6828 if ((vf >= adapter->vfs_allocated_count) ||
6829 (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
6830 (tx_rate < 0) || (tx_rate > actual_link_speed))
6831 return -EINVAL;
6832
6833 adapter->vf_rate_link_speed = actual_link_speed;
6834 adapter->vf_data[vf].tx_rate = (u16)tx_rate;
6835 igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
6836
6837 return 0;
8151d294
WM		 6838}
6839
6840static int igb_ndo_get_vf_config(struct net_device *netdev,
6841 int vf, struct ifla_vf_info *ivi)
6842{
6843 struct igb_adapter *adapter = netdev_priv(netdev);
6844 if (vf >= adapter->vfs_allocated_count)
6845 return -EINVAL;
6846 ivi->vf = vf;
6847 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
17dc566c 6848 ivi->tx_rate = adapter->vf_data[vf].tx_rate;
8151d294
WM		 6849 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6850 ivi->qos = adapter->vf_data[vf].pf_qos;
6851 return 0;
6852}
6853
4ae196df
AD		 6854static void igb_vmm_control(struct igb_adapter *adapter)
6855{
6856 struct e1000_hw *hw = &adapter->hw;
10d8e907 6857 u32 reg;
4ae196df 6858
52a1dd4d
AD		 6859 switch (hw->mac.type) {
6860 case e1000_82575:
6861 default:
6862 /* replication is not supported for 82575 */
4ae196df 6863 return;
52a1dd4d
AD		 6864 case e1000_82576:
6865 /* notify HW that the MAC is adding vlan tags */
6866 reg = rd32(E1000_DTXCTL);
6867 reg |= E1000_DTXCTL_VLAN_ADDED;
6868 wr32(E1000_DTXCTL, reg);
6869 case e1000_82580:
6870 /* enable replication vlan tag stripping */
6871 reg = rd32(E1000_RPLOLR);
6872 reg |= E1000_RPLOLR_STRVLAN;
6873 wr32(E1000_RPLOLR, reg);
d2ba2ed8
AD		 6874 case e1000_i350:
6875 /* none of the above registers are supported by i350 */
52a1dd4d
AD		 6876 break;
6877 }
10d8e907 6878
d4960307
AD		 6879 if (adapter->vfs_allocated_count) {
6880 igb_vmdq_set_loopback_pf(hw, true);
6881 igb_vmdq_set_replication_pf(hw, true);
13800469
GR		 6882 igb_vmdq_set_anti_spoofing_pf(hw, true,
6883 adapter->vfs_allocated_count);
d4960307
AD		 6884 } else {
6885 igb_vmdq_set_loopback_pf(hw, false);
6886 igb_vmdq_set_replication_pf(hw, false);
6887 }
4ae196df
AD		 6888}
6889
9d5c8243 6890/* igb_main.c */
ubuntu-zesty-kernel mirror