]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - drivers/net/igb/igb_main.c
projects / mirror_ubuntu-zesty-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
iseries_veth: don't use flush_scheduled_work()
[mirror_ubuntu-zesty-kernel.git] / drivers / net / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
86d5d38f 4 Copyright(c) 2007-2009 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
31#include <linux/vmalloc.h>
32#include <linux/pagemap.h>
33#include <linux/netdevice.h>
9d5c8243 34#include <linux/ipv6.h>
5a0e3ad6 35#include <linux/slab.h>
9d5c8243
AK		 36#include <net/checksum.h>
37#include <net/ip6_checksum.h>
c6cb090b 38#include <linux/net_tstamp.h>
9d5c8243
AK		 39#include <linux/mii.h>
40#include <linux/ethtool.h>
41#include <linux/if_vlan.h>
42#include <linux/pci.h>
c54106bb 43#include <linux/pci-aspm.h>
9d5c8243
AK		 44#include <linux/delay.h>
45#include <linux/interrupt.h>
46#include <linux/if_ether.h>
40a914fa 47#include <linux/aer.h>
421e02f0 48#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 49#include <linux/dca.h>
50#endif
9d5c8243
AK		 51#include "igb.h"
52
55cac248 53#define DRV_VERSION "2.1.0-k2"
9d5c8243
AK		 54char igb_driver_name[] = "igb";
55char igb_driver_version[] = DRV_VERSION;
56static const char igb_driver_string[] =
57 "Intel(R) Gigabit Ethernet Network Driver";
86d5d38f 58static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
9d5c8243 59
9d5c8243
AK		 60static const struct e1000_info *igb_info_tbl[] = {
61 [board_82575] = &e1000_82575_info,
62};
63
a3aa1884 64static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
d2ba2ed8
AD		 65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
55cac248
AD		 69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
308fb39a
JG		 74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
2d064c06 76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
b894fa26 82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
c8ea5ea9 83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
87 /* required last entry */
88 {0, }
89};
90
91MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
92
93void igb_reset(struct igb_adapter *);
94static int igb_setup_all_tx_resources(struct igb_adapter *);
95static int igb_setup_all_rx_resources(struct igb_adapter *);
96static void igb_free_all_tx_resources(struct igb_adapter *);
97static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 98static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 99static int igb_probe(struct pci_dev *, const struct pci_device_id *);
100static void __devexit igb_remove(struct pci_dev *pdev);
101static int igb_sw_init(struct igb_adapter *);
102static int igb_open(struct net_device *);
103static int igb_close(struct net_device *);
104static void igb_configure_tx(struct igb_adapter *);
105static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 106static void igb_clean_all_tx_rings(struct igb_adapter *);
107static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 108static void igb_clean_tx_ring(struct igb_ring *);
109static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 110static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 111static void igb_update_phy_info(unsigned long);
112static void igb_watchdog(unsigned long);
113static void igb_watchdog_task(struct work_struct *);
b1a436c3 114static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
12dcd86b
ED		 115static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
116 struct rtnl_link_stats64 *stats);
9d5c8243
AK		 117static int igb_change_mtu(struct net_device *, int);
118static int igb_set_mac(struct net_device *, void *);
68d480c4 119static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 120static irqreturn_t igb_intr(int irq, void *);
121static irqreturn_t igb_intr_msi(int irq, void *);
122static irqreturn_t igb_msix_other(int irq, void *);
047e0030 123static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 124#ifdef CONFIG_IGB_DCA
047e0030 125static void igb_update_dca(struct igb_q_vector *);
fe4506b6 126static void igb_setup_dca(struct igb_adapter *);
421e02f0 127#endif /* CONFIG_IGB_DCA */
047e0030 128static bool igb_clean_tx_irq(struct igb_q_vector *);
661086df 129static int igb_poll(struct napi_struct *, int);
047e0030 130static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
9d5c8243
AK		 131static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
132static void igb_tx_timeout(struct net_device *);
133static void igb_reset_task(struct work_struct *);
134static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
135static void igb_vlan_rx_add_vid(struct net_device *, u16);
136static void igb_vlan_rx_kill_vid(struct net_device *, u16);
137static void igb_restore_vlan(struct igb_adapter *);
26ad9178 138static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 139static void igb_ping_all_vfs(struct igb_adapter *);
140static void igb_msg_task(struct igb_adapter *);
4ae196df 141static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 142static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 143static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 144static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
145static int igb_ndo_set_vf_vlan(struct net_device *netdev,
146 int vf, u16 vlan, u8 qos);
147static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
148static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
149 struct ifla_vf_info *ivi);
9d5c8243 150
9d5c8243 151#ifdef CONFIG_PM
3fe7c4c9 152static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 153static int igb_resume(struct pci_dev *);
154#endif
155static void igb_shutdown(struct pci_dev *);
421e02f0 156#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 157static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
158static struct notifier_block dca_notifier = {
159 .notifier_call = igb_notify_dca,
160 .next = NULL,
161 .priority = 0
162};
163#endif
9d5c8243
AK		 164#ifdef CONFIG_NET_POLL_CONTROLLER
165/* for netdump / net console */
166static void igb_netpoll(struct net_device *);
167#endif
37680117 168#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 169static unsigned int max_vfs = 0;
170module_param(max_vfs, uint, 0);
171MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
172 "per physical function");
173#endif /* CONFIG_PCI_IOV */
174
9d5c8243
AK		 175static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
176 pci_channel_state_t);
177static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
178static void igb_io_resume(struct pci_dev *);
179
180static struct pci_error_handlers igb_err_handler = {
181 .error_detected = igb_io_error_detected,
182 .slot_reset = igb_io_slot_reset,
183 .resume = igb_io_resume,
184};
185
186
187static struct pci_driver igb_driver = {
188 .name = igb_driver_name,
189 .id_table = igb_pci_tbl,
190 .probe = igb_probe,
191 .remove = __devexit_p(igb_remove),
192#ifdef CONFIG_PM
193 /* Power Managment Hooks */
194 .suspend = igb_suspend,
195 .resume = igb_resume,
196#endif
197 .shutdown = igb_shutdown,
198 .err_handler = &igb_err_handler
199};
200
201MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
202MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
203MODULE_LICENSE("GPL");
204MODULE_VERSION(DRV_VERSION);
205
c97ec42a
TI		 206struct igb_reg_info {
207 u32 ofs;
208 char *name;
209};
210
211static const struct igb_reg_info igb_reg_info_tbl[] = {
212
213 /* General Registers */
214 {E1000_CTRL, "CTRL"},
215 {E1000_STATUS, "STATUS"},
216 {E1000_CTRL_EXT, "CTRL_EXT"},
217
218 /* Interrupt Registers */
219 {E1000_ICR, "ICR"},
220
221 /* RX Registers */
222 {E1000_RCTL, "RCTL"},
223 {E1000_RDLEN(0), "RDLEN"},
224 {E1000_RDH(0), "RDH"},
225 {E1000_RDT(0), "RDT"},
226 {E1000_RXDCTL(0), "RXDCTL"},
227 {E1000_RDBAL(0), "RDBAL"},
228 {E1000_RDBAH(0), "RDBAH"},
229
230 /* TX Registers */
231 {E1000_TCTL, "TCTL"},
232 {E1000_TDBAL(0), "TDBAL"},
233 {E1000_TDBAH(0), "TDBAH"},
234 {E1000_TDLEN(0), "TDLEN"},
235 {E1000_TDH(0), "TDH"},
236 {E1000_TDT(0), "TDT"},
237 {E1000_TXDCTL(0), "TXDCTL"},
238 {E1000_TDFH, "TDFH"},
239 {E1000_TDFT, "TDFT"},
240 {E1000_TDFHS, "TDFHS"},
241 {E1000_TDFPC, "TDFPC"},
242
243 /* List Terminator */
244 {}
245};
246
247/*
248 * igb_regdump - register printout routine
249 */
250static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
251{
252 int n = 0;
253 char rname[16];
254 u32 regs[8];
255
256 switch (reginfo->ofs) {
257 case E1000_RDLEN(0):
258 for (n = 0; n < 4; n++)
259 regs[n] = rd32(E1000_RDLEN(n));
260 break;
261 case E1000_RDH(0):
262 for (n = 0; n < 4; n++)
263 regs[n] = rd32(E1000_RDH(n));
264 break;
265 case E1000_RDT(0):
266 for (n = 0; n < 4; n++)
267 regs[n] = rd32(E1000_RDT(n));
268 break;
269 case E1000_RXDCTL(0):
270 for (n = 0; n < 4; n++)
271 regs[n] = rd32(E1000_RXDCTL(n));
272 break;
273 case E1000_RDBAL(0):
274 for (n = 0; n < 4; n++)
275 regs[n] = rd32(E1000_RDBAL(n));
276 break;
277 case E1000_RDBAH(0):
278 for (n = 0; n < 4; n++)
279 regs[n] = rd32(E1000_RDBAH(n));
280 break;
281 case E1000_TDBAL(0):
282 for (n = 0; n < 4; n++)
283 regs[n] = rd32(E1000_RDBAL(n));
284 break;
285 case E1000_TDBAH(0):
286 for (n = 0; n < 4; n++)
287 regs[n] = rd32(E1000_TDBAH(n));
288 break;
289 case E1000_TDLEN(0):
290 for (n = 0; n < 4; n++)
291 regs[n] = rd32(E1000_TDLEN(n));
292 break;
293 case E1000_TDH(0):
294 for (n = 0; n < 4; n++)
295 regs[n] = rd32(E1000_TDH(n));
296 break;
297 case E1000_TDT(0):
298 for (n = 0; n < 4; n++)
299 regs[n] = rd32(E1000_TDT(n));
300 break;
301 case E1000_TXDCTL(0):
302 for (n = 0; n < 4; n++)
303 regs[n] = rd32(E1000_TXDCTL(n));
304 break;
305 default:
306 printk(KERN_INFO "%-15s %08x\n",
307 reginfo->name, rd32(reginfo->ofs));
308 return;
309 }
310
311 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
312 printk(KERN_INFO "%-15s ", rname);
313 for (n = 0; n < 4; n++)
314 printk(KERN_CONT "%08x ", regs[n]);
315 printk(KERN_CONT "\n");
316}
317
318/*
319 * igb_dump - Print registers, tx-rings and rx-rings
320 */
321static void igb_dump(struct igb_adapter *adapter)
322{
323 struct net_device *netdev = adapter->netdev;
324 struct e1000_hw *hw = &adapter->hw;
325 struct igb_reg_info *reginfo;
326 int n = 0;
327 struct igb_ring *tx_ring;
328 union e1000_adv_tx_desc *tx_desc;
329 struct my_u0 { u64 a; u64 b; } *u0;
330 struct igb_buffer *buffer_info;
331 struct igb_ring *rx_ring;
332 union e1000_adv_rx_desc *rx_desc;
333 u32 staterr;
334 int i = 0;
335
336 if (!netif_msg_hw(adapter))
337 return;
338
339 /* Print netdevice Info */
340 if (netdev) {
341 dev_info(&adapter->pdev->dev, "Net device Info\n");
342 printk(KERN_INFO "Device Name state "
343 "trans_start last_rx\n");
344 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
345 netdev->name,
346 netdev->state,
347 netdev->trans_start,
348 netdev->last_rx);
349 }
350
351 /* Print Registers */
352 dev_info(&adapter->pdev->dev, "Register Dump\n");
353 printk(KERN_INFO " Register Name Value\n");
354 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
355 reginfo->name; reginfo++) {
356 igb_regdump(hw, reginfo);
357 }
358
359 /* Print TX Ring Summary */
360 if (!netdev || !netif_running(netdev))
361 goto exit;
362
363 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
364 printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma ]"
365 " leng ntw timestamp\n");
366 for (n = 0; n < adapter->num_tx_queues; n++) {
367 tx_ring = adapter->tx_ring[n];
368 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
369 printk(KERN_INFO " %5d %5X %5X %016llX %04X %3X %016llX\n",
370 n, tx_ring->next_to_use, tx_ring->next_to_clean,
371 (u64)buffer_info->dma,
372 buffer_info->length,
373 buffer_info->next_to_watch,
374 (u64)buffer_info->time_stamp);
375 }
376
377 /* Print TX Rings */
378 if (!netif_msg_tx_done(adapter))
379 goto rx_ring_summary;
380
381 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
382
383 /* Transmit Descriptor Formats
384 *
385 * Advanced Transmit Descriptor
386 * +--------------------------------------------------------------+
387 * 0 | Buffer Address [63:0] |
388 * +--------------------------------------------------------------+
389 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
390 * +--------------------------------------------------------------+
391 * 63 46 45 40 39 38 36 35 32 31 24 15 0
392 */
393
394 for (n = 0; n < adapter->num_tx_queues; n++) {
395 tx_ring = adapter->tx_ring[n];
396 printk(KERN_INFO "------------------------------------\n");
397 printk(KERN_INFO "TX QUEUE INDEX = %d\n", tx_ring->queue_index);
398 printk(KERN_INFO "------------------------------------\n");
399 printk(KERN_INFO "T [desc] [address 63:0 ] "
400 "[PlPOCIStDDM Ln] [bi->dma ] "
401 "leng ntw timestamp bi->skb\n");
402
403 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
404 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
405 buffer_info = &tx_ring->buffer_info[i];
406 u0 = (struct my_u0 *)tx_desc;
407 printk(KERN_INFO "T [0x%03X] %016llX %016llX %016llX"
408 " %04X %3X %016llX %p", i,
409 le64_to_cpu(u0->a),
410 le64_to_cpu(u0->b),
411 (u64)buffer_info->dma,
412 buffer_info->length,
413 buffer_info->next_to_watch,
414 (u64)buffer_info->time_stamp,
415 buffer_info->skb);
416 if (i == tx_ring->next_to_use &&
417 i == tx_ring->next_to_clean)
418 printk(KERN_CONT " NTC/U\n");
419 else if (i == tx_ring->next_to_use)
420 printk(KERN_CONT " NTU\n");
421 else if (i == tx_ring->next_to_clean)
422 printk(KERN_CONT " NTC\n");
423 else
424 printk(KERN_CONT "\n");
425
426 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
427 print_hex_dump(KERN_INFO, "",
428 DUMP_PREFIX_ADDRESS,
429 16, 1, phys_to_virt(buffer_info->dma),
430 buffer_info->length, true);
431 }
432 }
433
434 /* Print RX Rings Summary */
435rx_ring_summary:
436 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
437 printk(KERN_INFO "Queue [NTU] [NTC]\n");
438 for (n = 0; n < adapter->num_rx_queues; n++) {
439 rx_ring = adapter->rx_ring[n];
440 printk(KERN_INFO " %5d %5X %5X\n", n,
441 rx_ring->next_to_use, rx_ring->next_to_clean);
442 }
443
444 /* Print RX Rings */
445 if (!netif_msg_rx_status(adapter))
446 goto exit;
447
448 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
449
450 /* Advanced Receive Descriptor (Read) Format
451 * 63 1 0
452 * +-----------------------------------------------------+
453 * 0 | Packet Buffer Address [63:1] |A0/NSE|
454 * +----------------------------------------------+------+
455 * 8 | Header Buffer Address [63:1] | DD |
456 * +-----------------------------------------------------+
457 *
458 *
459 * Advanced Receive Descriptor (Write-Back) Format
460 *
461 * 63 48 47 32 31 30 21 20 17 16 4 3 0
462 * +------------------------------------------------------+
463 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
464 * | Checksum Ident | | | | Type | Type |
465 * +------------------------------------------------------+
466 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
467 * +------------------------------------------------------+
468 * 63 48 47 32 31 20 19 0
469 */
470
471 for (n = 0; n < adapter->num_rx_queues; n++) {
472 rx_ring = adapter->rx_ring[n];
473 printk(KERN_INFO "------------------------------------\n");
474 printk(KERN_INFO "RX QUEUE INDEX = %d\n", rx_ring->queue_index);
475 printk(KERN_INFO "------------------------------------\n");
476 printk(KERN_INFO "R [desc] [ PktBuf A0] "
477 "[ HeadBuf DD] [bi->dma ] [bi->skb] "
478 "<-- Adv Rx Read format\n");
479 printk(KERN_INFO "RWB[desc] [PcsmIpSHl PtRs] "
480 "[vl er S cks ln] ---------------- [bi->skb] "
481 "<-- Adv Rx Write-Back format\n");
482
483 for (i = 0; i < rx_ring->count; i++) {
484 buffer_info = &rx_ring->buffer_info[i];
485 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
486 u0 = (struct my_u0 *)rx_desc;
487 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
488 if (staterr & E1000_RXD_STAT_DD) {
489 /* Descriptor Done */
490 printk(KERN_INFO "RWB[0x%03X] %016llX "
491 "%016llX ---------------- %p", i,
492 le64_to_cpu(u0->a),
493 le64_to_cpu(u0->b),
494 buffer_info->skb);
495 } else {
496 printk(KERN_INFO "R [0x%03X] %016llX "
497 "%016llX %016llX %p", i,
498 le64_to_cpu(u0->a),
499 le64_to_cpu(u0->b),
500 (u64)buffer_info->dma,
501 buffer_info->skb);
502
503 if (netif_msg_pktdata(adapter)) {
504 print_hex_dump(KERN_INFO, "",
505 DUMP_PREFIX_ADDRESS,
506 16, 1,
507 phys_to_virt(buffer_info->dma),
508 rx_ring->rx_buffer_len, true);
509 if (rx_ring->rx_buffer_len
510 < IGB_RXBUFFER_1024)
511 print_hex_dump(KERN_INFO, "",
512 DUMP_PREFIX_ADDRESS,
513 16, 1,
514 phys_to_virt(
515 buffer_info->page_dma +
516 buffer_info->page_offset),
517 PAGE_SIZE/2, true);
518 }
519 }
520
521 if (i == rx_ring->next_to_use)
522 printk(KERN_CONT " NTU\n");
523 else if (i == rx_ring->next_to_clean)
524 printk(KERN_CONT " NTC\n");
525 else
526 printk(KERN_CONT "\n");
527
528 }
529 }
530
531exit:
532 return;
533}
534
535
38c845c7
PO		 536/**
537 * igb_read_clock - read raw cycle counter (to be used by time counter)
538 */
539static cycle_t igb_read_clock(const struct cyclecounter *tc)
540{
541 struct igb_adapter *adapter =
542 container_of(tc, struct igb_adapter, cycles);
543 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 544 u64 stamp = 0;
545 int shift = 0;
38c845c7 546
55cac248
AD		 547 /*
548 * The timestamp latches on lowest register read. For the 82580
549 * the lowest register is SYSTIMR instead of SYSTIML. However we never
550 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
551 */
552 if (hw->mac.type == e1000_82580) {
553 stamp = rd32(E1000_SYSTIMR) >> 8;
554 shift = IGB_82580_TSYNC_SHIFT;
555 }
556
c5b9bd5e
AD		 557 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
558 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 559 return stamp;
560}
561
9d5c8243 562/**
c041076a 563 * igb_get_hw_dev - return device
9d5c8243
AK		 564 * used by hardware layer to print debugging information
565 **/
c041076a 566struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
9d5c8243
AK		 567{
568 struct igb_adapter *adapter = hw->back;
c041076a 569 return adapter->netdev;
9d5c8243 570}
38c845c7 571
9d5c8243
AK		 572/**
573 * igb_init_module - Driver Registration Routine
574 *
575 * igb_init_module is the first routine called when the driver is
576 * loaded. All it does is register with the PCI subsystem.
577 **/
578static int __init igb_init_module(void)
579{
580 int ret;
581 printk(KERN_INFO "%s - version %s\n",
582 igb_driver_string, igb_driver_version);
583
584 printk(KERN_INFO "%s\n", igb_copyright);
585
421e02f0 586#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 587 dca_register_notify(&dca_notifier);
588#endif
bbd98fe4 589 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 590 return ret;
591}
592
593module_init(igb_init_module);
594
595/**
596 * igb_exit_module - Driver Exit Cleanup Routine
597 *
598 * igb_exit_module is called just before the driver is removed
599 * from memory.
600 **/
601static void __exit igb_exit_module(void)
602{
421e02f0 603#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 604 dca_unregister_notify(&dca_notifier);
605#endif
9d5c8243
AK		 606 pci_unregister_driver(&igb_driver);
607}
608
609module_exit(igb_exit_module);
610
26bc19ec
AD		 611#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
612/**
613 * igb_cache_ring_register - Descriptor ring to register mapping
614 * @adapter: board private structure to initialize
615 *
616 * Once we know the feature-set enabled for the device, we'll cache
617 * the register offset the descriptor ring is assigned to.
618 **/
619static void igb_cache_ring_register(struct igb_adapter *adapter)
620{
ee1b9f06 621 int i = 0, j = 0;
047e0030 622 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 623
624 switch (adapter->hw.mac.type) {
625 case e1000_82576:
626 /* The queues are allocated for virtualization such that VF 0
627 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
628 * In order to avoid collision we start at the first free queue
629 * and continue consuming queues in the same sequence
630 */
ee1b9f06 631 if (adapter->vfs_allocated_count) {
a99955fc 632 for (; i < adapter->rss_queues; i++)
3025a446
AD		 633 adapter->rx_ring[i]->reg_idx = rbase_offset +
634 Q_IDX_82576(i);
ee1b9f06 635 }
26bc19ec 636 case e1000_82575:
55cac248 637 case e1000_82580:
d2ba2ed8 638 case e1000_i350:
26bc19ec 639 default:
ee1b9f06 640 for (; i < adapter->num_rx_queues; i++)
3025a446 641 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 642 for (; j < adapter->num_tx_queues; j++)
3025a446 643 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 644 break;
645 }
646}
647
047e0030
AD		 648static void igb_free_queues(struct igb_adapter *adapter)
649{
3025a446 650 int i;
047e0030 651
3025a446
AD		 652 for (i = 0; i < adapter->num_tx_queues; i++) {
653 kfree(adapter->tx_ring[i]);
654 adapter->tx_ring[i] = NULL;
655 }
656 for (i = 0; i < adapter->num_rx_queues; i++) {
657 kfree(adapter->rx_ring[i]);
658 adapter->rx_ring[i] = NULL;
659 }
047e0030
AD		 660 adapter->num_rx_queues = 0;
661 adapter->num_tx_queues = 0;
662}
663
9d5c8243
AK		 664/**
665 * igb_alloc_queues - Allocate memory for all rings
666 * @adapter: board private structure to initialize
667 *
668 * We allocate one ring per queue at run-time since we don't know the
669 * number of queues at compile-time.
670 **/
671static int igb_alloc_queues(struct igb_adapter *adapter)
672{
3025a446 673 struct igb_ring *ring;
9d5c8243
AK		 674 int i;
675
661086df 676 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 677 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
678 if (!ring)
679 goto err;
68fd9910 680 ring->count = adapter->tx_ring_count;
661086df 681 ring->queue_index = i;
59d71989 682 ring->dev = &adapter->pdev->dev;
e694e964 683 ring->netdev = adapter->netdev;
85ad76b2
AD		 684 /* For 82575, context index must be unique per ring. */
685 if (adapter->hw.mac.type == e1000_82575)
686 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
3025a446 687 adapter->tx_ring[i] = ring;
661086df 688 }
85ad76b2 689
9d5c8243 690 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446
AD		 691 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
692 if (!ring)
693 goto err;
68fd9910 694 ring->count = adapter->rx_ring_count;
844290e5 695 ring->queue_index = i;
59d71989 696 ring->dev = &adapter->pdev->dev;
e694e964 697 ring->netdev = adapter->netdev;
4c844851 698 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85ad76b2
AD		 699 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
700 /* set flag indicating ring supports SCTP checksum offload */
701 if (adapter->hw.mac.type >= e1000_82576)
702 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
3025a446 703 adapter->rx_ring[i] = ring;
9d5c8243 704 }
26bc19ec
AD		 705
706 igb_cache_ring_register(adapter);
9d5c8243 707
047e0030 708 return 0;
a88f10ec 709
047e0030
AD		 710err:
711 igb_free_queues(adapter);
d1a8c9e1 712
047e0030 713 return -ENOMEM;
a88f10ec
AD		 714}
715
9d5c8243 716#define IGB_N0_QUEUE -1
047e0030 717static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 718{
719 u32 msixbm = 0;
047e0030 720 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 721 struct e1000_hw *hw = &adapter->hw;
2d064c06 722 u32 ivar, index;
047e0030
AD		 723 int rx_queue = IGB_N0_QUEUE;
724 int tx_queue = IGB_N0_QUEUE;
725
726 if (q_vector->rx_ring)
727 rx_queue = q_vector->rx_ring->reg_idx;
728 if (q_vector->tx_ring)
729 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 730
731 switch (hw->mac.type) {
732 case e1000_82575:
9d5c8243
AK		 733 /* The 82575 assigns vectors using a bitmask, which matches the
734 bitmask for the EICR/EIMS/EIMC registers. To assign one
735 or more queues to a vector, we write the appropriate bits
736 into the MSIXBM register for that vector. */
047e0030 737 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 738 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 739 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 740 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 741 if (!adapter->msix_entries && msix_vector == 0)
742 msixbm |= E1000_EIMS_OTHER;
9d5c8243 743 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 744 q_vector->eims_value = msixbm;
2d064c06
AD		 745 break;
746 case e1000_82576:
26bc19ec 747 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 748 Each queue has a single entry in the table to which we write
749 a vector number along with a "valid" bit. Sadly, the layout
750 of the table is somewhat counterintuitive. */
751 if (rx_queue > IGB_N0_QUEUE) {
047e0030 752 index = (rx_queue & 0x7);
2d064c06 753 ivar = array_rd32(E1000_IVAR0, index);
047e0030 754 if (rx_queue < 8) {
26bc19ec
AD		 755 /* vector goes into low byte of register */
756 ivar = ivar & 0xFFFFFF00;
757 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 758 } else {
759 /* vector goes into third byte of register */
760 ivar = ivar & 0xFF00FFFF;
761 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 762 }
2d064c06
AD		 763 array_wr32(E1000_IVAR0, index, ivar);
764 }
765 if (tx_queue > IGB_N0_QUEUE) {
047e0030 766 index = (tx_queue & 0x7);
2d064c06 767 ivar = array_rd32(E1000_IVAR0, index);
047e0030 768 if (tx_queue < 8) {
26bc19ec
AD		 769 /* vector goes into second byte of register */
770 ivar = ivar & 0xFFFF00FF;
771 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 772 } else {
773 /* vector goes into high byte of register */
774 ivar = ivar & 0x00FFFFFF;
775 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 776 }
2d064c06
AD		 777 array_wr32(E1000_IVAR0, index, ivar);
778 }
047e0030 779 q_vector->eims_value = 1 << msix_vector;
2d064c06 780 break;
55cac248 781 case e1000_82580:
d2ba2ed8 782 case e1000_i350:
55cac248
AD		 783 /* 82580 uses the same table-based approach as 82576 but has fewer
784 entries as a result we carry over for queues greater than 4. */
785 if (rx_queue > IGB_N0_QUEUE) {
786 index = (rx_queue >> 1);
787 ivar = array_rd32(E1000_IVAR0, index);
788 if (rx_queue & 0x1) {
789 /* vector goes into third byte of register */
790 ivar = ivar & 0xFF00FFFF;
791 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
792 } else {
793 /* vector goes into low byte of register */
794 ivar = ivar & 0xFFFFFF00;
795 ivar |= msix_vector | E1000_IVAR_VALID;
796 }
797 array_wr32(E1000_IVAR0, index, ivar);
798 }
799 if (tx_queue > IGB_N0_QUEUE) {
800 index = (tx_queue >> 1);
801 ivar = array_rd32(E1000_IVAR0, index);
802 if (tx_queue & 0x1) {
803 /* vector goes into high byte of register */
804 ivar = ivar & 0x00FFFFFF;
805 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
806 } else {
807 /* vector goes into second byte of register */
808 ivar = ivar & 0xFFFF00FF;
809 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
810 }
811 array_wr32(E1000_IVAR0, index, ivar);
812 }
813 q_vector->eims_value = 1 << msix_vector;
814 break;
2d064c06
AD		 815 default:
816 BUG();
817 break;
818 }
26b39276
AD		 819
820 /* add q_vector eims value to global eims_enable_mask */
821 adapter->eims_enable_mask |= q_vector->eims_value;
822
823 /* configure q_vector to set itr on first interrupt */
824 q_vector->set_itr = 1;
9d5c8243
AK		 825}
826
827/**
828 * igb_configure_msix - Configure MSI-X hardware
829 *
830 * igb_configure_msix sets up the hardware to properly
831 * generate MSI-X interrupts.
832 **/
833static void igb_configure_msix(struct igb_adapter *adapter)
834{
835 u32 tmp;
836 int i, vector = 0;
837 struct e1000_hw *hw = &adapter->hw;
838
839 adapter->eims_enable_mask = 0;
9d5c8243
AK		 840
841 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 842 switch (hw->mac.type) {
843 case e1000_82575:
9d5c8243
AK		 844 tmp = rd32(E1000_CTRL_EXT);
845 /* enable MSI-X PBA support*/
846 tmp |= E1000_CTRL_EXT_PBA_CLR;
847
848 /* Auto-Mask interrupts upon ICR read. */
849 tmp |= E1000_CTRL_EXT_EIAME;
850 tmp |= E1000_CTRL_EXT_IRCA;
851
852 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 853
854 /* enable msix_other interrupt */
855 array_wr32(E1000_MSIXBM(0), vector++,
856 E1000_EIMS_OTHER);
844290e5 857 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 858
2d064c06
AD		 859 break;
860
861 case e1000_82576:
55cac248 862 case e1000_82580:
d2ba2ed8 863 case e1000_i350:
047e0030
AD		 864 /* Turn on MSI-X capability first, or our settings
865 * won't stick. And it will take days to debug. */
866 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
867 E1000_GPIE_PBA | E1000_GPIE_EIAME |
868 E1000_GPIE_NSICR);
869
870 /* enable msix_other interrupt */
871 adapter->eims_other = 1 << vector;
2d064c06 872 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 873
047e0030 874 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 875 break;
876 default:
877 /* do nothing, since nothing else supports MSI-X */
878 break;
879 } /* switch (hw->mac.type) */
047e0030
AD		 880
881 adapter->eims_enable_mask |= adapter->eims_other;
882
26b39276
AD		 883 for (i = 0; i < adapter->num_q_vectors; i++)
884 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 885
9d5c8243
AK		 886 wrfl();
887}
888
889/**
890 * igb_request_msix - Initialize MSI-X interrupts
891 *
892 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
893 * kernel.
894 **/
895static int igb_request_msix(struct igb_adapter *adapter)
896{
897 struct net_device *netdev = adapter->netdev;
047e0030 898 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 899 int i, err = 0, vector = 0;
900
047e0030 901 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 902 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 903 if (err)
904 goto out;
905 vector++;
906
907 for (i = 0; i < adapter->num_q_vectors; i++) {
908 struct igb_q_vector *q_vector = adapter->q_vector[i];
909
910 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
911
912 if (q_vector->rx_ring && q_vector->tx_ring)
913 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
914 q_vector->rx_ring->queue_index);
915 else if (q_vector->tx_ring)
916 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
917 q_vector->tx_ring->queue_index);
918 else if (q_vector->rx_ring)
919 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
920 q_vector->rx_ring->queue_index);
9d5c8243 921 else
047e0030
AD		 922 sprintf(q_vector->name, "%s-unused", netdev->name);
923
9d5c8243 924 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 925 igb_msix_ring, 0, q_vector->name,
047e0030 926 q_vector);
9d5c8243
AK		 927 if (err)
928 goto out;
9d5c8243
AK		 929 vector++;
930 }
931
9d5c8243
AK		 932 igb_configure_msix(adapter);
933 return 0;
934out:
935 return err;
936}
937
938static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
939{
940 if (adapter->msix_entries) {
941 pci_disable_msix(adapter->pdev);
942 kfree(adapter->msix_entries);
943 adapter->msix_entries = NULL;
047e0030 944 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 945 pci_disable_msi(adapter->pdev);
047e0030 946 }
9d5c8243
AK		 947}
948
047e0030
AD		 949/**
950 * igb_free_q_vectors - Free memory allocated for interrupt vectors
951 * @adapter: board private structure to initialize
952 *
953 * This function frees the memory allocated to the q_vectors. In addition if
954 * NAPI is enabled it will delete any references to the NAPI struct prior
955 * to freeing the q_vector.
956 **/
957static void igb_free_q_vectors(struct igb_adapter *adapter)
958{
959 int v_idx;
960
961 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
962 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
963 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 964 if (!q_vector)
965 continue;
047e0030
AD		 966 netif_napi_del(&q_vector->napi);
967 kfree(q_vector);
968 }
969 adapter->num_q_vectors = 0;
970}
971
972/**
973 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
974 *
975 * This function resets the device so that it has 0 rx queues, tx queues, and
976 * MSI-X interrupts allocated.
977 */
978static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
979{
980 igb_free_queues(adapter);
981 igb_free_q_vectors(adapter);
982 igb_reset_interrupt_capability(adapter);
983}
9d5c8243
AK		 984
985/**
986 * igb_set_interrupt_capability - set MSI or MSI-X if supported
987 *
988 * Attempt to configure interrupts using the best available
989 * capabilities of the hardware and kernel.
990 **/
21adef3e 991static int igb_set_interrupt_capability(struct igb_adapter *adapter)
9d5c8243
AK		 992{
993 int err;
994 int numvecs, i;
995
83b7180d 996 /* Number of supported queues. */
a99955fc 997 adapter->num_rx_queues = adapter->rss_queues;
5fa8517f
GR		 998 if (adapter->vfs_allocated_count)
999 adapter->num_tx_queues = 1;
1000 else
1001 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 1002
047e0030
AD		 1003 /* start with one vector for every rx queue */
1004 numvecs = adapter->num_rx_queues;
1005
3ad2f3fb 1006 /* if tx handler is separate add 1 for every tx queue */
a99955fc
AD		 1007 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1008 numvecs += adapter->num_tx_queues;
047e0030
AD		 1009
1010 /* store the number of vectors reserved for queues */
1011 adapter->num_q_vectors = numvecs;
1012
1013 /* add 1 vector for link status interrupts */
1014 numvecs++;
9d5c8243
AK		 1015 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1016 GFP_KERNEL);
1017 if (!adapter->msix_entries)
1018 goto msi_only;
1019
1020 for (i = 0; i < numvecs; i++)
1021 adapter->msix_entries[i].entry = i;
1022
1023 err = pci_enable_msix(adapter->pdev,
1024 adapter->msix_entries,
1025 numvecs);
1026 if (err == 0)
34a20e89 1027 goto out;
9d5c8243
AK		 1028
1029 igb_reset_interrupt_capability(adapter);
1030
1031 /* If we can't do MSI-X, try MSI */
1032msi_only:
2a3abf6d
AD		 1033#ifdef CONFIG_PCI_IOV
1034 /* disable SR-IOV for non MSI-X configurations */
1035 if (adapter->vf_data) {
1036 struct e1000_hw *hw = &adapter->hw;
1037 /* disable iov and allow time for transactions to clear */
1038 pci_disable_sriov(adapter->pdev);
1039 msleep(500);
1040
1041 kfree(adapter->vf_data);
1042 adapter->vf_data = NULL;
1043 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1044 msleep(100);
1045 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1046 }
1047#endif
4fc82adf 1048 adapter->vfs_allocated_count = 0;
a99955fc 1049 adapter->rss_queues = 1;
4fc82adf 1050 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 1051 adapter->num_rx_queues = 1;
661086df 1052 adapter->num_tx_queues = 1;
047e0030 1053 adapter->num_q_vectors = 1;
9d5c8243 1054 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1055 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 1056out:
21adef3e
BH		 1057 /* Notify the stack of the (possibly) reduced queue counts. */
1058 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
1059 return netif_set_real_num_rx_queues(adapter->netdev,
1060 adapter->num_rx_queues);
9d5c8243
AK		 1061}
1062
047e0030
AD		 1063/**
1064 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1065 * @adapter: board private structure to initialize
1066 *
1067 * We allocate one q_vector per queue interrupt. If allocation fails we
1068 * return -ENOMEM.
1069 **/
1070static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1071{
1072 struct igb_q_vector *q_vector;
1073 struct e1000_hw *hw = &adapter->hw;
1074 int v_idx;
1075
1076 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1077 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
1078 if (!q_vector)
1079 goto err_out;
1080 q_vector->adapter = adapter;
047e0030
AD		 1081 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1082 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1083 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1084 adapter->q_vector[v_idx] = q_vector;
1085 }
1086 return 0;
1087
1088err_out:
fe0592b4 1089 igb_free_q_vectors(adapter);
047e0030
AD		 1090 return -ENOMEM;
1091}
1092
1093static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1094 int ring_idx, int v_idx)
1095{
3025a446 1096 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1097
3025a446 1098 q_vector->rx_ring = adapter->rx_ring[ring_idx];
047e0030 1099 q_vector->rx_ring->q_vector = q_vector;
4fc82adf
AD		 1100 q_vector->itr_val = adapter->rx_itr_setting;
1101 if (q_vector->itr_val && q_vector->itr_val <= 3)
1102 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1103}
1104
1105static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1106 int ring_idx, int v_idx)
1107{
3025a446 1108 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1109
3025a446 1110 q_vector->tx_ring = adapter->tx_ring[ring_idx];
047e0030 1111 q_vector->tx_ring->q_vector = q_vector;
4fc82adf
AD		 1112 q_vector->itr_val = adapter->tx_itr_setting;
1113 if (q_vector->itr_val && q_vector->itr_val <= 3)
1114 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1115}
1116
1117/**
1118 * igb_map_ring_to_vector - maps allocated queues to vectors
1119 *
1120 * This function maps the recently allocated queues to vectors.
1121 **/
1122static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1123{
1124 int i;
1125 int v_idx = 0;
1126
1127 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1128 (adapter->num_q_vectors < adapter->num_tx_queues))
1129 return -ENOMEM;
1130
1131 if (adapter->num_q_vectors >=
1132 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1133 for (i = 0; i < adapter->num_rx_queues; i++)
1134 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1135 for (i = 0; i < adapter->num_tx_queues; i++)
1136 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1137 } else {
1138 for (i = 0; i < adapter->num_rx_queues; i++) {
1139 if (i < adapter->num_tx_queues)
1140 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1141 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1142 }
1143 for (; i < adapter->num_tx_queues; i++)
1144 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1145 }
1146 return 0;
1147}
1148
1149/**
1150 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1151 *
1152 * This function initializes the interrupts and allocates all of the queues.
1153 **/
1154static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1155{
1156 struct pci_dev *pdev = adapter->pdev;
1157 int err;
1158
21adef3e
BH		 1159 err = igb_set_interrupt_capability(adapter);
1160 if (err)
1161 return err;
047e0030
AD		 1162
1163 err = igb_alloc_q_vectors(adapter);
1164 if (err) {
1165 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1166 goto err_alloc_q_vectors;
1167 }
1168
1169 err = igb_alloc_queues(adapter);
1170 if (err) {
1171 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1172 goto err_alloc_queues;
1173 }
1174
1175 err = igb_map_ring_to_vector(adapter);
1176 if (err) {
1177 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1178 goto err_map_queues;
1179 }
1180
1181
1182 return 0;
1183err_map_queues:
1184 igb_free_queues(adapter);
1185err_alloc_queues:
1186 igb_free_q_vectors(adapter);
1187err_alloc_q_vectors:
1188 igb_reset_interrupt_capability(adapter);
1189 return err;
1190}
1191
9d5c8243
AK		 1192/**
1193 * igb_request_irq - initialize interrupts
1194 *
1195 * Attempts to configure interrupts using the best available
1196 * capabilities of the hardware and kernel.
1197 **/
1198static int igb_request_irq(struct igb_adapter *adapter)
1199{
1200 struct net_device *netdev = adapter->netdev;
047e0030 1201 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1202 int err = 0;
1203
1204 if (adapter->msix_entries) {
1205 err = igb_request_msix(adapter);
844290e5 1206 if (!err)
9d5c8243 1207 goto request_done;
9d5c8243 1208 /* fall back to MSI */
047e0030 1209 igb_clear_interrupt_scheme(adapter);
9d5c8243 1210 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1211 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1212 igb_free_all_tx_resources(adapter);
1213 igb_free_all_rx_resources(adapter);
047e0030 1214 adapter->num_tx_queues = 1;
9d5c8243 1215 adapter->num_rx_queues = 1;
047e0030
AD		 1216 adapter->num_q_vectors = 1;
1217 err = igb_alloc_q_vectors(adapter);
1218 if (err) {
1219 dev_err(&pdev->dev,
1220 "Unable to allocate memory for vectors\n");
1221 goto request_done;
1222 }
1223 err = igb_alloc_queues(adapter);
1224 if (err) {
1225 dev_err(&pdev->dev,
1226 "Unable to allocate memory for queues\n");
1227 igb_free_q_vectors(adapter);
1228 goto request_done;
1229 }
1230 igb_setup_all_tx_resources(adapter);
1231 igb_setup_all_rx_resources(adapter);
844290e5 1232 } else {
feeb2721 1233 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243 1234 }
844290e5 1235
7dfc16fa 1236 if (adapter->flags & IGB_FLAG_HAS_MSI) {
a0607fd3 1237 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
047e0030 1238 netdev->name, adapter);
9d5c8243
AK		 1239 if (!err)
1240 goto request_done;
047e0030 1241
9d5c8243
AK		 1242 /* fall back to legacy interrupts */
1243 igb_reset_interrupt_capability(adapter);
7dfc16fa 1244 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1245 }
1246
a0607fd3 1247 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
047e0030 1248 netdev->name, adapter);
9d5c8243 1249
6cb5e577 1250 if (err)
9d5c8243
AK		 1251 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
1252 err);
9d5c8243
AK		 1253
1254request_done:
1255 return err;
1256}
1257
1258static void igb_free_irq(struct igb_adapter *adapter)
1259{
9d5c8243
AK		 1260 if (adapter->msix_entries) {
1261 int vector = 0, i;
1262
047e0030 1263 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 1264
047e0030
AD		 1265 for (i = 0; i < adapter->num_q_vectors; i++) {
1266 struct igb_q_vector *q_vector = adapter->q_vector[i];
1267 free_irq(adapter->msix_entries[vector++].vector,
1268 q_vector);
1269 }
1270 } else {
1271 free_irq(adapter->pdev->irq, adapter);
9d5c8243 1272 }
9d5c8243
AK		 1273}
1274
1275/**
1276 * igb_irq_disable - Mask off interrupt generation on the NIC
1277 * @adapter: board private structure
1278 **/
1279static void igb_irq_disable(struct igb_adapter *adapter)
1280{
1281 struct e1000_hw *hw = &adapter->hw;
1282
25568a53
AD		 1283 /*
1284 * we need to be careful when disabling interrupts. The VFs are also
1285 * mapped into these registers and so clearing the bits can cause
1286 * issues on the VF drivers so we only need to clear what we set
1287 */
9d5c8243 1288 if (adapter->msix_entries) {
2dfd1212
AD		 1289 u32 regval = rd32(E1000_EIAM);
1290 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1291 wr32(E1000_EIMC, adapter->eims_enable_mask);
1292 regval = rd32(E1000_EIAC);
1293 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 1294 }
844290e5
PW		 1295
1296 wr32(E1000_IAM, 0);
9d5c8243
AK		 1297 wr32(E1000_IMC, ~0);
1298 wrfl();
81a61859
ET		 1299 if (adapter->msix_entries) {
1300 int i;
1301 for (i = 0; i < adapter->num_q_vectors; i++)
1302 synchronize_irq(adapter->msix_entries[i].vector);
1303 } else {
1304 synchronize_irq(adapter->pdev->irq);
1305 }
9d5c8243
AK		 1306}
1307
1308/**
1309 * igb_irq_enable - Enable default interrupt generation settings
1310 * @adapter: board private structure
1311 **/
1312static void igb_irq_enable(struct igb_adapter *adapter)
1313{
1314 struct e1000_hw *hw = &adapter->hw;
1315
1316 if (adapter->msix_entries) {
25568a53 1317 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
2dfd1212
AD		 1318 u32 regval = rd32(E1000_EIAC);
1319 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1320 regval = rd32(E1000_EIAM);
1321 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 1322 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 1323 if (adapter->vfs_allocated_count) {
4ae196df 1324 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1325 ims |= E1000_IMS_VMMB;
1326 }
55cac248
AD		 1327 if (adapter->hw.mac.type == e1000_82580)
1328 ims |= E1000_IMS_DRSTA;
1329
25568a53 1330 wr32(E1000_IMS, ims);
844290e5 1331 } else {
55cac248
AD		 1332 wr32(E1000_IMS, IMS_ENABLE_MASK |
1333 E1000_IMS_DRSTA);
1334 wr32(E1000_IAM, IMS_ENABLE_MASK |
1335 E1000_IMS_DRSTA);
844290e5 1336 }
9d5c8243
AK		 1337}
1338
1339static void igb_update_mng_vlan(struct igb_adapter *adapter)
1340{
51466239 1341 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1342 u16 vid = adapter->hw.mng_cookie.vlan_id;
1343 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1344
1345 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1346 /* add VID to filter table */
1347 igb_vfta_set(hw, vid, true);
1348 adapter->mng_vlan_id = vid;
1349 } else {
1350 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1351 }
1352
1353 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1354 (vid != old_vid) &&
1355 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1356 /* remove VID from filter table */
1357 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1358 }
1359}
1360
1361/**
1362 * igb_release_hw_control - release control of the h/w to f/w
1363 * @adapter: address of board private structure
1364 *
1365 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1366 * For ASF and Pass Through versions of f/w this means that the
1367 * driver is no longer loaded.
1368 *
1369 **/
1370static void igb_release_hw_control(struct igb_adapter *adapter)
1371{
1372 struct e1000_hw *hw = &adapter->hw;
1373 u32 ctrl_ext;
1374
1375 /* Let firmware take over control of h/w */
1376 ctrl_ext = rd32(E1000_CTRL_EXT);
1377 wr32(E1000_CTRL_EXT,
1378 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1379}
1380
9d5c8243
AK		 1381/**
1382 * igb_get_hw_control - get control of the h/w from f/w
1383 * @adapter: address of board private structure
1384 *
1385 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1386 * For ASF and Pass Through versions of f/w this means that
1387 * the driver is loaded.
1388 *
1389 **/
1390static void igb_get_hw_control(struct igb_adapter *adapter)
1391{
1392 struct e1000_hw *hw = &adapter->hw;
1393 u32 ctrl_ext;
1394
1395 /* Let firmware know the driver has taken over */
1396 ctrl_ext = rd32(E1000_CTRL_EXT);
1397 wr32(E1000_CTRL_EXT,
1398 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1399}
1400
9d5c8243
AK		 1401/**
1402 * igb_configure - configure the hardware for RX and TX
1403 * @adapter: private board structure
1404 **/
1405static void igb_configure(struct igb_adapter *adapter)
1406{
1407 struct net_device *netdev = adapter->netdev;
1408 int i;
1409
1410 igb_get_hw_control(adapter);
ff41f8dc 1411 igb_set_rx_mode(netdev);
9d5c8243
AK		 1412
1413 igb_restore_vlan(adapter);
9d5c8243 1414
85b430b4 1415 igb_setup_tctl(adapter);
06cf2666 1416 igb_setup_mrqc(adapter);
9d5c8243 1417 igb_setup_rctl(adapter);
85b430b4
AD		 1418
1419 igb_configure_tx(adapter);
9d5c8243 1420 igb_configure_rx(adapter);
662d7205
AD		 1421
1422 igb_rx_fifo_flush_82575(&adapter->hw);
1423
c493ea45 1424 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1425 * at least 1 descriptor unused to make sure
1426 * next_to_use != next_to_clean */
1427 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1428 struct igb_ring *ring = adapter->rx_ring[i];
c493ea45 1429 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
9d5c8243 1430 }
9d5c8243
AK		 1431}
1432
88a268c1
NN		 1433/**
1434 * igb_power_up_link - Power up the phy/serdes link
1435 * @adapter: address of board private structure
1436 **/
1437void igb_power_up_link(struct igb_adapter *adapter)
1438{
1439 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1440 igb_power_up_phy_copper(&adapter->hw);
1441 else
1442 igb_power_up_serdes_link_82575(&adapter->hw);
1443}
1444
1445/**
1446 * igb_power_down_link - Power down the phy/serdes link
1447 * @adapter: address of board private structure
1448 */
1449static void igb_power_down_link(struct igb_adapter *adapter)
1450{
1451 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1452 igb_power_down_phy_copper_82575(&adapter->hw);
1453 else
1454 igb_shutdown_serdes_link_82575(&adapter->hw);
1455}
9d5c8243
AK		 1456
1457/**
1458 * igb_up - Open the interface and prepare it to handle traffic
1459 * @adapter: board private structure
1460 **/
9d5c8243
AK		 1461int igb_up(struct igb_adapter *adapter)
1462{
1463 struct e1000_hw *hw = &adapter->hw;
1464 int i;
1465
1466 /* hardware has been reset, we need to reload some things */
1467 igb_configure(adapter);
1468
1469 clear_bit(__IGB_DOWN, &adapter->state);
1470
047e0030
AD		 1471 for (i = 0; i < adapter->num_q_vectors; i++) {
1472 struct igb_q_vector *q_vector = adapter->q_vector[i];
1473 napi_enable(&q_vector->napi);
1474 }
844290e5 1475 if (adapter->msix_entries)
9d5c8243 1476 igb_configure_msix(adapter);
feeb2721
AD		 1477 else
1478 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1479
1480 /* Clear any pending interrupts. */
1481 rd32(E1000_ICR);
1482 igb_irq_enable(adapter);
1483
d4960307
AD		 1484 /* notify VFs that reset has been completed */
1485 if (adapter->vfs_allocated_count) {
1486 u32 reg_data = rd32(E1000_CTRL_EXT);
1487 reg_data |= E1000_CTRL_EXT_PFRSTD;
1488 wr32(E1000_CTRL_EXT, reg_data);
1489 }
1490
4cb9be7a
JB		 1491 netif_tx_start_all_queues(adapter->netdev);
1492
25568a53
AD		 1493 /* start the watchdog. */
1494 hw->mac.get_link_status = 1;
1495 schedule_work(&adapter->watchdog_task);
1496
9d5c8243
AK		 1497 return 0;
1498}
1499
1500void igb_down(struct igb_adapter *adapter)
1501{
9d5c8243 1502 struct net_device *netdev = adapter->netdev;
330a6d6a 1503 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1504 u32 tctl, rctl;
1505 int i;
1506
1507 /* signal that we're down so the interrupt handler does not
1508 * reschedule our watchdog timer */
1509 set_bit(__IGB_DOWN, &adapter->state);
1510
1511 /* disable receives in the hardware */
1512 rctl = rd32(E1000_RCTL);
1513 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1514 /* flush and sleep below */
1515
fd2ea0a7 1516 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1517
1518 /* disable transmits in the hardware */
1519 tctl = rd32(E1000_TCTL);
1520 tctl &= ~E1000_TCTL_EN;
1521 wr32(E1000_TCTL, tctl);
1522 /* flush both disables and wait for them to finish */
1523 wrfl();
1524 msleep(10);
1525
047e0030
AD		 1526 for (i = 0; i < adapter->num_q_vectors; i++) {
1527 struct igb_q_vector *q_vector = adapter->q_vector[i];
1528 napi_disable(&q_vector->napi);
1529 }
9d5c8243 1530
9d5c8243
AK		 1531 igb_irq_disable(adapter);
1532
1533 del_timer_sync(&adapter->watchdog_timer);
1534 del_timer_sync(&adapter->phy_info_timer);
1535
9d5c8243 1536 netif_carrier_off(netdev);
04fe6358
AD		 1537
1538 /* record the stats before reset*/
12dcd86b
ED		 1539 spin_lock(&adapter->stats64_lock);
1540 igb_update_stats(adapter, &adapter->stats64);
1541 spin_unlock(&adapter->stats64_lock);
04fe6358 1542
9d5c8243
AK		 1543 adapter->link_speed = 0;
1544 adapter->link_duplex = 0;
1545
3023682e
JK		 1546 if (!pci_channel_offline(adapter->pdev))
1547 igb_reset(adapter);
9d5c8243
AK		 1548 igb_clean_all_tx_rings(adapter);
1549 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1550#ifdef CONFIG_IGB_DCA
1551
1552 /* since we reset the hardware DCA settings were cleared */
1553 igb_setup_dca(adapter);
1554#endif
9d5c8243
AK		 1555}
1556
1557void igb_reinit_locked(struct igb_adapter *adapter)
1558{
1559 WARN_ON(in_interrupt());
1560 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1561 msleep(1);
1562 igb_down(adapter);
1563 igb_up(adapter);
1564 clear_bit(__IGB_RESETTING, &adapter->state);
1565}
1566
1567void igb_reset(struct igb_adapter *adapter)
1568{
090b1795 1569 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1570 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1571 struct e1000_mac_info *mac = &hw->mac;
1572 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1573 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1574 u16 hwm;
1575
1576 /* Repartition Pba for greater than 9k mtu
1577 * To take effect CTRL.RST is required.
1578 */
fa4dfae0 1579 switch (mac->type) {
d2ba2ed8 1580 case e1000_i350:
55cac248
AD		 1581 case e1000_82580:
1582 pba = rd32(E1000_RXPBS);
1583 pba = igb_rxpbs_adjust_82580(pba);
1584 break;
fa4dfae0 1585 case e1000_82576:
d249be54
AD		 1586 pba = rd32(E1000_RXPBS);
1587 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1588 break;
1589 case e1000_82575:
1590 default:
1591 pba = E1000_PBA_34K;
1592 break;
2d064c06 1593 }
9d5c8243 1594
2d064c06
AD		 1595 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1596 (mac->type < e1000_82576)) {
9d5c8243
AK		 1597 /* adjust PBA for jumbo frames */
1598 wr32(E1000_PBA, pba);
1599
1600 /* To maintain wire speed transmits, the Tx FIFO should be
1601 * large enough to accommodate two full transmit packets,
1602 * rounded up to the next 1KB and expressed in KB. Likewise,
1603 * the Rx FIFO should be large enough to accommodate at least
1604 * one full receive packet and is similarly rounded up and
1605 * expressed in KB. */
1606 pba = rd32(E1000_PBA);
1607 /* upper 16 bits has Tx packet buffer allocation size in KB */
1608 tx_space = pba >> 16;
1609 /* lower 16 bits has Rx packet buffer allocation size in KB */
1610 pba &= 0xffff;
1611 /* the tx fifo also stores 16 bytes of information about the tx
1612 * but don't include ethernet FCS because hardware appends it */
1613 min_tx_space = (adapter->max_frame_size +
85e8d004 1614 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1615 ETH_FCS_LEN) * 2;
1616 min_tx_space = ALIGN(min_tx_space, 1024);
1617 min_tx_space >>= 10;
1618 /* software strips receive CRC, so leave room for it */
1619 min_rx_space = adapter->max_frame_size;
1620 min_rx_space = ALIGN(min_rx_space, 1024);
1621 min_rx_space >>= 10;
1622
1623 /* If current Tx allocation is less than the min Tx FIFO size,
1624 * and the min Tx FIFO size is less than the current Rx FIFO
1625 * allocation, take space away from current Rx allocation */
1626 if (tx_space < min_tx_space &&
1627 ((min_tx_space - tx_space) < pba)) {
1628 pba = pba - (min_tx_space - tx_space);
1629
1630 /* if short on rx space, rx wins and must trump tx
1631 * adjustment */
1632 if (pba < min_rx_space)
1633 pba = min_rx_space;
1634 }
2d064c06 1635 wr32(E1000_PBA, pba);
9d5c8243 1636 }
9d5c8243
AK		 1637
1638 /* flow control settings */
1639 /* The high water mark must be low enough to fit one full frame
1640 * (or the size used for early receive) above it in the Rx FIFO.
1641 * Set it to the lower of:
1642 * - 90% of the Rx FIFO size, or
1643 * - the full Rx FIFO size minus one full frame */
1644 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1645 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1646
d405ea3e
AD		 1647 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1648 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1649 fc->pause_time = 0xFFFF;
1650 fc->send_xon = 1;
0cce119a 1651 fc->current_mode = fc->requested_mode;
9d5c8243 1652
4ae196df
AD		 1653 /* disable receive for all VFs and wait one second */
1654 if (adapter->vfs_allocated_count) {
1655 int i;
1656 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
f2ca0dbe 1657 adapter->vf_data[i].flags = 0;
4ae196df
AD		 1658
1659 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1660 igb_ping_all_vfs(adapter);
4ae196df
AD		 1661
1662 /* disable transmits and receives */
1663 wr32(E1000_VFRE, 0);
1664 wr32(E1000_VFTE, 0);
1665 }
1666
9d5c8243 1667 /* Allow time for pending master requests to run */
330a6d6a 1668 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1669 wr32(E1000_WUC, 0);
1670
330a6d6a 1671 if (hw->mac.ops.init_hw(hw))
090b1795 1672 dev_err(&pdev->dev, "Hardware Error\n");
9d5c8243 1673
55cac248
AD		 1674 if (hw->mac.type == e1000_82580) {
1675 u32 reg = rd32(E1000_PCIEMISC);
1676 wr32(E1000_PCIEMISC,
1677 reg & ~E1000_PCIEMISC_LX_DECISION);
1678 }
88a268c1
NN		 1679 if (!netif_running(adapter->netdev))
1680 igb_power_down_link(adapter);
1681
9d5c8243
AK		 1682 igb_update_mng_vlan(adapter);
1683
1684 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1685 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1686
330a6d6a 1687 igb_get_phy_info(hw);
9d5c8243
AK		 1688}
1689
2e5c6922 1690static const struct net_device_ops igb_netdev_ops = {
559e9c49 1691 .ndo_open = igb_open,
2e5c6922 1692 .ndo_stop = igb_close,
00829823 1693 .ndo_start_xmit = igb_xmit_frame_adv,
12dcd86b 1694 .ndo_get_stats64 = igb_get_stats64,
ff41f8dc
AD		 1695 .ndo_set_rx_mode = igb_set_rx_mode,
1696 .ndo_set_multicast_list = igb_set_rx_mode,
2e5c6922
SH		 1697 .ndo_set_mac_address = igb_set_mac,
1698 .ndo_change_mtu = igb_change_mtu,
1699 .ndo_do_ioctl = igb_ioctl,
1700 .ndo_tx_timeout = igb_tx_timeout,
1701 .ndo_validate_addr = eth_validate_addr,
1702 .ndo_vlan_rx_register = igb_vlan_rx_register,
1703 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1704 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1705 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1706 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1707 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1708 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1709#ifdef CONFIG_NET_POLL_CONTROLLER
1710 .ndo_poll_controller = igb_netpoll,
1711#endif
1712};
1713
9d5c8243
AK		 1714/**
1715 * igb_probe - Device Initialization Routine
1716 * @pdev: PCI device information struct
1717 * @ent: entry in igb_pci_tbl
1718 *
1719 * Returns 0 on success, negative on failure
1720 *
1721 * igb_probe initializes an adapter identified by a pci_dev structure.
1722 * The OS initialization, configuring of the adapter private structure,
1723 * and a hardware reset occur.
1724 **/
1725static int __devinit igb_probe(struct pci_dev *pdev,
1726 const struct pci_device_id *ent)
1727{
1728 struct net_device *netdev;
1729 struct igb_adapter *adapter;
1730 struct e1000_hw *hw;
4337e993 1731 u16 eeprom_data = 0;
9835fd73 1732 s32 ret_val;
4337e993 1733 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1734 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1735 unsigned long mmio_start, mmio_len;
2d6a5e95 1736 int err, pci_using_dac;
9d5c8243 1737 u16 eeprom_apme_mask = IGB_EEPROM_APME;
9835fd73 1738 u8 part_str[E1000_PBANUM_LENGTH];
9d5c8243 1739
bded64a7
AG		 1740 /* Catch broken hardware that put the wrong VF device ID in
1741 * the PCIe SR-IOV capability.
1742 */
1743 if (pdev->is_virtfn) {
1744 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1745 pci_name(pdev), pdev->vendor, pdev->device);
1746 return -EINVAL;
1747 }
1748
aed5dec3 1749 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1750 if (err)
1751 return err;
1752
1753 pci_using_dac = 0;
59d71989 1754 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243 1755 if (!err) {
59d71989 1756 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243
AK		 1757 if (!err)
1758 pci_using_dac = 1;
1759 } else {
59d71989 1760 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243 1761 if (err) {
59d71989 1762 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243
AK		 1763 if (err) {
1764 dev_err(&pdev->dev, "No usable DMA "
1765 "configuration, aborting\n");
1766 goto err_dma;
1767 }
1768 }
1769 }
1770
aed5dec3
AD		 1771 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1772 IORESOURCE_MEM),
1773 igb_driver_name);
9d5c8243
AK		 1774 if (err)
1775 goto err_pci_reg;
1776
19d5afd4 1777 pci_enable_pcie_error_reporting(pdev);
40a914fa 1778
9d5c8243 1779 pci_set_master(pdev);
c682fc23 1780 pci_save_state(pdev);
9d5c8243
AK		 1781
1782 err = -ENOMEM;
1bfaf07b
AD		 1783 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1784 IGB_ABS_MAX_TX_QUEUES);
9d5c8243
AK		 1785 if (!netdev)
1786 goto err_alloc_etherdev;
1787
1788 SET_NETDEV_DEV(netdev, &pdev->dev);
1789
1790 pci_set_drvdata(pdev, netdev);
1791 adapter = netdev_priv(netdev);
1792 adapter->netdev = netdev;
1793 adapter->pdev = pdev;
1794 hw = &adapter->hw;
1795 hw->back = adapter;
1796 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1797
1798 mmio_start = pci_resource_start(pdev, 0);
1799 mmio_len = pci_resource_len(pdev, 0);
1800
1801 err = -EIO;
28b0759c
AD		 1802 hw->hw_addr = ioremap(mmio_start, mmio_len);
1803 if (!hw->hw_addr)
9d5c8243
AK		 1804 goto err_ioremap;
1805
2e5c6922 1806 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1807 igb_set_ethtool_ops(netdev);
9d5c8243 1808 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1809
1810 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1811
1812 netdev->mem_start = mmio_start;
1813 netdev->mem_end = mmio_start + mmio_len;
1814
9d5c8243
AK		 1815 /* PCI config space info */
1816 hw->vendor_id = pdev->vendor;
1817 hw->device_id = pdev->device;
1818 hw->revision_id = pdev->revision;
1819 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1820 hw->subsystem_device_id = pdev->subsystem_device;
1821
9d5c8243
AK		 1822 /* Copy the default MAC, PHY and NVM function pointers */
1823 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1824 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1825 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1826 /* Initialize skew-specific constants */
1827 err = ei->get_invariants(hw);
1828 if (err)
450c87c8 1829 goto err_sw_init;
9d5c8243 1830
450c87c8 1831 /* setup the private structure */
9d5c8243
AK		 1832 err = igb_sw_init(adapter);
1833 if (err)
1834 goto err_sw_init;
1835
1836 igb_get_bus_info_pcie(hw);
1837
1838 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 1839
1840 /* Copper options */
1841 if (hw->phy.media_type == e1000_media_type_copper) {
1842 hw->phy.mdix = AUTO_ALL_MODES;
1843 hw->phy.disable_polarity_correction = false;
1844 hw->phy.ms_type = e1000_ms_hw_default;
1845 }
1846
1847 if (igb_check_reset_block(hw))
1848 dev_info(&pdev->dev,
1849 "PHY reset is blocked due to SOL/IDER session.\n");
1850
1851 netdev->features = NETIF_F_SG |
7d8eb29e 1852 NETIF_F_IP_CSUM |
9d5c8243
AK		 1853 NETIF_F_HW_VLAN_TX |
1854 NETIF_F_HW_VLAN_RX |
1855 NETIF_F_HW_VLAN_FILTER;
1856
7d8eb29e 1857 netdev->features |= NETIF_F_IPV6_CSUM;
9d5c8243 1858 netdev->features |= NETIF_F_TSO;
9d5c8243 1859 netdev->features |= NETIF_F_TSO6;
5c0999b7 1860 netdev->features |= NETIF_F_GRO;
d3352520 1861
48f29ffc
JK		 1862 netdev->vlan_features |= NETIF_F_TSO;
1863 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 1864 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 1865 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 1866 netdev->vlan_features |= NETIF_F_SG;
1867
7b872a55 1868 if (pci_using_dac) {
9d5c8243 1869 netdev->features |= NETIF_F_HIGHDMA;
7b872a55
YZ		 1870 netdev->vlan_features |= NETIF_F_HIGHDMA;
1871 }
9d5c8243 1872
5b043fb0 1873 if (hw->mac.type >= e1000_82576)
b9473560
JB		 1874 netdev->features |= NETIF_F_SCTP_CSUM;
1875
330a6d6a 1876 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 1877
1878 /* before reading the NVM, reset the controller to put the device in a
1879 * known good starting state */
1880 hw->mac.ops.reset_hw(hw);
1881
1882 /* make sure the NVM is good */
1883 if (igb_validate_nvm_checksum(hw) < 0) {
1884 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1885 err = -EIO;
1886 goto err_eeprom;
1887 }
1888
1889 /* copy the MAC address out of the NVM */
1890 if (hw->mac.ops.read_mac_addr(hw))
1891 dev_err(&pdev->dev, "NVM Read Error\n");
1892
1893 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1894 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1895
1896 if (!is_valid_ether_addr(netdev->perm_addr)) {
1897 dev_err(&pdev->dev, "Invalid MAC Address\n");
1898 err = -EIO;
1899 goto err_eeprom;
1900 }
1901
c061b18d 1902 setup_timer(&adapter->watchdog_timer, igb_watchdog,
0e340485 1903 (unsigned long) adapter);
c061b18d 1904 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
0e340485 1905 (unsigned long) adapter);
9d5c8243
AK		 1906
1907 INIT_WORK(&adapter->reset_task, igb_reset_task);
1908 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1909
450c87c8 1910 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 1911 adapter->fc_autoneg = true;
1912 hw->mac.autoneg = true;
1913 hw->phy.autoneg_advertised = 0x2f;
1914
0cce119a
AD		 1915 hw->fc.requested_mode = e1000_fc_default;
1916 hw->fc.current_mode = e1000_fc_default;
9d5c8243 1917
9d5c8243
AK		 1918 igb_validate_mdi_setting(hw);
1919
9d5c8243
AK		 1920 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1921 * enable the ACPI Magic Packet filter
1922 */
1923
a2cf8b6c 1924 if (hw->bus.func == 0)
312c75ae 1925 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
55cac248
AD		 1926 else if (hw->mac.type == e1000_82580)
1927 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1928 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1929 &eeprom_data);
a2cf8b6c
AD		 1930 else if (hw->bus.func == 1)
1931 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 1932
1933 if (eeprom_data & eeprom_apme_mask)
1934 adapter->eeprom_wol |= E1000_WUFC_MAG;
1935
1936 /* now that we have the eeprom settings, apply the special cases where
1937 * the eeprom may be wrong or the board simply won't support wake on
1938 * lan on a particular port */
1939 switch (pdev->device) {
1940 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1941 adapter->eeprom_wol = 0;
1942 break;
1943 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 1944 case E1000_DEV_ID_82576_FIBER:
1945 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 1946 /* Wake events only supported on port A for dual fiber
1947 * regardless of eeprom setting */
1948 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1949 adapter->eeprom_wol = 0;
1950 break;
c8ea5ea9 1951 case E1000_DEV_ID_82576_QUAD_COPPER:
d5aa2252 1952 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
c8ea5ea9
AD		 1953 /* if quad port adapter, disable WoL on all but port A */
1954 if (global_quad_port_a != 0)
1955 adapter->eeprom_wol = 0;
1956 else
1957 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1958 /* Reset for multiple quad port adapters */
1959 if (++global_quad_port_a == 4)
1960 global_quad_port_a = 0;
1961 break;
9d5c8243
AK		 1962 }
1963
1964 /* initialize the wol settings based on the eeprom settings */
1965 adapter->wol = adapter->eeprom_wol;
e1b86d84 1966 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 1967
1968 /* reset the hardware with the new settings */
1969 igb_reset(adapter);
1970
1971 /* let the f/w know that the h/w is now under the control of the
1972 * driver. */
1973 igb_get_hw_control(adapter);
1974
9d5c8243
AK		 1975 strcpy(netdev->name, "eth%d");
1976 err = register_netdev(netdev);
1977 if (err)
1978 goto err_register;
1979
b168dfc5
JB		 1980 /* carrier off reporting is important to ethtool even BEFORE open */
1981 netif_carrier_off(netdev);
1982
421e02f0 1983#ifdef CONFIG_IGB_DCA
bbd98fe4 1984 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 1985 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 1986 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 1987 igb_setup_dca(adapter);
1988 }
fe4506b6 1989
38c845c7 1990#endif
9d5c8243
AK		 1991 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1992 /* print bus type/speed/width info */
7c510e4b 1993 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 1994 netdev->name,
559e9c49 1995 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
ff846f52 1996 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
559e9c49 1997 "unknown"),
59c3de89
AD		 1998 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1999 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
2000 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2001 "unknown"),
7c510e4b 2002 netdev->dev_addr);
9d5c8243 2003
9835fd73
CW		 2004 ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH);
2005 if (ret_val)
2006 strcpy(part_str, "Unknown");
2007 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
9d5c8243
AK		 2008 dev_info(&pdev->dev,
2009 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2010 adapter->msix_entries ? "MSI-X" :
7dfc16fa 2011 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243
AK		 2012 adapter->num_rx_queues, adapter->num_tx_queues);
2013
9d5c8243
AK		 2014 return 0;
2015
2016err_register:
2017 igb_release_hw_control(adapter);
2018err_eeprom:
2019 if (!igb_check_reset_block(hw))
f5f4cf08 2020 igb_reset_phy(hw);
9d5c8243
AK		 2021
2022 if (hw->flash_address)
2023 iounmap(hw->flash_address);
9d5c8243 2024err_sw_init:
047e0030 2025 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 2026 iounmap(hw->hw_addr);
2027err_ioremap:
2028 free_netdev(netdev);
2029err_alloc_etherdev:
559e9c49
AD		 2030 pci_release_selected_regions(pdev,
2031 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2032err_pci_reg:
2033err_dma:
2034 pci_disable_device(pdev);
2035 return err;
2036}
2037
2038/**
2039 * igb_remove - Device Removal Routine
2040 * @pdev: PCI device information struct
2041 *
2042 * igb_remove is called by the PCI subsystem to alert the driver
2043 * that it should release a PCI device. The could be caused by a
2044 * Hot-Plug event, or because the driver is going to be removed from
2045 * memory.
2046 **/
2047static void __devexit igb_remove(struct pci_dev *pdev)
2048{
2049 struct net_device *netdev = pci_get_drvdata(pdev);
2050 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 2051 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2052
2053 /* flush_scheduled work may reschedule our watchdog task, so
2054 * explicitly disable watchdog tasks from being rescheduled */
2055 set_bit(__IGB_DOWN, &adapter->state);
2056 del_timer_sync(&adapter->watchdog_timer);
2057 del_timer_sync(&adapter->phy_info_timer);
2058
2059 flush_scheduled_work();
2060
421e02f0 2061#ifdef CONFIG_IGB_DCA
7dfc16fa 2062 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 2063 dev_info(&pdev->dev, "DCA disabled\n");
2064 dca_remove_requester(&pdev->dev);
7dfc16fa 2065 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 2066 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 2067 }
2068#endif
2069
9d5c8243
AK		 2070 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2071 * would have already happened in close and is redundant. */
2072 igb_release_hw_control(adapter);
2073
2074 unregister_netdev(netdev);
2075
047e0030 2076 igb_clear_interrupt_scheme(adapter);
9d5c8243 2077
37680117
AD		 2078#ifdef CONFIG_PCI_IOV
2079 /* reclaim resources allocated to VFs */
2080 if (adapter->vf_data) {
2081 /* disable iov and allow time for transactions to clear */
2082 pci_disable_sriov(pdev);
2083 msleep(500);
2084
2085 kfree(adapter->vf_data);
2086 adapter->vf_data = NULL;
2087 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
2088 msleep(100);
2089 dev_info(&pdev->dev, "IOV Disabled\n");
2090 }
2091#endif
559e9c49 2092
28b0759c
AD		 2093 iounmap(hw->hw_addr);
2094 if (hw->flash_address)
2095 iounmap(hw->flash_address);
559e9c49
AD		 2096 pci_release_selected_regions(pdev,
2097 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2098
2099 free_netdev(netdev);
2100
19d5afd4 2101 pci_disable_pcie_error_reporting(pdev);
40a914fa 2102
9d5c8243
AK		 2103 pci_disable_device(pdev);
2104}
2105
a6b623e0
AD		 2106/**
2107 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2108 * @adapter: board private structure to initialize
2109 *
2110 * This function initializes the vf specific data storage and then attempts to
2111 * allocate the VFs. The reason for ordering it this way is because it is much
2112 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2113 * the memory for the VFs.
2114 **/
2115static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2116{
2117#ifdef CONFIG_PCI_IOV
2118 struct pci_dev *pdev = adapter->pdev;
2119
a6b623e0
AD		 2120 if (adapter->vfs_allocated_count) {
2121 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2122 sizeof(struct vf_data_storage),
2123 GFP_KERNEL);
2124 /* if allocation failed then we do not support SR-IOV */
2125 if (!adapter->vf_data) {
2126 adapter->vfs_allocated_count = 0;
2127 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2128 "Data Storage\n");
2129 }
2130 }
2131
2132 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
2133 kfree(adapter->vf_data);
2134 adapter->vf_data = NULL;
2135#endif /* CONFIG_PCI_IOV */
2136 adapter->vfs_allocated_count = 0;
2137#ifdef CONFIG_PCI_IOV
2138 } else {
2139 unsigned char mac_addr[ETH_ALEN];
2140 int i;
2141 dev_info(&pdev->dev, "%d vfs allocated\n",
2142 adapter->vfs_allocated_count);
2143 for (i = 0; i < adapter->vfs_allocated_count; i++) {
2144 random_ether_addr(mac_addr);
2145 igb_set_vf_mac(adapter, i, mac_addr);
2146 }
2147 }
2148#endif /* CONFIG_PCI_IOV */
2149}
2150
115f459a
AD		 2151
2152/**
2153 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2154 * @adapter: board private structure to initialize
2155 *
2156 * igb_init_hw_timer initializes the function pointer and values for the hw
2157 * timer found in hardware.
2158 **/
2159static void igb_init_hw_timer(struct igb_adapter *adapter)
2160{
2161 struct e1000_hw *hw = &adapter->hw;
2162
2163 switch (hw->mac.type) {
d2ba2ed8 2164 case e1000_i350:
55cac248
AD		 2165 case e1000_82580:
2166 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2167 adapter->cycles.read = igb_read_clock;
2168 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2169 adapter->cycles.mult = 1;
2170 /*
2171 * The 82580 timesync updates the system timer every 8ns by 8ns
2172 * and the value cannot be shifted. Instead we need to shift
2173 * the registers to generate a 64bit timer value. As a result
2174 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2175 * 24 in order to generate a larger value for synchronization.
2176 */
2177 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2178 /* disable system timer temporarily by setting bit 31 */
2179 wr32(E1000_TSAUXC, 0x80000000);
2180 wrfl();
2181
2182 /* Set registers so that rollover occurs soon to test this. */
2183 wr32(E1000_SYSTIMR, 0x00000000);
2184 wr32(E1000_SYSTIML, 0x80000000);
2185 wr32(E1000_SYSTIMH, 0x000000FF);
2186 wrfl();
2187
2188 /* enable system timer by clearing bit 31 */
2189 wr32(E1000_TSAUXC, 0x0);
2190 wrfl();
2191
2192 timecounter_init(&adapter->clock,
2193 &adapter->cycles,
2194 ktime_to_ns(ktime_get_real()));
2195 /*
2196 * Synchronize our NIC clock against system wall clock. NIC
2197 * time stamp reading requires ~3us per sample, each sample
2198 * was pretty stable even under load => only require 10
2199 * samples for each offset comparison.
2200 */
2201 memset(&adapter->compare, 0, sizeof(adapter->compare));
2202 adapter->compare.source = &adapter->clock;
2203 adapter->compare.target = ktime_get_real;
2204 adapter->compare.num_samples = 10;
2205 timecompare_update(&adapter->compare, 0);
2206 break;
115f459a
AD		 2207 case e1000_82576:
2208 /*
2209 * Initialize hardware timer: we keep it running just in case
2210 * that some program needs it later on.
2211 */
2212 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2213 adapter->cycles.read = igb_read_clock;
2214 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2215 adapter->cycles.mult = 1;
2216 /**
2217 * Scale the NIC clock cycle by a large factor so that
2218 * relatively small clock corrections can be added or
2219 * substracted at each clock tick. The drawbacks of a large
2220 * factor are a) that the clock register overflows more quickly
2221 * (not such a big deal) and b) that the increment per tick has
2222 * to fit into 24 bits. As a result we need to use a shift of
2223 * 19 so we can fit a value of 16 into the TIMINCA register.
2224 */
2225 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2226 wr32(E1000_TIMINCA,
2227 (1 << E1000_TIMINCA_16NS_SHIFT) |
2228 (16 << IGB_82576_TSYNC_SHIFT));
2229
2230 /* Set registers so that rollover occurs soon to test this. */
2231 wr32(E1000_SYSTIML, 0x00000000);
2232 wr32(E1000_SYSTIMH, 0xFF800000);
2233 wrfl();
2234
2235 timecounter_init(&adapter->clock,
2236 &adapter->cycles,
2237 ktime_to_ns(ktime_get_real()));
2238 /*
2239 * Synchronize our NIC clock against system wall clock. NIC
2240 * time stamp reading requires ~3us per sample, each sample
2241 * was pretty stable even under load => only require 10
2242 * samples for each offset comparison.
2243 */
2244 memset(&adapter->compare, 0, sizeof(adapter->compare));
2245 adapter->compare.source = &adapter->clock;
2246 adapter->compare.target = ktime_get_real;
2247 adapter->compare.num_samples = 10;
2248 timecompare_update(&adapter->compare, 0);
2249 break;
2250 case e1000_82575:
2251 /* 82575 does not support timesync */
2252 default:
2253 break;
2254 }
2255
2256}
2257
9d5c8243
AK		 2258/**
2259 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2260 * @adapter: board private structure to initialize
2261 *
2262 * igb_sw_init initializes the Adapter private data structure.
2263 * Fields are initialized based on PCI device information and
2264 * OS network device settings (MTU size).
2265 **/
2266static int __devinit igb_sw_init(struct igb_adapter *adapter)
2267{
2268 struct e1000_hw *hw = &adapter->hw;
2269 struct net_device *netdev = adapter->netdev;
2270 struct pci_dev *pdev = adapter->pdev;
2271
2272 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2273
68fd9910
AD		 2274 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2275 adapter->rx_ring_count = IGB_DEFAULT_RXD;
4fc82adf
AD		 2276 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2277 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2278
9d5c8243
AK		 2279 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
2280 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2281
12dcd86b 2282 spin_lock_init(&adapter->stats64_lock);
a6b623e0
AD		 2283#ifdef CONFIG_PCI_IOV
2284 if (hw->mac.type == e1000_82576)
c0f2276f 2285 adapter->vfs_allocated_count = (max_vfs > 7) ? 7 : max_vfs;
a6b623e0
AD		 2286
2287#endif /* CONFIG_PCI_IOV */
a99955fc
AD		 2288 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
2289
2290 /*
2291 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2292 * then we should combine the queues into a queue pair in order to
2293 * conserve interrupts due to limited supply
2294 */
2295 if ((adapter->rss_queues > 4) ||
2296 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2297 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2298
a6b623e0 2299 /* This call may decrease the number of queues */
047e0030 2300 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 2301 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2302 return -ENOMEM;
2303 }
2304
115f459a 2305 igb_init_hw_timer(adapter);
a6b623e0
AD		 2306 igb_probe_vfs(adapter);
2307
9d5c8243
AK		 2308 /* Explicitly disable IRQ since the NIC can be in any state. */
2309 igb_irq_disable(adapter);
2310
2311 set_bit(__IGB_DOWN, &adapter->state);
2312 return 0;
2313}
2314
2315/**
2316 * igb_open - Called when a network interface is made active
2317 * @netdev: network interface device structure
2318 *
2319 * Returns 0 on success, negative value on failure
2320 *
2321 * The open entry point is called when a network interface is made
2322 * active by the system (IFF_UP). At this point all resources needed
2323 * for transmit and receive operations are allocated, the interrupt
2324 * handler is registered with the OS, the watchdog timer is started,
2325 * and the stack is notified that the interface is ready.
2326 **/
2327static int igb_open(struct net_device *netdev)
2328{
2329 struct igb_adapter *adapter = netdev_priv(netdev);
2330 struct e1000_hw *hw = &adapter->hw;
2331 int err;
2332 int i;
2333
2334 /* disallow open during test */
2335 if (test_bit(__IGB_TESTING, &adapter->state))
2336 return -EBUSY;
2337
b168dfc5
JB		 2338 netif_carrier_off(netdev);
2339
9d5c8243
AK		 2340 /* allocate transmit descriptors */
2341 err = igb_setup_all_tx_resources(adapter);
2342 if (err)
2343 goto err_setup_tx;
2344
2345 /* allocate receive descriptors */
2346 err = igb_setup_all_rx_resources(adapter);
2347 if (err)
2348 goto err_setup_rx;
2349
88a268c1 2350 igb_power_up_link(adapter);
9d5c8243 2351
9d5c8243
AK		 2352 /* before we allocate an interrupt, we must be ready to handle it.
2353 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2354 * as soon as we call pci_request_irq, so we have to setup our
2355 * clean_rx handler before we do so. */
2356 igb_configure(adapter);
2357
2358 err = igb_request_irq(adapter);
2359 if (err)
2360 goto err_req_irq;
2361
2362 /* From here on the code is the same as igb_up() */
2363 clear_bit(__IGB_DOWN, &adapter->state);
2364
047e0030
AD		 2365 for (i = 0; i < adapter->num_q_vectors; i++) {
2366 struct igb_q_vector *q_vector = adapter->q_vector[i];
2367 napi_enable(&q_vector->napi);
2368 }
9d5c8243
AK		 2369
2370 /* Clear any pending interrupts. */
2371 rd32(E1000_ICR);
844290e5
PW		 2372
2373 igb_irq_enable(adapter);
2374
d4960307
AD		 2375 /* notify VFs that reset has been completed */
2376 if (adapter->vfs_allocated_count) {
2377 u32 reg_data = rd32(E1000_CTRL_EXT);
2378 reg_data |= E1000_CTRL_EXT_PFRSTD;
2379 wr32(E1000_CTRL_EXT, reg_data);
2380 }
2381
d55b53ff
JK		 2382 netif_tx_start_all_queues(netdev);
2383
25568a53
AD		 2384 /* start the watchdog. */
2385 hw->mac.get_link_status = 1;
2386 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2387
2388 return 0;
2389
2390err_req_irq:
2391 igb_release_hw_control(adapter);
88a268c1 2392 igb_power_down_link(adapter);
9d5c8243
AK		 2393 igb_free_all_rx_resources(adapter);
2394err_setup_rx:
2395 igb_free_all_tx_resources(adapter);
2396err_setup_tx:
2397 igb_reset(adapter);
2398
2399 return err;
2400}
2401
2402/**
2403 * igb_close - Disables a network interface
2404 * @netdev: network interface device structure
2405 *
2406 * Returns 0, this is not allowed to fail
2407 *
2408 * The close entry point is called when an interface is de-activated
2409 * by the OS. The hardware is still under the driver's control, but
2410 * needs to be disabled. A global MAC reset is issued to stop the
2411 * hardware, and all transmit and receive resources are freed.
2412 **/
2413static int igb_close(struct net_device *netdev)
2414{
2415 struct igb_adapter *adapter = netdev_priv(netdev);
2416
2417 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2418 igb_down(adapter);
2419
2420 igb_free_irq(adapter);
2421
2422 igb_free_all_tx_resources(adapter);
2423 igb_free_all_rx_resources(adapter);
2424
9d5c8243
AK		 2425 return 0;
2426}
2427
2428/**
2429 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2430 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2431 *
2432 * Return 0 on success, negative on failure
2433 **/
80785298 2434int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2435{
59d71989 2436 struct device *dev = tx_ring->dev;
9d5c8243
AK		 2437 int size;
2438
2439 size = sizeof(struct igb_buffer) * tx_ring->count;
89bf67f1 2440 tx_ring->buffer_info = vzalloc(size);
9d5c8243
AK		 2441 if (!tx_ring->buffer_info)
2442 goto err;
9d5c8243
AK		 2443
2444 /* round up to nearest 4K */
85e8d004 2445 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2446 tx_ring->size = ALIGN(tx_ring->size, 4096);
2447
59d71989
AD		 2448 tx_ring->desc = dma_alloc_coherent(dev,
2449 tx_ring->size,
2450 &tx_ring->dma,
2451 GFP_KERNEL);
9d5c8243
AK		 2452
2453 if (!tx_ring->desc)
2454 goto err;
2455
9d5c8243
AK		 2456 tx_ring->next_to_use = 0;
2457 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2458 return 0;
2459
2460err:
2461 vfree(tx_ring->buffer_info);
59d71989 2462 dev_err(dev,
9d5c8243
AK		 2463 "Unable to allocate memory for the transmit descriptor ring\n");
2464 return -ENOMEM;
2465}
2466
2467/**
2468 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2469 * (Descriptors) for all queues
2470 * @adapter: board private structure
2471 *
2472 * Return 0 on success, negative on failure
2473 **/
2474static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2475{
439705e1 2476 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2477 int i, err = 0;
2478
2479 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2480 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2481 if (err) {
439705e1 2482 dev_err(&pdev->dev,
9d5c8243
AK		 2483 "Allocation for Tx Queue %u failed\n", i);
2484 for (i--; i >= 0; i--)
3025a446 2485 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2486 break;
2487 }
2488 }
2489
a99955fc 2490 for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
439705e1 2491 int r_idx = i % adapter->num_tx_queues;
3025a446 2492 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
eebbbdba 2493 }
9d5c8243
AK		 2494 return err;
2495}
2496
2497/**
85b430b4
AD		 2498 * igb_setup_tctl - configure the transmit control registers
2499 * @adapter: Board private structure
9d5c8243 2500 **/
d7ee5b3a 2501void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2502{
9d5c8243
AK		 2503 struct e1000_hw *hw = &adapter->hw;
2504 u32 tctl;
9d5c8243 2505
85b430b4
AD		 2506 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2507 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2508
2509 /* Program the Transmit Control Register */
9d5c8243
AK		 2510 tctl = rd32(E1000_TCTL);
2511 tctl &= ~E1000_TCTL_CT;
2512 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2513 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2514
2515 igb_config_collision_dist(hw);
2516
9d5c8243
AK		 2517 /* Enable transmits */
2518 tctl |= E1000_TCTL_EN;
2519
2520 wr32(E1000_TCTL, tctl);
2521}
2522
85b430b4
AD		 2523/**
2524 * igb_configure_tx_ring - Configure transmit ring after Reset
2525 * @adapter: board private structure
2526 * @ring: tx ring to configure
2527 *
2528 * Configure a transmit ring after a reset.
2529 **/
d7ee5b3a
AD		 2530void igb_configure_tx_ring(struct igb_adapter *adapter,
2531 struct igb_ring *ring)
85b430b4
AD		 2532{
2533 struct e1000_hw *hw = &adapter->hw;
2534 u32 txdctl;
2535 u64 tdba = ring->dma;
2536 int reg_idx = ring->reg_idx;
2537
2538 /* disable the queue */
2539 txdctl = rd32(E1000_TXDCTL(reg_idx));
2540 wr32(E1000_TXDCTL(reg_idx),
2541 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2542 wrfl();
2543 mdelay(10);
2544
2545 wr32(E1000_TDLEN(reg_idx),
2546 ring->count * sizeof(union e1000_adv_tx_desc));
2547 wr32(E1000_TDBAL(reg_idx),
2548 tdba & 0x00000000ffffffffULL);
2549 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2550
fce99e34
AD		 2551 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2552 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2553 writel(0, ring->head);
2554 writel(0, ring->tail);
85b430b4
AD		 2555
2556 txdctl |= IGB_TX_PTHRESH;
2557 txdctl |= IGB_TX_HTHRESH << 8;
2558 txdctl |= IGB_TX_WTHRESH << 16;
2559
2560 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2561 wr32(E1000_TXDCTL(reg_idx), txdctl);
2562}
2563
2564/**
2565 * igb_configure_tx - Configure transmit Unit after Reset
2566 * @adapter: board private structure
2567 *
2568 * Configure the Tx unit of the MAC after a reset.
2569 **/
2570static void igb_configure_tx(struct igb_adapter *adapter)
2571{
2572 int i;
2573
2574 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2575 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2576}
2577
9d5c8243
AK		 2578/**
2579 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2580 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2581 *
2582 * Returns 0 on success, negative on failure
2583 **/
80785298 2584int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2585{
59d71989 2586 struct device *dev = rx_ring->dev;
9d5c8243
AK		 2587 int size, desc_len;
2588
2589 size = sizeof(struct igb_buffer) * rx_ring->count;
89bf67f1 2590 rx_ring->buffer_info = vzalloc(size);
9d5c8243
AK		 2591 if (!rx_ring->buffer_info)
2592 goto err;
9d5c8243
AK		 2593
2594 desc_len = sizeof(union e1000_adv_rx_desc);
2595
2596 /* Round up to nearest 4K */
2597 rx_ring->size = rx_ring->count * desc_len;
2598 rx_ring->size = ALIGN(rx_ring->size, 4096);
2599
59d71989
AD		 2600 rx_ring->desc = dma_alloc_coherent(dev,
2601 rx_ring->size,
2602 &rx_ring->dma,
2603 GFP_KERNEL);
9d5c8243
AK		 2604
2605 if (!rx_ring->desc)
2606 goto err;
2607
2608 rx_ring->next_to_clean = 0;
2609 rx_ring->next_to_use = 0;
9d5c8243 2610
9d5c8243
AK		 2611 return 0;
2612
2613err:
2614 vfree(rx_ring->buffer_info);
439705e1 2615 rx_ring->buffer_info = NULL;
59d71989
AD		 2616 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2617 " ring\n");
9d5c8243
AK		 2618 return -ENOMEM;
2619}
2620
2621/**
2622 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2623 * (Descriptors) for all queues
2624 * @adapter: board private structure
2625 *
2626 * Return 0 on success, negative on failure
2627 **/
2628static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2629{
439705e1 2630 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2631 int i, err = 0;
2632
2633 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2634 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2635 if (err) {
439705e1 2636 dev_err(&pdev->dev,
9d5c8243
AK		 2637 "Allocation for Rx Queue %u failed\n", i);
2638 for (i--; i >= 0; i--)
3025a446 2639 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2640 break;
2641 }
2642 }
2643
2644 return err;
2645}
2646
06cf2666
AD		 2647/**
2648 * igb_setup_mrqc - configure the multiple receive queue control registers
2649 * @adapter: Board private structure
2650 **/
2651static void igb_setup_mrqc(struct igb_adapter *adapter)
2652{
2653 struct e1000_hw *hw = &adapter->hw;
2654 u32 mrqc, rxcsum;
2655 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2656 union e1000_reta {
2657 u32 dword;
2658 u8 bytes[4];
2659 } reta;
2660 static const u8 rsshash[40] = {
2661 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2662 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2663 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2664 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2665
2666 /* Fill out hash function seeds */
2667 for (j = 0; j < 10; j++) {
2668 u32 rsskey = rsshash[(j * 4)];
2669 rsskey |= rsshash[(j * 4) + 1] << 8;
2670 rsskey |= rsshash[(j * 4) + 2] << 16;
2671 rsskey |= rsshash[(j * 4) + 3] << 24;
2672 array_wr32(E1000_RSSRK(0), j, rsskey);
2673 }
2674
a99955fc 2675 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2676
2677 if (adapter->vfs_allocated_count) {
2678 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2679 switch (hw->mac.type) {
d2ba2ed8 2680 case e1000_i350:
55cac248
AD		 2681 case e1000_82580:
2682 num_rx_queues = 1;
2683 shift = 0;
2684 break;
06cf2666
AD		 2685 case e1000_82576:
2686 shift = 3;
2687 num_rx_queues = 2;
2688 break;
2689 case e1000_82575:
2690 shift = 2;
2691 shift2 = 6;
2692 default:
2693 break;
2694 }
2695 } else {
2696 if (hw->mac.type == e1000_82575)
2697 shift = 6;
2698 }
2699
2700 for (j = 0; j < (32 * 4); j++) {
2701 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2702 if (shift2)
2703 reta.bytes[j & 3] |= num_rx_queues << shift2;
2704 if ((j & 3) == 3)
2705 wr32(E1000_RETA(j >> 2), reta.dword);
2706 }
2707
2708 /*
2709 * Disable raw packet checksumming so that RSS hash is placed in
2710 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2711 * offloads as they are enabled by default
2712 */
2713 rxcsum = rd32(E1000_RXCSUM);
2714 rxcsum |= E1000_RXCSUM_PCSD;
2715
2716 if (adapter->hw.mac.type >= e1000_82576)
2717 /* Enable Receive Checksum Offload for SCTP */
2718 rxcsum |= E1000_RXCSUM_CRCOFL;
2719
2720 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2721 wr32(E1000_RXCSUM, rxcsum);
2722
2723 /* If VMDq is enabled then we set the appropriate mode for that, else
2724 * we default to RSS so that an RSS hash is calculated per packet even
2725 * if we are only using one queue */
2726 if (adapter->vfs_allocated_count) {
2727 if (hw->mac.type > e1000_82575) {
2728 /* Set the default pool for the PF's first queue */
2729 u32 vtctl = rd32(E1000_VT_CTL);
2730 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2731 E1000_VT_CTL_DISABLE_DEF_POOL);
2732 vtctl |= adapter->vfs_allocated_count <<
2733 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2734 wr32(E1000_VT_CTL, vtctl);
2735 }
a99955fc 2736 if (adapter->rss_queues > 1)
06cf2666
AD		 2737 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2738 else
2739 mrqc = E1000_MRQC_ENABLE_VMDQ;
2740 } else {
2741 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2742 }
2743 igb_vmm_control(adapter);
2744
4478a9cd
AD		 2745 /*
2746 * Generate RSS hash based on TCP port numbers and/or
2747 * IPv4/v6 src and dst addresses since UDP cannot be
2748 * hashed reliably due to IP fragmentation
2749 */
2750 mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
2751 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2752 E1000_MRQC_RSS_FIELD_IPV6 |
2753 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2754 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
06cf2666
AD		 2755
2756 wr32(E1000_MRQC, mrqc);
2757}
2758
9d5c8243
AK		 2759/**
2760 * igb_setup_rctl - configure the receive control registers
2761 * @adapter: Board private structure
2762 **/
d7ee5b3a 2763void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2764{
2765 struct e1000_hw *hw = &adapter->hw;
2766 u32 rctl;
9d5c8243
AK		 2767
2768 rctl = rd32(E1000_RCTL);
2769
2770 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2771 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2772
69d728ba 2773 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2774 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2775
87cb7e8c
AK		 2776 /*
2777 * enable stripping of CRC. It's unlikely this will break BMC
2778 * redirection as it did with e1000. Newer features require
2779 * that the HW strips the CRC.
73cd78f1 2780 */
87cb7e8c 2781 rctl |= E1000_RCTL_SECRC;
9d5c8243 2782
559e9c49 2783 /* disable store bad packets and clear size bits. */
ec54d7d6 2784 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2785
6ec43fe6
AD		 2786 /* enable LPE to prevent packets larger than max_frame_size */
2787 rctl |= E1000_RCTL_LPE;
9d5c8243 2788
952f72a8
AD		 2789 /* disable queue 0 to prevent tail write w/o re-config */
2790 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2791
e1739522
AD		 2792 /* Attention!!! For SR-IOV PF driver operations you must enable
2793 * queue drop for all VF and PF queues to prevent head of line blocking
2794 * if an un-trusted VF does not provide descriptors to hardware.
2795 */
2796 if (adapter->vfs_allocated_count) {
e1739522
AD		 2797 /* set all queue drop enable bits */
2798 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 2799 }
2800
9d5c8243
AK		 2801 wr32(E1000_RCTL, rctl);
2802}
2803
7d5753f0
AD		 2804static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2805 int vfn)
2806{
2807 struct e1000_hw *hw = &adapter->hw;
2808 u32 vmolr;
2809
2810 /* if it isn't the PF check to see if VFs are enabled and
2811 * increase the size to support vlan tags */
2812 if (vfn < adapter->vfs_allocated_count &&
2813 adapter->vf_data[vfn].vlans_enabled)
2814 size += VLAN_TAG_SIZE;
2815
2816 vmolr = rd32(E1000_VMOLR(vfn));
2817 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2818 vmolr |= size | E1000_VMOLR_LPE;
2819 wr32(E1000_VMOLR(vfn), vmolr);
2820
2821 return 0;
2822}
2823
e1739522
AD		 2824/**
2825 * igb_rlpml_set - set maximum receive packet size
2826 * @adapter: board private structure
2827 *
2828 * Configure maximum receivable packet size.
2829 **/
2830static void igb_rlpml_set(struct igb_adapter *adapter)
2831{
2832 u32 max_frame_size = adapter->max_frame_size;
2833 struct e1000_hw *hw = &adapter->hw;
2834 u16 pf_id = adapter->vfs_allocated_count;
2835
2836 if (adapter->vlgrp)
2837 max_frame_size += VLAN_TAG_SIZE;
2838
2839 /* if vfs are enabled we set RLPML to the largest possible request
2840 * size and set the VMOLR RLPML to the size we need */
2841 if (pf_id) {
2842 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
7d5753f0 2843 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 2844 }
2845
2846 wr32(E1000_RLPML, max_frame_size);
2847}
2848
8151d294
WM		 2849static inline void igb_set_vmolr(struct igb_adapter *adapter,
2850 int vfn, bool aupe)
7d5753f0
AD		 2851{
2852 struct e1000_hw *hw = &adapter->hw;
2853 u32 vmolr;
2854
2855 /*
2856 * This register exists only on 82576 and newer so if we are older then
2857 * we should exit and do nothing
2858 */
2859 if (hw->mac.type < e1000_82576)
2860 return;
2861
2862 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 2863 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2864 if (aupe)
2865 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
2866 else
2867 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 2868
2869 /* clear all bits that might not be set */
2870 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2871
a99955fc 2872 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 2873 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2874 /*
2875 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2876 * multicast packets
2877 */
2878 if (vfn <= adapter->vfs_allocated_count)
2879 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2880
2881 wr32(E1000_VMOLR(vfn), vmolr);
2882}
2883
85b430b4
AD		 2884/**
2885 * igb_configure_rx_ring - Configure a receive ring after Reset
2886 * @adapter: board private structure
2887 * @ring: receive ring to be configured
2888 *
2889 * Configure the Rx unit of the MAC after a reset.
2890 **/
d7ee5b3a
AD		 2891void igb_configure_rx_ring(struct igb_adapter *adapter,
2892 struct igb_ring *ring)
85b430b4
AD		 2893{
2894 struct e1000_hw *hw = &adapter->hw;
2895 u64 rdba = ring->dma;
2896 int reg_idx = ring->reg_idx;
952f72a8 2897 u32 srrctl, rxdctl;
85b430b4
AD		 2898
2899 /* disable the queue */
2900 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2901 wr32(E1000_RXDCTL(reg_idx),
2902 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2903
2904 /* Set DMA base address registers */
2905 wr32(E1000_RDBAL(reg_idx),
2906 rdba & 0x00000000ffffffffULL);
2907 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2908 wr32(E1000_RDLEN(reg_idx),
2909 ring->count * sizeof(union e1000_adv_rx_desc));
2910
2911 /* initialize head and tail */
fce99e34
AD		 2912 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2913 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2914 writel(0, ring->head);
2915 writel(0, ring->tail);
85b430b4 2916
952f72a8 2917 /* set descriptor configuration */
4c844851
AD		 2918 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2919 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
952f72a8
AD		 2920 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2921#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2922 srrctl |= IGB_RXBUFFER_16384 >>
2923 E1000_SRRCTL_BSIZEPKT_SHIFT;
2924#else
2925 srrctl |= (PAGE_SIZE / 2) >>
2926 E1000_SRRCTL_BSIZEPKT_SHIFT;
2927#endif
2928 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2929 } else {
4c844851 2930 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
952f72a8
AD		 2931 E1000_SRRCTL_BSIZEPKT_SHIFT;
2932 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2933 }
757b77e2
NN		 2934 if (hw->mac.type == e1000_82580)
2935 srrctl |= E1000_SRRCTL_TIMESTAMP;
e6bdb6fe
NN		 2936 /* Only set Drop Enable if we are supporting multiple queues */
2937 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
2938 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 2939
2940 wr32(E1000_SRRCTL(reg_idx), srrctl);
2941
7d5753f0 2942 /* set filtering for VMDQ pools */
8151d294 2943 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 2944
85b430b4
AD		 2945 /* enable receive descriptor fetching */
2946 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2947 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2948 rxdctl &= 0xFFF00000;
2949 rxdctl |= IGB_RX_PTHRESH;
2950 rxdctl |= IGB_RX_HTHRESH << 8;
2951 rxdctl |= IGB_RX_WTHRESH << 16;
2952 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2953}
2954
9d5c8243
AK		 2955/**
2956 * igb_configure_rx - Configure receive Unit after Reset
2957 * @adapter: board private structure
2958 *
2959 * Configure the Rx unit of the MAC after a reset.
2960 **/
2961static void igb_configure_rx(struct igb_adapter *adapter)
2962{
9107584e 2963 int i;
9d5c8243 2964
68d480c4
AD		 2965 /* set UTA to appropriate mode */
2966 igb_set_uta(adapter);
2967
26ad9178
AD		 2968 /* set the correct pool for the PF default MAC address in entry 0 */
2969 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2970 adapter->vfs_allocated_count);
2971
06cf2666
AD		 2972 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2973 * the Base and Length of the Rx Descriptor Ring */
2974 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2975 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 2976}
2977
2978/**
2979 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 2980 * @tx_ring: Tx descriptor ring for a specific queue
2981 *
2982 * Free all transmit software resources
2983 **/
68fd9910 2984void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2985{
3b644cf6 2986 igb_clean_tx_ring(tx_ring);
9d5c8243
AK		 2987
2988 vfree(tx_ring->buffer_info);
2989 tx_ring->buffer_info = NULL;
2990
439705e1
AD		 2991 /* if not set, then don't free */
2992 if (!tx_ring->desc)
2993 return;
2994
59d71989
AD		 2995 dma_free_coherent(tx_ring->dev, tx_ring->size,
2996 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 2997
2998 tx_ring->desc = NULL;
2999}
3000
3001/**
3002 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3003 * @adapter: board private structure
3004 *
3005 * Free all transmit software resources
3006 **/
3007static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3008{
3009 int i;
3010
3011 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3012 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 3013}
3014
b1a436c3
AD		 3015void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
3016 struct igb_buffer *buffer_info)
9d5c8243 3017{
6366ad33
AD		 3018 if (buffer_info->dma) {
3019 if (buffer_info->mapped_as_page)
59d71989 3020 dma_unmap_page(tx_ring->dev,
6366ad33
AD		 3021 buffer_info->dma,
3022 buffer_info->length,
59d71989 3023 DMA_TO_DEVICE);
6366ad33 3024 else
59d71989 3025 dma_unmap_single(tx_ring->dev,
6366ad33
AD		 3026 buffer_info->dma,
3027 buffer_info->length,
59d71989 3028 DMA_TO_DEVICE);
6366ad33
AD		 3029 buffer_info->dma = 0;
3030 }
9d5c8243
AK		 3031 if (buffer_info->skb) {
3032 dev_kfree_skb_any(buffer_info->skb);
3033 buffer_info->skb = NULL;
3034 }
3035 buffer_info->time_stamp = 0;
6366ad33
AD		 3036 buffer_info->length = 0;
3037 buffer_info->next_to_watch = 0;
3038 buffer_info->mapped_as_page = false;
9d5c8243
AK		 3039}
3040
3041/**
3042 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 3043 * @tx_ring: ring to be cleaned
3044 **/
3b644cf6 3045static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243
AK		 3046{
3047 struct igb_buffer *buffer_info;
3048 unsigned long size;
3049 unsigned int i;
3050
3051 if (!tx_ring->buffer_info)
3052 return;
3053 /* Free all the Tx ring sk_buffs */
3054
3055 for (i = 0; i < tx_ring->count; i++) {
3056 buffer_info = &tx_ring->buffer_info[i];
80785298 3057 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 3058 }
3059
3060 size = sizeof(struct igb_buffer) * tx_ring->count;
3061 memset(tx_ring->buffer_info, 0, size);
3062
3063 /* Zero out the descriptor ring */
9d5c8243
AK		 3064 memset(tx_ring->desc, 0, tx_ring->size);
3065
3066 tx_ring->next_to_use = 0;
3067 tx_ring->next_to_clean = 0;
9d5c8243
AK		 3068}
3069
3070/**
3071 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3072 * @adapter: board private structure
3073 **/
3074static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3075{
3076 int i;
3077
3078 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3079 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 3080}
3081
3082/**
3083 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 3084 * @rx_ring: ring to clean the resources from
3085 *
3086 * Free all receive software resources
3087 **/
68fd9910 3088void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 3089{
3b644cf6 3090 igb_clean_rx_ring(rx_ring);
9d5c8243
AK		 3091
3092 vfree(rx_ring->buffer_info);
3093 rx_ring->buffer_info = NULL;
3094
439705e1
AD		 3095 /* if not set, then don't free */
3096 if (!rx_ring->desc)
3097 return;
3098
59d71989
AD		 3099 dma_free_coherent(rx_ring->dev, rx_ring->size,
3100 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 3101
3102 rx_ring->desc = NULL;
3103}
3104
3105/**
3106 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3107 * @adapter: board private structure
3108 *
3109 * Free all receive software resources
3110 **/
3111static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3112{
3113 int i;
3114
3115 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3116 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 3117}
3118
3119/**
3120 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 3121 * @rx_ring: ring to free buffers from
3122 **/
3b644cf6 3123static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243
AK		 3124{
3125 struct igb_buffer *buffer_info;
9d5c8243
AK		 3126 unsigned long size;
3127 unsigned int i;
3128
3129 if (!rx_ring->buffer_info)
3130 return;
439705e1 3131
9d5c8243
AK		 3132 /* Free all the Rx ring sk_buffs */
3133 for (i = 0; i < rx_ring->count; i++) {
3134 buffer_info = &rx_ring->buffer_info[i];
3135 if (buffer_info->dma) {
59d71989 3136 dma_unmap_single(rx_ring->dev,
80785298 3137 buffer_info->dma,
4c844851 3138 rx_ring->rx_buffer_len,
59d71989 3139 DMA_FROM_DEVICE);
9d5c8243
AK		 3140 buffer_info->dma = 0;
3141 }
3142
3143 if (buffer_info->skb) {
3144 dev_kfree_skb(buffer_info->skb);
3145 buffer_info->skb = NULL;
3146 }
6ec43fe6 3147 if (buffer_info->page_dma) {
59d71989 3148 dma_unmap_page(rx_ring->dev,
80785298 3149 buffer_info->page_dma,
6ec43fe6 3150 PAGE_SIZE / 2,
59d71989 3151 DMA_FROM_DEVICE);
6ec43fe6
AD		 3152 buffer_info->page_dma = 0;
3153 }
9d5c8243 3154 if (buffer_info->page) {
9d5c8243
AK		 3155 put_page(buffer_info->page);
3156 buffer_info->page = NULL;
bf36c1a0 3157 buffer_info->page_offset = 0;
9d5c8243
AK		 3158 }
3159 }
3160
9d5c8243
AK		 3161 size = sizeof(struct igb_buffer) * rx_ring->count;
3162 memset(rx_ring->buffer_info, 0, size);
3163
3164 /* Zero out the descriptor ring */
3165 memset(rx_ring->desc, 0, rx_ring->size);
3166
3167 rx_ring->next_to_clean = 0;
3168 rx_ring->next_to_use = 0;
9d5c8243
AK		 3169}
3170
3171/**
3172 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3173 * @adapter: board private structure
3174 **/
3175static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3176{
3177 int i;
3178
3179 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3180 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 3181}
3182
3183/**
3184 * igb_set_mac - Change the Ethernet Address of the NIC
3185 * @netdev: network interface device structure
3186 * @p: pointer to an address structure
3187 *
3188 * Returns 0 on success, negative on failure
3189 **/
3190static int igb_set_mac(struct net_device *netdev, void *p)
3191{
3192 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 3193 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 3194 struct sockaddr *addr = p;
3195
3196 if (!is_valid_ether_addr(addr->sa_data))
3197 return -EADDRNOTAVAIL;
3198
3199 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 3200 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 3201
26ad9178
AD		 3202 /* set the correct pool for the new PF MAC address in entry 0 */
3203 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3204 adapter->vfs_allocated_count);
e1739522 3205
9d5c8243
AK		 3206 return 0;
3207}
3208
3209/**
68d480c4 3210 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 3211 * @netdev: network interface device structure
3212 *
68d480c4
AD		 3213 * Writes multicast address list to the MTA hash table.
3214 * Returns: -ENOMEM on failure
3215 * 0 on no addresses written
3216 * X on writing X addresses to MTA
9d5c8243 3217 **/
68d480c4 3218static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 3219{
3220 struct igb_adapter *adapter = netdev_priv(netdev);
3221 struct e1000_hw *hw = &adapter->hw;
22bedad3 3222 struct netdev_hw_addr *ha;
68d480c4 3223 u8 *mta_list;
9d5c8243
AK		 3224 int i;
3225
4cd24eaf 3226 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 3227 /* nothing to program, so clear mc list */
3228 igb_update_mc_addr_list(hw, NULL, 0);
3229 igb_restore_vf_multicasts(adapter);
3230 return 0;
3231 }
9d5c8243 3232
4cd24eaf 3233 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 3234 if (!mta_list)
3235 return -ENOMEM;
ff41f8dc 3236
68d480c4 3237 /* The shared function expects a packed array of only addresses. */
48e2f183 3238 i = 0;
22bedad3
JP		 3239 netdev_for_each_mc_addr(ha, netdev)
3240 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
68d480c4 3241
68d480c4
AD		 3242 igb_update_mc_addr_list(hw, mta_list, i);
3243 kfree(mta_list);
3244
4cd24eaf 3245 return netdev_mc_count(netdev);
68d480c4
AD		 3246}
3247
3248/**
3249 * igb_write_uc_addr_list - write unicast addresses to RAR table
3250 * @netdev: network interface device structure
3251 *
3252 * Writes unicast address list to the RAR table.
3253 * Returns: -ENOMEM on failure/insufficient address space
3254 * 0 on no addresses written
3255 * X on writing X addresses to the RAR table
3256 **/
3257static int igb_write_uc_addr_list(struct net_device *netdev)
3258{
3259 struct igb_adapter *adapter = netdev_priv(netdev);
3260 struct e1000_hw *hw = &adapter->hw;
3261 unsigned int vfn = adapter->vfs_allocated_count;
3262 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3263 int count = 0;
3264
3265 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 3266 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 3267 return -ENOMEM;
9d5c8243 3268
32e7bfc4 3269 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 3270 struct netdev_hw_addr *ha;
32e7bfc4
JP		 3271
3272 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 3273 if (!rar_entries)
3274 break;
26ad9178
AD		 3275 igb_rar_set_qsel(adapter, ha->addr,
3276 rar_entries--,
68d480c4
AD		 3277 vfn);
3278 count++;
ff41f8dc
AD		 3279 }
3280 }
3281 /* write the addresses in reverse order to avoid write combining */
3282 for (; rar_entries > 0 ; rar_entries--) {
3283 wr32(E1000_RAH(rar_entries), 0);
3284 wr32(E1000_RAL(rar_entries), 0);
3285 }
3286 wrfl();
3287
68d480c4
AD		 3288 return count;
3289}
3290
3291/**
3292 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3293 * @netdev: network interface device structure
3294 *
3295 * The set_rx_mode entry point is called whenever the unicast or multicast
3296 * address lists or the network interface flags are updated. This routine is
3297 * responsible for configuring the hardware for proper unicast, multicast,
3298 * promiscuous mode, and all-multi behavior.
3299 **/
3300static void igb_set_rx_mode(struct net_device *netdev)
3301{
3302 struct igb_adapter *adapter = netdev_priv(netdev);
3303 struct e1000_hw *hw = &adapter->hw;
3304 unsigned int vfn = adapter->vfs_allocated_count;
3305 u32 rctl, vmolr = 0;
3306 int count;
3307
3308 /* Check for Promiscuous and All Multicast modes */
3309 rctl = rd32(E1000_RCTL);
3310
3311 /* clear the effected bits */
3312 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3313
3314 if (netdev->flags & IFF_PROMISC) {
3315 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3316 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3317 } else {
3318 if (netdev->flags & IFF_ALLMULTI) {
3319 rctl |= E1000_RCTL_MPE;
3320 vmolr |= E1000_VMOLR_MPME;
3321 } else {
3322 /*
3323 * Write addresses to the MTA, if the attempt fails
3324 * then we should just turn on promiscous mode so
3325 * that we can at least receive multicast traffic
3326 */
3327 count = igb_write_mc_addr_list(netdev);
3328 if (count < 0) {
3329 rctl |= E1000_RCTL_MPE;
3330 vmolr |= E1000_VMOLR_MPME;
3331 } else if (count) {
3332 vmolr |= E1000_VMOLR_ROMPE;
3333 }
3334 }
3335 /*
3336 * Write addresses to available RAR registers, if there is not
3337 * sufficient space to store all the addresses then enable
3338 * unicast promiscous mode
3339 */
3340 count = igb_write_uc_addr_list(netdev);
3341 if (count < 0) {
3342 rctl |= E1000_RCTL_UPE;
3343 vmolr |= E1000_VMOLR_ROPE;
3344 }
3345 rctl |= E1000_RCTL_VFE;
28fc06f5 3346 }
68d480c4 3347 wr32(E1000_RCTL, rctl);
28fc06f5 3348
68d480c4
AD		 3349 /*
3350 * In order to support SR-IOV and eventually VMDq it is necessary to set
3351 * the VMOLR to enable the appropriate modes. Without this workaround
3352 * we will have issues with VLAN tag stripping not being done for frames
3353 * that are only arriving because we are the default pool
3354 */
3355 if (hw->mac.type < e1000_82576)
28fc06f5 3356 return;
9d5c8243 3357
68d480c4
AD		 3358 vmolr |= rd32(E1000_VMOLR(vfn)) &
3359 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3360 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3361 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3362}
3363
3364/* Need to wait a few seconds after link up to get diagnostic information from
3365 * the phy */
3366static void igb_update_phy_info(unsigned long data)
3367{
3368 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3369 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3370}
3371
4d6b725e
AD		 3372/**
3373 * igb_has_link - check shared code for link and determine up/down
3374 * @adapter: pointer to driver private info
3375 **/
3145535a 3376bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3377{
3378 struct e1000_hw *hw = &adapter->hw;
3379 bool link_active = false;
3380 s32 ret_val = 0;
3381
3382 /* get_link_status is set on LSC (link status) interrupt or
3383 * rx sequence error interrupt. get_link_status will stay
3384 * false until the e1000_check_for_link establishes link
3385 * for copper adapters ONLY
3386 */
3387 switch (hw->phy.media_type) {
3388 case e1000_media_type_copper:
3389 if (hw->mac.get_link_status) {
3390 ret_val = hw->mac.ops.check_for_link(hw);
3391 link_active = !hw->mac.get_link_status;
3392 } else {
3393 link_active = true;
3394 }
3395 break;
4d6b725e
AD		 3396 case e1000_media_type_internal_serdes:
3397 ret_val = hw->mac.ops.check_for_link(hw);
3398 link_active = hw->mac.serdes_has_link;
3399 break;
3400 default:
3401 case e1000_media_type_unknown:
3402 break;
3403 }
3404
3405 return link_active;
3406}
3407
9d5c8243
AK		 3408/**
3409 * igb_watchdog - Timer Call-back
3410 * @data: pointer to adapter cast into an unsigned long
3411 **/
3412static void igb_watchdog(unsigned long data)
3413{
3414 struct igb_adapter *adapter = (struct igb_adapter *)data;
3415 /* Do the rest outside of interrupt context */
3416 schedule_work(&adapter->watchdog_task);
3417}
3418
3419static void igb_watchdog_task(struct work_struct *work)
3420{
3421 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3422 struct igb_adapter,
3423 watchdog_task);
9d5c8243 3424 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3425 struct net_device *netdev = adapter->netdev;
9d5c8243 3426 u32 link;
7a6ea550 3427 int i;
9d5c8243 3428
4d6b725e 3429 link = igb_has_link(adapter);
9d5c8243
AK		 3430 if (link) {
3431 if (!netif_carrier_ok(netdev)) {
3432 u32 ctrl;
330a6d6a
AD		 3433 hw->mac.ops.get_speed_and_duplex(hw,
3434 &adapter->link_speed,
3435 &adapter->link_duplex);
9d5c8243
AK		 3436
3437 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 3438 /* Links status message must follow this format */
3439 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 3440 "Flow Control: %s\n",
559e9c49
AD		 3441 netdev->name,
3442 adapter->link_speed,
3443 adapter->link_duplex == FULL_DUPLEX ?
9d5c8243 3444 "Full Duplex" : "Half Duplex",
559e9c49
AD		 3445 ((ctrl & E1000_CTRL_TFCE) &&
3446 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3447 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3448 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
9d5c8243 3449
d07f3e37 3450 /* adjust timeout factor according to speed/duplex */
9d5c8243
AK		 3451 adapter->tx_timeout_factor = 1;
3452 switch (adapter->link_speed) {
3453 case SPEED_10:
9d5c8243
AK		 3454 adapter->tx_timeout_factor = 14;
3455 break;
3456 case SPEED_100:
9d5c8243
AK		 3457 /* maybe add some timeout factor ? */
3458 break;
3459 }
3460
3461 netif_carrier_on(netdev);
9d5c8243 3462
4ae196df
AD		 3463 igb_ping_all_vfs(adapter);
3464
4b1a9877 3465 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3466 if (!test_bit(__IGB_DOWN, &adapter->state))
3467 mod_timer(&adapter->phy_info_timer,
3468 round_jiffies(jiffies + 2 * HZ));
3469 }
3470 } else {
3471 if (netif_carrier_ok(netdev)) {
3472 adapter->link_speed = 0;
3473 adapter->link_duplex = 0;
527d47c1
AD		 3474 /* Links status message must follow this format */
3475 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3476 netdev->name);
9d5c8243 3477 netif_carrier_off(netdev);
4b1a9877 3478
4ae196df
AD		 3479 igb_ping_all_vfs(adapter);
3480
4b1a9877 3481 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3482 if (!test_bit(__IGB_DOWN, &adapter->state))
3483 mod_timer(&adapter->phy_info_timer,
3484 round_jiffies(jiffies + 2 * HZ));
3485 }
3486 }
3487
12dcd86b
ED		 3488 spin_lock(&adapter->stats64_lock);
3489 igb_update_stats(adapter, &adapter->stats64);
3490 spin_unlock(&adapter->stats64_lock);
9d5c8243 3491
dbabb065 3492 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3493 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3494 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3495 /* We've lost link, so the controller stops DMA,
3496 * but we've got queued Tx work that's never going
3497 * to get done, so reset controller to flush Tx.
3498 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3499 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3500 adapter->tx_timeout_count++;
3501 schedule_work(&adapter->reset_task);
3502 /* return immediately since reset is imminent */
3503 return;
3504 }
9d5c8243 3505 }
9d5c8243 3506
dbabb065
AD		 3507 /* Force detection of hung controller every watchdog period */
3508 tx_ring->detect_tx_hung = true;
3509 }
f7ba205e 3510
9d5c8243 3511 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3512 if (adapter->msix_entries) {
047e0030
AD		 3513 u32 eics = 0;
3514 for (i = 0; i < adapter->num_q_vectors; i++) {
3515 struct igb_q_vector *q_vector = adapter->q_vector[i];
3516 eics |= q_vector->eims_value;
3517 }
7a6ea550
AD		 3518 wr32(E1000_EICS, eics);
3519 } else {
3520 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3521 }
9d5c8243 3522
9d5c8243
AK		 3523 /* Reset the timer */
3524 if (!test_bit(__IGB_DOWN, &adapter->state))
3525 mod_timer(&adapter->watchdog_timer,
3526 round_jiffies(jiffies + 2 * HZ));
3527}
3528
3529enum latency_range {
3530 lowest_latency = 0,
3531 low_latency = 1,
3532 bulk_latency = 2,
3533 latency_invalid = 255
3534};
3535
6eb5a7f1
AD		 3536/**
3537 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3538 *
3539 * Stores a new ITR value based on strictly on packet size. This
3540 * algorithm is less sophisticated than that used in igb_update_itr,
3541 * due to the difficulty of synchronizing statistics across multiple
eef35c2d 3542 * receive rings. The divisors and thresholds used by this function
6eb5a7f1
AD		 3543 * were determined based on theoretical maximum wire speed and testing
3544 * data, in order to minimize response time while increasing bulk
3545 * throughput.
3546 * This functionality is controlled by the InterruptThrottleRate module
3547 * parameter (see igb_param.c)
3548 * NOTE: This function is called only when operating in a multiqueue
3549 * receive environment.
047e0030 3550 * @q_vector: pointer to q_vector
6eb5a7f1 3551 **/
047e0030 3552static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3553{
047e0030 3554 int new_val = q_vector->itr_val;
6eb5a7f1 3555 int avg_wire_size = 0;
047e0030 3556 struct igb_adapter *adapter = q_vector->adapter;
12dcd86b
ED		 3557 struct igb_ring *ring;
3558 unsigned int packets;
9d5c8243 3559
6eb5a7f1
AD		 3560 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3561 * ints/sec - ITR timer value of 120 ticks.
3562 */
3563 if (adapter->link_speed != SPEED_1000) {
047e0030 3564 new_val = 976;
6eb5a7f1 3565 goto set_itr_val;
9d5c8243 3566 }
047e0030 3567
12dcd86b
ED		 3568 ring = q_vector->rx_ring;
3569 if (ring) {
3570 packets = ACCESS_ONCE(ring->total_packets);
3571
3572 if (packets)
3573 avg_wire_size = ring->total_bytes / packets;
047e0030
AD		 3574 }
3575
12dcd86b
ED		 3576 ring = q_vector->tx_ring;
3577 if (ring) {
3578 packets = ACCESS_ONCE(ring->total_packets);
3579
3580 if (packets)
3581 avg_wire_size = max_t(u32, avg_wire_size,
3582 ring->total_bytes / packets);
047e0030
AD		 3583 }
3584
3585 /* if avg_wire_size isn't set no work was done */
3586 if (!avg_wire_size)
3587 goto clear_counts;
9d5c8243 3588
6eb5a7f1
AD		 3589 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3590 avg_wire_size += 24;
3591
3592 /* Don't starve jumbo frames */
3593 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3594
6eb5a7f1
AD		 3595 /* Give a little boost to mid-size frames */
3596 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3597 new_val = avg_wire_size / 3;
3598 else
3599 new_val = avg_wire_size / 2;
9d5c8243 3600
abe1c363
NN		 3601 /* when in itr mode 3 do not exceed 20K ints/sec */
3602 if (adapter->rx_itr_setting == 3 && new_val < 196)
3603 new_val = 196;
3604
6eb5a7f1 3605set_itr_val:
047e0030
AD		 3606 if (new_val != q_vector->itr_val) {
3607 q_vector->itr_val = new_val;
3608 q_vector->set_itr = 1;
9d5c8243 3609 }
6eb5a7f1 3610clear_counts:
047e0030
AD		 3611 if (q_vector->rx_ring) {
3612 q_vector->rx_ring->total_bytes = 0;
3613 q_vector->rx_ring->total_packets = 0;
3614 }
3615 if (q_vector->tx_ring) {
3616 q_vector->tx_ring->total_bytes = 0;
3617 q_vector->tx_ring->total_packets = 0;
3618 }
9d5c8243
AK		 3619}
3620
3621/**
3622 * igb_update_itr - update the dynamic ITR value based on statistics
3623 * Stores a new ITR value based on packets and byte
3624 * counts during the last interrupt. The advantage of per interrupt
3625 * computation is faster updates and more accurate ITR for the current
3626 * traffic pattern. Constants in this function were computed
3627 * based on theoretical maximum wire speed and thresholds were set based
3628 * on testing data as well as attempting to minimize response time
3629 * while increasing bulk throughput.
3630 * this functionality is controlled by the InterruptThrottleRate module
3631 * parameter (see igb_param.c)
3632 * NOTE: These calculations are only valid when operating in a single-
3633 * queue environment.
3634 * @adapter: pointer to adapter
047e0030 3635 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3636 * @packets: the number of packets during this measurement interval
3637 * @bytes: the number of bytes during this measurement interval
3638 **/
3639static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3640 int packets, int bytes)
3641{
3642 unsigned int retval = itr_setting;
3643
3644 if (packets == 0)
3645 goto update_itr_done;
3646
3647 switch (itr_setting) {
3648 case lowest_latency:
3649 /* handle TSO and jumbo frames */
3650 if (bytes/packets > 8000)
3651 retval = bulk_latency;
3652 else if ((packets < 5) && (bytes > 512))
3653 retval = low_latency;
3654 break;
3655 case low_latency: /* 50 usec aka 20000 ints/s */
3656 if (bytes > 10000) {
3657 /* this if handles the TSO accounting */
3658 if (bytes/packets > 8000) {
3659 retval = bulk_latency;
3660 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3661 retval = bulk_latency;
3662 } else if ((packets > 35)) {
3663 retval = lowest_latency;
3664 }
3665 } else if (bytes/packets > 2000) {
3666 retval = bulk_latency;
3667 } else if (packets <= 2 && bytes < 512) {
3668 retval = lowest_latency;
3669 }
3670 break;
3671 case bulk_latency: /* 250 usec aka 4000 ints/s */
3672 if (bytes > 25000) {
3673 if (packets > 35)
3674 retval = low_latency;
1e5c3d21 3675 } else if (bytes < 1500) {
9d5c8243
AK		 3676 retval = low_latency;
3677 }
3678 break;
3679 }
3680
3681update_itr_done:
3682 return retval;
3683}
3684
6eb5a7f1 3685static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3686{
047e0030 3687 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3688 u16 current_itr;
047e0030 3689 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3690
3691 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3692 if (adapter->link_speed != SPEED_1000) {
3693 current_itr = 0;
3694 new_itr = 4000;
3695 goto set_itr_now;
3696 }
3697
3698 adapter->rx_itr = igb_update_itr(adapter,
3699 adapter->rx_itr,
3025a446
AD		 3700 q_vector->rx_ring->total_packets,
3701 q_vector->rx_ring->total_bytes);
9d5c8243 3702
047e0030
AD		 3703 adapter->tx_itr = igb_update_itr(adapter,
3704 adapter->tx_itr,
3025a446
AD		 3705 q_vector->tx_ring->total_packets,
3706 q_vector->tx_ring->total_bytes);
047e0030 3707 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3708
6eb5a7f1 3709 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4fc82adf 3710 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3711 current_itr = low_latency;
3712
9d5c8243
AK		 3713 switch (current_itr) {
3714 /* counts and packets in update_itr are dependent on these numbers */
3715 case lowest_latency:
78b1f607 3716 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3717 break;
3718 case low_latency:
78b1f607 3719 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3720 break;
3721 case bulk_latency:
78b1f607 3722 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3723 break;
3724 default:
3725 break;
3726 }
3727
3728set_itr_now:
3025a446
AD		 3729 q_vector->rx_ring->total_bytes = 0;
3730 q_vector->rx_ring->total_packets = 0;
3731 q_vector->tx_ring->total_bytes = 0;
3732 q_vector->tx_ring->total_packets = 0;
6eb5a7f1 3733
047e0030 3734 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3735 /* this attempts to bias the interrupt rate towards Bulk
3736 * by adding intermediate steps when interrupt rate is
3737 * increasing */
047e0030
AD		 3738 new_itr = new_itr > q_vector->itr_val ?
3739 max((new_itr * q_vector->itr_val) /
3740 (new_itr + (q_vector->itr_val >> 2)),
3741 new_itr) :
9d5c8243
AK		 3742 new_itr;
3743 /* Don't write the value here; it resets the adapter's
3744 * internal timer, and causes us to delay far longer than
3745 * we should between interrupts. Instead, we write the ITR
3746 * value at the beginning of the next interrupt so the timing
3747 * ends up being correct.
3748 */
047e0030
AD		 3749 q_vector->itr_val = new_itr;
3750 q_vector->set_itr = 1;
9d5c8243 3751 }
9d5c8243
AK		 3752}
3753
9d5c8243
AK		 3754#define IGB_TX_FLAGS_CSUM 0x00000001
3755#define IGB_TX_FLAGS_VLAN 0x00000002
3756#define IGB_TX_FLAGS_TSO 0x00000004
3757#define IGB_TX_FLAGS_IPV4 0x00000008
cdfd01fc
AD		 3758#define IGB_TX_FLAGS_TSTAMP 0x00000010
3759#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3760#define IGB_TX_FLAGS_VLAN_SHIFT 16
9d5c8243 3761
85ad76b2 3762static inline int igb_tso_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3763 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3764{
3765 struct e1000_adv_tx_context_desc *context_desc;
3766 unsigned int i;
3767 int err;
3768 struct igb_buffer *buffer_info;
3769 u32 info = 0, tu_cmd = 0;
91d4ee33
NN		 3770 u32 mss_l4len_idx;
3771 u8 l4len;
9d5c8243
AK		 3772
3773 if (skb_header_cloned(skb)) {
3774 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3775 if (err)
3776 return err;
3777 }
3778
3779 l4len = tcp_hdrlen(skb);
3780 *hdr_len += l4len;
3781
3782 if (skb->protocol == htons(ETH_P_IP)) {
3783 struct iphdr *iph = ip_hdr(skb);
3784 iph->tot_len = 0;
3785 iph->check = 0;
3786 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3787 iph->daddr, 0,
3788 IPPROTO_TCP,
3789 0);
8e1e8a47 3790 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 3791 ipv6_hdr(skb)->payload_len = 0;
3792 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3793 &ipv6_hdr(skb)->daddr,
3794 0, IPPROTO_TCP, 0);
3795 }
3796
3797 i = tx_ring->next_to_use;
3798
3799 buffer_info = &tx_ring->buffer_info[i];
3800 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3801 /* VLAN MACLEN IPLEN */
3802 if (tx_flags & IGB_TX_FLAGS_VLAN)
3803 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3804 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3805 *hdr_len += skb_network_offset(skb);
3806 info |= skb_network_header_len(skb);
3807 *hdr_len += skb_network_header_len(skb);
3808 context_desc->vlan_macip_lens = cpu_to_le32(info);
3809
3810 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3811 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3812
3813 if (skb->protocol == htons(ETH_P_IP))
3814 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3815 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3816
3817 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3818
3819 /* MSS L4LEN IDX */
3820 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3821 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3822
73cd78f1 3823 /* For 82575, context index must be unique per ring. */
85ad76b2
AD		 3824 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3825 mss_l4len_idx |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3826
3827 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3828 context_desc->seqnum_seed = 0;
3829
3830 buffer_info->time_stamp = jiffies;
0e014cb1 3831 buffer_info->next_to_watch = i;
9d5c8243
AK		 3832 buffer_info->dma = 0;
3833 i++;
3834 if (i == tx_ring->count)
3835 i = 0;
3836
3837 tx_ring->next_to_use = i;
3838
3839 return true;
3840}
3841
85ad76b2
AD		 3842static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3843 struct sk_buff *skb, u32 tx_flags)
9d5c8243
AK		 3844{
3845 struct e1000_adv_tx_context_desc *context_desc;
59d71989 3846 struct device *dev = tx_ring->dev;
9d5c8243
AK		 3847 struct igb_buffer *buffer_info;
3848 u32 info = 0, tu_cmd = 0;
80785298 3849 unsigned int i;
9d5c8243
AK		 3850
3851 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3852 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3853 i = tx_ring->next_to_use;
3854 buffer_info = &tx_ring->buffer_info[i];
3855 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3856
3857 if (tx_flags & IGB_TX_FLAGS_VLAN)
3858 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
cdfd01fc 3859
9d5c8243
AK		 3860 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3861 if (skb->ip_summed == CHECKSUM_PARTIAL)
3862 info |= skb_network_header_len(skb);
3863
3864 context_desc->vlan_macip_lens = cpu_to_le32(info);
3865
3866 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3867
3868 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fa4a7ef3
AJ		 3869 __be16 protocol;
3870
3871 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3872 const struct vlan_ethhdr *vhdr =
3873 (const struct vlan_ethhdr*)skb->data;
3874
3875 protocol = vhdr->h_vlan_encapsulated_proto;
3876 } else {
3877 protocol = skb->protocol;
3878 }
3879
3880 switch (protocol) {
09640e63 3881 case cpu_to_be16(ETH_P_IP):
9d5c8243 3882 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
44b0cda3
MW		 3883 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3884 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3885 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3886 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3 3887 break;
09640e63 3888 case cpu_to_be16(ETH_P_IPV6):
44b0cda3
MW		 3889 /* XXX what about other V6 headers?? */
3890 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3891 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3892 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3893 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3
MW		 3894 break;
3895 default:
3896 if (unlikely(net_ratelimit()))
59d71989 3897 dev_warn(dev,
44b0cda3
MW		 3898 "partial checksum but proto=%x!\n",
3899 skb->protocol);
3900 break;
3901 }
9d5c8243
AK		 3902 }
3903
3904 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3905 context_desc->seqnum_seed = 0;
85ad76b2 3906 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
7dfc16fa 3907 context_desc->mss_l4len_idx =
85ad76b2 3908 cpu_to_le32(tx_ring->reg_idx << 4);
9d5c8243
AK		 3909
3910 buffer_info->time_stamp = jiffies;
0e014cb1 3911 buffer_info->next_to_watch = i;
9d5c8243
AK		 3912 buffer_info->dma = 0;
3913
3914 i++;
3915 if (i == tx_ring->count)
3916 i = 0;
3917 tx_ring->next_to_use = i;
3918
3919 return true;
3920 }
9d5c8243
AK		 3921 return false;
3922}
3923
3924#define IGB_MAX_TXD_PWR 16
3925#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3926
80785298 3927static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
0e014cb1 3928 unsigned int first)
9d5c8243
AK		 3929{
3930 struct igb_buffer *buffer_info;
59d71989 3931 struct device *dev = tx_ring->dev;
2873957d 3932 unsigned int hlen = skb_headlen(skb);
9d5c8243
AK		 3933 unsigned int count = 0, i;
3934 unsigned int f;
2873957d 3935 u16 gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
9d5c8243
AK		 3936
3937 i = tx_ring->next_to_use;
3938
3939 buffer_info = &tx_ring->buffer_info[i];
2873957d
NN		 3940 BUG_ON(hlen >= IGB_MAX_DATA_PER_TXD);
3941 buffer_info->length = hlen;
9d5c8243
AK		 3942 /* set time_stamp *before* dma to help avoid a possible race */
3943 buffer_info->time_stamp = jiffies;
0e014cb1 3944 buffer_info->next_to_watch = i;
2873957d 3945 buffer_info->dma = dma_map_single(dev, skb->data, hlen,
59d71989
AD		 3946 DMA_TO_DEVICE);
3947 if (dma_mapping_error(dev, buffer_info->dma))
6366ad33 3948 goto dma_error;
9d5c8243
AK		 3949
3950 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
2873957d
NN		 3951 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[f];
3952 unsigned int len = frag->size;
9d5c8243 3953
8581145f 3954 count++;
65689fef
AD		 3955 i++;
3956 if (i == tx_ring->count)
3957 i = 0;
3958
9d5c8243
AK		 3959 buffer_info = &tx_ring->buffer_info[i];
3960 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3961 buffer_info->length = len;
3962 buffer_info->time_stamp = jiffies;
0e014cb1 3963 buffer_info->next_to_watch = i;
6366ad33 3964 buffer_info->mapped_as_page = true;
59d71989 3965 buffer_info->dma = dma_map_page(dev,
6366ad33
AD		 3966 frag->page,
3967 frag->page_offset,
3968 len,
59d71989
AD		 3969 DMA_TO_DEVICE);
3970 if (dma_mapping_error(dev, buffer_info->dma))
6366ad33
AD		 3971 goto dma_error;
3972
9d5c8243
AK		 3973 }
3974
9d5c8243 3975 tx_ring->buffer_info[i].skb = skb;
2244d07b 3976 tx_ring->buffer_info[i].tx_flags = skb_shinfo(skb)->tx_flags;
2873957d
NN		 3977 /* multiply data chunks by size of headers */
3978 tx_ring->buffer_info[i].bytecount = ((gso_segs - 1) * hlen) + skb->len;
3979 tx_ring->buffer_info[i].gso_segs = gso_segs;
0e014cb1 3980 tx_ring->buffer_info[first].next_to_watch = i;
9d5c8243 3981
cdfd01fc 3982 return ++count;
6366ad33
AD		 3983
3984dma_error:
59d71989 3985 dev_err(dev, "TX DMA map failed\n");
6366ad33
AD		 3986
3987 /* clear timestamp and dma mappings for failed buffer_info mapping */
3988 buffer_info->dma = 0;
3989 buffer_info->time_stamp = 0;
3990 buffer_info->length = 0;
3991 buffer_info->next_to_watch = 0;
3992 buffer_info->mapped_as_page = false;
6366ad33
AD		 3993
3994 /* clear timestamp and dma mappings for remaining portion of packet */
a77ff709
NN		 3995 while (count--) {
3996 if (i == 0)
3997 i = tx_ring->count;
6366ad33 3998 i--;
6366ad33
AD		 3999 buffer_info = &tx_ring->buffer_info[i];
4000 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
4001 }
4002
4003 return 0;
9d5c8243
AK		 4004}
4005
85ad76b2 4006static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
91d4ee33 4007 u32 tx_flags, int count, u32 paylen,
9d5c8243
AK		 4008 u8 hdr_len)
4009{
cdfd01fc 4010 union e1000_adv_tx_desc *tx_desc;
9d5c8243
AK		 4011 struct igb_buffer *buffer_info;
4012 u32 olinfo_status = 0, cmd_type_len;
cdfd01fc 4013 unsigned int i = tx_ring->next_to_use;
9d5c8243
AK		 4014
4015 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
4016 E1000_ADVTXD_DCMD_DEXT);
4017
4018 if (tx_flags & IGB_TX_FLAGS_VLAN)
4019 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
4020
33af6bcc
PO		 4021 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4022 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
4023
9d5c8243
AK		 4024 if (tx_flags & IGB_TX_FLAGS_TSO) {
4025 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
4026
4027 /* insert tcp checksum */
4028 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4029
4030 /* insert ip checksum */
4031 if (tx_flags & IGB_TX_FLAGS_IPV4)
4032 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4033
4034 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
4035 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4036 }
4037
85ad76b2
AD		 4038 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
4039 (tx_flags & (IGB_TX_FLAGS_CSUM |
4040 IGB_TX_FLAGS_TSO |
7dfc16fa 4041 IGB_TX_FLAGS_VLAN)))
85ad76b2 4042 olinfo_status |= tx_ring->reg_idx << 4;
9d5c8243
AK		 4043
4044 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
4045
cdfd01fc 4046 do {
9d5c8243
AK		 4047 buffer_info = &tx_ring->buffer_info[i];
4048 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
4049 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
4050 tx_desc->read.cmd_type_len =
4051 cpu_to_le32(cmd_type_len | buffer_info->length);
4052 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
cdfd01fc 4053 count--;
9d5c8243
AK		 4054 i++;
4055 if (i == tx_ring->count)
4056 i = 0;
cdfd01fc 4057 } while (count > 0);
9d5c8243 4058
85ad76b2 4059 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
9d5c8243
AK		 4060 /* Force memory writes to complete before letting h/w
4061 * know there are new descriptors to fetch. (Only
4062 * applicable for weak-ordered memory model archs,
4063 * such as IA-64). */
4064 wmb();
4065
4066 tx_ring->next_to_use = i;
fce99e34 4067 writel(i, tx_ring->tail);
9d5c8243
AK		 4068 /* we need this if more than one processor can write to our tail
4069 * at a time, it syncronizes IO on IA64/Altix systems */
4070 mmiowb();
4071}
4072
e694e964 4073static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4074{
e694e964
AD		 4075 struct net_device *netdev = tx_ring->netdev;
4076
661086df 4077 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 4078
9d5c8243
AK		 4079 /* Herbert's original patch had:
4080 * smp_mb__after_netif_stop_queue();
4081 * but since that doesn't exist yet, just open code it. */
4082 smp_mb();
4083
4084 /* We need to check again in a case another CPU has just
4085 * made room available. */
c493ea45 4086 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 4087 return -EBUSY;
4088
4089 /* A reprieve! */
661086df 4090 netif_wake_subqueue(netdev, tx_ring->queue_index);
12dcd86b
ED		 4091
4092 u64_stats_update_begin(&tx_ring->tx_syncp2);
4093 tx_ring->tx_stats.restart_queue2++;
4094 u64_stats_update_end(&tx_ring->tx_syncp2);
4095
9d5c8243
AK		 4096 return 0;
4097}
4098
717ba089 4099static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4100{
c493ea45 4101 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 4102 return 0;
e694e964 4103 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 4104}
4105
b1a436c3
AD		 4106netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
4107 struct igb_ring *tx_ring)
9d5c8243 4108{
cdfd01fc 4109 int tso = 0, count;
91d4ee33
NN		 4110 u32 tx_flags = 0;
4111 u16 first;
4112 u8 hdr_len = 0;
9d5c8243 4113
9d5c8243
AK		 4114 /* need: 1 descriptor per page,
4115 * + 2 desc gap to keep tail from touching head,
4116 * + 1 desc for skb->data,
4117 * + 1 desc for context descriptor,
4118 * otherwise try next time */
e694e964 4119 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 4120 /* this is a hard error */
9d5c8243
AK		 4121 return NETDEV_TX_BUSY;
4122 }
33af6bcc 4123
2244d07b
OH		 4124 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4125 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
33af6bcc 4126 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 4127 }
9d5c8243 4128
eab6d18d 4129 if (vlan_tx_tag_present(skb)) {
9d5c8243
AK		 4130 tx_flags |= IGB_TX_FLAGS_VLAN;
4131 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4132 }
4133
661086df
PWJ		 4134 if (skb->protocol == htons(ETH_P_IP))
4135 tx_flags |= IGB_TX_FLAGS_IPV4;
4136
0e014cb1 4137 first = tx_ring->next_to_use;
85ad76b2
AD		 4138 if (skb_is_gso(skb)) {
4139 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
cdfd01fc 4140
85ad76b2
AD		 4141 if (tso < 0) {
4142 dev_kfree_skb_any(skb);
4143 return NETDEV_TX_OK;
4144 }
9d5c8243
AK		 4145 }
4146
4147 if (tso)
4148 tx_flags |= IGB_TX_FLAGS_TSO;
85ad76b2 4149 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
bc1cbd34
AD		 4150 (skb->ip_summed == CHECKSUM_PARTIAL))
4151 tx_flags |= IGB_TX_FLAGS_CSUM;
9d5c8243 4152
65689fef 4153 /*
cdfd01fc 4154 * count reflects descriptors mapped, if 0 or less then mapping error
65689fef
AD		 4155 * has occured and we need to rewind the descriptor queue
4156 */
80785298 4157 count = igb_tx_map_adv(tx_ring, skb, first);
6366ad33 4158 if (!count) {
65689fef
AD		 4159 dev_kfree_skb_any(skb);
4160 tx_ring->buffer_info[first].time_stamp = 0;
4161 tx_ring->next_to_use = first;
85ad76b2 4162 return NETDEV_TX_OK;
65689fef 4163 }
9d5c8243 4164
85ad76b2
AD		 4165 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
4166
4167 /* Make sure there is space in the ring for the next send. */
e694e964 4168 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 4169
9d5c8243
AK		 4170 return NETDEV_TX_OK;
4171}
4172
3b29a56d
SH		 4173static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
4174 struct net_device *netdev)
9d5c8243
AK		 4175{
4176 struct igb_adapter *adapter = netdev_priv(netdev);
661086df 4177 struct igb_ring *tx_ring;
661086df 4178 int r_idx = 0;
b1a436c3
AD		 4179
4180 if (test_bit(__IGB_DOWN, &adapter->state)) {
4181 dev_kfree_skb_any(skb);
4182 return NETDEV_TX_OK;
4183 }
4184
4185 if (skb->len <= 0) {
4186 dev_kfree_skb_any(skb);
4187 return NETDEV_TX_OK;
4188 }
4189
1bfaf07b 4190 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
661086df 4191 tx_ring = adapter->multi_tx_table[r_idx];
9d5c8243
AK		 4192
4193 /* This goes back to the question of how to logically map a tx queue
4194 * to a flow. Right now, performance is impacted slightly negatively
4195 * if using multiple tx queues. If the stack breaks away from a
4196 * single qdisc implementation, we can look at this again. */
e694e964 4197 return igb_xmit_frame_ring_adv(skb, tx_ring);
9d5c8243
AK		 4198}
4199
4200/**
4201 * igb_tx_timeout - Respond to a Tx Hang
4202 * @netdev: network interface device structure
4203 **/
4204static void igb_tx_timeout(struct net_device *netdev)
4205{
4206 struct igb_adapter *adapter = netdev_priv(netdev);
4207 struct e1000_hw *hw = &adapter->hw;
4208
4209 /* Do the reset outside of interrupt context */
4210 adapter->tx_timeout_count++;
f7ba205e 4211
55cac248
AD		 4212 if (hw->mac.type == e1000_82580)
4213 hw->dev_spec._82575.global_device_reset = true;
4214
9d5c8243 4215 schedule_work(&adapter->reset_task);
265de409
AD		 4216 wr32(E1000_EICS,
4217 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 4218}
4219
4220static void igb_reset_task(struct work_struct *work)
4221{
4222 struct igb_adapter *adapter;
4223 adapter = container_of(work, struct igb_adapter, reset_task);
4224
c97ec42a
TI		 4225 igb_dump(adapter);
4226 netdev_err(adapter->netdev, "Reset adapter\n");
9d5c8243
AK		 4227 igb_reinit_locked(adapter);
4228}
4229
4230/**
12dcd86b 4231 * igb_get_stats64 - Get System Network Statistics
9d5c8243 4232 * @netdev: network interface device structure
12dcd86b 4233 * @stats: rtnl_link_stats64 pointer
9d5c8243 4234 *
9d5c8243 4235 **/
12dcd86b
ED		 4236static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4237 struct rtnl_link_stats64 *stats)
9d5c8243 4238{
12dcd86b
ED		 4239 struct igb_adapter *adapter = netdev_priv(netdev);
4240
4241 spin_lock(&adapter->stats64_lock);
4242 igb_update_stats(adapter, &adapter->stats64);
4243 memcpy(stats, &adapter->stats64, sizeof(*stats));
4244 spin_unlock(&adapter->stats64_lock);
4245
4246 return stats;
9d5c8243
AK		 4247}
4248
4249/**
4250 * igb_change_mtu - Change the Maximum Transfer Unit
4251 * @netdev: network interface device structure
4252 * @new_mtu: new value for maximum frame size
4253 *
4254 * Returns 0 on success, negative on failure
4255 **/
4256static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4257{
4258 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4259 struct pci_dev *pdev = adapter->pdev;
9d5c8243 4260 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4c844851 4261 u32 rx_buffer_len, i;
9d5c8243 4262
c809d227 4263 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 4264 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 4265 return -EINVAL;
4266 }
4267
9d5c8243 4268 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 4269 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 4270 return -EINVAL;
4271 }
4272
4273 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4274 msleep(1);
73cd78f1 4275
9d5c8243
AK		 4276 /* igb_down has a dependency on max_frame_size */
4277 adapter->max_frame_size = max_frame;
559e9c49 4278
9d5c8243
AK		 4279 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
4280 * means we reserve 2 more, this pushes us to allocate from the next
4281 * larger slab size.
4282 * i.e. RXBUFFER_2048 --> size-4096 slab
4283 */
4284
757b77e2
NN		 4285 if (adapter->hw.mac.type == e1000_82580)
4286 max_frame += IGB_TS_HDR_LEN;
4287
7d95b717 4288 if (max_frame <= IGB_RXBUFFER_1024)
4c844851 4289 rx_buffer_len = IGB_RXBUFFER_1024;
6ec43fe6 4290 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4c844851 4291 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
6ec43fe6 4292 else
4c844851
AD		 4293 rx_buffer_len = IGB_RXBUFFER_128;
4294
757b77e2
NN		 4295 if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN + IGB_TS_HDR_LEN) ||
4296 (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN))
4297 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN;
4298
4299 if ((adapter->hw.mac.type == e1000_82580) &&
4300 (rx_buffer_len == IGB_RXBUFFER_128))
4301 rx_buffer_len += IGB_RXBUFFER_64;
4302
4c844851
AD		 4303 if (netif_running(netdev))
4304 igb_down(adapter);
9d5c8243 4305
090b1795 4306 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 4307 netdev->mtu, new_mtu);
4308 netdev->mtu = new_mtu;
4309
4c844851 4310 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 4311 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4c844851 4312
9d5c8243
AK		 4313 if (netif_running(netdev))
4314 igb_up(adapter);
4315 else
4316 igb_reset(adapter);
4317
4318 clear_bit(__IGB_RESETTING, &adapter->state);
4319
4320 return 0;
4321}
4322
4323/**
4324 * igb_update_stats - Update the board statistics counters
4325 * @adapter: board private structure
4326 **/
4327
12dcd86b
ED		 4328void igb_update_stats(struct igb_adapter *adapter,
4329 struct rtnl_link_stats64 *net_stats)
9d5c8243
AK		 4330{
4331 struct e1000_hw *hw = &adapter->hw;
4332 struct pci_dev *pdev = adapter->pdev;
fa3d9a6d 4333 u32 reg, mpc;
9d5c8243 4334 u16 phy_tmp;
3f9c0164
AD		 4335 int i;
4336 u64 bytes, packets;
12dcd86b
ED		 4337 unsigned int start;
4338 u64 _bytes, _packets;
9d5c8243
AK		 4339
4340#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4341
4342 /*
4343 * Prevent stats update while adapter is being reset, or if the pci
4344 * connection is down.
4345 */
4346 if (adapter->link_speed == 0)
4347 return;
4348 if (pci_channel_offline(pdev))
4349 return;
4350
3f9c0164
AD		 4351 bytes = 0;
4352 packets = 0;
4353 for (i = 0; i < adapter->num_rx_queues; i++) {
4354 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446 4355 struct igb_ring *ring = adapter->rx_ring[i];
12dcd86b 4356
3025a446 4357 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 4358 net_stats->rx_fifo_errors += rqdpc_tmp;
12dcd86b
ED		 4359
4360 do {
4361 start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4362 _bytes = ring->rx_stats.bytes;
4363 _packets = ring->rx_stats.packets;
4364 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4365 bytes += _bytes;
4366 packets += _packets;
3f9c0164
AD		 4367 }
4368
128e45eb
AD		 4369 net_stats->rx_bytes = bytes;
4370 net_stats->rx_packets = packets;
3f9c0164
AD		 4371
4372 bytes = 0;
4373 packets = 0;
4374 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 4375 struct igb_ring *ring = adapter->tx_ring[i];
12dcd86b
ED		 4376 do {
4377 start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4378 _bytes = ring->tx_stats.bytes;
4379 _packets = ring->tx_stats.packets;
4380 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4381 bytes += _bytes;
4382 packets += _packets;
3f9c0164 4383 }
128e45eb
AD		 4384 net_stats->tx_bytes = bytes;
4385 net_stats->tx_packets = packets;
3f9c0164
AD		 4386
4387 /* read stats registers */
9d5c8243
AK		 4388 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4389 adapter->stats.gprc += rd32(E1000_GPRC);
4390 adapter->stats.gorc += rd32(E1000_GORCL);
4391 rd32(E1000_GORCH); /* clear GORCL */
4392 adapter->stats.bprc += rd32(E1000_BPRC);
4393 adapter->stats.mprc += rd32(E1000_MPRC);
4394 adapter->stats.roc += rd32(E1000_ROC);
4395
4396 adapter->stats.prc64 += rd32(E1000_PRC64);
4397 adapter->stats.prc127 += rd32(E1000_PRC127);
4398 adapter->stats.prc255 += rd32(E1000_PRC255);
4399 adapter->stats.prc511 += rd32(E1000_PRC511);
4400 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4401 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4402 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4403 adapter->stats.sec += rd32(E1000_SEC);
4404
fa3d9a6d
MW		 4405 mpc = rd32(E1000_MPC);
4406 adapter->stats.mpc += mpc;
4407 net_stats->rx_fifo_errors += mpc;
9d5c8243
AK		 4408 adapter->stats.scc += rd32(E1000_SCC);
4409 adapter->stats.ecol += rd32(E1000_ECOL);
4410 adapter->stats.mcc += rd32(E1000_MCC);
4411 adapter->stats.latecol += rd32(E1000_LATECOL);
4412 adapter->stats.dc += rd32(E1000_DC);
4413 adapter->stats.rlec += rd32(E1000_RLEC);
4414 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4415 adapter->stats.xontxc += rd32(E1000_XONTXC);
4416 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4417 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4418 adapter->stats.fcruc += rd32(E1000_FCRUC);
4419 adapter->stats.gptc += rd32(E1000_GPTC);
4420 adapter->stats.gotc += rd32(E1000_GOTCL);
4421 rd32(E1000_GOTCH); /* clear GOTCL */
fa3d9a6d 4422 adapter->stats.rnbc += rd32(E1000_RNBC);
9d5c8243
AK		 4423 adapter->stats.ruc += rd32(E1000_RUC);
4424 adapter->stats.rfc += rd32(E1000_RFC);
4425 adapter->stats.rjc += rd32(E1000_RJC);
4426 adapter->stats.tor += rd32(E1000_TORH);
4427 adapter->stats.tot += rd32(E1000_TOTH);
4428 adapter->stats.tpr += rd32(E1000_TPR);
4429
4430 adapter->stats.ptc64 += rd32(E1000_PTC64);
4431 adapter->stats.ptc127 += rd32(E1000_PTC127);
4432 adapter->stats.ptc255 += rd32(E1000_PTC255);
4433 adapter->stats.ptc511 += rd32(E1000_PTC511);
4434 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4435 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4436
4437 adapter->stats.mptc += rd32(E1000_MPTC);
4438 adapter->stats.bptc += rd32(E1000_BPTC);
4439
2d0b0f69
NN		 4440 adapter->stats.tpt += rd32(E1000_TPT);
4441 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4442
4443 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4444 /* read internal phy specific stats */
4445 reg = rd32(E1000_CTRL_EXT);
4446 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4447 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4448 adapter->stats.tncrs += rd32(E1000_TNCRS);
4449 }
4450
9d5c8243
AK		 4451 adapter->stats.tsctc += rd32(E1000_TSCTC);
4452 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4453
4454 adapter->stats.iac += rd32(E1000_IAC);
4455 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4456 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4457 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4458 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4459 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4460 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4461 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4462 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4463
4464 /* Fill out the OS statistics structure */
128e45eb
AD		 4465 net_stats->multicast = adapter->stats.mprc;
4466 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4467
4468 /* Rx Errors */
4469
4470 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4471 * our own version based on RUC and ROC */
128e45eb 4472 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4473 adapter->stats.crcerrs + adapter->stats.algnerrc +
4474 adapter->stats.ruc + adapter->stats.roc +
4475 adapter->stats.cexterr;
128e45eb
AD		 4476 net_stats->rx_length_errors = adapter->stats.ruc +
4477 adapter->stats.roc;
4478 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4479 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4480 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4481
4482 /* Tx Errors */
128e45eb
AD		 4483 net_stats->tx_errors = adapter->stats.ecol +
4484 adapter->stats.latecol;
4485 net_stats->tx_aborted_errors = adapter->stats.ecol;
4486 net_stats->tx_window_errors = adapter->stats.latecol;
4487 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4488
4489 /* Tx Dropped needs to be maintained elsewhere */
4490
4491 /* Phy Stats */
4492 if (hw->phy.media_type == e1000_media_type_copper) {
4493 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4494 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4495 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4496 adapter->phy_stats.idle_errors += phy_tmp;
4497 }
4498 }
4499
4500 /* Management Stats */
4501 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4502 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4503 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4504}
4505
9d5c8243
AK		 4506static irqreturn_t igb_msix_other(int irq, void *data)
4507{
047e0030 4508 struct igb_adapter *adapter = data;
9d5c8243 4509 struct e1000_hw *hw = &adapter->hw;
844290e5 4510 u32 icr = rd32(E1000_ICR);
844290e5 4511 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4512
7f081d40
AD		 4513 if (icr & E1000_ICR_DRSTA)
4514 schedule_work(&adapter->reset_task);
4515
047e0030 4516 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4517 /* HW is reporting DMA is out of sync */
4518 adapter->stats.doosync++;
4519 }
eebbbdba 4520
4ae196df
AD		 4521 /* Check for a mailbox event */
4522 if (icr & E1000_ICR_VMMB)
4523 igb_msg_task(adapter);
4524
4525 if (icr & E1000_ICR_LSC) {
4526 hw->mac.get_link_status = 1;
4527 /* guard against interrupt when we're going down */
4528 if (!test_bit(__IGB_DOWN, &adapter->state))
4529 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4530 }
4531
25568a53
AD		 4532 if (adapter->vfs_allocated_count)
4533 wr32(E1000_IMS, E1000_IMS_LSC |
4534 E1000_IMS_VMMB |
4535 E1000_IMS_DOUTSYNC);
4536 else
4537 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
844290e5 4538 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4539
4540 return IRQ_HANDLED;
4541}
4542
047e0030 4543static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4544{
26b39276 4545 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4546 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4547
047e0030
AD		 4548 if (!q_vector->set_itr)
4549 return;
73cd78f1 4550
047e0030
AD		 4551 if (!itr_val)
4552 itr_val = 0x4;
661086df 4553
26b39276
AD		 4554 if (adapter->hw.mac.type == e1000_82575)
4555 itr_val |= itr_val << 16;
661086df 4556 else
047e0030 4557 itr_val |= 0x8000000;
661086df 4558
047e0030
AD		 4559 writel(itr_val, q_vector->itr_register);
4560 q_vector->set_itr = 0;
6eb5a7f1
AD		 4561}
4562
047e0030 4563static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4564{
047e0030 4565 struct igb_q_vector *q_vector = data;
9d5c8243 4566
047e0030
AD		 4567 /* Write the ITR value calculated from the previous interrupt. */
4568 igb_write_itr(q_vector);
9d5c8243 4569
047e0030 4570 napi_schedule(&q_vector->napi);
844290e5 4571
047e0030 4572 return IRQ_HANDLED;
fe4506b6
JC		 4573}
4574
421e02f0 4575#ifdef CONFIG_IGB_DCA
047e0030 4576static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4577{
047e0030 4578 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4579 struct e1000_hw *hw = &adapter->hw;
4580 int cpu = get_cpu();
fe4506b6 4581
047e0030
AD		 4582 if (q_vector->cpu == cpu)
4583 goto out_no_update;
4584
4585 if (q_vector->tx_ring) {
4586 int q = q_vector->tx_ring->reg_idx;
4587 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4588 if (hw->mac.type == e1000_82575) {
4589 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4590 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4591 } else {
047e0030
AD		 4592 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4593 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4594 E1000_DCA_TXCTRL_CPUID_SHIFT;
4595 }
4596 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4597 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4598 }
4599 if (q_vector->rx_ring) {
4600 int q = q_vector->rx_ring->reg_idx;
4601 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4602 if (hw->mac.type == e1000_82575) {
2d064c06 4603 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4604 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4605 } else {
4606 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4607 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4608 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4609 }
fe4506b6
JC		 4610 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4611 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4612 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4613 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4614 }
047e0030
AD		 4615 q_vector->cpu = cpu;
4616out_no_update:
fe4506b6
JC		 4617 put_cpu();
4618}
4619
4620static void igb_setup_dca(struct igb_adapter *adapter)
4621{
7e0e99ef 4622 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4623 int i;
4624
7dfc16fa 4625 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4626 return;
4627
7e0e99ef
AD		 4628 /* Always use CB2 mode, difference is masked in the CB driver. */
4629 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4630
047e0030 4631 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4632 adapter->q_vector[i]->cpu = -1;
4633 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4634 }
4635}
4636
4637static int __igb_notify_dca(struct device *dev, void *data)
4638{
4639 struct net_device *netdev = dev_get_drvdata(dev);
4640 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4641 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4642 struct e1000_hw *hw = &adapter->hw;
4643 unsigned long event = *(unsigned long *)data;
4644
4645 switch (event) {
4646 case DCA_PROVIDER_ADD:
4647 /* if already enabled, don't do it again */
7dfc16fa 4648 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4649 break;
fe4506b6 4650 if (dca_add_requester(dev) == 0) {
bbd98fe4 4651 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4652 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4653 igb_setup_dca(adapter);
4654 break;
4655 }
4656 /* Fall Through since DCA is disabled. */
4657 case DCA_PROVIDER_REMOVE:
7dfc16fa 4658 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4659 /* without this a class_device is left
047e0030 4660 * hanging around in the sysfs model */
fe4506b6 4661 dca_remove_requester(dev);
090b1795 4662 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4663 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4664 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4665 }
4666 break;
4667 }
bbd98fe4 4668
fe4506b6 4669 return 0;
9d5c8243
AK		 4670}
4671
fe4506b6
JC		 4672static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4673 void *p)
4674{
4675 int ret_val;
4676
4677 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4678 __igb_notify_dca);
4679
4680 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4681}
421e02f0 4682#endif /* CONFIG_IGB_DCA */
9d5c8243 4683
4ae196df
AD		 4684static void igb_ping_all_vfs(struct igb_adapter *adapter)
4685{
4686 struct e1000_hw *hw = &adapter->hw;
4687 u32 ping;
4688 int i;
4689
4690 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4691 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 4692 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 4693 ping |= E1000_VT_MSGTYPE_CTS;
4694 igb_write_mbx(hw, &ping, 1, i);
4695 }
4696}
4697
7d5753f0
AD		 4698static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4699{
4700 struct e1000_hw *hw = &adapter->hw;
4701 u32 vmolr = rd32(E1000_VMOLR(vf));
4702 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4703
d85b9004 4704 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
7d5753f0
AD		 4705 IGB_VF_FLAG_MULTI_PROMISC);
4706 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4707
4708 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4709 vmolr |= E1000_VMOLR_MPME;
d85b9004 4710 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
7d5753f0
AD		 4711 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4712 } else {
4713 /*
4714 * if we have hashes and we are clearing a multicast promisc
4715 * flag we need to write the hashes to the MTA as this step
4716 * was previously skipped
4717 */
4718 if (vf_data->num_vf_mc_hashes > 30) {
4719 vmolr |= E1000_VMOLR_MPME;
4720 } else if (vf_data->num_vf_mc_hashes) {
4721 int j;
4722 vmolr |= E1000_VMOLR_ROMPE;
4723 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4724 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4725 }
4726 }
4727
4728 wr32(E1000_VMOLR(vf), vmolr);
4729
4730 /* there are flags left unprocessed, likely not supported */
4731 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4732 return -EINVAL;
4733
4734 return 0;
4735
4736}
4737
4ae196df
AD		 4738static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4739 u32 *msgbuf, u32 vf)
4740{
4741 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4742 u16 *hash_list = (u16 *)&msgbuf[1];
4743 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4744 int i;
4745
7d5753f0 4746 /* salt away the number of multicast addresses assigned
4ae196df
AD		 4747 * to this VF for later use to restore when the PF multi cast
4748 * list changes
4749 */
4750 vf_data->num_vf_mc_hashes = n;
4751
7d5753f0
AD		 4752 /* only up to 30 hash values supported */
4753 if (n > 30)
4754 n = 30;
4755
4756 /* store the hashes for later use */
4ae196df 4757 for (i = 0; i < n; i++)
a419aef8 4758 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 4759
4760 /* Flush and reset the mta with the new values */
ff41f8dc 4761 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4762
4763 return 0;
4764}
4765
4766static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4767{
4768 struct e1000_hw *hw = &adapter->hw;
4769 struct vf_data_storage *vf_data;
4770 int i, j;
4771
4772 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 4773 u32 vmolr = rd32(E1000_VMOLR(i));
4774 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4775
4ae196df 4776 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 4777
4778 if ((vf_data->num_vf_mc_hashes > 30) ||
4779 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4780 vmolr |= E1000_VMOLR_MPME;
4781 } else if (vf_data->num_vf_mc_hashes) {
4782 vmolr |= E1000_VMOLR_ROMPE;
4783 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4784 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4785 }
4786 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 4787 }
4788}
4789
4790static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4791{
4792 struct e1000_hw *hw = &adapter->hw;
4793 u32 pool_mask, reg, vid;
4794 int i;
4795
4796 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4797
4798 /* Find the vlan filter for this id */
4799 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4800 reg = rd32(E1000_VLVF(i));
4801
4802 /* remove the vf from the pool */
4803 reg &= ~pool_mask;
4804
4805 /* if pool is empty then remove entry from vfta */
4806 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4807 (reg & E1000_VLVF_VLANID_ENABLE)) {
4808 reg = 0;
4809 vid = reg & E1000_VLVF_VLANID_MASK;
4810 igb_vfta_set(hw, vid, false);
4811 }
4812
4813 wr32(E1000_VLVF(i), reg);
4814 }
ae641bdc
AD		 4815
4816 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 4817}
4818
4819static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4820{
4821 struct e1000_hw *hw = &adapter->hw;
4822 u32 reg, i;
4823
51466239
AD		 4824 /* The vlvf table only exists on 82576 hardware and newer */
4825 if (hw->mac.type < e1000_82576)
4826 return -1;
4827
4828 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 4829 if (!adapter->vfs_allocated_count)
4830 return -1;
4831
4832 /* Find the vlan filter for this id */
4833 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4834 reg = rd32(E1000_VLVF(i));
4835 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4836 vid == (reg & E1000_VLVF_VLANID_MASK))
4837 break;
4838 }
4839
4840 if (add) {
4841 if (i == E1000_VLVF_ARRAY_SIZE) {
4842 /* Did not find a matching VLAN ID entry that was
4843 * enabled. Search for a free filter entry, i.e.
4844 * one without the enable bit set
4845 */
4846 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4847 reg = rd32(E1000_VLVF(i));
4848 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4849 break;
4850 }
4851 }
4852 if (i < E1000_VLVF_ARRAY_SIZE) {
4853 /* Found an enabled/available entry */
4854 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4855
4856 /* if !enabled we need to set this up in vfta */
4857 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 4858 /* add VID to filter table */
4859 igb_vfta_set(hw, vid, true);
4ae196df
AD		 4860 reg |= E1000_VLVF_VLANID_ENABLE;
4861 }
cad6d05f
AD		 4862 reg &= ~E1000_VLVF_VLANID_MASK;
4863 reg |= vid;
4ae196df 4864 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4865
4866 /* do not modify RLPML for PF devices */
4867 if (vf >= adapter->vfs_allocated_count)
4868 return 0;
4869
4870 if (!adapter->vf_data[vf].vlans_enabled) {
4871 u32 size;
4872 reg = rd32(E1000_VMOLR(vf));
4873 size = reg & E1000_VMOLR_RLPML_MASK;
4874 size += 4;
4875 reg &= ~E1000_VMOLR_RLPML_MASK;
4876 reg |= size;
4877 wr32(E1000_VMOLR(vf), reg);
4878 }
ae641bdc 4879
51466239 4880 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 4881 return 0;
4882 }
4883 } else {
4884 if (i < E1000_VLVF_ARRAY_SIZE) {
4885 /* remove vf from the pool */
4886 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4887 /* if pool is empty then remove entry from vfta */
4888 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4889 reg = 0;
4890 igb_vfta_set(hw, vid, false);
4891 }
4892 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4893
4894 /* do not modify RLPML for PF devices */
4895 if (vf >= adapter->vfs_allocated_count)
4896 return 0;
4897
4898 adapter->vf_data[vf].vlans_enabled--;
4899 if (!adapter->vf_data[vf].vlans_enabled) {
4900 u32 size;
4901 reg = rd32(E1000_VMOLR(vf));
4902 size = reg & E1000_VMOLR_RLPML_MASK;
4903 size -= 4;
4904 reg &= ~E1000_VMOLR_RLPML_MASK;
4905 reg |= size;
4906 wr32(E1000_VMOLR(vf), reg);
4907 }
4ae196df
AD		 4908 }
4909 }
8151d294
WM		 4910 return 0;
4911}
4912
4913static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
4914{
4915 struct e1000_hw *hw = &adapter->hw;
4916
4917 if (vid)
4918 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
4919 else
4920 wr32(E1000_VMVIR(vf), 0);
4921}
4922
4923static int igb_ndo_set_vf_vlan(struct net_device *netdev,
4924 int vf, u16 vlan, u8 qos)
4925{
4926 int err = 0;
4927 struct igb_adapter *adapter = netdev_priv(netdev);
4928
4929 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
4930 return -EINVAL;
4931 if (vlan || qos) {
4932 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
4933 if (err)
4934 goto out;
4935 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
4936 igb_set_vmolr(adapter, vf, !vlan);
4937 adapter->vf_data[vf].pf_vlan = vlan;
4938 adapter->vf_data[vf].pf_qos = qos;
4939 dev_info(&adapter->pdev->dev,
4940 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
4941 if (test_bit(__IGB_DOWN, &adapter->state)) {
4942 dev_warn(&adapter->pdev->dev,
4943 "The VF VLAN has been set,"
4944 " but the PF device is not up.\n");
4945 dev_warn(&adapter->pdev->dev,
4946 "Bring the PF device up before"
4947 " attempting to use the VF device.\n");
4948 }
4949 } else {
4950 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
4951 false, vf);
4952 igb_set_vmvir(adapter, vlan, vf);
4953 igb_set_vmolr(adapter, vf, true);
4954 adapter->vf_data[vf].pf_vlan = 0;
4955 adapter->vf_data[vf].pf_qos = 0;
4956 }
4957out:
4958 return err;
4ae196df
AD		 4959}
4960
4961static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4962{
4963 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4964 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4965
4966 return igb_vlvf_set(adapter, vid, add, vf);
4967}
4968
f2ca0dbe 4969static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 4970{
8151d294
WM		 4971 /* clear flags */
4972 adapter->vf_data[vf].flags &= ~(IGB_VF_FLAG_PF_SET_MAC);
f2ca0dbe 4973 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 4974
4975 /* reset offloads to defaults */
8151d294 4976 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 4977
4978 /* reset vlans for device */
4979 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 4980 if (adapter->vf_data[vf].pf_vlan)
4981 igb_ndo_set_vf_vlan(adapter->netdev, vf,
4982 adapter->vf_data[vf].pf_vlan,
4983 adapter->vf_data[vf].pf_qos);
4984 else
4985 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 4986
4987 /* reset multicast table array for vf */
4988 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4989
4990 /* Flush and reset the mta with the new values */
ff41f8dc 4991 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4992}
4993
f2ca0dbe
AD		 4994static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4995{
4996 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4997
4998 /* generate a new mac address as we were hotplug removed/added */
8151d294
WM		 4999 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5000 random_ether_addr(vf_mac);
f2ca0dbe
AD		 5001
5002 /* process remaining reset events */
5003 igb_vf_reset(adapter, vf);
5004}
5005
5006static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 5007{
5008 struct e1000_hw *hw = &adapter->hw;
5009 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 5010 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 5011 u32 reg, msgbuf[3];
5012 u8 *addr = (u8 *)(&msgbuf[1]);
5013
5014 /* process all the same items cleared in a function level reset */
f2ca0dbe 5015 igb_vf_reset(adapter, vf);
4ae196df
AD		 5016
5017 /* set vf mac address */
26ad9178 5018 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 5019
5020 /* enable transmit and receive for vf */
5021 reg = rd32(E1000_VFTE);
5022 wr32(E1000_VFTE, reg | (1 << vf));
5023 reg = rd32(E1000_VFRE);
5024 wr32(E1000_VFRE, reg | (1 << vf));
5025
f2ca0dbe 5026 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4ae196df
AD		 5027
5028 /* reply to reset with ack and vf mac address */
5029 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5030 memcpy(addr, vf_mac, 6);
5031 igb_write_mbx(hw, msgbuf, 3, vf);
5032}
5033
5034static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5035{
de42edde
GR		 5036 /*
5037 * The VF MAC Address is stored in a packed array of bytes
5038 * starting at the second 32 bit word of the msg array
5039 */
f2ca0dbe
AD		 5040 unsigned char *addr = (char *)&msg[1];
5041 int err = -1;
4ae196df 5042
f2ca0dbe
AD		 5043 if (is_valid_ether_addr(addr))
5044 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 5045
f2ca0dbe 5046 return err;
4ae196df
AD		 5047}
5048
5049static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5050{
5051 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5052 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5053 u32 msg = E1000_VT_MSGTYPE_NACK;
5054
5055 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 5056 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5057 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 5058 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 5059 vf_data->last_nack = jiffies;
4ae196df
AD		 5060 }
5061}
5062
f2ca0dbe 5063static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 5064{
f2ca0dbe
AD		 5065 struct pci_dev *pdev = adapter->pdev;
5066 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 5067 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5068 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5069 s32 retval;
5070
f2ca0dbe 5071 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 5072
fef45f4c
AD		 5073 if (retval) {
5074 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 5075 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 5076 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5077 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5078 return;
5079 goto out;
5080 }
4ae196df
AD		 5081
5082 /* this is a message we already processed, do nothing */
5083 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 5084 return;
4ae196df
AD		 5085
5086 /*
5087 * until the vf completes a reset it should not be
5088 * allowed to start any configuration.
5089 */
5090
5091 if (msgbuf[0] == E1000_VF_RESET) {
5092 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 5093 return;
4ae196df
AD		 5094 }
5095
f2ca0dbe 5096 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 5097 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5098 return;
5099 retval = -1;
5100 goto out;
4ae196df
AD		 5101 }
5102
5103 switch ((msgbuf[0] & 0xFFFF)) {
5104 case E1000_VF_SET_MAC_ADDR:
5105 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5106 break;
7d5753f0
AD		 5107 case E1000_VF_SET_PROMISC:
5108 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5109 break;
4ae196df
AD		 5110 case E1000_VF_SET_MULTICAST:
5111 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5112 break;
5113 case E1000_VF_SET_LPE:
5114 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5115 break;
5116 case E1000_VF_SET_VLAN:
8151d294
WM		 5117 if (adapter->vf_data[vf].pf_vlan)
5118 retval = -1;
5119 else
5120 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 5121 break;
5122 default:
090b1795 5123 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 5124 retval = -1;
5125 break;
5126 }
5127
fef45f4c
AD		 5128 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5129out:
4ae196df
AD		 5130 /* notify the VF of the results of what it sent us */
5131 if (retval)
5132 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5133 else
5134 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5135
4ae196df 5136 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 5137}
4ae196df 5138
f2ca0dbe
AD		 5139static void igb_msg_task(struct igb_adapter *adapter)
5140{
5141 struct e1000_hw *hw = &adapter->hw;
5142 u32 vf;
5143
5144 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5145 /* process any reset requests */
5146 if (!igb_check_for_rst(hw, vf))
5147 igb_vf_reset_event(adapter, vf);
5148
5149 /* process any messages pending */
5150 if (!igb_check_for_msg(hw, vf))
5151 igb_rcv_msg_from_vf(adapter, vf);
5152
5153 /* process any acks */
5154 if (!igb_check_for_ack(hw, vf))
5155 igb_rcv_ack_from_vf(adapter, vf);
5156 }
4ae196df
AD		 5157}
5158
68d480c4
AD		 5159/**
5160 * igb_set_uta - Set unicast filter table address
5161 * @adapter: board private structure
5162 *
5163 * The unicast table address is a register array of 32-bit registers.
5164 * The table is meant to be used in a way similar to how the MTA is used
5165 * however due to certain limitations in the hardware it is necessary to
5166 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
5167 * enable bit to allow vlan tag stripping when promiscous mode is enabled
5168 **/
5169static void igb_set_uta(struct igb_adapter *adapter)
5170{
5171 struct e1000_hw *hw = &adapter->hw;
5172 int i;
5173
5174 /* The UTA table only exists on 82576 hardware and newer */
5175 if (hw->mac.type < e1000_82576)
5176 return;
5177
5178 /* we only need to do this if VMDq is enabled */
5179 if (!adapter->vfs_allocated_count)
5180 return;
5181
5182 for (i = 0; i < hw->mac.uta_reg_count; i++)
5183 array_wr32(E1000_UTA, i, ~0);
5184}
5185
9d5c8243
AK		 5186/**
5187 * igb_intr_msi - Interrupt Handler
5188 * @irq: interrupt number
5189 * @data: pointer to a network interface device structure
5190 **/
5191static irqreturn_t igb_intr_msi(int irq, void *data)
5192{
047e0030
AD		 5193 struct igb_adapter *adapter = data;
5194 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5195 struct e1000_hw *hw = &adapter->hw;
5196 /* read ICR disables interrupts using IAM */
5197 u32 icr = rd32(E1000_ICR);
5198
047e0030 5199 igb_write_itr(q_vector);
9d5c8243 5200
7f081d40
AD		 5201 if (icr & E1000_ICR_DRSTA)
5202 schedule_work(&adapter->reset_task);
5203
047e0030 5204 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5205 /* HW is reporting DMA is out of sync */
5206 adapter->stats.doosync++;
5207 }
5208
9d5c8243
AK		 5209 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5210 hw->mac.get_link_status = 1;
5211 if (!test_bit(__IGB_DOWN, &adapter->state))
5212 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5213 }
5214
047e0030 5215 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5216
5217 return IRQ_HANDLED;
5218}
5219
5220/**
4a3c6433 5221 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 5222 * @irq: interrupt number
5223 * @data: pointer to a network interface device structure
5224 **/
5225static irqreturn_t igb_intr(int irq, void *data)
5226{
047e0030
AD		 5227 struct igb_adapter *adapter = data;
5228 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5229 struct e1000_hw *hw = &adapter->hw;
5230 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5231 * need for the IMC write */
5232 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 5233 if (!icr)
5234 return IRQ_NONE; /* Not our interrupt */
5235
047e0030 5236 igb_write_itr(q_vector);
9d5c8243
AK		 5237
5238 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5239 * not set, then the adapter didn't send an interrupt */
5240 if (!(icr & E1000_ICR_INT_ASSERTED))
5241 return IRQ_NONE;
5242
7f081d40
AD		 5243 if (icr & E1000_ICR_DRSTA)
5244 schedule_work(&adapter->reset_task);
5245
047e0030 5246 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5247 /* HW is reporting DMA is out of sync */
5248 adapter->stats.doosync++;
5249 }
5250
9d5c8243
AK		 5251 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5252 hw->mac.get_link_status = 1;
5253 /* guard against interrupt when we're going down */
5254 if (!test_bit(__IGB_DOWN, &adapter->state))
5255 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5256 }
5257
047e0030 5258 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5259
5260 return IRQ_HANDLED;
5261}
5262
047e0030 5263static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 5264{
047e0030 5265 struct igb_adapter *adapter = q_vector->adapter;
46544258 5266 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5267
4fc82adf
AD		 5268 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
5269 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
047e0030 5270 if (!adapter->msix_entries)
6eb5a7f1 5271 igb_set_itr(adapter);
46544258 5272 else
047e0030 5273 igb_update_ring_itr(q_vector);
9d5c8243
AK		 5274 }
5275
46544258
AD		 5276 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5277 if (adapter->msix_entries)
047e0030 5278 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 5279 else
5280 igb_irq_enable(adapter);
5281 }
9d5c8243
AK		 5282}
5283
46544258
AD		 5284/**
5285 * igb_poll - NAPI Rx polling callback
5286 * @napi: napi polling structure
5287 * @budget: count of how many packets we should handle
5288 **/
5289static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 5290{
047e0030
AD		 5291 struct igb_q_vector *q_vector = container_of(napi,
5292 struct igb_q_vector,
5293 napi);
5294 int tx_clean_complete = 1, work_done = 0;
9d5c8243 5295
421e02f0 5296#ifdef CONFIG_IGB_DCA
047e0030
AD		 5297 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5298 igb_update_dca(q_vector);
fe4506b6 5299#endif
047e0030
AD		 5300 if (q_vector->tx_ring)
5301 tx_clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 5302
047e0030
AD		 5303 if (q_vector->rx_ring)
5304 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
5305
5306 if (!tx_clean_complete)
5307 work_done = budget;
46544258 5308
9d5c8243 5309 /* If not enough Rx work done, exit the polling mode */
5e6d5b17 5310 if (work_done < budget) {
288379f0 5311 napi_complete(napi);
047e0030 5312 igb_ring_irq_enable(q_vector);
9d5c8243
AK		 5313 }
5314
46544258 5315 return work_done;
9d5c8243 5316}
6d8126f9 5317
33af6bcc 5318/**
c5b9bd5e 5319 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 5320 * @adapter: board private structure
c5b9bd5e
AD		 5321 * @shhwtstamps: timestamp structure to update
5322 * @regval: unsigned 64bit system time value.
5323 *
5324 * We need to convert the system time value stored in the RX/TXSTMP registers
5325 * into a hwtstamp which can be used by the upper level timestamping functions
5326 */
5327static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5328 struct skb_shared_hwtstamps *shhwtstamps,
5329 u64 regval)
5330{
5331 u64 ns;
5332
55cac248
AD		 5333 /*
5334 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5335 * 24 to match clock shift we setup earlier.
5336 */
5337 if (adapter->hw.mac.type == e1000_82580)
5338 regval <<= IGB_82580_TSYNC_SHIFT;
5339
c5b9bd5e
AD		 5340 ns = timecounter_cyc2time(&adapter->clock, regval);
5341 timecompare_update(&adapter->compare, ns);
5342 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5343 shhwtstamps->hwtstamp = ns_to_ktime(ns);
5344 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5345}
5346
5347/**
5348 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5349 * @q_vector: pointer to q_vector containing needed info
2873957d 5350 * @buffer: pointer to igb_buffer structure
33af6bcc
PO		 5351 *
5352 * If we were asked to do hardware stamping and such a time stamp is
5353 * available, then it must have been for this skb here because we only
5354 * allow only one such packet into the queue.
5355 */
2873957d 5356static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct igb_buffer *buffer_info)
33af6bcc 5357{
c5b9bd5e 5358 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc 5359 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 5360 struct skb_shared_hwtstamps shhwtstamps;
5361 u64 regval;
33af6bcc 5362
c5b9bd5e 5363 /* if skb does not support hw timestamp or TX stamp not valid exit */
2244d07b 5364 if (likely(!(buffer_info->tx_flags & SKBTX_HW_TSTAMP)) ||
c5b9bd5e
AD		 5365 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5366 return;
5367
5368 regval = rd32(E1000_TXSTMPL);
5369 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5370
5371 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
2873957d 5372 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
33af6bcc
PO		 5373}
5374
9d5c8243
AK		 5375/**
5376 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 5377 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 5378 * returns true if ring is completely cleaned
5379 **/
047e0030 5380static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 5381{
047e0030
AD		 5382 struct igb_adapter *adapter = q_vector->adapter;
5383 struct igb_ring *tx_ring = q_vector->tx_ring;
e694e964 5384 struct net_device *netdev = tx_ring->netdev;
0e014cb1 5385 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5386 struct igb_buffer *buffer_info;
0e014cb1 5387 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 5388 unsigned int total_bytes = 0, total_packets = 0;
0e014cb1
AD		 5389 unsigned int i, eop, count = 0;
5390 bool cleaned = false;
9d5c8243 5391
9d5c8243 5392 i = tx_ring->next_to_clean;
0e014cb1
AD		 5393 eop = tx_ring->buffer_info[i].next_to_watch;
5394 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5395
5396 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5397 (count < tx_ring->count)) {
2d0bb1c1 5398 rmb(); /* read buffer_info after eop_desc status */
0e014cb1
AD		 5399 for (cleaned = false; !cleaned; count++) {
5400 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
9d5c8243 5401 buffer_info = &tx_ring->buffer_info[i];
0e014cb1 5402 cleaned = (i == eop);
9d5c8243 5403
2873957d
NN		 5404 if (buffer_info->skb) {
5405 total_bytes += buffer_info->bytecount;
9d5c8243 5406 /* gso_segs is currently only valid for tcp */
2873957d
NN		 5407 total_packets += buffer_info->gso_segs;
5408 igb_tx_hwtstamp(q_vector, buffer_info);
9d5c8243
AK		 5409 }
5410
80785298 5411 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
0e014cb1 5412 tx_desc->wb.status = 0;
9d5c8243
AK		 5413
5414 i++;
5415 if (i == tx_ring->count)
5416 i = 0;
9d5c8243 5417 }
0e014cb1
AD		 5418 eop = tx_ring->buffer_info[i].next_to_watch;
5419 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5420 }
5421
9d5c8243
AK		 5422 tx_ring->next_to_clean = i;
5423
fc7d345d 5424 if (unlikely(count &&
9d5c8243 5425 netif_carrier_ok(netdev) &&
c493ea45 5426 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
9d5c8243
AK		 5427 /* Make sure that anybody stopping the queue after this
5428 * sees the new next_to_clean.
5429 */
5430 smp_mb();
661086df
PWJ		 5431 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5432 !(test_bit(__IGB_DOWN, &adapter->state))) {
5433 netif_wake_subqueue(netdev, tx_ring->queue_index);
12dcd86b
ED		 5434
5435 u64_stats_update_begin(&tx_ring->tx_syncp);
04a5fcaa 5436 tx_ring->tx_stats.restart_queue++;
12dcd86b 5437 u64_stats_update_end(&tx_ring->tx_syncp);
661086df 5438 }
9d5c8243
AK		 5439 }
5440
5441 if (tx_ring->detect_tx_hung) {
5442 /* Detect a transmit hang in hardware, this serializes the
5443 * check with the clearing of time_stamp and movement of i */
5444 tx_ring->detect_tx_hung = false;
5445 if (tx_ring->buffer_info[i].time_stamp &&
5446 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
8e95a202
JP		 5447 (adapter->tx_timeout_factor * HZ)) &&
5448 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5449
9d5c8243 5450 /* detected Tx unit hang */
59d71989 5451 dev_err(tx_ring->dev,
9d5c8243 5452 "Detected Tx Unit Hang\n"
2d064c06 5453 " Tx Queue <%d>\n"
9d5c8243
AK		 5454 " TDH <%x>\n"
5455 " TDT <%x>\n"
5456 " next_to_use <%x>\n"
5457 " next_to_clean <%x>\n"
9d5c8243
AK		 5458 "buffer_info[next_to_clean]\n"
5459 " time_stamp <%lx>\n"
0e014cb1 5460 " next_to_watch <%x>\n"
9d5c8243
AK		 5461 " jiffies <%lx>\n"
5462 " desc.status <%x>\n",
2d064c06 5463 tx_ring->queue_index,
fce99e34
AD		 5464 readl(tx_ring->head),
5465 readl(tx_ring->tail),
9d5c8243
AK		 5466 tx_ring->next_to_use,
5467 tx_ring->next_to_clean,
f7ba205e 5468 tx_ring->buffer_info[eop].time_stamp,
0e014cb1 5469 eop,
9d5c8243 5470 jiffies,
0e014cb1 5471 eop_desc->wb.status);
661086df 5472 netif_stop_subqueue(netdev, tx_ring->queue_index);
9d5c8243
AK		 5473 }
5474 }
5475 tx_ring->total_bytes += total_bytes;
5476 tx_ring->total_packets += total_packets;
12dcd86b 5477 u64_stats_update_begin(&tx_ring->tx_syncp);
e21ed353
AD		 5478 tx_ring->tx_stats.bytes += total_bytes;
5479 tx_ring->tx_stats.packets += total_packets;
12dcd86b 5480 u64_stats_update_end(&tx_ring->tx_syncp);
807540ba 5481 return count < tx_ring->count;
9d5c8243
AK		 5482}
5483
9d5c8243
AK		 5484/**
5485 * igb_receive_skb - helper function to handle rx indications
047e0030
AD		 5486 * @q_vector: structure containing interrupt and ring information
5487 * @skb: packet to send up
5488 * @vlan_tag: vlan tag for packet
9d5c8243 5489 **/
047e0030
AD		 5490static void igb_receive_skb(struct igb_q_vector *q_vector,
5491 struct sk_buff *skb,
5492 u16 vlan_tag)
5493{
5494 struct igb_adapter *adapter = q_vector->adapter;
5495
31b24b95 5496 if (vlan_tag && adapter->vlgrp)
047e0030
AD		 5497 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5498 vlan_tag, skb);
182ff8df 5499 else
047e0030 5500 napi_gro_receive(&q_vector->napi, skb);
9d5c8243
AK		 5501}
5502
04a5fcaa 5503static inline void igb_rx_checksum_adv(struct igb_ring *ring,
9d5c8243
AK		 5504 u32 status_err, struct sk_buff *skb)
5505{
bc8acf2c 5506 skb_checksum_none_assert(skb);
9d5c8243
AK		 5507
5508 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 5509 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5510 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 5511 return;
85ad76b2 5512
9d5c8243
AK		 5513 /* TCP/UDP checksum error bit is set */
5514 if (status_err &
5515 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5516 /*
5517 * work around errata with sctp packets where the TCPE aka
5518 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5519 * packets, (aka let the stack check the crc32c)
5520 */
85ad76b2 5521 if ((skb->len == 60) &&
12dcd86b
ED		 5522 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM)) {
5523 u64_stats_update_begin(&ring->rx_syncp);
04a5fcaa 5524 ring->rx_stats.csum_err++;
12dcd86b
ED		 5525 u64_stats_update_end(&ring->rx_syncp);
5526 }
9d5c8243 5527 /* let the stack verify checksum errors */
9d5c8243
AK		 5528 return;
5529 }
5530 /* It must be a TCP or UDP packet with a valid checksum */
5531 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5532 skb->ip_summed = CHECKSUM_UNNECESSARY;
5533
59d71989 5534 dev_dbg(ring->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 5535}
5536
757b77e2 5537static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
c5b9bd5e
AD		 5538 struct sk_buff *skb)
5539{
5540 struct igb_adapter *adapter = q_vector->adapter;
5541 struct e1000_hw *hw = &adapter->hw;
5542 u64 regval;
5543
5544 /*
5545 * If this bit is set, then the RX registers contain the time stamp. No
5546 * other packet will be time stamped until we read these registers, so
5547 * read the registers to make them available again. Because only one
5548 * packet can be time stamped at a time, we know that the register
5549 * values must belong to this one here and therefore we don't need to
5550 * compare any of the additional attributes stored for it.
5551 *
2244d07b 5552 * If nothing went wrong, then it should have a shared tx_flags that we
c5b9bd5e
AD		 5553 * can turn into a skb_shared_hwtstamps.
5554 */
757b77e2
NN		 5555 if (staterr & E1000_RXDADV_STAT_TSIP) {
5556 u32 *stamp = (u32 *)skb->data;
5557 regval = le32_to_cpu(*(stamp + 2));
5558 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5559 skb_pull(skb, IGB_TS_HDR_LEN);
5560 } else {
5561 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5562 return;
c5b9bd5e 5563
757b77e2
NN		 5564 regval = rd32(E1000_RXSTMPL);
5565 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5566 }
c5b9bd5e
AD		 5567
5568 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5569}
4c844851 5570static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
2d94d8ab
AD		 5571 union e1000_adv_rx_desc *rx_desc)
5572{
5573 /* HW will not DMA in data larger than the given buffer, even if it
5574 * parses the (NFS, of course) header to be larger. In that case, it
5575 * fills the header buffer and spills the rest into the page.
5576 */
5577 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5578 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4c844851
AD		 5579 if (hlen > rx_ring->rx_buffer_len)
5580 hlen = rx_ring->rx_buffer_len;
2d94d8ab
AD		 5581 return hlen;
5582}
5583
047e0030
AD		 5584static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5585 int *work_done, int budget)
9d5c8243 5586{
047e0030 5587 struct igb_ring *rx_ring = q_vector->rx_ring;
e694e964 5588 struct net_device *netdev = rx_ring->netdev;
59d71989 5589 struct device *dev = rx_ring->dev;
9d5c8243
AK		 5590 union e1000_adv_rx_desc *rx_desc , *next_rxd;
5591 struct igb_buffer *buffer_info , *next_buffer;
5592 struct sk_buff *skb;
9d5c8243
AK		 5593 bool cleaned = false;
5594 int cleaned_count = 0;
d1eff350 5595 int current_node = numa_node_id();
9d5c8243 5596 unsigned int total_bytes = 0, total_packets = 0;
73cd78f1 5597 unsigned int i;
2d94d8ab
AD		 5598 u32 staterr;
5599 u16 length;
047e0030 5600 u16 vlan_tag;
9d5c8243
AK		 5601
5602 i = rx_ring->next_to_clean;
69d3ca53 5603 buffer_info = &rx_ring->buffer_info[i];
9d5c8243
AK		 5604 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5605 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5606
5607 while (staterr & E1000_RXD_STAT_DD) {
5608 if (*work_done >= budget)
5609 break;
5610 (*work_done)++;
2d0bb1c1 5611 rmb(); /* read descriptor and rx_buffer_info after status DD */
9d5c8243 5612
69d3ca53
AD		 5613 skb = buffer_info->skb;
5614 prefetch(skb->data - NET_IP_ALIGN);
5615 buffer_info->skb = NULL;
5616
5617 i++;
5618 if (i == rx_ring->count)
5619 i = 0;
42d0781a 5620
69d3ca53
AD		 5621 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5622 prefetch(next_rxd);
5623 next_buffer = &rx_ring->buffer_info[i];
9d5c8243
AK		 5624
5625 length = le16_to_cpu(rx_desc->wb.upper.length);
5626 cleaned = true;
5627 cleaned_count++;
5628
2d94d8ab 5629 if (buffer_info->dma) {
59d71989 5630 dma_unmap_single(dev, buffer_info->dma,
4c844851 5631 rx_ring->rx_buffer_len,
59d71989 5632 DMA_FROM_DEVICE);
91615f76 5633 buffer_info->dma = 0;
4c844851 5634 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
6ec43fe6
AD		 5635 skb_put(skb, length);
5636 goto send_up;
5637 }
4c844851 5638 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
bf36c1a0
AD		 5639 }
5640
5641 if (length) {
59d71989
AD		 5642 dma_unmap_page(dev, buffer_info->page_dma,
5643 PAGE_SIZE / 2, DMA_FROM_DEVICE);
9d5c8243 5644 buffer_info->page_dma = 0;
bf36c1a0 5645
aa913403 5646 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
bf36c1a0
AD		 5647 buffer_info->page,
5648 buffer_info->page_offset,
5649 length);
5650
d1eff350
AD		 5651 if ((page_count(buffer_info->page) != 1) ||
5652 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 5653 buffer_info->page = NULL;
5654 else
5655 get_page(buffer_info->page);
9d5c8243
AK		 5656
5657 skb->len += length;
5658 skb->data_len += length;
bf36c1a0 5659 skb->truesize += length;
9d5c8243 5660 }
9d5c8243 5661
bf36c1a0 5662 if (!(staterr & E1000_RXD_STAT_EOP)) {
b2d56536
AD		 5663 buffer_info->skb = next_buffer->skb;
5664 buffer_info->dma = next_buffer->dma;
5665 next_buffer->skb = skb;
5666 next_buffer->dma = 0;
bf36c1a0
AD		 5667 goto next_desc;
5668 }
69d3ca53 5669send_up:
9d5c8243
AK		 5670 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5671 dev_kfree_skb_irq(skb);
5672 goto next_desc;
5673 }
9d5c8243 5674
757b77e2
NN		 5675 if (staterr & (E1000_RXDADV_STAT_TSIP | E1000_RXDADV_STAT_TS))
5676 igb_rx_hwtstamp(q_vector, staterr, skb);
9d5c8243
AK		 5677 total_bytes += skb->len;
5678 total_packets++;
5679
85ad76b2 5680 igb_rx_checksum_adv(rx_ring, staterr, skb);
9d5c8243
AK		 5681
5682 skb->protocol = eth_type_trans(skb, netdev);
047e0030
AD		 5683 skb_record_rx_queue(skb, rx_ring->queue_index);
5684
5685 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5686 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
9d5c8243 5687
047e0030 5688 igb_receive_skb(q_vector, skb, vlan_tag);
9d5c8243 5689
9d5c8243
AK		 5690next_desc:
5691 rx_desc->wb.upper.status_error = 0;
5692
5693 /* return some buffers to hardware, one at a time is too slow */
5694 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
3b644cf6 5695 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5696 cleaned_count = 0;
5697 }
5698
5699 /* use prefetched values */
5700 rx_desc = next_rxd;
5701 buffer_info = next_buffer;
9d5c8243
AK		 5702 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5703 }
bf36c1a0 5704
9d5c8243 5705 rx_ring->next_to_clean = i;
c493ea45 5706 cleaned_count = igb_desc_unused(rx_ring);
9d5c8243
AK		 5707
5708 if (cleaned_count)
3b644cf6 5709 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5710
5711 rx_ring->total_packets += total_packets;
5712 rx_ring->total_bytes += total_bytes;
12dcd86b 5713 u64_stats_update_begin(&rx_ring->rx_syncp);
9d5c8243
AK		 5714 rx_ring->rx_stats.packets += total_packets;
5715 rx_ring->rx_stats.bytes += total_bytes;
12dcd86b 5716 u64_stats_update_end(&rx_ring->rx_syncp);
9d5c8243
AK		 5717 return cleaned;
5718}
5719
9d5c8243
AK		 5720/**
5721 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5722 * @adapter: address of board private structure
5723 **/
d7ee5b3a 5724void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
9d5c8243 5725{
e694e964 5726 struct net_device *netdev = rx_ring->netdev;
9d5c8243
AK		 5727 union e1000_adv_rx_desc *rx_desc;
5728 struct igb_buffer *buffer_info;
5729 struct sk_buff *skb;
5730 unsigned int i;
db761762 5731 int bufsz;
9d5c8243
AK		 5732
5733 i = rx_ring->next_to_use;
5734 buffer_info = &rx_ring->buffer_info[i];
5735
4c844851 5736 bufsz = rx_ring->rx_buffer_len;
db761762 5737
9d5c8243
AK		 5738 while (cleaned_count--) {
5739 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5740
6ec43fe6 5741 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
9d5c8243 5742 if (!buffer_info->page) {
42d0781a 5743 buffer_info->page = netdev_alloc_page(netdev);
12dcd86b
ED		 5744 if (unlikely(!buffer_info->page)) {
5745 u64_stats_update_begin(&rx_ring->rx_syncp);
04a5fcaa 5746 rx_ring->rx_stats.alloc_failed++;
12dcd86b 5747 u64_stats_update_end(&rx_ring->rx_syncp);
bf36c1a0
AD		 5748 goto no_buffers;
5749 }
5750 buffer_info->page_offset = 0;
5751 } else {
5752 buffer_info->page_offset ^= PAGE_SIZE / 2;
9d5c8243
AK		 5753 }
5754 buffer_info->page_dma =
59d71989 5755 dma_map_page(rx_ring->dev, buffer_info->page,
bf36c1a0
AD		 5756 buffer_info->page_offset,
5757 PAGE_SIZE / 2,
59d71989
AD		 5758 DMA_FROM_DEVICE);
5759 if (dma_mapping_error(rx_ring->dev,
5760 buffer_info->page_dma)) {
42d0781a 5761 buffer_info->page_dma = 0;
12dcd86b 5762 u64_stats_update_begin(&rx_ring->rx_syncp);
42d0781a 5763 rx_ring->rx_stats.alloc_failed++;
12dcd86b 5764 u64_stats_update_end(&rx_ring->rx_syncp);
42d0781a
AD		 5765 goto no_buffers;
5766 }
9d5c8243
AK		 5767 }
5768
42d0781a
AD		 5769 skb = buffer_info->skb;
5770 if (!skb) {
89d71a66 5771 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
12dcd86b
ED		 5772 if (unlikely(!skb)) {
5773 u64_stats_update_begin(&rx_ring->rx_syncp);
04a5fcaa 5774 rx_ring->rx_stats.alloc_failed++;
12dcd86b 5775 u64_stats_update_end(&rx_ring->rx_syncp);
9d5c8243
AK		 5776 goto no_buffers;
5777 }
5778
9d5c8243 5779 buffer_info->skb = skb;
42d0781a
AD		 5780 }
5781 if (!buffer_info->dma) {
59d71989 5782 buffer_info->dma = dma_map_single(rx_ring->dev,
80785298 5783 skb->data,
9d5c8243 5784 bufsz,
59d71989
AD		 5785 DMA_FROM_DEVICE);
5786 if (dma_mapping_error(rx_ring->dev,
5787 buffer_info->dma)) {
42d0781a 5788 buffer_info->dma = 0;
12dcd86b 5789 u64_stats_update_begin(&rx_ring->rx_syncp);
42d0781a 5790 rx_ring->rx_stats.alloc_failed++;
12dcd86b 5791 u64_stats_update_end(&rx_ring->rx_syncp);
42d0781a
AD		 5792 goto no_buffers;
5793 }
9d5c8243
AK		 5794 }
5795 /* Refresh the desc even if buffer_addrs didn't change because
5796 * each write-back erases this info. */
6ec43fe6 5797 if (bufsz < IGB_RXBUFFER_1024) {
9d5c8243
AK		 5798 rx_desc->read.pkt_addr =
5799 cpu_to_le64(buffer_info->page_dma);
5800 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5801 } else {
42d0781a 5802 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
9d5c8243
AK		 5803 rx_desc->read.hdr_addr = 0;
5804 }
5805
5806 i++;
5807 if (i == rx_ring->count)
5808 i = 0;
5809 buffer_info = &rx_ring->buffer_info[i];
5810 }
5811
5812no_buffers:
5813 if (rx_ring->next_to_use != i) {
5814 rx_ring->next_to_use = i;
5815 if (i == 0)
5816 i = (rx_ring->count - 1);
5817 else
5818 i--;
5819
5820 /* Force memory writes to complete before letting h/w
5821 * know there are new descriptors to fetch. (Only
5822 * applicable for weak-ordered memory model archs,
5823 * such as IA-64). */
5824 wmb();
fce99e34 5825 writel(i, rx_ring->tail);
9d5c8243
AK		 5826 }
5827}
5828
5829/**
5830 * igb_mii_ioctl -
5831 * @netdev:
5832 * @ifreq:
5833 * @cmd:
5834 **/
5835static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5836{
5837 struct igb_adapter *adapter = netdev_priv(netdev);
5838 struct mii_ioctl_data *data = if_mii(ifr);
5839
5840 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5841 return -EOPNOTSUPP;
5842
5843 switch (cmd) {
5844 case SIOCGMIIPHY:
5845 data->phy_id = adapter->hw.phy.addr;
5846 break;
5847 case SIOCGMIIREG:
f5f4cf08
AD		 5848 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5849 &data->val_out))
9d5c8243
AK		 5850 return -EIO;
5851 break;
5852 case SIOCSMIIREG:
5853 default:
5854 return -EOPNOTSUPP;
5855 }
5856 return 0;
5857}
5858
c6cb090b
PO		 5859/**
5860 * igb_hwtstamp_ioctl - control hardware time stamping
5861 * @netdev:
5862 * @ifreq:
5863 * @cmd:
5864 *
33af6bcc
PO		 5865 * Outgoing time stamping can be enabled and disabled. Play nice and
5866 * disable it when requested, although it shouldn't case any overhead
5867 * when no packet needs it. At most one packet in the queue may be
5868 * marked for time stamping, otherwise it would be impossible to tell
5869 * for sure to which packet the hardware time stamp belongs.
5870 *
5871 * Incoming time stamping has to be configured via the hardware
5872 * filters. Not all combinations are supported, in particular event
5873 * type has to be specified. Matching the kind of event packet is
5874 * not supported, with the exception of "all V2 events regardless of
5875 * level 2 or 4".
5876 *
c6cb090b
PO		 5877 **/
5878static int igb_hwtstamp_ioctl(struct net_device *netdev,
5879 struct ifreq *ifr, int cmd)
5880{
33af6bcc
PO		 5881 struct igb_adapter *adapter = netdev_priv(netdev);
5882 struct e1000_hw *hw = &adapter->hw;
c6cb090b 5883 struct hwtstamp_config config;
c5b9bd5e
AD		 5884 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5885 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 5886 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 5887 bool is_l4 = false;
5888 bool is_l2 = false;
33af6bcc 5889 u32 regval;
c6cb090b
PO		 5890
5891 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5892 return -EFAULT;
5893
5894 /* reserved for future extensions */
5895 if (config.flags)
5896 return -EINVAL;
5897
33af6bcc
PO		 5898 switch (config.tx_type) {
5899 case HWTSTAMP_TX_OFF:
c5b9bd5e 5900 tsync_tx_ctl = 0;
33af6bcc 5901 case HWTSTAMP_TX_ON:
33af6bcc
PO		 5902 break;
5903 default:
5904 return -ERANGE;
5905 }
5906
5907 switch (config.rx_filter) {
5908 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 5909 tsync_rx_ctl = 0;
33af6bcc
PO		 5910 break;
5911 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5912 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5913 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5914 case HWTSTAMP_FILTER_ALL:
5915 /*
5916 * register TSYNCRXCFG must be set, therefore it is not
5917 * possible to time stamp both Sync and Delay_Req messages
5918 * => fall back to time stamping all packets
5919 */
c5b9bd5e 5920 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 5921 config.rx_filter = HWTSTAMP_FILTER_ALL;
5922 break;
5923 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 5924 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5925 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 5926 is_l4 = true;
33af6bcc
PO		 5927 break;
5928 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 5929 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5930 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 5931 is_l4 = true;
33af6bcc
PO		 5932 break;
5933 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5934 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 5935 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5936 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 5937 is_l2 = true;
5938 is_l4 = true;
33af6bcc
PO		 5939 config.rx_filter = HWTSTAMP_FILTER_SOME;
5940 break;
5941 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5942 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 5943 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5944 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 5945 is_l2 = true;
5946 is_l4 = true;
33af6bcc
PO		 5947 config.rx_filter = HWTSTAMP_FILTER_SOME;
5948 break;
5949 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5950 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5951 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 5952 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 5953 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 5954 is_l2 = true;
33af6bcc
PO		 5955 break;
5956 default:
5957 return -ERANGE;
5958 }
5959
c5b9bd5e
AD		 5960 if (hw->mac.type == e1000_82575) {
5961 if (tsync_rx_ctl | tsync_tx_ctl)
5962 return -EINVAL;
5963 return 0;
5964 }
5965
757b77e2
NN		 5966 /*
5967 * Per-packet timestamping only works if all packets are
5968 * timestamped, so enable timestamping in all packets as
5969 * long as one rx filter was configured.
5970 */
5971 if ((hw->mac.type == e1000_82580) && tsync_rx_ctl) {
5972 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5973 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5974 }
5975
33af6bcc
PO		 5976 /* enable/disable TX */
5977 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 5978 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5979 regval |= tsync_tx_ctl;
33af6bcc
PO		 5980 wr32(E1000_TSYNCTXCTL, regval);
5981
c5b9bd5e 5982 /* enable/disable RX */
33af6bcc 5983 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 5984 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5985 regval |= tsync_rx_ctl;
33af6bcc 5986 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 5987
c5b9bd5e
AD		 5988 /* define which PTP packets are time stamped */
5989 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 5990
c5b9bd5e
AD		 5991 /* define ethertype filter for timestamped packets */
5992 if (is_l2)
5993 wr32(E1000_ETQF(3),
5994 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5995 E1000_ETQF_1588 | /* enable timestamping */
5996 ETH_P_1588)); /* 1588 eth protocol type */
5997 else
5998 wr32(E1000_ETQF(3), 0);
5999
6000#define PTP_PORT 319
6001 /* L4 Queue Filter[3]: filter by destination port and protocol */
6002 if (is_l4) {
6003 u32 ftqf = (IPPROTO_UDP /* UDP */
6004 | E1000_FTQF_VF_BP /* VF not compared */
6005 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
6006 | E1000_FTQF_MASK); /* mask all inputs */
6007 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
6008
6009 wr32(E1000_IMIR(3), htons(PTP_PORT));
6010 wr32(E1000_IMIREXT(3),
6011 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
6012 if (hw->mac.type == e1000_82576) {
6013 /* enable source port check */
6014 wr32(E1000_SPQF(3), htons(PTP_PORT));
6015 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
6016 }
6017 wr32(E1000_FTQF(3), ftqf);
6018 } else {
6019 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
6020 }
33af6bcc
PO		 6021 wrfl();
6022
6023 adapter->hwtstamp_config = config;
6024
6025 /* clear TX/RX time stamp registers, just to be sure */
6026 regval = rd32(E1000_TXSTMPH);
6027 regval = rd32(E1000_RXSTMPH);
c6cb090b 6028
33af6bcc
PO		 6029 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
6030 -EFAULT : 0;
c6cb090b
PO		 6031}
6032
9d5c8243
AK		 6033/**
6034 * igb_ioctl -
6035 * @netdev:
6036 * @ifreq:
6037 * @cmd:
6038 **/
6039static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6040{
6041 switch (cmd) {
6042 case SIOCGMIIPHY:
6043 case SIOCGMIIREG:
6044 case SIOCSMIIREG:
6045 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 6046 case SIOCSHWTSTAMP:
6047 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 6048 default:
6049 return -EOPNOTSUPP;
6050 }
6051}
6052
009bc06e
AD		 6053s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6054{
6055 struct igb_adapter *adapter = hw->back;
6056 u16 cap_offset;
6057
6058 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
6059 if (!cap_offset)
6060 return -E1000_ERR_CONFIG;
6061
6062 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6063
6064 return 0;
6065}
6066
6067s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6068{
6069 struct igb_adapter *adapter = hw->back;
6070 u16 cap_offset;
6071
6072 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
6073 if (!cap_offset)
6074 return -E1000_ERR_CONFIG;
6075
6076 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6077
6078 return 0;
6079}
6080
9d5c8243
AK		 6081static void igb_vlan_rx_register(struct net_device *netdev,
6082 struct vlan_group *grp)
6083{
6084 struct igb_adapter *adapter = netdev_priv(netdev);
6085 struct e1000_hw *hw = &adapter->hw;
6086 u32 ctrl, rctl;
6087
6088 igb_irq_disable(adapter);
6089 adapter->vlgrp = grp;
6090
6091 if (grp) {
6092 /* enable VLAN tag insert/strip */
6093 ctrl = rd32(E1000_CTRL);
6094 ctrl |= E1000_CTRL_VME;
6095 wr32(E1000_CTRL, ctrl);
6096
51466239 6097 /* Disable CFI check */
9d5c8243 6098 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 6099 rctl &= ~E1000_RCTL_CFIEN;
6100 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 6101 } else {
6102 /* disable VLAN tag insert/strip */
6103 ctrl = rd32(E1000_CTRL);
6104 ctrl &= ~E1000_CTRL_VME;
6105 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 6106 }
6107
e1739522
AD		 6108 igb_rlpml_set(adapter);
6109
9d5c8243
AK		 6110 if (!test_bit(__IGB_DOWN, &adapter->state))
6111 igb_irq_enable(adapter);
6112}
6113
6114static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6115{
6116 struct igb_adapter *adapter = netdev_priv(netdev);
6117 struct e1000_hw *hw = &adapter->hw;
4ae196df 6118 int pf_id = adapter->vfs_allocated_count;
9d5c8243 6119
51466239
AD		 6120 /* attempt to add filter to vlvf array */
6121 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 6122
51466239
AD		 6123 /* add the filter since PF can receive vlans w/o entry in vlvf */
6124 igb_vfta_set(hw, vid, true);
9d5c8243
AK		 6125}
6126
6127static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6128{
6129 struct igb_adapter *adapter = netdev_priv(netdev);
6130 struct e1000_hw *hw = &adapter->hw;
4ae196df 6131 int pf_id = adapter->vfs_allocated_count;
51466239 6132 s32 err;
9d5c8243
AK		 6133
6134 igb_irq_disable(adapter);
6135 vlan_group_set_device(adapter->vlgrp, vid, NULL);
6136
6137 if (!test_bit(__IGB_DOWN, &adapter->state))
6138 igb_irq_enable(adapter);
6139
51466239
AD		 6140 /* remove vlan from VLVF table array */
6141 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 6142
51466239
AD		 6143 /* if vid was not present in VLVF just remove it from table */
6144 if (err)
4ae196df 6145 igb_vfta_set(hw, vid, false);
9d5c8243
AK		 6146}
6147
6148static void igb_restore_vlan(struct igb_adapter *adapter)
6149{
6150 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
6151
6152 if (adapter->vlgrp) {
6153 u16 vid;
b738127d 6154 for (vid = 0; vid < VLAN_N_VID; vid++) {
9d5c8243
AK		 6155 if (!vlan_group_get_device(adapter->vlgrp, vid))
6156 continue;
6157 igb_vlan_rx_add_vid(adapter->netdev, vid);
6158 }
6159 }
6160}
6161
6162int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
6163{
090b1795 6164 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 6165 struct e1000_mac_info *mac = &adapter->hw.mac;
6166
6167 mac->autoneg = 0;
6168
cd2638a8
CW		 6169 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6170 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
6171 spddplx != (SPEED_1000 + DUPLEX_FULL)) {
6172 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6173 return -EINVAL;
6174 }
6175
9d5c8243
AK		 6176 switch (spddplx) {
6177 case SPEED_10 + DUPLEX_HALF:
6178 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6179 break;
6180 case SPEED_10 + DUPLEX_FULL:
6181 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6182 break;
6183 case SPEED_100 + DUPLEX_HALF:
6184 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6185 break;
6186 case SPEED_100 + DUPLEX_FULL:
6187 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6188 break;
6189 case SPEED_1000 + DUPLEX_FULL:
6190 mac->autoneg = 1;
6191 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6192 break;
6193 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6194 default:
090b1795 6195 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
9d5c8243
AK		 6196 return -EINVAL;
6197 }
6198 return 0;
6199}
6200
3fe7c4c9 6201static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 6202{
6203 struct net_device *netdev = pci_get_drvdata(pdev);
6204 struct igb_adapter *adapter = netdev_priv(netdev);
6205 struct e1000_hw *hw = &adapter->hw;
2d064c06 6206 u32 ctrl, rctl, status;
9d5c8243
AK		 6207 u32 wufc = adapter->wol;
6208#ifdef CONFIG_PM
6209 int retval = 0;
6210#endif
6211
6212 netif_device_detach(netdev);
6213
a88f10ec
AD		 6214 if (netif_running(netdev))
6215 igb_close(netdev);
6216
047e0030 6217 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 6218
6219#ifdef CONFIG_PM
6220 retval = pci_save_state(pdev);
6221 if (retval)
6222 return retval;
6223#endif
6224
6225 status = rd32(E1000_STATUS);
6226 if (status & E1000_STATUS_LU)
6227 wufc &= ~E1000_WUFC_LNKC;
6228
6229 if (wufc) {
6230 igb_setup_rctl(adapter);
ff41f8dc 6231 igb_set_rx_mode(netdev);
9d5c8243
AK		 6232
6233 /* turn on all-multi mode if wake on multicast is enabled */
6234 if (wufc & E1000_WUFC_MC) {
6235 rctl = rd32(E1000_RCTL);
6236 rctl |= E1000_RCTL_MPE;
6237 wr32(E1000_RCTL, rctl);
6238 }
6239
6240 ctrl = rd32(E1000_CTRL);
6241 /* advertise wake from D3Cold */
6242 #define E1000_CTRL_ADVD3WUC 0x00100000
6243 /* phy power management enable */
6244 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6245 ctrl |= E1000_CTRL_ADVD3WUC;
6246 wr32(E1000_CTRL, ctrl);
6247
9d5c8243 6248 /* Allow time for pending master requests to run */
330a6d6a 6249 igb_disable_pcie_master(hw);
9d5c8243
AK		 6250
6251 wr32(E1000_WUC, E1000_WUC_PME_EN);
6252 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 6253 } else {
6254 wr32(E1000_WUC, 0);
6255 wr32(E1000_WUFC, 0);
9d5c8243
AK		 6256 }
6257
3fe7c4c9
RW		 6258 *enable_wake = wufc || adapter->en_mng_pt;
6259 if (!*enable_wake)
88a268c1
NN		 6260 igb_power_down_link(adapter);
6261 else
6262 igb_power_up_link(adapter);
9d5c8243
AK		 6263
6264 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6265 * would have already happened in close and is redundant. */
6266 igb_release_hw_control(adapter);
6267
6268 pci_disable_device(pdev);
6269
9d5c8243
AK		 6270 return 0;
6271}
6272
6273#ifdef CONFIG_PM
3fe7c4c9
RW		 6274static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
6275{
6276 int retval;
6277 bool wake;
6278
6279 retval = __igb_shutdown(pdev, &wake);
6280 if (retval)
6281 return retval;
6282
6283 if (wake) {
6284 pci_prepare_to_sleep(pdev);
6285 } else {
6286 pci_wake_from_d3(pdev, false);
6287 pci_set_power_state(pdev, PCI_D3hot);
6288 }
6289
6290 return 0;
6291}
6292
9d5c8243
AK		 6293static int igb_resume(struct pci_dev *pdev)
6294{
6295 struct net_device *netdev = pci_get_drvdata(pdev);
6296 struct igb_adapter *adapter = netdev_priv(netdev);
6297 struct e1000_hw *hw = &adapter->hw;
6298 u32 err;
6299
6300 pci_set_power_state(pdev, PCI_D0);
6301 pci_restore_state(pdev);
b94f2d77 6302 pci_save_state(pdev);
42bfd33a 6303
aed5dec3 6304 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 6305 if (err) {
6306 dev_err(&pdev->dev,
6307 "igb: Cannot enable PCI device from suspend\n");
6308 return err;
6309 }
6310 pci_set_master(pdev);
6311
6312 pci_enable_wake(pdev, PCI_D3hot, 0);
6313 pci_enable_wake(pdev, PCI_D3cold, 0);
6314
047e0030 6315 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 6316 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6317 return -ENOMEM;
9d5c8243
AK		 6318 }
6319
9d5c8243 6320 igb_reset(adapter);
a8564f03
AD		 6321
6322 /* let the f/w know that the h/w is now under the control of the
6323 * driver. */
6324 igb_get_hw_control(adapter);
6325
9d5c8243
AK		 6326 wr32(E1000_WUS, ~0);
6327
a88f10ec
AD		 6328 if (netif_running(netdev)) {
6329 err = igb_open(netdev);
6330 if (err)
6331 return err;
6332 }
9d5c8243
AK		 6333
6334 netif_device_attach(netdev);
6335
9d5c8243
AK		 6336 return 0;
6337}
6338#endif
6339
6340static void igb_shutdown(struct pci_dev *pdev)
6341{
3fe7c4c9
RW		 6342 bool wake;
6343
6344 __igb_shutdown(pdev, &wake);
6345
6346 if (system_state == SYSTEM_POWER_OFF) {
6347 pci_wake_from_d3(pdev, wake);
6348 pci_set_power_state(pdev, PCI_D3hot);
6349 }
9d5c8243
AK		 6350}
6351
6352#ifdef CONFIG_NET_POLL_CONTROLLER
6353/*
6354 * Polling 'interrupt' - used by things like netconsole to send skbs
6355 * without having to re-enable interrupts. It's not called while
6356 * the interrupt routine is executing.
6357 */
6358static void igb_netpoll(struct net_device *netdev)
6359{
6360 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 6361 struct e1000_hw *hw = &adapter->hw;
9d5c8243 6362 int i;
9d5c8243 6363
eebbbdba 6364 if (!adapter->msix_entries) {
047e0030 6365 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 6366 igb_irq_disable(adapter);
047e0030 6367 napi_schedule(&q_vector->napi);
eebbbdba
AD		 6368 return;
6369 }
9d5c8243 6370
047e0030
AD		 6371 for (i = 0; i < adapter->num_q_vectors; i++) {
6372 struct igb_q_vector *q_vector = adapter->q_vector[i];
6373 wr32(E1000_EIMC, q_vector->eims_value);
6374 napi_schedule(&q_vector->napi);
eebbbdba 6375 }
9d5c8243
AK		 6376}
6377#endif /* CONFIG_NET_POLL_CONTROLLER */
6378
6379/**
6380 * igb_io_error_detected - called when PCI error is detected
6381 * @pdev: Pointer to PCI device
6382 * @state: The current pci connection state
6383 *
6384 * This function is called after a PCI bus error affecting
6385 * this device has been detected.
6386 */
6387static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6388 pci_channel_state_t state)
6389{
6390 struct net_device *netdev = pci_get_drvdata(pdev);
6391 struct igb_adapter *adapter = netdev_priv(netdev);
6392
6393 netif_device_detach(netdev);
6394
59ed6eec
AD		 6395 if (state == pci_channel_io_perm_failure)
6396 return PCI_ERS_RESULT_DISCONNECT;
6397
9d5c8243
AK		 6398 if (netif_running(netdev))
6399 igb_down(adapter);
6400 pci_disable_device(pdev);
6401
6402 /* Request a slot slot reset. */
6403 return PCI_ERS_RESULT_NEED_RESET;
6404}
6405
6406/**
6407 * igb_io_slot_reset - called after the pci bus has been reset.
6408 * @pdev: Pointer to PCI device
6409 *
6410 * Restart the card from scratch, as if from a cold-boot. Implementation
6411 * resembles the first-half of the igb_resume routine.
6412 */
6413static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6414{
6415 struct net_device *netdev = pci_get_drvdata(pdev);
6416 struct igb_adapter *adapter = netdev_priv(netdev);
6417 struct e1000_hw *hw = &adapter->hw;
40a914fa 6418 pci_ers_result_t result;
42bfd33a 6419 int err;
9d5c8243 6420
aed5dec3 6421 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6422 dev_err(&pdev->dev,
6423 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6424 result = PCI_ERS_RESULT_DISCONNECT;
6425 } else {
6426 pci_set_master(pdev);
6427 pci_restore_state(pdev);
b94f2d77 6428 pci_save_state(pdev);
9d5c8243 6429
40a914fa
AD		 6430 pci_enable_wake(pdev, PCI_D3hot, 0);
6431 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6432
40a914fa
AD		 6433 igb_reset(adapter);
6434 wr32(E1000_WUS, ~0);
6435 result = PCI_ERS_RESULT_RECOVERED;
6436 }
9d5c8243 6437
ea943d41
JK		 6438 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6439 if (err) {
6440 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6441 "failed 0x%0x\n", err);
6442 /* non-fatal, continue */
6443 }
40a914fa
AD		 6444
6445 return result;
9d5c8243
AK		 6446}
6447
6448/**
6449 * igb_io_resume - called when traffic can start flowing again.
6450 * @pdev: Pointer to PCI device
6451 *
6452 * This callback is called when the error recovery driver tells us that
6453 * its OK to resume normal operation. Implementation resembles the
6454 * second-half of the igb_resume routine.
6455 */
6456static void igb_io_resume(struct pci_dev *pdev)
6457{
6458 struct net_device *netdev = pci_get_drvdata(pdev);
6459 struct igb_adapter *adapter = netdev_priv(netdev);
6460
9d5c8243
AK		 6461 if (netif_running(netdev)) {
6462 if (igb_up(adapter)) {
6463 dev_err(&pdev->dev, "igb_up failed after reset\n");
6464 return;
6465 }
6466 }
6467
6468 netif_device_attach(netdev);
6469
6470 /* let the f/w know that the h/w is now under the control of the
6471 * driver. */
6472 igb_get_hw_control(adapter);
9d5c8243
AK		 6473}
6474
26ad9178
AD		 6475static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6476 u8 qsel)
6477{
6478 u32 rar_low, rar_high;
6479 struct e1000_hw *hw = &adapter->hw;
6480
6481 /* HW expects these in little endian so we reverse the byte order
6482 * from network order (big endian) to little endian
6483 */
6484 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6485 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6486 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6487
6488 /* Indicate to hardware the Address is Valid. */
6489 rar_high |= E1000_RAH_AV;
6490
6491 if (hw->mac.type == e1000_82575)
6492 rar_high |= E1000_RAH_POOL_1 * qsel;
6493 else
6494 rar_high |= E1000_RAH_POOL_1 << qsel;
6495
6496 wr32(E1000_RAL(index), rar_low);
6497 wrfl();
6498 wr32(E1000_RAH(index), rar_high);
6499 wrfl();
6500}
6501
4ae196df
AD		 6502static int igb_set_vf_mac(struct igb_adapter *adapter,
6503 int vf, unsigned char *mac_addr)
6504{
6505 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 6506 /* VF MAC addresses start at end of receive addresses and moves
6507 * torwards the first, as a result a collision should not be possible */
6508 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 6509
37680117 6510 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 6511
26ad9178 6512 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 6513
6514 return 0;
6515}
6516
8151d294
WM		 6517static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6518{
6519 struct igb_adapter *adapter = netdev_priv(netdev);
6520 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6521 return -EINVAL;
6522 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6523 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6524 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6525 " change effective.");
6526 if (test_bit(__IGB_DOWN, &adapter->state)) {
6527 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6528 " but the PF device is not up.\n");
6529 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6530 " attempting to use the VF device.\n");
6531 }
6532 return igb_set_vf_mac(adapter, vf, mac);
6533}
6534
6535static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6536{
6537 return -EOPNOTSUPP;
6538}
6539
6540static int igb_ndo_get_vf_config(struct net_device *netdev,
6541 int vf, struct ifla_vf_info *ivi)
6542{
6543 struct igb_adapter *adapter = netdev_priv(netdev);
6544 if (vf >= adapter->vfs_allocated_count)
6545 return -EINVAL;
6546 ivi->vf = vf;
6547 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6548 ivi->tx_rate = 0;
6549 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6550 ivi->qos = adapter->vf_data[vf].pf_qos;
6551 return 0;
6552}
6553
4ae196df
AD		 6554static void igb_vmm_control(struct igb_adapter *adapter)
6555{
6556 struct e1000_hw *hw = &adapter->hw;
10d8e907 6557 u32 reg;
4ae196df 6558
52a1dd4d
AD		 6559 switch (hw->mac.type) {
6560 case e1000_82575:
6561 default:
6562 /* replication is not supported for 82575 */
4ae196df 6563 return;
52a1dd4d
AD		 6564 case e1000_82576:
6565 /* notify HW that the MAC is adding vlan tags */
6566 reg = rd32(E1000_DTXCTL);
6567 reg |= E1000_DTXCTL_VLAN_ADDED;
6568 wr32(E1000_DTXCTL, reg);
6569 case e1000_82580:
6570 /* enable replication vlan tag stripping */
6571 reg = rd32(E1000_RPLOLR);
6572 reg |= E1000_RPLOLR_STRVLAN;
6573 wr32(E1000_RPLOLR, reg);
d2ba2ed8
AD		 6574 case e1000_i350:
6575 /* none of the above registers are supported by i350 */
52a1dd4d
AD		 6576 break;
6577 }
10d8e907 6578
d4960307
AD		 6579 if (adapter->vfs_allocated_count) {
6580 igb_vmdq_set_loopback_pf(hw, true);
6581 igb_vmdq_set_replication_pf(hw, true);
6582 } else {
6583 igb_vmdq_set_loopback_pf(hw, false);
6584 igb_vmdq_set_replication_pf(hw, false);
6585 }
4ae196df
AD		 6586}
6587
9d5c8243 6588/* igb_main.c */
ubuntu-zesty-kernel mirror