]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/net/ethernet/intel/igb/igb_main.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net: introduce and use netdev_features_t for device features sets
[mirror_ubuntu-artful-kernel.git] / drivers / net / ethernet / intel / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
4297f99b 4 Copyright(c) 2007-2011 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
b2cb09b1 31#include <linux/bitops.h>
9d5c8243
AK		 32#include <linux/vmalloc.h>
33#include <linux/pagemap.h>
34#include <linux/netdevice.h>
9d5c8243 35#include <linux/ipv6.h>
5a0e3ad6 36#include <linux/slab.h>
9d5c8243
AK		 37#include <net/checksum.h>
38#include <net/ip6_checksum.h>
c6cb090b 39#include <linux/net_tstamp.h>
9d5c8243
AK		 40#include <linux/mii.h>
41#include <linux/ethtool.h>
01789349 42#include <linux/if.h>
9d5c8243
AK		 43#include <linux/if_vlan.h>
44#include <linux/pci.h>
c54106bb 45#include <linux/pci-aspm.h>
9d5c8243
AK		 46#include <linux/delay.h>
47#include <linux/interrupt.h>
7d13a7d0
AD		 48#include <linux/ip.h>
49#include <linux/tcp.h>
50#include <linux/sctp.h>
9d5c8243 51#include <linux/if_ether.h>
40a914fa 52#include <linux/aer.h>
70c71606 53#include <linux/prefetch.h>
421e02f0 54#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 55#include <linux/dca.h>
56#endif
9d5c8243
AK		 57#include "igb.h"
58
0d1fe82d 59#define MAJ 3
a28dc43f
CW		 60#define MIN 2
61#define BUILD 10
0d1fe82d 62#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
929dd047 63__stringify(BUILD) "-k"
9d5c8243
AK		 64char igb_driver_name[] = "igb";
65char igb_driver_version[] = DRV_VERSION;
66static const char igb_driver_string[] =
67 "Intel(R) Gigabit Ethernet Network Driver";
4c4b42cb 68static const char igb_copyright[] = "Copyright (c) 2007-2011 Intel Corporation.";
9d5c8243 69
9d5c8243
AK		 70static const struct e1000_info *igb_info_tbl[] = {
71 [board_82575] = &e1000_82575_info,
72};
73
a3aa1884 74static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
d2ba2ed8
AD		 75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
55cac248
AD		 79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
6493d24f 81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
55cac248
AD		 82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
308fb39a
JG		 85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
1b5dda33
GJ		 87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
2d064c06 89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
b894fa26 95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
c8ea5ea9 96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 97 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
98 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
99 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
100 /* required last entry */
101 {0, }
102};
103
104MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
105
106void igb_reset(struct igb_adapter *);
107static int igb_setup_all_tx_resources(struct igb_adapter *);
108static int igb_setup_all_rx_resources(struct igb_adapter *);
109static void igb_free_all_tx_resources(struct igb_adapter *);
110static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 111static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 112static int igb_probe(struct pci_dev *, const struct pci_device_id *);
113static void __devexit igb_remove(struct pci_dev *pdev);
673b8b70 114static void igb_init_hw_timer(struct igb_adapter *adapter);
9d5c8243
AK		 115static int igb_sw_init(struct igb_adapter *);
116static int igb_open(struct net_device *);
117static int igb_close(struct net_device *);
118static void igb_configure_tx(struct igb_adapter *);
119static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 120static void igb_clean_all_tx_rings(struct igb_adapter *);
121static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 122static void igb_clean_tx_ring(struct igb_ring *);
123static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 124static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 125static void igb_update_phy_info(unsigned long);
126static void igb_watchdog(unsigned long);
127static void igb_watchdog_task(struct work_struct *);
cd392f5c 128static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
12dcd86b
ED		 129static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
130 struct rtnl_link_stats64 *stats);
9d5c8243
AK		 131static int igb_change_mtu(struct net_device *, int);
132static int igb_set_mac(struct net_device *, void *);
68d480c4 133static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 134static irqreturn_t igb_intr(int irq, void *);
135static irqreturn_t igb_intr_msi(int irq, void *);
136static irqreturn_t igb_msix_other(int irq, void *);
047e0030 137static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 138#ifdef CONFIG_IGB_DCA
047e0030 139static void igb_update_dca(struct igb_q_vector *);
fe4506b6 140static void igb_setup_dca(struct igb_adapter *);
421e02f0 141#endif /* CONFIG_IGB_DCA */
661086df 142static int igb_poll(struct napi_struct *, int);
13fde97a 143static bool igb_clean_tx_irq(struct igb_q_vector *);
cd392f5c 144static bool igb_clean_rx_irq(struct igb_q_vector *, int);
9d5c8243
AK		 145static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
146static void igb_tx_timeout(struct net_device *);
147static void igb_reset_task(struct work_struct *);
c8f44aff 148static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features);
9d5c8243
AK		 149static void igb_vlan_rx_add_vid(struct net_device *, u16);
150static void igb_vlan_rx_kill_vid(struct net_device *, u16);
151static void igb_restore_vlan(struct igb_adapter *);
26ad9178 152static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 153static void igb_ping_all_vfs(struct igb_adapter *);
154static void igb_msg_task(struct igb_adapter *);
4ae196df 155static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 156static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 157static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 158static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
159static int igb_ndo_set_vf_vlan(struct net_device *netdev,
160 int vf, u16 vlan, u8 qos);
161static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
162static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
163 struct ifla_vf_info *ivi);
17dc566c 164static void igb_check_vf_rate_limit(struct igb_adapter *);
46a01698
RL		 165
166#ifdef CONFIG_PCI_IOV
0224d663
GR		 167static int igb_vf_configure(struct igb_adapter *adapter, int vf);
168static int igb_find_enabled_vfs(struct igb_adapter *adapter);
169static int igb_check_vf_assignment(struct igb_adapter *adapter);
46a01698 170#endif
9d5c8243 171
9d5c8243 172#ifdef CONFIG_PM
3fe7c4c9 173static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 174static int igb_resume(struct pci_dev *);
175#endif
176static void igb_shutdown(struct pci_dev *);
421e02f0 177#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 178static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
179static struct notifier_block dca_notifier = {
180 .notifier_call = igb_notify_dca,
181 .next = NULL,
182 .priority = 0
183};
184#endif
9d5c8243
AK		 185#ifdef CONFIG_NET_POLL_CONTROLLER
186/* for netdump / net console */
187static void igb_netpoll(struct net_device *);
188#endif
37680117 189#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 190static unsigned int max_vfs = 0;
191module_param(max_vfs, uint, 0);
192MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
193 "per physical function");
194#endif /* CONFIG_PCI_IOV */
195
9d5c8243
AK		 196static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
197 pci_channel_state_t);
198static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
199static void igb_io_resume(struct pci_dev *);
200
201static struct pci_error_handlers igb_err_handler = {
202 .error_detected = igb_io_error_detected,
203 .slot_reset = igb_io_slot_reset,
204 .resume = igb_io_resume,
205};
206
b6e0c419 207static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
9d5c8243
AK		 208
209static struct pci_driver igb_driver = {
210 .name = igb_driver_name,
211 .id_table = igb_pci_tbl,
212 .probe = igb_probe,
213 .remove = __devexit_p(igb_remove),
214#ifdef CONFIG_PM
25985edc 215 /* Power Management Hooks */
9d5c8243
AK		 216 .suspend = igb_suspend,
217 .resume = igb_resume,
218#endif
219 .shutdown = igb_shutdown,
220 .err_handler = &igb_err_handler
221};
222
223MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
224MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
225MODULE_LICENSE("GPL");
226MODULE_VERSION(DRV_VERSION);
227
c97ec42a
TI		 228struct igb_reg_info {
229 u32 ofs;
230 char *name;
231};
232
233static const struct igb_reg_info igb_reg_info_tbl[] = {
234
235 /* General Registers */
236 {E1000_CTRL, "CTRL"},
237 {E1000_STATUS, "STATUS"},
238 {E1000_CTRL_EXT, "CTRL_EXT"},
239
240 /* Interrupt Registers */
241 {E1000_ICR, "ICR"},
242
243 /* RX Registers */
244 {E1000_RCTL, "RCTL"},
245 {E1000_RDLEN(0), "RDLEN"},
246 {E1000_RDH(0), "RDH"},
247 {E1000_RDT(0), "RDT"},
248 {E1000_RXDCTL(0), "RXDCTL"},
249 {E1000_RDBAL(0), "RDBAL"},
250 {E1000_RDBAH(0), "RDBAH"},
251
252 /* TX Registers */
253 {E1000_TCTL, "TCTL"},
254 {E1000_TDBAL(0), "TDBAL"},
255 {E1000_TDBAH(0), "TDBAH"},
256 {E1000_TDLEN(0), "TDLEN"},
257 {E1000_TDH(0), "TDH"},
258 {E1000_TDT(0), "TDT"},
259 {E1000_TXDCTL(0), "TXDCTL"},
260 {E1000_TDFH, "TDFH"},
261 {E1000_TDFT, "TDFT"},
262 {E1000_TDFHS, "TDFHS"},
263 {E1000_TDFPC, "TDFPC"},
264
265 /* List Terminator */
266 {}
267};
268
269/*
270 * igb_regdump - register printout routine
271 */
272static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
273{
274 int n = 0;
275 char rname[16];
276 u32 regs[8];
277
278 switch (reginfo->ofs) {
279 case E1000_RDLEN(0):
280 for (n = 0; n < 4; n++)
281 regs[n] = rd32(E1000_RDLEN(n));
282 break;
283 case E1000_RDH(0):
284 for (n = 0; n < 4; n++)
285 regs[n] = rd32(E1000_RDH(n));
286 break;
287 case E1000_RDT(0):
288 for (n = 0; n < 4; n++)
289 regs[n] = rd32(E1000_RDT(n));
290 break;
291 case E1000_RXDCTL(0):
292 for (n = 0; n < 4; n++)
293 regs[n] = rd32(E1000_RXDCTL(n));
294 break;
295 case E1000_RDBAL(0):
296 for (n = 0; n < 4; n++)
297 regs[n] = rd32(E1000_RDBAL(n));
298 break;
299 case E1000_RDBAH(0):
300 for (n = 0; n < 4; n++)
301 regs[n] = rd32(E1000_RDBAH(n));
302 break;
303 case E1000_TDBAL(0):
304 for (n = 0; n < 4; n++)
305 regs[n] = rd32(E1000_RDBAL(n));
306 break;
307 case E1000_TDBAH(0):
308 for (n = 0; n < 4; n++)
309 regs[n] = rd32(E1000_TDBAH(n));
310 break;
311 case E1000_TDLEN(0):
312 for (n = 0; n < 4; n++)
313 regs[n] = rd32(E1000_TDLEN(n));
314 break;
315 case E1000_TDH(0):
316 for (n = 0; n < 4; n++)
317 regs[n] = rd32(E1000_TDH(n));
318 break;
319 case E1000_TDT(0):
320 for (n = 0; n < 4; n++)
321 regs[n] = rd32(E1000_TDT(n));
322 break;
323 case E1000_TXDCTL(0):
324 for (n = 0; n < 4; n++)
325 regs[n] = rd32(E1000_TXDCTL(n));
326 break;
327 default:
328 printk(KERN_INFO "%-15s %08x\n",
329 reginfo->name, rd32(reginfo->ofs));
330 return;
331 }
332
333 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
334 printk(KERN_INFO "%-15s ", rname);
335 for (n = 0; n < 4; n++)
336 printk(KERN_CONT "%08x ", regs[n]);
337 printk(KERN_CONT "\n");
338}
339
340/*
341 * igb_dump - Print registers, tx-rings and rx-rings
342 */
343static void igb_dump(struct igb_adapter *adapter)
344{
345 struct net_device *netdev = adapter->netdev;
346 struct e1000_hw *hw = &adapter->hw;
347 struct igb_reg_info *reginfo;
c97ec42a
TI		 348 struct igb_ring *tx_ring;
349 union e1000_adv_tx_desc *tx_desc;
350 struct my_u0 { u64 a; u64 b; } *u0;
c97ec42a
TI		 351 struct igb_ring *rx_ring;
352 union e1000_adv_rx_desc *rx_desc;
353 u32 staterr;
6ad4edfc 354 u16 i, n;
c97ec42a
TI		 355
356 if (!netif_msg_hw(adapter))
357 return;
358
359 /* Print netdevice Info */
360 if (netdev) {
361 dev_info(&adapter->pdev->dev, "Net device Info\n");
362 printk(KERN_INFO "Device Name state "
363 "trans_start last_rx\n");
364 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
365 netdev->name,
366 netdev->state,
367 netdev->trans_start,
368 netdev->last_rx);
369 }
370
371 /* Print Registers */
372 dev_info(&adapter->pdev->dev, "Register Dump\n");
373 printk(KERN_INFO " Register Name Value\n");
374 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
375 reginfo->name; reginfo++) {
376 igb_regdump(hw, reginfo);
377 }
378
379 /* Print TX Ring Summary */
380 if (!netdev || !netif_running(netdev))
381 goto exit;
382
383 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
384 printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma ]"
385 " leng ntw timestamp\n");
386 for (n = 0; n < adapter->num_tx_queues; n++) {
06034649 387 struct igb_tx_buffer *buffer_info;
c97ec42a 388 tx_ring = adapter->tx_ring[n];
06034649 389 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
8542db05 390 printk(KERN_INFO " %5d %5X %5X %016llX %04X %p %016llX\n",
c97ec42a
TI		 391 n, tx_ring->next_to_use, tx_ring->next_to_clean,
392 (u64)buffer_info->dma,
393 buffer_info->length,
394 buffer_info->next_to_watch,
395 (u64)buffer_info->time_stamp);
396 }
397
398 /* Print TX Rings */
399 if (!netif_msg_tx_done(adapter))
400 goto rx_ring_summary;
401
402 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
403
404 /* Transmit Descriptor Formats
405 *
406 * Advanced Transmit Descriptor
407 * +--------------------------------------------------------------+
408 * 0 | Buffer Address [63:0] |
409 * +--------------------------------------------------------------+
410 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
411 * +--------------------------------------------------------------+
412 * 63 46 45 40 39 38 36 35 32 31 24 15 0
413 */
414
415 for (n = 0; n < adapter->num_tx_queues; n++) {
416 tx_ring = adapter->tx_ring[n];
417 printk(KERN_INFO "------------------------------------\n");
418 printk(KERN_INFO "TX QUEUE INDEX = %d\n", tx_ring->queue_index);
419 printk(KERN_INFO "------------------------------------\n");
420 printk(KERN_INFO "T [desc] [address 63:0 ] "
421 "[PlPOCIStDDM Ln] [bi->dma ] "
422 "leng ntw timestamp bi->skb\n");
423
424 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
06034649 425 struct igb_tx_buffer *buffer_info;
60136906 426 tx_desc = IGB_TX_DESC(tx_ring, i);
06034649 427 buffer_info = &tx_ring->tx_buffer_info[i];
c97ec42a
TI		 428 u0 = (struct my_u0 *)tx_desc;
429 printk(KERN_INFO "T [0x%03X] %016llX %016llX %016llX"
8542db05 430 " %04X %p %016llX %p", i,
c97ec42a
TI		 431 le64_to_cpu(u0->a),
432 le64_to_cpu(u0->b),
433 (u64)buffer_info->dma,
434 buffer_info->length,
435 buffer_info->next_to_watch,
436 (u64)buffer_info->time_stamp,
437 buffer_info->skb);
438 if (i == tx_ring->next_to_use &&
439 i == tx_ring->next_to_clean)
440 printk(KERN_CONT " NTC/U\n");
441 else if (i == tx_ring->next_to_use)
442 printk(KERN_CONT " NTU\n");
443 else if (i == tx_ring->next_to_clean)
444 printk(KERN_CONT " NTC\n");
445 else
446 printk(KERN_CONT "\n");
447
448 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
449 print_hex_dump(KERN_INFO, "",
450 DUMP_PREFIX_ADDRESS,
451 16, 1, phys_to_virt(buffer_info->dma),
452 buffer_info->length, true);
453 }
454 }
455
456 /* Print RX Rings Summary */
457rx_ring_summary:
458 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
459 printk(KERN_INFO "Queue [NTU] [NTC]\n");
460 for (n = 0; n < adapter->num_rx_queues; n++) {
461 rx_ring = adapter->rx_ring[n];
462 printk(KERN_INFO " %5d %5X %5X\n", n,
463 rx_ring->next_to_use, rx_ring->next_to_clean);
464 }
465
466 /* Print RX Rings */
467 if (!netif_msg_rx_status(adapter))
468 goto exit;
469
470 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
471
472 /* Advanced Receive Descriptor (Read) Format
473 * 63 1 0
474 * +-----------------------------------------------------+
475 * 0 | Packet Buffer Address [63:1] |A0/NSE|
476 * +----------------------------------------------+------+
477 * 8 | Header Buffer Address [63:1] | DD |
478 * +-----------------------------------------------------+
479 *
480 *
481 * Advanced Receive Descriptor (Write-Back) Format
482 *
483 * 63 48 47 32 31 30 21 20 17 16 4 3 0
484 * +------------------------------------------------------+
485 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
486 * | Checksum Ident | | | | Type | Type |
487 * +------------------------------------------------------+
488 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
489 * +------------------------------------------------------+
490 * 63 48 47 32 31 20 19 0
491 */
492
493 for (n = 0; n < adapter->num_rx_queues; n++) {
494 rx_ring = adapter->rx_ring[n];
495 printk(KERN_INFO "------------------------------------\n");
496 printk(KERN_INFO "RX QUEUE INDEX = %d\n", rx_ring->queue_index);
497 printk(KERN_INFO "------------------------------------\n");
498 printk(KERN_INFO "R [desc] [ PktBuf A0] "
499 "[ HeadBuf DD] [bi->dma ] [bi->skb] "
500 "<-- Adv Rx Read format\n");
501 printk(KERN_INFO "RWB[desc] [PcsmIpSHl PtRs] "
502 "[vl er S cks ln] ---------------- [bi->skb] "
503 "<-- Adv Rx Write-Back format\n");
504
505 for (i = 0; i < rx_ring->count; i++) {
06034649
AD		 506 struct igb_rx_buffer *buffer_info;
507 buffer_info = &rx_ring->rx_buffer_info[i];
60136906 508 rx_desc = IGB_RX_DESC(rx_ring, i);
c97ec42a
TI		 509 u0 = (struct my_u0 *)rx_desc;
510 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
511 if (staterr & E1000_RXD_STAT_DD) {
512 /* Descriptor Done */
513 printk(KERN_INFO "RWB[0x%03X] %016llX "
514 "%016llX ---------------- %p", i,
515 le64_to_cpu(u0->a),
516 le64_to_cpu(u0->b),
517 buffer_info->skb);
518 } else {
519 printk(KERN_INFO "R [0x%03X] %016llX "
520 "%016llX %016llX %p", i,
521 le64_to_cpu(u0->a),
522 le64_to_cpu(u0->b),
523 (u64)buffer_info->dma,
524 buffer_info->skb);
525
526 if (netif_msg_pktdata(adapter)) {
527 print_hex_dump(KERN_INFO, "",
528 DUMP_PREFIX_ADDRESS,
529 16, 1,
530 phys_to_virt(buffer_info->dma),
44390ca6
AD		 531 IGB_RX_HDR_LEN, true);
532 print_hex_dump(KERN_INFO, "",
533 DUMP_PREFIX_ADDRESS,
534 16, 1,
535 phys_to_virt(
536 buffer_info->page_dma +
537 buffer_info->page_offset),
538 PAGE_SIZE/2, true);
c97ec42a
TI		 539 }
540 }
541
542 if (i == rx_ring->next_to_use)
543 printk(KERN_CONT " NTU\n");
544 else if (i == rx_ring->next_to_clean)
545 printk(KERN_CONT " NTC\n");
546 else
547 printk(KERN_CONT "\n");
548
549 }
550 }
551
552exit:
553 return;
554}
555
556
38c845c7
PO		 557/**
558 * igb_read_clock - read raw cycle counter (to be used by time counter)
559 */
560static cycle_t igb_read_clock(const struct cyclecounter *tc)
561{
562 struct igb_adapter *adapter =
563 container_of(tc, struct igb_adapter, cycles);
564 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 565 u64 stamp = 0;
566 int shift = 0;
38c845c7 567
55cac248
AD		 568 /*
569 * The timestamp latches on lowest register read. For the 82580
570 * the lowest register is SYSTIMR instead of SYSTIML. However we never
571 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
572 */
06218a8d 573 if (hw->mac.type >= e1000_82580) {
55cac248
AD		 574 stamp = rd32(E1000_SYSTIMR) >> 8;
575 shift = IGB_82580_TSYNC_SHIFT;
576 }
577
c5b9bd5e
AD		 578 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
579 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 580 return stamp;
581}
582
9d5c8243 583/**
c041076a 584 * igb_get_hw_dev - return device
9d5c8243
AK		 585 * used by hardware layer to print debugging information
586 **/
c041076a 587struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
9d5c8243
AK		 588{
589 struct igb_adapter *adapter = hw->back;
c041076a 590 return adapter->netdev;
9d5c8243 591}
38c845c7 592
9d5c8243
AK		 593/**
594 * igb_init_module - Driver Registration Routine
595 *
596 * igb_init_module is the first routine called when the driver is
597 * loaded. All it does is register with the PCI subsystem.
598 **/
599static int __init igb_init_module(void)
600{
601 int ret;
602 printk(KERN_INFO "%s - version %s\n",
603 igb_driver_string, igb_driver_version);
604
605 printk(KERN_INFO "%s\n", igb_copyright);
606
421e02f0 607#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 608 dca_register_notify(&dca_notifier);
609#endif
bbd98fe4 610 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 611 return ret;
612}
613
614module_init(igb_init_module);
615
616/**
617 * igb_exit_module - Driver Exit Cleanup Routine
618 *
619 * igb_exit_module is called just before the driver is removed
620 * from memory.
621 **/
622static void __exit igb_exit_module(void)
623{
421e02f0 624#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 625 dca_unregister_notify(&dca_notifier);
626#endif
9d5c8243
AK		 627 pci_unregister_driver(&igb_driver);
628}
629
630module_exit(igb_exit_module);
631
26bc19ec
AD		 632#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
633/**
634 * igb_cache_ring_register - Descriptor ring to register mapping
635 * @adapter: board private structure to initialize
636 *
637 * Once we know the feature-set enabled for the device, we'll cache
638 * the register offset the descriptor ring is assigned to.
639 **/
640static void igb_cache_ring_register(struct igb_adapter *adapter)
641{
ee1b9f06 642 int i = 0, j = 0;
047e0030 643 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 644
645 switch (adapter->hw.mac.type) {
646 case e1000_82576:
647 /* The queues are allocated for virtualization such that VF 0
648 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
649 * In order to avoid collision we start at the first free queue
650 * and continue consuming queues in the same sequence
651 */
ee1b9f06 652 if (adapter->vfs_allocated_count) {
a99955fc 653 for (; i < adapter->rss_queues; i++)
3025a446
AD		 654 adapter->rx_ring[i]->reg_idx = rbase_offset +
655 Q_IDX_82576(i);
ee1b9f06 656 }
26bc19ec 657 case e1000_82575:
55cac248 658 case e1000_82580:
d2ba2ed8 659 case e1000_i350:
26bc19ec 660 default:
ee1b9f06 661 for (; i < adapter->num_rx_queues; i++)
3025a446 662 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 663 for (; j < adapter->num_tx_queues; j++)
3025a446 664 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 665 break;
666 }
667}
668
047e0030
AD		 669static void igb_free_queues(struct igb_adapter *adapter)
670{
3025a446 671 int i;
047e0030 672
3025a446
AD		 673 for (i = 0; i < adapter->num_tx_queues; i++) {
674 kfree(adapter->tx_ring[i]);
675 adapter->tx_ring[i] = NULL;
676 }
677 for (i = 0; i < adapter->num_rx_queues; i++) {
678 kfree(adapter->rx_ring[i]);
679 adapter->rx_ring[i] = NULL;
680 }
047e0030
AD		 681 adapter->num_rx_queues = 0;
682 adapter->num_tx_queues = 0;
683}
684
9d5c8243
AK		 685/**
686 * igb_alloc_queues - Allocate memory for all rings
687 * @adapter: board private structure to initialize
688 *
689 * We allocate one ring per queue at run-time since we don't know the
690 * number of queues at compile-time.
691 **/
692static int igb_alloc_queues(struct igb_adapter *adapter)
693{
3025a446 694 struct igb_ring *ring;
9d5c8243 695 int i;
81c2fc22 696 int orig_node = adapter->node;
9d5c8243 697
661086df 698 for (i = 0; i < adapter->num_tx_queues; i++) {
81c2fc22
AD		 699 if (orig_node == -1) {
700 int cur_node = next_online_node(adapter->node);
701 if (cur_node == MAX_NUMNODES)
702 cur_node = first_online_node;
703 adapter->node = cur_node;
704 }
705 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
706 adapter->node);
707 if (!ring)
708 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
3025a446
AD		 709 if (!ring)
710 goto err;
68fd9910 711 ring->count = adapter->tx_ring_count;
661086df 712 ring->queue_index = i;
59d71989 713 ring->dev = &adapter->pdev->dev;
e694e964 714 ring->netdev = adapter->netdev;
81c2fc22 715 ring->numa_node = adapter->node;
85ad76b2
AD		 716 /* For 82575, context index must be unique per ring. */
717 if (adapter->hw.mac.type == e1000_82575)
866cff06 718 set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
3025a446 719 adapter->tx_ring[i] = ring;
661086df 720 }
81c2fc22
AD		 721 /* Restore the adapter's original node */
722 adapter->node = orig_node;
85ad76b2 723
9d5c8243 724 for (i = 0; i < adapter->num_rx_queues; i++) {
81c2fc22
AD		 725 if (orig_node == -1) {
726 int cur_node = next_online_node(adapter->node);
727 if (cur_node == MAX_NUMNODES)
728 cur_node = first_online_node;
729 adapter->node = cur_node;
730 }
731 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
732 adapter->node);
733 if (!ring)
734 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
3025a446
AD		 735 if (!ring)
736 goto err;
68fd9910 737 ring->count = adapter->rx_ring_count;
844290e5 738 ring->queue_index = i;
59d71989 739 ring->dev = &adapter->pdev->dev;
e694e964 740 ring->netdev = adapter->netdev;
81c2fc22 741 ring->numa_node = adapter->node;
85ad76b2
AD		 742 /* set flag indicating ring supports SCTP checksum offload */
743 if (adapter->hw.mac.type >= e1000_82576)
866cff06 744 set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
8be10e91
AD		 745
746 /* On i350, loopback VLAN packets have the tag byte-swapped. */
747 if (adapter->hw.mac.type == e1000_i350)
748 set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
749
3025a446 750 adapter->rx_ring[i] = ring;
9d5c8243 751 }
81c2fc22
AD		 752 /* Restore the adapter's original node */
753 adapter->node = orig_node;
26bc19ec
AD		 754
755 igb_cache_ring_register(adapter);
9d5c8243 756
047e0030 757 return 0;
a88f10ec 758
047e0030 759err:
81c2fc22
AD		 760 /* Restore the adapter's original node */
761 adapter->node = orig_node;
047e0030 762 igb_free_queues(adapter);
d1a8c9e1 763
047e0030 764 return -ENOMEM;
a88f10ec
AD		 765}
766
4be000c8
AD		 767/**
768 * igb_write_ivar - configure ivar for given MSI-X vector
769 * @hw: pointer to the HW structure
770 * @msix_vector: vector number we are allocating to a given ring
771 * @index: row index of IVAR register to write within IVAR table
772 * @offset: column offset of in IVAR, should be multiple of 8
773 *
774 * This function is intended to handle the writing of the IVAR register
775 * for adapters 82576 and newer. The IVAR table consists of 2 columns,
776 * each containing an cause allocation for an Rx and Tx ring, and a
777 * variable number of rows depending on the number of queues supported.
778 **/
779static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
780 int index, int offset)
781{
782 u32 ivar = array_rd32(E1000_IVAR0, index);
783
784 /* clear any bits that are currently set */
785 ivar &= ~((u32)0xFF << offset);
786
787 /* write vector and valid bit */
788 ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
789
790 array_wr32(E1000_IVAR0, index, ivar);
791}
792
9d5c8243 793#define IGB_N0_QUEUE -1
047e0030 794static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243 795{
047e0030 796 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 797 struct e1000_hw *hw = &adapter->hw;
047e0030
AD		 798 int rx_queue = IGB_N0_QUEUE;
799 int tx_queue = IGB_N0_QUEUE;
4be000c8 800 u32 msixbm = 0;
047e0030 801
0ba82994
AD		 802 if (q_vector->rx.ring)
803 rx_queue = q_vector->rx.ring->reg_idx;
804 if (q_vector->tx.ring)
805 tx_queue = q_vector->tx.ring->reg_idx;
2d064c06
AD		 806
807 switch (hw->mac.type) {
808 case e1000_82575:
9d5c8243
AK		 809 /* The 82575 assigns vectors using a bitmask, which matches the
810 bitmask for the EICR/EIMS/EIMC registers. To assign one
811 or more queues to a vector, we write the appropriate bits
812 into the MSIXBM register for that vector. */
047e0030 813 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 814 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 815 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 816 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 817 if (!adapter->msix_entries && msix_vector == 0)
818 msixbm |= E1000_EIMS_OTHER;
9d5c8243 819 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 820 q_vector->eims_value = msixbm;
2d064c06
AD		 821 break;
822 case e1000_82576:
4be000c8
AD		 823 /*
824 * 82576 uses a table that essentially consists of 2 columns
825 * with 8 rows. The ordering is column-major so we use the
826 * lower 3 bits as the row index, and the 4th bit as the
827 * column offset.
828 */
829 if (rx_queue > IGB_N0_QUEUE)
830 igb_write_ivar(hw, msix_vector,
831 rx_queue & 0x7,
832 (rx_queue & 0x8) << 1);
833 if (tx_queue > IGB_N0_QUEUE)
834 igb_write_ivar(hw, msix_vector,
835 tx_queue & 0x7,
836 ((tx_queue & 0x8) << 1) + 8);
047e0030 837 q_vector->eims_value = 1 << msix_vector;
2d064c06 838 break;
55cac248 839 case e1000_82580:
d2ba2ed8 840 case e1000_i350:
4be000c8
AD		 841 /*
842 * On 82580 and newer adapters the scheme is similar to 82576
843 * however instead of ordering column-major we have things
844 * ordered row-major. So we traverse the table by using
845 * bit 0 as the column offset, and the remaining bits as the
846 * row index.
847 */
848 if (rx_queue > IGB_N0_QUEUE)
849 igb_write_ivar(hw, msix_vector,
850 rx_queue >> 1,
851 (rx_queue & 0x1) << 4);
852 if (tx_queue > IGB_N0_QUEUE)
853 igb_write_ivar(hw, msix_vector,
854 tx_queue >> 1,
855 ((tx_queue & 0x1) << 4) + 8);
55cac248
AD		 856 q_vector->eims_value = 1 << msix_vector;
857 break;
2d064c06
AD		 858 default:
859 BUG();
860 break;
861 }
26b39276
AD		 862
863 /* add q_vector eims value to global eims_enable_mask */
864 adapter->eims_enable_mask |= q_vector->eims_value;
865
866 /* configure q_vector to set itr on first interrupt */
867 q_vector->set_itr = 1;
9d5c8243
AK		 868}
869
870/**
871 * igb_configure_msix - Configure MSI-X hardware
872 *
873 * igb_configure_msix sets up the hardware to properly
874 * generate MSI-X interrupts.
875 **/
876static void igb_configure_msix(struct igb_adapter *adapter)
877{
878 u32 tmp;
879 int i, vector = 0;
880 struct e1000_hw *hw = &adapter->hw;
881
882 adapter->eims_enable_mask = 0;
9d5c8243
AK		 883
884 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 885 switch (hw->mac.type) {
886 case e1000_82575:
9d5c8243
AK		 887 tmp = rd32(E1000_CTRL_EXT);
888 /* enable MSI-X PBA support*/
889 tmp |= E1000_CTRL_EXT_PBA_CLR;
890
891 /* Auto-Mask interrupts upon ICR read. */
892 tmp |= E1000_CTRL_EXT_EIAME;
893 tmp |= E1000_CTRL_EXT_IRCA;
894
895 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 896
897 /* enable msix_other interrupt */
898 array_wr32(E1000_MSIXBM(0), vector++,
899 E1000_EIMS_OTHER);
844290e5 900 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 901
2d064c06
AD		 902 break;
903
904 case e1000_82576:
55cac248 905 case e1000_82580:
d2ba2ed8 906 case e1000_i350:
047e0030
AD		 907 /* Turn on MSI-X capability first, or our settings
908 * won't stick. And it will take days to debug. */
909 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
910 E1000_GPIE_PBA | E1000_GPIE_EIAME |
911 E1000_GPIE_NSICR);
912
913 /* enable msix_other interrupt */
914 adapter->eims_other = 1 << vector;
2d064c06 915 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 916
047e0030 917 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 918 break;
919 default:
920 /* do nothing, since nothing else supports MSI-X */
921 break;
922 } /* switch (hw->mac.type) */
047e0030
AD		 923
924 adapter->eims_enable_mask |= adapter->eims_other;
925
26b39276
AD		 926 for (i = 0; i < adapter->num_q_vectors; i++)
927 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 928
9d5c8243
AK		 929 wrfl();
930}
931
932/**
933 * igb_request_msix - Initialize MSI-X interrupts
934 *
935 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
936 * kernel.
937 **/
938static int igb_request_msix(struct igb_adapter *adapter)
939{
940 struct net_device *netdev = adapter->netdev;
047e0030 941 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 942 int i, err = 0, vector = 0;
943
047e0030 944 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 945 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 946 if (err)
947 goto out;
948 vector++;
949
950 for (i = 0; i < adapter->num_q_vectors; i++) {
951 struct igb_q_vector *q_vector = adapter->q_vector[i];
952
953 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
954
0ba82994 955 if (q_vector->rx.ring && q_vector->tx.ring)
047e0030 956 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
0ba82994
AD		 957 q_vector->rx.ring->queue_index);
958 else if (q_vector->tx.ring)
047e0030 959 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
0ba82994
AD		 960 q_vector->tx.ring->queue_index);
961 else if (q_vector->rx.ring)
047e0030 962 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
0ba82994 963 q_vector->rx.ring->queue_index);
9d5c8243 964 else
047e0030
AD		 965 sprintf(q_vector->name, "%s-unused", netdev->name);
966
9d5c8243 967 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 968 igb_msix_ring, 0, q_vector->name,
047e0030 969 q_vector);
9d5c8243
AK		 970 if (err)
971 goto out;
9d5c8243
AK		 972 vector++;
973 }
974
9d5c8243
AK		 975 igb_configure_msix(adapter);
976 return 0;
977out:
978 return err;
979}
980
981static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
982{
983 if (adapter->msix_entries) {
984 pci_disable_msix(adapter->pdev);
985 kfree(adapter->msix_entries);
986 adapter->msix_entries = NULL;
047e0030 987 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 988 pci_disable_msi(adapter->pdev);
047e0030 989 }
9d5c8243
AK		 990}
991
047e0030
AD		 992/**
993 * igb_free_q_vectors - Free memory allocated for interrupt vectors
994 * @adapter: board private structure to initialize
995 *
996 * This function frees the memory allocated to the q_vectors. In addition if
997 * NAPI is enabled it will delete any references to the NAPI struct prior
998 * to freeing the q_vector.
999 **/
1000static void igb_free_q_vectors(struct igb_adapter *adapter)
1001{
1002 int v_idx;
1003
1004 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1005 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1006 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 1007 if (!q_vector)
1008 continue;
047e0030
AD		 1009 netif_napi_del(&q_vector->napi);
1010 kfree(q_vector);
1011 }
1012 adapter->num_q_vectors = 0;
1013}
1014
1015/**
1016 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
1017 *
1018 * This function resets the device so that it has 0 rx queues, tx queues, and
1019 * MSI-X interrupts allocated.
1020 */
1021static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
1022{
1023 igb_free_queues(adapter);
1024 igb_free_q_vectors(adapter);
1025 igb_reset_interrupt_capability(adapter);
1026}
9d5c8243
AK		 1027
1028/**
1029 * igb_set_interrupt_capability - set MSI or MSI-X if supported
1030 *
1031 * Attempt to configure interrupts using the best available
1032 * capabilities of the hardware and kernel.
1033 **/
21adef3e 1034static int igb_set_interrupt_capability(struct igb_adapter *adapter)
9d5c8243
AK		 1035{
1036 int err;
1037 int numvecs, i;
1038
83b7180d 1039 /* Number of supported queues. */
a99955fc 1040 adapter->num_rx_queues = adapter->rss_queues;
5fa8517f
GR		 1041 if (adapter->vfs_allocated_count)
1042 adapter->num_tx_queues = 1;
1043 else
1044 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 1045
047e0030
AD		 1046 /* start with one vector for every rx queue */
1047 numvecs = adapter->num_rx_queues;
1048
3ad2f3fb 1049 /* if tx handler is separate add 1 for every tx queue */
a99955fc
AD		 1050 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1051 numvecs += adapter->num_tx_queues;
047e0030
AD		 1052
1053 /* store the number of vectors reserved for queues */
1054 adapter->num_q_vectors = numvecs;
1055
1056 /* add 1 vector for link status interrupts */
1057 numvecs++;
9d5c8243
AK		 1058 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1059 GFP_KERNEL);
1060 if (!adapter->msix_entries)
1061 goto msi_only;
1062
1063 for (i = 0; i < numvecs; i++)
1064 adapter->msix_entries[i].entry = i;
1065
1066 err = pci_enable_msix(adapter->pdev,
1067 adapter->msix_entries,
1068 numvecs);
1069 if (err == 0)
34a20e89 1070 goto out;
9d5c8243
AK		 1071
1072 igb_reset_interrupt_capability(adapter);
1073
1074 /* If we can't do MSI-X, try MSI */
1075msi_only:
2a3abf6d
AD		 1076#ifdef CONFIG_PCI_IOV
1077 /* disable SR-IOV for non MSI-X configurations */
1078 if (adapter->vf_data) {
1079 struct e1000_hw *hw = &adapter->hw;
1080 /* disable iov and allow time for transactions to clear */
1081 pci_disable_sriov(adapter->pdev);
1082 msleep(500);
1083
1084 kfree(adapter->vf_data);
1085 adapter->vf_data = NULL;
1086 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 1087 wrfl();
2a3abf6d
AD		 1088 msleep(100);
1089 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1090 }
1091#endif
4fc82adf 1092 adapter->vfs_allocated_count = 0;
a99955fc 1093 adapter->rss_queues = 1;
4fc82adf 1094 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 1095 adapter->num_rx_queues = 1;
661086df 1096 adapter->num_tx_queues = 1;
047e0030 1097 adapter->num_q_vectors = 1;
9d5c8243 1098 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1099 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 1100out:
21adef3e
BH		 1101 /* Notify the stack of the (possibly) reduced queue counts. */
1102 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
1103 return netif_set_real_num_rx_queues(adapter->netdev,
1104 adapter->num_rx_queues);
9d5c8243
AK		 1105}
1106
047e0030
AD		 1107/**
1108 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1109 * @adapter: board private structure to initialize
1110 *
1111 * We allocate one q_vector per queue interrupt. If allocation fails we
1112 * return -ENOMEM.
1113 **/
1114static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1115{
1116 struct igb_q_vector *q_vector;
1117 struct e1000_hw *hw = &adapter->hw;
1118 int v_idx;
81c2fc22 1119 int orig_node = adapter->node;
047e0030
AD		 1120
1121 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
81c2fc22
AD		 1122 if ((adapter->num_q_vectors == (adapter->num_rx_queues +
1123 adapter->num_tx_queues)) &&
1124 (adapter->num_rx_queues == v_idx))
1125 adapter->node = orig_node;
1126 if (orig_node == -1) {
1127 int cur_node = next_online_node(adapter->node);
1128 if (cur_node == MAX_NUMNODES)
1129 cur_node = first_online_node;
1130 adapter->node = cur_node;
1131 }
1132 q_vector = kzalloc_node(sizeof(struct igb_q_vector), GFP_KERNEL,
1133 adapter->node);
1134 if (!q_vector)
1135 q_vector = kzalloc(sizeof(struct igb_q_vector),
1136 GFP_KERNEL);
047e0030
AD		 1137 if (!q_vector)
1138 goto err_out;
1139 q_vector->adapter = adapter;
047e0030
AD		 1140 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1141 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1142 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1143 adapter->q_vector[v_idx] = q_vector;
1144 }
81c2fc22
AD		 1145 /* Restore the adapter's original node */
1146 adapter->node = orig_node;
1147
047e0030
AD		 1148 return 0;
1149
1150err_out:
81c2fc22
AD		 1151 /* Restore the adapter's original node */
1152 adapter->node = orig_node;
fe0592b4 1153 igb_free_q_vectors(adapter);
047e0030
AD		 1154 return -ENOMEM;
1155}
1156
1157static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1158 int ring_idx, int v_idx)
1159{
3025a446 1160 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1161
0ba82994
AD		 1162 q_vector->rx.ring = adapter->rx_ring[ring_idx];
1163 q_vector->rx.ring->q_vector = q_vector;
1164 q_vector->rx.count++;
4fc82adf
AD		 1165 q_vector->itr_val = adapter->rx_itr_setting;
1166 if (q_vector->itr_val && q_vector->itr_val <= 3)
1167 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1168}
1169
1170static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1171 int ring_idx, int v_idx)
1172{
3025a446 1173 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1174
0ba82994
AD		 1175 q_vector->tx.ring = adapter->tx_ring[ring_idx];
1176 q_vector->tx.ring->q_vector = q_vector;
1177 q_vector->tx.count++;
4fc82adf 1178 q_vector->itr_val = adapter->tx_itr_setting;
0ba82994 1179 q_vector->tx.work_limit = adapter->tx_work_limit;
4fc82adf
AD		 1180 if (q_vector->itr_val && q_vector->itr_val <= 3)
1181 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1182}
1183
1184/**
1185 * igb_map_ring_to_vector - maps allocated queues to vectors
1186 *
1187 * This function maps the recently allocated queues to vectors.
1188 **/
1189static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1190{
1191 int i;
1192 int v_idx = 0;
1193
1194 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1195 (adapter->num_q_vectors < adapter->num_tx_queues))
1196 return -ENOMEM;
1197
1198 if (adapter->num_q_vectors >=
1199 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1200 for (i = 0; i < adapter->num_rx_queues; i++)
1201 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1202 for (i = 0; i < adapter->num_tx_queues; i++)
1203 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1204 } else {
1205 for (i = 0; i < adapter->num_rx_queues; i++) {
1206 if (i < adapter->num_tx_queues)
1207 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1208 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1209 }
1210 for (; i < adapter->num_tx_queues; i++)
1211 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1212 }
1213 return 0;
1214}
1215
1216/**
1217 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1218 *
1219 * This function initializes the interrupts and allocates all of the queues.
1220 **/
1221static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1222{
1223 struct pci_dev *pdev = adapter->pdev;
1224 int err;
1225
21adef3e
BH		 1226 err = igb_set_interrupt_capability(adapter);
1227 if (err)
1228 return err;
047e0030
AD		 1229
1230 err = igb_alloc_q_vectors(adapter);
1231 if (err) {
1232 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1233 goto err_alloc_q_vectors;
1234 }
1235
1236 err = igb_alloc_queues(adapter);
1237 if (err) {
1238 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1239 goto err_alloc_queues;
1240 }
1241
1242 err = igb_map_ring_to_vector(adapter);
1243 if (err) {
1244 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1245 goto err_map_queues;
1246 }
1247
1248
1249 return 0;
1250err_map_queues:
1251 igb_free_queues(adapter);
1252err_alloc_queues:
1253 igb_free_q_vectors(adapter);
1254err_alloc_q_vectors:
1255 igb_reset_interrupt_capability(adapter);
1256 return err;
1257}
1258
9d5c8243
AK		 1259/**
1260 * igb_request_irq - initialize interrupts
1261 *
1262 * Attempts to configure interrupts using the best available
1263 * capabilities of the hardware and kernel.
1264 **/
1265static int igb_request_irq(struct igb_adapter *adapter)
1266{
1267 struct net_device *netdev = adapter->netdev;
047e0030 1268 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1269 int err = 0;
1270
1271 if (adapter->msix_entries) {
1272 err = igb_request_msix(adapter);
844290e5 1273 if (!err)
9d5c8243 1274 goto request_done;
9d5c8243 1275 /* fall back to MSI */
047e0030 1276 igb_clear_interrupt_scheme(adapter);
c74d588e 1277 if (!pci_enable_msi(pdev))
7dfc16fa 1278 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1279 igb_free_all_tx_resources(adapter);
1280 igb_free_all_rx_resources(adapter);
047e0030 1281 adapter->num_tx_queues = 1;
9d5c8243 1282 adapter->num_rx_queues = 1;
047e0030
AD		 1283 adapter->num_q_vectors = 1;
1284 err = igb_alloc_q_vectors(adapter);
1285 if (err) {
1286 dev_err(&pdev->dev,
1287 "Unable to allocate memory for vectors\n");
1288 goto request_done;
1289 }
1290 err = igb_alloc_queues(adapter);
1291 if (err) {
1292 dev_err(&pdev->dev,
1293 "Unable to allocate memory for queues\n");
1294 igb_free_q_vectors(adapter);
1295 goto request_done;
1296 }
1297 igb_setup_all_tx_resources(adapter);
1298 igb_setup_all_rx_resources(adapter);
9d5c8243 1299 }
844290e5 1300
c74d588e
AD		 1301 igb_assign_vector(adapter->q_vector[0], 0);
1302
7dfc16fa 1303 if (adapter->flags & IGB_FLAG_HAS_MSI) {
c74d588e 1304 err = request_irq(pdev->irq, igb_intr_msi, 0,
047e0030 1305 netdev->name, adapter);
9d5c8243
AK		 1306 if (!err)
1307 goto request_done;
047e0030 1308
9d5c8243
AK		 1309 /* fall back to legacy interrupts */
1310 igb_reset_interrupt_capability(adapter);
7dfc16fa 1311 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1312 }
1313
c74d588e 1314 err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
047e0030 1315 netdev->name, adapter);
9d5c8243 1316
6cb5e577 1317 if (err)
c74d588e 1318 dev_err(&pdev->dev, "Error %d getting interrupt\n",
9d5c8243 1319 err);
9d5c8243
AK		 1320
1321request_done:
1322 return err;
1323}
1324
1325static void igb_free_irq(struct igb_adapter *adapter)
1326{
9d5c8243
AK		 1327 if (adapter->msix_entries) {
1328 int vector = 0, i;
1329
047e0030 1330 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 1331
0d1ae7f4 1332 for (i = 0; i < adapter->num_q_vectors; i++)
047e0030 1333 free_irq(adapter->msix_entries[vector++].vector,
0d1ae7f4 1334 adapter->q_vector[i]);
047e0030
AD		 1335 } else {
1336 free_irq(adapter->pdev->irq, adapter);
9d5c8243 1337 }
9d5c8243
AK		 1338}
1339
1340/**
1341 * igb_irq_disable - Mask off interrupt generation on the NIC
1342 * @adapter: board private structure
1343 **/
1344static void igb_irq_disable(struct igb_adapter *adapter)
1345{
1346 struct e1000_hw *hw = &adapter->hw;
1347
25568a53
AD		 1348 /*
1349 * we need to be careful when disabling interrupts. The VFs are also
1350 * mapped into these registers and so clearing the bits can cause
1351 * issues on the VF drivers so we only need to clear what we set
1352 */
9d5c8243 1353 if (adapter->msix_entries) {
2dfd1212
AD		 1354 u32 regval = rd32(E1000_EIAM);
1355 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1356 wr32(E1000_EIMC, adapter->eims_enable_mask);
1357 regval = rd32(E1000_EIAC);
1358 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 1359 }
844290e5
PW		 1360
1361 wr32(E1000_IAM, 0);
9d5c8243
AK		 1362 wr32(E1000_IMC, ~0);
1363 wrfl();
81a61859
ET		 1364 if (adapter->msix_entries) {
1365 int i;
1366 for (i = 0; i < adapter->num_q_vectors; i++)
1367 synchronize_irq(adapter->msix_entries[i].vector);
1368 } else {
1369 synchronize_irq(adapter->pdev->irq);
1370 }
9d5c8243
AK		 1371}
1372
1373/**
1374 * igb_irq_enable - Enable default interrupt generation settings
1375 * @adapter: board private structure
1376 **/
1377static void igb_irq_enable(struct igb_adapter *adapter)
1378{
1379 struct e1000_hw *hw = &adapter->hw;
1380
1381 if (adapter->msix_entries) {
06218a8d 1382 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
2dfd1212
AD		 1383 u32 regval = rd32(E1000_EIAC);
1384 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1385 regval = rd32(E1000_EIAM);
1386 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 1387 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 1388 if (adapter->vfs_allocated_count) {
4ae196df 1389 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1390 ims |= E1000_IMS_VMMB;
1391 }
1392 wr32(E1000_IMS, ims);
844290e5 1393 } else {
55cac248
AD		 1394 wr32(E1000_IMS, IMS_ENABLE_MASK |
1395 E1000_IMS_DRSTA);
1396 wr32(E1000_IAM, IMS_ENABLE_MASK |
1397 E1000_IMS_DRSTA);
844290e5 1398 }
9d5c8243
AK		 1399}
1400
1401static void igb_update_mng_vlan(struct igb_adapter *adapter)
1402{
51466239 1403 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1404 u16 vid = adapter->hw.mng_cookie.vlan_id;
1405 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1406
1407 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1408 /* add VID to filter table */
1409 igb_vfta_set(hw, vid, true);
1410 adapter->mng_vlan_id = vid;
1411 } else {
1412 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1413 }
1414
1415 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1416 (vid != old_vid) &&
b2cb09b1 1417 !test_bit(old_vid, adapter->active_vlans)) {
51466239
AD		 1418 /* remove VID from filter table */
1419 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1420 }
1421}
1422
1423/**
1424 * igb_release_hw_control - release control of the h/w to f/w
1425 * @adapter: address of board private structure
1426 *
1427 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1428 * For ASF and Pass Through versions of f/w this means that the
1429 * driver is no longer loaded.
1430 *
1431 **/
1432static void igb_release_hw_control(struct igb_adapter *adapter)
1433{
1434 struct e1000_hw *hw = &adapter->hw;
1435 u32 ctrl_ext;
1436
1437 /* Let firmware take over control of h/w */
1438 ctrl_ext = rd32(E1000_CTRL_EXT);
1439 wr32(E1000_CTRL_EXT,
1440 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1441}
1442
9d5c8243
AK		 1443/**
1444 * igb_get_hw_control - get control of the h/w from f/w
1445 * @adapter: address of board private structure
1446 *
1447 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1448 * For ASF and Pass Through versions of f/w this means that
1449 * the driver is loaded.
1450 *
1451 **/
1452static void igb_get_hw_control(struct igb_adapter *adapter)
1453{
1454 struct e1000_hw *hw = &adapter->hw;
1455 u32 ctrl_ext;
1456
1457 /* Let firmware know the driver has taken over */
1458 ctrl_ext = rd32(E1000_CTRL_EXT);
1459 wr32(E1000_CTRL_EXT,
1460 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1461}
1462
9d5c8243
AK		 1463/**
1464 * igb_configure - configure the hardware for RX and TX
1465 * @adapter: private board structure
1466 **/
1467static void igb_configure(struct igb_adapter *adapter)
1468{
1469 struct net_device *netdev = adapter->netdev;
1470 int i;
1471
1472 igb_get_hw_control(adapter);
ff41f8dc 1473 igb_set_rx_mode(netdev);
9d5c8243
AK		 1474
1475 igb_restore_vlan(adapter);
9d5c8243 1476
85b430b4 1477 igb_setup_tctl(adapter);
06cf2666 1478 igb_setup_mrqc(adapter);
9d5c8243 1479 igb_setup_rctl(adapter);
85b430b4
AD		 1480
1481 igb_configure_tx(adapter);
9d5c8243 1482 igb_configure_rx(adapter);
662d7205
AD		 1483
1484 igb_rx_fifo_flush_82575(&adapter->hw);
1485
c493ea45 1486 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1487 * at least 1 descriptor unused to make sure
1488 * next_to_use != next_to_clean */
1489 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1490 struct igb_ring *ring = adapter->rx_ring[i];
cd392f5c 1491 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
9d5c8243 1492 }
9d5c8243
AK		 1493}
1494
88a268c1
NN		 1495/**
1496 * igb_power_up_link - Power up the phy/serdes link
1497 * @adapter: address of board private structure
1498 **/
1499void igb_power_up_link(struct igb_adapter *adapter)
1500{
1501 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1502 igb_power_up_phy_copper(&adapter->hw);
1503 else
1504 igb_power_up_serdes_link_82575(&adapter->hw);
1505}
1506
1507/**
1508 * igb_power_down_link - Power down the phy/serdes link
1509 * @adapter: address of board private structure
1510 */
1511static void igb_power_down_link(struct igb_adapter *adapter)
1512{
1513 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1514 igb_power_down_phy_copper_82575(&adapter->hw);
1515 else
1516 igb_shutdown_serdes_link_82575(&adapter->hw);
1517}
9d5c8243
AK		 1518
1519/**
1520 * igb_up - Open the interface and prepare it to handle traffic
1521 * @adapter: board private structure
1522 **/
9d5c8243
AK		 1523int igb_up(struct igb_adapter *adapter)
1524{
1525 struct e1000_hw *hw = &adapter->hw;
1526 int i;
1527
1528 /* hardware has been reset, we need to reload some things */
1529 igb_configure(adapter);
1530
1531 clear_bit(__IGB_DOWN, &adapter->state);
1532
0d1ae7f4
AD		 1533 for (i = 0; i < adapter->num_q_vectors; i++)
1534 napi_enable(&(adapter->q_vector[i]->napi));
1535
844290e5 1536 if (adapter->msix_entries)
9d5c8243 1537 igb_configure_msix(adapter);
feeb2721
AD		 1538 else
1539 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1540
1541 /* Clear any pending interrupts. */
1542 rd32(E1000_ICR);
1543 igb_irq_enable(adapter);
1544
d4960307
AD		 1545 /* notify VFs that reset has been completed */
1546 if (adapter->vfs_allocated_count) {
1547 u32 reg_data = rd32(E1000_CTRL_EXT);
1548 reg_data |= E1000_CTRL_EXT_PFRSTD;
1549 wr32(E1000_CTRL_EXT, reg_data);
1550 }
1551
4cb9be7a
JB		 1552 netif_tx_start_all_queues(adapter->netdev);
1553
25568a53
AD		 1554 /* start the watchdog. */
1555 hw->mac.get_link_status = 1;
1556 schedule_work(&adapter->watchdog_task);
1557
9d5c8243
AK		 1558 return 0;
1559}
1560
1561void igb_down(struct igb_adapter *adapter)
1562{
9d5c8243 1563 struct net_device *netdev = adapter->netdev;
330a6d6a 1564 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1565 u32 tctl, rctl;
1566 int i;
1567
1568 /* signal that we're down so the interrupt handler does not
1569 * reschedule our watchdog timer */
1570 set_bit(__IGB_DOWN, &adapter->state);
1571
1572 /* disable receives in the hardware */
1573 rctl = rd32(E1000_RCTL);
1574 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1575 /* flush and sleep below */
1576
fd2ea0a7 1577 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1578
1579 /* disable transmits in the hardware */
1580 tctl = rd32(E1000_TCTL);
1581 tctl &= ~E1000_TCTL_EN;
1582 wr32(E1000_TCTL, tctl);
1583 /* flush both disables and wait for them to finish */
1584 wrfl();
1585 msleep(10);
1586
0d1ae7f4
AD		 1587 for (i = 0; i < adapter->num_q_vectors; i++)
1588 napi_disable(&(adapter->q_vector[i]->napi));
9d5c8243 1589
9d5c8243
AK		 1590 igb_irq_disable(adapter);
1591
1592 del_timer_sync(&adapter->watchdog_timer);
1593 del_timer_sync(&adapter->phy_info_timer);
1594
9d5c8243 1595 netif_carrier_off(netdev);
04fe6358
AD		 1596
1597 /* record the stats before reset*/
12dcd86b
ED		 1598 spin_lock(&adapter->stats64_lock);
1599 igb_update_stats(adapter, &adapter->stats64);
1600 spin_unlock(&adapter->stats64_lock);
04fe6358 1601
9d5c8243
AK		 1602 adapter->link_speed = 0;
1603 adapter->link_duplex = 0;
1604
3023682e
JK		 1605 if (!pci_channel_offline(adapter->pdev))
1606 igb_reset(adapter);
9d5c8243
AK		 1607 igb_clean_all_tx_rings(adapter);
1608 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1609#ifdef CONFIG_IGB_DCA
1610
1611 /* since we reset the hardware DCA settings were cleared */
1612 igb_setup_dca(adapter);
1613#endif
9d5c8243
AK		 1614}
1615
1616void igb_reinit_locked(struct igb_adapter *adapter)
1617{
1618 WARN_ON(in_interrupt());
1619 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1620 msleep(1);
1621 igb_down(adapter);
1622 igb_up(adapter);
1623 clear_bit(__IGB_RESETTING, &adapter->state);
1624}
1625
1626void igb_reset(struct igb_adapter *adapter)
1627{
090b1795 1628 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1629 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1630 struct e1000_mac_info *mac = &hw->mac;
1631 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1632 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1633 u16 hwm;
1634
1635 /* Repartition Pba for greater than 9k mtu
1636 * To take effect CTRL.RST is required.
1637 */
fa4dfae0 1638 switch (mac->type) {
d2ba2ed8 1639 case e1000_i350:
55cac248
AD		 1640 case e1000_82580:
1641 pba = rd32(E1000_RXPBS);
1642 pba = igb_rxpbs_adjust_82580(pba);
1643 break;
fa4dfae0 1644 case e1000_82576:
d249be54
AD		 1645 pba = rd32(E1000_RXPBS);
1646 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1647 break;
1648 case e1000_82575:
1649 default:
1650 pba = E1000_PBA_34K;
1651 break;
2d064c06 1652 }
9d5c8243 1653
2d064c06
AD		 1654 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1655 (mac->type < e1000_82576)) {
9d5c8243
AK		 1656 /* adjust PBA for jumbo frames */
1657 wr32(E1000_PBA, pba);
1658
1659 /* To maintain wire speed transmits, the Tx FIFO should be
1660 * large enough to accommodate two full transmit packets,
1661 * rounded up to the next 1KB and expressed in KB. Likewise,
1662 * the Rx FIFO should be large enough to accommodate at least
1663 * one full receive packet and is similarly rounded up and
1664 * expressed in KB. */
1665 pba = rd32(E1000_PBA);
1666 /* upper 16 bits has Tx packet buffer allocation size in KB */
1667 tx_space = pba >> 16;
1668 /* lower 16 bits has Rx packet buffer allocation size in KB */
1669 pba &= 0xffff;
1670 /* the tx fifo also stores 16 bytes of information about the tx
1671 * but don't include ethernet FCS because hardware appends it */
1672 min_tx_space = (adapter->max_frame_size +
85e8d004 1673 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1674 ETH_FCS_LEN) * 2;
1675 min_tx_space = ALIGN(min_tx_space, 1024);
1676 min_tx_space >>= 10;
1677 /* software strips receive CRC, so leave room for it */
1678 min_rx_space = adapter->max_frame_size;
1679 min_rx_space = ALIGN(min_rx_space, 1024);
1680 min_rx_space >>= 10;
1681
1682 /* If current Tx allocation is less than the min Tx FIFO size,
1683 * and the min Tx FIFO size is less than the current Rx FIFO
1684 * allocation, take space away from current Rx allocation */
1685 if (tx_space < min_tx_space &&
1686 ((min_tx_space - tx_space) < pba)) {
1687 pba = pba - (min_tx_space - tx_space);
1688
1689 /* if short on rx space, rx wins and must trump tx
1690 * adjustment */
1691 if (pba < min_rx_space)
1692 pba = min_rx_space;
1693 }
2d064c06 1694 wr32(E1000_PBA, pba);
9d5c8243 1695 }
9d5c8243
AK		 1696
1697 /* flow control settings */
1698 /* The high water mark must be low enough to fit one full frame
1699 * (or the size used for early receive) above it in the Rx FIFO.
1700 * Set it to the lower of:
1701 * - 90% of the Rx FIFO size, or
1702 * - the full Rx FIFO size minus one full frame */
1703 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1704 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1705
d405ea3e
AD		 1706 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1707 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1708 fc->pause_time = 0xFFFF;
1709 fc->send_xon = 1;
0cce119a 1710 fc->current_mode = fc->requested_mode;
9d5c8243 1711
4ae196df
AD		 1712 /* disable receive for all VFs and wait one second */
1713 if (adapter->vfs_allocated_count) {
1714 int i;
1715 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
8fa7e0f7 1716 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
4ae196df
AD		 1717
1718 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1719 igb_ping_all_vfs(adapter);
4ae196df
AD		 1720
1721 /* disable transmits and receives */
1722 wr32(E1000_VFRE, 0);
1723 wr32(E1000_VFTE, 0);
1724 }
1725
9d5c8243 1726 /* Allow time for pending master requests to run */
330a6d6a 1727 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1728 wr32(E1000_WUC, 0);
1729
330a6d6a 1730 if (hw->mac.ops.init_hw(hw))
090b1795 1731 dev_err(&pdev->dev, "Hardware Error\n");
831ec0b4 1732
b6e0c419 1733 igb_init_dmac(adapter, pba);
88a268c1
NN		 1734 if (!netif_running(adapter->netdev))
1735 igb_power_down_link(adapter);
1736
9d5c8243
AK		 1737 igb_update_mng_vlan(adapter);
1738
1739 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1740 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1741
330a6d6a 1742 igb_get_phy_info(hw);
9d5c8243
AK		 1743}
1744
c8f44aff
MM		 1745static netdev_features_t igb_fix_features(struct net_device *netdev,
1746 netdev_features_t features)
b2cb09b1
JP		 1747{
1748 /*
1749 * Since there is no support for separate rx/tx vlan accel
1750 * enable/disable make sure tx flag is always in same state as rx.
1751 */
1752 if (features & NETIF_F_HW_VLAN_RX)
1753 features |= NETIF_F_HW_VLAN_TX;
1754 else
1755 features &= ~NETIF_F_HW_VLAN_TX;
1756
1757 return features;
1758}
1759
c8f44aff
MM		 1760static int igb_set_features(struct net_device *netdev,
1761 netdev_features_t features)
ac52caa3 1762{
c8f44aff 1763 netdev_features_t changed = netdev->features ^ features;
ac52caa3 1764
b2cb09b1
JP		 1765 if (changed & NETIF_F_HW_VLAN_RX)
1766 igb_vlan_mode(netdev, features);
1767
ac52caa3
MM		 1768 return 0;
1769}
1770
2e5c6922 1771static const struct net_device_ops igb_netdev_ops = {
559e9c49 1772 .ndo_open = igb_open,
2e5c6922 1773 .ndo_stop = igb_close,
cd392f5c 1774 .ndo_start_xmit = igb_xmit_frame,
12dcd86b 1775 .ndo_get_stats64 = igb_get_stats64,
ff41f8dc 1776 .ndo_set_rx_mode = igb_set_rx_mode,
2e5c6922
SH		 1777 .ndo_set_mac_address = igb_set_mac,
1778 .ndo_change_mtu = igb_change_mtu,
1779 .ndo_do_ioctl = igb_ioctl,
1780 .ndo_tx_timeout = igb_tx_timeout,
1781 .ndo_validate_addr = eth_validate_addr,
2e5c6922
SH		 1782 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1783 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1784 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1785 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1786 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1787 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1788#ifdef CONFIG_NET_POLL_CONTROLLER
1789 .ndo_poll_controller = igb_netpoll,
1790#endif
b2cb09b1
JP		 1791 .ndo_fix_features = igb_fix_features,
1792 .ndo_set_features = igb_set_features,
2e5c6922
SH		 1793};
1794
9d5c8243
AK		 1795/**
1796 * igb_probe - Device Initialization Routine
1797 * @pdev: PCI device information struct
1798 * @ent: entry in igb_pci_tbl
1799 *
1800 * Returns 0 on success, negative on failure
1801 *
1802 * igb_probe initializes an adapter identified by a pci_dev structure.
1803 * The OS initialization, configuring of the adapter private structure,
1804 * and a hardware reset occur.
1805 **/
1806static int __devinit igb_probe(struct pci_dev *pdev,
1807 const struct pci_device_id *ent)
1808{
1809 struct net_device *netdev;
1810 struct igb_adapter *adapter;
1811 struct e1000_hw *hw;
4337e993 1812 u16 eeprom_data = 0;
9835fd73 1813 s32 ret_val;
4337e993 1814 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1815 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1816 unsigned long mmio_start, mmio_len;
2d6a5e95 1817 int err, pci_using_dac;
9d5c8243 1818 u16 eeprom_apme_mask = IGB_EEPROM_APME;
9835fd73 1819 u8 part_str[E1000_PBANUM_LENGTH];
9d5c8243 1820
bded64a7
AG		 1821 /* Catch broken hardware that put the wrong VF device ID in
1822 * the PCIe SR-IOV capability.
1823 */
1824 if (pdev->is_virtfn) {
1825 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1826 pci_name(pdev), pdev->vendor, pdev->device);
1827 return -EINVAL;
1828 }
1829
aed5dec3 1830 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1831 if (err)
1832 return err;
1833
1834 pci_using_dac = 0;
59d71989 1835 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243 1836 if (!err) {
59d71989 1837 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243
AK		 1838 if (!err)
1839 pci_using_dac = 1;
1840 } else {
59d71989 1841 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243 1842 if (err) {
59d71989 1843 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243
AK		 1844 if (err) {
1845 dev_err(&pdev->dev, "No usable DMA "
1846 "configuration, aborting\n");
1847 goto err_dma;
1848 }
1849 }
1850 }
1851
aed5dec3
AD		 1852 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1853 IORESOURCE_MEM),
1854 igb_driver_name);
9d5c8243
AK		 1855 if (err)
1856 goto err_pci_reg;
1857
19d5afd4 1858 pci_enable_pcie_error_reporting(pdev);
40a914fa 1859
9d5c8243 1860 pci_set_master(pdev);
c682fc23 1861 pci_save_state(pdev);
9d5c8243
AK		 1862
1863 err = -ENOMEM;
1bfaf07b 1864 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1cc3bd87 1865 IGB_MAX_TX_QUEUES);
9d5c8243
AK		 1866 if (!netdev)
1867 goto err_alloc_etherdev;
1868
1869 SET_NETDEV_DEV(netdev, &pdev->dev);
1870
1871 pci_set_drvdata(pdev, netdev);
1872 adapter = netdev_priv(netdev);
1873 adapter->netdev = netdev;
1874 adapter->pdev = pdev;
1875 hw = &adapter->hw;
1876 hw->back = adapter;
1877 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1878
1879 mmio_start = pci_resource_start(pdev, 0);
1880 mmio_len = pci_resource_len(pdev, 0);
1881
1882 err = -EIO;
28b0759c
AD		 1883 hw->hw_addr = ioremap(mmio_start, mmio_len);
1884 if (!hw->hw_addr)
9d5c8243
AK		 1885 goto err_ioremap;
1886
2e5c6922 1887 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1888 igb_set_ethtool_ops(netdev);
9d5c8243 1889 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1890
1891 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1892
1893 netdev->mem_start = mmio_start;
1894 netdev->mem_end = mmio_start + mmio_len;
1895
9d5c8243
AK		 1896 /* PCI config space info */
1897 hw->vendor_id = pdev->vendor;
1898 hw->device_id = pdev->device;
1899 hw->revision_id = pdev->revision;
1900 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1901 hw->subsystem_device_id = pdev->subsystem_device;
1902
9d5c8243
AK		 1903 /* Copy the default MAC, PHY and NVM function pointers */
1904 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1905 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1906 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1907 /* Initialize skew-specific constants */
1908 err = ei->get_invariants(hw);
1909 if (err)
450c87c8 1910 goto err_sw_init;
9d5c8243 1911
450c87c8 1912 /* setup the private structure */
9d5c8243
AK		 1913 err = igb_sw_init(adapter);
1914 if (err)
1915 goto err_sw_init;
1916
1917 igb_get_bus_info_pcie(hw);
1918
1919 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 1920
1921 /* Copper options */
1922 if (hw->phy.media_type == e1000_media_type_copper) {
1923 hw->phy.mdix = AUTO_ALL_MODES;
1924 hw->phy.disable_polarity_correction = false;
1925 hw->phy.ms_type = e1000_ms_hw_default;
1926 }
1927
1928 if (igb_check_reset_block(hw))
1929 dev_info(&pdev->dev,
1930 "PHY reset is blocked due to SOL/IDER session.\n");
1931
077887c3
AD		 1932 /*
1933 * features is initialized to 0 in allocation, it might have bits
1934 * set by igb_sw_init so we should use an or instead of an
1935 * assignment.
1936 */
1937 netdev->features |= NETIF_F_SG |
1938 NETIF_F_IP_CSUM |
1939 NETIF_F_IPV6_CSUM |
1940 NETIF_F_TSO |
1941 NETIF_F_TSO6 |
1942 NETIF_F_RXHASH |
1943 NETIF_F_RXCSUM |
1944 NETIF_F_HW_VLAN_RX |
1945 NETIF_F_HW_VLAN_TX;
1946
1947 /* copy netdev features into list of user selectable features */
1948 netdev->hw_features |= netdev->features;
1949
1950 /* set this bit last since it cannot be part of hw_features */
1951 netdev->features |= NETIF_F_HW_VLAN_FILTER;
1952
1953 netdev->vlan_features |= NETIF_F_TSO |
1954 NETIF_F_TSO6 |
1955 NETIF_F_IP_CSUM |
1956 NETIF_F_IPV6_CSUM |
1957 NETIF_F_SG;
48f29ffc 1958
7b872a55 1959 if (pci_using_dac) {
9d5c8243 1960 netdev->features |= NETIF_F_HIGHDMA;
7b872a55
YZ		 1961 netdev->vlan_features |= NETIF_F_HIGHDMA;
1962 }
9d5c8243 1963
ac52caa3
MM		 1964 if (hw->mac.type >= e1000_82576) {
1965 netdev->hw_features |= NETIF_F_SCTP_CSUM;
b9473560 1966 netdev->features |= NETIF_F_SCTP_CSUM;
ac52caa3 1967 }
b9473560 1968
01789349
JP		 1969 netdev->priv_flags |= IFF_UNICAST_FLT;
1970
330a6d6a 1971 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 1972
1973 /* before reading the NVM, reset the controller to put the device in a
1974 * known good starting state */
1975 hw->mac.ops.reset_hw(hw);
1976
1977 /* make sure the NVM is good */
4322e561 1978 if (hw->nvm.ops.validate(hw) < 0) {
9d5c8243
AK		 1979 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1980 err = -EIO;
1981 goto err_eeprom;
1982 }
1983
1984 /* copy the MAC address out of the NVM */
1985 if (hw->mac.ops.read_mac_addr(hw))
1986 dev_err(&pdev->dev, "NVM Read Error\n");
1987
1988 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1989 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1990
1991 if (!is_valid_ether_addr(netdev->perm_addr)) {
1992 dev_err(&pdev->dev, "Invalid MAC Address\n");
1993 err = -EIO;
1994 goto err_eeprom;
1995 }
1996
c061b18d 1997 setup_timer(&adapter->watchdog_timer, igb_watchdog,
0e340485 1998 (unsigned long) adapter);
c061b18d 1999 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
0e340485 2000 (unsigned long) adapter);
9d5c8243
AK		 2001
2002 INIT_WORK(&adapter->reset_task, igb_reset_task);
2003 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2004
450c87c8 2005 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 2006 adapter->fc_autoneg = true;
2007 hw->mac.autoneg = true;
2008 hw->phy.autoneg_advertised = 0x2f;
2009
0cce119a
AD		 2010 hw->fc.requested_mode = e1000_fc_default;
2011 hw->fc.current_mode = e1000_fc_default;
9d5c8243 2012
9d5c8243
AK		 2013 igb_validate_mdi_setting(hw);
2014
9d5c8243
AK		 2015 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2016 * enable the ACPI Magic Packet filter
2017 */
2018
a2cf8b6c 2019 if (hw->bus.func == 0)
312c75ae 2020 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
6d337dce 2021 else if (hw->mac.type >= e1000_82580)
55cac248
AD		 2022 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2023 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2024 &eeprom_data);
a2cf8b6c
AD		 2025 else if (hw->bus.func == 1)
2026 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 2027
2028 if (eeprom_data & eeprom_apme_mask)
2029 adapter->eeprom_wol |= E1000_WUFC_MAG;
2030
2031 /* now that we have the eeprom settings, apply the special cases where
2032 * the eeprom may be wrong or the board simply won't support wake on
2033 * lan on a particular port */
2034 switch (pdev->device) {
2035 case E1000_DEV_ID_82575GB_QUAD_COPPER:
2036 adapter->eeprom_wol = 0;
2037 break;
2038 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 2039 case E1000_DEV_ID_82576_FIBER:
2040 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 2041 /* Wake events only supported on port A for dual fiber
2042 * regardless of eeprom setting */
2043 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
2044 adapter->eeprom_wol = 0;
2045 break;
c8ea5ea9 2046 case E1000_DEV_ID_82576_QUAD_COPPER:
d5aa2252 2047 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
c8ea5ea9
AD		 2048 /* if quad port adapter, disable WoL on all but port A */
2049 if (global_quad_port_a != 0)
2050 adapter->eeprom_wol = 0;
2051 else
2052 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2053 /* Reset for multiple quad port adapters */
2054 if (++global_quad_port_a == 4)
2055 global_quad_port_a = 0;
2056 break;
9d5c8243
AK		 2057 }
2058
2059 /* initialize the wol settings based on the eeprom settings */
2060 adapter->wol = adapter->eeprom_wol;
e1b86d84 2061 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 2062
2063 /* reset the hardware with the new settings */
2064 igb_reset(adapter);
2065
2066 /* let the f/w know that the h/w is now under the control of the
2067 * driver. */
2068 igb_get_hw_control(adapter);
2069
9d5c8243
AK		 2070 strcpy(netdev->name, "eth%d");
2071 err = register_netdev(netdev);
2072 if (err)
2073 goto err_register;
2074
b168dfc5
JB		 2075 /* carrier off reporting is important to ethtool even BEFORE open */
2076 netif_carrier_off(netdev);
2077
421e02f0 2078#ifdef CONFIG_IGB_DCA
bbd98fe4 2079 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 2080 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 2081 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 2082 igb_setup_dca(adapter);
2083 }
fe4506b6 2084
38c845c7 2085#endif
673b8b70
AB		 2086 /* do hw tstamp init after resetting */
2087 igb_init_hw_timer(adapter);
2088
9d5c8243
AK		 2089 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
2090 /* print bus type/speed/width info */
7c510e4b 2091 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 2092 netdev->name,
559e9c49 2093 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
ff846f52 2094 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
559e9c49 2095 "unknown"),
59c3de89
AD		 2096 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
2097 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
2098 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2099 "unknown"),
7c510e4b 2100 netdev->dev_addr);
9d5c8243 2101
9835fd73
CW		 2102 ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH);
2103 if (ret_val)
2104 strcpy(part_str, "Unknown");
2105 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
9d5c8243
AK		 2106 dev_info(&pdev->dev,
2107 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2108 adapter->msix_entries ? "MSI-X" :
7dfc16fa 2109 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243 2110 adapter->num_rx_queues, adapter->num_tx_queues);
09b068d4
CW		 2111 switch (hw->mac.type) {
2112 case e1000_i350:
2113 igb_set_eee_i350(hw);
2114 break;
2115 default:
2116 break;
2117 }
9d5c8243
AK		 2118 return 0;
2119
2120err_register:
2121 igb_release_hw_control(adapter);
2122err_eeprom:
2123 if (!igb_check_reset_block(hw))
f5f4cf08 2124 igb_reset_phy(hw);
9d5c8243
AK		 2125
2126 if (hw->flash_address)
2127 iounmap(hw->flash_address);
9d5c8243 2128err_sw_init:
047e0030 2129 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 2130 iounmap(hw->hw_addr);
2131err_ioremap:
2132 free_netdev(netdev);
2133err_alloc_etherdev:
559e9c49
AD		 2134 pci_release_selected_regions(pdev,
2135 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2136err_pci_reg:
2137err_dma:
2138 pci_disable_device(pdev);
2139 return err;
2140}
2141
2142/**
2143 * igb_remove - Device Removal Routine
2144 * @pdev: PCI device information struct
2145 *
2146 * igb_remove is called by the PCI subsystem to alert the driver
2147 * that it should release a PCI device. The could be caused by a
2148 * Hot-Plug event, or because the driver is going to be removed from
2149 * memory.
2150 **/
2151static void __devexit igb_remove(struct pci_dev *pdev)
2152{
2153 struct net_device *netdev = pci_get_drvdata(pdev);
2154 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 2155 struct e1000_hw *hw = &adapter->hw;
9d5c8243 2156
760141a5
TH		 2157 /*
2158 * The watchdog timer may be rescheduled, so explicitly
2159 * disable watchdog from being rescheduled.
2160 */
9d5c8243
AK		 2161 set_bit(__IGB_DOWN, &adapter->state);
2162 del_timer_sync(&adapter->watchdog_timer);
2163 del_timer_sync(&adapter->phy_info_timer);
2164
760141a5
TH		 2165 cancel_work_sync(&adapter->reset_task);
2166 cancel_work_sync(&adapter->watchdog_task);
9d5c8243 2167
421e02f0 2168#ifdef CONFIG_IGB_DCA
7dfc16fa 2169 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 2170 dev_info(&pdev->dev, "DCA disabled\n");
2171 dca_remove_requester(&pdev->dev);
7dfc16fa 2172 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 2173 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 2174 }
2175#endif
2176
9d5c8243
AK		 2177 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2178 * would have already happened in close and is redundant. */
2179 igb_release_hw_control(adapter);
2180
2181 unregister_netdev(netdev);
2182
047e0030 2183 igb_clear_interrupt_scheme(adapter);
9d5c8243 2184
37680117
AD		 2185#ifdef CONFIG_PCI_IOV
2186 /* reclaim resources allocated to VFs */
2187 if (adapter->vf_data) {
2188 /* disable iov and allow time for transactions to clear */
0224d663
GR		 2189 if (!igb_check_vf_assignment(adapter)) {
2190 pci_disable_sriov(pdev);
2191 msleep(500);
2192 } else {
2193 dev_info(&pdev->dev, "VF(s) assigned to guests!\n");
2194 }
37680117
AD		 2195
2196 kfree(adapter->vf_data);
2197 adapter->vf_data = NULL;
2198 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 2199 wrfl();
37680117
AD		 2200 msleep(100);
2201 dev_info(&pdev->dev, "IOV Disabled\n");
2202 }
2203#endif
559e9c49 2204
28b0759c
AD		 2205 iounmap(hw->hw_addr);
2206 if (hw->flash_address)
2207 iounmap(hw->flash_address);
559e9c49
AD		 2208 pci_release_selected_regions(pdev,
2209 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243 2210
1128c756 2211 kfree(adapter->shadow_vfta);
9d5c8243
AK		 2212 free_netdev(netdev);
2213
19d5afd4 2214 pci_disable_pcie_error_reporting(pdev);
40a914fa 2215
9d5c8243
AK		 2216 pci_disable_device(pdev);
2217}
2218
a6b623e0
AD		 2219/**
2220 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2221 * @adapter: board private structure to initialize
2222 *
2223 * This function initializes the vf specific data storage and then attempts to
2224 * allocate the VFs. The reason for ordering it this way is because it is much
2225 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2226 * the memory for the VFs.
2227 **/
2228static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2229{
2230#ifdef CONFIG_PCI_IOV
2231 struct pci_dev *pdev = adapter->pdev;
0224d663
GR		 2232 int old_vfs = igb_find_enabled_vfs(adapter);
2233 int i;
a6b623e0 2234
0224d663
GR		 2235 if (old_vfs) {
2236 dev_info(&pdev->dev, "%d pre-allocated VFs found - override "
2237 "max_vfs setting of %d\n", old_vfs, max_vfs);
2238 adapter->vfs_allocated_count = old_vfs;
a6b623e0
AD		 2239 }
2240
0224d663
GR		 2241 if (!adapter->vfs_allocated_count)
2242 return;
2243
2244 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2245 sizeof(struct vf_data_storage), GFP_KERNEL);
2246 /* if allocation failed then we do not support SR-IOV */
2247 if (!adapter->vf_data) {
a6b623e0 2248 adapter->vfs_allocated_count = 0;
0224d663
GR		 2249 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2250 "Data Storage\n");
2251 goto out;
a6b623e0 2252 }
0224d663
GR		 2253
2254 if (!old_vfs) {
2255 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count))
2256 goto err_out;
2257 }
2258 dev_info(&pdev->dev, "%d VFs allocated\n",
2259 adapter->vfs_allocated_count);
2260 for (i = 0; i < adapter->vfs_allocated_count; i++)
2261 igb_vf_configure(adapter, i);
2262
2263 /* DMA Coalescing is not supported in IOV mode. */
2264 adapter->flags &= ~IGB_FLAG_DMAC;
2265 goto out;
2266err_out:
2267 kfree(adapter->vf_data);
2268 adapter->vf_data = NULL;
2269 adapter->vfs_allocated_count = 0;
2270out:
2271 return;
a6b623e0
AD		 2272#endif /* CONFIG_PCI_IOV */
2273}
2274
115f459a
AD		 2275/**
2276 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2277 * @adapter: board private structure to initialize
2278 *
2279 * igb_init_hw_timer initializes the function pointer and values for the hw
2280 * timer found in hardware.
2281 **/
2282static void igb_init_hw_timer(struct igb_adapter *adapter)
2283{
2284 struct e1000_hw *hw = &adapter->hw;
2285
2286 switch (hw->mac.type) {
d2ba2ed8 2287 case e1000_i350:
55cac248
AD		 2288 case e1000_82580:
2289 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2290 adapter->cycles.read = igb_read_clock;
2291 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2292 adapter->cycles.mult = 1;
2293 /*
2294 * The 82580 timesync updates the system timer every 8ns by 8ns
2295 * and the value cannot be shifted. Instead we need to shift
2296 * the registers to generate a 64bit timer value. As a result
2297 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2298 * 24 in order to generate a larger value for synchronization.
2299 */
2300 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2301 /* disable system timer temporarily by setting bit 31 */
2302 wr32(E1000_TSAUXC, 0x80000000);
2303 wrfl();
2304
2305 /* Set registers so that rollover occurs soon to test this. */
2306 wr32(E1000_SYSTIMR, 0x00000000);
2307 wr32(E1000_SYSTIML, 0x80000000);
2308 wr32(E1000_SYSTIMH, 0x000000FF);
2309 wrfl();
2310
2311 /* enable system timer by clearing bit 31 */
2312 wr32(E1000_TSAUXC, 0x0);
2313 wrfl();
2314
2315 timecounter_init(&adapter->clock,
2316 &adapter->cycles,
2317 ktime_to_ns(ktime_get_real()));
2318 /*
2319 * Synchronize our NIC clock against system wall clock. NIC
2320 * time stamp reading requires ~3us per sample, each sample
2321 * was pretty stable even under load => only require 10
2322 * samples for each offset comparison.
2323 */
2324 memset(&adapter->compare, 0, sizeof(adapter->compare));
2325 adapter->compare.source = &adapter->clock;
2326 adapter->compare.target = ktime_get_real;
2327 adapter->compare.num_samples = 10;
2328 timecompare_update(&adapter->compare, 0);
2329 break;
115f459a
AD		 2330 case e1000_82576:
2331 /*
2332 * Initialize hardware timer: we keep it running just in case
2333 * that some program needs it later on.
2334 */
2335 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2336 adapter->cycles.read = igb_read_clock;
2337 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2338 adapter->cycles.mult = 1;
2339 /**
2340 * Scale the NIC clock cycle by a large factor so that
2341 * relatively small clock corrections can be added or
25985edc 2342 * subtracted at each clock tick. The drawbacks of a large
115f459a
AD		 2343 * factor are a) that the clock register overflows more quickly
2344 * (not such a big deal) and b) that the increment per tick has
2345 * to fit into 24 bits. As a result we need to use a shift of
2346 * 19 so we can fit a value of 16 into the TIMINCA register.
2347 */
2348 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2349 wr32(E1000_TIMINCA,
2350 (1 << E1000_TIMINCA_16NS_SHIFT) |
2351 (16 << IGB_82576_TSYNC_SHIFT));
2352
2353 /* Set registers so that rollover occurs soon to test this. */
2354 wr32(E1000_SYSTIML, 0x00000000);
2355 wr32(E1000_SYSTIMH, 0xFF800000);
2356 wrfl();
2357
2358 timecounter_init(&adapter->clock,
2359 &adapter->cycles,
2360 ktime_to_ns(ktime_get_real()));
2361 /*
2362 * Synchronize our NIC clock against system wall clock. NIC
2363 * time stamp reading requires ~3us per sample, each sample
2364 * was pretty stable even under load => only require 10
2365 * samples for each offset comparison.
2366 */
2367 memset(&adapter->compare, 0, sizeof(adapter->compare));
2368 adapter->compare.source = &adapter->clock;
2369 adapter->compare.target = ktime_get_real;
2370 adapter->compare.num_samples = 10;
2371 timecompare_update(&adapter->compare, 0);
2372 break;
2373 case e1000_82575:
2374 /* 82575 does not support timesync */
2375 default:
2376 break;
2377 }
2378
2379}
2380
9d5c8243
AK		 2381/**
2382 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2383 * @adapter: board private structure to initialize
2384 *
2385 * igb_sw_init initializes the Adapter private data structure.
2386 * Fields are initialized based on PCI device information and
2387 * OS network device settings (MTU size).
2388 **/
2389static int __devinit igb_sw_init(struct igb_adapter *adapter)
2390{
2391 struct e1000_hw *hw = &adapter->hw;
2392 struct net_device *netdev = adapter->netdev;
2393 struct pci_dev *pdev = adapter->pdev;
2394
2395 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2396
13fde97a 2397 /* set default ring sizes */
68fd9910
AD		 2398 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2399 adapter->rx_ring_count = IGB_DEFAULT_RXD;
13fde97a
AD		 2400
2401 /* set default ITR values */
4fc82adf
AD		 2402 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2403 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2404
13fde97a
AD		 2405 /* set default work limits */
2406 adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
2407
153285f9
AD		 2408 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
2409 VLAN_HLEN;
9d5c8243
AK		 2410 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2411
81c2fc22
AD		 2412 adapter->node = -1;
2413
12dcd86b 2414 spin_lock_init(&adapter->stats64_lock);
a6b623e0 2415#ifdef CONFIG_PCI_IOV
6b78bb1d
CW		 2416 switch (hw->mac.type) {
2417 case e1000_82576:
2418 case e1000_i350:
9b082d73
SA		 2419 if (max_vfs > 7) {
2420 dev_warn(&pdev->dev,
2421 "Maximum of 7 VFs per PF, using max\n");
2422 adapter->vfs_allocated_count = 7;
2423 } else
2424 adapter->vfs_allocated_count = max_vfs;
6b78bb1d
CW		 2425 break;
2426 default:
2427 break;
2428 }
a6b623e0 2429#endif /* CONFIG_PCI_IOV */
a99955fc 2430 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
665c8c8e
WM		 2431 /* i350 cannot do RSS and SR-IOV at the same time */
2432 if (hw->mac.type == e1000_i350 && adapter->vfs_allocated_count)
2433 adapter->rss_queues = 1;
a99955fc
AD		 2434
2435 /*
2436 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2437 * then we should combine the queues into a queue pair in order to
2438 * conserve interrupts due to limited supply
2439 */
2440 if ((adapter->rss_queues > 4) ||
2441 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2442 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2443
1128c756
CW		 2444 /* Setup and initialize a copy of the hw vlan table array */
2445 adapter->shadow_vfta = kzalloc(sizeof(u32) *
2446 E1000_VLAN_FILTER_TBL_SIZE,
2447 GFP_ATOMIC);
2448
a6b623e0 2449 /* This call may decrease the number of queues */
047e0030 2450 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 2451 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2452 return -ENOMEM;
2453 }
2454
a6b623e0
AD		 2455 igb_probe_vfs(adapter);
2456
9d5c8243
AK		 2457 /* Explicitly disable IRQ since the NIC can be in any state. */
2458 igb_irq_disable(adapter);
2459
831ec0b4
CW		 2460 if (hw->mac.type == e1000_i350)
2461 adapter->flags &= ~IGB_FLAG_DMAC;
2462
9d5c8243
AK		 2463 set_bit(__IGB_DOWN, &adapter->state);
2464 return 0;
2465}
2466
2467/**
2468 * igb_open - Called when a network interface is made active
2469 * @netdev: network interface device structure
2470 *
2471 * Returns 0 on success, negative value on failure
2472 *
2473 * The open entry point is called when a network interface is made
2474 * active by the system (IFF_UP). At this point all resources needed
2475 * for transmit and receive operations are allocated, the interrupt
2476 * handler is registered with the OS, the watchdog timer is started,
2477 * and the stack is notified that the interface is ready.
2478 **/
2479static int igb_open(struct net_device *netdev)
2480{
2481 struct igb_adapter *adapter = netdev_priv(netdev);
2482 struct e1000_hw *hw = &adapter->hw;
2483 int err;
2484 int i;
2485
2486 /* disallow open during test */
2487 if (test_bit(__IGB_TESTING, &adapter->state))
2488 return -EBUSY;
2489
b168dfc5
JB		 2490 netif_carrier_off(netdev);
2491
9d5c8243
AK		 2492 /* allocate transmit descriptors */
2493 err = igb_setup_all_tx_resources(adapter);
2494 if (err)
2495 goto err_setup_tx;
2496
2497 /* allocate receive descriptors */
2498 err = igb_setup_all_rx_resources(adapter);
2499 if (err)
2500 goto err_setup_rx;
2501
88a268c1 2502 igb_power_up_link(adapter);
9d5c8243 2503
9d5c8243
AK		 2504 /* before we allocate an interrupt, we must be ready to handle it.
2505 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2506 * as soon as we call pci_request_irq, so we have to setup our
2507 * clean_rx handler before we do so. */
2508 igb_configure(adapter);
2509
2510 err = igb_request_irq(adapter);
2511 if (err)
2512 goto err_req_irq;
2513
2514 /* From here on the code is the same as igb_up() */
2515 clear_bit(__IGB_DOWN, &adapter->state);
2516
0d1ae7f4
AD		 2517 for (i = 0; i < adapter->num_q_vectors; i++)
2518 napi_enable(&(adapter->q_vector[i]->napi));
9d5c8243
AK		 2519
2520 /* Clear any pending interrupts. */
2521 rd32(E1000_ICR);
844290e5
PW		 2522
2523 igb_irq_enable(adapter);
2524
d4960307
AD		 2525 /* notify VFs that reset has been completed */
2526 if (adapter->vfs_allocated_count) {
2527 u32 reg_data = rd32(E1000_CTRL_EXT);
2528 reg_data |= E1000_CTRL_EXT_PFRSTD;
2529 wr32(E1000_CTRL_EXT, reg_data);
2530 }
2531
d55b53ff
JK		 2532 netif_tx_start_all_queues(netdev);
2533
25568a53
AD		 2534 /* start the watchdog. */
2535 hw->mac.get_link_status = 1;
2536 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2537
2538 return 0;
2539
2540err_req_irq:
2541 igb_release_hw_control(adapter);
88a268c1 2542 igb_power_down_link(adapter);
9d5c8243
AK		 2543 igb_free_all_rx_resources(adapter);
2544err_setup_rx:
2545 igb_free_all_tx_resources(adapter);
2546err_setup_tx:
2547 igb_reset(adapter);
2548
2549 return err;
2550}
2551
2552/**
2553 * igb_close - Disables a network interface
2554 * @netdev: network interface device structure
2555 *
2556 * Returns 0, this is not allowed to fail
2557 *
2558 * The close entry point is called when an interface is de-activated
2559 * by the OS. The hardware is still under the driver's control, but
2560 * needs to be disabled. A global MAC reset is issued to stop the
2561 * hardware, and all transmit and receive resources are freed.
2562 **/
2563static int igb_close(struct net_device *netdev)
2564{
2565 struct igb_adapter *adapter = netdev_priv(netdev);
2566
2567 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2568 igb_down(adapter);
2569
2570 igb_free_irq(adapter);
2571
2572 igb_free_all_tx_resources(adapter);
2573 igb_free_all_rx_resources(adapter);
2574
9d5c8243
AK		 2575 return 0;
2576}
2577
2578/**
2579 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2580 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2581 *
2582 * Return 0 on success, negative on failure
2583 **/
80785298 2584int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2585{
59d71989 2586 struct device *dev = tx_ring->dev;
81c2fc22 2587 int orig_node = dev_to_node(dev);
9d5c8243
AK		 2588 int size;
2589
06034649 2590 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
81c2fc22
AD		 2591 tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node);
2592 if (!tx_ring->tx_buffer_info)
2593 tx_ring->tx_buffer_info = vzalloc(size);
06034649 2594 if (!tx_ring->tx_buffer_info)
9d5c8243 2595 goto err;
9d5c8243
AK		 2596
2597 /* round up to nearest 4K */
85e8d004 2598 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2599 tx_ring->size = ALIGN(tx_ring->size, 4096);
2600
81c2fc22 2601 set_dev_node(dev, tx_ring->numa_node);
59d71989
AD		 2602 tx_ring->desc = dma_alloc_coherent(dev,
2603 tx_ring->size,
2604 &tx_ring->dma,
2605 GFP_KERNEL);
81c2fc22
AD		 2606 set_dev_node(dev, orig_node);
2607 if (!tx_ring->desc)
2608 tx_ring->desc = dma_alloc_coherent(dev,
2609 tx_ring->size,
2610 &tx_ring->dma,
2611 GFP_KERNEL);
9d5c8243
AK		 2612
2613 if (!tx_ring->desc)
2614 goto err;
2615
9d5c8243
AK		 2616 tx_ring->next_to_use = 0;
2617 tx_ring->next_to_clean = 0;
81c2fc22 2618
9d5c8243
AK		 2619 return 0;
2620
2621err:
06034649 2622 vfree(tx_ring->tx_buffer_info);
59d71989 2623 dev_err(dev,
9d5c8243
AK		 2624 "Unable to allocate memory for the transmit descriptor ring\n");
2625 return -ENOMEM;
2626}
2627
2628/**
2629 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2630 * (Descriptors) for all queues
2631 * @adapter: board private structure
2632 *
2633 * Return 0 on success, negative on failure
2634 **/
2635static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2636{
439705e1 2637 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2638 int i, err = 0;
2639
2640 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2641 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2642 if (err) {
439705e1 2643 dev_err(&pdev->dev,
9d5c8243
AK		 2644 "Allocation for Tx Queue %u failed\n", i);
2645 for (i--; i >= 0; i--)
3025a446 2646 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2647 break;
2648 }
2649 }
2650
2651 return err;
2652}
2653
2654/**
85b430b4
AD		 2655 * igb_setup_tctl - configure the transmit control registers
2656 * @adapter: Board private structure
9d5c8243 2657 **/
d7ee5b3a 2658void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2659{
9d5c8243
AK		 2660 struct e1000_hw *hw = &adapter->hw;
2661 u32 tctl;
9d5c8243 2662
85b430b4
AD		 2663 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2664 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2665
2666 /* Program the Transmit Control Register */
9d5c8243
AK		 2667 tctl = rd32(E1000_TCTL);
2668 tctl &= ~E1000_TCTL_CT;
2669 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2670 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2671
2672 igb_config_collision_dist(hw);
2673
9d5c8243
AK		 2674 /* Enable transmits */
2675 tctl |= E1000_TCTL_EN;
2676
2677 wr32(E1000_TCTL, tctl);
2678}
2679
85b430b4
AD		 2680/**
2681 * igb_configure_tx_ring - Configure transmit ring after Reset
2682 * @adapter: board private structure
2683 * @ring: tx ring to configure
2684 *
2685 * Configure a transmit ring after a reset.
2686 **/
d7ee5b3a
AD		 2687void igb_configure_tx_ring(struct igb_adapter *adapter,
2688 struct igb_ring *ring)
85b430b4
AD		 2689{
2690 struct e1000_hw *hw = &adapter->hw;
a74420e0 2691 u32 txdctl = 0;
85b430b4
AD		 2692 u64 tdba = ring->dma;
2693 int reg_idx = ring->reg_idx;
2694
2695 /* disable the queue */
a74420e0 2696 wr32(E1000_TXDCTL(reg_idx), 0);
85b430b4
AD		 2697 wrfl();
2698 mdelay(10);
2699
2700 wr32(E1000_TDLEN(reg_idx),
2701 ring->count * sizeof(union e1000_adv_tx_desc));
2702 wr32(E1000_TDBAL(reg_idx),
2703 tdba & 0x00000000ffffffffULL);
2704 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2705
fce99e34 2706 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
a74420e0 2707 wr32(E1000_TDH(reg_idx), 0);
fce99e34 2708 writel(0, ring->tail);
85b430b4
AD		 2709
2710 txdctl |= IGB_TX_PTHRESH;
2711 txdctl |= IGB_TX_HTHRESH << 8;
2712 txdctl |= IGB_TX_WTHRESH << 16;
2713
2714 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2715 wr32(E1000_TXDCTL(reg_idx), txdctl);
2716}
2717
2718/**
2719 * igb_configure_tx - Configure transmit Unit after Reset
2720 * @adapter: board private structure
2721 *
2722 * Configure the Tx unit of the MAC after a reset.
2723 **/
2724static void igb_configure_tx(struct igb_adapter *adapter)
2725{
2726 int i;
2727
2728 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2729 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2730}
2731
9d5c8243
AK		 2732/**
2733 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2734 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2735 *
2736 * Returns 0 on success, negative on failure
2737 **/
80785298 2738int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2739{
59d71989 2740 struct device *dev = rx_ring->dev;
81c2fc22 2741 int orig_node = dev_to_node(dev);
9d5c8243
AK		 2742 int size, desc_len;
2743
06034649 2744 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
81c2fc22
AD		 2745 rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node);
2746 if (!rx_ring->rx_buffer_info)
2747 rx_ring->rx_buffer_info = vzalloc(size);
06034649 2748 if (!rx_ring->rx_buffer_info)
9d5c8243 2749 goto err;
9d5c8243
AK		 2750
2751 desc_len = sizeof(union e1000_adv_rx_desc);
2752
2753 /* Round up to nearest 4K */
2754 rx_ring->size = rx_ring->count * desc_len;
2755 rx_ring->size = ALIGN(rx_ring->size, 4096);
2756
81c2fc22 2757 set_dev_node(dev, rx_ring->numa_node);
59d71989
AD		 2758 rx_ring->desc = dma_alloc_coherent(dev,
2759 rx_ring->size,
2760 &rx_ring->dma,
2761 GFP_KERNEL);
81c2fc22
AD		 2762 set_dev_node(dev, orig_node);
2763 if (!rx_ring->desc)
2764 rx_ring->desc = dma_alloc_coherent(dev,
2765 rx_ring->size,
2766 &rx_ring->dma,
2767 GFP_KERNEL);
9d5c8243
AK		 2768
2769 if (!rx_ring->desc)
2770 goto err;
2771
2772 rx_ring->next_to_clean = 0;
2773 rx_ring->next_to_use = 0;
9d5c8243 2774
9d5c8243
AK		 2775 return 0;
2776
2777err:
06034649
AD		 2778 vfree(rx_ring->rx_buffer_info);
2779 rx_ring->rx_buffer_info = NULL;
59d71989
AD		 2780 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2781 " ring\n");
9d5c8243
AK		 2782 return -ENOMEM;
2783}
2784
2785/**
2786 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2787 * (Descriptors) for all queues
2788 * @adapter: board private structure
2789 *
2790 * Return 0 on success, negative on failure
2791 **/
2792static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2793{
439705e1 2794 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2795 int i, err = 0;
2796
2797 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2798 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2799 if (err) {
439705e1 2800 dev_err(&pdev->dev,
9d5c8243
AK		 2801 "Allocation for Rx Queue %u failed\n", i);
2802 for (i--; i >= 0; i--)
3025a446 2803 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2804 break;
2805 }
2806 }
2807
2808 return err;
2809}
2810
06cf2666
AD		 2811/**
2812 * igb_setup_mrqc - configure the multiple receive queue control registers
2813 * @adapter: Board private structure
2814 **/
2815static void igb_setup_mrqc(struct igb_adapter *adapter)
2816{
2817 struct e1000_hw *hw = &adapter->hw;
2818 u32 mrqc, rxcsum;
2819 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2820 union e1000_reta {
2821 u32 dword;
2822 u8 bytes[4];
2823 } reta;
2824 static const u8 rsshash[40] = {
2825 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2826 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2827 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2828 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2829
2830 /* Fill out hash function seeds */
2831 for (j = 0; j < 10; j++) {
2832 u32 rsskey = rsshash[(j * 4)];
2833 rsskey |= rsshash[(j * 4) + 1] << 8;
2834 rsskey |= rsshash[(j * 4) + 2] << 16;
2835 rsskey |= rsshash[(j * 4) + 3] << 24;
2836 array_wr32(E1000_RSSRK(0), j, rsskey);
2837 }
2838
a99955fc 2839 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2840
2841 if (adapter->vfs_allocated_count) {
2842 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2843 switch (hw->mac.type) {
d2ba2ed8 2844 case e1000_i350:
55cac248
AD		 2845 case e1000_82580:
2846 num_rx_queues = 1;
2847 shift = 0;
2848 break;
06cf2666
AD		 2849 case e1000_82576:
2850 shift = 3;
2851 num_rx_queues = 2;
2852 break;
2853 case e1000_82575:
2854 shift = 2;
2855 shift2 = 6;
2856 default:
2857 break;
2858 }
2859 } else {
2860 if (hw->mac.type == e1000_82575)
2861 shift = 6;
2862 }
2863
2864 for (j = 0; j < (32 * 4); j++) {
2865 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2866 if (shift2)
2867 reta.bytes[j & 3] |= num_rx_queues << shift2;
2868 if ((j & 3) == 3)
2869 wr32(E1000_RETA(j >> 2), reta.dword);
2870 }
2871
2872 /*
2873 * Disable raw packet checksumming so that RSS hash is placed in
2874 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2875 * offloads as they are enabled by default
2876 */
2877 rxcsum = rd32(E1000_RXCSUM);
2878 rxcsum |= E1000_RXCSUM_PCSD;
2879
2880 if (adapter->hw.mac.type >= e1000_82576)
2881 /* Enable Receive Checksum Offload for SCTP */
2882 rxcsum |= E1000_RXCSUM_CRCOFL;
2883
2884 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2885 wr32(E1000_RXCSUM, rxcsum);
2886
2887 /* If VMDq is enabled then we set the appropriate mode for that, else
2888 * we default to RSS so that an RSS hash is calculated per packet even
2889 * if we are only using one queue */
2890 if (adapter->vfs_allocated_count) {
2891 if (hw->mac.type > e1000_82575) {
2892 /* Set the default pool for the PF's first queue */
2893 u32 vtctl = rd32(E1000_VT_CTL);
2894 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2895 E1000_VT_CTL_DISABLE_DEF_POOL);
2896 vtctl |= adapter->vfs_allocated_count <<
2897 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2898 wr32(E1000_VT_CTL, vtctl);
2899 }
a99955fc 2900 if (adapter->rss_queues > 1)
06cf2666
AD		 2901 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2902 else
2903 mrqc = E1000_MRQC_ENABLE_VMDQ;
2904 } else {
2905 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2906 }
2907 igb_vmm_control(adapter);
2908
4478a9cd
AD		 2909 /*
2910 * Generate RSS hash based on TCP port numbers and/or
2911 * IPv4/v6 src and dst addresses since UDP cannot be
2912 * hashed reliably due to IP fragmentation
2913 */
2914 mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
2915 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2916 E1000_MRQC_RSS_FIELD_IPV6 |
2917 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2918 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
06cf2666
AD		 2919
2920 wr32(E1000_MRQC, mrqc);
2921}
2922
9d5c8243
AK		 2923/**
2924 * igb_setup_rctl - configure the receive control registers
2925 * @adapter: Board private structure
2926 **/
d7ee5b3a 2927void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2928{
2929 struct e1000_hw *hw = &adapter->hw;
2930 u32 rctl;
9d5c8243
AK		 2931
2932 rctl = rd32(E1000_RCTL);
2933
2934 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2935 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2936
69d728ba 2937 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2938 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2939
87cb7e8c
AK		 2940 /*
2941 * enable stripping of CRC. It's unlikely this will break BMC
2942 * redirection as it did with e1000. Newer features require
2943 * that the HW strips the CRC.
73cd78f1 2944 */
87cb7e8c 2945 rctl |= E1000_RCTL_SECRC;
9d5c8243 2946
559e9c49 2947 /* disable store bad packets and clear size bits. */
ec54d7d6 2948 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2949
6ec43fe6
AD		 2950 /* enable LPE to prevent packets larger than max_frame_size */
2951 rctl |= E1000_RCTL_LPE;
9d5c8243 2952
952f72a8
AD		 2953 /* disable queue 0 to prevent tail write w/o re-config */
2954 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2955
e1739522
AD		 2956 /* Attention!!! For SR-IOV PF driver operations you must enable
2957 * queue drop for all VF and PF queues to prevent head of line blocking
2958 * if an un-trusted VF does not provide descriptors to hardware.
2959 */
2960 if (adapter->vfs_allocated_count) {
e1739522
AD		 2961 /* set all queue drop enable bits */
2962 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 2963 }
2964
9d5c8243
AK		 2965 wr32(E1000_RCTL, rctl);
2966}
2967
7d5753f0
AD		 2968static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2969 int vfn)
2970{
2971 struct e1000_hw *hw = &adapter->hw;
2972 u32 vmolr;
2973
2974 /* if it isn't the PF check to see if VFs are enabled and
2975 * increase the size to support vlan tags */
2976 if (vfn < adapter->vfs_allocated_count &&
2977 adapter->vf_data[vfn].vlans_enabled)
2978 size += VLAN_TAG_SIZE;
2979
2980 vmolr = rd32(E1000_VMOLR(vfn));
2981 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2982 vmolr |= size | E1000_VMOLR_LPE;
2983 wr32(E1000_VMOLR(vfn), vmolr);
2984
2985 return 0;
2986}
2987
e1739522
AD		 2988/**
2989 * igb_rlpml_set - set maximum receive packet size
2990 * @adapter: board private structure
2991 *
2992 * Configure maximum receivable packet size.
2993 **/
2994static void igb_rlpml_set(struct igb_adapter *adapter)
2995{
153285f9 2996 u32 max_frame_size = adapter->max_frame_size;
e1739522
AD		 2997 struct e1000_hw *hw = &adapter->hw;
2998 u16 pf_id = adapter->vfs_allocated_count;
2999
e1739522
AD		 3000 if (pf_id) {
3001 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
153285f9
AD		 3002 /*
3003 * If we're in VMDQ or SR-IOV mode, then set global RLPML
3004 * to our max jumbo frame size, in case we need to enable
3005 * jumbo frames on one of the rings later.
3006 * This will not pass over-length frames into the default
3007 * queue because it's gated by the VMOLR.RLPML.
3008 */
7d5753f0 3009 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 3010 }
3011
3012 wr32(E1000_RLPML, max_frame_size);
3013}
3014
8151d294
WM		 3015static inline void igb_set_vmolr(struct igb_adapter *adapter,
3016 int vfn, bool aupe)
7d5753f0
AD		 3017{
3018 struct e1000_hw *hw = &adapter->hw;
3019 u32 vmolr;
3020
3021 /*
3022 * This register exists only on 82576 and newer so if we are older then
3023 * we should exit and do nothing
3024 */
3025 if (hw->mac.type < e1000_82576)
3026 return;
3027
3028 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 3029 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
3030 if (aupe)
3031 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
3032 else
3033 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 3034
3035 /* clear all bits that might not be set */
3036 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
3037
a99955fc 3038 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 3039 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3040 /*
3041 * for VMDq only allow the VFs and pool 0 to accept broadcast and
3042 * multicast packets
3043 */
3044 if (vfn <= adapter->vfs_allocated_count)
3045 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
3046
3047 wr32(E1000_VMOLR(vfn), vmolr);
3048}
3049
85b430b4
AD		 3050/**
3051 * igb_configure_rx_ring - Configure a receive ring after Reset
3052 * @adapter: board private structure
3053 * @ring: receive ring to be configured
3054 *
3055 * Configure the Rx unit of the MAC after a reset.
3056 **/
d7ee5b3a
AD		 3057void igb_configure_rx_ring(struct igb_adapter *adapter,
3058 struct igb_ring *ring)
85b430b4
AD		 3059{
3060 struct e1000_hw *hw = &adapter->hw;
3061 u64 rdba = ring->dma;
3062 int reg_idx = ring->reg_idx;
a74420e0 3063 u32 srrctl = 0, rxdctl = 0;
85b430b4
AD		 3064
3065 /* disable the queue */
a74420e0 3066 wr32(E1000_RXDCTL(reg_idx), 0);
85b430b4
AD		 3067
3068 /* Set DMA base address registers */
3069 wr32(E1000_RDBAL(reg_idx),
3070 rdba & 0x00000000ffffffffULL);
3071 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
3072 wr32(E1000_RDLEN(reg_idx),
3073 ring->count * sizeof(union e1000_adv_rx_desc));
3074
3075 /* initialize head and tail */
fce99e34 3076 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
a74420e0 3077 wr32(E1000_RDH(reg_idx), 0);
fce99e34 3078 writel(0, ring->tail);
85b430b4 3079
952f72a8 3080 /* set descriptor configuration */
44390ca6 3081 srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
952f72a8 3082#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
44390ca6 3083 srrctl |= IGB_RXBUFFER_16384 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
952f72a8 3084#else
44390ca6 3085 srrctl |= (PAGE_SIZE / 2) >> E1000_SRRCTL_BSIZEPKT_SHIFT;
952f72a8 3086#endif
44390ca6 3087 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
06218a8d 3088 if (hw->mac.type >= e1000_82580)
757b77e2 3089 srrctl |= E1000_SRRCTL_TIMESTAMP;
e6bdb6fe
NN		 3090 /* Only set Drop Enable if we are supporting multiple queues */
3091 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
3092 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 3093
3094 wr32(E1000_SRRCTL(reg_idx), srrctl);
3095
7d5753f0 3096 /* set filtering for VMDQ pools */
8151d294 3097 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 3098
85b430b4
AD		 3099 rxdctl |= IGB_RX_PTHRESH;
3100 rxdctl |= IGB_RX_HTHRESH << 8;
3101 rxdctl |= IGB_RX_WTHRESH << 16;
a74420e0
AD		 3102
3103 /* enable receive descriptor fetching */
3104 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
85b430b4
AD		 3105 wr32(E1000_RXDCTL(reg_idx), rxdctl);
3106}
3107
9d5c8243
AK		 3108/**
3109 * igb_configure_rx - Configure receive Unit after Reset
3110 * @adapter: board private structure
3111 *
3112 * Configure the Rx unit of the MAC after a reset.
3113 **/
3114static void igb_configure_rx(struct igb_adapter *adapter)
3115{
9107584e 3116 int i;
9d5c8243 3117
68d480c4
AD		 3118 /* set UTA to appropriate mode */
3119 igb_set_uta(adapter);
3120
26ad9178
AD		 3121 /* set the correct pool for the PF default MAC address in entry 0 */
3122 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
3123 adapter->vfs_allocated_count);
3124
06cf2666
AD		 3125 /* Setup the HW Rx Head and Tail Descriptor Pointers and
3126 * the Base and Length of the Rx Descriptor Ring */
3127 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3128 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 3129}
3130
3131/**
3132 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 3133 * @tx_ring: Tx descriptor ring for a specific queue
3134 *
3135 * Free all transmit software resources
3136 **/
68fd9910 3137void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 3138{
3b644cf6 3139 igb_clean_tx_ring(tx_ring);
9d5c8243 3140
06034649
AD		 3141 vfree(tx_ring->tx_buffer_info);
3142 tx_ring->tx_buffer_info = NULL;
9d5c8243 3143
439705e1
AD		 3144 /* if not set, then don't free */
3145 if (!tx_ring->desc)
3146 return;
3147
59d71989
AD		 3148 dma_free_coherent(tx_ring->dev, tx_ring->size,
3149 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 3150
3151 tx_ring->desc = NULL;
3152}
3153
3154/**
3155 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3156 * @adapter: board private structure
3157 *
3158 * Free all transmit software resources
3159 **/
3160static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3161{
3162 int i;
3163
3164 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3165 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 3166}
3167
ebe42d16
AD		 3168void igb_unmap_and_free_tx_resource(struct igb_ring *ring,
3169 struct igb_tx_buffer *tx_buffer)
3170{
3171 if (tx_buffer->skb) {
3172 dev_kfree_skb_any(tx_buffer->skb);
3173 if (tx_buffer->dma)
3174 dma_unmap_single(ring->dev,
3175 tx_buffer->dma,
3176 tx_buffer->length,
3177 DMA_TO_DEVICE);
3178 } else if (tx_buffer->dma) {
3179 dma_unmap_page(ring->dev,
3180 tx_buffer->dma,
3181 tx_buffer->length,
3182 DMA_TO_DEVICE);
3183 }
3184 tx_buffer->next_to_watch = NULL;
3185 tx_buffer->skb = NULL;
3186 tx_buffer->dma = 0;
3187 /* buffer_info must be completely set up in the transmit path */
9d5c8243
AK		 3188}
3189
3190/**
3191 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 3192 * @tx_ring: ring to be cleaned
3193 **/
3b644cf6 3194static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243 3195{
06034649 3196 struct igb_tx_buffer *buffer_info;
9d5c8243 3197 unsigned long size;
6ad4edfc 3198 u16 i;
9d5c8243 3199
06034649 3200 if (!tx_ring->tx_buffer_info)
9d5c8243
AK		 3201 return;
3202 /* Free all the Tx ring sk_buffs */
3203
3204 for (i = 0; i < tx_ring->count; i++) {
06034649 3205 buffer_info = &tx_ring->tx_buffer_info[i];
80785298 3206 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 3207 }
3208
06034649
AD		 3209 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
3210 memset(tx_ring->tx_buffer_info, 0, size);
9d5c8243
AK		 3211
3212 /* Zero out the descriptor ring */
9d5c8243
AK		 3213 memset(tx_ring->desc, 0, tx_ring->size);
3214
3215 tx_ring->next_to_use = 0;
3216 tx_ring->next_to_clean = 0;
9d5c8243
AK		 3217}
3218
3219/**
3220 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3221 * @adapter: board private structure
3222 **/
3223static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3224{
3225 int i;
3226
3227 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3228 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 3229}
3230
3231/**
3232 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 3233 * @rx_ring: ring to clean the resources from
3234 *
3235 * Free all receive software resources
3236 **/
68fd9910 3237void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 3238{
3b644cf6 3239 igb_clean_rx_ring(rx_ring);
9d5c8243 3240
06034649
AD		 3241 vfree(rx_ring->rx_buffer_info);
3242 rx_ring->rx_buffer_info = NULL;
9d5c8243 3243
439705e1
AD		 3244 /* if not set, then don't free */
3245 if (!rx_ring->desc)
3246 return;
3247
59d71989
AD		 3248 dma_free_coherent(rx_ring->dev, rx_ring->size,
3249 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 3250
3251 rx_ring->desc = NULL;
3252}
3253
3254/**
3255 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3256 * @adapter: board private structure
3257 *
3258 * Free all receive software resources
3259 **/
3260static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3261{
3262 int i;
3263
3264 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3265 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 3266}
3267
3268/**
3269 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 3270 * @rx_ring: ring to free buffers from
3271 **/
3b644cf6 3272static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243 3273{
9d5c8243 3274 unsigned long size;
c023cd88 3275 u16 i;
9d5c8243 3276
06034649 3277 if (!rx_ring->rx_buffer_info)
9d5c8243 3278 return;
439705e1 3279
9d5c8243
AK		 3280 /* Free all the Rx ring sk_buffs */
3281 for (i = 0; i < rx_ring->count; i++) {
06034649 3282 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
9d5c8243 3283 if (buffer_info->dma) {
59d71989 3284 dma_unmap_single(rx_ring->dev,
80785298 3285 buffer_info->dma,
44390ca6 3286 IGB_RX_HDR_LEN,
59d71989 3287 DMA_FROM_DEVICE);
9d5c8243
AK		 3288 buffer_info->dma = 0;
3289 }
3290
3291 if (buffer_info->skb) {
3292 dev_kfree_skb(buffer_info->skb);
3293 buffer_info->skb = NULL;
3294 }
6ec43fe6 3295 if (buffer_info->page_dma) {
59d71989 3296 dma_unmap_page(rx_ring->dev,
80785298 3297 buffer_info->page_dma,
6ec43fe6 3298 PAGE_SIZE / 2,
59d71989 3299 DMA_FROM_DEVICE);
6ec43fe6
AD		 3300 buffer_info->page_dma = 0;
3301 }
9d5c8243 3302 if (buffer_info->page) {
9d5c8243
AK		 3303 put_page(buffer_info->page);
3304 buffer_info->page = NULL;
bf36c1a0 3305 buffer_info->page_offset = 0;
9d5c8243
AK		 3306 }
3307 }
3308
06034649
AD		 3309 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3310 memset(rx_ring->rx_buffer_info, 0, size);
9d5c8243
AK		 3311
3312 /* Zero out the descriptor ring */
3313 memset(rx_ring->desc, 0, rx_ring->size);
3314
3315 rx_ring->next_to_clean = 0;
3316 rx_ring->next_to_use = 0;
9d5c8243
AK		 3317}
3318
3319/**
3320 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3321 * @adapter: board private structure
3322 **/
3323static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3324{
3325 int i;
3326
3327 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3328 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 3329}
3330
3331/**
3332 * igb_set_mac - Change the Ethernet Address of the NIC
3333 * @netdev: network interface device structure
3334 * @p: pointer to an address structure
3335 *
3336 * Returns 0 on success, negative on failure
3337 **/
3338static int igb_set_mac(struct net_device *netdev, void *p)
3339{
3340 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 3341 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 3342 struct sockaddr *addr = p;
3343
3344 if (!is_valid_ether_addr(addr->sa_data))
3345 return -EADDRNOTAVAIL;
3346
3347 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 3348 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 3349
26ad9178
AD		 3350 /* set the correct pool for the new PF MAC address in entry 0 */
3351 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3352 adapter->vfs_allocated_count);
e1739522 3353
9d5c8243
AK		 3354 return 0;
3355}
3356
3357/**
68d480c4 3358 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 3359 * @netdev: network interface device structure
3360 *
68d480c4
AD		 3361 * Writes multicast address list to the MTA hash table.
3362 * Returns: -ENOMEM on failure
3363 * 0 on no addresses written
3364 * X on writing X addresses to MTA
9d5c8243 3365 **/
68d480c4 3366static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 3367{
3368 struct igb_adapter *adapter = netdev_priv(netdev);
3369 struct e1000_hw *hw = &adapter->hw;
22bedad3 3370 struct netdev_hw_addr *ha;
68d480c4 3371 u8 *mta_list;
9d5c8243
AK		 3372 int i;
3373
4cd24eaf 3374 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 3375 /* nothing to program, so clear mc list */
3376 igb_update_mc_addr_list(hw, NULL, 0);
3377 igb_restore_vf_multicasts(adapter);
3378 return 0;
3379 }
9d5c8243 3380
4cd24eaf 3381 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 3382 if (!mta_list)
3383 return -ENOMEM;
ff41f8dc 3384
68d480c4 3385 /* The shared function expects a packed array of only addresses. */
48e2f183 3386 i = 0;
22bedad3
JP		 3387 netdev_for_each_mc_addr(ha, netdev)
3388 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
68d480c4 3389
68d480c4
AD		 3390 igb_update_mc_addr_list(hw, mta_list, i);
3391 kfree(mta_list);
3392
4cd24eaf 3393 return netdev_mc_count(netdev);
68d480c4
AD		 3394}
3395
3396/**
3397 * igb_write_uc_addr_list - write unicast addresses to RAR table
3398 * @netdev: network interface device structure
3399 *
3400 * Writes unicast address list to the RAR table.
3401 * Returns: -ENOMEM on failure/insufficient address space
3402 * 0 on no addresses written
3403 * X on writing X addresses to the RAR table
3404 **/
3405static int igb_write_uc_addr_list(struct net_device *netdev)
3406{
3407 struct igb_adapter *adapter = netdev_priv(netdev);
3408 struct e1000_hw *hw = &adapter->hw;
3409 unsigned int vfn = adapter->vfs_allocated_count;
3410 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3411 int count = 0;
3412
3413 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 3414 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 3415 return -ENOMEM;
9d5c8243 3416
32e7bfc4 3417 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 3418 struct netdev_hw_addr *ha;
32e7bfc4
JP		 3419
3420 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 3421 if (!rar_entries)
3422 break;
26ad9178
AD		 3423 igb_rar_set_qsel(adapter, ha->addr,
3424 rar_entries--,
68d480c4
AD		 3425 vfn);
3426 count++;
ff41f8dc
AD		 3427 }
3428 }
3429 /* write the addresses in reverse order to avoid write combining */
3430 for (; rar_entries > 0 ; rar_entries--) {
3431 wr32(E1000_RAH(rar_entries), 0);
3432 wr32(E1000_RAL(rar_entries), 0);
3433 }
3434 wrfl();
3435
68d480c4
AD		 3436 return count;
3437}
3438
3439/**
3440 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3441 * @netdev: network interface device structure
3442 *
3443 * The set_rx_mode entry point is called whenever the unicast or multicast
3444 * address lists or the network interface flags are updated. This routine is
3445 * responsible for configuring the hardware for proper unicast, multicast,
3446 * promiscuous mode, and all-multi behavior.
3447 **/
3448static void igb_set_rx_mode(struct net_device *netdev)
3449{
3450 struct igb_adapter *adapter = netdev_priv(netdev);
3451 struct e1000_hw *hw = &adapter->hw;
3452 unsigned int vfn = adapter->vfs_allocated_count;
3453 u32 rctl, vmolr = 0;
3454 int count;
3455
3456 /* Check for Promiscuous and All Multicast modes */
3457 rctl = rd32(E1000_RCTL);
3458
3459 /* clear the effected bits */
3460 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3461
3462 if (netdev->flags & IFF_PROMISC) {
3463 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3464 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3465 } else {
3466 if (netdev->flags & IFF_ALLMULTI) {
3467 rctl |= E1000_RCTL_MPE;
3468 vmolr |= E1000_VMOLR_MPME;
3469 } else {
3470 /*
3471 * Write addresses to the MTA, if the attempt fails
25985edc 3472 * then we should just turn on promiscuous mode so
68d480c4
AD		 3473 * that we can at least receive multicast traffic
3474 */
3475 count = igb_write_mc_addr_list(netdev);
3476 if (count < 0) {
3477 rctl |= E1000_RCTL_MPE;
3478 vmolr |= E1000_VMOLR_MPME;
3479 } else if (count) {
3480 vmolr |= E1000_VMOLR_ROMPE;
3481 }
3482 }
3483 /*
3484 * Write addresses to available RAR registers, if there is not
3485 * sufficient space to store all the addresses then enable
25985edc 3486 * unicast promiscuous mode
68d480c4
AD		 3487 */
3488 count = igb_write_uc_addr_list(netdev);
3489 if (count < 0) {
3490 rctl |= E1000_RCTL_UPE;
3491 vmolr |= E1000_VMOLR_ROPE;
3492 }
3493 rctl |= E1000_RCTL_VFE;
28fc06f5 3494 }
68d480c4 3495 wr32(E1000_RCTL, rctl);
28fc06f5 3496
68d480c4
AD		 3497 /*
3498 * In order to support SR-IOV and eventually VMDq it is necessary to set
3499 * the VMOLR to enable the appropriate modes. Without this workaround
3500 * we will have issues with VLAN tag stripping not being done for frames
3501 * that are only arriving because we are the default pool
3502 */
3503 if (hw->mac.type < e1000_82576)
28fc06f5 3504 return;
9d5c8243 3505
68d480c4
AD		 3506 vmolr |= rd32(E1000_VMOLR(vfn)) &
3507 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3508 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3509 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3510}
3511
13800469
GR		 3512static void igb_check_wvbr(struct igb_adapter *adapter)
3513{
3514 struct e1000_hw *hw = &adapter->hw;
3515 u32 wvbr = 0;
3516
3517 switch (hw->mac.type) {
3518 case e1000_82576:
3519 case e1000_i350:
3520 if (!(wvbr = rd32(E1000_WVBR)))
3521 return;
3522 break;
3523 default:
3524 break;
3525 }
3526
3527 adapter->wvbr |= wvbr;
3528}
3529
3530#define IGB_STAGGERED_QUEUE_OFFSET 8
3531
3532static void igb_spoof_check(struct igb_adapter *adapter)
3533{
3534 int j;
3535
3536 if (!adapter->wvbr)
3537 return;
3538
3539 for(j = 0; j < adapter->vfs_allocated_count; j++) {
3540 if (adapter->wvbr & (1 << j) ||
3541 adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3542 dev_warn(&adapter->pdev->dev,
3543 "Spoof event(s) detected on VF %d\n", j);
3544 adapter->wvbr &=
3545 ~((1 << j) |
3546 (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3547 }
3548 }
3549}
3550
9d5c8243
AK		 3551/* Need to wait a few seconds after link up to get diagnostic information from
3552 * the phy */
3553static void igb_update_phy_info(unsigned long data)
3554{
3555 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3556 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3557}
3558
4d6b725e
AD		 3559/**
3560 * igb_has_link - check shared code for link and determine up/down
3561 * @adapter: pointer to driver private info
3562 **/
3145535a 3563bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3564{
3565 struct e1000_hw *hw = &adapter->hw;
3566 bool link_active = false;
3567 s32 ret_val = 0;
3568
3569 /* get_link_status is set on LSC (link status) interrupt or
3570 * rx sequence error interrupt. get_link_status will stay
3571 * false until the e1000_check_for_link establishes link
3572 * for copper adapters ONLY
3573 */
3574 switch (hw->phy.media_type) {
3575 case e1000_media_type_copper:
3576 if (hw->mac.get_link_status) {
3577 ret_val = hw->mac.ops.check_for_link(hw);
3578 link_active = !hw->mac.get_link_status;
3579 } else {
3580 link_active = true;
3581 }
3582 break;
4d6b725e
AD		 3583 case e1000_media_type_internal_serdes:
3584 ret_val = hw->mac.ops.check_for_link(hw);
3585 link_active = hw->mac.serdes_has_link;
3586 break;
3587 default:
3588 case e1000_media_type_unknown:
3589 break;
3590 }
3591
3592 return link_active;
3593}
3594
563988dc
SA		 3595static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
3596{
3597 bool ret = false;
3598 u32 ctrl_ext, thstat;
3599
3600 /* check for thermal sensor event on i350, copper only */
3601 if (hw->mac.type == e1000_i350) {
3602 thstat = rd32(E1000_THSTAT);
3603 ctrl_ext = rd32(E1000_CTRL_EXT);
3604
3605 if ((hw->phy.media_type == e1000_media_type_copper) &&
3606 !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3607 ret = !!(thstat & event);
3608 }
3609 }
3610
3611 return ret;
3612}
3613
9d5c8243
AK		 3614/**
3615 * igb_watchdog - Timer Call-back
3616 * @data: pointer to adapter cast into an unsigned long
3617 **/
3618static void igb_watchdog(unsigned long data)
3619{
3620 struct igb_adapter *adapter = (struct igb_adapter *)data;
3621 /* Do the rest outside of interrupt context */
3622 schedule_work(&adapter->watchdog_task);
3623}
3624
3625static void igb_watchdog_task(struct work_struct *work)
3626{
3627 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3628 struct igb_adapter,
3629 watchdog_task);
9d5c8243 3630 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3631 struct net_device *netdev = adapter->netdev;
563988dc 3632 u32 link;
7a6ea550 3633 int i;
9d5c8243 3634
4d6b725e 3635 link = igb_has_link(adapter);
9d5c8243
AK		 3636 if (link) {
3637 if (!netif_carrier_ok(netdev)) {
3638 u32 ctrl;
330a6d6a
AD		 3639 hw->mac.ops.get_speed_and_duplex(hw,
3640 &adapter->link_speed,
3641 &adapter->link_duplex);
9d5c8243
AK		 3642
3643 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 3644 /* Links status message must follow this format */
3645 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 3646 "Flow Control: %s\n",
559e9c49
AD		 3647 netdev->name,
3648 adapter->link_speed,
3649 adapter->link_duplex == FULL_DUPLEX ?
9d5c8243 3650 "Full Duplex" : "Half Duplex",
559e9c49
AD		 3651 ((ctrl & E1000_CTRL_TFCE) &&
3652 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3653 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3654 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
9d5c8243 3655
563988dc
SA		 3656 /* check for thermal sensor event */
3657 if (igb_thermal_sensor_event(hw, E1000_THSTAT_LINK_THROTTLE)) {
3658 printk(KERN_INFO "igb: %s The network adapter "
3659 "link speed was downshifted "
3660 "because it overheated.\n",
3661 netdev->name);
7ef5ed1c 3662 }
563988dc 3663
d07f3e37 3664 /* adjust timeout factor according to speed/duplex */
9d5c8243
AK		 3665 adapter->tx_timeout_factor = 1;
3666 switch (adapter->link_speed) {
3667 case SPEED_10:
9d5c8243
AK		 3668 adapter->tx_timeout_factor = 14;
3669 break;
3670 case SPEED_100:
9d5c8243
AK		 3671 /* maybe add some timeout factor ? */
3672 break;
3673 }
3674
3675 netif_carrier_on(netdev);
9d5c8243 3676
4ae196df 3677 igb_ping_all_vfs(adapter);
17dc566c 3678 igb_check_vf_rate_limit(adapter);
4ae196df 3679
4b1a9877 3680 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3681 if (!test_bit(__IGB_DOWN, &adapter->state))
3682 mod_timer(&adapter->phy_info_timer,
3683 round_jiffies(jiffies + 2 * HZ));
3684 }
3685 } else {
3686 if (netif_carrier_ok(netdev)) {
3687 adapter->link_speed = 0;
3688 adapter->link_duplex = 0;
563988dc
SA		 3689
3690 /* check for thermal sensor event */
3691 if (igb_thermal_sensor_event(hw, E1000_THSTAT_PWR_DOWN)) {
3692 printk(KERN_ERR "igb: %s The network adapter "
3693 "was stopped because it "
3694 "overheated.\n",
7ef5ed1c 3695 netdev->name);
7ef5ed1c 3696 }
563988dc 3697
527d47c1
AD		 3698 /* Links status message must follow this format */
3699 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3700 netdev->name);
9d5c8243 3701 netif_carrier_off(netdev);
4b1a9877 3702
4ae196df
AD		 3703 igb_ping_all_vfs(adapter);
3704
4b1a9877 3705 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3706 if (!test_bit(__IGB_DOWN, &adapter->state))
3707 mod_timer(&adapter->phy_info_timer,
3708 round_jiffies(jiffies + 2 * HZ));
3709 }
3710 }
3711
12dcd86b
ED		 3712 spin_lock(&adapter->stats64_lock);
3713 igb_update_stats(adapter, &adapter->stats64);
3714 spin_unlock(&adapter->stats64_lock);
9d5c8243 3715
dbabb065 3716 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3717 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3718 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3719 /* We've lost link, so the controller stops DMA,
3720 * but we've got queued Tx work that's never going
3721 * to get done, so reset controller to flush Tx.
3722 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3723 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3724 adapter->tx_timeout_count++;
3725 schedule_work(&adapter->reset_task);
3726 /* return immediately since reset is imminent */
3727 return;
3728 }
9d5c8243 3729 }
9d5c8243 3730
dbabb065 3731 /* Force detection of hung controller every watchdog period */
6d095fa8 3732 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
dbabb065 3733 }
f7ba205e 3734
9d5c8243 3735 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3736 if (adapter->msix_entries) {
047e0030 3737 u32 eics = 0;
0d1ae7f4
AD		 3738 for (i = 0; i < adapter->num_q_vectors; i++)
3739 eics |= adapter->q_vector[i]->eims_value;
7a6ea550
AD		 3740 wr32(E1000_EICS, eics);
3741 } else {
3742 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3743 }
9d5c8243 3744
13800469
GR		 3745 igb_spoof_check(adapter);
3746
9d5c8243
AK		 3747 /* Reset the timer */
3748 if (!test_bit(__IGB_DOWN, &adapter->state))
3749 mod_timer(&adapter->watchdog_timer,
3750 round_jiffies(jiffies + 2 * HZ));
3751}
3752
3753enum latency_range {
3754 lowest_latency = 0,
3755 low_latency = 1,
3756 bulk_latency = 2,
3757 latency_invalid = 255
3758};
3759
6eb5a7f1
AD		 3760/**
3761 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3762 *
3763 * Stores a new ITR value based on strictly on packet size. This
3764 * algorithm is less sophisticated than that used in igb_update_itr,
3765 * due to the difficulty of synchronizing statistics across multiple
eef35c2d 3766 * receive rings. The divisors and thresholds used by this function
6eb5a7f1
AD		 3767 * were determined based on theoretical maximum wire speed and testing
3768 * data, in order to minimize response time while increasing bulk
3769 * throughput.
3770 * This functionality is controlled by the InterruptThrottleRate module
3771 * parameter (see igb_param.c)
3772 * NOTE: This function is called only when operating in a multiqueue
3773 * receive environment.
047e0030 3774 * @q_vector: pointer to q_vector
6eb5a7f1 3775 **/
047e0030 3776static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3777{
047e0030 3778 int new_val = q_vector->itr_val;
6eb5a7f1 3779 int avg_wire_size = 0;
047e0030 3780 struct igb_adapter *adapter = q_vector->adapter;
12dcd86b 3781 unsigned int packets;
9d5c8243 3782
6eb5a7f1
AD		 3783 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3784 * ints/sec - ITR timer value of 120 ticks.
3785 */
3786 if (adapter->link_speed != SPEED_1000) {
0ba82994 3787 new_val = IGB_4K_ITR;
6eb5a7f1 3788 goto set_itr_val;
9d5c8243 3789 }
047e0030 3790
0ba82994
AD		 3791 packets = q_vector->rx.total_packets;
3792 if (packets)
3793 avg_wire_size = q_vector->rx.total_bytes / packets;
047e0030 3794
0ba82994
AD		 3795 packets = q_vector->tx.total_packets;
3796 if (packets)
3797 avg_wire_size = max_t(u32, avg_wire_size,
3798 q_vector->tx.total_bytes / packets);
047e0030
AD		 3799
3800 /* if avg_wire_size isn't set no work was done */
3801 if (!avg_wire_size)
3802 goto clear_counts;
9d5c8243 3803
6eb5a7f1
AD		 3804 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3805 avg_wire_size += 24;
3806
3807 /* Don't starve jumbo frames */
3808 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3809
6eb5a7f1
AD		 3810 /* Give a little boost to mid-size frames */
3811 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3812 new_val = avg_wire_size / 3;
3813 else
3814 new_val = avg_wire_size / 2;
9d5c8243 3815
0ba82994
AD		 3816 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3817 if (new_val < IGB_20K_ITR &&
3818 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3819 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3820 new_val = IGB_20K_ITR;
abe1c363 3821
6eb5a7f1 3822set_itr_val:
047e0030
AD		 3823 if (new_val != q_vector->itr_val) {
3824 q_vector->itr_val = new_val;
3825 q_vector->set_itr = 1;
9d5c8243 3826 }
6eb5a7f1 3827clear_counts:
0ba82994
AD		 3828 q_vector->rx.total_bytes = 0;
3829 q_vector->rx.total_packets = 0;
3830 q_vector->tx.total_bytes = 0;
3831 q_vector->tx.total_packets = 0;
9d5c8243
AK		 3832}
3833
3834/**
3835 * igb_update_itr - update the dynamic ITR value based on statistics
3836 * Stores a new ITR value based on packets and byte
3837 * counts during the last interrupt. The advantage of per interrupt
3838 * computation is faster updates and more accurate ITR for the current
3839 * traffic pattern. Constants in this function were computed
3840 * based on theoretical maximum wire speed and thresholds were set based
3841 * on testing data as well as attempting to minimize response time
3842 * while increasing bulk throughput.
3843 * this functionality is controlled by the InterruptThrottleRate module
3844 * parameter (see igb_param.c)
3845 * NOTE: These calculations are only valid when operating in a single-
3846 * queue environment.
0ba82994
AD		 3847 * @q_vector: pointer to q_vector
3848 * @ring_container: ring info to update the itr for
9d5c8243 3849 **/
0ba82994
AD		 3850static void igb_update_itr(struct igb_q_vector *q_vector,
3851 struct igb_ring_container *ring_container)
9d5c8243 3852{
0ba82994
AD		 3853 unsigned int packets = ring_container->total_packets;
3854 unsigned int bytes = ring_container->total_bytes;
3855 u8 itrval = ring_container->itr;
9d5c8243 3856
0ba82994 3857 /* no packets, exit with status unchanged */
9d5c8243 3858 if (packets == 0)
0ba82994 3859 return;
9d5c8243 3860
0ba82994 3861 switch (itrval) {
9d5c8243
AK		 3862 case lowest_latency:
3863 /* handle TSO and jumbo frames */
3864 if (bytes/packets > 8000)
0ba82994 3865 itrval = bulk_latency;
9d5c8243 3866 else if ((packets < 5) && (bytes > 512))
0ba82994 3867 itrval = low_latency;
9d5c8243
AK		 3868 break;
3869 case low_latency: /* 50 usec aka 20000 ints/s */
3870 if (bytes > 10000) {
3871 /* this if handles the TSO accounting */
3872 if (bytes/packets > 8000) {
0ba82994 3873 itrval = bulk_latency;
9d5c8243 3874 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
0ba82994 3875 itrval = bulk_latency;
9d5c8243 3876 } else if ((packets > 35)) {
0ba82994 3877 itrval = lowest_latency;
9d5c8243
AK		 3878 }
3879 } else if (bytes/packets > 2000) {
0ba82994 3880 itrval = bulk_latency;
9d5c8243 3881 } else if (packets <= 2 && bytes < 512) {
0ba82994 3882 itrval = lowest_latency;
9d5c8243
AK		 3883 }
3884 break;
3885 case bulk_latency: /* 250 usec aka 4000 ints/s */
3886 if (bytes > 25000) {
3887 if (packets > 35)
0ba82994 3888 itrval = low_latency;
1e5c3d21 3889 } else if (bytes < 1500) {
0ba82994 3890 itrval = low_latency;
9d5c8243
AK		 3891 }
3892 break;
3893 }
3894
0ba82994
AD		 3895 /* clear work counters since we have the values we need */
3896 ring_container->total_bytes = 0;
3897 ring_container->total_packets = 0;
3898
3899 /* write updated itr to ring container */
3900 ring_container->itr = itrval;
9d5c8243
AK		 3901}
3902
0ba82994 3903static void igb_set_itr(struct igb_q_vector *q_vector)
9d5c8243 3904{
0ba82994 3905 struct igb_adapter *adapter = q_vector->adapter;
047e0030 3906 u32 new_itr = q_vector->itr_val;
0ba82994 3907 u8 current_itr = 0;
9d5c8243
AK		 3908
3909 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3910 if (adapter->link_speed != SPEED_1000) {
3911 current_itr = 0;
0ba82994 3912 new_itr = IGB_4K_ITR;
9d5c8243
AK		 3913 goto set_itr_now;
3914 }
3915
0ba82994
AD		 3916 igb_update_itr(q_vector, &q_vector->tx);
3917 igb_update_itr(q_vector, &q_vector->rx);
9d5c8243 3918
0ba82994 3919 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
9d5c8243 3920
6eb5a7f1 3921 /* conservative mode (itr 3) eliminates the lowest_latency setting */
0ba82994
AD		 3922 if (current_itr == lowest_latency &&
3923 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3924 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
6eb5a7f1
AD		 3925 current_itr = low_latency;
3926
9d5c8243
AK		 3927 switch (current_itr) {
3928 /* counts and packets in update_itr are dependent on these numbers */
3929 case lowest_latency:
0ba82994 3930 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
9d5c8243
AK		 3931 break;
3932 case low_latency:
0ba82994 3933 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
9d5c8243
AK		 3934 break;
3935 case bulk_latency:
0ba82994 3936 new_itr = IGB_4K_ITR; /* 4,000 ints/sec */
9d5c8243
AK		 3937 break;
3938 default:
3939 break;
3940 }
3941
3942set_itr_now:
047e0030 3943 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3944 /* this attempts to bias the interrupt rate towards Bulk
3945 * by adding intermediate steps when interrupt rate is
3946 * increasing */
047e0030
AD		 3947 new_itr = new_itr > q_vector->itr_val ?
3948 max((new_itr * q_vector->itr_val) /
3949 (new_itr + (q_vector->itr_val >> 2)),
0ba82994 3950 new_itr) :
9d5c8243
AK		 3951 new_itr;
3952 /* Don't write the value here; it resets the adapter's
3953 * internal timer, and causes us to delay far longer than
3954 * we should between interrupts. Instead, we write the ITR
3955 * value at the beginning of the next interrupt so the timing
3956 * ends up being correct.
3957 */
047e0030
AD		 3958 q_vector->itr_val = new_itr;
3959 q_vector->set_itr = 1;
9d5c8243 3960 }
9d5c8243
AK		 3961}
3962
7d13a7d0
AD		 3963void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
3964 u32 type_tucmd, u32 mss_l4len_idx)
3965{
3966 struct e1000_adv_tx_context_desc *context_desc;
3967 u16 i = tx_ring->next_to_use;
3968
3969 context_desc = IGB_TX_CTXTDESC(tx_ring, i);
3970
3971 i++;
3972 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
3973
3974 /* set bits to identify this as an advanced context descriptor */
3975 type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3976
3977 /* For 82575, context index must be unique per ring. */
866cff06 3978 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
7d13a7d0
AD		 3979 mss_l4len_idx |= tx_ring->reg_idx << 4;
3980
3981 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
3982 context_desc->seqnum_seed = 0;
3983 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
3984 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3985}
3986
7af40ad9
AD		 3987static int igb_tso(struct igb_ring *tx_ring,
3988 struct igb_tx_buffer *first,
3989 u8 *hdr_len)
9d5c8243 3990{
7af40ad9 3991 struct sk_buff *skb = first->skb;
7d13a7d0
AD		 3992 u32 vlan_macip_lens, type_tucmd;
3993 u32 mss_l4len_idx, l4len;
3994
3995 if (!skb_is_gso(skb))
3996 return 0;
9d5c8243
AK		 3997
3998 if (skb_header_cloned(skb)) {
7af40ad9 3999 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
9d5c8243
AK		 4000 if (err)
4001 return err;
4002 }
4003
7d13a7d0
AD		 4004 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4005 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
9d5c8243 4006
7af40ad9 4007 if (first->protocol == __constant_htons(ETH_P_IP)) {
9d5c8243
AK		 4008 struct iphdr *iph = ip_hdr(skb);
4009 iph->tot_len = 0;
4010 iph->check = 0;
4011 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
4012 iph->daddr, 0,
4013 IPPROTO_TCP,
4014 0);
7d13a7d0 4015 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
7af40ad9
AD		 4016 first->tx_flags |= IGB_TX_FLAGS_TSO |
4017 IGB_TX_FLAGS_CSUM |
4018 IGB_TX_FLAGS_IPV4;
8e1e8a47 4019 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 4020 ipv6_hdr(skb)->payload_len = 0;
4021 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4022 &ipv6_hdr(skb)->daddr,
4023 0, IPPROTO_TCP, 0);
7af40ad9
AD		 4024 first->tx_flags |= IGB_TX_FLAGS_TSO |
4025 IGB_TX_FLAGS_CSUM;
9d5c8243
AK		 4026 }
4027
7af40ad9 4028 /* compute header lengths */
7d13a7d0
AD		 4029 l4len = tcp_hdrlen(skb);
4030 *hdr_len = skb_transport_offset(skb) + l4len;
9d5c8243 4031
7af40ad9
AD		 4032 /* update gso size and bytecount with header size */
4033 first->gso_segs = skb_shinfo(skb)->gso_segs;
4034 first->bytecount += (first->gso_segs - 1) * *hdr_len;
4035
9d5c8243 4036 /* MSS L4LEN IDX */
7d13a7d0
AD		 4037 mss_l4len_idx = l4len << E1000_ADVTXD_L4LEN_SHIFT;
4038 mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
9d5c8243 4039
7d13a7d0
AD		 4040 /* VLAN MACLEN IPLEN */
4041 vlan_macip_lens = skb_network_header_len(skb);
4042 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
7af40ad9 4043 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
9d5c8243 4044
7d13a7d0 4045 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
9d5c8243 4046
7d13a7d0 4047 return 1;
9d5c8243
AK		 4048}
4049
7af40ad9 4050static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
9d5c8243 4051{
7af40ad9 4052 struct sk_buff *skb = first->skb;
7d13a7d0
AD		 4053 u32 vlan_macip_lens = 0;
4054 u32 mss_l4len_idx = 0;
4055 u32 type_tucmd = 0;
9d5c8243 4056
7d13a7d0 4057 if (skb->ip_summed != CHECKSUM_PARTIAL) {
7af40ad9
AD		 4058 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN))
4059 return;
7d13a7d0
AD		 4060 } else {
4061 u8 l4_hdr = 0;
7af40ad9 4062 switch (first->protocol) {
7d13a7d0
AD		 4063 case __constant_htons(ETH_P_IP):
4064 vlan_macip_lens |= skb_network_header_len(skb);
4065 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4066 l4_hdr = ip_hdr(skb)->protocol;
4067 break;
4068 case __constant_htons(ETH_P_IPV6):
4069 vlan_macip_lens |= skb_network_header_len(skb);
4070 l4_hdr = ipv6_hdr(skb)->nexthdr;
4071 break;
4072 default:
4073 if (unlikely(net_ratelimit())) {
4074 dev_warn(tx_ring->dev,
4075 "partial checksum but proto=%x!\n",
7af40ad9 4076 first->protocol);
fa4a7ef3 4077 }
7d13a7d0
AD		 4078 break;
4079 }
fa4a7ef3 4080
7d13a7d0
AD		 4081 switch (l4_hdr) {
4082 case IPPROTO_TCP:
4083 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4084 mss_l4len_idx = tcp_hdrlen(skb) <<
4085 E1000_ADVTXD_L4LEN_SHIFT;
4086 break;
4087 case IPPROTO_SCTP:
4088 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
4089 mss_l4len_idx = sizeof(struct sctphdr) <<
4090 E1000_ADVTXD_L4LEN_SHIFT;
4091 break;
4092 case IPPROTO_UDP:
4093 mss_l4len_idx = sizeof(struct udphdr) <<
4094 E1000_ADVTXD_L4LEN_SHIFT;
4095 break;
4096 default:
4097 if (unlikely(net_ratelimit())) {
4098 dev_warn(tx_ring->dev,
4099 "partial checksum but l4 proto=%x!\n",
4100 l4_hdr);
44b0cda3 4101 }
7d13a7d0 4102 break;
9d5c8243 4103 }
7af40ad9
AD		 4104
4105 /* update TX checksum flag */
4106 first->tx_flags |= IGB_TX_FLAGS_CSUM;
7d13a7d0 4107 }
9d5c8243 4108
7d13a7d0 4109 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
7af40ad9 4110 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
9d5c8243 4111
7d13a7d0 4112 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
9d5c8243
AK		 4113}
4114
e032afc8
AD		 4115static __le32 igb_tx_cmd_type(u32 tx_flags)
4116{
4117 /* set type for advanced descriptor with frame checksum insertion */
4118 __le32 cmd_type = cpu_to_le32(E1000_ADVTXD_DTYP_DATA |
4119 E1000_ADVTXD_DCMD_IFCS |
4120 E1000_ADVTXD_DCMD_DEXT);
4121
4122 /* set HW vlan bit if vlan is present */
4123 if (tx_flags & IGB_TX_FLAGS_VLAN)
4124 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_VLE);
4125
4126 /* set timestamp bit if present */
4127 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4128 cmd_type |= cpu_to_le32(E1000_ADVTXD_MAC_TSTAMP);
4129
4130 /* set segmentation bits for TSO */
4131 if (tx_flags & IGB_TX_FLAGS_TSO)
4132 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_TSE);
4133
4134 return cmd_type;
4135}
4136
7af40ad9
AD		 4137static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
4138 union e1000_adv_tx_desc *tx_desc,
4139 u32 tx_flags, unsigned int paylen)
e032afc8
AD		 4140{
4141 u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
4142
4143 /* 82575 requires a unique index per ring if any offload is enabled */
4144 if ((tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_VLAN)) &&
866cff06 4145 test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
e032afc8
AD		 4146 olinfo_status |= tx_ring->reg_idx << 4;
4147
4148 /* insert L4 checksum */
4149 if (tx_flags & IGB_TX_FLAGS_CSUM) {
4150 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4151
4152 /* insert IPv4 checksum */
4153 if (tx_flags & IGB_TX_FLAGS_IPV4)
4154 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4155 }
4156
7af40ad9 4157 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
e032afc8
AD		 4158}
4159
ebe42d16
AD		 4160/*
4161 * The largest size we can write to the descriptor is 65535. In order to
4162 * maintain a power of two alignment we have to limit ourselves to 32K.
4163 */
4164#define IGB_MAX_TXD_PWR 15
7af40ad9 4165#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
9d5c8243 4166
7af40ad9
AD		 4167static void igb_tx_map(struct igb_ring *tx_ring,
4168 struct igb_tx_buffer *first,
ebe42d16 4169 const u8 hdr_len)
9d5c8243 4170{
7af40ad9 4171 struct sk_buff *skb = first->skb;
ebe42d16
AD		 4172 struct igb_tx_buffer *tx_buffer_info;
4173 union e1000_adv_tx_desc *tx_desc;
4174 dma_addr_t dma;
4175 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
4176 unsigned int data_len = skb->data_len;
4177 unsigned int size = skb_headlen(skb);
4178 unsigned int paylen = skb->len - hdr_len;
4179 __le32 cmd_type;
7af40ad9 4180 u32 tx_flags = first->tx_flags;
ebe42d16 4181 u16 i = tx_ring->next_to_use;
ebe42d16
AD		 4182
4183 tx_desc = IGB_TX_DESC(tx_ring, i);
4184
7af40ad9 4185 igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, paylen);
ebe42d16
AD		 4186 cmd_type = igb_tx_cmd_type(tx_flags);
4187
4188 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
4189 if (dma_mapping_error(tx_ring->dev, dma))
6366ad33 4190 goto dma_error;
9d5c8243 4191
ebe42d16
AD		 4192 /* record length, and DMA address */
4193 first->length = size;
4194 first->dma = dma;
ebe42d16
AD		 4195 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4196
4197 for (;;) {
4198 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
4199 tx_desc->read.cmd_type_len =
4200 cmd_type | cpu_to_le32(IGB_MAX_DATA_PER_TXD);
4201
4202 i++;
4203 tx_desc++;
4204 if (i == tx_ring->count) {
4205 tx_desc = IGB_TX_DESC(tx_ring, 0);
4206 i = 0;
4207 }
4208
4209 dma += IGB_MAX_DATA_PER_TXD;
4210 size -= IGB_MAX_DATA_PER_TXD;
4211
4212 tx_desc->read.olinfo_status = 0;
4213 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4214 }
4215
4216 if (likely(!data_len))
4217 break;
2bbfebe2 4218
ebe42d16 4219 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
9d5c8243 4220
65689fef 4221 i++;
ebe42d16
AD		 4222 tx_desc++;
4223 if (i == tx_ring->count) {
4224 tx_desc = IGB_TX_DESC(tx_ring, 0);
65689fef 4225 i = 0;
ebe42d16 4226 }
65689fef 4227
9e903e08 4228 size = skb_frag_size(frag);
ebe42d16
AD		 4229 data_len -= size;
4230
4231 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
4232 size, DMA_TO_DEVICE);
4233 if (dma_mapping_error(tx_ring->dev, dma))
6366ad33
AD		 4234 goto dma_error;
4235
ebe42d16
AD		 4236 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4237 tx_buffer_info->length = size;
4238 tx_buffer_info->dma = dma;
4239
4240 tx_desc->read.olinfo_status = 0;
4241 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4242
4243 frag++;
9d5c8243
AK		 4244 }
4245
ebe42d16
AD		 4246 /* write last descriptor with RS and EOP bits */
4247 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IGB_TXD_DCMD);
4248 tx_desc->read.cmd_type_len = cmd_type;
8542db05
AD		 4249
4250 /* set the timestamp */
4251 first->time_stamp = jiffies;
4252
ebe42d16
AD		 4253 /*
4254 * Force memory writes to complete before letting h/w know there
4255 * are new descriptors to fetch. (Only applicable for weak-ordered
4256 * memory model archs, such as IA-64).
4257 *
4258 * We also need this memory barrier to make certain all of the
4259 * status bits have been updated before next_to_watch is written.
4260 */
4261 wmb();
4262
8542db05 4263 /* set next_to_watch value indicating a packet is present */
ebe42d16 4264 first->next_to_watch = tx_desc;
9d5c8243 4265
ebe42d16
AD		 4266 i++;
4267 if (i == tx_ring->count)
4268 i = 0;
6366ad33 4269
ebe42d16 4270 tx_ring->next_to_use = i;
6366ad33 4271
ebe42d16 4272 writel(i, tx_ring->tail);
6366ad33 4273
ebe42d16
AD		 4274 /* we need this if more than one processor can write to our tail
4275 * at a time, it syncronizes IO on IA64/Altix systems */
4276 mmiowb();
4277
4278 return;
4279
4280dma_error:
4281 dev_err(tx_ring->dev, "TX DMA map failed\n");
4282
4283 /* clear dma mappings for failed tx_buffer_info map */
4284 for (;;) {
4285 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4286 igb_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
4287 if (tx_buffer_info == first)
4288 break;
a77ff709
NN		 4289 if (i == 0)
4290 i = tx_ring->count;
6366ad33 4291 i--;
6366ad33
AD		 4292 }
4293
9d5c8243 4294 tx_ring->next_to_use = i;
9d5c8243
AK		 4295}
4296
6ad4edfc 4297static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
9d5c8243 4298{
e694e964
AD		 4299 struct net_device *netdev = tx_ring->netdev;
4300
661086df 4301 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 4302
9d5c8243
AK		 4303 /* Herbert's original patch had:
4304 * smp_mb__after_netif_stop_queue();
4305 * but since that doesn't exist yet, just open code it. */
4306 smp_mb();
4307
4308 /* We need to check again in a case another CPU has just
4309 * made room available. */
c493ea45 4310 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 4311 return -EBUSY;
4312
4313 /* A reprieve! */
661086df 4314 netif_wake_subqueue(netdev, tx_ring->queue_index);
12dcd86b
ED		 4315
4316 u64_stats_update_begin(&tx_ring->tx_syncp2);
4317 tx_ring->tx_stats.restart_queue2++;
4318 u64_stats_update_end(&tx_ring->tx_syncp2);
4319
9d5c8243
AK		 4320 return 0;
4321}
4322
6ad4edfc 4323static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
9d5c8243 4324{
c493ea45 4325 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 4326 return 0;
e694e964 4327 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 4328}
4329
cd392f5c
AD		 4330netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
4331 struct igb_ring *tx_ring)
9d5c8243 4332{
8542db05 4333 struct igb_tx_buffer *first;
ebe42d16 4334 int tso;
91d4ee33 4335 u32 tx_flags = 0;
31f6adbb 4336 __be16 protocol = vlan_get_protocol(skb);
91d4ee33 4337 u8 hdr_len = 0;
9d5c8243 4338
9d5c8243
AK		 4339 /* need: 1 descriptor per page,
4340 * + 2 desc gap to keep tail from touching head,
4341 * + 1 desc for skb->data,
4342 * + 1 desc for context descriptor,
4343 * otherwise try next time */
e694e964 4344 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 4345 /* this is a hard error */
9d5c8243
AK		 4346 return NETDEV_TX_BUSY;
4347 }
33af6bcc 4348
7af40ad9
AD		 4349 /* record the location of the first descriptor for this packet */
4350 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
4351 first->skb = skb;
4352 first->bytecount = skb->len;
4353 first->gso_segs = 1;
4354
2244d07b
OH		 4355 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4356 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
33af6bcc 4357 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 4358 }
9d5c8243 4359
eab6d18d 4360 if (vlan_tx_tag_present(skb)) {
9d5c8243
AK		 4361 tx_flags |= IGB_TX_FLAGS_VLAN;
4362 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4363 }
4364
7af40ad9
AD		 4365 /* record initial flags and protocol */
4366 first->tx_flags = tx_flags;
4367 first->protocol = protocol;
cdfd01fc 4368
7af40ad9
AD		 4369 tso = igb_tso(tx_ring, first, &hdr_len);
4370 if (tso < 0)
7d13a7d0 4371 goto out_drop;
7af40ad9
AD		 4372 else if (!tso)
4373 igb_tx_csum(tx_ring, first);
9d5c8243 4374
7af40ad9 4375 igb_tx_map(tx_ring, first, hdr_len);
85ad76b2
AD		 4376
4377 /* Make sure there is space in the ring for the next send. */
e694e964 4378 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 4379
9d5c8243 4380 return NETDEV_TX_OK;
7d13a7d0
AD		 4381
4382out_drop:
7af40ad9
AD		 4383 igb_unmap_and_free_tx_resource(tx_ring, first);
4384
7d13a7d0 4385 return NETDEV_TX_OK;
9d5c8243
AK		 4386}
4387
1cc3bd87
AD		 4388static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
4389 struct sk_buff *skb)
4390{
4391 unsigned int r_idx = skb->queue_mapping;
4392
4393 if (r_idx >= adapter->num_tx_queues)
4394 r_idx = r_idx % adapter->num_tx_queues;
4395
4396 return adapter->tx_ring[r_idx];
4397}
4398
cd392f5c
AD		 4399static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
4400 struct net_device *netdev)
9d5c8243
AK		 4401{
4402 struct igb_adapter *adapter = netdev_priv(netdev);
b1a436c3
AD		 4403
4404 if (test_bit(__IGB_DOWN, &adapter->state)) {
4405 dev_kfree_skb_any(skb);
4406 return NETDEV_TX_OK;
4407 }
4408
4409 if (skb->len <= 0) {
4410 dev_kfree_skb_any(skb);
4411 return NETDEV_TX_OK;
4412 }
4413
1cc3bd87
AD		 4414 /*
4415 * The minimum packet size with TCTL.PSP set is 17 so pad the skb
4416 * in order to meet this minimum size requirement.
4417 */
4418 if (skb->len < 17) {
4419 if (skb_padto(skb, 17))
4420 return NETDEV_TX_OK;
4421 skb->len = 17;
4422 }
9d5c8243 4423
1cc3bd87 4424 return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
9d5c8243
AK		 4425}
4426
4427/**
4428 * igb_tx_timeout - Respond to a Tx Hang
4429 * @netdev: network interface device structure
4430 **/
4431static void igb_tx_timeout(struct net_device *netdev)
4432{
4433 struct igb_adapter *adapter = netdev_priv(netdev);
4434 struct e1000_hw *hw = &adapter->hw;
4435
4436 /* Do the reset outside of interrupt context */
4437 adapter->tx_timeout_count++;
f7ba205e 4438
06218a8d 4439 if (hw->mac.type >= e1000_82580)
55cac248
AD		 4440 hw->dev_spec._82575.global_device_reset = true;
4441
9d5c8243 4442 schedule_work(&adapter->reset_task);
265de409
AD		 4443 wr32(E1000_EICS,
4444 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 4445}
4446
4447static void igb_reset_task(struct work_struct *work)
4448{
4449 struct igb_adapter *adapter;
4450 adapter = container_of(work, struct igb_adapter, reset_task);
4451
c97ec42a
TI		 4452 igb_dump(adapter);
4453 netdev_err(adapter->netdev, "Reset adapter\n");
9d5c8243
AK		 4454 igb_reinit_locked(adapter);
4455}
4456
4457/**
12dcd86b 4458 * igb_get_stats64 - Get System Network Statistics
9d5c8243 4459 * @netdev: network interface device structure
12dcd86b 4460 * @stats: rtnl_link_stats64 pointer
9d5c8243 4461 *
9d5c8243 4462 **/
12dcd86b
ED		 4463static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4464 struct rtnl_link_stats64 *stats)
9d5c8243 4465{
12dcd86b
ED		 4466 struct igb_adapter *adapter = netdev_priv(netdev);
4467
4468 spin_lock(&adapter->stats64_lock);
4469 igb_update_stats(adapter, &adapter->stats64);
4470 memcpy(stats, &adapter->stats64, sizeof(*stats));
4471 spin_unlock(&adapter->stats64_lock);
4472
4473 return stats;
9d5c8243
AK		 4474}
4475
4476/**
4477 * igb_change_mtu - Change the Maximum Transfer Unit
4478 * @netdev: network interface device structure
4479 * @new_mtu: new value for maximum frame size
4480 *
4481 * Returns 0 on success, negative on failure
4482 **/
4483static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4484{
4485 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4486 struct pci_dev *pdev = adapter->pdev;
153285f9 4487 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
9d5c8243 4488
c809d227 4489 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 4490 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 4491 return -EINVAL;
4492 }
4493
153285f9 4494#define MAX_STD_JUMBO_FRAME_SIZE 9238
9d5c8243 4495 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 4496 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 4497 return -EINVAL;
4498 }
4499
4500 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4501 msleep(1);
73cd78f1 4502
9d5c8243
AK		 4503 /* igb_down has a dependency on max_frame_size */
4504 adapter->max_frame_size = max_frame;
559e9c49 4505
4c844851
AD		 4506 if (netif_running(netdev))
4507 igb_down(adapter);
9d5c8243 4508
090b1795 4509 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 4510 netdev->mtu, new_mtu);
4511 netdev->mtu = new_mtu;
4512
4513 if (netif_running(netdev))
4514 igb_up(adapter);
4515 else
4516 igb_reset(adapter);
4517
4518 clear_bit(__IGB_RESETTING, &adapter->state);
4519
4520 return 0;
4521}
4522
4523/**
4524 * igb_update_stats - Update the board statistics counters
4525 * @adapter: board private structure
4526 **/
4527
12dcd86b
ED		 4528void igb_update_stats(struct igb_adapter *adapter,
4529 struct rtnl_link_stats64 *net_stats)
9d5c8243
AK		 4530{
4531 struct e1000_hw *hw = &adapter->hw;
4532 struct pci_dev *pdev = adapter->pdev;
fa3d9a6d 4533 u32 reg, mpc;
9d5c8243 4534 u16 phy_tmp;
3f9c0164
AD		 4535 int i;
4536 u64 bytes, packets;
12dcd86b
ED		 4537 unsigned int start;
4538 u64 _bytes, _packets;
9d5c8243
AK		 4539
4540#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4541
4542 /*
4543 * Prevent stats update while adapter is being reset, or if the pci
4544 * connection is down.
4545 */
4546 if (adapter->link_speed == 0)
4547 return;
4548 if (pci_channel_offline(pdev))
4549 return;
4550
3f9c0164
AD		 4551 bytes = 0;
4552 packets = 0;
4553 for (i = 0; i < adapter->num_rx_queues; i++) {
4554 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446 4555 struct igb_ring *ring = adapter->rx_ring[i];
12dcd86b 4556
3025a446 4557 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 4558 net_stats->rx_fifo_errors += rqdpc_tmp;
12dcd86b
ED		 4559
4560 do {
4561 start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4562 _bytes = ring->rx_stats.bytes;
4563 _packets = ring->rx_stats.packets;
4564 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4565 bytes += _bytes;
4566 packets += _packets;
3f9c0164
AD		 4567 }
4568
128e45eb
AD		 4569 net_stats->rx_bytes = bytes;
4570 net_stats->rx_packets = packets;
3f9c0164
AD		 4571
4572 bytes = 0;
4573 packets = 0;
4574 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 4575 struct igb_ring *ring = adapter->tx_ring[i];
12dcd86b
ED		 4576 do {
4577 start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4578 _bytes = ring->tx_stats.bytes;
4579 _packets = ring->tx_stats.packets;
4580 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4581 bytes += _bytes;
4582 packets += _packets;
3f9c0164 4583 }
128e45eb
AD		 4584 net_stats->tx_bytes = bytes;
4585 net_stats->tx_packets = packets;
3f9c0164
AD		 4586
4587 /* read stats registers */
9d5c8243
AK		 4588 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4589 adapter->stats.gprc += rd32(E1000_GPRC);
4590 adapter->stats.gorc += rd32(E1000_GORCL);
4591 rd32(E1000_GORCH); /* clear GORCL */
4592 adapter->stats.bprc += rd32(E1000_BPRC);
4593 adapter->stats.mprc += rd32(E1000_MPRC);
4594 adapter->stats.roc += rd32(E1000_ROC);
4595
4596 adapter->stats.prc64 += rd32(E1000_PRC64);
4597 adapter->stats.prc127 += rd32(E1000_PRC127);
4598 adapter->stats.prc255 += rd32(E1000_PRC255);
4599 adapter->stats.prc511 += rd32(E1000_PRC511);
4600 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4601 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4602 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4603 adapter->stats.sec += rd32(E1000_SEC);
4604
fa3d9a6d
MW		 4605 mpc = rd32(E1000_MPC);
4606 adapter->stats.mpc += mpc;
4607 net_stats->rx_fifo_errors += mpc;
9d5c8243
AK		 4608 adapter->stats.scc += rd32(E1000_SCC);
4609 adapter->stats.ecol += rd32(E1000_ECOL);
4610 adapter->stats.mcc += rd32(E1000_MCC);
4611 adapter->stats.latecol += rd32(E1000_LATECOL);
4612 adapter->stats.dc += rd32(E1000_DC);
4613 adapter->stats.rlec += rd32(E1000_RLEC);
4614 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4615 adapter->stats.xontxc += rd32(E1000_XONTXC);
4616 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4617 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4618 adapter->stats.fcruc += rd32(E1000_FCRUC);
4619 adapter->stats.gptc += rd32(E1000_GPTC);
4620 adapter->stats.gotc += rd32(E1000_GOTCL);
4621 rd32(E1000_GOTCH); /* clear GOTCL */
fa3d9a6d 4622 adapter->stats.rnbc += rd32(E1000_RNBC);
9d5c8243
AK		 4623 adapter->stats.ruc += rd32(E1000_RUC);
4624 adapter->stats.rfc += rd32(E1000_RFC);
4625 adapter->stats.rjc += rd32(E1000_RJC);
4626 adapter->stats.tor += rd32(E1000_TORH);
4627 adapter->stats.tot += rd32(E1000_TOTH);
4628 adapter->stats.tpr += rd32(E1000_TPR);
4629
4630 adapter->stats.ptc64 += rd32(E1000_PTC64);
4631 adapter->stats.ptc127 += rd32(E1000_PTC127);
4632 adapter->stats.ptc255 += rd32(E1000_PTC255);
4633 adapter->stats.ptc511 += rd32(E1000_PTC511);
4634 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4635 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4636
4637 adapter->stats.mptc += rd32(E1000_MPTC);
4638 adapter->stats.bptc += rd32(E1000_BPTC);
4639
2d0b0f69
NN		 4640 adapter->stats.tpt += rd32(E1000_TPT);
4641 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4642
4643 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4644 /* read internal phy specific stats */
4645 reg = rd32(E1000_CTRL_EXT);
4646 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4647 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4648 adapter->stats.tncrs += rd32(E1000_TNCRS);
4649 }
4650
9d5c8243
AK		 4651 adapter->stats.tsctc += rd32(E1000_TSCTC);
4652 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4653
4654 adapter->stats.iac += rd32(E1000_IAC);
4655 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4656 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4657 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4658 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4659 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4660 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4661 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4662 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4663
4664 /* Fill out the OS statistics structure */
128e45eb
AD		 4665 net_stats->multicast = adapter->stats.mprc;
4666 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4667
4668 /* Rx Errors */
4669
4670 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4671 * our own version based on RUC and ROC */
128e45eb 4672 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4673 adapter->stats.crcerrs + adapter->stats.algnerrc +
4674 adapter->stats.ruc + adapter->stats.roc +
4675 adapter->stats.cexterr;
128e45eb
AD		 4676 net_stats->rx_length_errors = adapter->stats.ruc +
4677 adapter->stats.roc;
4678 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4679 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4680 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4681
4682 /* Tx Errors */
128e45eb
AD		 4683 net_stats->tx_errors = adapter->stats.ecol +
4684 adapter->stats.latecol;
4685 net_stats->tx_aborted_errors = adapter->stats.ecol;
4686 net_stats->tx_window_errors = adapter->stats.latecol;
4687 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4688
4689 /* Tx Dropped needs to be maintained elsewhere */
4690
4691 /* Phy Stats */
4692 if (hw->phy.media_type == e1000_media_type_copper) {
4693 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4694 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4695 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4696 adapter->phy_stats.idle_errors += phy_tmp;
4697 }
4698 }
4699
4700 /* Management Stats */
4701 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4702 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4703 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
0a915b95
CW		 4704
4705 /* OS2BMC Stats */
4706 reg = rd32(E1000_MANC);
4707 if (reg & E1000_MANC_EN_BMC2OS) {
4708 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
4709 adapter->stats.o2bspc += rd32(E1000_O2BSPC);
4710 adapter->stats.b2ospc += rd32(E1000_B2OSPC);
4711 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
4712 }
9d5c8243
AK		 4713}
4714
9d5c8243
AK		 4715static irqreturn_t igb_msix_other(int irq, void *data)
4716{
047e0030 4717 struct igb_adapter *adapter = data;
9d5c8243 4718 struct e1000_hw *hw = &adapter->hw;
844290e5 4719 u32 icr = rd32(E1000_ICR);
844290e5 4720 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4721
7f081d40
AD		 4722 if (icr & E1000_ICR_DRSTA)
4723 schedule_work(&adapter->reset_task);
4724
047e0030 4725 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4726 /* HW is reporting DMA is out of sync */
4727 adapter->stats.doosync++;
13800469
GR		 4728 /* The DMA Out of Sync is also indication of a spoof event
4729 * in IOV mode. Check the Wrong VM Behavior register to
4730 * see if it is really a spoof event. */
4731 igb_check_wvbr(adapter);
dda0e083 4732 }
eebbbdba 4733
4ae196df
AD		 4734 /* Check for a mailbox event */
4735 if (icr & E1000_ICR_VMMB)
4736 igb_msg_task(adapter);
4737
4738 if (icr & E1000_ICR_LSC) {
4739 hw->mac.get_link_status = 1;
4740 /* guard against interrupt when we're going down */
4741 if (!test_bit(__IGB_DOWN, &adapter->state))
4742 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4743 }
4744
844290e5 4745 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4746
4747 return IRQ_HANDLED;
4748}
4749
047e0030 4750static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4751{
26b39276 4752 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4753 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4754
047e0030
AD		 4755 if (!q_vector->set_itr)
4756 return;
73cd78f1 4757
047e0030
AD		 4758 if (!itr_val)
4759 itr_val = 0x4;
661086df 4760
26b39276
AD		 4761 if (adapter->hw.mac.type == e1000_82575)
4762 itr_val |= itr_val << 16;
661086df 4763 else
0ba82994 4764 itr_val |= E1000_EITR_CNT_IGNR;
661086df 4765
047e0030
AD		 4766 writel(itr_val, q_vector->itr_register);
4767 q_vector->set_itr = 0;
6eb5a7f1
AD		 4768}
4769
047e0030 4770static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4771{
047e0030 4772 struct igb_q_vector *q_vector = data;
9d5c8243 4773
047e0030
AD		 4774 /* Write the ITR value calculated from the previous interrupt. */
4775 igb_write_itr(q_vector);
9d5c8243 4776
047e0030 4777 napi_schedule(&q_vector->napi);
844290e5 4778
047e0030 4779 return IRQ_HANDLED;
fe4506b6
JC		 4780}
4781
421e02f0 4782#ifdef CONFIG_IGB_DCA
047e0030 4783static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4784{
047e0030 4785 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4786 struct e1000_hw *hw = &adapter->hw;
4787 int cpu = get_cpu();
fe4506b6 4788
047e0030
AD		 4789 if (q_vector->cpu == cpu)
4790 goto out_no_update;
4791
0ba82994
AD		 4792 if (q_vector->tx.ring) {
4793 int q = q_vector->tx.ring->reg_idx;
047e0030
AD		 4794 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4795 if (hw->mac.type == e1000_82575) {
4796 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4797 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4798 } else {
047e0030
AD		 4799 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4800 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4801 E1000_DCA_TXCTRL_CPUID_SHIFT;
4802 }
4803 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4804 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4805 }
0ba82994
AD		 4806 if (q_vector->rx.ring) {
4807 int q = q_vector->rx.ring->reg_idx;
047e0030
AD		 4808 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4809 if (hw->mac.type == e1000_82575) {
2d064c06 4810 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4811 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4812 } else {
4813 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4814 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4815 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4816 }
fe4506b6
JC		 4817 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4818 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4819 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4820 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4821 }
047e0030
AD		 4822 q_vector->cpu = cpu;
4823out_no_update:
fe4506b6
JC		 4824 put_cpu();
4825}
4826
4827static void igb_setup_dca(struct igb_adapter *adapter)
4828{
7e0e99ef 4829 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4830 int i;
4831
7dfc16fa 4832 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4833 return;
4834
7e0e99ef
AD		 4835 /* Always use CB2 mode, difference is masked in the CB driver. */
4836 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4837
047e0030 4838 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4839 adapter->q_vector[i]->cpu = -1;
4840 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4841 }
4842}
4843
4844static int __igb_notify_dca(struct device *dev, void *data)
4845{
4846 struct net_device *netdev = dev_get_drvdata(dev);
4847 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4848 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4849 struct e1000_hw *hw = &adapter->hw;
4850 unsigned long event = *(unsigned long *)data;
4851
4852 switch (event) {
4853 case DCA_PROVIDER_ADD:
4854 /* if already enabled, don't do it again */
7dfc16fa 4855 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4856 break;
fe4506b6 4857 if (dca_add_requester(dev) == 0) {
bbd98fe4 4858 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4859 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4860 igb_setup_dca(adapter);
4861 break;
4862 }
4863 /* Fall Through since DCA is disabled. */
4864 case DCA_PROVIDER_REMOVE:
7dfc16fa 4865 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4866 /* without this a class_device is left
047e0030 4867 * hanging around in the sysfs model */
fe4506b6 4868 dca_remove_requester(dev);
090b1795 4869 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4870 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4871 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4872 }
4873 break;
4874 }
bbd98fe4 4875
fe4506b6 4876 return 0;
9d5c8243
AK		 4877}
4878
fe4506b6
JC		 4879static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4880 void *p)
4881{
4882 int ret_val;
4883
4884 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4885 __igb_notify_dca);
4886
4887 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4888}
421e02f0 4889#endif /* CONFIG_IGB_DCA */
9d5c8243 4890
0224d663
GR		 4891#ifdef CONFIG_PCI_IOV
4892static int igb_vf_configure(struct igb_adapter *adapter, int vf)
4893{
4894 unsigned char mac_addr[ETH_ALEN];
4895 struct pci_dev *pdev = adapter->pdev;
4896 struct e1000_hw *hw = &adapter->hw;
4897 struct pci_dev *pvfdev;
4898 unsigned int device_id;
4899 u16 thisvf_devfn;
4900
4901 random_ether_addr(mac_addr);
4902 igb_set_vf_mac(adapter, vf, mac_addr);
4903
4904 switch (adapter->hw.mac.type) {
4905 case e1000_82576:
4906 device_id = IGB_82576_VF_DEV_ID;
4907 /* VF Stride for 82576 is 2 */
4908 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 1)) |
4909 (pdev->devfn & 1);
4910 break;
4911 case e1000_i350:
4912 device_id = IGB_I350_VF_DEV_ID;
4913 /* VF Stride for I350 is 4 */
4914 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 2)) |
4915 (pdev->devfn & 3);
4916 break;
4917 default:
4918 device_id = 0;
4919 thisvf_devfn = 0;
4920 break;
4921 }
4922
4923 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
4924 while (pvfdev) {
4925 if (pvfdev->devfn == thisvf_devfn)
4926 break;
4927 pvfdev = pci_get_device(hw->vendor_id,
4928 device_id, pvfdev);
4929 }
4930
4931 if (pvfdev)
4932 adapter->vf_data[vf].vfdev = pvfdev;
4933 else
4934 dev_err(&pdev->dev,
4935 "Couldn't find pci dev ptr for VF %4.4x\n",
4936 thisvf_devfn);
4937 return pvfdev != NULL;
4938}
4939
4940static int igb_find_enabled_vfs(struct igb_adapter *adapter)
4941{
4942 struct e1000_hw *hw = &adapter->hw;
4943 struct pci_dev *pdev = adapter->pdev;
4944 struct pci_dev *pvfdev;
4945 u16 vf_devfn = 0;
4946 u16 vf_stride;
4947 unsigned int device_id;
4948 int vfs_found = 0;
4949
4950 switch (adapter->hw.mac.type) {
4951 case e1000_82576:
4952 device_id = IGB_82576_VF_DEV_ID;
4953 /* VF Stride for 82576 is 2 */
4954 vf_stride = 2;
4955 break;
4956 case e1000_i350:
4957 device_id = IGB_I350_VF_DEV_ID;
4958 /* VF Stride for I350 is 4 */
4959 vf_stride = 4;
4960 break;
4961 default:
4962 device_id = 0;
4963 vf_stride = 0;
4964 break;
4965 }
4966
4967 vf_devfn = pdev->devfn + 0x80;
4968 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
4969 while (pvfdev) {
4970 if (pvfdev->devfn == vf_devfn)
4971 vfs_found++;
4972 vf_devfn += vf_stride;
4973 pvfdev = pci_get_device(hw->vendor_id,
4974 device_id, pvfdev);
4975 }
4976
4977 return vfs_found;
4978}
4979
4980static int igb_check_vf_assignment(struct igb_adapter *adapter)
4981{
4982 int i;
4983 for (i = 0; i < adapter->vfs_allocated_count; i++) {
4984 if (adapter->vf_data[i].vfdev) {
4985 if (adapter->vf_data[i].vfdev->dev_flags &
4986 PCI_DEV_FLAGS_ASSIGNED)
4987 return true;
4988 }
4989 }
4990 return false;
4991}
4992
4993#endif
4ae196df
AD		 4994static void igb_ping_all_vfs(struct igb_adapter *adapter)
4995{
4996 struct e1000_hw *hw = &adapter->hw;
4997 u32 ping;
4998 int i;
4999
5000 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
5001 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 5002 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 5003 ping |= E1000_VT_MSGTYPE_CTS;
5004 igb_write_mbx(hw, &ping, 1, i);
5005 }
5006}
5007
7d5753f0
AD		 5008static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5009{
5010 struct e1000_hw *hw = &adapter->hw;
5011 u32 vmolr = rd32(E1000_VMOLR(vf));
5012 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5013
d85b9004 5014 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
7d5753f0
AD		 5015 IGB_VF_FLAG_MULTI_PROMISC);
5016 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5017
5018 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
5019 vmolr |= E1000_VMOLR_MPME;
d85b9004 5020 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
7d5753f0
AD		 5021 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
5022 } else {
5023 /*
5024 * if we have hashes and we are clearing a multicast promisc
5025 * flag we need to write the hashes to the MTA as this step
5026 * was previously skipped
5027 */
5028 if (vf_data->num_vf_mc_hashes > 30) {
5029 vmolr |= E1000_VMOLR_MPME;
5030 } else if (vf_data->num_vf_mc_hashes) {
5031 int j;
5032 vmolr |= E1000_VMOLR_ROMPE;
5033 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5034 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5035 }
5036 }
5037
5038 wr32(E1000_VMOLR(vf), vmolr);
5039
5040 /* there are flags left unprocessed, likely not supported */
5041 if (*msgbuf & E1000_VT_MSGINFO_MASK)
5042 return -EINVAL;
5043
5044 return 0;
5045
5046}
5047
4ae196df
AD		 5048static int igb_set_vf_multicasts(struct igb_adapter *adapter,
5049 u32 *msgbuf, u32 vf)
5050{
5051 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5052 u16 *hash_list = (u16 *)&msgbuf[1];
5053 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5054 int i;
5055
7d5753f0 5056 /* salt away the number of multicast addresses assigned
4ae196df
AD		 5057 * to this VF for later use to restore when the PF multi cast
5058 * list changes
5059 */
5060 vf_data->num_vf_mc_hashes = n;
5061
7d5753f0
AD		 5062 /* only up to 30 hash values supported */
5063 if (n > 30)
5064 n = 30;
5065
5066 /* store the hashes for later use */
4ae196df 5067 for (i = 0; i < n; i++)
a419aef8 5068 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 5069
5070 /* Flush and reset the mta with the new values */
ff41f8dc 5071 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 5072
5073 return 0;
5074}
5075
5076static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
5077{
5078 struct e1000_hw *hw = &adapter->hw;
5079 struct vf_data_storage *vf_data;
5080 int i, j;
5081
5082 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 5083 u32 vmolr = rd32(E1000_VMOLR(i));
5084 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5085
4ae196df 5086 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 5087
5088 if ((vf_data->num_vf_mc_hashes > 30) ||
5089 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
5090 vmolr |= E1000_VMOLR_MPME;
5091 } else if (vf_data->num_vf_mc_hashes) {
5092 vmolr |= E1000_VMOLR_ROMPE;
5093 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5094 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5095 }
5096 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 5097 }
5098}
5099
5100static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
5101{
5102 struct e1000_hw *hw = &adapter->hw;
5103 u32 pool_mask, reg, vid;
5104 int i;
5105
5106 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5107
5108 /* Find the vlan filter for this id */
5109 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5110 reg = rd32(E1000_VLVF(i));
5111
5112 /* remove the vf from the pool */
5113 reg &= ~pool_mask;
5114
5115 /* if pool is empty then remove entry from vfta */
5116 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
5117 (reg & E1000_VLVF_VLANID_ENABLE)) {
5118 reg = 0;
5119 vid = reg & E1000_VLVF_VLANID_MASK;
5120 igb_vfta_set(hw, vid, false);
5121 }
5122
5123 wr32(E1000_VLVF(i), reg);
5124 }
ae641bdc
AD		 5125
5126 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 5127}
5128
5129static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
5130{
5131 struct e1000_hw *hw = &adapter->hw;
5132 u32 reg, i;
5133
51466239
AD		 5134 /* The vlvf table only exists on 82576 hardware and newer */
5135 if (hw->mac.type < e1000_82576)
5136 return -1;
5137
5138 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 5139 if (!adapter->vfs_allocated_count)
5140 return -1;
5141
5142 /* Find the vlan filter for this id */
5143 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5144 reg = rd32(E1000_VLVF(i));
5145 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
5146 vid == (reg & E1000_VLVF_VLANID_MASK))
5147 break;
5148 }
5149
5150 if (add) {
5151 if (i == E1000_VLVF_ARRAY_SIZE) {
5152 /* Did not find a matching VLAN ID entry that was
5153 * enabled. Search for a free filter entry, i.e.
5154 * one without the enable bit set
5155 */
5156 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5157 reg = rd32(E1000_VLVF(i));
5158 if (!(reg & E1000_VLVF_VLANID_ENABLE))
5159 break;
5160 }
5161 }
5162 if (i < E1000_VLVF_ARRAY_SIZE) {
5163 /* Found an enabled/available entry */
5164 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5165
5166 /* if !enabled we need to set this up in vfta */
5167 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 5168 /* add VID to filter table */
5169 igb_vfta_set(hw, vid, true);
4ae196df
AD		 5170 reg |= E1000_VLVF_VLANID_ENABLE;
5171 }
cad6d05f
AD		 5172 reg &= ~E1000_VLVF_VLANID_MASK;
5173 reg |= vid;
4ae196df 5174 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5175
5176 /* do not modify RLPML for PF devices */
5177 if (vf >= adapter->vfs_allocated_count)
5178 return 0;
5179
5180 if (!adapter->vf_data[vf].vlans_enabled) {
5181 u32 size;
5182 reg = rd32(E1000_VMOLR(vf));
5183 size = reg & E1000_VMOLR_RLPML_MASK;
5184 size += 4;
5185 reg &= ~E1000_VMOLR_RLPML_MASK;
5186 reg |= size;
5187 wr32(E1000_VMOLR(vf), reg);
5188 }
ae641bdc 5189
51466239 5190 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 5191 }
5192 } else {
5193 if (i < E1000_VLVF_ARRAY_SIZE) {
5194 /* remove vf from the pool */
5195 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
5196 /* if pool is empty then remove entry from vfta */
5197 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
5198 reg = 0;
5199 igb_vfta_set(hw, vid, false);
5200 }
5201 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5202
5203 /* do not modify RLPML for PF devices */
5204 if (vf >= adapter->vfs_allocated_count)
5205 return 0;
5206
5207 adapter->vf_data[vf].vlans_enabled--;
5208 if (!adapter->vf_data[vf].vlans_enabled) {
5209 u32 size;
5210 reg = rd32(E1000_VMOLR(vf));
5211 size = reg & E1000_VMOLR_RLPML_MASK;
5212 size -= 4;
5213 reg &= ~E1000_VMOLR_RLPML_MASK;
5214 reg |= size;
5215 wr32(E1000_VMOLR(vf), reg);
5216 }
4ae196df
AD		 5217 }
5218 }
8151d294
WM		 5219 return 0;
5220}
5221
5222static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
5223{
5224 struct e1000_hw *hw = &adapter->hw;
5225
5226 if (vid)
5227 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
5228 else
5229 wr32(E1000_VMVIR(vf), 0);
5230}
5231
5232static int igb_ndo_set_vf_vlan(struct net_device *netdev,
5233 int vf, u16 vlan, u8 qos)
5234{
5235 int err = 0;
5236 struct igb_adapter *adapter = netdev_priv(netdev);
5237
5238 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
5239 return -EINVAL;
5240 if (vlan || qos) {
5241 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
5242 if (err)
5243 goto out;
5244 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
5245 igb_set_vmolr(adapter, vf, !vlan);
5246 adapter->vf_data[vf].pf_vlan = vlan;
5247 adapter->vf_data[vf].pf_qos = qos;
5248 dev_info(&adapter->pdev->dev,
5249 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5250 if (test_bit(__IGB_DOWN, &adapter->state)) {
5251 dev_warn(&adapter->pdev->dev,
5252 "The VF VLAN has been set,"
5253 " but the PF device is not up.\n");
5254 dev_warn(&adapter->pdev->dev,
5255 "Bring the PF device up before"
5256 " attempting to use the VF device.\n");
5257 }
5258 } else {
5259 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5260 false, vf);
5261 igb_set_vmvir(adapter, vlan, vf);
5262 igb_set_vmolr(adapter, vf, true);
5263 adapter->vf_data[vf].pf_vlan = 0;
5264 adapter->vf_data[vf].pf_qos = 0;
5265 }
5266out:
5267 return err;
4ae196df
AD		 5268}
5269
5270static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5271{
5272 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5273 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5274
5275 return igb_vlvf_set(adapter, vid, add, vf);
5276}
5277
f2ca0dbe 5278static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 5279{
8fa7e0f7
GR		 5280 /* clear flags - except flag that indicates PF has set the MAC */
5281 adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
f2ca0dbe 5282 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 5283
5284 /* reset offloads to defaults */
8151d294 5285 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 5286
5287 /* reset vlans for device */
5288 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 5289 if (adapter->vf_data[vf].pf_vlan)
5290 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5291 adapter->vf_data[vf].pf_vlan,
5292 adapter->vf_data[vf].pf_qos);
5293 else
5294 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 5295
5296 /* reset multicast table array for vf */
5297 adapter->vf_data[vf].num_vf_mc_hashes = 0;
5298
5299 /* Flush and reset the mta with the new values */
ff41f8dc 5300 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 5301}
5302
f2ca0dbe
AD		 5303static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5304{
5305 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5306
5307 /* generate a new mac address as we were hotplug removed/added */
8151d294
WM		 5308 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5309 random_ether_addr(vf_mac);
f2ca0dbe
AD		 5310
5311 /* process remaining reset events */
5312 igb_vf_reset(adapter, vf);
5313}
5314
5315static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 5316{
5317 struct e1000_hw *hw = &adapter->hw;
5318 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 5319 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 5320 u32 reg, msgbuf[3];
5321 u8 *addr = (u8 *)(&msgbuf[1]);
5322
5323 /* process all the same items cleared in a function level reset */
f2ca0dbe 5324 igb_vf_reset(adapter, vf);
4ae196df
AD		 5325
5326 /* set vf mac address */
26ad9178 5327 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 5328
5329 /* enable transmit and receive for vf */
5330 reg = rd32(E1000_VFTE);
5331 wr32(E1000_VFTE, reg | (1 << vf));
5332 reg = rd32(E1000_VFRE);
5333 wr32(E1000_VFRE, reg | (1 << vf));
5334
8fa7e0f7 5335 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
4ae196df
AD		 5336
5337 /* reply to reset with ack and vf mac address */
5338 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5339 memcpy(addr, vf_mac, 6);
5340 igb_write_mbx(hw, msgbuf, 3, vf);
5341}
5342
5343static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5344{
de42edde
GR		 5345 /*
5346 * The VF MAC Address is stored in a packed array of bytes
5347 * starting at the second 32 bit word of the msg array
5348 */
f2ca0dbe
AD		 5349 unsigned char *addr = (char *)&msg[1];
5350 int err = -1;
4ae196df 5351
f2ca0dbe
AD		 5352 if (is_valid_ether_addr(addr))
5353 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 5354
f2ca0dbe 5355 return err;
4ae196df
AD		 5356}
5357
5358static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5359{
5360 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5361 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5362 u32 msg = E1000_VT_MSGTYPE_NACK;
5363
5364 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 5365 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5366 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 5367 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 5368 vf_data->last_nack = jiffies;
4ae196df
AD		 5369 }
5370}
5371
f2ca0dbe 5372static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 5373{
f2ca0dbe
AD		 5374 struct pci_dev *pdev = adapter->pdev;
5375 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 5376 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5377 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5378 s32 retval;
5379
f2ca0dbe 5380 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 5381
fef45f4c
AD		 5382 if (retval) {
5383 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 5384 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 5385 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5386 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5387 return;
5388 goto out;
5389 }
4ae196df
AD		 5390
5391 /* this is a message we already processed, do nothing */
5392 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 5393 return;
4ae196df
AD		 5394
5395 /*
5396 * until the vf completes a reset it should not be
5397 * allowed to start any configuration.
5398 */
5399
5400 if (msgbuf[0] == E1000_VF_RESET) {
5401 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 5402 return;
4ae196df
AD		 5403 }
5404
f2ca0dbe 5405 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 5406 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5407 return;
5408 retval = -1;
5409 goto out;
4ae196df
AD		 5410 }
5411
5412 switch ((msgbuf[0] & 0xFFFF)) {
5413 case E1000_VF_SET_MAC_ADDR:
a6b5ea35
GR		 5414 retval = -EINVAL;
5415 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
5416 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5417 else
5418 dev_warn(&pdev->dev,
5419 "VF %d attempted to override administratively "
5420 "set MAC address\nReload the VF driver to "
5421 "resume operations\n", vf);
4ae196df 5422 break;
7d5753f0
AD		 5423 case E1000_VF_SET_PROMISC:
5424 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5425 break;
4ae196df
AD		 5426 case E1000_VF_SET_MULTICAST:
5427 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5428 break;
5429 case E1000_VF_SET_LPE:
5430 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5431 break;
5432 case E1000_VF_SET_VLAN:
a6b5ea35
GR		 5433 retval = -1;
5434 if (vf_data->pf_vlan)
5435 dev_warn(&pdev->dev,
5436 "VF %d attempted to override administratively "
5437 "set VLAN tag\nReload the VF driver to "
5438 "resume operations\n", vf);
8151d294
WM		 5439 else
5440 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 5441 break;
5442 default:
090b1795 5443 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 5444 retval = -1;
5445 break;
5446 }
5447
fef45f4c
AD		 5448 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5449out:
4ae196df
AD		 5450 /* notify the VF of the results of what it sent us */
5451 if (retval)
5452 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5453 else
5454 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5455
4ae196df 5456 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 5457}
4ae196df 5458
f2ca0dbe
AD		 5459static void igb_msg_task(struct igb_adapter *adapter)
5460{
5461 struct e1000_hw *hw = &adapter->hw;
5462 u32 vf;
5463
5464 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5465 /* process any reset requests */
5466 if (!igb_check_for_rst(hw, vf))
5467 igb_vf_reset_event(adapter, vf);
5468
5469 /* process any messages pending */
5470 if (!igb_check_for_msg(hw, vf))
5471 igb_rcv_msg_from_vf(adapter, vf);
5472
5473 /* process any acks */
5474 if (!igb_check_for_ack(hw, vf))
5475 igb_rcv_ack_from_vf(adapter, vf);
5476 }
4ae196df
AD		 5477}
5478
68d480c4
AD		 5479/**
5480 * igb_set_uta - Set unicast filter table address
5481 * @adapter: board private structure
5482 *
5483 * The unicast table address is a register array of 32-bit registers.
5484 * The table is meant to be used in a way similar to how the MTA is used
5485 * however due to certain limitations in the hardware it is necessary to
25985edc
LDM		 5486 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
5487 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
68d480c4
AD		 5488 **/
5489static void igb_set_uta(struct igb_adapter *adapter)
5490{
5491 struct e1000_hw *hw = &adapter->hw;
5492 int i;
5493
5494 /* The UTA table only exists on 82576 hardware and newer */
5495 if (hw->mac.type < e1000_82576)
5496 return;
5497
5498 /* we only need to do this if VMDq is enabled */
5499 if (!adapter->vfs_allocated_count)
5500 return;
5501
5502 for (i = 0; i < hw->mac.uta_reg_count; i++)
5503 array_wr32(E1000_UTA, i, ~0);
5504}
5505
9d5c8243
AK		 5506/**
5507 * igb_intr_msi - Interrupt Handler
5508 * @irq: interrupt number
5509 * @data: pointer to a network interface device structure
5510 **/
5511static irqreturn_t igb_intr_msi(int irq, void *data)
5512{
047e0030
AD		 5513 struct igb_adapter *adapter = data;
5514 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5515 struct e1000_hw *hw = &adapter->hw;
5516 /* read ICR disables interrupts using IAM */
5517 u32 icr = rd32(E1000_ICR);
5518
047e0030 5519 igb_write_itr(q_vector);
9d5c8243 5520
7f081d40
AD		 5521 if (icr & E1000_ICR_DRSTA)
5522 schedule_work(&adapter->reset_task);
5523
047e0030 5524 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5525 /* HW is reporting DMA is out of sync */
5526 adapter->stats.doosync++;
5527 }
5528
9d5c8243
AK		 5529 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5530 hw->mac.get_link_status = 1;
5531 if (!test_bit(__IGB_DOWN, &adapter->state))
5532 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5533 }
5534
047e0030 5535 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5536
5537 return IRQ_HANDLED;
5538}
5539
5540/**
4a3c6433 5541 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 5542 * @irq: interrupt number
5543 * @data: pointer to a network interface device structure
5544 **/
5545static irqreturn_t igb_intr(int irq, void *data)
5546{
047e0030
AD		 5547 struct igb_adapter *adapter = data;
5548 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5549 struct e1000_hw *hw = &adapter->hw;
5550 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5551 * need for the IMC write */
5552 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 5553
5554 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5555 * not set, then the adapter didn't send an interrupt */
5556 if (!(icr & E1000_ICR_INT_ASSERTED))
5557 return IRQ_NONE;
5558
0ba82994
AD		 5559 igb_write_itr(q_vector);
5560
7f081d40
AD		 5561 if (icr & E1000_ICR_DRSTA)
5562 schedule_work(&adapter->reset_task);
5563
047e0030 5564 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5565 /* HW is reporting DMA is out of sync */
5566 adapter->stats.doosync++;
5567 }
5568
9d5c8243
AK		 5569 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5570 hw->mac.get_link_status = 1;
5571 /* guard against interrupt when we're going down */
5572 if (!test_bit(__IGB_DOWN, &adapter->state))
5573 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5574 }
5575
047e0030 5576 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5577
5578 return IRQ_HANDLED;
5579}
5580
0ba82994 5581void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 5582{
047e0030 5583 struct igb_adapter *adapter = q_vector->adapter;
46544258 5584 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5585
0ba82994
AD		 5586 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
5587 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
5588 if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
5589 igb_set_itr(q_vector);
46544258 5590 else
047e0030 5591 igb_update_ring_itr(q_vector);
9d5c8243
AK		 5592 }
5593
46544258
AD		 5594 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5595 if (adapter->msix_entries)
047e0030 5596 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 5597 else
5598 igb_irq_enable(adapter);
5599 }
9d5c8243
AK		 5600}
5601
46544258
AD		 5602/**
5603 * igb_poll - NAPI Rx polling callback
5604 * @napi: napi polling structure
5605 * @budget: count of how many packets we should handle
5606 **/
5607static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 5608{
047e0030
AD		 5609 struct igb_q_vector *q_vector = container_of(napi,
5610 struct igb_q_vector,
5611 napi);
16eb8815 5612 bool clean_complete = true;
9d5c8243 5613
421e02f0 5614#ifdef CONFIG_IGB_DCA
047e0030
AD		 5615 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5616 igb_update_dca(q_vector);
fe4506b6 5617#endif
0ba82994 5618 if (q_vector->tx.ring)
13fde97a 5619 clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 5620
0ba82994 5621 if (q_vector->rx.ring)
cd392f5c 5622 clean_complete &= igb_clean_rx_irq(q_vector, budget);
047e0030 5623
16eb8815
AD		 5624 /* If all work not completed, return budget and keep polling */
5625 if (!clean_complete)
5626 return budget;
46544258 5627
9d5c8243 5628 /* If not enough Rx work done, exit the polling mode */
16eb8815
AD		 5629 napi_complete(napi);
5630 igb_ring_irq_enable(q_vector);
9d5c8243 5631
16eb8815 5632 return 0;
9d5c8243 5633}
6d8126f9 5634
33af6bcc 5635/**
c5b9bd5e 5636 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 5637 * @adapter: board private structure
c5b9bd5e
AD		 5638 * @shhwtstamps: timestamp structure to update
5639 * @regval: unsigned 64bit system time value.
5640 *
5641 * We need to convert the system time value stored in the RX/TXSTMP registers
5642 * into a hwtstamp which can be used by the upper level timestamping functions
5643 */
5644static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5645 struct skb_shared_hwtstamps *shhwtstamps,
5646 u64 regval)
5647{
5648 u64 ns;
5649
55cac248
AD		 5650 /*
5651 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5652 * 24 to match clock shift we setup earlier.
5653 */
06218a8d 5654 if (adapter->hw.mac.type >= e1000_82580)
55cac248
AD		 5655 regval <<= IGB_82580_TSYNC_SHIFT;
5656
c5b9bd5e
AD		 5657 ns = timecounter_cyc2time(&adapter->clock, regval);
5658 timecompare_update(&adapter->compare, ns);
5659 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5660 shhwtstamps->hwtstamp = ns_to_ktime(ns);
5661 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5662}
5663
5664/**
5665 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5666 * @q_vector: pointer to q_vector containing needed info
06034649 5667 * @buffer: pointer to igb_tx_buffer structure
33af6bcc
PO		 5668 *
5669 * If we were asked to do hardware stamping and such a time stamp is
5670 * available, then it must have been for this skb here because we only
5671 * allow only one such packet into the queue.
5672 */
06034649
AD		 5673static void igb_tx_hwtstamp(struct igb_q_vector *q_vector,
5674 struct igb_tx_buffer *buffer_info)
33af6bcc 5675{
c5b9bd5e 5676 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc 5677 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 5678 struct skb_shared_hwtstamps shhwtstamps;
5679 u64 regval;
33af6bcc 5680
c5b9bd5e 5681 /* if skb does not support hw timestamp or TX stamp not valid exit */
2bbfebe2 5682 if (likely(!(buffer_info->tx_flags & IGB_TX_FLAGS_TSTAMP)) ||
c5b9bd5e
AD		 5683 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5684 return;
5685
5686 regval = rd32(E1000_TXSTMPL);
5687 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5688
5689 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
2873957d 5690 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
33af6bcc
PO		 5691}
5692
9d5c8243
AK		 5693/**
5694 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 5695 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 5696 * returns true if ring is completely cleaned
5697 **/
047e0030 5698static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 5699{
047e0030 5700 struct igb_adapter *adapter = q_vector->adapter;
0ba82994 5701 struct igb_ring *tx_ring = q_vector->tx.ring;
06034649 5702 struct igb_tx_buffer *tx_buffer;
8542db05 5703 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 5704 unsigned int total_bytes = 0, total_packets = 0;
0ba82994 5705 unsigned int budget = q_vector->tx.work_limit;
8542db05 5706 unsigned int i = tx_ring->next_to_clean;
9d5c8243 5707
13fde97a
AD		 5708 if (test_bit(__IGB_DOWN, &adapter->state))
5709 return true;
0e014cb1 5710
06034649 5711 tx_buffer = &tx_ring->tx_buffer_info[i];
13fde97a 5712 tx_desc = IGB_TX_DESC(tx_ring, i);
8542db05 5713 i -= tx_ring->count;
9d5c8243 5714
13fde97a 5715 for (; budget; budget--) {
8542db05 5716 eop_desc = tx_buffer->next_to_watch;
13fde97a 5717
8542db05
AD		 5718 /* prevent any other reads prior to eop_desc */
5719 rmb();
5720
5721 /* if next_to_watch is not set then there is no work pending */
5722 if (!eop_desc)
5723 break;
13fde97a
AD		 5724
5725 /* if DD is not set pending work has not been completed */
5726 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
5727 break;
5728
8542db05
AD		 5729 /* clear next_to_watch to prevent false hangs */
5730 tx_buffer->next_to_watch = NULL;
9d5c8243 5731
ebe42d16
AD		 5732 /* update the statistics for this packet */
5733 total_bytes += tx_buffer->bytecount;
5734 total_packets += tx_buffer->gso_segs;
13fde97a 5735
ebe42d16
AD		 5736 /* retrieve hardware timestamp */
5737 igb_tx_hwtstamp(q_vector, tx_buffer);
5738
5739 /* free the skb */
5740 dev_kfree_skb_any(tx_buffer->skb);
5741 tx_buffer->skb = NULL;
13fde97a 5742
ebe42d16
AD		 5743 /* unmap skb header data */
5744 dma_unmap_single(tx_ring->dev,
5745 tx_buffer->dma,
5746 tx_buffer->length,
5747 DMA_TO_DEVICE);
5748
5749 /* clear last DMA location and unmap remaining buffers */
5750 while (tx_desc != eop_desc) {
5751 tx_buffer->dma = 0;
9d5c8243 5752
13fde97a
AD		 5753 tx_buffer++;
5754 tx_desc++;
9d5c8243 5755 i++;
8542db05
AD		 5756 if (unlikely(!i)) {
5757 i -= tx_ring->count;
06034649 5758 tx_buffer = tx_ring->tx_buffer_info;
13fde97a
AD		 5759 tx_desc = IGB_TX_DESC(tx_ring, 0);
5760 }
ebe42d16
AD		 5761
5762 /* unmap any remaining paged data */
5763 if (tx_buffer->dma) {
5764 dma_unmap_page(tx_ring->dev,
5765 tx_buffer->dma,
5766 tx_buffer->length,
5767 DMA_TO_DEVICE);
5768 }
5769 }
5770
5771 /* clear last DMA location */
5772 tx_buffer->dma = 0;
5773
5774 /* move us one more past the eop_desc for start of next pkt */
5775 tx_buffer++;
5776 tx_desc++;
5777 i++;
5778 if (unlikely(!i)) {
5779 i -= tx_ring->count;
5780 tx_buffer = tx_ring->tx_buffer_info;
5781 tx_desc = IGB_TX_DESC(tx_ring, 0);
5782 }
0e014cb1
AD		 5783 }
5784
8542db05 5785 i += tx_ring->count;
9d5c8243 5786 tx_ring->next_to_clean = i;
13fde97a
AD		 5787 u64_stats_update_begin(&tx_ring->tx_syncp);
5788 tx_ring->tx_stats.bytes += total_bytes;
5789 tx_ring->tx_stats.packets += total_packets;
5790 u64_stats_update_end(&tx_ring->tx_syncp);
0ba82994
AD		 5791 q_vector->tx.total_bytes += total_bytes;
5792 q_vector->tx.total_packets += total_packets;
9d5c8243 5793
6d095fa8 5794 if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
13fde97a 5795 struct e1000_hw *hw = &adapter->hw;
12dcd86b 5796
8542db05 5797 eop_desc = tx_buffer->next_to_watch;
9d5c8243 5798
9d5c8243
AK		 5799 /* Detect a transmit hang in hardware, this serializes the
5800 * check with the clearing of time_stamp and movement of i */
6d095fa8 5801 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
8542db05
AD		 5802 if (eop_desc &&
5803 time_after(jiffies, tx_buffer->time_stamp +
8e95a202
JP		 5804 (adapter->tx_timeout_factor * HZ)) &&
5805 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5806
9d5c8243 5807 /* detected Tx unit hang */
59d71989 5808 dev_err(tx_ring->dev,
9d5c8243 5809 "Detected Tx Unit Hang\n"
2d064c06 5810 " Tx Queue <%d>\n"
9d5c8243
AK		 5811 " TDH <%x>\n"
5812 " TDT <%x>\n"
5813 " next_to_use <%x>\n"
5814 " next_to_clean <%x>\n"
9d5c8243
AK		 5815 "buffer_info[next_to_clean]\n"
5816 " time_stamp <%lx>\n"
8542db05 5817 " next_to_watch <%p>\n"
9d5c8243
AK		 5818 " jiffies <%lx>\n"
5819 " desc.status <%x>\n",
2d064c06 5820 tx_ring->queue_index,
238ac817 5821 rd32(E1000_TDH(tx_ring->reg_idx)),
fce99e34 5822 readl(tx_ring->tail),
9d5c8243
AK		 5823 tx_ring->next_to_use,
5824 tx_ring->next_to_clean,
8542db05
AD		 5825 tx_buffer->time_stamp,
5826 eop_desc,
9d5c8243 5827 jiffies,
0e014cb1 5828 eop_desc->wb.status);
13fde97a
AD		 5829 netif_stop_subqueue(tx_ring->netdev,
5830 tx_ring->queue_index);
5831
5832 /* we are about to reset, no point in enabling stuff */
5833 return true;
9d5c8243
AK		 5834 }
5835 }
13fde97a
AD		 5836
5837 if (unlikely(total_packets &&
5838 netif_carrier_ok(tx_ring->netdev) &&
5839 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5840 /* Make sure that anybody stopping the queue after this
5841 * sees the new next_to_clean.
5842 */
5843 smp_mb();
5844 if (__netif_subqueue_stopped(tx_ring->netdev,
5845 tx_ring->queue_index) &&
5846 !(test_bit(__IGB_DOWN, &adapter->state))) {
5847 netif_wake_subqueue(tx_ring->netdev,
5848 tx_ring->queue_index);
5849
5850 u64_stats_update_begin(&tx_ring->tx_syncp);
5851 tx_ring->tx_stats.restart_queue++;
5852 u64_stats_update_end(&tx_ring->tx_syncp);
5853 }
5854 }
5855
5856 return !!budget;
9d5c8243
AK		 5857}
5858
cd392f5c 5859static inline void igb_rx_checksum(struct igb_ring *ring,
3ceb90fd
AD		 5860 union e1000_adv_rx_desc *rx_desc,
5861 struct sk_buff *skb)
9d5c8243 5862{
bc8acf2c 5863 skb_checksum_none_assert(skb);
9d5c8243 5864
294e7d78 5865 /* Ignore Checksum bit is set */
3ceb90fd 5866 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
294e7d78
AD		 5867 return;
5868
5869 /* Rx checksum disabled via ethtool */
5870 if (!(ring->netdev->features & NETIF_F_RXCSUM))
9d5c8243 5871 return;
85ad76b2 5872
9d5c8243 5873 /* TCP/UDP checksum error bit is set */
3ceb90fd
AD		 5874 if (igb_test_staterr(rx_desc,
5875 E1000_RXDEXT_STATERR_TCPE |
5876 E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5877 /*
5878 * work around errata with sctp packets where the TCPE aka
5879 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5880 * packets, (aka let the stack check the crc32c)
5881 */
866cff06
AD		 5882 if (!((skb->len == 60) &&
5883 test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
12dcd86b 5884 u64_stats_update_begin(&ring->rx_syncp);
04a5fcaa 5885 ring->rx_stats.csum_err++;
12dcd86b
ED		 5886 u64_stats_update_end(&ring->rx_syncp);
5887 }
9d5c8243 5888 /* let the stack verify checksum errors */
9d5c8243
AK		 5889 return;
5890 }
5891 /* It must be a TCP or UDP packet with a valid checksum */
3ceb90fd
AD		 5892 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
5893 E1000_RXD_STAT_UDPCS))
9d5c8243
AK		 5894 skb->ip_summed = CHECKSUM_UNNECESSARY;
5895
3ceb90fd
AD		 5896 dev_dbg(ring->dev, "cksum success: bits %08X\n",
5897 le32_to_cpu(rx_desc->wb.upper.status_error));
9d5c8243
AK		 5898}
5899
077887c3
AD		 5900static inline void igb_rx_hash(struct igb_ring *ring,
5901 union e1000_adv_rx_desc *rx_desc,
5902 struct sk_buff *skb)
5903{
5904 if (ring->netdev->features & NETIF_F_RXHASH)
5905 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
5906}
5907
3ceb90fd
AD		 5908static void igb_rx_hwtstamp(struct igb_q_vector *q_vector,
5909 union e1000_adv_rx_desc *rx_desc,
5910 struct sk_buff *skb)
c5b9bd5e
AD		 5911{
5912 struct igb_adapter *adapter = q_vector->adapter;
5913 struct e1000_hw *hw = &adapter->hw;
5914 u64 regval;
5915
3ceb90fd
AD		 5916 if (!igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP |
5917 E1000_RXDADV_STAT_TS))
5918 return;
5919
c5b9bd5e
AD		 5920 /*
5921 * If this bit is set, then the RX registers contain the time stamp. No
5922 * other packet will be time stamped until we read these registers, so
5923 * read the registers to make them available again. Because only one
5924 * packet can be time stamped at a time, we know that the register
5925 * values must belong to this one here and therefore we don't need to
5926 * compare any of the additional attributes stored for it.
5927 *
2244d07b 5928 * If nothing went wrong, then it should have a shared tx_flags that we
c5b9bd5e
AD		 5929 * can turn into a skb_shared_hwtstamps.
5930 */
3ceb90fd 5931 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
757b77e2
NN		 5932 u32 *stamp = (u32 *)skb->data;
5933 regval = le32_to_cpu(*(stamp + 2));
5934 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5935 skb_pull(skb, IGB_TS_HDR_LEN);
5936 } else {
5937 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5938 return;
c5b9bd5e 5939
757b77e2
NN		 5940 regval = rd32(E1000_RXSTMPL);
5941 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5942 }
c5b9bd5e
AD		 5943
5944 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5945}
8be10e91
AD		 5946
5947static void igb_rx_vlan(struct igb_ring *ring,
5948 union e1000_adv_rx_desc *rx_desc,
5949 struct sk_buff *skb)
5950{
5951 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
5952 u16 vid;
5953 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
5954 test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags))
5955 vid = be16_to_cpu(rx_desc->wb.upper.vlan);
5956 else
5957 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
5958
5959 __vlan_hwaccel_put_tag(skb, vid);
5960 }
5961}
5962
44390ca6 5963static inline u16 igb_get_hlen(union e1000_adv_rx_desc *rx_desc)
2d94d8ab
AD		 5964{
5965 /* HW will not DMA in data larger than the given buffer, even if it
5966 * parses the (NFS, of course) header to be larger. In that case, it
5967 * fills the header buffer and spills the rest into the page.
5968 */
5969 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5970 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
44390ca6
AD		 5971 if (hlen > IGB_RX_HDR_LEN)
5972 hlen = IGB_RX_HDR_LEN;
2d94d8ab
AD		 5973 return hlen;
5974}
5975
cd392f5c 5976static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, int budget)
9d5c8243 5977{
0ba82994 5978 struct igb_ring *rx_ring = q_vector->rx.ring;
16eb8815
AD		 5979 union e1000_adv_rx_desc *rx_desc;
5980 const int current_node = numa_node_id();
9d5c8243 5981 unsigned int total_bytes = 0, total_packets = 0;
16eb8815
AD		 5982 u16 cleaned_count = igb_desc_unused(rx_ring);
5983 u16 i = rx_ring->next_to_clean;
9d5c8243 5984
60136906 5985 rx_desc = IGB_RX_DESC(rx_ring, i);
9d5c8243 5986
3ceb90fd 5987 while (igb_test_staterr(rx_desc, E1000_RXD_STAT_DD)) {
06034649 5988 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
16eb8815
AD		 5989 struct sk_buff *skb = buffer_info->skb;
5990 union e1000_adv_rx_desc *next_rxd;
9d5c8243 5991
69d3ca53 5992 buffer_info->skb = NULL;
16eb8815 5993 prefetch(skb->data);
69d3ca53
AD		 5994
5995 i++;
5996 if (i == rx_ring->count)
5997 i = 0;
42d0781a 5998
60136906 5999 next_rxd = IGB_RX_DESC(rx_ring, i);
69d3ca53 6000 prefetch(next_rxd);
9d5c8243 6001
16eb8815
AD		 6002 /*
6003 * This memory barrier is needed to keep us from reading
6004 * any other fields out of the rx_desc until we know the
6005 * RXD_STAT_DD bit is set
6006 */
6007 rmb();
9d5c8243 6008
16eb8815
AD		 6009 if (!skb_is_nonlinear(skb)) {
6010 __skb_put(skb, igb_get_hlen(rx_desc));
6011 dma_unmap_single(rx_ring->dev, buffer_info->dma,
44390ca6 6012 IGB_RX_HDR_LEN,
59d71989 6013 DMA_FROM_DEVICE);
91615f76 6014 buffer_info->dma = 0;
bf36c1a0
AD		 6015 }
6016
16eb8815
AD		 6017 if (rx_desc->wb.upper.length) {
6018 u16 length = le16_to_cpu(rx_desc->wb.upper.length);
bf36c1a0 6019
aa913403 6020 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
bf36c1a0
AD		 6021 buffer_info->page,
6022 buffer_info->page_offset,
6023 length);
6024
16eb8815
AD		 6025 skb->len += length;
6026 skb->data_len += length;
95b9c1df 6027 skb->truesize += PAGE_SIZE / 2;
16eb8815 6028
d1eff350
AD		 6029 if ((page_count(buffer_info->page) != 1) ||
6030 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 6031 buffer_info->page = NULL;
6032 else
6033 get_page(buffer_info->page);
9d5c8243 6034
16eb8815
AD		 6035 dma_unmap_page(rx_ring->dev, buffer_info->page_dma,
6036 PAGE_SIZE / 2, DMA_FROM_DEVICE);
6037 buffer_info->page_dma = 0;
9d5c8243 6038 }
9d5c8243 6039
3ceb90fd 6040 if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)) {
06034649
AD		 6041 struct igb_rx_buffer *next_buffer;
6042 next_buffer = &rx_ring->rx_buffer_info[i];
b2d56536
AD		 6043 buffer_info->skb = next_buffer->skb;
6044 buffer_info->dma = next_buffer->dma;
6045 next_buffer->skb = skb;
6046 next_buffer->dma = 0;
bf36c1a0
AD		 6047 goto next_desc;
6048 }
44390ca6 6049
3ceb90fd
AD		 6050 if (igb_test_staterr(rx_desc,
6051 E1000_RXDEXT_ERR_FRAME_ERR_MASK)) {
16eb8815 6052 dev_kfree_skb_any(skb);
9d5c8243
AK		 6053 goto next_desc;
6054 }
9d5c8243 6055
3ceb90fd 6056 igb_rx_hwtstamp(q_vector, rx_desc, skb);
077887c3 6057 igb_rx_hash(rx_ring, rx_desc, skb);
3ceb90fd 6058 igb_rx_checksum(rx_ring, rx_desc, skb);
8be10e91 6059 igb_rx_vlan(rx_ring, rx_desc, skb);
3ceb90fd
AD		 6060
6061 total_bytes += skb->len;
6062 total_packets++;
6063
6064 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
6065
b2cb09b1 6066 napi_gro_receive(&q_vector->napi, skb);
9d5c8243 6067
16eb8815 6068 budget--;
9d5c8243 6069next_desc:
16eb8815
AD		 6070 if (!budget)
6071 break;
6072
6073 cleaned_count++;
9d5c8243
AK		 6074 /* return some buffers to hardware, one at a time is too slow */
6075 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
cd392f5c 6076 igb_alloc_rx_buffers(rx_ring, cleaned_count);
9d5c8243
AK		 6077 cleaned_count = 0;
6078 }
6079
6080 /* use prefetched values */
6081 rx_desc = next_rxd;
9d5c8243 6082 }
bf36c1a0 6083
9d5c8243 6084 rx_ring->next_to_clean = i;
12dcd86b 6085 u64_stats_update_begin(&rx_ring->rx_syncp);
9d5c8243
AK		 6086 rx_ring->rx_stats.packets += total_packets;
6087 rx_ring->rx_stats.bytes += total_bytes;
12dcd86b 6088 u64_stats_update_end(&rx_ring->rx_syncp);
0ba82994
AD		 6089 q_vector->rx.total_packets += total_packets;
6090 q_vector->rx.total_bytes += total_bytes;
c023cd88
AD		 6091
6092 if (cleaned_count)
cd392f5c 6093 igb_alloc_rx_buffers(rx_ring, cleaned_count);
c023cd88 6094
16eb8815 6095 return !!budget;
9d5c8243
AK		 6096}
6097
c023cd88 6098static bool igb_alloc_mapped_skb(struct igb_ring *rx_ring,
06034649 6099 struct igb_rx_buffer *bi)
c023cd88
AD		 6100{
6101 struct sk_buff *skb = bi->skb;
6102 dma_addr_t dma = bi->dma;
6103
6104 if (dma)
6105 return true;
6106
6107 if (likely(!skb)) {
6108 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
6109 IGB_RX_HDR_LEN);
6110 bi->skb = skb;
6111 if (!skb) {
6112 rx_ring->rx_stats.alloc_failed++;
6113 return false;
6114 }
6115
6116 /* initialize skb for ring */
6117 skb_record_rx_queue(skb, rx_ring->queue_index);
6118 }
6119
6120 dma = dma_map_single(rx_ring->dev, skb->data,
6121 IGB_RX_HDR_LEN, DMA_FROM_DEVICE);
6122
6123 if (dma_mapping_error(rx_ring->dev, dma)) {
6124 rx_ring->rx_stats.alloc_failed++;
6125 return false;
6126 }
6127
6128 bi->dma = dma;
6129 return true;
6130}
6131
6132static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
06034649 6133 struct igb_rx_buffer *bi)
c023cd88
AD		 6134{
6135 struct page *page = bi->page;
6136 dma_addr_t page_dma = bi->page_dma;
6137 unsigned int page_offset = bi->page_offset ^ (PAGE_SIZE / 2);
6138
6139 if (page_dma)
6140 return true;
6141
6142 if (!page) {
6143 page = netdev_alloc_page(rx_ring->netdev);
6144 bi->page = page;
6145 if (unlikely(!page)) {
6146 rx_ring->rx_stats.alloc_failed++;
6147 return false;
6148 }
6149 }
6150
6151 page_dma = dma_map_page(rx_ring->dev, page,
6152 page_offset, PAGE_SIZE / 2,
6153 DMA_FROM_DEVICE);
6154
6155 if (dma_mapping_error(rx_ring->dev, page_dma)) {
6156 rx_ring->rx_stats.alloc_failed++;
6157 return false;
6158 }
6159
6160 bi->page_dma = page_dma;
6161 bi->page_offset = page_offset;
6162 return true;
6163}
6164
9d5c8243 6165/**
cd392f5c 6166 * igb_alloc_rx_buffers - Replace used receive buffers; packet split
9d5c8243
AK		 6167 * @adapter: address of board private structure
6168 **/
cd392f5c 6169void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
9d5c8243 6170{
9d5c8243 6171 union e1000_adv_rx_desc *rx_desc;
06034649 6172 struct igb_rx_buffer *bi;
c023cd88 6173 u16 i = rx_ring->next_to_use;
9d5c8243 6174
60136906 6175 rx_desc = IGB_RX_DESC(rx_ring, i);
06034649 6176 bi = &rx_ring->rx_buffer_info[i];
c023cd88 6177 i -= rx_ring->count;
9d5c8243
AK		 6178
6179 while (cleaned_count--) {
c023cd88
AD		 6180 if (!igb_alloc_mapped_skb(rx_ring, bi))
6181 break;
9d5c8243 6182
c023cd88
AD		 6183 /* Refresh the desc even if buffer_addrs didn't change
6184 * because each write-back erases this info. */
6185 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
9d5c8243 6186
c023cd88
AD		 6187 if (!igb_alloc_mapped_page(rx_ring, bi))
6188 break;
6189
6190 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
9d5c8243 6191
c023cd88
AD		 6192 rx_desc++;
6193 bi++;
9d5c8243 6194 i++;
c023cd88 6195 if (unlikely(!i)) {
60136906 6196 rx_desc = IGB_RX_DESC(rx_ring, 0);
06034649 6197 bi = rx_ring->rx_buffer_info;
c023cd88
AD		 6198 i -= rx_ring->count;
6199 }
6200
6201 /* clear the hdr_addr for the next_to_use descriptor */
6202 rx_desc->read.hdr_addr = 0;
9d5c8243
AK		 6203 }
6204
c023cd88
AD		 6205 i += rx_ring->count;
6206
9d5c8243
AK		 6207 if (rx_ring->next_to_use != i) {
6208 rx_ring->next_to_use = i;
9d5c8243
AK		 6209
6210 /* Force memory writes to complete before letting h/w
6211 * know there are new descriptors to fetch. (Only
6212 * applicable for weak-ordered memory model archs,
6213 * such as IA-64). */
6214 wmb();
fce99e34 6215 writel(i, rx_ring->tail);
9d5c8243
AK		 6216 }
6217}
6218
6219/**
6220 * igb_mii_ioctl -
6221 * @netdev:
6222 * @ifreq:
6223 * @cmd:
6224 **/
6225static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6226{
6227 struct igb_adapter *adapter = netdev_priv(netdev);
6228 struct mii_ioctl_data *data = if_mii(ifr);
6229
6230 if (adapter->hw.phy.media_type != e1000_media_type_copper)
6231 return -EOPNOTSUPP;
6232
6233 switch (cmd) {
6234 case SIOCGMIIPHY:
6235 data->phy_id = adapter->hw.phy.addr;
6236 break;
6237 case SIOCGMIIREG:
f5f4cf08
AD		 6238 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
6239 &data->val_out))
9d5c8243
AK		 6240 return -EIO;
6241 break;
6242 case SIOCSMIIREG:
6243 default:
6244 return -EOPNOTSUPP;
6245 }
6246 return 0;
6247}
6248
c6cb090b
PO		 6249/**
6250 * igb_hwtstamp_ioctl - control hardware time stamping
6251 * @netdev:
6252 * @ifreq:
6253 * @cmd:
6254 *
33af6bcc
PO		 6255 * Outgoing time stamping can be enabled and disabled. Play nice and
6256 * disable it when requested, although it shouldn't case any overhead
6257 * when no packet needs it. At most one packet in the queue may be
6258 * marked for time stamping, otherwise it would be impossible to tell
6259 * for sure to which packet the hardware time stamp belongs.
6260 *
6261 * Incoming time stamping has to be configured via the hardware
6262 * filters. Not all combinations are supported, in particular event
6263 * type has to be specified. Matching the kind of event packet is
6264 * not supported, with the exception of "all V2 events regardless of
6265 * level 2 or 4".
6266 *
c6cb090b
PO		 6267 **/
6268static int igb_hwtstamp_ioctl(struct net_device *netdev,
6269 struct ifreq *ifr, int cmd)
6270{
33af6bcc
PO		 6271 struct igb_adapter *adapter = netdev_priv(netdev);
6272 struct e1000_hw *hw = &adapter->hw;
c6cb090b 6273 struct hwtstamp_config config;
c5b9bd5e
AD		 6274 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
6275 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 6276 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 6277 bool is_l4 = false;
6278 bool is_l2 = false;
33af6bcc 6279 u32 regval;
c6cb090b
PO		 6280
6281 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
6282 return -EFAULT;
6283
6284 /* reserved for future extensions */
6285 if (config.flags)
6286 return -EINVAL;
6287
33af6bcc
PO		 6288 switch (config.tx_type) {
6289 case HWTSTAMP_TX_OFF:
c5b9bd5e 6290 tsync_tx_ctl = 0;
33af6bcc 6291 case HWTSTAMP_TX_ON:
33af6bcc
PO		 6292 break;
6293 default:
6294 return -ERANGE;
6295 }
6296
6297 switch (config.rx_filter) {
6298 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 6299 tsync_rx_ctl = 0;
33af6bcc
PO		 6300 break;
6301 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
6302 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
6303 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
6304 case HWTSTAMP_FILTER_ALL:
6305 /*
6306 * register TSYNCRXCFG must be set, therefore it is not
6307 * possible to time stamp both Sync and Delay_Req messages
6308 * => fall back to time stamping all packets
6309 */
c5b9bd5e 6310 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 6311 config.rx_filter = HWTSTAMP_FILTER_ALL;
6312 break;
6313 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 6314 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 6315 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 6316 is_l4 = true;
33af6bcc
PO		 6317 break;
6318 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 6319 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 6320 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 6321 is_l4 = true;
33af6bcc
PO		 6322 break;
6323 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
6324 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 6325 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 6326 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 6327 is_l2 = true;
6328 is_l4 = true;
33af6bcc
PO		 6329 config.rx_filter = HWTSTAMP_FILTER_SOME;
6330 break;
6331 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
6332 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 6333 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 6334 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 6335 is_l2 = true;
6336 is_l4 = true;
33af6bcc
PO		 6337 config.rx_filter = HWTSTAMP_FILTER_SOME;
6338 break;
6339 case HWTSTAMP_FILTER_PTP_V2_EVENT:
6340 case HWTSTAMP_FILTER_PTP_V2_SYNC:
6341 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 6342 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 6343 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 6344 is_l2 = true;
11ba69e8 6345 is_l4 = true;
33af6bcc
PO		 6346 break;
6347 default:
6348 return -ERANGE;
6349 }
6350
c5b9bd5e
AD		 6351 if (hw->mac.type == e1000_82575) {
6352 if (tsync_rx_ctl | tsync_tx_ctl)
6353 return -EINVAL;
6354 return 0;
6355 }
6356
757b77e2
NN		 6357 /*
6358 * Per-packet timestamping only works if all packets are
6359 * timestamped, so enable timestamping in all packets as
6360 * long as one rx filter was configured.
6361 */
06218a8d 6362 if ((hw->mac.type >= e1000_82580) && tsync_rx_ctl) {
757b77e2
NN		 6363 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
6364 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
6365 }
6366
33af6bcc
PO		 6367 /* enable/disable TX */
6368 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 6369 regval &= ~E1000_TSYNCTXCTL_ENABLED;
6370 regval |= tsync_tx_ctl;
33af6bcc
PO		 6371 wr32(E1000_TSYNCTXCTL, regval);
6372
c5b9bd5e 6373 /* enable/disable RX */
33af6bcc 6374 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 6375 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
6376 regval |= tsync_rx_ctl;
33af6bcc 6377 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 6378
c5b9bd5e
AD		 6379 /* define which PTP packets are time stamped */
6380 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 6381
c5b9bd5e
AD		 6382 /* define ethertype filter for timestamped packets */
6383 if (is_l2)
6384 wr32(E1000_ETQF(3),
6385 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
6386 E1000_ETQF_1588 | /* enable timestamping */
6387 ETH_P_1588)); /* 1588 eth protocol type */
6388 else
6389 wr32(E1000_ETQF(3), 0);
6390
6391#define PTP_PORT 319
6392 /* L4 Queue Filter[3]: filter by destination port and protocol */
6393 if (is_l4) {
6394 u32 ftqf = (IPPROTO_UDP /* UDP */
6395 | E1000_FTQF_VF_BP /* VF not compared */
6396 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
6397 | E1000_FTQF_MASK); /* mask all inputs */
6398 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
6399
6400 wr32(E1000_IMIR(3), htons(PTP_PORT));
6401 wr32(E1000_IMIREXT(3),
6402 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
6403 if (hw->mac.type == e1000_82576) {
6404 /* enable source port check */
6405 wr32(E1000_SPQF(3), htons(PTP_PORT));
6406 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
6407 }
6408 wr32(E1000_FTQF(3), ftqf);
6409 } else {
6410 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
6411 }
33af6bcc
PO		 6412 wrfl();
6413
6414 adapter->hwtstamp_config = config;
6415
6416 /* clear TX/RX time stamp registers, just to be sure */
6417 regval = rd32(E1000_TXSTMPH);
6418 regval = rd32(E1000_RXSTMPH);
c6cb090b 6419
33af6bcc
PO		 6420 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
6421 -EFAULT : 0;
c6cb090b
PO		 6422}
6423
9d5c8243
AK		 6424/**
6425 * igb_ioctl -
6426 * @netdev:
6427 * @ifreq:
6428 * @cmd:
6429 **/
6430static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6431{
6432 switch (cmd) {
6433 case SIOCGMIIPHY:
6434 case SIOCGMIIREG:
6435 case SIOCSMIIREG:
6436 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 6437 case SIOCSHWTSTAMP:
6438 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 6439 default:
6440 return -EOPNOTSUPP;
6441 }
6442}
6443
009bc06e
AD		 6444s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6445{
6446 struct igb_adapter *adapter = hw->back;
6447 u16 cap_offset;
6448
bdaae04c 6449 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6450 if (!cap_offset)
6451 return -E1000_ERR_CONFIG;
6452
6453 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6454
6455 return 0;
6456}
6457
6458s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6459{
6460 struct igb_adapter *adapter = hw->back;
6461 u16 cap_offset;
6462
bdaae04c 6463 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6464 if (!cap_offset)
6465 return -E1000_ERR_CONFIG;
6466
6467 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6468
6469 return 0;
6470}
6471
c8f44aff 6472static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features)
9d5c8243
AK		 6473{
6474 struct igb_adapter *adapter = netdev_priv(netdev);
6475 struct e1000_hw *hw = &adapter->hw;
6476 u32 ctrl, rctl;
5faf030c 6477 bool enable = !!(features & NETIF_F_HW_VLAN_RX);
9d5c8243 6478
5faf030c 6479 if (enable) {
9d5c8243
AK		 6480 /* enable VLAN tag insert/strip */
6481 ctrl = rd32(E1000_CTRL);
6482 ctrl |= E1000_CTRL_VME;
6483 wr32(E1000_CTRL, ctrl);
6484
51466239 6485 /* Disable CFI check */
9d5c8243 6486 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 6487 rctl &= ~E1000_RCTL_CFIEN;
6488 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 6489 } else {
6490 /* disable VLAN tag insert/strip */
6491 ctrl = rd32(E1000_CTRL);
6492 ctrl &= ~E1000_CTRL_VME;
6493 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 6494 }
6495
e1739522 6496 igb_rlpml_set(adapter);
9d5c8243
AK		 6497}
6498
6499static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6500{
6501 struct igb_adapter *adapter = netdev_priv(netdev);
6502 struct e1000_hw *hw = &adapter->hw;
4ae196df 6503 int pf_id = adapter->vfs_allocated_count;
9d5c8243 6504
51466239
AD		 6505 /* attempt to add filter to vlvf array */
6506 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 6507
51466239
AD		 6508 /* add the filter since PF can receive vlans w/o entry in vlvf */
6509 igb_vfta_set(hw, vid, true);
b2cb09b1
JP		 6510
6511 set_bit(vid, adapter->active_vlans);
9d5c8243
AK		 6512}
6513
6514static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6515{
6516 struct igb_adapter *adapter = netdev_priv(netdev);
6517 struct e1000_hw *hw = &adapter->hw;
4ae196df 6518 int pf_id = adapter->vfs_allocated_count;
51466239 6519 s32 err;
9d5c8243 6520
51466239
AD		 6521 /* remove vlan from VLVF table array */
6522 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 6523
51466239
AD		 6524 /* if vid was not present in VLVF just remove it from table */
6525 if (err)
4ae196df 6526 igb_vfta_set(hw, vid, false);
b2cb09b1
JP		 6527
6528 clear_bit(vid, adapter->active_vlans);
9d5c8243
AK		 6529}
6530
6531static void igb_restore_vlan(struct igb_adapter *adapter)
6532{
b2cb09b1 6533 u16 vid;
9d5c8243 6534
5faf030c
AD		 6535 igb_vlan_mode(adapter->netdev, adapter->netdev->features);
6536
b2cb09b1
JP		 6537 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
6538 igb_vlan_rx_add_vid(adapter->netdev, vid);
9d5c8243
AK		 6539}
6540
14ad2513 6541int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
9d5c8243 6542{
090b1795 6543 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 6544 struct e1000_mac_info *mac = &adapter->hw.mac;
6545
6546 mac->autoneg = 0;
6547
14ad2513
DD		 6548 /* Make sure dplx is at most 1 bit and lsb of speed is not set
6549 * for the switch() below to work */
6550 if ((spd & 1) || (dplx & ~1))
6551 goto err_inval;
6552
cd2638a8
CW		 6553 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6554 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
14ad2513
DD		 6555 spd != SPEED_1000 &&
6556 dplx != DUPLEX_FULL)
6557 goto err_inval;
cd2638a8 6558
14ad2513 6559 switch (spd + dplx) {
9d5c8243
AK		 6560 case SPEED_10 + DUPLEX_HALF:
6561 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6562 break;
6563 case SPEED_10 + DUPLEX_FULL:
6564 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6565 break;
6566 case SPEED_100 + DUPLEX_HALF:
6567 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6568 break;
6569 case SPEED_100 + DUPLEX_FULL:
6570 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6571 break;
6572 case SPEED_1000 + DUPLEX_FULL:
6573 mac->autoneg = 1;
6574 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6575 break;
6576 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6577 default:
14ad2513 6578 goto err_inval;
9d5c8243
AK		 6579 }
6580 return 0;
14ad2513
DD		 6581
6582err_inval:
6583 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6584 return -EINVAL;
9d5c8243
AK		 6585}
6586
3fe7c4c9 6587static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 6588{
6589 struct net_device *netdev = pci_get_drvdata(pdev);
6590 struct igb_adapter *adapter = netdev_priv(netdev);
6591 struct e1000_hw *hw = &adapter->hw;
2d064c06 6592 u32 ctrl, rctl, status;
9d5c8243
AK		 6593 u32 wufc = adapter->wol;
6594#ifdef CONFIG_PM
6595 int retval = 0;
6596#endif
6597
6598 netif_device_detach(netdev);
6599
a88f10ec
AD		 6600 if (netif_running(netdev))
6601 igb_close(netdev);
6602
047e0030 6603 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 6604
6605#ifdef CONFIG_PM
6606 retval = pci_save_state(pdev);
6607 if (retval)
6608 return retval;
6609#endif
6610
6611 status = rd32(E1000_STATUS);
6612 if (status & E1000_STATUS_LU)
6613 wufc &= ~E1000_WUFC_LNKC;
6614
6615 if (wufc) {
6616 igb_setup_rctl(adapter);
ff41f8dc 6617 igb_set_rx_mode(netdev);
9d5c8243
AK		 6618
6619 /* turn on all-multi mode if wake on multicast is enabled */
6620 if (wufc & E1000_WUFC_MC) {
6621 rctl = rd32(E1000_RCTL);
6622 rctl |= E1000_RCTL_MPE;
6623 wr32(E1000_RCTL, rctl);
6624 }
6625
6626 ctrl = rd32(E1000_CTRL);
6627 /* advertise wake from D3Cold */
6628 #define E1000_CTRL_ADVD3WUC 0x00100000
6629 /* phy power management enable */
6630 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6631 ctrl |= E1000_CTRL_ADVD3WUC;
6632 wr32(E1000_CTRL, ctrl);
6633
9d5c8243 6634 /* Allow time for pending master requests to run */
330a6d6a 6635 igb_disable_pcie_master(hw);
9d5c8243
AK		 6636
6637 wr32(E1000_WUC, E1000_WUC_PME_EN);
6638 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 6639 } else {
6640 wr32(E1000_WUC, 0);
6641 wr32(E1000_WUFC, 0);
9d5c8243
AK		 6642 }
6643
3fe7c4c9
RW		 6644 *enable_wake = wufc || adapter->en_mng_pt;
6645 if (!*enable_wake)
88a268c1
NN		 6646 igb_power_down_link(adapter);
6647 else
6648 igb_power_up_link(adapter);
9d5c8243
AK		 6649
6650 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6651 * would have already happened in close and is redundant. */
6652 igb_release_hw_control(adapter);
6653
6654 pci_disable_device(pdev);
6655
9d5c8243
AK		 6656 return 0;
6657}
6658
6659#ifdef CONFIG_PM
3fe7c4c9
RW		 6660static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
6661{
6662 int retval;
6663 bool wake;
6664
6665 retval = __igb_shutdown(pdev, &wake);
6666 if (retval)
6667 return retval;
6668
6669 if (wake) {
6670 pci_prepare_to_sleep(pdev);
6671 } else {
6672 pci_wake_from_d3(pdev, false);
6673 pci_set_power_state(pdev, PCI_D3hot);
6674 }
6675
6676 return 0;
6677}
6678
9d5c8243
AK		 6679static int igb_resume(struct pci_dev *pdev)
6680{
6681 struct net_device *netdev = pci_get_drvdata(pdev);
6682 struct igb_adapter *adapter = netdev_priv(netdev);
6683 struct e1000_hw *hw = &adapter->hw;
6684 u32 err;
6685
6686 pci_set_power_state(pdev, PCI_D0);
6687 pci_restore_state(pdev);
b94f2d77 6688 pci_save_state(pdev);
42bfd33a 6689
aed5dec3 6690 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 6691 if (err) {
6692 dev_err(&pdev->dev,
6693 "igb: Cannot enable PCI device from suspend\n");
6694 return err;
6695 }
6696 pci_set_master(pdev);
6697
6698 pci_enable_wake(pdev, PCI_D3hot, 0);
6699 pci_enable_wake(pdev, PCI_D3cold, 0);
6700
047e0030 6701 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 6702 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6703 return -ENOMEM;
9d5c8243
AK		 6704 }
6705
9d5c8243 6706 igb_reset(adapter);
a8564f03
AD		 6707
6708 /* let the f/w know that the h/w is now under the control of the
6709 * driver. */
6710 igb_get_hw_control(adapter);
6711
9d5c8243
AK		 6712 wr32(E1000_WUS, ~0);
6713
a88f10ec
AD		 6714 if (netif_running(netdev)) {
6715 err = igb_open(netdev);
6716 if (err)
6717 return err;
6718 }
9d5c8243
AK		 6719
6720 netif_device_attach(netdev);
6721
9d5c8243
AK		 6722 return 0;
6723}
6724#endif
6725
6726static void igb_shutdown(struct pci_dev *pdev)
6727{
3fe7c4c9
RW		 6728 bool wake;
6729
6730 __igb_shutdown(pdev, &wake);
6731
6732 if (system_state == SYSTEM_POWER_OFF) {
6733 pci_wake_from_d3(pdev, wake);
6734 pci_set_power_state(pdev, PCI_D3hot);
6735 }
9d5c8243
AK		 6736}
6737
6738#ifdef CONFIG_NET_POLL_CONTROLLER
6739/*
6740 * Polling 'interrupt' - used by things like netconsole to send skbs
6741 * without having to re-enable interrupts. It's not called while
6742 * the interrupt routine is executing.
6743 */
6744static void igb_netpoll(struct net_device *netdev)
6745{
6746 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 6747 struct e1000_hw *hw = &adapter->hw;
0d1ae7f4 6748 struct igb_q_vector *q_vector;
9d5c8243 6749 int i;
9d5c8243 6750
047e0030 6751 for (i = 0; i < adapter->num_q_vectors; i++) {
0d1ae7f4
AD		 6752 q_vector = adapter->q_vector[i];
6753 if (adapter->msix_entries)
6754 wr32(E1000_EIMC, q_vector->eims_value);
6755 else
6756 igb_irq_disable(adapter);
047e0030 6757 napi_schedule(&q_vector->napi);
eebbbdba 6758 }
9d5c8243
AK		 6759}
6760#endif /* CONFIG_NET_POLL_CONTROLLER */
6761
6762/**
6763 * igb_io_error_detected - called when PCI error is detected
6764 * @pdev: Pointer to PCI device
6765 * @state: The current pci connection state
6766 *
6767 * This function is called after a PCI bus error affecting
6768 * this device has been detected.
6769 */
6770static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6771 pci_channel_state_t state)
6772{
6773 struct net_device *netdev = pci_get_drvdata(pdev);
6774 struct igb_adapter *adapter = netdev_priv(netdev);
6775
6776 netif_device_detach(netdev);
6777
59ed6eec
AD		 6778 if (state == pci_channel_io_perm_failure)
6779 return PCI_ERS_RESULT_DISCONNECT;
6780
9d5c8243
AK		 6781 if (netif_running(netdev))
6782 igb_down(adapter);
6783 pci_disable_device(pdev);
6784
6785 /* Request a slot slot reset. */
6786 return PCI_ERS_RESULT_NEED_RESET;
6787}
6788
6789/**
6790 * igb_io_slot_reset - called after the pci bus has been reset.
6791 * @pdev: Pointer to PCI device
6792 *
6793 * Restart the card from scratch, as if from a cold-boot. Implementation
6794 * resembles the first-half of the igb_resume routine.
6795 */
6796static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6797{
6798 struct net_device *netdev = pci_get_drvdata(pdev);
6799 struct igb_adapter *adapter = netdev_priv(netdev);
6800 struct e1000_hw *hw = &adapter->hw;
40a914fa 6801 pci_ers_result_t result;
42bfd33a 6802 int err;
9d5c8243 6803
aed5dec3 6804 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6805 dev_err(&pdev->dev,
6806 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6807 result = PCI_ERS_RESULT_DISCONNECT;
6808 } else {
6809 pci_set_master(pdev);
6810 pci_restore_state(pdev);
b94f2d77 6811 pci_save_state(pdev);
9d5c8243 6812
40a914fa
AD		 6813 pci_enable_wake(pdev, PCI_D3hot, 0);
6814 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6815
40a914fa
AD		 6816 igb_reset(adapter);
6817 wr32(E1000_WUS, ~0);
6818 result = PCI_ERS_RESULT_RECOVERED;
6819 }
9d5c8243 6820
ea943d41
JK		 6821 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6822 if (err) {
6823 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6824 "failed 0x%0x\n", err);
6825 /* non-fatal, continue */
6826 }
40a914fa
AD		 6827
6828 return result;
9d5c8243
AK		 6829}
6830
6831/**
6832 * igb_io_resume - called when traffic can start flowing again.
6833 * @pdev: Pointer to PCI device
6834 *
6835 * This callback is called when the error recovery driver tells us that
6836 * its OK to resume normal operation. Implementation resembles the
6837 * second-half of the igb_resume routine.
6838 */
6839static void igb_io_resume(struct pci_dev *pdev)
6840{
6841 struct net_device *netdev = pci_get_drvdata(pdev);
6842 struct igb_adapter *adapter = netdev_priv(netdev);
6843
9d5c8243
AK		 6844 if (netif_running(netdev)) {
6845 if (igb_up(adapter)) {
6846 dev_err(&pdev->dev, "igb_up failed after reset\n");
6847 return;
6848 }
6849 }
6850
6851 netif_device_attach(netdev);
6852
6853 /* let the f/w know that the h/w is now under the control of the
6854 * driver. */
6855 igb_get_hw_control(adapter);
9d5c8243
AK		 6856}
6857
26ad9178
AD		 6858static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6859 u8 qsel)
6860{
6861 u32 rar_low, rar_high;
6862 struct e1000_hw *hw = &adapter->hw;
6863
6864 /* HW expects these in little endian so we reverse the byte order
6865 * from network order (big endian) to little endian
6866 */
6867 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6868 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6869 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6870
6871 /* Indicate to hardware the Address is Valid. */
6872 rar_high |= E1000_RAH_AV;
6873
6874 if (hw->mac.type == e1000_82575)
6875 rar_high |= E1000_RAH_POOL_1 * qsel;
6876 else
6877 rar_high |= E1000_RAH_POOL_1 << qsel;
6878
6879 wr32(E1000_RAL(index), rar_low);
6880 wrfl();
6881 wr32(E1000_RAH(index), rar_high);
6882 wrfl();
6883}
6884
4ae196df
AD		 6885static int igb_set_vf_mac(struct igb_adapter *adapter,
6886 int vf, unsigned char *mac_addr)
6887{
6888 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 6889 /* VF MAC addresses start at end of receive addresses and moves
6890 * torwards the first, as a result a collision should not be possible */
6891 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 6892
37680117 6893 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 6894
26ad9178 6895 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 6896
6897 return 0;
6898}
6899
8151d294
WM		 6900static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6901{
6902 struct igb_adapter *adapter = netdev_priv(netdev);
6903 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6904 return -EINVAL;
6905 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6906 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6907 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6908 " change effective.");
6909 if (test_bit(__IGB_DOWN, &adapter->state)) {
6910 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6911 " but the PF device is not up.\n");
6912 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6913 " attempting to use the VF device.\n");
6914 }
6915 return igb_set_vf_mac(adapter, vf, mac);
6916}
6917
17dc566c
LL		 6918static int igb_link_mbps(int internal_link_speed)
6919{
6920 switch (internal_link_speed) {
6921 case SPEED_100:
6922 return 100;
6923 case SPEED_1000:
6924 return 1000;
6925 default:
6926 return 0;
6927 }
6928}
6929
6930static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
6931 int link_speed)
6932{
6933 int rf_dec, rf_int;
6934 u32 bcnrc_val;
6935
6936 if (tx_rate != 0) {
6937 /* Calculate the rate factor values to set */
6938 rf_int = link_speed / tx_rate;
6939 rf_dec = (link_speed - (rf_int * tx_rate));
6940 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
6941
6942 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
6943 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
6944 E1000_RTTBCNRC_RF_INT_MASK);
6945 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
6946 } else {
6947 bcnrc_val = 0;
6948 }
6949
6950 wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
6951 wr32(E1000_RTTBCNRC, bcnrc_val);
6952}
6953
6954static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
6955{
6956 int actual_link_speed, i;
6957 bool reset_rate = false;
6958
6959 /* VF TX rate limit was not set or not supported */
6960 if ((adapter->vf_rate_link_speed == 0) ||
6961 (adapter->hw.mac.type != e1000_82576))
6962 return;
6963
6964 actual_link_speed = igb_link_mbps(adapter->link_speed);
6965 if (actual_link_speed != adapter->vf_rate_link_speed) {
6966 reset_rate = true;
6967 adapter->vf_rate_link_speed = 0;
6968 dev_info(&adapter->pdev->dev,
6969 "Link speed has been changed. VF Transmit "
6970 "rate is disabled\n");
6971 }
6972
6973 for (i = 0; i < adapter->vfs_allocated_count; i++) {
6974 if (reset_rate)
6975 adapter->vf_data[i].tx_rate = 0;
6976
6977 igb_set_vf_rate_limit(&adapter->hw, i,
6978 adapter->vf_data[i].tx_rate,
6979 actual_link_speed);
6980 }
6981}
6982
8151d294
WM		 6983static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6984{
17dc566c
LL		 6985 struct igb_adapter *adapter = netdev_priv(netdev);
6986 struct e1000_hw *hw = &adapter->hw;
6987 int actual_link_speed;
6988
6989 if (hw->mac.type != e1000_82576)
6990 return -EOPNOTSUPP;
6991
6992 actual_link_speed = igb_link_mbps(adapter->link_speed);
6993 if ((vf >= adapter->vfs_allocated_count) ||
6994 (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
6995 (tx_rate < 0) || (tx_rate > actual_link_speed))
6996 return -EINVAL;
6997
6998 adapter->vf_rate_link_speed = actual_link_speed;
6999 adapter->vf_data[vf].tx_rate = (u16)tx_rate;
7000 igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
7001
7002 return 0;
8151d294
WM		 7003}
7004
7005static int igb_ndo_get_vf_config(struct net_device *netdev,
7006 int vf, struct ifla_vf_info *ivi)
7007{
7008 struct igb_adapter *adapter = netdev_priv(netdev);
7009 if (vf >= adapter->vfs_allocated_count)
7010 return -EINVAL;
7011 ivi->vf = vf;
7012 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
17dc566c 7013 ivi->tx_rate = adapter->vf_data[vf].tx_rate;
8151d294
WM		 7014 ivi->vlan = adapter->vf_data[vf].pf_vlan;
7015 ivi->qos = adapter->vf_data[vf].pf_qos;
7016 return 0;
7017}
7018
4ae196df
AD		 7019static void igb_vmm_control(struct igb_adapter *adapter)
7020{
7021 struct e1000_hw *hw = &adapter->hw;
10d8e907 7022 u32 reg;
4ae196df 7023
52a1dd4d
AD		 7024 switch (hw->mac.type) {
7025 case e1000_82575:
7026 default:
7027 /* replication is not supported for 82575 */
4ae196df 7028 return;
52a1dd4d
AD		 7029 case e1000_82576:
7030 /* notify HW that the MAC is adding vlan tags */
7031 reg = rd32(E1000_DTXCTL);
7032 reg |= E1000_DTXCTL_VLAN_ADDED;
7033 wr32(E1000_DTXCTL, reg);
7034 case e1000_82580:
7035 /* enable replication vlan tag stripping */
7036 reg = rd32(E1000_RPLOLR);
7037 reg |= E1000_RPLOLR_STRVLAN;
7038 wr32(E1000_RPLOLR, reg);
d2ba2ed8
AD		 7039 case e1000_i350:
7040 /* none of the above registers are supported by i350 */
52a1dd4d
AD		 7041 break;
7042 }
10d8e907 7043
d4960307
AD		 7044 if (adapter->vfs_allocated_count) {
7045 igb_vmdq_set_loopback_pf(hw, true);
7046 igb_vmdq_set_replication_pf(hw, true);
13800469
GR		 7047 igb_vmdq_set_anti_spoofing_pf(hw, true,
7048 adapter->vfs_allocated_count);
d4960307
AD		 7049 } else {
7050 igb_vmdq_set_loopback_pf(hw, false);
7051 igb_vmdq_set_replication_pf(hw, false);
7052 }
4ae196df
AD		 7053}
7054
b6e0c419
CW		 7055static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
7056{
7057 struct e1000_hw *hw = &adapter->hw;
7058 u32 dmac_thr;
7059 u16 hwm;
7060
7061 if (hw->mac.type > e1000_82580) {
7062 if (adapter->flags & IGB_FLAG_DMAC) {
7063 u32 reg;
7064
7065 /* force threshold to 0. */
7066 wr32(E1000_DMCTXTH, 0);
7067
7068 /*
7069 * DMA Coalescing high water mark needs to be higher
7070 * than the RX threshold. set hwm to PBA - 2 * max
7071 * frame size
7072 */
7073 hwm = pba - (2 * adapter->max_frame_size);
7074 reg = rd32(E1000_DMACR);
7075 reg &= ~E1000_DMACR_DMACTHR_MASK;
7076 dmac_thr = pba - 4;
7077
7078 reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
7079 & E1000_DMACR_DMACTHR_MASK);
7080
7081 /* transition to L0x or L1 if available..*/
7082 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
7083
7084 /* watchdog timer= +-1000 usec in 32usec intervals */
7085 reg |= (1000 >> 5);
7086 wr32(E1000_DMACR, reg);
7087
7088 /*
7089 * no lower threshold to disable
7090 * coalescing(smart fifb)-UTRESH=0
7091 */
7092 wr32(E1000_DMCRTRH, 0);
7093 wr32(E1000_FCRTC, hwm);
7094
7095 reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
7096
7097 wr32(E1000_DMCTLX, reg);
7098
7099 /*
7100 * free space in tx packet buffer to wake from
7101 * DMA coal
7102 */
7103 wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
7104 (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
7105
7106 /*
7107 * make low power state decision controlled
7108 * by DMA coal
7109 */
7110 reg = rd32(E1000_PCIEMISC);
7111 reg &= ~E1000_PCIEMISC_LX_DECISION;
7112 wr32(E1000_PCIEMISC, reg);
7113 } /* endif adapter->dmac is not disabled */
7114 } else if (hw->mac.type == e1000_82580) {
7115 u32 reg = rd32(E1000_PCIEMISC);
7116 wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION);
7117 wr32(E1000_DMACR, 0);
7118 }
7119}
7120
9d5c8243 7121/* igb_main.c */
Ubuntu Artful kernel mirror