]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/net/igb/igb_main.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
igb: cleanup clean_rx_irq_adv and alloc_rx_buffers_adv
[mirror_ubuntu-artful-kernel.git] / drivers / net / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
86d5d38f 4 Copyright(c) 2007-2009 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
31#include <linux/vmalloc.h>
32#include <linux/pagemap.h>
33#include <linux/netdevice.h>
9d5c8243
AK		 34#include <linux/ipv6.h>
35#include <net/checksum.h>
36#include <net/ip6_checksum.h>
c6cb090b 37#include <linux/net_tstamp.h>
9d5c8243
AK		 38#include <linux/mii.h>
39#include <linux/ethtool.h>
40#include <linux/if_vlan.h>
41#include <linux/pci.h>
c54106bb 42#include <linux/pci-aspm.h>
9d5c8243
AK		 43#include <linux/delay.h>
44#include <linux/interrupt.h>
45#include <linux/if_ether.h>
40a914fa 46#include <linux/aer.h>
421e02f0 47#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 48#include <linux/dca.h>
49#endif
9d5c8243
AK		 50#include "igb.h"
51
86d5d38f 52#define DRV_VERSION "1.3.16-k2"
9d5c8243
AK		 53char igb_driver_name[] = "igb";
54char igb_driver_version[] = DRV_VERSION;
55static const char igb_driver_string[] =
56 "Intel(R) Gigabit Ethernet Network Driver";
86d5d38f 57static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
9d5c8243 58
9d5c8243
AK		 59static const struct e1000_info *igb_info_tbl[] = {
60 [board_82575] = &e1000_82575_info,
61};
62
63static struct pci_device_id igb_pci_tbl[] = {
2d064c06 64 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
c8ea5ea9 70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
74 /* required last entry */
75 {0, }
76};
77
78MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
79
80void igb_reset(struct igb_adapter *);
81static int igb_setup_all_tx_resources(struct igb_adapter *);
82static int igb_setup_all_rx_resources(struct igb_adapter *);
83static void igb_free_all_tx_resources(struct igb_adapter *);
84static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 85static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 86void igb_update_stats(struct igb_adapter *);
87static int igb_probe(struct pci_dev *, const struct pci_device_id *);
88static void __devexit igb_remove(struct pci_dev *pdev);
89static int igb_sw_init(struct igb_adapter *);
90static int igb_open(struct net_device *);
91static int igb_close(struct net_device *);
92static void igb_configure_tx(struct igb_adapter *);
93static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 94static void igb_clean_all_tx_rings(struct igb_adapter *);
95static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 96static void igb_clean_tx_ring(struct igb_ring *);
97static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 98static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 99static void igb_update_phy_info(unsigned long);
100static void igb_watchdog(unsigned long);
101static void igb_watchdog_task(struct work_struct *);
b1a436c3 102static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
9d5c8243
AK		 103static struct net_device_stats *igb_get_stats(struct net_device *);
104static int igb_change_mtu(struct net_device *, int);
105static int igb_set_mac(struct net_device *, void *);
68d480c4 106static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 107static irqreturn_t igb_intr(int irq, void *);
108static irqreturn_t igb_intr_msi(int irq, void *);
109static irqreturn_t igb_msix_other(int irq, void *);
047e0030 110static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 111#ifdef CONFIG_IGB_DCA
047e0030 112static void igb_update_dca(struct igb_q_vector *);
fe4506b6 113static void igb_setup_dca(struct igb_adapter *);
421e02f0 114#endif /* CONFIG_IGB_DCA */
047e0030 115static bool igb_clean_tx_irq(struct igb_q_vector *);
661086df 116static int igb_poll(struct napi_struct *, int);
047e0030 117static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
9d5c8243
AK		 118static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
119static void igb_tx_timeout(struct net_device *);
120static void igb_reset_task(struct work_struct *);
121static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
122static void igb_vlan_rx_add_vid(struct net_device *, u16);
123static void igb_vlan_rx_kill_vid(struct net_device *, u16);
124static void igb_restore_vlan(struct igb_adapter *);
26ad9178 125static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 126static void igb_ping_all_vfs(struct igb_adapter *);
127static void igb_msg_task(struct igb_adapter *);
4ae196df 128static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 129static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 130static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
9d5c8243 131
9d5c8243 132#ifdef CONFIG_PM
3fe7c4c9 133static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 134static int igb_resume(struct pci_dev *);
135#endif
136static void igb_shutdown(struct pci_dev *);
421e02f0 137#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 138static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
139static struct notifier_block dca_notifier = {
140 .notifier_call = igb_notify_dca,
141 .next = NULL,
142 .priority = 0
143};
144#endif
9d5c8243
AK		 145#ifdef CONFIG_NET_POLL_CONTROLLER
146/* for netdump / net console */
147static void igb_netpoll(struct net_device *);
148#endif
37680117 149#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 150static unsigned int max_vfs = 0;
151module_param(max_vfs, uint, 0);
152MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
153 "per physical function");
154#endif /* CONFIG_PCI_IOV */
155
9d5c8243
AK		 156static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
157 pci_channel_state_t);
158static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
159static void igb_io_resume(struct pci_dev *);
160
161static struct pci_error_handlers igb_err_handler = {
162 .error_detected = igb_io_error_detected,
163 .slot_reset = igb_io_slot_reset,
164 .resume = igb_io_resume,
165};
166
167
168static struct pci_driver igb_driver = {
169 .name = igb_driver_name,
170 .id_table = igb_pci_tbl,
171 .probe = igb_probe,
172 .remove = __devexit_p(igb_remove),
173#ifdef CONFIG_PM
174 /* Power Managment Hooks */
175 .suspend = igb_suspend,
176 .resume = igb_resume,
177#endif
178 .shutdown = igb_shutdown,
179 .err_handler = &igb_err_handler
180};
181
182MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
183MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
184MODULE_LICENSE("GPL");
185MODULE_VERSION(DRV_VERSION);
186
38c845c7
PO		 187/**
188 * igb_read_clock - read raw cycle counter (to be used by time counter)
189 */
190static cycle_t igb_read_clock(const struct cyclecounter *tc)
191{
192 struct igb_adapter *adapter =
193 container_of(tc, struct igb_adapter, cycles);
194 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 195 u64 stamp = 0;
196 int shift = 0;
38c845c7 197
c5b9bd5e
AD		 198 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
199 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 200 return stamp;
201}
202
9d5c8243
AK		 203#ifdef DEBUG
204/**
205 * igb_get_hw_dev_name - return device name string
206 * used by hardware layer to print debugging information
207 **/
208char *igb_get_hw_dev_name(struct e1000_hw *hw)
209{
210 struct igb_adapter *adapter = hw->back;
211 return adapter->netdev->name;
212}
38c845c7
PO		 213
214/**
215 * igb_get_time_str - format current NIC and system time as string
216 */
217static char *igb_get_time_str(struct igb_adapter *adapter,
218 char buffer[160])
219{
220 cycle_t hw = adapter->cycles.read(&adapter->cycles);
221 struct timespec nic = ns_to_timespec(timecounter_read(&adapter->clock));
222 struct timespec sys;
223 struct timespec delta;
224 getnstimeofday(&sys);
225
226 delta = timespec_sub(nic, sys);
227
228 sprintf(buffer,
33af6bcc
PO		 229 "HW %llu, NIC %ld.%09lus, SYS %ld.%09lus, NIC-SYS %lds + %09luns",
230 hw,
38c845c7
PO		 231 (long)nic.tv_sec, nic.tv_nsec,
232 (long)sys.tv_sec, sys.tv_nsec,
233 (long)delta.tv_sec, delta.tv_nsec);
234
235 return buffer;
236}
9d5c8243
AK		 237#endif
238
239/**
240 * igb_init_module - Driver Registration Routine
241 *
242 * igb_init_module is the first routine called when the driver is
243 * loaded. All it does is register with the PCI subsystem.
244 **/
245static int __init igb_init_module(void)
246{
247 int ret;
248 printk(KERN_INFO "%s - version %s\n",
249 igb_driver_string, igb_driver_version);
250
251 printk(KERN_INFO "%s\n", igb_copyright);
252
421e02f0 253#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 254 dca_register_notify(&dca_notifier);
255#endif
bbd98fe4 256 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 257 return ret;
258}
259
260module_init(igb_init_module);
261
262/**
263 * igb_exit_module - Driver Exit Cleanup Routine
264 *
265 * igb_exit_module is called just before the driver is removed
266 * from memory.
267 **/
268static void __exit igb_exit_module(void)
269{
421e02f0 270#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 271 dca_unregister_notify(&dca_notifier);
272#endif
9d5c8243
AK		 273 pci_unregister_driver(&igb_driver);
274}
275
276module_exit(igb_exit_module);
277
26bc19ec
AD		 278#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
279/**
280 * igb_cache_ring_register - Descriptor ring to register mapping
281 * @adapter: board private structure to initialize
282 *
283 * Once we know the feature-set enabled for the device, we'll cache
284 * the register offset the descriptor ring is assigned to.
285 **/
286static void igb_cache_ring_register(struct igb_adapter *adapter)
287{
ee1b9f06 288 int i = 0, j = 0;
047e0030 289 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 290
291 switch (adapter->hw.mac.type) {
292 case e1000_82576:
293 /* The queues are allocated for virtualization such that VF 0
294 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
295 * In order to avoid collision we start at the first free queue
296 * and continue consuming queues in the same sequence
297 */
ee1b9f06
AD		 298 if (adapter->vfs_allocated_count) {
299 for (; i < adapter->num_rx_queues; i++)
300 adapter->rx_ring[i].reg_idx = rbase_offset +
301 Q_IDX_82576(i);
302 for (; j < adapter->num_tx_queues; j++)
303 adapter->tx_ring[j].reg_idx = rbase_offset +
304 Q_IDX_82576(j);
305 }
26bc19ec
AD		 306 case e1000_82575:
307 default:
ee1b9f06
AD		 308 for (; i < adapter->num_rx_queues; i++)
309 adapter->rx_ring[i].reg_idx = rbase_offset + i;
310 for (; j < adapter->num_tx_queues; j++)
311 adapter->tx_ring[j].reg_idx = rbase_offset + j;
26bc19ec
AD		 312 break;
313 }
314}
315
047e0030
AD		 316static void igb_free_queues(struct igb_adapter *adapter)
317{
318 kfree(adapter->tx_ring);
319 kfree(adapter->rx_ring);
320
321 adapter->tx_ring = NULL;
322 adapter->rx_ring = NULL;
323
324 adapter->num_rx_queues = 0;
325 adapter->num_tx_queues = 0;
326}
327
9d5c8243
AK		 328/**
329 * igb_alloc_queues - Allocate memory for all rings
330 * @adapter: board private structure to initialize
331 *
332 * We allocate one ring per queue at run-time since we don't know the
333 * number of queues at compile-time.
334 **/
335static int igb_alloc_queues(struct igb_adapter *adapter)
336{
337 int i;
338
339 adapter->tx_ring = kcalloc(adapter->num_tx_queues,
340 sizeof(struct igb_ring), GFP_KERNEL);
341 if (!adapter->tx_ring)
047e0030 342 goto err;
9d5c8243
AK		 343
344 adapter->rx_ring = kcalloc(adapter->num_rx_queues,
345 sizeof(struct igb_ring), GFP_KERNEL);
047e0030
AD		 346 if (!adapter->rx_ring)
347 goto err;
6eb5a7f1 348
661086df
PWJ		 349 for (i = 0; i < adapter->num_tx_queues; i++) {
350 struct igb_ring *ring = &(adapter->tx_ring[i]);
68fd9910 351 ring->count = adapter->tx_ring_count;
661086df 352 ring->queue_index = i;
80785298 353 ring->pdev = adapter->pdev;
e694e964 354 ring->netdev = adapter->netdev;
85ad76b2
AD		 355 /* For 82575, context index must be unique per ring. */
356 if (adapter->hw.mac.type == e1000_82575)
357 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
661086df 358 }
85ad76b2 359
9d5c8243
AK		 360 for (i = 0; i < adapter->num_rx_queues; i++) {
361 struct igb_ring *ring = &(adapter->rx_ring[i]);
68fd9910 362 ring->count = adapter->rx_ring_count;
844290e5 363 ring->queue_index = i;
80785298 364 ring->pdev = adapter->pdev;
e694e964 365 ring->netdev = adapter->netdev;
4c844851 366 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85ad76b2
AD		 367 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
368 /* set flag indicating ring supports SCTP checksum offload */
369 if (adapter->hw.mac.type >= e1000_82576)
370 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
9d5c8243 371 }
26bc19ec
AD		 372
373 igb_cache_ring_register(adapter);
9d5c8243 374
047e0030 375 return 0;
a88f10ec 376
047e0030
AD		 377err:
378 igb_free_queues(adapter);
d1a8c9e1 379
047e0030 380 return -ENOMEM;
a88f10ec
AD		 381}
382
9d5c8243 383#define IGB_N0_QUEUE -1
047e0030 384static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 385{
386 u32 msixbm = 0;
047e0030 387 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 388 struct e1000_hw *hw = &adapter->hw;
2d064c06 389 u32 ivar, index;
047e0030
AD		 390 int rx_queue = IGB_N0_QUEUE;
391 int tx_queue = IGB_N0_QUEUE;
392
393 if (q_vector->rx_ring)
394 rx_queue = q_vector->rx_ring->reg_idx;
395 if (q_vector->tx_ring)
396 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 397
398 switch (hw->mac.type) {
399 case e1000_82575:
9d5c8243
AK		 400 /* The 82575 assigns vectors using a bitmask, which matches the
401 bitmask for the EICR/EIMS/EIMC registers. To assign one
402 or more queues to a vector, we write the appropriate bits
403 into the MSIXBM register for that vector. */
047e0030 404 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 405 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 406 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 407 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
9d5c8243 408 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 409 q_vector->eims_value = msixbm;
2d064c06
AD		 410 break;
411 case e1000_82576:
26bc19ec 412 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 413 Each queue has a single entry in the table to which we write
414 a vector number along with a "valid" bit. Sadly, the layout
415 of the table is somewhat counterintuitive. */
416 if (rx_queue > IGB_N0_QUEUE) {
047e0030 417 index = (rx_queue & 0x7);
2d064c06 418 ivar = array_rd32(E1000_IVAR0, index);
047e0030 419 if (rx_queue < 8) {
26bc19ec
AD		 420 /* vector goes into low byte of register */
421 ivar = ivar & 0xFFFFFF00;
422 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 423 } else {
424 /* vector goes into third byte of register */
425 ivar = ivar & 0xFF00FFFF;
426 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 427 }
2d064c06
AD		 428 array_wr32(E1000_IVAR0, index, ivar);
429 }
430 if (tx_queue > IGB_N0_QUEUE) {
047e0030 431 index = (tx_queue & 0x7);
2d064c06 432 ivar = array_rd32(E1000_IVAR0, index);
047e0030 433 if (tx_queue < 8) {
26bc19ec
AD		 434 /* vector goes into second byte of register */
435 ivar = ivar & 0xFFFF00FF;
436 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 437 } else {
438 /* vector goes into high byte of register */
439 ivar = ivar & 0x00FFFFFF;
440 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 441 }
2d064c06
AD		 442 array_wr32(E1000_IVAR0, index, ivar);
443 }
047e0030 444 q_vector->eims_value = 1 << msix_vector;
2d064c06
AD		 445 break;
446 default:
447 BUG();
448 break;
449 }
9d5c8243
AK		 450}
451
452/**
453 * igb_configure_msix - Configure MSI-X hardware
454 *
455 * igb_configure_msix sets up the hardware to properly
456 * generate MSI-X interrupts.
457 **/
458static void igb_configure_msix(struct igb_adapter *adapter)
459{
460 u32 tmp;
461 int i, vector = 0;
462 struct e1000_hw *hw = &adapter->hw;
463
464 adapter->eims_enable_mask = 0;
9d5c8243
AK		 465
466 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 467 switch (hw->mac.type) {
468 case e1000_82575:
9d5c8243
AK		 469 tmp = rd32(E1000_CTRL_EXT);
470 /* enable MSI-X PBA support*/
471 tmp |= E1000_CTRL_EXT_PBA_CLR;
472
473 /* Auto-Mask interrupts upon ICR read. */
474 tmp |= E1000_CTRL_EXT_EIAME;
475 tmp |= E1000_CTRL_EXT_IRCA;
476
477 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 478
479 /* enable msix_other interrupt */
480 array_wr32(E1000_MSIXBM(0), vector++,
481 E1000_EIMS_OTHER);
844290e5 482 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 483
2d064c06
AD		 484 break;
485
486 case e1000_82576:
047e0030
AD		 487 /* Turn on MSI-X capability first, or our settings
488 * won't stick. And it will take days to debug. */
489 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
490 E1000_GPIE_PBA | E1000_GPIE_EIAME |
491 E1000_GPIE_NSICR);
492
493 /* enable msix_other interrupt */
494 adapter->eims_other = 1 << vector;
2d064c06 495 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 496
047e0030 497 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 498 break;
499 default:
500 /* do nothing, since nothing else supports MSI-X */
501 break;
502 } /* switch (hw->mac.type) */
047e0030
AD		 503
504 adapter->eims_enable_mask |= adapter->eims_other;
505
506 for (i = 0; i < adapter->num_q_vectors; i++) {
507 struct igb_q_vector *q_vector = adapter->q_vector[i];
508 igb_assign_vector(q_vector, vector++);
509 adapter->eims_enable_mask |= q_vector->eims_value;
510 }
511
9d5c8243
AK		 512 wrfl();
513}
514
515/**
516 * igb_request_msix - Initialize MSI-X interrupts
517 *
518 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
519 * kernel.
520 **/
521static int igb_request_msix(struct igb_adapter *adapter)
522{
523 struct net_device *netdev = adapter->netdev;
047e0030 524 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 525 int i, err = 0, vector = 0;
526
047e0030
AD		 527 err = request_irq(adapter->msix_entries[vector].vector,
528 &igb_msix_other, 0, netdev->name, adapter);
529 if (err)
530 goto out;
531 vector++;
532
533 for (i = 0; i < adapter->num_q_vectors; i++) {
534 struct igb_q_vector *q_vector = adapter->q_vector[i];
535
536 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
537
538 if (q_vector->rx_ring && q_vector->tx_ring)
539 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
540 q_vector->rx_ring->queue_index);
541 else if (q_vector->tx_ring)
542 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
543 q_vector->tx_ring->queue_index);
544 else if (q_vector->rx_ring)
545 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
546 q_vector->rx_ring->queue_index);
9d5c8243 547 else
047e0030
AD		 548 sprintf(q_vector->name, "%s-unused", netdev->name);
549
9d5c8243 550 err = request_irq(adapter->msix_entries[vector].vector,
047e0030
AD		 551 &igb_msix_ring, 0, q_vector->name,
552 q_vector);
9d5c8243
AK		 553 if (err)
554 goto out;
9d5c8243
AK		 555 vector++;
556 }
557
9d5c8243
AK		 558 igb_configure_msix(adapter);
559 return 0;
560out:
561 return err;
562}
563
564static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
565{
566 if (adapter->msix_entries) {
567 pci_disable_msix(adapter->pdev);
568 kfree(adapter->msix_entries);
569 adapter->msix_entries = NULL;
047e0030 570 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 571 pci_disable_msi(adapter->pdev);
047e0030 572 }
9d5c8243
AK		 573}
574
047e0030
AD		 575/**
576 * igb_free_q_vectors - Free memory allocated for interrupt vectors
577 * @adapter: board private structure to initialize
578 *
579 * This function frees the memory allocated to the q_vectors. In addition if
580 * NAPI is enabled it will delete any references to the NAPI struct prior
581 * to freeing the q_vector.
582 **/
583static void igb_free_q_vectors(struct igb_adapter *adapter)
584{
585 int v_idx;
586
587 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
588 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
589 adapter->q_vector[v_idx] = NULL;
590 netif_napi_del(&q_vector->napi);
591 kfree(q_vector);
592 }
593 adapter->num_q_vectors = 0;
594}
595
596/**
597 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
598 *
599 * This function resets the device so that it has 0 rx queues, tx queues, and
600 * MSI-X interrupts allocated.
601 */
602static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
603{
604 igb_free_queues(adapter);
605 igb_free_q_vectors(adapter);
606 igb_reset_interrupt_capability(adapter);
607}
9d5c8243
AK		 608
609/**
610 * igb_set_interrupt_capability - set MSI or MSI-X if supported
611 *
612 * Attempt to configure interrupts using the best available
613 * capabilities of the hardware and kernel.
614 **/
615static void igb_set_interrupt_capability(struct igb_adapter *adapter)
616{
617 int err;
618 int numvecs, i;
619
83b7180d 620 /* Number of supported queues. */
83b7180d
AD		 621 adapter->num_rx_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
622 adapter->num_tx_queues = min_t(u32, IGB_MAX_TX_QUEUES, num_online_cpus());
623
047e0030
AD		 624 /* start with one vector for every rx queue */
625 numvecs = adapter->num_rx_queues;
626
627 /* if tx handler is seperate add 1 for every tx queue */
628 numvecs += adapter->num_tx_queues;
629
630 /* store the number of vectors reserved for queues */
631 adapter->num_q_vectors = numvecs;
632
633 /* add 1 vector for link status interrupts */
634 numvecs++;
9d5c8243
AK		 635 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
636 GFP_KERNEL);
637 if (!adapter->msix_entries)
638 goto msi_only;
639
640 for (i = 0; i < numvecs; i++)
641 adapter->msix_entries[i].entry = i;
642
643 err = pci_enable_msix(adapter->pdev,
644 adapter->msix_entries,
645 numvecs);
646 if (err == 0)
34a20e89 647 goto out;
9d5c8243
AK		 648
649 igb_reset_interrupt_capability(adapter);
650
651 /* If we can't do MSI-X, try MSI */
652msi_only:
2a3abf6d
AD		 653#ifdef CONFIG_PCI_IOV
654 /* disable SR-IOV for non MSI-X configurations */
655 if (adapter->vf_data) {
656 struct e1000_hw *hw = &adapter->hw;
657 /* disable iov and allow time for transactions to clear */
658 pci_disable_sriov(adapter->pdev);
659 msleep(500);
660
661 kfree(adapter->vf_data);
662 adapter->vf_data = NULL;
663 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
664 msleep(100);
665 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
666 }
667#endif
4fc82adf
AD		 668 adapter->vfs_allocated_count = 0;
669 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 670 adapter->num_rx_queues = 1;
661086df 671 adapter->num_tx_queues = 1;
047e0030 672 adapter->num_q_vectors = 1;
9d5c8243 673 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 674 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 675out:
661086df 676 /* Notify the stack of the (possibly) reduced Tx Queue count. */
fd2ea0a7 677 adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
9d5c8243
AK		 678 return;
679}
680
047e0030
AD		 681/**
682 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
683 * @adapter: board private structure to initialize
684 *
685 * We allocate one q_vector per queue interrupt. If allocation fails we
686 * return -ENOMEM.
687 **/
688static int igb_alloc_q_vectors(struct igb_adapter *adapter)
689{
690 struct igb_q_vector *q_vector;
691 struct e1000_hw *hw = &adapter->hw;
692 int v_idx;
693
694 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
695 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
696 if (!q_vector)
697 goto err_out;
698 q_vector->adapter = adapter;
699 q_vector->itr_shift = (hw->mac.type == e1000_82575) ? 16 : 0;
700 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
701 q_vector->itr_val = IGB_START_ITR;
702 q_vector->set_itr = 1;
703 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
704 adapter->q_vector[v_idx] = q_vector;
705 }
706 return 0;
707
708err_out:
709 while (v_idx) {
710 v_idx--;
711 q_vector = adapter->q_vector[v_idx];
712 netif_napi_del(&q_vector->napi);
713 kfree(q_vector);
714 adapter->q_vector[v_idx] = NULL;
715 }
716 return -ENOMEM;
717}
718
719static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
720 int ring_idx, int v_idx)
721{
722 struct igb_q_vector *q_vector;
723
724 q_vector = adapter->q_vector[v_idx];
725 q_vector->rx_ring = &adapter->rx_ring[ring_idx];
726 q_vector->rx_ring->q_vector = q_vector;
4fc82adf
AD		 727 q_vector->itr_val = adapter->rx_itr_setting;
728 if (q_vector->itr_val && q_vector->itr_val <= 3)
729 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 730}
731
732static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
733 int ring_idx, int v_idx)
734{
735 struct igb_q_vector *q_vector;
736
737 q_vector = adapter->q_vector[v_idx];
738 q_vector->tx_ring = &adapter->tx_ring[ring_idx];
739 q_vector->tx_ring->q_vector = q_vector;
4fc82adf
AD		 740 q_vector->itr_val = adapter->tx_itr_setting;
741 if (q_vector->itr_val && q_vector->itr_val <= 3)
742 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 743}
744
745/**
746 * igb_map_ring_to_vector - maps allocated queues to vectors
747 *
748 * This function maps the recently allocated queues to vectors.
749 **/
750static int igb_map_ring_to_vector(struct igb_adapter *adapter)
751{
752 int i;
753 int v_idx = 0;
754
755 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
756 (adapter->num_q_vectors < adapter->num_tx_queues))
757 return -ENOMEM;
758
759 if (adapter->num_q_vectors >=
760 (adapter->num_rx_queues + adapter->num_tx_queues)) {
761 for (i = 0; i < adapter->num_rx_queues; i++)
762 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
763 for (i = 0; i < adapter->num_tx_queues; i++)
764 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
765 } else {
766 for (i = 0; i < adapter->num_rx_queues; i++) {
767 if (i < adapter->num_tx_queues)
768 igb_map_tx_ring_to_vector(adapter, i, v_idx);
769 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
770 }
771 for (; i < adapter->num_tx_queues; i++)
772 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
773 }
774 return 0;
775}
776
777/**
778 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
779 *
780 * This function initializes the interrupts and allocates all of the queues.
781 **/
782static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
783{
784 struct pci_dev *pdev = adapter->pdev;
785 int err;
786
787 igb_set_interrupt_capability(adapter);
788
789 err = igb_alloc_q_vectors(adapter);
790 if (err) {
791 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
792 goto err_alloc_q_vectors;
793 }
794
795 err = igb_alloc_queues(adapter);
796 if (err) {
797 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
798 goto err_alloc_queues;
799 }
800
801 err = igb_map_ring_to_vector(adapter);
802 if (err) {
803 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
804 goto err_map_queues;
805 }
806
807
808 return 0;
809err_map_queues:
810 igb_free_queues(adapter);
811err_alloc_queues:
812 igb_free_q_vectors(adapter);
813err_alloc_q_vectors:
814 igb_reset_interrupt_capability(adapter);
815 return err;
816}
817
9d5c8243
AK		 818/**
819 * igb_request_irq - initialize interrupts
820 *
821 * Attempts to configure interrupts using the best available
822 * capabilities of the hardware and kernel.
823 **/
824static int igb_request_irq(struct igb_adapter *adapter)
825{
826 struct net_device *netdev = adapter->netdev;
047e0030 827 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 828 struct e1000_hw *hw = &adapter->hw;
829 int err = 0;
830
831 if (adapter->msix_entries) {
832 err = igb_request_msix(adapter);
844290e5 833 if (!err)
9d5c8243 834 goto request_done;
9d5c8243 835 /* fall back to MSI */
047e0030 836 igb_clear_interrupt_scheme(adapter);
9d5c8243 837 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 838 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 839 igb_free_all_tx_resources(adapter);
840 igb_free_all_rx_resources(adapter);
047e0030 841 adapter->num_tx_queues = 1;
9d5c8243 842 adapter->num_rx_queues = 1;
047e0030
AD		 843 adapter->num_q_vectors = 1;
844 err = igb_alloc_q_vectors(adapter);
845 if (err) {
846 dev_err(&pdev->dev,
847 "Unable to allocate memory for vectors\n");
848 goto request_done;
849 }
850 err = igb_alloc_queues(adapter);
851 if (err) {
852 dev_err(&pdev->dev,
853 "Unable to allocate memory for queues\n");
854 igb_free_q_vectors(adapter);
855 goto request_done;
856 }
857 igb_setup_all_tx_resources(adapter);
858 igb_setup_all_rx_resources(adapter);
844290e5 859 } else {
2d064c06
AD		 860 switch (hw->mac.type) {
861 case e1000_82575:
862 wr32(E1000_MSIXBM(0),
047e0030
AD		 863 (E1000_EICR_RX_QUEUE0 |
864 E1000_EICR_TX_QUEUE0 |
865 E1000_EIMS_OTHER));
2d064c06
AD		 866 break;
867 case e1000_82576:
868 wr32(E1000_IVAR0, E1000_IVAR_VALID);
869 break;
870 default:
871 break;
872 }
9d5c8243 873 }
844290e5 874
7dfc16fa 875 if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 876 err = request_irq(adapter->pdev->irq, &igb_intr_msi, 0,
047e0030 877 netdev->name, adapter);
9d5c8243
AK		 878 if (!err)
879 goto request_done;
047e0030 880
9d5c8243
AK		 881 /* fall back to legacy interrupts */
882 igb_reset_interrupt_capability(adapter);
7dfc16fa 883 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 884 }
885
886 err = request_irq(adapter->pdev->irq, &igb_intr, IRQF_SHARED,
047e0030 887 netdev->name, adapter);
9d5c8243 888
6cb5e577 889 if (err)
9d5c8243
AK		 890 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
891 err);
9d5c8243
AK		 892
893request_done:
894 return err;
895}
896
897static void igb_free_irq(struct igb_adapter *adapter)
898{
9d5c8243
AK		 899 if (adapter->msix_entries) {
900 int vector = 0, i;
901
047e0030 902 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 903
047e0030
AD		 904 for (i = 0; i < adapter->num_q_vectors; i++) {
905 struct igb_q_vector *q_vector = adapter->q_vector[i];
906 free_irq(adapter->msix_entries[vector++].vector,
907 q_vector);
908 }
909 } else {
910 free_irq(adapter->pdev->irq, adapter);
9d5c8243 911 }
9d5c8243
AK		 912}
913
914/**
915 * igb_irq_disable - Mask off interrupt generation on the NIC
916 * @adapter: board private structure
917 **/
918static void igb_irq_disable(struct igb_adapter *adapter)
919{
920 struct e1000_hw *hw = &adapter->hw;
921
25568a53
AD		 922 /*
923 * we need to be careful when disabling interrupts. The VFs are also
924 * mapped into these registers and so clearing the bits can cause
925 * issues on the VF drivers so we only need to clear what we set
926 */
9d5c8243 927 if (adapter->msix_entries) {
2dfd1212
AD		 928 u32 regval = rd32(E1000_EIAM);
929 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
930 wr32(E1000_EIMC, adapter->eims_enable_mask);
931 regval = rd32(E1000_EIAC);
932 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 933 }
844290e5
PW		 934
935 wr32(E1000_IAM, 0);
9d5c8243
AK		 936 wr32(E1000_IMC, ~0);
937 wrfl();
938 synchronize_irq(adapter->pdev->irq);
939}
940
941/**
942 * igb_irq_enable - Enable default interrupt generation settings
943 * @adapter: board private structure
944 **/
945static void igb_irq_enable(struct igb_adapter *adapter)
946{
947 struct e1000_hw *hw = &adapter->hw;
948
949 if (adapter->msix_entries) {
25568a53 950 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
2dfd1212
AD		 951 u32 regval = rd32(E1000_EIAC);
952 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
953 regval = rd32(E1000_EIAM);
954 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 955 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 956 if (adapter->vfs_allocated_count) {
4ae196df 957 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 958 ims |= E1000_IMS_VMMB;
959 }
960 wr32(E1000_IMS, ims);
844290e5
PW		 961 } else {
962 wr32(E1000_IMS, IMS_ENABLE_MASK);
963 wr32(E1000_IAM, IMS_ENABLE_MASK);
964 }
9d5c8243
AK		 965}
966
967static void igb_update_mng_vlan(struct igb_adapter *adapter)
968{
51466239 969 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 970 u16 vid = adapter->hw.mng_cookie.vlan_id;
971 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 972
973 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
974 /* add VID to filter table */
975 igb_vfta_set(hw, vid, true);
976 adapter->mng_vlan_id = vid;
977 } else {
978 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
979 }
980
981 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
982 (vid != old_vid) &&
983 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
984 /* remove VID from filter table */
985 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 986 }
987}
988
989/**
990 * igb_release_hw_control - release control of the h/w to f/w
991 * @adapter: address of board private structure
992 *
993 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
994 * For ASF and Pass Through versions of f/w this means that the
995 * driver is no longer loaded.
996 *
997 **/
998static void igb_release_hw_control(struct igb_adapter *adapter)
999{
1000 struct e1000_hw *hw = &adapter->hw;
1001 u32 ctrl_ext;
1002
1003 /* Let firmware take over control of h/w */
1004 ctrl_ext = rd32(E1000_CTRL_EXT);
1005 wr32(E1000_CTRL_EXT,
1006 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1007}
1008
1009
1010/**
1011 * igb_get_hw_control - get control of the h/w from f/w
1012 * @adapter: address of board private structure
1013 *
1014 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1015 * For ASF and Pass Through versions of f/w this means that
1016 * the driver is loaded.
1017 *
1018 **/
1019static void igb_get_hw_control(struct igb_adapter *adapter)
1020{
1021 struct e1000_hw *hw = &adapter->hw;
1022 u32 ctrl_ext;
1023
1024 /* Let firmware know the driver has taken over */
1025 ctrl_ext = rd32(E1000_CTRL_EXT);
1026 wr32(E1000_CTRL_EXT,
1027 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1028}
1029
9d5c8243
AK		 1030/**
1031 * igb_configure - configure the hardware for RX and TX
1032 * @adapter: private board structure
1033 **/
1034static void igb_configure(struct igb_adapter *adapter)
1035{
1036 struct net_device *netdev = adapter->netdev;
1037 int i;
1038
1039 igb_get_hw_control(adapter);
ff41f8dc 1040 igb_set_rx_mode(netdev);
9d5c8243
AK		 1041
1042 igb_restore_vlan(adapter);
9d5c8243 1043
85b430b4 1044 igb_setup_tctl(adapter);
06cf2666 1045 igb_setup_mrqc(adapter);
9d5c8243 1046 igb_setup_rctl(adapter);
85b430b4
AD		 1047
1048 igb_configure_tx(adapter);
9d5c8243 1049 igb_configure_rx(adapter);
662d7205
AD		 1050
1051 igb_rx_fifo_flush_82575(&adapter->hw);
1052
c493ea45 1053 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1054 * at least 1 descriptor unused to make sure
1055 * next_to_use != next_to_clean */
1056 for (i = 0; i < adapter->num_rx_queues; i++) {
1057 struct igb_ring *ring = &adapter->rx_ring[i];
c493ea45 1058 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
9d5c8243
AK		 1059 }
1060
1061
1062 adapter->tx_queue_len = netdev->tx_queue_len;
1063}
1064
1065
1066/**
1067 * igb_up - Open the interface and prepare it to handle traffic
1068 * @adapter: board private structure
1069 **/
1070
1071int igb_up(struct igb_adapter *adapter)
1072{
1073 struct e1000_hw *hw = &adapter->hw;
1074 int i;
1075
1076 /* hardware has been reset, we need to reload some things */
1077 igb_configure(adapter);
1078
1079 clear_bit(__IGB_DOWN, &adapter->state);
1080
047e0030
AD		 1081 for (i = 0; i < adapter->num_q_vectors; i++) {
1082 struct igb_q_vector *q_vector = adapter->q_vector[i];
1083 napi_enable(&q_vector->napi);
1084 }
844290e5 1085 if (adapter->msix_entries)
9d5c8243 1086 igb_configure_msix(adapter);
9d5c8243
AK		 1087
1088 /* Clear any pending interrupts. */
1089 rd32(E1000_ICR);
1090 igb_irq_enable(adapter);
1091
d4960307
AD		 1092 /* notify VFs that reset has been completed */
1093 if (adapter->vfs_allocated_count) {
1094 u32 reg_data = rd32(E1000_CTRL_EXT);
1095 reg_data |= E1000_CTRL_EXT_PFRSTD;
1096 wr32(E1000_CTRL_EXT, reg_data);
1097 }
1098
4cb9be7a
JB		 1099 netif_tx_start_all_queues(adapter->netdev);
1100
25568a53
AD		 1101 /* start the watchdog. */
1102 hw->mac.get_link_status = 1;
1103 schedule_work(&adapter->watchdog_task);
1104
9d5c8243
AK		 1105 return 0;
1106}
1107
1108void igb_down(struct igb_adapter *adapter)
1109{
1110 struct e1000_hw *hw = &adapter->hw;
1111 struct net_device *netdev = adapter->netdev;
1112 u32 tctl, rctl;
1113 int i;
1114
1115 /* signal that we're down so the interrupt handler does not
1116 * reschedule our watchdog timer */
1117 set_bit(__IGB_DOWN, &adapter->state);
1118
1119 /* disable receives in the hardware */
1120 rctl = rd32(E1000_RCTL);
1121 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1122 /* flush and sleep below */
1123
fd2ea0a7 1124 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1125
1126 /* disable transmits in the hardware */
1127 tctl = rd32(E1000_TCTL);
1128 tctl &= ~E1000_TCTL_EN;
1129 wr32(E1000_TCTL, tctl);
1130 /* flush both disables and wait for them to finish */
1131 wrfl();
1132 msleep(10);
1133
047e0030
AD		 1134 for (i = 0; i < adapter->num_q_vectors; i++) {
1135 struct igb_q_vector *q_vector = adapter->q_vector[i];
1136 napi_disable(&q_vector->napi);
1137 }
9d5c8243 1138
9d5c8243
AK		 1139 igb_irq_disable(adapter);
1140
1141 del_timer_sync(&adapter->watchdog_timer);
1142 del_timer_sync(&adapter->phy_info_timer);
1143
1144 netdev->tx_queue_len = adapter->tx_queue_len;
1145 netif_carrier_off(netdev);
04fe6358
AD		 1146
1147 /* record the stats before reset*/
1148 igb_update_stats(adapter);
1149
9d5c8243
AK		 1150 adapter->link_speed = 0;
1151 adapter->link_duplex = 0;
1152
3023682e
JK		 1153 if (!pci_channel_offline(adapter->pdev))
1154 igb_reset(adapter);
9d5c8243
AK		 1155 igb_clean_all_tx_rings(adapter);
1156 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1157#ifdef CONFIG_IGB_DCA
1158
1159 /* since we reset the hardware DCA settings were cleared */
1160 igb_setup_dca(adapter);
1161#endif
9d5c8243
AK		 1162}
1163
1164void igb_reinit_locked(struct igb_adapter *adapter)
1165{
1166 WARN_ON(in_interrupt());
1167 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1168 msleep(1);
1169 igb_down(adapter);
1170 igb_up(adapter);
1171 clear_bit(__IGB_RESETTING, &adapter->state);
1172}
1173
1174void igb_reset(struct igb_adapter *adapter)
1175{
1176 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1177 struct e1000_mac_info *mac = &hw->mac;
1178 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1179 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1180 u16 hwm;
1181
1182 /* Repartition Pba for greater than 9k mtu
1183 * To take effect CTRL.RST is required.
1184 */
fa4dfae0
AD		 1185 switch (mac->type) {
1186 case e1000_82576:
d249be54
AD		 1187 pba = rd32(E1000_RXPBS);
1188 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1189 break;
1190 case e1000_82575:
1191 default:
1192 pba = E1000_PBA_34K;
1193 break;
2d064c06 1194 }
9d5c8243 1195
2d064c06
AD		 1196 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1197 (mac->type < e1000_82576)) {
9d5c8243
AK		 1198 /* adjust PBA for jumbo frames */
1199 wr32(E1000_PBA, pba);
1200
1201 /* To maintain wire speed transmits, the Tx FIFO should be
1202 * large enough to accommodate two full transmit packets,
1203 * rounded up to the next 1KB and expressed in KB. Likewise,
1204 * the Rx FIFO should be large enough to accommodate at least
1205 * one full receive packet and is similarly rounded up and
1206 * expressed in KB. */
1207 pba = rd32(E1000_PBA);
1208 /* upper 16 bits has Tx packet buffer allocation size in KB */
1209 tx_space = pba >> 16;
1210 /* lower 16 bits has Rx packet buffer allocation size in KB */
1211 pba &= 0xffff;
1212 /* the tx fifo also stores 16 bytes of information about the tx
1213 * but don't include ethernet FCS because hardware appends it */
1214 min_tx_space = (adapter->max_frame_size +
85e8d004 1215 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1216 ETH_FCS_LEN) * 2;
1217 min_tx_space = ALIGN(min_tx_space, 1024);
1218 min_tx_space >>= 10;
1219 /* software strips receive CRC, so leave room for it */
1220 min_rx_space = adapter->max_frame_size;
1221 min_rx_space = ALIGN(min_rx_space, 1024);
1222 min_rx_space >>= 10;
1223
1224 /* If current Tx allocation is less than the min Tx FIFO size,
1225 * and the min Tx FIFO size is less than the current Rx FIFO
1226 * allocation, take space away from current Rx allocation */
1227 if (tx_space < min_tx_space &&
1228 ((min_tx_space - tx_space) < pba)) {
1229 pba = pba - (min_tx_space - tx_space);
1230
1231 /* if short on rx space, rx wins and must trump tx
1232 * adjustment */
1233 if (pba < min_rx_space)
1234 pba = min_rx_space;
1235 }
2d064c06 1236 wr32(E1000_PBA, pba);
9d5c8243 1237 }
9d5c8243
AK		 1238
1239 /* flow control settings */
1240 /* The high water mark must be low enough to fit one full frame
1241 * (or the size used for early receive) above it in the Rx FIFO.
1242 * Set it to the lower of:
1243 * - 90% of the Rx FIFO size, or
1244 * - the full Rx FIFO size minus one full frame */
1245 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1246 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1247
2d064c06
AD		 1248 if (mac->type < e1000_82576) {
1249 fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */
1250 fc->low_water = fc->high_water - 8;
1251 } else {
1252 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1253 fc->low_water = fc->high_water - 16;
1254 }
9d5c8243
AK		 1255 fc->pause_time = 0xFFFF;
1256 fc->send_xon = 1;
0cce119a 1257 fc->current_mode = fc->requested_mode;
9d5c8243 1258
4ae196df
AD		 1259 /* disable receive for all VFs and wait one second */
1260 if (adapter->vfs_allocated_count) {
1261 int i;
1262 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
f2ca0dbe 1263 adapter->vf_data[i].flags = 0;
4ae196df
AD		 1264
1265 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1266 igb_ping_all_vfs(adapter);
4ae196df
AD		 1267
1268 /* disable transmits and receives */
1269 wr32(E1000_VFRE, 0);
1270 wr32(E1000_VFTE, 0);
1271 }
1272
9d5c8243
AK		 1273 /* Allow time for pending master requests to run */
1274 adapter->hw.mac.ops.reset_hw(&adapter->hw);
1275 wr32(E1000_WUC, 0);
1276
1277 if (adapter->hw.mac.ops.init_hw(&adapter->hw))
1278 dev_err(&adapter->pdev->dev, "Hardware Error\n");
1279
1280 igb_update_mng_vlan(adapter);
1281
1282 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1283 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1284
1285 igb_reset_adaptive(&adapter->hw);
f5f4cf08 1286 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 1287}
1288
2e5c6922
SH		 1289static const struct net_device_ops igb_netdev_ops = {
1290 .ndo_open = igb_open,
1291 .ndo_stop = igb_close,
00829823 1292 .ndo_start_xmit = igb_xmit_frame_adv,
2e5c6922 1293 .ndo_get_stats = igb_get_stats,
ff41f8dc
AD		 1294 .ndo_set_rx_mode = igb_set_rx_mode,
1295 .ndo_set_multicast_list = igb_set_rx_mode,
2e5c6922
SH		 1296 .ndo_set_mac_address = igb_set_mac,
1297 .ndo_change_mtu = igb_change_mtu,
1298 .ndo_do_ioctl = igb_ioctl,
1299 .ndo_tx_timeout = igb_tx_timeout,
1300 .ndo_validate_addr = eth_validate_addr,
1301 .ndo_vlan_rx_register = igb_vlan_rx_register,
1302 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1303 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
1304#ifdef CONFIG_NET_POLL_CONTROLLER
1305 .ndo_poll_controller = igb_netpoll,
1306#endif
1307};
1308
9d5c8243
AK		 1309/**
1310 * igb_probe - Device Initialization Routine
1311 * @pdev: PCI device information struct
1312 * @ent: entry in igb_pci_tbl
1313 *
1314 * Returns 0 on success, negative on failure
1315 *
1316 * igb_probe initializes an adapter identified by a pci_dev structure.
1317 * The OS initialization, configuring of the adapter private structure,
1318 * and a hardware reset occur.
1319 **/
1320static int __devinit igb_probe(struct pci_dev *pdev,
1321 const struct pci_device_id *ent)
1322{
1323 struct net_device *netdev;
1324 struct igb_adapter *adapter;
1325 struct e1000_hw *hw;
4337e993
AD		 1326 u16 eeprom_data = 0;
1327 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1328 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1329 unsigned long mmio_start, mmio_len;
2d6a5e95 1330 int err, pci_using_dac;
9d5c8243
AK		 1331 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1332 u32 part_num;
1333
aed5dec3 1334 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1335 if (err)
1336 return err;
1337
1338 pci_using_dac = 0;
6a35528a 1339 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243 1340 if (!err) {
6a35528a 1341 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243
AK		 1342 if (!err)
1343 pci_using_dac = 1;
1344 } else {
284901a9 1345 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243 1346 if (err) {
284901a9 1347 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243
AK		 1348 if (err) {
1349 dev_err(&pdev->dev, "No usable DMA "
1350 "configuration, aborting\n");
1351 goto err_dma;
1352 }
1353 }
1354 }
1355
aed5dec3
AD		 1356 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1357 IORESOURCE_MEM),
1358 igb_driver_name);
9d5c8243
AK		 1359 if (err)
1360 goto err_pci_reg;
1361
19d5afd4 1362 pci_enable_pcie_error_reporting(pdev);
40a914fa 1363
9d5c8243 1364 pci_set_master(pdev);
c682fc23 1365 pci_save_state(pdev);
9d5c8243
AK		 1366
1367 err = -ENOMEM;
1bfaf07b
AD		 1368 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1369 IGB_ABS_MAX_TX_QUEUES);
9d5c8243
AK		 1370 if (!netdev)
1371 goto err_alloc_etherdev;
1372
1373 SET_NETDEV_DEV(netdev, &pdev->dev);
1374
1375 pci_set_drvdata(pdev, netdev);
1376 adapter = netdev_priv(netdev);
1377 adapter->netdev = netdev;
1378 adapter->pdev = pdev;
1379 hw = &adapter->hw;
1380 hw->back = adapter;
1381 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1382
1383 mmio_start = pci_resource_start(pdev, 0);
1384 mmio_len = pci_resource_len(pdev, 0);
1385
1386 err = -EIO;
28b0759c
AD		 1387 hw->hw_addr = ioremap(mmio_start, mmio_len);
1388 if (!hw->hw_addr)
9d5c8243
AK		 1389 goto err_ioremap;
1390
2e5c6922 1391 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1392 igb_set_ethtool_ops(netdev);
9d5c8243 1393 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1394
1395 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1396
1397 netdev->mem_start = mmio_start;
1398 netdev->mem_end = mmio_start + mmio_len;
1399
9d5c8243
AK		 1400 /* PCI config space info */
1401 hw->vendor_id = pdev->vendor;
1402 hw->device_id = pdev->device;
1403 hw->revision_id = pdev->revision;
1404 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1405 hw->subsystem_device_id = pdev->subsystem_device;
1406
1407 /* setup the private structure */
1408 hw->back = adapter;
1409 /* Copy the default MAC, PHY and NVM function pointers */
1410 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1411 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1412 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1413 /* Initialize skew-specific constants */
1414 err = ei->get_invariants(hw);
1415 if (err)
450c87c8 1416 goto err_sw_init;
9d5c8243 1417
450c87c8 1418 /* setup the private structure */
9d5c8243
AK		 1419 err = igb_sw_init(adapter);
1420 if (err)
1421 goto err_sw_init;
1422
1423 igb_get_bus_info_pcie(hw);
1424
1425 hw->phy.autoneg_wait_to_complete = false;
1426 hw->mac.adaptive_ifs = true;
1427
1428 /* Copper options */
1429 if (hw->phy.media_type == e1000_media_type_copper) {
1430 hw->phy.mdix = AUTO_ALL_MODES;
1431 hw->phy.disable_polarity_correction = false;
1432 hw->phy.ms_type = e1000_ms_hw_default;
1433 }
1434
1435 if (igb_check_reset_block(hw))
1436 dev_info(&pdev->dev,
1437 "PHY reset is blocked due to SOL/IDER session.\n");
1438
1439 netdev->features = NETIF_F_SG |
7d8eb29e 1440 NETIF_F_IP_CSUM |
9d5c8243
AK		 1441 NETIF_F_HW_VLAN_TX |
1442 NETIF_F_HW_VLAN_RX |
1443 NETIF_F_HW_VLAN_FILTER;
1444
7d8eb29e 1445 netdev->features |= NETIF_F_IPV6_CSUM;
9d5c8243 1446 netdev->features |= NETIF_F_TSO;
9d5c8243 1447 netdev->features |= NETIF_F_TSO6;
48f29ffc 1448
5c0999b7 1449 netdev->features |= NETIF_F_GRO;
d3352520 1450
48f29ffc
JK		 1451 netdev->vlan_features |= NETIF_F_TSO;
1452 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 1453 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 1454 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 1455 netdev->vlan_features |= NETIF_F_SG;
1456
9d5c8243
AK		 1457 if (pci_using_dac)
1458 netdev->features |= NETIF_F_HIGHDMA;
1459
b9473560
JB		 1460 if (adapter->hw.mac.type == e1000_82576)
1461 netdev->features |= NETIF_F_SCTP_CSUM;
1462
9d5c8243
AK		 1463 adapter->en_mng_pt = igb_enable_mng_pass_thru(&adapter->hw);
1464
1465 /* before reading the NVM, reset the controller to put the device in a
1466 * known good starting state */
1467 hw->mac.ops.reset_hw(hw);
1468
1469 /* make sure the NVM is good */
1470 if (igb_validate_nvm_checksum(hw) < 0) {
1471 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1472 err = -EIO;
1473 goto err_eeprom;
1474 }
1475
1476 /* copy the MAC address out of the NVM */
1477 if (hw->mac.ops.read_mac_addr(hw))
1478 dev_err(&pdev->dev, "NVM Read Error\n");
1479
1480 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1481 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1482
1483 if (!is_valid_ether_addr(netdev->perm_addr)) {
1484 dev_err(&pdev->dev, "Invalid MAC Address\n");
1485 err = -EIO;
1486 goto err_eeprom;
1487 }
1488
0e340485
AD		 1489 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1490 (unsigned long) adapter);
1491 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1492 (unsigned long) adapter);
9d5c8243
AK		 1493
1494 INIT_WORK(&adapter->reset_task, igb_reset_task);
1495 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1496
450c87c8 1497 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 1498 adapter->fc_autoneg = true;
1499 hw->mac.autoneg = true;
1500 hw->phy.autoneg_advertised = 0x2f;
1501
0cce119a
AD		 1502 hw->fc.requested_mode = e1000_fc_default;
1503 hw->fc.current_mode = e1000_fc_default;
9d5c8243 1504
9d5c8243
AK		 1505 igb_validate_mdi_setting(hw);
1506
9d5c8243
AK		 1507 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1508 * enable the ACPI Magic Packet filter
1509 */
1510
a2cf8b6c 1511 if (hw->bus.func == 0)
312c75ae 1512 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
a2cf8b6c
AD		 1513 else if (hw->bus.func == 1)
1514 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 1515
1516 if (eeprom_data & eeprom_apme_mask)
1517 adapter->eeprom_wol |= E1000_WUFC_MAG;
1518
1519 /* now that we have the eeprom settings, apply the special cases where
1520 * the eeprom may be wrong or the board simply won't support wake on
1521 * lan on a particular port */
1522 switch (pdev->device) {
1523 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1524 adapter->eeprom_wol = 0;
1525 break;
1526 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 1527 case E1000_DEV_ID_82576_FIBER:
1528 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 1529 /* Wake events only supported on port A for dual fiber
1530 * regardless of eeprom setting */
1531 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1532 adapter->eeprom_wol = 0;
1533 break;
c8ea5ea9
AD		 1534 case E1000_DEV_ID_82576_QUAD_COPPER:
1535 /* if quad port adapter, disable WoL on all but port A */
1536 if (global_quad_port_a != 0)
1537 adapter->eeprom_wol = 0;
1538 else
1539 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1540 /* Reset for multiple quad port adapters */
1541 if (++global_quad_port_a == 4)
1542 global_quad_port_a = 0;
1543 break;
9d5c8243
AK		 1544 }
1545
1546 /* initialize the wol settings based on the eeprom settings */
1547 adapter->wol = adapter->eeprom_wol;
e1b86d84 1548 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 1549
1550 /* reset the hardware with the new settings */
1551 igb_reset(adapter);
1552
1553 /* let the f/w know that the h/w is now under the control of the
1554 * driver. */
1555 igb_get_hw_control(adapter);
1556
9d5c8243
AK		 1557 strcpy(netdev->name, "eth%d");
1558 err = register_netdev(netdev);
1559 if (err)
1560 goto err_register;
1561
b168dfc5
JB		 1562 /* carrier off reporting is important to ethtool even BEFORE open */
1563 netif_carrier_off(netdev);
1564
421e02f0 1565#ifdef CONFIG_IGB_DCA
bbd98fe4 1566 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 1567 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 1568 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 1569 igb_setup_dca(adapter);
1570 }
fe4506b6 1571
38c845c7 1572#endif
38c845c7 1573
c5b9bd5e
AD		 1574 switch (hw->mac.type) {
1575 case e1000_82576:
1576 /*
1577 * Initialize hardware timer: we keep it running just in case
1578 * that some program needs it later on.
1579 */
1580 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1581 adapter->cycles.read = igb_read_clock;
1582 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1583 adapter->cycles.mult = 1;
1584 /**
1585 * Scale the NIC clock cycle by a large factor so that
1586 * relatively small clock corrections can be added or
1587 * substracted at each clock tick. The drawbacks of a large
1588 * factor are a) that the clock register overflows more quickly
1589 * (not such a big deal) and b) that the increment per tick has
1590 * to fit into 24 bits. As a result we need to use a shift of
1591 * 19 so we can fit a value of 16 into the TIMINCA register.
1592 */
1593 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
1594 wr32(E1000_TIMINCA,
1595 (1 << E1000_TIMINCA_16NS_SHIFT) |
1596 (16 << IGB_82576_TSYNC_SHIFT));
1597
1598 /* Set registers so that rollover occurs soon to test this. */
1599 wr32(E1000_SYSTIML, 0x00000000);
1600 wr32(E1000_SYSTIMH, 0xFF800000);
1601 wrfl();
1602
1603 timecounter_init(&adapter->clock,
1604 &adapter->cycles,
1605 ktime_to_ns(ktime_get_real()));
1606 /*
1607 * Synchronize our NIC clock against system wall clock. NIC
1608 * time stamp reading requires ~3us per sample, each sample
1609 * was pretty stable even under load => only require 10
1610 * samples for each offset comparison.
1611 */
1612 memset(&adapter->compare, 0, sizeof(adapter->compare));
1613 adapter->compare.source = &adapter->clock;
1614 adapter->compare.target = ktime_get_real;
1615 adapter->compare.num_samples = 10;
1616 timecompare_update(&adapter->compare, 0);
1617 break;
1618 case e1000_82575:
1619 /* 82575 does not support timesync */
1620 default:
1621 break;
38c845c7 1622 }
38c845c7 1623
9d5c8243
AK		 1624 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1625 /* print bus type/speed/width info */
7c510e4b 1626 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243
AK		 1627 netdev->name,
1628 ((hw->bus.speed == e1000_bus_speed_2500)
1629 ? "2.5Gb/s" : "unknown"),
59c3de89
AD		 1630 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1631 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1632 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1633 "unknown"),
7c510e4b 1634 netdev->dev_addr);
9d5c8243
AK		 1635
1636 igb_read_part_num(hw, &part_num);
1637 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1638 (part_num >> 8), (part_num & 0xff));
1639
1640 dev_info(&pdev->dev,
1641 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1642 adapter->msix_entries ? "MSI-X" :
7dfc16fa 1643 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243
AK		 1644 adapter->num_rx_queues, adapter->num_tx_queues);
1645
9d5c8243
AK		 1646 return 0;
1647
1648err_register:
1649 igb_release_hw_control(adapter);
1650err_eeprom:
1651 if (!igb_check_reset_block(hw))
f5f4cf08 1652 igb_reset_phy(hw);
9d5c8243
AK		 1653
1654 if (hw->flash_address)
1655 iounmap(hw->flash_address);
9d5c8243 1656err_sw_init:
047e0030 1657 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 1658 iounmap(hw->hw_addr);
1659err_ioremap:
1660 free_netdev(netdev);
1661err_alloc_etherdev:
aed5dec3
AD		 1662 pci_release_selected_regions(pdev, pci_select_bars(pdev,
1663 IORESOURCE_MEM));
9d5c8243
AK		 1664err_pci_reg:
1665err_dma:
1666 pci_disable_device(pdev);
1667 return err;
1668}
1669
1670/**
1671 * igb_remove - Device Removal Routine
1672 * @pdev: PCI device information struct
1673 *
1674 * igb_remove is called by the PCI subsystem to alert the driver
1675 * that it should release a PCI device. The could be caused by a
1676 * Hot-Plug event, or because the driver is going to be removed from
1677 * memory.
1678 **/
1679static void __devexit igb_remove(struct pci_dev *pdev)
1680{
1681 struct net_device *netdev = pci_get_drvdata(pdev);
1682 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 1683 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1684
1685 /* flush_scheduled work may reschedule our watchdog task, so
1686 * explicitly disable watchdog tasks from being rescheduled */
1687 set_bit(__IGB_DOWN, &adapter->state);
1688 del_timer_sync(&adapter->watchdog_timer);
1689 del_timer_sync(&adapter->phy_info_timer);
1690
1691 flush_scheduled_work();
1692
421e02f0 1693#ifdef CONFIG_IGB_DCA
7dfc16fa 1694 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 1695 dev_info(&pdev->dev, "DCA disabled\n");
1696 dca_remove_requester(&pdev->dev);
7dfc16fa 1697 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 1698 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 1699 }
1700#endif
1701
9d5c8243
AK		 1702 /* Release control of h/w to f/w. If f/w is AMT enabled, this
1703 * would have already happened in close and is redundant. */
1704 igb_release_hw_control(adapter);
1705
1706 unregister_netdev(netdev);
1707
f5f4cf08
AD		 1708 if (!igb_check_reset_block(&adapter->hw))
1709 igb_reset_phy(&adapter->hw);
9d5c8243 1710
047e0030 1711 igb_clear_interrupt_scheme(adapter);
9d5c8243 1712
37680117
AD		 1713#ifdef CONFIG_PCI_IOV
1714 /* reclaim resources allocated to VFs */
1715 if (adapter->vf_data) {
1716 /* disable iov and allow time for transactions to clear */
1717 pci_disable_sriov(pdev);
1718 msleep(500);
1719
1720 kfree(adapter->vf_data);
1721 adapter->vf_data = NULL;
1722 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1723 msleep(100);
1724 dev_info(&pdev->dev, "IOV Disabled\n");
1725 }
1726#endif
28b0759c
AD		 1727 iounmap(hw->hw_addr);
1728 if (hw->flash_address)
1729 iounmap(hw->flash_address);
aed5dec3
AD		 1730 pci_release_selected_regions(pdev, pci_select_bars(pdev,
1731 IORESOURCE_MEM));
9d5c8243
AK		 1732
1733 free_netdev(netdev);
1734
19d5afd4 1735 pci_disable_pcie_error_reporting(pdev);
40a914fa 1736
9d5c8243
AK		 1737 pci_disable_device(pdev);
1738}
1739
a6b623e0
AD		 1740/**
1741 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
1742 * @adapter: board private structure to initialize
1743 *
1744 * This function initializes the vf specific data storage and then attempts to
1745 * allocate the VFs. The reason for ordering it this way is because it is much
1746 * mor expensive time wise to disable SR-IOV than it is to allocate and free
1747 * the memory for the VFs.
1748 **/
1749static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
1750{
1751#ifdef CONFIG_PCI_IOV
1752 struct pci_dev *pdev = adapter->pdev;
1753
1754 if (adapter->vfs_allocated_count > 7)
1755 adapter->vfs_allocated_count = 7;
1756
1757 if (adapter->vfs_allocated_count) {
1758 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
1759 sizeof(struct vf_data_storage),
1760 GFP_KERNEL);
1761 /* if allocation failed then we do not support SR-IOV */
1762 if (!adapter->vf_data) {
1763 adapter->vfs_allocated_count = 0;
1764 dev_err(&pdev->dev, "Unable to allocate memory for VF "
1765 "Data Storage\n");
1766 }
1767 }
1768
1769 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
1770 kfree(adapter->vf_data);
1771 adapter->vf_data = NULL;
1772#endif /* CONFIG_PCI_IOV */
1773 adapter->vfs_allocated_count = 0;
1774#ifdef CONFIG_PCI_IOV
1775 } else {
1776 unsigned char mac_addr[ETH_ALEN];
1777 int i;
1778 dev_info(&pdev->dev, "%d vfs allocated\n",
1779 adapter->vfs_allocated_count);
1780 for (i = 0; i < adapter->vfs_allocated_count; i++) {
1781 random_ether_addr(mac_addr);
1782 igb_set_vf_mac(adapter, i, mac_addr);
1783 }
1784 }
1785#endif /* CONFIG_PCI_IOV */
1786}
1787
9d5c8243
AK		 1788/**
1789 * igb_sw_init - Initialize general software structures (struct igb_adapter)
1790 * @adapter: board private structure to initialize
1791 *
1792 * igb_sw_init initializes the Adapter private data structure.
1793 * Fields are initialized based on PCI device information and
1794 * OS network device settings (MTU size).
1795 **/
1796static int __devinit igb_sw_init(struct igb_adapter *adapter)
1797{
1798 struct e1000_hw *hw = &adapter->hw;
1799 struct net_device *netdev = adapter->netdev;
1800 struct pci_dev *pdev = adapter->pdev;
1801
1802 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
1803
68fd9910
AD		 1804 adapter->tx_ring_count = IGB_DEFAULT_TXD;
1805 adapter->rx_ring_count = IGB_DEFAULT_RXD;
4fc82adf
AD		 1806 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
1807 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
1808
9d5c8243
AK		 1809 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1810 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1811
a6b623e0
AD		 1812#ifdef CONFIG_PCI_IOV
1813 if (hw->mac.type == e1000_82576)
1814 adapter->vfs_allocated_count = max_vfs;
1815
1816#endif /* CONFIG_PCI_IOV */
1817 /* This call may decrease the number of queues */
047e0030 1818 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 1819 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1820 return -ENOMEM;
1821 }
1822
a6b623e0
AD		 1823 igb_probe_vfs(adapter);
1824
9d5c8243
AK		 1825 /* Explicitly disable IRQ since the NIC can be in any state. */
1826 igb_irq_disable(adapter);
1827
1828 set_bit(__IGB_DOWN, &adapter->state);
1829 return 0;
1830}
1831
1832/**
1833 * igb_open - Called when a network interface is made active
1834 * @netdev: network interface device structure
1835 *
1836 * Returns 0 on success, negative value on failure
1837 *
1838 * The open entry point is called when a network interface is made
1839 * active by the system (IFF_UP). At this point all resources needed
1840 * for transmit and receive operations are allocated, the interrupt
1841 * handler is registered with the OS, the watchdog timer is started,
1842 * and the stack is notified that the interface is ready.
1843 **/
1844static int igb_open(struct net_device *netdev)
1845{
1846 struct igb_adapter *adapter = netdev_priv(netdev);
1847 struct e1000_hw *hw = &adapter->hw;
1848 int err;
1849 int i;
1850
1851 /* disallow open during test */
1852 if (test_bit(__IGB_TESTING, &adapter->state))
1853 return -EBUSY;
1854
b168dfc5
JB		 1855 netif_carrier_off(netdev);
1856
9d5c8243
AK		 1857 /* allocate transmit descriptors */
1858 err = igb_setup_all_tx_resources(adapter);
1859 if (err)
1860 goto err_setup_tx;
1861
1862 /* allocate receive descriptors */
1863 err = igb_setup_all_rx_resources(adapter);
1864 if (err)
1865 goto err_setup_rx;
1866
1867 /* e1000_power_up_phy(adapter); */
1868
9d5c8243
AK		 1869 /* before we allocate an interrupt, we must be ready to handle it.
1870 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1871 * as soon as we call pci_request_irq, so we have to setup our
1872 * clean_rx handler before we do so. */
1873 igb_configure(adapter);
1874
1875 err = igb_request_irq(adapter);
1876 if (err)
1877 goto err_req_irq;
1878
1879 /* From here on the code is the same as igb_up() */
1880 clear_bit(__IGB_DOWN, &adapter->state);
1881
047e0030
AD		 1882 for (i = 0; i < adapter->num_q_vectors; i++) {
1883 struct igb_q_vector *q_vector = adapter->q_vector[i];
1884 napi_enable(&q_vector->napi);
1885 }
9d5c8243
AK		 1886
1887 /* Clear any pending interrupts. */
1888 rd32(E1000_ICR);
844290e5
PW		 1889
1890 igb_irq_enable(adapter);
1891
d4960307
AD		 1892 /* notify VFs that reset has been completed */
1893 if (adapter->vfs_allocated_count) {
1894 u32 reg_data = rd32(E1000_CTRL_EXT);
1895 reg_data |= E1000_CTRL_EXT_PFRSTD;
1896 wr32(E1000_CTRL_EXT, reg_data);
1897 }
1898
d55b53ff
JK		 1899 netif_tx_start_all_queues(netdev);
1900
25568a53
AD		 1901 /* start the watchdog. */
1902 hw->mac.get_link_status = 1;
1903 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 1904
1905 return 0;
1906
1907err_req_irq:
1908 igb_release_hw_control(adapter);
1909 /* e1000_power_down_phy(adapter); */
1910 igb_free_all_rx_resources(adapter);
1911err_setup_rx:
1912 igb_free_all_tx_resources(adapter);
1913err_setup_tx:
1914 igb_reset(adapter);
1915
1916 return err;
1917}
1918
1919/**
1920 * igb_close - Disables a network interface
1921 * @netdev: network interface device structure
1922 *
1923 * Returns 0, this is not allowed to fail
1924 *
1925 * The close entry point is called when an interface is de-activated
1926 * by the OS. The hardware is still under the driver's control, but
1927 * needs to be disabled. A global MAC reset is issued to stop the
1928 * hardware, and all transmit and receive resources are freed.
1929 **/
1930static int igb_close(struct net_device *netdev)
1931{
1932 struct igb_adapter *adapter = netdev_priv(netdev);
1933
1934 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
1935 igb_down(adapter);
1936
1937 igb_free_irq(adapter);
1938
1939 igb_free_all_tx_resources(adapter);
1940 igb_free_all_rx_resources(adapter);
1941
9d5c8243
AK		 1942 return 0;
1943}
1944
1945/**
1946 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 1947 * @tx_ring: tx descriptor ring (for a specific queue) to setup
1948 *
1949 * Return 0 on success, negative on failure
1950 **/
80785298 1951int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 1952{
80785298 1953 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 1954 int size;
1955
1956 size = sizeof(struct igb_buffer) * tx_ring->count;
1957 tx_ring->buffer_info = vmalloc(size);
1958 if (!tx_ring->buffer_info)
1959 goto err;
1960 memset(tx_ring->buffer_info, 0, size);
1961
1962 /* round up to nearest 4K */
85e8d004 1963 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 1964 tx_ring->size = ALIGN(tx_ring->size, 4096);
1965
439705e1
AD		 1966 tx_ring->desc = pci_alloc_consistent(pdev,
1967 tx_ring->size,
9d5c8243
AK		 1968 &tx_ring->dma);
1969
1970 if (!tx_ring->desc)
1971 goto err;
1972
9d5c8243
AK		 1973 tx_ring->next_to_use = 0;
1974 tx_ring->next_to_clean = 0;
9d5c8243
AK		 1975 return 0;
1976
1977err:
1978 vfree(tx_ring->buffer_info);
047e0030 1979 dev_err(&pdev->dev,
9d5c8243
AK		 1980 "Unable to allocate memory for the transmit descriptor ring\n");
1981 return -ENOMEM;
1982}
1983
1984/**
1985 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
1986 * (Descriptors) for all queues
1987 * @adapter: board private structure
1988 *
1989 * Return 0 on success, negative on failure
1990 **/
1991static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
1992{
439705e1 1993 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1994 int i, err = 0;
1995
1996 for (i = 0; i < adapter->num_tx_queues; i++) {
80785298 1997 err = igb_setup_tx_resources(&adapter->tx_ring[i]);
9d5c8243 1998 if (err) {
439705e1 1999 dev_err(&pdev->dev,
9d5c8243
AK		 2000 "Allocation for Tx Queue %u failed\n", i);
2001 for (i--; i >= 0; i--)
3b644cf6 2002 igb_free_tx_resources(&adapter->tx_ring[i]);
9d5c8243
AK		 2003 break;
2004 }
2005 }
2006
661086df 2007 for (i = 0; i < IGB_MAX_TX_QUEUES; i++) {
439705e1 2008 int r_idx = i % adapter->num_tx_queues;
661086df 2009 adapter->multi_tx_table[i] = &adapter->tx_ring[r_idx];
eebbbdba 2010 }
9d5c8243
AK		 2011 return err;
2012}
2013
2014/**
85b430b4
AD		 2015 * igb_setup_tctl - configure the transmit control registers
2016 * @adapter: Board private structure
9d5c8243 2017 **/
d7ee5b3a 2018void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2019{
9d5c8243
AK		 2020 struct e1000_hw *hw = &adapter->hw;
2021 u32 tctl;
9d5c8243 2022
85b430b4
AD		 2023 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2024 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2025
2026 /* Program the Transmit Control Register */
9d5c8243
AK		 2027 tctl = rd32(E1000_TCTL);
2028 tctl &= ~E1000_TCTL_CT;
2029 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2030 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2031
2032 igb_config_collision_dist(hw);
2033
9d5c8243
AK		 2034 /* Enable transmits */
2035 tctl |= E1000_TCTL_EN;
2036
2037 wr32(E1000_TCTL, tctl);
2038}
2039
85b430b4
AD		 2040/**
2041 * igb_configure_tx_ring - Configure transmit ring after Reset
2042 * @adapter: board private structure
2043 * @ring: tx ring to configure
2044 *
2045 * Configure a transmit ring after a reset.
2046 **/
d7ee5b3a
AD		 2047void igb_configure_tx_ring(struct igb_adapter *adapter,
2048 struct igb_ring *ring)
85b430b4
AD		 2049{
2050 struct e1000_hw *hw = &adapter->hw;
2051 u32 txdctl;
2052 u64 tdba = ring->dma;
2053 int reg_idx = ring->reg_idx;
2054
2055 /* disable the queue */
2056 txdctl = rd32(E1000_TXDCTL(reg_idx));
2057 wr32(E1000_TXDCTL(reg_idx),
2058 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2059 wrfl();
2060 mdelay(10);
2061
2062 wr32(E1000_TDLEN(reg_idx),
2063 ring->count * sizeof(union e1000_adv_tx_desc));
2064 wr32(E1000_TDBAL(reg_idx),
2065 tdba & 0x00000000ffffffffULL);
2066 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2067
fce99e34
AD		 2068 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2069 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2070 writel(0, ring->head);
2071 writel(0, ring->tail);
85b430b4
AD		 2072
2073 txdctl |= IGB_TX_PTHRESH;
2074 txdctl |= IGB_TX_HTHRESH << 8;
2075 txdctl |= IGB_TX_WTHRESH << 16;
2076
2077 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2078 wr32(E1000_TXDCTL(reg_idx), txdctl);
2079}
2080
2081/**
2082 * igb_configure_tx - Configure transmit Unit after Reset
2083 * @adapter: board private structure
2084 *
2085 * Configure the Tx unit of the MAC after a reset.
2086 **/
2087static void igb_configure_tx(struct igb_adapter *adapter)
2088{
2089 int i;
2090
2091 for (i = 0; i < adapter->num_tx_queues; i++)
2092 igb_configure_tx_ring(adapter, &adapter->tx_ring[i]);
85b430b4
AD		 2093}
2094
9d5c8243
AK		 2095/**
2096 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2097 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2098 *
2099 * Returns 0 on success, negative on failure
2100 **/
80785298 2101int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2102{
80785298 2103 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 2104 int size, desc_len;
2105
2106 size = sizeof(struct igb_buffer) * rx_ring->count;
2107 rx_ring->buffer_info = vmalloc(size);
2108 if (!rx_ring->buffer_info)
2109 goto err;
2110 memset(rx_ring->buffer_info, 0, size);
2111
2112 desc_len = sizeof(union e1000_adv_rx_desc);
2113
2114 /* Round up to nearest 4K */
2115 rx_ring->size = rx_ring->count * desc_len;
2116 rx_ring->size = ALIGN(rx_ring->size, 4096);
2117
2118 rx_ring->desc = pci_alloc_consistent(pdev, rx_ring->size,
2119 &rx_ring->dma);
2120
2121 if (!rx_ring->desc)
2122 goto err;
2123
2124 rx_ring->next_to_clean = 0;
2125 rx_ring->next_to_use = 0;
9d5c8243 2126
9d5c8243
AK		 2127 return 0;
2128
2129err:
2130 vfree(rx_ring->buffer_info);
439705e1 2131 rx_ring->buffer_info = NULL;
80785298 2132 dev_err(&pdev->dev, "Unable to allocate memory for "
9d5c8243
AK		 2133 "the receive descriptor ring\n");
2134 return -ENOMEM;
2135}
2136
2137/**
2138 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2139 * (Descriptors) for all queues
2140 * @adapter: board private structure
2141 *
2142 * Return 0 on success, negative on failure
2143 **/
2144static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2145{
439705e1 2146 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2147 int i, err = 0;
2148
2149 for (i = 0; i < adapter->num_rx_queues; i++) {
80785298 2150 err = igb_setup_rx_resources(&adapter->rx_ring[i]);
9d5c8243 2151 if (err) {
439705e1 2152 dev_err(&pdev->dev,
9d5c8243
AK		 2153 "Allocation for Rx Queue %u failed\n", i);
2154 for (i--; i >= 0; i--)
3b644cf6 2155 igb_free_rx_resources(&adapter->rx_ring[i]);
9d5c8243
AK		 2156 break;
2157 }
2158 }
2159
2160 return err;
2161}
2162
06cf2666
AD		 2163/**
2164 * igb_setup_mrqc - configure the multiple receive queue control registers
2165 * @adapter: Board private structure
2166 **/
2167static void igb_setup_mrqc(struct igb_adapter *adapter)
2168{
2169 struct e1000_hw *hw = &adapter->hw;
2170 u32 mrqc, rxcsum;
2171 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2172 union e1000_reta {
2173 u32 dword;
2174 u8 bytes[4];
2175 } reta;
2176 static const u8 rsshash[40] = {
2177 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2178 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2179 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2180 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2181
2182 /* Fill out hash function seeds */
2183 for (j = 0; j < 10; j++) {
2184 u32 rsskey = rsshash[(j * 4)];
2185 rsskey |= rsshash[(j * 4) + 1] << 8;
2186 rsskey |= rsshash[(j * 4) + 2] << 16;
2187 rsskey |= rsshash[(j * 4) + 3] << 24;
2188 array_wr32(E1000_RSSRK(0), j, rsskey);
2189 }
2190
2191 num_rx_queues = adapter->num_rx_queues;
2192
2193 if (adapter->vfs_allocated_count) {
2194 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2195 switch (hw->mac.type) {
2196 case e1000_82576:
2197 shift = 3;
2198 num_rx_queues = 2;
2199 break;
2200 case e1000_82575:
2201 shift = 2;
2202 shift2 = 6;
2203 default:
2204 break;
2205 }
2206 } else {
2207 if (hw->mac.type == e1000_82575)
2208 shift = 6;
2209 }
2210
2211 for (j = 0; j < (32 * 4); j++) {
2212 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2213 if (shift2)
2214 reta.bytes[j & 3] |= num_rx_queues << shift2;
2215 if ((j & 3) == 3)
2216 wr32(E1000_RETA(j >> 2), reta.dword);
2217 }
2218
2219 /*
2220 * Disable raw packet checksumming so that RSS hash is placed in
2221 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2222 * offloads as they are enabled by default
2223 */
2224 rxcsum = rd32(E1000_RXCSUM);
2225 rxcsum |= E1000_RXCSUM_PCSD;
2226
2227 if (adapter->hw.mac.type >= e1000_82576)
2228 /* Enable Receive Checksum Offload for SCTP */
2229 rxcsum |= E1000_RXCSUM_CRCOFL;
2230
2231 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2232 wr32(E1000_RXCSUM, rxcsum);
2233
2234 /* If VMDq is enabled then we set the appropriate mode for that, else
2235 * we default to RSS so that an RSS hash is calculated per packet even
2236 * if we are only using one queue */
2237 if (adapter->vfs_allocated_count) {
2238 if (hw->mac.type > e1000_82575) {
2239 /* Set the default pool for the PF's first queue */
2240 u32 vtctl = rd32(E1000_VT_CTL);
2241 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2242 E1000_VT_CTL_DISABLE_DEF_POOL);
2243 vtctl |= adapter->vfs_allocated_count <<
2244 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2245 wr32(E1000_VT_CTL, vtctl);
2246 }
2247 if (adapter->num_rx_queues > 1)
2248 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2249 else
2250 mrqc = E1000_MRQC_ENABLE_VMDQ;
2251 } else {
2252 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2253 }
2254 igb_vmm_control(adapter);
2255
2256 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2257 E1000_MRQC_RSS_FIELD_IPV4_TCP);
2258 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
2259 E1000_MRQC_RSS_FIELD_IPV6_TCP);
2260 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
2261 E1000_MRQC_RSS_FIELD_IPV6_UDP);
2262 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2263 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2264
2265 wr32(E1000_MRQC, mrqc);
2266}
2267
9d5c8243
AK		 2268/**
2269 * igb_setup_rctl - configure the receive control registers
2270 * @adapter: Board private structure
2271 **/
d7ee5b3a 2272void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2273{
2274 struct e1000_hw *hw = &adapter->hw;
2275 u32 rctl;
9d5c8243
AK		 2276
2277 rctl = rd32(E1000_RCTL);
2278
2279 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2280 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2281
69d728ba 2282 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2283 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2284
87cb7e8c
AK		 2285 /*
2286 * enable stripping of CRC. It's unlikely this will break BMC
2287 * redirection as it did with e1000. Newer features require
2288 * that the HW strips the CRC.
73cd78f1 2289 */
87cb7e8c 2290 rctl |= E1000_RCTL_SECRC;
9d5c8243 2291
9b07f3d3 2292 /*
ec54d7d6 2293 * disable store bad packets and clear size bits.
9b07f3d3 2294 */
ec54d7d6 2295 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2296
6ec43fe6
AD		 2297 /* enable LPE to prevent packets larger than max_frame_size */
2298 rctl |= E1000_RCTL_LPE;
9d5c8243 2299
952f72a8
AD		 2300 /* disable queue 0 to prevent tail write w/o re-config */
2301 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2302
e1739522
AD		 2303 /* Attention!!! For SR-IOV PF driver operations you must enable
2304 * queue drop for all VF and PF queues to prevent head of line blocking
2305 * if an un-trusted VF does not provide descriptors to hardware.
2306 */
2307 if (adapter->vfs_allocated_count) {
e1739522
AD		 2308 /* set all queue drop enable bits */
2309 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 2310 }
2311
9d5c8243
AK		 2312 wr32(E1000_RCTL, rctl);
2313}
2314
7d5753f0
AD		 2315static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2316 int vfn)
2317{
2318 struct e1000_hw *hw = &adapter->hw;
2319 u32 vmolr;
2320
2321 /* if it isn't the PF check to see if VFs are enabled and
2322 * increase the size to support vlan tags */
2323 if (vfn < adapter->vfs_allocated_count &&
2324 adapter->vf_data[vfn].vlans_enabled)
2325 size += VLAN_TAG_SIZE;
2326
2327 vmolr = rd32(E1000_VMOLR(vfn));
2328 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2329 vmolr |= size | E1000_VMOLR_LPE;
2330 wr32(E1000_VMOLR(vfn), vmolr);
2331
2332 return 0;
2333}
2334
e1739522
AD		 2335/**
2336 * igb_rlpml_set - set maximum receive packet size
2337 * @adapter: board private structure
2338 *
2339 * Configure maximum receivable packet size.
2340 **/
2341static void igb_rlpml_set(struct igb_adapter *adapter)
2342{
2343 u32 max_frame_size = adapter->max_frame_size;
2344 struct e1000_hw *hw = &adapter->hw;
2345 u16 pf_id = adapter->vfs_allocated_count;
2346
2347 if (adapter->vlgrp)
2348 max_frame_size += VLAN_TAG_SIZE;
2349
2350 /* if vfs are enabled we set RLPML to the largest possible request
2351 * size and set the VMOLR RLPML to the size we need */
2352 if (pf_id) {
2353 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
7d5753f0 2354 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 2355 }
2356
2357 wr32(E1000_RLPML, max_frame_size);
2358}
2359
7d5753f0
AD		 2360static inline void igb_set_vmolr(struct igb_adapter *adapter, int vfn)
2361{
2362 struct e1000_hw *hw = &adapter->hw;
2363 u32 vmolr;
2364
2365 /*
2366 * This register exists only on 82576 and newer so if we are older then
2367 * we should exit and do nothing
2368 */
2369 if (hw->mac.type < e1000_82576)
2370 return;
2371
2372 vmolr = rd32(E1000_VMOLR(vfn));
2373 vmolr |= E1000_VMOLR_AUPE | /* Accept untagged packets */
2374 E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2375
2376 /* clear all bits that might not be set */
2377 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2378
2379 if (adapter->num_rx_queues > 1 && vfn == adapter->vfs_allocated_count)
2380 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2381 /*
2382 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2383 * multicast packets
2384 */
2385 if (vfn <= adapter->vfs_allocated_count)
2386 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2387
2388 wr32(E1000_VMOLR(vfn), vmolr);
2389}
2390
85b430b4
AD		 2391/**
2392 * igb_configure_rx_ring - Configure a receive ring after Reset
2393 * @adapter: board private structure
2394 * @ring: receive ring to be configured
2395 *
2396 * Configure the Rx unit of the MAC after a reset.
2397 **/
d7ee5b3a
AD		 2398void igb_configure_rx_ring(struct igb_adapter *adapter,
2399 struct igb_ring *ring)
85b430b4
AD		 2400{
2401 struct e1000_hw *hw = &adapter->hw;
2402 u64 rdba = ring->dma;
2403 int reg_idx = ring->reg_idx;
952f72a8 2404 u32 srrctl, rxdctl;
85b430b4
AD		 2405
2406 /* disable the queue */
2407 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2408 wr32(E1000_RXDCTL(reg_idx),
2409 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2410
2411 /* Set DMA base address registers */
2412 wr32(E1000_RDBAL(reg_idx),
2413 rdba & 0x00000000ffffffffULL);
2414 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2415 wr32(E1000_RDLEN(reg_idx),
2416 ring->count * sizeof(union e1000_adv_rx_desc));
2417
2418 /* initialize head and tail */
fce99e34
AD		 2419 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2420 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2421 writel(0, ring->head);
2422 writel(0, ring->tail);
85b430b4 2423
952f72a8 2424 /* set descriptor configuration */
4c844851
AD		 2425 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2426 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
952f72a8
AD		 2427 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2428#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2429 srrctl |= IGB_RXBUFFER_16384 >>
2430 E1000_SRRCTL_BSIZEPKT_SHIFT;
2431#else
2432 srrctl |= (PAGE_SIZE / 2) >>
2433 E1000_SRRCTL_BSIZEPKT_SHIFT;
2434#endif
2435 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2436 } else {
4c844851 2437 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
952f72a8
AD		 2438 E1000_SRRCTL_BSIZEPKT_SHIFT;
2439 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2440 }
2441
2442 wr32(E1000_SRRCTL(reg_idx), srrctl);
2443
7d5753f0
AD		 2444 /* set filtering for VMDQ pools */
2445 igb_set_vmolr(adapter, reg_idx & 0x7);
2446
85b430b4
AD		 2447 /* enable receive descriptor fetching */
2448 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2449 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2450 rxdctl &= 0xFFF00000;
2451 rxdctl |= IGB_RX_PTHRESH;
2452 rxdctl |= IGB_RX_HTHRESH << 8;
2453 rxdctl |= IGB_RX_WTHRESH << 16;
2454 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2455}
2456
9d5c8243
AK		 2457/**
2458 * igb_configure_rx - Configure receive Unit after Reset
2459 * @adapter: board private structure
2460 *
2461 * Configure the Rx unit of the MAC after a reset.
2462 **/
2463static void igb_configure_rx(struct igb_adapter *adapter)
2464{
9107584e 2465 int i;
9d5c8243 2466
68d480c4
AD		 2467 /* set UTA to appropriate mode */
2468 igb_set_uta(adapter);
2469
26ad9178
AD		 2470 /* set the correct pool for the PF default MAC address in entry 0 */
2471 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2472 adapter->vfs_allocated_count);
2473
06cf2666
AD		 2474 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2475 * the Base and Length of the Rx Descriptor Ring */
2476 for (i = 0; i < adapter->num_rx_queues; i++)
2477 igb_configure_rx_ring(adapter, &adapter->rx_ring[i]);
9d5c8243
AK		 2478}
2479
2480/**
2481 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 2482 * @tx_ring: Tx descriptor ring for a specific queue
2483 *
2484 * Free all transmit software resources
2485 **/
68fd9910 2486void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2487{
3b644cf6 2488 igb_clean_tx_ring(tx_ring);
9d5c8243
AK		 2489
2490 vfree(tx_ring->buffer_info);
2491 tx_ring->buffer_info = NULL;
2492
439705e1
AD		 2493 /* if not set, then don't free */
2494 if (!tx_ring->desc)
2495 return;
2496
80785298
AD		 2497 pci_free_consistent(tx_ring->pdev, tx_ring->size,
2498 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 2499
2500 tx_ring->desc = NULL;
2501}
2502
2503/**
2504 * igb_free_all_tx_resources - Free Tx Resources for All Queues
2505 * @adapter: board private structure
2506 *
2507 * Free all transmit software resources
2508 **/
2509static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2510{
2511 int i;
2512
2513 for (i = 0; i < adapter->num_tx_queues; i++)
3b644cf6 2514 igb_free_tx_resources(&adapter->tx_ring[i]);
9d5c8243
AK		 2515}
2516
b1a436c3
AD		 2517void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
2518 struct igb_buffer *buffer_info)
9d5c8243 2519{
65689fef 2520 buffer_info->dma = 0;
9d5c8243 2521 if (buffer_info->skb) {
80785298
AD		 2522 skb_dma_unmap(&tx_ring->pdev->dev,
2523 buffer_info->skb,
65689fef 2524 DMA_TO_DEVICE);
9d5c8243
AK		 2525 dev_kfree_skb_any(buffer_info->skb);
2526 buffer_info->skb = NULL;
2527 }
2528 buffer_info->time_stamp = 0;
2529 /* buffer_info must be completely set up in the transmit path */
2530}
2531
2532/**
2533 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 2534 * @tx_ring: ring to be cleaned
2535 **/
3b644cf6 2536static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243
AK		 2537{
2538 struct igb_buffer *buffer_info;
2539 unsigned long size;
2540 unsigned int i;
2541
2542 if (!tx_ring->buffer_info)
2543 return;
2544 /* Free all the Tx ring sk_buffs */
2545
2546 for (i = 0; i < tx_ring->count; i++) {
2547 buffer_info = &tx_ring->buffer_info[i];
80785298 2548 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 2549 }
2550
2551 size = sizeof(struct igb_buffer) * tx_ring->count;
2552 memset(tx_ring->buffer_info, 0, size);
2553
2554 /* Zero out the descriptor ring */
9d5c8243
AK		 2555 memset(tx_ring->desc, 0, tx_ring->size);
2556
2557 tx_ring->next_to_use = 0;
2558 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2559}
2560
2561/**
2562 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
2563 * @adapter: board private structure
2564 **/
2565static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
2566{
2567 int i;
2568
2569 for (i = 0; i < adapter->num_tx_queues; i++)
3b644cf6 2570 igb_clean_tx_ring(&adapter->tx_ring[i]);
9d5c8243
AK		 2571}
2572
2573/**
2574 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 2575 * @rx_ring: ring to clean the resources from
2576 *
2577 * Free all receive software resources
2578 **/
68fd9910 2579void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2580{
3b644cf6 2581 igb_clean_rx_ring(rx_ring);
9d5c8243
AK		 2582
2583 vfree(rx_ring->buffer_info);
2584 rx_ring->buffer_info = NULL;
2585
439705e1
AD		 2586 /* if not set, then don't free */
2587 if (!rx_ring->desc)
2588 return;
2589
80785298
AD		 2590 pci_free_consistent(rx_ring->pdev, rx_ring->size,
2591 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 2592
2593 rx_ring->desc = NULL;
2594}
2595
2596/**
2597 * igb_free_all_rx_resources - Free Rx Resources for All Queues
2598 * @adapter: board private structure
2599 *
2600 * Free all receive software resources
2601 **/
2602static void igb_free_all_rx_resources(struct igb_adapter *adapter)
2603{
2604 int i;
2605
2606 for (i = 0; i < adapter->num_rx_queues; i++)
3b644cf6 2607 igb_free_rx_resources(&adapter->rx_ring[i]);
9d5c8243
AK		 2608}
2609
2610/**
2611 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 2612 * @rx_ring: ring to free buffers from
2613 **/
3b644cf6 2614static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243
AK		 2615{
2616 struct igb_buffer *buffer_info;
9d5c8243
AK		 2617 unsigned long size;
2618 unsigned int i;
2619
2620 if (!rx_ring->buffer_info)
2621 return;
439705e1 2622
9d5c8243
AK		 2623 /* Free all the Rx ring sk_buffs */
2624 for (i = 0; i < rx_ring->count; i++) {
2625 buffer_info = &rx_ring->buffer_info[i];
2626 if (buffer_info->dma) {
80785298
AD		 2627 pci_unmap_single(rx_ring->pdev,
2628 buffer_info->dma,
4c844851 2629 rx_ring->rx_buffer_len,
6ec43fe6 2630 PCI_DMA_FROMDEVICE);
9d5c8243
AK		 2631 buffer_info->dma = 0;
2632 }
2633
2634 if (buffer_info->skb) {
2635 dev_kfree_skb(buffer_info->skb);
2636 buffer_info->skb = NULL;
2637 }
6ec43fe6 2638 if (buffer_info->page_dma) {
80785298
AD		 2639 pci_unmap_page(rx_ring->pdev,
2640 buffer_info->page_dma,
6ec43fe6
AD		 2641 PAGE_SIZE / 2,
2642 PCI_DMA_FROMDEVICE);
2643 buffer_info->page_dma = 0;
2644 }
9d5c8243 2645 if (buffer_info->page) {
9d5c8243
AK		 2646 put_page(buffer_info->page);
2647 buffer_info->page = NULL;
bf36c1a0 2648 buffer_info->page_offset = 0;
9d5c8243
AK		 2649 }
2650 }
2651
9d5c8243
AK		 2652 size = sizeof(struct igb_buffer) * rx_ring->count;
2653 memset(rx_ring->buffer_info, 0, size);
2654
2655 /* Zero out the descriptor ring */
2656 memset(rx_ring->desc, 0, rx_ring->size);
2657
2658 rx_ring->next_to_clean = 0;
2659 rx_ring->next_to_use = 0;
9d5c8243
AK		 2660}
2661
2662/**
2663 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
2664 * @adapter: board private structure
2665 **/
2666static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
2667{
2668 int i;
2669
2670 for (i = 0; i < adapter->num_rx_queues; i++)
3b644cf6 2671 igb_clean_rx_ring(&adapter->rx_ring[i]);
9d5c8243
AK		 2672}
2673
2674/**
2675 * igb_set_mac - Change the Ethernet Address of the NIC
2676 * @netdev: network interface device structure
2677 * @p: pointer to an address structure
2678 *
2679 * Returns 0 on success, negative on failure
2680 **/
2681static int igb_set_mac(struct net_device *netdev, void *p)
2682{
2683 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 2684 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2685 struct sockaddr *addr = p;
2686
2687 if (!is_valid_ether_addr(addr->sa_data))
2688 return -EADDRNOTAVAIL;
2689
2690 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 2691 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 2692
26ad9178
AD		 2693 /* set the correct pool for the new PF MAC address in entry 0 */
2694 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
2695 adapter->vfs_allocated_count);
e1739522 2696
9d5c8243
AK		 2697 return 0;
2698}
2699
2700/**
68d480c4 2701 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 2702 * @netdev: network interface device structure
2703 *
68d480c4
AD		 2704 * Writes multicast address list to the MTA hash table.
2705 * Returns: -ENOMEM on failure
2706 * 0 on no addresses written
2707 * X on writing X addresses to MTA
9d5c8243 2708 **/
68d480c4 2709static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 2710{
2711 struct igb_adapter *adapter = netdev_priv(netdev);
2712 struct e1000_hw *hw = &adapter->hw;
ff41f8dc 2713 struct dev_mc_list *mc_ptr = netdev->mc_list;
68d480c4
AD		 2714 u8 *mta_list;
2715 u32 vmolr = 0;
9d5c8243
AK		 2716 int i;
2717
68d480c4
AD		 2718 if (!netdev->mc_count) {
2719 /* nothing to program, so clear mc list */
2720 igb_update_mc_addr_list(hw, NULL, 0);
2721 igb_restore_vf_multicasts(adapter);
2722 return 0;
2723 }
9d5c8243 2724
68d480c4
AD		 2725 mta_list = kzalloc(netdev->mc_count * 6, GFP_ATOMIC);
2726 if (!mta_list)
2727 return -ENOMEM;
ff41f8dc 2728
68d480c4
AD		 2729 /* set vmolr receive overflow multicast bit */
2730 vmolr |= E1000_VMOLR_ROMPE;
2731
2732 /* The shared function expects a packed array of only addresses. */
2733 mc_ptr = netdev->mc_list;
2734
2735 for (i = 0; i < netdev->mc_count; i++) {
2736 if (!mc_ptr)
2737 break;
2738 memcpy(mta_list + (i*ETH_ALEN), mc_ptr->dmi_addr, ETH_ALEN);
2739 mc_ptr = mc_ptr->next;
746b9f02 2740 }
68d480c4
AD		 2741 igb_update_mc_addr_list(hw, mta_list, i);
2742 kfree(mta_list);
2743
2744 return netdev->mc_count;
2745}
2746
2747/**
2748 * igb_write_uc_addr_list - write unicast addresses to RAR table
2749 * @netdev: network interface device structure
2750 *
2751 * Writes unicast address list to the RAR table.
2752 * Returns: -ENOMEM on failure/insufficient address space
2753 * 0 on no addresses written
2754 * X on writing X addresses to the RAR table
2755 **/
2756static int igb_write_uc_addr_list(struct net_device *netdev)
2757{
2758 struct igb_adapter *adapter = netdev_priv(netdev);
2759 struct e1000_hw *hw = &adapter->hw;
2760 unsigned int vfn = adapter->vfs_allocated_count;
2761 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
2762 int count = 0;
2763
2764 /* return ENOMEM indicating insufficient memory for addresses */
2765 if (netdev->uc.count > rar_entries)
2766 return -ENOMEM;
9d5c8243 2767
ff41f8dc
AD		 2768 if (netdev->uc.count && rar_entries) {
2769 struct netdev_hw_addr *ha;
2770 list_for_each_entry(ha, &netdev->uc.list, list) {
2771 if (!rar_entries)
2772 break;
26ad9178
AD		 2773 igb_rar_set_qsel(adapter, ha->addr,
2774 rar_entries--,
68d480c4
AD		 2775 vfn);
2776 count++;
ff41f8dc
AD		 2777 }
2778 }
2779 /* write the addresses in reverse order to avoid write combining */
2780 for (; rar_entries > 0 ; rar_entries--) {
2781 wr32(E1000_RAH(rar_entries), 0);
2782 wr32(E1000_RAL(rar_entries), 0);
2783 }
2784 wrfl();
2785
68d480c4
AD		 2786 return count;
2787}
2788
2789/**
2790 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2791 * @netdev: network interface device structure
2792 *
2793 * The set_rx_mode entry point is called whenever the unicast or multicast
2794 * address lists or the network interface flags are updated. This routine is
2795 * responsible for configuring the hardware for proper unicast, multicast,
2796 * promiscuous mode, and all-multi behavior.
2797 **/
2798static void igb_set_rx_mode(struct net_device *netdev)
2799{
2800 struct igb_adapter *adapter = netdev_priv(netdev);
2801 struct e1000_hw *hw = &adapter->hw;
2802 unsigned int vfn = adapter->vfs_allocated_count;
2803 u32 rctl, vmolr = 0;
2804 int count;
2805
2806 /* Check for Promiscuous and All Multicast modes */
2807 rctl = rd32(E1000_RCTL);
2808
2809 /* clear the effected bits */
2810 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
2811
2812 if (netdev->flags & IFF_PROMISC) {
2813 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2814 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
2815 } else {
2816 if (netdev->flags & IFF_ALLMULTI) {
2817 rctl |= E1000_RCTL_MPE;
2818 vmolr |= E1000_VMOLR_MPME;
2819 } else {
2820 /*
2821 * Write addresses to the MTA, if the attempt fails
2822 * then we should just turn on promiscous mode so
2823 * that we can at least receive multicast traffic
2824 */
2825 count = igb_write_mc_addr_list(netdev);
2826 if (count < 0) {
2827 rctl |= E1000_RCTL_MPE;
2828 vmolr |= E1000_VMOLR_MPME;
2829 } else if (count) {
2830 vmolr |= E1000_VMOLR_ROMPE;
2831 }
2832 }
2833 /*
2834 * Write addresses to available RAR registers, if there is not
2835 * sufficient space to store all the addresses then enable
2836 * unicast promiscous mode
2837 */
2838 count = igb_write_uc_addr_list(netdev);
2839 if (count < 0) {
2840 rctl |= E1000_RCTL_UPE;
2841 vmolr |= E1000_VMOLR_ROPE;
2842 }
2843 rctl |= E1000_RCTL_VFE;
28fc06f5 2844 }
68d480c4 2845 wr32(E1000_RCTL, rctl);
28fc06f5 2846
68d480c4
AD		 2847 /*
2848 * In order to support SR-IOV and eventually VMDq it is necessary to set
2849 * the VMOLR to enable the appropriate modes. Without this workaround
2850 * we will have issues with VLAN tag stripping not being done for frames
2851 * that are only arriving because we are the default pool
2852 */
2853 if (hw->mac.type < e1000_82576)
28fc06f5 2854 return;
9d5c8243 2855
68d480c4
AD		 2856 vmolr |= rd32(E1000_VMOLR(vfn)) &
2857 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
2858 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 2859 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 2860}
2861
2862/* Need to wait a few seconds after link up to get diagnostic information from
2863 * the phy */
2864static void igb_update_phy_info(unsigned long data)
2865{
2866 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 2867 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 2868}
2869
4d6b725e
AD		 2870/**
2871 * igb_has_link - check shared code for link and determine up/down
2872 * @adapter: pointer to driver private info
2873 **/
2874static bool igb_has_link(struct igb_adapter *adapter)
2875{
2876 struct e1000_hw *hw = &adapter->hw;
2877 bool link_active = false;
2878 s32 ret_val = 0;
2879
2880 /* get_link_status is set on LSC (link status) interrupt or
2881 * rx sequence error interrupt. get_link_status will stay
2882 * false until the e1000_check_for_link establishes link
2883 * for copper adapters ONLY
2884 */
2885 switch (hw->phy.media_type) {
2886 case e1000_media_type_copper:
2887 if (hw->mac.get_link_status) {
2888 ret_val = hw->mac.ops.check_for_link(hw);
2889 link_active = !hw->mac.get_link_status;
2890 } else {
2891 link_active = true;
2892 }
2893 break;
4d6b725e
AD		 2894 case e1000_media_type_internal_serdes:
2895 ret_val = hw->mac.ops.check_for_link(hw);
2896 link_active = hw->mac.serdes_has_link;
2897 break;
2898 default:
2899 case e1000_media_type_unknown:
2900 break;
2901 }
2902
2903 return link_active;
2904}
2905
9d5c8243
AK		 2906/**
2907 * igb_watchdog - Timer Call-back
2908 * @data: pointer to adapter cast into an unsigned long
2909 **/
2910static void igb_watchdog(unsigned long data)
2911{
2912 struct igb_adapter *adapter = (struct igb_adapter *)data;
2913 /* Do the rest outside of interrupt context */
2914 schedule_work(&adapter->watchdog_task);
2915}
2916
2917static void igb_watchdog_task(struct work_struct *work)
2918{
2919 struct igb_adapter *adapter = container_of(work,
2920 struct igb_adapter, watchdog_task);
2921 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2922 struct net_device *netdev = adapter->netdev;
2923 struct igb_ring *tx_ring = adapter->tx_ring;
9d5c8243 2924 u32 link;
7a6ea550 2925 int i;
9d5c8243 2926
4d6b725e 2927 link = igb_has_link(adapter);
9d5c8243
AK		 2928 if (link) {
2929 if (!netif_carrier_ok(netdev)) {
2930 u32 ctrl;
2931 hw->mac.ops.get_speed_and_duplex(&adapter->hw,
2932 &adapter->link_speed,
2933 &adapter->link_duplex);
2934
2935 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 2936 /* Links status message must follow this format */
2937 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 2938 "Flow Control: %s\n",
527d47c1 2939 netdev->name,
9d5c8243
AK		 2940 adapter->link_speed,
2941 adapter->link_duplex == FULL_DUPLEX ?
2942 "Full Duplex" : "Half Duplex",
2943 ((ctrl & E1000_CTRL_TFCE) && (ctrl &
2944 E1000_CTRL_RFCE)) ? "RX/TX" : ((ctrl &
2945 E1000_CTRL_RFCE) ? "RX" : ((ctrl &
2946 E1000_CTRL_TFCE) ? "TX" : "None")));
2947
2948 /* tweak tx_queue_len according to speed/duplex and
2949 * adjust the timeout factor */
2950 netdev->tx_queue_len = adapter->tx_queue_len;
2951 adapter->tx_timeout_factor = 1;
2952 switch (adapter->link_speed) {
2953 case SPEED_10:
2954 netdev->tx_queue_len = 10;
2955 adapter->tx_timeout_factor = 14;
2956 break;
2957 case SPEED_100:
2958 netdev->tx_queue_len = 100;
2959 /* maybe add some timeout factor ? */
2960 break;
2961 }
2962
2963 netif_carrier_on(netdev);
9d5c8243 2964
4ae196df
AD		 2965 igb_ping_all_vfs(adapter);
2966
4b1a9877 2967 /* link state has changed, schedule phy info update */
9d5c8243
AK		 2968 if (!test_bit(__IGB_DOWN, &adapter->state))
2969 mod_timer(&adapter->phy_info_timer,
2970 round_jiffies(jiffies + 2 * HZ));
2971 }
2972 } else {
2973 if (netif_carrier_ok(netdev)) {
2974 adapter->link_speed = 0;
2975 adapter->link_duplex = 0;
527d47c1
AD		 2976 /* Links status message must follow this format */
2977 printk(KERN_INFO "igb: %s NIC Link is Down\n",
2978 netdev->name);
9d5c8243 2979 netif_carrier_off(netdev);
4b1a9877 2980
4ae196df
AD		 2981 igb_ping_all_vfs(adapter);
2982
4b1a9877 2983 /* link state has changed, schedule phy info update */
9d5c8243
AK		 2984 if (!test_bit(__IGB_DOWN, &adapter->state))
2985 mod_timer(&adapter->phy_info_timer,
2986 round_jiffies(jiffies + 2 * HZ));
2987 }
2988 }
2989
9d5c8243 2990 igb_update_stats(adapter);
645a3abd 2991 igb_update_adaptive(hw);
9d5c8243
AK		 2992
2993 if (!netif_carrier_ok(netdev)) {
c493ea45 2994 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
9d5c8243
AK		 2995 /* We've lost link, so the controller stops DMA,
2996 * but we've got queued Tx work that's never going
2997 * to get done, so reset controller to flush Tx.
2998 * (Do the reset outside of interrupt context). */
2999 adapter->tx_timeout_count++;
3000 schedule_work(&adapter->reset_task);
c2d5ab49
JB		 3001 /* return immediately since reset is imminent */
3002 return;
9d5c8243
AK		 3003 }
3004 }
3005
f7ba205e
AD		 3006 /* Force detection of hung controller every watchdog period */
3007 for (i = 0; i < adapter->num_tx_queues; i++)
3008 adapter->tx_ring[i].detect_tx_hung = true;
3009
9d5c8243 3010 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3011 if (adapter->msix_entries) {
047e0030
AD		 3012 u32 eics = 0;
3013 for (i = 0; i < adapter->num_q_vectors; i++) {
3014 struct igb_q_vector *q_vector = adapter->q_vector[i];
3015 eics |= q_vector->eims_value;
3016 }
7a6ea550
AD		 3017 wr32(E1000_EICS, eics);
3018 } else {
3019 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3020 }
9d5c8243 3021
9d5c8243
AK		 3022 /* Reset the timer */
3023 if (!test_bit(__IGB_DOWN, &adapter->state))
3024 mod_timer(&adapter->watchdog_timer,
3025 round_jiffies(jiffies + 2 * HZ));
3026}
3027
3028enum latency_range {
3029 lowest_latency = 0,
3030 low_latency = 1,
3031 bulk_latency = 2,
3032 latency_invalid = 255
3033};
3034
6eb5a7f1
AD		 3035/**
3036 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3037 *
3038 * Stores a new ITR value based on strictly on packet size. This
3039 * algorithm is less sophisticated than that used in igb_update_itr,
3040 * due to the difficulty of synchronizing statistics across multiple
3041 * receive rings. The divisors and thresholds used by this fuction
3042 * were determined based on theoretical maximum wire speed and testing
3043 * data, in order to minimize response time while increasing bulk
3044 * throughput.
3045 * This functionality is controlled by the InterruptThrottleRate module
3046 * parameter (see igb_param.c)
3047 * NOTE: This function is called only when operating in a multiqueue
3048 * receive environment.
047e0030 3049 * @q_vector: pointer to q_vector
6eb5a7f1 3050 **/
047e0030 3051static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3052{
047e0030 3053 int new_val = q_vector->itr_val;
6eb5a7f1 3054 int avg_wire_size = 0;
047e0030 3055 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 3056
6eb5a7f1
AD		 3057 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3058 * ints/sec - ITR timer value of 120 ticks.
3059 */
3060 if (adapter->link_speed != SPEED_1000) {
047e0030 3061 new_val = 976;
6eb5a7f1 3062 goto set_itr_val;
9d5c8243 3063 }
047e0030
AD		 3064
3065 if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3066 struct igb_ring *ring = q_vector->rx_ring;
3067 avg_wire_size = ring->total_bytes / ring->total_packets;
3068 }
3069
3070 if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3071 struct igb_ring *ring = q_vector->tx_ring;
3072 avg_wire_size = max_t(u32, avg_wire_size,
3073 (ring->total_bytes /
3074 ring->total_packets));
3075 }
3076
3077 /* if avg_wire_size isn't set no work was done */
3078 if (!avg_wire_size)
3079 goto clear_counts;
9d5c8243 3080
6eb5a7f1
AD		 3081 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3082 avg_wire_size += 24;
3083
3084 /* Don't starve jumbo frames */
3085 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3086
6eb5a7f1
AD		 3087 /* Give a little boost to mid-size frames */
3088 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3089 new_val = avg_wire_size / 3;
3090 else
3091 new_val = avg_wire_size / 2;
9d5c8243 3092
6eb5a7f1 3093set_itr_val:
047e0030
AD		 3094 if (new_val != q_vector->itr_val) {
3095 q_vector->itr_val = new_val;
3096 q_vector->set_itr = 1;
9d5c8243 3097 }
6eb5a7f1 3098clear_counts:
047e0030
AD		 3099 if (q_vector->rx_ring) {
3100 q_vector->rx_ring->total_bytes = 0;
3101 q_vector->rx_ring->total_packets = 0;
3102 }
3103 if (q_vector->tx_ring) {
3104 q_vector->tx_ring->total_bytes = 0;
3105 q_vector->tx_ring->total_packets = 0;
3106 }
9d5c8243
AK		 3107}
3108
3109/**
3110 * igb_update_itr - update the dynamic ITR value based on statistics
3111 * Stores a new ITR value based on packets and byte
3112 * counts during the last interrupt. The advantage of per interrupt
3113 * computation is faster updates and more accurate ITR for the current
3114 * traffic pattern. Constants in this function were computed
3115 * based on theoretical maximum wire speed and thresholds were set based
3116 * on testing data as well as attempting to minimize response time
3117 * while increasing bulk throughput.
3118 * this functionality is controlled by the InterruptThrottleRate module
3119 * parameter (see igb_param.c)
3120 * NOTE: These calculations are only valid when operating in a single-
3121 * queue environment.
3122 * @adapter: pointer to adapter
047e0030 3123 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3124 * @packets: the number of packets during this measurement interval
3125 * @bytes: the number of bytes during this measurement interval
3126 **/
3127static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3128 int packets, int bytes)
3129{
3130 unsigned int retval = itr_setting;
3131
3132 if (packets == 0)
3133 goto update_itr_done;
3134
3135 switch (itr_setting) {
3136 case lowest_latency:
3137 /* handle TSO and jumbo frames */
3138 if (bytes/packets > 8000)
3139 retval = bulk_latency;
3140 else if ((packets < 5) && (bytes > 512))
3141 retval = low_latency;
3142 break;
3143 case low_latency: /* 50 usec aka 20000 ints/s */
3144 if (bytes > 10000) {
3145 /* this if handles the TSO accounting */
3146 if (bytes/packets > 8000) {
3147 retval = bulk_latency;
3148 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3149 retval = bulk_latency;
3150 } else if ((packets > 35)) {
3151 retval = lowest_latency;
3152 }
3153 } else if (bytes/packets > 2000) {
3154 retval = bulk_latency;
3155 } else if (packets <= 2 && bytes < 512) {
3156 retval = lowest_latency;
3157 }
3158 break;
3159 case bulk_latency: /* 250 usec aka 4000 ints/s */
3160 if (bytes > 25000) {
3161 if (packets > 35)
3162 retval = low_latency;
1e5c3d21 3163 } else if (bytes < 1500) {
9d5c8243
AK		 3164 retval = low_latency;
3165 }
3166 break;
3167 }
3168
3169update_itr_done:
3170 return retval;
3171}
3172
6eb5a7f1 3173static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3174{
047e0030 3175 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3176 u16 current_itr;
047e0030 3177 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3178
3179 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3180 if (adapter->link_speed != SPEED_1000) {
3181 current_itr = 0;
3182 new_itr = 4000;
3183 goto set_itr_now;
3184 }
3185
3186 adapter->rx_itr = igb_update_itr(adapter,
3187 adapter->rx_itr,
3188 adapter->rx_ring->total_packets,
3189 adapter->rx_ring->total_bytes);
9d5c8243 3190
047e0030
AD		 3191 adapter->tx_itr = igb_update_itr(adapter,
3192 adapter->tx_itr,
3193 adapter->tx_ring->total_packets,
3194 adapter->tx_ring->total_bytes);
3195 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3196
6eb5a7f1 3197 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4fc82adf 3198 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3199 current_itr = low_latency;
3200
9d5c8243
AK		 3201 switch (current_itr) {
3202 /* counts and packets in update_itr are dependent on these numbers */
3203 case lowest_latency:
78b1f607 3204 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3205 break;
3206 case low_latency:
78b1f607 3207 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3208 break;
3209 case bulk_latency:
78b1f607 3210 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3211 break;
3212 default:
3213 break;
3214 }
3215
3216set_itr_now:
6eb5a7f1
AD		 3217 adapter->rx_ring->total_bytes = 0;
3218 adapter->rx_ring->total_packets = 0;
047e0030
AD		 3219 adapter->tx_ring->total_bytes = 0;
3220 adapter->tx_ring->total_packets = 0;
6eb5a7f1 3221
047e0030 3222 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3223 /* this attempts to bias the interrupt rate towards Bulk
3224 * by adding intermediate steps when interrupt rate is
3225 * increasing */
047e0030
AD		 3226 new_itr = new_itr > q_vector->itr_val ?
3227 max((new_itr * q_vector->itr_val) /
3228 (new_itr + (q_vector->itr_val >> 2)),
3229 new_itr) :
9d5c8243
AK		 3230 new_itr;
3231 /* Don't write the value here; it resets the adapter's
3232 * internal timer, and causes us to delay far longer than
3233 * we should between interrupts. Instead, we write the ITR
3234 * value at the beginning of the next interrupt so the timing
3235 * ends up being correct.
3236 */
047e0030
AD		 3237 q_vector->itr_val = new_itr;
3238 q_vector->set_itr = 1;
9d5c8243
AK		 3239 }
3240
3241 return;
3242}
3243
9d5c8243
AK		 3244#define IGB_TX_FLAGS_CSUM 0x00000001
3245#define IGB_TX_FLAGS_VLAN 0x00000002
3246#define IGB_TX_FLAGS_TSO 0x00000004
3247#define IGB_TX_FLAGS_IPV4 0x00000008
cdfd01fc
AD		 3248#define IGB_TX_FLAGS_TSTAMP 0x00000010
3249#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3250#define IGB_TX_FLAGS_VLAN_SHIFT 16
9d5c8243 3251
85ad76b2 3252static inline int igb_tso_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3253 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3254{
3255 struct e1000_adv_tx_context_desc *context_desc;
3256 unsigned int i;
3257 int err;
3258 struct igb_buffer *buffer_info;
3259 u32 info = 0, tu_cmd = 0;
3260 u32 mss_l4len_idx, l4len;
3261 *hdr_len = 0;
3262
3263 if (skb_header_cloned(skb)) {
3264 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3265 if (err)
3266 return err;
3267 }
3268
3269 l4len = tcp_hdrlen(skb);
3270 *hdr_len += l4len;
3271
3272 if (skb->protocol == htons(ETH_P_IP)) {
3273 struct iphdr *iph = ip_hdr(skb);
3274 iph->tot_len = 0;
3275 iph->check = 0;
3276 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3277 iph->daddr, 0,
3278 IPPROTO_TCP,
3279 0);
3280 } else if (skb_shinfo(skb)->gso_type == SKB_GSO_TCPV6) {
3281 ipv6_hdr(skb)->payload_len = 0;
3282 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3283 &ipv6_hdr(skb)->daddr,
3284 0, IPPROTO_TCP, 0);
3285 }
3286
3287 i = tx_ring->next_to_use;
3288
3289 buffer_info = &tx_ring->buffer_info[i];
3290 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3291 /* VLAN MACLEN IPLEN */
3292 if (tx_flags & IGB_TX_FLAGS_VLAN)
3293 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3294 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3295 *hdr_len += skb_network_offset(skb);
3296 info |= skb_network_header_len(skb);
3297 *hdr_len += skb_network_header_len(skb);
3298 context_desc->vlan_macip_lens = cpu_to_le32(info);
3299
3300 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3301 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3302
3303 if (skb->protocol == htons(ETH_P_IP))
3304 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3305 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3306
3307 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3308
3309 /* MSS L4LEN IDX */
3310 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3311 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3312
73cd78f1 3313 /* For 82575, context index must be unique per ring. */
85ad76b2
AD		 3314 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3315 mss_l4len_idx |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3316
3317 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3318 context_desc->seqnum_seed = 0;
3319
3320 buffer_info->time_stamp = jiffies;
0e014cb1 3321 buffer_info->next_to_watch = i;
9d5c8243
AK		 3322 buffer_info->dma = 0;
3323 i++;
3324 if (i == tx_ring->count)
3325 i = 0;
3326
3327 tx_ring->next_to_use = i;
3328
3329 return true;
3330}
3331
85ad76b2
AD		 3332static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3333 struct sk_buff *skb, u32 tx_flags)
9d5c8243
AK		 3334{
3335 struct e1000_adv_tx_context_desc *context_desc;
80785298 3336 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3337 struct igb_buffer *buffer_info;
3338 u32 info = 0, tu_cmd = 0;
80785298 3339 unsigned int i;
9d5c8243
AK		 3340
3341 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3342 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3343 i = tx_ring->next_to_use;
3344 buffer_info = &tx_ring->buffer_info[i];
3345 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3346
3347 if (tx_flags & IGB_TX_FLAGS_VLAN)
3348 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
cdfd01fc 3349
9d5c8243
AK		 3350 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3351 if (skb->ip_summed == CHECKSUM_PARTIAL)
3352 info |= skb_network_header_len(skb);
3353
3354 context_desc->vlan_macip_lens = cpu_to_le32(info);
3355
3356 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3357
3358 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fa4a7ef3
AJ		 3359 __be16 protocol;
3360
3361 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3362 const struct vlan_ethhdr *vhdr =
3363 (const struct vlan_ethhdr*)skb->data;
3364
3365 protocol = vhdr->h_vlan_encapsulated_proto;
3366 } else {
3367 protocol = skb->protocol;
3368 }
3369
3370 switch (protocol) {
09640e63 3371 case cpu_to_be16(ETH_P_IP):
9d5c8243 3372 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
44b0cda3
MW		 3373 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3374 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3375 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3376 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3 3377 break;
09640e63 3378 case cpu_to_be16(ETH_P_IPV6):
44b0cda3
MW		 3379 /* XXX what about other V6 headers?? */
3380 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3381 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3382 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3383 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3
MW		 3384 break;
3385 default:
3386 if (unlikely(net_ratelimit()))
80785298 3387 dev_warn(&pdev->dev,
44b0cda3
MW		 3388 "partial checksum but proto=%x!\n",
3389 skb->protocol);
3390 break;
3391 }
9d5c8243
AK		 3392 }
3393
3394 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3395 context_desc->seqnum_seed = 0;
85ad76b2 3396 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
7dfc16fa 3397 context_desc->mss_l4len_idx =
85ad76b2 3398 cpu_to_le32(tx_ring->reg_idx << 4);
9d5c8243
AK		 3399
3400 buffer_info->time_stamp = jiffies;
0e014cb1 3401 buffer_info->next_to_watch = i;
9d5c8243
AK		 3402 buffer_info->dma = 0;
3403
3404 i++;
3405 if (i == tx_ring->count)
3406 i = 0;
3407 tx_ring->next_to_use = i;
3408
3409 return true;
3410 }
9d5c8243
AK		 3411 return false;
3412}
3413
3414#define IGB_MAX_TXD_PWR 16
3415#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3416
80785298 3417static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
0e014cb1 3418 unsigned int first)
9d5c8243
AK		 3419{
3420 struct igb_buffer *buffer_info;
80785298 3421 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3422 unsigned int len = skb_headlen(skb);
3423 unsigned int count = 0, i;
3424 unsigned int f;
65689fef 3425 dma_addr_t *map;
9d5c8243
AK		 3426
3427 i = tx_ring->next_to_use;
3428
80785298
AD		 3429 if (skb_dma_map(&pdev->dev, skb, DMA_TO_DEVICE)) {
3430 dev_err(&pdev->dev, "TX DMA map failed\n");
65689fef
AD		 3431 return 0;
3432 }
3433
3434 map = skb_shinfo(skb)->dma_maps;
3435
9d5c8243
AK		 3436 buffer_info = &tx_ring->buffer_info[i];
3437 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3438 buffer_info->length = len;
3439 /* set time_stamp *before* dma to help avoid a possible race */
3440 buffer_info->time_stamp = jiffies;
0e014cb1 3441 buffer_info->next_to_watch = i;
042a53a9 3442 buffer_info->dma = skb_shinfo(skb)->dma_head;
9d5c8243
AK		 3443
3444 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3445 struct skb_frag_struct *frag;
3446
65689fef
AD		 3447 i++;
3448 if (i == tx_ring->count)
3449 i = 0;
3450
9d5c8243
AK		 3451 frag = &skb_shinfo(skb)->frags[f];
3452 len = frag->size;
3453
3454 buffer_info = &tx_ring->buffer_info[i];
3455 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3456 buffer_info->length = len;
3457 buffer_info->time_stamp = jiffies;
0e014cb1 3458 buffer_info->next_to_watch = i;
65689fef 3459 buffer_info->dma = map[count];
9d5c8243 3460 count++;
9d5c8243
AK		 3461 }
3462
9d5c8243 3463 tx_ring->buffer_info[i].skb = skb;
0e014cb1 3464 tx_ring->buffer_info[first].next_to_watch = i;
9d5c8243 3465
cdfd01fc 3466 return ++count;
9d5c8243
AK		 3467}
3468
85ad76b2 3469static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3470 int tx_flags, int count, u32 paylen,
3471 u8 hdr_len)
3472{
cdfd01fc 3473 union e1000_adv_tx_desc *tx_desc;
9d5c8243
AK		 3474 struct igb_buffer *buffer_info;
3475 u32 olinfo_status = 0, cmd_type_len;
cdfd01fc 3476 unsigned int i = tx_ring->next_to_use;
9d5c8243
AK		 3477
3478 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3479 E1000_ADVTXD_DCMD_DEXT);
3480
3481 if (tx_flags & IGB_TX_FLAGS_VLAN)
3482 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3483
33af6bcc
PO		 3484 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3485 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
3486
9d5c8243
AK		 3487 if (tx_flags & IGB_TX_FLAGS_TSO) {
3488 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3489
3490 /* insert tcp checksum */
3491 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3492
3493 /* insert ip checksum */
3494 if (tx_flags & IGB_TX_FLAGS_IPV4)
3495 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3496
3497 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
3498 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3499 }
3500
85ad76b2
AD		 3501 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
3502 (tx_flags & (IGB_TX_FLAGS_CSUM |
3503 IGB_TX_FLAGS_TSO |
7dfc16fa 3504 IGB_TX_FLAGS_VLAN)))
85ad76b2 3505 olinfo_status |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3506
3507 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
3508
cdfd01fc 3509 do {
9d5c8243
AK		 3510 buffer_info = &tx_ring->buffer_info[i];
3511 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
3512 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
3513 tx_desc->read.cmd_type_len =
3514 cpu_to_le32(cmd_type_len | buffer_info->length);
3515 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
cdfd01fc 3516 count--;
9d5c8243
AK		 3517 i++;
3518 if (i == tx_ring->count)
3519 i = 0;
cdfd01fc 3520 } while (count > 0);
9d5c8243 3521
85ad76b2 3522 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
9d5c8243
AK		 3523 /* Force memory writes to complete before letting h/w
3524 * know there are new descriptors to fetch. (Only
3525 * applicable for weak-ordered memory model archs,
3526 * such as IA-64). */
3527 wmb();
3528
3529 tx_ring->next_to_use = i;
fce99e34 3530 writel(i, tx_ring->tail);
9d5c8243
AK		 3531 /* we need this if more than one processor can write to our tail
3532 * at a time, it syncronizes IO on IA64/Altix systems */
3533 mmiowb();
3534}
3535
e694e964 3536static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3537{
e694e964
AD		 3538 struct net_device *netdev = tx_ring->netdev;
3539
661086df 3540 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 3541
9d5c8243
AK		 3542 /* Herbert's original patch had:
3543 * smp_mb__after_netif_stop_queue();
3544 * but since that doesn't exist yet, just open code it. */
3545 smp_mb();
3546
3547 /* We need to check again in a case another CPU has just
3548 * made room available. */
c493ea45 3549 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 3550 return -EBUSY;
3551
3552 /* A reprieve! */
661086df 3553 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 3554 tx_ring->tx_stats.restart_queue++;
9d5c8243
AK		 3555 return 0;
3556}
3557
e694e964 3558static int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3559{
c493ea45 3560 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 3561 return 0;
e694e964 3562 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 3563}
3564
b1a436c3
AD		 3565netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
3566 struct igb_ring *tx_ring)
9d5c8243 3567{
e694e964 3568 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
0e014cb1 3569 unsigned int first;
9d5c8243 3570 unsigned int tx_flags = 0;
9d5c8243 3571 u8 hdr_len = 0;
cdfd01fc 3572 int tso = 0, count;
c5b9bd5e 3573 union skb_shared_tx *shtx = skb_tx(skb);
9d5c8243 3574
9d5c8243
AK		 3575 /* need: 1 descriptor per page,
3576 * + 2 desc gap to keep tail from touching head,
3577 * + 1 desc for skb->data,
3578 * + 1 desc for context descriptor,
3579 * otherwise try next time */
e694e964 3580 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 3581 /* this is a hard error */
9d5c8243
AK		 3582 return NETDEV_TX_BUSY;
3583 }
33af6bcc 3584
33af6bcc
PO		 3585 if (unlikely(shtx->hardware)) {
3586 shtx->in_progress = 1;
3587 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 3588 }
9d5c8243 3589
cdfd01fc 3590 if (vlan_tx_tag_present(skb) && adapter->vlgrp) {
9d5c8243
AK		 3591 tx_flags |= IGB_TX_FLAGS_VLAN;
3592 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
3593 }
3594
661086df
PWJ		 3595 if (skb->protocol == htons(ETH_P_IP))
3596 tx_flags |= IGB_TX_FLAGS_IPV4;
3597
0e014cb1 3598 first = tx_ring->next_to_use;
85ad76b2
AD		 3599 if (skb_is_gso(skb)) {
3600 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
cdfd01fc 3601
85ad76b2
AD		 3602 if (tso < 0) {
3603 dev_kfree_skb_any(skb);
3604 return NETDEV_TX_OK;
3605 }
9d5c8243
AK		 3606 }
3607
3608 if (tso)
3609 tx_flags |= IGB_TX_FLAGS_TSO;
85ad76b2 3610 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
bc1cbd34
AD		 3611 (skb->ip_summed == CHECKSUM_PARTIAL))
3612 tx_flags |= IGB_TX_FLAGS_CSUM;
9d5c8243 3613
65689fef 3614 /*
cdfd01fc 3615 * count reflects descriptors mapped, if 0 or less then mapping error
65689fef
AD		 3616 * has occured and we need to rewind the descriptor queue
3617 */
80785298 3618 count = igb_tx_map_adv(tx_ring, skb, first);
cdfd01fc 3619 if (count <= 0) {
65689fef
AD		 3620 dev_kfree_skb_any(skb);
3621 tx_ring->buffer_info[first].time_stamp = 0;
3622 tx_ring->next_to_use = first;
85ad76b2 3623 return NETDEV_TX_OK;
65689fef 3624 }
9d5c8243 3625
85ad76b2
AD		 3626 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
3627
3628 /* Make sure there is space in the ring for the next send. */
e694e964 3629 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 3630
9d5c8243
AK		 3631 return NETDEV_TX_OK;
3632}
3633
3b29a56d
SH		 3634static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
3635 struct net_device *netdev)
9d5c8243
AK		 3636{
3637 struct igb_adapter *adapter = netdev_priv(netdev);
661086df 3638 struct igb_ring *tx_ring;
661086df 3639 int r_idx = 0;
b1a436c3
AD		 3640
3641 if (test_bit(__IGB_DOWN, &adapter->state)) {
3642 dev_kfree_skb_any(skb);
3643 return NETDEV_TX_OK;
3644 }
3645
3646 if (skb->len <= 0) {
3647 dev_kfree_skb_any(skb);
3648 return NETDEV_TX_OK;
3649 }
3650
1bfaf07b 3651 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
661086df 3652 tx_ring = adapter->multi_tx_table[r_idx];
9d5c8243
AK		 3653
3654 /* This goes back to the question of how to logically map a tx queue
3655 * to a flow. Right now, performance is impacted slightly negatively
3656 * if using multiple tx queues. If the stack breaks away from a
3657 * single qdisc implementation, we can look at this again. */
e694e964 3658 return igb_xmit_frame_ring_adv(skb, tx_ring);
9d5c8243
AK		 3659}
3660
3661/**
3662 * igb_tx_timeout - Respond to a Tx Hang
3663 * @netdev: network interface device structure
3664 **/
3665static void igb_tx_timeout(struct net_device *netdev)
3666{
3667 struct igb_adapter *adapter = netdev_priv(netdev);
3668 struct e1000_hw *hw = &adapter->hw;
3669
3670 /* Do the reset outside of interrupt context */
3671 adapter->tx_timeout_count++;
f7ba205e 3672
9d5c8243 3673 schedule_work(&adapter->reset_task);
265de409
AD		 3674 wr32(E1000_EICS,
3675 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 3676}
3677
3678static void igb_reset_task(struct work_struct *work)
3679{
3680 struct igb_adapter *adapter;
3681 adapter = container_of(work, struct igb_adapter, reset_task);
3682
3683 igb_reinit_locked(adapter);
3684}
3685
3686/**
3687 * igb_get_stats - Get System Network Statistics
3688 * @netdev: network interface device structure
3689 *
3690 * Returns the address of the device statistics structure.
3691 * The statistics are actually updated from the timer callback.
3692 **/
73cd78f1 3693static struct net_device_stats *igb_get_stats(struct net_device *netdev)
9d5c8243 3694{
9d5c8243 3695 /* only return the current stats */
8d24e933 3696 return &netdev->stats;
9d5c8243
AK		 3697}
3698
3699/**
3700 * igb_change_mtu - Change the Maximum Transfer Unit
3701 * @netdev: network interface device structure
3702 * @new_mtu: new value for maximum frame size
3703 *
3704 * Returns 0 on success, negative on failure
3705 **/
3706static int igb_change_mtu(struct net_device *netdev, int new_mtu)
3707{
3708 struct igb_adapter *adapter = netdev_priv(netdev);
3709 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4c844851 3710 u32 rx_buffer_len, i;
9d5c8243
AK		 3711
3712 if ((max_frame < ETH_ZLEN + ETH_FCS_LEN) ||
3713 (max_frame > MAX_JUMBO_FRAME_SIZE)) {
3714 dev_err(&adapter->pdev->dev, "Invalid MTU setting\n");
3715 return -EINVAL;
3716 }
3717
9d5c8243
AK		 3718 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
3719 dev_err(&adapter->pdev->dev, "MTU > 9216 not supported.\n");
3720 return -EINVAL;
3721 }
3722
3723 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
3724 msleep(1);
73cd78f1 3725
9d5c8243
AK		 3726 /* igb_down has a dependency on max_frame_size */
3727 adapter->max_frame_size = max_frame;
9d5c8243
AK		 3728 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3729 * means we reserve 2 more, this pushes us to allocate from the next
3730 * larger slab size.
3731 * i.e. RXBUFFER_2048 --> size-4096 slab
3732 */
3733
7d95b717 3734 if (max_frame <= IGB_RXBUFFER_1024)
4c844851 3735 rx_buffer_len = IGB_RXBUFFER_1024;
6ec43fe6 3736 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4c844851 3737 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
6ec43fe6 3738 else
4c844851
AD		 3739 rx_buffer_len = IGB_RXBUFFER_128;
3740
3741 if (netif_running(netdev))
3742 igb_down(adapter);
9d5c8243
AK		 3743
3744 dev_info(&adapter->pdev->dev, "changing MTU from %d to %d\n",
3745 netdev->mtu, new_mtu);
3746 netdev->mtu = new_mtu;
3747
4c844851
AD		 3748 for (i = 0; i < adapter->num_rx_queues; i++)
3749 adapter->rx_ring[i].rx_buffer_len = rx_buffer_len;
3750
9d5c8243
AK		 3751 if (netif_running(netdev))
3752 igb_up(adapter);
3753 else
3754 igb_reset(adapter);
3755
3756 clear_bit(__IGB_RESETTING, &adapter->state);
3757
3758 return 0;
3759}
3760
3761/**
3762 * igb_update_stats - Update the board statistics counters
3763 * @adapter: board private structure
3764 **/
3765
3766void igb_update_stats(struct igb_adapter *adapter)
3767{
8d24e933 3768 struct net_device *netdev = adapter->netdev;
9d5c8243
AK		 3769 struct e1000_hw *hw = &adapter->hw;
3770 struct pci_dev *pdev = adapter->pdev;
3f9c0164 3771 u32 rnbc;
9d5c8243 3772 u16 phy_tmp;
3f9c0164
AD		 3773 int i;
3774 u64 bytes, packets;
9d5c8243
AK		 3775
3776#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3777
3778 /*
3779 * Prevent stats update while adapter is being reset, or if the pci
3780 * connection is down.
3781 */
3782 if (adapter->link_speed == 0)
3783 return;
3784 if (pci_channel_offline(pdev))
3785 return;
3786
3f9c0164
AD		 3787 bytes = 0;
3788 packets = 0;
3789 for (i = 0; i < adapter->num_rx_queues; i++) {
3790 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3791 adapter->rx_ring[i].rx_stats.drops += rqdpc_tmp;
3792 netdev->stats.rx_fifo_errors += rqdpc_tmp;
3793 bytes += adapter->rx_ring[i].rx_stats.bytes;
3794 packets += adapter->rx_ring[i].rx_stats.packets;
3795 }
3796
3797 netdev->stats.rx_bytes = bytes;
3798 netdev->stats.rx_packets = packets;
3799
3800 bytes = 0;
3801 packets = 0;
3802 for (i = 0; i < adapter->num_tx_queues; i++) {
3803 bytes += adapter->tx_ring[i].tx_stats.bytes;
3804 packets += adapter->tx_ring[i].tx_stats.packets;
3805 }
3806 netdev->stats.tx_bytes = bytes;
3807 netdev->stats.tx_packets = packets;
3808
3809 /* read stats registers */
9d5c8243
AK		 3810 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
3811 adapter->stats.gprc += rd32(E1000_GPRC);
3812 adapter->stats.gorc += rd32(E1000_GORCL);
3813 rd32(E1000_GORCH); /* clear GORCL */
3814 adapter->stats.bprc += rd32(E1000_BPRC);
3815 adapter->stats.mprc += rd32(E1000_MPRC);
3816 adapter->stats.roc += rd32(E1000_ROC);
3817
3818 adapter->stats.prc64 += rd32(E1000_PRC64);
3819 adapter->stats.prc127 += rd32(E1000_PRC127);
3820 adapter->stats.prc255 += rd32(E1000_PRC255);
3821 adapter->stats.prc511 += rd32(E1000_PRC511);
3822 adapter->stats.prc1023 += rd32(E1000_PRC1023);
3823 adapter->stats.prc1522 += rd32(E1000_PRC1522);
3824 adapter->stats.symerrs += rd32(E1000_SYMERRS);
3825 adapter->stats.sec += rd32(E1000_SEC);
3826
3827 adapter->stats.mpc += rd32(E1000_MPC);
3828 adapter->stats.scc += rd32(E1000_SCC);
3829 adapter->stats.ecol += rd32(E1000_ECOL);
3830 adapter->stats.mcc += rd32(E1000_MCC);
3831 adapter->stats.latecol += rd32(E1000_LATECOL);
3832 adapter->stats.dc += rd32(E1000_DC);
3833 adapter->stats.rlec += rd32(E1000_RLEC);
3834 adapter->stats.xonrxc += rd32(E1000_XONRXC);
3835 adapter->stats.xontxc += rd32(E1000_XONTXC);
3836 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
3837 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
3838 adapter->stats.fcruc += rd32(E1000_FCRUC);
3839 adapter->stats.gptc += rd32(E1000_GPTC);
3840 adapter->stats.gotc += rd32(E1000_GOTCL);
3841 rd32(E1000_GOTCH); /* clear GOTCL */
3f9c0164
AD		 3842 rnbc = rd32(E1000_RNBC);
3843 adapter->stats.rnbc += rnbc;
3844 netdev->stats.rx_fifo_errors += rnbc;
9d5c8243
AK		 3845 adapter->stats.ruc += rd32(E1000_RUC);
3846 adapter->stats.rfc += rd32(E1000_RFC);
3847 adapter->stats.rjc += rd32(E1000_RJC);
3848 adapter->stats.tor += rd32(E1000_TORH);
3849 adapter->stats.tot += rd32(E1000_TOTH);
3850 adapter->stats.tpr += rd32(E1000_TPR);
3851
3852 adapter->stats.ptc64 += rd32(E1000_PTC64);
3853 adapter->stats.ptc127 += rd32(E1000_PTC127);
3854 adapter->stats.ptc255 += rd32(E1000_PTC255);
3855 adapter->stats.ptc511 += rd32(E1000_PTC511);
3856 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
3857 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
3858
3859 adapter->stats.mptc += rd32(E1000_MPTC);
3860 adapter->stats.bptc += rd32(E1000_BPTC);
3861
3862 /* used for adaptive IFS */
9d5c8243
AK		 3863 hw->mac.tx_packet_delta = rd32(E1000_TPT);
3864 adapter->stats.tpt += hw->mac.tx_packet_delta;
3865 hw->mac.collision_delta = rd32(E1000_COLC);
3866 adapter->stats.colc += hw->mac.collision_delta;
3867
3868 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
3869 adapter->stats.rxerrc += rd32(E1000_RXERRC);
3870 adapter->stats.tncrs += rd32(E1000_TNCRS);
3871 adapter->stats.tsctc += rd32(E1000_TSCTC);
3872 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
3873
3874 adapter->stats.iac += rd32(E1000_IAC);
3875 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
3876 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
3877 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
3878 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
3879 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
3880 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
3881 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
3882 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
3883
3884 /* Fill out the OS statistics structure */
8d24e933
AK		 3885 netdev->stats.multicast = adapter->stats.mprc;
3886 netdev->stats.collisions = adapter->stats.colc;
9d5c8243
AK		 3887
3888 /* Rx Errors */
3889
3890 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 3891 * our own version based on RUC and ROC */
8d24e933 3892 netdev->stats.rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 3893 adapter->stats.crcerrs + adapter->stats.algnerrc +
3894 adapter->stats.ruc + adapter->stats.roc +
3895 adapter->stats.cexterr;
8d24e933 3896 netdev->stats.rx_length_errors = adapter->stats.ruc +
9d5c8243 3897 adapter->stats.roc;
8d24e933
AK		 3898 netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
3899 netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
3900 netdev->stats.rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 3901
3902 /* Tx Errors */
8d24e933 3903 netdev->stats.tx_errors = adapter->stats.ecol +
9d5c8243 3904 adapter->stats.latecol;
8d24e933
AK		 3905 netdev->stats.tx_aborted_errors = adapter->stats.ecol;
3906 netdev->stats.tx_window_errors = adapter->stats.latecol;
3907 netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 3908
3909 /* Tx Dropped needs to be maintained elsewhere */
3910
3911 /* Phy Stats */
3912 if (hw->phy.media_type == e1000_media_type_copper) {
3913 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 3914 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 3915 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
3916 adapter->phy_stats.idle_errors += phy_tmp;
3917 }
3918 }
3919
3920 /* Management Stats */
3921 adapter->stats.mgptc += rd32(E1000_MGTPTC);
3922 adapter->stats.mgprc += rd32(E1000_MGTPRC);
3923 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
3924}
3925
9d5c8243
AK		 3926static irqreturn_t igb_msix_other(int irq, void *data)
3927{
047e0030 3928 struct igb_adapter *adapter = data;
9d5c8243 3929 struct e1000_hw *hw = &adapter->hw;
844290e5 3930 u32 icr = rd32(E1000_ICR);
844290e5 3931 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 3932
047e0030 3933 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 3934 /* HW is reporting DMA is out of sync */
3935 adapter->stats.doosync++;
3936 }
eebbbdba 3937
4ae196df
AD		 3938 /* Check for a mailbox event */
3939 if (icr & E1000_ICR_VMMB)
3940 igb_msg_task(adapter);
3941
3942 if (icr & E1000_ICR_LSC) {
3943 hw->mac.get_link_status = 1;
3944 /* guard against interrupt when we're going down */
3945 if (!test_bit(__IGB_DOWN, &adapter->state))
3946 mod_timer(&adapter->watchdog_timer, jiffies + 1);
3947 }
3948
25568a53
AD		 3949 if (adapter->vfs_allocated_count)
3950 wr32(E1000_IMS, E1000_IMS_LSC |
3951 E1000_IMS_VMMB |
3952 E1000_IMS_DOUTSYNC);
3953 else
3954 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
844290e5 3955 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 3956
3957 return IRQ_HANDLED;
3958}
3959
047e0030 3960static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 3961{
047e0030 3962 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 3963
047e0030
AD		 3964 if (!q_vector->set_itr)
3965 return;
73cd78f1 3966
047e0030
AD		 3967 if (!itr_val)
3968 itr_val = 0x4;
661086df 3969
047e0030
AD		 3970 if (q_vector->itr_shift)
3971 itr_val |= itr_val << q_vector->itr_shift;
661086df 3972 else
047e0030 3973 itr_val |= 0x8000000;
661086df 3974
047e0030
AD		 3975 writel(itr_val, q_vector->itr_register);
3976 q_vector->set_itr = 0;
6eb5a7f1
AD		 3977}
3978
047e0030 3979static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 3980{
047e0030 3981 struct igb_q_vector *q_vector = data;
9d5c8243 3982
047e0030
AD		 3983 /* Write the ITR value calculated from the previous interrupt. */
3984 igb_write_itr(q_vector);
9d5c8243 3985
047e0030 3986 napi_schedule(&q_vector->napi);
844290e5 3987
047e0030 3988 return IRQ_HANDLED;
fe4506b6
JC		 3989}
3990
421e02f0 3991#ifdef CONFIG_IGB_DCA
047e0030 3992static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 3993{
047e0030 3994 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 3995 struct e1000_hw *hw = &adapter->hw;
3996 int cpu = get_cpu();
fe4506b6 3997
047e0030
AD		 3998 if (q_vector->cpu == cpu)
3999 goto out_no_update;
4000
4001 if (q_vector->tx_ring) {
4002 int q = q_vector->tx_ring->reg_idx;
4003 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4004 if (hw->mac.type == e1000_82575) {
4005 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4006 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4007 } else {
047e0030
AD		 4008 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4009 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4010 E1000_DCA_TXCTRL_CPUID_SHIFT;
4011 }
4012 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4013 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4014 }
4015 if (q_vector->rx_ring) {
4016 int q = q_vector->rx_ring->reg_idx;
4017 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4018 if (hw->mac.type == e1000_82575) {
2d064c06 4019 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4020 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4021 } else {
4022 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4023 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4024 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4025 }
fe4506b6
JC		 4026 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4027 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4028 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4029 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4030 }
047e0030
AD		 4031 q_vector->cpu = cpu;
4032out_no_update:
fe4506b6
JC		 4033 put_cpu();
4034}
4035
4036static void igb_setup_dca(struct igb_adapter *adapter)
4037{
7e0e99ef 4038 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4039 int i;
4040
7dfc16fa 4041 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4042 return;
4043
7e0e99ef
AD		 4044 /* Always use CB2 mode, difference is masked in the CB driver. */
4045 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4046
047e0030
AD		 4047 for (i = 0; i < adapter->num_q_vectors; i++) {
4048 struct igb_q_vector *q_vector = adapter->q_vector[i];
4049 q_vector->cpu = -1;
4050 igb_update_dca(q_vector);
fe4506b6
JC		 4051 }
4052}
4053
4054static int __igb_notify_dca(struct device *dev, void *data)
4055{
4056 struct net_device *netdev = dev_get_drvdata(dev);
4057 struct igb_adapter *adapter = netdev_priv(netdev);
4058 struct e1000_hw *hw = &adapter->hw;
4059 unsigned long event = *(unsigned long *)data;
4060
4061 switch (event) {
4062 case DCA_PROVIDER_ADD:
4063 /* if already enabled, don't do it again */
7dfc16fa 4064 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4065 break;
fe4506b6
JC		 4066 /* Always use CB2 mode, difference is masked
4067 * in the CB driver. */
cbd347ad 4068 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
fe4506b6 4069 if (dca_add_requester(dev) == 0) {
bbd98fe4 4070 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6
JC		 4071 dev_info(&adapter->pdev->dev, "DCA enabled\n");
4072 igb_setup_dca(adapter);
4073 break;
4074 }
4075 /* Fall Through since DCA is disabled. */
4076 case DCA_PROVIDER_REMOVE:
7dfc16fa 4077 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4078 /* without this a class_device is left
047e0030 4079 * hanging around in the sysfs model */
fe4506b6
JC		 4080 dca_remove_requester(dev);
4081 dev_info(&adapter->pdev->dev, "DCA disabled\n");
7dfc16fa 4082 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4083 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4084 }
4085 break;
4086 }
bbd98fe4 4087
fe4506b6 4088 return 0;
9d5c8243
AK		 4089}
4090
fe4506b6
JC		 4091static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4092 void *p)
4093{
4094 int ret_val;
4095
4096 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4097 __igb_notify_dca);
4098
4099 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4100}
421e02f0 4101#endif /* CONFIG_IGB_DCA */
9d5c8243 4102
4ae196df
AD		 4103static void igb_ping_all_vfs(struct igb_adapter *adapter)
4104{
4105 struct e1000_hw *hw = &adapter->hw;
4106 u32 ping;
4107 int i;
4108
4109 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4110 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 4111 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 4112 ping |= E1000_VT_MSGTYPE_CTS;
4113 igb_write_mbx(hw, &ping, 1, i);
4114 }
4115}
4116
7d5753f0
AD		 4117static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4118{
4119 struct e1000_hw *hw = &adapter->hw;
4120 u32 vmolr = rd32(E1000_VMOLR(vf));
4121 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4122
4123 vf_data->flags |= ~(IGB_VF_FLAG_UNI_PROMISC |
4124 IGB_VF_FLAG_MULTI_PROMISC);
4125 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4126
4127 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4128 vmolr |= E1000_VMOLR_MPME;
4129 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4130 } else {
4131 /*
4132 * if we have hashes and we are clearing a multicast promisc
4133 * flag we need to write the hashes to the MTA as this step
4134 * was previously skipped
4135 */
4136 if (vf_data->num_vf_mc_hashes > 30) {
4137 vmolr |= E1000_VMOLR_MPME;
4138 } else if (vf_data->num_vf_mc_hashes) {
4139 int j;
4140 vmolr |= E1000_VMOLR_ROMPE;
4141 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4142 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4143 }
4144 }
4145
4146 wr32(E1000_VMOLR(vf), vmolr);
4147
4148 /* there are flags left unprocessed, likely not supported */
4149 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4150 return -EINVAL;
4151
4152 return 0;
4153
4154}
4155
4ae196df
AD		 4156static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4157 u32 *msgbuf, u32 vf)
4158{
4159 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4160 u16 *hash_list = (u16 *)&msgbuf[1];
4161 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4162 int i;
4163
7d5753f0 4164 /* salt away the number of multicast addresses assigned
4ae196df
AD		 4165 * to this VF for later use to restore when the PF multi cast
4166 * list changes
4167 */
4168 vf_data->num_vf_mc_hashes = n;
4169
7d5753f0
AD		 4170 /* only up to 30 hash values supported */
4171 if (n > 30)
4172 n = 30;
4173
4174 /* store the hashes for later use */
4ae196df 4175 for (i = 0; i < n; i++)
a419aef8 4176 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 4177
4178 /* Flush and reset the mta with the new values */
ff41f8dc 4179 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4180
4181 return 0;
4182}
4183
4184static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4185{
4186 struct e1000_hw *hw = &adapter->hw;
4187 struct vf_data_storage *vf_data;
4188 int i, j;
4189
4190 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 4191 u32 vmolr = rd32(E1000_VMOLR(i));
4192 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4193
4ae196df 4194 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 4195
4196 if ((vf_data->num_vf_mc_hashes > 30) ||
4197 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4198 vmolr |= E1000_VMOLR_MPME;
4199 } else if (vf_data->num_vf_mc_hashes) {
4200 vmolr |= E1000_VMOLR_ROMPE;
4201 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4202 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4203 }
4204 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 4205 }
4206}
4207
4208static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4209{
4210 struct e1000_hw *hw = &adapter->hw;
4211 u32 pool_mask, reg, vid;
4212 int i;
4213
4214 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4215
4216 /* Find the vlan filter for this id */
4217 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4218 reg = rd32(E1000_VLVF(i));
4219
4220 /* remove the vf from the pool */
4221 reg &= ~pool_mask;
4222
4223 /* if pool is empty then remove entry from vfta */
4224 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4225 (reg & E1000_VLVF_VLANID_ENABLE)) {
4226 reg = 0;
4227 vid = reg & E1000_VLVF_VLANID_MASK;
4228 igb_vfta_set(hw, vid, false);
4229 }
4230
4231 wr32(E1000_VLVF(i), reg);
4232 }
ae641bdc
AD		 4233
4234 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 4235}
4236
4237static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4238{
4239 struct e1000_hw *hw = &adapter->hw;
4240 u32 reg, i;
4241
51466239
AD		 4242 /* The vlvf table only exists on 82576 hardware and newer */
4243 if (hw->mac.type < e1000_82576)
4244 return -1;
4245
4246 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 4247 if (!adapter->vfs_allocated_count)
4248 return -1;
4249
4250 /* Find the vlan filter for this id */
4251 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4252 reg = rd32(E1000_VLVF(i));
4253 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4254 vid == (reg & E1000_VLVF_VLANID_MASK))
4255 break;
4256 }
4257
4258 if (add) {
4259 if (i == E1000_VLVF_ARRAY_SIZE) {
4260 /* Did not find a matching VLAN ID entry that was
4261 * enabled. Search for a free filter entry, i.e.
4262 * one without the enable bit set
4263 */
4264 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4265 reg = rd32(E1000_VLVF(i));
4266 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4267 break;
4268 }
4269 }
4270 if (i < E1000_VLVF_ARRAY_SIZE) {
4271 /* Found an enabled/available entry */
4272 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4273
4274 /* if !enabled we need to set this up in vfta */
4275 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 4276 /* add VID to filter table */
4277 igb_vfta_set(hw, vid, true);
4ae196df
AD		 4278 reg |= E1000_VLVF_VLANID_ENABLE;
4279 }
cad6d05f
AD		 4280 reg &= ~E1000_VLVF_VLANID_MASK;
4281 reg |= vid;
4ae196df 4282 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4283
4284 /* do not modify RLPML for PF devices */
4285 if (vf >= adapter->vfs_allocated_count)
4286 return 0;
4287
4288 if (!adapter->vf_data[vf].vlans_enabled) {
4289 u32 size;
4290 reg = rd32(E1000_VMOLR(vf));
4291 size = reg & E1000_VMOLR_RLPML_MASK;
4292 size += 4;
4293 reg &= ~E1000_VMOLR_RLPML_MASK;
4294 reg |= size;
4295 wr32(E1000_VMOLR(vf), reg);
4296 }
ae641bdc 4297
51466239 4298 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 4299 return 0;
4300 }
4301 } else {
4302 if (i < E1000_VLVF_ARRAY_SIZE) {
4303 /* remove vf from the pool */
4304 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4305 /* if pool is empty then remove entry from vfta */
4306 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4307 reg = 0;
4308 igb_vfta_set(hw, vid, false);
4309 }
4310 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4311
4312 /* do not modify RLPML for PF devices */
4313 if (vf >= adapter->vfs_allocated_count)
4314 return 0;
4315
4316 adapter->vf_data[vf].vlans_enabled--;
4317 if (!adapter->vf_data[vf].vlans_enabled) {
4318 u32 size;
4319 reg = rd32(E1000_VMOLR(vf));
4320 size = reg & E1000_VMOLR_RLPML_MASK;
4321 size -= 4;
4322 reg &= ~E1000_VMOLR_RLPML_MASK;
4323 reg |= size;
4324 wr32(E1000_VMOLR(vf), reg);
4325 }
4ae196df
AD		 4326 return 0;
4327 }
4328 }
4329 return -1;
4330}
4331
4332static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4333{
4334 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4335 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4336
4337 return igb_vlvf_set(adapter, vid, add, vf);
4338}
4339
f2ca0dbe 4340static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 4341{
f2ca0dbe
AD		 4342 /* clear all flags */
4343 adapter->vf_data[vf].flags = 0;
4344 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 4345
4346 /* reset offloads to defaults */
7d5753f0 4347 igb_set_vmolr(adapter, vf);
4ae196df
AD		 4348
4349 /* reset vlans for device */
4350 igb_clear_vf_vfta(adapter, vf);
4351
4352 /* reset multicast table array for vf */
4353 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4354
4355 /* Flush and reset the mta with the new values */
ff41f8dc 4356 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4357}
4358
f2ca0dbe
AD		 4359static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4360{
4361 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4362
4363 /* generate a new mac address as we were hotplug removed/added */
4364 random_ether_addr(vf_mac);
4365
4366 /* process remaining reset events */
4367 igb_vf_reset(adapter, vf);
4368}
4369
4370static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 4371{
4372 struct e1000_hw *hw = &adapter->hw;
4373 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 4374 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 4375 u32 reg, msgbuf[3];
4376 u8 *addr = (u8 *)(&msgbuf[1]);
4377
4378 /* process all the same items cleared in a function level reset */
f2ca0dbe 4379 igb_vf_reset(adapter, vf);
4ae196df
AD		 4380
4381 /* set vf mac address */
26ad9178 4382 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 4383
4384 /* enable transmit and receive for vf */
4385 reg = rd32(E1000_VFTE);
4386 wr32(E1000_VFTE, reg | (1 << vf));
4387 reg = rd32(E1000_VFRE);
4388 wr32(E1000_VFRE, reg | (1 << vf));
4389
f2ca0dbe 4390 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4ae196df
AD		 4391
4392 /* reply to reset with ack and vf mac address */
4393 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4394 memcpy(addr, vf_mac, 6);
4395 igb_write_mbx(hw, msgbuf, 3, vf);
4396}
4397
4398static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4399{
f2ca0dbe
AD		 4400 unsigned char *addr = (char *)&msg[1];
4401 int err = -1;
4ae196df 4402
f2ca0dbe
AD		 4403 if (is_valid_ether_addr(addr))
4404 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 4405
f2ca0dbe 4406 return err;
4ae196df
AD		 4407}
4408
4409static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
4410{
4411 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 4412 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 4413 u32 msg = E1000_VT_MSGTYPE_NACK;
4414
4415 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 4416 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
4417 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 4418 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 4419 vf_data->last_nack = jiffies;
4ae196df
AD		 4420 }
4421}
4422
f2ca0dbe 4423static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 4424{
f2ca0dbe
AD		 4425 struct pci_dev *pdev = adapter->pdev;
4426 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 4427 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 4428 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 4429 s32 retval;
4430
f2ca0dbe 4431 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df
AD		 4432
4433 if (retval)
f2ca0dbe 4434 dev_err(&pdev->dev, "Error receiving message from VF\n");
4ae196df
AD		 4435
4436 /* this is a message we already processed, do nothing */
4437 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 4438 return;
4ae196df
AD		 4439
4440 /*
4441 * until the vf completes a reset it should not be
4442 * allowed to start any configuration.
4443 */
4444
4445 if (msgbuf[0] == E1000_VF_RESET) {
4446 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 4447 return;
4ae196df
AD		 4448 }
4449
f2ca0dbe
AD		 4450 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
4451 msgbuf[0] = E1000_VT_MSGTYPE_NACK;
4452 if (time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4453 igb_write_mbx(hw, msgbuf, 1, vf);
4454 vf_data->last_nack = jiffies;
4455 }
4456 return;
4ae196df
AD		 4457 }
4458
4459 switch ((msgbuf[0] & 0xFFFF)) {
4460 case E1000_VF_SET_MAC_ADDR:
4461 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
4462 break;
7d5753f0
AD		 4463 case E1000_VF_SET_PROMISC:
4464 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
4465 break;
4ae196df
AD		 4466 case E1000_VF_SET_MULTICAST:
4467 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
4468 break;
4469 case E1000_VF_SET_LPE:
4470 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
4471 break;
4472 case E1000_VF_SET_VLAN:
4473 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4474 break;
4475 default:
4476 dev_err(&adapter->pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4477 retval = -1;
4478 break;
4479 }
4480
4481 /* notify the VF of the results of what it sent us */
4482 if (retval)
4483 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
4484 else
4485 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
4486
4487 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
4488
4489 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 4490}
4ae196df 4491
f2ca0dbe
AD		 4492static void igb_msg_task(struct igb_adapter *adapter)
4493{
4494 struct e1000_hw *hw = &adapter->hw;
4495 u32 vf;
4496
4497 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
4498 /* process any reset requests */
4499 if (!igb_check_for_rst(hw, vf))
4500 igb_vf_reset_event(adapter, vf);
4501
4502 /* process any messages pending */
4503 if (!igb_check_for_msg(hw, vf))
4504 igb_rcv_msg_from_vf(adapter, vf);
4505
4506 /* process any acks */
4507 if (!igb_check_for_ack(hw, vf))
4508 igb_rcv_ack_from_vf(adapter, vf);
4509 }
4ae196df
AD		 4510}
4511
68d480c4
AD		 4512/**
4513 * igb_set_uta - Set unicast filter table address
4514 * @adapter: board private structure
4515 *
4516 * The unicast table address is a register array of 32-bit registers.
4517 * The table is meant to be used in a way similar to how the MTA is used
4518 * however due to certain limitations in the hardware it is necessary to
4519 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
4520 * enable bit to allow vlan tag stripping when promiscous mode is enabled
4521 **/
4522static void igb_set_uta(struct igb_adapter *adapter)
4523{
4524 struct e1000_hw *hw = &adapter->hw;
4525 int i;
4526
4527 /* The UTA table only exists on 82576 hardware and newer */
4528 if (hw->mac.type < e1000_82576)
4529 return;
4530
4531 /* we only need to do this if VMDq is enabled */
4532 if (!adapter->vfs_allocated_count)
4533 return;
4534
4535 for (i = 0; i < hw->mac.uta_reg_count; i++)
4536 array_wr32(E1000_UTA, i, ~0);
4537}
4538
9d5c8243
AK		 4539/**
4540 * igb_intr_msi - Interrupt Handler
4541 * @irq: interrupt number
4542 * @data: pointer to a network interface device structure
4543 **/
4544static irqreturn_t igb_intr_msi(int irq, void *data)
4545{
047e0030
AD		 4546 struct igb_adapter *adapter = data;
4547 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4548 struct e1000_hw *hw = &adapter->hw;
4549 /* read ICR disables interrupts using IAM */
4550 u32 icr = rd32(E1000_ICR);
4551
047e0030 4552 igb_write_itr(q_vector);
9d5c8243 4553
047e0030 4554 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4555 /* HW is reporting DMA is out of sync */
4556 adapter->stats.doosync++;
4557 }
4558
9d5c8243
AK		 4559 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4560 hw->mac.get_link_status = 1;
4561 if (!test_bit(__IGB_DOWN, &adapter->state))
4562 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4563 }
4564
047e0030 4565 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4566
4567 return IRQ_HANDLED;
4568}
4569
4570/**
4a3c6433 4571 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 4572 * @irq: interrupt number
4573 * @data: pointer to a network interface device structure
4574 **/
4575static irqreturn_t igb_intr(int irq, void *data)
4576{
047e0030
AD		 4577 struct igb_adapter *adapter = data;
4578 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4579 struct e1000_hw *hw = &adapter->hw;
4580 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
4581 * need for the IMC write */
4582 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 4583 if (!icr)
4584 return IRQ_NONE; /* Not our interrupt */
4585
047e0030 4586 igb_write_itr(q_vector);
9d5c8243
AK		 4587
4588 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
4589 * not set, then the adapter didn't send an interrupt */
4590 if (!(icr & E1000_ICR_INT_ASSERTED))
4591 return IRQ_NONE;
4592
047e0030 4593 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4594 /* HW is reporting DMA is out of sync */
4595 adapter->stats.doosync++;
4596 }
4597
9d5c8243
AK		 4598 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4599 hw->mac.get_link_status = 1;
4600 /* guard against interrupt when we're going down */
4601 if (!test_bit(__IGB_DOWN, &adapter->state))
4602 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4603 }
4604
047e0030 4605 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4606
4607 return IRQ_HANDLED;
4608}
4609
047e0030 4610static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 4611{
047e0030 4612 struct igb_adapter *adapter = q_vector->adapter;
46544258 4613 struct e1000_hw *hw = &adapter->hw;
9d5c8243 4614
4fc82adf
AD		 4615 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
4616 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
047e0030 4617 if (!adapter->msix_entries)
6eb5a7f1 4618 igb_set_itr(adapter);
46544258 4619 else
047e0030 4620 igb_update_ring_itr(q_vector);
9d5c8243
AK		 4621 }
4622
46544258
AD		 4623 if (!test_bit(__IGB_DOWN, &adapter->state)) {
4624 if (adapter->msix_entries)
047e0030 4625 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 4626 else
4627 igb_irq_enable(adapter);
4628 }
9d5c8243
AK		 4629}
4630
46544258
AD		 4631/**
4632 * igb_poll - NAPI Rx polling callback
4633 * @napi: napi polling structure
4634 * @budget: count of how many packets we should handle
4635 **/
4636static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 4637{
047e0030
AD		 4638 struct igb_q_vector *q_vector = container_of(napi,
4639 struct igb_q_vector,
4640 napi);
4641 int tx_clean_complete = 1, work_done = 0;
9d5c8243 4642
421e02f0 4643#ifdef CONFIG_IGB_DCA
047e0030
AD		 4644 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
4645 igb_update_dca(q_vector);
fe4506b6 4646#endif
047e0030
AD		 4647 if (q_vector->tx_ring)
4648 tx_clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 4649
047e0030
AD		 4650 if (q_vector->rx_ring)
4651 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
4652
4653 if (!tx_clean_complete)
4654 work_done = budget;
46544258 4655
9d5c8243 4656 /* If not enough Rx work done, exit the polling mode */
5e6d5b17 4657 if (work_done < budget) {
288379f0 4658 napi_complete(napi);
047e0030 4659 igb_ring_irq_enable(q_vector);
9d5c8243
AK		 4660 }
4661
46544258 4662 return work_done;
9d5c8243 4663}
6d8126f9 4664
33af6bcc 4665/**
c5b9bd5e 4666 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 4667 * @adapter: board private structure
c5b9bd5e
AD		 4668 * @shhwtstamps: timestamp structure to update
4669 * @regval: unsigned 64bit system time value.
4670 *
4671 * We need to convert the system time value stored in the RX/TXSTMP registers
4672 * into a hwtstamp which can be used by the upper level timestamping functions
4673 */
4674static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
4675 struct skb_shared_hwtstamps *shhwtstamps,
4676 u64 regval)
4677{
4678 u64 ns;
4679
4680 ns = timecounter_cyc2time(&adapter->clock, regval);
4681 timecompare_update(&adapter->compare, ns);
4682 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
4683 shhwtstamps->hwtstamp = ns_to_ktime(ns);
4684 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
4685}
4686
4687/**
4688 * igb_tx_hwtstamp - utility function which checks for TX time stamp
4689 * @q_vector: pointer to q_vector containing needed info
33af6bcc
PO		 4690 * @skb: packet that was just sent
4691 *
4692 * If we were asked to do hardware stamping and such a time stamp is
4693 * available, then it must have been for this skb here because we only
4694 * allow only one such packet into the queue.
4695 */
c5b9bd5e 4696static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct sk_buff *skb)
33af6bcc 4697{
c5b9bd5e 4698 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc
PO		 4699 union skb_shared_tx *shtx = skb_tx(skb);
4700 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 4701 struct skb_shared_hwtstamps shhwtstamps;
4702 u64 regval;
33af6bcc 4703
c5b9bd5e
AD		 4704 /* if skb does not support hw timestamp or TX stamp not valid exit */
4705 if (likely(!shtx->hardware) ||
4706 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
4707 return;
4708
4709 regval = rd32(E1000_TXSTMPL);
4710 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
4711
4712 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
4713 skb_tstamp_tx(skb, &shhwtstamps);
33af6bcc
PO		 4714}
4715
9d5c8243
AK		 4716/**
4717 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 4718 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 4719 * returns true if ring is completely cleaned
4720 **/
047e0030 4721static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 4722{
047e0030
AD		 4723 struct igb_adapter *adapter = q_vector->adapter;
4724 struct igb_ring *tx_ring = q_vector->tx_ring;
e694e964 4725 struct net_device *netdev = tx_ring->netdev;
0e014cb1 4726 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 4727 struct igb_buffer *buffer_info;
4728 struct sk_buff *skb;
0e014cb1 4729 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 4730 unsigned int total_bytes = 0, total_packets = 0;
0e014cb1
AD		 4731 unsigned int i, eop, count = 0;
4732 bool cleaned = false;
9d5c8243 4733
9d5c8243 4734 i = tx_ring->next_to_clean;
0e014cb1
AD		 4735 eop = tx_ring->buffer_info[i].next_to_watch;
4736 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
4737
4738 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
4739 (count < tx_ring->count)) {
4740 for (cleaned = false; !cleaned; count++) {
4741 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
9d5c8243 4742 buffer_info = &tx_ring->buffer_info[i];
0e014cb1 4743 cleaned = (i == eop);
9d5c8243
AK		 4744 skb = buffer_info->skb;
4745
4746 if (skb) {
4747 unsigned int segs, bytecount;
4748 /* gso_segs is currently only valid for tcp */
4749 segs = skb_shinfo(skb)->gso_segs ?: 1;
4750 /* multiply data chunks by size of headers */
4751 bytecount = ((segs - 1) * skb_headlen(skb)) +
4752 skb->len;
4753 total_packets += segs;
4754 total_bytes += bytecount;
33af6bcc 4755
c5b9bd5e 4756 igb_tx_hwtstamp(q_vector, skb);
9d5c8243
AK		 4757 }
4758
80785298 4759 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
0e014cb1 4760 tx_desc->wb.status = 0;
9d5c8243
AK		 4761
4762 i++;
4763 if (i == tx_ring->count)
4764 i = 0;
9d5c8243 4765 }
0e014cb1
AD		 4766 eop = tx_ring->buffer_info[i].next_to_watch;
4767 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
4768 }
4769
9d5c8243
AK		 4770 tx_ring->next_to_clean = i;
4771
fc7d345d 4772 if (unlikely(count &&
9d5c8243 4773 netif_carrier_ok(netdev) &&
c493ea45 4774 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
9d5c8243
AK		 4775 /* Make sure that anybody stopping the queue after this
4776 * sees the new next_to_clean.
4777 */
4778 smp_mb();
661086df
PWJ		 4779 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
4780 !(test_bit(__IGB_DOWN, &adapter->state))) {
4781 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 4782 tx_ring->tx_stats.restart_queue++;
661086df 4783 }
9d5c8243
AK		 4784 }
4785
4786 if (tx_ring->detect_tx_hung) {
4787 /* Detect a transmit hang in hardware, this serializes the
4788 * check with the clearing of time_stamp and movement of i */
4789 tx_ring->detect_tx_hung = false;
4790 if (tx_ring->buffer_info[i].time_stamp &&
4791 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
4792 (adapter->tx_timeout_factor * HZ))
4793 && !(rd32(E1000_STATUS) &
4794 E1000_STATUS_TXOFF)) {
4795
9d5c8243 4796 /* detected Tx unit hang */
80785298 4797 dev_err(&tx_ring->pdev->dev,
9d5c8243 4798 "Detected Tx Unit Hang\n"
2d064c06 4799 " Tx Queue <%d>\n"
9d5c8243
AK		 4800 " TDH <%x>\n"
4801 " TDT <%x>\n"
4802 " next_to_use <%x>\n"
4803 " next_to_clean <%x>\n"
9d5c8243
AK		 4804 "buffer_info[next_to_clean]\n"
4805 " time_stamp <%lx>\n"
0e014cb1 4806 " next_to_watch <%x>\n"
9d5c8243
AK		 4807 " jiffies <%lx>\n"
4808 " desc.status <%x>\n",
2d064c06 4809 tx_ring->queue_index,
fce99e34
AD		 4810 readl(tx_ring->head),
4811 readl(tx_ring->tail),
9d5c8243
AK		 4812 tx_ring->next_to_use,
4813 tx_ring->next_to_clean,
f7ba205e 4814 tx_ring->buffer_info[eop].time_stamp,
0e014cb1 4815 eop,
9d5c8243 4816 jiffies,
0e014cb1 4817 eop_desc->wb.status);
661086df 4818 netif_stop_subqueue(netdev, tx_ring->queue_index);
9d5c8243
AK		 4819 }
4820 }
4821 tx_ring->total_bytes += total_bytes;
4822 tx_ring->total_packets += total_packets;
e21ed353
AD		 4823 tx_ring->tx_stats.bytes += total_bytes;
4824 tx_ring->tx_stats.packets += total_packets;
0e014cb1 4825 return (count < tx_ring->count);
9d5c8243
AK		 4826}
4827
9d5c8243
AK		 4828/**
4829 * igb_receive_skb - helper function to handle rx indications
047e0030
AD		 4830 * @q_vector: structure containing interrupt and ring information
4831 * @skb: packet to send up
4832 * @vlan_tag: vlan tag for packet
9d5c8243 4833 **/
047e0030
AD		 4834static void igb_receive_skb(struct igb_q_vector *q_vector,
4835 struct sk_buff *skb,
4836 u16 vlan_tag)
4837{
4838 struct igb_adapter *adapter = q_vector->adapter;
4839
4840 if (vlan_tag)
4841 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
4842 vlan_tag, skb);
182ff8df 4843 else
047e0030 4844 napi_gro_receive(&q_vector->napi, skb);
9d5c8243
AK		 4845}
4846
04a5fcaa 4847static inline void igb_rx_checksum_adv(struct igb_ring *ring,
9d5c8243
AK		 4848 u32 status_err, struct sk_buff *skb)
4849{
4850 skb->ip_summed = CHECKSUM_NONE;
4851
4852 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 4853 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
4854 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 4855 return;
85ad76b2 4856
9d5c8243
AK		 4857 /* TCP/UDP checksum error bit is set */
4858 if (status_err &
4859 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 4860 /*
4861 * work around errata with sctp packets where the TCPE aka
4862 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
4863 * packets, (aka let the stack check the crc32c)
4864 */
85ad76b2
AD		 4865 if ((skb->len == 60) &&
4866 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
04a5fcaa 4867 ring->rx_stats.csum_err++;
85ad76b2 4868
9d5c8243 4869 /* let the stack verify checksum errors */
9d5c8243
AK		 4870 return;
4871 }
4872 /* It must be a TCP or UDP packet with a valid checksum */
4873 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
4874 skb->ip_summed = CHECKSUM_UNNECESSARY;
4875
85ad76b2 4876 dev_dbg(&ring->pdev->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 4877}
4878
c5b9bd5e
AD		 4879static inline void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
4880 struct sk_buff *skb)
4881{
4882 struct igb_adapter *adapter = q_vector->adapter;
4883 struct e1000_hw *hw = &adapter->hw;
4884 u64 regval;
4885
4886 /*
4887 * If this bit is set, then the RX registers contain the time stamp. No
4888 * other packet will be time stamped until we read these registers, so
4889 * read the registers to make them available again. Because only one
4890 * packet can be time stamped at a time, we know that the register
4891 * values must belong to this one here and therefore we don't need to
4892 * compare any of the additional attributes stored for it.
4893 *
4894 * If nothing went wrong, then it should have a skb_shared_tx that we
4895 * can turn into a skb_shared_hwtstamps.
4896 */
4897 if (likely(!(staterr & E1000_RXDADV_STAT_TS)))
4898 return;
4899 if (!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
4900 return;
4901
4902 regval = rd32(E1000_RXSTMPL);
4903 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
4904
4905 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
4906}
4c844851 4907static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
2d94d8ab
AD		 4908 union e1000_adv_rx_desc *rx_desc)
4909{
4910 /* HW will not DMA in data larger than the given buffer, even if it
4911 * parses the (NFS, of course) header to be larger. In that case, it
4912 * fills the header buffer and spills the rest into the page.
4913 */
4914 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
4915 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4c844851
AD		 4916 if (hlen > rx_ring->rx_buffer_len)
4917 hlen = rx_ring->rx_buffer_len;
2d94d8ab
AD		 4918 return hlen;
4919}
4920
047e0030
AD		 4921static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
4922 int *work_done, int budget)
9d5c8243 4923{
047e0030 4924 struct igb_ring *rx_ring = q_vector->rx_ring;
e694e964 4925 struct net_device *netdev = rx_ring->netdev;
80785298 4926 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 4927 union e1000_adv_rx_desc *rx_desc , *next_rxd;
4928 struct igb_buffer *buffer_info , *next_buffer;
4929 struct sk_buff *skb;
9d5c8243
AK		 4930 bool cleaned = false;
4931 int cleaned_count = 0;
4932 unsigned int total_bytes = 0, total_packets = 0;
73cd78f1 4933 unsigned int i;
2d94d8ab
AD		 4934 u32 staterr;
4935 u16 length;
047e0030 4936 u16 vlan_tag;
9d5c8243
AK		 4937
4938 i = rx_ring->next_to_clean;
69d3ca53 4939 buffer_info = &rx_ring->buffer_info[i];
9d5c8243
AK		 4940 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
4941 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
4942
4943 while (staterr & E1000_RXD_STAT_DD) {
4944 if (*work_done >= budget)
4945 break;
4946 (*work_done)++;
9d5c8243 4947
69d3ca53
AD		 4948 skb = buffer_info->skb;
4949 prefetch(skb->data - NET_IP_ALIGN);
4950 buffer_info->skb = NULL;
4951
4952 i++;
4953 if (i == rx_ring->count)
4954 i = 0;
42d0781a 4955
69d3ca53
AD		 4956 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
4957 prefetch(next_rxd);
4958 next_buffer = &rx_ring->buffer_info[i];
9d5c8243
AK		 4959
4960 length = le16_to_cpu(rx_desc->wb.upper.length);
4961 cleaned = true;
4962 cleaned_count++;
4963
2d94d8ab 4964 if (buffer_info->dma) {
bf36c1a0 4965 pci_unmap_single(pdev, buffer_info->dma,
4c844851 4966 rx_ring->rx_buffer_len,
bf36c1a0 4967 PCI_DMA_FROMDEVICE);
91615f76 4968 buffer_info->dma = 0;
4c844851 4969 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
6ec43fe6
AD		 4970 skb_put(skb, length);
4971 goto send_up;
4972 }
4c844851 4973 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
bf36c1a0
AD		 4974 }
4975
4976 if (length) {
9d5c8243 4977 pci_unmap_page(pdev, buffer_info->page_dma,
bf36c1a0 4978 PAGE_SIZE / 2, PCI_DMA_FROMDEVICE);
9d5c8243 4979 buffer_info->page_dma = 0;
bf36c1a0
AD		 4980
4981 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags++,
4982 buffer_info->page,
4983 buffer_info->page_offset,
4984 length);
4985
6ec43fe6 4986 if (page_count(buffer_info->page) != 1)
bf36c1a0
AD		 4987 buffer_info->page = NULL;
4988 else
4989 get_page(buffer_info->page);
9d5c8243
AK		 4990
4991 skb->len += length;
4992 skb->data_len += length;
bf36c1a0 4993 skb->truesize += length;
9d5c8243 4994 }
9d5c8243 4995
bf36c1a0 4996 if (!(staterr & E1000_RXD_STAT_EOP)) {
b2d56536
AD		 4997 buffer_info->skb = next_buffer->skb;
4998 buffer_info->dma = next_buffer->dma;
4999 next_buffer->skb = skb;
5000 next_buffer->dma = 0;
bf36c1a0
AD		 5001 goto next_desc;
5002 }
69d3ca53 5003send_up:
9d5c8243
AK		 5004 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5005 dev_kfree_skb_irq(skb);
5006 goto next_desc;
5007 }
9d5c8243 5008
c5b9bd5e 5009 igb_rx_hwtstamp(q_vector, staterr, skb);
9d5c8243
AK		 5010 total_bytes += skb->len;
5011 total_packets++;
5012
85ad76b2 5013 igb_rx_checksum_adv(rx_ring, staterr, skb);
9d5c8243
AK		 5014
5015 skb->protocol = eth_type_trans(skb, netdev);
047e0030
AD		 5016 skb_record_rx_queue(skb, rx_ring->queue_index);
5017
5018 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5019 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
9d5c8243 5020
047e0030 5021 igb_receive_skb(q_vector, skb, vlan_tag);
9d5c8243 5022
9d5c8243
AK		 5023next_desc:
5024 rx_desc->wb.upper.status_error = 0;
5025
5026 /* return some buffers to hardware, one at a time is too slow */
5027 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
3b644cf6 5028 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5029 cleaned_count = 0;
5030 }
5031
5032 /* use prefetched values */
5033 rx_desc = next_rxd;
5034 buffer_info = next_buffer;
9d5c8243
AK		 5035 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5036 }
bf36c1a0 5037
9d5c8243 5038 rx_ring->next_to_clean = i;
c493ea45 5039 cleaned_count = igb_desc_unused(rx_ring);
9d5c8243
AK		 5040
5041 if (cleaned_count)
3b644cf6 5042 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5043
5044 rx_ring->total_packets += total_packets;
5045 rx_ring->total_bytes += total_bytes;
5046 rx_ring->rx_stats.packets += total_packets;
5047 rx_ring->rx_stats.bytes += total_bytes;
9d5c8243
AK		 5048 return cleaned;
5049}
5050
9d5c8243
AK		 5051/**
5052 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5053 * @adapter: address of board private structure
5054 **/
d7ee5b3a 5055void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
9d5c8243 5056{
e694e964 5057 struct net_device *netdev = rx_ring->netdev;
9d5c8243
AK		 5058 union e1000_adv_rx_desc *rx_desc;
5059 struct igb_buffer *buffer_info;
5060 struct sk_buff *skb;
5061 unsigned int i;
db761762 5062 int bufsz;
9d5c8243
AK		 5063
5064 i = rx_ring->next_to_use;
5065 buffer_info = &rx_ring->buffer_info[i];
5066
4c844851 5067 bufsz = rx_ring->rx_buffer_len;
db761762 5068
9d5c8243
AK		 5069 while (cleaned_count--) {
5070 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5071
6ec43fe6 5072 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
9d5c8243 5073 if (!buffer_info->page) {
42d0781a 5074 buffer_info->page = netdev_alloc_page(netdev);
bf36c1a0 5075 if (!buffer_info->page) {
04a5fcaa 5076 rx_ring->rx_stats.alloc_failed++;
bf36c1a0
AD		 5077 goto no_buffers;
5078 }
5079 buffer_info->page_offset = 0;
5080 } else {
5081 buffer_info->page_offset ^= PAGE_SIZE / 2;
9d5c8243
AK		 5082 }
5083 buffer_info->page_dma =
80785298 5084 pci_map_page(rx_ring->pdev, buffer_info->page,
bf36c1a0
AD		 5085 buffer_info->page_offset,
5086 PAGE_SIZE / 2,
9d5c8243 5087 PCI_DMA_FROMDEVICE);
42d0781a
AD		 5088 if (pci_dma_mapping_error(rx_ring->pdev,
5089 buffer_info->page_dma)) {
5090 buffer_info->page_dma = 0;
5091 rx_ring->rx_stats.alloc_failed++;
5092 goto no_buffers;
5093 }
9d5c8243
AK		 5094 }
5095
42d0781a
AD		 5096 skb = buffer_info->skb;
5097 if (!skb) {
89d71a66 5098 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
9d5c8243 5099 if (!skb) {
04a5fcaa 5100 rx_ring->rx_stats.alloc_failed++;
9d5c8243
AK		 5101 goto no_buffers;
5102 }
5103
9d5c8243 5104 buffer_info->skb = skb;
42d0781a
AD		 5105 }
5106 if (!buffer_info->dma) {
80785298
AD		 5107 buffer_info->dma = pci_map_single(rx_ring->pdev,
5108 skb->data,
9d5c8243
AK		 5109 bufsz,
5110 PCI_DMA_FROMDEVICE);
42d0781a
AD		 5111 if (pci_dma_mapping_error(rx_ring->pdev,
5112 buffer_info->dma)) {
5113 buffer_info->dma = 0;
5114 rx_ring->rx_stats.alloc_failed++;
5115 goto no_buffers;
5116 }
9d5c8243
AK		 5117 }
5118 /* Refresh the desc even if buffer_addrs didn't change because
5119 * each write-back erases this info. */
6ec43fe6 5120 if (bufsz < IGB_RXBUFFER_1024) {
9d5c8243
AK		 5121 rx_desc->read.pkt_addr =
5122 cpu_to_le64(buffer_info->page_dma);
5123 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5124 } else {
42d0781a 5125 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
9d5c8243
AK		 5126 rx_desc->read.hdr_addr = 0;
5127 }
5128
5129 i++;
5130 if (i == rx_ring->count)
5131 i = 0;
5132 buffer_info = &rx_ring->buffer_info[i];
5133 }
5134
5135no_buffers:
5136 if (rx_ring->next_to_use != i) {
5137 rx_ring->next_to_use = i;
5138 if (i == 0)
5139 i = (rx_ring->count - 1);
5140 else
5141 i--;
5142
5143 /* Force memory writes to complete before letting h/w
5144 * know there are new descriptors to fetch. (Only
5145 * applicable for weak-ordered memory model archs,
5146 * such as IA-64). */
5147 wmb();
fce99e34 5148 writel(i, rx_ring->tail);
9d5c8243
AK		 5149 }
5150}
5151
5152/**
5153 * igb_mii_ioctl -
5154 * @netdev:
5155 * @ifreq:
5156 * @cmd:
5157 **/
5158static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5159{
5160 struct igb_adapter *adapter = netdev_priv(netdev);
5161 struct mii_ioctl_data *data = if_mii(ifr);
5162
5163 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5164 return -EOPNOTSUPP;
5165
5166 switch (cmd) {
5167 case SIOCGMIIPHY:
5168 data->phy_id = adapter->hw.phy.addr;
5169 break;
5170 case SIOCGMIIREG:
f5f4cf08
AD		 5171 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5172 &data->val_out))
9d5c8243
AK		 5173 return -EIO;
5174 break;
5175 case SIOCSMIIREG:
5176 default:
5177 return -EOPNOTSUPP;
5178 }
5179 return 0;
5180}
5181
c6cb090b
PO		 5182/**
5183 * igb_hwtstamp_ioctl - control hardware time stamping
5184 * @netdev:
5185 * @ifreq:
5186 * @cmd:
5187 *
33af6bcc
PO		 5188 * Outgoing time stamping can be enabled and disabled. Play nice and
5189 * disable it when requested, although it shouldn't case any overhead
5190 * when no packet needs it. At most one packet in the queue may be
5191 * marked for time stamping, otherwise it would be impossible to tell
5192 * for sure to which packet the hardware time stamp belongs.
5193 *
5194 * Incoming time stamping has to be configured via the hardware
5195 * filters. Not all combinations are supported, in particular event
5196 * type has to be specified. Matching the kind of event packet is
5197 * not supported, with the exception of "all V2 events regardless of
5198 * level 2 or 4".
5199 *
c6cb090b
PO		 5200 **/
5201static int igb_hwtstamp_ioctl(struct net_device *netdev,
5202 struct ifreq *ifr, int cmd)
5203{
33af6bcc
PO		 5204 struct igb_adapter *adapter = netdev_priv(netdev);
5205 struct e1000_hw *hw = &adapter->hw;
c6cb090b 5206 struct hwtstamp_config config;
c5b9bd5e
AD		 5207 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5208 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 5209 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 5210 bool is_l4 = false;
5211 bool is_l2 = false;
33af6bcc 5212 u32 regval;
c6cb090b
PO		 5213
5214 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5215 return -EFAULT;
5216
5217 /* reserved for future extensions */
5218 if (config.flags)
5219 return -EINVAL;
5220
33af6bcc
PO		 5221 switch (config.tx_type) {
5222 case HWTSTAMP_TX_OFF:
c5b9bd5e 5223 tsync_tx_ctl = 0;
33af6bcc 5224 case HWTSTAMP_TX_ON:
33af6bcc
PO		 5225 break;
5226 default:
5227 return -ERANGE;
5228 }
5229
5230 switch (config.rx_filter) {
5231 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 5232 tsync_rx_ctl = 0;
33af6bcc
PO		 5233 break;
5234 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5235 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5236 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5237 case HWTSTAMP_FILTER_ALL:
5238 /*
5239 * register TSYNCRXCFG must be set, therefore it is not
5240 * possible to time stamp both Sync and Delay_Req messages
5241 * => fall back to time stamping all packets
5242 */
c5b9bd5e 5243 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 5244 config.rx_filter = HWTSTAMP_FILTER_ALL;
5245 break;
5246 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 5247 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5248 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 5249 is_l4 = true;
33af6bcc
PO		 5250 break;
5251 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 5252 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5253 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 5254 is_l4 = true;
33af6bcc
PO		 5255 break;
5256 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5257 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 5258 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5259 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 5260 is_l2 = true;
5261 is_l4 = true;
33af6bcc
PO		 5262 config.rx_filter = HWTSTAMP_FILTER_SOME;
5263 break;
5264 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5265 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 5266 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5267 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 5268 is_l2 = true;
5269 is_l4 = true;
33af6bcc
PO		 5270 config.rx_filter = HWTSTAMP_FILTER_SOME;
5271 break;
5272 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5273 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5274 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 5275 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 5276 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 5277 is_l2 = true;
33af6bcc
PO		 5278 break;
5279 default:
5280 return -ERANGE;
5281 }
5282
c5b9bd5e
AD		 5283 if (hw->mac.type == e1000_82575) {
5284 if (tsync_rx_ctl | tsync_tx_ctl)
5285 return -EINVAL;
5286 return 0;
5287 }
5288
33af6bcc
PO		 5289 /* enable/disable TX */
5290 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 5291 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5292 regval |= tsync_tx_ctl;
33af6bcc
PO		 5293 wr32(E1000_TSYNCTXCTL, regval);
5294
c5b9bd5e 5295 /* enable/disable RX */
33af6bcc 5296 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 5297 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5298 regval |= tsync_rx_ctl;
33af6bcc 5299 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 5300
c5b9bd5e
AD		 5301 /* define which PTP packets are time stamped */
5302 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 5303
c5b9bd5e
AD		 5304 /* define ethertype filter for timestamped packets */
5305 if (is_l2)
5306 wr32(E1000_ETQF(3),
5307 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5308 E1000_ETQF_1588 | /* enable timestamping */
5309 ETH_P_1588)); /* 1588 eth protocol type */
5310 else
5311 wr32(E1000_ETQF(3), 0);
5312
5313#define PTP_PORT 319
5314 /* L4 Queue Filter[3]: filter by destination port and protocol */
5315 if (is_l4) {
5316 u32 ftqf = (IPPROTO_UDP /* UDP */
5317 | E1000_FTQF_VF_BP /* VF not compared */
5318 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5319 | E1000_FTQF_MASK); /* mask all inputs */
5320 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5321
5322 wr32(E1000_IMIR(3), htons(PTP_PORT));
5323 wr32(E1000_IMIREXT(3),
5324 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5325 if (hw->mac.type == e1000_82576) {
5326 /* enable source port check */
5327 wr32(E1000_SPQF(3), htons(PTP_PORT));
5328 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5329 }
5330 wr32(E1000_FTQF(3), ftqf);
5331 } else {
5332 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5333 }
33af6bcc
PO		 5334 wrfl();
5335
5336 adapter->hwtstamp_config = config;
5337
5338 /* clear TX/RX time stamp registers, just to be sure */
5339 regval = rd32(E1000_TXSTMPH);
5340 regval = rd32(E1000_RXSTMPH);
c6cb090b 5341
33af6bcc
PO		 5342 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5343 -EFAULT : 0;
c6cb090b
PO		 5344}
5345
9d5c8243
AK		 5346/**
5347 * igb_ioctl -
5348 * @netdev:
5349 * @ifreq:
5350 * @cmd:
5351 **/
5352static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5353{
5354 switch (cmd) {
5355 case SIOCGMIIPHY:
5356 case SIOCGMIIREG:
5357 case SIOCSMIIREG:
5358 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 5359 case SIOCSHWTSTAMP:
5360 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 5361 default:
5362 return -EOPNOTSUPP;
5363 }
5364}
5365
009bc06e
AD		 5366s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5367{
5368 struct igb_adapter *adapter = hw->back;
5369 u16 cap_offset;
5370
5371 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5372 if (!cap_offset)
5373 return -E1000_ERR_CONFIG;
5374
5375 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5376
5377 return 0;
5378}
5379
5380s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5381{
5382 struct igb_adapter *adapter = hw->back;
5383 u16 cap_offset;
5384
5385 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5386 if (!cap_offset)
5387 return -E1000_ERR_CONFIG;
5388
5389 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
5390
5391 return 0;
5392}
5393
9d5c8243
AK		 5394static void igb_vlan_rx_register(struct net_device *netdev,
5395 struct vlan_group *grp)
5396{
5397 struct igb_adapter *adapter = netdev_priv(netdev);
5398 struct e1000_hw *hw = &adapter->hw;
5399 u32 ctrl, rctl;
5400
5401 igb_irq_disable(adapter);
5402 adapter->vlgrp = grp;
5403
5404 if (grp) {
5405 /* enable VLAN tag insert/strip */
5406 ctrl = rd32(E1000_CTRL);
5407 ctrl |= E1000_CTRL_VME;
5408 wr32(E1000_CTRL, ctrl);
5409
51466239 5410 /* Disable CFI check */
9d5c8243 5411 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 5412 rctl &= ~E1000_RCTL_CFIEN;
5413 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 5414 } else {
5415 /* disable VLAN tag insert/strip */
5416 ctrl = rd32(E1000_CTRL);
5417 ctrl &= ~E1000_CTRL_VME;
5418 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 5419 }
5420
e1739522
AD		 5421 igb_rlpml_set(adapter);
5422
9d5c8243
AK		 5423 if (!test_bit(__IGB_DOWN, &adapter->state))
5424 igb_irq_enable(adapter);
5425}
5426
5427static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
5428{
5429 struct igb_adapter *adapter = netdev_priv(netdev);
5430 struct e1000_hw *hw = &adapter->hw;
4ae196df 5431 int pf_id = adapter->vfs_allocated_count;
9d5c8243 5432
51466239
AD		 5433 /* attempt to add filter to vlvf array */
5434 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 5435
51466239
AD		 5436 /* add the filter since PF can receive vlans w/o entry in vlvf */
5437 igb_vfta_set(hw, vid, true);
9d5c8243
AK		 5438}
5439
5440static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
5441{
5442 struct igb_adapter *adapter = netdev_priv(netdev);
5443 struct e1000_hw *hw = &adapter->hw;
4ae196df 5444 int pf_id = adapter->vfs_allocated_count;
51466239 5445 s32 err;
9d5c8243
AK		 5446
5447 igb_irq_disable(adapter);
5448 vlan_group_set_device(adapter->vlgrp, vid, NULL);
5449
5450 if (!test_bit(__IGB_DOWN, &adapter->state))
5451 igb_irq_enable(adapter);
5452
51466239
AD		 5453 /* remove vlan from VLVF table array */
5454 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 5455
51466239
AD		 5456 /* if vid was not present in VLVF just remove it from table */
5457 if (err)
4ae196df 5458 igb_vfta_set(hw, vid, false);
9d5c8243
AK		 5459}
5460
5461static void igb_restore_vlan(struct igb_adapter *adapter)
5462{
5463 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
5464
5465 if (adapter->vlgrp) {
5466 u16 vid;
5467 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
5468 if (!vlan_group_get_device(adapter->vlgrp, vid))
5469 continue;
5470 igb_vlan_rx_add_vid(adapter->netdev, vid);
5471 }
5472 }
5473}
5474
5475int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
5476{
5477 struct e1000_mac_info *mac = &adapter->hw.mac;
5478
5479 mac->autoneg = 0;
5480
9d5c8243
AK		 5481 switch (spddplx) {
5482 case SPEED_10 + DUPLEX_HALF:
5483 mac->forced_speed_duplex = ADVERTISE_10_HALF;
5484 break;
5485 case SPEED_10 + DUPLEX_FULL:
5486 mac->forced_speed_duplex = ADVERTISE_10_FULL;
5487 break;
5488 case SPEED_100 + DUPLEX_HALF:
5489 mac->forced_speed_duplex = ADVERTISE_100_HALF;
5490 break;
5491 case SPEED_100 + DUPLEX_FULL:
5492 mac->forced_speed_duplex = ADVERTISE_100_FULL;
5493 break;
5494 case SPEED_1000 + DUPLEX_FULL:
5495 mac->autoneg = 1;
5496 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
5497 break;
5498 case SPEED_1000 + DUPLEX_HALF: /* not supported */
5499 default:
5500 dev_err(&adapter->pdev->dev,
5501 "Unsupported Speed/Duplex configuration\n");
5502 return -EINVAL;
5503 }
5504 return 0;
5505}
5506
3fe7c4c9 5507static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 5508{
5509 struct net_device *netdev = pci_get_drvdata(pdev);
5510 struct igb_adapter *adapter = netdev_priv(netdev);
5511 struct e1000_hw *hw = &adapter->hw;
2d064c06 5512 u32 ctrl, rctl, status;
9d5c8243
AK		 5513 u32 wufc = adapter->wol;
5514#ifdef CONFIG_PM
5515 int retval = 0;
5516#endif
5517
5518 netif_device_detach(netdev);
5519
a88f10ec
AD		 5520 if (netif_running(netdev))
5521 igb_close(netdev);
5522
047e0030 5523 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 5524
5525#ifdef CONFIG_PM
5526 retval = pci_save_state(pdev);
5527 if (retval)
5528 return retval;
5529#endif
5530
5531 status = rd32(E1000_STATUS);
5532 if (status & E1000_STATUS_LU)
5533 wufc &= ~E1000_WUFC_LNKC;
5534
5535 if (wufc) {
5536 igb_setup_rctl(adapter);
ff41f8dc 5537 igb_set_rx_mode(netdev);
9d5c8243
AK		 5538
5539 /* turn on all-multi mode if wake on multicast is enabled */
5540 if (wufc & E1000_WUFC_MC) {
5541 rctl = rd32(E1000_RCTL);
5542 rctl |= E1000_RCTL_MPE;
5543 wr32(E1000_RCTL, rctl);
5544 }
5545
5546 ctrl = rd32(E1000_CTRL);
5547 /* advertise wake from D3Cold */
5548 #define E1000_CTRL_ADVD3WUC 0x00100000
5549 /* phy power management enable */
5550 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5551 ctrl |= E1000_CTRL_ADVD3WUC;
5552 wr32(E1000_CTRL, ctrl);
5553
9d5c8243
AK		 5554 /* Allow time for pending master requests to run */
5555 igb_disable_pcie_master(&adapter->hw);
5556
5557 wr32(E1000_WUC, E1000_WUC_PME_EN);
5558 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 5559 } else {
5560 wr32(E1000_WUC, 0);
5561 wr32(E1000_WUFC, 0);
9d5c8243
AK		 5562 }
5563
3fe7c4c9
RW		 5564 *enable_wake = wufc || adapter->en_mng_pt;
5565 if (!*enable_wake)
2fb02a26 5566 igb_shutdown_serdes_link_82575(hw);
9d5c8243
AK		 5567
5568 /* Release control of h/w to f/w. If f/w is AMT enabled, this
5569 * would have already happened in close and is redundant. */
5570 igb_release_hw_control(adapter);
5571
5572 pci_disable_device(pdev);
5573
9d5c8243
AK		 5574 return 0;
5575}
5576
5577#ifdef CONFIG_PM
3fe7c4c9
RW		 5578static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
5579{
5580 int retval;
5581 bool wake;
5582
5583 retval = __igb_shutdown(pdev, &wake);
5584 if (retval)
5585 return retval;
5586
5587 if (wake) {
5588 pci_prepare_to_sleep(pdev);
5589 } else {
5590 pci_wake_from_d3(pdev, false);
5591 pci_set_power_state(pdev, PCI_D3hot);
5592 }
5593
5594 return 0;
5595}
5596
9d5c8243
AK		 5597static int igb_resume(struct pci_dev *pdev)
5598{
5599 struct net_device *netdev = pci_get_drvdata(pdev);
5600 struct igb_adapter *adapter = netdev_priv(netdev);
5601 struct e1000_hw *hw = &adapter->hw;
5602 u32 err;
5603
5604 pci_set_power_state(pdev, PCI_D0);
5605 pci_restore_state(pdev);
42bfd33a 5606
aed5dec3 5607 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 5608 if (err) {
5609 dev_err(&pdev->dev,
5610 "igb: Cannot enable PCI device from suspend\n");
5611 return err;
5612 }
5613 pci_set_master(pdev);
5614
5615 pci_enable_wake(pdev, PCI_D3hot, 0);
5616 pci_enable_wake(pdev, PCI_D3cold, 0);
5617
047e0030 5618 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 5619 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
5620 return -ENOMEM;
9d5c8243
AK		 5621 }
5622
5623 /* e1000_power_up_phy(adapter); */
5624
5625 igb_reset(adapter);
a8564f03
AD		 5626
5627 /* let the f/w know that the h/w is now under the control of the
5628 * driver. */
5629 igb_get_hw_control(adapter);
5630
9d5c8243
AK		 5631 wr32(E1000_WUS, ~0);
5632
a88f10ec
AD		 5633 if (netif_running(netdev)) {
5634 err = igb_open(netdev);
5635 if (err)
5636 return err;
5637 }
9d5c8243
AK		 5638
5639 netif_device_attach(netdev);
5640
9d5c8243
AK		 5641 return 0;
5642}
5643#endif
5644
5645static void igb_shutdown(struct pci_dev *pdev)
5646{
3fe7c4c9
RW		 5647 bool wake;
5648
5649 __igb_shutdown(pdev, &wake);
5650
5651 if (system_state == SYSTEM_POWER_OFF) {
5652 pci_wake_from_d3(pdev, wake);
5653 pci_set_power_state(pdev, PCI_D3hot);
5654 }
9d5c8243
AK		 5655}
5656
5657#ifdef CONFIG_NET_POLL_CONTROLLER
5658/*
5659 * Polling 'interrupt' - used by things like netconsole to send skbs
5660 * without having to re-enable interrupts. It's not called while
5661 * the interrupt routine is executing.
5662 */
5663static void igb_netpoll(struct net_device *netdev)
5664{
5665 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 5666 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5667 int i;
9d5c8243 5668
eebbbdba 5669 if (!adapter->msix_entries) {
047e0030 5670 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 5671 igb_irq_disable(adapter);
047e0030 5672 napi_schedule(&q_vector->napi);
eebbbdba
AD		 5673 return;
5674 }
9d5c8243 5675
047e0030
AD		 5676 for (i = 0; i < adapter->num_q_vectors; i++) {
5677 struct igb_q_vector *q_vector = adapter->q_vector[i];
5678 wr32(E1000_EIMC, q_vector->eims_value);
5679 napi_schedule(&q_vector->napi);
eebbbdba 5680 }
9d5c8243
AK		 5681}
5682#endif /* CONFIG_NET_POLL_CONTROLLER */
5683
5684/**
5685 * igb_io_error_detected - called when PCI error is detected
5686 * @pdev: Pointer to PCI device
5687 * @state: The current pci connection state
5688 *
5689 * This function is called after a PCI bus error affecting
5690 * this device has been detected.
5691 */
5692static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
5693 pci_channel_state_t state)
5694{
5695 struct net_device *netdev = pci_get_drvdata(pdev);
5696 struct igb_adapter *adapter = netdev_priv(netdev);
5697
5698 netif_device_detach(netdev);
5699
59ed6eec
AD		 5700 if (state == pci_channel_io_perm_failure)
5701 return PCI_ERS_RESULT_DISCONNECT;
5702
9d5c8243
AK		 5703 if (netif_running(netdev))
5704 igb_down(adapter);
5705 pci_disable_device(pdev);
5706
5707 /* Request a slot slot reset. */
5708 return PCI_ERS_RESULT_NEED_RESET;
5709}
5710
5711/**
5712 * igb_io_slot_reset - called after the pci bus has been reset.
5713 * @pdev: Pointer to PCI device
5714 *
5715 * Restart the card from scratch, as if from a cold-boot. Implementation
5716 * resembles the first-half of the igb_resume routine.
5717 */
5718static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
5719{
5720 struct net_device *netdev = pci_get_drvdata(pdev);
5721 struct igb_adapter *adapter = netdev_priv(netdev);
5722 struct e1000_hw *hw = &adapter->hw;
40a914fa 5723 pci_ers_result_t result;
42bfd33a 5724 int err;
9d5c8243 5725
aed5dec3 5726 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 5727 dev_err(&pdev->dev,
5728 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 5729 result = PCI_ERS_RESULT_DISCONNECT;
5730 } else {
5731 pci_set_master(pdev);
5732 pci_restore_state(pdev);
9d5c8243 5733
40a914fa
AD		 5734 pci_enable_wake(pdev, PCI_D3hot, 0);
5735 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 5736
40a914fa
AD		 5737 igb_reset(adapter);
5738 wr32(E1000_WUS, ~0);
5739 result = PCI_ERS_RESULT_RECOVERED;
5740 }
9d5c8243 5741
ea943d41
JK		 5742 err = pci_cleanup_aer_uncorrect_error_status(pdev);
5743 if (err) {
5744 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
5745 "failed 0x%0x\n", err);
5746 /* non-fatal, continue */
5747 }
40a914fa
AD		 5748
5749 return result;
9d5c8243
AK		 5750}
5751
5752/**
5753 * igb_io_resume - called when traffic can start flowing again.
5754 * @pdev: Pointer to PCI device
5755 *
5756 * This callback is called when the error recovery driver tells us that
5757 * its OK to resume normal operation. Implementation resembles the
5758 * second-half of the igb_resume routine.
5759 */
5760static void igb_io_resume(struct pci_dev *pdev)
5761{
5762 struct net_device *netdev = pci_get_drvdata(pdev);
5763 struct igb_adapter *adapter = netdev_priv(netdev);
5764
9d5c8243
AK		 5765 if (netif_running(netdev)) {
5766 if (igb_up(adapter)) {
5767 dev_err(&pdev->dev, "igb_up failed after reset\n");
5768 return;
5769 }
5770 }
5771
5772 netif_device_attach(netdev);
5773
5774 /* let the f/w know that the h/w is now under the control of the
5775 * driver. */
5776 igb_get_hw_control(adapter);
9d5c8243
AK		 5777}
5778
26ad9178
AD		 5779static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
5780 u8 qsel)
5781{
5782 u32 rar_low, rar_high;
5783 struct e1000_hw *hw = &adapter->hw;
5784
5785 /* HW expects these in little endian so we reverse the byte order
5786 * from network order (big endian) to little endian
5787 */
5788 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
5789 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
5790 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
5791
5792 /* Indicate to hardware the Address is Valid. */
5793 rar_high |= E1000_RAH_AV;
5794
5795 if (hw->mac.type == e1000_82575)
5796 rar_high |= E1000_RAH_POOL_1 * qsel;
5797 else
5798 rar_high |= E1000_RAH_POOL_1 << qsel;
5799
5800 wr32(E1000_RAL(index), rar_low);
5801 wrfl();
5802 wr32(E1000_RAH(index), rar_high);
5803 wrfl();
5804}
5805
4ae196df
AD		 5806static int igb_set_vf_mac(struct igb_adapter *adapter,
5807 int vf, unsigned char *mac_addr)
5808{
5809 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 5810 /* VF MAC addresses start at end of receive addresses and moves
5811 * torwards the first, as a result a collision should not be possible */
5812 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 5813
37680117 5814 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 5815
26ad9178 5816 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 5817
5818 return 0;
5819}
5820
5821static void igb_vmm_control(struct igb_adapter *adapter)
5822{
5823 struct e1000_hw *hw = &adapter->hw;
10d8e907 5824 u32 reg;
4ae196df 5825
d4960307
AD		 5826 /* replication is not supported for 82575 */
5827 if (hw->mac.type == e1000_82575)
4ae196df
AD		 5828 return;
5829
10d8e907
AD		 5830 /* enable replication vlan tag stripping */
5831 reg = rd32(E1000_RPLOLR);
5832 reg |= E1000_RPLOLR_STRVLAN;
5833 wr32(E1000_RPLOLR, reg);
5834
5835 /* notify HW that the MAC is adding vlan tags */
5836 reg = rd32(E1000_DTXCTL);
5837 reg |= E1000_DTXCTL_VLAN_ADDED;
5838 wr32(E1000_DTXCTL, reg);
5839
d4960307
AD		 5840 if (adapter->vfs_allocated_count) {
5841 igb_vmdq_set_loopback_pf(hw, true);
5842 igb_vmdq_set_replication_pf(hw, true);
5843 } else {
5844 igb_vmdq_set_loopback_pf(hw, false);
5845 igb_vmdq_set_replication_pf(hw, false);
5846 }
4ae196df
AD		 5847}
5848
9d5c8243 5849/* igb_main.c */
Ubuntu Artful kernel mirror