]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - drivers/net/igb/igb_main.c
projects / mirror_ubuntu-zesty-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
e1000: convert to use netdev_for_each_mc_addr
[mirror_ubuntu-zesty-kernel.git] / drivers / net / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
86d5d38f 4 Copyright(c) 2007-2009 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
31#include <linux/vmalloc.h>
32#include <linux/pagemap.h>
33#include <linux/netdevice.h>
9d5c8243
AK		 34#include <linux/ipv6.h>
35#include <net/checksum.h>
36#include <net/ip6_checksum.h>
c6cb090b 37#include <linux/net_tstamp.h>
9d5c8243
AK		 38#include <linux/mii.h>
39#include <linux/ethtool.h>
40#include <linux/if_vlan.h>
41#include <linux/pci.h>
c54106bb 42#include <linux/pci-aspm.h>
9d5c8243
AK		 43#include <linux/delay.h>
44#include <linux/interrupt.h>
45#include <linux/if_ether.h>
40a914fa 46#include <linux/aer.h>
421e02f0 47#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 48#include <linux/dca.h>
49#endif
9d5c8243
AK		 50#include "igb.h"
51
55cac248 52#define DRV_VERSION "2.1.0-k2"
9d5c8243
AK		 53char igb_driver_name[] = "igb";
54char igb_driver_version[] = DRV_VERSION;
55static const char igb_driver_string[] =
56 "Intel(R) Gigabit Ethernet Network Driver";
86d5d38f 57static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
9d5c8243 58
9d5c8243
AK		 59static const struct e1000_info *igb_info_tbl[] = {
60 [board_82575] = &e1000_82575_info,
61};
62
a3aa1884 63static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
55cac248
AD		 64 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
2d064c06 69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
c8ea5ea9 75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
79 /* required last entry */
80 {0, }
81};
82
83MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
84
85void igb_reset(struct igb_adapter *);
86static int igb_setup_all_tx_resources(struct igb_adapter *);
87static int igb_setup_all_rx_resources(struct igb_adapter *);
88static void igb_free_all_tx_resources(struct igb_adapter *);
89static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 90static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 91void igb_update_stats(struct igb_adapter *);
92static int igb_probe(struct pci_dev *, const struct pci_device_id *);
93static void __devexit igb_remove(struct pci_dev *pdev);
94static int igb_sw_init(struct igb_adapter *);
95static int igb_open(struct net_device *);
96static int igb_close(struct net_device *);
97static void igb_configure_tx(struct igb_adapter *);
98static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 99static void igb_clean_all_tx_rings(struct igb_adapter *);
100static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 101static void igb_clean_tx_ring(struct igb_ring *);
102static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 103static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 104static void igb_update_phy_info(unsigned long);
105static void igb_watchdog(unsigned long);
106static void igb_watchdog_task(struct work_struct *);
b1a436c3 107static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
9d5c8243
AK		 108static struct net_device_stats *igb_get_stats(struct net_device *);
109static int igb_change_mtu(struct net_device *, int);
110static int igb_set_mac(struct net_device *, void *);
68d480c4 111static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 112static irqreturn_t igb_intr(int irq, void *);
113static irqreturn_t igb_intr_msi(int irq, void *);
114static irqreturn_t igb_msix_other(int irq, void *);
047e0030 115static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 116#ifdef CONFIG_IGB_DCA
047e0030 117static void igb_update_dca(struct igb_q_vector *);
fe4506b6 118static void igb_setup_dca(struct igb_adapter *);
421e02f0 119#endif /* CONFIG_IGB_DCA */
047e0030 120static bool igb_clean_tx_irq(struct igb_q_vector *);
661086df 121static int igb_poll(struct napi_struct *, int);
047e0030 122static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
9d5c8243
AK		 123static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
124static void igb_tx_timeout(struct net_device *);
125static void igb_reset_task(struct work_struct *);
126static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
127static void igb_vlan_rx_add_vid(struct net_device *, u16);
128static void igb_vlan_rx_kill_vid(struct net_device *, u16);
129static void igb_restore_vlan(struct igb_adapter *);
26ad9178 130static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 131static void igb_ping_all_vfs(struct igb_adapter *);
132static void igb_msg_task(struct igb_adapter *);
4ae196df 133static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 134static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 135static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 136static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
137static int igb_ndo_set_vf_vlan(struct net_device *netdev,
138 int vf, u16 vlan, u8 qos);
139static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
140static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
141 struct ifla_vf_info *ivi);
9d5c8243 142
9d5c8243 143#ifdef CONFIG_PM
3fe7c4c9 144static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 145static int igb_resume(struct pci_dev *);
146#endif
147static void igb_shutdown(struct pci_dev *);
421e02f0 148#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 149static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
150static struct notifier_block dca_notifier = {
151 .notifier_call = igb_notify_dca,
152 .next = NULL,
153 .priority = 0
154};
155#endif
9d5c8243
AK		 156#ifdef CONFIG_NET_POLL_CONTROLLER
157/* for netdump / net console */
158static void igb_netpoll(struct net_device *);
159#endif
37680117 160#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 161static unsigned int max_vfs = 0;
162module_param(max_vfs, uint, 0);
163MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
164 "per physical function");
165#endif /* CONFIG_PCI_IOV */
166
9d5c8243
AK		 167static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
168 pci_channel_state_t);
169static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
170static void igb_io_resume(struct pci_dev *);
171
172static struct pci_error_handlers igb_err_handler = {
173 .error_detected = igb_io_error_detected,
174 .slot_reset = igb_io_slot_reset,
175 .resume = igb_io_resume,
176};
177
178
179static struct pci_driver igb_driver = {
180 .name = igb_driver_name,
181 .id_table = igb_pci_tbl,
182 .probe = igb_probe,
183 .remove = __devexit_p(igb_remove),
184#ifdef CONFIG_PM
185 /* Power Managment Hooks */
186 .suspend = igb_suspend,
187 .resume = igb_resume,
188#endif
189 .shutdown = igb_shutdown,
190 .err_handler = &igb_err_handler
191};
192
193MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
194MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
195MODULE_LICENSE("GPL");
196MODULE_VERSION(DRV_VERSION);
197
38c845c7
PO		 198/**
199 * igb_read_clock - read raw cycle counter (to be used by time counter)
200 */
201static cycle_t igb_read_clock(const struct cyclecounter *tc)
202{
203 struct igb_adapter *adapter =
204 container_of(tc, struct igb_adapter, cycles);
205 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 206 u64 stamp = 0;
207 int shift = 0;
38c845c7 208
55cac248
AD		 209 /*
210 * The timestamp latches on lowest register read. For the 82580
211 * the lowest register is SYSTIMR instead of SYSTIML. However we never
212 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
213 */
214 if (hw->mac.type == e1000_82580) {
215 stamp = rd32(E1000_SYSTIMR) >> 8;
216 shift = IGB_82580_TSYNC_SHIFT;
217 }
218
c5b9bd5e
AD		 219 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
220 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 221 return stamp;
222}
223
9d5c8243
AK		 224#ifdef DEBUG
225/**
226 * igb_get_hw_dev_name - return device name string
227 * used by hardware layer to print debugging information
228 **/
229char *igb_get_hw_dev_name(struct e1000_hw *hw)
230{
231 struct igb_adapter *adapter = hw->back;
232 return adapter->netdev->name;
233}
38c845c7
PO		 234
235/**
236 * igb_get_time_str - format current NIC and system time as string
237 */
238static char *igb_get_time_str(struct igb_adapter *adapter,
239 char buffer[160])
240{
241 cycle_t hw = adapter->cycles.read(&adapter->cycles);
242 struct timespec nic = ns_to_timespec(timecounter_read(&adapter->clock));
243 struct timespec sys;
244 struct timespec delta;
245 getnstimeofday(&sys);
246
247 delta = timespec_sub(nic, sys);
248
249 sprintf(buffer,
33af6bcc
PO		 250 "HW %llu, NIC %ld.%09lus, SYS %ld.%09lus, NIC-SYS %lds + %09luns",
251 hw,
38c845c7
PO		 252 (long)nic.tv_sec, nic.tv_nsec,
253 (long)sys.tv_sec, sys.tv_nsec,
254 (long)delta.tv_sec, delta.tv_nsec);
255
256 return buffer;
257}
9d5c8243
AK		 258#endif
259
260/**
261 * igb_init_module - Driver Registration Routine
262 *
263 * igb_init_module is the first routine called when the driver is
264 * loaded. All it does is register with the PCI subsystem.
265 **/
266static int __init igb_init_module(void)
267{
268 int ret;
269 printk(KERN_INFO "%s - version %s\n",
270 igb_driver_string, igb_driver_version);
271
272 printk(KERN_INFO "%s\n", igb_copyright);
273
421e02f0 274#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 275 dca_register_notify(&dca_notifier);
276#endif
bbd98fe4 277 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 278 return ret;
279}
280
281module_init(igb_init_module);
282
283/**
284 * igb_exit_module - Driver Exit Cleanup Routine
285 *
286 * igb_exit_module is called just before the driver is removed
287 * from memory.
288 **/
289static void __exit igb_exit_module(void)
290{
421e02f0 291#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 292 dca_unregister_notify(&dca_notifier);
293#endif
9d5c8243
AK		 294 pci_unregister_driver(&igb_driver);
295}
296
297module_exit(igb_exit_module);
298
26bc19ec
AD		 299#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
300/**
301 * igb_cache_ring_register - Descriptor ring to register mapping
302 * @adapter: board private structure to initialize
303 *
304 * Once we know the feature-set enabled for the device, we'll cache
305 * the register offset the descriptor ring is assigned to.
306 **/
307static void igb_cache_ring_register(struct igb_adapter *adapter)
308{
ee1b9f06 309 int i = 0, j = 0;
047e0030 310 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 311
312 switch (adapter->hw.mac.type) {
313 case e1000_82576:
314 /* The queues are allocated for virtualization such that VF 0
315 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
316 * In order to avoid collision we start at the first free queue
317 * and continue consuming queues in the same sequence
318 */
ee1b9f06 319 if (adapter->vfs_allocated_count) {
a99955fc 320 for (; i < adapter->rss_queues; i++)
3025a446
AD		 321 adapter->rx_ring[i]->reg_idx = rbase_offset +
322 Q_IDX_82576(i);
a99955fc 323 for (; j < adapter->rss_queues; j++)
3025a446
AD		 324 adapter->tx_ring[j]->reg_idx = rbase_offset +
325 Q_IDX_82576(j);
ee1b9f06 326 }
26bc19ec 327 case e1000_82575:
55cac248 328 case e1000_82580:
26bc19ec 329 default:
ee1b9f06 330 for (; i < adapter->num_rx_queues; i++)
3025a446 331 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 332 for (; j < adapter->num_tx_queues; j++)
3025a446 333 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 334 break;
335 }
336}
337
047e0030
AD		 338static void igb_free_queues(struct igb_adapter *adapter)
339{
3025a446 340 int i;
047e0030 341
3025a446
AD		 342 for (i = 0; i < adapter->num_tx_queues; i++) {
343 kfree(adapter->tx_ring[i]);
344 adapter->tx_ring[i] = NULL;
345 }
346 for (i = 0; i < adapter->num_rx_queues; i++) {
347 kfree(adapter->rx_ring[i]);
348 adapter->rx_ring[i] = NULL;
349 }
047e0030
AD		 350 adapter->num_rx_queues = 0;
351 adapter->num_tx_queues = 0;
352}
353
9d5c8243
AK		 354/**
355 * igb_alloc_queues - Allocate memory for all rings
356 * @adapter: board private structure to initialize
357 *
358 * We allocate one ring per queue at run-time since we don't know the
359 * number of queues at compile-time.
360 **/
361static int igb_alloc_queues(struct igb_adapter *adapter)
362{
3025a446 363 struct igb_ring *ring;
9d5c8243
AK		 364 int i;
365
661086df 366 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 367 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
368 if (!ring)
369 goto err;
68fd9910 370 ring->count = adapter->tx_ring_count;
661086df 371 ring->queue_index = i;
80785298 372 ring->pdev = adapter->pdev;
e694e964 373 ring->netdev = adapter->netdev;
85ad76b2
AD		 374 /* For 82575, context index must be unique per ring. */
375 if (adapter->hw.mac.type == e1000_82575)
376 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
3025a446 377 adapter->tx_ring[i] = ring;
661086df 378 }
85ad76b2 379
9d5c8243 380 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446
AD		 381 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
382 if (!ring)
383 goto err;
68fd9910 384 ring->count = adapter->rx_ring_count;
844290e5 385 ring->queue_index = i;
80785298 386 ring->pdev = adapter->pdev;
e694e964 387 ring->netdev = adapter->netdev;
4c844851 388 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85ad76b2
AD		 389 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
390 /* set flag indicating ring supports SCTP checksum offload */
391 if (adapter->hw.mac.type >= e1000_82576)
392 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
3025a446 393 adapter->rx_ring[i] = ring;
9d5c8243 394 }
26bc19ec
AD		 395
396 igb_cache_ring_register(adapter);
9d5c8243 397
047e0030 398 return 0;
a88f10ec 399
047e0030
AD		 400err:
401 igb_free_queues(adapter);
d1a8c9e1 402
047e0030 403 return -ENOMEM;
a88f10ec
AD		 404}
405
9d5c8243 406#define IGB_N0_QUEUE -1
047e0030 407static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 408{
409 u32 msixbm = 0;
047e0030 410 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 411 struct e1000_hw *hw = &adapter->hw;
2d064c06 412 u32 ivar, index;
047e0030
AD		 413 int rx_queue = IGB_N0_QUEUE;
414 int tx_queue = IGB_N0_QUEUE;
415
416 if (q_vector->rx_ring)
417 rx_queue = q_vector->rx_ring->reg_idx;
418 if (q_vector->tx_ring)
419 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 420
421 switch (hw->mac.type) {
422 case e1000_82575:
9d5c8243
AK		 423 /* The 82575 assigns vectors using a bitmask, which matches the
424 bitmask for the EICR/EIMS/EIMC registers. To assign one
425 or more queues to a vector, we write the appropriate bits
426 into the MSIXBM register for that vector. */
047e0030 427 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 428 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 429 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 430 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 431 if (!adapter->msix_entries && msix_vector == 0)
432 msixbm |= E1000_EIMS_OTHER;
9d5c8243 433 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 434 q_vector->eims_value = msixbm;
2d064c06
AD		 435 break;
436 case e1000_82576:
26bc19ec 437 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 438 Each queue has a single entry in the table to which we write
439 a vector number along with a "valid" bit. Sadly, the layout
440 of the table is somewhat counterintuitive. */
441 if (rx_queue > IGB_N0_QUEUE) {
047e0030 442 index = (rx_queue & 0x7);
2d064c06 443 ivar = array_rd32(E1000_IVAR0, index);
047e0030 444 if (rx_queue < 8) {
26bc19ec
AD		 445 /* vector goes into low byte of register */
446 ivar = ivar & 0xFFFFFF00;
447 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 448 } else {
449 /* vector goes into third byte of register */
450 ivar = ivar & 0xFF00FFFF;
451 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 452 }
2d064c06
AD		 453 array_wr32(E1000_IVAR0, index, ivar);
454 }
455 if (tx_queue > IGB_N0_QUEUE) {
047e0030 456 index = (tx_queue & 0x7);
2d064c06 457 ivar = array_rd32(E1000_IVAR0, index);
047e0030 458 if (tx_queue < 8) {
26bc19ec
AD		 459 /* vector goes into second byte of register */
460 ivar = ivar & 0xFFFF00FF;
461 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 462 } else {
463 /* vector goes into high byte of register */
464 ivar = ivar & 0x00FFFFFF;
465 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 466 }
2d064c06
AD		 467 array_wr32(E1000_IVAR0, index, ivar);
468 }
047e0030 469 q_vector->eims_value = 1 << msix_vector;
2d064c06 470 break;
55cac248
AD		 471 case e1000_82580:
472 /* 82580 uses the same table-based approach as 82576 but has fewer
473 entries as a result we carry over for queues greater than 4. */
474 if (rx_queue > IGB_N0_QUEUE) {
475 index = (rx_queue >> 1);
476 ivar = array_rd32(E1000_IVAR0, index);
477 if (rx_queue & 0x1) {
478 /* vector goes into third byte of register */
479 ivar = ivar & 0xFF00FFFF;
480 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
481 } else {
482 /* vector goes into low byte of register */
483 ivar = ivar & 0xFFFFFF00;
484 ivar |= msix_vector | E1000_IVAR_VALID;
485 }
486 array_wr32(E1000_IVAR0, index, ivar);
487 }
488 if (tx_queue > IGB_N0_QUEUE) {
489 index = (tx_queue >> 1);
490 ivar = array_rd32(E1000_IVAR0, index);
491 if (tx_queue & 0x1) {
492 /* vector goes into high byte of register */
493 ivar = ivar & 0x00FFFFFF;
494 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
495 } else {
496 /* vector goes into second byte of register */
497 ivar = ivar & 0xFFFF00FF;
498 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
499 }
500 array_wr32(E1000_IVAR0, index, ivar);
501 }
502 q_vector->eims_value = 1 << msix_vector;
503 break;
2d064c06
AD		 504 default:
505 BUG();
506 break;
507 }
26b39276
AD		 508
509 /* add q_vector eims value to global eims_enable_mask */
510 adapter->eims_enable_mask |= q_vector->eims_value;
511
512 /* configure q_vector to set itr on first interrupt */
513 q_vector->set_itr = 1;
9d5c8243
AK		 514}
515
516/**
517 * igb_configure_msix - Configure MSI-X hardware
518 *
519 * igb_configure_msix sets up the hardware to properly
520 * generate MSI-X interrupts.
521 **/
522static void igb_configure_msix(struct igb_adapter *adapter)
523{
524 u32 tmp;
525 int i, vector = 0;
526 struct e1000_hw *hw = &adapter->hw;
527
528 adapter->eims_enable_mask = 0;
9d5c8243
AK		 529
530 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 531 switch (hw->mac.type) {
532 case e1000_82575:
9d5c8243
AK		 533 tmp = rd32(E1000_CTRL_EXT);
534 /* enable MSI-X PBA support*/
535 tmp |= E1000_CTRL_EXT_PBA_CLR;
536
537 /* Auto-Mask interrupts upon ICR read. */
538 tmp |= E1000_CTRL_EXT_EIAME;
539 tmp |= E1000_CTRL_EXT_IRCA;
540
541 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 542
543 /* enable msix_other interrupt */
544 array_wr32(E1000_MSIXBM(0), vector++,
545 E1000_EIMS_OTHER);
844290e5 546 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 547
2d064c06
AD		 548 break;
549
550 case e1000_82576:
55cac248 551 case e1000_82580:
047e0030
AD		 552 /* Turn on MSI-X capability first, or our settings
553 * won't stick. And it will take days to debug. */
554 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
555 E1000_GPIE_PBA | E1000_GPIE_EIAME |
556 E1000_GPIE_NSICR);
557
558 /* enable msix_other interrupt */
559 adapter->eims_other = 1 << vector;
2d064c06 560 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 561
047e0030 562 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 563 break;
564 default:
565 /* do nothing, since nothing else supports MSI-X */
566 break;
567 } /* switch (hw->mac.type) */
047e0030
AD		 568
569 adapter->eims_enable_mask |= adapter->eims_other;
570
26b39276
AD		 571 for (i = 0; i < adapter->num_q_vectors; i++)
572 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 573
9d5c8243
AK		 574 wrfl();
575}
576
577/**
578 * igb_request_msix - Initialize MSI-X interrupts
579 *
580 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
581 * kernel.
582 **/
583static int igb_request_msix(struct igb_adapter *adapter)
584{
585 struct net_device *netdev = adapter->netdev;
047e0030 586 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 587 int i, err = 0, vector = 0;
588
047e0030 589 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 590 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 591 if (err)
592 goto out;
593 vector++;
594
595 for (i = 0; i < adapter->num_q_vectors; i++) {
596 struct igb_q_vector *q_vector = adapter->q_vector[i];
597
598 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
599
600 if (q_vector->rx_ring && q_vector->tx_ring)
601 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
602 q_vector->rx_ring->queue_index);
603 else if (q_vector->tx_ring)
604 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
605 q_vector->tx_ring->queue_index);
606 else if (q_vector->rx_ring)
607 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
608 q_vector->rx_ring->queue_index);
9d5c8243 609 else
047e0030
AD		 610 sprintf(q_vector->name, "%s-unused", netdev->name);
611
9d5c8243 612 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 613 igb_msix_ring, 0, q_vector->name,
047e0030 614 q_vector);
9d5c8243
AK		 615 if (err)
616 goto out;
9d5c8243
AK		 617 vector++;
618 }
619
9d5c8243
AK		 620 igb_configure_msix(adapter);
621 return 0;
622out:
623 return err;
624}
625
626static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
627{
628 if (adapter->msix_entries) {
629 pci_disable_msix(adapter->pdev);
630 kfree(adapter->msix_entries);
631 adapter->msix_entries = NULL;
047e0030 632 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 633 pci_disable_msi(adapter->pdev);
047e0030 634 }
9d5c8243
AK		 635}
636
047e0030
AD		 637/**
638 * igb_free_q_vectors - Free memory allocated for interrupt vectors
639 * @adapter: board private structure to initialize
640 *
641 * This function frees the memory allocated to the q_vectors. In addition if
642 * NAPI is enabled it will delete any references to the NAPI struct prior
643 * to freeing the q_vector.
644 **/
645static void igb_free_q_vectors(struct igb_adapter *adapter)
646{
647 int v_idx;
648
649 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
650 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
651 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 652 if (!q_vector)
653 continue;
047e0030
AD		 654 netif_napi_del(&q_vector->napi);
655 kfree(q_vector);
656 }
657 adapter->num_q_vectors = 0;
658}
659
660/**
661 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
662 *
663 * This function resets the device so that it has 0 rx queues, tx queues, and
664 * MSI-X interrupts allocated.
665 */
666static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
667{
668 igb_free_queues(adapter);
669 igb_free_q_vectors(adapter);
670 igb_reset_interrupt_capability(adapter);
671}
9d5c8243
AK		 672
673/**
674 * igb_set_interrupt_capability - set MSI or MSI-X if supported
675 *
676 * Attempt to configure interrupts using the best available
677 * capabilities of the hardware and kernel.
678 **/
679static void igb_set_interrupt_capability(struct igb_adapter *adapter)
680{
681 int err;
682 int numvecs, i;
683
83b7180d 684 /* Number of supported queues. */
a99955fc
AD		 685 adapter->num_rx_queues = adapter->rss_queues;
686 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 687
047e0030
AD		 688 /* start with one vector for every rx queue */
689 numvecs = adapter->num_rx_queues;
690
691 /* if tx handler is seperate add 1 for every tx queue */
a99955fc
AD		 692 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
693 numvecs += adapter->num_tx_queues;
047e0030
AD		 694
695 /* store the number of vectors reserved for queues */
696 adapter->num_q_vectors = numvecs;
697
698 /* add 1 vector for link status interrupts */
699 numvecs++;
9d5c8243
AK		 700 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
701 GFP_KERNEL);
702 if (!adapter->msix_entries)
703 goto msi_only;
704
705 for (i = 0; i < numvecs; i++)
706 adapter->msix_entries[i].entry = i;
707
708 err = pci_enable_msix(adapter->pdev,
709 adapter->msix_entries,
710 numvecs);
711 if (err == 0)
34a20e89 712 goto out;
9d5c8243
AK		 713
714 igb_reset_interrupt_capability(adapter);
715
716 /* If we can't do MSI-X, try MSI */
717msi_only:
2a3abf6d
AD		 718#ifdef CONFIG_PCI_IOV
719 /* disable SR-IOV for non MSI-X configurations */
720 if (adapter->vf_data) {
721 struct e1000_hw *hw = &adapter->hw;
722 /* disable iov and allow time for transactions to clear */
723 pci_disable_sriov(adapter->pdev);
724 msleep(500);
725
726 kfree(adapter->vf_data);
727 adapter->vf_data = NULL;
728 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
729 msleep(100);
730 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
731 }
732#endif
4fc82adf 733 adapter->vfs_allocated_count = 0;
a99955fc 734 adapter->rss_queues = 1;
4fc82adf 735 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 736 adapter->num_rx_queues = 1;
661086df 737 adapter->num_tx_queues = 1;
047e0030 738 adapter->num_q_vectors = 1;
9d5c8243 739 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 740 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 741out:
661086df 742 /* Notify the stack of the (possibly) reduced Tx Queue count. */
fd2ea0a7 743 adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
9d5c8243
AK		 744 return;
745}
746
047e0030
AD		 747/**
748 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
749 * @adapter: board private structure to initialize
750 *
751 * We allocate one q_vector per queue interrupt. If allocation fails we
752 * return -ENOMEM.
753 **/
754static int igb_alloc_q_vectors(struct igb_adapter *adapter)
755{
756 struct igb_q_vector *q_vector;
757 struct e1000_hw *hw = &adapter->hw;
758 int v_idx;
759
760 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
761 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
762 if (!q_vector)
763 goto err_out;
764 q_vector->adapter = adapter;
047e0030
AD		 765 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
766 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 767 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
768 adapter->q_vector[v_idx] = q_vector;
769 }
770 return 0;
771
772err_out:
fe0592b4 773 igb_free_q_vectors(adapter);
047e0030
AD		 774 return -ENOMEM;
775}
776
777static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
778 int ring_idx, int v_idx)
779{
3025a446 780 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 781
3025a446 782 q_vector->rx_ring = adapter->rx_ring[ring_idx];
047e0030 783 q_vector->rx_ring->q_vector = q_vector;
4fc82adf
AD		 784 q_vector->itr_val = adapter->rx_itr_setting;
785 if (q_vector->itr_val && q_vector->itr_val <= 3)
786 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 787}
788
789static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
790 int ring_idx, int v_idx)
791{
3025a446 792 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 793
3025a446 794 q_vector->tx_ring = adapter->tx_ring[ring_idx];
047e0030 795 q_vector->tx_ring->q_vector = q_vector;
4fc82adf
AD		 796 q_vector->itr_val = adapter->tx_itr_setting;
797 if (q_vector->itr_val && q_vector->itr_val <= 3)
798 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 799}
800
801/**
802 * igb_map_ring_to_vector - maps allocated queues to vectors
803 *
804 * This function maps the recently allocated queues to vectors.
805 **/
806static int igb_map_ring_to_vector(struct igb_adapter *adapter)
807{
808 int i;
809 int v_idx = 0;
810
811 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
812 (adapter->num_q_vectors < adapter->num_tx_queues))
813 return -ENOMEM;
814
815 if (adapter->num_q_vectors >=
816 (adapter->num_rx_queues + adapter->num_tx_queues)) {
817 for (i = 0; i < adapter->num_rx_queues; i++)
818 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
819 for (i = 0; i < adapter->num_tx_queues; i++)
820 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
821 } else {
822 for (i = 0; i < adapter->num_rx_queues; i++) {
823 if (i < adapter->num_tx_queues)
824 igb_map_tx_ring_to_vector(adapter, i, v_idx);
825 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
826 }
827 for (; i < adapter->num_tx_queues; i++)
828 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
829 }
830 return 0;
831}
832
833/**
834 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
835 *
836 * This function initializes the interrupts and allocates all of the queues.
837 **/
838static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
839{
840 struct pci_dev *pdev = adapter->pdev;
841 int err;
842
843 igb_set_interrupt_capability(adapter);
844
845 err = igb_alloc_q_vectors(adapter);
846 if (err) {
847 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
848 goto err_alloc_q_vectors;
849 }
850
851 err = igb_alloc_queues(adapter);
852 if (err) {
853 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
854 goto err_alloc_queues;
855 }
856
857 err = igb_map_ring_to_vector(adapter);
858 if (err) {
859 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
860 goto err_map_queues;
861 }
862
863
864 return 0;
865err_map_queues:
866 igb_free_queues(adapter);
867err_alloc_queues:
868 igb_free_q_vectors(adapter);
869err_alloc_q_vectors:
870 igb_reset_interrupt_capability(adapter);
871 return err;
872}
873
9d5c8243
AK		 874/**
875 * igb_request_irq - initialize interrupts
876 *
877 * Attempts to configure interrupts using the best available
878 * capabilities of the hardware and kernel.
879 **/
880static int igb_request_irq(struct igb_adapter *adapter)
881{
882 struct net_device *netdev = adapter->netdev;
047e0030 883 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 884 int err = 0;
885
886 if (adapter->msix_entries) {
887 err = igb_request_msix(adapter);
844290e5 888 if (!err)
9d5c8243 889 goto request_done;
9d5c8243 890 /* fall back to MSI */
047e0030 891 igb_clear_interrupt_scheme(adapter);
9d5c8243 892 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 893 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 894 igb_free_all_tx_resources(adapter);
895 igb_free_all_rx_resources(adapter);
047e0030 896 adapter->num_tx_queues = 1;
9d5c8243 897 adapter->num_rx_queues = 1;
047e0030
AD		 898 adapter->num_q_vectors = 1;
899 err = igb_alloc_q_vectors(adapter);
900 if (err) {
901 dev_err(&pdev->dev,
902 "Unable to allocate memory for vectors\n");
903 goto request_done;
904 }
905 err = igb_alloc_queues(adapter);
906 if (err) {
907 dev_err(&pdev->dev,
908 "Unable to allocate memory for queues\n");
909 igb_free_q_vectors(adapter);
910 goto request_done;
911 }
912 igb_setup_all_tx_resources(adapter);
913 igb_setup_all_rx_resources(adapter);
844290e5 914 } else {
feeb2721 915 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243 916 }
844290e5 917
7dfc16fa 918 if (adapter->flags & IGB_FLAG_HAS_MSI) {
a0607fd3 919 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
047e0030 920 netdev->name, adapter);
9d5c8243
AK		 921 if (!err)
922 goto request_done;
047e0030 923
9d5c8243
AK		 924 /* fall back to legacy interrupts */
925 igb_reset_interrupt_capability(adapter);
7dfc16fa 926 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 927 }
928
a0607fd3 929 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
047e0030 930 netdev->name, adapter);
9d5c8243 931
6cb5e577 932 if (err)
9d5c8243
AK		 933 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
934 err);
9d5c8243
AK		 935
936request_done:
937 return err;
938}
939
940static void igb_free_irq(struct igb_adapter *adapter)
941{
9d5c8243
AK		 942 if (adapter->msix_entries) {
943 int vector = 0, i;
944
047e0030 945 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 946
047e0030
AD		 947 for (i = 0; i < adapter->num_q_vectors; i++) {
948 struct igb_q_vector *q_vector = adapter->q_vector[i];
949 free_irq(adapter->msix_entries[vector++].vector,
950 q_vector);
951 }
952 } else {
953 free_irq(adapter->pdev->irq, adapter);
9d5c8243 954 }
9d5c8243
AK		 955}
956
957/**
958 * igb_irq_disable - Mask off interrupt generation on the NIC
959 * @adapter: board private structure
960 **/
961static void igb_irq_disable(struct igb_adapter *adapter)
962{
963 struct e1000_hw *hw = &adapter->hw;
964
25568a53
AD		 965 /*
966 * we need to be careful when disabling interrupts. The VFs are also
967 * mapped into these registers and so clearing the bits can cause
968 * issues on the VF drivers so we only need to clear what we set
969 */
9d5c8243 970 if (adapter->msix_entries) {
2dfd1212
AD		 971 u32 regval = rd32(E1000_EIAM);
972 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
973 wr32(E1000_EIMC, adapter->eims_enable_mask);
974 regval = rd32(E1000_EIAC);
975 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 976 }
844290e5
PW		 977
978 wr32(E1000_IAM, 0);
9d5c8243
AK		 979 wr32(E1000_IMC, ~0);
980 wrfl();
981 synchronize_irq(adapter->pdev->irq);
982}
983
984/**
985 * igb_irq_enable - Enable default interrupt generation settings
986 * @adapter: board private structure
987 **/
988static void igb_irq_enable(struct igb_adapter *adapter)
989{
990 struct e1000_hw *hw = &adapter->hw;
991
992 if (adapter->msix_entries) {
25568a53 993 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
2dfd1212
AD		 994 u32 regval = rd32(E1000_EIAC);
995 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
996 regval = rd32(E1000_EIAM);
997 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 998 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 999 if (adapter->vfs_allocated_count) {
4ae196df 1000 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1001 ims |= E1000_IMS_VMMB;
1002 }
55cac248
AD		 1003 if (adapter->hw.mac.type == e1000_82580)
1004 ims |= E1000_IMS_DRSTA;
1005
25568a53 1006 wr32(E1000_IMS, ims);
844290e5 1007 } else {
55cac248
AD		 1008 wr32(E1000_IMS, IMS_ENABLE_MASK |
1009 E1000_IMS_DRSTA);
1010 wr32(E1000_IAM, IMS_ENABLE_MASK |
1011 E1000_IMS_DRSTA);
844290e5 1012 }
9d5c8243
AK		 1013}
1014
1015static void igb_update_mng_vlan(struct igb_adapter *adapter)
1016{
51466239 1017 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1018 u16 vid = adapter->hw.mng_cookie.vlan_id;
1019 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1020
1021 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1022 /* add VID to filter table */
1023 igb_vfta_set(hw, vid, true);
1024 adapter->mng_vlan_id = vid;
1025 } else {
1026 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1027 }
1028
1029 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1030 (vid != old_vid) &&
1031 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1032 /* remove VID from filter table */
1033 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1034 }
1035}
1036
1037/**
1038 * igb_release_hw_control - release control of the h/w to f/w
1039 * @adapter: address of board private structure
1040 *
1041 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1042 * For ASF and Pass Through versions of f/w this means that the
1043 * driver is no longer loaded.
1044 *
1045 **/
1046static void igb_release_hw_control(struct igb_adapter *adapter)
1047{
1048 struct e1000_hw *hw = &adapter->hw;
1049 u32 ctrl_ext;
1050
1051 /* Let firmware take over control of h/w */
1052 ctrl_ext = rd32(E1000_CTRL_EXT);
1053 wr32(E1000_CTRL_EXT,
1054 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1055}
1056
9d5c8243
AK		 1057/**
1058 * igb_get_hw_control - get control of the h/w from f/w
1059 * @adapter: address of board private structure
1060 *
1061 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1062 * For ASF and Pass Through versions of f/w this means that
1063 * the driver is loaded.
1064 *
1065 **/
1066static void igb_get_hw_control(struct igb_adapter *adapter)
1067{
1068 struct e1000_hw *hw = &adapter->hw;
1069 u32 ctrl_ext;
1070
1071 /* Let firmware know the driver has taken over */
1072 ctrl_ext = rd32(E1000_CTRL_EXT);
1073 wr32(E1000_CTRL_EXT,
1074 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1075}
1076
9d5c8243
AK		 1077/**
1078 * igb_configure - configure the hardware for RX and TX
1079 * @adapter: private board structure
1080 **/
1081static void igb_configure(struct igb_adapter *adapter)
1082{
1083 struct net_device *netdev = adapter->netdev;
1084 int i;
1085
1086 igb_get_hw_control(adapter);
ff41f8dc 1087 igb_set_rx_mode(netdev);
9d5c8243
AK		 1088
1089 igb_restore_vlan(adapter);
9d5c8243 1090
85b430b4 1091 igb_setup_tctl(adapter);
06cf2666 1092 igb_setup_mrqc(adapter);
9d5c8243 1093 igb_setup_rctl(adapter);
85b430b4
AD		 1094
1095 igb_configure_tx(adapter);
9d5c8243 1096 igb_configure_rx(adapter);
662d7205
AD		 1097
1098 igb_rx_fifo_flush_82575(&adapter->hw);
1099
c493ea45 1100 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1101 * at least 1 descriptor unused to make sure
1102 * next_to_use != next_to_clean */
1103 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1104 struct igb_ring *ring = adapter->rx_ring[i];
c493ea45 1105 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
9d5c8243
AK		 1106 }
1107
1108
1109 adapter->tx_queue_len = netdev->tx_queue_len;
1110}
1111
88a268c1
NN		 1112/**
1113 * igb_power_up_link - Power up the phy/serdes link
1114 * @adapter: address of board private structure
1115 **/
1116void igb_power_up_link(struct igb_adapter *adapter)
1117{
1118 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1119 igb_power_up_phy_copper(&adapter->hw);
1120 else
1121 igb_power_up_serdes_link_82575(&adapter->hw);
1122}
1123
1124/**
1125 * igb_power_down_link - Power down the phy/serdes link
1126 * @adapter: address of board private structure
1127 */
1128static void igb_power_down_link(struct igb_adapter *adapter)
1129{
1130 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1131 igb_power_down_phy_copper_82575(&adapter->hw);
1132 else
1133 igb_shutdown_serdes_link_82575(&adapter->hw);
1134}
9d5c8243
AK		 1135
1136/**
1137 * igb_up - Open the interface and prepare it to handle traffic
1138 * @adapter: board private structure
1139 **/
9d5c8243
AK		 1140int igb_up(struct igb_adapter *adapter)
1141{
1142 struct e1000_hw *hw = &adapter->hw;
1143 int i;
1144
1145 /* hardware has been reset, we need to reload some things */
1146 igb_configure(adapter);
1147
1148 clear_bit(__IGB_DOWN, &adapter->state);
1149
047e0030
AD		 1150 for (i = 0; i < adapter->num_q_vectors; i++) {
1151 struct igb_q_vector *q_vector = adapter->q_vector[i];
1152 napi_enable(&q_vector->napi);
1153 }
844290e5 1154 if (adapter->msix_entries)
9d5c8243 1155 igb_configure_msix(adapter);
feeb2721
AD		 1156 else
1157 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1158
1159 /* Clear any pending interrupts. */
1160 rd32(E1000_ICR);
1161 igb_irq_enable(adapter);
1162
d4960307
AD		 1163 /* notify VFs that reset has been completed */
1164 if (adapter->vfs_allocated_count) {
1165 u32 reg_data = rd32(E1000_CTRL_EXT);
1166 reg_data |= E1000_CTRL_EXT_PFRSTD;
1167 wr32(E1000_CTRL_EXT, reg_data);
1168 }
1169
4cb9be7a
JB		 1170 netif_tx_start_all_queues(adapter->netdev);
1171
25568a53
AD		 1172 /* start the watchdog. */
1173 hw->mac.get_link_status = 1;
1174 schedule_work(&adapter->watchdog_task);
1175
9d5c8243
AK		 1176 return 0;
1177}
1178
1179void igb_down(struct igb_adapter *adapter)
1180{
9d5c8243 1181 struct net_device *netdev = adapter->netdev;
330a6d6a 1182 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1183 u32 tctl, rctl;
1184 int i;
1185
1186 /* signal that we're down so the interrupt handler does not
1187 * reschedule our watchdog timer */
1188 set_bit(__IGB_DOWN, &adapter->state);
1189
1190 /* disable receives in the hardware */
1191 rctl = rd32(E1000_RCTL);
1192 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1193 /* flush and sleep below */
1194
fd2ea0a7 1195 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1196
1197 /* disable transmits in the hardware */
1198 tctl = rd32(E1000_TCTL);
1199 tctl &= ~E1000_TCTL_EN;
1200 wr32(E1000_TCTL, tctl);
1201 /* flush both disables and wait for them to finish */
1202 wrfl();
1203 msleep(10);
1204
047e0030
AD		 1205 for (i = 0; i < adapter->num_q_vectors; i++) {
1206 struct igb_q_vector *q_vector = adapter->q_vector[i];
1207 napi_disable(&q_vector->napi);
1208 }
9d5c8243 1209
9d5c8243
AK		 1210 igb_irq_disable(adapter);
1211
1212 del_timer_sync(&adapter->watchdog_timer);
1213 del_timer_sync(&adapter->phy_info_timer);
1214
1215 netdev->tx_queue_len = adapter->tx_queue_len;
1216 netif_carrier_off(netdev);
04fe6358
AD		 1217
1218 /* record the stats before reset*/
1219 igb_update_stats(adapter);
1220
9d5c8243
AK		 1221 adapter->link_speed = 0;
1222 adapter->link_duplex = 0;
1223
3023682e
JK		 1224 if (!pci_channel_offline(adapter->pdev))
1225 igb_reset(adapter);
9d5c8243
AK		 1226 igb_clean_all_tx_rings(adapter);
1227 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1228#ifdef CONFIG_IGB_DCA
1229
1230 /* since we reset the hardware DCA settings were cleared */
1231 igb_setup_dca(adapter);
1232#endif
9d5c8243
AK		 1233}
1234
1235void igb_reinit_locked(struct igb_adapter *adapter)
1236{
1237 WARN_ON(in_interrupt());
1238 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1239 msleep(1);
1240 igb_down(adapter);
1241 igb_up(adapter);
1242 clear_bit(__IGB_RESETTING, &adapter->state);
1243}
1244
1245void igb_reset(struct igb_adapter *adapter)
1246{
090b1795 1247 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1248 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1249 struct e1000_mac_info *mac = &hw->mac;
1250 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1251 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1252 u16 hwm;
1253
1254 /* Repartition Pba for greater than 9k mtu
1255 * To take effect CTRL.RST is required.
1256 */
fa4dfae0 1257 switch (mac->type) {
55cac248
AD		 1258 case e1000_82580:
1259 pba = rd32(E1000_RXPBS);
1260 pba = igb_rxpbs_adjust_82580(pba);
1261 break;
fa4dfae0 1262 case e1000_82576:
d249be54
AD		 1263 pba = rd32(E1000_RXPBS);
1264 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1265 break;
1266 case e1000_82575:
1267 default:
1268 pba = E1000_PBA_34K;
1269 break;
2d064c06 1270 }
9d5c8243 1271
2d064c06
AD		 1272 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1273 (mac->type < e1000_82576)) {
9d5c8243
AK		 1274 /* adjust PBA for jumbo frames */
1275 wr32(E1000_PBA, pba);
1276
1277 /* To maintain wire speed transmits, the Tx FIFO should be
1278 * large enough to accommodate two full transmit packets,
1279 * rounded up to the next 1KB and expressed in KB. Likewise,
1280 * the Rx FIFO should be large enough to accommodate at least
1281 * one full receive packet and is similarly rounded up and
1282 * expressed in KB. */
1283 pba = rd32(E1000_PBA);
1284 /* upper 16 bits has Tx packet buffer allocation size in KB */
1285 tx_space = pba >> 16;
1286 /* lower 16 bits has Rx packet buffer allocation size in KB */
1287 pba &= 0xffff;
1288 /* the tx fifo also stores 16 bytes of information about the tx
1289 * but don't include ethernet FCS because hardware appends it */
1290 min_tx_space = (adapter->max_frame_size +
85e8d004 1291 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1292 ETH_FCS_LEN) * 2;
1293 min_tx_space = ALIGN(min_tx_space, 1024);
1294 min_tx_space >>= 10;
1295 /* software strips receive CRC, so leave room for it */
1296 min_rx_space = adapter->max_frame_size;
1297 min_rx_space = ALIGN(min_rx_space, 1024);
1298 min_rx_space >>= 10;
1299
1300 /* If current Tx allocation is less than the min Tx FIFO size,
1301 * and the min Tx FIFO size is less than the current Rx FIFO
1302 * allocation, take space away from current Rx allocation */
1303 if (tx_space < min_tx_space &&
1304 ((min_tx_space - tx_space) < pba)) {
1305 pba = pba - (min_tx_space - tx_space);
1306
1307 /* if short on rx space, rx wins and must trump tx
1308 * adjustment */
1309 if (pba < min_rx_space)
1310 pba = min_rx_space;
1311 }
2d064c06 1312 wr32(E1000_PBA, pba);
9d5c8243 1313 }
9d5c8243
AK		 1314
1315 /* flow control settings */
1316 /* The high water mark must be low enough to fit one full frame
1317 * (or the size used for early receive) above it in the Rx FIFO.
1318 * Set it to the lower of:
1319 * - 90% of the Rx FIFO size, or
1320 * - the full Rx FIFO size minus one full frame */
1321 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1322 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1323
d405ea3e
AD		 1324 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1325 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1326 fc->pause_time = 0xFFFF;
1327 fc->send_xon = 1;
0cce119a 1328 fc->current_mode = fc->requested_mode;
9d5c8243 1329
4ae196df
AD		 1330 /* disable receive for all VFs and wait one second */
1331 if (adapter->vfs_allocated_count) {
1332 int i;
1333 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
f2ca0dbe 1334 adapter->vf_data[i].flags = 0;
4ae196df
AD		 1335
1336 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1337 igb_ping_all_vfs(adapter);
4ae196df
AD		 1338
1339 /* disable transmits and receives */
1340 wr32(E1000_VFRE, 0);
1341 wr32(E1000_VFTE, 0);
1342 }
1343
9d5c8243 1344 /* Allow time for pending master requests to run */
330a6d6a 1345 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1346 wr32(E1000_WUC, 0);
1347
330a6d6a 1348 if (hw->mac.ops.init_hw(hw))
090b1795 1349 dev_err(&pdev->dev, "Hardware Error\n");
9d5c8243 1350
55cac248
AD		 1351 if (hw->mac.type == e1000_82580) {
1352 u32 reg = rd32(E1000_PCIEMISC);
1353 wr32(E1000_PCIEMISC,
1354 reg & ~E1000_PCIEMISC_LX_DECISION);
1355 }
88a268c1
NN		 1356 if (!netif_running(adapter->netdev))
1357 igb_power_down_link(adapter);
1358
9d5c8243
AK		 1359 igb_update_mng_vlan(adapter);
1360
1361 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1362 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1363
330a6d6a 1364 igb_get_phy_info(hw);
9d5c8243
AK		 1365}
1366
2e5c6922 1367static const struct net_device_ops igb_netdev_ops = {
559e9c49 1368 .ndo_open = igb_open,
2e5c6922 1369 .ndo_stop = igb_close,
00829823 1370 .ndo_start_xmit = igb_xmit_frame_adv,
2e5c6922 1371 .ndo_get_stats = igb_get_stats,
ff41f8dc
AD		 1372 .ndo_set_rx_mode = igb_set_rx_mode,
1373 .ndo_set_multicast_list = igb_set_rx_mode,
2e5c6922
SH		 1374 .ndo_set_mac_address = igb_set_mac,
1375 .ndo_change_mtu = igb_change_mtu,
1376 .ndo_do_ioctl = igb_ioctl,
1377 .ndo_tx_timeout = igb_tx_timeout,
1378 .ndo_validate_addr = eth_validate_addr,
1379 .ndo_vlan_rx_register = igb_vlan_rx_register,
1380 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1381 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1382 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1383 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1384 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1385 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1386#ifdef CONFIG_NET_POLL_CONTROLLER
1387 .ndo_poll_controller = igb_netpoll,
1388#endif
1389};
1390
9d5c8243
AK		 1391/**
1392 * igb_probe - Device Initialization Routine
1393 * @pdev: PCI device information struct
1394 * @ent: entry in igb_pci_tbl
1395 *
1396 * Returns 0 on success, negative on failure
1397 *
1398 * igb_probe initializes an adapter identified by a pci_dev structure.
1399 * The OS initialization, configuring of the adapter private structure,
1400 * and a hardware reset occur.
1401 **/
1402static int __devinit igb_probe(struct pci_dev *pdev,
1403 const struct pci_device_id *ent)
1404{
1405 struct net_device *netdev;
1406 struct igb_adapter *adapter;
1407 struct e1000_hw *hw;
4337e993
AD		 1408 u16 eeprom_data = 0;
1409 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1410 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1411 unsigned long mmio_start, mmio_len;
2d6a5e95 1412 int err, pci_using_dac;
9d5c8243
AK		 1413 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1414 u32 part_num;
1415
aed5dec3 1416 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1417 if (err)
1418 return err;
1419
1420 pci_using_dac = 0;
6a35528a 1421 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243 1422 if (!err) {
6a35528a 1423 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243
AK		 1424 if (!err)
1425 pci_using_dac = 1;
1426 } else {
284901a9 1427 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243 1428 if (err) {
284901a9 1429 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243
AK		 1430 if (err) {
1431 dev_err(&pdev->dev, "No usable DMA "
1432 "configuration, aborting\n");
1433 goto err_dma;
1434 }
1435 }
1436 }
1437
aed5dec3
AD		 1438 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1439 IORESOURCE_MEM),
1440 igb_driver_name);
9d5c8243
AK		 1441 if (err)
1442 goto err_pci_reg;
1443
19d5afd4 1444 pci_enable_pcie_error_reporting(pdev);
40a914fa 1445
9d5c8243 1446 pci_set_master(pdev);
c682fc23 1447 pci_save_state(pdev);
9d5c8243
AK		 1448
1449 err = -ENOMEM;
1bfaf07b
AD		 1450 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1451 IGB_ABS_MAX_TX_QUEUES);
9d5c8243
AK		 1452 if (!netdev)
1453 goto err_alloc_etherdev;
1454
1455 SET_NETDEV_DEV(netdev, &pdev->dev);
1456
1457 pci_set_drvdata(pdev, netdev);
1458 adapter = netdev_priv(netdev);
1459 adapter->netdev = netdev;
1460 adapter->pdev = pdev;
1461 hw = &adapter->hw;
1462 hw->back = adapter;
1463 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1464
1465 mmio_start = pci_resource_start(pdev, 0);
1466 mmio_len = pci_resource_len(pdev, 0);
1467
1468 err = -EIO;
28b0759c
AD		 1469 hw->hw_addr = ioremap(mmio_start, mmio_len);
1470 if (!hw->hw_addr)
9d5c8243
AK		 1471 goto err_ioremap;
1472
2e5c6922 1473 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1474 igb_set_ethtool_ops(netdev);
9d5c8243 1475 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1476
1477 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1478
1479 netdev->mem_start = mmio_start;
1480 netdev->mem_end = mmio_start + mmio_len;
1481
9d5c8243
AK		 1482 /* PCI config space info */
1483 hw->vendor_id = pdev->vendor;
1484 hw->device_id = pdev->device;
1485 hw->revision_id = pdev->revision;
1486 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1487 hw->subsystem_device_id = pdev->subsystem_device;
1488
9d5c8243
AK		 1489 /* Copy the default MAC, PHY and NVM function pointers */
1490 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1491 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1492 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1493 /* Initialize skew-specific constants */
1494 err = ei->get_invariants(hw);
1495 if (err)
450c87c8 1496 goto err_sw_init;
9d5c8243 1497
450c87c8 1498 /* setup the private structure */
9d5c8243
AK		 1499 err = igb_sw_init(adapter);
1500 if (err)
1501 goto err_sw_init;
1502
1503 igb_get_bus_info_pcie(hw);
1504
1505 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 1506
1507 /* Copper options */
1508 if (hw->phy.media_type == e1000_media_type_copper) {
1509 hw->phy.mdix = AUTO_ALL_MODES;
1510 hw->phy.disable_polarity_correction = false;
1511 hw->phy.ms_type = e1000_ms_hw_default;
1512 }
1513
1514 if (igb_check_reset_block(hw))
1515 dev_info(&pdev->dev,
1516 "PHY reset is blocked due to SOL/IDER session.\n");
1517
1518 netdev->features = NETIF_F_SG |
7d8eb29e 1519 NETIF_F_IP_CSUM |
9d5c8243
AK		 1520 NETIF_F_HW_VLAN_TX |
1521 NETIF_F_HW_VLAN_RX |
1522 NETIF_F_HW_VLAN_FILTER;
1523
7d8eb29e 1524 netdev->features |= NETIF_F_IPV6_CSUM;
9d5c8243 1525 netdev->features |= NETIF_F_TSO;
9d5c8243 1526 netdev->features |= NETIF_F_TSO6;
5c0999b7 1527 netdev->features |= NETIF_F_GRO;
d3352520 1528
48f29ffc
JK		 1529 netdev->vlan_features |= NETIF_F_TSO;
1530 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 1531 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 1532 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 1533 netdev->vlan_features |= NETIF_F_SG;
1534
9d5c8243
AK		 1535 if (pci_using_dac)
1536 netdev->features |= NETIF_F_HIGHDMA;
1537
5b043fb0 1538 if (hw->mac.type >= e1000_82576)
b9473560
JB		 1539 netdev->features |= NETIF_F_SCTP_CSUM;
1540
330a6d6a 1541 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 1542
1543 /* before reading the NVM, reset the controller to put the device in a
1544 * known good starting state */
1545 hw->mac.ops.reset_hw(hw);
1546
1547 /* make sure the NVM is good */
1548 if (igb_validate_nvm_checksum(hw) < 0) {
1549 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1550 err = -EIO;
1551 goto err_eeprom;
1552 }
1553
1554 /* copy the MAC address out of the NVM */
1555 if (hw->mac.ops.read_mac_addr(hw))
1556 dev_err(&pdev->dev, "NVM Read Error\n");
1557
1558 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1559 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1560
1561 if (!is_valid_ether_addr(netdev->perm_addr)) {
1562 dev_err(&pdev->dev, "Invalid MAC Address\n");
1563 err = -EIO;
1564 goto err_eeprom;
1565 }
1566
0e340485
AD		 1567 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1568 (unsigned long) adapter);
1569 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1570 (unsigned long) adapter);
9d5c8243
AK		 1571
1572 INIT_WORK(&adapter->reset_task, igb_reset_task);
1573 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1574
450c87c8 1575 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 1576 adapter->fc_autoneg = true;
1577 hw->mac.autoneg = true;
1578 hw->phy.autoneg_advertised = 0x2f;
1579
0cce119a
AD		 1580 hw->fc.requested_mode = e1000_fc_default;
1581 hw->fc.current_mode = e1000_fc_default;
9d5c8243 1582
9d5c8243
AK		 1583 igb_validate_mdi_setting(hw);
1584
9d5c8243
AK		 1585 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1586 * enable the ACPI Magic Packet filter
1587 */
1588
a2cf8b6c 1589 if (hw->bus.func == 0)
312c75ae 1590 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
55cac248
AD		 1591 else if (hw->mac.type == e1000_82580)
1592 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1593 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1594 &eeprom_data);
a2cf8b6c
AD		 1595 else if (hw->bus.func == 1)
1596 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 1597
1598 if (eeprom_data & eeprom_apme_mask)
1599 adapter->eeprom_wol |= E1000_WUFC_MAG;
1600
1601 /* now that we have the eeprom settings, apply the special cases where
1602 * the eeprom may be wrong or the board simply won't support wake on
1603 * lan on a particular port */
1604 switch (pdev->device) {
1605 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1606 adapter->eeprom_wol = 0;
1607 break;
1608 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 1609 case E1000_DEV_ID_82576_FIBER:
1610 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 1611 /* Wake events only supported on port A for dual fiber
1612 * regardless of eeprom setting */
1613 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1614 adapter->eeprom_wol = 0;
1615 break;
c8ea5ea9
AD		 1616 case E1000_DEV_ID_82576_QUAD_COPPER:
1617 /* if quad port adapter, disable WoL on all but port A */
1618 if (global_quad_port_a != 0)
1619 adapter->eeprom_wol = 0;
1620 else
1621 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1622 /* Reset for multiple quad port adapters */
1623 if (++global_quad_port_a == 4)
1624 global_quad_port_a = 0;
1625 break;
9d5c8243
AK		 1626 }
1627
1628 /* initialize the wol settings based on the eeprom settings */
1629 adapter->wol = adapter->eeprom_wol;
e1b86d84 1630 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 1631
1632 /* reset the hardware with the new settings */
1633 igb_reset(adapter);
1634
1635 /* let the f/w know that the h/w is now under the control of the
1636 * driver. */
1637 igb_get_hw_control(adapter);
1638
9d5c8243
AK		 1639 strcpy(netdev->name, "eth%d");
1640 err = register_netdev(netdev);
1641 if (err)
1642 goto err_register;
1643
b168dfc5
JB		 1644 /* carrier off reporting is important to ethtool even BEFORE open */
1645 netif_carrier_off(netdev);
1646
421e02f0 1647#ifdef CONFIG_IGB_DCA
bbd98fe4 1648 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 1649 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 1650 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 1651 igb_setup_dca(adapter);
1652 }
fe4506b6 1653
38c845c7 1654#endif
9d5c8243
AK		 1655 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1656 /* print bus type/speed/width info */
7c510e4b 1657 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 1658 netdev->name,
559e9c49
AD		 1659 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
1660 "unknown"),
59c3de89
AD		 1661 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1662 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1663 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1664 "unknown"),
7c510e4b 1665 netdev->dev_addr);
9d5c8243
AK		 1666
1667 igb_read_part_num(hw, &part_num);
1668 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1669 (part_num >> 8), (part_num & 0xff));
1670
1671 dev_info(&pdev->dev,
1672 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1673 adapter->msix_entries ? "MSI-X" :
7dfc16fa 1674 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243
AK		 1675 adapter->num_rx_queues, adapter->num_tx_queues);
1676
9d5c8243
AK		 1677 return 0;
1678
1679err_register:
1680 igb_release_hw_control(adapter);
1681err_eeprom:
1682 if (!igb_check_reset_block(hw))
f5f4cf08 1683 igb_reset_phy(hw);
9d5c8243
AK		 1684
1685 if (hw->flash_address)
1686 iounmap(hw->flash_address);
9d5c8243 1687err_sw_init:
047e0030 1688 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 1689 iounmap(hw->hw_addr);
1690err_ioremap:
1691 free_netdev(netdev);
1692err_alloc_etherdev:
559e9c49
AD		 1693 pci_release_selected_regions(pdev,
1694 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 1695err_pci_reg:
1696err_dma:
1697 pci_disable_device(pdev);
1698 return err;
1699}
1700
1701/**
1702 * igb_remove - Device Removal Routine
1703 * @pdev: PCI device information struct
1704 *
1705 * igb_remove is called by the PCI subsystem to alert the driver
1706 * that it should release a PCI device. The could be caused by a
1707 * Hot-Plug event, or because the driver is going to be removed from
1708 * memory.
1709 **/
1710static void __devexit igb_remove(struct pci_dev *pdev)
1711{
1712 struct net_device *netdev = pci_get_drvdata(pdev);
1713 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 1714 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1715
1716 /* flush_scheduled work may reschedule our watchdog task, so
1717 * explicitly disable watchdog tasks from being rescheduled */
1718 set_bit(__IGB_DOWN, &adapter->state);
1719 del_timer_sync(&adapter->watchdog_timer);
1720 del_timer_sync(&adapter->phy_info_timer);
1721
1722 flush_scheduled_work();
1723
421e02f0 1724#ifdef CONFIG_IGB_DCA
7dfc16fa 1725 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 1726 dev_info(&pdev->dev, "DCA disabled\n");
1727 dca_remove_requester(&pdev->dev);
7dfc16fa 1728 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 1729 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 1730 }
1731#endif
1732
9d5c8243
AK		 1733 /* Release control of h/w to f/w. If f/w is AMT enabled, this
1734 * would have already happened in close and is redundant. */
1735 igb_release_hw_control(adapter);
1736
1737 unregister_netdev(netdev);
1738
047e0030 1739 igb_clear_interrupt_scheme(adapter);
9d5c8243 1740
37680117
AD		 1741#ifdef CONFIG_PCI_IOV
1742 /* reclaim resources allocated to VFs */
1743 if (adapter->vf_data) {
1744 /* disable iov and allow time for transactions to clear */
1745 pci_disable_sriov(pdev);
1746 msleep(500);
1747
1748 kfree(adapter->vf_data);
1749 adapter->vf_data = NULL;
1750 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1751 msleep(100);
1752 dev_info(&pdev->dev, "IOV Disabled\n");
1753 }
1754#endif
559e9c49 1755
28b0759c
AD		 1756 iounmap(hw->hw_addr);
1757 if (hw->flash_address)
1758 iounmap(hw->flash_address);
559e9c49
AD		 1759 pci_release_selected_regions(pdev,
1760 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 1761
1762 free_netdev(netdev);
1763
19d5afd4 1764 pci_disable_pcie_error_reporting(pdev);
40a914fa 1765
9d5c8243
AK		 1766 pci_disable_device(pdev);
1767}
1768
a6b623e0
AD		 1769/**
1770 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
1771 * @adapter: board private structure to initialize
1772 *
1773 * This function initializes the vf specific data storage and then attempts to
1774 * allocate the VFs. The reason for ordering it this way is because it is much
1775 * mor expensive time wise to disable SR-IOV than it is to allocate and free
1776 * the memory for the VFs.
1777 **/
1778static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
1779{
1780#ifdef CONFIG_PCI_IOV
1781 struct pci_dev *pdev = adapter->pdev;
1782
1783 if (adapter->vfs_allocated_count > 7)
1784 adapter->vfs_allocated_count = 7;
1785
1786 if (adapter->vfs_allocated_count) {
1787 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
1788 sizeof(struct vf_data_storage),
1789 GFP_KERNEL);
1790 /* if allocation failed then we do not support SR-IOV */
1791 if (!adapter->vf_data) {
1792 adapter->vfs_allocated_count = 0;
1793 dev_err(&pdev->dev, "Unable to allocate memory for VF "
1794 "Data Storage\n");
1795 }
1796 }
1797
1798 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
1799 kfree(adapter->vf_data);
1800 adapter->vf_data = NULL;
1801#endif /* CONFIG_PCI_IOV */
1802 adapter->vfs_allocated_count = 0;
1803#ifdef CONFIG_PCI_IOV
1804 } else {
1805 unsigned char mac_addr[ETH_ALEN];
1806 int i;
1807 dev_info(&pdev->dev, "%d vfs allocated\n",
1808 adapter->vfs_allocated_count);
1809 for (i = 0; i < adapter->vfs_allocated_count; i++) {
1810 random_ether_addr(mac_addr);
1811 igb_set_vf_mac(adapter, i, mac_addr);
1812 }
1813 }
1814#endif /* CONFIG_PCI_IOV */
1815}
1816
115f459a
AD		 1817
1818/**
1819 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
1820 * @adapter: board private structure to initialize
1821 *
1822 * igb_init_hw_timer initializes the function pointer and values for the hw
1823 * timer found in hardware.
1824 **/
1825static void igb_init_hw_timer(struct igb_adapter *adapter)
1826{
1827 struct e1000_hw *hw = &adapter->hw;
1828
1829 switch (hw->mac.type) {
55cac248
AD		 1830 case e1000_82580:
1831 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1832 adapter->cycles.read = igb_read_clock;
1833 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1834 adapter->cycles.mult = 1;
1835 /*
1836 * The 82580 timesync updates the system timer every 8ns by 8ns
1837 * and the value cannot be shifted. Instead we need to shift
1838 * the registers to generate a 64bit timer value. As a result
1839 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
1840 * 24 in order to generate a larger value for synchronization.
1841 */
1842 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
1843 /* disable system timer temporarily by setting bit 31 */
1844 wr32(E1000_TSAUXC, 0x80000000);
1845 wrfl();
1846
1847 /* Set registers so that rollover occurs soon to test this. */
1848 wr32(E1000_SYSTIMR, 0x00000000);
1849 wr32(E1000_SYSTIML, 0x80000000);
1850 wr32(E1000_SYSTIMH, 0x000000FF);
1851 wrfl();
1852
1853 /* enable system timer by clearing bit 31 */
1854 wr32(E1000_TSAUXC, 0x0);
1855 wrfl();
1856
1857 timecounter_init(&adapter->clock,
1858 &adapter->cycles,
1859 ktime_to_ns(ktime_get_real()));
1860 /*
1861 * Synchronize our NIC clock against system wall clock. NIC
1862 * time stamp reading requires ~3us per sample, each sample
1863 * was pretty stable even under load => only require 10
1864 * samples for each offset comparison.
1865 */
1866 memset(&adapter->compare, 0, sizeof(adapter->compare));
1867 adapter->compare.source = &adapter->clock;
1868 adapter->compare.target = ktime_get_real;
1869 adapter->compare.num_samples = 10;
1870 timecompare_update(&adapter->compare, 0);
1871 break;
115f459a
AD		 1872 case e1000_82576:
1873 /*
1874 * Initialize hardware timer: we keep it running just in case
1875 * that some program needs it later on.
1876 */
1877 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1878 adapter->cycles.read = igb_read_clock;
1879 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1880 adapter->cycles.mult = 1;
1881 /**
1882 * Scale the NIC clock cycle by a large factor so that
1883 * relatively small clock corrections can be added or
1884 * substracted at each clock tick. The drawbacks of a large
1885 * factor are a) that the clock register overflows more quickly
1886 * (not such a big deal) and b) that the increment per tick has
1887 * to fit into 24 bits. As a result we need to use a shift of
1888 * 19 so we can fit a value of 16 into the TIMINCA register.
1889 */
1890 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
1891 wr32(E1000_TIMINCA,
1892 (1 << E1000_TIMINCA_16NS_SHIFT) |
1893 (16 << IGB_82576_TSYNC_SHIFT));
1894
1895 /* Set registers so that rollover occurs soon to test this. */
1896 wr32(E1000_SYSTIML, 0x00000000);
1897 wr32(E1000_SYSTIMH, 0xFF800000);
1898 wrfl();
1899
1900 timecounter_init(&adapter->clock,
1901 &adapter->cycles,
1902 ktime_to_ns(ktime_get_real()));
1903 /*
1904 * Synchronize our NIC clock against system wall clock. NIC
1905 * time stamp reading requires ~3us per sample, each sample
1906 * was pretty stable even under load => only require 10
1907 * samples for each offset comparison.
1908 */
1909 memset(&adapter->compare, 0, sizeof(adapter->compare));
1910 adapter->compare.source = &adapter->clock;
1911 adapter->compare.target = ktime_get_real;
1912 adapter->compare.num_samples = 10;
1913 timecompare_update(&adapter->compare, 0);
1914 break;
1915 case e1000_82575:
1916 /* 82575 does not support timesync */
1917 default:
1918 break;
1919 }
1920
1921}
1922
9d5c8243
AK		 1923/**
1924 * igb_sw_init - Initialize general software structures (struct igb_adapter)
1925 * @adapter: board private structure to initialize
1926 *
1927 * igb_sw_init initializes the Adapter private data structure.
1928 * Fields are initialized based on PCI device information and
1929 * OS network device settings (MTU size).
1930 **/
1931static int __devinit igb_sw_init(struct igb_adapter *adapter)
1932{
1933 struct e1000_hw *hw = &adapter->hw;
1934 struct net_device *netdev = adapter->netdev;
1935 struct pci_dev *pdev = adapter->pdev;
1936
1937 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
1938
68fd9910
AD		 1939 adapter->tx_ring_count = IGB_DEFAULT_TXD;
1940 adapter->rx_ring_count = IGB_DEFAULT_RXD;
4fc82adf
AD		 1941 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
1942 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
1943
9d5c8243
AK		 1944 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1945 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1946
a6b623e0
AD		 1947#ifdef CONFIG_PCI_IOV
1948 if (hw->mac.type == e1000_82576)
1949 adapter->vfs_allocated_count = max_vfs;
1950
1951#endif /* CONFIG_PCI_IOV */
a99955fc
AD		 1952 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
1953
1954 /*
1955 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
1956 * then we should combine the queues into a queue pair in order to
1957 * conserve interrupts due to limited supply
1958 */
1959 if ((adapter->rss_queues > 4) ||
1960 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
1961 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1962
a6b623e0 1963 /* This call may decrease the number of queues */
047e0030 1964 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 1965 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1966 return -ENOMEM;
1967 }
1968
115f459a 1969 igb_init_hw_timer(adapter);
a6b623e0
AD		 1970 igb_probe_vfs(adapter);
1971
9d5c8243
AK		 1972 /* Explicitly disable IRQ since the NIC can be in any state. */
1973 igb_irq_disable(adapter);
1974
1975 set_bit(__IGB_DOWN, &adapter->state);
1976 return 0;
1977}
1978
1979/**
1980 * igb_open - Called when a network interface is made active
1981 * @netdev: network interface device structure
1982 *
1983 * Returns 0 on success, negative value on failure
1984 *
1985 * The open entry point is called when a network interface is made
1986 * active by the system (IFF_UP). At this point all resources needed
1987 * for transmit and receive operations are allocated, the interrupt
1988 * handler is registered with the OS, the watchdog timer is started,
1989 * and the stack is notified that the interface is ready.
1990 **/
1991static int igb_open(struct net_device *netdev)
1992{
1993 struct igb_adapter *adapter = netdev_priv(netdev);
1994 struct e1000_hw *hw = &adapter->hw;
1995 int err;
1996 int i;
1997
1998 /* disallow open during test */
1999 if (test_bit(__IGB_TESTING, &adapter->state))
2000 return -EBUSY;
2001
b168dfc5
JB		 2002 netif_carrier_off(netdev);
2003
9d5c8243
AK		 2004 /* allocate transmit descriptors */
2005 err = igb_setup_all_tx_resources(adapter);
2006 if (err)
2007 goto err_setup_tx;
2008
2009 /* allocate receive descriptors */
2010 err = igb_setup_all_rx_resources(adapter);
2011 if (err)
2012 goto err_setup_rx;
2013
88a268c1 2014 igb_power_up_link(adapter);
9d5c8243 2015
9d5c8243
AK		 2016 /* before we allocate an interrupt, we must be ready to handle it.
2017 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2018 * as soon as we call pci_request_irq, so we have to setup our
2019 * clean_rx handler before we do so. */
2020 igb_configure(adapter);
2021
2022 err = igb_request_irq(adapter);
2023 if (err)
2024 goto err_req_irq;
2025
2026 /* From here on the code is the same as igb_up() */
2027 clear_bit(__IGB_DOWN, &adapter->state);
2028
047e0030
AD		 2029 for (i = 0; i < adapter->num_q_vectors; i++) {
2030 struct igb_q_vector *q_vector = adapter->q_vector[i];
2031 napi_enable(&q_vector->napi);
2032 }
9d5c8243
AK		 2033
2034 /* Clear any pending interrupts. */
2035 rd32(E1000_ICR);
844290e5
PW		 2036
2037 igb_irq_enable(adapter);
2038
d4960307
AD		 2039 /* notify VFs that reset has been completed */
2040 if (adapter->vfs_allocated_count) {
2041 u32 reg_data = rd32(E1000_CTRL_EXT);
2042 reg_data |= E1000_CTRL_EXT_PFRSTD;
2043 wr32(E1000_CTRL_EXT, reg_data);
2044 }
2045
d55b53ff
JK		 2046 netif_tx_start_all_queues(netdev);
2047
25568a53
AD		 2048 /* start the watchdog. */
2049 hw->mac.get_link_status = 1;
2050 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2051
2052 return 0;
2053
2054err_req_irq:
2055 igb_release_hw_control(adapter);
88a268c1 2056 igb_power_down_link(adapter);
9d5c8243
AK		 2057 igb_free_all_rx_resources(adapter);
2058err_setup_rx:
2059 igb_free_all_tx_resources(adapter);
2060err_setup_tx:
2061 igb_reset(adapter);
2062
2063 return err;
2064}
2065
2066/**
2067 * igb_close - Disables a network interface
2068 * @netdev: network interface device structure
2069 *
2070 * Returns 0, this is not allowed to fail
2071 *
2072 * The close entry point is called when an interface is de-activated
2073 * by the OS. The hardware is still under the driver's control, but
2074 * needs to be disabled. A global MAC reset is issued to stop the
2075 * hardware, and all transmit and receive resources are freed.
2076 **/
2077static int igb_close(struct net_device *netdev)
2078{
2079 struct igb_adapter *adapter = netdev_priv(netdev);
2080
2081 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2082 igb_down(adapter);
2083
2084 igb_free_irq(adapter);
2085
2086 igb_free_all_tx_resources(adapter);
2087 igb_free_all_rx_resources(adapter);
2088
9d5c8243
AK		 2089 return 0;
2090}
2091
2092/**
2093 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2094 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2095 *
2096 * Return 0 on success, negative on failure
2097 **/
80785298 2098int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2099{
80785298 2100 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 2101 int size;
2102
2103 size = sizeof(struct igb_buffer) * tx_ring->count;
2104 tx_ring->buffer_info = vmalloc(size);
2105 if (!tx_ring->buffer_info)
2106 goto err;
2107 memset(tx_ring->buffer_info, 0, size);
2108
2109 /* round up to nearest 4K */
85e8d004 2110 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2111 tx_ring->size = ALIGN(tx_ring->size, 4096);
2112
439705e1
AD		 2113 tx_ring->desc = pci_alloc_consistent(pdev,
2114 tx_ring->size,
9d5c8243
AK		 2115 &tx_ring->dma);
2116
2117 if (!tx_ring->desc)
2118 goto err;
2119
9d5c8243
AK		 2120 tx_ring->next_to_use = 0;
2121 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2122 return 0;
2123
2124err:
2125 vfree(tx_ring->buffer_info);
047e0030 2126 dev_err(&pdev->dev,
9d5c8243
AK		 2127 "Unable to allocate memory for the transmit descriptor ring\n");
2128 return -ENOMEM;
2129}
2130
2131/**
2132 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2133 * (Descriptors) for all queues
2134 * @adapter: board private structure
2135 *
2136 * Return 0 on success, negative on failure
2137 **/
2138static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2139{
439705e1 2140 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2141 int i, err = 0;
2142
2143 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2144 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2145 if (err) {
439705e1 2146 dev_err(&pdev->dev,
9d5c8243
AK		 2147 "Allocation for Tx Queue %u failed\n", i);
2148 for (i--; i >= 0; i--)
3025a446 2149 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2150 break;
2151 }
2152 }
2153
a99955fc 2154 for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
439705e1 2155 int r_idx = i % adapter->num_tx_queues;
3025a446 2156 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
eebbbdba 2157 }
9d5c8243
AK		 2158 return err;
2159}
2160
2161/**
85b430b4
AD		 2162 * igb_setup_tctl - configure the transmit control registers
2163 * @adapter: Board private structure
9d5c8243 2164 **/
d7ee5b3a 2165void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2166{
9d5c8243
AK		 2167 struct e1000_hw *hw = &adapter->hw;
2168 u32 tctl;
9d5c8243 2169
85b430b4
AD		 2170 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2171 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2172
2173 /* Program the Transmit Control Register */
9d5c8243
AK		 2174 tctl = rd32(E1000_TCTL);
2175 tctl &= ~E1000_TCTL_CT;
2176 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2177 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2178
2179 igb_config_collision_dist(hw);
2180
9d5c8243
AK		 2181 /* Enable transmits */
2182 tctl |= E1000_TCTL_EN;
2183
2184 wr32(E1000_TCTL, tctl);
2185}
2186
85b430b4
AD		 2187/**
2188 * igb_configure_tx_ring - Configure transmit ring after Reset
2189 * @adapter: board private structure
2190 * @ring: tx ring to configure
2191 *
2192 * Configure a transmit ring after a reset.
2193 **/
d7ee5b3a
AD		 2194void igb_configure_tx_ring(struct igb_adapter *adapter,
2195 struct igb_ring *ring)
85b430b4
AD		 2196{
2197 struct e1000_hw *hw = &adapter->hw;
2198 u32 txdctl;
2199 u64 tdba = ring->dma;
2200 int reg_idx = ring->reg_idx;
2201
2202 /* disable the queue */
2203 txdctl = rd32(E1000_TXDCTL(reg_idx));
2204 wr32(E1000_TXDCTL(reg_idx),
2205 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2206 wrfl();
2207 mdelay(10);
2208
2209 wr32(E1000_TDLEN(reg_idx),
2210 ring->count * sizeof(union e1000_adv_tx_desc));
2211 wr32(E1000_TDBAL(reg_idx),
2212 tdba & 0x00000000ffffffffULL);
2213 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2214
fce99e34
AD		 2215 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2216 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2217 writel(0, ring->head);
2218 writel(0, ring->tail);
85b430b4
AD		 2219
2220 txdctl |= IGB_TX_PTHRESH;
2221 txdctl |= IGB_TX_HTHRESH << 8;
2222 txdctl |= IGB_TX_WTHRESH << 16;
2223
2224 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2225 wr32(E1000_TXDCTL(reg_idx), txdctl);
2226}
2227
2228/**
2229 * igb_configure_tx - Configure transmit Unit after Reset
2230 * @adapter: board private structure
2231 *
2232 * Configure the Tx unit of the MAC after a reset.
2233 **/
2234static void igb_configure_tx(struct igb_adapter *adapter)
2235{
2236 int i;
2237
2238 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2239 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2240}
2241
9d5c8243
AK		 2242/**
2243 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2244 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2245 *
2246 * Returns 0 on success, negative on failure
2247 **/
80785298 2248int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2249{
80785298 2250 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 2251 int size, desc_len;
2252
2253 size = sizeof(struct igb_buffer) * rx_ring->count;
2254 rx_ring->buffer_info = vmalloc(size);
2255 if (!rx_ring->buffer_info)
2256 goto err;
2257 memset(rx_ring->buffer_info, 0, size);
2258
2259 desc_len = sizeof(union e1000_adv_rx_desc);
2260
2261 /* Round up to nearest 4K */
2262 rx_ring->size = rx_ring->count * desc_len;
2263 rx_ring->size = ALIGN(rx_ring->size, 4096);
2264
2265 rx_ring->desc = pci_alloc_consistent(pdev, rx_ring->size,
2266 &rx_ring->dma);
2267
2268 if (!rx_ring->desc)
2269 goto err;
2270
2271 rx_ring->next_to_clean = 0;
2272 rx_ring->next_to_use = 0;
9d5c8243 2273
9d5c8243
AK		 2274 return 0;
2275
2276err:
2277 vfree(rx_ring->buffer_info);
439705e1 2278 rx_ring->buffer_info = NULL;
80785298 2279 dev_err(&pdev->dev, "Unable to allocate memory for "
9d5c8243
AK		 2280 "the receive descriptor ring\n");
2281 return -ENOMEM;
2282}
2283
2284/**
2285 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2286 * (Descriptors) for all queues
2287 * @adapter: board private structure
2288 *
2289 * Return 0 on success, negative on failure
2290 **/
2291static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2292{
439705e1 2293 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2294 int i, err = 0;
2295
2296 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2297 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2298 if (err) {
439705e1 2299 dev_err(&pdev->dev,
9d5c8243
AK		 2300 "Allocation for Rx Queue %u failed\n", i);
2301 for (i--; i >= 0; i--)
3025a446 2302 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2303 break;
2304 }
2305 }
2306
2307 return err;
2308}
2309
06cf2666
AD		 2310/**
2311 * igb_setup_mrqc - configure the multiple receive queue control registers
2312 * @adapter: Board private structure
2313 **/
2314static void igb_setup_mrqc(struct igb_adapter *adapter)
2315{
2316 struct e1000_hw *hw = &adapter->hw;
2317 u32 mrqc, rxcsum;
2318 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2319 union e1000_reta {
2320 u32 dword;
2321 u8 bytes[4];
2322 } reta;
2323 static const u8 rsshash[40] = {
2324 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2325 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2326 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2327 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2328
2329 /* Fill out hash function seeds */
2330 for (j = 0; j < 10; j++) {
2331 u32 rsskey = rsshash[(j * 4)];
2332 rsskey |= rsshash[(j * 4) + 1] << 8;
2333 rsskey |= rsshash[(j * 4) + 2] << 16;
2334 rsskey |= rsshash[(j * 4) + 3] << 24;
2335 array_wr32(E1000_RSSRK(0), j, rsskey);
2336 }
2337
a99955fc 2338 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2339
2340 if (adapter->vfs_allocated_count) {
2341 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2342 switch (hw->mac.type) {
55cac248
AD		 2343 case e1000_82580:
2344 num_rx_queues = 1;
2345 shift = 0;
2346 break;
06cf2666
AD		 2347 case e1000_82576:
2348 shift = 3;
2349 num_rx_queues = 2;
2350 break;
2351 case e1000_82575:
2352 shift = 2;
2353 shift2 = 6;
2354 default:
2355 break;
2356 }
2357 } else {
2358 if (hw->mac.type == e1000_82575)
2359 shift = 6;
2360 }
2361
2362 for (j = 0; j < (32 * 4); j++) {
2363 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2364 if (shift2)
2365 reta.bytes[j & 3] |= num_rx_queues << shift2;
2366 if ((j & 3) == 3)
2367 wr32(E1000_RETA(j >> 2), reta.dword);
2368 }
2369
2370 /*
2371 * Disable raw packet checksumming so that RSS hash is placed in
2372 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2373 * offloads as they are enabled by default
2374 */
2375 rxcsum = rd32(E1000_RXCSUM);
2376 rxcsum |= E1000_RXCSUM_PCSD;
2377
2378 if (adapter->hw.mac.type >= e1000_82576)
2379 /* Enable Receive Checksum Offload for SCTP */
2380 rxcsum |= E1000_RXCSUM_CRCOFL;
2381
2382 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2383 wr32(E1000_RXCSUM, rxcsum);
2384
2385 /* If VMDq is enabled then we set the appropriate mode for that, else
2386 * we default to RSS so that an RSS hash is calculated per packet even
2387 * if we are only using one queue */
2388 if (adapter->vfs_allocated_count) {
2389 if (hw->mac.type > e1000_82575) {
2390 /* Set the default pool for the PF's first queue */
2391 u32 vtctl = rd32(E1000_VT_CTL);
2392 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2393 E1000_VT_CTL_DISABLE_DEF_POOL);
2394 vtctl |= adapter->vfs_allocated_count <<
2395 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2396 wr32(E1000_VT_CTL, vtctl);
2397 }
a99955fc 2398 if (adapter->rss_queues > 1)
06cf2666
AD		 2399 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2400 else
2401 mrqc = E1000_MRQC_ENABLE_VMDQ;
2402 } else {
2403 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2404 }
2405 igb_vmm_control(adapter);
2406
2407 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2408 E1000_MRQC_RSS_FIELD_IPV4_TCP);
2409 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
2410 E1000_MRQC_RSS_FIELD_IPV6_TCP);
2411 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
2412 E1000_MRQC_RSS_FIELD_IPV6_UDP);
2413 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2414 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2415
2416 wr32(E1000_MRQC, mrqc);
2417}
2418
9d5c8243
AK		 2419/**
2420 * igb_setup_rctl - configure the receive control registers
2421 * @adapter: Board private structure
2422 **/
d7ee5b3a 2423void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2424{
2425 struct e1000_hw *hw = &adapter->hw;
2426 u32 rctl;
9d5c8243
AK		 2427
2428 rctl = rd32(E1000_RCTL);
2429
2430 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2431 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2432
69d728ba 2433 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2434 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2435
87cb7e8c
AK		 2436 /*
2437 * enable stripping of CRC. It's unlikely this will break BMC
2438 * redirection as it did with e1000. Newer features require
2439 * that the HW strips the CRC.
73cd78f1 2440 */
87cb7e8c 2441 rctl |= E1000_RCTL_SECRC;
9d5c8243 2442
559e9c49 2443 /* disable store bad packets and clear size bits. */
ec54d7d6 2444 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2445
6ec43fe6
AD		 2446 /* enable LPE to prevent packets larger than max_frame_size */
2447 rctl |= E1000_RCTL_LPE;
9d5c8243 2448
952f72a8
AD		 2449 /* disable queue 0 to prevent tail write w/o re-config */
2450 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2451
e1739522
AD		 2452 /* Attention!!! For SR-IOV PF driver operations you must enable
2453 * queue drop for all VF and PF queues to prevent head of line blocking
2454 * if an un-trusted VF does not provide descriptors to hardware.
2455 */
2456 if (adapter->vfs_allocated_count) {
e1739522
AD		 2457 /* set all queue drop enable bits */
2458 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 2459 }
2460
9d5c8243
AK		 2461 wr32(E1000_RCTL, rctl);
2462}
2463
7d5753f0
AD		 2464static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2465 int vfn)
2466{
2467 struct e1000_hw *hw = &adapter->hw;
2468 u32 vmolr;
2469
2470 /* if it isn't the PF check to see if VFs are enabled and
2471 * increase the size to support vlan tags */
2472 if (vfn < adapter->vfs_allocated_count &&
2473 adapter->vf_data[vfn].vlans_enabled)
2474 size += VLAN_TAG_SIZE;
2475
2476 vmolr = rd32(E1000_VMOLR(vfn));
2477 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2478 vmolr |= size | E1000_VMOLR_LPE;
2479 wr32(E1000_VMOLR(vfn), vmolr);
2480
2481 return 0;
2482}
2483
e1739522
AD		 2484/**
2485 * igb_rlpml_set - set maximum receive packet size
2486 * @adapter: board private structure
2487 *
2488 * Configure maximum receivable packet size.
2489 **/
2490static void igb_rlpml_set(struct igb_adapter *adapter)
2491{
2492 u32 max_frame_size = adapter->max_frame_size;
2493 struct e1000_hw *hw = &adapter->hw;
2494 u16 pf_id = adapter->vfs_allocated_count;
2495
2496 if (adapter->vlgrp)
2497 max_frame_size += VLAN_TAG_SIZE;
2498
2499 /* if vfs are enabled we set RLPML to the largest possible request
2500 * size and set the VMOLR RLPML to the size we need */
2501 if (pf_id) {
2502 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
7d5753f0 2503 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 2504 }
2505
2506 wr32(E1000_RLPML, max_frame_size);
2507}
2508
8151d294
WM		 2509static inline void igb_set_vmolr(struct igb_adapter *adapter,
2510 int vfn, bool aupe)
7d5753f0
AD		 2511{
2512 struct e1000_hw *hw = &adapter->hw;
2513 u32 vmolr;
2514
2515 /*
2516 * This register exists only on 82576 and newer so if we are older then
2517 * we should exit and do nothing
2518 */
2519 if (hw->mac.type < e1000_82576)
2520 return;
2521
2522 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 2523 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2524 if (aupe)
2525 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
2526 else
2527 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 2528
2529 /* clear all bits that might not be set */
2530 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2531
a99955fc 2532 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 2533 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2534 /*
2535 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2536 * multicast packets
2537 */
2538 if (vfn <= adapter->vfs_allocated_count)
2539 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2540
2541 wr32(E1000_VMOLR(vfn), vmolr);
2542}
2543
85b430b4
AD		 2544/**
2545 * igb_configure_rx_ring - Configure a receive ring after Reset
2546 * @adapter: board private structure
2547 * @ring: receive ring to be configured
2548 *
2549 * Configure the Rx unit of the MAC after a reset.
2550 **/
d7ee5b3a
AD		 2551void igb_configure_rx_ring(struct igb_adapter *adapter,
2552 struct igb_ring *ring)
85b430b4
AD		 2553{
2554 struct e1000_hw *hw = &adapter->hw;
2555 u64 rdba = ring->dma;
2556 int reg_idx = ring->reg_idx;
952f72a8 2557 u32 srrctl, rxdctl;
85b430b4
AD		 2558
2559 /* disable the queue */
2560 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2561 wr32(E1000_RXDCTL(reg_idx),
2562 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2563
2564 /* Set DMA base address registers */
2565 wr32(E1000_RDBAL(reg_idx),
2566 rdba & 0x00000000ffffffffULL);
2567 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2568 wr32(E1000_RDLEN(reg_idx),
2569 ring->count * sizeof(union e1000_adv_rx_desc));
2570
2571 /* initialize head and tail */
fce99e34
AD		 2572 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2573 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2574 writel(0, ring->head);
2575 writel(0, ring->tail);
85b430b4 2576
952f72a8 2577 /* set descriptor configuration */
4c844851
AD		 2578 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2579 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
952f72a8
AD		 2580 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2581#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2582 srrctl |= IGB_RXBUFFER_16384 >>
2583 E1000_SRRCTL_BSIZEPKT_SHIFT;
2584#else
2585 srrctl |= (PAGE_SIZE / 2) >>
2586 E1000_SRRCTL_BSIZEPKT_SHIFT;
2587#endif
2588 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2589 } else {
4c844851 2590 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
952f72a8
AD		 2591 E1000_SRRCTL_BSIZEPKT_SHIFT;
2592 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2593 }
e6bdb6fe
NN		 2594 /* Only set Drop Enable if we are supporting multiple queues */
2595 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
2596 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 2597
2598 wr32(E1000_SRRCTL(reg_idx), srrctl);
2599
7d5753f0 2600 /* set filtering for VMDQ pools */
8151d294 2601 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 2602
85b430b4
AD		 2603 /* enable receive descriptor fetching */
2604 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2605 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2606 rxdctl &= 0xFFF00000;
2607 rxdctl |= IGB_RX_PTHRESH;
2608 rxdctl |= IGB_RX_HTHRESH << 8;
2609 rxdctl |= IGB_RX_WTHRESH << 16;
2610 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2611}
2612
9d5c8243
AK		 2613/**
2614 * igb_configure_rx - Configure receive Unit after Reset
2615 * @adapter: board private structure
2616 *
2617 * Configure the Rx unit of the MAC after a reset.
2618 **/
2619static void igb_configure_rx(struct igb_adapter *adapter)
2620{
9107584e 2621 int i;
9d5c8243 2622
68d480c4
AD		 2623 /* set UTA to appropriate mode */
2624 igb_set_uta(adapter);
2625
26ad9178
AD		 2626 /* set the correct pool for the PF default MAC address in entry 0 */
2627 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2628 adapter->vfs_allocated_count);
2629
06cf2666
AD		 2630 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2631 * the Base and Length of the Rx Descriptor Ring */
2632 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2633 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 2634}
2635
2636/**
2637 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 2638 * @tx_ring: Tx descriptor ring for a specific queue
2639 *
2640 * Free all transmit software resources
2641 **/
68fd9910 2642void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2643{
3b644cf6 2644 igb_clean_tx_ring(tx_ring);
9d5c8243
AK		 2645
2646 vfree(tx_ring->buffer_info);
2647 tx_ring->buffer_info = NULL;
2648
439705e1
AD		 2649 /* if not set, then don't free */
2650 if (!tx_ring->desc)
2651 return;
2652
80785298
AD		 2653 pci_free_consistent(tx_ring->pdev, tx_ring->size,
2654 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 2655
2656 tx_ring->desc = NULL;
2657}
2658
2659/**
2660 * igb_free_all_tx_resources - Free Tx Resources for All Queues
2661 * @adapter: board private structure
2662 *
2663 * Free all transmit software resources
2664 **/
2665static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2666{
2667 int i;
2668
2669 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2670 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2671}
2672
b1a436c3
AD		 2673void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
2674 struct igb_buffer *buffer_info)
9d5c8243 2675{
6366ad33
AD		 2676 if (buffer_info->dma) {
2677 if (buffer_info->mapped_as_page)
2678 pci_unmap_page(tx_ring->pdev,
2679 buffer_info->dma,
2680 buffer_info->length,
2681 PCI_DMA_TODEVICE);
2682 else
2683 pci_unmap_single(tx_ring->pdev,
2684 buffer_info->dma,
2685 buffer_info->length,
2686 PCI_DMA_TODEVICE);
2687 buffer_info->dma = 0;
2688 }
9d5c8243
AK		 2689 if (buffer_info->skb) {
2690 dev_kfree_skb_any(buffer_info->skb);
2691 buffer_info->skb = NULL;
2692 }
2693 buffer_info->time_stamp = 0;
6366ad33
AD		 2694 buffer_info->length = 0;
2695 buffer_info->next_to_watch = 0;
2696 buffer_info->mapped_as_page = false;
9d5c8243
AK		 2697}
2698
2699/**
2700 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 2701 * @tx_ring: ring to be cleaned
2702 **/
3b644cf6 2703static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243
AK		 2704{
2705 struct igb_buffer *buffer_info;
2706 unsigned long size;
2707 unsigned int i;
2708
2709 if (!tx_ring->buffer_info)
2710 return;
2711 /* Free all the Tx ring sk_buffs */
2712
2713 for (i = 0; i < tx_ring->count; i++) {
2714 buffer_info = &tx_ring->buffer_info[i];
80785298 2715 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 2716 }
2717
2718 size = sizeof(struct igb_buffer) * tx_ring->count;
2719 memset(tx_ring->buffer_info, 0, size);
2720
2721 /* Zero out the descriptor ring */
9d5c8243
AK		 2722 memset(tx_ring->desc, 0, tx_ring->size);
2723
2724 tx_ring->next_to_use = 0;
2725 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2726}
2727
2728/**
2729 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
2730 * @adapter: board private structure
2731 **/
2732static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
2733{
2734 int i;
2735
2736 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2737 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 2738}
2739
2740/**
2741 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 2742 * @rx_ring: ring to clean the resources from
2743 *
2744 * Free all receive software resources
2745 **/
68fd9910 2746void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2747{
3b644cf6 2748 igb_clean_rx_ring(rx_ring);
9d5c8243
AK		 2749
2750 vfree(rx_ring->buffer_info);
2751 rx_ring->buffer_info = NULL;
2752
439705e1
AD		 2753 /* if not set, then don't free */
2754 if (!rx_ring->desc)
2755 return;
2756
80785298
AD		 2757 pci_free_consistent(rx_ring->pdev, rx_ring->size,
2758 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 2759
2760 rx_ring->desc = NULL;
2761}
2762
2763/**
2764 * igb_free_all_rx_resources - Free Rx Resources for All Queues
2765 * @adapter: board private structure
2766 *
2767 * Free all receive software resources
2768 **/
2769static void igb_free_all_rx_resources(struct igb_adapter *adapter)
2770{
2771 int i;
2772
2773 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2774 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2775}
2776
2777/**
2778 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 2779 * @rx_ring: ring to free buffers from
2780 **/
3b644cf6 2781static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243
AK		 2782{
2783 struct igb_buffer *buffer_info;
9d5c8243
AK		 2784 unsigned long size;
2785 unsigned int i;
2786
2787 if (!rx_ring->buffer_info)
2788 return;
439705e1 2789
9d5c8243
AK		 2790 /* Free all the Rx ring sk_buffs */
2791 for (i = 0; i < rx_ring->count; i++) {
2792 buffer_info = &rx_ring->buffer_info[i];
2793 if (buffer_info->dma) {
80785298
AD		 2794 pci_unmap_single(rx_ring->pdev,
2795 buffer_info->dma,
4c844851 2796 rx_ring->rx_buffer_len,
6ec43fe6 2797 PCI_DMA_FROMDEVICE);
9d5c8243
AK		 2798 buffer_info->dma = 0;
2799 }
2800
2801 if (buffer_info->skb) {
2802 dev_kfree_skb(buffer_info->skb);
2803 buffer_info->skb = NULL;
2804 }
6ec43fe6 2805 if (buffer_info->page_dma) {
80785298
AD		 2806 pci_unmap_page(rx_ring->pdev,
2807 buffer_info->page_dma,
6ec43fe6
AD		 2808 PAGE_SIZE / 2,
2809 PCI_DMA_FROMDEVICE);
2810 buffer_info->page_dma = 0;
2811 }
9d5c8243 2812 if (buffer_info->page) {
9d5c8243
AK		 2813 put_page(buffer_info->page);
2814 buffer_info->page = NULL;
bf36c1a0 2815 buffer_info->page_offset = 0;
9d5c8243
AK		 2816 }
2817 }
2818
9d5c8243
AK		 2819 size = sizeof(struct igb_buffer) * rx_ring->count;
2820 memset(rx_ring->buffer_info, 0, size);
2821
2822 /* Zero out the descriptor ring */
2823 memset(rx_ring->desc, 0, rx_ring->size);
2824
2825 rx_ring->next_to_clean = 0;
2826 rx_ring->next_to_use = 0;
9d5c8243
AK		 2827}
2828
2829/**
2830 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
2831 * @adapter: board private structure
2832 **/
2833static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
2834{
2835 int i;
2836
2837 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2838 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 2839}
2840
2841/**
2842 * igb_set_mac - Change the Ethernet Address of the NIC
2843 * @netdev: network interface device structure
2844 * @p: pointer to an address structure
2845 *
2846 * Returns 0 on success, negative on failure
2847 **/
2848static int igb_set_mac(struct net_device *netdev, void *p)
2849{
2850 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 2851 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2852 struct sockaddr *addr = p;
2853
2854 if (!is_valid_ether_addr(addr->sa_data))
2855 return -EADDRNOTAVAIL;
2856
2857 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 2858 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 2859
26ad9178
AD		 2860 /* set the correct pool for the new PF MAC address in entry 0 */
2861 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
2862 adapter->vfs_allocated_count);
e1739522 2863
9d5c8243
AK		 2864 return 0;
2865}
2866
2867/**
68d480c4 2868 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 2869 * @netdev: network interface device structure
2870 *
68d480c4
AD		 2871 * Writes multicast address list to the MTA hash table.
2872 * Returns: -ENOMEM on failure
2873 * 0 on no addresses written
2874 * X on writing X addresses to MTA
9d5c8243 2875 **/
68d480c4 2876static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 2877{
2878 struct igb_adapter *adapter = netdev_priv(netdev);
2879 struct e1000_hw *hw = &adapter->hw;
ff41f8dc 2880 struct dev_mc_list *mc_ptr = netdev->mc_list;
68d480c4 2881 u8 *mta_list;
9d5c8243
AK		 2882 int i;
2883
4cd24eaf 2884 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 2885 /* nothing to program, so clear mc list */
2886 igb_update_mc_addr_list(hw, NULL, 0);
2887 igb_restore_vf_multicasts(adapter);
2888 return 0;
2889 }
9d5c8243 2890
4cd24eaf 2891 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 2892 if (!mta_list)
2893 return -ENOMEM;
ff41f8dc 2894
68d480c4
AD		 2895 /* The shared function expects a packed array of only addresses. */
2896 mc_ptr = netdev->mc_list;
2897
4cd24eaf 2898 for (i = 0; i < netdev_mc_count(netdev); i++) {
68d480c4
AD		 2899 if (!mc_ptr)
2900 break;
2901 memcpy(mta_list + (i*ETH_ALEN), mc_ptr->dmi_addr, ETH_ALEN);
2902 mc_ptr = mc_ptr->next;
746b9f02 2903 }
68d480c4
AD		 2904 igb_update_mc_addr_list(hw, mta_list, i);
2905 kfree(mta_list);
2906
4cd24eaf 2907 return netdev_mc_count(netdev);
68d480c4
AD		 2908}
2909
2910/**
2911 * igb_write_uc_addr_list - write unicast addresses to RAR table
2912 * @netdev: network interface device structure
2913 *
2914 * Writes unicast address list to the RAR table.
2915 * Returns: -ENOMEM on failure/insufficient address space
2916 * 0 on no addresses written
2917 * X on writing X addresses to the RAR table
2918 **/
2919static int igb_write_uc_addr_list(struct net_device *netdev)
2920{
2921 struct igb_adapter *adapter = netdev_priv(netdev);
2922 struct e1000_hw *hw = &adapter->hw;
2923 unsigned int vfn = adapter->vfs_allocated_count;
2924 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
2925 int count = 0;
2926
2927 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 2928 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 2929 return -ENOMEM;
9d5c8243 2930
32e7bfc4 2931 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 2932 struct netdev_hw_addr *ha;
32e7bfc4
JP		 2933
2934 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 2935 if (!rar_entries)
2936 break;
26ad9178
AD		 2937 igb_rar_set_qsel(adapter, ha->addr,
2938 rar_entries--,
68d480c4
AD		 2939 vfn);
2940 count++;
ff41f8dc
AD		 2941 }
2942 }
2943 /* write the addresses in reverse order to avoid write combining */
2944 for (; rar_entries > 0 ; rar_entries--) {
2945 wr32(E1000_RAH(rar_entries), 0);
2946 wr32(E1000_RAL(rar_entries), 0);
2947 }
2948 wrfl();
2949
68d480c4
AD		 2950 return count;
2951}
2952
2953/**
2954 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2955 * @netdev: network interface device structure
2956 *
2957 * The set_rx_mode entry point is called whenever the unicast or multicast
2958 * address lists or the network interface flags are updated. This routine is
2959 * responsible for configuring the hardware for proper unicast, multicast,
2960 * promiscuous mode, and all-multi behavior.
2961 **/
2962static void igb_set_rx_mode(struct net_device *netdev)
2963{
2964 struct igb_adapter *adapter = netdev_priv(netdev);
2965 struct e1000_hw *hw = &adapter->hw;
2966 unsigned int vfn = adapter->vfs_allocated_count;
2967 u32 rctl, vmolr = 0;
2968 int count;
2969
2970 /* Check for Promiscuous and All Multicast modes */
2971 rctl = rd32(E1000_RCTL);
2972
2973 /* clear the effected bits */
2974 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
2975
2976 if (netdev->flags & IFF_PROMISC) {
2977 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2978 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
2979 } else {
2980 if (netdev->flags & IFF_ALLMULTI) {
2981 rctl |= E1000_RCTL_MPE;
2982 vmolr |= E1000_VMOLR_MPME;
2983 } else {
2984 /*
2985 * Write addresses to the MTA, if the attempt fails
2986 * then we should just turn on promiscous mode so
2987 * that we can at least receive multicast traffic
2988 */
2989 count = igb_write_mc_addr_list(netdev);
2990 if (count < 0) {
2991 rctl |= E1000_RCTL_MPE;
2992 vmolr |= E1000_VMOLR_MPME;
2993 } else if (count) {
2994 vmolr |= E1000_VMOLR_ROMPE;
2995 }
2996 }
2997 /*
2998 * Write addresses to available RAR registers, if there is not
2999 * sufficient space to store all the addresses then enable
3000 * unicast promiscous mode
3001 */
3002 count = igb_write_uc_addr_list(netdev);
3003 if (count < 0) {
3004 rctl |= E1000_RCTL_UPE;
3005 vmolr |= E1000_VMOLR_ROPE;
3006 }
3007 rctl |= E1000_RCTL_VFE;
28fc06f5 3008 }
68d480c4 3009 wr32(E1000_RCTL, rctl);
28fc06f5 3010
68d480c4
AD		 3011 /*
3012 * In order to support SR-IOV and eventually VMDq it is necessary to set
3013 * the VMOLR to enable the appropriate modes. Without this workaround
3014 * we will have issues with VLAN tag stripping not being done for frames
3015 * that are only arriving because we are the default pool
3016 */
3017 if (hw->mac.type < e1000_82576)
28fc06f5 3018 return;
9d5c8243 3019
68d480c4
AD		 3020 vmolr |= rd32(E1000_VMOLR(vfn)) &
3021 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3022 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3023 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3024}
3025
3026/* Need to wait a few seconds after link up to get diagnostic information from
3027 * the phy */
3028static void igb_update_phy_info(unsigned long data)
3029{
3030 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3031 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3032}
3033
4d6b725e
AD		 3034/**
3035 * igb_has_link - check shared code for link and determine up/down
3036 * @adapter: pointer to driver private info
3037 **/
3145535a 3038bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3039{
3040 struct e1000_hw *hw = &adapter->hw;
3041 bool link_active = false;
3042 s32 ret_val = 0;
3043
3044 /* get_link_status is set on LSC (link status) interrupt or
3045 * rx sequence error interrupt. get_link_status will stay
3046 * false until the e1000_check_for_link establishes link
3047 * for copper adapters ONLY
3048 */
3049 switch (hw->phy.media_type) {
3050 case e1000_media_type_copper:
3051 if (hw->mac.get_link_status) {
3052 ret_val = hw->mac.ops.check_for_link(hw);
3053 link_active = !hw->mac.get_link_status;
3054 } else {
3055 link_active = true;
3056 }
3057 break;
4d6b725e
AD		 3058 case e1000_media_type_internal_serdes:
3059 ret_val = hw->mac.ops.check_for_link(hw);
3060 link_active = hw->mac.serdes_has_link;
3061 break;
3062 default:
3063 case e1000_media_type_unknown:
3064 break;
3065 }
3066
3067 return link_active;
3068}
3069
9d5c8243
AK		 3070/**
3071 * igb_watchdog - Timer Call-back
3072 * @data: pointer to adapter cast into an unsigned long
3073 **/
3074static void igb_watchdog(unsigned long data)
3075{
3076 struct igb_adapter *adapter = (struct igb_adapter *)data;
3077 /* Do the rest outside of interrupt context */
3078 schedule_work(&adapter->watchdog_task);
3079}
3080
3081static void igb_watchdog_task(struct work_struct *work)
3082{
3083 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3084 struct igb_adapter,
3085 watchdog_task);
9d5c8243 3086 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3087 struct net_device *netdev = adapter->netdev;
9d5c8243 3088 u32 link;
7a6ea550 3089 int i;
9d5c8243 3090
4d6b725e 3091 link = igb_has_link(adapter);
9d5c8243
AK		 3092 if (link) {
3093 if (!netif_carrier_ok(netdev)) {
3094 u32 ctrl;
330a6d6a
AD		 3095 hw->mac.ops.get_speed_and_duplex(hw,
3096 &adapter->link_speed,
3097 &adapter->link_duplex);
9d5c8243
AK		 3098
3099 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 3100 /* Links status message must follow this format */
3101 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 3102 "Flow Control: %s\n",
559e9c49
AD		 3103 netdev->name,
3104 adapter->link_speed,
3105 adapter->link_duplex == FULL_DUPLEX ?
9d5c8243 3106 "Full Duplex" : "Half Duplex",
559e9c49
AD		 3107 ((ctrl & E1000_CTRL_TFCE) &&
3108 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3109 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3110 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
9d5c8243
AK		 3111
3112 /* tweak tx_queue_len according to speed/duplex and
3113 * adjust the timeout factor */
3114 netdev->tx_queue_len = adapter->tx_queue_len;
3115 adapter->tx_timeout_factor = 1;
3116 switch (adapter->link_speed) {
3117 case SPEED_10:
3118 netdev->tx_queue_len = 10;
3119 adapter->tx_timeout_factor = 14;
3120 break;
3121 case SPEED_100:
3122 netdev->tx_queue_len = 100;
3123 /* maybe add some timeout factor ? */
3124 break;
3125 }
3126
3127 netif_carrier_on(netdev);
9d5c8243 3128
4ae196df
AD		 3129 igb_ping_all_vfs(adapter);
3130
4b1a9877 3131 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3132 if (!test_bit(__IGB_DOWN, &adapter->state))
3133 mod_timer(&adapter->phy_info_timer,
3134 round_jiffies(jiffies + 2 * HZ));
3135 }
3136 } else {
3137 if (netif_carrier_ok(netdev)) {
3138 adapter->link_speed = 0;
3139 adapter->link_duplex = 0;
527d47c1
AD		 3140 /* Links status message must follow this format */
3141 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3142 netdev->name);
9d5c8243 3143 netif_carrier_off(netdev);
4b1a9877 3144
4ae196df
AD		 3145 igb_ping_all_vfs(adapter);
3146
4b1a9877 3147 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3148 if (!test_bit(__IGB_DOWN, &adapter->state))
3149 mod_timer(&adapter->phy_info_timer,
3150 round_jiffies(jiffies + 2 * HZ));
3151 }
3152 }
3153
9d5c8243 3154 igb_update_stats(adapter);
9d5c8243 3155
dbabb065 3156 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3157 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3158 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3159 /* We've lost link, so the controller stops DMA,
3160 * but we've got queued Tx work that's never going
3161 * to get done, so reset controller to flush Tx.
3162 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3163 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3164 adapter->tx_timeout_count++;
3165 schedule_work(&adapter->reset_task);
3166 /* return immediately since reset is imminent */
3167 return;
3168 }
9d5c8243 3169 }
9d5c8243 3170
dbabb065
AD		 3171 /* Force detection of hung controller every watchdog period */
3172 tx_ring->detect_tx_hung = true;
3173 }
f7ba205e 3174
9d5c8243 3175 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3176 if (adapter->msix_entries) {
047e0030
AD		 3177 u32 eics = 0;
3178 for (i = 0; i < adapter->num_q_vectors; i++) {
3179 struct igb_q_vector *q_vector = adapter->q_vector[i];
3180 eics |= q_vector->eims_value;
3181 }
7a6ea550
AD		 3182 wr32(E1000_EICS, eics);
3183 } else {
3184 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3185 }
9d5c8243 3186
9d5c8243
AK		 3187 /* Reset the timer */
3188 if (!test_bit(__IGB_DOWN, &adapter->state))
3189 mod_timer(&adapter->watchdog_timer,
3190 round_jiffies(jiffies + 2 * HZ));
3191}
3192
3193enum latency_range {
3194 lowest_latency = 0,
3195 low_latency = 1,
3196 bulk_latency = 2,
3197 latency_invalid = 255
3198};
3199
6eb5a7f1
AD		 3200/**
3201 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3202 *
3203 * Stores a new ITR value based on strictly on packet size. This
3204 * algorithm is less sophisticated than that used in igb_update_itr,
3205 * due to the difficulty of synchronizing statistics across multiple
3206 * receive rings. The divisors and thresholds used by this fuction
3207 * were determined based on theoretical maximum wire speed and testing
3208 * data, in order to minimize response time while increasing bulk
3209 * throughput.
3210 * This functionality is controlled by the InterruptThrottleRate module
3211 * parameter (see igb_param.c)
3212 * NOTE: This function is called only when operating in a multiqueue
3213 * receive environment.
047e0030 3214 * @q_vector: pointer to q_vector
6eb5a7f1 3215 **/
047e0030 3216static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3217{
047e0030 3218 int new_val = q_vector->itr_val;
6eb5a7f1 3219 int avg_wire_size = 0;
047e0030 3220 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 3221
6eb5a7f1
AD		 3222 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3223 * ints/sec - ITR timer value of 120 ticks.
3224 */
3225 if (adapter->link_speed != SPEED_1000) {
047e0030 3226 new_val = 976;
6eb5a7f1 3227 goto set_itr_val;
9d5c8243 3228 }
047e0030
AD		 3229
3230 if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3231 struct igb_ring *ring = q_vector->rx_ring;
3232 avg_wire_size = ring->total_bytes / ring->total_packets;
3233 }
3234
3235 if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3236 struct igb_ring *ring = q_vector->tx_ring;
3237 avg_wire_size = max_t(u32, avg_wire_size,
3238 (ring->total_bytes /
3239 ring->total_packets));
3240 }
3241
3242 /* if avg_wire_size isn't set no work was done */
3243 if (!avg_wire_size)
3244 goto clear_counts;
9d5c8243 3245
6eb5a7f1
AD		 3246 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3247 avg_wire_size += 24;
3248
3249 /* Don't starve jumbo frames */
3250 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3251
6eb5a7f1
AD		 3252 /* Give a little boost to mid-size frames */
3253 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3254 new_val = avg_wire_size / 3;
3255 else
3256 new_val = avg_wire_size / 2;
9d5c8243 3257
abe1c363
NN		 3258 /* when in itr mode 3 do not exceed 20K ints/sec */
3259 if (adapter->rx_itr_setting == 3 && new_val < 196)
3260 new_val = 196;
3261
6eb5a7f1 3262set_itr_val:
047e0030
AD		 3263 if (new_val != q_vector->itr_val) {
3264 q_vector->itr_val = new_val;
3265 q_vector->set_itr = 1;
9d5c8243 3266 }
6eb5a7f1 3267clear_counts:
047e0030
AD		 3268 if (q_vector->rx_ring) {
3269 q_vector->rx_ring->total_bytes = 0;
3270 q_vector->rx_ring->total_packets = 0;
3271 }
3272 if (q_vector->tx_ring) {
3273 q_vector->tx_ring->total_bytes = 0;
3274 q_vector->tx_ring->total_packets = 0;
3275 }
9d5c8243
AK		 3276}
3277
3278/**
3279 * igb_update_itr - update the dynamic ITR value based on statistics
3280 * Stores a new ITR value based on packets and byte
3281 * counts during the last interrupt. The advantage of per interrupt
3282 * computation is faster updates and more accurate ITR for the current
3283 * traffic pattern. Constants in this function were computed
3284 * based on theoretical maximum wire speed and thresholds were set based
3285 * on testing data as well as attempting to minimize response time
3286 * while increasing bulk throughput.
3287 * this functionality is controlled by the InterruptThrottleRate module
3288 * parameter (see igb_param.c)
3289 * NOTE: These calculations are only valid when operating in a single-
3290 * queue environment.
3291 * @adapter: pointer to adapter
047e0030 3292 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3293 * @packets: the number of packets during this measurement interval
3294 * @bytes: the number of bytes during this measurement interval
3295 **/
3296static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3297 int packets, int bytes)
3298{
3299 unsigned int retval = itr_setting;
3300
3301 if (packets == 0)
3302 goto update_itr_done;
3303
3304 switch (itr_setting) {
3305 case lowest_latency:
3306 /* handle TSO and jumbo frames */
3307 if (bytes/packets > 8000)
3308 retval = bulk_latency;
3309 else if ((packets < 5) && (bytes > 512))
3310 retval = low_latency;
3311 break;
3312 case low_latency: /* 50 usec aka 20000 ints/s */
3313 if (bytes > 10000) {
3314 /* this if handles the TSO accounting */
3315 if (bytes/packets > 8000) {
3316 retval = bulk_latency;
3317 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3318 retval = bulk_latency;
3319 } else if ((packets > 35)) {
3320 retval = lowest_latency;
3321 }
3322 } else if (bytes/packets > 2000) {
3323 retval = bulk_latency;
3324 } else if (packets <= 2 && bytes < 512) {
3325 retval = lowest_latency;
3326 }
3327 break;
3328 case bulk_latency: /* 250 usec aka 4000 ints/s */
3329 if (bytes > 25000) {
3330 if (packets > 35)
3331 retval = low_latency;
1e5c3d21 3332 } else if (bytes < 1500) {
9d5c8243
AK		 3333 retval = low_latency;
3334 }
3335 break;
3336 }
3337
3338update_itr_done:
3339 return retval;
3340}
3341
6eb5a7f1 3342static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3343{
047e0030 3344 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3345 u16 current_itr;
047e0030 3346 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3347
3348 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3349 if (adapter->link_speed != SPEED_1000) {
3350 current_itr = 0;
3351 new_itr = 4000;
3352 goto set_itr_now;
3353 }
3354
3355 adapter->rx_itr = igb_update_itr(adapter,
3356 adapter->rx_itr,
3025a446
AD		 3357 q_vector->rx_ring->total_packets,
3358 q_vector->rx_ring->total_bytes);
9d5c8243 3359
047e0030
AD		 3360 adapter->tx_itr = igb_update_itr(adapter,
3361 adapter->tx_itr,
3025a446
AD		 3362 q_vector->tx_ring->total_packets,
3363 q_vector->tx_ring->total_bytes);
047e0030 3364 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3365
6eb5a7f1 3366 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4fc82adf 3367 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3368 current_itr = low_latency;
3369
9d5c8243
AK		 3370 switch (current_itr) {
3371 /* counts and packets in update_itr are dependent on these numbers */
3372 case lowest_latency:
78b1f607 3373 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3374 break;
3375 case low_latency:
78b1f607 3376 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3377 break;
3378 case bulk_latency:
78b1f607 3379 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3380 break;
3381 default:
3382 break;
3383 }
3384
3385set_itr_now:
3025a446
AD		 3386 q_vector->rx_ring->total_bytes = 0;
3387 q_vector->rx_ring->total_packets = 0;
3388 q_vector->tx_ring->total_bytes = 0;
3389 q_vector->tx_ring->total_packets = 0;
6eb5a7f1 3390
047e0030 3391 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3392 /* this attempts to bias the interrupt rate towards Bulk
3393 * by adding intermediate steps when interrupt rate is
3394 * increasing */
047e0030
AD		 3395 new_itr = new_itr > q_vector->itr_val ?
3396 max((new_itr * q_vector->itr_val) /
3397 (new_itr + (q_vector->itr_val >> 2)),
3398 new_itr) :
9d5c8243
AK		 3399 new_itr;
3400 /* Don't write the value here; it resets the adapter's
3401 * internal timer, and causes us to delay far longer than
3402 * we should between interrupts. Instead, we write the ITR
3403 * value at the beginning of the next interrupt so the timing
3404 * ends up being correct.
3405 */
047e0030
AD		 3406 q_vector->itr_val = new_itr;
3407 q_vector->set_itr = 1;
9d5c8243
AK		 3408 }
3409
3410 return;
3411}
3412
9d5c8243
AK		 3413#define IGB_TX_FLAGS_CSUM 0x00000001
3414#define IGB_TX_FLAGS_VLAN 0x00000002
3415#define IGB_TX_FLAGS_TSO 0x00000004
3416#define IGB_TX_FLAGS_IPV4 0x00000008
cdfd01fc
AD		 3417#define IGB_TX_FLAGS_TSTAMP 0x00000010
3418#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3419#define IGB_TX_FLAGS_VLAN_SHIFT 16
9d5c8243 3420
85ad76b2 3421static inline int igb_tso_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3422 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3423{
3424 struct e1000_adv_tx_context_desc *context_desc;
3425 unsigned int i;
3426 int err;
3427 struct igb_buffer *buffer_info;
3428 u32 info = 0, tu_cmd = 0;
91d4ee33
NN		 3429 u32 mss_l4len_idx;
3430 u8 l4len;
9d5c8243
AK		 3431
3432 if (skb_header_cloned(skb)) {
3433 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3434 if (err)
3435 return err;
3436 }
3437
3438 l4len = tcp_hdrlen(skb);
3439 *hdr_len += l4len;
3440
3441 if (skb->protocol == htons(ETH_P_IP)) {
3442 struct iphdr *iph = ip_hdr(skb);
3443 iph->tot_len = 0;
3444 iph->check = 0;
3445 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3446 iph->daddr, 0,
3447 IPPROTO_TCP,
3448 0);
8e1e8a47 3449 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 3450 ipv6_hdr(skb)->payload_len = 0;
3451 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3452 &ipv6_hdr(skb)->daddr,
3453 0, IPPROTO_TCP, 0);
3454 }
3455
3456 i = tx_ring->next_to_use;
3457
3458 buffer_info = &tx_ring->buffer_info[i];
3459 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3460 /* VLAN MACLEN IPLEN */
3461 if (tx_flags & IGB_TX_FLAGS_VLAN)
3462 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3463 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3464 *hdr_len += skb_network_offset(skb);
3465 info |= skb_network_header_len(skb);
3466 *hdr_len += skb_network_header_len(skb);
3467 context_desc->vlan_macip_lens = cpu_to_le32(info);
3468
3469 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3470 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3471
3472 if (skb->protocol == htons(ETH_P_IP))
3473 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3474 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3475
3476 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3477
3478 /* MSS L4LEN IDX */
3479 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3480 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3481
73cd78f1 3482 /* For 82575, context index must be unique per ring. */
85ad76b2
AD		 3483 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3484 mss_l4len_idx |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3485
3486 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3487 context_desc->seqnum_seed = 0;
3488
3489 buffer_info->time_stamp = jiffies;
0e014cb1 3490 buffer_info->next_to_watch = i;
9d5c8243
AK		 3491 buffer_info->dma = 0;
3492 i++;
3493 if (i == tx_ring->count)
3494 i = 0;
3495
3496 tx_ring->next_to_use = i;
3497
3498 return true;
3499}
3500
85ad76b2
AD		 3501static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3502 struct sk_buff *skb, u32 tx_flags)
9d5c8243
AK		 3503{
3504 struct e1000_adv_tx_context_desc *context_desc;
80785298 3505 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3506 struct igb_buffer *buffer_info;
3507 u32 info = 0, tu_cmd = 0;
80785298 3508 unsigned int i;
9d5c8243
AK		 3509
3510 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3511 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3512 i = tx_ring->next_to_use;
3513 buffer_info = &tx_ring->buffer_info[i];
3514 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3515
3516 if (tx_flags & IGB_TX_FLAGS_VLAN)
3517 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
cdfd01fc 3518
9d5c8243
AK		 3519 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3520 if (skb->ip_summed == CHECKSUM_PARTIAL)
3521 info |= skb_network_header_len(skb);
3522
3523 context_desc->vlan_macip_lens = cpu_to_le32(info);
3524
3525 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3526
3527 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fa4a7ef3
AJ		 3528 __be16 protocol;
3529
3530 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3531 const struct vlan_ethhdr *vhdr =
3532 (const struct vlan_ethhdr*)skb->data;
3533
3534 protocol = vhdr->h_vlan_encapsulated_proto;
3535 } else {
3536 protocol = skb->protocol;
3537 }
3538
3539 switch (protocol) {
09640e63 3540 case cpu_to_be16(ETH_P_IP):
9d5c8243 3541 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
44b0cda3
MW		 3542 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3543 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3544 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3545 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3 3546 break;
09640e63 3547 case cpu_to_be16(ETH_P_IPV6):
44b0cda3
MW		 3548 /* XXX what about other V6 headers?? */
3549 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3550 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3551 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3552 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3
MW		 3553 break;
3554 default:
3555 if (unlikely(net_ratelimit()))
80785298 3556 dev_warn(&pdev->dev,
44b0cda3
MW		 3557 "partial checksum but proto=%x!\n",
3558 skb->protocol);
3559 break;
3560 }
9d5c8243
AK		 3561 }
3562
3563 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3564 context_desc->seqnum_seed = 0;
85ad76b2 3565 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
7dfc16fa 3566 context_desc->mss_l4len_idx =
85ad76b2 3567 cpu_to_le32(tx_ring->reg_idx << 4);
9d5c8243
AK		 3568
3569 buffer_info->time_stamp = jiffies;
0e014cb1 3570 buffer_info->next_to_watch = i;
9d5c8243
AK		 3571 buffer_info->dma = 0;
3572
3573 i++;
3574 if (i == tx_ring->count)
3575 i = 0;
3576 tx_ring->next_to_use = i;
3577
3578 return true;
3579 }
9d5c8243
AK		 3580 return false;
3581}
3582
3583#define IGB_MAX_TXD_PWR 16
3584#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3585
80785298 3586static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
0e014cb1 3587 unsigned int first)
9d5c8243
AK		 3588{
3589 struct igb_buffer *buffer_info;
80785298 3590 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3591 unsigned int len = skb_headlen(skb);
3592 unsigned int count = 0, i;
3593 unsigned int f;
3594
3595 i = tx_ring->next_to_use;
3596
3597 buffer_info = &tx_ring->buffer_info[i];
3598 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3599 buffer_info->length = len;
3600 /* set time_stamp *before* dma to help avoid a possible race */
3601 buffer_info->time_stamp = jiffies;
0e014cb1 3602 buffer_info->next_to_watch = i;
6366ad33
AD		 3603 buffer_info->dma = pci_map_single(pdev, skb->data, len,
3604 PCI_DMA_TODEVICE);
3605 if (pci_dma_mapping_error(pdev, buffer_info->dma))
3606 goto dma_error;
9d5c8243
AK		 3607
3608 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3609 struct skb_frag_struct *frag;
3610
8581145f 3611 count++;
65689fef
AD		 3612 i++;
3613 if (i == tx_ring->count)
3614 i = 0;
3615
9d5c8243
AK		 3616 frag = &skb_shinfo(skb)->frags[f];
3617 len = frag->size;
3618
3619 buffer_info = &tx_ring->buffer_info[i];
3620 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3621 buffer_info->length = len;
3622 buffer_info->time_stamp = jiffies;
0e014cb1 3623 buffer_info->next_to_watch = i;
6366ad33
AD		 3624 buffer_info->mapped_as_page = true;
3625 buffer_info->dma = pci_map_page(pdev,
3626 frag->page,
3627 frag->page_offset,
3628 len,
3629 PCI_DMA_TODEVICE);
3630 if (pci_dma_mapping_error(pdev, buffer_info->dma))
3631 goto dma_error;
3632
9d5c8243
AK		 3633 }
3634
9d5c8243 3635 tx_ring->buffer_info[i].skb = skb;
40e90c26 3636 tx_ring->buffer_info[i].gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
0e014cb1 3637 tx_ring->buffer_info[first].next_to_watch = i;
9d5c8243 3638
cdfd01fc 3639 return ++count;
6366ad33
AD		 3640
3641dma_error:
3642 dev_err(&pdev->dev, "TX DMA map failed\n");
3643
3644 /* clear timestamp and dma mappings for failed buffer_info mapping */
3645 buffer_info->dma = 0;
3646 buffer_info->time_stamp = 0;
3647 buffer_info->length = 0;
3648 buffer_info->next_to_watch = 0;
3649 buffer_info->mapped_as_page = false;
6366ad33
AD		 3650
3651 /* clear timestamp and dma mappings for remaining portion of packet */
a77ff709
NN		 3652 while (count--) {
3653 if (i == 0)
3654 i = tx_ring->count;
6366ad33 3655 i--;
6366ad33
AD		 3656 buffer_info = &tx_ring->buffer_info[i];
3657 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3658 }
3659
3660 return 0;
9d5c8243
AK		 3661}
3662
85ad76b2 3663static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
91d4ee33 3664 u32 tx_flags, int count, u32 paylen,
9d5c8243
AK		 3665 u8 hdr_len)
3666{
cdfd01fc 3667 union e1000_adv_tx_desc *tx_desc;
9d5c8243
AK		 3668 struct igb_buffer *buffer_info;
3669 u32 olinfo_status = 0, cmd_type_len;
cdfd01fc 3670 unsigned int i = tx_ring->next_to_use;
9d5c8243
AK		 3671
3672 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3673 E1000_ADVTXD_DCMD_DEXT);
3674
3675 if (tx_flags & IGB_TX_FLAGS_VLAN)
3676 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3677
33af6bcc
PO		 3678 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3679 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
3680
9d5c8243
AK		 3681 if (tx_flags & IGB_TX_FLAGS_TSO) {
3682 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3683
3684 /* insert tcp checksum */
3685 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3686
3687 /* insert ip checksum */
3688 if (tx_flags & IGB_TX_FLAGS_IPV4)
3689 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3690
3691 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
3692 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3693 }
3694
85ad76b2
AD		 3695 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
3696 (tx_flags & (IGB_TX_FLAGS_CSUM |
3697 IGB_TX_FLAGS_TSO |
7dfc16fa 3698 IGB_TX_FLAGS_VLAN)))
85ad76b2 3699 olinfo_status |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3700
3701 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
3702
cdfd01fc 3703 do {
9d5c8243
AK		 3704 buffer_info = &tx_ring->buffer_info[i];
3705 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
3706 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
3707 tx_desc->read.cmd_type_len =
3708 cpu_to_le32(cmd_type_len | buffer_info->length);
3709 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
cdfd01fc 3710 count--;
9d5c8243
AK		 3711 i++;
3712 if (i == tx_ring->count)
3713 i = 0;
cdfd01fc 3714 } while (count > 0);
9d5c8243 3715
85ad76b2 3716 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
9d5c8243
AK		 3717 /* Force memory writes to complete before letting h/w
3718 * know there are new descriptors to fetch. (Only
3719 * applicable for weak-ordered memory model archs,
3720 * such as IA-64). */
3721 wmb();
3722
3723 tx_ring->next_to_use = i;
fce99e34 3724 writel(i, tx_ring->tail);
9d5c8243
AK		 3725 /* we need this if more than one processor can write to our tail
3726 * at a time, it syncronizes IO on IA64/Altix systems */
3727 mmiowb();
3728}
3729
e694e964 3730static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3731{
e694e964
AD		 3732 struct net_device *netdev = tx_ring->netdev;
3733
661086df 3734 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 3735
9d5c8243
AK		 3736 /* Herbert's original patch had:
3737 * smp_mb__after_netif_stop_queue();
3738 * but since that doesn't exist yet, just open code it. */
3739 smp_mb();
3740
3741 /* We need to check again in a case another CPU has just
3742 * made room available. */
c493ea45 3743 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 3744 return -EBUSY;
3745
3746 /* A reprieve! */
661086df 3747 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 3748 tx_ring->tx_stats.restart_queue++;
9d5c8243
AK		 3749 return 0;
3750}
3751
717ba089 3752static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3753{
c493ea45 3754 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 3755 return 0;
e694e964 3756 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 3757}
3758
b1a436c3
AD		 3759netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
3760 struct igb_ring *tx_ring)
9d5c8243 3761{
e694e964 3762 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
cdfd01fc 3763 int tso = 0, count;
91d4ee33
NN		 3764 u32 tx_flags = 0;
3765 u16 first;
3766 u8 hdr_len = 0;
c5b9bd5e 3767 union skb_shared_tx *shtx = skb_tx(skb);
9d5c8243 3768
9d5c8243
AK		 3769 /* need: 1 descriptor per page,
3770 * + 2 desc gap to keep tail from touching head,
3771 * + 1 desc for skb->data,
3772 * + 1 desc for context descriptor,
3773 * otherwise try next time */
e694e964 3774 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 3775 /* this is a hard error */
9d5c8243
AK		 3776 return NETDEV_TX_BUSY;
3777 }
33af6bcc 3778
33af6bcc
PO		 3779 if (unlikely(shtx->hardware)) {
3780 shtx->in_progress = 1;
3781 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 3782 }
9d5c8243 3783
cdfd01fc 3784 if (vlan_tx_tag_present(skb) && adapter->vlgrp) {
9d5c8243
AK		 3785 tx_flags |= IGB_TX_FLAGS_VLAN;
3786 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
3787 }
3788
661086df
PWJ		 3789 if (skb->protocol == htons(ETH_P_IP))
3790 tx_flags |= IGB_TX_FLAGS_IPV4;
3791
0e014cb1 3792 first = tx_ring->next_to_use;
85ad76b2
AD		 3793 if (skb_is_gso(skb)) {
3794 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
cdfd01fc 3795
85ad76b2
AD		 3796 if (tso < 0) {
3797 dev_kfree_skb_any(skb);
3798 return NETDEV_TX_OK;
3799 }
9d5c8243
AK		 3800 }
3801
3802 if (tso)
3803 tx_flags |= IGB_TX_FLAGS_TSO;
85ad76b2 3804 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
bc1cbd34
AD		 3805 (skb->ip_summed == CHECKSUM_PARTIAL))
3806 tx_flags |= IGB_TX_FLAGS_CSUM;
9d5c8243 3807
65689fef 3808 /*
cdfd01fc 3809 * count reflects descriptors mapped, if 0 or less then mapping error
65689fef
AD		 3810 * has occured and we need to rewind the descriptor queue
3811 */
80785298 3812 count = igb_tx_map_adv(tx_ring, skb, first);
6366ad33 3813 if (!count) {
65689fef
AD		 3814 dev_kfree_skb_any(skb);
3815 tx_ring->buffer_info[first].time_stamp = 0;
3816 tx_ring->next_to_use = first;
85ad76b2 3817 return NETDEV_TX_OK;
65689fef 3818 }
9d5c8243 3819
85ad76b2
AD		 3820 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
3821
3822 /* Make sure there is space in the ring for the next send. */
e694e964 3823 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 3824
9d5c8243
AK		 3825 return NETDEV_TX_OK;
3826}
3827
3b29a56d
SH		 3828static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
3829 struct net_device *netdev)
9d5c8243
AK		 3830{
3831 struct igb_adapter *adapter = netdev_priv(netdev);
661086df 3832 struct igb_ring *tx_ring;
661086df 3833 int r_idx = 0;
b1a436c3
AD		 3834
3835 if (test_bit(__IGB_DOWN, &adapter->state)) {
3836 dev_kfree_skb_any(skb);
3837 return NETDEV_TX_OK;
3838 }
3839
3840 if (skb->len <= 0) {
3841 dev_kfree_skb_any(skb);
3842 return NETDEV_TX_OK;
3843 }
3844
1bfaf07b 3845 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
661086df 3846 tx_ring = adapter->multi_tx_table[r_idx];
9d5c8243
AK		 3847
3848 /* This goes back to the question of how to logically map a tx queue
3849 * to a flow. Right now, performance is impacted slightly negatively
3850 * if using multiple tx queues. If the stack breaks away from a
3851 * single qdisc implementation, we can look at this again. */
e694e964 3852 return igb_xmit_frame_ring_adv(skb, tx_ring);
9d5c8243
AK		 3853}
3854
3855/**
3856 * igb_tx_timeout - Respond to a Tx Hang
3857 * @netdev: network interface device structure
3858 **/
3859static void igb_tx_timeout(struct net_device *netdev)
3860{
3861 struct igb_adapter *adapter = netdev_priv(netdev);
3862 struct e1000_hw *hw = &adapter->hw;
3863
3864 /* Do the reset outside of interrupt context */
3865 adapter->tx_timeout_count++;
f7ba205e 3866
55cac248
AD		 3867 if (hw->mac.type == e1000_82580)
3868 hw->dev_spec._82575.global_device_reset = true;
3869
9d5c8243 3870 schedule_work(&adapter->reset_task);
265de409
AD		 3871 wr32(E1000_EICS,
3872 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 3873}
3874
3875static void igb_reset_task(struct work_struct *work)
3876{
3877 struct igb_adapter *adapter;
3878 adapter = container_of(work, struct igb_adapter, reset_task);
3879
3880 igb_reinit_locked(adapter);
3881}
3882
3883/**
3884 * igb_get_stats - Get System Network Statistics
3885 * @netdev: network interface device structure
3886 *
3887 * Returns the address of the device statistics structure.
3888 * The statistics are actually updated from the timer callback.
3889 **/
73cd78f1 3890static struct net_device_stats *igb_get_stats(struct net_device *netdev)
9d5c8243 3891{
9d5c8243 3892 /* only return the current stats */
8d24e933 3893 return &netdev->stats;
9d5c8243
AK		 3894}
3895
3896/**
3897 * igb_change_mtu - Change the Maximum Transfer Unit
3898 * @netdev: network interface device structure
3899 * @new_mtu: new value for maximum frame size
3900 *
3901 * Returns 0 on success, negative on failure
3902 **/
3903static int igb_change_mtu(struct net_device *netdev, int new_mtu)
3904{
3905 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 3906 struct pci_dev *pdev = adapter->pdev;
9d5c8243 3907 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4c844851 3908 u32 rx_buffer_len, i;
9d5c8243 3909
c809d227 3910 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 3911 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 3912 return -EINVAL;
3913 }
3914
9d5c8243 3915 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 3916 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 3917 return -EINVAL;
3918 }
3919
3920 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
3921 msleep(1);
73cd78f1 3922
9d5c8243
AK		 3923 /* igb_down has a dependency on max_frame_size */
3924 adapter->max_frame_size = max_frame;
559e9c49 3925
9d5c8243
AK		 3926 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3927 * means we reserve 2 more, this pushes us to allocate from the next
3928 * larger slab size.
3929 * i.e. RXBUFFER_2048 --> size-4096 slab
3930 */
3931
7d95b717 3932 if (max_frame <= IGB_RXBUFFER_1024)
4c844851 3933 rx_buffer_len = IGB_RXBUFFER_1024;
6ec43fe6 3934 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4c844851 3935 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
6ec43fe6 3936 else
4c844851
AD		 3937 rx_buffer_len = IGB_RXBUFFER_128;
3938
3939 if (netif_running(netdev))
3940 igb_down(adapter);
9d5c8243 3941
090b1795 3942 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 3943 netdev->mtu, new_mtu);
3944 netdev->mtu = new_mtu;
3945
4c844851 3946 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3947 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4c844851 3948
9d5c8243
AK		 3949 if (netif_running(netdev))
3950 igb_up(adapter);
3951 else
3952 igb_reset(adapter);
3953
3954 clear_bit(__IGB_RESETTING, &adapter->state);
3955
3956 return 0;
3957}
3958
3959/**
3960 * igb_update_stats - Update the board statistics counters
3961 * @adapter: board private structure
3962 **/
3963
3964void igb_update_stats(struct igb_adapter *adapter)
3965{
128e45eb 3966 struct net_device_stats *net_stats = igb_get_stats(adapter->netdev);
9d5c8243
AK		 3967 struct e1000_hw *hw = &adapter->hw;
3968 struct pci_dev *pdev = adapter->pdev;
43915c7c 3969 u32 rnbc, reg;
9d5c8243 3970 u16 phy_tmp;
3f9c0164
AD		 3971 int i;
3972 u64 bytes, packets;
9d5c8243
AK		 3973
3974#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3975
3976 /*
3977 * Prevent stats update while adapter is being reset, or if the pci
3978 * connection is down.
3979 */
3980 if (adapter->link_speed == 0)
3981 return;
3982 if (pci_channel_offline(pdev))
3983 return;
3984
3f9c0164
AD		 3985 bytes = 0;
3986 packets = 0;
3987 for (i = 0; i < adapter->num_rx_queues; i++) {
3988 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446
AD		 3989 struct igb_ring *ring = adapter->rx_ring[i];
3990 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 3991 net_stats->rx_fifo_errors += rqdpc_tmp;
3025a446
AD		 3992 bytes += ring->rx_stats.bytes;
3993 packets += ring->rx_stats.packets;
3f9c0164
AD		 3994 }
3995
128e45eb
AD		 3996 net_stats->rx_bytes = bytes;
3997 net_stats->rx_packets = packets;
3f9c0164
AD		 3998
3999 bytes = 0;
4000 packets = 0;
4001 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 4002 struct igb_ring *ring = adapter->tx_ring[i];
4003 bytes += ring->tx_stats.bytes;
4004 packets += ring->tx_stats.packets;
3f9c0164 4005 }
128e45eb
AD		 4006 net_stats->tx_bytes = bytes;
4007 net_stats->tx_packets = packets;
3f9c0164
AD		 4008
4009 /* read stats registers */
9d5c8243
AK		 4010 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4011 adapter->stats.gprc += rd32(E1000_GPRC);
4012 adapter->stats.gorc += rd32(E1000_GORCL);
4013 rd32(E1000_GORCH); /* clear GORCL */
4014 adapter->stats.bprc += rd32(E1000_BPRC);
4015 adapter->stats.mprc += rd32(E1000_MPRC);
4016 adapter->stats.roc += rd32(E1000_ROC);
4017
4018 adapter->stats.prc64 += rd32(E1000_PRC64);
4019 adapter->stats.prc127 += rd32(E1000_PRC127);
4020 adapter->stats.prc255 += rd32(E1000_PRC255);
4021 adapter->stats.prc511 += rd32(E1000_PRC511);
4022 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4023 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4024 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4025 adapter->stats.sec += rd32(E1000_SEC);
4026
4027 adapter->stats.mpc += rd32(E1000_MPC);
4028 adapter->stats.scc += rd32(E1000_SCC);
4029 adapter->stats.ecol += rd32(E1000_ECOL);
4030 adapter->stats.mcc += rd32(E1000_MCC);
4031 adapter->stats.latecol += rd32(E1000_LATECOL);
4032 adapter->stats.dc += rd32(E1000_DC);
4033 adapter->stats.rlec += rd32(E1000_RLEC);
4034 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4035 adapter->stats.xontxc += rd32(E1000_XONTXC);
4036 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4037 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4038 adapter->stats.fcruc += rd32(E1000_FCRUC);
4039 adapter->stats.gptc += rd32(E1000_GPTC);
4040 adapter->stats.gotc += rd32(E1000_GOTCL);
4041 rd32(E1000_GOTCH); /* clear GOTCL */
3f9c0164
AD		 4042 rnbc = rd32(E1000_RNBC);
4043 adapter->stats.rnbc += rnbc;
128e45eb 4044 net_stats->rx_fifo_errors += rnbc;
9d5c8243
AK		 4045 adapter->stats.ruc += rd32(E1000_RUC);
4046 adapter->stats.rfc += rd32(E1000_RFC);
4047 adapter->stats.rjc += rd32(E1000_RJC);
4048 adapter->stats.tor += rd32(E1000_TORH);
4049 adapter->stats.tot += rd32(E1000_TOTH);
4050 adapter->stats.tpr += rd32(E1000_TPR);
4051
4052 adapter->stats.ptc64 += rd32(E1000_PTC64);
4053 adapter->stats.ptc127 += rd32(E1000_PTC127);
4054 adapter->stats.ptc255 += rd32(E1000_PTC255);
4055 adapter->stats.ptc511 += rd32(E1000_PTC511);
4056 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4057 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4058
4059 adapter->stats.mptc += rd32(E1000_MPTC);
4060 adapter->stats.bptc += rd32(E1000_BPTC);
4061
2d0b0f69
NN		 4062 adapter->stats.tpt += rd32(E1000_TPT);
4063 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4064
4065 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4066 /* read internal phy specific stats */
4067 reg = rd32(E1000_CTRL_EXT);
4068 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4069 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4070 adapter->stats.tncrs += rd32(E1000_TNCRS);
4071 }
4072
9d5c8243
AK		 4073 adapter->stats.tsctc += rd32(E1000_TSCTC);
4074 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4075
4076 adapter->stats.iac += rd32(E1000_IAC);
4077 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4078 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4079 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4080 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4081 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4082 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4083 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4084 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4085
4086 /* Fill out the OS statistics structure */
128e45eb
AD		 4087 net_stats->multicast = adapter->stats.mprc;
4088 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4089
4090 /* Rx Errors */
4091
4092 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4093 * our own version based on RUC and ROC */
128e45eb 4094 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4095 adapter->stats.crcerrs + adapter->stats.algnerrc +
4096 adapter->stats.ruc + adapter->stats.roc +
4097 adapter->stats.cexterr;
128e45eb
AD		 4098 net_stats->rx_length_errors = adapter->stats.ruc +
4099 adapter->stats.roc;
4100 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4101 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4102 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4103
4104 /* Tx Errors */
128e45eb
AD		 4105 net_stats->tx_errors = adapter->stats.ecol +
4106 adapter->stats.latecol;
4107 net_stats->tx_aborted_errors = adapter->stats.ecol;
4108 net_stats->tx_window_errors = adapter->stats.latecol;
4109 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4110
4111 /* Tx Dropped needs to be maintained elsewhere */
4112
4113 /* Phy Stats */
4114 if (hw->phy.media_type == e1000_media_type_copper) {
4115 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4116 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4117 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4118 adapter->phy_stats.idle_errors += phy_tmp;
4119 }
4120 }
4121
4122 /* Management Stats */
4123 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4124 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4125 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4126}
4127
9d5c8243
AK		 4128static irqreturn_t igb_msix_other(int irq, void *data)
4129{
047e0030 4130 struct igb_adapter *adapter = data;
9d5c8243 4131 struct e1000_hw *hw = &adapter->hw;
844290e5 4132 u32 icr = rd32(E1000_ICR);
844290e5 4133 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4134
7f081d40
AD		 4135 if (icr & E1000_ICR_DRSTA)
4136 schedule_work(&adapter->reset_task);
4137
047e0030 4138 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4139 /* HW is reporting DMA is out of sync */
4140 adapter->stats.doosync++;
4141 }
eebbbdba 4142
4ae196df
AD		 4143 /* Check for a mailbox event */
4144 if (icr & E1000_ICR_VMMB)
4145 igb_msg_task(adapter);
4146
4147 if (icr & E1000_ICR_LSC) {
4148 hw->mac.get_link_status = 1;
4149 /* guard against interrupt when we're going down */
4150 if (!test_bit(__IGB_DOWN, &adapter->state))
4151 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4152 }
4153
25568a53
AD		 4154 if (adapter->vfs_allocated_count)
4155 wr32(E1000_IMS, E1000_IMS_LSC |
4156 E1000_IMS_VMMB |
4157 E1000_IMS_DOUTSYNC);
4158 else
4159 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
844290e5 4160 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4161
4162 return IRQ_HANDLED;
4163}
4164
047e0030 4165static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4166{
26b39276 4167 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4168 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4169
047e0030
AD		 4170 if (!q_vector->set_itr)
4171 return;
73cd78f1 4172
047e0030
AD		 4173 if (!itr_val)
4174 itr_val = 0x4;
661086df 4175
26b39276
AD		 4176 if (adapter->hw.mac.type == e1000_82575)
4177 itr_val |= itr_val << 16;
661086df 4178 else
047e0030 4179 itr_val |= 0x8000000;
661086df 4180
047e0030
AD		 4181 writel(itr_val, q_vector->itr_register);
4182 q_vector->set_itr = 0;
6eb5a7f1
AD		 4183}
4184
047e0030 4185static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4186{
047e0030 4187 struct igb_q_vector *q_vector = data;
9d5c8243 4188
047e0030
AD		 4189 /* Write the ITR value calculated from the previous interrupt. */
4190 igb_write_itr(q_vector);
9d5c8243 4191
047e0030 4192 napi_schedule(&q_vector->napi);
844290e5 4193
047e0030 4194 return IRQ_HANDLED;
fe4506b6
JC		 4195}
4196
421e02f0 4197#ifdef CONFIG_IGB_DCA
047e0030 4198static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4199{
047e0030 4200 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4201 struct e1000_hw *hw = &adapter->hw;
4202 int cpu = get_cpu();
fe4506b6 4203
047e0030
AD		 4204 if (q_vector->cpu == cpu)
4205 goto out_no_update;
4206
4207 if (q_vector->tx_ring) {
4208 int q = q_vector->tx_ring->reg_idx;
4209 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4210 if (hw->mac.type == e1000_82575) {
4211 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4212 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4213 } else {
047e0030
AD		 4214 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4215 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4216 E1000_DCA_TXCTRL_CPUID_SHIFT;
4217 }
4218 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4219 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4220 }
4221 if (q_vector->rx_ring) {
4222 int q = q_vector->rx_ring->reg_idx;
4223 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4224 if (hw->mac.type == e1000_82575) {
2d064c06 4225 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4226 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4227 } else {
4228 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4229 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4230 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4231 }
fe4506b6
JC		 4232 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4233 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4234 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4235 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4236 }
047e0030
AD		 4237 q_vector->cpu = cpu;
4238out_no_update:
fe4506b6
JC		 4239 put_cpu();
4240}
4241
4242static void igb_setup_dca(struct igb_adapter *adapter)
4243{
7e0e99ef 4244 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4245 int i;
4246
7dfc16fa 4247 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4248 return;
4249
7e0e99ef
AD		 4250 /* Always use CB2 mode, difference is masked in the CB driver. */
4251 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4252
047e0030 4253 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4254 adapter->q_vector[i]->cpu = -1;
4255 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4256 }
4257}
4258
4259static int __igb_notify_dca(struct device *dev, void *data)
4260{
4261 struct net_device *netdev = dev_get_drvdata(dev);
4262 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4263 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4264 struct e1000_hw *hw = &adapter->hw;
4265 unsigned long event = *(unsigned long *)data;
4266
4267 switch (event) {
4268 case DCA_PROVIDER_ADD:
4269 /* if already enabled, don't do it again */
7dfc16fa 4270 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4271 break;
fe4506b6 4272 if (dca_add_requester(dev) == 0) {
bbd98fe4 4273 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4274 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4275 igb_setup_dca(adapter);
4276 break;
4277 }
4278 /* Fall Through since DCA is disabled. */
4279 case DCA_PROVIDER_REMOVE:
7dfc16fa 4280 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4281 /* without this a class_device is left
047e0030 4282 * hanging around in the sysfs model */
fe4506b6 4283 dca_remove_requester(dev);
090b1795 4284 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4285 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4286 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4287 }
4288 break;
4289 }
bbd98fe4 4290
fe4506b6 4291 return 0;
9d5c8243
AK		 4292}
4293
fe4506b6
JC		 4294static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4295 void *p)
4296{
4297 int ret_val;
4298
4299 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4300 __igb_notify_dca);
4301
4302 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4303}
421e02f0 4304#endif /* CONFIG_IGB_DCA */
9d5c8243 4305
4ae196df
AD		 4306static void igb_ping_all_vfs(struct igb_adapter *adapter)
4307{
4308 struct e1000_hw *hw = &adapter->hw;
4309 u32 ping;
4310 int i;
4311
4312 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4313 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 4314 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 4315 ping |= E1000_VT_MSGTYPE_CTS;
4316 igb_write_mbx(hw, &ping, 1, i);
4317 }
4318}
4319
7d5753f0
AD		 4320static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4321{
4322 struct e1000_hw *hw = &adapter->hw;
4323 u32 vmolr = rd32(E1000_VMOLR(vf));
4324 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4325
4326 vf_data->flags |= ~(IGB_VF_FLAG_UNI_PROMISC |
4327 IGB_VF_FLAG_MULTI_PROMISC);
4328 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4329
4330 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4331 vmolr |= E1000_VMOLR_MPME;
4332 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4333 } else {
4334 /*
4335 * if we have hashes and we are clearing a multicast promisc
4336 * flag we need to write the hashes to the MTA as this step
4337 * was previously skipped
4338 */
4339 if (vf_data->num_vf_mc_hashes > 30) {
4340 vmolr |= E1000_VMOLR_MPME;
4341 } else if (vf_data->num_vf_mc_hashes) {
4342 int j;
4343 vmolr |= E1000_VMOLR_ROMPE;
4344 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4345 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4346 }
4347 }
4348
4349 wr32(E1000_VMOLR(vf), vmolr);
4350
4351 /* there are flags left unprocessed, likely not supported */
4352 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4353 return -EINVAL;
4354
4355 return 0;
4356
4357}
4358
4ae196df
AD		 4359static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4360 u32 *msgbuf, u32 vf)
4361{
4362 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4363 u16 *hash_list = (u16 *)&msgbuf[1];
4364 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4365 int i;
4366
7d5753f0 4367 /* salt away the number of multicast addresses assigned
4ae196df
AD		 4368 * to this VF for later use to restore when the PF multi cast
4369 * list changes
4370 */
4371 vf_data->num_vf_mc_hashes = n;
4372
7d5753f0
AD		 4373 /* only up to 30 hash values supported */
4374 if (n > 30)
4375 n = 30;
4376
4377 /* store the hashes for later use */
4ae196df 4378 for (i = 0; i < n; i++)
a419aef8 4379 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 4380
4381 /* Flush and reset the mta with the new values */
ff41f8dc 4382 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4383
4384 return 0;
4385}
4386
4387static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4388{
4389 struct e1000_hw *hw = &adapter->hw;
4390 struct vf_data_storage *vf_data;
4391 int i, j;
4392
4393 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 4394 u32 vmolr = rd32(E1000_VMOLR(i));
4395 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4396
4ae196df 4397 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 4398
4399 if ((vf_data->num_vf_mc_hashes > 30) ||
4400 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4401 vmolr |= E1000_VMOLR_MPME;
4402 } else if (vf_data->num_vf_mc_hashes) {
4403 vmolr |= E1000_VMOLR_ROMPE;
4404 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4405 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4406 }
4407 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 4408 }
4409}
4410
4411static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4412{
4413 struct e1000_hw *hw = &adapter->hw;
4414 u32 pool_mask, reg, vid;
4415 int i;
4416
4417 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4418
4419 /* Find the vlan filter for this id */
4420 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4421 reg = rd32(E1000_VLVF(i));
4422
4423 /* remove the vf from the pool */
4424 reg &= ~pool_mask;
4425
4426 /* if pool is empty then remove entry from vfta */
4427 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4428 (reg & E1000_VLVF_VLANID_ENABLE)) {
4429 reg = 0;
4430 vid = reg & E1000_VLVF_VLANID_MASK;
4431 igb_vfta_set(hw, vid, false);
4432 }
4433
4434 wr32(E1000_VLVF(i), reg);
4435 }
ae641bdc
AD		 4436
4437 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 4438}
4439
4440static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4441{
4442 struct e1000_hw *hw = &adapter->hw;
4443 u32 reg, i;
4444
51466239
AD		 4445 /* The vlvf table only exists on 82576 hardware and newer */
4446 if (hw->mac.type < e1000_82576)
4447 return -1;
4448
4449 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 4450 if (!adapter->vfs_allocated_count)
4451 return -1;
4452
4453 /* Find the vlan filter for this id */
4454 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4455 reg = rd32(E1000_VLVF(i));
4456 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4457 vid == (reg & E1000_VLVF_VLANID_MASK))
4458 break;
4459 }
4460
4461 if (add) {
4462 if (i == E1000_VLVF_ARRAY_SIZE) {
4463 /* Did not find a matching VLAN ID entry that was
4464 * enabled. Search for a free filter entry, i.e.
4465 * one without the enable bit set
4466 */
4467 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4468 reg = rd32(E1000_VLVF(i));
4469 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4470 break;
4471 }
4472 }
4473 if (i < E1000_VLVF_ARRAY_SIZE) {
4474 /* Found an enabled/available entry */
4475 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4476
4477 /* if !enabled we need to set this up in vfta */
4478 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 4479 /* add VID to filter table */
4480 igb_vfta_set(hw, vid, true);
4ae196df
AD		 4481 reg |= E1000_VLVF_VLANID_ENABLE;
4482 }
cad6d05f
AD		 4483 reg &= ~E1000_VLVF_VLANID_MASK;
4484 reg |= vid;
4ae196df 4485 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4486
4487 /* do not modify RLPML for PF devices */
4488 if (vf >= adapter->vfs_allocated_count)
4489 return 0;
4490
4491 if (!adapter->vf_data[vf].vlans_enabled) {
4492 u32 size;
4493 reg = rd32(E1000_VMOLR(vf));
4494 size = reg & E1000_VMOLR_RLPML_MASK;
4495 size += 4;
4496 reg &= ~E1000_VMOLR_RLPML_MASK;
4497 reg |= size;
4498 wr32(E1000_VMOLR(vf), reg);
4499 }
ae641bdc 4500
51466239 4501 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 4502 return 0;
4503 }
4504 } else {
4505 if (i < E1000_VLVF_ARRAY_SIZE) {
4506 /* remove vf from the pool */
4507 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4508 /* if pool is empty then remove entry from vfta */
4509 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4510 reg = 0;
4511 igb_vfta_set(hw, vid, false);
4512 }
4513 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4514
4515 /* do not modify RLPML for PF devices */
4516 if (vf >= adapter->vfs_allocated_count)
4517 return 0;
4518
4519 adapter->vf_data[vf].vlans_enabled--;
4520 if (!adapter->vf_data[vf].vlans_enabled) {
4521 u32 size;
4522 reg = rd32(E1000_VMOLR(vf));
4523 size = reg & E1000_VMOLR_RLPML_MASK;
4524 size -= 4;
4525 reg &= ~E1000_VMOLR_RLPML_MASK;
4526 reg |= size;
4527 wr32(E1000_VMOLR(vf), reg);
4528 }
4ae196df
AD		 4529 }
4530 }
8151d294
WM		 4531 return 0;
4532}
4533
4534static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
4535{
4536 struct e1000_hw *hw = &adapter->hw;
4537
4538 if (vid)
4539 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
4540 else
4541 wr32(E1000_VMVIR(vf), 0);
4542}
4543
4544static int igb_ndo_set_vf_vlan(struct net_device *netdev,
4545 int vf, u16 vlan, u8 qos)
4546{
4547 int err = 0;
4548 struct igb_adapter *adapter = netdev_priv(netdev);
4549
4550 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
4551 return -EINVAL;
4552 if (vlan || qos) {
4553 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
4554 if (err)
4555 goto out;
4556 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
4557 igb_set_vmolr(adapter, vf, !vlan);
4558 adapter->vf_data[vf].pf_vlan = vlan;
4559 adapter->vf_data[vf].pf_qos = qos;
4560 dev_info(&adapter->pdev->dev,
4561 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
4562 if (test_bit(__IGB_DOWN, &adapter->state)) {
4563 dev_warn(&adapter->pdev->dev,
4564 "The VF VLAN has been set,"
4565 " but the PF device is not up.\n");
4566 dev_warn(&adapter->pdev->dev,
4567 "Bring the PF device up before"
4568 " attempting to use the VF device.\n");
4569 }
4570 } else {
4571 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
4572 false, vf);
4573 igb_set_vmvir(adapter, vlan, vf);
4574 igb_set_vmolr(adapter, vf, true);
4575 adapter->vf_data[vf].pf_vlan = 0;
4576 adapter->vf_data[vf].pf_qos = 0;
4577 }
4578out:
4579 return err;
4ae196df
AD		 4580}
4581
4582static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4583{
4584 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4585 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4586
4587 return igb_vlvf_set(adapter, vid, add, vf);
4588}
4589
f2ca0dbe 4590static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 4591{
8151d294
WM		 4592 /* clear flags */
4593 adapter->vf_data[vf].flags &= ~(IGB_VF_FLAG_PF_SET_MAC);
f2ca0dbe 4594 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 4595
4596 /* reset offloads to defaults */
8151d294 4597 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 4598
4599 /* reset vlans for device */
4600 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 4601 if (adapter->vf_data[vf].pf_vlan)
4602 igb_ndo_set_vf_vlan(adapter->netdev, vf,
4603 adapter->vf_data[vf].pf_vlan,
4604 adapter->vf_data[vf].pf_qos);
4605 else
4606 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 4607
4608 /* reset multicast table array for vf */
4609 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4610
4611 /* Flush and reset the mta with the new values */
ff41f8dc 4612 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4613}
4614
f2ca0dbe
AD		 4615static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4616{
4617 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4618
4619 /* generate a new mac address as we were hotplug removed/added */
8151d294
WM		 4620 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
4621 random_ether_addr(vf_mac);
f2ca0dbe
AD		 4622
4623 /* process remaining reset events */
4624 igb_vf_reset(adapter, vf);
4625}
4626
4627static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 4628{
4629 struct e1000_hw *hw = &adapter->hw;
4630 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 4631 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 4632 u32 reg, msgbuf[3];
4633 u8 *addr = (u8 *)(&msgbuf[1]);
4634
4635 /* process all the same items cleared in a function level reset */
f2ca0dbe 4636 igb_vf_reset(adapter, vf);
4ae196df
AD		 4637
4638 /* set vf mac address */
26ad9178 4639 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 4640
4641 /* enable transmit and receive for vf */
4642 reg = rd32(E1000_VFTE);
4643 wr32(E1000_VFTE, reg | (1 << vf));
4644 reg = rd32(E1000_VFRE);
4645 wr32(E1000_VFRE, reg | (1 << vf));
4646
f2ca0dbe 4647 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4ae196df
AD		 4648
4649 /* reply to reset with ack and vf mac address */
4650 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4651 memcpy(addr, vf_mac, 6);
4652 igb_write_mbx(hw, msgbuf, 3, vf);
4653}
4654
4655static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4656{
f2ca0dbe
AD		 4657 unsigned char *addr = (char *)&msg[1];
4658 int err = -1;
4ae196df 4659
f2ca0dbe
AD		 4660 if (is_valid_ether_addr(addr))
4661 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 4662
f2ca0dbe 4663 return err;
4ae196df
AD		 4664}
4665
4666static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
4667{
4668 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 4669 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 4670 u32 msg = E1000_VT_MSGTYPE_NACK;
4671
4672 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 4673 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
4674 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 4675 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 4676 vf_data->last_nack = jiffies;
4ae196df
AD		 4677 }
4678}
4679
f2ca0dbe 4680static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 4681{
f2ca0dbe
AD		 4682 struct pci_dev *pdev = adapter->pdev;
4683 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 4684 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 4685 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 4686 s32 retval;
4687
f2ca0dbe 4688 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 4689
fef45f4c
AD		 4690 if (retval) {
4691 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 4692 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 4693 vf_data->flags &= ~IGB_VF_FLAG_CTS;
4694 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
4695 return;
4696 goto out;
4697 }
4ae196df
AD		 4698
4699 /* this is a message we already processed, do nothing */
4700 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 4701 return;
4ae196df
AD		 4702
4703 /*
4704 * until the vf completes a reset it should not be
4705 * allowed to start any configuration.
4706 */
4707
4708 if (msgbuf[0] == E1000_VF_RESET) {
4709 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 4710 return;
4ae196df
AD		 4711 }
4712
f2ca0dbe 4713 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 4714 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
4715 return;
4716 retval = -1;
4717 goto out;
4ae196df
AD		 4718 }
4719
4720 switch ((msgbuf[0] & 0xFFFF)) {
4721 case E1000_VF_SET_MAC_ADDR:
4722 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
4723 break;
7d5753f0
AD		 4724 case E1000_VF_SET_PROMISC:
4725 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
4726 break;
4ae196df
AD		 4727 case E1000_VF_SET_MULTICAST:
4728 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
4729 break;
4730 case E1000_VF_SET_LPE:
4731 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
4732 break;
4733 case E1000_VF_SET_VLAN:
8151d294
WM		 4734 if (adapter->vf_data[vf].pf_vlan)
4735 retval = -1;
4736 else
4737 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 4738 break;
4739 default:
090b1795 4740 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 4741 retval = -1;
4742 break;
4743 }
4744
fef45f4c
AD		 4745 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
4746out:
4ae196df
AD		 4747 /* notify the VF of the results of what it sent us */
4748 if (retval)
4749 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
4750 else
4751 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
4752
4ae196df 4753 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 4754}
4ae196df 4755
f2ca0dbe
AD		 4756static void igb_msg_task(struct igb_adapter *adapter)
4757{
4758 struct e1000_hw *hw = &adapter->hw;
4759 u32 vf;
4760
4761 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
4762 /* process any reset requests */
4763 if (!igb_check_for_rst(hw, vf))
4764 igb_vf_reset_event(adapter, vf);
4765
4766 /* process any messages pending */
4767 if (!igb_check_for_msg(hw, vf))
4768 igb_rcv_msg_from_vf(adapter, vf);
4769
4770 /* process any acks */
4771 if (!igb_check_for_ack(hw, vf))
4772 igb_rcv_ack_from_vf(adapter, vf);
4773 }
4ae196df
AD		 4774}
4775
68d480c4
AD		 4776/**
4777 * igb_set_uta - Set unicast filter table address
4778 * @adapter: board private structure
4779 *
4780 * The unicast table address is a register array of 32-bit registers.
4781 * The table is meant to be used in a way similar to how the MTA is used
4782 * however due to certain limitations in the hardware it is necessary to
4783 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
4784 * enable bit to allow vlan tag stripping when promiscous mode is enabled
4785 **/
4786static void igb_set_uta(struct igb_adapter *adapter)
4787{
4788 struct e1000_hw *hw = &adapter->hw;
4789 int i;
4790
4791 /* The UTA table only exists on 82576 hardware and newer */
4792 if (hw->mac.type < e1000_82576)
4793 return;
4794
4795 /* we only need to do this if VMDq is enabled */
4796 if (!adapter->vfs_allocated_count)
4797 return;
4798
4799 for (i = 0; i < hw->mac.uta_reg_count; i++)
4800 array_wr32(E1000_UTA, i, ~0);
4801}
4802
9d5c8243
AK		 4803/**
4804 * igb_intr_msi - Interrupt Handler
4805 * @irq: interrupt number
4806 * @data: pointer to a network interface device structure
4807 **/
4808static irqreturn_t igb_intr_msi(int irq, void *data)
4809{
047e0030
AD		 4810 struct igb_adapter *adapter = data;
4811 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4812 struct e1000_hw *hw = &adapter->hw;
4813 /* read ICR disables interrupts using IAM */
4814 u32 icr = rd32(E1000_ICR);
4815
047e0030 4816 igb_write_itr(q_vector);
9d5c8243 4817
7f081d40
AD		 4818 if (icr & E1000_ICR_DRSTA)
4819 schedule_work(&adapter->reset_task);
4820
047e0030 4821 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4822 /* HW is reporting DMA is out of sync */
4823 adapter->stats.doosync++;
4824 }
4825
9d5c8243
AK		 4826 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4827 hw->mac.get_link_status = 1;
4828 if (!test_bit(__IGB_DOWN, &adapter->state))
4829 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4830 }
4831
047e0030 4832 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4833
4834 return IRQ_HANDLED;
4835}
4836
4837/**
4a3c6433 4838 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 4839 * @irq: interrupt number
4840 * @data: pointer to a network interface device structure
4841 **/
4842static irqreturn_t igb_intr(int irq, void *data)
4843{
047e0030
AD		 4844 struct igb_adapter *adapter = data;
4845 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4846 struct e1000_hw *hw = &adapter->hw;
4847 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
4848 * need for the IMC write */
4849 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 4850 if (!icr)
4851 return IRQ_NONE; /* Not our interrupt */
4852
047e0030 4853 igb_write_itr(q_vector);
9d5c8243
AK		 4854
4855 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
4856 * not set, then the adapter didn't send an interrupt */
4857 if (!(icr & E1000_ICR_INT_ASSERTED))
4858 return IRQ_NONE;
4859
7f081d40
AD		 4860 if (icr & E1000_ICR_DRSTA)
4861 schedule_work(&adapter->reset_task);
4862
047e0030 4863 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4864 /* HW is reporting DMA is out of sync */
4865 adapter->stats.doosync++;
4866 }
4867
9d5c8243
AK		 4868 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4869 hw->mac.get_link_status = 1;
4870 /* guard against interrupt when we're going down */
4871 if (!test_bit(__IGB_DOWN, &adapter->state))
4872 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4873 }
4874
047e0030 4875 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4876
4877 return IRQ_HANDLED;
4878}
4879
047e0030 4880static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 4881{
047e0030 4882 struct igb_adapter *adapter = q_vector->adapter;
46544258 4883 struct e1000_hw *hw = &adapter->hw;
9d5c8243 4884
4fc82adf
AD		 4885 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
4886 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
047e0030 4887 if (!adapter->msix_entries)
6eb5a7f1 4888 igb_set_itr(adapter);
46544258 4889 else
047e0030 4890 igb_update_ring_itr(q_vector);
9d5c8243
AK		 4891 }
4892
46544258
AD		 4893 if (!test_bit(__IGB_DOWN, &adapter->state)) {
4894 if (adapter->msix_entries)
047e0030 4895 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 4896 else
4897 igb_irq_enable(adapter);
4898 }
9d5c8243
AK		 4899}
4900
46544258
AD		 4901/**
4902 * igb_poll - NAPI Rx polling callback
4903 * @napi: napi polling structure
4904 * @budget: count of how many packets we should handle
4905 **/
4906static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 4907{
047e0030
AD		 4908 struct igb_q_vector *q_vector = container_of(napi,
4909 struct igb_q_vector,
4910 napi);
4911 int tx_clean_complete = 1, work_done = 0;
9d5c8243 4912
421e02f0 4913#ifdef CONFIG_IGB_DCA
047e0030
AD		 4914 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
4915 igb_update_dca(q_vector);
fe4506b6 4916#endif
047e0030
AD		 4917 if (q_vector->tx_ring)
4918 tx_clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 4919
047e0030
AD		 4920 if (q_vector->rx_ring)
4921 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
4922
4923 if (!tx_clean_complete)
4924 work_done = budget;
46544258 4925
9d5c8243 4926 /* If not enough Rx work done, exit the polling mode */
5e6d5b17 4927 if (work_done < budget) {
288379f0 4928 napi_complete(napi);
047e0030 4929 igb_ring_irq_enable(q_vector);
9d5c8243
AK		 4930 }
4931
46544258 4932 return work_done;
9d5c8243 4933}
6d8126f9 4934
33af6bcc 4935/**
c5b9bd5e 4936 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 4937 * @adapter: board private structure
c5b9bd5e
AD		 4938 * @shhwtstamps: timestamp structure to update
4939 * @regval: unsigned 64bit system time value.
4940 *
4941 * We need to convert the system time value stored in the RX/TXSTMP registers
4942 * into a hwtstamp which can be used by the upper level timestamping functions
4943 */
4944static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
4945 struct skb_shared_hwtstamps *shhwtstamps,
4946 u64 regval)
4947{
4948 u64 ns;
4949
55cac248
AD		 4950 /*
4951 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
4952 * 24 to match clock shift we setup earlier.
4953 */
4954 if (adapter->hw.mac.type == e1000_82580)
4955 regval <<= IGB_82580_TSYNC_SHIFT;
4956
c5b9bd5e
AD		 4957 ns = timecounter_cyc2time(&adapter->clock, regval);
4958 timecompare_update(&adapter->compare, ns);
4959 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
4960 shhwtstamps->hwtstamp = ns_to_ktime(ns);
4961 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
4962}
4963
4964/**
4965 * igb_tx_hwtstamp - utility function which checks for TX time stamp
4966 * @q_vector: pointer to q_vector containing needed info
33af6bcc
PO		 4967 * @skb: packet that was just sent
4968 *
4969 * If we were asked to do hardware stamping and such a time stamp is
4970 * available, then it must have been for this skb here because we only
4971 * allow only one such packet into the queue.
4972 */
c5b9bd5e 4973static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct sk_buff *skb)
33af6bcc 4974{
c5b9bd5e 4975 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc
PO		 4976 union skb_shared_tx *shtx = skb_tx(skb);
4977 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 4978 struct skb_shared_hwtstamps shhwtstamps;
4979 u64 regval;
33af6bcc 4980
c5b9bd5e
AD		 4981 /* if skb does not support hw timestamp or TX stamp not valid exit */
4982 if (likely(!shtx->hardware) ||
4983 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
4984 return;
4985
4986 regval = rd32(E1000_TXSTMPL);
4987 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
4988
4989 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
4990 skb_tstamp_tx(skb, &shhwtstamps);
33af6bcc
PO		 4991}
4992
9d5c8243
AK		 4993/**
4994 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 4995 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 4996 * returns true if ring is completely cleaned
4997 **/
047e0030 4998static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 4999{
047e0030
AD		 5000 struct igb_adapter *adapter = q_vector->adapter;
5001 struct igb_ring *tx_ring = q_vector->tx_ring;
e694e964 5002 struct net_device *netdev = tx_ring->netdev;
0e014cb1 5003 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 5004 struct igb_buffer *buffer_info;
5005 struct sk_buff *skb;
0e014cb1 5006 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 5007 unsigned int total_bytes = 0, total_packets = 0;
0e014cb1
AD		 5008 unsigned int i, eop, count = 0;
5009 bool cleaned = false;
9d5c8243 5010
9d5c8243 5011 i = tx_ring->next_to_clean;
0e014cb1
AD		 5012 eop = tx_ring->buffer_info[i].next_to_watch;
5013 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5014
5015 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5016 (count < tx_ring->count)) {
5017 for (cleaned = false; !cleaned; count++) {
5018 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
9d5c8243 5019 buffer_info = &tx_ring->buffer_info[i];
0e014cb1 5020 cleaned = (i == eop);
9d5c8243
AK		 5021 skb = buffer_info->skb;
5022
5023 if (skb) {
5024 unsigned int segs, bytecount;
5025 /* gso_segs is currently only valid for tcp */
40e90c26 5026 segs = buffer_info->gso_segs;
9d5c8243
AK		 5027 /* multiply data chunks by size of headers */
5028 bytecount = ((segs - 1) * skb_headlen(skb)) +
5029 skb->len;
5030 total_packets += segs;
5031 total_bytes += bytecount;
33af6bcc 5032
c5b9bd5e 5033 igb_tx_hwtstamp(q_vector, skb);
9d5c8243
AK		 5034 }
5035
80785298 5036 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
0e014cb1 5037 tx_desc->wb.status = 0;
9d5c8243
AK		 5038
5039 i++;
5040 if (i == tx_ring->count)
5041 i = 0;
9d5c8243 5042 }
0e014cb1
AD		 5043 eop = tx_ring->buffer_info[i].next_to_watch;
5044 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5045 }
5046
9d5c8243
AK		 5047 tx_ring->next_to_clean = i;
5048
fc7d345d 5049 if (unlikely(count &&
9d5c8243 5050 netif_carrier_ok(netdev) &&
c493ea45 5051 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
9d5c8243
AK		 5052 /* Make sure that anybody stopping the queue after this
5053 * sees the new next_to_clean.
5054 */
5055 smp_mb();
661086df
PWJ		 5056 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5057 !(test_bit(__IGB_DOWN, &adapter->state))) {
5058 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 5059 tx_ring->tx_stats.restart_queue++;
661086df 5060 }
9d5c8243
AK		 5061 }
5062
5063 if (tx_ring->detect_tx_hung) {
5064 /* Detect a transmit hang in hardware, this serializes the
5065 * check with the clearing of time_stamp and movement of i */
5066 tx_ring->detect_tx_hung = false;
5067 if (tx_ring->buffer_info[i].time_stamp &&
5068 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
8e95a202
JP		 5069 (adapter->tx_timeout_factor * HZ)) &&
5070 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5071
9d5c8243 5072 /* detected Tx unit hang */
80785298 5073 dev_err(&tx_ring->pdev->dev,
9d5c8243 5074 "Detected Tx Unit Hang\n"
2d064c06 5075 " Tx Queue <%d>\n"
9d5c8243
AK		 5076 " TDH <%x>\n"
5077 " TDT <%x>\n"
5078 " next_to_use <%x>\n"
5079 " next_to_clean <%x>\n"
9d5c8243
AK		 5080 "buffer_info[next_to_clean]\n"
5081 " time_stamp <%lx>\n"
0e014cb1 5082 " next_to_watch <%x>\n"
9d5c8243
AK		 5083 " jiffies <%lx>\n"
5084 " desc.status <%x>\n",
2d064c06 5085 tx_ring->queue_index,
fce99e34
AD		 5086 readl(tx_ring->head),
5087 readl(tx_ring->tail),
9d5c8243
AK		 5088 tx_ring->next_to_use,
5089 tx_ring->next_to_clean,
f7ba205e 5090 tx_ring->buffer_info[eop].time_stamp,
0e014cb1 5091 eop,
9d5c8243 5092 jiffies,
0e014cb1 5093 eop_desc->wb.status);
661086df 5094 netif_stop_subqueue(netdev, tx_ring->queue_index);
9d5c8243
AK		 5095 }
5096 }
5097 tx_ring->total_bytes += total_bytes;
5098 tx_ring->total_packets += total_packets;
e21ed353
AD		 5099 tx_ring->tx_stats.bytes += total_bytes;
5100 tx_ring->tx_stats.packets += total_packets;
0e014cb1 5101 return (count < tx_ring->count);
9d5c8243
AK		 5102}
5103
9d5c8243
AK		 5104/**
5105 * igb_receive_skb - helper function to handle rx indications
047e0030
AD		 5106 * @q_vector: structure containing interrupt and ring information
5107 * @skb: packet to send up
5108 * @vlan_tag: vlan tag for packet
9d5c8243 5109 **/
047e0030
AD		 5110static void igb_receive_skb(struct igb_q_vector *q_vector,
5111 struct sk_buff *skb,
5112 u16 vlan_tag)
5113{
5114 struct igb_adapter *adapter = q_vector->adapter;
5115
5116 if (vlan_tag)
5117 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5118 vlan_tag, skb);
182ff8df 5119 else
047e0030 5120 napi_gro_receive(&q_vector->napi, skb);
9d5c8243
AK		 5121}
5122
04a5fcaa 5123static inline void igb_rx_checksum_adv(struct igb_ring *ring,
9d5c8243
AK		 5124 u32 status_err, struct sk_buff *skb)
5125{
5126 skb->ip_summed = CHECKSUM_NONE;
5127
5128 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 5129 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5130 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 5131 return;
85ad76b2 5132
9d5c8243
AK		 5133 /* TCP/UDP checksum error bit is set */
5134 if (status_err &
5135 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5136 /*
5137 * work around errata with sctp packets where the TCPE aka
5138 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5139 * packets, (aka let the stack check the crc32c)
5140 */
85ad76b2
AD		 5141 if ((skb->len == 60) &&
5142 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
04a5fcaa 5143 ring->rx_stats.csum_err++;
85ad76b2 5144
9d5c8243 5145 /* let the stack verify checksum errors */
9d5c8243
AK		 5146 return;
5147 }
5148 /* It must be a TCP or UDP packet with a valid checksum */
5149 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5150 skb->ip_summed = CHECKSUM_UNNECESSARY;
5151
85ad76b2 5152 dev_dbg(&ring->pdev->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 5153}
5154
c5b9bd5e
AD		 5155static inline void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
5156 struct sk_buff *skb)
5157{
5158 struct igb_adapter *adapter = q_vector->adapter;
5159 struct e1000_hw *hw = &adapter->hw;
5160 u64 regval;
5161
5162 /*
5163 * If this bit is set, then the RX registers contain the time stamp. No
5164 * other packet will be time stamped until we read these registers, so
5165 * read the registers to make them available again. Because only one
5166 * packet can be time stamped at a time, we know that the register
5167 * values must belong to this one here and therefore we don't need to
5168 * compare any of the additional attributes stored for it.
5169 *
5170 * If nothing went wrong, then it should have a skb_shared_tx that we
5171 * can turn into a skb_shared_hwtstamps.
5172 */
5173 if (likely(!(staterr & E1000_RXDADV_STAT_TS)))
5174 return;
5175 if (!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5176 return;
5177
5178 regval = rd32(E1000_RXSTMPL);
5179 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5180
5181 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5182}
4c844851 5183static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
2d94d8ab
AD		 5184 union e1000_adv_rx_desc *rx_desc)
5185{
5186 /* HW will not DMA in data larger than the given buffer, even if it
5187 * parses the (NFS, of course) header to be larger. In that case, it
5188 * fills the header buffer and spills the rest into the page.
5189 */
5190 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5191 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4c844851
AD		 5192 if (hlen > rx_ring->rx_buffer_len)
5193 hlen = rx_ring->rx_buffer_len;
2d94d8ab
AD		 5194 return hlen;
5195}
5196
047e0030
AD		 5197static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5198 int *work_done, int budget)
9d5c8243 5199{
047e0030 5200 struct igb_ring *rx_ring = q_vector->rx_ring;
e694e964 5201 struct net_device *netdev = rx_ring->netdev;
80785298 5202 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 5203 union e1000_adv_rx_desc *rx_desc , *next_rxd;
5204 struct igb_buffer *buffer_info , *next_buffer;
5205 struct sk_buff *skb;
9d5c8243
AK		 5206 bool cleaned = false;
5207 int cleaned_count = 0;
d1eff350 5208 int current_node = numa_node_id();
9d5c8243 5209 unsigned int total_bytes = 0, total_packets = 0;
73cd78f1 5210 unsigned int i;
2d94d8ab
AD		 5211 u32 staterr;
5212 u16 length;
047e0030 5213 u16 vlan_tag;
9d5c8243
AK		 5214
5215 i = rx_ring->next_to_clean;
69d3ca53 5216 buffer_info = &rx_ring->buffer_info[i];
9d5c8243
AK		 5217 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5218 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5219
5220 while (staterr & E1000_RXD_STAT_DD) {
5221 if (*work_done >= budget)
5222 break;
5223 (*work_done)++;
9d5c8243 5224
69d3ca53
AD		 5225 skb = buffer_info->skb;
5226 prefetch(skb->data - NET_IP_ALIGN);
5227 buffer_info->skb = NULL;
5228
5229 i++;
5230 if (i == rx_ring->count)
5231 i = 0;
42d0781a 5232
69d3ca53
AD		 5233 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5234 prefetch(next_rxd);
5235 next_buffer = &rx_ring->buffer_info[i];
9d5c8243
AK		 5236
5237 length = le16_to_cpu(rx_desc->wb.upper.length);
5238 cleaned = true;
5239 cleaned_count++;
5240
2d94d8ab 5241 if (buffer_info->dma) {
bf36c1a0 5242 pci_unmap_single(pdev, buffer_info->dma,
4c844851 5243 rx_ring->rx_buffer_len,
bf36c1a0 5244 PCI_DMA_FROMDEVICE);
91615f76 5245 buffer_info->dma = 0;
4c844851 5246 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
6ec43fe6
AD		 5247 skb_put(skb, length);
5248 goto send_up;
5249 }
4c844851 5250 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
bf36c1a0
AD		 5251 }
5252
5253 if (length) {
9d5c8243 5254 pci_unmap_page(pdev, buffer_info->page_dma,
bf36c1a0 5255 PAGE_SIZE / 2, PCI_DMA_FROMDEVICE);
9d5c8243 5256 buffer_info->page_dma = 0;
bf36c1a0
AD		 5257
5258 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags++,
5259 buffer_info->page,
5260 buffer_info->page_offset,
5261 length);
5262
d1eff350
AD		 5263 if ((page_count(buffer_info->page) != 1) ||
5264 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 5265 buffer_info->page = NULL;
5266 else
5267 get_page(buffer_info->page);
9d5c8243
AK		 5268
5269 skb->len += length;
5270 skb->data_len += length;
bf36c1a0 5271 skb->truesize += length;
9d5c8243 5272 }
9d5c8243 5273
bf36c1a0 5274 if (!(staterr & E1000_RXD_STAT_EOP)) {
b2d56536
AD		 5275 buffer_info->skb = next_buffer->skb;
5276 buffer_info->dma = next_buffer->dma;
5277 next_buffer->skb = skb;
5278 next_buffer->dma = 0;
bf36c1a0
AD		 5279 goto next_desc;
5280 }
69d3ca53 5281send_up:
9d5c8243
AK		 5282 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5283 dev_kfree_skb_irq(skb);
5284 goto next_desc;
5285 }
9d5c8243 5286
c5b9bd5e 5287 igb_rx_hwtstamp(q_vector, staterr, skb);
9d5c8243
AK		 5288 total_bytes += skb->len;
5289 total_packets++;
5290
85ad76b2 5291 igb_rx_checksum_adv(rx_ring, staterr, skb);
9d5c8243
AK		 5292
5293 skb->protocol = eth_type_trans(skb, netdev);
047e0030
AD		 5294 skb_record_rx_queue(skb, rx_ring->queue_index);
5295
5296 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5297 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
9d5c8243 5298
047e0030 5299 igb_receive_skb(q_vector, skb, vlan_tag);
9d5c8243 5300
9d5c8243
AK		 5301next_desc:
5302 rx_desc->wb.upper.status_error = 0;
5303
5304 /* return some buffers to hardware, one at a time is too slow */
5305 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
3b644cf6 5306 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5307 cleaned_count = 0;
5308 }
5309
5310 /* use prefetched values */
5311 rx_desc = next_rxd;
5312 buffer_info = next_buffer;
9d5c8243
AK		 5313 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5314 }
bf36c1a0 5315
9d5c8243 5316 rx_ring->next_to_clean = i;
c493ea45 5317 cleaned_count = igb_desc_unused(rx_ring);
9d5c8243
AK		 5318
5319 if (cleaned_count)
3b644cf6 5320 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5321
5322 rx_ring->total_packets += total_packets;
5323 rx_ring->total_bytes += total_bytes;
5324 rx_ring->rx_stats.packets += total_packets;
5325 rx_ring->rx_stats.bytes += total_bytes;
9d5c8243
AK		 5326 return cleaned;
5327}
5328
9d5c8243
AK		 5329/**
5330 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5331 * @adapter: address of board private structure
5332 **/
d7ee5b3a 5333void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
9d5c8243 5334{
e694e964 5335 struct net_device *netdev = rx_ring->netdev;
9d5c8243
AK		 5336 union e1000_adv_rx_desc *rx_desc;
5337 struct igb_buffer *buffer_info;
5338 struct sk_buff *skb;
5339 unsigned int i;
db761762 5340 int bufsz;
9d5c8243
AK		 5341
5342 i = rx_ring->next_to_use;
5343 buffer_info = &rx_ring->buffer_info[i];
5344
4c844851 5345 bufsz = rx_ring->rx_buffer_len;
db761762 5346
9d5c8243
AK		 5347 while (cleaned_count--) {
5348 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5349
6ec43fe6 5350 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
9d5c8243 5351 if (!buffer_info->page) {
42d0781a 5352 buffer_info->page = netdev_alloc_page(netdev);
bf36c1a0 5353 if (!buffer_info->page) {
04a5fcaa 5354 rx_ring->rx_stats.alloc_failed++;
bf36c1a0
AD		 5355 goto no_buffers;
5356 }
5357 buffer_info->page_offset = 0;
5358 } else {
5359 buffer_info->page_offset ^= PAGE_SIZE / 2;
9d5c8243
AK		 5360 }
5361 buffer_info->page_dma =
80785298 5362 pci_map_page(rx_ring->pdev, buffer_info->page,
bf36c1a0
AD		 5363 buffer_info->page_offset,
5364 PAGE_SIZE / 2,
9d5c8243 5365 PCI_DMA_FROMDEVICE);
42d0781a
AD		 5366 if (pci_dma_mapping_error(rx_ring->pdev,
5367 buffer_info->page_dma)) {
5368 buffer_info->page_dma = 0;
5369 rx_ring->rx_stats.alloc_failed++;
5370 goto no_buffers;
5371 }
9d5c8243
AK		 5372 }
5373
42d0781a
AD		 5374 skb = buffer_info->skb;
5375 if (!skb) {
89d71a66 5376 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
9d5c8243 5377 if (!skb) {
04a5fcaa 5378 rx_ring->rx_stats.alloc_failed++;
9d5c8243
AK		 5379 goto no_buffers;
5380 }
5381
9d5c8243 5382 buffer_info->skb = skb;
42d0781a
AD		 5383 }
5384 if (!buffer_info->dma) {
80785298
AD		 5385 buffer_info->dma = pci_map_single(rx_ring->pdev,
5386 skb->data,
9d5c8243
AK		 5387 bufsz,
5388 PCI_DMA_FROMDEVICE);
42d0781a
AD		 5389 if (pci_dma_mapping_error(rx_ring->pdev,
5390 buffer_info->dma)) {
5391 buffer_info->dma = 0;
5392 rx_ring->rx_stats.alloc_failed++;
5393 goto no_buffers;
5394 }
9d5c8243
AK		 5395 }
5396 /* Refresh the desc even if buffer_addrs didn't change because
5397 * each write-back erases this info. */
6ec43fe6 5398 if (bufsz < IGB_RXBUFFER_1024) {
9d5c8243
AK		 5399 rx_desc->read.pkt_addr =
5400 cpu_to_le64(buffer_info->page_dma);
5401 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5402 } else {
42d0781a 5403 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
9d5c8243
AK		 5404 rx_desc->read.hdr_addr = 0;
5405 }
5406
5407 i++;
5408 if (i == rx_ring->count)
5409 i = 0;
5410 buffer_info = &rx_ring->buffer_info[i];
5411 }
5412
5413no_buffers:
5414 if (rx_ring->next_to_use != i) {
5415 rx_ring->next_to_use = i;
5416 if (i == 0)
5417 i = (rx_ring->count - 1);
5418 else
5419 i--;
5420
5421 /* Force memory writes to complete before letting h/w
5422 * know there are new descriptors to fetch. (Only
5423 * applicable for weak-ordered memory model archs,
5424 * such as IA-64). */
5425 wmb();
fce99e34 5426 writel(i, rx_ring->tail);
9d5c8243
AK		 5427 }
5428}
5429
5430/**
5431 * igb_mii_ioctl -
5432 * @netdev:
5433 * @ifreq:
5434 * @cmd:
5435 **/
5436static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5437{
5438 struct igb_adapter *adapter = netdev_priv(netdev);
5439 struct mii_ioctl_data *data = if_mii(ifr);
5440
5441 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5442 return -EOPNOTSUPP;
5443
5444 switch (cmd) {
5445 case SIOCGMIIPHY:
5446 data->phy_id = adapter->hw.phy.addr;
5447 break;
5448 case SIOCGMIIREG:
f5f4cf08
AD		 5449 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5450 &data->val_out))
9d5c8243
AK		 5451 return -EIO;
5452 break;
5453 case SIOCSMIIREG:
5454 default:
5455 return -EOPNOTSUPP;
5456 }
5457 return 0;
5458}
5459
c6cb090b
PO		 5460/**
5461 * igb_hwtstamp_ioctl - control hardware time stamping
5462 * @netdev:
5463 * @ifreq:
5464 * @cmd:
5465 *
33af6bcc
PO		 5466 * Outgoing time stamping can be enabled and disabled. Play nice and
5467 * disable it when requested, although it shouldn't case any overhead
5468 * when no packet needs it. At most one packet in the queue may be
5469 * marked for time stamping, otherwise it would be impossible to tell
5470 * for sure to which packet the hardware time stamp belongs.
5471 *
5472 * Incoming time stamping has to be configured via the hardware
5473 * filters. Not all combinations are supported, in particular event
5474 * type has to be specified. Matching the kind of event packet is
5475 * not supported, with the exception of "all V2 events regardless of
5476 * level 2 or 4".
5477 *
c6cb090b
PO		 5478 **/
5479static int igb_hwtstamp_ioctl(struct net_device *netdev,
5480 struct ifreq *ifr, int cmd)
5481{
33af6bcc
PO		 5482 struct igb_adapter *adapter = netdev_priv(netdev);
5483 struct e1000_hw *hw = &adapter->hw;
c6cb090b 5484 struct hwtstamp_config config;
c5b9bd5e
AD		 5485 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5486 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 5487 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 5488 bool is_l4 = false;
5489 bool is_l2 = false;
33af6bcc 5490 u32 regval;
c6cb090b
PO		 5491
5492 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5493 return -EFAULT;
5494
5495 /* reserved for future extensions */
5496 if (config.flags)
5497 return -EINVAL;
5498
33af6bcc
PO		 5499 switch (config.tx_type) {
5500 case HWTSTAMP_TX_OFF:
c5b9bd5e 5501 tsync_tx_ctl = 0;
33af6bcc 5502 case HWTSTAMP_TX_ON:
33af6bcc
PO		 5503 break;
5504 default:
5505 return -ERANGE;
5506 }
5507
5508 switch (config.rx_filter) {
5509 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 5510 tsync_rx_ctl = 0;
33af6bcc
PO		 5511 break;
5512 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5513 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5514 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5515 case HWTSTAMP_FILTER_ALL:
5516 /*
5517 * register TSYNCRXCFG must be set, therefore it is not
5518 * possible to time stamp both Sync and Delay_Req messages
5519 * => fall back to time stamping all packets
5520 */
c5b9bd5e 5521 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 5522 config.rx_filter = HWTSTAMP_FILTER_ALL;
5523 break;
5524 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 5525 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5526 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 5527 is_l4 = true;
33af6bcc
PO		 5528 break;
5529 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 5530 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5531 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 5532 is_l4 = true;
33af6bcc
PO		 5533 break;
5534 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5535 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 5536 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5537 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 5538 is_l2 = true;
5539 is_l4 = true;
33af6bcc
PO		 5540 config.rx_filter = HWTSTAMP_FILTER_SOME;
5541 break;
5542 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5543 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 5544 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5545 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 5546 is_l2 = true;
5547 is_l4 = true;
33af6bcc
PO		 5548 config.rx_filter = HWTSTAMP_FILTER_SOME;
5549 break;
5550 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5551 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5552 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 5553 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 5554 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 5555 is_l2 = true;
33af6bcc
PO		 5556 break;
5557 default:
5558 return -ERANGE;
5559 }
5560
c5b9bd5e
AD		 5561 if (hw->mac.type == e1000_82575) {
5562 if (tsync_rx_ctl | tsync_tx_ctl)
5563 return -EINVAL;
5564 return 0;
5565 }
5566
33af6bcc
PO		 5567 /* enable/disable TX */
5568 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 5569 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5570 regval |= tsync_tx_ctl;
33af6bcc
PO		 5571 wr32(E1000_TSYNCTXCTL, regval);
5572
c5b9bd5e 5573 /* enable/disable RX */
33af6bcc 5574 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 5575 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5576 regval |= tsync_rx_ctl;
33af6bcc 5577 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 5578
c5b9bd5e
AD		 5579 /* define which PTP packets are time stamped */
5580 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 5581
c5b9bd5e
AD		 5582 /* define ethertype filter for timestamped packets */
5583 if (is_l2)
5584 wr32(E1000_ETQF(3),
5585 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5586 E1000_ETQF_1588 | /* enable timestamping */
5587 ETH_P_1588)); /* 1588 eth protocol type */
5588 else
5589 wr32(E1000_ETQF(3), 0);
5590
5591#define PTP_PORT 319
5592 /* L4 Queue Filter[3]: filter by destination port and protocol */
5593 if (is_l4) {
5594 u32 ftqf = (IPPROTO_UDP /* UDP */
5595 | E1000_FTQF_VF_BP /* VF not compared */
5596 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5597 | E1000_FTQF_MASK); /* mask all inputs */
5598 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5599
5600 wr32(E1000_IMIR(3), htons(PTP_PORT));
5601 wr32(E1000_IMIREXT(3),
5602 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5603 if (hw->mac.type == e1000_82576) {
5604 /* enable source port check */
5605 wr32(E1000_SPQF(3), htons(PTP_PORT));
5606 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5607 }
5608 wr32(E1000_FTQF(3), ftqf);
5609 } else {
5610 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5611 }
33af6bcc
PO		 5612 wrfl();
5613
5614 adapter->hwtstamp_config = config;
5615
5616 /* clear TX/RX time stamp registers, just to be sure */
5617 regval = rd32(E1000_TXSTMPH);
5618 regval = rd32(E1000_RXSTMPH);
c6cb090b 5619
33af6bcc
PO		 5620 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5621 -EFAULT : 0;
c6cb090b
PO		 5622}
5623
9d5c8243
AK		 5624/**
5625 * igb_ioctl -
5626 * @netdev:
5627 * @ifreq:
5628 * @cmd:
5629 **/
5630static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5631{
5632 switch (cmd) {
5633 case SIOCGMIIPHY:
5634 case SIOCGMIIREG:
5635 case SIOCSMIIREG:
5636 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 5637 case SIOCSHWTSTAMP:
5638 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 5639 default:
5640 return -EOPNOTSUPP;
5641 }
5642}
5643
009bc06e
AD		 5644s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5645{
5646 struct igb_adapter *adapter = hw->back;
5647 u16 cap_offset;
5648
5649 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5650 if (!cap_offset)
5651 return -E1000_ERR_CONFIG;
5652
5653 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5654
5655 return 0;
5656}
5657
5658s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5659{
5660 struct igb_adapter *adapter = hw->back;
5661 u16 cap_offset;
5662
5663 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5664 if (!cap_offset)
5665 return -E1000_ERR_CONFIG;
5666
5667 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
5668
5669 return 0;
5670}
5671
9d5c8243
AK		 5672static void igb_vlan_rx_register(struct net_device *netdev,
5673 struct vlan_group *grp)
5674{
5675 struct igb_adapter *adapter = netdev_priv(netdev);
5676 struct e1000_hw *hw = &adapter->hw;
5677 u32 ctrl, rctl;
5678
5679 igb_irq_disable(adapter);
5680 adapter->vlgrp = grp;
5681
5682 if (grp) {
5683 /* enable VLAN tag insert/strip */
5684 ctrl = rd32(E1000_CTRL);
5685 ctrl |= E1000_CTRL_VME;
5686 wr32(E1000_CTRL, ctrl);
5687
51466239 5688 /* Disable CFI check */
9d5c8243 5689 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 5690 rctl &= ~E1000_RCTL_CFIEN;
5691 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 5692 } else {
5693 /* disable VLAN tag insert/strip */
5694 ctrl = rd32(E1000_CTRL);
5695 ctrl &= ~E1000_CTRL_VME;
5696 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 5697 }
5698
e1739522
AD		 5699 igb_rlpml_set(adapter);
5700
9d5c8243
AK		 5701 if (!test_bit(__IGB_DOWN, &adapter->state))
5702 igb_irq_enable(adapter);
5703}
5704
5705static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
5706{
5707 struct igb_adapter *adapter = netdev_priv(netdev);
5708 struct e1000_hw *hw = &adapter->hw;
4ae196df 5709 int pf_id = adapter->vfs_allocated_count;
9d5c8243 5710
51466239
AD		 5711 /* attempt to add filter to vlvf array */
5712 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 5713
51466239
AD		 5714 /* add the filter since PF can receive vlans w/o entry in vlvf */
5715 igb_vfta_set(hw, vid, true);
9d5c8243
AK		 5716}
5717
5718static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
5719{
5720 struct igb_adapter *adapter = netdev_priv(netdev);
5721 struct e1000_hw *hw = &adapter->hw;
4ae196df 5722 int pf_id = adapter->vfs_allocated_count;
51466239 5723 s32 err;
9d5c8243
AK		 5724
5725 igb_irq_disable(adapter);
5726 vlan_group_set_device(adapter->vlgrp, vid, NULL);
5727
5728 if (!test_bit(__IGB_DOWN, &adapter->state))
5729 igb_irq_enable(adapter);
5730
51466239
AD		 5731 /* remove vlan from VLVF table array */
5732 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 5733
51466239
AD		 5734 /* if vid was not present in VLVF just remove it from table */
5735 if (err)
4ae196df 5736 igb_vfta_set(hw, vid, false);
9d5c8243
AK		 5737}
5738
5739static void igb_restore_vlan(struct igb_adapter *adapter)
5740{
5741 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
5742
5743 if (adapter->vlgrp) {
5744 u16 vid;
5745 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
5746 if (!vlan_group_get_device(adapter->vlgrp, vid))
5747 continue;
5748 igb_vlan_rx_add_vid(adapter->netdev, vid);
5749 }
5750 }
5751}
5752
5753int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
5754{
090b1795 5755 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 5756 struct e1000_mac_info *mac = &adapter->hw.mac;
5757
5758 mac->autoneg = 0;
5759
9d5c8243
AK		 5760 switch (spddplx) {
5761 case SPEED_10 + DUPLEX_HALF:
5762 mac->forced_speed_duplex = ADVERTISE_10_HALF;
5763 break;
5764 case SPEED_10 + DUPLEX_FULL:
5765 mac->forced_speed_duplex = ADVERTISE_10_FULL;
5766 break;
5767 case SPEED_100 + DUPLEX_HALF:
5768 mac->forced_speed_duplex = ADVERTISE_100_HALF;
5769 break;
5770 case SPEED_100 + DUPLEX_FULL:
5771 mac->forced_speed_duplex = ADVERTISE_100_FULL;
5772 break;
5773 case SPEED_1000 + DUPLEX_FULL:
5774 mac->autoneg = 1;
5775 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
5776 break;
5777 case SPEED_1000 + DUPLEX_HALF: /* not supported */
5778 default:
090b1795 5779 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
9d5c8243
AK		 5780 return -EINVAL;
5781 }
5782 return 0;
5783}
5784
3fe7c4c9 5785static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 5786{
5787 struct net_device *netdev = pci_get_drvdata(pdev);
5788 struct igb_adapter *adapter = netdev_priv(netdev);
5789 struct e1000_hw *hw = &adapter->hw;
2d064c06 5790 u32 ctrl, rctl, status;
9d5c8243
AK		 5791 u32 wufc = adapter->wol;
5792#ifdef CONFIG_PM
5793 int retval = 0;
5794#endif
5795
5796 netif_device_detach(netdev);
5797
a88f10ec
AD		 5798 if (netif_running(netdev))
5799 igb_close(netdev);
5800
047e0030 5801 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 5802
5803#ifdef CONFIG_PM
5804 retval = pci_save_state(pdev);
5805 if (retval)
5806 return retval;
5807#endif
5808
5809 status = rd32(E1000_STATUS);
5810 if (status & E1000_STATUS_LU)
5811 wufc &= ~E1000_WUFC_LNKC;
5812
5813 if (wufc) {
5814 igb_setup_rctl(adapter);
ff41f8dc 5815 igb_set_rx_mode(netdev);
9d5c8243
AK		 5816
5817 /* turn on all-multi mode if wake on multicast is enabled */
5818 if (wufc & E1000_WUFC_MC) {
5819 rctl = rd32(E1000_RCTL);
5820 rctl |= E1000_RCTL_MPE;
5821 wr32(E1000_RCTL, rctl);
5822 }
5823
5824 ctrl = rd32(E1000_CTRL);
5825 /* advertise wake from D3Cold */
5826 #define E1000_CTRL_ADVD3WUC 0x00100000
5827 /* phy power management enable */
5828 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5829 ctrl |= E1000_CTRL_ADVD3WUC;
5830 wr32(E1000_CTRL, ctrl);
5831
9d5c8243 5832 /* Allow time for pending master requests to run */
330a6d6a 5833 igb_disable_pcie_master(hw);
9d5c8243
AK		 5834
5835 wr32(E1000_WUC, E1000_WUC_PME_EN);
5836 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 5837 } else {
5838 wr32(E1000_WUC, 0);
5839 wr32(E1000_WUFC, 0);
9d5c8243
AK		 5840 }
5841
3fe7c4c9
RW		 5842 *enable_wake = wufc || adapter->en_mng_pt;
5843 if (!*enable_wake)
88a268c1
NN		 5844 igb_power_down_link(adapter);
5845 else
5846 igb_power_up_link(adapter);
9d5c8243
AK		 5847
5848 /* Release control of h/w to f/w. If f/w is AMT enabled, this
5849 * would have already happened in close and is redundant. */
5850 igb_release_hw_control(adapter);
5851
5852 pci_disable_device(pdev);
5853
9d5c8243
AK		 5854 return 0;
5855}
5856
5857#ifdef CONFIG_PM
3fe7c4c9
RW		 5858static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
5859{
5860 int retval;
5861 bool wake;
5862
5863 retval = __igb_shutdown(pdev, &wake);
5864 if (retval)
5865 return retval;
5866
5867 if (wake) {
5868 pci_prepare_to_sleep(pdev);
5869 } else {
5870 pci_wake_from_d3(pdev, false);
5871 pci_set_power_state(pdev, PCI_D3hot);
5872 }
5873
5874 return 0;
5875}
5876
9d5c8243
AK		 5877static int igb_resume(struct pci_dev *pdev)
5878{
5879 struct net_device *netdev = pci_get_drvdata(pdev);
5880 struct igb_adapter *adapter = netdev_priv(netdev);
5881 struct e1000_hw *hw = &adapter->hw;
5882 u32 err;
5883
5884 pci_set_power_state(pdev, PCI_D0);
5885 pci_restore_state(pdev);
b94f2d77 5886 pci_save_state(pdev);
42bfd33a 5887
aed5dec3 5888 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 5889 if (err) {
5890 dev_err(&pdev->dev,
5891 "igb: Cannot enable PCI device from suspend\n");
5892 return err;
5893 }
5894 pci_set_master(pdev);
5895
5896 pci_enable_wake(pdev, PCI_D3hot, 0);
5897 pci_enable_wake(pdev, PCI_D3cold, 0);
5898
047e0030 5899 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 5900 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
5901 return -ENOMEM;
9d5c8243
AK		 5902 }
5903
9d5c8243 5904 igb_reset(adapter);
a8564f03
AD		 5905
5906 /* let the f/w know that the h/w is now under the control of the
5907 * driver. */
5908 igb_get_hw_control(adapter);
5909
9d5c8243
AK		 5910 wr32(E1000_WUS, ~0);
5911
a88f10ec
AD		 5912 if (netif_running(netdev)) {
5913 err = igb_open(netdev);
5914 if (err)
5915 return err;
5916 }
9d5c8243
AK		 5917
5918 netif_device_attach(netdev);
5919
9d5c8243
AK		 5920 return 0;
5921}
5922#endif
5923
5924static void igb_shutdown(struct pci_dev *pdev)
5925{
3fe7c4c9
RW		 5926 bool wake;
5927
5928 __igb_shutdown(pdev, &wake);
5929
5930 if (system_state == SYSTEM_POWER_OFF) {
5931 pci_wake_from_d3(pdev, wake);
5932 pci_set_power_state(pdev, PCI_D3hot);
5933 }
9d5c8243
AK		 5934}
5935
5936#ifdef CONFIG_NET_POLL_CONTROLLER
5937/*
5938 * Polling 'interrupt' - used by things like netconsole to send skbs
5939 * without having to re-enable interrupts. It's not called while
5940 * the interrupt routine is executing.
5941 */
5942static void igb_netpoll(struct net_device *netdev)
5943{
5944 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 5945 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5946 int i;
9d5c8243 5947
eebbbdba 5948 if (!adapter->msix_entries) {
047e0030 5949 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 5950 igb_irq_disable(adapter);
047e0030 5951 napi_schedule(&q_vector->napi);
eebbbdba
AD		 5952 return;
5953 }
9d5c8243 5954
047e0030
AD		 5955 for (i = 0; i < adapter->num_q_vectors; i++) {
5956 struct igb_q_vector *q_vector = adapter->q_vector[i];
5957 wr32(E1000_EIMC, q_vector->eims_value);
5958 napi_schedule(&q_vector->napi);
eebbbdba 5959 }
9d5c8243
AK		 5960}
5961#endif /* CONFIG_NET_POLL_CONTROLLER */
5962
5963/**
5964 * igb_io_error_detected - called when PCI error is detected
5965 * @pdev: Pointer to PCI device
5966 * @state: The current pci connection state
5967 *
5968 * This function is called after a PCI bus error affecting
5969 * this device has been detected.
5970 */
5971static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
5972 pci_channel_state_t state)
5973{
5974 struct net_device *netdev = pci_get_drvdata(pdev);
5975 struct igb_adapter *adapter = netdev_priv(netdev);
5976
5977 netif_device_detach(netdev);
5978
59ed6eec
AD		 5979 if (state == pci_channel_io_perm_failure)
5980 return PCI_ERS_RESULT_DISCONNECT;
5981
9d5c8243
AK		 5982 if (netif_running(netdev))
5983 igb_down(adapter);
5984 pci_disable_device(pdev);
5985
5986 /* Request a slot slot reset. */
5987 return PCI_ERS_RESULT_NEED_RESET;
5988}
5989
5990/**
5991 * igb_io_slot_reset - called after the pci bus has been reset.
5992 * @pdev: Pointer to PCI device
5993 *
5994 * Restart the card from scratch, as if from a cold-boot. Implementation
5995 * resembles the first-half of the igb_resume routine.
5996 */
5997static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
5998{
5999 struct net_device *netdev = pci_get_drvdata(pdev);
6000 struct igb_adapter *adapter = netdev_priv(netdev);
6001 struct e1000_hw *hw = &adapter->hw;
40a914fa 6002 pci_ers_result_t result;
42bfd33a 6003 int err;
9d5c8243 6004
aed5dec3 6005 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6006 dev_err(&pdev->dev,
6007 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6008 result = PCI_ERS_RESULT_DISCONNECT;
6009 } else {
6010 pci_set_master(pdev);
6011 pci_restore_state(pdev);
b94f2d77 6012 pci_save_state(pdev);
9d5c8243 6013
40a914fa
AD		 6014 pci_enable_wake(pdev, PCI_D3hot, 0);
6015 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6016
40a914fa
AD		 6017 igb_reset(adapter);
6018 wr32(E1000_WUS, ~0);
6019 result = PCI_ERS_RESULT_RECOVERED;
6020 }
9d5c8243 6021
ea943d41
JK		 6022 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6023 if (err) {
6024 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6025 "failed 0x%0x\n", err);
6026 /* non-fatal, continue */
6027 }
40a914fa
AD		 6028
6029 return result;
9d5c8243
AK		 6030}
6031
6032/**
6033 * igb_io_resume - called when traffic can start flowing again.
6034 * @pdev: Pointer to PCI device
6035 *
6036 * This callback is called when the error recovery driver tells us that
6037 * its OK to resume normal operation. Implementation resembles the
6038 * second-half of the igb_resume routine.
6039 */
6040static void igb_io_resume(struct pci_dev *pdev)
6041{
6042 struct net_device *netdev = pci_get_drvdata(pdev);
6043 struct igb_adapter *adapter = netdev_priv(netdev);
6044
9d5c8243
AK		 6045 if (netif_running(netdev)) {
6046 if (igb_up(adapter)) {
6047 dev_err(&pdev->dev, "igb_up failed after reset\n");
6048 return;
6049 }
6050 }
6051
6052 netif_device_attach(netdev);
6053
6054 /* let the f/w know that the h/w is now under the control of the
6055 * driver. */
6056 igb_get_hw_control(adapter);
9d5c8243
AK		 6057}
6058
26ad9178
AD		 6059static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6060 u8 qsel)
6061{
6062 u32 rar_low, rar_high;
6063 struct e1000_hw *hw = &adapter->hw;
6064
6065 /* HW expects these in little endian so we reverse the byte order
6066 * from network order (big endian) to little endian
6067 */
6068 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6069 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6070 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6071
6072 /* Indicate to hardware the Address is Valid. */
6073 rar_high |= E1000_RAH_AV;
6074
6075 if (hw->mac.type == e1000_82575)
6076 rar_high |= E1000_RAH_POOL_1 * qsel;
6077 else
6078 rar_high |= E1000_RAH_POOL_1 << qsel;
6079
6080 wr32(E1000_RAL(index), rar_low);
6081 wrfl();
6082 wr32(E1000_RAH(index), rar_high);
6083 wrfl();
6084}
6085
4ae196df
AD		 6086static int igb_set_vf_mac(struct igb_adapter *adapter,
6087 int vf, unsigned char *mac_addr)
6088{
6089 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 6090 /* VF MAC addresses start at end of receive addresses and moves
6091 * torwards the first, as a result a collision should not be possible */
6092 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 6093
37680117 6094 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 6095
26ad9178 6096 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 6097
6098 return 0;
6099}
6100
8151d294
WM		 6101static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6102{
6103 struct igb_adapter *adapter = netdev_priv(netdev);
6104 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6105 return -EINVAL;
6106 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6107 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6108 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6109 " change effective.");
6110 if (test_bit(__IGB_DOWN, &adapter->state)) {
6111 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6112 " but the PF device is not up.\n");
6113 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6114 " attempting to use the VF device.\n");
6115 }
6116 return igb_set_vf_mac(adapter, vf, mac);
6117}
6118
6119static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6120{
6121 return -EOPNOTSUPP;
6122}
6123
6124static int igb_ndo_get_vf_config(struct net_device *netdev,
6125 int vf, struct ifla_vf_info *ivi)
6126{
6127 struct igb_adapter *adapter = netdev_priv(netdev);
6128 if (vf >= adapter->vfs_allocated_count)
6129 return -EINVAL;
6130 ivi->vf = vf;
6131 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6132 ivi->tx_rate = 0;
6133 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6134 ivi->qos = adapter->vf_data[vf].pf_qos;
6135 return 0;
6136}
6137
4ae196df
AD		 6138static void igb_vmm_control(struct igb_adapter *adapter)
6139{
6140 struct e1000_hw *hw = &adapter->hw;
10d8e907 6141 u32 reg;
4ae196df 6142
d4960307
AD		 6143 /* replication is not supported for 82575 */
6144 if (hw->mac.type == e1000_82575)
4ae196df
AD		 6145 return;
6146
10d8e907
AD		 6147 /* enable replication vlan tag stripping */
6148 reg = rd32(E1000_RPLOLR);
6149 reg |= E1000_RPLOLR_STRVLAN;
6150 wr32(E1000_RPLOLR, reg);
6151
6152 /* notify HW that the MAC is adding vlan tags */
6153 reg = rd32(E1000_DTXCTL);
6154 reg |= E1000_DTXCTL_VLAN_ADDED;
6155 wr32(E1000_DTXCTL, reg);
6156
d4960307
AD		 6157 if (adapter->vfs_allocated_count) {
6158 igb_vmdq_set_loopback_pf(hw, true);
6159 igb_vmdq_set_replication_pf(hw, true);
6160 } else {
6161 igb_vmdq_set_loopback_pf(hw, false);
6162 igb_vmdq_set_replication_pf(hw, false);
6163 }
4ae196df
AD		 6164}
6165
9d5c8243 6166/* igb_main.c */
ubuntu-zesty-kernel mirror