]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
projects / mirror_ubuntu-zesty-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ixgbevf: Check for adapter removal on register writes
[mirror_ubuntu-zesty-kernel.git] / drivers / net / ethernet / intel / ixgbevf / ixgbevf_main.c
CommitLineData
92915f71
GR		 1/*******************************************************************************
2
3 Intel 82599 Virtual Function driver
2e7cfbdd 4 Copyright(c) 1999 - 2014 Intel Corporation.
92915f71
GR		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28
29/******************************************************************************
30 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code
31******************************************************************************/
dbd9636e
JK		 32
33#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
34
92915f71 35#include <linux/types.h>
dadcd65f 36#include <linux/bitops.h>
92915f71
GR		 37#include <linux/module.h>
38#include <linux/pci.h>
39#include <linux/netdevice.h>
40#include <linux/vmalloc.h>
41#include <linux/string.h>
42#include <linux/in.h>
43#include <linux/ip.h>
44#include <linux/tcp.h>
70a10e25 45#include <linux/sctp.h>
92915f71 46#include <linux/ipv6.h>
5a0e3ad6 47#include <linux/slab.h>
92915f71
GR		 48#include <net/checksum.h>
49#include <net/ip6_checksum.h>
50#include <linux/ethtool.h>
01789349 51#include <linux/if.h>
92915f71 52#include <linux/if_vlan.h>
70c71606 53#include <linux/prefetch.h>
92915f71
GR		 54
55#include "ixgbevf.h"
56
3d8fe98f 57const char ixgbevf_driver_name[] = "ixgbevf";
92915f71 58static const char ixgbevf_driver_string[] =
422e05d1 59 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver";
92915f71 60
86f359f6 61#define DRV_VERSION "2.12.1-k"
92915f71 62const char ixgbevf_driver_version[] = DRV_VERSION;
66c87bd5 63static char ixgbevf_copyright[] =
5c47a2b6 64 "Copyright (c) 2009 - 2012 Intel Corporation.";
92915f71
GR		 65
66static const struct ixgbevf_info *ixgbevf_info_tbl[] = {
2316aa2a
GR		 67 [board_82599_vf] = &ixgbevf_82599_vf_info,
68 [board_X540_vf] = &ixgbevf_X540_vf_info,
92915f71
GR		 69};
70
71/* ixgbevf_pci_tbl - PCI Device ID Table
72 *
73 * Wildcard entries (PCI_ANY_ID) should come last
74 * Last entry must be all 0s
75 *
76 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
77 * Class, Class Mask, private data (not used) }
78 */
39ba22b4
SH		 79static DEFINE_PCI_DEVICE_TABLE(ixgbevf_pci_tbl) = {
80 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf },
81 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf },
92915f71
GR		 82 /* required last entry */
83 {0, }
84};
85MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl);
86
87MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
88MODULE_DESCRIPTION("Intel(R) 82599 Virtual Function Driver");
89MODULE_LICENSE("GPL");
90MODULE_VERSION(DRV_VERSION);
91
b3f4d599 92#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
93static int debug = -1;
94module_param(debug, int, 0);
95MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
92915f71
GR		 96
97/* forward decls */
220fe050 98static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter);
fa71ae27 99static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector);
56e94095 100static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter);
92915f71 101
dbf8b0d8
MR		 102static void ixgbevf_remove_adapter(struct ixgbe_hw *hw)
103{
104 struct ixgbevf_adapter *adapter = hw->back;
105
106 if (!hw->hw_addr)
107 return;
108 hw->hw_addr = NULL;
109 dev_err(&adapter->pdev->dev, "Adapter removed\n");
110}
111
112static void ixgbevf_check_remove(struct ixgbe_hw *hw, u32 reg)
113{
114 u32 value;
115
116 /* The following check not only optimizes a bit by not
117 * performing a read on the status register when the
118 * register just read was a status register read that
119 * returned IXGBE_FAILED_READ_REG. It also blocks any
120 * potential recursion.
121 */
122 if (reg == IXGBE_VFSTATUS) {
123 ixgbevf_remove_adapter(hw);
124 return;
125 }
126 value = ixgbe_read_reg(hw, IXGBE_VFSTATUS);
127 if (value == IXGBE_FAILED_READ_REG)
128 ixgbevf_remove_adapter(hw);
129}
130
131u32 ixgbe_read_reg(struct ixgbe_hw *hw, u32 reg)
132{
133 u8 __iomem *reg_addr = ACCESS_ONCE(hw->hw_addr);
134 u32 value;
135
136 if (IXGBE_REMOVED(reg_addr))
137 return IXGBE_FAILED_READ_REG;
138 value = readl(reg_addr + reg);
139 if (unlikely(value == IXGBE_FAILED_READ_REG))
140 ixgbevf_check_remove(hw, reg);
141 return value;
142}
143
5cdab2f6 144static inline void ixgbevf_release_rx_desc(struct ixgbevf_ring *rx_ring,
92915f71
GR		 145 u32 val)
146{
5cdab2f6
DS		 147 rx_ring->next_to_use = val;
148
92915f71
GR		 149 /*
150 * Force memory writes to complete before letting h/w
151 * know there are new descriptors to fetch. (Only
152 * applicable for weak-ordered memory model archs,
153 * such as IA-64).
154 */
155 wmb();
06380db6 156 ixgbevf_write_tail(rx_ring, val);
92915f71
GR		 157}
158
49ce9c2c 159/**
65d676c8 160 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors
92915f71
GR		 161 * @adapter: pointer to adapter struct
162 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
163 * @queue: queue to map the corresponding interrupt to
164 * @msix_vector: the vector to map to the corresponding queue
92915f71
GR		 165 */
166static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
167 u8 queue, u8 msix_vector)
168{
169 u32 ivar, index;
170 struct ixgbe_hw *hw = &adapter->hw;
171 if (direction == -1) {
172 /* other causes */
173 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
174 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
175 ivar &= ~0xFF;
176 ivar |= msix_vector;
177 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
178 } else {
179 /* tx or rx causes */
180 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
181 index = ((16 * (queue & 1)) + (8 * direction));
182 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
183 ivar &= ~(0xFF << index);
184 ivar |= (msix_vector << index);
185 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar);
186 }
187}
188
70a10e25 189static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring,
9bdfefd2
ET		 190 struct ixgbevf_tx_buffer *tx_buffer)
191{
192 if (tx_buffer->skb) {
193 dev_kfree_skb_any(tx_buffer->skb);
194 if (dma_unmap_len(tx_buffer, len))
70a10e25 195 dma_unmap_single(tx_ring->dev,
9bdfefd2
ET		 196 dma_unmap_addr(tx_buffer, dma),
197 dma_unmap_len(tx_buffer, len),
2a1f8794 198 DMA_TO_DEVICE);
9bdfefd2
ET		 199 } else if (dma_unmap_len(tx_buffer, len)) {
200 dma_unmap_page(tx_ring->dev,
201 dma_unmap_addr(tx_buffer, dma),
202 dma_unmap_len(tx_buffer, len),
203 DMA_TO_DEVICE);
92915f71 204 }
9bdfefd2
ET		 205 tx_buffer->next_to_watch = NULL;
206 tx_buffer->skb = NULL;
207 dma_unmap_len_set(tx_buffer, len, 0);
208 /* tx_buffer must be completely set up in the transmit path */
92915f71
GR		 209}
210
92915f71
GR		 211#define IXGBE_MAX_TXD_PWR 14
212#define IXGBE_MAX_DATA_PER_TXD (1 << IXGBE_MAX_TXD_PWR)
213
214/* Tx Descriptors needed, worst case */
3595990a
AD		 215#define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD)
216#define DESC_NEEDED (MAX_SKB_FRAGS + 4)
92915f71
GR		 217
218static void ixgbevf_tx_timeout(struct net_device *netdev);
219
220/**
221 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes
fa71ae27 222 * @q_vector: board private structure
92915f71
GR		 223 * @tx_ring: tx ring to clean
224 **/
fa71ae27 225static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
92915f71
GR		 226 struct ixgbevf_ring *tx_ring)
227{
fa71ae27 228 struct ixgbevf_adapter *adapter = q_vector->adapter;
7ad1a093
ET		 229 struct ixgbevf_tx_buffer *tx_buffer;
230 union ixgbe_adv_tx_desc *tx_desc;
92915f71 231 unsigned int total_bytes = 0, total_packets = 0;
7ad1a093
ET		 232 unsigned int budget = tx_ring->count / 2;
233 unsigned int i = tx_ring->next_to_clean;
92915f71 234
10cc1bdd
AD		 235 if (test_bit(__IXGBEVF_DOWN, &adapter->state))
236 return true;
237
7ad1a093
ET		 238 tx_buffer = &tx_ring->tx_buffer_info[i];
239 tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
240 i -= tx_ring->count;
92915f71 241
e757e3e1 242 do {
7ad1a093 243 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
e757e3e1
AD		 244
245 /* if next_to_watch is not set then there is no work pending */
246 if (!eop_desc)
247 break;
248
249 /* prevent any other reads prior to eop_desc */
250 read_barrier_depends();
251
252 /* if DD is not set pending work has not been completed */
253 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
254 break;
255
256 /* clear next_to_watch to prevent false hangs */
7ad1a093 257 tx_buffer->next_to_watch = NULL;
e757e3e1 258
7ad1a093
ET		 259 /* update the statistics for this packet */
260 total_bytes += tx_buffer->bytecount;
261 total_packets += tx_buffer->gso_segs;
92915f71 262
9bdfefd2
ET		 263 /* free the skb */
264 dev_kfree_skb_any(tx_buffer->skb);
265
266 /* unmap skb header data */
267 dma_unmap_single(tx_ring->dev,
268 dma_unmap_addr(tx_buffer, dma),
269 dma_unmap_len(tx_buffer, len),
270 DMA_TO_DEVICE);
271
7ad1a093 272 /* clear tx_buffer data */
9bdfefd2
ET		 273 tx_buffer->skb = NULL;
274 dma_unmap_len_set(tx_buffer, len, 0);
92915f71 275
7ad1a093
ET		 276 /* unmap remaining buffers */
277 while (tx_desc != eop_desc) {
7ad1a093
ET		 278 tx_buffer++;
279 tx_desc++;
92915f71 280 i++;
7ad1a093
ET		 281 if (unlikely(!i)) {
282 i -= tx_ring->count;
283 tx_buffer = tx_ring->tx_buffer_info;
284 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
285 }
e757e3e1 286
9bdfefd2
ET		 287 /* unmap any remaining paged data */
288 if (dma_unmap_len(tx_buffer, len)) {
289 dma_unmap_page(tx_ring->dev,
290 dma_unmap_addr(tx_buffer, dma),
291 dma_unmap_len(tx_buffer, len),
292 DMA_TO_DEVICE);
293 dma_unmap_len_set(tx_buffer, len, 0);
294 }
92915f71
GR		 295 }
296
7ad1a093
ET		 297 /* move us one more past the eop_desc for start of next pkt */
298 tx_buffer++;
299 tx_desc++;
300 i++;
301 if (unlikely(!i)) {
302 i -= tx_ring->count;
303 tx_buffer = tx_ring->tx_buffer_info;
304 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
305 }
306
307 /* issue prefetch for next Tx descriptor */
308 prefetch(tx_desc);
309
310 /* update budget accounting */
311 budget--;
312 } while (likely(budget));
313
314 i += tx_ring->count;
92915f71 315 tx_ring->next_to_clean = i;
7ad1a093
ET		 316 u64_stats_update_begin(&tx_ring->syncp);
317 tx_ring->stats.bytes += total_bytes;
318 tx_ring->stats.packets += total_packets;
319 u64_stats_update_end(&tx_ring->syncp);
320 q_vector->tx.total_bytes += total_bytes;
321 q_vector->tx.total_packets += total_packets;
92915f71
GR		 322
323#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
7ad1a093 324 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
f880d07b 325 (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
92915f71
GR		 326 /* Make sure that anybody stopping the queue after this
327 * sees the new next_to_clean.
328 */
329 smp_mb();
7ad1a093 330
fb40195c
AD		 331 if (__netif_subqueue_stopped(tx_ring->netdev,
332 tx_ring->queue_index) &&
92915f71 333 !test_bit(__IXGBEVF_DOWN, &adapter->state)) {
fb40195c
AD		 334 netif_wake_subqueue(tx_ring->netdev,
335 tx_ring->queue_index);
7ad1a093 336 ++tx_ring->tx_stats.restart_queue;
92915f71 337 }
92915f71
GR		 338 }
339
7ad1a093 340 return !!budget;
92915f71
GR		 341}
342
343/**
344 * ixgbevf_receive_skb - Send a completed packet up the stack
345 * @q_vector: structure containing interrupt and ring information
346 * @skb: packet to send up
347 * @status: hardware indication of status of receive
92915f71
GR		 348 * @rx_desc: rx descriptor
349 **/
350static void ixgbevf_receive_skb(struct ixgbevf_q_vector *q_vector,
351 struct sk_buff *skb, u8 status,
92915f71
GR		 352 union ixgbe_adv_rx_desc *rx_desc)
353{
354 struct ixgbevf_adapter *adapter = q_vector->adapter;
355 bool is_vlan = (status & IXGBE_RXD_STAT_VP);
dd1ed3b7 356 u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
92915f71 357
5d9a533b 358 if (is_vlan && test_bit(tag & VLAN_VID_MASK, adapter->active_vlans))
86a9bad3 359 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tag);
dadcd65f 360
366c1099
GR		 361 if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
362 napi_gro_receive(&q_vector->napi, skb);
363 else
364 netif_rx(skb);
92915f71
GR		 365}
366
08681618
JK		 367/**
368 * ixgbevf_rx_skb - Helper function to determine proper Rx method
369 * @q_vector: structure containing interrupt and ring information
370 * @skb: packet to send up
371 * @status: hardware indication of status of receive
372 * @rx_desc: rx descriptor
373 **/
374static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector,
375 struct sk_buff *skb, u8 status,
376 union ixgbe_adv_rx_desc *rx_desc)
377{
c777cdfa
JK		 378#ifdef CONFIG_NET_RX_BUSY_POLL
379 skb_mark_napi_id(skb, &q_vector->napi);
380
381 if (ixgbevf_qv_busy_polling(q_vector)) {
382 netif_receive_skb(skb);
383 /* exit early if we busy polled */
384 return;
385 }
386#endif /* CONFIG_NET_RX_BUSY_POLL */
387
08681618
JK		 388 ixgbevf_receive_skb(q_vector, skb, status, rx_desc);
389}
390
92915f71
GR		 391/**
392 * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
55fb277c 393 * @ring: pointer to Rx descriptor ring structure
92915f71
GR		 394 * @status_err: hardware indication of status of receive
395 * @skb: skb currently being received and modified
396 **/
55fb277c 397static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring,
92915f71
GR		 398 u32 status_err, struct sk_buff *skb)
399{
bc8acf2c 400 skb_checksum_none_assert(skb);
92915f71
GR		 401
402 /* Rx csum disabled */
fb40195c 403 if (!(ring->netdev->features & NETIF_F_RXCSUM))
92915f71
GR		 404 return;
405
406 /* if IP and error */
407 if ((status_err & IXGBE_RXD_STAT_IPCS) &&
408 (status_err & IXGBE_RXDADV_ERR_IPE)) {
095e2617 409 ring->rx_stats.csum_err++;
92915f71
GR		 410 return;
411 }
412
413 if (!(status_err & IXGBE_RXD_STAT_L4CS))
414 return;
415
416 if (status_err & IXGBE_RXDADV_ERR_TCPE) {
095e2617 417 ring->rx_stats.csum_err++;
92915f71
GR		 418 return;
419 }
420
421 /* It must be a TCP or UDP packet with a valid checksum */
422 skb->ip_summed = CHECKSUM_UNNECESSARY;
92915f71
GR		 423}
424
425/**
426 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
095e2617 427 * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on
92915f71 428 **/
095e2617 429static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring,
92915f71
GR		 430 int cleaned_count)
431{
92915f71
GR		 432 union ixgbe_adv_rx_desc *rx_desc;
433 struct ixgbevf_rx_buffer *bi;
fb40195c 434 unsigned int i = rx_ring->next_to_use;
92915f71 435
92915f71 436 while (cleaned_count--) {
908421f6 437 rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
05d063aa 438 bi = &rx_ring->rx_buffer_info[i];
b9dd245b
GR		 439
440 if (!bi->skb) {
441 struct sk_buff *skb;
442
fb40195c
AD		 443 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
444 rx_ring->rx_buf_len);
05d063aa 445 if (!skb)
92915f71 446 goto no_buffers;
05d063aa 447
92915f71 448 bi->skb = skb;
b9dd245b 449
05d063aa 450 bi->dma = dma_map_single(rx_ring->dev, skb->data,
92915f71 451 rx_ring->rx_buf_len,
2a1f8794 452 DMA_FROM_DEVICE);
05d063aa 453 if (dma_mapping_error(rx_ring->dev, bi->dma)) {
6132ee8a
GR		 454 dev_kfree_skb(skb);
455 bi->skb = NULL;
05d063aa 456 dev_err(rx_ring->dev, "Rx DMA map failed\n");
6132ee8a
GR		 457 break;
458 }
92915f71 459 }
77d5dfca 460 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
92915f71
GR		 461
462 i++;
463 if (i == rx_ring->count)
464 i = 0;
92915f71
GR		 465 }
466
467no_buffers:
095e2617 468 rx_ring->rx_stats.alloc_rx_buff_failed++;
5cdab2f6
DS		 469 if (rx_ring->next_to_use != i)
470 ixgbevf_release_rx_desc(rx_ring, i);
92915f71
GR		 471}
472
473static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
5f3600eb 474 u32 qmask)
92915f71 475{
92915f71
GR		 476 struct ixgbe_hw *hw = &adapter->hw;
477
5f3600eb 478 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask);
92915f71
GR		 479}
480
08e50a20
JK		 481static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
482 struct ixgbevf_ring *rx_ring,
483 int budget)
92915f71 484{
92915f71
GR		 485 union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
486 struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer;
487 struct sk_buff *skb;
488 unsigned int i;
489 u32 len, staterr;
92915f71
GR		 490 int cleaned_count = 0;
491 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
492
493 i = rx_ring->next_to_clean;
908421f6 494 rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
92915f71
GR		 495 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
496 rx_buffer_info = &rx_ring->rx_buffer_info[i];
497
498 while (staterr & IXGBE_RXD_STAT_DD) {
fa71ae27 499 if (!budget)
92915f71 500 break;
fa71ae27 501 budget--;
92915f71 502
2d0bb1c1 503 rmb(); /* read descriptor and rx_buffer_info after status DD */
77d5dfca 504 len = le16_to_cpu(rx_desc->wb.upper.length);
92915f71
GR		 505 skb = rx_buffer_info->skb;
506 prefetch(skb->data - NET_IP_ALIGN);
507 rx_buffer_info->skb = NULL;
508
509 if (rx_buffer_info->dma) {
05d063aa 510 dma_unmap_single(rx_ring->dev, rx_buffer_info->dma,
92915f71 511 rx_ring->rx_buf_len,
2a1f8794 512 DMA_FROM_DEVICE);
92915f71
GR		 513 rx_buffer_info->dma = 0;
514 skb_put(skb, len);
515 }
516
92915f71
GR		 517 i++;
518 if (i == rx_ring->count)
519 i = 0;
520
908421f6 521 next_rxd = IXGBEVF_RX_DESC(rx_ring, i);
92915f71
GR		 522 prefetch(next_rxd);
523 cleaned_count++;
524
525 next_buffer = &rx_ring->rx_buffer_info[i];
526
527 if (!(staterr & IXGBE_RXD_STAT_EOP)) {
77d5dfca 528 skb->next = next_buffer->skb;
5c60f81a 529 IXGBE_CB(skb->next)->prev = skb;
095e2617 530 rx_ring->rx_stats.non_eop_descs++;
92915f71
GR		 531 goto next_desc;
532 }
533
5c60f81a
AD		 534 /* we should not be chaining buffers, if we did drop the skb */
535 if (IXGBE_CB(skb)->prev) {
536 do {
537 struct sk_buff *this = skb;
538 skb = IXGBE_CB(skb)->prev;
539 dev_kfree_skb(this);
540 } while (skb);
541 goto next_desc;
542 }
543
92915f71
GR		 544 /* ERR_MASK will only have valid bits if EOP set */
545 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) {
546 dev_kfree_skb_irq(skb);
547 goto next_desc;
548 }
549
55fb277c 550 ixgbevf_rx_checksum(rx_ring, staterr, skb);
92915f71
GR		 551
552 /* probably a little skewed due to removing CRC */
553 total_rx_bytes += skb->len;
554 total_rx_packets++;
555
fb40195c 556 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
92915f71 557
815cccbf
JF		 558 /* Workaround hardware that can't do proper VEPA multicast
559 * source pruning.
560 */
bd9d5592
FF		 561 if ((skb->pkt_type == PACKET_BROADCAST ||
562 skb->pkt_type == PACKET_MULTICAST) &&
095e2617 563 ether_addr_equal(rx_ring->netdev->dev_addr,
7367d0b5 564 eth_hdr(skb)->h_source)) {
815cccbf
JF		 565 dev_kfree_skb_irq(skb);
566 goto next_desc;
567 }
568
08681618 569 ixgbevf_rx_skb(q_vector, skb, staterr, rx_desc);
92915f71
GR		 570
571next_desc:
572 rx_desc->wb.upper.status_error = 0;
573
574 /* return some buffers to hardware, one at a time is too slow */
575 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
095e2617 576 ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
92915f71
GR		 577 cleaned_count = 0;
578 }
579
580 /* use prefetched values */
581 rx_desc = next_rxd;
582 rx_buffer_info = &rx_ring->rx_buffer_info[i];
583
584 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
585 }
586
587 rx_ring->next_to_clean = i;
f880d07b 588 cleaned_count = ixgbevf_desc_unused(rx_ring);
92915f71
GR		 589
590 if (cleaned_count)
095e2617 591 ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
92915f71 592
4197aa7b 593 u64_stats_update_begin(&rx_ring->syncp);
095e2617
ET		 594 rx_ring->stats.packets += total_rx_packets;
595 rx_ring->stats.bytes += total_rx_bytes;
4197aa7b 596 u64_stats_update_end(&rx_ring->syncp);
ac6ed8f0
GR		 597 q_vector->rx.total_packets += total_rx_packets;
598 q_vector->rx.total_bytes += total_rx_bytes;
92915f71 599
08e50a20 600 return total_rx_packets;
92915f71
GR		 601}
602
603/**
fa71ae27 604 * ixgbevf_poll - NAPI polling calback
92915f71
GR		 605 * @napi: napi struct with our devices info in it
606 * @budget: amount of work driver is allowed to do this pass, in packets
607 *
fa71ae27 608 * This function will clean more than one or more rings associated with a
92915f71
GR		 609 * q_vector.
610 **/
fa71ae27 611static int ixgbevf_poll(struct napi_struct *napi, int budget)
92915f71
GR		 612{
613 struct ixgbevf_q_vector *q_vector =
614 container_of(napi, struct ixgbevf_q_vector, napi);
615 struct ixgbevf_adapter *adapter = q_vector->adapter;
fa71ae27
AD		 616 struct ixgbevf_ring *ring;
617 int per_ring_budget;
618 bool clean_complete = true;
619
620 ixgbevf_for_each_ring(ring, q_vector->tx)
621 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring);
92915f71 622
c777cdfa
JK		 623#ifdef CONFIG_NET_RX_BUSY_POLL
624 if (!ixgbevf_qv_lock_napi(q_vector))
625 return budget;
626#endif
627
92915f71
GR		 628 /* attempt to distribute budget to each queue fairly, but don't allow
629 * the budget to go below 1 because we'll exit polling */
fa71ae27
AD		 630 if (q_vector->rx.count > 1)
631 per_ring_budget = max(budget/q_vector->rx.count, 1);
632 else
633 per_ring_budget = budget;
634
366c1099 635 adapter->flags |= IXGBE_FLAG_IN_NETPOLL;
fa71ae27 636 ixgbevf_for_each_ring(ring, q_vector->rx)
08e50a20
JK		 637 clean_complete &= (ixgbevf_clean_rx_irq(q_vector, ring,
638 per_ring_budget)
639 < per_ring_budget);
366c1099 640 adapter->flags &= ~IXGBE_FLAG_IN_NETPOLL;
fa71ae27 641
c777cdfa
JK		 642#ifdef CONFIG_NET_RX_BUSY_POLL
643 ixgbevf_qv_unlock_napi(q_vector);
644#endif
645
fa71ae27
AD		 646 /* If all work not completed, return budget and keep polling */
647 if (!clean_complete)
648 return budget;
649 /* all work done, exit the polling mode */
650 napi_complete(napi);
651 if (adapter->rx_itr_setting & 1)
652 ixgbevf_set_itr(q_vector);
2e7cfbdd
MR		 653 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
654 !test_bit(__IXGBEVF_REMOVING, &adapter->state))
fa71ae27
AD		 655 ixgbevf_irq_enable_queues(adapter,
656 1 << q_vector->v_idx);
92915f71 657
fa71ae27 658 return 0;
92915f71
GR		 659}
660
ce422606
GR		 661/**
662 * ixgbevf_write_eitr - write VTEITR register in hardware specific way
663 * @q_vector: structure containing interrupt and ring information
664 */
3849623e 665void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector)
ce422606
GR		 666{
667 struct ixgbevf_adapter *adapter = q_vector->adapter;
668 struct ixgbe_hw *hw = &adapter->hw;
669 int v_idx = q_vector->v_idx;
670 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
671
672 /*
673 * set the WDIS bit to not clear the timer bits and cause an
674 * immediate assertion of the interrupt
675 */
676 itr_reg |= IXGBE_EITR_CNT_WDIS;
677
678 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg);
679}
92915f71 680
c777cdfa
JK		 681#ifdef CONFIG_NET_RX_BUSY_POLL
682/* must be called with local_bh_disable()d */
683static int ixgbevf_busy_poll_recv(struct napi_struct *napi)
684{
685 struct ixgbevf_q_vector *q_vector =
686 container_of(napi, struct ixgbevf_q_vector, napi);
687 struct ixgbevf_adapter *adapter = q_vector->adapter;
688 struct ixgbevf_ring *ring;
689 int found = 0;
690
691 if (test_bit(__IXGBEVF_DOWN, &adapter->state))
692 return LL_FLUSH_FAILED;
693
694 if (!ixgbevf_qv_lock_poll(q_vector))
695 return LL_FLUSH_BUSY;
696
697 ixgbevf_for_each_ring(ring, q_vector->rx) {
698 found = ixgbevf_clean_rx_irq(q_vector, ring, 4);
3b5dca26
JK		 699#ifdef BP_EXTENDED_STATS
700 if (found)
095e2617 701 ring->stats.cleaned += found;
3b5dca26 702 else
095e2617 703 ring->stats.misses++;
3b5dca26 704#endif
c777cdfa
JK		 705 if (found)
706 break;
707 }
708
709 ixgbevf_qv_unlock_poll(q_vector);
710
711 return found;
712}
713#endif /* CONFIG_NET_RX_BUSY_POLL */
714
92915f71
GR		 715/**
716 * ixgbevf_configure_msix - Configure MSI-X hardware
717 * @adapter: board private structure
718 *
719 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X
720 * interrupts.
721 **/
722static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter)
723{
724 struct ixgbevf_q_vector *q_vector;
6b43c446 725 int q_vectors, v_idx;
92915f71
GR		 726
727 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
5f3600eb 728 adapter->eims_enable_mask = 0;
92915f71
GR		 729
730 /*
731 * Populate the IVAR table and set the ITR values to the
732 * corresponding register.
733 */
734 for (v_idx = 0; v_idx < q_vectors; v_idx++) {
6b43c446 735 struct ixgbevf_ring *ring;
92915f71 736 q_vector = adapter->q_vector[v_idx];
6b43c446
AD		 737
738 ixgbevf_for_each_ring(ring, q_vector->rx)
739 ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx);
740
741 ixgbevf_for_each_ring(ring, q_vector->tx)
742 ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx);
92915f71 743
5f3600eb
AD		 744 if (q_vector->tx.ring && !q_vector->rx.ring) {
745 /* tx only vector */
746 if (adapter->tx_itr_setting == 1)
747 q_vector->itr = IXGBE_10K_ITR;
748 else
749 q_vector->itr = adapter->tx_itr_setting;
750 } else {
751 /* rx or rx/tx vector */
752 if (adapter->rx_itr_setting == 1)
753 q_vector->itr = IXGBE_20K_ITR;
754 else
755 q_vector->itr = adapter->rx_itr_setting;
756 }
757
758 /* add q_vector eims value to global eims_enable_mask */
759 adapter->eims_enable_mask |= 1 << v_idx;
92915f71 760
5f3600eb 761 ixgbevf_write_eitr(q_vector);
92915f71
GR		 762 }
763
764 ixgbevf_set_ivar(adapter, -1, 1, v_idx);
5f3600eb
AD		 765 /* setup eims_other and add value to global eims_enable_mask */
766 adapter->eims_other = 1 << v_idx;
767 adapter->eims_enable_mask |= adapter->eims_other;
92915f71
GR		 768}
769
770enum latency_range {
771 lowest_latency = 0,
772 low_latency = 1,
773 bulk_latency = 2,
774 latency_invalid = 255
775};
776
777/**
778 * ixgbevf_update_itr - update the dynamic ITR value based on statistics
5f3600eb
AD		 779 * @q_vector: structure containing interrupt and ring information
780 * @ring_container: structure containing ring performance data
92915f71
GR		 781 *
782 * Stores a new ITR value based on packets and byte
783 * counts during the last interrupt. The advantage of per interrupt
784 * computation is faster updates and more accurate ITR for the current
785 * traffic pattern. Constants in this function were computed
786 * based on theoretical maximum wire speed and thresholds were set based
787 * on testing data as well as attempting to minimize response time
788 * while increasing bulk throughput.
789 **/
5f3600eb
AD		 790static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector,
791 struct ixgbevf_ring_container *ring_container)
92915f71 792{
5f3600eb
AD		 793 int bytes = ring_container->total_bytes;
794 int packets = ring_container->total_packets;
92915f71
GR		 795 u32 timepassed_us;
796 u64 bytes_perint;
5f3600eb 797 u8 itr_setting = ring_container->itr;
92915f71
GR		 798
799 if (packets == 0)
5f3600eb 800 return;
92915f71
GR		 801
802 /* simple throttlerate management
803 * 0-20MB/s lowest (100000 ints/s)
804 * 20-100MB/s low (20000 ints/s)
805 * 100-1249MB/s bulk (8000 ints/s)
806 */
807 /* what was last interrupt timeslice? */
5f3600eb 808 timepassed_us = q_vector->itr >> 2;
92915f71
GR		 809 bytes_perint = bytes / timepassed_us; /* bytes/usec */
810
811 switch (itr_setting) {
812 case lowest_latency:
e2c28ce7 813 if (bytes_perint > 10)
5f3600eb 814 itr_setting = low_latency;
92915f71
GR		 815 break;
816 case low_latency:
e2c28ce7 817 if (bytes_perint > 20)
5f3600eb 818 itr_setting = bulk_latency;
e2c28ce7 819 else if (bytes_perint <= 10)
5f3600eb 820 itr_setting = lowest_latency;
92915f71
GR		 821 break;
822 case bulk_latency:
e2c28ce7 823 if (bytes_perint <= 20)
5f3600eb 824 itr_setting = low_latency;
92915f71
GR		 825 break;
826 }
827
5f3600eb
AD		 828 /* clear work counters since we have the values we need */
829 ring_container->total_bytes = 0;
830 ring_container->total_packets = 0;
831
832 /* write updated itr to ring container */
833 ring_container->itr = itr_setting;
92915f71
GR		 834}
835
fa71ae27 836static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector)
92915f71 837{
5f3600eb
AD		 838 u32 new_itr = q_vector->itr;
839 u8 current_itr;
92915f71 840
5f3600eb
AD		 841 ixgbevf_update_itr(q_vector, &q_vector->tx);
842 ixgbevf_update_itr(q_vector, &q_vector->rx);
92915f71 843
6b43c446 844 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
92915f71
GR		 845
846 switch (current_itr) {
847 /* counts and packets in update_itr are dependent on these numbers */
848 case lowest_latency:
5f3600eb 849 new_itr = IXGBE_100K_ITR;
92915f71
GR		 850 break;
851 case low_latency:
5f3600eb 852 new_itr = IXGBE_20K_ITR;
92915f71
GR		 853 break;
854 case bulk_latency:
855 default:
5f3600eb 856 new_itr = IXGBE_8K_ITR;
92915f71
GR		 857 break;
858 }
859
5f3600eb 860 if (new_itr != q_vector->itr) {
92915f71 861 /* do an exponential smoothing */
5f3600eb
AD		 862 new_itr = (10 * new_itr * q_vector->itr) /
863 ((9 * new_itr) + q_vector->itr);
864
865 /* save the algorithm value here */
866 q_vector->itr = new_itr;
867
868 ixgbevf_write_eitr(q_vector);
92915f71 869 }
92915f71
GR		 870}
871
4b2cd27f 872static irqreturn_t ixgbevf_msix_other(int irq, void *data)
92915f71 873{
fa71ae27 874 struct ixgbevf_adapter *adapter = data;
92915f71 875 struct ixgbe_hw *hw = &adapter->hw;
08259594 876
4b2cd27f 877 hw->mac.get_link_status = 1;
1e72bfc3 878
2e7cfbdd
MR		 879 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
880 !test_bit(__IXGBEVF_REMOVING, &adapter->state))
c7bb417d 881 mod_timer(&adapter->watchdog_timer, jiffies);
3a2c4033 882
5f3600eb
AD		 883 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other);
884
92915f71
GR		 885 return IRQ_HANDLED;
886}
887
92915f71 888/**
fa71ae27 889 * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues)
92915f71
GR		 890 * @irq: unused
891 * @data: pointer to our q_vector struct for this interrupt vector
892 **/
fa71ae27 893static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data)
92915f71
GR		 894{
895 struct ixgbevf_q_vector *q_vector = data;
92915f71 896
5f3600eb 897 /* EIAM disabled interrupts (on this vector) for us */
fa71ae27
AD		 898 if (q_vector->rx.ring || q_vector->tx.ring)
899 napi_schedule(&q_vector->napi);
92915f71
GR		 900
901 return IRQ_HANDLED;
902}
903
904static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx,
905 int r_idx)
906{
907 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
908
87e70ab9
DS		 909 a->rx_ring[r_idx]->next = q_vector->rx.ring;
910 q_vector->rx.ring = a->rx_ring[r_idx];
6b43c446 911 q_vector->rx.count++;
92915f71
GR		 912}
913
914static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx,
915 int t_idx)
916{
917 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
918
87e70ab9
DS		 919 a->tx_ring[t_idx]->next = q_vector->tx.ring;
920 q_vector->tx.ring = a->tx_ring[t_idx];
6b43c446 921 q_vector->tx.count++;
92915f71
GR		 922}
923
924/**
925 * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors
926 * @adapter: board private structure to initialize
927 *
928 * This function maps descriptor rings to the queue-specific vectors
929 * we were allotted through the MSI-X enabling code. Ideally, we'd have
930 * one vector per ring/queue, but on a constrained vector budget, we
931 * group the rings as "efficiently" as possible. You would add new
932 * mapping configurations in here.
933 **/
934static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter)
935{
936 int q_vectors;
937 int v_start = 0;
938 int rxr_idx = 0, txr_idx = 0;
939 int rxr_remaining = adapter->num_rx_queues;
940 int txr_remaining = adapter->num_tx_queues;
941 int i, j;
942 int rqpv, tqpv;
943 int err = 0;
944
945 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
946
947 /*
948 * The ideal configuration...
949 * We have enough vectors to map one per queue.
950 */
951 if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) {
952 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++)
953 map_vector_to_rxq(adapter, v_start, rxr_idx);
954
955 for (; txr_idx < txr_remaining; v_start++, txr_idx++)
956 map_vector_to_txq(adapter, v_start, txr_idx);
957 goto out;
958 }
959
960 /*
961 * If we don't have enough vectors for a 1-to-1
962 * mapping, we'll have to group them so there are
963 * multiple queues per vector.
964 */
965 /* Re-adjusting *qpv takes care of the remainder. */
966 for (i = v_start; i < q_vectors; i++) {
967 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i);
968 for (j = 0; j < rqpv; j++) {
969 map_vector_to_rxq(adapter, i, rxr_idx);
970 rxr_idx++;
971 rxr_remaining--;
972 }
973 }
974 for (i = v_start; i < q_vectors; i++) {
975 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i);
976 for (j = 0; j < tqpv; j++) {
977 map_vector_to_txq(adapter, i, txr_idx);
978 txr_idx++;
979 txr_remaining--;
980 }
981 }
982
983out:
984 return err;
985}
986
987/**
988 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts
989 * @adapter: board private structure
990 *
991 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests
992 * interrupts from the kernel.
993 **/
994static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter)
995{
996 struct net_device *netdev = adapter->netdev;
fa71ae27
AD		 997 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
998 int vector, err;
92915f71
GR		 999 int ri = 0, ti = 0;
1000
92915f71 1001 for (vector = 0; vector < q_vectors; vector++) {
fa71ae27
AD		 1002 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector];
1003 struct msix_entry *entry = &adapter->msix_entries[vector];
1004
1005 if (q_vector->tx.ring && q_vector->rx.ring) {
1006 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
1007 "%s-%s-%d", netdev->name, "TxRx", ri++);
1008 ti++;
1009 } else if (q_vector->rx.ring) {
1010 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
1011 "%s-%s-%d", netdev->name, "rx", ri++);
1012 } else if (q_vector->tx.ring) {
1013 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
1014 "%s-%s-%d", netdev->name, "tx", ti++);
92915f71
GR		 1015 } else {
1016 /* skip this unused q_vector */
1017 continue;
1018 }
fa71ae27
AD		 1019 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0,
1020 q_vector->name, q_vector);
92915f71
GR		 1021 if (err) {
1022 hw_dbg(&adapter->hw,
1023 "request_irq failed for MSIX interrupt "
1024 "Error: %d\n", err);
1025 goto free_queue_irqs;
1026 }
1027 }
1028
92915f71 1029 err = request_irq(adapter->msix_entries[vector].vector,
4b2cd27f 1030 &ixgbevf_msix_other, 0, netdev->name, adapter);
92915f71
GR		 1031 if (err) {
1032 hw_dbg(&adapter->hw,
4b2cd27f 1033 "request_irq for msix_other failed: %d\n", err);
92915f71
GR		 1034 goto free_queue_irqs;
1035 }
1036
1037 return 0;
1038
1039free_queue_irqs:
fa71ae27
AD		 1040 while (vector) {
1041 vector--;
1042 free_irq(adapter->msix_entries[vector].vector,
1043 adapter->q_vector[vector]);
1044 }
a1f6c6b1 1045 /* This failure is non-recoverable - it indicates the system is
1046 * out of MSIX vector resources and the VF driver cannot run
1047 * without them. Set the number of msix vectors to zero
1048 * indicating that not enough can be allocated. The error
1049 * will be returned to the user indicating device open failed.
1050 * Any further attempts to force the driver to open will also
1051 * fail. The only way to recover is to unload the driver and
1052 * reload it again. If the system has recovered some MSIX
1053 * vectors then it may succeed.
1054 */
1055 adapter->num_msix_vectors = 0;
92915f71
GR		 1056 return err;
1057}
1058
1059static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter)
1060{
1061 int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1062
1063 for (i = 0; i < q_vectors; i++) {
1064 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i];
6b43c446
AD		 1065 q_vector->rx.ring = NULL;
1066 q_vector->tx.ring = NULL;
1067 q_vector->rx.count = 0;
1068 q_vector->tx.count = 0;
92915f71
GR		 1069 }
1070}
1071
1072/**
1073 * ixgbevf_request_irq - initialize interrupts
1074 * @adapter: board private structure
1075 *
1076 * Attempts to configure interrupts using the best available
1077 * capabilities of the hardware and kernel.
1078 **/
1079static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter)
1080{
1081 int err = 0;
1082
1083 err = ixgbevf_request_msix_irqs(adapter);
1084
1085 if (err)
1086 hw_dbg(&adapter->hw,
1087 "request_irq failed, Error %d\n", err);
1088
1089 return err;
1090}
1091
1092static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter)
1093{
92915f71
GR		 1094 int i, q_vectors;
1095
1096 q_vectors = adapter->num_msix_vectors;
92915f71
GR		 1097 i = q_vectors - 1;
1098
fa71ae27 1099 free_irq(adapter->msix_entries[i].vector, adapter);
92915f71
GR		 1100 i--;
1101
1102 for (; i >= 0; i--) {
fa71ae27
AD		 1103 /* free only the irqs that were actually requested */
1104 if (!adapter->q_vector[i]->rx.ring &&
1105 !adapter->q_vector[i]->tx.ring)
1106 continue;
1107
92915f71
GR		 1108 free_irq(adapter->msix_entries[i].vector,
1109 adapter->q_vector[i]);
1110 }
1111
1112 ixgbevf_reset_q_vectors(adapter);
1113}
1114
1115/**
1116 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC
1117 * @adapter: board private structure
1118 **/
1119static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter)
1120{
92915f71 1121 struct ixgbe_hw *hw = &adapter->hw;
5f3600eb 1122 int i;
92915f71 1123
5f3600eb 1124 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0);
92915f71 1125 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0);
5f3600eb 1126 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0);
92915f71
GR		 1127
1128 IXGBE_WRITE_FLUSH(hw);
1129
1130 for (i = 0; i < adapter->num_msix_vectors; i++)
1131 synchronize_irq(adapter->msix_entries[i].vector);
1132}
1133
1134/**
1135 * ixgbevf_irq_enable - Enable default interrupt generation settings
1136 * @adapter: board private structure
1137 **/
5f3600eb 1138static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter)
92915f71
GR		 1139{
1140 struct ixgbe_hw *hw = &adapter->hw;
92915f71 1141
5f3600eb
AD		 1142 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask);
1143 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask);
1144 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask);
92915f71
GR		 1145}
1146
de02decb
DS		 1147/**
1148 * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset
1149 * @adapter: board private structure
1150 * @ring: structure containing ring specific data
1151 *
1152 * Configure the Tx descriptor ring after a reset.
1153 **/
1154static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter,
1155 struct ixgbevf_ring *ring)
1156{
1157 struct ixgbe_hw *hw = &adapter->hw;
1158 u64 tdba = ring->dma;
1159 int wait_loop = 10;
1160 u32 txdctl = IXGBE_TXDCTL_ENABLE;
1161 u8 reg_idx = ring->reg_idx;
1162
1163 /* disable queue to avoid issues while updating state */
1164 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
1165 IXGBE_WRITE_FLUSH(hw);
1166
1167 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(reg_idx), tdba & DMA_BIT_MASK(32));
1168 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(reg_idx), tdba >> 32);
1169 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(reg_idx),
1170 ring->count * sizeof(union ixgbe_adv_tx_desc));
1171
1172 /* disable head writeback */
1173 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAH(reg_idx), 0);
1174 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAL(reg_idx), 0);
1175
1176 /* enable relaxed ordering */
1177 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(reg_idx),
1178 (IXGBE_DCA_TXCTRL_DESC_RRO_EN |
1179 IXGBE_DCA_TXCTRL_DATA_RRO_EN));
1180
1181 /* reset head and tail pointers */
1182 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(reg_idx), 0);
1183 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(reg_idx), 0);
dbf8b0d8 1184 ring->tail = adapter->io_addr + IXGBE_VFTDT(reg_idx);
de02decb
DS		 1185
1186 /* reset ntu and ntc to place SW in sync with hardwdare */
1187 ring->next_to_clean = 0;
1188 ring->next_to_use = 0;
1189
1190 /* In order to avoid issues WTHRESH + PTHRESH should always be equal
1191 * to or less than the number of on chip descriptors, which is
1192 * currently 40.
1193 */
1194 txdctl |= (8 << 16); /* WTHRESH = 8 */
1195
1196 /* Setting PTHRESH to 32 both improves performance */
1197 txdctl |= (1 << 8) | /* HTHRESH = 1 */
1198 32; /* PTHRESH = 32 */
1199
1200 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl);
1201
1202 /* poll to verify queue is enabled */
1203 do {
1204 usleep_range(1000, 2000);
1205 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(reg_idx));
1206 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
1207 if (!wait_loop)
1208 pr_err("Could not enable Tx Queue %d\n", reg_idx);
1209}
1210
92915f71
GR		 1211/**
1212 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset
1213 * @adapter: board private structure
1214 *
1215 * Configure the Tx unit of the MAC after a reset.
1216 **/
1217static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter)
1218{
de02decb 1219 u32 i;
92915f71
GR		 1220
1221 /* Setup the HW Tx Head and Tail descriptor pointers */
de02decb
DS		 1222 for (i = 0; i < adapter->num_tx_queues; i++)
1223 ixgbevf_configure_tx_ring(adapter, adapter->tx_ring[i]);
92915f71
GR		 1224}
1225
1226#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
1227
1228static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index)
1229{
1230 struct ixgbevf_ring *rx_ring;
1231 struct ixgbe_hw *hw = &adapter->hw;
1232 u32 srrctl;
1233
87e70ab9 1234 rx_ring = adapter->rx_ring[index];
92915f71
GR		 1235
1236 srrctl = IXGBE_SRRCTL_DROP_EN;
1237
77d5dfca 1238 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
92915f71 1239
dd1fe113
AD		 1240 srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >>
1241 IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1242
92915f71
GR		 1243 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl);
1244}
1245
1bb9c639
DS		 1246static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter)
1247{
1248 struct ixgbe_hw *hw = &adapter->hw;
1249
1250 /* PSRTYPE must be initialized in 82599 */
1251 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
1252 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR |
1253 IXGBE_PSRTYPE_L2HDR;
1254
1255 if (adapter->num_rx_queues > 1)
1256 psrtype |= 1 << 29;
1257
1258 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
1259}
1260
dd1fe113
AD		 1261static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter)
1262{
1263 struct ixgbe_hw *hw = &adapter->hw;
1264 struct net_device *netdev = adapter->netdev;
1265 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1266 int i;
1267 u16 rx_buf_len;
1268
1269 /* notify the PF of our intent to use this size of frame */
1270 ixgbevf_rlpml_set_vf(hw, max_frame);
1271
1272 /* PF will allow an extra 4 bytes past for vlan tagged frames */
1273 max_frame += VLAN_HLEN;
1274
1275 /*
85624caf
GR		 1276 * Allocate buffer sizes that fit well into 32K and
1277 * take into account max frame size of 9.5K
dd1fe113
AD		 1278 */
1279 if ((hw->mac.type == ixgbe_mac_X540_vf) &&
1280 (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE))
1281 rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85624caf
GR		 1282 else if (max_frame <= IXGBEVF_RXBUFFER_2K)
1283 rx_buf_len = IXGBEVF_RXBUFFER_2K;
1284 else if (max_frame <= IXGBEVF_RXBUFFER_4K)
1285 rx_buf_len = IXGBEVF_RXBUFFER_4K;
1286 else if (max_frame <= IXGBEVF_RXBUFFER_8K)
1287 rx_buf_len = IXGBEVF_RXBUFFER_8K;
dd1fe113 1288 else
85624caf 1289 rx_buf_len = IXGBEVF_RXBUFFER_10K;
dd1fe113
AD		 1290
1291 for (i = 0; i < adapter->num_rx_queues; i++)
87e70ab9 1292 adapter->rx_ring[i]->rx_buf_len = rx_buf_len;
dd1fe113
AD		 1293}
1294
de02decb
DS		 1295#define IXGBEVF_MAX_RX_DESC_POLL 10
1296static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter,
1297 struct ixgbevf_ring *ring)
1298{
1299 struct ixgbe_hw *hw = &adapter->hw;
1300 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1301 u32 rxdctl;
1302 u8 reg_idx = ring->reg_idx;
1303
1304 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1305 rxdctl &= ~IXGBE_RXDCTL_ENABLE;
1306
1307 /* write value back with RXDCTL.ENABLE bit cleared */
1308 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
1309
1310 /* the hardware may take up to 100us to really disable the rx queue */
1311 do {
1312 udelay(10);
1313 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1314 } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
1315
1316 if (!wait_loop)
1317 pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n",
1318 reg_idx);
1319}
1320
1321static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
1322 struct ixgbevf_ring *ring)
1323{
1324 struct ixgbe_hw *hw = &adapter->hw;
1325 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1326 u32 rxdctl;
1327 u8 reg_idx = ring->reg_idx;
1328
1329 do {
1330 usleep_range(1000, 2000);
1331 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1332 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
1333
1334 if (!wait_loop)
1335 pr_err("RXDCTL.ENABLE queue %d not set while polling\n",
1336 reg_idx);
1337}
1338
1339static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter,
1340 struct ixgbevf_ring *ring)
1341{
1342 struct ixgbe_hw *hw = &adapter->hw;
1343 u64 rdba = ring->dma;
1344 u32 rxdctl;
1345 u8 reg_idx = ring->reg_idx;
1346
1347 /* disable queue to avoid issues while updating state */
1348 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1349 ixgbevf_disable_rx_queue(adapter, ring);
1350
1351 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(reg_idx), rdba & DMA_BIT_MASK(32));
1352 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(reg_idx), rdba >> 32);
1353 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(reg_idx),
1354 ring->count * sizeof(union ixgbe_adv_rx_desc));
1355
1356 /* enable relaxed ordering */
1357 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx),
1358 IXGBE_DCA_RXCTRL_DESC_RRO_EN);
1359
1360 /* reset head and tail pointers */
1361 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(reg_idx), 0);
1362 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0);
dbf8b0d8 1363 ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx);
de02decb
DS		 1364
1365 /* reset ntu and ntc to place SW in sync with hardwdare */
1366 ring->next_to_clean = 0;
1367 ring->next_to_use = 0;
1368
1369 ixgbevf_configure_srrctl(adapter, reg_idx);
1370
1371 /* prevent DMA from exceeding buffer space available */
1372 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
1373 rxdctl |= ring->rx_buf_len | IXGBE_RXDCTL_RLPML_EN;
1374 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
1375 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
1376
1377 ixgbevf_rx_desc_queue_enable(adapter, ring);
095e2617 1378 ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring));
de02decb
DS		 1379}
1380
92915f71
GR		 1381/**
1382 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset
1383 * @adapter: board private structure
1384 *
1385 * Configure the Rx unit of the MAC after a reset.
1386 **/
1387static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter)
1388{
de02decb 1389 int i;
92915f71 1390
1bb9c639 1391 ixgbevf_setup_psrtype(adapter);
dd1fe113
AD		 1392
1393 /* set_rx_buffer_len must be called before ring initialization */
1394 ixgbevf_set_rx_buffer_len(adapter);
92915f71 1395
92915f71
GR		 1396 /* Setup the HW Rx Head and Tail Descriptor Pointers and
1397 * the Base and Length of the Rx Descriptor Ring */
de02decb
DS		 1398 for (i = 0; i < adapter->num_rx_queues; i++)
1399 ixgbevf_configure_rx_ring(adapter, adapter->rx_ring[i]);
92915f71
GR		 1400}
1401
80d5c368
PM		 1402static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev,
1403 __be16 proto, u16 vid)
92915f71
GR		 1404{
1405 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1406 struct ixgbe_hw *hw = &adapter->hw;
2ddc7fe1
AD		 1407 int err;
1408
55fdd45b 1409 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1410
92915f71 1411 /* add VID to filter table */
2ddc7fe1 1412 err = hw->mac.ops.set_vfta(hw, vid, 0, true);
1c55ed76 1413
55fdd45b 1414 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 1415
2ddc7fe1
AD		 1416 /* translate error return types so error makes sense */
1417 if (err == IXGBE_ERR_MBX)
1418 return -EIO;
1419
1420 if (err == IXGBE_ERR_INVALID_ARGUMENT)
1421 return -EACCES;
1422
dadcd65f 1423 set_bit(vid, adapter->active_vlans);
8e586137 1424
2ddc7fe1 1425 return err;
92915f71
GR		 1426}
1427
80d5c368
PM		 1428static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev,
1429 __be16 proto, u16 vid)
92915f71
GR		 1430{
1431 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1432 struct ixgbe_hw *hw = &adapter->hw;
2ddc7fe1 1433 int err = -EOPNOTSUPP;
92915f71 1434
55fdd45b 1435 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1436
92915f71 1437 /* remove VID from filter table */
92fe0bf7 1438 err = hw->mac.ops.set_vfta(hw, vid, 0, false);
1c55ed76 1439
55fdd45b 1440 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 1441
dadcd65f 1442 clear_bit(vid, adapter->active_vlans);
8e586137 1443
2ddc7fe1 1444 return err;
92915f71
GR		 1445}
1446
1447static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter)
1448{
dadcd65f 1449 u16 vid;
92915f71 1450
dadcd65f 1451 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
80d5c368
PM		 1452 ixgbevf_vlan_rx_add_vid(adapter->netdev,
1453 htons(ETH_P_8021Q), vid);
92915f71
GR		 1454}
1455
46ec20ff
GR		 1456static int ixgbevf_write_uc_addr_list(struct net_device *netdev)
1457{
1458 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1459 struct ixgbe_hw *hw = &adapter->hw;
1460 int count = 0;
1461
1462 if ((netdev_uc_count(netdev)) > 10) {
dbd9636e 1463 pr_err("Too many unicast filters - No Space\n");
46ec20ff
GR		 1464 return -ENOSPC;
1465 }
1466
1467 if (!netdev_uc_empty(netdev)) {
1468 struct netdev_hw_addr *ha;
1469 netdev_for_each_uc_addr(ha, netdev) {
1470 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr);
1471 udelay(200);
1472 }
1473 } else {
1474 /*
1475 * If the list is empty then send message to PF driver to
1476 * clear all macvlans on this VF.
1477 */
1478 hw->mac.ops.set_uc_addr(hw, 0, NULL);
1479 }
1480
1481 return count;
1482}
1483
92915f71 1484/**
dee847f5 1485 * ixgbevf_set_rx_mode - Multicast and unicast set
92915f71
GR		 1486 * @netdev: network interface device structure
1487 *
1488 * The set_rx_method entry point is called whenever the multicast address
dee847f5
GR		 1489 * list, unicast address list or the network interface flags are updated.
1490 * This routine is responsible for configuring the hardware for proper
1491 * multicast mode and configuring requested unicast filters.
92915f71
GR		 1492 **/
1493static void ixgbevf_set_rx_mode(struct net_device *netdev)
1494{
1495 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1496 struct ixgbe_hw *hw = &adapter->hw;
92915f71 1497
55fdd45b 1498 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1499
92915f71 1500 /* reprogram multicast list */
92fe0bf7 1501 hw->mac.ops.update_mc_addr_list(hw, netdev);
46ec20ff
GR		 1502
1503 ixgbevf_write_uc_addr_list(netdev);
1c55ed76 1504
55fdd45b 1505 spin_unlock_bh(&adapter->mbx_lock);
92915f71
GR		 1506}
1507
1508static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter)
1509{
1510 int q_idx;
1511 struct ixgbevf_q_vector *q_vector;
1512 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1513
1514 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
92915f71 1515 q_vector = adapter->q_vector[q_idx];
c777cdfa
JK		 1516#ifdef CONFIG_NET_RX_BUSY_POLL
1517 ixgbevf_qv_init_lock(adapter->q_vector[q_idx]);
1518#endif
fa71ae27 1519 napi_enable(&q_vector->napi);
92915f71
GR		 1520 }
1521}
1522
1523static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter)
1524{
1525 int q_idx;
1526 struct ixgbevf_q_vector *q_vector;
1527 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1528
1529 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1530 q_vector = adapter->q_vector[q_idx];
92915f71 1531 napi_disable(&q_vector->napi);
c777cdfa
JK		 1532#ifdef CONFIG_NET_RX_BUSY_POLL
1533 while (!ixgbevf_qv_disable(adapter->q_vector[q_idx])) {
1534 pr_info("QV %d locked\n", q_idx);
1535 usleep_range(1000, 20000);
1536 }
1537#endif /* CONFIG_NET_RX_BUSY_POLL */
92915f71
GR		 1538 }
1539}
1540
220fe050
DS		 1541static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter)
1542{
1543 struct ixgbe_hw *hw = &adapter->hw;
1544 unsigned int def_q = 0;
1545 unsigned int num_tcs = 0;
1546 unsigned int num_rx_queues = 1;
1547 int err;
1548
1549 spin_lock_bh(&adapter->mbx_lock);
1550
1551 /* fetch queue configuration from the PF */
1552 err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
1553
1554 spin_unlock_bh(&adapter->mbx_lock);
1555
1556 if (err)
1557 return err;
1558
1559 if (num_tcs > 1) {
1560 /* update default Tx ring register index */
87e70ab9 1561 adapter->tx_ring[0]->reg_idx = def_q;
220fe050
DS		 1562
1563 /* we need as many queues as traffic classes */
1564 num_rx_queues = num_tcs;
1565 }
1566
1567 /* if we have a bad config abort request queue reset */
1568 if (adapter->num_rx_queues != num_rx_queues) {
1569 /* force mailbox timeout to prevent further messages */
1570 hw->mbx.timeout = 0;
1571
1572 /* wait for watchdog to come around and bail us out */
1573 adapter->flags |= IXGBEVF_FLAG_QUEUE_RESET_REQUESTED;
1574 }
1575
1576 return 0;
1577}
1578
92915f71
GR		 1579static void ixgbevf_configure(struct ixgbevf_adapter *adapter)
1580{
220fe050
DS		 1581 ixgbevf_configure_dcb(adapter);
1582
de02decb 1583 ixgbevf_set_rx_mode(adapter->netdev);
92915f71
GR		 1584
1585 ixgbevf_restore_vlan(adapter);
1586
1587 ixgbevf_configure_tx(adapter);
1588 ixgbevf_configure_rx(adapter);
92915f71
GR		 1589}
1590
33bd9f60
GR		 1591static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter)
1592{
1593 /* Only save pre-reset stats if there are some */
1594 if (adapter->stats.vfgprc || adapter->stats.vfgptc) {
1595 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc -
1596 adapter->stats.base_vfgprc;
1597 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc -
1598 adapter->stats.base_vfgptc;
1599 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc -
1600 adapter->stats.base_vfgorc;
1601 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc -
1602 adapter->stats.base_vfgotc;
1603 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc -
1604 adapter->stats.base_vfmprc;
1605 }
1606}
1607
1608static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter)
1609{
1610 struct ixgbe_hw *hw = &adapter->hw;
1611
1612 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
1613 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
1614 adapter->stats.last_vfgorc |=
1615 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
1616 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
1617 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
1618 adapter->stats.last_vfgotc |=
1619 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
1620 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
1621
1622 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc;
1623 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc;
1624 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc;
1625 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc;
1626 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc;
1627}
1628
31186785
AD		 1629static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter)
1630{
1631 struct ixgbe_hw *hw = &adapter->hw;
56e94095
AD		 1632 int api[] = { ixgbe_mbox_api_11,
1633 ixgbe_mbox_api_10,
31186785
AD		 1634 ixgbe_mbox_api_unknown };
1635 int err = 0, idx = 0;
1636
55fdd45b 1637 spin_lock_bh(&adapter->mbx_lock);
31186785
AD		 1638
1639 while (api[idx] != ixgbe_mbox_api_unknown) {
1640 err = ixgbevf_negotiate_api_version(hw, api[idx]);
1641 if (!err)
1642 break;
1643 idx++;
1644 }
1645
55fdd45b 1646 spin_unlock_bh(&adapter->mbx_lock);
31186785
AD		 1647}
1648
795180d8 1649static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter)
92915f71
GR		 1650{
1651 struct net_device *netdev = adapter->netdev;
1652 struct ixgbe_hw *hw = &adapter->hw;
92915f71
GR		 1653
1654 ixgbevf_configure_msix(adapter);
1655
55fdd45b 1656 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1657
92fe0bf7
GR		 1658 if (is_valid_ether_addr(hw->mac.addr))
1659 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
1660 else
1661 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0);
92915f71 1662
55fdd45b 1663 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 1664
5b346dc9 1665 smp_mb__before_clear_bit();
92915f71
GR		 1666 clear_bit(__IXGBEVF_DOWN, &adapter->state);
1667 ixgbevf_napi_enable_all(adapter);
1668
1669 /* enable transmits */
1670 netif_tx_start_all_queues(netdev);
1671
33bd9f60
GR		 1672 ixgbevf_save_reset_stats(adapter);
1673 ixgbevf_init_last_counter_stats(adapter);
1674
4b2cd27f 1675 hw->mac.get_link_status = 1;
92915f71 1676 mod_timer(&adapter->watchdog_timer, jiffies);
92915f71
GR		 1677}
1678
795180d8 1679void ixgbevf_up(struct ixgbevf_adapter *adapter)
92915f71 1680{
92915f71
GR		 1681 struct ixgbe_hw *hw = &adapter->hw;
1682
1683 ixgbevf_configure(adapter);
1684
795180d8 1685 ixgbevf_up_complete(adapter);
92915f71
GR		 1686
1687 /* clear any pending interrupts, may auto mask */
1688 IXGBE_READ_REG(hw, IXGBE_VTEICR);
1689
5f3600eb 1690 ixgbevf_irq_enable(adapter);
92915f71
GR		 1691}
1692
1693/**
1694 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue
92915f71
GR		 1695 * @rx_ring: ring to free buffers from
1696 **/
05d063aa 1697static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring)
92915f71 1698{
92915f71
GR		 1699 unsigned long size;
1700 unsigned int i;
1701
c0456c23
GR		 1702 if (!rx_ring->rx_buffer_info)
1703 return;
92915f71 1704
c0456c23 1705 /* Free all the Rx ring sk_buffs */
92915f71
GR		 1706 for (i = 0; i < rx_ring->count; i++) {
1707 struct ixgbevf_rx_buffer *rx_buffer_info;
1708
1709 rx_buffer_info = &rx_ring->rx_buffer_info[i];
1710 if (rx_buffer_info->dma) {
05d063aa 1711 dma_unmap_single(rx_ring->dev, rx_buffer_info->dma,
92915f71 1712 rx_ring->rx_buf_len,
2a1f8794 1713 DMA_FROM_DEVICE);
92915f71
GR		 1714 rx_buffer_info->dma = 0;
1715 }
1716 if (rx_buffer_info->skb) {
1717 struct sk_buff *skb = rx_buffer_info->skb;
1718 rx_buffer_info->skb = NULL;
1719 do {
1720 struct sk_buff *this = skb;
5c60f81a 1721 skb = IXGBE_CB(skb)->prev;
92915f71
GR		 1722 dev_kfree_skb(this);
1723 } while (skb);
1724 }
92915f71
GR		 1725 }
1726
1727 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
1728 memset(rx_ring->rx_buffer_info, 0, size);
1729
1730 /* Zero out the descriptor ring */
1731 memset(rx_ring->desc, 0, rx_ring->size);
92915f71
GR		 1732}
1733
1734/**
1735 * ixgbevf_clean_tx_ring - Free Tx Buffers
92915f71
GR		 1736 * @tx_ring: ring to be cleaned
1737 **/
05d063aa 1738static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring)
92915f71
GR		 1739{
1740 struct ixgbevf_tx_buffer *tx_buffer_info;
1741 unsigned long size;
1742 unsigned int i;
1743
c0456c23
GR		 1744 if (!tx_ring->tx_buffer_info)
1745 return;
1746
92915f71 1747 /* Free all the Tx ring sk_buffs */
92915f71
GR		 1748 for (i = 0; i < tx_ring->count; i++) {
1749 tx_buffer_info = &tx_ring->tx_buffer_info[i];
70a10e25 1750 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
92915f71
GR		 1751 }
1752
1753 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
1754 memset(tx_ring->tx_buffer_info, 0, size);
1755
1756 memset(tx_ring->desc, 0, tx_ring->size);
92915f71
GR		 1757}
1758
1759/**
1760 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues
1761 * @adapter: board private structure
1762 **/
1763static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter)
1764{
1765 int i;
1766
1767 for (i = 0; i < adapter->num_rx_queues; i++)
05d063aa 1768 ixgbevf_clean_rx_ring(adapter->rx_ring[i]);
92915f71
GR		 1769}
1770
1771/**
1772 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues
1773 * @adapter: board private structure
1774 **/
1775static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter)
1776{
1777 int i;
1778
1779 for (i = 0; i < adapter->num_tx_queues; i++)
05d063aa 1780 ixgbevf_clean_tx_ring(adapter->tx_ring[i]);
92915f71
GR		 1781}
1782
1783void ixgbevf_down(struct ixgbevf_adapter *adapter)
1784{
1785 struct net_device *netdev = adapter->netdev;
1786 struct ixgbe_hw *hw = &adapter->hw;
de02decb 1787 int i;
92915f71
GR		 1788
1789 /* signal that we are down to the interrupt handler */
5b346dc9
MR		 1790 if (test_and_set_bit(__IXGBEVF_DOWN, &adapter->state))
1791 return; /* do nothing if already down */
858c3dda
DS		 1792
1793 /* disable all enabled rx queues */
1794 for (i = 0; i < adapter->num_rx_queues; i++)
87e70ab9 1795 ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[i]);
92915f71
GR		 1796
1797 netif_tx_disable(netdev);
1798
1799 msleep(10);
1800
1801 netif_tx_stop_all_queues(netdev);
1802
1803 ixgbevf_irq_disable(adapter);
1804
1805 ixgbevf_napi_disable_all(adapter);
1806
1807 del_timer_sync(&adapter->watchdog_timer);
1808 /* can't call flush scheduled work here because it can deadlock
1809 * if linkwatch_event tries to acquire the rtnl_lock which we are
1810 * holding */
1811 while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK)
1812 msleep(1);
1813
1814 /* disable transmits in the hardware now that interrupts are off */
1815 for (i = 0; i < adapter->num_tx_queues; i++) {
de02decb
DS		 1816 u8 reg_idx = adapter->tx_ring[i]->reg_idx;
1817
1818 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx),
1819 IXGBE_TXDCTL_SWFLSH);
92915f71
GR		 1820 }
1821
1822 netif_carrier_off(netdev);
1823
1824 if (!pci_channel_offline(adapter->pdev))
1825 ixgbevf_reset(adapter);
1826
1827 ixgbevf_clean_all_tx_rings(adapter);
1828 ixgbevf_clean_all_rx_rings(adapter);
1829}
1830
1831void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter)
1832{
1833 WARN_ON(in_interrupt());
c0456c23 1834
92915f71
GR		 1835 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state))
1836 msleep(1);
1837
4b2cd27f
AD		 1838 ixgbevf_down(adapter);
1839 ixgbevf_up(adapter);
92915f71
GR		 1840
1841 clear_bit(__IXGBEVF_RESETTING, &adapter->state);
1842}
1843
1844void ixgbevf_reset(struct ixgbevf_adapter *adapter)
1845{
1846 struct ixgbe_hw *hw = &adapter->hw;
1847 struct net_device *netdev = adapter->netdev;
1848
798e381a 1849 if (hw->mac.ops.reset_hw(hw)) {
92915f71 1850 hw_dbg(hw, "PF still resetting\n");
798e381a 1851 } else {
92915f71 1852 hw->mac.ops.init_hw(hw);
798e381a
DS		 1853 ixgbevf_negotiate_api(adapter);
1854 }
92915f71
GR		 1855
1856 if (is_valid_ether_addr(adapter->hw.mac.addr)) {
1857 memcpy(netdev->dev_addr, adapter->hw.mac.addr,
1858 netdev->addr_len);
1859 memcpy(netdev->perm_addr, adapter->hw.mac.addr,
1860 netdev->addr_len);
1861 }
1862}
1863
e45dd5fe
JK		 1864static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter,
1865 int vectors)
92915f71 1866{
a5f9337b 1867 int vector_threshold;
92915f71 1868
fa71ae27
AD		 1869 /* We'll want at least 2 (vector_threshold):
1870 * 1) TxQ[0] + RxQ[0] handler
1871 * 2) Other (Link Status Change, etc.)
92915f71
GR		 1872 */
1873 vector_threshold = MIN_MSIX_COUNT;
1874
1875 /* The more we get, the more we will assign to Tx/Rx Cleanup
1876 * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1877 * Right now, we simply care about how many we'll get; we'll
1878 * set them up later while requesting irq's.
1879 */
5c1e3588
AG		 1880 vectors = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
1881 vector_threshold, vectors);
92915f71 1882
5c1e3588 1883 if (vectors < 0) {
e45dd5fe
JK		 1884 dev_err(&adapter->pdev->dev,
1885 "Unable to allocate MSI-X interrupts\n");
92915f71
GR		 1886 kfree(adapter->msix_entries);
1887 adapter->msix_entries = NULL;
5c1e3588 1888 return vectors;
92915f71 1889 }
dee847f5 1890
5c1e3588
AG		 1891 /* Adjust for only the vectors we'll use, which is minimum
1892 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
1893 * vectors we were allocated.
1894 */
1895 adapter->num_msix_vectors = vectors;
1896
1897 return 0;
92915f71
GR		 1898}
1899
49ce9c2c
BH		 1900/**
1901 * ixgbevf_set_num_queues - Allocate queues for device, feature dependent
92915f71
GR		 1902 * @adapter: board private structure to initialize
1903 *
1904 * This is the top level queue allocation routine. The order here is very
1905 * important, starting with the "most" number of features turned on at once,
1906 * and ending with the smallest set of features. This way large combinations
1907 * can be allocated if they're turned on, and smaller combinations are the
1908 * fallthrough conditions.
1909 *
1910 **/
1911static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter)
1912{
220fe050
DS		 1913 struct ixgbe_hw *hw = &adapter->hw;
1914 unsigned int def_q = 0;
1915 unsigned int num_tcs = 0;
1916 int err;
1917
92915f71
GR		 1918 /* Start with base case */
1919 adapter->num_rx_queues = 1;
1920 adapter->num_tx_queues = 1;
220fe050
DS		 1921
1922 spin_lock_bh(&adapter->mbx_lock);
1923
1924 /* fetch queue configuration from the PF */
1925 err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
1926
1927 spin_unlock_bh(&adapter->mbx_lock);
1928
1929 if (err)
1930 return;
1931
1932 /* we need as many queues as traffic classes */
1933 if (num_tcs > 1)
1934 adapter->num_rx_queues = num_tcs;
92915f71
GR		 1935}
1936
1937/**
1938 * ixgbevf_alloc_queues - Allocate memory for all rings
1939 * @adapter: board private structure to initialize
1940 *
1941 * We allocate one ring per queue at run-time since we don't know the
1942 * number of queues at compile-time. The polling_netdev array is
1943 * intended for Multiqueue, but should work fine with a single queue.
1944 **/
1945static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter)
1946{
87e70ab9
DS		 1947 struct ixgbevf_ring *ring;
1948 int rx = 0, tx = 0;
92915f71 1949
87e70ab9
DS		 1950 for (; tx < adapter->num_tx_queues; tx++) {
1951 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
1952 if (!ring)
1953 goto err_allocation;
92915f71 1954
87e70ab9
DS		 1955 ring->dev = &adapter->pdev->dev;
1956 ring->netdev = adapter->netdev;
1957 ring->count = adapter->tx_ring_count;
1958 ring->queue_index = tx;
1959 ring->reg_idx = tx;
92915f71 1960
87e70ab9 1961 adapter->tx_ring[tx] = ring;
92915f71
GR		 1962 }
1963
87e70ab9
DS		 1964 for (; rx < adapter->num_rx_queues; rx++) {
1965 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
1966 if (!ring)
1967 goto err_allocation;
1968
1969 ring->dev = &adapter->pdev->dev;
1970 ring->netdev = adapter->netdev;
1971
1972 ring->count = adapter->rx_ring_count;
1973 ring->queue_index = rx;
1974 ring->reg_idx = rx;
1975
1976 adapter->rx_ring[rx] = ring;
92915f71
GR		 1977 }
1978
1979 return 0;
1980
87e70ab9
DS		 1981err_allocation:
1982 while (tx) {
1983 kfree(adapter->tx_ring[--tx]);
1984 adapter->tx_ring[tx] = NULL;
1985 }
1986
1987 while (rx) {
1988 kfree(adapter->rx_ring[--rx]);
1989 adapter->rx_ring[rx] = NULL;
1990 }
92915f71
GR		 1991 return -ENOMEM;
1992}
1993
1994/**
1995 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported
1996 * @adapter: board private structure to initialize
1997 *
1998 * Attempt to configure the interrupts using the best available
1999 * capabilities of the hardware and the kernel.
2000 **/
2001static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter)
2002{
91e2b89b 2003 struct net_device *netdev = adapter->netdev;
92915f71
GR		 2004 int err = 0;
2005 int vector, v_budget;
2006
2007 /*
2008 * It's easy to be greedy for MSI-X vectors, but it really
2009 * doesn't do us much good if we have a lot more vectors
2010 * than CPU's. So let's be conservative and only ask for
fa71ae27
AD		 2011 * (roughly) the same number of vectors as there are CPU's.
2012 * The default is to use pairs of vectors.
92915f71 2013 */
fa71ae27
AD		 2014 v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues);
2015 v_budget = min_t(int, v_budget, num_online_cpus());
2016 v_budget += NON_Q_VECTORS;
92915f71
GR		 2017
2018 /* A failure in MSI-X entry allocation isn't fatal, but it does
2019 * mean we disable MSI-X capabilities of the adapter. */
2020 adapter->msix_entries = kcalloc(v_budget,
2021 sizeof(struct msix_entry), GFP_KERNEL);
2022 if (!adapter->msix_entries) {
2023 err = -ENOMEM;
2024 goto out;
2025 }
2026
2027 for (vector = 0; vector < v_budget; vector++)
2028 adapter->msix_entries[vector].entry = vector;
2029
e45dd5fe
JK		 2030 err = ixgbevf_acquire_msix_vectors(adapter, v_budget);
2031 if (err)
2032 goto out;
92915f71 2033
91e2b89b
GR		 2034 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues);
2035 if (err)
2036 goto out;
2037
2038 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues);
2039
92915f71
GR		 2040out:
2041 return err;
2042}
2043
2044/**
2045 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors
2046 * @adapter: board private structure to initialize
2047 *
2048 * We allocate one q_vector per queue interrupt. If allocation fails we
2049 * return -ENOMEM.
2050 **/
2051static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter)
2052{
2053 int q_idx, num_q_vectors;
2054 struct ixgbevf_q_vector *q_vector;
92915f71
GR		 2055
2056 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
92915f71
GR		 2057
2058 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
2059 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL);
2060 if (!q_vector)
2061 goto err_out;
2062 q_vector->adapter = adapter;
2063 q_vector->v_idx = q_idx;
fa71ae27
AD		 2064 netif_napi_add(adapter->netdev, &q_vector->napi,
2065 ixgbevf_poll, 64);
c777cdfa
JK		 2066#ifdef CONFIG_NET_RX_BUSY_POLL
2067 napi_hash_add(&q_vector->napi);
2068#endif
92915f71
GR		 2069 adapter->q_vector[q_idx] = q_vector;
2070 }
2071
2072 return 0;
2073
2074err_out:
2075 while (q_idx) {
2076 q_idx--;
2077 q_vector = adapter->q_vector[q_idx];
c777cdfa
JK		 2078#ifdef CONFIG_NET_RX_BUSY_POLL
2079 napi_hash_del(&q_vector->napi);
2080#endif
92915f71
GR		 2081 netif_napi_del(&q_vector->napi);
2082 kfree(q_vector);
2083 adapter->q_vector[q_idx] = NULL;
2084 }
2085 return -ENOMEM;
2086}
2087
2088/**
2089 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors
2090 * @adapter: board private structure to initialize
2091 *
2092 * This function frees the memory allocated to the q_vectors. In addition if
2093 * NAPI is enabled it will delete any references to the NAPI struct prior
2094 * to freeing the q_vector.
2095 **/
2096static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter)
2097{
f4477702 2098 int q_idx, num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
92915f71
GR		 2099
2100 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
2101 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx];
2102
2103 adapter->q_vector[q_idx] = NULL;
c777cdfa
JK		 2104#ifdef CONFIG_NET_RX_BUSY_POLL
2105 napi_hash_del(&q_vector->napi);
2106#endif
f4477702 2107 netif_napi_del(&q_vector->napi);
92915f71
GR		 2108 kfree(q_vector);
2109 }
2110}
2111
2112/**
2113 * ixgbevf_reset_interrupt_capability - Reset MSIX setup
2114 * @adapter: board private structure
2115 *
2116 **/
2117static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter)
2118{
2119 pci_disable_msix(adapter->pdev);
2120 kfree(adapter->msix_entries);
2121 adapter->msix_entries = NULL;
92915f71
GR		 2122}
2123
2124/**
2125 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init
2126 * @adapter: board private structure to initialize
2127 *
2128 **/
2129static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
2130{
2131 int err;
2132
2133 /* Number of supported queues */
2134 ixgbevf_set_num_queues(adapter);
2135
2136 err = ixgbevf_set_interrupt_capability(adapter);
2137 if (err) {
2138 hw_dbg(&adapter->hw,
2139 "Unable to setup interrupt capabilities\n");
2140 goto err_set_interrupt;
2141 }
2142
2143 err = ixgbevf_alloc_q_vectors(adapter);
2144 if (err) {
2145 hw_dbg(&adapter->hw, "Unable to allocate memory for queue "
2146 "vectors\n");
2147 goto err_alloc_q_vectors;
2148 }
2149
2150 err = ixgbevf_alloc_queues(adapter);
2151 if (err) {
dbd9636e 2152 pr_err("Unable to allocate memory for queues\n");
92915f71
GR		 2153 goto err_alloc_queues;
2154 }
2155
2156 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, "
2157 "Tx Queue count = %u\n",
2158 (adapter->num_rx_queues > 1) ? "Enabled" :
2159 "Disabled", adapter->num_rx_queues, adapter->num_tx_queues);
2160
2161 set_bit(__IXGBEVF_DOWN, &adapter->state);
2162
2163 return 0;
2164err_alloc_queues:
2165 ixgbevf_free_q_vectors(adapter);
2166err_alloc_q_vectors:
2167 ixgbevf_reset_interrupt_capability(adapter);
2168err_set_interrupt:
2169 return err;
2170}
2171
0ac1e8ce
AD		 2172/**
2173 * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings
2174 * @adapter: board private structure to clear interrupt scheme on
2175 *
2176 * We go through and clear interrupt specific resources and reset the structure
2177 * to pre-load conditions
2178 **/
2179static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter)
2180{
87e70ab9
DS		 2181 int i;
2182
2183 for (i = 0; i < adapter->num_tx_queues; i++) {
2184 kfree(adapter->tx_ring[i]);
2185 adapter->tx_ring[i] = NULL;
2186 }
2187 for (i = 0; i < adapter->num_rx_queues; i++) {
2188 kfree(adapter->rx_ring[i]);
2189 adapter->rx_ring[i] = NULL;
2190 }
2191
0ac1e8ce
AD		 2192 adapter->num_tx_queues = 0;
2193 adapter->num_rx_queues = 0;
2194
2195 ixgbevf_free_q_vectors(adapter);
2196 ixgbevf_reset_interrupt_capability(adapter);
2197}
2198
92915f71
GR		 2199/**
2200 * ixgbevf_sw_init - Initialize general software structures
2201 * (struct ixgbevf_adapter)
2202 * @adapter: board private structure to initialize
2203 *
2204 * ixgbevf_sw_init initializes the Adapter private data structure.
2205 * Fields are initialized based on PCI device information and
2206 * OS network device settings (MTU size).
2207 **/
9f9a12f8 2208static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter)
92915f71
GR		 2209{
2210 struct ixgbe_hw *hw = &adapter->hw;
2211 struct pci_dev *pdev = adapter->pdev;
e1941a74 2212 struct net_device *netdev = adapter->netdev;
92915f71
GR		 2213 int err;
2214
2215 /* PCI config space info */
2216
2217 hw->vendor_id = pdev->vendor;
2218 hw->device_id = pdev->device;
ff938e43 2219 hw->revision_id = pdev->revision;
92915f71
GR		 2220 hw->subsystem_vendor_id = pdev->subsystem_vendor;
2221 hw->subsystem_device_id = pdev->subsystem_device;
2222
2223 hw->mbx.ops.init_params(hw);
56e94095
AD		 2224
2225 /* assume legacy case in which PF would only give VF 2 queues */
2226 hw->mac.max_tx_queues = 2;
2227 hw->mac.max_rx_queues = 2;
2228
798e381a
DS		 2229 /* lock to protect mailbox accesses */
2230 spin_lock_init(&adapter->mbx_lock);
2231
92915f71
GR		 2232 err = hw->mac.ops.reset_hw(hw);
2233 if (err) {
2234 dev_info(&pdev->dev,
e1941a74 2235 "PF still in reset state. Is the PF interface up?\n");
92915f71
GR		 2236 } else {
2237 err = hw->mac.ops.init_hw(hw);
2238 if (err) {
dbd9636e 2239 pr_err("init_shared_code failed: %d\n", err);
92915f71
GR		 2240 goto out;
2241 }
798e381a 2242 ixgbevf_negotiate_api(adapter);
e1941a74
GR		 2243 err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr);
2244 if (err)
2245 dev_info(&pdev->dev, "Error reading MAC address\n");
2246 else if (is_zero_ether_addr(adapter->hw.mac.addr))
2247 dev_info(&pdev->dev,
2248 "MAC address not assigned by administrator.\n");
2249 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2250 }
2251
2252 if (!is_valid_ether_addr(netdev->dev_addr)) {
2253 dev_info(&pdev->dev, "Assigning random MAC address\n");
2254 eth_hw_addr_random(netdev);
2255 memcpy(hw->mac.addr, netdev->dev_addr, netdev->addr_len);
92915f71
GR		 2256 }
2257
2258 /* Enable dynamic interrupt throttling rates */
5f3600eb
AD		 2259 adapter->rx_itr_setting = 1;
2260 adapter->tx_itr_setting = 1;
92915f71 2261
92915f71
GR		 2262 /* set default ring sizes */
2263 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD;
2264 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD;
2265
92915f71 2266 set_bit(__IXGBEVF_DOWN, &adapter->state);
1a0d6ae5 2267 return 0;
92915f71
GR		 2268
2269out:
2270 return err;
2271}
2272
92915f71
GR		 2273#define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \
2274 { \
2275 u32 current_counter = IXGBE_READ_REG(hw, reg); \
2276 if (current_counter < last_counter) \
2277 counter += 0x100000000LL; \
2278 last_counter = current_counter; \
2279 counter &= 0xFFFFFFFF00000000LL; \
2280 counter |= current_counter; \
2281 }
2282
2283#define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
2284 { \
2285 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \
2286 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \
2287 u64 current_counter = (current_counter_msb << 32) | \
2288 current_counter_lsb; \
2289 if (current_counter < last_counter) \
2290 counter += 0x1000000000LL; \
2291 last_counter = current_counter; \
2292 counter &= 0xFFFFFFF000000000LL; \
2293 counter |= current_counter; \
2294 }
2295/**
2296 * ixgbevf_update_stats - Update the board statistics counters.
2297 * @adapter: board private structure
2298 **/
2299void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
2300{
2301 struct ixgbe_hw *hw = &adapter->hw;
55fb277c 2302 int i;
92915f71 2303
088245a3
GR		 2304 if (!adapter->link_up)
2305 return;
2306
92915f71
GR		 2307 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc,
2308 adapter->stats.vfgprc);
2309 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc,
2310 adapter->stats.vfgptc);
2311 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
2312 adapter->stats.last_vfgorc,
2313 adapter->stats.vfgorc);
2314 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
2315 adapter->stats.last_vfgotc,
2316 adapter->stats.vfgotc);
2317 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc,
2318 adapter->stats.vfmprc);
55fb277c
GR		 2319
2320 for (i = 0; i < adapter->num_rx_queues; i++) {
2321 adapter->hw_csum_rx_error +=
87e70ab9 2322 adapter->rx_ring[i]->hw_csum_rx_error;
87e70ab9 2323 adapter->rx_ring[i]->hw_csum_rx_error = 0;
55fb277c 2324 }
92915f71
GR		 2325}
2326
2327/**
2328 * ixgbevf_watchdog - Timer Call-back
2329 * @data: pointer to adapter cast into an unsigned long
2330 **/
2331static void ixgbevf_watchdog(unsigned long data)
2332{
2333 struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data;
2334 struct ixgbe_hw *hw = &adapter->hw;
5f3600eb 2335 u32 eics = 0;
92915f71
GR		 2336 int i;
2337
2338 /*
2339 * Do the watchdog outside of interrupt context due to the lovely
2340 * delays that some of the newer hardware requires
2341 */
2342
2343 if (test_bit(__IXGBEVF_DOWN, &adapter->state))
2344 goto watchdog_short_circuit;
2345
2346 /* get one bit for every active tx/rx interrupt vector */
2347 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
2348 struct ixgbevf_q_vector *qv = adapter->q_vector[i];
6b43c446 2349 if (qv->rx.ring || qv->tx.ring)
5f3600eb 2350 eics |= 1 << i;
92915f71
GR		 2351 }
2352
5f3600eb 2353 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics);
92915f71
GR		 2354
2355watchdog_short_circuit:
2356 schedule_work(&adapter->watchdog_task);
2357}
2358
2359/**
2360 * ixgbevf_tx_timeout - Respond to a Tx Hang
2361 * @netdev: network interface device structure
2362 **/
2363static void ixgbevf_tx_timeout(struct net_device *netdev)
2364{
2365 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2366
2367 /* Do the reset outside of interrupt context */
2368 schedule_work(&adapter->reset_task);
2369}
2370
2371static void ixgbevf_reset_task(struct work_struct *work)
2372{
2373 struct ixgbevf_adapter *adapter;
2374 adapter = container_of(work, struct ixgbevf_adapter, reset_task);
2375
2376 /* If we're already down or resetting, just bail */
2377 if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
2e7cfbdd 2378 test_bit(__IXGBEVF_REMOVING, &adapter->state) ||
92915f71
GR		 2379 test_bit(__IXGBEVF_RESETTING, &adapter->state))
2380 return;
2381
2382 adapter->tx_timeout_count++;
2383
2384 ixgbevf_reinit_locked(adapter);
2385}
2386
2387/**
2388 * ixgbevf_watchdog_task - worker thread to bring link up
2389 * @work: pointer to work_struct containing our data
2390 **/
2391static void ixgbevf_watchdog_task(struct work_struct *work)
2392{
2393 struct ixgbevf_adapter *adapter = container_of(work,
2394 struct ixgbevf_adapter,
2395 watchdog_task);
2396 struct net_device *netdev = adapter->netdev;
2397 struct ixgbe_hw *hw = &adapter->hw;
2398 u32 link_speed = adapter->link_speed;
2399 bool link_up = adapter->link_up;
92fe0bf7 2400 s32 need_reset;
92915f71 2401
220fe050
DS		 2402 ixgbevf_queue_reset_subtask(adapter);
2403
92915f71
GR		 2404 adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK;
2405
2406 /*
2407 * Always check the link on the watchdog because we have
2408 * no LSC interrupt
2409 */
92fe0bf7 2410 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 2411
92fe0bf7 2412 need_reset = hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
1c55ed76 2413
92fe0bf7 2414 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 2415
92fe0bf7
GR		 2416 if (need_reset) {
2417 adapter->link_up = link_up;
2418 adapter->link_speed = link_speed;
2419 netif_carrier_off(netdev);
2420 netif_tx_stop_all_queues(netdev);
2421 schedule_work(&adapter->reset_task);
2422 goto pf_has_reset;
92915f71
GR		 2423 }
2424 adapter->link_up = link_up;
2425 adapter->link_speed = link_speed;
2426
2427 if (link_up) {
2428 if (!netif_carrier_ok(netdev)) {
b876a744
GR		 2429 char *link_speed_string;
2430 switch (link_speed) {
2431 case IXGBE_LINK_SPEED_10GB_FULL:
2432 link_speed_string = "10 Gbps";
2433 break;
2434 case IXGBE_LINK_SPEED_1GB_FULL:
2435 link_speed_string = "1 Gbps";
2436 break;
2437 case IXGBE_LINK_SPEED_100_FULL:
2438 link_speed_string = "100 Mbps";
2439 break;
2440 default:
2441 link_speed_string = "unknown speed";
2442 break;
2443 }
6fe59675 2444 dev_info(&adapter->pdev->dev,
b876a744 2445 "NIC Link is Up, %s\n", link_speed_string);
92915f71
GR		 2446 netif_carrier_on(netdev);
2447 netif_tx_wake_all_queues(netdev);
92915f71
GR		 2448 }
2449 } else {
2450 adapter->link_up = false;
2451 adapter->link_speed = 0;
2452 if (netif_carrier_ok(netdev)) {
6fe59675 2453 dev_info(&adapter->pdev->dev, "NIC Link is Down\n");
92915f71
GR		 2454 netif_carrier_off(netdev);
2455 netif_tx_stop_all_queues(netdev);
2456 }
2457 }
2458
92915f71
GR		 2459 ixgbevf_update_stats(adapter);
2460
33bd9f60 2461pf_has_reset:
92915f71 2462 /* Reset the timer */
2e7cfbdd
MR		 2463 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
2464 !test_bit(__IXGBEVF_REMOVING, &adapter->state))
92915f71
GR		 2465 mod_timer(&adapter->watchdog_timer,
2466 round_jiffies(jiffies + (2 * HZ)));
2467
2468 adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK;
2469}
2470
2471/**
2472 * ixgbevf_free_tx_resources - Free Tx Resources per Queue
92915f71
GR		 2473 * @tx_ring: Tx descriptor ring for a specific queue
2474 *
2475 * Free all transmit software resources
2476 **/
05d063aa 2477void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring)
92915f71 2478{
05d063aa 2479 ixgbevf_clean_tx_ring(tx_ring);
92915f71
GR		 2480
2481 vfree(tx_ring->tx_buffer_info);
2482 tx_ring->tx_buffer_info = NULL;
2483
de02decb
DS		 2484 /* if not set, then don't free */
2485 if (!tx_ring->desc)
2486 return;
2487
05d063aa 2488 dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc,
2a1f8794 2489 tx_ring->dma);
92915f71
GR		 2490
2491 tx_ring->desc = NULL;
2492}
2493
2494/**
2495 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues
2496 * @adapter: board private structure
2497 *
2498 * Free all transmit software resources
2499 **/
2500static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter)
2501{
2502 int i;
2503
2504 for (i = 0; i < adapter->num_tx_queues; i++)
87e70ab9 2505 if (adapter->tx_ring[i]->desc)
05d063aa 2506 ixgbevf_free_tx_resources(adapter->tx_ring[i]);
92915f71
GR		 2507}
2508
2509/**
2510 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors)
92915f71
GR		 2511 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2512 *
2513 * Return 0 on success, negative on failure
2514 **/
05d063aa 2515int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring)
92915f71 2516{
92915f71
GR		 2517 int size;
2518
2519 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
89bf67f1 2520 tx_ring->tx_buffer_info = vzalloc(size);
92915f71
GR		 2521 if (!tx_ring->tx_buffer_info)
2522 goto err;
92915f71
GR		 2523
2524 /* round up to nearest 4K */
2525 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
2526 tx_ring->size = ALIGN(tx_ring->size, 4096);
2527
05d063aa 2528 tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size,
2a1f8794 2529 &tx_ring->dma, GFP_KERNEL);
92915f71
GR		 2530 if (!tx_ring->desc)
2531 goto err;
2532
92915f71
GR		 2533 return 0;
2534
2535err:
2536 vfree(tx_ring->tx_buffer_info);
2537 tx_ring->tx_buffer_info = NULL;
2538 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit "
2539 "descriptor ring\n");
2540 return -ENOMEM;
2541}
2542
2543/**
2544 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources
2545 * @adapter: board private structure
2546 *
2547 * If this function returns with an error, then it's possible one or
2548 * more of the rings is populated (while the rest are not). It is the
2549 * callers duty to clean those orphaned rings.
2550 *
2551 * Return 0 on success, negative on failure
2552 **/
2553static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter)
2554{
2555 int i, err = 0;
2556
2557 for (i = 0; i < adapter->num_tx_queues; i++) {
05d063aa 2558 err = ixgbevf_setup_tx_resources(adapter->tx_ring[i]);
92915f71
GR		 2559 if (!err)
2560 continue;
2561 hw_dbg(&adapter->hw,
2562 "Allocation for Tx Queue %u failed\n", i);
2563 break;
2564 }
2565
2566 return err;
2567}
2568
2569/**
2570 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors)
92915f71
GR		 2571 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2572 *
2573 * Returns 0 on success, negative on failure
2574 **/
05d063aa 2575int ixgbevf_setup_rx_resources(struct ixgbevf_ring *rx_ring)
92915f71 2576{
92915f71
GR		 2577 int size;
2578
2579 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
89bf67f1 2580 rx_ring->rx_buffer_info = vzalloc(size);
e404decb 2581 if (!rx_ring->rx_buffer_info)
05d063aa 2582 goto err;
92915f71
GR		 2583
2584 /* Round up to nearest 4K */
2585 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
2586 rx_ring->size = ALIGN(rx_ring->size, 4096);
2587
05d063aa 2588 rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size,
2a1f8794 2589 &rx_ring->dma, GFP_KERNEL);
92915f71 2590
05d063aa
ET		 2591 if (!rx_ring->desc)
2592 goto err;
92915f71 2593
92915f71 2594 return 0;
05d063aa
ET		 2595err:
2596 vfree(rx_ring->rx_buffer_info);
2597 rx_ring->rx_buffer_info = NULL;
2598 dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n");
92915f71
GR		 2599 return -ENOMEM;
2600}
2601
2602/**
2603 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources
2604 * @adapter: board private structure
2605 *
2606 * If this function returns with an error, then it's possible one or
2607 * more of the rings is populated (while the rest are not). It is the
2608 * callers duty to clean those orphaned rings.
2609 *
2610 * Return 0 on success, negative on failure
2611 **/
2612static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter)
2613{
2614 int i, err = 0;
2615
2616 for (i = 0; i < adapter->num_rx_queues; i++) {
05d063aa 2617 err = ixgbevf_setup_rx_resources(adapter->rx_ring[i]);
92915f71
GR		 2618 if (!err)
2619 continue;
2620 hw_dbg(&adapter->hw,
2621 "Allocation for Rx Queue %u failed\n", i);
2622 break;
2623 }
2624 return err;
2625}
2626
2627/**
2628 * ixgbevf_free_rx_resources - Free Rx Resources
92915f71
GR		 2629 * @rx_ring: ring to clean the resources from
2630 *
2631 * Free all receive software resources
2632 **/
05d063aa 2633void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring)
92915f71 2634{
05d063aa 2635 ixgbevf_clean_rx_ring(rx_ring);
92915f71
GR		 2636
2637 vfree(rx_ring->rx_buffer_info);
2638 rx_ring->rx_buffer_info = NULL;
2639
05d063aa 2640 dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc,
2a1f8794 2641 rx_ring->dma);
92915f71
GR		 2642
2643 rx_ring->desc = NULL;
2644}
2645
2646/**
2647 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues
2648 * @adapter: board private structure
2649 *
2650 * Free all receive software resources
2651 **/
2652static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter)
2653{
2654 int i;
2655
2656 for (i = 0; i < adapter->num_rx_queues; i++)
87e70ab9 2657 if (adapter->rx_ring[i]->desc)
05d063aa 2658 ixgbevf_free_rx_resources(adapter->rx_ring[i]);
92915f71
GR		 2659}
2660
2661/**
2662 * ixgbevf_open - Called when a network interface is made active
2663 * @netdev: network interface device structure
2664 *
2665 * Returns 0 on success, negative value on failure
2666 *
2667 * The open entry point is called when a network interface is made
2668 * active by the system (IFF_UP). At this point all resources needed
2669 * for transmit and receive operations are allocated, the interrupt
2670 * handler is registered with the OS, the watchdog timer is started,
2671 * and the stack is notified that the interface is ready.
2672 **/
2673static int ixgbevf_open(struct net_device *netdev)
2674{
2675 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2676 struct ixgbe_hw *hw = &adapter->hw;
2677 int err;
2678
a1f6c6b1 2679 /* A previous failure to open the device because of a lack of
2680 * available MSIX vector resources may have reset the number
2681 * of msix vectors variable to zero. The only way to recover
2682 * is to unload/reload the driver and hope that the system has
2683 * been able to recover some MSIX vector resources.
2684 */
2685 if (!adapter->num_msix_vectors)
2686 return -ENOMEM;
2687
92915f71
GR		 2688 /* disallow open during test */
2689 if (test_bit(__IXGBEVF_TESTING, &adapter->state))
2690 return -EBUSY;
2691
2692 if (hw->adapter_stopped) {
2693 ixgbevf_reset(adapter);
2694 /* if adapter is still stopped then PF isn't up and
2695 * the vf can't start. */
2696 if (hw->adapter_stopped) {
2697 err = IXGBE_ERR_MBX;
dbd9636e
JK		 2698 pr_err("Unable to start - perhaps the PF Driver isn't "
2699 "up yet\n");
92915f71
GR		 2700 goto err_setup_reset;
2701 }
2702 }
2703
2704 /* allocate transmit descriptors */
2705 err = ixgbevf_setup_all_tx_resources(adapter);
2706 if (err)
2707 goto err_setup_tx;
2708
2709 /* allocate receive descriptors */
2710 err = ixgbevf_setup_all_rx_resources(adapter);
2711 if (err)
2712 goto err_setup_rx;
2713
2714 ixgbevf_configure(adapter);
2715
2716 /*
2717 * Map the Tx/Rx rings to the vectors we were allotted.
2718 * if request_irq will be called in this function map_rings
2719 * must be called *before* up_complete
2720 */
2721 ixgbevf_map_rings_to_vectors(adapter);
2722
795180d8 2723 ixgbevf_up_complete(adapter);
92915f71
GR		 2724
2725 /* clear any pending interrupts, may auto mask */
2726 IXGBE_READ_REG(hw, IXGBE_VTEICR);
2727 err = ixgbevf_request_irq(adapter);
2728 if (err)
2729 goto err_req_irq;
2730
5f3600eb 2731 ixgbevf_irq_enable(adapter);
92915f71
GR		 2732
2733 return 0;
2734
2735err_req_irq:
2736 ixgbevf_down(adapter);
92915f71
GR		 2737err_setup_rx:
2738 ixgbevf_free_all_rx_resources(adapter);
2739err_setup_tx:
2740 ixgbevf_free_all_tx_resources(adapter);
2741 ixgbevf_reset(adapter);
2742
2743err_setup_reset:
2744
2745 return err;
2746}
2747
2748/**
2749 * ixgbevf_close - Disables a network interface
2750 * @netdev: network interface device structure
2751 *
2752 * Returns 0, this is not allowed to fail
2753 *
2754 * The close entry point is called when an interface is de-activated
2755 * by the OS. The hardware is still under the drivers control, but
2756 * needs to be disabled. A global MAC reset is issued to stop the
2757 * hardware, and all transmit and receive resources are freed.
2758 **/
2759static int ixgbevf_close(struct net_device *netdev)
2760{
2761 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2762
2763 ixgbevf_down(adapter);
2764 ixgbevf_free_irq(adapter);
2765
2766 ixgbevf_free_all_tx_resources(adapter);
2767 ixgbevf_free_all_rx_resources(adapter);
2768
2769 return 0;
2770}
2771
220fe050
DS		 2772static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter)
2773{
2774 struct net_device *dev = adapter->netdev;
2775
2776 if (!(adapter->flags & IXGBEVF_FLAG_QUEUE_RESET_REQUESTED))
2777 return;
2778
2779 adapter->flags &= ~IXGBEVF_FLAG_QUEUE_RESET_REQUESTED;
2780
2781 /* if interface is down do nothing */
2782 if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
2783 test_bit(__IXGBEVF_RESETTING, &adapter->state))
2784 return;
2785
2786 /* Hardware has to reinitialize queues and interrupts to
2787 * match packet buffer alignment. Unfortunately, the
2788 * hardware is not flexible enough to do this dynamically.
2789 */
2790 if (netif_running(dev))
2791 ixgbevf_close(dev);
2792
2793 ixgbevf_clear_interrupt_scheme(adapter);
2794 ixgbevf_init_interrupt_scheme(adapter);
2795
2796 if (netif_running(dev))
2797 ixgbevf_open(dev);
2798}
2799
70a10e25
AD		 2800static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring,
2801 u32 vlan_macip_lens, u32 type_tucmd,
2802 u32 mss_l4len_idx)
92915f71
GR		 2803{
2804 struct ixgbe_adv_tx_context_desc *context_desc;
70a10e25 2805 u16 i = tx_ring->next_to_use;
92915f71 2806
70a10e25 2807 context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i);
92915f71 2808
70a10e25
AD		 2809 i++;
2810 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
92915f71 2811
70a10e25
AD		 2812 /* set bits to identify this as an advanced context descriptor */
2813 type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
92915f71 2814
70a10e25
AD		 2815 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
2816 context_desc->seqnum_seed = 0;
2817 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
2818 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
2819}
2820
2821static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
7ad1a093
ET		 2822 struct ixgbevf_tx_buffer *first,
2823 u8 *hdr_len)
70a10e25 2824{
7ad1a093 2825 struct sk_buff *skb = first->skb;
70a10e25
AD		 2826 u32 vlan_macip_lens, type_tucmd;
2827 u32 mss_l4len_idx, l4len;
2828
01a545cf
ET		 2829 if (skb->ip_summed != CHECKSUM_PARTIAL)
2830 return 0;
2831
70a10e25
AD		 2832 if (!skb_is_gso(skb))
2833 return 0;
92915f71 2834
70a10e25
AD		 2835 if (skb_header_cloned(skb)) {
2836 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2837 if (err)
2838 return err;
92915f71
GR		 2839 }
2840
70a10e25
AD		 2841 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2842 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
2843
2844 if (skb->protocol == htons(ETH_P_IP)) {
2845 struct iphdr *iph = ip_hdr(skb);
2846 iph->tot_len = 0;
2847 iph->check = 0;
2848 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2849 iph->daddr, 0,
2850 IPPROTO_TCP,
2851 0);
2852 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
7ad1a093
ET		 2853 first->tx_flags |= IXGBE_TX_FLAGS_TSO |
2854 IXGBE_TX_FLAGS_CSUM |
2855 IXGBE_TX_FLAGS_IPV4;
70a10e25
AD		 2856 } else if (skb_is_gso_v6(skb)) {
2857 ipv6_hdr(skb)->payload_len = 0;
2858 tcp_hdr(skb)->check =
2859 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2860 &ipv6_hdr(skb)->daddr,
2861 0, IPPROTO_TCP, 0);
7ad1a093
ET		 2862 first->tx_flags |= IXGBE_TX_FLAGS_TSO |
2863 IXGBE_TX_FLAGS_CSUM;
70a10e25
AD		 2864 }
2865
2866 /* compute header lengths */
2867 l4len = tcp_hdrlen(skb);
2868 *hdr_len += l4len;
2869 *hdr_len = skb_transport_offset(skb) + l4len;
2870
7ad1a093
ET		 2871 /* update gso size and bytecount with header size */
2872 first->gso_segs = skb_shinfo(skb)->gso_segs;
2873 first->bytecount += (first->gso_segs - 1) * *hdr_len;
2874
70a10e25
AD		 2875 /* mss_l4len_id: use 1 as index for TSO */
2876 mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT;
2877 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
2878 mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT;
2879
2880 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
2881 vlan_macip_lens = skb_network_header_len(skb);
2882 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
7ad1a093 2883 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
70a10e25
AD		 2884
2885 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2886 type_tucmd, mss_l4len_idx);
2887
2888 return 1;
92915f71
GR		 2889}
2890
7ad1a093
ET		 2891static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
2892 struct ixgbevf_tx_buffer *first)
92915f71 2893{
7ad1a093 2894 struct sk_buff *skb = first->skb;
70a10e25
AD		 2895 u32 vlan_macip_lens = 0;
2896 u32 mss_l4len_idx = 0;
2897 u32 type_tucmd = 0;
92915f71 2898
70a10e25
AD		 2899 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2900 u8 l4_hdr = 0;
2901 switch (skb->protocol) {
0933ce4a 2902 case htons(ETH_P_IP):
70a10e25
AD		 2903 vlan_macip_lens |= skb_network_header_len(skb);
2904 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
2905 l4_hdr = ip_hdr(skb)->protocol;
2906 break;
0933ce4a 2907 case htons(ETH_P_IPV6):
70a10e25
AD		 2908 vlan_macip_lens |= skb_network_header_len(skb);
2909 l4_hdr = ipv6_hdr(skb)->nexthdr;
2910 break;
2911 default:
2912 if (unlikely(net_ratelimit())) {
2913 dev_warn(tx_ring->dev,
2914 "partial checksum but proto=%x!\n",
7ad1a093 2915 first->protocol);
70a10e25
AD		 2916 }
2917 break;
2918 }
92915f71 2919
70a10e25
AD		 2920 switch (l4_hdr) {
2921 case IPPROTO_TCP:
2922 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2923 mss_l4len_idx = tcp_hdrlen(skb) <<
2924 IXGBE_ADVTXD_L4LEN_SHIFT;
2925 break;
2926 case IPPROTO_SCTP:
2927 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
2928 mss_l4len_idx = sizeof(struct sctphdr) <<
2929 IXGBE_ADVTXD_L4LEN_SHIFT;
2930 break;
2931 case IPPROTO_UDP:
2932 mss_l4len_idx = sizeof(struct udphdr) <<
2933 IXGBE_ADVTXD_L4LEN_SHIFT;
2934 break;
2935 default:
2936 if (unlikely(net_ratelimit())) {
2937 dev_warn(tx_ring->dev,
2938 "partial checksum but l4 proto=%x!\n",
2939 l4_hdr);
2940 }
2941 break;
2942 }
7ad1a093
ET		 2943
2944 /* update TX checksum flag */
2945 first->tx_flags |= IXGBE_TX_FLAGS_CSUM;
92915f71
GR		 2946 }
2947
70a10e25
AD		 2948 /* vlan_macip_lens: MACLEN, VLAN tag */
2949 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
7ad1a093 2950 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
70a10e25
AD		 2951
2952 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2953 type_tucmd, mss_l4len_idx);
92915f71
GR		 2954}
2955
29d37fa1 2956static __le32 ixgbevf_tx_cmd_type(u32 tx_flags)
92915f71 2957{
29d37fa1
ET		 2958 /* set type for advanced descriptor with frame checksum insertion */
2959 __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA |
2960 IXGBE_ADVTXD_DCMD_IFCS |
2961 IXGBE_ADVTXD_DCMD_DEXT);
92915f71 2962
29d37fa1
ET		 2963 /* set HW vlan bit if vlan is present */
2964 if (tx_flags & IXGBE_TX_FLAGS_VLAN)
2965 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE);
92915f71 2966
29d37fa1
ET		 2967 /* set segmentation enable bits for TSO/FSO */
2968 if (tx_flags & IXGBE_TX_FLAGS_TSO)
2969 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE);
92915f71 2970
29d37fa1
ET		 2971 return cmd_type;
2972}
92915f71 2973
29d37fa1
ET		 2974static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc,
2975 u32 tx_flags, unsigned int paylen)
2976{
2977 __le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT);
9bdfefd2 2978
29d37fa1
ET		 2979 /* enable L4 checksum for TSO and TX checksum offload */
2980 if (tx_flags & IXGBE_TX_FLAGS_CSUM)
2981 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM);
92915f71 2982
29d37fa1
ET		 2983 /* enble IPv4 checksum for TSO */
2984 if (tx_flags & IXGBE_TX_FLAGS_IPV4)
2985 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM);
92915f71 2986
29d37fa1
ET		 2987 /* use index 1 context for TSO/FSO/FCOE */
2988 if (tx_flags & IXGBE_TX_FLAGS_TSO)
2989 olinfo_status |= cpu_to_le32(1 << IXGBE_ADVTXD_IDX_SHIFT);
92915f71 2990
29d37fa1
ET		 2991 /* Check Context must be set if Tx switch is enabled, which it
2992 * always is for case where virtual functions are running
2993 */
2994 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC);
92915f71 2995
29d37fa1
ET		 2996 tx_desc->read.olinfo_status = olinfo_status;
2997}
92915f71 2998
29d37fa1
ET		 2999static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
3000 struct ixgbevf_tx_buffer *first,
3001 const u8 hdr_len)
3002{
3003 dma_addr_t dma;
3004 struct sk_buff *skb = first->skb;
3005 struct ixgbevf_tx_buffer *tx_buffer;
3006 union ixgbe_adv_tx_desc *tx_desc;
3007 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
3008 unsigned int data_len = skb->data_len;
3009 unsigned int size = skb_headlen(skb);
3010 unsigned int paylen = skb->len - hdr_len;
3011 u32 tx_flags = first->tx_flags;
3012 __le32 cmd_type;
3013 u16 i = tx_ring->next_to_use;
9bdfefd2 3014
29d37fa1 3015 tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
92915f71 3016
29d37fa1
ET		 3017 ixgbevf_tx_olinfo_status(tx_desc, tx_flags, paylen);
3018 cmd_type = ixgbevf_tx_cmd_type(tx_flags);
7ad1a093 3019
29d37fa1
ET		 3020 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
3021 if (dma_mapping_error(tx_ring->dev, dma))
3022 goto dma_error;
92915f71 3023
29d37fa1
ET		 3024 /* record length, and DMA address */
3025 dma_unmap_len_set(first, len, size);
3026 dma_unmap_addr_set(first, dma, dma);
92915f71 3027
29d37fa1 3028 tx_desc->read.buffer_addr = cpu_to_le64(dma);
92915f71 3029
29d37fa1
ET		 3030 for (;;) {
3031 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) {
3032 tx_desc->read.cmd_type_len =
3033 cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD);
92915f71 3034
29d37fa1
ET		 3035 i++;
3036 tx_desc++;
3037 if (i == tx_ring->count) {
3038 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
3039 i = 0;
3040 }
92915f71 3041
29d37fa1
ET		 3042 dma += IXGBE_MAX_DATA_PER_TXD;
3043 size -= IXGBE_MAX_DATA_PER_TXD;
92915f71 3044
29d37fa1
ET		 3045 tx_desc->read.buffer_addr = cpu_to_le64(dma);
3046 tx_desc->read.olinfo_status = 0;
3047 }
92915f71 3048
29d37fa1
ET		 3049 if (likely(!data_len))
3050 break;
92915f71 3051
29d37fa1 3052 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
92915f71 3053
29d37fa1
ET		 3054 i++;
3055 tx_desc++;
3056 if (i == tx_ring->count) {
3057 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
3058 i = 0;
3059 }
92915f71 3060
29d37fa1
ET		 3061 size = skb_frag_size(frag);
3062 data_len -= size;
92915f71 3063
29d37fa1
ET		 3064 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
3065 DMA_TO_DEVICE);
3066 if (dma_mapping_error(tx_ring->dev, dma))
3067 goto dma_error;
70a10e25 3068
29d37fa1
ET		 3069 tx_buffer = &tx_ring->tx_buffer_info[i];
3070 dma_unmap_len_set(tx_buffer, len, size);
3071 dma_unmap_addr_set(tx_buffer, dma, dma);
92915f71 3072
29d37fa1
ET		 3073 tx_desc->read.buffer_addr = cpu_to_le64(dma);
3074 tx_desc->read.olinfo_status = 0;
3075
3076 frag++;
70a10e25 3077 }
92915f71 3078
29d37fa1
ET		 3079 /* write last descriptor with RS and EOP bits */
3080 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD);
3081 tx_desc->read.cmd_type_len = cmd_type;
3082
3083 /* set the timestamp */
3084 first->time_stamp = jiffies;
3085
3086 /* Force memory writes to complete before letting h/w know there
3087 * are new descriptors to fetch. (Only applicable for weak-ordered
3088 * memory model archs, such as IA-64).
3089 *
3090 * We also need this memory barrier (wmb) to make certain all of the
3091 * status bits have been updated before next_to_watch is written.
70a10e25 3092 */
29d37fa1 3093 wmb();
92915f71 3094
29d37fa1
ET		 3095 /* set next_to_watch value indicating a packet is present */
3096 first->next_to_watch = tx_desc;
92915f71 3097
29d37fa1
ET		 3098 i++;
3099 if (i == tx_ring->count)
3100 i = 0;
9bdfefd2 3101
29d37fa1 3102 tx_ring->next_to_use = i;
92915f71 3103
29d37fa1 3104 /* notify HW of packet */
06380db6 3105 ixgbevf_write_tail(tx_ring, i);
29d37fa1
ET		 3106
3107 return;
3108dma_error:
3109 dev_err(tx_ring->dev, "TX DMA map failed\n");
3110
3111 /* clear dma mappings for failed tx_buffer_info map */
3112 for (;;) {
3113 tx_buffer = &tx_ring->tx_buffer_info[i];
3114 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer);
3115 if (tx_buffer == first)
3116 break;
3117 if (i == 0)
3118 i = tx_ring->count;
3119 i--;
3120 }
92915f71 3121
92915f71 3122 tx_ring->next_to_use = i;
92915f71
GR		 3123}
3124
fb40195c 3125static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
92915f71 3126{
fb40195c 3127 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
92915f71
GR		 3128 /* Herbert's original patch had:
3129 * smp_mb__after_netif_stop_queue();
3130 * but since that doesn't exist yet, just open code it. */
3131 smp_mb();
3132
3133 /* We need to check again in a case another CPU has just
3134 * made room available. */
f880d07b 3135 if (likely(ixgbevf_desc_unused(tx_ring) < size))
92915f71
GR		 3136 return -EBUSY;
3137
3138 /* A reprieve! - use start_queue because it doesn't call schedule */
fb40195c 3139 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
095e2617
ET		 3140 ++tx_ring->tx_stats.restart_queue;
3141
92915f71
GR		 3142 return 0;
3143}
3144
fb40195c 3145static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
92915f71 3146{
f880d07b 3147 if (likely(ixgbevf_desc_unused(tx_ring) >= size))
92915f71 3148 return 0;
fb40195c 3149 return __ixgbevf_maybe_stop_tx(tx_ring, size);
92915f71
GR		 3150}
3151
3152static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
3153{
3154 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
7ad1a093 3155 struct ixgbevf_tx_buffer *first;
92915f71 3156 struct ixgbevf_ring *tx_ring;
7ad1a093
ET		 3157 int tso;
3158 u32 tx_flags = 0;
3595990a
AD		 3159 u16 count = TXD_USE_COUNT(skb_headlen(skb));
3160#if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
3161 unsigned short f;
3162#endif
7ad1a093 3163 u8 hdr_len = 0;
f9d08f16 3164 u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL);
7ad1a093 3165
46acc460 3166 if (!dst_mac || is_link_local_ether_addr(dst_mac)) {
f9d08f16
GR		 3167 dev_kfree_skb(skb);
3168 return NETDEV_TX_OK;
3169 }
92915f71 3170
7ad1a093 3171 tx_ring = adapter->tx_ring[skb->queue_mapping];
92915f71 3172
3595990a
AD		 3173 /*
3174 * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
3175 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,
3176 * + 2 desc gap to keep tail from touching head,
3177 * + 1 desc for context descriptor,
3178 * otherwise try next time
3179 */
3180#if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
3181 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
3182 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
3183#else
3184 count += skb_shinfo(skb)->nr_frags;
3185#endif
fb40195c 3186 if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) {
095e2617 3187 tx_ring->tx_stats.tx_busy++;
3595990a
AD		 3188 return NETDEV_TX_BUSY;
3189 }
3190
7ad1a093
ET		 3191 /* record the location of the first descriptor for this packet */
3192 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
3193 first->skb = skb;
3194 first->bytecount = skb->len;
3195 first->gso_segs = 1;
3196
eab6d18d 3197 if (vlan_tx_tag_present(skb)) {
92915f71
GR		 3198 tx_flags |= vlan_tx_tag_get(skb);
3199 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
3200 tx_flags |= IXGBE_TX_FLAGS_VLAN;
3201 }
3202
7ad1a093
ET		 3203 /* record initial flags and protocol */
3204 first->tx_flags = tx_flags;
3205 first->protocol = vlan_get_protocol(skb);
92915f71 3206
7ad1a093
ET		 3207 tso = ixgbevf_tso(tx_ring, first, &hdr_len);
3208 if (tso < 0)
3209 goto out_drop;
b5d217f3 3210 else if (!tso)
7ad1a093 3211 ixgbevf_tx_csum(tx_ring, first);
92915f71 3212
29d37fa1 3213 ixgbevf_tx_map(tx_ring, first, hdr_len);
70a10e25 3214
fb40195c 3215 ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED);
92915f71 3216
7ad1a093
ET		 3217 return NETDEV_TX_OK;
3218
3219out_drop:
3220 dev_kfree_skb_any(first->skb);
3221 first->skb = NULL;
3222
92915f71
GR		 3223 return NETDEV_TX_OK;
3224}
3225
92915f71
GR		 3226/**
3227 * ixgbevf_set_mac - Change the Ethernet Address of the NIC
3228 * @netdev: network interface device structure
3229 * @p: pointer to an address structure
3230 *
3231 * Returns 0 on success, negative on failure
3232 **/
3233static int ixgbevf_set_mac(struct net_device *netdev, void *p)
3234{
3235 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3236 struct ixgbe_hw *hw = &adapter->hw;
3237 struct sockaddr *addr = p;
3238
3239 if (!is_valid_ether_addr(addr->sa_data))
3240 return -EADDRNOTAVAIL;
3241
3242 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3243 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3244
55fdd45b 3245 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 3246
92fe0bf7 3247 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
92915f71 3248
55fdd45b 3249 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 3250
92915f71
GR		 3251 return 0;
3252}
3253
3254/**
3255 * ixgbevf_change_mtu - Change the Maximum Transfer Unit
3256 * @netdev: network interface device structure
3257 * @new_mtu: new value for maximum frame size
3258 *
3259 * Returns 0 on success, negative on failure
3260 **/
3261static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu)
3262{
3263 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3264 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
69bfbec4 3265 int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE;
69bfbec4 3266
56e94095
AD		 3267 switch (adapter->hw.api_version) {
3268 case ixgbe_mbox_api_11:
69bfbec4 3269 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
56e94095
AD		 3270 break;
3271 default:
3272 if (adapter->hw.mac.type == ixgbe_mac_X540_vf)
3273 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
3274 break;
3275 }
92915f71
GR		 3276
3277 /* MTU < 68 is an error and causes problems on some kernels */
69bfbec4 3278 if ((new_mtu < 68) || (max_frame > max_possible_frame))
92915f71
GR		 3279 return -EINVAL;
3280
3281 hw_dbg(&adapter->hw, "changing MTU from %d to %d\n",
3282 netdev->mtu, new_mtu);
3283 /* must set new MTU before calling down or up */
3284 netdev->mtu = new_mtu;
3285
3286 if (netif_running(netdev))
3287 ixgbevf_reinit_locked(adapter);
3288
3289 return 0;
3290}
3291
0ac1e8ce 3292static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state)
92915f71
GR		 3293{
3294 struct net_device *netdev = pci_get_drvdata(pdev);
3295 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
0ac1e8ce
AD		 3296#ifdef CONFIG_PM
3297 int retval = 0;
3298#endif
92915f71
GR		 3299
3300 netif_device_detach(netdev);
3301
3302 if (netif_running(netdev)) {
0ac1e8ce 3303 rtnl_lock();
92915f71
GR		 3304 ixgbevf_down(adapter);
3305 ixgbevf_free_irq(adapter);
3306 ixgbevf_free_all_tx_resources(adapter);
3307 ixgbevf_free_all_rx_resources(adapter);
0ac1e8ce 3308 rtnl_unlock();
92915f71
GR		 3309 }
3310
0ac1e8ce 3311 ixgbevf_clear_interrupt_scheme(adapter);
92915f71 3312
0ac1e8ce
AD		 3313#ifdef CONFIG_PM
3314 retval = pci_save_state(pdev);
3315 if (retval)
3316 return retval;
92915f71 3317
0ac1e8ce 3318#endif
92915f71 3319 pci_disable_device(pdev);
0ac1e8ce
AD		 3320
3321 return 0;
3322}
3323
3324#ifdef CONFIG_PM
3325static int ixgbevf_resume(struct pci_dev *pdev)
3326{
27ae2967
WY		 3327 struct net_device *netdev = pci_get_drvdata(pdev);
3328 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
0ac1e8ce
AD		 3329 u32 err;
3330
0ac1e8ce
AD		 3331 pci_restore_state(pdev);
3332 /*
3333 * pci_restore_state clears dev->state_saved so call
3334 * pci_save_state to restore it.
3335 */
3336 pci_save_state(pdev);
3337
3338 err = pci_enable_device_mem(pdev);
3339 if (err) {
3340 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n");
3341 return err;
3342 }
3343 pci_set_master(pdev);
3344
798e381a
DS		 3345 ixgbevf_reset(adapter);
3346
0ac1e8ce
AD		 3347 rtnl_lock();
3348 err = ixgbevf_init_interrupt_scheme(adapter);
3349 rtnl_unlock();
3350 if (err) {
3351 dev_err(&pdev->dev, "Cannot initialize interrupts\n");
3352 return err;
3353 }
3354
0ac1e8ce
AD		 3355 if (netif_running(netdev)) {
3356 err = ixgbevf_open(netdev);
3357 if (err)
3358 return err;
3359 }
3360
3361 netif_device_attach(netdev);
3362
3363 return err;
3364}
3365
3366#endif /* CONFIG_PM */
3367static void ixgbevf_shutdown(struct pci_dev *pdev)
3368{
3369 ixgbevf_suspend(pdev, PMSG_SUSPEND);
92915f71
GR		 3370}
3371
4197aa7b
ED		 3372static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev,
3373 struct rtnl_link_stats64 *stats)
3374{
3375 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3376 unsigned int start;
3377 u64 bytes, packets;
3378 const struct ixgbevf_ring *ring;
3379 int i;
3380
3381 ixgbevf_update_stats(adapter);
3382
3383 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc;
3384
3385 for (i = 0; i < adapter->num_rx_queues; i++) {
87e70ab9 3386 ring = adapter->rx_ring[i];
4197aa7b 3387 do {
57a7744e 3388 start = u64_stats_fetch_begin_irq(&ring->syncp);
095e2617
ET		 3389 bytes = ring->stats.bytes;
3390 packets = ring->stats.packets;
57a7744e 3391 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
4197aa7b
ED		 3392 stats->rx_bytes += bytes;
3393 stats->rx_packets += packets;
3394 }
3395
3396 for (i = 0; i < adapter->num_tx_queues; i++) {
87e70ab9 3397 ring = adapter->tx_ring[i];
4197aa7b 3398 do {
57a7744e 3399 start = u64_stats_fetch_begin_irq(&ring->syncp);
095e2617
ET		 3400 bytes = ring->stats.bytes;
3401 packets = ring->stats.packets;
57a7744e 3402 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
4197aa7b
ED		 3403 stats->tx_bytes += bytes;
3404 stats->tx_packets += packets;
3405 }
3406
3407 return stats;
3408}
3409
0ac1e8ce 3410static const struct net_device_ops ixgbevf_netdev_ops = {
c12db769
SH		 3411 .ndo_open = ixgbevf_open,
3412 .ndo_stop = ixgbevf_close,
3413 .ndo_start_xmit = ixgbevf_xmit_frame,
3414 .ndo_set_rx_mode = ixgbevf_set_rx_mode,
4197aa7b 3415 .ndo_get_stats64 = ixgbevf_get_stats,
92915f71 3416 .ndo_validate_addr = eth_validate_addr,
c12db769
SH		 3417 .ndo_set_mac_address = ixgbevf_set_mac,
3418 .ndo_change_mtu = ixgbevf_change_mtu,
3419 .ndo_tx_timeout = ixgbevf_tx_timeout,
c12db769
SH		 3420 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid,
3421 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid,
c777cdfa
JK		 3422#ifdef CONFIG_NET_RX_BUSY_POLL
3423 .ndo_busy_poll = ixgbevf_busy_poll_recv,
3424#endif
92915f71 3425};
92915f71
GR		 3426
3427static void ixgbevf_assign_netdev_ops(struct net_device *dev)
3428{
0ac1e8ce 3429 dev->netdev_ops = &ixgbevf_netdev_ops;
92915f71
GR		 3430 ixgbevf_set_ethtool_ops(dev);
3431 dev->watchdog_timeo = 5 * HZ;
3432}
3433
3434/**
3435 * ixgbevf_probe - Device Initialization Routine
3436 * @pdev: PCI device information struct
3437 * @ent: entry in ixgbevf_pci_tbl
3438 *
3439 * Returns 0 on success, negative on failure
3440 *
3441 * ixgbevf_probe initializes an adapter identified by a pci_dev structure.
3442 * The OS initialization, configuring of the adapter private structure,
3443 * and a hardware reset occur.
3444 **/
1dd06ae8 3445static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
92915f71
GR		 3446{
3447 struct net_device *netdev;
3448 struct ixgbevf_adapter *adapter = NULL;
3449 struct ixgbe_hw *hw = NULL;
3450 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data];
3451 static int cards_found;
3452 int err, pci_using_dac;
3453
3454 err = pci_enable_device(pdev);
3455 if (err)
3456 return err;
3457
53567aa4 3458 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
92915f71
GR		 3459 pci_using_dac = 1;
3460 } else {
53567aa4 3461 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
92915f71 3462 if (err) {
53567aa4
RK		 3463 dev_err(&pdev->dev, "No usable DMA "
3464 "configuration, aborting\n");
3465 goto err_dma;
92915f71
GR		 3466 }
3467 pci_using_dac = 0;
3468 }
3469
3470 err = pci_request_regions(pdev, ixgbevf_driver_name);
3471 if (err) {
3472 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err);
3473 goto err_pci_reg;
3474 }
3475
3476 pci_set_master(pdev);
3477
92915f71
GR		 3478 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter),
3479 MAX_TX_QUEUES);
92915f71
GR		 3480 if (!netdev) {
3481 err = -ENOMEM;
3482 goto err_alloc_etherdev;
3483 }
3484
3485 SET_NETDEV_DEV(netdev, &pdev->dev);
3486
3487 pci_set_drvdata(pdev, netdev);
3488 adapter = netdev_priv(netdev);
3489
3490 adapter->netdev = netdev;
3491 adapter->pdev = pdev;
3492 hw = &adapter->hw;
3493 hw->back = adapter;
b3f4d599 3494 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
92915f71
GR		 3495
3496 /*
3497 * call save state here in standalone driver because it relies on
3498 * adapter struct to exist, and needs to call netdev_priv
3499 */
3500 pci_save_state(pdev);
3501
3502 hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
3503 pci_resource_len(pdev, 0));
dbf8b0d8 3504 adapter->io_addr = hw->hw_addr;
92915f71
GR		 3505 if (!hw->hw_addr) {
3506 err = -EIO;
3507 goto err_ioremap;
3508 }
3509
3510 ixgbevf_assign_netdev_ops(netdev);
3511
3512 adapter->bd_number = cards_found;
3513
3514 /* Setup hw api */
3515 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
3516 hw->mac.type = ii->mac;
3517
3518 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops,
f416dfc0 3519 sizeof(struct ixgbe_mbx_operations));
92915f71 3520
92915f71
GR		 3521 /* setup the private structure */
3522 err = ixgbevf_sw_init(adapter);
1a0d6ae5
DK		 3523 if (err)
3524 goto err_sw_init;
3525
3526 /* The HW MAC address was set and/or determined in sw_init */
1a0d6ae5
DK		 3527 if (!is_valid_ether_addr(netdev->dev_addr)) {
3528 pr_err("invalid MAC address\n");
3529 err = -EIO;
3530 goto err_sw_init;
3531 }
92915f71 3532
471a76de 3533 netdev->hw_features = NETIF_F_SG |
92915f71 3534 NETIF_F_IP_CSUM |
471a76de
MM		 3535 NETIF_F_IPV6_CSUM |
3536 NETIF_F_TSO |
3537 NETIF_F_TSO6 |
3538 NETIF_F_RXCSUM;
3539
3540 netdev->features = netdev->hw_features |
f646968f
PM		 3541 NETIF_F_HW_VLAN_CTAG_TX |
3542 NETIF_F_HW_VLAN_CTAG_RX |
3543 NETIF_F_HW_VLAN_CTAG_FILTER;
92915f71 3544
92915f71
GR		 3545 netdev->vlan_features |= NETIF_F_TSO;
3546 netdev->vlan_features |= NETIF_F_TSO6;
3547 netdev->vlan_features |= NETIF_F_IP_CSUM;
3bfacf96 3548 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
92915f71
GR		 3549 netdev->vlan_features |= NETIF_F_SG;
3550
3551 if (pci_using_dac)
3552 netdev->features |= NETIF_F_HIGHDMA;
3553
01789349
JP		 3554 netdev->priv_flags |= IFF_UNICAST_FLT;
3555
92915f71 3556 init_timer(&adapter->watchdog_timer);
c061b18d 3557 adapter->watchdog_timer.function = ixgbevf_watchdog;
92915f71
GR		 3558 adapter->watchdog_timer.data = (unsigned long)adapter;
3559
3560 INIT_WORK(&adapter->reset_task, ixgbevf_reset_task);
3561 INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task);
3562
3563 err = ixgbevf_init_interrupt_scheme(adapter);
3564 if (err)
3565 goto err_sw_init;
3566
92915f71
GR		 3567 strcpy(netdev->name, "eth%d");
3568
3569 err = register_netdev(netdev);
3570 if (err)
3571 goto err_register;
3572
5d426ad1
GR		 3573 netif_carrier_off(netdev);
3574
33bd9f60
GR		 3575 ixgbevf_init_last_counter_stats(adapter);
3576
92915f71 3577 /* print the MAC address */
f794e7ef 3578 hw_dbg(hw, "%pM\n", netdev->dev_addr);
92915f71
GR		 3579
3580 hw_dbg(hw, "MAC: %d\n", hw->mac.type);
3581
92915f71
GR		 3582 hw_dbg(hw, "Intel(R) 82599 Virtual Function\n");
3583 cards_found++;
3584 return 0;
3585
3586err_register:
0ac1e8ce 3587 ixgbevf_clear_interrupt_scheme(adapter);
92915f71
GR		 3588err_sw_init:
3589 ixgbevf_reset_interrupt_capability(adapter);
dbf8b0d8 3590 iounmap(adapter->io_addr);
92915f71
GR		 3591err_ioremap:
3592 free_netdev(netdev);
3593err_alloc_etherdev:
3594 pci_release_regions(pdev);
3595err_pci_reg:
3596err_dma:
3597 pci_disable_device(pdev);
3598 return err;
3599}
3600
3601/**
3602 * ixgbevf_remove - Device Removal Routine
3603 * @pdev: PCI device information struct
3604 *
3605 * ixgbevf_remove is called by the PCI subsystem to alert the driver
3606 * that it should release a PCI device. The could be caused by a
3607 * Hot-Plug event, or because the driver is going to be removed from
3608 * memory.
3609 **/
9f9a12f8 3610static void ixgbevf_remove(struct pci_dev *pdev)
92915f71
GR		 3611{
3612 struct net_device *netdev = pci_get_drvdata(pdev);
3613 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3614
2e7cfbdd 3615 set_bit(__IXGBEVF_REMOVING, &adapter->state);
92915f71
GR		 3616
3617 del_timer_sync(&adapter->watchdog_timer);
3618
23f333a2 3619 cancel_work_sync(&adapter->reset_task);
92915f71
GR		 3620 cancel_work_sync(&adapter->watchdog_task);
3621
fd13a9ab 3622 if (netdev->reg_state == NETREG_REGISTERED)
92915f71 3623 unregister_netdev(netdev);
92915f71 3624
0ac1e8ce 3625 ixgbevf_clear_interrupt_scheme(adapter);
92915f71
GR		 3626 ixgbevf_reset_interrupt_capability(adapter);
3627
dbf8b0d8 3628 iounmap(adapter->io_addr);
92915f71
GR		 3629 pci_release_regions(pdev);
3630
3631 hw_dbg(&adapter->hw, "Remove complete\n");
3632
92915f71
GR		 3633 free_netdev(netdev);
3634
3635 pci_disable_device(pdev);
3636}
3637
9f19f31d
AD		 3638/**
3639 * ixgbevf_io_error_detected - called when PCI error is detected
3640 * @pdev: Pointer to PCI device
3641 * @state: The current pci connection state
3642 *
3643 * This function is called after a PCI bus error affecting
3644 * this device has been detected.
3645 */
3646static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev,
3647 pci_channel_state_t state)
3648{
3649 struct net_device *netdev = pci_get_drvdata(pdev);
3650 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3651
3652 netif_device_detach(netdev);
3653
3654 if (state == pci_channel_io_perm_failure)
3655 return PCI_ERS_RESULT_DISCONNECT;
3656
3657 if (netif_running(netdev))
3658 ixgbevf_down(adapter);
3659
3660 pci_disable_device(pdev);
3661
3662 /* Request a slot slot reset. */
3663 return PCI_ERS_RESULT_NEED_RESET;
3664}
3665
3666/**
3667 * ixgbevf_io_slot_reset - called after the pci bus has been reset.
3668 * @pdev: Pointer to PCI device
3669 *
3670 * Restart the card from scratch, as if from a cold-boot. Implementation
3671 * resembles the first-half of the ixgbevf_resume routine.
3672 */
3673static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev)
3674{
3675 struct net_device *netdev = pci_get_drvdata(pdev);
3676 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3677
3678 if (pci_enable_device_mem(pdev)) {
3679 dev_err(&pdev->dev,
3680 "Cannot re-enable PCI device after reset.\n");
3681 return PCI_ERS_RESULT_DISCONNECT;
3682 }
3683
3684 pci_set_master(pdev);
3685
3686 ixgbevf_reset(adapter);
3687
3688 return PCI_ERS_RESULT_RECOVERED;
3689}
3690
3691/**
3692 * ixgbevf_io_resume - called when traffic can start flowing again.
3693 * @pdev: Pointer to PCI device
3694 *
3695 * This callback is called when the error recovery driver tells us that
3696 * its OK to resume normal operation. Implementation resembles the
3697 * second-half of the ixgbevf_resume routine.
3698 */
3699static void ixgbevf_io_resume(struct pci_dev *pdev)
3700{
3701 struct net_device *netdev = pci_get_drvdata(pdev);
3702 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3703
3704 if (netif_running(netdev))
3705 ixgbevf_up(adapter);
3706
3707 netif_device_attach(netdev);
3708}
3709
3710/* PCI Error Recovery (ERS) */
3646f0e5 3711static const struct pci_error_handlers ixgbevf_err_handler = {
9f19f31d
AD		 3712 .error_detected = ixgbevf_io_error_detected,
3713 .slot_reset = ixgbevf_io_slot_reset,
3714 .resume = ixgbevf_io_resume,
3715};
3716
92915f71
GR		 3717static struct pci_driver ixgbevf_driver = {
3718 .name = ixgbevf_driver_name,
3719 .id_table = ixgbevf_pci_tbl,
3720 .probe = ixgbevf_probe,
9f9a12f8 3721 .remove = ixgbevf_remove,
0ac1e8ce
AD		 3722#ifdef CONFIG_PM
3723 /* Power Management Hooks */
3724 .suspend = ixgbevf_suspend,
3725 .resume = ixgbevf_resume,
3726#endif
92915f71 3727 .shutdown = ixgbevf_shutdown,
9f19f31d 3728 .err_handler = &ixgbevf_err_handler
92915f71
GR		 3729};
3730
3731/**
65d676c8 3732 * ixgbevf_init_module - Driver Registration Routine
92915f71 3733 *
65d676c8 3734 * ixgbevf_init_module is the first routine called when the driver is
92915f71
GR		 3735 * loaded. All it does is register with the PCI subsystem.
3736 **/
3737static int __init ixgbevf_init_module(void)
3738{
3739 int ret;
dbd9636e
JK		 3740 pr_info("%s - version %s\n", ixgbevf_driver_string,
3741 ixgbevf_driver_version);
92915f71 3742
dbd9636e 3743 pr_info("%s\n", ixgbevf_copyright);
92915f71
GR		 3744
3745 ret = pci_register_driver(&ixgbevf_driver);
3746 return ret;
3747}
3748
3749module_init(ixgbevf_init_module);
3750
3751/**
65d676c8 3752 * ixgbevf_exit_module - Driver Exit Cleanup Routine
92915f71 3753 *
65d676c8 3754 * ixgbevf_exit_module is called just before the driver is removed
92915f71
GR		 3755 * from memory.
3756 **/
3757static void __exit ixgbevf_exit_module(void)
3758{
3759 pci_unregister_driver(&ixgbevf_driver);
3760}
3761
3762#ifdef DEBUG
3763/**
65d676c8 3764 * ixgbevf_get_hw_dev_name - return device name string
92915f71
GR		 3765 * used by hardware layer to print debugging information
3766 **/
3767char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw)
3768{
3769 struct ixgbevf_adapter *adapter = hw->back;
3770 return adapter->netdev->name;
3771}
3772
3773#endif
3774module_exit(ixgbevf_exit_module);
3775
3776/* ixgbevf_main.c */
ubuntu-zesty-kernel mirror