]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ixgbevf: Indicate removal state explicitly
[mirror_ubuntu-artful-kernel.git] / drivers / net / ethernet / intel / ixgbevf / ixgbevf_main.c
CommitLineData
92915f71
GR		 1/*******************************************************************************
2
3 Intel 82599 Virtual Function driver
2e7cfbdd 4 Copyright(c) 1999 - 2014 Intel Corporation.
92915f71
GR		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28
29/******************************************************************************
30 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code
31******************************************************************************/
dbd9636e
JK		 32
33#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
34
92915f71 35#include <linux/types.h>
dadcd65f 36#include <linux/bitops.h>
92915f71
GR		 37#include <linux/module.h>
38#include <linux/pci.h>
39#include <linux/netdevice.h>
40#include <linux/vmalloc.h>
41#include <linux/string.h>
42#include <linux/in.h>
43#include <linux/ip.h>
44#include <linux/tcp.h>
70a10e25 45#include <linux/sctp.h>
92915f71 46#include <linux/ipv6.h>
5a0e3ad6 47#include <linux/slab.h>
92915f71
GR		 48#include <net/checksum.h>
49#include <net/ip6_checksum.h>
50#include <linux/ethtool.h>
01789349 51#include <linux/if.h>
92915f71 52#include <linux/if_vlan.h>
70c71606 53#include <linux/prefetch.h>
92915f71
GR		 54
55#include "ixgbevf.h"
56
3d8fe98f 57const char ixgbevf_driver_name[] = "ixgbevf";
92915f71 58static const char ixgbevf_driver_string[] =
422e05d1 59 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver";
92915f71 60
86f359f6 61#define DRV_VERSION "2.12.1-k"
92915f71 62const char ixgbevf_driver_version[] = DRV_VERSION;
66c87bd5 63static char ixgbevf_copyright[] =
5c47a2b6 64 "Copyright (c) 2009 - 2012 Intel Corporation.";
92915f71
GR		 65
66static const struct ixgbevf_info *ixgbevf_info_tbl[] = {
2316aa2a
GR		 67 [board_82599_vf] = &ixgbevf_82599_vf_info,
68 [board_X540_vf] = &ixgbevf_X540_vf_info,
92915f71
GR		 69};
70
71/* ixgbevf_pci_tbl - PCI Device ID Table
72 *
73 * Wildcard entries (PCI_ANY_ID) should come last
74 * Last entry must be all 0s
75 *
76 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
77 * Class, Class Mask, private data (not used) }
78 */
39ba22b4
SH		 79static DEFINE_PCI_DEVICE_TABLE(ixgbevf_pci_tbl) = {
80 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf },
81 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf },
92915f71
GR		 82 /* required last entry */
83 {0, }
84};
85MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl);
86
87MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
88MODULE_DESCRIPTION("Intel(R) 82599 Virtual Function Driver");
89MODULE_LICENSE("GPL");
90MODULE_VERSION(DRV_VERSION);
91
b3f4d599 92#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
93static int debug = -1;
94module_param(debug, int, 0);
95MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
92915f71
GR		 96
97/* forward decls */
220fe050 98static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter);
fa71ae27 99static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector);
56e94095 100static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter);
92915f71 101
5cdab2f6 102static inline void ixgbevf_release_rx_desc(struct ixgbevf_ring *rx_ring,
92915f71
GR		 103 u32 val)
104{
5cdab2f6
DS		 105 rx_ring->next_to_use = val;
106
92915f71
GR		 107 /*
108 * Force memory writes to complete before letting h/w
109 * know there are new descriptors to fetch. (Only
110 * applicable for weak-ordered memory model archs,
111 * such as IA-64).
112 */
113 wmb();
5cdab2f6 114 writel(val, rx_ring->tail);
92915f71
GR		 115}
116
49ce9c2c 117/**
65d676c8 118 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors
92915f71
GR		 119 * @adapter: pointer to adapter struct
120 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
121 * @queue: queue to map the corresponding interrupt to
122 * @msix_vector: the vector to map to the corresponding queue
92915f71
GR		 123 */
124static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
125 u8 queue, u8 msix_vector)
126{
127 u32 ivar, index;
128 struct ixgbe_hw *hw = &adapter->hw;
129 if (direction == -1) {
130 /* other causes */
131 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
132 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
133 ivar &= ~0xFF;
134 ivar |= msix_vector;
135 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
136 } else {
137 /* tx or rx causes */
138 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
139 index = ((16 * (queue & 1)) + (8 * direction));
140 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
141 ivar &= ~(0xFF << index);
142 ivar |= (msix_vector << index);
143 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar);
144 }
145}
146
70a10e25 147static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring,
9bdfefd2
ET		 148 struct ixgbevf_tx_buffer *tx_buffer)
149{
150 if (tx_buffer->skb) {
151 dev_kfree_skb_any(tx_buffer->skb);
152 if (dma_unmap_len(tx_buffer, len))
70a10e25 153 dma_unmap_single(tx_ring->dev,
9bdfefd2
ET		 154 dma_unmap_addr(tx_buffer, dma),
155 dma_unmap_len(tx_buffer, len),
2a1f8794 156 DMA_TO_DEVICE);
9bdfefd2
ET		 157 } else if (dma_unmap_len(tx_buffer, len)) {
158 dma_unmap_page(tx_ring->dev,
159 dma_unmap_addr(tx_buffer, dma),
160 dma_unmap_len(tx_buffer, len),
161 DMA_TO_DEVICE);
92915f71 162 }
9bdfefd2
ET		 163 tx_buffer->next_to_watch = NULL;
164 tx_buffer->skb = NULL;
165 dma_unmap_len_set(tx_buffer, len, 0);
166 /* tx_buffer must be completely set up in the transmit path */
92915f71
GR		 167}
168
92915f71
GR		 169#define IXGBE_MAX_TXD_PWR 14
170#define IXGBE_MAX_DATA_PER_TXD (1 << IXGBE_MAX_TXD_PWR)
171
172/* Tx Descriptors needed, worst case */
3595990a
AD		 173#define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD)
174#define DESC_NEEDED (MAX_SKB_FRAGS + 4)
92915f71
GR		 175
176static void ixgbevf_tx_timeout(struct net_device *netdev);
177
178/**
179 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes
fa71ae27 180 * @q_vector: board private structure
92915f71
GR		 181 * @tx_ring: tx ring to clean
182 **/
fa71ae27 183static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
92915f71
GR		 184 struct ixgbevf_ring *tx_ring)
185{
fa71ae27 186 struct ixgbevf_adapter *adapter = q_vector->adapter;
7ad1a093
ET		 187 struct ixgbevf_tx_buffer *tx_buffer;
188 union ixgbe_adv_tx_desc *tx_desc;
92915f71 189 unsigned int total_bytes = 0, total_packets = 0;
7ad1a093
ET		 190 unsigned int budget = tx_ring->count / 2;
191 unsigned int i = tx_ring->next_to_clean;
92915f71 192
10cc1bdd
AD		 193 if (test_bit(__IXGBEVF_DOWN, &adapter->state))
194 return true;
195
7ad1a093
ET		 196 tx_buffer = &tx_ring->tx_buffer_info[i];
197 tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
198 i -= tx_ring->count;
92915f71 199
e757e3e1 200 do {
7ad1a093 201 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
e757e3e1
AD		 202
203 /* if next_to_watch is not set then there is no work pending */
204 if (!eop_desc)
205 break;
206
207 /* prevent any other reads prior to eop_desc */
208 read_barrier_depends();
209
210 /* if DD is not set pending work has not been completed */
211 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
212 break;
213
214 /* clear next_to_watch to prevent false hangs */
7ad1a093 215 tx_buffer->next_to_watch = NULL;
e757e3e1 216
7ad1a093
ET		 217 /* update the statistics for this packet */
218 total_bytes += tx_buffer->bytecount;
219 total_packets += tx_buffer->gso_segs;
92915f71 220
9bdfefd2
ET		 221 /* free the skb */
222 dev_kfree_skb_any(tx_buffer->skb);
223
224 /* unmap skb header data */
225 dma_unmap_single(tx_ring->dev,
226 dma_unmap_addr(tx_buffer, dma),
227 dma_unmap_len(tx_buffer, len),
228 DMA_TO_DEVICE);
229
7ad1a093 230 /* clear tx_buffer data */
9bdfefd2
ET		 231 tx_buffer->skb = NULL;
232 dma_unmap_len_set(tx_buffer, len, 0);
92915f71 233
7ad1a093
ET		 234 /* unmap remaining buffers */
235 while (tx_desc != eop_desc) {
7ad1a093
ET		 236 tx_buffer++;
237 tx_desc++;
92915f71 238 i++;
7ad1a093
ET		 239 if (unlikely(!i)) {
240 i -= tx_ring->count;
241 tx_buffer = tx_ring->tx_buffer_info;
242 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
243 }
e757e3e1 244
9bdfefd2
ET		 245 /* unmap any remaining paged data */
246 if (dma_unmap_len(tx_buffer, len)) {
247 dma_unmap_page(tx_ring->dev,
248 dma_unmap_addr(tx_buffer, dma),
249 dma_unmap_len(tx_buffer, len),
250 DMA_TO_DEVICE);
251 dma_unmap_len_set(tx_buffer, len, 0);
252 }
92915f71
GR		 253 }
254
7ad1a093
ET		 255 /* move us one more past the eop_desc for start of next pkt */
256 tx_buffer++;
257 tx_desc++;
258 i++;
259 if (unlikely(!i)) {
260 i -= tx_ring->count;
261 tx_buffer = tx_ring->tx_buffer_info;
262 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
263 }
264
265 /* issue prefetch for next Tx descriptor */
266 prefetch(tx_desc);
267
268 /* update budget accounting */
269 budget--;
270 } while (likely(budget));
271
272 i += tx_ring->count;
92915f71 273 tx_ring->next_to_clean = i;
7ad1a093
ET		 274 u64_stats_update_begin(&tx_ring->syncp);
275 tx_ring->stats.bytes += total_bytes;
276 tx_ring->stats.packets += total_packets;
277 u64_stats_update_end(&tx_ring->syncp);
278 q_vector->tx.total_bytes += total_bytes;
279 q_vector->tx.total_packets += total_packets;
92915f71
GR		 280
281#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
7ad1a093 282 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
f880d07b 283 (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
92915f71
GR		 284 /* Make sure that anybody stopping the queue after this
285 * sees the new next_to_clean.
286 */
287 smp_mb();
7ad1a093 288
fb40195c
AD		 289 if (__netif_subqueue_stopped(tx_ring->netdev,
290 tx_ring->queue_index) &&
92915f71 291 !test_bit(__IXGBEVF_DOWN, &adapter->state)) {
fb40195c
AD		 292 netif_wake_subqueue(tx_ring->netdev,
293 tx_ring->queue_index);
7ad1a093 294 ++tx_ring->tx_stats.restart_queue;
92915f71 295 }
92915f71
GR		 296 }
297
7ad1a093 298 return !!budget;
92915f71
GR		 299}
300
301/**
302 * ixgbevf_receive_skb - Send a completed packet up the stack
303 * @q_vector: structure containing interrupt and ring information
304 * @skb: packet to send up
305 * @status: hardware indication of status of receive
92915f71
GR		 306 * @rx_desc: rx descriptor
307 **/
308static void ixgbevf_receive_skb(struct ixgbevf_q_vector *q_vector,
309 struct sk_buff *skb, u8 status,
92915f71
GR		 310 union ixgbe_adv_rx_desc *rx_desc)
311{
312 struct ixgbevf_adapter *adapter = q_vector->adapter;
313 bool is_vlan = (status & IXGBE_RXD_STAT_VP);
dd1ed3b7 314 u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
92915f71 315
5d9a533b 316 if (is_vlan && test_bit(tag & VLAN_VID_MASK, adapter->active_vlans))
86a9bad3 317 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tag);
dadcd65f 318
366c1099
GR		 319 if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
320 napi_gro_receive(&q_vector->napi, skb);
321 else
322 netif_rx(skb);
92915f71
GR		 323}
324
08681618
JK		 325/**
326 * ixgbevf_rx_skb - Helper function to determine proper Rx method
327 * @q_vector: structure containing interrupt and ring information
328 * @skb: packet to send up
329 * @status: hardware indication of status of receive
330 * @rx_desc: rx descriptor
331 **/
332static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector,
333 struct sk_buff *skb, u8 status,
334 union ixgbe_adv_rx_desc *rx_desc)
335{
c777cdfa
JK		 336#ifdef CONFIG_NET_RX_BUSY_POLL
337 skb_mark_napi_id(skb, &q_vector->napi);
338
339 if (ixgbevf_qv_busy_polling(q_vector)) {
340 netif_receive_skb(skb);
341 /* exit early if we busy polled */
342 return;
343 }
344#endif /* CONFIG_NET_RX_BUSY_POLL */
345
08681618
JK		 346 ixgbevf_receive_skb(q_vector, skb, status, rx_desc);
347}
348
92915f71
GR		 349/**
350 * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
55fb277c 351 * @ring: pointer to Rx descriptor ring structure
92915f71
GR		 352 * @status_err: hardware indication of status of receive
353 * @skb: skb currently being received and modified
354 **/
55fb277c 355static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring,
92915f71
GR		 356 u32 status_err, struct sk_buff *skb)
357{
bc8acf2c 358 skb_checksum_none_assert(skb);
92915f71
GR		 359
360 /* Rx csum disabled */
fb40195c 361 if (!(ring->netdev->features & NETIF_F_RXCSUM))
92915f71
GR		 362 return;
363
364 /* if IP and error */
365 if ((status_err & IXGBE_RXD_STAT_IPCS) &&
366 (status_err & IXGBE_RXDADV_ERR_IPE)) {
095e2617 367 ring->rx_stats.csum_err++;
92915f71
GR		 368 return;
369 }
370
371 if (!(status_err & IXGBE_RXD_STAT_L4CS))
372 return;
373
374 if (status_err & IXGBE_RXDADV_ERR_TCPE) {
095e2617 375 ring->rx_stats.csum_err++;
92915f71
GR		 376 return;
377 }
378
379 /* It must be a TCP or UDP packet with a valid checksum */
380 skb->ip_summed = CHECKSUM_UNNECESSARY;
92915f71
GR		 381}
382
383/**
384 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
095e2617 385 * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on
92915f71 386 **/
095e2617 387static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring,
92915f71
GR		 388 int cleaned_count)
389{
92915f71
GR		 390 union ixgbe_adv_rx_desc *rx_desc;
391 struct ixgbevf_rx_buffer *bi;
fb40195c 392 unsigned int i = rx_ring->next_to_use;
92915f71 393
92915f71 394 while (cleaned_count--) {
908421f6 395 rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
05d063aa 396 bi = &rx_ring->rx_buffer_info[i];
b9dd245b
GR		 397
398 if (!bi->skb) {
399 struct sk_buff *skb;
400
fb40195c
AD		 401 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
402 rx_ring->rx_buf_len);
05d063aa 403 if (!skb)
92915f71 404 goto no_buffers;
05d063aa 405
92915f71 406 bi->skb = skb;
b9dd245b 407
05d063aa 408 bi->dma = dma_map_single(rx_ring->dev, skb->data,
92915f71 409 rx_ring->rx_buf_len,
2a1f8794 410 DMA_FROM_DEVICE);
05d063aa 411 if (dma_mapping_error(rx_ring->dev, bi->dma)) {
6132ee8a
GR		 412 dev_kfree_skb(skb);
413 bi->skb = NULL;
05d063aa 414 dev_err(rx_ring->dev, "Rx DMA map failed\n");
6132ee8a
GR		 415 break;
416 }
92915f71 417 }
77d5dfca 418 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
92915f71
GR		 419
420 i++;
421 if (i == rx_ring->count)
422 i = 0;
92915f71
GR		 423 }
424
425no_buffers:
095e2617 426 rx_ring->rx_stats.alloc_rx_buff_failed++;
5cdab2f6
DS		 427 if (rx_ring->next_to_use != i)
428 ixgbevf_release_rx_desc(rx_ring, i);
92915f71
GR		 429}
430
431static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
5f3600eb 432 u32 qmask)
92915f71 433{
92915f71
GR		 434 struct ixgbe_hw *hw = &adapter->hw;
435
5f3600eb 436 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask);
92915f71
GR		 437}
438
08e50a20
JK		 439static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
440 struct ixgbevf_ring *rx_ring,
441 int budget)
92915f71 442{
92915f71
GR		 443 union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
444 struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer;
445 struct sk_buff *skb;
446 unsigned int i;
447 u32 len, staterr;
92915f71
GR		 448 int cleaned_count = 0;
449 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
450
451 i = rx_ring->next_to_clean;
908421f6 452 rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
92915f71
GR		 453 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
454 rx_buffer_info = &rx_ring->rx_buffer_info[i];
455
456 while (staterr & IXGBE_RXD_STAT_DD) {
fa71ae27 457 if (!budget)
92915f71 458 break;
fa71ae27 459 budget--;
92915f71 460
2d0bb1c1 461 rmb(); /* read descriptor and rx_buffer_info after status DD */
77d5dfca 462 len = le16_to_cpu(rx_desc->wb.upper.length);
92915f71
GR		 463 skb = rx_buffer_info->skb;
464 prefetch(skb->data - NET_IP_ALIGN);
465 rx_buffer_info->skb = NULL;
466
467 if (rx_buffer_info->dma) {
05d063aa 468 dma_unmap_single(rx_ring->dev, rx_buffer_info->dma,
92915f71 469 rx_ring->rx_buf_len,
2a1f8794 470 DMA_FROM_DEVICE);
92915f71
GR		 471 rx_buffer_info->dma = 0;
472 skb_put(skb, len);
473 }
474
92915f71
GR		 475 i++;
476 if (i == rx_ring->count)
477 i = 0;
478
908421f6 479 next_rxd = IXGBEVF_RX_DESC(rx_ring, i);
92915f71
GR		 480 prefetch(next_rxd);
481 cleaned_count++;
482
483 next_buffer = &rx_ring->rx_buffer_info[i];
484
485 if (!(staterr & IXGBE_RXD_STAT_EOP)) {
77d5dfca 486 skb->next = next_buffer->skb;
5c60f81a 487 IXGBE_CB(skb->next)->prev = skb;
095e2617 488 rx_ring->rx_stats.non_eop_descs++;
92915f71
GR		 489 goto next_desc;
490 }
491
5c60f81a
AD		 492 /* we should not be chaining buffers, if we did drop the skb */
493 if (IXGBE_CB(skb)->prev) {
494 do {
495 struct sk_buff *this = skb;
496 skb = IXGBE_CB(skb)->prev;
497 dev_kfree_skb(this);
498 } while (skb);
499 goto next_desc;
500 }
501
92915f71
GR		 502 /* ERR_MASK will only have valid bits if EOP set */
503 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) {
504 dev_kfree_skb_irq(skb);
505 goto next_desc;
506 }
507
55fb277c 508 ixgbevf_rx_checksum(rx_ring, staterr, skb);
92915f71
GR		 509
510 /* probably a little skewed due to removing CRC */
511 total_rx_bytes += skb->len;
512 total_rx_packets++;
513
fb40195c 514 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
92915f71 515
815cccbf
JF		 516 /* Workaround hardware that can't do proper VEPA multicast
517 * source pruning.
518 */
bd9d5592
FF		 519 if ((skb->pkt_type == PACKET_BROADCAST ||
520 skb->pkt_type == PACKET_MULTICAST) &&
095e2617 521 ether_addr_equal(rx_ring->netdev->dev_addr,
7367d0b5 522 eth_hdr(skb)->h_source)) {
815cccbf
JF		 523 dev_kfree_skb_irq(skb);
524 goto next_desc;
525 }
526
08681618 527 ixgbevf_rx_skb(q_vector, skb, staterr, rx_desc);
92915f71
GR		 528
529next_desc:
530 rx_desc->wb.upper.status_error = 0;
531
532 /* return some buffers to hardware, one at a time is too slow */
533 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
095e2617 534 ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
92915f71
GR		 535 cleaned_count = 0;
536 }
537
538 /* use prefetched values */
539 rx_desc = next_rxd;
540 rx_buffer_info = &rx_ring->rx_buffer_info[i];
541
542 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
543 }
544
545 rx_ring->next_to_clean = i;
f880d07b 546 cleaned_count = ixgbevf_desc_unused(rx_ring);
92915f71
GR		 547
548 if (cleaned_count)
095e2617 549 ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
92915f71 550
4197aa7b 551 u64_stats_update_begin(&rx_ring->syncp);
095e2617
ET		 552 rx_ring->stats.packets += total_rx_packets;
553 rx_ring->stats.bytes += total_rx_bytes;
4197aa7b 554 u64_stats_update_end(&rx_ring->syncp);
ac6ed8f0
GR		 555 q_vector->rx.total_packets += total_rx_packets;
556 q_vector->rx.total_bytes += total_rx_bytes;
92915f71 557
08e50a20 558 return total_rx_packets;
92915f71
GR		 559}
560
561/**
fa71ae27 562 * ixgbevf_poll - NAPI polling calback
92915f71
GR		 563 * @napi: napi struct with our devices info in it
564 * @budget: amount of work driver is allowed to do this pass, in packets
565 *
fa71ae27 566 * This function will clean more than one or more rings associated with a
92915f71
GR		 567 * q_vector.
568 **/
fa71ae27 569static int ixgbevf_poll(struct napi_struct *napi, int budget)
92915f71
GR		 570{
571 struct ixgbevf_q_vector *q_vector =
572 container_of(napi, struct ixgbevf_q_vector, napi);
573 struct ixgbevf_adapter *adapter = q_vector->adapter;
fa71ae27
AD		 574 struct ixgbevf_ring *ring;
575 int per_ring_budget;
576 bool clean_complete = true;
577
578 ixgbevf_for_each_ring(ring, q_vector->tx)
579 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring);
92915f71 580
c777cdfa
JK		 581#ifdef CONFIG_NET_RX_BUSY_POLL
582 if (!ixgbevf_qv_lock_napi(q_vector))
583 return budget;
584#endif
585
92915f71
GR		 586 /* attempt to distribute budget to each queue fairly, but don't allow
587 * the budget to go below 1 because we'll exit polling */
fa71ae27
AD		 588 if (q_vector->rx.count > 1)
589 per_ring_budget = max(budget/q_vector->rx.count, 1);
590 else
591 per_ring_budget = budget;
592
366c1099 593 adapter->flags |= IXGBE_FLAG_IN_NETPOLL;
fa71ae27 594 ixgbevf_for_each_ring(ring, q_vector->rx)
08e50a20
JK		 595 clean_complete &= (ixgbevf_clean_rx_irq(q_vector, ring,
596 per_ring_budget)
597 < per_ring_budget);
366c1099 598 adapter->flags &= ~IXGBE_FLAG_IN_NETPOLL;
fa71ae27 599
c777cdfa
JK		 600#ifdef CONFIG_NET_RX_BUSY_POLL
601 ixgbevf_qv_unlock_napi(q_vector);
602#endif
603
fa71ae27
AD		 604 /* If all work not completed, return budget and keep polling */
605 if (!clean_complete)
606 return budget;
607 /* all work done, exit the polling mode */
608 napi_complete(napi);
609 if (adapter->rx_itr_setting & 1)
610 ixgbevf_set_itr(q_vector);
2e7cfbdd
MR		 611 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
612 !test_bit(__IXGBEVF_REMOVING, &adapter->state))
fa71ae27
AD		 613 ixgbevf_irq_enable_queues(adapter,
614 1 << q_vector->v_idx);
92915f71 615
fa71ae27 616 return 0;
92915f71
GR		 617}
618
ce422606
GR		 619/**
620 * ixgbevf_write_eitr - write VTEITR register in hardware specific way
621 * @q_vector: structure containing interrupt and ring information
622 */
3849623e 623void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector)
ce422606
GR		 624{
625 struct ixgbevf_adapter *adapter = q_vector->adapter;
626 struct ixgbe_hw *hw = &adapter->hw;
627 int v_idx = q_vector->v_idx;
628 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
629
630 /*
631 * set the WDIS bit to not clear the timer bits and cause an
632 * immediate assertion of the interrupt
633 */
634 itr_reg |= IXGBE_EITR_CNT_WDIS;
635
636 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg);
637}
92915f71 638
c777cdfa
JK		 639#ifdef CONFIG_NET_RX_BUSY_POLL
640/* must be called with local_bh_disable()d */
641static int ixgbevf_busy_poll_recv(struct napi_struct *napi)
642{
643 struct ixgbevf_q_vector *q_vector =
644 container_of(napi, struct ixgbevf_q_vector, napi);
645 struct ixgbevf_adapter *adapter = q_vector->adapter;
646 struct ixgbevf_ring *ring;
647 int found = 0;
648
649 if (test_bit(__IXGBEVF_DOWN, &adapter->state))
650 return LL_FLUSH_FAILED;
651
652 if (!ixgbevf_qv_lock_poll(q_vector))
653 return LL_FLUSH_BUSY;
654
655 ixgbevf_for_each_ring(ring, q_vector->rx) {
656 found = ixgbevf_clean_rx_irq(q_vector, ring, 4);
3b5dca26
JK		 657#ifdef BP_EXTENDED_STATS
658 if (found)
095e2617 659 ring->stats.cleaned += found;
3b5dca26 660 else
095e2617 661 ring->stats.misses++;
3b5dca26 662#endif
c777cdfa
JK		 663 if (found)
664 break;
665 }
666
667 ixgbevf_qv_unlock_poll(q_vector);
668
669 return found;
670}
671#endif /* CONFIG_NET_RX_BUSY_POLL */
672
92915f71
GR		 673/**
674 * ixgbevf_configure_msix - Configure MSI-X hardware
675 * @adapter: board private structure
676 *
677 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X
678 * interrupts.
679 **/
680static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter)
681{
682 struct ixgbevf_q_vector *q_vector;
6b43c446 683 int q_vectors, v_idx;
92915f71
GR		 684
685 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
5f3600eb 686 adapter->eims_enable_mask = 0;
92915f71
GR		 687
688 /*
689 * Populate the IVAR table and set the ITR values to the
690 * corresponding register.
691 */
692 for (v_idx = 0; v_idx < q_vectors; v_idx++) {
6b43c446 693 struct ixgbevf_ring *ring;
92915f71 694 q_vector = adapter->q_vector[v_idx];
6b43c446
AD		 695
696 ixgbevf_for_each_ring(ring, q_vector->rx)
697 ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx);
698
699 ixgbevf_for_each_ring(ring, q_vector->tx)
700 ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx);
92915f71 701
5f3600eb
AD		 702 if (q_vector->tx.ring && !q_vector->rx.ring) {
703 /* tx only vector */
704 if (adapter->tx_itr_setting == 1)
705 q_vector->itr = IXGBE_10K_ITR;
706 else
707 q_vector->itr = adapter->tx_itr_setting;
708 } else {
709 /* rx or rx/tx vector */
710 if (adapter->rx_itr_setting == 1)
711 q_vector->itr = IXGBE_20K_ITR;
712 else
713 q_vector->itr = adapter->rx_itr_setting;
714 }
715
716 /* add q_vector eims value to global eims_enable_mask */
717 adapter->eims_enable_mask |= 1 << v_idx;
92915f71 718
5f3600eb 719 ixgbevf_write_eitr(q_vector);
92915f71
GR		 720 }
721
722 ixgbevf_set_ivar(adapter, -1, 1, v_idx);
5f3600eb
AD		 723 /* setup eims_other and add value to global eims_enable_mask */
724 adapter->eims_other = 1 << v_idx;
725 adapter->eims_enable_mask |= adapter->eims_other;
92915f71
GR		 726}
727
728enum latency_range {
729 lowest_latency = 0,
730 low_latency = 1,
731 bulk_latency = 2,
732 latency_invalid = 255
733};
734
735/**
736 * ixgbevf_update_itr - update the dynamic ITR value based on statistics
5f3600eb
AD		 737 * @q_vector: structure containing interrupt and ring information
738 * @ring_container: structure containing ring performance data
92915f71
GR		 739 *
740 * Stores a new ITR value based on packets and byte
741 * counts during the last interrupt. The advantage of per interrupt
742 * computation is faster updates and more accurate ITR for the current
743 * traffic pattern. Constants in this function were computed
744 * based on theoretical maximum wire speed and thresholds were set based
745 * on testing data as well as attempting to minimize response time
746 * while increasing bulk throughput.
747 **/
5f3600eb
AD		 748static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector,
749 struct ixgbevf_ring_container *ring_container)
92915f71 750{
5f3600eb
AD		 751 int bytes = ring_container->total_bytes;
752 int packets = ring_container->total_packets;
92915f71
GR		 753 u32 timepassed_us;
754 u64 bytes_perint;
5f3600eb 755 u8 itr_setting = ring_container->itr;
92915f71
GR		 756
757 if (packets == 0)
5f3600eb 758 return;
92915f71
GR		 759
760 /* simple throttlerate management
761 * 0-20MB/s lowest (100000 ints/s)
762 * 20-100MB/s low (20000 ints/s)
763 * 100-1249MB/s bulk (8000 ints/s)
764 */
765 /* what was last interrupt timeslice? */
5f3600eb 766 timepassed_us = q_vector->itr >> 2;
92915f71
GR		 767 bytes_perint = bytes / timepassed_us; /* bytes/usec */
768
769 switch (itr_setting) {
770 case lowest_latency:
e2c28ce7 771 if (bytes_perint > 10)
5f3600eb 772 itr_setting = low_latency;
92915f71
GR		 773 break;
774 case low_latency:
e2c28ce7 775 if (bytes_perint > 20)
5f3600eb 776 itr_setting = bulk_latency;
e2c28ce7 777 else if (bytes_perint <= 10)
5f3600eb 778 itr_setting = lowest_latency;
92915f71
GR		 779 break;
780 case bulk_latency:
e2c28ce7 781 if (bytes_perint <= 20)
5f3600eb 782 itr_setting = low_latency;
92915f71
GR		 783 break;
784 }
785
5f3600eb
AD		 786 /* clear work counters since we have the values we need */
787 ring_container->total_bytes = 0;
788 ring_container->total_packets = 0;
789
790 /* write updated itr to ring container */
791 ring_container->itr = itr_setting;
92915f71
GR		 792}
793
fa71ae27 794static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector)
92915f71 795{
5f3600eb
AD		 796 u32 new_itr = q_vector->itr;
797 u8 current_itr;
92915f71 798
5f3600eb
AD		 799 ixgbevf_update_itr(q_vector, &q_vector->tx);
800 ixgbevf_update_itr(q_vector, &q_vector->rx);
92915f71 801
6b43c446 802 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
92915f71
GR		 803
804 switch (current_itr) {
805 /* counts and packets in update_itr are dependent on these numbers */
806 case lowest_latency:
5f3600eb 807 new_itr = IXGBE_100K_ITR;
92915f71
GR		 808 break;
809 case low_latency:
5f3600eb 810 new_itr = IXGBE_20K_ITR;
92915f71
GR		 811 break;
812 case bulk_latency:
813 default:
5f3600eb 814 new_itr = IXGBE_8K_ITR;
92915f71
GR		 815 break;
816 }
817
5f3600eb 818 if (new_itr != q_vector->itr) {
92915f71 819 /* do an exponential smoothing */
5f3600eb
AD		 820 new_itr = (10 * new_itr * q_vector->itr) /
821 ((9 * new_itr) + q_vector->itr);
822
823 /* save the algorithm value here */
824 q_vector->itr = new_itr;
825
826 ixgbevf_write_eitr(q_vector);
92915f71 827 }
92915f71
GR		 828}
829
4b2cd27f 830static irqreturn_t ixgbevf_msix_other(int irq, void *data)
92915f71 831{
fa71ae27 832 struct ixgbevf_adapter *adapter = data;
92915f71 833 struct ixgbe_hw *hw = &adapter->hw;
08259594 834
4b2cd27f 835 hw->mac.get_link_status = 1;
1e72bfc3 836
2e7cfbdd
MR		 837 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
838 !test_bit(__IXGBEVF_REMOVING, &adapter->state))
c7bb417d 839 mod_timer(&adapter->watchdog_timer, jiffies);
3a2c4033 840
5f3600eb
AD		 841 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other);
842
92915f71
GR		 843 return IRQ_HANDLED;
844}
845
92915f71 846/**
fa71ae27 847 * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues)
92915f71
GR		 848 * @irq: unused
849 * @data: pointer to our q_vector struct for this interrupt vector
850 **/
fa71ae27 851static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data)
92915f71
GR		 852{
853 struct ixgbevf_q_vector *q_vector = data;
92915f71 854
5f3600eb 855 /* EIAM disabled interrupts (on this vector) for us */
fa71ae27
AD		 856 if (q_vector->rx.ring || q_vector->tx.ring)
857 napi_schedule(&q_vector->napi);
92915f71
GR		 858
859 return IRQ_HANDLED;
860}
861
862static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx,
863 int r_idx)
864{
865 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
866
87e70ab9
DS		 867 a->rx_ring[r_idx]->next = q_vector->rx.ring;
868 q_vector->rx.ring = a->rx_ring[r_idx];
6b43c446 869 q_vector->rx.count++;
92915f71
GR		 870}
871
872static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx,
873 int t_idx)
874{
875 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
876
87e70ab9
DS		 877 a->tx_ring[t_idx]->next = q_vector->tx.ring;
878 q_vector->tx.ring = a->tx_ring[t_idx];
6b43c446 879 q_vector->tx.count++;
92915f71
GR		 880}
881
882/**
883 * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors
884 * @adapter: board private structure to initialize
885 *
886 * This function maps descriptor rings to the queue-specific vectors
887 * we were allotted through the MSI-X enabling code. Ideally, we'd have
888 * one vector per ring/queue, but on a constrained vector budget, we
889 * group the rings as "efficiently" as possible. You would add new
890 * mapping configurations in here.
891 **/
892static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter)
893{
894 int q_vectors;
895 int v_start = 0;
896 int rxr_idx = 0, txr_idx = 0;
897 int rxr_remaining = adapter->num_rx_queues;
898 int txr_remaining = adapter->num_tx_queues;
899 int i, j;
900 int rqpv, tqpv;
901 int err = 0;
902
903 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
904
905 /*
906 * The ideal configuration...
907 * We have enough vectors to map one per queue.
908 */
909 if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) {
910 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++)
911 map_vector_to_rxq(adapter, v_start, rxr_idx);
912
913 for (; txr_idx < txr_remaining; v_start++, txr_idx++)
914 map_vector_to_txq(adapter, v_start, txr_idx);
915 goto out;
916 }
917
918 /*
919 * If we don't have enough vectors for a 1-to-1
920 * mapping, we'll have to group them so there are
921 * multiple queues per vector.
922 */
923 /* Re-adjusting *qpv takes care of the remainder. */
924 for (i = v_start; i < q_vectors; i++) {
925 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i);
926 for (j = 0; j < rqpv; j++) {
927 map_vector_to_rxq(adapter, i, rxr_idx);
928 rxr_idx++;
929 rxr_remaining--;
930 }
931 }
932 for (i = v_start; i < q_vectors; i++) {
933 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i);
934 for (j = 0; j < tqpv; j++) {
935 map_vector_to_txq(adapter, i, txr_idx);
936 txr_idx++;
937 txr_remaining--;
938 }
939 }
940
941out:
942 return err;
943}
944
945/**
946 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts
947 * @adapter: board private structure
948 *
949 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests
950 * interrupts from the kernel.
951 **/
952static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter)
953{
954 struct net_device *netdev = adapter->netdev;
fa71ae27
AD		 955 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
956 int vector, err;
92915f71
GR		 957 int ri = 0, ti = 0;
958
92915f71 959 for (vector = 0; vector < q_vectors; vector++) {
fa71ae27
AD		 960 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector];
961 struct msix_entry *entry = &adapter->msix_entries[vector];
962
963 if (q_vector->tx.ring && q_vector->rx.ring) {
964 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
965 "%s-%s-%d", netdev->name, "TxRx", ri++);
966 ti++;
967 } else if (q_vector->rx.ring) {
968 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
969 "%s-%s-%d", netdev->name, "rx", ri++);
970 } else if (q_vector->tx.ring) {
971 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
972 "%s-%s-%d", netdev->name, "tx", ti++);
92915f71
GR		 973 } else {
974 /* skip this unused q_vector */
975 continue;
976 }
fa71ae27
AD		 977 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0,
978 q_vector->name, q_vector);
92915f71
GR		 979 if (err) {
980 hw_dbg(&adapter->hw,
981 "request_irq failed for MSIX interrupt "
982 "Error: %d\n", err);
983 goto free_queue_irqs;
984 }
985 }
986
92915f71 987 err = request_irq(adapter->msix_entries[vector].vector,
4b2cd27f 988 &ixgbevf_msix_other, 0, netdev->name, adapter);
92915f71
GR		 989 if (err) {
990 hw_dbg(&adapter->hw,
4b2cd27f 991 "request_irq for msix_other failed: %d\n", err);
92915f71
GR		 992 goto free_queue_irqs;
993 }
994
995 return 0;
996
997free_queue_irqs:
fa71ae27
AD		 998 while (vector) {
999 vector--;
1000 free_irq(adapter->msix_entries[vector].vector,
1001 adapter->q_vector[vector]);
1002 }
a1f6c6b1 1003 /* This failure is non-recoverable - it indicates the system is
1004 * out of MSIX vector resources and the VF driver cannot run
1005 * without them. Set the number of msix vectors to zero
1006 * indicating that not enough can be allocated. The error
1007 * will be returned to the user indicating device open failed.
1008 * Any further attempts to force the driver to open will also
1009 * fail. The only way to recover is to unload the driver and
1010 * reload it again. If the system has recovered some MSIX
1011 * vectors then it may succeed.
1012 */
1013 adapter->num_msix_vectors = 0;
92915f71
GR		 1014 return err;
1015}
1016
1017static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter)
1018{
1019 int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1020
1021 for (i = 0; i < q_vectors; i++) {
1022 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i];
6b43c446
AD		 1023 q_vector->rx.ring = NULL;
1024 q_vector->tx.ring = NULL;
1025 q_vector->rx.count = 0;
1026 q_vector->tx.count = 0;
92915f71
GR		 1027 }
1028}
1029
1030/**
1031 * ixgbevf_request_irq - initialize interrupts
1032 * @adapter: board private structure
1033 *
1034 * Attempts to configure interrupts using the best available
1035 * capabilities of the hardware and kernel.
1036 **/
1037static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter)
1038{
1039 int err = 0;
1040
1041 err = ixgbevf_request_msix_irqs(adapter);
1042
1043 if (err)
1044 hw_dbg(&adapter->hw,
1045 "request_irq failed, Error %d\n", err);
1046
1047 return err;
1048}
1049
1050static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter)
1051{
92915f71
GR		 1052 int i, q_vectors;
1053
1054 q_vectors = adapter->num_msix_vectors;
92915f71
GR		 1055 i = q_vectors - 1;
1056
fa71ae27 1057 free_irq(adapter->msix_entries[i].vector, adapter);
92915f71
GR		 1058 i--;
1059
1060 for (; i >= 0; i--) {
fa71ae27
AD		 1061 /* free only the irqs that were actually requested */
1062 if (!adapter->q_vector[i]->rx.ring &&
1063 !adapter->q_vector[i]->tx.ring)
1064 continue;
1065
92915f71
GR		 1066 free_irq(adapter->msix_entries[i].vector,
1067 adapter->q_vector[i]);
1068 }
1069
1070 ixgbevf_reset_q_vectors(adapter);
1071}
1072
1073/**
1074 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC
1075 * @adapter: board private structure
1076 **/
1077static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter)
1078{
92915f71 1079 struct ixgbe_hw *hw = &adapter->hw;
5f3600eb 1080 int i;
92915f71 1081
5f3600eb 1082 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0);
92915f71 1083 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0);
5f3600eb 1084 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0);
92915f71
GR		 1085
1086 IXGBE_WRITE_FLUSH(hw);
1087
1088 for (i = 0; i < adapter->num_msix_vectors; i++)
1089 synchronize_irq(adapter->msix_entries[i].vector);
1090}
1091
1092/**
1093 * ixgbevf_irq_enable - Enable default interrupt generation settings
1094 * @adapter: board private structure
1095 **/
5f3600eb 1096static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter)
92915f71
GR		 1097{
1098 struct ixgbe_hw *hw = &adapter->hw;
92915f71 1099
5f3600eb
AD		 1100 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask);
1101 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask);
1102 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask);
92915f71
GR		 1103}
1104
de02decb
DS		 1105/**
1106 * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset
1107 * @adapter: board private structure
1108 * @ring: structure containing ring specific data
1109 *
1110 * Configure the Tx descriptor ring after a reset.
1111 **/
1112static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter,
1113 struct ixgbevf_ring *ring)
1114{
1115 struct ixgbe_hw *hw = &adapter->hw;
1116 u64 tdba = ring->dma;
1117 int wait_loop = 10;
1118 u32 txdctl = IXGBE_TXDCTL_ENABLE;
1119 u8 reg_idx = ring->reg_idx;
1120
1121 /* disable queue to avoid issues while updating state */
1122 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
1123 IXGBE_WRITE_FLUSH(hw);
1124
1125 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(reg_idx), tdba & DMA_BIT_MASK(32));
1126 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(reg_idx), tdba >> 32);
1127 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(reg_idx),
1128 ring->count * sizeof(union ixgbe_adv_tx_desc));
1129
1130 /* disable head writeback */
1131 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAH(reg_idx), 0);
1132 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAL(reg_idx), 0);
1133
1134 /* enable relaxed ordering */
1135 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(reg_idx),
1136 (IXGBE_DCA_TXCTRL_DESC_RRO_EN |
1137 IXGBE_DCA_TXCTRL_DATA_RRO_EN));
1138
1139 /* reset head and tail pointers */
1140 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(reg_idx), 0);
1141 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(reg_idx), 0);
1142 ring->tail = hw->hw_addr + IXGBE_VFTDT(reg_idx);
1143
1144 /* reset ntu and ntc to place SW in sync with hardwdare */
1145 ring->next_to_clean = 0;
1146 ring->next_to_use = 0;
1147
1148 /* In order to avoid issues WTHRESH + PTHRESH should always be equal
1149 * to or less than the number of on chip descriptors, which is
1150 * currently 40.
1151 */
1152 txdctl |= (8 << 16); /* WTHRESH = 8 */
1153
1154 /* Setting PTHRESH to 32 both improves performance */
1155 txdctl |= (1 << 8) | /* HTHRESH = 1 */
1156 32; /* PTHRESH = 32 */
1157
1158 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl);
1159
1160 /* poll to verify queue is enabled */
1161 do {
1162 usleep_range(1000, 2000);
1163 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(reg_idx));
1164 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
1165 if (!wait_loop)
1166 pr_err("Could not enable Tx Queue %d\n", reg_idx);
1167}
1168
92915f71
GR		 1169/**
1170 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset
1171 * @adapter: board private structure
1172 *
1173 * Configure the Tx unit of the MAC after a reset.
1174 **/
1175static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter)
1176{
de02decb 1177 u32 i;
92915f71
GR		 1178
1179 /* Setup the HW Tx Head and Tail descriptor pointers */
de02decb
DS		 1180 for (i = 0; i < adapter->num_tx_queues; i++)
1181 ixgbevf_configure_tx_ring(adapter, adapter->tx_ring[i]);
92915f71
GR		 1182}
1183
1184#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
1185
1186static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index)
1187{
1188 struct ixgbevf_ring *rx_ring;
1189 struct ixgbe_hw *hw = &adapter->hw;
1190 u32 srrctl;
1191
87e70ab9 1192 rx_ring = adapter->rx_ring[index];
92915f71
GR		 1193
1194 srrctl = IXGBE_SRRCTL_DROP_EN;
1195
77d5dfca 1196 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
92915f71 1197
dd1fe113
AD		 1198 srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >>
1199 IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1200
92915f71
GR		 1201 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl);
1202}
1203
1bb9c639
DS		 1204static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter)
1205{
1206 struct ixgbe_hw *hw = &adapter->hw;
1207
1208 /* PSRTYPE must be initialized in 82599 */
1209 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
1210 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR |
1211 IXGBE_PSRTYPE_L2HDR;
1212
1213 if (adapter->num_rx_queues > 1)
1214 psrtype |= 1 << 29;
1215
1216 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
1217}
1218
dd1fe113
AD		 1219static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter)
1220{
1221 struct ixgbe_hw *hw = &adapter->hw;
1222 struct net_device *netdev = adapter->netdev;
1223 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1224 int i;
1225 u16 rx_buf_len;
1226
1227 /* notify the PF of our intent to use this size of frame */
1228 ixgbevf_rlpml_set_vf(hw, max_frame);
1229
1230 /* PF will allow an extra 4 bytes past for vlan tagged frames */
1231 max_frame += VLAN_HLEN;
1232
1233 /*
85624caf
GR		 1234 * Allocate buffer sizes that fit well into 32K and
1235 * take into account max frame size of 9.5K
dd1fe113
AD		 1236 */
1237 if ((hw->mac.type == ixgbe_mac_X540_vf) &&
1238 (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE))
1239 rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85624caf
GR		 1240 else if (max_frame <= IXGBEVF_RXBUFFER_2K)
1241 rx_buf_len = IXGBEVF_RXBUFFER_2K;
1242 else if (max_frame <= IXGBEVF_RXBUFFER_4K)
1243 rx_buf_len = IXGBEVF_RXBUFFER_4K;
1244 else if (max_frame <= IXGBEVF_RXBUFFER_8K)
1245 rx_buf_len = IXGBEVF_RXBUFFER_8K;
dd1fe113 1246 else
85624caf 1247 rx_buf_len = IXGBEVF_RXBUFFER_10K;
dd1fe113
AD		 1248
1249 for (i = 0; i < adapter->num_rx_queues; i++)
87e70ab9 1250 adapter->rx_ring[i]->rx_buf_len = rx_buf_len;
dd1fe113
AD		 1251}
1252
de02decb
DS		 1253#define IXGBEVF_MAX_RX_DESC_POLL 10
1254static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter,
1255 struct ixgbevf_ring *ring)
1256{
1257 struct ixgbe_hw *hw = &adapter->hw;
1258 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1259 u32 rxdctl;
1260 u8 reg_idx = ring->reg_idx;
1261
1262 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1263 rxdctl &= ~IXGBE_RXDCTL_ENABLE;
1264
1265 /* write value back with RXDCTL.ENABLE bit cleared */
1266 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
1267
1268 /* the hardware may take up to 100us to really disable the rx queue */
1269 do {
1270 udelay(10);
1271 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1272 } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
1273
1274 if (!wait_loop)
1275 pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n",
1276 reg_idx);
1277}
1278
1279static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
1280 struct ixgbevf_ring *ring)
1281{
1282 struct ixgbe_hw *hw = &adapter->hw;
1283 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1284 u32 rxdctl;
1285 u8 reg_idx = ring->reg_idx;
1286
1287 do {
1288 usleep_range(1000, 2000);
1289 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1290 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
1291
1292 if (!wait_loop)
1293 pr_err("RXDCTL.ENABLE queue %d not set while polling\n",
1294 reg_idx);
1295}
1296
1297static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter,
1298 struct ixgbevf_ring *ring)
1299{
1300 struct ixgbe_hw *hw = &adapter->hw;
1301 u64 rdba = ring->dma;
1302 u32 rxdctl;
1303 u8 reg_idx = ring->reg_idx;
1304
1305 /* disable queue to avoid issues while updating state */
1306 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1307 ixgbevf_disable_rx_queue(adapter, ring);
1308
1309 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(reg_idx), rdba & DMA_BIT_MASK(32));
1310 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(reg_idx), rdba >> 32);
1311 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(reg_idx),
1312 ring->count * sizeof(union ixgbe_adv_rx_desc));
1313
1314 /* enable relaxed ordering */
1315 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx),
1316 IXGBE_DCA_RXCTRL_DESC_RRO_EN);
1317
1318 /* reset head and tail pointers */
1319 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(reg_idx), 0);
1320 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0);
1321 ring->tail = hw->hw_addr + IXGBE_VFRDT(reg_idx);
1322
1323 /* reset ntu and ntc to place SW in sync with hardwdare */
1324 ring->next_to_clean = 0;
1325 ring->next_to_use = 0;
1326
1327 ixgbevf_configure_srrctl(adapter, reg_idx);
1328
1329 /* prevent DMA from exceeding buffer space available */
1330 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
1331 rxdctl |= ring->rx_buf_len | IXGBE_RXDCTL_RLPML_EN;
1332 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
1333 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
1334
1335 ixgbevf_rx_desc_queue_enable(adapter, ring);
095e2617 1336 ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring));
de02decb
DS		 1337}
1338
92915f71
GR		 1339/**
1340 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset
1341 * @adapter: board private structure
1342 *
1343 * Configure the Rx unit of the MAC after a reset.
1344 **/
1345static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter)
1346{
de02decb 1347 int i;
92915f71 1348
1bb9c639 1349 ixgbevf_setup_psrtype(adapter);
dd1fe113
AD		 1350
1351 /* set_rx_buffer_len must be called before ring initialization */
1352 ixgbevf_set_rx_buffer_len(adapter);
92915f71 1353
92915f71
GR		 1354 /* Setup the HW Rx Head and Tail Descriptor Pointers and
1355 * the Base and Length of the Rx Descriptor Ring */
de02decb
DS		 1356 for (i = 0; i < adapter->num_rx_queues; i++)
1357 ixgbevf_configure_rx_ring(adapter, adapter->rx_ring[i]);
92915f71
GR		 1358}
1359
80d5c368
PM		 1360static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev,
1361 __be16 proto, u16 vid)
92915f71
GR		 1362{
1363 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1364 struct ixgbe_hw *hw = &adapter->hw;
2ddc7fe1
AD		 1365 int err;
1366
55fdd45b 1367 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1368
92915f71 1369 /* add VID to filter table */
2ddc7fe1 1370 err = hw->mac.ops.set_vfta(hw, vid, 0, true);
1c55ed76 1371
55fdd45b 1372 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 1373
2ddc7fe1
AD		 1374 /* translate error return types so error makes sense */
1375 if (err == IXGBE_ERR_MBX)
1376 return -EIO;
1377
1378 if (err == IXGBE_ERR_INVALID_ARGUMENT)
1379 return -EACCES;
1380
dadcd65f 1381 set_bit(vid, adapter->active_vlans);
8e586137 1382
2ddc7fe1 1383 return err;
92915f71
GR		 1384}
1385
80d5c368
PM		 1386static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev,
1387 __be16 proto, u16 vid)
92915f71
GR		 1388{
1389 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1390 struct ixgbe_hw *hw = &adapter->hw;
2ddc7fe1 1391 int err = -EOPNOTSUPP;
92915f71 1392
55fdd45b 1393 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1394
92915f71 1395 /* remove VID from filter table */
92fe0bf7 1396 err = hw->mac.ops.set_vfta(hw, vid, 0, false);
1c55ed76 1397
55fdd45b 1398 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 1399
dadcd65f 1400 clear_bit(vid, adapter->active_vlans);
8e586137 1401
2ddc7fe1 1402 return err;
92915f71
GR		 1403}
1404
1405static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter)
1406{
dadcd65f 1407 u16 vid;
92915f71 1408
dadcd65f 1409 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
80d5c368
PM		 1410 ixgbevf_vlan_rx_add_vid(adapter->netdev,
1411 htons(ETH_P_8021Q), vid);
92915f71
GR		 1412}
1413
46ec20ff
GR		 1414static int ixgbevf_write_uc_addr_list(struct net_device *netdev)
1415{
1416 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1417 struct ixgbe_hw *hw = &adapter->hw;
1418 int count = 0;
1419
1420 if ((netdev_uc_count(netdev)) > 10) {
dbd9636e 1421 pr_err("Too many unicast filters - No Space\n");
46ec20ff
GR		 1422 return -ENOSPC;
1423 }
1424
1425 if (!netdev_uc_empty(netdev)) {
1426 struct netdev_hw_addr *ha;
1427 netdev_for_each_uc_addr(ha, netdev) {
1428 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr);
1429 udelay(200);
1430 }
1431 } else {
1432 /*
1433 * If the list is empty then send message to PF driver to
1434 * clear all macvlans on this VF.
1435 */
1436 hw->mac.ops.set_uc_addr(hw, 0, NULL);
1437 }
1438
1439 return count;
1440}
1441
92915f71 1442/**
dee847f5 1443 * ixgbevf_set_rx_mode - Multicast and unicast set
92915f71
GR		 1444 * @netdev: network interface device structure
1445 *
1446 * The set_rx_method entry point is called whenever the multicast address
dee847f5
GR		 1447 * list, unicast address list or the network interface flags are updated.
1448 * This routine is responsible for configuring the hardware for proper
1449 * multicast mode and configuring requested unicast filters.
92915f71
GR		 1450 **/
1451static void ixgbevf_set_rx_mode(struct net_device *netdev)
1452{
1453 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1454 struct ixgbe_hw *hw = &adapter->hw;
92915f71 1455
55fdd45b 1456 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1457
92915f71 1458 /* reprogram multicast list */
92fe0bf7 1459 hw->mac.ops.update_mc_addr_list(hw, netdev);
46ec20ff
GR		 1460
1461 ixgbevf_write_uc_addr_list(netdev);
1c55ed76 1462
55fdd45b 1463 spin_unlock_bh(&adapter->mbx_lock);
92915f71
GR		 1464}
1465
1466static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter)
1467{
1468 int q_idx;
1469 struct ixgbevf_q_vector *q_vector;
1470 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1471
1472 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
92915f71 1473 q_vector = adapter->q_vector[q_idx];
c777cdfa
JK		 1474#ifdef CONFIG_NET_RX_BUSY_POLL
1475 ixgbevf_qv_init_lock(adapter->q_vector[q_idx]);
1476#endif
fa71ae27 1477 napi_enable(&q_vector->napi);
92915f71
GR		 1478 }
1479}
1480
1481static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter)
1482{
1483 int q_idx;
1484 struct ixgbevf_q_vector *q_vector;
1485 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1486
1487 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1488 q_vector = adapter->q_vector[q_idx];
92915f71 1489 napi_disable(&q_vector->napi);
c777cdfa
JK		 1490#ifdef CONFIG_NET_RX_BUSY_POLL
1491 while (!ixgbevf_qv_disable(adapter->q_vector[q_idx])) {
1492 pr_info("QV %d locked\n", q_idx);
1493 usleep_range(1000, 20000);
1494 }
1495#endif /* CONFIG_NET_RX_BUSY_POLL */
92915f71
GR		 1496 }
1497}
1498
220fe050
DS		 1499static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter)
1500{
1501 struct ixgbe_hw *hw = &adapter->hw;
1502 unsigned int def_q = 0;
1503 unsigned int num_tcs = 0;
1504 unsigned int num_rx_queues = 1;
1505 int err;
1506
1507 spin_lock_bh(&adapter->mbx_lock);
1508
1509 /* fetch queue configuration from the PF */
1510 err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
1511
1512 spin_unlock_bh(&adapter->mbx_lock);
1513
1514 if (err)
1515 return err;
1516
1517 if (num_tcs > 1) {
1518 /* update default Tx ring register index */
87e70ab9 1519 adapter->tx_ring[0]->reg_idx = def_q;
220fe050
DS		 1520
1521 /* we need as many queues as traffic classes */
1522 num_rx_queues = num_tcs;
1523 }
1524
1525 /* if we have a bad config abort request queue reset */
1526 if (adapter->num_rx_queues != num_rx_queues) {
1527 /* force mailbox timeout to prevent further messages */
1528 hw->mbx.timeout = 0;
1529
1530 /* wait for watchdog to come around and bail us out */
1531 adapter->flags |= IXGBEVF_FLAG_QUEUE_RESET_REQUESTED;
1532 }
1533
1534 return 0;
1535}
1536
92915f71
GR		 1537static void ixgbevf_configure(struct ixgbevf_adapter *adapter)
1538{
220fe050
DS		 1539 ixgbevf_configure_dcb(adapter);
1540
de02decb 1541 ixgbevf_set_rx_mode(adapter->netdev);
92915f71
GR		 1542
1543 ixgbevf_restore_vlan(adapter);
1544
1545 ixgbevf_configure_tx(adapter);
1546 ixgbevf_configure_rx(adapter);
92915f71
GR		 1547}
1548
33bd9f60
GR		 1549static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter)
1550{
1551 /* Only save pre-reset stats if there are some */
1552 if (adapter->stats.vfgprc || adapter->stats.vfgptc) {
1553 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc -
1554 adapter->stats.base_vfgprc;
1555 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc -
1556 adapter->stats.base_vfgptc;
1557 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc -
1558 adapter->stats.base_vfgorc;
1559 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc -
1560 adapter->stats.base_vfgotc;
1561 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc -
1562 adapter->stats.base_vfmprc;
1563 }
1564}
1565
1566static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter)
1567{
1568 struct ixgbe_hw *hw = &adapter->hw;
1569
1570 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
1571 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
1572 adapter->stats.last_vfgorc |=
1573 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
1574 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
1575 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
1576 adapter->stats.last_vfgotc |=
1577 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
1578 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
1579
1580 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc;
1581 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc;
1582 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc;
1583 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc;
1584 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc;
1585}
1586
31186785
AD		 1587static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter)
1588{
1589 struct ixgbe_hw *hw = &adapter->hw;
56e94095
AD		 1590 int api[] = { ixgbe_mbox_api_11,
1591 ixgbe_mbox_api_10,
31186785
AD		 1592 ixgbe_mbox_api_unknown };
1593 int err = 0, idx = 0;
1594
55fdd45b 1595 spin_lock_bh(&adapter->mbx_lock);
31186785
AD		 1596
1597 while (api[idx] != ixgbe_mbox_api_unknown) {
1598 err = ixgbevf_negotiate_api_version(hw, api[idx]);
1599 if (!err)
1600 break;
1601 idx++;
1602 }
1603
55fdd45b 1604 spin_unlock_bh(&adapter->mbx_lock);
31186785
AD		 1605}
1606
795180d8 1607static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter)
92915f71
GR		 1608{
1609 struct net_device *netdev = adapter->netdev;
1610 struct ixgbe_hw *hw = &adapter->hw;
92915f71
GR		 1611
1612 ixgbevf_configure_msix(adapter);
1613
55fdd45b 1614 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1615
92fe0bf7
GR		 1616 if (is_valid_ether_addr(hw->mac.addr))
1617 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
1618 else
1619 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0);
92915f71 1620
55fdd45b 1621 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 1622
92915f71
GR		 1623 clear_bit(__IXGBEVF_DOWN, &adapter->state);
1624 ixgbevf_napi_enable_all(adapter);
1625
1626 /* enable transmits */
1627 netif_tx_start_all_queues(netdev);
1628
33bd9f60
GR		 1629 ixgbevf_save_reset_stats(adapter);
1630 ixgbevf_init_last_counter_stats(adapter);
1631
4b2cd27f 1632 hw->mac.get_link_status = 1;
92915f71 1633 mod_timer(&adapter->watchdog_timer, jiffies);
92915f71
GR		 1634}
1635
795180d8 1636void ixgbevf_up(struct ixgbevf_adapter *adapter)
92915f71 1637{
92915f71
GR		 1638 struct ixgbe_hw *hw = &adapter->hw;
1639
1640 ixgbevf_configure(adapter);
1641
795180d8 1642 ixgbevf_up_complete(adapter);
92915f71
GR		 1643
1644 /* clear any pending interrupts, may auto mask */
1645 IXGBE_READ_REG(hw, IXGBE_VTEICR);
1646
5f3600eb 1647 ixgbevf_irq_enable(adapter);
92915f71
GR		 1648}
1649
1650/**
1651 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue
92915f71
GR		 1652 * @rx_ring: ring to free buffers from
1653 **/
05d063aa 1654static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring)
92915f71 1655{
92915f71
GR		 1656 unsigned long size;
1657 unsigned int i;
1658
c0456c23
GR		 1659 if (!rx_ring->rx_buffer_info)
1660 return;
92915f71 1661
c0456c23 1662 /* Free all the Rx ring sk_buffs */
92915f71
GR		 1663 for (i = 0; i < rx_ring->count; i++) {
1664 struct ixgbevf_rx_buffer *rx_buffer_info;
1665
1666 rx_buffer_info = &rx_ring->rx_buffer_info[i];
1667 if (rx_buffer_info->dma) {
05d063aa 1668 dma_unmap_single(rx_ring->dev, rx_buffer_info->dma,
92915f71 1669 rx_ring->rx_buf_len,
2a1f8794 1670 DMA_FROM_DEVICE);
92915f71
GR		 1671 rx_buffer_info->dma = 0;
1672 }
1673 if (rx_buffer_info->skb) {
1674 struct sk_buff *skb = rx_buffer_info->skb;
1675 rx_buffer_info->skb = NULL;
1676 do {
1677 struct sk_buff *this = skb;
5c60f81a 1678 skb = IXGBE_CB(skb)->prev;
92915f71
GR		 1679 dev_kfree_skb(this);
1680 } while (skb);
1681 }
92915f71
GR		 1682 }
1683
1684 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
1685 memset(rx_ring->rx_buffer_info, 0, size);
1686
1687 /* Zero out the descriptor ring */
1688 memset(rx_ring->desc, 0, rx_ring->size);
92915f71
GR		 1689}
1690
1691/**
1692 * ixgbevf_clean_tx_ring - Free Tx Buffers
92915f71
GR		 1693 * @tx_ring: ring to be cleaned
1694 **/
05d063aa 1695static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring)
92915f71
GR		 1696{
1697 struct ixgbevf_tx_buffer *tx_buffer_info;
1698 unsigned long size;
1699 unsigned int i;
1700
c0456c23
GR		 1701 if (!tx_ring->tx_buffer_info)
1702 return;
1703
92915f71 1704 /* Free all the Tx ring sk_buffs */
92915f71
GR		 1705 for (i = 0; i < tx_ring->count; i++) {
1706 tx_buffer_info = &tx_ring->tx_buffer_info[i];
70a10e25 1707 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
92915f71
GR		 1708 }
1709
1710 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
1711 memset(tx_ring->tx_buffer_info, 0, size);
1712
1713 memset(tx_ring->desc, 0, tx_ring->size);
92915f71
GR		 1714}
1715
1716/**
1717 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues
1718 * @adapter: board private structure
1719 **/
1720static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter)
1721{
1722 int i;
1723
1724 for (i = 0; i < adapter->num_rx_queues; i++)
05d063aa 1725 ixgbevf_clean_rx_ring(adapter->rx_ring[i]);
92915f71
GR		 1726}
1727
1728/**
1729 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues
1730 * @adapter: board private structure
1731 **/
1732static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter)
1733{
1734 int i;
1735
1736 for (i = 0; i < adapter->num_tx_queues; i++)
05d063aa 1737 ixgbevf_clean_tx_ring(adapter->tx_ring[i]);
92915f71
GR		 1738}
1739
1740void ixgbevf_down(struct ixgbevf_adapter *adapter)
1741{
1742 struct net_device *netdev = adapter->netdev;
1743 struct ixgbe_hw *hw = &adapter->hw;
de02decb 1744 int i;
92915f71
GR		 1745
1746 /* signal that we are down to the interrupt handler */
1747 set_bit(__IXGBEVF_DOWN, &adapter->state);
858c3dda
DS		 1748
1749 /* disable all enabled rx queues */
1750 for (i = 0; i < adapter->num_rx_queues; i++)
87e70ab9 1751 ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[i]);
92915f71
GR		 1752
1753 netif_tx_disable(netdev);
1754
1755 msleep(10);
1756
1757 netif_tx_stop_all_queues(netdev);
1758
1759 ixgbevf_irq_disable(adapter);
1760
1761 ixgbevf_napi_disable_all(adapter);
1762
1763 del_timer_sync(&adapter->watchdog_timer);
1764 /* can't call flush scheduled work here because it can deadlock
1765 * if linkwatch_event tries to acquire the rtnl_lock which we are
1766 * holding */
1767 while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK)
1768 msleep(1);
1769
1770 /* disable transmits in the hardware now that interrupts are off */
1771 for (i = 0; i < adapter->num_tx_queues; i++) {
de02decb
DS		 1772 u8 reg_idx = adapter->tx_ring[i]->reg_idx;
1773
1774 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx),
1775 IXGBE_TXDCTL_SWFLSH);
92915f71
GR		 1776 }
1777
1778 netif_carrier_off(netdev);
1779
1780 if (!pci_channel_offline(adapter->pdev))
1781 ixgbevf_reset(adapter);
1782
1783 ixgbevf_clean_all_tx_rings(adapter);
1784 ixgbevf_clean_all_rx_rings(adapter);
1785}
1786
1787void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter)
1788{
1789 WARN_ON(in_interrupt());
c0456c23 1790
92915f71
GR		 1791 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state))
1792 msleep(1);
1793
4b2cd27f
AD		 1794 ixgbevf_down(adapter);
1795 ixgbevf_up(adapter);
92915f71
GR		 1796
1797 clear_bit(__IXGBEVF_RESETTING, &adapter->state);
1798}
1799
1800void ixgbevf_reset(struct ixgbevf_adapter *adapter)
1801{
1802 struct ixgbe_hw *hw = &adapter->hw;
1803 struct net_device *netdev = adapter->netdev;
1804
798e381a 1805 if (hw->mac.ops.reset_hw(hw)) {
92915f71 1806 hw_dbg(hw, "PF still resetting\n");
798e381a 1807 } else {
92915f71 1808 hw->mac.ops.init_hw(hw);
798e381a
DS		 1809 ixgbevf_negotiate_api(adapter);
1810 }
92915f71
GR		 1811
1812 if (is_valid_ether_addr(adapter->hw.mac.addr)) {
1813 memcpy(netdev->dev_addr, adapter->hw.mac.addr,
1814 netdev->addr_len);
1815 memcpy(netdev->perm_addr, adapter->hw.mac.addr,
1816 netdev->addr_len);
1817 }
1818}
1819
e45dd5fe
JK		 1820static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter,
1821 int vectors)
92915f71 1822{
a5f9337b 1823 int vector_threshold;
92915f71 1824
fa71ae27
AD		 1825 /* We'll want at least 2 (vector_threshold):
1826 * 1) TxQ[0] + RxQ[0] handler
1827 * 2) Other (Link Status Change, etc.)
92915f71
GR		 1828 */
1829 vector_threshold = MIN_MSIX_COUNT;
1830
1831 /* The more we get, the more we will assign to Tx/Rx Cleanup
1832 * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1833 * Right now, we simply care about how many we'll get; we'll
1834 * set them up later while requesting irq's.
1835 */
5c1e3588
AG		 1836 vectors = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
1837 vector_threshold, vectors);
92915f71 1838
5c1e3588 1839 if (vectors < 0) {
e45dd5fe
JK		 1840 dev_err(&adapter->pdev->dev,
1841 "Unable to allocate MSI-X interrupts\n");
92915f71
GR		 1842 kfree(adapter->msix_entries);
1843 adapter->msix_entries = NULL;
5c1e3588 1844 return vectors;
92915f71 1845 }
dee847f5 1846
5c1e3588
AG		 1847 /* Adjust for only the vectors we'll use, which is minimum
1848 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
1849 * vectors we were allocated.
1850 */
1851 adapter->num_msix_vectors = vectors;
1852
1853 return 0;
92915f71
GR		 1854}
1855
49ce9c2c
BH		 1856/**
1857 * ixgbevf_set_num_queues - Allocate queues for device, feature dependent
92915f71
GR		 1858 * @adapter: board private structure to initialize
1859 *
1860 * This is the top level queue allocation routine. The order here is very
1861 * important, starting with the "most" number of features turned on at once,
1862 * and ending with the smallest set of features. This way large combinations
1863 * can be allocated if they're turned on, and smaller combinations are the
1864 * fallthrough conditions.
1865 *
1866 **/
1867static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter)
1868{
220fe050
DS		 1869 struct ixgbe_hw *hw = &adapter->hw;
1870 unsigned int def_q = 0;
1871 unsigned int num_tcs = 0;
1872 int err;
1873
92915f71
GR		 1874 /* Start with base case */
1875 adapter->num_rx_queues = 1;
1876 adapter->num_tx_queues = 1;
220fe050
DS		 1877
1878 spin_lock_bh(&adapter->mbx_lock);
1879
1880 /* fetch queue configuration from the PF */
1881 err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
1882
1883 spin_unlock_bh(&adapter->mbx_lock);
1884
1885 if (err)
1886 return;
1887
1888 /* we need as many queues as traffic classes */
1889 if (num_tcs > 1)
1890 adapter->num_rx_queues = num_tcs;
92915f71
GR		 1891}
1892
1893/**
1894 * ixgbevf_alloc_queues - Allocate memory for all rings
1895 * @adapter: board private structure to initialize
1896 *
1897 * We allocate one ring per queue at run-time since we don't know the
1898 * number of queues at compile-time. The polling_netdev array is
1899 * intended for Multiqueue, but should work fine with a single queue.
1900 **/
1901static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter)
1902{
87e70ab9
DS		 1903 struct ixgbevf_ring *ring;
1904 int rx = 0, tx = 0;
92915f71 1905
87e70ab9
DS		 1906 for (; tx < adapter->num_tx_queues; tx++) {
1907 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
1908 if (!ring)
1909 goto err_allocation;
92915f71 1910
87e70ab9
DS		 1911 ring->dev = &adapter->pdev->dev;
1912 ring->netdev = adapter->netdev;
1913 ring->count = adapter->tx_ring_count;
1914 ring->queue_index = tx;
1915 ring->reg_idx = tx;
92915f71 1916
87e70ab9 1917 adapter->tx_ring[tx] = ring;
92915f71
GR		 1918 }
1919
87e70ab9
DS		 1920 for (; rx < adapter->num_rx_queues; rx++) {
1921 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
1922 if (!ring)
1923 goto err_allocation;
1924
1925 ring->dev = &adapter->pdev->dev;
1926 ring->netdev = adapter->netdev;
1927
1928 ring->count = adapter->rx_ring_count;
1929 ring->queue_index = rx;
1930 ring->reg_idx = rx;
1931
1932 adapter->rx_ring[rx] = ring;
92915f71
GR		 1933 }
1934
1935 return 0;
1936
87e70ab9
DS		 1937err_allocation:
1938 while (tx) {
1939 kfree(adapter->tx_ring[--tx]);
1940 adapter->tx_ring[tx] = NULL;
1941 }
1942
1943 while (rx) {
1944 kfree(adapter->rx_ring[--rx]);
1945 adapter->rx_ring[rx] = NULL;
1946 }
92915f71
GR		 1947 return -ENOMEM;
1948}
1949
1950/**
1951 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported
1952 * @adapter: board private structure to initialize
1953 *
1954 * Attempt to configure the interrupts using the best available
1955 * capabilities of the hardware and the kernel.
1956 **/
1957static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter)
1958{
91e2b89b 1959 struct net_device *netdev = adapter->netdev;
92915f71
GR		 1960 int err = 0;
1961 int vector, v_budget;
1962
1963 /*
1964 * It's easy to be greedy for MSI-X vectors, but it really
1965 * doesn't do us much good if we have a lot more vectors
1966 * than CPU's. So let's be conservative and only ask for
fa71ae27
AD		 1967 * (roughly) the same number of vectors as there are CPU's.
1968 * The default is to use pairs of vectors.
92915f71 1969 */
fa71ae27
AD		 1970 v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues);
1971 v_budget = min_t(int, v_budget, num_online_cpus());
1972 v_budget += NON_Q_VECTORS;
92915f71
GR		 1973
1974 /* A failure in MSI-X entry allocation isn't fatal, but it does
1975 * mean we disable MSI-X capabilities of the adapter. */
1976 adapter->msix_entries = kcalloc(v_budget,
1977 sizeof(struct msix_entry), GFP_KERNEL);
1978 if (!adapter->msix_entries) {
1979 err = -ENOMEM;
1980 goto out;
1981 }
1982
1983 for (vector = 0; vector < v_budget; vector++)
1984 adapter->msix_entries[vector].entry = vector;
1985
e45dd5fe
JK		 1986 err = ixgbevf_acquire_msix_vectors(adapter, v_budget);
1987 if (err)
1988 goto out;
92915f71 1989
91e2b89b
GR		 1990 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues);
1991 if (err)
1992 goto out;
1993
1994 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues);
1995
92915f71
GR		 1996out:
1997 return err;
1998}
1999
2000/**
2001 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors
2002 * @adapter: board private structure to initialize
2003 *
2004 * We allocate one q_vector per queue interrupt. If allocation fails we
2005 * return -ENOMEM.
2006 **/
2007static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter)
2008{
2009 int q_idx, num_q_vectors;
2010 struct ixgbevf_q_vector *q_vector;
92915f71
GR		 2011
2012 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
92915f71
GR		 2013
2014 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
2015 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL);
2016 if (!q_vector)
2017 goto err_out;
2018 q_vector->adapter = adapter;
2019 q_vector->v_idx = q_idx;
fa71ae27
AD		 2020 netif_napi_add(adapter->netdev, &q_vector->napi,
2021 ixgbevf_poll, 64);
c777cdfa
JK		 2022#ifdef CONFIG_NET_RX_BUSY_POLL
2023 napi_hash_add(&q_vector->napi);
2024#endif
92915f71
GR		 2025 adapter->q_vector[q_idx] = q_vector;
2026 }
2027
2028 return 0;
2029
2030err_out:
2031 while (q_idx) {
2032 q_idx--;
2033 q_vector = adapter->q_vector[q_idx];
c777cdfa
JK		 2034#ifdef CONFIG_NET_RX_BUSY_POLL
2035 napi_hash_del(&q_vector->napi);
2036#endif
92915f71
GR		 2037 netif_napi_del(&q_vector->napi);
2038 kfree(q_vector);
2039 adapter->q_vector[q_idx] = NULL;
2040 }
2041 return -ENOMEM;
2042}
2043
2044/**
2045 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors
2046 * @adapter: board private structure to initialize
2047 *
2048 * This function frees the memory allocated to the q_vectors. In addition if
2049 * NAPI is enabled it will delete any references to the NAPI struct prior
2050 * to freeing the q_vector.
2051 **/
2052static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter)
2053{
f4477702 2054 int q_idx, num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
92915f71
GR		 2055
2056 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
2057 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx];
2058
2059 adapter->q_vector[q_idx] = NULL;
c777cdfa
JK		 2060#ifdef CONFIG_NET_RX_BUSY_POLL
2061 napi_hash_del(&q_vector->napi);
2062#endif
f4477702 2063 netif_napi_del(&q_vector->napi);
92915f71
GR		 2064 kfree(q_vector);
2065 }
2066}
2067
2068/**
2069 * ixgbevf_reset_interrupt_capability - Reset MSIX setup
2070 * @adapter: board private structure
2071 *
2072 **/
2073static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter)
2074{
2075 pci_disable_msix(adapter->pdev);
2076 kfree(adapter->msix_entries);
2077 adapter->msix_entries = NULL;
92915f71
GR		 2078}
2079
2080/**
2081 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init
2082 * @adapter: board private structure to initialize
2083 *
2084 **/
2085static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
2086{
2087 int err;
2088
2089 /* Number of supported queues */
2090 ixgbevf_set_num_queues(adapter);
2091
2092 err = ixgbevf_set_interrupt_capability(adapter);
2093 if (err) {
2094 hw_dbg(&adapter->hw,
2095 "Unable to setup interrupt capabilities\n");
2096 goto err_set_interrupt;
2097 }
2098
2099 err = ixgbevf_alloc_q_vectors(adapter);
2100 if (err) {
2101 hw_dbg(&adapter->hw, "Unable to allocate memory for queue "
2102 "vectors\n");
2103 goto err_alloc_q_vectors;
2104 }
2105
2106 err = ixgbevf_alloc_queues(adapter);
2107 if (err) {
dbd9636e 2108 pr_err("Unable to allocate memory for queues\n");
92915f71
GR		 2109 goto err_alloc_queues;
2110 }
2111
2112 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, "
2113 "Tx Queue count = %u\n",
2114 (adapter->num_rx_queues > 1) ? "Enabled" :
2115 "Disabled", adapter->num_rx_queues, adapter->num_tx_queues);
2116
2117 set_bit(__IXGBEVF_DOWN, &adapter->state);
2118
2119 return 0;
2120err_alloc_queues:
2121 ixgbevf_free_q_vectors(adapter);
2122err_alloc_q_vectors:
2123 ixgbevf_reset_interrupt_capability(adapter);
2124err_set_interrupt:
2125 return err;
2126}
2127
0ac1e8ce
AD		 2128/**
2129 * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings
2130 * @adapter: board private structure to clear interrupt scheme on
2131 *
2132 * We go through and clear interrupt specific resources and reset the structure
2133 * to pre-load conditions
2134 **/
2135static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter)
2136{
87e70ab9
DS		 2137 int i;
2138
2139 for (i = 0; i < adapter->num_tx_queues; i++) {
2140 kfree(adapter->tx_ring[i]);
2141 adapter->tx_ring[i] = NULL;
2142 }
2143 for (i = 0; i < adapter->num_rx_queues; i++) {
2144 kfree(adapter->rx_ring[i]);
2145 adapter->rx_ring[i] = NULL;
2146 }
2147
0ac1e8ce
AD		 2148 adapter->num_tx_queues = 0;
2149 adapter->num_rx_queues = 0;
2150
2151 ixgbevf_free_q_vectors(adapter);
2152 ixgbevf_reset_interrupt_capability(adapter);
2153}
2154
92915f71
GR		 2155/**
2156 * ixgbevf_sw_init - Initialize general software structures
2157 * (struct ixgbevf_adapter)
2158 * @adapter: board private structure to initialize
2159 *
2160 * ixgbevf_sw_init initializes the Adapter private data structure.
2161 * Fields are initialized based on PCI device information and
2162 * OS network device settings (MTU size).
2163 **/
9f9a12f8 2164static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter)
92915f71
GR		 2165{
2166 struct ixgbe_hw *hw = &adapter->hw;
2167 struct pci_dev *pdev = adapter->pdev;
e1941a74 2168 struct net_device *netdev = adapter->netdev;
92915f71
GR		 2169 int err;
2170
2171 /* PCI config space info */
2172
2173 hw->vendor_id = pdev->vendor;
2174 hw->device_id = pdev->device;
ff938e43 2175 hw->revision_id = pdev->revision;
92915f71
GR		 2176 hw->subsystem_vendor_id = pdev->subsystem_vendor;
2177 hw->subsystem_device_id = pdev->subsystem_device;
2178
2179 hw->mbx.ops.init_params(hw);
56e94095
AD		 2180
2181 /* assume legacy case in which PF would only give VF 2 queues */
2182 hw->mac.max_tx_queues = 2;
2183 hw->mac.max_rx_queues = 2;
2184
798e381a
DS		 2185 /* lock to protect mailbox accesses */
2186 spin_lock_init(&adapter->mbx_lock);
2187
92915f71
GR		 2188 err = hw->mac.ops.reset_hw(hw);
2189 if (err) {
2190 dev_info(&pdev->dev,
e1941a74 2191 "PF still in reset state. Is the PF interface up?\n");
92915f71
GR		 2192 } else {
2193 err = hw->mac.ops.init_hw(hw);
2194 if (err) {
dbd9636e 2195 pr_err("init_shared_code failed: %d\n", err);
92915f71
GR		 2196 goto out;
2197 }
798e381a 2198 ixgbevf_negotiate_api(adapter);
e1941a74
GR		 2199 err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr);
2200 if (err)
2201 dev_info(&pdev->dev, "Error reading MAC address\n");
2202 else if (is_zero_ether_addr(adapter->hw.mac.addr))
2203 dev_info(&pdev->dev,
2204 "MAC address not assigned by administrator.\n");
2205 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2206 }
2207
2208 if (!is_valid_ether_addr(netdev->dev_addr)) {
2209 dev_info(&pdev->dev, "Assigning random MAC address\n");
2210 eth_hw_addr_random(netdev);
2211 memcpy(hw->mac.addr, netdev->dev_addr, netdev->addr_len);
92915f71
GR		 2212 }
2213
2214 /* Enable dynamic interrupt throttling rates */
5f3600eb
AD		 2215 adapter->rx_itr_setting = 1;
2216 adapter->tx_itr_setting = 1;
92915f71 2217
92915f71
GR		 2218 /* set default ring sizes */
2219 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD;
2220 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD;
2221
92915f71 2222 set_bit(__IXGBEVF_DOWN, &adapter->state);
1a0d6ae5 2223 return 0;
92915f71
GR		 2224
2225out:
2226 return err;
2227}
2228
92915f71
GR		 2229#define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \
2230 { \
2231 u32 current_counter = IXGBE_READ_REG(hw, reg); \
2232 if (current_counter < last_counter) \
2233 counter += 0x100000000LL; \
2234 last_counter = current_counter; \
2235 counter &= 0xFFFFFFFF00000000LL; \
2236 counter |= current_counter; \
2237 }
2238
2239#define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
2240 { \
2241 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \
2242 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \
2243 u64 current_counter = (current_counter_msb << 32) | \
2244 current_counter_lsb; \
2245 if (current_counter < last_counter) \
2246 counter += 0x1000000000LL; \
2247 last_counter = current_counter; \
2248 counter &= 0xFFFFFFF000000000LL; \
2249 counter |= current_counter; \
2250 }
2251/**
2252 * ixgbevf_update_stats - Update the board statistics counters.
2253 * @adapter: board private structure
2254 **/
2255void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
2256{
2257 struct ixgbe_hw *hw = &adapter->hw;
55fb277c 2258 int i;
92915f71 2259
088245a3
GR		 2260 if (!adapter->link_up)
2261 return;
2262
92915f71
GR		 2263 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc,
2264 adapter->stats.vfgprc);
2265 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc,
2266 adapter->stats.vfgptc);
2267 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
2268 adapter->stats.last_vfgorc,
2269 adapter->stats.vfgorc);
2270 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
2271 adapter->stats.last_vfgotc,
2272 adapter->stats.vfgotc);
2273 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc,
2274 adapter->stats.vfmprc);
55fb277c
GR		 2275
2276 for (i = 0; i < adapter->num_rx_queues; i++) {
2277 adapter->hw_csum_rx_error +=
87e70ab9 2278 adapter->rx_ring[i]->hw_csum_rx_error;
87e70ab9 2279 adapter->rx_ring[i]->hw_csum_rx_error = 0;
55fb277c 2280 }
92915f71
GR		 2281}
2282
2283/**
2284 * ixgbevf_watchdog - Timer Call-back
2285 * @data: pointer to adapter cast into an unsigned long
2286 **/
2287static void ixgbevf_watchdog(unsigned long data)
2288{
2289 struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data;
2290 struct ixgbe_hw *hw = &adapter->hw;
5f3600eb 2291 u32 eics = 0;
92915f71
GR		 2292 int i;
2293
2294 /*
2295 * Do the watchdog outside of interrupt context due to the lovely
2296 * delays that some of the newer hardware requires
2297 */
2298
2299 if (test_bit(__IXGBEVF_DOWN, &adapter->state))
2300 goto watchdog_short_circuit;
2301
2302 /* get one bit for every active tx/rx interrupt vector */
2303 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
2304 struct ixgbevf_q_vector *qv = adapter->q_vector[i];
6b43c446 2305 if (qv->rx.ring || qv->tx.ring)
5f3600eb 2306 eics |= 1 << i;
92915f71
GR		 2307 }
2308
5f3600eb 2309 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics);
92915f71
GR		 2310
2311watchdog_short_circuit:
2312 schedule_work(&adapter->watchdog_task);
2313}
2314
2315/**
2316 * ixgbevf_tx_timeout - Respond to a Tx Hang
2317 * @netdev: network interface device structure
2318 **/
2319static void ixgbevf_tx_timeout(struct net_device *netdev)
2320{
2321 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2322
2323 /* Do the reset outside of interrupt context */
2324 schedule_work(&adapter->reset_task);
2325}
2326
2327static void ixgbevf_reset_task(struct work_struct *work)
2328{
2329 struct ixgbevf_adapter *adapter;
2330 adapter = container_of(work, struct ixgbevf_adapter, reset_task);
2331
2332 /* If we're already down or resetting, just bail */
2333 if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
2e7cfbdd 2334 test_bit(__IXGBEVF_REMOVING, &adapter->state) ||
92915f71
GR		 2335 test_bit(__IXGBEVF_RESETTING, &adapter->state))
2336 return;
2337
2338 adapter->tx_timeout_count++;
2339
2340 ixgbevf_reinit_locked(adapter);
2341}
2342
2343/**
2344 * ixgbevf_watchdog_task - worker thread to bring link up
2345 * @work: pointer to work_struct containing our data
2346 **/
2347static void ixgbevf_watchdog_task(struct work_struct *work)
2348{
2349 struct ixgbevf_adapter *adapter = container_of(work,
2350 struct ixgbevf_adapter,
2351 watchdog_task);
2352 struct net_device *netdev = adapter->netdev;
2353 struct ixgbe_hw *hw = &adapter->hw;
2354 u32 link_speed = adapter->link_speed;
2355 bool link_up = adapter->link_up;
92fe0bf7 2356 s32 need_reset;
92915f71 2357
220fe050
DS		 2358 ixgbevf_queue_reset_subtask(adapter);
2359
92915f71
GR		 2360 adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK;
2361
2362 /*
2363 * Always check the link on the watchdog because we have
2364 * no LSC interrupt
2365 */
92fe0bf7 2366 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 2367
92fe0bf7 2368 need_reset = hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
1c55ed76 2369
92fe0bf7 2370 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 2371
92fe0bf7
GR		 2372 if (need_reset) {
2373 adapter->link_up = link_up;
2374 adapter->link_speed = link_speed;
2375 netif_carrier_off(netdev);
2376 netif_tx_stop_all_queues(netdev);
2377 schedule_work(&adapter->reset_task);
2378 goto pf_has_reset;
92915f71
GR		 2379 }
2380 adapter->link_up = link_up;
2381 adapter->link_speed = link_speed;
2382
2383 if (link_up) {
2384 if (!netif_carrier_ok(netdev)) {
b876a744
GR		 2385 char *link_speed_string;
2386 switch (link_speed) {
2387 case IXGBE_LINK_SPEED_10GB_FULL:
2388 link_speed_string = "10 Gbps";
2389 break;
2390 case IXGBE_LINK_SPEED_1GB_FULL:
2391 link_speed_string = "1 Gbps";
2392 break;
2393 case IXGBE_LINK_SPEED_100_FULL:
2394 link_speed_string = "100 Mbps";
2395 break;
2396 default:
2397 link_speed_string = "unknown speed";
2398 break;
2399 }
6fe59675 2400 dev_info(&adapter->pdev->dev,
b876a744 2401 "NIC Link is Up, %s\n", link_speed_string);
92915f71
GR		 2402 netif_carrier_on(netdev);
2403 netif_tx_wake_all_queues(netdev);
92915f71
GR		 2404 }
2405 } else {
2406 adapter->link_up = false;
2407 adapter->link_speed = 0;
2408 if (netif_carrier_ok(netdev)) {
6fe59675 2409 dev_info(&adapter->pdev->dev, "NIC Link is Down\n");
92915f71
GR		 2410 netif_carrier_off(netdev);
2411 netif_tx_stop_all_queues(netdev);
2412 }
2413 }
2414
92915f71
GR		 2415 ixgbevf_update_stats(adapter);
2416
33bd9f60 2417pf_has_reset:
92915f71 2418 /* Reset the timer */
2e7cfbdd
MR		 2419 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
2420 !test_bit(__IXGBEVF_REMOVING, &adapter->state))
92915f71
GR		 2421 mod_timer(&adapter->watchdog_timer,
2422 round_jiffies(jiffies + (2 * HZ)));
2423
2424 adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK;
2425}
2426
2427/**
2428 * ixgbevf_free_tx_resources - Free Tx Resources per Queue
92915f71
GR		 2429 * @tx_ring: Tx descriptor ring for a specific queue
2430 *
2431 * Free all transmit software resources
2432 **/
05d063aa 2433void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring)
92915f71 2434{
05d063aa 2435 ixgbevf_clean_tx_ring(tx_ring);
92915f71
GR		 2436
2437 vfree(tx_ring->tx_buffer_info);
2438 tx_ring->tx_buffer_info = NULL;
2439
de02decb
DS		 2440 /* if not set, then don't free */
2441 if (!tx_ring->desc)
2442 return;
2443
05d063aa 2444 dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc,
2a1f8794 2445 tx_ring->dma);
92915f71
GR		 2446
2447 tx_ring->desc = NULL;
2448}
2449
2450/**
2451 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues
2452 * @adapter: board private structure
2453 *
2454 * Free all transmit software resources
2455 **/
2456static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter)
2457{
2458 int i;
2459
2460 for (i = 0; i < adapter->num_tx_queues; i++)
87e70ab9 2461 if (adapter->tx_ring[i]->desc)
05d063aa 2462 ixgbevf_free_tx_resources(adapter->tx_ring[i]);
92915f71
GR		 2463}
2464
2465/**
2466 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors)
92915f71
GR		 2467 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2468 *
2469 * Return 0 on success, negative on failure
2470 **/
05d063aa 2471int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring)
92915f71 2472{
92915f71
GR		 2473 int size;
2474
2475 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
89bf67f1 2476 tx_ring->tx_buffer_info = vzalloc(size);
92915f71
GR		 2477 if (!tx_ring->tx_buffer_info)
2478 goto err;
92915f71
GR		 2479
2480 /* round up to nearest 4K */
2481 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
2482 tx_ring->size = ALIGN(tx_ring->size, 4096);
2483
05d063aa 2484 tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size,
2a1f8794 2485 &tx_ring->dma, GFP_KERNEL);
92915f71
GR		 2486 if (!tx_ring->desc)
2487 goto err;
2488
92915f71
GR		 2489 return 0;
2490
2491err:
2492 vfree(tx_ring->tx_buffer_info);
2493 tx_ring->tx_buffer_info = NULL;
2494 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit "
2495 "descriptor ring\n");
2496 return -ENOMEM;
2497}
2498
2499/**
2500 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources
2501 * @adapter: board private structure
2502 *
2503 * If this function returns with an error, then it's possible one or
2504 * more of the rings is populated (while the rest are not). It is the
2505 * callers duty to clean those orphaned rings.
2506 *
2507 * Return 0 on success, negative on failure
2508 **/
2509static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter)
2510{
2511 int i, err = 0;
2512
2513 for (i = 0; i < adapter->num_tx_queues; i++) {
05d063aa 2514 err = ixgbevf_setup_tx_resources(adapter->tx_ring[i]);
92915f71
GR		 2515 if (!err)
2516 continue;
2517 hw_dbg(&adapter->hw,
2518 "Allocation for Tx Queue %u failed\n", i);
2519 break;
2520 }
2521
2522 return err;
2523}
2524
2525/**
2526 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors)
92915f71
GR		 2527 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2528 *
2529 * Returns 0 on success, negative on failure
2530 **/
05d063aa 2531int ixgbevf_setup_rx_resources(struct ixgbevf_ring *rx_ring)
92915f71 2532{
92915f71
GR		 2533 int size;
2534
2535 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
89bf67f1 2536 rx_ring->rx_buffer_info = vzalloc(size);
e404decb 2537 if (!rx_ring->rx_buffer_info)
05d063aa 2538 goto err;
92915f71
GR		 2539
2540 /* Round up to nearest 4K */
2541 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
2542 rx_ring->size = ALIGN(rx_ring->size, 4096);
2543
05d063aa 2544 rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size,
2a1f8794 2545 &rx_ring->dma, GFP_KERNEL);
92915f71 2546
05d063aa
ET		 2547 if (!rx_ring->desc)
2548 goto err;
92915f71 2549
92915f71 2550 return 0;
05d063aa
ET		 2551err:
2552 vfree(rx_ring->rx_buffer_info);
2553 rx_ring->rx_buffer_info = NULL;
2554 dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n");
92915f71
GR		 2555 return -ENOMEM;
2556}
2557
2558/**
2559 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources
2560 * @adapter: board private structure
2561 *
2562 * If this function returns with an error, then it's possible one or
2563 * more of the rings is populated (while the rest are not). It is the
2564 * callers duty to clean those orphaned rings.
2565 *
2566 * Return 0 on success, negative on failure
2567 **/
2568static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter)
2569{
2570 int i, err = 0;
2571
2572 for (i = 0; i < adapter->num_rx_queues; i++) {
05d063aa 2573 err = ixgbevf_setup_rx_resources(adapter->rx_ring[i]);
92915f71
GR		 2574 if (!err)
2575 continue;
2576 hw_dbg(&adapter->hw,
2577 "Allocation for Rx Queue %u failed\n", i);
2578 break;
2579 }
2580 return err;
2581}
2582
2583/**
2584 * ixgbevf_free_rx_resources - Free Rx Resources
92915f71
GR		 2585 * @rx_ring: ring to clean the resources from
2586 *
2587 * Free all receive software resources
2588 **/
05d063aa 2589void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring)
92915f71 2590{
05d063aa 2591 ixgbevf_clean_rx_ring(rx_ring);
92915f71
GR		 2592
2593 vfree(rx_ring->rx_buffer_info);
2594 rx_ring->rx_buffer_info = NULL;
2595
05d063aa 2596 dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc,
2a1f8794 2597 rx_ring->dma);
92915f71
GR		 2598
2599 rx_ring->desc = NULL;
2600}
2601
2602/**
2603 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues
2604 * @adapter: board private structure
2605 *
2606 * Free all receive software resources
2607 **/
2608static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter)
2609{
2610 int i;
2611
2612 for (i = 0; i < adapter->num_rx_queues; i++)
87e70ab9 2613 if (adapter->rx_ring[i]->desc)
05d063aa 2614 ixgbevf_free_rx_resources(adapter->rx_ring[i]);
92915f71
GR		 2615}
2616
2617/**
2618 * ixgbevf_open - Called when a network interface is made active
2619 * @netdev: network interface device structure
2620 *
2621 * Returns 0 on success, negative value on failure
2622 *
2623 * The open entry point is called when a network interface is made
2624 * active by the system (IFF_UP). At this point all resources needed
2625 * for transmit and receive operations are allocated, the interrupt
2626 * handler is registered with the OS, the watchdog timer is started,
2627 * and the stack is notified that the interface is ready.
2628 **/
2629static int ixgbevf_open(struct net_device *netdev)
2630{
2631 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2632 struct ixgbe_hw *hw = &adapter->hw;
2633 int err;
2634
a1f6c6b1 2635 /* A previous failure to open the device because of a lack of
2636 * available MSIX vector resources may have reset the number
2637 * of msix vectors variable to zero. The only way to recover
2638 * is to unload/reload the driver and hope that the system has
2639 * been able to recover some MSIX vector resources.
2640 */
2641 if (!adapter->num_msix_vectors)
2642 return -ENOMEM;
2643
92915f71
GR		 2644 /* disallow open during test */
2645 if (test_bit(__IXGBEVF_TESTING, &adapter->state))
2646 return -EBUSY;
2647
2648 if (hw->adapter_stopped) {
2649 ixgbevf_reset(adapter);
2650 /* if adapter is still stopped then PF isn't up and
2651 * the vf can't start. */
2652 if (hw->adapter_stopped) {
2653 err = IXGBE_ERR_MBX;
dbd9636e
JK		 2654 pr_err("Unable to start - perhaps the PF Driver isn't "
2655 "up yet\n");
92915f71
GR		 2656 goto err_setup_reset;
2657 }
2658 }
2659
2660 /* allocate transmit descriptors */
2661 err = ixgbevf_setup_all_tx_resources(adapter);
2662 if (err)
2663 goto err_setup_tx;
2664
2665 /* allocate receive descriptors */
2666 err = ixgbevf_setup_all_rx_resources(adapter);
2667 if (err)
2668 goto err_setup_rx;
2669
2670 ixgbevf_configure(adapter);
2671
2672 /*
2673 * Map the Tx/Rx rings to the vectors we were allotted.
2674 * if request_irq will be called in this function map_rings
2675 * must be called *before* up_complete
2676 */
2677 ixgbevf_map_rings_to_vectors(adapter);
2678
795180d8 2679 ixgbevf_up_complete(adapter);
92915f71
GR		 2680
2681 /* clear any pending interrupts, may auto mask */
2682 IXGBE_READ_REG(hw, IXGBE_VTEICR);
2683 err = ixgbevf_request_irq(adapter);
2684 if (err)
2685 goto err_req_irq;
2686
5f3600eb 2687 ixgbevf_irq_enable(adapter);
92915f71
GR		 2688
2689 return 0;
2690
2691err_req_irq:
2692 ixgbevf_down(adapter);
92915f71
GR		 2693err_setup_rx:
2694 ixgbevf_free_all_rx_resources(adapter);
2695err_setup_tx:
2696 ixgbevf_free_all_tx_resources(adapter);
2697 ixgbevf_reset(adapter);
2698
2699err_setup_reset:
2700
2701 return err;
2702}
2703
2704/**
2705 * ixgbevf_close - Disables a network interface
2706 * @netdev: network interface device structure
2707 *
2708 * Returns 0, this is not allowed to fail
2709 *
2710 * The close entry point is called when an interface is de-activated
2711 * by the OS. The hardware is still under the drivers control, but
2712 * needs to be disabled. A global MAC reset is issued to stop the
2713 * hardware, and all transmit and receive resources are freed.
2714 **/
2715static int ixgbevf_close(struct net_device *netdev)
2716{
2717 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2718
2719 ixgbevf_down(adapter);
2720 ixgbevf_free_irq(adapter);
2721
2722 ixgbevf_free_all_tx_resources(adapter);
2723 ixgbevf_free_all_rx_resources(adapter);
2724
2725 return 0;
2726}
2727
220fe050
DS		 2728static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter)
2729{
2730 struct net_device *dev = adapter->netdev;
2731
2732 if (!(adapter->flags & IXGBEVF_FLAG_QUEUE_RESET_REQUESTED))
2733 return;
2734
2735 adapter->flags &= ~IXGBEVF_FLAG_QUEUE_RESET_REQUESTED;
2736
2737 /* if interface is down do nothing */
2738 if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
2739 test_bit(__IXGBEVF_RESETTING, &adapter->state))
2740 return;
2741
2742 /* Hardware has to reinitialize queues and interrupts to
2743 * match packet buffer alignment. Unfortunately, the
2744 * hardware is not flexible enough to do this dynamically.
2745 */
2746 if (netif_running(dev))
2747 ixgbevf_close(dev);
2748
2749 ixgbevf_clear_interrupt_scheme(adapter);
2750 ixgbevf_init_interrupt_scheme(adapter);
2751
2752 if (netif_running(dev))
2753 ixgbevf_open(dev);
2754}
2755
70a10e25
AD		 2756static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring,
2757 u32 vlan_macip_lens, u32 type_tucmd,
2758 u32 mss_l4len_idx)
92915f71
GR		 2759{
2760 struct ixgbe_adv_tx_context_desc *context_desc;
70a10e25 2761 u16 i = tx_ring->next_to_use;
92915f71 2762
70a10e25 2763 context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i);
92915f71 2764
70a10e25
AD		 2765 i++;
2766 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
92915f71 2767
70a10e25
AD		 2768 /* set bits to identify this as an advanced context descriptor */
2769 type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
92915f71 2770
70a10e25
AD		 2771 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
2772 context_desc->seqnum_seed = 0;
2773 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
2774 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
2775}
2776
2777static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
7ad1a093
ET		 2778 struct ixgbevf_tx_buffer *first,
2779 u8 *hdr_len)
70a10e25 2780{
7ad1a093 2781 struct sk_buff *skb = first->skb;
70a10e25
AD		 2782 u32 vlan_macip_lens, type_tucmd;
2783 u32 mss_l4len_idx, l4len;
2784
01a545cf
ET		 2785 if (skb->ip_summed != CHECKSUM_PARTIAL)
2786 return 0;
2787
70a10e25
AD		 2788 if (!skb_is_gso(skb))
2789 return 0;
92915f71 2790
70a10e25
AD		 2791 if (skb_header_cloned(skb)) {
2792 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2793 if (err)
2794 return err;
92915f71
GR		 2795 }
2796
70a10e25
AD		 2797 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2798 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
2799
2800 if (skb->protocol == htons(ETH_P_IP)) {
2801 struct iphdr *iph = ip_hdr(skb);
2802 iph->tot_len = 0;
2803 iph->check = 0;
2804 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2805 iph->daddr, 0,
2806 IPPROTO_TCP,
2807 0);
2808 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
7ad1a093
ET		 2809 first->tx_flags |= IXGBE_TX_FLAGS_TSO |
2810 IXGBE_TX_FLAGS_CSUM |
2811 IXGBE_TX_FLAGS_IPV4;
70a10e25
AD		 2812 } else if (skb_is_gso_v6(skb)) {
2813 ipv6_hdr(skb)->payload_len = 0;
2814 tcp_hdr(skb)->check =
2815 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2816 &ipv6_hdr(skb)->daddr,
2817 0, IPPROTO_TCP, 0);
7ad1a093
ET		 2818 first->tx_flags |= IXGBE_TX_FLAGS_TSO |
2819 IXGBE_TX_FLAGS_CSUM;
70a10e25
AD		 2820 }
2821
2822 /* compute header lengths */
2823 l4len = tcp_hdrlen(skb);
2824 *hdr_len += l4len;
2825 *hdr_len = skb_transport_offset(skb) + l4len;
2826
7ad1a093
ET		 2827 /* update gso size and bytecount with header size */
2828 first->gso_segs = skb_shinfo(skb)->gso_segs;
2829 first->bytecount += (first->gso_segs - 1) * *hdr_len;
2830
70a10e25
AD		 2831 /* mss_l4len_id: use 1 as index for TSO */
2832 mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT;
2833 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
2834 mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT;
2835
2836 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
2837 vlan_macip_lens = skb_network_header_len(skb);
2838 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
7ad1a093 2839 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
70a10e25
AD		 2840
2841 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2842 type_tucmd, mss_l4len_idx);
2843
2844 return 1;
92915f71
GR		 2845}
2846
7ad1a093
ET		 2847static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
2848 struct ixgbevf_tx_buffer *first)
92915f71 2849{
7ad1a093 2850 struct sk_buff *skb = first->skb;
70a10e25
AD		 2851 u32 vlan_macip_lens = 0;
2852 u32 mss_l4len_idx = 0;
2853 u32 type_tucmd = 0;
92915f71 2854
70a10e25
AD		 2855 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2856 u8 l4_hdr = 0;
2857 switch (skb->protocol) {
2858 case __constant_htons(ETH_P_IP):
2859 vlan_macip_lens |= skb_network_header_len(skb);
2860 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
2861 l4_hdr = ip_hdr(skb)->protocol;
2862 break;
2863 case __constant_htons(ETH_P_IPV6):
2864 vlan_macip_lens |= skb_network_header_len(skb);
2865 l4_hdr = ipv6_hdr(skb)->nexthdr;
2866 break;
2867 default:
2868 if (unlikely(net_ratelimit())) {
2869 dev_warn(tx_ring->dev,
2870 "partial checksum but proto=%x!\n",
7ad1a093 2871 first->protocol);
70a10e25
AD		 2872 }
2873 break;
2874 }
92915f71 2875
70a10e25
AD		 2876 switch (l4_hdr) {
2877 case IPPROTO_TCP:
2878 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2879 mss_l4len_idx = tcp_hdrlen(skb) <<
2880 IXGBE_ADVTXD_L4LEN_SHIFT;
2881 break;
2882 case IPPROTO_SCTP:
2883 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
2884 mss_l4len_idx = sizeof(struct sctphdr) <<
2885 IXGBE_ADVTXD_L4LEN_SHIFT;
2886 break;
2887 case IPPROTO_UDP:
2888 mss_l4len_idx = sizeof(struct udphdr) <<
2889 IXGBE_ADVTXD_L4LEN_SHIFT;
2890 break;
2891 default:
2892 if (unlikely(net_ratelimit())) {
2893 dev_warn(tx_ring->dev,
2894 "partial checksum but l4 proto=%x!\n",
2895 l4_hdr);
2896 }
2897 break;
2898 }
7ad1a093
ET		 2899
2900 /* update TX checksum flag */
2901 first->tx_flags |= IXGBE_TX_FLAGS_CSUM;
92915f71
GR		 2902 }
2903
70a10e25
AD		 2904 /* vlan_macip_lens: MACLEN, VLAN tag */
2905 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
7ad1a093 2906 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
70a10e25
AD		 2907
2908 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2909 type_tucmd, mss_l4len_idx);
92915f71
GR		 2910}
2911
29d37fa1 2912static __le32 ixgbevf_tx_cmd_type(u32 tx_flags)
92915f71 2913{
29d37fa1
ET		 2914 /* set type for advanced descriptor with frame checksum insertion */
2915 __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA |
2916 IXGBE_ADVTXD_DCMD_IFCS |
2917 IXGBE_ADVTXD_DCMD_DEXT);
92915f71 2918
29d37fa1
ET		 2919 /* set HW vlan bit if vlan is present */
2920 if (tx_flags & IXGBE_TX_FLAGS_VLAN)
2921 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE);
92915f71 2922
29d37fa1
ET		 2923 /* set segmentation enable bits for TSO/FSO */
2924 if (tx_flags & IXGBE_TX_FLAGS_TSO)
2925 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE);
92915f71 2926
29d37fa1
ET		 2927 return cmd_type;
2928}
92915f71 2929
29d37fa1
ET		 2930static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc,
2931 u32 tx_flags, unsigned int paylen)
2932{
2933 __le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT);
9bdfefd2 2934
29d37fa1
ET		 2935 /* enable L4 checksum for TSO and TX checksum offload */
2936 if (tx_flags & IXGBE_TX_FLAGS_CSUM)
2937 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM);
92915f71 2938
29d37fa1
ET		 2939 /* enble IPv4 checksum for TSO */
2940 if (tx_flags & IXGBE_TX_FLAGS_IPV4)
2941 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM);
92915f71 2942
29d37fa1
ET		 2943 /* use index 1 context for TSO/FSO/FCOE */
2944 if (tx_flags & IXGBE_TX_FLAGS_TSO)
2945 olinfo_status |= cpu_to_le32(1 << IXGBE_ADVTXD_IDX_SHIFT);
92915f71 2946
29d37fa1
ET		 2947 /* Check Context must be set if Tx switch is enabled, which it
2948 * always is for case where virtual functions are running
2949 */
2950 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC);
92915f71 2951
29d37fa1
ET		 2952 tx_desc->read.olinfo_status = olinfo_status;
2953}
92915f71 2954
29d37fa1
ET		 2955static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
2956 struct ixgbevf_tx_buffer *first,
2957 const u8 hdr_len)
2958{
2959 dma_addr_t dma;
2960 struct sk_buff *skb = first->skb;
2961 struct ixgbevf_tx_buffer *tx_buffer;
2962 union ixgbe_adv_tx_desc *tx_desc;
2963 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
2964 unsigned int data_len = skb->data_len;
2965 unsigned int size = skb_headlen(skb);
2966 unsigned int paylen = skb->len - hdr_len;
2967 u32 tx_flags = first->tx_flags;
2968 __le32 cmd_type;
2969 u16 i = tx_ring->next_to_use;
9bdfefd2 2970
29d37fa1 2971 tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
92915f71 2972
29d37fa1
ET		 2973 ixgbevf_tx_olinfo_status(tx_desc, tx_flags, paylen);
2974 cmd_type = ixgbevf_tx_cmd_type(tx_flags);
7ad1a093 2975
29d37fa1
ET		 2976 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
2977 if (dma_mapping_error(tx_ring->dev, dma))
2978 goto dma_error;
92915f71 2979
29d37fa1
ET		 2980 /* record length, and DMA address */
2981 dma_unmap_len_set(first, len, size);
2982 dma_unmap_addr_set(first, dma, dma);
92915f71 2983
29d37fa1 2984 tx_desc->read.buffer_addr = cpu_to_le64(dma);
92915f71 2985
29d37fa1
ET		 2986 for (;;) {
2987 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) {
2988 tx_desc->read.cmd_type_len =
2989 cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD);
92915f71 2990
29d37fa1
ET		 2991 i++;
2992 tx_desc++;
2993 if (i == tx_ring->count) {
2994 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
2995 i = 0;
2996 }
92915f71 2997
29d37fa1
ET		 2998 dma += IXGBE_MAX_DATA_PER_TXD;
2999 size -= IXGBE_MAX_DATA_PER_TXD;
92915f71 3000
29d37fa1
ET		 3001 tx_desc->read.buffer_addr = cpu_to_le64(dma);
3002 tx_desc->read.olinfo_status = 0;
3003 }
92915f71 3004
29d37fa1
ET		 3005 if (likely(!data_len))
3006 break;
92915f71 3007
29d37fa1 3008 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
92915f71 3009
29d37fa1
ET		 3010 i++;
3011 tx_desc++;
3012 if (i == tx_ring->count) {
3013 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
3014 i = 0;
3015 }
92915f71 3016
29d37fa1
ET		 3017 size = skb_frag_size(frag);
3018 data_len -= size;
92915f71 3019
29d37fa1
ET		 3020 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
3021 DMA_TO_DEVICE);
3022 if (dma_mapping_error(tx_ring->dev, dma))
3023 goto dma_error;
70a10e25 3024
29d37fa1
ET		 3025 tx_buffer = &tx_ring->tx_buffer_info[i];
3026 dma_unmap_len_set(tx_buffer, len, size);
3027 dma_unmap_addr_set(tx_buffer, dma, dma);
92915f71 3028
29d37fa1
ET		 3029 tx_desc->read.buffer_addr = cpu_to_le64(dma);
3030 tx_desc->read.olinfo_status = 0;
3031
3032 frag++;
70a10e25 3033 }
92915f71 3034
29d37fa1
ET		 3035 /* write last descriptor with RS and EOP bits */
3036 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD);
3037 tx_desc->read.cmd_type_len = cmd_type;
3038
3039 /* set the timestamp */
3040 first->time_stamp = jiffies;
3041
3042 /* Force memory writes to complete before letting h/w know there
3043 * are new descriptors to fetch. (Only applicable for weak-ordered
3044 * memory model archs, such as IA-64).
3045 *
3046 * We also need this memory barrier (wmb) to make certain all of the
3047 * status bits have been updated before next_to_watch is written.
70a10e25 3048 */
29d37fa1 3049 wmb();
92915f71 3050
29d37fa1
ET		 3051 /* set next_to_watch value indicating a packet is present */
3052 first->next_to_watch = tx_desc;
92915f71 3053
29d37fa1
ET		 3054 i++;
3055 if (i == tx_ring->count)
3056 i = 0;
9bdfefd2 3057
29d37fa1 3058 tx_ring->next_to_use = i;
92915f71 3059
29d37fa1
ET		 3060 /* notify HW of packet */
3061 writel(i, tx_ring->tail);
3062
3063 return;
3064dma_error:
3065 dev_err(tx_ring->dev, "TX DMA map failed\n");
3066
3067 /* clear dma mappings for failed tx_buffer_info map */
3068 for (;;) {
3069 tx_buffer = &tx_ring->tx_buffer_info[i];
3070 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer);
3071 if (tx_buffer == first)
3072 break;
3073 if (i == 0)
3074 i = tx_ring->count;
3075 i--;
3076 }
92915f71 3077
92915f71 3078 tx_ring->next_to_use = i;
92915f71
GR		 3079}
3080
fb40195c 3081static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
92915f71 3082{
fb40195c 3083 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
92915f71
GR		 3084 /* Herbert's original patch had:
3085 * smp_mb__after_netif_stop_queue();
3086 * but since that doesn't exist yet, just open code it. */
3087 smp_mb();
3088
3089 /* We need to check again in a case another CPU has just
3090 * made room available. */
f880d07b 3091 if (likely(ixgbevf_desc_unused(tx_ring) < size))
92915f71
GR		 3092 return -EBUSY;
3093
3094 /* A reprieve! - use start_queue because it doesn't call schedule */
fb40195c 3095 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
095e2617
ET		 3096 ++tx_ring->tx_stats.restart_queue;
3097
92915f71
GR		 3098 return 0;
3099}
3100
fb40195c 3101static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
92915f71 3102{
f880d07b 3103 if (likely(ixgbevf_desc_unused(tx_ring) >= size))
92915f71 3104 return 0;
fb40195c 3105 return __ixgbevf_maybe_stop_tx(tx_ring, size);
92915f71
GR		 3106}
3107
3108static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
3109{
3110 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
7ad1a093 3111 struct ixgbevf_tx_buffer *first;
92915f71 3112 struct ixgbevf_ring *tx_ring;
7ad1a093
ET		 3113 int tso;
3114 u32 tx_flags = 0;
3595990a
AD		 3115 u16 count = TXD_USE_COUNT(skb_headlen(skb));
3116#if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
3117 unsigned short f;
3118#endif
7ad1a093 3119 u8 hdr_len = 0;
f9d08f16 3120 u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL);
7ad1a093 3121
46acc460 3122 if (!dst_mac || is_link_local_ether_addr(dst_mac)) {
f9d08f16
GR		 3123 dev_kfree_skb(skb);
3124 return NETDEV_TX_OK;
3125 }
92915f71 3126
7ad1a093 3127 tx_ring = adapter->tx_ring[skb->queue_mapping];
92915f71 3128
3595990a
AD		 3129 /*
3130 * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
3131 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,
3132 * + 2 desc gap to keep tail from touching head,
3133 * + 1 desc for context descriptor,
3134 * otherwise try next time
3135 */
3136#if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
3137 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
3138 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
3139#else
3140 count += skb_shinfo(skb)->nr_frags;
3141#endif
fb40195c 3142 if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) {
095e2617 3143 tx_ring->tx_stats.tx_busy++;
3595990a
AD		 3144 return NETDEV_TX_BUSY;
3145 }
3146
7ad1a093
ET		 3147 /* record the location of the first descriptor for this packet */
3148 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
3149 first->skb = skb;
3150 first->bytecount = skb->len;
3151 first->gso_segs = 1;
3152
eab6d18d 3153 if (vlan_tx_tag_present(skb)) {
92915f71
GR		 3154 tx_flags |= vlan_tx_tag_get(skb);
3155 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
3156 tx_flags |= IXGBE_TX_FLAGS_VLAN;
3157 }
3158
7ad1a093
ET		 3159 /* record initial flags and protocol */
3160 first->tx_flags = tx_flags;
3161 first->protocol = vlan_get_protocol(skb);
92915f71 3162
7ad1a093
ET		 3163 tso = ixgbevf_tso(tx_ring, first, &hdr_len);
3164 if (tso < 0)
3165 goto out_drop;
b5d217f3 3166 else if (!tso)
7ad1a093 3167 ixgbevf_tx_csum(tx_ring, first);
92915f71 3168
29d37fa1 3169 ixgbevf_tx_map(tx_ring, first, hdr_len);
70a10e25 3170
fb40195c 3171 ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED);
92915f71 3172
7ad1a093
ET		 3173 return NETDEV_TX_OK;
3174
3175out_drop:
3176 dev_kfree_skb_any(first->skb);
3177 first->skb = NULL;
3178
92915f71
GR		 3179 return NETDEV_TX_OK;
3180}
3181
92915f71
GR		 3182/**
3183 * ixgbevf_set_mac - Change the Ethernet Address of the NIC
3184 * @netdev: network interface device structure
3185 * @p: pointer to an address structure
3186 *
3187 * Returns 0 on success, negative on failure
3188 **/
3189static int ixgbevf_set_mac(struct net_device *netdev, void *p)
3190{
3191 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3192 struct ixgbe_hw *hw = &adapter->hw;
3193 struct sockaddr *addr = p;
3194
3195 if (!is_valid_ether_addr(addr->sa_data))
3196 return -EADDRNOTAVAIL;
3197
3198 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3199 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3200
55fdd45b 3201 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 3202
92fe0bf7 3203 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
92915f71 3204
55fdd45b 3205 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 3206
92915f71
GR		 3207 return 0;
3208}
3209
3210/**
3211 * ixgbevf_change_mtu - Change the Maximum Transfer Unit
3212 * @netdev: network interface device structure
3213 * @new_mtu: new value for maximum frame size
3214 *
3215 * Returns 0 on success, negative on failure
3216 **/
3217static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu)
3218{
3219 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3220 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
69bfbec4 3221 int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE;
69bfbec4 3222
56e94095
AD		 3223 switch (adapter->hw.api_version) {
3224 case ixgbe_mbox_api_11:
69bfbec4 3225 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
56e94095
AD		 3226 break;
3227 default:
3228 if (adapter->hw.mac.type == ixgbe_mac_X540_vf)
3229 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
3230 break;
3231 }
92915f71
GR		 3232
3233 /* MTU < 68 is an error and causes problems on some kernels */
69bfbec4 3234 if ((new_mtu < 68) || (max_frame > max_possible_frame))
92915f71
GR		 3235 return -EINVAL;
3236
3237 hw_dbg(&adapter->hw, "changing MTU from %d to %d\n",
3238 netdev->mtu, new_mtu);
3239 /* must set new MTU before calling down or up */
3240 netdev->mtu = new_mtu;
3241
3242 if (netif_running(netdev))
3243 ixgbevf_reinit_locked(adapter);
3244
3245 return 0;
3246}
3247
0ac1e8ce 3248static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state)
92915f71
GR		 3249{
3250 struct net_device *netdev = pci_get_drvdata(pdev);
3251 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
0ac1e8ce
AD		 3252#ifdef CONFIG_PM
3253 int retval = 0;
3254#endif
92915f71
GR		 3255
3256 netif_device_detach(netdev);
3257
3258 if (netif_running(netdev)) {
0ac1e8ce 3259 rtnl_lock();
92915f71
GR		 3260 ixgbevf_down(adapter);
3261 ixgbevf_free_irq(adapter);
3262 ixgbevf_free_all_tx_resources(adapter);
3263 ixgbevf_free_all_rx_resources(adapter);
0ac1e8ce 3264 rtnl_unlock();
92915f71
GR		 3265 }
3266
0ac1e8ce 3267 ixgbevf_clear_interrupt_scheme(adapter);
92915f71 3268
0ac1e8ce
AD		 3269#ifdef CONFIG_PM
3270 retval = pci_save_state(pdev);
3271 if (retval)
3272 return retval;
92915f71 3273
0ac1e8ce 3274#endif
92915f71 3275 pci_disable_device(pdev);
0ac1e8ce
AD		 3276
3277 return 0;
3278}
3279
3280#ifdef CONFIG_PM
3281static int ixgbevf_resume(struct pci_dev *pdev)
3282{
27ae2967
WY		 3283 struct net_device *netdev = pci_get_drvdata(pdev);
3284 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
0ac1e8ce
AD		 3285 u32 err;
3286
0ac1e8ce
AD		 3287 pci_restore_state(pdev);
3288 /*
3289 * pci_restore_state clears dev->state_saved so call
3290 * pci_save_state to restore it.
3291 */
3292 pci_save_state(pdev);
3293
3294 err = pci_enable_device_mem(pdev);
3295 if (err) {
3296 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n");
3297 return err;
3298 }
3299 pci_set_master(pdev);
3300
798e381a
DS		 3301 ixgbevf_reset(adapter);
3302
0ac1e8ce
AD		 3303 rtnl_lock();
3304 err = ixgbevf_init_interrupt_scheme(adapter);
3305 rtnl_unlock();
3306 if (err) {
3307 dev_err(&pdev->dev, "Cannot initialize interrupts\n");
3308 return err;
3309 }
3310
0ac1e8ce
AD		 3311 if (netif_running(netdev)) {
3312 err = ixgbevf_open(netdev);
3313 if (err)
3314 return err;
3315 }
3316
3317 netif_device_attach(netdev);
3318
3319 return err;
3320}
3321
3322#endif /* CONFIG_PM */
3323static void ixgbevf_shutdown(struct pci_dev *pdev)
3324{
3325 ixgbevf_suspend(pdev, PMSG_SUSPEND);
92915f71
GR		 3326}
3327
4197aa7b
ED		 3328static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev,
3329 struct rtnl_link_stats64 *stats)
3330{
3331 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3332 unsigned int start;
3333 u64 bytes, packets;
3334 const struct ixgbevf_ring *ring;
3335 int i;
3336
3337 ixgbevf_update_stats(adapter);
3338
3339 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc;
3340
3341 for (i = 0; i < adapter->num_rx_queues; i++) {
87e70ab9 3342 ring = adapter->rx_ring[i];
4197aa7b 3343 do {
57a7744e 3344 start = u64_stats_fetch_begin_irq(&ring->syncp);
095e2617
ET		 3345 bytes = ring->stats.bytes;
3346 packets = ring->stats.packets;
57a7744e 3347 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
4197aa7b
ED		 3348 stats->rx_bytes += bytes;
3349 stats->rx_packets += packets;
3350 }
3351
3352 for (i = 0; i < adapter->num_tx_queues; i++) {
87e70ab9 3353 ring = adapter->tx_ring[i];
4197aa7b 3354 do {
57a7744e 3355 start = u64_stats_fetch_begin_irq(&ring->syncp);
095e2617
ET		 3356 bytes = ring->stats.bytes;
3357 packets = ring->stats.packets;
57a7744e 3358 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
4197aa7b
ED		 3359 stats->tx_bytes += bytes;
3360 stats->tx_packets += packets;
3361 }
3362
3363 return stats;
3364}
3365
0ac1e8ce 3366static const struct net_device_ops ixgbevf_netdev_ops = {
c12db769
SH		 3367 .ndo_open = ixgbevf_open,
3368 .ndo_stop = ixgbevf_close,
3369 .ndo_start_xmit = ixgbevf_xmit_frame,
3370 .ndo_set_rx_mode = ixgbevf_set_rx_mode,
4197aa7b 3371 .ndo_get_stats64 = ixgbevf_get_stats,
92915f71 3372 .ndo_validate_addr = eth_validate_addr,
c12db769
SH		 3373 .ndo_set_mac_address = ixgbevf_set_mac,
3374 .ndo_change_mtu = ixgbevf_change_mtu,
3375 .ndo_tx_timeout = ixgbevf_tx_timeout,
c12db769
SH		 3376 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid,
3377 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid,
c777cdfa
JK		 3378#ifdef CONFIG_NET_RX_BUSY_POLL
3379 .ndo_busy_poll = ixgbevf_busy_poll_recv,
3380#endif
92915f71 3381};
92915f71
GR		 3382
3383static void ixgbevf_assign_netdev_ops(struct net_device *dev)
3384{
0ac1e8ce 3385 dev->netdev_ops = &ixgbevf_netdev_ops;
92915f71
GR		 3386 ixgbevf_set_ethtool_ops(dev);
3387 dev->watchdog_timeo = 5 * HZ;
3388}
3389
3390/**
3391 * ixgbevf_probe - Device Initialization Routine
3392 * @pdev: PCI device information struct
3393 * @ent: entry in ixgbevf_pci_tbl
3394 *
3395 * Returns 0 on success, negative on failure
3396 *
3397 * ixgbevf_probe initializes an adapter identified by a pci_dev structure.
3398 * The OS initialization, configuring of the adapter private structure,
3399 * and a hardware reset occur.
3400 **/
1dd06ae8 3401static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
92915f71
GR		 3402{
3403 struct net_device *netdev;
3404 struct ixgbevf_adapter *adapter = NULL;
3405 struct ixgbe_hw *hw = NULL;
3406 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data];
3407 static int cards_found;
3408 int err, pci_using_dac;
3409
3410 err = pci_enable_device(pdev);
3411 if (err)
3412 return err;
3413
53567aa4 3414 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
92915f71
GR		 3415 pci_using_dac = 1;
3416 } else {
53567aa4 3417 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
92915f71 3418 if (err) {
53567aa4
RK		 3419 dev_err(&pdev->dev, "No usable DMA "
3420 "configuration, aborting\n");
3421 goto err_dma;
92915f71
GR		 3422 }
3423 pci_using_dac = 0;
3424 }
3425
3426 err = pci_request_regions(pdev, ixgbevf_driver_name);
3427 if (err) {
3428 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err);
3429 goto err_pci_reg;
3430 }
3431
3432 pci_set_master(pdev);
3433
92915f71
GR		 3434 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter),
3435 MAX_TX_QUEUES);
92915f71
GR		 3436 if (!netdev) {
3437 err = -ENOMEM;
3438 goto err_alloc_etherdev;
3439 }
3440
3441 SET_NETDEV_DEV(netdev, &pdev->dev);
3442
3443 pci_set_drvdata(pdev, netdev);
3444 adapter = netdev_priv(netdev);
3445
3446 adapter->netdev = netdev;
3447 adapter->pdev = pdev;
3448 hw = &adapter->hw;
3449 hw->back = adapter;
b3f4d599 3450 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
92915f71
GR		 3451
3452 /*
3453 * call save state here in standalone driver because it relies on
3454 * adapter struct to exist, and needs to call netdev_priv
3455 */
3456 pci_save_state(pdev);
3457
3458 hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
3459 pci_resource_len(pdev, 0));
3460 if (!hw->hw_addr) {
3461 err = -EIO;
3462 goto err_ioremap;
3463 }
3464
3465 ixgbevf_assign_netdev_ops(netdev);
3466
3467 adapter->bd_number = cards_found;
3468
3469 /* Setup hw api */
3470 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
3471 hw->mac.type = ii->mac;
3472
3473 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops,
f416dfc0 3474 sizeof(struct ixgbe_mbx_operations));
92915f71 3475
92915f71
GR		 3476 /* setup the private structure */
3477 err = ixgbevf_sw_init(adapter);
1a0d6ae5
DK		 3478 if (err)
3479 goto err_sw_init;
3480
3481 /* The HW MAC address was set and/or determined in sw_init */
1a0d6ae5
DK		 3482 if (!is_valid_ether_addr(netdev->dev_addr)) {
3483 pr_err("invalid MAC address\n");
3484 err = -EIO;
3485 goto err_sw_init;
3486 }
92915f71 3487
471a76de 3488 netdev->hw_features = NETIF_F_SG |
92915f71 3489 NETIF_F_IP_CSUM |
471a76de
MM		 3490 NETIF_F_IPV6_CSUM |
3491 NETIF_F_TSO |
3492 NETIF_F_TSO6 |
3493 NETIF_F_RXCSUM;
3494
3495 netdev->features = netdev->hw_features |
f646968f
PM		 3496 NETIF_F_HW_VLAN_CTAG_TX |
3497 NETIF_F_HW_VLAN_CTAG_RX |
3498 NETIF_F_HW_VLAN_CTAG_FILTER;
92915f71 3499
92915f71
GR		 3500 netdev->vlan_features |= NETIF_F_TSO;
3501 netdev->vlan_features |= NETIF_F_TSO6;
3502 netdev->vlan_features |= NETIF_F_IP_CSUM;
3bfacf96 3503 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
92915f71
GR		 3504 netdev->vlan_features |= NETIF_F_SG;
3505
3506 if (pci_using_dac)
3507 netdev->features |= NETIF_F_HIGHDMA;
3508
01789349
JP		 3509 netdev->priv_flags |= IFF_UNICAST_FLT;
3510
92915f71 3511 init_timer(&adapter->watchdog_timer);
c061b18d 3512 adapter->watchdog_timer.function = ixgbevf_watchdog;
92915f71
GR		 3513 adapter->watchdog_timer.data = (unsigned long)adapter;
3514
3515 INIT_WORK(&adapter->reset_task, ixgbevf_reset_task);
3516 INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task);
3517
3518 err = ixgbevf_init_interrupt_scheme(adapter);
3519 if (err)
3520 goto err_sw_init;
3521
92915f71
GR		 3522 strcpy(netdev->name, "eth%d");
3523
3524 err = register_netdev(netdev);
3525 if (err)
3526 goto err_register;
3527
5d426ad1
GR		 3528 netif_carrier_off(netdev);
3529
33bd9f60
GR		 3530 ixgbevf_init_last_counter_stats(adapter);
3531
92915f71 3532 /* print the MAC address */
f794e7ef 3533 hw_dbg(hw, "%pM\n", netdev->dev_addr);
92915f71
GR		 3534
3535 hw_dbg(hw, "MAC: %d\n", hw->mac.type);
3536
92915f71
GR		 3537 hw_dbg(hw, "Intel(R) 82599 Virtual Function\n");
3538 cards_found++;
3539 return 0;
3540
3541err_register:
0ac1e8ce 3542 ixgbevf_clear_interrupt_scheme(adapter);
92915f71
GR		 3543err_sw_init:
3544 ixgbevf_reset_interrupt_capability(adapter);
3545 iounmap(hw->hw_addr);
3546err_ioremap:
3547 free_netdev(netdev);
3548err_alloc_etherdev:
3549 pci_release_regions(pdev);
3550err_pci_reg:
3551err_dma:
3552 pci_disable_device(pdev);
3553 return err;
3554}
3555
3556/**
3557 * ixgbevf_remove - Device Removal Routine
3558 * @pdev: PCI device information struct
3559 *
3560 * ixgbevf_remove is called by the PCI subsystem to alert the driver
3561 * that it should release a PCI device. The could be caused by a
3562 * Hot-Plug event, or because the driver is going to be removed from
3563 * memory.
3564 **/
9f9a12f8 3565static void ixgbevf_remove(struct pci_dev *pdev)
92915f71
GR		 3566{
3567 struct net_device *netdev = pci_get_drvdata(pdev);
3568 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3569
2e7cfbdd 3570 set_bit(__IXGBEVF_REMOVING, &adapter->state);
92915f71
GR		 3571
3572 del_timer_sync(&adapter->watchdog_timer);
3573
23f333a2 3574 cancel_work_sync(&adapter->reset_task);
92915f71
GR		 3575 cancel_work_sync(&adapter->watchdog_task);
3576
fd13a9ab 3577 if (netdev->reg_state == NETREG_REGISTERED)
92915f71 3578 unregister_netdev(netdev);
92915f71 3579
0ac1e8ce 3580 ixgbevf_clear_interrupt_scheme(adapter);
92915f71
GR		 3581 ixgbevf_reset_interrupt_capability(adapter);
3582
3583 iounmap(adapter->hw.hw_addr);
3584 pci_release_regions(pdev);
3585
3586 hw_dbg(&adapter->hw, "Remove complete\n");
3587
92915f71
GR		 3588 free_netdev(netdev);
3589
3590 pci_disable_device(pdev);
3591}
3592
9f19f31d
AD		 3593/**
3594 * ixgbevf_io_error_detected - called when PCI error is detected
3595 * @pdev: Pointer to PCI device
3596 * @state: The current pci connection state
3597 *
3598 * This function is called after a PCI bus error affecting
3599 * this device has been detected.
3600 */
3601static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev,
3602 pci_channel_state_t state)
3603{
3604 struct net_device *netdev = pci_get_drvdata(pdev);
3605 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3606
3607 netif_device_detach(netdev);
3608
3609 if (state == pci_channel_io_perm_failure)
3610 return PCI_ERS_RESULT_DISCONNECT;
3611
3612 if (netif_running(netdev))
3613 ixgbevf_down(adapter);
3614
3615 pci_disable_device(pdev);
3616
3617 /* Request a slot slot reset. */
3618 return PCI_ERS_RESULT_NEED_RESET;
3619}
3620
3621/**
3622 * ixgbevf_io_slot_reset - called after the pci bus has been reset.
3623 * @pdev: Pointer to PCI device
3624 *
3625 * Restart the card from scratch, as if from a cold-boot. Implementation
3626 * resembles the first-half of the ixgbevf_resume routine.
3627 */
3628static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev)
3629{
3630 struct net_device *netdev = pci_get_drvdata(pdev);
3631 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3632
3633 if (pci_enable_device_mem(pdev)) {
3634 dev_err(&pdev->dev,
3635 "Cannot re-enable PCI device after reset.\n");
3636 return PCI_ERS_RESULT_DISCONNECT;
3637 }
3638
3639 pci_set_master(pdev);
3640
3641 ixgbevf_reset(adapter);
3642
3643 return PCI_ERS_RESULT_RECOVERED;
3644}
3645
3646/**
3647 * ixgbevf_io_resume - called when traffic can start flowing again.
3648 * @pdev: Pointer to PCI device
3649 *
3650 * This callback is called when the error recovery driver tells us that
3651 * its OK to resume normal operation. Implementation resembles the
3652 * second-half of the ixgbevf_resume routine.
3653 */
3654static void ixgbevf_io_resume(struct pci_dev *pdev)
3655{
3656 struct net_device *netdev = pci_get_drvdata(pdev);
3657 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3658
3659 if (netif_running(netdev))
3660 ixgbevf_up(adapter);
3661
3662 netif_device_attach(netdev);
3663}
3664
3665/* PCI Error Recovery (ERS) */
3646f0e5 3666static const struct pci_error_handlers ixgbevf_err_handler = {
9f19f31d
AD		 3667 .error_detected = ixgbevf_io_error_detected,
3668 .slot_reset = ixgbevf_io_slot_reset,
3669 .resume = ixgbevf_io_resume,
3670};
3671
92915f71
GR		 3672static struct pci_driver ixgbevf_driver = {
3673 .name = ixgbevf_driver_name,
3674 .id_table = ixgbevf_pci_tbl,
3675 .probe = ixgbevf_probe,
9f9a12f8 3676 .remove = ixgbevf_remove,
0ac1e8ce
AD		 3677#ifdef CONFIG_PM
3678 /* Power Management Hooks */
3679 .suspend = ixgbevf_suspend,
3680 .resume = ixgbevf_resume,
3681#endif
92915f71 3682 .shutdown = ixgbevf_shutdown,
9f19f31d 3683 .err_handler = &ixgbevf_err_handler
92915f71
GR		 3684};
3685
3686/**
65d676c8 3687 * ixgbevf_init_module - Driver Registration Routine
92915f71 3688 *
65d676c8 3689 * ixgbevf_init_module is the first routine called when the driver is
92915f71
GR		 3690 * loaded. All it does is register with the PCI subsystem.
3691 **/
3692static int __init ixgbevf_init_module(void)
3693{
3694 int ret;
dbd9636e
JK		 3695 pr_info("%s - version %s\n", ixgbevf_driver_string,
3696 ixgbevf_driver_version);
92915f71 3697
dbd9636e 3698 pr_info("%s\n", ixgbevf_copyright);
92915f71
GR		 3699
3700 ret = pci_register_driver(&ixgbevf_driver);
3701 return ret;
3702}
3703
3704module_init(ixgbevf_init_module);
3705
3706/**
65d676c8 3707 * ixgbevf_exit_module - Driver Exit Cleanup Routine
92915f71 3708 *
65d676c8 3709 * ixgbevf_exit_module is called just before the driver is removed
92915f71
GR		 3710 * from memory.
3711 **/
3712static void __exit ixgbevf_exit_module(void)
3713{
3714 pci_unregister_driver(&ixgbevf_driver);
3715}
3716
3717#ifdef DEBUG
3718/**
65d676c8 3719 * ixgbevf_get_hw_dev_name - return device name string
92915f71
GR		 3720 * used by hardware layer to print debugging information
3721 **/
3722char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw)
3723{
3724 struct ixgbevf_adapter *adapter = hw->back;
3725 return adapter->netdev->name;
3726}
3727
3728#endif
3729module_exit(ixgbevf_exit_module);
3730
3731/* ixgbevf_main.c */
Ubuntu Artful kernel mirror