1 /****************************************************************************
2 * Driver for Solarflare network controllers and boards
3 * Copyright 2005-2006 Fen Systems Ltd.
4 * Copyright 2005-2013 Solarflare Communications Inc.
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation, incorporated herein by reference.
11 #include <linux/module.h>
12 #include <linux/pci.h>
13 #include <linux/netdevice.h>
14 #include <linux/etherdevice.h>
15 #include <linux/delay.h>
16 #include <linux/notifier.h>
18 #include <linux/tcp.h>
20 #include <linux/ethtool.h>
21 #include <linux/topology.h>
22 #include <linux/gfp.h>
23 #include <linux/aer.h>
24 #include <linux/interrupt.h>
25 #include "net_driver.h"
27 #include <net/udp_tunnel.h>
35 #include "mcdi_pcol.h"
36 #include "workarounds.h"
38 /**************************************************************************
42 **************************************************************************
45 /* Loopback mode names (see LOOPBACK_MODE()) */
46 const unsigned int efx_loopback_mode_max
= LOOPBACK_MAX
;
47 const char *const efx_loopback_mode_names
[] = {
48 [LOOPBACK_NONE
] = "NONE",
49 [LOOPBACK_DATA
] = "DATAPATH",
50 [LOOPBACK_GMAC
] = "GMAC",
51 [LOOPBACK_XGMII
] = "XGMII",
52 [LOOPBACK_XGXS
] = "XGXS",
53 [LOOPBACK_XAUI
] = "XAUI",
54 [LOOPBACK_GMII
] = "GMII",
55 [LOOPBACK_SGMII
] = "SGMII",
56 [LOOPBACK_XGBR
] = "XGBR",
57 [LOOPBACK_XFI
] = "XFI",
58 [LOOPBACK_XAUI_FAR
] = "XAUI_FAR",
59 [LOOPBACK_GMII_FAR
] = "GMII_FAR",
60 [LOOPBACK_SGMII_FAR
] = "SGMII_FAR",
61 [LOOPBACK_XFI_FAR
] = "XFI_FAR",
62 [LOOPBACK_GPHY
] = "GPHY",
63 [LOOPBACK_PHYXS
] = "PHYXS",
64 [LOOPBACK_PCS
] = "PCS",
65 [LOOPBACK_PMAPMD
] = "PMA/PMD",
66 [LOOPBACK_XPORT
] = "XPORT",
67 [LOOPBACK_XGMII_WS
] = "XGMII_WS",
68 [LOOPBACK_XAUI_WS
] = "XAUI_WS",
69 [LOOPBACK_XAUI_WS_FAR
] = "XAUI_WS_FAR",
70 [LOOPBACK_XAUI_WS_NEAR
] = "XAUI_WS_NEAR",
71 [LOOPBACK_GMII_WS
] = "GMII_WS",
72 [LOOPBACK_XFI_WS
] = "XFI_WS",
73 [LOOPBACK_XFI_WS_FAR
] = "XFI_WS_FAR",
74 [LOOPBACK_PHYXS_WS
] = "PHYXS_WS",
77 const unsigned int efx_reset_type_max
= RESET_TYPE_MAX
;
78 const char *const efx_reset_type_names
[] = {
79 [RESET_TYPE_INVISIBLE
] = "INVISIBLE",
80 [RESET_TYPE_ALL
] = "ALL",
81 [RESET_TYPE_RECOVER_OR_ALL
] = "RECOVER_OR_ALL",
82 [RESET_TYPE_WORLD
] = "WORLD",
83 [RESET_TYPE_RECOVER_OR_DISABLE
] = "RECOVER_OR_DISABLE",
84 [RESET_TYPE_DATAPATH
] = "DATAPATH",
85 [RESET_TYPE_MC_BIST
] = "MC_BIST",
86 [RESET_TYPE_DISABLE
] = "DISABLE",
87 [RESET_TYPE_TX_WATCHDOG
] = "TX_WATCHDOG",
88 [RESET_TYPE_INT_ERROR
] = "INT_ERROR",
89 [RESET_TYPE_DMA_ERROR
] = "DMA_ERROR",
90 [RESET_TYPE_TX_SKIP
] = "TX_SKIP",
91 [RESET_TYPE_MC_FAILURE
] = "MC_FAILURE",
92 [RESET_TYPE_MCDI_TIMEOUT
] = "MCDI_TIMEOUT (FLR)",
95 /* UDP tunnel type names */
96 static const char *const efx_udp_tunnel_type_names
[] = {
97 [TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN
] = "vxlan",
98 [TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE
] = "geneve",
101 void efx_get_udp_tunnel_type_name(u16 type
, char *buf
, size_t buflen
)
103 if (type
< ARRAY_SIZE(efx_udp_tunnel_type_names
) &&
104 efx_udp_tunnel_type_names
[type
] != NULL
)
105 snprintf(buf
, buflen
, "%s", efx_udp_tunnel_type_names
[type
]);
107 snprintf(buf
, buflen
, "type %d", type
);
110 /* Reset workqueue. If any NIC has a hardware failure then a reset will be
111 * queued onto this work queue. This is not a per-nic work queue, because
112 * efx_reset_work() acquires the rtnl lock, so resets are naturally serialised.
114 static struct workqueue_struct
*reset_workqueue
;
116 /* How often and how many times to poll for a reset while waiting for a
117 * BIST that another function started to complete.
119 #define BIST_WAIT_DELAY_MS 100
120 #define BIST_WAIT_DELAY_COUNT 100
122 /**************************************************************************
124 * Configurable values
126 *************************************************************************/
129 * Use separate channels for TX and RX events
131 * Set this to 1 to use separate channels for TX and RX. It allows us
132 * to control interrupt affinity separately for TX and RX.
134 * This is only used in MSI-X interrupt mode
136 bool efx_separate_tx_channels
;
137 module_param(efx_separate_tx_channels
, bool, 0444);
138 MODULE_PARM_DESC(efx_separate_tx_channels
,
139 "Use separate channels for TX and RX");
141 /* This is the weight assigned to each of the (per-channel) virtual
144 static int napi_weight
= 64;
146 /* This is the time (in jiffies) between invocations of the hardware
148 * On Falcon-based NICs, this will:
149 * - Check the on-board hardware monitor;
150 * - Poll the link state and reconfigure the hardware as necessary.
151 * On Siena-based NICs for power systems with EEH support, this will give EEH a
154 static unsigned int efx_monitor_interval
= 1 * HZ
;
156 /* Initial interrupt moderation settings. They can be modified after
157 * module load with ethtool.
159 * The default for RX should strike a balance between increasing the
160 * round-trip latency and reducing overhead.
162 static unsigned int rx_irq_mod_usec
= 60;
164 /* Initial interrupt moderation settings. They can be modified after
165 * module load with ethtool.
167 * This default is chosen to ensure that a 10G link does not go idle
168 * while a TX queue is stopped after it has become full. A queue is
169 * restarted when it drops below half full. The time this takes (assuming
170 * worst case 3 descriptors per packet and 1024 descriptors) is
171 * 512 / 3 * 1.2 = 205 usec.
173 static unsigned int tx_irq_mod_usec
= 150;
175 /* This is the first interrupt mode to try out of:
180 static unsigned int interrupt_mode
;
182 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
183 * i.e. the number of CPUs among which we may distribute simultaneous
184 * interrupt handling.
186 * Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
187 * The default (0) means to assign an interrupt to each core.
189 static unsigned int rss_cpus
;
190 module_param(rss_cpus
, uint
, 0444);
191 MODULE_PARM_DESC(rss_cpus
, "Number of CPUs to use for Receive-Side Scaling");
193 static bool phy_flash_cfg
;
194 module_param(phy_flash_cfg
, bool, 0644);
195 MODULE_PARM_DESC(phy_flash_cfg
, "Set PHYs into reflash mode initially");
197 static unsigned irq_adapt_low_thresh
= 8000;
198 module_param(irq_adapt_low_thresh
, uint
, 0644);
199 MODULE_PARM_DESC(irq_adapt_low_thresh
,
200 "Threshold score for reducing IRQ moderation");
202 static unsigned irq_adapt_high_thresh
= 16000;
203 module_param(irq_adapt_high_thresh
, uint
, 0644);
204 MODULE_PARM_DESC(irq_adapt_high_thresh
,
205 "Threshold score for increasing IRQ moderation");
207 static unsigned debug
= (NETIF_MSG_DRV
| NETIF_MSG_PROBE
|
208 NETIF_MSG_LINK
| NETIF_MSG_IFDOWN
|
209 NETIF_MSG_IFUP
| NETIF_MSG_RX_ERR
|
210 NETIF_MSG_TX_ERR
| NETIF_MSG_HW
);
211 module_param(debug
, uint
, 0);
212 MODULE_PARM_DESC(debug
, "Bitmapped debugging message enable value");
214 /**************************************************************************
216 * Utility functions and prototypes
218 *************************************************************************/
220 static int efx_soft_enable_interrupts(struct efx_nic
*efx
);
221 static void efx_soft_disable_interrupts(struct efx_nic
*efx
);
222 static void efx_remove_channel(struct efx_channel
*channel
);
223 static void efx_remove_channels(struct efx_nic
*efx
);
224 static const struct efx_channel_type efx_default_channel_type
;
225 static void efx_remove_port(struct efx_nic
*efx
);
226 static void efx_init_napi_channel(struct efx_channel
*channel
);
227 static void efx_fini_napi(struct efx_nic
*efx
);
228 static void efx_fini_napi_channel(struct efx_channel
*channel
);
229 static void efx_fini_struct(struct efx_nic
*efx
);
230 static void efx_start_all(struct efx_nic
*efx
);
231 static void efx_stop_all(struct efx_nic
*efx
);
233 #define EFX_ASSERT_RESET_SERIALISED(efx) \
235 if ((efx->state == STATE_READY) || \
236 (efx->state == STATE_RECOVERY) || \
237 (efx->state == STATE_DISABLED)) \
241 static int efx_check_disabled(struct efx_nic
*efx
)
243 if (efx
->state
== STATE_DISABLED
|| efx
->state
== STATE_RECOVERY
) {
244 netif_err(efx
, drv
, efx
->net_dev
,
245 "device is disabled due to earlier errors\n");
251 /**************************************************************************
253 * Event queue processing
255 *************************************************************************/
257 /* Process channel's event queue
259 * This function is responsible for processing the event queue of a
260 * single channel. The caller must guarantee that this function will
261 * never be concurrently called more than once on the same channel,
262 * though different channels may be being processed concurrently.
264 static int efx_process_channel(struct efx_channel
*channel
, int budget
)
266 struct efx_tx_queue
*tx_queue
;
267 struct list_head rx_list
;
270 if (unlikely(!channel
->enabled
))
273 /* Prepare the batch receive list */
274 EFX_WARN_ON_PARANOID(channel
->rx_list
!= NULL
);
275 INIT_LIST_HEAD(&rx_list
);
276 channel
->rx_list
= &rx_list
;
278 efx_for_each_channel_tx_queue(tx_queue
, channel
) {
279 tx_queue
->pkts_compl
= 0;
280 tx_queue
->bytes_compl
= 0;
283 spent
= efx_nic_process_eventq(channel
, budget
);
284 if (spent
&& efx_channel_has_rx_queue(channel
)) {
285 struct efx_rx_queue
*rx_queue
=
286 efx_channel_get_rx_queue(channel
);
288 efx_rx_flush_packet(channel
);
289 efx_fast_push_rx_descriptors(rx_queue
, true);
293 efx_for_each_channel_tx_queue(tx_queue
, channel
) {
294 if (tx_queue
->bytes_compl
) {
295 netdev_tx_completed_queue(tx_queue
->core_txq
,
296 tx_queue
->pkts_compl
, tx_queue
->bytes_compl
);
300 /* Receive any packets we queued up */
301 netif_receive_skb_list(channel
->rx_list
);
302 channel
->rx_list
= NULL
;
309 * NAPI guarantees serialisation of polls of the same device, which
310 * provides the guarantee required by efx_process_channel().
312 static void efx_update_irq_mod(struct efx_nic
*efx
, struct efx_channel
*channel
)
314 int step
= efx
->irq_mod_step_us
;
316 if (channel
->irq_mod_score
< irq_adapt_low_thresh
) {
317 if (channel
->irq_moderation_us
> step
) {
318 channel
->irq_moderation_us
-= step
;
319 efx
->type
->push_irq_moderation(channel
);
321 } else if (channel
->irq_mod_score
> irq_adapt_high_thresh
) {
322 if (channel
->irq_moderation_us
<
323 efx
->irq_rx_moderation_us
) {
324 channel
->irq_moderation_us
+= step
;
325 efx
->type
->push_irq_moderation(channel
);
329 channel
->irq_count
= 0;
330 channel
->irq_mod_score
= 0;
333 static int efx_poll(struct napi_struct
*napi
, int budget
)
335 struct efx_channel
*channel
=
336 container_of(napi
, struct efx_channel
, napi_str
);
337 struct efx_nic
*efx
= channel
->efx
;
340 netif_vdbg(efx
, intr
, efx
->net_dev
,
341 "channel %d NAPI poll executing on CPU %d\n",
342 channel
->channel
, raw_smp_processor_id());
344 spent
= efx_process_channel(channel
, budget
);
346 if (spent
< budget
) {
347 if (efx_channel_has_rx_queue(channel
) &&
348 efx
->irq_rx_adaptive
&&
349 unlikely(++channel
->irq_count
== 1000)) {
350 efx_update_irq_mod(efx
, channel
);
353 #ifdef CONFIG_RFS_ACCEL
354 /* Perhaps expire some ARFS filters */
355 schedule_work(&channel
->filter_work
);
358 /* There is no race here; although napi_disable() will
359 * only wait for napi_complete(), this isn't a problem
360 * since efx_nic_eventq_read_ack() will have no effect if
361 * interrupts have already been disabled.
363 if (napi_complete_done(napi
, spent
))
364 efx_nic_eventq_read_ack(channel
);
370 /* Create event queue
371 * Event queue memory allocations are done only once. If the channel
372 * is reset, the memory buffer will be reused; this guards against
373 * errors during channel reset and also simplifies interrupt handling.
375 static int efx_probe_eventq(struct efx_channel
*channel
)
377 struct efx_nic
*efx
= channel
->efx
;
378 unsigned long entries
;
380 netif_dbg(efx
, probe
, efx
->net_dev
,
381 "chan %d create event queue\n", channel
->channel
);
383 /* Build an event queue with room for one event per tx and rx buffer,
384 * plus some extra for link state events and MCDI completions. */
385 entries
= roundup_pow_of_two(efx
->rxq_entries
+ efx
->txq_entries
+ 128);
386 EFX_WARN_ON_PARANOID(entries
> EFX_MAX_EVQ_SIZE
);
387 channel
->eventq_mask
= max(entries
, EFX_MIN_EVQ_SIZE
) - 1;
389 return efx_nic_probe_eventq(channel
);
392 /* Prepare channel's event queue */
393 static int efx_init_eventq(struct efx_channel
*channel
)
395 struct efx_nic
*efx
= channel
->efx
;
398 EFX_WARN_ON_PARANOID(channel
->eventq_init
);
400 netif_dbg(efx
, drv
, efx
->net_dev
,
401 "chan %d init event queue\n", channel
->channel
);
403 rc
= efx_nic_init_eventq(channel
);
405 efx
->type
->push_irq_moderation(channel
);
406 channel
->eventq_read_ptr
= 0;
407 channel
->eventq_init
= true;
412 /* Enable event queue processing and NAPI */
413 void efx_start_eventq(struct efx_channel
*channel
)
415 netif_dbg(channel
->efx
, ifup
, channel
->efx
->net_dev
,
416 "chan %d start event queue\n", channel
->channel
);
418 /* Make sure the NAPI handler sees the enabled flag set */
419 channel
->enabled
= true;
422 napi_enable(&channel
->napi_str
);
423 efx_nic_eventq_read_ack(channel
);
426 /* Disable event queue processing and NAPI */
427 void efx_stop_eventq(struct efx_channel
*channel
)
429 if (!channel
->enabled
)
432 napi_disable(&channel
->napi_str
);
433 channel
->enabled
= false;
436 static void efx_fini_eventq(struct efx_channel
*channel
)
438 if (!channel
->eventq_init
)
441 netif_dbg(channel
->efx
, drv
, channel
->efx
->net_dev
,
442 "chan %d fini event queue\n", channel
->channel
);
444 efx_nic_fini_eventq(channel
);
445 channel
->eventq_init
= false;
448 static void efx_remove_eventq(struct efx_channel
*channel
)
450 netif_dbg(channel
->efx
, drv
, channel
->efx
->net_dev
,
451 "chan %d remove event queue\n", channel
->channel
);
453 efx_nic_remove_eventq(channel
);
456 /**************************************************************************
460 *************************************************************************/
462 /* Allocate and initialise a channel structure. */
463 static struct efx_channel
*
464 efx_alloc_channel(struct efx_nic
*efx
, int i
, struct efx_channel
*old_channel
)
466 struct efx_channel
*channel
;
467 struct efx_rx_queue
*rx_queue
;
468 struct efx_tx_queue
*tx_queue
;
471 channel
= kzalloc(sizeof(*channel
), GFP_KERNEL
);
476 channel
->channel
= i
;
477 channel
->type
= &efx_default_channel_type
;
479 for (j
= 0; j
< EFX_TXQ_TYPES
; j
++) {
480 tx_queue
= &channel
->tx_queue
[j
];
482 tx_queue
->queue
= i
* EFX_TXQ_TYPES
+ j
;
483 tx_queue
->channel
= channel
;
486 #ifdef CONFIG_RFS_ACCEL
487 INIT_WORK(&channel
->filter_work
, efx_filter_rfs_expire
);
490 rx_queue
= &channel
->rx_queue
;
492 timer_setup(&rx_queue
->slow_fill
, efx_rx_slow_fill
, 0);
497 /* Allocate and initialise a channel structure, copying parameters
498 * (but not resources) from an old channel structure.
500 static struct efx_channel
*
501 efx_copy_channel(const struct efx_channel
*old_channel
)
503 struct efx_channel
*channel
;
504 struct efx_rx_queue
*rx_queue
;
505 struct efx_tx_queue
*tx_queue
;
508 channel
= kmalloc(sizeof(*channel
), GFP_KERNEL
);
512 *channel
= *old_channel
;
514 channel
->napi_dev
= NULL
;
515 INIT_HLIST_NODE(&channel
->napi_str
.napi_hash_node
);
516 channel
->napi_str
.napi_id
= 0;
517 channel
->napi_str
.state
= 0;
518 memset(&channel
->eventq
, 0, sizeof(channel
->eventq
));
520 for (j
= 0; j
< EFX_TXQ_TYPES
; j
++) {
521 tx_queue
= &channel
->tx_queue
[j
];
522 if (tx_queue
->channel
)
523 tx_queue
->channel
= channel
;
524 tx_queue
->buffer
= NULL
;
525 memset(&tx_queue
->txd
, 0, sizeof(tx_queue
->txd
));
528 rx_queue
= &channel
->rx_queue
;
529 rx_queue
->buffer
= NULL
;
530 memset(&rx_queue
->rxd
, 0, sizeof(rx_queue
->rxd
));
531 timer_setup(&rx_queue
->slow_fill
, efx_rx_slow_fill
, 0);
532 #ifdef CONFIG_RFS_ACCEL
533 INIT_WORK(&channel
->filter_work
, efx_filter_rfs_expire
);
539 static int efx_probe_channel(struct efx_channel
*channel
)
541 struct efx_tx_queue
*tx_queue
;
542 struct efx_rx_queue
*rx_queue
;
545 netif_dbg(channel
->efx
, probe
, channel
->efx
->net_dev
,
546 "creating channel %d\n", channel
->channel
);
548 rc
= channel
->type
->pre_probe(channel
);
552 rc
= efx_probe_eventq(channel
);
556 efx_for_each_channel_tx_queue(tx_queue
, channel
) {
557 rc
= efx_probe_tx_queue(tx_queue
);
562 efx_for_each_channel_rx_queue(rx_queue
, channel
) {
563 rc
= efx_probe_rx_queue(rx_queue
);
568 channel
->rx_list
= NULL
;
573 efx_remove_channel(channel
);
578 efx_get_channel_name(struct efx_channel
*channel
, char *buf
, size_t len
)
580 struct efx_nic
*efx
= channel
->efx
;
584 number
= channel
->channel
;
585 if (efx
->tx_channel_offset
== 0) {
587 } else if (channel
->channel
< efx
->tx_channel_offset
) {
591 number
-= efx
->tx_channel_offset
;
593 snprintf(buf
, len
, "%s%s-%d", efx
->name
, type
, number
);
596 static void efx_set_channel_names(struct efx_nic
*efx
)
598 struct efx_channel
*channel
;
600 efx_for_each_channel(channel
, efx
)
601 channel
->type
->get_name(channel
,
602 efx
->msi_context
[channel
->channel
].name
,
603 sizeof(efx
->msi_context
[0].name
));
606 static int efx_probe_channels(struct efx_nic
*efx
)
608 struct efx_channel
*channel
;
611 /* Restart special buffer allocation */
612 efx
->next_buffer_table
= 0;
614 /* Probe channels in reverse, so that any 'extra' channels
615 * use the start of the buffer table. This allows the traffic
616 * channels to be resized without moving them or wasting the
617 * entries before them.
619 efx_for_each_channel_rev(channel
, efx
) {
620 rc
= efx_probe_channel(channel
);
622 netif_err(efx
, probe
, efx
->net_dev
,
623 "failed to create channel %d\n",
628 efx_set_channel_names(efx
);
633 efx_remove_channels(efx
);
637 /* Channels are shutdown and reinitialised whilst the NIC is running
638 * to propagate configuration changes (mtu, checksum offload), or
639 * to clear hardware error conditions
641 static void efx_start_datapath(struct efx_nic
*efx
)
643 netdev_features_t old_features
= efx
->net_dev
->features
;
644 bool old_rx_scatter
= efx
->rx_scatter
;
645 struct efx_tx_queue
*tx_queue
;
646 struct efx_rx_queue
*rx_queue
;
647 struct efx_channel
*channel
;
650 /* Calculate the rx buffer allocation parameters required to
651 * support the current MTU, including padding for header
652 * alignment and overruns.
654 efx
->rx_dma_len
= (efx
->rx_prefix_size
+
655 EFX_MAX_FRAME_LEN(efx
->net_dev
->mtu
) +
656 efx
->type
->rx_buffer_padding
);
657 rx_buf_len
= (sizeof(struct efx_rx_page_state
) +
658 efx
->rx_ip_align
+ efx
->rx_dma_len
);
659 if (rx_buf_len
<= PAGE_SIZE
) {
660 efx
->rx_scatter
= efx
->type
->always_rx_scatter
;
661 efx
->rx_buffer_order
= 0;
662 } else if (efx
->type
->can_rx_scatter
) {
663 BUILD_BUG_ON(EFX_RX_USR_BUF_SIZE
% L1_CACHE_BYTES
);
664 BUILD_BUG_ON(sizeof(struct efx_rx_page_state
) +
665 2 * ALIGN(NET_IP_ALIGN
+ EFX_RX_USR_BUF_SIZE
,
666 EFX_RX_BUF_ALIGNMENT
) >
668 efx
->rx_scatter
= true;
669 efx
->rx_dma_len
= EFX_RX_USR_BUF_SIZE
;
670 efx
->rx_buffer_order
= 0;
672 efx
->rx_scatter
= false;
673 efx
->rx_buffer_order
= get_order(rx_buf_len
);
676 efx_rx_config_page_split(efx
);
677 if (efx
->rx_buffer_order
)
678 netif_dbg(efx
, drv
, efx
->net_dev
,
679 "RX buf len=%u; page order=%u batch=%u\n",
680 efx
->rx_dma_len
, efx
->rx_buffer_order
,
681 efx
->rx_pages_per_batch
);
683 netif_dbg(efx
, drv
, efx
->net_dev
,
684 "RX buf len=%u step=%u bpp=%u; page batch=%u\n",
685 efx
->rx_dma_len
, efx
->rx_page_buf_step
,
686 efx
->rx_bufs_per_page
, efx
->rx_pages_per_batch
);
688 /* Restore previously fixed features in hw_features and remove
689 * features which are fixed now
691 efx
->net_dev
->hw_features
|= efx
->net_dev
->features
;
692 efx
->net_dev
->hw_features
&= ~efx
->fixed_features
;
693 efx
->net_dev
->features
|= efx
->fixed_features
;
694 if (efx
->net_dev
->features
!= old_features
)
695 netdev_features_change(efx
->net_dev
);
697 /* RX filters may also have scatter-enabled flags */
698 if (efx
->rx_scatter
!= old_rx_scatter
)
699 efx
->type
->filter_update_rx_scatter(efx
);
701 /* We must keep at least one descriptor in a TX ring empty.
702 * We could avoid this when the queue size does not exactly
703 * match the hardware ring size, but it's not that important.
704 * Therefore we stop the queue when one more skb might fill
705 * the ring completely. We wake it when half way back to
708 efx
->txq_stop_thresh
= efx
->txq_entries
- efx_tx_max_skb_descs(efx
);
709 efx
->txq_wake_thresh
= efx
->txq_stop_thresh
/ 2;
711 /* Initialise the channels */
712 efx_for_each_channel(channel
, efx
) {
713 efx_for_each_channel_tx_queue(tx_queue
, channel
) {
714 efx_init_tx_queue(tx_queue
);
715 atomic_inc(&efx
->active_queues
);
718 efx_for_each_channel_rx_queue(rx_queue
, channel
) {
719 efx_init_rx_queue(rx_queue
);
720 atomic_inc(&efx
->active_queues
);
721 efx_stop_eventq(channel
);
722 efx_fast_push_rx_descriptors(rx_queue
, false);
723 efx_start_eventq(channel
);
726 WARN_ON(channel
->rx_pkt_n_frags
);
729 efx_ptp_start_datapath(efx
);
731 if (netif_device_present(efx
->net_dev
))
732 netif_tx_wake_all_queues(efx
->net_dev
);
735 static void efx_stop_datapath(struct efx_nic
*efx
)
737 struct efx_channel
*channel
;
738 struct efx_tx_queue
*tx_queue
;
739 struct efx_rx_queue
*rx_queue
;
742 EFX_ASSERT_RESET_SERIALISED(efx
);
743 BUG_ON(efx
->port_enabled
);
745 efx_ptp_stop_datapath(efx
);
748 efx_for_each_channel(channel
, efx
) {
749 efx_for_each_channel_rx_queue(rx_queue
, channel
)
750 rx_queue
->refill_enabled
= false;
753 efx_for_each_channel(channel
, efx
) {
754 /* RX packet processing is pipelined, so wait for the
755 * NAPI handler to complete. At least event queue 0
756 * might be kept active by non-data events, so don't
757 * use napi_synchronize() but actually disable NAPI
760 if (efx_channel_has_rx_queue(channel
)) {
761 efx_stop_eventq(channel
);
762 efx_start_eventq(channel
);
766 rc
= efx
->type
->fini_dmaq(efx
);
768 netif_err(efx
, drv
, efx
->net_dev
, "failed to flush queues\n");
770 netif_dbg(efx
, drv
, efx
->net_dev
,
771 "successfully flushed all queues\n");
774 efx_for_each_channel(channel
, efx
) {
775 efx_for_each_channel_rx_queue(rx_queue
, channel
)
776 efx_fini_rx_queue(rx_queue
);
777 efx_for_each_possible_channel_tx_queue(tx_queue
, channel
)
778 efx_fini_tx_queue(tx_queue
);
782 static void efx_remove_channel(struct efx_channel
*channel
)
784 struct efx_tx_queue
*tx_queue
;
785 struct efx_rx_queue
*rx_queue
;
787 netif_dbg(channel
->efx
, drv
, channel
->efx
->net_dev
,
788 "destroy chan %d\n", channel
->channel
);
790 efx_for_each_channel_rx_queue(rx_queue
, channel
)
791 efx_remove_rx_queue(rx_queue
);
792 efx_for_each_possible_channel_tx_queue(tx_queue
, channel
)
793 efx_remove_tx_queue(tx_queue
);
794 efx_remove_eventq(channel
);
795 channel
->type
->post_remove(channel
);
798 static void efx_remove_channels(struct efx_nic
*efx
)
800 struct efx_channel
*channel
;
802 efx_for_each_channel(channel
, efx
)
803 efx_remove_channel(channel
);
807 efx_realloc_channels(struct efx_nic
*efx
, u32 rxq_entries
, u32 txq_entries
)
809 struct efx_channel
*other_channel
[EFX_MAX_CHANNELS
], *channel
;
810 u32 old_rxq_entries
, old_txq_entries
;
811 unsigned i
, next_buffer_table
= 0;
814 rc
= efx_check_disabled(efx
);
818 /* Not all channels should be reallocated. We must avoid
819 * reallocating their buffer table entries.
821 efx_for_each_channel(channel
, efx
) {
822 struct efx_rx_queue
*rx_queue
;
823 struct efx_tx_queue
*tx_queue
;
825 if (channel
->type
->copy
)
827 next_buffer_table
= max(next_buffer_table
,
828 channel
->eventq
.index
+
829 channel
->eventq
.entries
);
830 efx_for_each_channel_rx_queue(rx_queue
, channel
)
831 next_buffer_table
= max(next_buffer_table
,
832 rx_queue
->rxd
.index
+
833 rx_queue
->rxd
.entries
);
834 efx_for_each_channel_tx_queue(tx_queue
, channel
)
835 next_buffer_table
= max(next_buffer_table
,
836 tx_queue
->txd
.index
+
837 tx_queue
->txd
.entries
);
840 efx_device_detach_sync(efx
);
842 efx_soft_disable_interrupts(efx
);
844 /* Clone channels (where possible) */
845 memset(other_channel
, 0, sizeof(other_channel
));
846 for (i
= 0; i
< efx
->n_channels
; i
++) {
847 channel
= efx
->channel
[i
];
848 if (channel
->type
->copy
)
849 channel
= channel
->type
->copy(channel
);
854 other_channel
[i
] = channel
;
857 /* Swap entry counts and channel pointers */
858 old_rxq_entries
= efx
->rxq_entries
;
859 old_txq_entries
= efx
->txq_entries
;
860 efx
->rxq_entries
= rxq_entries
;
861 efx
->txq_entries
= txq_entries
;
862 for (i
= 0; i
< efx
->n_channels
; i
++) {
863 channel
= efx
->channel
[i
];
864 efx
->channel
[i
] = other_channel
[i
];
865 other_channel
[i
] = channel
;
868 /* Restart buffer table allocation */
869 efx
->next_buffer_table
= next_buffer_table
;
871 for (i
= 0; i
< efx
->n_channels
; i
++) {
872 channel
= efx
->channel
[i
];
873 if (!channel
->type
->copy
)
875 rc
= efx_probe_channel(channel
);
878 efx_init_napi_channel(efx
->channel
[i
]);
882 /* Destroy unused channel structures */
883 for (i
= 0; i
< efx
->n_channels
; i
++) {
884 channel
= other_channel
[i
];
885 if (channel
&& channel
->type
->copy
) {
886 efx_fini_napi_channel(channel
);
887 efx_remove_channel(channel
);
892 rc2
= efx_soft_enable_interrupts(efx
);
895 netif_err(efx
, drv
, efx
->net_dev
,
896 "unable to restart interrupts on channel reallocation\n");
897 efx_schedule_reset(efx
, RESET_TYPE_DISABLE
);
900 efx_device_attach_if_not_resetting(efx
);
906 efx
->rxq_entries
= old_rxq_entries
;
907 efx
->txq_entries
= old_txq_entries
;
908 for (i
= 0; i
< efx
->n_channels
; i
++) {
909 channel
= efx
->channel
[i
];
910 efx
->channel
[i
] = other_channel
[i
];
911 other_channel
[i
] = channel
;
916 void efx_schedule_slow_fill(struct efx_rx_queue
*rx_queue
)
918 mod_timer(&rx_queue
->slow_fill
, jiffies
+ msecs_to_jiffies(10));
921 static bool efx_default_channel_want_txqs(struct efx_channel
*channel
)
923 return channel
->channel
- channel
->efx
->tx_channel_offset
<
924 channel
->efx
->n_tx_channels
;
927 static const struct efx_channel_type efx_default_channel_type
= {
928 .pre_probe
= efx_channel_dummy_op_int
,
929 .post_remove
= efx_channel_dummy_op_void
,
930 .get_name
= efx_get_channel_name
,
931 .copy
= efx_copy_channel
,
932 .want_txqs
= efx_default_channel_want_txqs
,
933 .keep_eventq
= false,
937 int efx_channel_dummy_op_int(struct efx_channel
*channel
)
942 void efx_channel_dummy_op_void(struct efx_channel
*channel
)
946 /**************************************************************************
950 **************************************************************************/
952 /* This ensures that the kernel is kept informed (via
953 * netif_carrier_on/off) of the link status, and also maintains the
954 * link status's stop on the port's TX queue.
956 void efx_link_status_changed(struct efx_nic
*efx
)
958 struct efx_link_state
*link_state
= &efx
->link_state
;
960 /* SFC Bug 5356: A net_dev notifier is registered, so we must ensure
961 * that no events are triggered between unregister_netdev() and the
962 * driver unloading. A more general condition is that NETDEV_CHANGE
963 * can only be generated between NETDEV_UP and NETDEV_DOWN */
964 if (!netif_running(efx
->net_dev
))
967 if (link_state
->up
!= netif_carrier_ok(efx
->net_dev
)) {
968 efx
->n_link_state_changes
++;
971 netif_carrier_on(efx
->net_dev
);
973 netif_carrier_off(efx
->net_dev
);
976 /* Status message for kernel log */
978 netif_info(efx
, link
, efx
->net_dev
,
979 "link up at %uMbps %s-duplex (MTU %d)\n",
980 link_state
->speed
, link_state
->fd
? "full" : "half",
983 netif_info(efx
, link
, efx
->net_dev
, "link down\n");
986 void efx_link_set_advertising(struct efx_nic
*efx
,
987 const unsigned long *advertising
)
989 memcpy(efx
->link_advertising
, advertising
,
990 sizeof(__ETHTOOL_DECLARE_LINK_MODE_MASK()));
992 efx
->link_advertising
[0] |= ADVERTISED_Autoneg
;
993 if (advertising
[0] & ADVERTISED_Pause
)
994 efx
->wanted_fc
|= (EFX_FC_TX
| EFX_FC_RX
);
996 efx
->wanted_fc
&= ~(EFX_FC_TX
| EFX_FC_RX
);
997 if (advertising
[0] & ADVERTISED_Asym_Pause
)
998 efx
->wanted_fc
^= EFX_FC_TX
;
1001 /* Equivalent to efx_link_set_advertising with all-zeroes, except does not
1002 * force the Autoneg bit on.
1004 void efx_link_clear_advertising(struct efx_nic
*efx
)
1006 bitmap_zero(efx
->link_advertising
, __ETHTOOL_LINK_MODE_MASK_NBITS
);
1007 efx
->wanted_fc
&= ~(EFX_FC_TX
| EFX_FC_RX
);
1010 void efx_link_set_wanted_fc(struct efx_nic
*efx
, u8 wanted_fc
)
1012 efx
->wanted_fc
= wanted_fc
;
1013 if (efx
->link_advertising
[0]) {
1014 if (wanted_fc
& EFX_FC_RX
)
1015 efx
->link_advertising
[0] |= (ADVERTISED_Pause
|
1016 ADVERTISED_Asym_Pause
);
1018 efx
->link_advertising
[0] &= ~(ADVERTISED_Pause
|
1019 ADVERTISED_Asym_Pause
);
1020 if (wanted_fc
& EFX_FC_TX
)
1021 efx
->link_advertising
[0] ^= ADVERTISED_Asym_Pause
;
1025 static void efx_fini_port(struct efx_nic
*efx
);
1027 /* We assume that efx->type->reconfigure_mac will always try to sync RX
1028 * filters and therefore needs to read-lock the filter table against freeing
1030 void efx_mac_reconfigure(struct efx_nic
*efx
)
1032 down_read(&efx
->filter_sem
);
1033 efx
->type
->reconfigure_mac(efx
);
1034 up_read(&efx
->filter_sem
);
1037 /* Push loopback/power/transmit disable settings to the PHY, and reconfigure
1038 * the MAC appropriately. All other PHY configuration changes are pushed
1039 * through phy_op->set_settings(), and pushed asynchronously to the MAC
1040 * through efx_monitor().
1042 * Callers must hold the mac_lock
1044 int __efx_reconfigure_port(struct efx_nic
*efx
)
1046 enum efx_phy_mode phy_mode
;
1049 WARN_ON(!mutex_is_locked(&efx
->mac_lock
));
1051 /* Disable PHY transmit in mac level loopbacks */
1052 phy_mode
= efx
->phy_mode
;
1053 if (LOOPBACK_INTERNAL(efx
))
1054 efx
->phy_mode
|= PHY_MODE_TX_DISABLED
;
1056 efx
->phy_mode
&= ~PHY_MODE_TX_DISABLED
;
1058 rc
= efx
->type
->reconfigure_port(efx
);
1061 efx
->phy_mode
= phy_mode
;
1066 /* Reinitialise the MAC to pick up new PHY settings, even if the port is
1068 int efx_reconfigure_port(struct efx_nic
*efx
)
1072 EFX_ASSERT_RESET_SERIALISED(efx
);
1074 mutex_lock(&efx
->mac_lock
);
1075 rc
= __efx_reconfigure_port(efx
);
1076 mutex_unlock(&efx
->mac_lock
);
1081 /* Asynchronous work item for changing MAC promiscuity and multicast
1082 * hash. Avoid a drain/rx_ingress enable by reconfiguring the current
1084 static void efx_mac_work(struct work_struct
*data
)
1086 struct efx_nic
*efx
= container_of(data
, struct efx_nic
, mac_work
);
1088 mutex_lock(&efx
->mac_lock
);
1089 if (efx
->port_enabled
)
1090 efx_mac_reconfigure(efx
);
1091 mutex_unlock(&efx
->mac_lock
);
1094 static int efx_probe_port(struct efx_nic
*efx
)
1098 netif_dbg(efx
, probe
, efx
->net_dev
, "create port\n");
1101 efx
->phy_mode
= PHY_MODE_SPECIAL
;
1103 /* Connect up MAC/PHY operations table */
1104 rc
= efx
->type
->probe_port(efx
);
1108 /* Initialise MAC address to permanent address */
1109 ether_addr_copy(efx
->net_dev
->dev_addr
, efx
->net_dev
->perm_addr
);
1114 static int efx_init_port(struct efx_nic
*efx
)
1118 netif_dbg(efx
, drv
, efx
->net_dev
, "init port\n");
1120 mutex_lock(&efx
->mac_lock
);
1122 rc
= efx
->phy_op
->init(efx
);
1126 efx
->port_initialized
= true;
1128 /* Reconfigure the MAC before creating dma queues (required for
1129 * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */
1130 efx_mac_reconfigure(efx
);
1132 /* Ensure the PHY advertises the correct flow control settings */
1133 rc
= efx
->phy_op
->reconfigure(efx
);
1134 if (rc
&& rc
!= -EPERM
)
1137 mutex_unlock(&efx
->mac_lock
);
1141 efx
->phy_op
->fini(efx
);
1143 mutex_unlock(&efx
->mac_lock
);
1147 static void efx_start_port(struct efx_nic
*efx
)
1149 netif_dbg(efx
, ifup
, efx
->net_dev
, "start port\n");
1150 BUG_ON(efx
->port_enabled
);
1152 mutex_lock(&efx
->mac_lock
);
1153 efx
->port_enabled
= true;
1155 /* Ensure MAC ingress/egress is enabled */
1156 efx_mac_reconfigure(efx
);
1158 mutex_unlock(&efx
->mac_lock
);
1161 /* Cancel work for MAC reconfiguration, periodic hardware monitoring
1162 * and the async self-test, wait for them to finish and prevent them
1163 * being scheduled again. This doesn't cover online resets, which
1164 * should only be cancelled when removing the device.
1166 static void efx_stop_port(struct efx_nic
*efx
)
1168 netif_dbg(efx
, ifdown
, efx
->net_dev
, "stop port\n");
1170 EFX_ASSERT_RESET_SERIALISED(efx
);
1172 mutex_lock(&efx
->mac_lock
);
1173 efx
->port_enabled
= false;
1174 mutex_unlock(&efx
->mac_lock
);
1176 /* Serialise against efx_set_multicast_list() */
1177 netif_addr_lock_bh(efx
->net_dev
);
1178 netif_addr_unlock_bh(efx
->net_dev
);
1180 cancel_delayed_work_sync(&efx
->monitor_work
);
1181 efx_selftest_async_cancel(efx
);
1182 cancel_work_sync(&efx
->mac_work
);
1185 static void efx_fini_port(struct efx_nic
*efx
)
1187 netif_dbg(efx
, drv
, efx
->net_dev
, "shut down port\n");
1189 if (!efx
->port_initialized
)
1192 efx
->phy_op
->fini(efx
);
1193 efx
->port_initialized
= false;
1195 efx
->link_state
.up
= false;
1196 efx_link_status_changed(efx
);
1199 static void efx_remove_port(struct efx_nic
*efx
)
1201 netif_dbg(efx
, drv
, efx
->net_dev
, "destroying port\n");
1203 efx
->type
->remove_port(efx
);
1206 /**************************************************************************
1210 **************************************************************************/
1212 static LIST_HEAD(efx_primary_list
);
1213 static LIST_HEAD(efx_unassociated_list
);
1215 static bool efx_same_controller(struct efx_nic
*left
, struct efx_nic
*right
)
1217 return left
->type
== right
->type
&&
1218 left
->vpd_sn
&& right
->vpd_sn
&&
1219 !strcmp(left
->vpd_sn
, right
->vpd_sn
);
1222 static void efx_associate(struct efx_nic
*efx
)
1224 struct efx_nic
*other
, *next
;
1226 if (efx
->primary
== efx
) {
1227 /* Adding primary function; look for secondaries */
1229 netif_dbg(efx
, probe
, efx
->net_dev
, "adding to primary list\n");
1230 list_add_tail(&efx
->node
, &efx_primary_list
);
1232 list_for_each_entry_safe(other
, next
, &efx_unassociated_list
,
1234 if (efx_same_controller(efx
, other
)) {
1235 list_del(&other
->node
);
1236 netif_dbg(other
, probe
, other
->net_dev
,
1237 "moving to secondary list of %s %s\n",
1238 pci_name(efx
->pci_dev
),
1239 efx
->net_dev
->name
);
1240 list_add_tail(&other
->node
,
1241 &efx
->secondary_list
);
1242 other
->primary
= efx
;
1246 /* Adding secondary function; look for primary */
1248 list_for_each_entry(other
, &efx_primary_list
, node
) {
1249 if (efx_same_controller(efx
, other
)) {
1250 netif_dbg(efx
, probe
, efx
->net_dev
,
1251 "adding to secondary list of %s %s\n",
1252 pci_name(other
->pci_dev
),
1253 other
->net_dev
->name
);
1254 list_add_tail(&efx
->node
,
1255 &other
->secondary_list
);
1256 efx
->primary
= other
;
1261 netif_dbg(efx
, probe
, efx
->net_dev
,
1262 "adding to unassociated list\n");
1263 list_add_tail(&efx
->node
, &efx_unassociated_list
);
1267 static void efx_dissociate(struct efx_nic
*efx
)
1269 struct efx_nic
*other
, *next
;
1271 list_del(&efx
->node
);
1272 efx
->primary
= NULL
;
1274 list_for_each_entry_safe(other
, next
, &efx
->secondary_list
, node
) {
1275 list_del(&other
->node
);
1276 netif_dbg(other
, probe
, other
->net_dev
,
1277 "moving to unassociated list\n");
1278 list_add_tail(&other
->node
, &efx_unassociated_list
);
1279 other
->primary
= NULL
;
1283 /* This configures the PCI device to enable I/O and DMA. */
1284 static int efx_init_io(struct efx_nic
*efx
)
1286 struct pci_dev
*pci_dev
= efx
->pci_dev
;
1287 dma_addr_t dma_mask
= efx
->type
->max_dma_mask
;
1288 unsigned int mem_map_size
= efx
->type
->mem_map_size(efx
);
1291 netif_dbg(efx
, probe
, efx
->net_dev
, "initialising I/O\n");
1293 bar
= efx
->type
->mem_bar(efx
);
1295 rc
= pci_enable_device(pci_dev
);
1297 netif_err(efx
, probe
, efx
->net_dev
,
1298 "failed to enable PCI device\n");
1302 pci_set_master(pci_dev
);
1304 /* Set the PCI DMA mask. Try all possibilities from our genuine mask
1305 * down to 32 bits, because some architectures will allow 40 bit
1306 * masks event though they reject 46 bit masks.
1308 while (dma_mask
> 0x7fffffffUL
) {
1309 rc
= dma_set_mask_and_coherent(&pci_dev
->dev
, dma_mask
);
1315 netif_err(efx
, probe
, efx
->net_dev
,
1316 "could not find a suitable DMA mask\n");
1319 netif_dbg(efx
, probe
, efx
->net_dev
,
1320 "using DMA mask %llx\n", (unsigned long long) dma_mask
);
1322 efx
->membase_phys
= pci_resource_start(efx
->pci_dev
, bar
);
1323 rc
= pci_request_region(pci_dev
, bar
, "sfc");
1325 netif_err(efx
, probe
, efx
->net_dev
,
1326 "request for memory BAR failed\n");
1330 efx
->membase
= ioremap_nocache(efx
->membase_phys
, mem_map_size
);
1331 if (!efx
->membase
) {
1332 netif_err(efx
, probe
, efx
->net_dev
,
1333 "could not map memory BAR at %llx+%x\n",
1334 (unsigned long long)efx
->membase_phys
, mem_map_size
);
1338 netif_dbg(efx
, probe
, efx
->net_dev
,
1339 "memory BAR at %llx+%x (virtual %p)\n",
1340 (unsigned long long)efx
->membase_phys
, mem_map_size
,
1346 pci_release_region(efx
->pci_dev
, bar
);
1348 efx
->membase_phys
= 0;
1350 pci_disable_device(efx
->pci_dev
);
1355 static void efx_fini_io(struct efx_nic
*efx
)
1359 netif_dbg(efx
, drv
, efx
->net_dev
, "shutting down I/O\n");
1362 iounmap(efx
->membase
);
1363 efx
->membase
= NULL
;
1366 if (efx
->membase_phys
) {
1367 bar
= efx
->type
->mem_bar(efx
);
1368 pci_release_region(efx
->pci_dev
, bar
);
1369 efx
->membase_phys
= 0;
1372 /* Don't disable bus-mastering if VFs are assigned */
1373 if (!pci_vfs_assigned(efx
->pci_dev
))
1374 pci_disable_device(efx
->pci_dev
);
1377 void efx_set_default_rx_indir_table(struct efx_nic
*efx
,
1378 struct efx_rss_context
*ctx
)
1382 for (i
= 0; i
< ARRAY_SIZE(ctx
->rx_indir_table
); i
++)
1383 ctx
->rx_indir_table
[i
] =
1384 ethtool_rxfh_indir_default(i
, efx
->rss_spread
);
1387 static unsigned int efx_wanted_parallelism(struct efx_nic
*efx
)
1389 cpumask_var_t thread_mask
;
1396 if (unlikely(!zalloc_cpumask_var(&thread_mask
, GFP_KERNEL
))) {
1397 netif_warn(efx
, probe
, efx
->net_dev
,
1398 "RSS disabled due to allocation failure\n");
1403 for_each_online_cpu(cpu
) {
1404 if (!cpumask_test_cpu(cpu
, thread_mask
)) {
1406 cpumask_or(thread_mask
, thread_mask
,
1407 topology_sibling_cpumask(cpu
));
1411 free_cpumask_var(thread_mask
);
1414 if (count
> EFX_MAX_RX_QUEUES
) {
1415 netif_cond_dbg(efx
, probe
, efx
->net_dev
, !rss_cpus
, warn
,
1416 "Reducing number of rx queues from %u to %u.\n",
1417 count
, EFX_MAX_RX_QUEUES
);
1418 count
= EFX_MAX_RX_QUEUES
;
1421 /* If RSS is requested for the PF *and* VFs then we can't write RSS
1422 * table entries that are inaccessible to VFs
1424 #ifdef CONFIG_SFC_SRIOV
1425 if (efx
->type
->sriov_wanted
) {
1426 if (efx
->type
->sriov_wanted(efx
) && efx_vf_size(efx
) > 1 &&
1427 count
> efx_vf_size(efx
)) {
1428 netif_warn(efx
, probe
, efx
->net_dev
,
1429 "Reducing number of RSS channels from %u to %u for "
1430 "VF support. Increase vf-msix-limit to use more "
1431 "channels on the PF.\n",
1432 count
, efx_vf_size(efx
));
1433 count
= efx_vf_size(efx
);
1441 /* Probe the number and type of interrupts we are able to obtain, and
1442 * the resulting numbers of channels and RX queues.
1444 static int efx_probe_interrupts(struct efx_nic
*efx
)
1446 unsigned int extra_channels
= 0;
1450 for (i
= 0; i
< EFX_MAX_EXTRA_CHANNELS
; i
++)
1451 if (efx
->extra_channel_type
[i
])
1454 if (efx
->interrupt_mode
== EFX_INT_MODE_MSIX
) {
1455 struct msix_entry xentries
[EFX_MAX_CHANNELS
];
1456 unsigned int n_channels
;
1458 n_channels
= efx_wanted_parallelism(efx
);
1459 if (efx_separate_tx_channels
)
1461 n_channels
+= extra_channels
;
1462 n_channels
= min(n_channels
, efx
->max_channels
);
1464 for (i
= 0; i
< n_channels
; i
++)
1465 xentries
[i
].entry
= i
;
1466 rc
= pci_enable_msix_range(efx
->pci_dev
,
1467 xentries
, 1, n_channels
);
1469 /* Fall back to single channel MSI */
1470 netif_err(efx
, drv
, efx
->net_dev
,
1471 "could not enable MSI-X\n");
1472 if (efx
->type
->min_interrupt_mode
>= EFX_INT_MODE_MSI
)
1473 efx
->interrupt_mode
= EFX_INT_MODE_MSI
;
1476 } else if (rc
< n_channels
) {
1477 netif_err(efx
, drv
, efx
->net_dev
,
1478 "WARNING: Insufficient MSI-X vectors"
1479 " available (%d < %u).\n", rc
, n_channels
);
1480 netif_err(efx
, drv
, efx
->net_dev
,
1481 "WARNING: Performance may be reduced.\n");
1486 efx
->n_channels
= n_channels
;
1487 if (n_channels
> extra_channels
)
1488 n_channels
-= extra_channels
;
1489 if (efx_separate_tx_channels
) {
1490 efx
->n_tx_channels
= min(max(n_channels
/ 2,
1492 efx
->max_tx_channels
);
1493 efx
->n_rx_channels
= max(n_channels
-
1497 efx
->n_tx_channels
= min(n_channels
,
1498 efx
->max_tx_channels
);
1499 efx
->n_rx_channels
= n_channels
;
1501 for (i
= 0; i
< efx
->n_channels
; i
++)
1502 efx_get_channel(efx
, i
)->irq
=
1507 /* Try single interrupt MSI */
1508 if (efx
->interrupt_mode
== EFX_INT_MODE_MSI
) {
1509 efx
->n_channels
= 1;
1510 efx
->n_rx_channels
= 1;
1511 efx
->n_tx_channels
= 1;
1512 rc
= pci_enable_msi(efx
->pci_dev
);
1514 efx_get_channel(efx
, 0)->irq
= efx
->pci_dev
->irq
;
1516 netif_err(efx
, drv
, efx
->net_dev
,
1517 "could not enable MSI\n");
1518 if (efx
->type
->min_interrupt_mode
>= EFX_INT_MODE_LEGACY
)
1519 efx
->interrupt_mode
= EFX_INT_MODE_LEGACY
;
1525 /* Assume legacy interrupts */
1526 if (efx
->interrupt_mode
== EFX_INT_MODE_LEGACY
) {
1527 efx
->n_channels
= 1 + (efx_separate_tx_channels
? 1 : 0);
1528 efx
->n_rx_channels
= 1;
1529 efx
->n_tx_channels
= 1;
1530 efx
->legacy_irq
= efx
->pci_dev
->irq
;
1533 /* Assign extra channels if possible */
1534 efx
->n_extra_tx_channels
= 0;
1535 j
= efx
->n_channels
;
1536 for (i
= 0; i
< EFX_MAX_EXTRA_CHANNELS
; i
++) {
1537 if (!efx
->extra_channel_type
[i
])
1539 if (efx
->interrupt_mode
!= EFX_INT_MODE_MSIX
||
1540 efx
->n_channels
<= extra_channels
) {
1541 efx
->extra_channel_type
[i
]->handle_no_channel(efx
);
1544 efx_get_channel(efx
, j
)->type
=
1545 efx
->extra_channel_type
[i
];
1546 if (efx_channel_has_tx_queues(efx_get_channel(efx
, j
)))
1547 efx
->n_extra_tx_channels
++;
1551 /* RSS might be usable on VFs even if it is disabled on the PF */
1552 #ifdef CONFIG_SFC_SRIOV
1553 if (efx
->type
->sriov_wanted
) {
1554 efx
->rss_spread
= ((efx
->n_rx_channels
> 1 ||
1555 !efx
->type
->sriov_wanted(efx
)) ?
1556 efx
->n_rx_channels
: efx_vf_size(efx
));
1560 efx
->rss_spread
= efx
->n_rx_channels
;
1565 #if defined(CONFIG_SMP)
1566 static void efx_set_interrupt_affinity(struct efx_nic
*efx
)
1568 struct efx_channel
*channel
;
1571 efx_for_each_channel(channel
, efx
) {
1572 cpu
= cpumask_local_spread(channel
->channel
,
1573 pcibus_to_node(efx
->pci_dev
->bus
));
1574 irq_set_affinity_hint(channel
->irq
, cpumask_of(cpu
));
1578 static void efx_clear_interrupt_affinity(struct efx_nic
*efx
)
1580 struct efx_channel
*channel
;
1582 efx_for_each_channel(channel
, efx
)
1583 irq_set_affinity_hint(channel
->irq
, NULL
);
1587 efx_set_interrupt_affinity(struct efx_nic
*efx
__attribute__ ((unused
)))
1592 efx_clear_interrupt_affinity(struct efx_nic
*efx
__attribute__ ((unused
)))
1595 #endif /* CONFIG_SMP */
1597 static int efx_soft_enable_interrupts(struct efx_nic
*efx
)
1599 struct efx_channel
*channel
, *end_channel
;
1602 BUG_ON(efx
->state
== STATE_DISABLED
);
1604 efx
->irq_soft_enabled
= true;
1607 efx_for_each_channel(channel
, efx
) {
1608 if (!channel
->type
->keep_eventq
) {
1609 rc
= efx_init_eventq(channel
);
1613 efx_start_eventq(channel
);
1616 efx_mcdi_mode_event(efx
);
1620 end_channel
= channel
;
1621 efx_for_each_channel(channel
, efx
) {
1622 if (channel
== end_channel
)
1624 efx_stop_eventq(channel
);
1625 if (!channel
->type
->keep_eventq
)
1626 efx_fini_eventq(channel
);
1632 static void efx_soft_disable_interrupts(struct efx_nic
*efx
)
1634 struct efx_channel
*channel
;
1636 if (efx
->state
== STATE_DISABLED
)
1639 efx_mcdi_mode_poll(efx
);
1641 efx
->irq_soft_enabled
= false;
1644 if (efx
->legacy_irq
)
1645 synchronize_irq(efx
->legacy_irq
);
1647 efx_for_each_channel(channel
, efx
) {
1649 synchronize_irq(channel
->irq
);
1651 efx_stop_eventq(channel
);
1652 if (!channel
->type
->keep_eventq
)
1653 efx_fini_eventq(channel
);
1656 /* Flush the asynchronous MCDI request queue */
1657 efx_mcdi_flush_async(efx
);
1660 static int efx_enable_interrupts(struct efx_nic
*efx
)
1662 struct efx_channel
*channel
, *end_channel
;
1665 BUG_ON(efx
->state
== STATE_DISABLED
);
1667 if (efx
->eeh_disabled_legacy_irq
) {
1668 enable_irq(efx
->legacy_irq
);
1669 efx
->eeh_disabled_legacy_irq
= false;
1672 efx
->type
->irq_enable_master(efx
);
1674 efx_for_each_channel(channel
, efx
) {
1675 if (channel
->type
->keep_eventq
) {
1676 rc
= efx_init_eventq(channel
);
1682 rc
= efx_soft_enable_interrupts(efx
);
1689 end_channel
= channel
;
1690 efx_for_each_channel(channel
, efx
) {
1691 if (channel
== end_channel
)
1693 if (channel
->type
->keep_eventq
)
1694 efx_fini_eventq(channel
);
1697 efx
->type
->irq_disable_non_ev(efx
);
1702 static void efx_disable_interrupts(struct efx_nic
*efx
)
1704 struct efx_channel
*channel
;
1706 efx_soft_disable_interrupts(efx
);
1708 efx_for_each_channel(channel
, efx
) {
1709 if (channel
->type
->keep_eventq
)
1710 efx_fini_eventq(channel
);
1713 efx
->type
->irq_disable_non_ev(efx
);
1716 static void efx_remove_interrupts(struct efx_nic
*efx
)
1718 struct efx_channel
*channel
;
1720 /* Remove MSI/MSI-X interrupts */
1721 efx_for_each_channel(channel
, efx
)
1723 pci_disable_msi(efx
->pci_dev
);
1724 pci_disable_msix(efx
->pci_dev
);
1726 /* Remove legacy interrupt */
1727 efx
->legacy_irq
= 0;
1730 static void efx_set_channels(struct efx_nic
*efx
)
1732 struct efx_channel
*channel
;
1733 struct efx_tx_queue
*tx_queue
;
1735 efx
->tx_channel_offset
=
1736 efx_separate_tx_channels
?
1737 efx
->n_channels
- efx
->n_tx_channels
: 0;
1739 /* We need to mark which channels really have RX and TX
1740 * queues, and adjust the TX queue numbers if we have separate
1741 * RX-only and TX-only channels.
1743 efx_for_each_channel(channel
, efx
) {
1744 if (channel
->channel
< efx
->n_rx_channels
)
1745 channel
->rx_queue
.core_index
= channel
->channel
;
1747 channel
->rx_queue
.core_index
= -1;
1749 efx_for_each_channel_tx_queue(tx_queue
, channel
)
1750 tx_queue
->queue
-= (efx
->tx_channel_offset
*
1755 static int efx_probe_nic(struct efx_nic
*efx
)
1759 netif_dbg(efx
, probe
, efx
->net_dev
, "creating NIC\n");
1761 /* Carry out hardware-type specific initialisation */
1762 rc
= efx
->type
->probe(efx
);
1767 if (!efx
->max_channels
|| !efx
->max_tx_channels
) {
1768 netif_err(efx
, drv
, efx
->net_dev
,
1769 "Insufficient resources to allocate"
1775 /* Determine the number of channels and queues by trying
1776 * to hook in MSI-X interrupts.
1778 rc
= efx_probe_interrupts(efx
);
1782 efx_set_channels(efx
);
1784 /* dimension_resources can fail with EAGAIN */
1785 rc
= efx
->type
->dimension_resources(efx
);
1786 if (rc
!= 0 && rc
!= -EAGAIN
)
1790 /* try again with new max_channels */
1791 efx_remove_interrupts(efx
);
1793 } while (rc
== -EAGAIN
);
1795 if (efx
->n_channels
> 1)
1796 netdev_rss_key_fill(efx
->rss_context
.rx_hash_key
,
1797 sizeof(efx
->rss_context
.rx_hash_key
));
1798 efx_set_default_rx_indir_table(efx
, &efx
->rss_context
);
1800 netif_set_real_num_tx_queues(efx
->net_dev
, efx
->n_tx_channels
);
1801 netif_set_real_num_rx_queues(efx
->net_dev
, efx
->n_rx_channels
);
1803 /* Initialise the interrupt moderation settings */
1804 efx
->irq_mod_step_us
= DIV_ROUND_UP(efx
->timer_quantum_ns
, 1000);
1805 efx_init_irq_moderation(efx
, tx_irq_mod_usec
, rx_irq_mod_usec
, true,
1811 efx_remove_interrupts(efx
);
1813 efx
->type
->remove(efx
);
1817 static void efx_remove_nic(struct efx_nic
*efx
)
1819 netif_dbg(efx
, drv
, efx
->net_dev
, "destroying NIC\n");
1821 efx_remove_interrupts(efx
);
1822 efx
->type
->remove(efx
);
1825 static int efx_probe_filters(struct efx_nic
*efx
)
1829 init_rwsem(&efx
->filter_sem
);
1830 mutex_lock(&efx
->mac_lock
);
1831 down_write(&efx
->filter_sem
);
1832 rc
= efx
->type
->filter_table_probe(efx
);
1836 #ifdef CONFIG_RFS_ACCEL
1837 if (efx
->type
->offload_features
& NETIF_F_NTUPLE
) {
1838 struct efx_channel
*channel
;
1841 efx_for_each_channel(channel
, efx
) {
1842 channel
->rps_flow_id
=
1843 kcalloc(efx
->type
->max_rx_ip_filters
,
1844 sizeof(*channel
->rps_flow_id
),
1846 if (!channel
->rps_flow_id
)
1850 i
< efx
->type
->max_rx_ip_filters
;
1852 channel
->rps_flow_id
[i
] =
1853 RPS_FLOW_ID_INVALID
;
1857 efx_for_each_channel(channel
, efx
)
1858 kfree(channel
->rps_flow_id
);
1859 efx
->type
->filter_table_remove(efx
);
1864 efx
->rps_expire_index
= efx
->rps_expire_channel
= 0;
1868 up_write(&efx
->filter_sem
);
1869 mutex_unlock(&efx
->mac_lock
);
1873 static void efx_remove_filters(struct efx_nic
*efx
)
1875 #ifdef CONFIG_RFS_ACCEL
1876 struct efx_channel
*channel
;
1878 efx_for_each_channel(channel
, efx
)
1879 kfree(channel
->rps_flow_id
);
1881 down_write(&efx
->filter_sem
);
1882 efx
->type
->filter_table_remove(efx
);
1883 up_write(&efx
->filter_sem
);
1887 /**************************************************************************
1889 * NIC startup/shutdown
1891 *************************************************************************/
1893 static int efx_probe_all(struct efx_nic
*efx
)
1897 rc
= efx_probe_nic(efx
);
1899 netif_err(efx
, probe
, efx
->net_dev
, "failed to create NIC\n");
1903 rc
= efx_probe_port(efx
);
1905 netif_err(efx
, probe
, efx
->net_dev
, "failed to create port\n");
1909 BUILD_BUG_ON(EFX_DEFAULT_DMAQ_SIZE
< EFX_RXQ_MIN_ENT
);
1910 if (WARN_ON(EFX_DEFAULT_DMAQ_SIZE
< EFX_TXQ_MIN_ENT(efx
))) {
1914 efx
->rxq_entries
= efx
->txq_entries
= EFX_DEFAULT_DMAQ_SIZE
;
1916 #ifdef CONFIG_SFC_SRIOV
1917 rc
= efx
->type
->vswitching_probe(efx
);
1918 if (rc
) /* not fatal; the PF will still work fine */
1919 netif_warn(efx
, probe
, efx
->net_dev
,
1920 "failed to setup vswitching rc=%d;"
1921 " VFs may not function\n", rc
);
1924 rc
= efx_probe_filters(efx
);
1926 netif_err(efx
, probe
, efx
->net_dev
,
1927 "failed to create filter tables\n");
1931 rc
= efx_probe_channels(efx
);
1938 efx_remove_filters(efx
);
1940 #ifdef CONFIG_SFC_SRIOV
1941 efx
->type
->vswitching_remove(efx
);
1944 efx_remove_port(efx
);
1946 efx_remove_nic(efx
);
1951 /* If the interface is supposed to be running but is not, start
1952 * the hardware and software data path, regular activity for the port
1953 * (MAC statistics, link polling, etc.) and schedule the port to be
1954 * reconfigured. Interrupts must already be enabled. This function
1955 * is safe to call multiple times, so long as the NIC is not disabled.
1956 * Requires the RTNL lock.
1958 static void efx_start_all(struct efx_nic
*efx
)
1960 EFX_ASSERT_RESET_SERIALISED(efx
);
1961 BUG_ON(efx
->state
== STATE_DISABLED
);
1963 /* Check that it is appropriate to restart the interface. All
1964 * of these flags are safe to read under just the rtnl lock */
1965 if (efx
->port_enabled
|| !netif_running(efx
->net_dev
) ||
1969 efx_start_port(efx
);
1970 efx_start_datapath(efx
);
1972 /* Start the hardware monitor if there is one */
1973 if (efx
->type
->monitor
!= NULL
)
1974 queue_delayed_work(efx
->workqueue
, &efx
->monitor_work
,
1975 efx_monitor_interval
);
1977 /* Link state detection is normally event-driven; we have
1978 * to poll now because we could have missed a change
1980 mutex_lock(&efx
->mac_lock
);
1981 if (efx
->phy_op
->poll(efx
))
1982 efx_link_status_changed(efx
);
1983 mutex_unlock(&efx
->mac_lock
);
1985 efx
->type
->start_stats(efx
);
1986 efx
->type
->pull_stats(efx
);
1987 spin_lock_bh(&efx
->stats_lock
);
1988 efx
->type
->update_stats(efx
, NULL
, NULL
);
1989 spin_unlock_bh(&efx
->stats_lock
);
1992 /* Quiesce the hardware and software data path, and regular activity
1993 * for the port without bringing the link down. Safe to call multiple
1994 * times with the NIC in almost any state, but interrupts should be
1995 * enabled. Requires the RTNL lock.
1997 static void efx_stop_all(struct efx_nic
*efx
)
1999 EFX_ASSERT_RESET_SERIALISED(efx
);
2001 /* port_enabled can be read safely under the rtnl lock */
2002 if (!efx
->port_enabled
)
2005 /* update stats before we go down so we can accurately count
2008 efx
->type
->pull_stats(efx
);
2009 spin_lock_bh(&efx
->stats_lock
);
2010 efx
->type
->update_stats(efx
, NULL
, NULL
);
2011 spin_unlock_bh(&efx
->stats_lock
);
2012 efx
->type
->stop_stats(efx
);
2015 /* Stop the kernel transmit interface. This is only valid if
2016 * the device is stopped or detached; otherwise the watchdog
2017 * may fire immediately.
2019 WARN_ON(netif_running(efx
->net_dev
) &&
2020 netif_device_present(efx
->net_dev
));
2021 netif_tx_disable(efx
->net_dev
);
2023 efx_stop_datapath(efx
);
2026 static void efx_remove_all(struct efx_nic
*efx
)
2028 efx_remove_channels(efx
);
2029 efx_remove_filters(efx
);
2030 #ifdef CONFIG_SFC_SRIOV
2031 efx
->type
->vswitching_remove(efx
);
2033 efx_remove_port(efx
);
2034 efx_remove_nic(efx
);
2037 /**************************************************************************
2039 * Interrupt moderation
2041 **************************************************************************/
2042 unsigned int efx_usecs_to_ticks(struct efx_nic
*efx
, unsigned int usecs
)
2046 if (usecs
* 1000 < efx
->timer_quantum_ns
)
2047 return 1; /* never round down to 0 */
2048 return usecs
* 1000 / efx
->timer_quantum_ns
;
2051 unsigned int efx_ticks_to_usecs(struct efx_nic
*efx
, unsigned int ticks
)
2053 /* We must round up when converting ticks to microseconds
2054 * because we round down when converting the other way.
2056 return DIV_ROUND_UP(ticks
* efx
->timer_quantum_ns
, 1000);
2059 /* Set interrupt moderation parameters */
2060 int efx_init_irq_moderation(struct efx_nic
*efx
, unsigned int tx_usecs
,
2061 unsigned int rx_usecs
, bool rx_adaptive
,
2062 bool rx_may_override_tx
)
2064 struct efx_channel
*channel
;
2065 unsigned int timer_max_us
;
2067 EFX_ASSERT_RESET_SERIALISED(efx
);
2069 timer_max_us
= efx
->timer_max_ns
/ 1000;
2071 if (tx_usecs
> timer_max_us
|| rx_usecs
> timer_max_us
)
2074 if (tx_usecs
!= rx_usecs
&& efx
->tx_channel_offset
== 0 &&
2075 !rx_may_override_tx
) {
2076 netif_err(efx
, drv
, efx
->net_dev
, "Channels are shared. "
2077 "RX and TX IRQ moderation must be equal\n");
2081 efx
->irq_rx_adaptive
= rx_adaptive
;
2082 efx
->irq_rx_moderation_us
= rx_usecs
;
2083 efx_for_each_channel(channel
, efx
) {
2084 if (efx_channel_has_rx_queue(channel
))
2085 channel
->irq_moderation_us
= rx_usecs
;
2086 else if (efx_channel_has_tx_queues(channel
))
2087 channel
->irq_moderation_us
= tx_usecs
;
2093 void efx_get_irq_moderation(struct efx_nic
*efx
, unsigned int *tx_usecs
,
2094 unsigned int *rx_usecs
, bool *rx_adaptive
)
2096 *rx_adaptive
= efx
->irq_rx_adaptive
;
2097 *rx_usecs
= efx
->irq_rx_moderation_us
;
2099 /* If channels are shared between RX and TX, so is IRQ
2100 * moderation. Otherwise, IRQ moderation is the same for all
2101 * TX channels and is not adaptive.
2103 if (efx
->tx_channel_offset
== 0) {
2104 *tx_usecs
= *rx_usecs
;
2106 struct efx_channel
*tx_channel
;
2108 tx_channel
= efx
->channel
[efx
->tx_channel_offset
];
2109 *tx_usecs
= tx_channel
->irq_moderation_us
;
2113 /**************************************************************************
2117 **************************************************************************/
2119 /* Run periodically off the general workqueue */
2120 static void efx_monitor(struct work_struct
*data
)
2122 struct efx_nic
*efx
= container_of(data
, struct efx_nic
,
2125 netif_vdbg(efx
, timer
, efx
->net_dev
,
2126 "hardware monitor executing on CPU %d\n",
2127 raw_smp_processor_id());
2128 BUG_ON(efx
->type
->monitor
== NULL
);
2130 /* If the mac_lock is already held then it is likely a port
2131 * reconfiguration is already in place, which will likely do
2132 * most of the work of monitor() anyway. */
2133 if (mutex_trylock(&efx
->mac_lock
)) {
2134 if (efx
->port_enabled
)
2135 efx
->type
->monitor(efx
);
2136 mutex_unlock(&efx
->mac_lock
);
2139 queue_delayed_work(efx
->workqueue
, &efx
->monitor_work
,
2140 efx_monitor_interval
);
2143 /**************************************************************************
2147 *************************************************************************/
2150 * Context: process, rtnl_lock() held.
2152 static int efx_ioctl(struct net_device
*net_dev
, struct ifreq
*ifr
, int cmd
)
2154 struct efx_nic
*efx
= netdev_priv(net_dev
);
2155 struct mii_ioctl_data
*data
= if_mii(ifr
);
2157 if (cmd
== SIOCSHWTSTAMP
)
2158 return efx_ptp_set_ts_config(efx
, ifr
);
2159 if (cmd
== SIOCGHWTSTAMP
)
2160 return efx_ptp_get_ts_config(efx
, ifr
);
2162 /* Convert phy_id from older PRTAD/DEVAD format */
2163 if ((cmd
== SIOCGMIIREG
|| cmd
== SIOCSMIIREG
) &&
2164 (data
->phy_id
& 0xfc00) == 0x0400)
2165 data
->phy_id
^= MDIO_PHY_ID_C45
| 0x0400;
2167 return mdio_mii_ioctl(&efx
->mdio
, data
, cmd
);
2170 /**************************************************************************
2174 **************************************************************************/
2176 static void efx_init_napi_channel(struct efx_channel
*channel
)
2178 struct efx_nic
*efx
= channel
->efx
;
2180 channel
->napi_dev
= efx
->net_dev
;
2181 netif_napi_add(channel
->napi_dev
, &channel
->napi_str
,
2182 efx_poll
, napi_weight
);
2185 static void efx_init_napi(struct efx_nic
*efx
)
2187 struct efx_channel
*channel
;
2189 efx_for_each_channel(channel
, efx
)
2190 efx_init_napi_channel(channel
);
2193 static void efx_fini_napi_channel(struct efx_channel
*channel
)
2195 if (channel
->napi_dev
)
2196 netif_napi_del(&channel
->napi_str
);
2198 channel
->napi_dev
= NULL
;
2201 static void efx_fini_napi(struct efx_nic
*efx
)
2203 struct efx_channel
*channel
;
2205 efx_for_each_channel(channel
, efx
)
2206 efx_fini_napi_channel(channel
);
2209 /**************************************************************************
2211 * Kernel net device interface
2213 *************************************************************************/
2215 /* Context: process, rtnl_lock() held. */
2216 int efx_net_open(struct net_device
*net_dev
)
2218 struct efx_nic
*efx
= netdev_priv(net_dev
);
2221 netif_dbg(efx
, ifup
, efx
->net_dev
, "opening device on CPU %d\n",
2222 raw_smp_processor_id());
2224 rc
= efx_check_disabled(efx
);
2227 if (efx
->phy_mode
& PHY_MODE_SPECIAL
)
2229 if (efx_mcdi_poll_reboot(efx
) && efx_reset(efx
, RESET_TYPE_ALL
))
2232 /* Notify the kernel of the link state polled during driver load,
2233 * before the monitor starts running */
2234 efx_link_status_changed(efx
);
2237 if (efx
->state
== STATE_DISABLED
|| efx
->reset_pending
)
2238 netif_device_detach(efx
->net_dev
);
2239 efx_selftest_async_start(efx
);
2243 /* Context: process, rtnl_lock() held.
2244 * Note that the kernel will ignore our return code; this method
2245 * should really be a void.
2247 int efx_net_stop(struct net_device
*net_dev
)
2249 struct efx_nic
*efx
= netdev_priv(net_dev
);
2251 netif_dbg(efx
, ifdown
, efx
->net_dev
, "closing on CPU %d\n",
2252 raw_smp_processor_id());
2254 /* Stop the device and flush all the channels */
2260 /* Context: process, dev_base_lock or RTNL held, non-blocking. */
2261 static void efx_net_stats(struct net_device
*net_dev
,
2262 struct rtnl_link_stats64
*stats
)
2264 struct efx_nic
*efx
= netdev_priv(net_dev
);
2266 spin_lock_bh(&efx
->stats_lock
);
2267 efx
->type
->update_stats(efx
, NULL
, stats
);
2268 spin_unlock_bh(&efx
->stats_lock
);
2271 /* Context: netif_tx_lock held, BHs disabled. */
2272 static void efx_watchdog(struct net_device
*net_dev
)
2274 struct efx_nic
*efx
= netdev_priv(net_dev
);
2276 netif_err(efx
, tx_err
, efx
->net_dev
,
2277 "TX stuck with port_enabled=%d: resetting channels\n",
2280 efx_schedule_reset(efx
, RESET_TYPE_TX_WATCHDOG
);
2284 /* Context: process, rtnl_lock() held. */
2285 static int efx_change_mtu(struct net_device
*net_dev
, int new_mtu
)
2287 struct efx_nic
*efx
= netdev_priv(net_dev
);
2290 rc
= efx_check_disabled(efx
);
2294 netif_dbg(efx
, drv
, efx
->net_dev
, "changing MTU to %d\n", new_mtu
);
2296 efx_device_detach_sync(efx
);
2299 mutex_lock(&efx
->mac_lock
);
2300 net_dev
->mtu
= new_mtu
;
2301 efx_mac_reconfigure(efx
);
2302 mutex_unlock(&efx
->mac_lock
);
2305 efx_device_attach_if_not_resetting(efx
);
2309 static int efx_set_mac_address(struct net_device
*net_dev
, void *data
)
2311 struct efx_nic
*efx
= netdev_priv(net_dev
);
2312 struct sockaddr
*addr
= data
;
2313 u8
*new_addr
= addr
->sa_data
;
2317 if (!is_valid_ether_addr(new_addr
)) {
2318 netif_err(efx
, drv
, efx
->net_dev
,
2319 "invalid ethernet MAC address requested: %pM\n",
2321 return -EADDRNOTAVAIL
;
2324 /* save old address */
2325 ether_addr_copy(old_addr
, net_dev
->dev_addr
);
2326 ether_addr_copy(net_dev
->dev_addr
, new_addr
);
2327 if (efx
->type
->set_mac_address
) {
2328 rc
= efx
->type
->set_mac_address(efx
);
2330 ether_addr_copy(net_dev
->dev_addr
, old_addr
);
2335 /* Reconfigure the MAC */
2336 mutex_lock(&efx
->mac_lock
);
2337 efx_mac_reconfigure(efx
);
2338 mutex_unlock(&efx
->mac_lock
);
2343 /* Context: netif_addr_lock held, BHs disabled. */
2344 static void efx_set_rx_mode(struct net_device
*net_dev
)
2346 struct efx_nic
*efx
= netdev_priv(net_dev
);
2348 if (efx
->port_enabled
)
2349 queue_work(efx
->workqueue
, &efx
->mac_work
);
2350 /* Otherwise efx_start_port() will do this */
2353 static int efx_set_features(struct net_device
*net_dev
, netdev_features_t data
)
2355 struct efx_nic
*efx
= netdev_priv(net_dev
);
2358 /* If disabling RX n-tuple filtering, clear existing filters */
2359 if (net_dev
->features
& ~data
& NETIF_F_NTUPLE
) {
2360 rc
= efx
->type
->filter_clear_rx(efx
, EFX_FILTER_PRI_MANUAL
);
2365 /* If Rx VLAN filter is changed, update filters via mac_reconfigure.
2366 * If rx-fcs is changed, mac_reconfigure updates that too.
2368 if ((net_dev
->features
^ data
) & (NETIF_F_HW_VLAN_CTAG_FILTER
|
2370 /* efx_set_rx_mode() will schedule MAC work to update filters
2371 * when a new features are finally set in net_dev.
2373 efx_set_rx_mode(net_dev
);
2379 static int efx_get_phys_port_id(struct net_device
*net_dev
,
2380 struct netdev_phys_item_id
*ppid
)
2382 struct efx_nic
*efx
= netdev_priv(net_dev
);
2384 if (efx
->type
->get_phys_port_id
)
2385 return efx
->type
->get_phys_port_id(efx
, ppid
);
2390 static int efx_get_phys_port_name(struct net_device
*net_dev
,
2391 char *name
, size_t len
)
2393 struct efx_nic
*efx
= netdev_priv(net_dev
);
2395 if (snprintf(name
, len
, "p%u", efx
->port_num
) >= len
)
2400 static int efx_vlan_rx_add_vid(struct net_device
*net_dev
, __be16 proto
, u16 vid
)
2402 struct efx_nic
*efx
= netdev_priv(net_dev
);
2404 if (efx
->type
->vlan_rx_add_vid
)
2405 return efx
->type
->vlan_rx_add_vid(efx
, proto
, vid
);
2410 static int efx_vlan_rx_kill_vid(struct net_device
*net_dev
, __be16 proto
, u16 vid
)
2412 struct efx_nic
*efx
= netdev_priv(net_dev
);
2414 if (efx
->type
->vlan_rx_kill_vid
)
2415 return efx
->type
->vlan_rx_kill_vid(efx
, proto
, vid
);
2420 static int efx_udp_tunnel_type_map(enum udp_parsable_tunnel_type in
)
2423 case UDP_TUNNEL_TYPE_VXLAN
:
2424 return TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN
;
2425 case UDP_TUNNEL_TYPE_GENEVE
:
2426 return TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE
;
2432 static void efx_udp_tunnel_add(struct net_device
*dev
, struct udp_tunnel_info
*ti
)
2434 struct efx_nic
*efx
= netdev_priv(dev
);
2435 struct efx_udp_tunnel tnl
;
2436 int efx_tunnel_type
;
2438 efx_tunnel_type
= efx_udp_tunnel_type_map(ti
->type
);
2439 if (efx_tunnel_type
< 0)
2442 tnl
.type
= (u16
)efx_tunnel_type
;
2443 tnl
.port
= ti
->port
;
2445 if (efx
->type
->udp_tnl_add_port
)
2446 (void)efx
->type
->udp_tnl_add_port(efx
, tnl
);
2449 static void efx_udp_tunnel_del(struct net_device
*dev
, struct udp_tunnel_info
*ti
)
2451 struct efx_nic
*efx
= netdev_priv(dev
);
2452 struct efx_udp_tunnel tnl
;
2453 int efx_tunnel_type
;
2455 efx_tunnel_type
= efx_udp_tunnel_type_map(ti
->type
);
2456 if (efx_tunnel_type
< 0)
2459 tnl
.type
= (u16
)efx_tunnel_type
;
2460 tnl
.port
= ti
->port
;
2462 if (efx
->type
->udp_tnl_del_port
)
2463 (void)efx
->type
->udp_tnl_del_port(efx
, tnl
);
2466 static const struct net_device_ops efx_netdev_ops
= {
2467 .ndo_open
= efx_net_open
,
2468 .ndo_stop
= efx_net_stop
,
2469 .ndo_get_stats64
= efx_net_stats
,
2470 .ndo_tx_timeout
= efx_watchdog
,
2471 .ndo_start_xmit
= efx_hard_start_xmit
,
2472 .ndo_validate_addr
= eth_validate_addr
,
2473 .ndo_do_ioctl
= efx_ioctl
,
2474 .ndo_change_mtu
= efx_change_mtu
,
2475 .ndo_set_mac_address
= efx_set_mac_address
,
2476 .ndo_set_rx_mode
= efx_set_rx_mode
,
2477 .ndo_set_features
= efx_set_features
,
2478 .ndo_vlan_rx_add_vid
= efx_vlan_rx_add_vid
,
2479 .ndo_vlan_rx_kill_vid
= efx_vlan_rx_kill_vid
,
2480 #ifdef CONFIG_SFC_SRIOV
2481 .ndo_set_vf_mac
= efx_sriov_set_vf_mac
,
2482 .ndo_set_vf_vlan
= efx_sriov_set_vf_vlan
,
2483 .ndo_set_vf_spoofchk
= efx_sriov_set_vf_spoofchk
,
2484 .ndo_get_vf_config
= efx_sriov_get_vf_config
,
2485 .ndo_set_vf_link_state
= efx_sriov_set_vf_link_state
,
2487 .ndo_get_phys_port_id
= efx_get_phys_port_id
,
2488 .ndo_get_phys_port_name
= efx_get_phys_port_name
,
2489 .ndo_setup_tc
= efx_setup_tc
,
2490 #ifdef CONFIG_RFS_ACCEL
2491 .ndo_rx_flow_steer
= efx_filter_rfs
,
2493 .ndo_udp_tunnel_add
= efx_udp_tunnel_add
,
2494 .ndo_udp_tunnel_del
= efx_udp_tunnel_del
,
2497 static void efx_update_name(struct efx_nic
*efx
)
2499 strcpy(efx
->name
, efx
->net_dev
->name
);
2500 efx_mtd_rename(efx
);
2501 efx_set_channel_names(efx
);
2504 static int efx_netdev_event(struct notifier_block
*this,
2505 unsigned long event
, void *ptr
)
2507 struct net_device
*net_dev
= netdev_notifier_info_to_dev(ptr
);
2509 if ((net_dev
->netdev_ops
== &efx_netdev_ops
) &&
2510 event
== NETDEV_CHANGENAME
)
2511 efx_update_name(netdev_priv(net_dev
));
2516 static struct notifier_block efx_netdev_notifier
= {
2517 .notifier_call
= efx_netdev_event
,
2521 show_phy_type(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2523 struct efx_nic
*efx
= pci_get_drvdata(to_pci_dev(dev
));
2524 return sprintf(buf
, "%d\n", efx
->phy_type
);
2526 static DEVICE_ATTR(phy_type
, 0444, show_phy_type
, NULL
);
2528 #ifdef CONFIG_SFC_MCDI_LOGGING
2529 static ssize_t
show_mcdi_log(struct device
*dev
, struct device_attribute
*attr
,
2532 struct efx_nic
*efx
= pci_get_drvdata(to_pci_dev(dev
));
2533 struct efx_mcdi_iface
*mcdi
= efx_mcdi(efx
);
2535 return scnprintf(buf
, PAGE_SIZE
, "%d\n", mcdi
->logging_enabled
);
2537 static ssize_t
set_mcdi_log(struct device
*dev
, struct device_attribute
*attr
,
2538 const char *buf
, size_t count
)
2540 struct efx_nic
*efx
= pci_get_drvdata(to_pci_dev(dev
));
2541 struct efx_mcdi_iface
*mcdi
= efx_mcdi(efx
);
2542 bool enable
= count
> 0 && *buf
!= '0';
2544 mcdi
->logging_enabled
= enable
;
2547 static DEVICE_ATTR(mcdi_logging
, 0644, show_mcdi_log
, set_mcdi_log
);
2550 static int efx_register_netdev(struct efx_nic
*efx
)
2552 struct net_device
*net_dev
= efx
->net_dev
;
2553 struct efx_channel
*channel
;
2556 net_dev
->watchdog_timeo
= 5 * HZ
;
2557 net_dev
->irq
= efx
->pci_dev
->irq
;
2558 net_dev
->netdev_ops
= &efx_netdev_ops
;
2559 if (efx_nic_rev(efx
) >= EFX_REV_HUNT_A0
)
2560 net_dev
->priv_flags
|= IFF_UNICAST_FLT
;
2561 net_dev
->ethtool_ops
= &efx_ethtool_ops
;
2562 net_dev
->gso_max_segs
= EFX_TSO_MAX_SEGS
;
2563 net_dev
->min_mtu
= EFX_MIN_MTU
;
2564 net_dev
->max_mtu
= EFX_MAX_MTU
;
2568 /* Enable resets to be scheduled and check whether any were
2569 * already requested. If so, the NIC is probably hosed so we
2572 efx
->state
= STATE_READY
;
2573 smp_mb(); /* ensure we change state before checking reset_pending */
2574 if (efx
->reset_pending
) {
2575 netif_err(efx
, probe
, efx
->net_dev
,
2576 "aborting probe due to scheduled reset\n");
2581 rc
= dev_alloc_name(net_dev
, net_dev
->name
);
2584 efx_update_name(efx
);
2586 /* Always start with carrier off; PHY events will detect the link */
2587 netif_carrier_off(net_dev
);
2589 rc
= register_netdevice(net_dev
);
2593 efx_for_each_channel(channel
, efx
) {
2594 struct efx_tx_queue
*tx_queue
;
2595 efx_for_each_channel_tx_queue(tx_queue
, channel
)
2596 efx_init_tx_queue_core_txq(tx_queue
);
2603 rc
= device_create_file(&efx
->pci_dev
->dev
, &dev_attr_phy_type
);
2605 netif_err(efx
, drv
, efx
->net_dev
,
2606 "failed to init net dev attributes\n");
2607 goto fail_registered
;
2609 #ifdef CONFIG_SFC_MCDI_LOGGING
2610 rc
= device_create_file(&efx
->pci_dev
->dev
, &dev_attr_mcdi_logging
);
2612 netif_err(efx
, drv
, efx
->net_dev
,
2613 "failed to init net dev attributes\n");
2614 goto fail_attr_mcdi_logging
;
2620 #ifdef CONFIG_SFC_MCDI_LOGGING
2621 fail_attr_mcdi_logging
:
2622 device_remove_file(&efx
->pci_dev
->dev
, &dev_attr_phy_type
);
2626 efx_dissociate(efx
);
2627 unregister_netdevice(net_dev
);
2629 efx
->state
= STATE_UNINIT
;
2631 netif_err(efx
, drv
, efx
->net_dev
, "could not register net dev\n");
2635 static void efx_unregister_netdev(struct efx_nic
*efx
)
2640 BUG_ON(netdev_priv(efx
->net_dev
) != efx
);
2642 if (efx_dev_registered(efx
)) {
2643 strlcpy(efx
->name
, pci_name(efx
->pci_dev
), sizeof(efx
->name
));
2644 #ifdef CONFIG_SFC_MCDI_LOGGING
2645 device_remove_file(&efx
->pci_dev
->dev
, &dev_attr_mcdi_logging
);
2647 device_remove_file(&efx
->pci_dev
->dev
, &dev_attr_phy_type
);
2648 unregister_netdev(efx
->net_dev
);
2652 /**************************************************************************
2654 * Device reset and suspend
2656 **************************************************************************/
2658 /* Tears down the entire software state and most of the hardware state
2660 void efx_reset_down(struct efx_nic
*efx
, enum reset_type method
)
2662 EFX_ASSERT_RESET_SERIALISED(efx
);
2664 if (method
== RESET_TYPE_MCDI_TIMEOUT
)
2665 efx
->type
->prepare_flr(efx
);
2668 efx_disable_interrupts(efx
);
2670 mutex_lock(&efx
->mac_lock
);
2671 down_write(&efx
->filter_sem
);
2672 mutex_lock(&efx
->rss_lock
);
2673 if (efx
->port_initialized
&& method
!= RESET_TYPE_INVISIBLE
&&
2674 method
!= RESET_TYPE_DATAPATH
)
2675 efx
->phy_op
->fini(efx
);
2676 efx
->type
->fini(efx
);
2679 /* This function will always ensure that the locks acquired in
2680 * efx_reset_down() are released. A failure return code indicates
2681 * that we were unable to reinitialise the hardware, and the
2682 * driver should be disabled. If ok is false, then the rx and tx
2683 * engines are not restarted, pending a RESET_DISABLE. */
2684 int efx_reset_up(struct efx_nic
*efx
, enum reset_type method
, bool ok
)
2688 EFX_ASSERT_RESET_SERIALISED(efx
);
2690 if (method
== RESET_TYPE_MCDI_TIMEOUT
)
2691 efx
->type
->finish_flr(efx
);
2693 /* Ensure that SRAM is initialised even if we're disabling the device */
2694 rc
= efx
->type
->init(efx
);
2696 netif_err(efx
, drv
, efx
->net_dev
, "failed to initialise NIC\n");
2703 if (efx
->port_initialized
&& method
!= RESET_TYPE_INVISIBLE
&&
2704 method
!= RESET_TYPE_DATAPATH
) {
2705 rc
= efx
->phy_op
->init(efx
);
2708 rc
= efx
->phy_op
->reconfigure(efx
);
2709 if (rc
&& rc
!= -EPERM
)
2710 netif_err(efx
, drv
, efx
->net_dev
,
2711 "could not restore PHY settings\n");
2714 rc
= efx_enable_interrupts(efx
);
2718 #ifdef CONFIG_SFC_SRIOV
2719 rc
= efx
->type
->vswitching_restore(efx
);
2720 if (rc
) /* not fatal; the PF will still work fine */
2721 netif_warn(efx
, probe
, efx
->net_dev
,
2722 "failed to restore vswitching rc=%d;"
2723 " VFs may not function\n", rc
);
2726 if (efx
->type
->rx_restore_rss_contexts
)
2727 efx
->type
->rx_restore_rss_contexts(efx
);
2728 mutex_unlock(&efx
->rss_lock
);
2729 efx
->type
->filter_table_restore(efx
);
2730 up_write(&efx
->filter_sem
);
2731 if (efx
->type
->sriov_reset
)
2732 efx
->type
->sriov_reset(efx
);
2734 mutex_unlock(&efx
->mac_lock
);
2738 if (efx
->type
->udp_tnl_push_ports
)
2739 efx
->type
->udp_tnl_push_ports(efx
);
2744 efx
->port_initialized
= false;
2746 mutex_unlock(&efx
->rss_lock
);
2747 up_write(&efx
->filter_sem
);
2748 mutex_unlock(&efx
->mac_lock
);
2753 /* Reset the NIC using the specified method. Note that the reset may
2754 * fail, in which case the card will be left in an unusable state.
2756 * Caller must hold the rtnl_lock.
2758 int efx_reset(struct efx_nic
*efx
, enum reset_type method
)
2763 netif_info(efx
, drv
, efx
->net_dev
, "resetting (%s)\n",
2764 RESET_TYPE(method
));
2766 efx_device_detach_sync(efx
);
2767 efx_reset_down(efx
, method
);
2769 rc
= efx
->type
->reset(efx
, method
);
2771 netif_err(efx
, drv
, efx
->net_dev
, "failed to reset hardware\n");
2775 /* Clear flags for the scopes we covered. We assume the NIC and
2776 * driver are now quiescent so that there is no race here.
2778 if (method
< RESET_TYPE_MAX_METHOD
)
2779 efx
->reset_pending
&= -(1 << (method
+ 1));
2780 else /* it doesn't fit into the well-ordered scope hierarchy */
2781 __clear_bit(method
, &efx
->reset_pending
);
2783 /* Reinitialise bus-mastering, which may have been turned off before
2784 * the reset was scheduled. This is still appropriate, even in the
2785 * RESET_TYPE_DISABLE since this driver generally assumes the hardware
2786 * can respond to requests. */
2787 pci_set_master(efx
->pci_dev
);
2790 /* Leave device stopped if necessary */
2792 method
== RESET_TYPE_DISABLE
||
2793 method
== RESET_TYPE_RECOVER_OR_DISABLE
;
2794 rc2
= efx_reset_up(efx
, method
, !disabled
);
2802 dev_close(efx
->net_dev
);
2803 netif_err(efx
, drv
, efx
->net_dev
, "has been disabled\n");
2804 efx
->state
= STATE_DISABLED
;
2806 netif_dbg(efx
, drv
, efx
->net_dev
, "reset complete\n");
2807 efx_device_attach_if_not_resetting(efx
);
2812 /* Try recovery mechanisms.
2813 * For now only EEH is supported.
2814 * Returns 0 if the recovery mechanisms are unsuccessful.
2815 * Returns a non-zero value otherwise.
2817 int efx_try_recovery(struct efx_nic
*efx
)
2820 /* A PCI error can occur and not be seen by EEH because nothing
2821 * happens on the PCI bus. In this case the driver may fail and
2822 * schedule a 'recover or reset', leading to this recovery handler.
2823 * Manually call the eeh failure check function.
2825 struct eeh_dev
*eehdev
= pci_dev_to_eeh_dev(efx
->pci_dev
);
2826 if (eeh_dev_check_failure(eehdev
)) {
2827 /* The EEH mechanisms will handle the error and reset the
2828 * device if necessary.
2836 static void efx_wait_for_bist_end(struct efx_nic
*efx
)
2840 for (i
= 0; i
< BIST_WAIT_DELAY_COUNT
; ++i
) {
2841 if (efx_mcdi_poll_reboot(efx
))
2843 msleep(BIST_WAIT_DELAY_MS
);
2846 netif_err(efx
, drv
, efx
->net_dev
, "Warning: No MC reboot after BIST mode\n");
2848 /* Either way unset the BIST flag. If we found no reboot we probably
2849 * won't recover, but we should try.
2851 efx
->mc_bist_for_other_fn
= false;
2854 /* The worker thread exists so that code that cannot sleep can
2855 * schedule a reset for later.
2857 static void efx_reset_work(struct work_struct
*data
)
2859 struct efx_nic
*efx
= container_of(data
, struct efx_nic
, reset_work
);
2860 unsigned long pending
;
2861 enum reset_type method
;
2863 pending
= READ_ONCE(efx
->reset_pending
);
2864 method
= fls(pending
) - 1;
2866 if (method
== RESET_TYPE_MC_BIST
)
2867 efx_wait_for_bist_end(efx
);
2869 if ((method
== RESET_TYPE_RECOVER_OR_DISABLE
||
2870 method
== RESET_TYPE_RECOVER_OR_ALL
) &&
2871 efx_try_recovery(efx
))
2879 /* We checked the state in efx_schedule_reset() but it may
2880 * have changed by now. Now that we have the RTNL lock,
2881 * it cannot change again.
2883 if (efx
->state
== STATE_READY
)
2884 (void)efx_reset(efx
, method
);
2889 void efx_schedule_reset(struct efx_nic
*efx
, enum reset_type type
)
2891 enum reset_type method
;
2893 if (efx
->state
== STATE_RECOVERY
) {
2894 netif_dbg(efx
, drv
, efx
->net_dev
,
2895 "recovering: skip scheduling %s reset\n",
2901 case RESET_TYPE_INVISIBLE
:
2902 case RESET_TYPE_ALL
:
2903 case RESET_TYPE_RECOVER_OR_ALL
:
2904 case RESET_TYPE_WORLD
:
2905 case RESET_TYPE_DISABLE
:
2906 case RESET_TYPE_RECOVER_OR_DISABLE
:
2907 case RESET_TYPE_DATAPATH
:
2908 case RESET_TYPE_MC_BIST
:
2909 case RESET_TYPE_MCDI_TIMEOUT
:
2911 netif_dbg(efx
, drv
, efx
->net_dev
, "scheduling %s reset\n",
2912 RESET_TYPE(method
));
2915 method
= efx
->type
->map_reset_reason(type
);
2916 netif_dbg(efx
, drv
, efx
->net_dev
,
2917 "scheduling %s reset for %s\n",
2918 RESET_TYPE(method
), RESET_TYPE(type
));
2922 set_bit(method
, &efx
->reset_pending
);
2923 smp_mb(); /* ensure we change reset_pending before checking state */
2925 /* If we're not READY then just leave the flags set as the cue
2926 * to abort probing or reschedule the reset later.
2928 if (READ_ONCE(efx
->state
) != STATE_READY
)
2931 /* efx_process_channel() will no longer read events once a
2932 * reset is scheduled. So switch back to poll'd MCDI completions. */
2933 efx_mcdi_mode_poll(efx
);
2935 queue_work(reset_workqueue
, &efx
->reset_work
);
2938 /**************************************************************************
2940 * List of NICs we support
2942 **************************************************************************/
2944 /* PCI device ID table */
2945 static const struct pci_device_id efx_pci_table
[] = {
2946 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE
, 0x0803), /* SFC9020 */
2947 .driver_data
= (unsigned long) &siena_a0_nic_type
},
2948 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE
, 0x0813), /* SFL9021 */
2949 .driver_data
= (unsigned long) &siena_a0_nic_type
},
2950 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE
, 0x0903), /* SFC9120 PF */
2951 .driver_data
= (unsigned long) &efx_hunt_a0_nic_type
},
2952 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE
, 0x1903), /* SFC9120 VF */
2953 .driver_data
= (unsigned long) &efx_hunt_a0_vf_nic_type
},
2954 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE
, 0x0923), /* SFC9140 PF */
2955 .driver_data
= (unsigned long) &efx_hunt_a0_nic_type
},
2956 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE
, 0x1923), /* SFC9140 VF */
2957 .driver_data
= (unsigned long) &efx_hunt_a0_vf_nic_type
},
2958 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE
, 0x0a03), /* SFC9220 PF */
2959 .driver_data
= (unsigned long) &efx_hunt_a0_nic_type
},
2960 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE
, 0x1a03), /* SFC9220 VF */
2961 .driver_data
= (unsigned long) &efx_hunt_a0_vf_nic_type
},
2962 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE
, 0x0b03), /* SFC9250 PF */
2963 .driver_data
= (unsigned long) &efx_hunt_a0_nic_type
},
2964 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE
, 0x1b03), /* SFC9250 VF */
2965 .driver_data
= (unsigned long) &efx_hunt_a0_vf_nic_type
},
2966 {0} /* end of list */
2969 /**************************************************************************
2971 * Dummy PHY/MAC operations
2973 * Can be used for some unimplemented operations
2974 * Needed so all function pointers are valid and do not have to be tested
2977 **************************************************************************/
2978 int efx_port_dummy_op_int(struct efx_nic
*efx
)
2982 void efx_port_dummy_op_void(struct efx_nic
*efx
) {}
2984 static bool efx_port_dummy_op_poll(struct efx_nic
*efx
)
2989 static const struct efx_phy_operations efx_dummy_phy_operations
= {
2990 .init
= efx_port_dummy_op_int
,
2991 .reconfigure
= efx_port_dummy_op_int
,
2992 .poll
= efx_port_dummy_op_poll
,
2993 .fini
= efx_port_dummy_op_void
,
2996 /**************************************************************************
3000 **************************************************************************/
3002 /* This zeroes out and then fills in the invariants in a struct
3003 * efx_nic (including all sub-structures).
3005 static int efx_init_struct(struct efx_nic
*efx
,
3006 struct pci_dev
*pci_dev
, struct net_device
*net_dev
)
3008 int rc
= -ENOMEM
, i
;
3010 /* Initialise common structures */
3011 INIT_LIST_HEAD(&efx
->node
);
3012 INIT_LIST_HEAD(&efx
->secondary_list
);
3013 spin_lock_init(&efx
->biu_lock
);
3014 #ifdef CONFIG_SFC_MTD
3015 INIT_LIST_HEAD(&efx
->mtd_list
);
3017 INIT_WORK(&efx
->reset_work
, efx_reset_work
);
3018 INIT_DELAYED_WORK(&efx
->monitor_work
, efx_monitor
);
3019 INIT_DELAYED_WORK(&efx
->selftest_work
, efx_selftest_async_work
);
3020 efx
->pci_dev
= pci_dev
;
3021 efx
->msg_enable
= debug
;
3022 efx
->state
= STATE_UNINIT
;
3023 strlcpy(efx
->name
, pci_name(pci_dev
), sizeof(efx
->name
));
3025 efx
->net_dev
= net_dev
;
3026 efx
->rx_prefix_size
= efx
->type
->rx_prefix_size
;
3028 NET_IP_ALIGN
? (efx
->rx_prefix_size
+ NET_IP_ALIGN
) % 4 : 0;
3029 efx
->rx_packet_hash_offset
=
3030 efx
->type
->rx_hash_offset
- efx
->type
->rx_prefix_size
;
3031 efx
->rx_packet_ts_offset
=
3032 efx
->type
->rx_ts_offset
- efx
->type
->rx_prefix_size
;
3033 INIT_LIST_HEAD(&efx
->rss_context
.list
);
3034 mutex_init(&efx
->rss_lock
);
3035 spin_lock_init(&efx
->stats_lock
);
3036 efx
->vi_stride
= EFX_DEFAULT_VI_STRIDE
;
3037 efx
->num_mac_stats
= MC_CMD_MAC_NSTATS
;
3038 BUILD_BUG_ON(MC_CMD_MAC_NSTATS
- 1 != MC_CMD_MAC_GENERATION_END
);
3039 mutex_init(&efx
->mac_lock
);
3040 #ifdef CONFIG_RFS_ACCEL
3041 mutex_init(&efx
->rps_mutex
);
3042 spin_lock_init(&efx
->rps_hash_lock
);
3043 /* Failure to allocate is not fatal, but may degrade ARFS performance */
3044 efx
->rps_hash_table
= kcalloc(EFX_ARFS_HASH_TABLE_SIZE
,
3045 sizeof(*efx
->rps_hash_table
), GFP_KERNEL
);
3047 efx
->phy_op
= &efx_dummy_phy_operations
;
3048 efx
->mdio
.dev
= net_dev
;
3049 INIT_WORK(&efx
->mac_work
, efx_mac_work
);
3050 init_waitqueue_head(&efx
->flush_wq
);
3052 for (i
= 0; i
< EFX_MAX_CHANNELS
; i
++) {
3053 efx
->channel
[i
] = efx_alloc_channel(efx
, i
, NULL
);
3054 if (!efx
->channel
[i
])
3056 efx
->msi_context
[i
].efx
= efx
;
3057 efx
->msi_context
[i
].index
= i
;
3060 /* Higher numbered interrupt modes are less capable! */
3061 if (WARN_ON_ONCE(efx
->type
->max_interrupt_mode
>
3062 efx
->type
->min_interrupt_mode
)) {
3066 efx
->interrupt_mode
= max(efx
->type
->max_interrupt_mode
,
3068 efx
->interrupt_mode
= min(efx
->type
->min_interrupt_mode
,
3071 /* Would be good to use the net_dev name, but we're too early */
3072 snprintf(efx
->workqueue_name
, sizeof(efx
->workqueue_name
), "sfc%s",
3074 efx
->workqueue
= create_singlethread_workqueue(efx
->workqueue_name
);
3075 if (!efx
->workqueue
)
3081 efx_fini_struct(efx
);
3085 static void efx_fini_struct(struct efx_nic
*efx
)
3089 #ifdef CONFIG_RFS_ACCEL
3090 kfree(efx
->rps_hash_table
);
3093 for (i
= 0; i
< EFX_MAX_CHANNELS
; i
++)
3094 kfree(efx
->channel
[i
]);
3098 if (efx
->workqueue
) {
3099 destroy_workqueue(efx
->workqueue
);
3100 efx
->workqueue
= NULL
;
3104 void efx_update_sw_stats(struct efx_nic
*efx
, u64
*stats
)
3106 u64 n_rx_nodesc_trunc
= 0;
3107 struct efx_channel
*channel
;
3109 efx_for_each_channel(channel
, efx
)
3110 n_rx_nodesc_trunc
+= channel
->n_rx_nodesc_trunc
;
3111 stats
[GENERIC_STAT_rx_nodesc_trunc
] = n_rx_nodesc_trunc
;
3112 stats
[GENERIC_STAT_rx_noskb_drops
] = atomic_read(&efx
->n_rx_noskb_drops
);
3115 bool efx_filter_spec_equal(const struct efx_filter_spec
*left
,
3116 const struct efx_filter_spec
*right
)
3118 if ((left
->match_flags
^ right
->match_flags
) |
3119 ((left
->flags
^ right
->flags
) &
3120 (EFX_FILTER_FLAG_RX
| EFX_FILTER_FLAG_TX
)))
3123 return memcmp(&left
->outer_vid
, &right
->outer_vid
,
3124 sizeof(struct efx_filter_spec
) -
3125 offsetof(struct efx_filter_spec
, outer_vid
)) == 0;
3128 u32
efx_filter_spec_hash(const struct efx_filter_spec
*spec
)
3130 BUILD_BUG_ON(offsetof(struct efx_filter_spec
, outer_vid
) & 3);
3131 return jhash2((const u32
*)&spec
->outer_vid
,
3132 (sizeof(struct efx_filter_spec
) -
3133 offsetof(struct efx_filter_spec
, outer_vid
)) / 4,
3137 #ifdef CONFIG_RFS_ACCEL
3138 bool efx_rps_check_rule(struct efx_arfs_rule
*rule
, unsigned int filter_idx
,
3141 if (rule
->filter_id
== EFX_ARFS_FILTER_ID_PENDING
) {
3142 /* ARFS is currently updating this entry, leave it */
3145 if (rule
->filter_id
== EFX_ARFS_FILTER_ID_ERROR
) {
3146 /* ARFS tried and failed to update this, so it's probably out
3147 * of date. Remove the filter and the ARFS rule entry.
3149 rule
->filter_id
= EFX_ARFS_FILTER_ID_REMOVING
;
3152 } else if (WARN_ON(rule
->filter_id
!= filter_idx
)) { /* can't happen */
3153 /* ARFS has moved on, so old filter is not needed. Since we did
3154 * not mark the rule with EFX_ARFS_FILTER_ID_REMOVING, it will
3155 * not be removed by efx_rps_hash_del() subsequently.
3160 /* Remove it iff ARFS wants to. */
3165 struct hlist_head
*efx_rps_hash_bucket(struct efx_nic
*efx
,
3166 const struct efx_filter_spec
*spec
)
3168 u32 hash
= efx_filter_spec_hash(spec
);
3170 lockdep_assert_held(&efx
->rps_hash_lock
);
3171 if (!efx
->rps_hash_table
)
3173 return &efx
->rps_hash_table
[hash
% EFX_ARFS_HASH_TABLE_SIZE
];
3176 struct efx_arfs_rule
*efx_rps_hash_find(struct efx_nic
*efx
,
3177 const struct efx_filter_spec
*spec
)
3179 struct efx_arfs_rule
*rule
;
3180 struct hlist_head
*head
;
3181 struct hlist_node
*node
;
3183 head
= efx_rps_hash_bucket(efx
, spec
);
3186 hlist_for_each(node
, head
) {
3187 rule
= container_of(node
, struct efx_arfs_rule
, node
);
3188 if (efx_filter_spec_equal(spec
, &rule
->spec
))
3194 struct efx_arfs_rule
*efx_rps_hash_add(struct efx_nic
*efx
,
3195 const struct efx_filter_spec
*spec
,
3198 struct efx_arfs_rule
*rule
;
3199 struct hlist_head
*head
;
3200 struct hlist_node
*node
;
3202 head
= efx_rps_hash_bucket(efx
, spec
);
3205 hlist_for_each(node
, head
) {
3206 rule
= container_of(node
, struct efx_arfs_rule
, node
);
3207 if (efx_filter_spec_equal(spec
, &rule
->spec
)) {
3212 rule
= kmalloc(sizeof(*rule
), GFP_ATOMIC
);
3215 memcpy(&rule
->spec
, spec
, sizeof(rule
->spec
));
3216 hlist_add_head(&rule
->node
, head
);
3221 void efx_rps_hash_del(struct efx_nic
*efx
, const struct efx_filter_spec
*spec
)
3223 struct efx_arfs_rule
*rule
;
3224 struct hlist_head
*head
;
3225 struct hlist_node
*node
;
3227 head
= efx_rps_hash_bucket(efx
, spec
);
3230 hlist_for_each(node
, head
) {
3231 rule
= container_of(node
, struct efx_arfs_rule
, node
);
3232 if (efx_filter_spec_equal(spec
, &rule
->spec
)) {
3233 /* Someone already reused the entry. We know that if
3234 * this check doesn't fire (i.e. filter_id == REMOVING)
3235 * then the REMOVING mark was put there by our caller,
3236 * because caller is holding a lock on filter table and
3237 * only holders of that lock set REMOVING.
3239 if (rule
->filter_id
!= EFX_ARFS_FILTER_ID_REMOVING
)
3246 /* We didn't find it. */
3251 /* RSS contexts. We're using linked lists and crappy O(n) algorithms, because
3252 * (a) this is an infrequent control-plane operation and (b) n is small (max 64)
3254 struct efx_rss_context
*efx_alloc_rss_context_entry(struct efx_nic
*efx
)
3256 struct list_head
*head
= &efx
->rss_context
.list
;
3257 struct efx_rss_context
*ctx
, *new;
3258 u32 id
= 1; /* Don't use zero, that refers to the master RSS context */
3260 WARN_ON(!mutex_is_locked(&efx
->rss_lock
));
3262 /* Search for first gap in the numbering */
3263 list_for_each_entry(ctx
, head
, list
) {
3264 if (ctx
->user_id
!= id
)
3267 /* Check for wrap. If this happens, we have nearly 2^32
3268 * allocated RSS contexts, which seems unlikely.
3270 if (WARN_ON_ONCE(!id
))
3274 /* Create the new entry */
3275 new = kmalloc(sizeof(struct efx_rss_context
), GFP_KERNEL
);
3278 new->context_id
= EFX_EF10_RSS_CONTEXT_INVALID
;
3279 new->rx_hash_udp_4tuple
= false;
3281 /* Insert the new entry into the gap */
3283 list_add_tail(&new->list
, &ctx
->list
);
3287 struct efx_rss_context
*efx_find_rss_context_entry(struct efx_nic
*efx
, u32 id
)
3289 struct list_head
*head
= &efx
->rss_context
.list
;
3290 struct efx_rss_context
*ctx
;
3292 WARN_ON(!mutex_is_locked(&efx
->rss_lock
));
3294 list_for_each_entry(ctx
, head
, list
)
3295 if (ctx
->user_id
== id
)
3300 void efx_free_rss_context_entry(struct efx_rss_context
*ctx
)
3302 list_del(&ctx
->list
);
3306 /**************************************************************************
3310 **************************************************************************/
3312 /* Main body of final NIC shutdown code
3313 * This is called only at module unload (or hotplug removal).
3315 static void efx_pci_remove_main(struct efx_nic
*efx
)
3317 /* Flush reset_work. It can no longer be scheduled since we
3320 BUG_ON(efx
->state
== STATE_READY
);
3321 cancel_work_sync(&efx
->reset_work
);
3323 efx_disable_interrupts(efx
);
3324 efx_clear_interrupt_affinity(efx
);
3325 efx_nic_fini_interrupt(efx
);
3327 efx
->type
->fini(efx
);
3329 efx_remove_all(efx
);
3332 /* Final NIC shutdown
3333 * This is called only at module unload (or hotplug removal). A PF can call
3334 * this on its VFs to ensure they are unbound first.
3336 static void efx_pci_remove(struct pci_dev
*pci_dev
)
3338 struct efx_nic
*efx
;
3340 efx
= pci_get_drvdata(pci_dev
);
3344 /* Mark the NIC as fini, then stop the interface */
3346 efx_dissociate(efx
);
3347 dev_close(efx
->net_dev
);
3348 efx_disable_interrupts(efx
);
3349 efx
->state
= STATE_UNINIT
;
3352 if (efx
->type
->sriov_fini
)
3353 efx
->type
->sriov_fini(efx
);
3355 efx_unregister_netdev(efx
);
3357 efx_mtd_remove(efx
);
3359 efx_pci_remove_main(efx
);
3362 netif_dbg(efx
, drv
, efx
->net_dev
, "shutdown successful\n");
3364 efx_fini_struct(efx
);
3365 free_netdev(efx
->net_dev
);
3367 pci_disable_pcie_error_reporting(pci_dev
);
3370 /* NIC VPD information
3371 * Called during probe to display the part number of the
3372 * installed NIC. VPD is potentially very large but this should
3373 * always appear within the first 512 bytes.
3375 #define SFC_VPD_LEN 512
3376 static void efx_probe_vpd_strings(struct efx_nic
*efx
)
3378 struct pci_dev
*dev
= efx
->pci_dev
;
3379 char vpd_data
[SFC_VPD_LEN
];
3381 int ro_start
, ro_size
, i
, j
;
3383 /* Get the vpd data from the device */
3384 vpd_size
= pci_read_vpd(dev
, 0, sizeof(vpd_data
), vpd_data
);
3385 if (vpd_size
<= 0) {
3386 netif_err(efx
, drv
, efx
->net_dev
, "Unable to read VPD\n");
3390 /* Get the Read only section */
3391 ro_start
= pci_vpd_find_tag(vpd_data
, 0, vpd_size
, PCI_VPD_LRDT_RO_DATA
);
3393 netif_err(efx
, drv
, efx
->net_dev
, "VPD Read-only not found\n");
3397 ro_size
= pci_vpd_lrdt_size(&vpd_data
[ro_start
]);
3399 i
= ro_start
+ PCI_VPD_LRDT_TAG_SIZE
;
3400 if (i
+ j
> vpd_size
)
3403 /* Get the Part number */
3404 i
= pci_vpd_find_info_keyword(vpd_data
, i
, j
, "PN");
3406 netif_err(efx
, drv
, efx
->net_dev
, "Part number not found\n");
3410 j
= pci_vpd_info_field_size(&vpd_data
[i
]);
3411 i
+= PCI_VPD_INFO_FLD_HDR_SIZE
;
3412 if (i
+ j
> vpd_size
) {
3413 netif_err(efx
, drv
, efx
->net_dev
, "Incomplete part number\n");
3417 netif_info(efx
, drv
, efx
->net_dev
,
3418 "Part Number : %.*s\n", j
, &vpd_data
[i
]);
3420 i
= ro_start
+ PCI_VPD_LRDT_TAG_SIZE
;
3422 i
= pci_vpd_find_info_keyword(vpd_data
, i
, j
, "SN");
3424 netif_err(efx
, drv
, efx
->net_dev
, "Serial number not found\n");
3428 j
= pci_vpd_info_field_size(&vpd_data
[i
]);
3429 i
+= PCI_VPD_INFO_FLD_HDR_SIZE
;
3430 if (i
+ j
> vpd_size
) {
3431 netif_err(efx
, drv
, efx
->net_dev
, "Incomplete serial number\n");
3435 efx
->vpd_sn
= kmalloc(j
+ 1, GFP_KERNEL
);
3439 snprintf(efx
->vpd_sn
, j
+ 1, "%s", &vpd_data
[i
]);
3443 /* Main body of NIC initialisation
3444 * This is called at module load (or hotplug insertion, theoretically).
3446 static int efx_pci_probe_main(struct efx_nic
*efx
)
3450 /* Do start-of-day initialisation */
3451 rc
= efx_probe_all(efx
);
3457 down_write(&efx
->filter_sem
);
3458 rc
= efx
->type
->init(efx
);
3459 up_write(&efx
->filter_sem
);
3461 netif_err(efx
, probe
, efx
->net_dev
,
3462 "failed to initialise NIC\n");
3466 rc
= efx_init_port(efx
);
3468 netif_err(efx
, probe
, efx
->net_dev
,
3469 "failed to initialise port\n");
3473 rc
= efx_nic_init_interrupt(efx
);
3477 efx_set_interrupt_affinity(efx
);
3478 rc
= efx_enable_interrupts(efx
);
3485 efx_clear_interrupt_affinity(efx
);
3486 efx_nic_fini_interrupt(efx
);
3490 efx
->type
->fini(efx
);
3493 efx_remove_all(efx
);
3498 static int efx_pci_probe_post_io(struct efx_nic
*efx
)
3500 struct net_device
*net_dev
= efx
->net_dev
;
3501 int rc
= efx_pci_probe_main(efx
);
3506 if (efx
->type
->sriov_init
) {
3507 rc
= efx
->type
->sriov_init(efx
);
3509 netif_err(efx
, probe
, efx
->net_dev
,
3510 "SR-IOV can't be enabled rc %d\n", rc
);
3513 /* Determine netdevice features */
3514 net_dev
->features
|= (efx
->type
->offload_features
| NETIF_F_SG
|
3515 NETIF_F_TSO
| NETIF_F_RXCSUM
| NETIF_F_RXALL
);
3516 if (efx
->type
->offload_features
& (NETIF_F_IPV6_CSUM
| NETIF_F_HW_CSUM
))
3517 net_dev
->features
|= NETIF_F_TSO6
;
3518 /* Check whether device supports TSO */
3519 if (!efx
->type
->tso_versions
|| !efx
->type
->tso_versions(efx
))
3520 net_dev
->features
&= ~NETIF_F_ALL_TSO
;
3521 /* Mask for features that also apply to VLAN devices */
3522 net_dev
->vlan_features
|= (NETIF_F_HW_CSUM
| NETIF_F_SG
|
3523 NETIF_F_HIGHDMA
| NETIF_F_ALL_TSO
|
3526 net_dev
->hw_features
|= net_dev
->features
& ~efx
->fixed_features
;
3528 /* Disable receiving frames with bad FCS, by default. */
3529 net_dev
->features
&= ~NETIF_F_RXALL
;
3531 /* Disable VLAN filtering by default. It may be enforced if
3532 * the feature is fixed (i.e. VLAN filters are required to
3533 * receive VLAN tagged packets due to vPort restrictions).
3535 net_dev
->features
&= ~NETIF_F_HW_VLAN_CTAG_FILTER
;
3536 net_dev
->features
|= efx
->fixed_features
;
3538 rc
= efx_register_netdev(efx
);
3542 efx_pci_remove_main(efx
);
3546 /* NIC initialisation
3548 * This is called at module load (or hotplug insertion,
3549 * theoretically). It sets up PCI mappings, resets the NIC,
3550 * sets up and registers the network devices with the kernel and hooks
3551 * the interrupt service routine. It does not prepare the device for
3552 * transmission; this is left to the first time one of the network
3553 * interfaces is brought up (i.e. efx_net_open).
3555 static int efx_pci_probe(struct pci_dev
*pci_dev
,
3556 const struct pci_device_id
*entry
)
3558 struct net_device
*net_dev
;
3559 struct efx_nic
*efx
;
3562 /* Allocate and initialise a struct net_device and struct efx_nic */
3563 net_dev
= alloc_etherdev_mqs(sizeof(*efx
), EFX_MAX_CORE_TX_QUEUES
,
3567 efx
= netdev_priv(net_dev
);
3568 efx
->type
= (const struct efx_nic_type
*) entry
->driver_data
;
3569 efx
->fixed_features
|= NETIF_F_HIGHDMA
;
3571 pci_set_drvdata(pci_dev
, efx
);
3572 SET_NETDEV_DEV(net_dev
, &pci_dev
->dev
);
3573 rc
= efx_init_struct(efx
, pci_dev
, net_dev
);
3577 netif_info(efx
, probe
, efx
->net_dev
,
3578 "Solarflare NIC detected\n");
3580 if (!efx
->type
->is_vf
)
3581 efx_probe_vpd_strings(efx
);
3583 /* Set up basic I/O (BAR mappings etc) */
3584 rc
= efx_init_io(efx
);
3588 rc
= efx_pci_probe_post_io(efx
);
3590 /* On failure, retry once immediately.
3591 * If we aborted probe due to a scheduled reset, dismiss it.
3593 efx
->reset_pending
= 0;
3594 rc
= efx_pci_probe_post_io(efx
);
3596 /* On another failure, retry once more
3597 * after a 50-305ms delay.
3601 get_random_bytes(&r
, 1);
3602 msleep((unsigned int)r
+ 50);
3603 efx
->reset_pending
= 0;
3604 rc
= efx_pci_probe_post_io(efx
);
3610 netif_dbg(efx
, probe
, efx
->net_dev
, "initialisation successful\n");
3612 /* Try to create MTDs, but allow this to fail */
3614 rc
= efx_mtd_probe(efx
);
3616 if (rc
&& rc
!= -EPERM
)
3617 netif_warn(efx
, probe
, efx
->net_dev
,
3618 "failed to create MTDs (%d)\n", rc
);
3620 rc
= pci_enable_pcie_error_reporting(pci_dev
);
3621 if (rc
&& rc
!= -EINVAL
)
3622 netif_notice(efx
, probe
, efx
->net_dev
,
3623 "PCIE error reporting unavailable (%d).\n",
3626 if (efx
->type
->udp_tnl_push_ports
)
3627 efx
->type
->udp_tnl_push_ports(efx
);
3634 efx_fini_struct(efx
);
3637 netif_dbg(efx
, drv
, efx
->net_dev
, "initialisation failed. rc=%d\n", rc
);
3638 free_netdev(net_dev
);
3642 /* efx_pci_sriov_configure returns the actual number of Virtual Functions
3643 * enabled on success
3645 #ifdef CONFIG_SFC_SRIOV
3646 static int efx_pci_sriov_configure(struct pci_dev
*dev
, int num_vfs
)
3649 struct efx_nic
*efx
= pci_get_drvdata(dev
);
3651 if (efx
->type
->sriov_configure
) {
3652 rc
= efx
->type
->sriov_configure(efx
, num_vfs
);
3662 static int efx_pm_freeze(struct device
*dev
)
3664 struct efx_nic
*efx
= pci_get_drvdata(to_pci_dev(dev
));
3668 if (efx
->state
!= STATE_DISABLED
) {
3669 efx
->state
= STATE_UNINIT
;
3671 efx_device_detach_sync(efx
);
3674 efx_disable_interrupts(efx
);
3682 static int efx_pm_thaw(struct device
*dev
)
3685 struct efx_nic
*efx
= pci_get_drvdata(to_pci_dev(dev
));
3689 if (efx
->state
!= STATE_DISABLED
) {
3690 rc
= efx_enable_interrupts(efx
);
3694 mutex_lock(&efx
->mac_lock
);
3695 efx
->phy_op
->reconfigure(efx
);
3696 mutex_unlock(&efx
->mac_lock
);
3700 efx_device_attach_if_not_resetting(efx
);
3702 efx
->state
= STATE_READY
;
3704 efx
->type
->resume_wol(efx
);
3709 /* Reschedule any quenched resets scheduled during efx_pm_freeze() */
3710 queue_work(reset_workqueue
, &efx
->reset_work
);
3720 static int efx_pm_poweroff(struct device
*dev
)
3722 struct pci_dev
*pci_dev
= to_pci_dev(dev
);
3723 struct efx_nic
*efx
= pci_get_drvdata(pci_dev
);
3725 efx
->type
->fini(efx
);
3727 efx
->reset_pending
= 0;
3729 pci_save_state(pci_dev
);
3730 return pci_set_power_state(pci_dev
, PCI_D3hot
);
3733 /* Used for both resume and restore */
3734 static int efx_pm_resume(struct device
*dev
)
3736 struct pci_dev
*pci_dev
= to_pci_dev(dev
);
3737 struct efx_nic
*efx
= pci_get_drvdata(pci_dev
);
3740 rc
= pci_set_power_state(pci_dev
, PCI_D0
);
3743 pci_restore_state(pci_dev
);
3744 rc
= pci_enable_device(pci_dev
);
3747 pci_set_master(efx
->pci_dev
);
3748 rc
= efx
->type
->reset(efx
, RESET_TYPE_ALL
);
3751 down_write(&efx
->filter_sem
);
3752 rc
= efx
->type
->init(efx
);
3753 up_write(&efx
->filter_sem
);
3756 rc
= efx_pm_thaw(dev
);
3760 static int efx_pm_suspend(struct device
*dev
)
3765 rc
= efx_pm_poweroff(dev
);
3771 static const struct dev_pm_ops efx_pm_ops
= {
3772 .suspend
= efx_pm_suspend
,
3773 .resume
= efx_pm_resume
,
3774 .freeze
= efx_pm_freeze
,
3775 .thaw
= efx_pm_thaw
,
3776 .poweroff
= efx_pm_poweroff
,
3777 .restore
= efx_pm_resume
,
3780 /* A PCI error affecting this device was detected.
3781 * At this point MMIO and DMA may be disabled.
3782 * Stop the software path and request a slot reset.
3784 static pci_ers_result_t
efx_io_error_detected(struct pci_dev
*pdev
,
3785 enum pci_channel_state state
)
3787 pci_ers_result_t status
= PCI_ERS_RESULT_RECOVERED
;
3788 struct efx_nic
*efx
= pci_get_drvdata(pdev
);
3790 if (state
== pci_channel_io_perm_failure
)
3791 return PCI_ERS_RESULT_DISCONNECT
;
3795 if (efx
->state
!= STATE_DISABLED
) {
3796 efx
->state
= STATE_RECOVERY
;
3797 efx
->reset_pending
= 0;
3799 efx_device_detach_sync(efx
);
3802 efx_disable_interrupts(efx
);
3804 status
= PCI_ERS_RESULT_NEED_RESET
;
3806 /* If the interface is disabled we don't want to do anything
3809 status
= PCI_ERS_RESULT_RECOVERED
;
3814 pci_disable_device(pdev
);
3819 /* Fake a successful reset, which will be performed later in efx_io_resume. */
3820 static pci_ers_result_t
efx_io_slot_reset(struct pci_dev
*pdev
)
3822 struct efx_nic
*efx
= pci_get_drvdata(pdev
);
3823 pci_ers_result_t status
= PCI_ERS_RESULT_RECOVERED
;
3825 if (pci_enable_device(pdev
)) {
3826 netif_err(efx
, hw
, efx
->net_dev
,
3827 "Cannot re-enable PCI device after reset.\n");
3828 status
= PCI_ERS_RESULT_DISCONNECT
;
3834 /* Perform the actual reset and resume I/O operations. */
3835 static void efx_io_resume(struct pci_dev
*pdev
)
3837 struct efx_nic
*efx
= pci_get_drvdata(pdev
);
3842 if (efx
->state
== STATE_DISABLED
)
3845 rc
= efx_reset(efx
, RESET_TYPE_ALL
);
3847 netif_err(efx
, hw
, efx
->net_dev
,
3848 "efx_reset failed after PCI error (%d)\n", rc
);
3850 efx
->state
= STATE_READY
;
3851 netif_dbg(efx
, hw
, efx
->net_dev
,
3852 "Done resetting and resuming IO after PCI error.\n");
3859 /* For simplicity and reliability, we always require a slot reset and try to
3860 * reset the hardware when a pci error affecting the device is detected.
3861 * We leave both the link_reset and mmio_enabled callback unimplemented:
3862 * with our request for slot reset the mmio_enabled callback will never be
3863 * called, and the link_reset callback is not used by AER or EEH mechanisms.
3865 static const struct pci_error_handlers efx_err_handlers
= {
3866 .error_detected
= efx_io_error_detected
,
3867 .slot_reset
= efx_io_slot_reset
,
3868 .resume
= efx_io_resume
,
3871 static struct pci_driver efx_pci_driver
= {
3872 .name
= KBUILD_MODNAME
,
3873 .id_table
= efx_pci_table
,
3874 .probe
= efx_pci_probe
,
3875 .remove
= efx_pci_remove
,
3876 .driver
.pm
= &efx_pm_ops
,
3877 .err_handler
= &efx_err_handlers
,
3878 #ifdef CONFIG_SFC_SRIOV
3879 .sriov_configure
= efx_pci_sriov_configure
,
3883 /**************************************************************************
3885 * Kernel module interface
3887 *************************************************************************/
3889 module_param(interrupt_mode
, uint
, 0444);
3890 MODULE_PARM_DESC(interrupt_mode
,
3891 "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)");
3893 static int __init
efx_init_module(void)
3897 printk(KERN_INFO
"Solarflare NET driver v" EFX_DRIVER_VERSION
"\n");
3899 rc
= register_netdevice_notifier(&efx_netdev_notifier
);
3903 #ifdef CONFIG_SFC_SRIOV
3904 rc
= efx_init_sriov();
3909 reset_workqueue
= create_singlethread_workqueue("sfc_reset");
3910 if (!reset_workqueue
) {
3915 rc
= pci_register_driver(&efx_pci_driver
);
3922 destroy_workqueue(reset_workqueue
);
3924 #ifdef CONFIG_SFC_SRIOV
3928 unregister_netdevice_notifier(&efx_netdev_notifier
);
3933 static void __exit
efx_exit_module(void)
3935 printk(KERN_INFO
"Solarflare NET driver unloading\n");
3937 pci_unregister_driver(&efx_pci_driver
);
3938 destroy_workqueue(reset_workqueue
);
3939 #ifdef CONFIG_SFC_SRIOV
3942 unregister_netdevice_notifier(&efx_netdev_notifier
);
3946 module_init(efx_init_module
);
3947 module_exit(efx_exit_module
);
3949 MODULE_AUTHOR("Solarflare Communications and "
3950 "Michael Brown <mbrown@fensystems.co.uk>");
3951 MODULE_DESCRIPTION("Solarflare network driver");
3952 MODULE_LICENSE("GPL");
3953 MODULE_DEVICE_TABLE(pci
, efx_pci_table
);
3954 MODULE_VERSION(EFX_DRIVER_VERSION
);