1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /**************************************************************************/
4 /* IBM System i and System p Virtual NIC Device Driver */
5 /* Copyright (C) 2014 IBM Corp. */
6 /* Santiago Leon (santi_leon@yahoo.com) */
7 /* Thomas Falcon (tlfalcon@linux.vnet.ibm.com) */
8 /* John Allen (jallen@linux.vnet.ibm.com) */
11 /* This module contains the implementation of a virtual ethernet device */
12 /* for use with IBM i/p Series LPAR Linux. It utilizes the logical LAN */
13 /* option of the RS/6000 Platform Architecture to interface with virtual */
14 /* ethernet NICs that are presented to the partition by the hypervisor. */
16 /* Messages are passed between the VNIC driver and the VNIC server using */
17 /* Command/Response Queues (CRQs) and sub CRQs (sCRQs). CRQs are used to */
18 /* issue and receive commands that initiate communication with the server */
19 /* on driver initialization. Sub CRQs (sCRQs) are similar to CRQs, but */
20 /* are used by the driver to notify the server that a packet is */
21 /* ready for transmission or that a buffer has been added to receive a */
22 /* packet. Subsequently, sCRQs are used by the server to notify the */
23 /* driver that a packet transmission has been completed or that a packet */
24 /* has been received and placed in a waiting buffer. */
26 /* In lieu of a more conventional "on-the-fly" DMA mapping strategy in */
27 /* which skbs are DMA mapped and immediately unmapped when the transmit */
28 /* or receive has been completed, the VNIC driver is required to use */
29 /* "long term mapping". This entails that large, continuous DMA mapped */
30 /* buffers are allocated on driver initialization and these buffers are */
31 /* then continuously reused to pass skbs to and from the VNIC server. */
33 /**************************************************************************/
35 #include <linux/module.h>
36 #include <linux/moduleparam.h>
37 #include <linux/types.h>
38 #include <linux/errno.h>
39 #include <linux/completion.h>
40 #include <linux/ioport.h>
41 #include <linux/dma-mapping.h>
42 #include <linux/kernel.h>
43 #include <linux/netdevice.h>
44 #include <linux/etherdevice.h>
45 #include <linux/skbuff.h>
46 #include <linux/init.h>
47 #include <linux/delay.h>
49 #include <linux/ethtool.h>
50 #include <linux/proc_fs.h>
51 #include <linux/if_arp.h>
54 #include <linux/ipv6.h>
55 #include <linux/irq.h>
56 #include <linux/kthread.h>
57 #include <linux/seq_file.h>
58 #include <linux/interrupt.h>
59 #include <net/net_namespace.h>
60 #include <asm/hvcall.h>
61 #include <linux/atomic.h>
63 #include <asm/iommu.h>
64 #include <linux/uaccess.h>
65 #include <asm/firmware.h>
66 #include <linux/workqueue.h>
67 #include <linux/if_vlan.h>
68 #include <linux/utsname.h>
72 static const char ibmvnic_driver_name
[] = "ibmvnic";
73 static const char ibmvnic_driver_string
[] = "IBM System i/p Virtual NIC Driver";
75 MODULE_AUTHOR("Santiago Leon");
76 MODULE_DESCRIPTION("IBM System i/p Virtual NIC Driver");
77 MODULE_LICENSE("GPL");
78 MODULE_VERSION(IBMVNIC_DRIVER_VERSION
);
80 static int ibmvnic_version
= IBMVNIC_INITIAL_VERSION
;
81 static void release_sub_crqs(struct ibmvnic_adapter
*, bool);
82 static int ibmvnic_reset_crq(struct ibmvnic_adapter
*);
83 static int ibmvnic_send_crq_init(struct ibmvnic_adapter
*);
84 static int ibmvnic_reenable_crq_queue(struct ibmvnic_adapter
*);
85 static int ibmvnic_send_crq(struct ibmvnic_adapter
*, union ibmvnic_crq
*);
86 static int send_subcrq_indirect(struct ibmvnic_adapter
*, u64
, u64
, u64
);
87 static irqreturn_t
ibmvnic_interrupt_rx(int irq
, void *instance
);
88 static int enable_scrq_irq(struct ibmvnic_adapter
*,
89 struct ibmvnic_sub_crq_queue
*);
90 static int disable_scrq_irq(struct ibmvnic_adapter
*,
91 struct ibmvnic_sub_crq_queue
*);
92 static int pending_scrq(struct ibmvnic_adapter
*,
93 struct ibmvnic_sub_crq_queue
*);
94 static union sub_crq
*ibmvnic_next_scrq(struct ibmvnic_adapter
*,
95 struct ibmvnic_sub_crq_queue
*);
96 static int ibmvnic_poll(struct napi_struct
*napi
, int data
);
97 static void send_query_map(struct ibmvnic_adapter
*adapter
);
98 static int send_request_map(struct ibmvnic_adapter
*, dma_addr_t
, u32
, u8
);
99 static int send_request_unmap(struct ibmvnic_adapter
*, u8
);
100 static int send_login(struct ibmvnic_adapter
*adapter
);
101 static void send_query_cap(struct ibmvnic_adapter
*adapter
);
102 static int init_sub_crqs(struct ibmvnic_adapter
*);
103 static int init_sub_crq_irqs(struct ibmvnic_adapter
*adapter
);
104 static int ibmvnic_reset_init(struct ibmvnic_adapter
*, bool reset
);
105 static void release_crq_queue(struct ibmvnic_adapter
*);
106 static int __ibmvnic_set_mac(struct net_device
*, u8
*);
107 static int init_crq_queue(struct ibmvnic_adapter
*adapter
);
108 static int send_query_phys_parms(struct ibmvnic_adapter
*adapter
);
109 static void ibmvnic_tx_scrq_clean_buffer(struct ibmvnic_adapter
*adapter
,
110 struct ibmvnic_sub_crq_queue
*tx_scrq
);
112 struct ibmvnic_stat
{
113 char name
[ETH_GSTRING_LEN
];
117 #define IBMVNIC_STAT_OFF(stat) (offsetof(struct ibmvnic_adapter, stats) + \
118 offsetof(struct ibmvnic_statistics, stat))
119 #define IBMVNIC_GET_STAT(a, off) (*((u64 *)(((unsigned long)(a)) + (off))))
121 static const struct ibmvnic_stat ibmvnic_stats
[] = {
122 {"rx_packets", IBMVNIC_STAT_OFF(rx_packets
)},
123 {"rx_bytes", IBMVNIC_STAT_OFF(rx_bytes
)},
124 {"tx_packets", IBMVNIC_STAT_OFF(tx_packets
)},
125 {"tx_bytes", IBMVNIC_STAT_OFF(tx_bytes
)},
126 {"ucast_tx_packets", IBMVNIC_STAT_OFF(ucast_tx_packets
)},
127 {"ucast_rx_packets", IBMVNIC_STAT_OFF(ucast_rx_packets
)},
128 {"mcast_tx_packets", IBMVNIC_STAT_OFF(mcast_tx_packets
)},
129 {"mcast_rx_packets", IBMVNIC_STAT_OFF(mcast_rx_packets
)},
130 {"bcast_tx_packets", IBMVNIC_STAT_OFF(bcast_tx_packets
)},
131 {"bcast_rx_packets", IBMVNIC_STAT_OFF(bcast_rx_packets
)},
132 {"align_errors", IBMVNIC_STAT_OFF(align_errors
)},
133 {"fcs_errors", IBMVNIC_STAT_OFF(fcs_errors
)},
134 {"single_collision_frames", IBMVNIC_STAT_OFF(single_collision_frames
)},
135 {"multi_collision_frames", IBMVNIC_STAT_OFF(multi_collision_frames
)},
136 {"sqe_test_errors", IBMVNIC_STAT_OFF(sqe_test_errors
)},
137 {"deferred_tx", IBMVNIC_STAT_OFF(deferred_tx
)},
138 {"late_collisions", IBMVNIC_STAT_OFF(late_collisions
)},
139 {"excess_collisions", IBMVNIC_STAT_OFF(excess_collisions
)},
140 {"internal_mac_tx_errors", IBMVNIC_STAT_OFF(internal_mac_tx_errors
)},
141 {"carrier_sense", IBMVNIC_STAT_OFF(carrier_sense
)},
142 {"too_long_frames", IBMVNIC_STAT_OFF(too_long_frames
)},
143 {"internal_mac_rx_errors", IBMVNIC_STAT_OFF(internal_mac_rx_errors
)},
146 static int send_crq_init_complete(struct ibmvnic_adapter
*adapter
)
148 union ibmvnic_crq crq
;
150 memset(&crq
, 0, sizeof(crq
));
151 crq
.generic
.first
= IBMVNIC_CRQ_INIT_CMD
;
152 crq
.generic
.cmd
= IBMVNIC_CRQ_INIT_COMPLETE
;
154 return ibmvnic_send_crq(adapter
, &crq
);
157 static int send_version_xchg(struct ibmvnic_adapter
*adapter
)
159 union ibmvnic_crq crq
;
161 memset(&crq
, 0, sizeof(crq
));
162 crq
.version_exchange
.first
= IBMVNIC_CRQ_CMD
;
163 crq
.version_exchange
.cmd
= VERSION_EXCHANGE
;
164 crq
.version_exchange
.version
= cpu_to_be16(ibmvnic_version
);
166 return ibmvnic_send_crq(adapter
, &crq
);
169 static long h_reg_sub_crq(unsigned long unit_address
, unsigned long token
,
170 unsigned long length
, unsigned long *number
,
173 unsigned long retbuf
[PLPAR_HCALL_BUFSIZE
];
176 rc
= plpar_hcall(H_REG_SUB_CRQ
, retbuf
, unit_address
, token
, length
);
184 * ibmvnic_wait_for_completion - Check device state and wait for completion
185 * @adapter: private device data
186 * @comp_done: completion structure to wait for
187 * @timeout: time to wait in milliseconds
189 * Wait for a completion signal or until the timeout limit is reached
190 * while checking that the device is still active.
192 static int ibmvnic_wait_for_completion(struct ibmvnic_adapter
*adapter
,
193 struct completion
*comp_done
,
194 unsigned long timeout
)
196 struct net_device
*netdev
;
197 unsigned long div_timeout
;
200 netdev
= adapter
->netdev
;
202 div_timeout
= msecs_to_jiffies(timeout
/ retry
);
204 if (!adapter
->crq
.active
) {
205 netdev_err(netdev
, "Device down!\n");
210 if (wait_for_completion_timeout(comp_done
, div_timeout
))
213 netdev_err(netdev
, "Operation timed out.\n");
217 static int alloc_long_term_buff(struct ibmvnic_adapter
*adapter
,
218 struct ibmvnic_long_term_buff
*ltb
, int size
)
220 struct device
*dev
= &adapter
->vdev
->dev
;
224 ltb
->buff
= dma_alloc_coherent(dev
, ltb
->size
, <b
->addr
,
228 dev_err(dev
, "Couldn't alloc long term buffer\n");
231 ltb
->map_id
= adapter
->map_id
;
234 mutex_lock(&adapter
->fw_lock
);
235 adapter
->fw_done_rc
= 0;
236 reinit_completion(&adapter
->fw_done
);
238 rc
= send_request_map(adapter
, ltb
->addr
, ltb
->size
, ltb
->map_id
);
240 dev_err(dev
, "send_request_map failed, rc = %d\n", rc
);
244 rc
= ibmvnic_wait_for_completion(adapter
, &adapter
->fw_done
, 10000);
247 "Long term map request aborted or timed out,rc = %d\n",
252 if (adapter
->fw_done_rc
) {
253 dev_err(dev
, "Couldn't map long term buffer,rc = %d\n",
254 adapter
->fw_done_rc
);
261 dma_free_coherent(dev
, ltb
->size
, ltb
->buff
, ltb
->addr
);
264 mutex_unlock(&adapter
->fw_lock
);
268 static void free_long_term_buff(struct ibmvnic_adapter
*adapter
,
269 struct ibmvnic_long_term_buff
*ltb
)
271 struct device
*dev
= &adapter
->vdev
->dev
;
276 /* VIOS automatically unmaps the long term buffer at remote
277 * end for the following resets:
278 * FAILOVER, MOBILITY, TIMEOUT.
280 if (adapter
->reset_reason
!= VNIC_RESET_FAILOVER
&&
281 adapter
->reset_reason
!= VNIC_RESET_MOBILITY
&&
282 adapter
->reset_reason
!= VNIC_RESET_TIMEOUT
)
283 send_request_unmap(adapter
, ltb
->map_id
);
284 dma_free_coherent(dev
, ltb
->size
, ltb
->buff
, ltb
->addr
);
289 static int reset_long_term_buff(struct ibmvnic_adapter
*adapter
,
290 struct ibmvnic_long_term_buff
*ltb
)
292 struct device
*dev
= &adapter
->vdev
->dev
;
295 memset(ltb
->buff
, 0, ltb
->size
);
297 mutex_lock(&adapter
->fw_lock
);
298 adapter
->fw_done_rc
= 0;
300 reinit_completion(&adapter
->fw_done
);
301 rc
= send_request_map(adapter
, ltb
->addr
, ltb
->size
, ltb
->map_id
);
303 mutex_unlock(&adapter
->fw_lock
);
307 rc
= ibmvnic_wait_for_completion(adapter
, &adapter
->fw_done
, 10000);
310 "Reset failed, long term map request timed out or aborted\n");
311 mutex_unlock(&adapter
->fw_lock
);
315 if (adapter
->fw_done_rc
) {
317 "Reset failed, attempting to free and reallocate buffer\n");
318 free_long_term_buff(adapter
, ltb
);
319 mutex_unlock(&adapter
->fw_lock
);
320 return alloc_long_term_buff(adapter
, ltb
, ltb
->size
);
322 mutex_unlock(&adapter
->fw_lock
);
326 static void deactivate_rx_pools(struct ibmvnic_adapter
*adapter
)
330 for (i
= 0; i
< adapter
->num_active_rx_pools
; i
++)
331 adapter
->rx_pool
[i
].active
= 0;
334 static void replenish_rx_pool(struct ibmvnic_adapter
*adapter
,
335 struct ibmvnic_rx_pool
*pool
)
337 int count
= pool
->size
- atomic_read(&pool
->available
);
338 u64 handle
= adapter
->rx_scrq
[pool
->index
]->handle
;
339 struct device
*dev
= &adapter
->vdev
->dev
;
340 struct ibmvnic_ind_xmit_queue
*ind_bufp
;
341 struct ibmvnic_sub_crq_queue
*rx_scrq
;
342 union sub_crq
*sub_crq
;
343 int buffers_added
= 0;
344 unsigned long lpar_rc
;
356 rx_scrq
= adapter
->rx_scrq
[pool
->index
];
357 ind_bufp
= &rx_scrq
->ind_buf
;
359 /* netdev_skb_alloc() could have failed after we saved a few skbs
360 * in the indir_buf and we would not have sent them to VIOS yet.
361 * To account for them, start the loop at ind_bufp->index rather
362 * than 0. If we pushed all the skbs to VIOS, ind_bufp->index will
365 for (i
= ind_bufp
->index
; i
< count
; ++i
) {
366 skb
= netdev_alloc_skb(adapter
->netdev
, pool
->buff_size
);
368 dev_err(dev
, "Couldn't replenish rx buff\n");
369 adapter
->replenish_no_mem
++;
373 index
= pool
->free_map
[pool
->next_free
];
375 if (pool
->rx_buff
[index
].skb
)
376 dev_err(dev
, "Inconsistent free_map!\n");
378 /* Copy the skb to the long term mapped DMA buffer */
379 offset
= index
* pool
->buff_size
;
380 dst
= pool
->long_term_buff
.buff
+ offset
;
381 memset(dst
, 0, pool
->buff_size
);
382 dma_addr
= pool
->long_term_buff
.addr
+ offset
;
383 pool
->rx_buff
[index
].data
= dst
;
385 pool
->free_map
[pool
->next_free
] = IBMVNIC_INVALID_MAP
;
386 pool
->rx_buff
[index
].dma
= dma_addr
;
387 pool
->rx_buff
[index
].skb
= skb
;
388 pool
->rx_buff
[index
].pool_index
= pool
->index
;
389 pool
->rx_buff
[index
].size
= pool
->buff_size
;
391 sub_crq
= &ind_bufp
->indir_arr
[ind_bufp
->index
++];
392 memset(sub_crq
, 0, sizeof(*sub_crq
));
393 sub_crq
->rx_add
.first
= IBMVNIC_CRQ_CMD
;
394 sub_crq
->rx_add
.correlator
=
395 cpu_to_be64((u64
)&pool
->rx_buff
[index
]);
396 sub_crq
->rx_add
.ioba
= cpu_to_be32(dma_addr
);
397 sub_crq
->rx_add
.map_id
= pool
->long_term_buff
.map_id
;
399 /* The length field of the sCRQ is defined to be 24 bits so the
400 * buffer size needs to be left shifted by a byte before it is
401 * converted to big endian to prevent the last byte from being
404 #ifdef __LITTLE_ENDIAN__
407 sub_crq
->rx_add
.len
= cpu_to_be32(pool
->buff_size
<< shift
);
408 pool
->next_free
= (pool
->next_free
+ 1) % pool
->size
;
409 if (ind_bufp
->index
== IBMVNIC_MAX_IND_DESCS
||
412 send_subcrq_indirect(adapter
, handle
,
413 (u64
)ind_bufp
->indir_dma
,
414 (u64
)ind_bufp
->index
);
415 if (lpar_rc
!= H_SUCCESS
)
417 buffers_added
+= ind_bufp
->index
;
418 adapter
->replenish_add_buff_success
+= ind_bufp
->index
;
422 atomic_add(buffers_added
, &pool
->available
);
426 if (lpar_rc
!= H_PARAMETER
&& lpar_rc
!= H_CLOSED
)
427 dev_err_ratelimited(dev
, "rx: replenish packet buffer failed\n");
428 for (i
= ind_bufp
->index
- 1; i
>= 0; --i
) {
429 struct ibmvnic_rx_buff
*rx_buff
;
431 pool
->next_free
= pool
->next_free
== 0 ?
432 pool
->size
- 1 : pool
->next_free
- 1;
433 sub_crq
= &ind_bufp
->indir_arr
[i
];
434 rx_buff
= (struct ibmvnic_rx_buff
*)
435 be64_to_cpu(sub_crq
->rx_add
.correlator
);
436 index
= (int)(rx_buff
- pool
->rx_buff
);
437 pool
->free_map
[pool
->next_free
] = index
;
438 dev_kfree_skb_any(pool
->rx_buff
[index
].skb
);
439 pool
->rx_buff
[index
].skb
= NULL
;
441 adapter
->replenish_add_buff_failure
+= ind_bufp
->index
;
442 atomic_add(buffers_added
, &pool
->available
);
444 if (lpar_rc
== H_CLOSED
|| adapter
->failover_pending
) {
445 /* Disable buffer pool replenishment and report carrier off if
446 * queue is closed or pending failover.
447 * Firmware guarantees that a signal will be sent to the
448 * driver, triggering a reset.
450 deactivate_rx_pools(adapter
);
451 netif_carrier_off(adapter
->netdev
);
455 static void replenish_pools(struct ibmvnic_adapter
*adapter
)
459 adapter
->replenish_task_cycles
++;
460 for (i
= 0; i
< adapter
->num_active_rx_pools
; i
++) {
461 if (adapter
->rx_pool
[i
].active
)
462 replenish_rx_pool(adapter
, &adapter
->rx_pool
[i
]);
465 netdev_dbg(adapter
->netdev
, "Replenished %d pools\n", i
);
468 static void release_stats_buffers(struct ibmvnic_adapter
*adapter
)
470 kfree(adapter
->tx_stats_buffers
);
471 kfree(adapter
->rx_stats_buffers
);
472 adapter
->tx_stats_buffers
= NULL
;
473 adapter
->rx_stats_buffers
= NULL
;
476 static int init_stats_buffers(struct ibmvnic_adapter
*adapter
)
478 adapter
->tx_stats_buffers
=
479 kcalloc(IBMVNIC_MAX_QUEUES
,
480 sizeof(struct ibmvnic_tx_queue_stats
),
482 if (!adapter
->tx_stats_buffers
)
485 adapter
->rx_stats_buffers
=
486 kcalloc(IBMVNIC_MAX_QUEUES
,
487 sizeof(struct ibmvnic_rx_queue_stats
),
489 if (!adapter
->rx_stats_buffers
)
495 static void release_stats_token(struct ibmvnic_adapter
*adapter
)
497 struct device
*dev
= &adapter
->vdev
->dev
;
499 if (!adapter
->stats_token
)
502 dma_unmap_single(dev
, adapter
->stats_token
,
503 sizeof(struct ibmvnic_statistics
),
505 adapter
->stats_token
= 0;
508 static int init_stats_token(struct ibmvnic_adapter
*adapter
)
510 struct device
*dev
= &adapter
->vdev
->dev
;
513 stok
= dma_map_single(dev
, &adapter
->stats
,
514 sizeof(struct ibmvnic_statistics
),
516 if (dma_mapping_error(dev
, stok
)) {
517 dev_err(dev
, "Couldn't map stats buffer\n");
521 adapter
->stats_token
= stok
;
522 netdev_dbg(adapter
->netdev
, "Stats token initialized (%llx)\n", stok
);
526 static int reset_rx_pools(struct ibmvnic_adapter
*adapter
)
528 struct ibmvnic_rx_pool
*rx_pool
;
533 if (!adapter
->rx_pool
)
536 buff_size
= adapter
->cur_rx_buf_sz
;
537 rx_scrqs
= adapter
->num_active_rx_pools
;
538 for (i
= 0; i
< rx_scrqs
; i
++) {
539 rx_pool
= &adapter
->rx_pool
[i
];
541 netdev_dbg(adapter
->netdev
, "Re-setting rx_pool[%d]\n", i
);
543 if (rx_pool
->buff_size
!= buff_size
) {
544 free_long_term_buff(adapter
, &rx_pool
->long_term_buff
);
545 rx_pool
->buff_size
= ALIGN(buff_size
, L1_CACHE_BYTES
);
546 rc
= alloc_long_term_buff(adapter
,
547 &rx_pool
->long_term_buff
,
551 rc
= reset_long_term_buff(adapter
,
552 &rx_pool
->long_term_buff
);
558 for (j
= 0; j
< rx_pool
->size
; j
++)
559 rx_pool
->free_map
[j
] = j
;
561 memset(rx_pool
->rx_buff
, 0,
562 rx_pool
->size
* sizeof(struct ibmvnic_rx_buff
));
564 atomic_set(&rx_pool
->available
, 0);
565 rx_pool
->next_alloc
= 0;
566 rx_pool
->next_free
= 0;
573 static void release_rx_pools(struct ibmvnic_adapter
*adapter
)
575 struct ibmvnic_rx_pool
*rx_pool
;
578 if (!adapter
->rx_pool
)
581 for (i
= 0; i
< adapter
->num_active_rx_pools
; i
++) {
582 rx_pool
= &adapter
->rx_pool
[i
];
584 netdev_dbg(adapter
->netdev
, "Releasing rx_pool[%d]\n", i
);
586 kfree(rx_pool
->free_map
);
587 free_long_term_buff(adapter
, &rx_pool
->long_term_buff
);
589 if (!rx_pool
->rx_buff
)
592 for (j
= 0; j
< rx_pool
->size
; j
++) {
593 if (rx_pool
->rx_buff
[j
].skb
) {
594 dev_kfree_skb_any(rx_pool
->rx_buff
[j
].skb
);
595 rx_pool
->rx_buff
[j
].skb
= NULL
;
599 kfree(rx_pool
->rx_buff
);
602 kfree(adapter
->rx_pool
);
603 adapter
->rx_pool
= NULL
;
604 adapter
->num_active_rx_pools
= 0;
607 static int init_rx_pools(struct net_device
*netdev
)
609 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
610 struct device
*dev
= &adapter
->vdev
->dev
;
611 struct ibmvnic_rx_pool
*rx_pool
;
616 rxadd_subcrqs
= adapter
->num_active_rx_scrqs
;
617 buff_size
= adapter
->cur_rx_buf_sz
;
619 adapter
->rx_pool
= kcalloc(rxadd_subcrqs
,
620 sizeof(struct ibmvnic_rx_pool
),
622 if (!adapter
->rx_pool
) {
623 dev_err(dev
, "Failed to allocate rx pools\n");
627 adapter
->num_active_rx_pools
= rxadd_subcrqs
;
629 for (i
= 0; i
< rxadd_subcrqs
; i
++) {
630 rx_pool
= &adapter
->rx_pool
[i
];
632 netdev_dbg(adapter
->netdev
,
633 "Initializing rx_pool[%d], %lld buffs, %lld bytes each\n",
634 i
, adapter
->req_rx_add_entries_per_subcrq
,
637 rx_pool
->size
= adapter
->req_rx_add_entries_per_subcrq
;
639 rx_pool
->buff_size
= ALIGN(buff_size
, L1_CACHE_BYTES
);
642 rx_pool
->free_map
= kcalloc(rx_pool
->size
, sizeof(int),
644 if (!rx_pool
->free_map
) {
645 release_rx_pools(adapter
);
649 rx_pool
->rx_buff
= kcalloc(rx_pool
->size
,
650 sizeof(struct ibmvnic_rx_buff
),
652 if (!rx_pool
->rx_buff
) {
653 dev_err(dev
, "Couldn't alloc rx buffers\n");
654 release_rx_pools(adapter
);
658 if (alloc_long_term_buff(adapter
, &rx_pool
->long_term_buff
,
659 rx_pool
->size
* rx_pool
->buff_size
)) {
660 release_rx_pools(adapter
);
664 for (j
= 0; j
< rx_pool
->size
; ++j
)
665 rx_pool
->free_map
[j
] = j
;
667 atomic_set(&rx_pool
->available
, 0);
668 rx_pool
->next_alloc
= 0;
669 rx_pool
->next_free
= 0;
675 static int reset_one_tx_pool(struct ibmvnic_adapter
*adapter
,
676 struct ibmvnic_tx_pool
*tx_pool
)
680 rc
= reset_long_term_buff(adapter
, &tx_pool
->long_term_buff
);
684 memset(tx_pool
->tx_buff
, 0,
685 tx_pool
->num_buffers
*
686 sizeof(struct ibmvnic_tx_buff
));
688 for (i
= 0; i
< tx_pool
->num_buffers
; i
++)
689 tx_pool
->free_map
[i
] = i
;
691 tx_pool
->consumer_index
= 0;
692 tx_pool
->producer_index
= 0;
697 static int reset_tx_pools(struct ibmvnic_adapter
*adapter
)
702 if (!adapter
->tx_pool
)
705 tx_scrqs
= adapter
->num_active_tx_pools
;
706 for (i
= 0; i
< tx_scrqs
; i
++) {
707 ibmvnic_tx_scrq_clean_buffer(adapter
, adapter
->tx_scrq
[i
]);
708 rc
= reset_one_tx_pool(adapter
, &adapter
->tso_pool
[i
]);
711 rc
= reset_one_tx_pool(adapter
, &adapter
->tx_pool
[i
]);
719 static void release_vpd_data(struct ibmvnic_adapter
*adapter
)
724 kfree(adapter
->vpd
->buff
);
730 static void release_one_tx_pool(struct ibmvnic_adapter
*adapter
,
731 struct ibmvnic_tx_pool
*tx_pool
)
733 kfree(tx_pool
->tx_buff
);
734 kfree(tx_pool
->free_map
);
735 free_long_term_buff(adapter
, &tx_pool
->long_term_buff
);
738 static void release_tx_pools(struct ibmvnic_adapter
*adapter
)
742 if (!adapter
->tx_pool
)
745 for (i
= 0; i
< adapter
->num_active_tx_pools
; i
++) {
746 release_one_tx_pool(adapter
, &adapter
->tx_pool
[i
]);
747 release_one_tx_pool(adapter
, &adapter
->tso_pool
[i
]);
750 kfree(adapter
->tx_pool
);
751 adapter
->tx_pool
= NULL
;
752 kfree(adapter
->tso_pool
);
753 adapter
->tso_pool
= NULL
;
754 adapter
->num_active_tx_pools
= 0;
757 static int init_one_tx_pool(struct net_device
*netdev
,
758 struct ibmvnic_tx_pool
*tx_pool
,
759 int num_entries
, int buf_size
)
761 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
764 tx_pool
->tx_buff
= kcalloc(num_entries
,
765 sizeof(struct ibmvnic_tx_buff
),
767 if (!tx_pool
->tx_buff
)
770 if (alloc_long_term_buff(adapter
, &tx_pool
->long_term_buff
,
771 num_entries
* buf_size
))
774 tx_pool
->free_map
= kcalloc(num_entries
, sizeof(int), GFP_KERNEL
);
775 if (!tx_pool
->free_map
)
778 for (i
= 0; i
< num_entries
; i
++)
779 tx_pool
->free_map
[i
] = i
;
781 tx_pool
->consumer_index
= 0;
782 tx_pool
->producer_index
= 0;
783 tx_pool
->num_buffers
= num_entries
;
784 tx_pool
->buf_size
= buf_size
;
789 static int init_tx_pools(struct net_device
*netdev
)
791 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
796 tx_subcrqs
= adapter
->num_active_tx_scrqs
;
797 adapter
->tx_pool
= kcalloc(tx_subcrqs
,
798 sizeof(struct ibmvnic_tx_pool
), GFP_KERNEL
);
799 if (!adapter
->tx_pool
)
802 adapter
->tso_pool
= kcalloc(tx_subcrqs
,
803 sizeof(struct ibmvnic_tx_pool
), GFP_KERNEL
);
804 if (!adapter
->tso_pool
) {
805 kfree(adapter
->tx_pool
);
806 adapter
->tx_pool
= NULL
;
810 adapter
->num_active_tx_pools
= tx_subcrqs
;
812 for (i
= 0; i
< tx_subcrqs
; i
++) {
813 buff_size
= adapter
->req_mtu
+ VLAN_HLEN
;
814 buff_size
= ALIGN(buff_size
, L1_CACHE_BYTES
);
815 rc
= init_one_tx_pool(netdev
, &adapter
->tx_pool
[i
],
816 adapter
->req_tx_entries_per_subcrq
,
819 release_tx_pools(adapter
);
823 rc
= init_one_tx_pool(netdev
, &adapter
->tso_pool
[i
],
827 release_tx_pools(adapter
);
835 static void ibmvnic_napi_enable(struct ibmvnic_adapter
*adapter
)
839 if (adapter
->napi_enabled
)
842 for (i
= 0; i
< adapter
->req_rx_queues
; i
++)
843 napi_enable(&adapter
->napi
[i
]);
845 adapter
->napi_enabled
= true;
848 static void ibmvnic_napi_disable(struct ibmvnic_adapter
*adapter
)
852 if (!adapter
->napi_enabled
)
855 for (i
= 0; i
< adapter
->req_rx_queues
; i
++) {
856 netdev_dbg(adapter
->netdev
, "Disabling napi[%d]\n", i
);
857 napi_disable(&adapter
->napi
[i
]);
860 adapter
->napi_enabled
= false;
863 static int init_napi(struct ibmvnic_adapter
*adapter
)
867 adapter
->napi
= kcalloc(adapter
->req_rx_queues
,
868 sizeof(struct napi_struct
), GFP_KERNEL
);
872 for (i
= 0; i
< adapter
->req_rx_queues
; i
++) {
873 netdev_dbg(adapter
->netdev
, "Adding napi[%d]\n", i
);
874 netif_napi_add(adapter
->netdev
, &adapter
->napi
[i
],
875 ibmvnic_poll
, NAPI_POLL_WEIGHT
);
878 adapter
->num_active_rx_napi
= adapter
->req_rx_queues
;
882 static void release_napi(struct ibmvnic_adapter
*adapter
)
889 for (i
= 0; i
< adapter
->num_active_rx_napi
; i
++) {
890 netdev_dbg(adapter
->netdev
, "Releasing napi[%d]\n", i
);
891 netif_napi_del(&adapter
->napi
[i
]);
894 kfree(adapter
->napi
);
895 adapter
->napi
= NULL
;
896 adapter
->num_active_rx_napi
= 0;
897 adapter
->napi_enabled
= false;
900 static const char *adapter_state_to_string(enum vnic_state state
)
925 static int ibmvnic_login(struct net_device
*netdev
)
927 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
928 unsigned long timeout
= msecs_to_jiffies(20000);
936 if (retry_count
> retries
) {
937 netdev_warn(netdev
, "Login attempts exceeded\n");
941 adapter
->init_done_rc
= 0;
942 reinit_completion(&adapter
->init_done
);
943 rc
= send_login(adapter
);
947 if (!wait_for_completion_timeout(&adapter
->init_done
,
949 netdev_warn(netdev
, "Login timed out, retrying...\n");
951 adapter
->init_done_rc
= 0;
956 if (adapter
->init_done_rc
== ABORTED
) {
957 netdev_warn(netdev
, "Login aborted, retrying...\n");
959 adapter
->init_done_rc
= 0;
961 /* FW or device may be busy, so
962 * wait a bit before retrying login
965 } else if (adapter
->init_done_rc
== PARTIALSUCCESS
) {
967 release_sub_crqs(adapter
, 1);
971 "Received partial success, retrying...\n");
972 adapter
->init_done_rc
= 0;
973 reinit_completion(&adapter
->init_done
);
974 send_query_cap(adapter
);
975 if (!wait_for_completion_timeout(&adapter
->init_done
,
978 "Capabilities query timed out\n");
982 rc
= init_sub_crqs(adapter
);
985 "SCRQ initialization failed\n");
989 rc
= init_sub_crq_irqs(adapter
);
992 "SCRQ irq initialization failed\n");
995 } else if (adapter
->init_done_rc
) {
996 netdev_warn(netdev
, "Adapter login failed\n");
1001 __ibmvnic_set_mac(netdev
, adapter
->mac_addr
);
1003 netdev_dbg(netdev
, "[S:%s] Login succeeded\n", adapter_state_to_string(adapter
->state
));
1007 static void release_login_buffer(struct ibmvnic_adapter
*adapter
)
1009 kfree(adapter
->login_buf
);
1010 adapter
->login_buf
= NULL
;
1013 static void release_login_rsp_buffer(struct ibmvnic_adapter
*adapter
)
1015 kfree(adapter
->login_rsp_buf
);
1016 adapter
->login_rsp_buf
= NULL
;
1019 static void release_resources(struct ibmvnic_adapter
*adapter
)
1021 release_vpd_data(adapter
);
1023 release_tx_pools(adapter
);
1024 release_rx_pools(adapter
);
1026 release_napi(adapter
);
1027 release_login_buffer(adapter
);
1028 release_login_rsp_buffer(adapter
);
1031 static int set_link_state(struct ibmvnic_adapter
*adapter
, u8 link_state
)
1033 struct net_device
*netdev
= adapter
->netdev
;
1034 unsigned long timeout
= msecs_to_jiffies(20000);
1035 union ibmvnic_crq crq
;
1039 netdev_dbg(netdev
, "setting link state %d\n", link_state
);
1041 memset(&crq
, 0, sizeof(crq
));
1042 crq
.logical_link_state
.first
= IBMVNIC_CRQ_CMD
;
1043 crq
.logical_link_state
.cmd
= LOGICAL_LINK_STATE
;
1044 crq
.logical_link_state
.link_state
= link_state
;
1049 reinit_completion(&adapter
->init_done
);
1050 rc
= ibmvnic_send_crq(adapter
, &crq
);
1052 netdev_err(netdev
, "Failed to set link state\n");
1056 if (!wait_for_completion_timeout(&adapter
->init_done
,
1058 netdev_err(netdev
, "timeout setting link state\n");
1062 if (adapter
->init_done_rc
== PARTIALSUCCESS
) {
1063 /* Partuial success, delay and re-send */
1066 } else if (adapter
->init_done_rc
) {
1067 netdev_warn(netdev
, "Unable to set link state, rc=%d\n",
1068 adapter
->init_done_rc
);
1069 return adapter
->init_done_rc
;
1076 static int set_real_num_queues(struct net_device
*netdev
)
1078 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
1081 netdev_dbg(netdev
, "Setting real tx/rx queues (%llx/%llx)\n",
1082 adapter
->req_tx_queues
, adapter
->req_rx_queues
);
1084 rc
= netif_set_real_num_tx_queues(netdev
, adapter
->req_tx_queues
);
1086 netdev_err(netdev
, "failed to set the number of tx queues\n");
1090 rc
= netif_set_real_num_rx_queues(netdev
, adapter
->req_rx_queues
);
1092 netdev_err(netdev
, "failed to set the number of rx queues\n");
1097 static int ibmvnic_get_vpd(struct ibmvnic_adapter
*adapter
)
1099 struct device
*dev
= &adapter
->vdev
->dev
;
1100 union ibmvnic_crq crq
;
1104 if (adapter
->vpd
->buff
)
1105 len
= adapter
->vpd
->len
;
1107 mutex_lock(&adapter
->fw_lock
);
1108 adapter
->fw_done_rc
= 0;
1109 reinit_completion(&adapter
->fw_done
);
1111 crq
.get_vpd_size
.first
= IBMVNIC_CRQ_CMD
;
1112 crq
.get_vpd_size
.cmd
= GET_VPD_SIZE
;
1113 rc
= ibmvnic_send_crq(adapter
, &crq
);
1115 mutex_unlock(&adapter
->fw_lock
);
1119 rc
= ibmvnic_wait_for_completion(adapter
, &adapter
->fw_done
, 10000);
1121 dev_err(dev
, "Could not retrieve VPD size, rc = %d\n", rc
);
1122 mutex_unlock(&adapter
->fw_lock
);
1125 mutex_unlock(&adapter
->fw_lock
);
1127 if (!adapter
->vpd
->len
)
1130 if (!adapter
->vpd
->buff
)
1131 adapter
->vpd
->buff
= kzalloc(adapter
->vpd
->len
, GFP_KERNEL
);
1132 else if (adapter
->vpd
->len
!= len
)
1133 adapter
->vpd
->buff
=
1134 krealloc(adapter
->vpd
->buff
,
1135 adapter
->vpd
->len
, GFP_KERNEL
);
1137 if (!adapter
->vpd
->buff
) {
1138 dev_err(dev
, "Could allocate VPD buffer\n");
1142 adapter
->vpd
->dma_addr
=
1143 dma_map_single(dev
, adapter
->vpd
->buff
, adapter
->vpd
->len
,
1145 if (dma_mapping_error(dev
, adapter
->vpd
->dma_addr
)) {
1146 dev_err(dev
, "Could not map VPD buffer\n");
1147 kfree(adapter
->vpd
->buff
);
1148 adapter
->vpd
->buff
= NULL
;
1152 mutex_lock(&adapter
->fw_lock
);
1153 adapter
->fw_done_rc
= 0;
1154 reinit_completion(&adapter
->fw_done
);
1156 crq
.get_vpd
.first
= IBMVNIC_CRQ_CMD
;
1157 crq
.get_vpd
.cmd
= GET_VPD
;
1158 crq
.get_vpd
.ioba
= cpu_to_be32(adapter
->vpd
->dma_addr
);
1159 crq
.get_vpd
.len
= cpu_to_be32((u32
)adapter
->vpd
->len
);
1160 rc
= ibmvnic_send_crq(adapter
, &crq
);
1162 kfree(adapter
->vpd
->buff
);
1163 adapter
->vpd
->buff
= NULL
;
1164 mutex_unlock(&adapter
->fw_lock
);
1168 rc
= ibmvnic_wait_for_completion(adapter
, &adapter
->fw_done
, 10000);
1170 dev_err(dev
, "Unable to retrieve VPD, rc = %d\n", rc
);
1171 kfree(adapter
->vpd
->buff
);
1172 adapter
->vpd
->buff
= NULL
;
1173 mutex_unlock(&adapter
->fw_lock
);
1177 mutex_unlock(&adapter
->fw_lock
);
1181 static int init_resources(struct ibmvnic_adapter
*adapter
)
1183 struct net_device
*netdev
= adapter
->netdev
;
1186 rc
= set_real_num_queues(netdev
);
1190 adapter
->vpd
= kzalloc(sizeof(*adapter
->vpd
), GFP_KERNEL
);
1194 /* Vital Product Data (VPD) */
1195 rc
= ibmvnic_get_vpd(adapter
);
1197 netdev_err(netdev
, "failed to initialize Vital Product Data (VPD)\n");
1201 adapter
->map_id
= 1;
1203 rc
= init_napi(adapter
);
1207 send_query_map(adapter
);
1209 rc
= init_rx_pools(netdev
);
1213 rc
= init_tx_pools(netdev
);
1217 static int __ibmvnic_open(struct net_device
*netdev
)
1219 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
1220 enum vnic_state prev_state
= adapter
->state
;
1223 adapter
->state
= VNIC_OPENING
;
1224 replenish_pools(adapter
);
1225 ibmvnic_napi_enable(adapter
);
1227 /* We're ready to receive frames, enable the sub-crq interrupts and
1228 * set the logical link state to up
1230 for (i
= 0; i
< adapter
->req_rx_queues
; i
++) {
1231 netdev_dbg(netdev
, "Enabling rx_scrq[%d] irq\n", i
);
1232 if (prev_state
== VNIC_CLOSED
)
1233 enable_irq(adapter
->rx_scrq
[i
]->irq
);
1234 enable_scrq_irq(adapter
, adapter
->rx_scrq
[i
]);
1237 for (i
= 0; i
< adapter
->req_tx_queues
; i
++) {
1238 netdev_dbg(netdev
, "Enabling tx_scrq[%d] irq\n", i
);
1239 if (prev_state
== VNIC_CLOSED
)
1240 enable_irq(adapter
->tx_scrq
[i
]->irq
);
1241 enable_scrq_irq(adapter
, adapter
->tx_scrq
[i
]);
1242 netdev_tx_reset_queue(netdev_get_tx_queue(netdev
, i
));
1245 rc
= set_link_state(adapter
, IBMVNIC_LOGICAL_LNK_UP
);
1247 ibmvnic_napi_disable(adapter
);
1248 release_resources(adapter
);
1252 netif_tx_start_all_queues(netdev
);
1254 if (prev_state
== VNIC_CLOSED
) {
1255 for (i
= 0; i
< adapter
->req_rx_queues
; i
++)
1256 napi_schedule(&adapter
->napi
[i
]);
1259 adapter
->state
= VNIC_OPEN
;
1263 static int ibmvnic_open(struct net_device
*netdev
)
1265 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
1270 /* If device failover is pending or we are about to reset, just set
1271 * device state and return. Device operation will be handled by reset
1274 * It should be safe to overwrite the adapter->state here. Since
1275 * we hold the rtnl, either the reset has not actually started or
1276 * the rtnl got dropped during the set_link_state() in do_reset().
1277 * In the former case, no one else is changing the state (again we
1278 * have the rtnl) and in the latter case, do_reset() will detect and
1279 * honor our setting below.
1281 if (adapter
->failover_pending
|| (test_bit(0, &adapter
->resetting
))) {
1282 netdev_dbg(netdev
, "[S:%s FOP:%d] Resetting, deferring open\n",
1283 adapter_state_to_string(adapter
->state
),
1284 adapter
->failover_pending
);
1285 adapter
->state
= VNIC_OPEN
;
1290 if (adapter
->state
!= VNIC_CLOSED
) {
1291 rc
= ibmvnic_login(netdev
);
1295 rc
= init_resources(adapter
);
1297 netdev_err(netdev
, "failed to initialize resources\n");
1298 release_resources(adapter
);
1303 rc
= __ibmvnic_open(netdev
);
1306 /* If open failed and there is a pending failover or in-progress reset,
1307 * set device state and return. Device operation will be handled by
1308 * reset routine. See also comments above regarding rtnl.
1311 (adapter
->failover_pending
|| (test_bit(0, &adapter
->resetting
)))) {
1312 adapter
->state
= VNIC_OPEN
;
1318 static void clean_rx_pools(struct ibmvnic_adapter
*adapter
)
1320 struct ibmvnic_rx_pool
*rx_pool
;
1321 struct ibmvnic_rx_buff
*rx_buff
;
1326 if (!adapter
->rx_pool
)
1329 rx_scrqs
= adapter
->num_active_rx_pools
;
1330 rx_entries
= adapter
->req_rx_add_entries_per_subcrq
;
1332 /* Free any remaining skbs in the rx buffer pools */
1333 for (i
= 0; i
< rx_scrqs
; i
++) {
1334 rx_pool
= &adapter
->rx_pool
[i
];
1335 if (!rx_pool
|| !rx_pool
->rx_buff
)
1338 netdev_dbg(adapter
->netdev
, "Cleaning rx_pool[%d]\n", i
);
1339 for (j
= 0; j
< rx_entries
; j
++) {
1340 rx_buff
= &rx_pool
->rx_buff
[j
];
1341 if (rx_buff
&& rx_buff
->skb
) {
1342 dev_kfree_skb_any(rx_buff
->skb
);
1343 rx_buff
->skb
= NULL
;
1349 static void clean_one_tx_pool(struct ibmvnic_adapter
*adapter
,
1350 struct ibmvnic_tx_pool
*tx_pool
)
1352 struct ibmvnic_tx_buff
*tx_buff
;
1356 if (!tx_pool
|| !tx_pool
->tx_buff
)
1359 tx_entries
= tx_pool
->num_buffers
;
1361 for (i
= 0; i
< tx_entries
; i
++) {
1362 tx_buff
= &tx_pool
->tx_buff
[i
];
1363 if (tx_buff
&& tx_buff
->skb
) {
1364 dev_kfree_skb_any(tx_buff
->skb
);
1365 tx_buff
->skb
= NULL
;
1370 static void clean_tx_pools(struct ibmvnic_adapter
*adapter
)
1375 if (!adapter
->tx_pool
|| !adapter
->tso_pool
)
1378 tx_scrqs
= adapter
->num_active_tx_pools
;
1380 /* Free any remaining skbs in the tx buffer pools */
1381 for (i
= 0; i
< tx_scrqs
; i
++) {
1382 netdev_dbg(adapter
->netdev
, "Cleaning tx_pool[%d]\n", i
);
1383 clean_one_tx_pool(adapter
, &adapter
->tx_pool
[i
]);
1384 clean_one_tx_pool(adapter
, &adapter
->tso_pool
[i
]);
1388 static void ibmvnic_disable_irqs(struct ibmvnic_adapter
*adapter
)
1390 struct net_device
*netdev
= adapter
->netdev
;
1393 if (adapter
->tx_scrq
) {
1394 for (i
= 0; i
< adapter
->req_tx_queues
; i
++)
1395 if (adapter
->tx_scrq
[i
]->irq
) {
1397 "Disabling tx_scrq[%d] irq\n", i
);
1398 disable_scrq_irq(adapter
, adapter
->tx_scrq
[i
]);
1399 disable_irq(adapter
->tx_scrq
[i
]->irq
);
1403 if (adapter
->rx_scrq
) {
1404 for (i
= 0; i
< adapter
->req_rx_queues
; i
++) {
1405 if (adapter
->rx_scrq
[i
]->irq
) {
1407 "Disabling rx_scrq[%d] irq\n", i
);
1408 disable_scrq_irq(adapter
, adapter
->rx_scrq
[i
]);
1409 disable_irq(adapter
->rx_scrq
[i
]->irq
);
1415 static void ibmvnic_cleanup(struct net_device
*netdev
)
1417 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
1419 /* ensure that transmissions are stopped if called by do_reset */
1420 if (test_bit(0, &adapter
->resetting
))
1421 netif_tx_disable(netdev
);
1423 netif_tx_stop_all_queues(netdev
);
1425 ibmvnic_napi_disable(adapter
);
1426 ibmvnic_disable_irqs(adapter
);
1428 clean_rx_pools(adapter
);
1429 clean_tx_pools(adapter
);
1432 static int __ibmvnic_close(struct net_device
*netdev
)
1434 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
1437 adapter
->state
= VNIC_CLOSING
;
1438 rc
= set_link_state(adapter
, IBMVNIC_LOGICAL_LNK_DN
);
1439 adapter
->state
= VNIC_CLOSED
;
1443 static int ibmvnic_close(struct net_device
*netdev
)
1445 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
1448 netdev_dbg(netdev
, "[S:%s FOP:%d FRR:%d] Closing\n",
1449 adapter_state_to_string(adapter
->state
),
1450 adapter
->failover_pending
,
1451 adapter
->force_reset_recovery
);
1453 /* If device failover is pending, just set device state and return.
1454 * Device operation will be handled by reset routine.
1456 if (adapter
->failover_pending
) {
1457 adapter
->state
= VNIC_CLOSED
;
1461 rc
= __ibmvnic_close(netdev
);
1462 ibmvnic_cleanup(netdev
);
1468 * build_hdr_data - creates L2/L3/L4 header data buffer
1469 * @hdr_field: bitfield determining needed headers
1470 * @skb: socket buffer
1471 * @hdr_len: array of header lengths
1472 * @hdr_data: buffer to write the header to
1474 * Reads hdr_field to determine which headers are needed by firmware.
1475 * Builds a buffer containing these headers. Saves individual header
1476 * lengths and total buffer length to be used to build descriptors.
1478 static int build_hdr_data(u8 hdr_field
, struct sk_buff
*skb
,
1479 int *hdr_len
, u8
*hdr_data
)
1484 if (skb_vlan_tagged(skb
) && !skb_vlan_tag_present(skb
))
1485 hdr_len
[0] = sizeof(struct vlan_ethhdr
);
1487 hdr_len
[0] = sizeof(struct ethhdr
);
1489 if (skb
->protocol
== htons(ETH_P_IP
)) {
1490 hdr_len
[1] = ip_hdr(skb
)->ihl
* 4;
1491 if (ip_hdr(skb
)->protocol
== IPPROTO_TCP
)
1492 hdr_len
[2] = tcp_hdrlen(skb
);
1493 else if (ip_hdr(skb
)->protocol
== IPPROTO_UDP
)
1494 hdr_len
[2] = sizeof(struct udphdr
);
1495 } else if (skb
->protocol
== htons(ETH_P_IPV6
)) {
1496 hdr_len
[1] = sizeof(struct ipv6hdr
);
1497 if (ipv6_hdr(skb
)->nexthdr
== IPPROTO_TCP
)
1498 hdr_len
[2] = tcp_hdrlen(skb
);
1499 else if (ipv6_hdr(skb
)->nexthdr
== IPPROTO_UDP
)
1500 hdr_len
[2] = sizeof(struct udphdr
);
1501 } else if (skb
->protocol
== htons(ETH_P_ARP
)) {
1502 hdr_len
[1] = arp_hdr_len(skb
->dev
);
1506 memset(hdr_data
, 0, 120);
1507 if ((hdr_field
>> 6) & 1) {
1508 hdr
= skb_mac_header(skb
);
1509 memcpy(hdr_data
, hdr
, hdr_len
[0]);
1513 if ((hdr_field
>> 5) & 1) {
1514 hdr
= skb_network_header(skb
);
1515 memcpy(hdr_data
+ len
, hdr
, hdr_len
[1]);
1519 if ((hdr_field
>> 4) & 1) {
1520 hdr
= skb_transport_header(skb
);
1521 memcpy(hdr_data
+ len
, hdr
, hdr_len
[2]);
1528 * create_hdr_descs - create header and header extension descriptors
1529 * @hdr_field: bitfield determining needed headers
1530 * @hdr_data: buffer containing header data
1531 * @len: length of data buffer
1532 * @hdr_len: array of individual header lengths
1533 * @scrq_arr: descriptor array
1535 * Creates header and, if needed, header extension descriptors and
1536 * places them in a descriptor array, scrq_arr
1539 static int create_hdr_descs(u8 hdr_field
, u8
*hdr_data
, int len
, int *hdr_len
,
1540 union sub_crq
*scrq_arr
)
1542 union sub_crq hdr_desc
;
1548 while (tmp_len
> 0) {
1549 cur
= hdr_data
+ len
- tmp_len
;
1551 memset(&hdr_desc
, 0, sizeof(hdr_desc
));
1552 if (cur
!= hdr_data
) {
1553 data
= hdr_desc
.hdr_ext
.data
;
1554 tmp
= tmp_len
> 29 ? 29 : tmp_len
;
1555 hdr_desc
.hdr_ext
.first
= IBMVNIC_CRQ_CMD
;
1556 hdr_desc
.hdr_ext
.type
= IBMVNIC_HDR_EXT_DESC
;
1557 hdr_desc
.hdr_ext
.len
= tmp
;
1559 data
= hdr_desc
.hdr
.data
;
1560 tmp
= tmp_len
> 24 ? 24 : tmp_len
;
1561 hdr_desc
.hdr
.first
= IBMVNIC_CRQ_CMD
;
1562 hdr_desc
.hdr
.type
= IBMVNIC_HDR_DESC
;
1563 hdr_desc
.hdr
.len
= tmp
;
1564 hdr_desc
.hdr
.l2_len
= (u8
)hdr_len
[0];
1565 hdr_desc
.hdr
.l3_len
= cpu_to_be16((u16
)hdr_len
[1]);
1566 hdr_desc
.hdr
.l4_len
= (u8
)hdr_len
[2];
1567 hdr_desc
.hdr
.flag
= hdr_field
<< 1;
1569 memcpy(data
, cur
, tmp
);
1571 *scrq_arr
= hdr_desc
;
1580 * build_hdr_descs_arr - build a header descriptor array
1581 * @skb: tx socket buffer
1582 * @indir_arr: indirect array
1583 * @num_entries: number of descriptors to be sent
1584 * @hdr_field: bit field determining which headers will be sent
1586 * This function will build a TX descriptor array with applicable
1587 * L2/L3/L4 packet header descriptors to be sent by send_subcrq_indirect.
1590 static void build_hdr_descs_arr(struct sk_buff
*skb
,
1591 union sub_crq
*indir_arr
,
1592 int *num_entries
, u8 hdr_field
)
1594 int hdr_len
[3] = {0, 0, 0};
1595 u8 hdr_data
[140] = {0};
1598 tot_len
= build_hdr_data(hdr_field
, skb
, hdr_len
,
1600 *num_entries
+= create_hdr_descs(hdr_field
, hdr_data
, tot_len
, hdr_len
,
1604 static int ibmvnic_xmit_workarounds(struct sk_buff
*skb
,
1605 struct net_device
*netdev
)
1607 /* For some backing devices, mishandling of small packets
1608 * can result in a loss of connection or TX stall. Device
1609 * architects recommend that no packet should be smaller
1610 * than the minimum MTU value provided to the driver, so
1611 * pad any packets to that length
1613 if (skb
->len
< netdev
->min_mtu
)
1614 return skb_put_padto(skb
, netdev
->min_mtu
);
1619 static void ibmvnic_tx_scrq_clean_buffer(struct ibmvnic_adapter
*adapter
,
1620 struct ibmvnic_sub_crq_queue
*tx_scrq
)
1622 struct ibmvnic_ind_xmit_queue
*ind_bufp
;
1623 struct ibmvnic_tx_buff
*tx_buff
;
1624 struct ibmvnic_tx_pool
*tx_pool
;
1625 union sub_crq tx_scrq_entry
;
1631 ind_bufp
= &tx_scrq
->ind_buf
;
1632 entries
= (u64
)ind_bufp
->index
;
1633 queue_num
= tx_scrq
->pool_index
;
1635 for (i
= entries
- 1; i
>= 0; --i
) {
1636 tx_scrq_entry
= ind_bufp
->indir_arr
[i
];
1637 if (tx_scrq_entry
.v1
.type
!= IBMVNIC_TX_DESC
)
1639 index
= be32_to_cpu(tx_scrq_entry
.v1
.correlator
);
1640 if (index
& IBMVNIC_TSO_POOL_MASK
) {
1641 tx_pool
= &adapter
->tso_pool
[queue_num
];
1642 index
&= ~IBMVNIC_TSO_POOL_MASK
;
1644 tx_pool
= &adapter
->tx_pool
[queue_num
];
1646 tx_pool
->free_map
[tx_pool
->consumer_index
] = index
;
1647 tx_pool
->consumer_index
= tx_pool
->consumer_index
== 0 ?
1648 tx_pool
->num_buffers
- 1 :
1649 tx_pool
->consumer_index
- 1;
1650 tx_buff
= &tx_pool
->tx_buff
[index
];
1651 adapter
->netdev
->stats
.tx_packets
--;
1652 adapter
->netdev
->stats
.tx_bytes
-= tx_buff
->skb
->len
;
1653 adapter
->tx_stats_buffers
[queue_num
].packets
--;
1654 adapter
->tx_stats_buffers
[queue_num
].bytes
-=
1656 dev_kfree_skb_any(tx_buff
->skb
);
1657 tx_buff
->skb
= NULL
;
1658 adapter
->netdev
->stats
.tx_dropped
++;
1660 ind_bufp
->index
= 0;
1661 if (atomic_sub_return(entries
, &tx_scrq
->used
) <=
1662 (adapter
->req_tx_entries_per_subcrq
/ 2) &&
1663 __netif_subqueue_stopped(adapter
->netdev
, queue_num
) &&
1664 !test_bit(0, &adapter
->resetting
)) {
1665 netif_wake_subqueue(adapter
->netdev
, queue_num
);
1666 netdev_dbg(adapter
->netdev
, "Started queue %d\n",
1671 static int ibmvnic_tx_scrq_flush(struct ibmvnic_adapter
*adapter
,
1672 struct ibmvnic_sub_crq_queue
*tx_scrq
)
1674 struct ibmvnic_ind_xmit_queue
*ind_bufp
;
1680 ind_bufp
= &tx_scrq
->ind_buf
;
1681 dma_addr
= (u64
)ind_bufp
->indir_dma
;
1682 entries
= (u64
)ind_bufp
->index
;
1683 handle
= tx_scrq
->handle
;
1687 rc
= send_subcrq_indirect(adapter
, handle
, dma_addr
, entries
);
1689 ibmvnic_tx_scrq_clean_buffer(adapter
, tx_scrq
);
1691 ind_bufp
->index
= 0;
1695 static netdev_tx_t
ibmvnic_xmit(struct sk_buff
*skb
, struct net_device
*netdev
)
1697 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
1698 int queue_num
= skb_get_queue_mapping(skb
);
1699 u8
*hdrs
= (u8
*)&adapter
->tx_rx_desc_req
;
1700 struct device
*dev
= &adapter
->vdev
->dev
;
1701 struct ibmvnic_ind_xmit_queue
*ind_bufp
;
1702 struct ibmvnic_tx_buff
*tx_buff
= NULL
;
1703 struct ibmvnic_sub_crq_queue
*tx_scrq
;
1704 struct ibmvnic_tx_pool
*tx_pool
;
1705 unsigned int tx_send_failed
= 0;
1706 netdev_tx_t ret
= NETDEV_TX_OK
;
1707 unsigned int tx_map_failed
= 0;
1708 union sub_crq indir_arr
[16];
1709 unsigned int tx_dropped
= 0;
1710 unsigned int tx_packets
= 0;
1711 unsigned int tx_bytes
= 0;
1712 dma_addr_t data_dma_addr
;
1713 struct netdev_queue
*txq
;
1714 unsigned long lpar_rc
;
1715 union sub_crq tx_crq
;
1716 unsigned int offset
;
1717 int num_entries
= 1;
1722 tx_scrq
= adapter
->tx_scrq
[queue_num
];
1723 txq
= netdev_get_tx_queue(netdev
, queue_num
);
1724 ind_bufp
= &tx_scrq
->ind_buf
;
1726 if (test_bit(0, &adapter
->resetting
)) {
1727 dev_kfree_skb_any(skb
);
1735 if (ibmvnic_xmit_workarounds(skb
, netdev
)) {
1739 ibmvnic_tx_scrq_flush(adapter
, tx_scrq
);
1742 if (skb_is_gso(skb
))
1743 tx_pool
= &adapter
->tso_pool
[queue_num
];
1745 tx_pool
= &adapter
->tx_pool
[queue_num
];
1747 index
= tx_pool
->free_map
[tx_pool
->consumer_index
];
1749 if (index
== IBMVNIC_INVALID_MAP
) {
1750 dev_kfree_skb_any(skb
);
1753 ibmvnic_tx_scrq_flush(adapter
, tx_scrq
);
1758 tx_pool
->free_map
[tx_pool
->consumer_index
] = IBMVNIC_INVALID_MAP
;
1760 offset
= index
* tx_pool
->buf_size
;
1761 dst
= tx_pool
->long_term_buff
.buff
+ offset
;
1762 memset(dst
, 0, tx_pool
->buf_size
);
1763 data_dma_addr
= tx_pool
->long_term_buff
.addr
+ offset
;
1765 if (skb_shinfo(skb
)->nr_frags
) {
1769 skb_copy_from_linear_data(skb
, dst
, skb_headlen(skb
));
1770 cur
= skb_headlen(skb
);
1772 /* Copy the frags */
1773 for (i
= 0; i
< skb_shinfo(skb
)->nr_frags
; i
++) {
1774 const skb_frag_t
*frag
= &skb_shinfo(skb
)->frags
[i
];
1776 memcpy(dst
+ cur
, skb_frag_address(frag
),
1777 skb_frag_size(frag
));
1778 cur
+= skb_frag_size(frag
);
1781 skb_copy_from_linear_data(skb
, dst
, skb
->len
);
1784 /* post changes to long_term_buff *dst before VIOS accessing it */
1787 tx_pool
->consumer_index
=
1788 (tx_pool
->consumer_index
+ 1) % tx_pool
->num_buffers
;
1790 tx_buff
= &tx_pool
->tx_buff
[index
];
1792 tx_buff
->index
= index
;
1793 tx_buff
->pool_index
= queue_num
;
1795 memset(&tx_crq
, 0, sizeof(tx_crq
));
1796 tx_crq
.v1
.first
= IBMVNIC_CRQ_CMD
;
1797 tx_crq
.v1
.type
= IBMVNIC_TX_DESC
;
1798 tx_crq
.v1
.n_crq_elem
= 1;
1799 tx_crq
.v1
.n_sge
= 1;
1800 tx_crq
.v1
.flags1
= IBMVNIC_TX_COMP_NEEDED
;
1802 if (skb_is_gso(skb
))
1803 tx_crq
.v1
.correlator
=
1804 cpu_to_be32(index
| IBMVNIC_TSO_POOL_MASK
);
1806 tx_crq
.v1
.correlator
= cpu_to_be32(index
);
1807 tx_crq
.v1
.dma_reg
= cpu_to_be16(tx_pool
->long_term_buff
.map_id
);
1808 tx_crq
.v1
.sge_len
= cpu_to_be32(skb
->len
);
1809 tx_crq
.v1
.ioba
= cpu_to_be64(data_dma_addr
);
1811 if (adapter
->vlan_header_insertion
&& skb_vlan_tag_present(skb
)) {
1812 tx_crq
.v1
.flags2
|= IBMVNIC_TX_VLAN_INSERT
;
1813 tx_crq
.v1
.vlan_id
= cpu_to_be16(skb
->vlan_tci
);
1816 if (skb
->protocol
== htons(ETH_P_IP
)) {
1817 tx_crq
.v1
.flags1
|= IBMVNIC_TX_PROT_IPV4
;
1818 proto
= ip_hdr(skb
)->protocol
;
1819 } else if (skb
->protocol
== htons(ETH_P_IPV6
)) {
1820 tx_crq
.v1
.flags1
|= IBMVNIC_TX_PROT_IPV6
;
1821 proto
= ipv6_hdr(skb
)->nexthdr
;
1824 if (proto
== IPPROTO_TCP
)
1825 tx_crq
.v1
.flags1
|= IBMVNIC_TX_PROT_TCP
;
1826 else if (proto
== IPPROTO_UDP
)
1827 tx_crq
.v1
.flags1
|= IBMVNIC_TX_PROT_UDP
;
1829 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
1830 tx_crq
.v1
.flags1
|= IBMVNIC_TX_CHKSUM_OFFLOAD
;
1833 if (skb_is_gso(skb
)) {
1834 tx_crq
.v1
.flags1
|= IBMVNIC_TX_LSO
;
1835 tx_crq
.v1
.mss
= cpu_to_be16(skb_shinfo(skb
)->gso_size
);
1839 if ((*hdrs
>> 7) & 1)
1840 build_hdr_descs_arr(skb
, indir_arr
, &num_entries
, *hdrs
);
1842 tx_crq
.v1
.n_crq_elem
= num_entries
;
1843 tx_buff
->num_entries
= num_entries
;
1844 /* flush buffer if current entry can not fit */
1845 if (num_entries
+ ind_bufp
->index
> IBMVNIC_MAX_IND_DESCS
) {
1846 lpar_rc
= ibmvnic_tx_scrq_flush(adapter
, tx_scrq
);
1847 if (lpar_rc
!= H_SUCCESS
)
1851 indir_arr
[0] = tx_crq
;
1852 memcpy(&ind_bufp
->indir_arr
[ind_bufp
->index
], &indir_arr
[0],
1853 num_entries
* sizeof(struct ibmvnic_generic_scrq
));
1854 ind_bufp
->index
+= num_entries
;
1855 if (__netdev_tx_sent_queue(txq
, skb
->len
,
1856 netdev_xmit_more() &&
1857 ind_bufp
->index
< IBMVNIC_MAX_IND_DESCS
)) {
1858 lpar_rc
= ibmvnic_tx_scrq_flush(adapter
, tx_scrq
);
1859 if (lpar_rc
!= H_SUCCESS
)
1863 if (atomic_add_return(num_entries
, &tx_scrq
->used
)
1864 >= adapter
->req_tx_entries_per_subcrq
) {
1865 netdev_dbg(netdev
, "Stopping queue %d\n", queue_num
);
1866 netif_stop_subqueue(netdev
, queue_num
);
1870 tx_bytes
+= skb
->len
;
1871 txq
->trans_start
= jiffies
;
1876 dev_kfree_skb_any(skb
);
1877 tx_buff
->skb
= NULL
;
1878 tx_pool
->consumer_index
= tx_pool
->consumer_index
== 0 ?
1879 tx_pool
->num_buffers
- 1 :
1880 tx_pool
->consumer_index
- 1;
1883 if (lpar_rc
!= H_CLOSED
&& lpar_rc
!= H_PARAMETER
)
1884 dev_err_ratelimited(dev
, "tx: send failed\n");
1886 if (lpar_rc
== H_CLOSED
|| adapter
->failover_pending
) {
1887 /* Disable TX and report carrier off if queue is closed
1888 * or pending failover.
1889 * Firmware guarantees that a signal will be sent to the
1890 * driver, triggering a reset or some other action.
1892 netif_tx_stop_all_queues(netdev
);
1893 netif_carrier_off(netdev
);
1896 netdev
->stats
.tx_dropped
+= tx_dropped
;
1897 netdev
->stats
.tx_bytes
+= tx_bytes
;
1898 netdev
->stats
.tx_packets
+= tx_packets
;
1899 adapter
->tx_send_failed
+= tx_send_failed
;
1900 adapter
->tx_map_failed
+= tx_map_failed
;
1901 adapter
->tx_stats_buffers
[queue_num
].packets
+= tx_packets
;
1902 adapter
->tx_stats_buffers
[queue_num
].bytes
+= tx_bytes
;
1903 adapter
->tx_stats_buffers
[queue_num
].dropped_packets
+= tx_dropped
;
1908 static void ibmvnic_set_multi(struct net_device
*netdev
)
1910 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
1911 struct netdev_hw_addr
*ha
;
1912 union ibmvnic_crq crq
;
1914 memset(&crq
, 0, sizeof(crq
));
1915 crq
.request_capability
.first
= IBMVNIC_CRQ_CMD
;
1916 crq
.request_capability
.cmd
= REQUEST_CAPABILITY
;
1918 if (netdev
->flags
& IFF_PROMISC
) {
1919 if (!adapter
->promisc_supported
)
1922 if (netdev
->flags
& IFF_ALLMULTI
) {
1923 /* Accept all multicast */
1924 memset(&crq
, 0, sizeof(crq
));
1925 crq
.multicast_ctrl
.first
= IBMVNIC_CRQ_CMD
;
1926 crq
.multicast_ctrl
.cmd
= MULTICAST_CTRL
;
1927 crq
.multicast_ctrl
.flags
= IBMVNIC_ENABLE_ALL
;
1928 ibmvnic_send_crq(adapter
, &crq
);
1929 } else if (netdev_mc_empty(netdev
)) {
1930 /* Reject all multicast */
1931 memset(&crq
, 0, sizeof(crq
));
1932 crq
.multicast_ctrl
.first
= IBMVNIC_CRQ_CMD
;
1933 crq
.multicast_ctrl
.cmd
= MULTICAST_CTRL
;
1934 crq
.multicast_ctrl
.flags
= IBMVNIC_DISABLE_ALL
;
1935 ibmvnic_send_crq(adapter
, &crq
);
1937 /* Accept one or more multicast(s) */
1938 netdev_for_each_mc_addr(ha
, netdev
) {
1939 memset(&crq
, 0, sizeof(crq
));
1940 crq
.multicast_ctrl
.first
= IBMVNIC_CRQ_CMD
;
1941 crq
.multicast_ctrl
.cmd
= MULTICAST_CTRL
;
1942 crq
.multicast_ctrl
.flags
= IBMVNIC_ENABLE_MC
;
1943 ether_addr_copy(&crq
.multicast_ctrl
.mac_addr
[0],
1945 ibmvnic_send_crq(adapter
, &crq
);
1951 static int __ibmvnic_set_mac(struct net_device
*netdev
, u8
*dev_addr
)
1953 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
1954 union ibmvnic_crq crq
;
1957 if (!is_valid_ether_addr(dev_addr
)) {
1958 rc
= -EADDRNOTAVAIL
;
1962 memset(&crq
, 0, sizeof(crq
));
1963 crq
.change_mac_addr
.first
= IBMVNIC_CRQ_CMD
;
1964 crq
.change_mac_addr
.cmd
= CHANGE_MAC_ADDR
;
1965 ether_addr_copy(&crq
.change_mac_addr
.mac_addr
[0], dev_addr
);
1967 mutex_lock(&adapter
->fw_lock
);
1968 adapter
->fw_done_rc
= 0;
1969 reinit_completion(&adapter
->fw_done
);
1971 rc
= ibmvnic_send_crq(adapter
, &crq
);
1974 mutex_unlock(&adapter
->fw_lock
);
1978 rc
= ibmvnic_wait_for_completion(adapter
, &adapter
->fw_done
, 10000);
1979 /* netdev->dev_addr is changed in handle_change_mac_rsp function */
1980 if (rc
|| adapter
->fw_done_rc
) {
1982 mutex_unlock(&adapter
->fw_lock
);
1985 mutex_unlock(&adapter
->fw_lock
);
1988 ether_addr_copy(adapter
->mac_addr
, netdev
->dev_addr
);
1992 static int ibmvnic_set_mac(struct net_device
*netdev
, void *p
)
1994 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
1995 struct sockaddr
*addr
= p
;
1999 if (!is_valid_ether_addr(addr
->sa_data
))
2000 return -EADDRNOTAVAIL
;
2002 ether_addr_copy(adapter
->mac_addr
, addr
->sa_data
);
2003 if (adapter
->state
!= VNIC_PROBED
)
2004 rc
= __ibmvnic_set_mac(netdev
, addr
->sa_data
);
2009 static const char *reset_reason_to_string(enum ibmvnic_reset_reason reason
)
2012 case VNIC_RESET_FAILOVER
:
2014 case VNIC_RESET_MOBILITY
:
2016 case VNIC_RESET_FATAL
:
2018 case VNIC_RESET_NON_FATAL
:
2020 case VNIC_RESET_TIMEOUT
:
2022 case VNIC_RESET_CHANGE_PARAM
:
2023 return "CHANGE_PARAM";
2024 case VNIC_RESET_PASSIVE_INIT
:
2025 return "PASSIVE_INIT";
2031 * do_reset returns zero if we are able to keep processing reset events, or
2032 * non-zero if we hit a fatal error and must halt.
2034 static int do_reset(struct ibmvnic_adapter
*adapter
,
2035 struct ibmvnic_rwi
*rwi
, u32 reset_state
)
2037 u64 old_num_rx_queues
, old_num_tx_queues
;
2038 u64 old_num_rx_slots
, old_num_tx_slots
;
2039 struct net_device
*netdev
= adapter
->netdev
;
2042 netdev_dbg(adapter
->netdev
,
2043 "[S:%s FOP:%d] Reset reason: %s, reset_state: %s\n",
2044 adapter_state_to_string(adapter
->state
),
2045 adapter
->failover_pending
,
2046 reset_reason_to_string(rwi
->reset_reason
),
2047 adapter_state_to_string(reset_state
));
2049 adapter
->reset_reason
= rwi
->reset_reason
;
2050 /* requestor of VNIC_RESET_CHANGE_PARAM already has the rtnl lock */
2051 if (!(adapter
->reset_reason
== VNIC_RESET_CHANGE_PARAM
))
2054 /* Now that we have the rtnl lock, clear any pending failover.
2055 * This will ensure ibmvnic_open() has either completed or will
2056 * block until failover is complete.
2058 if (rwi
->reset_reason
== VNIC_RESET_FAILOVER
)
2059 adapter
->failover_pending
= false;
2061 /* read the state and check (again) after getting rtnl */
2062 reset_state
= adapter
->state
;
2064 if (reset_state
== VNIC_REMOVING
|| reset_state
== VNIC_REMOVED
) {
2069 netif_carrier_off(netdev
);
2071 old_num_rx_queues
= adapter
->req_rx_queues
;
2072 old_num_tx_queues
= adapter
->req_tx_queues
;
2073 old_num_rx_slots
= adapter
->req_rx_add_entries_per_subcrq
;
2074 old_num_tx_slots
= adapter
->req_tx_entries_per_subcrq
;
2076 ibmvnic_cleanup(netdev
);
2078 if (reset_state
== VNIC_OPEN
&&
2079 adapter
->reset_reason
!= VNIC_RESET_MOBILITY
&&
2080 adapter
->reset_reason
!= VNIC_RESET_FAILOVER
) {
2081 if (adapter
->reset_reason
== VNIC_RESET_CHANGE_PARAM
) {
2082 rc
= __ibmvnic_close(netdev
);
2086 adapter
->state
= VNIC_CLOSING
;
2088 /* Release the RTNL lock before link state change and
2089 * re-acquire after the link state change to allow
2090 * linkwatch_event to grab the RTNL lock and run during
2094 rc
= set_link_state(adapter
, IBMVNIC_LOGICAL_LNK_DN
);
2099 if (adapter
->state
== VNIC_OPEN
) {
2100 /* When we dropped rtnl, ibmvnic_open() got
2101 * it and noticed that we are resetting and
2102 * set the adapter state to OPEN. Update our
2103 * new "target" state, and resume the reset
2104 * from VNIC_CLOSING state.
2107 "Open changed state from %s, updating.\n",
2108 adapter_state_to_string(reset_state
));
2109 reset_state
= VNIC_OPEN
;
2110 adapter
->state
= VNIC_CLOSING
;
2113 if (adapter
->state
!= VNIC_CLOSING
) {
2114 /* If someone else changed the adapter state
2115 * when we dropped the rtnl, fail the reset
2120 adapter
->state
= VNIC_CLOSED
;
2124 if (adapter
->reset_reason
== VNIC_RESET_CHANGE_PARAM
) {
2125 release_resources(adapter
);
2126 release_sub_crqs(adapter
, 1);
2127 release_crq_queue(adapter
);
2130 if (adapter
->reset_reason
!= VNIC_RESET_NON_FATAL
) {
2131 /* remove the closed state so when we call open it appears
2132 * we are coming from the probed state.
2134 adapter
->state
= VNIC_PROBED
;
2136 if (adapter
->reset_reason
== VNIC_RESET_CHANGE_PARAM
) {
2137 rc
= init_crq_queue(adapter
);
2138 } else if (adapter
->reset_reason
== VNIC_RESET_MOBILITY
) {
2139 rc
= ibmvnic_reenable_crq_queue(adapter
);
2140 release_sub_crqs(adapter
, 1);
2142 rc
= ibmvnic_reset_crq(adapter
);
2143 if (rc
== H_CLOSED
|| rc
== H_SUCCESS
) {
2144 rc
= vio_enable_interrupts(adapter
->vdev
);
2146 netdev_err(adapter
->netdev
,
2147 "Reset failed to enable interrupts. rc=%d\n",
2153 netdev_err(adapter
->netdev
,
2154 "Reset couldn't initialize crq. rc=%d\n", rc
);
2158 rc
= ibmvnic_reset_init(adapter
, true);
2160 rc
= IBMVNIC_INIT_FAILED
;
2164 /* If the adapter was in PROBE or DOWN state prior to the reset,
2167 if (reset_state
== VNIC_PROBED
|| reset_state
== VNIC_DOWN
) {
2172 rc
= ibmvnic_login(netdev
);
2176 if (adapter
->reset_reason
== VNIC_RESET_CHANGE_PARAM
) {
2177 rc
= init_resources(adapter
);
2180 } else if (adapter
->req_rx_queues
!= old_num_rx_queues
||
2181 adapter
->req_tx_queues
!= old_num_tx_queues
||
2182 adapter
->req_rx_add_entries_per_subcrq
!=
2184 adapter
->req_tx_entries_per_subcrq
!=
2186 !adapter
->rx_pool
||
2187 !adapter
->tso_pool
||
2188 !adapter
->tx_pool
) {
2189 release_rx_pools(adapter
);
2190 release_tx_pools(adapter
);
2191 release_napi(adapter
);
2192 release_vpd_data(adapter
);
2194 rc
= init_resources(adapter
);
2199 rc
= reset_tx_pools(adapter
);
2201 netdev_dbg(adapter
->netdev
, "reset tx pools failed (%d)\n",
2206 rc
= reset_rx_pools(adapter
);
2208 netdev_dbg(adapter
->netdev
, "reset rx pools failed (%d)\n",
2213 ibmvnic_disable_irqs(adapter
);
2215 adapter
->state
= VNIC_CLOSED
;
2217 if (reset_state
== VNIC_CLOSED
) {
2222 rc
= __ibmvnic_open(netdev
);
2224 rc
= IBMVNIC_OPEN_FAILED
;
2228 /* refresh device's multicast list */
2229 ibmvnic_set_multi(netdev
);
2231 if (adapter
->reset_reason
== VNIC_RESET_FAILOVER
||
2232 adapter
->reset_reason
== VNIC_RESET_MOBILITY
)
2233 __netdev_notify_peers(netdev
);
2238 /* restore the adapter state if reset failed */
2240 adapter
->state
= reset_state
;
2241 /* requestor of VNIC_RESET_CHANGE_PARAM should still hold the rtnl lock */
2242 if (!(adapter
->reset_reason
== VNIC_RESET_CHANGE_PARAM
))
2245 netdev_dbg(adapter
->netdev
, "[S:%s FOP:%d] Reset done, rc %d\n",
2246 adapter_state_to_string(adapter
->state
),
2247 adapter
->failover_pending
, rc
);
2251 static int do_hard_reset(struct ibmvnic_adapter
*adapter
,
2252 struct ibmvnic_rwi
*rwi
, u32 reset_state
)
2254 struct net_device
*netdev
= adapter
->netdev
;
2257 netdev_dbg(adapter
->netdev
, "Hard resetting driver (%s)\n",
2258 reset_reason_to_string(rwi
->reset_reason
));
2260 /* read the state and check (again) after getting rtnl */
2261 reset_state
= adapter
->state
;
2263 if (reset_state
== VNIC_REMOVING
|| reset_state
== VNIC_REMOVED
) {
2268 netif_carrier_off(netdev
);
2269 adapter
->reset_reason
= rwi
->reset_reason
;
2271 ibmvnic_cleanup(netdev
);
2272 release_resources(adapter
);
2273 release_sub_crqs(adapter
, 0);
2274 release_crq_queue(adapter
);
2276 /* remove the closed state so when we call open it appears
2277 * we are coming from the probed state.
2279 adapter
->state
= VNIC_PROBED
;
2281 reinit_completion(&adapter
->init_done
);
2282 rc
= init_crq_queue(adapter
);
2284 netdev_err(adapter
->netdev
,
2285 "Couldn't initialize crq. rc=%d\n", rc
);
2289 rc
= ibmvnic_reset_init(adapter
, false);
2293 /* If the adapter was in PROBE or DOWN state prior to the reset,
2296 if (reset_state
== VNIC_PROBED
|| reset_state
== VNIC_DOWN
)
2299 rc
= ibmvnic_login(netdev
);
2303 rc
= init_resources(adapter
);
2307 ibmvnic_disable_irqs(adapter
);
2308 adapter
->state
= VNIC_CLOSED
;
2310 if (reset_state
== VNIC_CLOSED
)
2313 rc
= __ibmvnic_open(netdev
);
2315 rc
= IBMVNIC_OPEN_FAILED
;
2319 __netdev_notify_peers(netdev
);
2321 /* restore adapter state if reset failed */
2323 adapter
->state
= reset_state
;
2324 netdev_dbg(adapter
->netdev
, "[S:%s FOP:%d] Hard reset done, rc %d\n",
2325 adapter_state_to_string(adapter
->state
),
2326 adapter
->failover_pending
, rc
);
2330 static struct ibmvnic_rwi
*get_next_rwi(struct ibmvnic_adapter
*adapter
)
2332 struct ibmvnic_rwi
*rwi
;
2333 unsigned long flags
;
2335 spin_lock_irqsave(&adapter
->rwi_lock
, flags
);
2337 if (!list_empty(&adapter
->rwi_list
)) {
2338 rwi
= list_first_entry(&adapter
->rwi_list
, struct ibmvnic_rwi
,
2340 list_del(&rwi
->list
);
2345 spin_unlock_irqrestore(&adapter
->rwi_lock
, flags
);
2350 * do_passive_init - complete probing when partner device is detected.
2351 * @adapter: ibmvnic_adapter struct
2353 * If the ibmvnic device does not have a partner device to communicate with at boot
2354 * and that partner device comes online at a later time, this function is called
2355 * to complete the initialization process of ibmvnic device.
2356 * Caller is expected to hold rtnl_lock().
2358 * Returns non-zero if sub-CRQs are not initialized properly leaving the device
2359 * in the down state.
2360 * Returns 0 upon success and the device is in PROBED state.
2363 static int do_passive_init(struct ibmvnic_adapter
*adapter
)
2365 unsigned long timeout
= msecs_to_jiffies(30000);
2366 struct net_device
*netdev
= adapter
->netdev
;
2367 struct device
*dev
= &adapter
->vdev
->dev
;
2370 netdev_dbg(netdev
, "Partner device found, probing.\n");
2372 adapter
->state
= VNIC_PROBING
;
2373 reinit_completion(&adapter
->init_done
);
2374 adapter
->init_done_rc
= 0;
2375 adapter
->crq
.active
= true;
2377 rc
= send_crq_init_complete(adapter
);
2381 rc
= send_version_xchg(adapter
);
2383 netdev_dbg(adapter
->netdev
, "send_version_xchg failed, rc=%d\n", rc
);
2385 if (!wait_for_completion_timeout(&adapter
->init_done
, timeout
)) {
2386 dev_err(dev
, "Initialization sequence timed out\n");
2391 rc
= init_sub_crqs(adapter
);
2393 dev_err(dev
, "Initialization of sub crqs failed, rc=%d\n", rc
);
2397 rc
= init_sub_crq_irqs(adapter
);
2399 dev_err(dev
, "Failed to initialize sub crq irqs\n, rc=%d", rc
);
2403 netdev
->mtu
= adapter
->req_mtu
- ETH_HLEN
;
2404 netdev
->min_mtu
= adapter
->min_mtu
- ETH_HLEN
;
2405 netdev
->max_mtu
= adapter
->max_mtu
- ETH_HLEN
;
2407 adapter
->state
= VNIC_PROBED
;
2408 netdev_dbg(netdev
, "Probed successfully. Waiting for signal from partner device.\n");
2413 release_sub_crqs(adapter
, 1);
2415 adapter
->state
= VNIC_DOWN
;
2419 static void __ibmvnic_reset(struct work_struct
*work
)
2421 struct ibmvnic_adapter
*adapter
;
2422 bool saved_state
= false;
2423 struct ibmvnic_rwi
*tmprwi
;
2424 struct ibmvnic_rwi
*rwi
;
2425 unsigned long flags
;
2430 adapter
= container_of(work
, struct ibmvnic_adapter
, ibmvnic_reset
);
2432 if (test_and_set_bit_lock(0, &adapter
->resetting
)) {
2433 queue_delayed_work(system_long_wq
,
2434 &adapter
->ibmvnic_delayed_reset
,
2435 IBMVNIC_RESET_DELAY
);
2439 rwi
= get_next_rwi(adapter
);
2441 spin_lock_irqsave(&adapter
->state_lock
, flags
);
2443 if (adapter
->state
== VNIC_REMOVING
||
2444 adapter
->state
== VNIC_REMOVED
) {
2445 spin_unlock_irqrestore(&adapter
->state_lock
, flags
);
2452 reset_state
= adapter
->state
;
2455 spin_unlock_irqrestore(&adapter
->state_lock
, flags
);
2457 if (rwi
->reset_reason
== VNIC_RESET_PASSIVE_INIT
) {
2459 rc
= do_passive_init(adapter
);
2462 netif_carrier_on(adapter
->netdev
);
2463 } else if (adapter
->force_reset_recovery
) {
2464 /* Since we are doing a hard reset now, clear the
2465 * failover_pending flag so we don't ignore any
2466 * future MOBILITY or other resets.
2468 adapter
->failover_pending
= false;
2470 /* Transport event occurred during previous reset */
2471 if (adapter
->wait_for_reset
) {
2472 /* Previous was CHANGE_PARAM; caller locked */
2473 adapter
->force_reset_recovery
= false;
2474 rc
= do_hard_reset(adapter
, rwi
, reset_state
);
2477 adapter
->force_reset_recovery
= false;
2478 rc
= do_hard_reset(adapter
, rwi
, reset_state
);
2486 /* If auto-priority-failover is enabled we can get
2487 * back to back failovers during resets, resulting
2488 * in at least two failed resets (from high-priority
2489 * backing device to low-priority one and then back)
2490 * If resets continue to fail beyond that, give the
2491 * adapter some time to settle down before retrying.
2493 if (num_fails
>= 3) {
2494 netdev_dbg(adapter
->netdev
,
2495 "[S:%s] Hard reset failed %d times, waiting 60 secs\n",
2496 adapter_state_to_string(adapter
->state
),
2498 set_current_state(TASK_UNINTERRUPTIBLE
);
2499 schedule_timeout(60 * HZ
);
2502 rc
= do_reset(adapter
, rwi
, reset_state
);
2505 adapter
->last_reset_time
= jiffies
;
2508 netdev_dbg(adapter
->netdev
, "Reset failed, rc=%d\n", rc
);
2510 rwi
= get_next_rwi(adapter
);
2513 * If there is another reset queued, free the previous rwi
2514 * and process the new reset even if previous reset failed
2515 * (the previous reset could have failed because of a fail
2516 * over for instance, so process the fail over).
2518 * If there are no resets queued and the previous reset failed,
2519 * the adapter would be in an undefined state. So retry the
2520 * previous reset as a hard reset.
2527 if (rwi
&& (rwi
->reset_reason
== VNIC_RESET_FAILOVER
||
2528 rwi
->reset_reason
== VNIC_RESET_MOBILITY
|| rc
))
2529 adapter
->force_reset_recovery
= true;
2532 if (adapter
->wait_for_reset
) {
2533 adapter
->reset_done_rc
= rc
;
2534 complete(&adapter
->reset_done
);
2537 clear_bit_unlock(0, &adapter
->resetting
);
2539 netdev_dbg(adapter
->netdev
,
2540 "[S:%s FRR:%d WFR:%d] Done processing resets\n",
2541 adapter_state_to_string(adapter
->state
),
2542 adapter
->force_reset_recovery
,
2543 adapter
->wait_for_reset
);
2546 static void __ibmvnic_delayed_reset(struct work_struct
*work
)
2548 struct ibmvnic_adapter
*adapter
;
2550 adapter
= container_of(work
, struct ibmvnic_adapter
,
2551 ibmvnic_delayed_reset
.work
);
2552 __ibmvnic_reset(&adapter
->ibmvnic_reset
);
2555 static int ibmvnic_reset(struct ibmvnic_adapter
*adapter
,
2556 enum ibmvnic_reset_reason reason
)
2558 struct list_head
*entry
, *tmp_entry
;
2559 struct ibmvnic_rwi
*rwi
, *tmp
;
2560 struct net_device
*netdev
= adapter
->netdev
;
2561 unsigned long flags
;
2564 spin_lock_irqsave(&adapter
->rwi_lock
, flags
);
2566 /* If failover is pending don't schedule any other reset.
2567 * Instead let the failover complete. If there is already a
2568 * a failover reset scheduled, we will detect and drop the
2569 * duplicate reset when walking the ->rwi_list below.
2571 if (adapter
->state
== VNIC_REMOVING
||
2572 adapter
->state
== VNIC_REMOVED
||
2573 (adapter
->failover_pending
&& reason
!= VNIC_RESET_FAILOVER
)) {
2575 netdev_dbg(netdev
, "Adapter removing or pending failover, skipping reset\n");
2579 if (adapter
->state
== VNIC_PROBING
) {
2580 netdev_warn(netdev
, "Adapter reset during probe\n");
2581 adapter
->init_done_rc
= -EAGAIN
;
2586 list_for_each_entry(tmp
, &adapter
->rwi_list
, list
) {
2587 if (tmp
->reset_reason
== reason
) {
2588 netdev_dbg(netdev
, "Skipping matching reset, reason=%s\n",
2589 reset_reason_to_string(reason
));
2595 rwi
= kzalloc(sizeof(*rwi
), GFP_ATOMIC
);
2600 /* if we just received a transport event,
2601 * flush reset queue and process this reset
2603 if (adapter
->force_reset_recovery
&& !list_empty(&adapter
->rwi_list
)) {
2604 list_for_each_safe(entry
, tmp_entry
, &adapter
->rwi_list
)
2607 rwi
->reset_reason
= reason
;
2608 list_add_tail(&rwi
->list
, &adapter
->rwi_list
);
2609 netdev_dbg(adapter
->netdev
, "Scheduling reset (reason %s)\n",
2610 reset_reason_to_string(reason
));
2611 queue_work(system_long_wq
, &adapter
->ibmvnic_reset
);
2615 /* ibmvnic_close() below can block, so drop the lock first */
2616 spin_unlock_irqrestore(&adapter
->rwi_lock
, flags
);
2619 ibmvnic_close(netdev
);
2624 static void ibmvnic_tx_timeout(struct net_device
*dev
, unsigned int txqueue
)
2626 struct ibmvnic_adapter
*adapter
= netdev_priv(dev
);
2628 if (test_bit(0, &adapter
->resetting
)) {
2629 netdev_err(adapter
->netdev
,
2630 "Adapter is resetting, skip timeout reset\n");
2633 /* No queuing up reset until at least 5 seconds (default watchdog val)
2636 if (time_before(jiffies
, (adapter
->last_reset_time
+ dev
->watchdog_timeo
))) {
2637 netdev_dbg(dev
, "Not yet time to tx timeout.\n");
2640 ibmvnic_reset(adapter
, VNIC_RESET_TIMEOUT
);
2643 static void remove_buff_from_pool(struct ibmvnic_adapter
*adapter
,
2644 struct ibmvnic_rx_buff
*rx_buff
)
2646 struct ibmvnic_rx_pool
*pool
= &adapter
->rx_pool
[rx_buff
->pool_index
];
2648 rx_buff
->skb
= NULL
;
2650 pool
->free_map
[pool
->next_alloc
] = (int)(rx_buff
- pool
->rx_buff
);
2651 pool
->next_alloc
= (pool
->next_alloc
+ 1) % pool
->size
;
2653 atomic_dec(&pool
->available
);
2656 static int ibmvnic_poll(struct napi_struct
*napi
, int budget
)
2658 struct ibmvnic_sub_crq_queue
*rx_scrq
;
2659 struct ibmvnic_adapter
*adapter
;
2660 struct net_device
*netdev
;
2661 int frames_processed
;
2665 adapter
= netdev_priv(netdev
);
2666 scrq_num
= (int)(napi
- adapter
->napi
);
2667 frames_processed
= 0;
2668 rx_scrq
= adapter
->rx_scrq
[scrq_num
];
2671 while (frames_processed
< budget
) {
2672 struct sk_buff
*skb
;
2673 struct ibmvnic_rx_buff
*rx_buff
;
2674 union sub_crq
*next
;
2679 if (unlikely(test_bit(0, &adapter
->resetting
) &&
2680 adapter
->reset_reason
!= VNIC_RESET_NON_FATAL
)) {
2681 enable_scrq_irq(adapter
, rx_scrq
);
2682 napi_complete_done(napi
, frames_processed
);
2683 return frames_processed
;
2686 if (!pending_scrq(adapter
, rx_scrq
))
2688 next
= ibmvnic_next_scrq(adapter
, rx_scrq
);
2689 rx_buff
= (struct ibmvnic_rx_buff
*)
2690 be64_to_cpu(next
->rx_comp
.correlator
);
2691 /* do error checking */
2692 if (next
->rx_comp
.rc
) {
2693 netdev_dbg(netdev
, "rx buffer returned with rc %x\n",
2694 be16_to_cpu(next
->rx_comp
.rc
));
2695 /* free the entry */
2696 next
->rx_comp
.first
= 0;
2697 dev_kfree_skb_any(rx_buff
->skb
);
2698 remove_buff_from_pool(adapter
, rx_buff
);
2700 } else if (!rx_buff
->skb
) {
2701 /* free the entry */
2702 next
->rx_comp
.first
= 0;
2703 remove_buff_from_pool(adapter
, rx_buff
);
2707 length
= be32_to_cpu(next
->rx_comp
.len
);
2708 offset
= be16_to_cpu(next
->rx_comp
.off_frame_data
);
2709 flags
= next
->rx_comp
.flags
;
2711 /* load long_term_buff before copying to skb */
2713 skb_copy_to_linear_data(skb
, rx_buff
->data
+ offset
,
2716 /* VLAN Header has been stripped by the system firmware and
2717 * needs to be inserted by the driver
2719 if (adapter
->rx_vlan_header_insertion
&&
2720 (flags
& IBMVNIC_VLAN_STRIPPED
))
2721 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
),
2722 ntohs(next
->rx_comp
.vlan_tci
));
2724 /* free the entry */
2725 next
->rx_comp
.first
= 0;
2726 remove_buff_from_pool(adapter
, rx_buff
);
2728 skb_put(skb
, length
);
2729 skb
->protocol
= eth_type_trans(skb
, netdev
);
2730 skb_record_rx_queue(skb
, scrq_num
);
2732 if (flags
& IBMVNIC_IP_CHKSUM_GOOD
&&
2733 flags
& IBMVNIC_TCP_UDP_CHKSUM_GOOD
) {
2734 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
2738 napi_gro_receive(napi
, skb
); /* send it up */
2739 netdev
->stats
.rx_packets
++;
2740 netdev
->stats
.rx_bytes
+= length
;
2741 adapter
->rx_stats_buffers
[scrq_num
].packets
++;
2742 adapter
->rx_stats_buffers
[scrq_num
].bytes
+= length
;
2746 if (adapter
->state
!= VNIC_CLOSING
&&
2747 ((atomic_read(&adapter
->rx_pool
[scrq_num
].available
) <
2748 adapter
->req_rx_add_entries_per_subcrq
/ 2) ||
2749 frames_processed
< budget
))
2750 replenish_rx_pool(adapter
, &adapter
->rx_pool
[scrq_num
]);
2751 if (frames_processed
< budget
) {
2752 if (napi_complete_done(napi
, frames_processed
)) {
2753 enable_scrq_irq(adapter
, rx_scrq
);
2754 if (pending_scrq(adapter
, rx_scrq
)) {
2755 if (napi_reschedule(napi
)) {
2756 disable_scrq_irq(adapter
, rx_scrq
);
2762 return frames_processed
;
2765 static int wait_for_reset(struct ibmvnic_adapter
*adapter
)
2769 adapter
->fallback
.mtu
= adapter
->req_mtu
;
2770 adapter
->fallback
.rx_queues
= adapter
->req_rx_queues
;
2771 adapter
->fallback
.tx_queues
= adapter
->req_tx_queues
;
2772 adapter
->fallback
.rx_entries
= adapter
->req_rx_add_entries_per_subcrq
;
2773 adapter
->fallback
.tx_entries
= adapter
->req_tx_entries_per_subcrq
;
2775 reinit_completion(&adapter
->reset_done
);
2776 adapter
->wait_for_reset
= true;
2777 rc
= ibmvnic_reset(adapter
, VNIC_RESET_CHANGE_PARAM
);
2783 rc
= ibmvnic_wait_for_completion(adapter
, &adapter
->reset_done
, 60000);
2790 if (adapter
->reset_done_rc
) {
2792 adapter
->desired
.mtu
= adapter
->fallback
.mtu
;
2793 adapter
->desired
.rx_queues
= adapter
->fallback
.rx_queues
;
2794 adapter
->desired
.tx_queues
= adapter
->fallback
.tx_queues
;
2795 adapter
->desired
.rx_entries
= adapter
->fallback
.rx_entries
;
2796 adapter
->desired
.tx_entries
= adapter
->fallback
.tx_entries
;
2798 reinit_completion(&adapter
->reset_done
);
2799 adapter
->wait_for_reset
= true;
2800 rc
= ibmvnic_reset(adapter
, VNIC_RESET_CHANGE_PARAM
);
2805 rc
= ibmvnic_wait_for_completion(adapter
, &adapter
->reset_done
,
2813 adapter
->wait_for_reset
= false;
2818 static int ibmvnic_change_mtu(struct net_device
*netdev
, int new_mtu
)
2820 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
2822 adapter
->desired
.mtu
= new_mtu
+ ETH_HLEN
;
2824 return wait_for_reset(adapter
);
2827 static netdev_features_t
ibmvnic_features_check(struct sk_buff
*skb
,
2828 struct net_device
*dev
,
2829 netdev_features_t features
)
2831 /* Some backing hardware adapters can not
2832 * handle packets with a MSS less than 224
2833 * or with only one segment.
2835 if (skb_is_gso(skb
)) {
2836 if (skb_shinfo(skb
)->gso_size
< 224 ||
2837 skb_shinfo(skb
)->gso_segs
== 1)
2838 features
&= ~NETIF_F_GSO_MASK
;
2844 static const struct net_device_ops ibmvnic_netdev_ops
= {
2845 .ndo_open
= ibmvnic_open
,
2846 .ndo_stop
= ibmvnic_close
,
2847 .ndo_start_xmit
= ibmvnic_xmit
,
2848 .ndo_set_rx_mode
= ibmvnic_set_multi
,
2849 .ndo_set_mac_address
= ibmvnic_set_mac
,
2850 .ndo_validate_addr
= eth_validate_addr
,
2851 .ndo_tx_timeout
= ibmvnic_tx_timeout
,
2852 .ndo_change_mtu
= ibmvnic_change_mtu
,
2853 .ndo_features_check
= ibmvnic_features_check
,
2856 /* ethtool functions */
2858 static int ibmvnic_get_link_ksettings(struct net_device
*netdev
,
2859 struct ethtool_link_ksettings
*cmd
)
2861 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
2864 rc
= send_query_phys_parms(adapter
);
2866 adapter
->speed
= SPEED_UNKNOWN
;
2867 adapter
->duplex
= DUPLEX_UNKNOWN
;
2869 cmd
->base
.speed
= adapter
->speed
;
2870 cmd
->base
.duplex
= adapter
->duplex
;
2871 cmd
->base
.port
= PORT_FIBRE
;
2872 cmd
->base
.phy_address
= 0;
2873 cmd
->base
.autoneg
= AUTONEG_ENABLE
;
2878 static void ibmvnic_get_drvinfo(struct net_device
*netdev
,
2879 struct ethtool_drvinfo
*info
)
2881 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
2883 strscpy(info
->driver
, ibmvnic_driver_name
, sizeof(info
->driver
));
2884 strscpy(info
->version
, IBMVNIC_DRIVER_VERSION
, sizeof(info
->version
));
2885 strscpy(info
->fw_version
, adapter
->fw_version
,
2886 sizeof(info
->fw_version
));
2889 static u32
ibmvnic_get_msglevel(struct net_device
*netdev
)
2891 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
2893 return adapter
->msg_enable
;
2896 static void ibmvnic_set_msglevel(struct net_device
*netdev
, u32 data
)
2898 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
2900 adapter
->msg_enable
= data
;
2903 static u32
ibmvnic_get_link(struct net_device
*netdev
)
2905 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
2907 /* Don't need to send a query because we request a logical link up at
2908 * init and then we wait for link state indications
2910 return adapter
->logical_link_state
;
2913 static void ibmvnic_get_ringparam(struct net_device
*netdev
,
2914 struct ethtool_ringparam
*ring
)
2916 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
2918 if (adapter
->priv_flags
& IBMVNIC_USE_SERVER_MAXES
) {
2919 ring
->rx_max_pending
= adapter
->max_rx_add_entries_per_subcrq
;
2920 ring
->tx_max_pending
= adapter
->max_tx_entries_per_subcrq
;
2922 ring
->rx_max_pending
= IBMVNIC_MAX_QUEUE_SZ
;
2923 ring
->tx_max_pending
= IBMVNIC_MAX_QUEUE_SZ
;
2925 ring
->rx_mini_max_pending
= 0;
2926 ring
->rx_jumbo_max_pending
= 0;
2927 ring
->rx_pending
= adapter
->req_rx_add_entries_per_subcrq
;
2928 ring
->tx_pending
= adapter
->req_tx_entries_per_subcrq
;
2929 ring
->rx_mini_pending
= 0;
2930 ring
->rx_jumbo_pending
= 0;
2933 static int ibmvnic_set_ringparam(struct net_device
*netdev
,
2934 struct ethtool_ringparam
*ring
)
2936 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
2940 adapter
->desired
.rx_entries
= ring
->rx_pending
;
2941 adapter
->desired
.tx_entries
= ring
->tx_pending
;
2943 ret
= wait_for_reset(adapter
);
2946 (adapter
->req_rx_add_entries_per_subcrq
!= ring
->rx_pending
||
2947 adapter
->req_tx_entries_per_subcrq
!= ring
->tx_pending
))
2949 "Could not match full ringsize request. Requested: RX %d, TX %d; Allowed: RX %llu, TX %llu\n",
2950 ring
->rx_pending
, ring
->tx_pending
,
2951 adapter
->req_rx_add_entries_per_subcrq
,
2952 adapter
->req_tx_entries_per_subcrq
);
2956 static void ibmvnic_get_channels(struct net_device
*netdev
,
2957 struct ethtool_channels
*channels
)
2959 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
2961 if (adapter
->priv_flags
& IBMVNIC_USE_SERVER_MAXES
) {
2962 channels
->max_rx
= adapter
->max_rx_queues
;
2963 channels
->max_tx
= adapter
->max_tx_queues
;
2965 channels
->max_rx
= IBMVNIC_MAX_QUEUES
;
2966 channels
->max_tx
= IBMVNIC_MAX_QUEUES
;
2969 channels
->max_other
= 0;
2970 channels
->max_combined
= 0;
2971 channels
->rx_count
= adapter
->req_rx_queues
;
2972 channels
->tx_count
= adapter
->req_tx_queues
;
2973 channels
->other_count
= 0;
2974 channels
->combined_count
= 0;
2977 static int ibmvnic_set_channels(struct net_device
*netdev
,
2978 struct ethtool_channels
*channels
)
2980 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
2984 adapter
->desired
.rx_queues
= channels
->rx_count
;
2985 adapter
->desired
.tx_queues
= channels
->tx_count
;
2987 ret
= wait_for_reset(adapter
);
2990 (adapter
->req_rx_queues
!= channels
->rx_count
||
2991 adapter
->req_tx_queues
!= channels
->tx_count
))
2993 "Could not match full channels request. Requested: RX %d, TX %d; Allowed: RX %llu, TX %llu\n",
2994 channels
->rx_count
, channels
->tx_count
,
2995 adapter
->req_rx_queues
, adapter
->req_tx_queues
);
2999 static void ibmvnic_get_strings(struct net_device
*dev
, u32 stringset
, u8
*data
)
3001 struct ibmvnic_adapter
*adapter
= netdev_priv(dev
);
3004 switch (stringset
) {
3006 for (i
= 0; i
< ARRAY_SIZE(ibmvnic_stats
);
3007 i
++, data
+= ETH_GSTRING_LEN
)
3008 memcpy(data
, ibmvnic_stats
[i
].name
, ETH_GSTRING_LEN
);
3010 for (i
= 0; i
< adapter
->req_tx_queues
; i
++) {
3011 snprintf(data
, ETH_GSTRING_LEN
, "tx%d_packets", i
);
3012 data
+= ETH_GSTRING_LEN
;
3014 snprintf(data
, ETH_GSTRING_LEN
, "tx%d_bytes", i
);
3015 data
+= ETH_GSTRING_LEN
;
3017 snprintf(data
, ETH_GSTRING_LEN
,
3018 "tx%d_dropped_packets", i
);
3019 data
+= ETH_GSTRING_LEN
;
3022 for (i
= 0; i
< adapter
->req_rx_queues
; i
++) {
3023 snprintf(data
, ETH_GSTRING_LEN
, "rx%d_packets", i
);
3024 data
+= ETH_GSTRING_LEN
;
3026 snprintf(data
, ETH_GSTRING_LEN
, "rx%d_bytes", i
);
3027 data
+= ETH_GSTRING_LEN
;
3029 snprintf(data
, ETH_GSTRING_LEN
, "rx%d_interrupts", i
);
3030 data
+= ETH_GSTRING_LEN
;
3034 case ETH_SS_PRIV_FLAGS
:
3035 for (i
= 0; i
< ARRAY_SIZE(ibmvnic_priv_flags
); i
++)
3036 strcpy(data
+ i
* ETH_GSTRING_LEN
,
3037 ibmvnic_priv_flags
[i
]);
3044 static int ibmvnic_get_sset_count(struct net_device
*dev
, int sset
)
3046 struct ibmvnic_adapter
*adapter
= netdev_priv(dev
);
3050 return ARRAY_SIZE(ibmvnic_stats
) +
3051 adapter
->req_tx_queues
* NUM_TX_STATS
+
3052 adapter
->req_rx_queues
* NUM_RX_STATS
;
3053 case ETH_SS_PRIV_FLAGS
:
3054 return ARRAY_SIZE(ibmvnic_priv_flags
);
3060 static void ibmvnic_get_ethtool_stats(struct net_device
*dev
,
3061 struct ethtool_stats
*stats
, u64
*data
)
3063 struct ibmvnic_adapter
*adapter
= netdev_priv(dev
);
3064 union ibmvnic_crq crq
;
3068 memset(&crq
, 0, sizeof(crq
));
3069 crq
.request_statistics
.first
= IBMVNIC_CRQ_CMD
;
3070 crq
.request_statistics
.cmd
= REQUEST_STATISTICS
;
3071 crq
.request_statistics
.ioba
= cpu_to_be32(adapter
->stats_token
);
3072 crq
.request_statistics
.len
=
3073 cpu_to_be32(sizeof(struct ibmvnic_statistics
));
3075 /* Wait for data to be written */
3076 reinit_completion(&adapter
->stats_done
);
3077 rc
= ibmvnic_send_crq(adapter
, &crq
);
3080 rc
= ibmvnic_wait_for_completion(adapter
, &adapter
->stats_done
, 10000);
3084 for (i
= 0; i
< ARRAY_SIZE(ibmvnic_stats
); i
++)
3085 data
[i
] = be64_to_cpu(IBMVNIC_GET_STAT
3086 (adapter
, ibmvnic_stats
[i
].offset
));
3088 for (j
= 0; j
< adapter
->req_tx_queues
; j
++) {
3089 data
[i
] = adapter
->tx_stats_buffers
[j
].packets
;
3091 data
[i
] = adapter
->tx_stats_buffers
[j
].bytes
;
3093 data
[i
] = adapter
->tx_stats_buffers
[j
].dropped_packets
;
3097 for (j
= 0; j
< adapter
->req_rx_queues
; j
++) {
3098 data
[i
] = adapter
->rx_stats_buffers
[j
].packets
;
3100 data
[i
] = adapter
->rx_stats_buffers
[j
].bytes
;
3102 data
[i
] = adapter
->rx_stats_buffers
[j
].interrupts
;
3107 static u32
ibmvnic_get_priv_flags(struct net_device
*netdev
)
3109 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
3111 return adapter
->priv_flags
;
3114 static int ibmvnic_set_priv_flags(struct net_device
*netdev
, u32 flags
)
3116 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
3117 bool which_maxes
= !!(flags
& IBMVNIC_USE_SERVER_MAXES
);
3120 adapter
->priv_flags
|= IBMVNIC_USE_SERVER_MAXES
;
3122 adapter
->priv_flags
&= ~IBMVNIC_USE_SERVER_MAXES
;
3127 static const struct ethtool_ops ibmvnic_ethtool_ops
= {
3128 .get_drvinfo
= ibmvnic_get_drvinfo
,
3129 .get_msglevel
= ibmvnic_get_msglevel
,
3130 .set_msglevel
= ibmvnic_set_msglevel
,
3131 .get_link
= ibmvnic_get_link
,
3132 .get_ringparam
= ibmvnic_get_ringparam
,
3133 .set_ringparam
= ibmvnic_set_ringparam
,
3134 .get_channels
= ibmvnic_get_channels
,
3135 .set_channels
= ibmvnic_set_channels
,
3136 .get_strings
= ibmvnic_get_strings
,
3137 .get_sset_count
= ibmvnic_get_sset_count
,
3138 .get_ethtool_stats
= ibmvnic_get_ethtool_stats
,
3139 .get_link_ksettings
= ibmvnic_get_link_ksettings
,
3140 .get_priv_flags
= ibmvnic_get_priv_flags
,
3141 .set_priv_flags
= ibmvnic_set_priv_flags
,
3144 /* Routines for managing CRQs/sCRQs */
3146 static int reset_one_sub_crq_queue(struct ibmvnic_adapter
*adapter
,
3147 struct ibmvnic_sub_crq_queue
*scrq
)
3152 netdev_dbg(adapter
->netdev
, "Invalid scrq reset.\n");
3157 free_irq(scrq
->irq
, scrq
);
3158 irq_dispose_mapping(scrq
->irq
);
3163 memset(scrq
->msgs
, 0, 4 * PAGE_SIZE
);
3164 atomic_set(&scrq
->used
, 0);
3166 scrq
->ind_buf
.index
= 0;
3168 netdev_dbg(adapter
->netdev
, "Invalid scrq reset\n");
3172 rc
= h_reg_sub_crq(adapter
->vdev
->unit_address
, scrq
->msg_token
,
3173 4 * PAGE_SIZE
, &scrq
->crq_num
, &scrq
->hw_irq
);
3177 static int reset_sub_crq_queues(struct ibmvnic_adapter
*adapter
)
3181 if (!adapter
->tx_scrq
|| !adapter
->rx_scrq
)
3184 for (i
= 0; i
< adapter
->req_tx_queues
; i
++) {
3185 netdev_dbg(adapter
->netdev
, "Re-setting tx_scrq[%d]\n", i
);
3186 rc
= reset_one_sub_crq_queue(adapter
, adapter
->tx_scrq
[i
]);
3191 for (i
= 0; i
< adapter
->req_rx_queues
; i
++) {
3192 netdev_dbg(adapter
->netdev
, "Re-setting rx_scrq[%d]\n", i
);
3193 rc
= reset_one_sub_crq_queue(adapter
, adapter
->rx_scrq
[i
]);
3201 static void release_sub_crq_queue(struct ibmvnic_adapter
*adapter
,
3202 struct ibmvnic_sub_crq_queue
*scrq
,
3205 struct device
*dev
= &adapter
->vdev
->dev
;
3208 netdev_dbg(adapter
->netdev
, "Releasing sub-CRQ\n");
3211 /* Close the sub-crqs */
3213 rc
= plpar_hcall_norets(H_FREE_SUB_CRQ
,
3214 adapter
->vdev
->unit_address
,
3216 } while (rc
== H_BUSY
|| H_IS_LONG_BUSY(rc
));
3219 netdev_err(adapter
->netdev
,
3220 "Failed to release sub-CRQ %16lx, rc = %ld\n",
3225 dma_free_coherent(dev
,
3227 scrq
->ind_buf
.indir_arr
,
3228 scrq
->ind_buf
.indir_dma
);
3230 dma_unmap_single(dev
, scrq
->msg_token
, 4 * PAGE_SIZE
,
3232 free_pages((unsigned long)scrq
->msgs
, 2);
3236 static struct ibmvnic_sub_crq_queue
*init_sub_crq_queue(struct ibmvnic_adapter
3239 struct device
*dev
= &adapter
->vdev
->dev
;
3240 struct ibmvnic_sub_crq_queue
*scrq
;
3243 scrq
= kzalloc(sizeof(*scrq
), GFP_KERNEL
);
3248 (union sub_crq
*)__get_free_pages(GFP_KERNEL
| __GFP_ZERO
, 2);
3250 dev_warn(dev
, "Couldn't allocate crq queue messages page\n");
3251 goto zero_page_failed
;
3254 scrq
->msg_token
= dma_map_single(dev
, scrq
->msgs
, 4 * PAGE_SIZE
,
3256 if (dma_mapping_error(dev
, scrq
->msg_token
)) {
3257 dev_warn(dev
, "Couldn't map crq queue messages page\n");
3261 rc
= h_reg_sub_crq(adapter
->vdev
->unit_address
, scrq
->msg_token
,
3262 4 * PAGE_SIZE
, &scrq
->crq_num
, &scrq
->hw_irq
);
3264 if (rc
== H_RESOURCE
)
3265 rc
= ibmvnic_reset_crq(adapter
);
3267 if (rc
== H_CLOSED
) {
3268 dev_warn(dev
, "Partner adapter not ready, waiting.\n");
3270 dev_warn(dev
, "Error %d registering sub-crq\n", rc
);
3274 scrq
->adapter
= adapter
;
3275 scrq
->size
= 4 * PAGE_SIZE
/ sizeof(*scrq
->msgs
);
3276 scrq
->ind_buf
.index
= 0;
3278 scrq
->ind_buf
.indir_arr
=
3279 dma_alloc_coherent(dev
,
3281 &scrq
->ind_buf
.indir_dma
,
3284 if (!scrq
->ind_buf
.indir_arr
)
3287 spin_lock_init(&scrq
->lock
);
3289 netdev_dbg(adapter
->netdev
,
3290 "sub-crq initialized, num %lx, hw_irq=%lx, irq=%x\n",
3291 scrq
->crq_num
, scrq
->hw_irq
, scrq
->irq
);
3297 rc
= plpar_hcall_norets(H_FREE_SUB_CRQ
,
3298 adapter
->vdev
->unit_address
,
3300 } while (rc
== H_BUSY
|| rc
== H_IS_LONG_BUSY(rc
));
3302 dma_unmap_single(dev
, scrq
->msg_token
, 4 * PAGE_SIZE
,
3305 free_pages((unsigned long)scrq
->msgs
, 2);
3312 static void release_sub_crqs(struct ibmvnic_adapter
*adapter
, bool do_h_free
)
3316 if (adapter
->tx_scrq
) {
3317 for (i
= 0; i
< adapter
->num_active_tx_scrqs
; i
++) {
3318 if (!adapter
->tx_scrq
[i
])
3321 netdev_dbg(adapter
->netdev
, "Releasing tx_scrq[%d]\n",
3323 ibmvnic_tx_scrq_clean_buffer(adapter
, adapter
->tx_scrq
[i
]);
3324 if (adapter
->tx_scrq
[i
]->irq
) {
3325 free_irq(adapter
->tx_scrq
[i
]->irq
,
3326 adapter
->tx_scrq
[i
]);
3327 irq_dispose_mapping(adapter
->tx_scrq
[i
]->irq
);
3328 adapter
->tx_scrq
[i
]->irq
= 0;
3331 release_sub_crq_queue(adapter
, adapter
->tx_scrq
[i
],
3335 kfree(adapter
->tx_scrq
);
3336 adapter
->tx_scrq
= NULL
;
3337 adapter
->num_active_tx_scrqs
= 0;
3340 if (adapter
->rx_scrq
) {
3341 for (i
= 0; i
< adapter
->num_active_rx_scrqs
; i
++) {
3342 if (!adapter
->rx_scrq
[i
])
3345 netdev_dbg(adapter
->netdev
, "Releasing rx_scrq[%d]\n",
3347 if (adapter
->rx_scrq
[i
]->irq
) {
3348 free_irq(adapter
->rx_scrq
[i
]->irq
,
3349 adapter
->rx_scrq
[i
]);
3350 irq_dispose_mapping(adapter
->rx_scrq
[i
]->irq
);
3351 adapter
->rx_scrq
[i
]->irq
= 0;
3354 release_sub_crq_queue(adapter
, adapter
->rx_scrq
[i
],
3358 kfree(adapter
->rx_scrq
);
3359 adapter
->rx_scrq
= NULL
;
3360 adapter
->num_active_rx_scrqs
= 0;
3364 static int disable_scrq_irq(struct ibmvnic_adapter
*adapter
,
3365 struct ibmvnic_sub_crq_queue
*scrq
)
3367 struct device
*dev
= &adapter
->vdev
->dev
;
3370 rc
= plpar_hcall_norets(H_VIOCTL
, adapter
->vdev
->unit_address
,
3371 H_DISABLE_VIO_INTERRUPT
, scrq
->hw_irq
, 0, 0);
3373 dev_err(dev
, "Couldn't disable scrq irq 0x%lx. rc=%ld\n",
3378 static int enable_scrq_irq(struct ibmvnic_adapter
*adapter
,
3379 struct ibmvnic_sub_crq_queue
*scrq
)
3381 struct device
*dev
= &adapter
->vdev
->dev
;
3384 if (scrq
->hw_irq
> 0x100000000ULL
) {
3385 dev_err(dev
, "bad hw_irq = %lx\n", scrq
->hw_irq
);
3389 if (test_bit(0, &adapter
->resetting
) &&
3390 adapter
->reset_reason
== VNIC_RESET_MOBILITY
) {
3391 u64 val
= (0xff000000) | scrq
->hw_irq
;
3393 rc
= plpar_hcall_norets(H_EOI
, val
);
3394 /* H_EOI would fail with rc = H_FUNCTION when running
3395 * in XIVE mode which is expected, but not an error.
3397 if (rc
&& (rc
!= H_FUNCTION
))
3398 dev_err(dev
, "H_EOI FAILED irq 0x%llx. rc=%ld\n",
3402 rc
= plpar_hcall_norets(H_VIOCTL
, adapter
->vdev
->unit_address
,
3403 H_ENABLE_VIO_INTERRUPT
, scrq
->hw_irq
, 0, 0);
3405 dev_err(dev
, "Couldn't enable scrq irq 0x%lx. rc=%ld\n",
3410 static int ibmvnic_complete_tx(struct ibmvnic_adapter
*adapter
,
3411 struct ibmvnic_sub_crq_queue
*scrq
)
3413 struct device
*dev
= &adapter
->vdev
->dev
;
3414 struct ibmvnic_tx_pool
*tx_pool
;
3415 struct ibmvnic_tx_buff
*txbuff
;
3416 struct netdev_queue
*txq
;
3417 union sub_crq
*next
;
3422 while (pending_scrq(adapter
, scrq
)) {
3423 unsigned int pool
= scrq
->pool_index
;
3424 int num_entries
= 0;
3425 int total_bytes
= 0;
3426 int num_packets
= 0;
3428 next
= ibmvnic_next_scrq(adapter
, scrq
);
3429 for (i
= 0; i
< next
->tx_comp
.num_comps
; i
++) {
3430 index
= be32_to_cpu(next
->tx_comp
.correlators
[i
]);
3431 if (index
& IBMVNIC_TSO_POOL_MASK
) {
3432 tx_pool
= &adapter
->tso_pool
[pool
];
3433 index
&= ~IBMVNIC_TSO_POOL_MASK
;
3435 tx_pool
= &adapter
->tx_pool
[pool
];
3438 txbuff
= &tx_pool
->tx_buff
[index
];
3440 num_entries
+= txbuff
->num_entries
;
3442 total_bytes
+= txbuff
->skb
->len
;
3443 if (next
->tx_comp
.rcs
[i
]) {
3444 dev_err(dev
, "tx error %x\n",
3445 next
->tx_comp
.rcs
[i
]);
3446 dev_kfree_skb_irq(txbuff
->skb
);
3448 dev_consume_skb_irq(txbuff
->skb
);
3452 netdev_warn(adapter
->netdev
,
3453 "TX completion received with NULL socket buffer\n");
3455 tx_pool
->free_map
[tx_pool
->producer_index
] = index
;
3456 tx_pool
->producer_index
=
3457 (tx_pool
->producer_index
+ 1) %
3458 tx_pool
->num_buffers
;
3460 /* remove tx_comp scrq*/
3461 next
->tx_comp
.first
= 0;
3463 txq
= netdev_get_tx_queue(adapter
->netdev
, scrq
->pool_index
);
3464 netdev_tx_completed_queue(txq
, num_packets
, total_bytes
);
3466 if (atomic_sub_return(num_entries
, &scrq
->used
) <=
3467 (adapter
->req_tx_entries_per_subcrq
/ 2) &&
3468 __netif_subqueue_stopped(adapter
->netdev
,
3469 scrq
->pool_index
)) {
3470 netif_wake_subqueue(adapter
->netdev
, scrq
->pool_index
);
3471 netdev_dbg(adapter
->netdev
, "Started queue %d\n",
3476 enable_scrq_irq(adapter
, scrq
);
3478 if (pending_scrq(adapter
, scrq
)) {
3479 disable_scrq_irq(adapter
, scrq
);
3486 static irqreturn_t
ibmvnic_interrupt_tx(int irq
, void *instance
)
3488 struct ibmvnic_sub_crq_queue
*scrq
= instance
;
3489 struct ibmvnic_adapter
*adapter
= scrq
->adapter
;
3491 disable_scrq_irq(adapter
, scrq
);
3492 ibmvnic_complete_tx(adapter
, scrq
);
3497 static irqreturn_t
ibmvnic_interrupt_rx(int irq
, void *instance
)
3499 struct ibmvnic_sub_crq_queue
*scrq
= instance
;
3500 struct ibmvnic_adapter
*adapter
= scrq
->adapter
;
3502 /* When booting a kdump kernel we can hit pending interrupts
3503 * prior to completing driver initialization.
3505 if (unlikely(adapter
->state
!= VNIC_OPEN
))
3508 adapter
->rx_stats_buffers
[scrq
->scrq_num
].interrupts
++;
3510 if (napi_schedule_prep(&adapter
->napi
[scrq
->scrq_num
])) {
3511 disable_scrq_irq(adapter
, scrq
);
3512 __napi_schedule(&adapter
->napi
[scrq
->scrq_num
]);
3518 static int init_sub_crq_irqs(struct ibmvnic_adapter
*adapter
)
3520 struct device
*dev
= &adapter
->vdev
->dev
;
3521 struct ibmvnic_sub_crq_queue
*scrq
;
3525 for (i
= 0; i
< adapter
->req_tx_queues
; i
++) {
3526 netdev_dbg(adapter
->netdev
, "Initializing tx_scrq[%d] irq\n",
3528 scrq
= adapter
->tx_scrq
[i
];
3529 scrq
->irq
= irq_create_mapping(NULL
, scrq
->hw_irq
);
3533 dev_err(dev
, "Error mapping irq\n");
3534 goto req_tx_irq_failed
;
3537 snprintf(scrq
->name
, sizeof(scrq
->name
), "ibmvnic-%x-tx%d",
3538 adapter
->vdev
->unit_address
, i
);
3539 rc
= request_irq(scrq
->irq
, ibmvnic_interrupt_tx
,
3540 0, scrq
->name
, scrq
);
3543 dev_err(dev
, "Couldn't register tx irq 0x%x. rc=%d\n",
3545 irq_dispose_mapping(scrq
->irq
);
3546 goto req_tx_irq_failed
;
3550 for (i
= 0; i
< adapter
->req_rx_queues
; i
++) {
3551 netdev_dbg(adapter
->netdev
, "Initializing rx_scrq[%d] irq\n",
3553 scrq
= adapter
->rx_scrq
[i
];
3554 scrq
->irq
= irq_create_mapping(NULL
, scrq
->hw_irq
);
3557 dev_err(dev
, "Error mapping irq\n");
3558 goto req_rx_irq_failed
;
3560 snprintf(scrq
->name
, sizeof(scrq
->name
), "ibmvnic-%x-rx%d",
3561 adapter
->vdev
->unit_address
, i
);
3562 rc
= request_irq(scrq
->irq
, ibmvnic_interrupt_rx
,
3563 0, scrq
->name
, scrq
);
3565 dev_err(dev
, "Couldn't register rx irq 0x%x. rc=%d\n",
3567 irq_dispose_mapping(scrq
->irq
);
3568 goto req_rx_irq_failed
;
3574 for (j
= 0; j
< i
; j
++) {
3575 free_irq(adapter
->rx_scrq
[j
]->irq
, adapter
->rx_scrq
[j
]);
3576 irq_dispose_mapping(adapter
->rx_scrq
[j
]->irq
);
3578 i
= adapter
->req_tx_queues
;
3580 for (j
= 0; j
< i
; j
++) {
3581 free_irq(adapter
->tx_scrq
[j
]->irq
, adapter
->tx_scrq
[j
]);
3582 irq_dispose_mapping(adapter
->tx_scrq
[j
]->irq
);
3584 release_sub_crqs(adapter
, 1);
3588 static int init_sub_crqs(struct ibmvnic_adapter
*adapter
)
3590 struct device
*dev
= &adapter
->vdev
->dev
;
3591 struct ibmvnic_sub_crq_queue
**allqueues
;
3592 int registered_queues
= 0;
3597 total_queues
= adapter
->req_tx_queues
+ adapter
->req_rx_queues
;
3599 allqueues
= kcalloc(total_queues
, sizeof(*allqueues
), GFP_KERNEL
);
3603 for (i
= 0; i
< total_queues
; i
++) {
3604 allqueues
[i
] = init_sub_crq_queue(adapter
);
3605 if (!allqueues
[i
]) {
3606 dev_warn(dev
, "Couldn't allocate all sub-crqs\n");
3609 registered_queues
++;
3612 /* Make sure we were able to register the minimum number of queues */
3613 if (registered_queues
<
3614 adapter
->min_tx_queues
+ adapter
->min_rx_queues
) {
3615 dev_err(dev
, "Fatal: Couldn't init min number of sub-crqs\n");
3619 /* Distribute the failed allocated queues*/
3620 for (i
= 0; i
< total_queues
- registered_queues
+ more
; i
++) {
3621 netdev_dbg(adapter
->netdev
, "Reducing number of queues\n");
3624 if (adapter
->req_rx_queues
> adapter
->min_rx_queues
)
3625 adapter
->req_rx_queues
--;
3630 if (adapter
->req_tx_queues
> adapter
->min_tx_queues
)
3631 adapter
->req_tx_queues
--;
3638 adapter
->tx_scrq
= kcalloc(adapter
->req_tx_queues
,
3639 sizeof(*adapter
->tx_scrq
), GFP_KERNEL
);
3640 if (!adapter
->tx_scrq
)
3643 for (i
= 0; i
< adapter
->req_tx_queues
; i
++) {
3644 adapter
->tx_scrq
[i
] = allqueues
[i
];
3645 adapter
->tx_scrq
[i
]->pool_index
= i
;
3646 adapter
->num_active_tx_scrqs
++;
3649 adapter
->rx_scrq
= kcalloc(adapter
->req_rx_queues
,
3650 sizeof(*adapter
->rx_scrq
), GFP_KERNEL
);
3651 if (!adapter
->rx_scrq
)
3654 for (i
= 0; i
< adapter
->req_rx_queues
; i
++) {
3655 adapter
->rx_scrq
[i
] = allqueues
[i
+ adapter
->req_tx_queues
];
3656 adapter
->rx_scrq
[i
]->scrq_num
= i
;
3657 adapter
->num_active_rx_scrqs
++;
3664 kfree(adapter
->tx_scrq
);
3665 adapter
->tx_scrq
= NULL
;
3667 for (i
= 0; i
< registered_queues
; i
++)
3668 release_sub_crq_queue(adapter
, allqueues
[i
], 1);
3673 static void send_request_cap(struct ibmvnic_adapter
*adapter
, int retry
)
3675 struct device
*dev
= &adapter
->vdev
->dev
;
3676 union ibmvnic_crq crq
;
3680 /* Sub-CRQ entries are 32 byte long */
3681 int entries_page
= 4 * PAGE_SIZE
/ (sizeof(u64
) * 4);
3683 if (adapter
->min_tx_entries_per_subcrq
> entries_page
||
3684 adapter
->min_rx_add_entries_per_subcrq
> entries_page
) {
3685 dev_err(dev
, "Fatal, invalid entries per sub-crq\n");
3689 if (adapter
->desired
.mtu
)
3690 adapter
->req_mtu
= adapter
->desired
.mtu
;
3692 adapter
->req_mtu
= adapter
->netdev
->mtu
+ ETH_HLEN
;
3694 if (!adapter
->desired
.tx_entries
)
3695 adapter
->desired
.tx_entries
=
3696 adapter
->max_tx_entries_per_subcrq
;
3697 if (!adapter
->desired
.rx_entries
)
3698 adapter
->desired
.rx_entries
=
3699 adapter
->max_rx_add_entries_per_subcrq
;
3701 max_entries
= IBMVNIC_MAX_LTB_SIZE
/
3702 (adapter
->req_mtu
+ IBMVNIC_BUFFER_HLEN
);
3704 if ((adapter
->req_mtu
+ IBMVNIC_BUFFER_HLEN
) *
3705 adapter
->desired
.tx_entries
> IBMVNIC_MAX_LTB_SIZE
) {
3706 adapter
->desired
.tx_entries
= max_entries
;
3709 if ((adapter
->req_mtu
+ IBMVNIC_BUFFER_HLEN
) *
3710 adapter
->desired
.rx_entries
> IBMVNIC_MAX_LTB_SIZE
) {
3711 adapter
->desired
.rx_entries
= max_entries
;
3714 if (adapter
->desired
.tx_entries
)
3715 adapter
->req_tx_entries_per_subcrq
=
3716 adapter
->desired
.tx_entries
;
3718 adapter
->req_tx_entries_per_subcrq
=
3719 adapter
->max_tx_entries_per_subcrq
;
3721 if (adapter
->desired
.rx_entries
)
3722 adapter
->req_rx_add_entries_per_subcrq
=
3723 adapter
->desired
.rx_entries
;
3725 adapter
->req_rx_add_entries_per_subcrq
=
3726 adapter
->max_rx_add_entries_per_subcrq
;
3728 if (adapter
->desired
.tx_queues
)
3729 adapter
->req_tx_queues
=
3730 adapter
->desired
.tx_queues
;
3732 adapter
->req_tx_queues
=
3733 adapter
->opt_tx_comp_sub_queues
;
3735 if (adapter
->desired
.rx_queues
)
3736 adapter
->req_rx_queues
=
3737 adapter
->desired
.rx_queues
;
3739 adapter
->req_rx_queues
=
3740 adapter
->opt_rx_comp_queues
;
3742 adapter
->req_rx_add_queues
= adapter
->max_rx_add_queues
;
3745 memset(&crq
, 0, sizeof(crq
));
3746 crq
.request_capability
.first
= IBMVNIC_CRQ_CMD
;
3747 crq
.request_capability
.cmd
= REQUEST_CAPABILITY
;
3749 crq
.request_capability
.capability
= cpu_to_be16(REQ_TX_QUEUES
);
3750 crq
.request_capability
.number
= cpu_to_be64(adapter
->req_tx_queues
);
3751 atomic_inc(&adapter
->running_cap_crqs
);
3752 ibmvnic_send_crq(adapter
, &crq
);
3754 crq
.request_capability
.capability
= cpu_to_be16(REQ_RX_QUEUES
);
3755 crq
.request_capability
.number
= cpu_to_be64(adapter
->req_rx_queues
);
3756 atomic_inc(&adapter
->running_cap_crqs
);
3757 ibmvnic_send_crq(adapter
, &crq
);
3759 crq
.request_capability
.capability
= cpu_to_be16(REQ_RX_ADD_QUEUES
);
3760 crq
.request_capability
.number
= cpu_to_be64(adapter
->req_rx_add_queues
);
3761 atomic_inc(&adapter
->running_cap_crqs
);
3762 ibmvnic_send_crq(adapter
, &crq
);
3764 crq
.request_capability
.capability
=
3765 cpu_to_be16(REQ_TX_ENTRIES_PER_SUBCRQ
);
3766 crq
.request_capability
.number
=
3767 cpu_to_be64(adapter
->req_tx_entries_per_subcrq
);
3768 atomic_inc(&adapter
->running_cap_crqs
);
3769 ibmvnic_send_crq(adapter
, &crq
);
3771 crq
.request_capability
.capability
=
3772 cpu_to_be16(REQ_RX_ADD_ENTRIES_PER_SUBCRQ
);
3773 crq
.request_capability
.number
=
3774 cpu_to_be64(adapter
->req_rx_add_entries_per_subcrq
);
3775 atomic_inc(&adapter
->running_cap_crqs
);
3776 ibmvnic_send_crq(adapter
, &crq
);
3778 crq
.request_capability
.capability
= cpu_to_be16(REQ_MTU
);
3779 crq
.request_capability
.number
= cpu_to_be64(adapter
->req_mtu
);
3780 atomic_inc(&adapter
->running_cap_crqs
);
3781 ibmvnic_send_crq(adapter
, &crq
);
3783 if (adapter
->netdev
->flags
& IFF_PROMISC
) {
3784 if (adapter
->promisc_supported
) {
3785 crq
.request_capability
.capability
=
3786 cpu_to_be16(PROMISC_REQUESTED
);
3787 crq
.request_capability
.number
= cpu_to_be64(1);
3788 atomic_inc(&adapter
->running_cap_crqs
);
3789 ibmvnic_send_crq(adapter
, &crq
);
3792 crq
.request_capability
.capability
=
3793 cpu_to_be16(PROMISC_REQUESTED
);
3794 crq
.request_capability
.number
= cpu_to_be64(0);
3795 atomic_inc(&adapter
->running_cap_crqs
);
3796 ibmvnic_send_crq(adapter
, &crq
);
3800 static int pending_scrq(struct ibmvnic_adapter
*adapter
,
3801 struct ibmvnic_sub_crq_queue
*scrq
)
3803 union sub_crq
*entry
= &scrq
->msgs
[scrq
->cur
];
3806 rc
= !!(entry
->generic
.first
& IBMVNIC_CRQ_CMD_RSP
);
3808 /* Ensure that the SCRQ valid flag is loaded prior to loading the
3809 * contents of the SCRQ descriptor
3816 static union sub_crq
*ibmvnic_next_scrq(struct ibmvnic_adapter
*adapter
,
3817 struct ibmvnic_sub_crq_queue
*scrq
)
3819 union sub_crq
*entry
;
3820 unsigned long flags
;
3822 spin_lock_irqsave(&scrq
->lock
, flags
);
3823 entry
= &scrq
->msgs
[scrq
->cur
];
3824 if (entry
->generic
.first
& IBMVNIC_CRQ_CMD_RSP
) {
3825 if (++scrq
->cur
== scrq
->size
)
3830 spin_unlock_irqrestore(&scrq
->lock
, flags
);
3832 /* Ensure that the SCRQ valid flag is loaded prior to loading the
3833 * contents of the SCRQ descriptor
3840 static union ibmvnic_crq
*ibmvnic_next_crq(struct ibmvnic_adapter
*adapter
)
3842 struct ibmvnic_crq_queue
*queue
= &adapter
->crq
;
3843 union ibmvnic_crq
*crq
;
3845 crq
= &queue
->msgs
[queue
->cur
];
3846 if (crq
->generic
.first
& IBMVNIC_CRQ_CMD_RSP
) {
3847 if (++queue
->cur
== queue
->size
)
3856 static void print_subcrq_error(struct device
*dev
, int rc
, const char *func
)
3860 dev_warn_ratelimited(dev
,
3861 "%s failed: Send request is malformed or adapter failover pending. (rc=%d)\n",
3865 dev_warn_ratelimited(dev
,
3866 "%s failed: Backing queue closed. Adapter is down or failover pending. (rc=%d)\n",
3870 dev_err_ratelimited(dev
, "%s failed: (rc=%d)\n", func
, rc
);
3875 static int send_subcrq_indirect(struct ibmvnic_adapter
*adapter
,
3876 u64 remote_handle
, u64 ioba
, u64 num_entries
)
3878 unsigned int ua
= adapter
->vdev
->unit_address
;
3879 struct device
*dev
= &adapter
->vdev
->dev
;
3882 /* Make sure the hypervisor sees the complete request */
3884 rc
= plpar_hcall_norets(H_SEND_SUB_CRQ_INDIRECT
, ua
,
3885 cpu_to_be64(remote_handle
),
3889 print_subcrq_error(dev
, rc
, __func__
);
3894 static int ibmvnic_send_crq(struct ibmvnic_adapter
*adapter
,
3895 union ibmvnic_crq
*crq
)
3897 unsigned int ua
= adapter
->vdev
->unit_address
;
3898 struct device
*dev
= &adapter
->vdev
->dev
;
3899 u64
*u64_crq
= (u64
*)crq
;
3902 netdev_dbg(adapter
->netdev
, "Sending CRQ: %016lx %016lx\n",
3903 (unsigned long)cpu_to_be64(u64_crq
[0]),
3904 (unsigned long)cpu_to_be64(u64_crq
[1]));
3906 if (!adapter
->crq
.active
&&
3907 crq
->generic
.first
!= IBMVNIC_CRQ_INIT_CMD
) {
3908 dev_warn(dev
, "Invalid request detected while CRQ is inactive, possible device state change during reset\n");
3912 /* Make sure the hypervisor sees the complete request */
3915 rc
= plpar_hcall_norets(H_SEND_CRQ
, ua
,
3916 cpu_to_be64(u64_crq
[0]),
3917 cpu_to_be64(u64_crq
[1]));
3920 if (rc
== H_CLOSED
) {
3921 dev_warn(dev
, "CRQ Queue closed\n");
3922 /* do not reset, report the fail, wait for passive init from server */
3925 dev_warn(dev
, "Send error (rc=%d)\n", rc
);
3931 static int ibmvnic_send_crq_init(struct ibmvnic_adapter
*adapter
)
3933 struct device
*dev
= &adapter
->vdev
->dev
;
3934 union ibmvnic_crq crq
;
3938 memset(&crq
, 0, sizeof(crq
));
3939 crq
.generic
.first
= IBMVNIC_CRQ_INIT_CMD
;
3940 crq
.generic
.cmd
= IBMVNIC_CRQ_INIT
;
3941 netdev_dbg(adapter
->netdev
, "Sending CRQ init\n");
3944 rc
= ibmvnic_send_crq(adapter
, &crq
);
3950 } while (retries
> 0);
3953 dev_err(dev
, "Failed to send init request, rc = %d\n", rc
);
3960 struct vnic_login_client_data
{
3966 static int vnic_client_data_len(struct ibmvnic_adapter
*adapter
)
3970 /* Calculate the amount of buffer space needed for the
3971 * vnic client data in the login buffer. There are four entries,
3972 * OS name, LPAR name, device name, and a null last entry.
3974 len
= 4 * sizeof(struct vnic_login_client_data
);
3975 len
+= 6; /* "Linux" plus NULL */
3976 len
+= strlen(utsname()->nodename
) + 1;
3977 len
+= strlen(adapter
->netdev
->name
) + 1;
3982 static void vnic_add_client_data(struct ibmvnic_adapter
*adapter
,
3983 struct vnic_login_client_data
*vlcd
)
3985 const char *os_name
= "Linux";
3988 /* Type 1 - LPAR OS */
3990 len
= strlen(os_name
) + 1;
3991 vlcd
->len
= cpu_to_be16(len
);
3992 strscpy(vlcd
->name
, os_name
, len
);
3993 vlcd
= (struct vnic_login_client_data
*)(vlcd
->name
+ len
);
3995 /* Type 2 - LPAR name */
3997 len
= strlen(utsname()->nodename
) + 1;
3998 vlcd
->len
= cpu_to_be16(len
);
3999 strscpy(vlcd
->name
, utsname()->nodename
, len
);
4000 vlcd
= (struct vnic_login_client_data
*)(vlcd
->name
+ len
);
4002 /* Type 3 - device name */
4004 len
= strlen(adapter
->netdev
->name
) + 1;
4005 vlcd
->len
= cpu_to_be16(len
);
4006 strscpy(vlcd
->name
, adapter
->netdev
->name
, len
);
4009 static int send_login(struct ibmvnic_adapter
*adapter
)
4011 struct ibmvnic_login_rsp_buffer
*login_rsp_buffer
;
4012 struct ibmvnic_login_buffer
*login_buffer
;
4013 struct device
*dev
= &adapter
->vdev
->dev
;
4014 struct vnic_login_client_data
*vlcd
;
4015 dma_addr_t rsp_buffer_token
;
4016 dma_addr_t buffer_token
;
4017 size_t rsp_buffer_size
;
4018 union ibmvnic_crq crq
;
4019 int client_data_len
;
4026 if (!adapter
->tx_scrq
|| !adapter
->rx_scrq
) {
4027 netdev_err(adapter
->netdev
,
4028 "RX or TX queues are not allocated, device login failed\n");
4032 release_login_buffer(adapter
);
4033 release_login_rsp_buffer(adapter
);
4035 client_data_len
= vnic_client_data_len(adapter
);
4038 sizeof(struct ibmvnic_login_buffer
) +
4039 sizeof(u64
) * (adapter
->req_tx_queues
+ adapter
->req_rx_queues
) +
4042 login_buffer
= kzalloc(buffer_size
, GFP_ATOMIC
);
4044 goto buf_alloc_failed
;
4046 buffer_token
= dma_map_single(dev
, login_buffer
, buffer_size
,
4048 if (dma_mapping_error(dev
, buffer_token
)) {
4049 dev_err(dev
, "Couldn't map login buffer\n");
4050 goto buf_map_failed
;
4053 rsp_buffer_size
= sizeof(struct ibmvnic_login_rsp_buffer
) +
4054 sizeof(u64
) * adapter
->req_tx_queues
+
4055 sizeof(u64
) * adapter
->req_rx_queues
+
4056 sizeof(u64
) * adapter
->req_rx_queues
+
4057 sizeof(u8
) * IBMVNIC_TX_DESC_VERSIONS
;
4059 login_rsp_buffer
= kmalloc(rsp_buffer_size
, GFP_ATOMIC
);
4060 if (!login_rsp_buffer
)
4061 goto buf_rsp_alloc_failed
;
4063 rsp_buffer_token
= dma_map_single(dev
, login_rsp_buffer
,
4064 rsp_buffer_size
, DMA_FROM_DEVICE
);
4065 if (dma_mapping_error(dev
, rsp_buffer_token
)) {
4066 dev_err(dev
, "Couldn't map login rsp buffer\n");
4067 goto buf_rsp_map_failed
;
4070 adapter
->login_buf
= login_buffer
;
4071 adapter
->login_buf_token
= buffer_token
;
4072 adapter
->login_buf_sz
= buffer_size
;
4073 adapter
->login_rsp_buf
= login_rsp_buffer
;
4074 adapter
->login_rsp_buf_token
= rsp_buffer_token
;
4075 adapter
->login_rsp_buf_sz
= rsp_buffer_size
;
4077 login_buffer
->len
= cpu_to_be32(buffer_size
);
4078 login_buffer
->version
= cpu_to_be32(INITIAL_VERSION_LB
);
4079 login_buffer
->num_txcomp_subcrqs
= cpu_to_be32(adapter
->req_tx_queues
);
4080 login_buffer
->off_txcomp_subcrqs
=
4081 cpu_to_be32(sizeof(struct ibmvnic_login_buffer
));
4082 login_buffer
->num_rxcomp_subcrqs
= cpu_to_be32(adapter
->req_rx_queues
);
4083 login_buffer
->off_rxcomp_subcrqs
=
4084 cpu_to_be32(sizeof(struct ibmvnic_login_buffer
) +
4085 sizeof(u64
) * adapter
->req_tx_queues
);
4086 login_buffer
->login_rsp_ioba
= cpu_to_be32(rsp_buffer_token
);
4087 login_buffer
->login_rsp_len
= cpu_to_be32(rsp_buffer_size
);
4089 tx_list_p
= (__be64
*)((char *)login_buffer
+
4090 sizeof(struct ibmvnic_login_buffer
));
4091 rx_list_p
= (__be64
*)((char *)login_buffer
+
4092 sizeof(struct ibmvnic_login_buffer
) +
4093 sizeof(u64
) * adapter
->req_tx_queues
);
4095 for (i
= 0; i
< adapter
->req_tx_queues
; i
++) {
4096 if (adapter
->tx_scrq
[i
]) {
4098 cpu_to_be64(adapter
->tx_scrq
[i
]->crq_num
);
4102 for (i
= 0; i
< adapter
->req_rx_queues
; i
++) {
4103 if (adapter
->rx_scrq
[i
]) {
4105 cpu_to_be64(adapter
->rx_scrq
[i
]->crq_num
);
4109 /* Insert vNIC login client data */
4110 vlcd
= (struct vnic_login_client_data
*)
4111 ((char *)rx_list_p
+ (sizeof(u64
) * adapter
->req_rx_queues
));
4112 login_buffer
->client_data_offset
=
4113 cpu_to_be32((char *)vlcd
- (char *)login_buffer
);
4114 login_buffer
->client_data_len
= cpu_to_be32(client_data_len
);
4116 vnic_add_client_data(adapter
, vlcd
);
4118 netdev_dbg(adapter
->netdev
, "Login Buffer:\n");
4119 for (i
= 0; i
< (adapter
->login_buf_sz
- 1) / 8 + 1; i
++) {
4120 netdev_dbg(adapter
->netdev
, "%016lx\n",
4121 ((unsigned long *)(adapter
->login_buf
))[i
]);
4124 memset(&crq
, 0, sizeof(crq
));
4125 crq
.login
.first
= IBMVNIC_CRQ_CMD
;
4126 crq
.login
.cmd
= LOGIN
;
4127 crq
.login
.ioba
= cpu_to_be32(buffer_token
);
4128 crq
.login
.len
= cpu_to_be32(buffer_size
);
4130 adapter
->login_pending
= true;
4131 rc
= ibmvnic_send_crq(adapter
, &crq
);
4133 adapter
->login_pending
= false;
4134 netdev_err(adapter
->netdev
, "Failed to send login, rc=%d\n", rc
);
4135 goto buf_rsp_map_failed
;
4141 kfree(login_rsp_buffer
);
4142 adapter
->login_rsp_buf
= NULL
;
4143 buf_rsp_alloc_failed
:
4144 dma_unmap_single(dev
, buffer_token
, buffer_size
, DMA_TO_DEVICE
);
4146 kfree(login_buffer
);
4147 adapter
->login_buf
= NULL
;
4152 static int send_request_map(struct ibmvnic_adapter
*adapter
, dma_addr_t addr
,
4155 union ibmvnic_crq crq
;
4157 memset(&crq
, 0, sizeof(crq
));
4158 crq
.request_map
.first
= IBMVNIC_CRQ_CMD
;
4159 crq
.request_map
.cmd
= REQUEST_MAP
;
4160 crq
.request_map
.map_id
= map_id
;
4161 crq
.request_map
.ioba
= cpu_to_be32(addr
);
4162 crq
.request_map
.len
= cpu_to_be32(len
);
4163 return ibmvnic_send_crq(adapter
, &crq
);
4166 static int send_request_unmap(struct ibmvnic_adapter
*adapter
, u8 map_id
)
4168 union ibmvnic_crq crq
;
4170 memset(&crq
, 0, sizeof(crq
));
4171 crq
.request_unmap
.first
= IBMVNIC_CRQ_CMD
;
4172 crq
.request_unmap
.cmd
= REQUEST_UNMAP
;
4173 crq
.request_unmap
.map_id
= map_id
;
4174 return ibmvnic_send_crq(adapter
, &crq
);
4177 static void send_query_map(struct ibmvnic_adapter
*adapter
)
4179 union ibmvnic_crq crq
;
4181 memset(&crq
, 0, sizeof(crq
));
4182 crq
.query_map
.first
= IBMVNIC_CRQ_CMD
;
4183 crq
.query_map
.cmd
= QUERY_MAP
;
4184 ibmvnic_send_crq(adapter
, &crq
);
4187 /* Send a series of CRQs requesting various capabilities of the VNIC server */
4188 static void send_query_cap(struct ibmvnic_adapter
*adapter
)
4190 union ibmvnic_crq crq
;
4192 atomic_set(&adapter
->running_cap_crqs
, 0);
4193 memset(&crq
, 0, sizeof(crq
));
4194 crq
.query_capability
.first
= IBMVNIC_CRQ_CMD
;
4195 crq
.query_capability
.cmd
= QUERY_CAPABILITY
;
4197 crq
.query_capability
.capability
= cpu_to_be16(MIN_TX_QUEUES
);
4198 atomic_inc(&adapter
->running_cap_crqs
);
4199 ibmvnic_send_crq(adapter
, &crq
);
4201 crq
.query_capability
.capability
= cpu_to_be16(MIN_RX_QUEUES
);
4202 atomic_inc(&adapter
->running_cap_crqs
);
4203 ibmvnic_send_crq(adapter
, &crq
);
4205 crq
.query_capability
.capability
= cpu_to_be16(MIN_RX_ADD_QUEUES
);
4206 atomic_inc(&adapter
->running_cap_crqs
);
4207 ibmvnic_send_crq(adapter
, &crq
);
4209 crq
.query_capability
.capability
= cpu_to_be16(MAX_TX_QUEUES
);
4210 atomic_inc(&adapter
->running_cap_crqs
);
4211 ibmvnic_send_crq(adapter
, &crq
);
4213 crq
.query_capability
.capability
= cpu_to_be16(MAX_RX_QUEUES
);
4214 atomic_inc(&adapter
->running_cap_crqs
);
4215 ibmvnic_send_crq(adapter
, &crq
);
4217 crq
.query_capability
.capability
= cpu_to_be16(MAX_RX_ADD_QUEUES
);
4218 atomic_inc(&adapter
->running_cap_crqs
);
4219 ibmvnic_send_crq(adapter
, &crq
);
4221 crq
.query_capability
.capability
=
4222 cpu_to_be16(MIN_TX_ENTRIES_PER_SUBCRQ
);
4223 atomic_inc(&adapter
->running_cap_crqs
);
4224 ibmvnic_send_crq(adapter
, &crq
);
4226 crq
.query_capability
.capability
=
4227 cpu_to_be16(MIN_RX_ADD_ENTRIES_PER_SUBCRQ
);
4228 atomic_inc(&adapter
->running_cap_crqs
);
4229 ibmvnic_send_crq(adapter
, &crq
);
4231 crq
.query_capability
.capability
=
4232 cpu_to_be16(MAX_TX_ENTRIES_PER_SUBCRQ
);
4233 atomic_inc(&adapter
->running_cap_crqs
);
4234 ibmvnic_send_crq(adapter
, &crq
);
4236 crq
.query_capability
.capability
=
4237 cpu_to_be16(MAX_RX_ADD_ENTRIES_PER_SUBCRQ
);
4238 atomic_inc(&adapter
->running_cap_crqs
);
4239 ibmvnic_send_crq(adapter
, &crq
);
4241 crq
.query_capability
.capability
= cpu_to_be16(TCP_IP_OFFLOAD
);
4242 atomic_inc(&adapter
->running_cap_crqs
);
4243 ibmvnic_send_crq(adapter
, &crq
);
4245 crq
.query_capability
.capability
= cpu_to_be16(PROMISC_SUPPORTED
);
4246 atomic_inc(&adapter
->running_cap_crqs
);
4247 ibmvnic_send_crq(adapter
, &crq
);
4249 crq
.query_capability
.capability
= cpu_to_be16(MIN_MTU
);
4250 atomic_inc(&adapter
->running_cap_crqs
);
4251 ibmvnic_send_crq(adapter
, &crq
);
4253 crq
.query_capability
.capability
= cpu_to_be16(MAX_MTU
);
4254 atomic_inc(&adapter
->running_cap_crqs
);
4255 ibmvnic_send_crq(adapter
, &crq
);
4257 crq
.query_capability
.capability
= cpu_to_be16(MAX_MULTICAST_FILTERS
);
4258 atomic_inc(&adapter
->running_cap_crqs
);
4259 ibmvnic_send_crq(adapter
, &crq
);
4261 crq
.query_capability
.capability
= cpu_to_be16(VLAN_HEADER_INSERTION
);
4262 atomic_inc(&adapter
->running_cap_crqs
);
4263 ibmvnic_send_crq(adapter
, &crq
);
4265 crq
.query_capability
.capability
= cpu_to_be16(RX_VLAN_HEADER_INSERTION
);
4266 atomic_inc(&adapter
->running_cap_crqs
);
4267 ibmvnic_send_crq(adapter
, &crq
);
4269 crq
.query_capability
.capability
= cpu_to_be16(MAX_TX_SG_ENTRIES
);
4270 atomic_inc(&adapter
->running_cap_crqs
);
4271 ibmvnic_send_crq(adapter
, &crq
);
4273 crq
.query_capability
.capability
= cpu_to_be16(RX_SG_SUPPORTED
);
4274 atomic_inc(&adapter
->running_cap_crqs
);
4275 ibmvnic_send_crq(adapter
, &crq
);
4277 crq
.query_capability
.capability
= cpu_to_be16(OPT_TX_COMP_SUB_QUEUES
);
4278 atomic_inc(&adapter
->running_cap_crqs
);
4279 ibmvnic_send_crq(adapter
, &crq
);
4281 crq
.query_capability
.capability
= cpu_to_be16(OPT_RX_COMP_QUEUES
);
4282 atomic_inc(&adapter
->running_cap_crqs
);
4283 ibmvnic_send_crq(adapter
, &crq
);
4285 crq
.query_capability
.capability
=
4286 cpu_to_be16(OPT_RX_BUFADD_Q_PER_RX_COMP_Q
);
4287 atomic_inc(&adapter
->running_cap_crqs
);
4288 ibmvnic_send_crq(adapter
, &crq
);
4290 crq
.query_capability
.capability
=
4291 cpu_to_be16(OPT_TX_ENTRIES_PER_SUBCRQ
);
4292 atomic_inc(&adapter
->running_cap_crqs
);
4293 ibmvnic_send_crq(adapter
, &crq
);
4295 crq
.query_capability
.capability
=
4296 cpu_to_be16(OPT_RXBA_ENTRIES_PER_SUBCRQ
);
4297 atomic_inc(&adapter
->running_cap_crqs
);
4298 ibmvnic_send_crq(adapter
, &crq
);
4300 crq
.query_capability
.capability
= cpu_to_be16(TX_RX_DESC_REQ
);
4301 atomic_inc(&adapter
->running_cap_crqs
);
4302 ibmvnic_send_crq(adapter
, &crq
);
4305 static void send_query_ip_offload(struct ibmvnic_adapter
*adapter
)
4307 int buf_sz
= sizeof(struct ibmvnic_query_ip_offload_buffer
);
4308 struct device
*dev
= &adapter
->vdev
->dev
;
4309 union ibmvnic_crq crq
;
4311 adapter
->ip_offload_tok
=
4313 &adapter
->ip_offload_buf
,
4317 if (dma_mapping_error(dev
, adapter
->ip_offload_tok
)) {
4318 if (!firmware_has_feature(FW_FEATURE_CMO
))
4319 dev_err(dev
, "Couldn't map offload buffer\n");
4323 memset(&crq
, 0, sizeof(crq
));
4324 crq
.query_ip_offload
.first
= IBMVNIC_CRQ_CMD
;
4325 crq
.query_ip_offload
.cmd
= QUERY_IP_OFFLOAD
;
4326 crq
.query_ip_offload
.len
= cpu_to_be32(buf_sz
);
4327 crq
.query_ip_offload
.ioba
=
4328 cpu_to_be32(adapter
->ip_offload_tok
);
4330 ibmvnic_send_crq(adapter
, &crq
);
4333 static void send_control_ip_offload(struct ibmvnic_adapter
*adapter
)
4335 struct ibmvnic_control_ip_offload_buffer
*ctrl_buf
= &adapter
->ip_offload_ctrl
;
4336 struct ibmvnic_query_ip_offload_buffer
*buf
= &adapter
->ip_offload_buf
;
4337 struct device
*dev
= &adapter
->vdev
->dev
;
4338 netdev_features_t old_hw_features
= 0;
4339 union ibmvnic_crq crq
;
4341 adapter
->ip_offload_ctrl_tok
=
4344 sizeof(adapter
->ip_offload_ctrl
),
4347 if (dma_mapping_error(dev
, adapter
->ip_offload_ctrl_tok
)) {
4348 dev_err(dev
, "Couldn't map ip offload control buffer\n");
4352 ctrl_buf
->len
= cpu_to_be32(sizeof(adapter
->ip_offload_ctrl
));
4353 ctrl_buf
->version
= cpu_to_be32(INITIAL_VERSION_IOB
);
4354 ctrl_buf
->ipv4_chksum
= buf
->ipv4_chksum
;
4355 ctrl_buf
->ipv6_chksum
= buf
->ipv6_chksum
;
4356 ctrl_buf
->tcp_ipv4_chksum
= buf
->tcp_ipv4_chksum
;
4357 ctrl_buf
->udp_ipv4_chksum
= buf
->udp_ipv4_chksum
;
4358 ctrl_buf
->tcp_ipv6_chksum
= buf
->tcp_ipv6_chksum
;
4359 ctrl_buf
->udp_ipv6_chksum
= buf
->udp_ipv6_chksum
;
4360 ctrl_buf
->large_tx_ipv4
= buf
->large_tx_ipv4
;
4361 ctrl_buf
->large_tx_ipv6
= buf
->large_tx_ipv6
;
4363 /* large_rx disabled for now, additional features needed */
4364 ctrl_buf
->large_rx_ipv4
= 0;
4365 ctrl_buf
->large_rx_ipv6
= 0;
4367 if (adapter
->state
!= VNIC_PROBING
) {
4368 old_hw_features
= adapter
->netdev
->hw_features
;
4369 adapter
->netdev
->hw_features
= 0;
4372 adapter
->netdev
->hw_features
= NETIF_F_SG
| NETIF_F_GSO
| NETIF_F_GRO
;
4374 if (buf
->tcp_ipv4_chksum
|| buf
->udp_ipv4_chksum
)
4375 adapter
->netdev
->hw_features
|= NETIF_F_IP_CSUM
;
4377 if (buf
->tcp_ipv6_chksum
|| buf
->udp_ipv6_chksum
)
4378 adapter
->netdev
->hw_features
|= NETIF_F_IPV6_CSUM
;
4380 if ((adapter
->netdev
->features
&
4381 (NETIF_F_IP_CSUM
| NETIF_F_IPV6_CSUM
)))
4382 adapter
->netdev
->hw_features
|= NETIF_F_RXCSUM
;
4384 if (buf
->large_tx_ipv4
)
4385 adapter
->netdev
->hw_features
|= NETIF_F_TSO
;
4386 if (buf
->large_tx_ipv6
)
4387 adapter
->netdev
->hw_features
|= NETIF_F_TSO6
;
4389 if (adapter
->state
== VNIC_PROBING
) {
4390 adapter
->netdev
->features
|= adapter
->netdev
->hw_features
;
4391 } else if (old_hw_features
!= adapter
->netdev
->hw_features
) {
4392 netdev_features_t tmp
= 0;
4394 /* disable features no longer supported */
4395 adapter
->netdev
->features
&= adapter
->netdev
->hw_features
;
4396 /* turn on features now supported if previously enabled */
4397 tmp
= (old_hw_features
^ adapter
->netdev
->hw_features
) &
4398 adapter
->netdev
->hw_features
;
4399 adapter
->netdev
->features
|=
4400 tmp
& adapter
->netdev
->wanted_features
;
4403 memset(&crq
, 0, sizeof(crq
));
4404 crq
.control_ip_offload
.first
= IBMVNIC_CRQ_CMD
;
4405 crq
.control_ip_offload
.cmd
= CONTROL_IP_OFFLOAD
;
4406 crq
.control_ip_offload
.len
=
4407 cpu_to_be32(sizeof(adapter
->ip_offload_ctrl
));
4408 crq
.control_ip_offload
.ioba
= cpu_to_be32(adapter
->ip_offload_ctrl_tok
);
4409 ibmvnic_send_crq(adapter
, &crq
);
4412 static void handle_vpd_size_rsp(union ibmvnic_crq
*crq
,
4413 struct ibmvnic_adapter
*adapter
)
4415 struct device
*dev
= &adapter
->vdev
->dev
;
4417 if (crq
->get_vpd_size_rsp
.rc
.code
) {
4418 dev_err(dev
, "Error retrieving VPD size, rc=%x\n",
4419 crq
->get_vpd_size_rsp
.rc
.code
);
4420 complete(&adapter
->fw_done
);
4424 adapter
->vpd
->len
= be64_to_cpu(crq
->get_vpd_size_rsp
.len
);
4425 complete(&adapter
->fw_done
);
4428 static void handle_vpd_rsp(union ibmvnic_crq
*crq
,
4429 struct ibmvnic_adapter
*adapter
)
4431 struct device
*dev
= &adapter
->vdev
->dev
;
4432 unsigned char *substr
= NULL
;
4433 u8 fw_level_len
= 0;
4435 memset(adapter
->fw_version
, 0, 32);
4437 dma_unmap_single(dev
, adapter
->vpd
->dma_addr
, adapter
->vpd
->len
,
4440 if (crq
->get_vpd_rsp
.rc
.code
) {
4441 dev_err(dev
, "Error retrieving VPD from device, rc=%x\n",
4442 crq
->get_vpd_rsp
.rc
.code
);
4446 /* get the position of the firmware version info
4447 * located after the ASCII 'RM' substring in the buffer
4449 substr
= strnstr(adapter
->vpd
->buff
, "RM", adapter
->vpd
->len
);
4451 dev_info(dev
, "Warning - No FW level has been provided in the VPD buffer by the VIOS Server\n");
4455 /* get length of firmware level ASCII substring */
4456 if ((substr
+ 2) < (adapter
->vpd
->buff
+ adapter
->vpd
->len
)) {
4457 fw_level_len
= *(substr
+ 2);
4459 dev_info(dev
, "Length of FW substr extrapolated VDP buff\n");
4463 /* copy firmware version string from vpd into adapter */
4464 if ((substr
+ 3 + fw_level_len
) <
4465 (adapter
->vpd
->buff
+ adapter
->vpd
->len
)) {
4466 strncpy((char *)adapter
->fw_version
, substr
+ 3, fw_level_len
);
4468 dev_info(dev
, "FW substr extrapolated VPD buff\n");
4472 if (adapter
->fw_version
[0] == '\0')
4473 strscpy((char *)adapter
->fw_version
, "N/A", sizeof(adapter
->fw_version
));
4474 complete(&adapter
->fw_done
);
4477 static void handle_query_ip_offload_rsp(struct ibmvnic_adapter
*adapter
)
4479 struct device
*dev
= &adapter
->vdev
->dev
;
4480 struct ibmvnic_query_ip_offload_buffer
*buf
= &adapter
->ip_offload_buf
;
4483 dma_unmap_single(dev
, adapter
->ip_offload_tok
,
4484 sizeof(adapter
->ip_offload_buf
), DMA_FROM_DEVICE
);
4486 netdev_dbg(adapter
->netdev
, "Query IP Offload Buffer:\n");
4487 for (i
= 0; i
< (sizeof(adapter
->ip_offload_buf
) - 1) / 8 + 1; i
++)
4488 netdev_dbg(adapter
->netdev
, "%016lx\n",
4489 ((unsigned long *)(buf
))[i
]);
4491 netdev_dbg(adapter
->netdev
, "ipv4_chksum = %d\n", buf
->ipv4_chksum
);
4492 netdev_dbg(adapter
->netdev
, "ipv6_chksum = %d\n", buf
->ipv6_chksum
);
4493 netdev_dbg(adapter
->netdev
, "tcp_ipv4_chksum = %d\n",
4494 buf
->tcp_ipv4_chksum
);
4495 netdev_dbg(adapter
->netdev
, "tcp_ipv6_chksum = %d\n",
4496 buf
->tcp_ipv6_chksum
);
4497 netdev_dbg(adapter
->netdev
, "udp_ipv4_chksum = %d\n",
4498 buf
->udp_ipv4_chksum
);
4499 netdev_dbg(adapter
->netdev
, "udp_ipv6_chksum = %d\n",
4500 buf
->udp_ipv6_chksum
);
4501 netdev_dbg(adapter
->netdev
, "large_tx_ipv4 = %d\n",
4502 buf
->large_tx_ipv4
);
4503 netdev_dbg(adapter
->netdev
, "large_tx_ipv6 = %d\n",
4504 buf
->large_tx_ipv6
);
4505 netdev_dbg(adapter
->netdev
, "large_rx_ipv4 = %d\n",
4506 buf
->large_rx_ipv4
);
4507 netdev_dbg(adapter
->netdev
, "large_rx_ipv6 = %d\n",
4508 buf
->large_rx_ipv6
);
4509 netdev_dbg(adapter
->netdev
, "max_ipv4_hdr_sz = %d\n",
4510 buf
->max_ipv4_header_size
);
4511 netdev_dbg(adapter
->netdev
, "max_ipv6_hdr_sz = %d\n",
4512 buf
->max_ipv6_header_size
);
4513 netdev_dbg(adapter
->netdev
, "max_tcp_hdr_size = %d\n",
4514 buf
->max_tcp_header_size
);
4515 netdev_dbg(adapter
->netdev
, "max_udp_hdr_size = %d\n",
4516 buf
->max_udp_header_size
);
4517 netdev_dbg(adapter
->netdev
, "max_large_tx_size = %d\n",
4518 buf
->max_large_tx_size
);
4519 netdev_dbg(adapter
->netdev
, "max_large_rx_size = %d\n",
4520 buf
->max_large_rx_size
);
4521 netdev_dbg(adapter
->netdev
, "ipv6_ext_hdr = %d\n",
4522 buf
->ipv6_extension_header
);
4523 netdev_dbg(adapter
->netdev
, "tcp_pseudosum_req = %d\n",
4524 buf
->tcp_pseudosum_req
);
4525 netdev_dbg(adapter
->netdev
, "num_ipv6_ext_hd = %d\n",
4526 buf
->num_ipv6_ext_headers
);
4527 netdev_dbg(adapter
->netdev
, "off_ipv6_ext_hd = %d\n",
4528 buf
->off_ipv6_ext_headers
);
4530 send_control_ip_offload(adapter
);
4533 static const char *ibmvnic_fw_err_cause(u16 cause
)
4536 case ADAPTER_PROBLEM
:
4537 return "adapter problem";
4539 return "bus problem";
4541 return "firmware problem";
4543 return "device driver problem";
4545 return "EEH recovery";
4547 return "firmware updated";
4549 return "low Memory";
4555 static void handle_error_indication(union ibmvnic_crq
*crq
,
4556 struct ibmvnic_adapter
*adapter
)
4558 struct device
*dev
= &adapter
->vdev
->dev
;
4561 cause
= be16_to_cpu(crq
->error_indication
.error_cause
);
4563 dev_warn_ratelimited(dev
,
4564 "Firmware reports %serror, cause: %s. Starting recovery...\n",
4565 crq
->error_indication
.flags
4566 & IBMVNIC_FATAL_ERROR
? "FATAL " : "",
4567 ibmvnic_fw_err_cause(cause
));
4569 if (crq
->error_indication
.flags
& IBMVNIC_FATAL_ERROR
)
4570 ibmvnic_reset(adapter
, VNIC_RESET_FATAL
);
4572 ibmvnic_reset(adapter
, VNIC_RESET_NON_FATAL
);
4575 static int handle_change_mac_rsp(union ibmvnic_crq
*crq
,
4576 struct ibmvnic_adapter
*adapter
)
4578 struct net_device
*netdev
= adapter
->netdev
;
4579 struct device
*dev
= &adapter
->vdev
->dev
;
4582 rc
= crq
->change_mac_addr_rsp
.rc
.code
;
4584 dev_err(dev
, "Error %ld in CHANGE_MAC_ADDR_RSP\n", rc
);
4587 /* crq->change_mac_addr.mac_addr is the requested one
4588 * crq->change_mac_addr_rsp.mac_addr is the returned valid one.
4590 ether_addr_copy(netdev
->dev_addr
,
4591 &crq
->change_mac_addr_rsp
.mac_addr
[0]);
4592 ether_addr_copy(adapter
->mac_addr
,
4593 &crq
->change_mac_addr_rsp
.mac_addr
[0]);
4595 complete(&adapter
->fw_done
);
4599 static void handle_request_cap_rsp(union ibmvnic_crq
*crq
,
4600 struct ibmvnic_adapter
*adapter
)
4602 struct device
*dev
= &adapter
->vdev
->dev
;
4606 atomic_dec(&adapter
->running_cap_crqs
);
4607 switch (be16_to_cpu(crq
->request_capability_rsp
.capability
)) {
4609 req_value
= &adapter
->req_tx_queues
;
4613 req_value
= &adapter
->req_rx_queues
;
4616 case REQ_RX_ADD_QUEUES
:
4617 req_value
= &adapter
->req_rx_add_queues
;
4620 case REQ_TX_ENTRIES_PER_SUBCRQ
:
4621 req_value
= &adapter
->req_tx_entries_per_subcrq
;
4622 name
= "tx_entries_per_subcrq";
4624 case REQ_RX_ADD_ENTRIES_PER_SUBCRQ
:
4625 req_value
= &adapter
->req_rx_add_entries_per_subcrq
;
4626 name
= "rx_add_entries_per_subcrq";
4629 req_value
= &adapter
->req_mtu
;
4632 case PROMISC_REQUESTED
:
4633 req_value
= &adapter
->promisc
;
4637 dev_err(dev
, "Got invalid cap request rsp %d\n",
4638 crq
->request_capability
.capability
);
4642 switch (crq
->request_capability_rsp
.rc
.code
) {
4645 case PARTIALSUCCESS
:
4646 dev_info(dev
, "req=%lld, rsp=%ld in %s queue, retrying.\n",
4648 (long)be64_to_cpu(crq
->request_capability_rsp
.number
),
4651 if (be16_to_cpu(crq
->request_capability_rsp
.capability
) ==
4653 pr_err("mtu of %llu is not supported. Reverting.\n",
4655 *req_value
= adapter
->fallback
.mtu
;
4658 be64_to_cpu(crq
->request_capability_rsp
.number
);
4661 send_request_cap(adapter
, 1);
4664 dev_err(dev
, "Error %d in request cap rsp\n",
4665 crq
->request_capability_rsp
.rc
.code
);
4669 /* Done receiving requested capabilities, query IP offload support */
4670 if (atomic_read(&adapter
->running_cap_crqs
) == 0) {
4671 adapter
->wait_capability
= false;
4672 send_query_ip_offload(adapter
);
4676 static int handle_login_rsp(union ibmvnic_crq
*login_rsp_crq
,
4677 struct ibmvnic_adapter
*adapter
)
4679 struct device
*dev
= &adapter
->vdev
->dev
;
4680 struct net_device
*netdev
= adapter
->netdev
;
4681 struct ibmvnic_login_rsp_buffer
*login_rsp
= adapter
->login_rsp_buf
;
4682 struct ibmvnic_login_buffer
*login
= adapter
->login_buf
;
4683 u64
*tx_handle_array
;
4684 u64
*rx_handle_array
;
4690 /* CHECK: Test/set of login_pending does not need to be atomic
4691 * because only ibmvnic_tasklet tests/clears this.
4693 if (!adapter
->login_pending
) {
4694 netdev_warn(netdev
, "Ignoring unexpected login response\n");
4697 adapter
->login_pending
= false;
4699 dma_unmap_single(dev
, adapter
->login_buf_token
, adapter
->login_buf_sz
,
4701 dma_unmap_single(dev
, adapter
->login_rsp_buf_token
,
4702 adapter
->login_rsp_buf_sz
, DMA_FROM_DEVICE
);
4704 /* If the number of queues requested can't be allocated by the
4705 * server, the login response will return with code 1. We will need
4706 * to resend the login buffer with fewer queues requested.
4708 if (login_rsp_crq
->generic
.rc
.code
) {
4709 adapter
->init_done_rc
= login_rsp_crq
->generic
.rc
.code
;
4710 complete(&adapter
->init_done
);
4714 if (adapter
->failover_pending
) {
4715 adapter
->init_done_rc
= -EAGAIN
;
4716 netdev_dbg(netdev
, "Failover pending, ignoring login response\n");
4717 complete(&adapter
->init_done
);
4718 /* login response buffer will be released on reset */
4722 netdev
->mtu
= adapter
->req_mtu
- ETH_HLEN
;
4724 netdev_dbg(adapter
->netdev
, "Login Response Buffer:\n");
4725 for (i
= 0; i
< (adapter
->login_rsp_buf_sz
- 1) / 8 + 1; i
++) {
4726 netdev_dbg(adapter
->netdev
, "%016lx\n",
4727 ((unsigned long *)(adapter
->login_rsp_buf
))[i
]);
4731 if (login
->num_txcomp_subcrqs
!= login_rsp
->num_txsubm_subcrqs
||
4732 (be32_to_cpu(login
->num_rxcomp_subcrqs
) *
4733 adapter
->req_rx_add_queues
!=
4734 be32_to_cpu(login_rsp
->num_rxadd_subcrqs
))) {
4735 dev_err(dev
, "FATAL: Inconsistent login and login rsp\n");
4736 ibmvnic_reset(adapter
, VNIC_RESET_FATAL
);
4739 size_array
= (u64
*)((u8
*)(adapter
->login_rsp_buf
) +
4740 be32_to_cpu(adapter
->login_rsp_buf
->off_rxadd_buff_size
));
4741 /* variable buffer sizes are not supported, so just read the
4744 adapter
->cur_rx_buf_sz
= be64_to_cpu(size_array
[0]);
4746 num_tx_pools
= be32_to_cpu(adapter
->login_rsp_buf
->num_txsubm_subcrqs
);
4747 num_rx_pools
= be32_to_cpu(adapter
->login_rsp_buf
->num_rxadd_subcrqs
);
4749 tx_handle_array
= (u64
*)((u8
*)(adapter
->login_rsp_buf
) +
4750 be32_to_cpu(adapter
->login_rsp_buf
->off_txsubm_subcrqs
));
4751 rx_handle_array
= (u64
*)((u8
*)(adapter
->login_rsp_buf
) +
4752 be32_to_cpu(adapter
->login_rsp_buf
->off_rxadd_subcrqs
));
4754 for (i
= 0; i
< num_tx_pools
; i
++)
4755 adapter
->tx_scrq
[i
]->handle
= tx_handle_array
[i
];
4757 for (i
= 0; i
< num_rx_pools
; i
++)
4758 adapter
->rx_scrq
[i
]->handle
= rx_handle_array
[i
];
4760 adapter
->num_active_tx_scrqs
= num_tx_pools
;
4761 adapter
->num_active_rx_scrqs
= num_rx_pools
;
4762 release_login_rsp_buffer(adapter
);
4763 release_login_buffer(adapter
);
4764 complete(&adapter
->init_done
);
4769 static void handle_request_unmap_rsp(union ibmvnic_crq
*crq
,
4770 struct ibmvnic_adapter
*adapter
)
4772 struct device
*dev
= &adapter
->vdev
->dev
;
4775 rc
= crq
->request_unmap_rsp
.rc
.code
;
4777 dev_err(dev
, "Error %ld in REQUEST_UNMAP_RSP\n", rc
);
4780 static void handle_query_map_rsp(union ibmvnic_crq
*crq
,
4781 struct ibmvnic_adapter
*adapter
)
4783 struct net_device
*netdev
= adapter
->netdev
;
4784 struct device
*dev
= &adapter
->vdev
->dev
;
4787 rc
= crq
->query_map_rsp
.rc
.code
;
4789 dev_err(dev
, "Error %ld in QUERY_MAP_RSP\n", rc
);
4792 netdev_dbg(netdev
, "page_size = %d\ntot_pages = %d\nfree_pages = %d\n",
4793 crq
->query_map_rsp
.page_size
, crq
->query_map_rsp
.tot_pages
,
4794 crq
->query_map_rsp
.free_pages
);
4797 static void handle_query_cap_rsp(union ibmvnic_crq
*crq
,
4798 struct ibmvnic_adapter
*adapter
)
4800 struct net_device
*netdev
= adapter
->netdev
;
4801 struct device
*dev
= &adapter
->vdev
->dev
;
4804 atomic_dec(&adapter
->running_cap_crqs
);
4805 netdev_dbg(netdev
, "Outstanding queries: %d\n",
4806 atomic_read(&adapter
->running_cap_crqs
));
4807 rc
= crq
->query_capability
.rc
.code
;
4809 dev_err(dev
, "Error %ld in QUERY_CAP_RSP\n", rc
);
4813 switch (be16_to_cpu(crq
->query_capability
.capability
)) {
4815 adapter
->min_tx_queues
=
4816 be64_to_cpu(crq
->query_capability
.number
);
4817 netdev_dbg(netdev
, "min_tx_queues = %lld\n",
4818 adapter
->min_tx_queues
);
4821 adapter
->min_rx_queues
=
4822 be64_to_cpu(crq
->query_capability
.number
);
4823 netdev_dbg(netdev
, "min_rx_queues = %lld\n",
4824 adapter
->min_rx_queues
);
4826 case MIN_RX_ADD_QUEUES
:
4827 adapter
->min_rx_add_queues
=
4828 be64_to_cpu(crq
->query_capability
.number
);
4829 netdev_dbg(netdev
, "min_rx_add_queues = %lld\n",
4830 adapter
->min_rx_add_queues
);
4833 adapter
->max_tx_queues
=
4834 be64_to_cpu(crq
->query_capability
.number
);
4835 netdev_dbg(netdev
, "max_tx_queues = %lld\n",
4836 adapter
->max_tx_queues
);
4839 adapter
->max_rx_queues
=
4840 be64_to_cpu(crq
->query_capability
.number
);
4841 netdev_dbg(netdev
, "max_rx_queues = %lld\n",
4842 adapter
->max_rx_queues
);
4844 case MAX_RX_ADD_QUEUES
:
4845 adapter
->max_rx_add_queues
=
4846 be64_to_cpu(crq
->query_capability
.number
);
4847 netdev_dbg(netdev
, "max_rx_add_queues = %lld\n",
4848 adapter
->max_rx_add_queues
);
4850 case MIN_TX_ENTRIES_PER_SUBCRQ
:
4851 adapter
->min_tx_entries_per_subcrq
=
4852 be64_to_cpu(crq
->query_capability
.number
);
4853 netdev_dbg(netdev
, "min_tx_entries_per_subcrq = %lld\n",
4854 adapter
->min_tx_entries_per_subcrq
);
4856 case MIN_RX_ADD_ENTRIES_PER_SUBCRQ
:
4857 adapter
->min_rx_add_entries_per_subcrq
=
4858 be64_to_cpu(crq
->query_capability
.number
);
4859 netdev_dbg(netdev
, "min_rx_add_entrs_per_subcrq = %lld\n",
4860 adapter
->min_rx_add_entries_per_subcrq
);
4862 case MAX_TX_ENTRIES_PER_SUBCRQ
:
4863 adapter
->max_tx_entries_per_subcrq
=
4864 be64_to_cpu(crq
->query_capability
.number
);
4865 netdev_dbg(netdev
, "max_tx_entries_per_subcrq = %lld\n",
4866 adapter
->max_tx_entries_per_subcrq
);
4868 case MAX_RX_ADD_ENTRIES_PER_SUBCRQ
:
4869 adapter
->max_rx_add_entries_per_subcrq
=
4870 be64_to_cpu(crq
->query_capability
.number
);
4871 netdev_dbg(netdev
, "max_rx_add_entrs_per_subcrq = %lld\n",
4872 adapter
->max_rx_add_entries_per_subcrq
);
4874 case TCP_IP_OFFLOAD
:
4875 adapter
->tcp_ip_offload
=
4876 be64_to_cpu(crq
->query_capability
.number
);
4877 netdev_dbg(netdev
, "tcp_ip_offload = %lld\n",
4878 adapter
->tcp_ip_offload
);
4880 case PROMISC_SUPPORTED
:
4881 adapter
->promisc_supported
=
4882 be64_to_cpu(crq
->query_capability
.number
);
4883 netdev_dbg(netdev
, "promisc_supported = %lld\n",
4884 adapter
->promisc_supported
);
4887 adapter
->min_mtu
= be64_to_cpu(crq
->query_capability
.number
);
4888 netdev
->min_mtu
= adapter
->min_mtu
- ETH_HLEN
;
4889 netdev_dbg(netdev
, "min_mtu = %lld\n", adapter
->min_mtu
);
4892 adapter
->max_mtu
= be64_to_cpu(crq
->query_capability
.number
);
4893 netdev
->max_mtu
= adapter
->max_mtu
- ETH_HLEN
;
4894 netdev_dbg(netdev
, "max_mtu = %lld\n", adapter
->max_mtu
);
4896 case MAX_MULTICAST_FILTERS
:
4897 adapter
->max_multicast_filters
=
4898 be64_to_cpu(crq
->query_capability
.number
);
4899 netdev_dbg(netdev
, "max_multicast_filters = %lld\n",
4900 adapter
->max_multicast_filters
);
4902 case VLAN_HEADER_INSERTION
:
4903 adapter
->vlan_header_insertion
=
4904 be64_to_cpu(crq
->query_capability
.number
);
4905 if (adapter
->vlan_header_insertion
)
4906 netdev
->features
|= NETIF_F_HW_VLAN_STAG_TX
;
4907 netdev_dbg(netdev
, "vlan_header_insertion = %lld\n",
4908 adapter
->vlan_header_insertion
);
4910 case RX_VLAN_HEADER_INSERTION
:
4911 adapter
->rx_vlan_header_insertion
=
4912 be64_to_cpu(crq
->query_capability
.number
);
4913 netdev_dbg(netdev
, "rx_vlan_header_insertion = %lld\n",
4914 adapter
->rx_vlan_header_insertion
);
4916 case MAX_TX_SG_ENTRIES
:
4917 adapter
->max_tx_sg_entries
=
4918 be64_to_cpu(crq
->query_capability
.number
);
4919 netdev_dbg(netdev
, "max_tx_sg_entries = %lld\n",
4920 adapter
->max_tx_sg_entries
);
4922 case RX_SG_SUPPORTED
:
4923 adapter
->rx_sg_supported
=
4924 be64_to_cpu(crq
->query_capability
.number
);
4925 netdev_dbg(netdev
, "rx_sg_supported = %lld\n",
4926 adapter
->rx_sg_supported
);
4928 case OPT_TX_COMP_SUB_QUEUES
:
4929 adapter
->opt_tx_comp_sub_queues
=
4930 be64_to_cpu(crq
->query_capability
.number
);
4931 netdev_dbg(netdev
, "opt_tx_comp_sub_queues = %lld\n",
4932 adapter
->opt_tx_comp_sub_queues
);
4934 case OPT_RX_COMP_QUEUES
:
4935 adapter
->opt_rx_comp_queues
=
4936 be64_to_cpu(crq
->query_capability
.number
);
4937 netdev_dbg(netdev
, "opt_rx_comp_queues = %lld\n",
4938 adapter
->opt_rx_comp_queues
);
4940 case OPT_RX_BUFADD_Q_PER_RX_COMP_Q
:
4941 adapter
->opt_rx_bufadd_q_per_rx_comp_q
=
4942 be64_to_cpu(crq
->query_capability
.number
);
4943 netdev_dbg(netdev
, "opt_rx_bufadd_q_per_rx_comp_q = %lld\n",
4944 adapter
->opt_rx_bufadd_q_per_rx_comp_q
);
4946 case OPT_TX_ENTRIES_PER_SUBCRQ
:
4947 adapter
->opt_tx_entries_per_subcrq
=
4948 be64_to_cpu(crq
->query_capability
.number
);
4949 netdev_dbg(netdev
, "opt_tx_entries_per_subcrq = %lld\n",
4950 adapter
->opt_tx_entries_per_subcrq
);
4952 case OPT_RXBA_ENTRIES_PER_SUBCRQ
:
4953 adapter
->opt_rxba_entries_per_subcrq
=
4954 be64_to_cpu(crq
->query_capability
.number
);
4955 netdev_dbg(netdev
, "opt_rxba_entries_per_subcrq = %lld\n",
4956 adapter
->opt_rxba_entries_per_subcrq
);
4958 case TX_RX_DESC_REQ
:
4959 adapter
->tx_rx_desc_req
= crq
->query_capability
.number
;
4960 netdev_dbg(netdev
, "tx_rx_desc_req = %llx\n",
4961 adapter
->tx_rx_desc_req
);
4965 netdev_err(netdev
, "Got invalid cap rsp %d\n",
4966 crq
->query_capability
.capability
);
4970 if (atomic_read(&adapter
->running_cap_crqs
) == 0) {
4971 adapter
->wait_capability
= false;
4972 send_request_cap(adapter
, 0);
4976 static int send_query_phys_parms(struct ibmvnic_adapter
*adapter
)
4978 union ibmvnic_crq crq
;
4981 memset(&crq
, 0, sizeof(crq
));
4982 crq
.query_phys_parms
.first
= IBMVNIC_CRQ_CMD
;
4983 crq
.query_phys_parms
.cmd
= QUERY_PHYS_PARMS
;
4985 mutex_lock(&adapter
->fw_lock
);
4986 adapter
->fw_done_rc
= 0;
4987 reinit_completion(&adapter
->fw_done
);
4989 rc
= ibmvnic_send_crq(adapter
, &crq
);
4991 mutex_unlock(&adapter
->fw_lock
);
4995 rc
= ibmvnic_wait_for_completion(adapter
, &adapter
->fw_done
, 10000);
4997 mutex_unlock(&adapter
->fw_lock
);
5001 mutex_unlock(&adapter
->fw_lock
);
5002 return adapter
->fw_done_rc
? -EIO
: 0;
5005 static int handle_query_phys_parms_rsp(union ibmvnic_crq
*crq
,
5006 struct ibmvnic_adapter
*adapter
)
5008 struct net_device
*netdev
= adapter
->netdev
;
5010 __be32 rspeed
= cpu_to_be32(crq
->query_phys_parms_rsp
.speed
);
5012 rc
= crq
->query_phys_parms_rsp
.rc
.code
;
5014 netdev_err(netdev
, "Error %d in QUERY_PHYS_PARMS\n", rc
);
5018 case IBMVNIC_10MBPS
:
5019 adapter
->speed
= SPEED_10
;
5021 case IBMVNIC_100MBPS
:
5022 adapter
->speed
= SPEED_100
;
5025 adapter
->speed
= SPEED_1000
;
5027 case IBMVNIC_10GBPS
:
5028 adapter
->speed
= SPEED_10000
;
5030 case IBMVNIC_25GBPS
:
5031 adapter
->speed
= SPEED_25000
;
5033 case IBMVNIC_40GBPS
:
5034 adapter
->speed
= SPEED_40000
;
5036 case IBMVNIC_50GBPS
:
5037 adapter
->speed
= SPEED_50000
;
5039 case IBMVNIC_100GBPS
:
5040 adapter
->speed
= SPEED_100000
;
5042 case IBMVNIC_200GBPS
:
5043 adapter
->speed
= SPEED_200000
;
5046 if (netif_carrier_ok(netdev
))
5047 netdev_warn(netdev
, "Unknown speed 0x%08x\n", rspeed
);
5048 adapter
->speed
= SPEED_UNKNOWN
;
5050 if (crq
->query_phys_parms_rsp
.flags1
& IBMVNIC_FULL_DUPLEX
)
5051 adapter
->duplex
= DUPLEX_FULL
;
5052 else if (crq
->query_phys_parms_rsp
.flags1
& IBMVNIC_HALF_DUPLEX
)
5053 adapter
->duplex
= DUPLEX_HALF
;
5055 adapter
->duplex
= DUPLEX_UNKNOWN
;
5060 static void ibmvnic_handle_crq(union ibmvnic_crq
*crq
,
5061 struct ibmvnic_adapter
*adapter
)
5063 struct ibmvnic_generic_crq
*gen_crq
= &crq
->generic
;
5064 struct net_device
*netdev
= adapter
->netdev
;
5065 struct device
*dev
= &adapter
->vdev
->dev
;
5066 u64
*u64_crq
= (u64
*)crq
;
5069 netdev_dbg(netdev
, "Handling CRQ: %016lx %016lx\n",
5070 (unsigned long)cpu_to_be64(u64_crq
[0]),
5071 (unsigned long)cpu_to_be64(u64_crq
[1]));
5072 switch (gen_crq
->first
) {
5073 case IBMVNIC_CRQ_INIT_RSP
:
5074 switch (gen_crq
->cmd
) {
5075 case IBMVNIC_CRQ_INIT
:
5076 dev_info(dev
, "Partner initialized\n");
5077 adapter
->from_passive_init
= true;
5078 /* Discard any stale login responses from prev reset.
5079 * CHECK: should we clear even on INIT_COMPLETE?
5081 adapter
->login_pending
= false;
5083 if (adapter
->state
== VNIC_DOWN
)
5084 rc
= ibmvnic_reset(adapter
, VNIC_RESET_PASSIVE_INIT
);
5086 rc
= ibmvnic_reset(adapter
, VNIC_RESET_FAILOVER
);
5088 if (rc
&& rc
!= -EBUSY
) {
5089 /* We were unable to schedule the failover
5090 * reset either because the adapter was still
5091 * probing (eg: during kexec) or we could not
5092 * allocate memory. Clear the failover_pending
5093 * flag since no one else will. We ignore
5094 * EBUSY because it means either FAILOVER reset
5095 * is already scheduled or the adapter is
5099 "Error %ld scheduling failover reset\n",
5101 adapter
->failover_pending
= false;
5104 if (!completion_done(&adapter
->init_done
)) {
5105 complete(&adapter
->init_done
);
5106 if (!adapter
->init_done_rc
)
5107 adapter
->init_done_rc
= -EAGAIN
;
5111 case IBMVNIC_CRQ_INIT_COMPLETE
:
5112 dev_info(dev
, "Partner initialization complete\n");
5113 adapter
->crq
.active
= true;
5114 send_version_xchg(adapter
);
5117 dev_err(dev
, "Unknown crq cmd: %d\n", gen_crq
->cmd
);
5120 case IBMVNIC_CRQ_XPORT_EVENT
:
5121 netif_carrier_off(netdev
);
5122 adapter
->crq
.active
= false;
5123 /* terminate any thread waiting for a response
5126 if (!completion_done(&adapter
->fw_done
)) {
5127 adapter
->fw_done_rc
= -EIO
;
5128 complete(&adapter
->fw_done
);
5130 if (!completion_done(&adapter
->stats_done
))
5131 complete(&adapter
->stats_done
);
5132 if (test_bit(0, &adapter
->resetting
))
5133 adapter
->force_reset_recovery
= true;
5134 if (gen_crq
->cmd
== IBMVNIC_PARTITION_MIGRATED
) {
5135 dev_info(dev
, "Migrated, re-enabling adapter\n");
5136 ibmvnic_reset(adapter
, VNIC_RESET_MOBILITY
);
5137 } else if (gen_crq
->cmd
== IBMVNIC_DEVICE_FAILOVER
) {
5138 dev_info(dev
, "Backing device failover detected\n");
5139 adapter
->failover_pending
= true;
5141 /* The adapter lost the connection */
5142 dev_err(dev
, "Virtual Adapter failed (rc=%d)\n",
5144 ibmvnic_reset(adapter
, VNIC_RESET_FATAL
);
5147 case IBMVNIC_CRQ_CMD_RSP
:
5150 dev_err(dev
, "Got an invalid msg type 0x%02x\n",
5155 switch (gen_crq
->cmd
) {
5156 case VERSION_EXCHANGE_RSP
:
5157 rc
= crq
->version_exchange_rsp
.rc
.code
;
5159 dev_err(dev
, "Error %ld in VERSION_EXCHG_RSP\n", rc
);
5163 be16_to_cpu(crq
->version_exchange_rsp
.version
);
5164 dev_info(dev
, "Partner protocol version is %d\n",
5166 send_query_cap(adapter
);
5168 case QUERY_CAPABILITY_RSP
:
5169 handle_query_cap_rsp(crq
, adapter
);
5172 handle_query_map_rsp(crq
, adapter
);
5174 case REQUEST_MAP_RSP
:
5175 adapter
->fw_done_rc
= crq
->request_map_rsp
.rc
.code
;
5176 complete(&adapter
->fw_done
);
5178 case REQUEST_UNMAP_RSP
:
5179 handle_request_unmap_rsp(crq
, adapter
);
5181 case REQUEST_CAPABILITY_RSP
:
5182 handle_request_cap_rsp(crq
, adapter
);
5185 netdev_dbg(netdev
, "Got Login Response\n");
5186 handle_login_rsp(crq
, adapter
);
5188 case LOGICAL_LINK_STATE_RSP
:
5190 "Got Logical Link State Response, state: %d rc: %d\n",
5191 crq
->logical_link_state_rsp
.link_state
,
5192 crq
->logical_link_state_rsp
.rc
.code
);
5193 adapter
->logical_link_state
=
5194 crq
->logical_link_state_rsp
.link_state
;
5195 adapter
->init_done_rc
= crq
->logical_link_state_rsp
.rc
.code
;
5196 complete(&adapter
->init_done
);
5198 case LINK_STATE_INDICATION
:
5199 netdev_dbg(netdev
, "Got Logical Link State Indication\n");
5200 adapter
->phys_link_state
=
5201 crq
->link_state_indication
.phys_link_state
;
5202 adapter
->logical_link_state
=
5203 crq
->link_state_indication
.logical_link_state
;
5204 if (adapter
->phys_link_state
&& adapter
->logical_link_state
)
5205 netif_carrier_on(netdev
);
5207 netif_carrier_off(netdev
);
5209 case CHANGE_MAC_ADDR_RSP
:
5210 netdev_dbg(netdev
, "Got MAC address change Response\n");
5211 adapter
->fw_done_rc
= handle_change_mac_rsp(crq
, adapter
);
5213 case ERROR_INDICATION
:
5214 netdev_dbg(netdev
, "Got Error Indication\n");
5215 handle_error_indication(crq
, adapter
);
5217 case REQUEST_STATISTICS_RSP
:
5218 netdev_dbg(netdev
, "Got Statistics Response\n");
5219 complete(&adapter
->stats_done
);
5221 case QUERY_IP_OFFLOAD_RSP
:
5222 netdev_dbg(netdev
, "Got Query IP offload Response\n");
5223 handle_query_ip_offload_rsp(adapter
);
5225 case MULTICAST_CTRL_RSP
:
5226 netdev_dbg(netdev
, "Got multicast control Response\n");
5228 case CONTROL_IP_OFFLOAD_RSP
:
5229 netdev_dbg(netdev
, "Got Control IP offload Response\n");
5230 dma_unmap_single(dev
, adapter
->ip_offload_ctrl_tok
,
5231 sizeof(adapter
->ip_offload_ctrl
),
5233 complete(&adapter
->init_done
);
5235 case COLLECT_FW_TRACE_RSP
:
5236 netdev_dbg(netdev
, "Got Collect firmware trace Response\n");
5237 complete(&adapter
->fw_done
);
5239 case GET_VPD_SIZE_RSP
:
5240 handle_vpd_size_rsp(crq
, adapter
);
5243 handle_vpd_rsp(crq
, adapter
);
5245 case QUERY_PHYS_PARMS_RSP
:
5246 adapter
->fw_done_rc
= handle_query_phys_parms_rsp(crq
, adapter
);
5247 complete(&adapter
->fw_done
);
5250 netdev_err(netdev
, "Got an invalid cmd type 0x%02x\n",
5255 static irqreturn_t
ibmvnic_interrupt(int irq
, void *instance
)
5257 struct ibmvnic_adapter
*adapter
= instance
;
5259 tasklet_schedule(&adapter
->tasklet
);
5263 static void ibmvnic_tasklet(struct tasklet_struct
*t
)
5265 struct ibmvnic_adapter
*adapter
= from_tasklet(adapter
, t
, tasklet
);
5266 struct ibmvnic_crq_queue
*queue
= &adapter
->crq
;
5267 union ibmvnic_crq
*crq
;
5268 unsigned long flags
;
5271 spin_lock_irqsave(&queue
->lock
, flags
);
5273 /* Pull all the valid messages off the CRQ */
5274 while ((crq
= ibmvnic_next_crq(adapter
)) != NULL
) {
5275 /* This barrier makes sure ibmvnic_next_crq()'s
5276 * crq->generic.first & IBMVNIC_CRQ_CMD_RSP is loaded
5277 * before ibmvnic_handle_crq()'s
5278 * switch(gen_crq->first) and switch(gen_crq->cmd).
5281 ibmvnic_handle_crq(crq
, adapter
);
5282 crq
->generic
.first
= 0;
5285 /* remain in tasklet until all
5286 * capabilities responses are received
5288 if (!adapter
->wait_capability
)
5291 /* if capabilities CRQ's were sent in this tasklet, the following
5292 * tasklet must wait until all responses are received
5294 if (atomic_read(&adapter
->running_cap_crqs
) != 0)
5295 adapter
->wait_capability
= true;
5296 spin_unlock_irqrestore(&queue
->lock
, flags
);
5299 static int ibmvnic_reenable_crq_queue(struct ibmvnic_adapter
*adapter
)
5301 struct vio_dev
*vdev
= adapter
->vdev
;
5305 rc
= plpar_hcall_norets(H_ENABLE_CRQ
, vdev
->unit_address
);
5306 } while (rc
== H_IN_PROGRESS
|| rc
== H_BUSY
|| H_IS_LONG_BUSY(rc
));
5309 dev_err(&vdev
->dev
, "Error enabling adapter (rc=%d)\n", rc
);
5314 static int ibmvnic_reset_crq(struct ibmvnic_adapter
*adapter
)
5316 struct ibmvnic_crq_queue
*crq
= &adapter
->crq
;
5317 struct device
*dev
= &adapter
->vdev
->dev
;
5318 struct vio_dev
*vdev
= adapter
->vdev
;
5323 rc
= plpar_hcall_norets(H_FREE_CRQ
, vdev
->unit_address
);
5324 } while (rc
== H_BUSY
|| H_IS_LONG_BUSY(rc
));
5326 /* Clean out the queue */
5330 memset(crq
->msgs
, 0, PAGE_SIZE
);
5332 crq
->active
= false;
5334 /* And re-open it again */
5335 rc
= plpar_hcall_norets(H_REG_CRQ
, vdev
->unit_address
,
5336 crq
->msg_token
, PAGE_SIZE
);
5339 /* Adapter is good, but other end is not ready */
5340 dev_warn(dev
, "Partner adapter not ready\n");
5342 dev_warn(dev
, "Couldn't register crq (rc=%d)\n", rc
);
5347 static void release_crq_queue(struct ibmvnic_adapter
*adapter
)
5349 struct ibmvnic_crq_queue
*crq
= &adapter
->crq
;
5350 struct vio_dev
*vdev
= adapter
->vdev
;
5356 netdev_dbg(adapter
->netdev
, "Releasing CRQ\n");
5357 free_irq(vdev
->irq
, adapter
);
5358 tasklet_kill(&adapter
->tasklet
);
5360 rc
= plpar_hcall_norets(H_FREE_CRQ
, vdev
->unit_address
);
5361 } while (rc
== H_BUSY
|| H_IS_LONG_BUSY(rc
));
5363 dma_unmap_single(&vdev
->dev
, crq
->msg_token
, PAGE_SIZE
,
5365 free_page((unsigned long)crq
->msgs
);
5367 crq
->active
= false;
5370 static int init_crq_queue(struct ibmvnic_adapter
*adapter
)
5372 struct ibmvnic_crq_queue
*crq
= &adapter
->crq
;
5373 struct device
*dev
= &adapter
->vdev
->dev
;
5374 struct vio_dev
*vdev
= adapter
->vdev
;
5375 int rc
, retrc
= -ENOMEM
;
5380 crq
->msgs
= (union ibmvnic_crq
*)get_zeroed_page(GFP_KERNEL
);
5381 /* Should we allocate more than one page? */
5386 crq
->size
= PAGE_SIZE
/ sizeof(*crq
->msgs
);
5387 crq
->msg_token
= dma_map_single(dev
, crq
->msgs
, PAGE_SIZE
,
5389 if (dma_mapping_error(dev
, crq
->msg_token
))
5392 rc
= plpar_hcall_norets(H_REG_CRQ
, vdev
->unit_address
,
5393 crq
->msg_token
, PAGE_SIZE
);
5395 if (rc
== H_RESOURCE
)
5396 /* maybe kexecing and resource is busy. try a reset */
5397 rc
= ibmvnic_reset_crq(adapter
);
5400 if (rc
== H_CLOSED
) {
5401 dev_warn(dev
, "Partner adapter not ready\n");
5403 dev_warn(dev
, "Error %d opening adapter\n", rc
);
5404 goto reg_crq_failed
;
5409 tasklet_setup(&adapter
->tasklet
, (void *)ibmvnic_tasklet
);
5411 netdev_dbg(adapter
->netdev
, "registering irq 0x%x\n", vdev
->irq
);
5412 snprintf(crq
->name
, sizeof(crq
->name
), "ibmvnic-%x",
5413 adapter
->vdev
->unit_address
);
5414 rc
= request_irq(vdev
->irq
, ibmvnic_interrupt
, 0, crq
->name
, adapter
);
5416 dev_err(dev
, "Couldn't register irq 0x%x. rc=%d\n",
5418 goto req_irq_failed
;
5421 rc
= vio_enable_interrupts(vdev
);
5423 dev_err(dev
, "Error %d enabling interrupts\n", rc
);
5424 goto req_irq_failed
;
5428 spin_lock_init(&crq
->lock
);
5430 /* process any CRQs that were queued before we enabled interrupts */
5431 tasklet_schedule(&adapter
->tasklet
);
5436 tasklet_kill(&adapter
->tasklet
);
5438 rc
= plpar_hcall_norets(H_FREE_CRQ
, vdev
->unit_address
);
5439 } while (rc
== H_BUSY
|| H_IS_LONG_BUSY(rc
));
5441 dma_unmap_single(dev
, crq
->msg_token
, PAGE_SIZE
, DMA_BIDIRECTIONAL
);
5443 free_page((unsigned long)crq
->msgs
);
5448 static int ibmvnic_reset_init(struct ibmvnic_adapter
*adapter
, bool reset
)
5450 struct device
*dev
= &adapter
->vdev
->dev
;
5451 unsigned long timeout
= msecs_to_jiffies(20000);
5452 u64 old_num_rx_queues
= adapter
->req_rx_queues
;
5453 u64 old_num_tx_queues
= adapter
->req_tx_queues
;
5456 adapter
->from_passive_init
= false;
5459 reinit_completion(&adapter
->init_done
);
5461 adapter
->init_done_rc
= 0;
5462 rc
= ibmvnic_send_crq_init(adapter
);
5464 dev_err(dev
, "Send crq init failed with error %d\n", rc
);
5468 if (!wait_for_completion_timeout(&adapter
->init_done
, timeout
)) {
5469 dev_err(dev
, "Initialization sequence timed out\n");
5473 if (adapter
->init_done_rc
) {
5474 release_crq_queue(adapter
);
5475 return adapter
->init_done_rc
;
5478 if (adapter
->from_passive_init
) {
5479 adapter
->state
= VNIC_OPEN
;
5480 adapter
->from_passive_init
= false;
5485 test_bit(0, &adapter
->resetting
) && !adapter
->wait_for_reset
&&
5486 adapter
->reset_reason
!= VNIC_RESET_MOBILITY
) {
5487 if (adapter
->req_rx_queues
!= old_num_rx_queues
||
5488 adapter
->req_tx_queues
!= old_num_tx_queues
) {
5489 release_sub_crqs(adapter
, 0);
5490 rc
= init_sub_crqs(adapter
);
5492 rc
= reset_sub_crq_queues(adapter
);
5495 rc
= init_sub_crqs(adapter
);
5499 dev_err(dev
, "Initialization of sub crqs failed\n");
5500 release_crq_queue(adapter
);
5504 rc
= init_sub_crq_irqs(adapter
);
5506 dev_err(dev
, "Failed to initialize sub crq irqs\n");
5507 release_crq_queue(adapter
);
5513 static struct device_attribute dev_attr_failover
;
5515 static int ibmvnic_probe(struct vio_dev
*dev
, const struct vio_device_id
*id
)
5517 struct ibmvnic_adapter
*adapter
;
5518 struct net_device
*netdev
;
5519 unsigned char *mac_addr_p
;
5523 dev_dbg(&dev
->dev
, "entering ibmvnic_probe for UA 0x%x\n",
5526 mac_addr_p
= (unsigned char *)vio_get_attribute(dev
,
5527 VETH_MAC_ADDR
, NULL
);
5530 "(%s:%3.3d) ERROR: Can't find MAC_ADDR attribute\n",
5531 __FILE__
, __LINE__
);
5535 netdev
= alloc_etherdev_mq(sizeof(struct ibmvnic_adapter
),
5536 IBMVNIC_MAX_QUEUES
);
5540 adapter
= netdev_priv(netdev
);
5541 adapter
->state
= VNIC_PROBING
;
5542 dev_set_drvdata(&dev
->dev
, netdev
);
5543 adapter
->vdev
= dev
;
5544 adapter
->netdev
= netdev
;
5545 adapter
->login_pending
= false;
5547 ether_addr_copy(adapter
->mac_addr
, mac_addr_p
);
5548 ether_addr_copy(netdev
->dev_addr
, adapter
->mac_addr
);
5549 netdev
->irq
= dev
->irq
;
5550 netdev
->netdev_ops
= &ibmvnic_netdev_ops
;
5551 netdev
->ethtool_ops
= &ibmvnic_ethtool_ops
;
5552 SET_NETDEV_DEV(netdev
, &dev
->dev
);
5554 INIT_WORK(&adapter
->ibmvnic_reset
, __ibmvnic_reset
);
5555 INIT_DELAYED_WORK(&adapter
->ibmvnic_delayed_reset
,
5556 __ibmvnic_delayed_reset
);
5557 INIT_LIST_HEAD(&adapter
->rwi_list
);
5558 spin_lock_init(&adapter
->rwi_lock
);
5559 spin_lock_init(&adapter
->state_lock
);
5560 mutex_init(&adapter
->fw_lock
);
5561 init_completion(&adapter
->init_done
);
5562 init_completion(&adapter
->fw_done
);
5563 init_completion(&adapter
->reset_done
);
5564 init_completion(&adapter
->stats_done
);
5565 clear_bit(0, &adapter
->resetting
);
5567 init_success
= false;
5569 rc
= init_crq_queue(adapter
);
5571 dev_err(&dev
->dev
, "Couldn't initialize crq. rc=%d\n",
5573 goto ibmvnic_init_fail
;
5576 rc
= ibmvnic_reset_init(adapter
, false);
5577 } while (rc
== -EAGAIN
);
5579 /* We are ignoring the error from ibmvnic_reset_init() assuming that the
5580 * partner is not ready. CRQ is not active. When the partner becomes
5581 * ready, we will do the passive init reset.
5585 init_success
= true;
5587 rc
= init_stats_buffers(adapter
);
5589 goto ibmvnic_init_fail
;
5591 rc
= init_stats_token(adapter
);
5593 goto ibmvnic_stats_fail
;
5595 rc
= device_create_file(&dev
->dev
, &dev_attr_failover
);
5597 goto ibmvnic_dev_file_err
;
5599 netif_carrier_off(netdev
);
5600 rc
= register_netdev(netdev
);
5602 dev_err(&dev
->dev
, "failed to register netdev rc=%d\n", rc
);
5603 goto ibmvnic_register_fail
;
5605 dev_info(&dev
->dev
, "ibmvnic registered\n");
5608 adapter
->state
= VNIC_PROBED
;
5609 netdev
->mtu
= adapter
->req_mtu
- ETH_HLEN
;
5610 netdev
->min_mtu
= adapter
->min_mtu
- ETH_HLEN
;
5611 netdev
->max_mtu
= adapter
->max_mtu
- ETH_HLEN
;
5613 adapter
->state
= VNIC_DOWN
;
5616 adapter
->wait_for_reset
= false;
5617 adapter
->last_reset_time
= jiffies
;
5620 ibmvnic_register_fail
:
5621 device_remove_file(&dev
->dev
, &dev_attr_failover
);
5623 ibmvnic_dev_file_err
:
5624 release_stats_token(adapter
);
5627 release_stats_buffers(adapter
);
5630 release_sub_crqs(adapter
, 1);
5631 release_crq_queue(adapter
);
5632 mutex_destroy(&adapter
->fw_lock
);
5633 free_netdev(netdev
);
5638 static void ibmvnic_remove(struct vio_dev
*dev
)
5640 struct net_device
*netdev
= dev_get_drvdata(&dev
->dev
);
5641 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
5642 unsigned long flags
;
5644 spin_lock_irqsave(&adapter
->state_lock
, flags
);
5646 /* If ibmvnic_reset() is scheduling a reset, wait for it to
5647 * finish. Then, set the state to REMOVING to prevent it from
5648 * scheduling any more work and to have reset functions ignore
5649 * any resets that have already been scheduled. Drop the lock
5650 * after setting state, so __ibmvnic_reset() which is called
5651 * from the flush_work() below, can make progress.
5653 spin_lock(&adapter
->rwi_lock
);
5654 adapter
->state
= VNIC_REMOVING
;
5655 spin_unlock(&adapter
->rwi_lock
);
5657 spin_unlock_irqrestore(&adapter
->state_lock
, flags
);
5659 flush_work(&adapter
->ibmvnic_reset
);
5660 flush_delayed_work(&adapter
->ibmvnic_delayed_reset
);
5663 unregister_netdevice(netdev
);
5665 release_resources(adapter
);
5666 release_sub_crqs(adapter
, 1);
5667 release_crq_queue(adapter
);
5669 release_stats_token(adapter
);
5670 release_stats_buffers(adapter
);
5672 adapter
->state
= VNIC_REMOVED
;
5675 mutex_destroy(&adapter
->fw_lock
);
5676 device_remove_file(&dev
->dev
, &dev_attr_failover
);
5677 free_netdev(netdev
);
5678 dev_set_drvdata(&dev
->dev
, NULL
);
5681 static ssize_t
failover_store(struct device
*dev
, struct device_attribute
*attr
,
5682 const char *buf
, size_t count
)
5684 struct net_device
*netdev
= dev_get_drvdata(dev
);
5685 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
5686 unsigned long retbuf
[PLPAR_HCALL_BUFSIZE
];
5687 __be64 session_token
;
5690 if (!sysfs_streq(buf
, "1"))
5693 rc
= plpar_hcall(H_VIOCTL
, retbuf
, adapter
->vdev
->unit_address
,
5694 H_GET_SESSION_TOKEN
, 0, 0, 0);
5696 netdev_err(netdev
, "Couldn't retrieve session token, rc %ld\n",
5701 session_token
= (__be64
)retbuf
[0];
5702 netdev_dbg(netdev
, "Initiating client failover, session id %llx\n",
5703 be64_to_cpu(session_token
));
5704 rc
= plpar_hcall_norets(H_VIOCTL
, adapter
->vdev
->unit_address
,
5705 H_SESSION_ERR_DETECTED
, session_token
, 0, 0);
5708 "H_VIOCTL initiated failover failed, rc %ld\n",
5712 netdev_dbg(netdev
, "Trying to send CRQ_CMD, the last resort\n");
5713 ibmvnic_reset(adapter
, VNIC_RESET_FAILOVER
);
5717 static DEVICE_ATTR_WO(failover
);
5719 static unsigned long ibmvnic_get_desired_dma(struct vio_dev
*vdev
)
5721 struct net_device
*netdev
= dev_get_drvdata(&vdev
->dev
);
5722 struct ibmvnic_adapter
*adapter
;
5723 struct iommu_table
*tbl
;
5724 unsigned long ret
= 0;
5727 tbl
= get_iommu_table_base(&vdev
->dev
);
5729 /* netdev inits at probe time along with the structures we need below*/
5731 return IOMMU_PAGE_ALIGN(IBMVNIC_IO_ENTITLEMENT_DEFAULT
, tbl
);
5733 adapter
= netdev_priv(netdev
);
5735 ret
+= PAGE_SIZE
; /* the crq message queue */
5736 ret
+= IOMMU_PAGE_ALIGN(sizeof(struct ibmvnic_statistics
), tbl
);
5738 for (i
= 0; i
< adapter
->req_tx_queues
+ adapter
->req_rx_queues
; i
++)
5739 ret
+= 4 * PAGE_SIZE
; /* the scrq message queue */
5741 for (i
= 0; i
< adapter
->num_active_rx_pools
; i
++)
5742 ret
+= adapter
->rx_pool
[i
].size
*
5743 IOMMU_PAGE_ALIGN(adapter
->rx_pool
[i
].buff_size
, tbl
);
5748 static int ibmvnic_resume(struct device
*dev
)
5750 struct net_device
*netdev
= dev_get_drvdata(dev
);
5751 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
5753 if (adapter
->state
!= VNIC_OPEN
)
5756 tasklet_schedule(&adapter
->tasklet
);
5761 static const struct vio_device_id ibmvnic_device_table
[] = {
5762 {"network", "IBM,vnic"},
5765 MODULE_DEVICE_TABLE(vio
, ibmvnic_device_table
);
5767 static const struct dev_pm_ops ibmvnic_pm_ops
= {
5768 .resume
= ibmvnic_resume
5771 static struct vio_driver ibmvnic_driver
= {
5772 .id_table
= ibmvnic_device_table
,
5773 .probe
= ibmvnic_probe
,
5774 .remove
= ibmvnic_remove
,
5775 .get_desired_dma
= ibmvnic_get_desired_dma
,
5776 .name
= ibmvnic_driver_name
,
5777 .pm
= &ibmvnic_pm_ops
,
5780 /* module functions */
5781 static int __init
ibmvnic_module_init(void)
5783 pr_info("%s: %s %s\n", ibmvnic_driver_name
, ibmvnic_driver_string
,
5784 IBMVNIC_DRIVER_VERSION
);
5786 return vio_register_driver(&ibmvnic_driver
);
5789 static void __exit
ibmvnic_module_exit(void)
5791 vio_unregister_driver(&ibmvnic_driver
);
5794 module_init(ibmvnic_module_init
);
5795 module_exit(ibmvnic_module_exit
);