1 /**************************************************************************/
3 /* IBM System i and System p Virtual NIC Device Driver */
4 /* Copyright (C) 2014 IBM Corp. */
5 /* Santiago Leon (santi_leon@yahoo.com) */
6 /* Thomas Falcon (tlfalcon@linux.vnet.ibm.com) */
7 /* John Allen (jallen@linux.vnet.ibm.com) */
9 /* This program is free software; you can redistribute it and/or modify */
10 /* it under the terms of the GNU General Public License as published by */
11 /* the Free Software Foundation; either version 2 of the License, or */
12 /* (at your option) any later version. */
14 /* This program is distributed in the hope that it will be useful, */
15 /* but WITHOUT ANY WARRANTY; without even the implied warranty of */
16 /* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the */
17 /* GNU General Public License for more details. */
19 /* You should have received a copy of the GNU General Public License */
20 /* along with this program. */
22 /* This module contains the implementation of a virtual ethernet device */
23 /* for use with IBM i/p Series LPAR Linux. It utilizes the logical LAN */
24 /* option of the RS/6000 Platform Architecture to interface with virtual */
25 /* ethernet NICs that are presented to the partition by the hypervisor. */
27 /* Messages are passed between the VNIC driver and the VNIC server using */
28 /* Command/Response Queues (CRQs) and sub CRQs (sCRQs). CRQs are used to */
29 /* issue and receive commands that initiate communication with the server */
30 /* on driver initialization. Sub CRQs (sCRQs) are similar to CRQs, but */
31 /* are used by the driver to notify the server that a packet is */
32 /* ready for transmission or that a buffer has been added to receive a */
33 /* packet. Subsequently, sCRQs are used by the server to notify the */
34 /* driver that a packet transmission has been completed or that a packet */
35 /* has been received and placed in a waiting buffer. */
37 /* In lieu of a more conventional "on-the-fly" DMA mapping strategy in */
38 /* which skbs are DMA mapped and immediately unmapped when the transmit */
39 /* or receive has been completed, the VNIC driver is required to use */
40 /* "long term mapping". This entails that large, continuous DMA mapped */
41 /* buffers are allocated on driver initialization and these buffers are */
42 /* then continuously reused to pass skbs to and from the VNIC server. */
44 /**************************************************************************/
46 #include <linux/module.h>
47 #include <linux/moduleparam.h>
48 #include <linux/types.h>
49 #include <linux/errno.h>
50 #include <linux/completion.h>
51 #include <linux/ioport.h>
52 #include <linux/dma-mapping.h>
53 #include <linux/kernel.h>
54 #include <linux/netdevice.h>
55 #include <linux/etherdevice.h>
56 #include <linux/skbuff.h>
57 #include <linux/init.h>
58 #include <linux/delay.h>
60 #include <linux/ethtool.h>
61 #include <linux/proc_fs.h>
64 #include <linux/ipv6.h>
65 #include <linux/irq.h>
66 #include <linux/kthread.h>
67 #include <linux/seq_file.h>
68 #include <linux/interrupt.h>
69 #include <net/net_namespace.h>
70 #include <asm/hvcall.h>
71 #include <linux/atomic.h>
73 #include <asm/iommu.h>
74 #include <linux/uaccess.h>
75 #include <asm/firmware.h>
76 #include <linux/workqueue.h>
77 #include <linux/if_vlan.h>
81 static const char ibmvnic_driver_name
[] = "ibmvnic";
82 static const char ibmvnic_driver_string
[] = "IBM System i/p Virtual NIC Driver";
84 MODULE_AUTHOR("Santiago Leon <santi_leon@yahoo.com>");
85 MODULE_DESCRIPTION("IBM System i/p Virtual NIC Driver");
86 MODULE_LICENSE("GPL");
87 MODULE_VERSION(IBMVNIC_DRIVER_VERSION
);
89 static int ibmvnic_version
= IBMVNIC_INITIAL_VERSION
;
90 static int ibmvnic_remove(struct vio_dev
*);
91 static void release_sub_crqs(struct ibmvnic_adapter
*);
92 static int ibmvnic_reset_crq(struct ibmvnic_adapter
*);
93 static int ibmvnic_send_crq_init(struct ibmvnic_adapter
*);
94 static int ibmvnic_reenable_crq_queue(struct ibmvnic_adapter
*);
95 static int ibmvnic_send_crq(struct ibmvnic_adapter
*, union ibmvnic_crq
*);
96 static int send_subcrq(struct ibmvnic_adapter
*adapter
, u64 remote_handle
,
97 union sub_crq
*sub_crq
);
98 static int send_subcrq_indirect(struct ibmvnic_adapter
*, u64
, u64
, u64
);
99 static irqreturn_t
ibmvnic_interrupt_rx(int irq
, void *instance
);
100 static int enable_scrq_irq(struct ibmvnic_adapter
*,
101 struct ibmvnic_sub_crq_queue
*);
102 static int disable_scrq_irq(struct ibmvnic_adapter
*,
103 struct ibmvnic_sub_crq_queue
*);
104 static int pending_scrq(struct ibmvnic_adapter
*,
105 struct ibmvnic_sub_crq_queue
*);
106 static union sub_crq
*ibmvnic_next_scrq(struct ibmvnic_adapter
*,
107 struct ibmvnic_sub_crq_queue
*);
108 static int ibmvnic_poll(struct napi_struct
*napi
, int data
);
109 static void send_map_query(struct ibmvnic_adapter
*adapter
);
110 static void send_request_map(struct ibmvnic_adapter
*, dma_addr_t
, __be32
, u8
);
111 static void send_request_unmap(struct ibmvnic_adapter
*, u8
);
112 static void send_login(struct ibmvnic_adapter
*adapter
);
113 static void send_cap_queries(struct ibmvnic_adapter
*adapter
);
114 static int init_sub_crq_irqs(struct ibmvnic_adapter
*adapter
);
115 static int ibmvnic_init(struct ibmvnic_adapter
*);
116 static void release_crq_queue(struct ibmvnic_adapter
*);
118 struct ibmvnic_stat
{
119 char name
[ETH_GSTRING_LEN
];
123 #define IBMVNIC_STAT_OFF(stat) (offsetof(struct ibmvnic_adapter, stats) + \
124 offsetof(struct ibmvnic_statistics, stat))
125 #define IBMVNIC_GET_STAT(a, off) (*((u64 *)(((unsigned long)(a)) + off)))
127 static const struct ibmvnic_stat ibmvnic_stats
[] = {
128 {"rx_packets", IBMVNIC_STAT_OFF(rx_packets
)},
129 {"rx_bytes", IBMVNIC_STAT_OFF(rx_bytes
)},
130 {"tx_packets", IBMVNIC_STAT_OFF(tx_packets
)},
131 {"tx_bytes", IBMVNIC_STAT_OFF(tx_bytes
)},
132 {"ucast_tx_packets", IBMVNIC_STAT_OFF(ucast_tx_packets
)},
133 {"ucast_rx_packets", IBMVNIC_STAT_OFF(ucast_rx_packets
)},
134 {"mcast_tx_packets", IBMVNIC_STAT_OFF(mcast_tx_packets
)},
135 {"mcast_rx_packets", IBMVNIC_STAT_OFF(mcast_rx_packets
)},
136 {"bcast_tx_packets", IBMVNIC_STAT_OFF(bcast_tx_packets
)},
137 {"bcast_rx_packets", IBMVNIC_STAT_OFF(bcast_rx_packets
)},
138 {"align_errors", IBMVNIC_STAT_OFF(align_errors
)},
139 {"fcs_errors", IBMVNIC_STAT_OFF(fcs_errors
)},
140 {"single_collision_frames", IBMVNIC_STAT_OFF(single_collision_frames
)},
141 {"multi_collision_frames", IBMVNIC_STAT_OFF(multi_collision_frames
)},
142 {"sqe_test_errors", IBMVNIC_STAT_OFF(sqe_test_errors
)},
143 {"deferred_tx", IBMVNIC_STAT_OFF(deferred_tx
)},
144 {"late_collisions", IBMVNIC_STAT_OFF(late_collisions
)},
145 {"excess_collisions", IBMVNIC_STAT_OFF(excess_collisions
)},
146 {"internal_mac_tx_errors", IBMVNIC_STAT_OFF(internal_mac_tx_errors
)},
147 {"carrier_sense", IBMVNIC_STAT_OFF(carrier_sense
)},
148 {"too_long_frames", IBMVNIC_STAT_OFF(too_long_frames
)},
149 {"internal_mac_rx_errors", IBMVNIC_STAT_OFF(internal_mac_rx_errors
)},
152 static long h_reg_sub_crq(unsigned long unit_address
, unsigned long token
,
153 unsigned long length
, unsigned long *number
,
156 unsigned long retbuf
[PLPAR_HCALL_BUFSIZE
];
159 rc
= plpar_hcall(H_REG_SUB_CRQ
, retbuf
, unit_address
, token
, length
);
166 static int alloc_long_term_buff(struct ibmvnic_adapter
*adapter
,
167 struct ibmvnic_long_term_buff
*ltb
, int size
)
169 struct device
*dev
= &adapter
->vdev
->dev
;
172 ltb
->buff
= dma_alloc_coherent(dev
, ltb
->size
, <b
->addr
,
176 dev_err(dev
, "Couldn't alloc long term buffer\n");
179 ltb
->map_id
= adapter
->map_id
;
182 init_completion(&adapter
->fw_done
);
183 send_request_map(adapter
, ltb
->addr
,
184 ltb
->size
, ltb
->map_id
);
185 wait_for_completion(&adapter
->fw_done
);
189 static void free_long_term_buff(struct ibmvnic_adapter
*adapter
,
190 struct ibmvnic_long_term_buff
*ltb
)
192 struct device
*dev
= &adapter
->vdev
->dev
;
197 if (adapter
->reset_reason
!= VNIC_RESET_FAILOVER
&&
198 adapter
->reset_reason
!= VNIC_RESET_MOBILITY
)
199 send_request_unmap(adapter
, ltb
->map_id
);
200 dma_free_coherent(dev
, ltb
->size
, ltb
->buff
, ltb
->addr
);
203 static void replenish_rx_pool(struct ibmvnic_adapter
*adapter
,
204 struct ibmvnic_rx_pool
*pool
)
206 int count
= pool
->size
- atomic_read(&pool
->available
);
207 struct device
*dev
= &adapter
->vdev
->dev
;
208 int buffers_added
= 0;
209 unsigned long lpar_rc
;
210 union sub_crq sub_crq
;
220 handle_array
= (u64
*)((u8
*)(adapter
->login_rsp_buf
) +
221 be32_to_cpu(adapter
->login_rsp_buf
->
224 for (i
= 0; i
< count
; ++i
) {
225 skb
= alloc_skb(pool
->buff_size
, GFP_ATOMIC
);
227 dev_err(dev
, "Couldn't replenish rx buff\n");
228 adapter
->replenish_no_mem
++;
232 index
= pool
->free_map
[pool
->next_free
];
234 if (pool
->rx_buff
[index
].skb
)
235 dev_err(dev
, "Inconsistent free_map!\n");
237 /* Copy the skb to the long term mapped DMA buffer */
238 offset
= index
* pool
->buff_size
;
239 dst
= pool
->long_term_buff
.buff
+ offset
;
240 memset(dst
, 0, pool
->buff_size
);
241 dma_addr
= pool
->long_term_buff
.addr
+ offset
;
242 pool
->rx_buff
[index
].data
= dst
;
244 pool
->free_map
[pool
->next_free
] = IBMVNIC_INVALID_MAP
;
245 pool
->rx_buff
[index
].dma
= dma_addr
;
246 pool
->rx_buff
[index
].skb
= skb
;
247 pool
->rx_buff
[index
].pool_index
= pool
->index
;
248 pool
->rx_buff
[index
].size
= pool
->buff_size
;
250 memset(&sub_crq
, 0, sizeof(sub_crq
));
251 sub_crq
.rx_add
.first
= IBMVNIC_CRQ_CMD
;
252 sub_crq
.rx_add
.correlator
=
253 cpu_to_be64((u64
)&pool
->rx_buff
[index
]);
254 sub_crq
.rx_add
.ioba
= cpu_to_be32(dma_addr
);
255 sub_crq
.rx_add
.map_id
= pool
->long_term_buff
.map_id
;
257 /* The length field of the sCRQ is defined to be 24 bits so the
258 * buffer size needs to be left shifted by a byte before it is
259 * converted to big endian to prevent the last byte from being
262 #ifdef __LITTLE_ENDIAN__
265 sub_crq
.rx_add
.len
= cpu_to_be32(pool
->buff_size
<< shift
);
267 lpar_rc
= send_subcrq(adapter
, handle_array
[pool
->index
],
269 if (lpar_rc
!= H_SUCCESS
)
273 adapter
->replenish_add_buff_success
++;
274 pool
->next_free
= (pool
->next_free
+ 1) % pool
->size
;
276 atomic_add(buffers_added
, &pool
->available
);
280 dev_info(dev
, "replenish pools failure\n");
281 pool
->free_map
[pool
->next_free
] = index
;
282 pool
->rx_buff
[index
].skb
= NULL
;
283 if (!dma_mapping_error(dev
, dma_addr
))
284 dma_unmap_single(dev
, dma_addr
, pool
->buff_size
,
287 dev_kfree_skb_any(skb
);
288 adapter
->replenish_add_buff_failure
++;
289 atomic_add(buffers_added
, &pool
->available
);
292 static void replenish_pools(struct ibmvnic_adapter
*adapter
)
296 adapter
->replenish_task_cycles
++;
297 for (i
= 0; i
< be32_to_cpu(adapter
->login_rsp_buf
->num_rxadd_subcrqs
);
299 if (adapter
->rx_pool
[i
].active
)
300 replenish_rx_pool(adapter
, &adapter
->rx_pool
[i
]);
304 static void release_stats_token(struct ibmvnic_adapter
*adapter
)
306 struct device
*dev
= &adapter
->vdev
->dev
;
308 if (!adapter
->stats_token
)
311 dma_unmap_single(dev
, adapter
->stats_token
,
312 sizeof(struct ibmvnic_statistics
),
314 adapter
->stats_token
= 0;
317 static int init_stats_token(struct ibmvnic_adapter
*adapter
)
319 struct device
*dev
= &adapter
->vdev
->dev
;
322 stok
= dma_map_single(dev
, &adapter
->stats
,
323 sizeof(struct ibmvnic_statistics
),
325 if (dma_mapping_error(dev
, stok
)) {
326 dev_err(dev
, "Couldn't map stats buffer\n");
330 adapter
->stats_token
= stok
;
334 static void release_rx_pools(struct ibmvnic_adapter
*adapter
)
336 struct ibmvnic_rx_pool
*rx_pool
;
340 if (!adapter
->rx_pool
)
343 rx_scrqs
= be32_to_cpu(adapter
->login_rsp_buf
->num_rxadd_subcrqs
);
344 for (i
= 0; i
< rx_scrqs
; i
++) {
345 rx_pool
= &adapter
->rx_pool
[i
];
347 kfree(rx_pool
->free_map
);
348 free_long_term_buff(adapter
, &rx_pool
->long_term_buff
);
350 if (!rx_pool
->rx_buff
)
353 for (j
= 0; j
< rx_pool
->size
; j
++) {
354 if (rx_pool
->rx_buff
[j
].skb
) {
355 dev_kfree_skb_any(rx_pool
->rx_buff
[i
].skb
);
356 rx_pool
->rx_buff
[i
].skb
= NULL
;
360 kfree(rx_pool
->rx_buff
);
363 kfree(adapter
->rx_pool
);
364 adapter
->rx_pool
= NULL
;
367 static int init_rx_pools(struct net_device
*netdev
)
369 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
370 struct device
*dev
= &adapter
->vdev
->dev
;
371 struct ibmvnic_rx_pool
*rx_pool
;
377 be32_to_cpu(adapter
->login_rsp_buf
->num_rxadd_subcrqs
);
378 size_array
= (u64
*)((u8
*)(adapter
->login_rsp_buf
) +
379 be32_to_cpu(adapter
->login_rsp_buf
->off_rxadd_buff_size
));
381 adapter
->rx_pool
= kcalloc(rxadd_subcrqs
,
382 sizeof(struct ibmvnic_rx_pool
),
384 if (!adapter
->rx_pool
) {
385 dev_err(dev
, "Failed to allocate rx pools\n");
389 for (i
= 0; i
< rxadd_subcrqs
; i
++) {
390 rx_pool
= &adapter
->rx_pool
[i
];
392 netdev_dbg(adapter
->netdev
,
393 "Initializing rx_pool %d, %lld buffs, %lld bytes each\n",
394 i
, adapter
->req_rx_add_entries_per_subcrq
,
395 be64_to_cpu(size_array
[i
]));
397 rx_pool
->size
= adapter
->req_rx_add_entries_per_subcrq
;
399 rx_pool
->buff_size
= be64_to_cpu(size_array
[i
]);
402 rx_pool
->free_map
= kcalloc(rx_pool
->size
, sizeof(int),
404 if (!rx_pool
->free_map
) {
405 release_rx_pools(adapter
);
409 rx_pool
->rx_buff
= kcalloc(rx_pool
->size
,
410 sizeof(struct ibmvnic_rx_buff
),
412 if (!rx_pool
->rx_buff
) {
413 dev_err(dev
, "Couldn't alloc rx buffers\n");
414 release_rx_pools(adapter
);
418 if (alloc_long_term_buff(adapter
, &rx_pool
->long_term_buff
,
419 rx_pool
->size
* rx_pool
->buff_size
)) {
420 release_rx_pools(adapter
);
424 for (j
= 0; j
< rx_pool
->size
; ++j
)
425 rx_pool
->free_map
[j
] = j
;
427 atomic_set(&rx_pool
->available
, 0);
428 rx_pool
->next_alloc
= 0;
429 rx_pool
->next_free
= 0;
435 static void release_tx_pools(struct ibmvnic_adapter
*adapter
)
437 struct ibmvnic_tx_pool
*tx_pool
;
440 if (!adapter
->tx_pool
)
443 tx_scrqs
= be32_to_cpu(adapter
->login_rsp_buf
->num_txsubm_subcrqs
);
444 for (i
= 0; i
< tx_scrqs
; i
++) {
445 tx_pool
= &adapter
->tx_pool
[i
];
446 kfree(tx_pool
->tx_buff
);
447 free_long_term_buff(adapter
, &tx_pool
->long_term_buff
);
448 kfree(tx_pool
->free_map
);
451 kfree(adapter
->tx_pool
);
452 adapter
->tx_pool
= NULL
;
455 static int init_tx_pools(struct net_device
*netdev
)
457 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
458 struct device
*dev
= &adapter
->vdev
->dev
;
459 struct ibmvnic_tx_pool
*tx_pool
;
463 tx_subcrqs
= be32_to_cpu(adapter
->login_rsp_buf
->num_txsubm_subcrqs
);
464 adapter
->tx_pool
= kcalloc(tx_subcrqs
,
465 sizeof(struct ibmvnic_tx_pool
), GFP_KERNEL
);
466 if (!adapter
->tx_pool
)
469 for (i
= 0; i
< tx_subcrqs
; i
++) {
470 tx_pool
= &adapter
->tx_pool
[i
];
471 tx_pool
->tx_buff
= kcalloc(adapter
->req_tx_entries_per_subcrq
,
472 sizeof(struct ibmvnic_tx_buff
),
474 if (!tx_pool
->tx_buff
) {
475 dev_err(dev
, "tx pool buffer allocation failed\n");
476 release_tx_pools(adapter
);
480 if (alloc_long_term_buff(adapter
, &tx_pool
->long_term_buff
,
481 adapter
->req_tx_entries_per_subcrq
*
483 release_tx_pools(adapter
);
487 tx_pool
->free_map
= kcalloc(adapter
->req_tx_entries_per_subcrq
,
488 sizeof(int), GFP_KERNEL
);
489 if (!tx_pool
->free_map
) {
490 release_tx_pools(adapter
);
494 for (j
= 0; j
< adapter
->req_tx_entries_per_subcrq
; j
++)
495 tx_pool
->free_map
[j
] = j
;
497 tx_pool
->consumer_index
= 0;
498 tx_pool
->producer_index
= 0;
504 static void release_error_buffers(struct ibmvnic_adapter
*adapter
)
506 struct device
*dev
= &adapter
->vdev
->dev
;
507 struct ibmvnic_error_buff
*error_buff
, *tmp
;
510 spin_lock_irqsave(&adapter
->error_list_lock
, flags
);
511 list_for_each_entry_safe(error_buff
, tmp
, &adapter
->errors
, list
) {
512 list_del(&error_buff
->list
);
513 dma_unmap_single(dev
, error_buff
->dma
, error_buff
->len
,
515 kfree(error_buff
->buff
);
518 spin_unlock_irqrestore(&adapter
->error_list_lock
, flags
);
521 static int ibmvnic_login(struct net_device
*netdev
)
523 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
524 unsigned long timeout
= msecs_to_jiffies(30000);
525 struct device
*dev
= &adapter
->vdev
->dev
;
528 if (adapter
->renegotiate
) {
529 adapter
->renegotiate
= false;
530 release_sub_crqs(adapter
);
532 reinit_completion(&adapter
->init_done
);
533 send_cap_queries(adapter
);
534 if (!wait_for_completion_timeout(&adapter
->init_done
,
536 dev_err(dev
, "Capabilities query timeout\n");
541 reinit_completion(&adapter
->init_done
);
543 if (!wait_for_completion_timeout(&adapter
->init_done
,
545 dev_err(dev
, "Login timeout\n");
548 } while (adapter
->renegotiate
);
553 static void release_resources(struct ibmvnic_adapter
*adapter
)
557 release_tx_pools(adapter
);
558 release_rx_pools(adapter
);
560 release_stats_token(adapter
);
561 release_error_buffers(adapter
);
564 for (i
= 0; i
< adapter
->req_rx_queues
; i
++) {
565 if (&adapter
->napi
[i
])
566 netif_napi_del(&adapter
->napi
[i
]);
571 static int set_link_state(struct ibmvnic_adapter
*adapter
, u8 link_state
)
573 struct net_device
*netdev
= adapter
->netdev
;
574 unsigned long timeout
= msecs_to_jiffies(30000);
575 union ibmvnic_crq crq
;
579 netdev_err(netdev
, "setting link state %d\n", link_state
);
580 memset(&crq
, 0, sizeof(crq
));
581 crq
.logical_link_state
.first
= IBMVNIC_CRQ_CMD
;
582 crq
.logical_link_state
.cmd
= LOGICAL_LINK_STATE
;
583 crq
.logical_link_state
.link_state
= link_state
;
588 reinit_completion(&adapter
->init_done
);
589 rc
= ibmvnic_send_crq(adapter
, &crq
);
591 netdev_err(netdev
, "Failed to set link state\n");
595 if (!wait_for_completion_timeout(&adapter
->init_done
,
597 netdev_err(netdev
, "timeout setting link state\n");
601 if (adapter
->init_done_rc
== 1) {
602 /* Partuial success, delay and re-send */
611 static int set_real_num_queues(struct net_device
*netdev
)
613 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
616 rc
= netif_set_real_num_tx_queues(netdev
, adapter
->req_tx_queues
);
618 netdev_err(netdev
, "failed to set the number of tx queues\n");
622 rc
= netif_set_real_num_rx_queues(netdev
, adapter
->req_rx_queues
);
624 netdev_err(netdev
, "failed to set the number of rx queues\n");
629 static int init_resources(struct ibmvnic_adapter
*adapter
)
631 struct net_device
*netdev
= adapter
->netdev
;
634 rc
= set_real_num_queues(netdev
);
638 rc
= init_sub_crq_irqs(adapter
);
640 netdev_err(netdev
, "failed to initialize sub crq irqs\n");
644 rc
= init_stats_token(adapter
);
649 adapter
->napi
= kcalloc(adapter
->req_rx_queues
,
650 sizeof(struct napi_struct
), GFP_KERNEL
);
654 for (i
= 0; i
< adapter
->req_rx_queues
; i
++) {
655 netif_napi_add(netdev
, &adapter
->napi
[i
], ibmvnic_poll
,
659 send_map_query(adapter
);
661 rc
= init_rx_pools(netdev
);
665 rc
= init_tx_pools(netdev
);
669 static int __ibmvnic_open(struct net_device
*netdev
)
671 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
672 enum vnic_state prev_state
= adapter
->state
;
675 adapter
->state
= VNIC_OPENING
;
676 replenish_pools(adapter
);
678 for (i
= 0; i
< adapter
->req_rx_queues
; i
++)
679 napi_enable(&adapter
->napi
[i
]);
681 /* We're ready to receive frames, enable the sub-crq interrupts and
682 * set the logical link state to up
684 for (i
= 0; i
< adapter
->req_rx_queues
; i
++) {
685 if (prev_state
== VNIC_CLOSED
)
686 enable_irq(adapter
->rx_scrq
[i
]->irq
);
688 enable_scrq_irq(adapter
, adapter
->rx_scrq
[i
]);
691 for (i
= 0; i
< adapter
->req_tx_queues
; i
++) {
692 if (prev_state
== VNIC_CLOSED
)
693 enable_irq(adapter
->tx_scrq
[i
]->irq
);
695 enable_scrq_irq(adapter
, adapter
->tx_scrq
[i
]);
698 rc
= set_link_state(adapter
, IBMVNIC_LOGICAL_LNK_UP
);
700 for (i
= 0; i
< adapter
->req_rx_queues
; i
++)
701 napi_disable(&adapter
->napi
[i
]);
702 release_resources(adapter
);
706 netif_tx_start_all_queues(netdev
);
708 if (prev_state
== VNIC_CLOSED
) {
709 for (i
= 0; i
< adapter
->req_rx_queues
; i
++)
710 napi_schedule(&adapter
->napi
[i
]);
713 adapter
->state
= VNIC_OPEN
;
717 static int ibmvnic_open(struct net_device
*netdev
)
719 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
722 mutex_lock(&adapter
->reset_lock
);
724 if (adapter
->state
!= VNIC_CLOSED
) {
725 rc
= ibmvnic_login(netdev
);
727 mutex_unlock(&adapter
->reset_lock
);
731 rc
= init_resources(adapter
);
733 netdev_err(netdev
, "failed to initialize resources\n");
734 release_resources(adapter
);
735 mutex_unlock(&adapter
->reset_lock
);
740 rc
= __ibmvnic_open(netdev
);
741 mutex_unlock(&adapter
->reset_lock
);
746 static void clean_tx_pools(struct ibmvnic_adapter
*adapter
)
748 struct ibmvnic_tx_pool
*tx_pool
;
753 if (!adapter
->tx_pool
)
756 tx_scrqs
= be32_to_cpu(adapter
->login_rsp_buf
->num_txsubm_subcrqs
);
757 tx_entries
= adapter
->req_tx_entries_per_subcrq
;
759 /* Free any remaining skbs in the tx buffer pools */
760 for (i
= 0; i
< tx_scrqs
; i
++) {
761 tx_pool
= &adapter
->tx_pool
[i
];
765 for (j
= 0; j
< tx_entries
; j
++) {
766 if (tx_pool
->tx_buff
[j
].skb
) {
767 dev_kfree_skb_any(tx_pool
->tx_buff
[j
].skb
);
768 tx_pool
->tx_buff
[j
].skb
= NULL
;
774 static int __ibmvnic_close(struct net_device
*netdev
)
776 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
780 adapter
->state
= VNIC_CLOSING
;
781 netif_tx_stop_all_queues(netdev
);
784 for (i
= 0; i
< adapter
->req_rx_queues
; i
++)
785 napi_disable(&adapter
->napi
[i
]);
788 clean_tx_pools(adapter
);
790 if (adapter
->tx_scrq
) {
791 for (i
= 0; i
< adapter
->req_tx_queues
; i
++)
792 if (adapter
->tx_scrq
[i
]->irq
)
793 disable_irq(adapter
->tx_scrq
[i
]->irq
);
796 rc
= set_link_state(adapter
, IBMVNIC_LOGICAL_LNK_DN
);
800 if (adapter
->rx_scrq
) {
801 for (i
= 0; i
< adapter
->req_rx_queues
; i
++) {
804 while (pending_scrq(adapter
, adapter
->rx_scrq
[i
])) {
812 if (adapter
->rx_scrq
[i
]->irq
)
813 disable_irq(adapter
->rx_scrq
[i
]->irq
);
817 adapter
->state
= VNIC_CLOSED
;
821 static int ibmvnic_close(struct net_device
*netdev
)
823 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
826 mutex_lock(&adapter
->reset_lock
);
827 rc
= __ibmvnic_close(netdev
);
828 mutex_unlock(&adapter
->reset_lock
);
834 * build_hdr_data - creates L2/L3/L4 header data buffer
835 * @hdr_field - bitfield determining needed headers
836 * @skb - socket buffer
837 * @hdr_len - array of header lengths
838 * @tot_len - total length of data
840 * Reads hdr_field to determine which headers are needed by firmware.
841 * Builds a buffer containing these headers. Saves individual header
842 * lengths and total buffer length to be used to build descriptors.
844 static int build_hdr_data(u8 hdr_field
, struct sk_buff
*skb
,
845 int *hdr_len
, u8
*hdr_data
)
850 hdr_len
[0] = sizeof(struct ethhdr
);
852 if (skb
->protocol
== htons(ETH_P_IP
)) {
853 hdr_len
[1] = ip_hdr(skb
)->ihl
* 4;
854 if (ip_hdr(skb
)->protocol
== IPPROTO_TCP
)
855 hdr_len
[2] = tcp_hdrlen(skb
);
856 else if (ip_hdr(skb
)->protocol
== IPPROTO_UDP
)
857 hdr_len
[2] = sizeof(struct udphdr
);
858 } else if (skb
->protocol
== htons(ETH_P_IPV6
)) {
859 hdr_len
[1] = sizeof(struct ipv6hdr
);
860 if (ipv6_hdr(skb
)->nexthdr
== IPPROTO_TCP
)
861 hdr_len
[2] = tcp_hdrlen(skb
);
862 else if (ipv6_hdr(skb
)->nexthdr
== IPPROTO_UDP
)
863 hdr_len
[2] = sizeof(struct udphdr
);
866 memset(hdr_data
, 0, 120);
867 if ((hdr_field
>> 6) & 1) {
868 hdr
= skb_mac_header(skb
);
869 memcpy(hdr_data
, hdr
, hdr_len
[0]);
873 if ((hdr_field
>> 5) & 1) {
874 hdr
= skb_network_header(skb
);
875 memcpy(hdr_data
+ len
, hdr
, hdr_len
[1]);
879 if ((hdr_field
>> 4) & 1) {
880 hdr
= skb_transport_header(skb
);
881 memcpy(hdr_data
+ len
, hdr
, hdr_len
[2]);
888 * create_hdr_descs - create header and header extension descriptors
889 * @hdr_field - bitfield determining needed headers
890 * @data - buffer containing header data
891 * @len - length of data buffer
892 * @hdr_len - array of individual header lengths
893 * @scrq_arr - descriptor array
895 * Creates header and, if needed, header extension descriptors and
896 * places them in a descriptor array, scrq_arr
899 static void create_hdr_descs(u8 hdr_field
, u8
*hdr_data
, int len
, int *hdr_len
,
900 union sub_crq
*scrq_arr
)
902 union sub_crq hdr_desc
;
907 while (tmp_len
> 0) {
908 cur
= hdr_data
+ len
- tmp_len
;
910 memset(&hdr_desc
, 0, sizeof(hdr_desc
));
911 if (cur
!= hdr_data
) {
912 data
= hdr_desc
.hdr_ext
.data
;
913 tmp
= tmp_len
> 29 ? 29 : tmp_len
;
914 hdr_desc
.hdr_ext
.first
= IBMVNIC_CRQ_CMD
;
915 hdr_desc
.hdr_ext
.type
= IBMVNIC_HDR_EXT_DESC
;
916 hdr_desc
.hdr_ext
.len
= tmp
;
918 data
= hdr_desc
.hdr
.data
;
919 tmp
= tmp_len
> 24 ? 24 : tmp_len
;
920 hdr_desc
.hdr
.first
= IBMVNIC_CRQ_CMD
;
921 hdr_desc
.hdr
.type
= IBMVNIC_HDR_DESC
;
922 hdr_desc
.hdr
.len
= tmp
;
923 hdr_desc
.hdr
.l2_len
= (u8
)hdr_len
[0];
924 hdr_desc
.hdr
.l3_len
= cpu_to_be16((u16
)hdr_len
[1]);
925 hdr_desc
.hdr
.l4_len
= (u8
)hdr_len
[2];
926 hdr_desc
.hdr
.flag
= hdr_field
<< 1;
928 memcpy(data
, cur
, tmp
);
930 *scrq_arr
= hdr_desc
;
936 * build_hdr_descs_arr - build a header descriptor array
937 * @skb - socket buffer
938 * @num_entries - number of descriptors to be sent
939 * @subcrq - first TX descriptor
940 * @hdr_field - bit field determining which headers will be sent
942 * This function will build a TX descriptor array with applicable
943 * L2/L3/L4 packet header descriptors to be sent by send_subcrq_indirect.
946 static void build_hdr_descs_arr(struct ibmvnic_tx_buff
*txbuff
,
947 int *num_entries
, u8 hdr_field
)
949 int hdr_len
[3] = {0, 0, 0};
951 u8
*hdr_data
= txbuff
->hdr_data
;
953 tot_len
= build_hdr_data(hdr_field
, txbuff
->skb
, hdr_len
,
958 num_entries
+= len
% 29 ? len
/ 29 + 1 : len
/ 29;
959 create_hdr_descs(hdr_field
, hdr_data
, tot_len
, hdr_len
,
960 txbuff
->indir_arr
+ 1);
963 static int ibmvnic_xmit(struct sk_buff
*skb
, struct net_device
*netdev
)
965 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
966 int queue_num
= skb_get_queue_mapping(skb
);
967 u8
*hdrs
= (u8
*)&adapter
->tx_rx_desc_req
;
968 struct device
*dev
= &adapter
->vdev
->dev
;
969 struct ibmvnic_tx_buff
*tx_buff
= NULL
;
970 struct ibmvnic_sub_crq_queue
*tx_scrq
;
971 struct ibmvnic_tx_pool
*tx_pool
;
972 unsigned int tx_send_failed
= 0;
973 unsigned int tx_map_failed
= 0;
974 unsigned int tx_dropped
= 0;
975 unsigned int tx_packets
= 0;
976 unsigned int tx_bytes
= 0;
977 dma_addr_t data_dma_addr
;
978 struct netdev_queue
*txq
;
979 unsigned long lpar_rc
;
980 union sub_crq tx_crq
;
988 if (adapter
->resetting
) {
989 if (!netif_subqueue_stopped(netdev
, skb
))
990 netif_stop_subqueue(netdev
, queue_num
);
991 dev_kfree_skb_any(skb
);
999 tx_pool
= &adapter
->tx_pool
[queue_num
];
1000 tx_scrq
= adapter
->tx_scrq
[queue_num
];
1001 txq
= netdev_get_tx_queue(netdev
, skb_get_queue_mapping(skb
));
1002 handle_array
= (u64
*)((u8
*)(adapter
->login_rsp_buf
) +
1003 be32_to_cpu(adapter
->login_rsp_buf
->off_txsubm_subcrqs
));
1005 index
= tx_pool
->free_map
[tx_pool
->consumer_index
];
1006 offset
= index
* adapter
->req_mtu
;
1007 dst
= tx_pool
->long_term_buff
.buff
+ offset
;
1008 memset(dst
, 0, adapter
->req_mtu
);
1009 skb_copy_from_linear_data(skb
, dst
, skb
->len
);
1010 data_dma_addr
= tx_pool
->long_term_buff
.addr
+ offset
;
1012 tx_pool
->consumer_index
=
1013 (tx_pool
->consumer_index
+ 1) %
1014 adapter
->req_tx_entries_per_subcrq
;
1016 tx_buff
= &tx_pool
->tx_buff
[index
];
1018 tx_buff
->data_dma
[0] = data_dma_addr
;
1019 tx_buff
->data_len
[0] = skb
->len
;
1020 tx_buff
->index
= index
;
1021 tx_buff
->pool_index
= queue_num
;
1022 tx_buff
->last_frag
= true;
1024 memset(&tx_crq
, 0, sizeof(tx_crq
));
1025 tx_crq
.v1
.first
= IBMVNIC_CRQ_CMD
;
1026 tx_crq
.v1
.type
= IBMVNIC_TX_DESC
;
1027 tx_crq
.v1
.n_crq_elem
= 1;
1028 tx_crq
.v1
.n_sge
= 1;
1029 tx_crq
.v1
.flags1
= IBMVNIC_TX_COMP_NEEDED
;
1030 tx_crq
.v1
.correlator
= cpu_to_be32(index
);
1031 tx_crq
.v1
.dma_reg
= cpu_to_be16(tx_pool
->long_term_buff
.map_id
);
1032 tx_crq
.v1
.sge_len
= cpu_to_be32(skb
->len
);
1033 tx_crq
.v1
.ioba
= cpu_to_be64(data_dma_addr
);
1035 if (adapter
->vlan_header_insertion
) {
1036 tx_crq
.v1
.flags2
|= IBMVNIC_TX_VLAN_INSERT
;
1037 tx_crq
.v1
.vlan_id
= cpu_to_be16(skb
->vlan_tci
);
1040 if (skb
->protocol
== htons(ETH_P_IP
)) {
1041 if (ip_hdr(skb
)->version
== 4)
1042 tx_crq
.v1
.flags1
|= IBMVNIC_TX_PROT_IPV4
;
1043 else if (ip_hdr(skb
)->version
== 6)
1044 tx_crq
.v1
.flags1
|= IBMVNIC_TX_PROT_IPV6
;
1046 if (ip_hdr(skb
)->protocol
== IPPROTO_TCP
)
1047 tx_crq
.v1
.flags1
|= IBMVNIC_TX_PROT_TCP
;
1048 else if (ip_hdr(skb
)->protocol
!= IPPROTO_TCP
)
1049 tx_crq
.v1
.flags1
|= IBMVNIC_TX_PROT_UDP
;
1052 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
1053 tx_crq
.v1
.flags1
|= IBMVNIC_TX_CHKSUM_OFFLOAD
;
1056 /* determine if l2/3/4 headers are sent to firmware */
1057 if ((*hdrs
>> 7) & 1 &&
1058 (skb
->protocol
== htons(ETH_P_IP
) ||
1059 skb
->protocol
== htons(ETH_P_IPV6
))) {
1060 build_hdr_descs_arr(tx_buff
, &num_entries
, *hdrs
);
1061 tx_crq
.v1
.n_crq_elem
= num_entries
;
1062 tx_buff
->indir_arr
[0] = tx_crq
;
1063 tx_buff
->indir_dma
= dma_map_single(dev
, tx_buff
->indir_arr
,
1064 sizeof(tx_buff
->indir_arr
),
1066 if (dma_mapping_error(dev
, tx_buff
->indir_dma
)) {
1067 dev_kfree_skb_any(skb
);
1068 tx_buff
->skb
= NULL
;
1069 if (!firmware_has_feature(FW_FEATURE_CMO
))
1070 dev_err(dev
, "tx: unable to map descriptor array\n");
1076 lpar_rc
= send_subcrq_indirect(adapter
, handle_array
[queue_num
],
1077 (u64
)tx_buff
->indir_dma
,
1080 lpar_rc
= send_subcrq(adapter
, handle_array
[queue_num
],
1083 if (lpar_rc
!= H_SUCCESS
) {
1084 dev_err(dev
, "tx failed with code %ld\n", lpar_rc
);
1086 if (tx_pool
->consumer_index
== 0)
1087 tx_pool
->consumer_index
=
1088 adapter
->req_tx_entries_per_subcrq
- 1;
1090 tx_pool
->consumer_index
--;
1092 dev_kfree_skb_any(skb
);
1093 tx_buff
->skb
= NULL
;
1095 if (lpar_rc
== H_CLOSED
)
1096 netif_stop_subqueue(netdev
, queue_num
);
1104 if (atomic_inc_return(&tx_scrq
->used
)
1105 >= adapter
->req_tx_entries_per_subcrq
) {
1106 netdev_info(netdev
, "Stopping queue %d\n", queue_num
);
1107 netif_stop_subqueue(netdev
, queue_num
);
1111 tx_bytes
+= skb
->len
;
1112 txq
->trans_start
= jiffies
;
1116 netdev
->stats
.tx_dropped
+= tx_dropped
;
1117 netdev
->stats
.tx_bytes
+= tx_bytes
;
1118 netdev
->stats
.tx_packets
+= tx_packets
;
1119 adapter
->tx_send_failed
+= tx_send_failed
;
1120 adapter
->tx_map_failed
+= tx_map_failed
;
1125 static void ibmvnic_set_multi(struct net_device
*netdev
)
1127 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
1128 struct netdev_hw_addr
*ha
;
1129 union ibmvnic_crq crq
;
1131 memset(&crq
, 0, sizeof(crq
));
1132 crq
.request_capability
.first
= IBMVNIC_CRQ_CMD
;
1133 crq
.request_capability
.cmd
= REQUEST_CAPABILITY
;
1135 if (netdev
->flags
& IFF_PROMISC
) {
1136 if (!adapter
->promisc_supported
)
1139 if (netdev
->flags
& IFF_ALLMULTI
) {
1140 /* Accept all multicast */
1141 memset(&crq
, 0, sizeof(crq
));
1142 crq
.multicast_ctrl
.first
= IBMVNIC_CRQ_CMD
;
1143 crq
.multicast_ctrl
.cmd
= MULTICAST_CTRL
;
1144 crq
.multicast_ctrl
.flags
= IBMVNIC_ENABLE_ALL
;
1145 ibmvnic_send_crq(adapter
, &crq
);
1146 } else if (netdev_mc_empty(netdev
)) {
1147 /* Reject all multicast */
1148 memset(&crq
, 0, sizeof(crq
));
1149 crq
.multicast_ctrl
.first
= IBMVNIC_CRQ_CMD
;
1150 crq
.multicast_ctrl
.cmd
= MULTICAST_CTRL
;
1151 crq
.multicast_ctrl
.flags
= IBMVNIC_DISABLE_ALL
;
1152 ibmvnic_send_crq(adapter
, &crq
);
1154 /* Accept one or more multicast(s) */
1155 netdev_for_each_mc_addr(ha
, netdev
) {
1156 memset(&crq
, 0, sizeof(crq
));
1157 crq
.multicast_ctrl
.first
= IBMVNIC_CRQ_CMD
;
1158 crq
.multicast_ctrl
.cmd
= MULTICAST_CTRL
;
1159 crq
.multicast_ctrl
.flags
= IBMVNIC_ENABLE_MC
;
1160 ether_addr_copy(&crq
.multicast_ctrl
.mac_addr
[0],
1162 ibmvnic_send_crq(adapter
, &crq
);
1168 static int ibmvnic_set_mac(struct net_device
*netdev
, void *p
)
1170 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
1171 struct sockaddr
*addr
= p
;
1172 union ibmvnic_crq crq
;
1174 if (!is_valid_ether_addr(addr
->sa_data
))
1175 return -EADDRNOTAVAIL
;
1177 memset(&crq
, 0, sizeof(crq
));
1178 crq
.change_mac_addr
.first
= IBMVNIC_CRQ_CMD
;
1179 crq
.change_mac_addr
.cmd
= CHANGE_MAC_ADDR
;
1180 ether_addr_copy(&crq
.change_mac_addr
.mac_addr
[0], addr
->sa_data
);
1181 ibmvnic_send_crq(adapter
, &crq
);
1182 /* netdev->dev_addr is changed in handle_change_mac_rsp function */
1187 * do_reset returns zero if we are able to keep processing reset events, or
1188 * non-zero if we hit a fatal error and must halt.
1190 static int do_reset(struct ibmvnic_adapter
*adapter
,
1191 struct ibmvnic_rwi
*rwi
, u32 reset_state
)
1193 struct net_device
*netdev
= adapter
->netdev
;
1196 netif_carrier_off(netdev
);
1197 adapter
->reset_reason
= rwi
->reset_reason
;
1199 if (rwi
->reset_reason
== VNIC_RESET_MOBILITY
) {
1200 rc
= ibmvnic_reenable_crq_queue(adapter
);
1205 rc
= __ibmvnic_close(netdev
);
1209 /* remove the closed state so when we call open it appears
1210 * we are coming from the probed state.
1212 adapter
->state
= VNIC_PROBED
;
1214 release_resources(adapter
);
1215 release_sub_crqs(adapter
);
1216 release_crq_queue(adapter
);
1218 rc
= ibmvnic_init(adapter
);
1222 /* If the adapter was in PROBE state prior to the reset, exit here. */
1223 if (reset_state
== VNIC_PROBED
)
1226 rc
= ibmvnic_login(netdev
);
1228 adapter
->state
= VNIC_PROBED
;
1233 rc
= init_resources(adapter
);
1238 if (reset_state
== VNIC_CLOSED
)
1241 rc
= __ibmvnic_open(netdev
);
1243 if (list_empty(&adapter
->rwi_list
))
1244 adapter
->state
= VNIC_CLOSED
;
1246 adapter
->state
= reset_state
;
1251 netif_carrier_on(netdev
);
1254 for (i
= 0; i
< adapter
->req_rx_queues
; i
++)
1255 napi_schedule(&adapter
->napi
[i
]);
1260 static struct ibmvnic_rwi
*get_next_rwi(struct ibmvnic_adapter
*adapter
)
1262 struct ibmvnic_rwi
*rwi
;
1264 mutex_lock(&adapter
->rwi_lock
);
1266 if (!list_empty(&adapter
->rwi_list
)) {
1267 rwi
= list_first_entry(&adapter
->rwi_list
, struct ibmvnic_rwi
,
1269 list_del(&rwi
->list
);
1274 mutex_unlock(&adapter
->rwi_lock
);
1278 static void free_all_rwi(struct ibmvnic_adapter
*adapter
)
1280 struct ibmvnic_rwi
*rwi
;
1282 rwi
= get_next_rwi(adapter
);
1285 rwi
= get_next_rwi(adapter
);
1289 static void __ibmvnic_reset(struct work_struct
*work
)
1291 struct ibmvnic_rwi
*rwi
;
1292 struct ibmvnic_adapter
*adapter
;
1293 struct net_device
*netdev
;
1297 adapter
= container_of(work
, struct ibmvnic_adapter
, ibmvnic_reset
);
1298 netdev
= adapter
->netdev
;
1300 mutex_lock(&adapter
->reset_lock
);
1301 adapter
->resetting
= true;
1302 reset_state
= adapter
->state
;
1304 rwi
= get_next_rwi(adapter
);
1306 rc
= do_reset(adapter
, rwi
, reset_state
);
1311 rwi
= get_next_rwi(adapter
);
1315 free_all_rwi(adapter
);
1319 adapter
->resetting
= false;
1320 mutex_unlock(&adapter
->reset_lock
);
1323 static void ibmvnic_reset(struct ibmvnic_adapter
*adapter
,
1324 enum ibmvnic_reset_reason reason
)
1326 struct ibmvnic_rwi
*rwi
, *tmp
;
1327 struct net_device
*netdev
= adapter
->netdev
;
1328 struct list_head
*entry
;
1330 if (adapter
->state
== VNIC_REMOVING
||
1331 adapter
->state
== VNIC_REMOVED
) {
1332 netdev_dbg(netdev
, "Adapter removing, skipping reset\n");
1336 mutex_lock(&adapter
->rwi_lock
);
1338 list_for_each(entry
, &adapter
->rwi_list
) {
1339 tmp
= list_entry(entry
, struct ibmvnic_rwi
, list
);
1340 if (tmp
->reset_reason
== reason
) {
1341 netdev_err(netdev
, "Matching reset found, skipping\n");
1342 mutex_unlock(&adapter
->rwi_lock
);
1347 rwi
= kzalloc(sizeof(*rwi
), GFP_KERNEL
);
1349 mutex_unlock(&adapter
->rwi_lock
);
1350 ibmvnic_close(netdev
);
1354 rwi
->reset_reason
= reason
;
1355 list_add_tail(&rwi
->list
, &adapter
->rwi_list
);
1356 mutex_unlock(&adapter
->rwi_lock
);
1357 schedule_work(&adapter
->ibmvnic_reset
);
1360 static void ibmvnic_tx_timeout(struct net_device
*dev
)
1362 struct ibmvnic_adapter
*adapter
= netdev_priv(dev
);
1364 ibmvnic_reset(adapter
, VNIC_RESET_TIMEOUT
);
1367 static void remove_buff_from_pool(struct ibmvnic_adapter
*adapter
,
1368 struct ibmvnic_rx_buff
*rx_buff
)
1370 struct ibmvnic_rx_pool
*pool
= &adapter
->rx_pool
[rx_buff
->pool_index
];
1372 rx_buff
->skb
= NULL
;
1374 pool
->free_map
[pool
->next_alloc
] = (int)(rx_buff
- pool
->rx_buff
);
1375 pool
->next_alloc
= (pool
->next_alloc
+ 1) % pool
->size
;
1377 atomic_dec(&pool
->available
);
1380 static int ibmvnic_poll(struct napi_struct
*napi
, int budget
)
1382 struct net_device
*netdev
= napi
->dev
;
1383 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
1384 int scrq_num
= (int)(napi
- adapter
->napi
);
1385 int frames_processed
= 0;
1387 while (frames_processed
< budget
) {
1388 struct sk_buff
*skb
;
1389 struct ibmvnic_rx_buff
*rx_buff
;
1390 union sub_crq
*next
;
1395 if (!pending_scrq(adapter
, adapter
->rx_scrq
[scrq_num
]))
1397 next
= ibmvnic_next_scrq(adapter
, adapter
->rx_scrq
[scrq_num
]);
1399 (struct ibmvnic_rx_buff
*)be64_to_cpu(next
->
1400 rx_comp
.correlator
);
1401 /* do error checking */
1402 if (next
->rx_comp
.rc
) {
1403 netdev_err(netdev
, "rx error %x\n", next
->rx_comp
.rc
);
1404 /* free the entry */
1405 next
->rx_comp
.first
= 0;
1406 remove_buff_from_pool(adapter
, rx_buff
);
1410 length
= be32_to_cpu(next
->rx_comp
.len
);
1411 offset
= be16_to_cpu(next
->rx_comp
.off_frame_data
);
1412 flags
= next
->rx_comp
.flags
;
1414 skb_copy_to_linear_data(skb
, rx_buff
->data
+ offset
,
1417 /* VLAN Header has been stripped by the system firmware and
1418 * needs to be inserted by the driver
1420 if (adapter
->rx_vlan_header_insertion
&&
1421 (flags
& IBMVNIC_VLAN_STRIPPED
))
1422 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
),
1423 ntohs(next
->rx_comp
.vlan_tci
));
1425 /* free the entry */
1426 next
->rx_comp
.first
= 0;
1427 remove_buff_from_pool(adapter
, rx_buff
);
1429 skb_put(skb
, length
);
1430 skb
->protocol
= eth_type_trans(skb
, netdev
);
1431 skb_record_rx_queue(skb
, scrq_num
);
1433 if (flags
& IBMVNIC_IP_CHKSUM_GOOD
&&
1434 flags
& IBMVNIC_TCP_UDP_CHKSUM_GOOD
) {
1435 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
1439 napi_gro_receive(napi
, skb
); /* send it up */
1440 netdev
->stats
.rx_packets
++;
1441 netdev
->stats
.rx_bytes
+= length
;
1444 replenish_rx_pool(adapter
, &adapter
->rx_pool
[scrq_num
]);
1446 if (frames_processed
< budget
) {
1447 enable_scrq_irq(adapter
, adapter
->rx_scrq
[scrq_num
]);
1448 napi_complete_done(napi
, frames_processed
);
1449 if (pending_scrq(adapter
, adapter
->rx_scrq
[scrq_num
]) &&
1450 napi_reschedule(napi
)) {
1451 disable_scrq_irq(adapter
, adapter
->rx_scrq
[scrq_num
]);
1455 return frames_processed
;
1458 #ifdef CONFIG_NET_POLL_CONTROLLER
1459 static void ibmvnic_netpoll_controller(struct net_device
*dev
)
1461 struct ibmvnic_adapter
*adapter
= netdev_priv(dev
);
1464 replenish_pools(netdev_priv(dev
));
1465 for (i
= 0; i
< adapter
->req_rx_queues
; i
++)
1466 ibmvnic_interrupt_rx(adapter
->rx_scrq
[i
]->irq
,
1467 adapter
->rx_scrq
[i
]);
1471 static const struct net_device_ops ibmvnic_netdev_ops
= {
1472 .ndo_open
= ibmvnic_open
,
1473 .ndo_stop
= ibmvnic_close
,
1474 .ndo_start_xmit
= ibmvnic_xmit
,
1475 .ndo_set_rx_mode
= ibmvnic_set_multi
,
1476 .ndo_set_mac_address
= ibmvnic_set_mac
,
1477 .ndo_validate_addr
= eth_validate_addr
,
1478 .ndo_tx_timeout
= ibmvnic_tx_timeout
,
1479 #ifdef CONFIG_NET_POLL_CONTROLLER
1480 .ndo_poll_controller
= ibmvnic_netpoll_controller
,
1484 /* ethtool functions */
1486 static int ibmvnic_get_link_ksettings(struct net_device
*netdev
,
1487 struct ethtool_link_ksettings
*cmd
)
1489 u32 supported
, advertising
;
1491 supported
= (SUPPORTED_1000baseT_Full
| SUPPORTED_Autoneg
|
1493 advertising
= (ADVERTISED_1000baseT_Full
| ADVERTISED_Autoneg
|
1495 cmd
->base
.speed
= SPEED_1000
;
1496 cmd
->base
.duplex
= DUPLEX_FULL
;
1497 cmd
->base
.port
= PORT_FIBRE
;
1498 cmd
->base
.phy_address
= 0;
1499 cmd
->base
.autoneg
= AUTONEG_ENABLE
;
1501 ethtool_convert_legacy_u32_to_link_mode(cmd
->link_modes
.supported
,
1503 ethtool_convert_legacy_u32_to_link_mode(cmd
->link_modes
.advertising
,
1509 static void ibmvnic_get_drvinfo(struct net_device
*dev
,
1510 struct ethtool_drvinfo
*info
)
1512 strlcpy(info
->driver
, ibmvnic_driver_name
, sizeof(info
->driver
));
1513 strlcpy(info
->version
, IBMVNIC_DRIVER_VERSION
, sizeof(info
->version
));
1516 static u32
ibmvnic_get_msglevel(struct net_device
*netdev
)
1518 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
1520 return adapter
->msg_enable
;
1523 static void ibmvnic_set_msglevel(struct net_device
*netdev
, u32 data
)
1525 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
1527 adapter
->msg_enable
= data
;
1530 static u32
ibmvnic_get_link(struct net_device
*netdev
)
1532 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
1534 /* Don't need to send a query because we request a logical link up at
1535 * init and then we wait for link state indications
1537 return adapter
->logical_link_state
;
1540 static void ibmvnic_get_ringparam(struct net_device
*netdev
,
1541 struct ethtool_ringparam
*ring
)
1543 ring
->rx_max_pending
= 0;
1544 ring
->tx_max_pending
= 0;
1545 ring
->rx_mini_max_pending
= 0;
1546 ring
->rx_jumbo_max_pending
= 0;
1547 ring
->rx_pending
= 0;
1548 ring
->tx_pending
= 0;
1549 ring
->rx_mini_pending
= 0;
1550 ring
->rx_jumbo_pending
= 0;
1553 static void ibmvnic_get_strings(struct net_device
*dev
, u32 stringset
, u8
*data
)
1557 if (stringset
!= ETH_SS_STATS
)
1560 for (i
= 0; i
< ARRAY_SIZE(ibmvnic_stats
); i
++, data
+= ETH_GSTRING_LEN
)
1561 memcpy(data
, ibmvnic_stats
[i
].name
, ETH_GSTRING_LEN
);
1564 static int ibmvnic_get_sset_count(struct net_device
*dev
, int sset
)
1568 return ARRAY_SIZE(ibmvnic_stats
);
1574 static void ibmvnic_get_ethtool_stats(struct net_device
*dev
,
1575 struct ethtool_stats
*stats
, u64
*data
)
1577 struct ibmvnic_adapter
*adapter
= netdev_priv(dev
);
1578 union ibmvnic_crq crq
;
1581 memset(&crq
, 0, sizeof(crq
));
1582 crq
.request_statistics
.first
= IBMVNIC_CRQ_CMD
;
1583 crq
.request_statistics
.cmd
= REQUEST_STATISTICS
;
1584 crq
.request_statistics
.ioba
= cpu_to_be32(adapter
->stats_token
);
1585 crq
.request_statistics
.len
=
1586 cpu_to_be32(sizeof(struct ibmvnic_statistics
));
1588 /* Wait for data to be written */
1589 init_completion(&adapter
->stats_done
);
1590 ibmvnic_send_crq(adapter
, &crq
);
1591 wait_for_completion(&adapter
->stats_done
);
1593 for (i
= 0; i
< ARRAY_SIZE(ibmvnic_stats
); i
++)
1594 data
[i
] = IBMVNIC_GET_STAT(adapter
, ibmvnic_stats
[i
].offset
);
1597 static const struct ethtool_ops ibmvnic_ethtool_ops
= {
1598 .get_drvinfo
= ibmvnic_get_drvinfo
,
1599 .get_msglevel
= ibmvnic_get_msglevel
,
1600 .set_msglevel
= ibmvnic_set_msglevel
,
1601 .get_link
= ibmvnic_get_link
,
1602 .get_ringparam
= ibmvnic_get_ringparam
,
1603 .get_strings
= ibmvnic_get_strings
,
1604 .get_sset_count
= ibmvnic_get_sset_count
,
1605 .get_ethtool_stats
= ibmvnic_get_ethtool_stats
,
1606 .get_link_ksettings
= ibmvnic_get_link_ksettings
,
1609 /* Routines for managing CRQs/sCRQs */
1611 static void release_sub_crq_queue(struct ibmvnic_adapter
*adapter
,
1612 struct ibmvnic_sub_crq_queue
*scrq
)
1614 struct device
*dev
= &adapter
->vdev
->dev
;
1617 netdev_dbg(adapter
->netdev
, "Releasing sub-CRQ\n");
1619 /* Close the sub-crqs */
1621 rc
= plpar_hcall_norets(H_FREE_SUB_CRQ
,
1622 adapter
->vdev
->unit_address
,
1624 } while (rc
== H_BUSY
|| H_IS_LONG_BUSY(rc
));
1627 netdev_err(adapter
->netdev
,
1628 "Failed to release sub-CRQ %16lx, rc = %ld\n",
1632 dma_unmap_single(dev
, scrq
->msg_token
, 4 * PAGE_SIZE
,
1634 free_pages((unsigned long)scrq
->msgs
, 2);
1638 static struct ibmvnic_sub_crq_queue
*init_sub_crq_queue(struct ibmvnic_adapter
1641 struct device
*dev
= &adapter
->vdev
->dev
;
1642 struct ibmvnic_sub_crq_queue
*scrq
;
1645 scrq
= kzalloc(sizeof(*scrq
), GFP_KERNEL
);
1650 (union sub_crq
*)__get_free_pages(GFP_KERNEL
| __GFP_ZERO
, 2);
1652 dev_warn(dev
, "Couldn't allocate crq queue messages page\n");
1653 goto zero_page_failed
;
1656 scrq
->msg_token
= dma_map_single(dev
, scrq
->msgs
, 4 * PAGE_SIZE
,
1658 if (dma_mapping_error(dev
, scrq
->msg_token
)) {
1659 dev_warn(dev
, "Couldn't map crq queue messages page\n");
1663 rc
= h_reg_sub_crq(adapter
->vdev
->unit_address
, scrq
->msg_token
,
1664 4 * PAGE_SIZE
, &scrq
->crq_num
, &scrq
->hw_irq
);
1666 if (rc
== H_RESOURCE
)
1667 rc
= ibmvnic_reset_crq(adapter
);
1669 if (rc
== H_CLOSED
) {
1670 dev_warn(dev
, "Partner adapter not ready, waiting.\n");
1672 dev_warn(dev
, "Error %d registering sub-crq\n", rc
);
1676 scrq
->adapter
= adapter
;
1677 scrq
->size
= 4 * PAGE_SIZE
/ sizeof(*scrq
->msgs
);
1678 spin_lock_init(&scrq
->lock
);
1680 netdev_dbg(adapter
->netdev
,
1681 "sub-crq initialized, num %lx, hw_irq=%lx, irq=%x\n",
1682 scrq
->crq_num
, scrq
->hw_irq
, scrq
->irq
);
1687 dma_unmap_single(dev
, scrq
->msg_token
, 4 * PAGE_SIZE
,
1690 free_pages((unsigned long)scrq
->msgs
, 2);
1697 static void release_sub_crqs(struct ibmvnic_adapter
*adapter
)
1701 if (adapter
->tx_scrq
) {
1702 for (i
= 0; i
< adapter
->req_tx_queues
; i
++) {
1703 if (!adapter
->tx_scrq
[i
])
1706 if (adapter
->tx_scrq
[i
]->irq
) {
1707 free_irq(adapter
->tx_scrq
[i
]->irq
,
1708 adapter
->tx_scrq
[i
]);
1709 irq_dispose_mapping(adapter
->tx_scrq
[i
]->irq
);
1710 adapter
->tx_scrq
[i
]->irq
= 0;
1713 release_sub_crq_queue(adapter
, adapter
->tx_scrq
[i
]);
1716 kfree(adapter
->tx_scrq
);
1717 adapter
->tx_scrq
= NULL
;
1720 if (adapter
->rx_scrq
) {
1721 for (i
= 0; i
< adapter
->req_rx_queues
; i
++) {
1722 if (!adapter
->rx_scrq
[i
])
1725 if (adapter
->rx_scrq
[i
]->irq
) {
1726 free_irq(adapter
->rx_scrq
[i
]->irq
,
1727 adapter
->rx_scrq
[i
]);
1728 irq_dispose_mapping(adapter
->rx_scrq
[i
]->irq
);
1729 adapter
->rx_scrq
[i
]->irq
= 0;
1732 release_sub_crq_queue(adapter
, adapter
->rx_scrq
[i
]);
1735 kfree(adapter
->rx_scrq
);
1736 adapter
->rx_scrq
= NULL
;
1740 static int disable_scrq_irq(struct ibmvnic_adapter
*adapter
,
1741 struct ibmvnic_sub_crq_queue
*scrq
)
1743 struct device
*dev
= &adapter
->vdev
->dev
;
1746 rc
= plpar_hcall_norets(H_VIOCTL
, adapter
->vdev
->unit_address
,
1747 H_DISABLE_VIO_INTERRUPT
, scrq
->hw_irq
, 0, 0);
1749 dev_err(dev
, "Couldn't disable scrq irq 0x%lx. rc=%ld\n",
1754 static int enable_scrq_irq(struct ibmvnic_adapter
*adapter
,
1755 struct ibmvnic_sub_crq_queue
*scrq
)
1757 struct device
*dev
= &adapter
->vdev
->dev
;
1760 if (scrq
->hw_irq
> 0x100000000ULL
) {
1761 dev_err(dev
, "bad hw_irq = %lx\n", scrq
->hw_irq
);
1765 rc
= plpar_hcall_norets(H_VIOCTL
, adapter
->vdev
->unit_address
,
1766 H_ENABLE_VIO_INTERRUPT
, scrq
->hw_irq
, 0, 0);
1768 dev_err(dev
, "Couldn't enable scrq irq 0x%lx. rc=%ld\n",
1773 static int ibmvnic_complete_tx(struct ibmvnic_adapter
*adapter
,
1774 struct ibmvnic_sub_crq_queue
*scrq
)
1776 struct device
*dev
= &adapter
->vdev
->dev
;
1777 struct ibmvnic_tx_buff
*txbuff
;
1778 union sub_crq
*next
;
1784 while (pending_scrq(adapter
, scrq
)) {
1785 unsigned int pool
= scrq
->pool_index
;
1787 next
= ibmvnic_next_scrq(adapter
, scrq
);
1788 for (i
= 0; i
< next
->tx_comp
.num_comps
; i
++) {
1789 if (next
->tx_comp
.rcs
[i
]) {
1790 dev_err(dev
, "tx error %x\n",
1791 next
->tx_comp
.rcs
[i
]);
1794 index
= be32_to_cpu(next
->tx_comp
.correlators
[i
]);
1795 txbuff
= &adapter
->tx_pool
[pool
].tx_buff
[index
];
1797 for (j
= 0; j
< IBMVNIC_MAX_FRAGS_PER_CRQ
; j
++) {
1798 if (!txbuff
->data_dma
[j
])
1801 txbuff
->data_dma
[j
] = 0;
1803 /* if sub_crq was sent indirectly */
1804 first
= txbuff
->indir_arr
[0].generic
.first
;
1805 if (first
== IBMVNIC_CRQ_CMD
) {
1806 dma_unmap_single(dev
, txbuff
->indir_dma
,
1807 sizeof(txbuff
->indir_arr
),
1811 if (txbuff
->last_frag
) {
1812 dev_kfree_skb_any(txbuff
->skb
);
1816 adapter
->tx_pool
[pool
].free_map
[adapter
->tx_pool
[pool
].
1817 producer_index
] = index
;
1818 adapter
->tx_pool
[pool
].producer_index
=
1819 (adapter
->tx_pool
[pool
].producer_index
+ 1) %
1820 adapter
->req_tx_entries_per_subcrq
;
1822 /* remove tx_comp scrq*/
1823 next
->tx_comp
.first
= 0;
1825 if (atomic_sub_return(next
->tx_comp
.num_comps
, &scrq
->used
) <=
1826 (adapter
->req_tx_entries_per_subcrq
/ 2) &&
1827 __netif_subqueue_stopped(adapter
->netdev
,
1828 scrq
->pool_index
)) {
1829 netif_wake_subqueue(adapter
->netdev
, scrq
->pool_index
);
1830 netdev_info(adapter
->netdev
, "Started queue %d\n",
1835 enable_scrq_irq(adapter
, scrq
);
1837 if (pending_scrq(adapter
, scrq
)) {
1838 disable_scrq_irq(adapter
, scrq
);
1845 static irqreturn_t
ibmvnic_interrupt_tx(int irq
, void *instance
)
1847 struct ibmvnic_sub_crq_queue
*scrq
= instance
;
1848 struct ibmvnic_adapter
*adapter
= scrq
->adapter
;
1850 disable_scrq_irq(adapter
, scrq
);
1851 ibmvnic_complete_tx(adapter
, scrq
);
1856 static irqreturn_t
ibmvnic_interrupt_rx(int irq
, void *instance
)
1858 struct ibmvnic_sub_crq_queue
*scrq
= instance
;
1859 struct ibmvnic_adapter
*adapter
= scrq
->adapter
;
1861 if (napi_schedule_prep(&adapter
->napi
[scrq
->scrq_num
])) {
1862 disable_scrq_irq(adapter
, scrq
);
1863 __napi_schedule(&adapter
->napi
[scrq
->scrq_num
]);
1869 static int init_sub_crq_irqs(struct ibmvnic_adapter
*adapter
)
1871 struct device
*dev
= &adapter
->vdev
->dev
;
1872 struct ibmvnic_sub_crq_queue
*scrq
;
1876 for (i
= 0; i
< adapter
->req_tx_queues
; i
++) {
1877 scrq
= adapter
->tx_scrq
[i
];
1878 scrq
->irq
= irq_create_mapping(NULL
, scrq
->hw_irq
);
1882 dev_err(dev
, "Error mapping irq\n");
1883 goto req_tx_irq_failed
;
1886 rc
= request_irq(scrq
->irq
, ibmvnic_interrupt_tx
,
1887 0, "ibmvnic_tx", scrq
);
1890 dev_err(dev
, "Couldn't register tx irq 0x%x. rc=%d\n",
1892 irq_dispose_mapping(scrq
->irq
);
1893 goto req_rx_irq_failed
;
1897 for (i
= 0; i
< adapter
->req_rx_queues
; i
++) {
1898 scrq
= adapter
->rx_scrq
[i
];
1899 scrq
->irq
= irq_create_mapping(NULL
, scrq
->hw_irq
);
1902 dev_err(dev
, "Error mapping irq\n");
1903 goto req_rx_irq_failed
;
1905 rc
= request_irq(scrq
->irq
, ibmvnic_interrupt_rx
,
1906 0, "ibmvnic_rx", scrq
);
1908 dev_err(dev
, "Couldn't register rx irq 0x%x. rc=%d\n",
1910 irq_dispose_mapping(scrq
->irq
);
1911 goto req_rx_irq_failed
;
1917 for (j
= 0; j
< i
; j
++) {
1918 free_irq(adapter
->rx_scrq
[j
]->irq
, adapter
->rx_scrq
[j
]);
1919 irq_dispose_mapping(adapter
->rx_scrq
[j
]->irq
);
1921 i
= adapter
->req_tx_queues
;
1923 for (j
= 0; j
< i
; j
++) {
1924 free_irq(adapter
->tx_scrq
[j
]->irq
, adapter
->tx_scrq
[j
]);
1925 irq_dispose_mapping(adapter
->rx_scrq
[j
]->irq
);
1927 release_sub_crqs(adapter
);
1931 static int init_sub_crqs(struct ibmvnic_adapter
*adapter
)
1933 struct device
*dev
= &adapter
->vdev
->dev
;
1934 struct ibmvnic_sub_crq_queue
**allqueues
;
1935 int registered_queues
= 0;
1940 total_queues
= adapter
->req_tx_queues
+ adapter
->req_rx_queues
;
1942 allqueues
= kcalloc(total_queues
, sizeof(*allqueues
), GFP_KERNEL
);
1946 for (i
= 0; i
< total_queues
; i
++) {
1947 allqueues
[i
] = init_sub_crq_queue(adapter
);
1948 if (!allqueues
[i
]) {
1949 dev_warn(dev
, "Couldn't allocate all sub-crqs\n");
1952 registered_queues
++;
1955 /* Make sure we were able to register the minimum number of queues */
1956 if (registered_queues
<
1957 adapter
->min_tx_queues
+ adapter
->min_rx_queues
) {
1958 dev_err(dev
, "Fatal: Couldn't init min number of sub-crqs\n");
1962 /* Distribute the failed allocated queues*/
1963 for (i
= 0; i
< total_queues
- registered_queues
+ more
; i
++) {
1964 netdev_dbg(adapter
->netdev
, "Reducing number of queues\n");
1967 if (adapter
->req_rx_queues
> adapter
->min_rx_queues
)
1968 adapter
->req_rx_queues
--;
1973 if (adapter
->req_tx_queues
> adapter
->min_tx_queues
)
1974 adapter
->req_tx_queues
--;
1981 adapter
->tx_scrq
= kcalloc(adapter
->req_tx_queues
,
1982 sizeof(*adapter
->tx_scrq
), GFP_KERNEL
);
1983 if (!adapter
->tx_scrq
)
1986 for (i
= 0; i
< adapter
->req_tx_queues
; i
++) {
1987 adapter
->tx_scrq
[i
] = allqueues
[i
];
1988 adapter
->tx_scrq
[i
]->pool_index
= i
;
1991 adapter
->rx_scrq
= kcalloc(adapter
->req_rx_queues
,
1992 sizeof(*adapter
->rx_scrq
), GFP_KERNEL
);
1993 if (!adapter
->rx_scrq
)
1996 for (i
= 0; i
< adapter
->req_rx_queues
; i
++) {
1997 adapter
->rx_scrq
[i
] = allqueues
[i
+ adapter
->req_tx_queues
];
1998 adapter
->rx_scrq
[i
]->scrq_num
= i
;
2005 kfree(adapter
->tx_scrq
);
2006 adapter
->tx_scrq
= NULL
;
2008 for (i
= 0; i
< registered_queues
; i
++)
2009 release_sub_crq_queue(adapter
, allqueues
[i
]);
2014 static void ibmvnic_send_req_caps(struct ibmvnic_adapter
*adapter
, int retry
)
2016 struct device
*dev
= &adapter
->vdev
->dev
;
2017 union ibmvnic_crq crq
;
2020 /* Sub-CRQ entries are 32 byte long */
2021 int entries_page
= 4 * PAGE_SIZE
/ (sizeof(u64
) * 4);
2023 if (adapter
->min_tx_entries_per_subcrq
> entries_page
||
2024 adapter
->min_rx_add_entries_per_subcrq
> entries_page
) {
2025 dev_err(dev
, "Fatal, invalid entries per sub-crq\n");
2029 /* Get the minimum between the queried max and the entries
2030 * that fit in our PAGE_SIZE
2032 adapter
->req_tx_entries_per_subcrq
=
2033 adapter
->max_tx_entries_per_subcrq
> entries_page
?
2034 entries_page
: adapter
->max_tx_entries_per_subcrq
;
2035 adapter
->req_rx_add_entries_per_subcrq
=
2036 adapter
->max_rx_add_entries_per_subcrq
> entries_page
?
2037 entries_page
: adapter
->max_rx_add_entries_per_subcrq
;
2039 adapter
->req_tx_queues
= adapter
->opt_tx_comp_sub_queues
;
2040 adapter
->req_rx_queues
= adapter
->opt_rx_comp_queues
;
2041 adapter
->req_rx_add_queues
= adapter
->max_rx_add_queues
;
2043 adapter
->req_mtu
= adapter
->netdev
->mtu
+ ETH_HLEN
;
2046 memset(&crq
, 0, sizeof(crq
));
2047 crq
.request_capability
.first
= IBMVNIC_CRQ_CMD
;
2048 crq
.request_capability
.cmd
= REQUEST_CAPABILITY
;
2050 crq
.request_capability
.capability
= cpu_to_be16(REQ_TX_QUEUES
);
2051 crq
.request_capability
.number
= cpu_to_be64(adapter
->req_tx_queues
);
2052 atomic_inc(&adapter
->running_cap_crqs
);
2053 ibmvnic_send_crq(adapter
, &crq
);
2055 crq
.request_capability
.capability
= cpu_to_be16(REQ_RX_QUEUES
);
2056 crq
.request_capability
.number
= cpu_to_be64(adapter
->req_rx_queues
);
2057 atomic_inc(&adapter
->running_cap_crqs
);
2058 ibmvnic_send_crq(adapter
, &crq
);
2060 crq
.request_capability
.capability
= cpu_to_be16(REQ_RX_ADD_QUEUES
);
2061 crq
.request_capability
.number
= cpu_to_be64(adapter
->req_rx_add_queues
);
2062 atomic_inc(&adapter
->running_cap_crqs
);
2063 ibmvnic_send_crq(adapter
, &crq
);
2065 crq
.request_capability
.capability
=
2066 cpu_to_be16(REQ_TX_ENTRIES_PER_SUBCRQ
);
2067 crq
.request_capability
.number
=
2068 cpu_to_be64(adapter
->req_tx_entries_per_subcrq
);
2069 atomic_inc(&adapter
->running_cap_crqs
);
2070 ibmvnic_send_crq(adapter
, &crq
);
2072 crq
.request_capability
.capability
=
2073 cpu_to_be16(REQ_RX_ADD_ENTRIES_PER_SUBCRQ
);
2074 crq
.request_capability
.number
=
2075 cpu_to_be64(adapter
->req_rx_add_entries_per_subcrq
);
2076 atomic_inc(&adapter
->running_cap_crqs
);
2077 ibmvnic_send_crq(adapter
, &crq
);
2079 crq
.request_capability
.capability
= cpu_to_be16(REQ_MTU
);
2080 crq
.request_capability
.number
= cpu_to_be64(adapter
->req_mtu
);
2081 atomic_inc(&adapter
->running_cap_crqs
);
2082 ibmvnic_send_crq(adapter
, &crq
);
2084 if (adapter
->netdev
->flags
& IFF_PROMISC
) {
2085 if (adapter
->promisc_supported
) {
2086 crq
.request_capability
.capability
=
2087 cpu_to_be16(PROMISC_REQUESTED
);
2088 crq
.request_capability
.number
= cpu_to_be64(1);
2089 atomic_inc(&adapter
->running_cap_crqs
);
2090 ibmvnic_send_crq(adapter
, &crq
);
2093 crq
.request_capability
.capability
=
2094 cpu_to_be16(PROMISC_REQUESTED
);
2095 crq
.request_capability
.number
= cpu_to_be64(0);
2096 atomic_inc(&adapter
->running_cap_crqs
);
2097 ibmvnic_send_crq(adapter
, &crq
);
2101 static int pending_scrq(struct ibmvnic_adapter
*adapter
,
2102 struct ibmvnic_sub_crq_queue
*scrq
)
2104 union sub_crq
*entry
= &scrq
->msgs
[scrq
->cur
];
2106 if (entry
->generic
.first
& IBMVNIC_CRQ_CMD_RSP
||
2107 adapter
->state
== VNIC_CLOSING
)
2113 static union sub_crq
*ibmvnic_next_scrq(struct ibmvnic_adapter
*adapter
,
2114 struct ibmvnic_sub_crq_queue
*scrq
)
2116 union sub_crq
*entry
;
2117 unsigned long flags
;
2119 spin_lock_irqsave(&scrq
->lock
, flags
);
2120 entry
= &scrq
->msgs
[scrq
->cur
];
2121 if (entry
->generic
.first
& IBMVNIC_CRQ_CMD_RSP
) {
2122 if (++scrq
->cur
== scrq
->size
)
2127 spin_unlock_irqrestore(&scrq
->lock
, flags
);
2132 static union ibmvnic_crq
*ibmvnic_next_crq(struct ibmvnic_adapter
*adapter
)
2134 struct ibmvnic_crq_queue
*queue
= &adapter
->crq
;
2135 union ibmvnic_crq
*crq
;
2137 crq
= &queue
->msgs
[queue
->cur
];
2138 if (crq
->generic
.first
& IBMVNIC_CRQ_CMD_RSP
) {
2139 if (++queue
->cur
== queue
->size
)
2148 static int send_subcrq(struct ibmvnic_adapter
*adapter
, u64 remote_handle
,
2149 union sub_crq
*sub_crq
)
2151 unsigned int ua
= adapter
->vdev
->unit_address
;
2152 struct device
*dev
= &adapter
->vdev
->dev
;
2153 u64
*u64_crq
= (u64
*)sub_crq
;
2156 netdev_dbg(adapter
->netdev
,
2157 "Sending sCRQ %016lx: %016lx %016lx %016lx %016lx\n",
2158 (unsigned long int)cpu_to_be64(remote_handle
),
2159 (unsigned long int)cpu_to_be64(u64_crq
[0]),
2160 (unsigned long int)cpu_to_be64(u64_crq
[1]),
2161 (unsigned long int)cpu_to_be64(u64_crq
[2]),
2162 (unsigned long int)cpu_to_be64(u64_crq
[3]));
2164 /* Make sure the hypervisor sees the complete request */
2167 rc
= plpar_hcall_norets(H_SEND_SUB_CRQ
, ua
,
2168 cpu_to_be64(remote_handle
),
2169 cpu_to_be64(u64_crq
[0]),
2170 cpu_to_be64(u64_crq
[1]),
2171 cpu_to_be64(u64_crq
[2]),
2172 cpu_to_be64(u64_crq
[3]));
2176 dev_warn(dev
, "CRQ Queue closed\n");
2177 dev_err(dev
, "Send error (rc=%d)\n", rc
);
2183 static int send_subcrq_indirect(struct ibmvnic_adapter
*adapter
,
2184 u64 remote_handle
, u64 ioba
, u64 num_entries
)
2186 unsigned int ua
= adapter
->vdev
->unit_address
;
2187 struct device
*dev
= &adapter
->vdev
->dev
;
2190 /* Make sure the hypervisor sees the complete request */
2192 rc
= plpar_hcall_norets(H_SEND_SUB_CRQ_INDIRECT
, ua
,
2193 cpu_to_be64(remote_handle
),
2198 dev_warn(dev
, "CRQ Queue closed\n");
2199 dev_err(dev
, "Send (indirect) error (rc=%d)\n", rc
);
2205 static int ibmvnic_send_crq(struct ibmvnic_adapter
*adapter
,
2206 union ibmvnic_crq
*crq
)
2208 unsigned int ua
= adapter
->vdev
->unit_address
;
2209 struct device
*dev
= &adapter
->vdev
->dev
;
2210 u64
*u64_crq
= (u64
*)crq
;
2213 netdev_dbg(adapter
->netdev
, "Sending CRQ: %016lx %016lx\n",
2214 (unsigned long int)cpu_to_be64(u64_crq
[0]),
2215 (unsigned long int)cpu_to_be64(u64_crq
[1]));
2217 /* Make sure the hypervisor sees the complete request */
2220 rc
= plpar_hcall_norets(H_SEND_CRQ
, ua
,
2221 cpu_to_be64(u64_crq
[0]),
2222 cpu_to_be64(u64_crq
[1]));
2226 dev_warn(dev
, "CRQ Queue closed\n");
2227 dev_warn(dev
, "Send error (rc=%d)\n", rc
);
2233 static int ibmvnic_send_crq_init(struct ibmvnic_adapter
*adapter
)
2235 union ibmvnic_crq crq
;
2237 memset(&crq
, 0, sizeof(crq
));
2238 crq
.generic
.first
= IBMVNIC_CRQ_INIT_CMD
;
2239 crq
.generic
.cmd
= IBMVNIC_CRQ_INIT
;
2240 netdev_dbg(adapter
->netdev
, "Sending CRQ init\n");
2242 return ibmvnic_send_crq(adapter
, &crq
);
2245 static int send_version_xchg(struct ibmvnic_adapter
*adapter
)
2247 union ibmvnic_crq crq
;
2249 memset(&crq
, 0, sizeof(crq
));
2250 crq
.version_exchange
.first
= IBMVNIC_CRQ_CMD
;
2251 crq
.version_exchange
.cmd
= VERSION_EXCHANGE
;
2252 crq
.version_exchange
.version
= cpu_to_be16(ibmvnic_version
);
2254 return ibmvnic_send_crq(adapter
, &crq
);
2257 static void send_login(struct ibmvnic_adapter
*adapter
)
2259 struct ibmvnic_login_rsp_buffer
*login_rsp_buffer
;
2260 struct ibmvnic_login_buffer
*login_buffer
;
2261 struct device
*dev
= &adapter
->vdev
->dev
;
2262 dma_addr_t rsp_buffer_token
;
2263 dma_addr_t buffer_token
;
2264 size_t rsp_buffer_size
;
2265 union ibmvnic_crq crq
;
2272 sizeof(struct ibmvnic_login_buffer
) +
2273 sizeof(u64
) * (adapter
->req_tx_queues
+ adapter
->req_rx_queues
);
2275 login_buffer
= kmalloc(buffer_size
, GFP_ATOMIC
);
2277 goto buf_alloc_failed
;
2279 buffer_token
= dma_map_single(dev
, login_buffer
, buffer_size
,
2281 if (dma_mapping_error(dev
, buffer_token
)) {
2282 dev_err(dev
, "Couldn't map login buffer\n");
2283 goto buf_map_failed
;
2286 rsp_buffer_size
= sizeof(struct ibmvnic_login_rsp_buffer
) +
2287 sizeof(u64
) * adapter
->req_tx_queues
+
2288 sizeof(u64
) * adapter
->req_rx_queues
+
2289 sizeof(u64
) * adapter
->req_rx_queues
+
2290 sizeof(u8
) * IBMVNIC_TX_DESC_VERSIONS
;
2292 login_rsp_buffer
= kmalloc(rsp_buffer_size
, GFP_ATOMIC
);
2293 if (!login_rsp_buffer
)
2294 goto buf_rsp_alloc_failed
;
2296 rsp_buffer_token
= dma_map_single(dev
, login_rsp_buffer
,
2297 rsp_buffer_size
, DMA_FROM_DEVICE
);
2298 if (dma_mapping_error(dev
, rsp_buffer_token
)) {
2299 dev_err(dev
, "Couldn't map login rsp buffer\n");
2300 goto buf_rsp_map_failed
;
2303 adapter
->login_buf
= login_buffer
;
2304 adapter
->login_buf_token
= buffer_token
;
2305 adapter
->login_buf_sz
= buffer_size
;
2306 adapter
->login_rsp_buf
= login_rsp_buffer
;
2307 adapter
->login_rsp_buf_token
= rsp_buffer_token
;
2308 adapter
->login_rsp_buf_sz
= rsp_buffer_size
;
2310 login_buffer
->len
= cpu_to_be32(buffer_size
);
2311 login_buffer
->version
= cpu_to_be32(INITIAL_VERSION_LB
);
2312 login_buffer
->num_txcomp_subcrqs
= cpu_to_be32(adapter
->req_tx_queues
);
2313 login_buffer
->off_txcomp_subcrqs
=
2314 cpu_to_be32(sizeof(struct ibmvnic_login_buffer
));
2315 login_buffer
->num_rxcomp_subcrqs
= cpu_to_be32(adapter
->req_rx_queues
);
2316 login_buffer
->off_rxcomp_subcrqs
=
2317 cpu_to_be32(sizeof(struct ibmvnic_login_buffer
) +
2318 sizeof(u64
) * adapter
->req_tx_queues
);
2319 login_buffer
->login_rsp_ioba
= cpu_to_be32(rsp_buffer_token
);
2320 login_buffer
->login_rsp_len
= cpu_to_be32(rsp_buffer_size
);
2322 tx_list_p
= (__be64
*)((char *)login_buffer
+
2323 sizeof(struct ibmvnic_login_buffer
));
2324 rx_list_p
= (__be64
*)((char *)login_buffer
+
2325 sizeof(struct ibmvnic_login_buffer
) +
2326 sizeof(u64
) * adapter
->req_tx_queues
);
2328 for (i
= 0; i
< adapter
->req_tx_queues
; i
++) {
2329 if (adapter
->tx_scrq
[i
]) {
2330 tx_list_p
[i
] = cpu_to_be64(adapter
->tx_scrq
[i
]->
2335 for (i
= 0; i
< adapter
->req_rx_queues
; i
++) {
2336 if (adapter
->rx_scrq
[i
]) {
2337 rx_list_p
[i
] = cpu_to_be64(adapter
->rx_scrq
[i
]->
2342 netdev_dbg(adapter
->netdev
, "Login Buffer:\n");
2343 for (i
= 0; i
< (adapter
->login_buf_sz
- 1) / 8 + 1; i
++) {
2344 netdev_dbg(adapter
->netdev
, "%016lx\n",
2345 ((unsigned long int *)(adapter
->login_buf
))[i
]);
2348 memset(&crq
, 0, sizeof(crq
));
2349 crq
.login
.first
= IBMVNIC_CRQ_CMD
;
2350 crq
.login
.cmd
= LOGIN
;
2351 crq
.login
.ioba
= cpu_to_be32(buffer_token
);
2352 crq
.login
.len
= cpu_to_be32(buffer_size
);
2353 ibmvnic_send_crq(adapter
, &crq
);
2358 kfree(login_rsp_buffer
);
2359 buf_rsp_alloc_failed
:
2360 dma_unmap_single(dev
, buffer_token
, buffer_size
, DMA_TO_DEVICE
);
2362 kfree(login_buffer
);
2367 static void send_request_map(struct ibmvnic_adapter
*adapter
, dma_addr_t addr
,
2370 union ibmvnic_crq crq
;
2372 memset(&crq
, 0, sizeof(crq
));
2373 crq
.request_map
.first
= IBMVNIC_CRQ_CMD
;
2374 crq
.request_map
.cmd
= REQUEST_MAP
;
2375 crq
.request_map
.map_id
= map_id
;
2376 crq
.request_map
.ioba
= cpu_to_be32(addr
);
2377 crq
.request_map
.len
= cpu_to_be32(len
);
2378 ibmvnic_send_crq(adapter
, &crq
);
2381 static void send_request_unmap(struct ibmvnic_adapter
*adapter
, u8 map_id
)
2383 union ibmvnic_crq crq
;
2385 memset(&crq
, 0, sizeof(crq
));
2386 crq
.request_unmap
.first
= IBMVNIC_CRQ_CMD
;
2387 crq
.request_unmap
.cmd
= REQUEST_UNMAP
;
2388 crq
.request_unmap
.map_id
= map_id
;
2389 ibmvnic_send_crq(adapter
, &crq
);
2392 static void send_map_query(struct ibmvnic_adapter
*adapter
)
2394 union ibmvnic_crq crq
;
2396 memset(&crq
, 0, sizeof(crq
));
2397 crq
.query_map
.first
= IBMVNIC_CRQ_CMD
;
2398 crq
.query_map
.cmd
= QUERY_MAP
;
2399 ibmvnic_send_crq(adapter
, &crq
);
2402 /* Send a series of CRQs requesting various capabilities of the VNIC server */
2403 static void send_cap_queries(struct ibmvnic_adapter
*adapter
)
2405 union ibmvnic_crq crq
;
2407 atomic_set(&adapter
->running_cap_crqs
, 0);
2408 memset(&crq
, 0, sizeof(crq
));
2409 crq
.query_capability
.first
= IBMVNIC_CRQ_CMD
;
2410 crq
.query_capability
.cmd
= QUERY_CAPABILITY
;
2412 crq
.query_capability
.capability
= cpu_to_be16(MIN_TX_QUEUES
);
2413 atomic_inc(&adapter
->running_cap_crqs
);
2414 ibmvnic_send_crq(adapter
, &crq
);
2416 crq
.query_capability
.capability
= cpu_to_be16(MIN_RX_QUEUES
);
2417 atomic_inc(&adapter
->running_cap_crqs
);
2418 ibmvnic_send_crq(adapter
, &crq
);
2420 crq
.query_capability
.capability
= cpu_to_be16(MIN_RX_ADD_QUEUES
);
2421 atomic_inc(&adapter
->running_cap_crqs
);
2422 ibmvnic_send_crq(adapter
, &crq
);
2424 crq
.query_capability
.capability
= cpu_to_be16(MAX_TX_QUEUES
);
2425 atomic_inc(&adapter
->running_cap_crqs
);
2426 ibmvnic_send_crq(adapter
, &crq
);
2428 crq
.query_capability
.capability
= cpu_to_be16(MAX_RX_QUEUES
);
2429 atomic_inc(&adapter
->running_cap_crqs
);
2430 ibmvnic_send_crq(adapter
, &crq
);
2432 crq
.query_capability
.capability
= cpu_to_be16(MAX_RX_ADD_QUEUES
);
2433 atomic_inc(&adapter
->running_cap_crqs
);
2434 ibmvnic_send_crq(adapter
, &crq
);
2436 crq
.query_capability
.capability
=
2437 cpu_to_be16(MIN_TX_ENTRIES_PER_SUBCRQ
);
2438 atomic_inc(&adapter
->running_cap_crqs
);
2439 ibmvnic_send_crq(adapter
, &crq
);
2441 crq
.query_capability
.capability
=
2442 cpu_to_be16(MIN_RX_ADD_ENTRIES_PER_SUBCRQ
);
2443 atomic_inc(&adapter
->running_cap_crqs
);
2444 ibmvnic_send_crq(adapter
, &crq
);
2446 crq
.query_capability
.capability
=
2447 cpu_to_be16(MAX_TX_ENTRIES_PER_SUBCRQ
);
2448 atomic_inc(&adapter
->running_cap_crqs
);
2449 ibmvnic_send_crq(adapter
, &crq
);
2451 crq
.query_capability
.capability
=
2452 cpu_to_be16(MAX_RX_ADD_ENTRIES_PER_SUBCRQ
);
2453 atomic_inc(&adapter
->running_cap_crqs
);
2454 ibmvnic_send_crq(adapter
, &crq
);
2456 crq
.query_capability
.capability
= cpu_to_be16(TCP_IP_OFFLOAD
);
2457 atomic_inc(&adapter
->running_cap_crqs
);
2458 ibmvnic_send_crq(adapter
, &crq
);
2460 crq
.query_capability
.capability
= cpu_to_be16(PROMISC_SUPPORTED
);
2461 atomic_inc(&adapter
->running_cap_crqs
);
2462 ibmvnic_send_crq(adapter
, &crq
);
2464 crq
.query_capability
.capability
= cpu_to_be16(MIN_MTU
);
2465 atomic_inc(&adapter
->running_cap_crqs
);
2466 ibmvnic_send_crq(adapter
, &crq
);
2468 crq
.query_capability
.capability
= cpu_to_be16(MAX_MTU
);
2469 atomic_inc(&adapter
->running_cap_crqs
);
2470 ibmvnic_send_crq(adapter
, &crq
);
2472 crq
.query_capability
.capability
= cpu_to_be16(MAX_MULTICAST_FILTERS
);
2473 atomic_inc(&adapter
->running_cap_crqs
);
2474 ibmvnic_send_crq(adapter
, &crq
);
2476 crq
.query_capability
.capability
= cpu_to_be16(VLAN_HEADER_INSERTION
);
2477 atomic_inc(&adapter
->running_cap_crqs
);
2478 ibmvnic_send_crq(adapter
, &crq
);
2480 crq
.query_capability
.capability
= cpu_to_be16(RX_VLAN_HEADER_INSERTION
);
2481 atomic_inc(&adapter
->running_cap_crqs
);
2482 ibmvnic_send_crq(adapter
, &crq
);
2484 crq
.query_capability
.capability
= cpu_to_be16(MAX_TX_SG_ENTRIES
);
2485 atomic_inc(&adapter
->running_cap_crqs
);
2486 ibmvnic_send_crq(adapter
, &crq
);
2488 crq
.query_capability
.capability
= cpu_to_be16(RX_SG_SUPPORTED
);
2489 atomic_inc(&adapter
->running_cap_crqs
);
2490 ibmvnic_send_crq(adapter
, &crq
);
2492 crq
.query_capability
.capability
= cpu_to_be16(OPT_TX_COMP_SUB_QUEUES
);
2493 atomic_inc(&adapter
->running_cap_crqs
);
2494 ibmvnic_send_crq(adapter
, &crq
);
2496 crq
.query_capability
.capability
= cpu_to_be16(OPT_RX_COMP_QUEUES
);
2497 atomic_inc(&adapter
->running_cap_crqs
);
2498 ibmvnic_send_crq(adapter
, &crq
);
2500 crq
.query_capability
.capability
=
2501 cpu_to_be16(OPT_RX_BUFADD_Q_PER_RX_COMP_Q
);
2502 atomic_inc(&adapter
->running_cap_crqs
);
2503 ibmvnic_send_crq(adapter
, &crq
);
2505 crq
.query_capability
.capability
=
2506 cpu_to_be16(OPT_TX_ENTRIES_PER_SUBCRQ
);
2507 atomic_inc(&adapter
->running_cap_crqs
);
2508 ibmvnic_send_crq(adapter
, &crq
);
2510 crq
.query_capability
.capability
=
2511 cpu_to_be16(OPT_RXBA_ENTRIES_PER_SUBCRQ
);
2512 atomic_inc(&adapter
->running_cap_crqs
);
2513 ibmvnic_send_crq(adapter
, &crq
);
2515 crq
.query_capability
.capability
= cpu_to_be16(TX_RX_DESC_REQ
);
2516 atomic_inc(&adapter
->running_cap_crqs
);
2517 ibmvnic_send_crq(adapter
, &crq
);
2520 static void handle_query_ip_offload_rsp(struct ibmvnic_adapter
*adapter
)
2522 struct device
*dev
= &adapter
->vdev
->dev
;
2523 struct ibmvnic_query_ip_offload_buffer
*buf
= &adapter
->ip_offload_buf
;
2524 union ibmvnic_crq crq
;
2527 dma_unmap_single(dev
, adapter
->ip_offload_tok
,
2528 sizeof(adapter
->ip_offload_buf
), DMA_FROM_DEVICE
);
2530 netdev_dbg(adapter
->netdev
, "Query IP Offload Buffer:\n");
2531 for (i
= 0; i
< (sizeof(adapter
->ip_offload_buf
) - 1) / 8 + 1; i
++)
2532 netdev_dbg(adapter
->netdev
, "%016lx\n",
2533 ((unsigned long int *)(buf
))[i
]);
2535 netdev_dbg(adapter
->netdev
, "ipv4_chksum = %d\n", buf
->ipv4_chksum
);
2536 netdev_dbg(adapter
->netdev
, "ipv6_chksum = %d\n", buf
->ipv6_chksum
);
2537 netdev_dbg(adapter
->netdev
, "tcp_ipv4_chksum = %d\n",
2538 buf
->tcp_ipv4_chksum
);
2539 netdev_dbg(adapter
->netdev
, "tcp_ipv6_chksum = %d\n",
2540 buf
->tcp_ipv6_chksum
);
2541 netdev_dbg(adapter
->netdev
, "udp_ipv4_chksum = %d\n",
2542 buf
->udp_ipv4_chksum
);
2543 netdev_dbg(adapter
->netdev
, "udp_ipv6_chksum = %d\n",
2544 buf
->udp_ipv6_chksum
);
2545 netdev_dbg(adapter
->netdev
, "large_tx_ipv4 = %d\n",
2546 buf
->large_tx_ipv4
);
2547 netdev_dbg(adapter
->netdev
, "large_tx_ipv6 = %d\n",
2548 buf
->large_tx_ipv6
);
2549 netdev_dbg(adapter
->netdev
, "large_rx_ipv4 = %d\n",
2550 buf
->large_rx_ipv4
);
2551 netdev_dbg(adapter
->netdev
, "large_rx_ipv6 = %d\n",
2552 buf
->large_rx_ipv6
);
2553 netdev_dbg(adapter
->netdev
, "max_ipv4_hdr_sz = %d\n",
2554 buf
->max_ipv4_header_size
);
2555 netdev_dbg(adapter
->netdev
, "max_ipv6_hdr_sz = %d\n",
2556 buf
->max_ipv6_header_size
);
2557 netdev_dbg(adapter
->netdev
, "max_tcp_hdr_size = %d\n",
2558 buf
->max_tcp_header_size
);
2559 netdev_dbg(adapter
->netdev
, "max_udp_hdr_size = %d\n",
2560 buf
->max_udp_header_size
);
2561 netdev_dbg(adapter
->netdev
, "max_large_tx_size = %d\n",
2562 buf
->max_large_tx_size
);
2563 netdev_dbg(adapter
->netdev
, "max_large_rx_size = %d\n",
2564 buf
->max_large_rx_size
);
2565 netdev_dbg(adapter
->netdev
, "ipv6_ext_hdr = %d\n",
2566 buf
->ipv6_extension_header
);
2567 netdev_dbg(adapter
->netdev
, "tcp_pseudosum_req = %d\n",
2568 buf
->tcp_pseudosum_req
);
2569 netdev_dbg(adapter
->netdev
, "num_ipv6_ext_hd = %d\n",
2570 buf
->num_ipv6_ext_headers
);
2571 netdev_dbg(adapter
->netdev
, "off_ipv6_ext_hd = %d\n",
2572 buf
->off_ipv6_ext_headers
);
2574 adapter
->ip_offload_ctrl_tok
=
2575 dma_map_single(dev
, &adapter
->ip_offload_ctrl
,
2576 sizeof(adapter
->ip_offload_ctrl
), DMA_TO_DEVICE
);
2578 if (dma_mapping_error(dev
, adapter
->ip_offload_ctrl_tok
)) {
2579 dev_err(dev
, "Couldn't map ip offload control buffer\n");
2583 adapter
->ip_offload_ctrl
.version
= cpu_to_be32(INITIAL_VERSION_IOB
);
2584 adapter
->ip_offload_ctrl
.tcp_ipv4_chksum
= buf
->tcp_ipv4_chksum
;
2585 adapter
->ip_offload_ctrl
.udp_ipv4_chksum
= buf
->udp_ipv4_chksum
;
2586 adapter
->ip_offload_ctrl
.tcp_ipv6_chksum
= buf
->tcp_ipv6_chksum
;
2587 adapter
->ip_offload_ctrl
.udp_ipv6_chksum
= buf
->udp_ipv6_chksum
;
2589 /* large_tx/rx disabled for now, additional features needed */
2590 adapter
->ip_offload_ctrl
.large_tx_ipv4
= 0;
2591 adapter
->ip_offload_ctrl
.large_tx_ipv6
= 0;
2592 adapter
->ip_offload_ctrl
.large_rx_ipv4
= 0;
2593 adapter
->ip_offload_ctrl
.large_rx_ipv6
= 0;
2595 adapter
->netdev
->features
= NETIF_F_GSO
;
2597 if (buf
->tcp_ipv4_chksum
|| buf
->udp_ipv4_chksum
)
2598 adapter
->netdev
->features
|= NETIF_F_IP_CSUM
;
2600 if (buf
->tcp_ipv6_chksum
|| buf
->udp_ipv6_chksum
)
2601 adapter
->netdev
->features
|= NETIF_F_IPV6_CSUM
;
2603 if ((adapter
->netdev
->features
&
2604 (NETIF_F_IP_CSUM
| NETIF_F_IPV6_CSUM
)))
2605 adapter
->netdev
->features
|= NETIF_F_RXCSUM
;
2607 memset(&crq
, 0, sizeof(crq
));
2608 crq
.control_ip_offload
.first
= IBMVNIC_CRQ_CMD
;
2609 crq
.control_ip_offload
.cmd
= CONTROL_IP_OFFLOAD
;
2610 crq
.control_ip_offload
.len
=
2611 cpu_to_be32(sizeof(adapter
->ip_offload_ctrl
));
2612 crq
.control_ip_offload
.ioba
= cpu_to_be32(adapter
->ip_offload_ctrl_tok
);
2613 ibmvnic_send_crq(adapter
, &crq
);
2616 static void handle_error_info_rsp(union ibmvnic_crq
*crq
,
2617 struct ibmvnic_adapter
*adapter
)
2619 struct device
*dev
= &adapter
->vdev
->dev
;
2620 struct ibmvnic_error_buff
*error_buff
, *tmp
;
2621 unsigned long flags
;
2625 if (!crq
->request_error_rsp
.rc
.code
) {
2626 dev_info(dev
, "Request Error Rsp returned with rc=%x\n",
2627 crq
->request_error_rsp
.rc
.code
);
2631 spin_lock_irqsave(&adapter
->error_list_lock
, flags
);
2632 list_for_each_entry_safe(error_buff
, tmp
, &adapter
->errors
, list
)
2633 if (error_buff
->error_id
== crq
->request_error_rsp
.error_id
) {
2635 list_del(&error_buff
->list
);
2638 spin_unlock_irqrestore(&adapter
->error_list_lock
, flags
);
2641 dev_err(dev
, "Couldn't find error id %x\n",
2642 be32_to_cpu(crq
->request_error_rsp
.error_id
));
2646 dev_err(dev
, "Detailed info for error id %x:",
2647 be32_to_cpu(crq
->request_error_rsp
.error_id
));
2649 for (i
= 0; i
< error_buff
->len
; i
++) {
2650 pr_cont("%02x", (int)error_buff
->buff
[i
]);
2656 dma_unmap_single(dev
, error_buff
->dma
, error_buff
->len
,
2658 kfree(error_buff
->buff
);
2662 static void request_error_information(struct ibmvnic_adapter
*adapter
,
2663 union ibmvnic_crq
*err_crq
)
2665 struct device
*dev
= &adapter
->vdev
->dev
;
2666 struct net_device
*netdev
= adapter
->netdev
;
2667 struct ibmvnic_error_buff
*error_buff
;
2668 unsigned long timeout
= msecs_to_jiffies(30000);
2669 union ibmvnic_crq crq
;
2670 unsigned long flags
;
2673 error_buff
= kmalloc(sizeof(*error_buff
), GFP_ATOMIC
);
2677 detail_len
= be32_to_cpu(err_crq
->error_indication
.detail_error_sz
);
2678 error_buff
->buff
= kmalloc(detail_len
, GFP_ATOMIC
);
2679 if (!error_buff
->buff
) {
2684 error_buff
->dma
= dma_map_single(dev
, error_buff
->buff
, detail_len
,
2686 if (dma_mapping_error(dev
, error_buff
->dma
)) {
2687 netdev_err(netdev
, "Couldn't map error buffer\n");
2688 kfree(error_buff
->buff
);
2693 error_buff
->len
= detail_len
;
2694 error_buff
->error_id
= err_crq
->error_indication
.error_id
;
2696 spin_lock_irqsave(&adapter
->error_list_lock
, flags
);
2697 list_add_tail(&error_buff
->list
, &adapter
->errors
);
2698 spin_unlock_irqrestore(&adapter
->error_list_lock
, flags
);
2700 memset(&crq
, 0, sizeof(crq
));
2701 crq
.request_error_info
.first
= IBMVNIC_CRQ_CMD
;
2702 crq
.request_error_info
.cmd
= REQUEST_ERROR_INFO
;
2703 crq
.request_error_info
.ioba
= cpu_to_be32(error_buff
->dma
);
2704 crq
.request_error_info
.len
= cpu_to_be32(detail_len
);
2705 crq
.request_error_info
.error_id
= err_crq
->error_indication
.error_id
;
2707 rc
= ibmvnic_send_crq(adapter
, &crq
);
2709 netdev_err(netdev
, "failed to request error information\n");
2713 if (!wait_for_completion_timeout(&adapter
->init_done
, timeout
)) {
2714 netdev_err(netdev
, "timeout waiting for error information\n");
2721 spin_lock_irqsave(&adapter
->error_list_lock
, flags
);
2722 list_del(&error_buff
->list
);
2723 spin_unlock_irqrestore(&adapter
->error_list_lock
, flags
);
2725 kfree(error_buff
->buff
);
2729 static void handle_error_indication(union ibmvnic_crq
*crq
,
2730 struct ibmvnic_adapter
*adapter
)
2732 struct device
*dev
= &adapter
->vdev
->dev
;
2734 dev_err(dev
, "Firmware reports %serror id %x, cause %d\n",
2735 crq
->error_indication
.flags
2736 & IBMVNIC_FATAL_ERROR
? "FATAL " : "",
2737 be32_to_cpu(crq
->error_indication
.error_id
),
2738 be16_to_cpu(crq
->error_indication
.error_cause
));
2740 if (be32_to_cpu(crq
->error_indication
.error_id
))
2741 request_error_information(adapter
, crq
);
2743 if (crq
->error_indication
.flags
& IBMVNIC_FATAL_ERROR
)
2744 ibmvnic_reset(adapter
, VNIC_RESET_FATAL
);
2747 static void handle_change_mac_rsp(union ibmvnic_crq
*crq
,
2748 struct ibmvnic_adapter
*adapter
)
2750 struct net_device
*netdev
= adapter
->netdev
;
2751 struct device
*dev
= &adapter
->vdev
->dev
;
2754 rc
= crq
->change_mac_addr_rsp
.rc
.code
;
2756 dev_err(dev
, "Error %ld in CHANGE_MAC_ADDR_RSP\n", rc
);
2759 memcpy(netdev
->dev_addr
, &crq
->change_mac_addr_rsp
.mac_addr
[0],
2763 static void handle_request_cap_rsp(union ibmvnic_crq
*crq
,
2764 struct ibmvnic_adapter
*adapter
)
2766 struct device
*dev
= &adapter
->vdev
->dev
;
2770 atomic_dec(&adapter
->running_cap_crqs
);
2771 switch (be16_to_cpu(crq
->request_capability_rsp
.capability
)) {
2773 req_value
= &adapter
->req_tx_queues
;
2777 req_value
= &adapter
->req_rx_queues
;
2780 case REQ_RX_ADD_QUEUES
:
2781 req_value
= &adapter
->req_rx_add_queues
;
2784 case REQ_TX_ENTRIES_PER_SUBCRQ
:
2785 req_value
= &adapter
->req_tx_entries_per_subcrq
;
2786 name
= "tx_entries_per_subcrq";
2788 case REQ_RX_ADD_ENTRIES_PER_SUBCRQ
:
2789 req_value
= &adapter
->req_rx_add_entries_per_subcrq
;
2790 name
= "rx_add_entries_per_subcrq";
2793 req_value
= &adapter
->req_mtu
;
2796 case PROMISC_REQUESTED
:
2797 req_value
= &adapter
->promisc
;
2801 dev_err(dev
, "Got invalid cap request rsp %d\n",
2802 crq
->request_capability
.capability
);
2806 switch (crq
->request_capability_rsp
.rc
.code
) {
2809 case PARTIALSUCCESS
:
2810 dev_info(dev
, "req=%lld, rsp=%ld in %s queue, retrying.\n",
2812 (long int)be64_to_cpu(crq
->request_capability_rsp
.
2814 release_sub_crqs(adapter
);
2815 *req_value
= be64_to_cpu(crq
->request_capability_rsp
.number
);
2816 ibmvnic_send_req_caps(adapter
, 1);
2819 dev_err(dev
, "Error %d in request cap rsp\n",
2820 crq
->request_capability_rsp
.rc
.code
);
2824 /* Done receiving requested capabilities, query IP offload support */
2825 if (atomic_read(&adapter
->running_cap_crqs
) == 0) {
2826 union ibmvnic_crq newcrq
;
2827 int buf_sz
= sizeof(struct ibmvnic_query_ip_offload_buffer
);
2828 struct ibmvnic_query_ip_offload_buffer
*ip_offload_buf
=
2829 &adapter
->ip_offload_buf
;
2831 adapter
->wait_capability
= false;
2832 adapter
->ip_offload_tok
= dma_map_single(dev
, ip_offload_buf
,
2836 if (dma_mapping_error(dev
, adapter
->ip_offload_tok
)) {
2837 if (!firmware_has_feature(FW_FEATURE_CMO
))
2838 dev_err(dev
, "Couldn't map offload buffer\n");
2842 memset(&newcrq
, 0, sizeof(newcrq
));
2843 newcrq
.query_ip_offload
.first
= IBMVNIC_CRQ_CMD
;
2844 newcrq
.query_ip_offload
.cmd
= QUERY_IP_OFFLOAD
;
2845 newcrq
.query_ip_offload
.len
= cpu_to_be32(buf_sz
);
2846 newcrq
.query_ip_offload
.ioba
=
2847 cpu_to_be32(adapter
->ip_offload_tok
);
2849 ibmvnic_send_crq(adapter
, &newcrq
);
2853 static int handle_login_rsp(union ibmvnic_crq
*login_rsp_crq
,
2854 struct ibmvnic_adapter
*adapter
)
2856 struct device
*dev
= &adapter
->vdev
->dev
;
2857 struct ibmvnic_login_rsp_buffer
*login_rsp
= adapter
->login_rsp_buf
;
2858 struct ibmvnic_login_buffer
*login
= adapter
->login_buf
;
2861 dma_unmap_single(dev
, adapter
->login_buf_token
, adapter
->login_buf_sz
,
2863 dma_unmap_single(dev
, adapter
->login_rsp_buf_token
,
2864 adapter
->login_rsp_buf_sz
, DMA_BIDIRECTIONAL
);
2866 /* If the number of queues requested can't be allocated by the
2867 * server, the login response will return with code 1. We will need
2868 * to resend the login buffer with fewer queues requested.
2870 if (login_rsp_crq
->generic
.rc
.code
) {
2871 adapter
->renegotiate
= true;
2872 complete(&adapter
->init_done
);
2876 netdev_dbg(adapter
->netdev
, "Login Response Buffer:\n");
2877 for (i
= 0; i
< (adapter
->login_rsp_buf_sz
- 1) / 8 + 1; i
++) {
2878 netdev_dbg(adapter
->netdev
, "%016lx\n",
2879 ((unsigned long int *)(adapter
->login_rsp_buf
))[i
]);
2883 if (login
->num_txcomp_subcrqs
!= login_rsp
->num_txsubm_subcrqs
||
2884 (be32_to_cpu(login
->num_rxcomp_subcrqs
) *
2885 adapter
->req_rx_add_queues
!=
2886 be32_to_cpu(login_rsp
->num_rxadd_subcrqs
))) {
2887 dev_err(dev
, "FATAL: Inconsistent login and login rsp\n");
2888 ibmvnic_remove(adapter
->vdev
);
2891 complete(&adapter
->init_done
);
2896 static void handle_request_map_rsp(union ibmvnic_crq
*crq
,
2897 struct ibmvnic_adapter
*adapter
)
2899 struct device
*dev
= &adapter
->vdev
->dev
;
2900 u8 map_id
= crq
->request_map_rsp
.map_id
;
2906 tx_subcrqs
= be32_to_cpu(adapter
->login_rsp_buf
->num_txsubm_subcrqs
);
2907 rx_subcrqs
= be32_to_cpu(adapter
->login_rsp_buf
->num_rxadd_subcrqs
);
2909 rc
= crq
->request_map_rsp
.rc
.code
;
2911 dev_err(dev
, "Error %ld in REQUEST_MAP_RSP\n", rc
);
2913 /* need to find and zero tx/rx_pool map_id */
2914 for (i
= 0; i
< tx_subcrqs
; i
++) {
2915 if (adapter
->tx_pool
[i
].long_term_buff
.map_id
== map_id
)
2916 adapter
->tx_pool
[i
].long_term_buff
.map_id
= 0;
2918 for (i
= 0; i
< rx_subcrqs
; i
++) {
2919 if (adapter
->rx_pool
[i
].long_term_buff
.map_id
== map_id
)
2920 adapter
->rx_pool
[i
].long_term_buff
.map_id
= 0;
2923 complete(&adapter
->fw_done
);
2926 static void handle_request_unmap_rsp(union ibmvnic_crq
*crq
,
2927 struct ibmvnic_adapter
*adapter
)
2929 struct device
*dev
= &adapter
->vdev
->dev
;
2932 rc
= crq
->request_unmap_rsp
.rc
.code
;
2934 dev_err(dev
, "Error %ld in REQUEST_UNMAP_RSP\n", rc
);
2937 static void handle_query_map_rsp(union ibmvnic_crq
*crq
,
2938 struct ibmvnic_adapter
*adapter
)
2940 struct net_device
*netdev
= adapter
->netdev
;
2941 struct device
*dev
= &adapter
->vdev
->dev
;
2944 rc
= crq
->query_map_rsp
.rc
.code
;
2946 dev_err(dev
, "Error %ld in QUERY_MAP_RSP\n", rc
);
2949 netdev_dbg(netdev
, "page_size = %d\ntot_pages = %d\nfree_pages = %d\n",
2950 crq
->query_map_rsp
.page_size
, crq
->query_map_rsp
.tot_pages
,
2951 crq
->query_map_rsp
.free_pages
);
2954 static void handle_query_cap_rsp(union ibmvnic_crq
*crq
,
2955 struct ibmvnic_adapter
*adapter
)
2957 struct net_device
*netdev
= adapter
->netdev
;
2958 struct device
*dev
= &adapter
->vdev
->dev
;
2961 atomic_dec(&adapter
->running_cap_crqs
);
2962 netdev_dbg(netdev
, "Outstanding queries: %d\n",
2963 atomic_read(&adapter
->running_cap_crqs
));
2964 rc
= crq
->query_capability
.rc
.code
;
2966 dev_err(dev
, "Error %ld in QUERY_CAP_RSP\n", rc
);
2970 switch (be16_to_cpu(crq
->query_capability
.capability
)) {
2972 adapter
->min_tx_queues
=
2973 be64_to_cpu(crq
->query_capability
.number
);
2974 netdev_dbg(netdev
, "min_tx_queues = %lld\n",
2975 adapter
->min_tx_queues
);
2978 adapter
->min_rx_queues
=
2979 be64_to_cpu(crq
->query_capability
.number
);
2980 netdev_dbg(netdev
, "min_rx_queues = %lld\n",
2981 adapter
->min_rx_queues
);
2983 case MIN_RX_ADD_QUEUES
:
2984 adapter
->min_rx_add_queues
=
2985 be64_to_cpu(crq
->query_capability
.number
);
2986 netdev_dbg(netdev
, "min_rx_add_queues = %lld\n",
2987 adapter
->min_rx_add_queues
);
2990 adapter
->max_tx_queues
=
2991 be64_to_cpu(crq
->query_capability
.number
);
2992 netdev_dbg(netdev
, "max_tx_queues = %lld\n",
2993 adapter
->max_tx_queues
);
2996 adapter
->max_rx_queues
=
2997 be64_to_cpu(crq
->query_capability
.number
);
2998 netdev_dbg(netdev
, "max_rx_queues = %lld\n",
2999 adapter
->max_rx_queues
);
3001 case MAX_RX_ADD_QUEUES
:
3002 adapter
->max_rx_add_queues
=
3003 be64_to_cpu(crq
->query_capability
.number
);
3004 netdev_dbg(netdev
, "max_rx_add_queues = %lld\n",
3005 adapter
->max_rx_add_queues
);
3007 case MIN_TX_ENTRIES_PER_SUBCRQ
:
3008 adapter
->min_tx_entries_per_subcrq
=
3009 be64_to_cpu(crq
->query_capability
.number
);
3010 netdev_dbg(netdev
, "min_tx_entries_per_subcrq = %lld\n",
3011 adapter
->min_tx_entries_per_subcrq
);
3013 case MIN_RX_ADD_ENTRIES_PER_SUBCRQ
:
3014 adapter
->min_rx_add_entries_per_subcrq
=
3015 be64_to_cpu(crq
->query_capability
.number
);
3016 netdev_dbg(netdev
, "min_rx_add_entrs_per_subcrq = %lld\n",
3017 adapter
->min_rx_add_entries_per_subcrq
);
3019 case MAX_TX_ENTRIES_PER_SUBCRQ
:
3020 adapter
->max_tx_entries_per_subcrq
=
3021 be64_to_cpu(crq
->query_capability
.number
);
3022 netdev_dbg(netdev
, "max_tx_entries_per_subcrq = %lld\n",
3023 adapter
->max_tx_entries_per_subcrq
);
3025 case MAX_RX_ADD_ENTRIES_PER_SUBCRQ
:
3026 adapter
->max_rx_add_entries_per_subcrq
=
3027 be64_to_cpu(crq
->query_capability
.number
);
3028 netdev_dbg(netdev
, "max_rx_add_entrs_per_subcrq = %lld\n",
3029 adapter
->max_rx_add_entries_per_subcrq
);
3031 case TCP_IP_OFFLOAD
:
3032 adapter
->tcp_ip_offload
=
3033 be64_to_cpu(crq
->query_capability
.number
);
3034 netdev_dbg(netdev
, "tcp_ip_offload = %lld\n",
3035 adapter
->tcp_ip_offload
);
3037 case PROMISC_SUPPORTED
:
3038 adapter
->promisc_supported
=
3039 be64_to_cpu(crq
->query_capability
.number
);
3040 netdev_dbg(netdev
, "promisc_supported = %lld\n",
3041 adapter
->promisc_supported
);
3044 adapter
->min_mtu
= be64_to_cpu(crq
->query_capability
.number
);
3045 netdev
->min_mtu
= adapter
->min_mtu
- ETH_HLEN
;
3046 netdev_dbg(netdev
, "min_mtu = %lld\n", adapter
->min_mtu
);
3049 adapter
->max_mtu
= be64_to_cpu(crq
->query_capability
.number
);
3050 netdev
->max_mtu
= adapter
->max_mtu
- ETH_HLEN
;
3051 netdev_dbg(netdev
, "max_mtu = %lld\n", adapter
->max_mtu
);
3053 case MAX_MULTICAST_FILTERS
:
3054 adapter
->max_multicast_filters
=
3055 be64_to_cpu(crq
->query_capability
.number
);
3056 netdev_dbg(netdev
, "max_multicast_filters = %lld\n",
3057 adapter
->max_multicast_filters
);
3059 case VLAN_HEADER_INSERTION
:
3060 adapter
->vlan_header_insertion
=
3061 be64_to_cpu(crq
->query_capability
.number
);
3062 if (adapter
->vlan_header_insertion
)
3063 netdev
->features
|= NETIF_F_HW_VLAN_STAG_TX
;
3064 netdev_dbg(netdev
, "vlan_header_insertion = %lld\n",
3065 adapter
->vlan_header_insertion
);
3067 case RX_VLAN_HEADER_INSERTION
:
3068 adapter
->rx_vlan_header_insertion
=
3069 be64_to_cpu(crq
->query_capability
.number
);
3070 netdev_dbg(netdev
, "rx_vlan_header_insertion = %lld\n",
3071 adapter
->rx_vlan_header_insertion
);
3073 case MAX_TX_SG_ENTRIES
:
3074 adapter
->max_tx_sg_entries
=
3075 be64_to_cpu(crq
->query_capability
.number
);
3076 netdev_dbg(netdev
, "max_tx_sg_entries = %lld\n",
3077 adapter
->max_tx_sg_entries
);
3079 case RX_SG_SUPPORTED
:
3080 adapter
->rx_sg_supported
=
3081 be64_to_cpu(crq
->query_capability
.number
);
3082 netdev_dbg(netdev
, "rx_sg_supported = %lld\n",
3083 adapter
->rx_sg_supported
);
3085 case OPT_TX_COMP_SUB_QUEUES
:
3086 adapter
->opt_tx_comp_sub_queues
=
3087 be64_to_cpu(crq
->query_capability
.number
);
3088 netdev_dbg(netdev
, "opt_tx_comp_sub_queues = %lld\n",
3089 adapter
->opt_tx_comp_sub_queues
);
3091 case OPT_RX_COMP_QUEUES
:
3092 adapter
->opt_rx_comp_queues
=
3093 be64_to_cpu(crq
->query_capability
.number
);
3094 netdev_dbg(netdev
, "opt_rx_comp_queues = %lld\n",
3095 adapter
->opt_rx_comp_queues
);
3097 case OPT_RX_BUFADD_Q_PER_RX_COMP_Q
:
3098 adapter
->opt_rx_bufadd_q_per_rx_comp_q
=
3099 be64_to_cpu(crq
->query_capability
.number
);
3100 netdev_dbg(netdev
, "opt_rx_bufadd_q_per_rx_comp_q = %lld\n",
3101 adapter
->opt_rx_bufadd_q_per_rx_comp_q
);
3103 case OPT_TX_ENTRIES_PER_SUBCRQ
:
3104 adapter
->opt_tx_entries_per_subcrq
=
3105 be64_to_cpu(crq
->query_capability
.number
);
3106 netdev_dbg(netdev
, "opt_tx_entries_per_subcrq = %lld\n",
3107 adapter
->opt_tx_entries_per_subcrq
);
3109 case OPT_RXBA_ENTRIES_PER_SUBCRQ
:
3110 adapter
->opt_rxba_entries_per_subcrq
=
3111 be64_to_cpu(crq
->query_capability
.number
);
3112 netdev_dbg(netdev
, "opt_rxba_entries_per_subcrq = %lld\n",
3113 adapter
->opt_rxba_entries_per_subcrq
);
3115 case TX_RX_DESC_REQ
:
3116 adapter
->tx_rx_desc_req
= crq
->query_capability
.number
;
3117 netdev_dbg(netdev
, "tx_rx_desc_req = %llx\n",
3118 adapter
->tx_rx_desc_req
);
3122 netdev_err(netdev
, "Got invalid cap rsp %d\n",
3123 crq
->query_capability
.capability
);
3127 if (atomic_read(&adapter
->running_cap_crqs
) == 0) {
3128 adapter
->wait_capability
= false;
3129 ibmvnic_send_req_caps(adapter
, 0);
3133 static void ibmvnic_handle_crq(union ibmvnic_crq
*crq
,
3134 struct ibmvnic_adapter
*adapter
)
3136 struct ibmvnic_generic_crq
*gen_crq
= &crq
->generic
;
3137 struct net_device
*netdev
= adapter
->netdev
;
3138 struct device
*dev
= &adapter
->vdev
->dev
;
3139 u64
*u64_crq
= (u64
*)crq
;
3142 netdev_dbg(netdev
, "Handling CRQ: %016lx %016lx\n",
3143 (unsigned long int)cpu_to_be64(u64_crq
[0]),
3144 (unsigned long int)cpu_to_be64(u64_crq
[1]));
3145 switch (gen_crq
->first
) {
3146 case IBMVNIC_CRQ_INIT_RSP
:
3147 switch (gen_crq
->cmd
) {
3148 case IBMVNIC_CRQ_INIT
:
3149 dev_info(dev
, "Partner initialized\n");
3151 case IBMVNIC_CRQ_INIT_COMPLETE
:
3152 dev_info(dev
, "Partner initialization complete\n");
3153 send_version_xchg(adapter
);
3156 dev_err(dev
, "Unknown crq cmd: %d\n", gen_crq
->cmd
);
3159 case IBMVNIC_CRQ_XPORT_EVENT
:
3160 netif_carrier_off(netdev
);
3161 if (gen_crq
->cmd
== IBMVNIC_PARTITION_MIGRATED
) {
3162 dev_info(dev
, "Migrated, re-enabling adapter\n");
3163 ibmvnic_reset(adapter
, VNIC_RESET_MOBILITY
);
3164 } else if (gen_crq
->cmd
== IBMVNIC_DEVICE_FAILOVER
) {
3165 dev_info(dev
, "Backing device failover detected\n");
3166 ibmvnic_reset(adapter
, VNIC_RESET_FAILOVER
);
3168 /* The adapter lost the connection */
3169 dev_err(dev
, "Virtual Adapter failed (rc=%d)\n",
3171 ibmvnic_reset(adapter
, VNIC_RESET_FATAL
);
3174 case IBMVNIC_CRQ_CMD_RSP
:
3177 dev_err(dev
, "Got an invalid msg type 0x%02x\n",
3182 switch (gen_crq
->cmd
) {
3183 case VERSION_EXCHANGE_RSP
:
3184 rc
= crq
->version_exchange_rsp
.rc
.code
;
3186 dev_err(dev
, "Error %ld in VERSION_EXCHG_RSP\n", rc
);
3189 dev_info(dev
, "Partner protocol version is %d\n",
3190 crq
->version_exchange_rsp
.version
);
3191 if (be16_to_cpu(crq
->version_exchange_rsp
.version
) <
3194 be16_to_cpu(crq
->version_exchange_rsp
.version
);
3195 send_cap_queries(adapter
);
3197 case QUERY_CAPABILITY_RSP
:
3198 handle_query_cap_rsp(crq
, adapter
);
3201 handle_query_map_rsp(crq
, adapter
);
3203 case REQUEST_MAP_RSP
:
3204 handle_request_map_rsp(crq
, adapter
);
3206 case REQUEST_UNMAP_RSP
:
3207 handle_request_unmap_rsp(crq
, adapter
);
3209 case REQUEST_CAPABILITY_RSP
:
3210 handle_request_cap_rsp(crq
, adapter
);
3213 netdev_dbg(netdev
, "Got Login Response\n");
3214 handle_login_rsp(crq
, adapter
);
3216 case LOGICAL_LINK_STATE_RSP
:
3218 "Got Logical Link State Response, state: %d rc: %d\n",
3219 crq
->logical_link_state_rsp
.link_state
,
3220 crq
->logical_link_state_rsp
.rc
.code
);
3221 adapter
->logical_link_state
=
3222 crq
->logical_link_state_rsp
.link_state
;
3223 adapter
->init_done_rc
= crq
->logical_link_state_rsp
.rc
.code
;
3224 complete(&adapter
->init_done
);
3226 case LINK_STATE_INDICATION
:
3227 netdev_dbg(netdev
, "Got Logical Link State Indication\n");
3228 adapter
->phys_link_state
=
3229 crq
->link_state_indication
.phys_link_state
;
3230 adapter
->logical_link_state
=
3231 crq
->link_state_indication
.logical_link_state
;
3233 case CHANGE_MAC_ADDR_RSP
:
3234 netdev_dbg(netdev
, "Got MAC address change Response\n");
3235 handle_change_mac_rsp(crq
, adapter
);
3237 case ERROR_INDICATION
:
3238 netdev_dbg(netdev
, "Got Error Indication\n");
3239 handle_error_indication(crq
, adapter
);
3241 case REQUEST_ERROR_RSP
:
3242 netdev_dbg(netdev
, "Got Error Detail Response\n");
3243 handle_error_info_rsp(crq
, adapter
);
3245 case REQUEST_STATISTICS_RSP
:
3246 netdev_dbg(netdev
, "Got Statistics Response\n");
3247 complete(&adapter
->stats_done
);
3249 case QUERY_IP_OFFLOAD_RSP
:
3250 netdev_dbg(netdev
, "Got Query IP offload Response\n");
3251 handle_query_ip_offload_rsp(adapter
);
3253 case MULTICAST_CTRL_RSP
:
3254 netdev_dbg(netdev
, "Got multicast control Response\n");
3256 case CONTROL_IP_OFFLOAD_RSP
:
3257 netdev_dbg(netdev
, "Got Control IP offload Response\n");
3258 dma_unmap_single(dev
, adapter
->ip_offload_ctrl_tok
,
3259 sizeof(adapter
->ip_offload_ctrl
),
3261 complete(&adapter
->init_done
);
3263 case COLLECT_FW_TRACE_RSP
:
3264 netdev_dbg(netdev
, "Got Collect firmware trace Response\n");
3265 complete(&adapter
->fw_done
);
3268 netdev_err(netdev
, "Got an invalid cmd type 0x%02x\n",
3273 static irqreturn_t
ibmvnic_interrupt(int irq
, void *instance
)
3275 struct ibmvnic_adapter
*adapter
= instance
;
3277 tasklet_schedule(&adapter
->tasklet
);
3281 static void ibmvnic_tasklet(void *data
)
3283 struct ibmvnic_adapter
*adapter
= data
;
3284 struct ibmvnic_crq_queue
*queue
= &adapter
->crq
;
3285 union ibmvnic_crq
*crq
;
3286 unsigned long flags
;
3289 spin_lock_irqsave(&queue
->lock
, flags
);
3291 /* Pull all the valid messages off the CRQ */
3292 while ((crq
= ibmvnic_next_crq(adapter
)) != NULL
) {
3293 ibmvnic_handle_crq(crq
, adapter
);
3294 crq
->generic
.first
= 0;
3297 /* remain in tasklet until all
3298 * capabilities responses are received
3300 if (!adapter
->wait_capability
)
3303 /* if capabilities CRQ's were sent in this tasklet, the following
3304 * tasklet must wait until all responses are received
3306 if (atomic_read(&adapter
->running_cap_crqs
) != 0)
3307 adapter
->wait_capability
= true;
3308 spin_unlock_irqrestore(&queue
->lock
, flags
);
3311 static int ibmvnic_reenable_crq_queue(struct ibmvnic_adapter
*adapter
)
3313 struct vio_dev
*vdev
= adapter
->vdev
;
3317 rc
= plpar_hcall_norets(H_ENABLE_CRQ
, vdev
->unit_address
);
3318 } while (rc
== H_IN_PROGRESS
|| rc
== H_BUSY
|| H_IS_LONG_BUSY(rc
));
3321 dev_err(&vdev
->dev
, "Error enabling adapter (rc=%d)\n", rc
);
3326 static int ibmvnic_reset_crq(struct ibmvnic_adapter
*adapter
)
3328 struct ibmvnic_crq_queue
*crq
= &adapter
->crq
;
3329 struct device
*dev
= &adapter
->vdev
->dev
;
3330 struct vio_dev
*vdev
= adapter
->vdev
;
3335 rc
= plpar_hcall_norets(H_FREE_CRQ
, vdev
->unit_address
);
3336 } while (rc
== H_BUSY
|| H_IS_LONG_BUSY(rc
));
3338 /* Clean out the queue */
3339 memset(crq
->msgs
, 0, PAGE_SIZE
);
3342 /* And re-open it again */
3343 rc
= plpar_hcall_norets(H_REG_CRQ
, vdev
->unit_address
,
3344 crq
->msg_token
, PAGE_SIZE
);
3347 /* Adapter is good, but other end is not ready */
3348 dev_warn(dev
, "Partner adapter not ready\n");
3350 dev_warn(dev
, "Couldn't register crq (rc=%d)\n", rc
);
3355 static void release_crq_queue(struct ibmvnic_adapter
*adapter
)
3357 struct ibmvnic_crq_queue
*crq
= &adapter
->crq
;
3358 struct vio_dev
*vdev
= adapter
->vdev
;
3364 netdev_dbg(adapter
->netdev
, "Releasing CRQ\n");
3365 free_irq(vdev
->irq
, adapter
);
3366 tasklet_kill(&adapter
->tasklet
);
3368 rc
= plpar_hcall_norets(H_FREE_CRQ
, vdev
->unit_address
);
3369 } while (rc
== H_BUSY
|| H_IS_LONG_BUSY(rc
));
3371 dma_unmap_single(&vdev
->dev
, crq
->msg_token
, PAGE_SIZE
,
3373 free_page((unsigned long)crq
->msgs
);
3377 static int init_crq_queue(struct ibmvnic_adapter
*adapter
)
3379 struct ibmvnic_crq_queue
*crq
= &adapter
->crq
;
3380 struct device
*dev
= &adapter
->vdev
->dev
;
3381 struct vio_dev
*vdev
= adapter
->vdev
;
3382 int rc
, retrc
= -ENOMEM
;
3387 crq
->msgs
= (union ibmvnic_crq
*)get_zeroed_page(GFP_KERNEL
);
3388 /* Should we allocate more than one page? */
3393 crq
->size
= PAGE_SIZE
/ sizeof(*crq
->msgs
);
3394 crq
->msg_token
= dma_map_single(dev
, crq
->msgs
, PAGE_SIZE
,
3396 if (dma_mapping_error(dev
, crq
->msg_token
))
3399 rc
= plpar_hcall_norets(H_REG_CRQ
, vdev
->unit_address
,
3400 crq
->msg_token
, PAGE_SIZE
);
3402 if (rc
== H_RESOURCE
)
3403 /* maybe kexecing and resource is busy. try a reset */
3404 rc
= ibmvnic_reset_crq(adapter
);
3407 if (rc
== H_CLOSED
) {
3408 dev_warn(dev
, "Partner adapter not ready\n");
3410 dev_warn(dev
, "Error %d opening adapter\n", rc
);
3411 goto reg_crq_failed
;
3416 tasklet_init(&adapter
->tasklet
, (void *)ibmvnic_tasklet
,
3417 (unsigned long)adapter
);
3419 netdev_dbg(adapter
->netdev
, "registering irq 0x%x\n", vdev
->irq
);
3420 rc
= request_irq(vdev
->irq
, ibmvnic_interrupt
, 0, IBMVNIC_NAME
,
3423 dev_err(dev
, "Couldn't register irq 0x%x. rc=%d\n",
3425 goto req_irq_failed
;
3428 rc
= vio_enable_interrupts(vdev
);
3430 dev_err(dev
, "Error %d enabling interrupts\n", rc
);
3431 goto req_irq_failed
;
3435 spin_lock_init(&crq
->lock
);
3440 tasklet_kill(&adapter
->tasklet
);
3442 rc
= plpar_hcall_norets(H_FREE_CRQ
, vdev
->unit_address
);
3443 } while (rc
== H_BUSY
|| H_IS_LONG_BUSY(rc
));
3445 dma_unmap_single(dev
, crq
->msg_token
, PAGE_SIZE
, DMA_BIDIRECTIONAL
);
3447 free_page((unsigned long)crq
->msgs
);
3452 static int ibmvnic_init(struct ibmvnic_adapter
*adapter
)
3454 struct device
*dev
= &adapter
->vdev
->dev
;
3455 unsigned long timeout
= msecs_to_jiffies(30000);
3458 rc
= init_crq_queue(adapter
);
3460 dev_err(dev
, "Couldn't initialize crq. rc=%d\n", rc
);
3464 init_completion(&adapter
->init_done
);
3465 ibmvnic_send_crq_init(adapter
);
3466 if (!wait_for_completion_timeout(&adapter
->init_done
, timeout
)) {
3467 dev_err(dev
, "Initialization sequence timed out\n");
3468 release_crq_queue(adapter
);
3472 rc
= init_sub_crqs(adapter
);
3474 dev_err(dev
, "Initialization of sub crqs failed\n");
3475 release_crq_queue(adapter
);
3481 static int ibmvnic_probe(struct vio_dev
*dev
, const struct vio_device_id
*id
)
3483 struct ibmvnic_adapter
*adapter
;
3484 struct net_device
*netdev
;
3485 unsigned char *mac_addr_p
;
3488 dev_dbg(&dev
->dev
, "entering ibmvnic_probe for UA 0x%x\n",
3491 mac_addr_p
= (unsigned char *)vio_get_attribute(dev
,
3492 VETH_MAC_ADDR
, NULL
);
3495 "(%s:%3.3d) ERROR: Can't find MAC_ADDR attribute\n",
3496 __FILE__
, __LINE__
);
3500 netdev
= alloc_etherdev_mq(sizeof(struct ibmvnic_adapter
),
3501 IBMVNIC_MAX_TX_QUEUES
);
3505 adapter
= netdev_priv(netdev
);
3506 adapter
->state
= VNIC_PROBING
;
3507 dev_set_drvdata(&dev
->dev
, netdev
);
3508 adapter
->vdev
= dev
;
3509 adapter
->netdev
= netdev
;
3511 ether_addr_copy(adapter
->mac_addr
, mac_addr_p
);
3512 ether_addr_copy(netdev
->dev_addr
, adapter
->mac_addr
);
3513 netdev
->irq
= dev
->irq
;
3514 netdev
->netdev_ops
= &ibmvnic_netdev_ops
;
3515 netdev
->ethtool_ops
= &ibmvnic_ethtool_ops
;
3516 SET_NETDEV_DEV(netdev
, &dev
->dev
);
3518 spin_lock_init(&adapter
->stats_lock
);
3520 INIT_LIST_HEAD(&adapter
->errors
);
3521 spin_lock_init(&adapter
->error_list_lock
);
3523 INIT_WORK(&adapter
->ibmvnic_reset
, __ibmvnic_reset
);
3524 INIT_LIST_HEAD(&adapter
->rwi_list
);
3525 mutex_init(&adapter
->reset_lock
);
3526 mutex_init(&adapter
->rwi_lock
);
3527 adapter
->resetting
= false;
3529 rc
= ibmvnic_init(adapter
);
3531 free_netdev(netdev
);
3535 netdev
->mtu
= adapter
->req_mtu
- ETH_HLEN
;
3537 rc
= register_netdev(netdev
);
3539 dev_err(&dev
->dev
, "failed to register netdev rc=%d\n", rc
);
3540 free_netdev(netdev
);
3543 dev_info(&dev
->dev
, "ibmvnic registered\n");
3545 adapter
->state
= VNIC_PROBED
;
3549 static int ibmvnic_remove(struct vio_dev
*dev
)
3551 struct net_device
*netdev
= dev_get_drvdata(&dev
->dev
);
3552 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
3554 adapter
->state
= VNIC_REMOVING
;
3555 unregister_netdev(netdev
);
3556 mutex_lock(&adapter
->reset_lock
);
3558 release_resources(adapter
);
3559 release_sub_crqs(adapter
);
3560 release_crq_queue(adapter
);
3562 adapter
->state
= VNIC_REMOVED
;
3564 mutex_unlock(&adapter
->reset_lock
);
3565 free_netdev(netdev
);
3566 dev_set_drvdata(&dev
->dev
, NULL
);
3571 static unsigned long ibmvnic_get_desired_dma(struct vio_dev
*vdev
)
3573 struct net_device
*netdev
= dev_get_drvdata(&vdev
->dev
);
3574 struct ibmvnic_adapter
*adapter
;
3575 struct iommu_table
*tbl
;
3576 unsigned long ret
= 0;
3579 tbl
= get_iommu_table_base(&vdev
->dev
);
3581 /* netdev inits at probe time along with the structures we need below*/
3583 return IOMMU_PAGE_ALIGN(IBMVNIC_IO_ENTITLEMENT_DEFAULT
, tbl
);
3585 adapter
= netdev_priv(netdev
);
3587 ret
+= PAGE_SIZE
; /* the crq message queue */
3588 ret
+= IOMMU_PAGE_ALIGN(sizeof(struct ibmvnic_statistics
), tbl
);
3590 for (i
= 0; i
< adapter
->req_tx_queues
+ adapter
->req_rx_queues
; i
++)
3591 ret
+= 4 * PAGE_SIZE
; /* the scrq message queue */
3593 for (i
= 0; i
< be32_to_cpu(adapter
->login_rsp_buf
->num_rxadd_subcrqs
);
3595 ret
+= adapter
->rx_pool
[i
].size
*
3596 IOMMU_PAGE_ALIGN(adapter
->rx_pool
[i
].buff_size
, tbl
);
3601 static int ibmvnic_resume(struct device
*dev
)
3603 struct net_device
*netdev
= dev_get_drvdata(dev
);
3604 struct ibmvnic_adapter
*adapter
= netdev_priv(netdev
);
3607 /* kick the interrupt handlers just in case we lost an interrupt */
3608 for (i
= 0; i
< adapter
->req_rx_queues
; i
++)
3609 ibmvnic_interrupt_rx(adapter
->rx_scrq
[i
]->irq
,
3610 adapter
->rx_scrq
[i
]);
3615 static struct vio_device_id ibmvnic_device_table
[] = {
3616 {"network", "IBM,vnic"},
3619 MODULE_DEVICE_TABLE(vio
, ibmvnic_device_table
);
3621 static const struct dev_pm_ops ibmvnic_pm_ops
= {
3622 .resume
= ibmvnic_resume
3625 static struct vio_driver ibmvnic_driver
= {
3626 .id_table
= ibmvnic_device_table
,
3627 .probe
= ibmvnic_probe
,
3628 .remove
= ibmvnic_remove
,
3629 .get_desired_dma
= ibmvnic_get_desired_dma
,
3630 .name
= ibmvnic_driver_name
,
3631 .pm
= &ibmvnic_pm_ops
,
3634 /* module functions */
3635 static int __init
ibmvnic_module_init(void)
3637 pr_info("%s: %s %s\n", ibmvnic_driver_name
, ibmvnic_driver_string
,
3638 IBMVNIC_DRIVER_VERSION
);
3640 return vio_register_driver(&ibmvnic_driver
);
3643 static void __exit
ibmvnic_module_exit(void)
3645 vio_unregister_driver(&ibmvnic_driver
);
3648 module_init(ibmvnic_module_init
);
3649 module_exit(ibmvnic_module_exit
);