2 * Copyright (c) 2012-2016 VMware, Inc. All rights reserved.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of EITHER the GNU General Public License
6 * version 2 as published by the Free Software Foundation or the BSD
7 * 2-Clause License. This program is distributed in the hope that it
8 * will be useful, but WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED
9 * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
10 * See the GNU General Public License version 2 for more details at
11 * http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html.
13 * You should have received a copy of the GNU General Public License
14 * along with this program available in the file COPYING in the main
15 * directory of this source tree.
17 * The BSD 2-Clause License
19 * Redistribution and use in source and binary forms, with or
20 * without modification, are permitted provided that the following
23 * - Redistributions of source code must retain the above
24 * copyright notice, this list of conditions and the following
27 * - Redistributions in binary form must reproduce the above
28 * copyright notice, this list of conditions and the following
29 * disclaimer in the documentation and/or other materials
30 * provided with the distribution.
32 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
33 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
34 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
35 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
36 * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
37 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
38 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
39 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
40 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
41 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
42 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
43 * OF THE POSSIBILITY OF SUCH DAMAGE.
46 #include <linux/errno.h>
47 #include <linux/inetdevice.h>
48 #include <linux/init.h>
49 #include <linux/module.h>
50 #include <linux/slab.h>
51 #include <rdma/ib_addr.h>
52 #include <rdma/ib_smi.h>
53 #include <rdma/ib_user_verbs.h>
54 #include <net/addrconf.h>
58 #define DRV_NAME "vmw_pvrdma"
59 #define DRV_VERSION "1.0.0.0-k"
61 static DEFINE_MUTEX(pvrdma_device_list_lock
);
62 static LIST_HEAD(pvrdma_device_list
);
63 static struct workqueue_struct
*event_wq
;
65 static int pvrdma_add_gid(struct ib_device
*ibdev
,
68 const union ib_gid
*gid
,
69 const struct ib_gid_attr
*attr
,
71 static int pvrdma_del_gid(struct ib_device
*ibdev
,
77 static ssize_t
show_hca(struct device
*device
, struct device_attribute
*attr
,
80 return sprintf(buf
, "VMW_PVRDMA-%s\n", DRV_VERSION
);
83 static ssize_t
show_rev(struct device
*device
, struct device_attribute
*attr
,
86 return sprintf(buf
, "%d\n", PVRDMA_REV_ID
);
89 static ssize_t
show_board(struct device
*device
, struct device_attribute
*attr
,
92 return sprintf(buf
, "%d\n", PVRDMA_BOARD_ID
);
95 static DEVICE_ATTR(hw_rev
, S_IRUGO
, show_rev
, NULL
);
96 static DEVICE_ATTR(hca_type
, S_IRUGO
, show_hca
, NULL
);
97 static DEVICE_ATTR(board_id
, S_IRUGO
, show_board
, NULL
);
99 static struct device_attribute
*pvrdma_class_attributes
[] = {
105 static void pvrdma_get_fw_ver_str(struct ib_device
*device
, char *str
,
108 struct pvrdma_dev
*dev
=
109 container_of(device
, struct pvrdma_dev
, ib_dev
);
110 snprintf(str
, str_len
, "%d.%d.%d\n",
111 (int) (dev
->dsr
->caps
.fw_ver
>> 32),
112 (int) (dev
->dsr
->caps
.fw_ver
>> 16) & 0xffff,
113 (int) dev
->dsr
->caps
.fw_ver
& 0xffff);
116 static int pvrdma_init_device(struct pvrdma_dev
*dev
)
118 /* Initialize some device related stuff */
119 spin_lock_init(&dev
->cmd_lock
);
120 sema_init(&dev
->cmd_sema
, 1);
121 atomic_set(&dev
->num_qps
, 0);
122 atomic_set(&dev
->num_cqs
, 0);
123 atomic_set(&dev
->num_pds
, 0);
124 atomic_set(&dev
->num_ahs
, 0);
129 static int pvrdma_port_immutable(struct ib_device
*ibdev
, u8 port_num
,
130 struct ib_port_immutable
*immutable
)
132 struct ib_port_attr attr
;
135 err
= pvrdma_query_port(ibdev
, port_num
, &attr
);
139 immutable
->pkey_tbl_len
= attr
.pkey_tbl_len
;
140 immutable
->gid_tbl_len
= attr
.gid_tbl_len
;
141 immutable
->core_cap_flags
= RDMA_CORE_PORT_IBA_ROCE
;
142 immutable
->max_mad_size
= IB_MGMT_MAD_SIZE
;
146 static struct net_device
*pvrdma_get_netdev(struct ib_device
*ibdev
,
149 struct net_device
*netdev
;
150 struct pvrdma_dev
*dev
= to_vdev(ibdev
);
156 netdev
= dev
->netdev
;
164 static int pvrdma_register_device(struct pvrdma_dev
*dev
)
169 strlcpy(dev
->ib_dev
.name
, "vmw_pvrdma%d", IB_DEVICE_NAME_MAX
);
170 dev
->ib_dev
.node_guid
= dev
->dsr
->caps
.node_guid
;
171 dev
->sys_image_guid
= dev
->dsr
->caps
.sys_image_guid
;
173 dev
->ib_dev
.owner
= THIS_MODULE
;
174 dev
->ib_dev
.num_comp_vectors
= 1;
175 dev
->ib_dev
.dma_device
= &dev
->pdev
->dev
;
176 dev
->ib_dev
.uverbs_abi_ver
= PVRDMA_UVERBS_ABI_VERSION
;
177 dev
->ib_dev
.uverbs_cmd_mask
=
178 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT
) |
179 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE
) |
180 (1ull << IB_USER_VERBS_CMD_QUERY_PORT
) |
181 (1ull << IB_USER_VERBS_CMD_ALLOC_PD
) |
182 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD
) |
183 (1ull << IB_USER_VERBS_CMD_REG_MR
) |
184 (1ull << IB_USER_VERBS_CMD_DEREG_MR
) |
185 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL
) |
186 (1ull << IB_USER_VERBS_CMD_CREATE_CQ
) |
187 (1ull << IB_USER_VERBS_CMD_POLL_CQ
) |
188 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ
) |
189 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ
) |
190 (1ull << IB_USER_VERBS_CMD_CREATE_QP
) |
191 (1ull << IB_USER_VERBS_CMD_MODIFY_QP
) |
192 (1ull << IB_USER_VERBS_CMD_QUERY_QP
) |
193 (1ull << IB_USER_VERBS_CMD_DESTROY_QP
) |
194 (1ull << IB_USER_VERBS_CMD_POST_SEND
) |
195 (1ull << IB_USER_VERBS_CMD_POST_RECV
) |
196 (1ull << IB_USER_VERBS_CMD_CREATE_AH
) |
197 (1ull << IB_USER_VERBS_CMD_DESTROY_AH
);
199 dev
->ib_dev
.node_type
= RDMA_NODE_IB_CA
;
200 dev
->ib_dev
.phys_port_cnt
= dev
->dsr
->caps
.phys_port_cnt
;
202 dev
->ib_dev
.query_device
= pvrdma_query_device
;
203 dev
->ib_dev
.query_port
= pvrdma_query_port
;
204 dev
->ib_dev
.query_gid
= pvrdma_query_gid
;
205 dev
->ib_dev
.query_pkey
= pvrdma_query_pkey
;
206 dev
->ib_dev
.modify_port
= pvrdma_modify_port
;
207 dev
->ib_dev
.alloc_ucontext
= pvrdma_alloc_ucontext
;
208 dev
->ib_dev
.dealloc_ucontext
= pvrdma_dealloc_ucontext
;
209 dev
->ib_dev
.mmap
= pvrdma_mmap
;
210 dev
->ib_dev
.alloc_pd
= pvrdma_alloc_pd
;
211 dev
->ib_dev
.dealloc_pd
= pvrdma_dealloc_pd
;
212 dev
->ib_dev
.create_ah
= pvrdma_create_ah
;
213 dev
->ib_dev
.destroy_ah
= pvrdma_destroy_ah
;
214 dev
->ib_dev
.create_qp
= pvrdma_create_qp
;
215 dev
->ib_dev
.modify_qp
= pvrdma_modify_qp
;
216 dev
->ib_dev
.query_qp
= pvrdma_query_qp
;
217 dev
->ib_dev
.destroy_qp
= pvrdma_destroy_qp
;
218 dev
->ib_dev
.post_send
= pvrdma_post_send
;
219 dev
->ib_dev
.post_recv
= pvrdma_post_recv
;
220 dev
->ib_dev
.create_cq
= pvrdma_create_cq
;
221 dev
->ib_dev
.modify_cq
= pvrdma_modify_cq
;
222 dev
->ib_dev
.resize_cq
= pvrdma_resize_cq
;
223 dev
->ib_dev
.destroy_cq
= pvrdma_destroy_cq
;
224 dev
->ib_dev
.poll_cq
= pvrdma_poll_cq
;
225 dev
->ib_dev
.req_notify_cq
= pvrdma_req_notify_cq
;
226 dev
->ib_dev
.get_dma_mr
= pvrdma_get_dma_mr
;
227 dev
->ib_dev
.reg_user_mr
= pvrdma_reg_user_mr
;
228 dev
->ib_dev
.dereg_mr
= pvrdma_dereg_mr
;
229 dev
->ib_dev
.alloc_mr
= pvrdma_alloc_mr
;
230 dev
->ib_dev
.map_mr_sg
= pvrdma_map_mr_sg
;
231 dev
->ib_dev
.add_gid
= pvrdma_add_gid
;
232 dev
->ib_dev
.del_gid
= pvrdma_del_gid
;
233 dev
->ib_dev
.get_netdev
= pvrdma_get_netdev
;
234 dev
->ib_dev
.get_port_immutable
= pvrdma_port_immutable
;
235 dev
->ib_dev
.get_link_layer
= pvrdma_port_link_layer
;
236 dev
->ib_dev
.get_dev_fw_str
= pvrdma_get_fw_ver_str
;
238 mutex_init(&dev
->port_mutex
);
239 spin_lock_init(&dev
->desc_lock
);
241 dev
->cq_tbl
= kcalloc(dev
->dsr
->caps
.max_cq
, sizeof(void *),
245 spin_lock_init(&dev
->cq_tbl_lock
);
247 dev
->qp_tbl
= kcalloc(dev
->dsr
->caps
.max_qp
, sizeof(void *),
251 spin_lock_init(&dev
->qp_tbl_lock
);
253 ret
= ib_register_device(&dev
->ib_dev
, NULL
);
257 for (i
= 0; i
< ARRAY_SIZE(pvrdma_class_attributes
); ++i
) {
258 ret
= device_create_file(&dev
->ib_dev
.dev
,
259 pvrdma_class_attributes
[i
]);
264 dev
->ib_active
= true;
269 ib_unregister_device(&dev
->ib_dev
);
278 static irqreturn_t
pvrdma_intr0_handler(int irq
, void *dev_id
)
280 u32 icr
= PVRDMA_INTR_CAUSE_RESPONSE
;
281 struct pvrdma_dev
*dev
= dev_id
;
283 dev_dbg(&dev
->pdev
->dev
, "interrupt 0 (response) handler\n");
285 if (!dev
->pdev
->msix_enabled
) {
287 icr
= pvrdma_read_reg(dev
, PVRDMA_REG_ICR
);
292 if (icr
== PVRDMA_INTR_CAUSE_RESPONSE
)
293 complete(&dev
->cmd_done
);
298 static void pvrdma_qp_event(struct pvrdma_dev
*dev
, u32 qpn
, int type
)
300 struct pvrdma_qp
*qp
;
303 spin_lock_irqsave(&dev
->qp_tbl_lock
, flags
);
304 qp
= dev
->qp_tbl
[qpn
% dev
->dsr
->caps
.max_qp
];
306 atomic_inc(&qp
->refcnt
);
307 spin_unlock_irqrestore(&dev
->qp_tbl_lock
, flags
);
309 if (qp
&& qp
->ibqp
.event_handler
) {
310 struct ib_qp
*ibqp
= &qp
->ibqp
;
313 e
.device
= ibqp
->device
;
315 e
.event
= type
; /* 1:1 mapping for now. */
316 ibqp
->event_handler(&e
, ibqp
->qp_context
);
319 atomic_dec(&qp
->refcnt
);
320 if (atomic_read(&qp
->refcnt
) == 0)
325 static void pvrdma_cq_event(struct pvrdma_dev
*dev
, u32 cqn
, int type
)
327 struct pvrdma_cq
*cq
;
330 spin_lock_irqsave(&dev
->cq_tbl_lock
, flags
);
331 cq
= dev
->cq_tbl
[cqn
% dev
->dsr
->caps
.max_cq
];
333 atomic_inc(&cq
->refcnt
);
334 spin_unlock_irqrestore(&dev
->cq_tbl_lock
, flags
);
336 if (cq
&& cq
->ibcq
.event_handler
) {
337 struct ib_cq
*ibcq
= &cq
->ibcq
;
340 e
.device
= ibcq
->device
;
342 e
.event
= type
; /* 1:1 mapping for now. */
343 ibcq
->event_handler(&e
, ibcq
->cq_context
);
346 atomic_dec(&cq
->refcnt
);
347 if (atomic_read(&cq
->refcnt
) == 0)
352 static void pvrdma_dispatch_event(struct pvrdma_dev
*dev
, int port
,
353 enum ib_event_type event
)
355 struct ib_event ib_event
;
357 memset(&ib_event
, 0, sizeof(ib_event
));
358 ib_event
.device
= &dev
->ib_dev
;
359 ib_event
.element
.port_num
= port
;
360 ib_event
.event
= event
;
361 ib_dispatch_event(&ib_event
);
364 static void pvrdma_dev_event(struct pvrdma_dev
*dev
, u8 port
, int type
)
366 if (port
< 1 || port
> dev
->dsr
->caps
.phys_port_cnt
) {
367 dev_warn(&dev
->pdev
->dev
, "event on port %d\n", port
);
371 pvrdma_dispatch_event(dev
, port
, type
);
374 static inline struct pvrdma_eqe
*get_eqe(struct pvrdma_dev
*dev
, unsigned int i
)
376 return (struct pvrdma_eqe
*)pvrdma_page_dir_get_ptr(
379 sizeof(struct pvrdma_eqe
) * i
);
382 static irqreturn_t
pvrdma_intr1_handler(int irq
, void *dev_id
)
384 struct pvrdma_dev
*dev
= dev_id
;
385 struct pvrdma_ring
*ring
= &dev
->async_ring_state
->rx
;
386 int ring_slots
= (dev
->dsr
->async_ring_pages
.num_pages
- 1) *
387 PAGE_SIZE
/ sizeof(struct pvrdma_eqe
);
390 dev_dbg(&dev
->pdev
->dev
, "interrupt 1 (async event) handler\n");
393 * Don't process events until the IB device is registered. Otherwise
394 * we'll try to ib_dispatch_event() on an invalid device.
399 while (pvrdma_idx_ring_has_data(ring
, ring_slots
, &head
) > 0) {
400 struct pvrdma_eqe
*eqe
;
402 eqe
= get_eqe(dev
, head
);
405 case PVRDMA_EVENT_QP_FATAL
:
406 case PVRDMA_EVENT_QP_REQ_ERR
:
407 case PVRDMA_EVENT_QP_ACCESS_ERR
:
408 case PVRDMA_EVENT_COMM_EST
:
409 case PVRDMA_EVENT_SQ_DRAINED
:
410 case PVRDMA_EVENT_PATH_MIG
:
411 case PVRDMA_EVENT_PATH_MIG_ERR
:
412 case PVRDMA_EVENT_QP_LAST_WQE_REACHED
:
413 pvrdma_qp_event(dev
, eqe
->info
, eqe
->type
);
416 case PVRDMA_EVENT_CQ_ERR
:
417 pvrdma_cq_event(dev
, eqe
->info
, eqe
->type
);
420 case PVRDMA_EVENT_SRQ_ERR
:
421 case PVRDMA_EVENT_SRQ_LIMIT_REACHED
:
424 case PVRDMA_EVENT_PORT_ACTIVE
:
425 case PVRDMA_EVENT_PORT_ERR
:
426 case PVRDMA_EVENT_LID_CHANGE
:
427 case PVRDMA_EVENT_PKEY_CHANGE
:
428 case PVRDMA_EVENT_SM_CHANGE
:
429 case PVRDMA_EVENT_CLIENT_REREGISTER
:
430 case PVRDMA_EVENT_GID_CHANGE
:
431 pvrdma_dev_event(dev
, eqe
->info
, eqe
->type
);
434 case PVRDMA_EVENT_DEVICE_FATAL
:
435 pvrdma_dev_event(dev
, 1, eqe
->type
);
442 pvrdma_idx_ring_inc(&ring
->cons_head
, ring_slots
);
448 static inline struct pvrdma_cqne
*get_cqne(struct pvrdma_dev
*dev
,
451 return (struct pvrdma_cqne
*)pvrdma_page_dir_get_ptr(
454 sizeof(struct pvrdma_cqne
) * i
);
457 static irqreturn_t
pvrdma_intrx_handler(int irq
, void *dev_id
)
459 struct pvrdma_dev
*dev
= dev_id
;
460 struct pvrdma_ring
*ring
= &dev
->cq_ring_state
->rx
;
461 int ring_slots
= (dev
->dsr
->cq_ring_pages
.num_pages
- 1) * PAGE_SIZE
/
462 sizeof(struct pvrdma_cqne
);
466 dev_dbg(&dev
->pdev
->dev
, "interrupt x (completion) handler\n");
468 while (pvrdma_idx_ring_has_data(ring
, ring_slots
, &head
) > 0) {
469 struct pvrdma_cqne
*cqne
;
470 struct pvrdma_cq
*cq
;
472 cqne
= get_cqne(dev
, head
);
473 spin_lock_irqsave(&dev
->cq_tbl_lock
, flags
);
474 cq
= dev
->cq_tbl
[cqne
->info
% dev
->dsr
->caps
.max_cq
];
476 atomic_inc(&cq
->refcnt
);
477 spin_unlock_irqrestore(&dev
->cq_tbl_lock
, flags
);
479 if (cq
&& cq
->ibcq
.comp_handler
)
480 cq
->ibcq
.comp_handler(&cq
->ibcq
, cq
->ibcq
.cq_context
);
482 atomic_dec(&cq
->refcnt
);
483 if (atomic_read(&cq
->refcnt
))
486 pvrdma_idx_ring_inc(&ring
->cons_head
, ring_slots
);
492 static void pvrdma_free_irq(struct pvrdma_dev
*dev
)
496 dev_dbg(&dev
->pdev
->dev
, "freeing interrupts\n");
497 for (i
= 0; i
< dev
->nr_vectors
; i
++)
498 free_irq(pci_irq_vector(dev
->pdev
, i
), dev
);
501 static void pvrdma_enable_intrs(struct pvrdma_dev
*dev
)
503 dev_dbg(&dev
->pdev
->dev
, "enable interrupts\n");
504 pvrdma_write_reg(dev
, PVRDMA_REG_IMR
, 0);
507 static void pvrdma_disable_intrs(struct pvrdma_dev
*dev
)
509 dev_dbg(&dev
->pdev
->dev
, "disable interrupts\n");
510 pvrdma_write_reg(dev
, PVRDMA_REG_IMR
, ~0);
513 static int pvrdma_alloc_intrs(struct pvrdma_dev
*dev
)
515 struct pci_dev
*pdev
= dev
->pdev
;
518 ret
= pci_alloc_irq_vectors(pdev
, 1, PVRDMA_MAX_INTERRUPTS
,
521 ret
= pci_alloc_irq_vectors(pdev
, 1, 1,
522 PCI_IRQ_MSI
| PCI_IRQ_LEGACY
);
526 dev
->nr_vectors
= ret
;
528 ret
= request_irq(pci_irq_vector(dev
->pdev
, 0), pvrdma_intr0_handler
,
529 pdev
->msix_enabled
? 0 : IRQF_SHARED
, DRV_NAME
, dev
);
531 dev_err(&dev
->pdev
->dev
,
532 "failed to request interrupt 0\n");
533 goto out_free_vectors
;
536 for (i
= 1; i
< dev
->nr_vectors
; i
++) {
537 ret
= request_irq(pci_irq_vector(dev
->pdev
, i
),
538 i
== 1 ? pvrdma_intr1_handler
:
539 pvrdma_intrx_handler
,
542 dev_err(&dev
->pdev
->dev
,
543 "failed to request interrupt %d\n", i
);
552 free_irq(pci_irq_vector(dev
->pdev
, i
), dev
);
554 pci_free_irq_vectors(pdev
);
558 static void pvrdma_free_slots(struct pvrdma_dev
*dev
)
560 struct pci_dev
*pdev
= dev
->pdev
;
563 dma_free_coherent(&pdev
->dev
, PAGE_SIZE
, dev
->resp_slot
,
564 dev
->dsr
->resp_slot_dma
);
566 dma_free_coherent(&pdev
->dev
, PAGE_SIZE
, dev
->cmd_slot
,
567 dev
->dsr
->cmd_slot_dma
);
570 static int pvrdma_add_gid_at_index(struct pvrdma_dev
*dev
,
571 const union ib_gid
*gid
,
575 union pvrdma_cmd_req req
;
576 struct pvrdma_cmd_create_bind
*cmd_bind
= &req
.create_bind
;
578 if (!dev
->sgid_tbl
) {
579 dev_warn(&dev
->pdev
->dev
, "sgid table not initialized\n");
583 memset(cmd_bind
, 0, sizeof(*cmd_bind
));
584 cmd_bind
->hdr
.cmd
= PVRDMA_CMD_CREATE_BIND
;
585 memcpy(cmd_bind
->new_gid
, gid
->raw
, 16);
586 cmd_bind
->mtu
= ib_mtu_enum_to_int(IB_MTU_1024
);
587 cmd_bind
->vlan
= 0xfff;
588 cmd_bind
->index
= index
;
589 cmd_bind
->gid_type
= PVRDMA_GID_TYPE_FLAG_ROCE_V1
;
591 ret
= pvrdma_cmd_post(dev
, &req
, NULL
, 0);
593 dev_warn(&dev
->pdev
->dev
,
594 "could not create binding, error: %d\n", ret
);
597 memcpy(&dev
->sgid_tbl
[index
], gid
, sizeof(*gid
));
601 static int pvrdma_add_gid(struct ib_device
*ibdev
,
604 const union ib_gid
*gid
,
605 const struct ib_gid_attr
*attr
,
608 struct pvrdma_dev
*dev
= to_vdev(ibdev
);
610 return pvrdma_add_gid_at_index(dev
, gid
, index
);
613 static int pvrdma_del_gid_at_index(struct pvrdma_dev
*dev
, int index
)
616 union pvrdma_cmd_req req
;
617 struct pvrdma_cmd_destroy_bind
*cmd_dest
= &req
.destroy_bind
;
619 /* Update sgid table. */
620 if (!dev
->sgid_tbl
) {
621 dev_warn(&dev
->pdev
->dev
, "sgid table not initialized\n");
625 memset(cmd_dest
, 0, sizeof(*cmd_dest
));
626 cmd_dest
->hdr
.cmd
= PVRDMA_CMD_DESTROY_BIND
;
627 memcpy(cmd_dest
->dest_gid
, &dev
->sgid_tbl
[index
], 16);
628 cmd_dest
->index
= index
;
630 ret
= pvrdma_cmd_post(dev
, &req
, NULL
, 0);
632 dev_warn(&dev
->pdev
->dev
,
633 "could not destroy binding, error: %d\n", ret
);
636 memset(&dev
->sgid_tbl
[index
], 0, 16);
640 static int pvrdma_del_gid(struct ib_device
*ibdev
,
645 struct pvrdma_dev
*dev
= to_vdev(ibdev
);
647 dev_dbg(&dev
->pdev
->dev
, "removing gid at index %u from %s",
648 index
, dev
->netdev
->name
);
650 return pvrdma_del_gid_at_index(dev
, index
);
653 static void pvrdma_netdevice_event_handle(struct pvrdma_dev
*dev
,
659 pvrdma_dispatch_event(dev
, 1, IB_EVENT_PORT_ERR
);
662 pvrdma_dispatch_event(dev
, 1, IB_EVENT_PORT_ACTIVE
);
665 dev_dbg(&dev
->pdev
->dev
, "ignore netdevice event %ld on %s\n",
666 event
, dev
->ib_dev
.name
);
671 static void pvrdma_netdevice_event_work(struct work_struct
*work
)
673 struct pvrdma_netdevice_work
*netdev_work
;
674 struct pvrdma_dev
*dev
;
676 netdev_work
= container_of(work
, struct pvrdma_netdevice_work
, work
);
678 mutex_lock(&pvrdma_device_list_lock
);
679 list_for_each_entry(dev
, &pvrdma_device_list
, device_link
) {
680 if (dev
->netdev
== netdev_work
->event_netdev
) {
681 pvrdma_netdevice_event_handle(dev
, netdev_work
->event
);
685 mutex_unlock(&pvrdma_device_list_lock
);
690 static int pvrdma_netdevice_event(struct notifier_block
*this,
691 unsigned long event
, void *ptr
)
693 struct net_device
*event_netdev
= netdev_notifier_info_to_dev(ptr
);
694 struct pvrdma_netdevice_work
*netdev_work
;
696 netdev_work
= kmalloc(sizeof(*netdev_work
), GFP_ATOMIC
);
700 INIT_WORK(&netdev_work
->work
, pvrdma_netdevice_event_work
);
701 netdev_work
->event_netdev
= event_netdev
;
702 netdev_work
->event
= event
;
703 queue_work(event_wq
, &netdev_work
->work
);
708 static int pvrdma_pci_probe(struct pci_dev
*pdev
,
709 const struct pci_device_id
*id
)
711 struct pci_dev
*pdev_net
;
712 struct pvrdma_dev
*dev
;
716 unsigned int version
;
717 dma_addr_t slot_dma
= 0;
719 dev_dbg(&pdev
->dev
, "initializing driver %s\n", pci_name(pdev
));
721 /* Allocate zero-out device */
722 dev
= (struct pvrdma_dev
*)ib_alloc_device(sizeof(*dev
));
724 dev_err(&pdev
->dev
, "failed to allocate IB device\n");
728 mutex_lock(&pvrdma_device_list_lock
);
729 list_add(&dev
->device_link
, &pvrdma_device_list
);
730 mutex_unlock(&pvrdma_device_list_lock
);
732 ret
= pvrdma_init_device(dev
);
734 goto err_free_device
;
737 pci_set_drvdata(pdev
, dev
);
739 ret
= pci_enable_device(pdev
);
741 dev_err(&pdev
->dev
, "cannot enable PCI device\n");
742 goto err_free_device
;
745 dev_dbg(&pdev
->dev
, "PCI resource flags BAR0 %#lx\n",
746 pci_resource_flags(pdev
, 0));
747 dev_dbg(&pdev
->dev
, "PCI resource len %#llx\n",
748 (unsigned long long)pci_resource_len(pdev
, 0));
749 dev_dbg(&pdev
->dev
, "PCI resource start %#llx\n",
750 (unsigned long long)pci_resource_start(pdev
, 0));
751 dev_dbg(&pdev
->dev
, "PCI resource flags BAR1 %#lx\n",
752 pci_resource_flags(pdev
, 1));
753 dev_dbg(&pdev
->dev
, "PCI resource len %#llx\n",
754 (unsigned long long)pci_resource_len(pdev
, 1));
755 dev_dbg(&pdev
->dev
, "PCI resource start %#llx\n",
756 (unsigned long long)pci_resource_start(pdev
, 1));
758 if (!(pci_resource_flags(pdev
, 0) & IORESOURCE_MEM
) ||
759 !(pci_resource_flags(pdev
, 1) & IORESOURCE_MEM
)) {
760 dev_err(&pdev
->dev
, "PCI BAR region not MMIO\n");
762 goto err_free_device
;
765 ret
= pci_request_regions(pdev
, DRV_NAME
);
767 dev_err(&pdev
->dev
, "cannot request PCI resources\n");
768 goto err_disable_pdev
;
771 /* Enable 64-Bit DMA */
772 if (pci_set_dma_mask(pdev
, DMA_BIT_MASK(64)) == 0) {
773 ret
= pci_set_consistent_dma_mask(pdev
, DMA_BIT_MASK(64));
776 "pci_set_consistent_dma_mask failed\n");
777 goto err_free_resource
;
780 ret
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(32));
783 "pci_set_dma_mask failed\n");
784 goto err_free_resource
;
788 pci_set_master(pdev
);
790 /* Map register space */
791 start
= pci_resource_start(dev
->pdev
, PVRDMA_PCI_RESOURCE_REG
);
792 len
= pci_resource_len(dev
->pdev
, PVRDMA_PCI_RESOURCE_REG
);
793 dev
->regs
= ioremap(start
, len
);
795 dev_err(&pdev
->dev
, "register mapping failed\n");
797 goto err_free_resource
;
800 /* Setup per-device UAR. */
801 dev
->driver_uar
.index
= 0;
802 dev
->driver_uar
.pfn
=
803 pci_resource_start(dev
->pdev
, PVRDMA_PCI_RESOURCE_UAR
) >>
805 dev
->driver_uar
.map
=
806 ioremap(dev
->driver_uar
.pfn
<< PAGE_SHIFT
, PAGE_SIZE
);
807 if (!dev
->driver_uar
.map
) {
808 dev_err(&pdev
->dev
, "failed to remap UAR pages\n");
813 version
= pvrdma_read_reg(dev
, PVRDMA_REG_VERSION
);
814 dev_info(&pdev
->dev
, "device version %d, driver version %d\n",
815 version
, PVRDMA_VERSION
);
816 if (version
< PVRDMA_VERSION
) {
817 dev_err(&pdev
->dev
, "incompatible device version\n");
821 dev
->dsr
= dma_alloc_coherent(&pdev
->dev
, sizeof(*dev
->dsr
),
822 &dev
->dsrbase
, GFP_KERNEL
);
824 dev_err(&pdev
->dev
, "failed to allocate shared region\n");
829 /* Setup the shared region */
830 memset(dev
->dsr
, 0, sizeof(*dev
->dsr
));
831 dev
->dsr
->driver_version
= PVRDMA_VERSION
;
832 dev
->dsr
->gos_info
.gos_bits
= sizeof(void *) == 4 ?
835 dev
->dsr
->gos_info
.gos_type
= PVRDMA_GOS_TYPE_LINUX
;
836 dev
->dsr
->gos_info
.gos_ver
= 1;
837 dev
->dsr
->uar_pfn
= dev
->driver_uar
.pfn
;
840 dev
->cmd_slot
= dma_alloc_coherent(&pdev
->dev
, PAGE_SIZE
,
841 &slot_dma
, GFP_KERNEL
);
842 if (!dev
->cmd_slot
) {
847 dev
->dsr
->cmd_slot_dma
= (u64
)slot_dma
;
850 dev
->resp_slot
= dma_alloc_coherent(&pdev
->dev
, PAGE_SIZE
,
851 &slot_dma
, GFP_KERNEL
);
852 if (!dev
->resp_slot
) {
857 dev
->dsr
->resp_slot_dma
= (u64
)slot_dma
;
859 /* Async event ring */
860 dev
->dsr
->async_ring_pages
.num_pages
= 4;
861 ret
= pvrdma_page_dir_init(dev
, &dev
->async_pdir
,
862 dev
->dsr
->async_ring_pages
.num_pages
, true);
865 dev
->async_ring_state
= dev
->async_pdir
.pages
[0];
866 dev
->dsr
->async_ring_pages
.pdir_dma
= dev
->async_pdir
.dir_dma
;
868 /* CQ notification ring */
869 dev
->dsr
->cq_ring_pages
.num_pages
= 4;
870 ret
= pvrdma_page_dir_init(dev
, &dev
->cq_pdir
,
871 dev
->dsr
->cq_ring_pages
.num_pages
, true);
873 goto err_free_async_ring
;
874 dev
->cq_ring_state
= dev
->cq_pdir
.pages
[0];
875 dev
->dsr
->cq_ring_pages
.pdir_dma
= dev
->cq_pdir
.dir_dma
;
878 * Write the PA of the shared region to the device. The writes must be
879 * ordered such that the high bits are written last. When the writes
880 * complete, the device will have filled out the capabilities.
883 pvrdma_write_reg(dev
, PVRDMA_REG_DSRLOW
, (u32
)dev
->dsrbase
);
884 pvrdma_write_reg(dev
, PVRDMA_REG_DSRHIGH
,
885 (u32
)((u64
)(dev
->dsrbase
) >> 32));
887 /* Make sure the write is complete before reading status. */
890 /* Currently, the driver only supports RoCE mode. */
891 if (dev
->dsr
->caps
.mode
!= PVRDMA_DEVICE_MODE_ROCE
) {
892 dev_err(&pdev
->dev
, "unsupported transport %d\n",
893 dev
->dsr
->caps
.mode
);
895 goto err_free_cq_ring
;
898 /* Currently, the driver only supports RoCE V1. */
899 if (!(dev
->dsr
->caps
.gid_types
& PVRDMA_GID_TYPE_FLAG_ROCE_V1
)) {
900 dev_err(&pdev
->dev
, "driver needs RoCE v1 support\n");
902 goto err_free_cq_ring
;
905 /* Paired vmxnet3 will have same bus, slot. But func will be 0 */
906 pdev_net
= pci_get_slot(pdev
->bus
, PCI_DEVFN(PCI_SLOT(pdev
->devfn
), 0));
908 dev_err(&pdev
->dev
, "failed to find paired net device\n");
910 goto err_free_cq_ring
;
913 if (pdev_net
->vendor
!= PCI_VENDOR_ID_VMWARE
||
914 pdev_net
->device
!= PCI_DEVICE_ID_VMWARE_VMXNET3
) {
915 dev_err(&pdev
->dev
, "failed to find paired vmxnet3 device\n");
916 pci_dev_put(pdev_net
);
918 goto err_free_cq_ring
;
921 dev
->netdev
= pci_get_drvdata(pdev_net
);
922 pci_dev_put(pdev_net
);
924 dev_err(&pdev
->dev
, "failed to get vmxnet3 device\n");
926 goto err_free_cq_ring
;
929 dev_info(&pdev
->dev
, "paired device to %s\n", dev
->netdev
->name
);
931 /* Interrupt setup */
932 ret
= pvrdma_alloc_intrs(dev
);
934 dev_err(&pdev
->dev
, "failed to allocate interrupts\n");
936 goto err_free_cq_ring
;
939 /* Allocate UAR table. */
940 ret
= pvrdma_uar_table_init(dev
);
942 dev_err(&pdev
->dev
, "failed to allocate UAR table\n");
947 /* Allocate GID table */
948 dev
->sgid_tbl
= kcalloc(dev
->dsr
->caps
.gid_tbl_len
,
949 sizeof(union ib_gid
), GFP_KERNEL
);
950 if (!dev
->sgid_tbl
) {
952 goto err_free_uar_table
;
954 dev_dbg(&pdev
->dev
, "gid table len %d\n", dev
->dsr
->caps
.gid_tbl_len
);
956 pvrdma_enable_intrs(dev
);
958 /* Activate pvrdma device */
959 pvrdma_write_reg(dev
, PVRDMA_REG_CTL
, PVRDMA_DEVICE_CTL_ACTIVATE
);
961 /* Make sure the write is complete before reading status. */
964 /* Check if device was successfully activated */
965 ret
= pvrdma_read_reg(dev
, PVRDMA_REG_ERR
);
967 dev_err(&pdev
->dev
, "failed to activate device\n");
969 goto err_disable_intr
;
972 /* Register IB device */
973 ret
= pvrdma_register_device(dev
);
975 dev_err(&pdev
->dev
, "failed to register IB device\n");
976 goto err_disable_intr
;
979 dev
->nb_netdev
.notifier_call
= pvrdma_netdevice_event
;
980 ret
= register_netdevice_notifier(&dev
->nb_netdev
);
982 dev_err(&pdev
->dev
, "failed to register netdevice events\n");
983 goto err_unreg_ibdev
;
986 dev_info(&pdev
->dev
, "attached to device\n");
990 ib_unregister_device(&dev
->ib_dev
);
992 pvrdma_disable_intrs(dev
);
993 kfree(dev
->sgid_tbl
);
995 pvrdma_uar_table_cleanup(dev
);
997 pvrdma_free_irq(dev
);
998 pci_free_irq_vectors(pdev
);
1000 pvrdma_page_dir_cleanup(dev
, &dev
->cq_pdir
);
1001 err_free_async_ring
:
1002 pvrdma_page_dir_cleanup(dev
, &dev
->async_pdir
);
1004 pvrdma_free_slots(dev
);
1006 dma_free_coherent(&pdev
->dev
, sizeof(*dev
->dsr
), dev
->dsr
,
1009 iounmap(dev
->driver_uar
.map
);
1013 pci_release_regions(pdev
);
1015 pci_disable_device(pdev
);
1016 pci_set_drvdata(pdev
, NULL
);
1018 mutex_lock(&pvrdma_device_list_lock
);
1019 list_del(&dev
->device_link
);
1020 mutex_unlock(&pvrdma_device_list_lock
);
1021 ib_dealloc_device(&dev
->ib_dev
);
1025 static void pvrdma_pci_remove(struct pci_dev
*pdev
)
1027 struct pvrdma_dev
*dev
= pci_get_drvdata(pdev
);
1032 dev_info(&pdev
->dev
, "detaching from device\n");
1034 unregister_netdevice_notifier(&dev
->nb_netdev
);
1035 dev
->nb_netdev
.notifier_call
= NULL
;
1037 flush_workqueue(event_wq
);
1039 /* Unregister ib device */
1040 ib_unregister_device(&dev
->ib_dev
);
1042 mutex_lock(&pvrdma_device_list_lock
);
1043 list_del(&dev
->device_link
);
1044 mutex_unlock(&pvrdma_device_list_lock
);
1046 pvrdma_disable_intrs(dev
);
1047 pvrdma_free_irq(dev
);
1048 pci_free_irq_vectors(pdev
);
1050 /* Deactivate pvrdma device */
1051 pvrdma_write_reg(dev
, PVRDMA_REG_CTL
, PVRDMA_DEVICE_CTL_RESET
);
1052 pvrdma_page_dir_cleanup(dev
, &dev
->cq_pdir
);
1053 pvrdma_page_dir_cleanup(dev
, &dev
->async_pdir
);
1054 pvrdma_free_slots(dev
);
1057 kfree(dev
->sgid_tbl
);
1060 pvrdma_uar_table_cleanup(dev
);
1061 iounmap(dev
->driver_uar
.map
);
1063 ib_dealloc_device(&dev
->ib_dev
);
1065 /* Free pci resources */
1066 pci_release_regions(pdev
);
1067 pci_disable_device(pdev
);
1068 pci_set_drvdata(pdev
, NULL
);
1071 static struct pci_device_id pvrdma_pci_table
[] = {
1072 { PCI_DEVICE(PCI_VENDOR_ID_VMWARE
, PCI_DEVICE_ID_VMWARE_PVRDMA
), },
1076 MODULE_DEVICE_TABLE(pci
, pvrdma_pci_table
);
1078 static struct pci_driver pvrdma_driver
= {
1080 .id_table
= pvrdma_pci_table
,
1081 .probe
= pvrdma_pci_probe
,
1082 .remove
= pvrdma_pci_remove
,
1085 static int __init
pvrdma_init(void)
1089 event_wq
= alloc_ordered_workqueue("pvrdma_event_wq", WQ_MEM_RECLAIM
);
1093 err
= pci_register_driver(&pvrdma_driver
);
1095 destroy_workqueue(event_wq
);
1100 static void __exit
pvrdma_cleanup(void)
1102 pci_unregister_driver(&pvrdma_driver
);
1104 destroy_workqueue(event_wq
);
1107 module_init(pvrdma_init
);
1108 module_exit(pvrdma_cleanup
);
1110 MODULE_AUTHOR("VMware, Inc");
1111 MODULE_DESCRIPTION("VMware Paravirtual RDMA driver");
1112 MODULE_VERSION(DRV_VERSION
);
1113 MODULE_LICENSE("Dual BSD/GPL");