2 * Copyright (c) 2012-2016 VMware, Inc. All rights reserved.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of EITHER the GNU General Public License
6 * version 2 as published by the Free Software Foundation or the BSD
7 * 2-Clause License. This program is distributed in the hope that it
8 * will be useful, but WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED
9 * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
10 * See the GNU General Public License version 2 for more details at
11 * http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html.
13 * You should have received a copy of the GNU General Public License
14 * along with this program available in the file COPYING in the main
15 * directory of this source tree.
17 * The BSD 2-Clause License
19 * Redistribution and use in source and binary forms, with or
20 * without modification, are permitted provided that the following
23 * - Redistributions of source code must retain the above
24 * copyright notice, this list of conditions and the following
27 * - Redistributions in binary form must reproduce the above
28 * copyright notice, this list of conditions and the following
29 * disclaimer in the documentation and/or other materials
30 * provided with the distribution.
32 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
33 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
34 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
35 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
36 * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
37 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
38 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
39 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
40 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
41 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
42 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
43 * OF THE POSSIBILITY OF SUCH DAMAGE.
46 #include <linux/errno.h>
47 #include <linux/inetdevice.h>
48 #include <linux/init.h>
49 #include <linux/module.h>
50 #include <linux/slab.h>
51 #include <rdma/ib_addr.h>
52 #include <rdma/ib_smi.h>
53 #include <rdma/ib_user_verbs.h>
54 #include <net/addrconf.h>
58 #define DRV_NAME "vmw_pvrdma"
59 #define DRV_VERSION "1.0.0.0-k"
61 static DEFINE_MUTEX(pvrdma_device_list_lock
);
62 static LIST_HEAD(pvrdma_device_list
);
63 static struct workqueue_struct
*event_wq
;
65 static int pvrdma_add_gid(struct ib_device
*ibdev
,
68 const union ib_gid
*gid
,
69 const struct ib_gid_attr
*attr
,
71 static int pvrdma_del_gid(struct ib_device
*ibdev
,
77 static ssize_t
show_hca(struct device
*device
, struct device_attribute
*attr
,
80 return sprintf(buf
, "VMW_PVRDMA-%s\n", DRV_VERSION
);
83 static ssize_t
show_rev(struct device
*device
, struct device_attribute
*attr
,
86 return sprintf(buf
, "%d\n", PVRDMA_REV_ID
);
89 static ssize_t
show_board(struct device
*device
, struct device_attribute
*attr
,
92 return sprintf(buf
, "%d\n", PVRDMA_BOARD_ID
);
95 static DEVICE_ATTR(hw_rev
, S_IRUGO
, show_rev
, NULL
);
96 static DEVICE_ATTR(hca_type
, S_IRUGO
, show_hca
, NULL
);
97 static DEVICE_ATTR(board_id
, S_IRUGO
, show_board
, NULL
);
99 static struct device_attribute
*pvrdma_class_attributes
[] = {
105 static void pvrdma_get_fw_ver_str(struct ib_device
*device
, char *str
,
108 struct pvrdma_dev
*dev
=
109 container_of(device
, struct pvrdma_dev
, ib_dev
);
110 snprintf(str
, str_len
, "%d.%d.%d\n",
111 (int) (dev
->dsr
->caps
.fw_ver
>> 32),
112 (int) (dev
->dsr
->caps
.fw_ver
>> 16) & 0xffff,
113 (int) dev
->dsr
->caps
.fw_ver
& 0xffff);
116 static int pvrdma_init_device(struct pvrdma_dev
*dev
)
118 /* Initialize some device related stuff */
119 spin_lock_init(&dev
->cmd_lock
);
120 sema_init(&dev
->cmd_sema
, 1);
121 atomic_set(&dev
->num_qps
, 0);
122 atomic_set(&dev
->num_cqs
, 0);
123 atomic_set(&dev
->num_pds
, 0);
124 atomic_set(&dev
->num_ahs
, 0);
129 static int pvrdma_port_immutable(struct ib_device
*ibdev
, u8 port_num
,
130 struct ib_port_immutable
*immutable
)
132 struct ib_port_attr attr
;
135 immutable
->core_cap_flags
= RDMA_CORE_PORT_IBA_ROCE
;
137 err
= ib_query_port(ibdev
, port_num
, &attr
);
141 immutable
->pkey_tbl_len
= attr
.pkey_tbl_len
;
142 immutable
->gid_tbl_len
= attr
.gid_tbl_len
;
143 immutable
->max_mad_size
= IB_MGMT_MAD_SIZE
;
147 static struct net_device
*pvrdma_get_netdev(struct ib_device
*ibdev
,
150 struct net_device
*netdev
;
151 struct pvrdma_dev
*dev
= to_vdev(ibdev
);
157 netdev
= dev
->netdev
;
165 static int pvrdma_register_device(struct pvrdma_dev
*dev
)
170 strlcpy(dev
->ib_dev
.name
, "vmw_pvrdma%d", IB_DEVICE_NAME_MAX
);
171 dev
->ib_dev
.node_guid
= dev
->dsr
->caps
.node_guid
;
172 dev
->sys_image_guid
= dev
->dsr
->caps
.sys_image_guid
;
174 dev
->ib_dev
.owner
= THIS_MODULE
;
175 dev
->ib_dev
.num_comp_vectors
= 1;
176 dev
->ib_dev
.dev
.parent
= &dev
->pdev
->dev
;
177 dev
->ib_dev
.uverbs_abi_ver
= PVRDMA_UVERBS_ABI_VERSION
;
178 dev
->ib_dev
.uverbs_cmd_mask
=
179 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT
) |
180 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE
) |
181 (1ull << IB_USER_VERBS_CMD_QUERY_PORT
) |
182 (1ull << IB_USER_VERBS_CMD_ALLOC_PD
) |
183 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD
) |
184 (1ull << IB_USER_VERBS_CMD_REG_MR
) |
185 (1ull << IB_USER_VERBS_CMD_DEREG_MR
) |
186 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL
) |
187 (1ull << IB_USER_VERBS_CMD_CREATE_CQ
) |
188 (1ull << IB_USER_VERBS_CMD_POLL_CQ
) |
189 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ
) |
190 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ
) |
191 (1ull << IB_USER_VERBS_CMD_CREATE_QP
) |
192 (1ull << IB_USER_VERBS_CMD_MODIFY_QP
) |
193 (1ull << IB_USER_VERBS_CMD_QUERY_QP
) |
194 (1ull << IB_USER_VERBS_CMD_DESTROY_QP
) |
195 (1ull << IB_USER_VERBS_CMD_POST_SEND
) |
196 (1ull << IB_USER_VERBS_CMD_POST_RECV
) |
197 (1ull << IB_USER_VERBS_CMD_CREATE_AH
) |
198 (1ull << IB_USER_VERBS_CMD_DESTROY_AH
);
200 dev
->ib_dev
.node_type
= RDMA_NODE_IB_CA
;
201 dev
->ib_dev
.phys_port_cnt
= dev
->dsr
->caps
.phys_port_cnt
;
203 dev
->ib_dev
.query_device
= pvrdma_query_device
;
204 dev
->ib_dev
.query_port
= pvrdma_query_port
;
205 dev
->ib_dev
.query_gid
= pvrdma_query_gid
;
206 dev
->ib_dev
.query_pkey
= pvrdma_query_pkey
;
207 dev
->ib_dev
.modify_port
= pvrdma_modify_port
;
208 dev
->ib_dev
.alloc_ucontext
= pvrdma_alloc_ucontext
;
209 dev
->ib_dev
.dealloc_ucontext
= pvrdma_dealloc_ucontext
;
210 dev
->ib_dev
.mmap
= pvrdma_mmap
;
211 dev
->ib_dev
.alloc_pd
= pvrdma_alloc_pd
;
212 dev
->ib_dev
.dealloc_pd
= pvrdma_dealloc_pd
;
213 dev
->ib_dev
.create_ah
= pvrdma_create_ah
;
214 dev
->ib_dev
.destroy_ah
= pvrdma_destroy_ah
;
215 dev
->ib_dev
.create_qp
= pvrdma_create_qp
;
216 dev
->ib_dev
.modify_qp
= pvrdma_modify_qp
;
217 dev
->ib_dev
.query_qp
= pvrdma_query_qp
;
218 dev
->ib_dev
.destroy_qp
= pvrdma_destroy_qp
;
219 dev
->ib_dev
.post_send
= pvrdma_post_send
;
220 dev
->ib_dev
.post_recv
= pvrdma_post_recv
;
221 dev
->ib_dev
.create_cq
= pvrdma_create_cq
;
222 dev
->ib_dev
.modify_cq
= pvrdma_modify_cq
;
223 dev
->ib_dev
.resize_cq
= pvrdma_resize_cq
;
224 dev
->ib_dev
.destroy_cq
= pvrdma_destroy_cq
;
225 dev
->ib_dev
.poll_cq
= pvrdma_poll_cq
;
226 dev
->ib_dev
.req_notify_cq
= pvrdma_req_notify_cq
;
227 dev
->ib_dev
.get_dma_mr
= pvrdma_get_dma_mr
;
228 dev
->ib_dev
.reg_user_mr
= pvrdma_reg_user_mr
;
229 dev
->ib_dev
.dereg_mr
= pvrdma_dereg_mr
;
230 dev
->ib_dev
.alloc_mr
= pvrdma_alloc_mr
;
231 dev
->ib_dev
.map_mr_sg
= pvrdma_map_mr_sg
;
232 dev
->ib_dev
.add_gid
= pvrdma_add_gid
;
233 dev
->ib_dev
.del_gid
= pvrdma_del_gid
;
234 dev
->ib_dev
.get_netdev
= pvrdma_get_netdev
;
235 dev
->ib_dev
.get_port_immutable
= pvrdma_port_immutable
;
236 dev
->ib_dev
.get_link_layer
= pvrdma_port_link_layer
;
237 dev
->ib_dev
.get_dev_fw_str
= pvrdma_get_fw_ver_str
;
239 mutex_init(&dev
->port_mutex
);
240 spin_lock_init(&dev
->desc_lock
);
242 dev
->cq_tbl
= kcalloc(dev
->dsr
->caps
.max_cq
, sizeof(void *),
246 spin_lock_init(&dev
->cq_tbl_lock
);
248 dev
->qp_tbl
= kcalloc(dev
->dsr
->caps
.max_qp
, sizeof(void *),
252 spin_lock_init(&dev
->qp_tbl_lock
);
254 ret
= ib_register_device(&dev
->ib_dev
, NULL
);
258 for (i
= 0; i
< ARRAY_SIZE(pvrdma_class_attributes
); ++i
) {
259 ret
= device_create_file(&dev
->ib_dev
.dev
,
260 pvrdma_class_attributes
[i
]);
265 dev
->ib_active
= true;
270 ib_unregister_device(&dev
->ib_dev
);
279 static irqreturn_t
pvrdma_intr0_handler(int irq
, void *dev_id
)
281 u32 icr
= PVRDMA_INTR_CAUSE_RESPONSE
;
282 struct pvrdma_dev
*dev
= dev_id
;
284 dev_dbg(&dev
->pdev
->dev
, "interrupt 0 (response) handler\n");
286 if (!dev
->pdev
->msix_enabled
) {
288 icr
= pvrdma_read_reg(dev
, PVRDMA_REG_ICR
);
293 if (icr
== PVRDMA_INTR_CAUSE_RESPONSE
)
294 complete(&dev
->cmd_done
);
299 static void pvrdma_qp_event(struct pvrdma_dev
*dev
, u32 qpn
, int type
)
301 struct pvrdma_qp
*qp
;
304 spin_lock_irqsave(&dev
->qp_tbl_lock
, flags
);
305 qp
= dev
->qp_tbl
[qpn
% dev
->dsr
->caps
.max_qp
];
307 atomic_inc(&qp
->refcnt
);
308 spin_unlock_irqrestore(&dev
->qp_tbl_lock
, flags
);
310 if (qp
&& qp
->ibqp
.event_handler
) {
311 struct ib_qp
*ibqp
= &qp
->ibqp
;
314 e
.device
= ibqp
->device
;
316 e
.event
= type
; /* 1:1 mapping for now. */
317 ibqp
->event_handler(&e
, ibqp
->qp_context
);
320 atomic_dec(&qp
->refcnt
);
321 if (atomic_read(&qp
->refcnt
) == 0)
326 static void pvrdma_cq_event(struct pvrdma_dev
*dev
, u32 cqn
, int type
)
328 struct pvrdma_cq
*cq
;
331 spin_lock_irqsave(&dev
->cq_tbl_lock
, flags
);
332 cq
= dev
->cq_tbl
[cqn
% dev
->dsr
->caps
.max_cq
];
334 atomic_inc(&cq
->refcnt
);
335 spin_unlock_irqrestore(&dev
->cq_tbl_lock
, flags
);
337 if (cq
&& cq
->ibcq
.event_handler
) {
338 struct ib_cq
*ibcq
= &cq
->ibcq
;
341 e
.device
= ibcq
->device
;
343 e
.event
= type
; /* 1:1 mapping for now. */
344 ibcq
->event_handler(&e
, ibcq
->cq_context
);
347 atomic_dec(&cq
->refcnt
);
348 if (atomic_read(&cq
->refcnt
) == 0)
353 static void pvrdma_dispatch_event(struct pvrdma_dev
*dev
, int port
,
354 enum ib_event_type event
)
356 struct ib_event ib_event
;
358 memset(&ib_event
, 0, sizeof(ib_event
));
359 ib_event
.device
= &dev
->ib_dev
;
360 ib_event
.element
.port_num
= port
;
361 ib_event
.event
= event
;
362 ib_dispatch_event(&ib_event
);
365 static void pvrdma_dev_event(struct pvrdma_dev
*dev
, u8 port
, int type
)
367 if (port
< 1 || port
> dev
->dsr
->caps
.phys_port_cnt
) {
368 dev_warn(&dev
->pdev
->dev
, "event on port %d\n", port
);
372 pvrdma_dispatch_event(dev
, port
, type
);
375 static inline struct pvrdma_eqe
*get_eqe(struct pvrdma_dev
*dev
, unsigned int i
)
377 return (struct pvrdma_eqe
*)pvrdma_page_dir_get_ptr(
380 sizeof(struct pvrdma_eqe
) * i
);
383 static irqreturn_t
pvrdma_intr1_handler(int irq
, void *dev_id
)
385 struct pvrdma_dev
*dev
= dev_id
;
386 struct pvrdma_ring
*ring
= &dev
->async_ring_state
->rx
;
387 int ring_slots
= (dev
->dsr
->async_ring_pages
.num_pages
- 1) *
388 PAGE_SIZE
/ sizeof(struct pvrdma_eqe
);
391 dev_dbg(&dev
->pdev
->dev
, "interrupt 1 (async event) handler\n");
394 * Don't process events until the IB device is registered. Otherwise
395 * we'll try to ib_dispatch_event() on an invalid device.
400 while (pvrdma_idx_ring_has_data(ring
, ring_slots
, &head
) > 0) {
401 struct pvrdma_eqe
*eqe
;
403 eqe
= get_eqe(dev
, head
);
406 case PVRDMA_EVENT_QP_FATAL
:
407 case PVRDMA_EVENT_QP_REQ_ERR
:
408 case PVRDMA_EVENT_QP_ACCESS_ERR
:
409 case PVRDMA_EVENT_COMM_EST
:
410 case PVRDMA_EVENT_SQ_DRAINED
:
411 case PVRDMA_EVENT_PATH_MIG
:
412 case PVRDMA_EVENT_PATH_MIG_ERR
:
413 case PVRDMA_EVENT_QP_LAST_WQE_REACHED
:
414 pvrdma_qp_event(dev
, eqe
->info
, eqe
->type
);
417 case PVRDMA_EVENT_CQ_ERR
:
418 pvrdma_cq_event(dev
, eqe
->info
, eqe
->type
);
421 case PVRDMA_EVENT_SRQ_ERR
:
422 case PVRDMA_EVENT_SRQ_LIMIT_REACHED
:
425 case PVRDMA_EVENT_PORT_ACTIVE
:
426 case PVRDMA_EVENT_PORT_ERR
:
427 case PVRDMA_EVENT_LID_CHANGE
:
428 case PVRDMA_EVENT_PKEY_CHANGE
:
429 case PVRDMA_EVENT_SM_CHANGE
:
430 case PVRDMA_EVENT_CLIENT_REREGISTER
:
431 case PVRDMA_EVENT_GID_CHANGE
:
432 pvrdma_dev_event(dev
, eqe
->info
, eqe
->type
);
435 case PVRDMA_EVENT_DEVICE_FATAL
:
436 pvrdma_dev_event(dev
, 1, eqe
->type
);
443 pvrdma_idx_ring_inc(&ring
->cons_head
, ring_slots
);
449 static inline struct pvrdma_cqne
*get_cqne(struct pvrdma_dev
*dev
,
452 return (struct pvrdma_cqne
*)pvrdma_page_dir_get_ptr(
455 sizeof(struct pvrdma_cqne
) * i
);
458 static irqreturn_t
pvrdma_intrx_handler(int irq
, void *dev_id
)
460 struct pvrdma_dev
*dev
= dev_id
;
461 struct pvrdma_ring
*ring
= &dev
->cq_ring_state
->rx
;
462 int ring_slots
= (dev
->dsr
->cq_ring_pages
.num_pages
- 1) * PAGE_SIZE
/
463 sizeof(struct pvrdma_cqne
);
467 dev_dbg(&dev
->pdev
->dev
, "interrupt x (completion) handler\n");
469 while (pvrdma_idx_ring_has_data(ring
, ring_slots
, &head
) > 0) {
470 struct pvrdma_cqne
*cqne
;
471 struct pvrdma_cq
*cq
;
473 cqne
= get_cqne(dev
, head
);
474 spin_lock_irqsave(&dev
->cq_tbl_lock
, flags
);
475 cq
= dev
->cq_tbl
[cqne
->info
% dev
->dsr
->caps
.max_cq
];
477 atomic_inc(&cq
->refcnt
);
478 spin_unlock_irqrestore(&dev
->cq_tbl_lock
, flags
);
480 if (cq
&& cq
->ibcq
.comp_handler
)
481 cq
->ibcq
.comp_handler(&cq
->ibcq
, cq
->ibcq
.cq_context
);
483 atomic_dec(&cq
->refcnt
);
484 if (atomic_read(&cq
->refcnt
))
487 pvrdma_idx_ring_inc(&ring
->cons_head
, ring_slots
);
493 static void pvrdma_free_irq(struct pvrdma_dev
*dev
)
497 dev_dbg(&dev
->pdev
->dev
, "freeing interrupts\n");
498 for (i
= 0; i
< dev
->nr_vectors
; i
++)
499 free_irq(pci_irq_vector(dev
->pdev
, i
), dev
);
502 static void pvrdma_enable_intrs(struct pvrdma_dev
*dev
)
504 dev_dbg(&dev
->pdev
->dev
, "enable interrupts\n");
505 pvrdma_write_reg(dev
, PVRDMA_REG_IMR
, 0);
508 static void pvrdma_disable_intrs(struct pvrdma_dev
*dev
)
510 dev_dbg(&dev
->pdev
->dev
, "disable interrupts\n");
511 pvrdma_write_reg(dev
, PVRDMA_REG_IMR
, ~0);
514 static int pvrdma_alloc_intrs(struct pvrdma_dev
*dev
)
516 struct pci_dev
*pdev
= dev
->pdev
;
519 ret
= pci_alloc_irq_vectors(pdev
, 1, PVRDMA_MAX_INTERRUPTS
,
522 ret
= pci_alloc_irq_vectors(pdev
, 1, 1,
523 PCI_IRQ_MSI
| PCI_IRQ_LEGACY
);
527 dev
->nr_vectors
= ret
;
529 ret
= request_irq(pci_irq_vector(dev
->pdev
, 0), pvrdma_intr0_handler
,
530 pdev
->msix_enabled
? 0 : IRQF_SHARED
, DRV_NAME
, dev
);
532 dev_err(&dev
->pdev
->dev
,
533 "failed to request interrupt 0\n");
534 goto out_free_vectors
;
537 for (i
= 1; i
< dev
->nr_vectors
; i
++) {
538 ret
= request_irq(pci_irq_vector(dev
->pdev
, i
),
539 i
== 1 ? pvrdma_intr1_handler
:
540 pvrdma_intrx_handler
,
543 dev_err(&dev
->pdev
->dev
,
544 "failed to request interrupt %d\n", i
);
553 free_irq(pci_irq_vector(dev
->pdev
, i
), dev
);
555 pci_free_irq_vectors(pdev
);
559 static void pvrdma_free_slots(struct pvrdma_dev
*dev
)
561 struct pci_dev
*pdev
= dev
->pdev
;
564 dma_free_coherent(&pdev
->dev
, PAGE_SIZE
, dev
->resp_slot
,
565 dev
->dsr
->resp_slot_dma
);
567 dma_free_coherent(&pdev
->dev
, PAGE_SIZE
, dev
->cmd_slot
,
568 dev
->dsr
->cmd_slot_dma
);
571 static int pvrdma_add_gid_at_index(struct pvrdma_dev
*dev
,
572 const union ib_gid
*gid
,
576 union pvrdma_cmd_req req
;
577 struct pvrdma_cmd_create_bind
*cmd_bind
= &req
.create_bind
;
579 if (!dev
->sgid_tbl
) {
580 dev_warn(&dev
->pdev
->dev
, "sgid table not initialized\n");
584 memset(cmd_bind
, 0, sizeof(*cmd_bind
));
585 cmd_bind
->hdr
.cmd
= PVRDMA_CMD_CREATE_BIND
;
586 memcpy(cmd_bind
->new_gid
, gid
->raw
, 16);
587 cmd_bind
->mtu
= ib_mtu_enum_to_int(IB_MTU_1024
);
588 cmd_bind
->vlan
= 0xfff;
589 cmd_bind
->index
= index
;
590 cmd_bind
->gid_type
= PVRDMA_GID_TYPE_FLAG_ROCE_V1
;
592 ret
= pvrdma_cmd_post(dev
, &req
, NULL
, 0);
594 dev_warn(&dev
->pdev
->dev
,
595 "could not create binding, error: %d\n", ret
);
598 memcpy(&dev
->sgid_tbl
[index
], gid
, sizeof(*gid
));
602 static int pvrdma_add_gid(struct ib_device
*ibdev
,
605 const union ib_gid
*gid
,
606 const struct ib_gid_attr
*attr
,
609 struct pvrdma_dev
*dev
= to_vdev(ibdev
);
611 return pvrdma_add_gid_at_index(dev
, gid
, index
);
614 static int pvrdma_del_gid_at_index(struct pvrdma_dev
*dev
, int index
)
617 union pvrdma_cmd_req req
;
618 struct pvrdma_cmd_destroy_bind
*cmd_dest
= &req
.destroy_bind
;
620 /* Update sgid table. */
621 if (!dev
->sgid_tbl
) {
622 dev_warn(&dev
->pdev
->dev
, "sgid table not initialized\n");
626 memset(cmd_dest
, 0, sizeof(*cmd_dest
));
627 cmd_dest
->hdr
.cmd
= PVRDMA_CMD_DESTROY_BIND
;
628 memcpy(cmd_dest
->dest_gid
, &dev
->sgid_tbl
[index
], 16);
629 cmd_dest
->index
= index
;
631 ret
= pvrdma_cmd_post(dev
, &req
, NULL
, 0);
633 dev_warn(&dev
->pdev
->dev
,
634 "could not destroy binding, error: %d\n", ret
);
637 memset(&dev
->sgid_tbl
[index
], 0, 16);
641 static int pvrdma_del_gid(struct ib_device
*ibdev
,
646 struct pvrdma_dev
*dev
= to_vdev(ibdev
);
648 dev_dbg(&dev
->pdev
->dev
, "removing gid at index %u from %s",
649 index
, dev
->netdev
->name
);
651 return pvrdma_del_gid_at_index(dev
, index
);
654 static void pvrdma_netdevice_event_handle(struct pvrdma_dev
*dev
,
660 pvrdma_dispatch_event(dev
, 1, IB_EVENT_PORT_ERR
);
663 pvrdma_dispatch_event(dev
, 1, IB_EVENT_PORT_ACTIVE
);
666 dev_dbg(&dev
->pdev
->dev
, "ignore netdevice event %ld on %s\n",
667 event
, dev
->ib_dev
.name
);
672 static void pvrdma_netdevice_event_work(struct work_struct
*work
)
674 struct pvrdma_netdevice_work
*netdev_work
;
675 struct pvrdma_dev
*dev
;
677 netdev_work
= container_of(work
, struct pvrdma_netdevice_work
, work
);
679 mutex_lock(&pvrdma_device_list_lock
);
680 list_for_each_entry(dev
, &pvrdma_device_list
, device_link
) {
681 if (dev
->netdev
== netdev_work
->event_netdev
) {
682 pvrdma_netdevice_event_handle(dev
, netdev_work
->event
);
686 mutex_unlock(&pvrdma_device_list_lock
);
691 static int pvrdma_netdevice_event(struct notifier_block
*this,
692 unsigned long event
, void *ptr
)
694 struct net_device
*event_netdev
= netdev_notifier_info_to_dev(ptr
);
695 struct pvrdma_netdevice_work
*netdev_work
;
697 netdev_work
= kmalloc(sizeof(*netdev_work
), GFP_ATOMIC
);
701 INIT_WORK(&netdev_work
->work
, pvrdma_netdevice_event_work
);
702 netdev_work
->event_netdev
= event_netdev
;
703 netdev_work
->event
= event
;
704 queue_work(event_wq
, &netdev_work
->work
);
709 static int pvrdma_pci_probe(struct pci_dev
*pdev
,
710 const struct pci_device_id
*id
)
712 struct pci_dev
*pdev_net
;
713 struct pvrdma_dev
*dev
;
717 unsigned int version
;
718 dma_addr_t slot_dma
= 0;
720 dev_dbg(&pdev
->dev
, "initializing driver %s\n", pci_name(pdev
));
722 /* Allocate zero-out device */
723 dev
= (struct pvrdma_dev
*)ib_alloc_device(sizeof(*dev
));
725 dev_err(&pdev
->dev
, "failed to allocate IB device\n");
729 mutex_lock(&pvrdma_device_list_lock
);
730 list_add(&dev
->device_link
, &pvrdma_device_list
);
731 mutex_unlock(&pvrdma_device_list_lock
);
733 ret
= pvrdma_init_device(dev
);
735 goto err_free_device
;
738 pci_set_drvdata(pdev
, dev
);
740 ret
= pci_enable_device(pdev
);
742 dev_err(&pdev
->dev
, "cannot enable PCI device\n");
743 goto err_free_device
;
746 dev_dbg(&pdev
->dev
, "PCI resource flags BAR0 %#lx\n",
747 pci_resource_flags(pdev
, 0));
748 dev_dbg(&pdev
->dev
, "PCI resource len %#llx\n",
749 (unsigned long long)pci_resource_len(pdev
, 0));
750 dev_dbg(&pdev
->dev
, "PCI resource start %#llx\n",
751 (unsigned long long)pci_resource_start(pdev
, 0));
752 dev_dbg(&pdev
->dev
, "PCI resource flags BAR1 %#lx\n",
753 pci_resource_flags(pdev
, 1));
754 dev_dbg(&pdev
->dev
, "PCI resource len %#llx\n",
755 (unsigned long long)pci_resource_len(pdev
, 1));
756 dev_dbg(&pdev
->dev
, "PCI resource start %#llx\n",
757 (unsigned long long)pci_resource_start(pdev
, 1));
759 if (!(pci_resource_flags(pdev
, 0) & IORESOURCE_MEM
) ||
760 !(pci_resource_flags(pdev
, 1) & IORESOURCE_MEM
)) {
761 dev_err(&pdev
->dev
, "PCI BAR region not MMIO\n");
763 goto err_free_device
;
766 ret
= pci_request_regions(pdev
, DRV_NAME
);
768 dev_err(&pdev
->dev
, "cannot request PCI resources\n");
769 goto err_disable_pdev
;
772 /* Enable 64-Bit DMA */
773 if (pci_set_dma_mask(pdev
, DMA_BIT_MASK(64)) == 0) {
774 ret
= pci_set_consistent_dma_mask(pdev
, DMA_BIT_MASK(64));
777 "pci_set_consistent_dma_mask failed\n");
778 goto err_free_resource
;
781 ret
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(32));
784 "pci_set_dma_mask failed\n");
785 goto err_free_resource
;
789 pci_set_master(pdev
);
791 /* Map register space */
792 start
= pci_resource_start(dev
->pdev
, PVRDMA_PCI_RESOURCE_REG
);
793 len
= pci_resource_len(dev
->pdev
, PVRDMA_PCI_RESOURCE_REG
);
794 dev
->regs
= ioremap(start
, len
);
796 dev_err(&pdev
->dev
, "register mapping failed\n");
798 goto err_free_resource
;
801 /* Setup per-device UAR. */
802 dev
->driver_uar
.index
= 0;
803 dev
->driver_uar
.pfn
=
804 pci_resource_start(dev
->pdev
, PVRDMA_PCI_RESOURCE_UAR
) >>
806 dev
->driver_uar
.map
=
807 ioremap(dev
->driver_uar
.pfn
<< PAGE_SHIFT
, PAGE_SIZE
);
808 if (!dev
->driver_uar
.map
) {
809 dev_err(&pdev
->dev
, "failed to remap UAR pages\n");
814 version
= pvrdma_read_reg(dev
, PVRDMA_REG_VERSION
);
815 dev_info(&pdev
->dev
, "device version %d, driver version %d\n",
816 version
, PVRDMA_VERSION
);
817 if (version
< PVRDMA_VERSION
) {
818 dev_err(&pdev
->dev
, "incompatible device version\n");
822 dev
->dsr
= dma_alloc_coherent(&pdev
->dev
, sizeof(*dev
->dsr
),
823 &dev
->dsrbase
, GFP_KERNEL
);
825 dev_err(&pdev
->dev
, "failed to allocate shared region\n");
830 /* Setup the shared region */
831 memset(dev
->dsr
, 0, sizeof(*dev
->dsr
));
832 dev
->dsr
->driver_version
= PVRDMA_VERSION
;
833 dev
->dsr
->gos_info
.gos_bits
= sizeof(void *) == 4 ?
836 dev
->dsr
->gos_info
.gos_type
= PVRDMA_GOS_TYPE_LINUX
;
837 dev
->dsr
->gos_info
.gos_ver
= 1;
838 dev
->dsr
->uar_pfn
= dev
->driver_uar
.pfn
;
841 dev
->cmd_slot
= dma_alloc_coherent(&pdev
->dev
, PAGE_SIZE
,
842 &slot_dma
, GFP_KERNEL
);
843 if (!dev
->cmd_slot
) {
848 dev
->dsr
->cmd_slot_dma
= (u64
)slot_dma
;
851 dev
->resp_slot
= dma_alloc_coherent(&pdev
->dev
, PAGE_SIZE
,
852 &slot_dma
, GFP_KERNEL
);
853 if (!dev
->resp_slot
) {
858 dev
->dsr
->resp_slot_dma
= (u64
)slot_dma
;
860 /* Async event ring */
861 dev
->dsr
->async_ring_pages
.num_pages
= 4;
862 ret
= pvrdma_page_dir_init(dev
, &dev
->async_pdir
,
863 dev
->dsr
->async_ring_pages
.num_pages
, true);
866 dev
->async_ring_state
= dev
->async_pdir
.pages
[0];
867 dev
->dsr
->async_ring_pages
.pdir_dma
= dev
->async_pdir
.dir_dma
;
869 /* CQ notification ring */
870 dev
->dsr
->cq_ring_pages
.num_pages
= 4;
871 ret
= pvrdma_page_dir_init(dev
, &dev
->cq_pdir
,
872 dev
->dsr
->cq_ring_pages
.num_pages
, true);
874 goto err_free_async_ring
;
875 dev
->cq_ring_state
= dev
->cq_pdir
.pages
[0];
876 dev
->dsr
->cq_ring_pages
.pdir_dma
= dev
->cq_pdir
.dir_dma
;
879 * Write the PA of the shared region to the device. The writes must be
880 * ordered such that the high bits are written last. When the writes
881 * complete, the device will have filled out the capabilities.
884 pvrdma_write_reg(dev
, PVRDMA_REG_DSRLOW
, (u32
)dev
->dsrbase
);
885 pvrdma_write_reg(dev
, PVRDMA_REG_DSRHIGH
,
886 (u32
)((u64
)(dev
->dsrbase
) >> 32));
888 /* Make sure the write is complete before reading status. */
891 /* Currently, the driver only supports RoCE mode. */
892 if (dev
->dsr
->caps
.mode
!= PVRDMA_DEVICE_MODE_ROCE
) {
893 dev_err(&pdev
->dev
, "unsupported transport %d\n",
894 dev
->dsr
->caps
.mode
);
896 goto err_free_cq_ring
;
899 /* Currently, the driver only supports RoCE V1. */
900 if (!(dev
->dsr
->caps
.gid_types
& PVRDMA_GID_TYPE_FLAG_ROCE_V1
)) {
901 dev_err(&pdev
->dev
, "driver needs RoCE v1 support\n");
903 goto err_free_cq_ring
;
906 /* Paired vmxnet3 will have same bus, slot. But func will be 0 */
907 pdev_net
= pci_get_slot(pdev
->bus
, PCI_DEVFN(PCI_SLOT(pdev
->devfn
), 0));
909 dev_err(&pdev
->dev
, "failed to find paired net device\n");
911 goto err_free_cq_ring
;
914 if (pdev_net
->vendor
!= PCI_VENDOR_ID_VMWARE
||
915 pdev_net
->device
!= PCI_DEVICE_ID_VMWARE_VMXNET3
) {
916 dev_err(&pdev
->dev
, "failed to find paired vmxnet3 device\n");
917 pci_dev_put(pdev_net
);
919 goto err_free_cq_ring
;
922 dev
->netdev
= pci_get_drvdata(pdev_net
);
923 pci_dev_put(pdev_net
);
925 dev_err(&pdev
->dev
, "failed to get vmxnet3 device\n");
927 goto err_free_cq_ring
;
930 dev_info(&pdev
->dev
, "paired device to %s\n", dev
->netdev
->name
);
932 /* Interrupt setup */
933 ret
= pvrdma_alloc_intrs(dev
);
935 dev_err(&pdev
->dev
, "failed to allocate interrupts\n");
937 goto err_free_cq_ring
;
940 /* Allocate UAR table. */
941 ret
= pvrdma_uar_table_init(dev
);
943 dev_err(&pdev
->dev
, "failed to allocate UAR table\n");
948 /* Allocate GID table */
949 dev
->sgid_tbl
= kcalloc(dev
->dsr
->caps
.gid_tbl_len
,
950 sizeof(union ib_gid
), GFP_KERNEL
);
951 if (!dev
->sgid_tbl
) {
953 goto err_free_uar_table
;
955 dev_dbg(&pdev
->dev
, "gid table len %d\n", dev
->dsr
->caps
.gid_tbl_len
);
957 pvrdma_enable_intrs(dev
);
959 /* Activate pvrdma device */
960 pvrdma_write_reg(dev
, PVRDMA_REG_CTL
, PVRDMA_DEVICE_CTL_ACTIVATE
);
962 /* Make sure the write is complete before reading status. */
965 /* Check if device was successfully activated */
966 ret
= pvrdma_read_reg(dev
, PVRDMA_REG_ERR
);
968 dev_err(&pdev
->dev
, "failed to activate device\n");
970 goto err_disable_intr
;
973 /* Register IB device */
974 ret
= pvrdma_register_device(dev
);
976 dev_err(&pdev
->dev
, "failed to register IB device\n");
977 goto err_disable_intr
;
980 dev
->nb_netdev
.notifier_call
= pvrdma_netdevice_event
;
981 ret
= register_netdevice_notifier(&dev
->nb_netdev
);
983 dev_err(&pdev
->dev
, "failed to register netdevice events\n");
984 goto err_unreg_ibdev
;
987 dev_info(&pdev
->dev
, "attached to device\n");
991 ib_unregister_device(&dev
->ib_dev
);
993 pvrdma_disable_intrs(dev
);
994 kfree(dev
->sgid_tbl
);
996 pvrdma_uar_table_cleanup(dev
);
998 pvrdma_free_irq(dev
);
999 pci_free_irq_vectors(pdev
);
1001 pvrdma_page_dir_cleanup(dev
, &dev
->cq_pdir
);
1002 err_free_async_ring
:
1003 pvrdma_page_dir_cleanup(dev
, &dev
->async_pdir
);
1005 pvrdma_free_slots(dev
);
1007 dma_free_coherent(&pdev
->dev
, sizeof(*dev
->dsr
), dev
->dsr
,
1010 iounmap(dev
->driver_uar
.map
);
1014 pci_release_regions(pdev
);
1016 pci_disable_device(pdev
);
1017 pci_set_drvdata(pdev
, NULL
);
1019 mutex_lock(&pvrdma_device_list_lock
);
1020 list_del(&dev
->device_link
);
1021 mutex_unlock(&pvrdma_device_list_lock
);
1022 ib_dealloc_device(&dev
->ib_dev
);
1026 static void pvrdma_pci_remove(struct pci_dev
*pdev
)
1028 struct pvrdma_dev
*dev
= pci_get_drvdata(pdev
);
1033 dev_info(&pdev
->dev
, "detaching from device\n");
1035 unregister_netdevice_notifier(&dev
->nb_netdev
);
1036 dev
->nb_netdev
.notifier_call
= NULL
;
1038 flush_workqueue(event_wq
);
1040 /* Unregister ib device */
1041 ib_unregister_device(&dev
->ib_dev
);
1043 mutex_lock(&pvrdma_device_list_lock
);
1044 list_del(&dev
->device_link
);
1045 mutex_unlock(&pvrdma_device_list_lock
);
1047 pvrdma_disable_intrs(dev
);
1048 pvrdma_free_irq(dev
);
1049 pci_free_irq_vectors(pdev
);
1051 /* Deactivate pvrdma device */
1052 pvrdma_write_reg(dev
, PVRDMA_REG_CTL
, PVRDMA_DEVICE_CTL_RESET
);
1053 pvrdma_page_dir_cleanup(dev
, &dev
->cq_pdir
);
1054 pvrdma_page_dir_cleanup(dev
, &dev
->async_pdir
);
1055 pvrdma_free_slots(dev
);
1058 kfree(dev
->sgid_tbl
);
1061 pvrdma_uar_table_cleanup(dev
);
1062 iounmap(dev
->driver_uar
.map
);
1064 ib_dealloc_device(&dev
->ib_dev
);
1066 /* Free pci resources */
1067 pci_release_regions(pdev
);
1068 pci_disable_device(pdev
);
1069 pci_set_drvdata(pdev
, NULL
);
1072 static struct pci_device_id pvrdma_pci_table
[] = {
1073 { PCI_DEVICE(PCI_VENDOR_ID_VMWARE
, PCI_DEVICE_ID_VMWARE_PVRDMA
), },
1077 MODULE_DEVICE_TABLE(pci
, pvrdma_pci_table
);
1079 static struct pci_driver pvrdma_driver
= {
1081 .id_table
= pvrdma_pci_table
,
1082 .probe
= pvrdma_pci_probe
,
1083 .remove
= pvrdma_pci_remove
,
1086 static int __init
pvrdma_init(void)
1090 event_wq
= alloc_ordered_workqueue("pvrdma_event_wq", WQ_MEM_RECLAIM
);
1094 err
= pci_register_driver(&pvrdma_driver
);
1096 destroy_workqueue(event_wq
);
1101 static void __exit
pvrdma_cleanup(void)
1103 pci_unregister_driver(&pvrdma_driver
);
1105 destroy_workqueue(event_wq
);
1108 module_init(pvrdma_init
);
1109 module_exit(pvrdma_cleanup
);
1111 MODULE_AUTHOR("VMware, Inc");
1112 MODULE_DESCRIPTION("VMware Paravirtual RDMA driver");
1113 MODULE_VERSION(DRV_VERSION
);
1114 MODULE_LICENSE("Dual BSD/GPL");