1 /* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 #include <linux/module.h>
33 #include <rdma/ib_verbs.h>
34 #include <rdma/ib_addr.h>
35 #include <rdma/ib_user_verbs.h>
36 #include <rdma/iw_cm.h>
37 #include <rdma/ib_mad.h>
38 #include <linux/netdevice.h>
39 #include <linux/iommu.h>
40 #include <linux/pci.h>
41 #include <net/addrconf.h>
43 #include <linux/qed/qed_chain.h>
44 #include <linux/qed/qed_if.h>
47 #include <rdma/qedr-abi.h>
48 #include "qedr_iw_cm.h"
50 MODULE_DESCRIPTION("QLogic 40G/100G ROCE Driver");
51 MODULE_AUTHOR("QLogic Corporation");
52 MODULE_LICENSE("Dual BSD/GPL");
54 #define QEDR_WQ_MULTIPLIER_DFT (3)
56 static void qedr_ib_dispatch_event(struct qedr_dev
*dev
, u8 port_num
,
57 enum ib_event_type type
)
61 ibev
.device
= &dev
->ibdev
;
62 ibev
.element
.port_num
= port_num
;
65 ib_dispatch_event(&ibev
);
68 static enum rdma_link_layer
qedr_link_layer(struct ib_device
*device
,
71 return IB_LINK_LAYER_ETHERNET
;
74 static void qedr_get_dev_fw_str(struct ib_device
*ibdev
, char *str
)
76 struct qedr_dev
*qedr
= get_qedr_dev(ibdev
);
77 u32 fw_ver
= (u32
)qedr
->attr
.fw_ver
;
79 snprintf(str
, IB_FW_VERSION_NAME_MAX
, "%d. %d. %d. %d",
80 (fw_ver
>> 24) & 0xFF, (fw_ver
>> 16) & 0xFF,
81 (fw_ver
>> 8) & 0xFF, fw_ver
& 0xFF);
84 static int qedr_roce_port_immutable(struct ib_device
*ibdev
, u8 port_num
,
85 struct ib_port_immutable
*immutable
)
87 struct ib_port_attr attr
;
90 err
= qedr_query_port(ibdev
, port_num
, &attr
);
94 immutable
->pkey_tbl_len
= attr
.pkey_tbl_len
;
95 immutable
->gid_tbl_len
= attr
.gid_tbl_len
;
96 immutable
->core_cap_flags
= RDMA_CORE_PORT_IBA_ROCE
|
97 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP
;
98 immutable
->max_mad_size
= IB_MGMT_MAD_SIZE
;
103 static int qedr_iw_port_immutable(struct ib_device
*ibdev
, u8 port_num
,
104 struct ib_port_immutable
*immutable
)
106 struct ib_port_attr attr
;
109 err
= qedr_query_port(ibdev
, port_num
, &attr
);
113 immutable
->pkey_tbl_len
= 1;
114 immutable
->gid_tbl_len
= 1;
115 immutable
->core_cap_flags
= RDMA_CORE_PORT_IWARP
;
116 immutable
->max_mad_size
= 0;
121 /* QEDR sysfs interface */
122 static ssize_t
hw_rev_show(struct device
*device
, struct device_attribute
*attr
,
125 struct qedr_dev
*dev
=
126 rdma_device_to_drv_device(device
, struct qedr_dev
, ibdev
);
128 return scnprintf(buf
, PAGE_SIZE
, "0x%x\n", dev
->pdev
->vendor
);
130 static DEVICE_ATTR_RO(hw_rev
);
132 static ssize_t
hca_type_show(struct device
*device
,
133 struct device_attribute
*attr
, char *buf
)
135 return scnprintf(buf
, PAGE_SIZE
, "%s\n", "HCA_TYPE_TO_SET");
137 static DEVICE_ATTR_RO(hca_type
);
139 static struct attribute
*qedr_attributes
[] = {
140 &dev_attr_hw_rev
.attr
,
141 &dev_attr_hca_type
.attr
,
145 static const struct attribute_group qedr_attr_group
= {
146 .attrs
= qedr_attributes
,
149 static const struct ib_device_ops qedr_iw_dev_ops
= {
150 .get_port_immutable
= qedr_iw_port_immutable
,
151 .iw_accept
= qedr_iw_accept
,
152 .iw_add_ref
= qedr_iw_qp_add_ref
,
153 .iw_connect
= qedr_iw_connect
,
154 .iw_create_listen
= qedr_iw_create_listen
,
155 .iw_destroy_listen
= qedr_iw_destroy_listen
,
156 .iw_get_qp
= qedr_iw_get_qp
,
157 .iw_reject
= qedr_iw_reject
,
158 .iw_rem_ref
= qedr_iw_qp_rem_ref
,
159 .query_gid
= qedr_iw_query_gid
,
162 static int qedr_iw_register_device(struct qedr_dev
*dev
)
164 dev
->ibdev
.node_type
= RDMA_NODE_RNIC
;
166 ib_set_device_ops(&dev
->ibdev
, &qedr_iw_dev_ops
);
168 memcpy(dev
->ibdev
.iw_ifname
,
169 dev
->ndev
->name
, sizeof(dev
->ibdev
.iw_ifname
));
174 static const struct ib_device_ops qedr_roce_dev_ops
= {
175 .get_port_immutable
= qedr_roce_port_immutable
,
178 static void qedr_roce_register_device(struct qedr_dev
*dev
)
180 dev
->ibdev
.node_type
= RDMA_NODE_IB_CA
;
182 ib_set_device_ops(&dev
->ibdev
, &qedr_roce_dev_ops
);
185 static const struct ib_device_ops qedr_dev_ops
= {
186 .owner
= THIS_MODULE
,
187 .driver_id
= RDMA_DRIVER_QEDR
,
188 .uverbs_abi_ver
= QEDR_ABI_VERSION
,
190 .alloc_mr
= qedr_alloc_mr
,
191 .alloc_pd
= qedr_alloc_pd
,
192 .alloc_ucontext
= qedr_alloc_ucontext
,
193 .create_ah
= qedr_create_ah
,
194 .create_cq
= qedr_create_cq
,
195 .create_qp
= qedr_create_qp
,
196 .create_srq
= qedr_create_srq
,
197 .dealloc_pd
= qedr_dealloc_pd
,
198 .dealloc_ucontext
= qedr_dealloc_ucontext
,
199 .dereg_mr
= qedr_dereg_mr
,
200 .destroy_ah
= qedr_destroy_ah
,
201 .destroy_cq
= qedr_destroy_cq
,
202 .destroy_qp
= qedr_destroy_qp
,
203 .destroy_srq
= qedr_destroy_srq
,
204 .get_dev_fw_str
= qedr_get_dev_fw_str
,
205 .get_dma_mr
= qedr_get_dma_mr
,
206 .get_link_layer
= qedr_link_layer
,
207 .map_mr_sg
= qedr_map_mr_sg
,
209 .modify_port
= qedr_modify_port
,
210 .modify_qp
= qedr_modify_qp
,
211 .modify_srq
= qedr_modify_srq
,
212 .poll_cq
= qedr_poll_cq
,
213 .post_recv
= qedr_post_recv
,
214 .post_send
= qedr_post_send
,
215 .post_srq_recv
= qedr_post_srq_recv
,
216 .process_mad
= qedr_process_mad
,
217 .query_device
= qedr_query_device
,
218 .query_pkey
= qedr_query_pkey
,
219 .query_port
= qedr_query_port
,
220 .query_qp
= qedr_query_qp
,
221 .query_srq
= qedr_query_srq
,
222 .reg_user_mr
= qedr_reg_user_mr
,
223 .req_notify_cq
= qedr_arm_cq
,
224 .resize_cq
= qedr_resize_cq
,
226 INIT_RDMA_OBJ_SIZE(ib_ah
, qedr_ah
, ibah
),
227 INIT_RDMA_OBJ_SIZE(ib_cq
, qedr_cq
, ibcq
),
228 INIT_RDMA_OBJ_SIZE(ib_pd
, qedr_pd
, ibpd
),
229 INIT_RDMA_OBJ_SIZE(ib_srq
, qedr_srq
, ibsrq
),
230 INIT_RDMA_OBJ_SIZE(ib_ucontext
, qedr_ucontext
, ibucontext
),
233 static int qedr_register_device(struct qedr_dev
*dev
)
237 dev
->ibdev
.node_guid
= dev
->attr
.node_guid
;
238 memcpy(dev
->ibdev
.node_desc
, QEDR_NODE_DESC
, sizeof(QEDR_NODE_DESC
));
240 dev
->ibdev
.uverbs_cmd_mask
= QEDR_UVERBS(GET_CONTEXT
) |
241 QEDR_UVERBS(QUERY_DEVICE
) |
242 QEDR_UVERBS(QUERY_PORT
) |
243 QEDR_UVERBS(ALLOC_PD
) |
244 QEDR_UVERBS(DEALLOC_PD
) |
245 QEDR_UVERBS(CREATE_COMP_CHANNEL
) |
246 QEDR_UVERBS(CREATE_CQ
) |
247 QEDR_UVERBS(RESIZE_CQ
) |
248 QEDR_UVERBS(DESTROY_CQ
) |
249 QEDR_UVERBS(REQ_NOTIFY_CQ
) |
250 QEDR_UVERBS(CREATE_QP
) |
251 QEDR_UVERBS(MODIFY_QP
) |
252 QEDR_UVERBS(QUERY_QP
) |
253 QEDR_UVERBS(DESTROY_QP
) |
254 QEDR_UVERBS(CREATE_SRQ
) |
255 QEDR_UVERBS(DESTROY_SRQ
) |
256 QEDR_UVERBS(QUERY_SRQ
) |
257 QEDR_UVERBS(MODIFY_SRQ
) |
258 QEDR_UVERBS(POST_SRQ_RECV
) |
259 QEDR_UVERBS(REG_MR
) |
260 QEDR_UVERBS(DEREG_MR
) |
261 QEDR_UVERBS(POLL_CQ
) |
262 QEDR_UVERBS(POST_SEND
) |
263 QEDR_UVERBS(POST_RECV
);
266 rc
= qedr_iw_register_device(dev
);
270 qedr_roce_register_device(dev
);
273 dev
->ibdev
.phys_port_cnt
= 1;
274 dev
->ibdev
.num_comp_vectors
= dev
->num_cnq
;
275 dev
->ibdev
.dev
.parent
= &dev
->pdev
->dev
;
277 rdma_set_device_sysfs_group(&dev
->ibdev
, &qedr_attr_group
);
278 ib_set_device_ops(&dev
->ibdev
, &qedr_dev_ops
);
280 rc
= ib_device_set_netdev(&dev
->ibdev
, dev
->ndev
, 1);
284 return ib_register_device(&dev
->ibdev
, "qedr%d");
287 /* This function allocates fast-path status block memory */
288 static int qedr_alloc_mem_sb(struct qedr_dev
*dev
,
289 struct qed_sb_info
*sb_info
, u16 sb_id
)
291 struct status_block_e4
*sb_virt
;
295 sb_virt
= dma_alloc_coherent(&dev
->pdev
->dev
,
296 sizeof(*sb_virt
), &sb_phys
, GFP_KERNEL
);
300 rc
= dev
->ops
->common
->sb_init(dev
->cdev
, sb_info
,
301 sb_virt
, sb_phys
, sb_id
,
304 pr_err("Status block initialization failed\n");
305 dma_free_coherent(&dev
->pdev
->dev
, sizeof(*sb_virt
),
313 static void qedr_free_mem_sb(struct qedr_dev
*dev
,
314 struct qed_sb_info
*sb_info
, int sb_id
)
316 if (sb_info
->sb_virt
) {
317 dev
->ops
->common
->sb_release(dev
->cdev
, sb_info
, sb_id
,
319 dma_free_coherent(&dev
->pdev
->dev
, sizeof(*sb_info
->sb_virt
),
320 (void *)sb_info
->sb_virt
, sb_info
->sb_phys
);
324 static void qedr_free_resources(struct qedr_dev
*dev
)
329 destroy_workqueue(dev
->iwarp_wq
);
331 for (i
= 0; i
< dev
->num_cnq
; i
++) {
332 qedr_free_mem_sb(dev
, &dev
->sb_array
[i
], dev
->sb_start
+ i
);
333 dev
->ops
->common
->chain_free(dev
->cdev
, &dev
->cnq_array
[i
].pbl
);
336 kfree(dev
->cnq_array
);
337 kfree(dev
->sb_array
);
338 kfree(dev
->sgid_tbl
);
341 static int qedr_alloc_resources(struct qedr_dev
*dev
)
343 struct qedr_cnq
*cnq
;
348 dev
->sgid_tbl
= kcalloc(QEDR_MAX_SGID
, sizeof(union ib_gid
),
353 spin_lock_init(&dev
->sgid_lock
);
356 xa_init_flags(&dev
->qps
, XA_FLAGS_LOCK_IRQ
);
357 dev
->iwarp_wq
= create_singlethread_workqueue("qedr_iwarpq");
360 /* Allocate Status blocks for CNQ */
361 dev
->sb_array
= kcalloc(dev
->num_cnq
, sizeof(*dev
->sb_array
),
363 if (!dev
->sb_array
) {
368 dev
->cnq_array
= kcalloc(dev
->num_cnq
,
369 sizeof(*dev
->cnq_array
), GFP_KERNEL
);
370 if (!dev
->cnq_array
) {
375 dev
->sb_start
= dev
->ops
->rdma_get_start_sb(dev
->cdev
);
377 /* Allocate CNQ PBLs */
378 n_entries
= min_t(u32
, QED_RDMA_MAX_CNQ_SIZE
, QEDR_ROCE_MAX_CNQ_SIZE
);
379 for (i
= 0; i
< dev
->num_cnq
; i
++) {
380 cnq
= &dev
->cnq_array
[i
];
382 rc
= qedr_alloc_mem_sb(dev
, &dev
->sb_array
[i
],
387 rc
= dev
->ops
->common
->chain_alloc(dev
->cdev
,
388 QED_CHAIN_USE_TO_CONSUME
,
390 QED_CHAIN_CNT_TYPE_U16
,
392 sizeof(struct regpair
*),
398 cnq
->sb
= &dev
->sb_array
[i
];
399 cons_pi
= dev
->sb_array
[i
].sb_virt
->pi_array
;
400 cnq
->hw_cons_ptr
= &cons_pi
[QED_ROCE_PROTOCOL_INDEX
];
402 sprintf(cnq
->name
, "qedr%d@pci:%s", i
, pci_name(dev
->pdev
));
404 DP_DEBUG(dev
, QEDR_MSG_INIT
, "cnq[%d].cons=%d\n",
405 i
, qed_chain_get_cons_idx(&cnq
->pbl
));
410 qedr_free_mem_sb(dev
, &dev
->sb_array
[i
], dev
->sb_start
+ i
);
412 for (--i
; i
>= 0; i
--) {
413 dev
->ops
->common
->chain_free(dev
->cdev
, &dev
->cnq_array
[i
].pbl
);
414 qedr_free_mem_sb(dev
, &dev
->sb_array
[i
], dev
->sb_start
+ i
);
416 kfree(dev
->cnq_array
);
418 kfree(dev
->sb_array
);
420 kfree(dev
->sgid_tbl
);
424 static void qedr_pci_set_atomic(struct qedr_dev
*dev
, struct pci_dev
*pdev
)
426 int rc
= pci_enable_atomic_ops_to_root(pdev
,
427 PCI_EXP_DEVCAP2_ATOMIC_COMP64
);
430 dev
->atomic_cap
= IB_ATOMIC_NONE
;
431 DP_DEBUG(dev
, QEDR_MSG_INIT
, "Atomic capability disabled\n");
433 dev
->atomic_cap
= IB_ATOMIC_GLOB
;
434 DP_DEBUG(dev
, QEDR_MSG_INIT
, "Atomic capability enabled\n");
438 static const struct qed_rdma_ops
*qed_ops
;
440 #define HILO_U64(hi, lo) ((((u64)(hi)) << 32) + (lo))
442 static irqreturn_t
qedr_irq_handler(int irq
, void *handle
)
444 u16 hw_comp_cons
, sw_comp_cons
;
445 struct qedr_cnq
*cnq
= handle
;
446 struct regpair
*cq_handle
;
449 qed_sb_ack(cnq
->sb
, IGU_INT_DISABLE
, 0);
451 qed_sb_update_sb_idx(cnq
->sb
);
453 hw_comp_cons
= le16_to_cpu(*cnq
->hw_cons_ptr
);
454 sw_comp_cons
= qed_chain_get_cons_idx(&cnq
->pbl
);
456 /* Align protocol-index and chain reads */
459 while (sw_comp_cons
!= hw_comp_cons
) {
460 cq_handle
= (struct regpair
*)qed_chain_consume(&cnq
->pbl
);
461 cq
= (struct qedr_cq
*)(uintptr_t)HILO_U64(cq_handle
->hi
,
466 "Received NULL CQ cq_handle->hi=%d cq_handle->lo=%d sw_comp_cons=%d hw_comp_cons=%d\n",
467 cq_handle
->hi
, cq_handle
->lo
, sw_comp_cons
,
473 if (cq
->sig
!= QEDR_CQ_MAGIC_NUMBER
) {
475 "Problem with cq signature, cq_handle->hi=%d ch_handle->lo=%d cq=%p\n",
476 cq_handle
->hi
, cq_handle
->lo
, cq
);
482 if (!cq
->destroyed
&& cq
->ibcq
.comp_handler
)
483 (*cq
->ibcq
.comp_handler
)
484 (&cq
->ibcq
, cq
->ibcq
.cq_context
);
486 /* The CQ's CNQ notification counter is checked before
487 * destroying the CQ in a busy-wait loop that waits for all of
488 * the CQ's CNQ interrupts to be processed. It is increased
489 * here, only after the completion handler, to ensure that the
490 * the handler is not running when the CQ is destroyed.
494 sw_comp_cons
= qed_chain_get_cons_idx(&cnq
->pbl
);
499 qed_ops
->rdma_cnq_prod_update(cnq
->dev
->rdma_ctx
, cnq
->index
,
502 qed_sb_ack(cnq
->sb
, IGU_INT_ENABLE
, 1);
507 static void qedr_sync_free_irqs(struct qedr_dev
*dev
)
513 for (i
= 0; i
< dev
->int_info
.used_cnt
; i
++) {
514 if (dev
->int_info
.msix_cnt
) {
515 idx
= i
* dev
->num_hwfns
+ dev
->affin_hwfn_idx
;
516 vector
= dev
->int_info
.msix
[idx
].vector
;
517 synchronize_irq(vector
);
518 free_irq(vector
, &dev
->cnq_array
[i
]);
522 dev
->int_info
.used_cnt
= 0;
525 static int qedr_req_msix_irqs(struct qedr_dev
*dev
)
530 if (dev
->num_cnq
> dev
->int_info
.msix_cnt
) {
532 "Interrupt mismatch: %d CNQ queues > %d MSI-x vectors\n",
533 dev
->num_cnq
, dev
->int_info
.msix_cnt
);
537 for (i
= 0; i
< dev
->num_cnq
; i
++) {
538 idx
= i
* dev
->num_hwfns
+ dev
->affin_hwfn_idx
;
539 rc
= request_irq(dev
->int_info
.msix
[idx
].vector
,
540 qedr_irq_handler
, 0, dev
->cnq_array
[i
].name
,
543 DP_ERR(dev
, "Request cnq %d irq failed\n", i
);
544 qedr_sync_free_irqs(dev
);
546 DP_DEBUG(dev
, QEDR_MSG_INIT
,
547 "Requested cnq irq for %s [entry %d]. Cookie is at %p\n",
548 dev
->cnq_array
[i
].name
, i
,
550 dev
->int_info
.used_cnt
++;
557 static int qedr_setup_irqs(struct qedr_dev
*dev
)
561 DP_DEBUG(dev
, QEDR_MSG_INIT
, "qedr_setup_irqs\n");
563 /* Learn Interrupt configuration */
564 rc
= dev
->ops
->rdma_set_rdma_int(dev
->cdev
, dev
->num_cnq
);
568 rc
= dev
->ops
->rdma_get_rdma_int(dev
->cdev
, &dev
->int_info
);
570 DP_DEBUG(dev
, QEDR_MSG_INIT
, "get_rdma_int failed\n");
574 if (dev
->int_info
.msix_cnt
) {
575 DP_DEBUG(dev
, QEDR_MSG_INIT
, "rdma msix_cnt = %d\n",
576 dev
->int_info
.msix_cnt
);
577 rc
= qedr_req_msix_irqs(dev
);
582 DP_DEBUG(dev
, QEDR_MSG_INIT
, "qedr_setup_irqs succeeded\n");
587 static int qedr_set_device_attr(struct qedr_dev
*dev
)
589 struct qed_rdma_device
*qed_attr
;
590 struct qedr_device_attr
*attr
;
593 /* Part 1 - query core capabilities */
594 qed_attr
= dev
->ops
->rdma_query_device(dev
->rdma_ctx
);
596 /* Part 2 - check capabilities */
597 page_size
= ~dev
->attr
.page_size_caps
+ 1;
598 if (page_size
> PAGE_SIZE
) {
600 "Kernel PAGE_SIZE is %ld which is smaller than minimum page size (%d) required by qedr\n",
601 PAGE_SIZE
, page_size
);
605 /* Part 3 - copy and update capabilities */
607 attr
->vendor_id
= qed_attr
->vendor_id
;
608 attr
->vendor_part_id
= qed_attr
->vendor_part_id
;
609 attr
->hw_ver
= qed_attr
->hw_ver
;
610 attr
->fw_ver
= qed_attr
->fw_ver
;
611 attr
->node_guid
= qed_attr
->node_guid
;
612 attr
->sys_image_guid
= qed_attr
->sys_image_guid
;
613 attr
->max_cnq
= qed_attr
->max_cnq
;
614 attr
->max_sge
= qed_attr
->max_sge
;
615 attr
->max_inline
= qed_attr
->max_inline
;
616 attr
->max_sqe
= min_t(u32
, qed_attr
->max_wqe
, QEDR_MAX_SQE
);
617 attr
->max_rqe
= min_t(u32
, qed_attr
->max_wqe
, QEDR_MAX_RQE
);
618 attr
->max_qp_resp_rd_atomic_resc
= qed_attr
->max_qp_resp_rd_atomic_resc
;
619 attr
->max_qp_req_rd_atomic_resc
= qed_attr
->max_qp_req_rd_atomic_resc
;
620 attr
->max_dev_resp_rd_atomic_resc
=
621 qed_attr
->max_dev_resp_rd_atomic_resc
;
622 attr
->max_cq
= qed_attr
->max_cq
;
623 attr
->max_qp
= qed_attr
->max_qp
;
624 attr
->max_mr
= qed_attr
->max_mr
;
625 attr
->max_mr_size
= qed_attr
->max_mr_size
;
626 attr
->max_cqe
= min_t(u64
, qed_attr
->max_cqe
, QEDR_MAX_CQES
);
627 attr
->max_mw
= qed_attr
->max_mw
;
628 attr
->max_fmr
= qed_attr
->max_fmr
;
629 attr
->max_mr_mw_fmr_pbl
= qed_attr
->max_mr_mw_fmr_pbl
;
630 attr
->max_mr_mw_fmr_size
= qed_attr
->max_mr_mw_fmr_size
;
631 attr
->max_pd
= qed_attr
->max_pd
;
632 attr
->max_ah
= qed_attr
->max_ah
;
633 attr
->max_pkey
= qed_attr
->max_pkey
;
634 attr
->max_srq
= qed_attr
->max_srq
;
635 attr
->max_srq_wr
= qed_attr
->max_srq_wr
;
636 attr
->dev_caps
= qed_attr
->dev_caps
;
637 attr
->page_size_caps
= qed_attr
->page_size_caps
;
638 attr
->dev_ack_delay
= qed_attr
->dev_ack_delay
;
639 attr
->reserved_lkey
= qed_attr
->reserved_lkey
;
640 attr
->bad_pkey_counter
= qed_attr
->bad_pkey_counter
;
641 attr
->max_stats_queues
= qed_attr
->max_stats_queues
;
646 static void qedr_unaffiliated_event(void *context
, u8 event_code
)
648 pr_err("unaffiliated event not implemented yet\n");
651 static void qedr_affiliated_event(void *context
, u8 e_code
, void *fw_handle
)
653 #define EVENT_TYPE_NOT_DEFINED 0
654 #define EVENT_TYPE_CQ 1
655 #define EVENT_TYPE_QP 2
656 #define EVENT_TYPE_SRQ 3
657 struct qedr_dev
*dev
= (struct qedr_dev
*)context
;
658 struct regpair
*async_handle
= (struct regpair
*)fw_handle
;
659 u64 roce_handle64
= ((u64
) async_handle
->hi
<< 32) + async_handle
->lo
;
660 u8 event_type
= EVENT_TYPE_NOT_DEFINED
;
661 struct ib_event event
;
662 struct ib_srq
*ibsrq
;
663 struct qedr_srq
*srq
;
673 case ROCE_ASYNC_EVENT_CQ_OVERFLOW_ERR
:
674 event
.event
= IB_EVENT_CQ_ERR
;
675 event_type
= EVENT_TYPE_CQ
;
677 case ROCE_ASYNC_EVENT_SQ_DRAINED
:
678 event
.event
= IB_EVENT_SQ_DRAINED
;
679 event_type
= EVENT_TYPE_QP
;
681 case ROCE_ASYNC_EVENT_QP_CATASTROPHIC_ERR
:
682 event
.event
= IB_EVENT_QP_FATAL
;
683 event_type
= EVENT_TYPE_QP
;
685 case ROCE_ASYNC_EVENT_LOCAL_INVALID_REQUEST_ERR
:
686 event
.event
= IB_EVENT_QP_REQ_ERR
;
687 event_type
= EVENT_TYPE_QP
;
689 case ROCE_ASYNC_EVENT_LOCAL_ACCESS_ERR
:
690 event
.event
= IB_EVENT_QP_ACCESS_ERR
;
691 event_type
= EVENT_TYPE_QP
;
693 case ROCE_ASYNC_EVENT_SRQ_LIMIT
:
694 event
.event
= IB_EVENT_SRQ_LIMIT_REACHED
;
695 event_type
= EVENT_TYPE_SRQ
;
697 case ROCE_ASYNC_EVENT_SRQ_EMPTY
:
698 event
.event
= IB_EVENT_SRQ_ERR
;
699 event_type
= EVENT_TYPE_SRQ
;
702 DP_ERR(dev
, "unsupported event %d on handle=%llx\n",
703 e_code
, roce_handle64
);
707 case QED_IWARP_EVENT_SRQ_LIMIT
:
708 event
.event
= IB_EVENT_SRQ_LIMIT_REACHED
;
709 event_type
= EVENT_TYPE_SRQ
;
711 case QED_IWARP_EVENT_SRQ_EMPTY
:
712 event
.event
= IB_EVENT_SRQ_ERR
;
713 event_type
= EVENT_TYPE_SRQ
;
716 DP_ERR(dev
, "unsupported event %d on handle=%llx\n", e_code
,
720 switch (event_type
) {
722 cq
= (struct qedr_cq
*)(uintptr_t)roce_handle64
;
725 if (ibcq
->event_handler
) {
726 event
.device
= ibcq
->device
;
727 event
.element
.cq
= ibcq
;
728 ibcq
->event_handler(&event
, ibcq
->cq_context
);
732 "Error: CQ event with NULL pointer ibcq. Handle=%llx\n",
735 DP_ERR(dev
, "CQ event %d on handle %p\n", e_code
, cq
);
738 qp
= (struct qedr_qp
*)(uintptr_t)roce_handle64
;
741 if (ibqp
->event_handler
) {
742 event
.device
= ibqp
->device
;
743 event
.element
.qp
= ibqp
;
744 ibqp
->event_handler(&event
, ibqp
->qp_context
);
748 "Error: QP event with NULL pointer ibqp. Handle=%llx\n",
751 DP_ERR(dev
, "QP event %d on handle %p\n", e_code
, qp
);
754 srq_id
= (u16
)roce_handle64
;
755 xa_lock_irqsave(&dev
->srqs
, flags
);
756 srq
= xa_load(&dev
->srqs
, srq_id
);
759 if (ibsrq
->event_handler
) {
760 event
.device
= ibsrq
->device
;
761 event
.element
.srq
= ibsrq
;
762 ibsrq
->event_handler(&event
,
767 "SRQ event with NULL pointer ibsrq. Handle=%llx\n",
770 xa_unlock_irqrestore(&dev
->srqs
, flags
);
771 DP_NOTICE(dev
, "SRQ event %d on handle %p\n", e_code
, srq
);
777 static int qedr_init_hw(struct qedr_dev
*dev
)
779 struct qed_rdma_add_user_out_params out_params
;
780 struct qed_rdma_start_in_params
*in_params
;
781 struct qed_rdma_cnq_params
*cur_pbl
;
782 struct qed_rdma_events events
;
783 dma_addr_t p_phys_table
;
788 in_params
= kzalloc(sizeof(*in_params
), GFP_KERNEL
);
794 in_params
->desired_cnq
= dev
->num_cnq
;
795 for (i
= 0; i
< dev
->num_cnq
; i
++) {
796 cur_pbl
= &in_params
->cnq_pbl_list
[i
];
798 page_cnt
= qed_chain_get_page_cnt(&dev
->cnq_array
[i
].pbl
);
799 cur_pbl
->num_pbl_pages
= page_cnt
;
801 p_phys_table
= qed_chain_get_pbl_phys(&dev
->cnq_array
[i
].pbl
);
802 cur_pbl
->pbl_ptr
= (u64
)p_phys_table
;
805 events
.affiliated_event
= qedr_affiliated_event
;
806 events
.unaffiliated_event
= qedr_unaffiliated_event
;
807 events
.context
= dev
;
809 in_params
->events
= &events
;
810 in_params
->cq_mode
= QED_RDMA_CQ_MODE_32_BITS
;
811 in_params
->max_mtu
= dev
->ndev
->mtu
;
812 dev
->iwarp_max_mtu
= dev
->ndev
->mtu
;
813 ether_addr_copy(&in_params
->mac_addr
[0], dev
->ndev
->dev_addr
);
815 rc
= dev
->ops
->rdma_init(dev
->cdev
, in_params
);
819 rc
= dev
->ops
->rdma_add_user(dev
->rdma_ctx
, &out_params
);
823 dev
->db_addr
= (void __iomem
*)(uintptr_t)out_params
.dpi_addr
;
824 dev
->db_phys_addr
= out_params
.dpi_phys_addr
;
825 dev
->db_size
= out_params
.dpi_size
;
826 dev
->dpi
= out_params
.dpi
;
828 rc
= qedr_set_device_attr(dev
);
832 DP_ERR(dev
, "Init HW Failed rc = %d\n", rc
);
837 static void qedr_stop_hw(struct qedr_dev
*dev
)
839 dev
->ops
->rdma_remove_user(dev
->rdma_ctx
, dev
->dpi
);
840 dev
->ops
->rdma_stop(dev
->rdma_ctx
);
843 static struct qedr_dev
*qedr_add(struct qed_dev
*cdev
, struct pci_dev
*pdev
,
844 struct net_device
*ndev
)
846 struct qed_dev_rdma_info dev_info
;
847 struct qedr_dev
*dev
;
850 dev
= ib_alloc_device(qedr_dev
, ibdev
);
852 pr_err("Unable to allocate ib device\n");
856 DP_DEBUG(dev
, QEDR_MSG_INIT
, "qedr add device called\n");
862 qed_ops
= qed_get_rdma_ops();
864 DP_ERR(dev
, "Failed to get qed roce operations\n");
869 rc
= qed_ops
->fill_dev_info(cdev
, &dev_info
);
873 dev
->user_dpm_enabled
= dev_info
.user_dpm_enabled
;
874 dev
->rdma_type
= dev_info
.rdma_type
;
875 dev
->num_hwfns
= dev_info
.common
.num_hwfns
;
877 if (IS_IWARP(dev
) && QEDR_IS_CMT(dev
)) {
878 rc
= dev
->ops
->iwarp_set_engine_affin(cdev
, false);
880 DP_ERR(dev
, "iWARP is disabled over a 100g device Enabling it may impact L2 performance. To enable it run devlink dev param set <dev> name iwarp_cmt value true cmode runtime\n");
884 dev
->affin_hwfn_idx
= dev
->ops
->common
->get_affin_hwfn_idx(cdev
);
886 dev
->rdma_ctx
= dev
->ops
->rdma_get_rdma_ctx(cdev
);
888 dev
->num_cnq
= dev
->ops
->rdma_get_min_cnq_msix(cdev
);
890 DP_ERR(dev
, "Failed. At least one CNQ is required.\n");
895 dev
->wq_multiplier
= QEDR_WQ_MULTIPLIER_DFT
;
897 qedr_pci_set_atomic(dev
, pdev
);
899 rc
= qedr_alloc_resources(dev
);
903 rc
= qedr_init_hw(dev
);
907 rc
= qedr_setup_irqs(dev
);
911 rc
= qedr_register_device(dev
);
913 DP_ERR(dev
, "Unable to allocate register device\n");
917 if (!test_and_set_bit(QEDR_ENET_STATE_BIT
, &dev
->enet_state
))
918 qedr_ib_dispatch_event(dev
, QEDR_PORT
, IB_EVENT_PORT_ACTIVE
);
920 DP_DEBUG(dev
, QEDR_MSG_INIT
, "qedr driver loaded successfully\n");
924 qedr_sync_free_irqs(dev
);
928 qedr_free_resources(dev
);
930 ib_dealloc_device(&dev
->ibdev
);
931 DP_ERR(dev
, "qedr driver load failed rc=%d\n", rc
);
936 static void qedr_remove(struct qedr_dev
*dev
)
938 /* First unregister with stack to stop all the active traffic
939 * of the registered clients.
941 ib_unregister_device(&dev
->ibdev
);
944 qedr_sync_free_irqs(dev
);
945 qedr_free_resources(dev
);
947 if (IS_IWARP(dev
) && QEDR_IS_CMT(dev
))
948 dev
->ops
->iwarp_set_engine_affin(dev
->cdev
, true);
950 ib_dealloc_device(&dev
->ibdev
);
953 static void qedr_close(struct qedr_dev
*dev
)
955 if (test_and_clear_bit(QEDR_ENET_STATE_BIT
, &dev
->enet_state
))
956 qedr_ib_dispatch_event(dev
, QEDR_PORT
, IB_EVENT_PORT_ERR
);
959 static void qedr_shutdown(struct qedr_dev
*dev
)
965 static void qedr_open(struct qedr_dev
*dev
)
967 if (!test_and_set_bit(QEDR_ENET_STATE_BIT
, &dev
->enet_state
))
968 qedr_ib_dispatch_event(dev
, QEDR_PORT
, IB_EVENT_PORT_ACTIVE
);
971 static void qedr_mac_address_change(struct qedr_dev
*dev
)
973 union ib_gid
*sgid
= &dev
->sgid_tbl
[0];
974 u8 guid
[8], mac_addr
[6];
978 ether_addr_copy(&mac_addr
[0], dev
->ndev
->dev_addr
);
979 guid
[0] = mac_addr
[0] ^ 2;
980 guid
[1] = mac_addr
[1];
981 guid
[2] = mac_addr
[2];
984 guid
[5] = mac_addr
[3];
985 guid
[6] = mac_addr
[4];
986 guid
[7] = mac_addr
[5];
987 sgid
->global
.subnet_prefix
= cpu_to_be64(0xfe80000000000000LL
);
988 memcpy(&sgid
->raw
[8], guid
, sizeof(guid
));
991 rc
= dev
->ops
->ll2_set_mac_filter(dev
->cdev
,
992 dev
->gsi_ll2_mac_address
,
993 dev
->ndev
->dev_addr
);
995 ether_addr_copy(dev
->gsi_ll2_mac_address
, dev
->ndev
->dev_addr
);
997 qedr_ib_dispatch_event(dev
, QEDR_PORT
, IB_EVENT_GID_CHANGE
);
1000 DP_ERR(dev
, "Error updating mac filter\n");
1003 /* event handling via NIC driver ensures that all the NIC specific
1004 * initialization done before RoCE driver notifies
1007 static void qedr_notify(struct qedr_dev
*dev
, enum qede_rdma_event event
)
1019 case QEDE_CHANGE_ADDR
:
1020 qedr_mac_address_change(dev
);
1023 pr_err("Event not supported\n");
1027 static struct qedr_driver qedr_drv
= {
1028 .name
= "qedr_driver",
1030 .remove
= qedr_remove
,
1031 .notify
= qedr_notify
,
1034 static int __init
qedr_init_module(void)
1036 return qede_rdma_register_driver(&qedr_drv
);
1039 static void __exit
qedr_exit_module(void)
1041 qede_rdma_unregister_driver(&qedr_drv
);
1044 module_init(qedr_init_module
);
1045 module_exit(qedr_exit_module
);