2 * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved.
4 * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/kernel.h>
35 #include <linux/module.h>
36 #include <linux/slab.h>
37 #include <linux/delay.h>
39 #include "iscsi_iser.h"
41 #define ISCSI_ISER_MAX_CONN 8
42 #define ISER_MAX_RX_LEN (ISER_QP_MAX_RECV_DTOS * ISCSI_ISER_MAX_CONN)
43 #define ISER_MAX_TX_LEN (ISER_QP_MAX_REQ_DTOS * ISCSI_ISER_MAX_CONN)
44 #define ISER_MAX_CQ_LEN (ISER_MAX_RX_LEN + ISER_MAX_TX_LEN + \
47 static void iser_qp_event_callback(struct ib_event
*cause
, void *context
)
49 iser_err("qp event %s (%d)\n",
50 ib_event_msg(cause
->event
), cause
->event
);
53 static void iser_event_handler(struct ib_event_handler
*handler
,
54 struct ib_event
*event
)
56 iser_err("async event %s (%d) on device %s port %d\n",
57 ib_event_msg(event
->event
), event
->event
,
58 event
->device
->name
, event
->element
.port_num
);
62 * iser_create_device_ib_res - creates Protection Domain (PD), Completion
63 * Queue (CQ), DMA Memory Region (DMA MR) with the device associated with
66 * returns 0 on success, -1 on failure
68 static int iser_create_device_ib_res(struct iser_device
*device
)
70 struct ib_device
*ib_dev
= device
->ib_device
;
73 ret
= iser_assign_reg_ops(device
);
77 device
->comps_used
= min_t(int, num_online_cpus(),
78 ib_dev
->num_comp_vectors
);
80 device
->comps
= kcalloc(device
->comps_used
, sizeof(*device
->comps
),
85 max_cqe
= min(ISER_MAX_CQ_LEN
, ib_dev
->attrs
.max_cqe
);
87 iser_info("using %d CQs, device %s supports %d vectors max_cqe %d\n",
88 device
->comps_used
, ib_dev
->name
,
89 ib_dev
->num_comp_vectors
, max_cqe
);
91 device
->pd
= ib_alloc_pd(ib_dev
,
92 iser_always_reg
? 0 : IB_PD_UNSAFE_GLOBAL_RKEY
);
93 if (IS_ERR(device
->pd
))
96 for (i
= 0; i
< device
->comps_used
; i
++) {
97 struct iser_comp
*comp
= &device
->comps
[i
];
99 comp
->cq
= ib_alloc_cq(ib_dev
, comp
, max_cqe
, i
,
101 if (IS_ERR(comp
->cq
)) {
107 INIT_IB_EVENT_HANDLER(&device
->event_handler
, ib_dev
,
109 if (ib_register_event_handler(&device
->event_handler
))
115 for (i
= 0; i
< device
->comps_used
; i
++) {
116 struct iser_comp
*comp
= &device
->comps
[i
];
119 ib_free_cq(comp
->cq
);
121 ib_dealloc_pd(device
->pd
);
123 kfree(device
->comps
);
125 iser_err("failed to allocate an IB resource\n");
130 * iser_free_device_ib_res - destroy/dealloc/dereg the DMA MR,
131 * CQ and PD created with the device associated with the adapator.
133 static void iser_free_device_ib_res(struct iser_device
*device
)
137 for (i
= 0; i
< device
->comps_used
; i
++) {
138 struct iser_comp
*comp
= &device
->comps
[i
];
140 ib_free_cq(comp
->cq
);
144 (void)ib_unregister_event_handler(&device
->event_handler
);
145 ib_dealloc_pd(device
->pd
);
147 kfree(device
->comps
);
148 device
->comps
= NULL
;
153 * iser_alloc_fmr_pool - Creates FMR pool and page_vector
155 * returns 0 on success, or errno code on failure
157 int iser_alloc_fmr_pool(struct ib_conn
*ib_conn
,
161 struct iser_device
*device
= ib_conn
->device
;
162 struct iser_fr_pool
*fr_pool
= &ib_conn
->fr_pool
;
163 struct iser_page_vec
*page_vec
;
164 struct iser_fr_desc
*desc
;
165 struct ib_fmr_pool
*fmr_pool
;
166 struct ib_fmr_pool_param params
;
169 INIT_LIST_HEAD(&fr_pool
->list
);
170 spin_lock_init(&fr_pool
->lock
);
172 desc
= kzalloc(sizeof(*desc
), GFP_KERNEL
);
176 page_vec
= kmalloc(sizeof(*page_vec
) + (sizeof(u64
) * size
),
183 page_vec
->pages
= (u64
*)(page_vec
+ 1);
185 params
.page_shift
= SHIFT_4K
;
186 params
.max_pages_per_fmr
= size
;
187 /* make the pool size twice the max number of SCSI commands *
188 * the ML is expected to queue, watermark for unmap at 50% */
189 params
.pool_size
= cmds_max
* 2;
190 params
.dirty_watermark
= cmds_max
;
192 params
.flush_function
= NULL
;
193 params
.access
= (IB_ACCESS_LOCAL_WRITE
|
194 IB_ACCESS_REMOTE_WRITE
|
195 IB_ACCESS_REMOTE_READ
);
197 fmr_pool
= ib_create_fmr_pool(device
->pd
, ¶ms
);
198 if (IS_ERR(fmr_pool
)) {
199 ret
= PTR_ERR(fmr_pool
);
200 iser_err("FMR allocation failed, err %d\n", ret
);
204 desc
->rsc
.page_vec
= page_vec
;
205 desc
->rsc
.fmr_pool
= fmr_pool
;
206 list_add(&desc
->list
, &fr_pool
->list
);
219 * iser_free_fmr_pool - releases the FMR pool and page vec
221 void iser_free_fmr_pool(struct ib_conn
*ib_conn
)
223 struct iser_fr_pool
*fr_pool
= &ib_conn
->fr_pool
;
224 struct iser_fr_desc
*desc
;
226 desc
= list_first_entry(&fr_pool
->list
,
227 struct iser_fr_desc
, list
);
228 list_del(&desc
->list
);
230 iser_info("freeing conn %p fmr pool %p\n",
231 ib_conn
, desc
->rsc
.fmr_pool
);
233 ib_destroy_fmr_pool(desc
->rsc
.fmr_pool
);
234 kfree(desc
->rsc
.page_vec
);
239 iser_alloc_reg_res(struct iser_device
*device
,
241 struct iser_reg_resources
*res
,
244 struct ib_device
*ib_dev
= device
->ib_device
;
245 enum ib_mr_type mr_type
;
248 if (ib_dev
->attrs
.device_cap_flags
& IB_DEVICE_SG_GAPS_REG
)
249 mr_type
= IB_MR_TYPE_SG_GAPS
;
251 mr_type
= IB_MR_TYPE_MEM_REG
;
253 res
->mr
= ib_alloc_mr(pd
, mr_type
, size
);
254 if (IS_ERR(res
->mr
)) {
255 ret
= PTR_ERR(res
->mr
);
256 iser_err("Failed to allocate ib_fast_reg_mr err=%d\n", ret
);
265 iser_free_reg_res(struct iser_reg_resources
*rsc
)
267 ib_dereg_mr(rsc
->mr
);
271 iser_alloc_pi_ctx(struct iser_device
*device
,
273 struct iser_fr_desc
*desc
,
276 struct iser_pi_context
*pi_ctx
= NULL
;
279 desc
->pi_ctx
= kzalloc(sizeof(*desc
->pi_ctx
), GFP_KERNEL
);
283 pi_ctx
= desc
->pi_ctx
;
285 ret
= iser_alloc_reg_res(device
, pd
, &pi_ctx
->rsc
, size
);
287 iser_err("failed to allocate reg_resources\n");
288 goto alloc_reg_res_err
;
291 pi_ctx
->sig_mr
= ib_alloc_mr(pd
, IB_MR_TYPE_SIGNATURE
, 2);
292 if (IS_ERR(pi_ctx
->sig_mr
)) {
293 ret
= PTR_ERR(pi_ctx
->sig_mr
);
296 pi_ctx
->sig_mr_valid
= 0;
297 desc
->pi_ctx
->sig_protected
= 0;
302 iser_free_reg_res(&pi_ctx
->rsc
);
310 iser_free_pi_ctx(struct iser_pi_context
*pi_ctx
)
312 iser_free_reg_res(&pi_ctx
->rsc
);
313 ib_dereg_mr(pi_ctx
->sig_mr
);
317 static struct iser_fr_desc
*
318 iser_create_fastreg_desc(struct iser_device
*device
,
323 struct iser_fr_desc
*desc
;
326 desc
= kzalloc(sizeof(*desc
), GFP_KERNEL
);
328 return ERR_PTR(-ENOMEM
);
330 ret
= iser_alloc_reg_res(device
, pd
, &desc
->rsc
, size
);
332 goto reg_res_alloc_failure
;
335 ret
= iser_alloc_pi_ctx(device
, pd
, desc
, size
);
337 goto pi_ctx_alloc_failure
;
342 pi_ctx_alloc_failure
:
343 iser_free_reg_res(&desc
->rsc
);
344 reg_res_alloc_failure
:
351 * iser_alloc_fastreg_pool - Creates pool of fast_reg descriptors
352 * for fast registration work requests.
353 * returns 0 on success, or errno code on failure
355 int iser_alloc_fastreg_pool(struct ib_conn
*ib_conn
,
359 struct iser_device
*device
= ib_conn
->device
;
360 struct iser_fr_pool
*fr_pool
= &ib_conn
->fr_pool
;
361 struct iser_fr_desc
*desc
;
364 INIT_LIST_HEAD(&fr_pool
->list
);
365 INIT_LIST_HEAD(&fr_pool
->all_list
);
366 spin_lock_init(&fr_pool
->lock
);
368 for (i
= 0; i
< cmds_max
; i
++) {
369 desc
= iser_create_fastreg_desc(device
, device
->pd
,
370 ib_conn
->pi_support
, size
);
376 list_add_tail(&desc
->list
, &fr_pool
->list
);
377 list_add_tail(&desc
->all_list
, &fr_pool
->all_list
);
384 iser_free_fastreg_pool(ib_conn
);
389 * iser_free_fastreg_pool - releases the pool of fast_reg descriptors
391 void iser_free_fastreg_pool(struct ib_conn
*ib_conn
)
393 struct iser_fr_pool
*fr_pool
= &ib_conn
->fr_pool
;
394 struct iser_fr_desc
*desc
, *tmp
;
397 if (list_empty(&fr_pool
->all_list
))
400 iser_info("freeing conn %p fr pool\n", ib_conn
);
402 list_for_each_entry_safe(desc
, tmp
, &fr_pool
->all_list
, all_list
) {
403 list_del(&desc
->all_list
);
404 iser_free_reg_res(&desc
->rsc
);
406 iser_free_pi_ctx(desc
->pi_ctx
);
411 if (i
< fr_pool
->size
)
412 iser_warn("pool still has %d regions registered\n",
417 * iser_create_ib_conn_res - Queue-Pair (QP)
419 * returns 0 on success, -1 on failure
421 static int iser_create_ib_conn_res(struct ib_conn
*ib_conn
)
423 struct iser_conn
*iser_conn
= to_iser_conn(ib_conn
);
424 struct iser_device
*device
;
425 struct ib_device
*ib_dev
;
426 struct ib_qp_init_attr init_attr
;
428 int index
, min_index
= 0;
430 BUG_ON(ib_conn
->device
== NULL
);
432 device
= ib_conn
->device
;
433 ib_dev
= device
->ib_device
;
435 memset(&init_attr
, 0, sizeof init_attr
);
437 mutex_lock(&ig
.connlist_mutex
);
438 /* select the CQ with the minimal number of usages */
439 for (index
= 0; index
< device
->comps_used
; index
++) {
440 if (device
->comps
[index
].active_qps
<
441 device
->comps
[min_index
].active_qps
)
444 ib_conn
->comp
= &device
->comps
[min_index
];
445 ib_conn
->comp
->active_qps
++;
446 mutex_unlock(&ig
.connlist_mutex
);
447 iser_info("cq index %d used for ib_conn %p\n", min_index
, ib_conn
);
449 init_attr
.event_handler
= iser_qp_event_callback
;
450 init_attr
.qp_context
= (void *)ib_conn
;
451 init_attr
.send_cq
= ib_conn
->comp
->cq
;
452 init_attr
.recv_cq
= ib_conn
->comp
->cq
;
453 init_attr
.cap
.max_recv_wr
= ISER_QP_MAX_RECV_DTOS
;
454 init_attr
.cap
.max_send_sge
= 2;
455 init_attr
.cap
.max_recv_sge
= 1;
456 init_attr
.sq_sig_type
= IB_SIGNAL_REQ_WR
;
457 init_attr
.qp_type
= IB_QPT_RC
;
458 if (ib_conn
->pi_support
) {
459 init_attr
.cap
.max_send_wr
= ISER_QP_SIG_MAX_REQ_DTOS
+ 1;
460 init_attr
.create_flags
|= IB_QP_CREATE_SIGNATURE_EN
;
461 iser_conn
->max_cmds
=
462 ISER_GET_MAX_XMIT_CMDS(ISER_QP_SIG_MAX_REQ_DTOS
);
464 if (ib_dev
->attrs
.max_qp_wr
> ISER_QP_MAX_REQ_DTOS
) {
465 init_attr
.cap
.max_send_wr
= ISER_QP_MAX_REQ_DTOS
+ 1;
466 iser_conn
->max_cmds
=
467 ISER_GET_MAX_XMIT_CMDS(ISER_QP_MAX_REQ_DTOS
);
469 init_attr
.cap
.max_send_wr
= ib_dev
->attrs
.max_qp_wr
;
470 iser_conn
->max_cmds
=
471 ISER_GET_MAX_XMIT_CMDS(ib_dev
->attrs
.max_qp_wr
);
472 iser_dbg("device %s supports max_send_wr %d\n",
473 device
->ib_device
->name
, ib_dev
->attrs
.max_qp_wr
);
477 ret
= rdma_create_qp(ib_conn
->cma_id
, device
->pd
, &init_attr
);
481 ib_conn
->qp
= ib_conn
->cma_id
->qp
;
482 iser_info("setting conn %p cma_id %p qp %p\n",
483 ib_conn
, ib_conn
->cma_id
,
484 ib_conn
->cma_id
->qp
);
488 mutex_lock(&ig
.connlist_mutex
);
489 ib_conn
->comp
->active_qps
--;
490 mutex_unlock(&ig
.connlist_mutex
);
491 iser_err("unable to alloc mem or create resource, err %d\n", ret
);
497 * based on the resolved device node GUID see if there already allocated
498 * device for this device. If there's no such, create one.
501 struct iser_device
*iser_device_find_by_ib_device(struct rdma_cm_id
*cma_id
)
503 struct iser_device
*device
;
505 mutex_lock(&ig
.device_list_mutex
);
507 list_for_each_entry(device
, &ig
.device_list
, ig_list
)
508 /* find if there's a match using the node GUID */
509 if (device
->ib_device
->node_guid
== cma_id
->device
->node_guid
)
512 device
= kzalloc(sizeof *device
, GFP_KERNEL
);
516 /* assign this device to the device */
517 device
->ib_device
= cma_id
->device
;
518 /* init the device and link it into ig device list */
519 if (iser_create_device_ib_res(device
)) {
524 list_add(&device
->ig_list
, &ig
.device_list
);
529 mutex_unlock(&ig
.device_list_mutex
);
533 /* if there's no demand for this device, release it */
534 static void iser_device_try_release(struct iser_device
*device
)
536 mutex_lock(&ig
.device_list_mutex
);
538 iser_info("device %p refcount %d\n", device
, device
->refcount
);
539 if (!device
->refcount
) {
540 iser_free_device_ib_res(device
);
541 list_del(&device
->ig_list
);
544 mutex_unlock(&ig
.device_list_mutex
);
548 * Called with state mutex held
550 static int iser_conn_state_comp_exch(struct iser_conn
*iser_conn
,
551 enum iser_conn_state comp
,
552 enum iser_conn_state exch
)
556 ret
= (iser_conn
->state
== comp
);
558 iser_conn
->state
= exch
;
563 void iser_release_work(struct work_struct
*work
)
565 struct iser_conn
*iser_conn
;
567 iser_conn
= container_of(work
, struct iser_conn
, release_work
);
569 /* Wait for conn_stop to complete */
570 wait_for_completion(&iser_conn
->stop_completion
);
571 /* Wait for IB resouces cleanup to complete */
572 wait_for_completion(&iser_conn
->ib_completion
);
574 mutex_lock(&iser_conn
->state_mutex
);
575 iser_conn
->state
= ISER_CONN_DOWN
;
576 mutex_unlock(&iser_conn
->state_mutex
);
578 iser_conn_release(iser_conn
);
582 * iser_free_ib_conn_res - release IB related resources
583 * @iser_conn: iser connection struct
584 * @destroy: indicator if we need to try to release the
585 * iser device and memory regoins pool (only iscsi
586 * shutdown and DEVICE_REMOVAL will use this).
588 * This routine is called with the iser state mutex held
589 * so the cm_id removal is out of here. It is Safe to
590 * be invoked multiple times.
592 static void iser_free_ib_conn_res(struct iser_conn
*iser_conn
,
595 struct ib_conn
*ib_conn
= &iser_conn
->ib_conn
;
596 struct iser_device
*device
= ib_conn
->device
;
598 iser_info("freeing conn %p cma_id %p qp %p\n",
599 iser_conn
, ib_conn
->cma_id
, ib_conn
->qp
);
601 if (ib_conn
->qp
!= NULL
) {
602 mutex_lock(&ig
.connlist_mutex
);
603 ib_conn
->comp
->active_qps
--;
604 mutex_unlock(&ig
.connlist_mutex
);
605 rdma_destroy_qp(ib_conn
->cma_id
);
610 if (iser_conn
->rx_descs
)
611 iser_free_rx_descriptors(iser_conn
);
613 if (device
!= NULL
) {
614 iser_device_try_release(device
);
615 ib_conn
->device
= NULL
;
621 * Frees all conn objects and deallocs conn descriptor
623 void iser_conn_release(struct iser_conn
*iser_conn
)
625 struct ib_conn
*ib_conn
= &iser_conn
->ib_conn
;
627 mutex_lock(&ig
.connlist_mutex
);
628 list_del(&iser_conn
->conn_list
);
629 mutex_unlock(&ig
.connlist_mutex
);
631 mutex_lock(&iser_conn
->state_mutex
);
632 /* In case we endup here without ep_disconnect being invoked. */
633 if (iser_conn
->state
!= ISER_CONN_DOWN
) {
634 iser_warn("iser conn %p state %d, expected state down.\n",
635 iser_conn
, iser_conn
->state
);
636 iscsi_destroy_endpoint(iser_conn
->ep
);
637 iser_conn
->state
= ISER_CONN_DOWN
;
640 * In case we never got to bind stage, we still need to
641 * release IB resources (which is safe to call more than once).
643 iser_free_ib_conn_res(iser_conn
, true);
644 mutex_unlock(&iser_conn
->state_mutex
);
646 if (ib_conn
->cma_id
!= NULL
) {
647 rdma_destroy_id(ib_conn
->cma_id
);
648 ib_conn
->cma_id
= NULL
;
655 * triggers start of the disconnect procedures and wait for them to be done
656 * Called with state mutex held
658 int iser_conn_terminate(struct iser_conn
*iser_conn
)
660 struct ib_conn
*ib_conn
= &iser_conn
->ib_conn
;
663 /* terminate the iser conn only if the conn state is UP */
664 if (!iser_conn_state_comp_exch(iser_conn
, ISER_CONN_UP
,
665 ISER_CONN_TERMINATING
))
668 iser_info("iser_conn %p state %d\n", iser_conn
, iser_conn
->state
);
670 /* suspend queuing of new iscsi commands */
671 if (iser_conn
->iscsi_conn
)
672 iscsi_suspend_queue(iser_conn
->iscsi_conn
);
675 * In case we didn't already clean up the cma_id (peer initiated
676 * a disconnection), we need to Cause the CMA to change the QP
679 if (ib_conn
->cma_id
) {
680 err
= rdma_disconnect(ib_conn
->cma_id
);
682 iser_err("Failed to disconnect, conn: 0x%p err %d\n",
685 /* block until all flush errors are consumed */
686 ib_drain_sq(ib_conn
->qp
);
693 * Called with state mutex held
695 static void iser_connect_error(struct rdma_cm_id
*cma_id
)
697 struct iser_conn
*iser_conn
;
699 iser_conn
= (struct iser_conn
*)cma_id
->context
;
700 iser_conn
->state
= ISER_CONN_TERMINATING
;
704 iser_calc_scsi_params(struct iser_conn
*iser_conn
,
705 unsigned int max_sectors
)
707 struct iser_device
*device
= iser_conn
->ib_conn
.device
;
708 unsigned short sg_tablesize
, sup_sg_tablesize
;
710 sg_tablesize
= DIV_ROUND_UP(max_sectors
* 512, SIZE_4K
);
711 if (device
->ib_device
->attrs
.device_cap_flags
&
712 IB_DEVICE_MEM_MGT_EXTENSIONS
)
715 uint
, ISCSI_ISER_MAX_SG_TABLESIZE
,
716 device
->ib_device
->attrs
.max_fast_reg_page_list_len
);
718 sup_sg_tablesize
= ISCSI_ISER_MAX_SG_TABLESIZE
;
720 iser_conn
->scsi_sg_tablesize
= min(sg_tablesize
, sup_sg_tablesize
);
724 * Called with state mutex held
726 static void iser_addr_handler(struct rdma_cm_id
*cma_id
)
728 struct iser_device
*device
;
729 struct iser_conn
*iser_conn
;
730 struct ib_conn
*ib_conn
;
733 iser_conn
= (struct iser_conn
*)cma_id
->context
;
734 if (iser_conn
->state
!= ISER_CONN_PENDING
)
738 ib_conn
= &iser_conn
->ib_conn
;
739 device
= iser_device_find_by_ib_device(cma_id
);
741 iser_err("device lookup/creation failed\n");
742 iser_connect_error(cma_id
);
746 ib_conn
->device
= device
;
748 /* connection T10-PI support */
749 if (iser_pi_enable
) {
750 if (!(device
->ib_device
->attrs
.device_cap_flags
&
751 IB_DEVICE_SIGNATURE_HANDOVER
)) {
752 iser_warn("T10-PI requested but not supported on %s, "
753 "continue without T10-PI\n",
754 ib_conn
->device
->ib_device
->name
);
755 ib_conn
->pi_support
= false;
757 ib_conn
->pi_support
= true;
761 iser_calc_scsi_params(iser_conn
, iser_max_sectors
);
763 ret
= rdma_resolve_route(cma_id
, 1000);
765 iser_err("resolve route failed: %d\n", ret
);
766 iser_connect_error(cma_id
);
772 * Called with state mutex held
774 static void iser_route_handler(struct rdma_cm_id
*cma_id
)
776 struct rdma_conn_param conn_param
;
778 struct iser_cm_hdr req_hdr
;
779 struct iser_conn
*iser_conn
= (struct iser_conn
*)cma_id
->context
;
780 struct ib_conn
*ib_conn
= &iser_conn
->ib_conn
;
781 struct iser_device
*device
= ib_conn
->device
;
783 if (iser_conn
->state
!= ISER_CONN_PENDING
)
787 ret
= iser_create_ib_conn_res(ib_conn
);
791 memset(&conn_param
, 0, sizeof conn_param
);
792 conn_param
.responder_resources
= device
->ib_device
->attrs
.max_qp_rd_atom
;
793 conn_param
.initiator_depth
= 1;
794 conn_param
.retry_count
= 7;
795 conn_param
.rnr_retry_count
= 6;
797 memset(&req_hdr
, 0, sizeof(req_hdr
));
798 req_hdr
.flags
= ISER_ZBVA_NOT_SUP
;
799 if (!device
->remote_inv_sup
)
800 req_hdr
.flags
|= ISER_SEND_W_INV_NOT_SUP
;
801 conn_param
.private_data
= (void *)&req_hdr
;
802 conn_param
.private_data_len
= sizeof(struct iser_cm_hdr
);
804 ret
= rdma_connect(cma_id
, &conn_param
);
806 iser_err("failure connecting: %d\n", ret
);
812 iser_connect_error(cma_id
);
815 static void iser_connected_handler(struct rdma_cm_id
*cma_id
,
816 const void *private_data
)
818 struct iser_conn
*iser_conn
;
819 struct ib_qp_attr attr
;
820 struct ib_qp_init_attr init_attr
;
822 iser_conn
= (struct iser_conn
*)cma_id
->context
;
823 if (iser_conn
->state
!= ISER_CONN_PENDING
)
827 (void)ib_query_qp(cma_id
->qp
, &attr
, ~0, &init_attr
);
828 iser_info("remote qpn:%x my qpn:%x\n", attr
.dest_qp_num
, cma_id
->qp
->qp_num
);
831 u8 flags
= *(u8
*)private_data
;
833 iser_conn
->snd_w_inv
= !(flags
& ISER_SEND_W_INV_NOT_SUP
);
836 iser_info("conn %p: negotiated %s invalidation\n",
837 iser_conn
, iser_conn
->snd_w_inv
? "remote" : "local");
839 iser_conn
->state
= ISER_CONN_UP
;
840 complete(&iser_conn
->up_completion
);
843 static void iser_disconnected_handler(struct rdma_cm_id
*cma_id
)
845 struct iser_conn
*iser_conn
= (struct iser_conn
*)cma_id
->context
;
847 if (iser_conn_terminate(iser_conn
)) {
848 if (iser_conn
->iscsi_conn
)
849 iscsi_conn_failure(iser_conn
->iscsi_conn
,
850 ISCSI_ERR_CONN_FAILED
);
852 iser_err("iscsi_iser connection isn't bound\n");
856 static void iser_cleanup_handler(struct rdma_cm_id
*cma_id
,
859 struct iser_conn
*iser_conn
= (struct iser_conn
*)cma_id
->context
;
862 * We are not guaranteed that we visited disconnected_handler
863 * by now, call it here to be safe that we handle CM drep
866 iser_disconnected_handler(cma_id
);
867 iser_free_ib_conn_res(iser_conn
, destroy
);
868 complete(&iser_conn
->ib_completion
);
871 static int iser_cma_handler(struct rdma_cm_id
*cma_id
, struct rdma_cm_event
*event
)
873 struct iser_conn
*iser_conn
;
876 iser_conn
= (struct iser_conn
*)cma_id
->context
;
877 iser_info("%s (%d): status %d conn %p id %p\n",
878 rdma_event_msg(event
->event
), event
->event
,
879 event
->status
, cma_id
->context
, cma_id
);
881 mutex_lock(&iser_conn
->state_mutex
);
882 switch (event
->event
) {
883 case RDMA_CM_EVENT_ADDR_RESOLVED
:
884 iser_addr_handler(cma_id
);
886 case RDMA_CM_EVENT_ROUTE_RESOLVED
:
887 iser_route_handler(cma_id
);
889 case RDMA_CM_EVENT_ESTABLISHED
:
890 iser_connected_handler(cma_id
, event
->param
.conn
.private_data
);
892 case RDMA_CM_EVENT_REJECTED
:
893 iser_info("Connection rejected: %s\n",
894 rdma_reject_msg(cma_id
, event
->status
));
896 case RDMA_CM_EVENT_ADDR_ERROR
:
897 case RDMA_CM_EVENT_ROUTE_ERROR
:
898 case RDMA_CM_EVENT_CONNECT_ERROR
:
899 case RDMA_CM_EVENT_UNREACHABLE
:
900 iser_connect_error(cma_id
);
902 case RDMA_CM_EVENT_DISCONNECTED
:
903 case RDMA_CM_EVENT_ADDR_CHANGE
:
904 case RDMA_CM_EVENT_TIMEWAIT_EXIT
:
905 iser_cleanup_handler(cma_id
, false);
907 case RDMA_CM_EVENT_DEVICE_REMOVAL
:
909 * we *must* destroy the device as we cannot rely
910 * on iscsid to be around to initiate error handling.
911 * also if we are not in state DOWN implicitly destroy
914 iser_cleanup_handler(cma_id
, true);
915 if (iser_conn
->state
!= ISER_CONN_DOWN
) {
916 iser_conn
->ib_conn
.cma_id
= NULL
;
921 iser_err("Unexpected RDMA CM event: %s (%d)\n",
922 rdma_event_msg(event
->event
), event
->event
);
925 mutex_unlock(&iser_conn
->state_mutex
);
930 void iser_conn_init(struct iser_conn
*iser_conn
)
932 struct ib_conn
*ib_conn
= &iser_conn
->ib_conn
;
934 iser_conn
->state
= ISER_CONN_INIT
;
935 init_completion(&iser_conn
->stop_completion
);
936 init_completion(&iser_conn
->ib_completion
);
937 init_completion(&iser_conn
->up_completion
);
938 INIT_LIST_HEAD(&iser_conn
->conn_list
);
939 mutex_init(&iser_conn
->state_mutex
);
941 ib_conn
->post_recv_buf_count
= 0;
942 ib_conn
->reg_cqe
.done
= iser_reg_comp
;
946 * starts the process of connecting to the target
947 * sleeps until the connection is established or rejected
949 int iser_connect(struct iser_conn
*iser_conn
,
950 struct sockaddr
*src_addr
,
951 struct sockaddr
*dst_addr
,
954 struct ib_conn
*ib_conn
= &iser_conn
->ib_conn
;
957 mutex_lock(&iser_conn
->state_mutex
);
959 sprintf(iser_conn
->name
, "%pISp", dst_addr
);
961 iser_info("connecting to: %s\n", iser_conn
->name
);
963 /* the device is known only --after-- address resolution */
964 ib_conn
->device
= NULL
;
966 iser_conn
->state
= ISER_CONN_PENDING
;
968 ib_conn
->cma_id
= rdma_create_id(&init_net
, iser_cma_handler
,
970 RDMA_PS_TCP
, IB_QPT_RC
);
971 if (IS_ERR(ib_conn
->cma_id
)) {
972 err
= PTR_ERR(ib_conn
->cma_id
);
973 iser_err("rdma_create_id failed: %d\n", err
);
977 err
= rdma_resolve_addr(ib_conn
->cma_id
, src_addr
, dst_addr
, 1000);
979 iser_err("rdma_resolve_addr failed: %d\n", err
);
984 wait_for_completion_interruptible(&iser_conn
->up_completion
);
986 if (iser_conn
->state
!= ISER_CONN_UP
) {
988 goto connect_failure
;
991 mutex_unlock(&iser_conn
->state_mutex
);
993 mutex_lock(&ig
.connlist_mutex
);
994 list_add(&iser_conn
->conn_list
, &ig
.connlist
);
995 mutex_unlock(&ig
.connlist_mutex
);
999 ib_conn
->cma_id
= NULL
;
1001 iser_conn
->state
= ISER_CONN_DOWN
;
1003 mutex_unlock(&iser_conn
->state_mutex
);
1004 iser_conn_release(iser_conn
);
1008 int iser_post_recvl(struct iser_conn
*iser_conn
)
1010 struct ib_conn
*ib_conn
= &iser_conn
->ib_conn
;
1011 struct iser_login_desc
*desc
= &iser_conn
->login_desc
;
1012 struct ib_recv_wr wr
, *wr_failed
;
1015 desc
->sge
.addr
= desc
->rsp_dma
;
1016 desc
->sge
.length
= ISER_RX_LOGIN_SIZE
;
1017 desc
->sge
.lkey
= ib_conn
->device
->pd
->local_dma_lkey
;
1019 desc
->cqe
.done
= iser_login_rsp
;
1020 wr
.wr_cqe
= &desc
->cqe
;
1021 wr
.sg_list
= &desc
->sge
;
1025 ib_conn
->post_recv_buf_count
++;
1026 ib_ret
= ib_post_recv(ib_conn
->qp
, &wr
, &wr_failed
);
1028 iser_err("ib_post_recv failed ret=%d\n", ib_ret
);
1029 ib_conn
->post_recv_buf_count
--;
1035 int iser_post_recvm(struct iser_conn
*iser_conn
, int count
)
1037 struct ib_conn
*ib_conn
= &iser_conn
->ib_conn
;
1038 unsigned int my_rx_head
= iser_conn
->rx_desc_head
;
1039 struct iser_rx_desc
*rx_desc
;
1040 struct ib_recv_wr
*wr
, *wr_failed
;
1043 for (wr
= ib_conn
->rx_wr
, i
= 0; i
< count
; i
++, wr
++) {
1044 rx_desc
= &iser_conn
->rx_descs
[my_rx_head
];
1045 rx_desc
->cqe
.done
= iser_task_rsp
;
1046 wr
->wr_cqe
= &rx_desc
->cqe
;
1047 wr
->sg_list
= &rx_desc
->rx_sg
;
1050 my_rx_head
= (my_rx_head
+ 1) & iser_conn
->qp_max_recv_dtos_mask
;
1054 wr
->next
= NULL
; /* mark end of work requests list */
1056 ib_conn
->post_recv_buf_count
+= count
;
1057 ib_ret
= ib_post_recv(ib_conn
->qp
, ib_conn
->rx_wr
, &wr_failed
);
1059 iser_err("ib_post_recv failed ret=%d\n", ib_ret
);
1060 ib_conn
->post_recv_buf_count
-= count
;
1062 iser_conn
->rx_desc_head
= my_rx_head
;
1069 * iser_start_send - Initiate a Send DTO operation
1071 * returns 0 on success, -1 on failure
1073 int iser_post_send(struct ib_conn
*ib_conn
, struct iser_tx_desc
*tx_desc
,
1076 struct ib_send_wr
*bad_wr
, *wr
= iser_tx_next_wr(tx_desc
);
1079 ib_dma_sync_single_for_device(ib_conn
->device
->ib_device
,
1080 tx_desc
->dma_addr
, ISER_HEADERS_LEN
,
1084 wr
->wr_cqe
= &tx_desc
->cqe
;
1085 wr
->sg_list
= tx_desc
->tx_sg
;
1086 wr
->num_sge
= tx_desc
->num_sge
;
1087 wr
->opcode
= IB_WR_SEND
;
1088 wr
->send_flags
= signal
? IB_SEND_SIGNALED
: 0;
1090 ib_ret
= ib_post_send(ib_conn
->qp
, &tx_desc
->wrs
[0].send
, &bad_wr
);
1092 iser_err("ib_post_send failed, ret:%d opcode:%d\n",
1093 ib_ret
, bad_wr
->opcode
);
1098 u8
iser_check_task_pi_status(struct iscsi_iser_task
*iser_task
,
1099 enum iser_data_dir cmd_dir
, sector_t
*sector
)
1101 struct iser_mem_reg
*reg
= &iser_task
->rdma_reg
[cmd_dir
];
1102 struct iser_fr_desc
*desc
= reg
->mem_h
;
1103 unsigned long sector_size
= iser_task
->sc
->device
->sector_size
;
1104 struct ib_mr_status mr_status
;
1107 if (desc
&& desc
->pi_ctx
->sig_protected
) {
1108 desc
->pi_ctx
->sig_protected
= 0;
1109 ret
= ib_check_mr_status(desc
->pi_ctx
->sig_mr
,
1110 IB_MR_CHECK_SIG_STATUS
, &mr_status
);
1112 pr_err("ib_check_mr_status failed, ret %d\n", ret
);
1116 if (mr_status
.fail_status
& IB_MR_CHECK_SIG_STATUS
) {
1117 sector_t sector_off
= mr_status
.sig_err
.sig_err_offset
;
1119 sector_div(sector_off
, sector_size
+ 8);
1120 *sector
= scsi_get_lba(iser_task
->sc
) + sector_off
;
1122 pr_err("PI error found type %d at sector %llx "
1123 "expected %x vs actual %x\n",
1124 mr_status
.sig_err
.err_type
,
1125 (unsigned long long)*sector
,
1126 mr_status
.sig_err
.expected
,
1127 mr_status
.sig_err
.actual
);
1129 switch (mr_status
.sig_err
.err_type
) {
1130 case IB_SIG_BAD_GUARD
:
1132 case IB_SIG_BAD_REFTAG
:
1134 case IB_SIG_BAD_APPTAG
:
1142 /* Not alot we can do here, return ambiguous guard error */
1146 void iser_err_comp(struct ib_wc
*wc
, const char *type
)
1148 if (wc
->status
!= IB_WC_WR_FLUSH_ERR
) {
1149 struct iser_conn
*iser_conn
= to_iser_conn(wc
->qp
->qp_context
);
1151 iser_err("%s failure: %s (%d) vend_err %x\n", type
,
1152 ib_wc_status_msg(wc
->status
), wc
->status
,
1155 if (iser_conn
->iscsi_conn
)
1156 iscsi_conn_failure(iser_conn
->iscsi_conn
,
1157 ISCSI_ERR_CONN_FAILED
);
1159 iser_dbg("%s failure: %s (%d)\n", type
,
1160 ib_wc_status_msg(wc
->status
), wc
->status
);