1 /*******************************************************************************
2 * This file contains iSCSI extentions for RDMA (iSER) Verbs
4 * (c) Copyright 2013 Datera, Inc.
6 * Nicholas A. Bellinger <nab@linux-iscsi.org>
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 ****************************************************************************/
19 #include <linux/string.h>
20 #include <linux/module.h>
21 #include <linux/scatterlist.h>
22 #include <linux/socket.h>
24 #include <linux/in6.h>
25 #include <rdma/ib_verbs.h>
26 #include <rdma/rdma_cm.h>
27 #include <target/target_core_base.h>
28 #include <target/target_core_fabric.h>
29 #include <target/iscsi/iscsi_transport.h>
30 #include <linux/semaphore.h>
32 #include "isert_proto.h"
35 #define ISERT_MAX_CONN 8
36 #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN)
37 #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN)
38 #define ISER_MAX_CQ_LEN (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \
41 static int isert_debug_level
;
42 module_param_named(debug_level
, isert_debug_level
, int, 0644);
43 MODULE_PARM_DESC(debug_level
, "Enable debug tracing if > 0 (default:0)");
45 static DEFINE_MUTEX(device_list_mutex
);
46 static LIST_HEAD(device_list
);
47 static struct workqueue_struct
*isert_comp_wq
;
48 static struct workqueue_struct
*isert_release_wq
;
51 isert_unmap_cmd(struct isert_cmd
*isert_cmd
, struct isert_conn
*isert_conn
);
53 isert_map_rdma(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
,
54 struct isert_rdma_wr
*wr
);
56 isert_unreg_rdma(struct isert_cmd
*isert_cmd
, struct isert_conn
*isert_conn
);
58 isert_reg_rdma(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
,
59 struct isert_rdma_wr
*wr
);
61 isert_put_response(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
);
63 isert_rdma_post_recvl(struct isert_conn
*isert_conn
);
65 isert_rdma_accept(struct isert_conn
*isert_conn
);
66 struct rdma_cm_id
*isert_setup_id(struct isert_np
*isert_np
);
69 isert_prot_cmd(struct isert_conn
*conn
, struct se_cmd
*cmd
)
71 return (conn
->pi_support
&&
72 cmd
->prot_op
!= TARGET_PROT_NORMAL
);
77 isert_qp_event_callback(struct ib_event
*e
, void *context
)
79 struct isert_conn
*isert_conn
= context
;
81 isert_err("conn %p event: %d\n", isert_conn
, e
->event
);
83 case IB_EVENT_COMM_EST
:
84 rdma_notify(isert_conn
->conn_cm_id
, IB_EVENT_COMM_EST
);
86 case IB_EVENT_QP_LAST_WQE_REACHED
:
87 isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n");
95 isert_query_device(struct ib_device
*ib_dev
, struct ib_device_attr
*devattr
)
99 ret
= ib_query_device(ib_dev
, devattr
);
101 isert_err("ib_query_device() failed: %d\n", ret
);
104 isert_dbg("devattr->max_sge: %d\n", devattr
->max_sge
);
105 isert_dbg("devattr->max_sge_rd: %d\n", devattr
->max_sge_rd
);
110 static struct isert_comp
*
111 isert_comp_get(struct isert_conn
*isert_conn
)
113 struct isert_device
*device
= isert_conn
->conn_device
;
114 struct isert_comp
*comp
;
117 mutex_lock(&device_list_mutex
);
118 for (i
= 0; i
< device
->comps_used
; i
++)
119 if (device
->comps
[i
].active_qps
<
120 device
->comps
[min
].active_qps
)
122 comp
= &device
->comps
[min
];
124 mutex_unlock(&device_list_mutex
);
126 isert_info("conn %p, using comp %p min_index: %d\n",
127 isert_conn
, comp
, min
);
133 isert_comp_put(struct isert_comp
*comp
)
135 mutex_lock(&device_list_mutex
);
137 mutex_unlock(&device_list_mutex
);
140 static struct ib_qp
*
141 isert_create_qp(struct isert_conn
*isert_conn
,
142 struct isert_comp
*comp
,
143 struct rdma_cm_id
*cma_id
)
145 struct isert_device
*device
= isert_conn
->conn_device
;
146 struct ib_qp_init_attr attr
;
149 memset(&attr
, 0, sizeof(struct ib_qp_init_attr
));
150 attr
.event_handler
= isert_qp_event_callback
;
151 attr
.qp_context
= isert_conn
;
152 attr
.send_cq
= comp
->cq
;
153 attr
.recv_cq
= comp
->cq
;
154 attr
.cap
.max_send_wr
= ISERT_QP_MAX_REQ_DTOS
;
155 attr
.cap
.max_recv_wr
= ISERT_QP_MAX_RECV_DTOS
+ 1;
157 * FIXME: Use devattr.max_sge - 2 for max_send_sge as
158 * work-around for RDMA_READs with ConnectX-2.
160 * Also, still make sure to have at least two SGEs for
161 * outgoing control PDU responses.
163 attr
.cap
.max_send_sge
= max(2, device
->dev_attr
.max_sge
- 2);
164 isert_conn
->max_sge
= attr
.cap
.max_send_sge
;
166 attr
.cap
.max_recv_sge
= 1;
167 attr
.sq_sig_type
= IB_SIGNAL_REQ_WR
;
168 attr
.qp_type
= IB_QPT_RC
;
169 if (device
->pi_capable
)
170 attr
.create_flags
|= IB_QP_CREATE_SIGNATURE_EN
;
172 ret
= rdma_create_qp(cma_id
, device
->pd
, &attr
);
174 isert_err("rdma_create_qp failed for cma_id %d\n", ret
);
182 isert_conn_setup_qp(struct isert_conn
*isert_conn
, struct rdma_cm_id
*cma_id
)
184 struct isert_comp
*comp
;
187 comp
= isert_comp_get(isert_conn
);
188 isert_conn
->conn_qp
= isert_create_qp(isert_conn
, comp
, cma_id
);
189 if (IS_ERR(isert_conn
->conn_qp
)) {
190 ret
= PTR_ERR(isert_conn
->conn_qp
);
196 isert_comp_put(comp
);
201 isert_cq_event_callback(struct ib_event
*e
, void *context
)
203 isert_dbg("event: %d\n", e
->event
);
207 isert_alloc_rx_descriptors(struct isert_conn
*isert_conn
)
209 struct isert_device
*device
= isert_conn
->conn_device
;
210 struct ib_device
*ib_dev
= device
->ib_device
;
211 struct iser_rx_desc
*rx_desc
;
212 struct ib_sge
*rx_sg
;
216 isert_conn
->conn_rx_descs
= kzalloc(ISERT_QP_MAX_RECV_DTOS
*
217 sizeof(struct iser_rx_desc
), GFP_KERNEL
);
218 if (!isert_conn
->conn_rx_descs
)
221 rx_desc
= isert_conn
->conn_rx_descs
;
223 for (i
= 0; i
< ISERT_QP_MAX_RECV_DTOS
; i
++, rx_desc
++) {
224 dma_addr
= ib_dma_map_single(ib_dev
, (void *)rx_desc
,
225 ISER_RX_PAYLOAD_SIZE
, DMA_FROM_DEVICE
);
226 if (ib_dma_mapping_error(ib_dev
, dma_addr
))
229 rx_desc
->dma_addr
= dma_addr
;
231 rx_sg
= &rx_desc
->rx_sg
;
232 rx_sg
->addr
= rx_desc
->dma_addr
;
233 rx_sg
->length
= ISER_RX_PAYLOAD_SIZE
;
234 rx_sg
->lkey
= device
->mr
->lkey
;
237 isert_conn
->conn_rx_desc_head
= 0;
242 rx_desc
= isert_conn
->conn_rx_descs
;
243 for (j
= 0; j
< i
; j
++, rx_desc
++) {
244 ib_dma_unmap_single(ib_dev
, rx_desc
->dma_addr
,
245 ISER_RX_PAYLOAD_SIZE
, DMA_FROM_DEVICE
);
247 kfree(isert_conn
->conn_rx_descs
);
248 isert_conn
->conn_rx_descs
= NULL
;
250 isert_err("conn %p failed to allocate rx descriptors\n", isert_conn
);
256 isert_free_rx_descriptors(struct isert_conn
*isert_conn
)
258 struct ib_device
*ib_dev
= isert_conn
->conn_device
->ib_device
;
259 struct iser_rx_desc
*rx_desc
;
262 if (!isert_conn
->conn_rx_descs
)
265 rx_desc
= isert_conn
->conn_rx_descs
;
266 for (i
= 0; i
< ISERT_QP_MAX_RECV_DTOS
; i
++, rx_desc
++) {
267 ib_dma_unmap_single(ib_dev
, rx_desc
->dma_addr
,
268 ISER_RX_PAYLOAD_SIZE
, DMA_FROM_DEVICE
);
271 kfree(isert_conn
->conn_rx_descs
);
272 isert_conn
->conn_rx_descs
= NULL
;
275 static void isert_cq_work(struct work_struct
*);
276 static void isert_cq_callback(struct ib_cq
*, void *);
279 isert_create_device_ib_res(struct isert_device
*device
)
281 struct ib_device
*ib_dev
= device
->ib_device
;
282 struct ib_device_attr
*dev_attr
;
286 dev_attr
= &device
->dev_attr
;
287 ret
= isert_query_device(ib_dev
, dev_attr
);
291 max_cqe
= min(ISER_MAX_CQ_LEN
, dev_attr
->max_cqe
);
293 /* asign function handlers */
294 if (dev_attr
->device_cap_flags
& IB_DEVICE_MEM_MGT_EXTENSIONS
&&
295 dev_attr
->device_cap_flags
& IB_DEVICE_SIGNATURE_HANDOVER
) {
296 device
->use_fastreg
= 1;
297 device
->reg_rdma_mem
= isert_reg_rdma
;
298 device
->unreg_rdma_mem
= isert_unreg_rdma
;
300 device
->use_fastreg
= 0;
301 device
->reg_rdma_mem
= isert_map_rdma
;
302 device
->unreg_rdma_mem
= isert_unmap_cmd
;
305 /* Check signature cap */
306 device
->pi_capable
= dev_attr
->device_cap_flags
&
307 IB_DEVICE_SIGNATURE_HANDOVER
? true : false;
309 device
->comps_used
= min(ISERT_MAX_CQ
, min_t(int, num_online_cpus(),
310 device
->ib_device
->num_comp_vectors
));
311 isert_info("Using %d CQs, %s supports %d vectors support "
312 "Fast registration %d pi_capable %d\n",
313 device
->comps_used
, device
->ib_device
->name
,
314 device
->ib_device
->num_comp_vectors
, device
->use_fastreg
,
317 device
->comps
= kcalloc(device
->comps_used
, sizeof(struct isert_comp
),
319 if (!device
->comps
) {
320 isert_err("Unable to allocate completion contexts\n");
324 for (i
= 0; i
< device
->comps_used
; i
++) {
325 struct isert_comp
*comp
= &device
->comps
[i
];
327 comp
->device
= device
;
328 INIT_WORK(&comp
->work
, isert_cq_work
);
329 comp
->cq
= ib_create_cq(device
->ib_device
,
331 isert_cq_event_callback
,
334 if (IS_ERR(comp
->cq
)) {
335 ret
= PTR_ERR(comp
->cq
);
340 ret
= ib_req_notify_cq(comp
->cq
, IB_CQ_NEXT_COMP
);
345 device
->pd
= ib_alloc_pd(device
->ib_device
);
346 if (IS_ERR(device
->pd
)) {
347 ret
= PTR_ERR(device
->pd
);
348 isert_err("failed to allocate pd, device %p, ret=%d\n",
353 device
->mr
= ib_get_dma_mr(device
->pd
, IB_ACCESS_LOCAL_WRITE
);
354 if (IS_ERR(device
->mr
)) {
355 ret
= PTR_ERR(device
->mr
);
356 isert_err("failed to create dma mr, device %p, ret=%d\n",
365 ib_dealloc_pd(device
->pd
);
367 for (i
= 0; i
< device
->comps_used
; i
++) {
368 struct isert_comp
*comp
= &device
->comps
[i
];
371 cancel_work_sync(&comp
->work
);
372 ib_destroy_cq(comp
->cq
);
375 kfree(device
->comps
);
381 isert_free_device_ib_res(struct isert_device
*device
)
385 isert_info("device %p\n", device
);
387 ib_dereg_mr(device
->mr
);
388 ib_dealloc_pd(device
->pd
);
389 for (i
= 0; i
< device
->comps_used
; i
++) {
390 struct isert_comp
*comp
= &device
->comps
[i
];
392 cancel_work_sync(&comp
->work
);
393 ib_destroy_cq(comp
->cq
);
396 kfree(device
->comps
);
400 isert_device_try_release(struct isert_device
*device
)
402 mutex_lock(&device_list_mutex
);
404 isert_info("device %p refcount %d\n", device
, device
->refcount
);
405 if (!device
->refcount
) {
406 isert_free_device_ib_res(device
);
407 list_del(&device
->dev_node
);
410 mutex_unlock(&device_list_mutex
);
413 static struct isert_device
*
414 isert_device_find_by_ib_dev(struct rdma_cm_id
*cma_id
)
416 struct isert_device
*device
;
419 mutex_lock(&device_list_mutex
);
420 list_for_each_entry(device
, &device_list
, dev_node
) {
421 if (device
->ib_device
->node_guid
== cma_id
->device
->node_guid
) {
423 isert_info("Found iser device %p refcount %d\n",
424 device
, device
->refcount
);
425 mutex_unlock(&device_list_mutex
);
430 device
= kzalloc(sizeof(struct isert_device
), GFP_KERNEL
);
432 mutex_unlock(&device_list_mutex
);
433 return ERR_PTR(-ENOMEM
);
436 INIT_LIST_HEAD(&device
->dev_node
);
438 device
->ib_device
= cma_id
->device
;
439 ret
= isert_create_device_ib_res(device
);
442 mutex_unlock(&device_list_mutex
);
447 list_add_tail(&device
->dev_node
, &device_list
);
448 isert_info("Created a new iser device %p refcount %d\n",
449 device
, device
->refcount
);
450 mutex_unlock(&device_list_mutex
);
456 isert_conn_free_fastreg_pool(struct isert_conn
*isert_conn
)
458 struct fast_reg_descriptor
*fr_desc
, *tmp
;
461 if (list_empty(&isert_conn
->conn_fr_pool
))
464 isert_info("Freeing conn %p fastreg pool", isert_conn
);
466 list_for_each_entry_safe(fr_desc
, tmp
,
467 &isert_conn
->conn_fr_pool
, list
) {
468 list_del(&fr_desc
->list
);
469 ib_free_fast_reg_page_list(fr_desc
->data_frpl
);
470 ib_dereg_mr(fr_desc
->data_mr
);
471 if (fr_desc
->pi_ctx
) {
472 ib_free_fast_reg_page_list(fr_desc
->pi_ctx
->prot_frpl
);
473 ib_dereg_mr(fr_desc
->pi_ctx
->prot_mr
);
474 ib_destroy_mr(fr_desc
->pi_ctx
->sig_mr
);
475 kfree(fr_desc
->pi_ctx
);
481 if (i
< isert_conn
->conn_fr_pool_size
)
482 isert_warn("Pool still has %d regions registered\n",
483 isert_conn
->conn_fr_pool_size
- i
);
487 isert_create_pi_ctx(struct fast_reg_descriptor
*desc
,
488 struct ib_device
*device
,
491 struct ib_mr_init_attr mr_init_attr
;
492 struct pi_context
*pi_ctx
;
495 pi_ctx
= kzalloc(sizeof(*desc
->pi_ctx
), GFP_KERNEL
);
497 isert_err("Failed to allocate pi context\n");
501 pi_ctx
->prot_frpl
= ib_alloc_fast_reg_page_list(device
,
502 ISCSI_ISER_SG_TABLESIZE
);
503 if (IS_ERR(pi_ctx
->prot_frpl
)) {
504 isert_err("Failed to allocate prot frpl err=%ld\n",
505 PTR_ERR(pi_ctx
->prot_frpl
));
506 ret
= PTR_ERR(pi_ctx
->prot_frpl
);
510 pi_ctx
->prot_mr
= ib_alloc_fast_reg_mr(pd
, ISCSI_ISER_SG_TABLESIZE
);
511 if (IS_ERR(pi_ctx
->prot_mr
)) {
512 isert_err("Failed to allocate prot frmr err=%ld\n",
513 PTR_ERR(pi_ctx
->prot_mr
));
514 ret
= PTR_ERR(pi_ctx
->prot_mr
);
517 desc
->ind
|= ISERT_PROT_KEY_VALID
;
519 memset(&mr_init_attr
, 0, sizeof(mr_init_attr
));
520 mr_init_attr
.max_reg_descriptors
= 2;
521 mr_init_attr
.flags
|= IB_MR_SIGNATURE_EN
;
522 pi_ctx
->sig_mr
= ib_create_mr(pd
, &mr_init_attr
);
523 if (IS_ERR(pi_ctx
->sig_mr
)) {
524 isert_err("Failed to allocate signature enabled mr err=%ld\n",
525 PTR_ERR(pi_ctx
->sig_mr
));
526 ret
= PTR_ERR(pi_ctx
->sig_mr
);
530 desc
->pi_ctx
= pi_ctx
;
531 desc
->ind
|= ISERT_SIG_KEY_VALID
;
532 desc
->ind
&= ~ISERT_PROTECTED
;
537 ib_dereg_mr(desc
->pi_ctx
->prot_mr
);
539 ib_free_fast_reg_page_list(desc
->pi_ctx
->prot_frpl
);
547 isert_create_fr_desc(struct ib_device
*ib_device
, struct ib_pd
*pd
,
548 struct fast_reg_descriptor
*fr_desc
)
552 fr_desc
->data_frpl
= ib_alloc_fast_reg_page_list(ib_device
,
553 ISCSI_ISER_SG_TABLESIZE
);
554 if (IS_ERR(fr_desc
->data_frpl
)) {
555 isert_err("Failed to allocate data frpl err=%ld\n",
556 PTR_ERR(fr_desc
->data_frpl
));
557 return PTR_ERR(fr_desc
->data_frpl
);
560 fr_desc
->data_mr
= ib_alloc_fast_reg_mr(pd
, ISCSI_ISER_SG_TABLESIZE
);
561 if (IS_ERR(fr_desc
->data_mr
)) {
562 isert_err("Failed to allocate data frmr err=%ld\n",
563 PTR_ERR(fr_desc
->data_mr
));
564 ret
= PTR_ERR(fr_desc
->data_mr
);
567 fr_desc
->ind
|= ISERT_DATA_KEY_VALID
;
569 isert_dbg("Created fr_desc %p\n", fr_desc
);
574 ib_free_fast_reg_page_list(fr_desc
->data_frpl
);
580 isert_conn_create_fastreg_pool(struct isert_conn
*isert_conn
)
582 struct fast_reg_descriptor
*fr_desc
;
583 struct isert_device
*device
= isert_conn
->conn_device
;
584 struct se_session
*se_sess
= isert_conn
->conn
->sess
->se_sess
;
585 struct se_node_acl
*se_nacl
= se_sess
->se_node_acl
;
588 * Setup the number of FRMRs based upon the number of tags
589 * available to session in iscsi_target_locate_portal().
591 tag_num
= max_t(u32
, ISCSIT_MIN_TAGS
, se_nacl
->queue_depth
);
592 tag_num
= (tag_num
* 2) + ISCSIT_EXTRA_TAGS
;
594 isert_conn
->conn_fr_pool_size
= 0;
595 for (i
= 0; i
< tag_num
; i
++) {
596 fr_desc
= kzalloc(sizeof(*fr_desc
), GFP_KERNEL
);
598 isert_err("Failed to allocate fast_reg descriptor\n");
603 ret
= isert_create_fr_desc(device
->ib_device
,
604 device
->pd
, fr_desc
);
606 isert_err("Failed to create fastreg descriptor err=%d\n",
612 list_add_tail(&fr_desc
->list
, &isert_conn
->conn_fr_pool
);
613 isert_conn
->conn_fr_pool_size
++;
616 isert_dbg("Creating conn %p fastreg pool size=%d",
617 isert_conn
, isert_conn
->conn_fr_pool_size
);
622 isert_conn_free_fastreg_pool(isert_conn
);
627 isert_connect_request(struct rdma_cm_id
*cma_id
, struct rdma_cm_event
*event
)
629 struct isert_np
*isert_np
= cma_id
->context
;
630 struct iscsi_np
*np
= isert_np
->np
;
631 struct isert_conn
*isert_conn
;
632 struct isert_device
*device
;
633 struct ib_device
*ib_dev
= cma_id
->device
;
636 spin_lock_bh(&np
->np_thread_lock
);
638 spin_unlock_bh(&np
->np_thread_lock
);
639 isert_dbg("iscsi_np is not enabled, reject connect request\n");
640 return rdma_reject(cma_id
, NULL
, 0);
642 spin_unlock_bh(&np
->np_thread_lock
);
644 isert_dbg("cma_id: %p, portal: %p\n",
645 cma_id
, cma_id
->context
);
647 isert_conn
= kzalloc(sizeof(struct isert_conn
), GFP_KERNEL
);
649 isert_err("Unable to allocate isert_conn\n");
652 isert_conn
->state
= ISER_CONN_INIT
;
653 INIT_LIST_HEAD(&isert_conn
->conn_accept_node
);
654 init_completion(&isert_conn
->conn_login_comp
);
655 init_completion(&isert_conn
->login_req_comp
);
656 init_completion(&isert_conn
->conn_wait
);
657 kref_init(&isert_conn
->conn_kref
);
658 mutex_init(&isert_conn
->conn_mutex
);
659 spin_lock_init(&isert_conn
->conn_lock
);
660 INIT_LIST_HEAD(&isert_conn
->conn_fr_pool
);
662 isert_conn
->conn_cm_id
= cma_id
;
664 isert_conn
->login_buf
= kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN
+
665 ISER_RX_LOGIN_SIZE
, GFP_KERNEL
);
666 if (!isert_conn
->login_buf
) {
667 isert_err("Unable to allocate isert_conn->login_buf\n");
672 isert_conn
->login_req_buf
= isert_conn
->login_buf
;
673 isert_conn
->login_rsp_buf
= isert_conn
->login_buf
+
674 ISCSI_DEF_MAX_RECV_SEG_LEN
;
675 isert_dbg("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n",
676 isert_conn
->login_buf
, isert_conn
->login_req_buf
,
677 isert_conn
->login_rsp_buf
);
679 isert_conn
->login_req_dma
= ib_dma_map_single(ib_dev
,
680 (void *)isert_conn
->login_req_buf
,
681 ISCSI_DEF_MAX_RECV_SEG_LEN
, DMA_FROM_DEVICE
);
683 ret
= ib_dma_mapping_error(ib_dev
, isert_conn
->login_req_dma
);
685 isert_err("ib_dma_mapping_error failed for login_req_dma: %d\n",
687 isert_conn
->login_req_dma
= 0;
691 isert_conn
->login_rsp_dma
= ib_dma_map_single(ib_dev
,
692 (void *)isert_conn
->login_rsp_buf
,
693 ISER_RX_LOGIN_SIZE
, DMA_TO_DEVICE
);
695 ret
= ib_dma_mapping_error(ib_dev
, isert_conn
->login_rsp_dma
);
697 isert_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n",
699 isert_conn
->login_rsp_dma
= 0;
700 goto out_req_dma_map
;
703 device
= isert_device_find_by_ib_dev(cma_id
);
704 if (IS_ERR(device
)) {
705 ret
= PTR_ERR(device
);
706 goto out_rsp_dma_map
;
709 /* Set max inflight RDMA READ requests */
710 isert_conn
->initiator_depth
= min_t(u8
,
711 event
->param
.conn
.initiator_depth
,
712 device
->dev_attr
.max_qp_init_rd_atom
);
713 isert_dbg("Using initiator_depth: %u\n", isert_conn
->initiator_depth
);
715 isert_conn
->conn_device
= device
;
717 ret
= isert_conn_setup_qp(isert_conn
, cma_id
);
721 ret
= isert_rdma_post_recvl(isert_conn
);
725 ret
= isert_rdma_accept(isert_conn
);
729 mutex_lock(&isert_np
->np_accept_mutex
);
730 list_add_tail(&isert_conn
->conn_accept_node
, &isert_np
->np_accept_list
);
731 mutex_unlock(&isert_np
->np_accept_mutex
);
733 isert_info("np %p: Allow accept_np to continue\n", np
);
734 up(&isert_np
->np_sem
);
738 isert_device_try_release(device
);
740 ib_dma_unmap_single(ib_dev
, isert_conn
->login_rsp_dma
,
741 ISER_RX_LOGIN_SIZE
, DMA_TO_DEVICE
);
743 ib_dma_unmap_single(ib_dev
, isert_conn
->login_req_dma
,
744 ISCSI_DEF_MAX_RECV_SEG_LEN
, DMA_FROM_DEVICE
);
746 kfree(isert_conn
->login_buf
);
749 rdma_reject(cma_id
, NULL
, 0);
754 isert_connect_release(struct isert_conn
*isert_conn
)
756 struct isert_device
*device
= isert_conn
->conn_device
;
757 struct ib_device
*ib_dev
= device
->ib_device
;
759 isert_dbg("conn %p\n", isert_conn
);
761 if (device
&& device
->use_fastreg
)
762 isert_conn_free_fastreg_pool(isert_conn
);
764 isert_free_rx_descriptors(isert_conn
);
765 if (isert_conn
->conn_cm_id
)
766 rdma_destroy_id(isert_conn
->conn_cm_id
);
768 if (isert_conn
->conn_qp
) {
769 struct isert_comp
*comp
= isert_conn
->conn_qp
->recv_cq
->cq_context
;
771 isert_comp_put(comp
);
772 ib_destroy_qp(isert_conn
->conn_qp
);
775 if (isert_conn
->login_buf
) {
776 ib_dma_unmap_single(ib_dev
, isert_conn
->login_rsp_dma
,
777 ISER_RX_LOGIN_SIZE
, DMA_TO_DEVICE
);
778 ib_dma_unmap_single(ib_dev
, isert_conn
->login_req_dma
,
779 ISCSI_DEF_MAX_RECV_SEG_LEN
,
781 kfree(isert_conn
->login_buf
);
786 isert_device_try_release(device
);
790 isert_connected_handler(struct rdma_cm_id
*cma_id
)
792 struct isert_conn
*isert_conn
= cma_id
->qp
->qp_context
;
794 isert_info("conn %p\n", isert_conn
);
796 if (!kref_get_unless_zero(&isert_conn
->conn_kref
)) {
797 isert_warn("conn %p connect_release is running\n", isert_conn
);
801 mutex_lock(&isert_conn
->conn_mutex
);
802 if (isert_conn
->state
!= ISER_CONN_FULL_FEATURE
)
803 isert_conn
->state
= ISER_CONN_UP
;
804 mutex_unlock(&isert_conn
->conn_mutex
);
808 isert_release_conn_kref(struct kref
*kref
)
810 struct isert_conn
*isert_conn
= container_of(kref
,
811 struct isert_conn
, conn_kref
);
813 isert_info("conn %p final kref %s/%d\n", isert_conn
, current
->comm
,
816 isert_connect_release(isert_conn
);
820 isert_put_conn(struct isert_conn
*isert_conn
)
822 kref_put(&isert_conn
->conn_kref
, isert_release_conn_kref
);
826 * isert_conn_terminate() - Initiate connection termination
827 * @isert_conn: isert connection struct
830 * In case the connection state is FULL_FEATURE, move state
831 * to TEMINATING and start teardown sequence (rdma_disconnect).
832 * In case the connection state is UP, complete flush as well.
834 * This routine must be called with conn_mutex held. Thus it is
835 * safe to call multiple times.
838 isert_conn_terminate(struct isert_conn
*isert_conn
)
842 switch (isert_conn
->state
) {
843 case ISER_CONN_TERMINATING
:
846 case ISER_CONN_FULL_FEATURE
: /* FALLTHRU */
847 isert_info("Terminating conn %p state %d\n",
848 isert_conn
, isert_conn
->state
);
849 isert_conn
->state
= ISER_CONN_TERMINATING
;
850 err
= rdma_disconnect(isert_conn
->conn_cm_id
);
852 isert_warn("Failed rdma_disconnect isert_conn %p\n",
856 isert_warn("conn %p teminating in state %d\n",
857 isert_conn
, isert_conn
->state
);
862 isert_np_cma_handler(struct isert_np
*isert_np
,
863 enum rdma_cm_event_type event
)
865 isert_dbg("isert np %p, handling event %d\n", isert_np
, event
);
868 case RDMA_CM_EVENT_DEVICE_REMOVAL
:
869 isert_np
->np_cm_id
= NULL
;
871 case RDMA_CM_EVENT_ADDR_CHANGE
:
872 isert_np
->np_cm_id
= isert_setup_id(isert_np
);
873 if (IS_ERR(isert_np
->np_cm_id
)) {
874 isert_err("isert np %p setup id failed: %ld\n",
875 isert_np
, PTR_ERR(isert_np
->np_cm_id
));
876 isert_np
->np_cm_id
= NULL
;
880 isert_err("isert np %p Unexpected event %d\n",
888 isert_disconnected_handler(struct rdma_cm_id
*cma_id
,
889 enum rdma_cm_event_type event
)
891 struct isert_np
*isert_np
= cma_id
->context
;
892 struct isert_conn
*isert_conn
;
894 if (isert_np
->np_cm_id
== cma_id
)
895 return isert_np_cma_handler(cma_id
->context
, event
);
897 isert_conn
= cma_id
->qp
->qp_context
;
899 mutex_lock(&isert_conn
->conn_mutex
);
900 isert_conn_terminate(isert_conn
);
901 mutex_unlock(&isert_conn
->conn_mutex
);
903 isert_info("conn %p completing conn_wait\n", isert_conn
);
904 complete(&isert_conn
->conn_wait
);
910 isert_connect_error(struct rdma_cm_id
*cma_id
)
912 struct isert_conn
*isert_conn
= cma_id
->qp
->qp_context
;
914 isert_conn
->conn_cm_id
= NULL
;
915 isert_put_conn(isert_conn
);
921 isert_cma_handler(struct rdma_cm_id
*cma_id
, struct rdma_cm_event
*event
)
925 isert_info("event %d status %d id %p np %p\n", event
->event
,
926 event
->status
, cma_id
, cma_id
->context
);
928 switch (event
->event
) {
929 case RDMA_CM_EVENT_CONNECT_REQUEST
:
930 ret
= isert_connect_request(cma_id
, event
);
932 isert_err("failed handle connect request %d\n", ret
);
934 case RDMA_CM_EVENT_ESTABLISHED
:
935 isert_connected_handler(cma_id
);
937 case RDMA_CM_EVENT_ADDR_CHANGE
: /* FALLTHRU */
938 case RDMA_CM_EVENT_DISCONNECTED
: /* FALLTHRU */
939 case RDMA_CM_EVENT_DEVICE_REMOVAL
: /* FALLTHRU */
940 case RDMA_CM_EVENT_TIMEWAIT_EXIT
: /* FALLTHRU */
941 ret
= isert_disconnected_handler(cma_id
, event
->event
);
943 case RDMA_CM_EVENT_REJECTED
: /* FALLTHRU */
944 case RDMA_CM_EVENT_UNREACHABLE
: /* FALLTHRU */
945 case RDMA_CM_EVENT_CONNECT_ERROR
:
946 ret
= isert_connect_error(cma_id
);
949 isert_err("Unhandled RDMA CMA event: %d\n", event
->event
);
957 isert_post_recv(struct isert_conn
*isert_conn
, u32 count
)
959 struct ib_recv_wr
*rx_wr
, *rx_wr_failed
;
961 unsigned int rx_head
= isert_conn
->conn_rx_desc_head
;
962 struct iser_rx_desc
*rx_desc
;
964 for (rx_wr
= isert_conn
->conn_rx_wr
, i
= 0; i
< count
; i
++, rx_wr
++) {
965 rx_desc
= &isert_conn
->conn_rx_descs
[rx_head
];
966 rx_wr
->wr_id
= (uintptr_t)rx_desc
;
967 rx_wr
->sg_list
= &rx_desc
->rx_sg
;
969 rx_wr
->next
= rx_wr
+ 1;
970 rx_head
= (rx_head
+ 1) & (ISERT_QP_MAX_RECV_DTOS
- 1);
974 rx_wr
->next
= NULL
; /* mark end of work requests list */
976 isert_conn
->post_recv_buf_count
+= count
;
977 ret
= ib_post_recv(isert_conn
->conn_qp
, isert_conn
->conn_rx_wr
,
980 isert_err("ib_post_recv() failed with ret: %d\n", ret
);
981 isert_conn
->post_recv_buf_count
-= count
;
983 isert_dbg("Posted %d RX buffers\n", count
);
984 isert_conn
->conn_rx_desc_head
= rx_head
;
990 isert_post_send(struct isert_conn
*isert_conn
, struct iser_tx_desc
*tx_desc
)
992 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
993 struct ib_send_wr send_wr
, *send_wr_failed
;
996 ib_dma_sync_single_for_device(ib_dev
, tx_desc
->dma_addr
,
997 ISER_HEADERS_LEN
, DMA_TO_DEVICE
);
1000 send_wr
.wr_id
= (uintptr_t)tx_desc
;
1001 send_wr
.sg_list
= tx_desc
->tx_sg
;
1002 send_wr
.num_sge
= tx_desc
->num_sge
;
1003 send_wr
.opcode
= IB_WR_SEND
;
1004 send_wr
.send_flags
= IB_SEND_SIGNALED
;
1006 ret
= ib_post_send(isert_conn
->conn_qp
, &send_wr
, &send_wr_failed
);
1008 isert_err("ib_post_send() failed, ret: %d\n", ret
);
1014 isert_create_send_desc(struct isert_conn
*isert_conn
,
1015 struct isert_cmd
*isert_cmd
,
1016 struct iser_tx_desc
*tx_desc
)
1018 struct isert_device
*device
= isert_conn
->conn_device
;
1019 struct ib_device
*ib_dev
= device
->ib_device
;
1021 ib_dma_sync_single_for_cpu(ib_dev
, tx_desc
->dma_addr
,
1022 ISER_HEADERS_LEN
, DMA_TO_DEVICE
);
1024 memset(&tx_desc
->iser_header
, 0, sizeof(struct iser_hdr
));
1025 tx_desc
->iser_header
.flags
= ISER_VER
;
1027 tx_desc
->num_sge
= 1;
1028 tx_desc
->isert_cmd
= isert_cmd
;
1030 if (tx_desc
->tx_sg
[0].lkey
!= device
->mr
->lkey
) {
1031 tx_desc
->tx_sg
[0].lkey
= device
->mr
->lkey
;
1032 isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc
);
1037 isert_init_tx_hdrs(struct isert_conn
*isert_conn
,
1038 struct iser_tx_desc
*tx_desc
)
1040 struct isert_device
*device
= isert_conn
->conn_device
;
1041 struct ib_device
*ib_dev
= device
->ib_device
;
1044 dma_addr
= ib_dma_map_single(ib_dev
, (void *)tx_desc
,
1045 ISER_HEADERS_LEN
, DMA_TO_DEVICE
);
1046 if (ib_dma_mapping_error(ib_dev
, dma_addr
)) {
1047 isert_err("ib_dma_mapping_error() failed\n");
1051 tx_desc
->dma_addr
= dma_addr
;
1052 tx_desc
->tx_sg
[0].addr
= tx_desc
->dma_addr
;
1053 tx_desc
->tx_sg
[0].length
= ISER_HEADERS_LEN
;
1054 tx_desc
->tx_sg
[0].lkey
= device
->mr
->lkey
;
1056 isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n",
1057 tx_desc
->tx_sg
[0].addr
, tx_desc
->tx_sg
[0].length
,
1058 tx_desc
->tx_sg
[0].lkey
);
1064 isert_init_send_wr(struct isert_conn
*isert_conn
, struct isert_cmd
*isert_cmd
,
1065 struct ib_send_wr
*send_wr
)
1067 struct iser_tx_desc
*tx_desc
= &isert_cmd
->tx_desc
;
1069 isert_cmd
->rdma_wr
.iser_ib_op
= ISER_IB_SEND
;
1070 send_wr
->wr_id
= (uintptr_t)&isert_cmd
->tx_desc
;
1071 send_wr
->opcode
= IB_WR_SEND
;
1072 send_wr
->sg_list
= &tx_desc
->tx_sg
[0];
1073 send_wr
->num_sge
= isert_cmd
->tx_desc
.num_sge
;
1074 send_wr
->send_flags
= IB_SEND_SIGNALED
;
1078 isert_rdma_post_recvl(struct isert_conn
*isert_conn
)
1080 struct ib_recv_wr rx_wr
, *rx_wr_fail
;
1084 memset(&sge
, 0, sizeof(struct ib_sge
));
1085 sge
.addr
= isert_conn
->login_req_dma
;
1086 sge
.length
= ISER_RX_LOGIN_SIZE
;
1087 sge
.lkey
= isert_conn
->conn_device
->mr
->lkey
;
1089 isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n",
1090 sge
.addr
, sge
.length
, sge
.lkey
);
1092 memset(&rx_wr
, 0, sizeof(struct ib_recv_wr
));
1093 rx_wr
.wr_id
= (uintptr_t)isert_conn
->login_req_buf
;
1094 rx_wr
.sg_list
= &sge
;
1097 isert_conn
->post_recv_buf_count
++;
1098 ret
= ib_post_recv(isert_conn
->conn_qp
, &rx_wr
, &rx_wr_fail
);
1100 isert_err("ib_post_recv() failed: %d\n", ret
);
1101 isert_conn
->post_recv_buf_count
--;
1108 isert_put_login_tx(struct iscsi_conn
*conn
, struct iscsi_login
*login
,
1111 struct isert_conn
*isert_conn
= conn
->context
;
1112 struct isert_device
*device
= isert_conn
->conn_device
;
1113 struct ib_device
*ib_dev
= device
->ib_device
;
1114 struct iser_tx_desc
*tx_desc
= &isert_conn
->conn_login_tx_desc
;
1117 isert_create_send_desc(isert_conn
, NULL
, tx_desc
);
1119 memcpy(&tx_desc
->iscsi_header
, &login
->rsp
[0],
1120 sizeof(struct iscsi_hdr
));
1122 isert_init_tx_hdrs(isert_conn
, tx_desc
);
1125 struct ib_sge
*tx_dsg
= &tx_desc
->tx_sg
[1];
1127 ib_dma_sync_single_for_cpu(ib_dev
, isert_conn
->login_rsp_dma
,
1128 length
, DMA_TO_DEVICE
);
1130 memcpy(isert_conn
->login_rsp_buf
, login
->rsp_buf
, length
);
1132 ib_dma_sync_single_for_device(ib_dev
, isert_conn
->login_rsp_dma
,
1133 length
, DMA_TO_DEVICE
);
1135 tx_dsg
->addr
= isert_conn
->login_rsp_dma
;
1136 tx_dsg
->length
= length
;
1137 tx_dsg
->lkey
= isert_conn
->conn_device
->mr
->lkey
;
1138 tx_desc
->num_sge
= 2;
1140 if (!login
->login_failed
) {
1141 if (login
->login_complete
) {
1142 if (!conn
->sess
->sess_ops
->SessionType
&&
1143 isert_conn
->conn_device
->use_fastreg
) {
1144 ret
= isert_conn_create_fastreg_pool(isert_conn
);
1146 isert_err("Conn: %p failed to create"
1147 " fastreg pool\n", isert_conn
);
1152 ret
= isert_alloc_rx_descriptors(isert_conn
);
1156 ret
= isert_post_recv(isert_conn
, ISERT_MIN_POSTED_RX
);
1160 /* Now we are in FULL_FEATURE phase */
1161 mutex_lock(&isert_conn
->conn_mutex
);
1162 isert_conn
->state
= ISER_CONN_FULL_FEATURE
;
1163 mutex_unlock(&isert_conn
->conn_mutex
);
1167 ret
= isert_rdma_post_recvl(isert_conn
);
1172 ret
= isert_post_send(isert_conn
, tx_desc
);
1180 isert_rx_login_req(struct isert_conn
*isert_conn
)
1182 struct iser_rx_desc
*rx_desc
= (void *)isert_conn
->login_req_buf
;
1183 int rx_buflen
= isert_conn
->login_req_len
;
1184 struct iscsi_conn
*conn
= isert_conn
->conn
;
1185 struct iscsi_login
*login
= conn
->conn_login
;
1188 isert_info("conn %p\n", isert_conn
);
1190 WARN_ON_ONCE(!login
);
1192 if (login
->first_request
) {
1193 struct iscsi_login_req
*login_req
=
1194 (struct iscsi_login_req
*)&rx_desc
->iscsi_header
;
1196 * Setup the initial iscsi_login values from the leading
1197 * login request PDU.
1199 login
->leading_connection
= (!login_req
->tsih
) ? 1 : 0;
1200 login
->current_stage
=
1201 (login_req
->flags
& ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK
)
1203 login
->version_min
= login_req
->min_version
;
1204 login
->version_max
= login_req
->max_version
;
1205 memcpy(login
->isid
, login_req
->isid
, 6);
1206 login
->cmd_sn
= be32_to_cpu(login_req
->cmdsn
);
1207 login
->init_task_tag
= login_req
->itt
;
1208 login
->initial_exp_statsn
= be32_to_cpu(login_req
->exp_statsn
);
1209 login
->cid
= be16_to_cpu(login_req
->cid
);
1210 login
->tsih
= be16_to_cpu(login_req
->tsih
);
1213 memcpy(&login
->req
[0], (void *)&rx_desc
->iscsi_header
, ISCSI_HDR_LEN
);
1215 size
= min(rx_buflen
, MAX_KEY_VALUE_PAIRS
);
1216 isert_dbg("Using login payload size: %d, rx_buflen: %d "
1217 "MAX_KEY_VALUE_PAIRS: %d\n", size
, rx_buflen
,
1218 MAX_KEY_VALUE_PAIRS
);
1219 memcpy(login
->req_buf
, &rx_desc
->data
[0], size
);
1221 if (login
->first_request
) {
1222 complete(&isert_conn
->conn_login_comp
);
1225 schedule_delayed_work(&conn
->login_work
, 0);
1228 static struct iscsi_cmd
1229 *isert_allocate_cmd(struct iscsi_conn
*conn
)
1231 struct isert_conn
*isert_conn
= conn
->context
;
1232 struct isert_cmd
*isert_cmd
;
1233 struct iscsi_cmd
*cmd
;
1235 cmd
= iscsit_allocate_cmd(conn
, TASK_INTERRUPTIBLE
);
1237 isert_err("Unable to allocate iscsi_cmd + isert_cmd\n");
1240 isert_cmd
= iscsit_priv_cmd(cmd
);
1241 isert_cmd
->conn
= isert_conn
;
1242 isert_cmd
->iscsi_cmd
= cmd
;
1248 isert_handle_scsi_cmd(struct isert_conn
*isert_conn
,
1249 struct isert_cmd
*isert_cmd
, struct iscsi_cmd
*cmd
,
1250 struct iser_rx_desc
*rx_desc
, unsigned char *buf
)
1252 struct iscsi_conn
*conn
= isert_conn
->conn
;
1253 struct iscsi_scsi_req
*hdr
= (struct iscsi_scsi_req
*)buf
;
1254 struct scatterlist
*sg
;
1255 int imm_data
, imm_data_len
, unsol_data
, sg_nents
, rc
;
1256 bool dump_payload
= false;
1258 rc
= iscsit_setup_scsi_cmd(conn
, cmd
, buf
);
1262 imm_data
= cmd
->immediate_data
;
1263 imm_data_len
= cmd
->first_burst_len
;
1264 unsol_data
= cmd
->unsolicited_data
;
1266 rc
= iscsit_process_scsi_cmd(conn
, cmd
, hdr
);
1269 } else if (rc
> 0) {
1270 dump_payload
= true;
1277 sg
= &cmd
->se_cmd
.t_data_sg
[0];
1278 sg_nents
= max(1UL, DIV_ROUND_UP(imm_data_len
, PAGE_SIZE
));
1280 isert_dbg("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n",
1281 sg
, sg_nents
, &rx_desc
->data
[0], imm_data_len
);
1283 sg_copy_from_buffer(sg
, sg_nents
, &rx_desc
->data
[0], imm_data_len
);
1285 cmd
->write_data_done
+= imm_data_len
;
1287 if (cmd
->write_data_done
== cmd
->se_cmd
.data_length
) {
1288 spin_lock_bh(&cmd
->istate_lock
);
1289 cmd
->cmd_flags
|= ICF_GOT_LAST_DATAOUT
;
1290 cmd
->i_state
= ISTATE_RECEIVED_LAST_DATAOUT
;
1291 spin_unlock_bh(&cmd
->istate_lock
);
1295 rc
= iscsit_sequence_cmd(conn
, cmd
, buf
, hdr
->cmdsn
);
1297 if (!rc
&& dump_payload
== false && unsol_data
)
1298 iscsit_set_unsoliticed_dataout(cmd
);
1299 else if (dump_payload
&& imm_data
)
1300 target_put_sess_cmd(conn
->sess
->se_sess
, &cmd
->se_cmd
);
1306 isert_handle_iscsi_dataout(struct isert_conn
*isert_conn
,
1307 struct iser_rx_desc
*rx_desc
, unsigned char *buf
)
1309 struct scatterlist
*sg_start
;
1310 struct iscsi_conn
*conn
= isert_conn
->conn
;
1311 struct iscsi_cmd
*cmd
= NULL
;
1312 struct iscsi_data
*hdr
= (struct iscsi_data
*)buf
;
1313 u32 unsol_data_len
= ntoh24(hdr
->dlength
);
1314 int rc
, sg_nents
, sg_off
, page_off
;
1316 rc
= iscsit_check_dataout_hdr(conn
, buf
, &cmd
);
1322 * FIXME: Unexpected unsolicited_data out
1324 if (!cmd
->unsolicited_data
) {
1325 isert_err("Received unexpected solicited data payload\n");
1330 isert_dbg("Unsolicited DataOut unsol_data_len: %u, "
1331 "write_data_done: %u, data_length: %u\n",
1332 unsol_data_len
, cmd
->write_data_done
,
1333 cmd
->se_cmd
.data_length
);
1335 sg_off
= cmd
->write_data_done
/ PAGE_SIZE
;
1336 sg_start
= &cmd
->se_cmd
.t_data_sg
[sg_off
];
1337 sg_nents
= max(1UL, DIV_ROUND_UP(unsol_data_len
, PAGE_SIZE
));
1338 page_off
= cmd
->write_data_done
% PAGE_SIZE
;
1340 * FIXME: Non page-aligned unsolicited_data out
1343 isert_err("unexpected non-page aligned data payload\n");
1347 isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u "
1348 "sg_nents: %u from %p %u\n", sg_start
, sg_off
,
1349 sg_nents
, &rx_desc
->data
[0], unsol_data_len
);
1351 sg_copy_from_buffer(sg_start
, sg_nents
, &rx_desc
->data
[0],
1354 rc
= iscsit_check_dataout_payload(cmd
, hdr
, false);
1362 isert_handle_nop_out(struct isert_conn
*isert_conn
, struct isert_cmd
*isert_cmd
,
1363 struct iscsi_cmd
*cmd
, struct iser_rx_desc
*rx_desc
,
1366 struct iscsi_conn
*conn
= isert_conn
->conn
;
1367 struct iscsi_nopout
*hdr
= (struct iscsi_nopout
*)buf
;
1370 rc
= iscsit_setup_nop_out(conn
, cmd
, hdr
);
1374 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload
1377 return iscsit_process_nop_out(conn
, cmd
, hdr
);
1381 isert_handle_text_cmd(struct isert_conn
*isert_conn
, struct isert_cmd
*isert_cmd
,
1382 struct iscsi_cmd
*cmd
, struct iser_rx_desc
*rx_desc
,
1383 struct iscsi_text
*hdr
)
1385 struct iscsi_conn
*conn
= isert_conn
->conn
;
1386 u32 payload_length
= ntoh24(hdr
->dlength
);
1388 unsigned char *text_in
= NULL
;
1390 rc
= iscsit_setup_text_cmd(conn
, cmd
, hdr
);
1394 if (payload_length
) {
1395 text_in
= kzalloc(payload_length
, GFP_KERNEL
);
1397 isert_err("Unable to allocate text_in of payload_length: %u\n",
1402 cmd
->text_in_ptr
= text_in
;
1404 memcpy(cmd
->text_in_ptr
, &rx_desc
->data
[0], payload_length
);
1406 return iscsit_process_text_cmd(conn
, cmd
, hdr
);
1410 isert_rx_opcode(struct isert_conn
*isert_conn
, struct iser_rx_desc
*rx_desc
,
1411 uint32_t read_stag
, uint64_t read_va
,
1412 uint32_t write_stag
, uint64_t write_va
)
1414 struct iscsi_hdr
*hdr
= &rx_desc
->iscsi_header
;
1415 struct iscsi_conn
*conn
= isert_conn
->conn
;
1416 struct iscsi_cmd
*cmd
;
1417 struct isert_cmd
*isert_cmd
;
1419 u8 opcode
= (hdr
->opcode
& ISCSI_OPCODE_MASK
);
1421 if (conn
->sess
->sess_ops
->SessionType
&&
1422 (!(opcode
& ISCSI_OP_TEXT
) || !(opcode
& ISCSI_OP_LOGOUT
))) {
1423 isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery,"
1424 " ignoring\n", opcode
);
1429 case ISCSI_OP_SCSI_CMD
:
1430 cmd
= isert_allocate_cmd(conn
);
1434 isert_cmd
= iscsit_priv_cmd(cmd
);
1435 isert_cmd
->read_stag
= read_stag
;
1436 isert_cmd
->read_va
= read_va
;
1437 isert_cmd
->write_stag
= write_stag
;
1438 isert_cmd
->write_va
= write_va
;
1440 ret
= isert_handle_scsi_cmd(isert_conn
, isert_cmd
, cmd
,
1441 rx_desc
, (unsigned char *)hdr
);
1443 case ISCSI_OP_NOOP_OUT
:
1444 cmd
= isert_allocate_cmd(conn
);
1448 isert_cmd
= iscsit_priv_cmd(cmd
);
1449 ret
= isert_handle_nop_out(isert_conn
, isert_cmd
, cmd
,
1450 rx_desc
, (unsigned char *)hdr
);
1452 case ISCSI_OP_SCSI_DATA_OUT
:
1453 ret
= isert_handle_iscsi_dataout(isert_conn
, rx_desc
,
1454 (unsigned char *)hdr
);
1456 case ISCSI_OP_SCSI_TMFUNC
:
1457 cmd
= isert_allocate_cmd(conn
);
1461 ret
= iscsit_handle_task_mgt_cmd(conn
, cmd
,
1462 (unsigned char *)hdr
);
1464 case ISCSI_OP_LOGOUT
:
1465 cmd
= isert_allocate_cmd(conn
);
1469 ret
= iscsit_handle_logout_cmd(conn
, cmd
, (unsigned char *)hdr
);
1472 if (be32_to_cpu(hdr
->ttt
) != 0xFFFFFFFF) {
1473 cmd
= iscsit_find_cmd_from_itt(conn
, hdr
->itt
);
1477 cmd
= isert_allocate_cmd(conn
);
1482 isert_cmd
= iscsit_priv_cmd(cmd
);
1483 ret
= isert_handle_text_cmd(isert_conn
, isert_cmd
, cmd
,
1484 rx_desc
, (struct iscsi_text
*)hdr
);
1487 isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode
);
1496 isert_rx_do_work(struct iser_rx_desc
*rx_desc
, struct isert_conn
*isert_conn
)
1498 struct iser_hdr
*iser_hdr
= &rx_desc
->iser_header
;
1499 uint64_t read_va
= 0, write_va
= 0;
1500 uint32_t read_stag
= 0, write_stag
= 0;
1503 switch (iser_hdr
->flags
& 0xF0) {
1505 if (iser_hdr
->flags
& ISER_RSV
) {
1506 read_stag
= be32_to_cpu(iser_hdr
->read_stag
);
1507 read_va
= be64_to_cpu(iser_hdr
->read_va
);
1508 isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n",
1509 read_stag
, (unsigned long long)read_va
);
1511 if (iser_hdr
->flags
& ISER_WSV
) {
1512 write_stag
= be32_to_cpu(iser_hdr
->write_stag
);
1513 write_va
= be64_to_cpu(iser_hdr
->write_va
);
1514 isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n",
1515 write_stag
, (unsigned long long)write_va
);
1518 isert_dbg("ISER ISCSI_CTRL PDU\n");
1521 isert_err("iSER Hello message\n");
1524 isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr
->flags
);
1528 rc
= isert_rx_opcode(isert_conn
, rx_desc
,
1529 read_stag
, read_va
, write_stag
, write_va
);
1533 isert_rx_completion(struct iser_rx_desc
*desc
, struct isert_conn
*isert_conn
,
1536 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
1537 struct iscsi_hdr
*hdr
;
1539 int rx_buflen
, outstanding
;
1541 if ((char *)desc
== isert_conn
->login_req_buf
) {
1542 rx_dma
= isert_conn
->login_req_dma
;
1543 rx_buflen
= ISER_RX_LOGIN_SIZE
;
1544 isert_dbg("login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n",
1547 rx_dma
= desc
->dma_addr
;
1548 rx_buflen
= ISER_RX_PAYLOAD_SIZE
;
1549 isert_dbg("req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n",
1553 ib_dma_sync_single_for_cpu(ib_dev
, rx_dma
, rx_buflen
, DMA_FROM_DEVICE
);
1555 hdr
= &desc
->iscsi_header
;
1556 isert_dbg("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n",
1557 hdr
->opcode
, hdr
->itt
, hdr
->flags
,
1558 (int)(xfer_len
- ISER_HEADERS_LEN
));
1560 if ((char *)desc
== isert_conn
->login_req_buf
) {
1561 isert_conn
->login_req_len
= xfer_len
- ISER_HEADERS_LEN
;
1562 if (isert_conn
->conn
) {
1563 struct iscsi_login
*login
= isert_conn
->conn
->conn_login
;
1565 if (login
&& !login
->first_request
)
1566 isert_rx_login_req(isert_conn
);
1568 mutex_lock(&isert_conn
->conn_mutex
);
1569 complete(&isert_conn
->login_req_comp
);
1570 mutex_unlock(&isert_conn
->conn_mutex
);
1572 isert_rx_do_work(desc
, isert_conn
);
1575 ib_dma_sync_single_for_device(ib_dev
, rx_dma
, rx_buflen
,
1578 isert_conn
->post_recv_buf_count
--;
1579 isert_dbg("Decremented post_recv_buf_count: %d\n",
1580 isert_conn
->post_recv_buf_count
);
1582 if ((char *)desc
== isert_conn
->login_req_buf
)
1585 outstanding
= isert_conn
->post_recv_buf_count
;
1586 if (outstanding
+ ISERT_MIN_POSTED_RX
<= ISERT_QP_MAX_RECV_DTOS
) {
1587 int err
, count
= min(ISERT_QP_MAX_RECV_DTOS
- outstanding
,
1588 ISERT_MIN_POSTED_RX
);
1589 err
= isert_post_recv(isert_conn
, count
);
1591 isert_err("isert_post_recv() count: %d failed, %d\n",
1598 isert_map_data_buf(struct isert_conn
*isert_conn
, struct isert_cmd
*isert_cmd
,
1599 struct scatterlist
*sg
, u32 nents
, u32 length
, u32 offset
,
1600 enum iser_ib_op_code op
, struct isert_data_buf
*data
)
1602 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
1604 data
->dma_dir
= op
== ISER_IB_RDMA_WRITE
?
1605 DMA_TO_DEVICE
: DMA_FROM_DEVICE
;
1607 data
->len
= length
- offset
;
1608 data
->offset
= offset
;
1609 data
->sg_off
= data
->offset
/ PAGE_SIZE
;
1611 data
->sg
= &sg
[data
->sg_off
];
1612 data
->nents
= min_t(unsigned int, nents
- data
->sg_off
,
1613 ISCSI_ISER_SG_TABLESIZE
);
1614 data
->len
= min_t(unsigned int, data
->len
, ISCSI_ISER_SG_TABLESIZE
*
1617 data
->dma_nents
= ib_dma_map_sg(ib_dev
, data
->sg
, data
->nents
,
1619 if (unlikely(!data
->dma_nents
)) {
1620 isert_err("Cmd: unable to dma map SGs %p\n", sg
);
1624 isert_dbg("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n",
1625 isert_cmd
, data
->dma_nents
, data
->sg
, data
->nents
, data
->len
);
1631 isert_unmap_data_buf(struct isert_conn
*isert_conn
, struct isert_data_buf
*data
)
1633 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
1635 ib_dma_unmap_sg(ib_dev
, data
->sg
, data
->nents
, data
->dma_dir
);
1636 memset(data
, 0, sizeof(*data
));
1642 isert_unmap_cmd(struct isert_cmd
*isert_cmd
, struct isert_conn
*isert_conn
)
1644 struct isert_rdma_wr
*wr
= &isert_cmd
->rdma_wr
;
1646 isert_dbg("Cmd %p\n", isert_cmd
);
1649 isert_dbg("Cmd %p unmap_sg op\n", isert_cmd
);
1650 isert_unmap_data_buf(isert_conn
, &wr
->data
);
1654 isert_dbg("Cmd %p free send_wr\n", isert_cmd
);
1660 isert_dbg("Cmd %p free ib_sge\n", isert_cmd
);
1667 isert_unreg_rdma(struct isert_cmd
*isert_cmd
, struct isert_conn
*isert_conn
)
1669 struct isert_rdma_wr
*wr
= &isert_cmd
->rdma_wr
;
1671 isert_dbg("Cmd %p\n", isert_cmd
);
1674 isert_dbg("Cmd %p free fr_desc %p\n", isert_cmd
, wr
->fr_desc
);
1675 if (wr
->fr_desc
->ind
& ISERT_PROTECTED
) {
1676 isert_unmap_data_buf(isert_conn
, &wr
->prot
);
1677 wr
->fr_desc
->ind
&= ~ISERT_PROTECTED
;
1679 spin_lock_bh(&isert_conn
->conn_lock
);
1680 list_add_tail(&wr
->fr_desc
->list
, &isert_conn
->conn_fr_pool
);
1681 spin_unlock_bh(&isert_conn
->conn_lock
);
1686 isert_dbg("Cmd %p unmap_sg op\n", isert_cmd
);
1687 isert_unmap_data_buf(isert_conn
, &wr
->data
);
1695 isert_put_cmd(struct isert_cmd
*isert_cmd
, bool comp_err
)
1697 struct iscsi_cmd
*cmd
= isert_cmd
->iscsi_cmd
;
1698 struct isert_conn
*isert_conn
= isert_cmd
->conn
;
1699 struct iscsi_conn
*conn
= isert_conn
->conn
;
1700 struct isert_device
*device
= isert_conn
->conn_device
;
1701 struct iscsi_text_rsp
*hdr
;
1703 isert_dbg("Cmd %p\n", isert_cmd
);
1705 switch (cmd
->iscsi_opcode
) {
1706 case ISCSI_OP_SCSI_CMD
:
1707 spin_lock_bh(&conn
->cmd_lock
);
1708 if (!list_empty(&cmd
->i_conn_node
))
1709 list_del_init(&cmd
->i_conn_node
);
1710 spin_unlock_bh(&conn
->cmd_lock
);
1712 if (cmd
->data_direction
== DMA_TO_DEVICE
) {
1713 iscsit_stop_dataout_timer(cmd
);
1715 * Check for special case during comp_err where
1716 * WRITE_PENDING has been handed off from core,
1717 * but requires an extra target_put_sess_cmd()
1718 * before transport_generic_free_cmd() below.
1721 cmd
->se_cmd
.t_state
== TRANSPORT_WRITE_PENDING
) {
1722 struct se_cmd
*se_cmd
= &cmd
->se_cmd
;
1724 target_put_sess_cmd(se_cmd
->se_sess
, se_cmd
);
1728 device
->unreg_rdma_mem(isert_cmd
, isert_conn
);
1729 transport_generic_free_cmd(&cmd
->se_cmd
, 0);
1731 case ISCSI_OP_SCSI_TMFUNC
:
1732 spin_lock_bh(&conn
->cmd_lock
);
1733 if (!list_empty(&cmd
->i_conn_node
))
1734 list_del_init(&cmd
->i_conn_node
);
1735 spin_unlock_bh(&conn
->cmd_lock
);
1737 transport_generic_free_cmd(&cmd
->se_cmd
, 0);
1739 case ISCSI_OP_REJECT
:
1740 case ISCSI_OP_NOOP_OUT
:
1742 hdr
= (struct iscsi_text_rsp
*)&isert_cmd
->tx_desc
.iscsi_header
;
1743 /* If the continue bit is on, keep the command alive */
1744 if (hdr
->flags
& ISCSI_FLAG_TEXT_CONTINUE
)
1747 spin_lock_bh(&conn
->cmd_lock
);
1748 if (!list_empty(&cmd
->i_conn_node
))
1749 list_del_init(&cmd
->i_conn_node
);
1750 spin_unlock_bh(&conn
->cmd_lock
);
1753 * Handle special case for REJECT when iscsi_add_reject*() has
1754 * overwritten the original iscsi_opcode assignment, and the
1755 * associated cmd->se_cmd needs to be released.
1757 if (cmd
->se_cmd
.se_tfo
!= NULL
) {
1758 isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n",
1760 transport_generic_free_cmd(&cmd
->se_cmd
, 0);
1767 iscsit_release_cmd(cmd
);
1773 isert_unmap_tx_desc(struct iser_tx_desc
*tx_desc
, struct ib_device
*ib_dev
)
1775 if (tx_desc
->dma_addr
!= 0) {
1776 isert_dbg("unmap single for tx_desc->dma_addr\n");
1777 ib_dma_unmap_single(ib_dev
, tx_desc
->dma_addr
,
1778 ISER_HEADERS_LEN
, DMA_TO_DEVICE
);
1779 tx_desc
->dma_addr
= 0;
1784 isert_completion_put(struct iser_tx_desc
*tx_desc
, struct isert_cmd
*isert_cmd
,
1785 struct ib_device
*ib_dev
, bool comp_err
)
1787 if (isert_cmd
->pdu_buf_dma
!= 0) {
1788 isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n");
1789 ib_dma_unmap_single(ib_dev
, isert_cmd
->pdu_buf_dma
,
1790 isert_cmd
->pdu_buf_len
, DMA_TO_DEVICE
);
1791 isert_cmd
->pdu_buf_dma
= 0;
1794 isert_unmap_tx_desc(tx_desc
, ib_dev
);
1795 isert_put_cmd(isert_cmd
, comp_err
);
1799 isert_check_pi_status(struct se_cmd
*se_cmd
, struct ib_mr
*sig_mr
)
1801 struct ib_mr_status mr_status
;
1804 ret
= ib_check_mr_status(sig_mr
, IB_MR_CHECK_SIG_STATUS
, &mr_status
);
1806 isert_err("ib_check_mr_status failed, ret %d\n", ret
);
1807 goto fail_mr_status
;
1810 if (mr_status
.fail_status
& IB_MR_CHECK_SIG_STATUS
) {
1812 u32 block_size
= se_cmd
->se_dev
->dev_attrib
.block_size
+ 8;
1814 switch (mr_status
.sig_err
.err_type
) {
1815 case IB_SIG_BAD_GUARD
:
1816 se_cmd
->pi_err
= TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED
;
1818 case IB_SIG_BAD_REFTAG
:
1819 se_cmd
->pi_err
= TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED
;
1821 case IB_SIG_BAD_APPTAG
:
1822 se_cmd
->pi_err
= TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED
;
1825 sec_offset_err
= mr_status
.sig_err
.sig_err_offset
;
1826 do_div(sec_offset_err
, block_size
);
1827 se_cmd
->bad_sector
= sec_offset_err
+ se_cmd
->t_task_lba
;
1829 isert_err("PI error found type %d at sector 0x%llx "
1830 "expected 0x%x vs actual 0x%x\n",
1831 mr_status
.sig_err
.err_type
,
1832 (unsigned long long)se_cmd
->bad_sector
,
1833 mr_status
.sig_err
.expected
,
1834 mr_status
.sig_err
.actual
);
1843 isert_completion_rdma_write(struct iser_tx_desc
*tx_desc
,
1844 struct isert_cmd
*isert_cmd
)
1846 struct isert_rdma_wr
*wr
= &isert_cmd
->rdma_wr
;
1847 struct iscsi_cmd
*cmd
= isert_cmd
->iscsi_cmd
;
1848 struct se_cmd
*se_cmd
= &cmd
->se_cmd
;
1849 struct isert_conn
*isert_conn
= isert_cmd
->conn
;
1850 struct isert_device
*device
= isert_conn
->conn_device
;
1853 if (wr
->fr_desc
&& wr
->fr_desc
->ind
& ISERT_PROTECTED
) {
1854 ret
= isert_check_pi_status(se_cmd
,
1855 wr
->fr_desc
->pi_ctx
->sig_mr
);
1856 wr
->fr_desc
->ind
&= ~ISERT_PROTECTED
;
1859 device
->unreg_rdma_mem(isert_cmd
, isert_conn
);
1860 wr
->send_wr_num
= 0;
1862 transport_send_check_condition_and_sense(se_cmd
,
1865 isert_put_response(isert_conn
->conn
, cmd
);
1869 isert_completion_rdma_read(struct iser_tx_desc
*tx_desc
,
1870 struct isert_cmd
*isert_cmd
)
1872 struct isert_rdma_wr
*wr
= &isert_cmd
->rdma_wr
;
1873 struct iscsi_cmd
*cmd
= isert_cmd
->iscsi_cmd
;
1874 struct se_cmd
*se_cmd
= &cmd
->se_cmd
;
1875 struct isert_conn
*isert_conn
= isert_cmd
->conn
;
1876 struct isert_device
*device
= isert_conn
->conn_device
;
1879 if (wr
->fr_desc
&& wr
->fr_desc
->ind
& ISERT_PROTECTED
) {
1880 ret
= isert_check_pi_status(se_cmd
,
1881 wr
->fr_desc
->pi_ctx
->sig_mr
);
1882 wr
->fr_desc
->ind
&= ~ISERT_PROTECTED
;
1885 iscsit_stop_dataout_timer(cmd
);
1886 device
->unreg_rdma_mem(isert_cmd
, isert_conn
);
1887 cmd
->write_data_done
= wr
->data
.len
;
1888 wr
->send_wr_num
= 0;
1890 isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd
);
1891 spin_lock_bh(&cmd
->istate_lock
);
1892 cmd
->cmd_flags
|= ICF_GOT_LAST_DATAOUT
;
1893 cmd
->i_state
= ISTATE_RECEIVED_LAST_DATAOUT
;
1894 spin_unlock_bh(&cmd
->istate_lock
);
1897 target_put_sess_cmd(se_cmd
->se_sess
, se_cmd
);
1898 transport_send_check_condition_and_sense(se_cmd
,
1901 target_execute_cmd(se_cmd
);
1906 isert_do_control_comp(struct work_struct
*work
)
1908 struct isert_cmd
*isert_cmd
= container_of(work
,
1909 struct isert_cmd
, comp_work
);
1910 struct isert_conn
*isert_conn
= isert_cmd
->conn
;
1911 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
1912 struct iscsi_cmd
*cmd
= isert_cmd
->iscsi_cmd
;
1914 isert_dbg("Cmd %p i_state %d\n", isert_cmd
, cmd
->i_state
);
1916 switch (cmd
->i_state
) {
1917 case ISTATE_SEND_TASKMGTRSP
:
1918 iscsit_tmr_post_handler(cmd
, cmd
->conn
);
1919 case ISTATE_SEND_REJECT
: /* FALLTHRU */
1920 case ISTATE_SEND_TEXTRSP
: /* FALLTHRU */
1921 cmd
->i_state
= ISTATE_SENT_STATUS
;
1922 isert_completion_put(&isert_cmd
->tx_desc
, isert_cmd
,
1925 case ISTATE_SEND_LOGOUTRSP
:
1926 iscsit_logout_post_handler(cmd
, cmd
->conn
);
1929 isert_err("Unknown i_state %d\n", cmd
->i_state
);
1936 isert_response_completion(struct iser_tx_desc
*tx_desc
,
1937 struct isert_cmd
*isert_cmd
,
1938 struct isert_conn
*isert_conn
,
1939 struct ib_device
*ib_dev
)
1941 struct iscsi_cmd
*cmd
= isert_cmd
->iscsi_cmd
;
1943 if (cmd
->i_state
== ISTATE_SEND_TASKMGTRSP
||
1944 cmd
->i_state
== ISTATE_SEND_LOGOUTRSP
||
1945 cmd
->i_state
== ISTATE_SEND_REJECT
||
1946 cmd
->i_state
== ISTATE_SEND_TEXTRSP
) {
1947 isert_unmap_tx_desc(tx_desc
, ib_dev
);
1949 INIT_WORK(&isert_cmd
->comp_work
, isert_do_control_comp
);
1950 queue_work(isert_comp_wq
, &isert_cmd
->comp_work
);
1954 cmd
->i_state
= ISTATE_SENT_STATUS
;
1955 isert_completion_put(tx_desc
, isert_cmd
, ib_dev
, false);
1959 isert_send_completion(struct iser_tx_desc
*tx_desc
,
1960 struct isert_conn
*isert_conn
)
1962 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
1963 struct isert_cmd
*isert_cmd
= tx_desc
->isert_cmd
;
1964 struct isert_rdma_wr
*wr
;
1967 isert_unmap_tx_desc(tx_desc
, ib_dev
);
1970 wr
= &isert_cmd
->rdma_wr
;
1972 isert_dbg("Cmd %p iser_ib_op %d\n", isert_cmd
, wr
->iser_ib_op
);
1974 switch (wr
->iser_ib_op
) {
1976 isert_response_completion(tx_desc
, isert_cmd
,
1977 isert_conn
, ib_dev
);
1979 case ISER_IB_RDMA_WRITE
:
1980 isert_completion_rdma_write(tx_desc
, isert_cmd
);
1982 case ISER_IB_RDMA_READ
:
1983 isert_completion_rdma_read(tx_desc
, isert_cmd
);
1986 isert_err("Unknown wr->iser_ib_op: 0x%x\n", wr
->iser_ib_op
);
1993 * is_isert_tx_desc() - Indicate if the completion wr_id
1994 * is a TX descriptor or not.
1995 * @isert_conn: iser connection
1996 * @wr_id: completion WR identifier
1998 * Since we cannot rely on wc opcode in FLUSH errors
1999 * we must work around it by checking if the wr_id address
2000 * falls in the iser connection rx_descs buffer. If so
2001 * it is an RX descriptor, otherwize it is a TX.
2004 is_isert_tx_desc(struct isert_conn
*isert_conn
, void *wr_id
)
2006 void *start
= isert_conn
->conn_rx_descs
;
2007 int len
= ISERT_QP_MAX_RECV_DTOS
* sizeof(*isert_conn
->conn_rx_descs
);
2009 if (wr_id
>= start
&& wr_id
< start
+ len
)
2016 isert_cq_comp_err(struct isert_conn
*isert_conn
, struct ib_wc
*wc
)
2018 if (wc
->wr_id
== ISER_BEACON_WRID
) {
2019 isert_info("conn %p completing conn_wait_comp_err\n",
2021 complete(&isert_conn
->conn_wait_comp_err
);
2022 } else if (is_isert_tx_desc(isert_conn
, (void *)(uintptr_t)wc
->wr_id
)) {
2023 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
2024 struct isert_cmd
*isert_cmd
;
2025 struct iser_tx_desc
*desc
;
2027 desc
= (struct iser_tx_desc
*)(uintptr_t)wc
->wr_id
;
2028 isert_cmd
= desc
->isert_cmd
;
2030 isert_unmap_tx_desc(desc
, ib_dev
);
2032 isert_completion_put(desc
, isert_cmd
, ib_dev
, true);
2034 isert_conn
->post_recv_buf_count
--;
2035 if (!isert_conn
->post_recv_buf_count
)
2036 iscsit_cause_connection_reinstatement(isert_conn
->conn
, 0);
2041 isert_handle_wc(struct ib_wc
*wc
)
2043 struct isert_conn
*isert_conn
;
2044 struct iser_tx_desc
*tx_desc
;
2045 struct iser_rx_desc
*rx_desc
;
2047 isert_conn
= wc
->qp
->qp_context
;
2048 if (likely(wc
->status
== IB_WC_SUCCESS
)) {
2049 if (wc
->opcode
== IB_WC_RECV
) {
2050 rx_desc
= (struct iser_rx_desc
*)(uintptr_t)wc
->wr_id
;
2051 isert_rx_completion(rx_desc
, isert_conn
, wc
->byte_len
);
2053 tx_desc
= (struct iser_tx_desc
*)(uintptr_t)wc
->wr_id
;
2054 isert_send_completion(tx_desc
, isert_conn
);
2057 if (wc
->status
!= IB_WC_WR_FLUSH_ERR
)
2058 isert_err("wr id %llx status %d vend_err %x\n",
2059 wc
->wr_id
, wc
->status
, wc
->vendor_err
);
2061 isert_dbg("flush error: wr id %llx\n", wc
->wr_id
);
2063 if (wc
->wr_id
!= ISER_FASTREG_LI_WRID
)
2064 isert_cq_comp_err(isert_conn
, wc
);
2069 isert_cq_work(struct work_struct
*work
)
2071 enum { isert_poll_budget
= 65536 };
2072 struct isert_comp
*comp
= container_of(work
, struct isert_comp
,
2074 struct ib_wc
*const wcs
= comp
->wcs
;
2075 int i
, n
, completed
= 0;
2077 while ((n
= ib_poll_cq(comp
->cq
, ARRAY_SIZE(comp
->wcs
), wcs
)) > 0) {
2078 for (i
= 0; i
< n
; i
++)
2079 isert_handle_wc(&wcs
[i
]);
2082 if (completed
>= isert_poll_budget
)
2086 ib_req_notify_cq(comp
->cq
, IB_CQ_NEXT_COMP
);
2090 isert_cq_callback(struct ib_cq
*cq
, void *context
)
2092 struct isert_comp
*comp
= context
;
2094 queue_work(isert_comp_wq
, &comp
->work
);
2098 isert_post_response(struct isert_conn
*isert_conn
, struct isert_cmd
*isert_cmd
)
2100 struct ib_send_wr
*wr_failed
;
2103 ret
= ib_post_send(isert_conn
->conn_qp
, &isert_cmd
->tx_desc
.send_wr
,
2106 isert_err("ib_post_send failed with %d\n", ret
);
2113 isert_put_response(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
)
2115 struct isert_cmd
*isert_cmd
= iscsit_priv_cmd(cmd
);
2116 struct isert_conn
*isert_conn
= conn
->context
;
2117 struct ib_send_wr
*send_wr
= &isert_cmd
->tx_desc
.send_wr
;
2118 struct iscsi_scsi_rsp
*hdr
= (struct iscsi_scsi_rsp
*)
2119 &isert_cmd
->tx_desc
.iscsi_header
;
2121 isert_create_send_desc(isert_conn
, isert_cmd
, &isert_cmd
->tx_desc
);
2122 iscsit_build_rsp_pdu(cmd
, conn
, true, hdr
);
2123 isert_init_tx_hdrs(isert_conn
, &isert_cmd
->tx_desc
);
2125 * Attach SENSE DATA payload to iSCSI Response PDU
2127 if (cmd
->se_cmd
.sense_buffer
&&
2128 ((cmd
->se_cmd
.se_cmd_flags
& SCF_TRANSPORT_TASK_SENSE
) ||
2129 (cmd
->se_cmd
.se_cmd_flags
& SCF_EMULATED_TASK_SENSE
))) {
2130 struct isert_device
*device
= isert_conn
->conn_device
;
2131 struct ib_device
*ib_dev
= device
->ib_device
;
2132 struct ib_sge
*tx_dsg
= &isert_cmd
->tx_desc
.tx_sg
[1];
2133 u32 padding
, pdu_len
;
2135 put_unaligned_be16(cmd
->se_cmd
.scsi_sense_length
,
2137 cmd
->se_cmd
.scsi_sense_length
+= sizeof(__be16
);
2139 padding
= -(cmd
->se_cmd
.scsi_sense_length
) & 3;
2140 hton24(hdr
->dlength
, (u32
)cmd
->se_cmd
.scsi_sense_length
);
2141 pdu_len
= cmd
->se_cmd
.scsi_sense_length
+ padding
;
2143 isert_cmd
->pdu_buf_dma
= ib_dma_map_single(ib_dev
,
2144 (void *)cmd
->sense_buffer
, pdu_len
,
2147 isert_cmd
->pdu_buf_len
= pdu_len
;
2148 tx_dsg
->addr
= isert_cmd
->pdu_buf_dma
;
2149 tx_dsg
->length
= pdu_len
;
2150 tx_dsg
->lkey
= device
->mr
->lkey
;
2151 isert_cmd
->tx_desc
.num_sge
= 2;
2154 isert_init_send_wr(isert_conn
, isert_cmd
, send_wr
);
2156 isert_dbg("Posting SCSI Response\n");
2158 return isert_post_response(isert_conn
, isert_cmd
);
2162 isert_aborted_task(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
)
2164 struct isert_cmd
*isert_cmd
= iscsit_priv_cmd(cmd
);
2165 struct isert_conn
*isert_conn
= conn
->context
;
2166 struct isert_device
*device
= isert_conn
->conn_device
;
2168 spin_lock_bh(&conn
->cmd_lock
);
2169 if (!list_empty(&cmd
->i_conn_node
))
2170 list_del_init(&cmd
->i_conn_node
);
2171 spin_unlock_bh(&conn
->cmd_lock
);
2173 if (cmd
->data_direction
== DMA_TO_DEVICE
)
2174 iscsit_stop_dataout_timer(cmd
);
2176 device
->unreg_rdma_mem(isert_cmd
, isert_conn
);
2179 static enum target_prot_op
2180 isert_get_sup_prot_ops(struct iscsi_conn
*conn
)
2182 struct isert_conn
*isert_conn
= conn
->context
;
2183 struct isert_device
*device
= isert_conn
->conn_device
;
2185 if (conn
->tpg
->tpg_attrib
.t10_pi
) {
2186 if (device
->pi_capable
) {
2187 isert_info("conn %p PI offload enabled\n", isert_conn
);
2188 isert_conn
->pi_support
= true;
2189 return TARGET_PROT_ALL
;
2193 isert_info("conn %p PI offload disabled\n", isert_conn
);
2194 isert_conn
->pi_support
= false;
2196 return TARGET_PROT_NORMAL
;
2200 isert_put_nopin(struct iscsi_cmd
*cmd
, struct iscsi_conn
*conn
,
2201 bool nopout_response
)
2203 struct isert_cmd
*isert_cmd
= iscsit_priv_cmd(cmd
);
2204 struct isert_conn
*isert_conn
= conn
->context
;
2205 struct ib_send_wr
*send_wr
= &isert_cmd
->tx_desc
.send_wr
;
2207 isert_create_send_desc(isert_conn
, isert_cmd
, &isert_cmd
->tx_desc
);
2208 iscsit_build_nopin_rsp(cmd
, conn
, (struct iscsi_nopin
*)
2209 &isert_cmd
->tx_desc
.iscsi_header
,
2211 isert_init_tx_hdrs(isert_conn
, &isert_cmd
->tx_desc
);
2212 isert_init_send_wr(isert_conn
, isert_cmd
, send_wr
);
2214 isert_dbg("conn %p Posting NOPIN Response\n", isert_conn
);
2216 return isert_post_response(isert_conn
, isert_cmd
);
2220 isert_put_logout_rsp(struct iscsi_cmd
*cmd
, struct iscsi_conn
*conn
)
2222 struct isert_cmd
*isert_cmd
= iscsit_priv_cmd(cmd
);
2223 struct isert_conn
*isert_conn
= conn
->context
;
2224 struct ib_send_wr
*send_wr
= &isert_cmd
->tx_desc
.send_wr
;
2226 isert_create_send_desc(isert_conn
, isert_cmd
, &isert_cmd
->tx_desc
);
2227 iscsit_build_logout_rsp(cmd
, conn
, (struct iscsi_logout_rsp
*)
2228 &isert_cmd
->tx_desc
.iscsi_header
);
2229 isert_init_tx_hdrs(isert_conn
, &isert_cmd
->tx_desc
);
2230 isert_init_send_wr(isert_conn
, isert_cmd
, send_wr
);
2232 isert_dbg("conn %p Posting Logout Response\n", isert_conn
);
2234 return isert_post_response(isert_conn
, isert_cmd
);
2238 isert_put_tm_rsp(struct iscsi_cmd
*cmd
, struct iscsi_conn
*conn
)
2240 struct isert_cmd
*isert_cmd
= iscsit_priv_cmd(cmd
);
2241 struct isert_conn
*isert_conn
= conn
->context
;
2242 struct ib_send_wr
*send_wr
= &isert_cmd
->tx_desc
.send_wr
;
2244 isert_create_send_desc(isert_conn
, isert_cmd
, &isert_cmd
->tx_desc
);
2245 iscsit_build_task_mgt_rsp(cmd
, conn
, (struct iscsi_tm_rsp
*)
2246 &isert_cmd
->tx_desc
.iscsi_header
);
2247 isert_init_tx_hdrs(isert_conn
, &isert_cmd
->tx_desc
);
2248 isert_init_send_wr(isert_conn
, isert_cmd
, send_wr
);
2250 isert_dbg("conn %p Posting Task Management Response\n", isert_conn
);
2252 return isert_post_response(isert_conn
, isert_cmd
);
2256 isert_put_reject(struct iscsi_cmd
*cmd
, struct iscsi_conn
*conn
)
2258 struct isert_cmd
*isert_cmd
= iscsit_priv_cmd(cmd
);
2259 struct isert_conn
*isert_conn
= conn
->context
;
2260 struct ib_send_wr
*send_wr
= &isert_cmd
->tx_desc
.send_wr
;
2261 struct isert_device
*device
= isert_conn
->conn_device
;
2262 struct ib_device
*ib_dev
= device
->ib_device
;
2263 struct ib_sge
*tx_dsg
= &isert_cmd
->tx_desc
.tx_sg
[1];
2264 struct iscsi_reject
*hdr
=
2265 (struct iscsi_reject
*)&isert_cmd
->tx_desc
.iscsi_header
;
2267 isert_create_send_desc(isert_conn
, isert_cmd
, &isert_cmd
->tx_desc
);
2268 iscsit_build_reject(cmd
, conn
, hdr
);
2269 isert_init_tx_hdrs(isert_conn
, &isert_cmd
->tx_desc
);
2271 hton24(hdr
->dlength
, ISCSI_HDR_LEN
);
2272 isert_cmd
->pdu_buf_dma
= ib_dma_map_single(ib_dev
,
2273 (void *)cmd
->buf_ptr
, ISCSI_HDR_LEN
,
2275 isert_cmd
->pdu_buf_len
= ISCSI_HDR_LEN
;
2276 tx_dsg
->addr
= isert_cmd
->pdu_buf_dma
;
2277 tx_dsg
->length
= ISCSI_HDR_LEN
;
2278 tx_dsg
->lkey
= device
->mr
->lkey
;
2279 isert_cmd
->tx_desc
.num_sge
= 2;
2281 isert_init_send_wr(isert_conn
, isert_cmd
, send_wr
);
2283 isert_dbg("conn %p Posting Reject\n", isert_conn
);
2285 return isert_post_response(isert_conn
, isert_cmd
);
2289 isert_put_text_rsp(struct iscsi_cmd
*cmd
, struct iscsi_conn
*conn
)
2291 struct isert_cmd
*isert_cmd
= iscsit_priv_cmd(cmd
);
2292 struct isert_conn
*isert_conn
= conn
->context
;
2293 struct ib_send_wr
*send_wr
= &isert_cmd
->tx_desc
.send_wr
;
2294 struct iscsi_text_rsp
*hdr
=
2295 (struct iscsi_text_rsp
*)&isert_cmd
->tx_desc
.iscsi_header
;
2299 isert_create_send_desc(isert_conn
, isert_cmd
, &isert_cmd
->tx_desc
);
2300 rc
= iscsit_build_text_rsp(cmd
, conn
, hdr
, ISCSI_INFINIBAND
);
2305 isert_init_tx_hdrs(isert_conn
, &isert_cmd
->tx_desc
);
2308 struct isert_device
*device
= isert_conn
->conn_device
;
2309 struct ib_device
*ib_dev
= device
->ib_device
;
2310 struct ib_sge
*tx_dsg
= &isert_cmd
->tx_desc
.tx_sg
[1];
2311 void *txt_rsp_buf
= cmd
->buf_ptr
;
2313 isert_cmd
->pdu_buf_dma
= ib_dma_map_single(ib_dev
,
2314 txt_rsp_buf
, txt_rsp_len
, DMA_TO_DEVICE
);
2316 isert_cmd
->pdu_buf_len
= txt_rsp_len
;
2317 tx_dsg
->addr
= isert_cmd
->pdu_buf_dma
;
2318 tx_dsg
->length
= txt_rsp_len
;
2319 tx_dsg
->lkey
= device
->mr
->lkey
;
2320 isert_cmd
->tx_desc
.num_sge
= 2;
2322 isert_init_send_wr(isert_conn
, isert_cmd
, send_wr
);
2324 isert_dbg("conn %p Text Response\n", isert_conn
);
2326 return isert_post_response(isert_conn
, isert_cmd
);
2330 isert_build_rdma_wr(struct isert_conn
*isert_conn
, struct isert_cmd
*isert_cmd
,
2331 struct ib_sge
*ib_sge
, struct ib_send_wr
*send_wr
,
2332 u32 data_left
, u32 offset
)
2334 struct iscsi_cmd
*cmd
= isert_cmd
->iscsi_cmd
;
2335 struct scatterlist
*sg_start
, *tmp_sg
;
2336 struct isert_device
*device
= isert_conn
->conn_device
;
2337 struct ib_device
*ib_dev
= device
->ib_device
;
2338 u32 sg_off
, page_off
;
2339 int i
= 0, sg_nents
;
2341 sg_off
= offset
/ PAGE_SIZE
;
2342 sg_start
= &cmd
->se_cmd
.t_data_sg
[sg_off
];
2343 sg_nents
= min(cmd
->se_cmd
.t_data_nents
- sg_off
, isert_conn
->max_sge
);
2344 page_off
= offset
% PAGE_SIZE
;
2346 send_wr
->sg_list
= ib_sge
;
2347 send_wr
->num_sge
= sg_nents
;
2348 send_wr
->wr_id
= (uintptr_t)&isert_cmd
->tx_desc
;
2350 * Perform mapping of TCM scatterlist memory ib_sge dma_addr.
2352 for_each_sg(sg_start
, tmp_sg
, sg_nents
, i
) {
2353 isert_dbg("RDMA from SGL dma_addr: 0x%llx dma_len: %u, "
2355 (unsigned long long)tmp_sg
->dma_address
,
2356 tmp_sg
->length
, page_off
);
2358 ib_sge
->addr
= ib_sg_dma_address(ib_dev
, tmp_sg
) + page_off
;
2359 ib_sge
->length
= min_t(u32
, data_left
,
2360 ib_sg_dma_len(ib_dev
, tmp_sg
) - page_off
);
2361 ib_sge
->lkey
= device
->mr
->lkey
;
2363 isert_dbg("RDMA ib_sge: addr: 0x%llx length: %u lkey: %x\n",
2364 ib_sge
->addr
, ib_sge
->length
, ib_sge
->lkey
);
2366 data_left
-= ib_sge
->length
;
2368 isert_dbg("Incrementing ib_sge pointer to %p\n", ib_sge
);
2371 isert_dbg("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n",
2372 send_wr
->sg_list
, send_wr
->num_sge
);
2378 isert_map_rdma(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
,
2379 struct isert_rdma_wr
*wr
)
2381 struct se_cmd
*se_cmd
= &cmd
->se_cmd
;
2382 struct isert_cmd
*isert_cmd
= iscsit_priv_cmd(cmd
);
2383 struct isert_conn
*isert_conn
= conn
->context
;
2384 struct isert_data_buf
*data
= &wr
->data
;
2385 struct ib_send_wr
*send_wr
;
2386 struct ib_sge
*ib_sge
;
2387 u32 offset
, data_len
, data_left
, rdma_write_max
, va_offset
= 0;
2388 int ret
= 0, i
, ib_sge_cnt
;
2390 isert_cmd
->tx_desc
.isert_cmd
= isert_cmd
;
2392 offset
= wr
->iser_ib_op
== ISER_IB_RDMA_READ
? cmd
->write_data_done
: 0;
2393 ret
= isert_map_data_buf(isert_conn
, isert_cmd
, se_cmd
->t_data_sg
,
2394 se_cmd
->t_data_nents
, se_cmd
->data_length
,
2395 offset
, wr
->iser_ib_op
, &wr
->data
);
2399 data_left
= data
->len
;
2400 offset
= data
->offset
;
2402 ib_sge
= kzalloc(sizeof(struct ib_sge
) * data
->nents
, GFP_KERNEL
);
2404 isert_warn("Unable to allocate ib_sge\n");
2408 wr
->ib_sge
= ib_sge
;
2410 wr
->send_wr_num
= DIV_ROUND_UP(data
->nents
, isert_conn
->max_sge
);
2411 wr
->send_wr
= kzalloc(sizeof(struct ib_send_wr
) * wr
->send_wr_num
,
2414 isert_dbg("Unable to allocate wr->send_wr\n");
2419 wr
->isert_cmd
= isert_cmd
;
2420 rdma_write_max
= isert_conn
->max_sge
* PAGE_SIZE
;
2422 for (i
= 0; i
< wr
->send_wr_num
; i
++) {
2423 send_wr
= &isert_cmd
->rdma_wr
.send_wr
[i
];
2424 data_len
= min(data_left
, rdma_write_max
);
2426 send_wr
->send_flags
= 0;
2427 if (wr
->iser_ib_op
== ISER_IB_RDMA_WRITE
) {
2428 send_wr
->opcode
= IB_WR_RDMA_WRITE
;
2429 send_wr
->wr
.rdma
.remote_addr
= isert_cmd
->read_va
+ offset
;
2430 send_wr
->wr
.rdma
.rkey
= isert_cmd
->read_stag
;
2431 if (i
+ 1 == wr
->send_wr_num
)
2432 send_wr
->next
= &isert_cmd
->tx_desc
.send_wr
;
2434 send_wr
->next
= &wr
->send_wr
[i
+ 1];
2436 send_wr
->opcode
= IB_WR_RDMA_READ
;
2437 send_wr
->wr
.rdma
.remote_addr
= isert_cmd
->write_va
+ va_offset
;
2438 send_wr
->wr
.rdma
.rkey
= isert_cmd
->write_stag
;
2439 if (i
+ 1 == wr
->send_wr_num
)
2440 send_wr
->send_flags
= IB_SEND_SIGNALED
;
2442 send_wr
->next
= &wr
->send_wr
[i
+ 1];
2445 ib_sge_cnt
= isert_build_rdma_wr(isert_conn
, isert_cmd
, ib_sge
,
2446 send_wr
, data_len
, offset
);
2447 ib_sge
+= ib_sge_cnt
;
2450 va_offset
+= data_len
;
2451 data_left
-= data_len
;
2456 isert_unmap_data_buf(isert_conn
, data
);
2462 isert_map_fr_pagelist(struct ib_device
*ib_dev
,
2463 struct scatterlist
*sg_start
, int sg_nents
, u64
*fr_pl
)
2465 u64 start_addr
, end_addr
, page
, chunk_start
= 0;
2466 struct scatterlist
*tmp_sg
;
2467 int i
= 0, new_chunk
, last_ent
, n_pages
;
2471 last_ent
= sg_nents
- 1;
2472 for_each_sg(sg_start
, tmp_sg
, sg_nents
, i
) {
2473 start_addr
= ib_sg_dma_address(ib_dev
, tmp_sg
);
2475 chunk_start
= start_addr
;
2476 end_addr
= start_addr
+ ib_sg_dma_len(ib_dev
, tmp_sg
);
2478 isert_dbg("SGL[%d] dma_addr: 0x%llx len: %u\n",
2479 i
, (unsigned long long)tmp_sg
->dma_address
,
2482 if ((end_addr
& ~PAGE_MASK
) && i
< last_ent
) {
2488 page
= chunk_start
& PAGE_MASK
;
2490 fr_pl
[n_pages
++] = page
;
2491 isert_dbg("Mapped page_list[%d] page_addr: 0x%llx\n",
2494 } while (page
< end_addr
);
2501 isert_inv_rkey(struct ib_send_wr
*inv_wr
, struct ib_mr
*mr
)
2505 memset(inv_wr
, 0, sizeof(*inv_wr
));
2506 inv_wr
->wr_id
= ISER_FASTREG_LI_WRID
;
2507 inv_wr
->opcode
= IB_WR_LOCAL_INV
;
2508 inv_wr
->ex
.invalidate_rkey
= mr
->rkey
;
2511 rkey
= ib_inc_rkey(mr
->rkey
);
2512 ib_update_fast_reg_key(mr
, rkey
);
2516 isert_fast_reg_mr(struct isert_conn
*isert_conn
,
2517 struct fast_reg_descriptor
*fr_desc
,
2518 struct isert_data_buf
*mem
,
2519 enum isert_indicator ind
,
2522 struct isert_device
*device
= isert_conn
->conn_device
;
2523 struct ib_device
*ib_dev
= device
->ib_device
;
2525 struct ib_fast_reg_page_list
*frpl
;
2526 struct ib_send_wr fr_wr
, inv_wr
;
2527 struct ib_send_wr
*bad_wr
, *wr
= NULL
;
2528 int ret
, pagelist_len
;
2531 if (mem
->dma_nents
== 1) {
2532 sge
->lkey
= device
->mr
->lkey
;
2533 sge
->addr
= ib_sg_dma_address(ib_dev
, &mem
->sg
[0]);
2534 sge
->length
= ib_sg_dma_len(ib_dev
, &mem
->sg
[0]);
2535 isert_dbg("sge: addr: 0x%llx length: %u lkey: %x\n",
2536 sge
->addr
, sge
->length
, sge
->lkey
);
2540 if (ind
== ISERT_DATA_KEY_VALID
) {
2541 /* Registering data buffer */
2542 mr
= fr_desc
->data_mr
;
2543 frpl
= fr_desc
->data_frpl
;
2545 /* Registering protection buffer */
2546 mr
= fr_desc
->pi_ctx
->prot_mr
;
2547 frpl
= fr_desc
->pi_ctx
->prot_frpl
;
2550 page_off
= mem
->offset
% PAGE_SIZE
;
2552 isert_dbg("Use fr_desc %p sg_nents %d offset %u\n",
2553 fr_desc
, mem
->nents
, mem
->offset
);
2555 pagelist_len
= isert_map_fr_pagelist(ib_dev
, mem
->sg
, mem
->nents
,
2556 &frpl
->page_list
[0]);
2558 if (!(fr_desc
->ind
& ind
)) {
2559 isert_inv_rkey(&inv_wr
, mr
);
2563 /* Prepare FASTREG WR */
2564 memset(&fr_wr
, 0, sizeof(fr_wr
));
2565 fr_wr
.wr_id
= ISER_FASTREG_LI_WRID
;
2566 fr_wr
.opcode
= IB_WR_FAST_REG_MR
;
2567 fr_wr
.wr
.fast_reg
.iova_start
= frpl
->page_list
[0] + page_off
;
2568 fr_wr
.wr
.fast_reg
.page_list
= frpl
;
2569 fr_wr
.wr
.fast_reg
.page_list_len
= pagelist_len
;
2570 fr_wr
.wr
.fast_reg
.page_shift
= PAGE_SHIFT
;
2571 fr_wr
.wr
.fast_reg
.length
= mem
->len
;
2572 fr_wr
.wr
.fast_reg
.rkey
= mr
->rkey
;
2573 fr_wr
.wr
.fast_reg
.access_flags
= IB_ACCESS_LOCAL_WRITE
;
2580 ret
= ib_post_send(isert_conn
->conn_qp
, wr
, &bad_wr
);
2582 isert_err("fast registration failed, ret:%d\n", ret
);
2585 fr_desc
->ind
&= ~ind
;
2587 sge
->lkey
= mr
->lkey
;
2588 sge
->addr
= frpl
->page_list
[0] + page_off
;
2589 sge
->length
= mem
->len
;
2591 isert_dbg("sge: addr: 0x%llx length: %u lkey: %x\n",
2592 sge
->addr
, sge
->length
, sge
->lkey
);
2598 isert_set_dif_domain(struct se_cmd
*se_cmd
, struct ib_sig_attrs
*sig_attrs
,
2599 struct ib_sig_domain
*domain
)
2601 domain
->sig_type
= IB_SIG_TYPE_T10_DIF
;
2602 domain
->sig
.dif
.bg_type
= IB_T10DIF_CRC
;
2603 domain
->sig
.dif
.pi_interval
= se_cmd
->se_dev
->dev_attrib
.block_size
;
2604 domain
->sig
.dif
.ref_tag
= se_cmd
->reftag_seed
;
2606 * At the moment we hard code those, but if in the future
2607 * the target core would like to use it, we will take it
2610 domain
->sig
.dif
.apptag_check_mask
= 0xffff;
2611 domain
->sig
.dif
.app_escape
= true;
2612 domain
->sig
.dif
.ref_escape
= true;
2613 if (se_cmd
->prot_type
== TARGET_DIF_TYPE1_PROT
||
2614 se_cmd
->prot_type
== TARGET_DIF_TYPE2_PROT
)
2615 domain
->sig
.dif
.ref_remap
= true;
2619 isert_set_sig_attrs(struct se_cmd
*se_cmd
, struct ib_sig_attrs
*sig_attrs
)
2621 switch (se_cmd
->prot_op
) {
2622 case TARGET_PROT_DIN_INSERT
:
2623 case TARGET_PROT_DOUT_STRIP
:
2624 sig_attrs
->mem
.sig_type
= IB_SIG_TYPE_NONE
;
2625 isert_set_dif_domain(se_cmd
, sig_attrs
, &sig_attrs
->wire
);
2627 case TARGET_PROT_DOUT_INSERT
:
2628 case TARGET_PROT_DIN_STRIP
:
2629 sig_attrs
->wire
.sig_type
= IB_SIG_TYPE_NONE
;
2630 isert_set_dif_domain(se_cmd
, sig_attrs
, &sig_attrs
->mem
);
2632 case TARGET_PROT_DIN_PASS
:
2633 case TARGET_PROT_DOUT_PASS
:
2634 isert_set_dif_domain(se_cmd
, sig_attrs
, &sig_attrs
->wire
);
2635 isert_set_dif_domain(se_cmd
, sig_attrs
, &sig_attrs
->mem
);
2638 isert_err("Unsupported PI operation %d\n", se_cmd
->prot_op
);
2646 isert_set_prot_checks(u8 prot_checks
)
2648 return (prot_checks
& TARGET_DIF_CHECK_GUARD
? 0xc0 : 0) |
2649 (prot_checks
& TARGET_DIF_CHECK_REFTAG
? 0x30 : 0) |
2650 (prot_checks
& TARGET_DIF_CHECK_REFTAG
? 0x0f : 0);
2654 isert_reg_sig_mr(struct isert_conn
*isert_conn
,
2655 struct se_cmd
*se_cmd
,
2656 struct isert_rdma_wr
*rdma_wr
,
2657 struct fast_reg_descriptor
*fr_desc
)
2659 struct ib_send_wr sig_wr
, inv_wr
;
2660 struct ib_send_wr
*bad_wr
, *wr
= NULL
;
2661 struct pi_context
*pi_ctx
= fr_desc
->pi_ctx
;
2662 struct ib_sig_attrs sig_attrs
;
2665 memset(&sig_attrs
, 0, sizeof(sig_attrs
));
2666 ret
= isert_set_sig_attrs(se_cmd
, &sig_attrs
);
2670 sig_attrs
.check_mask
= isert_set_prot_checks(se_cmd
->prot_checks
);
2672 if (!(fr_desc
->ind
& ISERT_SIG_KEY_VALID
)) {
2673 isert_inv_rkey(&inv_wr
, pi_ctx
->sig_mr
);
2677 memset(&sig_wr
, 0, sizeof(sig_wr
));
2678 sig_wr
.opcode
= IB_WR_REG_SIG_MR
;
2679 sig_wr
.wr_id
= ISER_FASTREG_LI_WRID
;
2680 sig_wr
.sg_list
= &rdma_wr
->ib_sg
[DATA
];
2682 sig_wr
.wr
.sig_handover
.access_flags
= IB_ACCESS_LOCAL_WRITE
;
2683 sig_wr
.wr
.sig_handover
.sig_attrs
= &sig_attrs
;
2684 sig_wr
.wr
.sig_handover
.sig_mr
= pi_ctx
->sig_mr
;
2685 if (se_cmd
->t_prot_sg
)
2686 sig_wr
.wr
.sig_handover
.prot
= &rdma_wr
->ib_sg
[PROT
];
2693 ret
= ib_post_send(isert_conn
->conn_qp
, wr
, &bad_wr
);
2695 isert_err("fast registration failed, ret:%d\n", ret
);
2698 fr_desc
->ind
&= ~ISERT_SIG_KEY_VALID
;
2700 rdma_wr
->ib_sg
[SIG
].lkey
= pi_ctx
->sig_mr
->lkey
;
2701 rdma_wr
->ib_sg
[SIG
].addr
= 0;
2702 rdma_wr
->ib_sg
[SIG
].length
= se_cmd
->data_length
;
2703 if (se_cmd
->prot_op
!= TARGET_PROT_DIN_STRIP
&&
2704 se_cmd
->prot_op
!= TARGET_PROT_DOUT_INSERT
)
2706 * We have protection guards on the wire
2707 * so we need to set a larget transfer
2709 rdma_wr
->ib_sg
[SIG
].length
+= se_cmd
->prot_length
;
2711 isert_dbg("sig_sge: addr: 0x%llx length: %u lkey: %x\n",
2712 rdma_wr
->ib_sg
[SIG
].addr
, rdma_wr
->ib_sg
[SIG
].length
,
2713 rdma_wr
->ib_sg
[SIG
].lkey
);
2719 isert_handle_prot_cmd(struct isert_conn
*isert_conn
,
2720 struct isert_cmd
*isert_cmd
,
2721 struct isert_rdma_wr
*wr
)
2723 struct isert_device
*device
= isert_conn
->conn_device
;
2724 struct se_cmd
*se_cmd
= &isert_cmd
->iscsi_cmd
->se_cmd
;
2727 if (!wr
->fr_desc
->pi_ctx
) {
2728 ret
= isert_create_pi_ctx(wr
->fr_desc
,
2732 isert_err("conn %p failed to allocate pi_ctx\n",
2738 if (se_cmd
->t_prot_sg
) {
2739 ret
= isert_map_data_buf(isert_conn
, isert_cmd
,
2741 se_cmd
->t_prot_nents
,
2742 se_cmd
->prot_length
,
2743 0, wr
->iser_ib_op
, &wr
->prot
);
2745 isert_err("conn %p failed to map protection buffer\n",
2750 memset(&wr
->ib_sg
[PROT
], 0, sizeof(wr
->ib_sg
[PROT
]));
2751 ret
= isert_fast_reg_mr(isert_conn
, wr
->fr_desc
, &wr
->prot
,
2752 ISERT_PROT_KEY_VALID
, &wr
->ib_sg
[PROT
]);
2754 isert_err("conn %p failed to fast reg mr\n",
2756 goto unmap_prot_cmd
;
2760 ret
= isert_reg_sig_mr(isert_conn
, se_cmd
, wr
, wr
->fr_desc
);
2762 isert_err("conn %p failed to fast reg mr\n",
2764 goto unmap_prot_cmd
;
2766 wr
->fr_desc
->ind
|= ISERT_PROTECTED
;
2771 if (se_cmd
->t_prot_sg
)
2772 isert_unmap_data_buf(isert_conn
, &wr
->prot
);
2778 isert_reg_rdma(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
,
2779 struct isert_rdma_wr
*wr
)
2781 struct se_cmd
*se_cmd
= &cmd
->se_cmd
;
2782 struct isert_cmd
*isert_cmd
= iscsit_priv_cmd(cmd
);
2783 struct isert_conn
*isert_conn
= conn
->context
;
2784 struct fast_reg_descriptor
*fr_desc
= NULL
;
2785 struct ib_send_wr
*send_wr
;
2786 struct ib_sge
*ib_sg
;
2789 unsigned long flags
;
2791 isert_cmd
->tx_desc
.isert_cmd
= isert_cmd
;
2793 offset
= wr
->iser_ib_op
== ISER_IB_RDMA_READ
? cmd
->write_data_done
: 0;
2794 ret
= isert_map_data_buf(isert_conn
, isert_cmd
, se_cmd
->t_data_sg
,
2795 se_cmd
->t_data_nents
, se_cmd
->data_length
,
2796 offset
, wr
->iser_ib_op
, &wr
->data
);
2800 if (wr
->data
.dma_nents
!= 1 || isert_prot_cmd(isert_conn
, se_cmd
)) {
2801 spin_lock_irqsave(&isert_conn
->conn_lock
, flags
);
2802 fr_desc
= list_first_entry(&isert_conn
->conn_fr_pool
,
2803 struct fast_reg_descriptor
, list
);
2804 list_del(&fr_desc
->list
);
2805 spin_unlock_irqrestore(&isert_conn
->conn_lock
, flags
);
2806 wr
->fr_desc
= fr_desc
;
2809 ret
= isert_fast_reg_mr(isert_conn
, fr_desc
, &wr
->data
,
2810 ISERT_DATA_KEY_VALID
, &wr
->ib_sg
[DATA
]);
2814 if (isert_prot_cmd(isert_conn
, se_cmd
)) {
2815 ret
= isert_handle_prot_cmd(isert_conn
, isert_cmd
, wr
);
2819 ib_sg
= &wr
->ib_sg
[SIG
];
2821 ib_sg
= &wr
->ib_sg
[DATA
];
2824 memcpy(&wr
->s_ib_sge
, ib_sg
, sizeof(*ib_sg
));
2825 wr
->ib_sge
= &wr
->s_ib_sge
;
2826 wr
->send_wr_num
= 1;
2827 memset(&wr
->s_send_wr
, 0, sizeof(*send_wr
));
2828 wr
->send_wr
= &wr
->s_send_wr
;
2829 wr
->isert_cmd
= isert_cmd
;
2831 send_wr
= &isert_cmd
->rdma_wr
.s_send_wr
;
2832 send_wr
->sg_list
= &wr
->s_ib_sge
;
2833 send_wr
->num_sge
= 1;
2834 send_wr
->wr_id
= (uintptr_t)&isert_cmd
->tx_desc
;
2835 if (wr
->iser_ib_op
== ISER_IB_RDMA_WRITE
) {
2836 send_wr
->opcode
= IB_WR_RDMA_WRITE
;
2837 send_wr
->wr
.rdma
.remote_addr
= isert_cmd
->read_va
;
2838 send_wr
->wr
.rdma
.rkey
= isert_cmd
->read_stag
;
2839 send_wr
->send_flags
= !isert_prot_cmd(isert_conn
, se_cmd
) ?
2840 0 : IB_SEND_SIGNALED
;
2842 send_wr
->opcode
= IB_WR_RDMA_READ
;
2843 send_wr
->wr
.rdma
.remote_addr
= isert_cmd
->write_va
;
2844 send_wr
->wr
.rdma
.rkey
= isert_cmd
->write_stag
;
2845 send_wr
->send_flags
= IB_SEND_SIGNALED
;
2852 spin_lock_irqsave(&isert_conn
->conn_lock
, flags
);
2853 list_add_tail(&fr_desc
->list
, &isert_conn
->conn_fr_pool
);
2854 spin_unlock_irqrestore(&isert_conn
->conn_lock
, flags
);
2856 isert_unmap_data_buf(isert_conn
, &wr
->data
);
2862 isert_put_datain(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
)
2864 struct se_cmd
*se_cmd
= &cmd
->se_cmd
;
2865 struct isert_cmd
*isert_cmd
= iscsit_priv_cmd(cmd
);
2866 struct isert_rdma_wr
*wr
= &isert_cmd
->rdma_wr
;
2867 struct isert_conn
*isert_conn
= conn
->context
;
2868 struct isert_device
*device
= isert_conn
->conn_device
;
2869 struct ib_send_wr
*wr_failed
;
2872 isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n",
2873 isert_cmd
, se_cmd
->data_length
);
2875 wr
->iser_ib_op
= ISER_IB_RDMA_WRITE
;
2876 rc
= device
->reg_rdma_mem(conn
, cmd
, wr
);
2878 isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd
);
2882 if (!isert_prot_cmd(isert_conn
, se_cmd
)) {
2884 * Build isert_conn->tx_desc for iSCSI response PDU and attach
2886 isert_create_send_desc(isert_conn
, isert_cmd
,
2887 &isert_cmd
->tx_desc
);
2888 iscsit_build_rsp_pdu(cmd
, conn
, true, (struct iscsi_scsi_rsp
*)
2889 &isert_cmd
->tx_desc
.iscsi_header
);
2890 isert_init_tx_hdrs(isert_conn
, &isert_cmd
->tx_desc
);
2891 isert_init_send_wr(isert_conn
, isert_cmd
,
2892 &isert_cmd
->tx_desc
.send_wr
);
2893 isert_cmd
->rdma_wr
.s_send_wr
.next
= &isert_cmd
->tx_desc
.send_wr
;
2894 wr
->send_wr_num
+= 1;
2897 rc
= ib_post_send(isert_conn
->conn_qp
, wr
->send_wr
, &wr_failed
);
2899 isert_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n");
2901 if (!isert_prot_cmd(isert_conn
, se_cmd
))
2902 isert_dbg("Cmd: %p posted RDMA_WRITE + Response for iSER Data "
2903 "READ\n", isert_cmd
);
2905 isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ\n",
2912 isert_get_dataout(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
, bool recovery
)
2914 struct se_cmd
*se_cmd
= &cmd
->se_cmd
;
2915 struct isert_cmd
*isert_cmd
= iscsit_priv_cmd(cmd
);
2916 struct isert_rdma_wr
*wr
= &isert_cmd
->rdma_wr
;
2917 struct isert_conn
*isert_conn
= conn
->context
;
2918 struct isert_device
*device
= isert_conn
->conn_device
;
2919 struct ib_send_wr
*wr_failed
;
2922 isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n",
2923 isert_cmd
, se_cmd
->data_length
, cmd
->write_data_done
);
2924 wr
->iser_ib_op
= ISER_IB_RDMA_READ
;
2925 rc
= device
->reg_rdma_mem(conn
, cmd
, wr
);
2927 isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd
);
2931 rc
= ib_post_send(isert_conn
->conn_qp
, wr
->send_wr
, &wr_failed
);
2933 isert_warn("ib_post_send() failed for IB_WR_RDMA_READ\n");
2935 isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n",
2942 isert_immediate_queue(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
, int state
)
2947 case ISTATE_SEND_NOPIN_WANT_RESPONSE
:
2948 ret
= isert_put_nopin(cmd
, conn
, false);
2951 isert_err("Unknown immediate state: 0x%02x\n", state
);
2960 isert_response_queue(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
, int state
)
2962 struct isert_conn
*isert_conn
= conn
->context
;
2966 case ISTATE_SEND_LOGOUTRSP
:
2967 ret
= isert_put_logout_rsp(cmd
, conn
);
2969 isert_conn
->logout_posted
= true;
2971 case ISTATE_SEND_NOPIN
:
2972 ret
= isert_put_nopin(cmd
, conn
, true);
2974 case ISTATE_SEND_TASKMGTRSP
:
2975 ret
= isert_put_tm_rsp(cmd
, conn
);
2977 case ISTATE_SEND_REJECT
:
2978 ret
= isert_put_reject(cmd
, conn
);
2980 case ISTATE_SEND_TEXTRSP
:
2981 ret
= isert_put_text_rsp(cmd
, conn
);
2983 case ISTATE_SEND_STATUS
:
2985 * Special case for sending non GOOD SCSI status from TX thread
2986 * context during pre se_cmd excecution failure.
2988 ret
= isert_put_response(conn
, cmd
);
2991 isert_err("Unknown response state: 0x%02x\n", state
);
3000 isert_setup_id(struct isert_np
*isert_np
)
3002 struct iscsi_np
*np
= isert_np
->np
;
3003 struct rdma_cm_id
*id
;
3004 struct sockaddr
*sa
;
3007 sa
= (struct sockaddr
*)&np
->np_sockaddr
;
3008 isert_dbg("ksockaddr: %p, sa: %p\n", &np
->np_sockaddr
, sa
);
3010 id
= rdma_create_id(isert_cma_handler
, isert_np
,
3011 RDMA_PS_TCP
, IB_QPT_RC
);
3013 isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id
));
3017 isert_dbg("id %p context %p\n", id
, id
->context
);
3019 ret
= rdma_bind_addr(id
, sa
);
3021 isert_err("rdma_bind_addr() failed: %d\n", ret
);
3025 ret
= rdma_listen(id
, ISERT_RDMA_LISTEN_BACKLOG
);
3027 isert_err("rdma_listen() failed: %d\n", ret
);
3033 rdma_destroy_id(id
);
3035 return ERR_PTR(ret
);
3039 isert_setup_np(struct iscsi_np
*np
,
3040 struct __kernel_sockaddr_storage
*ksockaddr
)
3042 struct isert_np
*isert_np
;
3043 struct rdma_cm_id
*isert_lid
;
3046 isert_np
= kzalloc(sizeof(struct isert_np
), GFP_KERNEL
);
3048 isert_err("Unable to allocate struct isert_np\n");
3051 sema_init(&isert_np
->np_sem
, 0);
3052 mutex_init(&isert_np
->np_accept_mutex
);
3053 INIT_LIST_HEAD(&isert_np
->np_accept_list
);
3054 init_completion(&isert_np
->np_login_comp
);
3058 * Setup the np->np_sockaddr from the passed sockaddr setup
3059 * in iscsi_target_configfs.c code..
3061 memcpy(&np
->np_sockaddr
, ksockaddr
,
3062 sizeof(struct __kernel_sockaddr_storage
));
3064 isert_lid
= isert_setup_id(isert_np
);
3065 if (IS_ERR(isert_lid
)) {
3066 ret
= PTR_ERR(isert_lid
);
3070 isert_np
->np_cm_id
= isert_lid
;
3071 np
->np_context
= isert_np
;
3082 isert_rdma_accept(struct isert_conn
*isert_conn
)
3084 struct rdma_cm_id
*cm_id
= isert_conn
->conn_cm_id
;
3085 struct rdma_conn_param cp
;
3088 memset(&cp
, 0, sizeof(struct rdma_conn_param
));
3089 cp
.initiator_depth
= isert_conn
->initiator_depth
;
3091 cp
.rnr_retry_count
= 7;
3093 ret
= rdma_accept(cm_id
, &cp
);
3095 isert_err("rdma_accept() failed with: %d\n", ret
);
3103 isert_get_login_rx(struct iscsi_conn
*conn
, struct iscsi_login
*login
)
3105 struct isert_conn
*isert_conn
= conn
->context
;
3108 isert_info("before login_req comp conn: %p\n", isert_conn
);
3109 ret
= wait_for_completion_interruptible(&isert_conn
->login_req_comp
);
3111 isert_err("isert_conn %p interrupted before got login req\n",
3115 reinit_completion(&isert_conn
->login_req_comp
);
3118 * For login requests after the first PDU, isert_rx_login_req() will
3119 * kick schedule_delayed_work(&conn->login_work) as the packet is
3120 * received, which turns this callback from iscsi_target_do_login_rx()
3123 if (!login
->first_request
)
3126 isert_rx_login_req(isert_conn
);
3128 isert_info("before conn_login_comp conn: %p\n", conn
);
3129 ret
= wait_for_completion_interruptible(&isert_conn
->conn_login_comp
);
3133 isert_info("processing login->req: %p\n", login
->req
);
3139 isert_set_conn_info(struct iscsi_np
*np
, struct iscsi_conn
*conn
,
3140 struct isert_conn
*isert_conn
)
3142 struct rdma_cm_id
*cm_id
= isert_conn
->conn_cm_id
;
3143 struct rdma_route
*cm_route
= &cm_id
->route
;
3144 struct sockaddr_in
*sock_in
;
3145 struct sockaddr_in6
*sock_in6
;
3147 conn
->login_family
= np
->np_sockaddr
.ss_family
;
3149 if (np
->np_sockaddr
.ss_family
== AF_INET6
) {
3150 sock_in6
= (struct sockaddr_in6
*)&cm_route
->addr
.dst_addr
;
3151 snprintf(conn
->login_ip
, sizeof(conn
->login_ip
), "%pI6c",
3152 &sock_in6
->sin6_addr
.in6_u
);
3153 conn
->login_port
= ntohs(sock_in6
->sin6_port
);
3155 sock_in6
= (struct sockaddr_in6
*)&cm_route
->addr
.src_addr
;
3156 snprintf(conn
->local_ip
, sizeof(conn
->local_ip
), "%pI6c",
3157 &sock_in6
->sin6_addr
.in6_u
);
3158 conn
->local_port
= ntohs(sock_in6
->sin6_port
);
3160 sock_in
= (struct sockaddr_in
*)&cm_route
->addr
.dst_addr
;
3161 sprintf(conn
->login_ip
, "%pI4",
3162 &sock_in
->sin_addr
.s_addr
);
3163 conn
->login_port
= ntohs(sock_in
->sin_port
);
3165 sock_in
= (struct sockaddr_in
*)&cm_route
->addr
.src_addr
;
3166 sprintf(conn
->local_ip
, "%pI4",
3167 &sock_in
->sin_addr
.s_addr
);
3168 conn
->local_port
= ntohs(sock_in
->sin_port
);
3173 isert_accept_np(struct iscsi_np
*np
, struct iscsi_conn
*conn
)
3175 struct isert_np
*isert_np
= np
->np_context
;
3176 struct isert_conn
*isert_conn
;
3177 int max_accept
= 0, ret
;
3180 ret
= down_interruptible(&isert_np
->np_sem
);
3181 if (ret
|| max_accept
> 5)
3184 spin_lock_bh(&np
->np_thread_lock
);
3185 if (np
->np_thread_state
>= ISCSI_NP_THREAD_RESET
) {
3186 spin_unlock_bh(&np
->np_thread_lock
);
3187 isert_dbg("np_thread_state %d\n",
3188 np
->np_thread_state
);
3190 * No point in stalling here when np_thread
3191 * is in state RESET/SHUTDOWN/EXIT - bail
3195 spin_unlock_bh(&np
->np_thread_lock
);
3197 mutex_lock(&isert_np
->np_accept_mutex
);
3198 if (list_empty(&isert_np
->np_accept_list
)) {
3199 mutex_unlock(&isert_np
->np_accept_mutex
);
3203 isert_conn
= list_first_entry(&isert_np
->np_accept_list
,
3204 struct isert_conn
, conn_accept_node
);
3205 list_del_init(&isert_conn
->conn_accept_node
);
3206 mutex_unlock(&isert_np
->np_accept_mutex
);
3208 conn
->context
= isert_conn
;
3209 isert_conn
->conn
= conn
;
3212 isert_set_conn_info(np
, conn
, isert_conn
);
3214 isert_dbg("Processing isert_conn: %p\n", isert_conn
);
3220 isert_free_np(struct iscsi_np
*np
)
3222 struct isert_np
*isert_np
= np
->np_context
;
3223 struct isert_conn
*isert_conn
, *n
;
3225 if (isert_np
->np_cm_id
)
3226 rdma_destroy_id(isert_np
->np_cm_id
);
3229 * FIXME: At this point we don't have a good way to insure
3230 * that at this point we don't have hanging connections that
3231 * completed RDMA establishment but didn't start iscsi login
3232 * process. So work-around this by cleaning up what ever piled
3233 * up in np_accept_list.
3235 mutex_lock(&isert_np
->np_accept_mutex
);
3236 if (!list_empty(&isert_np
->np_accept_list
)) {
3237 isert_info("Still have isert connections, cleaning up...\n");
3238 list_for_each_entry_safe(isert_conn
, n
,
3239 &isert_np
->np_accept_list
,
3241 isert_info("cleaning isert_conn %p state (%d)\n",
3242 isert_conn
, isert_conn
->state
);
3243 isert_connect_release(isert_conn
);
3246 mutex_unlock(&isert_np
->np_accept_mutex
);
3248 np
->np_context
= NULL
;
3252 static void isert_release_work(struct work_struct
*work
)
3254 struct isert_conn
*isert_conn
= container_of(work
,
3258 isert_info("Starting release conn %p\n", isert_conn
);
3260 wait_for_completion(&isert_conn
->conn_wait
);
3262 mutex_lock(&isert_conn
->conn_mutex
);
3263 isert_conn
->state
= ISER_CONN_DOWN
;
3264 mutex_unlock(&isert_conn
->conn_mutex
);
3266 isert_info("Destroying conn %p\n", isert_conn
);
3267 isert_put_conn(isert_conn
);
3271 isert_wait4logout(struct isert_conn
*isert_conn
)
3273 struct iscsi_conn
*conn
= isert_conn
->conn
;
3275 isert_info("conn %p\n", isert_conn
);
3277 if (isert_conn
->logout_posted
) {
3278 isert_info("conn %p wait for conn_logout_comp\n", isert_conn
);
3279 wait_for_completion_timeout(&conn
->conn_logout_comp
,
3280 SECONDS_FOR_LOGOUT_COMP
* HZ
);
3285 isert_wait4cmds(struct iscsi_conn
*conn
)
3287 isert_info("iscsi_conn %p\n", conn
);
3290 target_sess_cmd_list_set_waiting(conn
->sess
->se_sess
);
3291 target_wait_for_sess_cmds(conn
->sess
->se_sess
);
3296 isert_wait4flush(struct isert_conn
*isert_conn
)
3298 struct ib_recv_wr
*bad_wr
;
3300 isert_info("conn %p\n", isert_conn
);
3302 init_completion(&isert_conn
->conn_wait_comp_err
);
3303 isert_conn
->beacon
.wr_id
= ISER_BEACON_WRID
;
3304 /* post an indication that all flush errors were consumed */
3305 if (ib_post_recv(isert_conn
->conn_qp
, &isert_conn
->beacon
, &bad_wr
)) {
3306 isert_err("conn %p failed to post beacon", isert_conn
);
3310 wait_for_completion(&isert_conn
->conn_wait_comp_err
);
3313 static void isert_wait_conn(struct iscsi_conn
*conn
)
3315 struct isert_conn
*isert_conn
= conn
->context
;
3317 isert_info("Starting conn %p\n", isert_conn
);
3319 mutex_lock(&isert_conn
->conn_mutex
);
3321 * Only wait for conn_wait_comp_err if the isert_conn made it
3322 * into full feature phase..
3324 if (isert_conn
->state
== ISER_CONN_INIT
) {
3325 mutex_unlock(&isert_conn
->conn_mutex
);
3328 isert_conn_terminate(isert_conn
);
3329 mutex_unlock(&isert_conn
->conn_mutex
);
3331 isert_wait4cmds(conn
);
3332 isert_wait4flush(isert_conn
);
3333 isert_wait4logout(isert_conn
);
3335 INIT_WORK(&isert_conn
->release_work
, isert_release_work
);
3336 queue_work(isert_release_wq
, &isert_conn
->release_work
);
3339 static void isert_free_conn(struct iscsi_conn
*conn
)
3341 struct isert_conn
*isert_conn
= conn
->context
;
3343 isert_put_conn(isert_conn
);
3346 static struct iscsit_transport iser_target_transport
= {
3348 .transport_type
= ISCSI_INFINIBAND
,
3349 .priv_size
= sizeof(struct isert_cmd
),
3350 .owner
= THIS_MODULE
,
3351 .iscsit_setup_np
= isert_setup_np
,
3352 .iscsit_accept_np
= isert_accept_np
,
3353 .iscsit_free_np
= isert_free_np
,
3354 .iscsit_wait_conn
= isert_wait_conn
,
3355 .iscsit_free_conn
= isert_free_conn
,
3356 .iscsit_get_login_rx
= isert_get_login_rx
,
3357 .iscsit_put_login_tx
= isert_put_login_tx
,
3358 .iscsit_immediate_queue
= isert_immediate_queue
,
3359 .iscsit_response_queue
= isert_response_queue
,
3360 .iscsit_get_dataout
= isert_get_dataout
,
3361 .iscsit_queue_data_in
= isert_put_datain
,
3362 .iscsit_queue_status
= isert_put_response
,
3363 .iscsit_aborted_task
= isert_aborted_task
,
3364 .iscsit_get_sup_prot_ops
= isert_get_sup_prot_ops
,
3367 static int __init
isert_init(void)
3371 isert_comp_wq
= alloc_workqueue("isert_comp_wq",
3372 WQ_UNBOUND
| WQ_HIGHPRI
, 0);
3373 if (!isert_comp_wq
) {
3374 isert_err("Unable to allocate isert_comp_wq\n");
3379 isert_release_wq
= alloc_workqueue("isert_release_wq", WQ_UNBOUND
,
3380 WQ_UNBOUND_MAX_ACTIVE
);
3381 if (!isert_release_wq
) {
3382 isert_err("Unable to allocate isert_release_wq\n");
3384 goto destroy_comp_wq
;
3387 iscsit_register_transport(&iser_target_transport
);
3388 isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n");
3393 destroy_workqueue(isert_comp_wq
);
3398 static void __exit
isert_exit(void)
3400 flush_scheduled_work();
3401 destroy_workqueue(isert_release_wq
);
3402 destroy_workqueue(isert_comp_wq
);
3403 iscsit_unregister_transport(&iser_target_transport
);
3404 isert_info("iSER_TARGET[0] - Released iser_target_transport\n");
3407 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure");
3408 MODULE_VERSION("0.1");
3409 MODULE_AUTHOR("nab@Linux-iSCSI.org");
3410 MODULE_LICENSE("GPL");
3412 module_init(isert_init
);
3413 module_exit(isert_exit
);