2 * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/kernel.h>
34 #include <linux/module.h>
35 #include <linux/delay.h>
36 #include <linux/version.h>
38 #include "iscsi_iser.h"
40 #define ISCSI_ISER_MAX_CONN 8
41 #define ISER_MAX_CQ_LEN ((ISER_QP_MAX_RECV_DTOS + \
42 ISER_QP_MAX_REQ_DTOS) * \
45 static void iser_cq_tasklet_fn(unsigned long data
);
46 static void iser_cq_callback(struct ib_cq
*cq
, void *cq_context
);
48 static void iser_cq_event_callback(struct ib_event
*cause
, void *context
)
50 iser_err("got cq event %d \n", cause
->event
);
53 static void iser_qp_event_callback(struct ib_event
*cause
, void *context
)
55 iser_err("got qp event %d\n",cause
->event
);
59 * iser_create_device_ib_res - creates Protection Domain (PD), Completion
60 * Queue (CQ), DMA Memory Region (DMA MR) with the device associated with
63 * returns 0 on success, -1 on failure
65 static int iser_create_device_ib_res(struct iser_device
*device
)
67 device
->pd
= ib_alloc_pd(device
->ib_device
);
68 if (IS_ERR(device
->pd
))
71 device
->cq
= ib_create_cq(device
->ib_device
,
73 iser_cq_event_callback
,
76 if (IS_ERR(device
->cq
))
79 if (ib_req_notify_cq(device
->cq
, IB_CQ_NEXT_COMP
))
82 tasklet_init(&device
->cq_tasklet
,
84 (unsigned long)device
);
86 device
->mr
= ib_get_dma_mr(device
->pd
, IB_ACCESS_LOCAL_WRITE
|
87 IB_ACCESS_REMOTE_WRITE
|
88 IB_ACCESS_REMOTE_READ
);
89 if (IS_ERR(device
->mr
))
95 tasklet_kill(&device
->cq_tasklet
);
97 ib_destroy_cq(device
->cq
);
99 ib_dealloc_pd(device
->pd
);
101 iser_err("failed to allocate an IB resource\n");
106 * iser_free_device_ib_res - destroy/dealloc/dereg the DMA MR,
107 * CQ and PD created with the device associated with the adapator.
109 static void iser_free_device_ib_res(struct iser_device
*device
)
111 BUG_ON(device
->mr
== NULL
);
113 tasklet_kill(&device
->cq_tasklet
);
115 (void)ib_dereg_mr(device
->mr
);
116 (void)ib_destroy_cq(device
->cq
);
117 (void)ib_dealloc_pd(device
->pd
);
125 * iser_create_ib_conn_res - Creates FMR pool and Queue-Pair (QP)
127 * returns 0 on success, -1 on failure
129 static int iser_create_ib_conn_res(struct iser_conn
*ib_conn
)
131 struct iser_device
*device
;
132 struct ib_qp_init_attr init_attr
;
134 struct ib_fmr_pool_param params
;
136 BUG_ON(ib_conn
->device
== NULL
);
138 device
= ib_conn
->device
;
140 ib_conn
->page_vec
= kmalloc(sizeof(struct iser_page_vec
) +
141 (sizeof(u64
) * (ISCSI_ISER_SG_TABLESIZE
+1)),
143 if (!ib_conn
->page_vec
) {
147 ib_conn
->page_vec
->pages
= (u64
*) (ib_conn
->page_vec
+ 1);
149 params
.page_shift
= SHIFT_4K
;
150 /* when the first/last SG element are not start/end *
151 * page aligned, the map whould be of N+1 pages */
152 params
.max_pages_per_fmr
= ISCSI_ISER_SG_TABLESIZE
+ 1;
153 /* make the pool size twice the max number of SCSI commands *
154 * the ML is expected to queue, watermark for unmap at 50% */
155 params
.pool_size
= ISCSI_DEF_XMIT_CMDS_MAX
* 2;
156 params
.dirty_watermark
= ISCSI_DEF_XMIT_CMDS_MAX
;
158 params
.flush_function
= NULL
;
159 params
.access
= (IB_ACCESS_LOCAL_WRITE
|
160 IB_ACCESS_REMOTE_WRITE
|
161 IB_ACCESS_REMOTE_READ
);
163 ib_conn
->fmr_pool
= ib_create_fmr_pool(device
->pd
, ¶ms
);
164 if (IS_ERR(ib_conn
->fmr_pool
)) {
165 ret
= PTR_ERR(ib_conn
->fmr_pool
);
169 memset(&init_attr
, 0, sizeof init_attr
);
171 init_attr
.event_handler
= iser_qp_event_callback
;
172 init_attr
.qp_context
= (void *)ib_conn
;
173 init_attr
.send_cq
= device
->cq
;
174 init_attr
.recv_cq
= device
->cq
;
175 init_attr
.cap
.max_send_wr
= ISER_QP_MAX_REQ_DTOS
;
176 init_attr
.cap
.max_recv_wr
= ISER_QP_MAX_RECV_DTOS
;
177 init_attr
.cap
.max_send_sge
= MAX_REGD_BUF_VECTOR_LEN
;
178 init_attr
.cap
.max_recv_sge
= 2;
179 init_attr
.sq_sig_type
= IB_SIGNAL_REQ_WR
;
180 init_attr
.qp_type
= IB_QPT_RC
;
182 ret
= rdma_create_qp(ib_conn
->cma_id
, device
->pd
, &init_attr
);
186 ib_conn
->qp
= ib_conn
->cma_id
->qp
;
187 iser_err("setting conn %p cma_id %p: fmr_pool %p qp %p\n",
188 ib_conn
, ib_conn
->cma_id
,
189 ib_conn
->fmr_pool
, ib_conn
->cma_id
->qp
);
193 (void)ib_destroy_fmr_pool(ib_conn
->fmr_pool
);
195 kfree(ib_conn
->page_vec
);
197 iser_err("unable to alloc mem or create resource, err %d\n", ret
);
202 * releases the FMR pool, QP and CMA ID objects, returns 0 on success,
205 static int iser_free_ib_conn_res(struct iser_conn
*ib_conn
)
207 BUG_ON(ib_conn
== NULL
);
209 iser_err("freeing conn %p cma_id %p fmr pool %p qp %p\n",
210 ib_conn
, ib_conn
->cma_id
,
211 ib_conn
->fmr_pool
, ib_conn
->qp
);
213 /* qp is created only once both addr & route are resolved */
214 if (ib_conn
->fmr_pool
!= NULL
)
215 ib_destroy_fmr_pool(ib_conn
->fmr_pool
);
217 if (ib_conn
->qp
!= NULL
)
218 rdma_destroy_qp(ib_conn
->cma_id
);
220 if (ib_conn
->cma_id
!= NULL
)
221 rdma_destroy_id(ib_conn
->cma_id
);
223 ib_conn
->fmr_pool
= NULL
;
225 ib_conn
->cma_id
= NULL
;
226 kfree(ib_conn
->page_vec
);
232 * based on the resolved device node GUID see if there already allocated
233 * device for this device. If there's no such, create one.
236 struct iser_device
*iser_device_find_by_ib_device(struct rdma_cm_id
*cma_id
)
238 struct iser_device
*device
;
240 mutex_lock(&ig
.device_list_mutex
);
242 list_for_each_entry(device
, &ig
.device_list
, ig_list
)
243 /* find if there's a match using the node GUID */
244 if (device
->ib_device
->node_guid
== cma_id
->device
->node_guid
)
247 device
= kzalloc(sizeof *device
, GFP_KERNEL
);
251 /* assign this device to the device */
252 device
->ib_device
= cma_id
->device
;
253 /* init the device and link it into ig device list */
254 if (iser_create_device_ib_res(device
)) {
259 list_add(&device
->ig_list
, &ig
.device_list
);
264 mutex_unlock(&ig
.device_list_mutex
);
268 /* if there's no demand for this device, release it */
269 static void iser_device_try_release(struct iser_device
*device
)
271 mutex_lock(&ig
.device_list_mutex
);
273 iser_err("device %p refcount %d\n",device
,device
->refcount
);
274 if (!device
->refcount
) {
275 iser_free_device_ib_res(device
);
276 list_del(&device
->ig_list
);
279 mutex_unlock(&ig
.device_list_mutex
);
282 int iser_conn_state_comp(struct iser_conn
*ib_conn
,
283 enum iser_ib_conn_state comp
)
287 spin_lock_bh(&ib_conn
->lock
);
288 ret
= (ib_conn
->state
== comp
);
289 spin_unlock_bh(&ib_conn
->lock
);
293 static int iser_conn_state_comp_exch(struct iser_conn
*ib_conn
,
294 enum iser_ib_conn_state comp
,
295 enum iser_ib_conn_state exch
)
299 spin_lock_bh(&ib_conn
->lock
);
300 if ((ret
= (ib_conn
->state
== comp
)))
301 ib_conn
->state
= exch
;
302 spin_unlock_bh(&ib_conn
->lock
);
307 * Frees all conn objects and deallocs conn descriptor
309 static void iser_conn_release(struct iser_conn
*ib_conn
)
311 struct iser_device
*device
= ib_conn
->device
;
313 BUG_ON(ib_conn
->state
!= ISER_CONN_DOWN
);
315 mutex_lock(&ig
.connlist_mutex
);
316 list_del(&ib_conn
->conn_list
);
317 mutex_unlock(&ig
.connlist_mutex
);
319 iser_free_ib_conn_res(ib_conn
);
320 ib_conn
->device
= NULL
;
321 /* on EVENT_ADDR_ERROR there's no device yet for this conn */
323 iser_device_try_release(device
);
324 if (ib_conn
->iser_conn
)
325 ib_conn
->iser_conn
->ib_conn
= NULL
;
326 iscsi_destroy_endpoint(ib_conn
->ep
);
329 void iser_conn_get(struct iser_conn
*ib_conn
)
331 atomic_inc(&ib_conn
->refcount
);
334 void iser_conn_put(struct iser_conn
*ib_conn
)
336 if (atomic_dec_and_test(&ib_conn
->refcount
))
337 iser_conn_release(ib_conn
);
341 * triggers start of the disconnect procedures and wait for them to be done
343 void iser_conn_terminate(struct iser_conn
*ib_conn
)
347 /* change the ib conn state only if the conn is UP, however always call
348 * rdma_disconnect since this is the only way to cause the CMA to change
349 * the QP state to ERROR
352 iser_conn_state_comp_exch(ib_conn
, ISER_CONN_UP
, ISER_CONN_TERMINATING
);
353 err
= rdma_disconnect(ib_conn
->cma_id
);
355 iser_err("Failed to disconnect, conn: 0x%p err %d\n",
358 wait_event_interruptible(ib_conn
->wait
,
359 ib_conn
->state
== ISER_CONN_DOWN
);
361 iser_conn_put(ib_conn
);
364 static void iser_connect_error(struct rdma_cm_id
*cma_id
)
366 struct iser_conn
*ib_conn
;
367 ib_conn
= (struct iser_conn
*)cma_id
->context
;
369 ib_conn
->state
= ISER_CONN_DOWN
;
370 wake_up_interruptible(&ib_conn
->wait
);
373 static void iser_addr_handler(struct rdma_cm_id
*cma_id
)
375 struct iser_device
*device
;
376 struct iser_conn
*ib_conn
;
379 device
= iser_device_find_by_ib_device(cma_id
);
381 iser_err("device lookup/creation failed\n");
382 iser_connect_error(cma_id
);
386 ib_conn
= (struct iser_conn
*)cma_id
->context
;
387 ib_conn
->device
= device
;
389 ret
= rdma_resolve_route(cma_id
, 1000);
391 iser_err("resolve route failed: %d\n", ret
);
392 iser_connect_error(cma_id
);
396 static void iser_route_handler(struct rdma_cm_id
*cma_id
)
398 struct rdma_conn_param conn_param
;
401 ret
= iser_create_ib_conn_res((struct iser_conn
*)cma_id
->context
);
405 iser_dbg("path.mtu is %d setting it to %d\n",
406 cma_id
->route
.path_rec
->mtu
, IB_MTU_1024
);
408 /* we must set the MTU to 1024 as this is what the target is assuming */
409 if (cma_id
->route
.path_rec
->mtu
> IB_MTU_1024
)
410 cma_id
->route
.path_rec
->mtu
= IB_MTU_1024
;
412 memset(&conn_param
, 0, sizeof conn_param
);
413 conn_param
.responder_resources
= 4;
414 conn_param
.initiator_depth
= 1;
415 conn_param
.retry_count
= 7;
416 conn_param
.rnr_retry_count
= 6;
418 ret
= rdma_connect(cma_id
, &conn_param
);
420 iser_err("failure connecting: %d\n", ret
);
426 iser_connect_error(cma_id
);
429 static void iser_connected_handler(struct rdma_cm_id
*cma_id
)
431 struct iser_conn
*ib_conn
;
433 ib_conn
= (struct iser_conn
*)cma_id
->context
;
434 ib_conn
->state
= ISER_CONN_UP
;
435 wake_up_interruptible(&ib_conn
->wait
);
438 static void iser_disconnected_handler(struct rdma_cm_id
*cma_id
)
440 struct iser_conn
*ib_conn
;
442 ib_conn
= (struct iser_conn
*)cma_id
->context
;
443 ib_conn
->disc_evt_flag
= 1;
445 /* getting here when the state is UP means that the conn is being *
446 * terminated asynchronously from the iSCSI layer's perspective. */
447 if (iser_conn_state_comp_exch(ib_conn
, ISER_CONN_UP
,
448 ISER_CONN_TERMINATING
))
449 iscsi_conn_failure(ib_conn
->iser_conn
->iscsi_conn
,
450 ISCSI_ERR_CONN_FAILED
);
452 /* Complete the termination process if no posts are pending */
453 if ((atomic_read(&ib_conn
->post_recv_buf_count
) == 0) &&
454 (atomic_read(&ib_conn
->post_send_buf_count
) == 0)) {
455 ib_conn
->state
= ISER_CONN_DOWN
;
456 wake_up_interruptible(&ib_conn
->wait
);
460 static int iser_cma_handler(struct rdma_cm_id
*cma_id
, struct rdma_cm_event
*event
)
464 iser_err("event %d conn %p id %p\n",event
->event
,cma_id
->context
,cma_id
);
466 switch (event
->event
) {
467 case RDMA_CM_EVENT_ADDR_RESOLVED
:
468 iser_addr_handler(cma_id
);
470 case RDMA_CM_EVENT_ROUTE_RESOLVED
:
471 iser_route_handler(cma_id
);
473 case RDMA_CM_EVENT_ESTABLISHED
:
474 iser_connected_handler(cma_id
);
476 case RDMA_CM_EVENT_ADDR_ERROR
:
477 case RDMA_CM_EVENT_ROUTE_ERROR
:
478 case RDMA_CM_EVENT_CONNECT_ERROR
:
479 case RDMA_CM_EVENT_UNREACHABLE
:
480 case RDMA_CM_EVENT_REJECTED
:
481 iser_err("event: %d, error: %d\n", event
->event
, event
->status
);
482 iser_connect_error(cma_id
);
484 case RDMA_CM_EVENT_DISCONNECTED
:
485 case RDMA_CM_EVENT_DEVICE_REMOVAL
:
486 iser_disconnected_handler(cma_id
);
489 iser_err("Unexpected RDMA CM event (%d)\n", event
->event
);
495 void iser_conn_init(struct iser_conn
*ib_conn
)
497 ib_conn
->state
= ISER_CONN_INIT
;
498 init_waitqueue_head(&ib_conn
->wait
);
499 atomic_set(&ib_conn
->post_recv_buf_count
, 0);
500 atomic_set(&ib_conn
->post_send_buf_count
, 0);
501 atomic_set(&ib_conn
->refcount
, 1);
502 INIT_LIST_HEAD(&ib_conn
->conn_list
);
503 spin_lock_init(&ib_conn
->lock
);
507 * starts the process of connecting to the target
508 * sleeps untill the connection is established or rejected
510 int iser_connect(struct iser_conn
*ib_conn
,
511 struct sockaddr_in
*src_addr
,
512 struct sockaddr_in
*dst_addr
,
515 struct sockaddr
*src
, *dst
;
518 sprintf(ib_conn
->name
,"%d.%d.%d.%d:%d",
519 NIPQUAD(dst_addr
->sin_addr
.s_addr
), dst_addr
->sin_port
);
521 /* the device is known only --after-- address resolution */
522 ib_conn
->device
= NULL
;
524 iser_err("connecting to: %d.%d.%d.%d, port 0x%x\n",
525 NIPQUAD(dst_addr
->sin_addr
), dst_addr
->sin_port
);
527 ib_conn
->state
= ISER_CONN_PENDING
;
529 ib_conn
->cma_id
= rdma_create_id(iser_cma_handler
,
532 if (IS_ERR(ib_conn
->cma_id
)) {
533 err
= PTR_ERR(ib_conn
->cma_id
);
534 iser_err("rdma_create_id failed: %d\n", err
);
538 src
= (struct sockaddr
*)src_addr
;
539 dst
= (struct sockaddr
*)dst_addr
;
540 err
= rdma_resolve_addr(ib_conn
->cma_id
, src
, dst
, 1000);
542 iser_err("rdma_resolve_addr failed: %d\n", err
);
547 wait_event_interruptible(ib_conn
->wait
,
548 (ib_conn
->state
!= ISER_CONN_PENDING
));
550 if (ib_conn
->state
!= ISER_CONN_UP
) {
552 goto connect_failure
;
556 mutex_lock(&ig
.connlist_mutex
);
557 list_add(&ib_conn
->conn_list
, &ig
.connlist
);
558 mutex_unlock(&ig
.connlist_mutex
);
562 ib_conn
->cma_id
= NULL
;
564 ib_conn
->state
= ISER_CONN_DOWN
;
566 iser_conn_release(ib_conn
);
571 * iser_reg_page_vec - Register physical memory
573 * returns: 0 on success, errno code on failure
575 int iser_reg_page_vec(struct iser_conn
*ib_conn
,
576 struct iser_page_vec
*page_vec
,
577 struct iser_mem_reg
*mem_reg
)
579 struct ib_pool_fmr
*mem
;
584 page_list
= page_vec
->pages
;
585 io_addr
= page_list
[0];
587 mem
= ib_fmr_pool_map_phys(ib_conn
->fmr_pool
,
593 status
= (int)PTR_ERR(mem
);
594 iser_err("ib_fmr_pool_map_phys failed: %d\n", status
);
598 mem_reg
->lkey
= mem
->fmr
->lkey
;
599 mem_reg
->rkey
= mem
->fmr
->rkey
;
600 mem_reg
->len
= page_vec
->length
* SIZE_4K
;
601 mem_reg
->va
= io_addr
;
603 mem_reg
->mem_h
= (void *)mem
;
605 mem_reg
->va
+= page_vec
->offset
;
606 mem_reg
->len
= page_vec
->data_size
;
608 iser_dbg("PHYSICAL Mem.register, [PHYS p_array: 0x%p, sz: %d, "
609 "entry[0]: (0x%08lx,%ld)] -> "
610 "[lkey: 0x%08X mem_h: 0x%p va: 0x%08lX sz: %ld]\n",
611 page_vec
, page_vec
->length
,
612 (unsigned long)page_vec
->pages
[0],
613 (unsigned long)page_vec
->data_size
,
614 (unsigned int)mem_reg
->lkey
, mem_reg
->mem_h
,
615 (unsigned long)mem_reg
->va
, (unsigned long)mem_reg
->len
);
620 * Unregister (previosuly registered) memory.
622 void iser_unreg_mem(struct iser_mem_reg
*reg
)
626 iser_dbg("PHYSICAL Mem.Unregister mem_h %p\n",reg
->mem_h
);
628 ret
= ib_fmr_pool_unmap((struct ib_pool_fmr
*)reg
->mem_h
);
630 iser_err("ib_fmr_pool_unmap failed %d\n", ret
);
636 * iser_dto_to_iov - builds IOV from a dto descriptor
638 static void iser_dto_to_iov(struct iser_dto
*dto
, struct ib_sge
*iov
, int iov_len
)
642 struct iser_regd_buf
*regd_buf
;
644 if (dto
->regd_vector_len
> iov_len
) {
645 iser_err("iov size %d too small for posting dto of len %d\n",
646 iov_len
, dto
->regd_vector_len
);
650 for (i
= 0; i
< dto
->regd_vector_len
; i
++) {
652 regd_buf
= dto
->regd
[i
];
654 sge
->addr
= regd_buf
->reg
.va
;
655 sge
->length
= regd_buf
->reg
.len
;
656 sge
->lkey
= regd_buf
->reg
.lkey
;
658 if (dto
->used_sz
[i
] > 0) /* Adjust size */
659 sge
->length
= dto
->used_sz
[i
];
661 /* offset and length should not exceed the regd buf length */
662 if (sge
->length
+ dto
->offset
[i
] > regd_buf
->reg
.len
) {
663 iser_err("Used len:%ld + offset:%d, exceed reg.buf.len:"
664 "%ld in dto:0x%p [%d], va:0x%08lX\n",
665 (unsigned long)sge
->length
, dto
->offset
[i
],
666 (unsigned long)regd_buf
->reg
.len
, dto
, i
,
667 (unsigned long)sge
->addr
);
671 sge
->addr
+= dto
->offset
[i
]; /* Adjust offset */
676 * iser_post_recv - Posts a receive buffer.
678 * returns 0 on success, -1 on failure
680 int iser_post_recv(struct iser_desc
*rx_desc
)
682 int ib_ret
, ret_val
= 0;
683 struct ib_recv_wr recv_wr
, *recv_wr_failed
;
684 struct ib_sge iov
[2];
685 struct iser_conn
*ib_conn
;
686 struct iser_dto
*recv_dto
= &rx_desc
->dto
;
689 ib_conn
= recv_dto
->ib_conn
;
691 iser_dto_to_iov(recv_dto
, iov
, 2);
694 recv_wr
.sg_list
= iov
;
695 recv_wr
.num_sge
= recv_dto
->regd_vector_len
;
696 recv_wr
.wr_id
= (unsigned long)rx_desc
;
698 atomic_inc(&ib_conn
->post_recv_buf_count
);
699 ib_ret
= ib_post_recv(ib_conn
->qp
, &recv_wr
, &recv_wr_failed
);
701 iser_err("ib_post_recv failed ret=%d\n", ib_ret
);
702 atomic_dec(&ib_conn
->post_recv_buf_count
);
710 * iser_start_send - Initiate a Send DTO operation
712 * returns 0 on success, -1 on failure
714 int iser_post_send(struct iser_desc
*tx_desc
)
716 int ib_ret
, ret_val
= 0;
717 struct ib_send_wr send_wr
, *send_wr_failed
;
718 struct ib_sge iov
[MAX_REGD_BUF_VECTOR_LEN
];
719 struct iser_conn
*ib_conn
;
720 struct iser_dto
*dto
= &tx_desc
->dto
;
722 ib_conn
= dto
->ib_conn
;
724 iser_dto_to_iov(dto
, iov
, MAX_REGD_BUF_VECTOR_LEN
);
727 send_wr
.wr_id
= (unsigned long)tx_desc
;
728 send_wr
.sg_list
= iov
;
729 send_wr
.num_sge
= dto
->regd_vector_len
;
730 send_wr
.opcode
= IB_WR_SEND
;
731 send_wr
.send_flags
= dto
->notify_enable
? IB_SEND_SIGNALED
: 0;
733 atomic_inc(&ib_conn
->post_send_buf_count
);
735 ib_ret
= ib_post_send(ib_conn
->qp
, &send_wr
, &send_wr_failed
);
737 iser_err("Failed to start SEND DTO, dto: 0x%p, IOV len: %d\n",
738 dto
, dto
->regd_vector_len
);
739 iser_err("ib_post_send failed, ret:%d\n", ib_ret
);
740 atomic_dec(&ib_conn
->post_send_buf_count
);
747 static void iser_handle_comp_error(struct iser_desc
*desc
)
749 struct iser_dto
*dto
= &desc
->dto
;
750 struct iser_conn
*ib_conn
= dto
->ib_conn
;
752 iser_dto_buffs_release(dto
);
754 if (desc
->type
== ISCSI_RX
) {
756 kmem_cache_free(ig
.desc_cache
, desc
);
757 atomic_dec(&ib_conn
->post_recv_buf_count
);
758 } else { /* type is TX control/command/dataout */
759 if (desc
->type
== ISCSI_TX_DATAOUT
)
760 kmem_cache_free(ig
.desc_cache
, desc
);
761 atomic_dec(&ib_conn
->post_send_buf_count
);
764 if (atomic_read(&ib_conn
->post_recv_buf_count
) == 0 &&
765 atomic_read(&ib_conn
->post_send_buf_count
) == 0) {
766 /* getting here when the state is UP means that the conn is *
767 * being terminated asynchronously from the iSCSI layer's *
769 if (iser_conn_state_comp_exch(ib_conn
, ISER_CONN_UP
,
770 ISER_CONN_TERMINATING
))
771 iscsi_conn_failure(ib_conn
->iser_conn
->iscsi_conn
,
772 ISCSI_ERR_CONN_FAILED
);
774 /* complete the termination process if disconnect event was delivered *
775 * note there are no more non completed posts to the QP */
776 if (ib_conn
->disc_evt_flag
) {
777 ib_conn
->state
= ISER_CONN_DOWN
;
778 wake_up_interruptible(&ib_conn
->wait
);
783 static void iser_cq_tasklet_fn(unsigned long data
)
785 struct iser_device
*device
= (struct iser_device
*)data
;
786 struct ib_cq
*cq
= device
->cq
;
788 struct iser_desc
*desc
;
789 unsigned long xfer_len
;
791 while (ib_poll_cq(cq
, 1, &wc
) == 1) {
792 desc
= (struct iser_desc
*) (unsigned long) wc
.wr_id
;
793 BUG_ON(desc
== NULL
);
795 if (wc
.status
== IB_WC_SUCCESS
) {
796 if (desc
->type
== ISCSI_RX
) {
797 xfer_len
= (unsigned long)wc
.byte_len
;
798 iser_rcv_completion(desc
, xfer_len
);
799 } else /* type == ISCSI_TX_CONTROL/SCSI_CMD/DOUT */
800 iser_snd_completion(desc
);
802 iser_err("comp w. error op %d status %d\n",desc
->type
,wc
.status
);
803 iser_handle_comp_error(desc
);
806 /* #warning "it is assumed here that arming CQ only once its empty" *
807 * " would not cause interrupts to be missed" */
808 ib_req_notify_cq(cq
, IB_CQ_NEXT_COMP
);
811 static void iser_cq_callback(struct ib_cq
*cq
, void *cq_context
)
813 struct iser_device
*device
= (struct iser_device
*)cq_context
;
815 tasklet_schedule(&device
->cq_tasklet
);