2 * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/kernel.h>
34 #include <linux/slab.h>
36 #include <linux/scatterlist.h>
37 #include <linux/kfifo.h>
38 #include <scsi/scsi_cmnd.h>
39 #include <scsi/scsi_host.h>
41 #include "iscsi_iser.h"
43 /* Register user buffer memory and initialize passive rdma
44 * dto descriptor. Data size is stored in
45 * task->data[ISER_DIR_IN].data_len, Protection size
46 * os stored in task->prot[ISER_DIR_IN].data_len
48 static int iser_prepare_read_cmd(struct iscsi_task
*task
)
51 struct iscsi_iser_task
*iser_task
= task
->dd_data
;
52 struct iser_mem_reg
*mem_reg
;
54 struct iser_ctrl
*hdr
= &iser_task
->desc
.iser_header
;
55 struct iser_data_buf
*buf_in
= &iser_task
->data
[ISER_DIR_IN
];
57 err
= iser_dma_map_task_data(iser_task
,
64 if (scsi_prot_sg_count(iser_task
->sc
)) {
65 struct iser_data_buf
*pbuf_in
= &iser_task
->prot
[ISER_DIR_IN
];
67 err
= iser_dma_map_task_data(iser_task
,
75 err
= iser_reg_rdma_mem(iser_task
, ISER_DIR_IN
, false);
77 iser_err("Failed to set up Data-IN RDMA\n");
80 mem_reg
= &iser_task
->rdma_reg
[ISER_DIR_IN
];
82 hdr
->flags
|= ISER_RSV
;
83 hdr
->read_stag
= cpu_to_be32(mem_reg
->rkey
);
84 hdr
->read_va
= cpu_to_be64(mem_reg
->sge
.addr
);
86 iser_dbg("Cmd itt:%d READ tags RKEY:%#.4X VA:%#llX\n",
87 task
->itt
, mem_reg
->rkey
,
88 (unsigned long long)mem_reg
->sge
.addr
);
93 /* Register user buffer memory and initialize passive rdma
94 * dto descriptor. Data size is stored in
95 * task->data[ISER_DIR_OUT].data_len, Protection size
96 * is stored at task->prot[ISER_DIR_OUT].data_len
99 iser_prepare_write_cmd(struct iscsi_task
*task
,
101 unsigned int unsol_sz
,
104 struct iscsi_iser_task
*iser_task
= task
->dd_data
;
105 struct iser_mem_reg
*mem_reg
;
107 struct iser_ctrl
*hdr
= &iser_task
->desc
.iser_header
;
108 struct iser_data_buf
*buf_out
= &iser_task
->data
[ISER_DIR_OUT
];
109 struct ib_sge
*tx_dsg
= &iser_task
->desc
.tx_sg
[1];
111 err
= iser_dma_map_task_data(iser_task
,
118 if (scsi_prot_sg_count(iser_task
->sc
)) {
119 struct iser_data_buf
*pbuf_out
= &iser_task
->prot
[ISER_DIR_OUT
];
121 err
= iser_dma_map_task_data(iser_task
,
129 err
= iser_reg_rdma_mem(iser_task
, ISER_DIR_OUT
,
130 buf_out
->data_len
== imm_sz
);
132 iser_err("Failed to register write cmd RDMA mem\n");
136 mem_reg
= &iser_task
->rdma_reg
[ISER_DIR_OUT
];
138 if (unsol_sz
< edtl
) {
139 hdr
->flags
|= ISER_WSV
;
140 if (buf_out
->data_len
> imm_sz
) {
141 hdr
->write_stag
= cpu_to_be32(mem_reg
->rkey
);
142 hdr
->write_va
= cpu_to_be64(mem_reg
->sge
.addr
+ unsol_sz
);
145 iser_dbg("Cmd itt:%d, WRITE tags, RKEY:%#.4X "
146 "VA:%#llX + unsol:%d\n",
147 task
->itt
, mem_reg
->rkey
,
148 (unsigned long long)mem_reg
->sge
.addr
, unsol_sz
);
152 iser_dbg("Cmd itt:%d, WRITE, adding imm.data sz: %d\n",
154 tx_dsg
->addr
= mem_reg
->sge
.addr
;
155 tx_dsg
->length
= imm_sz
;
156 tx_dsg
->lkey
= mem_reg
->sge
.lkey
;
157 iser_task
->desc
.num_sge
= 2;
163 /* creates a new tx descriptor and adds header regd buffer */
164 static void iser_create_send_desc(struct iser_conn
*iser_conn
,
165 struct iser_tx_desc
*tx_desc
)
167 struct iser_device
*device
= iser_conn
->ib_conn
.device
;
169 ib_dma_sync_single_for_cpu(device
->ib_device
,
170 tx_desc
->dma_addr
, ISER_HEADERS_LEN
, DMA_TO_DEVICE
);
172 memset(&tx_desc
->iser_header
, 0, sizeof(struct iser_ctrl
));
173 tx_desc
->iser_header
.flags
= ISER_VER
;
174 tx_desc
->num_sge
= 1;
177 static void iser_free_login_buf(struct iser_conn
*iser_conn
)
179 struct iser_device
*device
= iser_conn
->ib_conn
.device
;
180 struct iser_login_desc
*desc
= &iser_conn
->login_desc
;
185 ib_dma_unmap_single(device
->ib_device
, desc
->req_dma
,
186 ISCSI_DEF_MAX_RECV_SEG_LEN
, DMA_TO_DEVICE
);
188 ib_dma_unmap_single(device
->ib_device
, desc
->rsp_dma
,
189 ISER_RX_LOGIN_SIZE
, DMA_FROM_DEVICE
);
194 /* make sure we never redo any unmapping */
199 static int iser_alloc_login_buf(struct iser_conn
*iser_conn
)
201 struct iser_device
*device
= iser_conn
->ib_conn
.device
;
202 struct iser_login_desc
*desc
= &iser_conn
->login_desc
;
204 desc
->req
= kmalloc(ISCSI_DEF_MAX_RECV_SEG_LEN
, GFP_KERNEL
);
208 desc
->req_dma
= ib_dma_map_single(device
->ib_device
, desc
->req
,
209 ISCSI_DEF_MAX_RECV_SEG_LEN
,
211 if (ib_dma_mapping_error(device
->ib_device
,
215 desc
->rsp
= kmalloc(ISER_RX_LOGIN_SIZE
, GFP_KERNEL
);
219 desc
->rsp_dma
= ib_dma_map_single(device
->ib_device
, desc
->rsp
,
222 if (ib_dma_mapping_error(device
->ib_device
,
231 ib_dma_unmap_single(device
->ib_device
, desc
->req_dma
,
232 ISCSI_DEF_MAX_RECV_SEG_LEN
,
240 int iser_alloc_rx_descriptors(struct iser_conn
*iser_conn
,
241 struct iscsi_session
*session
)
245 struct iser_rx_desc
*rx_desc
;
246 struct ib_sge
*rx_sg
;
247 struct ib_conn
*ib_conn
= &iser_conn
->ib_conn
;
248 struct iser_device
*device
= ib_conn
->device
;
250 iser_conn
->qp_max_recv_dtos
= session
->cmds_max
;
251 iser_conn
->qp_max_recv_dtos_mask
= session
->cmds_max
- 1; /* cmds_max is 2^N */
252 iser_conn
->min_posted_rx
= iser_conn
->qp_max_recv_dtos
>> 2;
254 if (device
->reg_ops
->alloc_reg_res(ib_conn
, session
->scsi_cmds_max
,
255 iser_conn
->scsi_sg_tablesize
))
256 goto create_rdma_reg_res_failed
;
258 if (iser_alloc_login_buf(iser_conn
))
259 goto alloc_login_buf_fail
;
261 iser_conn
->num_rx_descs
= session
->cmds_max
;
262 iser_conn
->rx_descs
= kmalloc(iser_conn
->num_rx_descs
*
263 sizeof(struct iser_rx_desc
), GFP_KERNEL
);
264 if (!iser_conn
->rx_descs
)
265 goto rx_desc_alloc_fail
;
267 rx_desc
= iser_conn
->rx_descs
;
269 for (i
= 0; i
< iser_conn
->qp_max_recv_dtos
; i
++, rx_desc
++) {
270 dma_addr
= ib_dma_map_single(device
->ib_device
, (void *)rx_desc
,
271 ISER_RX_PAYLOAD_SIZE
, DMA_FROM_DEVICE
);
272 if (ib_dma_mapping_error(device
->ib_device
, dma_addr
))
273 goto rx_desc_dma_map_failed
;
275 rx_desc
->dma_addr
= dma_addr
;
276 rx_desc
->cqe
.done
= iser_task_rsp
;
277 rx_sg
= &rx_desc
->rx_sg
;
278 rx_sg
->addr
= rx_desc
->dma_addr
;
279 rx_sg
->length
= ISER_RX_PAYLOAD_SIZE
;
280 rx_sg
->lkey
= device
->pd
->local_dma_lkey
;
283 iser_conn
->rx_desc_head
= 0;
286 rx_desc_dma_map_failed
:
287 rx_desc
= iser_conn
->rx_descs
;
288 for (j
= 0; j
< i
; j
++, rx_desc
++)
289 ib_dma_unmap_single(device
->ib_device
, rx_desc
->dma_addr
,
290 ISER_RX_PAYLOAD_SIZE
, DMA_FROM_DEVICE
);
291 kfree(iser_conn
->rx_descs
);
292 iser_conn
->rx_descs
= NULL
;
294 iser_free_login_buf(iser_conn
);
295 alloc_login_buf_fail
:
296 device
->reg_ops
->free_reg_res(ib_conn
);
297 create_rdma_reg_res_failed
:
298 iser_err("failed allocating rx descriptors / data buffers\n");
302 void iser_free_rx_descriptors(struct iser_conn
*iser_conn
)
305 struct iser_rx_desc
*rx_desc
;
306 struct ib_conn
*ib_conn
= &iser_conn
->ib_conn
;
307 struct iser_device
*device
= ib_conn
->device
;
309 if (device
->reg_ops
->free_reg_res
)
310 device
->reg_ops
->free_reg_res(ib_conn
);
312 rx_desc
= iser_conn
->rx_descs
;
313 for (i
= 0; i
< iser_conn
->qp_max_recv_dtos
; i
++, rx_desc
++)
314 ib_dma_unmap_single(device
->ib_device
, rx_desc
->dma_addr
,
315 ISER_RX_PAYLOAD_SIZE
, DMA_FROM_DEVICE
);
316 kfree(iser_conn
->rx_descs
);
317 /* make sure we never redo any unmapping */
318 iser_conn
->rx_descs
= NULL
;
320 iser_free_login_buf(iser_conn
);
323 static int iser_post_rx_bufs(struct iscsi_conn
*conn
, struct iscsi_hdr
*req
)
325 struct iser_conn
*iser_conn
= conn
->dd_data
;
326 struct ib_conn
*ib_conn
= &iser_conn
->ib_conn
;
327 struct iscsi_session
*session
= conn
->session
;
329 iser_dbg("req op %x flags %x\n", req
->opcode
, req
->flags
);
330 /* check if this is the last login - going to full feature phase */
331 if ((req
->flags
& ISCSI_FULL_FEATURE_PHASE
) != ISCSI_FULL_FEATURE_PHASE
)
335 * Check that there is one posted recv buffer
336 * (for the last login response).
338 WARN_ON(ib_conn
->post_recv_buf_count
!= 1);
340 if (session
->discovery_sess
) {
341 iser_info("Discovery session, re-using login RX buffer\n");
344 iser_info("Normal session, posting batch of RX %d buffers\n",
345 iser_conn
->min_posted_rx
);
347 /* Initial post receive buffers */
348 if (iser_post_recvm(iser_conn
, iser_conn
->min_posted_rx
))
354 static inline bool iser_signal_comp(u8 sig_count
)
356 return ((sig_count
% ISER_SIGNAL_CMD_COUNT
) == 0);
360 * iser_send_command - send command PDU
362 int iser_send_command(struct iscsi_conn
*conn
,
363 struct iscsi_task
*task
)
365 struct iser_conn
*iser_conn
= conn
->dd_data
;
366 struct iscsi_iser_task
*iser_task
= task
->dd_data
;
369 struct iser_data_buf
*data_buf
, *prot_buf
;
370 struct iscsi_scsi_req
*hdr
= (struct iscsi_scsi_req
*)task
->hdr
;
371 struct scsi_cmnd
*sc
= task
->sc
;
372 struct iser_tx_desc
*tx_desc
= &iser_task
->desc
;
373 u8 sig_count
= ++iser_conn
->ib_conn
.sig_count
;
375 edtl
= ntohl(hdr
->data_length
);
377 /* build the tx desc regd header and add it to the tx desc dto */
378 tx_desc
->type
= ISCSI_TX_SCSI_COMMAND
;
379 tx_desc
->cqe
.done
= iser_cmd_comp
;
380 iser_create_send_desc(iser_conn
, tx_desc
);
382 if (hdr
->flags
& ISCSI_FLAG_CMD_READ
) {
383 data_buf
= &iser_task
->data
[ISER_DIR_IN
];
384 prot_buf
= &iser_task
->prot
[ISER_DIR_IN
];
386 data_buf
= &iser_task
->data
[ISER_DIR_OUT
];
387 prot_buf
= &iser_task
->prot
[ISER_DIR_OUT
];
390 if (scsi_sg_count(sc
)) { /* using a scatter list */
391 data_buf
->sg
= scsi_sglist(sc
);
392 data_buf
->size
= scsi_sg_count(sc
);
394 data_buf
->data_len
= scsi_bufflen(sc
);
396 if (scsi_prot_sg_count(sc
)) {
397 prot_buf
->sg
= scsi_prot_sglist(sc
);
398 prot_buf
->size
= scsi_prot_sg_count(sc
);
399 prot_buf
->data_len
= (data_buf
->data_len
>>
400 ilog2(sc
->device
->sector_size
)) * 8;
403 if (hdr
->flags
& ISCSI_FLAG_CMD_READ
) {
404 err
= iser_prepare_read_cmd(task
);
406 goto send_command_error
;
408 if (hdr
->flags
& ISCSI_FLAG_CMD_WRITE
) {
409 err
= iser_prepare_write_cmd(task
,
412 task
->unsol_r2t
.data_length
,
415 goto send_command_error
;
418 iser_task
->status
= ISER_TASK_STATUS_STARTED
;
420 err
= iser_post_send(&iser_conn
->ib_conn
, tx_desc
,
421 iser_signal_comp(sig_count
));
426 iser_err("conn %p failed task->itt %d err %d\n",conn
, task
->itt
, err
);
431 * iser_send_data_out - send data out PDU
433 int iser_send_data_out(struct iscsi_conn
*conn
,
434 struct iscsi_task
*task
,
435 struct iscsi_data
*hdr
)
437 struct iser_conn
*iser_conn
= conn
->dd_data
;
438 struct iscsi_iser_task
*iser_task
= task
->dd_data
;
439 struct iser_tx_desc
*tx_desc
= NULL
;
440 struct iser_mem_reg
*mem_reg
;
441 unsigned long buf_offset
;
442 unsigned long data_seg_len
;
445 struct ib_sge
*tx_dsg
;
447 itt
= (__force
uint32_t)hdr
->itt
;
448 data_seg_len
= ntoh24(hdr
->dlength
);
449 buf_offset
= ntohl(hdr
->offset
);
451 iser_dbg("%s itt %d dseg_len %d offset %d\n",
452 __func__
,(int)itt
,(int)data_seg_len
,(int)buf_offset
);
454 tx_desc
= kmem_cache_zalloc(ig
.desc_cache
, GFP_ATOMIC
);
455 if (tx_desc
== NULL
) {
456 iser_err("Failed to alloc desc for post dataout\n");
460 tx_desc
->type
= ISCSI_TX_DATAOUT
;
461 tx_desc
->cqe
.done
= iser_dataout_comp
;
462 tx_desc
->iser_header
.flags
= ISER_VER
;
463 memcpy(&tx_desc
->iscsi_header
, hdr
, sizeof(struct iscsi_hdr
));
465 /* build the tx desc */
466 err
= iser_initialize_task_headers(task
, tx_desc
);
468 goto send_data_out_error
;
470 mem_reg
= &iser_task
->rdma_reg
[ISER_DIR_OUT
];
471 tx_dsg
= &tx_desc
->tx_sg
[1];
472 tx_dsg
->addr
= mem_reg
->sge
.addr
+ buf_offset
;
473 tx_dsg
->length
= data_seg_len
;
474 tx_dsg
->lkey
= mem_reg
->sge
.lkey
;
475 tx_desc
->num_sge
= 2;
477 if (buf_offset
+ data_seg_len
> iser_task
->data
[ISER_DIR_OUT
].data_len
) {
478 iser_err("Offset:%ld & DSL:%ld in Data-Out "
479 "inconsistent with total len:%ld, itt:%d\n",
480 buf_offset
, data_seg_len
,
481 iser_task
->data
[ISER_DIR_OUT
].data_len
, itt
);
483 goto send_data_out_error
;
485 iser_dbg("data-out itt: %d, offset: %ld, sz: %ld\n",
486 itt
, buf_offset
, data_seg_len
);
489 err
= iser_post_send(&iser_conn
->ib_conn
, tx_desc
, true);
494 kmem_cache_free(ig
.desc_cache
, tx_desc
);
495 iser_err("conn %p failed err %d\n", conn
, err
);
499 int iser_send_control(struct iscsi_conn
*conn
,
500 struct iscsi_task
*task
)
502 struct iser_conn
*iser_conn
= conn
->dd_data
;
503 struct iscsi_iser_task
*iser_task
= task
->dd_data
;
504 struct iser_tx_desc
*mdesc
= &iser_task
->desc
;
505 unsigned long data_seg_len
;
507 struct iser_device
*device
;
509 /* build the tx desc regd header and add it to the tx desc dto */
510 mdesc
->type
= ISCSI_TX_CONTROL
;
511 mdesc
->cqe
.done
= iser_ctrl_comp
;
512 iser_create_send_desc(iser_conn
, mdesc
);
514 device
= iser_conn
->ib_conn
.device
;
516 data_seg_len
= ntoh24(task
->hdr
->dlength
);
518 if (data_seg_len
> 0) {
519 struct iser_login_desc
*desc
= &iser_conn
->login_desc
;
520 struct ib_sge
*tx_dsg
= &mdesc
->tx_sg
[1];
522 if (task
!= conn
->login_task
) {
523 iser_err("data present on non login task!!!\n");
524 goto send_control_error
;
527 ib_dma_sync_single_for_cpu(device
->ib_device
, desc
->req_dma
,
528 task
->data_count
, DMA_TO_DEVICE
);
530 memcpy(desc
->req
, task
->data
, task
->data_count
);
532 ib_dma_sync_single_for_device(device
->ib_device
, desc
->req_dma
,
533 task
->data_count
, DMA_TO_DEVICE
);
535 tx_dsg
->addr
= desc
->req_dma
;
536 tx_dsg
->length
= task
->data_count
;
537 tx_dsg
->lkey
= device
->pd
->local_dma_lkey
;
541 if (task
== conn
->login_task
) {
542 iser_dbg("op %x dsl %lx, posting login rx buffer\n",
543 task
->hdr
->opcode
, data_seg_len
);
544 err
= iser_post_recvl(iser_conn
);
546 goto send_control_error
;
547 err
= iser_post_rx_bufs(conn
, task
->hdr
);
549 goto send_control_error
;
552 err
= iser_post_send(&iser_conn
->ib_conn
, mdesc
, true);
557 iser_err("conn %p failed err %d\n",conn
, err
);
561 void iser_login_rsp(struct ib_cq
*cq
, struct ib_wc
*wc
)
563 struct ib_conn
*ib_conn
= wc
->qp
->qp_context
;
564 struct iser_conn
*iser_conn
= to_iser_conn(ib_conn
);
565 struct iser_login_desc
*desc
= iser_login(wc
->wr_cqe
);
566 struct iscsi_hdr
*hdr
;
570 if (unlikely(wc
->status
!= IB_WC_SUCCESS
)) {
571 iser_err_comp(wc
, "login_rsp");
575 ib_dma_sync_single_for_cpu(ib_conn
->device
->ib_device
,
576 desc
->rsp_dma
, ISER_RX_LOGIN_SIZE
,
579 hdr
= desc
->rsp
+ sizeof(struct iser_ctrl
);
580 data
= desc
->rsp
+ ISER_HEADERS_LEN
;
581 length
= wc
->byte_len
- ISER_HEADERS_LEN
;
583 iser_dbg("op 0x%x itt 0x%x dlen %d\n", hdr
->opcode
,
586 iscsi_iser_recv(iser_conn
->iscsi_conn
, hdr
, data
, length
);
588 ib_dma_sync_single_for_device(ib_conn
->device
->ib_device
,
589 desc
->rsp_dma
, ISER_RX_LOGIN_SIZE
,
592 ib_conn
->post_recv_buf_count
--;
596 iser_inv_desc(struct iser_fr_desc
*desc
, u32 rkey
)
598 if (likely(rkey
== desc
->rsc
.mr
->rkey
))
599 desc
->rsc
.mr_valid
= 0;
600 else if (likely(rkey
== desc
->pi_ctx
->sig_mr
->rkey
))
601 desc
->pi_ctx
->sig_mr_valid
= 0;
605 iser_check_remote_inv(struct iser_conn
*iser_conn
,
607 struct iscsi_hdr
*hdr
)
609 if (wc
->wc_flags
& IB_WC_WITH_INVALIDATE
) {
610 struct iscsi_task
*task
;
611 u32 rkey
= wc
->ex
.invalidate_rkey
;
613 iser_dbg("conn %p: remote invalidation for rkey %#x\n",
616 if (unlikely(!iser_conn
->snd_w_inv
)) {
617 iser_err("conn %p: unexpected remote invalidation, "
618 "terminating connection\n", iser_conn
);
622 task
= iscsi_itt_to_ctask(iser_conn
->iscsi_conn
, hdr
->itt
);
624 struct iscsi_iser_task
*iser_task
= task
->dd_data
;
625 struct iser_fr_desc
*desc
;
627 if (iser_task
->dir
[ISER_DIR_IN
]) {
628 desc
= iser_task
->rdma_reg
[ISER_DIR_IN
].mem_h
;
629 iser_inv_desc(desc
, rkey
);
632 if (iser_task
->dir
[ISER_DIR_OUT
]) {
633 desc
= iser_task
->rdma_reg
[ISER_DIR_OUT
].mem_h
;
634 iser_inv_desc(desc
, rkey
);
637 iser_err("failed to get task for itt=%d\n", hdr
->itt
);
646 void iser_task_rsp(struct ib_cq
*cq
, struct ib_wc
*wc
)
648 struct ib_conn
*ib_conn
= wc
->qp
->qp_context
;
649 struct iser_conn
*iser_conn
= to_iser_conn(ib_conn
);
650 struct iser_rx_desc
*desc
= iser_rx(wc
->wr_cqe
);
651 struct iscsi_hdr
*hdr
;
653 int outstanding
, count
, err
;
655 if (unlikely(wc
->status
!= IB_WC_SUCCESS
)) {
656 iser_err_comp(wc
, "task_rsp");
660 ib_dma_sync_single_for_cpu(ib_conn
->device
->ib_device
,
661 desc
->dma_addr
, ISER_RX_PAYLOAD_SIZE
,
664 hdr
= &desc
->iscsi_header
;
665 length
= wc
->byte_len
- ISER_HEADERS_LEN
;
667 iser_dbg("op 0x%x itt 0x%x dlen %d\n", hdr
->opcode
,
670 if (iser_check_remote_inv(iser_conn
, wc
, hdr
)) {
671 iscsi_conn_failure(iser_conn
->iscsi_conn
,
672 ISCSI_ERR_CONN_FAILED
);
676 iscsi_iser_recv(iser_conn
->iscsi_conn
, hdr
, desc
->data
, length
);
678 ib_dma_sync_single_for_device(ib_conn
->device
->ib_device
,
679 desc
->dma_addr
, ISER_RX_PAYLOAD_SIZE
,
682 /* decrementing conn->post_recv_buf_count only --after-- freeing the *
683 * task eliminates the need to worry on tasks which are completed in *
684 * parallel to the execution of iser_conn_term. So the code that waits *
685 * for the posted rx bufs refcount to become zero handles everything */
686 ib_conn
->post_recv_buf_count
--;
688 outstanding
= ib_conn
->post_recv_buf_count
;
689 if (outstanding
+ iser_conn
->min_posted_rx
<= iser_conn
->qp_max_recv_dtos
) {
690 count
= min(iser_conn
->qp_max_recv_dtos
- outstanding
,
691 iser_conn
->min_posted_rx
);
692 err
= iser_post_recvm(iser_conn
, count
);
694 iser_err("posting %d rx bufs err %d\n", count
, err
);
698 void iser_cmd_comp(struct ib_cq
*cq
, struct ib_wc
*wc
)
700 if (unlikely(wc
->status
!= IB_WC_SUCCESS
))
701 iser_err_comp(wc
, "command");
704 void iser_ctrl_comp(struct ib_cq
*cq
, struct ib_wc
*wc
)
706 struct iser_tx_desc
*desc
= iser_tx(wc
->wr_cqe
);
707 struct iscsi_task
*task
;
709 if (unlikely(wc
->status
!= IB_WC_SUCCESS
)) {
710 iser_err_comp(wc
, "control");
714 /* this arithmetic is legal by libiscsi dd_data allocation */
715 task
= (void *)desc
- sizeof(struct iscsi_task
);
716 if (task
->hdr
->itt
== RESERVED_ITT
)
717 iscsi_put_task(task
);
720 void iser_dataout_comp(struct ib_cq
*cq
, struct ib_wc
*wc
)
722 struct iser_tx_desc
*desc
= iser_tx(wc
->wr_cqe
);
723 struct ib_conn
*ib_conn
= wc
->qp
->qp_context
;
724 struct iser_device
*device
= ib_conn
->device
;
726 if (unlikely(wc
->status
!= IB_WC_SUCCESS
))
727 iser_err_comp(wc
, "dataout");
729 ib_dma_unmap_single(device
->ib_device
, desc
->dma_addr
,
730 ISER_HEADERS_LEN
, DMA_TO_DEVICE
);
731 kmem_cache_free(ig
.desc_cache
, desc
);
734 void iser_task_rdma_init(struct iscsi_iser_task
*iser_task
)
737 iser_task
->status
= ISER_TASK_STATUS_INIT
;
739 iser_task
->dir
[ISER_DIR_IN
] = 0;
740 iser_task
->dir
[ISER_DIR_OUT
] = 0;
742 iser_task
->data
[ISER_DIR_IN
].data_len
= 0;
743 iser_task
->data
[ISER_DIR_OUT
].data_len
= 0;
745 iser_task
->prot
[ISER_DIR_IN
].data_len
= 0;
746 iser_task
->prot
[ISER_DIR_OUT
].data_len
= 0;
748 memset(&iser_task
->rdma_reg
[ISER_DIR_IN
], 0,
749 sizeof(struct iser_mem_reg
));
750 memset(&iser_task
->rdma_reg
[ISER_DIR_OUT
], 0,
751 sizeof(struct iser_mem_reg
));
754 void iser_task_rdma_finalize(struct iscsi_iser_task
*iser_task
)
756 int prot_count
= scsi_prot_sg_count(iser_task
->sc
);
758 if (iser_task
->dir
[ISER_DIR_IN
]) {
759 iser_unreg_rdma_mem(iser_task
, ISER_DIR_IN
);
760 iser_dma_unmap_task_data(iser_task
,
761 &iser_task
->data
[ISER_DIR_IN
],
764 iser_dma_unmap_task_data(iser_task
,
765 &iser_task
->prot
[ISER_DIR_IN
],
769 if (iser_task
->dir
[ISER_DIR_OUT
]) {
770 iser_unreg_rdma_mem(iser_task
, ISER_DIR_OUT
);
771 iser_dma_unmap_task_data(iser_task
,
772 &iser_task
->data
[ISER_DIR_OUT
],
775 iser_dma_unmap_task_data(iser_task
,
776 &iser_task
->prot
[ISER_DIR_OUT
],