2 * QLogic iSCSI Offload Driver
3 * Copyright (c) 2016 Cavium Inc.
5 * This software is available under the terms of the GNU General Public License
6 * (GPL) Version 2, available from the file COPYING in the main directory of
10 #include <linux/blkdev.h>
11 #include <scsi/scsi_tcq.h>
12 #include <linux/delay.h>
15 #include "qedi_iscsi.h"
18 static int qedi_send_iscsi_tmf(struct qedi_conn
*qedi_conn
,
19 struct iscsi_task
*mtask
);
21 void qedi_iscsi_unmap_sg_list(struct qedi_cmd
*cmd
)
23 struct scsi_cmnd
*sc
= cmd
->scsi_cmd
;
25 if (cmd
->io_tbl
.sge_valid
&& sc
) {
26 cmd
->io_tbl
.sge_valid
= 0;
31 static void qedi_process_logout_resp(struct qedi_ctx
*qedi
,
33 struct iscsi_task
*task
,
34 struct qedi_conn
*qedi_conn
)
36 struct iscsi_conn
*conn
= qedi_conn
->cls_conn
->dd_data
;
37 struct iscsi_logout_rsp
*resp_hdr
;
38 struct iscsi_session
*session
= conn
->session
;
39 struct iscsi_logout_response_hdr
*cqe_logout_response
;
42 cmd
= (struct qedi_cmd
*)task
->dd_data
;
43 cqe_logout_response
= &cqe
->cqe_common
.iscsi_hdr
.logout_response
;
44 spin_lock(&session
->back_lock
);
45 resp_hdr
= (struct iscsi_logout_rsp
*)&qedi_conn
->gen_pdu
.resp_hdr
;
46 memset(resp_hdr
, 0, sizeof(struct iscsi_hdr
));
47 resp_hdr
->opcode
= cqe_logout_response
->opcode
;
48 resp_hdr
->flags
= cqe_logout_response
->flags
;
49 resp_hdr
->hlength
= 0;
51 resp_hdr
->itt
= build_itt(cqe
->cqe_solicited
.itid
, conn
->session
->age
);
52 resp_hdr
->statsn
= cpu_to_be32(cqe_logout_response
->stat_sn
);
53 resp_hdr
->exp_cmdsn
= cpu_to_be32(cqe_logout_response
->exp_cmd_sn
);
54 resp_hdr
->max_cmdsn
= cpu_to_be32(cqe_logout_response
->max_cmd_sn
);
56 resp_hdr
->t2wait
= cpu_to_be32(cqe_logout_response
->time2wait
);
57 resp_hdr
->t2retain
= cpu_to_be32(cqe_logout_response
->time2retain
);
59 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_TID
,
60 "Freeing tid=0x%x for cid=0x%x\n",
61 cmd
->task_id
, qedi_conn
->iscsi_conn_id
);
63 if (likely(cmd
->io_cmd_in_list
)) {
64 cmd
->io_cmd_in_list
= false;
65 list_del_init(&cmd
->io_cmd
);
66 qedi_conn
->active_cmd_count
--;
68 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_INFO
,
69 "Active cmd list node already deleted, tid=0x%x, cid=0x%x, io_cmd_node=%p\n",
70 cmd
->task_id
, qedi_conn
->iscsi_conn_id
,
74 cmd
->state
= RESPONSE_RECEIVED
;
75 qedi_clear_task_idx(qedi
, cmd
->task_id
);
76 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)resp_hdr
, NULL
, 0);
78 spin_unlock(&session
->back_lock
);
81 static void qedi_process_text_resp(struct qedi_ctx
*qedi
,
83 struct iscsi_task
*task
,
84 struct qedi_conn
*qedi_conn
)
86 struct iscsi_conn
*conn
= qedi_conn
->cls_conn
->dd_data
;
87 struct iscsi_session
*session
= conn
->session
;
88 struct iscsi_task_context
*task_ctx
;
89 struct iscsi_text_rsp
*resp_hdr_ptr
;
90 struct iscsi_text_response_hdr
*cqe_text_response
;
95 cmd
= (struct qedi_cmd
*)task
->dd_data
;
96 task_ctx
= qedi_get_task_mem(&qedi
->tasks
, cmd
->task_id
);
98 cqe_text_response
= &cqe
->cqe_common
.iscsi_hdr
.text_response
;
99 spin_lock(&session
->back_lock
);
100 resp_hdr_ptr
= (struct iscsi_text_rsp
*)&qedi_conn
->gen_pdu
.resp_hdr
;
101 memset(resp_hdr_ptr
, 0, sizeof(struct iscsi_hdr
));
102 resp_hdr_ptr
->opcode
= cqe_text_response
->opcode
;
103 resp_hdr_ptr
->flags
= cqe_text_response
->flags
;
104 resp_hdr_ptr
->hlength
= 0;
106 hton24(resp_hdr_ptr
->dlength
,
107 (cqe_text_response
->hdr_second_dword
&
108 ISCSI_TEXT_RESPONSE_HDR_DATA_SEG_LEN_MASK
));
109 tmp
= (u32
*)resp_hdr_ptr
->dlength
;
111 resp_hdr_ptr
->itt
= build_itt(cqe
->cqe_solicited
.itid
,
113 resp_hdr_ptr
->ttt
= cqe_text_response
->ttt
;
114 resp_hdr_ptr
->statsn
= cpu_to_be32(cqe_text_response
->stat_sn
);
115 resp_hdr_ptr
->exp_cmdsn
= cpu_to_be32(cqe_text_response
->exp_cmd_sn
);
116 resp_hdr_ptr
->max_cmdsn
= cpu_to_be32(cqe_text_response
->max_cmd_sn
);
118 pld_len
= cqe_text_response
->hdr_second_dword
&
119 ISCSI_TEXT_RESPONSE_HDR_DATA_SEG_LEN_MASK
;
120 qedi_conn
->gen_pdu
.resp_wr_ptr
= qedi_conn
->gen_pdu
.resp_buf
+ pld_len
;
122 memset(task_ctx
, '\0', sizeof(*task_ctx
));
124 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_TID
,
125 "Freeing tid=0x%x for cid=0x%x\n",
126 cmd
->task_id
, qedi_conn
->iscsi_conn_id
);
128 if (likely(cmd
->io_cmd_in_list
)) {
129 cmd
->io_cmd_in_list
= false;
130 list_del_init(&cmd
->io_cmd
);
131 qedi_conn
->active_cmd_count
--;
133 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_INFO
,
134 "Active cmd list node already deleted, tid=0x%x, cid=0x%x, io_cmd_node=%p\n",
135 cmd
->task_id
, qedi_conn
->iscsi_conn_id
,
139 cmd
->state
= RESPONSE_RECEIVED
;
140 qedi_clear_task_idx(qedi
, cmd
->task_id
);
142 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)resp_hdr_ptr
,
143 qedi_conn
->gen_pdu
.resp_buf
,
144 (qedi_conn
->gen_pdu
.resp_wr_ptr
-
145 qedi_conn
->gen_pdu
.resp_buf
));
146 spin_unlock(&session
->back_lock
);
149 static void qedi_tmf_resp_work(struct work_struct
*work
)
151 struct qedi_cmd
*qedi_cmd
=
152 container_of(work
, struct qedi_cmd
, tmf_work
);
153 struct qedi_conn
*qedi_conn
= qedi_cmd
->conn
;
154 struct qedi_ctx
*qedi
= qedi_conn
->qedi
;
155 struct iscsi_conn
*conn
= qedi_conn
->cls_conn
->dd_data
;
156 struct iscsi_session
*session
= conn
->session
;
157 struct iscsi_tm_rsp
*resp_hdr_ptr
;
158 struct iscsi_cls_session
*cls_sess
;
161 set_bit(QEDI_CONN_FW_CLEANUP
, &qedi_conn
->flags
);
162 resp_hdr_ptr
= (struct iscsi_tm_rsp
*)qedi_cmd
->tmf_resp_buf
;
163 cls_sess
= iscsi_conn_to_session(qedi_conn
->cls_conn
);
165 iscsi_block_session(session
->cls_session
);
166 rval
= qedi_cleanup_all_io(qedi
, qedi_conn
, qedi_cmd
->task
, true);
168 qedi_clear_task_idx(qedi
, qedi_cmd
->task_id
);
169 iscsi_unblock_session(session
->cls_session
);
173 iscsi_unblock_session(session
->cls_session
);
174 qedi_clear_task_idx(qedi
, qedi_cmd
->task_id
);
176 spin_lock(&session
->back_lock
);
177 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)resp_hdr_ptr
, NULL
, 0);
178 spin_unlock(&session
->back_lock
);
182 clear_bit(QEDI_CONN_FW_CLEANUP
, &qedi_conn
->flags
);
185 static void qedi_process_tmf_resp(struct qedi_ctx
*qedi
,
186 union iscsi_cqe
*cqe
,
187 struct iscsi_task
*task
,
188 struct qedi_conn
*qedi_conn
)
191 struct iscsi_conn
*conn
= qedi_conn
->cls_conn
->dd_data
;
192 struct iscsi_session
*session
= conn
->session
;
193 struct iscsi_tmf_response_hdr
*cqe_tmp_response
;
194 struct iscsi_tm_rsp
*resp_hdr_ptr
;
195 struct iscsi_tm
*tmf_hdr
;
196 struct qedi_cmd
*qedi_cmd
= NULL
;
199 cqe_tmp_response
= &cqe
->cqe_common
.iscsi_hdr
.tmf_response
;
201 qedi_cmd
= task
->dd_data
;
202 qedi_cmd
->tmf_resp_buf
= kzalloc(sizeof(*resp_hdr_ptr
), GFP_KERNEL
);
203 if (!qedi_cmd
->tmf_resp_buf
) {
204 QEDI_ERR(&qedi
->dbg_ctx
,
205 "Failed to allocate resp buf, cid=0x%x\n",
206 qedi_conn
->iscsi_conn_id
);
210 spin_lock(&session
->back_lock
);
211 resp_hdr_ptr
= (struct iscsi_tm_rsp
*)qedi_cmd
->tmf_resp_buf
;
212 memset(resp_hdr_ptr
, 0, sizeof(struct iscsi_tm_rsp
));
214 /* Fill up the header */
215 resp_hdr_ptr
->opcode
= cqe_tmp_response
->opcode
;
216 resp_hdr_ptr
->flags
= cqe_tmp_response
->hdr_flags
;
217 resp_hdr_ptr
->response
= cqe_tmp_response
->hdr_response
;
218 resp_hdr_ptr
->hlength
= 0;
220 hton24(resp_hdr_ptr
->dlength
,
221 (cqe_tmp_response
->hdr_second_dword
&
222 ISCSI_TMF_RESPONSE_HDR_DATA_SEG_LEN_MASK
));
223 tmp
= (u32
*)resp_hdr_ptr
->dlength
;
224 resp_hdr_ptr
->itt
= build_itt(cqe
->cqe_solicited
.itid
,
226 resp_hdr_ptr
->statsn
= cpu_to_be32(cqe_tmp_response
->stat_sn
);
227 resp_hdr_ptr
->exp_cmdsn
= cpu_to_be32(cqe_tmp_response
->exp_cmd_sn
);
228 resp_hdr_ptr
->max_cmdsn
= cpu_to_be32(cqe_tmp_response
->max_cmd_sn
);
230 tmf_hdr
= (struct iscsi_tm
*)qedi_cmd
->task
->hdr
;
232 if (likely(qedi_cmd
->io_cmd_in_list
)) {
233 qedi_cmd
->io_cmd_in_list
= false;
234 list_del_init(&qedi_cmd
->io_cmd
);
235 qedi_conn
->active_cmd_count
--;
238 if (((tmf_hdr
->flags
& ISCSI_FLAG_TM_FUNC_MASK
) ==
239 ISCSI_TM_FUNC_LOGICAL_UNIT_RESET
) ||
240 ((tmf_hdr
->flags
& ISCSI_FLAG_TM_FUNC_MASK
) ==
241 ISCSI_TM_FUNC_TARGET_WARM_RESET
) ||
242 ((tmf_hdr
->flags
& ISCSI_FLAG_TM_FUNC_MASK
) ==
243 ISCSI_TM_FUNC_TARGET_COLD_RESET
)) {
244 INIT_WORK(&qedi_cmd
->tmf_work
, qedi_tmf_resp_work
);
245 queue_work(qedi
->tmf_thread
, &qedi_cmd
->tmf_work
);
249 qedi_clear_task_idx(qedi
, qedi_cmd
->task_id
);
251 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)resp_hdr_ptr
, NULL
, 0);
255 spin_unlock(&session
->back_lock
);
258 static void qedi_process_login_resp(struct qedi_ctx
*qedi
,
259 union iscsi_cqe
*cqe
,
260 struct iscsi_task
*task
,
261 struct qedi_conn
*qedi_conn
)
263 struct iscsi_conn
*conn
= qedi_conn
->cls_conn
->dd_data
;
264 struct iscsi_session
*session
= conn
->session
;
265 struct iscsi_task_context
*task_ctx
;
266 struct iscsi_login_rsp
*resp_hdr_ptr
;
267 struct iscsi_login_response_hdr
*cqe_login_response
;
268 struct qedi_cmd
*cmd
;
272 cmd
= (struct qedi_cmd
*)task
->dd_data
;
274 cqe_login_response
= &cqe
->cqe_common
.iscsi_hdr
.login_response
;
275 task_ctx
= qedi_get_task_mem(&qedi
->tasks
, cmd
->task_id
);
277 spin_lock(&session
->back_lock
);
278 resp_hdr_ptr
= (struct iscsi_login_rsp
*)&qedi_conn
->gen_pdu
.resp_hdr
;
279 memset(resp_hdr_ptr
, 0, sizeof(struct iscsi_login_rsp
));
280 resp_hdr_ptr
->opcode
= cqe_login_response
->opcode
;
281 resp_hdr_ptr
->flags
= cqe_login_response
->flags_attr
;
282 resp_hdr_ptr
->hlength
= 0;
284 hton24(resp_hdr_ptr
->dlength
,
285 (cqe_login_response
->hdr_second_dword
&
286 ISCSI_LOGIN_RESPONSE_HDR_DATA_SEG_LEN_MASK
));
287 tmp
= (u32
*)resp_hdr_ptr
->dlength
;
288 resp_hdr_ptr
->itt
= build_itt(cqe
->cqe_solicited
.itid
,
290 resp_hdr_ptr
->tsih
= cqe_login_response
->tsih
;
291 resp_hdr_ptr
->statsn
= cpu_to_be32(cqe_login_response
->stat_sn
);
292 resp_hdr_ptr
->exp_cmdsn
= cpu_to_be32(cqe_login_response
->exp_cmd_sn
);
293 resp_hdr_ptr
->max_cmdsn
= cpu_to_be32(cqe_login_response
->max_cmd_sn
);
294 resp_hdr_ptr
->status_class
= cqe_login_response
->status_class
;
295 resp_hdr_ptr
->status_detail
= cqe_login_response
->status_detail
;
296 pld_len
= cqe_login_response
->hdr_second_dword
&
297 ISCSI_LOGIN_RESPONSE_HDR_DATA_SEG_LEN_MASK
;
298 qedi_conn
->gen_pdu
.resp_wr_ptr
= qedi_conn
->gen_pdu
.resp_buf
+ pld_len
;
300 if (likely(cmd
->io_cmd_in_list
)) {
301 cmd
->io_cmd_in_list
= false;
302 list_del_init(&cmd
->io_cmd
);
303 qedi_conn
->active_cmd_count
--;
306 memset(task_ctx
, '\0', sizeof(*task_ctx
));
308 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)resp_hdr_ptr
,
309 qedi_conn
->gen_pdu
.resp_buf
,
310 (qedi_conn
->gen_pdu
.resp_wr_ptr
-
311 qedi_conn
->gen_pdu
.resp_buf
));
313 spin_unlock(&session
->back_lock
);
314 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_TID
,
315 "Freeing tid=0x%x for cid=0x%x\n",
316 cmd
->task_id
, qedi_conn
->iscsi_conn_id
);
317 cmd
->state
= RESPONSE_RECEIVED
;
318 qedi_clear_task_idx(qedi
, cmd
->task_id
);
321 static void qedi_get_rq_bdq_buf(struct qedi_ctx
*qedi
,
322 struct iscsi_cqe_unsolicited
*cqe
,
327 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_CONN
,
328 "pld_len [%d], bdq_prod_idx [%d], idx [%d]\n",
329 len
, qedi
->bdq_prod_idx
,
330 (qedi
->bdq_prod_idx
% qedi
->rq_num_entries
));
332 /* Obtain buffer address from rqe_opaque */
333 idx
= cqe
->rqe_opaque
.lo
;
334 if ((idx
< 0) || (idx
> (QEDI_BDQ_NUM
- 1))) {
335 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_CONN
,
336 "wrong idx %d returned by FW, dropping the unsolicited pkt\n",
341 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_CONN
,
342 "rqe_opaque.lo [0x%p], rqe_opaque.hi [0x%p], idx [%d]\n",
343 cqe
->rqe_opaque
.lo
, cqe
->rqe_opaque
.hi
, idx
);
345 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_CONN
,
346 "unsol_cqe_type = %d\n", cqe
->unsol_cqe_type
);
347 switch (cqe
->unsol_cqe_type
) {
348 case ISCSI_CQE_UNSOLICITED_SINGLE
:
349 case ISCSI_CQE_UNSOLICITED_FIRST
:
351 memcpy(ptr
, (void *)qedi
->bdq
[idx
].buf_addr
, len
);
353 case ISCSI_CQE_UNSOLICITED_MIDDLE
:
354 case ISCSI_CQE_UNSOLICITED_LAST
:
361 static void qedi_put_rq_bdq_buf(struct qedi_ctx
*qedi
,
362 struct iscsi_cqe_unsolicited
*cqe
,
369 /* Obtain buffer address from rqe_opaque */
370 idx
= cqe
->rqe_opaque
.lo
;
371 if ((idx
< 0) || (idx
> (QEDI_BDQ_NUM
- 1))) {
372 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_CONN
,
373 "wrong idx %d returned by FW, dropping the unsolicited pkt\n",
378 pbl
= (struct scsi_bd
*)qedi
->bdq_pbl
;
379 pbl
+= (qedi
->bdq_prod_idx
% qedi
->rq_num_entries
);
380 pbl
->address
.hi
= cpu_to_le32(QEDI_U64_HI(qedi
->bdq
[idx
].buf_dma
));
381 pbl
->address
.lo
= cpu_to_le32(QEDI_U64_LO(qedi
->bdq
[idx
].buf_dma
));
382 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_CONN
,
383 "pbl [0x%p] pbl->address hi [0x%llx] lo [0x%llx] idx [%d]\n",
384 pbl
, pbl
->address
.hi
, pbl
->address
.lo
, idx
);
386 pbl
->opaque
.lo
= cpu_to_le32(QEDI_U64_LO(idx
));
388 /* Increment producer to let f/w know we've handled the frame */
389 qedi
->bdq_prod_idx
+= count
;
391 writew(qedi
->bdq_prod_idx
, qedi
->bdq_primary_prod
);
392 tmp
= readw(qedi
->bdq_primary_prod
);
394 writew(qedi
->bdq_prod_idx
, qedi
->bdq_secondary_prod
);
395 tmp
= readw(qedi
->bdq_secondary_prod
);
398 static void qedi_unsol_pdu_adjust_bdq(struct qedi_ctx
*qedi
,
399 struct iscsi_cqe_unsolicited
*cqe
,
400 u32 pdu_len
, u32 num_bdqs
,
403 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_CONN
,
404 "num_bdqs [%d]\n", num_bdqs
);
406 qedi_get_rq_bdq_buf(qedi
, cqe
, bdq_data
, pdu_len
);
407 qedi_put_rq_bdq_buf(qedi
, cqe
, (num_bdqs
+ 1));
410 static int qedi_process_nopin_mesg(struct qedi_ctx
*qedi
,
411 union iscsi_cqe
*cqe
,
412 struct iscsi_task
*task
,
413 struct qedi_conn
*qedi_conn
, u16 que_idx
)
415 struct iscsi_conn
*conn
= qedi_conn
->cls_conn
->dd_data
;
416 struct iscsi_session
*session
= conn
->session
;
417 struct iscsi_nop_in_hdr
*cqe_nop_in
;
418 struct iscsi_nopin
*hdr
;
419 struct qedi_cmd
*cmd
;
420 int tgt_async_nop
= 0;
422 u32 pdu_len
, num_bdqs
;
423 char bdq_data
[QEDI_BDQ_BUF_SIZE
];
426 spin_lock_bh(&session
->back_lock
);
427 cqe_nop_in
= &cqe
->cqe_common
.iscsi_hdr
.nop_in
;
429 pdu_len
= cqe_nop_in
->hdr_second_dword
&
430 ISCSI_NOP_IN_HDR_DATA_SEG_LEN_MASK
;
431 num_bdqs
= pdu_len
/ QEDI_BDQ_BUF_SIZE
;
433 hdr
= (struct iscsi_nopin
*)&qedi_conn
->gen_pdu
.resp_hdr
;
434 memset(hdr
, 0, sizeof(struct iscsi_hdr
));
435 hdr
->opcode
= cqe_nop_in
->opcode
;
436 hdr
->max_cmdsn
= cpu_to_be32(cqe_nop_in
->max_cmd_sn
);
437 hdr
->exp_cmdsn
= cpu_to_be32(cqe_nop_in
->exp_cmd_sn
);
438 hdr
->statsn
= cpu_to_be32(cqe_nop_in
->stat_sn
);
439 hdr
->ttt
= cpu_to_be32(cqe_nop_in
->ttt
);
441 if (cqe
->cqe_common
.cqe_type
== ISCSI_CQE_TYPE_UNSOLICITED
) {
442 spin_lock_irqsave(&qedi
->hba_lock
, flags
);
443 qedi_unsol_pdu_adjust_bdq(qedi
, &cqe
->cqe_unsolicited
,
444 pdu_len
, num_bdqs
, bdq_data
);
445 hdr
->itt
= RESERVED_ITT
;
447 spin_unlock_irqrestore(&qedi
->hba_lock
, flags
);
451 /* Response to one of our nop-outs */
454 hdr
->flags
= ISCSI_FLAG_CMD_FINAL
;
455 hdr
->itt
= build_itt(cqe
->cqe_solicited
.itid
,
459 memcpy(&hdr
->lun
, lun
, sizeof(struct scsi_lun
));
460 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_TID
,
461 "Freeing tid=0x%x for cid=0x%x\n",
462 cmd
->task_id
, qedi_conn
->iscsi_conn_id
);
463 cmd
->state
= RESPONSE_RECEIVED
;
464 spin_lock(&qedi_conn
->list_lock
);
465 if (likely(cmd
->io_cmd_in_list
)) {
466 cmd
->io_cmd_in_list
= false;
467 list_del_init(&cmd
->io_cmd
);
468 qedi_conn
->active_cmd_count
--;
471 spin_unlock(&qedi_conn
->list_lock
);
472 qedi_clear_task_idx(qedi
, cmd
->task_id
);
476 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)hdr
, bdq_data
, pdu_len
);
478 spin_unlock_bh(&session
->back_lock
);
479 return tgt_async_nop
;
482 static void qedi_process_async_mesg(struct qedi_ctx
*qedi
,
483 union iscsi_cqe
*cqe
,
484 struct iscsi_task
*task
,
485 struct qedi_conn
*qedi_conn
,
488 struct iscsi_conn
*conn
= qedi_conn
->cls_conn
->dd_data
;
489 struct iscsi_session
*session
= conn
->session
;
490 struct iscsi_async_msg_hdr
*cqe_async_msg
;
491 struct iscsi_async
*resp_hdr
;
493 u32 pdu_len
, num_bdqs
;
494 char bdq_data
[QEDI_BDQ_BUF_SIZE
];
497 spin_lock_bh(&session
->back_lock
);
499 cqe_async_msg
= &cqe
->cqe_common
.iscsi_hdr
.async_msg
;
500 pdu_len
= cqe_async_msg
->hdr_second_dword
&
501 ISCSI_ASYNC_MSG_HDR_DATA_SEG_LEN_MASK
;
502 num_bdqs
= pdu_len
/ QEDI_BDQ_BUF_SIZE
;
504 if (cqe
->cqe_common
.cqe_type
== ISCSI_CQE_TYPE_UNSOLICITED
) {
505 spin_lock_irqsave(&qedi
->hba_lock
, flags
);
506 qedi_unsol_pdu_adjust_bdq(qedi
, &cqe
->cqe_unsolicited
,
507 pdu_len
, num_bdqs
, bdq_data
);
508 spin_unlock_irqrestore(&qedi
->hba_lock
, flags
);
511 resp_hdr
= (struct iscsi_async
*)&qedi_conn
->gen_pdu
.resp_hdr
;
512 memset(resp_hdr
, 0, sizeof(struct iscsi_hdr
));
513 resp_hdr
->opcode
= cqe_async_msg
->opcode
;
514 resp_hdr
->flags
= 0x80;
516 lun
[0] = cpu_to_be32(cqe_async_msg
->lun
.lo
);
517 lun
[1] = cpu_to_be32(cqe_async_msg
->lun
.hi
);
518 memcpy(&resp_hdr
->lun
, lun
, sizeof(struct scsi_lun
));
519 resp_hdr
->exp_cmdsn
= cpu_to_be32(cqe_async_msg
->exp_cmd_sn
);
520 resp_hdr
->max_cmdsn
= cpu_to_be32(cqe_async_msg
->max_cmd_sn
);
521 resp_hdr
->statsn
= cpu_to_be32(cqe_async_msg
->stat_sn
);
523 resp_hdr
->async_event
= cqe_async_msg
->async_event
;
524 resp_hdr
->async_vcode
= cqe_async_msg
->async_vcode
;
526 resp_hdr
->param1
= cpu_to_be16(cqe_async_msg
->param1_rsrv
);
527 resp_hdr
->param2
= cpu_to_be16(cqe_async_msg
->param2_rsrv
);
528 resp_hdr
->param3
= cpu_to_be16(cqe_async_msg
->param3_rsrv
);
530 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)resp_hdr
, bdq_data
,
533 spin_unlock_bh(&session
->back_lock
);
536 static void qedi_process_reject_mesg(struct qedi_ctx
*qedi
,
537 union iscsi_cqe
*cqe
,
538 struct iscsi_task
*task
,
539 struct qedi_conn
*qedi_conn
,
542 struct iscsi_conn
*conn
= qedi_conn
->cls_conn
->dd_data
;
543 struct iscsi_session
*session
= conn
->session
;
544 struct iscsi_reject_hdr
*cqe_reject
;
545 struct iscsi_reject
*hdr
;
546 u32 pld_len
, num_bdqs
;
549 spin_lock_bh(&session
->back_lock
);
550 cqe_reject
= &cqe
->cqe_common
.iscsi_hdr
.reject
;
551 pld_len
= cqe_reject
->hdr_second_dword
&
552 ISCSI_REJECT_HDR_DATA_SEG_LEN_MASK
;
553 num_bdqs
= pld_len
/ QEDI_BDQ_BUF_SIZE
;
555 if (cqe
->cqe_common
.cqe_type
== ISCSI_CQE_TYPE_UNSOLICITED
) {
556 spin_lock_irqsave(&qedi
->hba_lock
, flags
);
557 qedi_unsol_pdu_adjust_bdq(qedi
, &cqe
->cqe_unsolicited
,
558 pld_len
, num_bdqs
, conn
->data
);
559 spin_unlock_irqrestore(&qedi
->hba_lock
, flags
);
561 hdr
= (struct iscsi_reject
*)&qedi_conn
->gen_pdu
.resp_hdr
;
562 memset(hdr
, 0, sizeof(struct iscsi_hdr
));
563 hdr
->opcode
= cqe_reject
->opcode
;
564 hdr
->reason
= cqe_reject
->hdr_reason
;
565 hdr
->flags
= cqe_reject
->hdr_flags
;
566 hton24(hdr
->dlength
, (cqe_reject
->hdr_second_dword
&
567 ISCSI_REJECT_HDR_DATA_SEG_LEN_MASK
));
568 hdr
->max_cmdsn
= cpu_to_be32(cqe_reject
->max_cmd_sn
);
569 hdr
->exp_cmdsn
= cpu_to_be32(cqe_reject
->exp_cmd_sn
);
570 hdr
->statsn
= cpu_to_be32(cqe_reject
->stat_sn
);
571 hdr
->ffffffff
= cpu_to_be32(0xffffffff);
573 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)hdr
,
574 conn
->data
, pld_len
);
575 spin_unlock_bh(&session
->back_lock
);
578 static void qedi_scsi_completion(struct qedi_ctx
*qedi
,
579 union iscsi_cqe
*cqe
,
580 struct iscsi_task
*task
,
581 struct iscsi_conn
*conn
)
583 struct scsi_cmnd
*sc_cmd
;
584 struct qedi_cmd
*cmd
= task
->dd_data
;
585 struct iscsi_session
*session
= conn
->session
;
586 struct iscsi_scsi_rsp
*hdr
;
587 struct iscsi_data_in_hdr
*cqe_data_in
;
589 struct qedi_conn
*qedi_conn
;
591 bool mark_cmd_node_deleted
= false;
594 iscsi_cid
= cqe
->cqe_common
.conn_id
;
595 qedi_conn
= qedi
->cid_que
.conn_cid_tbl
[iscsi_cid
];
597 cqe_data_in
= &cqe
->cqe_common
.iscsi_hdr
.data_in
;
599 cqe
->cqe_common
.error_bitmap
.error_bits
.cqe_error_status_bits
;
601 spin_lock_bh(&session
->back_lock
);
602 /* get the scsi command */
603 sc_cmd
= cmd
->scsi_cmd
;
606 QEDI_WARN(&qedi
->dbg_ctx
, "sc_cmd is NULL!\n");
610 if (!sc_cmd
->SCp
.ptr
) {
611 QEDI_WARN(&qedi
->dbg_ctx
,
612 "SCp.ptr is NULL, returned in another context.\n");
616 if (!sc_cmd
->request
) {
617 QEDI_WARN(&qedi
->dbg_ctx
,
618 "sc_cmd->request is NULL, sc_cmd=%p.\n",
623 if (!sc_cmd
->request
->special
) {
624 QEDI_WARN(&qedi
->dbg_ctx
,
625 "request->special is NULL so request not valid, sc_cmd=%p.\n",
630 if (!sc_cmd
->request
->q
) {
631 QEDI_WARN(&qedi
->dbg_ctx
,
632 "request->q is NULL so request is not valid, sc_cmd=%p.\n",
637 qedi_iscsi_unmap_sg_list(cmd
);
639 hdr
= (struct iscsi_scsi_rsp
*)task
->hdr
;
640 hdr
->opcode
= cqe_data_in
->opcode
;
641 hdr
->max_cmdsn
= cpu_to_be32(cqe_data_in
->max_cmd_sn
);
642 hdr
->exp_cmdsn
= cpu_to_be32(cqe_data_in
->exp_cmd_sn
);
643 hdr
->itt
= build_itt(cqe
->cqe_solicited
.itid
, conn
->session
->age
);
644 hdr
->response
= cqe_data_in
->reserved1
;
645 hdr
->cmd_status
= cqe_data_in
->status_rsvd
;
646 hdr
->flags
= cqe_data_in
->flags
;
647 hdr
->residual_count
= cpu_to_be32(cqe_data_in
->residual_count
);
649 if (hdr
->cmd_status
== SAM_STAT_CHECK_CONDITION
) {
650 datalen
= cqe_data_in
->reserved2
&
651 ISCSI_COMMON_HDR_DATA_SEG_LEN_MASK
;
652 memcpy((char *)conn
->data
, (char *)cmd
->sense_buffer
, datalen
);
655 /* If f/w reports data underrun err then set residual to IO transfer
656 * length, set Underrun flag and clear Overrun flag explicitly
658 if (unlikely(cqe_err_bits
&&
659 GET_FIELD(cqe_err_bits
, CQE_ERROR_BITMAP_UNDER_RUN_ERR
))) {
660 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_INFO
,
661 "Under flow itt=0x%x proto flags=0x%x tid=0x%x cid 0x%x fw resid 0x%x sc dlen 0x%x\n",
662 hdr
->itt
, cqe_data_in
->flags
, cmd
->task_id
,
663 qedi_conn
->iscsi_conn_id
, hdr
->residual_count
,
664 scsi_bufflen(sc_cmd
));
665 hdr
->residual_count
= cpu_to_be32(scsi_bufflen(sc_cmd
));
666 hdr
->flags
|= ISCSI_FLAG_CMD_UNDERFLOW
;
667 hdr
->flags
&= (~ISCSI_FLAG_CMD_OVERFLOW
);
670 spin_lock(&qedi_conn
->list_lock
);
671 if (likely(cmd
->io_cmd_in_list
)) {
672 cmd
->io_cmd_in_list
= false;
673 list_del_init(&cmd
->io_cmd
);
674 qedi_conn
->active_cmd_count
--;
675 mark_cmd_node_deleted
= true;
677 spin_unlock(&qedi_conn
->list_lock
);
679 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_TID
,
680 "Freeing tid=0x%x for cid=0x%x\n",
681 cmd
->task_id
, qedi_conn
->iscsi_conn_id
);
682 cmd
->state
= RESPONSE_RECEIVED
;
684 qedi_trace_io(qedi
, task
, cmd
->task_id
, QEDI_IO_TRACE_RSP
);
686 qedi_clear_task_idx(qedi
, cmd
->task_id
);
687 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)hdr
,
688 conn
->data
, datalen
);
690 spin_unlock_bh(&session
->back_lock
);
693 static void qedi_mtask_completion(struct qedi_ctx
*qedi
,
694 union iscsi_cqe
*cqe
,
695 struct iscsi_task
*task
,
696 struct qedi_conn
*conn
, uint16_t que_idx
)
698 struct iscsi_conn
*iscsi_conn
;
701 hdr_opcode
= cqe
->cqe_common
.iscsi_hdr
.common
.hdr_first_byte
;
702 iscsi_conn
= conn
->cls_conn
->dd_data
;
704 switch (hdr_opcode
) {
705 case ISCSI_OPCODE_SCSI_RESPONSE
:
706 case ISCSI_OPCODE_DATA_IN
:
707 qedi_scsi_completion(qedi
, cqe
, task
, iscsi_conn
);
709 case ISCSI_OPCODE_LOGIN_RESPONSE
:
710 qedi_process_login_resp(qedi
, cqe
, task
, conn
);
712 case ISCSI_OPCODE_TMF_RESPONSE
:
713 qedi_process_tmf_resp(qedi
, cqe
, task
, conn
);
715 case ISCSI_OPCODE_TEXT_RESPONSE
:
716 qedi_process_text_resp(qedi
, cqe
, task
, conn
);
718 case ISCSI_OPCODE_LOGOUT_RESPONSE
:
719 qedi_process_logout_resp(qedi
, cqe
, task
, conn
);
721 case ISCSI_OPCODE_NOP_IN
:
722 qedi_process_nopin_mesg(qedi
, cqe
, task
, conn
, que_idx
);
725 QEDI_ERR(&qedi
->dbg_ctx
, "unknown opcode\n");
729 static void qedi_process_nopin_local_cmpl(struct qedi_ctx
*qedi
,
730 struct iscsi_cqe_solicited
*cqe
,
731 struct iscsi_task
*task
,
732 struct qedi_conn
*qedi_conn
)
734 struct iscsi_conn
*conn
= qedi_conn
->cls_conn
->dd_data
;
735 struct iscsi_session
*session
= conn
->session
;
736 struct qedi_cmd
*cmd
= task
->dd_data
;
738 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_UNSOL
,
739 "itid=0x%x, cmd task id=0x%x\n",
740 cqe
->itid
, cmd
->task_id
);
742 cmd
->state
= RESPONSE_RECEIVED
;
743 qedi_clear_task_idx(qedi
, cmd
->task_id
);
745 spin_lock_bh(&session
->back_lock
);
746 __iscsi_put_task(task
);
747 spin_unlock_bh(&session
->back_lock
);
750 static void qedi_process_cmd_cleanup_resp(struct qedi_ctx
*qedi
,
751 struct iscsi_cqe_solicited
*cqe
,
752 struct iscsi_task
*task
,
753 struct iscsi_conn
*conn
)
755 struct qedi_work_map
*work
, *work_tmp
;
756 u32 proto_itt
= cqe
->itid
;
760 struct qedi_cmd
*qedi_cmd
= NULL
;
763 struct qedi_conn
*qedi_conn
;
764 struct qedi_cmd
*cmd_new
, *dbg_cmd
;
765 struct iscsi_task
*mtask
;
766 struct iscsi_tm
*tmf_hdr
= NULL
;
768 iscsi_cid
= cqe
->conn_id
;
769 qedi_conn
= qedi
->cid_que
.conn_cid_tbl
[iscsi_cid
];
771 /* Based on this itt get the corresponding qedi_cmd */
772 spin_lock_bh(&qedi_conn
->tmf_work_lock
);
773 list_for_each_entry_safe(work
, work_tmp
, &qedi_conn
->tmf_work_list
,
775 if (work
->rtid
== proto_itt
) {
776 /* We found the command */
777 qedi_cmd
= work
->qedi_cmd
;
778 if (!qedi_cmd
->list_tmf_work
) {
779 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_SCSI_TM
,
780 "TMF work not found, cqe->tid=0x%x, cid=0x%x\n",
781 proto_itt
, qedi_conn
->iscsi_conn_id
);
785 mtask
= qedi_cmd
->task
;
786 tmf_hdr
= (struct iscsi_tm
*)mtask
->hdr
;
789 list_del_init(&work
->list
);
791 qedi_cmd
->list_tmf_work
= NULL
;
794 spin_unlock_bh(&qedi_conn
->tmf_work_lock
);
797 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_SCSI_TM
,
798 "TMF work, cqe->tid=0x%x, tmf flags=0x%x, cid=0x%x\n",
799 proto_itt
, tmf_hdr
->flags
, qedi_conn
->iscsi_conn_id
);
801 if ((tmf_hdr
->flags
& ISCSI_FLAG_TM_FUNC_MASK
) ==
802 ISCSI_TM_FUNC_ABORT_TASK
) {
803 spin_lock_bh(&conn
->session
->back_lock
);
805 protoitt
= build_itt(get_itt(tmf_hdr
->rtt
),
807 task
= iscsi_itt_to_task(conn
, protoitt
);
809 spin_unlock_bh(&conn
->session
->back_lock
);
812 QEDI_NOTICE(&qedi
->dbg_ctx
,
813 "IO task completed, tmf rtt=0x%x, cid=0x%x\n",
814 get_itt(tmf_hdr
->rtt
),
815 qedi_conn
->iscsi_conn_id
);
819 dbg_cmd
= task
->dd_data
;
821 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_SCSI_TM
,
822 "Abort tmf rtt=0x%x, i/o itt=0x%x, i/o tid=0x%x, cid=0x%x\n",
823 get_itt(tmf_hdr
->rtt
), get_itt(task
->itt
),
824 dbg_cmd
->task_id
, qedi_conn
->iscsi_conn_id
);
826 if (qedi_cmd
->state
== CLEANUP_WAIT_FAILED
)
827 qedi_cmd
->state
= CLEANUP_RECV
;
829 qedi_clear_task_idx(qedi_conn
->qedi
, rtid
);
831 spin_lock(&qedi_conn
->list_lock
);
832 list_del_init(&dbg_cmd
->io_cmd
);
833 qedi_conn
->active_cmd_count
--;
834 spin_unlock(&qedi_conn
->list_lock
);
835 qedi_cmd
->state
= CLEANUP_RECV
;
836 wake_up_interruptible(&qedi_conn
->wait_queue
);
838 } else if (qedi_conn
->cmd_cleanup_req
> 0) {
839 spin_lock_bh(&conn
->session
->back_lock
);
840 qedi_get_proto_itt(qedi
, cqe
->itid
, &ptmp_itt
);
841 protoitt
= build_itt(ptmp_itt
, conn
->session
->age
);
842 task
= iscsi_itt_to_task(conn
, protoitt
);
843 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_SCSI_TM
,
844 "cleanup io itid=0x%x, protoitt=0x%x, cmd_cleanup_cmpl=%d, cid=0x%x\n",
845 cqe
->itid
, protoitt
, qedi_conn
->cmd_cleanup_cmpl
,
846 qedi_conn
->iscsi_conn_id
);
848 spin_unlock_bh(&conn
->session
->back_lock
);
850 QEDI_NOTICE(&qedi
->dbg_ctx
,
851 "task is null, itid=0x%x, cid=0x%x\n",
852 cqe
->itid
, qedi_conn
->iscsi_conn_id
);
855 qedi_conn
->cmd_cleanup_cmpl
++;
856 wake_up(&qedi_conn
->wait_queue
);
857 cmd_new
= task
->dd_data
;
859 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_TID
,
860 "Freeing tid=0x%x for cid=0x%x\n",
861 cqe
->itid
, qedi_conn
->iscsi_conn_id
);
862 qedi_clear_task_idx(qedi_conn
->qedi
, cqe
->itid
);
865 qedi_get_proto_itt(qedi
, cqe
->itid
, &ptmp_itt
);
866 protoitt
= build_itt(ptmp_itt
, conn
->session
->age
);
867 task
= iscsi_itt_to_task(conn
, protoitt
);
868 QEDI_ERR(&qedi
->dbg_ctx
,
869 "Delayed or untracked cleanup response, itt=0x%x, tid=0x%x, cid=0x%x, task=%p\n",
870 protoitt
, cqe
->itid
, qedi_conn
->iscsi_conn_id
, task
);
875 void qedi_fp_process_cqes(struct qedi_work
*work
)
877 struct qedi_ctx
*qedi
= work
->qedi
;
878 union iscsi_cqe
*cqe
= &work
->cqe
;
879 struct iscsi_task
*task
= NULL
;
880 struct iscsi_nopout
*nopout_hdr
;
881 struct qedi_conn
*q_conn
;
882 struct iscsi_conn
*conn
;
883 struct qedi_cmd
*qedi_cmd
;
887 u16 que_idx
= work
->que_idx
;
890 comp_type
= cqe
->cqe_common
.cqe_type
;
891 hdr_opcode
= cqe
->cqe_common
.iscsi_hdr
.common
.hdr_first_byte
;
893 cqe
->cqe_common
.error_bitmap
.error_bits
.cqe_error_status_bits
;
895 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_CONN
,
896 "fw_cid=0x%x, cqe type=0x%x, opcode=0x%x\n",
897 cqe
->cqe_common
.conn_id
, comp_type
, hdr_opcode
);
899 if (comp_type
>= MAX_ISCSI_CQES_TYPE
) {
900 QEDI_WARN(&qedi
->dbg_ctx
, "Invalid CqE type\n");
904 iscsi_cid
= cqe
->cqe_common
.conn_id
;
905 q_conn
= qedi
->cid_que
.conn_cid_tbl
[iscsi_cid
];
907 QEDI_WARN(&qedi
->dbg_ctx
,
908 "Session no longer exists for cid=0x%x!!\n",
913 conn
= q_conn
->cls_conn
->dd_data
;
915 if (unlikely(cqe_err_bits
&&
916 GET_FIELD(cqe_err_bits
,
917 CQE_ERROR_BITMAP_DATA_DIGEST_ERR
))) {
918 iscsi_conn_failure(conn
, ISCSI_ERR_DATA_DGST
);
923 case ISCSI_CQE_TYPE_SOLICITED
:
924 case ISCSI_CQE_TYPE_SOLICITED_WITH_SENSE
:
925 qedi_cmd
= container_of(work
, struct qedi_cmd
, cqe_work
);
926 task
= qedi_cmd
->task
;
928 QEDI_WARN(&qedi
->dbg_ctx
, "task is NULL\n");
932 /* Process NOPIN local completion */
933 nopout_hdr
= (struct iscsi_nopout
*)task
->hdr
;
934 if ((nopout_hdr
->itt
== RESERVED_ITT
) &&
935 (cqe
->cqe_solicited
.itid
!= (u16
)RESERVED_ITT
)) {
936 qedi_process_nopin_local_cmpl(qedi
, &cqe
->cqe_solicited
,
939 cqe
->cqe_solicited
.itid
=
940 qedi_get_itt(cqe
->cqe_solicited
);
941 /* Process other solicited responses */
942 qedi_mtask_completion(qedi
, cqe
, task
, q_conn
, que_idx
);
945 case ISCSI_CQE_TYPE_UNSOLICITED
:
946 switch (hdr_opcode
) {
947 case ISCSI_OPCODE_NOP_IN
:
948 qedi_process_nopin_mesg(qedi
, cqe
, task
, q_conn
,
951 case ISCSI_OPCODE_ASYNC_MSG
:
952 qedi_process_async_mesg(qedi
, cqe
, task
, q_conn
,
955 case ISCSI_OPCODE_REJECT
:
956 qedi_process_reject_mesg(qedi
, cqe
, task
, q_conn
,
960 goto exit_fp_process
;
961 case ISCSI_CQE_TYPE_DUMMY
:
962 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_SCSI_TM
, "Dummy CqE\n");
963 goto exit_fp_process
;
964 case ISCSI_CQE_TYPE_TASK_CLEANUP
:
965 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_SCSI_TM
, "CleanUp CqE\n");
966 qedi_process_cmd_cleanup_resp(qedi
, &cqe
->cqe_solicited
, task
,
968 goto exit_fp_process
;
970 QEDI_ERR(&qedi
->dbg_ctx
, "Error cqe.\n");
978 static void qedi_add_to_sq(struct qedi_conn
*qedi_conn
, struct iscsi_task
*task
,
979 u16 tid
, uint16_t ptu_invalidate
, int is_cleanup
)
981 struct iscsi_wqe
*wqe
;
982 struct iscsi_wqe_field
*cont_field
;
983 struct qedi_endpoint
*ep
;
984 struct scsi_cmnd
*sc
= task
->sc
;
985 struct iscsi_login_req
*login_hdr
;
986 struct qedi_cmd
*cmd
= task
->dd_data
;
988 login_hdr
= (struct iscsi_login_req
*)task
->hdr
;
990 wqe
= &ep
->sq
[ep
->sq_prod_idx
];
992 memset(wqe
, 0, sizeof(*wqe
));
995 ep
->fw_sq_prod_idx
++;
996 if (ep
->sq_prod_idx
== QEDI_SQ_SIZE
)
1000 SET_FIELD(wqe
->flags
, ISCSI_WQE_WQE_TYPE
,
1001 ISCSI_WQE_TYPE_TASK_CLEANUP
);
1006 if (ptu_invalidate
) {
1007 SET_FIELD(wqe
->flags
, ISCSI_WQE_PTU_INVALIDATE
,
1008 ISCSI_WQE_SET_PTU_INVALIDATE
);
1011 cont_field
= &wqe
->cont_prevtid_union
.cont_field
;
1013 switch (task
->hdr
->opcode
& ISCSI_OPCODE_MASK
) {
1014 case ISCSI_OP_LOGIN
:
1016 SET_FIELD(wqe
->flags
, ISCSI_WQE_WQE_TYPE
,
1017 ISCSI_WQE_TYPE_MIDDLE_PATH
);
1018 SET_FIELD(wqe
->flags
, ISCSI_WQE_NUM_FAST_SGES
,
1020 cont_field
->contlen_cdbsize_field
= ntoh24(login_hdr
->dlength
);
1022 case ISCSI_OP_LOGOUT
:
1023 case ISCSI_OP_NOOP_OUT
:
1024 case ISCSI_OP_SCSI_TMFUNC
:
1025 SET_FIELD(wqe
->flags
, ISCSI_WQE_WQE_TYPE
,
1026 ISCSI_WQE_TYPE_NORMAL
);
1032 SET_FIELD(wqe
->flags
, ISCSI_WQE_WQE_TYPE
,
1033 ISCSI_WQE_TYPE_NORMAL
);
1034 cont_field
->contlen_cdbsize_field
=
1035 (sc
->sc_data_direction
== DMA_TO_DEVICE
) ?
1036 scsi_bufflen(sc
) : 0;
1037 if (cmd
->use_slowpath
)
1038 SET_FIELD(wqe
->flags
, ISCSI_WQE_NUM_FAST_SGES
, 0);
1040 SET_FIELD(wqe
->flags
, ISCSI_WQE_NUM_FAST_SGES
,
1041 (sc
->sc_data_direction
==
1043 min((u16
)QEDI_FAST_SGE_COUNT
,
1044 (u16
)cmd
->io_tbl
.sge_valid
) : 0);
1049 /* Make sure SQ data is coherent */
1053 static void qedi_ring_doorbell(struct qedi_conn
*qedi_conn
)
1055 struct iscsi_db_data dbell
= { 0 };
1057 dbell
.agg_flags
= 0;
1059 dbell
.params
|= DB_DEST_XCM
<< ISCSI_DB_DATA_DEST_SHIFT
;
1060 dbell
.params
|= DB_AGG_CMD_SET
<< ISCSI_DB_DATA_AGG_CMD_SHIFT
;
1062 DQ_XCM_ISCSI_SQ_PROD_CMD
<< ISCSI_DB_DATA_AGG_VAL_SEL_SHIFT
;
1064 dbell
.sq_prod
= qedi_conn
->ep
->fw_sq_prod_idx
;
1065 writel(*(u32
*)&dbell
, qedi_conn
->ep
->p_doorbell
);
1067 /* Make sure fw write idx is coherent, and include both memory barriers
1068 * as a failsafe as for some architectures the call is the same but on
1069 * others they are two different assembly operations.
1073 QEDI_INFO(&qedi_conn
->qedi
->dbg_ctx
, QEDI_LOG_MP_REQ
,
1074 "prod_idx=0x%x, fw_prod_idx=0x%x, cid=0x%x\n",
1075 qedi_conn
->ep
->sq_prod_idx
, qedi_conn
->ep
->fw_sq_prod_idx
,
1076 qedi_conn
->iscsi_conn_id
);
1079 int qedi_send_iscsi_login(struct qedi_conn
*qedi_conn
,
1080 struct iscsi_task
*task
)
1082 struct qedi_ctx
*qedi
= qedi_conn
->qedi
;
1083 struct iscsi_task_context
*fw_task_ctx
;
1084 struct iscsi_login_req
*login_hdr
;
1085 struct iscsi_login_req_hdr
*fw_login_req
= NULL
;
1086 struct iscsi_cached_sge_ctx
*cached_sge
= NULL
;
1087 struct iscsi_sge
*single_sge
= NULL
;
1088 struct iscsi_sge
*req_sge
= NULL
;
1089 struct iscsi_sge
*resp_sge
= NULL
;
1090 struct qedi_cmd
*qedi_cmd
;
1091 s16 ptu_invalidate
= 0;
1094 req_sge
= (struct iscsi_sge
*)qedi_conn
->gen_pdu
.req_bd_tbl
;
1095 resp_sge
= (struct iscsi_sge
*)qedi_conn
->gen_pdu
.resp_bd_tbl
;
1096 qedi_cmd
= (struct qedi_cmd
*)task
->dd_data
;
1097 login_hdr
= (struct iscsi_login_req
*)task
->hdr
;
1099 tid
= qedi_get_task_idx(qedi
);
1103 fw_task_ctx
= qedi_get_task_mem(&qedi
->tasks
, tid
);
1104 memset(fw_task_ctx
, 0, sizeof(struct iscsi_task_context
));
1106 qedi_cmd
->task_id
= tid
;
1108 /* Ystorm context */
1109 fw_login_req
= &fw_task_ctx
->ystorm_st_context
.pdu_hdr
.login_req
;
1110 fw_login_req
->opcode
= login_hdr
->opcode
;
1111 fw_login_req
->version_min
= login_hdr
->min_version
;
1112 fw_login_req
->version_max
= login_hdr
->max_version
;
1113 fw_login_req
->flags_attr
= login_hdr
->flags
;
1114 fw_login_req
->isid_tabc
= *((u16
*)login_hdr
->isid
+ 2);
1115 fw_login_req
->isid_d
= *((u32
*)login_hdr
->isid
);
1116 fw_login_req
->tsih
= login_hdr
->tsih
;
1117 qedi_update_itt_map(qedi
, tid
, task
->itt
, qedi_cmd
);
1118 fw_login_req
->itt
= qedi_set_itt(tid
, get_itt(task
->itt
));
1119 fw_login_req
->cid
= qedi_conn
->iscsi_conn_id
;
1120 fw_login_req
->cmd_sn
= be32_to_cpu(login_hdr
->cmdsn
);
1121 fw_login_req
->exp_stat_sn
= be32_to_cpu(login_hdr
->exp_statsn
);
1122 fw_login_req
->exp_stat_sn
= 0;
1124 if (qedi
->tid_reuse_count
[tid
] == QEDI_MAX_TASK_NUM
) {
1126 qedi
->tid_reuse_count
[tid
] = 0;
1129 fw_task_ctx
->ystorm_st_context
.state
.reuse_count
=
1130 qedi
->tid_reuse_count
[tid
];
1131 fw_task_ctx
->mstorm_st_context
.reuse_count
=
1132 qedi
->tid_reuse_count
[tid
]++;
1134 &fw_task_ctx
->ystorm_st_context
.state
.sgl_ctx_union
.cached_sge
;
1135 cached_sge
->sge
.sge_len
= req_sge
->sge_len
;
1136 cached_sge
->sge
.sge_addr
.lo
= (u32
)(qedi_conn
->gen_pdu
.req_dma_addr
);
1137 cached_sge
->sge
.sge_addr
.hi
=
1138 (u32
)((u64
)qedi_conn
->gen_pdu
.req_dma_addr
>> 32);
1140 /* Mstorm context */
1141 single_sge
= &fw_task_ctx
->mstorm_st_context
.sgl_union
.single_sge
;
1142 fw_task_ctx
->mstorm_st_context
.task_type
= 0x2;
1143 fw_task_ctx
->mstorm_ag_context
.task_cid
= (u16
)qedi_conn
->iscsi_conn_id
;
1144 single_sge
->sge_addr
.lo
= resp_sge
->sge_addr
.lo
;
1145 single_sge
->sge_addr
.hi
= resp_sge
->sge_addr
.hi
;
1146 single_sge
->sge_len
= resp_sge
->sge_len
;
1148 SET_FIELD(fw_task_ctx
->mstorm_st_context
.flags
.mflags
,
1149 ISCSI_MFLAGS_SINGLE_SGE
, 1);
1150 SET_FIELD(fw_task_ctx
->mstorm_st_context
.flags
.mflags
,
1151 ISCSI_MFLAGS_SLOW_IO
, 0);
1152 fw_task_ctx
->mstorm_st_context
.sgl_size
= 1;
1153 fw_task_ctx
->mstorm_st_context
.rem_task_size
= resp_sge
->sge_len
;
1155 /* Ustorm context */
1156 fw_task_ctx
->ustorm_st_context
.rem_rcv_len
= resp_sge
->sge_len
;
1157 fw_task_ctx
->ustorm_st_context
.exp_data_transfer_len
=
1158 ntoh24(login_hdr
->dlength
);
1159 fw_task_ctx
->ustorm_st_context
.exp_data_sn
= 0;
1160 fw_task_ctx
->ustorm_st_context
.cq_rss_number
= 0;
1161 fw_task_ctx
->ustorm_st_context
.task_type
= 0x2;
1162 fw_task_ctx
->ustorm_ag_context
.icid
= (u16
)qedi_conn
->iscsi_conn_id
;
1163 fw_task_ctx
->ustorm_ag_context
.exp_data_acked
=
1164 ntoh24(login_hdr
->dlength
);
1165 SET_FIELD(fw_task_ctx
->ustorm_ag_context
.flags1
,
1166 USTORM_ISCSI_TASK_AG_CTX_R2T2RECV
, 1);
1167 SET_FIELD(fw_task_ctx
->ustorm_st_context
.flags
,
1168 USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP
, 0);
1170 spin_lock(&qedi_conn
->list_lock
);
1171 list_add_tail(&qedi_cmd
->io_cmd
, &qedi_conn
->active_cmd_list
);
1172 qedi_cmd
->io_cmd_in_list
= true;
1173 qedi_conn
->active_cmd_count
++;
1174 spin_unlock(&qedi_conn
->list_lock
);
1176 qedi_add_to_sq(qedi_conn
, task
, tid
, ptu_invalidate
, false);
1177 qedi_ring_doorbell(qedi_conn
);
1181 int qedi_send_iscsi_logout(struct qedi_conn
*qedi_conn
,
1182 struct iscsi_task
*task
)
1184 struct qedi_ctx
*qedi
= qedi_conn
->qedi
;
1185 struct iscsi_logout_req_hdr
*fw_logout_req
= NULL
;
1186 struct iscsi_task_context
*fw_task_ctx
= NULL
;
1187 struct iscsi_logout
*logout_hdr
= NULL
;
1188 struct qedi_cmd
*qedi_cmd
= NULL
;
1190 s16 ptu_invalidate
= 0;
1192 qedi_cmd
= (struct qedi_cmd
*)task
->dd_data
;
1193 logout_hdr
= (struct iscsi_logout
*)task
->hdr
;
1195 tid
= qedi_get_task_idx(qedi
);
1199 fw_task_ctx
= qedi_get_task_mem(&qedi
->tasks
, tid
);
1201 memset(fw_task_ctx
, 0, sizeof(struct iscsi_task_context
));
1202 qedi_cmd
->task_id
= tid
;
1204 /* Ystorm context */
1205 fw_logout_req
= &fw_task_ctx
->ystorm_st_context
.pdu_hdr
.logout_req
;
1206 fw_logout_req
->opcode
= ISCSI_OPCODE_LOGOUT_REQUEST
;
1207 fw_logout_req
->reason_code
= 0x80 | logout_hdr
->flags
;
1208 qedi_update_itt_map(qedi
, tid
, task
->itt
, qedi_cmd
);
1209 fw_logout_req
->itt
= qedi_set_itt(tid
, get_itt(task
->itt
));
1210 fw_logout_req
->exp_stat_sn
= be32_to_cpu(logout_hdr
->exp_statsn
);
1211 fw_logout_req
->cmd_sn
= be32_to_cpu(logout_hdr
->cmdsn
);
1213 if (qedi
->tid_reuse_count
[tid
] == QEDI_MAX_TASK_NUM
) {
1215 qedi
->tid_reuse_count
[tid
] = 0;
1217 fw_task_ctx
->ystorm_st_context
.state
.reuse_count
=
1218 qedi
->tid_reuse_count
[tid
];
1219 fw_task_ctx
->mstorm_st_context
.reuse_count
=
1220 qedi
->tid_reuse_count
[tid
]++;
1221 fw_logout_req
->cid
= qedi_conn
->iscsi_conn_id
;
1222 fw_task_ctx
->ystorm_st_context
.state
.buffer_offset
[0] = 0;
1224 /* Mstorm context */
1225 fw_task_ctx
->mstorm_st_context
.task_type
= ISCSI_TASK_TYPE_MIDPATH
;
1226 fw_task_ctx
->mstorm_ag_context
.task_cid
= (u16
)qedi_conn
->iscsi_conn_id
;
1228 /* Ustorm context */
1229 fw_task_ctx
->ustorm_st_context
.rem_rcv_len
= 0;
1230 fw_task_ctx
->ustorm_st_context
.exp_data_transfer_len
= 0;
1231 fw_task_ctx
->ustorm_st_context
.exp_data_sn
= 0;
1232 fw_task_ctx
->ustorm_st_context
.task_type
= ISCSI_TASK_TYPE_MIDPATH
;
1233 fw_task_ctx
->ustorm_st_context
.cq_rss_number
= 0;
1235 SET_FIELD(fw_task_ctx
->ustorm_st_context
.flags
,
1236 USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP
, 0);
1237 SET_FIELD(fw_task_ctx
->ustorm_st_context
.reg1
.reg1_map
,
1238 ISCSI_REG1_NUM_FAST_SGES
, 0);
1240 fw_task_ctx
->ustorm_ag_context
.icid
= (u16
)qedi_conn
->iscsi_conn_id
;
1241 SET_FIELD(fw_task_ctx
->ustorm_ag_context
.flags1
,
1242 USTORM_ISCSI_TASK_AG_CTX_R2T2RECV
, 1);
1244 spin_lock(&qedi_conn
->list_lock
);
1245 list_add_tail(&qedi_cmd
->io_cmd
, &qedi_conn
->active_cmd_list
);
1246 qedi_cmd
->io_cmd_in_list
= true;
1247 qedi_conn
->active_cmd_count
++;
1248 spin_unlock(&qedi_conn
->list_lock
);
1250 qedi_add_to_sq(qedi_conn
, task
, tid
, ptu_invalidate
, false);
1251 qedi_ring_doorbell(qedi_conn
);
1256 int qedi_cleanup_all_io(struct qedi_ctx
*qedi
, struct qedi_conn
*qedi_conn
,
1257 struct iscsi_task
*task
, bool in_recovery
)
1260 struct iscsi_task
*ctask
;
1261 struct qedi_cmd
*cmd
, *cmd_tmp
;
1262 struct iscsi_tm
*tmf_hdr
;
1263 unsigned int lun
= 0;
1264 bool lun_reset
= false;
1265 struct iscsi_conn
*conn
= qedi_conn
->cls_conn
->dd_data
;
1266 struct iscsi_session
*session
= conn
->session
;
1268 /* From recovery, task is NULL or from tmf resp valid task */
1270 tmf_hdr
= (struct iscsi_tm
*)task
->hdr
;
1272 if ((tmf_hdr
->flags
& ISCSI_FLAG_TM_FUNC_MASK
) ==
1273 ISCSI_TM_FUNC_LOGICAL_UNIT_RESET
) {
1275 lun
= scsilun_to_int(&tmf_hdr
->lun
);
1279 qedi_conn
->cmd_cleanup_req
= 0;
1280 qedi_conn
->cmd_cleanup_cmpl
= 0;
1282 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_SCSI_TM
,
1283 "active_cmd_count=%d, cid=0x%x, in_recovery=%d, lun_reset=%d\n",
1284 qedi_conn
->active_cmd_count
, qedi_conn
->iscsi_conn_id
,
1285 in_recovery
, lun_reset
);
1288 spin_lock_bh(&session
->back_lock
);
1290 spin_lock(&qedi_conn
->list_lock
);
1292 list_for_each_entry_safe(cmd
, cmd_tmp
, &qedi_conn
->active_cmd_list
,
1299 if (cmd
->scsi_cmd
&& cmd
->scsi_cmd
->device
) {
1300 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_SCSI_TM
,
1301 "tid=0x%x itt=0x%x scsi_cmd_ptr=%p device=%p task_state=%d cmd_state=0%x cid=0x%x\n",
1302 cmd
->task_id
, get_itt(ctask
->itt
),
1303 cmd
->scsi_cmd
, cmd
->scsi_cmd
->device
,
1304 ctask
->state
, cmd
->state
,
1305 qedi_conn
->iscsi_conn_id
);
1306 if (cmd
->scsi_cmd
->device
->lun
!= lun
)
1310 qedi_conn
->cmd_cleanup_req
++;
1311 qedi_iscsi_cleanup_task(ctask
, true);
1313 list_del_init(&cmd
->io_cmd
);
1314 qedi_conn
->active_cmd_count
--;
1315 QEDI_WARN(&qedi
->dbg_ctx
,
1316 "Deleted active cmd list node io_cmd=%p, cid=0x%x\n",
1317 &cmd
->io_cmd
, qedi_conn
->iscsi_conn_id
);
1320 spin_unlock(&qedi_conn
->list_lock
);
1323 spin_unlock_bh(&session
->back_lock
);
1325 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_SCSI_TM
,
1326 "cmd_cleanup_req=%d, cid=0x%x\n",
1327 qedi_conn
->cmd_cleanup_req
,
1328 qedi_conn
->iscsi_conn_id
);
1330 rval
= wait_event_interruptible_timeout(qedi_conn
->wait_queue
,
1331 ((qedi_conn
->cmd_cleanup_req
==
1332 qedi_conn
->cmd_cleanup_cmpl
) ||
1336 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_SCSI_TM
,
1337 "i/o cmd_cleanup_req=%d, equal to cmd_cleanup_cmpl=%d, cid=0x%x\n",
1338 qedi_conn
->cmd_cleanup_req
,
1339 qedi_conn
->cmd_cleanup_cmpl
,
1340 qedi_conn
->iscsi_conn_id
);
1345 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_SCSI_TM
,
1346 "i/o cmd_cleanup_req=%d, not equal to cmd_cleanup_cmpl=%d, cid=0x%x\n",
1347 qedi_conn
->cmd_cleanup_req
,
1348 qedi_conn
->cmd_cleanup_cmpl
,
1349 qedi_conn
->iscsi_conn_id
);
1351 iscsi_host_for_each_session(qedi
->shost
,
1352 qedi_mark_device_missing
);
1353 qedi_ops
->common
->drain(qedi
->cdev
);
1355 /* Enable IOs for all other sessions except current.*/
1356 if (!wait_event_interruptible_timeout(qedi_conn
->wait_queue
,
1357 (qedi_conn
->cmd_cleanup_req
==
1358 qedi_conn
->cmd_cleanup_cmpl
),
1360 iscsi_host_for_each_session(qedi
->shost
,
1361 qedi_mark_device_available
);
1365 iscsi_host_for_each_session(qedi
->shost
,
1366 qedi_mark_device_available
);
1371 void qedi_clearsq(struct qedi_ctx
*qedi
, struct qedi_conn
*qedi_conn
,
1372 struct iscsi_task
*task
)
1374 struct qedi_endpoint
*qedi_ep
;
1377 qedi_ep
= qedi_conn
->ep
;
1378 qedi_conn
->cmd_cleanup_req
= 0;
1379 qedi_conn
->cmd_cleanup_cmpl
= 0;
1382 QEDI_WARN(&qedi
->dbg_ctx
,
1383 "Cannot proceed, ep already disconnected, cid=0x%x\n",
1384 qedi_conn
->iscsi_conn_id
);
1388 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_INFO
,
1389 "Clearing SQ for cid=0x%x, conn=%p, ep=%p\n",
1390 qedi_conn
->iscsi_conn_id
, qedi_conn
, qedi_ep
);
1392 qedi_ops
->clear_sq(qedi
->cdev
, qedi_ep
->handle
);
1394 rval
= qedi_cleanup_all_io(qedi
, qedi_conn
, task
, true);
1396 QEDI_ERR(&qedi
->dbg_ctx
,
1397 "fatal error, need hard reset, cid=0x%x\n",
1398 qedi_conn
->iscsi_conn_id
);
1403 static int qedi_wait_for_cleanup_request(struct qedi_ctx
*qedi
,
1404 struct qedi_conn
*qedi_conn
,
1405 struct iscsi_task
*task
,
1406 struct qedi_cmd
*qedi_cmd
,
1407 struct qedi_work_map
*list_work
)
1409 struct qedi_cmd
*cmd
= (struct qedi_cmd
*)task
->dd_data
;
1412 wait
= wait_event_interruptible_timeout(qedi_conn
->wait_queue
,
1413 ((qedi_cmd
->state
==
1415 ((qedi_cmd
->type
== TYPEIO
) &&
1417 RESPONSE_RECEIVED
))),
1420 qedi_cmd
->state
= CLEANUP_WAIT_FAILED
;
1422 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_SCSI_TM
,
1423 "Cleanup timedout tid=0x%x, issue connection recovery, cid=0x%x\n",
1424 cmd
->task_id
, qedi_conn
->iscsi_conn_id
);
1431 static void qedi_tmf_work(struct work_struct
*work
)
1433 struct qedi_cmd
*qedi_cmd
=
1434 container_of(work
, struct qedi_cmd
, tmf_work
);
1435 struct qedi_conn
*qedi_conn
= qedi_cmd
->conn
;
1436 struct qedi_ctx
*qedi
= qedi_conn
->qedi
;
1437 struct iscsi_conn
*conn
= qedi_conn
->cls_conn
->dd_data
;
1438 struct iscsi_cls_session
*cls_sess
;
1439 struct qedi_work_map
*list_work
= NULL
;
1440 struct iscsi_task
*mtask
;
1441 struct qedi_cmd
*cmd
;
1442 struct iscsi_task
*ctask
;
1443 struct iscsi_tm
*tmf_hdr
;
1447 mtask
= qedi_cmd
->task
;
1448 tmf_hdr
= (struct iscsi_tm
*)mtask
->hdr
;
1449 cls_sess
= iscsi_conn_to_session(qedi_conn
->cls_conn
);
1450 set_bit(QEDI_CONN_FW_CLEANUP
, &qedi_conn
->flags
);
1452 ctask
= iscsi_itt_to_task(conn
, tmf_hdr
->rtt
);
1453 if (!ctask
|| !ctask
->sc
) {
1454 QEDI_ERR(&qedi
->dbg_ctx
, "Task already completed\n");
1458 cmd
= (struct qedi_cmd
*)ctask
->dd_data
;
1459 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_INFO
,
1460 "Abort tmf rtt=0x%x, cmd itt=0x%x, cmd tid=0x%x, cid=0x%x\n",
1461 get_itt(tmf_hdr
->rtt
), get_itt(ctask
->itt
), cmd
->task_id
,
1462 qedi_conn
->iscsi_conn_id
);
1464 if (do_not_recover
) {
1465 QEDI_ERR(&qedi
->dbg_ctx
, "DONT SEND CLEANUP/ABORT %d\n",
1470 list_work
= kzalloc(sizeof(*list_work
), GFP_ATOMIC
);
1472 QEDI_ERR(&qedi
->dbg_ctx
, "Memory alloction failed\n");
1476 qedi_cmd
->type
= TYPEIO
;
1477 list_work
->qedi_cmd
= qedi_cmd
;
1478 list_work
->rtid
= cmd
->task_id
;
1479 list_work
->state
= QEDI_WORK_SCHEDULED
;
1480 qedi_cmd
->list_tmf_work
= list_work
;
1482 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_SCSI_TM
,
1483 "Queue tmf work=%p, list node=%p, cid=0x%x, tmf flags=0x%x\n",
1484 list_work
->ptr_tmf_work
, list_work
, qedi_conn
->iscsi_conn_id
,
1487 spin_lock_bh(&qedi_conn
->tmf_work_lock
);
1488 list_add_tail(&list_work
->list
, &qedi_conn
->tmf_work_list
);
1489 spin_unlock_bh(&qedi_conn
->tmf_work_lock
);
1491 qedi_iscsi_cleanup_task(ctask
, false);
1493 rval
= qedi_wait_for_cleanup_request(qedi
, qedi_conn
, ctask
, qedi_cmd
,
1496 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_INFO
,
1497 "FW cleanup got escalated, cid=0x%x\n",
1498 qedi_conn
->iscsi_conn_id
);
1502 tid
= qedi_get_task_idx(qedi
);
1504 QEDI_ERR(&qedi
->dbg_ctx
, "Invalid tid, cid=0x%x\n",
1505 qedi_conn
->iscsi_conn_id
);
1509 qedi_cmd
->task_id
= tid
;
1510 qedi_send_iscsi_tmf(qedi_conn
, qedi_cmd
->task
);
1513 clear_bit(QEDI_CONN_FW_CLEANUP
, &qedi_conn
->flags
);
1517 spin_lock_bh(&qedi_conn
->tmf_work_lock
);
1518 if (!qedi_cmd
->list_tmf_work
) {
1519 list_del_init(&list_work
->list
);
1520 qedi_cmd
->list_tmf_work
= NULL
;
1523 spin_unlock_bh(&qedi_conn
->tmf_work_lock
);
1525 spin_lock(&qedi_conn
->list_lock
);
1526 list_del_init(&cmd
->io_cmd
);
1527 qedi_conn
->active_cmd_count
--;
1528 spin_unlock(&qedi_conn
->list_lock
);
1530 clear_bit(QEDI_CONN_FW_CLEANUP
, &qedi_conn
->flags
);
1533 static int qedi_send_iscsi_tmf(struct qedi_conn
*qedi_conn
,
1534 struct iscsi_task
*mtask
)
1536 struct iscsi_conn
*conn
= qedi_conn
->cls_conn
->dd_data
;
1537 struct qedi_ctx
*qedi
= qedi_conn
->qedi
;
1538 struct iscsi_task_context
*fw_task_ctx
;
1539 struct iscsi_tmf_request_hdr
*fw_tmf_request
;
1540 struct iscsi_sge
*single_sge
;
1541 struct qedi_cmd
*qedi_cmd
;
1542 struct qedi_cmd
*cmd
;
1543 struct iscsi_task
*ctask
;
1544 struct iscsi_tm
*tmf_hdr
;
1545 struct iscsi_sge
*req_sge
;
1546 struct iscsi_sge
*resp_sge
;
1548 s16 tid
= 0, ptu_invalidate
= 0;
1550 req_sge
= (struct iscsi_sge
*)qedi_conn
->gen_pdu
.req_bd_tbl
;
1551 resp_sge
= (struct iscsi_sge
*)qedi_conn
->gen_pdu
.resp_bd_tbl
;
1552 qedi_cmd
= (struct qedi_cmd
*)mtask
->dd_data
;
1553 tmf_hdr
= (struct iscsi_tm
*)mtask
->hdr
;
1555 tid
= qedi_cmd
->task_id
;
1556 qedi_update_itt_map(qedi
, tid
, mtask
->itt
, qedi_cmd
);
1558 fw_task_ctx
= qedi_get_task_mem(&qedi
->tasks
, tid
);
1559 memset(fw_task_ctx
, 0, sizeof(struct iscsi_task_context
));
1561 fw_tmf_request
= &fw_task_ctx
->ystorm_st_context
.pdu_hdr
.tmf_request
;
1562 fw_tmf_request
->itt
= qedi_set_itt(tid
, get_itt(mtask
->itt
));
1563 fw_tmf_request
->cmd_sn
= be32_to_cpu(tmf_hdr
->cmdsn
);
1565 memcpy(lun
, &tmf_hdr
->lun
, sizeof(struct scsi_lun
));
1566 fw_tmf_request
->lun
.lo
= be32_to_cpu(lun
[0]);
1567 fw_tmf_request
->lun
.hi
= be32_to_cpu(lun
[1]);
1569 if (qedi
->tid_reuse_count
[tid
] == QEDI_MAX_TASK_NUM
) {
1571 qedi
->tid_reuse_count
[tid
] = 0;
1573 fw_task_ctx
->ystorm_st_context
.state
.reuse_count
=
1574 qedi
->tid_reuse_count
[tid
];
1575 fw_task_ctx
->mstorm_st_context
.reuse_count
=
1576 qedi
->tid_reuse_count
[tid
]++;
1578 if ((tmf_hdr
->flags
& ISCSI_FLAG_TM_FUNC_MASK
) ==
1579 ISCSI_TM_FUNC_ABORT_TASK
) {
1580 ctask
= iscsi_itt_to_task(conn
, tmf_hdr
->rtt
);
1581 if (!ctask
|| !ctask
->sc
) {
1582 QEDI_ERR(&qedi
->dbg_ctx
,
1583 "Could not get reference task\n");
1586 cmd
= (struct qedi_cmd
*)ctask
->dd_data
;
1587 fw_tmf_request
->rtt
=
1588 qedi_set_itt(cmd
->task_id
,
1589 get_itt(tmf_hdr
->rtt
));
1591 fw_tmf_request
->rtt
= ISCSI_RESERVED_TAG
;
1594 fw_tmf_request
->opcode
= tmf_hdr
->opcode
;
1595 fw_tmf_request
->function
= tmf_hdr
->flags
;
1596 fw_tmf_request
->hdr_second_dword
= ntoh24(tmf_hdr
->dlength
);
1597 fw_tmf_request
->ref_cmd_sn
= be32_to_cpu(tmf_hdr
->refcmdsn
);
1599 single_sge
= &fw_task_ctx
->mstorm_st_context
.sgl_union
.single_sge
;
1600 fw_task_ctx
->mstorm_st_context
.task_type
= ISCSI_TASK_TYPE_MIDPATH
;
1601 fw_task_ctx
->mstorm_ag_context
.task_cid
= (u16
)qedi_conn
->iscsi_conn_id
;
1602 single_sge
->sge_addr
.lo
= resp_sge
->sge_addr
.lo
;
1603 single_sge
->sge_addr
.hi
= resp_sge
->sge_addr
.hi
;
1604 single_sge
->sge_len
= resp_sge
->sge_len
;
1606 SET_FIELD(fw_task_ctx
->mstorm_st_context
.flags
.mflags
,
1607 ISCSI_MFLAGS_SINGLE_SGE
, 1);
1608 SET_FIELD(fw_task_ctx
->mstorm_st_context
.flags
.mflags
,
1609 ISCSI_MFLAGS_SLOW_IO
, 0);
1610 fw_task_ctx
->mstorm_st_context
.sgl_size
= 1;
1611 fw_task_ctx
->mstorm_st_context
.rem_task_size
= resp_sge
->sge_len
;
1613 /* Ustorm context */
1614 fw_task_ctx
->ustorm_st_context
.rem_rcv_len
= 0;
1615 fw_task_ctx
->ustorm_st_context
.exp_data_transfer_len
= 0;
1616 fw_task_ctx
->ustorm_st_context
.exp_data_sn
= 0;
1617 fw_task_ctx
->ustorm_st_context
.task_type
= ISCSI_TASK_TYPE_MIDPATH
;
1618 fw_task_ctx
->ustorm_st_context
.cq_rss_number
= 0;
1620 SET_FIELD(fw_task_ctx
->ustorm_st_context
.flags
,
1621 USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP
, 0);
1622 SET_FIELD(fw_task_ctx
->ustorm_st_context
.reg1
.reg1_map
,
1623 ISCSI_REG1_NUM_FAST_SGES
, 0);
1625 fw_task_ctx
->ustorm_ag_context
.icid
= (u16
)qedi_conn
->iscsi_conn_id
;
1626 SET_FIELD(fw_task_ctx
->ustorm_ag_context
.flags1
,
1627 USTORM_ISCSI_TASK_AG_CTX_R2T2RECV
, 1);
1628 fw_task_ctx
->ustorm_st_context
.lun
.lo
= be32_to_cpu(lun
[0]);
1629 fw_task_ctx
->ustorm_st_context
.lun
.hi
= be32_to_cpu(lun
[1]);
1631 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_SCSI_TM
,
1632 "Add TMF to SQ, tmf tid=0x%x, itt=0x%x, cid=0x%x\n",
1633 tid
, mtask
->itt
, qedi_conn
->iscsi_conn_id
);
1635 spin_lock(&qedi_conn
->list_lock
);
1636 list_add_tail(&qedi_cmd
->io_cmd
, &qedi_conn
->active_cmd_list
);
1637 qedi_cmd
->io_cmd_in_list
= true;
1638 qedi_conn
->active_cmd_count
++;
1639 spin_unlock(&qedi_conn
->list_lock
);
1641 qedi_add_to_sq(qedi_conn
, mtask
, tid
, ptu_invalidate
, false);
1642 qedi_ring_doorbell(qedi_conn
);
1646 int qedi_iscsi_abort_work(struct qedi_conn
*qedi_conn
,
1647 struct iscsi_task
*mtask
)
1649 struct qedi_ctx
*qedi
= qedi_conn
->qedi
;
1650 struct iscsi_tm
*tmf_hdr
;
1651 struct qedi_cmd
*qedi_cmd
= (struct qedi_cmd
*)mtask
->dd_data
;
1654 tmf_hdr
= (struct iscsi_tm
*)mtask
->hdr
;
1655 qedi_cmd
->task
= mtask
;
1657 /* If abort task then schedule the work and return */
1658 if ((tmf_hdr
->flags
& ISCSI_FLAG_TM_FUNC_MASK
) ==
1659 ISCSI_TM_FUNC_ABORT_TASK
) {
1660 qedi_cmd
->state
= CLEANUP_WAIT
;
1661 INIT_WORK(&qedi_cmd
->tmf_work
, qedi_tmf_work
);
1662 queue_work(qedi
->tmf_thread
, &qedi_cmd
->tmf_work
);
1664 } else if (((tmf_hdr
->flags
& ISCSI_FLAG_TM_FUNC_MASK
) ==
1665 ISCSI_TM_FUNC_LOGICAL_UNIT_RESET
) ||
1666 ((tmf_hdr
->flags
& ISCSI_FLAG_TM_FUNC_MASK
) ==
1667 ISCSI_TM_FUNC_TARGET_WARM_RESET
) ||
1668 ((tmf_hdr
->flags
& ISCSI_FLAG_TM_FUNC_MASK
) ==
1669 ISCSI_TM_FUNC_TARGET_COLD_RESET
)) {
1670 tid
= qedi_get_task_idx(qedi
);
1672 QEDI_ERR(&qedi
->dbg_ctx
, "Invalid tid, cid=0x%x\n",
1673 qedi_conn
->iscsi_conn_id
);
1676 qedi_cmd
->task_id
= tid
;
1678 qedi_send_iscsi_tmf(qedi_conn
, qedi_cmd
->task
);
1681 QEDI_ERR(&qedi
->dbg_ctx
, "Invalid tmf, cid=0x%x\n",
1682 qedi_conn
->iscsi_conn_id
);
1689 int qedi_send_iscsi_text(struct qedi_conn
*qedi_conn
,
1690 struct iscsi_task
*task
)
1692 struct qedi_ctx
*qedi
= qedi_conn
->qedi
;
1693 struct iscsi_task_context
*fw_task_ctx
;
1694 struct iscsi_text_request_hdr
*fw_text_request
;
1695 struct iscsi_cached_sge_ctx
*cached_sge
;
1696 struct iscsi_sge
*single_sge
;
1697 struct qedi_cmd
*qedi_cmd
;
1698 /* For 6.5 hdr iscsi_hdr */
1699 struct iscsi_text
*text_hdr
;
1700 struct iscsi_sge
*req_sge
;
1701 struct iscsi_sge
*resp_sge
;
1702 s16 ptu_invalidate
= 0;
1705 req_sge
= (struct iscsi_sge
*)qedi_conn
->gen_pdu
.req_bd_tbl
;
1706 resp_sge
= (struct iscsi_sge
*)qedi_conn
->gen_pdu
.resp_bd_tbl
;
1707 qedi_cmd
= (struct qedi_cmd
*)task
->dd_data
;
1708 text_hdr
= (struct iscsi_text
*)task
->hdr
;
1710 tid
= qedi_get_task_idx(qedi
);
1714 fw_task_ctx
= qedi_get_task_mem(&qedi
->tasks
, tid
);
1715 memset(fw_task_ctx
, 0, sizeof(struct iscsi_task_context
));
1717 qedi_cmd
->task_id
= tid
;
1719 /* Ystorm context */
1721 &fw_task_ctx
->ystorm_st_context
.pdu_hdr
.text_request
;
1722 fw_text_request
->opcode
= text_hdr
->opcode
;
1723 fw_text_request
->flags_attr
= text_hdr
->flags
;
1725 qedi_update_itt_map(qedi
, tid
, task
->itt
, qedi_cmd
);
1726 fw_text_request
->itt
= qedi_set_itt(tid
, get_itt(task
->itt
));
1727 fw_text_request
->ttt
= text_hdr
->ttt
;
1728 fw_text_request
->cmd_sn
= be32_to_cpu(text_hdr
->cmdsn
);
1729 fw_text_request
->exp_stat_sn
= be32_to_cpu(text_hdr
->exp_statsn
);
1730 fw_text_request
->hdr_second_dword
= ntoh24(text_hdr
->dlength
);
1732 if (qedi
->tid_reuse_count
[tid
] == QEDI_MAX_TASK_NUM
) {
1734 qedi
->tid_reuse_count
[tid
] = 0;
1736 fw_task_ctx
->ystorm_st_context
.state
.reuse_count
=
1737 qedi
->tid_reuse_count
[tid
];
1738 fw_task_ctx
->mstorm_st_context
.reuse_count
=
1739 qedi
->tid_reuse_count
[tid
]++;
1742 &fw_task_ctx
->ystorm_st_context
.state
.sgl_ctx_union
.cached_sge
;
1743 cached_sge
->sge
.sge_len
= req_sge
->sge_len
;
1744 cached_sge
->sge
.sge_addr
.lo
= (u32
)(qedi_conn
->gen_pdu
.req_dma_addr
);
1745 cached_sge
->sge
.sge_addr
.hi
=
1746 (u32
)((u64
)qedi_conn
->gen_pdu
.req_dma_addr
>> 32);
1748 /* Mstorm context */
1749 single_sge
= &fw_task_ctx
->mstorm_st_context
.sgl_union
.single_sge
;
1750 fw_task_ctx
->mstorm_st_context
.task_type
= 0x2;
1751 fw_task_ctx
->mstorm_ag_context
.task_cid
= (u16
)qedi_conn
->iscsi_conn_id
;
1752 single_sge
->sge_addr
.lo
= resp_sge
->sge_addr
.lo
;
1753 single_sge
->sge_addr
.hi
= resp_sge
->sge_addr
.hi
;
1754 single_sge
->sge_len
= resp_sge
->sge_len
;
1756 SET_FIELD(fw_task_ctx
->mstorm_st_context
.flags
.mflags
,
1757 ISCSI_MFLAGS_SINGLE_SGE
, 1);
1758 SET_FIELD(fw_task_ctx
->mstorm_st_context
.flags
.mflags
,
1759 ISCSI_MFLAGS_SLOW_IO
, 0);
1760 fw_task_ctx
->mstorm_st_context
.sgl_size
= 1;
1761 fw_task_ctx
->mstorm_st_context
.rem_task_size
= resp_sge
->sge_len
;
1763 /* Ustorm context */
1764 fw_task_ctx
->ustorm_ag_context
.exp_data_acked
=
1765 ntoh24(text_hdr
->dlength
);
1766 fw_task_ctx
->ustorm_st_context
.rem_rcv_len
= resp_sge
->sge_len
;
1767 fw_task_ctx
->ustorm_st_context
.exp_data_transfer_len
=
1768 ntoh24(text_hdr
->dlength
);
1769 fw_task_ctx
->ustorm_st_context
.exp_data_sn
=
1770 be32_to_cpu(text_hdr
->exp_statsn
);
1771 fw_task_ctx
->ustorm_st_context
.cq_rss_number
= 0;
1772 fw_task_ctx
->ustorm_st_context
.task_type
= 0x2;
1773 fw_task_ctx
->ustorm_ag_context
.icid
= (u16
)qedi_conn
->iscsi_conn_id
;
1774 SET_FIELD(fw_task_ctx
->ustorm_ag_context
.flags1
,
1775 USTORM_ISCSI_TASK_AG_CTX_R2T2RECV
, 1);
1777 /* Add command in active command list */
1778 spin_lock(&qedi_conn
->list_lock
);
1779 list_add_tail(&qedi_cmd
->io_cmd
, &qedi_conn
->active_cmd_list
);
1780 qedi_cmd
->io_cmd_in_list
= true;
1781 qedi_conn
->active_cmd_count
++;
1782 spin_unlock(&qedi_conn
->list_lock
);
1784 qedi_add_to_sq(qedi_conn
, task
, tid
, ptu_invalidate
, false);
1785 qedi_ring_doorbell(qedi_conn
);
1790 int qedi_send_iscsi_nopout(struct qedi_conn
*qedi_conn
,
1791 struct iscsi_task
*task
,
1792 char *datap
, int data_len
, int unsol
)
1794 struct qedi_ctx
*qedi
= qedi_conn
->qedi
;
1795 struct iscsi_task_context
*fw_task_ctx
;
1796 struct iscsi_nop_out_hdr
*fw_nop_out
;
1797 struct qedi_cmd
*qedi_cmd
;
1798 /* For 6.5 hdr iscsi_hdr */
1799 struct iscsi_nopout
*nopout_hdr
;
1800 struct iscsi_cached_sge_ctx
*cached_sge
;
1801 struct iscsi_sge
*single_sge
;
1802 struct iscsi_sge
*req_sge
;
1803 struct iscsi_sge
*resp_sge
;
1805 s16 ptu_invalidate
= 0;
1808 req_sge
= (struct iscsi_sge
*)qedi_conn
->gen_pdu
.req_bd_tbl
;
1809 resp_sge
= (struct iscsi_sge
*)qedi_conn
->gen_pdu
.resp_bd_tbl
;
1810 qedi_cmd
= (struct qedi_cmd
*)task
->dd_data
;
1811 nopout_hdr
= (struct iscsi_nopout
*)task
->hdr
;
1813 tid
= qedi_get_task_idx(qedi
);
1815 QEDI_WARN(&qedi
->dbg_ctx
, "Invalid tid\n");
1819 fw_task_ctx
= qedi_get_task_mem(&qedi
->tasks
, tid
);
1821 memset(fw_task_ctx
, 0, sizeof(struct iscsi_task_context
));
1822 qedi_cmd
->task_id
= tid
;
1824 /* Ystorm context */
1825 fw_nop_out
= &fw_task_ctx
->ystorm_st_context
.pdu_hdr
.nop_out
;
1826 SET_FIELD(fw_nop_out
->flags_attr
, ISCSI_NOP_OUT_HDR_CONST1
, 1);
1827 SET_FIELD(fw_nop_out
->flags_attr
, ISCSI_NOP_OUT_HDR_RSRV
, 0);
1829 memcpy(lun
, &nopout_hdr
->lun
, sizeof(struct scsi_lun
));
1830 fw_nop_out
->lun
.lo
= be32_to_cpu(lun
[0]);
1831 fw_nop_out
->lun
.hi
= be32_to_cpu(lun
[1]);
1833 qedi_update_itt_map(qedi
, tid
, task
->itt
, qedi_cmd
);
1835 if (nopout_hdr
->ttt
!= ISCSI_TTT_ALL_ONES
) {
1836 fw_nop_out
->itt
= be32_to_cpu(nopout_hdr
->itt
);
1837 fw_nop_out
->ttt
= be32_to_cpu(nopout_hdr
->ttt
);
1838 fw_task_ctx
->ystorm_st_context
.state
.buffer_offset
[0] = 0;
1839 fw_task_ctx
->ystorm_st_context
.state
.local_comp
= 1;
1840 SET_FIELD(fw_task_ctx
->ustorm_st_context
.flags
,
1841 USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP
, 1);
1843 fw_nop_out
->itt
= qedi_set_itt(tid
, get_itt(task
->itt
));
1844 fw_nop_out
->ttt
= ISCSI_TTT_ALL_ONES
;
1845 fw_task_ctx
->ystorm_st_context
.state
.buffer_offset
[0] = 0;
1847 spin_lock(&qedi_conn
->list_lock
);
1848 list_add_tail(&qedi_cmd
->io_cmd
, &qedi_conn
->active_cmd_list
);
1849 qedi_cmd
->io_cmd_in_list
= true;
1850 qedi_conn
->active_cmd_count
++;
1851 spin_unlock(&qedi_conn
->list_lock
);
1854 fw_nop_out
->opcode
= ISCSI_OPCODE_NOP_OUT
;
1855 fw_nop_out
->cmd_sn
= be32_to_cpu(nopout_hdr
->cmdsn
);
1856 fw_nop_out
->exp_stat_sn
= be32_to_cpu(nopout_hdr
->exp_statsn
);
1859 &fw_task_ctx
->ystorm_st_context
.state
.sgl_ctx_union
.cached_sge
;
1860 cached_sge
->sge
.sge_len
= req_sge
->sge_len
;
1861 cached_sge
->sge
.sge_addr
.lo
= (u32
)(qedi_conn
->gen_pdu
.req_dma_addr
);
1862 cached_sge
->sge
.sge_addr
.hi
=
1863 (u32
)((u64
)qedi_conn
->gen_pdu
.req_dma_addr
>> 32);
1865 /* Mstorm context */
1866 fw_task_ctx
->mstorm_st_context
.task_type
= ISCSI_TASK_TYPE_MIDPATH
;
1867 fw_task_ctx
->mstorm_ag_context
.task_cid
= (u16
)qedi_conn
->iscsi_conn_id
;
1869 single_sge
= &fw_task_ctx
->mstorm_st_context
.sgl_union
.single_sge
;
1870 single_sge
->sge_addr
.lo
= resp_sge
->sge_addr
.lo
;
1871 single_sge
->sge_addr
.hi
= resp_sge
->sge_addr
.hi
;
1872 single_sge
->sge_len
= resp_sge
->sge_len
;
1873 fw_task_ctx
->mstorm_st_context
.rem_task_size
= resp_sge
->sge_len
;
1875 if (qedi
->tid_reuse_count
[tid
] == QEDI_MAX_TASK_NUM
) {
1877 qedi
->tid_reuse_count
[tid
] = 0;
1879 fw_task_ctx
->ystorm_st_context
.state
.reuse_count
=
1880 qedi
->tid_reuse_count
[tid
];
1881 fw_task_ctx
->mstorm_st_context
.reuse_count
=
1882 qedi
->tid_reuse_count
[tid
]++;
1883 /* Ustorm context */
1884 fw_task_ctx
->ustorm_st_context
.rem_rcv_len
= resp_sge
->sge_len
;
1885 fw_task_ctx
->ustorm_st_context
.exp_data_transfer_len
= data_len
;
1886 fw_task_ctx
->ustorm_st_context
.exp_data_sn
= 0;
1887 fw_task_ctx
->ustorm_st_context
.task_type
= ISCSI_TASK_TYPE_MIDPATH
;
1888 fw_task_ctx
->ustorm_st_context
.cq_rss_number
= 0;
1890 SET_FIELD(fw_task_ctx
->ustorm_st_context
.reg1
.reg1_map
,
1891 ISCSI_REG1_NUM_FAST_SGES
, 0);
1893 fw_task_ctx
->ustorm_ag_context
.icid
= (u16
)qedi_conn
->iscsi_conn_id
;
1894 SET_FIELD(fw_task_ctx
->ustorm_ag_context
.flags1
,
1895 USTORM_ISCSI_TASK_AG_CTX_R2T2RECV
, 1);
1897 fw_task_ctx
->ustorm_st_context
.lun
.lo
= be32_to_cpu(lun
[0]);
1898 fw_task_ctx
->ustorm_st_context
.lun
.hi
= be32_to_cpu(lun
[1]);
1900 qedi_add_to_sq(qedi_conn
, task
, tid
, ptu_invalidate
, false);
1901 qedi_ring_doorbell(qedi_conn
);
1905 static int qedi_split_bd(struct qedi_cmd
*cmd
, u64 addr
, int sg_len
,
1908 struct iscsi_sge
*bd
= cmd
->io_tbl
.sge_tbl
;
1909 int frag_size
, sg_frags
;
1914 if (addr
% QEDI_PAGE_SIZE
)
1916 (QEDI_PAGE_SIZE
- (addr
% QEDI_PAGE_SIZE
));
1918 frag_size
= (sg_len
> QEDI_BD_SPLIT_SZ
) ? 0 :
1919 (sg_len
% QEDI_BD_SPLIT_SZ
);
1922 frag_size
= QEDI_BD_SPLIT_SZ
;
1924 bd
[bd_index
+ sg_frags
].sge_addr
.lo
= (addr
& 0xffffffff);
1925 bd
[bd_index
+ sg_frags
].sge_addr
.hi
= (addr
>> 32);
1926 bd
[bd_index
+ sg_frags
].sge_len
= (u16
)frag_size
;
1927 QEDI_INFO(&cmd
->conn
->qedi
->dbg_ctx
, QEDI_LOG_IO
,
1928 "split sge %d: addr=%llx, len=%x",
1929 (bd_index
+ sg_frags
), addr
, frag_size
);
1931 addr
+= (u64
)frag_size
;
1933 sg_len
-= frag_size
;
1938 static int qedi_map_scsi_sg(struct qedi_ctx
*qedi
, struct qedi_cmd
*cmd
)
1940 struct scsi_cmnd
*sc
= cmd
->scsi_cmd
;
1941 struct iscsi_sge
*bd
= cmd
->io_tbl
.sge_tbl
;
1942 struct scatterlist
*sg
;
1951 WARN_ON(scsi_sg_count(sc
) > QEDI_ISCSI_MAX_BDS_PER_CMD
);
1953 sg_count
= dma_map_sg(&qedi
->pdev
->dev
, scsi_sglist(sc
),
1954 scsi_sg_count(sc
), sc
->sc_data_direction
);
1957 * New condition to send single SGE as cached-SGL.
1958 * Single SGE with length less than 64K.
1960 sg
= scsi_sglist(sc
);
1961 if ((sg_count
== 1) && (sg_dma_len(sg
) <= MAX_SGLEN_FOR_CACHESGL
)) {
1962 sg_len
= sg_dma_len(sg
);
1963 addr
= (u64
)sg_dma_address(sg
);
1965 bd
[bd_count
].sge_addr
.lo
= (addr
& 0xffffffff);
1966 bd
[bd_count
].sge_addr
.hi
= (addr
>> 32);
1967 bd
[bd_count
].sge_len
= (u16
)sg_len
;
1969 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_IO
,
1970 "single-cashed-sgl: bd_count:%d addr=%llx, len=%x",
1971 sg_count
, addr
, sg_len
);
1976 scsi_for_each_sg(sc
, sg
, sg_count
, i
) {
1977 sg_len
= sg_dma_len(sg
);
1978 addr
= (u64
)sg_dma_address(sg
);
1979 end_addr
= (addr
+ sg_len
);
1982 * first sg elem in the 'list',
1983 * check if end addr is page-aligned.
1985 if ((i
== 0) && (sg_count
> 1) && (end_addr
% QEDI_PAGE_SIZE
))
1986 cmd
->use_slowpath
= true;
1989 * last sg elem in the 'list',
1990 * check if start addr is page-aligned.
1992 else if ((i
== (sg_count
- 1)) &&
1993 (sg_count
> 1) && (addr
% QEDI_PAGE_SIZE
))
1994 cmd
->use_slowpath
= true;
1997 * middle sg elements in list,
1998 * check if start and end addr is page-aligned
2000 else if ((i
!= 0) && (i
!= (sg_count
- 1)) &&
2001 ((addr
% QEDI_PAGE_SIZE
) ||
2002 (end_addr
% QEDI_PAGE_SIZE
)))
2003 cmd
->use_slowpath
= true;
2005 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_IO
, "sg[%d] size=0x%x",
2008 if (sg_len
> QEDI_BD_SPLIT_SZ
) {
2009 sg_frags
= qedi_split_bd(cmd
, addr
, sg_len
, bd_count
);
2012 bd
[bd_count
].sge_addr
.lo
= addr
& 0xffffffff;
2013 bd
[bd_count
].sge_addr
.hi
= addr
>> 32;
2014 bd
[bd_count
].sge_len
= sg_len
;
2016 byte_count
+= sg_len
;
2017 bd_count
+= sg_frags
;
2020 if (byte_count
!= scsi_bufflen(sc
))
2021 QEDI_ERR(&qedi
->dbg_ctx
,
2022 "byte_count = %d != scsi_bufflen = %d\n", byte_count
,
2025 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_IO
, "byte_count = %d\n",
2028 WARN_ON(byte_count
!= scsi_bufflen(sc
));
2033 static void qedi_iscsi_map_sg_list(struct qedi_cmd
*cmd
)
2036 struct scsi_cmnd
*sc
= cmd
->scsi_cmd
;
2038 if (scsi_sg_count(sc
)) {
2039 bd_count
= qedi_map_scsi_sg(cmd
->conn
->qedi
, cmd
);
2043 struct iscsi_sge
*bd
= cmd
->io_tbl
.sge_tbl
;
2045 bd
[0].sge_addr
.lo
= 0;
2046 bd
[0].sge_addr
.hi
= 0;
2050 cmd
->io_tbl
.sge_valid
= bd_count
;
2053 static void qedi_cpy_scsi_cdb(struct scsi_cmnd
*sc
, u32
*dstp
)
2059 lpcnt
= sc
->cmd_len
/ sizeof(dword
);
2060 srcp
= (u8
*)sc
->cmnd
;
2062 memcpy(&dword
, (const void *)srcp
, 4);
2063 *dstp
= cpu_to_be32(dword
);
2067 if (sc
->cmd_len
& 0x3) {
2068 dword
= (u32
)srcp
[0] | ((u32
)srcp
[1] << 8);
2069 *dstp
= cpu_to_be32(dword
);
2073 void qedi_trace_io(struct qedi_ctx
*qedi
, struct iscsi_task
*task
,
2074 u16 tid
, int8_t direction
)
2076 struct qedi_io_log
*io_log
;
2077 struct iscsi_conn
*conn
= task
->conn
;
2078 struct qedi_conn
*qedi_conn
= conn
->dd_data
;
2079 struct scsi_cmnd
*sc_cmd
= task
->sc
;
2080 unsigned long flags
;
2083 spin_lock_irqsave(&qedi
->io_trace_lock
, flags
);
2085 io_log
= &qedi
->io_trace_buf
[qedi
->io_trace_idx
];
2086 io_log
->direction
= direction
;
2087 io_log
->task_id
= tid
;
2088 io_log
->cid
= qedi_conn
->iscsi_conn_id
;
2089 io_log
->lun
= sc_cmd
->device
->lun
;
2090 io_log
->op
= sc_cmd
->cmnd
[0];
2091 op
= sc_cmd
->cmnd
[0];
2092 io_log
->lba
[0] = sc_cmd
->cmnd
[2];
2093 io_log
->lba
[1] = sc_cmd
->cmnd
[3];
2094 io_log
->lba
[2] = sc_cmd
->cmnd
[4];
2095 io_log
->lba
[3] = sc_cmd
->cmnd
[5];
2096 io_log
->bufflen
= scsi_bufflen(sc_cmd
);
2097 io_log
->sg_count
= scsi_sg_count(sc_cmd
);
2098 io_log
->fast_sgs
= qedi
->fast_sgls
;
2099 io_log
->cached_sgs
= qedi
->cached_sgls
;
2100 io_log
->slow_sgs
= qedi
->slow_sgls
;
2101 io_log
->cached_sge
= qedi
->use_cached_sge
;
2102 io_log
->slow_sge
= qedi
->use_slow_sge
;
2103 io_log
->fast_sge
= qedi
->use_fast_sge
;
2104 io_log
->result
= sc_cmd
->result
;
2105 io_log
->jiffies
= jiffies
;
2106 io_log
->blk_req_cpu
= smp_processor_id();
2108 if (direction
== QEDI_IO_TRACE_REQ
) {
2109 /* For requests we only care about the submission CPU */
2110 io_log
->req_cpu
= smp_processor_id() % qedi
->num_queues
;
2111 io_log
->intr_cpu
= 0;
2112 io_log
->blk_rsp_cpu
= 0;
2113 } else if (direction
== QEDI_IO_TRACE_RSP
) {
2114 io_log
->req_cpu
= smp_processor_id() % qedi
->num_queues
;
2115 io_log
->intr_cpu
= qedi
->intr_cpu
;
2116 io_log
->blk_rsp_cpu
= smp_processor_id();
2119 qedi
->io_trace_idx
++;
2120 if (qedi
->io_trace_idx
== QEDI_IO_TRACE_SIZE
)
2121 qedi
->io_trace_idx
= 0;
2123 qedi
->use_cached_sge
= false;
2124 qedi
->use_slow_sge
= false;
2125 qedi
->use_fast_sge
= false;
2127 spin_unlock_irqrestore(&qedi
->io_trace_lock
, flags
);
2130 int qedi_iscsi_send_ioreq(struct iscsi_task
*task
)
2132 struct iscsi_conn
*conn
= task
->conn
;
2133 struct iscsi_session
*session
= conn
->session
;
2134 struct Scsi_Host
*shost
= iscsi_session_to_shost(session
->cls_session
);
2135 struct qedi_ctx
*qedi
= iscsi_host_priv(shost
);
2136 struct qedi_conn
*qedi_conn
= conn
->dd_data
;
2137 struct qedi_cmd
*cmd
= task
->dd_data
;
2138 struct scsi_cmnd
*sc
= task
->sc
;
2139 struct iscsi_task_context
*fw_task_ctx
;
2140 struct iscsi_cached_sge_ctx
*cached_sge
;
2141 struct iscsi_phys_sgl_ctx
*phys_sgl
;
2142 struct iscsi_virt_sgl_ctx
*virt_sgl
;
2143 struct ystorm_iscsi_task_st_ctx
*yst_cxt
;
2144 struct mstorm_iscsi_task_st_ctx
*mst_cxt
;
2145 struct iscsi_sgl
*sgl_struct
;
2146 struct iscsi_sge
*single_sge
;
2147 struct iscsi_scsi_req
*hdr
= (struct iscsi_scsi_req
*)task
->hdr
;
2148 struct iscsi_sge
*bd
= cmd
->io_tbl
.sge_tbl
;
2149 enum iscsi_task_type task_type
;
2150 struct iscsi_cmd_hdr
*fw_cmd
;
2153 u16 cq_idx
= smp_processor_id() % qedi
->num_queues
;
2154 s16 ptu_invalidate
= 0;
2158 tid
= qedi_get_task_idx(qedi
);
2162 qedi_iscsi_map_sg_list(cmd
);
2164 int_to_scsilun(sc
->device
->lun
, (struct scsi_lun
*)lun
);
2165 fw_task_ctx
= qedi_get_task_mem(&qedi
->tasks
, tid
);
2167 memset(fw_task_ctx
, 0, sizeof(struct iscsi_task_context
));
2170 /* Ystorm context */
2171 fw_cmd
= &fw_task_ctx
->ystorm_st_context
.pdu_hdr
.cmd
;
2172 SET_FIELD(fw_cmd
->flags_attr
, ISCSI_CMD_HDR_ATTR
, ISCSI_ATTR_SIMPLE
);
2174 if (sc
->sc_data_direction
== DMA_TO_DEVICE
) {
2175 if (conn
->session
->initial_r2t_en
) {
2176 exp_data
= min((conn
->session
->imm_data_en
*
2177 conn
->max_xmit_dlength
),
2178 conn
->session
->first_burst
);
2179 exp_data
= min(exp_data
, scsi_bufflen(sc
));
2180 fw_task_ctx
->ustorm_ag_context
.exp_data_acked
=
2181 cpu_to_le32(exp_data
);
2183 fw_task_ctx
->ustorm_ag_context
.exp_data_acked
=
2184 min(conn
->session
->first_burst
, scsi_bufflen(sc
));
2187 SET_FIELD(fw_cmd
->flags_attr
, ISCSI_CMD_HDR_WRITE
, 1);
2188 task_type
= ISCSI_TASK_TYPE_INITIATOR_WRITE
;
2190 if (scsi_bufflen(sc
))
2191 SET_FIELD(fw_cmd
->flags_attr
, ISCSI_CMD_HDR_READ
, 1);
2192 task_type
= ISCSI_TASK_TYPE_INITIATOR_READ
;
2195 fw_cmd
->lun
.lo
= be32_to_cpu(lun
[0]);
2196 fw_cmd
->lun
.hi
= be32_to_cpu(lun
[1]);
2198 qedi_update_itt_map(qedi
, tid
, task
->itt
, cmd
);
2199 fw_cmd
->itt
= qedi_set_itt(tid
, get_itt(task
->itt
));
2200 fw_cmd
->expected_transfer_length
= scsi_bufflen(sc
);
2201 fw_cmd
->cmd_sn
= be32_to_cpu(hdr
->cmdsn
);
2202 fw_cmd
->opcode
= hdr
->opcode
;
2203 qedi_cpy_scsi_cdb(sc
, (u32
*)fw_cmd
->cdb
);
2205 /* Mstorm context */
2206 fw_task_ctx
->mstorm_st_context
.sense_db
.lo
= (u32
)cmd
->sense_buffer_dma
;
2207 fw_task_ctx
->mstorm_st_context
.sense_db
.hi
=
2208 (u32
)((u64
)cmd
->sense_buffer_dma
>> 32);
2209 fw_task_ctx
->mstorm_ag_context
.task_cid
= qedi_conn
->iscsi_conn_id
;
2210 fw_task_ctx
->mstorm_st_context
.task_type
= task_type
;
2212 if (qedi
->tid_reuse_count
[tid
] == QEDI_MAX_TASK_NUM
) {
2214 qedi
->tid_reuse_count
[tid
] = 0;
2216 fw_task_ctx
->ystorm_st_context
.state
.reuse_count
=
2217 qedi
->tid_reuse_count
[tid
];
2218 fw_task_ctx
->mstorm_st_context
.reuse_count
=
2219 qedi
->tid_reuse_count
[tid
]++;
2221 /* Ustorm context */
2222 fw_task_ctx
->ustorm_st_context
.rem_rcv_len
= scsi_bufflen(sc
);
2223 fw_task_ctx
->ustorm_st_context
.exp_data_transfer_len
= scsi_bufflen(sc
);
2224 fw_task_ctx
->ustorm_st_context
.exp_data_sn
=
2225 be32_to_cpu(hdr
->exp_statsn
);
2226 fw_task_ctx
->ustorm_st_context
.task_type
= task_type
;
2227 fw_task_ctx
->ustorm_st_context
.cq_rss_number
= cq_idx
;
2228 fw_task_ctx
->ustorm_ag_context
.icid
= (u16
)qedi_conn
->iscsi_conn_id
;
2230 SET_FIELD(fw_task_ctx
->ustorm_ag_context
.flags1
,
2231 USTORM_ISCSI_TASK_AG_CTX_R2T2RECV
, 1);
2232 SET_FIELD(fw_task_ctx
->ustorm_st_context
.flags
,
2233 USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP
, 0);
2235 num_fast_sgs
= (cmd
->io_tbl
.sge_valid
?
2236 min((u16
)QEDI_FAST_SGE_COUNT
,
2237 (u16
)cmd
->io_tbl
.sge_valid
) : 0);
2238 SET_FIELD(fw_task_ctx
->ustorm_st_context
.reg1
.reg1_map
,
2239 ISCSI_REG1_NUM_FAST_SGES
, num_fast_sgs
);
2241 fw_task_ctx
->ustorm_st_context
.lun
.lo
= be32_to_cpu(lun
[0]);
2242 fw_task_ctx
->ustorm_st_context
.lun
.hi
= be32_to_cpu(lun
[1]);
2244 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_IO
, "Total sge count [%d]\n",
2245 cmd
->io_tbl
.sge_valid
);
2247 yst_cxt
= &fw_task_ctx
->ystorm_st_context
;
2248 mst_cxt
= &fw_task_ctx
->mstorm_st_context
;
2250 if (task_type
== ISCSI_TASK_TYPE_INITIATOR_WRITE
) {
2251 /* not considering superIO or FastIO */
2252 if (cmd
->io_tbl
.sge_valid
== 1) {
2253 cached_sge
= &yst_cxt
->state
.sgl_ctx_union
.cached_sge
;
2254 cached_sge
->sge
.sge_addr
.lo
= bd
[0].sge_addr
.lo
;
2255 cached_sge
->sge
.sge_addr
.hi
= bd
[0].sge_addr
.hi
;
2256 cached_sge
->sge
.sge_len
= bd
[0].sge_len
;
2257 qedi
->cached_sgls
++;
2258 } else if ((cmd
->io_tbl
.sge_valid
!= 1) && cmd
->use_slowpath
) {
2259 SET_FIELD(fw_task_ctx
->mstorm_st_context
.flags
.mflags
,
2260 ISCSI_MFLAGS_SLOW_IO
, 1);
2261 SET_FIELD(fw_task_ctx
->ustorm_st_context
.reg1
.reg1_map
,
2262 ISCSI_REG1_NUM_FAST_SGES
, 0);
2263 phys_sgl
= &yst_cxt
->state
.sgl_ctx_union
.phys_sgl
;
2264 phys_sgl
->sgl_base
.lo
= (u32
)(cmd
->io_tbl
.sge_tbl_dma
);
2265 phys_sgl
->sgl_base
.hi
=
2266 (u32
)((u64
)cmd
->io_tbl
.sge_tbl_dma
>> 32);
2267 phys_sgl
->sgl_size
= cmd
->io_tbl
.sge_valid
;
2269 } else if ((cmd
->io_tbl
.sge_valid
!= 1) && !cmd
->use_slowpath
) {
2270 SET_FIELD(fw_task_ctx
->mstorm_st_context
.flags
.mflags
,
2271 ISCSI_MFLAGS_SLOW_IO
, 0);
2272 SET_FIELD(fw_task_ctx
->ustorm_st_context
.reg1
.reg1_map
,
2273 ISCSI_REG1_NUM_FAST_SGES
,
2274 min((u16
)QEDI_FAST_SGE_COUNT
,
2275 (u16
)cmd
->io_tbl
.sge_valid
));
2276 virt_sgl
= &yst_cxt
->state
.sgl_ctx_union
.virt_sgl
;
2277 virt_sgl
->sgl_base
.lo
= (u32
)(cmd
->io_tbl
.sge_tbl_dma
);
2278 virt_sgl
->sgl_base
.hi
=
2279 (u32
)((u64
)cmd
->io_tbl
.sge_tbl_dma
>> 32);
2280 virt_sgl
->sgl_initial_offset
=
2281 (u32
)bd
[0].sge_addr
.lo
& (QEDI_PAGE_SIZE
- 1);
2284 fw_task_ctx
->mstorm_st_context
.sgl_size
= cmd
->io_tbl
.sge_valid
;
2285 fw_task_ctx
->mstorm_st_context
.rem_task_size
= scsi_bufflen(sc
);
2288 if (cmd
->io_tbl
.sge_valid
== 1) {
2289 SET_FIELD(fw_task_ctx
->mstorm_st_context
.flags
.mflags
,
2290 ISCSI_MFLAGS_SLOW_IO
, 0);
2291 SET_FIELD(fw_task_ctx
->mstorm_st_context
.flags
.mflags
,
2292 ISCSI_MFLAGS_SINGLE_SGE
, 1);
2293 single_sge
= &mst_cxt
->sgl_union
.single_sge
;
2294 single_sge
->sge_addr
.lo
= bd
[0].sge_addr
.lo
;
2295 single_sge
->sge_addr
.hi
= bd
[0].sge_addr
.hi
;
2296 single_sge
->sge_len
= bd
[0].sge_len
;
2297 qedi
->cached_sgls
++;
2298 } else if ((cmd
->io_tbl
.sge_valid
!= 1) && cmd
->use_slowpath
) {
2299 sgl_struct
= &mst_cxt
->sgl_union
.sgl_struct
;
2300 sgl_struct
->sgl_addr
.lo
=
2301 (u32
)(cmd
->io_tbl
.sge_tbl_dma
);
2302 sgl_struct
->sgl_addr
.hi
=
2303 (u32
)((u64
)cmd
->io_tbl
.sge_tbl_dma
>> 32);
2304 SET_FIELD(fw_task_ctx
->mstorm_st_context
.flags
.mflags
,
2305 ISCSI_MFLAGS_SLOW_IO
, 1);
2306 SET_FIELD(fw_task_ctx
->ustorm_st_context
.reg1
.reg1_map
,
2307 ISCSI_REG1_NUM_FAST_SGES
, 0);
2308 sgl_struct
->updated_sge_size
= 0;
2309 sgl_struct
->updated_sge_offset
= 0;
2311 } else if ((cmd
->io_tbl
.sge_valid
!= 1) && !cmd
->use_slowpath
) {
2312 sgl_struct
= &mst_cxt
->sgl_union
.sgl_struct
;
2313 sgl_struct
->sgl_addr
.lo
=
2314 (u32
)(cmd
->io_tbl
.sge_tbl_dma
);
2315 sgl_struct
->sgl_addr
.hi
=
2316 (u32
)((u64
)cmd
->io_tbl
.sge_tbl_dma
>> 32);
2317 sgl_struct
->byte_offset
=
2318 (u32
)bd
[0].sge_addr
.lo
& (QEDI_PAGE_SIZE
- 1);
2319 SET_FIELD(fw_task_ctx
->mstorm_st_context
.flags
.mflags
,
2320 ISCSI_MFLAGS_SLOW_IO
, 0);
2321 SET_FIELD(fw_task_ctx
->ustorm_st_context
.reg1
.reg1_map
,
2322 ISCSI_REG1_NUM_FAST_SGES
, 0);
2323 sgl_struct
->updated_sge_size
= 0;
2324 sgl_struct
->updated_sge_offset
= 0;
2327 fw_task_ctx
->mstorm_st_context
.sgl_size
= cmd
->io_tbl
.sge_valid
;
2328 fw_task_ctx
->mstorm_st_context
.rem_task_size
= scsi_bufflen(sc
);
2331 if (cmd
->io_tbl
.sge_valid
== 1)
2333 qedi
->use_cached_sge
= true;
2335 if (cmd
->use_slowpath
)
2336 qedi
->use_slow_sge
= true;
2338 qedi
->use_fast_sge
= true;
2340 QEDI_INFO(&qedi
->dbg_ctx
, QEDI_LOG_IO
,
2341 "%s: %s-SGL: num_sges=0x%x first-sge-lo=0x%x first-sge-hi=0x%x",
2342 (task_type
== ISCSI_TASK_TYPE_INITIATOR_WRITE
) ?
2343 "Write " : "Read ", (cmd
->io_tbl
.sge_valid
== 1) ?
2344 "Single" : (cmd
->use_slowpath
? "SLOW" : "FAST"),
2345 (u16
)cmd
->io_tbl
.sge_valid
, (u32
)(cmd
->io_tbl
.sge_tbl_dma
),
2346 (u32
)((u64
)cmd
->io_tbl
.sge_tbl_dma
>> 32));
2348 /* Add command in active command list */
2349 spin_lock(&qedi_conn
->list_lock
);
2350 list_add_tail(&cmd
->io_cmd
, &qedi_conn
->active_cmd_list
);
2351 cmd
->io_cmd_in_list
= true;
2352 qedi_conn
->active_cmd_count
++;
2353 spin_unlock(&qedi_conn
->list_lock
);
2355 qedi_add_to_sq(qedi_conn
, task
, tid
, ptu_invalidate
, false);
2356 qedi_ring_doorbell(qedi_conn
);
2357 if (qedi_io_tracing
)
2358 qedi_trace_io(qedi
, task
, tid
, QEDI_IO_TRACE_REQ
);
2363 int qedi_iscsi_cleanup_task(struct iscsi_task
*task
, bool mark_cmd_node_deleted
)
2365 struct iscsi_conn
*conn
= task
->conn
;
2366 struct qedi_conn
*qedi_conn
= conn
->dd_data
;
2367 struct qedi_cmd
*cmd
= task
->dd_data
;
2368 s16 ptu_invalidate
= 0;
2370 QEDI_INFO(&qedi_conn
->qedi
->dbg_ctx
, QEDI_LOG_SCSI_TM
,
2371 "issue cleanup tid=0x%x itt=0x%x task_state=%d cmd_state=0%x cid=0x%x\n",
2372 cmd
->task_id
, get_itt(task
->itt
), task
->state
,
2373 cmd
->state
, qedi_conn
->iscsi_conn_id
);
2375 qedi_add_to_sq(qedi_conn
, task
, cmd
->task_id
, ptu_invalidate
, true);
2376 qedi_ring_doorbell(qedi_conn
);