]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/scsi/qedi/qedi_fw.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
ASoC: wm_adsp: add support for DSP region lock
[mirror_ubuntu-bionic-kernel.git] / drivers / scsi / qedi / qedi_fw.c
1 /*
2 * QLogic iSCSI Offload Driver
3 * Copyright (c) 2016 Cavium Inc.
4 *
5 * This software is available under the terms of the GNU General Public License
6 * (GPL) Version 2, available from the file COPYING in the main directory of
7 * this source tree.
8 */
9
10 #include <linux/blkdev.h>
11 #include <scsi/scsi_tcq.h>
12 #include <linux/delay.h>
13
14 #include "qedi.h"
15 #include "qedi_iscsi.h"
16 #include "qedi_gbl.h"
17
18 static int qedi_send_iscsi_tmf(struct qedi_conn *qedi_conn,
19 struct iscsi_task *mtask);
20
21 void qedi_iscsi_unmap_sg_list(struct qedi_cmd *cmd)
22 {
23 struct scsi_cmnd *sc = cmd->scsi_cmd;
24
25 if (cmd->io_tbl.sge_valid && sc) {
26 cmd->io_tbl.sge_valid = 0;
27 scsi_dma_unmap(sc);
28 }
29 }
30
31 static void qedi_process_logout_resp(struct qedi_ctx *qedi,
32 union iscsi_cqe *cqe,
33 struct iscsi_task *task,
34 struct qedi_conn *qedi_conn)
35 {
36 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
37 struct iscsi_logout_rsp *resp_hdr;
38 struct iscsi_session *session = conn->session;
39 struct iscsi_logout_response_hdr *cqe_logout_response;
40 struct qedi_cmd *cmd;
41
42 cmd = (struct qedi_cmd *)task->dd_data;
43 cqe_logout_response = &cqe->cqe_common.iscsi_hdr.logout_response;
44 spin_lock(&session->back_lock);
45 resp_hdr = (struct iscsi_logout_rsp *)&qedi_conn->gen_pdu.resp_hdr;
46 memset(resp_hdr, 0, sizeof(struct iscsi_hdr));
47 resp_hdr->opcode = cqe_logout_response->opcode;
48 resp_hdr->flags = cqe_logout_response->flags;
49 resp_hdr->hlength = 0;
50
51 resp_hdr->itt = build_itt(cqe->cqe_solicited.itid, conn->session->age);
52 resp_hdr->statsn = cpu_to_be32(cqe_logout_response->stat_sn);
53 resp_hdr->exp_cmdsn = cpu_to_be32(cqe_logout_response->exp_cmd_sn);
54 resp_hdr->max_cmdsn = cpu_to_be32(cqe_logout_response->max_cmd_sn);
55
56 resp_hdr->t2wait = cpu_to_be32(cqe_logout_response->time2wait);
57 resp_hdr->t2retain = cpu_to_be32(cqe_logout_response->time2retain);
58
59 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID,
60 "Freeing tid=0x%x for cid=0x%x\n",
61 cmd->task_id, qedi_conn->iscsi_conn_id);
62
63 if (likely(cmd->io_cmd_in_list)) {
64 cmd->io_cmd_in_list = false;
65 list_del_init(&cmd->io_cmd);
66 qedi_conn->active_cmd_count--;
67 } else {
68 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
69 "Active cmd list node already deleted, tid=0x%x, cid=0x%x, io_cmd_node=%p\n",
70 cmd->task_id, qedi_conn->iscsi_conn_id,
71 &cmd->io_cmd);
72 }
73
74 cmd->state = RESPONSE_RECEIVED;
75 qedi_clear_task_idx(qedi, cmd->task_id);
76 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr, NULL, 0);
77
78 spin_unlock(&session->back_lock);
79 }
80
81 static void qedi_process_text_resp(struct qedi_ctx *qedi,
82 union iscsi_cqe *cqe,
83 struct iscsi_task *task,
84 struct qedi_conn *qedi_conn)
85 {
86 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
87 struct iscsi_session *session = conn->session;
88 struct iscsi_task_context *task_ctx;
89 struct iscsi_text_rsp *resp_hdr_ptr;
90 struct iscsi_text_response_hdr *cqe_text_response;
91 struct qedi_cmd *cmd;
92 int pld_len;
93 u32 *tmp;
94
95 cmd = (struct qedi_cmd *)task->dd_data;
96 task_ctx = qedi_get_task_mem(&qedi->tasks, cmd->task_id);
97
98 cqe_text_response = &cqe->cqe_common.iscsi_hdr.text_response;
99 spin_lock(&session->back_lock);
100 resp_hdr_ptr = (struct iscsi_text_rsp *)&qedi_conn->gen_pdu.resp_hdr;
101 memset(resp_hdr_ptr, 0, sizeof(struct iscsi_hdr));
102 resp_hdr_ptr->opcode = cqe_text_response->opcode;
103 resp_hdr_ptr->flags = cqe_text_response->flags;
104 resp_hdr_ptr->hlength = 0;
105
106 hton24(resp_hdr_ptr->dlength,
107 (cqe_text_response->hdr_second_dword &
108 ISCSI_TEXT_RESPONSE_HDR_DATA_SEG_LEN_MASK));
109 tmp = (u32 *)resp_hdr_ptr->dlength;
110
111 resp_hdr_ptr->itt = build_itt(cqe->cqe_solicited.itid,
112 conn->session->age);
113 resp_hdr_ptr->ttt = cqe_text_response->ttt;
114 resp_hdr_ptr->statsn = cpu_to_be32(cqe_text_response->stat_sn);
115 resp_hdr_ptr->exp_cmdsn = cpu_to_be32(cqe_text_response->exp_cmd_sn);
116 resp_hdr_ptr->max_cmdsn = cpu_to_be32(cqe_text_response->max_cmd_sn);
117
118 pld_len = cqe_text_response->hdr_second_dword &
119 ISCSI_TEXT_RESPONSE_HDR_DATA_SEG_LEN_MASK;
120 qedi_conn->gen_pdu.resp_wr_ptr = qedi_conn->gen_pdu.resp_buf + pld_len;
121
122 memset(task_ctx, '\0', sizeof(*task_ctx));
123
124 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID,
125 "Freeing tid=0x%x for cid=0x%x\n",
126 cmd->task_id, qedi_conn->iscsi_conn_id);
127
128 if (likely(cmd->io_cmd_in_list)) {
129 cmd->io_cmd_in_list = false;
130 list_del_init(&cmd->io_cmd);
131 qedi_conn->active_cmd_count--;
132 } else {
133 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
134 "Active cmd list node already deleted, tid=0x%x, cid=0x%x, io_cmd_node=%p\n",
135 cmd->task_id, qedi_conn->iscsi_conn_id,
136 &cmd->io_cmd);
137 }
138
139 cmd->state = RESPONSE_RECEIVED;
140 qedi_clear_task_idx(qedi, cmd->task_id);
141
142 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr_ptr,
143 qedi_conn->gen_pdu.resp_buf,
144 (qedi_conn->gen_pdu.resp_wr_ptr -
145 qedi_conn->gen_pdu.resp_buf));
146 spin_unlock(&session->back_lock);
147 }
148
149 static void qedi_tmf_resp_work(struct work_struct *work)
150 {
151 struct qedi_cmd *qedi_cmd =
152 container_of(work, struct qedi_cmd, tmf_work);
153 struct qedi_conn *qedi_conn = qedi_cmd->conn;
154 struct qedi_ctx *qedi = qedi_conn->qedi;
155 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
156 struct iscsi_session *session = conn->session;
157 struct iscsi_tm_rsp *resp_hdr_ptr;
158 struct iscsi_cls_session *cls_sess;
159 int rval = 0;
160
161 set_bit(QEDI_CONN_FW_CLEANUP, &qedi_conn->flags);
162 resp_hdr_ptr = (struct iscsi_tm_rsp *)qedi_cmd->tmf_resp_buf;
163 cls_sess = iscsi_conn_to_session(qedi_conn->cls_conn);
164
165 iscsi_block_session(session->cls_session);
166 rval = qedi_cleanup_all_io(qedi, qedi_conn, qedi_cmd->task, true);
167 if (rval) {
168 qedi_clear_task_idx(qedi, qedi_cmd->task_id);
169 iscsi_unblock_session(session->cls_session);
170 goto exit_tmf_resp;
171 }
172
173 iscsi_unblock_session(session->cls_session);
174 qedi_clear_task_idx(qedi, qedi_cmd->task_id);
175
176 spin_lock(&session->back_lock);
177 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr_ptr, NULL, 0);
178 spin_unlock(&session->back_lock);
179
180 exit_tmf_resp:
181 kfree(resp_hdr_ptr);
182 clear_bit(QEDI_CONN_FW_CLEANUP, &qedi_conn->flags);
183 }
184
185 static void qedi_process_tmf_resp(struct qedi_ctx *qedi,
186 union iscsi_cqe *cqe,
187 struct iscsi_task *task,
188 struct qedi_conn *qedi_conn)
189
190 {
191 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
192 struct iscsi_session *session = conn->session;
193 struct iscsi_tmf_response_hdr *cqe_tmp_response;
194 struct iscsi_tm_rsp *resp_hdr_ptr;
195 struct iscsi_tm *tmf_hdr;
196 struct qedi_cmd *qedi_cmd = NULL;
197 u32 *tmp;
198
199 cqe_tmp_response = &cqe->cqe_common.iscsi_hdr.tmf_response;
200
201 qedi_cmd = task->dd_data;
202 qedi_cmd->tmf_resp_buf = kzalloc(sizeof(*resp_hdr_ptr), GFP_KERNEL);
203 if (!qedi_cmd->tmf_resp_buf) {
204 QEDI_ERR(&qedi->dbg_ctx,
205 "Failed to allocate resp buf, cid=0x%x\n",
206 qedi_conn->iscsi_conn_id);
207 return;
208 }
209
210 spin_lock(&session->back_lock);
211 resp_hdr_ptr = (struct iscsi_tm_rsp *)qedi_cmd->tmf_resp_buf;
212 memset(resp_hdr_ptr, 0, sizeof(struct iscsi_tm_rsp));
213
214 /* Fill up the header */
215 resp_hdr_ptr->opcode = cqe_tmp_response->opcode;
216 resp_hdr_ptr->flags = cqe_tmp_response->hdr_flags;
217 resp_hdr_ptr->response = cqe_tmp_response->hdr_response;
218 resp_hdr_ptr->hlength = 0;
219
220 hton24(resp_hdr_ptr->dlength,
221 (cqe_tmp_response->hdr_second_dword &
222 ISCSI_TMF_RESPONSE_HDR_DATA_SEG_LEN_MASK));
223 tmp = (u32 *)resp_hdr_ptr->dlength;
224 resp_hdr_ptr->itt = build_itt(cqe->cqe_solicited.itid,
225 conn->session->age);
226 resp_hdr_ptr->statsn = cpu_to_be32(cqe_tmp_response->stat_sn);
227 resp_hdr_ptr->exp_cmdsn = cpu_to_be32(cqe_tmp_response->exp_cmd_sn);
228 resp_hdr_ptr->max_cmdsn = cpu_to_be32(cqe_tmp_response->max_cmd_sn);
229
230 tmf_hdr = (struct iscsi_tm *)qedi_cmd->task->hdr;
231
232 if (likely(qedi_cmd->io_cmd_in_list)) {
233 qedi_cmd->io_cmd_in_list = false;
234 list_del_init(&qedi_cmd->io_cmd);
235 qedi_conn->active_cmd_count--;
236 }
237
238 if (((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) ==
239 ISCSI_TM_FUNC_LOGICAL_UNIT_RESET) ||
240 ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) ==
241 ISCSI_TM_FUNC_TARGET_WARM_RESET) ||
242 ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) ==
243 ISCSI_TM_FUNC_TARGET_COLD_RESET)) {
244 INIT_WORK(&qedi_cmd->tmf_work, qedi_tmf_resp_work);
245 queue_work(qedi->tmf_thread, &qedi_cmd->tmf_work);
246 goto unblock_sess;
247 }
248
249 qedi_clear_task_idx(qedi, qedi_cmd->task_id);
250
251 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr_ptr, NULL, 0);
252 kfree(resp_hdr_ptr);
253
254 unblock_sess:
255 spin_unlock(&session->back_lock);
256 }
257
258 static void qedi_process_login_resp(struct qedi_ctx *qedi,
259 union iscsi_cqe *cqe,
260 struct iscsi_task *task,
261 struct qedi_conn *qedi_conn)
262 {
263 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
264 struct iscsi_session *session = conn->session;
265 struct iscsi_task_context *task_ctx;
266 struct iscsi_login_rsp *resp_hdr_ptr;
267 struct iscsi_login_response_hdr *cqe_login_response;
268 struct qedi_cmd *cmd;
269 int pld_len;
270 u32 *tmp;
271
272 cmd = (struct qedi_cmd *)task->dd_data;
273
274 cqe_login_response = &cqe->cqe_common.iscsi_hdr.login_response;
275 task_ctx = qedi_get_task_mem(&qedi->tasks, cmd->task_id);
276
277 spin_lock(&session->back_lock);
278 resp_hdr_ptr = (struct iscsi_login_rsp *)&qedi_conn->gen_pdu.resp_hdr;
279 memset(resp_hdr_ptr, 0, sizeof(struct iscsi_login_rsp));
280 resp_hdr_ptr->opcode = cqe_login_response->opcode;
281 resp_hdr_ptr->flags = cqe_login_response->flags_attr;
282 resp_hdr_ptr->hlength = 0;
283
284 hton24(resp_hdr_ptr->dlength,
285 (cqe_login_response->hdr_second_dword &
286 ISCSI_LOGIN_RESPONSE_HDR_DATA_SEG_LEN_MASK));
287 tmp = (u32 *)resp_hdr_ptr->dlength;
288 resp_hdr_ptr->itt = build_itt(cqe->cqe_solicited.itid,
289 conn->session->age);
290 resp_hdr_ptr->tsih = cqe_login_response->tsih;
291 resp_hdr_ptr->statsn = cpu_to_be32(cqe_login_response->stat_sn);
292 resp_hdr_ptr->exp_cmdsn = cpu_to_be32(cqe_login_response->exp_cmd_sn);
293 resp_hdr_ptr->max_cmdsn = cpu_to_be32(cqe_login_response->max_cmd_sn);
294 resp_hdr_ptr->status_class = cqe_login_response->status_class;
295 resp_hdr_ptr->status_detail = cqe_login_response->status_detail;
296 pld_len = cqe_login_response->hdr_second_dword &
297 ISCSI_LOGIN_RESPONSE_HDR_DATA_SEG_LEN_MASK;
298 qedi_conn->gen_pdu.resp_wr_ptr = qedi_conn->gen_pdu.resp_buf + pld_len;
299
300 if (likely(cmd->io_cmd_in_list)) {
301 cmd->io_cmd_in_list = false;
302 list_del_init(&cmd->io_cmd);
303 qedi_conn->active_cmd_count--;
304 }
305
306 memset(task_ctx, '\0', sizeof(*task_ctx));
307
308 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr_ptr,
309 qedi_conn->gen_pdu.resp_buf,
310 (qedi_conn->gen_pdu.resp_wr_ptr -
311 qedi_conn->gen_pdu.resp_buf));
312
313 spin_unlock(&session->back_lock);
314 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID,
315 "Freeing tid=0x%x for cid=0x%x\n",
316 cmd->task_id, qedi_conn->iscsi_conn_id);
317 cmd->state = RESPONSE_RECEIVED;
318 qedi_clear_task_idx(qedi, cmd->task_id);
319 }
320
321 static void qedi_get_rq_bdq_buf(struct qedi_ctx *qedi,
322 struct iscsi_cqe_unsolicited *cqe,
323 char *ptr, int len)
324 {
325 u16 idx = 0;
326
327 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
328 "pld_len [%d], bdq_prod_idx [%d], idx [%d]\n",
329 len, qedi->bdq_prod_idx,
330 (qedi->bdq_prod_idx % qedi->rq_num_entries));
331
332 /* Obtain buffer address from rqe_opaque */
333 idx = cqe->rqe_opaque.lo;
334 if ((idx < 0) || (idx > (QEDI_BDQ_NUM - 1))) {
335 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
336 "wrong idx %d returned by FW, dropping the unsolicited pkt\n",
337 idx);
338 return;
339 }
340
341 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
342 "rqe_opaque.lo [0x%p], rqe_opaque.hi [0x%p], idx [%d]\n",
343 cqe->rqe_opaque.lo, cqe->rqe_opaque.hi, idx);
344
345 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
346 "unsol_cqe_type = %d\n", cqe->unsol_cqe_type);
347 switch (cqe->unsol_cqe_type) {
348 case ISCSI_CQE_UNSOLICITED_SINGLE:
349 case ISCSI_CQE_UNSOLICITED_FIRST:
350 if (len)
351 memcpy(ptr, (void *)qedi->bdq[idx].buf_addr, len);
352 break;
353 case ISCSI_CQE_UNSOLICITED_MIDDLE:
354 case ISCSI_CQE_UNSOLICITED_LAST:
355 break;
356 default:
357 break;
358 }
359 }
360
361 static void qedi_put_rq_bdq_buf(struct qedi_ctx *qedi,
362 struct iscsi_cqe_unsolicited *cqe,
363 int count)
364 {
365 u16 tmp;
366 u16 idx = 0;
367 struct scsi_bd *pbl;
368
369 /* Obtain buffer address from rqe_opaque */
370 idx = cqe->rqe_opaque.lo;
371 if ((idx < 0) || (idx > (QEDI_BDQ_NUM - 1))) {
372 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
373 "wrong idx %d returned by FW, dropping the unsolicited pkt\n",
374 idx);
375 return;
376 }
377
378 pbl = (struct scsi_bd *)qedi->bdq_pbl;
379 pbl += (qedi->bdq_prod_idx % qedi->rq_num_entries);
380 pbl->address.hi = cpu_to_le32(QEDI_U64_HI(qedi->bdq[idx].buf_dma));
381 pbl->address.lo = cpu_to_le32(QEDI_U64_LO(qedi->bdq[idx].buf_dma));
382 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
383 "pbl [0x%p] pbl->address hi [0x%llx] lo [0x%llx] idx [%d]\n",
384 pbl, pbl->address.hi, pbl->address.lo, idx);
385 pbl->opaque.hi = 0;
386 pbl->opaque.lo = cpu_to_le32(QEDI_U64_LO(idx));
387
388 /* Increment producer to let f/w know we've handled the frame */
389 qedi->bdq_prod_idx += count;
390
391 writew(qedi->bdq_prod_idx, qedi->bdq_primary_prod);
392 tmp = readw(qedi->bdq_primary_prod);
393
394 writew(qedi->bdq_prod_idx, qedi->bdq_secondary_prod);
395 tmp = readw(qedi->bdq_secondary_prod);
396 }
397
398 static void qedi_unsol_pdu_adjust_bdq(struct qedi_ctx *qedi,
399 struct iscsi_cqe_unsolicited *cqe,
400 u32 pdu_len, u32 num_bdqs,
401 char *bdq_data)
402 {
403 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
404 "num_bdqs [%d]\n", num_bdqs);
405
406 qedi_get_rq_bdq_buf(qedi, cqe, bdq_data, pdu_len);
407 qedi_put_rq_bdq_buf(qedi, cqe, (num_bdqs + 1));
408 }
409
410 static int qedi_process_nopin_mesg(struct qedi_ctx *qedi,
411 union iscsi_cqe *cqe,
412 struct iscsi_task *task,
413 struct qedi_conn *qedi_conn, u16 que_idx)
414 {
415 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
416 struct iscsi_session *session = conn->session;
417 struct iscsi_nop_in_hdr *cqe_nop_in;
418 struct iscsi_nopin *hdr;
419 struct qedi_cmd *cmd;
420 int tgt_async_nop = 0;
421 u32 lun[2];
422 u32 pdu_len, num_bdqs;
423 char bdq_data[QEDI_BDQ_BUF_SIZE];
424 unsigned long flags;
425
426 spin_lock_bh(&session->back_lock);
427 cqe_nop_in = &cqe->cqe_common.iscsi_hdr.nop_in;
428
429 pdu_len = cqe_nop_in->hdr_second_dword &
430 ISCSI_NOP_IN_HDR_DATA_SEG_LEN_MASK;
431 num_bdqs = pdu_len / QEDI_BDQ_BUF_SIZE;
432
433 hdr = (struct iscsi_nopin *)&qedi_conn->gen_pdu.resp_hdr;
434 memset(hdr, 0, sizeof(struct iscsi_hdr));
435 hdr->opcode = cqe_nop_in->opcode;
436 hdr->max_cmdsn = cpu_to_be32(cqe_nop_in->max_cmd_sn);
437 hdr->exp_cmdsn = cpu_to_be32(cqe_nop_in->exp_cmd_sn);
438 hdr->statsn = cpu_to_be32(cqe_nop_in->stat_sn);
439 hdr->ttt = cpu_to_be32(cqe_nop_in->ttt);
440
441 if (cqe->cqe_common.cqe_type == ISCSI_CQE_TYPE_UNSOLICITED) {
442 spin_lock_irqsave(&qedi->hba_lock, flags);
443 qedi_unsol_pdu_adjust_bdq(qedi, &cqe->cqe_unsolicited,
444 pdu_len, num_bdqs, bdq_data);
445 hdr->itt = RESERVED_ITT;
446 tgt_async_nop = 1;
447 spin_unlock_irqrestore(&qedi->hba_lock, flags);
448 goto done;
449 }
450
451 /* Response to one of our nop-outs */
452 if (task) {
453 cmd = task->dd_data;
454 hdr->flags = ISCSI_FLAG_CMD_FINAL;
455 hdr->itt = build_itt(cqe->cqe_solicited.itid,
456 conn->session->age);
457 lun[0] = 0xffffffff;
458 lun[1] = 0xffffffff;
459 memcpy(&hdr->lun, lun, sizeof(struct scsi_lun));
460 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID,
461 "Freeing tid=0x%x for cid=0x%x\n",
462 cmd->task_id, qedi_conn->iscsi_conn_id);
463 cmd->state = RESPONSE_RECEIVED;
464 spin_lock(&qedi_conn->list_lock);
465 if (likely(cmd->io_cmd_in_list)) {
466 cmd->io_cmd_in_list = false;
467 list_del_init(&cmd->io_cmd);
468 qedi_conn->active_cmd_count--;
469 }
470
471 spin_unlock(&qedi_conn->list_lock);
472 qedi_clear_task_idx(qedi, cmd->task_id);
473 }
474
475 done:
476 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, bdq_data, pdu_len);
477
478 spin_unlock_bh(&session->back_lock);
479 return tgt_async_nop;
480 }
481
482 static void qedi_process_async_mesg(struct qedi_ctx *qedi,
483 union iscsi_cqe *cqe,
484 struct iscsi_task *task,
485 struct qedi_conn *qedi_conn,
486 u16 que_idx)
487 {
488 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
489 struct iscsi_session *session = conn->session;
490 struct iscsi_async_msg_hdr *cqe_async_msg;
491 struct iscsi_async *resp_hdr;
492 u32 lun[2];
493 u32 pdu_len, num_bdqs;
494 char bdq_data[QEDI_BDQ_BUF_SIZE];
495 unsigned long flags;
496
497 spin_lock_bh(&session->back_lock);
498
499 cqe_async_msg = &cqe->cqe_common.iscsi_hdr.async_msg;
500 pdu_len = cqe_async_msg->hdr_second_dword &
501 ISCSI_ASYNC_MSG_HDR_DATA_SEG_LEN_MASK;
502 num_bdqs = pdu_len / QEDI_BDQ_BUF_SIZE;
503
504 if (cqe->cqe_common.cqe_type == ISCSI_CQE_TYPE_UNSOLICITED) {
505 spin_lock_irqsave(&qedi->hba_lock, flags);
506 qedi_unsol_pdu_adjust_bdq(qedi, &cqe->cqe_unsolicited,
507 pdu_len, num_bdqs, bdq_data);
508 spin_unlock_irqrestore(&qedi->hba_lock, flags);
509 }
510
511 resp_hdr = (struct iscsi_async *)&qedi_conn->gen_pdu.resp_hdr;
512 memset(resp_hdr, 0, sizeof(struct iscsi_hdr));
513 resp_hdr->opcode = cqe_async_msg->opcode;
514 resp_hdr->flags = 0x80;
515
516 lun[0] = cpu_to_be32(cqe_async_msg->lun.lo);
517 lun[1] = cpu_to_be32(cqe_async_msg->lun.hi);
518 memcpy(&resp_hdr->lun, lun, sizeof(struct scsi_lun));
519 resp_hdr->exp_cmdsn = cpu_to_be32(cqe_async_msg->exp_cmd_sn);
520 resp_hdr->max_cmdsn = cpu_to_be32(cqe_async_msg->max_cmd_sn);
521 resp_hdr->statsn = cpu_to_be32(cqe_async_msg->stat_sn);
522
523 resp_hdr->async_event = cqe_async_msg->async_event;
524 resp_hdr->async_vcode = cqe_async_msg->async_vcode;
525
526 resp_hdr->param1 = cpu_to_be16(cqe_async_msg->param1_rsrv);
527 resp_hdr->param2 = cpu_to_be16(cqe_async_msg->param2_rsrv);
528 resp_hdr->param3 = cpu_to_be16(cqe_async_msg->param3_rsrv);
529
530 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr, bdq_data,
531 pdu_len);
532
533 spin_unlock_bh(&session->back_lock);
534 }
535
536 static void qedi_process_reject_mesg(struct qedi_ctx *qedi,
537 union iscsi_cqe *cqe,
538 struct iscsi_task *task,
539 struct qedi_conn *qedi_conn,
540 uint16_t que_idx)
541 {
542 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
543 struct iscsi_session *session = conn->session;
544 struct iscsi_reject_hdr *cqe_reject;
545 struct iscsi_reject *hdr;
546 u32 pld_len, num_bdqs;
547 unsigned long flags;
548
549 spin_lock_bh(&session->back_lock);
550 cqe_reject = &cqe->cqe_common.iscsi_hdr.reject;
551 pld_len = cqe_reject->hdr_second_dword &
552 ISCSI_REJECT_HDR_DATA_SEG_LEN_MASK;
553 num_bdqs = pld_len / QEDI_BDQ_BUF_SIZE;
554
555 if (cqe->cqe_common.cqe_type == ISCSI_CQE_TYPE_UNSOLICITED) {
556 spin_lock_irqsave(&qedi->hba_lock, flags);
557 qedi_unsol_pdu_adjust_bdq(qedi, &cqe->cqe_unsolicited,
558 pld_len, num_bdqs, conn->data);
559 spin_unlock_irqrestore(&qedi->hba_lock, flags);
560 }
561 hdr = (struct iscsi_reject *)&qedi_conn->gen_pdu.resp_hdr;
562 memset(hdr, 0, sizeof(struct iscsi_hdr));
563 hdr->opcode = cqe_reject->opcode;
564 hdr->reason = cqe_reject->hdr_reason;
565 hdr->flags = cqe_reject->hdr_flags;
566 hton24(hdr->dlength, (cqe_reject->hdr_second_dword &
567 ISCSI_REJECT_HDR_DATA_SEG_LEN_MASK));
568 hdr->max_cmdsn = cpu_to_be32(cqe_reject->max_cmd_sn);
569 hdr->exp_cmdsn = cpu_to_be32(cqe_reject->exp_cmd_sn);
570 hdr->statsn = cpu_to_be32(cqe_reject->stat_sn);
571 hdr->ffffffff = cpu_to_be32(0xffffffff);
572
573 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr,
574 conn->data, pld_len);
575 spin_unlock_bh(&session->back_lock);
576 }
577
578 static void qedi_scsi_completion(struct qedi_ctx *qedi,
579 union iscsi_cqe *cqe,
580 struct iscsi_task *task,
581 struct iscsi_conn *conn)
582 {
583 struct scsi_cmnd *sc_cmd;
584 struct qedi_cmd *cmd = task->dd_data;
585 struct iscsi_session *session = conn->session;
586 struct iscsi_scsi_rsp *hdr;
587 struct iscsi_data_in_hdr *cqe_data_in;
588 int datalen = 0;
589 struct qedi_conn *qedi_conn;
590 u32 iscsi_cid;
591 bool mark_cmd_node_deleted = false;
592 u8 cqe_err_bits = 0;
593
594 iscsi_cid = cqe->cqe_common.conn_id;
595 qedi_conn = qedi->cid_que.conn_cid_tbl[iscsi_cid];
596
597 cqe_data_in = &cqe->cqe_common.iscsi_hdr.data_in;
598 cqe_err_bits =
599 cqe->cqe_common.error_bitmap.error_bits.cqe_error_status_bits;
600
601 spin_lock_bh(&session->back_lock);
602 /* get the scsi command */
603 sc_cmd = cmd->scsi_cmd;
604
605 if (!sc_cmd) {
606 QEDI_WARN(&qedi->dbg_ctx, "sc_cmd is NULL!\n");
607 goto error;
608 }
609
610 if (!sc_cmd->SCp.ptr) {
611 QEDI_WARN(&qedi->dbg_ctx,
612 "SCp.ptr is NULL, returned in another context.\n");
613 goto error;
614 }
615
616 if (!sc_cmd->request) {
617 QEDI_WARN(&qedi->dbg_ctx,
618 "sc_cmd->request is NULL, sc_cmd=%p.\n",
619 sc_cmd);
620 goto error;
621 }
622
623 if (!sc_cmd->request->special) {
624 QEDI_WARN(&qedi->dbg_ctx,
625 "request->special is NULL so request not valid, sc_cmd=%p.\n",
626 sc_cmd);
627 goto error;
628 }
629
630 if (!sc_cmd->request->q) {
631 QEDI_WARN(&qedi->dbg_ctx,
632 "request->q is NULL so request is not valid, sc_cmd=%p.\n",
633 sc_cmd);
634 goto error;
635 }
636
637 qedi_iscsi_unmap_sg_list(cmd);
638
639 hdr = (struct iscsi_scsi_rsp *)task->hdr;
640 hdr->opcode = cqe_data_in->opcode;
641 hdr->max_cmdsn = cpu_to_be32(cqe_data_in->max_cmd_sn);
642 hdr->exp_cmdsn = cpu_to_be32(cqe_data_in->exp_cmd_sn);
643 hdr->itt = build_itt(cqe->cqe_solicited.itid, conn->session->age);
644 hdr->response = cqe_data_in->reserved1;
645 hdr->cmd_status = cqe_data_in->status_rsvd;
646 hdr->flags = cqe_data_in->flags;
647 hdr->residual_count = cpu_to_be32(cqe_data_in->residual_count);
648
649 if (hdr->cmd_status == SAM_STAT_CHECK_CONDITION) {
650 datalen = cqe_data_in->reserved2 &
651 ISCSI_COMMON_HDR_DATA_SEG_LEN_MASK;
652 memcpy((char *)conn->data, (char *)cmd->sense_buffer, datalen);
653 }
654
655 /* If f/w reports data underrun err then set residual to IO transfer
656 * length, set Underrun flag and clear Overrun flag explicitly
657 */
658 if (unlikely(cqe_err_bits &&
659 GET_FIELD(cqe_err_bits, CQE_ERROR_BITMAP_UNDER_RUN_ERR))) {
660 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
661 "Under flow itt=0x%x proto flags=0x%x tid=0x%x cid 0x%x fw resid 0x%x sc dlen 0x%x\n",
662 hdr->itt, cqe_data_in->flags, cmd->task_id,
663 qedi_conn->iscsi_conn_id, hdr->residual_count,
664 scsi_bufflen(sc_cmd));
665 hdr->residual_count = cpu_to_be32(scsi_bufflen(sc_cmd));
666 hdr->flags |= ISCSI_FLAG_CMD_UNDERFLOW;
667 hdr->flags &= (~ISCSI_FLAG_CMD_OVERFLOW);
668 }
669
670 spin_lock(&qedi_conn->list_lock);
671 if (likely(cmd->io_cmd_in_list)) {
672 cmd->io_cmd_in_list = false;
673 list_del_init(&cmd->io_cmd);
674 qedi_conn->active_cmd_count--;
675 mark_cmd_node_deleted = true;
676 }
677 spin_unlock(&qedi_conn->list_lock);
678
679 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID,
680 "Freeing tid=0x%x for cid=0x%x\n",
681 cmd->task_id, qedi_conn->iscsi_conn_id);
682 cmd->state = RESPONSE_RECEIVED;
683 if (qedi_io_tracing)
684 qedi_trace_io(qedi, task, cmd->task_id, QEDI_IO_TRACE_RSP);
685
686 qedi_clear_task_idx(qedi, cmd->task_id);
687 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr,
688 conn->data, datalen);
689 error:
690 spin_unlock_bh(&session->back_lock);
691 }
692
693 static void qedi_mtask_completion(struct qedi_ctx *qedi,
694 union iscsi_cqe *cqe,
695 struct iscsi_task *task,
696 struct qedi_conn *conn, uint16_t que_idx)
697 {
698 struct iscsi_conn *iscsi_conn;
699 u32 hdr_opcode;
700
701 hdr_opcode = cqe->cqe_common.iscsi_hdr.common.hdr_first_byte;
702 iscsi_conn = conn->cls_conn->dd_data;
703
704 switch (hdr_opcode) {
705 case ISCSI_OPCODE_SCSI_RESPONSE:
706 case ISCSI_OPCODE_DATA_IN:
707 qedi_scsi_completion(qedi, cqe, task, iscsi_conn);
708 break;
709 case ISCSI_OPCODE_LOGIN_RESPONSE:
710 qedi_process_login_resp(qedi, cqe, task, conn);
711 break;
712 case ISCSI_OPCODE_TMF_RESPONSE:
713 qedi_process_tmf_resp(qedi, cqe, task, conn);
714 break;
715 case ISCSI_OPCODE_TEXT_RESPONSE:
716 qedi_process_text_resp(qedi, cqe, task, conn);
717 break;
718 case ISCSI_OPCODE_LOGOUT_RESPONSE:
719 qedi_process_logout_resp(qedi, cqe, task, conn);
720 break;
721 case ISCSI_OPCODE_NOP_IN:
722 qedi_process_nopin_mesg(qedi, cqe, task, conn, que_idx);
723 break;
724 default:
725 QEDI_ERR(&qedi->dbg_ctx, "unknown opcode\n");
726 }
727 }
728
729 static void qedi_process_nopin_local_cmpl(struct qedi_ctx *qedi,
730 struct iscsi_cqe_solicited *cqe,
731 struct iscsi_task *task,
732 struct qedi_conn *qedi_conn)
733 {
734 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
735 struct iscsi_session *session = conn->session;
736 struct qedi_cmd *cmd = task->dd_data;
737
738 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_UNSOL,
739 "itid=0x%x, cmd task id=0x%x\n",
740 cqe->itid, cmd->task_id);
741
742 cmd->state = RESPONSE_RECEIVED;
743 qedi_clear_task_idx(qedi, cmd->task_id);
744
745 spin_lock_bh(&session->back_lock);
746 __iscsi_put_task(task);
747 spin_unlock_bh(&session->back_lock);
748 }
749
750 static void qedi_process_cmd_cleanup_resp(struct qedi_ctx *qedi,
751 struct iscsi_cqe_solicited *cqe,
752 struct iscsi_task *task,
753 struct iscsi_conn *conn)
754 {
755 struct qedi_work_map *work, *work_tmp;
756 u32 proto_itt = cqe->itid;
757 u32 ptmp_itt = 0;
758 itt_t protoitt = 0;
759 int found = 0;
760 struct qedi_cmd *qedi_cmd = NULL;
761 u32 rtid = 0;
762 u32 iscsi_cid;
763 struct qedi_conn *qedi_conn;
764 struct qedi_cmd *cmd_new, *dbg_cmd;
765 struct iscsi_task *mtask;
766 struct iscsi_tm *tmf_hdr = NULL;
767
768 iscsi_cid = cqe->conn_id;
769 qedi_conn = qedi->cid_que.conn_cid_tbl[iscsi_cid];
770
771 /* Based on this itt get the corresponding qedi_cmd */
772 spin_lock_bh(&qedi_conn->tmf_work_lock);
773 list_for_each_entry_safe(work, work_tmp, &qedi_conn->tmf_work_list,
774 list) {
775 if (work->rtid == proto_itt) {
776 /* We found the command */
777 qedi_cmd = work->qedi_cmd;
778 if (!qedi_cmd->list_tmf_work) {
779 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
780 "TMF work not found, cqe->tid=0x%x, cid=0x%x\n",
781 proto_itt, qedi_conn->iscsi_conn_id);
782 WARN_ON(1);
783 }
784 found = 1;
785 mtask = qedi_cmd->task;
786 tmf_hdr = (struct iscsi_tm *)mtask->hdr;
787 rtid = work->rtid;
788
789 list_del_init(&work->list);
790 kfree(work);
791 qedi_cmd->list_tmf_work = NULL;
792 }
793 }
794 spin_unlock_bh(&qedi_conn->tmf_work_lock);
795
796 if (found) {
797 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
798 "TMF work, cqe->tid=0x%x, tmf flags=0x%x, cid=0x%x\n",
799 proto_itt, tmf_hdr->flags, qedi_conn->iscsi_conn_id);
800
801 if ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) ==
802 ISCSI_TM_FUNC_ABORT_TASK) {
803 spin_lock_bh(&conn->session->back_lock);
804
805 protoitt = build_itt(get_itt(tmf_hdr->rtt),
806 conn->session->age);
807 task = iscsi_itt_to_task(conn, protoitt);
808
809 spin_unlock_bh(&conn->session->back_lock);
810
811 if (!task) {
812 QEDI_NOTICE(&qedi->dbg_ctx,
813 "IO task completed, tmf rtt=0x%x, cid=0x%x\n",
814 get_itt(tmf_hdr->rtt),
815 qedi_conn->iscsi_conn_id);
816 return;
817 }
818
819 dbg_cmd = task->dd_data;
820
821 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
822 "Abort tmf rtt=0x%x, i/o itt=0x%x, i/o tid=0x%x, cid=0x%x\n",
823 get_itt(tmf_hdr->rtt), get_itt(task->itt),
824 dbg_cmd->task_id, qedi_conn->iscsi_conn_id);
825
826 if (qedi_cmd->state == CLEANUP_WAIT_FAILED)
827 qedi_cmd->state = CLEANUP_RECV;
828
829 qedi_clear_task_idx(qedi_conn->qedi, rtid);
830
831 spin_lock(&qedi_conn->list_lock);
832 list_del_init(&dbg_cmd->io_cmd);
833 qedi_conn->active_cmd_count--;
834 spin_unlock(&qedi_conn->list_lock);
835 qedi_cmd->state = CLEANUP_RECV;
836 wake_up_interruptible(&qedi_conn->wait_queue);
837 }
838 } else if (qedi_conn->cmd_cleanup_req > 0) {
839 spin_lock_bh(&conn->session->back_lock);
840 qedi_get_proto_itt(qedi, cqe->itid, &ptmp_itt);
841 protoitt = build_itt(ptmp_itt, conn->session->age);
842 task = iscsi_itt_to_task(conn, protoitt);
843 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
844 "cleanup io itid=0x%x, protoitt=0x%x, cmd_cleanup_cmpl=%d, cid=0x%x\n",
845 cqe->itid, protoitt, qedi_conn->cmd_cleanup_cmpl,
846 qedi_conn->iscsi_conn_id);
847
848 spin_unlock_bh(&conn->session->back_lock);
849 if (!task) {
850 QEDI_NOTICE(&qedi->dbg_ctx,
851 "task is null, itid=0x%x, cid=0x%x\n",
852 cqe->itid, qedi_conn->iscsi_conn_id);
853 return;
854 }
855 qedi_conn->cmd_cleanup_cmpl++;
856 wake_up(&qedi_conn->wait_queue);
857 cmd_new = task->dd_data;
858
859 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID,
860 "Freeing tid=0x%x for cid=0x%x\n",
861 cqe->itid, qedi_conn->iscsi_conn_id);
862 qedi_clear_task_idx(qedi_conn->qedi, cqe->itid);
863
864 } else {
865 qedi_get_proto_itt(qedi, cqe->itid, &ptmp_itt);
866 protoitt = build_itt(ptmp_itt, conn->session->age);
867 task = iscsi_itt_to_task(conn, protoitt);
868 QEDI_ERR(&qedi->dbg_ctx,
869 "Delayed or untracked cleanup response, itt=0x%x, tid=0x%x, cid=0x%x, task=%p\n",
870 protoitt, cqe->itid, qedi_conn->iscsi_conn_id, task);
871 WARN_ON(1);
872 }
873 }
874
875 void qedi_fp_process_cqes(struct qedi_work *work)
876 {
877 struct qedi_ctx *qedi = work->qedi;
878 union iscsi_cqe *cqe = &work->cqe;
879 struct iscsi_task *task = NULL;
880 struct iscsi_nopout *nopout_hdr;
881 struct qedi_conn *q_conn;
882 struct iscsi_conn *conn;
883 struct qedi_cmd *qedi_cmd;
884 u32 comp_type;
885 u32 iscsi_cid;
886 u32 hdr_opcode;
887 u16 que_idx = work->que_idx;
888 u8 cqe_err_bits = 0;
889
890 comp_type = cqe->cqe_common.cqe_type;
891 hdr_opcode = cqe->cqe_common.iscsi_hdr.common.hdr_first_byte;
892 cqe_err_bits =
893 cqe->cqe_common.error_bitmap.error_bits.cqe_error_status_bits;
894
895 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
896 "fw_cid=0x%x, cqe type=0x%x, opcode=0x%x\n",
897 cqe->cqe_common.conn_id, comp_type, hdr_opcode);
898
899 if (comp_type >= MAX_ISCSI_CQES_TYPE) {
900 QEDI_WARN(&qedi->dbg_ctx, "Invalid CqE type\n");
901 return;
902 }
903
904 iscsi_cid = cqe->cqe_common.conn_id;
905 q_conn = qedi->cid_que.conn_cid_tbl[iscsi_cid];
906 if (!q_conn) {
907 QEDI_WARN(&qedi->dbg_ctx,
908 "Session no longer exists for cid=0x%x!!\n",
909 iscsi_cid);
910 return;
911 }
912
913 conn = q_conn->cls_conn->dd_data;
914
915 if (unlikely(cqe_err_bits &&
916 GET_FIELD(cqe_err_bits,
917 CQE_ERROR_BITMAP_DATA_DIGEST_ERR))) {
918 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
919 return;
920 }
921
922 switch (comp_type) {
923 case ISCSI_CQE_TYPE_SOLICITED:
924 case ISCSI_CQE_TYPE_SOLICITED_WITH_SENSE:
925 qedi_cmd = container_of(work, struct qedi_cmd, cqe_work);
926 task = qedi_cmd->task;
927 if (!task) {
928 QEDI_WARN(&qedi->dbg_ctx, "task is NULL\n");
929 return;
930 }
931
932 /* Process NOPIN local completion */
933 nopout_hdr = (struct iscsi_nopout *)task->hdr;
934 if ((nopout_hdr->itt == RESERVED_ITT) &&
935 (cqe->cqe_solicited.itid != (u16)RESERVED_ITT)) {
936 qedi_process_nopin_local_cmpl(qedi, &cqe->cqe_solicited,
937 task, q_conn);
938 } else {
939 cqe->cqe_solicited.itid =
940 qedi_get_itt(cqe->cqe_solicited);
941 /* Process other solicited responses */
942 qedi_mtask_completion(qedi, cqe, task, q_conn, que_idx);
943 }
944 break;
945 case ISCSI_CQE_TYPE_UNSOLICITED:
946 switch (hdr_opcode) {
947 case ISCSI_OPCODE_NOP_IN:
948 qedi_process_nopin_mesg(qedi, cqe, task, q_conn,
949 que_idx);
950 break;
951 case ISCSI_OPCODE_ASYNC_MSG:
952 qedi_process_async_mesg(qedi, cqe, task, q_conn,
953 que_idx);
954 break;
955 case ISCSI_OPCODE_REJECT:
956 qedi_process_reject_mesg(qedi, cqe, task, q_conn,
957 que_idx);
958 break;
959 }
960 goto exit_fp_process;
961 case ISCSI_CQE_TYPE_DUMMY:
962 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM, "Dummy CqE\n");
963 goto exit_fp_process;
964 case ISCSI_CQE_TYPE_TASK_CLEANUP:
965 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM, "CleanUp CqE\n");
966 qedi_process_cmd_cleanup_resp(qedi, &cqe->cqe_solicited, task,
967 conn);
968 goto exit_fp_process;
969 default:
970 QEDI_ERR(&qedi->dbg_ctx, "Error cqe.\n");
971 break;
972 }
973
974 exit_fp_process:
975 return;
976 }
977
978 static void qedi_add_to_sq(struct qedi_conn *qedi_conn, struct iscsi_task *task,
979 u16 tid, uint16_t ptu_invalidate, int is_cleanup)
980 {
981 struct iscsi_wqe *wqe;
982 struct iscsi_wqe_field *cont_field;
983 struct qedi_endpoint *ep;
984 struct scsi_cmnd *sc = task->sc;
985 struct iscsi_login_req *login_hdr;
986 struct qedi_cmd *cmd = task->dd_data;
987
988 login_hdr = (struct iscsi_login_req *)task->hdr;
989 ep = qedi_conn->ep;
990 wqe = &ep->sq[ep->sq_prod_idx];
991
992 memset(wqe, 0, sizeof(*wqe));
993
994 ep->sq_prod_idx++;
995 ep->fw_sq_prod_idx++;
996 if (ep->sq_prod_idx == QEDI_SQ_SIZE)
997 ep->sq_prod_idx = 0;
998
999 if (is_cleanup) {
1000 SET_FIELD(wqe->flags, ISCSI_WQE_WQE_TYPE,
1001 ISCSI_WQE_TYPE_TASK_CLEANUP);
1002 wqe->task_id = tid;
1003 return;
1004 }
1005
1006 if (ptu_invalidate) {
1007 SET_FIELD(wqe->flags, ISCSI_WQE_PTU_INVALIDATE,
1008 ISCSI_WQE_SET_PTU_INVALIDATE);
1009 }
1010
1011 cont_field = &wqe->cont_prevtid_union.cont_field;
1012
1013 switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
1014 case ISCSI_OP_LOGIN:
1015 case ISCSI_OP_TEXT:
1016 SET_FIELD(wqe->flags, ISCSI_WQE_WQE_TYPE,
1017 ISCSI_WQE_TYPE_MIDDLE_PATH);
1018 SET_FIELD(wqe->flags, ISCSI_WQE_NUM_FAST_SGES,
1019 1);
1020 cont_field->contlen_cdbsize_field = ntoh24(login_hdr->dlength);
1021 break;
1022 case ISCSI_OP_LOGOUT:
1023 case ISCSI_OP_NOOP_OUT:
1024 case ISCSI_OP_SCSI_TMFUNC:
1025 SET_FIELD(wqe->flags, ISCSI_WQE_WQE_TYPE,
1026 ISCSI_WQE_TYPE_NORMAL);
1027 break;
1028 default:
1029 if (!sc)
1030 break;
1031
1032 SET_FIELD(wqe->flags, ISCSI_WQE_WQE_TYPE,
1033 ISCSI_WQE_TYPE_NORMAL);
1034 cont_field->contlen_cdbsize_field =
1035 (sc->sc_data_direction == DMA_TO_DEVICE) ?
1036 scsi_bufflen(sc) : 0;
1037 if (cmd->use_slowpath)
1038 SET_FIELD(wqe->flags, ISCSI_WQE_NUM_FAST_SGES, 0);
1039 else
1040 SET_FIELD(wqe->flags, ISCSI_WQE_NUM_FAST_SGES,
1041 (sc->sc_data_direction ==
1042 DMA_TO_DEVICE) ?
1043 min((u16)QEDI_FAST_SGE_COUNT,
1044 (u16)cmd->io_tbl.sge_valid) : 0);
1045 break;
1046 }
1047
1048 wqe->task_id = tid;
1049 /* Make sure SQ data is coherent */
1050 wmb();
1051 }
1052
1053 static void qedi_ring_doorbell(struct qedi_conn *qedi_conn)
1054 {
1055 struct iscsi_db_data dbell = { 0 };
1056
1057 dbell.agg_flags = 0;
1058
1059 dbell.params |= DB_DEST_XCM << ISCSI_DB_DATA_DEST_SHIFT;
1060 dbell.params |= DB_AGG_CMD_SET << ISCSI_DB_DATA_AGG_CMD_SHIFT;
1061 dbell.params |=
1062 DQ_XCM_ISCSI_SQ_PROD_CMD << ISCSI_DB_DATA_AGG_VAL_SEL_SHIFT;
1063
1064 dbell.sq_prod = qedi_conn->ep->fw_sq_prod_idx;
1065 writel(*(u32 *)&dbell, qedi_conn->ep->p_doorbell);
1066
1067 /* Make sure fw write idx is coherent, and include both memory barriers
1068 * as a failsafe as for some architectures the call is the same but on
1069 * others they are two different assembly operations.
1070 */
1071 wmb();
1072 mmiowb();
1073 QEDI_INFO(&qedi_conn->qedi->dbg_ctx, QEDI_LOG_MP_REQ,
1074 "prod_idx=0x%x, fw_prod_idx=0x%x, cid=0x%x\n",
1075 qedi_conn->ep->sq_prod_idx, qedi_conn->ep->fw_sq_prod_idx,
1076 qedi_conn->iscsi_conn_id);
1077 }
1078
1079 int qedi_send_iscsi_login(struct qedi_conn *qedi_conn,
1080 struct iscsi_task *task)
1081 {
1082 struct qedi_ctx *qedi = qedi_conn->qedi;
1083 struct iscsi_task_context *fw_task_ctx;
1084 struct iscsi_login_req *login_hdr;
1085 struct iscsi_login_req_hdr *fw_login_req = NULL;
1086 struct iscsi_cached_sge_ctx *cached_sge = NULL;
1087 struct iscsi_sge *single_sge = NULL;
1088 struct iscsi_sge *req_sge = NULL;
1089 struct iscsi_sge *resp_sge = NULL;
1090 struct qedi_cmd *qedi_cmd;
1091 s16 ptu_invalidate = 0;
1092 s16 tid = 0;
1093
1094 req_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.req_bd_tbl;
1095 resp_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.resp_bd_tbl;
1096 qedi_cmd = (struct qedi_cmd *)task->dd_data;
1097 login_hdr = (struct iscsi_login_req *)task->hdr;
1098
1099 tid = qedi_get_task_idx(qedi);
1100 if (tid == -1)
1101 return -ENOMEM;
1102
1103 fw_task_ctx = qedi_get_task_mem(&qedi->tasks, tid);
1104 memset(fw_task_ctx, 0, sizeof(struct iscsi_task_context));
1105
1106 qedi_cmd->task_id = tid;
1107
1108 /* Ystorm context */
1109 fw_login_req = &fw_task_ctx->ystorm_st_context.pdu_hdr.login_req;
1110 fw_login_req->opcode = login_hdr->opcode;
1111 fw_login_req->version_min = login_hdr->min_version;
1112 fw_login_req->version_max = login_hdr->max_version;
1113 fw_login_req->flags_attr = login_hdr->flags;
1114 fw_login_req->isid_tabc = *((u16 *)login_hdr->isid + 2);
1115 fw_login_req->isid_d = *((u32 *)login_hdr->isid);
1116 fw_login_req->tsih = login_hdr->tsih;
1117 qedi_update_itt_map(qedi, tid, task->itt, qedi_cmd);
1118 fw_login_req->itt = qedi_set_itt(tid, get_itt(task->itt));
1119 fw_login_req->cid = qedi_conn->iscsi_conn_id;
1120 fw_login_req->cmd_sn = be32_to_cpu(login_hdr->cmdsn);
1121 fw_login_req->exp_stat_sn = be32_to_cpu(login_hdr->exp_statsn);
1122 fw_login_req->exp_stat_sn = 0;
1123
1124 if (qedi->tid_reuse_count[tid] == QEDI_MAX_TASK_NUM) {
1125 ptu_invalidate = 1;
1126 qedi->tid_reuse_count[tid] = 0;
1127 }
1128
1129 fw_task_ctx->ystorm_st_context.state.reuse_count =
1130 qedi->tid_reuse_count[tid];
1131 fw_task_ctx->mstorm_st_context.reuse_count =
1132 qedi->tid_reuse_count[tid]++;
1133 cached_sge =
1134 &fw_task_ctx->ystorm_st_context.state.sgl_ctx_union.cached_sge;
1135 cached_sge->sge.sge_len = req_sge->sge_len;
1136 cached_sge->sge.sge_addr.lo = (u32)(qedi_conn->gen_pdu.req_dma_addr);
1137 cached_sge->sge.sge_addr.hi =
1138 (u32)((u64)qedi_conn->gen_pdu.req_dma_addr >> 32);
1139
1140 /* Mstorm context */
1141 single_sge = &fw_task_ctx->mstorm_st_context.sgl_union.single_sge;
1142 fw_task_ctx->mstorm_st_context.task_type = 0x2;
1143 fw_task_ctx->mstorm_ag_context.task_cid = (u16)qedi_conn->iscsi_conn_id;
1144 single_sge->sge_addr.lo = resp_sge->sge_addr.lo;
1145 single_sge->sge_addr.hi = resp_sge->sge_addr.hi;
1146 single_sge->sge_len = resp_sge->sge_len;
1147
1148 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
1149 ISCSI_MFLAGS_SINGLE_SGE, 1);
1150 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
1151 ISCSI_MFLAGS_SLOW_IO, 0);
1152 fw_task_ctx->mstorm_st_context.sgl_size = 1;
1153 fw_task_ctx->mstorm_st_context.rem_task_size = resp_sge->sge_len;
1154
1155 /* Ustorm context */
1156 fw_task_ctx->ustorm_st_context.rem_rcv_len = resp_sge->sge_len;
1157 fw_task_ctx->ustorm_st_context.exp_data_transfer_len =
1158 ntoh24(login_hdr->dlength);
1159 fw_task_ctx->ustorm_st_context.exp_data_sn = 0;
1160 fw_task_ctx->ustorm_st_context.cq_rss_number = 0;
1161 fw_task_ctx->ustorm_st_context.task_type = 0x2;
1162 fw_task_ctx->ustorm_ag_context.icid = (u16)qedi_conn->iscsi_conn_id;
1163 fw_task_ctx->ustorm_ag_context.exp_data_acked =
1164 ntoh24(login_hdr->dlength);
1165 SET_FIELD(fw_task_ctx->ustorm_ag_context.flags1,
1166 USTORM_ISCSI_TASK_AG_CTX_R2T2RECV, 1);
1167 SET_FIELD(fw_task_ctx->ustorm_st_context.flags,
1168 USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP, 0);
1169
1170 spin_lock(&qedi_conn->list_lock);
1171 list_add_tail(&qedi_cmd->io_cmd, &qedi_conn->active_cmd_list);
1172 qedi_cmd->io_cmd_in_list = true;
1173 qedi_conn->active_cmd_count++;
1174 spin_unlock(&qedi_conn->list_lock);
1175
1176 qedi_add_to_sq(qedi_conn, task, tid, ptu_invalidate, false);
1177 qedi_ring_doorbell(qedi_conn);
1178 return 0;
1179 }
1180
1181 int qedi_send_iscsi_logout(struct qedi_conn *qedi_conn,
1182 struct iscsi_task *task)
1183 {
1184 struct qedi_ctx *qedi = qedi_conn->qedi;
1185 struct iscsi_logout_req_hdr *fw_logout_req = NULL;
1186 struct iscsi_task_context *fw_task_ctx = NULL;
1187 struct iscsi_logout *logout_hdr = NULL;
1188 struct qedi_cmd *qedi_cmd = NULL;
1189 s16 tid = 0;
1190 s16 ptu_invalidate = 0;
1191
1192 qedi_cmd = (struct qedi_cmd *)task->dd_data;
1193 logout_hdr = (struct iscsi_logout *)task->hdr;
1194
1195 tid = qedi_get_task_idx(qedi);
1196 if (tid == -1)
1197 return -ENOMEM;
1198
1199 fw_task_ctx = qedi_get_task_mem(&qedi->tasks, tid);
1200
1201 memset(fw_task_ctx, 0, sizeof(struct iscsi_task_context));
1202 qedi_cmd->task_id = tid;
1203
1204 /* Ystorm context */
1205 fw_logout_req = &fw_task_ctx->ystorm_st_context.pdu_hdr.logout_req;
1206 fw_logout_req->opcode = ISCSI_OPCODE_LOGOUT_REQUEST;
1207 fw_logout_req->reason_code = 0x80 | logout_hdr->flags;
1208 qedi_update_itt_map(qedi, tid, task->itt, qedi_cmd);
1209 fw_logout_req->itt = qedi_set_itt(tid, get_itt(task->itt));
1210 fw_logout_req->exp_stat_sn = be32_to_cpu(logout_hdr->exp_statsn);
1211 fw_logout_req->cmd_sn = be32_to_cpu(logout_hdr->cmdsn);
1212
1213 if (qedi->tid_reuse_count[tid] == QEDI_MAX_TASK_NUM) {
1214 ptu_invalidate = 1;
1215 qedi->tid_reuse_count[tid] = 0;
1216 }
1217 fw_task_ctx->ystorm_st_context.state.reuse_count =
1218 qedi->tid_reuse_count[tid];
1219 fw_task_ctx->mstorm_st_context.reuse_count =
1220 qedi->tid_reuse_count[tid]++;
1221 fw_logout_req->cid = qedi_conn->iscsi_conn_id;
1222 fw_task_ctx->ystorm_st_context.state.buffer_offset[0] = 0;
1223
1224 /* Mstorm context */
1225 fw_task_ctx->mstorm_st_context.task_type = ISCSI_TASK_TYPE_MIDPATH;
1226 fw_task_ctx->mstorm_ag_context.task_cid = (u16)qedi_conn->iscsi_conn_id;
1227
1228 /* Ustorm context */
1229 fw_task_ctx->ustorm_st_context.rem_rcv_len = 0;
1230 fw_task_ctx->ustorm_st_context.exp_data_transfer_len = 0;
1231 fw_task_ctx->ustorm_st_context.exp_data_sn = 0;
1232 fw_task_ctx->ustorm_st_context.task_type = ISCSI_TASK_TYPE_MIDPATH;
1233 fw_task_ctx->ustorm_st_context.cq_rss_number = 0;
1234
1235 SET_FIELD(fw_task_ctx->ustorm_st_context.flags,
1236 USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP, 0);
1237 SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map,
1238 ISCSI_REG1_NUM_FAST_SGES, 0);
1239
1240 fw_task_ctx->ustorm_ag_context.icid = (u16)qedi_conn->iscsi_conn_id;
1241 SET_FIELD(fw_task_ctx->ustorm_ag_context.flags1,
1242 USTORM_ISCSI_TASK_AG_CTX_R2T2RECV, 1);
1243
1244 spin_lock(&qedi_conn->list_lock);
1245 list_add_tail(&qedi_cmd->io_cmd, &qedi_conn->active_cmd_list);
1246 qedi_cmd->io_cmd_in_list = true;
1247 qedi_conn->active_cmd_count++;
1248 spin_unlock(&qedi_conn->list_lock);
1249
1250 qedi_add_to_sq(qedi_conn, task, tid, ptu_invalidate, false);
1251 qedi_ring_doorbell(qedi_conn);
1252
1253 return 0;
1254 }
1255
1256 int qedi_cleanup_all_io(struct qedi_ctx *qedi, struct qedi_conn *qedi_conn,
1257 struct iscsi_task *task, bool in_recovery)
1258 {
1259 int rval;
1260 struct iscsi_task *ctask;
1261 struct qedi_cmd *cmd, *cmd_tmp;
1262 struct iscsi_tm *tmf_hdr;
1263 unsigned int lun = 0;
1264 bool lun_reset = false;
1265 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
1266 struct iscsi_session *session = conn->session;
1267
1268 /* From recovery, task is NULL or from tmf resp valid task */
1269 if (task) {
1270 tmf_hdr = (struct iscsi_tm *)task->hdr;
1271
1272 if ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) ==
1273 ISCSI_TM_FUNC_LOGICAL_UNIT_RESET) {
1274 lun_reset = true;
1275 lun = scsilun_to_int(&tmf_hdr->lun);
1276 }
1277 }
1278
1279 qedi_conn->cmd_cleanup_req = 0;
1280 qedi_conn->cmd_cleanup_cmpl = 0;
1281
1282 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
1283 "active_cmd_count=%d, cid=0x%x, in_recovery=%d, lun_reset=%d\n",
1284 qedi_conn->active_cmd_count, qedi_conn->iscsi_conn_id,
1285 in_recovery, lun_reset);
1286
1287 if (lun_reset)
1288 spin_lock_bh(&session->back_lock);
1289
1290 spin_lock(&qedi_conn->list_lock);
1291
1292 list_for_each_entry_safe(cmd, cmd_tmp, &qedi_conn->active_cmd_list,
1293 io_cmd) {
1294 ctask = cmd->task;
1295 if (ctask == task)
1296 continue;
1297
1298 if (lun_reset) {
1299 if (cmd->scsi_cmd && cmd->scsi_cmd->device) {
1300 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
1301 "tid=0x%x itt=0x%x scsi_cmd_ptr=%p device=%p task_state=%d cmd_state=0%x cid=0x%x\n",
1302 cmd->task_id, get_itt(ctask->itt),
1303 cmd->scsi_cmd, cmd->scsi_cmd->device,
1304 ctask->state, cmd->state,
1305 qedi_conn->iscsi_conn_id);
1306 if (cmd->scsi_cmd->device->lun != lun)
1307 continue;
1308 }
1309 }
1310 qedi_conn->cmd_cleanup_req++;
1311 qedi_iscsi_cleanup_task(ctask, true);
1312
1313 list_del_init(&cmd->io_cmd);
1314 qedi_conn->active_cmd_count--;
1315 QEDI_WARN(&qedi->dbg_ctx,
1316 "Deleted active cmd list node io_cmd=%p, cid=0x%x\n",
1317 &cmd->io_cmd, qedi_conn->iscsi_conn_id);
1318 }
1319
1320 spin_unlock(&qedi_conn->list_lock);
1321
1322 if (lun_reset)
1323 spin_unlock_bh(&session->back_lock);
1324
1325 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
1326 "cmd_cleanup_req=%d, cid=0x%x\n",
1327 qedi_conn->cmd_cleanup_req,
1328 qedi_conn->iscsi_conn_id);
1329
1330 rval = wait_event_interruptible_timeout(qedi_conn->wait_queue,
1331 ((qedi_conn->cmd_cleanup_req ==
1332 qedi_conn->cmd_cleanup_cmpl) ||
1333 qedi_conn->ep),
1334 5 * HZ);
1335 if (rval) {
1336 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
1337 "i/o cmd_cleanup_req=%d, equal to cmd_cleanup_cmpl=%d, cid=0x%x\n",
1338 qedi_conn->cmd_cleanup_req,
1339 qedi_conn->cmd_cleanup_cmpl,
1340 qedi_conn->iscsi_conn_id);
1341
1342 return 0;
1343 }
1344
1345 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
1346 "i/o cmd_cleanup_req=%d, not equal to cmd_cleanup_cmpl=%d, cid=0x%x\n",
1347 qedi_conn->cmd_cleanup_req,
1348 qedi_conn->cmd_cleanup_cmpl,
1349 qedi_conn->iscsi_conn_id);
1350
1351 iscsi_host_for_each_session(qedi->shost,
1352 qedi_mark_device_missing);
1353 qedi_ops->common->drain(qedi->cdev);
1354
1355 /* Enable IOs for all other sessions except current.*/
1356 if (!wait_event_interruptible_timeout(qedi_conn->wait_queue,
1357 (qedi_conn->cmd_cleanup_req ==
1358 qedi_conn->cmd_cleanup_cmpl),
1359 5 * HZ)) {
1360 iscsi_host_for_each_session(qedi->shost,
1361 qedi_mark_device_available);
1362 return -1;
1363 }
1364
1365 iscsi_host_for_each_session(qedi->shost,
1366 qedi_mark_device_available);
1367
1368 return 0;
1369 }
1370
1371 void qedi_clearsq(struct qedi_ctx *qedi, struct qedi_conn *qedi_conn,
1372 struct iscsi_task *task)
1373 {
1374 struct qedi_endpoint *qedi_ep;
1375 int rval;
1376
1377 qedi_ep = qedi_conn->ep;
1378 qedi_conn->cmd_cleanup_req = 0;
1379 qedi_conn->cmd_cleanup_cmpl = 0;
1380
1381 if (!qedi_ep) {
1382 QEDI_WARN(&qedi->dbg_ctx,
1383 "Cannot proceed, ep already disconnected, cid=0x%x\n",
1384 qedi_conn->iscsi_conn_id);
1385 return;
1386 }
1387
1388 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1389 "Clearing SQ for cid=0x%x, conn=%p, ep=%p\n",
1390 qedi_conn->iscsi_conn_id, qedi_conn, qedi_ep);
1391
1392 qedi_ops->clear_sq(qedi->cdev, qedi_ep->handle);
1393
1394 rval = qedi_cleanup_all_io(qedi, qedi_conn, task, true);
1395 if (rval) {
1396 QEDI_ERR(&qedi->dbg_ctx,
1397 "fatal error, need hard reset, cid=0x%x\n",
1398 qedi_conn->iscsi_conn_id);
1399 WARN_ON(1);
1400 }
1401 }
1402
1403 static int qedi_wait_for_cleanup_request(struct qedi_ctx *qedi,
1404 struct qedi_conn *qedi_conn,
1405 struct iscsi_task *task,
1406 struct qedi_cmd *qedi_cmd,
1407 struct qedi_work_map *list_work)
1408 {
1409 struct qedi_cmd *cmd = (struct qedi_cmd *)task->dd_data;
1410 int wait;
1411
1412 wait = wait_event_interruptible_timeout(qedi_conn->wait_queue,
1413 ((qedi_cmd->state ==
1414 CLEANUP_RECV) ||
1415 ((qedi_cmd->type == TYPEIO) &&
1416 (cmd->state ==
1417 RESPONSE_RECEIVED))),
1418 5 * HZ);
1419 if (!wait) {
1420 qedi_cmd->state = CLEANUP_WAIT_FAILED;
1421
1422 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
1423 "Cleanup timedout tid=0x%x, issue connection recovery, cid=0x%x\n",
1424 cmd->task_id, qedi_conn->iscsi_conn_id);
1425
1426 return -1;
1427 }
1428 return 0;
1429 }
1430
1431 static void qedi_tmf_work(struct work_struct *work)
1432 {
1433 struct qedi_cmd *qedi_cmd =
1434 container_of(work, struct qedi_cmd, tmf_work);
1435 struct qedi_conn *qedi_conn = qedi_cmd->conn;
1436 struct qedi_ctx *qedi = qedi_conn->qedi;
1437 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
1438 struct iscsi_cls_session *cls_sess;
1439 struct qedi_work_map *list_work = NULL;
1440 struct iscsi_task *mtask;
1441 struct qedi_cmd *cmd;
1442 struct iscsi_task *ctask;
1443 struct iscsi_tm *tmf_hdr;
1444 s16 rval = 0;
1445 s16 tid = 0;
1446
1447 mtask = qedi_cmd->task;
1448 tmf_hdr = (struct iscsi_tm *)mtask->hdr;
1449 cls_sess = iscsi_conn_to_session(qedi_conn->cls_conn);
1450 set_bit(QEDI_CONN_FW_CLEANUP, &qedi_conn->flags);
1451
1452 ctask = iscsi_itt_to_task(conn, tmf_hdr->rtt);
1453 if (!ctask || !ctask->sc) {
1454 QEDI_ERR(&qedi->dbg_ctx, "Task already completed\n");
1455 goto abort_ret;
1456 }
1457
1458 cmd = (struct qedi_cmd *)ctask->dd_data;
1459 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1460 "Abort tmf rtt=0x%x, cmd itt=0x%x, cmd tid=0x%x, cid=0x%x\n",
1461 get_itt(tmf_hdr->rtt), get_itt(ctask->itt), cmd->task_id,
1462 qedi_conn->iscsi_conn_id);
1463
1464 if (do_not_recover) {
1465 QEDI_ERR(&qedi->dbg_ctx, "DONT SEND CLEANUP/ABORT %d\n",
1466 do_not_recover);
1467 goto abort_ret;
1468 }
1469
1470 list_work = kzalloc(sizeof(*list_work), GFP_ATOMIC);
1471 if (!list_work) {
1472 QEDI_ERR(&qedi->dbg_ctx, "Memory alloction failed\n");
1473 goto abort_ret;
1474 }
1475
1476 qedi_cmd->type = TYPEIO;
1477 list_work->qedi_cmd = qedi_cmd;
1478 list_work->rtid = cmd->task_id;
1479 list_work->state = QEDI_WORK_SCHEDULED;
1480 qedi_cmd->list_tmf_work = list_work;
1481
1482 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
1483 "Queue tmf work=%p, list node=%p, cid=0x%x, tmf flags=0x%x\n",
1484 list_work->ptr_tmf_work, list_work, qedi_conn->iscsi_conn_id,
1485 tmf_hdr->flags);
1486
1487 spin_lock_bh(&qedi_conn->tmf_work_lock);
1488 list_add_tail(&list_work->list, &qedi_conn->tmf_work_list);
1489 spin_unlock_bh(&qedi_conn->tmf_work_lock);
1490
1491 qedi_iscsi_cleanup_task(ctask, false);
1492
1493 rval = qedi_wait_for_cleanup_request(qedi, qedi_conn, ctask, qedi_cmd,
1494 list_work);
1495 if (rval == -1) {
1496 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1497 "FW cleanup got escalated, cid=0x%x\n",
1498 qedi_conn->iscsi_conn_id);
1499 goto ldel_exit;
1500 }
1501
1502 tid = qedi_get_task_idx(qedi);
1503 if (tid == -1) {
1504 QEDI_ERR(&qedi->dbg_ctx, "Invalid tid, cid=0x%x\n",
1505 qedi_conn->iscsi_conn_id);
1506 goto ldel_exit;
1507 }
1508
1509 qedi_cmd->task_id = tid;
1510 qedi_send_iscsi_tmf(qedi_conn, qedi_cmd->task);
1511
1512 abort_ret:
1513 clear_bit(QEDI_CONN_FW_CLEANUP, &qedi_conn->flags);
1514 return;
1515
1516 ldel_exit:
1517 spin_lock_bh(&qedi_conn->tmf_work_lock);
1518 if (!qedi_cmd->list_tmf_work) {
1519 list_del_init(&list_work->list);
1520 qedi_cmd->list_tmf_work = NULL;
1521 kfree(list_work);
1522 }
1523 spin_unlock_bh(&qedi_conn->tmf_work_lock);
1524
1525 spin_lock(&qedi_conn->list_lock);
1526 list_del_init(&cmd->io_cmd);
1527 qedi_conn->active_cmd_count--;
1528 spin_unlock(&qedi_conn->list_lock);
1529
1530 clear_bit(QEDI_CONN_FW_CLEANUP, &qedi_conn->flags);
1531 }
1532
1533 static int qedi_send_iscsi_tmf(struct qedi_conn *qedi_conn,
1534 struct iscsi_task *mtask)
1535 {
1536 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
1537 struct qedi_ctx *qedi = qedi_conn->qedi;
1538 struct iscsi_task_context *fw_task_ctx;
1539 struct iscsi_tmf_request_hdr *fw_tmf_request;
1540 struct iscsi_sge *single_sge;
1541 struct qedi_cmd *qedi_cmd;
1542 struct qedi_cmd *cmd;
1543 struct iscsi_task *ctask;
1544 struct iscsi_tm *tmf_hdr;
1545 struct iscsi_sge *req_sge;
1546 struct iscsi_sge *resp_sge;
1547 u32 lun[2];
1548 s16 tid = 0, ptu_invalidate = 0;
1549
1550 req_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.req_bd_tbl;
1551 resp_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.resp_bd_tbl;
1552 qedi_cmd = (struct qedi_cmd *)mtask->dd_data;
1553 tmf_hdr = (struct iscsi_tm *)mtask->hdr;
1554
1555 tid = qedi_cmd->task_id;
1556 qedi_update_itt_map(qedi, tid, mtask->itt, qedi_cmd);
1557
1558 fw_task_ctx = qedi_get_task_mem(&qedi->tasks, tid);
1559 memset(fw_task_ctx, 0, sizeof(struct iscsi_task_context));
1560
1561 fw_tmf_request = &fw_task_ctx->ystorm_st_context.pdu_hdr.tmf_request;
1562 fw_tmf_request->itt = qedi_set_itt(tid, get_itt(mtask->itt));
1563 fw_tmf_request->cmd_sn = be32_to_cpu(tmf_hdr->cmdsn);
1564
1565 memcpy(lun, &tmf_hdr->lun, sizeof(struct scsi_lun));
1566 fw_tmf_request->lun.lo = be32_to_cpu(lun[0]);
1567 fw_tmf_request->lun.hi = be32_to_cpu(lun[1]);
1568
1569 if (qedi->tid_reuse_count[tid] == QEDI_MAX_TASK_NUM) {
1570 ptu_invalidate = 1;
1571 qedi->tid_reuse_count[tid] = 0;
1572 }
1573 fw_task_ctx->ystorm_st_context.state.reuse_count =
1574 qedi->tid_reuse_count[tid];
1575 fw_task_ctx->mstorm_st_context.reuse_count =
1576 qedi->tid_reuse_count[tid]++;
1577
1578 if ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) ==
1579 ISCSI_TM_FUNC_ABORT_TASK) {
1580 ctask = iscsi_itt_to_task(conn, tmf_hdr->rtt);
1581 if (!ctask || !ctask->sc) {
1582 QEDI_ERR(&qedi->dbg_ctx,
1583 "Could not get reference task\n");
1584 return 0;
1585 }
1586 cmd = (struct qedi_cmd *)ctask->dd_data;
1587 fw_tmf_request->rtt =
1588 qedi_set_itt(cmd->task_id,
1589 get_itt(tmf_hdr->rtt));
1590 } else {
1591 fw_tmf_request->rtt = ISCSI_RESERVED_TAG;
1592 }
1593
1594 fw_tmf_request->opcode = tmf_hdr->opcode;
1595 fw_tmf_request->function = tmf_hdr->flags;
1596 fw_tmf_request->hdr_second_dword = ntoh24(tmf_hdr->dlength);
1597 fw_tmf_request->ref_cmd_sn = be32_to_cpu(tmf_hdr->refcmdsn);
1598
1599 single_sge = &fw_task_ctx->mstorm_st_context.sgl_union.single_sge;
1600 fw_task_ctx->mstorm_st_context.task_type = ISCSI_TASK_TYPE_MIDPATH;
1601 fw_task_ctx->mstorm_ag_context.task_cid = (u16)qedi_conn->iscsi_conn_id;
1602 single_sge->sge_addr.lo = resp_sge->sge_addr.lo;
1603 single_sge->sge_addr.hi = resp_sge->sge_addr.hi;
1604 single_sge->sge_len = resp_sge->sge_len;
1605
1606 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
1607 ISCSI_MFLAGS_SINGLE_SGE, 1);
1608 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
1609 ISCSI_MFLAGS_SLOW_IO, 0);
1610 fw_task_ctx->mstorm_st_context.sgl_size = 1;
1611 fw_task_ctx->mstorm_st_context.rem_task_size = resp_sge->sge_len;
1612
1613 /* Ustorm context */
1614 fw_task_ctx->ustorm_st_context.rem_rcv_len = 0;
1615 fw_task_ctx->ustorm_st_context.exp_data_transfer_len = 0;
1616 fw_task_ctx->ustorm_st_context.exp_data_sn = 0;
1617 fw_task_ctx->ustorm_st_context.task_type = ISCSI_TASK_TYPE_MIDPATH;
1618 fw_task_ctx->ustorm_st_context.cq_rss_number = 0;
1619
1620 SET_FIELD(fw_task_ctx->ustorm_st_context.flags,
1621 USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP, 0);
1622 SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map,
1623 ISCSI_REG1_NUM_FAST_SGES, 0);
1624
1625 fw_task_ctx->ustorm_ag_context.icid = (u16)qedi_conn->iscsi_conn_id;
1626 SET_FIELD(fw_task_ctx->ustorm_ag_context.flags1,
1627 USTORM_ISCSI_TASK_AG_CTX_R2T2RECV, 1);
1628 fw_task_ctx->ustorm_st_context.lun.lo = be32_to_cpu(lun[0]);
1629 fw_task_ctx->ustorm_st_context.lun.hi = be32_to_cpu(lun[1]);
1630
1631 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
1632 "Add TMF to SQ, tmf tid=0x%x, itt=0x%x, cid=0x%x\n",
1633 tid, mtask->itt, qedi_conn->iscsi_conn_id);
1634
1635 spin_lock(&qedi_conn->list_lock);
1636 list_add_tail(&qedi_cmd->io_cmd, &qedi_conn->active_cmd_list);
1637 qedi_cmd->io_cmd_in_list = true;
1638 qedi_conn->active_cmd_count++;
1639 spin_unlock(&qedi_conn->list_lock);
1640
1641 qedi_add_to_sq(qedi_conn, mtask, tid, ptu_invalidate, false);
1642 qedi_ring_doorbell(qedi_conn);
1643 return 0;
1644 }
1645
1646 int qedi_iscsi_abort_work(struct qedi_conn *qedi_conn,
1647 struct iscsi_task *mtask)
1648 {
1649 struct qedi_ctx *qedi = qedi_conn->qedi;
1650 struct iscsi_tm *tmf_hdr;
1651 struct qedi_cmd *qedi_cmd = (struct qedi_cmd *)mtask->dd_data;
1652 s16 tid = 0;
1653
1654 tmf_hdr = (struct iscsi_tm *)mtask->hdr;
1655 qedi_cmd->task = mtask;
1656
1657 /* If abort task then schedule the work and return */
1658 if ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) ==
1659 ISCSI_TM_FUNC_ABORT_TASK) {
1660 qedi_cmd->state = CLEANUP_WAIT;
1661 INIT_WORK(&qedi_cmd->tmf_work, qedi_tmf_work);
1662 queue_work(qedi->tmf_thread, &qedi_cmd->tmf_work);
1663
1664 } else if (((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) ==
1665 ISCSI_TM_FUNC_LOGICAL_UNIT_RESET) ||
1666 ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) ==
1667 ISCSI_TM_FUNC_TARGET_WARM_RESET) ||
1668 ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) ==
1669 ISCSI_TM_FUNC_TARGET_COLD_RESET)) {
1670 tid = qedi_get_task_idx(qedi);
1671 if (tid == -1) {
1672 QEDI_ERR(&qedi->dbg_ctx, "Invalid tid, cid=0x%x\n",
1673 qedi_conn->iscsi_conn_id);
1674 return -1;
1675 }
1676 qedi_cmd->task_id = tid;
1677
1678 qedi_send_iscsi_tmf(qedi_conn, qedi_cmd->task);
1679
1680 } else {
1681 QEDI_ERR(&qedi->dbg_ctx, "Invalid tmf, cid=0x%x\n",
1682 qedi_conn->iscsi_conn_id);
1683 return -1;
1684 }
1685
1686 return 0;
1687 }
1688
1689 int qedi_send_iscsi_text(struct qedi_conn *qedi_conn,
1690 struct iscsi_task *task)
1691 {
1692 struct qedi_ctx *qedi = qedi_conn->qedi;
1693 struct iscsi_task_context *fw_task_ctx;
1694 struct iscsi_text_request_hdr *fw_text_request;
1695 struct iscsi_cached_sge_ctx *cached_sge;
1696 struct iscsi_sge *single_sge;
1697 struct qedi_cmd *qedi_cmd;
1698 /* For 6.5 hdr iscsi_hdr */
1699 struct iscsi_text *text_hdr;
1700 struct iscsi_sge *req_sge;
1701 struct iscsi_sge *resp_sge;
1702 s16 ptu_invalidate = 0;
1703 s16 tid = 0;
1704
1705 req_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.req_bd_tbl;
1706 resp_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.resp_bd_tbl;
1707 qedi_cmd = (struct qedi_cmd *)task->dd_data;
1708 text_hdr = (struct iscsi_text *)task->hdr;
1709
1710 tid = qedi_get_task_idx(qedi);
1711 if (tid == -1)
1712 return -ENOMEM;
1713
1714 fw_task_ctx = qedi_get_task_mem(&qedi->tasks, tid);
1715 memset(fw_task_ctx, 0, sizeof(struct iscsi_task_context));
1716
1717 qedi_cmd->task_id = tid;
1718
1719 /* Ystorm context */
1720 fw_text_request =
1721 &fw_task_ctx->ystorm_st_context.pdu_hdr.text_request;
1722 fw_text_request->opcode = text_hdr->opcode;
1723 fw_text_request->flags_attr = text_hdr->flags;
1724
1725 qedi_update_itt_map(qedi, tid, task->itt, qedi_cmd);
1726 fw_text_request->itt = qedi_set_itt(tid, get_itt(task->itt));
1727 fw_text_request->ttt = text_hdr->ttt;
1728 fw_text_request->cmd_sn = be32_to_cpu(text_hdr->cmdsn);
1729 fw_text_request->exp_stat_sn = be32_to_cpu(text_hdr->exp_statsn);
1730 fw_text_request->hdr_second_dword = ntoh24(text_hdr->dlength);
1731
1732 if (qedi->tid_reuse_count[tid] == QEDI_MAX_TASK_NUM) {
1733 ptu_invalidate = 1;
1734 qedi->tid_reuse_count[tid] = 0;
1735 }
1736 fw_task_ctx->ystorm_st_context.state.reuse_count =
1737 qedi->tid_reuse_count[tid];
1738 fw_task_ctx->mstorm_st_context.reuse_count =
1739 qedi->tid_reuse_count[tid]++;
1740
1741 cached_sge =
1742 &fw_task_ctx->ystorm_st_context.state.sgl_ctx_union.cached_sge;
1743 cached_sge->sge.sge_len = req_sge->sge_len;
1744 cached_sge->sge.sge_addr.lo = (u32)(qedi_conn->gen_pdu.req_dma_addr);
1745 cached_sge->sge.sge_addr.hi =
1746 (u32)((u64)qedi_conn->gen_pdu.req_dma_addr >> 32);
1747
1748 /* Mstorm context */
1749 single_sge = &fw_task_ctx->mstorm_st_context.sgl_union.single_sge;
1750 fw_task_ctx->mstorm_st_context.task_type = 0x2;
1751 fw_task_ctx->mstorm_ag_context.task_cid = (u16)qedi_conn->iscsi_conn_id;
1752 single_sge->sge_addr.lo = resp_sge->sge_addr.lo;
1753 single_sge->sge_addr.hi = resp_sge->sge_addr.hi;
1754 single_sge->sge_len = resp_sge->sge_len;
1755
1756 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
1757 ISCSI_MFLAGS_SINGLE_SGE, 1);
1758 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
1759 ISCSI_MFLAGS_SLOW_IO, 0);
1760 fw_task_ctx->mstorm_st_context.sgl_size = 1;
1761 fw_task_ctx->mstorm_st_context.rem_task_size = resp_sge->sge_len;
1762
1763 /* Ustorm context */
1764 fw_task_ctx->ustorm_ag_context.exp_data_acked =
1765 ntoh24(text_hdr->dlength);
1766 fw_task_ctx->ustorm_st_context.rem_rcv_len = resp_sge->sge_len;
1767 fw_task_ctx->ustorm_st_context.exp_data_transfer_len =
1768 ntoh24(text_hdr->dlength);
1769 fw_task_ctx->ustorm_st_context.exp_data_sn =
1770 be32_to_cpu(text_hdr->exp_statsn);
1771 fw_task_ctx->ustorm_st_context.cq_rss_number = 0;
1772 fw_task_ctx->ustorm_st_context.task_type = 0x2;
1773 fw_task_ctx->ustorm_ag_context.icid = (u16)qedi_conn->iscsi_conn_id;
1774 SET_FIELD(fw_task_ctx->ustorm_ag_context.flags1,
1775 USTORM_ISCSI_TASK_AG_CTX_R2T2RECV, 1);
1776
1777 /* Add command in active command list */
1778 spin_lock(&qedi_conn->list_lock);
1779 list_add_tail(&qedi_cmd->io_cmd, &qedi_conn->active_cmd_list);
1780 qedi_cmd->io_cmd_in_list = true;
1781 qedi_conn->active_cmd_count++;
1782 spin_unlock(&qedi_conn->list_lock);
1783
1784 qedi_add_to_sq(qedi_conn, task, tid, ptu_invalidate, false);
1785 qedi_ring_doorbell(qedi_conn);
1786
1787 return 0;
1788 }
1789
1790 int qedi_send_iscsi_nopout(struct qedi_conn *qedi_conn,
1791 struct iscsi_task *task,
1792 char *datap, int data_len, int unsol)
1793 {
1794 struct qedi_ctx *qedi = qedi_conn->qedi;
1795 struct iscsi_task_context *fw_task_ctx;
1796 struct iscsi_nop_out_hdr *fw_nop_out;
1797 struct qedi_cmd *qedi_cmd;
1798 /* For 6.5 hdr iscsi_hdr */
1799 struct iscsi_nopout *nopout_hdr;
1800 struct iscsi_cached_sge_ctx *cached_sge;
1801 struct iscsi_sge *single_sge;
1802 struct iscsi_sge *req_sge;
1803 struct iscsi_sge *resp_sge;
1804 u32 lun[2];
1805 s16 ptu_invalidate = 0;
1806 s16 tid = 0;
1807
1808 req_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.req_bd_tbl;
1809 resp_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.resp_bd_tbl;
1810 qedi_cmd = (struct qedi_cmd *)task->dd_data;
1811 nopout_hdr = (struct iscsi_nopout *)task->hdr;
1812
1813 tid = qedi_get_task_idx(qedi);
1814 if (tid == -1) {
1815 QEDI_WARN(&qedi->dbg_ctx, "Invalid tid\n");
1816 return -ENOMEM;
1817 }
1818
1819 fw_task_ctx = qedi_get_task_mem(&qedi->tasks, tid);
1820
1821 memset(fw_task_ctx, 0, sizeof(struct iscsi_task_context));
1822 qedi_cmd->task_id = tid;
1823
1824 /* Ystorm context */
1825 fw_nop_out = &fw_task_ctx->ystorm_st_context.pdu_hdr.nop_out;
1826 SET_FIELD(fw_nop_out->flags_attr, ISCSI_NOP_OUT_HDR_CONST1, 1);
1827 SET_FIELD(fw_nop_out->flags_attr, ISCSI_NOP_OUT_HDR_RSRV, 0);
1828
1829 memcpy(lun, &nopout_hdr->lun, sizeof(struct scsi_lun));
1830 fw_nop_out->lun.lo = be32_to_cpu(lun[0]);
1831 fw_nop_out->lun.hi = be32_to_cpu(lun[1]);
1832
1833 qedi_update_itt_map(qedi, tid, task->itt, qedi_cmd);
1834
1835 if (nopout_hdr->ttt != ISCSI_TTT_ALL_ONES) {
1836 fw_nop_out->itt = be32_to_cpu(nopout_hdr->itt);
1837 fw_nop_out->ttt = be32_to_cpu(nopout_hdr->ttt);
1838 fw_task_ctx->ystorm_st_context.state.buffer_offset[0] = 0;
1839 fw_task_ctx->ystorm_st_context.state.local_comp = 1;
1840 SET_FIELD(fw_task_ctx->ustorm_st_context.flags,
1841 USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP, 1);
1842 } else {
1843 fw_nop_out->itt = qedi_set_itt(tid, get_itt(task->itt));
1844 fw_nop_out->ttt = ISCSI_TTT_ALL_ONES;
1845 fw_task_ctx->ystorm_st_context.state.buffer_offset[0] = 0;
1846
1847 spin_lock(&qedi_conn->list_lock);
1848 list_add_tail(&qedi_cmd->io_cmd, &qedi_conn->active_cmd_list);
1849 qedi_cmd->io_cmd_in_list = true;
1850 qedi_conn->active_cmd_count++;
1851 spin_unlock(&qedi_conn->list_lock);
1852 }
1853
1854 fw_nop_out->opcode = ISCSI_OPCODE_NOP_OUT;
1855 fw_nop_out->cmd_sn = be32_to_cpu(nopout_hdr->cmdsn);
1856 fw_nop_out->exp_stat_sn = be32_to_cpu(nopout_hdr->exp_statsn);
1857
1858 cached_sge =
1859 &fw_task_ctx->ystorm_st_context.state.sgl_ctx_union.cached_sge;
1860 cached_sge->sge.sge_len = req_sge->sge_len;
1861 cached_sge->sge.sge_addr.lo = (u32)(qedi_conn->gen_pdu.req_dma_addr);
1862 cached_sge->sge.sge_addr.hi =
1863 (u32)((u64)qedi_conn->gen_pdu.req_dma_addr >> 32);
1864
1865 /* Mstorm context */
1866 fw_task_ctx->mstorm_st_context.task_type = ISCSI_TASK_TYPE_MIDPATH;
1867 fw_task_ctx->mstorm_ag_context.task_cid = (u16)qedi_conn->iscsi_conn_id;
1868
1869 single_sge = &fw_task_ctx->mstorm_st_context.sgl_union.single_sge;
1870 single_sge->sge_addr.lo = resp_sge->sge_addr.lo;
1871 single_sge->sge_addr.hi = resp_sge->sge_addr.hi;
1872 single_sge->sge_len = resp_sge->sge_len;
1873 fw_task_ctx->mstorm_st_context.rem_task_size = resp_sge->sge_len;
1874
1875 if (qedi->tid_reuse_count[tid] == QEDI_MAX_TASK_NUM) {
1876 ptu_invalidate = 1;
1877 qedi->tid_reuse_count[tid] = 0;
1878 }
1879 fw_task_ctx->ystorm_st_context.state.reuse_count =
1880 qedi->tid_reuse_count[tid];
1881 fw_task_ctx->mstorm_st_context.reuse_count =
1882 qedi->tid_reuse_count[tid]++;
1883 /* Ustorm context */
1884 fw_task_ctx->ustorm_st_context.rem_rcv_len = resp_sge->sge_len;
1885 fw_task_ctx->ustorm_st_context.exp_data_transfer_len = data_len;
1886 fw_task_ctx->ustorm_st_context.exp_data_sn = 0;
1887 fw_task_ctx->ustorm_st_context.task_type = ISCSI_TASK_TYPE_MIDPATH;
1888 fw_task_ctx->ustorm_st_context.cq_rss_number = 0;
1889
1890 SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map,
1891 ISCSI_REG1_NUM_FAST_SGES, 0);
1892
1893 fw_task_ctx->ustorm_ag_context.icid = (u16)qedi_conn->iscsi_conn_id;
1894 SET_FIELD(fw_task_ctx->ustorm_ag_context.flags1,
1895 USTORM_ISCSI_TASK_AG_CTX_R2T2RECV, 1);
1896
1897 fw_task_ctx->ustorm_st_context.lun.lo = be32_to_cpu(lun[0]);
1898 fw_task_ctx->ustorm_st_context.lun.hi = be32_to_cpu(lun[1]);
1899
1900 qedi_add_to_sq(qedi_conn, task, tid, ptu_invalidate, false);
1901 qedi_ring_doorbell(qedi_conn);
1902 return 0;
1903 }
1904
1905 static int qedi_split_bd(struct qedi_cmd *cmd, u64 addr, int sg_len,
1906 int bd_index)
1907 {
1908 struct iscsi_sge *bd = cmd->io_tbl.sge_tbl;
1909 int frag_size, sg_frags;
1910
1911 sg_frags = 0;
1912
1913 while (sg_len) {
1914 if (addr % QEDI_PAGE_SIZE)
1915 frag_size =
1916 (QEDI_PAGE_SIZE - (addr % QEDI_PAGE_SIZE));
1917 else
1918 frag_size = (sg_len > QEDI_BD_SPLIT_SZ) ? 0 :
1919 (sg_len % QEDI_BD_SPLIT_SZ);
1920
1921 if (frag_size == 0)
1922 frag_size = QEDI_BD_SPLIT_SZ;
1923
1924 bd[bd_index + sg_frags].sge_addr.lo = (addr & 0xffffffff);
1925 bd[bd_index + sg_frags].sge_addr.hi = (addr >> 32);
1926 bd[bd_index + sg_frags].sge_len = (u16)frag_size;
1927 QEDI_INFO(&cmd->conn->qedi->dbg_ctx, QEDI_LOG_IO,
1928 "split sge %d: addr=%llx, len=%x",
1929 (bd_index + sg_frags), addr, frag_size);
1930
1931 addr += (u64)frag_size;
1932 sg_frags++;
1933 sg_len -= frag_size;
1934 }
1935 return sg_frags;
1936 }
1937
1938 static int qedi_map_scsi_sg(struct qedi_ctx *qedi, struct qedi_cmd *cmd)
1939 {
1940 struct scsi_cmnd *sc = cmd->scsi_cmd;
1941 struct iscsi_sge *bd = cmd->io_tbl.sge_tbl;
1942 struct scatterlist *sg;
1943 int byte_count = 0;
1944 int bd_count = 0;
1945 int sg_count;
1946 int sg_len;
1947 int sg_frags;
1948 u64 addr, end_addr;
1949 int i;
1950
1951 WARN_ON(scsi_sg_count(sc) > QEDI_ISCSI_MAX_BDS_PER_CMD);
1952
1953 sg_count = dma_map_sg(&qedi->pdev->dev, scsi_sglist(sc),
1954 scsi_sg_count(sc), sc->sc_data_direction);
1955
1956 /*
1957 * New condition to send single SGE as cached-SGL.
1958 * Single SGE with length less than 64K.
1959 */
1960 sg = scsi_sglist(sc);
1961 if ((sg_count == 1) && (sg_dma_len(sg) <= MAX_SGLEN_FOR_CACHESGL)) {
1962 sg_len = sg_dma_len(sg);
1963 addr = (u64)sg_dma_address(sg);
1964
1965 bd[bd_count].sge_addr.lo = (addr & 0xffffffff);
1966 bd[bd_count].sge_addr.hi = (addr >> 32);
1967 bd[bd_count].sge_len = (u16)sg_len;
1968
1969 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO,
1970 "single-cashed-sgl: bd_count:%d addr=%llx, len=%x",
1971 sg_count, addr, sg_len);
1972
1973 return ++bd_count;
1974 }
1975
1976 scsi_for_each_sg(sc, sg, sg_count, i) {
1977 sg_len = sg_dma_len(sg);
1978 addr = (u64)sg_dma_address(sg);
1979 end_addr = (addr + sg_len);
1980
1981 /*
1982 * first sg elem in the 'list',
1983 * check if end addr is page-aligned.
1984 */
1985 if ((i == 0) && (sg_count > 1) && (end_addr % QEDI_PAGE_SIZE))
1986 cmd->use_slowpath = true;
1987
1988 /*
1989 * last sg elem in the 'list',
1990 * check if start addr is page-aligned.
1991 */
1992 else if ((i == (sg_count - 1)) &&
1993 (sg_count > 1) && (addr % QEDI_PAGE_SIZE))
1994 cmd->use_slowpath = true;
1995
1996 /*
1997 * middle sg elements in list,
1998 * check if start and end addr is page-aligned
1999 */
2000 else if ((i != 0) && (i != (sg_count - 1)) &&
2001 ((addr % QEDI_PAGE_SIZE) ||
2002 (end_addr % QEDI_PAGE_SIZE)))
2003 cmd->use_slowpath = true;
2004
2005 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO, "sg[%d] size=0x%x",
2006 i, sg_len);
2007
2008 if (sg_len > QEDI_BD_SPLIT_SZ) {
2009 sg_frags = qedi_split_bd(cmd, addr, sg_len, bd_count);
2010 } else {
2011 sg_frags = 1;
2012 bd[bd_count].sge_addr.lo = addr & 0xffffffff;
2013 bd[bd_count].sge_addr.hi = addr >> 32;
2014 bd[bd_count].sge_len = sg_len;
2015 }
2016 byte_count += sg_len;
2017 bd_count += sg_frags;
2018 }
2019
2020 if (byte_count != scsi_bufflen(sc))
2021 QEDI_ERR(&qedi->dbg_ctx,
2022 "byte_count = %d != scsi_bufflen = %d\n", byte_count,
2023 scsi_bufflen(sc));
2024 else
2025 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO, "byte_count = %d\n",
2026 byte_count);
2027
2028 WARN_ON(byte_count != scsi_bufflen(sc));
2029
2030 return bd_count;
2031 }
2032
2033 static void qedi_iscsi_map_sg_list(struct qedi_cmd *cmd)
2034 {
2035 int bd_count;
2036 struct scsi_cmnd *sc = cmd->scsi_cmd;
2037
2038 if (scsi_sg_count(sc)) {
2039 bd_count = qedi_map_scsi_sg(cmd->conn->qedi, cmd);
2040 if (bd_count == 0)
2041 return;
2042 } else {
2043 struct iscsi_sge *bd = cmd->io_tbl.sge_tbl;
2044
2045 bd[0].sge_addr.lo = 0;
2046 bd[0].sge_addr.hi = 0;
2047 bd[0].sge_len = 0;
2048 bd_count = 0;
2049 }
2050 cmd->io_tbl.sge_valid = bd_count;
2051 }
2052
2053 static void qedi_cpy_scsi_cdb(struct scsi_cmnd *sc, u32 *dstp)
2054 {
2055 u32 dword;
2056 int lpcnt;
2057 u8 *srcp;
2058
2059 lpcnt = sc->cmd_len / sizeof(dword);
2060 srcp = (u8 *)sc->cmnd;
2061 while (lpcnt--) {
2062 memcpy(&dword, (const void *)srcp, 4);
2063 *dstp = cpu_to_be32(dword);
2064 srcp += 4;
2065 dstp++;
2066 }
2067 if (sc->cmd_len & 0x3) {
2068 dword = (u32)srcp[0] | ((u32)srcp[1] << 8);
2069 *dstp = cpu_to_be32(dword);
2070 }
2071 }
2072
2073 void qedi_trace_io(struct qedi_ctx *qedi, struct iscsi_task *task,
2074 u16 tid, int8_t direction)
2075 {
2076 struct qedi_io_log *io_log;
2077 struct iscsi_conn *conn = task->conn;
2078 struct qedi_conn *qedi_conn = conn->dd_data;
2079 struct scsi_cmnd *sc_cmd = task->sc;
2080 unsigned long flags;
2081 u8 op;
2082
2083 spin_lock_irqsave(&qedi->io_trace_lock, flags);
2084
2085 io_log = &qedi->io_trace_buf[qedi->io_trace_idx];
2086 io_log->direction = direction;
2087 io_log->task_id = tid;
2088 io_log->cid = qedi_conn->iscsi_conn_id;
2089 io_log->lun = sc_cmd->device->lun;
2090 io_log->op = sc_cmd->cmnd[0];
2091 op = sc_cmd->cmnd[0];
2092 io_log->lba[0] = sc_cmd->cmnd[2];
2093 io_log->lba[1] = sc_cmd->cmnd[3];
2094 io_log->lba[2] = sc_cmd->cmnd[4];
2095 io_log->lba[3] = sc_cmd->cmnd[5];
2096 io_log->bufflen = scsi_bufflen(sc_cmd);
2097 io_log->sg_count = scsi_sg_count(sc_cmd);
2098 io_log->fast_sgs = qedi->fast_sgls;
2099 io_log->cached_sgs = qedi->cached_sgls;
2100 io_log->slow_sgs = qedi->slow_sgls;
2101 io_log->cached_sge = qedi->use_cached_sge;
2102 io_log->slow_sge = qedi->use_slow_sge;
2103 io_log->fast_sge = qedi->use_fast_sge;
2104 io_log->result = sc_cmd->result;
2105 io_log->jiffies = jiffies;
2106 io_log->blk_req_cpu = smp_processor_id();
2107
2108 if (direction == QEDI_IO_TRACE_REQ) {
2109 /* For requests we only care about the submission CPU */
2110 io_log->req_cpu = smp_processor_id() % qedi->num_queues;
2111 io_log->intr_cpu = 0;
2112 io_log->blk_rsp_cpu = 0;
2113 } else if (direction == QEDI_IO_TRACE_RSP) {
2114 io_log->req_cpu = smp_processor_id() % qedi->num_queues;
2115 io_log->intr_cpu = qedi->intr_cpu;
2116 io_log->blk_rsp_cpu = smp_processor_id();
2117 }
2118
2119 qedi->io_trace_idx++;
2120 if (qedi->io_trace_idx == QEDI_IO_TRACE_SIZE)
2121 qedi->io_trace_idx = 0;
2122
2123 qedi->use_cached_sge = false;
2124 qedi->use_slow_sge = false;
2125 qedi->use_fast_sge = false;
2126
2127 spin_unlock_irqrestore(&qedi->io_trace_lock, flags);
2128 }
2129
2130 int qedi_iscsi_send_ioreq(struct iscsi_task *task)
2131 {
2132 struct iscsi_conn *conn = task->conn;
2133 struct iscsi_session *session = conn->session;
2134 struct Scsi_Host *shost = iscsi_session_to_shost(session->cls_session);
2135 struct qedi_ctx *qedi = iscsi_host_priv(shost);
2136 struct qedi_conn *qedi_conn = conn->dd_data;
2137 struct qedi_cmd *cmd = task->dd_data;
2138 struct scsi_cmnd *sc = task->sc;
2139 struct iscsi_task_context *fw_task_ctx;
2140 struct iscsi_cached_sge_ctx *cached_sge;
2141 struct iscsi_phys_sgl_ctx *phys_sgl;
2142 struct iscsi_virt_sgl_ctx *virt_sgl;
2143 struct ystorm_iscsi_task_st_ctx *yst_cxt;
2144 struct mstorm_iscsi_task_st_ctx *mst_cxt;
2145 struct iscsi_sgl *sgl_struct;
2146 struct iscsi_sge *single_sge;
2147 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)task->hdr;
2148 struct iscsi_sge *bd = cmd->io_tbl.sge_tbl;
2149 enum iscsi_task_type task_type;
2150 struct iscsi_cmd_hdr *fw_cmd;
2151 u32 lun[2];
2152 u32 exp_data;
2153 u16 cq_idx = smp_processor_id() % qedi->num_queues;
2154 s16 ptu_invalidate = 0;
2155 s16 tid = 0;
2156 u8 num_fast_sgs;
2157
2158 tid = qedi_get_task_idx(qedi);
2159 if (tid == -1)
2160 return -ENOMEM;
2161
2162 qedi_iscsi_map_sg_list(cmd);
2163
2164 int_to_scsilun(sc->device->lun, (struct scsi_lun *)lun);
2165 fw_task_ctx = qedi_get_task_mem(&qedi->tasks, tid);
2166
2167 memset(fw_task_ctx, 0, sizeof(struct iscsi_task_context));
2168 cmd->task_id = tid;
2169
2170 /* Ystorm context */
2171 fw_cmd = &fw_task_ctx->ystorm_st_context.pdu_hdr.cmd;
2172 SET_FIELD(fw_cmd->flags_attr, ISCSI_CMD_HDR_ATTR, ISCSI_ATTR_SIMPLE);
2173
2174 if (sc->sc_data_direction == DMA_TO_DEVICE) {
2175 if (conn->session->initial_r2t_en) {
2176 exp_data = min((conn->session->imm_data_en *
2177 conn->max_xmit_dlength),
2178 conn->session->first_burst);
2179 exp_data = min(exp_data, scsi_bufflen(sc));
2180 fw_task_ctx->ustorm_ag_context.exp_data_acked =
2181 cpu_to_le32(exp_data);
2182 } else {
2183 fw_task_ctx->ustorm_ag_context.exp_data_acked =
2184 min(conn->session->first_burst, scsi_bufflen(sc));
2185 }
2186
2187 SET_FIELD(fw_cmd->flags_attr, ISCSI_CMD_HDR_WRITE, 1);
2188 task_type = ISCSI_TASK_TYPE_INITIATOR_WRITE;
2189 } else {
2190 if (scsi_bufflen(sc))
2191 SET_FIELD(fw_cmd->flags_attr, ISCSI_CMD_HDR_READ, 1);
2192 task_type = ISCSI_TASK_TYPE_INITIATOR_READ;
2193 }
2194
2195 fw_cmd->lun.lo = be32_to_cpu(lun[0]);
2196 fw_cmd->lun.hi = be32_to_cpu(lun[1]);
2197
2198 qedi_update_itt_map(qedi, tid, task->itt, cmd);
2199 fw_cmd->itt = qedi_set_itt(tid, get_itt(task->itt));
2200 fw_cmd->expected_transfer_length = scsi_bufflen(sc);
2201 fw_cmd->cmd_sn = be32_to_cpu(hdr->cmdsn);
2202 fw_cmd->opcode = hdr->opcode;
2203 qedi_cpy_scsi_cdb(sc, (u32 *)fw_cmd->cdb);
2204
2205 /* Mstorm context */
2206 fw_task_ctx->mstorm_st_context.sense_db.lo = (u32)cmd->sense_buffer_dma;
2207 fw_task_ctx->mstorm_st_context.sense_db.hi =
2208 (u32)((u64)cmd->sense_buffer_dma >> 32);
2209 fw_task_ctx->mstorm_ag_context.task_cid = qedi_conn->iscsi_conn_id;
2210 fw_task_ctx->mstorm_st_context.task_type = task_type;
2211
2212 if (qedi->tid_reuse_count[tid] == QEDI_MAX_TASK_NUM) {
2213 ptu_invalidate = 1;
2214 qedi->tid_reuse_count[tid] = 0;
2215 }
2216 fw_task_ctx->ystorm_st_context.state.reuse_count =
2217 qedi->tid_reuse_count[tid];
2218 fw_task_ctx->mstorm_st_context.reuse_count =
2219 qedi->tid_reuse_count[tid]++;
2220
2221 /* Ustorm context */
2222 fw_task_ctx->ustorm_st_context.rem_rcv_len = scsi_bufflen(sc);
2223 fw_task_ctx->ustorm_st_context.exp_data_transfer_len = scsi_bufflen(sc);
2224 fw_task_ctx->ustorm_st_context.exp_data_sn =
2225 be32_to_cpu(hdr->exp_statsn);
2226 fw_task_ctx->ustorm_st_context.task_type = task_type;
2227 fw_task_ctx->ustorm_st_context.cq_rss_number = cq_idx;
2228 fw_task_ctx->ustorm_ag_context.icid = (u16)qedi_conn->iscsi_conn_id;
2229
2230 SET_FIELD(fw_task_ctx->ustorm_ag_context.flags1,
2231 USTORM_ISCSI_TASK_AG_CTX_R2T2RECV, 1);
2232 SET_FIELD(fw_task_ctx->ustorm_st_context.flags,
2233 USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP, 0);
2234
2235 num_fast_sgs = (cmd->io_tbl.sge_valid ?
2236 min((u16)QEDI_FAST_SGE_COUNT,
2237 (u16)cmd->io_tbl.sge_valid) : 0);
2238 SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map,
2239 ISCSI_REG1_NUM_FAST_SGES, num_fast_sgs);
2240
2241 fw_task_ctx->ustorm_st_context.lun.lo = be32_to_cpu(lun[0]);
2242 fw_task_ctx->ustorm_st_context.lun.hi = be32_to_cpu(lun[1]);
2243
2244 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO, "Total sge count [%d]\n",
2245 cmd->io_tbl.sge_valid);
2246
2247 yst_cxt = &fw_task_ctx->ystorm_st_context;
2248 mst_cxt = &fw_task_ctx->mstorm_st_context;
2249 /* Tx path */
2250 if (task_type == ISCSI_TASK_TYPE_INITIATOR_WRITE) {
2251 /* not considering superIO or FastIO */
2252 if (cmd->io_tbl.sge_valid == 1) {
2253 cached_sge = &yst_cxt->state.sgl_ctx_union.cached_sge;
2254 cached_sge->sge.sge_addr.lo = bd[0].sge_addr.lo;
2255 cached_sge->sge.sge_addr.hi = bd[0].sge_addr.hi;
2256 cached_sge->sge.sge_len = bd[0].sge_len;
2257 qedi->cached_sgls++;
2258 } else if ((cmd->io_tbl.sge_valid != 1) && cmd->use_slowpath) {
2259 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
2260 ISCSI_MFLAGS_SLOW_IO, 1);
2261 SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map,
2262 ISCSI_REG1_NUM_FAST_SGES, 0);
2263 phys_sgl = &yst_cxt->state.sgl_ctx_union.phys_sgl;
2264 phys_sgl->sgl_base.lo = (u32)(cmd->io_tbl.sge_tbl_dma);
2265 phys_sgl->sgl_base.hi =
2266 (u32)((u64)cmd->io_tbl.sge_tbl_dma >> 32);
2267 phys_sgl->sgl_size = cmd->io_tbl.sge_valid;
2268 qedi->slow_sgls++;
2269 } else if ((cmd->io_tbl.sge_valid != 1) && !cmd->use_slowpath) {
2270 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
2271 ISCSI_MFLAGS_SLOW_IO, 0);
2272 SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map,
2273 ISCSI_REG1_NUM_FAST_SGES,
2274 min((u16)QEDI_FAST_SGE_COUNT,
2275 (u16)cmd->io_tbl.sge_valid));
2276 virt_sgl = &yst_cxt->state.sgl_ctx_union.virt_sgl;
2277 virt_sgl->sgl_base.lo = (u32)(cmd->io_tbl.sge_tbl_dma);
2278 virt_sgl->sgl_base.hi =
2279 (u32)((u64)cmd->io_tbl.sge_tbl_dma >> 32);
2280 virt_sgl->sgl_initial_offset =
2281 (u32)bd[0].sge_addr.lo & (QEDI_PAGE_SIZE - 1);
2282 qedi->fast_sgls++;
2283 }
2284 fw_task_ctx->mstorm_st_context.sgl_size = cmd->io_tbl.sge_valid;
2285 fw_task_ctx->mstorm_st_context.rem_task_size = scsi_bufflen(sc);
2286 } else {
2287 /* Rx path */
2288 if (cmd->io_tbl.sge_valid == 1) {
2289 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
2290 ISCSI_MFLAGS_SLOW_IO, 0);
2291 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
2292 ISCSI_MFLAGS_SINGLE_SGE, 1);
2293 single_sge = &mst_cxt->sgl_union.single_sge;
2294 single_sge->sge_addr.lo = bd[0].sge_addr.lo;
2295 single_sge->sge_addr.hi = bd[0].sge_addr.hi;
2296 single_sge->sge_len = bd[0].sge_len;
2297 qedi->cached_sgls++;
2298 } else if ((cmd->io_tbl.sge_valid != 1) && cmd->use_slowpath) {
2299 sgl_struct = &mst_cxt->sgl_union.sgl_struct;
2300 sgl_struct->sgl_addr.lo =
2301 (u32)(cmd->io_tbl.sge_tbl_dma);
2302 sgl_struct->sgl_addr.hi =
2303 (u32)((u64)cmd->io_tbl.sge_tbl_dma >> 32);
2304 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
2305 ISCSI_MFLAGS_SLOW_IO, 1);
2306 SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map,
2307 ISCSI_REG1_NUM_FAST_SGES, 0);
2308 sgl_struct->updated_sge_size = 0;
2309 sgl_struct->updated_sge_offset = 0;
2310 qedi->slow_sgls++;
2311 } else if ((cmd->io_tbl.sge_valid != 1) && !cmd->use_slowpath) {
2312 sgl_struct = &mst_cxt->sgl_union.sgl_struct;
2313 sgl_struct->sgl_addr.lo =
2314 (u32)(cmd->io_tbl.sge_tbl_dma);
2315 sgl_struct->sgl_addr.hi =
2316 (u32)((u64)cmd->io_tbl.sge_tbl_dma >> 32);
2317 sgl_struct->byte_offset =
2318 (u32)bd[0].sge_addr.lo & (QEDI_PAGE_SIZE - 1);
2319 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
2320 ISCSI_MFLAGS_SLOW_IO, 0);
2321 SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map,
2322 ISCSI_REG1_NUM_FAST_SGES, 0);
2323 sgl_struct->updated_sge_size = 0;
2324 sgl_struct->updated_sge_offset = 0;
2325 qedi->fast_sgls++;
2326 }
2327 fw_task_ctx->mstorm_st_context.sgl_size = cmd->io_tbl.sge_valid;
2328 fw_task_ctx->mstorm_st_context.rem_task_size = scsi_bufflen(sc);
2329 }
2330
2331 if (cmd->io_tbl.sge_valid == 1)
2332 /* Singel-SGL */
2333 qedi->use_cached_sge = true;
2334 else {
2335 if (cmd->use_slowpath)
2336 qedi->use_slow_sge = true;
2337 else
2338 qedi->use_fast_sge = true;
2339 }
2340 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO,
2341 "%s: %s-SGL: num_sges=0x%x first-sge-lo=0x%x first-sge-hi=0x%x",
2342 (task_type == ISCSI_TASK_TYPE_INITIATOR_WRITE) ?
2343 "Write " : "Read ", (cmd->io_tbl.sge_valid == 1) ?
2344 "Single" : (cmd->use_slowpath ? "SLOW" : "FAST"),
2345 (u16)cmd->io_tbl.sge_valid, (u32)(cmd->io_tbl.sge_tbl_dma),
2346 (u32)((u64)cmd->io_tbl.sge_tbl_dma >> 32));
2347
2348 /* Add command in active command list */
2349 spin_lock(&qedi_conn->list_lock);
2350 list_add_tail(&cmd->io_cmd, &qedi_conn->active_cmd_list);
2351 cmd->io_cmd_in_list = true;
2352 qedi_conn->active_cmd_count++;
2353 spin_unlock(&qedi_conn->list_lock);
2354
2355 qedi_add_to_sq(qedi_conn, task, tid, ptu_invalidate, false);
2356 qedi_ring_doorbell(qedi_conn);
2357 if (qedi_io_tracing)
2358 qedi_trace_io(qedi, task, tid, QEDI_IO_TRACE_REQ);
2359
2360 return 0;
2361 }
2362
2363 int qedi_iscsi_cleanup_task(struct iscsi_task *task, bool mark_cmd_node_deleted)
2364 {
2365 struct iscsi_conn *conn = task->conn;
2366 struct qedi_conn *qedi_conn = conn->dd_data;
2367 struct qedi_cmd *cmd = task->dd_data;
2368 s16 ptu_invalidate = 0;
2369
2370 QEDI_INFO(&qedi_conn->qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
2371 "issue cleanup tid=0x%x itt=0x%x task_state=%d cmd_state=0%x cid=0x%x\n",
2372 cmd->task_id, get_itt(task->itt), task->state,
2373 cmd->state, qedi_conn->iscsi_conn_id);
2374
2375 qedi_add_to_sq(qedi_conn, task, cmd->task_id, ptu_invalidate, true);
2376 qedi_ring_doorbell(qedi_conn);
2377
2378 return 0;
2379 }
ubuntu-bionic-kernel mirror