1 /*******************************************************************
2 * This file is part of the Emulex Linux Device Driver for *
3 * Fibre Channel Host Bus Adapters. *
4 * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
5 * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
6 * Copyright (C) 2004-2016 Emulex. All rights reserved. *
7 * EMULEX and SLI are trademarks of Emulex. *
9 * Portions Copyright (C) 2004-2005 Christoph Hellwig *
11 * This program is free software; you can redistribute it and/or *
12 * modify it under the terms of version 2 of the GNU General *
13 * Public License as published by the Free Software Foundation. *
14 * This program is distributed in the hope that it will be useful. *
15 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND *
16 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, *
17 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE *
18 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
19 * TO BE LEGALLY INVALID. See the GNU General Public License for *
20 * more details, a copy of which can be found in the file COPYING *
21 * included with this package. *
22 ********************************************************************/
23 #include <linux/pci.h>
24 #include <linux/slab.h>
25 #include <linux/interrupt.h>
26 #include <linux/delay.h>
27 #include <asm/unaligned.h>
28 #include <linux/crc-t10dif.h>
29 #include <net/checksum.h>
31 #include <scsi/scsi.h>
32 #include <scsi/scsi_device.h>
33 #include <scsi/scsi_eh.h>
34 #include <scsi/scsi_host.h>
35 #include <scsi/scsi_tcq.h>
36 #include <scsi/scsi_transport_fc.h>
37 #include <scsi/fc/fc_fs.h>
39 #include <linux/nvme.h>
40 #include <linux/nvme-fc-driver.h>
41 #include <linux/nvme-fc.h>
42 #include "lpfc_version.h"
46 #include "lpfc_sli4.h"
48 #include "lpfc_disc.h"
50 #include "lpfc_nvme.h"
51 #include "lpfc_scsi.h"
52 #include "lpfc_logmsg.h"
53 #include "lpfc_crtn.h"
54 #include "lpfc_vport.h"
55 #include "lpfc_debugfs.h"
57 /* NVME initiator-based functions */
59 static struct lpfc_nvme_buf
*
60 lpfc_get_nvme_buf(struct lpfc_hba
*phba
, struct lpfc_nodelist
*ndlp
);
63 lpfc_release_nvme_buf(struct lpfc_hba
*, struct lpfc_nvme_buf
*);
67 * lpfc_nvme_create_queue -
68 * @lpfc_pnvme: Pointer to the driver's nvme instance data
69 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
70 * @handle: An opaque driver handle used in follow-up calls.
72 * Driver registers this routine to preallocate and initialize any
73 * internal data structures to bind the @qidx to its internal IO queues.
74 * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ.
78 * -EINVAL - Unsupported input value.
79 * -ENOMEM - Could not alloc necessary memory
82 lpfc_nvme_create_queue(struct nvme_fc_local_port
*pnvme_lport
,
83 unsigned int qidx
, u16 qsize
,
86 struct lpfc_nvme_lport
*lport
;
87 struct lpfc_vport
*vport
;
88 struct lpfc_nvme_qhandle
*qhandle
;
91 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
93 qhandle
= kzalloc(sizeof(struct lpfc_nvme_qhandle
), GFP_KERNEL
);
97 qhandle
->cpu_id
= smp_processor_id();
100 * NVME qidx == 0 is the admin queue, so both admin queue
101 * and first IO queue will use MSI-X vector and associated
102 * EQ/CQ/WQ at index 0. After that they are sequentially assigned.
105 str
= "IO "; /* IO queue */
106 qhandle
->index
= ((qidx
- 1) %
107 vport
->phba
->cfg_nvme_io_channel
);
109 str
= "ADM"; /* Admin queue */
110 qhandle
->index
= qidx
;
113 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
,
114 "6073 Binding %s HdwQueue %d (cpu %d) to "
115 "io_channel %d qhandle %p\n", str
,
116 qidx
, qhandle
->cpu_id
, qhandle
->index
, qhandle
);
117 *handle
= (void *)qhandle
;
122 * lpfc_nvme_delete_queue -
123 * @lpfc_pnvme: Pointer to the driver's nvme instance data
124 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
125 * @handle: An opaque driver handle from lpfc_nvme_create_queue
127 * Driver registers this routine to free
128 * any internal data structures to bind the @qidx to its internal
133 * TODO: What are the failure codes.
136 lpfc_nvme_delete_queue(struct nvme_fc_local_port
*pnvme_lport
,
140 struct lpfc_nvme_lport
*lport
;
141 struct lpfc_vport
*vport
;
143 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
144 vport
= lport
->vport
;
146 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
,
147 "6001 ENTER. lpfc_pnvme %p, qidx x%xi qhandle %p\n",
148 lport
, qidx
, handle
);
153 lpfc_nvme_localport_delete(struct nvme_fc_local_port
*localport
)
155 struct lpfc_nvme_lport
*lport
= localport
->private;
157 lpfc_printf_vlog(lport
->vport
, KERN_INFO
, LOG_NVME
,
158 "6173 localport %p delete complete\n",
161 /* release any threads waiting for the unreg to complete */
162 complete(&lport
->lport_unreg_done
);
165 /* lpfc_nvme_remoteport_delete
167 * @remoteport: Pointer to an nvme transport remoteport instance.
169 * This is a template downcall. NVME transport calls this function
170 * when it has completed the unregistration of a previously
171 * registered remoteport.
177 lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port
*remoteport
)
179 struct lpfc_nvme_rport
*rport
= remoteport
->private;
180 struct lpfc_vport
*vport
;
181 struct lpfc_nodelist
*ndlp
;
191 /* Remove this rport from the lport's list - memory is owned by the
192 * transport. Remove the ndlp reference for the NVME transport before
193 * calling state machine to remove the node.
195 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
196 "6146 remoteport delete complete %p\n",
202 /* This call has to execute as long as the rport is valid.
203 * Release any threads waiting for the unreg to complete.
205 complete(&rport
->rport_unreg_done
);
209 lpfc_nvme_cmpl_gen_req(struct lpfc_hba
*phba
, struct lpfc_iocbq
*cmdwqe
,
210 struct lpfc_wcqe_complete
*wcqe
)
212 struct lpfc_vport
*vport
= cmdwqe
->vport
;
214 struct nvmefc_ls_req
*pnvme_lsreq
;
215 struct lpfc_dmabuf
*buf_ptr
;
216 struct lpfc_nodelist
*ndlp
;
218 atomic_inc(&vport
->phba
->fc4NvmeLsCmpls
);
220 pnvme_lsreq
= (struct nvmefc_ls_req
*)cmdwqe
->context2
;
221 status
= bf_get(lpfc_wcqe_c_status
, wcqe
) & LPFC_IOCB_STATUS_MASK
;
222 ndlp
= (struct lpfc_nodelist
*)cmdwqe
->context1
;
223 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
224 "6047 nvme cmpl Enter "
225 "Data %p DID %x Xri: %x status %x cmd:%p lsreg:%p "
227 pnvme_lsreq
, ndlp
? ndlp
->nlp_DID
: 0,
228 cmdwqe
->sli4_xritag
, status
,
229 cmdwqe
, pnvme_lsreq
, cmdwqe
->context3
, ndlp
);
231 lpfc_nvmeio_data(phba
, "NVME LS CMPL: xri x%x stat x%x parm x%x\n",
232 cmdwqe
->sli4_xritag
, status
, wcqe
->parameter
);
234 if (cmdwqe
->context3
) {
235 buf_ptr
= (struct lpfc_dmabuf
*)cmdwqe
->context3
;
236 lpfc_mbuf_free(phba
, buf_ptr
->virt
, buf_ptr
->phys
);
238 cmdwqe
->context3
= NULL
;
240 if (pnvme_lsreq
->done
)
241 pnvme_lsreq
->done(pnvme_lsreq
, status
);
243 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
244 "6046 nvme cmpl without done call back? "
245 "Data %p DID %x Xri: %x status %x\n",
246 pnvme_lsreq
, ndlp
? ndlp
->nlp_DID
: 0,
247 cmdwqe
->sli4_xritag
, status
);
250 cmdwqe
->context1
= NULL
;
252 lpfc_sli_release_iocbq(phba
, cmdwqe
);
256 lpfc_nvme_gen_req(struct lpfc_vport
*vport
, struct lpfc_dmabuf
*bmp
,
257 struct lpfc_dmabuf
*inp
,
258 struct nvmefc_ls_req
*pnvme_lsreq
,
259 void (*cmpl
)(struct lpfc_hba
*, struct lpfc_iocbq
*,
260 struct lpfc_wcqe_complete
*),
261 struct lpfc_nodelist
*ndlp
, uint32_t num_entry
,
262 uint32_t tmo
, uint8_t retry
)
264 struct lpfc_hba
*phba
= vport
->phba
;
266 struct lpfc_iocbq
*genwqe
;
267 struct ulp_bde64
*bpl
;
268 struct ulp_bde64 bde
;
269 int i
, rc
, xmit_len
, first_len
;
271 /* Allocate buffer for command WQE */
272 genwqe
= lpfc_sli_get_iocbq(phba
);
277 memset(wqe
, 0, sizeof(union lpfc_wqe
));
279 genwqe
->context3
= (uint8_t *)bmp
;
280 genwqe
->iocb_flag
|= LPFC_IO_NVME_LS
;
282 /* Save for completion so we can release these resources */
283 genwqe
->context1
= lpfc_nlp_get(ndlp
);
284 genwqe
->context2
= (uint8_t *)pnvme_lsreq
;
285 /* Fill in payload, bp points to frame payload */
288 /* FC spec states we need 3 * ratov for CT requests */
289 tmo
= (3 * phba
->fc_ratov
);
291 /* For this command calculate the xmit length of the request bde. */
294 bpl
= (struct ulp_bde64
*)bmp
->virt
;
295 for (i
= 0; i
< num_entry
; i
++) {
296 bde
.tus
.w
= bpl
[i
].tus
.w
;
297 if (bde
.tus
.f
.bdeFlags
!= BUFF_TYPE_BDE_64
)
299 xmit_len
+= bde
.tus
.f
.bdeSize
;
301 first_len
= xmit_len
;
304 genwqe
->rsvd2
= num_entry
;
305 genwqe
->hba_wqidx
= 0;
308 wqe
->generic
.bde
.tus
.f
.bdeFlags
= BUFF_TYPE_BDE_64
;
309 wqe
->generic
.bde
.tus
.f
.bdeSize
= first_len
;
310 wqe
->generic
.bde
.addrLow
= bpl
[0].addrLow
;
311 wqe
->generic
.bde
.addrHigh
= bpl
[0].addrHigh
;
314 wqe
->gen_req
.request_payload_len
= first_len
;
319 bf_set(wqe_dfctl
, &wqe
->gen_req
.wge_ctl
, 0);
320 bf_set(wqe_si
, &wqe
->gen_req
.wge_ctl
, 1);
321 bf_set(wqe_la
, &wqe
->gen_req
.wge_ctl
, 1);
322 bf_set(wqe_rctl
, &wqe
->gen_req
.wge_ctl
, FC_RCTL_ELS4_REQ
);
323 bf_set(wqe_type
, &wqe
->gen_req
.wge_ctl
, FC_TYPE_NVME
);
326 bf_set(wqe_ctxt_tag
, &wqe
->gen_req
.wqe_com
,
327 phba
->sli4_hba
.rpi_ids
[ndlp
->nlp_rpi
]);
328 bf_set(wqe_xri_tag
, &wqe
->gen_req
.wqe_com
, genwqe
->sli4_xritag
);
331 bf_set(wqe_tmo
, &wqe
->gen_req
.wqe_com
, (vport
->phba
->fc_ratov
-1));
332 bf_set(wqe_class
, &wqe
->gen_req
.wqe_com
, CLASS3
);
333 bf_set(wqe_cmnd
, &wqe
->gen_req
.wqe_com
, CMD_GEN_REQUEST64_WQE
);
334 bf_set(wqe_ct
, &wqe
->gen_req
.wqe_com
, SLI4_CT_RPI
);
337 wqe
->gen_req
.wqe_com
.abort_tag
= genwqe
->iotag
;
340 bf_set(wqe_reqtag
, &wqe
->gen_req
.wqe_com
, genwqe
->iotag
);
343 bf_set(wqe_dbde
, &wqe
->gen_req
.wqe_com
, 1);
344 bf_set(wqe_iod
, &wqe
->gen_req
.wqe_com
, LPFC_WQE_IOD_READ
);
345 bf_set(wqe_qosd
, &wqe
->gen_req
.wqe_com
, 1);
346 bf_set(wqe_lenloc
, &wqe
->gen_req
.wqe_com
, LPFC_WQE_LENLOC_NONE
);
347 bf_set(wqe_ebde_cnt
, &wqe
->gen_req
.wqe_com
, 0);
350 bf_set(wqe_cqid
, &wqe
->gen_req
.wqe_com
, LPFC_WQE_CQ_ID_DEFAULT
);
351 bf_set(wqe_cmd_type
, &wqe
->gen_req
.wqe_com
, OTHER_COMMAND
);
354 /* Issue GEN REQ WQE for NPORT <did> */
355 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_ELS
,
356 "6050 Issue GEN REQ WQE to NPORT x%x "
357 "Data: x%x x%x wq:%p lsreq:%p bmp:%p xmit:%d 1st:%d\n",
358 ndlp
->nlp_DID
, genwqe
->iotag
,
360 genwqe
, pnvme_lsreq
, bmp
, xmit_len
, first_len
);
361 genwqe
->wqe_cmpl
= cmpl
;
362 genwqe
->iocb_cmpl
= NULL
;
363 genwqe
->drvrTimeout
= tmo
+ LPFC_DRVR_TIMEOUT
;
364 genwqe
->vport
= vport
;
365 genwqe
->retry
= retry
;
367 lpfc_nvmeio_data(phba
, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n",
368 genwqe
->sli4_xritag
, genwqe
->iotag
, ndlp
->nlp_DID
);
370 rc
= lpfc_sli4_issue_wqe(phba
, LPFC_ELS_RING
, genwqe
);
372 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_ELS
,
373 "6045 Issue GEN REQ WQE to NPORT x%x "
375 ndlp
->nlp_DID
, genwqe
->iotag
,
377 lpfc_sli_release_iocbq(phba
, genwqe
);
384 * lpfc_nvme_ls_req - Issue an Link Service request
385 * @lpfc_pnvme: Pointer to the driver's nvme instance data
386 * @lpfc_nvme_lport: Pointer to the driver's local port data
387 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
389 * Driver registers this routine to handle any link service request
390 * from the nvme_fc transport to a remote nvme-aware port.
394 * TODO: What are the failure codes.
397 lpfc_nvme_ls_req(struct nvme_fc_local_port
*pnvme_lport
,
398 struct nvme_fc_remote_port
*pnvme_rport
,
399 struct nvmefc_ls_req
*pnvme_lsreq
)
402 struct lpfc_nvme_lport
*lport
;
403 struct lpfc_vport
*vport
;
404 struct lpfc_nodelist
*ndlp
;
405 struct ulp_bde64
*bpl
;
406 struct lpfc_dmabuf
*bmp
;
407 uint16_t ntype
, nstate
;
409 /* there are two dma buf in the request, actually there is one and
410 * the second one is just the start address + cmd size.
411 * Before calling lpfc_nvme_gen_req these buffers need to be wrapped
412 * in a lpfc_dmabuf struct. When freeing we just free the wrapper
413 * because the nvem layer owns the data bufs.
414 * We do not have to break these packets open, we don't care what is in
415 * them. And we do not have to look at the resonse data, we only care
416 * that we got a response. All of the caring is going to happen in the
420 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
421 vport
= lport
->vport
;
423 if (vport
->load_flag
& FC_UNLOADING
)
426 if (vport
->load_flag
& FC_UNLOADING
)
429 ndlp
= lpfc_findnode_did(vport
, pnvme_rport
->port_id
);
430 if (!ndlp
|| !NLP_CHK_NODE_ACT(ndlp
)) {
431 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
432 "6051 DID x%06x not an active rport.\n",
433 pnvme_rport
->port_id
);
437 /* The remote node has to be a mapped nvme target or an
438 * unmapped nvme initiator or it's an error.
440 ntype
= ndlp
->nlp_type
;
441 nstate
= ndlp
->nlp_state
;
442 if ((ntype
& NLP_NVME_TARGET
&& nstate
!= NLP_STE_MAPPED_NODE
) ||
443 (ntype
& NLP_NVME_INITIATOR
&& nstate
!= NLP_STE_UNMAPPED_NODE
)) {
444 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
445 "6088 DID x%06x not ready for "
446 "IO. State x%x, Type x%x\n",
447 pnvme_rport
->port_id
,
448 ndlp
->nlp_state
, ndlp
->nlp_type
);
451 bmp
= kmalloc(sizeof(struct lpfc_dmabuf
), GFP_KERNEL
);
454 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
455 "6044 Could not find node for DID %x\n",
456 pnvme_rport
->port_id
);
459 INIT_LIST_HEAD(&bmp
->list
);
460 bmp
->virt
= lpfc_mbuf_alloc(vport
->phba
, MEM_PRI
, &(bmp
->phys
));
462 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
463 "6042 Could not find node for DID %x\n",
464 pnvme_rport
->port_id
);
468 bpl
= (struct ulp_bde64
*)bmp
->virt
;
469 bpl
->addrHigh
= le32_to_cpu(putPaddrHigh(pnvme_lsreq
->rqstdma
));
470 bpl
->addrLow
= le32_to_cpu(putPaddrLow(pnvme_lsreq
->rqstdma
));
471 bpl
->tus
.f
.bdeFlags
= 0;
472 bpl
->tus
.f
.bdeSize
= pnvme_lsreq
->rqstlen
;
473 bpl
->tus
.w
= le32_to_cpu(bpl
->tus
.w
);
476 bpl
->addrHigh
= le32_to_cpu(putPaddrHigh(pnvme_lsreq
->rspdma
));
477 bpl
->addrLow
= le32_to_cpu(putPaddrLow(pnvme_lsreq
->rspdma
));
478 bpl
->tus
.f
.bdeFlags
= BUFF_TYPE_BDE_64I
;
479 bpl
->tus
.f
.bdeSize
= pnvme_lsreq
->rsplen
;
480 bpl
->tus
.w
= le32_to_cpu(bpl
->tus
.w
);
482 /* Expand print to include key fields. */
483 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
484 "6149 ENTER. lport %p, rport %p lsreq%p rqstlen:%d "
485 "rsplen:%d %pad %pad\n",
486 pnvme_lport
, pnvme_rport
,
487 pnvme_lsreq
, pnvme_lsreq
->rqstlen
,
488 pnvme_lsreq
->rsplen
, &pnvme_lsreq
->rqstdma
,
489 &pnvme_lsreq
->rspdma
);
491 atomic_inc(&vport
->phba
->fc4NvmeLsRequests
);
493 /* Hardcode the wait to 30 seconds. Connections are failing otherwise.
494 * This code allows it all to work.
496 ret
= lpfc_nvme_gen_req(vport
, bmp
, pnvme_lsreq
->rqstaddr
,
497 pnvme_lsreq
, lpfc_nvme_cmpl_gen_req
,
499 if (ret
!= WQE_SUCCESS
) {
500 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
501 "6052 EXIT. issue ls wqe failed lport %p, "
502 "rport %p lsreq%p Status %x DID %x\n",
503 pnvme_lport
, pnvme_rport
, pnvme_lsreq
,
505 lpfc_mbuf_free(vport
->phba
, bmp
->virt
, bmp
->phys
);
510 /* Stub in routine and return 0 for now. */
515 * lpfc_nvme_ls_abort - Issue an Link Service request
516 * @lpfc_pnvme: Pointer to the driver's nvme instance data
517 * @lpfc_nvme_lport: Pointer to the driver's local port data
518 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
520 * Driver registers this routine to handle any link service request
521 * from the nvme_fc transport to a remote nvme-aware port.
525 * TODO: What are the failure codes.
528 lpfc_nvme_ls_abort(struct nvme_fc_local_port
*pnvme_lport
,
529 struct nvme_fc_remote_port
*pnvme_rport
,
530 struct nvmefc_ls_req
*pnvme_lsreq
)
532 struct lpfc_nvme_lport
*lport
;
533 struct lpfc_vport
*vport
;
534 struct lpfc_hba
*phba
;
535 struct lpfc_nodelist
*ndlp
;
536 LIST_HEAD(abort_list
);
537 struct lpfc_sli_ring
*pring
;
538 struct lpfc_iocbq
*wqe
, *next_wqe
;
540 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
541 vport
= lport
->vport
;
544 if (vport
->load_flag
& FC_UNLOADING
)
547 ndlp
= lpfc_findnode_did(vport
, pnvme_rport
->port_id
);
549 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
550 "6049 Could not find node for DID %x\n",
551 pnvme_rport
->port_id
);
555 /* Expand print to include key fields. */
556 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_ABTS
,
557 "6040 ENTER. lport %p, rport %p lsreq %p rqstlen:%d "
558 "rsplen:%d %pad %pad\n",
559 pnvme_lport
, pnvme_rport
,
560 pnvme_lsreq
, pnvme_lsreq
->rqstlen
,
561 pnvme_lsreq
->rsplen
, &pnvme_lsreq
->rqstdma
,
562 &pnvme_lsreq
->rspdma
);
565 * Lock the ELS ring txcmplq and build a local list of all ELS IOs
566 * that need an ABTS. The IOs need to stay on the txcmplq so that
567 * the abort operation completes them successfully.
569 pring
= phba
->sli4_hba
.nvmels_wq
->pring
;
570 spin_lock_irq(&phba
->hbalock
);
571 spin_lock(&pring
->ring_lock
);
572 list_for_each_entry_safe(wqe
, next_wqe
, &pring
->txcmplq
, list
) {
573 /* Add to abort_list on on NDLP match. */
574 if (lpfc_check_sli_ndlp(phba
, pring
, wqe
, ndlp
)) {
575 wqe
->iocb_flag
|= LPFC_DRIVER_ABORTED
;
576 list_add_tail(&wqe
->dlist
, &abort_list
);
579 spin_unlock(&pring
->ring_lock
);
580 spin_unlock_irq(&phba
->hbalock
);
582 /* Abort the targeted IOs and remove them from the abort list. */
583 list_for_each_entry_safe(wqe
, next_wqe
, &abort_list
, dlist
) {
584 spin_lock_irq(&phba
->hbalock
);
585 list_del_init(&wqe
->dlist
);
586 lpfc_sli_issue_abort_iotag(phba
, pring
, wqe
);
587 spin_unlock_irq(&phba
->hbalock
);
591 /* Fix up the existing sgls for NVME IO. */
593 lpfc_nvme_adj_fcp_sgls(struct lpfc_vport
*vport
,
594 struct lpfc_nvme_buf
*lpfc_ncmd
,
595 struct nvmefc_fcp_req
*nCmd
)
597 struct sli4_sge
*sgl
;
598 union lpfc_wqe128
*wqe
;
599 uint32_t *wptr
, *dptr
;
602 * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to
603 * match NVME. NVME sends 96 bytes. Also, use the
604 * nvme commands command and response dma addresses
605 * rather than the virtual memory to ease the restore
608 sgl
= lpfc_ncmd
->nvme_sgl
;
609 sgl
->sge_len
= cpu_to_le32(nCmd
->cmdlen
);
613 /* Setup the physical region for the FCP RSP */
614 sgl
->addr_hi
= cpu_to_le32(putPaddrHigh(nCmd
->rspdma
));
615 sgl
->addr_lo
= cpu_to_le32(putPaddrLow(nCmd
->rspdma
));
616 sgl
->word2
= le32_to_cpu(sgl
->word2
);
618 bf_set(lpfc_sli4_sge_last
, sgl
, 0);
620 bf_set(lpfc_sli4_sge_last
, sgl
, 1);
621 sgl
->word2
= cpu_to_le32(sgl
->word2
);
622 sgl
->sge_len
= cpu_to_le32(nCmd
->rsplen
);
625 * Get a local pointer to the built-in wqe and correct
626 * the cmd size to match NVME's 96 bytes and fix
630 /* 128 byte wqe support here */
631 wqe
= (union lpfc_wqe128
*)&lpfc_ncmd
->cur_iocbq
.wqe
;
633 /* Word 0-2 - NVME CMND IU (embedded payload) */
634 wqe
->generic
.bde
.tus
.f
.bdeFlags
= BUFF_TYPE_BDE_IMMED
;
635 wqe
->generic
.bde
.tus
.f
.bdeSize
= 60;
636 wqe
->generic
.bde
.addrHigh
= 0;
637 wqe
->generic
.bde
.addrLow
= 64; /* Word 16 */
640 bf_set(payload_offset_len
, &wqe
->fcp_icmd
,
641 (nCmd
->rsplen
+ nCmd
->cmdlen
));
644 bf_set(wqe_nvme
, &wqe
->fcp_icmd
.wqe_com
, 1);
645 bf_set(wqe_wqes
, &wqe
->fcp_icmd
.wqe_com
, 1);
648 * Embed the payload in the last half of the WQE
649 * WQE words 16-30 get the NVME CMD IU payload
651 * WQE words 16-19 get payload Words 1-4
652 * WQE words 20-21 get payload Words 6-7
653 * WQE words 22-29 get payload Words 16-23
655 wptr
= &wqe
->words
[16]; /* WQE ptr */
656 dptr
= (uint32_t *)nCmd
->cmdaddr
; /* payload ptr */
657 dptr
++; /* Skip Word 0 in payload */
659 *wptr
++ = *dptr
++; /* Word 1 */
660 *wptr
++ = *dptr
++; /* Word 2 */
661 *wptr
++ = *dptr
++; /* Word 3 */
662 *wptr
++ = *dptr
++; /* Word 4 */
663 dptr
++; /* Skip Word 5 in payload */
664 *wptr
++ = *dptr
++; /* Word 6 */
665 *wptr
++ = *dptr
++; /* Word 7 */
666 dptr
+= 8; /* Skip Words 8-15 in payload */
667 *wptr
++ = *dptr
++; /* Word 16 */
668 *wptr
++ = *dptr
++; /* Word 17 */
669 *wptr
++ = *dptr
++; /* Word 18 */
670 *wptr
++ = *dptr
++; /* Word 19 */
671 *wptr
++ = *dptr
++; /* Word 20 */
672 *wptr
++ = *dptr
++; /* Word 21 */
673 *wptr
++ = *dptr
++; /* Word 22 */
674 *wptr
= *dptr
; /* Word 23 */
677 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
679 lpfc_nvme_ktime(struct lpfc_hba
*phba
,
680 struct lpfc_nvme_buf
*lpfc_ncmd
)
682 uint64_t seg1
, seg2
, seg3
, seg4
;
685 if (!lpfc_ncmd
->ts_last_cmd
||
686 !lpfc_ncmd
->ts_cmd_start
||
687 !lpfc_ncmd
->ts_cmd_wqput
||
688 !lpfc_ncmd
->ts_isr_cmpl
||
689 !lpfc_ncmd
->ts_data_nvme
)
692 if (lpfc_ncmd
->ts_data_nvme
< lpfc_ncmd
->ts_cmd_start
)
694 if (lpfc_ncmd
->ts_cmd_start
< lpfc_ncmd
->ts_last_cmd
)
696 if (lpfc_ncmd
->ts_cmd_wqput
< lpfc_ncmd
->ts_cmd_start
)
698 if (lpfc_ncmd
->ts_isr_cmpl
< lpfc_ncmd
->ts_cmd_wqput
)
700 if (lpfc_ncmd
->ts_data_nvme
< lpfc_ncmd
->ts_isr_cmpl
)
703 * Segment 1 - Time from Last FCP command cmpl is handed
704 * off to NVME Layer to start of next command.
705 * Segment 2 - Time from Driver receives a IO cmd start
706 * from NVME Layer to WQ put is done on IO cmd.
707 * Segment 3 - Time from Driver WQ put is done on IO cmd
708 * to MSI-X ISR for IO cmpl.
709 * Segment 4 - Time from MSI-X ISR for IO cmpl to when
710 * cmpl is handled off to the NVME Layer.
712 seg1
= lpfc_ncmd
->ts_cmd_start
- lpfc_ncmd
->ts_last_cmd
;
713 if (seg1
> 5000000) /* 5 ms - for sequential IOs only */
716 /* Calculate times relative to start of IO */
717 seg2
= (lpfc_ncmd
->ts_cmd_wqput
- lpfc_ncmd
->ts_cmd_start
);
719 seg3
= lpfc_ncmd
->ts_isr_cmpl
- lpfc_ncmd
->ts_cmd_start
;
725 seg4
= lpfc_ncmd
->ts_data_nvme
- lpfc_ncmd
->ts_cmd_start
;
730 phba
->ktime_data_samples
++;
731 phba
->ktime_seg1_total
+= seg1
;
732 if (seg1
< phba
->ktime_seg1_min
)
733 phba
->ktime_seg1_min
= seg1
;
734 else if (seg1
> phba
->ktime_seg1_max
)
735 phba
->ktime_seg1_max
= seg1
;
736 phba
->ktime_seg2_total
+= seg2
;
737 if (seg2
< phba
->ktime_seg2_min
)
738 phba
->ktime_seg2_min
= seg2
;
739 else if (seg2
> phba
->ktime_seg2_max
)
740 phba
->ktime_seg2_max
= seg2
;
741 phba
->ktime_seg3_total
+= seg3
;
742 if (seg3
< phba
->ktime_seg3_min
)
743 phba
->ktime_seg3_min
= seg3
;
744 else if (seg3
> phba
->ktime_seg3_max
)
745 phba
->ktime_seg3_max
= seg3
;
746 phba
->ktime_seg4_total
+= seg4
;
747 if (seg4
< phba
->ktime_seg4_min
)
748 phba
->ktime_seg4_min
= seg4
;
749 else if (seg4
> phba
->ktime_seg4_max
)
750 phba
->ktime_seg4_max
= seg4
;
752 lpfc_ncmd
->ts_last_cmd
= 0;
753 lpfc_ncmd
->ts_cmd_start
= 0;
754 lpfc_ncmd
->ts_cmd_wqput
= 0;
755 lpfc_ncmd
->ts_isr_cmpl
= 0;
756 lpfc_ncmd
->ts_data_nvme
= 0;
761 * lpfc_nvme_io_cmd_wqe_cmpl - Complete an NVME-over-FCP IO
762 * @lpfc_pnvme: Pointer to the driver's nvme instance data
763 * @lpfc_nvme_lport: Pointer to the driver's local port data
764 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
766 * Driver registers this routine as it io request handler. This
767 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
768 * data structure to the rport indicated in @lpfc_nvme_rport.
772 * TODO: What are the failure codes.
775 lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba
*phba
, struct lpfc_iocbq
*pwqeIn
,
776 struct lpfc_wcqe_complete
*wcqe
)
778 struct lpfc_nvme_buf
*lpfc_ncmd
=
779 (struct lpfc_nvme_buf
*)pwqeIn
->context1
;
780 struct lpfc_vport
*vport
= pwqeIn
->vport
;
781 struct nvmefc_fcp_req
*nCmd
;
782 struct nvme_fc_ersp_iu
*ep
;
783 struct nvme_fc_cmd_iu
*cp
;
784 struct lpfc_nvme_rport
*rport
;
785 struct lpfc_nodelist
*ndlp
;
786 struct lpfc_nvme_fcpreq_priv
*freqpriv
;
789 uint16_t cid
, sqhd
, data
;
792 /* Sanity check on return of outstanding command */
793 if (!lpfc_ncmd
|| !lpfc_ncmd
->nvmeCmd
|| !lpfc_ncmd
->nrport
) {
794 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
795 "6071 Completion pointers bad on wqe %p.\n",
799 atomic_inc(&phba
->fc4NvmeIoCmpls
);
801 nCmd
= lpfc_ncmd
->nvmeCmd
;
802 rport
= lpfc_ncmd
->nrport
;
804 lpfc_nvmeio_data(phba
, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n",
805 lpfc_ncmd
->cur_iocbq
.sli4_xritag
,
806 bf_get(lpfc_wcqe_c_status
, wcqe
), wcqe
->parameter
);
808 * Catch race where our node has transitioned, but the
809 * transport is still transitioning.
812 if (!ndlp
|| !NLP_CHK_NODE_ACT(ndlp
)) {
813 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
814 "6061 rport %p, DID x%06x node not ready.\n",
815 rport
, rport
->remoteport
->port_id
);
817 ndlp
= lpfc_findnode_did(vport
, rport
->remoteport
->port_id
);
819 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
820 "6062 Ignoring NVME cmpl. No ndlp\n");
825 code
= bf_get(lpfc_wcqe_c_code
, wcqe
);
826 if (code
== CQE_CODE_NVME_ERSP
) {
827 /* For this type of CQE, we need to rebuild the rsp */
828 ep
= (struct nvme_fc_ersp_iu
*)nCmd
->rspaddr
;
831 * Get Command Id from cmd to plug into response. This
832 * code is not needed in the next NVME Transport drop.
834 cp
= (struct nvme_fc_cmd_iu
*)nCmd
->cmdaddr
;
835 cid
= cp
->sqe
.common
.command_id
;
838 * RSN is in CQE word 2
839 * SQHD is in CQE Word 3 bits 15:0
840 * Cmd Specific info is in CQE Word 1
841 * and in CQE Word 0 bits 15:0
843 sqhd
= bf_get(lpfc_wcqe_c_sqhead
, wcqe
);
845 /* Now lets build the NVME ERSP IU */
846 ep
->iu_len
= cpu_to_be16(8);
847 ep
->rsn
= wcqe
->parameter
;
848 ep
->xfrd_len
= cpu_to_be32(nCmd
->payload_length
);
850 ptr
= (uint32_t *)&ep
->cqe
.result
.u64
;
851 *ptr
++ = wcqe
->total_data_placed
;
852 data
= bf_get(lpfc_wcqe_c_ersp0
, wcqe
);
853 *ptr
= (uint32_t)data
;
854 ep
->cqe
.sq_head
= sqhd
;
855 ep
->cqe
.sq_id
= nCmd
->sqid
;
856 ep
->cqe
.command_id
= cid
;
859 lpfc_ncmd
->status
= IOSTAT_SUCCESS
;
860 lpfc_ncmd
->result
= 0;
861 nCmd
->rcv_rsplen
= LPFC_NVME_ERSP_LEN
;
862 nCmd
->transferred_length
= nCmd
->payload_length
;
864 lpfc_ncmd
->status
= (bf_get(lpfc_wcqe_c_status
, wcqe
) &
865 LPFC_IOCB_STATUS_MASK
);
866 lpfc_ncmd
->result
= (wcqe
->parameter
& IOERR_PARAM_MASK
);
868 /* For NVME, the only failure path that results in an
869 * IO error is when the adapter rejects it. All other
870 * conditions are a success case and resolved by the
872 * IOSTAT_FCP_RSP_ERROR means:
873 * 1. Length of data received doesn't match total
874 * transfer length in WQE
875 * 2. If the RSP payload does NOT match these cases:
876 * a. RSP length 12/24 bytes and all zeros
879 switch (lpfc_ncmd
->status
) {
881 nCmd
->transferred_length
= wcqe
->total_data_placed
;
882 nCmd
->rcv_rsplen
= 0;
885 case IOSTAT_FCP_RSP_ERROR
:
886 nCmd
->transferred_length
= wcqe
->total_data_placed
;
887 nCmd
->rcv_rsplen
= wcqe
->parameter
;
890 if (nCmd
->rcv_rsplen
== LPFC_NVME_ERSP_LEN
)
892 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
893 "6081 NVME Completion Protocol Error: "
894 "xri %x status x%x result x%x "
896 lpfc_ncmd
->cur_iocbq
.sli4_xritag
,
897 lpfc_ncmd
->status
, lpfc_ncmd
->result
,
898 wcqe
->total_data_placed
);
900 case IOSTAT_LOCAL_REJECT
:
901 /* Let fall through to set command final state. */
902 if (lpfc_ncmd
->result
== IOERR_ABORT_REQUESTED
)
903 lpfc_printf_vlog(vport
, KERN_INFO
,
905 "6032 Delay Aborted cmd %p "
906 "nvme cmd %p, xri x%x, "
909 lpfc_ncmd
->cur_iocbq
.sli4_xritag
,
910 bf_get(lpfc_wcqe_c_xb
, wcqe
));
913 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_IOERR
,
914 "6072 NVME Completion Error: xri %x "
915 "status x%x result x%x placed x%x\n",
916 lpfc_ncmd
->cur_iocbq
.sli4_xritag
,
917 lpfc_ncmd
->status
, lpfc_ncmd
->result
,
918 wcqe
->total_data_placed
);
919 nCmd
->transferred_length
= 0;
920 nCmd
->rcv_rsplen
= 0;
921 nCmd
->status
= NVME_SC_INTERNAL
;
925 /* pick up SLI4 exhange busy condition */
926 if (bf_get(lpfc_wcqe_c_xb
, wcqe
))
927 lpfc_ncmd
->flags
|= LPFC_SBUF_XBUSY
;
929 lpfc_ncmd
->flags
&= ~LPFC_SBUF_XBUSY
;
931 if (ndlp
&& NLP_CHK_NODE_ACT(ndlp
))
932 atomic_dec(&ndlp
->cmd_pending
);
934 /* Update stats and complete the IO. There is
935 * no need for dma unprep because the nvme_transport
936 * owns the dma address.
938 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
939 if (lpfc_ncmd
->ts_cmd_start
) {
940 lpfc_ncmd
->ts_isr_cmpl
= pwqeIn
->isr_timestamp
;
941 lpfc_ncmd
->ts_data_nvme
= ktime_get_ns();
942 phba
->ktime_last_cmd
= lpfc_ncmd
->ts_data_nvme
;
943 lpfc_nvme_ktime(phba
, lpfc_ncmd
);
945 if (phba
->cpucheck_on
& LPFC_CHECK_NVME_IO
) {
946 if (lpfc_ncmd
->cpu
!= smp_processor_id())
947 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
948 "6701 CPU Check cmpl: "
949 "cpu %d expect %d\n",
950 smp_processor_id(), lpfc_ncmd
->cpu
);
951 if (lpfc_ncmd
->cpu
< LPFC_CHECK_CPU_CNT
)
952 phba
->cpucheck_cmpl_io
[lpfc_ncmd
->cpu
]++;
955 freqpriv
= nCmd
->private;
956 freqpriv
->nvme_buf
= NULL
;
958 /* NVME targets need completion held off until the abort exchange
959 * completes unless the NVME Rport is getting unregistered.
961 if (!(lpfc_ncmd
->flags
& LPFC_SBUF_XBUSY
) ||
962 ndlp
->upcall_flags
& NLP_WAIT_FOR_UNREG
) {
963 /* Clear the XBUSY flag to prevent double completions.
964 * The nvme rport is getting unregistered and there is
965 * no need to defer the IO.
967 if (lpfc_ncmd
->flags
& LPFC_SBUF_XBUSY
)
968 lpfc_ncmd
->flags
&= ~LPFC_SBUF_XBUSY
;
973 spin_lock_irqsave(&phba
->hbalock
, flags
);
974 lpfc_ncmd
->nrport
= NULL
;
975 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
977 /* Call release with XB=1 to queue the IO into the abort list. */
978 lpfc_release_nvme_buf(phba
, lpfc_ncmd
);
983 * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO
984 * @lpfc_pnvme: Pointer to the driver's nvme instance data
985 * @lpfc_nvme_lport: Pointer to the driver's local port data
986 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
987 * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
988 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
990 * Driver registers this routine as it io request handler. This
991 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
992 * data structure to the rport indicated in @lpfc_nvme_rport.
996 * TODO: What are the failure codes.
999 lpfc_nvme_prep_io_cmd(struct lpfc_vport
*vport
,
1000 struct lpfc_nvme_buf
*lpfc_ncmd
,
1001 struct lpfc_nodelist
*pnode
)
1003 struct lpfc_hba
*phba
= vport
->phba
;
1004 struct nvmefc_fcp_req
*nCmd
= lpfc_ncmd
->nvmeCmd
;
1005 struct lpfc_iocbq
*pwqeq
= &(lpfc_ncmd
->cur_iocbq
);
1006 union lpfc_wqe128
*wqe
= (union lpfc_wqe128
*)&pwqeq
->wqe
;
1009 if (!pnode
|| !NLP_CHK_NODE_ACT(pnode
))
1013 * There are three possibilities here - use scatter-gather segment, use
1014 * the single mapping, or neither.
1016 wqe
->fcp_iwrite
.initial_xfer_len
= 0;
1018 if (nCmd
->io_dir
== NVMEFC_FCP_WRITE
) {
1020 if ((phba
->cfg_nvme_enable_fb
) &&
1021 (pnode
->nlp_flag
& NLP_FIRSTBURST
)) {
1022 req_len
= lpfc_ncmd
->nvmeCmd
->payload_length
;
1023 if (req_len
< pnode
->nvme_fb_size
)
1024 wqe
->fcp_iwrite
.initial_xfer_len
=
1027 wqe
->fcp_iwrite
.initial_xfer_len
=
1028 pnode
->nvme_fb_size
;
1032 bf_set(wqe_cmnd
, &wqe
->generic
.wqe_com
,
1033 CMD_FCP_IWRITE64_WQE
);
1034 bf_set(wqe_pu
, &wqe
->generic
.wqe_com
,
1038 bf_set(wqe_qosd
, &wqe
->fcp_iwrite
.wqe_com
, 0);
1039 bf_set(wqe_iod
, &wqe
->fcp_iwrite
.wqe_com
,
1040 LPFC_WQE_IOD_WRITE
);
1041 bf_set(wqe_lenloc
, &wqe
->fcp_iwrite
.wqe_com
,
1042 LPFC_WQE_LENLOC_WORD4
);
1043 if (phba
->cfg_nvme_oas
)
1044 bf_set(wqe_oas
, &wqe
->fcp_iwrite
.wqe_com
, 1);
1047 bf_set(wqe_cmd_type
, &wqe
->generic
.wqe_com
,
1050 atomic_inc(&phba
->fc4NvmeOutputRequests
);
1053 bf_set(wqe_cmnd
, &wqe
->generic
.wqe_com
,
1054 CMD_FCP_IREAD64_WQE
);
1055 bf_set(wqe_pu
, &wqe
->generic
.wqe_com
,
1059 bf_set(wqe_qosd
, &wqe
->fcp_iread
.wqe_com
, 0);
1060 bf_set(wqe_iod
, &wqe
->fcp_iread
.wqe_com
,
1062 bf_set(wqe_lenloc
, &wqe
->fcp_iread
.wqe_com
,
1063 LPFC_WQE_LENLOC_WORD4
);
1064 if (phba
->cfg_nvme_oas
)
1065 bf_set(wqe_oas
, &wqe
->fcp_iread
.wqe_com
, 1);
1068 bf_set(wqe_cmd_type
, &wqe
->generic
.wqe_com
,
1071 atomic_inc(&phba
->fc4NvmeInputRequests
);
1075 wqe
->fcp_icmd
.rsrvd4
= 0;
1078 bf_set(wqe_cmnd
, &wqe
->generic
.wqe_com
, CMD_FCP_ICMND64_WQE
);
1079 bf_set(wqe_pu
, &wqe
->generic
.wqe_com
, 0);
1082 bf_set(wqe_qosd
, &wqe
->fcp_icmd
.wqe_com
, 1);
1083 bf_set(wqe_iod
, &wqe
->fcp_icmd
.wqe_com
, LPFC_WQE_IOD_WRITE
);
1084 bf_set(wqe_lenloc
, &wqe
->fcp_icmd
.wqe_com
,
1085 LPFC_WQE_LENLOC_NONE
);
1086 if (phba
->cfg_nvme_oas
)
1087 bf_set(wqe_oas
, &wqe
->fcp_icmd
.wqe_com
, 1);
1090 bf_set(wqe_cmd_type
, &wqe
->generic
.wqe_com
, NVME_READ_CMD
);
1092 atomic_inc(&phba
->fc4NvmeControlRequests
);
1095 * Finish initializing those WQE fields that are independent
1096 * of the nvme_cmnd request_buffer
1100 bf_set(wqe_ctxt_tag
, &wqe
->generic
.wqe_com
,
1101 phba
->sli4_hba
.rpi_ids
[pnode
->nlp_rpi
]);
1102 bf_set(wqe_xri_tag
, &wqe
->generic
.wqe_com
, pwqeq
->sli4_xritag
);
1105 /* Preserve Class data in the ndlp. */
1106 bf_set(wqe_class
, &wqe
->generic
.wqe_com
,
1107 (pnode
->nlp_fcp_info
& 0x0f));
1110 wqe
->generic
.wqe_com
.abort_tag
= pwqeq
->iotag
;
1113 bf_set(wqe_reqtag
, &wqe
->generic
.wqe_com
, pwqeq
->iotag
);
1116 bf_set(wqe_cqid
, &wqe
->generic
.wqe_com
, LPFC_WQE_CQ_ID_DEFAULT
);
1118 pwqeq
->vport
= vport
;
1124 * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO
1125 * @lpfc_pnvme: Pointer to the driver's nvme instance data
1126 * @lpfc_nvme_lport: Pointer to the driver's local port data
1127 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1128 * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1129 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1131 * Driver registers this routine as it io request handler. This
1132 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1133 * data structure to the rport indicated in @lpfc_nvme_rport.
1137 * TODO: What are the failure codes.
1140 lpfc_nvme_prep_io_dma(struct lpfc_vport
*vport
,
1141 struct lpfc_nvme_buf
*lpfc_ncmd
)
1143 struct lpfc_hba
*phba
= vport
->phba
;
1144 struct nvmefc_fcp_req
*nCmd
= lpfc_ncmd
->nvmeCmd
;
1145 union lpfc_wqe128
*wqe
= (union lpfc_wqe128
*)&lpfc_ncmd
->cur_iocbq
.wqe
;
1146 struct sli4_sge
*sgl
= lpfc_ncmd
->nvme_sgl
;
1147 struct scatterlist
*data_sg
;
1148 struct sli4_sge
*first_data_sgl
;
1149 dma_addr_t physaddr
;
1150 uint32_t num_bde
= 0;
1152 uint32_t dma_offset
= 0;
1155 /* Fix up the command and response DMA stuff. */
1156 lpfc_nvme_adj_fcp_sgls(vport
, lpfc_ncmd
, nCmd
);
1159 * There are three possibilities here - use scatter-gather segment, use
1160 * the single mapping, or neither.
1164 * Jump over the cmd and rsp SGEs. The fix routine
1165 * has already adjusted for this.
1169 first_data_sgl
= sgl
;
1170 lpfc_ncmd
->seg_cnt
= nCmd
->sg_cnt
;
1171 if (lpfc_ncmd
->seg_cnt
> phba
->cfg_nvme_seg_cnt
+ 1) {
1172 lpfc_printf_log(phba
, KERN_ERR
, LOG_NVME_IOERR
,
1173 "6058 Too many sg segments from "
1174 "NVME Transport. Max %d, "
1175 "nvmeIO sg_cnt %d\n",
1176 phba
->cfg_nvme_seg_cnt
+ 1,
1177 lpfc_ncmd
->seg_cnt
);
1178 lpfc_ncmd
->seg_cnt
= 0;
1183 * The driver established a maximum scatter-gather segment count
1184 * during probe that limits the number of sg elements in any
1185 * single nvme command. Just run through the seg_cnt and format
1188 nseg
= nCmd
->sg_cnt
;
1189 data_sg
= nCmd
->first_sgl
;
1190 for (i
= 0; i
< nseg
; i
++) {
1191 if (data_sg
== NULL
) {
1192 lpfc_printf_log(phba
, KERN_ERR
, LOG_NVME_IOERR
,
1193 "6059 dptr err %d, nseg %d\n",
1195 lpfc_ncmd
->seg_cnt
= 0;
1198 physaddr
= data_sg
->dma_address
;
1199 dma_len
= data_sg
->length
;
1200 sgl
->addr_lo
= cpu_to_le32(putPaddrLow(physaddr
));
1201 sgl
->addr_hi
= cpu_to_le32(putPaddrHigh(physaddr
));
1202 sgl
->word2
= le32_to_cpu(sgl
->word2
);
1203 if ((num_bde
+ 1) == nseg
)
1204 bf_set(lpfc_sli4_sge_last
, sgl
, 1);
1206 bf_set(lpfc_sli4_sge_last
, sgl
, 0);
1207 bf_set(lpfc_sli4_sge_offset
, sgl
, dma_offset
);
1208 bf_set(lpfc_sli4_sge_type
, sgl
, LPFC_SGE_TYPE_DATA
);
1209 sgl
->word2
= cpu_to_le32(sgl
->word2
);
1210 sgl
->sge_len
= cpu_to_le32(dma_len
);
1212 dma_offset
+= dma_len
;
1213 data_sg
= sg_next(data_sg
);
1217 /* For this clause to be valid, the payload_length
1218 * and sg_cnt must zero.
1220 if (nCmd
->payload_length
!= 0) {
1221 lpfc_printf_log(phba
, KERN_ERR
, LOG_NVME_IOERR
,
1222 "6063 NVME DMA Prep Err: sg_cnt %d "
1223 "payload_length x%x\n",
1224 nCmd
->sg_cnt
, nCmd
->payload_length
);
1230 * Due to difference in data length between DIF/non-DIF paths,
1231 * we need to set word 4 of WQE here
1233 wqe
->fcp_iread
.total_xfer_len
= nCmd
->payload_length
;
1238 * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO
1239 * @lpfc_pnvme: Pointer to the driver's nvme instance data
1240 * @lpfc_nvme_lport: Pointer to the driver's local port data
1241 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1242 * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1243 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1245 * Driver registers this routine as it io request handler. This
1246 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1247 * data structure to the rport
1248 indicated in @lpfc_nvme_rport.
1252 * TODO: What are the failure codes.
1255 lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port
*pnvme_lport
,
1256 struct nvme_fc_remote_port
*pnvme_rport
,
1257 void *hw_queue_handle
,
1258 struct nvmefc_fcp_req
*pnvme_fcreq
)
1261 struct lpfc_nvme_lport
*lport
;
1262 struct lpfc_vport
*vport
;
1263 struct lpfc_hba
*phba
;
1264 struct lpfc_nodelist
*ndlp
;
1265 struct lpfc_nvme_buf
*lpfc_ncmd
;
1266 struct lpfc_nvme_rport
*rport
;
1267 struct lpfc_nvme_qhandle
*lpfc_queue_info
;
1268 struct lpfc_nvme_fcpreq_priv
*freqpriv
= pnvme_fcreq
->private;
1269 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1273 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
1274 vport
= lport
->vport
;
1277 if (vport
->load_flag
& FC_UNLOADING
) {
1282 if (vport
->load_flag
& FC_UNLOADING
) {
1287 /* Validate pointers. */
1288 if (!pnvme_lport
|| !pnvme_rport
|| !freqpriv
) {
1289 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_IOERR
| LOG_NODE
,
1290 "6117 No Send:IO submit ptrs NULL, lport %p, "
1291 "rport %p fcreq_priv %p\n",
1292 pnvme_lport
, pnvme_rport
, freqpriv
);
1297 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1299 start
= ktime_get_ns();
1301 rport
= (struct lpfc_nvme_rport
*)pnvme_rport
->private;
1302 lpfc_queue_info
= (struct lpfc_nvme_qhandle
*)hw_queue_handle
;
1305 * Catch race where our node has transitioned, but the
1306 * transport is still transitioning.
1309 if (!ndlp
|| !NLP_CHK_NODE_ACT(ndlp
)) {
1310 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
1311 "6053 rport %p, ndlp %p, DID x%06x "
1312 "ndlp not ready.\n",
1313 rport
, ndlp
, pnvme_rport
->port_id
);
1315 ndlp
= lpfc_findnode_did(vport
, pnvme_rport
->port_id
);
1317 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
1318 "6066 Missing node for DID %x\n",
1319 pnvme_rport
->port_id
);
1325 /* The remote node has to be a mapped target or it's an error. */
1326 if ((ndlp
->nlp_type
& NLP_NVME_TARGET
) &&
1327 (ndlp
->nlp_state
!= NLP_STE_MAPPED_NODE
)) {
1328 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
1329 "6036 rport %p, DID x%06x not ready for "
1330 "IO. State x%x, Type x%x\n",
1331 rport
, pnvme_rport
->port_id
,
1332 ndlp
->nlp_state
, ndlp
->nlp_type
);
1338 /* The node is shared with FCP IO, make sure the IO pending count does
1339 * not exceed the programmed depth.
1341 if (atomic_read(&ndlp
->cmd_pending
) >= ndlp
->cmd_qdepth
) {
1346 lpfc_ncmd
= lpfc_get_nvme_buf(phba
, ndlp
);
1347 if (lpfc_ncmd
== NULL
) {
1348 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_IOERR
,
1349 "6065 driver's buffer pool is empty, "
1354 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1356 lpfc_ncmd
->ts_cmd_start
= start
;
1357 lpfc_ncmd
->ts_last_cmd
= phba
->ktime_last_cmd
;
1359 lpfc_ncmd
->ts_cmd_start
= 0;
1364 * Store the data needed by the driver to issue, abort, and complete
1366 * Do not let the IO hang out forever. There is no midlayer issuing
1367 * an abort so inform the FW of the maximum IO pending time.
1369 freqpriv
->nvme_buf
= lpfc_ncmd
;
1370 lpfc_ncmd
->nvmeCmd
= pnvme_fcreq
;
1371 lpfc_ncmd
->nrport
= rport
;
1372 lpfc_ncmd
->ndlp
= ndlp
;
1373 lpfc_ncmd
->start_time
= jiffies
;
1375 lpfc_nvme_prep_io_cmd(vport
, lpfc_ncmd
, ndlp
);
1376 ret
= lpfc_nvme_prep_io_dma(vport
, lpfc_ncmd
);
1379 goto out_free_nvme_buf
;
1382 atomic_inc(&ndlp
->cmd_pending
);
1385 * Issue the IO on the WQ indicated by index in the hw_queue_handle.
1386 * This identfier was create in our hardware queue create callback
1387 * routine. The driver now is dependent on the IO queue steering from
1388 * the transport. We are trusting the upper NVME layers know which
1389 * index to use and that they have affinitized a CPU to this hardware
1390 * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ.
1392 lpfc_ncmd
->cur_iocbq
.hba_wqidx
= lpfc_queue_info
->index
;
1394 lpfc_nvmeio_data(phba
, "NVME FCP XMIT: xri x%x idx %d to %06x\n",
1395 lpfc_ncmd
->cur_iocbq
.sli4_xritag
,
1396 lpfc_queue_info
->index
, ndlp
->nlp_DID
);
1398 ret
= lpfc_sli4_issue_wqe(phba
, LPFC_FCP_RING
, &lpfc_ncmd
->cur_iocbq
);
1400 atomic_dec(&ndlp
->cmd_pending
);
1401 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_IOERR
,
1402 "6113 FCP could not issue WQE err %x "
1403 "sid: x%x did: x%x oxid: x%x\n",
1404 ret
, vport
->fc_myDID
, ndlp
->nlp_DID
,
1405 lpfc_ncmd
->cur_iocbq
.sli4_xritag
);
1406 goto out_free_nvme_buf
;
1409 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1410 if (lpfc_ncmd
->ts_cmd_start
)
1411 lpfc_ncmd
->ts_cmd_wqput
= ktime_get_ns();
1413 if (phba
->cpucheck_on
& LPFC_CHECK_NVME_IO
) {
1414 lpfc_ncmd
->cpu
= smp_processor_id();
1415 if (lpfc_ncmd
->cpu
!= lpfc_queue_info
->index
) {
1416 /* Check for admin queue */
1417 if (lpfc_queue_info
->qidx
) {
1418 lpfc_printf_vlog(vport
,
1419 KERN_ERR
, LOG_NVME_IOERR
,
1420 "6702 CPU Check cmd: "
1423 lpfc_queue_info
->index
);
1425 lpfc_ncmd
->cpu
= lpfc_queue_info
->index
;
1427 if (lpfc_ncmd
->cpu
< LPFC_CHECK_CPU_CNT
)
1428 phba
->cpucheck_xmt_io
[lpfc_ncmd
->cpu
]++;
1434 if (lpfc_ncmd
->nvmeCmd
->sg_cnt
) {
1435 if (lpfc_ncmd
->nvmeCmd
->io_dir
== NVMEFC_FCP_WRITE
)
1436 atomic_dec(&phba
->fc4NvmeOutputRequests
);
1438 atomic_dec(&phba
->fc4NvmeInputRequests
);
1440 atomic_dec(&phba
->fc4NvmeControlRequests
);
1441 lpfc_release_nvme_buf(phba
, lpfc_ncmd
);
1447 * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request.
1448 * @phba: Pointer to HBA context object
1449 * @cmdiocb: Pointer to command iocb object.
1450 * @rspiocb: Pointer to response iocb object.
1452 * This is the callback function for any NVME FCP IO that was aborted.
1458 lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba
*phba
, struct lpfc_iocbq
*cmdiocb
,
1459 struct lpfc_wcqe_complete
*abts_cmpl
)
1461 lpfc_printf_log(phba
, KERN_INFO
, LOG_NVME
,
1462 "6145 ABORT_XRI_CN completing on rpi x%x "
1463 "original iotag x%x, abort cmd iotag x%x "
1464 "req_tag x%x, status x%x, hwstatus x%x\n",
1465 cmdiocb
->iocb
.un
.acxri
.abortContextTag
,
1466 cmdiocb
->iocb
.un
.acxri
.abortIoTag
,
1468 bf_get(lpfc_wcqe_c_request_tag
, abts_cmpl
),
1469 bf_get(lpfc_wcqe_c_status
, abts_cmpl
),
1470 bf_get(lpfc_wcqe_c_hw_status
, abts_cmpl
));
1471 lpfc_sli_release_iocbq(phba
, cmdiocb
);
1475 * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS
1476 * @lpfc_pnvme: Pointer to the driver's nvme instance data
1477 * @lpfc_nvme_lport: Pointer to the driver's local port data
1478 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1479 * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1480 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1482 * Driver registers this routine as its nvme request io abort handler. This
1483 * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq
1484 * data structure to the rport indicated in @lpfc_nvme_rport. This routine
1485 * is executed asynchronously - one the target is validated as "MAPPED" and
1486 * ready for IO, the driver issues the abort request and returns.
1492 lpfc_nvme_fcp_abort(struct nvme_fc_local_port
*pnvme_lport
,
1493 struct nvme_fc_remote_port
*pnvme_rport
,
1494 void *hw_queue_handle
,
1495 struct nvmefc_fcp_req
*pnvme_fcreq
)
1497 struct lpfc_nvme_lport
*lport
;
1498 struct lpfc_vport
*vport
;
1499 struct lpfc_hba
*phba
;
1500 struct lpfc_nvme_rport
*rport
;
1501 struct lpfc_nvme_buf
*lpfc_nbuf
;
1502 struct lpfc_iocbq
*abts_buf
;
1503 struct lpfc_iocbq
*nvmereq_wqe
;
1504 struct lpfc_nvme_fcpreq_priv
*freqpriv
= pnvme_fcreq
->private;
1505 union lpfc_wqe
*abts_wqe
;
1506 unsigned long flags
;
1509 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
1510 rport
= (struct lpfc_nvme_rport
*)pnvme_rport
->private;
1511 vport
= lport
->vport
;
1514 if (vport
->load_flag
& FC_UNLOADING
)
1517 /* Announce entry to new IO submit field. */
1518 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_ABTS
,
1519 "6002 Abort Request to rport DID x%06x "
1520 "for nvme_fc_req %p\n",
1521 pnvme_rport
->port_id
,
1524 /* If the hba is getting reset, this flag is set. It is
1525 * cleared when the reset is complete and rings reestablished.
1527 spin_lock_irqsave(&phba
->hbalock
, flags
);
1528 /* driver queued commands are in process of being flushed */
1529 if (phba
->hba_flag
& HBA_NVME_IOQ_FLUSH
) {
1530 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1531 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1532 "6139 Driver in reset cleanup - flushing "
1533 "NVME Req now. hba_flag x%x\n",
1538 lpfc_nbuf
= freqpriv
->nvme_buf
;
1540 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1541 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1542 "6140 NVME IO req has no matching lpfc nvme "
1543 "io buffer. Skipping abort req.\n");
1545 } else if (!lpfc_nbuf
->nvmeCmd
) {
1546 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1547 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1548 "6141 lpfc NVME IO req has no nvme_fcreq "
1549 "io buffer. Skipping abort req.\n");
1552 nvmereq_wqe
= &lpfc_nbuf
->cur_iocbq
;
1555 * The lpfc_nbuf and the mapped nvme_fcreq in the driver's
1556 * state must match the nvme_fcreq passed by the nvme
1557 * transport. If they don't match, it is likely the driver
1558 * has already completed the NVME IO and the nvme transport
1559 * has not seen it yet.
1561 if (lpfc_nbuf
->nvmeCmd
!= pnvme_fcreq
) {
1562 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1563 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1564 "6143 NVME req mismatch: "
1565 "lpfc_nbuf %p nvmeCmd %p, "
1566 "pnvme_fcreq %p. Skipping Abort xri x%x\n",
1567 lpfc_nbuf
, lpfc_nbuf
->nvmeCmd
,
1568 pnvme_fcreq
, nvmereq_wqe
->sli4_xritag
);
1572 /* Don't abort IOs no longer on the pending queue. */
1573 if (!(nvmereq_wqe
->iocb_flag
& LPFC_IO_ON_TXCMPLQ
)) {
1574 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1575 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1576 "6142 NVME IO req %p not queued - skipping "
1577 "abort req xri x%x\n",
1578 pnvme_fcreq
, nvmereq_wqe
->sli4_xritag
);
1582 lpfc_nvmeio_data(phba
, "NVME FCP ABORT: xri x%x idx %d to %06x\n",
1583 nvmereq_wqe
->sli4_xritag
,
1584 nvmereq_wqe
->hba_wqidx
, pnvme_rport
->port_id
);
1586 /* Outstanding abort is in progress */
1587 if (nvmereq_wqe
->iocb_flag
& LPFC_DRIVER_ABORTED
) {
1588 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1589 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1590 "6144 Outstanding NVME I/O Abort Request "
1591 "still pending on nvme_fcreq %p, "
1592 "lpfc_ncmd %p xri x%x\n",
1593 pnvme_fcreq
, lpfc_nbuf
,
1594 nvmereq_wqe
->sli4_xritag
);
1598 abts_buf
= __lpfc_sli_get_iocbq(phba
);
1600 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1601 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1602 "6136 No available abort wqes. Skipping "
1603 "Abts req for nvme_fcreq %p xri x%x\n",
1604 pnvme_fcreq
, nvmereq_wqe
->sli4_xritag
);
1608 /* Ready - mark outstanding as aborted by driver. */
1609 nvmereq_wqe
->iocb_flag
|= LPFC_DRIVER_ABORTED
;
1611 /* Complete prepping the abort wqe and issue to the FW. */
1612 abts_wqe
= &abts_buf
->wqe
;
1614 /* WQEs are reused. Clear stale data and set key fields to
1615 * zero like ia, iaab, iaar, xri_tag, and ctxt_tag.
1617 memset(abts_wqe
, 0, sizeof(union lpfc_wqe
));
1618 bf_set(abort_cmd_criteria
, &abts_wqe
->abort_cmd
, T_XRI_TAG
);
1621 bf_set(wqe_ct
, &abts_wqe
->abort_cmd
.wqe_com
, 0);
1622 bf_set(wqe_cmnd
, &abts_wqe
->abort_cmd
.wqe_com
, CMD_ABORT_XRI_CX
);
1623 bf_set(wqe_class
, &abts_wqe
->abort_cmd
.wqe_com
,
1624 nvmereq_wqe
->iocb
.ulpClass
);
1626 /* word 8 - tell the FW to abort the IO associated with this
1627 * outstanding exchange ID.
1629 abts_wqe
->abort_cmd
.wqe_com
.abort_tag
= nvmereq_wqe
->sli4_xritag
;
1631 /* word 9 - this is the iotag for the abts_wqe completion. */
1632 bf_set(wqe_reqtag
, &abts_wqe
->abort_cmd
.wqe_com
,
1636 bf_set(wqe_wqid
, &abts_wqe
->abort_cmd
.wqe_com
, nvmereq_wqe
->hba_wqidx
);
1637 bf_set(wqe_qosd
, &abts_wqe
->abort_cmd
.wqe_com
, 1);
1638 bf_set(wqe_lenloc
, &abts_wqe
->abort_cmd
.wqe_com
, LPFC_WQE_LENLOC_NONE
);
1641 bf_set(wqe_cmd_type
, &abts_wqe
->abort_cmd
.wqe_com
, OTHER_COMMAND
);
1642 bf_set(wqe_wqec
, &abts_wqe
->abort_cmd
.wqe_com
, 1);
1643 bf_set(wqe_cqid
, &abts_wqe
->abort_cmd
.wqe_com
, LPFC_WQE_CQ_ID_DEFAULT
);
1645 /* ABTS WQE must go to the same WQ as the WQE to be aborted */
1646 abts_buf
->iocb_flag
|= LPFC_IO_NVME
;
1647 abts_buf
->hba_wqidx
= nvmereq_wqe
->hba_wqidx
;
1648 abts_buf
->vport
= vport
;
1649 abts_buf
->wqe_cmpl
= lpfc_nvme_abort_fcreq_cmpl
;
1650 ret_val
= lpfc_sli4_issue_wqe(phba
, LPFC_FCP_RING
, abts_buf
);
1651 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1653 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1654 "6137 Failed abts issue_wqe with status x%x "
1655 "for nvme_fcreq %p.\n",
1656 ret_val
, pnvme_fcreq
);
1657 lpfc_sli_release_iocbq(phba
, abts_buf
);
1661 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_ABTS
,
1662 "6138 Transport Abort NVME Request Issued for "
1663 "ox_id x%x on reqtag x%x\n",
1664 nvmereq_wqe
->sli4_xritag
,
1668 /* Declare and initialization an instance of the FC NVME template. */
1669 static struct nvme_fc_port_template lpfc_nvme_template
= {
1670 /* initiator-based functions */
1671 .localport_delete
= lpfc_nvme_localport_delete
,
1672 .remoteport_delete
= lpfc_nvme_remoteport_delete
,
1673 .create_queue
= lpfc_nvme_create_queue
,
1674 .delete_queue
= lpfc_nvme_delete_queue
,
1675 .ls_req
= lpfc_nvme_ls_req
,
1676 .fcp_io
= lpfc_nvme_fcp_io_submit
,
1677 .ls_abort
= lpfc_nvme_ls_abort
,
1678 .fcp_abort
= lpfc_nvme_fcp_abort
,
1681 .max_sgl_segments
= LPFC_NVME_DEFAULT_SEGS
,
1682 .max_dif_sgl_segments
= LPFC_NVME_DEFAULT_SEGS
,
1683 .dma_boundary
= 0xFFFFFFFF,
1685 /* Sizes of additional private data for data structures.
1686 * No use for the last two sizes at this time.
1688 .local_priv_sz
= sizeof(struct lpfc_nvme_lport
),
1689 .remote_priv_sz
= sizeof(struct lpfc_nvme_rport
),
1690 .lsrqst_priv_sz
= 0,
1691 .fcprqst_priv_sz
= sizeof(struct lpfc_nvme_fcpreq_priv
),
1695 * lpfc_sli4_post_nvme_sgl_block - post a block of nvme sgl list to firmware
1696 * @phba: pointer to lpfc hba data structure.
1697 * @nblist: pointer to nvme buffer list.
1698 * @count: number of scsi buffers on the list.
1700 * This routine is invoked to post a block of @count scsi sgl pages from a
1701 * SCSI buffer list @nblist to the HBA using non-embedded mailbox command.
1706 lpfc_sli4_post_nvme_sgl_block(struct lpfc_hba
*phba
,
1707 struct list_head
*nblist
,
1710 struct lpfc_nvme_buf
*lpfc_ncmd
;
1711 struct lpfc_mbx_post_uembed_sgl_page1
*sgl
;
1712 struct sgl_page_pairs
*sgl_pg_pairs
;
1715 uint32_t reqlen
, alloclen
, pg_pairs
;
1717 uint16_t xritag_start
= 0;
1719 uint32_t shdr_status
, shdr_add_status
;
1720 dma_addr_t pdma_phys_bpl1
;
1721 union lpfc_sli4_cfg_shdr
*shdr
;
1723 /* Calculate the requested length of the dma memory */
1724 reqlen
= count
* sizeof(struct sgl_page_pairs
) +
1725 sizeof(union lpfc_sli4_cfg_shdr
) + sizeof(uint32_t);
1726 if (reqlen
> SLI4_PAGE_SIZE
) {
1727 lpfc_printf_log(phba
, KERN_WARNING
, LOG_INIT
,
1728 "6118 Block sgl registration required DMA "
1729 "size (%d) great than a page\n", reqlen
);
1732 mbox
= mempool_alloc(phba
->mbox_mem_pool
, GFP_KERNEL
);
1734 lpfc_printf_log(phba
, KERN_ERR
, LOG_INIT
,
1735 "6119 Failed to allocate mbox cmd memory\n");
1739 /* Allocate DMA memory and set up the non-embedded mailbox command */
1740 alloclen
= lpfc_sli4_config(phba
, mbox
, LPFC_MBOX_SUBSYSTEM_FCOE
,
1741 LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES
, reqlen
,
1742 LPFC_SLI4_MBX_NEMBED
);
1744 if (alloclen
< reqlen
) {
1745 lpfc_printf_log(phba
, KERN_ERR
, LOG_INIT
,
1746 "6120 Allocated DMA memory size (%d) is "
1747 "less than the requested DMA memory "
1748 "size (%d)\n", alloclen
, reqlen
);
1749 lpfc_sli4_mbox_cmd_free(phba
, mbox
);
1753 /* Get the first SGE entry from the non-embedded DMA memory */
1754 viraddr
= mbox
->sge_array
->addr
[0];
1756 /* Set up the SGL pages in the non-embedded DMA pages */
1757 sgl
= (struct lpfc_mbx_post_uembed_sgl_page1
*)viraddr
;
1758 sgl_pg_pairs
= &sgl
->sgl_pg_pairs
;
1761 list_for_each_entry(lpfc_ncmd
, nblist
, list
) {
1762 /* Set up the sge entry */
1763 sgl_pg_pairs
->sgl_pg0_addr_lo
=
1764 cpu_to_le32(putPaddrLow(lpfc_ncmd
->dma_phys_sgl
));
1765 sgl_pg_pairs
->sgl_pg0_addr_hi
=
1766 cpu_to_le32(putPaddrHigh(lpfc_ncmd
->dma_phys_sgl
));
1767 if (phba
->cfg_sg_dma_buf_size
> SGL_PAGE_SIZE
)
1768 pdma_phys_bpl1
= lpfc_ncmd
->dma_phys_sgl
+
1772 sgl_pg_pairs
->sgl_pg1_addr_lo
=
1773 cpu_to_le32(putPaddrLow(pdma_phys_bpl1
));
1774 sgl_pg_pairs
->sgl_pg1_addr_hi
=
1775 cpu_to_le32(putPaddrHigh(pdma_phys_bpl1
));
1776 /* Keep the first xritag on the list */
1778 xritag_start
= lpfc_ncmd
->cur_iocbq
.sli4_xritag
;
1782 bf_set(lpfc_post_sgl_pages_xri
, sgl
, xritag_start
);
1783 bf_set(lpfc_post_sgl_pages_xricnt
, sgl
, pg_pairs
);
1784 /* Perform endian conversion if necessary */
1785 sgl
->word0
= cpu_to_le32(sgl
->word0
);
1787 if (!phba
->sli4_hba
.intr_enable
)
1788 rc
= lpfc_sli_issue_mbox(phba
, mbox
, MBX_POLL
);
1790 mbox_tmo
= lpfc_mbox_tmo_val(phba
, mbox
);
1791 rc
= lpfc_sli_issue_mbox_wait(phba
, mbox
, mbox_tmo
);
1793 shdr
= (union lpfc_sli4_cfg_shdr
*)&sgl
->cfg_shdr
;
1794 shdr_status
= bf_get(lpfc_mbox_hdr_status
, &shdr
->response
);
1795 shdr_add_status
= bf_get(lpfc_mbox_hdr_add_status
, &shdr
->response
);
1796 if (rc
!= MBX_TIMEOUT
)
1797 lpfc_sli4_mbox_cmd_free(phba
, mbox
);
1798 if (shdr_status
|| shdr_add_status
|| rc
) {
1799 lpfc_printf_log(phba
, KERN_ERR
, LOG_SLI
,
1800 "6125 POST_SGL_BLOCK mailbox command failed "
1801 "status x%x add_status x%x mbx status x%x\n",
1802 shdr_status
, shdr_add_status
, rc
);
1809 * lpfc_post_nvme_sgl_list - Post blocks of nvme buffer sgls from a list
1810 * @phba: pointer to lpfc hba data structure.
1811 * @post_nblist: pointer to the nvme buffer list.
1813 * This routine walks a list of nvme buffers that was passed in. It attempts
1814 * to construct blocks of nvme buffer sgls which contains contiguous xris and
1815 * uses the non-embedded SGL block post mailbox commands to post to the port.
1816 * For single NVME buffer sgl with non-contiguous xri, if any, it shall use
1817 * embedded SGL post mailbox command for posting. The @post_nblist passed in
1818 * must be local list, thus no lock is needed when manipulate the list.
1820 * Returns: 0 = failure, non-zero number of successfully posted buffers.
1823 lpfc_post_nvme_sgl_list(struct lpfc_hba
*phba
,
1824 struct list_head
*post_nblist
, int sb_count
)
1826 struct lpfc_nvme_buf
*lpfc_ncmd
, *lpfc_ncmd_next
;
1827 int status
, sgl_size
;
1828 int post_cnt
= 0, block_cnt
= 0, num_posting
= 0, num_posted
= 0;
1829 dma_addr_t pdma_phys_sgl1
;
1830 int last_xritag
= NO_XRI
;
1832 LIST_HEAD(prep_nblist
);
1833 LIST_HEAD(blck_nblist
);
1834 LIST_HEAD(nvme_nblist
);
1840 sgl_size
= phba
->cfg_sg_dma_buf_size
;
1842 list_for_each_entry_safe(lpfc_ncmd
, lpfc_ncmd_next
, post_nblist
, list
) {
1843 list_del_init(&lpfc_ncmd
->list
);
1845 if ((last_xritag
!= NO_XRI
) &&
1846 (lpfc_ncmd
->cur_iocbq
.sli4_xritag
!= last_xritag
+ 1)) {
1847 /* a hole in xri block, form a sgl posting block */
1848 list_splice_init(&prep_nblist
, &blck_nblist
);
1849 post_cnt
= block_cnt
- 1;
1850 /* prepare list for next posting block */
1851 list_add_tail(&lpfc_ncmd
->list
, &prep_nblist
);
1854 /* prepare list for next posting block */
1855 list_add_tail(&lpfc_ncmd
->list
, &prep_nblist
);
1856 /* enough sgls for non-embed sgl mbox command */
1857 if (block_cnt
== LPFC_NEMBED_MBOX_SGL_CNT
) {
1858 list_splice_init(&prep_nblist
, &blck_nblist
);
1859 post_cnt
= block_cnt
;
1864 last_xritag
= lpfc_ncmd
->cur_iocbq
.sli4_xritag
;
1866 /* end of repost sgl list condition for NVME buffers */
1867 if (num_posting
== sb_count
) {
1868 if (post_cnt
== 0) {
1869 /* last sgl posting block */
1870 list_splice_init(&prep_nblist
, &blck_nblist
);
1871 post_cnt
= block_cnt
;
1872 } else if (block_cnt
== 1) {
1873 /* last single sgl with non-contiguous xri */
1874 if (sgl_size
> SGL_PAGE_SIZE
)
1876 lpfc_ncmd
->dma_phys_sgl
+
1880 cur_xritag
= lpfc_ncmd
->cur_iocbq
.sli4_xritag
;
1881 status
= lpfc_sli4_post_sgl(phba
,
1882 lpfc_ncmd
->dma_phys_sgl
,
1883 pdma_phys_sgl1
, cur_xritag
);
1885 /* failure, put on abort nvme list */
1886 lpfc_ncmd
->flags
|= LPFC_SBUF_XBUSY
;
1888 /* success, put on NVME buffer list */
1889 lpfc_ncmd
->flags
&= ~LPFC_SBUF_XBUSY
;
1890 lpfc_ncmd
->status
= IOSTAT_SUCCESS
;
1893 /* success, put on NVME buffer sgl list */
1894 list_add_tail(&lpfc_ncmd
->list
, &nvme_nblist
);
1898 /* continue until a nembed page worth of sgls */
1902 /* post block of NVME buffer list sgls */
1903 status
= lpfc_sli4_post_nvme_sgl_block(phba
, &blck_nblist
,
1906 /* don't reset xirtag due to hole in xri block */
1908 last_xritag
= NO_XRI
;
1910 /* reset NVME buffer post count for next round of posting */
1913 /* put posted NVME buffer-sgl posted on NVME buffer sgl list */
1914 while (!list_empty(&blck_nblist
)) {
1915 list_remove_head(&blck_nblist
, lpfc_ncmd
,
1916 struct lpfc_nvme_buf
, list
);
1918 /* failure, put on abort nvme list */
1919 lpfc_ncmd
->flags
|= LPFC_SBUF_XBUSY
;
1921 /* success, put on NVME buffer list */
1922 lpfc_ncmd
->flags
&= ~LPFC_SBUF_XBUSY
;
1923 lpfc_ncmd
->status
= IOSTAT_SUCCESS
;
1926 list_add_tail(&lpfc_ncmd
->list
, &nvme_nblist
);
1929 /* Push NVME buffers with sgl posted to the available list */
1930 while (!list_empty(&nvme_nblist
)) {
1931 list_remove_head(&nvme_nblist
, lpfc_ncmd
,
1932 struct lpfc_nvme_buf
, list
);
1933 lpfc_release_nvme_buf(phba
, lpfc_ncmd
);
1939 * lpfc_repost_nvme_sgl_list - Repost all the allocated nvme buffer sgls
1940 * @phba: pointer to lpfc hba data structure.
1942 * This routine walks the list of nvme buffers that have been allocated and
1943 * repost them to the port by using SGL block post. This is needed after a
1944 * pci_function_reset/warm_start or start. The lpfc_hba_down_post_s4 routine
1945 * is responsible for moving all nvme buffers on the lpfc_abts_nvme_sgl_list
1946 * to the lpfc_nvme_buf_list. If the repost fails, reject all nvme buffers.
1948 * Returns: 0 = success, non-zero failure.
1951 lpfc_repost_nvme_sgl_list(struct lpfc_hba
*phba
)
1953 LIST_HEAD(post_nblist
);
1954 int num_posted
, rc
= 0;
1956 /* get all NVME buffers need to repost to a local list */
1957 spin_lock_irq(&phba
->nvme_buf_list_get_lock
);
1958 spin_lock(&phba
->nvme_buf_list_put_lock
);
1959 list_splice_init(&phba
->lpfc_nvme_buf_list_get
, &post_nblist
);
1960 list_splice(&phba
->lpfc_nvme_buf_list_put
, &post_nblist
);
1961 spin_unlock(&phba
->nvme_buf_list_put_lock
);
1962 spin_unlock_irq(&phba
->nvme_buf_list_get_lock
);
1964 /* post the list of nvme buffer sgls to port if available */
1965 if (!list_empty(&post_nblist
)) {
1966 num_posted
= lpfc_post_nvme_sgl_list(phba
, &post_nblist
,
1967 phba
->sli4_hba
.nvme_xri_cnt
);
1968 /* failed to post any nvme buffer, return error */
1969 if (num_posted
== 0)
1976 * lpfc_new_nvme_buf - Scsi buffer allocator for HBA with SLI4 IF spec
1977 * @vport: The virtual port for which this call being executed.
1978 * @num_to_allocate: The requested number of buffers to allocate.
1980 * This routine allocates nvme buffers for device with SLI-4 interface spec,
1981 * the nvme buffer contains all the necessary information needed to initiate
1982 * a NVME I/O. After allocating up to @num_to_allocate NVME buffers and put
1983 * them on a list, it post them to the port by using SGL block post.
1986 * int - number of nvme buffers that were allocated and posted.
1987 * 0 = failure, less than num_to_alloc is a partial failure.
1990 lpfc_new_nvme_buf(struct lpfc_vport
*vport
, int num_to_alloc
)
1992 struct lpfc_hba
*phba
= vport
->phba
;
1993 struct lpfc_nvme_buf
*lpfc_ncmd
;
1994 struct lpfc_iocbq
*pwqeq
;
1995 union lpfc_wqe128
*wqe
;
1996 struct sli4_sge
*sgl
;
1997 dma_addr_t pdma_phys_sgl
;
1998 uint16_t iotag
, lxri
= 0;
1999 int bcnt
, num_posted
, sgl_size
;
2000 LIST_HEAD(prep_nblist
);
2001 LIST_HEAD(post_nblist
);
2002 LIST_HEAD(nvme_nblist
);
2004 sgl_size
= phba
->cfg_sg_dma_buf_size
;
2006 for (bcnt
= 0; bcnt
< num_to_alloc
; bcnt
++) {
2007 lpfc_ncmd
= kzalloc(sizeof(struct lpfc_nvme_buf
), GFP_KERNEL
);
2011 * Get memory from the pci pool to map the virt space to
2012 * pci bus space for an I/O. The DMA buffer includes the
2013 * number of SGE's necessary to support the sg_tablesize.
2015 lpfc_ncmd
->data
= dma_pool_zalloc(phba
->lpfc_sg_dma_buf_pool
,
2017 &lpfc_ncmd
->dma_handle
);
2018 if (!lpfc_ncmd
->data
) {
2023 lxri
= lpfc_sli4_next_xritag(phba
);
2024 if (lxri
== NO_XRI
) {
2025 dma_pool_free(phba
->lpfc_sg_dma_buf_pool
,
2026 lpfc_ncmd
->data
, lpfc_ncmd
->dma_handle
);
2030 pwqeq
= &(lpfc_ncmd
->cur_iocbq
);
2031 wqe
= (union lpfc_wqe128
*)&pwqeq
->wqe
;
2033 /* Allocate iotag for lpfc_ncmd->cur_iocbq. */
2034 iotag
= lpfc_sli_next_iotag(phba
, pwqeq
);
2036 dma_pool_free(phba
->lpfc_sg_dma_buf_pool
,
2037 lpfc_ncmd
->data
, lpfc_ncmd
->dma_handle
);
2039 lpfc_printf_log(phba
, KERN_ERR
, LOG_NVME_IOERR
,
2040 "6121 Failed to allocated IOTAG for"
2041 " XRI:0x%x\n", lxri
);
2042 lpfc_sli4_free_xri(phba
, lxri
);
2045 pwqeq
->sli4_lxritag
= lxri
;
2046 pwqeq
->sli4_xritag
= phba
->sli4_hba
.xri_ids
[lxri
];
2047 pwqeq
->iocb_flag
|= LPFC_IO_NVME
;
2048 pwqeq
->context1
= lpfc_ncmd
;
2049 pwqeq
->wqe_cmpl
= lpfc_nvme_io_cmd_wqe_cmpl
;
2051 /* Initialize local short-hand pointers. */
2052 lpfc_ncmd
->nvme_sgl
= lpfc_ncmd
->data
;
2053 sgl
= lpfc_ncmd
->nvme_sgl
;
2054 pdma_phys_sgl
= lpfc_ncmd
->dma_handle
;
2055 lpfc_ncmd
->dma_phys_sgl
= pdma_phys_sgl
;
2057 /* Rsp SGE will be filled in when we rcv an IO
2058 * from the NVME Layer to be sent.
2059 * The cmd is going to be embedded so we need a SKIP SGE.
2061 bf_set(lpfc_sli4_sge_type
, sgl
, LPFC_SGE_TYPE_SKIP
);
2062 bf_set(lpfc_sli4_sge_last
, sgl
, 0);
2063 sgl
->word2
= cpu_to_le32(sgl
->word2
);
2064 /* Fill in word 3 / sgl_len during cmd submission */
2066 lpfc_ncmd
->cur_iocbq
.context1
= lpfc_ncmd
;
2069 bf_set(wqe_erp
, &wqe
->generic
.wqe_com
, 0);
2070 /* NVME upper layers will time things out, if needed */
2071 bf_set(wqe_tmo
, &wqe
->generic
.wqe_com
, 0);
2074 bf_set(wqe_ebde_cnt
, &wqe
->generic
.wqe_com
, 0);
2075 bf_set(wqe_dbde
, &wqe
->generic
.wqe_com
, 1);
2077 /* add the nvme buffer to a post list */
2078 list_add_tail(&lpfc_ncmd
->list
, &post_nblist
);
2079 spin_lock_irq(&phba
->nvme_buf_list_get_lock
);
2080 phba
->sli4_hba
.nvme_xri_cnt
++;
2081 spin_unlock_irq(&phba
->nvme_buf_list_get_lock
);
2083 lpfc_printf_log(phba
, KERN_INFO
, LOG_NVME
,
2084 "6114 Allocate %d out of %d requested new NVME "
2085 "buffers\n", bcnt
, num_to_alloc
);
2087 /* post the list of nvme buffer sgls to port if available */
2088 if (!list_empty(&post_nblist
))
2089 num_posted
= lpfc_post_nvme_sgl_list(phba
,
2090 &post_nblist
, bcnt
);
2098 * lpfc_get_nvme_buf - Get a nvme buffer from lpfc_nvme_buf_list of the HBA
2099 * @phba: The HBA for which this call is being executed.
2101 * This routine removes a nvme buffer from head of @phba lpfc_nvme_buf_list list
2102 * and returns to caller.
2106 * Pointer to lpfc_nvme_buf - Success
2108 static struct lpfc_nvme_buf
*
2109 lpfc_get_nvme_buf(struct lpfc_hba
*phba
, struct lpfc_nodelist
*ndlp
)
2111 struct lpfc_nvme_buf
*lpfc_ncmd
, *lpfc_ncmd_next
;
2112 unsigned long iflag
= 0;
2115 spin_lock_irqsave(&phba
->nvme_buf_list_get_lock
, iflag
);
2116 list_for_each_entry_safe(lpfc_ncmd
, lpfc_ncmd_next
,
2117 &phba
->lpfc_nvme_buf_list_get
, list
) {
2118 list_del_init(&lpfc_ncmd
->list
);
2123 spin_lock(&phba
->nvme_buf_list_put_lock
);
2124 list_splice(&phba
->lpfc_nvme_buf_list_put
,
2125 &phba
->lpfc_nvme_buf_list_get
);
2126 INIT_LIST_HEAD(&phba
->lpfc_nvme_buf_list_put
);
2127 spin_unlock(&phba
->nvme_buf_list_put_lock
);
2128 list_for_each_entry_safe(lpfc_ncmd
, lpfc_ncmd_next
,
2129 &phba
->lpfc_nvme_buf_list_get
, list
) {
2130 list_del_init(&lpfc_ncmd
->list
);
2135 spin_unlock_irqrestore(&phba
->nvme_buf_list_get_lock
, iflag
);
2142 * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list.
2143 * @phba: The Hba for which this call is being executed.
2144 * @lpfc_ncmd: The nvme buffer which is being released.
2146 * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba
2147 * lpfc_nvme_buf_list list. For SLI4 XRI's are tied to the nvme buffer
2148 * and cannot be reused for at least RA_TOV amount of time if it was
2152 lpfc_release_nvme_buf(struct lpfc_hba
*phba
, struct lpfc_nvme_buf
*lpfc_ncmd
)
2154 unsigned long iflag
= 0;
2156 lpfc_ncmd
->nonsg_phys
= 0;
2157 if (lpfc_ncmd
->flags
& LPFC_SBUF_XBUSY
) {
2158 lpfc_printf_log(phba
, KERN_INFO
, LOG_NVME_ABTS
,
2159 "6310 XB release deferred for "
2160 "ox_id x%x on reqtag x%x\n",
2161 lpfc_ncmd
->cur_iocbq
.sli4_xritag
,
2162 lpfc_ncmd
->cur_iocbq
.iotag
);
2164 spin_lock_irqsave(&phba
->sli4_hba
.abts_nvme_buf_list_lock
,
2166 list_add_tail(&lpfc_ncmd
->list
,
2167 &phba
->sli4_hba
.lpfc_abts_nvme_buf_list
);
2168 spin_unlock_irqrestore(&phba
->sli4_hba
.abts_nvme_buf_list_lock
,
2171 lpfc_ncmd
->nvmeCmd
= NULL
;
2172 lpfc_ncmd
->cur_iocbq
.iocb_flag
= LPFC_IO_NVME
;
2173 spin_lock_irqsave(&phba
->nvme_buf_list_put_lock
, iflag
);
2174 list_add_tail(&lpfc_ncmd
->list
, &phba
->lpfc_nvme_buf_list_put
);
2175 spin_unlock_irqrestore(&phba
->nvme_buf_list_put_lock
, iflag
);
2180 * lpfc_nvme_create_localport - Create/Bind an nvme localport instance.
2181 * @pvport - the lpfc_vport instance requesting a localport.
2183 * This routine is invoked to create an nvme localport instance to bind
2184 * to the nvme_fc_transport. It is called once during driver load
2185 * like lpfc_create_shost after all other services are initialized.
2186 * It requires a vport, vpi, and wwns at call time. Other localport
2187 * parameters are modified as the driver's FCID and the Fabric WWN
2192 * -ENOMEM - no heap memory available
2193 * other values - from nvme registration upcall
2196 lpfc_nvme_create_localport(struct lpfc_vport
*vport
)
2199 struct lpfc_hba
*phba
= vport
->phba
;
2200 struct nvme_fc_port_info nfcp_info
;
2201 struct nvme_fc_local_port
*localport
;
2202 struct lpfc_nvme_lport
*lport
;
2205 /* Initialize this localport instance. The vport wwn usage ensures
2206 * that NPIV is accounted for.
2208 memset(&nfcp_info
, 0, sizeof(struct nvme_fc_port_info
));
2209 nfcp_info
.port_role
= FC_PORT_ROLE_NVME_INITIATOR
;
2210 nfcp_info
.node_name
= wwn_to_u64(vport
->fc_nodename
.u
.wwn
);
2211 nfcp_info
.port_name
= wwn_to_u64(vport
->fc_portname
.u
.wwn
);
2213 /* Limit to LPFC_MAX_NVME_SEG_CNT.
2214 * For now need + 1 to get around NVME transport logic.
2216 if (phba
->cfg_sg_seg_cnt
> LPFC_MAX_NVME_SEG_CNT
) {
2217 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
| LOG_INIT
,
2218 "6300 Reducing sg segment cnt to %d\n",
2219 LPFC_MAX_NVME_SEG_CNT
);
2220 phba
->cfg_nvme_seg_cnt
= LPFC_MAX_NVME_SEG_CNT
;
2222 phba
->cfg_nvme_seg_cnt
= phba
->cfg_sg_seg_cnt
;
2224 lpfc_nvme_template
.max_sgl_segments
= phba
->cfg_nvme_seg_cnt
+ 1;
2225 lpfc_nvme_template
.max_hw_queues
= phba
->cfg_nvme_io_channel
;
2227 /* localport is allocated from the stack, but the registration
2228 * call allocates heap memory as well as the private area.
2230 #if (IS_ENABLED(CONFIG_NVME_FC))
2231 ret
= nvme_fc_register_localport(&nfcp_info
, &lpfc_nvme_template
,
2232 &vport
->phba
->pcidev
->dev
, &localport
);
2237 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
| LOG_NVME_DISC
,
2238 "6005 Successfully registered local "
2239 "NVME port num %d, localP %p, private %p, "
2241 localport
->port_num
, localport
,
2243 lpfc_nvme_template
.max_sgl_segments
);
2245 /* Private is our lport size declared in the template. */
2246 lport
= (struct lpfc_nvme_lport
*)localport
->private;
2247 vport
->localport
= localport
;
2248 lport
->vport
= vport
;
2249 vport
->nvmei_support
= 1;
2251 /* Don't post more new bufs if repost already recovered
2254 if (phba
->sli4_hba
.nvme_xri_cnt
== 0) {
2255 len
= lpfc_new_nvme_buf(vport
,
2256 phba
->sli4_hba
.nvme_xri_max
);
2257 vport
->phba
->total_nvme_bufs
+= len
;
2264 /* lpfc_nvme_lport_unreg_wait - Wait for the host to complete an lport unreg.
2266 * The driver has to wait for the host nvme transport to callback
2267 * indicating the localport has successfully unregistered all
2268 * resources. Since this is an uninterruptible wait, loop every ten
2269 * seconds and print a message indicating no progress.
2271 * An uninterruptible wait is used because of the risk of transport-to-
2272 * driver state mismatch.
2275 lpfc_nvme_lport_unreg_wait(struct lpfc_vport
*vport
,
2276 struct lpfc_nvme_lport
*lport
)
2278 #if (IS_ENABLED(CONFIG_NVME_FC))
2282 /* Host transport has to clean up and confirm requiring an indefinite
2283 * wait. Print a message if a 10 second wait expires and renew the
2284 * wait. This is unexpected.
2286 wait_tmo
= msecs_to_jiffies(LPFC_NVME_WAIT_TMO
* 1000);
2288 ret
= wait_for_completion_timeout(&lport
->lport_unreg_done
,
2290 if (unlikely(!ret
)) {
2291 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
2292 "6176 Lport %p Localport %p wait "
2293 "timed out. Renewing.\n",
2294 lport
, vport
->localport
);
2299 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_IOERR
,
2300 "6177 Lport %p Localport %p Complete Success\n",
2301 lport
, vport
->localport
);
2306 * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport.
2307 * @pnvme: pointer to lpfc nvme data structure.
2309 * This routine is invoked to destroy all lports bound to the phba.
2310 * The lport memory was allocated by the nvme fc transport and is
2311 * released there. This routine ensures all rports bound to the
2312 * lport have been disconnected.
2316 lpfc_nvme_destroy_localport(struct lpfc_vport
*vport
)
2318 #if (IS_ENABLED(CONFIG_NVME_FC))
2319 struct nvme_fc_local_port
*localport
;
2320 struct lpfc_nvme_lport
*lport
;
2323 if (vport
->nvmei_support
== 0)
2326 localport
= vport
->localport
;
2327 vport
->localport
= NULL
;
2328 lport
= (struct lpfc_nvme_lport
*)localport
->private;
2330 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
,
2331 "6011 Destroying NVME localport %p\n",
2334 /* lport's rport list is clear. Unregister
2335 * lport and release resources.
2337 init_completion(&lport
->lport_unreg_done
);
2338 ret
= nvme_fc_unregister_localport(localport
);
2340 /* Wait for completion. This either blocks
2341 * indefinitely or succeeds
2343 lpfc_nvme_lport_unreg_wait(vport
, lport
);
2345 /* Regardless of the unregister upcall response, clear
2346 * nvmei_support. All rports are unregistered and the
2347 * driver will clean up.
2349 vport
->nvmei_support
= 0;
2351 lpfc_printf_vlog(vport
,
2352 KERN_INFO
, LOG_NVME_DISC
,
2353 "6009 Unregistered lport Success\n");
2355 lpfc_printf_vlog(vport
,
2356 KERN_INFO
, LOG_NVME_DISC
,
2357 "6010 Unregistered lport "
2358 "Failed, status x%x\n",
2365 lpfc_nvme_update_localport(struct lpfc_vport
*vport
)
2367 #if (IS_ENABLED(CONFIG_NVME_FC))
2368 struct nvme_fc_local_port
*localport
;
2369 struct lpfc_nvme_lport
*lport
;
2371 localport
= vport
->localport
;
2373 lpfc_printf_vlog(vport
, KERN_WARNING
, LOG_NVME
,
2374 "6710 Update NVME fail. No localport\n");
2377 lport
= (struct lpfc_nvme_lport
*)localport
->private;
2379 lpfc_printf_vlog(vport
, KERN_WARNING
, LOG_NVME
,
2380 "6171 Update NVME fail. localP %p, No lport\n",
2384 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
,
2385 "6012 Update NVME lport %p did x%x\n",
2386 localport
, vport
->fc_myDID
);
2388 localport
->port_id
= vport
->fc_myDID
;
2389 if (localport
->port_id
== 0)
2390 localport
->port_role
= FC_PORT_ROLE_NVME_DISCOVERY
;
2392 localport
->port_role
= FC_PORT_ROLE_NVME_INITIATOR
;
2394 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
2395 "6030 bound lport %p to DID x%06x\n",
2396 lport
, localport
->port_id
);
2401 lpfc_nvme_register_port(struct lpfc_vport
*vport
, struct lpfc_nodelist
*ndlp
)
2403 #if (IS_ENABLED(CONFIG_NVME_FC))
2405 struct nvme_fc_local_port
*localport
;
2406 struct lpfc_nvme_lport
*lport
;
2407 struct lpfc_nvme_rport
*rport
;
2408 struct nvme_fc_remote_port
*remote_port
;
2409 struct nvme_fc_port_info rpinfo
;
2410 struct lpfc_nodelist
*prev_ndlp
;
2412 lpfc_printf_vlog(ndlp
->vport
, KERN_INFO
, LOG_NVME_DISC
,
2413 "6006 Register NVME PORT. DID x%06x nlptype x%x\n",
2414 ndlp
->nlp_DID
, ndlp
->nlp_type
);
2416 localport
= vport
->localport
;
2420 lport
= (struct lpfc_nvme_lport
*)localport
->private;
2422 /* NVME rports are not preserved across devloss.
2423 * Just register this instance. Note, rpinfo->dev_loss_tmo
2424 * is left 0 to indicate accept transport defaults. The
2425 * driver communicates port role capabilities consistent
2426 * with the PRLI response data.
2428 memset(&rpinfo
, 0, sizeof(struct nvme_fc_port_info
));
2429 rpinfo
.port_id
= ndlp
->nlp_DID
;
2430 if (ndlp
->nlp_type
& NLP_NVME_TARGET
)
2431 rpinfo
.port_role
|= FC_PORT_ROLE_NVME_TARGET
;
2432 if (ndlp
->nlp_type
& NLP_NVME_INITIATOR
)
2433 rpinfo
.port_role
|= FC_PORT_ROLE_NVME_INITIATOR
;
2435 if (ndlp
->nlp_type
& NLP_NVME_DISCOVERY
)
2436 rpinfo
.port_role
|= FC_PORT_ROLE_NVME_DISCOVERY
;
2438 rpinfo
.port_name
= wwn_to_u64(ndlp
->nlp_portname
.u
.wwn
);
2439 rpinfo
.node_name
= wwn_to_u64(ndlp
->nlp_nodename
.u
.wwn
);
2440 ret
= nvme_fc_register_remoteport(localport
, &rpinfo
, &remote_port
);
2442 /* If the ndlp already has an nrport, this is just
2443 * a resume of the existing rport. Else this is a
2446 rport
= remote_port
->private;
2448 lpfc_printf_vlog(ndlp
->vport
, KERN_INFO
,
2450 "6014 Rebinding lport to "
2451 "rport wwpn 0x%llx, "
2452 "Data: x%x x%x x%x x%06x\n",
2453 remote_port
->port_name
,
2454 remote_port
->port_id
,
2455 remote_port
->port_role
,
2458 prev_ndlp
= rport
->ndlp
;
2460 /* Sever the ndlp<->rport connection before dropping
2461 * the ndlp ref from register.
2463 ndlp
->nrport
= NULL
;
2469 /* Clean bind the rport to the ndlp. */
2470 rport
->remoteport
= remote_port
;
2471 rport
->lport
= lport
;
2472 rport
->ndlp
= lpfc_nlp_get(ndlp
);
2475 ndlp
->nrport
= rport
;
2476 lpfc_printf_vlog(vport
, KERN_INFO
,
2477 LOG_NVME_DISC
| LOG_NODE
,
2478 "6022 Binding new rport to "
2479 "lport %p Rport WWNN 0x%llx, "
2480 "Rport WWPN 0x%llx DID "
2483 rpinfo
.node_name
, rpinfo
.port_name
,
2484 rpinfo
.port_id
, rpinfo
.port_role
);
2486 lpfc_printf_vlog(vport
, KERN_ERR
,
2487 LOG_NVME_DISC
| LOG_NODE
,
2488 "6031 RemotePort Registration failed "
2489 "err: %d, DID x%06x\n",
2490 ret
, ndlp
->nlp_DID
);
2499 /* lpfc_nvme_rport_unreg_wait - Wait for the host to complete an rport unreg.
2501 * The driver has to wait for the host nvme transport to callback
2502 * indicating the remoteport has successfully unregistered all
2503 * resources. Since this is an uninterruptible wait, loop every ten
2504 * seconds and print a message indicating no progress.
2506 * An uninterruptible wait is used because of the risk of transport-to-
2507 * driver state mismatch.
2510 lpfc_nvme_rport_unreg_wait(struct lpfc_vport
*vport
,
2511 struct lpfc_nvme_rport
*rport
)
2513 #if (IS_ENABLED(CONFIG_NVME_FC))
2517 /* Host transport has to clean up and confirm requiring an indefinite
2518 * wait. Print a message if a 10 second wait expires and renew the
2519 * wait. This is unexpected.
2521 wait_tmo
= msecs_to_jiffies(LPFC_NVME_WAIT_TMO
* 1000);
2523 ret
= wait_for_completion_timeout(&rport
->rport_unreg_done
,
2525 if (unlikely(!ret
)) {
2526 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
2527 "6174 Rport %p Remoteport %p wait "
2528 "timed out. Renewing.\n",
2529 rport
, rport
->remoteport
);
2534 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_IOERR
,
2535 "6175 Rport %p Remoteport %p Complete Success\n",
2536 rport
, rport
->remoteport
);
2540 /* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport.
2542 * There is no notion of Devloss or rport recovery from the current
2543 * nvme_transport perspective. Loss of an rport just means IO cannot
2544 * be sent and recovery is completely up to the initator.
2545 * For now, the driver just unbinds the DID and port_role so that
2546 * no further IO can be issued. Changes are planned for later.
2548 * Notes - the ndlp reference count is not decremented here since
2549 * since there is no nvme_transport api for devloss. Node ref count
2550 * is only adjusted in driver unload.
2553 lpfc_nvme_unregister_port(struct lpfc_vport
*vport
, struct lpfc_nodelist
*ndlp
)
2555 #if (IS_ENABLED(CONFIG_NVME_FC))
2557 struct nvme_fc_local_port
*localport
;
2558 struct lpfc_nvme_lport
*lport
;
2559 struct lpfc_nvme_rport
*rport
;
2560 struct nvme_fc_remote_port
*remoteport
;
2562 localport
= vport
->localport
;
2564 /* This is fundamental error. The localport is always
2565 * available until driver unload. Just exit.
2570 lport
= (struct lpfc_nvme_lport
*)localport
->private;
2574 rport
= ndlp
->nrport
;
2578 remoteport
= rport
->remoteport
;
2579 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
2580 "6033 Unreg nvme remoteport %p, portname x%llx, "
2581 "port_id x%06x, portstate x%x port type x%x\n",
2582 remoteport
, remoteport
->port_name
,
2583 remoteport
->port_id
, remoteport
->port_state
,
2586 /* Sanity check ndlp type. Only call for NVME ports. Don't
2587 * clear any rport state until the transport calls back.
2590 if (ndlp
->nlp_type
& NLP_NVME_TARGET
) {
2591 init_completion(&rport
->rport_unreg_done
);
2593 /* No concern about the role change on the nvme remoteport.
2594 * The transport will update it.
2596 ndlp
->upcall_flags
|= NLP_WAIT_FOR_UNREG
;
2597 ret
= nvme_fc_unregister_remoteport(remoteport
);
2599 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
2600 "6167 NVME unregister failed %d "
2602 ret
, remoteport
->port_state
);
2604 /* Wait for completion. This either blocks
2605 * indefinitely or succeeds
2607 lpfc_nvme_rport_unreg_wait(vport
, rport
);
2608 ndlp
->upcall_flags
&= ~NLP_WAIT_FOR_UNREG
;
2614 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
2615 "6168 State error: lport %p, rport%p FCID x%06x\n",
2616 vport
->localport
, ndlp
->rport
, ndlp
->nlp_DID
);
2620 * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort
2621 * @phba: pointer to lpfc hba data structure.
2622 * @axri: pointer to the fcp xri abort wcqe structure.
2624 * This routine is invoked by the worker thread to process a SLI4 fast-path
2625 * NVME aborted xri. Aborted NVME IO commands are completed to the transport
2629 lpfc_sli4_nvme_xri_aborted(struct lpfc_hba
*phba
,
2630 struct sli4_wcqe_xri_aborted
*axri
)
2632 uint16_t xri
= bf_get(lpfc_wcqe_xa_xri
, axri
);
2633 struct lpfc_nvme_buf
*lpfc_ncmd
, *next_lpfc_ncmd
;
2634 struct nvmefc_fcp_req
*nvme_cmd
= NULL
;
2635 struct lpfc_nodelist
*ndlp
;
2636 unsigned long iflag
= 0;
2638 if (!(phba
->cfg_enable_fc4_type
& LPFC_ENABLE_NVME
))
2640 spin_lock_irqsave(&phba
->hbalock
, iflag
);
2641 spin_lock(&phba
->sli4_hba
.abts_nvme_buf_list_lock
);
2642 list_for_each_entry_safe(lpfc_ncmd
, next_lpfc_ncmd
,
2643 &phba
->sli4_hba
.lpfc_abts_nvme_buf_list
,
2645 if (lpfc_ncmd
->cur_iocbq
.sli4_xritag
== xri
) {
2646 list_del_init(&lpfc_ncmd
->list
);
2647 lpfc_ncmd
->flags
&= ~LPFC_SBUF_XBUSY
;
2648 lpfc_ncmd
->status
= IOSTAT_SUCCESS
;
2650 &phba
->sli4_hba
.abts_nvme_buf_list_lock
);
2652 spin_unlock_irqrestore(&phba
->hbalock
, iflag
);
2653 ndlp
= lpfc_ncmd
->ndlp
;
2655 lpfc_sli4_abts_err_handler(phba
, ndlp
, axri
);
2657 lpfc_printf_log(phba
, KERN_INFO
, LOG_NVME_ABTS
,
2658 "6311 nvme_cmd %p xri x%x tag x%x "
2659 "abort complete and xri released\n",
2660 lpfc_ncmd
->nvmeCmd
, xri
,
2661 lpfc_ncmd
->cur_iocbq
.iotag
);
2663 /* Aborted NVME commands are required to not complete
2664 * before the abort exchange command fully completes.
2665 * Once completed, it is available via the put list.
2667 nvme_cmd
= lpfc_ncmd
->nvmeCmd
;
2668 nvme_cmd
->done(nvme_cmd
);
2669 lpfc_release_nvme_buf(phba
, lpfc_ncmd
);
2673 spin_unlock(&phba
->sli4_hba
.abts_nvme_buf_list_lock
);
2674 spin_unlock_irqrestore(&phba
->hbalock
, iflag
);
2676 lpfc_printf_log(phba
, KERN_INFO
, LOG_NVME_ABTS
,
2677 "6312 XRI Aborted xri x%x not found\n", xri
);