1 /*******************************************************************
2 * This file is part of the Emulex Linux Device Driver for *
3 * Fibre Channel Host Bus Adapters. *
4 * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
5 * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
6 * Copyright (C) 2004-2016 Emulex. All rights reserved. *
7 * EMULEX and SLI are trademarks of Emulex. *
9 * Portions Copyright (C) 2004-2005 Christoph Hellwig *
11 * This program is free software; you can redistribute it and/or *
12 * modify it under the terms of version 2 of the GNU General *
13 * Public License as published by the Free Software Foundation. *
14 * This program is distributed in the hope that it will be useful. *
15 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND *
16 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, *
17 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE *
18 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
19 * TO BE LEGALLY INVALID. See the GNU General Public License for *
20 * more details, a copy of which can be found in the file COPYING *
21 * included with this package. *
22 ********************************************************************/
23 #include <linux/pci.h>
24 #include <linux/slab.h>
25 #include <linux/interrupt.h>
26 #include <linux/delay.h>
27 #include <asm/unaligned.h>
28 #include <linux/crc-t10dif.h>
29 #include <net/checksum.h>
31 #include <scsi/scsi.h>
32 #include <scsi/scsi_device.h>
33 #include <scsi/scsi_eh.h>
34 #include <scsi/scsi_host.h>
35 #include <scsi/scsi_tcq.h>
36 #include <scsi/scsi_transport_fc.h>
37 #include <scsi/fc/fc_fs.h>
39 #include <linux/nvme.h>
40 #include <linux/nvme-fc-driver.h>
41 #include <linux/nvme-fc.h>
42 #include "lpfc_version.h"
46 #include "lpfc_sli4.h"
48 #include "lpfc_disc.h"
50 #include "lpfc_nvme.h"
51 #include "lpfc_scsi.h"
52 #include "lpfc_logmsg.h"
53 #include "lpfc_crtn.h"
54 #include "lpfc_vport.h"
55 #include "lpfc_debugfs.h"
57 /* NVME initiator-based functions */
59 static struct lpfc_nvme_buf
*
60 lpfc_get_nvme_buf(struct lpfc_hba
*phba
, struct lpfc_nodelist
*ndlp
);
63 lpfc_release_nvme_buf(struct lpfc_hba
*, struct lpfc_nvme_buf
*);
67 * lpfc_nvme_create_queue -
68 * @lpfc_pnvme: Pointer to the driver's nvme instance data
69 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
70 * @handle: An opaque driver handle used in follow-up calls.
72 * Driver registers this routine to preallocate and initialize any
73 * internal data structures to bind the @qidx to its internal IO queues.
74 * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ.
78 * -EINVAL - Unsupported input value.
79 * -ENOMEM - Could not alloc necessary memory
82 lpfc_nvme_create_queue(struct nvme_fc_local_port
*pnvme_lport
,
83 unsigned int qidx
, u16 qsize
,
86 struct lpfc_nvme_lport
*lport
;
87 struct lpfc_vport
*vport
;
88 struct lpfc_nvme_qhandle
*qhandle
;
91 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
93 qhandle
= kzalloc(sizeof(struct lpfc_nvme_qhandle
), GFP_KERNEL
);
97 qhandle
->cpu_id
= smp_processor_id();
100 * NVME qidx == 0 is the admin queue, so both admin queue
101 * and first IO queue will use MSI-X vector and associated
102 * EQ/CQ/WQ at index 0. After that they are sequentially assigned.
105 str
= "IO "; /* IO queue */
106 qhandle
->index
= ((qidx
- 1) %
107 vport
->phba
->cfg_nvme_io_channel
);
109 str
= "ADM"; /* Admin queue */
110 qhandle
->index
= qidx
;
113 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
,
114 "6073 Binding %s HdwQueue %d (cpu %d) to "
115 "io_channel %d qhandle %p\n", str
,
116 qidx
, qhandle
->cpu_id
, qhandle
->index
, qhandle
);
117 *handle
= (void *)qhandle
;
122 * lpfc_nvme_delete_queue -
123 * @lpfc_pnvme: Pointer to the driver's nvme instance data
124 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
125 * @handle: An opaque driver handle from lpfc_nvme_create_queue
127 * Driver registers this routine to free
128 * any internal data structures to bind the @qidx to its internal
133 * TODO: What are the failure codes.
136 lpfc_nvme_delete_queue(struct nvme_fc_local_port
*pnvme_lport
,
140 struct lpfc_nvme_lport
*lport
;
141 struct lpfc_vport
*vport
;
143 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
144 vport
= lport
->vport
;
146 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
,
147 "6001 ENTER. lpfc_pnvme %p, qidx x%xi qhandle %p\n",
148 lport
, qidx
, handle
);
153 lpfc_nvme_localport_delete(struct nvme_fc_local_port
*localport
)
155 struct lpfc_nvme_lport
*lport
= localport
->private;
157 /* release any threads waiting for the unreg to complete */
158 complete(&lport
->lport_unreg_done
);
161 /* lpfc_nvme_remoteport_delete
163 * @remoteport: Pointer to an nvme transport remoteport instance.
165 * This is a template downcall. NVME transport calls this function
166 * when it has completed the unregistration of a previously
167 * registered remoteport.
173 lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port
*remoteport
)
175 struct lpfc_nvme_rport
*rport
= remoteport
->private;
176 struct lpfc_vport
*vport
;
177 struct lpfc_nodelist
*ndlp
;
187 /* Remove this rport from the lport's list - memory is owned by the
188 * transport. Remove the ndlp reference for the NVME transport before
189 * calling state machine to remove the node.
191 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
192 "6146 remoteport delete complete %p\n",
198 /* This call has to execute as long as the rport is valid.
199 * Release any threads waiting for the unreg to complete.
201 complete(&rport
->rport_unreg_done
);
205 lpfc_nvme_cmpl_gen_req(struct lpfc_hba
*phba
, struct lpfc_iocbq
*cmdwqe
,
206 struct lpfc_wcqe_complete
*wcqe
)
208 struct lpfc_vport
*vport
= cmdwqe
->vport
;
210 struct nvmefc_ls_req
*pnvme_lsreq
;
211 struct lpfc_dmabuf
*buf_ptr
;
212 struct lpfc_nodelist
*ndlp
;
214 atomic_inc(&vport
->phba
->fc4NvmeLsCmpls
);
216 pnvme_lsreq
= (struct nvmefc_ls_req
*)cmdwqe
->context2
;
217 status
= bf_get(lpfc_wcqe_c_status
, wcqe
) & LPFC_IOCB_STATUS_MASK
;
218 ndlp
= (struct lpfc_nodelist
*)cmdwqe
->context1
;
219 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
220 "6047 nvme cmpl Enter "
221 "Data %p DID %x Xri: %x status %x cmd:%p lsreg:%p "
223 pnvme_lsreq
, ndlp
? ndlp
->nlp_DID
: 0,
224 cmdwqe
->sli4_xritag
, status
,
225 cmdwqe
, pnvme_lsreq
, cmdwqe
->context3
, ndlp
);
227 lpfc_nvmeio_data(phba
, "NVME LS CMPL: xri x%x stat x%x parm x%x\n",
228 cmdwqe
->sli4_xritag
, status
, wcqe
->parameter
);
230 if (cmdwqe
->context3
) {
231 buf_ptr
= (struct lpfc_dmabuf
*)cmdwqe
->context3
;
232 lpfc_mbuf_free(phba
, buf_ptr
->virt
, buf_ptr
->phys
);
234 cmdwqe
->context3
= NULL
;
236 if (pnvme_lsreq
->done
)
237 pnvme_lsreq
->done(pnvme_lsreq
, status
);
239 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
240 "6046 nvme cmpl without done call back? "
241 "Data %p DID %x Xri: %x status %x\n",
242 pnvme_lsreq
, ndlp
? ndlp
->nlp_DID
: 0,
243 cmdwqe
->sli4_xritag
, status
);
246 cmdwqe
->context1
= NULL
;
248 lpfc_sli_release_iocbq(phba
, cmdwqe
);
252 lpfc_nvme_gen_req(struct lpfc_vport
*vport
, struct lpfc_dmabuf
*bmp
,
253 struct lpfc_dmabuf
*inp
,
254 struct nvmefc_ls_req
*pnvme_lsreq
,
255 void (*cmpl
)(struct lpfc_hba
*, struct lpfc_iocbq
*,
256 struct lpfc_wcqe_complete
*),
257 struct lpfc_nodelist
*ndlp
, uint32_t num_entry
,
258 uint32_t tmo
, uint8_t retry
)
260 struct lpfc_hba
*phba
= vport
->phba
;
262 struct lpfc_iocbq
*genwqe
;
263 struct ulp_bde64
*bpl
;
264 struct ulp_bde64 bde
;
265 int i
, rc
, xmit_len
, first_len
;
267 /* Allocate buffer for command WQE */
268 genwqe
= lpfc_sli_get_iocbq(phba
);
273 memset(wqe
, 0, sizeof(union lpfc_wqe
));
275 genwqe
->context3
= (uint8_t *)bmp
;
276 genwqe
->iocb_flag
|= LPFC_IO_NVME_LS
;
278 /* Save for completion so we can release these resources */
279 genwqe
->context1
= lpfc_nlp_get(ndlp
);
280 genwqe
->context2
= (uint8_t *)pnvme_lsreq
;
281 /* Fill in payload, bp points to frame payload */
284 /* FC spec states we need 3 * ratov for CT requests */
285 tmo
= (3 * phba
->fc_ratov
);
287 /* For this command calculate the xmit length of the request bde. */
290 bpl
= (struct ulp_bde64
*)bmp
->virt
;
291 for (i
= 0; i
< num_entry
; i
++) {
292 bde
.tus
.w
= bpl
[i
].tus
.w
;
293 if (bde
.tus
.f
.bdeFlags
!= BUFF_TYPE_BDE_64
)
295 xmit_len
+= bde
.tus
.f
.bdeSize
;
297 first_len
= xmit_len
;
300 genwqe
->rsvd2
= num_entry
;
301 genwqe
->hba_wqidx
= 0;
304 wqe
->generic
.bde
.tus
.f
.bdeFlags
= BUFF_TYPE_BDE_64
;
305 wqe
->generic
.bde
.tus
.f
.bdeSize
= first_len
;
306 wqe
->generic
.bde
.addrLow
= bpl
[0].addrLow
;
307 wqe
->generic
.bde
.addrHigh
= bpl
[0].addrHigh
;
310 wqe
->gen_req
.request_payload_len
= first_len
;
315 bf_set(wqe_dfctl
, &wqe
->gen_req
.wge_ctl
, 0);
316 bf_set(wqe_si
, &wqe
->gen_req
.wge_ctl
, 1);
317 bf_set(wqe_la
, &wqe
->gen_req
.wge_ctl
, 1);
318 bf_set(wqe_rctl
, &wqe
->gen_req
.wge_ctl
, FC_RCTL_ELS4_REQ
);
319 bf_set(wqe_type
, &wqe
->gen_req
.wge_ctl
, FC_TYPE_NVME
);
322 bf_set(wqe_ctxt_tag
, &wqe
->gen_req
.wqe_com
,
323 phba
->sli4_hba
.rpi_ids
[ndlp
->nlp_rpi
]);
324 bf_set(wqe_xri_tag
, &wqe
->gen_req
.wqe_com
, genwqe
->sli4_xritag
);
327 bf_set(wqe_tmo
, &wqe
->gen_req
.wqe_com
, (vport
->phba
->fc_ratov
-1));
328 bf_set(wqe_class
, &wqe
->gen_req
.wqe_com
, CLASS3
);
329 bf_set(wqe_cmnd
, &wqe
->gen_req
.wqe_com
, CMD_GEN_REQUEST64_WQE
);
330 bf_set(wqe_ct
, &wqe
->gen_req
.wqe_com
, SLI4_CT_RPI
);
333 wqe
->gen_req
.wqe_com
.abort_tag
= genwqe
->iotag
;
336 bf_set(wqe_reqtag
, &wqe
->gen_req
.wqe_com
, genwqe
->iotag
);
339 bf_set(wqe_dbde
, &wqe
->gen_req
.wqe_com
, 1);
340 bf_set(wqe_iod
, &wqe
->gen_req
.wqe_com
, LPFC_WQE_IOD_READ
);
341 bf_set(wqe_qosd
, &wqe
->gen_req
.wqe_com
, 1);
342 bf_set(wqe_lenloc
, &wqe
->gen_req
.wqe_com
, LPFC_WQE_LENLOC_NONE
);
343 bf_set(wqe_ebde_cnt
, &wqe
->gen_req
.wqe_com
, 0);
346 bf_set(wqe_cqid
, &wqe
->gen_req
.wqe_com
, LPFC_WQE_CQ_ID_DEFAULT
);
347 bf_set(wqe_cmd_type
, &wqe
->gen_req
.wqe_com
, OTHER_COMMAND
);
350 /* Issue GEN REQ WQE for NPORT <did> */
351 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_ELS
,
352 "6050 Issue GEN REQ WQE to NPORT x%x "
353 "Data: x%x x%x wq:%p lsreq:%p bmp:%p xmit:%d 1st:%d\n",
354 ndlp
->nlp_DID
, genwqe
->iotag
,
356 genwqe
, pnvme_lsreq
, bmp
, xmit_len
, first_len
);
357 genwqe
->wqe_cmpl
= cmpl
;
358 genwqe
->iocb_cmpl
= NULL
;
359 genwqe
->drvrTimeout
= tmo
+ LPFC_DRVR_TIMEOUT
;
360 genwqe
->vport
= vport
;
361 genwqe
->retry
= retry
;
363 lpfc_nvmeio_data(phba
, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n",
364 genwqe
->sli4_xritag
, genwqe
->iotag
, ndlp
->nlp_DID
);
366 rc
= lpfc_sli4_issue_wqe(phba
, LPFC_ELS_RING
, genwqe
);
368 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_ELS
,
369 "6045 Issue GEN REQ WQE to NPORT x%x "
371 ndlp
->nlp_DID
, genwqe
->iotag
,
373 lpfc_sli_release_iocbq(phba
, genwqe
);
380 * lpfc_nvme_ls_req - Issue an Link Service request
381 * @lpfc_pnvme: Pointer to the driver's nvme instance data
382 * @lpfc_nvme_lport: Pointer to the driver's local port data
383 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
385 * Driver registers this routine to handle any link service request
386 * from the nvme_fc transport to a remote nvme-aware port.
390 * TODO: What are the failure codes.
393 lpfc_nvme_ls_req(struct nvme_fc_local_port
*pnvme_lport
,
394 struct nvme_fc_remote_port
*pnvme_rport
,
395 struct nvmefc_ls_req
*pnvme_lsreq
)
398 struct lpfc_nvme_lport
*lport
;
399 struct lpfc_vport
*vport
;
400 struct lpfc_nodelist
*ndlp
;
401 struct ulp_bde64
*bpl
;
402 struct lpfc_dmabuf
*bmp
;
403 uint16_t ntype
, nstate
;
405 /* there are two dma buf in the request, actually there is one and
406 * the second one is just the start address + cmd size.
407 * Before calling lpfc_nvme_gen_req these buffers need to be wrapped
408 * in a lpfc_dmabuf struct. When freeing we just free the wrapper
409 * because the nvem layer owns the data bufs.
410 * We do not have to break these packets open, we don't care what is in
411 * them. And we do not have to look at the resonse data, we only care
412 * that we got a response. All of the caring is going to happen in the
416 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
417 vport
= lport
->vport
;
419 ndlp
= lpfc_findnode_did(vport
, pnvme_rport
->port_id
);
420 if (!ndlp
|| !NLP_CHK_NODE_ACT(ndlp
)) {
421 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
422 "6051 DID x%06x not an active rport.\n",
423 pnvme_rport
->port_id
);
427 /* The remote node has to be a mapped nvme target or an
428 * unmapped nvme initiator or it's an error.
430 ntype
= ndlp
->nlp_type
;
431 nstate
= ndlp
->nlp_state
;
432 if ((ntype
& NLP_NVME_TARGET
&& nstate
!= NLP_STE_MAPPED_NODE
) ||
433 (ntype
& NLP_NVME_INITIATOR
&& nstate
!= NLP_STE_UNMAPPED_NODE
)) {
434 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
435 "6088 DID x%06x not ready for "
436 "IO. State x%x, Type x%x\n",
437 pnvme_rport
->port_id
,
438 ndlp
->nlp_state
, ndlp
->nlp_type
);
441 bmp
= kmalloc(sizeof(struct lpfc_dmabuf
), GFP_KERNEL
);
444 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
445 "6044 Could not find node for DID %x\n",
446 pnvme_rport
->port_id
);
449 INIT_LIST_HEAD(&bmp
->list
);
450 bmp
->virt
= lpfc_mbuf_alloc(vport
->phba
, MEM_PRI
, &(bmp
->phys
));
452 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
453 "6042 Could not find node for DID %x\n",
454 pnvme_rport
->port_id
);
458 bpl
= (struct ulp_bde64
*)bmp
->virt
;
459 bpl
->addrHigh
= le32_to_cpu(putPaddrHigh(pnvme_lsreq
->rqstdma
));
460 bpl
->addrLow
= le32_to_cpu(putPaddrLow(pnvme_lsreq
->rqstdma
));
461 bpl
->tus
.f
.bdeFlags
= 0;
462 bpl
->tus
.f
.bdeSize
= pnvme_lsreq
->rqstlen
;
463 bpl
->tus
.w
= le32_to_cpu(bpl
->tus
.w
);
466 bpl
->addrHigh
= le32_to_cpu(putPaddrHigh(pnvme_lsreq
->rspdma
));
467 bpl
->addrLow
= le32_to_cpu(putPaddrLow(pnvme_lsreq
->rspdma
));
468 bpl
->tus
.f
.bdeFlags
= BUFF_TYPE_BDE_64I
;
469 bpl
->tus
.f
.bdeSize
= pnvme_lsreq
->rsplen
;
470 bpl
->tus
.w
= le32_to_cpu(bpl
->tus
.w
);
472 /* Expand print to include key fields. */
473 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
474 "6149 ENTER. lport %p, rport %p lsreq%p rqstlen:%d "
475 "rsplen:%d %pad %pad\n",
476 pnvme_lport
, pnvme_rport
,
477 pnvme_lsreq
, pnvme_lsreq
->rqstlen
,
478 pnvme_lsreq
->rsplen
, &pnvme_lsreq
->rqstdma
,
479 &pnvme_lsreq
->rspdma
);
481 atomic_inc(&vport
->phba
->fc4NvmeLsRequests
);
483 /* Hardcode the wait to 30 seconds. Connections are failing otherwise.
484 * This code allows it all to work.
486 ret
= lpfc_nvme_gen_req(vport
, bmp
, pnvme_lsreq
->rqstaddr
,
487 pnvme_lsreq
, lpfc_nvme_cmpl_gen_req
,
489 if (ret
!= WQE_SUCCESS
) {
490 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
491 "6052 EXIT. issue ls wqe failed lport %p, "
492 "rport %p lsreq%p Status %x DID %x\n",
493 pnvme_lport
, pnvme_rport
, pnvme_lsreq
,
495 lpfc_mbuf_free(vport
->phba
, bmp
->virt
, bmp
->phys
);
500 /* Stub in routine and return 0 for now. */
505 * lpfc_nvme_ls_abort - Issue an Link Service request
506 * @lpfc_pnvme: Pointer to the driver's nvme instance data
507 * @lpfc_nvme_lport: Pointer to the driver's local port data
508 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
510 * Driver registers this routine to handle any link service request
511 * from the nvme_fc transport to a remote nvme-aware port.
515 * TODO: What are the failure codes.
518 lpfc_nvme_ls_abort(struct nvme_fc_local_port
*pnvme_lport
,
519 struct nvme_fc_remote_port
*pnvme_rport
,
520 struct nvmefc_ls_req
*pnvme_lsreq
)
522 struct lpfc_nvme_lport
*lport
;
523 struct lpfc_vport
*vport
;
524 struct lpfc_hba
*phba
;
525 struct lpfc_nodelist
*ndlp
;
526 LIST_HEAD(abort_list
);
527 struct lpfc_sli_ring
*pring
;
528 struct lpfc_iocbq
*wqe
, *next_wqe
;
530 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
531 vport
= lport
->vport
;
534 ndlp
= lpfc_findnode_did(vport
, pnvme_rport
->port_id
);
536 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
537 "6049 Could not find node for DID %x\n",
538 pnvme_rport
->port_id
);
542 /* Expand print to include key fields. */
543 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_ABTS
,
544 "6040 ENTER. lport %p, rport %p lsreq %p rqstlen:%d "
545 "rsplen:%d %pad %pad\n",
546 pnvme_lport
, pnvme_rport
,
547 pnvme_lsreq
, pnvme_lsreq
->rqstlen
,
548 pnvme_lsreq
->rsplen
, &pnvme_lsreq
->rqstdma
,
549 &pnvme_lsreq
->rspdma
);
552 * Lock the ELS ring txcmplq and build a local list of all ELS IOs
553 * that need an ABTS. The IOs need to stay on the txcmplq so that
554 * the abort operation completes them successfully.
556 pring
= phba
->sli4_hba
.nvmels_wq
->pring
;
557 spin_lock_irq(&phba
->hbalock
);
558 spin_lock(&pring
->ring_lock
);
559 list_for_each_entry_safe(wqe
, next_wqe
, &pring
->txcmplq
, list
) {
560 /* Add to abort_list on on NDLP match. */
561 if (lpfc_check_sli_ndlp(phba
, pring
, wqe
, ndlp
)) {
562 wqe
->iocb_flag
|= LPFC_DRIVER_ABORTED
;
563 list_add_tail(&wqe
->dlist
, &abort_list
);
566 spin_unlock(&pring
->ring_lock
);
567 spin_unlock_irq(&phba
->hbalock
);
569 /* Abort the targeted IOs and remove them from the abort list. */
570 list_for_each_entry_safe(wqe
, next_wqe
, &abort_list
, dlist
) {
571 spin_lock_irq(&phba
->hbalock
);
572 list_del_init(&wqe
->dlist
);
573 lpfc_sli_issue_abort_iotag(phba
, pring
, wqe
);
574 spin_unlock_irq(&phba
->hbalock
);
578 /* Fix up the existing sgls for NVME IO. */
580 lpfc_nvme_adj_fcp_sgls(struct lpfc_vport
*vport
,
581 struct lpfc_nvme_buf
*lpfc_ncmd
,
582 struct nvmefc_fcp_req
*nCmd
)
584 struct sli4_sge
*sgl
;
585 union lpfc_wqe128
*wqe
;
586 uint32_t *wptr
, *dptr
;
589 * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to
590 * match NVME. NVME sends 96 bytes. Also, use the
591 * nvme commands command and response dma addresses
592 * rather than the virtual memory to ease the restore
595 sgl
= lpfc_ncmd
->nvme_sgl
;
596 sgl
->sge_len
= cpu_to_le32(nCmd
->cmdlen
);
600 /* Setup the physical region for the FCP RSP */
601 sgl
->addr_hi
= cpu_to_le32(putPaddrHigh(nCmd
->rspdma
));
602 sgl
->addr_lo
= cpu_to_le32(putPaddrLow(nCmd
->rspdma
));
603 sgl
->word2
= le32_to_cpu(sgl
->word2
);
605 bf_set(lpfc_sli4_sge_last
, sgl
, 0);
607 bf_set(lpfc_sli4_sge_last
, sgl
, 1);
608 sgl
->word2
= cpu_to_le32(sgl
->word2
);
609 sgl
->sge_len
= cpu_to_le32(nCmd
->rsplen
);
612 * Get a local pointer to the built-in wqe and correct
613 * the cmd size to match NVME's 96 bytes and fix
617 /* 128 byte wqe support here */
618 wqe
= (union lpfc_wqe128
*)&lpfc_ncmd
->cur_iocbq
.wqe
;
620 /* Word 0-2 - NVME CMND IU (embedded payload) */
621 wqe
->generic
.bde
.tus
.f
.bdeFlags
= BUFF_TYPE_BDE_IMMED
;
622 wqe
->generic
.bde
.tus
.f
.bdeSize
= 60;
623 wqe
->generic
.bde
.addrHigh
= 0;
624 wqe
->generic
.bde
.addrLow
= 64; /* Word 16 */
627 bf_set(payload_offset_len
, &wqe
->fcp_icmd
,
628 (nCmd
->rsplen
+ nCmd
->cmdlen
));
631 bf_set(wqe_nvme
, &wqe
->fcp_icmd
.wqe_com
, 1);
632 bf_set(wqe_wqes
, &wqe
->fcp_icmd
.wqe_com
, 1);
635 * Embed the payload in the last half of the WQE
636 * WQE words 16-30 get the NVME CMD IU payload
638 * WQE words 16-19 get payload Words 1-4
639 * WQE words 20-21 get payload Words 6-7
640 * WQE words 22-29 get payload Words 16-23
642 wptr
= &wqe
->words
[16]; /* WQE ptr */
643 dptr
= (uint32_t *)nCmd
->cmdaddr
; /* payload ptr */
644 dptr
++; /* Skip Word 0 in payload */
646 *wptr
++ = *dptr
++; /* Word 1 */
647 *wptr
++ = *dptr
++; /* Word 2 */
648 *wptr
++ = *dptr
++; /* Word 3 */
649 *wptr
++ = *dptr
++; /* Word 4 */
650 dptr
++; /* Skip Word 5 in payload */
651 *wptr
++ = *dptr
++; /* Word 6 */
652 *wptr
++ = *dptr
++; /* Word 7 */
653 dptr
+= 8; /* Skip Words 8-15 in payload */
654 *wptr
++ = *dptr
++; /* Word 16 */
655 *wptr
++ = *dptr
++; /* Word 17 */
656 *wptr
++ = *dptr
++; /* Word 18 */
657 *wptr
++ = *dptr
++; /* Word 19 */
658 *wptr
++ = *dptr
++; /* Word 20 */
659 *wptr
++ = *dptr
++; /* Word 21 */
660 *wptr
++ = *dptr
++; /* Word 22 */
661 *wptr
= *dptr
; /* Word 23 */
664 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
666 lpfc_nvme_ktime(struct lpfc_hba
*phba
,
667 struct lpfc_nvme_buf
*lpfc_ncmd
)
669 uint64_t seg1
, seg2
, seg3
, seg4
;
673 if (!lpfc_ncmd
->ts_last_cmd
||
674 !lpfc_ncmd
->ts_cmd_start
||
675 !lpfc_ncmd
->ts_cmd_wqput
||
676 !lpfc_ncmd
->ts_isr_cmpl
||
677 !lpfc_ncmd
->ts_data_nvme
)
679 if (lpfc_ncmd
->ts_cmd_start
< lpfc_ncmd
->ts_last_cmd
)
681 if (lpfc_ncmd
->ts_cmd_wqput
< lpfc_ncmd
->ts_cmd_start
)
683 if (lpfc_ncmd
->ts_isr_cmpl
< lpfc_ncmd
->ts_cmd_wqput
)
685 if (lpfc_ncmd
->ts_data_nvme
< lpfc_ncmd
->ts_isr_cmpl
)
688 * Segment 1 - Time from Last FCP command cmpl is handed
689 * off to NVME Layer to start of next command.
690 * Segment 2 - Time from Driver receives a IO cmd start
691 * from NVME Layer to WQ put is done on IO cmd.
692 * Segment 3 - Time from Driver WQ put is done on IO cmd
693 * to MSI-X ISR for IO cmpl.
694 * Segment 4 - Time from MSI-X ISR for IO cmpl to when
695 * cmpl is handled off to the NVME Layer.
697 seg1
= lpfc_ncmd
->ts_cmd_start
- lpfc_ncmd
->ts_last_cmd
;
698 if (seg1
> 5000000) /* 5 ms - for sequential IOs */
701 /* Calculate times relative to start of IO */
702 seg2
= (lpfc_ncmd
->ts_cmd_wqput
- lpfc_ncmd
->ts_cmd_start
);
703 seg3
= (lpfc_ncmd
->ts_isr_cmpl
-
704 lpfc_ncmd
->ts_cmd_start
) - seg2
;
705 seg4
= (lpfc_ncmd
->ts_data_nvme
-
706 lpfc_ncmd
->ts_cmd_start
) - seg2
- seg3
;
707 phba
->ktime_data_samples
++;
708 phba
->ktime_seg1_total
+= seg1
;
709 if (seg1
< phba
->ktime_seg1_min
)
710 phba
->ktime_seg1_min
= seg1
;
711 else if (seg1
> phba
->ktime_seg1_max
)
712 phba
->ktime_seg1_max
= seg1
;
713 phba
->ktime_seg2_total
+= seg2
;
714 if (seg2
< phba
->ktime_seg2_min
)
715 phba
->ktime_seg2_min
= seg2
;
716 else if (seg2
> phba
->ktime_seg2_max
)
717 phba
->ktime_seg2_max
= seg2
;
718 phba
->ktime_seg3_total
+= seg3
;
719 if (seg3
< phba
->ktime_seg3_min
)
720 phba
->ktime_seg3_min
= seg3
;
721 else if (seg3
> phba
->ktime_seg3_max
)
722 phba
->ktime_seg3_max
= seg3
;
723 phba
->ktime_seg4_total
+= seg4
;
724 if (seg4
< phba
->ktime_seg4_min
)
725 phba
->ktime_seg4_min
= seg4
;
726 else if (seg4
> phba
->ktime_seg4_max
)
727 phba
->ktime_seg4_max
= seg4
;
729 lpfc_ncmd
->ts_last_cmd
= 0;
730 lpfc_ncmd
->ts_cmd_start
= 0;
731 lpfc_ncmd
->ts_cmd_wqput
= 0;
732 lpfc_ncmd
->ts_isr_cmpl
= 0;
733 lpfc_ncmd
->ts_data_nvme
= 0;
738 * lpfc_nvme_io_cmd_wqe_cmpl - Complete an NVME-over-FCP IO
739 * @lpfc_pnvme: Pointer to the driver's nvme instance data
740 * @lpfc_nvme_lport: Pointer to the driver's local port data
741 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
743 * Driver registers this routine as it io request handler. This
744 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
745 * data structure to the rport indicated in @lpfc_nvme_rport.
749 * TODO: What are the failure codes.
752 lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba
*phba
, struct lpfc_iocbq
*pwqeIn
,
753 struct lpfc_wcqe_complete
*wcqe
)
755 struct lpfc_nvme_buf
*lpfc_ncmd
=
756 (struct lpfc_nvme_buf
*)pwqeIn
->context1
;
757 struct lpfc_vport
*vport
= pwqeIn
->vport
;
758 struct nvmefc_fcp_req
*nCmd
;
759 struct nvme_fc_ersp_iu
*ep
;
760 struct nvme_fc_cmd_iu
*cp
;
761 struct lpfc_nvme_rport
*rport
;
762 struct lpfc_nodelist
*ndlp
;
763 struct lpfc_nvme_fcpreq_priv
*freqpriv
;
766 uint16_t cid
, sqhd
, data
;
769 /* Sanity check on return of outstanding command */
770 if (!lpfc_ncmd
|| !lpfc_ncmd
->nvmeCmd
|| !lpfc_ncmd
->nrport
) {
771 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
772 "6071 Completion pointers bad on wqe %p.\n",
776 atomic_inc(&phba
->fc4NvmeIoCmpls
);
778 nCmd
= lpfc_ncmd
->nvmeCmd
;
779 rport
= lpfc_ncmd
->nrport
;
781 lpfc_nvmeio_data(phba
, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n",
782 lpfc_ncmd
->cur_iocbq
.sli4_xritag
,
783 bf_get(lpfc_wcqe_c_status
, wcqe
), wcqe
->parameter
);
785 * Catch race where our node has transitioned, but the
786 * transport is still transitioning.
789 if (!ndlp
|| !NLP_CHK_NODE_ACT(ndlp
)) {
790 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
791 "6061 rport %p, DID x%06x node not ready.\n",
792 rport
, rport
->remoteport
->port_id
);
794 ndlp
= lpfc_findnode_did(vport
, rport
->remoteport
->port_id
);
796 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
797 "6062 Ignoring NVME cmpl. No ndlp\n");
802 code
= bf_get(lpfc_wcqe_c_code
, wcqe
);
803 if (code
== CQE_CODE_NVME_ERSP
) {
804 /* For this type of CQE, we need to rebuild the rsp */
805 ep
= (struct nvme_fc_ersp_iu
*)nCmd
->rspaddr
;
808 * Get Command Id from cmd to plug into response. This
809 * code is not needed in the next NVME Transport drop.
811 cp
= (struct nvme_fc_cmd_iu
*)nCmd
->cmdaddr
;
812 cid
= cp
->sqe
.common
.command_id
;
815 * RSN is in CQE word 2
816 * SQHD is in CQE Word 3 bits 15:0
817 * Cmd Specific info is in CQE Word 1
818 * and in CQE Word 0 bits 15:0
820 sqhd
= bf_get(lpfc_wcqe_c_sqhead
, wcqe
);
822 /* Now lets build the NVME ERSP IU */
823 ep
->iu_len
= cpu_to_be16(8);
824 ep
->rsn
= wcqe
->parameter
;
825 ep
->xfrd_len
= cpu_to_be32(nCmd
->payload_length
);
827 ptr
= (uint32_t *)&ep
->cqe
.result
.u64
;
828 *ptr
++ = wcqe
->total_data_placed
;
829 data
= bf_get(lpfc_wcqe_c_ersp0
, wcqe
);
830 *ptr
= (uint32_t)data
;
831 ep
->cqe
.sq_head
= sqhd
;
832 ep
->cqe
.sq_id
= nCmd
->sqid
;
833 ep
->cqe
.command_id
= cid
;
836 lpfc_ncmd
->status
= IOSTAT_SUCCESS
;
837 lpfc_ncmd
->result
= 0;
838 nCmd
->rcv_rsplen
= LPFC_NVME_ERSP_LEN
;
839 nCmd
->transferred_length
= nCmd
->payload_length
;
841 lpfc_ncmd
->status
= (bf_get(lpfc_wcqe_c_status
, wcqe
) &
842 LPFC_IOCB_STATUS_MASK
);
843 lpfc_ncmd
->result
= wcqe
->parameter
;
845 /* For NVME, the only failure path that results in an
846 * IO error is when the adapter rejects it. All other
847 * conditions are a success case and resolved by the
849 * IOSTAT_FCP_RSP_ERROR means:
850 * 1. Length of data received doesn't match total
851 * transfer length in WQE
852 * 2. If the RSP payload does NOT match these cases:
853 * a. RSP length 12/24 bytes and all zeros
856 switch (lpfc_ncmd
->status
) {
858 nCmd
->transferred_length
= wcqe
->total_data_placed
;
859 nCmd
->rcv_rsplen
= 0;
862 case IOSTAT_FCP_RSP_ERROR
:
863 nCmd
->transferred_length
= wcqe
->total_data_placed
;
864 nCmd
->rcv_rsplen
= wcqe
->parameter
;
867 if (nCmd
->rcv_rsplen
== LPFC_NVME_ERSP_LEN
)
869 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
870 "6081 NVME Completion Protocol Error: "
871 "xri %x status x%x result x%x "
873 lpfc_ncmd
->cur_iocbq
.sli4_xritag
,
874 lpfc_ncmd
->status
, lpfc_ncmd
->result
,
875 wcqe
->total_data_placed
);
879 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
880 "6072 NVME Completion Error: xri %x "
881 "status x%x result x%x placed x%x\n",
882 lpfc_ncmd
->cur_iocbq
.sli4_xritag
,
883 lpfc_ncmd
->status
, lpfc_ncmd
->result
,
884 wcqe
->total_data_placed
);
885 nCmd
->transferred_length
= 0;
886 nCmd
->rcv_rsplen
= 0;
887 nCmd
->status
= NVME_SC_INTERNAL
;
891 /* pick up SLI4 exhange busy condition */
892 if (bf_get(lpfc_wcqe_c_xb
, wcqe
))
893 lpfc_ncmd
->flags
|= LPFC_SBUF_XBUSY
;
895 lpfc_ncmd
->flags
&= ~LPFC_SBUF_XBUSY
;
897 if (ndlp
&& NLP_CHK_NODE_ACT(ndlp
))
898 atomic_dec(&ndlp
->cmd_pending
);
900 /* Update stats and complete the IO. There is
901 * no need for dma unprep because the nvme_transport
902 * owns the dma address.
904 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
905 if (phba
->ktime_on
) {
906 lpfc_ncmd
->ts_isr_cmpl
= pwqeIn
->isr_timestamp
;
907 lpfc_ncmd
->ts_data_nvme
= ktime_get_ns();
908 phba
->ktime_last_cmd
= lpfc_ncmd
->ts_data_nvme
;
909 lpfc_nvme_ktime(phba
, lpfc_ncmd
);
911 if (phba
->cpucheck_on
& LPFC_CHECK_NVME_IO
) {
912 if (lpfc_ncmd
->cpu
!= smp_processor_id())
913 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
914 "6701 CPU Check cmpl: "
915 "cpu %d expect %d\n",
916 smp_processor_id(), lpfc_ncmd
->cpu
);
917 if (lpfc_ncmd
->cpu
< LPFC_CHECK_CPU_CNT
)
918 phba
->cpucheck_cmpl_io
[lpfc_ncmd
->cpu
]++;
921 freqpriv
= nCmd
->private;
922 freqpriv
->nvme_buf
= NULL
;
925 spin_lock_irqsave(&phba
->hbalock
, flags
);
926 lpfc_ncmd
->nrport
= NULL
;
927 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
929 lpfc_release_nvme_buf(phba
, lpfc_ncmd
);
934 * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO
935 * @lpfc_pnvme: Pointer to the driver's nvme instance data
936 * @lpfc_nvme_lport: Pointer to the driver's local port data
937 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
938 * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
939 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
941 * Driver registers this routine as it io request handler. This
942 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
943 * data structure to the rport indicated in @lpfc_nvme_rport.
947 * TODO: What are the failure codes.
950 lpfc_nvme_prep_io_cmd(struct lpfc_vport
*vport
,
951 struct lpfc_nvme_buf
*lpfc_ncmd
,
952 struct lpfc_nodelist
*pnode
)
954 struct lpfc_hba
*phba
= vport
->phba
;
955 struct nvmefc_fcp_req
*nCmd
= lpfc_ncmd
->nvmeCmd
;
956 struct lpfc_iocbq
*pwqeq
= &(lpfc_ncmd
->cur_iocbq
);
957 union lpfc_wqe128
*wqe
= (union lpfc_wqe128
*)&pwqeq
->wqe
;
960 if (!pnode
|| !NLP_CHK_NODE_ACT(pnode
))
964 * There are three possibilities here - use scatter-gather segment, use
965 * the single mapping, or neither.
967 wqe
->fcp_iwrite
.initial_xfer_len
= 0;
969 if (nCmd
->io_dir
== NVMEFC_FCP_WRITE
) {
971 if ((phba
->cfg_nvme_enable_fb
) &&
972 (pnode
->nlp_flag
& NLP_FIRSTBURST
)) {
973 req_len
= lpfc_ncmd
->nvmeCmd
->payload_length
;
974 if (req_len
< pnode
->nvme_fb_size
)
975 wqe
->fcp_iwrite
.initial_xfer_len
=
978 wqe
->fcp_iwrite
.initial_xfer_len
=
983 bf_set(wqe_cmnd
, &wqe
->generic
.wqe_com
,
984 CMD_FCP_IWRITE64_WQE
);
985 bf_set(wqe_pu
, &wqe
->generic
.wqe_com
,
989 bf_set(wqe_qosd
, &wqe
->fcp_iwrite
.wqe_com
, 0);
990 bf_set(wqe_iod
, &wqe
->fcp_iwrite
.wqe_com
,
992 bf_set(wqe_lenloc
, &wqe
->fcp_iwrite
.wqe_com
,
993 LPFC_WQE_LENLOC_WORD4
);
994 if (phba
->cfg_nvme_oas
)
995 bf_set(wqe_oas
, &wqe
->fcp_iwrite
.wqe_com
, 1);
998 bf_set(wqe_cmd_type
, &wqe
->generic
.wqe_com
,
1001 atomic_inc(&phba
->fc4NvmeOutputRequests
);
1004 bf_set(wqe_cmnd
, &wqe
->generic
.wqe_com
,
1005 CMD_FCP_IREAD64_WQE
);
1006 bf_set(wqe_pu
, &wqe
->generic
.wqe_com
,
1010 bf_set(wqe_qosd
, &wqe
->fcp_iread
.wqe_com
, 0);
1011 bf_set(wqe_iod
, &wqe
->fcp_iread
.wqe_com
,
1013 bf_set(wqe_lenloc
, &wqe
->fcp_iread
.wqe_com
,
1014 LPFC_WQE_LENLOC_WORD4
);
1015 if (phba
->cfg_nvme_oas
)
1016 bf_set(wqe_oas
, &wqe
->fcp_iread
.wqe_com
, 1);
1019 bf_set(wqe_cmd_type
, &wqe
->generic
.wqe_com
,
1022 atomic_inc(&phba
->fc4NvmeInputRequests
);
1026 wqe
->fcp_icmd
.rsrvd4
= 0;
1029 bf_set(wqe_cmnd
, &wqe
->generic
.wqe_com
, CMD_FCP_ICMND64_WQE
);
1030 bf_set(wqe_pu
, &wqe
->generic
.wqe_com
, 0);
1033 bf_set(wqe_qosd
, &wqe
->fcp_icmd
.wqe_com
, 1);
1034 bf_set(wqe_iod
, &wqe
->fcp_icmd
.wqe_com
, LPFC_WQE_IOD_WRITE
);
1035 bf_set(wqe_lenloc
, &wqe
->fcp_icmd
.wqe_com
,
1036 LPFC_WQE_LENLOC_NONE
);
1037 if (phba
->cfg_nvme_oas
)
1038 bf_set(wqe_oas
, &wqe
->fcp_icmd
.wqe_com
, 1);
1041 bf_set(wqe_cmd_type
, &wqe
->generic
.wqe_com
, NVME_READ_CMD
);
1043 atomic_inc(&phba
->fc4NvmeControlRequests
);
1046 * Finish initializing those WQE fields that are independent
1047 * of the nvme_cmnd request_buffer
1051 bf_set(wqe_ctxt_tag
, &wqe
->generic
.wqe_com
,
1052 phba
->sli4_hba
.rpi_ids
[pnode
->nlp_rpi
]);
1053 bf_set(wqe_xri_tag
, &wqe
->generic
.wqe_com
, pwqeq
->sli4_xritag
);
1056 /* Preserve Class data in the ndlp. */
1057 bf_set(wqe_class
, &wqe
->generic
.wqe_com
,
1058 (pnode
->nlp_fcp_info
& 0x0f));
1061 wqe
->generic
.wqe_com
.abort_tag
= pwqeq
->iotag
;
1064 bf_set(wqe_reqtag
, &wqe
->generic
.wqe_com
, pwqeq
->iotag
);
1067 bf_set(wqe_cqid
, &wqe
->generic
.wqe_com
, LPFC_WQE_CQ_ID_DEFAULT
);
1069 pwqeq
->vport
= vport
;
1075 * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO
1076 * @lpfc_pnvme: Pointer to the driver's nvme instance data
1077 * @lpfc_nvme_lport: Pointer to the driver's local port data
1078 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1079 * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1080 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1082 * Driver registers this routine as it io request handler. This
1083 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1084 * data structure to the rport indicated in @lpfc_nvme_rport.
1088 * TODO: What are the failure codes.
1091 lpfc_nvme_prep_io_dma(struct lpfc_vport
*vport
,
1092 struct lpfc_nvme_buf
*lpfc_ncmd
)
1094 struct lpfc_hba
*phba
= vport
->phba
;
1095 struct nvmefc_fcp_req
*nCmd
= lpfc_ncmd
->nvmeCmd
;
1096 union lpfc_wqe128
*wqe
= (union lpfc_wqe128
*)&lpfc_ncmd
->cur_iocbq
.wqe
;
1097 struct sli4_sge
*sgl
= lpfc_ncmd
->nvme_sgl
;
1098 struct scatterlist
*data_sg
;
1099 struct sli4_sge
*first_data_sgl
;
1100 dma_addr_t physaddr
;
1101 uint32_t num_bde
= 0;
1103 uint32_t dma_offset
= 0;
1106 /* Fix up the command and response DMA stuff. */
1107 lpfc_nvme_adj_fcp_sgls(vport
, lpfc_ncmd
, nCmd
);
1110 * There are three possibilities here - use scatter-gather segment, use
1111 * the single mapping, or neither.
1115 * Jump over the cmd and rsp SGEs. The fix routine
1116 * has already adjusted for this.
1120 first_data_sgl
= sgl
;
1121 lpfc_ncmd
->seg_cnt
= nCmd
->sg_cnt
;
1122 if (lpfc_ncmd
->seg_cnt
> phba
->cfg_nvme_seg_cnt
) {
1123 lpfc_printf_log(phba
, KERN_ERR
, LOG_NVME_IOERR
,
1124 "6058 Too many sg segments from "
1125 "NVME Transport. Max %d, "
1126 "nvmeIO sg_cnt %d\n",
1127 phba
->cfg_nvme_seg_cnt
,
1128 lpfc_ncmd
->seg_cnt
);
1129 lpfc_ncmd
->seg_cnt
= 0;
1134 * The driver established a maximum scatter-gather segment count
1135 * during probe that limits the number of sg elements in any
1136 * single nvme command. Just run through the seg_cnt and format
1139 nseg
= nCmd
->sg_cnt
;
1140 data_sg
= nCmd
->first_sgl
;
1141 for (i
= 0; i
< nseg
; i
++) {
1142 if (data_sg
== NULL
) {
1143 lpfc_printf_log(phba
, KERN_ERR
, LOG_NVME_IOERR
,
1144 "6059 dptr err %d, nseg %d\n",
1146 lpfc_ncmd
->seg_cnt
= 0;
1149 physaddr
= data_sg
->dma_address
;
1150 dma_len
= data_sg
->length
;
1151 sgl
->addr_lo
= cpu_to_le32(putPaddrLow(physaddr
));
1152 sgl
->addr_hi
= cpu_to_le32(putPaddrHigh(physaddr
));
1153 sgl
->word2
= le32_to_cpu(sgl
->word2
);
1154 if ((num_bde
+ 1) == nseg
)
1155 bf_set(lpfc_sli4_sge_last
, sgl
, 1);
1157 bf_set(lpfc_sli4_sge_last
, sgl
, 0);
1158 bf_set(lpfc_sli4_sge_offset
, sgl
, dma_offset
);
1159 bf_set(lpfc_sli4_sge_type
, sgl
, LPFC_SGE_TYPE_DATA
);
1160 sgl
->word2
= cpu_to_le32(sgl
->word2
);
1161 sgl
->sge_len
= cpu_to_le32(dma_len
);
1163 dma_offset
+= dma_len
;
1164 data_sg
= sg_next(data_sg
);
1168 /* For this clause to be valid, the payload_length
1169 * and sg_cnt must zero.
1171 if (nCmd
->payload_length
!= 0) {
1172 lpfc_printf_log(phba
, KERN_ERR
, LOG_NVME_IOERR
,
1173 "6063 NVME DMA Prep Err: sg_cnt %d "
1174 "payload_length x%x\n",
1175 nCmd
->sg_cnt
, nCmd
->payload_length
);
1181 * Due to difference in data length between DIF/non-DIF paths,
1182 * we need to set word 4 of WQE here
1184 wqe
->fcp_iread
.total_xfer_len
= nCmd
->payload_length
;
1189 * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO
1190 * @lpfc_pnvme: Pointer to the driver's nvme instance data
1191 * @lpfc_nvme_lport: Pointer to the driver's local port data
1192 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1193 * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1194 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1196 * Driver registers this routine as it io request handler. This
1197 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1198 * data structure to the rport
1199 indicated in @lpfc_nvme_rport.
1203 * TODO: What are the failure codes.
1206 lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port
*pnvme_lport
,
1207 struct nvme_fc_remote_port
*pnvme_rport
,
1208 void *hw_queue_handle
,
1209 struct nvmefc_fcp_req
*pnvme_fcreq
)
1212 struct lpfc_nvme_lport
*lport
;
1213 struct lpfc_vport
*vport
;
1214 struct lpfc_hba
*phba
;
1215 struct lpfc_nodelist
*ndlp
;
1216 struct lpfc_nvme_buf
*lpfc_ncmd
;
1217 struct lpfc_nvme_rport
*rport
;
1218 struct lpfc_nvme_qhandle
*lpfc_queue_info
;
1219 struct lpfc_nvme_fcpreq_priv
*freqpriv
= pnvme_fcreq
->private;
1220 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1224 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
1225 vport
= lport
->vport
;
1228 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1230 start
= ktime_get_ns();
1232 rport
= (struct lpfc_nvme_rport
*)pnvme_rport
->private;
1233 lpfc_queue_info
= (struct lpfc_nvme_qhandle
*)hw_queue_handle
;
1236 * Catch race where our node has transitioned, but the
1237 * transport is still transitioning.
1240 if (!ndlp
|| !NLP_CHK_NODE_ACT(ndlp
)) {
1241 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
1242 "6053 rport %p, ndlp %p, DID x%06x "
1243 "ndlp not ready.\n",
1244 rport
, ndlp
, pnvme_rport
->port_id
);
1246 ndlp
= lpfc_findnode_did(vport
, pnvme_rport
->port_id
);
1248 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
1249 "6066 Missing node for DID %x\n",
1250 pnvme_rport
->port_id
);
1256 /* The remote node has to be a mapped target or it's an error. */
1257 if ((ndlp
->nlp_type
& NLP_NVME_TARGET
) &&
1258 (ndlp
->nlp_state
!= NLP_STE_MAPPED_NODE
)) {
1259 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
1260 "6036 rport %p, DID x%06x not ready for "
1261 "IO. State x%x, Type x%x\n",
1262 rport
, pnvme_rport
->port_id
,
1263 ndlp
->nlp_state
, ndlp
->nlp_type
);
1269 /* The node is shared with FCP IO, make sure the IO pending count does
1270 * not exceed the programmed depth.
1272 if (atomic_read(&ndlp
->cmd_pending
) >= ndlp
->cmd_qdepth
) {
1277 lpfc_ncmd
= lpfc_get_nvme_buf(phba
, ndlp
);
1278 if (lpfc_ncmd
== NULL
) {
1279 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_IOERR
,
1280 "6065 driver's buffer pool is empty, "
1285 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1286 if (phba
->ktime_on
) {
1287 lpfc_ncmd
->ts_cmd_start
= start
;
1288 lpfc_ncmd
->ts_last_cmd
= phba
->ktime_last_cmd
;
1293 * Store the data needed by the driver to issue, abort, and complete
1295 * Do not let the IO hang out forever. There is no midlayer issuing
1296 * an abort so inform the FW of the maximum IO pending time.
1298 freqpriv
->nvme_buf
= lpfc_ncmd
;
1299 lpfc_ncmd
->nvmeCmd
= pnvme_fcreq
;
1300 lpfc_ncmd
->nrport
= rport
;
1301 lpfc_ncmd
->ndlp
= ndlp
;
1302 lpfc_ncmd
->start_time
= jiffies
;
1304 lpfc_nvme_prep_io_cmd(vport
, lpfc_ncmd
, ndlp
);
1305 ret
= lpfc_nvme_prep_io_dma(vport
, lpfc_ncmd
);
1308 goto out_free_nvme_buf
;
1311 atomic_inc(&ndlp
->cmd_pending
);
1314 * Issue the IO on the WQ indicated by index in the hw_queue_handle.
1315 * This identfier was create in our hardware queue create callback
1316 * routine. The driver now is dependent on the IO queue steering from
1317 * the transport. We are trusting the upper NVME layers know which
1318 * index to use and that they have affinitized a CPU to this hardware
1319 * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ.
1321 lpfc_ncmd
->cur_iocbq
.hba_wqidx
= lpfc_queue_info
->index
;
1323 lpfc_nvmeio_data(phba
, "NVME FCP XMIT: xri x%x idx %d to %06x\n",
1324 lpfc_ncmd
->cur_iocbq
.sli4_xritag
,
1325 lpfc_queue_info
->index
, ndlp
->nlp_DID
);
1327 ret
= lpfc_sli4_issue_wqe(phba
, LPFC_FCP_RING
, &lpfc_ncmd
->cur_iocbq
);
1329 atomic_dec(&ndlp
->cmd_pending
);
1330 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
1331 "6113 FCP could not issue WQE err %x "
1332 "sid: x%x did: x%x oxid: x%x\n",
1333 ret
, vport
->fc_myDID
, ndlp
->nlp_DID
,
1334 lpfc_ncmd
->cur_iocbq
.sli4_xritag
);
1335 goto out_free_nvme_buf
;
1338 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1340 lpfc_ncmd
->ts_cmd_wqput
= ktime_get_ns();
1342 if (phba
->cpucheck_on
& LPFC_CHECK_NVME_IO
) {
1343 lpfc_ncmd
->cpu
= smp_processor_id();
1344 if (lpfc_ncmd
->cpu
!= lpfc_queue_info
->index
) {
1345 /* Check for admin queue */
1346 if (lpfc_queue_info
->qidx
) {
1347 lpfc_printf_vlog(vport
,
1348 KERN_ERR
, LOG_NVME_IOERR
,
1349 "6702 CPU Check cmd: "
1352 lpfc_queue_info
->index
);
1354 lpfc_ncmd
->cpu
= lpfc_queue_info
->index
;
1356 if (lpfc_ncmd
->cpu
< LPFC_CHECK_CPU_CNT
)
1357 phba
->cpucheck_xmt_io
[lpfc_ncmd
->cpu
]++;
1363 if (lpfc_ncmd
->nvmeCmd
->sg_cnt
) {
1364 if (lpfc_ncmd
->nvmeCmd
->io_dir
== NVMEFC_FCP_WRITE
)
1365 atomic_dec(&phba
->fc4NvmeOutputRequests
);
1367 atomic_dec(&phba
->fc4NvmeInputRequests
);
1369 atomic_dec(&phba
->fc4NvmeControlRequests
);
1370 lpfc_release_nvme_buf(phba
, lpfc_ncmd
);
1376 * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request.
1377 * @phba: Pointer to HBA context object
1378 * @cmdiocb: Pointer to command iocb object.
1379 * @rspiocb: Pointer to response iocb object.
1381 * This is the callback function for any NVME FCP IO that was aborted.
1387 lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba
*phba
, struct lpfc_iocbq
*cmdiocb
,
1388 struct lpfc_wcqe_complete
*abts_cmpl
)
1390 lpfc_printf_log(phba
, KERN_ERR
, LOG_NVME
,
1391 "6145 ABORT_XRI_CN completing on rpi x%x "
1392 "original iotag x%x, abort cmd iotag x%x "
1393 "req_tag x%x, status x%x, hwstatus x%x\n",
1394 cmdiocb
->iocb
.un
.acxri
.abortContextTag
,
1395 cmdiocb
->iocb
.un
.acxri
.abortIoTag
,
1397 bf_get(lpfc_wcqe_c_request_tag
, abts_cmpl
),
1398 bf_get(lpfc_wcqe_c_status
, abts_cmpl
),
1399 bf_get(lpfc_wcqe_c_hw_status
, abts_cmpl
));
1400 lpfc_sli_release_iocbq(phba
, cmdiocb
);
1404 * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS
1405 * @lpfc_pnvme: Pointer to the driver's nvme instance data
1406 * @lpfc_nvme_lport: Pointer to the driver's local port data
1407 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1408 * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1409 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1411 * Driver registers this routine as its nvme request io abort handler. This
1412 * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq
1413 * data structure to the rport indicated in @lpfc_nvme_rport. This routine
1414 * is executed asynchronously - one the target is validated as "MAPPED" and
1415 * ready for IO, the driver issues the abort request and returns.
1421 lpfc_nvme_fcp_abort(struct nvme_fc_local_port
*pnvme_lport
,
1422 struct nvme_fc_remote_port
*pnvme_rport
,
1423 void *hw_queue_handle
,
1424 struct nvmefc_fcp_req
*pnvme_fcreq
)
1426 struct lpfc_nvme_lport
*lport
;
1427 struct lpfc_vport
*vport
;
1428 struct lpfc_hba
*phba
;
1429 struct lpfc_nvme_rport
*rport
;
1430 struct lpfc_nvme_buf
*lpfc_nbuf
;
1431 struct lpfc_iocbq
*abts_buf
;
1432 struct lpfc_iocbq
*nvmereq_wqe
;
1433 struct lpfc_nvme_fcpreq_priv
*freqpriv
= pnvme_fcreq
->private;
1434 union lpfc_wqe
*abts_wqe
;
1435 unsigned long flags
;
1438 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
1439 rport
= (struct lpfc_nvme_rport
*)pnvme_rport
->private;
1440 vport
= lport
->vport
;
1443 /* Announce entry to new IO submit field. */
1444 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_ABTS
,
1445 "6002 Abort Request to rport DID x%06x "
1446 "for nvme_fc_req %p\n",
1447 pnvme_rport
->port_id
,
1450 /* If the hba is getting reset, this flag is set. It is
1451 * cleared when the reset is complete and rings reestablished.
1453 spin_lock_irqsave(&phba
->hbalock
, flags
);
1454 /* driver queued commands are in process of being flushed */
1455 if (phba
->hba_flag
& HBA_NVME_IOQ_FLUSH
) {
1456 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1457 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1458 "6139 Driver in reset cleanup - flushing "
1459 "NVME Req now. hba_flag x%x\n",
1464 lpfc_nbuf
= freqpriv
->nvme_buf
;
1466 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1467 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1468 "6140 NVME IO req has no matching lpfc nvme "
1469 "io buffer. Skipping abort req.\n");
1471 } else if (!lpfc_nbuf
->nvmeCmd
) {
1472 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1473 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1474 "6141 lpfc NVME IO req has no nvme_fcreq "
1475 "io buffer. Skipping abort req.\n");
1478 nvmereq_wqe
= &lpfc_nbuf
->cur_iocbq
;
1481 * The lpfc_nbuf and the mapped nvme_fcreq in the driver's
1482 * state must match the nvme_fcreq passed by the nvme
1483 * transport. If they don't match, it is likely the driver
1484 * has already completed the NVME IO and the nvme transport
1485 * has not seen it yet.
1487 if (lpfc_nbuf
->nvmeCmd
!= pnvme_fcreq
) {
1488 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1489 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1490 "6143 NVME req mismatch: "
1491 "lpfc_nbuf %p nvmeCmd %p, "
1492 "pnvme_fcreq %p. Skipping Abort xri x%x\n",
1493 lpfc_nbuf
, lpfc_nbuf
->nvmeCmd
,
1494 pnvme_fcreq
, nvmereq_wqe
->sli4_xritag
);
1498 /* Don't abort IOs no longer on the pending queue. */
1499 if (!(nvmereq_wqe
->iocb_flag
& LPFC_IO_ON_TXCMPLQ
)) {
1500 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1501 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1502 "6142 NVME IO req %p not queued - skipping "
1503 "abort req xri x%x\n",
1504 pnvme_fcreq
, nvmereq_wqe
->sli4_xritag
);
1508 lpfc_nvmeio_data(phba
, "NVME FCP ABORT: xri x%x idx %d to %06x\n",
1509 nvmereq_wqe
->sli4_xritag
,
1510 nvmereq_wqe
->hba_wqidx
, pnvme_rport
->port_id
);
1512 /* Outstanding abort is in progress */
1513 if (nvmereq_wqe
->iocb_flag
& LPFC_DRIVER_ABORTED
) {
1514 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1515 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1516 "6144 Outstanding NVME I/O Abort Request "
1517 "still pending on nvme_fcreq %p, "
1518 "lpfc_ncmd %p xri x%x\n",
1519 pnvme_fcreq
, lpfc_nbuf
,
1520 nvmereq_wqe
->sli4_xritag
);
1524 abts_buf
= __lpfc_sli_get_iocbq(phba
);
1526 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1527 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1528 "6136 No available abort wqes. Skipping "
1529 "Abts req for nvme_fcreq %p xri x%x\n",
1530 pnvme_fcreq
, nvmereq_wqe
->sli4_xritag
);
1534 /* Ready - mark outstanding as aborted by driver. */
1535 nvmereq_wqe
->iocb_flag
|= LPFC_DRIVER_ABORTED
;
1537 /* Complete prepping the abort wqe and issue to the FW. */
1538 abts_wqe
= &abts_buf
->wqe
;
1540 /* WQEs are reused. Clear stale data and set key fields to
1541 * zero like ia, iaab, iaar, xri_tag, and ctxt_tag.
1543 memset(abts_wqe
, 0, sizeof(union lpfc_wqe
));
1544 bf_set(abort_cmd_criteria
, &abts_wqe
->abort_cmd
, T_XRI_TAG
);
1547 bf_set(wqe_ct
, &abts_wqe
->abort_cmd
.wqe_com
, 0);
1548 bf_set(wqe_cmnd
, &abts_wqe
->abort_cmd
.wqe_com
, CMD_ABORT_XRI_CX
);
1549 bf_set(wqe_class
, &abts_wqe
->abort_cmd
.wqe_com
,
1550 nvmereq_wqe
->iocb
.ulpClass
);
1552 /* word 8 - tell the FW to abort the IO associated with this
1553 * outstanding exchange ID.
1555 abts_wqe
->abort_cmd
.wqe_com
.abort_tag
= nvmereq_wqe
->sli4_xritag
;
1557 /* word 9 - this is the iotag for the abts_wqe completion. */
1558 bf_set(wqe_reqtag
, &abts_wqe
->abort_cmd
.wqe_com
,
1562 bf_set(wqe_wqid
, &abts_wqe
->abort_cmd
.wqe_com
, nvmereq_wqe
->hba_wqidx
);
1563 bf_set(wqe_qosd
, &abts_wqe
->abort_cmd
.wqe_com
, 1);
1564 bf_set(wqe_lenloc
, &abts_wqe
->abort_cmd
.wqe_com
, LPFC_WQE_LENLOC_NONE
);
1567 bf_set(wqe_cmd_type
, &abts_wqe
->abort_cmd
.wqe_com
, OTHER_COMMAND
);
1568 bf_set(wqe_wqec
, &abts_wqe
->abort_cmd
.wqe_com
, 1);
1569 bf_set(wqe_cqid
, &abts_wqe
->abort_cmd
.wqe_com
, LPFC_WQE_CQ_ID_DEFAULT
);
1571 /* ABTS WQE must go to the same WQ as the WQE to be aborted */
1572 abts_buf
->iocb_flag
|= LPFC_IO_NVME
;
1573 abts_buf
->hba_wqidx
= nvmereq_wqe
->hba_wqidx
;
1574 abts_buf
->vport
= vport
;
1575 abts_buf
->wqe_cmpl
= lpfc_nvme_abort_fcreq_cmpl
;
1576 ret_val
= lpfc_sli4_issue_wqe(phba
, LPFC_FCP_RING
, abts_buf
);
1577 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1579 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1580 "6137 Failed abts issue_wqe with status x%x "
1581 "for nvme_fcreq %p.\n",
1582 ret_val
, pnvme_fcreq
);
1583 lpfc_sli_release_iocbq(phba
, abts_buf
);
1587 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_ABTS
,
1588 "6138 Transport Abort NVME Request Issued for "
1589 "ox_id x%x on reqtag x%x\n",
1590 nvmereq_wqe
->sli4_xritag
,
1594 /* Declare and initialization an instance of the FC NVME template. */
1595 static struct nvme_fc_port_template lpfc_nvme_template
= {
1596 /* initiator-based functions */
1597 .localport_delete
= lpfc_nvme_localport_delete
,
1598 .remoteport_delete
= lpfc_nvme_remoteport_delete
,
1599 .create_queue
= lpfc_nvme_create_queue
,
1600 .delete_queue
= lpfc_nvme_delete_queue
,
1601 .ls_req
= lpfc_nvme_ls_req
,
1602 .fcp_io
= lpfc_nvme_fcp_io_submit
,
1603 .ls_abort
= lpfc_nvme_ls_abort
,
1604 .fcp_abort
= lpfc_nvme_fcp_abort
,
1607 .max_sgl_segments
= LPFC_NVME_DEFAULT_SEGS
,
1608 .max_dif_sgl_segments
= LPFC_NVME_DEFAULT_SEGS
,
1609 .dma_boundary
= 0xFFFFFFFF,
1611 /* Sizes of additional private data for data structures.
1612 * No use for the last two sizes at this time.
1614 .local_priv_sz
= sizeof(struct lpfc_nvme_lport
),
1615 .remote_priv_sz
= sizeof(struct lpfc_nvme_rport
),
1616 .lsrqst_priv_sz
= 0,
1617 .fcprqst_priv_sz
= sizeof(struct lpfc_nvme_fcpreq_priv
),
1621 * lpfc_sli4_post_nvme_sgl_block - post a block of nvme sgl list to firmware
1622 * @phba: pointer to lpfc hba data structure.
1623 * @nblist: pointer to nvme buffer list.
1624 * @count: number of scsi buffers on the list.
1626 * This routine is invoked to post a block of @count scsi sgl pages from a
1627 * SCSI buffer list @nblist to the HBA using non-embedded mailbox command.
1632 lpfc_sli4_post_nvme_sgl_block(struct lpfc_hba
*phba
,
1633 struct list_head
*nblist
,
1636 struct lpfc_nvme_buf
*lpfc_ncmd
;
1637 struct lpfc_mbx_post_uembed_sgl_page1
*sgl
;
1638 struct sgl_page_pairs
*sgl_pg_pairs
;
1641 uint32_t reqlen
, alloclen
, pg_pairs
;
1643 uint16_t xritag_start
= 0;
1645 uint32_t shdr_status
, shdr_add_status
;
1646 dma_addr_t pdma_phys_bpl1
;
1647 union lpfc_sli4_cfg_shdr
*shdr
;
1649 /* Calculate the requested length of the dma memory */
1650 reqlen
= count
* sizeof(struct sgl_page_pairs
) +
1651 sizeof(union lpfc_sli4_cfg_shdr
) + sizeof(uint32_t);
1652 if (reqlen
> SLI4_PAGE_SIZE
) {
1653 lpfc_printf_log(phba
, KERN_WARNING
, LOG_INIT
,
1654 "6118 Block sgl registration required DMA "
1655 "size (%d) great than a page\n", reqlen
);
1658 mbox
= mempool_alloc(phba
->mbox_mem_pool
, GFP_KERNEL
);
1660 lpfc_printf_log(phba
, KERN_ERR
, LOG_INIT
,
1661 "6119 Failed to allocate mbox cmd memory\n");
1665 /* Allocate DMA memory and set up the non-embedded mailbox command */
1666 alloclen
= lpfc_sli4_config(phba
, mbox
, LPFC_MBOX_SUBSYSTEM_FCOE
,
1667 LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES
, reqlen
,
1668 LPFC_SLI4_MBX_NEMBED
);
1670 if (alloclen
< reqlen
) {
1671 lpfc_printf_log(phba
, KERN_ERR
, LOG_INIT
,
1672 "6120 Allocated DMA memory size (%d) is "
1673 "less than the requested DMA memory "
1674 "size (%d)\n", alloclen
, reqlen
);
1675 lpfc_sli4_mbox_cmd_free(phba
, mbox
);
1679 /* Get the first SGE entry from the non-embedded DMA memory */
1680 viraddr
= mbox
->sge_array
->addr
[0];
1682 /* Set up the SGL pages in the non-embedded DMA pages */
1683 sgl
= (struct lpfc_mbx_post_uembed_sgl_page1
*)viraddr
;
1684 sgl_pg_pairs
= &sgl
->sgl_pg_pairs
;
1687 list_for_each_entry(lpfc_ncmd
, nblist
, list
) {
1688 /* Set up the sge entry */
1689 sgl_pg_pairs
->sgl_pg0_addr_lo
=
1690 cpu_to_le32(putPaddrLow(lpfc_ncmd
->dma_phys_sgl
));
1691 sgl_pg_pairs
->sgl_pg0_addr_hi
=
1692 cpu_to_le32(putPaddrHigh(lpfc_ncmd
->dma_phys_sgl
));
1693 if (phba
->cfg_sg_dma_buf_size
> SGL_PAGE_SIZE
)
1694 pdma_phys_bpl1
= lpfc_ncmd
->dma_phys_sgl
+
1698 sgl_pg_pairs
->sgl_pg1_addr_lo
=
1699 cpu_to_le32(putPaddrLow(pdma_phys_bpl1
));
1700 sgl_pg_pairs
->sgl_pg1_addr_hi
=
1701 cpu_to_le32(putPaddrHigh(pdma_phys_bpl1
));
1702 /* Keep the first xritag on the list */
1704 xritag_start
= lpfc_ncmd
->cur_iocbq
.sli4_xritag
;
1708 bf_set(lpfc_post_sgl_pages_xri
, sgl
, xritag_start
);
1709 bf_set(lpfc_post_sgl_pages_xricnt
, sgl
, pg_pairs
);
1710 /* Perform endian conversion if necessary */
1711 sgl
->word0
= cpu_to_le32(sgl
->word0
);
1713 if (!phba
->sli4_hba
.intr_enable
)
1714 rc
= lpfc_sli_issue_mbox(phba
, mbox
, MBX_POLL
);
1716 mbox_tmo
= lpfc_mbox_tmo_val(phba
, mbox
);
1717 rc
= lpfc_sli_issue_mbox_wait(phba
, mbox
, mbox_tmo
);
1719 shdr
= (union lpfc_sli4_cfg_shdr
*)&sgl
->cfg_shdr
;
1720 shdr_status
= bf_get(lpfc_mbox_hdr_status
, &shdr
->response
);
1721 shdr_add_status
= bf_get(lpfc_mbox_hdr_add_status
, &shdr
->response
);
1722 if (rc
!= MBX_TIMEOUT
)
1723 lpfc_sli4_mbox_cmd_free(phba
, mbox
);
1724 if (shdr_status
|| shdr_add_status
|| rc
) {
1725 lpfc_printf_log(phba
, KERN_ERR
, LOG_SLI
,
1726 "6125 POST_SGL_BLOCK mailbox command failed "
1727 "status x%x add_status x%x mbx status x%x\n",
1728 shdr_status
, shdr_add_status
, rc
);
1735 * lpfc_post_nvme_sgl_list - Post blocks of nvme buffer sgls from a list
1736 * @phba: pointer to lpfc hba data structure.
1737 * @post_nblist: pointer to the nvme buffer list.
1739 * This routine walks a list of nvme buffers that was passed in. It attempts
1740 * to construct blocks of nvme buffer sgls which contains contiguous xris and
1741 * uses the non-embedded SGL block post mailbox commands to post to the port.
1742 * For single NVME buffer sgl with non-contiguous xri, if any, it shall use
1743 * embedded SGL post mailbox command for posting. The @post_nblist passed in
1744 * must be local list, thus no lock is needed when manipulate the list.
1746 * Returns: 0 = failure, non-zero number of successfully posted buffers.
1749 lpfc_post_nvme_sgl_list(struct lpfc_hba
*phba
,
1750 struct list_head
*post_nblist
, int sb_count
)
1752 struct lpfc_nvme_buf
*lpfc_ncmd
, *lpfc_ncmd_next
;
1753 int status
, sgl_size
;
1754 int post_cnt
= 0, block_cnt
= 0, num_posting
= 0, num_posted
= 0;
1755 dma_addr_t pdma_phys_sgl1
;
1756 int last_xritag
= NO_XRI
;
1758 LIST_HEAD(prep_nblist
);
1759 LIST_HEAD(blck_nblist
);
1760 LIST_HEAD(nvme_nblist
);
1766 sgl_size
= phba
->cfg_sg_dma_buf_size
;
1768 list_for_each_entry_safe(lpfc_ncmd
, lpfc_ncmd_next
, post_nblist
, list
) {
1769 list_del_init(&lpfc_ncmd
->list
);
1771 if ((last_xritag
!= NO_XRI
) &&
1772 (lpfc_ncmd
->cur_iocbq
.sli4_xritag
!= last_xritag
+ 1)) {
1773 /* a hole in xri block, form a sgl posting block */
1774 list_splice_init(&prep_nblist
, &blck_nblist
);
1775 post_cnt
= block_cnt
- 1;
1776 /* prepare list for next posting block */
1777 list_add_tail(&lpfc_ncmd
->list
, &prep_nblist
);
1780 /* prepare list for next posting block */
1781 list_add_tail(&lpfc_ncmd
->list
, &prep_nblist
);
1782 /* enough sgls for non-embed sgl mbox command */
1783 if (block_cnt
== LPFC_NEMBED_MBOX_SGL_CNT
) {
1784 list_splice_init(&prep_nblist
, &blck_nblist
);
1785 post_cnt
= block_cnt
;
1790 last_xritag
= lpfc_ncmd
->cur_iocbq
.sli4_xritag
;
1792 /* end of repost sgl list condition for NVME buffers */
1793 if (num_posting
== sb_count
) {
1794 if (post_cnt
== 0) {
1795 /* last sgl posting block */
1796 list_splice_init(&prep_nblist
, &blck_nblist
);
1797 post_cnt
= block_cnt
;
1798 } else if (block_cnt
== 1) {
1799 /* last single sgl with non-contiguous xri */
1800 if (sgl_size
> SGL_PAGE_SIZE
)
1802 lpfc_ncmd
->dma_phys_sgl
+
1806 cur_xritag
= lpfc_ncmd
->cur_iocbq
.sli4_xritag
;
1807 status
= lpfc_sli4_post_sgl(phba
,
1808 lpfc_ncmd
->dma_phys_sgl
,
1809 pdma_phys_sgl1
, cur_xritag
);
1811 /* failure, put on abort nvme list */
1812 lpfc_ncmd
->flags
|= LPFC_SBUF_XBUSY
;
1814 /* success, put on NVME buffer list */
1815 lpfc_ncmd
->flags
&= ~LPFC_SBUF_XBUSY
;
1816 lpfc_ncmd
->status
= IOSTAT_SUCCESS
;
1819 /* success, put on NVME buffer sgl list */
1820 list_add_tail(&lpfc_ncmd
->list
, &nvme_nblist
);
1824 /* continue until a nembed page worth of sgls */
1828 /* post block of NVME buffer list sgls */
1829 status
= lpfc_sli4_post_nvme_sgl_block(phba
, &blck_nblist
,
1832 /* don't reset xirtag due to hole in xri block */
1834 last_xritag
= NO_XRI
;
1836 /* reset NVME buffer post count for next round of posting */
1839 /* put posted NVME buffer-sgl posted on NVME buffer sgl list */
1840 while (!list_empty(&blck_nblist
)) {
1841 list_remove_head(&blck_nblist
, lpfc_ncmd
,
1842 struct lpfc_nvme_buf
, list
);
1844 /* failure, put on abort nvme list */
1845 lpfc_ncmd
->flags
|= LPFC_SBUF_XBUSY
;
1847 /* success, put on NVME buffer list */
1848 lpfc_ncmd
->flags
&= ~LPFC_SBUF_XBUSY
;
1849 lpfc_ncmd
->status
= IOSTAT_SUCCESS
;
1852 list_add_tail(&lpfc_ncmd
->list
, &nvme_nblist
);
1855 /* Push NVME buffers with sgl posted to the available list */
1856 while (!list_empty(&nvme_nblist
)) {
1857 list_remove_head(&nvme_nblist
, lpfc_ncmd
,
1858 struct lpfc_nvme_buf
, list
);
1859 lpfc_release_nvme_buf(phba
, lpfc_ncmd
);
1865 * lpfc_repost_nvme_sgl_list - Repost all the allocated nvme buffer sgls
1866 * @phba: pointer to lpfc hba data structure.
1868 * This routine walks the list of nvme buffers that have been allocated and
1869 * repost them to the port by using SGL block post. This is needed after a
1870 * pci_function_reset/warm_start or start. The lpfc_hba_down_post_s4 routine
1871 * is responsible for moving all nvme buffers on the lpfc_abts_nvme_sgl_list
1872 * to the lpfc_nvme_buf_list. If the repost fails, reject all nvme buffers.
1874 * Returns: 0 = success, non-zero failure.
1877 lpfc_repost_nvme_sgl_list(struct lpfc_hba
*phba
)
1879 LIST_HEAD(post_nblist
);
1880 int num_posted
, rc
= 0;
1882 /* get all NVME buffers need to repost to a local list */
1883 spin_lock_irq(&phba
->nvme_buf_list_get_lock
);
1884 spin_lock(&phba
->nvme_buf_list_put_lock
);
1885 list_splice_init(&phba
->lpfc_nvme_buf_list_get
, &post_nblist
);
1886 list_splice(&phba
->lpfc_nvme_buf_list_put
, &post_nblist
);
1887 spin_unlock(&phba
->nvme_buf_list_put_lock
);
1888 spin_unlock_irq(&phba
->nvme_buf_list_get_lock
);
1890 /* post the list of nvme buffer sgls to port if available */
1891 if (!list_empty(&post_nblist
)) {
1892 num_posted
= lpfc_post_nvme_sgl_list(phba
, &post_nblist
,
1893 phba
->sli4_hba
.nvme_xri_cnt
);
1894 /* failed to post any nvme buffer, return error */
1895 if (num_posted
== 0)
1902 * lpfc_new_nvme_buf - Scsi buffer allocator for HBA with SLI4 IF spec
1903 * @vport: The virtual port for which this call being executed.
1904 * @num_to_allocate: The requested number of buffers to allocate.
1906 * This routine allocates nvme buffers for device with SLI-4 interface spec,
1907 * the nvme buffer contains all the necessary information needed to initiate
1908 * a NVME I/O. After allocating up to @num_to_allocate NVME buffers and put
1909 * them on a list, it post them to the port by using SGL block post.
1912 * int - number of nvme buffers that were allocated and posted.
1913 * 0 = failure, less than num_to_alloc is a partial failure.
1916 lpfc_new_nvme_buf(struct lpfc_vport
*vport
, int num_to_alloc
)
1918 struct lpfc_hba
*phba
= vport
->phba
;
1919 struct lpfc_nvme_buf
*lpfc_ncmd
;
1920 struct lpfc_iocbq
*pwqeq
;
1921 union lpfc_wqe128
*wqe
;
1922 struct sli4_sge
*sgl
;
1923 dma_addr_t pdma_phys_sgl
;
1924 uint16_t iotag
, lxri
= 0;
1925 int bcnt
, num_posted
, sgl_size
;
1926 LIST_HEAD(prep_nblist
);
1927 LIST_HEAD(post_nblist
);
1928 LIST_HEAD(nvme_nblist
);
1930 sgl_size
= phba
->cfg_sg_dma_buf_size
;
1932 for (bcnt
= 0; bcnt
< num_to_alloc
; bcnt
++) {
1933 lpfc_ncmd
= kzalloc(sizeof(struct lpfc_nvme_buf
), GFP_KERNEL
);
1937 * Get memory from the pci pool to map the virt space to
1938 * pci bus space for an I/O. The DMA buffer includes the
1939 * number of SGE's necessary to support the sg_tablesize.
1941 lpfc_ncmd
->data
= dma_pool_alloc(phba
->lpfc_sg_dma_buf_pool
,
1943 &lpfc_ncmd
->dma_handle
);
1944 if (!lpfc_ncmd
->data
) {
1948 memset(lpfc_ncmd
->data
, 0, phba
->cfg_sg_dma_buf_size
);
1950 lxri
= lpfc_sli4_next_xritag(phba
);
1951 if (lxri
== NO_XRI
) {
1952 dma_pool_free(phba
->lpfc_sg_dma_buf_pool
,
1953 lpfc_ncmd
->data
, lpfc_ncmd
->dma_handle
);
1957 pwqeq
= &(lpfc_ncmd
->cur_iocbq
);
1958 wqe
= (union lpfc_wqe128
*)&pwqeq
->wqe
;
1960 /* Allocate iotag for lpfc_ncmd->cur_iocbq. */
1961 iotag
= lpfc_sli_next_iotag(phba
, pwqeq
);
1963 dma_pool_free(phba
->lpfc_sg_dma_buf_pool
,
1964 lpfc_ncmd
->data
, lpfc_ncmd
->dma_handle
);
1966 lpfc_printf_log(phba
, KERN_ERR
, LOG_NVME_IOERR
,
1967 "6121 Failed to allocated IOTAG for"
1968 " XRI:0x%x\n", lxri
);
1969 lpfc_sli4_free_xri(phba
, lxri
);
1972 pwqeq
->sli4_lxritag
= lxri
;
1973 pwqeq
->sli4_xritag
= phba
->sli4_hba
.xri_ids
[lxri
];
1974 pwqeq
->iocb_flag
|= LPFC_IO_NVME
;
1975 pwqeq
->context1
= lpfc_ncmd
;
1976 pwqeq
->wqe_cmpl
= lpfc_nvme_io_cmd_wqe_cmpl
;
1978 /* Initialize local short-hand pointers. */
1979 lpfc_ncmd
->nvme_sgl
= lpfc_ncmd
->data
;
1980 sgl
= lpfc_ncmd
->nvme_sgl
;
1981 pdma_phys_sgl
= lpfc_ncmd
->dma_handle
;
1982 lpfc_ncmd
->dma_phys_sgl
= pdma_phys_sgl
;
1984 /* Rsp SGE will be filled in when we rcv an IO
1985 * from the NVME Layer to be sent.
1986 * The cmd is going to be embedded so we need a SKIP SGE.
1988 bf_set(lpfc_sli4_sge_type
, sgl
, LPFC_SGE_TYPE_SKIP
);
1989 bf_set(lpfc_sli4_sge_last
, sgl
, 0);
1990 sgl
->word2
= cpu_to_le32(sgl
->word2
);
1991 /* Fill in word 3 / sgl_len during cmd submission */
1993 lpfc_ncmd
->cur_iocbq
.context1
= lpfc_ncmd
;
1996 bf_set(wqe_erp
, &wqe
->generic
.wqe_com
, 0);
1997 /* NVME upper layers will time things out, if needed */
1998 bf_set(wqe_tmo
, &wqe
->generic
.wqe_com
, 0);
2001 bf_set(wqe_ebde_cnt
, &wqe
->generic
.wqe_com
, 0);
2002 bf_set(wqe_dbde
, &wqe
->generic
.wqe_com
, 1);
2004 /* add the nvme buffer to a post list */
2005 list_add_tail(&lpfc_ncmd
->list
, &post_nblist
);
2006 spin_lock_irq(&phba
->nvme_buf_list_get_lock
);
2007 phba
->sli4_hba
.nvme_xri_cnt
++;
2008 spin_unlock_irq(&phba
->nvme_buf_list_get_lock
);
2010 lpfc_printf_log(phba
, KERN_INFO
, LOG_NVME
,
2011 "6114 Allocate %d out of %d requested new NVME "
2012 "buffers\n", bcnt
, num_to_alloc
);
2014 /* post the list of nvme buffer sgls to port if available */
2015 if (!list_empty(&post_nblist
))
2016 num_posted
= lpfc_post_nvme_sgl_list(phba
,
2017 &post_nblist
, bcnt
);
2025 * lpfc_get_nvme_buf - Get a nvme buffer from lpfc_nvme_buf_list of the HBA
2026 * @phba: The HBA for which this call is being executed.
2028 * This routine removes a nvme buffer from head of @phba lpfc_nvme_buf_list list
2029 * and returns to caller.
2033 * Pointer to lpfc_nvme_buf - Success
2035 static struct lpfc_nvme_buf
*
2036 lpfc_get_nvme_buf(struct lpfc_hba
*phba
, struct lpfc_nodelist
*ndlp
)
2038 struct lpfc_nvme_buf
*lpfc_ncmd
, *lpfc_ncmd_next
;
2039 unsigned long iflag
= 0;
2042 spin_lock_irqsave(&phba
->nvme_buf_list_get_lock
, iflag
);
2043 list_for_each_entry_safe(lpfc_ncmd
, lpfc_ncmd_next
,
2044 &phba
->lpfc_nvme_buf_list_get
, list
) {
2045 if (lpfc_test_rrq_active(phba
, ndlp
,
2046 lpfc_ncmd
->cur_iocbq
.sli4_lxritag
))
2048 list_del_init(&lpfc_ncmd
->list
);
2053 spin_lock(&phba
->nvme_buf_list_put_lock
);
2054 list_splice(&phba
->lpfc_nvme_buf_list_put
,
2055 &phba
->lpfc_nvme_buf_list_get
);
2056 INIT_LIST_HEAD(&phba
->lpfc_nvme_buf_list_put
);
2057 spin_unlock(&phba
->nvme_buf_list_put_lock
);
2058 list_for_each_entry_safe(lpfc_ncmd
, lpfc_ncmd_next
,
2059 &phba
->lpfc_nvme_buf_list_get
, list
) {
2060 if (lpfc_test_rrq_active(
2061 phba
, ndlp
, lpfc_ncmd
->cur_iocbq
.sli4_lxritag
))
2063 list_del_init(&lpfc_ncmd
->list
);
2068 spin_unlock_irqrestore(&phba
->nvme_buf_list_get_lock
, iflag
);
2075 * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list.
2076 * @phba: The Hba for which this call is being executed.
2077 * @lpfc_ncmd: The nvme buffer which is being released.
2079 * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba
2080 * lpfc_nvme_buf_list list. For SLI4 XRI's are tied to the nvme buffer
2081 * and cannot be reused for at least RA_TOV amount of time if it was
2085 lpfc_release_nvme_buf(struct lpfc_hba
*phba
, struct lpfc_nvme_buf
*lpfc_ncmd
)
2087 unsigned long iflag
= 0;
2089 lpfc_ncmd
->nonsg_phys
= 0;
2090 if (lpfc_ncmd
->flags
& LPFC_SBUF_XBUSY
) {
2091 lpfc_printf_log(phba
, KERN_INFO
, LOG_NVME_ABTS
,
2092 "6310 XB release deferred for "
2093 "ox_id x%x on reqtag x%x\n",
2094 lpfc_ncmd
->cur_iocbq
.sli4_xritag
,
2095 lpfc_ncmd
->cur_iocbq
.iotag
);
2097 spin_lock_irqsave(&phba
->sli4_hba
.abts_nvme_buf_list_lock
,
2099 lpfc_ncmd
->nvmeCmd
= NULL
;
2100 list_add_tail(&lpfc_ncmd
->list
,
2101 &phba
->sli4_hba
.lpfc_abts_nvme_buf_list
);
2102 spin_unlock_irqrestore(&phba
->sli4_hba
.abts_nvme_buf_list_lock
,
2105 lpfc_ncmd
->nvmeCmd
= NULL
;
2106 lpfc_ncmd
->cur_iocbq
.iocb_flag
= LPFC_IO_NVME
;
2107 spin_lock_irqsave(&phba
->nvme_buf_list_put_lock
, iflag
);
2108 list_add_tail(&lpfc_ncmd
->list
, &phba
->lpfc_nvme_buf_list_put
);
2109 spin_unlock_irqrestore(&phba
->nvme_buf_list_put_lock
, iflag
);
2114 * lpfc_nvme_create_localport - Create/Bind an nvme localport instance.
2115 * @pvport - the lpfc_vport instance requesting a localport.
2117 * This routine is invoked to create an nvme localport instance to bind
2118 * to the nvme_fc_transport. It is called once during driver load
2119 * like lpfc_create_shost after all other services are initialized.
2120 * It requires a vport, vpi, and wwns at call time. Other localport
2121 * parameters are modified as the driver's FCID and the Fabric WWN
2126 * -ENOMEM - no heap memory available
2127 * other values - from nvme registration upcall
2130 lpfc_nvme_create_localport(struct lpfc_vport
*vport
)
2133 struct lpfc_hba
*phba
= vport
->phba
;
2134 struct nvme_fc_port_info nfcp_info
;
2135 struct nvme_fc_local_port
*localport
;
2136 struct lpfc_nvme_lport
*lport
;
2139 /* Initialize this localport instance. The vport wwn usage ensures
2140 * that NPIV is accounted for.
2142 memset(&nfcp_info
, 0, sizeof(struct nvme_fc_port_info
));
2143 nfcp_info
.port_role
= FC_PORT_ROLE_NVME_INITIATOR
;
2144 nfcp_info
.node_name
= wwn_to_u64(vport
->fc_nodename
.u
.wwn
);
2145 nfcp_info
.port_name
= wwn_to_u64(vport
->fc_portname
.u
.wwn
);
2147 /* Limit to LPFC_MAX_NVME_SEG_CNT.
2148 * For now need + 1 to get around NVME transport logic.
2150 if (phba
->cfg_sg_seg_cnt
> LPFC_MAX_NVME_SEG_CNT
) {
2151 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
| LOG_INIT
,
2152 "6300 Reducing sg segment cnt to %d\n",
2153 LPFC_MAX_NVME_SEG_CNT
);
2154 phba
->cfg_nvme_seg_cnt
= LPFC_MAX_NVME_SEG_CNT
;
2156 phba
->cfg_nvme_seg_cnt
= phba
->cfg_sg_seg_cnt
;
2158 lpfc_nvme_template
.max_sgl_segments
= phba
->cfg_nvme_seg_cnt
+ 1;
2159 lpfc_nvme_template
.max_hw_queues
= phba
->cfg_nvme_io_channel
;
2161 /* localport is allocated from the stack, but the registration
2162 * call allocates heap memory as well as the private area.
2164 #if (IS_ENABLED(CONFIG_NVME_FC))
2165 ret
= nvme_fc_register_localport(&nfcp_info
, &lpfc_nvme_template
,
2166 &vport
->phba
->pcidev
->dev
, &localport
);
2171 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
| LOG_NVME_DISC
,
2172 "6005 Successfully registered local "
2173 "NVME port num %d, localP %p, private %p, "
2175 localport
->port_num
, localport
,
2177 lpfc_nvme_template
.max_sgl_segments
);
2179 /* Private is our lport size declared in the template. */
2180 lport
= (struct lpfc_nvme_lport
*)localport
->private;
2181 vport
->localport
= localport
;
2182 lport
->vport
= vport
;
2183 vport
->nvmei_support
= 1;
2185 /* Don't post more new bufs if repost already recovered
2188 if (phba
->sli4_hba
.nvme_xri_cnt
== 0) {
2189 len
= lpfc_new_nvme_buf(vport
,
2190 phba
->sli4_hba
.nvme_xri_max
);
2191 vport
->phba
->total_nvme_bufs
+= len
;
2199 * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport.
2200 * @pnvme: pointer to lpfc nvme data structure.
2202 * This routine is invoked to destroy all lports bound to the phba.
2203 * The lport memory was allocated by the nvme fc transport and is
2204 * released there. This routine ensures all rports bound to the
2205 * lport have been disconnected.
2209 lpfc_nvme_destroy_localport(struct lpfc_vport
*vport
)
2211 #if (IS_ENABLED(CONFIG_NVME_FC))
2212 struct nvme_fc_local_port
*localport
;
2213 struct lpfc_nvme_lport
*lport
;
2216 if (vport
->nvmei_support
== 0)
2219 localport
= vport
->localport
;
2220 vport
->localport
= NULL
;
2221 lport
= (struct lpfc_nvme_lport
*)localport
->private;
2223 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
,
2224 "6011 Destroying NVME localport %p\n",
2227 /* lport's rport list is clear. Unregister
2228 * lport and release resources.
2230 init_completion(&lport
->lport_unreg_done
);
2231 ret
= nvme_fc_unregister_localport(localport
);
2232 wait_for_completion_timeout(&lport
->lport_unreg_done
, 5);
2234 /* Regardless of the unregister upcall response, clear
2235 * nvmei_support. All rports are unregistered and the
2236 * driver will clean up.
2238 vport
->nvmei_support
= 0;
2240 lpfc_printf_vlog(vport
,
2241 KERN_INFO
, LOG_NVME_DISC
,
2242 "6009 Unregistered lport Success\n");
2244 lpfc_printf_vlog(vport
,
2245 KERN_INFO
, LOG_NVME_DISC
,
2246 "6010 Unregistered lport "
2247 "Failed, status x%x\n",
2254 lpfc_nvme_update_localport(struct lpfc_vport
*vport
)
2256 #if (IS_ENABLED(CONFIG_NVME_FC))
2257 struct nvme_fc_local_port
*localport
;
2258 struct lpfc_nvme_lport
*lport
;
2260 localport
= vport
->localport
;
2262 lpfc_printf_vlog(vport
, KERN_WARNING
, LOG_NVME
,
2263 "6710 Update NVME fail. No localport\n");
2266 lport
= (struct lpfc_nvme_lport
*)localport
->private;
2268 lpfc_printf_vlog(vport
, KERN_WARNING
, LOG_NVME
,
2269 "6171 Update NVME fail. localP %p, No lport\n",
2273 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
,
2274 "6012 Update NVME lport %p did x%x\n",
2275 localport
, vport
->fc_myDID
);
2277 localport
->port_id
= vport
->fc_myDID
;
2278 if (localport
->port_id
== 0)
2279 localport
->port_role
= FC_PORT_ROLE_NVME_DISCOVERY
;
2281 localport
->port_role
= FC_PORT_ROLE_NVME_INITIATOR
;
2283 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
2284 "6030 bound lport %p to DID x%06x\n",
2285 lport
, localport
->port_id
);
2290 lpfc_nvme_register_port(struct lpfc_vport
*vport
, struct lpfc_nodelist
*ndlp
)
2292 #if (IS_ENABLED(CONFIG_NVME_FC))
2294 struct nvme_fc_local_port
*localport
;
2295 struct lpfc_nvme_lport
*lport
;
2296 struct lpfc_nvme_rport
*rport
;
2297 struct nvme_fc_remote_port
*remote_port
;
2298 struct nvme_fc_port_info rpinfo
;
2300 lpfc_printf_vlog(ndlp
->vport
, KERN_INFO
, LOG_NVME_DISC
,
2301 "6006 Register NVME PORT. DID x%06x nlptype x%x\n",
2302 ndlp
->nlp_DID
, ndlp
->nlp_type
);
2304 localport
= vport
->localport
;
2308 lport
= (struct lpfc_nvme_lport
*)localport
->private;
2310 /* NVME rports are not preserved across devloss.
2311 * Just register this instance. Note, rpinfo->dev_loss_tmo
2312 * is left 0 to indicate accept transport defaults. The
2313 * driver communicates port role capabilities consistent
2314 * with the PRLI response data.
2316 memset(&rpinfo
, 0, sizeof(struct nvme_fc_port_info
));
2317 rpinfo
.port_id
= ndlp
->nlp_DID
;
2318 if (ndlp
->nlp_type
& NLP_NVME_TARGET
)
2319 rpinfo
.port_role
|= FC_PORT_ROLE_NVME_TARGET
;
2320 if (ndlp
->nlp_type
& NLP_NVME_INITIATOR
)
2321 rpinfo
.port_role
|= FC_PORT_ROLE_NVME_INITIATOR
;
2323 if (ndlp
->nlp_type
& NLP_NVME_DISCOVERY
)
2324 rpinfo
.port_role
|= FC_PORT_ROLE_NVME_DISCOVERY
;
2326 rpinfo
.port_name
= wwn_to_u64(ndlp
->nlp_portname
.u
.wwn
);
2327 rpinfo
.node_name
= wwn_to_u64(ndlp
->nlp_nodename
.u
.wwn
);
2328 ret
= nvme_fc_register_remoteport(localport
, &rpinfo
, &remote_port
);
2330 /* If the ndlp already has an nrport, this is just
2331 * a resume of the existing rport. Else this is a
2334 rport
= remote_port
->private;
2335 if (ndlp
->nrport
== rport
) {
2336 lpfc_printf_vlog(ndlp
->vport
, KERN_INFO
,
2338 "6014 Rebinding lport to "
2339 "rport wwpn 0x%llx, "
2340 "Data: x%x x%x x%x x%06x\n",
2341 remote_port
->port_name
,
2342 remote_port
->port_id
,
2343 remote_port
->port_role
,
2348 rport
->remoteport
= remote_port
;
2349 rport
->lport
= lport
;
2350 rport
->ndlp
= lpfc_nlp_get(ndlp
);
2353 ndlp
->nrport
= rport
;
2354 lpfc_printf_vlog(vport
, KERN_INFO
,
2355 LOG_NVME_DISC
| LOG_NODE
,
2356 "6022 Binding new rport to "
2357 "lport %p Rport WWNN 0x%llx, "
2358 "Rport WWPN 0x%llx DID "
2361 rpinfo
.node_name
, rpinfo
.port_name
,
2362 rpinfo
.port_id
, rpinfo
.port_role
);
2365 lpfc_printf_vlog(vport
, KERN_ERR
,
2366 LOG_NVME_DISC
| LOG_NODE
,
2367 "6031 RemotePort Registration failed "
2368 "err: %d, DID x%06x\n",
2369 ret
, ndlp
->nlp_DID
);
2378 /* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport.
2380 * There is no notion of Devloss or rport recovery from the current
2381 * nvme_transport perspective. Loss of an rport just means IO cannot
2382 * be sent and recovery is completely up to the initator.
2383 * For now, the driver just unbinds the DID and port_role so that
2384 * no further IO can be issued. Changes are planned for later.
2386 * Notes - the ndlp reference count is not decremented here since
2387 * since there is no nvme_transport api for devloss. Node ref count
2388 * is only adjusted in driver unload.
2391 lpfc_nvme_unregister_port(struct lpfc_vport
*vport
, struct lpfc_nodelist
*ndlp
)
2393 #if (IS_ENABLED(CONFIG_NVME_FC))
2395 struct nvme_fc_local_port
*localport
;
2396 struct lpfc_nvme_lport
*lport
;
2397 struct lpfc_nvme_rport
*rport
;
2398 struct nvme_fc_remote_port
*remoteport
;
2400 localport
= vport
->localport
;
2402 /* This is fundamental error. The localport is always
2403 * available until driver unload. Just exit.
2408 lport
= (struct lpfc_nvme_lport
*)localport
->private;
2412 rport
= ndlp
->nrport
;
2416 remoteport
= rport
->remoteport
;
2417 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
2418 "6033 Unreg nvme remoteport %p, portname x%llx, "
2419 "port_id x%06x, portstate x%x port type x%x\n",
2420 remoteport
, remoteport
->port_name
,
2421 remoteport
->port_id
, remoteport
->port_state
,
2424 /* Sanity check ndlp type. Only call for NVME ports. Don't
2425 * clear any rport state until the transport calls back.
2427 if (ndlp
->nlp_type
& (NLP_NVME_TARGET
| NLP_NVME_INITIATOR
)) {
2428 init_completion(&rport
->rport_unreg_done
);
2430 /* No concern about the role change on the nvme remoteport.
2431 * The transport will update it.
2433 ret
= nvme_fc_unregister_remoteport(remoteport
);
2435 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
2436 "6167 NVME unregister failed %d "
2438 ret
, remoteport
->port_state
);
2446 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
2447 "6168 State error: lport %p, rport%p FCID x%06x\n",
2448 vport
->localport
, ndlp
->rport
, ndlp
->nlp_DID
);
2452 * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort
2453 * @phba: pointer to lpfc hba data structure.
2454 * @axri: pointer to the fcp xri abort wcqe structure.
2456 * This routine is invoked by the worker thread to process a SLI4 fast-path
2460 lpfc_sli4_nvme_xri_aborted(struct lpfc_hba
*phba
,
2461 struct sli4_wcqe_xri_aborted
*axri
)
2463 uint16_t xri
= bf_get(lpfc_wcqe_xa_xri
, axri
);
2464 uint16_t rxid
= bf_get(lpfc_wcqe_xa_remote_xid
, axri
);
2465 struct lpfc_nvme_buf
*lpfc_ncmd
, *next_lpfc_ncmd
;
2466 struct lpfc_nodelist
*ndlp
;
2467 unsigned long iflag
= 0;
2470 if (!(phba
->cfg_enable_fc4_type
& LPFC_ENABLE_NVME
))
2472 spin_lock_irqsave(&phba
->hbalock
, iflag
);
2473 spin_lock(&phba
->sli4_hba
.abts_nvme_buf_list_lock
);
2474 list_for_each_entry_safe(lpfc_ncmd
, next_lpfc_ncmd
,
2475 &phba
->sli4_hba
.lpfc_abts_nvme_buf_list
,
2477 if (lpfc_ncmd
->cur_iocbq
.sli4_xritag
== xri
) {
2478 list_del_init(&lpfc_ncmd
->list
);
2479 lpfc_ncmd
->flags
&= ~LPFC_SBUF_XBUSY
;
2480 lpfc_ncmd
->status
= IOSTAT_SUCCESS
;
2482 &phba
->sli4_hba
.abts_nvme_buf_list_lock
);
2484 rrq_empty
= list_empty(&phba
->active_rrq_list
);
2485 spin_unlock_irqrestore(&phba
->hbalock
, iflag
);
2486 ndlp
= lpfc_ncmd
->ndlp
;
2488 lpfc_set_rrq_active(
2490 lpfc_ncmd
->cur_iocbq
.sli4_lxritag
,
2492 lpfc_sli4_abts_err_handler(phba
, ndlp
, axri
);
2495 lpfc_printf_log(phba
, KERN_INFO
, LOG_NVME_ABTS
,
2496 "6311 XRI Aborted xri x%x tag x%x "
2498 xri
, lpfc_ncmd
->cur_iocbq
.iotag
);
2500 lpfc_release_nvme_buf(phba
, lpfc_ncmd
);
2502 lpfc_worker_wake_up(phba
);
2506 spin_unlock(&phba
->sli4_hba
.abts_nvme_buf_list_lock
);
2507 spin_unlock_irqrestore(&phba
->hbalock
, iflag
);
2509 lpfc_printf_log(phba
, KERN_INFO
, LOG_NVME_ABTS
,
2510 "6312 XRI Aborted xri x%x not found\n", xri
);