2 * Copyright (c) 2016 Avago Technologies. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful.
9 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES,
10 * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A
11 * PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE DISCLAIMED, EXCEPT TO
12 * THE EXTENT THAT SUCH DISCLAIMERS ARE HELD TO BE LEGALLY INVALID.
13 * See the GNU General Public License for more details, a copy of which
14 * can be found in the file COPYING included with this package
17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
18 #include <linux/module.h>
19 #include <linux/parser.h>
20 #include <uapi/scsi/fc/fc_fs.h>
21 #include <uapi/scsi/fc/fc_els.h>
25 #include <linux/nvme-fc-driver.h>
26 #include <linux/nvme-fc.h>
29 /* *************************** Data Structures/Defines ****************** */
33 * We handle AEN commands ourselves and don't even let the
34 * block layer know about them.
36 #define NVME_FC_NR_AEN_COMMANDS 1
37 #define NVME_FC_AQ_BLKMQ_DEPTH \
38 (NVMF_AQ_DEPTH - NVME_FC_NR_AEN_COMMANDS)
39 #define AEN_CMDID_BASE (NVME_FC_AQ_BLKMQ_DEPTH + 1)
41 enum nvme_fc_queue_flags
{
42 NVME_FC_Q_CONNECTED
= (1 << 0),
45 #define NVMEFC_QUEUE_DELAY 3 /* ms units */
47 struct nvme_fc_queue
{
48 struct nvme_fc_ctrl
*ctrl
;
50 struct blk_mq_hw_ctx
*hctx
;
53 size_t cmnd_capsule_len
;
62 } __aligned(sizeof(u64
)); /* alignment for other things alloc'd with */
64 struct nvmefc_ls_req_op
{
65 struct nvmefc_ls_req ls_req
;
67 struct nvme_fc_ctrl
*ctrl
;
68 struct nvme_fc_queue
*queue
;
72 struct completion ls_done
;
73 struct list_head lsreq_list
; /* ctrl->ls_req_list */
77 enum nvme_fcpop_state
{
78 FCPOP_STATE_UNINIT
= 0,
80 FCPOP_STATE_ACTIVE
= 2,
81 FCPOP_STATE_ABORTED
= 3,
84 struct nvme_fc_fcp_op
{
85 struct nvme_request nreq
; /*
88 * the 1st element in the
93 struct nvmefc_fcp_req fcp_req
;
95 struct nvme_fc_ctrl
*ctrl
;
96 struct nvme_fc_queue
*queue
;
103 struct nvme_fc_cmd_iu cmd_iu
;
104 struct nvme_fc_ersp_iu rsp_iu
;
107 struct nvme_fc_lport
{
108 struct nvme_fc_local_port localport
;
111 struct list_head port_list
; /* nvme_fc_port_list */
112 struct list_head endp_list
;
113 struct device
*dev
; /* physical device for dma */
114 struct nvme_fc_port_template
*ops
;
116 } __aligned(sizeof(u64
)); /* alignment for other things alloc'd with */
118 struct nvme_fc_rport
{
119 struct nvme_fc_remote_port remoteport
;
121 struct list_head endp_list
; /* for lport->endp_list */
122 struct list_head ctrl_list
;
125 } __aligned(sizeof(u64
)); /* alignment for other things alloc'd with */
127 enum nvme_fcctrl_state
{
132 struct nvme_fc_ctrl
{
134 struct nvme_fc_queue
*queues
;
138 struct nvme_fc_lport
*lport
;
139 struct nvme_fc_rport
*rport
;
146 struct list_head ctrl_list
; /* rport->ctrl_list */
147 struct list_head ls_req_list
;
149 struct blk_mq_tag_set admin_tag_set
;
150 struct blk_mq_tag_set tag_set
;
152 struct work_struct delete_work
;
156 struct nvme_fc_fcp_op aen_ops
[NVME_FC_NR_AEN_COMMANDS
];
158 struct nvme_ctrl ctrl
;
161 static inline struct nvme_fc_ctrl
*
162 to_fc_ctrl(struct nvme_ctrl
*ctrl
)
164 return container_of(ctrl
, struct nvme_fc_ctrl
, ctrl
);
167 static inline struct nvme_fc_lport
*
168 localport_to_lport(struct nvme_fc_local_port
*portptr
)
170 return container_of(portptr
, struct nvme_fc_lport
, localport
);
173 static inline struct nvme_fc_rport
*
174 remoteport_to_rport(struct nvme_fc_remote_port
*portptr
)
176 return container_of(portptr
, struct nvme_fc_rport
, remoteport
);
179 static inline struct nvmefc_ls_req_op
*
180 ls_req_to_lsop(struct nvmefc_ls_req
*lsreq
)
182 return container_of(lsreq
, struct nvmefc_ls_req_op
, ls_req
);
185 static inline struct nvme_fc_fcp_op
*
186 fcp_req_to_fcp_op(struct nvmefc_fcp_req
*fcpreq
)
188 return container_of(fcpreq
, struct nvme_fc_fcp_op
, fcp_req
);
193 /* *************************** Globals **************************** */
196 static DEFINE_SPINLOCK(nvme_fc_lock
);
198 static LIST_HEAD(nvme_fc_lport_list
);
199 static DEFINE_IDA(nvme_fc_local_port_cnt
);
200 static DEFINE_IDA(nvme_fc_ctrl_cnt
);
202 static struct workqueue_struct
*nvme_fc_wq
;
206 /* *********************** FC-NVME Port Management ************************ */
208 static int __nvme_fc_del_ctrl(struct nvme_fc_ctrl
*);
209 static void __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl
*,
210 struct nvme_fc_queue
*, unsigned int);
214 * nvme_fc_register_localport - transport entry point called by an
215 * LLDD to register the existence of a NVME
217 * @pinfo: pointer to information about the port to be registered
218 * @template: LLDD entrypoints and operational parameters for the port
219 * @dev: physical hardware device node port corresponds to. Will be
220 * used for DMA mappings
221 * @lport_p: pointer to a local port pointer. Upon success, the routine
222 * will allocate a nvme_fc_local_port structure and place its
223 * address in the local port pointer. Upon failure, local port
224 * pointer will be set to 0.
227 * a completion status. Must be 0 upon success; a negative errno
228 * (ex: -ENXIO) upon failure.
231 nvme_fc_register_localport(struct nvme_fc_port_info
*pinfo
,
232 struct nvme_fc_port_template
*template,
234 struct nvme_fc_local_port
**portptr
)
236 struct nvme_fc_lport
*newrec
;
240 if (!template->localport_delete
|| !template->remoteport_delete
||
241 !template->ls_req
|| !template->fcp_io
||
242 !template->ls_abort
|| !template->fcp_abort
||
243 !template->max_hw_queues
|| !template->max_sgl_segments
||
244 !template->max_dif_sgl_segments
|| !template->dma_boundary
) {
246 goto out_reghost_failed
;
249 newrec
= kmalloc((sizeof(*newrec
) + template->local_priv_sz
),
253 goto out_reghost_failed
;
256 idx
= ida_simple_get(&nvme_fc_local_port_cnt
, 0, 0, GFP_KERNEL
);
262 if (!get_device(dev
) && dev
) {
267 INIT_LIST_HEAD(&newrec
->port_list
);
268 INIT_LIST_HEAD(&newrec
->endp_list
);
269 kref_init(&newrec
->ref
);
270 newrec
->ops
= template;
272 ida_init(&newrec
->endp_cnt
);
273 newrec
->localport
.private = &newrec
[1];
274 newrec
->localport
.node_name
= pinfo
->node_name
;
275 newrec
->localport
.port_name
= pinfo
->port_name
;
276 newrec
->localport
.port_role
= pinfo
->port_role
;
277 newrec
->localport
.port_id
= pinfo
->port_id
;
278 newrec
->localport
.port_state
= FC_OBJSTATE_ONLINE
;
279 newrec
->localport
.port_num
= idx
;
281 spin_lock_irqsave(&nvme_fc_lock
, flags
);
282 list_add_tail(&newrec
->port_list
, &nvme_fc_lport_list
);
283 spin_unlock_irqrestore(&nvme_fc_lock
, flags
);
286 dma_set_seg_boundary(dev
, template->dma_boundary
);
288 *portptr
= &newrec
->localport
;
292 ida_simple_remove(&nvme_fc_local_port_cnt
, idx
);
300 EXPORT_SYMBOL_GPL(nvme_fc_register_localport
);
303 nvme_fc_free_lport(struct kref
*ref
)
305 struct nvme_fc_lport
*lport
=
306 container_of(ref
, struct nvme_fc_lport
, ref
);
309 WARN_ON(lport
->localport
.port_state
!= FC_OBJSTATE_DELETED
);
310 WARN_ON(!list_empty(&lport
->endp_list
));
312 /* remove from transport list */
313 spin_lock_irqsave(&nvme_fc_lock
, flags
);
314 list_del(&lport
->port_list
);
315 spin_unlock_irqrestore(&nvme_fc_lock
, flags
);
317 /* let the LLDD know we've finished tearing it down */
318 lport
->ops
->localport_delete(&lport
->localport
);
320 ida_simple_remove(&nvme_fc_local_port_cnt
, lport
->localport
.port_num
);
321 ida_destroy(&lport
->endp_cnt
);
323 put_device(lport
->dev
);
329 nvme_fc_lport_put(struct nvme_fc_lport
*lport
)
331 kref_put(&lport
->ref
, nvme_fc_free_lport
);
335 nvme_fc_lport_get(struct nvme_fc_lport
*lport
)
337 return kref_get_unless_zero(&lport
->ref
);
341 * nvme_fc_unregister_localport - transport entry point called by an
342 * LLDD to deregister/remove a previously
343 * registered a NVME host FC port.
344 * @localport: pointer to the (registered) local port that is to be
348 * a completion status. Must be 0 upon success; a negative errno
349 * (ex: -ENXIO) upon failure.
352 nvme_fc_unregister_localport(struct nvme_fc_local_port
*portptr
)
354 struct nvme_fc_lport
*lport
= localport_to_lport(portptr
);
360 spin_lock_irqsave(&nvme_fc_lock
, flags
);
362 if (portptr
->port_state
!= FC_OBJSTATE_ONLINE
) {
363 spin_unlock_irqrestore(&nvme_fc_lock
, flags
);
366 portptr
->port_state
= FC_OBJSTATE_DELETED
;
368 spin_unlock_irqrestore(&nvme_fc_lock
, flags
);
370 nvme_fc_lport_put(lport
);
374 EXPORT_SYMBOL_GPL(nvme_fc_unregister_localport
);
377 * nvme_fc_register_remoteport - transport entry point called by an
378 * LLDD to register the existence of a NVME
379 * subsystem FC port on its fabric.
380 * @localport: pointer to the (registered) local port that the remote
381 * subsystem port is connected to.
382 * @pinfo: pointer to information about the port to be registered
383 * @rport_p: pointer to a remote port pointer. Upon success, the routine
384 * will allocate a nvme_fc_remote_port structure and place its
385 * address in the remote port pointer. Upon failure, remote port
386 * pointer will be set to 0.
389 * a completion status. Must be 0 upon success; a negative errno
390 * (ex: -ENXIO) upon failure.
393 nvme_fc_register_remoteport(struct nvme_fc_local_port
*localport
,
394 struct nvme_fc_port_info
*pinfo
,
395 struct nvme_fc_remote_port
**portptr
)
397 struct nvme_fc_lport
*lport
= localport_to_lport(localport
);
398 struct nvme_fc_rport
*newrec
;
402 newrec
= kmalloc((sizeof(*newrec
) + lport
->ops
->remote_priv_sz
),
406 goto out_reghost_failed
;
409 if (!nvme_fc_lport_get(lport
)) {
411 goto out_kfree_rport
;
414 idx
= ida_simple_get(&lport
->endp_cnt
, 0, 0, GFP_KERNEL
);
420 INIT_LIST_HEAD(&newrec
->endp_list
);
421 INIT_LIST_HEAD(&newrec
->ctrl_list
);
422 kref_init(&newrec
->ref
);
423 spin_lock_init(&newrec
->lock
);
424 newrec
->remoteport
.localport
= &lport
->localport
;
425 newrec
->remoteport
.private = &newrec
[1];
426 newrec
->remoteport
.port_role
= pinfo
->port_role
;
427 newrec
->remoteport
.node_name
= pinfo
->node_name
;
428 newrec
->remoteport
.port_name
= pinfo
->port_name
;
429 newrec
->remoteport
.port_id
= pinfo
->port_id
;
430 newrec
->remoteport
.port_state
= FC_OBJSTATE_ONLINE
;
431 newrec
->remoteport
.port_num
= idx
;
433 spin_lock_irqsave(&nvme_fc_lock
, flags
);
434 list_add_tail(&newrec
->endp_list
, &lport
->endp_list
);
435 spin_unlock_irqrestore(&nvme_fc_lock
, flags
);
437 *portptr
= &newrec
->remoteport
;
441 nvme_fc_lport_put(lport
);
449 EXPORT_SYMBOL_GPL(nvme_fc_register_remoteport
);
452 nvme_fc_free_rport(struct kref
*ref
)
454 struct nvme_fc_rport
*rport
=
455 container_of(ref
, struct nvme_fc_rport
, ref
);
456 struct nvme_fc_lport
*lport
=
457 localport_to_lport(rport
->remoteport
.localport
);
460 WARN_ON(rport
->remoteport
.port_state
!= FC_OBJSTATE_DELETED
);
461 WARN_ON(!list_empty(&rport
->ctrl_list
));
463 /* remove from lport list */
464 spin_lock_irqsave(&nvme_fc_lock
, flags
);
465 list_del(&rport
->endp_list
);
466 spin_unlock_irqrestore(&nvme_fc_lock
, flags
);
468 /* let the LLDD know we've finished tearing it down */
469 lport
->ops
->remoteport_delete(&rport
->remoteport
);
471 ida_simple_remove(&lport
->endp_cnt
, rport
->remoteport
.port_num
);
475 nvme_fc_lport_put(lport
);
479 nvme_fc_rport_put(struct nvme_fc_rport
*rport
)
481 kref_put(&rport
->ref
, nvme_fc_free_rport
);
485 nvme_fc_rport_get(struct nvme_fc_rport
*rport
)
487 return kref_get_unless_zero(&rport
->ref
);
491 * nvme_fc_unregister_remoteport - transport entry point called by an
492 * LLDD to deregister/remove a previously
493 * registered a NVME subsystem FC port.
494 * @remoteport: pointer to the (registered) remote port that is to be
498 * a completion status. Must be 0 upon success; a negative errno
499 * (ex: -ENXIO) upon failure.
502 nvme_fc_unregister_remoteport(struct nvme_fc_remote_port
*portptr
)
504 struct nvme_fc_rport
*rport
= remoteport_to_rport(portptr
);
505 struct nvme_fc_ctrl
*ctrl
;
511 spin_lock_irqsave(&rport
->lock
, flags
);
513 if (portptr
->port_state
!= FC_OBJSTATE_ONLINE
) {
514 spin_unlock_irqrestore(&rport
->lock
, flags
);
517 portptr
->port_state
= FC_OBJSTATE_DELETED
;
519 /* tear down all associations to the remote port */
520 list_for_each_entry(ctrl
, &rport
->ctrl_list
, ctrl_list
)
521 __nvme_fc_del_ctrl(ctrl
);
523 spin_unlock_irqrestore(&rport
->lock
, flags
);
525 nvme_fc_rport_put(rport
);
528 EXPORT_SYMBOL_GPL(nvme_fc_unregister_remoteport
);
531 /* *********************** FC-NVME DMA Handling **************************** */
534 * The fcloop device passes in a NULL device pointer. Real LLD's will
535 * pass in a valid device pointer. If NULL is passed to the dma mapping
536 * routines, depending on the platform, it may or may not succeed, and
540 * Wrapper all the dma routines and check the dev pointer.
542 * If simple mappings (return just a dma address, we'll noop them,
543 * returning a dma address of 0.
545 * On more complex mappings (dma_map_sg), a pseudo routine fills
546 * in the scatter list, setting all dma addresses to 0.
549 static inline dma_addr_t
550 fc_dma_map_single(struct device
*dev
, void *ptr
, size_t size
,
551 enum dma_data_direction dir
)
553 return dev
? dma_map_single(dev
, ptr
, size
, dir
) : (dma_addr_t
)0L;
557 fc_dma_mapping_error(struct device
*dev
, dma_addr_t dma_addr
)
559 return dev
? dma_mapping_error(dev
, dma_addr
) : 0;
563 fc_dma_unmap_single(struct device
*dev
, dma_addr_t addr
, size_t size
,
564 enum dma_data_direction dir
)
567 dma_unmap_single(dev
, addr
, size
, dir
);
571 fc_dma_sync_single_for_cpu(struct device
*dev
, dma_addr_t addr
, size_t size
,
572 enum dma_data_direction dir
)
575 dma_sync_single_for_cpu(dev
, addr
, size
, dir
);
579 fc_dma_sync_single_for_device(struct device
*dev
, dma_addr_t addr
, size_t size
,
580 enum dma_data_direction dir
)
583 dma_sync_single_for_device(dev
, addr
, size
, dir
);
586 /* pseudo dma_map_sg call */
588 fc_map_sg(struct scatterlist
*sg
, int nents
)
590 struct scatterlist
*s
;
593 WARN_ON(nents
== 0 || sg
[0].length
== 0);
595 for_each_sg(sg
, s
, nents
, i
) {
597 #ifdef CONFIG_NEED_SG_DMA_LENGTH
598 s
->dma_length
= s
->length
;
605 fc_dma_map_sg(struct device
*dev
, struct scatterlist
*sg
, int nents
,
606 enum dma_data_direction dir
)
608 return dev
? dma_map_sg(dev
, sg
, nents
, dir
) : fc_map_sg(sg
, nents
);
612 fc_dma_unmap_sg(struct device
*dev
, struct scatterlist
*sg
, int nents
,
613 enum dma_data_direction dir
)
616 dma_unmap_sg(dev
, sg
, nents
, dir
);
620 /* *********************** FC-NVME LS Handling **************************** */
622 static void nvme_fc_ctrl_put(struct nvme_fc_ctrl
*);
623 static int nvme_fc_ctrl_get(struct nvme_fc_ctrl
*);
627 __nvme_fc_finish_ls_req(struct nvme_fc_ctrl
*ctrl
,
628 struct nvmefc_ls_req_op
*lsop
)
630 struct nvmefc_ls_req
*lsreq
= &lsop
->ls_req
;
633 spin_lock_irqsave(&ctrl
->lock
, flags
);
635 if (!lsop
->req_queued
) {
636 spin_unlock_irqrestore(&ctrl
->lock
, flags
);
640 list_del(&lsop
->lsreq_list
);
642 lsop
->req_queued
= false;
644 spin_unlock_irqrestore(&ctrl
->lock
, flags
);
646 fc_dma_unmap_single(ctrl
->dev
, lsreq
->rqstdma
,
647 (lsreq
->rqstlen
+ lsreq
->rsplen
),
650 nvme_fc_ctrl_put(ctrl
);
654 __nvme_fc_send_ls_req(struct nvme_fc_ctrl
*ctrl
,
655 struct nvmefc_ls_req_op
*lsop
,
656 void (*done
)(struct nvmefc_ls_req
*req
, int status
))
658 struct nvmefc_ls_req
*lsreq
= &lsop
->ls_req
;
662 if (!nvme_fc_ctrl_get(ctrl
))
667 lsop
->req_queued
= false;
668 INIT_LIST_HEAD(&lsop
->lsreq_list
);
669 init_completion(&lsop
->ls_done
);
671 lsreq
->rqstdma
= fc_dma_map_single(ctrl
->dev
, lsreq
->rqstaddr
,
672 lsreq
->rqstlen
+ lsreq
->rsplen
,
674 if (fc_dma_mapping_error(ctrl
->dev
, lsreq
->rqstdma
)) {
675 nvme_fc_ctrl_put(ctrl
);
677 "els request command failed EFAULT.\n");
680 lsreq
->rspdma
= lsreq
->rqstdma
+ lsreq
->rqstlen
;
682 spin_lock_irqsave(&ctrl
->lock
, flags
);
684 list_add_tail(&lsop
->lsreq_list
, &ctrl
->ls_req_list
);
686 lsop
->req_queued
= true;
688 spin_unlock_irqrestore(&ctrl
->lock
, flags
);
690 ret
= ctrl
->lport
->ops
->ls_req(&ctrl
->lport
->localport
,
691 &ctrl
->rport
->remoteport
, lsreq
);
693 lsop
->ls_error
= ret
;
699 nvme_fc_send_ls_req_done(struct nvmefc_ls_req
*lsreq
, int status
)
701 struct nvmefc_ls_req_op
*lsop
= ls_req_to_lsop(lsreq
);
703 lsop
->ls_error
= status
;
704 complete(&lsop
->ls_done
);
708 nvme_fc_send_ls_req(struct nvme_fc_ctrl
*ctrl
, struct nvmefc_ls_req_op
*lsop
)
710 struct nvmefc_ls_req
*lsreq
= &lsop
->ls_req
;
711 struct fcnvme_ls_rjt
*rjt
= lsreq
->rspaddr
;
714 ret
= __nvme_fc_send_ls_req(ctrl
, lsop
, nvme_fc_send_ls_req_done
);
718 * No timeout/not interruptible as we need the struct
719 * to exist until the lldd calls us back. Thus mandate
720 * wait until driver calls back. lldd responsible for
723 wait_for_completion(&lsop
->ls_done
);
725 __nvme_fc_finish_ls_req(ctrl
, lsop
);
729 "ls request command failed (%d).\n", ret
);
733 /* ACC or RJT payload ? */
734 if (rjt
->w0
.ls_cmd
== FCNVME_LS_RJT
)
741 nvme_fc_send_ls_req_async(struct nvme_fc_ctrl
*ctrl
,
742 struct nvmefc_ls_req_op
*lsop
,
743 void (*done
)(struct nvmefc_ls_req
*req
, int status
))
747 ret
= __nvme_fc_send_ls_req(ctrl
, lsop
, done
);
749 /* don't wait for completion */
752 done(&lsop
->ls_req
, ret
);
755 /* Validation Error indexes into the string table below */
759 VERR_LSDESC_RQST
= 2,
760 VERR_LSDESC_RQST_LEN
= 3,
762 VERR_ASSOC_ID_LEN
= 5,
764 VERR_CONN_ID_LEN
= 7,
766 VERR_CR_ASSOC_ACC_LEN
= 9,
768 VERR_CR_CONN_ACC_LEN
= 11,
770 VERR_DISCONN_ACC_LEN
= 13,
773 static char *validation_errors
[] = {
777 "Bad LSDESC_RQST Length",
778 "Not Association ID",
779 "Bad Association ID Length",
781 "Bad Connection ID Length",
783 "Bad CR_ASSOC ACC Length",
785 "Bad CR_CONN ACC Length",
786 "Not Disconnect Rqst",
787 "Bad Disconnect ACC Length",
791 nvme_fc_connect_admin_queue(struct nvme_fc_ctrl
*ctrl
,
792 struct nvme_fc_queue
*queue
, u16 qsize
, u16 ersp_ratio
)
794 struct nvmefc_ls_req_op
*lsop
;
795 struct nvmefc_ls_req
*lsreq
;
796 struct fcnvme_ls_cr_assoc_rqst
*assoc_rqst
;
797 struct fcnvme_ls_cr_assoc_acc
*assoc_acc
;
800 lsop
= kzalloc((sizeof(*lsop
) +
801 ctrl
->lport
->ops
->lsrqst_priv_sz
+
802 sizeof(*assoc_rqst
) + sizeof(*assoc_acc
)), GFP_KERNEL
);
807 lsreq
= &lsop
->ls_req
;
809 lsreq
->private = (void *)&lsop
[1];
810 assoc_rqst
= (struct fcnvme_ls_cr_assoc_rqst
*)
811 (lsreq
->private + ctrl
->lport
->ops
->lsrqst_priv_sz
);
812 assoc_acc
= (struct fcnvme_ls_cr_assoc_acc
*)&assoc_rqst
[1];
814 assoc_rqst
->w0
.ls_cmd
= FCNVME_LS_CREATE_ASSOCIATION
;
815 assoc_rqst
->desc_list_len
=
816 cpu_to_be32(sizeof(struct fcnvme_lsdesc_cr_assoc_cmd
));
818 assoc_rqst
->assoc_cmd
.desc_tag
=
819 cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD
);
820 assoc_rqst
->assoc_cmd
.desc_len
=
822 sizeof(struct fcnvme_lsdesc_cr_assoc_cmd
));
824 assoc_rqst
->assoc_cmd
.ersp_ratio
= cpu_to_be16(ersp_ratio
);
825 assoc_rqst
->assoc_cmd
.sqsize
= cpu_to_be16(qsize
);
826 /* Linux supports only Dynamic controllers */
827 assoc_rqst
->assoc_cmd
.cntlid
= cpu_to_be16(0xffff);
828 memcpy(&assoc_rqst
->assoc_cmd
.hostid
, &ctrl
->ctrl
.opts
->host
->id
,
829 min_t(size_t, FCNVME_ASSOC_HOSTID_LEN
, sizeof(uuid_be
)));
830 strncpy(assoc_rqst
->assoc_cmd
.hostnqn
, ctrl
->ctrl
.opts
->host
->nqn
,
831 min(FCNVME_ASSOC_HOSTNQN_LEN
, NVMF_NQN_SIZE
));
832 strncpy(assoc_rqst
->assoc_cmd
.subnqn
, ctrl
->ctrl
.opts
->subsysnqn
,
833 min(FCNVME_ASSOC_SUBNQN_LEN
, NVMF_NQN_SIZE
));
836 lsreq
->rqstaddr
= assoc_rqst
;
837 lsreq
->rqstlen
= sizeof(*assoc_rqst
);
838 lsreq
->rspaddr
= assoc_acc
;
839 lsreq
->rsplen
= sizeof(*assoc_acc
);
840 lsreq
->timeout
= NVME_FC_CONNECT_TIMEOUT_SEC
;
842 ret
= nvme_fc_send_ls_req(ctrl
, lsop
);
844 goto out_free_buffer
;
846 /* process connect LS completion */
848 /* validate the ACC response */
849 if (assoc_acc
->hdr
.w0
.ls_cmd
!= FCNVME_LS_ACC
)
851 if (assoc_acc
->hdr
.desc_list_len
!=
853 sizeof(struct fcnvme_ls_cr_assoc_acc
)))
854 fcret
= VERR_CR_ASSOC_ACC_LEN
;
855 if (assoc_acc
->hdr
.rqst
.desc_tag
!= cpu_to_be32(FCNVME_LSDESC_RQST
))
856 fcret
= VERR_LSDESC_RQST
;
857 else if (assoc_acc
->hdr
.rqst
.desc_len
!=
858 fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst
)))
859 fcret
= VERR_LSDESC_RQST_LEN
;
860 else if (assoc_acc
->hdr
.rqst
.w0
.ls_cmd
!= FCNVME_LS_CREATE_ASSOCIATION
)
861 fcret
= VERR_CR_ASSOC
;
862 else if (assoc_acc
->associd
.desc_tag
!=
863 cpu_to_be32(FCNVME_LSDESC_ASSOC_ID
))
864 fcret
= VERR_ASSOC_ID
;
865 else if (assoc_acc
->associd
.desc_len
!=
867 sizeof(struct fcnvme_lsdesc_assoc_id
)))
868 fcret
= VERR_ASSOC_ID_LEN
;
869 else if (assoc_acc
->connectid
.desc_tag
!=
870 cpu_to_be32(FCNVME_LSDESC_CONN_ID
))
871 fcret
= VERR_CONN_ID
;
872 else if (assoc_acc
->connectid
.desc_len
!=
873 fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id
)))
874 fcret
= VERR_CONN_ID_LEN
;
879 "q %d connect failed: %s\n",
880 queue
->qnum
, validation_errors
[fcret
]);
882 ctrl
->association_id
=
883 be64_to_cpu(assoc_acc
->associd
.association_id
);
884 queue
->connection_id
=
885 be64_to_cpu(assoc_acc
->connectid
.connection_id
);
886 set_bit(NVME_FC_Q_CONNECTED
, &queue
->flags
);
894 "queue %d connect admin queue failed (%d).\n",
900 nvme_fc_connect_queue(struct nvme_fc_ctrl
*ctrl
, struct nvme_fc_queue
*queue
,
901 u16 qsize
, u16 ersp_ratio
)
903 struct nvmefc_ls_req_op
*lsop
;
904 struct nvmefc_ls_req
*lsreq
;
905 struct fcnvme_ls_cr_conn_rqst
*conn_rqst
;
906 struct fcnvme_ls_cr_conn_acc
*conn_acc
;
909 lsop
= kzalloc((sizeof(*lsop
) +
910 ctrl
->lport
->ops
->lsrqst_priv_sz
+
911 sizeof(*conn_rqst
) + sizeof(*conn_acc
)), GFP_KERNEL
);
916 lsreq
= &lsop
->ls_req
;
918 lsreq
->private = (void *)&lsop
[1];
919 conn_rqst
= (struct fcnvme_ls_cr_conn_rqst
*)
920 (lsreq
->private + ctrl
->lport
->ops
->lsrqst_priv_sz
);
921 conn_acc
= (struct fcnvme_ls_cr_conn_acc
*)&conn_rqst
[1];
923 conn_rqst
->w0
.ls_cmd
= FCNVME_LS_CREATE_CONNECTION
;
924 conn_rqst
->desc_list_len
= cpu_to_be32(
925 sizeof(struct fcnvme_lsdesc_assoc_id
) +
926 sizeof(struct fcnvme_lsdesc_cr_conn_cmd
));
928 conn_rqst
->associd
.desc_tag
= cpu_to_be32(FCNVME_LSDESC_ASSOC_ID
);
929 conn_rqst
->associd
.desc_len
=
931 sizeof(struct fcnvme_lsdesc_assoc_id
));
932 conn_rqst
->associd
.association_id
= cpu_to_be64(ctrl
->association_id
);
933 conn_rqst
->connect_cmd
.desc_tag
=
934 cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD
);
935 conn_rqst
->connect_cmd
.desc_len
=
937 sizeof(struct fcnvme_lsdesc_cr_conn_cmd
));
938 conn_rqst
->connect_cmd
.ersp_ratio
= cpu_to_be16(ersp_ratio
);
939 conn_rqst
->connect_cmd
.qid
= cpu_to_be16(queue
->qnum
);
940 conn_rqst
->connect_cmd
.sqsize
= cpu_to_be16(qsize
);
943 lsreq
->rqstaddr
= conn_rqst
;
944 lsreq
->rqstlen
= sizeof(*conn_rqst
);
945 lsreq
->rspaddr
= conn_acc
;
946 lsreq
->rsplen
= sizeof(*conn_acc
);
947 lsreq
->timeout
= NVME_FC_CONNECT_TIMEOUT_SEC
;
949 ret
= nvme_fc_send_ls_req(ctrl
, lsop
);
951 goto out_free_buffer
;
953 /* process connect LS completion */
955 /* validate the ACC response */
956 if (conn_acc
->hdr
.w0
.ls_cmd
!= FCNVME_LS_ACC
)
958 if (conn_acc
->hdr
.desc_list_len
!=
959 fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc
)))
960 fcret
= VERR_CR_CONN_ACC_LEN
;
961 if (conn_acc
->hdr
.rqst
.desc_tag
!= cpu_to_be32(FCNVME_LSDESC_RQST
))
962 fcret
= VERR_LSDESC_RQST
;
963 else if (conn_acc
->hdr
.rqst
.desc_len
!=
964 fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst
)))
965 fcret
= VERR_LSDESC_RQST_LEN
;
966 else if (conn_acc
->hdr
.rqst
.w0
.ls_cmd
!= FCNVME_LS_CREATE_CONNECTION
)
967 fcret
= VERR_CR_CONN
;
968 else if (conn_acc
->connectid
.desc_tag
!=
969 cpu_to_be32(FCNVME_LSDESC_CONN_ID
))
970 fcret
= VERR_CONN_ID
;
971 else if (conn_acc
->connectid
.desc_len
!=
972 fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id
)))
973 fcret
= VERR_CONN_ID_LEN
;
978 "q %d connect failed: %s\n",
979 queue
->qnum
, validation_errors
[fcret
]);
981 queue
->connection_id
=
982 be64_to_cpu(conn_acc
->connectid
.connection_id
);
983 set_bit(NVME_FC_Q_CONNECTED
, &queue
->flags
);
991 "queue %d connect command failed (%d).\n",
997 nvme_fc_disconnect_assoc_done(struct nvmefc_ls_req
*lsreq
, int status
)
999 struct nvmefc_ls_req_op
*lsop
= ls_req_to_lsop(lsreq
);
1000 struct nvme_fc_ctrl
*ctrl
= lsop
->ctrl
;
1002 __nvme_fc_finish_ls_req(ctrl
, lsop
);
1006 "disconnect assoc ls request command failed (%d).\n",
1009 /* fc-nvme iniator doesn't care about success or failure of cmd */
1015 * This routine sends a FC-NVME LS to disconnect (aka terminate)
1016 * the FC-NVME Association. Terminating the association also
1017 * terminates the FC-NVME connections (per queue, both admin and io
1018 * queues) that are part of the association. E.g. things are torn
1019 * down, and the related FC-NVME Association ID and Connection IDs
1022 * The behavior of the fc-nvme initiator is such that it's
1023 * understanding of the association and connections will implicitly
1024 * be torn down. The action is implicit as it may be due to a loss of
1025 * connectivity with the fc-nvme target, so you may never get a
1026 * response even if you tried. As such, the action of this routine
1027 * is to asynchronously send the LS, ignore any results of the LS, and
1028 * continue on with terminating the association. If the fc-nvme target
1029 * is present and receives the LS, it too can tear down.
1032 nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl
*ctrl
)
1034 struct fcnvme_ls_disconnect_rqst
*discon_rqst
;
1035 struct fcnvme_ls_disconnect_acc
*discon_acc
;
1036 struct nvmefc_ls_req_op
*lsop
;
1037 struct nvmefc_ls_req
*lsreq
;
1039 lsop
= kzalloc((sizeof(*lsop
) +
1040 ctrl
->lport
->ops
->lsrqst_priv_sz
+
1041 sizeof(*discon_rqst
) + sizeof(*discon_acc
)),
1044 /* couldn't sent it... too bad */
1047 lsreq
= &lsop
->ls_req
;
1049 lsreq
->private = (void *)&lsop
[1];
1050 discon_rqst
= (struct fcnvme_ls_disconnect_rqst
*)
1051 (lsreq
->private + ctrl
->lport
->ops
->lsrqst_priv_sz
);
1052 discon_acc
= (struct fcnvme_ls_disconnect_acc
*)&discon_rqst
[1];
1054 discon_rqst
->w0
.ls_cmd
= FCNVME_LS_DISCONNECT
;
1055 discon_rqst
->desc_list_len
= cpu_to_be32(
1056 sizeof(struct fcnvme_lsdesc_assoc_id
) +
1057 sizeof(struct fcnvme_lsdesc_disconn_cmd
));
1059 discon_rqst
->associd
.desc_tag
= cpu_to_be32(FCNVME_LSDESC_ASSOC_ID
);
1060 discon_rqst
->associd
.desc_len
=
1062 sizeof(struct fcnvme_lsdesc_assoc_id
));
1064 discon_rqst
->associd
.association_id
= cpu_to_be64(ctrl
->association_id
);
1066 discon_rqst
->discon_cmd
.desc_tag
= cpu_to_be32(
1067 FCNVME_LSDESC_DISCONN_CMD
);
1068 discon_rqst
->discon_cmd
.desc_len
=
1070 sizeof(struct fcnvme_lsdesc_disconn_cmd
));
1071 discon_rqst
->discon_cmd
.scope
= FCNVME_DISCONN_ASSOCIATION
;
1072 discon_rqst
->discon_cmd
.id
= cpu_to_be64(ctrl
->association_id
);
1074 lsreq
->rqstaddr
= discon_rqst
;
1075 lsreq
->rqstlen
= sizeof(*discon_rqst
);
1076 lsreq
->rspaddr
= discon_acc
;
1077 lsreq
->rsplen
= sizeof(*discon_acc
);
1078 lsreq
->timeout
= NVME_FC_CONNECT_TIMEOUT_SEC
;
1080 nvme_fc_send_ls_req_async(ctrl
, lsop
, nvme_fc_disconnect_assoc_done
);
1082 /* only meaningful part to terminating the association */
1083 ctrl
->association_id
= 0;
1087 /* *********************** NVME Ctrl Routines **************************** */
1091 nvme_fc_reinit_request(void *data
, struct request
*rq
)
1093 struct nvme_fc_fcp_op
*op
= blk_mq_rq_to_pdu(rq
);
1094 struct nvme_fc_cmd_iu
*cmdiu
= &op
->cmd_iu
;
1096 memset(cmdiu
, 0, sizeof(*cmdiu
));
1097 cmdiu
->scsi_id
= NVME_CMD_SCSI_ID
;
1098 cmdiu
->fc_id
= NVME_CMD_FC_ID
;
1099 cmdiu
->iu_len
= cpu_to_be16(sizeof(*cmdiu
) / sizeof(u32
));
1100 memset(&op
->rsp_iu
, 0, sizeof(op
->rsp_iu
));
1106 __nvme_fc_exit_request(struct nvme_fc_ctrl
*ctrl
,
1107 struct nvme_fc_fcp_op
*op
)
1109 fc_dma_unmap_single(ctrl
->lport
->dev
, op
->fcp_req
.rspdma
,
1110 sizeof(op
->rsp_iu
), DMA_FROM_DEVICE
);
1111 fc_dma_unmap_single(ctrl
->lport
->dev
, op
->fcp_req
.cmddma
,
1112 sizeof(op
->cmd_iu
), DMA_TO_DEVICE
);
1114 atomic_set(&op
->state
, FCPOP_STATE_UNINIT
);
1118 nvme_fc_exit_request(void *data
, struct request
*rq
,
1119 unsigned int hctx_idx
, unsigned int rq_idx
)
1121 struct nvme_fc_fcp_op
*op
= blk_mq_rq_to_pdu(rq
);
1123 return __nvme_fc_exit_request(data
, op
);
1127 nvme_fc_exit_aen_ops(struct nvme_fc_ctrl
*ctrl
)
1129 struct nvme_fc_fcp_op
*aen_op
= ctrl
->aen_ops
;
1132 for (i
= 0; i
< NVME_FC_NR_AEN_COMMANDS
; i
++, aen_op
++) {
1133 if (atomic_read(&aen_op
->state
) == FCPOP_STATE_UNINIT
)
1135 __nvme_fc_exit_request(ctrl
, aen_op
);
1136 nvme_fc_ctrl_put(ctrl
);
1141 nvme_fc_fcpio_done(struct nvmefc_fcp_req
*req
)
1143 struct nvme_fc_fcp_op
*op
= fcp_req_to_fcp_op(req
);
1144 struct request
*rq
= op
->rq
;
1145 struct nvmefc_fcp_req
*freq
= &op
->fcp_req
;
1146 struct nvme_fc_ctrl
*ctrl
= op
->ctrl
;
1147 struct nvme_fc_queue
*queue
= op
->queue
;
1148 struct nvme_completion
*cqe
= &op
->rsp_iu
.cqe
;
1153 * The current linux implementation of a nvme controller
1154 * allocates a single tag set for all io queues and sizes
1155 * the io queues to fully hold all possible tags. Thus, the
1156 * implementation does not reference or care about the sqhd
1157 * value as it never needs to use the sqhd/sqtail pointers
1158 * for submission pacing.
1160 * This affects the FC-NVME implementation in two ways:
1161 * 1) As the value doesn't matter, we don't need to waste
1162 * cycles extracting it from ERSPs and stamping it in the
1163 * cases where the transport fabricates CQEs on successful
1165 * 2) The FC-NVME implementation requires that delivery of
1166 * ERSP completions are to go back to the nvme layer in order
1167 * relative to the rsn, such that the sqhd value will always
1168 * be "in order" for the nvme layer. As the nvme layer in
1169 * linux doesn't care about sqhd, there's no need to return
1173 * As the core nvme layer in linux currently does not look at
1174 * every field in the cqe - in cases where the FC transport must
1175 * fabricate a CQE, the following fields will not be set as they
1176 * are not referenced:
1177 * cqe.sqid, cqe.sqhd, cqe.command_id
1180 fc_dma_sync_single_for_cpu(ctrl
->lport
->dev
, op
->fcp_req
.rspdma
,
1181 sizeof(op
->rsp_iu
), DMA_FROM_DEVICE
);
1183 if (atomic_read(&op
->state
) == FCPOP_STATE_ABORTED
)
1184 status
= NVME_SC_ABORT_REQ
| NVME_SC_DNR
;
1186 status
= freq
->status
;
1189 * For the linux implementation, if we have an unsuccesful
1190 * status, they blk-mq layer can typically be called with the
1191 * non-zero status and the content of the cqe isn't important.
1197 * command completed successfully relative to the wire
1198 * protocol. However, validate anything received and
1199 * extract the status and result from the cqe (create it
1203 switch (freq
->rcv_rsplen
) {
1206 case NVME_FC_SIZEOF_ZEROS_RSP
:
1208 * No response payload or 12 bytes of payload (which
1209 * should all be zeros) are considered successful and
1210 * no payload in the CQE by the transport.
1212 if (freq
->transferred_length
!=
1213 be32_to_cpu(op
->cmd_iu
.data_len
)) {
1217 op
->nreq
.result
.u64
= 0;
1220 case sizeof(struct nvme_fc_ersp_iu
):
1222 * The ERSP IU contains a full completion with CQE.
1223 * Validate ERSP IU and look at cqe.
1225 if (unlikely(be16_to_cpu(op
->rsp_iu
.iu_len
) !=
1226 (freq
->rcv_rsplen
/ 4) ||
1227 be32_to_cpu(op
->rsp_iu
.xfrd_len
) !=
1228 freq
->transferred_length
||
1229 op
->rqno
!= le16_to_cpu(cqe
->command_id
))) {
1233 op
->nreq
.result
= cqe
->result
;
1234 status
= le16_to_cpu(cqe
->status
) >> 1;
1243 if (!queue
->qnum
&& op
->rqno
>= AEN_CMDID_BASE
) {
1244 nvme_complete_async_event(&queue
->ctrl
->ctrl
, status
,
1246 nvme_fc_ctrl_put(ctrl
);
1250 blk_mq_complete_request(rq
, status
);
1254 __nvme_fc_init_request(struct nvme_fc_ctrl
*ctrl
,
1255 struct nvme_fc_queue
*queue
, struct nvme_fc_fcp_op
*op
,
1256 struct request
*rq
, u32 rqno
)
1258 struct nvme_fc_cmd_iu
*cmdiu
= &op
->cmd_iu
;
1261 memset(op
, 0, sizeof(*op
));
1262 op
->fcp_req
.cmdaddr
= &op
->cmd_iu
;
1263 op
->fcp_req
.cmdlen
= sizeof(op
->cmd_iu
);
1264 op
->fcp_req
.rspaddr
= &op
->rsp_iu
;
1265 op
->fcp_req
.rsplen
= sizeof(op
->rsp_iu
);
1266 op
->fcp_req
.done
= nvme_fc_fcpio_done
;
1267 op
->fcp_req
.first_sgl
= (struct scatterlist
*)&op
[1];
1268 op
->fcp_req
.private = &op
->fcp_req
.first_sgl
[SG_CHUNK_SIZE
];
1274 cmdiu
->scsi_id
= NVME_CMD_SCSI_ID
;
1275 cmdiu
->fc_id
= NVME_CMD_FC_ID
;
1276 cmdiu
->iu_len
= cpu_to_be16(sizeof(*cmdiu
) / sizeof(u32
));
1278 op
->fcp_req
.cmddma
= fc_dma_map_single(ctrl
->lport
->dev
,
1279 &op
->cmd_iu
, sizeof(op
->cmd_iu
), DMA_TO_DEVICE
);
1280 if (fc_dma_mapping_error(ctrl
->lport
->dev
, op
->fcp_req
.cmddma
)) {
1282 "FCP Op failed - cmdiu dma mapping failed.\n");
1287 op
->fcp_req
.rspdma
= fc_dma_map_single(ctrl
->lport
->dev
,
1288 &op
->rsp_iu
, sizeof(op
->rsp_iu
),
1290 if (fc_dma_mapping_error(ctrl
->lport
->dev
, op
->fcp_req
.rspdma
)) {
1292 "FCP Op failed - rspiu dma mapping failed.\n");
1296 atomic_set(&op
->state
, FCPOP_STATE_IDLE
);
1302 nvme_fc_init_request(void *data
, struct request
*rq
,
1303 unsigned int hctx_idx
, unsigned int rq_idx
,
1304 unsigned int numa_node
)
1306 struct nvme_fc_ctrl
*ctrl
= data
;
1307 struct nvme_fc_fcp_op
*op
= blk_mq_rq_to_pdu(rq
);
1308 struct nvme_fc_queue
*queue
= &ctrl
->queues
[hctx_idx
+1];
1310 return __nvme_fc_init_request(ctrl
, queue
, op
, rq
, queue
->rqcnt
++);
1314 nvme_fc_init_admin_request(void *data
, struct request
*rq
,
1315 unsigned int hctx_idx
, unsigned int rq_idx
,
1316 unsigned int numa_node
)
1318 struct nvme_fc_ctrl
*ctrl
= data
;
1319 struct nvme_fc_fcp_op
*op
= blk_mq_rq_to_pdu(rq
);
1320 struct nvme_fc_queue
*queue
= &ctrl
->queues
[0];
1322 return __nvme_fc_init_request(ctrl
, queue
, op
, rq
, queue
->rqcnt
++);
1326 nvme_fc_init_aen_ops(struct nvme_fc_ctrl
*ctrl
)
1328 struct nvme_fc_fcp_op
*aen_op
;
1329 struct nvme_fc_cmd_iu
*cmdiu
;
1330 struct nvme_command
*sqe
;
1333 aen_op
= ctrl
->aen_ops
;
1334 for (i
= 0; i
< NVME_FC_NR_AEN_COMMANDS
; i
++, aen_op
++) {
1335 cmdiu
= &aen_op
->cmd_iu
;
1337 ret
= __nvme_fc_init_request(ctrl
, &ctrl
->queues
[0],
1338 aen_op
, (struct request
*)NULL
,
1339 (AEN_CMDID_BASE
+ i
));
1343 memset(sqe
, 0, sizeof(*sqe
));
1344 sqe
->common
.opcode
= nvme_admin_async_event
;
1345 sqe
->common
.command_id
= AEN_CMDID_BASE
+ i
;
1352 __nvme_fc_init_hctx(struct blk_mq_hw_ctx
*hctx
, struct nvme_fc_ctrl
*ctrl
,
1355 struct nvme_fc_queue
*queue
= &ctrl
->queues
[qidx
];
1357 hctx
->driver_data
= queue
;
1362 nvme_fc_init_hctx(struct blk_mq_hw_ctx
*hctx
, void *data
,
1363 unsigned int hctx_idx
)
1365 struct nvme_fc_ctrl
*ctrl
= data
;
1367 __nvme_fc_init_hctx(hctx
, ctrl
, hctx_idx
+ 1);
1373 nvme_fc_init_admin_hctx(struct blk_mq_hw_ctx
*hctx
, void *data
,
1374 unsigned int hctx_idx
)
1376 struct nvme_fc_ctrl
*ctrl
= data
;
1378 __nvme_fc_init_hctx(hctx
, ctrl
, hctx_idx
);
1384 nvme_fc_init_queue(struct nvme_fc_ctrl
*ctrl
, int idx
, size_t queue_size
)
1386 struct nvme_fc_queue
*queue
;
1388 queue
= &ctrl
->queues
[idx
];
1389 memset(queue
, 0, sizeof(*queue
));
1392 atomic_set(&queue
->csn
, 1);
1393 queue
->dev
= ctrl
->dev
;
1396 queue
->cmnd_capsule_len
= ctrl
->ctrl
.ioccsz
* 16;
1398 queue
->cmnd_capsule_len
= sizeof(struct nvme_command
);
1400 queue
->queue_size
= queue_size
;
1403 * Considered whether we should allocate buffers for all SQEs
1404 * and CQEs and dma map them - mapping their respective entries
1405 * into the request structures (kernel vm addr and dma address)
1406 * thus the driver could use the buffers/mappings directly.
1407 * It only makes sense if the LLDD would use them for its
1408 * messaging api. It's very unlikely most adapter api's would use
1409 * a native NVME sqe/cqe. More reasonable if FC-NVME IU payload
1410 * structures were used instead.
1415 * This routine terminates a queue at the transport level.
1416 * The transport has already ensured that all outstanding ios on
1417 * the queue have been terminated.
1418 * The transport will send a Disconnect LS request to terminate
1419 * the queue's connection. Termination of the admin queue will also
1420 * terminate the association at the target.
1423 nvme_fc_free_queue(struct nvme_fc_queue
*queue
)
1425 if (!test_and_clear_bit(NVME_FC_Q_CONNECTED
, &queue
->flags
))
1429 * Current implementation never disconnects a single queue.
1430 * It always terminates a whole association. So there is never
1431 * a disconnect(queue) LS sent to the target.
1434 queue
->connection_id
= 0;
1435 clear_bit(NVME_FC_Q_CONNECTED
, &queue
->flags
);
1439 __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl
*ctrl
,
1440 struct nvme_fc_queue
*queue
, unsigned int qidx
)
1442 if (ctrl
->lport
->ops
->delete_queue
)
1443 ctrl
->lport
->ops
->delete_queue(&ctrl
->lport
->localport
, qidx
,
1444 queue
->lldd_handle
);
1445 queue
->lldd_handle
= NULL
;
1449 nvme_fc_destroy_admin_queue(struct nvme_fc_ctrl
*ctrl
)
1451 __nvme_fc_delete_hw_queue(ctrl
, &ctrl
->queues
[0], 0);
1452 blk_cleanup_queue(ctrl
->ctrl
.admin_q
);
1453 blk_mq_free_tag_set(&ctrl
->admin_tag_set
);
1454 nvme_fc_free_queue(&ctrl
->queues
[0]);
1458 nvme_fc_free_io_queues(struct nvme_fc_ctrl
*ctrl
)
1462 for (i
= 1; i
< ctrl
->queue_count
; i
++)
1463 nvme_fc_free_queue(&ctrl
->queues
[i
]);
1467 __nvme_fc_create_hw_queue(struct nvme_fc_ctrl
*ctrl
,
1468 struct nvme_fc_queue
*queue
, unsigned int qidx
, u16 qsize
)
1472 queue
->lldd_handle
= NULL
;
1473 if (ctrl
->lport
->ops
->create_queue
)
1474 ret
= ctrl
->lport
->ops
->create_queue(&ctrl
->lport
->localport
,
1475 qidx
, qsize
, &queue
->lldd_handle
);
1481 nvme_fc_delete_hw_io_queues(struct nvme_fc_ctrl
*ctrl
)
1483 struct nvme_fc_queue
*queue
= &ctrl
->queues
[ctrl
->queue_count
- 1];
1486 for (i
= ctrl
->queue_count
- 1; i
>= 1; i
--, queue
--)
1487 __nvme_fc_delete_hw_queue(ctrl
, queue
, i
);
1491 nvme_fc_create_hw_io_queues(struct nvme_fc_ctrl
*ctrl
, u16 qsize
)
1493 struct nvme_fc_queue
*queue
= &ctrl
->queues
[1];
1496 for (i
= 1; i
< ctrl
->queue_count
; i
++, queue
++) {
1497 ret
= __nvme_fc_create_hw_queue(ctrl
, queue
, i
, qsize
);
1506 __nvme_fc_delete_hw_queue(ctrl
, &ctrl
->queues
[i
], i
);
1511 nvme_fc_connect_io_queues(struct nvme_fc_ctrl
*ctrl
, u16 qsize
)
1515 for (i
= 1; i
< ctrl
->queue_count
; i
++) {
1516 ret
= nvme_fc_connect_queue(ctrl
, &ctrl
->queues
[i
], qsize
,
1520 ret
= nvmf_connect_io_queue(&ctrl
->ctrl
, i
);
1529 nvme_fc_init_io_queues(struct nvme_fc_ctrl
*ctrl
)
1533 for (i
= 1; i
< ctrl
->queue_count
; i
++)
1534 nvme_fc_init_queue(ctrl
, i
, ctrl
->ctrl
.sqsize
);
1538 nvme_fc_ctrl_free(struct kref
*ref
)
1540 struct nvme_fc_ctrl
*ctrl
=
1541 container_of(ref
, struct nvme_fc_ctrl
, ref
);
1542 unsigned long flags
;
1544 if (ctrl
->state
!= FCCTRL_INIT
) {
1545 /* remove from rport list */
1546 spin_lock_irqsave(&ctrl
->rport
->lock
, flags
);
1547 list_del(&ctrl
->ctrl_list
);
1548 spin_unlock_irqrestore(&ctrl
->rport
->lock
, flags
);
1551 put_device(ctrl
->dev
);
1552 nvme_fc_rport_put(ctrl
->rport
);
1554 kfree(ctrl
->queues
);
1555 ida_simple_remove(&nvme_fc_ctrl_cnt
, ctrl
->cnum
);
1556 nvmf_free_options(ctrl
->ctrl
.opts
);
1561 nvme_fc_ctrl_put(struct nvme_fc_ctrl
*ctrl
)
1563 kref_put(&ctrl
->ref
, nvme_fc_ctrl_free
);
1567 nvme_fc_ctrl_get(struct nvme_fc_ctrl
*ctrl
)
1569 return kref_get_unless_zero(&ctrl
->ref
);
1573 * All accesses from nvme core layer done - can now free the
1574 * controller. Called after last nvme_put_ctrl() call
1577 nvme_fc_free_nvme_ctrl(struct nvme_ctrl
*nctrl
)
1579 struct nvme_fc_ctrl
*ctrl
= to_fc_ctrl(nctrl
);
1581 WARN_ON(nctrl
!= &ctrl
->ctrl
);
1584 * Tear down the association, which will generate link
1585 * traffic to terminate connections
1588 if (ctrl
->state
!= FCCTRL_INIT
) {
1589 /* send a Disconnect(association) LS to fc-nvme target */
1590 nvme_fc_xmt_disconnect_assoc(ctrl
);
1592 if (ctrl
->ctrl
.tagset
) {
1593 blk_cleanup_queue(ctrl
->ctrl
.connect_q
);
1594 blk_mq_free_tag_set(&ctrl
->tag_set
);
1595 nvme_fc_delete_hw_io_queues(ctrl
);
1596 nvme_fc_free_io_queues(ctrl
);
1599 nvme_fc_exit_aen_ops(ctrl
);
1601 nvme_fc_destroy_admin_queue(ctrl
);
1604 nvme_fc_ctrl_put(ctrl
);
1609 __nvme_fc_abort_op(struct nvme_fc_ctrl
*ctrl
, struct nvme_fc_fcp_op
*op
)
1613 state
= atomic_xchg(&op
->state
, FCPOP_STATE_ABORTED
);
1614 if (state
!= FCPOP_STATE_ACTIVE
) {
1615 atomic_set(&op
->state
, state
);
1616 return -ECANCELED
; /* fail */
1619 ctrl
->lport
->ops
->fcp_abort(&ctrl
->lport
->localport
,
1620 &ctrl
->rport
->remoteport
,
1621 op
->queue
->lldd_handle
,
1627 enum blk_eh_timer_return
1628 nvme_fc_timeout(struct request
*rq
, bool reserved
)
1630 struct nvme_fc_fcp_op
*op
= blk_mq_rq_to_pdu(rq
);
1631 struct nvme_fc_ctrl
*ctrl
= op
->ctrl
;
1635 return BLK_EH_RESET_TIMER
;
1637 ret
= __nvme_fc_abort_op(ctrl
, op
);
1639 /* io wasn't active to abort consider it done */
1640 return BLK_EH_HANDLED
;
1643 * TODO: force a controller reset
1644 * when that happens, queues will be torn down and outstanding
1645 * ios will be terminated, and the above abort, on a single io
1646 * will no longer be needed.
1649 return BLK_EH_HANDLED
;
1653 nvme_fc_map_data(struct nvme_fc_ctrl
*ctrl
, struct request
*rq
,
1654 struct nvme_fc_fcp_op
*op
)
1656 struct nvmefc_fcp_req
*freq
= &op
->fcp_req
;
1657 enum dma_data_direction dir
;
1662 if (!blk_rq_payload_bytes(rq
))
1665 freq
->sg_table
.sgl
= freq
->first_sgl
;
1666 ret
= sg_alloc_table_chained(&freq
->sg_table
,
1667 blk_rq_nr_phys_segments(rq
), freq
->sg_table
.sgl
);
1671 op
->nents
= blk_rq_map_sg(rq
->q
, rq
, freq
->sg_table
.sgl
);
1672 WARN_ON(op
->nents
> blk_rq_nr_phys_segments(rq
));
1673 dir
= (rq_data_dir(rq
) == WRITE
) ? DMA_TO_DEVICE
: DMA_FROM_DEVICE
;
1674 freq
->sg_cnt
= fc_dma_map_sg(ctrl
->lport
->dev
, freq
->sg_table
.sgl
,
1676 if (unlikely(freq
->sg_cnt
<= 0)) {
1677 sg_free_table_chained(&freq
->sg_table
, true);
1683 * TODO: blk_integrity_rq(rq) for DIF
1689 nvme_fc_unmap_data(struct nvme_fc_ctrl
*ctrl
, struct request
*rq
,
1690 struct nvme_fc_fcp_op
*op
)
1692 struct nvmefc_fcp_req
*freq
= &op
->fcp_req
;
1697 fc_dma_unmap_sg(ctrl
->lport
->dev
, freq
->sg_table
.sgl
, op
->nents
,
1698 ((rq_data_dir(rq
) == WRITE
) ?
1699 DMA_TO_DEVICE
: DMA_FROM_DEVICE
));
1701 nvme_cleanup_cmd(rq
);
1703 sg_free_table_chained(&freq
->sg_table
, true);
1709 * In FC, the queue is a logical thing. At transport connect, the target
1710 * creates its "queue" and returns a handle that is to be given to the
1711 * target whenever it posts something to the corresponding SQ. When an
1712 * SQE is sent on a SQ, FC effectively considers the SQE, or rather the
1713 * command contained within the SQE, an io, and assigns a FC exchange
1714 * to it. The SQE and the associated SQ handle are sent in the initial
1715 * CMD IU sents on the exchange. All transfers relative to the io occur
1716 * as part of the exchange. The CQE is the last thing for the io,
1717 * which is transferred (explicitly or implicitly) with the RSP IU
1718 * sent on the exchange. After the CQE is received, the FC exchange is
1719 * terminaed and the Exchange may be used on a different io.
1721 * The transport to LLDD api has the transport making a request for a
1722 * new fcp io request to the LLDD. The LLDD then allocates a FC exchange
1723 * resource and transfers the command. The LLDD will then process all
1724 * steps to complete the io. Upon completion, the transport done routine
1727 * So - while the operation is outstanding to the LLDD, there is a link
1728 * level FC exchange resource that is also outstanding. This must be
1729 * considered in all cleanup operations.
1732 nvme_fc_start_fcp_op(struct nvme_fc_ctrl
*ctrl
, struct nvme_fc_queue
*queue
,
1733 struct nvme_fc_fcp_op
*op
, u32 data_len
,
1734 enum nvmefc_fcp_datadir io_dir
)
1736 struct nvme_fc_cmd_iu
*cmdiu
= &op
->cmd_iu
;
1737 struct nvme_command
*sqe
= &cmdiu
->sqe
;
1741 if (!nvme_fc_ctrl_get(ctrl
))
1742 return BLK_MQ_RQ_QUEUE_ERROR
;
1744 /* format the FC-NVME CMD IU and fcp_req */
1745 cmdiu
->connection_id
= cpu_to_be64(queue
->connection_id
);
1746 csn
= atomic_inc_return(&queue
->csn
);
1747 cmdiu
->csn
= cpu_to_be32(csn
);
1748 cmdiu
->data_len
= cpu_to_be32(data_len
);
1750 case NVMEFC_FCP_WRITE
:
1751 cmdiu
->flags
= FCNVME_CMD_FLAGS_WRITE
;
1753 case NVMEFC_FCP_READ
:
1754 cmdiu
->flags
= FCNVME_CMD_FLAGS_READ
;
1756 case NVMEFC_FCP_NODATA
:
1760 op
->fcp_req
.payload_length
= data_len
;
1761 op
->fcp_req
.io_dir
= io_dir
;
1762 op
->fcp_req
.transferred_length
= 0;
1763 op
->fcp_req
.rcv_rsplen
= 0;
1764 op
->fcp_req
.status
= 0;
1765 op
->fcp_req
.sqid
= cpu_to_le16(queue
->qnum
);
1768 * validate per fabric rules, set fields mandated by fabric spec
1769 * as well as those by FC-NVME spec.
1771 WARN_ON_ONCE(sqe
->common
.metadata
);
1772 WARN_ON_ONCE(sqe
->common
.dptr
.prp1
);
1773 WARN_ON_ONCE(sqe
->common
.dptr
.prp2
);
1774 sqe
->common
.flags
|= NVME_CMD_SGL_METABUF
;
1777 * format SQE DPTR field per FC-NVME rules
1778 * type=data block descr; subtype=offset;
1779 * offset is currently 0.
1781 sqe
->rw
.dptr
.sgl
.type
= NVME_SGL_FMT_OFFSET
;
1782 sqe
->rw
.dptr
.sgl
.length
= cpu_to_le32(data_len
);
1783 sqe
->rw
.dptr
.sgl
.addr
= 0;
1785 /* odd that we set the command_id - should come from nvme-fabrics */
1786 WARN_ON_ONCE(sqe
->common
.command_id
!= cpu_to_le16(op
->rqno
));
1788 if (op
->rq
) { /* skipped on aens */
1789 ret
= nvme_fc_map_data(ctrl
, op
->rq
, op
);
1791 dev_err(queue
->ctrl
->ctrl
.device
,
1792 "Failed to map data (%d)\n", ret
);
1793 nvme_cleanup_cmd(op
->rq
);
1794 nvme_fc_ctrl_put(ctrl
);
1795 return (ret
== -ENOMEM
|| ret
== -EAGAIN
) ?
1796 BLK_MQ_RQ_QUEUE_BUSY
: BLK_MQ_RQ_QUEUE_ERROR
;
1800 fc_dma_sync_single_for_device(ctrl
->lport
->dev
, op
->fcp_req
.cmddma
,
1801 sizeof(op
->cmd_iu
), DMA_TO_DEVICE
);
1803 atomic_set(&op
->state
, FCPOP_STATE_ACTIVE
);
1806 blk_mq_start_request(op
->rq
);
1808 ret
= ctrl
->lport
->ops
->fcp_io(&ctrl
->lport
->localport
,
1809 &ctrl
->rport
->remoteport
,
1810 queue
->lldd_handle
, &op
->fcp_req
);
1814 "Send nvme command failed - lldd returned %d.\n", ret
);
1816 if (op
->rq
) { /* normal request */
1817 nvme_fc_unmap_data(ctrl
, op
->rq
, op
);
1818 nvme_cleanup_cmd(op
->rq
);
1820 /* else - aen. no cleanup needed */
1822 nvme_fc_ctrl_put(ctrl
);
1825 return BLK_MQ_RQ_QUEUE_ERROR
;
1828 blk_mq_stop_hw_queues(op
->rq
->q
);
1829 blk_mq_delay_queue(queue
->hctx
, NVMEFC_QUEUE_DELAY
);
1831 return BLK_MQ_RQ_QUEUE_BUSY
;
1834 return BLK_MQ_RQ_QUEUE_OK
;
1838 nvme_fc_queue_rq(struct blk_mq_hw_ctx
*hctx
,
1839 const struct blk_mq_queue_data
*bd
)
1841 struct nvme_ns
*ns
= hctx
->queue
->queuedata
;
1842 struct nvme_fc_queue
*queue
= hctx
->driver_data
;
1843 struct nvme_fc_ctrl
*ctrl
= queue
->ctrl
;
1844 struct request
*rq
= bd
->rq
;
1845 struct nvme_fc_fcp_op
*op
= blk_mq_rq_to_pdu(rq
);
1846 struct nvme_fc_cmd_iu
*cmdiu
= &op
->cmd_iu
;
1847 struct nvme_command
*sqe
= &cmdiu
->sqe
;
1848 enum nvmefc_fcp_datadir io_dir
;
1852 ret
= nvme_setup_cmd(ns
, rq
, sqe
);
1856 data_len
= blk_rq_payload_bytes(rq
);
1858 io_dir
= ((rq_data_dir(rq
) == WRITE
) ?
1859 NVMEFC_FCP_WRITE
: NVMEFC_FCP_READ
);
1861 io_dir
= NVMEFC_FCP_NODATA
;
1863 return nvme_fc_start_fcp_op(ctrl
, queue
, op
, data_len
, io_dir
);
1866 static struct blk_mq_tags
*
1867 nvme_fc_tagset(struct nvme_fc_queue
*queue
)
1869 if (queue
->qnum
== 0)
1870 return queue
->ctrl
->admin_tag_set
.tags
[queue
->qnum
];
1872 return queue
->ctrl
->tag_set
.tags
[queue
->qnum
- 1];
1876 nvme_fc_poll(struct blk_mq_hw_ctx
*hctx
, unsigned int tag
)
1879 struct nvme_fc_queue
*queue
= hctx
->driver_data
;
1880 struct nvme_fc_ctrl
*ctrl
= queue
->ctrl
;
1881 struct request
*req
;
1882 struct nvme_fc_fcp_op
*op
;
1884 req
= blk_mq_tag_to_rq(nvme_fc_tagset(queue
), tag
);
1886 dev_err(queue
->ctrl
->ctrl
.device
,
1887 "tag 0x%x on QNum %#x not found\n",
1892 op
= blk_mq_rq_to_pdu(req
);
1894 if ((atomic_read(&op
->state
) == FCPOP_STATE_ACTIVE
) &&
1895 (ctrl
->lport
->ops
->poll_queue
))
1896 ctrl
->lport
->ops
->poll_queue(&ctrl
->lport
->localport
,
1897 queue
->lldd_handle
);
1899 return ((atomic_read(&op
->state
) != FCPOP_STATE_ACTIVE
));
1903 nvme_fc_submit_async_event(struct nvme_ctrl
*arg
, int aer_idx
)
1905 struct nvme_fc_ctrl
*ctrl
= to_fc_ctrl(arg
);
1906 struct nvme_fc_fcp_op
*aen_op
;
1909 if (aer_idx
> NVME_FC_NR_AEN_COMMANDS
)
1912 aen_op
= &ctrl
->aen_ops
[aer_idx
];
1914 ret
= nvme_fc_start_fcp_op(ctrl
, aen_op
->queue
, aen_op
, 0,
1917 dev_err(ctrl
->ctrl
.device
,
1918 "failed async event work [%d]\n", aer_idx
);
1922 nvme_fc_complete_rq(struct request
*rq
)
1924 struct nvme_fc_fcp_op
*op
= blk_mq_rq_to_pdu(rq
);
1925 struct nvme_fc_ctrl
*ctrl
= op
->ctrl
;
1926 int error
= 0, state
;
1928 state
= atomic_xchg(&op
->state
, FCPOP_STATE_IDLE
);
1930 nvme_cleanup_cmd(rq
);
1932 nvme_fc_unmap_data(ctrl
, rq
, op
);
1934 if (unlikely(rq
->errors
)) {
1935 if (nvme_req_needs_retry(rq
, rq
->errors
)) {
1936 nvme_requeue_req(rq
);
1940 if (rq
->cmd_type
== REQ_TYPE_DRV_PRIV
)
1943 error
= nvme_error_status(rq
->errors
);
1946 nvme_fc_ctrl_put(ctrl
);
1948 blk_mq_end_request(rq
, error
);
1951 static struct blk_mq_ops nvme_fc_mq_ops
= {
1952 .queue_rq
= nvme_fc_queue_rq
,
1953 .complete
= nvme_fc_complete_rq
,
1954 .init_request
= nvme_fc_init_request
,
1955 .exit_request
= nvme_fc_exit_request
,
1956 .reinit_request
= nvme_fc_reinit_request
,
1957 .init_hctx
= nvme_fc_init_hctx
,
1958 .poll
= nvme_fc_poll
,
1959 .timeout
= nvme_fc_timeout
,
1962 static struct blk_mq_ops nvme_fc_admin_mq_ops
= {
1963 .queue_rq
= nvme_fc_queue_rq
,
1964 .complete
= nvme_fc_complete_rq
,
1965 .init_request
= nvme_fc_init_admin_request
,
1966 .exit_request
= nvme_fc_exit_request
,
1967 .reinit_request
= nvme_fc_reinit_request
,
1968 .init_hctx
= nvme_fc_init_admin_hctx
,
1969 .timeout
= nvme_fc_timeout
,
1973 nvme_fc_configure_admin_queue(struct nvme_fc_ctrl
*ctrl
)
1978 nvme_fc_init_queue(ctrl
, 0, NVME_FC_AQ_BLKMQ_DEPTH
);
1980 error
= nvme_fc_connect_admin_queue(ctrl
, &ctrl
->queues
[0],
1981 NVME_FC_AQ_BLKMQ_DEPTH
,
1982 (NVME_FC_AQ_BLKMQ_DEPTH
/ 4));
1986 memset(&ctrl
->admin_tag_set
, 0, sizeof(ctrl
->admin_tag_set
));
1987 ctrl
->admin_tag_set
.ops
= &nvme_fc_admin_mq_ops
;
1988 ctrl
->admin_tag_set
.queue_depth
= NVME_FC_AQ_BLKMQ_DEPTH
;
1989 ctrl
->admin_tag_set
.reserved_tags
= 2; /* fabric connect + Keep-Alive */
1990 ctrl
->admin_tag_set
.numa_node
= NUMA_NO_NODE
;
1991 ctrl
->admin_tag_set
.cmd_size
= sizeof(struct nvme_fc_fcp_op
) +
1993 sizeof(struct scatterlist
)) +
1994 ctrl
->lport
->ops
->fcprqst_priv_sz
;
1995 ctrl
->admin_tag_set
.driver_data
= ctrl
;
1996 ctrl
->admin_tag_set
.nr_hw_queues
= 1;
1997 ctrl
->admin_tag_set
.timeout
= ADMIN_TIMEOUT
;
1999 error
= blk_mq_alloc_tag_set(&ctrl
->admin_tag_set
);
2001 goto out_free_queue
;
2003 ctrl
->ctrl
.admin_q
= blk_mq_init_queue(&ctrl
->admin_tag_set
);
2004 if (IS_ERR(ctrl
->ctrl
.admin_q
)) {
2005 error
= PTR_ERR(ctrl
->ctrl
.admin_q
);
2006 goto out_free_tagset
;
2009 error
= __nvme_fc_create_hw_queue(ctrl
, &ctrl
->queues
[0], 0,
2010 NVME_FC_AQ_BLKMQ_DEPTH
);
2012 goto out_cleanup_queue
;
2014 error
= nvmf_connect_admin_queue(&ctrl
->ctrl
);
2016 goto out_delete_hw_queue
;
2018 error
= nvmf_reg_read64(&ctrl
->ctrl
, NVME_REG_CAP
, &ctrl
->cap
);
2020 dev_err(ctrl
->ctrl
.device
,
2021 "prop_get NVME_REG_CAP failed\n");
2022 goto out_delete_hw_queue
;
2026 min_t(int, NVME_CAP_MQES(ctrl
->cap
) + 1, ctrl
->ctrl
.sqsize
);
2028 error
= nvme_enable_ctrl(&ctrl
->ctrl
, ctrl
->cap
);
2030 goto out_delete_hw_queue
;
2032 segs
= min_t(u32
, NVME_FC_MAX_SEGMENTS
,
2033 ctrl
->lport
->ops
->max_sgl_segments
);
2034 ctrl
->ctrl
.max_hw_sectors
= (segs
- 1) << (PAGE_SHIFT
- 9);
2036 error
= nvme_init_identify(&ctrl
->ctrl
);
2038 goto out_delete_hw_queue
;
2040 nvme_start_keep_alive(&ctrl
->ctrl
);
2044 out_delete_hw_queue
:
2045 __nvme_fc_delete_hw_queue(ctrl
, &ctrl
->queues
[0], 0);
2047 blk_cleanup_queue(ctrl
->ctrl
.admin_q
);
2049 blk_mq_free_tag_set(&ctrl
->admin_tag_set
);
2051 nvme_fc_free_queue(&ctrl
->queues
[0]);
2056 * This routine is used by the transport when it needs to find active
2057 * io on a queue that is to be terminated. The transport uses
2058 * blk_mq_tagset_busy_itr() to find the busy requests, which then invoke
2059 * this routine to kill them on a 1 by 1 basis.
2061 * As FC allocates FC exchange for each io, the transport must contact
2062 * the LLDD to terminate the exchange, thus releasing the FC exchange.
2063 * After terminating the exchange the LLDD will call the transport's
2064 * normal io done path for the request, but it will have an aborted
2065 * status. The done path will return the io request back to the block
2066 * layer with an error status.
2069 nvme_fc_terminate_exchange(struct request
*req
, void *data
, bool reserved
)
2071 struct nvme_ctrl
*nctrl
= data
;
2072 struct nvme_fc_ctrl
*ctrl
= to_fc_ctrl(nctrl
);
2073 struct nvme_fc_fcp_op
*op
= blk_mq_rq_to_pdu(req
);
2076 if (!blk_mq_request_started(req
))
2079 /* this performs an ABTS-LS on the FC exchange for the io */
2080 status
= __nvme_fc_abort_op(ctrl
, op
);
2082 * if __nvme_fc_abort_op failed: io wasn't active to abort
2083 * consider it done. Assume completion path already completing
2087 /* io wasn't active to abort consider it done */
2088 /* assume completion path already completing in parallel */
2094 * This routine stops operation of the controller. Admin and IO queues
2095 * are stopped, outstanding ios on them terminated, and the nvme ctrl
2099 nvme_fc_shutdown_ctrl(struct nvme_fc_ctrl
*ctrl
)
2102 * If io queues are present, stop them and terminate all outstanding
2103 * ios on them. As FC allocates FC exchange for each io, the
2104 * transport must contact the LLDD to terminate the exchange,
2105 * thus releasing the FC exchange. We use blk_mq_tagset_busy_itr()
2106 * to tell us what io's are busy and invoke a transport routine
2107 * to kill them with the LLDD. After terminating the exchange
2108 * the LLDD will call the transport's normal io done path, but it
2109 * will have an aborted status. The done path will return the
2110 * io requests back to the block layer as part of normal completions
2111 * (but with error status).
2113 if (ctrl
->queue_count
> 1) {
2114 nvme_stop_queues(&ctrl
->ctrl
);
2115 blk_mq_tagset_busy_iter(&ctrl
->tag_set
,
2116 nvme_fc_terminate_exchange
, &ctrl
->ctrl
);
2119 if (ctrl
->ctrl
.state
== NVME_CTRL_LIVE
)
2120 nvme_shutdown_ctrl(&ctrl
->ctrl
);
2123 * now clean up the admin queue. Same thing as above.
2124 * use blk_mq_tagset_busy_itr() and the transport routine to
2125 * terminate the exchanges.
2127 blk_mq_stop_hw_queues(ctrl
->ctrl
.admin_q
);
2128 blk_mq_tagset_busy_iter(&ctrl
->admin_tag_set
,
2129 nvme_fc_terminate_exchange
, &ctrl
->ctrl
);
2133 * Called to teardown an association.
2134 * May be called with association fully in place or partially in place.
2137 __nvme_fc_remove_ctrl(struct nvme_fc_ctrl
*ctrl
)
2139 nvme_stop_keep_alive(&ctrl
->ctrl
);
2141 /* stop and terminate ios on admin and io queues */
2142 nvme_fc_shutdown_ctrl(ctrl
);
2145 * tear down the controller
2146 * This will result in the last reference on the nvme ctrl to
2147 * expire, calling the transport nvme_fc_free_nvme_ctrl() callback.
2148 * From there, the transport will tear down it's logical queues and
2151 nvme_uninit_ctrl(&ctrl
->ctrl
);
2153 nvme_put_ctrl(&ctrl
->ctrl
);
2157 nvme_fc_del_ctrl_work(struct work_struct
*work
)
2159 struct nvme_fc_ctrl
*ctrl
=
2160 container_of(work
, struct nvme_fc_ctrl
, delete_work
);
2162 __nvme_fc_remove_ctrl(ctrl
);
2166 __nvme_fc_del_ctrl(struct nvme_fc_ctrl
*ctrl
)
2168 if (!nvme_change_ctrl_state(&ctrl
->ctrl
, NVME_CTRL_DELETING
))
2171 if (!queue_work(nvme_fc_wq
, &ctrl
->delete_work
))
2178 * Request from nvme core layer to delete the controller
2181 nvme_fc_del_nvme_ctrl(struct nvme_ctrl
*nctrl
)
2183 struct nvme_fc_ctrl
*ctrl
= to_fc_ctrl(nctrl
);
2184 struct nvme_fc_rport
*rport
= ctrl
->rport
;
2185 unsigned long flags
;
2188 spin_lock_irqsave(&rport
->lock
, flags
);
2189 ret
= __nvme_fc_del_ctrl(ctrl
);
2190 spin_unlock_irqrestore(&rport
->lock
, flags
);
2194 flush_work(&ctrl
->delete_work
);
2200 nvme_fc_reset_nvme_ctrl(struct nvme_ctrl
*nctrl
)
2205 static const struct nvme_ctrl_ops nvme_fc_ctrl_ops
= {
2207 .module
= THIS_MODULE
,
2209 .reg_read32
= nvmf_reg_read32
,
2210 .reg_read64
= nvmf_reg_read64
,
2211 .reg_write32
= nvmf_reg_write32
,
2212 .reset_ctrl
= nvme_fc_reset_nvme_ctrl
,
2213 .free_ctrl
= nvme_fc_free_nvme_ctrl
,
2214 .submit_async_event
= nvme_fc_submit_async_event
,
2215 .delete_ctrl
= nvme_fc_del_nvme_ctrl
,
2216 .get_subsysnqn
= nvmf_get_subsysnqn
,
2217 .get_address
= nvmf_get_address
,
2221 nvme_fc_create_io_queues(struct nvme_fc_ctrl
*ctrl
)
2223 struct nvmf_ctrl_options
*opts
= ctrl
->ctrl
.opts
;
2226 ret
= nvme_set_queue_count(&ctrl
->ctrl
, &opts
->nr_io_queues
);
2228 dev_info(ctrl
->ctrl
.device
,
2229 "set_queue_count failed: %d\n", ret
);
2233 ctrl
->queue_count
= opts
->nr_io_queues
+ 1;
2234 if (!opts
->nr_io_queues
)
2237 dev_info(ctrl
->ctrl
.device
, "creating %d I/O queues.\n",
2238 opts
->nr_io_queues
);
2240 nvme_fc_init_io_queues(ctrl
);
2242 memset(&ctrl
->tag_set
, 0, sizeof(ctrl
->tag_set
));
2243 ctrl
->tag_set
.ops
= &nvme_fc_mq_ops
;
2244 ctrl
->tag_set
.queue_depth
= ctrl
->ctrl
.opts
->queue_size
;
2245 ctrl
->tag_set
.reserved_tags
= 1; /* fabric connect */
2246 ctrl
->tag_set
.numa_node
= NUMA_NO_NODE
;
2247 ctrl
->tag_set
.flags
= BLK_MQ_F_SHOULD_MERGE
;
2248 ctrl
->tag_set
.cmd_size
= sizeof(struct nvme_fc_fcp_op
) +
2250 sizeof(struct scatterlist
)) +
2251 ctrl
->lport
->ops
->fcprqst_priv_sz
;
2252 ctrl
->tag_set
.driver_data
= ctrl
;
2253 ctrl
->tag_set
.nr_hw_queues
= ctrl
->queue_count
- 1;
2254 ctrl
->tag_set
.timeout
= NVME_IO_TIMEOUT
;
2256 ret
= blk_mq_alloc_tag_set(&ctrl
->tag_set
);
2260 ctrl
->ctrl
.tagset
= &ctrl
->tag_set
;
2262 ctrl
->ctrl
.connect_q
= blk_mq_init_queue(&ctrl
->tag_set
);
2263 if (IS_ERR(ctrl
->ctrl
.connect_q
)) {
2264 ret
= PTR_ERR(ctrl
->ctrl
.connect_q
);
2265 goto out_free_tag_set
;
2268 ret
= nvme_fc_create_hw_io_queues(ctrl
, ctrl
->ctrl
.opts
->queue_size
);
2270 goto out_cleanup_blk_queue
;
2272 ret
= nvme_fc_connect_io_queues(ctrl
, ctrl
->ctrl
.opts
->queue_size
);
2274 goto out_delete_hw_queues
;
2278 out_delete_hw_queues
:
2279 nvme_fc_delete_hw_io_queues(ctrl
);
2280 out_cleanup_blk_queue
:
2281 nvme_stop_keep_alive(&ctrl
->ctrl
);
2282 blk_cleanup_queue(ctrl
->ctrl
.connect_q
);
2284 blk_mq_free_tag_set(&ctrl
->tag_set
);
2285 nvme_fc_free_io_queues(ctrl
);
2287 /* force put free routine to ignore io queues */
2288 ctrl
->ctrl
.tagset
= NULL
;
2294 static struct nvme_ctrl
*
2295 __nvme_fc_create_ctrl(struct device
*dev
, struct nvmf_ctrl_options
*opts
,
2296 struct nvme_fc_lport
*lport
, struct nvme_fc_rport
*rport
)
2298 struct nvme_fc_ctrl
*ctrl
;
2299 unsigned long flags
;
2303 ctrl
= kzalloc(sizeof(*ctrl
), GFP_KERNEL
);
2309 idx
= ida_simple_get(&nvme_fc_ctrl_cnt
, 0, 0, GFP_KERNEL
);
2315 ctrl
->ctrl
.opts
= opts
;
2316 INIT_LIST_HEAD(&ctrl
->ctrl_list
);
2317 INIT_LIST_HEAD(&ctrl
->ls_req_list
);
2318 ctrl
->lport
= lport
;
2319 ctrl
->rport
= rport
;
2320 ctrl
->dev
= lport
->dev
;
2321 ctrl
->state
= FCCTRL_INIT
;
2324 ret
= nvme_init_ctrl(&ctrl
->ctrl
, dev
, &nvme_fc_ctrl_ops
, 0);
2328 get_device(ctrl
->dev
);
2329 kref_init(&ctrl
->ref
);
2331 INIT_WORK(&ctrl
->delete_work
, nvme_fc_del_ctrl_work
);
2332 spin_lock_init(&ctrl
->lock
);
2334 /* io queue count */
2335 ctrl
->queue_count
= min_t(unsigned int,
2337 lport
->ops
->max_hw_queues
);
2338 opts
->nr_io_queues
= ctrl
->queue_count
; /* so opts has valid value */
2339 ctrl
->queue_count
++; /* +1 for admin queue */
2341 ctrl
->ctrl
.sqsize
= opts
->queue_size
- 1;
2342 ctrl
->ctrl
.kato
= opts
->kato
;
2345 ctrl
->queues
= kcalloc(ctrl
->queue_count
, sizeof(struct nvme_fc_queue
),
2348 goto out_uninit_ctrl
;
2350 ret
= nvme_fc_configure_admin_queue(ctrl
);
2352 goto out_uninit_ctrl
;
2356 /* FC-NVME supports 64-byte SQE only */
2357 if (ctrl
->ctrl
.ioccsz
!= 4) {
2358 dev_err(ctrl
->ctrl
.device
, "ioccsz %d is not supported!\n",
2360 goto out_remove_admin_queue
;
2362 /* FC-NVME supports 16-byte CQE only */
2363 if (ctrl
->ctrl
.iorcsz
!= 1) {
2364 dev_err(ctrl
->ctrl
.device
, "iorcsz %d is not supported!\n",
2366 goto out_remove_admin_queue
;
2368 /* FC-NVME does not have other data in the capsule */
2369 if (ctrl
->ctrl
.icdoff
) {
2370 dev_err(ctrl
->ctrl
.device
, "icdoff %d is not supported!\n",
2372 goto out_remove_admin_queue
;
2375 /* FC-NVME supports normal SGL Data Block Descriptors */
2377 if (opts
->queue_size
> ctrl
->ctrl
.maxcmd
) {
2378 /* warn if maxcmd is lower than queue_size */
2379 dev_warn(ctrl
->ctrl
.device
,
2380 "queue_size %zu > ctrl maxcmd %u, reducing "
2382 opts
->queue_size
, ctrl
->ctrl
.maxcmd
);
2383 opts
->queue_size
= ctrl
->ctrl
.maxcmd
;
2386 ret
= nvme_fc_init_aen_ops(ctrl
);
2388 goto out_exit_aen_ops
;
2390 if (ctrl
->queue_count
> 1) {
2391 ret
= nvme_fc_create_io_queues(ctrl
);
2393 goto out_exit_aen_ops
;
2396 spin_lock_irqsave(&ctrl
->lock
, flags
);
2397 ctrl
->state
= FCCTRL_ACTIVE
;
2398 spin_unlock_irqrestore(&ctrl
->lock
, flags
);
2400 changed
= nvme_change_ctrl_state(&ctrl
->ctrl
, NVME_CTRL_LIVE
);
2401 WARN_ON_ONCE(!changed
);
2403 dev_info(ctrl
->ctrl
.device
,
2404 "NVME-FC{%d}: new ctrl: NQN \"%s\"\n",
2405 ctrl
->cnum
, ctrl
->ctrl
.opts
->subsysnqn
);
2407 kref_get(&ctrl
->ctrl
.kref
);
2409 spin_lock_irqsave(&rport
->lock
, flags
);
2410 list_add_tail(&ctrl
->ctrl_list
, &rport
->ctrl_list
);
2411 spin_unlock_irqrestore(&rport
->lock
, flags
);
2413 if (opts
->nr_io_queues
) {
2414 nvme_queue_scan(&ctrl
->ctrl
);
2415 nvme_queue_async_events(&ctrl
->ctrl
);
2421 nvme_fc_exit_aen_ops(ctrl
);
2422 out_remove_admin_queue
:
2423 /* send a Disconnect(association) LS to fc-nvme target */
2424 nvme_fc_xmt_disconnect_assoc(ctrl
);
2425 nvme_stop_keep_alive(&ctrl
->ctrl
);
2426 nvme_fc_destroy_admin_queue(ctrl
);
2428 nvme_uninit_ctrl(&ctrl
->ctrl
);
2429 nvme_put_ctrl(&ctrl
->ctrl
);
2432 /* exit via here will follow ctlr ref point callbacks to free */
2433 return ERR_PTR(ret
);
2436 ida_simple_remove(&nvme_fc_ctrl_cnt
, ctrl
->cnum
);
2440 nvme_fc_rport_put(rport
);
2441 /* exit via here doesn't follow ctlr ref points */
2442 return ERR_PTR(ret
);
2447 FCT_TRADDR_WWNN
= 1 << 0,
2448 FCT_TRADDR_WWPN
= 1 << 1,
2451 struct nvmet_fc_traddr
{
2456 static const match_table_t traddr_opt_tokens
= {
2457 { FCT_TRADDR_WWNN
, "nn-%s" },
2458 { FCT_TRADDR_WWPN
, "pn-%s" },
2459 { FCT_TRADDR_ERR
, NULL
}
2463 nvme_fc_parse_address(struct nvmet_fc_traddr
*traddr
, char *buf
)
2465 substring_t args
[MAX_OPT_ARGS
];
2466 char *options
, *o
, *p
;
2470 options
= o
= kstrdup(buf
, GFP_KERNEL
);
2474 while ((p
= strsep(&o
, ":\n")) != NULL
) {
2478 token
= match_token(p
, traddr_opt_tokens
, args
);
2480 case FCT_TRADDR_WWNN
:
2481 if (match_u64(args
, &token64
)) {
2485 traddr
->nn
= token64
;
2487 case FCT_TRADDR_WWPN
:
2488 if (match_u64(args
, &token64
)) {
2492 traddr
->pn
= token64
;
2495 pr_warn("unknown traddr token or missing value '%s'\n",
2507 static struct nvme_ctrl
*
2508 nvme_fc_create_ctrl(struct device
*dev
, struct nvmf_ctrl_options
*opts
)
2510 struct nvme_fc_lport
*lport
;
2511 struct nvme_fc_rport
*rport
;
2512 struct nvmet_fc_traddr laddr
= { 0L, 0L };
2513 struct nvmet_fc_traddr raddr
= { 0L, 0L };
2514 unsigned long flags
;
2517 ret
= nvme_fc_parse_address(&raddr
, opts
->traddr
);
2518 if (ret
|| !raddr
.nn
|| !raddr
.pn
)
2519 return ERR_PTR(-EINVAL
);
2521 ret
= nvme_fc_parse_address(&laddr
, opts
->host_traddr
);
2522 if (ret
|| !laddr
.nn
|| !laddr
.pn
)
2523 return ERR_PTR(-EINVAL
);
2525 /* find the host and remote ports to connect together */
2526 spin_lock_irqsave(&nvme_fc_lock
, flags
);
2527 list_for_each_entry(lport
, &nvme_fc_lport_list
, port_list
) {
2528 if (lport
->localport
.node_name
!= laddr
.nn
||
2529 lport
->localport
.port_name
!= laddr
.pn
)
2532 list_for_each_entry(rport
, &lport
->endp_list
, endp_list
) {
2533 if (rport
->remoteport
.node_name
!= raddr
.nn
||
2534 rport
->remoteport
.port_name
!= raddr
.pn
)
2537 /* if fail to get reference fall through. Will error */
2538 if (!nvme_fc_rport_get(rport
))
2541 spin_unlock_irqrestore(&nvme_fc_lock
, flags
);
2543 return __nvme_fc_create_ctrl(dev
, opts
, lport
, rport
);
2546 spin_unlock_irqrestore(&nvme_fc_lock
, flags
);
2548 return ERR_PTR(-ENOENT
);
2552 static struct nvmf_transport_ops nvme_fc_transport
= {
2554 .required_opts
= NVMF_OPT_TRADDR
| NVMF_OPT_HOST_TRADDR
,
2555 .allowed_opts
= NVMF_OPT_RECONNECT_DELAY
,
2556 .create_ctrl
= nvme_fc_create_ctrl
,
2559 static int __init
nvme_fc_init_module(void)
2561 nvme_fc_wq
= create_workqueue("nvme_fc_wq");
2565 nvmf_register_transport(&nvme_fc_transport
);
2569 static void __exit
nvme_fc_exit_module(void)
2571 /* sanity check - all lports should be removed */
2572 if (!list_empty(&nvme_fc_lport_list
))
2573 pr_warn("%s: localport list not empty\n", __func__
);
2575 nvmf_unregister_transport(&nvme_fc_transport
);
2577 destroy_workqueue(nvme_fc_wq
);
2579 ida_destroy(&nvme_fc_local_port_cnt
);
2580 ida_destroy(&nvme_fc_ctrl_cnt
);
2583 module_init(nvme_fc_init_module
);
2584 module_exit(nvme_fc_exit_module
);
2586 MODULE_LICENSE("GPL v2");