2 * NVMe over Fabrics RDMA host code.
3 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
15 #include <linux/module.h>
16 #include <linux/init.h>
17 #include <linux/slab.h>
18 #include <linux/err.h>
19 #include <linux/string.h>
20 #include <linux/atomic.h>
21 #include <linux/blk-mq.h>
22 #include <linux/blk-mq-rdma.h>
23 #include <linux/types.h>
24 #include <linux/list.h>
25 #include <linux/mutex.h>
26 #include <linux/scatterlist.h>
27 #include <linux/nvme.h>
28 #include <asm/unaligned.h>
30 #include <rdma/ib_verbs.h>
31 #include <rdma/rdma_cm.h>
32 #include <linux/nvme-rdma.h>
38 #define NVME_RDMA_CONNECT_TIMEOUT_MS 3000 /* 3 second */
40 #define NVME_RDMA_MAX_SEGMENTS 256
42 #define NVME_RDMA_MAX_INLINE_SEGMENTS 1
44 struct nvme_rdma_device
{
45 struct ib_device
*dev
;
48 struct list_head entry
;
57 struct nvme_rdma_queue
;
58 struct nvme_rdma_request
{
59 struct nvme_request req
;
61 struct nvme_rdma_qe sqe
;
62 struct ib_sge sge
[1 + NVME_RDMA_MAX_INLINE_SEGMENTS
];
66 struct ib_reg_wr reg_wr
;
67 struct ib_cqe reg_cqe
;
68 struct nvme_rdma_queue
*queue
;
69 struct sg_table sg_table
;
70 struct scatterlist first_sgl
[];
73 enum nvme_rdma_queue_flags
{
74 NVME_RDMA_Q_ALLOCATED
= 0,
78 struct nvme_rdma_queue
{
79 struct nvme_rdma_qe
*rsp_ring
;
82 size_t cmnd_capsule_len
;
83 struct nvme_rdma_ctrl
*ctrl
;
84 struct nvme_rdma_device
*device
;
89 struct rdma_cm_id
*cm_id
;
91 struct completion cm_done
;
94 struct nvme_rdma_ctrl
{
95 /* read only in the hot path */
96 struct nvme_rdma_queue
*queues
;
98 /* other member variables */
99 struct blk_mq_tag_set tag_set
;
100 struct work_struct err_work
;
102 struct nvme_rdma_qe async_event_sqe
;
104 struct delayed_work reconnect_work
;
106 struct list_head list
;
108 struct blk_mq_tag_set admin_tag_set
;
109 struct nvme_rdma_device
*device
;
113 struct sockaddr_storage addr
;
114 struct sockaddr_storage src_addr
;
116 struct nvme_ctrl ctrl
;
119 static inline struct nvme_rdma_ctrl
*to_rdma_ctrl(struct nvme_ctrl
*ctrl
)
121 return container_of(ctrl
, struct nvme_rdma_ctrl
, ctrl
);
124 static LIST_HEAD(device_list
);
125 static DEFINE_MUTEX(device_list_mutex
);
127 static LIST_HEAD(nvme_rdma_ctrl_list
);
128 static DEFINE_MUTEX(nvme_rdma_ctrl_mutex
);
131 * Disabling this option makes small I/O goes faster, but is fundamentally
132 * unsafe. With it turned off we will have to register a global rkey that
133 * allows read and write access to all physical memory.
135 static bool register_always
= true;
136 module_param(register_always
, bool, 0444);
137 MODULE_PARM_DESC(register_always
,
138 "Use memory registration even for contiguous memory regions");
140 static int nvme_rdma_cm_handler(struct rdma_cm_id
*cm_id
,
141 struct rdma_cm_event
*event
);
142 static void nvme_rdma_recv_done(struct ib_cq
*cq
, struct ib_wc
*wc
);
144 static const struct blk_mq_ops nvme_rdma_mq_ops
;
145 static const struct blk_mq_ops nvme_rdma_admin_mq_ops
;
147 /* XXX: really should move to a generic header sooner or later.. */
148 static inline void put_unaligned_le24(u32 val
, u8
*p
)
155 static inline int nvme_rdma_queue_idx(struct nvme_rdma_queue
*queue
)
157 return queue
- queue
->ctrl
->queues
;
160 static inline size_t nvme_rdma_inline_data_size(struct nvme_rdma_queue
*queue
)
162 return queue
->cmnd_capsule_len
- sizeof(struct nvme_command
);
165 static void nvme_rdma_free_qe(struct ib_device
*ibdev
, struct nvme_rdma_qe
*qe
,
166 size_t capsule_size
, enum dma_data_direction dir
)
168 ib_dma_unmap_single(ibdev
, qe
->dma
, capsule_size
, dir
);
172 static int nvme_rdma_alloc_qe(struct ib_device
*ibdev
, struct nvme_rdma_qe
*qe
,
173 size_t capsule_size
, enum dma_data_direction dir
)
175 qe
->data
= kzalloc(capsule_size
, GFP_KERNEL
);
179 qe
->dma
= ib_dma_map_single(ibdev
, qe
->data
, capsule_size
, dir
);
180 if (ib_dma_mapping_error(ibdev
, qe
->dma
)) {
188 static void nvme_rdma_free_ring(struct ib_device
*ibdev
,
189 struct nvme_rdma_qe
*ring
, size_t ib_queue_size
,
190 size_t capsule_size
, enum dma_data_direction dir
)
194 for (i
= 0; i
< ib_queue_size
; i
++)
195 nvme_rdma_free_qe(ibdev
, &ring
[i
], capsule_size
, dir
);
199 static struct nvme_rdma_qe
*nvme_rdma_alloc_ring(struct ib_device
*ibdev
,
200 size_t ib_queue_size
, size_t capsule_size
,
201 enum dma_data_direction dir
)
203 struct nvme_rdma_qe
*ring
;
206 ring
= kcalloc(ib_queue_size
, sizeof(struct nvme_rdma_qe
), GFP_KERNEL
);
210 for (i
= 0; i
< ib_queue_size
; i
++) {
211 if (nvme_rdma_alloc_qe(ibdev
, &ring
[i
], capsule_size
, dir
))
218 nvme_rdma_free_ring(ibdev
, ring
, i
, capsule_size
, dir
);
222 static void nvme_rdma_qp_event(struct ib_event
*event
, void *context
)
224 pr_debug("QP event %s (%d)\n",
225 ib_event_msg(event
->event
), event
->event
);
229 static int nvme_rdma_wait_for_cm(struct nvme_rdma_queue
*queue
)
231 wait_for_completion_interruptible_timeout(&queue
->cm_done
,
232 msecs_to_jiffies(NVME_RDMA_CONNECT_TIMEOUT_MS
) + 1);
233 return queue
->cm_error
;
236 static int nvme_rdma_create_qp(struct nvme_rdma_queue
*queue
, const int factor
)
238 struct nvme_rdma_device
*dev
= queue
->device
;
239 struct ib_qp_init_attr init_attr
;
242 memset(&init_attr
, 0, sizeof(init_attr
));
243 init_attr
.event_handler
= nvme_rdma_qp_event
;
245 init_attr
.cap
.max_send_wr
= factor
* queue
->queue_size
+ 1;
247 init_attr
.cap
.max_recv_wr
= queue
->queue_size
+ 1;
248 init_attr
.cap
.max_recv_sge
= 1;
249 init_attr
.cap
.max_send_sge
= 1 + NVME_RDMA_MAX_INLINE_SEGMENTS
;
250 init_attr
.sq_sig_type
= IB_SIGNAL_REQ_WR
;
251 init_attr
.qp_type
= IB_QPT_RC
;
252 init_attr
.send_cq
= queue
->ib_cq
;
253 init_attr
.recv_cq
= queue
->ib_cq
;
255 ret
= rdma_create_qp(queue
->cm_id
, dev
->pd
, &init_attr
);
257 queue
->qp
= queue
->cm_id
->qp
;
261 static int nvme_rdma_reinit_request(void *data
, struct request
*rq
)
263 struct nvme_rdma_ctrl
*ctrl
= data
;
264 struct nvme_rdma_device
*dev
= ctrl
->device
;
265 struct nvme_rdma_request
*req
= blk_mq_rq_to_pdu(rq
);
268 if (WARN_ON_ONCE(!req
->mr
))
271 ib_dereg_mr(req
->mr
);
273 req
->mr
= ib_alloc_mr(dev
->pd
, IB_MR_TYPE_MEM_REG
,
275 if (IS_ERR(req
->mr
)) {
276 ret
= PTR_ERR(req
->mr
);
281 req
->mr
->need_inval
= false;
287 static void nvme_rdma_exit_request(struct blk_mq_tag_set
*set
,
288 struct request
*rq
, unsigned int hctx_idx
)
290 struct nvme_rdma_ctrl
*ctrl
= set
->driver_data
;
291 struct nvme_rdma_request
*req
= blk_mq_rq_to_pdu(rq
);
292 int queue_idx
= (set
== &ctrl
->tag_set
) ? hctx_idx
+ 1 : 0;
293 struct nvme_rdma_queue
*queue
= &ctrl
->queues
[queue_idx
];
294 struct nvme_rdma_device
*dev
= queue
->device
;
297 ib_dereg_mr(req
->mr
);
299 nvme_rdma_free_qe(dev
->dev
, &req
->sqe
, sizeof(struct nvme_command
),
303 static int nvme_rdma_init_request(struct blk_mq_tag_set
*set
,
304 struct request
*rq
, unsigned int hctx_idx
,
305 unsigned int numa_node
)
307 struct nvme_rdma_ctrl
*ctrl
= set
->driver_data
;
308 struct nvme_rdma_request
*req
= blk_mq_rq_to_pdu(rq
);
309 int queue_idx
= (set
== &ctrl
->tag_set
) ? hctx_idx
+ 1 : 0;
310 struct nvme_rdma_queue
*queue
= &ctrl
->queues
[queue_idx
];
311 struct nvme_rdma_device
*dev
= queue
->device
;
312 struct ib_device
*ibdev
= dev
->dev
;
315 ret
= nvme_rdma_alloc_qe(ibdev
, &req
->sqe
, sizeof(struct nvme_command
),
320 req
->mr
= ib_alloc_mr(dev
->pd
, IB_MR_TYPE_MEM_REG
,
322 if (IS_ERR(req
->mr
)) {
323 ret
= PTR_ERR(req
->mr
);
332 nvme_rdma_free_qe(dev
->dev
, &req
->sqe
, sizeof(struct nvme_command
),
337 static int nvme_rdma_init_hctx(struct blk_mq_hw_ctx
*hctx
, void *data
,
338 unsigned int hctx_idx
)
340 struct nvme_rdma_ctrl
*ctrl
= data
;
341 struct nvme_rdma_queue
*queue
= &ctrl
->queues
[hctx_idx
+ 1];
343 BUG_ON(hctx_idx
>= ctrl
->ctrl
.queue_count
);
345 hctx
->driver_data
= queue
;
349 static int nvme_rdma_init_admin_hctx(struct blk_mq_hw_ctx
*hctx
, void *data
,
350 unsigned int hctx_idx
)
352 struct nvme_rdma_ctrl
*ctrl
= data
;
353 struct nvme_rdma_queue
*queue
= &ctrl
->queues
[0];
355 BUG_ON(hctx_idx
!= 0);
357 hctx
->driver_data
= queue
;
361 static void nvme_rdma_free_dev(struct kref
*ref
)
363 struct nvme_rdma_device
*ndev
=
364 container_of(ref
, struct nvme_rdma_device
, ref
);
366 mutex_lock(&device_list_mutex
);
367 list_del(&ndev
->entry
);
368 mutex_unlock(&device_list_mutex
);
370 ib_dealloc_pd(ndev
->pd
);
374 static void nvme_rdma_dev_put(struct nvme_rdma_device
*dev
)
376 kref_put(&dev
->ref
, nvme_rdma_free_dev
);
379 static int nvme_rdma_dev_get(struct nvme_rdma_device
*dev
)
381 return kref_get_unless_zero(&dev
->ref
);
384 static struct nvme_rdma_device
*
385 nvme_rdma_find_get_device(struct rdma_cm_id
*cm_id
)
387 struct nvme_rdma_device
*ndev
;
389 mutex_lock(&device_list_mutex
);
390 list_for_each_entry(ndev
, &device_list
, entry
) {
391 if (ndev
->dev
->node_guid
== cm_id
->device
->node_guid
&&
392 nvme_rdma_dev_get(ndev
))
396 ndev
= kzalloc(sizeof(*ndev
), GFP_KERNEL
);
400 ndev
->dev
= cm_id
->device
;
401 kref_init(&ndev
->ref
);
403 ndev
->pd
= ib_alloc_pd(ndev
->dev
,
404 register_always
? 0 : IB_PD_UNSAFE_GLOBAL_RKEY
);
405 if (IS_ERR(ndev
->pd
))
408 if (!(ndev
->dev
->attrs
.device_cap_flags
&
409 IB_DEVICE_MEM_MGT_EXTENSIONS
)) {
410 dev_err(&ndev
->dev
->dev
,
411 "Memory registrations not supported.\n");
415 list_add(&ndev
->entry
, &device_list
);
417 mutex_unlock(&device_list_mutex
);
421 ib_dealloc_pd(ndev
->pd
);
425 mutex_unlock(&device_list_mutex
);
429 static void nvme_rdma_destroy_queue_ib(struct nvme_rdma_queue
*queue
)
431 struct nvme_rdma_device
*dev
= queue
->device
;
432 struct ib_device
*ibdev
= dev
->dev
;
434 rdma_destroy_qp(queue
->cm_id
);
435 ib_free_cq(queue
->ib_cq
);
437 nvme_rdma_free_ring(ibdev
, queue
->rsp_ring
, queue
->queue_size
,
438 sizeof(struct nvme_completion
), DMA_FROM_DEVICE
);
440 nvme_rdma_dev_put(dev
);
443 static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue
*queue
)
445 struct ib_device
*ibdev
;
446 const int send_wr_factor
= 3; /* MR, SEND, INV */
447 const int cq_factor
= send_wr_factor
+ 1; /* + RECV */
448 int comp_vector
, idx
= nvme_rdma_queue_idx(queue
);
451 queue
->device
= nvme_rdma_find_get_device(queue
->cm_id
);
452 if (!queue
->device
) {
453 dev_err(queue
->cm_id
->device
->dev
.parent
,
454 "no client data found!\n");
455 return -ECONNREFUSED
;
457 ibdev
= queue
->device
->dev
;
460 * Spread I/O queues completion vectors according their queue index.
461 * Admin queues can always go on completion vector 0.
463 comp_vector
= idx
== 0 ? idx
: idx
- 1;
465 /* +1 for ib_stop_cq */
466 queue
->ib_cq
= ib_alloc_cq(ibdev
, queue
,
467 cq_factor
* queue
->queue_size
+ 1,
468 comp_vector
, IB_POLL_SOFTIRQ
);
469 if (IS_ERR(queue
->ib_cq
)) {
470 ret
= PTR_ERR(queue
->ib_cq
);
474 ret
= nvme_rdma_create_qp(queue
, send_wr_factor
);
476 goto out_destroy_ib_cq
;
478 queue
->rsp_ring
= nvme_rdma_alloc_ring(ibdev
, queue
->queue_size
,
479 sizeof(struct nvme_completion
), DMA_FROM_DEVICE
);
480 if (!queue
->rsp_ring
) {
488 rdma_destroy_qp(queue
->cm_id
);
490 ib_free_cq(queue
->ib_cq
);
492 nvme_rdma_dev_put(queue
->device
);
496 static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl
*ctrl
,
497 int idx
, size_t queue_size
)
499 struct nvme_rdma_queue
*queue
;
500 struct sockaddr
*src_addr
= NULL
;
503 queue
= &ctrl
->queues
[idx
];
505 init_completion(&queue
->cm_done
);
508 queue
->cmnd_capsule_len
= ctrl
->ctrl
.ioccsz
* 16;
510 queue
->cmnd_capsule_len
= sizeof(struct nvme_command
);
512 queue
->queue_size
= queue_size
;
513 atomic_set(&queue
->sig_count
, 0);
515 queue
->cm_id
= rdma_create_id(&init_net
, nvme_rdma_cm_handler
, queue
,
516 RDMA_PS_TCP
, IB_QPT_RC
);
517 if (IS_ERR(queue
->cm_id
)) {
518 dev_info(ctrl
->ctrl
.device
,
519 "failed to create CM ID: %ld\n", PTR_ERR(queue
->cm_id
));
520 return PTR_ERR(queue
->cm_id
);
523 if (ctrl
->ctrl
.opts
->mask
& NVMF_OPT_HOST_TRADDR
)
524 src_addr
= (struct sockaddr
*)&ctrl
->src_addr
;
526 queue
->cm_error
= -ETIMEDOUT
;
527 ret
= rdma_resolve_addr(queue
->cm_id
, src_addr
,
528 (struct sockaddr
*)&ctrl
->addr
,
529 NVME_RDMA_CONNECT_TIMEOUT_MS
);
531 dev_info(ctrl
->ctrl
.device
,
532 "rdma_resolve_addr failed (%d).\n", ret
);
533 goto out_destroy_cm_id
;
536 ret
= nvme_rdma_wait_for_cm(queue
);
538 dev_info(ctrl
->ctrl
.device
,
539 "rdma connection establishment failed (%d)\n", ret
);
540 goto out_destroy_cm_id
;
543 set_bit(NVME_RDMA_Q_ALLOCATED
, &queue
->flags
);
548 rdma_destroy_id(queue
->cm_id
);
552 static void nvme_rdma_stop_queue(struct nvme_rdma_queue
*queue
)
554 if (!test_and_clear_bit(NVME_RDMA_Q_LIVE
, &queue
->flags
))
557 rdma_disconnect(queue
->cm_id
);
558 ib_drain_qp(queue
->qp
);
561 static void nvme_rdma_free_queue(struct nvme_rdma_queue
*queue
)
563 if (!test_and_clear_bit(NVME_RDMA_Q_ALLOCATED
, &queue
->flags
))
566 if (nvme_rdma_queue_idx(queue
) == 0) {
567 nvme_rdma_free_qe(queue
->device
->dev
,
568 &queue
->ctrl
->async_event_sqe
,
569 sizeof(struct nvme_command
), DMA_TO_DEVICE
);
572 nvme_rdma_destroy_queue_ib(queue
);
573 rdma_destroy_id(queue
->cm_id
);
576 static void nvme_rdma_free_io_queues(struct nvme_rdma_ctrl
*ctrl
)
580 for (i
= 1; i
< ctrl
->ctrl
.queue_count
; i
++)
581 nvme_rdma_free_queue(&ctrl
->queues
[i
]);
584 static void nvme_rdma_stop_io_queues(struct nvme_rdma_ctrl
*ctrl
)
588 for (i
= 1; i
< ctrl
->ctrl
.queue_count
; i
++)
589 nvme_rdma_stop_queue(&ctrl
->queues
[i
]);
592 static int nvme_rdma_start_queue(struct nvme_rdma_ctrl
*ctrl
, int idx
)
597 ret
= nvmf_connect_io_queue(&ctrl
->ctrl
, idx
);
599 ret
= nvmf_connect_admin_queue(&ctrl
->ctrl
);
602 set_bit(NVME_RDMA_Q_LIVE
, &ctrl
->queues
[idx
].flags
);
604 dev_info(ctrl
->ctrl
.device
,
605 "failed to connect queue: %d ret=%d\n", idx
, ret
);
609 static int nvme_rdma_start_io_queues(struct nvme_rdma_ctrl
*ctrl
)
613 for (i
= 1; i
< ctrl
->ctrl
.queue_count
; i
++) {
614 ret
= nvme_rdma_start_queue(ctrl
, i
);
616 goto out_stop_queues
;
622 for (i
--; i
>= 1; i
--)
623 nvme_rdma_stop_queue(&ctrl
->queues
[i
]);
627 static int nvme_rdma_alloc_io_queues(struct nvme_rdma_ctrl
*ctrl
)
629 struct nvmf_ctrl_options
*opts
= ctrl
->ctrl
.opts
;
630 struct ib_device
*ibdev
= ctrl
->device
->dev
;
631 unsigned int nr_io_queues
;
634 nr_io_queues
= min(opts
->nr_io_queues
, num_online_cpus());
637 * we map queues according to the device irq vectors for
638 * optimal locality so we don't need more queues than
639 * completion vectors.
641 nr_io_queues
= min_t(unsigned int, nr_io_queues
,
642 ibdev
->num_comp_vectors
);
644 ret
= nvme_set_queue_count(&ctrl
->ctrl
, &nr_io_queues
);
648 ctrl
->ctrl
.queue_count
= nr_io_queues
+ 1;
649 if (ctrl
->ctrl
.queue_count
< 2)
652 dev_info(ctrl
->ctrl
.device
,
653 "creating %d I/O queues.\n", nr_io_queues
);
655 for (i
= 1; i
< ctrl
->ctrl
.queue_count
; i
++) {
656 ret
= nvme_rdma_alloc_queue(ctrl
, i
,
657 ctrl
->ctrl
.sqsize
+ 1);
659 goto out_free_queues
;
665 for (i
--; i
>= 1; i
--)
666 nvme_rdma_free_queue(&ctrl
->queues
[i
]);
671 static void nvme_rdma_free_tagset(struct nvme_ctrl
*nctrl
,
672 struct blk_mq_tag_set
*set
)
674 struct nvme_rdma_ctrl
*ctrl
= to_rdma_ctrl(nctrl
);
676 blk_mq_free_tag_set(set
);
677 nvme_rdma_dev_put(ctrl
->device
);
680 static struct blk_mq_tag_set
*nvme_rdma_alloc_tagset(struct nvme_ctrl
*nctrl
,
683 struct nvme_rdma_ctrl
*ctrl
= to_rdma_ctrl(nctrl
);
684 struct blk_mq_tag_set
*set
;
688 set
= &ctrl
->admin_tag_set
;
689 memset(set
, 0, sizeof(*set
));
690 set
->ops
= &nvme_rdma_admin_mq_ops
;
691 set
->queue_depth
= NVME_AQ_MQ_TAG_DEPTH
;
692 set
->reserved_tags
= 2; /* connect + keep-alive */
693 set
->numa_node
= NUMA_NO_NODE
;
694 set
->cmd_size
= sizeof(struct nvme_rdma_request
) +
695 SG_CHUNK_SIZE
* sizeof(struct scatterlist
);
696 set
->driver_data
= ctrl
;
697 set
->nr_hw_queues
= 1;
698 set
->timeout
= ADMIN_TIMEOUT
;
699 set
->flags
= BLK_MQ_F_NO_SCHED
;
701 set
= &ctrl
->tag_set
;
702 memset(set
, 0, sizeof(*set
));
703 set
->ops
= &nvme_rdma_mq_ops
;
704 set
->queue_depth
= nctrl
->opts
->queue_size
;
705 set
->reserved_tags
= 1; /* fabric connect */
706 set
->numa_node
= NUMA_NO_NODE
;
707 set
->flags
= BLK_MQ_F_SHOULD_MERGE
;
708 set
->cmd_size
= sizeof(struct nvme_rdma_request
) +
709 SG_CHUNK_SIZE
* sizeof(struct scatterlist
);
710 set
->driver_data
= ctrl
;
711 set
->nr_hw_queues
= nctrl
->queue_count
- 1;
712 set
->timeout
= NVME_IO_TIMEOUT
;
715 ret
= blk_mq_alloc_tag_set(set
);
720 * We need a reference on the device as long as the tag_set is alive,
721 * as the MRs in the request structures need a valid ib_device.
723 ret
= nvme_rdma_dev_get(ctrl
->device
);
726 goto out_free_tagset
;
732 blk_mq_free_tag_set(set
);
737 static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl
*ctrl
,
740 nvme_rdma_stop_queue(&ctrl
->queues
[0]);
742 blk_cleanup_queue(ctrl
->ctrl
.admin_q
);
743 nvme_rdma_free_tagset(&ctrl
->ctrl
, ctrl
->ctrl
.admin_tagset
);
745 nvme_rdma_free_queue(&ctrl
->queues
[0]);
748 static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl
*ctrl
,
753 error
= nvme_rdma_alloc_queue(ctrl
, 0, NVME_AQ_DEPTH
);
757 ctrl
->device
= ctrl
->queues
[0].device
;
759 ctrl
->max_fr_pages
= min_t(u32
, NVME_RDMA_MAX_SEGMENTS
,
760 ctrl
->device
->dev
->attrs
.max_fast_reg_page_list_len
);
763 ctrl
->ctrl
.admin_tagset
= nvme_rdma_alloc_tagset(&ctrl
->ctrl
, true);
764 if (IS_ERR(ctrl
->ctrl
.admin_tagset
)) {
765 error
= PTR_ERR(ctrl
->ctrl
.admin_tagset
);
769 ctrl
->ctrl
.admin_q
= blk_mq_init_queue(&ctrl
->admin_tag_set
);
770 if (IS_ERR(ctrl
->ctrl
.admin_q
)) {
771 error
= PTR_ERR(ctrl
->ctrl
.admin_q
);
772 goto out_free_tagset
;
775 error
= nvme_reinit_tagset(&ctrl
->ctrl
, ctrl
->ctrl
.admin_tagset
);
780 error
= nvme_rdma_start_queue(ctrl
, 0);
782 goto out_cleanup_queue
;
784 error
= ctrl
->ctrl
.ops
->reg_read64(&ctrl
->ctrl
, NVME_REG_CAP
,
787 dev_err(ctrl
->ctrl
.device
,
788 "prop_get NVME_REG_CAP failed\n");
789 goto out_cleanup_queue
;
793 min_t(int, NVME_CAP_MQES(ctrl
->ctrl
.cap
), ctrl
->ctrl
.sqsize
);
795 error
= nvme_enable_ctrl(&ctrl
->ctrl
, ctrl
->ctrl
.cap
);
797 goto out_cleanup_queue
;
799 ctrl
->ctrl
.max_hw_sectors
=
800 (ctrl
->max_fr_pages
- 1) << (ilog2(SZ_4K
) - 9);
802 error
= nvme_init_identify(&ctrl
->ctrl
);
804 goto out_cleanup_queue
;
806 error
= nvme_rdma_alloc_qe(ctrl
->queues
[0].device
->dev
,
807 &ctrl
->async_event_sqe
, sizeof(struct nvme_command
),
810 goto out_cleanup_queue
;
816 blk_cleanup_queue(ctrl
->ctrl
.admin_q
);
819 nvme_rdma_free_tagset(&ctrl
->ctrl
, ctrl
->ctrl
.admin_tagset
);
821 nvme_rdma_free_queue(&ctrl
->queues
[0]);
825 static void nvme_rdma_destroy_io_queues(struct nvme_rdma_ctrl
*ctrl
,
828 nvme_rdma_stop_io_queues(ctrl
);
830 blk_cleanup_queue(ctrl
->ctrl
.connect_q
);
831 nvme_rdma_free_tagset(&ctrl
->ctrl
, ctrl
->ctrl
.tagset
);
833 nvme_rdma_free_io_queues(ctrl
);
836 static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl
*ctrl
, bool new)
840 ret
= nvme_rdma_alloc_io_queues(ctrl
);
845 ctrl
->ctrl
.tagset
= nvme_rdma_alloc_tagset(&ctrl
->ctrl
, false);
846 if (IS_ERR(ctrl
->ctrl
.tagset
)) {
847 ret
= PTR_ERR(ctrl
->ctrl
.tagset
);
848 goto out_free_io_queues
;
851 ctrl
->ctrl
.connect_q
= blk_mq_init_queue(&ctrl
->tag_set
);
852 if (IS_ERR(ctrl
->ctrl
.connect_q
)) {
853 ret
= PTR_ERR(ctrl
->ctrl
.connect_q
);
854 goto out_free_tag_set
;
857 ret
= nvme_reinit_tagset(&ctrl
->ctrl
, ctrl
->ctrl
.tagset
);
859 goto out_free_io_queues
;
861 blk_mq_update_nr_hw_queues(&ctrl
->tag_set
,
862 ctrl
->ctrl
.queue_count
- 1);
865 ret
= nvme_rdma_start_io_queues(ctrl
);
867 goto out_cleanup_connect_q
;
871 out_cleanup_connect_q
:
873 blk_cleanup_queue(ctrl
->ctrl
.connect_q
);
876 nvme_rdma_free_tagset(&ctrl
->ctrl
, ctrl
->ctrl
.tagset
);
878 nvme_rdma_free_io_queues(ctrl
);
882 static void nvme_rdma_free_ctrl(struct nvme_ctrl
*nctrl
)
884 struct nvme_rdma_ctrl
*ctrl
= to_rdma_ctrl(nctrl
);
886 if (list_empty(&ctrl
->list
))
889 mutex_lock(&nvme_rdma_ctrl_mutex
);
890 list_del(&ctrl
->list
);
891 mutex_unlock(&nvme_rdma_ctrl_mutex
);
894 nvmf_free_options(nctrl
->opts
);
899 static void nvme_rdma_reconnect_or_remove(struct nvme_rdma_ctrl
*ctrl
)
901 /* If we are resetting/deleting then do nothing */
902 if (ctrl
->ctrl
.state
!= NVME_CTRL_RECONNECTING
) {
903 WARN_ON_ONCE(ctrl
->ctrl
.state
== NVME_CTRL_NEW
||
904 ctrl
->ctrl
.state
== NVME_CTRL_LIVE
);
908 if (nvmf_should_reconnect(&ctrl
->ctrl
)) {
909 dev_info(ctrl
->ctrl
.device
, "Reconnecting in %d seconds...\n",
910 ctrl
->ctrl
.opts
->reconnect_delay
);
911 queue_delayed_work(nvme_wq
, &ctrl
->reconnect_work
,
912 ctrl
->ctrl
.opts
->reconnect_delay
* HZ
);
914 dev_info(ctrl
->ctrl
.device
, "Removing controller...\n");
915 nvme_delete_ctrl(&ctrl
->ctrl
);
919 static void nvme_rdma_reconnect_ctrl_work(struct work_struct
*work
)
921 struct nvme_rdma_ctrl
*ctrl
= container_of(to_delayed_work(work
),
922 struct nvme_rdma_ctrl
, reconnect_work
);
926 ++ctrl
->ctrl
.nr_reconnects
;
928 ret
= nvme_rdma_configure_admin_queue(ctrl
, false);
932 if (ctrl
->ctrl
.queue_count
> 1) {
933 ret
= nvme_rdma_configure_io_queues(ctrl
, false);
938 changed
= nvme_change_ctrl_state(&ctrl
->ctrl
, NVME_CTRL_LIVE
);
940 /* state change failure is ok if we're in DELETING state */
941 WARN_ON_ONCE(ctrl
->ctrl
.state
!= NVME_CTRL_DELETING
);
945 nvme_start_ctrl(&ctrl
->ctrl
);
947 dev_info(ctrl
->ctrl
.device
, "Successfully reconnected (%d attempts)\n",
948 ctrl
->ctrl
.nr_reconnects
);
950 ctrl
->ctrl
.nr_reconnects
= 0;
955 nvme_rdma_destroy_admin_queue(ctrl
, false);
957 dev_info(ctrl
->ctrl
.device
, "Failed reconnect attempt %d\n",
958 ctrl
->ctrl
.nr_reconnects
);
959 nvme_rdma_reconnect_or_remove(ctrl
);
962 static void nvme_rdma_error_recovery_work(struct work_struct
*work
)
964 struct nvme_rdma_ctrl
*ctrl
= container_of(work
,
965 struct nvme_rdma_ctrl
, err_work
);
967 nvme_stop_keep_alive(&ctrl
->ctrl
);
969 if (ctrl
->ctrl
.queue_count
> 1) {
970 nvme_stop_queues(&ctrl
->ctrl
);
971 blk_mq_tagset_busy_iter(&ctrl
->tag_set
,
972 nvme_cancel_request
, &ctrl
->ctrl
);
973 nvme_rdma_destroy_io_queues(ctrl
, false);
976 blk_mq_quiesce_queue(ctrl
->ctrl
.admin_q
);
977 blk_mq_tagset_busy_iter(&ctrl
->admin_tag_set
,
978 nvme_cancel_request
, &ctrl
->ctrl
);
979 nvme_rdma_destroy_admin_queue(ctrl
, false);
982 * queues are not a live anymore, so restart the queues to fail fast
985 blk_mq_unquiesce_queue(ctrl
->ctrl
.admin_q
);
986 nvme_start_queues(&ctrl
->ctrl
);
988 nvme_rdma_reconnect_or_remove(ctrl
);
991 static void nvme_rdma_error_recovery(struct nvme_rdma_ctrl
*ctrl
)
993 if (!nvme_change_ctrl_state(&ctrl
->ctrl
, NVME_CTRL_RECONNECTING
))
996 queue_work(nvme_wq
, &ctrl
->err_work
);
999 static void nvme_rdma_wr_error(struct ib_cq
*cq
, struct ib_wc
*wc
,
1002 struct nvme_rdma_queue
*queue
= cq
->cq_context
;
1003 struct nvme_rdma_ctrl
*ctrl
= queue
->ctrl
;
1005 if (ctrl
->ctrl
.state
== NVME_CTRL_LIVE
)
1006 dev_info(ctrl
->ctrl
.device
,
1007 "%s for CQE 0x%p failed with status %s (%d)\n",
1009 ib_wc_status_msg(wc
->status
), wc
->status
);
1010 nvme_rdma_error_recovery(ctrl
);
1013 static void nvme_rdma_memreg_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
1015 if (unlikely(wc
->status
!= IB_WC_SUCCESS
))
1016 nvme_rdma_wr_error(cq
, wc
, "MEMREG");
1019 static void nvme_rdma_inv_rkey_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
1021 if (unlikely(wc
->status
!= IB_WC_SUCCESS
))
1022 nvme_rdma_wr_error(cq
, wc
, "LOCAL_INV");
1025 static int nvme_rdma_inv_rkey(struct nvme_rdma_queue
*queue
,
1026 struct nvme_rdma_request
*req
)
1028 struct ib_send_wr
*bad_wr
;
1029 struct ib_send_wr wr
= {
1030 .opcode
= IB_WR_LOCAL_INV
,
1034 .ex
.invalidate_rkey
= req
->mr
->rkey
,
1037 req
->reg_cqe
.done
= nvme_rdma_inv_rkey_done
;
1038 wr
.wr_cqe
= &req
->reg_cqe
;
1040 return ib_post_send(queue
->qp
, &wr
, &bad_wr
);
1043 static void nvme_rdma_unmap_data(struct nvme_rdma_queue
*queue
,
1046 struct nvme_rdma_request
*req
= blk_mq_rq_to_pdu(rq
);
1047 struct nvme_rdma_ctrl
*ctrl
= queue
->ctrl
;
1048 struct nvme_rdma_device
*dev
= queue
->device
;
1049 struct ib_device
*ibdev
= dev
->dev
;
1052 if (!blk_rq_bytes(rq
))
1055 if (req
->mr
->need_inval
&& test_bit(NVME_RDMA_Q_LIVE
, &req
->queue
->flags
)) {
1056 res
= nvme_rdma_inv_rkey(queue
, req
);
1057 if (unlikely(res
< 0)) {
1058 dev_err(ctrl
->ctrl
.device
,
1059 "Queueing INV WR for rkey %#x failed (%d)\n",
1060 req
->mr
->rkey
, res
);
1061 nvme_rdma_error_recovery(queue
->ctrl
);
1065 ib_dma_unmap_sg(ibdev
, req
->sg_table
.sgl
,
1066 req
->nents
, rq_data_dir(rq
) ==
1067 WRITE
? DMA_TO_DEVICE
: DMA_FROM_DEVICE
);
1069 nvme_cleanup_cmd(rq
);
1070 sg_free_table_chained(&req
->sg_table
, true);
1073 static int nvme_rdma_set_sg_null(struct nvme_command
*c
)
1075 struct nvme_keyed_sgl_desc
*sg
= &c
->common
.dptr
.ksgl
;
1078 put_unaligned_le24(0, sg
->length
);
1079 put_unaligned_le32(0, sg
->key
);
1080 sg
->type
= NVME_KEY_SGL_FMT_DATA_DESC
<< 4;
1084 static int nvme_rdma_map_sg_inline(struct nvme_rdma_queue
*queue
,
1085 struct nvme_rdma_request
*req
, struct nvme_command
*c
)
1087 struct nvme_sgl_desc
*sg
= &c
->common
.dptr
.sgl
;
1089 req
->sge
[1].addr
= sg_dma_address(req
->sg_table
.sgl
);
1090 req
->sge
[1].length
= sg_dma_len(req
->sg_table
.sgl
);
1091 req
->sge
[1].lkey
= queue
->device
->pd
->local_dma_lkey
;
1093 sg
->addr
= cpu_to_le64(queue
->ctrl
->ctrl
.icdoff
);
1094 sg
->length
= cpu_to_le32(sg_dma_len(req
->sg_table
.sgl
));
1095 sg
->type
= (NVME_SGL_FMT_DATA_DESC
<< 4) | NVME_SGL_FMT_OFFSET
;
1097 req
->inline_data
= true;
1102 static int nvme_rdma_map_sg_single(struct nvme_rdma_queue
*queue
,
1103 struct nvme_rdma_request
*req
, struct nvme_command
*c
)
1105 struct nvme_keyed_sgl_desc
*sg
= &c
->common
.dptr
.ksgl
;
1107 sg
->addr
= cpu_to_le64(sg_dma_address(req
->sg_table
.sgl
));
1108 put_unaligned_le24(sg_dma_len(req
->sg_table
.sgl
), sg
->length
);
1109 put_unaligned_le32(queue
->device
->pd
->unsafe_global_rkey
, sg
->key
);
1110 sg
->type
= NVME_KEY_SGL_FMT_DATA_DESC
<< 4;
1114 static int nvme_rdma_map_sg_fr(struct nvme_rdma_queue
*queue
,
1115 struct nvme_rdma_request
*req
, struct nvme_command
*c
,
1118 struct nvme_keyed_sgl_desc
*sg
= &c
->common
.dptr
.ksgl
;
1122 * Align the MR to a 4K page size to match the ctrl page size and
1123 * the block virtual boundary.
1125 nr
= ib_map_mr_sg(req
->mr
, req
->sg_table
.sgl
, count
, NULL
, SZ_4K
);
1126 if (unlikely(nr
< count
)) {
1132 ib_update_fast_reg_key(req
->mr
, ib_inc_rkey(req
->mr
->rkey
));
1134 req
->reg_cqe
.done
= nvme_rdma_memreg_done
;
1135 memset(&req
->reg_wr
, 0, sizeof(req
->reg_wr
));
1136 req
->reg_wr
.wr
.opcode
= IB_WR_REG_MR
;
1137 req
->reg_wr
.wr
.wr_cqe
= &req
->reg_cqe
;
1138 req
->reg_wr
.wr
.num_sge
= 0;
1139 req
->reg_wr
.mr
= req
->mr
;
1140 req
->reg_wr
.key
= req
->mr
->rkey
;
1141 req
->reg_wr
.access
= IB_ACCESS_LOCAL_WRITE
|
1142 IB_ACCESS_REMOTE_READ
|
1143 IB_ACCESS_REMOTE_WRITE
;
1145 req
->mr
->need_inval
= true;
1147 sg
->addr
= cpu_to_le64(req
->mr
->iova
);
1148 put_unaligned_le24(req
->mr
->length
, sg
->length
);
1149 put_unaligned_le32(req
->mr
->rkey
, sg
->key
);
1150 sg
->type
= (NVME_KEY_SGL_FMT_DATA_DESC
<< 4) |
1151 NVME_SGL_FMT_INVALIDATE
;
1156 static int nvme_rdma_map_data(struct nvme_rdma_queue
*queue
,
1157 struct request
*rq
, struct nvme_command
*c
)
1159 struct nvme_rdma_request
*req
= blk_mq_rq_to_pdu(rq
);
1160 struct nvme_rdma_device
*dev
= queue
->device
;
1161 struct ib_device
*ibdev
= dev
->dev
;
1165 req
->inline_data
= false;
1166 req
->mr
->need_inval
= false;
1168 c
->common
.flags
|= NVME_CMD_SGL_METABUF
;
1170 if (!blk_rq_bytes(rq
))
1171 return nvme_rdma_set_sg_null(c
);
1173 req
->sg_table
.sgl
= req
->first_sgl
;
1174 ret
= sg_alloc_table_chained(&req
->sg_table
,
1175 blk_rq_nr_phys_segments(rq
), req
->sg_table
.sgl
);
1179 req
->nents
= blk_rq_map_sg(rq
->q
, rq
, req
->sg_table
.sgl
);
1181 count
= ib_dma_map_sg(ibdev
, req
->sg_table
.sgl
, req
->nents
,
1182 rq_data_dir(rq
) == WRITE
? DMA_TO_DEVICE
: DMA_FROM_DEVICE
);
1183 if (unlikely(count
<= 0)) {
1184 sg_free_table_chained(&req
->sg_table
, true);
1189 if (rq_data_dir(rq
) == WRITE
&& nvme_rdma_queue_idx(queue
) &&
1190 blk_rq_payload_bytes(rq
) <=
1191 nvme_rdma_inline_data_size(queue
))
1192 return nvme_rdma_map_sg_inline(queue
, req
, c
);
1194 if (dev
->pd
->flags
& IB_PD_UNSAFE_GLOBAL_RKEY
)
1195 return nvme_rdma_map_sg_single(queue
, req
, c
);
1198 return nvme_rdma_map_sg_fr(queue
, req
, c
, count
);
1201 static void nvme_rdma_send_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
1203 if (unlikely(wc
->status
!= IB_WC_SUCCESS
))
1204 nvme_rdma_wr_error(cq
, wc
, "SEND");
1208 * We want to signal completion at least every queue depth/2. This returns the
1209 * largest power of two that is not above half of (queue size + 1) to optimize
1210 * (avoid divisions).
1212 static inline bool nvme_rdma_queue_sig_limit(struct nvme_rdma_queue
*queue
)
1214 int limit
= 1 << ilog2((queue
->queue_size
+ 1) / 2);
1216 return (atomic_inc_return(&queue
->sig_count
) & (limit
- 1)) == 0;
1219 static int nvme_rdma_post_send(struct nvme_rdma_queue
*queue
,
1220 struct nvme_rdma_qe
*qe
, struct ib_sge
*sge
, u32 num_sge
,
1221 struct ib_send_wr
*first
, bool flush
)
1223 struct ib_send_wr wr
, *bad_wr
;
1226 sge
->addr
= qe
->dma
;
1227 sge
->length
= sizeof(struct nvme_command
),
1228 sge
->lkey
= queue
->device
->pd
->local_dma_lkey
;
1230 qe
->cqe
.done
= nvme_rdma_send_done
;
1233 wr
.wr_cqe
= &qe
->cqe
;
1235 wr
.num_sge
= num_sge
;
1236 wr
.opcode
= IB_WR_SEND
;
1240 * Unsignalled send completions are another giant desaster in the
1241 * IB Verbs spec: If we don't regularly post signalled sends
1242 * the send queue will fill up and only a QP reset will rescue us.
1243 * Would have been way to obvious to handle this in hardware or
1244 * at least the RDMA stack..
1246 * Always signal the flushes. The magic request used for the flush
1247 * sequencer is not allocated in our driver's tagset and it's
1248 * triggered to be freed by blk_cleanup_queue(). So we need to
1249 * always mark it as signaled to ensure that the "wr_cqe", which is
1250 * embedded in request's payload, is not freed when __ib_process_cq()
1251 * calls wr_cqe->done().
1253 if (nvme_rdma_queue_sig_limit(queue
) || flush
)
1254 wr
.send_flags
|= IB_SEND_SIGNALED
;
1261 ret
= ib_post_send(queue
->qp
, first
, &bad_wr
);
1262 if (unlikely(ret
)) {
1263 dev_err(queue
->ctrl
->ctrl
.device
,
1264 "%s failed with error code %d\n", __func__
, ret
);
1269 static int nvme_rdma_post_recv(struct nvme_rdma_queue
*queue
,
1270 struct nvme_rdma_qe
*qe
)
1272 struct ib_recv_wr wr
, *bad_wr
;
1276 list
.addr
= qe
->dma
;
1277 list
.length
= sizeof(struct nvme_completion
);
1278 list
.lkey
= queue
->device
->pd
->local_dma_lkey
;
1280 qe
->cqe
.done
= nvme_rdma_recv_done
;
1283 wr
.wr_cqe
= &qe
->cqe
;
1287 ret
= ib_post_recv(queue
->qp
, &wr
, &bad_wr
);
1288 if (unlikely(ret
)) {
1289 dev_err(queue
->ctrl
->ctrl
.device
,
1290 "%s failed with error code %d\n", __func__
, ret
);
1295 static struct blk_mq_tags
*nvme_rdma_tagset(struct nvme_rdma_queue
*queue
)
1297 u32 queue_idx
= nvme_rdma_queue_idx(queue
);
1300 return queue
->ctrl
->admin_tag_set
.tags
[queue_idx
];
1301 return queue
->ctrl
->tag_set
.tags
[queue_idx
- 1];
1304 static void nvme_rdma_submit_async_event(struct nvme_ctrl
*arg
)
1306 struct nvme_rdma_ctrl
*ctrl
= to_rdma_ctrl(arg
);
1307 struct nvme_rdma_queue
*queue
= &ctrl
->queues
[0];
1308 struct ib_device
*dev
= queue
->device
->dev
;
1309 struct nvme_rdma_qe
*sqe
= &ctrl
->async_event_sqe
;
1310 struct nvme_command
*cmd
= sqe
->data
;
1314 ib_dma_sync_single_for_cpu(dev
, sqe
->dma
, sizeof(*cmd
), DMA_TO_DEVICE
);
1316 memset(cmd
, 0, sizeof(*cmd
));
1317 cmd
->common
.opcode
= nvme_admin_async_event
;
1318 cmd
->common
.command_id
= NVME_AQ_BLK_MQ_DEPTH
;
1319 cmd
->common
.flags
|= NVME_CMD_SGL_METABUF
;
1320 nvme_rdma_set_sg_null(cmd
);
1322 ib_dma_sync_single_for_device(dev
, sqe
->dma
, sizeof(*cmd
),
1325 ret
= nvme_rdma_post_send(queue
, sqe
, &sge
, 1, NULL
, false);
1329 static int nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue
*queue
,
1330 struct nvme_completion
*cqe
, struct ib_wc
*wc
, int tag
)
1333 struct nvme_rdma_request
*req
;
1336 rq
= blk_mq_tag_to_rq(nvme_rdma_tagset(queue
), cqe
->command_id
);
1338 dev_err(queue
->ctrl
->ctrl
.device
,
1339 "tag 0x%x on QP %#x not found\n",
1340 cqe
->command_id
, queue
->qp
->qp_num
);
1341 nvme_rdma_error_recovery(queue
->ctrl
);
1344 req
= blk_mq_rq_to_pdu(rq
);
1349 if ((wc
->wc_flags
& IB_WC_WITH_INVALIDATE
) &&
1350 wc
->ex
.invalidate_rkey
== req
->mr
->rkey
)
1351 req
->mr
->need_inval
= false;
1353 nvme_end_request(rq
, cqe
->status
, cqe
->result
);
1357 static int __nvme_rdma_recv_done(struct ib_cq
*cq
, struct ib_wc
*wc
, int tag
)
1359 struct nvme_rdma_qe
*qe
=
1360 container_of(wc
->wr_cqe
, struct nvme_rdma_qe
, cqe
);
1361 struct nvme_rdma_queue
*queue
= cq
->cq_context
;
1362 struct ib_device
*ibdev
= queue
->device
->dev
;
1363 struct nvme_completion
*cqe
= qe
->data
;
1364 const size_t len
= sizeof(struct nvme_completion
);
1367 if (unlikely(wc
->status
!= IB_WC_SUCCESS
)) {
1368 nvme_rdma_wr_error(cq
, wc
, "RECV");
1372 ib_dma_sync_single_for_cpu(ibdev
, qe
->dma
, len
, DMA_FROM_DEVICE
);
1374 * AEN requests are special as they don't time out and can
1375 * survive any kind of queue freeze and often don't respond to
1376 * aborts. We don't even bother to allocate a struct request
1377 * for them but rather special case them here.
1379 if (unlikely(nvme_rdma_queue_idx(queue
) == 0 &&
1380 cqe
->command_id
>= NVME_AQ_BLK_MQ_DEPTH
))
1381 nvme_complete_async_event(&queue
->ctrl
->ctrl
, cqe
->status
,
1384 ret
= nvme_rdma_process_nvme_rsp(queue
, cqe
, wc
, tag
);
1385 ib_dma_sync_single_for_device(ibdev
, qe
->dma
, len
, DMA_FROM_DEVICE
);
1387 nvme_rdma_post_recv(queue
, qe
);
1391 static void nvme_rdma_recv_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
1393 __nvme_rdma_recv_done(cq
, wc
, -1);
1396 static int nvme_rdma_conn_established(struct nvme_rdma_queue
*queue
)
1400 for (i
= 0; i
< queue
->queue_size
; i
++) {
1401 ret
= nvme_rdma_post_recv(queue
, &queue
->rsp_ring
[i
]);
1403 goto out_destroy_queue_ib
;
1408 out_destroy_queue_ib
:
1409 nvme_rdma_destroy_queue_ib(queue
);
1413 static int nvme_rdma_conn_rejected(struct nvme_rdma_queue
*queue
,
1414 struct rdma_cm_event
*ev
)
1416 struct rdma_cm_id
*cm_id
= queue
->cm_id
;
1417 int status
= ev
->status
;
1418 const char *rej_msg
;
1419 const struct nvme_rdma_cm_rej
*rej_data
;
1422 rej_msg
= rdma_reject_msg(cm_id
, status
);
1423 rej_data
= rdma_consumer_reject_data(cm_id
, ev
, &rej_data_len
);
1425 if (rej_data
&& rej_data_len
>= sizeof(u16
)) {
1426 u16 sts
= le16_to_cpu(rej_data
->sts
);
1428 dev_err(queue
->ctrl
->ctrl
.device
,
1429 "Connect rejected: status %d (%s) nvme status %d (%s).\n",
1430 status
, rej_msg
, sts
, nvme_rdma_cm_msg(sts
));
1432 dev_err(queue
->ctrl
->ctrl
.device
,
1433 "Connect rejected: status %d (%s).\n", status
, rej_msg
);
1439 static int nvme_rdma_addr_resolved(struct nvme_rdma_queue
*queue
)
1443 ret
= nvme_rdma_create_queue_ib(queue
);
1447 ret
= rdma_resolve_route(queue
->cm_id
, NVME_RDMA_CONNECT_TIMEOUT_MS
);
1449 dev_err(queue
->ctrl
->ctrl
.device
,
1450 "rdma_resolve_route failed (%d).\n",
1452 goto out_destroy_queue
;
1458 nvme_rdma_destroy_queue_ib(queue
);
1462 static int nvme_rdma_route_resolved(struct nvme_rdma_queue
*queue
)
1464 struct nvme_rdma_ctrl
*ctrl
= queue
->ctrl
;
1465 struct rdma_conn_param param
= { };
1466 struct nvme_rdma_cm_req priv
= { };
1469 param
.qp_num
= queue
->qp
->qp_num
;
1470 param
.flow_control
= 1;
1472 param
.responder_resources
= queue
->device
->dev
->attrs
.max_qp_rd_atom
;
1473 /* maximum retry count */
1474 param
.retry_count
= 7;
1475 param
.rnr_retry_count
= 7;
1476 param
.private_data
= &priv
;
1477 param
.private_data_len
= sizeof(priv
);
1479 priv
.recfmt
= cpu_to_le16(NVME_RDMA_CM_FMT_1_0
);
1480 priv
.qid
= cpu_to_le16(nvme_rdma_queue_idx(queue
));
1482 * set the admin queue depth to the minimum size
1483 * specified by the Fabrics standard.
1485 if (priv
.qid
== 0) {
1486 priv
.hrqsize
= cpu_to_le16(NVME_AQ_DEPTH
);
1487 priv
.hsqsize
= cpu_to_le16(NVME_AQ_DEPTH
- 1);
1490 * current interpretation of the fabrics spec
1491 * is at minimum you make hrqsize sqsize+1, or a
1492 * 1's based representation of sqsize.
1494 priv
.hrqsize
= cpu_to_le16(queue
->queue_size
);
1495 priv
.hsqsize
= cpu_to_le16(queue
->ctrl
->ctrl
.sqsize
);
1498 ret
= rdma_connect(queue
->cm_id
, ¶m
);
1500 dev_err(ctrl
->ctrl
.device
,
1501 "rdma_connect failed (%d).\n", ret
);
1502 goto out_destroy_queue_ib
;
1507 out_destroy_queue_ib
:
1508 nvme_rdma_destroy_queue_ib(queue
);
1512 static int nvme_rdma_cm_handler(struct rdma_cm_id
*cm_id
,
1513 struct rdma_cm_event
*ev
)
1515 struct nvme_rdma_queue
*queue
= cm_id
->context
;
1518 dev_dbg(queue
->ctrl
->ctrl
.device
, "%s (%d): status %d id %p\n",
1519 rdma_event_msg(ev
->event
), ev
->event
,
1522 switch (ev
->event
) {
1523 case RDMA_CM_EVENT_ADDR_RESOLVED
:
1524 cm_error
= nvme_rdma_addr_resolved(queue
);
1526 case RDMA_CM_EVENT_ROUTE_RESOLVED
:
1527 cm_error
= nvme_rdma_route_resolved(queue
);
1529 case RDMA_CM_EVENT_ESTABLISHED
:
1530 queue
->cm_error
= nvme_rdma_conn_established(queue
);
1531 /* complete cm_done regardless of success/failure */
1532 complete(&queue
->cm_done
);
1534 case RDMA_CM_EVENT_REJECTED
:
1535 nvme_rdma_destroy_queue_ib(queue
);
1536 cm_error
= nvme_rdma_conn_rejected(queue
, ev
);
1538 case RDMA_CM_EVENT_ROUTE_ERROR
:
1539 case RDMA_CM_EVENT_CONNECT_ERROR
:
1540 case RDMA_CM_EVENT_UNREACHABLE
:
1541 nvme_rdma_destroy_queue_ib(queue
);
1542 case RDMA_CM_EVENT_ADDR_ERROR
:
1543 dev_dbg(queue
->ctrl
->ctrl
.device
,
1544 "CM error event %d\n", ev
->event
);
1545 cm_error
= -ECONNRESET
;
1547 case RDMA_CM_EVENT_DISCONNECTED
:
1548 case RDMA_CM_EVENT_ADDR_CHANGE
:
1549 case RDMA_CM_EVENT_TIMEWAIT_EXIT
:
1550 dev_dbg(queue
->ctrl
->ctrl
.device
,
1551 "disconnect received - connection closed\n");
1552 nvme_rdma_error_recovery(queue
->ctrl
);
1554 case RDMA_CM_EVENT_DEVICE_REMOVAL
:
1555 /* device removal is handled via the ib_client API */
1558 dev_err(queue
->ctrl
->ctrl
.device
,
1559 "Unexpected RDMA CM event (%d)\n", ev
->event
);
1560 nvme_rdma_error_recovery(queue
->ctrl
);
1565 queue
->cm_error
= cm_error
;
1566 complete(&queue
->cm_done
);
1572 static enum blk_eh_timer_return
1573 nvme_rdma_timeout(struct request
*rq
, bool reserved
)
1575 struct nvme_rdma_request
*req
= blk_mq_rq_to_pdu(rq
);
1577 dev_warn(req
->queue
->ctrl
->ctrl
.device
,
1578 "I/O %d QID %d timeout, reset controller\n",
1579 rq
->tag
, nvme_rdma_queue_idx(req
->queue
));
1581 /* queue error recovery */
1582 nvme_rdma_error_recovery(req
->queue
->ctrl
);
1584 /* fail with DNR on cmd timeout */
1585 nvme_req(rq
)->status
= NVME_SC_ABORT_REQ
| NVME_SC_DNR
;
1587 return BLK_EH_HANDLED
;
1591 * We cannot accept any other command until the Connect command has completed.
1593 static inline blk_status_t
1594 nvme_rdma_queue_is_ready(struct nvme_rdma_queue
*queue
, struct request
*rq
)
1596 if (unlikely(!test_bit(NVME_RDMA_Q_LIVE
, &queue
->flags
))) {
1597 struct nvme_command
*cmd
= nvme_req(rq
)->cmd
;
1599 if (!blk_rq_is_passthrough(rq
) ||
1600 cmd
->common
.opcode
!= nvme_fabrics_command
||
1601 cmd
->fabrics
.fctype
!= nvme_fabrics_type_connect
) {
1603 * reconnecting state means transport disruption, which
1604 * can take a long time and even might fail permanently,
1605 * fail fast to give upper layers a chance to failover.
1606 * deleting state means that the ctrl will never accept
1607 * commands again, fail it permanently.
1609 if (queue
->ctrl
->ctrl
.state
== NVME_CTRL_RECONNECTING
||
1610 queue
->ctrl
->ctrl
.state
== NVME_CTRL_DELETING
) {
1611 nvme_req(rq
)->status
= NVME_SC_ABORT_REQ
;
1612 return BLK_STS_IOERR
;
1614 return BLK_STS_RESOURCE
; /* try again later */
1621 static blk_status_t
nvme_rdma_queue_rq(struct blk_mq_hw_ctx
*hctx
,
1622 const struct blk_mq_queue_data
*bd
)
1624 struct nvme_ns
*ns
= hctx
->queue
->queuedata
;
1625 struct nvme_rdma_queue
*queue
= hctx
->driver_data
;
1626 struct request
*rq
= bd
->rq
;
1627 struct nvme_rdma_request
*req
= blk_mq_rq_to_pdu(rq
);
1628 struct nvme_rdma_qe
*sqe
= &req
->sqe
;
1629 struct nvme_command
*c
= sqe
->data
;
1631 struct ib_device
*dev
;
1635 WARN_ON_ONCE(rq
->tag
< 0);
1637 ret
= nvme_rdma_queue_is_ready(queue
, rq
);
1641 dev
= queue
->device
->dev
;
1642 ib_dma_sync_single_for_cpu(dev
, sqe
->dma
,
1643 sizeof(struct nvme_command
), DMA_TO_DEVICE
);
1645 ret
= nvme_setup_cmd(ns
, rq
, c
);
1649 blk_mq_start_request(rq
);
1651 err
= nvme_rdma_map_data(queue
, rq
, c
);
1652 if (unlikely(err
< 0)) {
1653 dev_err(queue
->ctrl
->ctrl
.device
,
1654 "Failed to map data (%d)\n", err
);
1655 nvme_cleanup_cmd(rq
);
1659 ib_dma_sync_single_for_device(dev
, sqe
->dma
,
1660 sizeof(struct nvme_command
), DMA_TO_DEVICE
);
1662 if (req_op(rq
) == REQ_OP_FLUSH
)
1664 err
= nvme_rdma_post_send(queue
, sqe
, req
->sge
, req
->num_sge
,
1665 req
->mr
->need_inval
? &req
->reg_wr
.wr
: NULL
, flush
);
1666 if (unlikely(err
)) {
1667 nvme_rdma_unmap_data(queue
, rq
);
1673 if (err
== -ENOMEM
|| err
== -EAGAIN
)
1674 return BLK_STS_RESOURCE
;
1675 return BLK_STS_IOERR
;
1678 static int nvme_rdma_poll(struct blk_mq_hw_ctx
*hctx
, unsigned int tag
)
1680 struct nvme_rdma_queue
*queue
= hctx
->driver_data
;
1681 struct ib_cq
*cq
= queue
->ib_cq
;
1685 while (ib_poll_cq(cq
, 1, &wc
) > 0) {
1686 struct ib_cqe
*cqe
= wc
.wr_cqe
;
1689 if (cqe
->done
== nvme_rdma_recv_done
)
1690 found
|= __nvme_rdma_recv_done(cq
, &wc
, tag
);
1699 static void nvme_rdma_complete_rq(struct request
*rq
)
1701 struct nvme_rdma_request
*req
= blk_mq_rq_to_pdu(rq
);
1703 nvme_rdma_unmap_data(req
->queue
, rq
);
1704 nvme_complete_rq(rq
);
1707 static int nvme_rdma_map_queues(struct blk_mq_tag_set
*set
)
1709 struct nvme_rdma_ctrl
*ctrl
= set
->driver_data
;
1711 return blk_mq_rdma_map_queues(set
, ctrl
->device
->dev
, 0);
1714 static const struct blk_mq_ops nvme_rdma_mq_ops
= {
1715 .queue_rq
= nvme_rdma_queue_rq
,
1716 .complete
= nvme_rdma_complete_rq
,
1717 .init_request
= nvme_rdma_init_request
,
1718 .exit_request
= nvme_rdma_exit_request
,
1719 .init_hctx
= nvme_rdma_init_hctx
,
1720 .poll
= nvme_rdma_poll
,
1721 .timeout
= nvme_rdma_timeout
,
1722 .map_queues
= nvme_rdma_map_queues
,
1725 static const struct blk_mq_ops nvme_rdma_admin_mq_ops
= {
1726 .queue_rq
= nvme_rdma_queue_rq
,
1727 .complete
= nvme_rdma_complete_rq
,
1728 .init_request
= nvme_rdma_init_request
,
1729 .exit_request
= nvme_rdma_exit_request
,
1730 .init_hctx
= nvme_rdma_init_admin_hctx
,
1731 .timeout
= nvme_rdma_timeout
,
1734 static void nvme_rdma_shutdown_ctrl(struct nvme_rdma_ctrl
*ctrl
, bool shutdown
)
1736 cancel_work_sync(&ctrl
->err_work
);
1737 cancel_delayed_work_sync(&ctrl
->reconnect_work
);
1739 if (ctrl
->ctrl
.queue_count
> 1) {
1740 nvme_stop_queues(&ctrl
->ctrl
);
1741 blk_mq_tagset_busy_iter(&ctrl
->tag_set
,
1742 nvme_cancel_request
, &ctrl
->ctrl
);
1743 nvme_rdma_destroy_io_queues(ctrl
, shutdown
);
1747 nvme_shutdown_ctrl(&ctrl
->ctrl
);
1749 nvme_disable_ctrl(&ctrl
->ctrl
, ctrl
->ctrl
.cap
);
1751 blk_mq_quiesce_queue(ctrl
->ctrl
.admin_q
);
1752 blk_mq_tagset_busy_iter(&ctrl
->admin_tag_set
,
1753 nvme_cancel_request
, &ctrl
->ctrl
);
1754 blk_mq_unquiesce_queue(ctrl
->ctrl
.admin_q
);
1755 nvme_rdma_destroy_admin_queue(ctrl
, shutdown
);
1758 static void nvme_rdma_delete_ctrl(struct nvme_ctrl
*ctrl
)
1760 nvme_rdma_shutdown_ctrl(to_rdma_ctrl(ctrl
), true);
1763 static void nvme_rdma_reset_ctrl_work(struct work_struct
*work
)
1765 struct nvme_rdma_ctrl
*ctrl
=
1766 container_of(work
, struct nvme_rdma_ctrl
, ctrl
.reset_work
);
1770 nvme_stop_ctrl(&ctrl
->ctrl
);
1771 nvme_rdma_shutdown_ctrl(ctrl
, false);
1773 ret
= nvme_rdma_configure_admin_queue(ctrl
, false);
1777 if (ctrl
->ctrl
.queue_count
> 1) {
1778 ret
= nvme_rdma_configure_io_queues(ctrl
, false);
1783 changed
= nvme_change_ctrl_state(&ctrl
->ctrl
, NVME_CTRL_LIVE
);
1785 /* state change failure is ok if we're in DELETING state */
1786 WARN_ON_ONCE(ctrl
->ctrl
.state
!= NVME_CTRL_DELETING
);
1790 nvme_start_ctrl(&ctrl
->ctrl
);
1795 dev_warn(ctrl
->ctrl
.device
, "Removing after reset failure\n");
1796 nvme_remove_namespaces(&ctrl
->ctrl
);
1797 nvme_rdma_shutdown_ctrl(ctrl
, true);
1798 nvme_uninit_ctrl(&ctrl
->ctrl
);
1799 nvme_put_ctrl(&ctrl
->ctrl
);
1802 static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops
= {
1804 .module
= THIS_MODULE
,
1805 .flags
= NVME_F_FABRICS
,
1806 .reg_read32
= nvmf_reg_read32
,
1807 .reg_read64
= nvmf_reg_read64
,
1808 .reg_write32
= nvmf_reg_write32
,
1809 .free_ctrl
= nvme_rdma_free_ctrl
,
1810 .submit_async_event
= nvme_rdma_submit_async_event
,
1811 .delete_ctrl
= nvme_rdma_delete_ctrl
,
1812 .get_address
= nvmf_get_address
,
1813 .reinit_request
= nvme_rdma_reinit_request
,
1817 __nvme_rdma_options_match(struct nvme_rdma_ctrl
*ctrl
,
1818 struct nvmf_ctrl_options
*opts
)
1820 char *stdport
= __stringify(NVME_RDMA_IP_PORT
);
1823 if (!nvmf_ctlr_matches_baseopts(&ctrl
->ctrl
, opts
) ||
1824 strcmp(opts
->traddr
, ctrl
->ctrl
.opts
->traddr
))
1827 if (opts
->mask
& NVMF_OPT_TRSVCID
&&
1828 ctrl
->ctrl
.opts
->mask
& NVMF_OPT_TRSVCID
) {
1829 if (strcmp(opts
->trsvcid
, ctrl
->ctrl
.opts
->trsvcid
))
1831 } else if (opts
->mask
& NVMF_OPT_TRSVCID
) {
1832 if (strcmp(opts
->trsvcid
, stdport
))
1834 } else if (ctrl
->ctrl
.opts
->mask
& NVMF_OPT_TRSVCID
) {
1835 if (strcmp(stdport
, ctrl
->ctrl
.opts
->trsvcid
))
1838 /* else, it's a match as both have stdport. Fall to next checks */
1841 * checking the local address is rough. In most cases, one
1842 * is not specified and the host port is selected by the stack.
1844 * Assume no match if:
1845 * local address is specified and address is not the same
1846 * local address is not specified but remote is, or vice versa
1847 * (admin using specific host_traddr when it matters).
1849 if (opts
->mask
& NVMF_OPT_HOST_TRADDR
&&
1850 ctrl
->ctrl
.opts
->mask
& NVMF_OPT_HOST_TRADDR
) {
1851 if (strcmp(opts
->host_traddr
, ctrl
->ctrl
.opts
->host_traddr
))
1853 } else if (opts
->mask
& NVMF_OPT_HOST_TRADDR
||
1854 ctrl
->ctrl
.opts
->mask
& NVMF_OPT_HOST_TRADDR
)
1857 * if neither controller had an host port specified, assume it's
1858 * a match as everything else matched.
1865 * Fails a connection request if it matches an existing controller
1866 * (association) with the same tuple:
1867 * <Host NQN, Host ID, local address, remote address, remote port, SUBSYS NQN>
1869 * if local address is not specified in the request, it will match an
1870 * existing controller with all the other parameters the same and no
1871 * local port address specified as well.
1873 * The ports don't need to be compared as they are intrinsically
1874 * already matched by the port pointers supplied.
1877 nvme_rdma_existing_controller(struct nvmf_ctrl_options
*opts
)
1879 struct nvme_rdma_ctrl
*ctrl
;
1882 mutex_lock(&nvme_rdma_ctrl_mutex
);
1883 list_for_each_entry(ctrl
, &nvme_rdma_ctrl_list
, list
) {
1884 found
= __nvme_rdma_options_match(ctrl
, opts
);
1888 mutex_unlock(&nvme_rdma_ctrl_mutex
);
1893 static struct nvme_ctrl
*nvme_rdma_create_ctrl(struct device
*dev
,
1894 struct nvmf_ctrl_options
*opts
)
1896 struct nvme_rdma_ctrl
*ctrl
;
1901 ctrl
= kzalloc(sizeof(*ctrl
), GFP_KERNEL
);
1903 return ERR_PTR(-ENOMEM
);
1904 ctrl
->ctrl
.opts
= opts
;
1905 INIT_LIST_HEAD(&ctrl
->list
);
1907 if (opts
->mask
& NVMF_OPT_TRSVCID
)
1908 port
= opts
->trsvcid
;
1910 port
= __stringify(NVME_RDMA_IP_PORT
);
1912 ret
= inet_pton_with_scope(&init_net
, AF_UNSPEC
,
1913 opts
->traddr
, port
, &ctrl
->addr
);
1915 pr_err("malformed address passed: %s:%s\n", opts
->traddr
, port
);
1919 if (opts
->mask
& NVMF_OPT_HOST_TRADDR
) {
1920 ret
= inet_pton_with_scope(&init_net
, AF_UNSPEC
,
1921 opts
->host_traddr
, NULL
, &ctrl
->src_addr
);
1923 pr_err("malformed src address passed: %s\n",
1929 if (!opts
->duplicate_connect
&& nvme_rdma_existing_controller(opts
)) {
1934 ret
= nvme_init_ctrl(&ctrl
->ctrl
, dev
, &nvme_rdma_ctrl_ops
,
1935 0 /* no quirks, we're perfect! */);
1939 INIT_DELAYED_WORK(&ctrl
->reconnect_work
,
1940 nvme_rdma_reconnect_ctrl_work
);
1941 INIT_WORK(&ctrl
->err_work
, nvme_rdma_error_recovery_work
);
1942 INIT_WORK(&ctrl
->ctrl
.reset_work
, nvme_rdma_reset_ctrl_work
);
1944 ctrl
->ctrl
.queue_count
= opts
->nr_io_queues
+ 1; /* +1 for admin queue */
1945 ctrl
->ctrl
.sqsize
= opts
->queue_size
- 1;
1946 ctrl
->ctrl
.kato
= opts
->kato
;
1949 ctrl
->queues
= kcalloc(ctrl
->ctrl
.queue_count
, sizeof(*ctrl
->queues
),
1952 goto out_uninit_ctrl
;
1954 ret
= nvme_rdma_configure_admin_queue(ctrl
, true);
1956 goto out_kfree_queues
;
1958 /* sanity check icdoff */
1959 if (ctrl
->ctrl
.icdoff
) {
1960 dev_err(ctrl
->ctrl
.device
, "icdoff is not supported!\n");
1962 goto out_remove_admin_queue
;
1965 /* sanity check keyed sgls */
1966 if (!(ctrl
->ctrl
.sgls
& (1 << 20))) {
1967 dev_err(ctrl
->ctrl
.device
, "Mandatory keyed sgls are not support\n");
1969 goto out_remove_admin_queue
;
1972 if (opts
->queue_size
> ctrl
->ctrl
.maxcmd
) {
1973 /* warn if maxcmd is lower than queue_size */
1974 dev_warn(ctrl
->ctrl
.device
,
1975 "queue_size %zu > ctrl maxcmd %u, clamping down\n",
1976 opts
->queue_size
, ctrl
->ctrl
.maxcmd
);
1977 opts
->queue_size
= ctrl
->ctrl
.maxcmd
;
1980 if (opts
->queue_size
> ctrl
->ctrl
.sqsize
+ 1) {
1981 /* warn if sqsize is lower than queue_size */
1982 dev_warn(ctrl
->ctrl
.device
,
1983 "queue_size %zu > ctrl sqsize %u, clamping down\n",
1984 opts
->queue_size
, ctrl
->ctrl
.sqsize
+ 1);
1985 opts
->queue_size
= ctrl
->ctrl
.sqsize
+ 1;
1988 if (opts
->nr_io_queues
) {
1989 ret
= nvme_rdma_configure_io_queues(ctrl
, true);
1991 goto out_remove_admin_queue
;
1994 changed
= nvme_change_ctrl_state(&ctrl
->ctrl
, NVME_CTRL_LIVE
);
1995 WARN_ON_ONCE(!changed
);
1997 dev_info(ctrl
->ctrl
.device
, "new ctrl: NQN \"%s\", addr %pISpcs\n",
1998 ctrl
->ctrl
.opts
->subsysnqn
, &ctrl
->addr
);
2000 nvme_get_ctrl(&ctrl
->ctrl
);
2002 mutex_lock(&nvme_rdma_ctrl_mutex
);
2003 list_add_tail(&ctrl
->list
, &nvme_rdma_ctrl_list
);
2004 mutex_unlock(&nvme_rdma_ctrl_mutex
);
2006 nvme_start_ctrl(&ctrl
->ctrl
);
2010 out_remove_admin_queue
:
2011 nvme_rdma_destroy_admin_queue(ctrl
, true);
2013 kfree(ctrl
->queues
);
2015 nvme_uninit_ctrl(&ctrl
->ctrl
);
2016 nvme_put_ctrl(&ctrl
->ctrl
);
2019 return ERR_PTR(ret
);
2022 return ERR_PTR(ret
);
2025 static struct nvmf_transport_ops nvme_rdma_transport
= {
2027 .required_opts
= NVMF_OPT_TRADDR
,
2028 .allowed_opts
= NVMF_OPT_TRSVCID
| NVMF_OPT_RECONNECT_DELAY
|
2029 NVMF_OPT_HOST_TRADDR
| NVMF_OPT_CTRL_LOSS_TMO
,
2030 .create_ctrl
= nvme_rdma_create_ctrl
,
2033 static void nvme_rdma_remove_one(struct ib_device
*ib_device
, void *client_data
)
2035 struct nvme_rdma_ctrl
*ctrl
;
2037 /* Delete all controllers using this device */
2038 mutex_lock(&nvme_rdma_ctrl_mutex
);
2039 list_for_each_entry(ctrl
, &nvme_rdma_ctrl_list
, list
) {
2040 if (ctrl
->device
->dev
!= ib_device
)
2042 dev_info(ctrl
->ctrl
.device
,
2043 "Removing ctrl: NQN \"%s\", addr %pISp\n",
2044 ctrl
->ctrl
.opts
->subsysnqn
, &ctrl
->addr
);
2045 nvme_delete_ctrl(&ctrl
->ctrl
);
2047 mutex_unlock(&nvme_rdma_ctrl_mutex
);
2049 flush_workqueue(nvme_wq
);
2052 static struct ib_client nvme_rdma_ib_client
= {
2053 .name
= "nvme_rdma",
2054 .remove
= nvme_rdma_remove_one
2057 static int __init
nvme_rdma_init_module(void)
2061 ret
= ib_register_client(&nvme_rdma_ib_client
);
2065 ret
= nvmf_register_transport(&nvme_rdma_transport
);
2067 goto err_unreg_client
;
2072 ib_unregister_client(&nvme_rdma_ib_client
);
2076 static void __exit
nvme_rdma_cleanup_module(void)
2078 nvmf_unregister_transport(&nvme_rdma_transport
);
2079 ib_unregister_client(&nvme_rdma_ib_client
);
2082 module_init(nvme_rdma_init_module
);
2083 module_exit(nvme_rdma_cleanup_module
);
2085 MODULE_LICENSE("GPL v2");