2 * NVMe over Fabrics RDMA host code.
3 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
15 #include <linux/module.h>
16 #include <linux/init.h>
17 #include <linux/slab.h>
18 #include <rdma/mr_pool.h>
19 #include <linux/err.h>
20 #include <linux/string.h>
21 #include <linux/atomic.h>
22 #include <linux/blk-mq.h>
23 #include <linux/blk-mq-rdma.h>
24 #include <linux/types.h>
25 #include <linux/list.h>
26 #include <linux/mutex.h>
27 #include <linux/scatterlist.h>
28 #include <linux/nvme.h>
29 #include <asm/unaligned.h>
31 #include <rdma/ib_verbs.h>
32 #include <rdma/rdma_cm.h>
33 #include <linux/nvme-rdma.h>
39 #define NVME_RDMA_CONNECT_TIMEOUT_MS 3000 /* 3 second */
41 #define NVME_RDMA_MAX_SEGMENTS 256
43 #define NVME_RDMA_MAX_INLINE_SEGMENTS 4
45 struct nvme_rdma_device
{
46 struct ib_device
*dev
;
49 struct list_head entry
;
50 unsigned int num_inline_segments
;
59 struct nvme_rdma_queue
;
60 struct nvme_rdma_request
{
61 struct nvme_request req
;
63 struct nvme_rdma_qe sqe
;
64 union nvme_result result
;
67 struct ib_sge sge
[1 + NVME_RDMA_MAX_INLINE_SEGMENTS
];
70 struct ib_reg_wr reg_wr
;
71 struct ib_cqe reg_cqe
;
72 struct nvme_rdma_queue
*queue
;
73 struct sg_table sg_table
;
74 struct scatterlist first_sgl
[];
77 enum nvme_rdma_queue_flags
{
78 NVME_RDMA_Q_ALLOCATED
= 0,
80 NVME_RDMA_Q_TR_READY
= 2,
83 struct nvme_rdma_queue
{
84 struct nvme_rdma_qe
*rsp_ring
;
86 size_t cmnd_capsule_len
;
87 struct nvme_rdma_ctrl
*ctrl
;
88 struct nvme_rdma_device
*device
;
93 struct rdma_cm_id
*cm_id
;
95 struct completion cm_done
;
98 struct nvme_rdma_ctrl
{
99 /* read only in the hot path */
100 struct nvme_rdma_queue
*queues
;
102 /* other member variables */
103 struct blk_mq_tag_set tag_set
;
104 struct work_struct err_work
;
106 struct nvme_rdma_qe async_event_sqe
;
108 struct delayed_work reconnect_work
;
110 struct list_head list
;
112 struct blk_mq_tag_set admin_tag_set
;
113 struct nvme_rdma_device
*device
;
117 struct sockaddr_storage addr
;
118 struct sockaddr_storage src_addr
;
120 struct nvme_ctrl ctrl
;
121 bool use_inline_data
;
124 static inline struct nvme_rdma_ctrl
*to_rdma_ctrl(struct nvme_ctrl
*ctrl
)
126 return container_of(ctrl
, struct nvme_rdma_ctrl
, ctrl
);
129 static LIST_HEAD(device_list
);
130 static DEFINE_MUTEX(device_list_mutex
);
132 static LIST_HEAD(nvme_rdma_ctrl_list
);
133 static DEFINE_MUTEX(nvme_rdma_ctrl_mutex
);
136 * Disabling this option makes small I/O goes faster, but is fundamentally
137 * unsafe. With it turned off we will have to register a global rkey that
138 * allows read and write access to all physical memory.
140 static bool register_always
= true;
141 module_param(register_always
, bool, 0444);
142 MODULE_PARM_DESC(register_always
,
143 "Use memory registration even for contiguous memory regions");
145 static int nvme_rdma_cm_handler(struct rdma_cm_id
*cm_id
,
146 struct rdma_cm_event
*event
);
147 static void nvme_rdma_recv_done(struct ib_cq
*cq
, struct ib_wc
*wc
);
149 static const struct blk_mq_ops nvme_rdma_mq_ops
;
150 static const struct blk_mq_ops nvme_rdma_admin_mq_ops
;
152 /* XXX: really should move to a generic header sooner or later.. */
153 static inline void put_unaligned_le24(u32 val
, u8
*p
)
160 static inline int nvme_rdma_queue_idx(struct nvme_rdma_queue
*queue
)
162 return queue
- queue
->ctrl
->queues
;
165 static inline size_t nvme_rdma_inline_data_size(struct nvme_rdma_queue
*queue
)
167 return queue
->cmnd_capsule_len
- sizeof(struct nvme_command
);
170 static void nvme_rdma_free_qe(struct ib_device
*ibdev
, struct nvme_rdma_qe
*qe
,
171 size_t capsule_size
, enum dma_data_direction dir
)
173 ib_dma_unmap_single(ibdev
, qe
->dma
, capsule_size
, dir
);
177 static int nvme_rdma_alloc_qe(struct ib_device
*ibdev
, struct nvme_rdma_qe
*qe
,
178 size_t capsule_size
, enum dma_data_direction dir
)
180 qe
->data
= kzalloc(capsule_size
, GFP_KERNEL
);
184 qe
->dma
= ib_dma_map_single(ibdev
, qe
->data
, capsule_size
, dir
);
185 if (ib_dma_mapping_error(ibdev
, qe
->dma
)) {
193 static void nvme_rdma_free_ring(struct ib_device
*ibdev
,
194 struct nvme_rdma_qe
*ring
, size_t ib_queue_size
,
195 size_t capsule_size
, enum dma_data_direction dir
)
199 for (i
= 0; i
< ib_queue_size
; i
++)
200 nvme_rdma_free_qe(ibdev
, &ring
[i
], capsule_size
, dir
);
204 static struct nvme_rdma_qe
*nvme_rdma_alloc_ring(struct ib_device
*ibdev
,
205 size_t ib_queue_size
, size_t capsule_size
,
206 enum dma_data_direction dir
)
208 struct nvme_rdma_qe
*ring
;
211 ring
= kcalloc(ib_queue_size
, sizeof(struct nvme_rdma_qe
), GFP_KERNEL
);
215 for (i
= 0; i
< ib_queue_size
; i
++) {
216 if (nvme_rdma_alloc_qe(ibdev
, &ring
[i
], capsule_size
, dir
))
223 nvme_rdma_free_ring(ibdev
, ring
, i
, capsule_size
, dir
);
227 static void nvme_rdma_qp_event(struct ib_event
*event
, void *context
)
229 pr_debug("QP event %s (%d)\n",
230 ib_event_msg(event
->event
), event
->event
);
234 static int nvme_rdma_wait_for_cm(struct nvme_rdma_queue
*queue
)
236 wait_for_completion_interruptible_timeout(&queue
->cm_done
,
237 msecs_to_jiffies(NVME_RDMA_CONNECT_TIMEOUT_MS
) + 1);
238 return queue
->cm_error
;
241 static int nvme_rdma_create_qp(struct nvme_rdma_queue
*queue
, const int factor
)
243 struct nvme_rdma_device
*dev
= queue
->device
;
244 struct ib_qp_init_attr init_attr
;
247 memset(&init_attr
, 0, sizeof(init_attr
));
248 init_attr
.event_handler
= nvme_rdma_qp_event
;
250 init_attr
.cap
.max_send_wr
= factor
* queue
->queue_size
+ 1;
252 init_attr
.cap
.max_recv_wr
= queue
->queue_size
+ 1;
253 init_attr
.cap
.max_recv_sge
= 1;
254 init_attr
.cap
.max_send_sge
= 1 + dev
->num_inline_segments
;
255 init_attr
.sq_sig_type
= IB_SIGNAL_REQ_WR
;
256 init_attr
.qp_type
= IB_QPT_RC
;
257 init_attr
.send_cq
= queue
->ib_cq
;
258 init_attr
.recv_cq
= queue
->ib_cq
;
260 ret
= rdma_create_qp(queue
->cm_id
, dev
->pd
, &init_attr
);
262 queue
->qp
= queue
->cm_id
->qp
;
266 static void nvme_rdma_exit_request(struct blk_mq_tag_set
*set
,
267 struct request
*rq
, unsigned int hctx_idx
)
269 struct nvme_rdma_ctrl
*ctrl
= set
->driver_data
;
270 struct nvme_rdma_request
*req
= blk_mq_rq_to_pdu(rq
);
271 int queue_idx
= (set
== &ctrl
->tag_set
) ? hctx_idx
+ 1 : 0;
272 struct nvme_rdma_queue
*queue
= &ctrl
->queues
[queue_idx
];
273 struct nvme_rdma_device
*dev
= queue
->device
;
275 nvme_rdma_free_qe(dev
->dev
, &req
->sqe
, sizeof(struct nvme_command
),
279 static int nvme_rdma_init_request(struct blk_mq_tag_set
*set
,
280 struct request
*rq
, unsigned int hctx_idx
,
281 unsigned int numa_node
)
283 struct nvme_rdma_ctrl
*ctrl
= set
->driver_data
;
284 struct nvme_rdma_request
*req
= blk_mq_rq_to_pdu(rq
);
285 int queue_idx
= (set
== &ctrl
->tag_set
) ? hctx_idx
+ 1 : 0;
286 struct nvme_rdma_queue
*queue
= &ctrl
->queues
[queue_idx
];
287 struct nvme_rdma_device
*dev
= queue
->device
;
288 struct ib_device
*ibdev
= dev
->dev
;
291 nvme_req(rq
)->ctrl
= &ctrl
->ctrl
;
292 ret
= nvme_rdma_alloc_qe(ibdev
, &req
->sqe
, sizeof(struct nvme_command
),
302 static int nvme_rdma_init_hctx(struct blk_mq_hw_ctx
*hctx
, void *data
,
303 unsigned int hctx_idx
)
305 struct nvme_rdma_ctrl
*ctrl
= data
;
306 struct nvme_rdma_queue
*queue
= &ctrl
->queues
[hctx_idx
+ 1];
308 BUG_ON(hctx_idx
>= ctrl
->ctrl
.queue_count
);
310 hctx
->driver_data
= queue
;
314 static int nvme_rdma_init_admin_hctx(struct blk_mq_hw_ctx
*hctx
, void *data
,
315 unsigned int hctx_idx
)
317 struct nvme_rdma_ctrl
*ctrl
= data
;
318 struct nvme_rdma_queue
*queue
= &ctrl
->queues
[0];
320 BUG_ON(hctx_idx
!= 0);
322 hctx
->driver_data
= queue
;
326 static void nvme_rdma_free_dev(struct kref
*ref
)
328 struct nvme_rdma_device
*ndev
=
329 container_of(ref
, struct nvme_rdma_device
, ref
);
331 mutex_lock(&device_list_mutex
);
332 list_del(&ndev
->entry
);
333 mutex_unlock(&device_list_mutex
);
335 ib_dealloc_pd(ndev
->pd
);
339 static void nvme_rdma_dev_put(struct nvme_rdma_device
*dev
)
341 kref_put(&dev
->ref
, nvme_rdma_free_dev
);
344 static int nvme_rdma_dev_get(struct nvme_rdma_device
*dev
)
346 return kref_get_unless_zero(&dev
->ref
);
349 static struct nvme_rdma_device
*
350 nvme_rdma_find_get_device(struct rdma_cm_id
*cm_id
)
352 struct nvme_rdma_device
*ndev
;
354 mutex_lock(&device_list_mutex
);
355 list_for_each_entry(ndev
, &device_list
, entry
) {
356 if (ndev
->dev
->node_guid
== cm_id
->device
->node_guid
&&
357 nvme_rdma_dev_get(ndev
))
361 ndev
= kzalloc(sizeof(*ndev
), GFP_KERNEL
);
365 ndev
->dev
= cm_id
->device
;
366 kref_init(&ndev
->ref
);
368 ndev
->pd
= ib_alloc_pd(ndev
->dev
,
369 register_always
? 0 : IB_PD_UNSAFE_GLOBAL_RKEY
);
370 if (IS_ERR(ndev
->pd
))
373 if (!(ndev
->dev
->attrs
.device_cap_flags
&
374 IB_DEVICE_MEM_MGT_EXTENSIONS
)) {
375 dev_err(&ndev
->dev
->dev
,
376 "Memory registrations not supported.\n");
380 ndev
->num_inline_segments
= min(NVME_RDMA_MAX_INLINE_SEGMENTS
,
381 ndev
->dev
->attrs
.max_sge
- 1);
382 list_add(&ndev
->entry
, &device_list
);
384 mutex_unlock(&device_list_mutex
);
388 ib_dealloc_pd(ndev
->pd
);
392 mutex_unlock(&device_list_mutex
);
396 static void nvme_rdma_destroy_queue_ib(struct nvme_rdma_queue
*queue
)
398 struct nvme_rdma_device
*dev
;
399 struct ib_device
*ibdev
;
401 if (!test_and_clear_bit(NVME_RDMA_Q_TR_READY
, &queue
->flags
))
407 ib_mr_pool_destroy(queue
->qp
, &queue
->qp
->rdma_mrs
);
410 * The cm_id object might have been destroyed during RDMA connection
411 * establishment error flow to avoid getting other cma events, thus
412 * the destruction of the QP shouldn't use rdma_cm API.
414 ib_destroy_qp(queue
->qp
);
415 ib_free_cq(queue
->ib_cq
);
417 nvme_rdma_free_ring(ibdev
, queue
->rsp_ring
, queue
->queue_size
,
418 sizeof(struct nvme_completion
), DMA_FROM_DEVICE
);
420 nvme_rdma_dev_put(dev
);
423 static int nvme_rdma_get_max_fr_pages(struct ib_device
*ibdev
)
425 return min_t(u32
, NVME_RDMA_MAX_SEGMENTS
,
426 ibdev
->attrs
.max_fast_reg_page_list_len
);
429 static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue
*queue
)
431 struct ib_device
*ibdev
;
432 const int send_wr_factor
= 3; /* MR, SEND, INV */
433 const int cq_factor
= send_wr_factor
+ 1; /* + RECV */
434 int comp_vector
, idx
= nvme_rdma_queue_idx(queue
);
437 queue
->device
= nvme_rdma_find_get_device(queue
->cm_id
);
438 if (!queue
->device
) {
439 dev_err(queue
->cm_id
->device
->dev
.parent
,
440 "no client data found!\n");
441 return -ECONNREFUSED
;
443 ibdev
= queue
->device
->dev
;
446 * Spread I/O queues completion vectors according their queue index.
447 * Admin queues can always go on completion vector 0.
449 comp_vector
= idx
== 0 ? idx
: idx
- 1;
451 /* +1 for ib_stop_cq */
452 queue
->ib_cq
= ib_alloc_cq(ibdev
, queue
,
453 cq_factor
* queue
->queue_size
+ 1,
454 comp_vector
, IB_POLL_SOFTIRQ
);
455 if (IS_ERR(queue
->ib_cq
)) {
456 ret
= PTR_ERR(queue
->ib_cq
);
460 ret
= nvme_rdma_create_qp(queue
, send_wr_factor
);
462 goto out_destroy_ib_cq
;
464 queue
->rsp_ring
= nvme_rdma_alloc_ring(ibdev
, queue
->queue_size
,
465 sizeof(struct nvme_completion
), DMA_FROM_DEVICE
);
466 if (!queue
->rsp_ring
) {
471 ret
= ib_mr_pool_init(queue
->qp
, &queue
->qp
->rdma_mrs
,
474 nvme_rdma_get_max_fr_pages(ibdev
));
476 dev_err(queue
->ctrl
->ctrl
.device
,
477 "failed to initialize MR pool sized %d for QID %d\n",
478 queue
->queue_size
, idx
);
479 goto out_destroy_ring
;
482 set_bit(NVME_RDMA_Q_TR_READY
, &queue
->flags
);
487 nvme_rdma_free_ring(ibdev
, queue
->rsp_ring
, queue
->queue_size
,
488 sizeof(struct nvme_completion
), DMA_FROM_DEVICE
);
490 rdma_destroy_qp(queue
->cm_id
);
492 ib_free_cq(queue
->ib_cq
);
494 nvme_rdma_dev_put(queue
->device
);
498 static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl
*ctrl
,
499 int idx
, size_t queue_size
)
501 struct nvme_rdma_queue
*queue
;
502 struct sockaddr
*src_addr
= NULL
;
505 queue
= &ctrl
->queues
[idx
];
507 init_completion(&queue
->cm_done
);
510 queue
->cmnd_capsule_len
= ctrl
->ctrl
.ioccsz
* 16;
512 queue
->cmnd_capsule_len
= sizeof(struct nvme_command
);
514 queue
->queue_size
= queue_size
;
516 queue
->cm_id
= rdma_create_id(&init_net
, nvme_rdma_cm_handler
, queue
,
517 RDMA_PS_TCP
, IB_QPT_RC
);
518 if (IS_ERR(queue
->cm_id
)) {
519 dev_info(ctrl
->ctrl
.device
,
520 "failed to create CM ID: %ld\n", PTR_ERR(queue
->cm_id
));
521 return PTR_ERR(queue
->cm_id
);
524 if (ctrl
->ctrl
.opts
->mask
& NVMF_OPT_HOST_TRADDR
)
525 src_addr
= (struct sockaddr
*)&ctrl
->src_addr
;
527 queue
->cm_error
= -ETIMEDOUT
;
528 ret
= rdma_resolve_addr(queue
->cm_id
, src_addr
,
529 (struct sockaddr
*)&ctrl
->addr
,
530 NVME_RDMA_CONNECT_TIMEOUT_MS
);
532 dev_info(ctrl
->ctrl
.device
,
533 "rdma_resolve_addr failed (%d).\n", ret
);
534 goto out_destroy_cm_id
;
537 ret
= nvme_rdma_wait_for_cm(queue
);
539 dev_info(ctrl
->ctrl
.device
,
540 "rdma connection establishment failed (%d)\n", ret
);
541 goto out_destroy_cm_id
;
544 set_bit(NVME_RDMA_Q_ALLOCATED
, &queue
->flags
);
549 rdma_destroy_id(queue
->cm_id
);
550 nvme_rdma_destroy_queue_ib(queue
);
554 static void nvme_rdma_stop_queue(struct nvme_rdma_queue
*queue
)
556 if (!test_and_clear_bit(NVME_RDMA_Q_LIVE
, &queue
->flags
))
559 rdma_disconnect(queue
->cm_id
);
560 ib_drain_qp(queue
->qp
);
563 static void nvme_rdma_free_queue(struct nvme_rdma_queue
*queue
)
565 if (!test_and_clear_bit(NVME_RDMA_Q_ALLOCATED
, &queue
->flags
))
568 nvme_rdma_destroy_queue_ib(queue
);
569 rdma_destroy_id(queue
->cm_id
);
572 static void nvme_rdma_free_io_queues(struct nvme_rdma_ctrl
*ctrl
)
576 for (i
= 1; i
< ctrl
->ctrl
.queue_count
; i
++)
577 nvme_rdma_free_queue(&ctrl
->queues
[i
]);
580 static void nvme_rdma_stop_io_queues(struct nvme_rdma_ctrl
*ctrl
)
584 for (i
= 1; i
< ctrl
->ctrl
.queue_count
; i
++)
585 nvme_rdma_stop_queue(&ctrl
->queues
[i
]);
588 static int nvme_rdma_start_queue(struct nvme_rdma_ctrl
*ctrl
, int idx
)
593 ret
= nvmf_connect_io_queue(&ctrl
->ctrl
, idx
);
595 ret
= nvmf_connect_admin_queue(&ctrl
->ctrl
);
598 set_bit(NVME_RDMA_Q_LIVE
, &ctrl
->queues
[idx
].flags
);
600 dev_info(ctrl
->ctrl
.device
,
601 "failed to connect queue: %d ret=%d\n", idx
, ret
);
605 static int nvme_rdma_start_io_queues(struct nvme_rdma_ctrl
*ctrl
)
609 for (i
= 1; i
< ctrl
->ctrl
.queue_count
; i
++) {
610 ret
= nvme_rdma_start_queue(ctrl
, i
);
612 goto out_stop_queues
;
618 for (i
--; i
>= 1; i
--)
619 nvme_rdma_stop_queue(&ctrl
->queues
[i
]);
623 static int nvme_rdma_alloc_io_queues(struct nvme_rdma_ctrl
*ctrl
)
625 struct nvmf_ctrl_options
*opts
= ctrl
->ctrl
.opts
;
626 struct ib_device
*ibdev
= ctrl
->device
->dev
;
627 unsigned int nr_io_queues
;
630 nr_io_queues
= min(opts
->nr_io_queues
, num_online_cpus());
633 * we map queues according to the device irq vectors for
634 * optimal locality so we don't need more queues than
635 * completion vectors.
637 nr_io_queues
= min_t(unsigned int, nr_io_queues
,
638 ibdev
->num_comp_vectors
);
640 ret
= nvme_set_queue_count(&ctrl
->ctrl
, &nr_io_queues
);
644 ctrl
->ctrl
.queue_count
= nr_io_queues
+ 1;
645 if (ctrl
->ctrl
.queue_count
< 2)
648 dev_info(ctrl
->ctrl
.device
,
649 "creating %d I/O queues.\n", nr_io_queues
);
651 for (i
= 1; i
< ctrl
->ctrl
.queue_count
; i
++) {
652 ret
= nvme_rdma_alloc_queue(ctrl
, i
,
653 ctrl
->ctrl
.sqsize
+ 1);
655 goto out_free_queues
;
661 for (i
--; i
>= 1; i
--)
662 nvme_rdma_free_queue(&ctrl
->queues
[i
]);
667 static void nvme_rdma_free_tagset(struct nvme_ctrl
*nctrl
,
668 struct blk_mq_tag_set
*set
)
670 struct nvme_rdma_ctrl
*ctrl
= to_rdma_ctrl(nctrl
);
672 blk_mq_free_tag_set(set
);
673 nvme_rdma_dev_put(ctrl
->device
);
676 static struct blk_mq_tag_set
*nvme_rdma_alloc_tagset(struct nvme_ctrl
*nctrl
,
679 struct nvme_rdma_ctrl
*ctrl
= to_rdma_ctrl(nctrl
);
680 struct blk_mq_tag_set
*set
;
684 set
= &ctrl
->admin_tag_set
;
685 memset(set
, 0, sizeof(*set
));
686 set
->ops
= &nvme_rdma_admin_mq_ops
;
687 set
->queue_depth
= NVME_AQ_MQ_TAG_DEPTH
;
688 set
->reserved_tags
= 2; /* connect + keep-alive */
689 set
->numa_node
= NUMA_NO_NODE
;
690 set
->cmd_size
= sizeof(struct nvme_rdma_request
) +
691 SG_CHUNK_SIZE
* sizeof(struct scatterlist
);
692 set
->driver_data
= ctrl
;
693 set
->nr_hw_queues
= 1;
694 set
->timeout
= ADMIN_TIMEOUT
;
695 set
->flags
= BLK_MQ_F_NO_SCHED
;
697 set
= &ctrl
->tag_set
;
698 memset(set
, 0, sizeof(*set
));
699 set
->ops
= &nvme_rdma_mq_ops
;
700 set
->queue_depth
= nctrl
->sqsize
+ 1;
701 set
->reserved_tags
= 1; /* fabric connect */
702 set
->numa_node
= NUMA_NO_NODE
;
703 set
->flags
= BLK_MQ_F_SHOULD_MERGE
;
704 set
->cmd_size
= sizeof(struct nvme_rdma_request
) +
705 SG_CHUNK_SIZE
* sizeof(struct scatterlist
);
706 set
->driver_data
= ctrl
;
707 set
->nr_hw_queues
= nctrl
->queue_count
- 1;
708 set
->timeout
= NVME_IO_TIMEOUT
;
711 ret
= blk_mq_alloc_tag_set(set
);
716 * We need a reference on the device as long as the tag_set is alive,
717 * as the MRs in the request structures need a valid ib_device.
719 ret
= nvme_rdma_dev_get(ctrl
->device
);
722 goto out_free_tagset
;
728 blk_mq_free_tag_set(set
);
733 static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl
*ctrl
,
737 blk_cleanup_queue(ctrl
->ctrl
.admin_q
);
738 nvme_rdma_free_tagset(&ctrl
->ctrl
, ctrl
->ctrl
.admin_tagset
);
740 if (ctrl
->async_event_sqe
.data
) {
741 nvme_rdma_free_qe(ctrl
->device
->dev
, &ctrl
->async_event_sqe
,
742 sizeof(struct nvme_command
), DMA_TO_DEVICE
);
743 ctrl
->async_event_sqe
.data
= NULL
;
745 nvme_rdma_free_queue(&ctrl
->queues
[0]);
748 static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl
*ctrl
,
753 error
= nvme_rdma_alloc_queue(ctrl
, 0, NVME_AQ_DEPTH
);
757 ctrl
->device
= ctrl
->queues
[0].device
;
759 ctrl
->max_fr_pages
= nvme_rdma_get_max_fr_pages(ctrl
->device
->dev
);
761 error
= nvme_rdma_alloc_qe(ctrl
->device
->dev
, &ctrl
->async_event_sqe
,
762 sizeof(struct nvme_command
), DMA_TO_DEVICE
);
767 ctrl
->ctrl
.admin_tagset
= nvme_rdma_alloc_tagset(&ctrl
->ctrl
, true);
768 if (IS_ERR(ctrl
->ctrl
.admin_tagset
)) {
769 error
= PTR_ERR(ctrl
->ctrl
.admin_tagset
);
770 goto out_free_async_qe
;
773 ctrl
->ctrl
.admin_q
= blk_mq_init_queue(&ctrl
->admin_tag_set
);
774 if (IS_ERR(ctrl
->ctrl
.admin_q
)) {
775 error
= PTR_ERR(ctrl
->ctrl
.admin_q
);
776 goto out_free_tagset
;
780 error
= nvme_rdma_start_queue(ctrl
, 0);
782 goto out_cleanup_queue
;
784 error
= ctrl
->ctrl
.ops
->reg_read64(&ctrl
->ctrl
, NVME_REG_CAP
,
787 dev_err(ctrl
->ctrl
.device
,
788 "prop_get NVME_REG_CAP failed\n");
793 min_t(int, NVME_CAP_MQES(ctrl
->ctrl
.cap
), ctrl
->ctrl
.sqsize
);
795 error
= nvme_enable_ctrl(&ctrl
->ctrl
, ctrl
->ctrl
.cap
);
799 ctrl
->ctrl
.max_hw_sectors
=
800 (ctrl
->max_fr_pages
- 1) << (ilog2(SZ_4K
) - 9);
802 error
= nvme_init_identify(&ctrl
->ctrl
);
809 nvme_rdma_stop_queue(&ctrl
->queues
[0]);
812 blk_cleanup_queue(ctrl
->ctrl
.admin_q
);
815 nvme_rdma_free_tagset(&ctrl
->ctrl
, ctrl
->ctrl
.admin_tagset
);
817 nvme_rdma_free_qe(ctrl
->device
->dev
, &ctrl
->async_event_sqe
,
818 sizeof(struct nvme_command
), DMA_TO_DEVICE
);
820 nvme_rdma_free_queue(&ctrl
->queues
[0]);
824 static void nvme_rdma_destroy_io_queues(struct nvme_rdma_ctrl
*ctrl
,
828 blk_cleanup_queue(ctrl
->ctrl
.connect_q
);
829 nvme_rdma_free_tagset(&ctrl
->ctrl
, ctrl
->ctrl
.tagset
);
831 nvme_rdma_free_io_queues(ctrl
);
834 static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl
*ctrl
, bool new)
838 ret
= nvme_rdma_alloc_io_queues(ctrl
);
843 ctrl
->ctrl
.tagset
= nvme_rdma_alloc_tagset(&ctrl
->ctrl
, false);
844 if (IS_ERR(ctrl
->ctrl
.tagset
)) {
845 ret
= PTR_ERR(ctrl
->ctrl
.tagset
);
846 goto out_free_io_queues
;
849 ctrl
->ctrl
.connect_q
= blk_mq_init_queue(&ctrl
->tag_set
);
850 if (IS_ERR(ctrl
->ctrl
.connect_q
)) {
851 ret
= PTR_ERR(ctrl
->ctrl
.connect_q
);
852 goto out_free_tag_set
;
855 blk_mq_update_nr_hw_queues(&ctrl
->tag_set
,
856 ctrl
->ctrl
.queue_count
- 1);
859 ret
= nvme_rdma_start_io_queues(ctrl
);
861 goto out_cleanup_connect_q
;
865 out_cleanup_connect_q
:
867 blk_cleanup_queue(ctrl
->ctrl
.connect_q
);
870 nvme_rdma_free_tagset(&ctrl
->ctrl
, ctrl
->ctrl
.tagset
);
872 nvme_rdma_free_io_queues(ctrl
);
876 static void nvme_rdma_teardown_admin_queue(struct nvme_rdma_ctrl
*ctrl
,
879 blk_mq_quiesce_queue(ctrl
->ctrl
.admin_q
);
880 nvme_rdma_stop_queue(&ctrl
->queues
[0]);
881 blk_mq_tagset_busy_iter(&ctrl
->admin_tag_set
, nvme_cancel_request
,
883 blk_mq_unquiesce_queue(ctrl
->ctrl
.admin_q
);
884 nvme_rdma_destroy_admin_queue(ctrl
, remove
);
887 static void nvme_rdma_teardown_io_queues(struct nvme_rdma_ctrl
*ctrl
,
890 if (ctrl
->ctrl
.queue_count
> 1) {
891 nvme_stop_queues(&ctrl
->ctrl
);
892 nvme_rdma_stop_io_queues(ctrl
);
893 blk_mq_tagset_busy_iter(&ctrl
->tag_set
, nvme_cancel_request
,
896 nvme_start_queues(&ctrl
->ctrl
);
897 nvme_rdma_destroy_io_queues(ctrl
, remove
);
901 static void nvme_rdma_stop_ctrl(struct nvme_ctrl
*nctrl
)
903 struct nvme_rdma_ctrl
*ctrl
= to_rdma_ctrl(nctrl
);
905 cancel_work_sync(&ctrl
->err_work
);
906 cancel_delayed_work_sync(&ctrl
->reconnect_work
);
909 static void nvme_rdma_free_ctrl(struct nvme_ctrl
*nctrl
)
911 struct nvme_rdma_ctrl
*ctrl
= to_rdma_ctrl(nctrl
);
913 if (list_empty(&ctrl
->list
))
916 mutex_lock(&nvme_rdma_ctrl_mutex
);
917 list_del(&ctrl
->list
);
918 mutex_unlock(&nvme_rdma_ctrl_mutex
);
920 nvmf_free_options(nctrl
->opts
);
926 static void nvme_rdma_reconnect_or_remove(struct nvme_rdma_ctrl
*ctrl
)
928 /* If we are resetting/deleting then do nothing */
929 if (ctrl
->ctrl
.state
!= NVME_CTRL_CONNECTING
) {
930 WARN_ON_ONCE(ctrl
->ctrl
.state
== NVME_CTRL_NEW
||
931 ctrl
->ctrl
.state
== NVME_CTRL_LIVE
);
935 if (nvmf_should_reconnect(&ctrl
->ctrl
)) {
936 dev_info(ctrl
->ctrl
.device
, "Reconnecting in %d seconds...\n",
937 ctrl
->ctrl
.opts
->reconnect_delay
);
938 queue_delayed_work(nvme_wq
, &ctrl
->reconnect_work
,
939 ctrl
->ctrl
.opts
->reconnect_delay
* HZ
);
941 nvme_delete_ctrl(&ctrl
->ctrl
);
945 static int nvme_rdma_setup_ctrl(struct nvme_rdma_ctrl
*ctrl
, bool new)
950 ret
= nvme_rdma_configure_admin_queue(ctrl
, new);
954 if (ctrl
->ctrl
.icdoff
) {
955 dev_err(ctrl
->ctrl
.device
, "icdoff is not supported!\n");
959 if (!(ctrl
->ctrl
.sgls
& (1 << 2))) {
960 dev_err(ctrl
->ctrl
.device
,
961 "Mandatory keyed sgls are not supported!\n");
965 if (ctrl
->ctrl
.opts
->queue_size
> ctrl
->ctrl
.sqsize
+ 1) {
966 dev_warn(ctrl
->ctrl
.device
,
967 "queue_size %zu > ctrl sqsize %u, clamping down\n",
968 ctrl
->ctrl
.opts
->queue_size
, ctrl
->ctrl
.sqsize
+ 1);
971 if (ctrl
->ctrl
.sqsize
+ 1 > ctrl
->ctrl
.maxcmd
) {
972 dev_warn(ctrl
->ctrl
.device
,
973 "sqsize %u > ctrl maxcmd %u, clamping down\n",
974 ctrl
->ctrl
.sqsize
+ 1, ctrl
->ctrl
.maxcmd
);
975 ctrl
->ctrl
.sqsize
= ctrl
->ctrl
.maxcmd
- 1;
978 if (ctrl
->ctrl
.sgls
& (1 << 20))
979 ctrl
->use_inline_data
= true;
981 if (ctrl
->ctrl
.queue_count
> 1) {
982 ret
= nvme_rdma_configure_io_queues(ctrl
, new);
987 changed
= nvme_change_ctrl_state(&ctrl
->ctrl
, NVME_CTRL_LIVE
);
989 /* state change failure is ok if we're in DELETING state */
990 WARN_ON_ONCE(ctrl
->ctrl
.state
!= NVME_CTRL_DELETING
);
995 nvme_start_ctrl(&ctrl
->ctrl
);
999 if (ctrl
->ctrl
.queue_count
> 1)
1000 nvme_rdma_destroy_io_queues(ctrl
, new);
1002 nvme_rdma_stop_queue(&ctrl
->queues
[0]);
1003 nvme_rdma_destroy_admin_queue(ctrl
, new);
1007 static void nvme_rdma_reconnect_ctrl_work(struct work_struct
*work
)
1009 struct nvme_rdma_ctrl
*ctrl
= container_of(to_delayed_work(work
),
1010 struct nvme_rdma_ctrl
, reconnect_work
);
1012 ++ctrl
->ctrl
.nr_reconnects
;
1014 if (nvme_rdma_setup_ctrl(ctrl
, false))
1017 dev_info(ctrl
->ctrl
.device
, "Successfully reconnected (%d attempts)\n",
1018 ctrl
->ctrl
.nr_reconnects
);
1020 ctrl
->ctrl
.nr_reconnects
= 0;
1025 dev_info(ctrl
->ctrl
.device
, "Failed reconnect attempt %d\n",
1026 ctrl
->ctrl
.nr_reconnects
);
1027 nvme_rdma_reconnect_or_remove(ctrl
);
1030 static void nvme_rdma_error_recovery_work(struct work_struct
*work
)
1032 struct nvme_rdma_ctrl
*ctrl
= container_of(work
,
1033 struct nvme_rdma_ctrl
, err_work
);
1035 nvme_stop_keep_alive(&ctrl
->ctrl
);
1036 nvme_rdma_teardown_io_queues(ctrl
, false);
1037 nvme_start_queues(&ctrl
->ctrl
);
1038 nvme_rdma_teardown_admin_queue(ctrl
, false);
1040 if (!nvme_change_ctrl_state(&ctrl
->ctrl
, NVME_CTRL_CONNECTING
)) {
1041 /* state change failure is ok if we're in DELETING state */
1042 WARN_ON_ONCE(ctrl
->ctrl
.state
!= NVME_CTRL_DELETING
);
1046 nvme_rdma_reconnect_or_remove(ctrl
);
1049 static void nvme_rdma_error_recovery(struct nvme_rdma_ctrl
*ctrl
)
1051 if (!nvme_change_ctrl_state(&ctrl
->ctrl
, NVME_CTRL_RESETTING
))
1054 queue_work(nvme_wq
, &ctrl
->err_work
);
1057 static void nvme_rdma_wr_error(struct ib_cq
*cq
, struct ib_wc
*wc
,
1060 struct nvme_rdma_queue
*queue
= cq
->cq_context
;
1061 struct nvme_rdma_ctrl
*ctrl
= queue
->ctrl
;
1063 if (ctrl
->ctrl
.state
== NVME_CTRL_LIVE
)
1064 dev_info(ctrl
->ctrl
.device
,
1065 "%s for CQE 0x%p failed with status %s (%d)\n",
1067 ib_wc_status_msg(wc
->status
), wc
->status
);
1068 nvme_rdma_error_recovery(ctrl
);
1071 static void nvme_rdma_memreg_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
1073 if (unlikely(wc
->status
!= IB_WC_SUCCESS
))
1074 nvme_rdma_wr_error(cq
, wc
, "MEMREG");
1077 static void nvme_rdma_inv_rkey_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
1079 struct nvme_rdma_request
*req
=
1080 container_of(wc
->wr_cqe
, struct nvme_rdma_request
, reg_cqe
);
1081 struct request
*rq
= blk_mq_rq_from_pdu(req
);
1083 if (unlikely(wc
->status
!= IB_WC_SUCCESS
)) {
1084 nvme_rdma_wr_error(cq
, wc
, "LOCAL_INV");
1088 if (refcount_dec_and_test(&req
->ref
))
1089 nvme_end_request(rq
, req
->status
, req
->result
);
1093 static int nvme_rdma_inv_rkey(struct nvme_rdma_queue
*queue
,
1094 struct nvme_rdma_request
*req
)
1096 struct ib_send_wr
*bad_wr
;
1097 struct ib_send_wr wr
= {
1098 .opcode
= IB_WR_LOCAL_INV
,
1101 .send_flags
= IB_SEND_SIGNALED
,
1102 .ex
.invalidate_rkey
= req
->mr
->rkey
,
1105 req
->reg_cqe
.done
= nvme_rdma_inv_rkey_done
;
1106 wr
.wr_cqe
= &req
->reg_cqe
;
1108 return ib_post_send(queue
->qp
, &wr
, &bad_wr
);
1111 static void nvme_rdma_unmap_data(struct nvme_rdma_queue
*queue
,
1114 struct nvme_rdma_request
*req
= blk_mq_rq_to_pdu(rq
);
1115 struct nvme_rdma_device
*dev
= queue
->device
;
1116 struct ib_device
*ibdev
= dev
->dev
;
1118 if (!blk_rq_payload_bytes(rq
))
1122 ib_mr_pool_put(queue
->qp
, &queue
->qp
->rdma_mrs
, req
->mr
);
1126 ib_dma_unmap_sg(ibdev
, req
->sg_table
.sgl
,
1127 req
->nents
, rq_data_dir(rq
) ==
1128 WRITE
? DMA_TO_DEVICE
: DMA_FROM_DEVICE
);
1130 nvme_cleanup_cmd(rq
);
1131 sg_free_table_chained(&req
->sg_table
, true);
1134 static int nvme_rdma_set_sg_null(struct nvme_command
*c
)
1136 struct nvme_keyed_sgl_desc
*sg
= &c
->common
.dptr
.ksgl
;
1139 put_unaligned_le24(0, sg
->length
);
1140 put_unaligned_le32(0, sg
->key
);
1141 sg
->type
= NVME_KEY_SGL_FMT_DATA_DESC
<< 4;
1145 static int nvme_rdma_map_sg_inline(struct nvme_rdma_queue
*queue
,
1146 struct nvme_rdma_request
*req
, struct nvme_command
*c
,
1149 struct nvme_sgl_desc
*sg
= &c
->common
.dptr
.sgl
;
1150 struct scatterlist
*sgl
= req
->sg_table
.sgl
;
1151 struct ib_sge
*sge
= &req
->sge
[1];
1155 for (i
= 0; i
< count
; i
++, sgl
++, sge
++) {
1156 sge
->addr
= sg_dma_address(sgl
);
1157 sge
->length
= sg_dma_len(sgl
);
1158 sge
->lkey
= queue
->device
->pd
->local_dma_lkey
;
1162 sg
->addr
= cpu_to_le64(queue
->ctrl
->ctrl
.icdoff
);
1163 sg
->length
= cpu_to_le32(len
);
1164 sg
->type
= (NVME_SGL_FMT_DATA_DESC
<< 4) | NVME_SGL_FMT_OFFSET
;
1166 req
->num_sge
+= count
;
1170 static int nvme_rdma_map_sg_single(struct nvme_rdma_queue
*queue
,
1171 struct nvme_rdma_request
*req
, struct nvme_command
*c
)
1173 struct nvme_keyed_sgl_desc
*sg
= &c
->common
.dptr
.ksgl
;
1175 sg
->addr
= cpu_to_le64(sg_dma_address(req
->sg_table
.sgl
));
1176 put_unaligned_le24(sg_dma_len(req
->sg_table
.sgl
), sg
->length
);
1177 put_unaligned_le32(queue
->device
->pd
->unsafe_global_rkey
, sg
->key
);
1178 sg
->type
= NVME_KEY_SGL_FMT_DATA_DESC
<< 4;
1182 static int nvme_rdma_map_sg_fr(struct nvme_rdma_queue
*queue
,
1183 struct nvme_rdma_request
*req
, struct nvme_command
*c
,
1186 struct nvme_keyed_sgl_desc
*sg
= &c
->common
.dptr
.ksgl
;
1189 req
->mr
= ib_mr_pool_get(queue
->qp
, &queue
->qp
->rdma_mrs
);
1190 if (WARN_ON_ONCE(!req
->mr
))
1194 * Align the MR to a 4K page size to match the ctrl page size and
1195 * the block virtual boundary.
1197 nr
= ib_map_mr_sg(req
->mr
, req
->sg_table
.sgl
, count
, NULL
, SZ_4K
);
1198 if (unlikely(nr
< count
)) {
1199 ib_mr_pool_put(queue
->qp
, &queue
->qp
->rdma_mrs
, req
->mr
);
1206 ib_update_fast_reg_key(req
->mr
, ib_inc_rkey(req
->mr
->rkey
));
1208 req
->reg_cqe
.done
= nvme_rdma_memreg_done
;
1209 memset(&req
->reg_wr
, 0, sizeof(req
->reg_wr
));
1210 req
->reg_wr
.wr
.opcode
= IB_WR_REG_MR
;
1211 req
->reg_wr
.wr
.wr_cqe
= &req
->reg_cqe
;
1212 req
->reg_wr
.wr
.num_sge
= 0;
1213 req
->reg_wr
.mr
= req
->mr
;
1214 req
->reg_wr
.key
= req
->mr
->rkey
;
1215 req
->reg_wr
.access
= IB_ACCESS_LOCAL_WRITE
|
1216 IB_ACCESS_REMOTE_READ
|
1217 IB_ACCESS_REMOTE_WRITE
;
1219 sg
->addr
= cpu_to_le64(req
->mr
->iova
);
1220 put_unaligned_le24(req
->mr
->length
, sg
->length
);
1221 put_unaligned_le32(req
->mr
->rkey
, sg
->key
);
1222 sg
->type
= (NVME_KEY_SGL_FMT_DATA_DESC
<< 4) |
1223 NVME_SGL_FMT_INVALIDATE
;
1228 static int nvme_rdma_map_data(struct nvme_rdma_queue
*queue
,
1229 struct request
*rq
, struct nvme_command
*c
)
1231 struct nvme_rdma_request
*req
= blk_mq_rq_to_pdu(rq
);
1232 struct nvme_rdma_device
*dev
= queue
->device
;
1233 struct ib_device
*ibdev
= dev
->dev
;
1237 refcount_set(&req
->ref
, 2); /* send and recv completions */
1239 c
->common
.flags
|= NVME_CMD_SGL_METABUF
;
1241 if (!blk_rq_payload_bytes(rq
))
1242 return nvme_rdma_set_sg_null(c
);
1244 req
->sg_table
.sgl
= req
->first_sgl
;
1245 ret
= sg_alloc_table_chained(&req
->sg_table
,
1246 blk_rq_nr_phys_segments(rq
), req
->sg_table
.sgl
);
1250 req
->nents
= blk_rq_map_sg(rq
->q
, rq
, req
->sg_table
.sgl
);
1252 count
= ib_dma_map_sg(ibdev
, req
->sg_table
.sgl
, req
->nents
,
1253 rq_data_dir(rq
) == WRITE
? DMA_TO_DEVICE
: DMA_FROM_DEVICE
);
1254 if (unlikely(count
<= 0)) {
1256 goto out_free_table
;
1259 if (count
<= dev
->num_inline_segments
) {
1260 if (rq_data_dir(rq
) == WRITE
&& nvme_rdma_queue_idx(queue
) &&
1261 queue
->ctrl
->use_inline_data
&&
1262 blk_rq_payload_bytes(rq
) <=
1263 nvme_rdma_inline_data_size(queue
)) {
1264 ret
= nvme_rdma_map_sg_inline(queue
, req
, c
, count
);
1268 if (count
== 1 && dev
->pd
->flags
& IB_PD_UNSAFE_GLOBAL_RKEY
) {
1269 ret
= nvme_rdma_map_sg_single(queue
, req
, c
);
1274 ret
= nvme_rdma_map_sg_fr(queue
, req
, c
, count
);
1282 ib_dma_unmap_sg(ibdev
, req
->sg_table
.sgl
,
1283 req
->nents
, rq_data_dir(rq
) ==
1284 WRITE
? DMA_TO_DEVICE
: DMA_FROM_DEVICE
);
1286 sg_free_table_chained(&req
->sg_table
, true);
1290 static void nvme_rdma_send_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
1292 struct nvme_rdma_qe
*qe
=
1293 container_of(wc
->wr_cqe
, struct nvme_rdma_qe
, cqe
);
1294 struct nvme_rdma_request
*req
=
1295 container_of(qe
, struct nvme_rdma_request
, sqe
);
1296 struct request
*rq
= blk_mq_rq_from_pdu(req
);
1298 if (unlikely(wc
->status
!= IB_WC_SUCCESS
)) {
1299 nvme_rdma_wr_error(cq
, wc
, "SEND");
1303 if (refcount_dec_and_test(&req
->ref
))
1304 nvme_end_request(rq
, req
->status
, req
->result
);
1307 static int nvme_rdma_post_send(struct nvme_rdma_queue
*queue
,
1308 struct nvme_rdma_qe
*qe
, struct ib_sge
*sge
, u32 num_sge
,
1309 struct ib_send_wr
*first
)
1311 struct ib_send_wr wr
, *bad_wr
;
1314 sge
->addr
= qe
->dma
;
1315 sge
->length
= sizeof(struct nvme_command
),
1316 sge
->lkey
= queue
->device
->pd
->local_dma_lkey
;
1319 wr
.wr_cqe
= &qe
->cqe
;
1321 wr
.num_sge
= num_sge
;
1322 wr
.opcode
= IB_WR_SEND
;
1323 wr
.send_flags
= IB_SEND_SIGNALED
;
1330 ret
= ib_post_send(queue
->qp
, first
, &bad_wr
);
1331 if (unlikely(ret
)) {
1332 dev_err(queue
->ctrl
->ctrl
.device
,
1333 "%s failed with error code %d\n", __func__
, ret
);
1338 static int nvme_rdma_post_recv(struct nvme_rdma_queue
*queue
,
1339 struct nvme_rdma_qe
*qe
)
1341 struct ib_recv_wr wr
, *bad_wr
;
1345 list
.addr
= qe
->dma
;
1346 list
.length
= sizeof(struct nvme_completion
);
1347 list
.lkey
= queue
->device
->pd
->local_dma_lkey
;
1349 qe
->cqe
.done
= nvme_rdma_recv_done
;
1352 wr
.wr_cqe
= &qe
->cqe
;
1356 ret
= ib_post_recv(queue
->qp
, &wr
, &bad_wr
);
1357 if (unlikely(ret
)) {
1358 dev_err(queue
->ctrl
->ctrl
.device
,
1359 "%s failed with error code %d\n", __func__
, ret
);
1364 static struct blk_mq_tags
*nvme_rdma_tagset(struct nvme_rdma_queue
*queue
)
1366 u32 queue_idx
= nvme_rdma_queue_idx(queue
);
1369 return queue
->ctrl
->admin_tag_set
.tags
[queue_idx
];
1370 return queue
->ctrl
->tag_set
.tags
[queue_idx
- 1];
1373 static void nvme_rdma_async_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
1375 if (unlikely(wc
->status
!= IB_WC_SUCCESS
))
1376 nvme_rdma_wr_error(cq
, wc
, "ASYNC");
1379 static void nvme_rdma_submit_async_event(struct nvme_ctrl
*arg
)
1381 struct nvme_rdma_ctrl
*ctrl
= to_rdma_ctrl(arg
);
1382 struct nvme_rdma_queue
*queue
= &ctrl
->queues
[0];
1383 struct ib_device
*dev
= queue
->device
->dev
;
1384 struct nvme_rdma_qe
*sqe
= &ctrl
->async_event_sqe
;
1385 struct nvme_command
*cmd
= sqe
->data
;
1389 ib_dma_sync_single_for_cpu(dev
, sqe
->dma
, sizeof(*cmd
), DMA_TO_DEVICE
);
1391 memset(cmd
, 0, sizeof(*cmd
));
1392 cmd
->common
.opcode
= nvme_admin_async_event
;
1393 cmd
->common
.command_id
= NVME_AQ_BLK_MQ_DEPTH
;
1394 cmd
->common
.flags
|= NVME_CMD_SGL_METABUF
;
1395 nvme_rdma_set_sg_null(cmd
);
1397 sqe
->cqe
.done
= nvme_rdma_async_done
;
1399 ib_dma_sync_single_for_device(dev
, sqe
->dma
, sizeof(*cmd
),
1402 ret
= nvme_rdma_post_send(queue
, sqe
, &sge
, 1, NULL
);
1406 static int nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue
*queue
,
1407 struct nvme_completion
*cqe
, struct ib_wc
*wc
, int tag
)
1410 struct nvme_rdma_request
*req
;
1413 rq
= blk_mq_tag_to_rq(nvme_rdma_tagset(queue
), cqe
->command_id
);
1415 dev_err(queue
->ctrl
->ctrl
.device
,
1416 "tag 0x%x on QP %#x not found\n",
1417 cqe
->command_id
, queue
->qp
->qp_num
);
1418 nvme_rdma_error_recovery(queue
->ctrl
);
1421 req
= blk_mq_rq_to_pdu(rq
);
1423 req
->status
= cqe
->status
;
1424 req
->result
= cqe
->result
;
1426 if (wc
->wc_flags
& IB_WC_WITH_INVALIDATE
) {
1427 if (unlikely(wc
->ex
.invalidate_rkey
!= req
->mr
->rkey
)) {
1428 dev_err(queue
->ctrl
->ctrl
.device
,
1429 "Bogus remote invalidation for rkey %#x\n",
1431 nvme_rdma_error_recovery(queue
->ctrl
);
1433 } else if (req
->mr
) {
1434 ret
= nvme_rdma_inv_rkey(queue
, req
);
1435 if (unlikely(ret
< 0)) {
1436 dev_err(queue
->ctrl
->ctrl
.device
,
1437 "Queueing INV WR for rkey %#x failed (%d)\n",
1438 req
->mr
->rkey
, ret
);
1439 nvme_rdma_error_recovery(queue
->ctrl
);
1441 /* the local invalidation completion will end the request */
1445 if (refcount_dec_and_test(&req
->ref
)) {
1448 nvme_end_request(rq
, req
->status
, req
->result
);
1454 static int __nvme_rdma_recv_done(struct ib_cq
*cq
, struct ib_wc
*wc
, int tag
)
1456 struct nvme_rdma_qe
*qe
=
1457 container_of(wc
->wr_cqe
, struct nvme_rdma_qe
, cqe
);
1458 struct nvme_rdma_queue
*queue
= cq
->cq_context
;
1459 struct ib_device
*ibdev
= queue
->device
->dev
;
1460 struct nvme_completion
*cqe
= qe
->data
;
1461 const size_t len
= sizeof(struct nvme_completion
);
1464 if (unlikely(wc
->status
!= IB_WC_SUCCESS
)) {
1465 nvme_rdma_wr_error(cq
, wc
, "RECV");
1469 ib_dma_sync_single_for_cpu(ibdev
, qe
->dma
, len
, DMA_FROM_DEVICE
);
1471 * AEN requests are special as they don't time out and can
1472 * survive any kind of queue freeze and often don't respond to
1473 * aborts. We don't even bother to allocate a struct request
1474 * for them but rather special case them here.
1476 if (unlikely(nvme_rdma_queue_idx(queue
) == 0 &&
1477 cqe
->command_id
>= NVME_AQ_BLK_MQ_DEPTH
))
1478 nvme_complete_async_event(&queue
->ctrl
->ctrl
, cqe
->status
,
1481 ret
= nvme_rdma_process_nvme_rsp(queue
, cqe
, wc
, tag
);
1482 ib_dma_sync_single_for_device(ibdev
, qe
->dma
, len
, DMA_FROM_DEVICE
);
1484 nvme_rdma_post_recv(queue
, qe
);
1488 static void nvme_rdma_recv_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
1490 __nvme_rdma_recv_done(cq
, wc
, -1);
1493 static int nvme_rdma_conn_established(struct nvme_rdma_queue
*queue
)
1497 for (i
= 0; i
< queue
->queue_size
; i
++) {
1498 ret
= nvme_rdma_post_recv(queue
, &queue
->rsp_ring
[i
]);
1500 goto out_destroy_queue_ib
;
1505 out_destroy_queue_ib
:
1506 nvme_rdma_destroy_queue_ib(queue
);
1510 static int nvme_rdma_conn_rejected(struct nvme_rdma_queue
*queue
,
1511 struct rdma_cm_event
*ev
)
1513 struct rdma_cm_id
*cm_id
= queue
->cm_id
;
1514 int status
= ev
->status
;
1515 const char *rej_msg
;
1516 const struct nvme_rdma_cm_rej
*rej_data
;
1519 rej_msg
= rdma_reject_msg(cm_id
, status
);
1520 rej_data
= rdma_consumer_reject_data(cm_id
, ev
, &rej_data_len
);
1522 if (rej_data
&& rej_data_len
>= sizeof(u16
)) {
1523 u16 sts
= le16_to_cpu(rej_data
->sts
);
1525 dev_err(queue
->ctrl
->ctrl
.device
,
1526 "Connect rejected: status %d (%s) nvme status %d (%s).\n",
1527 status
, rej_msg
, sts
, nvme_rdma_cm_msg(sts
));
1529 dev_err(queue
->ctrl
->ctrl
.device
,
1530 "Connect rejected: status %d (%s).\n", status
, rej_msg
);
1536 static int nvme_rdma_addr_resolved(struct nvme_rdma_queue
*queue
)
1540 ret
= nvme_rdma_create_queue_ib(queue
);
1544 ret
= rdma_resolve_route(queue
->cm_id
, NVME_RDMA_CONNECT_TIMEOUT_MS
);
1546 dev_err(queue
->ctrl
->ctrl
.device
,
1547 "rdma_resolve_route failed (%d).\n",
1549 goto out_destroy_queue
;
1555 nvme_rdma_destroy_queue_ib(queue
);
1559 static int nvme_rdma_route_resolved(struct nvme_rdma_queue
*queue
)
1561 struct nvme_rdma_ctrl
*ctrl
= queue
->ctrl
;
1562 struct rdma_conn_param param
= { };
1563 struct nvme_rdma_cm_req priv
= { };
1566 param
.qp_num
= queue
->qp
->qp_num
;
1567 param
.flow_control
= 1;
1569 param
.responder_resources
= queue
->device
->dev
->attrs
.max_qp_rd_atom
;
1570 /* maximum retry count */
1571 param
.retry_count
= 7;
1572 param
.rnr_retry_count
= 7;
1573 param
.private_data
= &priv
;
1574 param
.private_data_len
= sizeof(priv
);
1576 priv
.recfmt
= cpu_to_le16(NVME_RDMA_CM_FMT_1_0
);
1577 priv
.qid
= cpu_to_le16(nvme_rdma_queue_idx(queue
));
1579 * set the admin queue depth to the minimum size
1580 * specified by the Fabrics standard.
1582 if (priv
.qid
== 0) {
1583 priv
.hrqsize
= cpu_to_le16(NVME_AQ_DEPTH
);
1584 priv
.hsqsize
= cpu_to_le16(NVME_AQ_DEPTH
- 1);
1587 * current interpretation of the fabrics spec
1588 * is at minimum you make hrqsize sqsize+1, or a
1589 * 1's based representation of sqsize.
1591 priv
.hrqsize
= cpu_to_le16(queue
->queue_size
);
1592 priv
.hsqsize
= cpu_to_le16(queue
->ctrl
->ctrl
.sqsize
);
1595 ret
= rdma_connect(queue
->cm_id
, ¶m
);
1597 dev_err(ctrl
->ctrl
.device
,
1598 "rdma_connect failed (%d).\n", ret
);
1599 goto out_destroy_queue_ib
;
1604 out_destroy_queue_ib
:
1605 nvme_rdma_destroy_queue_ib(queue
);
1609 static int nvme_rdma_cm_handler(struct rdma_cm_id
*cm_id
,
1610 struct rdma_cm_event
*ev
)
1612 struct nvme_rdma_queue
*queue
= cm_id
->context
;
1615 dev_dbg(queue
->ctrl
->ctrl
.device
, "%s (%d): status %d id %p\n",
1616 rdma_event_msg(ev
->event
), ev
->event
,
1619 switch (ev
->event
) {
1620 case RDMA_CM_EVENT_ADDR_RESOLVED
:
1621 cm_error
= nvme_rdma_addr_resolved(queue
);
1623 case RDMA_CM_EVENT_ROUTE_RESOLVED
:
1624 cm_error
= nvme_rdma_route_resolved(queue
);
1626 case RDMA_CM_EVENT_ESTABLISHED
:
1627 queue
->cm_error
= nvme_rdma_conn_established(queue
);
1628 /* complete cm_done regardless of success/failure */
1629 complete(&queue
->cm_done
);
1631 case RDMA_CM_EVENT_REJECTED
:
1632 nvme_rdma_destroy_queue_ib(queue
);
1633 cm_error
= nvme_rdma_conn_rejected(queue
, ev
);
1635 case RDMA_CM_EVENT_ROUTE_ERROR
:
1636 case RDMA_CM_EVENT_CONNECT_ERROR
:
1637 case RDMA_CM_EVENT_UNREACHABLE
:
1638 nvme_rdma_destroy_queue_ib(queue
);
1640 case RDMA_CM_EVENT_ADDR_ERROR
:
1641 dev_dbg(queue
->ctrl
->ctrl
.device
,
1642 "CM error event %d\n", ev
->event
);
1643 cm_error
= -ECONNRESET
;
1645 case RDMA_CM_EVENT_DISCONNECTED
:
1646 case RDMA_CM_EVENT_ADDR_CHANGE
:
1647 case RDMA_CM_EVENT_TIMEWAIT_EXIT
:
1648 dev_dbg(queue
->ctrl
->ctrl
.device
,
1649 "disconnect received - connection closed\n");
1650 nvme_rdma_error_recovery(queue
->ctrl
);
1652 case RDMA_CM_EVENT_DEVICE_REMOVAL
:
1653 /* device removal is handled via the ib_client API */
1656 dev_err(queue
->ctrl
->ctrl
.device
,
1657 "Unexpected RDMA CM event (%d)\n", ev
->event
);
1658 nvme_rdma_error_recovery(queue
->ctrl
);
1663 queue
->cm_error
= cm_error
;
1664 complete(&queue
->cm_done
);
1670 static enum blk_eh_timer_return
1671 nvme_rdma_timeout(struct request
*rq
, bool reserved
)
1673 struct nvme_rdma_request
*req
= blk_mq_rq_to_pdu(rq
);
1675 dev_warn(req
->queue
->ctrl
->ctrl
.device
,
1676 "I/O %d QID %d timeout, reset controller\n",
1677 rq
->tag
, nvme_rdma_queue_idx(req
->queue
));
1679 /* queue error recovery */
1680 nvme_rdma_error_recovery(req
->queue
->ctrl
);
1682 /* fail with DNR on cmd timeout */
1683 nvme_req(rq
)->status
= NVME_SC_ABORT_REQ
| NVME_SC_DNR
;
1688 static blk_status_t
nvme_rdma_queue_rq(struct blk_mq_hw_ctx
*hctx
,
1689 const struct blk_mq_queue_data
*bd
)
1691 struct nvme_ns
*ns
= hctx
->queue
->queuedata
;
1692 struct nvme_rdma_queue
*queue
= hctx
->driver_data
;
1693 struct request
*rq
= bd
->rq
;
1694 struct nvme_rdma_request
*req
= blk_mq_rq_to_pdu(rq
);
1695 struct nvme_rdma_qe
*sqe
= &req
->sqe
;
1696 struct nvme_command
*c
= sqe
->data
;
1697 struct ib_device
*dev
;
1698 bool queue_ready
= test_bit(NVME_RDMA_Q_LIVE
, &queue
->flags
);
1702 WARN_ON_ONCE(rq
->tag
< 0);
1704 if (!nvmf_check_ready(&queue
->ctrl
->ctrl
, rq
, queue_ready
))
1705 return nvmf_fail_nonready_command(&queue
->ctrl
->ctrl
, rq
);
1707 dev
= queue
->device
->dev
;
1708 ib_dma_sync_single_for_cpu(dev
, sqe
->dma
,
1709 sizeof(struct nvme_command
), DMA_TO_DEVICE
);
1711 ret
= nvme_setup_cmd(ns
, rq
, c
);
1715 blk_mq_start_request(rq
);
1717 err
= nvme_rdma_map_data(queue
, rq
, c
);
1718 if (unlikely(err
< 0)) {
1719 dev_err(queue
->ctrl
->ctrl
.device
,
1720 "Failed to map data (%d)\n", err
);
1721 nvme_cleanup_cmd(rq
);
1725 sqe
->cqe
.done
= nvme_rdma_send_done
;
1727 ib_dma_sync_single_for_device(dev
, sqe
->dma
,
1728 sizeof(struct nvme_command
), DMA_TO_DEVICE
);
1730 err
= nvme_rdma_post_send(queue
, sqe
, req
->sge
, req
->num_sge
,
1731 req
->mr
? &req
->reg_wr
.wr
: NULL
);
1732 if (unlikely(err
)) {
1733 nvme_rdma_unmap_data(queue
, rq
);
1739 if (err
== -ENOMEM
|| err
== -EAGAIN
)
1740 return BLK_STS_RESOURCE
;
1741 return BLK_STS_IOERR
;
1744 static int nvme_rdma_poll(struct blk_mq_hw_ctx
*hctx
, unsigned int tag
)
1746 struct nvme_rdma_queue
*queue
= hctx
->driver_data
;
1747 struct ib_cq
*cq
= queue
->ib_cq
;
1751 while (ib_poll_cq(cq
, 1, &wc
) > 0) {
1752 struct ib_cqe
*cqe
= wc
.wr_cqe
;
1755 if (cqe
->done
== nvme_rdma_recv_done
)
1756 found
|= __nvme_rdma_recv_done(cq
, &wc
, tag
);
1765 static void nvme_rdma_complete_rq(struct request
*rq
)
1767 struct nvme_rdma_request
*req
= blk_mq_rq_to_pdu(rq
);
1769 nvme_rdma_unmap_data(req
->queue
, rq
);
1770 nvme_complete_rq(rq
);
1773 static int nvme_rdma_map_queues(struct blk_mq_tag_set
*set
)
1775 struct nvme_rdma_ctrl
*ctrl
= set
->driver_data
;
1777 return blk_mq_rdma_map_queues(set
, ctrl
->device
->dev
, 0);
1780 static const struct blk_mq_ops nvme_rdma_mq_ops
= {
1781 .queue_rq
= nvme_rdma_queue_rq
,
1782 .complete
= nvme_rdma_complete_rq
,
1783 .init_request
= nvme_rdma_init_request
,
1784 .exit_request
= nvme_rdma_exit_request
,
1785 .init_hctx
= nvme_rdma_init_hctx
,
1786 .poll
= nvme_rdma_poll
,
1787 .timeout
= nvme_rdma_timeout
,
1788 .map_queues
= nvme_rdma_map_queues
,
1791 static const struct blk_mq_ops nvme_rdma_admin_mq_ops
= {
1792 .queue_rq
= nvme_rdma_queue_rq
,
1793 .complete
= nvme_rdma_complete_rq
,
1794 .init_request
= nvme_rdma_init_request
,
1795 .exit_request
= nvme_rdma_exit_request
,
1796 .init_hctx
= nvme_rdma_init_admin_hctx
,
1797 .timeout
= nvme_rdma_timeout
,
1800 static void nvme_rdma_shutdown_ctrl(struct nvme_rdma_ctrl
*ctrl
, bool shutdown
)
1802 nvme_rdma_teardown_io_queues(ctrl
, shutdown
);
1804 nvme_shutdown_ctrl(&ctrl
->ctrl
);
1806 nvme_disable_ctrl(&ctrl
->ctrl
, ctrl
->ctrl
.cap
);
1807 nvme_rdma_teardown_admin_queue(ctrl
, shutdown
);
1810 static void nvme_rdma_delete_ctrl(struct nvme_ctrl
*ctrl
)
1812 nvme_rdma_shutdown_ctrl(to_rdma_ctrl(ctrl
), true);
1815 static void nvme_rdma_reset_ctrl_work(struct work_struct
*work
)
1817 struct nvme_rdma_ctrl
*ctrl
=
1818 container_of(work
, struct nvme_rdma_ctrl
, ctrl
.reset_work
);
1820 nvme_stop_ctrl(&ctrl
->ctrl
);
1821 nvme_rdma_shutdown_ctrl(ctrl
, false);
1823 if (!nvme_change_ctrl_state(&ctrl
->ctrl
, NVME_CTRL_CONNECTING
)) {
1824 /* state change failure should never happen */
1829 if (nvme_rdma_setup_ctrl(ctrl
, false))
1835 ++ctrl
->ctrl
.nr_reconnects
;
1836 nvme_rdma_reconnect_or_remove(ctrl
);
1839 static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops
= {
1841 .module
= THIS_MODULE
,
1842 .flags
= NVME_F_FABRICS
,
1843 .reg_read32
= nvmf_reg_read32
,
1844 .reg_read64
= nvmf_reg_read64
,
1845 .reg_write32
= nvmf_reg_write32
,
1846 .free_ctrl
= nvme_rdma_free_ctrl
,
1847 .submit_async_event
= nvme_rdma_submit_async_event
,
1848 .delete_ctrl
= nvme_rdma_delete_ctrl
,
1849 .get_address
= nvmf_get_address
,
1850 .stop_ctrl
= nvme_rdma_stop_ctrl
,
1854 __nvme_rdma_options_match(struct nvme_rdma_ctrl
*ctrl
,
1855 struct nvmf_ctrl_options
*opts
)
1857 char *stdport
= __stringify(NVME_RDMA_IP_PORT
);
1860 if (!nvmf_ctlr_matches_baseopts(&ctrl
->ctrl
, opts
) ||
1861 strcmp(opts
->traddr
, ctrl
->ctrl
.opts
->traddr
))
1864 if (opts
->mask
& NVMF_OPT_TRSVCID
&&
1865 ctrl
->ctrl
.opts
->mask
& NVMF_OPT_TRSVCID
) {
1866 if (strcmp(opts
->trsvcid
, ctrl
->ctrl
.opts
->trsvcid
))
1868 } else if (opts
->mask
& NVMF_OPT_TRSVCID
) {
1869 if (strcmp(opts
->trsvcid
, stdport
))
1871 } else if (ctrl
->ctrl
.opts
->mask
& NVMF_OPT_TRSVCID
) {
1872 if (strcmp(stdport
, ctrl
->ctrl
.opts
->trsvcid
))
1875 /* else, it's a match as both have stdport. Fall to next checks */
1878 * checking the local address is rough. In most cases, one
1879 * is not specified and the host port is selected by the stack.
1881 * Assume no match if:
1882 * local address is specified and address is not the same
1883 * local address is not specified but remote is, or vice versa
1884 * (admin using specific host_traddr when it matters).
1886 if (opts
->mask
& NVMF_OPT_HOST_TRADDR
&&
1887 ctrl
->ctrl
.opts
->mask
& NVMF_OPT_HOST_TRADDR
) {
1888 if (strcmp(opts
->host_traddr
, ctrl
->ctrl
.opts
->host_traddr
))
1890 } else if (opts
->mask
& NVMF_OPT_HOST_TRADDR
||
1891 ctrl
->ctrl
.opts
->mask
& NVMF_OPT_HOST_TRADDR
)
1894 * if neither controller had an host port specified, assume it's
1895 * a match as everything else matched.
1902 * Fails a connection request if it matches an existing controller
1903 * (association) with the same tuple:
1904 * <Host NQN, Host ID, local address, remote address, remote port, SUBSYS NQN>
1906 * if local address is not specified in the request, it will match an
1907 * existing controller with all the other parameters the same and no
1908 * local port address specified as well.
1910 * The ports don't need to be compared as they are intrinsically
1911 * already matched by the port pointers supplied.
1914 nvme_rdma_existing_controller(struct nvmf_ctrl_options
*opts
)
1916 struct nvme_rdma_ctrl
*ctrl
;
1919 mutex_lock(&nvme_rdma_ctrl_mutex
);
1920 list_for_each_entry(ctrl
, &nvme_rdma_ctrl_list
, list
) {
1921 found
= __nvme_rdma_options_match(ctrl
, opts
);
1925 mutex_unlock(&nvme_rdma_ctrl_mutex
);
1930 static struct nvme_ctrl
*nvme_rdma_create_ctrl(struct device
*dev
,
1931 struct nvmf_ctrl_options
*opts
)
1933 struct nvme_rdma_ctrl
*ctrl
;
1938 ctrl
= kzalloc(sizeof(*ctrl
), GFP_KERNEL
);
1940 return ERR_PTR(-ENOMEM
);
1941 ctrl
->ctrl
.opts
= opts
;
1942 INIT_LIST_HEAD(&ctrl
->list
);
1944 if (opts
->mask
& NVMF_OPT_TRSVCID
)
1945 port
= opts
->trsvcid
;
1947 port
= __stringify(NVME_RDMA_IP_PORT
);
1949 ret
= inet_pton_with_scope(&init_net
, AF_UNSPEC
,
1950 opts
->traddr
, port
, &ctrl
->addr
);
1952 pr_err("malformed address passed: %s:%s\n", opts
->traddr
, port
);
1956 if (opts
->mask
& NVMF_OPT_HOST_TRADDR
) {
1957 ret
= inet_pton_with_scope(&init_net
, AF_UNSPEC
,
1958 opts
->host_traddr
, NULL
, &ctrl
->src_addr
);
1960 pr_err("malformed src address passed: %s\n",
1966 if (!opts
->duplicate_connect
&& nvme_rdma_existing_controller(opts
)) {
1971 INIT_DELAYED_WORK(&ctrl
->reconnect_work
,
1972 nvme_rdma_reconnect_ctrl_work
);
1973 INIT_WORK(&ctrl
->err_work
, nvme_rdma_error_recovery_work
);
1974 INIT_WORK(&ctrl
->ctrl
.reset_work
, nvme_rdma_reset_ctrl_work
);
1976 ctrl
->ctrl
.queue_count
= opts
->nr_io_queues
+ 1; /* +1 for admin queue */
1977 ctrl
->ctrl
.sqsize
= opts
->queue_size
- 1;
1978 ctrl
->ctrl
.kato
= opts
->kato
;
1981 ctrl
->queues
= kcalloc(ctrl
->ctrl
.queue_count
, sizeof(*ctrl
->queues
),
1986 ret
= nvme_init_ctrl(&ctrl
->ctrl
, dev
, &nvme_rdma_ctrl_ops
,
1987 0 /* no quirks, we're perfect! */);
1989 goto out_kfree_queues
;
1991 changed
= nvme_change_ctrl_state(&ctrl
->ctrl
, NVME_CTRL_CONNECTING
);
1992 WARN_ON_ONCE(!changed
);
1994 ret
= nvme_rdma_setup_ctrl(ctrl
, true);
1996 goto out_uninit_ctrl
;
1998 dev_info(ctrl
->ctrl
.device
, "new ctrl: NQN \"%s\", addr %pISpcs\n",
1999 ctrl
->ctrl
.opts
->subsysnqn
, &ctrl
->addr
);
2001 nvme_get_ctrl(&ctrl
->ctrl
);
2003 mutex_lock(&nvme_rdma_ctrl_mutex
);
2004 list_add_tail(&ctrl
->list
, &nvme_rdma_ctrl_list
);
2005 mutex_unlock(&nvme_rdma_ctrl_mutex
);
2010 nvme_uninit_ctrl(&ctrl
->ctrl
);
2011 nvme_put_ctrl(&ctrl
->ctrl
);
2014 return ERR_PTR(ret
);
2016 kfree(ctrl
->queues
);
2019 return ERR_PTR(ret
);
2022 static struct nvmf_transport_ops nvme_rdma_transport
= {
2024 .module
= THIS_MODULE
,
2025 .required_opts
= NVMF_OPT_TRADDR
,
2026 .allowed_opts
= NVMF_OPT_TRSVCID
| NVMF_OPT_RECONNECT_DELAY
|
2027 NVMF_OPT_HOST_TRADDR
| NVMF_OPT_CTRL_LOSS_TMO
,
2028 .create_ctrl
= nvme_rdma_create_ctrl
,
2031 static void nvme_rdma_remove_one(struct ib_device
*ib_device
, void *client_data
)
2033 struct nvme_rdma_ctrl
*ctrl
;
2034 struct nvme_rdma_device
*ndev
;
2037 mutex_lock(&device_list_mutex
);
2038 list_for_each_entry(ndev
, &device_list
, entry
) {
2039 if (ndev
->dev
== ib_device
) {
2044 mutex_unlock(&device_list_mutex
);
2049 /* Delete all controllers using this device */
2050 mutex_lock(&nvme_rdma_ctrl_mutex
);
2051 list_for_each_entry(ctrl
, &nvme_rdma_ctrl_list
, list
) {
2052 if (ctrl
->device
->dev
!= ib_device
)
2054 nvme_delete_ctrl(&ctrl
->ctrl
);
2056 mutex_unlock(&nvme_rdma_ctrl_mutex
);
2058 flush_workqueue(nvme_delete_wq
);
2061 static struct ib_client nvme_rdma_ib_client
= {
2062 .name
= "nvme_rdma",
2063 .remove
= nvme_rdma_remove_one
2066 static int __init
nvme_rdma_init_module(void)
2070 ret
= ib_register_client(&nvme_rdma_ib_client
);
2074 ret
= nvmf_register_transport(&nvme_rdma_transport
);
2076 goto err_unreg_client
;
2081 ib_unregister_client(&nvme_rdma_ib_client
);
2085 static void __exit
nvme_rdma_cleanup_module(void)
2087 nvmf_unregister_transport(&nvme_rdma_transport
);
2088 ib_unregister_client(&nvme_rdma_ib_client
);
2091 module_init(nvme_rdma_init_module
);
2092 module_exit(nvme_rdma_cleanup_module
);
2094 MODULE_LICENSE("GPL v2");