1 // SPDX-License-Identifier: GPL-2.0
3 * NVMe over Fabrics RDMA target.
4 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 #include <linux/atomic.h>
8 #include <linux/ctype.h>
9 #include <linux/delay.h>
10 #include <linux/err.h>
11 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/nvme.h>
14 #include <linux/slab.h>
15 #include <linux/string.h>
16 #include <linux/wait.h>
17 #include <linux/inet.h>
18 #include <asm/unaligned.h>
20 #include <rdma/ib_verbs.h>
21 #include <rdma/rdma_cm.h>
24 #include <linux/nvme-rdma.h>
28 * We allow at least 1 page, up to 4 SGEs, and up to 16KB of inline data
30 #define NVMET_RDMA_DEFAULT_INLINE_DATA_SIZE PAGE_SIZE
31 #define NVMET_RDMA_MAX_INLINE_SGE 4
32 #define NVMET_RDMA_MAX_INLINE_DATA_SIZE max_t(int, SZ_16K, PAGE_SIZE)
34 struct nvmet_rdma_cmd
{
35 struct ib_sge sge
[NVMET_RDMA_MAX_INLINE_SGE
+ 1];
38 struct scatterlist inline_sg
[NVMET_RDMA_MAX_INLINE_SGE
];
39 struct nvme_command
*nvme_cmd
;
40 struct nvmet_rdma_queue
*queue
;
44 NVMET_RDMA_REQ_INLINE_DATA
= (1 << 0),
45 NVMET_RDMA_REQ_INVALIDATE_RKEY
= (1 << 1),
48 struct nvmet_rdma_rsp
{
49 struct ib_sge send_sge
;
50 struct ib_cqe send_cqe
;
51 struct ib_send_wr send_wr
;
53 struct nvmet_rdma_cmd
*cmd
;
54 struct nvmet_rdma_queue
*queue
;
56 struct ib_cqe read_cqe
;
57 struct rdma_rw_ctx rw
;
66 struct list_head wait_list
;
67 struct list_head free_list
;
70 enum nvmet_rdma_queue_state
{
71 NVMET_RDMA_Q_CONNECTING
,
73 NVMET_RDMA_Q_DISCONNECTING
,
76 struct nvmet_rdma_queue
{
77 struct rdma_cm_id
*cm_id
;
78 struct nvmet_port
*port
;
81 struct nvmet_rdma_device
*dev
;
82 spinlock_t state_lock
;
83 enum nvmet_rdma_queue_state state
;
84 struct nvmet_cq nvme_cq
;
85 struct nvmet_sq nvme_sq
;
87 struct nvmet_rdma_rsp
*rsps
;
88 struct list_head free_rsps
;
90 struct nvmet_rdma_cmd
*cmds
;
92 struct work_struct release_work
;
93 struct list_head rsp_wait_list
;
94 struct list_head rsp_wr_wait_list
;
95 spinlock_t rsp_wr_wait_lock
;
102 struct list_head queue_list
;
105 struct nvmet_rdma_device
{
106 struct ib_device
*device
;
109 struct nvmet_rdma_cmd
*srq_cmds
;
112 struct list_head entry
;
113 int inline_data_size
;
114 int inline_page_count
;
117 static bool nvmet_rdma_use_srq
;
118 module_param_named(use_srq
, nvmet_rdma_use_srq
, bool, 0444);
119 MODULE_PARM_DESC(use_srq
, "Use shared receive queue.");
121 static DEFINE_IDA(nvmet_rdma_queue_ida
);
122 static LIST_HEAD(nvmet_rdma_queue_list
);
123 static DEFINE_MUTEX(nvmet_rdma_queue_mutex
);
125 static LIST_HEAD(device_list
);
126 static DEFINE_MUTEX(device_list_mutex
);
128 static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp
*rsp
);
129 static void nvmet_rdma_send_done(struct ib_cq
*cq
, struct ib_wc
*wc
);
130 static void nvmet_rdma_recv_done(struct ib_cq
*cq
, struct ib_wc
*wc
);
131 static void nvmet_rdma_read_data_done(struct ib_cq
*cq
, struct ib_wc
*wc
);
132 static void nvmet_rdma_qp_event(struct ib_event
*event
, void *priv
);
133 static void nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue
*queue
);
134 static void nvmet_rdma_free_rsp(struct nvmet_rdma_device
*ndev
,
135 struct nvmet_rdma_rsp
*r
);
136 static int nvmet_rdma_alloc_rsp(struct nvmet_rdma_device
*ndev
,
137 struct nvmet_rdma_rsp
*r
);
139 static const struct nvmet_fabrics_ops nvmet_rdma_ops
;
141 static int num_pages(int len
)
143 return 1 + (((len
- 1) & PAGE_MASK
) >> PAGE_SHIFT
);
146 /* XXX: really should move to a generic header sooner or later.. */
147 static inline u32
get_unaligned_le24(const u8
*p
)
149 return (u32
)p
[0] | (u32
)p
[1] << 8 | (u32
)p
[2] << 16;
152 static inline bool nvmet_rdma_need_data_in(struct nvmet_rdma_rsp
*rsp
)
154 return nvme_is_write(rsp
->req
.cmd
) &&
155 rsp
->req
.transfer_len
&&
156 !(rsp
->flags
& NVMET_RDMA_REQ_INLINE_DATA
);
159 static inline bool nvmet_rdma_need_data_out(struct nvmet_rdma_rsp
*rsp
)
161 return !nvme_is_write(rsp
->req
.cmd
) &&
162 rsp
->req
.transfer_len
&&
163 !rsp
->req
.rsp
->status
&&
164 !(rsp
->flags
& NVMET_RDMA_REQ_INLINE_DATA
);
167 static inline struct nvmet_rdma_rsp
*
168 nvmet_rdma_get_rsp(struct nvmet_rdma_queue
*queue
)
170 struct nvmet_rdma_rsp
*rsp
;
173 spin_lock_irqsave(&queue
->rsps_lock
, flags
);
174 rsp
= list_first_entry_or_null(&queue
->free_rsps
,
175 struct nvmet_rdma_rsp
, free_list
);
177 list_del(&rsp
->free_list
);
178 spin_unlock_irqrestore(&queue
->rsps_lock
, flags
);
180 if (unlikely(!rsp
)) {
183 rsp
= kzalloc(sizeof(*rsp
), GFP_KERNEL
);
186 ret
= nvmet_rdma_alloc_rsp(queue
->dev
, rsp
);
192 rsp
->allocated
= true;
199 nvmet_rdma_put_rsp(struct nvmet_rdma_rsp
*rsp
)
203 if (unlikely(rsp
->allocated
)) {
204 nvmet_rdma_free_rsp(rsp
->queue
->dev
, rsp
);
209 spin_lock_irqsave(&rsp
->queue
->rsps_lock
, flags
);
210 list_add_tail(&rsp
->free_list
, &rsp
->queue
->free_rsps
);
211 spin_unlock_irqrestore(&rsp
->queue
->rsps_lock
, flags
);
214 static void nvmet_rdma_free_inline_pages(struct nvmet_rdma_device
*ndev
,
215 struct nvmet_rdma_cmd
*c
)
217 struct scatterlist
*sg
;
221 if (!ndev
->inline_data_size
)
227 for (i
= 0; i
< ndev
->inline_page_count
; i
++, sg
++, sge
++) {
229 ib_dma_unmap_page(ndev
->device
, sge
->addr
,
230 sge
->length
, DMA_FROM_DEVICE
);
232 __free_page(sg_page(sg
));
236 static int nvmet_rdma_alloc_inline_pages(struct nvmet_rdma_device
*ndev
,
237 struct nvmet_rdma_cmd
*c
)
239 struct scatterlist
*sg
;
245 if (!ndev
->inline_data_size
)
249 sg_init_table(sg
, ndev
->inline_page_count
);
251 len
= ndev
->inline_data_size
;
253 for (i
= 0; i
< ndev
->inline_page_count
; i
++, sg
++, sge
++) {
254 pg
= alloc_page(GFP_KERNEL
);
257 sg_assign_page(sg
, pg
);
258 sge
->addr
= ib_dma_map_page(ndev
->device
,
259 pg
, 0, PAGE_SIZE
, DMA_FROM_DEVICE
);
260 if (ib_dma_mapping_error(ndev
->device
, sge
->addr
))
262 sge
->length
= min_t(int, len
, PAGE_SIZE
);
263 sge
->lkey
= ndev
->pd
->local_dma_lkey
;
269 for (; i
>= 0; i
--, sg
--, sge
--) {
271 ib_dma_unmap_page(ndev
->device
, sge
->addr
,
272 sge
->length
, DMA_FROM_DEVICE
);
274 __free_page(sg_page(sg
));
279 static int nvmet_rdma_alloc_cmd(struct nvmet_rdma_device
*ndev
,
280 struct nvmet_rdma_cmd
*c
, bool admin
)
282 /* NVMe command / RDMA RECV */
283 c
->nvme_cmd
= kmalloc(sizeof(*c
->nvme_cmd
), GFP_KERNEL
);
287 c
->sge
[0].addr
= ib_dma_map_single(ndev
->device
, c
->nvme_cmd
,
288 sizeof(*c
->nvme_cmd
), DMA_FROM_DEVICE
);
289 if (ib_dma_mapping_error(ndev
->device
, c
->sge
[0].addr
))
292 c
->sge
[0].length
= sizeof(*c
->nvme_cmd
);
293 c
->sge
[0].lkey
= ndev
->pd
->local_dma_lkey
;
295 if (!admin
&& nvmet_rdma_alloc_inline_pages(ndev
, c
))
298 c
->cqe
.done
= nvmet_rdma_recv_done
;
300 c
->wr
.wr_cqe
= &c
->cqe
;
301 c
->wr
.sg_list
= c
->sge
;
302 c
->wr
.num_sge
= admin
? 1 : ndev
->inline_page_count
+ 1;
307 ib_dma_unmap_single(ndev
->device
, c
->sge
[0].addr
,
308 sizeof(*c
->nvme_cmd
), DMA_FROM_DEVICE
);
316 static void nvmet_rdma_free_cmd(struct nvmet_rdma_device
*ndev
,
317 struct nvmet_rdma_cmd
*c
, bool admin
)
320 nvmet_rdma_free_inline_pages(ndev
, c
);
321 ib_dma_unmap_single(ndev
->device
, c
->sge
[0].addr
,
322 sizeof(*c
->nvme_cmd
), DMA_FROM_DEVICE
);
326 static struct nvmet_rdma_cmd
*
327 nvmet_rdma_alloc_cmds(struct nvmet_rdma_device
*ndev
,
328 int nr_cmds
, bool admin
)
330 struct nvmet_rdma_cmd
*cmds
;
331 int ret
= -EINVAL
, i
;
333 cmds
= kcalloc(nr_cmds
, sizeof(struct nvmet_rdma_cmd
), GFP_KERNEL
);
337 for (i
= 0; i
< nr_cmds
; i
++) {
338 ret
= nvmet_rdma_alloc_cmd(ndev
, cmds
+ i
, admin
);
347 nvmet_rdma_free_cmd(ndev
, cmds
+ i
, admin
);
353 static void nvmet_rdma_free_cmds(struct nvmet_rdma_device
*ndev
,
354 struct nvmet_rdma_cmd
*cmds
, int nr_cmds
, bool admin
)
358 for (i
= 0; i
< nr_cmds
; i
++)
359 nvmet_rdma_free_cmd(ndev
, cmds
+ i
, admin
);
363 static int nvmet_rdma_alloc_rsp(struct nvmet_rdma_device
*ndev
,
364 struct nvmet_rdma_rsp
*r
)
366 /* NVMe CQE / RDMA SEND */
367 r
->req
.rsp
= kmalloc(sizeof(*r
->req
.rsp
), GFP_KERNEL
);
371 r
->send_sge
.addr
= ib_dma_map_single(ndev
->device
, r
->req
.rsp
,
372 sizeof(*r
->req
.rsp
), DMA_TO_DEVICE
);
373 if (ib_dma_mapping_error(ndev
->device
, r
->send_sge
.addr
))
376 r
->send_sge
.length
= sizeof(*r
->req
.rsp
);
377 r
->send_sge
.lkey
= ndev
->pd
->local_dma_lkey
;
379 r
->send_cqe
.done
= nvmet_rdma_send_done
;
381 r
->send_wr
.wr_cqe
= &r
->send_cqe
;
382 r
->send_wr
.sg_list
= &r
->send_sge
;
383 r
->send_wr
.num_sge
= 1;
384 r
->send_wr
.send_flags
= IB_SEND_SIGNALED
;
386 /* Data In / RDMA READ */
387 r
->read_cqe
.done
= nvmet_rdma_read_data_done
;
396 static void nvmet_rdma_free_rsp(struct nvmet_rdma_device
*ndev
,
397 struct nvmet_rdma_rsp
*r
)
399 ib_dma_unmap_single(ndev
->device
, r
->send_sge
.addr
,
400 sizeof(*r
->req
.rsp
), DMA_TO_DEVICE
);
405 nvmet_rdma_alloc_rsps(struct nvmet_rdma_queue
*queue
)
407 struct nvmet_rdma_device
*ndev
= queue
->dev
;
408 int nr_rsps
= queue
->recv_queue_size
* 2;
409 int ret
= -EINVAL
, i
;
411 queue
->rsps
= kcalloc(nr_rsps
, sizeof(struct nvmet_rdma_rsp
),
416 for (i
= 0; i
< nr_rsps
; i
++) {
417 struct nvmet_rdma_rsp
*rsp
= &queue
->rsps
[i
];
419 ret
= nvmet_rdma_alloc_rsp(ndev
, rsp
);
423 list_add_tail(&rsp
->free_list
, &queue
->free_rsps
);
430 struct nvmet_rdma_rsp
*rsp
= &queue
->rsps
[i
];
432 list_del(&rsp
->free_list
);
433 nvmet_rdma_free_rsp(ndev
, rsp
);
440 static void nvmet_rdma_free_rsps(struct nvmet_rdma_queue
*queue
)
442 struct nvmet_rdma_device
*ndev
= queue
->dev
;
443 int i
, nr_rsps
= queue
->recv_queue_size
* 2;
445 for (i
= 0; i
< nr_rsps
; i
++) {
446 struct nvmet_rdma_rsp
*rsp
= &queue
->rsps
[i
];
448 list_del(&rsp
->free_list
);
449 nvmet_rdma_free_rsp(ndev
, rsp
);
454 static int nvmet_rdma_post_recv(struct nvmet_rdma_device
*ndev
,
455 struct nvmet_rdma_cmd
*cmd
)
459 ib_dma_sync_single_for_device(ndev
->device
,
460 cmd
->sge
[0].addr
, cmd
->sge
[0].length
,
464 ret
= ib_post_srq_recv(ndev
->srq
, &cmd
->wr
, NULL
);
466 ret
= ib_post_recv(cmd
->queue
->cm_id
->qp
, &cmd
->wr
, NULL
);
469 pr_err("post_recv cmd failed\n");
474 static void nvmet_rdma_process_wr_wait_list(struct nvmet_rdma_queue
*queue
)
476 spin_lock(&queue
->rsp_wr_wait_lock
);
477 while (!list_empty(&queue
->rsp_wr_wait_list
)) {
478 struct nvmet_rdma_rsp
*rsp
;
481 rsp
= list_entry(queue
->rsp_wr_wait_list
.next
,
482 struct nvmet_rdma_rsp
, wait_list
);
483 list_del(&rsp
->wait_list
);
485 spin_unlock(&queue
->rsp_wr_wait_lock
);
486 ret
= nvmet_rdma_execute_command(rsp
);
487 spin_lock(&queue
->rsp_wr_wait_lock
);
490 list_add(&rsp
->wait_list
, &queue
->rsp_wr_wait_list
);
494 spin_unlock(&queue
->rsp_wr_wait_lock
);
498 static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp
*rsp
)
500 struct nvmet_rdma_queue
*queue
= rsp
->queue
;
502 atomic_add(1 + rsp
->n_rdma
, &queue
->sq_wr_avail
);
505 rdma_rw_ctx_destroy(&rsp
->rw
, queue
->cm_id
->qp
,
506 queue
->cm_id
->port_num
, rsp
->req
.sg
,
507 rsp
->req
.sg_cnt
, nvmet_data_dir(&rsp
->req
));
510 if (rsp
->req
.sg
!= rsp
->cmd
->inline_sg
)
511 nvmet_req_free_sgl(&rsp
->req
);
513 if (unlikely(!list_empty_careful(&queue
->rsp_wr_wait_list
)))
514 nvmet_rdma_process_wr_wait_list(queue
);
516 nvmet_rdma_put_rsp(rsp
);
519 static void nvmet_rdma_error_comp(struct nvmet_rdma_queue
*queue
)
521 if (queue
->nvme_sq
.ctrl
) {
522 nvmet_ctrl_fatal_error(queue
->nvme_sq
.ctrl
);
525 * we didn't setup the controller yet in case
526 * of admin connect error, just disconnect and
529 nvmet_rdma_queue_disconnect(queue
);
533 static void nvmet_rdma_send_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
535 struct nvmet_rdma_rsp
*rsp
=
536 container_of(wc
->wr_cqe
, struct nvmet_rdma_rsp
, send_cqe
);
537 struct nvmet_rdma_queue
*queue
= cq
->cq_context
;
539 nvmet_rdma_release_rsp(rsp
);
541 if (unlikely(wc
->status
!= IB_WC_SUCCESS
&&
542 wc
->status
!= IB_WC_WR_FLUSH_ERR
)) {
543 pr_err("SEND for CQE 0x%p failed with status %s (%d).\n",
544 wc
->wr_cqe
, ib_wc_status_msg(wc
->status
), wc
->status
);
545 nvmet_rdma_error_comp(queue
);
549 static void nvmet_rdma_queue_response(struct nvmet_req
*req
)
551 struct nvmet_rdma_rsp
*rsp
=
552 container_of(req
, struct nvmet_rdma_rsp
, req
);
553 struct rdma_cm_id
*cm_id
= rsp
->queue
->cm_id
;
554 struct ib_send_wr
*first_wr
;
556 if (rsp
->flags
& NVMET_RDMA_REQ_INVALIDATE_RKEY
) {
557 rsp
->send_wr
.opcode
= IB_WR_SEND_WITH_INV
;
558 rsp
->send_wr
.ex
.invalidate_rkey
= rsp
->invalidate_rkey
;
560 rsp
->send_wr
.opcode
= IB_WR_SEND
;
563 if (nvmet_rdma_need_data_out(rsp
))
564 first_wr
= rdma_rw_ctx_wrs(&rsp
->rw
, cm_id
->qp
,
565 cm_id
->port_num
, NULL
, &rsp
->send_wr
);
567 first_wr
= &rsp
->send_wr
;
569 nvmet_rdma_post_recv(rsp
->queue
->dev
, rsp
->cmd
);
571 ib_dma_sync_single_for_device(rsp
->queue
->dev
->device
,
572 rsp
->send_sge
.addr
, rsp
->send_sge
.length
,
575 if (unlikely(ib_post_send(cm_id
->qp
, first_wr
, NULL
))) {
576 pr_err("sending cmd response failed\n");
577 nvmet_rdma_release_rsp(rsp
);
581 static void nvmet_rdma_read_data_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
583 struct nvmet_rdma_rsp
*rsp
=
584 container_of(wc
->wr_cqe
, struct nvmet_rdma_rsp
, read_cqe
);
585 struct nvmet_rdma_queue
*queue
= cq
->cq_context
;
587 WARN_ON(rsp
->n_rdma
<= 0);
588 atomic_add(rsp
->n_rdma
, &queue
->sq_wr_avail
);
589 rdma_rw_ctx_destroy(&rsp
->rw
, queue
->cm_id
->qp
,
590 queue
->cm_id
->port_num
, rsp
->req
.sg
,
591 rsp
->req
.sg_cnt
, nvmet_data_dir(&rsp
->req
));
594 if (unlikely(wc
->status
!= IB_WC_SUCCESS
)) {
595 nvmet_req_uninit(&rsp
->req
);
596 nvmet_rdma_release_rsp(rsp
);
597 if (wc
->status
!= IB_WC_WR_FLUSH_ERR
) {
598 pr_info("RDMA READ for CQE 0x%p failed with status %s (%d).\n",
599 wc
->wr_cqe
, ib_wc_status_msg(wc
->status
), wc
->status
);
600 nvmet_rdma_error_comp(queue
);
605 nvmet_req_execute(&rsp
->req
);
608 static void nvmet_rdma_use_inline_sg(struct nvmet_rdma_rsp
*rsp
, u32 len
,
611 int sg_count
= num_pages(len
);
612 struct scatterlist
*sg
;
615 sg
= rsp
->cmd
->inline_sg
;
616 for (i
= 0; i
< sg_count
; i
++, sg
++) {
617 if (i
< sg_count
- 1)
622 sg
->length
= min_t(int, len
, PAGE_SIZE
- off
);
628 rsp
->req
.sg
= rsp
->cmd
->inline_sg
;
629 rsp
->req
.sg_cnt
= sg_count
;
632 static u16
nvmet_rdma_map_sgl_inline(struct nvmet_rdma_rsp
*rsp
)
634 struct nvme_sgl_desc
*sgl
= &rsp
->req
.cmd
->common
.dptr
.sgl
;
635 u64 off
= le64_to_cpu(sgl
->addr
);
636 u32 len
= le32_to_cpu(sgl
->length
);
638 if (!nvme_is_write(rsp
->req
.cmd
)) {
640 offsetof(struct nvme_common_command
, opcode
);
641 return NVME_SC_INVALID_FIELD
| NVME_SC_DNR
;
644 if (off
+ len
> rsp
->queue
->dev
->inline_data_size
) {
645 pr_err("invalid inline data offset!\n");
646 return NVME_SC_SGL_INVALID_OFFSET
| NVME_SC_DNR
;
649 /* no data command? */
653 nvmet_rdma_use_inline_sg(rsp
, len
, off
);
654 rsp
->flags
|= NVMET_RDMA_REQ_INLINE_DATA
;
655 rsp
->req
.transfer_len
+= len
;
659 static u16
nvmet_rdma_map_sgl_keyed(struct nvmet_rdma_rsp
*rsp
,
660 struct nvme_keyed_sgl_desc
*sgl
, bool invalidate
)
662 struct rdma_cm_id
*cm_id
= rsp
->queue
->cm_id
;
663 u64 addr
= le64_to_cpu(sgl
->addr
);
664 u32 key
= get_unaligned_le32(sgl
->key
);
667 rsp
->req
.transfer_len
= get_unaligned_le24(sgl
->length
);
669 /* no data command? */
670 if (!rsp
->req
.transfer_len
)
673 ret
= nvmet_req_alloc_sgl(&rsp
->req
);
677 ret
= rdma_rw_ctx_init(&rsp
->rw
, cm_id
->qp
, cm_id
->port_num
,
678 rsp
->req
.sg
, rsp
->req
.sg_cnt
, 0, addr
, key
,
679 nvmet_data_dir(&rsp
->req
));
685 rsp
->invalidate_rkey
= key
;
686 rsp
->flags
|= NVMET_RDMA_REQ_INVALIDATE_RKEY
;
692 rsp
->req
.transfer_len
= 0;
693 return NVME_SC_INTERNAL
;
696 static u16
nvmet_rdma_map_sgl(struct nvmet_rdma_rsp
*rsp
)
698 struct nvme_keyed_sgl_desc
*sgl
= &rsp
->req
.cmd
->common
.dptr
.ksgl
;
700 switch (sgl
->type
>> 4) {
701 case NVME_SGL_FMT_DATA_DESC
:
702 switch (sgl
->type
& 0xf) {
703 case NVME_SGL_FMT_OFFSET
:
704 return nvmet_rdma_map_sgl_inline(rsp
);
706 pr_err("invalid SGL subtype: %#x\n", sgl
->type
);
708 offsetof(struct nvme_common_command
, dptr
);
709 return NVME_SC_INVALID_FIELD
| NVME_SC_DNR
;
711 case NVME_KEY_SGL_FMT_DATA_DESC
:
712 switch (sgl
->type
& 0xf) {
713 case NVME_SGL_FMT_ADDRESS
| NVME_SGL_FMT_INVALIDATE
:
714 return nvmet_rdma_map_sgl_keyed(rsp
, sgl
, true);
715 case NVME_SGL_FMT_ADDRESS
:
716 return nvmet_rdma_map_sgl_keyed(rsp
, sgl
, false);
718 pr_err("invalid SGL subtype: %#x\n", sgl
->type
);
720 offsetof(struct nvme_common_command
, dptr
);
721 return NVME_SC_INVALID_FIELD
| NVME_SC_DNR
;
724 pr_err("invalid SGL type: %#x\n", sgl
->type
);
725 rsp
->req
.error_loc
= offsetof(struct nvme_common_command
, dptr
);
726 return NVME_SC_SGL_INVALID_TYPE
| NVME_SC_DNR
;
730 static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp
*rsp
)
732 struct nvmet_rdma_queue
*queue
= rsp
->queue
;
734 if (unlikely(atomic_sub_return(1 + rsp
->n_rdma
,
735 &queue
->sq_wr_avail
) < 0)) {
736 pr_debug("IB send queue full (needed %d): queue %u cntlid %u\n",
737 1 + rsp
->n_rdma
, queue
->idx
,
738 queue
->nvme_sq
.ctrl
->cntlid
);
739 atomic_add(1 + rsp
->n_rdma
, &queue
->sq_wr_avail
);
743 if (nvmet_rdma_need_data_in(rsp
)) {
744 if (rdma_rw_ctx_post(&rsp
->rw
, queue
->cm_id
->qp
,
745 queue
->cm_id
->port_num
, &rsp
->read_cqe
, NULL
))
746 nvmet_req_complete(&rsp
->req
, NVME_SC_DATA_XFER_ERROR
);
748 nvmet_req_execute(&rsp
->req
);
754 static void nvmet_rdma_handle_command(struct nvmet_rdma_queue
*queue
,
755 struct nvmet_rdma_rsp
*cmd
)
759 ib_dma_sync_single_for_cpu(queue
->dev
->device
,
760 cmd
->cmd
->sge
[0].addr
, cmd
->cmd
->sge
[0].length
,
762 ib_dma_sync_single_for_cpu(queue
->dev
->device
,
763 cmd
->send_sge
.addr
, cmd
->send_sge
.length
,
766 cmd
->req
.p2p_client
= &queue
->dev
->device
->dev
;
768 if (!nvmet_req_init(&cmd
->req
, &queue
->nvme_cq
,
769 &queue
->nvme_sq
, &nvmet_rdma_ops
))
772 status
= nvmet_rdma_map_sgl(cmd
);
776 if (unlikely(!nvmet_rdma_execute_command(cmd
))) {
777 spin_lock(&queue
->rsp_wr_wait_lock
);
778 list_add_tail(&cmd
->wait_list
, &queue
->rsp_wr_wait_list
);
779 spin_unlock(&queue
->rsp_wr_wait_lock
);
785 nvmet_req_complete(&cmd
->req
, status
);
788 static void nvmet_rdma_recv_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
790 struct nvmet_rdma_cmd
*cmd
=
791 container_of(wc
->wr_cqe
, struct nvmet_rdma_cmd
, cqe
);
792 struct nvmet_rdma_queue
*queue
= cq
->cq_context
;
793 struct nvmet_rdma_rsp
*rsp
;
795 if (unlikely(wc
->status
!= IB_WC_SUCCESS
)) {
796 if (wc
->status
!= IB_WC_WR_FLUSH_ERR
) {
797 pr_err("RECV for CQE 0x%p failed with status %s (%d)\n",
798 wc
->wr_cqe
, ib_wc_status_msg(wc
->status
),
800 nvmet_rdma_error_comp(queue
);
805 if (unlikely(wc
->byte_len
< sizeof(struct nvme_command
))) {
806 pr_err("Ctrl Fatal Error: capsule size less than 64 bytes\n");
807 nvmet_rdma_error_comp(queue
);
812 rsp
= nvmet_rdma_get_rsp(queue
);
813 if (unlikely(!rsp
)) {
815 * we get here only under memory pressure,
816 * silently drop and have the host retry
817 * as we can't even fail it.
819 nvmet_rdma_post_recv(queue
->dev
, cmd
);
825 rsp
->req
.cmd
= cmd
->nvme_cmd
;
826 rsp
->req
.port
= queue
->port
;
829 if (unlikely(queue
->state
!= NVMET_RDMA_Q_LIVE
)) {
832 spin_lock_irqsave(&queue
->state_lock
, flags
);
833 if (queue
->state
== NVMET_RDMA_Q_CONNECTING
)
834 list_add_tail(&rsp
->wait_list
, &queue
->rsp_wait_list
);
836 nvmet_rdma_put_rsp(rsp
);
837 spin_unlock_irqrestore(&queue
->state_lock
, flags
);
841 nvmet_rdma_handle_command(queue
, rsp
);
844 static void nvmet_rdma_destroy_srq(struct nvmet_rdma_device
*ndev
)
849 nvmet_rdma_free_cmds(ndev
, ndev
->srq_cmds
, ndev
->srq_size
, false);
850 ib_destroy_srq(ndev
->srq
);
853 static int nvmet_rdma_init_srq(struct nvmet_rdma_device
*ndev
)
855 struct ib_srq_init_attr srq_attr
= { NULL
, };
860 srq_size
= 4095; /* XXX: tune */
862 srq_attr
.attr
.max_wr
= srq_size
;
863 srq_attr
.attr
.max_sge
= 1 + ndev
->inline_page_count
;
864 srq_attr
.attr
.srq_limit
= 0;
865 srq_attr
.srq_type
= IB_SRQT_BASIC
;
866 srq
= ib_create_srq(ndev
->pd
, &srq_attr
);
869 * If SRQs aren't supported we just go ahead and use normal
870 * non-shared receive queues.
872 pr_info("SRQ requested but not supported.\n");
876 ndev
->srq_cmds
= nvmet_rdma_alloc_cmds(ndev
, srq_size
, false);
877 if (IS_ERR(ndev
->srq_cmds
)) {
878 ret
= PTR_ERR(ndev
->srq_cmds
);
879 goto out_destroy_srq
;
883 ndev
->srq_size
= srq_size
;
885 for (i
= 0; i
< srq_size
; i
++) {
886 ret
= nvmet_rdma_post_recv(ndev
, &ndev
->srq_cmds
[i
]);
894 nvmet_rdma_free_cmds(ndev
, ndev
->srq_cmds
, ndev
->srq_size
, false);
900 static void nvmet_rdma_free_dev(struct kref
*ref
)
902 struct nvmet_rdma_device
*ndev
=
903 container_of(ref
, struct nvmet_rdma_device
, ref
);
905 mutex_lock(&device_list_mutex
);
906 list_del(&ndev
->entry
);
907 mutex_unlock(&device_list_mutex
);
909 nvmet_rdma_destroy_srq(ndev
);
910 ib_dealloc_pd(ndev
->pd
);
915 static struct nvmet_rdma_device
*
916 nvmet_rdma_find_get_device(struct rdma_cm_id
*cm_id
)
918 struct nvmet_port
*port
= cm_id
->context
;
919 struct nvmet_rdma_device
*ndev
;
920 int inline_page_count
;
921 int inline_sge_count
;
924 mutex_lock(&device_list_mutex
);
925 list_for_each_entry(ndev
, &device_list
, entry
) {
926 if (ndev
->device
->node_guid
== cm_id
->device
->node_guid
&&
927 kref_get_unless_zero(&ndev
->ref
))
931 ndev
= kzalloc(sizeof(*ndev
), GFP_KERNEL
);
935 inline_page_count
= num_pages(port
->inline_data_size
);
936 inline_sge_count
= max(cm_id
->device
->attrs
.max_sge_rd
,
937 cm_id
->device
->attrs
.max_recv_sge
) - 1;
938 if (inline_page_count
> inline_sge_count
) {
939 pr_warn("inline_data_size %d cannot be supported by device %s. Reducing to %lu.\n",
940 port
->inline_data_size
, cm_id
->device
->name
,
941 inline_sge_count
* PAGE_SIZE
);
942 port
->inline_data_size
= inline_sge_count
* PAGE_SIZE
;
943 inline_page_count
= inline_sge_count
;
945 ndev
->inline_data_size
= port
->inline_data_size
;
946 ndev
->inline_page_count
= inline_page_count
;
947 ndev
->device
= cm_id
->device
;
948 kref_init(&ndev
->ref
);
950 ndev
->pd
= ib_alloc_pd(ndev
->device
, 0);
951 if (IS_ERR(ndev
->pd
))
954 if (nvmet_rdma_use_srq
) {
955 ret
= nvmet_rdma_init_srq(ndev
);
960 list_add(&ndev
->entry
, &device_list
);
962 mutex_unlock(&device_list_mutex
);
963 pr_debug("added %s.\n", ndev
->device
->name
);
967 ib_dealloc_pd(ndev
->pd
);
971 mutex_unlock(&device_list_mutex
);
975 static int nvmet_rdma_create_queue_ib(struct nvmet_rdma_queue
*queue
)
977 struct ib_qp_init_attr qp_attr
;
978 struct nvmet_rdma_device
*ndev
= queue
->dev
;
979 int comp_vector
, nr_cqe
, ret
, i
;
982 * Spread the io queues across completion vectors,
983 * but still keep all admin queues on vector 0.
985 comp_vector
= !queue
->host_qid
? 0 :
986 queue
->idx
% ndev
->device
->num_comp_vectors
;
989 * Reserve CQ slots for RECV + RDMA_READ/RDMA_WRITE + RDMA_SEND.
991 nr_cqe
= queue
->recv_queue_size
+ 2 * queue
->send_queue_size
;
993 queue
->cq
= ib_alloc_cq(ndev
->device
, queue
,
994 nr_cqe
+ 1, comp_vector
,
996 if (IS_ERR(queue
->cq
)) {
997 ret
= PTR_ERR(queue
->cq
);
998 pr_err("failed to create CQ cqe= %d ret= %d\n",
1003 memset(&qp_attr
, 0, sizeof(qp_attr
));
1004 qp_attr
.qp_context
= queue
;
1005 qp_attr
.event_handler
= nvmet_rdma_qp_event
;
1006 qp_attr
.send_cq
= queue
->cq
;
1007 qp_attr
.recv_cq
= queue
->cq
;
1008 qp_attr
.sq_sig_type
= IB_SIGNAL_REQ_WR
;
1009 qp_attr
.qp_type
= IB_QPT_RC
;
1011 qp_attr
.cap
.max_send_wr
= queue
->send_queue_size
+ 1;
1012 qp_attr
.cap
.max_rdma_ctxs
= queue
->send_queue_size
;
1013 qp_attr
.cap
.max_send_sge
= max(ndev
->device
->attrs
.max_sge_rd
,
1014 ndev
->device
->attrs
.max_send_sge
);
1017 qp_attr
.srq
= ndev
->srq
;
1020 qp_attr
.cap
.max_recv_wr
= 1 + queue
->recv_queue_size
;
1021 qp_attr
.cap
.max_recv_sge
= 1 + ndev
->inline_page_count
;
1024 ret
= rdma_create_qp(queue
->cm_id
, ndev
->pd
, &qp_attr
);
1026 pr_err("failed to create_qp ret= %d\n", ret
);
1027 goto err_destroy_cq
;
1030 atomic_set(&queue
->sq_wr_avail
, qp_attr
.cap
.max_send_wr
);
1032 pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d cm_id= %p\n",
1033 __func__
, queue
->cq
->cqe
, qp_attr
.cap
.max_send_sge
,
1034 qp_attr
.cap
.max_send_wr
, queue
->cm_id
);
1037 for (i
= 0; i
< queue
->recv_queue_size
; i
++) {
1038 queue
->cmds
[i
].queue
= queue
;
1039 ret
= nvmet_rdma_post_recv(ndev
, &queue
->cmds
[i
]);
1041 goto err_destroy_qp
;
1049 rdma_destroy_qp(queue
->cm_id
);
1051 ib_free_cq(queue
->cq
);
1055 static void nvmet_rdma_destroy_queue_ib(struct nvmet_rdma_queue
*queue
)
1057 struct ib_qp
*qp
= queue
->cm_id
->qp
;
1060 rdma_destroy_id(queue
->cm_id
);
1062 ib_free_cq(queue
->cq
);
1065 static void nvmet_rdma_free_queue(struct nvmet_rdma_queue
*queue
)
1067 pr_debug("freeing queue %d\n", queue
->idx
);
1069 nvmet_sq_destroy(&queue
->nvme_sq
);
1071 nvmet_rdma_destroy_queue_ib(queue
);
1072 if (!queue
->dev
->srq
) {
1073 nvmet_rdma_free_cmds(queue
->dev
, queue
->cmds
,
1074 queue
->recv_queue_size
,
1077 nvmet_rdma_free_rsps(queue
);
1078 ida_simple_remove(&nvmet_rdma_queue_ida
, queue
->idx
);
1082 static void nvmet_rdma_release_queue_work(struct work_struct
*w
)
1084 struct nvmet_rdma_queue
*queue
=
1085 container_of(w
, struct nvmet_rdma_queue
, release_work
);
1086 struct nvmet_rdma_device
*dev
= queue
->dev
;
1088 nvmet_rdma_free_queue(queue
);
1090 kref_put(&dev
->ref
, nvmet_rdma_free_dev
);
1094 nvmet_rdma_parse_cm_connect_req(struct rdma_conn_param
*conn
,
1095 struct nvmet_rdma_queue
*queue
)
1097 struct nvme_rdma_cm_req
*req
;
1099 req
= (struct nvme_rdma_cm_req
*)conn
->private_data
;
1100 if (!req
|| conn
->private_data_len
== 0)
1101 return NVME_RDMA_CM_INVALID_LEN
;
1103 if (le16_to_cpu(req
->recfmt
) != NVME_RDMA_CM_FMT_1_0
)
1104 return NVME_RDMA_CM_INVALID_RECFMT
;
1106 queue
->host_qid
= le16_to_cpu(req
->qid
);
1109 * req->hsqsize corresponds to our recv queue size plus 1
1110 * req->hrqsize corresponds to our send queue size
1112 queue
->recv_queue_size
= le16_to_cpu(req
->hsqsize
) + 1;
1113 queue
->send_queue_size
= le16_to_cpu(req
->hrqsize
);
1115 if (!queue
->host_qid
&& queue
->recv_queue_size
> NVME_AQ_DEPTH
)
1116 return NVME_RDMA_CM_INVALID_HSQSIZE
;
1118 /* XXX: Should we enforce some kind of max for IO queues? */
1123 static int nvmet_rdma_cm_reject(struct rdma_cm_id
*cm_id
,
1124 enum nvme_rdma_cm_status status
)
1126 struct nvme_rdma_cm_rej rej
;
1128 pr_debug("rejecting connect request: status %d (%s)\n",
1129 status
, nvme_rdma_cm_msg(status
));
1131 rej
.recfmt
= cpu_to_le16(NVME_RDMA_CM_FMT_1_0
);
1132 rej
.sts
= cpu_to_le16(status
);
1134 return rdma_reject(cm_id
, (void *)&rej
, sizeof(rej
));
1137 static struct nvmet_rdma_queue
*
1138 nvmet_rdma_alloc_queue(struct nvmet_rdma_device
*ndev
,
1139 struct rdma_cm_id
*cm_id
,
1140 struct rdma_cm_event
*event
)
1142 struct nvmet_rdma_queue
*queue
;
1145 queue
= kzalloc(sizeof(*queue
), GFP_KERNEL
);
1147 ret
= NVME_RDMA_CM_NO_RSC
;
1151 ret
= nvmet_sq_init(&queue
->nvme_sq
);
1153 ret
= NVME_RDMA_CM_NO_RSC
;
1154 goto out_free_queue
;
1157 ret
= nvmet_rdma_parse_cm_connect_req(&event
->param
.conn
, queue
);
1159 goto out_destroy_sq
;
1162 * Schedules the actual release because calling rdma_destroy_id from
1163 * inside a CM callback would trigger a deadlock. (great API design..)
1165 INIT_WORK(&queue
->release_work
, nvmet_rdma_release_queue_work
);
1167 queue
->cm_id
= cm_id
;
1169 spin_lock_init(&queue
->state_lock
);
1170 queue
->state
= NVMET_RDMA_Q_CONNECTING
;
1171 INIT_LIST_HEAD(&queue
->rsp_wait_list
);
1172 INIT_LIST_HEAD(&queue
->rsp_wr_wait_list
);
1173 spin_lock_init(&queue
->rsp_wr_wait_lock
);
1174 INIT_LIST_HEAD(&queue
->free_rsps
);
1175 spin_lock_init(&queue
->rsps_lock
);
1176 INIT_LIST_HEAD(&queue
->queue_list
);
1178 queue
->idx
= ida_simple_get(&nvmet_rdma_queue_ida
, 0, 0, GFP_KERNEL
);
1179 if (queue
->idx
< 0) {
1180 ret
= NVME_RDMA_CM_NO_RSC
;
1181 goto out_destroy_sq
;
1184 ret
= nvmet_rdma_alloc_rsps(queue
);
1186 ret
= NVME_RDMA_CM_NO_RSC
;
1187 goto out_ida_remove
;
1191 queue
->cmds
= nvmet_rdma_alloc_cmds(ndev
,
1192 queue
->recv_queue_size
,
1194 if (IS_ERR(queue
->cmds
)) {
1195 ret
= NVME_RDMA_CM_NO_RSC
;
1196 goto out_free_responses
;
1200 ret
= nvmet_rdma_create_queue_ib(queue
);
1202 pr_err("%s: creating RDMA queue failed (%d).\n",
1204 ret
= NVME_RDMA_CM_NO_RSC
;
1212 nvmet_rdma_free_cmds(queue
->dev
, queue
->cmds
,
1213 queue
->recv_queue_size
,
1217 nvmet_rdma_free_rsps(queue
);
1219 ida_simple_remove(&nvmet_rdma_queue_ida
, queue
->idx
);
1221 nvmet_sq_destroy(&queue
->nvme_sq
);
1225 nvmet_rdma_cm_reject(cm_id
, ret
);
1229 static void nvmet_rdma_qp_event(struct ib_event
*event
, void *priv
)
1231 struct nvmet_rdma_queue
*queue
= priv
;
1233 switch (event
->event
) {
1234 case IB_EVENT_COMM_EST
:
1235 rdma_notify(queue
->cm_id
, event
->event
);
1238 pr_err("received IB QP event: %s (%d)\n",
1239 ib_event_msg(event
->event
), event
->event
);
1244 static int nvmet_rdma_cm_accept(struct rdma_cm_id
*cm_id
,
1245 struct nvmet_rdma_queue
*queue
,
1246 struct rdma_conn_param
*p
)
1248 struct rdma_conn_param param
= { };
1249 struct nvme_rdma_cm_rep priv
= { };
1252 param
.rnr_retry_count
= 7;
1253 param
.flow_control
= 1;
1254 param
.initiator_depth
= min_t(u8
, p
->initiator_depth
,
1255 queue
->dev
->device
->attrs
.max_qp_init_rd_atom
);
1256 param
.private_data
= &priv
;
1257 param
.private_data_len
= sizeof(priv
);
1258 priv
.recfmt
= cpu_to_le16(NVME_RDMA_CM_FMT_1_0
);
1259 priv
.crqsize
= cpu_to_le16(queue
->recv_queue_size
);
1261 ret
= rdma_accept(cm_id
, ¶m
);
1263 pr_err("rdma_accept failed (error code = %d)\n", ret
);
1268 static int nvmet_rdma_queue_connect(struct rdma_cm_id
*cm_id
,
1269 struct rdma_cm_event
*event
)
1271 struct nvmet_rdma_device
*ndev
;
1272 struct nvmet_rdma_queue
*queue
;
1275 ndev
= nvmet_rdma_find_get_device(cm_id
);
1277 nvmet_rdma_cm_reject(cm_id
, NVME_RDMA_CM_NO_RSC
);
1278 return -ECONNREFUSED
;
1281 queue
= nvmet_rdma_alloc_queue(ndev
, cm_id
, event
);
1286 queue
->port
= cm_id
->context
;
1288 if (queue
->host_qid
== 0) {
1289 /* Let inflight controller teardown complete */
1290 flush_scheduled_work();
1293 ret
= nvmet_rdma_cm_accept(cm_id
, queue
, &event
->param
.conn
);
1295 schedule_work(&queue
->release_work
);
1296 /* Destroying rdma_cm id is not needed here */
1300 mutex_lock(&nvmet_rdma_queue_mutex
);
1301 list_add_tail(&queue
->queue_list
, &nvmet_rdma_queue_list
);
1302 mutex_unlock(&nvmet_rdma_queue_mutex
);
1307 kref_put(&ndev
->ref
, nvmet_rdma_free_dev
);
1312 static void nvmet_rdma_queue_established(struct nvmet_rdma_queue
*queue
)
1314 unsigned long flags
;
1316 spin_lock_irqsave(&queue
->state_lock
, flags
);
1317 if (queue
->state
!= NVMET_RDMA_Q_CONNECTING
) {
1318 pr_warn("trying to establish a connected queue\n");
1321 queue
->state
= NVMET_RDMA_Q_LIVE
;
1323 while (!list_empty(&queue
->rsp_wait_list
)) {
1324 struct nvmet_rdma_rsp
*cmd
;
1326 cmd
= list_first_entry(&queue
->rsp_wait_list
,
1327 struct nvmet_rdma_rsp
, wait_list
);
1328 list_del(&cmd
->wait_list
);
1330 spin_unlock_irqrestore(&queue
->state_lock
, flags
);
1331 nvmet_rdma_handle_command(queue
, cmd
);
1332 spin_lock_irqsave(&queue
->state_lock
, flags
);
1336 spin_unlock_irqrestore(&queue
->state_lock
, flags
);
1339 static void __nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue
*queue
)
1341 bool disconnect
= false;
1342 unsigned long flags
;
1344 pr_debug("cm_id= %p queue->state= %d\n", queue
->cm_id
, queue
->state
);
1346 spin_lock_irqsave(&queue
->state_lock
, flags
);
1347 switch (queue
->state
) {
1348 case NVMET_RDMA_Q_CONNECTING
:
1349 case NVMET_RDMA_Q_LIVE
:
1350 queue
->state
= NVMET_RDMA_Q_DISCONNECTING
;
1353 case NVMET_RDMA_Q_DISCONNECTING
:
1356 spin_unlock_irqrestore(&queue
->state_lock
, flags
);
1359 rdma_disconnect(queue
->cm_id
);
1360 schedule_work(&queue
->release_work
);
1364 static void nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue
*queue
)
1366 bool disconnect
= false;
1368 mutex_lock(&nvmet_rdma_queue_mutex
);
1369 if (!list_empty(&queue
->queue_list
)) {
1370 list_del_init(&queue
->queue_list
);
1373 mutex_unlock(&nvmet_rdma_queue_mutex
);
1376 __nvmet_rdma_queue_disconnect(queue
);
1379 static void nvmet_rdma_queue_connect_fail(struct rdma_cm_id
*cm_id
,
1380 struct nvmet_rdma_queue
*queue
)
1382 WARN_ON_ONCE(queue
->state
!= NVMET_RDMA_Q_CONNECTING
);
1384 mutex_lock(&nvmet_rdma_queue_mutex
);
1385 if (!list_empty(&queue
->queue_list
))
1386 list_del_init(&queue
->queue_list
);
1387 mutex_unlock(&nvmet_rdma_queue_mutex
);
1389 pr_err("failed to connect queue %d\n", queue
->idx
);
1390 schedule_work(&queue
->release_work
);
1394 * nvme_rdma_device_removal() - Handle RDMA device removal
1395 * @cm_id: rdma_cm id, used for nvmet port
1396 * @queue: nvmet rdma queue (cm id qp_context)
1398 * DEVICE_REMOVAL event notifies us that the RDMA device is about
1399 * to unplug. Note that this event can be generated on a normal
1400 * queue cm_id and/or a device bound listener cm_id (where in this
1401 * case queue will be null).
1403 * We registered an ib_client to handle device removal for queues,
1404 * so we only need to handle the listening port cm_ids. In this case
1405 * we nullify the priv to prevent double cm_id destruction and destroying
1406 * the cm_id implicitely by returning a non-zero rc to the callout.
1408 static int nvmet_rdma_device_removal(struct rdma_cm_id
*cm_id
,
1409 struct nvmet_rdma_queue
*queue
)
1411 struct nvmet_port
*port
;
1415 * This is a queue cm_id. we have registered
1416 * an ib_client to handle queues removal
1417 * so don't interfear and just return.
1422 port
= cm_id
->context
;
1425 * This is a listener cm_id. Make sure that
1426 * future remove_port won't invoke a double
1427 * cm_id destroy. use atomic xchg to make sure
1428 * we don't compete with remove_port.
1430 if (xchg(&port
->priv
, NULL
) != cm_id
)
1434 * We need to return 1 so that the core will destroy
1435 * it's own ID. What a great API design..
1440 static int nvmet_rdma_cm_handler(struct rdma_cm_id
*cm_id
,
1441 struct rdma_cm_event
*event
)
1443 struct nvmet_rdma_queue
*queue
= NULL
;
1447 queue
= cm_id
->qp
->qp_context
;
1449 pr_debug("%s (%d): status %d id %p\n",
1450 rdma_event_msg(event
->event
), event
->event
,
1451 event
->status
, cm_id
);
1453 switch (event
->event
) {
1454 case RDMA_CM_EVENT_CONNECT_REQUEST
:
1455 ret
= nvmet_rdma_queue_connect(cm_id
, event
);
1457 case RDMA_CM_EVENT_ESTABLISHED
:
1458 nvmet_rdma_queue_established(queue
);
1460 case RDMA_CM_EVENT_ADDR_CHANGE
:
1461 case RDMA_CM_EVENT_DISCONNECTED
:
1462 case RDMA_CM_EVENT_TIMEWAIT_EXIT
:
1463 nvmet_rdma_queue_disconnect(queue
);
1465 case RDMA_CM_EVENT_DEVICE_REMOVAL
:
1466 ret
= nvmet_rdma_device_removal(cm_id
, queue
);
1468 case RDMA_CM_EVENT_REJECTED
:
1469 pr_debug("Connection rejected: %s\n",
1470 rdma_reject_msg(cm_id
, event
->status
));
1472 case RDMA_CM_EVENT_UNREACHABLE
:
1473 case RDMA_CM_EVENT_CONNECT_ERROR
:
1474 nvmet_rdma_queue_connect_fail(cm_id
, queue
);
1477 pr_err("received unrecognized RDMA CM event %d\n",
1485 static void nvmet_rdma_delete_ctrl(struct nvmet_ctrl
*ctrl
)
1487 struct nvmet_rdma_queue
*queue
;
1490 mutex_lock(&nvmet_rdma_queue_mutex
);
1491 list_for_each_entry(queue
, &nvmet_rdma_queue_list
, queue_list
) {
1492 if (queue
->nvme_sq
.ctrl
== ctrl
) {
1493 list_del_init(&queue
->queue_list
);
1494 mutex_unlock(&nvmet_rdma_queue_mutex
);
1496 __nvmet_rdma_queue_disconnect(queue
);
1500 mutex_unlock(&nvmet_rdma_queue_mutex
);
1503 static int nvmet_rdma_add_port(struct nvmet_port
*port
)
1505 struct rdma_cm_id
*cm_id
;
1506 struct sockaddr_storage addr
= { };
1507 __kernel_sa_family_t af
;
1510 switch (port
->disc_addr
.adrfam
) {
1511 case NVMF_ADDR_FAMILY_IP4
:
1514 case NVMF_ADDR_FAMILY_IP6
:
1518 pr_err("address family %d not supported\n",
1519 port
->disc_addr
.adrfam
);
1523 if (port
->inline_data_size
< 0) {
1524 port
->inline_data_size
= NVMET_RDMA_DEFAULT_INLINE_DATA_SIZE
;
1525 } else if (port
->inline_data_size
> NVMET_RDMA_MAX_INLINE_DATA_SIZE
) {
1526 pr_warn("inline_data_size %u is too large, reducing to %u\n",
1527 port
->inline_data_size
,
1528 NVMET_RDMA_MAX_INLINE_DATA_SIZE
);
1529 port
->inline_data_size
= NVMET_RDMA_MAX_INLINE_DATA_SIZE
;
1532 ret
= inet_pton_with_scope(&init_net
, af
, port
->disc_addr
.traddr
,
1533 port
->disc_addr
.trsvcid
, &addr
);
1535 pr_err("malformed ip/port passed: %s:%s\n",
1536 port
->disc_addr
.traddr
, port
->disc_addr
.trsvcid
);
1540 cm_id
= rdma_create_id(&init_net
, nvmet_rdma_cm_handler
, port
,
1541 RDMA_PS_TCP
, IB_QPT_RC
);
1542 if (IS_ERR(cm_id
)) {
1543 pr_err("CM ID creation failed\n");
1544 return PTR_ERR(cm_id
);
1548 * Allow both IPv4 and IPv6 sockets to bind a single port
1551 ret
= rdma_set_afonly(cm_id
, 1);
1553 pr_err("rdma_set_afonly failed (%d)\n", ret
);
1554 goto out_destroy_id
;
1557 ret
= rdma_bind_addr(cm_id
, (struct sockaddr
*)&addr
);
1559 pr_err("binding CM ID to %pISpcs failed (%d)\n",
1560 (struct sockaddr
*)&addr
, ret
);
1561 goto out_destroy_id
;
1564 ret
= rdma_listen(cm_id
, 128);
1566 pr_err("listening to %pISpcs failed (%d)\n",
1567 (struct sockaddr
*)&addr
, ret
);
1568 goto out_destroy_id
;
1571 pr_info("enabling port %d (%pISpcs)\n",
1572 le16_to_cpu(port
->disc_addr
.portid
), (struct sockaddr
*)&addr
);
1577 rdma_destroy_id(cm_id
);
1581 static void nvmet_rdma_remove_port(struct nvmet_port
*port
)
1583 struct rdma_cm_id
*cm_id
= xchg(&port
->priv
, NULL
);
1586 rdma_destroy_id(cm_id
);
1589 static void nvmet_rdma_disc_port_addr(struct nvmet_req
*req
,
1590 struct nvmet_port
*port
, char *traddr
)
1592 struct rdma_cm_id
*cm_id
= port
->priv
;
1594 if (inet_addr_is_any((struct sockaddr
*)&cm_id
->route
.addr
.src_addr
)) {
1595 struct nvmet_rdma_rsp
*rsp
=
1596 container_of(req
, struct nvmet_rdma_rsp
, req
);
1597 struct rdma_cm_id
*req_cm_id
= rsp
->queue
->cm_id
;
1598 struct sockaddr
*addr
= (void *)&req_cm_id
->route
.addr
.src_addr
;
1600 sprintf(traddr
, "%pISc", addr
);
1602 memcpy(traddr
, port
->disc_addr
.traddr
, NVMF_TRADDR_SIZE
);
1606 static const struct nvmet_fabrics_ops nvmet_rdma_ops
= {
1607 .owner
= THIS_MODULE
,
1608 .type
= NVMF_TRTYPE_RDMA
,
1610 .has_keyed_sgls
= 1,
1611 .add_port
= nvmet_rdma_add_port
,
1612 .remove_port
= nvmet_rdma_remove_port
,
1613 .queue_response
= nvmet_rdma_queue_response
,
1614 .delete_ctrl
= nvmet_rdma_delete_ctrl
,
1615 .disc_traddr
= nvmet_rdma_disc_port_addr
,
1618 static void nvmet_rdma_remove_one(struct ib_device
*ib_device
, void *client_data
)
1620 struct nvmet_rdma_queue
*queue
, *tmp
;
1621 struct nvmet_rdma_device
*ndev
;
1624 mutex_lock(&device_list_mutex
);
1625 list_for_each_entry(ndev
, &device_list
, entry
) {
1626 if (ndev
->device
== ib_device
) {
1631 mutex_unlock(&device_list_mutex
);
1637 * IB Device that is used by nvmet controllers is being removed,
1638 * delete all queues using this device.
1640 mutex_lock(&nvmet_rdma_queue_mutex
);
1641 list_for_each_entry_safe(queue
, tmp
, &nvmet_rdma_queue_list
,
1643 if (queue
->dev
->device
!= ib_device
)
1646 pr_info("Removing queue %d\n", queue
->idx
);
1647 list_del_init(&queue
->queue_list
);
1648 __nvmet_rdma_queue_disconnect(queue
);
1650 mutex_unlock(&nvmet_rdma_queue_mutex
);
1652 flush_scheduled_work();
1655 static struct ib_client nvmet_rdma_ib_client
= {
1656 .name
= "nvmet_rdma",
1657 .remove
= nvmet_rdma_remove_one
1660 static int __init
nvmet_rdma_init(void)
1664 ret
= ib_register_client(&nvmet_rdma_ib_client
);
1668 ret
= nvmet_register_transport(&nvmet_rdma_ops
);
1675 ib_unregister_client(&nvmet_rdma_ib_client
);
1679 static void __exit
nvmet_rdma_exit(void)
1681 nvmet_unregister_transport(&nvmet_rdma_ops
);
1682 ib_unregister_client(&nvmet_rdma_ib_client
);
1683 WARN_ON_ONCE(!list_empty(&nvmet_rdma_queue_list
));
1684 ida_destroy(&nvmet_rdma_queue_ida
);
1687 module_init(nvmet_rdma_init
);
1688 module_exit(nvmet_rdma_exit
);
1690 MODULE_LICENSE("GPL v2");
1691 MODULE_ALIAS("nvmet-transport-1"); /* 1 == NVMF_TRTYPE_RDMA */