2 * NVMe over Fabrics RDMA target.
3 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
15 #include <linux/atomic.h>
16 #include <linux/ctype.h>
17 #include <linux/delay.h>
18 #include <linux/err.h>
19 #include <linux/init.h>
20 #include <linux/module.h>
21 #include <linux/nvme.h>
22 #include <linux/slab.h>
23 #include <linux/string.h>
24 #include <linux/wait.h>
25 #include <linux/inet.h>
26 #include <asm/unaligned.h>
28 #include <rdma/ib_verbs.h>
29 #include <rdma/rdma_cm.h>
32 #include <linux/nvme-rdma.h>
36 * We allow up to a page of inline data to go with the SQE
38 #define NVMET_RDMA_INLINE_DATA_SIZE PAGE_SIZE
40 struct nvmet_rdma_cmd
{
44 struct scatterlist inline_sg
;
45 struct page
*inline_page
;
46 struct nvme_command
*nvme_cmd
;
47 struct nvmet_rdma_queue
*queue
;
51 NVMET_RDMA_REQ_INLINE_DATA
= (1 << 0),
52 NVMET_RDMA_REQ_INVALIDATE_RKEY
= (1 << 1),
55 struct nvmet_rdma_rsp
{
56 struct ib_sge send_sge
;
57 struct ib_cqe send_cqe
;
58 struct ib_send_wr send_wr
;
60 struct nvmet_rdma_cmd
*cmd
;
61 struct nvmet_rdma_queue
*queue
;
63 struct ib_cqe read_cqe
;
64 struct rdma_rw_ctx rw
;
72 struct list_head wait_list
;
73 struct list_head free_list
;
76 enum nvmet_rdma_queue_state
{
77 NVMET_RDMA_Q_CONNECTING
,
79 NVMET_RDMA_Q_DISCONNECTING
,
80 NVMET_RDMA_IN_DEVICE_REMOVAL
,
83 struct nvmet_rdma_queue
{
84 struct rdma_cm_id
*cm_id
;
85 struct nvmet_port
*port
;
88 struct nvmet_rdma_device
*dev
;
89 spinlock_t state_lock
;
90 enum nvmet_rdma_queue_state state
;
91 struct nvmet_cq nvme_cq
;
92 struct nvmet_sq nvme_sq
;
94 struct nvmet_rdma_rsp
*rsps
;
95 struct list_head free_rsps
;
97 struct nvmet_rdma_cmd
*cmds
;
99 struct work_struct release_work
;
100 struct list_head rsp_wait_list
;
101 struct list_head rsp_wr_wait_list
;
102 spinlock_t rsp_wr_wait_lock
;
109 struct list_head queue_list
;
112 struct nvmet_rdma_device
{
113 struct ib_device
*device
;
116 struct nvmet_rdma_cmd
*srq_cmds
;
119 struct list_head entry
;
122 static bool nvmet_rdma_use_srq
;
123 module_param_named(use_srq
, nvmet_rdma_use_srq
, bool, 0444);
124 MODULE_PARM_DESC(use_srq
, "Use shared receive queue.");
126 static DEFINE_IDA(nvmet_rdma_queue_ida
);
127 static LIST_HEAD(nvmet_rdma_queue_list
);
128 static DEFINE_MUTEX(nvmet_rdma_queue_mutex
);
130 static LIST_HEAD(device_list
);
131 static DEFINE_MUTEX(device_list_mutex
);
133 static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp
*rsp
);
134 static void nvmet_rdma_send_done(struct ib_cq
*cq
, struct ib_wc
*wc
);
135 static void nvmet_rdma_recv_done(struct ib_cq
*cq
, struct ib_wc
*wc
);
136 static void nvmet_rdma_read_data_done(struct ib_cq
*cq
, struct ib_wc
*wc
);
137 static void nvmet_rdma_qp_event(struct ib_event
*event
, void *priv
);
138 static void nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue
*queue
);
140 static struct nvmet_fabrics_ops nvmet_rdma_ops
;
142 /* XXX: really should move to a generic header sooner or later.. */
143 static inline u32
get_unaligned_le24(const u8
*p
)
145 return (u32
)p
[0] | (u32
)p
[1] << 8 | (u32
)p
[2] << 16;
148 static inline bool nvmet_rdma_need_data_in(struct nvmet_rdma_rsp
*rsp
)
150 return nvme_is_write(rsp
->req
.cmd
) &&
152 !(rsp
->flags
& NVMET_RDMA_REQ_INLINE_DATA
);
155 static inline bool nvmet_rdma_need_data_out(struct nvmet_rdma_rsp
*rsp
)
157 return !nvme_is_write(rsp
->req
.cmd
) &&
159 !rsp
->req
.rsp
->status
&&
160 !(rsp
->flags
& NVMET_RDMA_REQ_INLINE_DATA
);
163 static inline struct nvmet_rdma_rsp
*
164 nvmet_rdma_get_rsp(struct nvmet_rdma_queue
*queue
)
166 struct nvmet_rdma_rsp
*rsp
;
169 spin_lock_irqsave(&queue
->rsps_lock
, flags
);
170 rsp
= list_first_entry(&queue
->free_rsps
,
171 struct nvmet_rdma_rsp
, free_list
);
172 list_del(&rsp
->free_list
);
173 spin_unlock_irqrestore(&queue
->rsps_lock
, flags
);
179 nvmet_rdma_put_rsp(struct nvmet_rdma_rsp
*rsp
)
183 spin_lock_irqsave(&rsp
->queue
->rsps_lock
, flags
);
184 list_add_tail(&rsp
->free_list
, &rsp
->queue
->free_rsps
);
185 spin_unlock_irqrestore(&rsp
->queue
->rsps_lock
, flags
);
188 static void nvmet_rdma_free_sgl(struct scatterlist
*sgl
, unsigned int nents
)
190 struct scatterlist
*sg
;
196 for_each_sg(sgl
, sg
, nents
, count
)
197 __free_page(sg_page(sg
));
201 static int nvmet_rdma_alloc_sgl(struct scatterlist
**sgl
, unsigned int *nents
,
204 struct scatterlist
*sg
;
209 nent
= DIV_ROUND_UP(length
, PAGE_SIZE
);
210 sg
= kmalloc_array(nent
, sizeof(struct scatterlist
), GFP_KERNEL
);
214 sg_init_table(sg
, nent
);
217 u32 page_len
= min_t(u32
, length
, PAGE_SIZE
);
219 page
= alloc_page(GFP_KERNEL
);
223 sg_set_page(&sg
[i
], page
, page_len
, 0);
234 __free_page(sg_page(&sg
[i
]));
238 return NVME_SC_INTERNAL
;
241 static int nvmet_rdma_alloc_cmd(struct nvmet_rdma_device
*ndev
,
242 struct nvmet_rdma_cmd
*c
, bool admin
)
244 /* NVMe command / RDMA RECV */
245 c
->nvme_cmd
= kmalloc(sizeof(*c
->nvme_cmd
), GFP_KERNEL
);
249 c
->sge
[0].addr
= ib_dma_map_single(ndev
->device
, c
->nvme_cmd
,
250 sizeof(*c
->nvme_cmd
), DMA_FROM_DEVICE
);
251 if (ib_dma_mapping_error(ndev
->device
, c
->sge
[0].addr
))
254 c
->sge
[0].length
= sizeof(*c
->nvme_cmd
);
255 c
->sge
[0].lkey
= ndev
->pd
->local_dma_lkey
;
258 c
->inline_page
= alloc_pages(GFP_KERNEL
,
259 get_order(NVMET_RDMA_INLINE_DATA_SIZE
));
262 c
->sge
[1].addr
= ib_dma_map_page(ndev
->device
,
263 c
->inline_page
, 0, NVMET_RDMA_INLINE_DATA_SIZE
,
265 if (ib_dma_mapping_error(ndev
->device
, c
->sge
[1].addr
))
266 goto out_free_inline_page
;
267 c
->sge
[1].length
= NVMET_RDMA_INLINE_DATA_SIZE
;
268 c
->sge
[1].lkey
= ndev
->pd
->local_dma_lkey
;
271 c
->cqe
.done
= nvmet_rdma_recv_done
;
273 c
->wr
.wr_cqe
= &c
->cqe
;
274 c
->wr
.sg_list
= c
->sge
;
275 c
->wr
.num_sge
= admin
? 1 : 2;
279 out_free_inline_page
:
281 __free_pages(c
->inline_page
,
282 get_order(NVMET_RDMA_INLINE_DATA_SIZE
));
285 ib_dma_unmap_single(ndev
->device
, c
->sge
[0].addr
,
286 sizeof(*c
->nvme_cmd
), DMA_FROM_DEVICE
);
294 static void nvmet_rdma_free_cmd(struct nvmet_rdma_device
*ndev
,
295 struct nvmet_rdma_cmd
*c
, bool admin
)
298 ib_dma_unmap_page(ndev
->device
, c
->sge
[1].addr
,
299 NVMET_RDMA_INLINE_DATA_SIZE
, DMA_FROM_DEVICE
);
300 __free_pages(c
->inline_page
,
301 get_order(NVMET_RDMA_INLINE_DATA_SIZE
));
303 ib_dma_unmap_single(ndev
->device
, c
->sge
[0].addr
,
304 sizeof(*c
->nvme_cmd
), DMA_FROM_DEVICE
);
308 static struct nvmet_rdma_cmd
*
309 nvmet_rdma_alloc_cmds(struct nvmet_rdma_device
*ndev
,
310 int nr_cmds
, bool admin
)
312 struct nvmet_rdma_cmd
*cmds
;
313 int ret
= -EINVAL
, i
;
315 cmds
= kcalloc(nr_cmds
, sizeof(struct nvmet_rdma_cmd
), GFP_KERNEL
);
319 for (i
= 0; i
< nr_cmds
; i
++) {
320 ret
= nvmet_rdma_alloc_cmd(ndev
, cmds
+ i
, admin
);
329 nvmet_rdma_free_cmd(ndev
, cmds
+ i
, admin
);
335 static void nvmet_rdma_free_cmds(struct nvmet_rdma_device
*ndev
,
336 struct nvmet_rdma_cmd
*cmds
, int nr_cmds
, bool admin
)
340 for (i
= 0; i
< nr_cmds
; i
++)
341 nvmet_rdma_free_cmd(ndev
, cmds
+ i
, admin
);
345 static int nvmet_rdma_alloc_rsp(struct nvmet_rdma_device
*ndev
,
346 struct nvmet_rdma_rsp
*r
)
348 /* NVMe CQE / RDMA SEND */
349 r
->req
.rsp
= kmalloc(sizeof(*r
->req
.rsp
), GFP_KERNEL
);
353 r
->send_sge
.addr
= ib_dma_map_single(ndev
->device
, r
->req
.rsp
,
354 sizeof(*r
->req
.rsp
), DMA_TO_DEVICE
);
355 if (ib_dma_mapping_error(ndev
->device
, r
->send_sge
.addr
))
358 r
->send_sge
.length
= sizeof(*r
->req
.rsp
);
359 r
->send_sge
.lkey
= ndev
->pd
->local_dma_lkey
;
361 r
->send_cqe
.done
= nvmet_rdma_send_done
;
363 r
->send_wr
.wr_cqe
= &r
->send_cqe
;
364 r
->send_wr
.sg_list
= &r
->send_sge
;
365 r
->send_wr
.num_sge
= 1;
366 r
->send_wr
.send_flags
= IB_SEND_SIGNALED
;
368 /* Data In / RDMA READ */
369 r
->read_cqe
.done
= nvmet_rdma_read_data_done
;
378 static void nvmet_rdma_free_rsp(struct nvmet_rdma_device
*ndev
,
379 struct nvmet_rdma_rsp
*r
)
381 ib_dma_unmap_single(ndev
->device
, r
->send_sge
.addr
,
382 sizeof(*r
->req
.rsp
), DMA_TO_DEVICE
);
387 nvmet_rdma_alloc_rsps(struct nvmet_rdma_queue
*queue
)
389 struct nvmet_rdma_device
*ndev
= queue
->dev
;
390 int nr_rsps
= queue
->recv_queue_size
* 2;
391 int ret
= -EINVAL
, i
;
393 queue
->rsps
= kcalloc(nr_rsps
, sizeof(struct nvmet_rdma_rsp
),
398 for (i
= 0; i
< nr_rsps
; i
++) {
399 struct nvmet_rdma_rsp
*rsp
= &queue
->rsps
[i
];
401 ret
= nvmet_rdma_alloc_rsp(ndev
, rsp
);
405 list_add_tail(&rsp
->free_list
, &queue
->free_rsps
);
412 struct nvmet_rdma_rsp
*rsp
= &queue
->rsps
[i
];
414 list_del(&rsp
->free_list
);
415 nvmet_rdma_free_rsp(ndev
, rsp
);
422 static void nvmet_rdma_free_rsps(struct nvmet_rdma_queue
*queue
)
424 struct nvmet_rdma_device
*ndev
= queue
->dev
;
425 int i
, nr_rsps
= queue
->recv_queue_size
* 2;
427 for (i
= 0; i
< nr_rsps
; i
++) {
428 struct nvmet_rdma_rsp
*rsp
= &queue
->rsps
[i
];
430 list_del(&rsp
->free_list
);
431 nvmet_rdma_free_rsp(ndev
, rsp
);
436 static int nvmet_rdma_post_recv(struct nvmet_rdma_device
*ndev
,
437 struct nvmet_rdma_cmd
*cmd
)
439 struct ib_recv_wr
*bad_wr
;
441 ib_dma_sync_single_for_device(ndev
->device
,
442 cmd
->sge
[0].addr
, cmd
->sge
[0].length
,
446 return ib_post_srq_recv(ndev
->srq
, &cmd
->wr
, &bad_wr
);
447 return ib_post_recv(cmd
->queue
->cm_id
->qp
, &cmd
->wr
, &bad_wr
);
450 static void nvmet_rdma_process_wr_wait_list(struct nvmet_rdma_queue
*queue
)
452 spin_lock(&queue
->rsp_wr_wait_lock
);
453 while (!list_empty(&queue
->rsp_wr_wait_list
)) {
454 struct nvmet_rdma_rsp
*rsp
;
457 rsp
= list_entry(queue
->rsp_wr_wait_list
.next
,
458 struct nvmet_rdma_rsp
, wait_list
);
459 list_del(&rsp
->wait_list
);
461 spin_unlock(&queue
->rsp_wr_wait_lock
);
462 ret
= nvmet_rdma_execute_command(rsp
);
463 spin_lock(&queue
->rsp_wr_wait_lock
);
466 list_add(&rsp
->wait_list
, &queue
->rsp_wr_wait_list
);
470 spin_unlock(&queue
->rsp_wr_wait_lock
);
474 static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp
*rsp
)
476 struct nvmet_rdma_queue
*queue
= rsp
->queue
;
478 atomic_add(1 + rsp
->n_rdma
, &queue
->sq_wr_avail
);
481 rdma_rw_ctx_destroy(&rsp
->rw
, queue
->cm_id
->qp
,
482 queue
->cm_id
->port_num
, rsp
->req
.sg
,
483 rsp
->req
.sg_cnt
, nvmet_data_dir(&rsp
->req
));
486 if (rsp
->req
.sg
!= &rsp
->cmd
->inline_sg
)
487 nvmet_rdma_free_sgl(rsp
->req
.sg
, rsp
->req
.sg_cnt
);
489 if (unlikely(!list_empty_careful(&queue
->rsp_wr_wait_list
)))
490 nvmet_rdma_process_wr_wait_list(queue
);
492 nvmet_rdma_put_rsp(rsp
);
495 static void nvmet_rdma_error_comp(struct nvmet_rdma_queue
*queue
)
497 if (queue
->nvme_sq
.ctrl
) {
498 nvmet_ctrl_fatal_error(queue
->nvme_sq
.ctrl
);
501 * we didn't setup the controller yet in case
502 * of admin connect error, just disconnect and
505 nvmet_rdma_queue_disconnect(queue
);
509 static void nvmet_rdma_send_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
511 struct nvmet_rdma_rsp
*rsp
=
512 container_of(wc
->wr_cqe
, struct nvmet_rdma_rsp
, send_cqe
);
514 nvmet_rdma_release_rsp(rsp
);
516 if (unlikely(wc
->status
!= IB_WC_SUCCESS
&&
517 wc
->status
!= IB_WC_WR_FLUSH_ERR
)) {
518 pr_err("SEND for CQE 0x%p failed with status %s (%d).\n",
519 wc
->wr_cqe
, ib_wc_status_msg(wc
->status
), wc
->status
);
520 nvmet_rdma_error_comp(rsp
->queue
);
524 static void nvmet_rdma_queue_response(struct nvmet_req
*req
)
526 struct nvmet_rdma_rsp
*rsp
=
527 container_of(req
, struct nvmet_rdma_rsp
, req
);
528 struct rdma_cm_id
*cm_id
= rsp
->queue
->cm_id
;
529 struct ib_send_wr
*first_wr
, *bad_wr
;
531 if (rsp
->flags
& NVMET_RDMA_REQ_INVALIDATE_RKEY
) {
532 rsp
->send_wr
.opcode
= IB_WR_SEND_WITH_INV
;
533 rsp
->send_wr
.ex
.invalidate_rkey
= rsp
->invalidate_rkey
;
535 rsp
->send_wr
.opcode
= IB_WR_SEND
;
538 if (nvmet_rdma_need_data_out(rsp
))
539 first_wr
= rdma_rw_ctx_wrs(&rsp
->rw
, cm_id
->qp
,
540 cm_id
->port_num
, NULL
, &rsp
->send_wr
);
542 first_wr
= &rsp
->send_wr
;
544 nvmet_rdma_post_recv(rsp
->queue
->dev
, rsp
->cmd
);
546 ib_dma_sync_single_for_device(rsp
->queue
->dev
->device
,
547 rsp
->send_sge
.addr
, rsp
->send_sge
.length
,
550 if (ib_post_send(cm_id
->qp
, first_wr
, &bad_wr
)) {
551 pr_err("sending cmd response failed\n");
552 nvmet_rdma_release_rsp(rsp
);
556 static void nvmet_rdma_read_data_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
558 struct nvmet_rdma_rsp
*rsp
=
559 container_of(wc
->wr_cqe
, struct nvmet_rdma_rsp
, read_cqe
);
560 struct nvmet_rdma_queue
*queue
= cq
->cq_context
;
562 WARN_ON(rsp
->n_rdma
<= 0);
563 atomic_add(rsp
->n_rdma
, &queue
->sq_wr_avail
);
564 rdma_rw_ctx_destroy(&rsp
->rw
, queue
->cm_id
->qp
,
565 queue
->cm_id
->port_num
, rsp
->req
.sg
,
566 rsp
->req
.sg_cnt
, nvmet_data_dir(&rsp
->req
));
569 if (unlikely(wc
->status
!= IB_WC_SUCCESS
)) {
570 nvmet_rdma_release_rsp(rsp
);
571 if (wc
->status
!= IB_WC_WR_FLUSH_ERR
) {
572 pr_info("RDMA READ for CQE 0x%p failed with status %s (%d).\n",
573 wc
->wr_cqe
, ib_wc_status_msg(wc
->status
), wc
->status
);
574 nvmet_rdma_error_comp(queue
);
579 rsp
->req
.execute(&rsp
->req
);
582 static void nvmet_rdma_use_inline_sg(struct nvmet_rdma_rsp
*rsp
, u32 len
,
585 sg_init_table(&rsp
->cmd
->inline_sg
, 1);
586 sg_set_page(&rsp
->cmd
->inline_sg
, rsp
->cmd
->inline_page
, len
, off
);
587 rsp
->req
.sg
= &rsp
->cmd
->inline_sg
;
591 static u16
nvmet_rdma_map_sgl_inline(struct nvmet_rdma_rsp
*rsp
)
593 struct nvme_sgl_desc
*sgl
= &rsp
->req
.cmd
->common
.dptr
.sgl
;
594 u64 off
= le64_to_cpu(sgl
->addr
);
595 u32 len
= le32_to_cpu(sgl
->length
);
597 if (!nvme_is_write(rsp
->req
.cmd
))
598 return NVME_SC_INVALID_FIELD
| NVME_SC_DNR
;
600 if (off
+ len
> NVMET_RDMA_INLINE_DATA_SIZE
) {
601 pr_err("invalid inline data offset!\n");
602 return NVME_SC_SGL_INVALID_OFFSET
| NVME_SC_DNR
;
605 /* no data command? */
609 nvmet_rdma_use_inline_sg(rsp
, len
, off
);
610 rsp
->flags
|= NVMET_RDMA_REQ_INLINE_DATA
;
614 static u16
nvmet_rdma_map_sgl_keyed(struct nvmet_rdma_rsp
*rsp
,
615 struct nvme_keyed_sgl_desc
*sgl
, bool invalidate
)
617 struct rdma_cm_id
*cm_id
= rsp
->queue
->cm_id
;
618 u64 addr
= le64_to_cpu(sgl
->addr
);
619 u32 len
= get_unaligned_le24(sgl
->length
);
620 u32 key
= get_unaligned_le32(sgl
->key
);
624 /* no data command? */
628 status
= nvmet_rdma_alloc_sgl(&rsp
->req
.sg
, &rsp
->req
.sg_cnt
,
633 ret
= rdma_rw_ctx_init(&rsp
->rw
, cm_id
->qp
, cm_id
->port_num
,
634 rsp
->req
.sg
, rsp
->req
.sg_cnt
, 0, addr
, key
,
635 nvmet_data_dir(&rsp
->req
));
637 return NVME_SC_INTERNAL
;
641 rsp
->invalidate_rkey
= key
;
642 rsp
->flags
|= NVMET_RDMA_REQ_INVALIDATE_RKEY
;
648 static u16
nvmet_rdma_map_sgl(struct nvmet_rdma_rsp
*rsp
)
650 struct nvme_keyed_sgl_desc
*sgl
= &rsp
->req
.cmd
->common
.dptr
.ksgl
;
652 switch (sgl
->type
>> 4) {
653 case NVME_SGL_FMT_DATA_DESC
:
654 switch (sgl
->type
& 0xf) {
655 case NVME_SGL_FMT_OFFSET
:
656 return nvmet_rdma_map_sgl_inline(rsp
);
658 pr_err("invalid SGL subtype: %#x\n", sgl
->type
);
659 return NVME_SC_INVALID_FIELD
| NVME_SC_DNR
;
661 case NVME_KEY_SGL_FMT_DATA_DESC
:
662 switch (sgl
->type
& 0xf) {
663 case NVME_SGL_FMT_ADDRESS
| NVME_SGL_FMT_INVALIDATE
:
664 return nvmet_rdma_map_sgl_keyed(rsp
, sgl
, true);
665 case NVME_SGL_FMT_ADDRESS
:
666 return nvmet_rdma_map_sgl_keyed(rsp
, sgl
, false);
668 pr_err("invalid SGL subtype: %#x\n", sgl
->type
);
669 return NVME_SC_INVALID_FIELD
| NVME_SC_DNR
;
672 pr_err("invalid SGL type: %#x\n", sgl
->type
);
673 return NVME_SC_SGL_INVALID_TYPE
| NVME_SC_DNR
;
677 static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp
*rsp
)
679 struct nvmet_rdma_queue
*queue
= rsp
->queue
;
681 if (unlikely(atomic_sub_return(1 + rsp
->n_rdma
,
682 &queue
->sq_wr_avail
) < 0)) {
683 pr_debug("IB send queue full (needed %d): queue %u cntlid %u\n",
684 1 + rsp
->n_rdma
, queue
->idx
,
685 queue
->nvme_sq
.ctrl
->cntlid
);
686 atomic_add(1 + rsp
->n_rdma
, &queue
->sq_wr_avail
);
690 if (nvmet_rdma_need_data_in(rsp
)) {
691 if (rdma_rw_ctx_post(&rsp
->rw
, queue
->cm_id
->qp
,
692 queue
->cm_id
->port_num
, &rsp
->read_cqe
, NULL
))
693 nvmet_req_complete(&rsp
->req
, NVME_SC_DATA_XFER_ERROR
);
695 rsp
->req
.execute(&rsp
->req
);
701 static void nvmet_rdma_handle_command(struct nvmet_rdma_queue
*queue
,
702 struct nvmet_rdma_rsp
*cmd
)
706 ib_dma_sync_single_for_cpu(queue
->dev
->device
,
707 cmd
->cmd
->sge
[0].addr
, cmd
->cmd
->sge
[0].length
,
709 ib_dma_sync_single_for_cpu(queue
->dev
->device
,
710 cmd
->send_sge
.addr
, cmd
->send_sge
.length
,
713 if (!nvmet_req_init(&cmd
->req
, &queue
->nvme_cq
,
714 &queue
->nvme_sq
, &nvmet_rdma_ops
))
717 status
= nvmet_rdma_map_sgl(cmd
);
721 if (unlikely(!nvmet_rdma_execute_command(cmd
))) {
722 spin_lock(&queue
->rsp_wr_wait_lock
);
723 list_add_tail(&cmd
->wait_list
, &queue
->rsp_wr_wait_list
);
724 spin_unlock(&queue
->rsp_wr_wait_lock
);
730 nvmet_req_complete(&cmd
->req
, status
);
733 static void nvmet_rdma_recv_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
735 struct nvmet_rdma_cmd
*cmd
=
736 container_of(wc
->wr_cqe
, struct nvmet_rdma_cmd
, cqe
);
737 struct nvmet_rdma_queue
*queue
= cq
->cq_context
;
738 struct nvmet_rdma_rsp
*rsp
;
740 if (unlikely(wc
->status
!= IB_WC_SUCCESS
)) {
741 if (wc
->status
!= IB_WC_WR_FLUSH_ERR
) {
742 pr_err("RECV for CQE 0x%p failed with status %s (%d)\n",
743 wc
->wr_cqe
, ib_wc_status_msg(wc
->status
),
745 nvmet_rdma_error_comp(queue
);
750 if (unlikely(wc
->byte_len
< sizeof(struct nvme_command
))) {
751 pr_err("Ctrl Fatal Error: capsule size less than 64 bytes\n");
752 nvmet_rdma_error_comp(queue
);
757 rsp
= nvmet_rdma_get_rsp(queue
);
761 rsp
->req
.cmd
= cmd
->nvme_cmd
;
762 rsp
->req
.port
= queue
->port
;
765 if (unlikely(queue
->state
!= NVMET_RDMA_Q_LIVE
)) {
768 spin_lock_irqsave(&queue
->state_lock
, flags
);
769 if (queue
->state
== NVMET_RDMA_Q_CONNECTING
)
770 list_add_tail(&rsp
->wait_list
, &queue
->rsp_wait_list
);
772 nvmet_rdma_put_rsp(rsp
);
773 spin_unlock_irqrestore(&queue
->state_lock
, flags
);
777 nvmet_rdma_handle_command(queue
, rsp
);
780 static void nvmet_rdma_destroy_srq(struct nvmet_rdma_device
*ndev
)
785 nvmet_rdma_free_cmds(ndev
, ndev
->srq_cmds
, ndev
->srq_size
, false);
786 ib_destroy_srq(ndev
->srq
);
789 static int nvmet_rdma_init_srq(struct nvmet_rdma_device
*ndev
)
791 struct ib_srq_init_attr srq_attr
= { NULL
, };
796 srq_size
= 4095; /* XXX: tune */
798 srq_attr
.attr
.max_wr
= srq_size
;
799 srq_attr
.attr
.max_sge
= 2;
800 srq_attr
.attr
.srq_limit
= 0;
801 srq_attr
.srq_type
= IB_SRQT_BASIC
;
802 srq
= ib_create_srq(ndev
->pd
, &srq_attr
);
805 * If SRQs aren't supported we just go ahead and use normal
806 * non-shared receive queues.
808 pr_info("SRQ requested but not supported.\n");
812 ndev
->srq_cmds
= nvmet_rdma_alloc_cmds(ndev
, srq_size
, false);
813 if (IS_ERR(ndev
->srq_cmds
)) {
814 ret
= PTR_ERR(ndev
->srq_cmds
);
815 goto out_destroy_srq
;
819 ndev
->srq_size
= srq_size
;
821 for (i
= 0; i
< srq_size
; i
++)
822 nvmet_rdma_post_recv(ndev
, &ndev
->srq_cmds
[i
]);
831 static void nvmet_rdma_free_dev(struct kref
*ref
)
833 struct nvmet_rdma_device
*ndev
=
834 container_of(ref
, struct nvmet_rdma_device
, ref
);
836 mutex_lock(&device_list_mutex
);
837 list_del(&ndev
->entry
);
838 mutex_unlock(&device_list_mutex
);
840 nvmet_rdma_destroy_srq(ndev
);
841 ib_dealloc_pd(ndev
->pd
);
846 static struct nvmet_rdma_device
*
847 nvmet_rdma_find_get_device(struct rdma_cm_id
*cm_id
)
849 struct nvmet_rdma_device
*ndev
;
852 mutex_lock(&device_list_mutex
);
853 list_for_each_entry(ndev
, &device_list
, entry
) {
854 if (ndev
->device
->node_guid
== cm_id
->device
->node_guid
&&
855 kref_get_unless_zero(&ndev
->ref
))
859 ndev
= kzalloc(sizeof(*ndev
), GFP_KERNEL
);
863 ndev
->device
= cm_id
->device
;
864 kref_init(&ndev
->ref
);
866 ndev
->pd
= ib_alloc_pd(ndev
->device
, 0);
867 if (IS_ERR(ndev
->pd
))
870 if (nvmet_rdma_use_srq
) {
871 ret
= nvmet_rdma_init_srq(ndev
);
876 list_add(&ndev
->entry
, &device_list
);
878 mutex_unlock(&device_list_mutex
);
879 pr_debug("added %s.\n", ndev
->device
->name
);
883 ib_dealloc_pd(ndev
->pd
);
887 mutex_unlock(&device_list_mutex
);
891 static int nvmet_rdma_create_queue_ib(struct nvmet_rdma_queue
*queue
)
893 struct ib_qp_init_attr qp_attr
;
894 struct nvmet_rdma_device
*ndev
= queue
->dev
;
895 int comp_vector
, nr_cqe
, ret
, i
;
898 * Spread the io queues across completion vectors,
899 * but still keep all admin queues on vector 0.
901 comp_vector
= !queue
->host_qid
? 0 :
902 queue
->idx
% ndev
->device
->num_comp_vectors
;
905 * Reserve CQ slots for RECV + RDMA_READ/RDMA_WRITE + RDMA_SEND.
907 nr_cqe
= queue
->recv_queue_size
+ 2 * queue
->send_queue_size
;
909 queue
->cq
= ib_alloc_cq(ndev
->device
, queue
,
910 nr_cqe
+ 1, comp_vector
,
912 if (IS_ERR(queue
->cq
)) {
913 ret
= PTR_ERR(queue
->cq
);
914 pr_err("failed to create CQ cqe= %d ret= %d\n",
919 memset(&qp_attr
, 0, sizeof(qp_attr
));
920 qp_attr
.qp_context
= queue
;
921 qp_attr
.event_handler
= nvmet_rdma_qp_event
;
922 qp_attr
.send_cq
= queue
->cq
;
923 qp_attr
.recv_cq
= queue
->cq
;
924 qp_attr
.sq_sig_type
= IB_SIGNAL_REQ_WR
;
925 qp_attr
.qp_type
= IB_QPT_RC
;
927 qp_attr
.cap
.max_send_wr
= queue
->send_queue_size
+ 1;
928 qp_attr
.cap
.max_rdma_ctxs
= queue
->send_queue_size
;
929 qp_attr
.cap
.max_send_sge
= max(ndev
->device
->attrs
.max_sge_rd
,
930 ndev
->device
->attrs
.max_sge
);
933 qp_attr
.srq
= ndev
->srq
;
936 qp_attr
.cap
.max_recv_wr
= 1 + queue
->recv_queue_size
;
937 qp_attr
.cap
.max_recv_sge
= 2;
940 ret
= rdma_create_qp(queue
->cm_id
, ndev
->pd
, &qp_attr
);
942 pr_err("failed to create_qp ret= %d\n", ret
);
946 atomic_set(&queue
->sq_wr_avail
, qp_attr
.cap
.max_send_wr
);
948 pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d cm_id= %p\n",
949 __func__
, queue
->cq
->cqe
, qp_attr
.cap
.max_send_sge
,
950 qp_attr
.cap
.max_send_wr
, queue
->cm_id
);
953 for (i
= 0; i
< queue
->recv_queue_size
; i
++) {
954 queue
->cmds
[i
].queue
= queue
;
955 nvmet_rdma_post_recv(ndev
, &queue
->cmds
[i
]);
963 ib_free_cq(queue
->cq
);
967 static void nvmet_rdma_destroy_queue_ib(struct nvmet_rdma_queue
*queue
)
969 ib_drain_qp(queue
->cm_id
->qp
);
970 rdma_destroy_qp(queue
->cm_id
);
971 ib_free_cq(queue
->cq
);
974 static void nvmet_rdma_free_queue(struct nvmet_rdma_queue
*queue
)
976 pr_info("freeing queue %d\n", queue
->idx
);
978 nvmet_sq_destroy(&queue
->nvme_sq
);
980 nvmet_rdma_destroy_queue_ib(queue
);
981 if (!queue
->dev
->srq
) {
982 nvmet_rdma_free_cmds(queue
->dev
, queue
->cmds
,
983 queue
->recv_queue_size
,
986 nvmet_rdma_free_rsps(queue
);
987 ida_simple_remove(&nvmet_rdma_queue_ida
, queue
->idx
);
991 static void nvmet_rdma_release_queue_work(struct work_struct
*w
)
993 struct nvmet_rdma_queue
*queue
=
994 container_of(w
, struct nvmet_rdma_queue
, release_work
);
995 struct rdma_cm_id
*cm_id
= queue
->cm_id
;
996 struct nvmet_rdma_device
*dev
= queue
->dev
;
997 enum nvmet_rdma_queue_state state
= queue
->state
;
999 nvmet_rdma_free_queue(queue
);
1001 if (state
!= NVMET_RDMA_IN_DEVICE_REMOVAL
)
1002 rdma_destroy_id(cm_id
);
1004 kref_put(&dev
->ref
, nvmet_rdma_free_dev
);
1008 nvmet_rdma_parse_cm_connect_req(struct rdma_conn_param
*conn
,
1009 struct nvmet_rdma_queue
*queue
)
1011 struct nvme_rdma_cm_req
*req
;
1013 req
= (struct nvme_rdma_cm_req
*)conn
->private_data
;
1014 if (!req
|| conn
->private_data_len
== 0)
1015 return NVME_RDMA_CM_INVALID_LEN
;
1017 if (le16_to_cpu(req
->recfmt
) != NVME_RDMA_CM_FMT_1_0
)
1018 return NVME_RDMA_CM_INVALID_RECFMT
;
1020 queue
->host_qid
= le16_to_cpu(req
->qid
);
1023 * req->hsqsize corresponds to our recv queue size plus 1
1024 * req->hrqsize corresponds to our send queue size
1026 queue
->recv_queue_size
= le16_to_cpu(req
->hsqsize
) + 1;
1027 queue
->send_queue_size
= le16_to_cpu(req
->hrqsize
);
1029 if (!queue
->host_qid
&& queue
->recv_queue_size
> NVMF_AQ_DEPTH
)
1030 return NVME_RDMA_CM_INVALID_HSQSIZE
;
1032 /* XXX: Should we enforce some kind of max for IO queues? */
1037 static int nvmet_rdma_cm_reject(struct rdma_cm_id
*cm_id
,
1038 enum nvme_rdma_cm_status status
)
1040 struct nvme_rdma_cm_rej rej
;
1042 pr_debug("rejecting connect request: status %d (%s)\n",
1043 status
, nvme_rdma_cm_msg(status
));
1045 rej
.recfmt
= cpu_to_le16(NVME_RDMA_CM_FMT_1_0
);
1046 rej
.sts
= cpu_to_le16(status
);
1048 return rdma_reject(cm_id
, (void *)&rej
, sizeof(rej
));
1051 static struct nvmet_rdma_queue
*
1052 nvmet_rdma_alloc_queue(struct nvmet_rdma_device
*ndev
,
1053 struct rdma_cm_id
*cm_id
,
1054 struct rdma_cm_event
*event
)
1056 struct nvmet_rdma_queue
*queue
;
1059 queue
= kzalloc(sizeof(*queue
), GFP_KERNEL
);
1061 ret
= NVME_RDMA_CM_NO_RSC
;
1065 ret
= nvmet_sq_init(&queue
->nvme_sq
);
1067 ret
= NVME_RDMA_CM_NO_RSC
;
1068 goto out_free_queue
;
1071 ret
= nvmet_rdma_parse_cm_connect_req(&event
->param
.conn
, queue
);
1073 goto out_destroy_sq
;
1076 * Schedules the actual release because calling rdma_destroy_id from
1077 * inside a CM callback would trigger a deadlock. (great API design..)
1079 INIT_WORK(&queue
->release_work
, nvmet_rdma_release_queue_work
);
1081 queue
->cm_id
= cm_id
;
1083 spin_lock_init(&queue
->state_lock
);
1084 queue
->state
= NVMET_RDMA_Q_CONNECTING
;
1085 INIT_LIST_HEAD(&queue
->rsp_wait_list
);
1086 INIT_LIST_HEAD(&queue
->rsp_wr_wait_list
);
1087 spin_lock_init(&queue
->rsp_wr_wait_lock
);
1088 INIT_LIST_HEAD(&queue
->free_rsps
);
1089 spin_lock_init(&queue
->rsps_lock
);
1090 INIT_LIST_HEAD(&queue
->queue_list
);
1092 queue
->idx
= ida_simple_get(&nvmet_rdma_queue_ida
, 0, 0, GFP_KERNEL
);
1093 if (queue
->idx
< 0) {
1094 ret
= NVME_RDMA_CM_NO_RSC
;
1095 goto out_destroy_sq
;
1098 ret
= nvmet_rdma_alloc_rsps(queue
);
1100 ret
= NVME_RDMA_CM_NO_RSC
;
1101 goto out_ida_remove
;
1105 queue
->cmds
= nvmet_rdma_alloc_cmds(ndev
,
1106 queue
->recv_queue_size
,
1108 if (IS_ERR(queue
->cmds
)) {
1109 ret
= NVME_RDMA_CM_NO_RSC
;
1110 goto out_free_responses
;
1114 ret
= nvmet_rdma_create_queue_ib(queue
);
1116 pr_err("%s: creating RDMA queue failed (%d).\n",
1118 ret
= NVME_RDMA_CM_NO_RSC
;
1126 nvmet_rdma_free_cmds(queue
->dev
, queue
->cmds
,
1127 queue
->recv_queue_size
,
1131 nvmet_rdma_free_rsps(queue
);
1133 ida_simple_remove(&nvmet_rdma_queue_ida
, queue
->idx
);
1135 nvmet_sq_destroy(&queue
->nvme_sq
);
1139 nvmet_rdma_cm_reject(cm_id
, ret
);
1143 static void nvmet_rdma_qp_event(struct ib_event
*event
, void *priv
)
1145 struct nvmet_rdma_queue
*queue
= priv
;
1147 switch (event
->event
) {
1148 case IB_EVENT_COMM_EST
:
1149 rdma_notify(queue
->cm_id
, event
->event
);
1152 pr_err("received IB QP event: %s (%d)\n",
1153 ib_event_msg(event
->event
), event
->event
);
1158 static int nvmet_rdma_cm_accept(struct rdma_cm_id
*cm_id
,
1159 struct nvmet_rdma_queue
*queue
,
1160 struct rdma_conn_param
*p
)
1162 struct rdma_conn_param param
= { };
1163 struct nvme_rdma_cm_rep priv
= { };
1166 param
.rnr_retry_count
= 7;
1167 param
.flow_control
= 1;
1168 param
.initiator_depth
= min_t(u8
, p
->initiator_depth
,
1169 queue
->dev
->device
->attrs
.max_qp_init_rd_atom
);
1170 param
.private_data
= &priv
;
1171 param
.private_data_len
= sizeof(priv
);
1172 priv
.recfmt
= cpu_to_le16(NVME_RDMA_CM_FMT_1_0
);
1173 priv
.crqsize
= cpu_to_le16(queue
->recv_queue_size
);
1175 ret
= rdma_accept(cm_id
, ¶m
);
1177 pr_err("rdma_accept failed (error code = %d)\n", ret
);
1182 static int nvmet_rdma_queue_connect(struct rdma_cm_id
*cm_id
,
1183 struct rdma_cm_event
*event
)
1185 struct nvmet_rdma_device
*ndev
;
1186 struct nvmet_rdma_queue
*queue
;
1189 ndev
= nvmet_rdma_find_get_device(cm_id
);
1191 nvmet_rdma_cm_reject(cm_id
, NVME_RDMA_CM_NO_RSC
);
1192 return -ECONNREFUSED
;
1195 queue
= nvmet_rdma_alloc_queue(ndev
, cm_id
, event
);
1200 queue
->port
= cm_id
->context
;
1202 if (queue
->host_qid
== 0) {
1203 /* Let inflight controller teardown complete */
1204 flush_scheduled_work();
1207 ret
= nvmet_rdma_cm_accept(cm_id
, queue
, &event
->param
.conn
);
1211 mutex_lock(&nvmet_rdma_queue_mutex
);
1212 list_add_tail(&queue
->queue_list
, &nvmet_rdma_queue_list
);
1213 mutex_unlock(&nvmet_rdma_queue_mutex
);
1218 nvmet_rdma_free_queue(queue
);
1220 kref_put(&ndev
->ref
, nvmet_rdma_free_dev
);
1225 static void nvmet_rdma_queue_established(struct nvmet_rdma_queue
*queue
)
1227 unsigned long flags
;
1229 spin_lock_irqsave(&queue
->state_lock
, flags
);
1230 if (queue
->state
!= NVMET_RDMA_Q_CONNECTING
) {
1231 pr_warn("trying to establish a connected queue\n");
1234 queue
->state
= NVMET_RDMA_Q_LIVE
;
1236 while (!list_empty(&queue
->rsp_wait_list
)) {
1237 struct nvmet_rdma_rsp
*cmd
;
1239 cmd
= list_first_entry(&queue
->rsp_wait_list
,
1240 struct nvmet_rdma_rsp
, wait_list
);
1241 list_del(&cmd
->wait_list
);
1243 spin_unlock_irqrestore(&queue
->state_lock
, flags
);
1244 nvmet_rdma_handle_command(queue
, cmd
);
1245 spin_lock_irqsave(&queue
->state_lock
, flags
);
1249 spin_unlock_irqrestore(&queue
->state_lock
, flags
);
1252 static void __nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue
*queue
)
1254 bool disconnect
= false;
1255 unsigned long flags
;
1257 pr_debug("cm_id= %p queue->state= %d\n", queue
->cm_id
, queue
->state
);
1259 spin_lock_irqsave(&queue
->state_lock
, flags
);
1260 switch (queue
->state
) {
1261 case NVMET_RDMA_Q_CONNECTING
:
1262 case NVMET_RDMA_Q_LIVE
:
1263 queue
->state
= NVMET_RDMA_Q_DISCONNECTING
;
1264 case NVMET_RDMA_IN_DEVICE_REMOVAL
:
1267 case NVMET_RDMA_Q_DISCONNECTING
:
1270 spin_unlock_irqrestore(&queue
->state_lock
, flags
);
1273 rdma_disconnect(queue
->cm_id
);
1274 schedule_work(&queue
->release_work
);
1278 static void nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue
*queue
)
1280 bool disconnect
= false;
1282 mutex_lock(&nvmet_rdma_queue_mutex
);
1283 if (!list_empty(&queue
->queue_list
)) {
1284 list_del_init(&queue
->queue_list
);
1287 mutex_unlock(&nvmet_rdma_queue_mutex
);
1290 __nvmet_rdma_queue_disconnect(queue
);
1293 static void nvmet_rdma_queue_connect_fail(struct rdma_cm_id
*cm_id
,
1294 struct nvmet_rdma_queue
*queue
)
1296 WARN_ON_ONCE(queue
->state
!= NVMET_RDMA_Q_CONNECTING
);
1298 mutex_lock(&nvmet_rdma_queue_mutex
);
1299 if (!list_empty(&queue
->queue_list
))
1300 list_del_init(&queue
->queue_list
);
1301 mutex_unlock(&nvmet_rdma_queue_mutex
);
1303 pr_err("failed to connect queue %d\n", queue
->idx
);
1304 schedule_work(&queue
->release_work
);
1308 * nvme_rdma_device_removal() - Handle RDMA device removal
1309 * @queue: nvmet rdma queue (cm id qp_context)
1310 * @addr: nvmet address (cm_id context)
1312 * DEVICE_REMOVAL event notifies us that the RDMA device is about
1313 * to unplug so we should take care of destroying our RDMA resources.
1314 * This event will be generated for each allocated cm_id.
1316 * Note that this event can be generated on a normal queue cm_id
1317 * and/or a device bound listener cm_id (where in this case
1318 * queue will be null).
1320 * we claim ownership on destroying the cm_id. For queues we move
1321 * the queue state to NVMET_RDMA_IN_DEVICE_REMOVAL and for port
1322 * we nullify the priv to prevent double cm_id destruction and destroying
1323 * the cm_id implicitely by returning a non-zero rc to the callout.
1325 static int nvmet_rdma_device_removal(struct rdma_cm_id
*cm_id
,
1326 struct nvmet_rdma_queue
*queue
)
1328 unsigned long flags
;
1331 struct nvmet_port
*port
= cm_id
->context
;
1334 * This is a listener cm_id. Make sure that
1335 * future remove_port won't invoke a double
1336 * cm_id destroy. use atomic xchg to make sure
1337 * we don't compete with remove_port.
1339 if (xchg(&port
->priv
, NULL
) != cm_id
)
1343 * This is a queue cm_id. Make sure that
1344 * release queue will not destroy the cm_id
1345 * and schedule all ctrl queues removal (only
1346 * if the queue is not disconnecting already).
1348 spin_lock_irqsave(&queue
->state_lock
, flags
);
1349 if (queue
->state
!= NVMET_RDMA_Q_DISCONNECTING
)
1350 queue
->state
= NVMET_RDMA_IN_DEVICE_REMOVAL
;
1351 spin_unlock_irqrestore(&queue
->state_lock
, flags
);
1352 nvmet_rdma_queue_disconnect(queue
);
1353 flush_scheduled_work();
1357 * We need to return 1 so that the core will destroy
1358 * it's own ID. What a great API design..
1363 static int nvmet_rdma_cm_handler(struct rdma_cm_id
*cm_id
,
1364 struct rdma_cm_event
*event
)
1366 struct nvmet_rdma_queue
*queue
= NULL
;
1370 queue
= cm_id
->qp
->qp_context
;
1372 pr_debug("%s (%d): status %d id %p\n",
1373 rdma_event_msg(event
->event
), event
->event
,
1374 event
->status
, cm_id
);
1376 switch (event
->event
) {
1377 case RDMA_CM_EVENT_CONNECT_REQUEST
:
1378 ret
= nvmet_rdma_queue_connect(cm_id
, event
);
1380 case RDMA_CM_EVENT_ESTABLISHED
:
1381 nvmet_rdma_queue_established(queue
);
1383 case RDMA_CM_EVENT_ADDR_CHANGE
:
1384 case RDMA_CM_EVENT_DISCONNECTED
:
1385 case RDMA_CM_EVENT_TIMEWAIT_EXIT
:
1387 * We might end up here when we already freed the qp
1388 * which means queue release sequence is in progress,
1389 * so don't get in the way...
1392 nvmet_rdma_queue_disconnect(queue
);
1394 case RDMA_CM_EVENT_DEVICE_REMOVAL
:
1395 ret
= nvmet_rdma_device_removal(cm_id
, queue
);
1397 case RDMA_CM_EVENT_REJECTED
:
1398 pr_debug("Connection rejected: %s\n",
1399 rdma_reject_msg(cm_id
, event
->status
));
1401 case RDMA_CM_EVENT_UNREACHABLE
:
1402 case RDMA_CM_EVENT_CONNECT_ERROR
:
1403 nvmet_rdma_queue_connect_fail(cm_id
, queue
);
1406 pr_err("received unrecognized RDMA CM event %d\n",
1414 static void nvmet_rdma_delete_ctrl(struct nvmet_ctrl
*ctrl
)
1416 struct nvmet_rdma_queue
*queue
;
1419 mutex_lock(&nvmet_rdma_queue_mutex
);
1420 list_for_each_entry(queue
, &nvmet_rdma_queue_list
, queue_list
) {
1421 if (queue
->nvme_sq
.ctrl
== ctrl
) {
1422 list_del_init(&queue
->queue_list
);
1423 mutex_unlock(&nvmet_rdma_queue_mutex
);
1425 __nvmet_rdma_queue_disconnect(queue
);
1429 mutex_unlock(&nvmet_rdma_queue_mutex
);
1432 static int nvmet_rdma_add_port(struct nvmet_port
*port
)
1434 struct rdma_cm_id
*cm_id
;
1435 struct sockaddr_storage addr
= { };
1436 __kernel_sa_family_t af
;
1439 switch (port
->disc_addr
.adrfam
) {
1440 case NVMF_ADDR_FAMILY_IP4
:
1443 case NVMF_ADDR_FAMILY_IP6
:
1447 pr_err("address family %d not supported\n",
1448 port
->disc_addr
.adrfam
);
1452 ret
= inet_pton_with_scope(&init_net
, af
, port
->disc_addr
.traddr
,
1453 port
->disc_addr
.trsvcid
, &addr
);
1455 pr_err("malformed ip/port passed: %s:%s\n",
1456 port
->disc_addr
.traddr
, port
->disc_addr
.trsvcid
);
1460 cm_id
= rdma_create_id(&init_net
, nvmet_rdma_cm_handler
, port
,
1461 RDMA_PS_TCP
, IB_QPT_RC
);
1462 if (IS_ERR(cm_id
)) {
1463 pr_err("CM ID creation failed\n");
1464 return PTR_ERR(cm_id
);
1468 * Allow both IPv4 and IPv6 sockets to bind a single port
1471 ret
= rdma_set_afonly(cm_id
, 1);
1473 pr_err("rdma_set_afonly failed (%d)\n", ret
);
1474 goto out_destroy_id
;
1477 ret
= rdma_bind_addr(cm_id
, (struct sockaddr
*)&addr
);
1479 pr_err("binding CM ID to %pISpcs failed (%d)\n",
1480 (struct sockaddr
*)&addr
, ret
);
1481 goto out_destroy_id
;
1484 ret
= rdma_listen(cm_id
, 128);
1486 pr_err("listening to %pISpcs failed (%d)\n",
1487 (struct sockaddr
*)&addr
, ret
);
1488 goto out_destroy_id
;
1491 pr_info("enabling port %d (%pISpcs)\n",
1492 le16_to_cpu(port
->disc_addr
.portid
), (struct sockaddr
*)&addr
);
1497 rdma_destroy_id(cm_id
);
1501 static void nvmet_rdma_remove_port(struct nvmet_port
*port
)
1503 struct rdma_cm_id
*cm_id
= xchg(&port
->priv
, NULL
);
1506 rdma_destroy_id(cm_id
);
1509 static struct nvmet_fabrics_ops nvmet_rdma_ops
= {
1510 .owner
= THIS_MODULE
,
1511 .type
= NVMF_TRTYPE_RDMA
,
1512 .sqe_inline_size
= NVMET_RDMA_INLINE_DATA_SIZE
,
1514 .has_keyed_sgls
= 1,
1515 .add_port
= nvmet_rdma_add_port
,
1516 .remove_port
= nvmet_rdma_remove_port
,
1517 .queue_response
= nvmet_rdma_queue_response
,
1518 .delete_ctrl
= nvmet_rdma_delete_ctrl
,
1521 static int __init
nvmet_rdma_init(void)
1523 return nvmet_register_transport(&nvmet_rdma_ops
);
1526 static void __exit
nvmet_rdma_exit(void)
1528 struct nvmet_rdma_queue
*queue
;
1530 nvmet_unregister_transport(&nvmet_rdma_ops
);
1532 flush_scheduled_work();
1534 mutex_lock(&nvmet_rdma_queue_mutex
);
1535 while ((queue
= list_first_entry_or_null(&nvmet_rdma_queue_list
,
1536 struct nvmet_rdma_queue
, queue_list
))) {
1537 list_del_init(&queue
->queue_list
);
1539 mutex_unlock(&nvmet_rdma_queue_mutex
);
1540 __nvmet_rdma_queue_disconnect(queue
);
1541 mutex_lock(&nvmet_rdma_queue_mutex
);
1543 mutex_unlock(&nvmet_rdma_queue_mutex
);
1545 flush_scheduled_work();
1546 ida_destroy(&nvmet_rdma_queue_ida
);
1549 module_init(nvmet_rdma_init
);
1550 module_exit(nvmet_rdma_exit
);
1552 MODULE_LICENSE("GPL v2");
1553 MODULE_ALIAS("nvmet-transport-1"); /* 1 == NVMF_TRTYPE_RDMA */