2 * NVMe over Fabrics RDMA target.
3 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
15 #include <linux/atomic.h>
16 #include <linux/ctype.h>
17 #include <linux/delay.h>
18 #include <linux/err.h>
19 #include <linux/init.h>
20 #include <linux/module.h>
21 #include <linux/nvme.h>
22 #include <linux/slab.h>
23 #include <linux/string.h>
24 #include <linux/wait.h>
25 #include <linux/inet.h>
26 #include <asm/unaligned.h>
28 #include <rdma/ib_verbs.h>
29 #include <rdma/rdma_cm.h>
32 #include <linux/nvme-rdma.h>
36 * We allow up to a page of inline data to go with the SQE
38 #define NVMET_RDMA_INLINE_DATA_SIZE PAGE_SIZE
40 struct nvmet_rdma_cmd
{
44 struct scatterlist inline_sg
;
45 struct page
*inline_page
;
46 struct nvme_command
*nvme_cmd
;
47 struct nvmet_rdma_queue
*queue
;
51 NVMET_RDMA_REQ_INLINE_DATA
= (1 << 0),
52 NVMET_RDMA_REQ_INVALIDATE_RKEY
= (1 << 1),
55 struct nvmet_rdma_rsp
{
56 struct ib_sge send_sge
;
57 struct ib_cqe send_cqe
;
58 struct ib_send_wr send_wr
;
60 struct nvmet_rdma_cmd
*cmd
;
61 struct nvmet_rdma_queue
*queue
;
63 struct ib_cqe read_cqe
;
64 struct rdma_rw_ctx rw
;
72 struct list_head wait_list
;
73 struct list_head free_list
;
76 enum nvmet_rdma_queue_state
{
77 NVMET_RDMA_Q_CONNECTING
,
79 NVMET_RDMA_Q_DISCONNECTING
,
80 NVMET_RDMA_IN_DEVICE_REMOVAL
,
83 struct nvmet_rdma_queue
{
84 struct rdma_cm_id
*cm_id
;
85 struct nvmet_port
*port
;
88 struct nvmet_rdma_device
*dev
;
89 spinlock_t state_lock
;
90 enum nvmet_rdma_queue_state state
;
91 struct nvmet_cq nvme_cq
;
92 struct nvmet_sq nvme_sq
;
94 struct nvmet_rdma_rsp
*rsps
;
95 struct list_head free_rsps
;
97 struct nvmet_rdma_cmd
*cmds
;
99 struct work_struct release_work
;
100 struct list_head rsp_wait_list
;
101 struct list_head rsp_wr_wait_list
;
102 spinlock_t rsp_wr_wait_lock
;
109 struct list_head queue_list
;
112 struct nvmet_rdma_device
{
113 struct ib_device
*device
;
116 struct nvmet_rdma_cmd
*srq_cmds
;
119 struct list_head entry
;
122 static bool nvmet_rdma_use_srq
;
123 module_param_named(use_srq
, nvmet_rdma_use_srq
, bool, 0444);
124 MODULE_PARM_DESC(use_srq
, "Use shared receive queue.");
126 static DEFINE_IDA(nvmet_rdma_queue_ida
);
127 static LIST_HEAD(nvmet_rdma_queue_list
);
128 static DEFINE_MUTEX(nvmet_rdma_queue_mutex
);
130 static LIST_HEAD(device_list
);
131 static DEFINE_MUTEX(device_list_mutex
);
133 static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp
*rsp
);
134 static void nvmet_rdma_send_done(struct ib_cq
*cq
, struct ib_wc
*wc
);
135 static void nvmet_rdma_recv_done(struct ib_cq
*cq
, struct ib_wc
*wc
);
136 static void nvmet_rdma_read_data_done(struct ib_cq
*cq
, struct ib_wc
*wc
);
137 static void nvmet_rdma_qp_event(struct ib_event
*event
, void *priv
);
138 static void nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue
*queue
);
140 static struct nvmet_fabrics_ops nvmet_rdma_ops
;
142 /* XXX: really should move to a generic header sooner or later.. */
143 static inline u32
get_unaligned_le24(const u8
*p
)
145 return (u32
)p
[0] | (u32
)p
[1] << 8 | (u32
)p
[2] << 16;
148 static inline bool nvmet_rdma_need_data_in(struct nvmet_rdma_rsp
*rsp
)
150 return nvme_is_write(rsp
->req
.cmd
) &&
152 !(rsp
->flags
& NVMET_RDMA_REQ_INLINE_DATA
);
155 static inline bool nvmet_rdma_need_data_out(struct nvmet_rdma_rsp
*rsp
)
157 return !nvme_is_write(rsp
->req
.cmd
) &&
159 !rsp
->req
.rsp
->status
&&
160 !(rsp
->flags
& NVMET_RDMA_REQ_INLINE_DATA
);
163 static inline struct nvmet_rdma_rsp
*
164 nvmet_rdma_get_rsp(struct nvmet_rdma_queue
*queue
)
166 struct nvmet_rdma_rsp
*rsp
;
169 spin_lock_irqsave(&queue
->rsps_lock
, flags
);
170 rsp
= list_first_entry(&queue
->free_rsps
,
171 struct nvmet_rdma_rsp
, free_list
);
172 list_del(&rsp
->free_list
);
173 spin_unlock_irqrestore(&queue
->rsps_lock
, flags
);
179 nvmet_rdma_put_rsp(struct nvmet_rdma_rsp
*rsp
)
183 spin_lock_irqsave(&rsp
->queue
->rsps_lock
, flags
);
184 list_add_tail(&rsp
->free_list
, &rsp
->queue
->free_rsps
);
185 spin_unlock_irqrestore(&rsp
->queue
->rsps_lock
, flags
);
188 static void nvmet_rdma_free_sgl(struct scatterlist
*sgl
, unsigned int nents
)
190 struct scatterlist
*sg
;
196 for_each_sg(sgl
, sg
, nents
, count
)
197 __free_page(sg_page(sg
));
201 static int nvmet_rdma_alloc_sgl(struct scatterlist
**sgl
, unsigned int *nents
,
204 struct scatterlist
*sg
;
209 nent
= DIV_ROUND_UP(length
, PAGE_SIZE
);
210 sg
= kmalloc_array(nent
, sizeof(struct scatterlist
), GFP_KERNEL
);
214 sg_init_table(sg
, nent
);
217 u32 page_len
= min_t(u32
, length
, PAGE_SIZE
);
219 page
= alloc_page(GFP_KERNEL
);
223 sg_set_page(&sg
[i
], page
, page_len
, 0);
234 __free_page(sg_page(&sg
[i
]));
238 return NVME_SC_INTERNAL
;
241 static int nvmet_rdma_alloc_cmd(struct nvmet_rdma_device
*ndev
,
242 struct nvmet_rdma_cmd
*c
, bool admin
)
244 /* NVMe command / RDMA RECV */
245 c
->nvme_cmd
= kmalloc(sizeof(*c
->nvme_cmd
), GFP_KERNEL
);
249 c
->sge
[0].addr
= ib_dma_map_single(ndev
->device
, c
->nvme_cmd
,
250 sizeof(*c
->nvme_cmd
), DMA_FROM_DEVICE
);
251 if (ib_dma_mapping_error(ndev
->device
, c
->sge
[0].addr
))
254 c
->sge
[0].length
= sizeof(*c
->nvme_cmd
);
255 c
->sge
[0].lkey
= ndev
->pd
->local_dma_lkey
;
258 c
->inline_page
= alloc_pages(GFP_KERNEL
,
259 get_order(NVMET_RDMA_INLINE_DATA_SIZE
));
262 c
->sge
[1].addr
= ib_dma_map_page(ndev
->device
,
263 c
->inline_page
, 0, NVMET_RDMA_INLINE_DATA_SIZE
,
265 if (ib_dma_mapping_error(ndev
->device
, c
->sge
[1].addr
))
266 goto out_free_inline_page
;
267 c
->sge
[1].length
= NVMET_RDMA_INLINE_DATA_SIZE
;
268 c
->sge
[1].lkey
= ndev
->pd
->local_dma_lkey
;
271 c
->cqe
.done
= nvmet_rdma_recv_done
;
273 c
->wr
.wr_cqe
= &c
->cqe
;
274 c
->wr
.sg_list
= c
->sge
;
275 c
->wr
.num_sge
= admin
? 1 : 2;
279 out_free_inline_page
:
281 __free_pages(c
->inline_page
,
282 get_order(NVMET_RDMA_INLINE_DATA_SIZE
));
285 ib_dma_unmap_single(ndev
->device
, c
->sge
[0].addr
,
286 sizeof(*c
->nvme_cmd
), DMA_FROM_DEVICE
);
294 static void nvmet_rdma_free_cmd(struct nvmet_rdma_device
*ndev
,
295 struct nvmet_rdma_cmd
*c
, bool admin
)
298 ib_dma_unmap_page(ndev
->device
, c
->sge
[1].addr
,
299 NVMET_RDMA_INLINE_DATA_SIZE
, DMA_FROM_DEVICE
);
300 __free_pages(c
->inline_page
,
301 get_order(NVMET_RDMA_INLINE_DATA_SIZE
));
303 ib_dma_unmap_single(ndev
->device
, c
->sge
[0].addr
,
304 sizeof(*c
->nvme_cmd
), DMA_FROM_DEVICE
);
308 static struct nvmet_rdma_cmd
*
309 nvmet_rdma_alloc_cmds(struct nvmet_rdma_device
*ndev
,
310 int nr_cmds
, bool admin
)
312 struct nvmet_rdma_cmd
*cmds
;
313 int ret
= -EINVAL
, i
;
315 cmds
= kcalloc(nr_cmds
, sizeof(struct nvmet_rdma_cmd
), GFP_KERNEL
);
319 for (i
= 0; i
< nr_cmds
; i
++) {
320 ret
= nvmet_rdma_alloc_cmd(ndev
, cmds
+ i
, admin
);
329 nvmet_rdma_free_cmd(ndev
, cmds
+ i
, admin
);
335 static void nvmet_rdma_free_cmds(struct nvmet_rdma_device
*ndev
,
336 struct nvmet_rdma_cmd
*cmds
, int nr_cmds
, bool admin
)
340 for (i
= 0; i
< nr_cmds
; i
++)
341 nvmet_rdma_free_cmd(ndev
, cmds
+ i
, admin
);
345 static int nvmet_rdma_alloc_rsp(struct nvmet_rdma_device
*ndev
,
346 struct nvmet_rdma_rsp
*r
)
348 /* NVMe CQE / RDMA SEND */
349 r
->req
.rsp
= kmalloc(sizeof(*r
->req
.rsp
), GFP_KERNEL
);
353 r
->send_sge
.addr
= ib_dma_map_single(ndev
->device
, r
->req
.rsp
,
354 sizeof(*r
->req
.rsp
), DMA_TO_DEVICE
);
355 if (ib_dma_mapping_error(ndev
->device
, r
->send_sge
.addr
))
358 r
->send_sge
.length
= sizeof(*r
->req
.rsp
);
359 r
->send_sge
.lkey
= ndev
->pd
->local_dma_lkey
;
361 r
->send_cqe
.done
= nvmet_rdma_send_done
;
363 r
->send_wr
.wr_cqe
= &r
->send_cqe
;
364 r
->send_wr
.sg_list
= &r
->send_sge
;
365 r
->send_wr
.num_sge
= 1;
366 r
->send_wr
.send_flags
= IB_SEND_SIGNALED
;
368 /* Data In / RDMA READ */
369 r
->read_cqe
.done
= nvmet_rdma_read_data_done
;
378 static void nvmet_rdma_free_rsp(struct nvmet_rdma_device
*ndev
,
379 struct nvmet_rdma_rsp
*r
)
381 ib_dma_unmap_single(ndev
->device
, r
->send_sge
.addr
,
382 sizeof(*r
->req
.rsp
), DMA_TO_DEVICE
);
387 nvmet_rdma_alloc_rsps(struct nvmet_rdma_queue
*queue
)
389 struct nvmet_rdma_device
*ndev
= queue
->dev
;
390 int nr_rsps
= queue
->recv_queue_size
* 2;
391 int ret
= -EINVAL
, i
;
393 queue
->rsps
= kcalloc(nr_rsps
, sizeof(struct nvmet_rdma_rsp
),
398 for (i
= 0; i
< nr_rsps
; i
++) {
399 struct nvmet_rdma_rsp
*rsp
= &queue
->rsps
[i
];
401 ret
= nvmet_rdma_alloc_rsp(ndev
, rsp
);
405 list_add_tail(&rsp
->free_list
, &queue
->free_rsps
);
412 struct nvmet_rdma_rsp
*rsp
= &queue
->rsps
[i
];
414 list_del(&rsp
->free_list
);
415 nvmet_rdma_free_rsp(ndev
, rsp
);
422 static void nvmet_rdma_free_rsps(struct nvmet_rdma_queue
*queue
)
424 struct nvmet_rdma_device
*ndev
= queue
->dev
;
425 int i
, nr_rsps
= queue
->recv_queue_size
* 2;
427 for (i
= 0; i
< nr_rsps
; i
++) {
428 struct nvmet_rdma_rsp
*rsp
= &queue
->rsps
[i
];
430 list_del(&rsp
->free_list
);
431 nvmet_rdma_free_rsp(ndev
, rsp
);
436 static int nvmet_rdma_post_recv(struct nvmet_rdma_device
*ndev
,
437 struct nvmet_rdma_cmd
*cmd
)
439 struct ib_recv_wr
*bad_wr
;
441 ib_dma_sync_single_for_device(ndev
->device
,
442 cmd
->sge
[0].addr
, cmd
->sge
[0].length
,
446 return ib_post_srq_recv(ndev
->srq
, &cmd
->wr
, &bad_wr
);
447 return ib_post_recv(cmd
->queue
->cm_id
->qp
, &cmd
->wr
, &bad_wr
);
450 static void nvmet_rdma_process_wr_wait_list(struct nvmet_rdma_queue
*queue
)
452 spin_lock(&queue
->rsp_wr_wait_lock
);
453 while (!list_empty(&queue
->rsp_wr_wait_list
)) {
454 struct nvmet_rdma_rsp
*rsp
;
457 rsp
= list_entry(queue
->rsp_wr_wait_list
.next
,
458 struct nvmet_rdma_rsp
, wait_list
);
459 list_del(&rsp
->wait_list
);
461 spin_unlock(&queue
->rsp_wr_wait_lock
);
462 ret
= nvmet_rdma_execute_command(rsp
);
463 spin_lock(&queue
->rsp_wr_wait_lock
);
466 list_add(&rsp
->wait_list
, &queue
->rsp_wr_wait_list
);
470 spin_unlock(&queue
->rsp_wr_wait_lock
);
474 static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp
*rsp
)
476 struct nvmet_rdma_queue
*queue
= rsp
->queue
;
478 atomic_add(1 + rsp
->n_rdma
, &queue
->sq_wr_avail
);
481 rdma_rw_ctx_destroy(&rsp
->rw
, queue
->cm_id
->qp
,
482 queue
->cm_id
->port_num
, rsp
->req
.sg
,
483 rsp
->req
.sg_cnt
, nvmet_data_dir(&rsp
->req
));
486 if (rsp
->req
.sg
!= &rsp
->cmd
->inline_sg
)
487 nvmet_rdma_free_sgl(rsp
->req
.sg
, rsp
->req
.sg_cnt
);
489 if (unlikely(!list_empty_careful(&queue
->rsp_wr_wait_list
)))
490 nvmet_rdma_process_wr_wait_list(queue
);
492 nvmet_rdma_put_rsp(rsp
);
495 static void nvmet_rdma_error_comp(struct nvmet_rdma_queue
*queue
)
497 if (queue
->nvme_sq
.ctrl
) {
498 nvmet_ctrl_fatal_error(queue
->nvme_sq
.ctrl
);
501 * we didn't setup the controller yet in case
502 * of admin connect error, just disconnect and
505 nvmet_rdma_queue_disconnect(queue
);
509 static void nvmet_rdma_send_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
511 struct nvmet_rdma_rsp
*rsp
=
512 container_of(wc
->wr_cqe
, struct nvmet_rdma_rsp
, send_cqe
);
514 nvmet_rdma_release_rsp(rsp
);
516 if (unlikely(wc
->status
!= IB_WC_SUCCESS
&&
517 wc
->status
!= IB_WC_WR_FLUSH_ERR
)) {
518 pr_err("SEND for CQE 0x%p failed with status %s (%d).\n",
519 wc
->wr_cqe
, ib_wc_status_msg(wc
->status
), wc
->status
);
520 nvmet_rdma_error_comp(rsp
->queue
);
524 static void nvmet_rdma_queue_response(struct nvmet_req
*req
)
526 struct nvmet_rdma_rsp
*rsp
=
527 container_of(req
, struct nvmet_rdma_rsp
, req
);
528 struct rdma_cm_id
*cm_id
= rsp
->queue
->cm_id
;
529 struct ib_send_wr
*first_wr
, *bad_wr
;
531 if (rsp
->flags
& NVMET_RDMA_REQ_INVALIDATE_RKEY
) {
532 rsp
->send_wr
.opcode
= IB_WR_SEND_WITH_INV
;
533 rsp
->send_wr
.ex
.invalidate_rkey
= rsp
->invalidate_rkey
;
535 rsp
->send_wr
.opcode
= IB_WR_SEND
;
538 if (nvmet_rdma_need_data_out(rsp
))
539 first_wr
= rdma_rw_ctx_wrs(&rsp
->rw
, cm_id
->qp
,
540 cm_id
->port_num
, NULL
, &rsp
->send_wr
);
542 first_wr
= &rsp
->send_wr
;
544 nvmet_rdma_post_recv(rsp
->queue
->dev
, rsp
->cmd
);
546 ib_dma_sync_single_for_device(rsp
->queue
->dev
->device
,
547 rsp
->send_sge
.addr
, rsp
->send_sge
.length
,
550 if (ib_post_send(cm_id
->qp
, first_wr
, &bad_wr
)) {
551 pr_err("sending cmd response failed\n");
552 nvmet_rdma_release_rsp(rsp
);
556 static void nvmet_rdma_read_data_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
558 struct nvmet_rdma_rsp
*rsp
=
559 container_of(wc
->wr_cqe
, struct nvmet_rdma_rsp
, read_cqe
);
560 struct nvmet_rdma_queue
*queue
= cq
->cq_context
;
562 WARN_ON(rsp
->n_rdma
<= 0);
563 atomic_add(rsp
->n_rdma
, &queue
->sq_wr_avail
);
564 rdma_rw_ctx_destroy(&rsp
->rw
, queue
->cm_id
->qp
,
565 queue
->cm_id
->port_num
, rsp
->req
.sg
,
566 rsp
->req
.sg_cnt
, nvmet_data_dir(&rsp
->req
));
569 if (unlikely(wc
->status
!= IB_WC_SUCCESS
)) {
570 nvmet_req_uninit(&rsp
->req
);
571 nvmet_rdma_release_rsp(rsp
);
572 if (wc
->status
!= IB_WC_WR_FLUSH_ERR
) {
573 pr_info("RDMA READ for CQE 0x%p failed with status %s (%d).\n",
574 wc
->wr_cqe
, ib_wc_status_msg(wc
->status
), wc
->status
);
575 nvmet_rdma_error_comp(queue
);
580 rsp
->req
.execute(&rsp
->req
);
583 static void nvmet_rdma_use_inline_sg(struct nvmet_rdma_rsp
*rsp
, u32 len
,
586 sg_init_table(&rsp
->cmd
->inline_sg
, 1);
587 sg_set_page(&rsp
->cmd
->inline_sg
, rsp
->cmd
->inline_page
, len
, off
);
588 rsp
->req
.sg
= &rsp
->cmd
->inline_sg
;
592 static u16
nvmet_rdma_map_sgl_inline(struct nvmet_rdma_rsp
*rsp
)
594 struct nvme_sgl_desc
*sgl
= &rsp
->req
.cmd
->common
.dptr
.sgl
;
595 u64 off
= le64_to_cpu(sgl
->addr
);
596 u32 len
= le32_to_cpu(sgl
->length
);
598 if (!nvme_is_write(rsp
->req
.cmd
))
599 return NVME_SC_INVALID_FIELD
| NVME_SC_DNR
;
601 if (off
+ len
> NVMET_RDMA_INLINE_DATA_SIZE
) {
602 pr_err("invalid inline data offset!\n");
603 return NVME_SC_SGL_INVALID_OFFSET
| NVME_SC_DNR
;
606 /* no data command? */
610 nvmet_rdma_use_inline_sg(rsp
, len
, off
);
611 rsp
->flags
|= NVMET_RDMA_REQ_INLINE_DATA
;
615 static u16
nvmet_rdma_map_sgl_keyed(struct nvmet_rdma_rsp
*rsp
,
616 struct nvme_keyed_sgl_desc
*sgl
, bool invalidate
)
618 struct rdma_cm_id
*cm_id
= rsp
->queue
->cm_id
;
619 u64 addr
= le64_to_cpu(sgl
->addr
);
620 u32 len
= get_unaligned_le24(sgl
->length
);
621 u32 key
= get_unaligned_le32(sgl
->key
);
625 /* no data command? */
629 status
= nvmet_rdma_alloc_sgl(&rsp
->req
.sg
, &rsp
->req
.sg_cnt
,
634 ret
= rdma_rw_ctx_init(&rsp
->rw
, cm_id
->qp
, cm_id
->port_num
,
635 rsp
->req
.sg
, rsp
->req
.sg_cnt
, 0, addr
, key
,
636 nvmet_data_dir(&rsp
->req
));
638 return NVME_SC_INTERNAL
;
642 rsp
->invalidate_rkey
= key
;
643 rsp
->flags
|= NVMET_RDMA_REQ_INVALIDATE_RKEY
;
649 static u16
nvmet_rdma_map_sgl(struct nvmet_rdma_rsp
*rsp
)
651 struct nvme_keyed_sgl_desc
*sgl
= &rsp
->req
.cmd
->common
.dptr
.ksgl
;
653 switch (sgl
->type
>> 4) {
654 case NVME_SGL_FMT_DATA_DESC
:
655 switch (sgl
->type
& 0xf) {
656 case NVME_SGL_FMT_OFFSET
:
657 return nvmet_rdma_map_sgl_inline(rsp
);
659 pr_err("invalid SGL subtype: %#x\n", sgl
->type
);
660 return NVME_SC_INVALID_FIELD
| NVME_SC_DNR
;
662 case NVME_KEY_SGL_FMT_DATA_DESC
:
663 switch (sgl
->type
& 0xf) {
664 case NVME_SGL_FMT_ADDRESS
| NVME_SGL_FMT_INVALIDATE
:
665 return nvmet_rdma_map_sgl_keyed(rsp
, sgl
, true);
666 case NVME_SGL_FMT_ADDRESS
:
667 return nvmet_rdma_map_sgl_keyed(rsp
, sgl
, false);
669 pr_err("invalid SGL subtype: %#x\n", sgl
->type
);
670 return NVME_SC_INVALID_FIELD
| NVME_SC_DNR
;
673 pr_err("invalid SGL type: %#x\n", sgl
->type
);
674 return NVME_SC_SGL_INVALID_TYPE
| NVME_SC_DNR
;
678 static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp
*rsp
)
680 struct nvmet_rdma_queue
*queue
= rsp
->queue
;
682 if (unlikely(atomic_sub_return(1 + rsp
->n_rdma
,
683 &queue
->sq_wr_avail
) < 0)) {
684 pr_debug("IB send queue full (needed %d): queue %u cntlid %u\n",
685 1 + rsp
->n_rdma
, queue
->idx
,
686 queue
->nvme_sq
.ctrl
->cntlid
);
687 atomic_add(1 + rsp
->n_rdma
, &queue
->sq_wr_avail
);
691 if (nvmet_rdma_need_data_in(rsp
)) {
692 if (rdma_rw_ctx_post(&rsp
->rw
, queue
->cm_id
->qp
,
693 queue
->cm_id
->port_num
, &rsp
->read_cqe
, NULL
))
694 nvmet_req_complete(&rsp
->req
, NVME_SC_DATA_XFER_ERROR
);
696 rsp
->req
.execute(&rsp
->req
);
702 static void nvmet_rdma_handle_command(struct nvmet_rdma_queue
*queue
,
703 struct nvmet_rdma_rsp
*cmd
)
707 ib_dma_sync_single_for_cpu(queue
->dev
->device
,
708 cmd
->cmd
->sge
[0].addr
, cmd
->cmd
->sge
[0].length
,
710 ib_dma_sync_single_for_cpu(queue
->dev
->device
,
711 cmd
->send_sge
.addr
, cmd
->send_sge
.length
,
714 if (!nvmet_req_init(&cmd
->req
, &queue
->nvme_cq
,
715 &queue
->nvme_sq
, &nvmet_rdma_ops
))
718 status
= nvmet_rdma_map_sgl(cmd
);
722 if (unlikely(!nvmet_rdma_execute_command(cmd
))) {
723 spin_lock(&queue
->rsp_wr_wait_lock
);
724 list_add_tail(&cmd
->wait_list
, &queue
->rsp_wr_wait_list
);
725 spin_unlock(&queue
->rsp_wr_wait_lock
);
731 nvmet_req_complete(&cmd
->req
, status
);
734 static void nvmet_rdma_recv_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
736 struct nvmet_rdma_cmd
*cmd
=
737 container_of(wc
->wr_cqe
, struct nvmet_rdma_cmd
, cqe
);
738 struct nvmet_rdma_queue
*queue
= cq
->cq_context
;
739 struct nvmet_rdma_rsp
*rsp
;
741 if (unlikely(wc
->status
!= IB_WC_SUCCESS
)) {
742 if (wc
->status
!= IB_WC_WR_FLUSH_ERR
) {
743 pr_err("RECV for CQE 0x%p failed with status %s (%d)\n",
744 wc
->wr_cqe
, ib_wc_status_msg(wc
->status
),
746 nvmet_rdma_error_comp(queue
);
751 if (unlikely(wc
->byte_len
< sizeof(struct nvme_command
))) {
752 pr_err("Ctrl Fatal Error: capsule size less than 64 bytes\n");
753 nvmet_rdma_error_comp(queue
);
758 rsp
= nvmet_rdma_get_rsp(queue
);
762 rsp
->req
.cmd
= cmd
->nvme_cmd
;
763 rsp
->req
.port
= queue
->port
;
766 if (unlikely(queue
->state
!= NVMET_RDMA_Q_LIVE
)) {
769 spin_lock_irqsave(&queue
->state_lock
, flags
);
770 if (queue
->state
== NVMET_RDMA_Q_CONNECTING
)
771 list_add_tail(&rsp
->wait_list
, &queue
->rsp_wait_list
);
773 nvmet_rdma_put_rsp(rsp
);
774 spin_unlock_irqrestore(&queue
->state_lock
, flags
);
778 nvmet_rdma_handle_command(queue
, rsp
);
781 static void nvmet_rdma_destroy_srq(struct nvmet_rdma_device
*ndev
)
786 nvmet_rdma_free_cmds(ndev
, ndev
->srq_cmds
, ndev
->srq_size
, false);
787 ib_destroy_srq(ndev
->srq
);
790 static int nvmet_rdma_init_srq(struct nvmet_rdma_device
*ndev
)
792 struct ib_srq_init_attr srq_attr
= { NULL
, };
797 srq_size
= 4095; /* XXX: tune */
799 srq_attr
.attr
.max_wr
= srq_size
;
800 srq_attr
.attr
.max_sge
= 2;
801 srq_attr
.attr
.srq_limit
= 0;
802 srq_attr
.srq_type
= IB_SRQT_BASIC
;
803 srq
= ib_create_srq(ndev
->pd
, &srq_attr
);
806 * If SRQs aren't supported we just go ahead and use normal
807 * non-shared receive queues.
809 pr_info("SRQ requested but not supported.\n");
813 ndev
->srq_cmds
= nvmet_rdma_alloc_cmds(ndev
, srq_size
, false);
814 if (IS_ERR(ndev
->srq_cmds
)) {
815 ret
= PTR_ERR(ndev
->srq_cmds
);
816 goto out_destroy_srq
;
820 ndev
->srq_size
= srq_size
;
822 for (i
= 0; i
< srq_size
; i
++)
823 nvmet_rdma_post_recv(ndev
, &ndev
->srq_cmds
[i
]);
832 static void nvmet_rdma_free_dev(struct kref
*ref
)
834 struct nvmet_rdma_device
*ndev
=
835 container_of(ref
, struct nvmet_rdma_device
, ref
);
837 mutex_lock(&device_list_mutex
);
838 list_del(&ndev
->entry
);
839 mutex_unlock(&device_list_mutex
);
841 nvmet_rdma_destroy_srq(ndev
);
842 ib_dealloc_pd(ndev
->pd
);
847 static struct nvmet_rdma_device
*
848 nvmet_rdma_find_get_device(struct rdma_cm_id
*cm_id
)
850 struct nvmet_rdma_device
*ndev
;
853 mutex_lock(&device_list_mutex
);
854 list_for_each_entry(ndev
, &device_list
, entry
) {
855 if (ndev
->device
->node_guid
== cm_id
->device
->node_guid
&&
856 kref_get_unless_zero(&ndev
->ref
))
860 ndev
= kzalloc(sizeof(*ndev
), GFP_KERNEL
);
864 ndev
->device
= cm_id
->device
;
865 kref_init(&ndev
->ref
);
867 ndev
->pd
= ib_alloc_pd(ndev
->device
, 0);
868 if (IS_ERR(ndev
->pd
))
871 if (nvmet_rdma_use_srq
) {
872 ret
= nvmet_rdma_init_srq(ndev
);
877 list_add(&ndev
->entry
, &device_list
);
879 mutex_unlock(&device_list_mutex
);
880 pr_debug("added %s.\n", ndev
->device
->name
);
884 ib_dealloc_pd(ndev
->pd
);
888 mutex_unlock(&device_list_mutex
);
892 static int nvmet_rdma_create_queue_ib(struct nvmet_rdma_queue
*queue
)
894 struct ib_qp_init_attr qp_attr
;
895 struct nvmet_rdma_device
*ndev
= queue
->dev
;
896 int comp_vector
, nr_cqe
, ret
, i
;
899 * Spread the io queues across completion vectors,
900 * but still keep all admin queues on vector 0.
902 comp_vector
= !queue
->host_qid
? 0 :
903 queue
->idx
% ndev
->device
->num_comp_vectors
;
906 * Reserve CQ slots for RECV + RDMA_READ/RDMA_WRITE + RDMA_SEND.
908 nr_cqe
= queue
->recv_queue_size
+ 2 * queue
->send_queue_size
;
910 queue
->cq
= ib_alloc_cq(ndev
->device
, queue
,
911 nr_cqe
+ 1, comp_vector
,
913 if (IS_ERR(queue
->cq
)) {
914 ret
= PTR_ERR(queue
->cq
);
915 pr_err("failed to create CQ cqe= %d ret= %d\n",
920 memset(&qp_attr
, 0, sizeof(qp_attr
));
921 qp_attr
.qp_context
= queue
;
922 qp_attr
.event_handler
= nvmet_rdma_qp_event
;
923 qp_attr
.send_cq
= queue
->cq
;
924 qp_attr
.recv_cq
= queue
->cq
;
925 qp_attr
.sq_sig_type
= IB_SIGNAL_REQ_WR
;
926 qp_attr
.qp_type
= IB_QPT_RC
;
928 qp_attr
.cap
.max_send_wr
= queue
->send_queue_size
+ 1;
929 qp_attr
.cap
.max_rdma_ctxs
= queue
->send_queue_size
;
930 qp_attr
.cap
.max_send_sge
= max(ndev
->device
->attrs
.max_sge_rd
,
931 ndev
->device
->attrs
.max_sge
);
934 qp_attr
.srq
= ndev
->srq
;
937 qp_attr
.cap
.max_recv_wr
= 1 + queue
->recv_queue_size
;
938 qp_attr
.cap
.max_recv_sge
= 2;
941 ret
= rdma_create_qp(queue
->cm_id
, ndev
->pd
, &qp_attr
);
943 pr_err("failed to create_qp ret= %d\n", ret
);
947 atomic_set(&queue
->sq_wr_avail
, qp_attr
.cap
.max_send_wr
);
949 pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d cm_id= %p\n",
950 __func__
, queue
->cq
->cqe
, qp_attr
.cap
.max_send_sge
,
951 qp_attr
.cap
.max_send_wr
, queue
->cm_id
);
954 for (i
= 0; i
< queue
->recv_queue_size
; i
++) {
955 queue
->cmds
[i
].queue
= queue
;
956 nvmet_rdma_post_recv(ndev
, &queue
->cmds
[i
]);
964 ib_free_cq(queue
->cq
);
968 static void nvmet_rdma_destroy_queue_ib(struct nvmet_rdma_queue
*queue
)
970 ib_drain_qp(queue
->cm_id
->qp
);
971 rdma_destroy_qp(queue
->cm_id
);
972 ib_free_cq(queue
->cq
);
975 static void nvmet_rdma_free_queue(struct nvmet_rdma_queue
*queue
)
977 pr_info("freeing queue %d\n", queue
->idx
);
979 nvmet_sq_destroy(&queue
->nvme_sq
);
981 nvmet_rdma_destroy_queue_ib(queue
);
982 if (!queue
->dev
->srq
) {
983 nvmet_rdma_free_cmds(queue
->dev
, queue
->cmds
,
984 queue
->recv_queue_size
,
987 nvmet_rdma_free_rsps(queue
);
988 ida_simple_remove(&nvmet_rdma_queue_ida
, queue
->idx
);
992 static void nvmet_rdma_release_queue_work(struct work_struct
*w
)
994 struct nvmet_rdma_queue
*queue
=
995 container_of(w
, struct nvmet_rdma_queue
, release_work
);
996 struct rdma_cm_id
*cm_id
= queue
->cm_id
;
997 struct nvmet_rdma_device
*dev
= queue
->dev
;
998 enum nvmet_rdma_queue_state state
= queue
->state
;
1000 nvmet_rdma_free_queue(queue
);
1002 if (state
!= NVMET_RDMA_IN_DEVICE_REMOVAL
)
1003 rdma_destroy_id(cm_id
);
1005 kref_put(&dev
->ref
, nvmet_rdma_free_dev
);
1009 nvmet_rdma_parse_cm_connect_req(struct rdma_conn_param
*conn
,
1010 struct nvmet_rdma_queue
*queue
)
1012 struct nvme_rdma_cm_req
*req
;
1014 req
= (struct nvme_rdma_cm_req
*)conn
->private_data
;
1015 if (!req
|| conn
->private_data_len
== 0)
1016 return NVME_RDMA_CM_INVALID_LEN
;
1018 if (le16_to_cpu(req
->recfmt
) != NVME_RDMA_CM_FMT_1_0
)
1019 return NVME_RDMA_CM_INVALID_RECFMT
;
1021 queue
->host_qid
= le16_to_cpu(req
->qid
);
1024 * req->hsqsize corresponds to our recv queue size plus 1
1025 * req->hrqsize corresponds to our send queue size
1027 queue
->recv_queue_size
= le16_to_cpu(req
->hsqsize
) + 1;
1028 queue
->send_queue_size
= le16_to_cpu(req
->hrqsize
);
1030 if (!queue
->host_qid
&& queue
->recv_queue_size
> NVME_AQ_DEPTH
)
1031 return NVME_RDMA_CM_INVALID_HSQSIZE
;
1033 /* XXX: Should we enforce some kind of max for IO queues? */
1038 static int nvmet_rdma_cm_reject(struct rdma_cm_id
*cm_id
,
1039 enum nvme_rdma_cm_status status
)
1041 struct nvme_rdma_cm_rej rej
;
1043 pr_debug("rejecting connect request: status %d (%s)\n",
1044 status
, nvme_rdma_cm_msg(status
));
1046 rej
.recfmt
= cpu_to_le16(NVME_RDMA_CM_FMT_1_0
);
1047 rej
.sts
= cpu_to_le16(status
);
1049 return rdma_reject(cm_id
, (void *)&rej
, sizeof(rej
));
1052 static struct nvmet_rdma_queue
*
1053 nvmet_rdma_alloc_queue(struct nvmet_rdma_device
*ndev
,
1054 struct rdma_cm_id
*cm_id
,
1055 struct rdma_cm_event
*event
)
1057 struct nvmet_rdma_queue
*queue
;
1060 queue
= kzalloc(sizeof(*queue
), GFP_KERNEL
);
1062 ret
= NVME_RDMA_CM_NO_RSC
;
1066 ret
= nvmet_sq_init(&queue
->nvme_sq
);
1068 ret
= NVME_RDMA_CM_NO_RSC
;
1069 goto out_free_queue
;
1072 ret
= nvmet_rdma_parse_cm_connect_req(&event
->param
.conn
, queue
);
1074 goto out_destroy_sq
;
1077 * Schedules the actual release because calling rdma_destroy_id from
1078 * inside a CM callback would trigger a deadlock. (great API design..)
1080 INIT_WORK(&queue
->release_work
, nvmet_rdma_release_queue_work
);
1082 queue
->cm_id
= cm_id
;
1084 spin_lock_init(&queue
->state_lock
);
1085 queue
->state
= NVMET_RDMA_Q_CONNECTING
;
1086 INIT_LIST_HEAD(&queue
->rsp_wait_list
);
1087 INIT_LIST_HEAD(&queue
->rsp_wr_wait_list
);
1088 spin_lock_init(&queue
->rsp_wr_wait_lock
);
1089 INIT_LIST_HEAD(&queue
->free_rsps
);
1090 spin_lock_init(&queue
->rsps_lock
);
1091 INIT_LIST_HEAD(&queue
->queue_list
);
1093 queue
->idx
= ida_simple_get(&nvmet_rdma_queue_ida
, 0, 0, GFP_KERNEL
);
1094 if (queue
->idx
< 0) {
1095 ret
= NVME_RDMA_CM_NO_RSC
;
1096 goto out_destroy_sq
;
1099 ret
= nvmet_rdma_alloc_rsps(queue
);
1101 ret
= NVME_RDMA_CM_NO_RSC
;
1102 goto out_ida_remove
;
1106 queue
->cmds
= nvmet_rdma_alloc_cmds(ndev
,
1107 queue
->recv_queue_size
,
1109 if (IS_ERR(queue
->cmds
)) {
1110 ret
= NVME_RDMA_CM_NO_RSC
;
1111 goto out_free_responses
;
1115 ret
= nvmet_rdma_create_queue_ib(queue
);
1117 pr_err("%s: creating RDMA queue failed (%d).\n",
1119 ret
= NVME_RDMA_CM_NO_RSC
;
1127 nvmet_rdma_free_cmds(queue
->dev
, queue
->cmds
,
1128 queue
->recv_queue_size
,
1132 nvmet_rdma_free_rsps(queue
);
1134 ida_simple_remove(&nvmet_rdma_queue_ida
, queue
->idx
);
1136 nvmet_sq_destroy(&queue
->nvme_sq
);
1140 nvmet_rdma_cm_reject(cm_id
, ret
);
1144 static void nvmet_rdma_qp_event(struct ib_event
*event
, void *priv
)
1146 struct nvmet_rdma_queue
*queue
= priv
;
1148 switch (event
->event
) {
1149 case IB_EVENT_COMM_EST
:
1150 rdma_notify(queue
->cm_id
, event
->event
);
1153 pr_err("received IB QP event: %s (%d)\n",
1154 ib_event_msg(event
->event
), event
->event
);
1159 static int nvmet_rdma_cm_accept(struct rdma_cm_id
*cm_id
,
1160 struct nvmet_rdma_queue
*queue
,
1161 struct rdma_conn_param
*p
)
1163 struct rdma_conn_param param
= { };
1164 struct nvme_rdma_cm_rep priv
= { };
1167 param
.rnr_retry_count
= 7;
1168 param
.flow_control
= 1;
1169 param
.initiator_depth
= min_t(u8
, p
->initiator_depth
,
1170 queue
->dev
->device
->attrs
.max_qp_init_rd_atom
);
1171 param
.private_data
= &priv
;
1172 param
.private_data_len
= sizeof(priv
);
1173 priv
.recfmt
= cpu_to_le16(NVME_RDMA_CM_FMT_1_0
);
1174 priv
.crqsize
= cpu_to_le16(queue
->recv_queue_size
);
1176 ret
= rdma_accept(cm_id
, ¶m
);
1178 pr_err("rdma_accept failed (error code = %d)\n", ret
);
1183 static int nvmet_rdma_queue_connect(struct rdma_cm_id
*cm_id
,
1184 struct rdma_cm_event
*event
)
1186 struct nvmet_rdma_device
*ndev
;
1187 struct nvmet_rdma_queue
*queue
;
1190 ndev
= nvmet_rdma_find_get_device(cm_id
);
1192 nvmet_rdma_cm_reject(cm_id
, NVME_RDMA_CM_NO_RSC
);
1193 return -ECONNREFUSED
;
1196 queue
= nvmet_rdma_alloc_queue(ndev
, cm_id
, event
);
1201 queue
->port
= cm_id
->context
;
1203 if (queue
->host_qid
== 0) {
1204 /* Let inflight controller teardown complete */
1205 flush_scheduled_work();
1208 ret
= nvmet_rdma_cm_accept(cm_id
, queue
, &event
->param
.conn
);
1212 mutex_lock(&nvmet_rdma_queue_mutex
);
1213 list_add_tail(&queue
->queue_list
, &nvmet_rdma_queue_list
);
1214 mutex_unlock(&nvmet_rdma_queue_mutex
);
1219 nvmet_rdma_free_queue(queue
);
1221 kref_put(&ndev
->ref
, nvmet_rdma_free_dev
);
1226 static void nvmet_rdma_queue_established(struct nvmet_rdma_queue
*queue
)
1228 unsigned long flags
;
1230 spin_lock_irqsave(&queue
->state_lock
, flags
);
1231 if (queue
->state
!= NVMET_RDMA_Q_CONNECTING
) {
1232 pr_warn("trying to establish a connected queue\n");
1235 queue
->state
= NVMET_RDMA_Q_LIVE
;
1237 while (!list_empty(&queue
->rsp_wait_list
)) {
1238 struct nvmet_rdma_rsp
*cmd
;
1240 cmd
= list_first_entry(&queue
->rsp_wait_list
,
1241 struct nvmet_rdma_rsp
, wait_list
);
1242 list_del(&cmd
->wait_list
);
1244 spin_unlock_irqrestore(&queue
->state_lock
, flags
);
1245 nvmet_rdma_handle_command(queue
, cmd
);
1246 spin_lock_irqsave(&queue
->state_lock
, flags
);
1250 spin_unlock_irqrestore(&queue
->state_lock
, flags
);
1253 static void __nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue
*queue
)
1255 bool disconnect
= false;
1256 unsigned long flags
;
1258 pr_debug("cm_id= %p queue->state= %d\n", queue
->cm_id
, queue
->state
);
1260 spin_lock_irqsave(&queue
->state_lock
, flags
);
1261 switch (queue
->state
) {
1262 case NVMET_RDMA_Q_CONNECTING
:
1263 case NVMET_RDMA_Q_LIVE
:
1264 queue
->state
= NVMET_RDMA_Q_DISCONNECTING
;
1265 case NVMET_RDMA_IN_DEVICE_REMOVAL
:
1268 case NVMET_RDMA_Q_DISCONNECTING
:
1271 spin_unlock_irqrestore(&queue
->state_lock
, flags
);
1274 rdma_disconnect(queue
->cm_id
);
1275 schedule_work(&queue
->release_work
);
1279 static void nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue
*queue
)
1281 bool disconnect
= false;
1283 mutex_lock(&nvmet_rdma_queue_mutex
);
1284 if (!list_empty(&queue
->queue_list
)) {
1285 list_del_init(&queue
->queue_list
);
1288 mutex_unlock(&nvmet_rdma_queue_mutex
);
1291 __nvmet_rdma_queue_disconnect(queue
);
1294 static void nvmet_rdma_queue_connect_fail(struct rdma_cm_id
*cm_id
,
1295 struct nvmet_rdma_queue
*queue
)
1297 WARN_ON_ONCE(queue
->state
!= NVMET_RDMA_Q_CONNECTING
);
1299 mutex_lock(&nvmet_rdma_queue_mutex
);
1300 if (!list_empty(&queue
->queue_list
))
1301 list_del_init(&queue
->queue_list
);
1302 mutex_unlock(&nvmet_rdma_queue_mutex
);
1304 pr_err("failed to connect queue %d\n", queue
->idx
);
1305 schedule_work(&queue
->release_work
);
1309 * nvme_rdma_device_removal() - Handle RDMA device removal
1310 * @cm_id: rdma_cm id, used for nvmet port
1311 * @queue: nvmet rdma queue (cm id qp_context)
1313 * DEVICE_REMOVAL event notifies us that the RDMA device is about
1314 * to unplug. Note that this event can be generated on a normal
1315 * queue cm_id and/or a device bound listener cm_id (where in this
1316 * case queue will be null).
1318 * We registered an ib_client to handle device removal for queues,
1319 * so we only need to handle the listening port cm_ids. In this case
1320 * we nullify the priv to prevent double cm_id destruction and destroying
1321 * the cm_id implicitely by returning a non-zero rc to the callout.
1323 static int nvmet_rdma_device_removal(struct rdma_cm_id
*cm_id
,
1324 struct nvmet_rdma_queue
*queue
)
1326 struct nvmet_port
*port
;
1330 * This is a queue cm_id. we have registered
1331 * an ib_client to handle queues removal
1332 * so don't interfear and just return.
1337 port
= cm_id
->context
;
1340 * This is a listener cm_id. Make sure that
1341 * future remove_port won't invoke a double
1342 * cm_id destroy. use atomic xchg to make sure
1343 * we don't compete with remove_port.
1345 if (xchg(&port
->priv
, NULL
) != cm_id
)
1349 * We need to return 1 so that the core will destroy
1350 * it's own ID. What a great API design..
1355 static int nvmet_rdma_cm_handler(struct rdma_cm_id
*cm_id
,
1356 struct rdma_cm_event
*event
)
1358 struct nvmet_rdma_queue
*queue
= NULL
;
1362 queue
= cm_id
->qp
->qp_context
;
1364 pr_debug("%s (%d): status %d id %p\n",
1365 rdma_event_msg(event
->event
), event
->event
,
1366 event
->status
, cm_id
);
1368 switch (event
->event
) {
1369 case RDMA_CM_EVENT_CONNECT_REQUEST
:
1370 ret
= nvmet_rdma_queue_connect(cm_id
, event
);
1372 case RDMA_CM_EVENT_ESTABLISHED
:
1373 nvmet_rdma_queue_established(queue
);
1375 case RDMA_CM_EVENT_ADDR_CHANGE
:
1376 case RDMA_CM_EVENT_DISCONNECTED
:
1377 case RDMA_CM_EVENT_TIMEWAIT_EXIT
:
1379 * We might end up here when we already freed the qp
1380 * which means queue release sequence is in progress,
1381 * so don't get in the way...
1384 nvmet_rdma_queue_disconnect(queue
);
1386 case RDMA_CM_EVENT_DEVICE_REMOVAL
:
1387 ret
= nvmet_rdma_device_removal(cm_id
, queue
);
1389 case RDMA_CM_EVENT_REJECTED
:
1390 pr_debug("Connection rejected: %s\n",
1391 rdma_reject_msg(cm_id
, event
->status
));
1393 case RDMA_CM_EVENT_UNREACHABLE
:
1394 case RDMA_CM_EVENT_CONNECT_ERROR
:
1395 nvmet_rdma_queue_connect_fail(cm_id
, queue
);
1398 pr_err("received unrecognized RDMA CM event %d\n",
1406 static void nvmet_rdma_delete_ctrl(struct nvmet_ctrl
*ctrl
)
1408 struct nvmet_rdma_queue
*queue
;
1411 mutex_lock(&nvmet_rdma_queue_mutex
);
1412 list_for_each_entry(queue
, &nvmet_rdma_queue_list
, queue_list
) {
1413 if (queue
->nvme_sq
.ctrl
== ctrl
) {
1414 list_del_init(&queue
->queue_list
);
1415 mutex_unlock(&nvmet_rdma_queue_mutex
);
1417 __nvmet_rdma_queue_disconnect(queue
);
1421 mutex_unlock(&nvmet_rdma_queue_mutex
);
1424 static int nvmet_rdma_add_port(struct nvmet_port
*port
)
1426 struct rdma_cm_id
*cm_id
;
1427 struct sockaddr_storage addr
= { };
1428 __kernel_sa_family_t af
;
1431 switch (port
->disc_addr
.adrfam
) {
1432 case NVMF_ADDR_FAMILY_IP4
:
1435 case NVMF_ADDR_FAMILY_IP6
:
1439 pr_err("address family %d not supported\n",
1440 port
->disc_addr
.adrfam
);
1444 ret
= inet_pton_with_scope(&init_net
, af
, port
->disc_addr
.traddr
,
1445 port
->disc_addr
.trsvcid
, &addr
);
1447 pr_err("malformed ip/port passed: %s:%s\n",
1448 port
->disc_addr
.traddr
, port
->disc_addr
.trsvcid
);
1452 cm_id
= rdma_create_id(&init_net
, nvmet_rdma_cm_handler
, port
,
1453 RDMA_PS_TCP
, IB_QPT_RC
);
1454 if (IS_ERR(cm_id
)) {
1455 pr_err("CM ID creation failed\n");
1456 return PTR_ERR(cm_id
);
1460 * Allow both IPv4 and IPv6 sockets to bind a single port
1463 ret
= rdma_set_afonly(cm_id
, 1);
1465 pr_err("rdma_set_afonly failed (%d)\n", ret
);
1466 goto out_destroy_id
;
1469 ret
= rdma_bind_addr(cm_id
, (struct sockaddr
*)&addr
);
1471 pr_err("binding CM ID to %pISpcs failed (%d)\n",
1472 (struct sockaddr
*)&addr
, ret
);
1473 goto out_destroy_id
;
1476 ret
= rdma_listen(cm_id
, 128);
1478 pr_err("listening to %pISpcs failed (%d)\n",
1479 (struct sockaddr
*)&addr
, ret
);
1480 goto out_destroy_id
;
1483 pr_info("enabling port %d (%pISpcs)\n",
1484 le16_to_cpu(port
->disc_addr
.portid
), (struct sockaddr
*)&addr
);
1489 rdma_destroy_id(cm_id
);
1493 static void nvmet_rdma_remove_port(struct nvmet_port
*port
)
1495 struct rdma_cm_id
*cm_id
= xchg(&port
->priv
, NULL
);
1498 rdma_destroy_id(cm_id
);
1501 static struct nvmet_fabrics_ops nvmet_rdma_ops
= {
1502 .owner
= THIS_MODULE
,
1503 .type
= NVMF_TRTYPE_RDMA
,
1504 .sqe_inline_size
= NVMET_RDMA_INLINE_DATA_SIZE
,
1506 .has_keyed_sgls
= 1,
1507 .add_port
= nvmet_rdma_add_port
,
1508 .remove_port
= nvmet_rdma_remove_port
,
1509 .queue_response
= nvmet_rdma_queue_response
,
1510 .delete_ctrl
= nvmet_rdma_delete_ctrl
,
1513 static void nvmet_rdma_add_one(struct ib_device
*ib_device
)
1517 static void nvmet_rdma_remove_one(struct ib_device
*ib_device
, void *client_data
)
1519 struct nvmet_rdma_queue
*queue
;
1521 /* Device is being removed, delete all queues using this device */
1522 mutex_lock(&nvmet_rdma_queue_mutex
);
1523 list_for_each_entry(queue
, &nvmet_rdma_queue_list
, queue_list
) {
1524 if (queue
->dev
->device
!= ib_device
)
1527 pr_info("Removing queue %d\n", queue
->idx
);
1528 __nvmet_rdma_queue_disconnect(queue
);
1530 mutex_unlock(&nvmet_rdma_queue_mutex
);
1532 flush_scheduled_work();
1535 static struct ib_client nvmet_rdma_ib_client
= {
1536 .name
= "nvmet_rdma",
1537 .add
= nvmet_rdma_add_one
,
1538 .remove
= nvmet_rdma_remove_one
1541 static int __init
nvmet_rdma_init(void)
1545 ret
= ib_register_client(&nvmet_rdma_ib_client
);
1549 ret
= nvmet_register_transport(&nvmet_rdma_ops
);
1556 ib_unregister_client(&nvmet_rdma_ib_client
);
1560 static void __exit
nvmet_rdma_exit(void)
1562 struct nvmet_rdma_queue
*queue
;
1564 nvmet_unregister_transport(&nvmet_rdma_ops
);
1566 flush_scheduled_work();
1568 mutex_lock(&nvmet_rdma_queue_mutex
);
1569 while ((queue
= list_first_entry_or_null(&nvmet_rdma_queue_list
,
1570 struct nvmet_rdma_queue
, queue_list
))) {
1571 list_del_init(&queue
->queue_list
);
1573 mutex_unlock(&nvmet_rdma_queue_mutex
);
1574 __nvmet_rdma_queue_disconnect(queue
);
1575 mutex_lock(&nvmet_rdma_queue_mutex
);
1577 mutex_unlock(&nvmet_rdma_queue_mutex
);
1579 flush_scheduled_work();
1580 ib_unregister_client(&nvmet_rdma_ib_client
);
1581 ida_destroy(&nvmet_rdma_queue_ida
);
1584 module_init(nvmet_rdma_init
);
1585 module_exit(nvmet_rdma_exit
);
1587 MODULE_LICENSE("GPL v2");
1588 MODULE_ALIAS("nvmet-transport-1"); /* 1 == NVMF_TRTYPE_RDMA */