1 // SPDX-License-Identifier: GPL-2.0
3 * NVMe over Fabrics RDMA target.
4 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 #include <linux/atomic.h>
8 #include <linux/ctype.h>
9 #include <linux/delay.h>
10 #include <linux/err.h>
11 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/nvme.h>
14 #include <linux/slab.h>
15 #include <linux/string.h>
16 #include <linux/wait.h>
17 #include <linux/inet.h>
18 #include <asm/unaligned.h>
20 #include <rdma/ib_verbs.h>
21 #include <rdma/rdma_cm.h>
23 #include <rdma/ib_cm.h>
25 #include <linux/nvme-rdma.h>
29 * We allow at least 1 page, up to 4 SGEs, and up to 16KB of inline data
31 #define NVMET_RDMA_DEFAULT_INLINE_DATA_SIZE PAGE_SIZE
32 #define NVMET_RDMA_MAX_INLINE_SGE 4
33 #define NVMET_RDMA_MAX_INLINE_DATA_SIZE max_t(int, SZ_16K, PAGE_SIZE)
35 /* Assume mpsmin == device_page_size == 4KB */
36 #define NVMET_RDMA_MAX_MDTS 8
37 #define NVMET_RDMA_MAX_METADATA_MDTS 5
39 struct nvmet_rdma_srq
;
41 struct nvmet_rdma_cmd
{
42 struct ib_sge sge
[NVMET_RDMA_MAX_INLINE_SGE
+ 1];
45 struct scatterlist inline_sg
[NVMET_RDMA_MAX_INLINE_SGE
];
46 struct nvme_command
*nvme_cmd
;
47 struct nvmet_rdma_queue
*queue
;
48 struct nvmet_rdma_srq
*nsrq
;
52 NVMET_RDMA_REQ_INLINE_DATA
= (1 << 0),
53 NVMET_RDMA_REQ_INVALIDATE_RKEY
= (1 << 1),
56 struct nvmet_rdma_rsp
{
57 struct ib_sge send_sge
;
58 struct ib_cqe send_cqe
;
59 struct ib_send_wr send_wr
;
61 struct nvmet_rdma_cmd
*cmd
;
62 struct nvmet_rdma_queue
*queue
;
64 struct ib_cqe read_cqe
;
65 struct ib_cqe write_cqe
;
66 struct rdma_rw_ctx rw
;
75 struct list_head wait_list
;
76 struct list_head free_list
;
79 enum nvmet_rdma_queue_state
{
80 NVMET_RDMA_Q_CONNECTING
,
82 NVMET_RDMA_Q_DISCONNECTING
,
85 struct nvmet_rdma_queue
{
86 struct rdma_cm_id
*cm_id
;
88 struct nvmet_port
*port
;
91 struct nvmet_rdma_device
*dev
;
92 struct nvmet_rdma_srq
*nsrq
;
93 spinlock_t state_lock
;
94 enum nvmet_rdma_queue_state state
;
95 struct nvmet_cq nvme_cq
;
96 struct nvmet_sq nvme_sq
;
98 struct nvmet_rdma_rsp
*rsps
;
99 struct list_head free_rsps
;
100 spinlock_t rsps_lock
;
101 struct nvmet_rdma_cmd
*cmds
;
103 struct work_struct release_work
;
104 struct list_head rsp_wait_list
;
105 struct list_head rsp_wr_wait_list
;
106 spinlock_t rsp_wr_wait_lock
;
114 struct list_head queue_list
;
117 struct nvmet_rdma_port
{
118 struct nvmet_port
*nport
;
119 struct sockaddr_storage addr
;
120 struct rdma_cm_id
*cm_id
;
121 struct delayed_work repair_work
;
124 struct nvmet_rdma_srq
{
126 struct nvmet_rdma_cmd
*cmds
;
127 struct nvmet_rdma_device
*ndev
;
130 struct nvmet_rdma_device
{
131 struct ib_device
*device
;
133 struct nvmet_rdma_srq
**srqs
;
137 struct list_head entry
;
138 int inline_data_size
;
139 int inline_page_count
;
142 static bool nvmet_rdma_use_srq
;
143 module_param_named(use_srq
, nvmet_rdma_use_srq
, bool, 0444);
144 MODULE_PARM_DESC(use_srq
, "Use shared receive queue.");
146 static int srq_size_set(const char *val
, const struct kernel_param
*kp
);
147 static const struct kernel_param_ops srq_size_ops
= {
149 .get
= param_get_int
,
152 static int nvmet_rdma_srq_size
= 1024;
153 module_param_cb(srq_size
, &srq_size_ops
, &nvmet_rdma_srq_size
, 0644);
154 MODULE_PARM_DESC(srq_size
, "set Shared Receive Queue (SRQ) size, should >= 256 (default: 1024)");
156 static DEFINE_IDA(nvmet_rdma_queue_ida
);
157 static LIST_HEAD(nvmet_rdma_queue_list
);
158 static DEFINE_MUTEX(nvmet_rdma_queue_mutex
);
160 static LIST_HEAD(device_list
);
161 static DEFINE_MUTEX(device_list_mutex
);
163 static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp
*rsp
);
164 static void nvmet_rdma_send_done(struct ib_cq
*cq
, struct ib_wc
*wc
);
165 static void nvmet_rdma_recv_done(struct ib_cq
*cq
, struct ib_wc
*wc
);
166 static void nvmet_rdma_read_data_done(struct ib_cq
*cq
, struct ib_wc
*wc
);
167 static void nvmet_rdma_write_data_done(struct ib_cq
*cq
, struct ib_wc
*wc
);
168 static void nvmet_rdma_qp_event(struct ib_event
*event
, void *priv
);
169 static void nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue
*queue
);
170 static void nvmet_rdma_free_rsp(struct nvmet_rdma_device
*ndev
,
171 struct nvmet_rdma_rsp
*r
);
172 static int nvmet_rdma_alloc_rsp(struct nvmet_rdma_device
*ndev
,
173 struct nvmet_rdma_rsp
*r
);
175 static const struct nvmet_fabrics_ops nvmet_rdma_ops
;
177 static int srq_size_set(const char *val
, const struct kernel_param
*kp
)
181 ret
= kstrtoint(val
, 10, &n
);
182 if (ret
!= 0 || n
< 256)
185 return param_set_int(val
, kp
);
188 static int num_pages(int len
)
190 return 1 + (((len
- 1) & PAGE_MASK
) >> PAGE_SHIFT
);
193 static inline bool nvmet_rdma_need_data_in(struct nvmet_rdma_rsp
*rsp
)
195 return nvme_is_write(rsp
->req
.cmd
) &&
196 rsp
->req
.transfer_len
&&
197 !(rsp
->flags
& NVMET_RDMA_REQ_INLINE_DATA
);
200 static inline bool nvmet_rdma_need_data_out(struct nvmet_rdma_rsp
*rsp
)
202 return !nvme_is_write(rsp
->req
.cmd
) &&
203 rsp
->req
.transfer_len
&&
204 !rsp
->req
.cqe
->status
&&
205 !(rsp
->flags
& NVMET_RDMA_REQ_INLINE_DATA
);
208 static inline struct nvmet_rdma_rsp
*
209 nvmet_rdma_get_rsp(struct nvmet_rdma_queue
*queue
)
211 struct nvmet_rdma_rsp
*rsp
;
214 spin_lock_irqsave(&queue
->rsps_lock
, flags
);
215 rsp
= list_first_entry_or_null(&queue
->free_rsps
,
216 struct nvmet_rdma_rsp
, free_list
);
218 list_del(&rsp
->free_list
);
219 spin_unlock_irqrestore(&queue
->rsps_lock
, flags
);
221 if (unlikely(!rsp
)) {
224 rsp
= kzalloc(sizeof(*rsp
), GFP_KERNEL
);
227 ret
= nvmet_rdma_alloc_rsp(queue
->dev
, rsp
);
233 rsp
->allocated
= true;
240 nvmet_rdma_put_rsp(struct nvmet_rdma_rsp
*rsp
)
244 if (unlikely(rsp
->allocated
)) {
245 nvmet_rdma_free_rsp(rsp
->queue
->dev
, rsp
);
250 spin_lock_irqsave(&rsp
->queue
->rsps_lock
, flags
);
251 list_add_tail(&rsp
->free_list
, &rsp
->queue
->free_rsps
);
252 spin_unlock_irqrestore(&rsp
->queue
->rsps_lock
, flags
);
255 static void nvmet_rdma_free_inline_pages(struct nvmet_rdma_device
*ndev
,
256 struct nvmet_rdma_cmd
*c
)
258 struct scatterlist
*sg
;
262 if (!ndev
->inline_data_size
)
268 for (i
= 0; i
< ndev
->inline_page_count
; i
++, sg
++, sge
++) {
270 ib_dma_unmap_page(ndev
->device
, sge
->addr
,
271 sge
->length
, DMA_FROM_DEVICE
);
273 __free_page(sg_page(sg
));
277 static int nvmet_rdma_alloc_inline_pages(struct nvmet_rdma_device
*ndev
,
278 struct nvmet_rdma_cmd
*c
)
280 struct scatterlist
*sg
;
286 if (!ndev
->inline_data_size
)
290 sg_init_table(sg
, ndev
->inline_page_count
);
292 len
= ndev
->inline_data_size
;
294 for (i
= 0; i
< ndev
->inline_page_count
; i
++, sg
++, sge
++) {
295 pg
= alloc_page(GFP_KERNEL
);
298 sg_assign_page(sg
, pg
);
299 sge
->addr
= ib_dma_map_page(ndev
->device
,
300 pg
, 0, PAGE_SIZE
, DMA_FROM_DEVICE
);
301 if (ib_dma_mapping_error(ndev
->device
, sge
->addr
))
303 sge
->length
= min_t(int, len
, PAGE_SIZE
);
304 sge
->lkey
= ndev
->pd
->local_dma_lkey
;
310 for (; i
>= 0; i
--, sg
--, sge
--) {
312 ib_dma_unmap_page(ndev
->device
, sge
->addr
,
313 sge
->length
, DMA_FROM_DEVICE
);
315 __free_page(sg_page(sg
));
320 static int nvmet_rdma_alloc_cmd(struct nvmet_rdma_device
*ndev
,
321 struct nvmet_rdma_cmd
*c
, bool admin
)
323 /* NVMe command / RDMA RECV */
324 c
->nvme_cmd
= kmalloc(sizeof(*c
->nvme_cmd
), GFP_KERNEL
);
328 c
->sge
[0].addr
= ib_dma_map_single(ndev
->device
, c
->nvme_cmd
,
329 sizeof(*c
->nvme_cmd
), DMA_FROM_DEVICE
);
330 if (ib_dma_mapping_error(ndev
->device
, c
->sge
[0].addr
))
333 c
->sge
[0].length
= sizeof(*c
->nvme_cmd
);
334 c
->sge
[0].lkey
= ndev
->pd
->local_dma_lkey
;
336 if (!admin
&& nvmet_rdma_alloc_inline_pages(ndev
, c
))
339 c
->cqe
.done
= nvmet_rdma_recv_done
;
341 c
->wr
.wr_cqe
= &c
->cqe
;
342 c
->wr
.sg_list
= c
->sge
;
343 c
->wr
.num_sge
= admin
? 1 : ndev
->inline_page_count
+ 1;
348 ib_dma_unmap_single(ndev
->device
, c
->sge
[0].addr
,
349 sizeof(*c
->nvme_cmd
), DMA_FROM_DEVICE
);
357 static void nvmet_rdma_free_cmd(struct nvmet_rdma_device
*ndev
,
358 struct nvmet_rdma_cmd
*c
, bool admin
)
361 nvmet_rdma_free_inline_pages(ndev
, c
);
362 ib_dma_unmap_single(ndev
->device
, c
->sge
[0].addr
,
363 sizeof(*c
->nvme_cmd
), DMA_FROM_DEVICE
);
367 static struct nvmet_rdma_cmd
*
368 nvmet_rdma_alloc_cmds(struct nvmet_rdma_device
*ndev
,
369 int nr_cmds
, bool admin
)
371 struct nvmet_rdma_cmd
*cmds
;
372 int ret
= -EINVAL
, i
;
374 cmds
= kcalloc(nr_cmds
, sizeof(struct nvmet_rdma_cmd
), GFP_KERNEL
);
378 for (i
= 0; i
< nr_cmds
; i
++) {
379 ret
= nvmet_rdma_alloc_cmd(ndev
, cmds
+ i
, admin
);
388 nvmet_rdma_free_cmd(ndev
, cmds
+ i
, admin
);
394 static void nvmet_rdma_free_cmds(struct nvmet_rdma_device
*ndev
,
395 struct nvmet_rdma_cmd
*cmds
, int nr_cmds
, bool admin
)
399 for (i
= 0; i
< nr_cmds
; i
++)
400 nvmet_rdma_free_cmd(ndev
, cmds
+ i
, admin
);
404 static int nvmet_rdma_alloc_rsp(struct nvmet_rdma_device
*ndev
,
405 struct nvmet_rdma_rsp
*r
)
407 /* NVMe CQE / RDMA SEND */
408 r
->req
.cqe
= kmalloc(sizeof(*r
->req
.cqe
), GFP_KERNEL
);
412 r
->send_sge
.addr
= ib_dma_map_single(ndev
->device
, r
->req
.cqe
,
413 sizeof(*r
->req
.cqe
), DMA_TO_DEVICE
);
414 if (ib_dma_mapping_error(ndev
->device
, r
->send_sge
.addr
))
417 if (!ib_uses_virt_dma(ndev
->device
))
418 r
->req
.p2p_client
= &ndev
->device
->dev
;
419 r
->send_sge
.length
= sizeof(*r
->req
.cqe
);
420 r
->send_sge
.lkey
= ndev
->pd
->local_dma_lkey
;
422 r
->send_cqe
.done
= nvmet_rdma_send_done
;
424 r
->send_wr
.wr_cqe
= &r
->send_cqe
;
425 r
->send_wr
.sg_list
= &r
->send_sge
;
426 r
->send_wr
.num_sge
= 1;
427 r
->send_wr
.send_flags
= IB_SEND_SIGNALED
;
429 /* Data In / RDMA READ */
430 r
->read_cqe
.done
= nvmet_rdma_read_data_done
;
431 /* Data Out / RDMA WRITE */
432 r
->write_cqe
.done
= nvmet_rdma_write_data_done
;
442 static void nvmet_rdma_free_rsp(struct nvmet_rdma_device
*ndev
,
443 struct nvmet_rdma_rsp
*r
)
445 ib_dma_unmap_single(ndev
->device
, r
->send_sge
.addr
,
446 sizeof(*r
->req
.cqe
), DMA_TO_DEVICE
);
451 nvmet_rdma_alloc_rsps(struct nvmet_rdma_queue
*queue
)
453 struct nvmet_rdma_device
*ndev
= queue
->dev
;
454 int nr_rsps
= queue
->recv_queue_size
* 2;
455 int ret
= -EINVAL
, i
;
457 queue
->rsps
= kcalloc(nr_rsps
, sizeof(struct nvmet_rdma_rsp
),
462 for (i
= 0; i
< nr_rsps
; i
++) {
463 struct nvmet_rdma_rsp
*rsp
= &queue
->rsps
[i
];
465 ret
= nvmet_rdma_alloc_rsp(ndev
, rsp
);
469 list_add_tail(&rsp
->free_list
, &queue
->free_rsps
);
476 struct nvmet_rdma_rsp
*rsp
= &queue
->rsps
[i
];
478 list_del(&rsp
->free_list
);
479 nvmet_rdma_free_rsp(ndev
, rsp
);
486 static void nvmet_rdma_free_rsps(struct nvmet_rdma_queue
*queue
)
488 struct nvmet_rdma_device
*ndev
= queue
->dev
;
489 int i
, nr_rsps
= queue
->recv_queue_size
* 2;
491 for (i
= 0; i
< nr_rsps
; i
++) {
492 struct nvmet_rdma_rsp
*rsp
= &queue
->rsps
[i
];
494 list_del(&rsp
->free_list
);
495 nvmet_rdma_free_rsp(ndev
, rsp
);
500 static int nvmet_rdma_post_recv(struct nvmet_rdma_device
*ndev
,
501 struct nvmet_rdma_cmd
*cmd
)
505 ib_dma_sync_single_for_device(ndev
->device
,
506 cmd
->sge
[0].addr
, cmd
->sge
[0].length
,
510 ret
= ib_post_srq_recv(cmd
->nsrq
->srq
, &cmd
->wr
, NULL
);
512 ret
= ib_post_recv(cmd
->queue
->qp
, &cmd
->wr
, NULL
);
515 pr_err("post_recv cmd failed\n");
520 static void nvmet_rdma_process_wr_wait_list(struct nvmet_rdma_queue
*queue
)
522 spin_lock(&queue
->rsp_wr_wait_lock
);
523 while (!list_empty(&queue
->rsp_wr_wait_list
)) {
524 struct nvmet_rdma_rsp
*rsp
;
527 rsp
= list_entry(queue
->rsp_wr_wait_list
.next
,
528 struct nvmet_rdma_rsp
, wait_list
);
529 list_del(&rsp
->wait_list
);
531 spin_unlock(&queue
->rsp_wr_wait_lock
);
532 ret
= nvmet_rdma_execute_command(rsp
);
533 spin_lock(&queue
->rsp_wr_wait_lock
);
536 list_add(&rsp
->wait_list
, &queue
->rsp_wr_wait_list
);
540 spin_unlock(&queue
->rsp_wr_wait_lock
);
543 static u16
nvmet_rdma_check_pi_status(struct ib_mr
*sig_mr
)
545 struct ib_mr_status mr_status
;
549 ret
= ib_check_mr_status(sig_mr
, IB_MR_CHECK_SIG_STATUS
, &mr_status
);
551 pr_err("ib_check_mr_status failed, ret %d\n", ret
);
552 return NVME_SC_INVALID_PI
;
555 if (mr_status
.fail_status
& IB_MR_CHECK_SIG_STATUS
) {
556 switch (mr_status
.sig_err
.err_type
) {
557 case IB_SIG_BAD_GUARD
:
558 status
= NVME_SC_GUARD_CHECK
;
560 case IB_SIG_BAD_REFTAG
:
561 status
= NVME_SC_REFTAG_CHECK
;
563 case IB_SIG_BAD_APPTAG
:
564 status
= NVME_SC_APPTAG_CHECK
;
567 pr_err("PI error found type %d expected 0x%x vs actual 0x%x\n",
568 mr_status
.sig_err
.err_type
,
569 mr_status
.sig_err
.expected
,
570 mr_status
.sig_err
.actual
);
576 static void nvmet_rdma_set_sig_domain(struct blk_integrity
*bi
,
577 struct nvme_command
*cmd
, struct ib_sig_domain
*domain
,
578 u16 control
, u8 pi_type
)
580 domain
->sig_type
= IB_SIG_TYPE_T10_DIF
;
581 domain
->sig
.dif
.bg_type
= IB_T10DIF_CRC
;
582 domain
->sig
.dif
.pi_interval
= 1 << bi
->interval_exp
;
583 domain
->sig
.dif
.ref_tag
= le32_to_cpu(cmd
->rw
.reftag
);
584 if (control
& NVME_RW_PRINFO_PRCHK_REF
)
585 domain
->sig
.dif
.ref_remap
= true;
587 domain
->sig
.dif
.app_tag
= le16_to_cpu(cmd
->rw
.apptag
);
588 domain
->sig
.dif
.apptag_check_mask
= le16_to_cpu(cmd
->rw
.appmask
);
589 domain
->sig
.dif
.app_escape
= true;
590 if (pi_type
== NVME_NS_DPS_PI_TYPE3
)
591 domain
->sig
.dif
.ref_escape
= true;
594 static void nvmet_rdma_set_sig_attrs(struct nvmet_req
*req
,
595 struct ib_sig_attrs
*sig_attrs
)
597 struct nvme_command
*cmd
= req
->cmd
;
598 u16 control
= le16_to_cpu(cmd
->rw
.control
);
599 u8 pi_type
= req
->ns
->pi_type
;
600 struct blk_integrity
*bi
;
602 bi
= bdev_get_integrity(req
->ns
->bdev
);
604 memset(sig_attrs
, 0, sizeof(*sig_attrs
));
606 if (control
& NVME_RW_PRINFO_PRACT
) {
607 /* for WRITE_INSERT/READ_STRIP no wire domain */
608 sig_attrs
->wire
.sig_type
= IB_SIG_TYPE_NONE
;
609 nvmet_rdma_set_sig_domain(bi
, cmd
, &sig_attrs
->mem
, control
,
611 /* Clear the PRACT bit since HCA will generate/verify the PI */
612 control
&= ~NVME_RW_PRINFO_PRACT
;
613 cmd
->rw
.control
= cpu_to_le16(control
);
614 /* PI is added by the HW */
615 req
->transfer_len
+= req
->metadata_len
;
617 /* for WRITE_PASS/READ_PASS both wire/memory domains exist */
618 nvmet_rdma_set_sig_domain(bi
, cmd
, &sig_attrs
->wire
, control
,
620 nvmet_rdma_set_sig_domain(bi
, cmd
, &sig_attrs
->mem
, control
,
624 if (control
& NVME_RW_PRINFO_PRCHK_REF
)
625 sig_attrs
->check_mask
|= IB_SIG_CHECK_REFTAG
;
626 if (control
& NVME_RW_PRINFO_PRCHK_GUARD
)
627 sig_attrs
->check_mask
|= IB_SIG_CHECK_GUARD
;
628 if (control
& NVME_RW_PRINFO_PRCHK_APP
)
629 sig_attrs
->check_mask
|= IB_SIG_CHECK_APPTAG
;
632 static int nvmet_rdma_rw_ctx_init(struct nvmet_rdma_rsp
*rsp
, u64 addr
, u32 key
,
633 struct ib_sig_attrs
*sig_attrs
)
635 struct rdma_cm_id
*cm_id
= rsp
->queue
->cm_id
;
636 struct nvmet_req
*req
= &rsp
->req
;
639 if (req
->metadata_len
)
640 ret
= rdma_rw_ctx_signature_init(&rsp
->rw
, cm_id
->qp
,
641 cm_id
->port_num
, req
->sg
, req
->sg_cnt
,
642 req
->metadata_sg
, req
->metadata_sg_cnt
, sig_attrs
,
643 addr
, key
, nvmet_data_dir(req
));
645 ret
= rdma_rw_ctx_init(&rsp
->rw
, cm_id
->qp
, cm_id
->port_num
,
646 req
->sg
, req
->sg_cnt
, 0, addr
, key
,
647 nvmet_data_dir(req
));
652 static void nvmet_rdma_rw_ctx_destroy(struct nvmet_rdma_rsp
*rsp
)
654 struct rdma_cm_id
*cm_id
= rsp
->queue
->cm_id
;
655 struct nvmet_req
*req
= &rsp
->req
;
657 if (req
->metadata_len
)
658 rdma_rw_ctx_destroy_signature(&rsp
->rw
, cm_id
->qp
,
659 cm_id
->port_num
, req
->sg
, req
->sg_cnt
,
660 req
->metadata_sg
, req
->metadata_sg_cnt
,
661 nvmet_data_dir(req
));
663 rdma_rw_ctx_destroy(&rsp
->rw
, cm_id
->qp
, cm_id
->port_num
,
664 req
->sg
, req
->sg_cnt
, nvmet_data_dir(req
));
667 static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp
*rsp
)
669 struct nvmet_rdma_queue
*queue
= rsp
->queue
;
671 atomic_add(1 + rsp
->n_rdma
, &queue
->sq_wr_avail
);
674 nvmet_rdma_rw_ctx_destroy(rsp
);
676 if (rsp
->req
.sg
!= rsp
->cmd
->inline_sg
)
677 nvmet_req_free_sgls(&rsp
->req
);
679 if (unlikely(!list_empty_careful(&queue
->rsp_wr_wait_list
)))
680 nvmet_rdma_process_wr_wait_list(queue
);
682 nvmet_rdma_put_rsp(rsp
);
685 static void nvmet_rdma_error_comp(struct nvmet_rdma_queue
*queue
)
687 if (queue
->nvme_sq
.ctrl
) {
688 nvmet_ctrl_fatal_error(queue
->nvme_sq
.ctrl
);
691 * we didn't setup the controller yet in case
692 * of admin connect error, just disconnect and
695 nvmet_rdma_queue_disconnect(queue
);
699 static void nvmet_rdma_send_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
701 struct nvmet_rdma_rsp
*rsp
=
702 container_of(wc
->wr_cqe
, struct nvmet_rdma_rsp
, send_cqe
);
703 struct nvmet_rdma_queue
*queue
= wc
->qp
->qp_context
;
705 nvmet_rdma_release_rsp(rsp
);
707 if (unlikely(wc
->status
!= IB_WC_SUCCESS
&&
708 wc
->status
!= IB_WC_WR_FLUSH_ERR
)) {
709 pr_err("SEND for CQE 0x%p failed with status %s (%d).\n",
710 wc
->wr_cqe
, ib_wc_status_msg(wc
->status
), wc
->status
);
711 nvmet_rdma_error_comp(queue
);
715 static void nvmet_rdma_queue_response(struct nvmet_req
*req
)
717 struct nvmet_rdma_rsp
*rsp
=
718 container_of(req
, struct nvmet_rdma_rsp
, req
);
719 struct rdma_cm_id
*cm_id
= rsp
->queue
->cm_id
;
720 struct ib_send_wr
*first_wr
;
722 if (rsp
->flags
& NVMET_RDMA_REQ_INVALIDATE_RKEY
) {
723 rsp
->send_wr
.opcode
= IB_WR_SEND_WITH_INV
;
724 rsp
->send_wr
.ex
.invalidate_rkey
= rsp
->invalidate_rkey
;
726 rsp
->send_wr
.opcode
= IB_WR_SEND
;
729 if (nvmet_rdma_need_data_out(rsp
)) {
730 if (rsp
->req
.metadata_len
)
731 first_wr
= rdma_rw_ctx_wrs(&rsp
->rw
, cm_id
->qp
,
732 cm_id
->port_num
, &rsp
->write_cqe
, NULL
);
734 first_wr
= rdma_rw_ctx_wrs(&rsp
->rw
, cm_id
->qp
,
735 cm_id
->port_num
, NULL
, &rsp
->send_wr
);
737 first_wr
= &rsp
->send_wr
;
740 nvmet_rdma_post_recv(rsp
->queue
->dev
, rsp
->cmd
);
742 ib_dma_sync_single_for_device(rsp
->queue
->dev
->device
,
743 rsp
->send_sge
.addr
, rsp
->send_sge
.length
,
746 if (unlikely(ib_post_send(cm_id
->qp
, first_wr
, NULL
))) {
747 pr_err("sending cmd response failed\n");
748 nvmet_rdma_release_rsp(rsp
);
752 static void nvmet_rdma_read_data_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
754 struct nvmet_rdma_rsp
*rsp
=
755 container_of(wc
->wr_cqe
, struct nvmet_rdma_rsp
, read_cqe
);
756 struct nvmet_rdma_queue
*queue
= wc
->qp
->qp_context
;
759 WARN_ON(rsp
->n_rdma
<= 0);
760 atomic_add(rsp
->n_rdma
, &queue
->sq_wr_avail
);
763 if (unlikely(wc
->status
!= IB_WC_SUCCESS
)) {
764 nvmet_rdma_rw_ctx_destroy(rsp
);
765 nvmet_req_uninit(&rsp
->req
);
766 nvmet_rdma_release_rsp(rsp
);
767 if (wc
->status
!= IB_WC_WR_FLUSH_ERR
) {
768 pr_info("RDMA READ for CQE 0x%p failed with status %s (%d).\n",
769 wc
->wr_cqe
, ib_wc_status_msg(wc
->status
), wc
->status
);
770 nvmet_rdma_error_comp(queue
);
775 if (rsp
->req
.metadata_len
)
776 status
= nvmet_rdma_check_pi_status(rsp
->rw
.reg
->mr
);
777 nvmet_rdma_rw_ctx_destroy(rsp
);
779 if (unlikely(status
))
780 nvmet_req_complete(&rsp
->req
, status
);
782 rsp
->req
.execute(&rsp
->req
);
785 static void nvmet_rdma_write_data_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
787 struct nvmet_rdma_rsp
*rsp
=
788 container_of(wc
->wr_cqe
, struct nvmet_rdma_rsp
, write_cqe
);
789 struct nvmet_rdma_queue
*queue
= wc
->qp
->qp_context
;
790 struct rdma_cm_id
*cm_id
= rsp
->queue
->cm_id
;
793 if (!IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY
))
796 WARN_ON(rsp
->n_rdma
<= 0);
797 atomic_add(rsp
->n_rdma
, &queue
->sq_wr_avail
);
800 if (unlikely(wc
->status
!= IB_WC_SUCCESS
)) {
801 nvmet_rdma_rw_ctx_destroy(rsp
);
802 nvmet_req_uninit(&rsp
->req
);
803 nvmet_rdma_release_rsp(rsp
);
804 if (wc
->status
!= IB_WC_WR_FLUSH_ERR
) {
805 pr_info("RDMA WRITE for CQE failed with status %s (%d).\n",
806 ib_wc_status_msg(wc
->status
), wc
->status
);
807 nvmet_rdma_error_comp(queue
);
813 * Upon RDMA completion check the signature status
814 * - if succeeded send good NVMe response
815 * - if failed send bad NVMe response with appropriate error
817 status
= nvmet_rdma_check_pi_status(rsp
->rw
.reg
->mr
);
818 if (unlikely(status
))
819 rsp
->req
.cqe
->status
= cpu_to_le16(status
<< 1);
820 nvmet_rdma_rw_ctx_destroy(rsp
);
822 if (unlikely(ib_post_send(cm_id
->qp
, &rsp
->send_wr
, NULL
))) {
823 pr_err("sending cmd response failed\n");
824 nvmet_rdma_release_rsp(rsp
);
828 static void nvmet_rdma_use_inline_sg(struct nvmet_rdma_rsp
*rsp
, u32 len
,
831 int sg_count
= num_pages(len
);
832 struct scatterlist
*sg
;
835 sg
= rsp
->cmd
->inline_sg
;
836 for (i
= 0; i
< sg_count
; i
++, sg
++) {
837 if (i
< sg_count
- 1)
842 sg
->length
= min_t(int, len
, PAGE_SIZE
- off
);
848 rsp
->req
.sg
= rsp
->cmd
->inline_sg
;
849 rsp
->req
.sg_cnt
= sg_count
;
852 static u16
nvmet_rdma_map_sgl_inline(struct nvmet_rdma_rsp
*rsp
)
854 struct nvme_sgl_desc
*sgl
= &rsp
->req
.cmd
->common
.dptr
.sgl
;
855 u64 off
= le64_to_cpu(sgl
->addr
);
856 u32 len
= le32_to_cpu(sgl
->length
);
858 if (!nvme_is_write(rsp
->req
.cmd
)) {
860 offsetof(struct nvme_common_command
, opcode
);
861 return NVME_SC_INVALID_FIELD
| NVME_SC_DNR
;
864 if (off
+ len
> rsp
->queue
->dev
->inline_data_size
) {
865 pr_err("invalid inline data offset!\n");
866 return NVME_SC_SGL_INVALID_OFFSET
| NVME_SC_DNR
;
869 /* no data command? */
873 nvmet_rdma_use_inline_sg(rsp
, len
, off
);
874 rsp
->flags
|= NVMET_RDMA_REQ_INLINE_DATA
;
875 rsp
->req
.transfer_len
+= len
;
879 static u16
nvmet_rdma_map_sgl_keyed(struct nvmet_rdma_rsp
*rsp
,
880 struct nvme_keyed_sgl_desc
*sgl
, bool invalidate
)
882 u64 addr
= le64_to_cpu(sgl
->addr
);
883 u32 key
= get_unaligned_le32(sgl
->key
);
884 struct ib_sig_attrs sig_attrs
;
887 rsp
->req
.transfer_len
= get_unaligned_le24(sgl
->length
);
889 /* no data command? */
890 if (!rsp
->req
.transfer_len
)
893 if (rsp
->req
.metadata_len
)
894 nvmet_rdma_set_sig_attrs(&rsp
->req
, &sig_attrs
);
896 ret
= nvmet_req_alloc_sgls(&rsp
->req
);
897 if (unlikely(ret
< 0))
900 ret
= nvmet_rdma_rw_ctx_init(rsp
, addr
, key
, &sig_attrs
);
901 if (unlikely(ret
< 0))
906 rsp
->invalidate_rkey
= key
;
907 rsp
->flags
|= NVMET_RDMA_REQ_INVALIDATE_RKEY
;
913 rsp
->req
.transfer_len
= 0;
914 return NVME_SC_INTERNAL
;
917 static u16
nvmet_rdma_map_sgl(struct nvmet_rdma_rsp
*rsp
)
919 struct nvme_keyed_sgl_desc
*sgl
= &rsp
->req
.cmd
->common
.dptr
.ksgl
;
921 switch (sgl
->type
>> 4) {
922 case NVME_SGL_FMT_DATA_DESC
:
923 switch (sgl
->type
& 0xf) {
924 case NVME_SGL_FMT_OFFSET
:
925 return nvmet_rdma_map_sgl_inline(rsp
);
927 pr_err("invalid SGL subtype: %#x\n", sgl
->type
);
929 offsetof(struct nvme_common_command
, dptr
);
930 return NVME_SC_INVALID_FIELD
| NVME_SC_DNR
;
932 case NVME_KEY_SGL_FMT_DATA_DESC
:
933 switch (sgl
->type
& 0xf) {
934 case NVME_SGL_FMT_ADDRESS
| NVME_SGL_FMT_INVALIDATE
:
935 return nvmet_rdma_map_sgl_keyed(rsp
, sgl
, true);
936 case NVME_SGL_FMT_ADDRESS
:
937 return nvmet_rdma_map_sgl_keyed(rsp
, sgl
, false);
939 pr_err("invalid SGL subtype: %#x\n", sgl
->type
);
941 offsetof(struct nvme_common_command
, dptr
);
942 return NVME_SC_INVALID_FIELD
| NVME_SC_DNR
;
945 pr_err("invalid SGL type: %#x\n", sgl
->type
);
946 rsp
->req
.error_loc
= offsetof(struct nvme_common_command
, dptr
);
947 return NVME_SC_SGL_INVALID_TYPE
| NVME_SC_DNR
;
951 static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp
*rsp
)
953 struct nvmet_rdma_queue
*queue
= rsp
->queue
;
955 if (unlikely(atomic_sub_return(1 + rsp
->n_rdma
,
956 &queue
->sq_wr_avail
) < 0)) {
957 pr_debug("IB send queue full (needed %d): queue %u cntlid %u\n",
958 1 + rsp
->n_rdma
, queue
->idx
,
959 queue
->nvme_sq
.ctrl
->cntlid
);
960 atomic_add(1 + rsp
->n_rdma
, &queue
->sq_wr_avail
);
964 if (nvmet_rdma_need_data_in(rsp
)) {
965 if (rdma_rw_ctx_post(&rsp
->rw
, queue
->qp
,
966 queue
->cm_id
->port_num
, &rsp
->read_cqe
, NULL
))
967 nvmet_req_complete(&rsp
->req
, NVME_SC_DATA_XFER_ERROR
);
969 rsp
->req
.execute(&rsp
->req
);
975 static void nvmet_rdma_handle_command(struct nvmet_rdma_queue
*queue
,
976 struct nvmet_rdma_rsp
*cmd
)
980 ib_dma_sync_single_for_cpu(queue
->dev
->device
,
981 cmd
->cmd
->sge
[0].addr
, cmd
->cmd
->sge
[0].length
,
983 ib_dma_sync_single_for_cpu(queue
->dev
->device
,
984 cmd
->send_sge
.addr
, cmd
->send_sge
.length
,
987 if (!nvmet_req_init(&cmd
->req
, &queue
->nvme_cq
,
988 &queue
->nvme_sq
, &nvmet_rdma_ops
))
991 status
= nvmet_rdma_map_sgl(cmd
);
995 if (unlikely(!nvmet_rdma_execute_command(cmd
))) {
996 spin_lock(&queue
->rsp_wr_wait_lock
);
997 list_add_tail(&cmd
->wait_list
, &queue
->rsp_wr_wait_list
);
998 spin_unlock(&queue
->rsp_wr_wait_lock
);
1004 nvmet_req_complete(&cmd
->req
, status
);
1007 static void nvmet_rdma_recv_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
1009 struct nvmet_rdma_cmd
*cmd
=
1010 container_of(wc
->wr_cqe
, struct nvmet_rdma_cmd
, cqe
);
1011 struct nvmet_rdma_queue
*queue
= wc
->qp
->qp_context
;
1012 struct nvmet_rdma_rsp
*rsp
;
1014 if (unlikely(wc
->status
!= IB_WC_SUCCESS
)) {
1015 if (wc
->status
!= IB_WC_WR_FLUSH_ERR
) {
1016 pr_err("RECV for CQE 0x%p failed with status %s (%d)\n",
1017 wc
->wr_cqe
, ib_wc_status_msg(wc
->status
),
1019 nvmet_rdma_error_comp(queue
);
1024 if (unlikely(wc
->byte_len
< sizeof(struct nvme_command
))) {
1025 pr_err("Ctrl Fatal Error: capsule size less than 64 bytes\n");
1026 nvmet_rdma_error_comp(queue
);
1031 rsp
= nvmet_rdma_get_rsp(queue
);
1032 if (unlikely(!rsp
)) {
1034 * we get here only under memory pressure,
1035 * silently drop and have the host retry
1036 * as we can't even fail it.
1038 nvmet_rdma_post_recv(queue
->dev
, cmd
);
1044 rsp
->req
.cmd
= cmd
->nvme_cmd
;
1045 rsp
->req
.port
= queue
->port
;
1048 if (unlikely(queue
->state
!= NVMET_RDMA_Q_LIVE
)) {
1049 unsigned long flags
;
1051 spin_lock_irqsave(&queue
->state_lock
, flags
);
1052 if (queue
->state
== NVMET_RDMA_Q_CONNECTING
)
1053 list_add_tail(&rsp
->wait_list
, &queue
->rsp_wait_list
);
1055 nvmet_rdma_put_rsp(rsp
);
1056 spin_unlock_irqrestore(&queue
->state_lock
, flags
);
1060 nvmet_rdma_handle_command(queue
, rsp
);
1063 static void nvmet_rdma_destroy_srq(struct nvmet_rdma_srq
*nsrq
)
1065 nvmet_rdma_free_cmds(nsrq
->ndev
, nsrq
->cmds
, nsrq
->ndev
->srq_size
,
1067 ib_destroy_srq(nsrq
->srq
);
1072 static void nvmet_rdma_destroy_srqs(struct nvmet_rdma_device
*ndev
)
1079 for (i
= 0; i
< ndev
->srq_count
; i
++)
1080 nvmet_rdma_destroy_srq(ndev
->srqs
[i
]);
1085 static struct nvmet_rdma_srq
*
1086 nvmet_rdma_init_srq(struct nvmet_rdma_device
*ndev
)
1088 struct ib_srq_init_attr srq_attr
= { NULL
, };
1089 size_t srq_size
= ndev
->srq_size
;
1090 struct nvmet_rdma_srq
*nsrq
;
1094 nsrq
= kzalloc(sizeof(*nsrq
), GFP_KERNEL
);
1096 return ERR_PTR(-ENOMEM
);
1098 srq_attr
.attr
.max_wr
= srq_size
;
1099 srq_attr
.attr
.max_sge
= 1 + ndev
->inline_page_count
;
1100 srq_attr
.attr
.srq_limit
= 0;
1101 srq_attr
.srq_type
= IB_SRQT_BASIC
;
1102 srq
= ib_create_srq(ndev
->pd
, &srq_attr
);
1108 nsrq
->cmds
= nvmet_rdma_alloc_cmds(ndev
, srq_size
, false);
1109 if (IS_ERR(nsrq
->cmds
)) {
1110 ret
= PTR_ERR(nsrq
->cmds
);
1111 goto out_destroy_srq
;
1117 for (i
= 0; i
< srq_size
; i
++) {
1118 nsrq
->cmds
[i
].nsrq
= nsrq
;
1119 ret
= nvmet_rdma_post_recv(ndev
, &nsrq
->cmds
[i
]);
1127 nvmet_rdma_free_cmds(ndev
, nsrq
->cmds
, srq_size
, false);
1129 ib_destroy_srq(srq
);
1132 return ERR_PTR(ret
);
1135 static int nvmet_rdma_init_srqs(struct nvmet_rdma_device
*ndev
)
1139 if (!ndev
->device
->attrs
.max_srq_wr
|| !ndev
->device
->attrs
.max_srq
) {
1141 * If SRQs aren't supported we just go ahead and use normal
1142 * non-shared receive queues.
1144 pr_info("SRQ requested but not supported.\n");
1148 ndev
->srq_size
= min(ndev
->device
->attrs
.max_srq_wr
,
1149 nvmet_rdma_srq_size
);
1150 ndev
->srq_count
= min(ndev
->device
->num_comp_vectors
,
1151 ndev
->device
->attrs
.max_srq
);
1153 ndev
->srqs
= kcalloc(ndev
->srq_count
, sizeof(*ndev
->srqs
), GFP_KERNEL
);
1157 for (i
= 0; i
< ndev
->srq_count
; i
++) {
1158 ndev
->srqs
[i
] = nvmet_rdma_init_srq(ndev
);
1159 if (IS_ERR(ndev
->srqs
[i
])) {
1160 ret
= PTR_ERR(ndev
->srqs
[i
]);
1169 nvmet_rdma_destroy_srq(ndev
->srqs
[i
]);
1174 static void nvmet_rdma_free_dev(struct kref
*ref
)
1176 struct nvmet_rdma_device
*ndev
=
1177 container_of(ref
, struct nvmet_rdma_device
, ref
);
1179 mutex_lock(&device_list_mutex
);
1180 list_del(&ndev
->entry
);
1181 mutex_unlock(&device_list_mutex
);
1183 nvmet_rdma_destroy_srqs(ndev
);
1184 ib_dealloc_pd(ndev
->pd
);
1189 static struct nvmet_rdma_device
*
1190 nvmet_rdma_find_get_device(struct rdma_cm_id
*cm_id
)
1192 struct nvmet_rdma_port
*port
= cm_id
->context
;
1193 struct nvmet_port
*nport
= port
->nport
;
1194 struct nvmet_rdma_device
*ndev
;
1195 int inline_page_count
;
1196 int inline_sge_count
;
1199 mutex_lock(&device_list_mutex
);
1200 list_for_each_entry(ndev
, &device_list
, entry
) {
1201 if (ndev
->device
->node_guid
== cm_id
->device
->node_guid
&&
1202 kref_get_unless_zero(&ndev
->ref
))
1206 ndev
= kzalloc(sizeof(*ndev
), GFP_KERNEL
);
1210 inline_page_count
= num_pages(nport
->inline_data_size
);
1211 inline_sge_count
= max(cm_id
->device
->attrs
.max_sge_rd
,
1212 cm_id
->device
->attrs
.max_recv_sge
) - 1;
1213 if (inline_page_count
> inline_sge_count
) {
1214 pr_warn("inline_data_size %d cannot be supported by device %s. Reducing to %lu.\n",
1215 nport
->inline_data_size
, cm_id
->device
->name
,
1216 inline_sge_count
* PAGE_SIZE
);
1217 nport
->inline_data_size
= inline_sge_count
* PAGE_SIZE
;
1218 inline_page_count
= inline_sge_count
;
1220 ndev
->inline_data_size
= nport
->inline_data_size
;
1221 ndev
->inline_page_count
= inline_page_count
;
1223 if (nport
->pi_enable
&& !(cm_id
->device
->attrs
.device_cap_flags
&
1224 IB_DEVICE_INTEGRITY_HANDOVER
)) {
1225 pr_warn("T10-PI is not supported by device %s. Disabling it\n",
1226 cm_id
->device
->name
);
1227 nport
->pi_enable
= false;
1230 ndev
->device
= cm_id
->device
;
1231 kref_init(&ndev
->ref
);
1233 ndev
->pd
= ib_alloc_pd(ndev
->device
, 0);
1234 if (IS_ERR(ndev
->pd
))
1237 if (nvmet_rdma_use_srq
) {
1238 ret
= nvmet_rdma_init_srqs(ndev
);
1243 list_add(&ndev
->entry
, &device_list
);
1245 mutex_unlock(&device_list_mutex
);
1246 pr_debug("added %s.\n", ndev
->device
->name
);
1250 ib_dealloc_pd(ndev
->pd
);
1254 mutex_unlock(&device_list_mutex
);
1258 static int nvmet_rdma_create_queue_ib(struct nvmet_rdma_queue
*queue
)
1260 struct ib_qp_init_attr qp_attr
;
1261 struct nvmet_rdma_device
*ndev
= queue
->dev
;
1262 int nr_cqe
, ret
, i
, factor
;
1265 * Reserve CQ slots for RECV + RDMA_READ/RDMA_WRITE + RDMA_SEND.
1267 nr_cqe
= queue
->recv_queue_size
+ 2 * queue
->send_queue_size
;
1269 queue
->cq
= ib_cq_pool_get(ndev
->device
, nr_cqe
+ 1,
1270 queue
->comp_vector
, IB_POLL_WORKQUEUE
);
1271 if (IS_ERR(queue
->cq
)) {
1272 ret
= PTR_ERR(queue
->cq
);
1273 pr_err("failed to create CQ cqe= %d ret= %d\n",
1278 memset(&qp_attr
, 0, sizeof(qp_attr
));
1279 qp_attr
.qp_context
= queue
;
1280 qp_attr
.event_handler
= nvmet_rdma_qp_event
;
1281 qp_attr
.send_cq
= queue
->cq
;
1282 qp_attr
.recv_cq
= queue
->cq
;
1283 qp_attr
.sq_sig_type
= IB_SIGNAL_REQ_WR
;
1284 qp_attr
.qp_type
= IB_QPT_RC
;
1286 qp_attr
.cap
.max_send_wr
= queue
->send_queue_size
+ 1;
1287 factor
= rdma_rw_mr_factor(ndev
->device
, queue
->cm_id
->port_num
,
1288 1 << NVMET_RDMA_MAX_MDTS
);
1289 qp_attr
.cap
.max_rdma_ctxs
= queue
->send_queue_size
* factor
;
1290 qp_attr
.cap
.max_send_sge
= max(ndev
->device
->attrs
.max_sge_rd
,
1291 ndev
->device
->attrs
.max_send_sge
);
1294 qp_attr
.srq
= queue
->nsrq
->srq
;
1297 qp_attr
.cap
.max_recv_wr
= 1 + queue
->recv_queue_size
;
1298 qp_attr
.cap
.max_recv_sge
= 1 + ndev
->inline_page_count
;
1301 if (queue
->port
->pi_enable
&& queue
->host_qid
)
1302 qp_attr
.create_flags
|= IB_QP_CREATE_INTEGRITY_EN
;
1304 ret
= rdma_create_qp(queue
->cm_id
, ndev
->pd
, &qp_attr
);
1306 pr_err("failed to create_qp ret= %d\n", ret
);
1307 goto err_destroy_cq
;
1309 queue
->qp
= queue
->cm_id
->qp
;
1311 atomic_set(&queue
->sq_wr_avail
, qp_attr
.cap
.max_send_wr
);
1313 pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d cm_id= %p\n",
1314 __func__
, queue
->cq
->cqe
, qp_attr
.cap
.max_send_sge
,
1315 qp_attr
.cap
.max_send_wr
, queue
->cm_id
);
1318 for (i
= 0; i
< queue
->recv_queue_size
; i
++) {
1319 queue
->cmds
[i
].queue
= queue
;
1320 ret
= nvmet_rdma_post_recv(ndev
, &queue
->cmds
[i
]);
1322 goto err_destroy_qp
;
1330 rdma_destroy_qp(queue
->cm_id
);
1332 ib_cq_pool_put(queue
->cq
, nr_cqe
+ 1);
1336 static void nvmet_rdma_destroy_queue_ib(struct nvmet_rdma_queue
*queue
)
1338 ib_drain_qp(queue
->qp
);
1340 rdma_destroy_id(queue
->cm_id
);
1341 ib_destroy_qp(queue
->qp
);
1342 ib_cq_pool_put(queue
->cq
, queue
->recv_queue_size
+ 2 *
1343 queue
->send_queue_size
+ 1);
1346 static void nvmet_rdma_free_queue(struct nvmet_rdma_queue
*queue
)
1348 pr_debug("freeing queue %d\n", queue
->idx
);
1350 nvmet_sq_destroy(&queue
->nvme_sq
);
1352 nvmet_rdma_destroy_queue_ib(queue
);
1354 nvmet_rdma_free_cmds(queue
->dev
, queue
->cmds
,
1355 queue
->recv_queue_size
,
1358 nvmet_rdma_free_rsps(queue
);
1359 ida_simple_remove(&nvmet_rdma_queue_ida
, queue
->idx
);
1363 static void nvmet_rdma_release_queue_work(struct work_struct
*w
)
1365 struct nvmet_rdma_queue
*queue
=
1366 container_of(w
, struct nvmet_rdma_queue
, release_work
);
1367 struct nvmet_rdma_device
*dev
= queue
->dev
;
1369 nvmet_rdma_free_queue(queue
);
1371 kref_put(&dev
->ref
, nvmet_rdma_free_dev
);
1375 nvmet_rdma_parse_cm_connect_req(struct rdma_conn_param
*conn
,
1376 struct nvmet_rdma_queue
*queue
)
1378 struct nvme_rdma_cm_req
*req
;
1380 req
= (struct nvme_rdma_cm_req
*)conn
->private_data
;
1381 if (!req
|| conn
->private_data_len
== 0)
1382 return NVME_RDMA_CM_INVALID_LEN
;
1384 if (le16_to_cpu(req
->recfmt
) != NVME_RDMA_CM_FMT_1_0
)
1385 return NVME_RDMA_CM_INVALID_RECFMT
;
1387 queue
->host_qid
= le16_to_cpu(req
->qid
);
1390 * req->hsqsize corresponds to our recv queue size plus 1
1391 * req->hrqsize corresponds to our send queue size
1393 queue
->recv_queue_size
= le16_to_cpu(req
->hsqsize
) + 1;
1394 queue
->send_queue_size
= le16_to_cpu(req
->hrqsize
);
1396 if (!queue
->host_qid
&& queue
->recv_queue_size
> NVME_AQ_DEPTH
)
1397 return NVME_RDMA_CM_INVALID_HSQSIZE
;
1399 /* XXX: Should we enforce some kind of max for IO queues? */
1404 static int nvmet_rdma_cm_reject(struct rdma_cm_id
*cm_id
,
1405 enum nvme_rdma_cm_status status
)
1407 struct nvme_rdma_cm_rej rej
;
1409 pr_debug("rejecting connect request: status %d (%s)\n",
1410 status
, nvme_rdma_cm_msg(status
));
1412 rej
.recfmt
= cpu_to_le16(NVME_RDMA_CM_FMT_1_0
);
1413 rej
.sts
= cpu_to_le16(status
);
1415 return rdma_reject(cm_id
, (void *)&rej
, sizeof(rej
),
1416 IB_CM_REJ_CONSUMER_DEFINED
);
1419 static struct nvmet_rdma_queue
*
1420 nvmet_rdma_alloc_queue(struct nvmet_rdma_device
*ndev
,
1421 struct rdma_cm_id
*cm_id
,
1422 struct rdma_cm_event
*event
)
1424 struct nvmet_rdma_port
*port
= cm_id
->context
;
1425 struct nvmet_rdma_queue
*queue
;
1428 queue
= kzalloc(sizeof(*queue
), GFP_KERNEL
);
1430 ret
= NVME_RDMA_CM_NO_RSC
;
1434 ret
= nvmet_sq_init(&queue
->nvme_sq
);
1436 ret
= NVME_RDMA_CM_NO_RSC
;
1437 goto out_free_queue
;
1440 ret
= nvmet_rdma_parse_cm_connect_req(&event
->param
.conn
, queue
);
1442 goto out_destroy_sq
;
1445 * Schedules the actual release because calling rdma_destroy_id from
1446 * inside a CM callback would trigger a deadlock. (great API design..)
1448 INIT_WORK(&queue
->release_work
, nvmet_rdma_release_queue_work
);
1450 queue
->cm_id
= cm_id
;
1451 queue
->port
= port
->nport
;
1453 spin_lock_init(&queue
->state_lock
);
1454 queue
->state
= NVMET_RDMA_Q_CONNECTING
;
1455 INIT_LIST_HEAD(&queue
->rsp_wait_list
);
1456 INIT_LIST_HEAD(&queue
->rsp_wr_wait_list
);
1457 spin_lock_init(&queue
->rsp_wr_wait_lock
);
1458 INIT_LIST_HEAD(&queue
->free_rsps
);
1459 spin_lock_init(&queue
->rsps_lock
);
1460 INIT_LIST_HEAD(&queue
->queue_list
);
1462 queue
->idx
= ida_simple_get(&nvmet_rdma_queue_ida
, 0, 0, GFP_KERNEL
);
1463 if (queue
->idx
< 0) {
1464 ret
= NVME_RDMA_CM_NO_RSC
;
1465 goto out_destroy_sq
;
1469 * Spread the io queues across completion vectors,
1470 * but still keep all admin queues on vector 0.
1472 queue
->comp_vector
= !queue
->host_qid
? 0 :
1473 queue
->idx
% ndev
->device
->num_comp_vectors
;
1476 ret
= nvmet_rdma_alloc_rsps(queue
);
1478 ret
= NVME_RDMA_CM_NO_RSC
;
1479 goto out_ida_remove
;
1483 queue
->nsrq
= ndev
->srqs
[queue
->comp_vector
% ndev
->srq_count
];
1485 queue
->cmds
= nvmet_rdma_alloc_cmds(ndev
,
1486 queue
->recv_queue_size
,
1488 if (IS_ERR(queue
->cmds
)) {
1489 ret
= NVME_RDMA_CM_NO_RSC
;
1490 goto out_free_responses
;
1494 ret
= nvmet_rdma_create_queue_ib(queue
);
1496 pr_err("%s: creating RDMA queue failed (%d).\n",
1498 ret
= NVME_RDMA_CM_NO_RSC
;
1506 nvmet_rdma_free_cmds(queue
->dev
, queue
->cmds
,
1507 queue
->recv_queue_size
,
1511 nvmet_rdma_free_rsps(queue
);
1513 ida_simple_remove(&nvmet_rdma_queue_ida
, queue
->idx
);
1515 nvmet_sq_destroy(&queue
->nvme_sq
);
1519 nvmet_rdma_cm_reject(cm_id
, ret
);
1523 static void nvmet_rdma_qp_event(struct ib_event
*event
, void *priv
)
1525 struct nvmet_rdma_queue
*queue
= priv
;
1527 switch (event
->event
) {
1528 case IB_EVENT_COMM_EST
:
1529 rdma_notify(queue
->cm_id
, event
->event
);
1531 case IB_EVENT_QP_LAST_WQE_REACHED
:
1532 pr_debug("received last WQE reached event for queue=0x%p\n",
1536 pr_err("received IB QP event: %s (%d)\n",
1537 ib_event_msg(event
->event
), event
->event
);
1542 static int nvmet_rdma_cm_accept(struct rdma_cm_id
*cm_id
,
1543 struct nvmet_rdma_queue
*queue
,
1544 struct rdma_conn_param
*p
)
1546 struct rdma_conn_param param
= { };
1547 struct nvme_rdma_cm_rep priv
= { };
1550 param
.rnr_retry_count
= 7;
1551 param
.flow_control
= 1;
1552 param
.initiator_depth
= min_t(u8
, p
->initiator_depth
,
1553 queue
->dev
->device
->attrs
.max_qp_init_rd_atom
);
1554 param
.private_data
= &priv
;
1555 param
.private_data_len
= sizeof(priv
);
1556 priv
.recfmt
= cpu_to_le16(NVME_RDMA_CM_FMT_1_0
);
1557 priv
.crqsize
= cpu_to_le16(queue
->recv_queue_size
);
1559 ret
= rdma_accept(cm_id
, ¶m
);
1561 pr_err("rdma_accept failed (error code = %d)\n", ret
);
1566 static int nvmet_rdma_queue_connect(struct rdma_cm_id
*cm_id
,
1567 struct rdma_cm_event
*event
)
1569 struct nvmet_rdma_device
*ndev
;
1570 struct nvmet_rdma_queue
*queue
;
1573 ndev
= nvmet_rdma_find_get_device(cm_id
);
1575 nvmet_rdma_cm_reject(cm_id
, NVME_RDMA_CM_NO_RSC
);
1576 return -ECONNREFUSED
;
1579 queue
= nvmet_rdma_alloc_queue(ndev
, cm_id
, event
);
1585 if (queue
->host_qid
== 0) {
1586 /* Let inflight controller teardown complete */
1587 flush_scheduled_work();
1590 ret
= nvmet_rdma_cm_accept(cm_id
, queue
, &event
->param
.conn
);
1593 * Don't destroy the cm_id in free path, as we implicitly
1594 * destroy the cm_id here with non-zero ret code.
1596 queue
->cm_id
= NULL
;
1600 mutex_lock(&nvmet_rdma_queue_mutex
);
1601 list_add_tail(&queue
->queue_list
, &nvmet_rdma_queue_list
);
1602 mutex_unlock(&nvmet_rdma_queue_mutex
);
1607 nvmet_rdma_free_queue(queue
);
1609 kref_put(&ndev
->ref
, nvmet_rdma_free_dev
);
1614 static void nvmet_rdma_queue_established(struct nvmet_rdma_queue
*queue
)
1616 unsigned long flags
;
1618 spin_lock_irqsave(&queue
->state_lock
, flags
);
1619 if (queue
->state
!= NVMET_RDMA_Q_CONNECTING
) {
1620 pr_warn("trying to establish a connected queue\n");
1623 queue
->state
= NVMET_RDMA_Q_LIVE
;
1625 while (!list_empty(&queue
->rsp_wait_list
)) {
1626 struct nvmet_rdma_rsp
*cmd
;
1628 cmd
= list_first_entry(&queue
->rsp_wait_list
,
1629 struct nvmet_rdma_rsp
, wait_list
);
1630 list_del(&cmd
->wait_list
);
1632 spin_unlock_irqrestore(&queue
->state_lock
, flags
);
1633 nvmet_rdma_handle_command(queue
, cmd
);
1634 spin_lock_irqsave(&queue
->state_lock
, flags
);
1638 spin_unlock_irqrestore(&queue
->state_lock
, flags
);
1641 static void __nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue
*queue
)
1643 bool disconnect
= false;
1644 unsigned long flags
;
1646 pr_debug("cm_id= %p queue->state= %d\n", queue
->cm_id
, queue
->state
);
1648 spin_lock_irqsave(&queue
->state_lock
, flags
);
1649 switch (queue
->state
) {
1650 case NVMET_RDMA_Q_CONNECTING
:
1651 while (!list_empty(&queue
->rsp_wait_list
)) {
1652 struct nvmet_rdma_rsp
*rsp
;
1654 rsp
= list_first_entry(&queue
->rsp_wait_list
,
1655 struct nvmet_rdma_rsp
,
1657 list_del(&rsp
->wait_list
);
1658 nvmet_rdma_put_rsp(rsp
);
1661 case NVMET_RDMA_Q_LIVE
:
1662 queue
->state
= NVMET_RDMA_Q_DISCONNECTING
;
1665 case NVMET_RDMA_Q_DISCONNECTING
:
1668 spin_unlock_irqrestore(&queue
->state_lock
, flags
);
1671 rdma_disconnect(queue
->cm_id
);
1672 schedule_work(&queue
->release_work
);
1676 static void nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue
*queue
)
1678 bool disconnect
= false;
1680 mutex_lock(&nvmet_rdma_queue_mutex
);
1681 if (!list_empty(&queue
->queue_list
)) {
1682 list_del_init(&queue
->queue_list
);
1685 mutex_unlock(&nvmet_rdma_queue_mutex
);
1688 __nvmet_rdma_queue_disconnect(queue
);
1691 static void nvmet_rdma_queue_connect_fail(struct rdma_cm_id
*cm_id
,
1692 struct nvmet_rdma_queue
*queue
)
1694 WARN_ON_ONCE(queue
->state
!= NVMET_RDMA_Q_CONNECTING
);
1696 mutex_lock(&nvmet_rdma_queue_mutex
);
1697 if (!list_empty(&queue
->queue_list
))
1698 list_del_init(&queue
->queue_list
);
1699 mutex_unlock(&nvmet_rdma_queue_mutex
);
1701 pr_err("failed to connect queue %d\n", queue
->idx
);
1702 schedule_work(&queue
->release_work
);
1706 * nvme_rdma_device_removal() - Handle RDMA device removal
1707 * @cm_id: rdma_cm id, used for nvmet port
1708 * @queue: nvmet rdma queue (cm id qp_context)
1710 * DEVICE_REMOVAL event notifies us that the RDMA device is about
1711 * to unplug. Note that this event can be generated on a normal
1712 * queue cm_id and/or a device bound listener cm_id (where in this
1713 * case queue will be null).
1715 * We registered an ib_client to handle device removal for queues,
1716 * so we only need to handle the listening port cm_ids. In this case
1717 * we nullify the priv to prevent double cm_id destruction and destroying
1718 * the cm_id implicitely by returning a non-zero rc to the callout.
1720 static int nvmet_rdma_device_removal(struct rdma_cm_id
*cm_id
,
1721 struct nvmet_rdma_queue
*queue
)
1723 struct nvmet_rdma_port
*port
;
1727 * This is a queue cm_id. we have registered
1728 * an ib_client to handle queues removal
1729 * so don't interfear and just return.
1734 port
= cm_id
->context
;
1737 * This is a listener cm_id. Make sure that
1738 * future remove_port won't invoke a double
1739 * cm_id destroy. use atomic xchg to make sure
1740 * we don't compete with remove_port.
1742 if (xchg(&port
->cm_id
, NULL
) != cm_id
)
1746 * We need to return 1 so that the core will destroy
1747 * it's own ID. What a great API design..
1752 static int nvmet_rdma_cm_handler(struct rdma_cm_id
*cm_id
,
1753 struct rdma_cm_event
*event
)
1755 struct nvmet_rdma_queue
*queue
= NULL
;
1759 queue
= cm_id
->qp
->qp_context
;
1761 pr_debug("%s (%d): status %d id %p\n",
1762 rdma_event_msg(event
->event
), event
->event
,
1763 event
->status
, cm_id
);
1765 switch (event
->event
) {
1766 case RDMA_CM_EVENT_CONNECT_REQUEST
:
1767 ret
= nvmet_rdma_queue_connect(cm_id
, event
);
1769 case RDMA_CM_EVENT_ESTABLISHED
:
1770 nvmet_rdma_queue_established(queue
);
1772 case RDMA_CM_EVENT_ADDR_CHANGE
:
1774 struct nvmet_rdma_port
*port
= cm_id
->context
;
1776 schedule_delayed_work(&port
->repair_work
, 0);
1780 case RDMA_CM_EVENT_DISCONNECTED
:
1781 case RDMA_CM_EVENT_TIMEWAIT_EXIT
:
1782 nvmet_rdma_queue_disconnect(queue
);
1784 case RDMA_CM_EVENT_DEVICE_REMOVAL
:
1785 ret
= nvmet_rdma_device_removal(cm_id
, queue
);
1787 case RDMA_CM_EVENT_REJECTED
:
1788 pr_debug("Connection rejected: %s\n",
1789 rdma_reject_msg(cm_id
, event
->status
));
1791 case RDMA_CM_EVENT_UNREACHABLE
:
1792 case RDMA_CM_EVENT_CONNECT_ERROR
:
1793 nvmet_rdma_queue_connect_fail(cm_id
, queue
);
1796 pr_err("received unrecognized RDMA CM event %d\n",
1804 static void nvmet_rdma_delete_ctrl(struct nvmet_ctrl
*ctrl
)
1806 struct nvmet_rdma_queue
*queue
;
1809 mutex_lock(&nvmet_rdma_queue_mutex
);
1810 list_for_each_entry(queue
, &nvmet_rdma_queue_list
, queue_list
) {
1811 if (queue
->nvme_sq
.ctrl
== ctrl
) {
1812 list_del_init(&queue
->queue_list
);
1813 mutex_unlock(&nvmet_rdma_queue_mutex
);
1815 __nvmet_rdma_queue_disconnect(queue
);
1819 mutex_unlock(&nvmet_rdma_queue_mutex
);
1822 static void nvmet_rdma_disable_port(struct nvmet_rdma_port
*port
)
1824 struct rdma_cm_id
*cm_id
= xchg(&port
->cm_id
, NULL
);
1827 rdma_destroy_id(cm_id
);
1830 static int nvmet_rdma_enable_port(struct nvmet_rdma_port
*port
)
1832 struct sockaddr
*addr
= (struct sockaddr
*)&port
->addr
;
1833 struct rdma_cm_id
*cm_id
;
1836 cm_id
= rdma_create_id(&init_net
, nvmet_rdma_cm_handler
, port
,
1837 RDMA_PS_TCP
, IB_QPT_RC
);
1838 if (IS_ERR(cm_id
)) {
1839 pr_err("CM ID creation failed\n");
1840 return PTR_ERR(cm_id
);
1844 * Allow both IPv4 and IPv6 sockets to bind a single port
1847 ret
= rdma_set_afonly(cm_id
, 1);
1849 pr_err("rdma_set_afonly failed (%d)\n", ret
);
1850 goto out_destroy_id
;
1853 ret
= rdma_bind_addr(cm_id
, addr
);
1855 pr_err("binding CM ID to %pISpcs failed (%d)\n", addr
, ret
);
1856 goto out_destroy_id
;
1859 ret
= rdma_listen(cm_id
, 128);
1861 pr_err("listening to %pISpcs failed (%d)\n", addr
, ret
);
1862 goto out_destroy_id
;
1865 port
->cm_id
= cm_id
;
1869 rdma_destroy_id(cm_id
);
1873 static void nvmet_rdma_repair_port_work(struct work_struct
*w
)
1875 struct nvmet_rdma_port
*port
= container_of(to_delayed_work(w
),
1876 struct nvmet_rdma_port
, repair_work
);
1879 nvmet_rdma_disable_port(port
);
1880 ret
= nvmet_rdma_enable_port(port
);
1882 schedule_delayed_work(&port
->repair_work
, 5 * HZ
);
1885 static int nvmet_rdma_add_port(struct nvmet_port
*nport
)
1887 struct nvmet_rdma_port
*port
;
1888 __kernel_sa_family_t af
;
1891 port
= kzalloc(sizeof(*port
), GFP_KERNEL
);
1896 port
->nport
= nport
;
1897 INIT_DELAYED_WORK(&port
->repair_work
, nvmet_rdma_repair_port_work
);
1899 switch (nport
->disc_addr
.adrfam
) {
1900 case NVMF_ADDR_FAMILY_IP4
:
1903 case NVMF_ADDR_FAMILY_IP6
:
1907 pr_err("address family %d not supported\n",
1908 nport
->disc_addr
.adrfam
);
1913 if (nport
->inline_data_size
< 0) {
1914 nport
->inline_data_size
= NVMET_RDMA_DEFAULT_INLINE_DATA_SIZE
;
1915 } else if (nport
->inline_data_size
> NVMET_RDMA_MAX_INLINE_DATA_SIZE
) {
1916 pr_warn("inline_data_size %u is too large, reducing to %u\n",
1917 nport
->inline_data_size
,
1918 NVMET_RDMA_MAX_INLINE_DATA_SIZE
);
1919 nport
->inline_data_size
= NVMET_RDMA_MAX_INLINE_DATA_SIZE
;
1922 ret
= inet_pton_with_scope(&init_net
, af
, nport
->disc_addr
.traddr
,
1923 nport
->disc_addr
.trsvcid
, &port
->addr
);
1925 pr_err("malformed ip/port passed: %s:%s\n",
1926 nport
->disc_addr
.traddr
, nport
->disc_addr
.trsvcid
);
1930 ret
= nvmet_rdma_enable_port(port
);
1934 pr_info("enabling port %d (%pISpcs)\n",
1935 le16_to_cpu(nport
->disc_addr
.portid
),
1936 (struct sockaddr
*)&port
->addr
);
1945 static void nvmet_rdma_remove_port(struct nvmet_port
*nport
)
1947 struct nvmet_rdma_port
*port
= nport
->priv
;
1949 cancel_delayed_work_sync(&port
->repair_work
);
1950 nvmet_rdma_disable_port(port
);
1954 static void nvmet_rdma_disc_port_addr(struct nvmet_req
*req
,
1955 struct nvmet_port
*nport
, char *traddr
)
1957 struct nvmet_rdma_port
*port
= nport
->priv
;
1958 struct rdma_cm_id
*cm_id
= port
->cm_id
;
1960 if (inet_addr_is_any((struct sockaddr
*)&cm_id
->route
.addr
.src_addr
)) {
1961 struct nvmet_rdma_rsp
*rsp
=
1962 container_of(req
, struct nvmet_rdma_rsp
, req
);
1963 struct rdma_cm_id
*req_cm_id
= rsp
->queue
->cm_id
;
1964 struct sockaddr
*addr
= (void *)&req_cm_id
->route
.addr
.src_addr
;
1966 sprintf(traddr
, "%pISc", addr
);
1968 memcpy(traddr
, nport
->disc_addr
.traddr
, NVMF_TRADDR_SIZE
);
1972 static u8
nvmet_rdma_get_mdts(const struct nvmet_ctrl
*ctrl
)
1974 if (ctrl
->pi_support
)
1975 return NVMET_RDMA_MAX_METADATA_MDTS
;
1976 return NVMET_RDMA_MAX_MDTS
;
1979 static const struct nvmet_fabrics_ops nvmet_rdma_ops
= {
1980 .owner
= THIS_MODULE
,
1981 .type
= NVMF_TRTYPE_RDMA
,
1983 .flags
= NVMF_KEYED_SGLS
| NVMF_METADATA_SUPPORTED
,
1984 .add_port
= nvmet_rdma_add_port
,
1985 .remove_port
= nvmet_rdma_remove_port
,
1986 .queue_response
= nvmet_rdma_queue_response
,
1987 .delete_ctrl
= nvmet_rdma_delete_ctrl
,
1988 .disc_traddr
= nvmet_rdma_disc_port_addr
,
1989 .get_mdts
= nvmet_rdma_get_mdts
,
1992 static void nvmet_rdma_remove_one(struct ib_device
*ib_device
, void *client_data
)
1994 struct nvmet_rdma_queue
*queue
, *tmp
;
1995 struct nvmet_rdma_device
*ndev
;
1998 mutex_lock(&device_list_mutex
);
1999 list_for_each_entry(ndev
, &device_list
, entry
) {
2000 if (ndev
->device
== ib_device
) {
2005 mutex_unlock(&device_list_mutex
);
2011 * IB Device that is used by nvmet controllers is being removed,
2012 * delete all queues using this device.
2014 mutex_lock(&nvmet_rdma_queue_mutex
);
2015 list_for_each_entry_safe(queue
, tmp
, &nvmet_rdma_queue_list
,
2017 if (queue
->dev
->device
!= ib_device
)
2020 pr_info("Removing queue %d\n", queue
->idx
);
2021 list_del_init(&queue
->queue_list
);
2022 __nvmet_rdma_queue_disconnect(queue
);
2024 mutex_unlock(&nvmet_rdma_queue_mutex
);
2026 flush_scheduled_work();
2029 static struct ib_client nvmet_rdma_ib_client
= {
2030 .name
= "nvmet_rdma",
2031 .remove
= nvmet_rdma_remove_one
2034 static int __init
nvmet_rdma_init(void)
2038 ret
= ib_register_client(&nvmet_rdma_ib_client
);
2042 ret
= nvmet_register_transport(&nvmet_rdma_ops
);
2049 ib_unregister_client(&nvmet_rdma_ib_client
);
2053 static void __exit
nvmet_rdma_exit(void)
2055 nvmet_unregister_transport(&nvmet_rdma_ops
);
2056 ib_unregister_client(&nvmet_rdma_ib_client
);
2057 WARN_ON_ONCE(!list_empty(&nvmet_rdma_queue_list
));
2058 ida_destroy(&nvmet_rdma_queue_ida
);
2061 module_init(nvmet_rdma_init
);
2062 module_exit(nvmet_rdma_exit
);
2064 MODULE_LICENSE("GPL v2");
2065 MODULE_ALIAS("nvmet-transport-1"); /* 1 == NVMF_TRTYPE_RDMA */