]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - drivers/nvme/host/rdma.c
projects / mirror_ubuntu-jammy-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'timers-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
[mirror_ubuntu-jammy-kernel.git] / drivers / nvme / host / rdma.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * NVMe over Fabrics RDMA host code.
4 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
5 */
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 #include <linux/module.h>
8 #include <linux/init.h>
9 #include <linux/slab.h>
10 #include <rdma/mr_pool.h>
11 #include <linux/err.h>
12 #include <linux/string.h>
13 #include <linux/atomic.h>
14 #include <linux/blk-mq.h>
15 #include <linux/blk-mq-rdma.h>
16 #include <linux/types.h>
17 #include <linux/list.h>
18 #include <linux/mutex.h>
19 #include <linux/scatterlist.h>
20 #include <linux/nvme.h>
21 #include <asm/unaligned.h>
22
23 #include <rdma/ib_verbs.h>
24 #include <rdma/rdma_cm.h>
25 #include <linux/nvme-rdma.h>
26
27 #include "nvme.h"
28 #include "fabrics.h"
29
30
31 #define NVME_RDMA_CONNECT_TIMEOUT_MS 3000 /* 3 second */
32
33 #define NVME_RDMA_MAX_SEGMENTS 256
34
35 #define NVME_RDMA_MAX_INLINE_SEGMENTS 4
36
37 struct nvme_rdma_device {
38 struct ib_device *dev;
39 struct ib_pd *pd;
40 struct kref ref;
41 struct list_head entry;
42 unsigned int num_inline_segments;
43 };
44
45 struct nvme_rdma_qe {
46 struct ib_cqe cqe;
47 void *data;
48 u64 dma;
49 };
50
51 struct nvme_rdma_queue;
52 struct nvme_rdma_request {
53 struct nvme_request req;
54 struct ib_mr *mr;
55 struct nvme_rdma_qe sqe;
56 union nvme_result result;
57 __le16 status;
58 refcount_t ref;
59 struct ib_sge sge[1 + NVME_RDMA_MAX_INLINE_SEGMENTS];
60 u32 num_sge;
61 int nents;
62 struct ib_reg_wr reg_wr;
63 struct ib_cqe reg_cqe;
64 struct nvme_rdma_queue *queue;
65 struct sg_table sg_table;
66 struct scatterlist first_sgl[];
67 };
68
69 enum nvme_rdma_queue_flags {
70 NVME_RDMA_Q_ALLOCATED = 0,
71 NVME_RDMA_Q_LIVE = 1,
72 NVME_RDMA_Q_TR_READY = 2,
73 };
74
75 struct nvme_rdma_queue {
76 struct nvme_rdma_qe *rsp_ring;
77 int queue_size;
78 size_t cmnd_capsule_len;
79 struct nvme_rdma_ctrl *ctrl;
80 struct nvme_rdma_device *device;
81 struct ib_cq *ib_cq;
82 struct ib_qp *qp;
83
84 unsigned long flags;
85 struct rdma_cm_id *cm_id;
86 int cm_error;
87 struct completion cm_done;
88 };
89
90 struct nvme_rdma_ctrl {
91 /* read only in the hot path */
92 struct nvme_rdma_queue *queues;
93
94 /* other member variables */
95 struct blk_mq_tag_set tag_set;
96 struct work_struct err_work;
97
98 struct nvme_rdma_qe async_event_sqe;
99
100 struct delayed_work reconnect_work;
101
102 struct list_head list;
103
104 struct blk_mq_tag_set admin_tag_set;
105 struct nvme_rdma_device *device;
106
107 u32 max_fr_pages;
108
109 struct sockaddr_storage addr;
110 struct sockaddr_storage src_addr;
111
112 struct nvme_ctrl ctrl;
113 bool use_inline_data;
114 u32 io_queues[HCTX_MAX_TYPES];
115 };
116
117 static inline struct nvme_rdma_ctrl *to_rdma_ctrl(struct nvme_ctrl *ctrl)
118 {
119 return container_of(ctrl, struct nvme_rdma_ctrl, ctrl);
120 }
121
122 static LIST_HEAD(device_list);
123 static DEFINE_MUTEX(device_list_mutex);
124
125 static LIST_HEAD(nvme_rdma_ctrl_list);
126 static DEFINE_MUTEX(nvme_rdma_ctrl_mutex);
127
128 /*
129 * Disabling this option makes small I/O goes faster, but is fundamentally
130 * unsafe. With it turned off we will have to register a global rkey that
131 * allows read and write access to all physical memory.
132 */
133 static bool register_always = true;
134 module_param(register_always, bool, 0444);
135 MODULE_PARM_DESC(register_always,
136 "Use memory registration even for contiguous memory regions");
137
138 static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
139 struct rdma_cm_event *event);
140 static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc);
141
142 static const struct blk_mq_ops nvme_rdma_mq_ops;
143 static const struct blk_mq_ops nvme_rdma_admin_mq_ops;
144
145 /* XXX: really should move to a generic header sooner or later.. */
146 static inline void put_unaligned_le24(u32 val, u8 *p)
147 {
148 *p++ = val;
149 *p++ = val >> 8;
150 *p++ = val >> 16;
151 }
152
153 static inline int nvme_rdma_queue_idx(struct nvme_rdma_queue *queue)
154 {
155 return queue - queue->ctrl->queues;
156 }
157
158 static bool nvme_rdma_poll_queue(struct nvme_rdma_queue *queue)
159 {
160 return nvme_rdma_queue_idx(queue) >
161 queue->ctrl->io_queues[HCTX_TYPE_DEFAULT] +
162 queue->ctrl->io_queues[HCTX_TYPE_READ];
163 }
164
165 static inline size_t nvme_rdma_inline_data_size(struct nvme_rdma_queue *queue)
166 {
167 return queue->cmnd_capsule_len - sizeof(struct nvme_command);
168 }
169
170 static void nvme_rdma_free_qe(struct ib_device *ibdev, struct nvme_rdma_qe *qe,
171 size_t capsule_size, enum dma_data_direction dir)
172 {
173 ib_dma_unmap_single(ibdev, qe->dma, capsule_size, dir);
174 kfree(qe->data);
175 }
176
177 static int nvme_rdma_alloc_qe(struct ib_device *ibdev, struct nvme_rdma_qe *qe,
178 size_t capsule_size, enum dma_data_direction dir)
179 {
180 qe->data = kzalloc(capsule_size, GFP_KERNEL);
181 if (!qe->data)
182 return -ENOMEM;
183
184 qe->dma = ib_dma_map_single(ibdev, qe->data, capsule_size, dir);
185 if (ib_dma_mapping_error(ibdev, qe->dma)) {
186 kfree(qe->data);
187 qe->data = NULL;
188 return -ENOMEM;
189 }
190
191 return 0;
192 }
193
194 static void nvme_rdma_free_ring(struct ib_device *ibdev,
195 struct nvme_rdma_qe *ring, size_t ib_queue_size,
196 size_t capsule_size, enum dma_data_direction dir)
197 {
198 int i;
199
200 for (i = 0; i < ib_queue_size; i++)
201 nvme_rdma_free_qe(ibdev, &ring[i], capsule_size, dir);
202 kfree(ring);
203 }
204
205 static struct nvme_rdma_qe *nvme_rdma_alloc_ring(struct ib_device *ibdev,
206 size_t ib_queue_size, size_t capsule_size,
207 enum dma_data_direction dir)
208 {
209 struct nvme_rdma_qe *ring;
210 int i;
211
212 ring = kcalloc(ib_queue_size, sizeof(struct nvme_rdma_qe), GFP_KERNEL);
213 if (!ring)
214 return NULL;
215
216 /*
217 * Bind the CQEs (post recv buffers) DMA mapping to the RDMA queue
218 * lifetime. It's safe, since any chage in the underlying RDMA device
219 * will issue error recovery and queue re-creation.
220 */
221 for (i = 0; i < ib_queue_size; i++) {
222 if (nvme_rdma_alloc_qe(ibdev, &ring[i], capsule_size, dir))
223 goto out_free_ring;
224 }
225
226 return ring;
227
228 out_free_ring:
229 nvme_rdma_free_ring(ibdev, ring, i, capsule_size, dir);
230 return NULL;
231 }
232
233 static void nvme_rdma_qp_event(struct ib_event *event, void *context)
234 {
235 pr_debug("QP event %s (%d)\n",
236 ib_event_msg(event->event), event->event);
237
238 }
239
240 static int nvme_rdma_wait_for_cm(struct nvme_rdma_queue *queue)
241 {
242 int ret;
243
244 ret = wait_for_completion_interruptible_timeout(&queue->cm_done,
245 msecs_to_jiffies(NVME_RDMA_CONNECT_TIMEOUT_MS) + 1);
246 if (ret < 0)
247 return ret;
248 if (ret == 0)
249 return -ETIMEDOUT;
250 WARN_ON_ONCE(queue->cm_error > 0);
251 return queue->cm_error;
252 }
253
254 static int nvme_rdma_create_qp(struct nvme_rdma_queue *queue, const int factor)
255 {
256 struct nvme_rdma_device *dev = queue->device;
257 struct ib_qp_init_attr init_attr;
258 int ret;
259
260 memset(&init_attr, 0, sizeof(init_attr));
261 init_attr.event_handler = nvme_rdma_qp_event;
262 /* +1 for drain */
263 init_attr.cap.max_send_wr = factor * queue->queue_size + 1;
264 /* +1 for drain */
265 init_attr.cap.max_recv_wr = queue->queue_size + 1;
266 init_attr.cap.max_recv_sge = 1;
267 init_attr.cap.max_send_sge = 1 + dev->num_inline_segments;
268 init_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
269 init_attr.qp_type = IB_QPT_RC;
270 init_attr.send_cq = queue->ib_cq;
271 init_attr.recv_cq = queue->ib_cq;
272
273 ret = rdma_create_qp(queue->cm_id, dev->pd, &init_attr);
274
275 queue->qp = queue->cm_id->qp;
276 return ret;
277 }
278
279 static void nvme_rdma_exit_request(struct blk_mq_tag_set *set,
280 struct request *rq, unsigned int hctx_idx)
281 {
282 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
283
284 kfree(req->sqe.data);
285 }
286
287 static int nvme_rdma_init_request(struct blk_mq_tag_set *set,
288 struct request *rq, unsigned int hctx_idx,
289 unsigned int numa_node)
290 {
291 struct nvme_rdma_ctrl *ctrl = set->driver_data;
292 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
293 int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
294 struct nvme_rdma_queue *queue = &ctrl->queues[queue_idx];
295
296 nvme_req(rq)->ctrl = &ctrl->ctrl;
297 req->sqe.data = kzalloc(sizeof(struct nvme_command), GFP_KERNEL);
298 if (!req->sqe.data)
299 return -ENOMEM;
300
301 req->queue = queue;
302
303 return 0;
304 }
305
306 static int nvme_rdma_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
307 unsigned int hctx_idx)
308 {
309 struct nvme_rdma_ctrl *ctrl = data;
310 struct nvme_rdma_queue *queue = &ctrl->queues[hctx_idx + 1];
311
312 BUG_ON(hctx_idx >= ctrl->ctrl.queue_count);
313
314 hctx->driver_data = queue;
315 return 0;
316 }
317
318 static int nvme_rdma_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
319 unsigned int hctx_idx)
320 {
321 struct nvme_rdma_ctrl *ctrl = data;
322 struct nvme_rdma_queue *queue = &ctrl->queues[0];
323
324 BUG_ON(hctx_idx != 0);
325
326 hctx->driver_data = queue;
327 return 0;
328 }
329
330 static void nvme_rdma_free_dev(struct kref *ref)
331 {
332 struct nvme_rdma_device *ndev =
333 container_of(ref, struct nvme_rdma_device, ref);
334
335 mutex_lock(&device_list_mutex);
336 list_del(&ndev->entry);
337 mutex_unlock(&device_list_mutex);
338
339 ib_dealloc_pd(ndev->pd);
340 kfree(ndev);
341 }
342
343 static void nvme_rdma_dev_put(struct nvme_rdma_device *dev)
344 {
345 kref_put(&dev->ref, nvme_rdma_free_dev);
346 }
347
348 static int nvme_rdma_dev_get(struct nvme_rdma_device *dev)
349 {
350 return kref_get_unless_zero(&dev->ref);
351 }
352
353 static struct nvme_rdma_device *
354 nvme_rdma_find_get_device(struct rdma_cm_id *cm_id)
355 {
356 struct nvme_rdma_device *ndev;
357
358 mutex_lock(&device_list_mutex);
359 list_for_each_entry(ndev, &device_list, entry) {
360 if (ndev->dev->node_guid == cm_id->device->node_guid &&
361 nvme_rdma_dev_get(ndev))
362 goto out_unlock;
363 }
364
365 ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
366 if (!ndev)
367 goto out_err;
368
369 ndev->dev = cm_id->device;
370 kref_init(&ndev->ref);
371
372 ndev->pd = ib_alloc_pd(ndev->dev,
373 register_always ? 0 : IB_PD_UNSAFE_GLOBAL_RKEY);
374 if (IS_ERR(ndev->pd))
375 goto out_free_dev;
376
377 if (!(ndev->dev->attrs.device_cap_flags &
378 IB_DEVICE_MEM_MGT_EXTENSIONS)) {
379 dev_err(&ndev->dev->dev,
380 "Memory registrations not supported.\n");
381 goto out_free_pd;
382 }
383
384 ndev->num_inline_segments = min(NVME_RDMA_MAX_INLINE_SEGMENTS,
385 ndev->dev->attrs.max_send_sge - 1);
386 list_add(&ndev->entry, &device_list);
387 out_unlock:
388 mutex_unlock(&device_list_mutex);
389 return ndev;
390
391 out_free_pd:
392 ib_dealloc_pd(ndev->pd);
393 out_free_dev:
394 kfree(ndev);
395 out_err:
396 mutex_unlock(&device_list_mutex);
397 return NULL;
398 }
399
400 static void nvme_rdma_destroy_queue_ib(struct nvme_rdma_queue *queue)
401 {
402 struct nvme_rdma_device *dev;
403 struct ib_device *ibdev;
404
405 if (!test_and_clear_bit(NVME_RDMA_Q_TR_READY, &queue->flags))
406 return;
407
408 dev = queue->device;
409 ibdev = dev->dev;
410
411 ib_mr_pool_destroy(queue->qp, &queue->qp->rdma_mrs);
412
413 /*
414 * The cm_id object might have been destroyed during RDMA connection
415 * establishment error flow to avoid getting other cma events, thus
416 * the destruction of the QP shouldn't use rdma_cm API.
417 */
418 ib_destroy_qp(queue->qp);
419 ib_free_cq(queue->ib_cq);
420
421 nvme_rdma_free_ring(ibdev, queue->rsp_ring, queue->queue_size,
422 sizeof(struct nvme_completion), DMA_FROM_DEVICE);
423
424 nvme_rdma_dev_put(dev);
425 }
426
427 static int nvme_rdma_get_max_fr_pages(struct ib_device *ibdev)
428 {
429 return min_t(u32, NVME_RDMA_MAX_SEGMENTS,
430 ibdev->attrs.max_fast_reg_page_list_len);
431 }
432
433 static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue *queue)
434 {
435 struct ib_device *ibdev;
436 const int send_wr_factor = 3; /* MR, SEND, INV */
437 const int cq_factor = send_wr_factor + 1; /* + RECV */
438 int comp_vector, idx = nvme_rdma_queue_idx(queue);
439 enum ib_poll_context poll_ctx;
440 int ret;
441
442 queue->device = nvme_rdma_find_get_device(queue->cm_id);
443 if (!queue->device) {
444 dev_err(queue->cm_id->device->dev.parent,
445 "no client data found!\n");
446 return -ECONNREFUSED;
447 }
448 ibdev = queue->device->dev;
449
450 /*
451 * Spread I/O queues completion vectors according their queue index.
452 * Admin queues can always go on completion vector 0.
453 */
454 comp_vector = idx == 0 ? idx : idx - 1;
455
456 /* Polling queues need direct cq polling context */
457 if (nvme_rdma_poll_queue(queue))
458 poll_ctx = IB_POLL_DIRECT;
459 else
460 poll_ctx = IB_POLL_SOFTIRQ;
461
462 /* +1 for ib_stop_cq */
463 queue->ib_cq = ib_alloc_cq(ibdev, queue,
464 cq_factor * queue->queue_size + 1,
465 comp_vector, poll_ctx);
466 if (IS_ERR(queue->ib_cq)) {
467 ret = PTR_ERR(queue->ib_cq);
468 goto out_put_dev;
469 }
470
471 ret = nvme_rdma_create_qp(queue, send_wr_factor);
472 if (ret)
473 goto out_destroy_ib_cq;
474
475 queue->rsp_ring = nvme_rdma_alloc_ring(ibdev, queue->queue_size,
476 sizeof(struct nvme_completion), DMA_FROM_DEVICE);
477 if (!queue->rsp_ring) {
478 ret = -ENOMEM;
479 goto out_destroy_qp;
480 }
481
482 ret = ib_mr_pool_init(queue->qp, &queue->qp->rdma_mrs,
483 queue->queue_size,
484 IB_MR_TYPE_MEM_REG,
485 nvme_rdma_get_max_fr_pages(ibdev));
486 if (ret) {
487 dev_err(queue->ctrl->ctrl.device,
488 "failed to initialize MR pool sized %d for QID %d\n",
489 queue->queue_size, idx);
490 goto out_destroy_ring;
491 }
492
493 set_bit(NVME_RDMA_Q_TR_READY, &queue->flags);
494
495 return 0;
496
497 out_destroy_ring:
498 nvme_rdma_free_ring(ibdev, queue->rsp_ring, queue->queue_size,
499 sizeof(struct nvme_completion), DMA_FROM_DEVICE);
500 out_destroy_qp:
501 rdma_destroy_qp(queue->cm_id);
502 out_destroy_ib_cq:
503 ib_free_cq(queue->ib_cq);
504 out_put_dev:
505 nvme_rdma_dev_put(queue->device);
506 return ret;
507 }
508
509 static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl,
510 int idx, size_t queue_size)
511 {
512 struct nvme_rdma_queue *queue;
513 struct sockaddr *src_addr = NULL;
514 int ret;
515
516 queue = &ctrl->queues[idx];
517 queue->ctrl = ctrl;
518 init_completion(&queue->cm_done);
519
520 if (idx > 0)
521 queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;
522 else
523 queue->cmnd_capsule_len = sizeof(struct nvme_command);
524
525 queue->queue_size = queue_size;
526
527 queue->cm_id = rdma_create_id(&init_net, nvme_rdma_cm_handler, queue,
528 RDMA_PS_TCP, IB_QPT_RC);
529 if (IS_ERR(queue->cm_id)) {
530 dev_info(ctrl->ctrl.device,
531 "failed to create CM ID: %ld\n", PTR_ERR(queue->cm_id));
532 return PTR_ERR(queue->cm_id);
533 }
534
535 if (ctrl->ctrl.opts->mask & NVMF_OPT_HOST_TRADDR)
536 src_addr = (struct sockaddr *)&ctrl->src_addr;
537
538 queue->cm_error = -ETIMEDOUT;
539 ret = rdma_resolve_addr(queue->cm_id, src_addr,
540 (struct sockaddr *)&ctrl->addr,
541 NVME_RDMA_CONNECT_TIMEOUT_MS);
542 if (ret) {
543 dev_info(ctrl->ctrl.device,
544 "rdma_resolve_addr failed (%d).\n", ret);
545 goto out_destroy_cm_id;
546 }
547
548 ret = nvme_rdma_wait_for_cm(queue);
549 if (ret) {
550 dev_info(ctrl->ctrl.device,
551 "rdma connection establishment failed (%d)\n", ret);
552 goto out_destroy_cm_id;
553 }
554
555 set_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags);
556
557 return 0;
558
559 out_destroy_cm_id:
560 rdma_destroy_id(queue->cm_id);
561 nvme_rdma_destroy_queue_ib(queue);
562 return ret;
563 }
564
565 static void nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
566 {
567 if (!test_and_clear_bit(NVME_RDMA_Q_LIVE, &queue->flags))
568 return;
569
570 rdma_disconnect(queue->cm_id);
571 ib_drain_qp(queue->qp);
572 }
573
574 static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)
575 {
576 if (!test_and_clear_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags))
577 return;
578
579 nvme_rdma_destroy_queue_ib(queue);
580 rdma_destroy_id(queue->cm_id);
581 }
582
583 static void nvme_rdma_free_io_queues(struct nvme_rdma_ctrl *ctrl)
584 {
585 int i;
586
587 for (i = 1; i < ctrl->ctrl.queue_count; i++)
588 nvme_rdma_free_queue(&ctrl->queues[i]);
589 }
590
591 static void nvme_rdma_stop_io_queues(struct nvme_rdma_ctrl *ctrl)
592 {
593 int i;
594
595 for (i = 1; i < ctrl->ctrl.queue_count; i++)
596 nvme_rdma_stop_queue(&ctrl->queues[i]);
597 }
598
599 static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx)
600 {
601 struct nvme_rdma_queue *queue = &ctrl->queues[idx];
602 bool poll = nvme_rdma_poll_queue(queue);
603 int ret;
604
605 if (idx)
606 ret = nvmf_connect_io_queue(&ctrl->ctrl, idx, poll);
607 else
608 ret = nvmf_connect_admin_queue(&ctrl->ctrl);
609
610 if (!ret)
611 set_bit(NVME_RDMA_Q_LIVE, &queue->flags);
612 else
613 dev_info(ctrl->ctrl.device,
614 "failed to connect queue: %d ret=%d\n", idx, ret);
615 return ret;
616 }
617
618 static int nvme_rdma_start_io_queues(struct nvme_rdma_ctrl *ctrl)
619 {
620 int i, ret = 0;
621
622 for (i = 1; i < ctrl->ctrl.queue_count; i++) {
623 ret = nvme_rdma_start_queue(ctrl, i);
624 if (ret)
625 goto out_stop_queues;
626 }
627
628 return 0;
629
630 out_stop_queues:
631 for (i--; i >= 1; i--)
632 nvme_rdma_stop_queue(&ctrl->queues[i]);
633 return ret;
634 }
635
636 static int nvme_rdma_alloc_io_queues(struct nvme_rdma_ctrl *ctrl)
637 {
638 struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
639 struct ib_device *ibdev = ctrl->device->dev;
640 unsigned int nr_io_queues, nr_default_queues;
641 unsigned int nr_read_queues, nr_poll_queues;
642 int i, ret;
643
644 nr_read_queues = min_t(unsigned int, ibdev->num_comp_vectors,
645 min(opts->nr_io_queues, num_online_cpus()));
646 nr_default_queues = min_t(unsigned int, ibdev->num_comp_vectors,
647 min(opts->nr_write_queues, num_online_cpus()));
648 nr_poll_queues = min(opts->nr_poll_queues, num_online_cpus());
649 nr_io_queues = nr_read_queues + nr_default_queues + nr_poll_queues;
650
651 ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
652 if (ret)
653 return ret;
654
655 ctrl->ctrl.queue_count = nr_io_queues + 1;
656 if (ctrl->ctrl.queue_count < 2)
657 return 0;
658
659 dev_info(ctrl->ctrl.device,
660 "creating %d I/O queues.\n", nr_io_queues);
661
662 if (opts->nr_write_queues && nr_read_queues < nr_io_queues) {
663 /*
664 * separate read/write queues
665 * hand out dedicated default queues only after we have
666 * sufficient read queues.
667 */
668 ctrl->io_queues[HCTX_TYPE_READ] = nr_read_queues;
669 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_READ];
670 ctrl->io_queues[HCTX_TYPE_DEFAULT] =
671 min(nr_default_queues, nr_io_queues);
672 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
673 } else {
674 /*
675 * shared read/write queues
676 * either no write queues were requested, or we don't have
677 * sufficient queue count to have dedicated default queues.
678 */
679 ctrl->io_queues[HCTX_TYPE_DEFAULT] =
680 min(nr_read_queues, nr_io_queues);
681 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
682 }
683
684 if (opts->nr_poll_queues && nr_io_queues) {
685 /* map dedicated poll queues only if we have queues left */
686 ctrl->io_queues[HCTX_TYPE_POLL] =
687 min(nr_poll_queues, nr_io_queues);
688 }
689
690 for (i = 1; i < ctrl->ctrl.queue_count; i++) {
691 ret = nvme_rdma_alloc_queue(ctrl, i,
692 ctrl->ctrl.sqsize + 1);
693 if (ret)
694 goto out_free_queues;
695 }
696
697 return 0;
698
699 out_free_queues:
700 for (i--; i >= 1; i--)
701 nvme_rdma_free_queue(&ctrl->queues[i]);
702
703 return ret;
704 }
705
706 static struct blk_mq_tag_set *nvme_rdma_alloc_tagset(struct nvme_ctrl *nctrl,
707 bool admin)
708 {
709 struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
710 struct blk_mq_tag_set *set;
711 int ret;
712
713 if (admin) {
714 set = &ctrl->admin_tag_set;
715 memset(set, 0, sizeof(*set));
716 set->ops = &nvme_rdma_admin_mq_ops;
717 set->queue_depth = NVME_AQ_MQ_TAG_DEPTH;
718 set->reserved_tags = 2; /* connect + keep-alive */
719 set->numa_node = nctrl->numa_node;
720 set->cmd_size = sizeof(struct nvme_rdma_request) +
721 SG_CHUNK_SIZE * sizeof(struct scatterlist);
722 set->driver_data = ctrl;
723 set->nr_hw_queues = 1;
724 set->timeout = ADMIN_TIMEOUT;
725 set->flags = BLK_MQ_F_NO_SCHED;
726 } else {
727 set = &ctrl->tag_set;
728 memset(set, 0, sizeof(*set));
729 set->ops = &nvme_rdma_mq_ops;
730 set->queue_depth = nctrl->sqsize + 1;
731 set->reserved_tags = 1; /* fabric connect */
732 set->numa_node = nctrl->numa_node;
733 set->flags = BLK_MQ_F_SHOULD_MERGE;
734 set->cmd_size = sizeof(struct nvme_rdma_request) +
735 SG_CHUNK_SIZE * sizeof(struct scatterlist);
736 set->driver_data = ctrl;
737 set->nr_hw_queues = nctrl->queue_count - 1;
738 set->timeout = NVME_IO_TIMEOUT;
739 set->nr_maps = nctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2;
740 }
741
742 ret = blk_mq_alloc_tag_set(set);
743 if (ret)
744 return ERR_PTR(ret);
745
746 return set;
747 }
748
749 static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl,
750 bool remove)
751 {
752 if (remove) {
753 blk_cleanup_queue(ctrl->ctrl.admin_q);
754 blk_mq_free_tag_set(ctrl->ctrl.admin_tagset);
755 }
756 if (ctrl->async_event_sqe.data) {
757 nvme_rdma_free_qe(ctrl->device->dev, &ctrl->async_event_sqe,
758 sizeof(struct nvme_command), DMA_TO_DEVICE);
759 ctrl->async_event_sqe.data = NULL;
760 }
761 nvme_rdma_free_queue(&ctrl->queues[0]);
762 }
763
764 static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
765 bool new)
766 {
767 int error;
768
769 error = nvme_rdma_alloc_queue(ctrl, 0, NVME_AQ_DEPTH);
770 if (error)
771 return error;
772
773 ctrl->device = ctrl->queues[0].device;
774 ctrl->ctrl.numa_node = dev_to_node(ctrl->device->dev->dma_device);
775
776 ctrl->max_fr_pages = nvme_rdma_get_max_fr_pages(ctrl->device->dev);
777
778 /*
779 * Bind the async event SQE DMA mapping to the admin queue lifetime.
780 * It's safe, since any chage in the underlying RDMA device will issue
781 * error recovery and queue re-creation.
782 */
783 error = nvme_rdma_alloc_qe(ctrl->device->dev, &ctrl->async_event_sqe,
784 sizeof(struct nvme_command), DMA_TO_DEVICE);
785 if (error)
786 goto out_free_queue;
787
788 if (new) {
789 ctrl->ctrl.admin_tagset = nvme_rdma_alloc_tagset(&ctrl->ctrl, true);
790 if (IS_ERR(ctrl->ctrl.admin_tagset)) {
791 error = PTR_ERR(ctrl->ctrl.admin_tagset);
792 goto out_free_async_qe;
793 }
794
795 ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);
796 if (IS_ERR(ctrl->ctrl.admin_q)) {
797 error = PTR_ERR(ctrl->ctrl.admin_q);
798 goto out_free_tagset;
799 }
800 }
801
802 error = nvme_rdma_start_queue(ctrl, 0);
803 if (error)
804 goto out_cleanup_queue;
805
806 error = ctrl->ctrl.ops->reg_read64(&ctrl->ctrl, NVME_REG_CAP,
807 &ctrl->ctrl.cap);
808 if (error) {
809 dev_err(ctrl->ctrl.device,
810 "prop_get NVME_REG_CAP failed\n");
811 goto out_stop_queue;
812 }
813
814 ctrl->ctrl.sqsize =
815 min_t(int, NVME_CAP_MQES(ctrl->ctrl.cap), ctrl->ctrl.sqsize);
816
817 error = nvme_enable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap);
818 if (error)
819 goto out_stop_queue;
820
821 ctrl->ctrl.max_hw_sectors =
822 (ctrl->max_fr_pages - 1) << (ilog2(SZ_4K) - 9);
823
824 error = nvme_init_identify(&ctrl->ctrl);
825 if (error)
826 goto out_stop_queue;
827
828 return 0;
829
830 out_stop_queue:
831 nvme_rdma_stop_queue(&ctrl->queues[0]);
832 out_cleanup_queue:
833 if (new)
834 blk_cleanup_queue(ctrl->ctrl.admin_q);
835 out_free_tagset:
836 if (new)
837 blk_mq_free_tag_set(ctrl->ctrl.admin_tagset);
838 out_free_async_qe:
839 nvme_rdma_free_qe(ctrl->device->dev, &ctrl->async_event_sqe,
840 sizeof(struct nvme_command), DMA_TO_DEVICE);
841 ctrl->async_event_sqe.data = NULL;
842 out_free_queue:
843 nvme_rdma_free_queue(&ctrl->queues[0]);
844 return error;
845 }
846
847 static void nvme_rdma_destroy_io_queues(struct nvme_rdma_ctrl *ctrl,
848 bool remove)
849 {
850 if (remove) {
851 blk_cleanup_queue(ctrl->ctrl.connect_q);
852 blk_mq_free_tag_set(ctrl->ctrl.tagset);
853 }
854 nvme_rdma_free_io_queues(ctrl);
855 }
856
857 static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new)
858 {
859 int ret;
860
861 ret = nvme_rdma_alloc_io_queues(ctrl);
862 if (ret)
863 return ret;
864
865 if (new) {
866 ctrl->ctrl.tagset = nvme_rdma_alloc_tagset(&ctrl->ctrl, false);
867 if (IS_ERR(ctrl->ctrl.tagset)) {
868 ret = PTR_ERR(ctrl->ctrl.tagset);
869 goto out_free_io_queues;
870 }
871
872 ctrl->ctrl.connect_q = blk_mq_init_queue(&ctrl->tag_set);
873 if (IS_ERR(ctrl->ctrl.connect_q)) {
874 ret = PTR_ERR(ctrl->ctrl.connect_q);
875 goto out_free_tag_set;
876 }
877 } else {
878 blk_mq_update_nr_hw_queues(&ctrl->tag_set,
879 ctrl->ctrl.queue_count - 1);
880 }
881
882 ret = nvme_rdma_start_io_queues(ctrl);
883 if (ret)
884 goto out_cleanup_connect_q;
885
886 return 0;
887
888 out_cleanup_connect_q:
889 if (new)
890 blk_cleanup_queue(ctrl->ctrl.connect_q);
891 out_free_tag_set:
892 if (new)
893 blk_mq_free_tag_set(ctrl->ctrl.tagset);
894 out_free_io_queues:
895 nvme_rdma_free_io_queues(ctrl);
896 return ret;
897 }
898
899 static void nvme_rdma_teardown_admin_queue(struct nvme_rdma_ctrl *ctrl,
900 bool remove)
901 {
902 blk_mq_quiesce_queue(ctrl->ctrl.admin_q);
903 nvme_rdma_stop_queue(&ctrl->queues[0]);
904 if (ctrl->ctrl.admin_tagset)
905 blk_mq_tagset_busy_iter(ctrl->ctrl.admin_tagset,
906 nvme_cancel_request, &ctrl->ctrl);
907 blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);
908 nvme_rdma_destroy_admin_queue(ctrl, remove);
909 }
910
911 static void nvme_rdma_teardown_io_queues(struct nvme_rdma_ctrl *ctrl,
912 bool remove)
913 {
914 if (ctrl->ctrl.queue_count > 1) {
915 nvme_stop_queues(&ctrl->ctrl);
916 nvme_rdma_stop_io_queues(ctrl);
917 if (ctrl->ctrl.tagset)
918 blk_mq_tagset_busy_iter(ctrl->ctrl.tagset,
919 nvme_cancel_request, &ctrl->ctrl);
920 if (remove)
921 nvme_start_queues(&ctrl->ctrl);
922 nvme_rdma_destroy_io_queues(ctrl, remove);
923 }
924 }
925
926 static void nvme_rdma_free_ctrl(struct nvme_ctrl *nctrl)
927 {
928 struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
929
930 if (list_empty(&ctrl->list))
931 goto free_ctrl;
932
933 mutex_lock(&nvme_rdma_ctrl_mutex);
934 list_del(&ctrl->list);
935 mutex_unlock(&nvme_rdma_ctrl_mutex);
936
937 nvmf_free_options(nctrl->opts);
938 free_ctrl:
939 kfree(ctrl->queues);
940 kfree(ctrl);
941 }
942
943 static void nvme_rdma_reconnect_or_remove(struct nvme_rdma_ctrl *ctrl)
944 {
945 /* If we are resetting/deleting then do nothing */
946 if (ctrl->ctrl.state != NVME_CTRL_CONNECTING) {
947 WARN_ON_ONCE(ctrl->ctrl.state == NVME_CTRL_NEW ||
948 ctrl->ctrl.state == NVME_CTRL_LIVE);
949 return;
950 }
951
952 if (nvmf_should_reconnect(&ctrl->ctrl)) {
953 dev_info(ctrl->ctrl.device, "Reconnecting in %d seconds...\n",
954 ctrl->ctrl.opts->reconnect_delay);
955 queue_delayed_work(nvme_wq, &ctrl->reconnect_work,
956 ctrl->ctrl.opts->reconnect_delay * HZ);
957 } else {
958 nvme_delete_ctrl(&ctrl->ctrl);
959 }
960 }
961
962 static int nvme_rdma_setup_ctrl(struct nvme_rdma_ctrl *ctrl, bool new)
963 {
964 int ret = -EINVAL;
965 bool changed;
966
967 ret = nvme_rdma_configure_admin_queue(ctrl, new);
968 if (ret)
969 return ret;
970
971 if (ctrl->ctrl.icdoff) {
972 dev_err(ctrl->ctrl.device, "icdoff is not supported!\n");
973 goto destroy_admin;
974 }
975
976 if (!(ctrl->ctrl.sgls & (1 << 2))) {
977 dev_err(ctrl->ctrl.device,
978 "Mandatory keyed sgls are not supported!\n");
979 goto destroy_admin;
980 }
981
982 if (ctrl->ctrl.opts->queue_size > ctrl->ctrl.sqsize + 1) {
983 dev_warn(ctrl->ctrl.device,
984 "queue_size %zu > ctrl sqsize %u, clamping down\n",
985 ctrl->ctrl.opts->queue_size, ctrl->ctrl.sqsize + 1);
986 }
987
988 if (ctrl->ctrl.sqsize + 1 > ctrl->ctrl.maxcmd) {
989 dev_warn(ctrl->ctrl.device,
990 "sqsize %u > ctrl maxcmd %u, clamping down\n",
991 ctrl->ctrl.sqsize + 1, ctrl->ctrl.maxcmd);
992 ctrl->ctrl.sqsize = ctrl->ctrl.maxcmd - 1;
993 }
994
995 if (ctrl->ctrl.sgls & (1 << 20))
996 ctrl->use_inline_data = true;
997
998 if (ctrl->ctrl.queue_count > 1) {
999 ret = nvme_rdma_configure_io_queues(ctrl, new);
1000 if (ret)
1001 goto destroy_admin;
1002 }
1003
1004 changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
1005 if (!changed) {
1006 /* state change failure is ok if we're in DELETING state */
1007 WARN_ON_ONCE(ctrl->ctrl.state != NVME_CTRL_DELETING);
1008 ret = -EINVAL;
1009 goto destroy_io;
1010 }
1011
1012 nvme_start_ctrl(&ctrl->ctrl);
1013 return 0;
1014
1015 destroy_io:
1016 if (ctrl->ctrl.queue_count > 1)
1017 nvme_rdma_destroy_io_queues(ctrl, new);
1018 destroy_admin:
1019 nvme_rdma_stop_queue(&ctrl->queues[0]);
1020 nvme_rdma_destroy_admin_queue(ctrl, new);
1021 return ret;
1022 }
1023
1024 static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
1025 {
1026 struct nvme_rdma_ctrl *ctrl = container_of(to_delayed_work(work),
1027 struct nvme_rdma_ctrl, reconnect_work);
1028
1029 ++ctrl->ctrl.nr_reconnects;
1030
1031 if (nvme_rdma_setup_ctrl(ctrl, false))
1032 goto requeue;
1033
1034 dev_info(ctrl->ctrl.device, "Successfully reconnected (%d attempts)\n",
1035 ctrl->ctrl.nr_reconnects);
1036
1037 ctrl->ctrl.nr_reconnects = 0;
1038
1039 return;
1040
1041 requeue:
1042 dev_info(ctrl->ctrl.device, "Failed reconnect attempt %d\n",
1043 ctrl->ctrl.nr_reconnects);
1044 nvme_rdma_reconnect_or_remove(ctrl);
1045 }
1046
1047 static void nvme_rdma_error_recovery_work(struct work_struct *work)
1048 {
1049 struct nvme_rdma_ctrl *ctrl = container_of(work,
1050 struct nvme_rdma_ctrl, err_work);
1051
1052 nvme_stop_keep_alive(&ctrl->ctrl);
1053 nvme_rdma_teardown_io_queues(ctrl, false);
1054 nvme_start_queues(&ctrl->ctrl);
1055 nvme_rdma_teardown_admin_queue(ctrl, false);
1056
1057 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
1058 /* state change failure is ok if we're in DELETING state */
1059 WARN_ON_ONCE(ctrl->ctrl.state != NVME_CTRL_DELETING);
1060 return;
1061 }
1062
1063 nvme_rdma_reconnect_or_remove(ctrl);
1064 }
1065
1066 static void nvme_rdma_error_recovery(struct nvme_rdma_ctrl *ctrl)
1067 {
1068 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RESETTING))
1069 return;
1070
1071 queue_work(nvme_wq, &ctrl->err_work);
1072 }
1073
1074 static void nvme_rdma_wr_error(struct ib_cq *cq, struct ib_wc *wc,
1075 const char *op)
1076 {
1077 struct nvme_rdma_queue *queue = cq->cq_context;
1078 struct nvme_rdma_ctrl *ctrl = queue->ctrl;
1079
1080 if (ctrl->ctrl.state == NVME_CTRL_LIVE)
1081 dev_info(ctrl->ctrl.device,
1082 "%s for CQE 0x%p failed with status %s (%d)\n",
1083 op, wc->wr_cqe,
1084 ib_wc_status_msg(wc->status), wc->status);
1085 nvme_rdma_error_recovery(ctrl);
1086 }
1087
1088 static void nvme_rdma_memreg_done(struct ib_cq *cq, struct ib_wc *wc)
1089 {
1090 if (unlikely(wc->status != IB_WC_SUCCESS))
1091 nvme_rdma_wr_error(cq, wc, "MEMREG");
1092 }
1093
1094 static void nvme_rdma_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc)
1095 {
1096 struct nvme_rdma_request *req =
1097 container_of(wc->wr_cqe, struct nvme_rdma_request, reg_cqe);
1098 struct request *rq = blk_mq_rq_from_pdu(req);
1099
1100 if (unlikely(wc->status != IB_WC_SUCCESS)) {
1101 nvme_rdma_wr_error(cq, wc, "LOCAL_INV");
1102 return;
1103 }
1104
1105 if (refcount_dec_and_test(&req->ref))
1106 nvme_end_request(rq, req->status, req->result);
1107
1108 }
1109
1110 static int nvme_rdma_inv_rkey(struct nvme_rdma_queue *queue,
1111 struct nvme_rdma_request *req)
1112 {
1113 struct ib_send_wr wr = {
1114 .opcode = IB_WR_LOCAL_INV,
1115 .next = NULL,
1116 .num_sge = 0,
1117 .send_flags = IB_SEND_SIGNALED,
1118 .ex.invalidate_rkey = req->mr->rkey,
1119 };
1120
1121 req->reg_cqe.done = nvme_rdma_inv_rkey_done;
1122 wr.wr_cqe = &req->reg_cqe;
1123
1124 return ib_post_send(queue->qp, &wr, NULL);
1125 }
1126
1127 static void nvme_rdma_unmap_data(struct nvme_rdma_queue *queue,
1128 struct request *rq)
1129 {
1130 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1131 struct nvme_rdma_device *dev = queue->device;
1132 struct ib_device *ibdev = dev->dev;
1133
1134 if (!blk_rq_nr_phys_segments(rq))
1135 return;
1136
1137 if (req->mr) {
1138 ib_mr_pool_put(queue->qp, &queue->qp->rdma_mrs, req->mr);
1139 req->mr = NULL;
1140 }
1141
1142 ib_dma_unmap_sg(ibdev, req->sg_table.sgl,
1143 req->nents, rq_data_dir(rq) ==
1144 WRITE ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
1145
1146 nvme_cleanup_cmd(rq);
1147 sg_free_table_chained(&req->sg_table, true);
1148 }
1149
1150 static int nvme_rdma_set_sg_null(struct nvme_command *c)
1151 {
1152 struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
1153
1154 sg->addr = 0;
1155 put_unaligned_le24(0, sg->length);
1156 put_unaligned_le32(0, sg->key);
1157 sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4;
1158 return 0;
1159 }
1160
1161 static int nvme_rdma_map_sg_inline(struct nvme_rdma_queue *queue,
1162 struct nvme_rdma_request *req, struct nvme_command *c,
1163 int count)
1164 {
1165 struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
1166 struct scatterlist *sgl = req->sg_table.sgl;
1167 struct ib_sge *sge = &req->sge[1];
1168 u32 len = 0;
1169 int i;
1170
1171 for (i = 0; i < count; i++, sgl++, sge++) {
1172 sge->addr = sg_dma_address(sgl);
1173 sge->length = sg_dma_len(sgl);
1174 sge->lkey = queue->device->pd->local_dma_lkey;
1175 len += sge->length;
1176 }
1177
1178 sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);
1179 sg->length = cpu_to_le32(len);
1180 sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET;
1181
1182 req->num_sge += count;
1183 return 0;
1184 }
1185
1186 static int nvme_rdma_map_sg_single(struct nvme_rdma_queue *queue,
1187 struct nvme_rdma_request *req, struct nvme_command *c)
1188 {
1189 struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
1190
1191 sg->addr = cpu_to_le64(sg_dma_address(req->sg_table.sgl));
1192 put_unaligned_le24(sg_dma_len(req->sg_table.sgl), sg->length);
1193 put_unaligned_le32(queue->device->pd->unsafe_global_rkey, sg->key);
1194 sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4;
1195 return 0;
1196 }
1197
1198 static int nvme_rdma_map_sg_fr(struct nvme_rdma_queue *queue,
1199 struct nvme_rdma_request *req, struct nvme_command *c,
1200 int count)
1201 {
1202 struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
1203 int nr;
1204
1205 req->mr = ib_mr_pool_get(queue->qp, &queue->qp->rdma_mrs);
1206 if (WARN_ON_ONCE(!req->mr))
1207 return -EAGAIN;
1208
1209 /*
1210 * Align the MR to a 4K page size to match the ctrl page size and
1211 * the block virtual boundary.
1212 */
1213 nr = ib_map_mr_sg(req->mr, req->sg_table.sgl, count, NULL, SZ_4K);
1214 if (unlikely(nr < count)) {
1215 ib_mr_pool_put(queue->qp, &queue->qp->rdma_mrs, req->mr);
1216 req->mr = NULL;
1217 if (nr < 0)
1218 return nr;
1219 return -EINVAL;
1220 }
1221
1222 ib_update_fast_reg_key(req->mr, ib_inc_rkey(req->mr->rkey));
1223
1224 req->reg_cqe.done = nvme_rdma_memreg_done;
1225 memset(&req->reg_wr, 0, sizeof(req->reg_wr));
1226 req->reg_wr.wr.opcode = IB_WR_REG_MR;
1227 req->reg_wr.wr.wr_cqe = &req->reg_cqe;
1228 req->reg_wr.wr.num_sge = 0;
1229 req->reg_wr.mr = req->mr;
1230 req->reg_wr.key = req->mr->rkey;
1231 req->reg_wr.access = IB_ACCESS_LOCAL_WRITE |
1232 IB_ACCESS_REMOTE_READ |
1233 IB_ACCESS_REMOTE_WRITE;
1234
1235 sg->addr = cpu_to_le64(req->mr->iova);
1236 put_unaligned_le24(req->mr->length, sg->length);
1237 put_unaligned_le32(req->mr->rkey, sg->key);
1238 sg->type = (NVME_KEY_SGL_FMT_DATA_DESC << 4) |
1239 NVME_SGL_FMT_INVALIDATE;
1240
1241 return 0;
1242 }
1243
1244 static int nvme_rdma_map_data(struct nvme_rdma_queue *queue,
1245 struct request *rq, struct nvme_command *c)
1246 {
1247 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1248 struct nvme_rdma_device *dev = queue->device;
1249 struct ib_device *ibdev = dev->dev;
1250 int count, ret;
1251
1252 req->num_sge = 1;
1253 refcount_set(&req->ref, 2); /* send and recv completions */
1254
1255 c->common.flags |= NVME_CMD_SGL_METABUF;
1256
1257 if (!blk_rq_nr_phys_segments(rq))
1258 return nvme_rdma_set_sg_null(c);
1259
1260 req->sg_table.sgl = req->first_sgl;
1261 ret = sg_alloc_table_chained(&req->sg_table,
1262 blk_rq_nr_phys_segments(rq), req->sg_table.sgl);
1263 if (ret)
1264 return -ENOMEM;
1265
1266 req->nents = blk_rq_map_sg(rq->q, rq, req->sg_table.sgl);
1267
1268 count = ib_dma_map_sg(ibdev, req->sg_table.sgl, req->nents,
1269 rq_data_dir(rq) == WRITE ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
1270 if (unlikely(count <= 0)) {
1271 ret = -EIO;
1272 goto out_free_table;
1273 }
1274
1275 if (count <= dev->num_inline_segments) {
1276 if (rq_data_dir(rq) == WRITE && nvme_rdma_queue_idx(queue) &&
1277 queue->ctrl->use_inline_data &&
1278 blk_rq_payload_bytes(rq) <=
1279 nvme_rdma_inline_data_size(queue)) {
1280 ret = nvme_rdma_map_sg_inline(queue, req, c, count);
1281 goto out;
1282 }
1283
1284 if (count == 1 && dev->pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY) {
1285 ret = nvme_rdma_map_sg_single(queue, req, c);
1286 goto out;
1287 }
1288 }
1289
1290 ret = nvme_rdma_map_sg_fr(queue, req, c, count);
1291 out:
1292 if (unlikely(ret))
1293 goto out_unmap_sg;
1294
1295 return 0;
1296
1297 out_unmap_sg:
1298 ib_dma_unmap_sg(ibdev, req->sg_table.sgl,
1299 req->nents, rq_data_dir(rq) ==
1300 WRITE ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
1301 out_free_table:
1302 sg_free_table_chained(&req->sg_table, true);
1303 return ret;
1304 }
1305
1306 static void nvme_rdma_send_done(struct ib_cq *cq, struct ib_wc *wc)
1307 {
1308 struct nvme_rdma_qe *qe =
1309 container_of(wc->wr_cqe, struct nvme_rdma_qe, cqe);
1310 struct nvme_rdma_request *req =
1311 container_of(qe, struct nvme_rdma_request, sqe);
1312 struct request *rq = blk_mq_rq_from_pdu(req);
1313
1314 if (unlikely(wc->status != IB_WC_SUCCESS)) {
1315 nvme_rdma_wr_error(cq, wc, "SEND");
1316 return;
1317 }
1318
1319 if (refcount_dec_and_test(&req->ref))
1320 nvme_end_request(rq, req->status, req->result);
1321 }
1322
1323 static int nvme_rdma_post_send(struct nvme_rdma_queue *queue,
1324 struct nvme_rdma_qe *qe, struct ib_sge *sge, u32 num_sge,
1325 struct ib_send_wr *first)
1326 {
1327 struct ib_send_wr wr;
1328 int ret;
1329
1330 sge->addr = qe->dma;
1331 sge->length = sizeof(struct nvme_command),
1332 sge->lkey = queue->device->pd->local_dma_lkey;
1333
1334 wr.next = NULL;
1335 wr.wr_cqe = &qe->cqe;
1336 wr.sg_list = sge;
1337 wr.num_sge = num_sge;
1338 wr.opcode = IB_WR_SEND;
1339 wr.send_flags = IB_SEND_SIGNALED;
1340
1341 if (first)
1342 first->next = &wr;
1343 else
1344 first = &wr;
1345
1346 ret = ib_post_send(queue->qp, first, NULL);
1347 if (unlikely(ret)) {
1348 dev_err(queue->ctrl->ctrl.device,
1349 "%s failed with error code %d\n", __func__, ret);
1350 }
1351 return ret;
1352 }
1353
1354 static int nvme_rdma_post_recv(struct nvme_rdma_queue *queue,
1355 struct nvme_rdma_qe *qe)
1356 {
1357 struct ib_recv_wr wr;
1358 struct ib_sge list;
1359 int ret;
1360
1361 list.addr = qe->dma;
1362 list.length = sizeof(struct nvme_completion);
1363 list.lkey = queue->device->pd->local_dma_lkey;
1364
1365 qe->cqe.done = nvme_rdma_recv_done;
1366
1367 wr.next = NULL;
1368 wr.wr_cqe = &qe->cqe;
1369 wr.sg_list = &list;
1370 wr.num_sge = 1;
1371
1372 ret = ib_post_recv(queue->qp, &wr, NULL);
1373 if (unlikely(ret)) {
1374 dev_err(queue->ctrl->ctrl.device,
1375 "%s failed with error code %d\n", __func__, ret);
1376 }
1377 return ret;
1378 }
1379
1380 static struct blk_mq_tags *nvme_rdma_tagset(struct nvme_rdma_queue *queue)
1381 {
1382 u32 queue_idx = nvme_rdma_queue_idx(queue);
1383
1384 if (queue_idx == 0)
1385 return queue->ctrl->admin_tag_set.tags[queue_idx];
1386 return queue->ctrl->tag_set.tags[queue_idx - 1];
1387 }
1388
1389 static void nvme_rdma_async_done(struct ib_cq *cq, struct ib_wc *wc)
1390 {
1391 if (unlikely(wc->status != IB_WC_SUCCESS))
1392 nvme_rdma_wr_error(cq, wc, "ASYNC");
1393 }
1394
1395 static void nvme_rdma_submit_async_event(struct nvme_ctrl *arg)
1396 {
1397 struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(arg);
1398 struct nvme_rdma_queue *queue = &ctrl->queues[0];
1399 struct ib_device *dev = queue->device->dev;
1400 struct nvme_rdma_qe *sqe = &ctrl->async_event_sqe;
1401 struct nvme_command *cmd = sqe->data;
1402 struct ib_sge sge;
1403 int ret;
1404
1405 ib_dma_sync_single_for_cpu(dev, sqe->dma, sizeof(*cmd), DMA_TO_DEVICE);
1406
1407 memset(cmd, 0, sizeof(*cmd));
1408 cmd->common.opcode = nvme_admin_async_event;
1409 cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH;
1410 cmd->common.flags |= NVME_CMD_SGL_METABUF;
1411 nvme_rdma_set_sg_null(cmd);
1412
1413 sqe->cqe.done = nvme_rdma_async_done;
1414
1415 ib_dma_sync_single_for_device(dev, sqe->dma, sizeof(*cmd),
1416 DMA_TO_DEVICE);
1417
1418 ret = nvme_rdma_post_send(queue, sqe, &sge, 1, NULL);
1419 WARN_ON_ONCE(ret);
1420 }
1421
1422 static void nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue,
1423 struct nvme_completion *cqe, struct ib_wc *wc)
1424 {
1425 struct request *rq;
1426 struct nvme_rdma_request *req;
1427
1428 rq = blk_mq_tag_to_rq(nvme_rdma_tagset(queue), cqe->command_id);
1429 if (!rq) {
1430 dev_err(queue->ctrl->ctrl.device,
1431 "tag 0x%x on QP %#x not found\n",
1432 cqe->command_id, queue->qp->qp_num);
1433 nvme_rdma_error_recovery(queue->ctrl);
1434 return;
1435 }
1436 req = blk_mq_rq_to_pdu(rq);
1437
1438 req->status = cqe->status;
1439 req->result = cqe->result;
1440
1441 if (wc->wc_flags & IB_WC_WITH_INVALIDATE) {
1442 if (unlikely(wc->ex.invalidate_rkey != req->mr->rkey)) {
1443 dev_err(queue->ctrl->ctrl.device,
1444 "Bogus remote invalidation for rkey %#x\n",
1445 req->mr->rkey);
1446 nvme_rdma_error_recovery(queue->ctrl);
1447 }
1448 } else if (req->mr) {
1449 int ret;
1450
1451 ret = nvme_rdma_inv_rkey(queue, req);
1452 if (unlikely(ret < 0)) {
1453 dev_err(queue->ctrl->ctrl.device,
1454 "Queueing INV WR for rkey %#x failed (%d)\n",
1455 req->mr->rkey, ret);
1456 nvme_rdma_error_recovery(queue->ctrl);
1457 }
1458 /* the local invalidation completion will end the request */
1459 return;
1460 }
1461
1462 if (refcount_dec_and_test(&req->ref))
1463 nvme_end_request(rq, req->status, req->result);
1464 }
1465
1466 static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc)
1467 {
1468 struct nvme_rdma_qe *qe =
1469 container_of(wc->wr_cqe, struct nvme_rdma_qe, cqe);
1470 struct nvme_rdma_queue *queue = cq->cq_context;
1471 struct ib_device *ibdev = queue->device->dev;
1472 struct nvme_completion *cqe = qe->data;
1473 const size_t len = sizeof(struct nvme_completion);
1474
1475 if (unlikely(wc->status != IB_WC_SUCCESS)) {
1476 nvme_rdma_wr_error(cq, wc, "RECV");
1477 return;
1478 }
1479
1480 ib_dma_sync_single_for_cpu(ibdev, qe->dma, len, DMA_FROM_DEVICE);
1481 /*
1482 * AEN requests are special as they don't time out and can
1483 * survive any kind of queue freeze and often don't respond to
1484 * aborts. We don't even bother to allocate a struct request
1485 * for them but rather special case them here.
1486 */
1487 if (unlikely(nvme_rdma_queue_idx(queue) == 0 &&
1488 cqe->command_id >= NVME_AQ_BLK_MQ_DEPTH))
1489 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
1490 &cqe->result);
1491 else
1492 nvme_rdma_process_nvme_rsp(queue, cqe, wc);
1493 ib_dma_sync_single_for_device(ibdev, qe->dma, len, DMA_FROM_DEVICE);
1494
1495 nvme_rdma_post_recv(queue, qe);
1496 }
1497
1498 static int nvme_rdma_conn_established(struct nvme_rdma_queue *queue)
1499 {
1500 int ret, i;
1501
1502 for (i = 0; i < queue->queue_size; i++) {
1503 ret = nvme_rdma_post_recv(queue, &queue->rsp_ring[i]);
1504 if (ret)
1505 goto out_destroy_queue_ib;
1506 }
1507
1508 return 0;
1509
1510 out_destroy_queue_ib:
1511 nvme_rdma_destroy_queue_ib(queue);
1512 return ret;
1513 }
1514
1515 static int nvme_rdma_conn_rejected(struct nvme_rdma_queue *queue,
1516 struct rdma_cm_event *ev)
1517 {
1518 struct rdma_cm_id *cm_id = queue->cm_id;
1519 int status = ev->status;
1520 const char *rej_msg;
1521 const struct nvme_rdma_cm_rej *rej_data;
1522 u8 rej_data_len;
1523
1524 rej_msg = rdma_reject_msg(cm_id, status);
1525 rej_data = rdma_consumer_reject_data(cm_id, ev, &rej_data_len);
1526
1527 if (rej_data && rej_data_len >= sizeof(u16)) {
1528 u16 sts = le16_to_cpu(rej_data->sts);
1529
1530 dev_err(queue->ctrl->ctrl.device,
1531 "Connect rejected: status %d (%s) nvme status %d (%s).\n",
1532 status, rej_msg, sts, nvme_rdma_cm_msg(sts));
1533 } else {
1534 dev_err(queue->ctrl->ctrl.device,
1535 "Connect rejected: status %d (%s).\n", status, rej_msg);
1536 }
1537
1538 return -ECONNRESET;
1539 }
1540
1541 static int nvme_rdma_addr_resolved(struct nvme_rdma_queue *queue)
1542 {
1543 int ret;
1544
1545 ret = nvme_rdma_create_queue_ib(queue);
1546 if (ret)
1547 return ret;
1548
1549 ret = rdma_resolve_route(queue->cm_id, NVME_RDMA_CONNECT_TIMEOUT_MS);
1550 if (ret) {
1551 dev_err(queue->ctrl->ctrl.device,
1552 "rdma_resolve_route failed (%d).\n",
1553 queue->cm_error);
1554 goto out_destroy_queue;
1555 }
1556
1557 return 0;
1558
1559 out_destroy_queue:
1560 nvme_rdma_destroy_queue_ib(queue);
1561 return ret;
1562 }
1563
1564 static int nvme_rdma_route_resolved(struct nvme_rdma_queue *queue)
1565 {
1566 struct nvme_rdma_ctrl *ctrl = queue->ctrl;
1567 struct rdma_conn_param param = { };
1568 struct nvme_rdma_cm_req priv = { };
1569 int ret;
1570
1571 param.qp_num = queue->qp->qp_num;
1572 param.flow_control = 1;
1573
1574 param.responder_resources = queue->device->dev->attrs.max_qp_rd_atom;
1575 /* maximum retry count */
1576 param.retry_count = 7;
1577 param.rnr_retry_count = 7;
1578 param.private_data = &priv;
1579 param.private_data_len = sizeof(priv);
1580
1581 priv.recfmt = cpu_to_le16(NVME_RDMA_CM_FMT_1_0);
1582 priv.qid = cpu_to_le16(nvme_rdma_queue_idx(queue));
1583 /*
1584 * set the admin queue depth to the minimum size
1585 * specified by the Fabrics standard.
1586 */
1587 if (priv.qid == 0) {
1588 priv.hrqsize = cpu_to_le16(NVME_AQ_DEPTH);
1589 priv.hsqsize = cpu_to_le16(NVME_AQ_DEPTH - 1);
1590 } else {
1591 /*
1592 * current interpretation of the fabrics spec
1593 * is at minimum you make hrqsize sqsize+1, or a
1594 * 1's based representation of sqsize.
1595 */
1596 priv.hrqsize = cpu_to_le16(queue->queue_size);
1597 priv.hsqsize = cpu_to_le16(queue->ctrl->ctrl.sqsize);
1598 }
1599
1600 ret = rdma_connect(queue->cm_id, &param);
1601 if (ret) {
1602 dev_err(ctrl->ctrl.device,
1603 "rdma_connect failed (%d).\n", ret);
1604 goto out_destroy_queue_ib;
1605 }
1606
1607 return 0;
1608
1609 out_destroy_queue_ib:
1610 nvme_rdma_destroy_queue_ib(queue);
1611 return ret;
1612 }
1613
1614 static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
1615 struct rdma_cm_event *ev)
1616 {
1617 struct nvme_rdma_queue *queue = cm_id->context;
1618 int cm_error = 0;
1619
1620 dev_dbg(queue->ctrl->ctrl.device, "%s (%d): status %d id %p\n",
1621 rdma_event_msg(ev->event), ev->event,
1622 ev->status, cm_id);
1623
1624 switch (ev->event) {
1625 case RDMA_CM_EVENT_ADDR_RESOLVED:
1626 cm_error = nvme_rdma_addr_resolved(queue);
1627 break;
1628 case RDMA_CM_EVENT_ROUTE_RESOLVED:
1629 cm_error = nvme_rdma_route_resolved(queue);
1630 break;
1631 case RDMA_CM_EVENT_ESTABLISHED:
1632 queue->cm_error = nvme_rdma_conn_established(queue);
1633 /* complete cm_done regardless of success/failure */
1634 complete(&queue->cm_done);
1635 return 0;
1636 case RDMA_CM_EVENT_REJECTED:
1637 nvme_rdma_destroy_queue_ib(queue);
1638 cm_error = nvme_rdma_conn_rejected(queue, ev);
1639 break;
1640 case RDMA_CM_EVENT_ROUTE_ERROR:
1641 case RDMA_CM_EVENT_CONNECT_ERROR:
1642 case RDMA_CM_EVENT_UNREACHABLE:
1643 nvme_rdma_destroy_queue_ib(queue);
1644 /* fall through */
1645 case RDMA_CM_EVENT_ADDR_ERROR:
1646 dev_dbg(queue->ctrl->ctrl.device,
1647 "CM error event %d\n", ev->event);
1648 cm_error = -ECONNRESET;
1649 break;
1650 case RDMA_CM_EVENT_DISCONNECTED:
1651 case RDMA_CM_EVENT_ADDR_CHANGE:
1652 case RDMA_CM_EVENT_TIMEWAIT_EXIT:
1653 dev_dbg(queue->ctrl->ctrl.device,
1654 "disconnect received - connection closed\n");
1655 nvme_rdma_error_recovery(queue->ctrl);
1656 break;
1657 case RDMA_CM_EVENT_DEVICE_REMOVAL:
1658 /* device removal is handled via the ib_client API */
1659 break;
1660 default:
1661 dev_err(queue->ctrl->ctrl.device,
1662 "Unexpected RDMA CM event (%d)\n", ev->event);
1663 nvme_rdma_error_recovery(queue->ctrl);
1664 break;
1665 }
1666
1667 if (cm_error) {
1668 queue->cm_error = cm_error;
1669 complete(&queue->cm_done);
1670 }
1671
1672 return 0;
1673 }
1674
1675 static enum blk_eh_timer_return
1676 nvme_rdma_timeout(struct request *rq, bool reserved)
1677 {
1678 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1679 struct nvme_rdma_queue *queue = req->queue;
1680 struct nvme_rdma_ctrl *ctrl = queue->ctrl;
1681
1682 dev_warn(ctrl->ctrl.device, "I/O %d QID %d timeout\n",
1683 rq->tag, nvme_rdma_queue_idx(queue));
1684
1685 if (ctrl->ctrl.state != NVME_CTRL_LIVE) {
1686 /*
1687 * Teardown immediately if controller times out while starting
1688 * or we are already started error recovery. all outstanding
1689 * requests are completed on shutdown, so we return BLK_EH_DONE.
1690 */
1691 flush_work(&ctrl->err_work);
1692 nvme_rdma_teardown_io_queues(ctrl, false);
1693 nvme_rdma_teardown_admin_queue(ctrl, false);
1694 return BLK_EH_DONE;
1695 }
1696
1697 dev_warn(ctrl->ctrl.device, "starting error recovery\n");
1698 nvme_rdma_error_recovery(ctrl);
1699
1700 return BLK_EH_RESET_TIMER;
1701 }
1702
1703 static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx,
1704 const struct blk_mq_queue_data *bd)
1705 {
1706 struct nvme_ns *ns = hctx->queue->queuedata;
1707 struct nvme_rdma_queue *queue = hctx->driver_data;
1708 struct request *rq = bd->rq;
1709 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1710 struct nvme_rdma_qe *sqe = &req->sqe;
1711 struct nvme_command *c = sqe->data;
1712 struct ib_device *dev;
1713 bool queue_ready = test_bit(NVME_RDMA_Q_LIVE, &queue->flags);
1714 blk_status_t ret;
1715 int err;
1716
1717 WARN_ON_ONCE(rq->tag < 0);
1718
1719 if (!nvmf_check_ready(&queue->ctrl->ctrl, rq, queue_ready))
1720 return nvmf_fail_nonready_command(&queue->ctrl->ctrl, rq);
1721
1722 dev = queue->device->dev;
1723
1724 req->sqe.dma = ib_dma_map_single(dev, req->sqe.data,
1725 sizeof(struct nvme_command),
1726 DMA_TO_DEVICE);
1727 err = ib_dma_mapping_error(dev, req->sqe.dma);
1728 if (unlikely(err))
1729 return BLK_STS_RESOURCE;
1730
1731 ib_dma_sync_single_for_cpu(dev, sqe->dma,
1732 sizeof(struct nvme_command), DMA_TO_DEVICE);
1733
1734 ret = nvme_setup_cmd(ns, rq, c);
1735 if (ret)
1736 goto unmap_qe;
1737
1738 blk_mq_start_request(rq);
1739
1740 err = nvme_rdma_map_data(queue, rq, c);
1741 if (unlikely(err < 0)) {
1742 dev_err(queue->ctrl->ctrl.device,
1743 "Failed to map data (%d)\n", err);
1744 nvme_cleanup_cmd(rq);
1745 goto err;
1746 }
1747
1748 sqe->cqe.done = nvme_rdma_send_done;
1749
1750 ib_dma_sync_single_for_device(dev, sqe->dma,
1751 sizeof(struct nvme_command), DMA_TO_DEVICE);
1752
1753 err = nvme_rdma_post_send(queue, sqe, req->sge, req->num_sge,
1754 req->mr ? &req->reg_wr.wr : NULL);
1755 if (unlikely(err)) {
1756 nvme_rdma_unmap_data(queue, rq);
1757 goto err;
1758 }
1759
1760 return BLK_STS_OK;
1761
1762 err:
1763 if (err == -ENOMEM || err == -EAGAIN)
1764 ret = BLK_STS_RESOURCE;
1765 else
1766 ret = BLK_STS_IOERR;
1767 unmap_qe:
1768 ib_dma_unmap_single(dev, req->sqe.dma, sizeof(struct nvme_command),
1769 DMA_TO_DEVICE);
1770 return ret;
1771 }
1772
1773 static int nvme_rdma_poll(struct blk_mq_hw_ctx *hctx)
1774 {
1775 struct nvme_rdma_queue *queue = hctx->driver_data;
1776
1777 return ib_process_cq_direct(queue->ib_cq, -1);
1778 }
1779
1780 static void nvme_rdma_complete_rq(struct request *rq)
1781 {
1782 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1783 struct nvme_rdma_queue *queue = req->queue;
1784 struct ib_device *ibdev = queue->device->dev;
1785
1786 nvme_rdma_unmap_data(queue, rq);
1787 ib_dma_unmap_single(ibdev, req->sqe.dma, sizeof(struct nvme_command),
1788 DMA_TO_DEVICE);
1789 nvme_complete_rq(rq);
1790 }
1791
1792 static int nvme_rdma_map_queues(struct blk_mq_tag_set *set)
1793 {
1794 struct nvme_rdma_ctrl *ctrl = set->driver_data;
1795 struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
1796
1797 if (opts->nr_write_queues && ctrl->io_queues[HCTX_TYPE_READ]) {
1798 /* separate read/write queues */
1799 set->map[HCTX_TYPE_DEFAULT].nr_queues =
1800 ctrl->io_queues[HCTX_TYPE_DEFAULT];
1801 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
1802 set->map[HCTX_TYPE_READ].nr_queues =
1803 ctrl->io_queues[HCTX_TYPE_READ];
1804 set->map[HCTX_TYPE_READ].queue_offset =
1805 ctrl->io_queues[HCTX_TYPE_DEFAULT];
1806 } else {
1807 /* shared read/write queues */
1808 set->map[HCTX_TYPE_DEFAULT].nr_queues =
1809 ctrl->io_queues[HCTX_TYPE_DEFAULT];
1810 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
1811 set->map[HCTX_TYPE_READ].nr_queues =
1812 ctrl->io_queues[HCTX_TYPE_DEFAULT];
1813 set->map[HCTX_TYPE_READ].queue_offset = 0;
1814 }
1815 blk_mq_rdma_map_queues(&set->map[HCTX_TYPE_DEFAULT],
1816 ctrl->device->dev, 0);
1817 blk_mq_rdma_map_queues(&set->map[HCTX_TYPE_READ],
1818 ctrl->device->dev, 0);
1819
1820 if (opts->nr_poll_queues && ctrl->io_queues[HCTX_TYPE_POLL]) {
1821 /* map dedicated poll queues only if we have queues left */
1822 set->map[HCTX_TYPE_POLL].nr_queues =
1823 ctrl->io_queues[HCTX_TYPE_POLL];
1824 set->map[HCTX_TYPE_POLL].queue_offset =
1825 ctrl->io_queues[HCTX_TYPE_DEFAULT] +
1826 ctrl->io_queues[HCTX_TYPE_READ];
1827 blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]);
1828 }
1829
1830 dev_info(ctrl->ctrl.device,
1831 "mapped %d/%d/%d default/read/poll queues.\n",
1832 ctrl->io_queues[HCTX_TYPE_DEFAULT],
1833 ctrl->io_queues[HCTX_TYPE_READ],
1834 ctrl->io_queues[HCTX_TYPE_POLL]);
1835
1836 return 0;
1837 }
1838
1839 static const struct blk_mq_ops nvme_rdma_mq_ops = {
1840 .queue_rq = nvme_rdma_queue_rq,
1841 .complete = nvme_rdma_complete_rq,
1842 .init_request = nvme_rdma_init_request,
1843 .exit_request = nvme_rdma_exit_request,
1844 .init_hctx = nvme_rdma_init_hctx,
1845 .timeout = nvme_rdma_timeout,
1846 .map_queues = nvme_rdma_map_queues,
1847 .poll = nvme_rdma_poll,
1848 };
1849
1850 static const struct blk_mq_ops nvme_rdma_admin_mq_ops = {
1851 .queue_rq = nvme_rdma_queue_rq,
1852 .complete = nvme_rdma_complete_rq,
1853 .init_request = nvme_rdma_init_request,
1854 .exit_request = nvme_rdma_exit_request,
1855 .init_hctx = nvme_rdma_init_admin_hctx,
1856 .timeout = nvme_rdma_timeout,
1857 };
1858
1859 static void nvme_rdma_shutdown_ctrl(struct nvme_rdma_ctrl *ctrl, bool shutdown)
1860 {
1861 cancel_work_sync(&ctrl->err_work);
1862 cancel_delayed_work_sync(&ctrl->reconnect_work);
1863
1864 nvme_rdma_teardown_io_queues(ctrl, shutdown);
1865 if (shutdown)
1866 nvme_shutdown_ctrl(&ctrl->ctrl);
1867 else
1868 nvme_disable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap);
1869 nvme_rdma_teardown_admin_queue(ctrl, shutdown);
1870 }
1871
1872 static void nvme_rdma_delete_ctrl(struct nvme_ctrl *ctrl)
1873 {
1874 nvme_rdma_shutdown_ctrl(to_rdma_ctrl(ctrl), true);
1875 }
1876
1877 static void nvme_rdma_reset_ctrl_work(struct work_struct *work)
1878 {
1879 struct nvme_rdma_ctrl *ctrl =
1880 container_of(work, struct nvme_rdma_ctrl, ctrl.reset_work);
1881
1882 nvme_stop_ctrl(&ctrl->ctrl);
1883 nvme_rdma_shutdown_ctrl(ctrl, false);
1884
1885 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
1886 /* state change failure should never happen */
1887 WARN_ON_ONCE(1);
1888 return;
1889 }
1890
1891 if (nvme_rdma_setup_ctrl(ctrl, false))
1892 goto out_fail;
1893
1894 return;
1895
1896 out_fail:
1897 ++ctrl->ctrl.nr_reconnects;
1898 nvme_rdma_reconnect_or_remove(ctrl);
1899 }
1900
1901 static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = {
1902 .name = "rdma",
1903 .module = THIS_MODULE,
1904 .flags = NVME_F_FABRICS,
1905 .reg_read32 = nvmf_reg_read32,
1906 .reg_read64 = nvmf_reg_read64,
1907 .reg_write32 = nvmf_reg_write32,
1908 .free_ctrl = nvme_rdma_free_ctrl,
1909 .submit_async_event = nvme_rdma_submit_async_event,
1910 .delete_ctrl = nvme_rdma_delete_ctrl,
1911 .get_address = nvmf_get_address,
1912 };
1913
1914 /*
1915 * Fails a connection request if it matches an existing controller
1916 * (association) with the same tuple:
1917 * <Host NQN, Host ID, local address, remote address, remote port, SUBSYS NQN>
1918 *
1919 * if local address is not specified in the request, it will match an
1920 * existing controller with all the other parameters the same and no
1921 * local port address specified as well.
1922 *
1923 * The ports don't need to be compared as they are intrinsically
1924 * already matched by the port pointers supplied.
1925 */
1926 static bool
1927 nvme_rdma_existing_controller(struct nvmf_ctrl_options *opts)
1928 {
1929 struct nvme_rdma_ctrl *ctrl;
1930 bool found = false;
1931
1932 mutex_lock(&nvme_rdma_ctrl_mutex);
1933 list_for_each_entry(ctrl, &nvme_rdma_ctrl_list, list) {
1934 found = nvmf_ip_options_match(&ctrl->ctrl, opts);
1935 if (found)
1936 break;
1937 }
1938 mutex_unlock(&nvme_rdma_ctrl_mutex);
1939
1940 return found;
1941 }
1942
1943 static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,
1944 struct nvmf_ctrl_options *opts)
1945 {
1946 struct nvme_rdma_ctrl *ctrl;
1947 int ret;
1948 bool changed;
1949
1950 ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
1951 if (!ctrl)
1952 return ERR_PTR(-ENOMEM);
1953 ctrl->ctrl.opts = opts;
1954 INIT_LIST_HEAD(&ctrl->list);
1955
1956 if (!(opts->mask & NVMF_OPT_TRSVCID)) {
1957 opts->trsvcid =
1958 kstrdup(__stringify(NVME_RDMA_IP_PORT), GFP_KERNEL);
1959 if (!opts->trsvcid) {
1960 ret = -ENOMEM;
1961 goto out_free_ctrl;
1962 }
1963 opts->mask |= NVMF_OPT_TRSVCID;
1964 }
1965
1966 ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
1967 opts->traddr, opts->trsvcid, &ctrl->addr);
1968 if (ret) {
1969 pr_err("malformed address passed: %s:%s\n",
1970 opts->traddr, opts->trsvcid);
1971 goto out_free_ctrl;
1972 }
1973
1974 if (opts->mask & NVMF_OPT_HOST_TRADDR) {
1975 ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
1976 opts->host_traddr, NULL, &ctrl->src_addr);
1977 if (ret) {
1978 pr_err("malformed src address passed: %s\n",
1979 opts->host_traddr);
1980 goto out_free_ctrl;
1981 }
1982 }
1983
1984 if (!opts->duplicate_connect && nvme_rdma_existing_controller(opts)) {
1985 ret = -EALREADY;
1986 goto out_free_ctrl;
1987 }
1988
1989 INIT_DELAYED_WORK(&ctrl->reconnect_work,
1990 nvme_rdma_reconnect_ctrl_work);
1991 INIT_WORK(&ctrl->err_work, nvme_rdma_error_recovery_work);
1992 INIT_WORK(&ctrl->ctrl.reset_work, nvme_rdma_reset_ctrl_work);
1993
1994 ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues +
1995 opts->nr_poll_queues + 1;
1996 ctrl->ctrl.sqsize = opts->queue_size - 1;
1997 ctrl->ctrl.kato = opts->kato;
1998
1999 ret = -ENOMEM;
2000 ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues),
2001 GFP_KERNEL);
2002 if (!ctrl->queues)
2003 goto out_free_ctrl;
2004
2005 ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_rdma_ctrl_ops,
2006 0 /* no quirks, we're perfect! */);
2007 if (ret)
2008 goto out_kfree_queues;
2009
2010 changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING);
2011 WARN_ON_ONCE(!changed);
2012
2013 ret = nvme_rdma_setup_ctrl(ctrl, true);
2014 if (ret)
2015 goto out_uninit_ctrl;
2016
2017 dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISpcs\n",
2018 ctrl->ctrl.opts->subsysnqn, &ctrl->addr);
2019
2020 nvme_get_ctrl(&ctrl->ctrl);
2021
2022 mutex_lock(&nvme_rdma_ctrl_mutex);
2023 list_add_tail(&ctrl->list, &nvme_rdma_ctrl_list);
2024 mutex_unlock(&nvme_rdma_ctrl_mutex);
2025
2026 return &ctrl->ctrl;
2027
2028 out_uninit_ctrl:
2029 nvme_uninit_ctrl(&ctrl->ctrl);
2030 nvme_put_ctrl(&ctrl->ctrl);
2031 if (ret > 0)
2032 ret = -EIO;
2033 return ERR_PTR(ret);
2034 out_kfree_queues:
2035 kfree(ctrl->queues);
2036 out_free_ctrl:
2037 kfree(ctrl);
2038 return ERR_PTR(ret);
2039 }
2040
2041 static struct nvmf_transport_ops nvme_rdma_transport = {
2042 .name = "rdma",
2043 .module = THIS_MODULE,
2044 .required_opts = NVMF_OPT_TRADDR,
2045 .allowed_opts = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY |
2046 NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO |
2047 NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES,
2048 .create_ctrl = nvme_rdma_create_ctrl,
2049 };
2050
2051 static void nvme_rdma_remove_one(struct ib_device *ib_device, void *client_data)
2052 {
2053 struct nvme_rdma_ctrl *ctrl;
2054 struct nvme_rdma_device *ndev;
2055 bool found = false;
2056
2057 mutex_lock(&device_list_mutex);
2058 list_for_each_entry(ndev, &device_list, entry) {
2059 if (ndev->dev == ib_device) {
2060 found = true;
2061 break;
2062 }
2063 }
2064 mutex_unlock(&device_list_mutex);
2065
2066 if (!found)
2067 return;
2068
2069 /* Delete all controllers using this device */
2070 mutex_lock(&nvme_rdma_ctrl_mutex);
2071 list_for_each_entry(ctrl, &nvme_rdma_ctrl_list, list) {
2072 if (ctrl->device->dev != ib_device)
2073 continue;
2074 nvme_delete_ctrl(&ctrl->ctrl);
2075 }
2076 mutex_unlock(&nvme_rdma_ctrl_mutex);
2077
2078 flush_workqueue(nvme_delete_wq);
2079 }
2080
2081 static struct ib_client nvme_rdma_ib_client = {
2082 .name = "nvme_rdma",
2083 .remove = nvme_rdma_remove_one
2084 };
2085
2086 static int __init nvme_rdma_init_module(void)
2087 {
2088 int ret;
2089
2090 ret = ib_register_client(&nvme_rdma_ib_client);
2091 if (ret)
2092 return ret;
2093
2094 ret = nvmf_register_transport(&nvme_rdma_transport);
2095 if (ret)
2096 goto err_unreg_client;
2097
2098 return 0;
2099
2100 err_unreg_client:
2101 ib_unregister_client(&nvme_rdma_ib_client);
2102 return ret;
2103 }
2104
2105 static void __exit nvme_rdma_cleanup_module(void)
2106 {
2107 nvmf_unregister_transport(&nvme_rdma_transport);
2108 ib_unregister_client(&nvme_rdma_ib_client);
2109 }
2110
2111 module_init(nvme_rdma_init_module);
2112 module_exit(nvme_rdma_cleanup_module);
2113
2114 MODULE_LICENSE("GPL v2");
Ubuntu Jammy Linux kernel mirror