]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/nvme/host/rdma.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
nvmet: Fix controller serial number inconsistency
[mirror_ubuntu-artful-kernel.git] / drivers / nvme / host / rdma.c
CommitLineData
71102307
CH		 1/*
2 * NVMe over Fabrics RDMA host code.
3 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 */
14#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
15#include <linux/delay.h>
16#include <linux/module.h>
17#include <linux/init.h>
18#include <linux/slab.h>
19#include <linux/err.h>
20#include <linux/string.h>
21#include <linux/jiffies.h>
22#include <linux/atomic.h>
23#include <linux/blk-mq.h>
24#include <linux/types.h>
25#include <linux/list.h>
26#include <linux/mutex.h>
27#include <linux/scatterlist.h>
28#include <linux/nvme.h>
29#include <linux/t10-pi.h>
30#include <asm/unaligned.h>
31
32#include <rdma/ib_verbs.h>
33#include <rdma/rdma_cm.h>
34#include <rdma/ib_cm.h>
35#include <linux/nvme-rdma.h>
36
37#include "nvme.h"
38#include "fabrics.h"
39
40
41#define NVME_RDMA_CONNECT_TIMEOUT_MS 1000 /* 1 second */
42
43#define NVME_RDMA_MAX_SEGMENT_SIZE 0xffffff /* 24-bit SGL field */
44
45#define NVME_RDMA_MAX_SEGMENTS 256
46
47#define NVME_RDMA_MAX_INLINE_SEGMENTS 1
48
49#define NVME_RDMA_MAX_PAGES_PER_MR 512
50
51#define NVME_RDMA_DEF_RECONNECT_DELAY 20
52
53/*
54 * We handle AEN commands ourselves and don't even let the
55 * block layer know about them.
56 */
57#define NVME_RDMA_NR_AEN_COMMANDS 1
58#define NVME_RDMA_AQ_BLKMQ_DEPTH \
59 (NVMF_AQ_DEPTH - NVME_RDMA_NR_AEN_COMMANDS)
60
61struct nvme_rdma_device {
62 struct ib_device *dev;
63 struct ib_pd *pd;
64 struct ib_mr *mr;
65 struct kref ref;
66 struct list_head entry;
67};
68
69struct nvme_rdma_qe {
70 struct ib_cqe cqe;
71 void *data;
72 u64 dma;
73};
74
75struct nvme_rdma_queue;
76struct nvme_rdma_request {
77 struct ib_mr *mr;
78 struct nvme_rdma_qe sqe;
79 struct ib_sge sge[1 + NVME_RDMA_MAX_INLINE_SEGMENTS];
80 u32 num_sge;
81 int nents;
82 bool inline_data;
83 bool need_inval;
84 struct ib_reg_wr reg_wr;
85 struct ib_cqe reg_cqe;
86 struct nvme_rdma_queue *queue;
87 struct sg_table sg_table;
88 struct scatterlist first_sgl[];
89};
90
91enum nvme_rdma_queue_flags {
92 NVME_RDMA_Q_CONNECTED = (1 << 0),
93};
94
95struct nvme_rdma_queue {
96 struct nvme_rdma_qe *rsp_ring;
97 u8 sig_count;
98 int queue_size;
99 size_t cmnd_capsule_len;
100 struct nvme_rdma_ctrl *ctrl;
101 struct nvme_rdma_device *device;
102 struct ib_cq *ib_cq;
103 struct ib_qp *qp;
104
105 unsigned long flags;
106 struct rdma_cm_id *cm_id;
107 int cm_error;
108 struct completion cm_done;
109};
110
111struct nvme_rdma_ctrl {
112 /* read and written in the hot path */
113 spinlock_t lock;
114
115 /* read only in the hot path */
116 struct nvme_rdma_queue *queues;
117 u32 queue_count;
118
119 /* other member variables */
71102307
CH		 120 struct blk_mq_tag_set tag_set;
121 struct work_struct delete_work;
122 struct work_struct reset_work;
123 struct work_struct err_work;
124
125 struct nvme_rdma_qe async_event_sqe;
126
127 int reconnect_delay;
128 struct delayed_work reconnect_work;
129
130 struct list_head list;
131
132 struct blk_mq_tag_set admin_tag_set;
133 struct nvme_rdma_device *device;
134
135 u64 cap;
136 u32 max_fr_pages;
137
138 union {
139 struct sockaddr addr;
140 struct sockaddr_in addr_in;
141 };
142
143 struct nvme_ctrl ctrl;
144};
145
146static inline struct nvme_rdma_ctrl *to_rdma_ctrl(struct nvme_ctrl *ctrl)
147{
148 return container_of(ctrl, struct nvme_rdma_ctrl, ctrl);
149}
150
151static LIST_HEAD(device_list);
152static DEFINE_MUTEX(device_list_mutex);
153
154static LIST_HEAD(nvme_rdma_ctrl_list);
155static DEFINE_MUTEX(nvme_rdma_ctrl_mutex);
156
157static struct workqueue_struct *nvme_rdma_wq;
158
159/*
160 * Disabling this option makes small I/O goes faster, but is fundamentally
161 * unsafe. With it turned off we will have to register a global rkey that
162 * allows read and write access to all physical memory.
163 */
164static bool register_always = true;
165module_param(register_always, bool, 0444);
166MODULE_PARM_DESC(register_always,
167 "Use memory registration even for contiguous memory regions");
168
169static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
170 struct rdma_cm_event *event);
171static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc);
71102307
CH		 172
173/* XXX: really should move to a generic header sooner or later.. */
174static inline void put_unaligned_le24(u32 val, u8 *p)
175{
176 *p++ = val;
177 *p++ = val >> 8;
178 *p++ = val >> 16;
179}
180
181static inline int nvme_rdma_queue_idx(struct nvme_rdma_queue *queue)
182{
183 return queue - queue->ctrl->queues;
184}
185
186static inline size_t nvme_rdma_inline_data_size(struct nvme_rdma_queue *queue)
187{
188 return queue->cmnd_capsule_len - sizeof(struct nvme_command);
189}
190
191static void nvme_rdma_free_qe(struct ib_device *ibdev, struct nvme_rdma_qe *qe,
192 size_t capsule_size, enum dma_data_direction dir)
193{
194 ib_dma_unmap_single(ibdev, qe->dma, capsule_size, dir);
195 kfree(qe->data);
196}
197
198static int nvme_rdma_alloc_qe(struct ib_device *ibdev, struct nvme_rdma_qe *qe,
199 size_t capsule_size, enum dma_data_direction dir)
200{
201 qe->data = kzalloc(capsule_size, GFP_KERNEL);
202 if (!qe->data)
203 return -ENOMEM;
204
205 qe->dma = ib_dma_map_single(ibdev, qe->data, capsule_size, dir);
206 if (ib_dma_mapping_error(ibdev, qe->dma)) {
207 kfree(qe->data);
208 return -ENOMEM;
209 }
210
211 return 0;
212}
213
214static void nvme_rdma_free_ring(struct ib_device *ibdev,
215 struct nvme_rdma_qe *ring, size_t ib_queue_size,
216 size_t capsule_size, enum dma_data_direction dir)
217{
218 int i;
219
220 for (i = 0; i < ib_queue_size; i++)
221 nvme_rdma_free_qe(ibdev, &ring[i], capsule_size, dir);
222 kfree(ring);
223}
224
225static struct nvme_rdma_qe *nvme_rdma_alloc_ring(struct ib_device *ibdev,
226 size_t ib_queue_size, size_t capsule_size,
227 enum dma_data_direction dir)
228{
229 struct nvme_rdma_qe *ring;
230 int i;
231
232 ring = kcalloc(ib_queue_size, sizeof(struct nvme_rdma_qe), GFP_KERNEL);
233 if (!ring)
234 return NULL;
235
236 for (i = 0; i < ib_queue_size; i++) {
237 if (nvme_rdma_alloc_qe(ibdev, &ring[i], capsule_size, dir))
238 goto out_free_ring;
239 }
240
241 return ring;
242
243out_free_ring:
244 nvme_rdma_free_ring(ibdev, ring, i, capsule_size, dir);
245 return NULL;
246}
247
248static void nvme_rdma_qp_event(struct ib_event *event, void *context)
249{
250 pr_debug("QP event %d\n", event->event);
251}
252
253static int nvme_rdma_wait_for_cm(struct nvme_rdma_queue *queue)
254{
255 wait_for_completion_interruptible_timeout(&queue->cm_done,
256 msecs_to_jiffies(NVME_RDMA_CONNECT_TIMEOUT_MS) + 1);
257 return queue->cm_error;
258}
259
260static int nvme_rdma_create_qp(struct nvme_rdma_queue *queue, const int factor)
261{
262 struct nvme_rdma_device *dev = queue->device;
263 struct ib_qp_init_attr init_attr;
264 int ret;
265
266 memset(&init_attr, 0, sizeof(init_attr));
267 init_attr.event_handler = nvme_rdma_qp_event;
268 /* +1 for drain */
269 init_attr.cap.max_send_wr = factor * queue->queue_size + 1;
270 /* +1 for drain */
271 init_attr.cap.max_recv_wr = queue->queue_size + 1;
272 init_attr.cap.max_recv_sge = 1;
273 init_attr.cap.max_send_sge = 1 + NVME_RDMA_MAX_INLINE_SEGMENTS;
274 init_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
275 init_attr.qp_type = IB_QPT_RC;
276 init_attr.send_cq = queue->ib_cq;
277 init_attr.recv_cq = queue->ib_cq;
278
279 ret = rdma_create_qp(queue->cm_id, dev->pd, &init_attr);
280
281 queue->qp = queue->cm_id->qp;
282 return ret;
283}
284
285static int nvme_rdma_reinit_request(void *data, struct request *rq)
286{
287 struct nvme_rdma_ctrl *ctrl = data;
288 struct nvme_rdma_device *dev = ctrl->device;
289 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
290 int ret = 0;
291
292 if (!req->need_inval)
293 goto out;
294
295 ib_dereg_mr(req->mr);
296
297 req->mr = ib_alloc_mr(dev->pd, IB_MR_TYPE_MEM_REG,
298 ctrl->max_fr_pages);
299 if (IS_ERR(req->mr)) {
71102307 300 ret = PTR_ERR(req->mr);
458a9632 301 req->mr = NULL;
71102307
CH		 302 }
303
304 req->need_inval = false;
305
306out:
307 return ret;
308}
309
310static void __nvme_rdma_exit_request(struct nvme_rdma_ctrl *ctrl,
311 struct request *rq, unsigned int queue_idx)
312{
313 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
314 struct nvme_rdma_queue *queue = &ctrl->queues[queue_idx];
315 struct nvme_rdma_device *dev = queue->device;
316
317 if (req->mr)
318 ib_dereg_mr(req->mr);
319
320 nvme_rdma_free_qe(dev->dev, &req->sqe, sizeof(struct nvme_command),
321 DMA_TO_DEVICE);
322}
323
324static void nvme_rdma_exit_request(void *data, struct request *rq,
325 unsigned int hctx_idx, unsigned int rq_idx)
326{
327 return __nvme_rdma_exit_request(data, rq, hctx_idx + 1);
328}
329
330static void nvme_rdma_exit_admin_request(void *data, struct request *rq,
331 unsigned int hctx_idx, unsigned int rq_idx)
332{
333 return __nvme_rdma_exit_request(data, rq, 0);
334}
335
336static int __nvme_rdma_init_request(struct nvme_rdma_ctrl *ctrl,
337 struct request *rq, unsigned int queue_idx)
338{
339 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
340 struct nvme_rdma_queue *queue = &ctrl->queues[queue_idx];
341 struct nvme_rdma_device *dev = queue->device;
342 struct ib_device *ibdev = dev->dev;
343 int ret;
344
345 BUG_ON(queue_idx >= ctrl->queue_count);
346
347 ret = nvme_rdma_alloc_qe(ibdev, &req->sqe, sizeof(struct nvme_command),
348 DMA_TO_DEVICE);
349 if (ret)
350 return ret;
351
352 req->mr = ib_alloc_mr(dev->pd, IB_MR_TYPE_MEM_REG,
353 ctrl->max_fr_pages);
354 if (IS_ERR(req->mr)) {
355 ret = PTR_ERR(req->mr);
356 goto out_free_qe;
357 }
358
359 req->queue = queue;
360
361 return 0;
362
363out_free_qe:
364 nvme_rdma_free_qe(dev->dev, &req->sqe, sizeof(struct nvme_command),
365 DMA_TO_DEVICE);
366 return -ENOMEM;
367}
368
369static int nvme_rdma_init_request(void *data, struct request *rq,
370 unsigned int hctx_idx, unsigned int rq_idx,
371 unsigned int numa_node)
372{
373 return __nvme_rdma_init_request(data, rq, hctx_idx + 1);
374}
375
376static int nvme_rdma_init_admin_request(void *data, struct request *rq,
377 unsigned int hctx_idx, unsigned int rq_idx,
378 unsigned int numa_node)
379{
380 return __nvme_rdma_init_request(data, rq, 0);
381}
382
383static int nvme_rdma_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
384 unsigned int hctx_idx)
385{
386 struct nvme_rdma_ctrl *ctrl = data;
387 struct nvme_rdma_queue *queue = &ctrl->queues[hctx_idx + 1];
388
389 BUG_ON(hctx_idx >= ctrl->queue_count);
390
391 hctx->driver_data = queue;
392 return 0;
393}
394
395static int nvme_rdma_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
396 unsigned int hctx_idx)
397{
398 struct nvme_rdma_ctrl *ctrl = data;
399 struct nvme_rdma_queue *queue = &ctrl->queues[0];
400
401 BUG_ON(hctx_idx != 0);
402
403 hctx->driver_data = queue;
404 return 0;
405}
406
407static void nvme_rdma_free_dev(struct kref *ref)
408{
409 struct nvme_rdma_device *ndev =
410 container_of(ref, struct nvme_rdma_device, ref);
411
412 mutex_lock(&device_list_mutex);
413 list_del(&ndev->entry);
414 mutex_unlock(&device_list_mutex);
415
416 if (!register_always)
417 ib_dereg_mr(ndev->mr);
418 ib_dealloc_pd(ndev->pd);
419
420 kfree(ndev);
421}
422
423static void nvme_rdma_dev_put(struct nvme_rdma_device *dev)
424{
425 kref_put(&dev->ref, nvme_rdma_free_dev);
426}
427
428static int nvme_rdma_dev_get(struct nvme_rdma_device *dev)
429{
430 return kref_get_unless_zero(&dev->ref);
431}
432
433static struct nvme_rdma_device *
434nvme_rdma_find_get_device(struct rdma_cm_id *cm_id)
435{
436 struct nvme_rdma_device *ndev;
437
438 mutex_lock(&device_list_mutex);
439 list_for_each_entry(ndev, &device_list, entry) {
440 if (ndev->dev->node_guid == cm_id->device->node_guid &&
441 nvme_rdma_dev_get(ndev))
442 goto out_unlock;
443 }
444
445 ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
446 if (!ndev)
447 goto out_err;
448
449 ndev->dev = cm_id->device;
450 kref_init(&ndev->ref);
451
452 ndev->pd = ib_alloc_pd(ndev->dev);
453 if (IS_ERR(ndev->pd))
454 goto out_free_dev;
455
456 if (!register_always) {
457 ndev->mr = ib_get_dma_mr(ndev->pd,
458 IB_ACCESS_LOCAL_WRITE |
459 IB_ACCESS_REMOTE_READ |
460 IB_ACCESS_REMOTE_WRITE);
461 if (IS_ERR(ndev->mr))
462 goto out_free_pd;
463 }
464
465 if (!(ndev->dev->attrs.device_cap_flags &
466 IB_DEVICE_MEM_MGT_EXTENSIONS)) {
467 dev_err(&ndev->dev->dev,
468 "Memory registrations not supported.\n");
469 goto out_free_mr;
470 }
471
472 list_add(&ndev->entry, &device_list);
473out_unlock:
474 mutex_unlock(&device_list_mutex);
475 return ndev;
476
477out_free_mr:
478 if (!register_always)
479 ib_dereg_mr(ndev->mr);
480out_free_pd:
481 ib_dealloc_pd(ndev->pd);
482out_free_dev:
483 kfree(ndev);
484out_err:
485 mutex_unlock(&device_list_mutex);
486 return NULL;
487}
488
489static void nvme_rdma_destroy_queue_ib(struct nvme_rdma_queue *queue)
490{
491 struct nvme_rdma_device *dev = queue->device;
492 struct ib_device *ibdev = dev->dev;
493
494 rdma_destroy_qp(queue->cm_id);
495 ib_free_cq(queue->ib_cq);
496
497 nvme_rdma_free_ring(ibdev, queue->rsp_ring, queue->queue_size,
498 sizeof(struct nvme_completion), DMA_FROM_DEVICE);
499
500 nvme_rdma_dev_put(dev);
501}
502
503static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue *queue,
504 struct nvme_rdma_device *dev)
505{
506 struct ib_device *ibdev = dev->dev;
507 const int send_wr_factor = 3; /* MR, SEND, INV */
508 const int cq_factor = send_wr_factor + 1; /* + RECV */
509 int comp_vector, idx = nvme_rdma_queue_idx(queue);
510
511 int ret;
512
513 queue->device = dev;
514
515 /*
516 * The admin queue is barely used once the controller is live, so don't
517 * bother to spread it out.
518 */
519 if (idx == 0)
520 comp_vector = 0;
521 else
522 comp_vector = idx % ibdev->num_comp_vectors;
523
524
525 /* +1 for ib_stop_cq */
526 queue->ib_cq = ib_alloc_cq(dev->dev, queue,
527 cq_factor * queue->queue_size + 1, comp_vector,
528 IB_POLL_SOFTIRQ);
529 if (IS_ERR(queue->ib_cq)) {
530 ret = PTR_ERR(queue->ib_cq);
531 goto out;
532 }
533
534 ret = nvme_rdma_create_qp(queue, send_wr_factor);
535 if (ret)
536 goto out_destroy_ib_cq;
537
538 queue->rsp_ring = nvme_rdma_alloc_ring(ibdev, queue->queue_size,
539 sizeof(struct nvme_completion), DMA_FROM_DEVICE);
540 if (!queue->rsp_ring) {
541 ret = -ENOMEM;
542 goto out_destroy_qp;
543 }
544
545 return 0;
546
547out_destroy_qp:
548 ib_destroy_qp(queue->qp);
549out_destroy_ib_cq:
550 ib_free_cq(queue->ib_cq);
551out:
552 return ret;
553}
554
555static int nvme_rdma_init_queue(struct nvme_rdma_ctrl *ctrl,
556 int idx, size_t queue_size)
557{
558 struct nvme_rdma_queue *queue;
559 int ret;
560
561 queue = &ctrl->queues[idx];
562 queue->ctrl = ctrl;
563 init_completion(&queue->cm_done);
564
565 if (idx > 0)
566 queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;
567 else
568 queue->cmnd_capsule_len = sizeof(struct nvme_command);
569
570 queue->queue_size = queue_size;
571
572 queue->cm_id = rdma_create_id(&init_net, nvme_rdma_cm_handler, queue,
573 RDMA_PS_TCP, IB_QPT_RC);
574 if (IS_ERR(queue->cm_id)) {
575 dev_info(ctrl->ctrl.device,
576 "failed to create CM ID: %ld\n", PTR_ERR(queue->cm_id));
577 return PTR_ERR(queue->cm_id);
578 }
579
580 queue->cm_error = -ETIMEDOUT;
581 ret = rdma_resolve_addr(queue->cm_id, NULL, &ctrl->addr,
582 NVME_RDMA_CONNECT_TIMEOUT_MS);
583 if (ret) {
584 dev_info(ctrl->ctrl.device,
585 "rdma_resolve_addr failed (%d).\n", ret);
586 goto out_destroy_cm_id;
587 }
588
589 ret = nvme_rdma_wait_for_cm(queue);
590 if (ret) {
591 dev_info(ctrl->ctrl.device,
592 "rdma_resolve_addr wait failed (%d).\n", ret);
593 goto out_destroy_cm_id;
594 }
595
596 set_bit(NVME_RDMA_Q_CONNECTED, &queue->flags);
597
598 return 0;
599
600out_destroy_cm_id:
601 rdma_destroy_id(queue->cm_id);
602 return ret;
603}
604
605static void nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
606{
607 rdma_disconnect(queue->cm_id);
608 ib_drain_qp(queue->qp);
609}
610
611static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)
612{
613 nvme_rdma_destroy_queue_ib(queue);
614 rdma_destroy_id(queue->cm_id);
615}
616
617static void nvme_rdma_stop_and_free_queue(struct nvme_rdma_queue *queue)
618{
619 if (!test_and_clear_bit(NVME_RDMA_Q_CONNECTED, &queue->flags))
620 return;
621 nvme_rdma_stop_queue(queue);
622 nvme_rdma_free_queue(queue);
623}
624
625static void nvme_rdma_free_io_queues(struct nvme_rdma_ctrl *ctrl)
626{
627 int i;
628
629 for (i = 1; i < ctrl->queue_count; i++)
630 nvme_rdma_stop_and_free_queue(&ctrl->queues[i]);
631}
632
633static int nvme_rdma_connect_io_queues(struct nvme_rdma_ctrl *ctrl)
634{
635 int i, ret = 0;
636
637 for (i = 1; i < ctrl->queue_count; i++) {
638 ret = nvmf_connect_io_queue(&ctrl->ctrl, i);
639 if (ret)
640 break;
641 }
642
643 return ret;
644}
645
646static int nvme_rdma_init_io_queues(struct nvme_rdma_ctrl *ctrl)
647{
648 int i, ret;
649
650 for (i = 1; i < ctrl->queue_count; i++) {
651 ret = nvme_rdma_init_queue(ctrl, i, ctrl->ctrl.sqsize);
652 if (ret) {
653 dev_info(ctrl->ctrl.device,
654 "failed to initialize i/o queue: %d\n", ret);
655 goto out_free_queues;
656 }
657 }
658
659 return 0;
660
661out_free_queues:
662 for (; i >= 1; i--)
663 nvme_rdma_stop_and_free_queue(&ctrl->queues[i]);
664
665 return ret;
666}
667
668static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl)
669{
670 nvme_rdma_free_qe(ctrl->queues[0].device->dev, &ctrl->async_event_sqe,
671 sizeof(struct nvme_command), DMA_TO_DEVICE);
672 nvme_rdma_stop_and_free_queue(&ctrl->queues[0]);
673 blk_cleanup_queue(ctrl->ctrl.admin_q);
674 blk_mq_free_tag_set(&ctrl->admin_tag_set);
675 nvme_rdma_dev_put(ctrl->device);
676}
677
678static void nvme_rdma_free_ctrl(struct nvme_ctrl *nctrl)
679{
680 struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
681
682 if (list_empty(&ctrl->list))
683 goto free_ctrl;
684
685 mutex_lock(&nvme_rdma_ctrl_mutex);
686 list_del(&ctrl->list);
687 mutex_unlock(&nvme_rdma_ctrl_mutex);
688
71102307
CH		 689 kfree(ctrl->queues);
690 nvmf_free_options(nctrl->opts);
691free_ctrl:
692 kfree(ctrl);
693}
694
695static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
696{
697 struct nvme_rdma_ctrl *ctrl = container_of(to_delayed_work(work),
698 struct nvme_rdma_ctrl, reconnect_work);
699 bool changed;
700 int ret;
701
702 if (ctrl->queue_count > 1) {
703 nvme_rdma_free_io_queues(ctrl);
704
705 ret = blk_mq_reinit_tagset(&ctrl->tag_set);
706 if (ret)
707 goto requeue;
708 }
709
710 nvme_rdma_stop_and_free_queue(&ctrl->queues[0]);
711
712 ret = blk_mq_reinit_tagset(&ctrl->admin_tag_set);
713 if (ret)
714 goto requeue;
715
716 ret = nvme_rdma_init_queue(ctrl, 0, NVMF_AQ_DEPTH);
717 if (ret)
718 goto requeue;
719
720 blk_mq_start_stopped_hw_queues(ctrl->ctrl.admin_q, true);
721
722 ret = nvmf_connect_admin_queue(&ctrl->ctrl);
723 if (ret)
724 goto stop_admin_q;
725
726 ret = nvme_enable_ctrl(&ctrl->ctrl, ctrl->cap);
727 if (ret)
728 goto stop_admin_q;
729
730 nvme_start_keep_alive(&ctrl->ctrl);
731
732 if (ctrl->queue_count > 1) {
733 ret = nvme_rdma_init_io_queues(ctrl);
734 if (ret)
735 goto stop_admin_q;
736
737 ret = nvme_rdma_connect_io_queues(ctrl);
738 if (ret)
739 goto stop_admin_q;
740 }
741
742 changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
743 WARN_ON_ONCE(!changed);
744
5f372eb3 745 if (ctrl->queue_count > 1) {
71102307 746 nvme_start_queues(&ctrl->ctrl);
5f372eb3
SG		 747 nvme_queue_scan(&ctrl->ctrl);
748 }
71102307
CH		 749
750 dev_info(ctrl->ctrl.device, "Successfully reconnected\n");
751
752 return;
753
754stop_admin_q:
755 blk_mq_stop_hw_queues(ctrl->ctrl.admin_q);
756requeue:
757 /* Make sure we are not resetting/deleting */
758 if (ctrl->ctrl.state == NVME_CTRL_RECONNECTING) {
759 dev_info(ctrl->ctrl.device,
760 "Failed reconnect attempt, requeueing...\n");
761 queue_delayed_work(nvme_rdma_wq, &ctrl->reconnect_work,
762 ctrl->reconnect_delay * HZ);
763 }
764}
765
766static void nvme_rdma_error_recovery_work(struct work_struct *work)
767{
768 struct nvme_rdma_ctrl *ctrl = container_of(work,
769 struct nvme_rdma_ctrl, err_work);
770
771 nvme_stop_keep_alive(&ctrl->ctrl);
772 if (ctrl->queue_count > 1)
773 nvme_stop_queues(&ctrl->ctrl);
774 blk_mq_stop_hw_queues(ctrl->ctrl.admin_q);
775
776 /* We must take care of fastfail/requeue all our inflight requests */
777 if (ctrl->queue_count > 1)
778 blk_mq_tagset_busy_iter(&ctrl->tag_set,
779 nvme_cancel_request, &ctrl->ctrl);
780 blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
781 nvme_cancel_request, &ctrl->ctrl);
782
783 dev_info(ctrl->ctrl.device, "reconnecting in %d seconds\n",
784 ctrl->reconnect_delay);
785
786 queue_delayed_work(nvme_rdma_wq, &ctrl->reconnect_work,
787 ctrl->reconnect_delay * HZ);
788}
789
790static void nvme_rdma_error_recovery(struct nvme_rdma_ctrl *ctrl)
791{
792 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RECONNECTING))
793 return;
794
795 queue_work(nvme_rdma_wq, &ctrl->err_work);
796}
797
798static void nvme_rdma_wr_error(struct ib_cq *cq, struct ib_wc *wc,
799 const char *op)
800{
801 struct nvme_rdma_queue *queue = cq->cq_context;
802 struct nvme_rdma_ctrl *ctrl = queue->ctrl;
803
804 if (ctrl->ctrl.state == NVME_CTRL_LIVE)
805 dev_info(ctrl->ctrl.device,
806 "%s for CQE 0x%p failed with status %s (%d)\n",
807 op, wc->wr_cqe,
808 ib_wc_status_msg(wc->status), wc->status);
809 nvme_rdma_error_recovery(ctrl);
810}
811
812static void nvme_rdma_memreg_done(struct ib_cq *cq, struct ib_wc *wc)
813{
814 if (unlikely(wc->status != IB_WC_SUCCESS))
815 nvme_rdma_wr_error(cq, wc, "MEMREG");
816}
817
818static void nvme_rdma_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc)
819{
820 if (unlikely(wc->status != IB_WC_SUCCESS))
821 nvme_rdma_wr_error(cq, wc, "LOCAL_INV");
822}
823
824static int nvme_rdma_inv_rkey(struct nvme_rdma_queue *queue,
825 struct nvme_rdma_request *req)
826{
827 struct ib_send_wr *bad_wr;
828 struct ib_send_wr wr = {
829 .opcode = IB_WR_LOCAL_INV,
830 .next = NULL,
831 .num_sge = 0,
832 .send_flags = 0,
833 .ex.invalidate_rkey = req->mr->rkey,
834 };
835
836 req->reg_cqe.done = nvme_rdma_inv_rkey_done;
837 wr.wr_cqe = &req->reg_cqe;
838
839 return ib_post_send(queue->qp, &wr, &bad_wr);
840}
841
842static void nvme_rdma_unmap_data(struct nvme_rdma_queue *queue,
843 struct request *rq)
844{
845 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
846 struct nvme_rdma_ctrl *ctrl = queue->ctrl;
847 struct nvme_rdma_device *dev = queue->device;
848 struct ib_device *ibdev = dev->dev;
849 int res;
850
851 if (!blk_rq_bytes(rq))
852 return;
853
854 if (req->need_inval) {
855 res = nvme_rdma_inv_rkey(queue, req);
856 if (res < 0) {
857 dev_err(ctrl->ctrl.device,
858 "Queueing INV WR for rkey %#x failed (%d)\n",
859 req->mr->rkey, res);
860 nvme_rdma_error_recovery(queue->ctrl);
861 }
862 }
863
864 ib_dma_unmap_sg(ibdev, req->sg_table.sgl,
865 req->nents, rq_data_dir(rq) ==
866 WRITE ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
867
868 nvme_cleanup_cmd(rq);
869 sg_free_table_chained(&req->sg_table, true);
870}
871
872static int nvme_rdma_set_sg_null(struct nvme_command *c)
873{
874 struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
875
876 sg->addr = 0;
877 put_unaligned_le24(0, sg->length);
878 put_unaligned_le32(0, sg->key);
879 sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4;
880 return 0;
881}
882
883static int nvme_rdma_map_sg_inline(struct nvme_rdma_queue *queue,
884 struct nvme_rdma_request *req, struct nvme_command *c)
885{
886 struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
887
888 req->sge[1].addr = sg_dma_address(req->sg_table.sgl);
889 req->sge[1].length = sg_dma_len(req->sg_table.sgl);
890 req->sge[1].lkey = queue->device->pd->local_dma_lkey;
891
892 sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);
893 sg->length = cpu_to_le32(sg_dma_len(req->sg_table.sgl));
894 sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET;
895
896 req->inline_data = true;
897 req->num_sge++;
898 return 0;
899}
900
901static int nvme_rdma_map_sg_single(struct nvme_rdma_queue *queue,
902 struct nvme_rdma_request *req, struct nvme_command *c)
903{
904 struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
905
906 sg->addr = cpu_to_le64(sg_dma_address(req->sg_table.sgl));
907 put_unaligned_le24(sg_dma_len(req->sg_table.sgl), sg->length);
908 put_unaligned_le32(queue->device->mr->rkey, sg->key);
909 sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4;
910 return 0;
911}
912
913static int nvme_rdma_map_sg_fr(struct nvme_rdma_queue *queue,
914 struct nvme_rdma_request *req, struct nvme_command *c,
915 int count)
916{
917 struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
918 int nr;
919
920 nr = ib_map_mr_sg(req->mr, req->sg_table.sgl, count, NULL, PAGE_SIZE);
921 if (nr < count) {
922 if (nr < 0)
923 return nr;
924 return -EINVAL;
925 }
926
927 ib_update_fast_reg_key(req->mr, ib_inc_rkey(req->mr->rkey));
928
929 req->reg_cqe.done = nvme_rdma_memreg_done;
930 memset(&req->reg_wr, 0, sizeof(req->reg_wr));
931 req->reg_wr.wr.opcode = IB_WR_REG_MR;
932 req->reg_wr.wr.wr_cqe = &req->reg_cqe;
933 req->reg_wr.wr.num_sge = 0;
934 req->reg_wr.mr = req->mr;
935 req->reg_wr.key = req->mr->rkey;
936 req->reg_wr.access = IB_ACCESS_LOCAL_WRITE |
937 IB_ACCESS_REMOTE_READ |
938 IB_ACCESS_REMOTE_WRITE;
939
940 req->need_inval = true;
941
942 sg->addr = cpu_to_le64(req->mr->iova);
943 put_unaligned_le24(req->mr->length, sg->length);
944 put_unaligned_le32(req->mr->rkey, sg->key);
945 sg->type = (NVME_KEY_SGL_FMT_DATA_DESC << 4) |
946 NVME_SGL_FMT_INVALIDATE;
947
948 return 0;
949}
950
951static int nvme_rdma_map_data(struct nvme_rdma_queue *queue,
952 struct request *rq, unsigned int map_len,
953 struct nvme_command *c)
954{
955 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
956 struct nvme_rdma_device *dev = queue->device;
957 struct ib_device *ibdev = dev->dev;
958 int nents, count;
959 int ret;
960
961 req->num_sge = 1;
962 req->inline_data = false;
963 req->need_inval = false;
964
965 c->common.flags |= NVME_CMD_SGL_METABUF;
966
967 if (!blk_rq_bytes(rq))
968 return nvme_rdma_set_sg_null(c);
969
970 req->sg_table.sgl = req->first_sgl;
971 ret = sg_alloc_table_chained(&req->sg_table, rq->nr_phys_segments,
972 req->sg_table.sgl);
973 if (ret)
974 return -ENOMEM;
975
976 nents = blk_rq_map_sg(rq->q, rq, req->sg_table.sgl);
977 BUG_ON(nents > rq->nr_phys_segments);
978 req->nents = nents;
979
980 count = ib_dma_map_sg(ibdev, req->sg_table.sgl, nents,
981 rq_data_dir(rq) == WRITE ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
982 if (unlikely(count <= 0)) {
983 sg_free_table_chained(&req->sg_table, true);
984 return -EIO;
985 }
986
987 if (count == 1) {
988 if (rq_data_dir(rq) == WRITE &&
989 map_len <= nvme_rdma_inline_data_size(queue) &&
990 nvme_rdma_queue_idx(queue))
991 return nvme_rdma_map_sg_inline(queue, req, c);
992
993 if (!register_always)
994 return nvme_rdma_map_sg_single(queue, req, c);
995 }
996
997 return nvme_rdma_map_sg_fr(queue, req, c, count);
998}
999
1000static void nvme_rdma_send_done(struct ib_cq *cq, struct ib_wc *wc)
1001{
1002 if (unlikely(wc->status != IB_WC_SUCCESS))
1003 nvme_rdma_wr_error(cq, wc, "SEND");
1004}
1005
1006static int nvme_rdma_post_send(struct nvme_rdma_queue *queue,
1007 struct nvme_rdma_qe *qe, struct ib_sge *sge, u32 num_sge,
1008 struct ib_send_wr *first, bool flush)
1009{
1010 struct ib_send_wr wr, *bad_wr;
1011 int ret;
1012
1013 sge->addr = qe->dma;
1014 sge->length = sizeof(struct nvme_command),
1015 sge->lkey = queue->device->pd->local_dma_lkey;
1016
1017 qe->cqe.done = nvme_rdma_send_done;
1018
1019 wr.next = NULL;
1020 wr.wr_cqe = &qe->cqe;
1021 wr.sg_list = sge;
1022 wr.num_sge = num_sge;
1023 wr.opcode = IB_WR_SEND;
1024 wr.send_flags = 0;
1025
1026 /*
1027 * Unsignalled send completions are another giant desaster in the
1028 * IB Verbs spec: If we don't regularly post signalled sends
1029 * the send queue will fill up and only a QP reset will rescue us.
1030 * Would have been way to obvious to handle this in hardware or
1031 * at least the RDMA stack..
1032 *
1033 * This messy and racy code sniplet is copy and pasted from the iSER
1034 * initiator, and the magic '32' comes from there as well.
1035 *
1036 * Always signal the flushes. The magic request used for the flush
1037 * sequencer is not allocated in our driver's tagset and it's
1038 * triggered to be freed by blk_cleanup_queue(). So we need to
1039 * always mark it as signaled to ensure that the "wr_cqe", which is
1040 * embeded in request's payload, is not freed when __ib_process_cq()
1041 * calls wr_cqe->done().
1042 */
1043 if ((++queue->sig_count % 32) == 0 || flush)
1044 wr.send_flags |= IB_SEND_SIGNALED;
1045
1046 if (first)
1047 first->next = &wr;
1048 else
1049 first = &wr;
1050
1051 ret = ib_post_send(queue->qp, first, &bad_wr);
1052 if (ret) {
1053 dev_err(queue->ctrl->ctrl.device,
1054 "%s failed with error code %d\n", __func__, ret);
1055 }
1056 return ret;
1057}
1058
1059static int nvme_rdma_post_recv(struct nvme_rdma_queue *queue,
1060 struct nvme_rdma_qe *qe)
1061{
1062 struct ib_recv_wr wr, *bad_wr;
1063 struct ib_sge list;
1064 int ret;
1065
1066 list.addr = qe->dma;
1067 list.length = sizeof(struct nvme_completion);
1068 list.lkey = queue->device->pd->local_dma_lkey;
1069
1070 qe->cqe.done = nvme_rdma_recv_done;
1071
1072 wr.next = NULL;
1073 wr.wr_cqe = &qe->cqe;
1074 wr.sg_list = &list;
1075 wr.num_sge = 1;
1076
1077 ret = ib_post_recv(queue->qp, &wr, &bad_wr);
1078 if (ret) {
1079 dev_err(queue->ctrl->ctrl.device,
1080 "%s failed with error code %d\n", __func__, ret);
1081 }
1082 return ret;
1083}
1084
1085static struct blk_mq_tags *nvme_rdma_tagset(struct nvme_rdma_queue *queue)
1086{
1087 u32 queue_idx = nvme_rdma_queue_idx(queue);
1088
1089 if (queue_idx == 0)
1090 return queue->ctrl->admin_tag_set.tags[queue_idx];
1091 return queue->ctrl->tag_set.tags[queue_idx - 1];
1092}
1093
1094static void nvme_rdma_submit_async_event(struct nvme_ctrl *arg, int aer_idx)
1095{
1096 struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(arg);
1097 struct nvme_rdma_queue *queue = &ctrl->queues[0];
1098 struct ib_device *dev = queue->device->dev;
1099 struct nvme_rdma_qe *sqe = &ctrl->async_event_sqe;
1100 struct nvme_command *cmd = sqe->data;
1101 struct ib_sge sge;
1102 int ret;
1103
1104 if (WARN_ON_ONCE(aer_idx != 0))
1105 return;
1106
1107 ib_dma_sync_single_for_cpu(dev, sqe->dma, sizeof(*cmd), DMA_TO_DEVICE);
1108
1109 memset(cmd, 0, sizeof(*cmd));
1110 cmd->common.opcode = nvme_admin_async_event;
1111 cmd->common.command_id = NVME_RDMA_AQ_BLKMQ_DEPTH;
1112 cmd->common.flags |= NVME_CMD_SGL_METABUF;
1113 nvme_rdma_set_sg_null(cmd);
1114
1115 ib_dma_sync_single_for_device(dev, sqe->dma, sizeof(*cmd),
1116 DMA_TO_DEVICE);
1117
1118 ret = nvme_rdma_post_send(queue, sqe, &sge, 1, NULL, false);
1119 WARN_ON_ONCE(ret);
1120}
1121
1122static int nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue,
1123 struct nvme_completion *cqe, struct ib_wc *wc, int tag)
1124{
1125 u16 status = le16_to_cpu(cqe->status);
1126 struct request *rq;
1127 struct nvme_rdma_request *req;
1128 int ret = 0;
1129
1130 status >>= 1;
1131
1132 rq = blk_mq_tag_to_rq(nvme_rdma_tagset(queue), cqe->command_id);
1133 if (!rq) {
1134 dev_err(queue->ctrl->ctrl.device,
1135 "tag 0x%x on QP %#x not found\n",
1136 cqe->command_id, queue->qp->qp_num);
1137 nvme_rdma_error_recovery(queue->ctrl);
1138 return ret;
1139 }
1140 req = blk_mq_rq_to_pdu(rq);
1141
1142 if (rq->cmd_type == REQ_TYPE_DRV_PRIV && rq->special)
1143 memcpy(rq->special, cqe, sizeof(*cqe));
1144
1145 if (rq->tag == tag)
1146 ret = 1;
1147
1148 if ((wc->wc_flags & IB_WC_WITH_INVALIDATE) &&
1149 wc->ex.invalidate_rkey == req->mr->rkey)
1150 req->need_inval = false;
1151
1152 blk_mq_complete_request(rq, status);
1153
1154 return ret;
1155}
1156
1157static int __nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc, int tag)
1158{
1159 struct nvme_rdma_qe *qe =
1160 container_of(wc->wr_cqe, struct nvme_rdma_qe, cqe);
1161 struct nvme_rdma_queue *queue = cq->cq_context;
1162 struct ib_device *ibdev = queue->device->dev;
1163 struct nvme_completion *cqe = qe->data;
1164 const size_t len = sizeof(struct nvme_completion);
1165 int ret = 0;
1166
1167 if (unlikely(wc->status != IB_WC_SUCCESS)) {
1168 nvme_rdma_wr_error(cq, wc, "RECV");
1169 return 0;
1170 }
1171
1172 ib_dma_sync_single_for_cpu(ibdev, qe->dma, len, DMA_FROM_DEVICE);
1173 /*
1174 * AEN requests are special as they don't time out and can
1175 * survive any kind of queue freeze and often don't respond to
1176 * aborts. We don't even bother to allocate a struct request
1177 * for them but rather special case them here.
1178 */
1179 if (unlikely(nvme_rdma_queue_idx(queue) == 0 &&
1180 cqe->command_id >= NVME_RDMA_AQ_BLKMQ_DEPTH))
1181 nvme_complete_async_event(&queue->ctrl->ctrl, cqe);
1182 else
1183 ret = nvme_rdma_process_nvme_rsp(queue, cqe, wc, tag);
1184 ib_dma_sync_single_for_device(ibdev, qe->dma, len, DMA_FROM_DEVICE);
1185
1186 nvme_rdma_post_recv(queue, qe);
1187 return ret;
1188}
1189
1190static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc)
1191{
1192 __nvme_rdma_recv_done(cq, wc, -1);
1193}
1194
1195static int nvme_rdma_conn_established(struct nvme_rdma_queue *queue)
1196{
1197 int ret, i;
1198
1199 for (i = 0; i < queue->queue_size; i++) {
1200 ret = nvme_rdma_post_recv(queue, &queue->rsp_ring[i]);
1201 if (ret)
1202 goto out_destroy_queue_ib;
1203 }
1204
1205 return 0;
1206
1207out_destroy_queue_ib:
1208 nvme_rdma_destroy_queue_ib(queue);
1209 return ret;
1210}
1211
1212static int nvme_rdma_conn_rejected(struct nvme_rdma_queue *queue,
1213 struct rdma_cm_event *ev)
1214{
1215 if (ev->param.conn.private_data_len) {
1216 struct nvme_rdma_cm_rej *rej =
1217 (struct nvme_rdma_cm_rej *)ev->param.conn.private_data;
1218
1219 dev_err(queue->ctrl->ctrl.device,
1220 "Connect rejected, status %d.", le16_to_cpu(rej->sts));
1221 /* XXX: Think of something clever to do here... */
1222 } else {
1223 dev_err(queue->ctrl->ctrl.device,
1224 "Connect rejected, no private data.\n");
1225 }
1226
1227 return -ECONNRESET;
1228}
1229
1230static int nvme_rdma_addr_resolved(struct nvme_rdma_queue *queue)
1231{
1232 struct nvme_rdma_device *dev;
1233 int ret;
1234
1235 dev = nvme_rdma_find_get_device(queue->cm_id);
1236 if (!dev) {
1237 dev_err(queue->cm_id->device->dma_device,
1238 "no client data found!\n");
1239 return -ECONNREFUSED;
1240 }
1241
1242 ret = nvme_rdma_create_queue_ib(queue, dev);
1243 if (ret) {
1244 nvme_rdma_dev_put(dev);
1245 goto out;
1246 }
1247
1248 ret = rdma_resolve_route(queue->cm_id, NVME_RDMA_CONNECT_TIMEOUT_MS);
1249 if (ret) {
1250 dev_err(queue->ctrl->ctrl.device,
1251 "rdma_resolve_route failed (%d).\n",
1252 queue->cm_error);
1253 goto out_destroy_queue;
1254 }
1255
1256 return 0;
1257
1258out_destroy_queue:
1259 nvme_rdma_destroy_queue_ib(queue);
1260out:
1261 return ret;
1262}
1263
1264static int nvme_rdma_route_resolved(struct nvme_rdma_queue *queue)
1265{
1266 struct nvme_rdma_ctrl *ctrl = queue->ctrl;
1267 struct rdma_conn_param param = { };
0b857b44 1268 struct nvme_rdma_cm_req priv = { };
71102307
CH		 1269 int ret;
1270
1271 param.qp_num = queue->qp->qp_num;
1272 param.flow_control = 1;
1273
1274 param.responder_resources = queue->device->dev->attrs.max_qp_rd_atom;
2ac17c28
SG		 1275 /* maximum retry count */
1276 param.retry_count = 7;
71102307
CH		 1277 param.rnr_retry_count = 7;
1278 param.private_data = &priv;
1279 param.private_data_len = sizeof(priv);
1280
1281 priv.recfmt = cpu_to_le16(NVME_RDMA_CM_FMT_1_0);
1282 priv.qid = cpu_to_le16(nvme_rdma_queue_idx(queue));
1283 priv.hrqsize = cpu_to_le16(queue->queue_size);
1284 priv.hsqsize = cpu_to_le16(queue->queue_size);
1285
1286 ret = rdma_connect(queue->cm_id, &param);
1287 if (ret) {
1288 dev_err(ctrl->ctrl.device,
1289 "rdma_connect failed (%d).\n", ret);
1290 goto out_destroy_queue_ib;
1291 }
1292
1293 return 0;
1294
1295out_destroy_queue_ib:
1296 nvme_rdma_destroy_queue_ib(queue);
1297 return ret;
1298}
1299
1300/**
1301 * nvme_rdma_device_unplug() - Handle RDMA device unplug
1302 * @queue: Queue that owns the cm_id that caught the event
1303 *
1304 * DEVICE_REMOVAL event notifies us that the RDMA device is about
1305 * to unplug so we should take care of destroying our RDMA resources.
1306 * This event will be generated for each allocated cm_id.
1307 *
1308 * In our case, the RDMA resources are managed per controller and not
1309 * only per queue. So the way we handle this is we trigger an implicit
1310 * controller deletion upon the first DEVICE_REMOVAL event we see, and
1311 * hold the event inflight until the controller deletion is completed.
1312 *
1313 * One exception that we need to handle is the destruction of the cm_id
1314 * that caught the event. Since we hold the callout until the controller
1315 * deletion is completed, we'll deadlock if the controller deletion will
1316 * call rdma_destroy_id on this queue's cm_id. Thus, we claim ownership
57de5a0a
SG		 1317 * of destroying this queue before-hand, destroy the queue resources,
1318 * then queue the controller deletion which won't destroy this queue and
1319 * we destroy the cm_id implicitely by returning a non-zero rc to the callout.
71102307
CH		 1320 */
1321static int nvme_rdma_device_unplug(struct nvme_rdma_queue *queue)
1322{
1323 struct nvme_rdma_ctrl *ctrl = queue->ctrl;
57de5a0a 1324 int ret;
71102307 1325
57de5a0a
SG		 1326 /* Own the controller deletion */
1327 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING))
1328 return 0;
71102307 1329
57de5a0a
SG		 1330 dev_warn(ctrl->ctrl.device,
1331 "Got rdma device removal event, deleting ctrl\n");
71102307 1332
57de5a0a
SG		 1333 /* Get rid of reconnect work if its running */
1334 cancel_delayed_work_sync(&ctrl->reconnect_work);
71102307 1335
57de5a0a
SG		 1336 /* Disable the queue so ctrl delete won't free it */
1337 if (test_and_clear_bit(NVME_RDMA_Q_CONNECTED, &queue->flags)) {
71102307 1338 /* Free this queue ourselves */
57de5a0a 1339 nvme_rdma_stop_queue(queue);
71102307 1340 nvme_rdma_destroy_queue_ib(queue);
57de5a0a
SG		 1341
1342 /* Return non-zero so the cm_id will destroy implicitly */
1343 ret = 1;
71102307
CH		 1344 }
1345
57de5a0a
SG		 1346 /* Queue controller deletion */
1347 queue_work(nvme_rdma_wq, &ctrl->delete_work);
1348 flush_work(&ctrl->delete_work);
1349 return ret;
71102307
CH		 1350}
1351
1352static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
1353 struct rdma_cm_event *ev)
1354{
1355 struct nvme_rdma_queue *queue = cm_id->context;
1356 int cm_error = 0;
1357
1358 dev_dbg(queue->ctrl->ctrl.device, "%s (%d): status %d id %p\n",
1359 rdma_event_msg(ev->event), ev->event,
1360 ev->status, cm_id);
1361
1362 switch (ev->event) {
1363 case RDMA_CM_EVENT_ADDR_RESOLVED:
1364 cm_error = nvme_rdma_addr_resolved(queue);
1365 break;
1366 case RDMA_CM_EVENT_ROUTE_RESOLVED:
1367 cm_error = nvme_rdma_route_resolved(queue);
1368 break;
1369 case RDMA_CM_EVENT_ESTABLISHED:
1370 queue->cm_error = nvme_rdma_conn_established(queue);
1371 /* complete cm_done regardless of success/failure */
1372 complete(&queue->cm_done);
1373 return 0;
1374 case RDMA_CM_EVENT_REJECTED:
1375 cm_error = nvme_rdma_conn_rejected(queue, ev);
1376 break;
1377 case RDMA_CM_EVENT_ADDR_ERROR:
1378 case RDMA_CM_EVENT_ROUTE_ERROR:
1379 case RDMA_CM_EVENT_CONNECT_ERROR:
1380 case RDMA_CM_EVENT_UNREACHABLE:
1381 dev_dbg(queue->ctrl->ctrl.device,
1382 "CM error event %d\n", ev->event);
1383 cm_error = -ECONNRESET;
1384 break;
1385 case RDMA_CM_EVENT_DISCONNECTED:
1386 case RDMA_CM_EVENT_ADDR_CHANGE:
1387 case RDMA_CM_EVENT_TIMEWAIT_EXIT:
1388 dev_dbg(queue->ctrl->ctrl.device,
1389 "disconnect received - connection closed\n");
1390 nvme_rdma_error_recovery(queue->ctrl);
1391 break;
1392 case RDMA_CM_EVENT_DEVICE_REMOVAL:
1393 /* return 1 means impliciy CM ID destroy */
1394 return nvme_rdma_device_unplug(queue);
1395 default:
1396 dev_err(queue->ctrl->ctrl.device,
1397 "Unexpected RDMA CM event (%d)\n", ev->event);
1398 nvme_rdma_error_recovery(queue->ctrl);
1399 break;
1400 }
1401
1402 if (cm_error) {
1403 queue->cm_error = cm_error;
1404 complete(&queue->cm_done);
1405 }
1406
1407 return 0;
1408}
1409
1410static enum blk_eh_timer_return
1411nvme_rdma_timeout(struct request *rq, bool reserved)
1412{
1413 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1414
1415 /* queue error recovery */
1416 nvme_rdma_error_recovery(req->queue->ctrl);
1417
1418 /* fail with DNR on cmd timeout */
1419 rq->errors = NVME_SC_ABORT_REQ | NVME_SC_DNR;
1420
1421 return BLK_EH_HANDLED;
1422}
1423
1424static int nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx,
1425 const struct blk_mq_queue_data *bd)
1426{
1427 struct nvme_ns *ns = hctx->queue->queuedata;
1428 struct nvme_rdma_queue *queue = hctx->driver_data;
1429 struct request *rq = bd->rq;
1430 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1431 struct nvme_rdma_qe *sqe = &req->sqe;
1432 struct nvme_command *c = sqe->data;
1433 bool flush = false;
1434 struct ib_device *dev;
1435 unsigned int map_len;
1436 int ret;
1437
1438 WARN_ON_ONCE(rq->tag < 0);
1439
1440 dev = queue->device->dev;
1441 ib_dma_sync_single_for_cpu(dev, sqe->dma,
1442 sizeof(struct nvme_command), DMA_TO_DEVICE);
1443
1444 ret = nvme_setup_cmd(ns, rq, c);
1445 if (ret)
1446 return ret;
1447
1448 c->common.command_id = rq->tag;
1449 blk_mq_start_request(rq);
1450
1451 map_len = nvme_map_len(rq);
1452 ret = nvme_rdma_map_data(queue, rq, map_len, c);
1453 if (ret < 0) {
1454 dev_err(queue->ctrl->ctrl.device,
1455 "Failed to map data (%d)\n", ret);
1456 nvme_cleanup_cmd(rq);
1457 goto err;
1458 }
1459
1460 ib_dma_sync_single_for_device(dev, sqe->dma,
1461 sizeof(struct nvme_command), DMA_TO_DEVICE);
1462
1463 if (rq->cmd_type == REQ_TYPE_FS && req_op(rq) == REQ_OP_FLUSH)
1464 flush = true;
1465 ret = nvme_rdma_post_send(queue, sqe, req->sge, req->num_sge,
1466 req->need_inval ? &req->reg_wr.wr : NULL, flush);
1467 if (ret) {
1468 nvme_rdma_unmap_data(queue, rq);
1469 goto err;
1470 }
1471
1472 return BLK_MQ_RQ_QUEUE_OK;
1473err:
1474 return (ret == -ENOMEM || ret == -EAGAIN) ?
1475 BLK_MQ_RQ_QUEUE_BUSY : BLK_MQ_RQ_QUEUE_ERROR;
1476}
1477
1478static int nvme_rdma_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
1479{
1480 struct nvme_rdma_queue *queue = hctx->driver_data;
1481 struct ib_cq *cq = queue->ib_cq;
1482 struct ib_wc wc;
1483 int found = 0;
1484
1485 ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
1486 while (ib_poll_cq(cq, 1, &wc) > 0) {
1487 struct ib_cqe *cqe = wc.wr_cqe;
1488
1489 if (cqe) {
1490 if (cqe->done == nvme_rdma_recv_done)
1491 found |= __nvme_rdma_recv_done(cq, &wc, tag);
1492 else
1493 cqe->done(cq, &wc);
1494 }
1495 }
1496
1497 return found;
1498}
1499
1500static void nvme_rdma_complete_rq(struct request *rq)
1501{
1502 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1503 struct nvme_rdma_queue *queue = req->queue;
1504 int error = 0;
1505
1506 nvme_rdma_unmap_data(queue, rq);
1507
1508 if (unlikely(rq->errors)) {
1509 if (nvme_req_needs_retry(rq, rq->errors)) {
1510 nvme_requeue_req(rq);
1511 return;
1512 }
1513
1514 if (rq->cmd_type == REQ_TYPE_DRV_PRIV)
1515 error = rq->errors;
1516 else
1517 error = nvme_error_status(rq->errors);
1518 }
1519
1520 blk_mq_end_request(rq, error);
1521}
1522
1523static struct blk_mq_ops nvme_rdma_mq_ops = {
1524 .queue_rq = nvme_rdma_queue_rq,
1525 .complete = nvme_rdma_complete_rq,
1526 .map_queue = blk_mq_map_queue,
1527 .init_request = nvme_rdma_init_request,
1528 .exit_request = nvme_rdma_exit_request,
1529 .reinit_request = nvme_rdma_reinit_request,
1530 .init_hctx = nvme_rdma_init_hctx,
1531 .poll = nvme_rdma_poll,
1532 .timeout = nvme_rdma_timeout,
1533};
1534
1535static struct blk_mq_ops nvme_rdma_admin_mq_ops = {
1536 .queue_rq = nvme_rdma_queue_rq,
1537 .complete = nvme_rdma_complete_rq,
1538 .map_queue = blk_mq_map_queue,
1539 .init_request = nvme_rdma_init_admin_request,
1540 .exit_request = nvme_rdma_exit_admin_request,
1541 .reinit_request = nvme_rdma_reinit_request,
1542 .init_hctx = nvme_rdma_init_admin_hctx,
1543 .timeout = nvme_rdma_timeout,
1544};
1545
1546static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl)
1547{
1548 int error;
1549
1550 error = nvme_rdma_init_queue(ctrl, 0, NVMF_AQ_DEPTH);
1551 if (error)
1552 return error;
1553
1554 ctrl->device = ctrl->queues[0].device;
1555
1556 /*
1557 * We need a reference on the device as long as the tag_set is alive,
1558 * as the MRs in the request structures need a valid ib_device.
1559 */
1560 error = -EINVAL;
1561 if (!nvme_rdma_dev_get(ctrl->device))
1562 goto out_free_queue;
1563
1564 ctrl->max_fr_pages = min_t(u32, NVME_RDMA_MAX_SEGMENTS,
1565 ctrl->device->dev->attrs.max_fast_reg_page_list_len);
1566
1567 memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set));
1568 ctrl->admin_tag_set.ops = &nvme_rdma_admin_mq_ops;
1569 ctrl->admin_tag_set.queue_depth = NVME_RDMA_AQ_BLKMQ_DEPTH;
1570 ctrl->admin_tag_set.reserved_tags = 2; /* connect + keep-alive */
1571 ctrl->admin_tag_set.numa_node = NUMA_NO_NODE;
1572 ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_rdma_request) +
1573 SG_CHUNK_SIZE * sizeof(struct scatterlist);
1574 ctrl->admin_tag_set.driver_data = ctrl;
1575 ctrl->admin_tag_set.nr_hw_queues = 1;
1576 ctrl->admin_tag_set.timeout = ADMIN_TIMEOUT;
1577
1578 error = blk_mq_alloc_tag_set(&ctrl->admin_tag_set);
1579 if (error)
1580 goto out_put_dev;
1581
1582 ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);
1583 if (IS_ERR(ctrl->ctrl.admin_q)) {
1584 error = PTR_ERR(ctrl->ctrl.admin_q);
1585 goto out_free_tagset;
1586 }
1587
1588 error = nvmf_connect_admin_queue(&ctrl->ctrl);
1589 if (error)
1590 goto out_cleanup_queue;
1591
1592 error = nvmf_reg_read64(&ctrl->ctrl, NVME_REG_CAP, &ctrl->cap);
1593 if (error) {
1594 dev_err(ctrl->ctrl.device,
1595 "prop_get NVME_REG_CAP failed\n");
1596 goto out_cleanup_queue;
1597 }
1598
1599 ctrl->ctrl.sqsize =
1600 min_t(int, NVME_CAP_MQES(ctrl->cap) + 1, ctrl->ctrl.sqsize);
1601
1602 error = nvme_enable_ctrl(&ctrl->ctrl, ctrl->cap);
1603 if (error)
1604 goto out_cleanup_queue;
1605
1606 ctrl->ctrl.max_hw_sectors =
1607 (ctrl->max_fr_pages - 1) << (PAGE_SHIFT - 9);
1608
1609 error = nvme_init_identify(&ctrl->ctrl);
1610 if (error)
1611 goto out_cleanup_queue;
1612
1613 error = nvme_rdma_alloc_qe(ctrl->queues[0].device->dev,
1614 &ctrl->async_event_sqe, sizeof(struct nvme_command),
1615 DMA_TO_DEVICE);
1616 if (error)
1617 goto out_cleanup_queue;
1618
1619 nvme_start_keep_alive(&ctrl->ctrl);
1620
1621 return 0;
1622
1623out_cleanup_queue:
1624 blk_cleanup_queue(ctrl->ctrl.admin_q);
1625out_free_tagset:
1626 /* disconnect and drain the queue before freeing the tagset */
1627 nvme_rdma_stop_queue(&ctrl->queues[0]);
1628 blk_mq_free_tag_set(&ctrl->admin_tag_set);
1629out_put_dev:
1630 nvme_rdma_dev_put(ctrl->device);
1631out_free_queue:
1632 nvme_rdma_free_queue(&ctrl->queues[0]);
1633 return error;
1634}
1635
1636static void nvme_rdma_shutdown_ctrl(struct nvme_rdma_ctrl *ctrl)
1637{
1638 nvme_stop_keep_alive(&ctrl->ctrl);
1639 cancel_work_sync(&ctrl->err_work);
1640 cancel_delayed_work_sync(&ctrl->reconnect_work);
1641
1642 if (ctrl->queue_count > 1) {
1643 nvme_stop_queues(&ctrl->ctrl);
1644 blk_mq_tagset_busy_iter(&ctrl->tag_set,
1645 nvme_cancel_request, &ctrl->ctrl);
1646 nvme_rdma_free_io_queues(ctrl);
1647 }
1648
45862ebc 1649 if (test_bit(NVME_RDMA_Q_CONNECTED, &ctrl->queues[0].flags))
71102307
CH		 1650 nvme_shutdown_ctrl(&ctrl->ctrl);
1651
1652 blk_mq_stop_hw_queues(ctrl->ctrl.admin_q);
1653 blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
1654 nvme_cancel_request, &ctrl->ctrl);
1655 nvme_rdma_destroy_admin_queue(ctrl);
1656}
1657
2461a8dd
SG		 1658static void __nvme_rdma_remove_ctrl(struct nvme_rdma_ctrl *ctrl, bool shutdown)
1659{
1660 nvme_uninit_ctrl(&ctrl->ctrl);
1661 if (shutdown)
1662 nvme_rdma_shutdown_ctrl(ctrl);
a34ca17a
SG		 1663
1664 if (ctrl->ctrl.tagset) {
1665 blk_cleanup_queue(ctrl->ctrl.connect_q);
1666 blk_mq_free_tag_set(&ctrl->tag_set);
1667 nvme_rdma_dev_put(ctrl->device);
1668 }
1669
2461a8dd
SG		 1670 nvme_put_ctrl(&ctrl->ctrl);
1671}
1672
71102307
CH		 1673static void nvme_rdma_del_ctrl_work(struct work_struct *work)
1674{
1675 struct nvme_rdma_ctrl *ctrl = container_of(work,
1676 struct nvme_rdma_ctrl, delete_work);
1677
2461a8dd 1678 __nvme_rdma_remove_ctrl(ctrl, true);
71102307
CH		 1679}
1680
1681static int __nvme_rdma_del_ctrl(struct nvme_rdma_ctrl *ctrl)
1682{
1683 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING))
1684 return -EBUSY;
1685
1686 if (!queue_work(nvme_rdma_wq, &ctrl->delete_work))
1687 return -EBUSY;
1688
1689 return 0;
1690}
1691
1692static int nvme_rdma_del_ctrl(struct nvme_ctrl *nctrl)
1693{
1694 struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
1695 int ret;
1696
1697 ret = __nvme_rdma_del_ctrl(ctrl);
1698 if (ret)
1699 return ret;
1700
1701 flush_work(&ctrl->delete_work);
1702
1703 return 0;
1704}
1705
1706static void nvme_rdma_remove_ctrl_work(struct work_struct *work)
1707{
1708 struct nvme_rdma_ctrl *ctrl = container_of(work,
1709 struct nvme_rdma_ctrl, delete_work);
1710
2461a8dd 1711 __nvme_rdma_remove_ctrl(ctrl, false);
71102307
CH		 1712}
1713
1714static void nvme_rdma_reset_ctrl_work(struct work_struct *work)
1715{
1716 struct nvme_rdma_ctrl *ctrl = container_of(work,
1717 struct nvme_rdma_ctrl, reset_work);
1718 int ret;
1719 bool changed;
1720
1721 nvme_rdma_shutdown_ctrl(ctrl);
1722
1723 ret = nvme_rdma_configure_admin_queue(ctrl);
1724 if (ret) {
1725 /* ctrl is already shutdown, just remove the ctrl */
1726 INIT_WORK(&ctrl->delete_work, nvme_rdma_remove_ctrl_work);
1727 goto del_dead_ctrl;
1728 }
1729
1730 if (ctrl->queue_count > 1) {
1731 ret = blk_mq_reinit_tagset(&ctrl->tag_set);
1732 if (ret)
1733 goto del_dead_ctrl;
1734
1735 ret = nvme_rdma_init_io_queues(ctrl);
1736 if (ret)
1737 goto del_dead_ctrl;
1738
1739 ret = nvme_rdma_connect_io_queues(ctrl);
1740 if (ret)
1741 goto del_dead_ctrl;
1742 }
1743
1744 changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
1745 WARN_ON_ONCE(!changed);
1746
1747 if (ctrl->queue_count > 1) {
1748 nvme_start_queues(&ctrl->ctrl);
1749 nvme_queue_scan(&ctrl->ctrl);
1750 }
1751
1752 return;
1753
1754del_dead_ctrl:
1755 /* Deleting this dead controller... */
1756 dev_warn(ctrl->ctrl.device, "Removing after reset failure\n");
1757 WARN_ON(!queue_work(nvme_rdma_wq, &ctrl->delete_work));
1758}
1759
1760static int nvme_rdma_reset_ctrl(struct nvme_ctrl *nctrl)
1761{
1762 struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
1763
1764 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RESETTING))
1765 return -EBUSY;
1766
1767 if (!queue_work(nvme_rdma_wq, &ctrl->reset_work))
1768 return -EBUSY;
1769
1770 flush_work(&ctrl->reset_work);
1771
1772 return 0;
1773}
1774
1775static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = {
1776 .name = "rdma",
1777 .module = THIS_MODULE,
1778 .is_fabrics = true,
1779 .reg_read32 = nvmf_reg_read32,
1780 .reg_read64 = nvmf_reg_read64,
1781 .reg_write32 = nvmf_reg_write32,
1782 .reset_ctrl = nvme_rdma_reset_ctrl,
1783 .free_ctrl = nvme_rdma_free_ctrl,
1784 .submit_async_event = nvme_rdma_submit_async_event,
1785 .delete_ctrl = nvme_rdma_del_ctrl,
1786 .get_subsysnqn = nvmf_get_subsysnqn,
1787 .get_address = nvmf_get_address,
1788};
1789
1790static int nvme_rdma_create_io_queues(struct nvme_rdma_ctrl *ctrl)
1791{
1792 struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
1793 int ret;
1794
1795 ret = nvme_set_queue_count(&ctrl->ctrl, &opts->nr_io_queues);
1796 if (ret)
1797 return ret;
1798
1799 ctrl->queue_count = opts->nr_io_queues + 1;
1800 if (ctrl->queue_count < 2)
1801 return 0;
1802
1803 dev_info(ctrl->ctrl.device,
1804 "creating %d I/O queues.\n", opts->nr_io_queues);
1805
1806 ret = nvme_rdma_init_io_queues(ctrl);
1807 if (ret)
1808 return ret;
1809
1810 /*
1811 * We need a reference on the device as long as the tag_set is alive,
1812 * as the MRs in the request structures need a valid ib_device.
1813 */
1814 ret = -EINVAL;
1815 if (!nvme_rdma_dev_get(ctrl->device))
1816 goto out_free_io_queues;
1817
1818 memset(&ctrl->tag_set, 0, sizeof(ctrl->tag_set));
1819 ctrl->tag_set.ops = &nvme_rdma_mq_ops;
1820 ctrl->tag_set.queue_depth = ctrl->ctrl.sqsize;
1821 ctrl->tag_set.reserved_tags = 1; /* fabric connect */
1822 ctrl->tag_set.numa_node = NUMA_NO_NODE;
1823 ctrl->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
1824 ctrl->tag_set.cmd_size = sizeof(struct nvme_rdma_request) +
1825 SG_CHUNK_SIZE * sizeof(struct scatterlist);
1826 ctrl->tag_set.driver_data = ctrl;
1827 ctrl->tag_set.nr_hw_queues = ctrl->queue_count - 1;
1828 ctrl->tag_set.timeout = NVME_IO_TIMEOUT;
1829
1830 ret = blk_mq_alloc_tag_set(&ctrl->tag_set);
1831 if (ret)
1832 goto out_put_dev;
1833 ctrl->ctrl.tagset = &ctrl->tag_set;
1834
1835 ctrl->ctrl.connect_q = blk_mq_init_queue(&ctrl->tag_set);
1836 if (IS_ERR(ctrl->ctrl.connect_q)) {
1837 ret = PTR_ERR(ctrl->ctrl.connect_q);
1838 goto out_free_tag_set;
1839 }
1840
1841 ret = nvme_rdma_connect_io_queues(ctrl);
1842 if (ret)
1843 goto out_cleanup_connect_q;
1844
1845 return 0;
1846
1847out_cleanup_connect_q:
1848 blk_cleanup_queue(ctrl->ctrl.connect_q);
1849out_free_tag_set:
1850 blk_mq_free_tag_set(&ctrl->tag_set);
1851out_put_dev:
1852 nvme_rdma_dev_put(ctrl->device);
1853out_free_io_queues:
1854 nvme_rdma_free_io_queues(ctrl);
1855 return ret;
1856}
1857
1858static int nvme_rdma_parse_ipaddr(struct sockaddr_in *in_addr, char *p)
1859{
1860 u8 *addr = (u8 *)&in_addr->sin_addr.s_addr;
1861 size_t buflen = strlen(p);
1862
1863 /* XXX: handle IPv6 addresses */
1864
1865 if (buflen > INET_ADDRSTRLEN)
1866 return -EINVAL;
1867 if (in4_pton(p, buflen, addr, '\0', NULL) == 0)
1868 return -EINVAL;
1869 in_addr->sin_family = AF_INET;
1870 return 0;
1871}
1872
1873static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,
1874 struct nvmf_ctrl_options *opts)
1875{
1876 struct nvme_rdma_ctrl *ctrl;
1877 int ret;
1878 bool changed;
1879
1880 ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
1881 if (!ctrl)
1882 return ERR_PTR(-ENOMEM);
1883 ctrl->ctrl.opts = opts;
1884 INIT_LIST_HEAD(&ctrl->list);
1885
1886 ret = nvme_rdma_parse_ipaddr(&ctrl->addr_in, opts->traddr);
1887 if (ret) {
1888 pr_err("malformed IP address passed: %s\n", opts->traddr);
1889 goto out_free_ctrl;
1890 }
1891
1892 if (opts->mask & NVMF_OPT_TRSVCID) {
1893 u16 port;
1894
1895 ret = kstrtou16(opts->trsvcid, 0, &port);
1896 if (ret)
1897 goto out_free_ctrl;
1898
1899 ctrl->addr_in.sin_port = cpu_to_be16(port);
1900 } else {
1901 ctrl->addr_in.sin_port = cpu_to_be16(NVME_RDMA_IP_PORT);
1902 }
1903
1904 ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_rdma_ctrl_ops,
1905 0 /* no quirks, we're perfect! */);
1906 if (ret)
1907 goto out_free_ctrl;
1908
1909 ctrl->reconnect_delay = opts->reconnect_delay;
1910 INIT_DELAYED_WORK(&ctrl->reconnect_work,
1911 nvme_rdma_reconnect_ctrl_work);
1912 INIT_WORK(&ctrl->err_work, nvme_rdma_error_recovery_work);
1913 INIT_WORK(&ctrl->delete_work, nvme_rdma_del_ctrl_work);
1914 INIT_WORK(&ctrl->reset_work, nvme_rdma_reset_ctrl_work);
1915 spin_lock_init(&ctrl->lock);
1916
1917 ctrl->queue_count = opts->nr_io_queues + 1; /* +1 for admin queue */
1918 ctrl->ctrl.sqsize = opts->queue_size;
71102307
CH		 1919 ctrl->ctrl.kato = opts->kato;
1920
1921 ret = -ENOMEM;
1922 ctrl->queues = kcalloc(ctrl->queue_count, sizeof(*ctrl->queues),
1923 GFP_KERNEL);
1924 if (!ctrl->queues)
1925 goto out_uninit_ctrl;
1926
1927 ret = nvme_rdma_configure_admin_queue(ctrl);
1928 if (ret)
1929 goto out_kfree_queues;
1930
1931 /* sanity check icdoff */
1932 if (ctrl->ctrl.icdoff) {
1933 dev_err(ctrl->ctrl.device, "icdoff is not supported!\n");
1934 goto out_remove_admin_queue;
1935 }
1936
1937 /* sanity check keyed sgls */
1938 if (!(ctrl->ctrl.sgls & (1 << 20))) {
1939 dev_err(ctrl->ctrl.device, "Mandatory keyed sgls are not support\n");
1940 goto out_remove_admin_queue;
1941 }
1942
1943 if (opts->queue_size > ctrl->ctrl.maxcmd) {
1944 /* warn if maxcmd is lower than queue_size */
1945 dev_warn(ctrl->ctrl.device,
1946 "queue_size %zu > ctrl maxcmd %u, clamping down\n",
1947 opts->queue_size, ctrl->ctrl.maxcmd);
1948 opts->queue_size = ctrl->ctrl.maxcmd;
1949 }
1950
1951 if (opts->nr_io_queues) {
1952 ret = nvme_rdma_create_io_queues(ctrl);
1953 if (ret)
1954 goto out_remove_admin_queue;
1955 }
1956
1957 changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
1958 WARN_ON_ONCE(!changed);
1959
1960 dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n",
1961 ctrl->ctrl.opts->subsysnqn, &ctrl->addr);
1962
1963 kref_get(&ctrl->ctrl.kref);
1964
1965 mutex_lock(&nvme_rdma_ctrl_mutex);
1966 list_add_tail(&ctrl->list, &nvme_rdma_ctrl_list);
1967 mutex_unlock(&nvme_rdma_ctrl_mutex);
1968
1969 if (opts->nr_io_queues) {
1970 nvme_queue_scan(&ctrl->ctrl);
1971 nvme_queue_async_events(&ctrl->ctrl);
1972 }
1973
1974 return &ctrl->ctrl;
1975
1976out_remove_admin_queue:
1977 nvme_stop_keep_alive(&ctrl->ctrl);
1978 nvme_rdma_destroy_admin_queue(ctrl);
1979out_kfree_queues:
1980 kfree(ctrl->queues);
1981out_uninit_ctrl:
1982 nvme_uninit_ctrl(&ctrl->ctrl);
1983 nvme_put_ctrl(&ctrl->ctrl);
1984 if (ret > 0)
1985 ret = -EIO;
1986 return ERR_PTR(ret);
1987out_free_ctrl:
1988 kfree(ctrl);
1989 return ERR_PTR(ret);
1990}
1991
1992static struct nvmf_transport_ops nvme_rdma_transport = {
1993 .name = "rdma",
1994 .required_opts = NVMF_OPT_TRADDR,
2ac17c28 1995 .allowed_opts = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY,
71102307
CH		 1996 .create_ctrl = nvme_rdma_create_ctrl,
1997};
1998
1999static int __init nvme_rdma_init_module(void)
2000{
2001 nvme_rdma_wq = create_workqueue("nvme_rdma_wq");
2002 if (!nvme_rdma_wq)
2003 return -ENOMEM;
2004
2005 nvmf_register_transport(&nvme_rdma_transport);
2006 return 0;
2007}
2008
2009static void __exit nvme_rdma_cleanup_module(void)
2010{
2011 struct nvme_rdma_ctrl *ctrl;
2012
2013 nvmf_unregister_transport(&nvme_rdma_transport);
2014
2015 mutex_lock(&nvme_rdma_ctrl_mutex);
2016 list_for_each_entry(ctrl, &nvme_rdma_ctrl_list, list)
2017 __nvme_rdma_del_ctrl(ctrl);
2018 mutex_unlock(&nvme_rdma_ctrl_mutex);
2019
2020 destroy_workqueue(nvme_rdma_wq);
2021}
2022
2023module_init(nvme_rdma_init_module);
2024module_exit(nvme_rdma_cleanup_module);
2025
2026MODULE_LICENSE("GPL v2");
Ubuntu Artful kernel mirror