nvme-rdma: Don't local invalidate if the queue is not live

[mirror_ubuntu-bionic-kernel.git] / drivers / nvme / host / rdma.c

diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
index 58983000964be4c2ce3280ca9f4dda768979ed76..a58dd0c77a2950c4203d92be77723671fa42be64 100644 (file)
--- a/drivers/nvme/host/rdma.c
+++ b/drivers/nvme/host/rdma.c
@@ -274,6 +274,9 @@ static int nvme_rdma_reinit_request(void *data, struct request *rq)
        struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
        int ret = 0;
 
+       if (WARN_ON_ONCE(!req->mr))
+               return 0;
+
        ib_dereg_mr(req->mr);
 
        req->mr = ib_alloc_mr(dev->pd, IB_MR_TYPE_MEM_REG,
@@ -434,11 +437,9 @@ out_err:
 
 static void nvme_rdma_destroy_queue_ib(struct nvme_rdma_queue *queue)
 {
-       struct nvme_rdma_device *dev;
-       struct ib_device *ibdev;
+       struct nvme_rdma_device *dev = queue->device;
+       struct ib_device *ibdev = dev->dev;
 
-       dev = queue->device;
-       ibdev = dev->dev;
        rdma_destroy_qp(queue->cm_id);
        ib_free_cq(queue->ib_cq);
 
@@ -544,7 +545,7 @@ static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl,
        ret = nvme_rdma_wait_for_cm(queue);
        if (ret) {
                dev_info(ctrl->ctrl.device,
-                       "rdma_resolve_addr wait failed (%d).\n", ret);
+                       "rdma connection establishment failed (%d)\n", ret);
                goto out_destroy_cm_id;
        }
 
@@ -670,11 +671,10 @@ out_free_queues:
        return ret;
 }
 
-static void nvme_rdma_free_tagset(struct nvme_ctrl *nctrl, bool admin)
+static void nvme_rdma_free_tagset(struct nvme_ctrl *nctrl,
+               struct blk_mq_tag_set *set)
 {
        struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
-       struct blk_mq_tag_set *set = admin ?
-                       &ctrl->admin_tag_set : &ctrl->tag_set;
 
        blk_mq_free_tag_set(set);
        nvme_rdma_dev_put(ctrl->device);
@@ -744,7 +744,7 @@ static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl,
        nvme_rdma_stop_queue(&ctrl->queues[0]);
        if (remove) {
                blk_cleanup_queue(ctrl->ctrl.admin_q);
-               nvme_rdma_free_tagset(&ctrl->ctrl, true);
+               nvme_rdma_free_tagset(&ctrl->ctrl, ctrl->ctrl.admin_tagset);
        }
        nvme_rdma_free_queue(&ctrl->queues[0]);
 }
@@ -774,8 +774,7 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
                        goto out_free_tagset;
                }
        } else {
-               error = blk_mq_reinit_tagset(&ctrl->admin_tag_set,
-                                            nvme_rdma_reinit_request);
+               error = nvme_reinit_tagset(&ctrl->ctrl, ctrl->ctrl.admin_tagset);
                if (error)
                        goto out_free_queue;
        }
@@ -819,7 +818,7 @@ out_cleanup_queue:
                blk_cleanup_queue(ctrl->ctrl.admin_q);
 out_free_tagset:
        if (new)
-               nvme_rdma_free_tagset(&ctrl->ctrl, true);
+               nvme_rdma_free_tagset(&ctrl->ctrl, ctrl->ctrl.admin_tagset);
 out_free_queue:
        nvme_rdma_free_queue(&ctrl->queues[0]);
        return error;
@@ -831,7 +830,7 @@ static void nvme_rdma_destroy_io_queues(struct nvme_rdma_ctrl *ctrl,
        nvme_rdma_stop_io_queues(ctrl);
        if (remove) {
                blk_cleanup_queue(ctrl->ctrl.connect_q);
-               nvme_rdma_free_tagset(&ctrl->ctrl, false);
+               nvme_rdma_free_tagset(&ctrl->ctrl, ctrl->ctrl.tagset);
        }
        nvme_rdma_free_io_queues(ctrl);
 }
@@ -855,8 +854,7 @@ static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new)
                        goto out_free_tag_set;
                }
        } else {
-               ret = blk_mq_reinit_tagset(&ctrl->tag_set,
-                                          nvme_rdma_reinit_request);
+               ret = nvme_reinit_tagset(&ctrl->ctrl, ctrl->ctrl.tagset);
                if (ret)
                        goto out_free_io_queues;
 
@@ -875,7 +873,7 @@ out_cleanup_connect_q:
                blk_cleanup_queue(ctrl->ctrl.connect_q);
 out_free_tag_set:
        if (new)
-               nvme_rdma_free_tagset(&ctrl->ctrl, false);
+               nvme_rdma_free_tagset(&ctrl->ctrl, ctrl->ctrl.tagset);
 out_free_io_queues:
        nvme_rdma_free_io_queues(ctrl);
        return ret;
@@ -927,10 +925,6 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
 
        ++ctrl->ctrl.nr_reconnects;
 
-       if (ctrl->ctrl.queue_count > 1)
-               nvme_rdma_destroy_io_queues(ctrl, false);
-
-       nvme_rdma_destroy_admin_queue(ctrl, false);
        ret = nvme_rdma_configure_admin_queue(ctrl, false);
        if (ret)
                goto requeue;
@@ -938,19 +932,27 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
        if (ctrl->ctrl.queue_count > 1) {
                ret = nvme_rdma_configure_io_queues(ctrl, false);
                if (ret)
-                       goto requeue;
+                       goto destroy_admin;
        }
 
        changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
-       WARN_ON_ONCE(!changed);
-       ctrl->ctrl.nr_reconnects = 0;
+       if (!changed) {
+               /* state change failure is ok if we're in DELETING state */
+               WARN_ON_ONCE(ctrl->ctrl.state != NVME_CTRL_DELETING);
+               return;
+       }
 
        nvme_start_ctrl(&ctrl->ctrl);
 
-       dev_info(ctrl->ctrl.device, "Successfully reconnected\n");
+       dev_info(ctrl->ctrl.device, "Successfully reconnected (%d attempts)\n",
+                       ctrl->ctrl.nr_reconnects);
+
+       ctrl->ctrl.nr_reconnects = 0;
 
        return;
 
+destroy_admin:
+       nvme_rdma_destroy_admin_queue(ctrl, false);
 requeue:
        dev_info(ctrl->ctrl.device, "Failed reconnect attempt %d\n",
                        ctrl->ctrl.nr_reconnects);
@@ -962,21 +964,19 @@ static void nvme_rdma_error_recovery_work(struct work_struct *work)
        struct nvme_rdma_ctrl *ctrl = container_of(work,
                        struct nvme_rdma_ctrl, err_work);
 
-       nvme_stop_ctrl(&ctrl->ctrl);
+       nvme_stop_keep_alive(&ctrl->ctrl);
 
        if (ctrl->ctrl.queue_count > 1) {
                nvme_stop_queues(&ctrl->ctrl);
-               nvme_rdma_stop_io_queues(ctrl);
-       }
-       blk_mq_quiesce_queue(ctrl->ctrl.admin_q);
-       nvme_rdma_stop_queue(&ctrl->queues[0]);
-
-       /* We must take care of fastfail/requeue all our inflight requests */
-       if (ctrl->ctrl.queue_count > 1)
                blk_mq_tagset_busy_iter(&ctrl->tag_set,
                                        nvme_cancel_request, &ctrl->ctrl);
+               nvme_rdma_destroy_io_queues(ctrl, false);
+       }
+
+       blk_mq_quiesce_queue(ctrl->ctrl.admin_q);
        blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
                                nvme_cancel_request, &ctrl->ctrl);
+       nvme_rdma_destroy_admin_queue(ctrl, false);
 
        /*
         * queues are not a live anymore, so restart the queues to fail fast
@@ -1052,7 +1052,7 @@ static void nvme_rdma_unmap_data(struct nvme_rdma_queue *queue,
        if (!blk_rq_bytes(rq))
                return;
 
-       if (req->mr->need_inval) {
+       if (req->mr->need_inval && test_bit(NVME_RDMA_Q_LIVE, &req->queue->flags)) {
                res = nvme_rdma_inv_rkey(queue, req);
                if (unlikely(res < 0)) {
                        dev_err(ctrl->ctrl.device,
@@ -1840,6 +1840,7 @@ static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = {
        .submit_async_event     = nvme_rdma_submit_async_event,
        .delete_ctrl            = nvme_rdma_del_ctrl,
        .get_address            = nvmf_get_address,
+       .reinit_request         = nvme_rdma_reinit_request,
 };
 
 static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,