]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/infiniband/ulp/srp/ib_srp.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
IB/cma: Accept connection without a valid netdev on RoCE
[mirror_ubuntu-artful-kernel.git] / drivers / infiniband / ulp / srp / ib_srp.c
CommitLineData
aef9ec39
RD		 1/*
2 * Copyright (c) 2005 Cisco Systems. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
aef9ec39
RD		 31 */
32
d236cd0e 33#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
e0bda7d8 34
aef9ec39
RD		 35#include <linux/module.h>
36#include <linux/init.h>
37#include <linux/slab.h>
38#include <linux/err.h>
39#include <linux/string.h>
40#include <linux/parser.h>
41#include <linux/random.h>
de25968c 42#include <linux/jiffies.h>
56b5390c 43#include <rdma/ib_cache.h>
aef9ec39 44
60063497 45#include <linux/atomic.h>
aef9ec39
RD		 46
47#include <scsi/scsi.h>
48#include <scsi/scsi_device.h>
49#include <scsi/scsi_dbg.h>
71444b97 50#include <scsi/scsi_tcq.h>
aef9ec39 51#include <scsi/srp.h>
3236822b 52#include <scsi/scsi_transport_srp.h>
aef9ec39 53
aef9ec39
RD		 54#include "ib_srp.h"
55
56#define DRV_NAME "ib_srp"
57#define PFX DRV_NAME ": "
713ef24e
BVA		 58#define DRV_VERSION "2.0"
59#define DRV_RELDATE "July 26, 2015"
aef9ec39
RD		 60
61MODULE_AUTHOR("Roland Dreier");
33ab3e5b 62MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator");
aef9ec39 63MODULE_LICENSE("Dual BSD/GPL");
33ab3e5b
BVA		 64MODULE_VERSION(DRV_VERSION);
65MODULE_INFO(release_date, DRV_RELDATE);
aef9ec39 66
49248644
DD		 67static unsigned int srp_sg_tablesize;
68static unsigned int cmd_sg_entries;
c07d424d
DD		 69static unsigned int indirect_sg_entries;
70static bool allow_ext_sg;
03f6fb93
BVA		 71static bool prefer_fr = true;
72static bool register_always = true;
49248644 73static int topspin_workarounds = 1;
74b0a15b 74
49248644
DD		 75module_param(srp_sg_tablesize, uint, 0444);
76MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries");
74b0a15b 77
49248644
DD		 78module_param(cmd_sg_entries, uint, 0444);
79MODULE_PARM_DESC(cmd_sg_entries,
80 "Default number of gather/scatter entries in the SRP command (default is 12, max 255)");
aef9ec39 81
c07d424d
DD		 82module_param(indirect_sg_entries, uint, 0444);
83MODULE_PARM_DESC(indirect_sg_entries,
84 "Default max number of gather/scatter entries (default is 12, max is " __stringify(SCSI_MAX_SG_CHAIN_SEGMENTS) ")");
85
86module_param(allow_ext_sg, bool, 0444);
87MODULE_PARM_DESC(allow_ext_sg,
88 "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)");
89
aef9ec39
RD		 90module_param(topspin_workarounds, int, 0444);
91MODULE_PARM_DESC(topspin_workarounds,
92 "Enable workarounds for Topspin/Cisco SRP target bugs if != 0");
93
5cfb1782
BVA		 94module_param(prefer_fr, bool, 0444);
95MODULE_PARM_DESC(prefer_fr,
96"Whether to use fast registration if both FMR and fast registration are supported");
97
b1b8854d
BVA		 98module_param(register_always, bool, 0444);
99MODULE_PARM_DESC(register_always,
100 "Use memory registration even for contiguous memory regions");
101
9c27847d 102static const struct kernel_param_ops srp_tmo_ops;
ed9b2264 103
a95cadb9
BVA		 104static int srp_reconnect_delay = 10;
105module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay,
106 S_IRUGO | S_IWUSR);
107MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts");
108
ed9b2264
BVA		 109static int srp_fast_io_fail_tmo = 15;
110module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo,
111 S_IRUGO | S_IWUSR);
112MODULE_PARM_DESC(fast_io_fail_tmo,
113 "Number of seconds between the observation of a transport"
114 " layer error and failing all I/O. \"off\" means that this"
115 " functionality is disabled.");
116
a95cadb9 117static int srp_dev_loss_tmo = 600;
ed9b2264
BVA		 118module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo,
119 S_IRUGO | S_IWUSR);
120MODULE_PARM_DESC(dev_loss_tmo,
121 "Maximum number of seconds that the SRP transport should"
122 " insulate transport layer errors. After this time has been"
123 " exceeded the SCSI host is removed. Should be"
124 " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT)
125 " if fast_io_fail_tmo has not been set. \"off\" means that"
126 " this functionality is disabled.");
127
d92c0da7
BVA		 128static unsigned ch_count;
129module_param(ch_count, uint, 0444);
130MODULE_PARM_DESC(ch_count,
131 "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA.");
132
aef9ec39 133static void srp_add_one(struct ib_device *device);
7c1eb45a 134static void srp_remove_one(struct ib_device *device, void *client_data);
509c07bc
BVA		 135static void srp_recv_completion(struct ib_cq *cq, void *ch_ptr);
136static void srp_send_completion(struct ib_cq *cq, void *ch_ptr);
aef9ec39
RD		 137static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
138
3236822b 139static struct scsi_transport_template *ib_srp_transport_template;
bcc05910 140static struct workqueue_struct *srp_remove_wq;
3236822b 141
aef9ec39
RD		 142static struct ib_client srp_client = {
143 .name = "srp",
144 .add = srp_add_one,
145 .remove = srp_remove_one
146};
147
c1a0b23b
MT		 148static struct ib_sa_client srp_sa_client;
149
ed9b2264
BVA		 150static int srp_tmo_get(char *buffer, const struct kernel_param *kp)
151{
152 int tmo = *(int *)kp->arg;
153
154 if (tmo >= 0)
155 return sprintf(buffer, "%d", tmo);
156 else
157 return sprintf(buffer, "off");
158}
159
160static int srp_tmo_set(const char *val, const struct kernel_param *kp)
161{
162 int tmo, res;
163
3fdf70ac
SG		 164 res = srp_parse_tmo(&tmo, val);
165 if (res)
166 goto out;
167
a95cadb9
BVA		 168 if (kp->arg == &srp_reconnect_delay)
169 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo,
170 srp_dev_loss_tmo);
171 else if (kp->arg == &srp_fast_io_fail_tmo)
172 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo);
ed9b2264 173 else
a95cadb9
BVA		 174 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo,
175 tmo);
ed9b2264
BVA		 176 if (res)
177 goto out;
178 *(int *)kp->arg = tmo;
179
180out:
181 return res;
182}
183
9c27847d 184static const struct kernel_param_ops srp_tmo_ops = {
ed9b2264
BVA		 185 .get = srp_tmo_get,
186 .set = srp_tmo_set,
187};
188
aef9ec39
RD		 189static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
190{
191 return (struct srp_target_port *) host->hostdata;
192}
193
194static const char *srp_target_info(struct Scsi_Host *host)
195{
196 return host_to_target(host)->target_name;
197}
198
5d7cbfd6
RD		 199static int srp_target_is_topspin(struct srp_target_port *target)
200{
201 static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad };
3d1ff48d 202 static const u8 cisco_oui[3] = { 0x00, 0x1b, 0x0d };
5d7cbfd6
RD		 203
204 return topspin_workarounds &&
3d1ff48d
RK		 205 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) ||
206 !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui));
5d7cbfd6
RD		 207}
208
aef9ec39
RD		 209static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size,
210 gfp_t gfp_mask,
211 enum dma_data_direction direction)
212{
213 struct srp_iu *iu;
214
215 iu = kmalloc(sizeof *iu, gfp_mask);
216 if (!iu)
217 goto out;
218
219 iu->buf = kzalloc(size, gfp_mask);
220 if (!iu->buf)
221 goto out_free_iu;
222
05321937
GKH		 223 iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size,
224 direction);
225 if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma))
aef9ec39
RD		 226 goto out_free_buf;
227
228 iu->size = size;
229 iu->direction = direction;
230
231 return iu;
232
233out_free_buf:
234 kfree(iu->buf);
235out_free_iu:
236 kfree(iu);
237out:
238 return NULL;
239}
240
241static void srp_free_iu(struct srp_host *host, struct srp_iu *iu)
242{
243 if (!iu)
244 return;
245
05321937
GKH		 246 ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size,
247 iu->direction);
aef9ec39
RD		 248 kfree(iu->buf);
249 kfree(iu);
250}
251
252static void srp_qp_event(struct ib_event *event, void *context)
253{
57363d98
SG		 254 pr_debug("QP event %s (%d)\n",
255 ib_event_msg(event->event), event->event);
aef9ec39
RD		 256}
257
258static int srp_init_qp(struct srp_target_port *target,
259 struct ib_qp *qp)
260{
261 struct ib_qp_attr *attr;
262 int ret;
263
264 attr = kmalloc(sizeof *attr, GFP_KERNEL);
265 if (!attr)
266 return -ENOMEM;
267
56b5390c
BVA		 268 ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev,
269 target->srp_host->port,
270 be16_to_cpu(target->pkey),
271 &attr->pkey_index);
aef9ec39
RD		 272 if (ret)
273 goto out;
274
275 attr->qp_state = IB_QPS_INIT;
276 attr->qp_access_flags = (IB_ACCESS_REMOTE_READ |
277 IB_ACCESS_REMOTE_WRITE);
278 attr->port_num = target->srp_host->port;
279
280 ret = ib_modify_qp(qp, attr,
281 IB_QP_STATE |
282 IB_QP_PKEY_INDEX |
283 IB_QP_ACCESS_FLAGS |
284 IB_QP_PORT);
285
286out:
287 kfree(attr);
288 return ret;
289}
290
509c07bc 291static int srp_new_cm_id(struct srp_rdma_ch *ch)
9fe4bcf4 292{
509c07bc 293 struct srp_target_port *target = ch->target;
9fe4bcf4
DD		 294 struct ib_cm_id *new_cm_id;
295
05321937 296 new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
509c07bc 297 srp_cm_handler, ch);
9fe4bcf4
DD		 298 if (IS_ERR(new_cm_id))
299 return PTR_ERR(new_cm_id);
300
509c07bc
BVA		 301 if (ch->cm_id)
302 ib_destroy_cm_id(ch->cm_id);
303 ch->cm_id = new_cm_id;
304 ch->path.sgid = target->sgid;
305 ch->path.dgid = target->orig_dgid;
306 ch->path.pkey = target->pkey;
307 ch->path.service_id = target->service_id;
9fe4bcf4
DD		 308
309 return 0;
310}
311
d1b4289e
BVA		 312static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target)
313{
314 struct srp_device *dev = target->srp_host->srp_dev;
315 struct ib_fmr_pool_param fmr_param;
316
317 memset(&fmr_param, 0, sizeof(fmr_param));
318 fmr_param.pool_size = target->scsi_host->can_queue;
319 fmr_param.dirty_watermark = fmr_param.pool_size / 4;
320 fmr_param.cache = 1;
52ede08f
BVA		 321 fmr_param.max_pages_per_fmr = dev->max_pages_per_mr;
322 fmr_param.page_shift = ilog2(dev->mr_page_size);
d1b4289e
BVA		 323 fmr_param.access = (IB_ACCESS_LOCAL_WRITE |
324 IB_ACCESS_REMOTE_WRITE |
325 IB_ACCESS_REMOTE_READ);
326
327 return ib_create_fmr_pool(dev->pd, &fmr_param);
328}
329
5cfb1782
BVA		 330/**
331 * srp_destroy_fr_pool() - free the resources owned by a pool
332 * @pool: Fast registration pool to be destroyed.
333 */
334static void srp_destroy_fr_pool(struct srp_fr_pool *pool)
335{
336 int i;
337 struct srp_fr_desc *d;
338
339 if (!pool)
340 return;
341
342 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
343 if (d->frpl)
344 ib_free_fast_reg_page_list(d->frpl);
345 if (d->mr)
346 ib_dereg_mr(d->mr);
347 }
348 kfree(pool);
349}
350
351/**
352 * srp_create_fr_pool() - allocate and initialize a pool for fast registration
353 * @device: IB device to allocate fast registration descriptors for.
354 * @pd: Protection domain associated with the FR descriptors.
355 * @pool_size: Number of descriptors to allocate.
356 * @max_page_list_len: Maximum fast registration work request page list length.
357 */
358static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
359 struct ib_pd *pd, int pool_size,
360 int max_page_list_len)
361{
362 struct srp_fr_pool *pool;
363 struct srp_fr_desc *d;
364 struct ib_mr *mr;
365 struct ib_fast_reg_page_list *frpl;
366 int i, ret = -EINVAL;
367
368 if (pool_size <= 0)
369 goto err;
370 ret = -ENOMEM;
371 pool = kzalloc(sizeof(struct srp_fr_pool) +
372 pool_size * sizeof(struct srp_fr_desc), GFP_KERNEL);
373 if (!pool)
374 goto err;
375 pool->size = pool_size;
376 pool->max_page_list_len = max_page_list_len;
377 spin_lock_init(&pool->lock);
378 INIT_LIST_HEAD(&pool->free_list);
379
380 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
563b67c5
SG		 381 mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG,
382 max_page_list_len);
5cfb1782
BVA		 383 if (IS_ERR(mr)) {
384 ret = PTR_ERR(mr);
385 goto destroy_pool;
386 }
387 d->mr = mr;
388 frpl = ib_alloc_fast_reg_page_list(device, max_page_list_len);
389 if (IS_ERR(frpl)) {
390 ret = PTR_ERR(frpl);
391 goto destroy_pool;
392 }
393 d->frpl = frpl;
394 list_add_tail(&d->entry, &pool->free_list);
395 }
396
397out:
398 return pool;
399
400destroy_pool:
401 srp_destroy_fr_pool(pool);
402
403err:
404 pool = ERR_PTR(ret);
405 goto out;
406}
407
408/**
409 * srp_fr_pool_get() - obtain a descriptor suitable for fast registration
410 * @pool: Pool to obtain descriptor from.
411 */
412static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool)
413{
414 struct srp_fr_desc *d = NULL;
415 unsigned long flags;
416
417 spin_lock_irqsave(&pool->lock, flags);
418 if (!list_empty(&pool->free_list)) {
419 d = list_first_entry(&pool->free_list, typeof(*d), entry);
420 list_del(&d->entry);
421 }
422 spin_unlock_irqrestore(&pool->lock, flags);
423
424 return d;
425}
426
427/**
428 * srp_fr_pool_put() - put an FR descriptor back in the free list
429 * @pool: Pool the descriptor was allocated from.
430 * @desc: Pointer to an array of fast registration descriptor pointers.
431 * @n: Number of descriptors to put back.
432 *
433 * Note: The caller must already have queued an invalidation request for
434 * desc->mr->rkey before calling this function.
435 */
436static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc,
437 int n)
438{
439 unsigned long flags;
440 int i;
441
442 spin_lock_irqsave(&pool->lock, flags);
443 for (i = 0; i < n; i++)
444 list_add(&desc[i]->entry, &pool->free_list);
445 spin_unlock_irqrestore(&pool->lock, flags);
446}
447
448static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
449{
450 struct srp_device *dev = target->srp_host->srp_dev;
451
452 return srp_create_fr_pool(dev->dev, dev->pd,
453 target->scsi_host->can_queue,
454 dev->max_pages_per_mr);
455}
456
7dad6b2e
BVA		 457/**
458 * srp_destroy_qp() - destroy an RDMA queue pair
459 * @ch: SRP RDMA channel.
460 *
461 * Change a queue pair into the error state and wait until all receive
462 * completions have been processed before destroying it. This avoids that
463 * the receive completion handler can access the queue pair while it is
464 * being destroyed.
465 */
466static void srp_destroy_qp(struct srp_rdma_ch *ch)
467{
7dad6b2e
BVA		 468 static struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
469 static struct ib_recv_wr wr = { .wr_id = SRP_LAST_WR_ID };
470 struct ib_recv_wr *bad_wr;
471 int ret;
472
473 /* Destroying a QP and reusing ch->done is only safe if not connected */
c014c8cd 474 WARN_ON_ONCE(ch->connected);
7dad6b2e
BVA		 475
476 ret = ib_modify_qp(ch->qp, &attr, IB_QP_STATE);
477 WARN_ONCE(ret, "ib_cm_init_qp_attr() returned %d\n", ret);
478 if (ret)
479 goto out;
480
481 init_completion(&ch->done);
482 ret = ib_post_recv(ch->qp, &wr, &bad_wr);
483 WARN_ONCE(ret, "ib_post_recv() returned %d\n", ret);
484 if (ret == 0)
485 wait_for_completion(&ch->done);
486
487out:
488 ib_destroy_qp(ch->qp);
489}
490
509c07bc 491static int srp_create_ch_ib(struct srp_rdma_ch *ch)
aef9ec39 492{
509c07bc 493 struct srp_target_port *target = ch->target;
62154b2e 494 struct srp_device *dev = target->srp_host->srp_dev;
aef9ec39 495 struct ib_qp_init_attr *init_attr;
73aa89ed
IR		 496 struct ib_cq *recv_cq, *send_cq;
497 struct ib_qp *qp;
d1b4289e 498 struct ib_fmr_pool *fmr_pool = NULL;
5cfb1782
BVA		 499 struct srp_fr_pool *fr_pool = NULL;
500 const int m = 1 + dev->use_fast_reg;
8e37210b 501 struct ib_cq_init_attr cq_attr = {};
aef9ec39
RD		 502 int ret;
503
504 init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
505 if (!init_attr)
506 return -ENOMEM;
507
7dad6b2e 508 /* + 1 for SRP_LAST_WR_ID */
8e37210b
MB		 509 cq_attr.cqe = target->queue_size + 1;
510 cq_attr.comp_vector = ch->comp_vector;
509c07bc 511 recv_cq = ib_create_cq(dev->dev, srp_recv_completion, NULL, ch,
8e37210b 512 &cq_attr);
73aa89ed
IR		 513 if (IS_ERR(recv_cq)) {
514 ret = PTR_ERR(recv_cq);
da9d2f07 515 goto err;
aef9ec39
RD		 516 }
517
8e37210b
MB		 518 cq_attr.cqe = m * target->queue_size;
519 cq_attr.comp_vector = ch->comp_vector;
509c07bc 520 send_cq = ib_create_cq(dev->dev, srp_send_completion, NULL, ch,
8e37210b 521 &cq_attr);
73aa89ed
IR		 522 if (IS_ERR(send_cq)) {
523 ret = PTR_ERR(send_cq);
da9d2f07 524 goto err_recv_cq;
9c03dc9f
BVA		 525 }
526
73aa89ed 527 ib_req_notify_cq(recv_cq, IB_CQ_NEXT_COMP);
aef9ec39
RD		 528
529 init_attr->event_handler = srp_qp_event;
5cfb1782 530 init_attr->cap.max_send_wr = m * target->queue_size;
7dad6b2e 531 init_attr->cap.max_recv_wr = target->queue_size + 1;
aef9ec39
RD		 532 init_attr->cap.max_recv_sge = 1;
533 init_attr->cap.max_send_sge = 1;
5cfb1782 534 init_attr->sq_sig_type = IB_SIGNAL_REQ_WR;
aef9ec39 535 init_attr->qp_type = IB_QPT_RC;
73aa89ed
IR		 536 init_attr->send_cq = send_cq;
537 init_attr->recv_cq = recv_cq;
aef9ec39 538
62154b2e 539 qp = ib_create_qp(dev->pd, init_attr);
73aa89ed
IR		 540 if (IS_ERR(qp)) {
541 ret = PTR_ERR(qp);
da9d2f07 542 goto err_send_cq;
aef9ec39
RD		 543 }
544
73aa89ed 545 ret = srp_init_qp(target, qp);
da9d2f07
RD		 546 if (ret)
547 goto err_qp;
aef9ec39 548
002f1567 549 if (dev->use_fast_reg) {
5cfb1782
BVA		 550 fr_pool = srp_alloc_fr_pool(target);
551 if (IS_ERR(fr_pool)) {
552 ret = PTR_ERR(fr_pool);
553 shost_printk(KERN_WARNING, target->scsi_host, PFX
554 "FR pool allocation failed (%d)\n", ret);
555 goto err_qp;
556 }
002f1567 557 } else if (dev->use_fmr) {
d1b4289e
BVA		 558 fmr_pool = srp_alloc_fmr_pool(target);
559 if (IS_ERR(fmr_pool)) {
560 ret = PTR_ERR(fmr_pool);
561 shost_printk(KERN_WARNING, target->scsi_host, PFX
562 "FMR pool allocation failed (%d)\n", ret);
563 goto err_qp;
564 }
d1b4289e
BVA		 565 }
566
509c07bc 567 if (ch->qp)
7dad6b2e 568 srp_destroy_qp(ch);
509c07bc
BVA		 569 if (ch->recv_cq)
570 ib_destroy_cq(ch->recv_cq);
571 if (ch->send_cq)
572 ib_destroy_cq(ch->send_cq);
73aa89ed 573
509c07bc
BVA		 574 ch->qp = qp;
575 ch->recv_cq = recv_cq;
576 ch->send_cq = send_cq;
73aa89ed 577
7fbc67df
SG		 578 if (dev->use_fast_reg) {
579 if (ch->fr_pool)
580 srp_destroy_fr_pool(ch->fr_pool);
581 ch->fr_pool = fr_pool;
582 } else if (dev->use_fmr) {
583 if (ch->fmr_pool)
584 ib_destroy_fmr_pool(ch->fmr_pool);
585 ch->fmr_pool = fmr_pool;
586 }
587
da9d2f07
RD		 588 kfree(init_attr);
589 return 0;
590
591err_qp:
73aa89ed 592 ib_destroy_qp(qp);
da9d2f07
RD		 593
594err_send_cq:
73aa89ed 595 ib_destroy_cq(send_cq);
da9d2f07
RD		 596
597err_recv_cq:
73aa89ed 598 ib_destroy_cq(recv_cq);
da9d2f07
RD		 599
600err:
aef9ec39
RD		 601 kfree(init_attr);
602 return ret;
603}
604
4d73f95f
BVA		 605/*
606 * Note: this function may be called without srp_alloc_iu_bufs() having been
509c07bc 607 * invoked. Hence the ch->[rt]x_ring checks.
4d73f95f 608 */
509c07bc
BVA		 609static void srp_free_ch_ib(struct srp_target_port *target,
610 struct srp_rdma_ch *ch)
aef9ec39 611{
5cfb1782 612 struct srp_device *dev = target->srp_host->srp_dev;
aef9ec39
RD		 613 int i;
614
d92c0da7
BVA		 615 if (!ch->target)
616 return;
617
509c07bc
BVA		 618 if (ch->cm_id) {
619 ib_destroy_cm_id(ch->cm_id);
620 ch->cm_id = NULL;
394c595e
BVA		 621 }
622
d92c0da7
BVA		 623 /* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */
624 if (!ch->qp)
625 return;
626
5cfb1782 627 if (dev->use_fast_reg) {
509c07bc
BVA		 628 if (ch->fr_pool)
629 srp_destroy_fr_pool(ch->fr_pool);
002f1567 630 } else if (dev->use_fmr) {
509c07bc
BVA		 631 if (ch->fmr_pool)
632 ib_destroy_fmr_pool(ch->fmr_pool);
5cfb1782 633 }
7dad6b2e 634 srp_destroy_qp(ch);
509c07bc
BVA		 635 ib_destroy_cq(ch->send_cq);
636 ib_destroy_cq(ch->recv_cq);
aef9ec39 637
d92c0da7
BVA		 638 /*
639 * Avoid that the SCSI error handler tries to use this channel after
640 * it has been freed. The SCSI error handler can namely continue
641 * trying to perform recovery actions after scsi_remove_host()
642 * returned.
643 */
644 ch->target = NULL;
645
509c07bc
BVA		 646 ch->qp = NULL;
647 ch->send_cq = ch->recv_cq = NULL;
73aa89ed 648
509c07bc 649 if (ch->rx_ring) {
4d73f95f 650 for (i = 0; i < target->queue_size; ++i)
509c07bc
BVA		 651 srp_free_iu(target->srp_host, ch->rx_ring[i]);
652 kfree(ch->rx_ring);
653 ch->rx_ring = NULL;
4d73f95f 654 }
509c07bc 655 if (ch->tx_ring) {
4d73f95f 656 for (i = 0; i < target->queue_size; ++i)
509c07bc
BVA		 657 srp_free_iu(target->srp_host, ch->tx_ring[i]);
658 kfree(ch->tx_ring);
659 ch->tx_ring = NULL;
4d73f95f 660 }
aef9ec39
RD		 661}
662
663static void srp_path_rec_completion(int status,
664 struct ib_sa_path_rec *pathrec,
509c07bc 665 void *ch_ptr)
aef9ec39 666{
509c07bc
BVA		 667 struct srp_rdma_ch *ch = ch_ptr;
668 struct srp_target_port *target = ch->target;
aef9ec39 669
509c07bc 670 ch->status = status;
aef9ec39 671 if (status)
7aa54bd7
DD		 672 shost_printk(KERN_ERR, target->scsi_host,
673 PFX "Got failed path rec status %d\n", status);
aef9ec39 674 else
509c07bc
BVA		 675 ch->path = *pathrec;
676 complete(&ch->done);
aef9ec39
RD		 677}
678
509c07bc 679static int srp_lookup_path(struct srp_rdma_ch *ch)
aef9ec39 680{
509c07bc 681 struct srp_target_port *target = ch->target;
a702adce
BVA		 682 int ret;
683
509c07bc
BVA		 684 ch->path.numb_path = 1;
685
686 init_completion(&ch->done);
687
688 ch->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
689 target->srp_host->srp_dev->dev,
690 target->srp_host->port,
691 &ch->path,
692 IB_SA_PATH_REC_SERVICE_ID |
693 IB_SA_PATH_REC_DGID |
694 IB_SA_PATH_REC_SGID |
695 IB_SA_PATH_REC_NUMB_PATH |
696 IB_SA_PATH_REC_PKEY,
697 SRP_PATH_REC_TIMEOUT_MS,
698 GFP_KERNEL,
699 srp_path_rec_completion,
700 ch, &ch->path_query);
701 if (ch->path_query_id < 0)
702 return ch->path_query_id;
703
704 ret = wait_for_completion_interruptible(&ch->done);
a702adce
BVA		 705 if (ret < 0)
706 return ret;
aef9ec39 707
509c07bc 708 if (ch->status < 0)
7aa54bd7
DD		 709 shost_printk(KERN_WARNING, target->scsi_host,
710 PFX "Path record query failed\n");
aef9ec39 711
509c07bc 712 return ch->status;
aef9ec39
RD		 713}
714
d92c0da7 715static int srp_send_req(struct srp_rdma_ch *ch, bool multich)
aef9ec39 716{
509c07bc 717 struct srp_target_port *target = ch->target;
aef9ec39
RD		 718 struct {
719 struct ib_cm_req_param param;
720 struct srp_login_req priv;
721 } *req = NULL;
722 int status;
723
724 req = kzalloc(sizeof *req, GFP_KERNEL);
725 if (!req)
726 return -ENOMEM;
727
509c07bc 728 req->param.primary_path = &ch->path;
aef9ec39
RD		 729 req->param.alternate_path = NULL;
730 req->param.service_id = target->service_id;
509c07bc
BVA		 731 req->param.qp_num = ch->qp->qp_num;
732 req->param.qp_type = ch->qp->qp_type;
aef9ec39
RD		 733 req->param.private_data = &req->priv;
734 req->param.private_data_len = sizeof req->priv;
735 req->param.flow_control = 1;
736
737 get_random_bytes(&req->param.starting_psn, 4);
738 req->param.starting_psn &= 0xffffff;
739
740 /*
741 * Pick some arbitrary defaults here; we could make these
742 * module parameters if anyone cared about setting them.
743 */
744 req->param.responder_resources = 4;
745 req->param.remote_cm_response_timeout = 20;
746 req->param.local_cm_response_timeout = 20;
7bb312e4 747 req->param.retry_count = target->tl_retry_count;
aef9ec39
RD		 748 req->param.rnr_retry_count = 7;
749 req->param.max_cm_retries = 15;
750
751 req->priv.opcode = SRP_LOGIN_REQ;
752 req->priv.tag = 0;
49248644 753 req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len);
aef9ec39
RD		 754 req->priv.req_buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
755 SRP_BUF_FORMAT_INDIRECT);
d92c0da7
BVA		 756 req->priv.req_flags = (multich ? SRP_MULTICHAN_MULTI :
757 SRP_MULTICHAN_SINGLE);
0c0450db 758 /*
3cd96564 759 * In the published SRP specification (draft rev. 16a), the
0c0450db
R		 760 * port identifier format is 8 bytes of ID extension followed
761 * by 8 bytes of GUID. Older drafts put the two halves in the
762 * opposite order, so that the GUID comes first.
763 *
764 * Targets conforming to these obsolete drafts can be
765 * recognized by the I/O Class they report.
766 */
767 if (target->io_class == SRP_REV10_IB_IO_CLASS) {
768 memcpy(req->priv.initiator_port_id,
747fe000 769 &target->sgid.global.interface_id, 8);
0c0450db 770 memcpy(req->priv.initiator_port_id + 8,
01cb9bcb 771 &target->initiator_ext, 8);
0c0450db
R		 772 memcpy(req->priv.target_port_id, &target->ioc_guid, 8);
773 memcpy(req->priv.target_port_id + 8, &target->id_ext, 8);
774 } else {
775 memcpy(req->priv.initiator_port_id,
01cb9bcb
IR		 776 &target->initiator_ext, 8);
777 memcpy(req->priv.initiator_port_id + 8,
747fe000 778 &target->sgid.global.interface_id, 8);
0c0450db
R		 779 memcpy(req->priv.target_port_id, &target->id_ext, 8);
780 memcpy(req->priv.target_port_id + 8, &target->ioc_guid, 8);
781 }
782
aef9ec39
RD		 783 /*
784 * Topspin/Cisco SRP targets will reject our login unless we
01cb9bcb
IR		 785 * zero out the first 8 bytes of our initiator port ID and set
786 * the second 8 bytes to the local node GUID.
aef9ec39 787 */
5d7cbfd6 788 if (srp_target_is_topspin(target)) {
7aa54bd7
DD		 789 shost_printk(KERN_DEBUG, target->scsi_host,
790 PFX "Topspin/Cisco initiator port ID workaround "
791 "activated for target GUID %016llx\n",
45c37cad 792 be64_to_cpu(target->ioc_guid));
aef9ec39 793 memset(req->priv.initiator_port_id, 0, 8);
01cb9bcb 794 memcpy(req->priv.initiator_port_id + 8,
05321937 795 &target->srp_host->srp_dev->dev->node_guid, 8);
aef9ec39 796 }
aef9ec39 797
509c07bc 798 status = ib_send_cm_req(ch->cm_id, &req->param);
aef9ec39
RD		 799
800 kfree(req);
801
802 return status;
803}
804
ef6c49d8
BVA		 805static bool srp_queue_remove_work(struct srp_target_port *target)
806{
807 bool changed = false;
808
809 spin_lock_irq(&target->lock);
810 if (target->state != SRP_TARGET_REMOVED) {
811 target->state = SRP_TARGET_REMOVED;
812 changed = true;
813 }
814 spin_unlock_irq(&target->lock);
815
816 if (changed)
bcc05910 817 queue_work(srp_remove_wq, &target->remove_work);
ef6c49d8
BVA		 818
819 return changed;
820}
821
aef9ec39
RD		 822static void srp_disconnect_target(struct srp_target_port *target)
823{
d92c0da7
BVA		 824 struct srp_rdma_ch *ch;
825 int i;
509c07bc 826
c014c8cd 827 /* XXX should send SRP_I_LOGOUT request */
aef9ec39 828
c014c8cd
BVA		 829 for (i = 0; i < target->ch_count; i++) {
830 ch = &target->ch[i];
831 ch->connected = false;
832 if (ch->cm_id && ib_send_cm_dreq(ch->cm_id, NULL, 0)) {
833 shost_printk(KERN_DEBUG, target->scsi_host,
834 PFX "Sending CM DREQ failed\n");
294c875a 835 }
e6581056 836 }
aef9ec39
RD		 837}
838
509c07bc
BVA		 839static void srp_free_req_data(struct srp_target_port *target,
840 struct srp_rdma_ch *ch)
8f26c9ff 841{
5cfb1782
BVA		 842 struct srp_device *dev = target->srp_host->srp_dev;
843 struct ib_device *ibdev = dev->dev;
8f26c9ff
DD		 844 struct srp_request *req;
845 int i;
846
47513cf4 847 if (!ch->req_ring)
4d73f95f
BVA		 848 return;
849
850 for (i = 0; i < target->req_ring_size; ++i) {
509c07bc 851 req = &ch->req_ring[i];
5cfb1782
BVA		 852 if (dev->use_fast_reg)
853 kfree(req->fr_list);
854 else
855 kfree(req->fmr_list);
8f26c9ff 856 kfree(req->map_page);
c07d424d
DD		 857 if (req->indirect_dma_addr) {
858 ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
859 target->indirect_size,
860 DMA_TO_DEVICE);
861 }
862 kfree(req->indirect_desc);
8f26c9ff 863 }
4d73f95f 864
509c07bc
BVA		 865 kfree(ch->req_ring);
866 ch->req_ring = NULL;
8f26c9ff
DD		 867}
868
509c07bc 869static int srp_alloc_req_data(struct srp_rdma_ch *ch)
b81d00bd 870{
509c07bc 871 struct srp_target_port *target = ch->target;
b81d00bd
BVA		 872 struct srp_device *srp_dev = target->srp_host->srp_dev;
873 struct ib_device *ibdev = srp_dev->dev;
874 struct srp_request *req;
5cfb1782 875 void *mr_list;
b81d00bd
BVA		 876 dma_addr_t dma_addr;
877 int i, ret = -ENOMEM;
878
509c07bc
BVA		 879 ch->req_ring = kcalloc(target->req_ring_size, sizeof(*ch->req_ring),
880 GFP_KERNEL);
881 if (!ch->req_ring)
4d73f95f
BVA		 882 goto out;
883
884 for (i = 0; i < target->req_ring_size; ++i) {
509c07bc 885 req = &ch->req_ring[i];
5cfb1782
BVA		 886 mr_list = kmalloc(target->cmd_sg_cnt * sizeof(void *),
887 GFP_KERNEL);
888 if (!mr_list)
889 goto out;
890 if (srp_dev->use_fast_reg)
891 req->fr_list = mr_list;
892 else
893 req->fmr_list = mr_list;
52ede08f 894 req->map_page = kmalloc(srp_dev->max_pages_per_mr *
d1b4289e 895 sizeof(void *), GFP_KERNEL);
5cfb1782
BVA		 896 if (!req->map_page)
897 goto out;
b81d00bd 898 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
5cfb1782 899 if (!req->indirect_desc)
b81d00bd
BVA		 900 goto out;
901
902 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
903 target->indirect_size,
904 DMA_TO_DEVICE);
905 if (ib_dma_mapping_error(ibdev, dma_addr))
906 goto out;
907
908 req->indirect_dma_addr = dma_addr;
b81d00bd
BVA		 909 }
910 ret = 0;
911
912out:
913 return ret;
914}
915
683b159a
BVA		 916/**
917 * srp_del_scsi_host_attr() - Remove attributes defined in the host template.
918 * @shost: SCSI host whose attributes to remove from sysfs.
919 *
920 * Note: Any attributes defined in the host template and that did not exist
921 * before invocation of this function will be ignored.
922 */
923static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
924{
925 struct device_attribute **attr;
926
927 for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr)
928 device_remove_file(&shost->shost_dev, *attr);
929}
930
ee12d6a8
BVA		 931static void srp_remove_target(struct srp_target_port *target)
932{
d92c0da7
BVA		 933 struct srp_rdma_ch *ch;
934 int i;
509c07bc 935
ef6c49d8
BVA		 936 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
937
ee12d6a8 938 srp_del_scsi_host_attr(target->scsi_host);
9dd69a60 939 srp_rport_get(target->rport);
ee12d6a8
BVA		 940 srp_remove_host(target->scsi_host);
941 scsi_remove_host(target->scsi_host);
93079162 942 srp_stop_rport_timers(target->rport);
ef6c49d8 943 srp_disconnect_target(target);
d92c0da7
BVA		 944 for (i = 0; i < target->ch_count; i++) {
945 ch = &target->ch[i];
946 srp_free_ch_ib(target, ch);
947 }
c1120f89 948 cancel_work_sync(&target->tl_err_work);
9dd69a60 949 srp_rport_put(target->rport);
d92c0da7
BVA		 950 for (i = 0; i < target->ch_count; i++) {
951 ch = &target->ch[i];
952 srp_free_req_data(target, ch);
953 }
954 kfree(target->ch);
955 target->ch = NULL;
65d7dd2f
VP		 956
957 spin_lock(&target->srp_host->target_lock);
958 list_del(&target->list);
959 spin_unlock(&target->srp_host->target_lock);
960
ee12d6a8
BVA		 961 scsi_host_put(target->scsi_host);
962}
963
c4028958 964static void srp_remove_work(struct work_struct *work)
aef9ec39 965{
c4028958 966 struct srp_target_port *target =
ef6c49d8 967 container_of(work, struct srp_target_port, remove_work);
aef9ec39 968
ef6c49d8 969 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
aef9ec39 970
96fc248a 971 srp_remove_target(target);
aef9ec39
RD		 972}
973
dc1bdbd9
BVA		 974static void srp_rport_delete(struct srp_rport *rport)
975{
976 struct srp_target_port *target = rport->lld_data;
977
978 srp_queue_remove_work(target);
979}
980
c014c8cd
BVA		 981/**
982 * srp_connected_ch() - number of connected channels
983 * @target: SRP target port.
984 */
985static int srp_connected_ch(struct srp_target_port *target)
986{
987 int i, c = 0;
988
989 for (i = 0; i < target->ch_count; i++)
990 c += target->ch[i].connected;
991
992 return c;
993}
994
d92c0da7 995static int srp_connect_ch(struct srp_rdma_ch *ch, bool multich)
aef9ec39 996{
509c07bc 997 struct srp_target_port *target = ch->target;
aef9ec39
RD		 998 int ret;
999
c014c8cd 1000 WARN_ON_ONCE(!multich && srp_connected_ch(target) > 0);
294c875a 1001
509c07bc 1002 ret = srp_lookup_path(ch);
aef9ec39
RD		 1003 if (ret)
1004 return ret;
1005
1006 while (1) {
509c07bc 1007 init_completion(&ch->done);
d92c0da7 1008 ret = srp_send_req(ch, multich);
aef9ec39
RD		 1009 if (ret)
1010 return ret;
509c07bc 1011 ret = wait_for_completion_interruptible(&ch->done);
a702adce
BVA		 1012 if (ret < 0)
1013 return ret;
aef9ec39
RD		 1014
1015 /*
1016 * The CM event handling code will set status to
1017 * SRP_PORT_REDIRECT if we get a port redirect REJ
1018 * back, or SRP_DLID_REDIRECT if we get a lid/qp
1019 * redirect REJ back.
1020 */
509c07bc 1021 switch (ch->status) {
aef9ec39 1022 case 0:
c014c8cd 1023 ch->connected = true;
aef9ec39
RD		 1024 return 0;
1025
1026 case SRP_PORT_REDIRECT:
509c07bc 1027 ret = srp_lookup_path(ch);
aef9ec39
RD		 1028 if (ret)
1029 return ret;
1030 break;
1031
1032 case SRP_DLID_REDIRECT:
1033 break;
1034
9fe4bcf4 1035 case SRP_STALE_CONN:
9fe4bcf4 1036 shost_printk(KERN_ERR, target->scsi_host, PFX
205619f2 1037 "giving up on stale connection\n");
509c07bc
BVA		 1038 ch->status = -ECONNRESET;
1039 return ch->status;
9fe4bcf4 1040
aef9ec39 1041 default:
509c07bc 1042 return ch->status;
aef9ec39
RD		 1043 }
1044 }
1045}
1046
509c07bc 1047static int srp_inv_rkey(struct srp_rdma_ch *ch, u32 rkey)
5cfb1782
BVA		 1048{
1049 struct ib_send_wr *bad_wr;
1050 struct ib_send_wr wr = {
1051 .opcode = IB_WR_LOCAL_INV,
1052 .wr_id = LOCAL_INV_WR_ID_MASK,
1053 .next = NULL,
1054 .num_sge = 0,
1055 .send_flags = 0,
1056 .ex.invalidate_rkey = rkey,
1057 };
1058
509c07bc 1059 return ib_post_send(ch->qp, &wr, &bad_wr);
5cfb1782
BVA		 1060}
1061
d945e1df 1062static void srp_unmap_data(struct scsi_cmnd *scmnd,
509c07bc 1063 struct srp_rdma_ch *ch,
d945e1df
RD		 1064 struct srp_request *req)
1065{
509c07bc 1066 struct srp_target_port *target = ch->target;
5cfb1782
BVA		 1067 struct srp_device *dev = target->srp_host->srp_dev;
1068 struct ib_device *ibdev = dev->dev;
1069 int i, res;
8f26c9ff 1070
bb350d1d 1071 if (!scsi_sglist(scmnd) ||
d945e1df
RD		 1072 (scmnd->sc_data_direction != DMA_TO_DEVICE &&
1073 scmnd->sc_data_direction != DMA_FROM_DEVICE))
1074 return;
1075
5cfb1782
BVA		 1076 if (dev->use_fast_reg) {
1077 struct srp_fr_desc **pfr;
1078
1079 for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) {
509c07bc 1080 res = srp_inv_rkey(ch, (*pfr)->mr->rkey);
5cfb1782
BVA		 1081 if (res < 0) {
1082 shost_printk(KERN_ERR, target->scsi_host, PFX
1083 "Queueing INV WR for rkey %#x failed (%d)\n",
1084 (*pfr)->mr->rkey, res);
1085 queue_work(system_long_wq,
1086 &target->tl_err_work);
1087 }
1088 }
1089 if (req->nmdesc)
509c07bc 1090 srp_fr_pool_put(ch->fr_pool, req->fr_list,
5cfb1782 1091 req->nmdesc);
002f1567 1092 } else if (dev->use_fmr) {
5cfb1782
BVA		 1093 struct ib_pool_fmr **pfmr;
1094
1095 for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++)
1096 ib_fmr_pool_unmap(*pfmr);
1097 }
f5358a17 1098
8f26c9ff
DD		 1099 ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
1100 scmnd->sc_data_direction);
d945e1df
RD		 1101}
1102
22032991
BVA		 1103/**
1104 * srp_claim_req - Take ownership of the scmnd associated with a request.
509c07bc 1105 * @ch: SRP RDMA channel.
22032991 1106 * @req: SRP request.
b3fe628d 1107 * @sdev: If not NULL, only take ownership for this SCSI device.
22032991
BVA		 1108 * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take
1109 * ownership of @req->scmnd if it equals @scmnd.
1110 *
1111 * Return value:
1112 * Either NULL or a pointer to the SCSI command the caller became owner of.
1113 */
509c07bc 1114static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch,
22032991 1115 struct srp_request *req,
b3fe628d 1116 struct scsi_device *sdev,
22032991
BVA		 1117 struct scsi_cmnd *scmnd)
1118{
1119 unsigned long flags;
1120
509c07bc 1121 spin_lock_irqsave(&ch->lock, flags);
b3fe628d
BVA		 1122 if (req->scmnd &&
1123 (!sdev || req->scmnd->device == sdev) &&
1124 (!scmnd || req->scmnd == scmnd)) {
22032991
BVA		 1125 scmnd = req->scmnd;
1126 req->scmnd = NULL;
22032991
BVA		 1127 } else {
1128 scmnd = NULL;
1129 }
509c07bc 1130 spin_unlock_irqrestore(&ch->lock, flags);
22032991
BVA		 1131
1132 return scmnd;
1133}
1134
1135/**
1136 * srp_free_req() - Unmap data and add request to the free request list.
509c07bc 1137 * @ch: SRP RDMA channel.
af24663b
BVA		 1138 * @req: Request to be freed.
1139 * @scmnd: SCSI command associated with @req.
1140 * @req_lim_delta: Amount to be added to @target->req_lim.
22032991 1141 */
509c07bc
BVA		 1142static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req,
1143 struct scsi_cmnd *scmnd, s32 req_lim_delta)
526b4caa 1144{
94a9174c
BVA		 1145 unsigned long flags;
1146
509c07bc 1147 srp_unmap_data(scmnd, ch, req);
22032991 1148
509c07bc
BVA		 1149 spin_lock_irqsave(&ch->lock, flags);
1150 ch->req_lim += req_lim_delta;
509c07bc 1151 spin_unlock_irqrestore(&ch->lock, flags);
526b4caa
IR		 1152}
1153
509c07bc
BVA		 1154static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req,
1155 struct scsi_device *sdev, int result)
526b4caa 1156{
509c07bc 1157 struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL);
22032991
BVA		 1158
1159 if (scmnd) {
509c07bc 1160 srp_free_req(ch, req, scmnd, 0);
ed9b2264 1161 scmnd->result = result;
22032991 1162 scmnd->scsi_done(scmnd);
22032991 1163 }
526b4caa
IR		 1164}
1165
ed9b2264 1166static void srp_terminate_io(struct srp_rport *rport)
aef9ec39 1167{
ed9b2264 1168 struct srp_target_port *target = rport->lld_data;
d92c0da7 1169 struct srp_rdma_ch *ch;
b3fe628d
BVA		 1170 struct Scsi_Host *shost = target->scsi_host;
1171 struct scsi_device *sdev;
d92c0da7 1172 int i, j;
ed9b2264 1173
b3fe628d
BVA		 1174 /*
1175 * Invoking srp_terminate_io() while srp_queuecommand() is running
1176 * is not safe. Hence the warning statement below.
1177 */
1178 shost_for_each_device(sdev, shost)
1179 WARN_ON_ONCE(sdev->request_queue->request_fn_active);
1180
d92c0da7
BVA		 1181 for (i = 0; i < target->ch_count; i++) {
1182 ch = &target->ch[i];
509c07bc 1183
d92c0da7
BVA		 1184 for (j = 0; j < target->req_ring_size; ++j) {
1185 struct srp_request *req = &ch->req_ring[j];
1186
1187 srp_finish_req(ch, req, NULL,
1188 DID_TRANSPORT_FAILFAST << 16);
1189 }
ed9b2264
BVA		 1190 }
1191}
aef9ec39 1192
ed9b2264
BVA		 1193/*
1194 * It is up to the caller to ensure that srp_rport_reconnect() calls are
1195 * serialized and that no concurrent srp_queuecommand(), srp_abort(),
1196 * srp_reset_device() or srp_reset_host() calls will occur while this function
1197 * is in progress. One way to realize that is not to call this function
1198 * directly but to call srp_reconnect_rport() instead since that last function
1199 * serializes calls of this function via rport->mutex and also blocks
1200 * srp_queuecommand() calls before invoking this function.
1201 */
1202static int srp_rport_reconnect(struct srp_rport *rport)
1203{
1204 struct srp_target_port *target = rport->lld_data;
d92c0da7
BVA		 1205 struct srp_rdma_ch *ch;
1206 int i, j, ret = 0;
1207 bool multich = false;
09be70a2 1208
aef9ec39 1209 srp_disconnect_target(target);
34aa654e
BVA		 1210
1211 if (target->state == SRP_TARGET_SCANNING)
1212 return -ENODEV;
1213
aef9ec39 1214 /*
c7c4e7ff
BVA		 1215 * Now get a new local CM ID so that we avoid confusing the target in
1216 * case things are really fouled up. Doing so also ensures that all CM
1217 * callbacks will have finished before a new QP is allocated.
aef9ec39 1218 */
d92c0da7
BVA		 1219 for (i = 0; i < target->ch_count; i++) {
1220 ch = &target->ch[i];
d92c0da7 1221 ret += srp_new_cm_id(ch);
536ae14e 1222 }
d92c0da7
BVA		 1223 for (i = 0; i < target->ch_count; i++) {
1224 ch = &target->ch[i];
d92c0da7
BVA		 1225 for (j = 0; j < target->req_ring_size; ++j) {
1226 struct srp_request *req = &ch->req_ring[j];
aef9ec39 1227
d92c0da7
BVA		 1228 srp_finish_req(ch, req, NULL, DID_RESET << 16);
1229 }
1230 }
1231 for (i = 0; i < target->ch_count; i++) {
1232 ch = &target->ch[i];
d92c0da7
BVA		 1233 /*
1234 * Whether or not creating a new CM ID succeeded, create a new
1235 * QP. This guarantees that all completion callback function
1236 * invocations have finished before request resetting starts.
1237 */
1238 ret += srp_create_ch_ib(ch);
aef9ec39 1239
d92c0da7
BVA		 1240 INIT_LIST_HEAD(&ch->free_tx);
1241 for (j = 0; j < target->queue_size; ++j)
1242 list_add(&ch->tx_ring[j]->list, &ch->free_tx);
1243 }
8de9fe3a
BVA		 1244
1245 target->qp_in_error = false;
1246
d92c0da7
BVA		 1247 for (i = 0; i < target->ch_count; i++) {
1248 ch = &target->ch[i];
bbac5ccf 1249 if (ret)
d92c0da7 1250 break;
d92c0da7
BVA		 1251 ret = srp_connect_ch(ch, multich);
1252 multich = true;
1253 }
09be70a2 1254
ed9b2264
BVA		 1255 if (ret == 0)
1256 shost_printk(KERN_INFO, target->scsi_host,
1257 PFX "reconnect succeeded\n");
aef9ec39
RD		 1258
1259 return ret;
1260}
1261
8f26c9ff
DD		 1262static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
1263 unsigned int dma_len, u32 rkey)
f5358a17 1264{
8f26c9ff 1265 struct srp_direct_buf *desc = state->desc;
f5358a17 1266
3ae95da8
BVA		 1267 WARN_ON_ONCE(!dma_len);
1268
8f26c9ff
DD		 1269 desc->va = cpu_to_be64(dma_addr);
1270 desc->key = cpu_to_be32(rkey);
1271 desc->len = cpu_to_be32(dma_len);
f5358a17 1272
8f26c9ff
DD		 1273 state->total_len += dma_len;
1274 state->desc++;
1275 state->ndesc++;
1276}
559ce8f1 1277
8f26c9ff 1278static int srp_map_finish_fmr(struct srp_map_state *state,
509c07bc 1279 struct srp_rdma_ch *ch)
8f26c9ff 1280{
186fbc66
BVA		 1281 struct srp_target_port *target = ch->target;
1282 struct srp_device *dev = target->srp_host->srp_dev;
8f26c9ff
DD		 1283 struct ib_pool_fmr *fmr;
1284 u64 io_addr = 0;
85507bcc 1285
f731ed62
BVA		 1286 if (state->fmr.next >= state->fmr.end)
1287 return -ENOMEM;
1288
509c07bc 1289 fmr = ib_fmr_pool_map_phys(ch->fmr_pool, state->pages,
8f26c9ff
DD		 1290 state->npages, io_addr);
1291 if (IS_ERR(fmr))
1292 return PTR_ERR(fmr);
f5358a17 1293
f731ed62 1294 *state->fmr.next++ = fmr;
52ede08f 1295 state->nmdesc++;
f5358a17 1296
186fbc66
BVA		 1297 srp_map_desc(state, state->base_dma_addr & ~dev->mr_page_mask,
1298 state->dma_len, fmr->fmr->rkey);
539dde6f 1299
8f26c9ff
DD		 1300 return 0;
1301}
1302
5cfb1782 1303static int srp_map_finish_fr(struct srp_map_state *state,
509c07bc 1304 struct srp_rdma_ch *ch)
5cfb1782 1305{
509c07bc 1306 struct srp_target_port *target = ch->target;
5cfb1782
BVA		 1307 struct srp_device *dev = target->srp_host->srp_dev;
1308 struct ib_send_wr *bad_wr;
1309 struct ib_send_wr wr;
1310 struct srp_fr_desc *desc;
1311 u32 rkey;
1312
f731ed62
BVA		 1313 if (state->fr.next >= state->fr.end)
1314 return -ENOMEM;
1315
509c07bc 1316 desc = srp_fr_pool_get(ch->fr_pool);
5cfb1782
BVA		 1317 if (!desc)
1318 return -ENOMEM;
1319
1320 rkey = ib_inc_rkey(desc->mr->rkey);
1321 ib_update_fast_reg_key(desc->mr, rkey);
1322
1323 memcpy(desc->frpl->page_list, state->pages,
1324 sizeof(state->pages[0]) * state->npages);
1325
1326 memset(&wr, 0, sizeof(wr));
1327 wr.opcode = IB_WR_FAST_REG_MR;
1328 wr.wr_id = FAST_REG_WR_ID_MASK;
1329 wr.wr.fast_reg.iova_start = state->base_dma_addr;
1330 wr.wr.fast_reg.page_list = desc->frpl;
1331 wr.wr.fast_reg.page_list_len = state->npages;
1332 wr.wr.fast_reg.page_shift = ilog2(dev->mr_page_size);
1333 wr.wr.fast_reg.length = state->dma_len;
1334 wr.wr.fast_reg.access_flags = (IB_ACCESS_LOCAL_WRITE |
1335 IB_ACCESS_REMOTE_READ |
1336 IB_ACCESS_REMOTE_WRITE);
1337 wr.wr.fast_reg.rkey = desc->mr->lkey;
1338
f731ed62 1339 *state->fr.next++ = desc;
5cfb1782
BVA		 1340 state->nmdesc++;
1341
1342 srp_map_desc(state, state->base_dma_addr, state->dma_len,
1343 desc->mr->rkey);
1344
509c07bc 1345 return ib_post_send(ch->qp, &wr, &bad_wr);
5cfb1782
BVA		 1346}
1347
539dde6f 1348static int srp_finish_mapping(struct srp_map_state *state,
509c07bc 1349 struct srp_rdma_ch *ch)
539dde6f 1350{
509c07bc 1351 struct srp_target_port *target = ch->target;
002f1567 1352 struct srp_device *dev = target->srp_host->srp_dev;
539dde6f
BVA		 1353 int ret = 0;
1354
002f1567
BVA		 1355 WARN_ON_ONCE(!dev->use_fast_reg && !dev->use_fmr);
1356
539dde6f
BVA		 1357 if (state->npages == 0)
1358 return 0;
1359
03f6fb93 1360 if (state->npages == 1 && target->global_mr)
52ede08f 1361 srp_map_desc(state, state->base_dma_addr, state->dma_len,
03f6fb93 1362 target->global_mr->rkey);
539dde6f 1363 else
002f1567 1364 ret = dev->use_fast_reg ? srp_map_finish_fr(state, ch) :
509c07bc 1365 srp_map_finish_fmr(state, ch);
539dde6f
BVA		 1366
1367 if (ret == 0) {
1368 state->npages = 0;
52ede08f 1369 state->dma_len = 0;
539dde6f
BVA		 1370 }
1371
1372 return ret;
1373}
1374
8f26c9ff 1375static int srp_map_sg_entry(struct srp_map_state *state,
509c07bc 1376 struct srp_rdma_ch *ch,
3ae95da8 1377 struct scatterlist *sg, int sg_index)
8f26c9ff 1378{
509c07bc 1379 struct srp_target_port *target = ch->target;
8f26c9ff
DD		 1380 struct srp_device *dev = target->srp_host->srp_dev;
1381 struct ib_device *ibdev = dev->dev;
1382 dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg);
1383 unsigned int dma_len = ib_sg_dma_len(ibdev, sg);
3ae95da8 1384 unsigned int len = 0;
8f26c9ff
DD		 1385 int ret;
1386
3ae95da8 1387 WARN_ON_ONCE(!dma_len);
f5358a17 1388
8f26c9ff 1389 while (dma_len) {
5cfb1782
BVA		 1390 unsigned offset = dma_addr & ~dev->mr_page_mask;
1391 if (state->npages == dev->max_pages_per_mr || offset != 0) {
509c07bc 1392 ret = srp_finish_mapping(state, ch);
8f26c9ff
DD		 1393 if (ret)
1394 return ret;
8f26c9ff
DD		 1395 }
1396
5cfb1782 1397 len = min_t(unsigned int, dma_len, dev->mr_page_size - offset);
f5358a17 1398
8f26c9ff
DD		 1399 if (!state->npages)
1400 state->base_dma_addr = dma_addr;
5cfb1782 1401 state->pages[state->npages++] = dma_addr & dev->mr_page_mask;
52ede08f 1402 state->dma_len += len;
8f26c9ff
DD		 1403 dma_addr += len;
1404 dma_len -= len;
1405 }
1406
5cfb1782
BVA		 1407 /*
1408 * If the last entry of the MR wasn't a full page, then we need to
8f26c9ff
DD		 1409 * close it out and start a new one -- we can only merge at page
1410 * boundries.
1411 */
1412 ret = 0;
0e0d3a48 1413 if (len != dev->mr_page_size)
509c07bc 1414 ret = srp_finish_mapping(state, ch);
f5358a17
RD		 1415 return ret;
1416}
1417
509c07bc
BVA		 1418static int srp_map_sg(struct srp_map_state *state, struct srp_rdma_ch *ch,
1419 struct srp_request *req, struct scatterlist *scat,
1420 int count)
76bc1e1d 1421{
509c07bc 1422 struct srp_target_port *target = ch->target;
76bc1e1d 1423 struct srp_device *dev = target->srp_host->srp_dev;
76bc1e1d 1424 struct scatterlist *sg;
0e0d3a48 1425 int i, ret;
76bc1e1d
BVA		 1426
1427 state->desc = req->indirect_desc;
1428 state->pages = req->map_page;
5cfb1782 1429 if (dev->use_fast_reg) {
f731ed62
BVA		 1430 state->fr.next = req->fr_list;
1431 state->fr.end = req->fr_list + target->cmd_sg_cnt;
002f1567 1432 } else if (dev->use_fmr) {
f731ed62
BVA		 1433 state->fmr.next = req->fmr_list;
1434 state->fmr.end = req->fmr_list + target->cmd_sg_cnt;
5cfb1782 1435 }
76bc1e1d 1436
002f1567 1437 if (dev->use_fast_reg || dev->use_fmr) {
3ae95da8
BVA		 1438 for_each_sg(scat, sg, count, i) {
1439 ret = srp_map_sg_entry(state, ch, sg, i);
1440 if (ret)
1441 goto out;
1442 }
0e0d3a48
BVA		 1443 ret = srp_finish_mapping(state, ch);
1444 if (ret)
1445 goto out;
3ae95da8
BVA		 1446 } else {
1447 for_each_sg(scat, sg, count, i) {
1448 srp_map_desc(state, ib_sg_dma_address(dev->dev, sg),
03f6fb93
BVA		 1449 ib_sg_dma_len(dev->dev, sg),
1450 target->global_mr->rkey);
3ae95da8 1451 }
0e0d3a48 1452 }
76bc1e1d 1453
52ede08f 1454 req->nmdesc = state->nmdesc;
0e0d3a48 1455 ret = 0;
5cfb1782 1456
0e0d3a48
BVA		 1457out:
1458 return ret;
76bc1e1d
BVA		 1459}
1460
330179f2
BVA		 1461/*
1462 * Register the indirect data buffer descriptor with the HCA.
1463 *
1464 * Note: since the indirect data buffer descriptor has been allocated with
1465 * kmalloc() it is guaranteed that this buffer is a physically contiguous
1466 * memory buffer.
1467 */
1468static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req,
1469 void **next_mr, void **end_mr, u32 idb_len,
1470 __be32 *idb_rkey)
1471{
1472 struct srp_target_port *target = ch->target;
1473 struct srp_device *dev = target->srp_host->srp_dev;
1474 struct srp_map_state state;
1475 struct srp_direct_buf idb_desc;
1476 u64 idb_pages[1];
1477 int ret;
1478
1479 memset(&state, 0, sizeof(state));
1480 memset(&idb_desc, 0, sizeof(idb_desc));
1481 state.gen.next = next_mr;
1482 state.gen.end = end_mr;
1483 state.desc = &idb_desc;
1484 state.pages = idb_pages;
1485 state.pages[0] = (req->indirect_dma_addr &
1486 dev->mr_page_mask);
1487 state.npages = 1;
1488 state.base_dma_addr = req->indirect_dma_addr;
1489 state.dma_len = idb_len;
1490 ret = srp_finish_mapping(&state, ch);
1491 if (ret < 0)
1492 goto out;
1493
1494 *idb_rkey = idb_desc.key;
1495
1496out:
1497 return ret;
1498}
1499
509c07bc 1500static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
aef9ec39
RD		 1501 struct srp_request *req)
1502{
509c07bc 1503 struct srp_target_port *target = ch->target;
76bc1e1d 1504 struct scatterlist *scat;
aef9ec39 1505 struct srp_cmd *cmd = req->cmd->buf;
330179f2 1506 int len, nents, count, ret;
85507bcc
RC		 1507 struct srp_device *dev;
1508 struct ib_device *ibdev;
8f26c9ff
DD		 1509 struct srp_map_state state;
1510 struct srp_indirect_buf *indirect_hdr;
330179f2
BVA		 1511 u32 idb_len, table_len;
1512 __be32 idb_rkey;
8f26c9ff 1513 u8 fmt;
aef9ec39 1514
bb350d1d 1515 if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE)
aef9ec39
RD		 1516 return sizeof (struct srp_cmd);
1517
1518 if (scmnd->sc_data_direction != DMA_FROM_DEVICE &&
1519 scmnd->sc_data_direction != DMA_TO_DEVICE) {
7aa54bd7
DD		 1520 shost_printk(KERN_WARNING, target->scsi_host,
1521 PFX "Unhandled data direction %d\n",
1522 scmnd->sc_data_direction);
aef9ec39
RD		 1523 return -EINVAL;
1524 }
1525
bb350d1d
FT		 1526 nents = scsi_sg_count(scmnd);
1527 scat = scsi_sglist(scmnd);
aef9ec39 1528
05321937 1529 dev = target->srp_host->srp_dev;
85507bcc
RC		 1530 ibdev = dev->dev;
1531
1532 count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction);
8f26c9ff
DD		 1533 if (unlikely(count == 0))
1534 return -EIO;
f5358a17
RD		 1535
1536 fmt = SRP_DATA_DESC_DIRECT;
1537 len = sizeof (struct srp_cmd) + sizeof (struct srp_direct_buf);
aef9ec39 1538
03f6fb93 1539 if (count == 1 && target->global_mr) {
f5358a17
RD		 1540 /*
1541 * The midlayer only generated a single gather/scatter
1542 * entry, or DMA mapping coalesced everything to a
1543 * single entry. So a direct descriptor along with
1544 * the DMA MR suffices.
1545 */
cf368713 1546 struct srp_direct_buf *buf = (void *) cmd->add_data;
aef9ec39 1547
85507bcc 1548 buf->va = cpu_to_be64(ib_sg_dma_address(ibdev, scat));
03f6fb93 1549 buf->key = cpu_to_be32(target->global_mr->rkey);
85507bcc 1550 buf->len = cpu_to_be32(ib_sg_dma_len(ibdev, scat));
8f26c9ff 1551
52ede08f 1552 req->nmdesc = 0;
8f26c9ff
DD		 1553 goto map_complete;
1554 }
1555
5cfb1782
BVA		 1556 /*
1557 * We have more than one scatter/gather entry, so build our indirect
1558 * descriptor table, trying to merge as many entries as we can.
8f26c9ff
DD		 1559 */
1560 indirect_hdr = (void *) cmd->add_data;
1561
c07d424d
DD		 1562 ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr,
1563 target->indirect_size, DMA_TO_DEVICE);
1564
8f26c9ff 1565 memset(&state, 0, sizeof(state));
509c07bc 1566 srp_map_sg(&state, ch, req, scat, count);
cf368713 1567
c07d424d
DD		 1568 /* We've mapped the request, now pull as much of the indirect
1569 * descriptor table as we can into the command buffer. If this
1570 * target is not using an external indirect table, we are
1571 * guaranteed to fit into the command, as the SCSI layer won't
1572 * give us more S/G entries than we allow.
8f26c9ff 1573 */
8f26c9ff 1574 if (state.ndesc == 1) {
5cfb1782
BVA		 1575 /*
1576 * Memory registration collapsed the sg-list into one entry,
8f26c9ff
DD		 1577 * so use a direct descriptor.
1578 */
1579 struct srp_direct_buf *buf = (void *) cmd->add_data;
cf368713 1580
c07d424d 1581 *buf = req->indirect_desc[0];
8f26c9ff 1582 goto map_complete;
aef9ec39
RD		 1583 }
1584
c07d424d
DD		 1585 if (unlikely(target->cmd_sg_cnt < state.ndesc &&
1586 !target->allow_ext_sg)) {
1587 shost_printk(KERN_ERR, target->scsi_host,
1588 "Could not fit S/G list into SRP_CMD\n");
1589 return -EIO;
1590 }
1591
1592 count = min(state.ndesc, target->cmd_sg_cnt);
8f26c9ff 1593 table_len = state.ndesc * sizeof (struct srp_direct_buf);
330179f2 1594 idb_len = sizeof(struct srp_indirect_buf) + table_len;
8f26c9ff
DD		 1595
1596 fmt = SRP_DATA_DESC_INDIRECT;
1597 len = sizeof(struct srp_cmd) + sizeof (struct srp_indirect_buf);
c07d424d 1598 len += count * sizeof (struct srp_direct_buf);
8f26c9ff 1599
c07d424d
DD		 1600 memcpy(indirect_hdr->desc_list, req->indirect_desc,
1601 count * sizeof (struct srp_direct_buf));
8f26c9ff 1602
03f6fb93 1603 if (!target->global_mr) {
330179f2
BVA		 1604 ret = srp_map_idb(ch, req, state.gen.next, state.gen.end,
1605 idb_len, &idb_rkey);
1606 if (ret < 0)
1607 return ret;
1608 req->nmdesc++;
1609 } else {
03f6fb93 1610 idb_rkey = target->global_mr->rkey;
330179f2
BVA		 1611 }
1612
c07d424d 1613 indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr);
330179f2 1614 indirect_hdr->table_desc.key = idb_rkey;
8f26c9ff
DD		 1615 indirect_hdr->table_desc.len = cpu_to_be32(table_len);
1616 indirect_hdr->len = cpu_to_be32(state.total_len);
1617
1618 if (scmnd->sc_data_direction == DMA_TO_DEVICE)
c07d424d 1619 cmd->data_out_desc_cnt = count;
8f26c9ff 1620 else
c07d424d
DD		 1621 cmd->data_in_desc_cnt = count;
1622
1623 ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len,
1624 DMA_TO_DEVICE);
8f26c9ff
DD		 1625
1626map_complete:
aef9ec39
RD		 1627 if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1628 cmd->buf_fmt = fmt << 4;
1629 else
1630 cmd->buf_fmt = fmt;
1631
aef9ec39
RD		 1632 return len;
1633}
1634
76c75b25
BVA		 1635/*
1636 * Return an IU and possible credit to the free pool
1637 */
509c07bc 1638static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu,
76c75b25
BVA		 1639 enum srp_iu_type iu_type)
1640{
1641 unsigned long flags;
1642
509c07bc
BVA		 1643 spin_lock_irqsave(&ch->lock, flags);
1644 list_add(&iu->list, &ch->free_tx);
76c75b25 1645 if (iu_type != SRP_IU_RSP)
509c07bc
BVA		 1646 ++ch->req_lim;
1647 spin_unlock_irqrestore(&ch->lock, flags);
76c75b25
BVA		 1648}
1649
05a1d750 1650/*
509c07bc 1651 * Must be called with ch->lock held to protect req_lim and free_tx.
e9684678 1652 * If IU is not sent, it must be returned using srp_put_tx_iu().
05a1d750
DD		 1653 *
1654 * Note:
1655 * An upper limit for the number of allocated information units for each
1656 * request type is:
1657 * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues
1658 * more than Scsi_Host.can_queue requests.
1659 * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE.
1660 * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than
1661 * one unanswered SRP request to an initiator.
1662 */
509c07bc 1663static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch,
05a1d750
DD		 1664 enum srp_iu_type iu_type)
1665{
509c07bc 1666 struct srp_target_port *target = ch->target;
05a1d750
DD		 1667 s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
1668 struct srp_iu *iu;
1669
509c07bc 1670 srp_send_completion(ch->send_cq, ch);
05a1d750 1671
509c07bc 1672 if (list_empty(&ch->free_tx))
05a1d750
DD		 1673 return NULL;
1674
1675 /* Initiator responses to target requests do not consume credits */
76c75b25 1676 if (iu_type != SRP_IU_RSP) {
509c07bc 1677 if (ch->req_lim <= rsv) {
76c75b25
BVA		 1678 ++target->zero_req_lim;
1679 return NULL;
1680 }
1681
509c07bc 1682 --ch->req_lim;
05a1d750
DD		 1683 }
1684
509c07bc 1685 iu = list_first_entry(&ch->free_tx, struct srp_iu, list);
76c75b25 1686 list_del(&iu->list);
05a1d750
DD		 1687 return iu;
1688}
1689
509c07bc 1690static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len)
05a1d750 1691{
509c07bc 1692 struct srp_target_port *target = ch->target;
05a1d750
DD		 1693 struct ib_sge list;
1694 struct ib_send_wr wr, *bad_wr;
05a1d750
DD		 1695
1696 list.addr = iu->dma;
1697 list.length = len;
9af76271 1698 list.lkey = target->lkey;
05a1d750
DD		 1699
1700 wr.next = NULL;
dcb4cb85 1701 wr.wr_id = (uintptr_t) iu;
05a1d750
DD		 1702 wr.sg_list = &list;
1703 wr.num_sge = 1;
1704 wr.opcode = IB_WR_SEND;
1705 wr.send_flags = IB_SEND_SIGNALED;
1706
509c07bc 1707 return ib_post_send(ch->qp, &wr, &bad_wr);
05a1d750
DD		 1708}
1709
509c07bc 1710static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu)
c996bb47 1711{
509c07bc 1712 struct srp_target_port *target = ch->target;
c996bb47 1713 struct ib_recv_wr wr, *bad_wr;
dcb4cb85 1714 struct ib_sge list;
c996bb47
BVA		 1715
1716 list.addr = iu->dma;
1717 list.length = iu->size;
9af76271 1718 list.lkey = target->lkey;
c996bb47
BVA		 1719
1720 wr.next = NULL;
dcb4cb85 1721 wr.wr_id = (uintptr_t) iu;
c996bb47
BVA		 1722 wr.sg_list = &list;
1723 wr.num_sge = 1;
1724
509c07bc 1725 return ib_post_recv(ch->qp, &wr, &bad_wr);
c996bb47
BVA		 1726}
1727
509c07bc 1728static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp)
aef9ec39 1729{
509c07bc 1730 struct srp_target_port *target = ch->target;
aef9ec39
RD		 1731 struct srp_request *req;
1732 struct scsi_cmnd *scmnd;
1733 unsigned long flags;
aef9ec39 1734
aef9ec39 1735 if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) {
509c07bc
BVA		 1736 spin_lock_irqsave(&ch->lock, flags);
1737 ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1738 spin_unlock_irqrestore(&ch->lock, flags);
94a9174c 1739
509c07bc 1740 ch->tsk_mgmt_status = -1;
f8b6e31e 1741 if (be32_to_cpu(rsp->resp_data_len) >= 4)
509c07bc
BVA		 1742 ch->tsk_mgmt_status = rsp->data[3];
1743 complete(&ch->tsk_mgmt_done);
aef9ec39 1744 } else {
77f2c1a4
BVA		 1745 scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag);
1746 if (scmnd) {
1747 req = (void *)scmnd->host_scribble;
1748 scmnd = srp_claim_req(ch, req, NULL, scmnd);
1749 }
22032991 1750 if (!scmnd) {
7aa54bd7 1751 shost_printk(KERN_ERR, target->scsi_host,
d92c0da7
BVA		 1752 "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n",
1753 rsp->tag, ch - target->ch, ch->qp->qp_num);
22032991 1754
509c07bc
BVA		 1755 spin_lock_irqsave(&ch->lock, flags);
1756 ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1757 spin_unlock_irqrestore(&ch->lock, flags);
22032991
BVA		 1758
1759 return;
1760 }
aef9ec39
RD		 1761 scmnd->result = rsp->status;
1762
1763 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) {
1764 memcpy(scmnd->sense_buffer, rsp->data +
1765 be32_to_cpu(rsp->resp_data_len),
1766 min_t(int, be32_to_cpu(rsp->sense_data_len),
1767 SCSI_SENSE_BUFFERSIZE));
1768 }
1769
e714531a 1770 if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER))
bb350d1d 1771 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt));
e714531a
BVA		 1772 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER))
1773 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt));
1774 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER))
1775 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt));
1776 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER))
1777 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt));
aef9ec39 1778
509c07bc 1779 srp_free_req(ch, req, scmnd,
22032991
BVA		 1780 be32_to_cpu(rsp->req_lim_delta));
1781
f8b6e31e
DD		 1782 scmnd->host_scribble = NULL;
1783 scmnd->scsi_done(scmnd);
aef9ec39 1784 }
aef9ec39
RD		 1785}
1786
509c07bc 1787static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta,
bb12588a
DD		 1788 void *rsp, int len)
1789{
509c07bc 1790 struct srp_target_port *target = ch->target;
76c75b25 1791 struct ib_device *dev = target->srp_host->srp_dev->dev;
bb12588a
DD		 1792 unsigned long flags;
1793 struct srp_iu *iu;
76c75b25 1794 int err;
bb12588a 1795
509c07bc
BVA		 1796 spin_lock_irqsave(&ch->lock, flags);
1797 ch->req_lim += req_delta;
1798 iu = __srp_get_tx_iu(ch, SRP_IU_RSP);
1799 spin_unlock_irqrestore(&ch->lock, flags);
76c75b25 1800
bb12588a
DD		 1801 if (!iu) {
1802 shost_printk(KERN_ERR, target->scsi_host, PFX
1803 "no IU available to send response\n");
76c75b25 1804 return 1;
bb12588a
DD		 1805 }
1806
1807 ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE);
1808 memcpy(iu->buf, rsp, len);
1809 ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE);
1810
509c07bc 1811 err = srp_post_send(ch, iu, len);
76c75b25 1812 if (err) {
bb12588a
DD		 1813 shost_printk(KERN_ERR, target->scsi_host, PFX
1814 "unable to post response: %d\n", err);
509c07bc 1815 srp_put_tx_iu(ch, iu, SRP_IU_RSP);
76c75b25 1816 }
bb12588a 1817
bb12588a
DD		 1818 return err;
1819}
1820
509c07bc 1821static void srp_process_cred_req(struct srp_rdma_ch *ch,
bb12588a
DD		 1822 struct srp_cred_req *req)
1823{
1824 struct srp_cred_rsp rsp = {
1825 .opcode = SRP_CRED_RSP,
1826 .tag = req->tag,
1827 };
1828 s32 delta = be32_to_cpu(req->req_lim_delta);
1829
509c07bc
BVA		 1830 if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
1831 shost_printk(KERN_ERR, ch->target->scsi_host, PFX
bb12588a
DD		 1832 "problems processing SRP_CRED_REQ\n");
1833}
1834
509c07bc 1835static void srp_process_aer_req(struct srp_rdma_ch *ch,
bb12588a
DD		 1836 struct srp_aer_req *req)
1837{
509c07bc 1838 struct srp_target_port *target = ch->target;
bb12588a
DD		 1839 struct srp_aer_rsp rsp = {
1840 .opcode = SRP_AER_RSP,
1841 .tag = req->tag,
1842 };
1843 s32 delta = be32_to_cpu(req->req_lim_delta);
1844
1845 shost_printk(KERN_ERR, target->scsi_host, PFX
985aa495 1846 "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun));
bb12588a 1847
509c07bc 1848 if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
bb12588a
DD		 1849 shost_printk(KERN_ERR, target->scsi_host, PFX
1850 "problems processing SRP_AER_REQ\n");
1851}
1852
509c07bc 1853static void srp_handle_recv(struct srp_rdma_ch *ch, struct ib_wc *wc)
aef9ec39 1854{
509c07bc 1855 struct srp_target_port *target = ch->target;
dcb4cb85 1856 struct ib_device *dev = target->srp_host->srp_dev->dev;
737b94eb 1857 struct srp_iu *iu = (struct srp_iu *) (uintptr_t) wc->wr_id;
c996bb47 1858 int res;
aef9ec39
RD		 1859 u8 opcode;
1860
509c07bc 1861 ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len,
85507bcc 1862 DMA_FROM_DEVICE);
aef9ec39
RD		 1863
1864 opcode = *(u8 *) iu->buf;
1865
1866 if (0) {
7aa54bd7
DD		 1867 shost_printk(KERN_ERR, target->scsi_host,
1868 PFX "recv completion, opcode 0x%02x\n", opcode);
7a700811
BVA		 1869 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1,
1870 iu->buf, wc->byte_len, true);
aef9ec39
RD		 1871 }
1872
1873 switch (opcode) {
1874 case SRP_RSP:
509c07bc 1875 srp_process_rsp(ch, iu->buf);
aef9ec39
RD		 1876 break;
1877
bb12588a 1878 case SRP_CRED_REQ:
509c07bc 1879 srp_process_cred_req(ch, iu->buf);
bb12588a
DD		 1880 break;
1881
1882 case SRP_AER_REQ:
509c07bc 1883 srp_process_aer_req(ch, iu->buf);
bb12588a
DD		 1884 break;
1885
aef9ec39
RD		 1886 case SRP_T_LOGOUT:
1887 /* XXX Handle target logout */
7aa54bd7
DD		 1888 shost_printk(KERN_WARNING, target->scsi_host,
1889 PFX "Got target logout request\n");
aef9ec39
RD		 1890 break;
1891
1892 default:
7aa54bd7
DD		 1893 shost_printk(KERN_WARNING, target->scsi_host,
1894 PFX "Unhandled SRP opcode 0x%02x\n", opcode);
aef9ec39
RD		 1895 break;
1896 }
1897
509c07bc 1898 ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len,
85507bcc 1899 DMA_FROM_DEVICE);
c996bb47 1900
509c07bc 1901 res = srp_post_recv(ch, iu);
c996bb47
BVA		 1902 if (res != 0)
1903 shost_printk(KERN_ERR, target->scsi_host,
1904 PFX "Recv failed with error code %d\n", res);
aef9ec39
RD		 1905}
1906
c1120f89
BVA		 1907/**
1908 * srp_tl_err_work() - handle a transport layer error
af24663b 1909 * @work: Work structure embedded in an SRP target port.
c1120f89
BVA		 1910 *
1911 * Note: This function may get invoked before the rport has been created,
1912 * hence the target->rport test.
1913 */
1914static void srp_tl_err_work(struct work_struct *work)
1915{
1916 struct srp_target_port *target;
1917
1918 target = container_of(work, struct srp_target_port, tl_err_work);
1919 if (target->rport)
1920 srp_start_tl_fail_timers(target->rport);
1921}
1922
5cfb1782 1923static void srp_handle_qp_err(u64 wr_id, enum ib_wc_status wc_status,
7dad6b2e 1924 bool send_err, struct srp_rdma_ch *ch)
948d1e88 1925{
7dad6b2e
BVA		 1926 struct srp_target_port *target = ch->target;
1927
1928 if (wr_id == SRP_LAST_WR_ID) {
1929 complete(&ch->done);
1930 return;
1931 }
1932
c014c8cd 1933 if (ch->connected && !target->qp_in_error) {
5cfb1782
BVA		 1934 if (wr_id & LOCAL_INV_WR_ID_MASK) {
1935 shost_printk(KERN_ERR, target->scsi_host, PFX
57363d98
SG		 1936 "LOCAL_INV failed with status %s (%d)\n",
1937 ib_wc_status_msg(wc_status), wc_status);
5cfb1782
BVA		 1938 } else if (wr_id & FAST_REG_WR_ID_MASK) {
1939 shost_printk(KERN_ERR, target->scsi_host, PFX
57363d98
SG		 1940 "FAST_REG_MR failed status %s (%d)\n",
1941 ib_wc_status_msg(wc_status), wc_status);
5cfb1782
BVA		 1942 } else {
1943 shost_printk(KERN_ERR, target->scsi_host,
57363d98 1944 PFX "failed %s status %s (%d) for iu %p\n",
5cfb1782 1945 send_err ? "send" : "receive",
57363d98
SG		 1946 ib_wc_status_msg(wc_status), wc_status,
1947 (void *)(uintptr_t)wr_id);
5cfb1782 1948 }
c1120f89 1949 queue_work(system_long_wq, &target->tl_err_work);
4f0af697 1950 }
948d1e88
BVA		 1951 target->qp_in_error = true;
1952}
1953
509c07bc 1954static void srp_recv_completion(struct ib_cq *cq, void *ch_ptr)
aef9ec39 1955{
509c07bc 1956 struct srp_rdma_ch *ch = ch_ptr;
aef9ec39 1957 struct ib_wc wc;
aef9ec39
RD		 1958
1959 ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
1960 while (ib_poll_cq(cq, 1, &wc) > 0) {
948d1e88 1961 if (likely(wc.status == IB_WC_SUCCESS)) {
509c07bc 1962 srp_handle_recv(ch, &wc);
948d1e88 1963 } else {
7dad6b2e 1964 srp_handle_qp_err(wc.wr_id, wc.status, false, ch);
aef9ec39 1965 }
9c03dc9f
BVA		 1966 }
1967}
1968
509c07bc 1969static void srp_send_completion(struct ib_cq *cq, void *ch_ptr)
9c03dc9f 1970{
509c07bc 1971 struct srp_rdma_ch *ch = ch_ptr;
9c03dc9f 1972 struct ib_wc wc;
dcb4cb85 1973 struct srp_iu *iu;
9c03dc9f
BVA		 1974
1975 while (ib_poll_cq(cq, 1, &wc) > 0) {
948d1e88
BVA		 1976 if (likely(wc.status == IB_WC_SUCCESS)) {
1977 iu = (struct srp_iu *) (uintptr_t) wc.wr_id;
509c07bc 1978 list_add(&iu->list, &ch->free_tx);
948d1e88 1979 } else {
7dad6b2e 1980 srp_handle_qp_err(wc.wr_id, wc.status, true, ch);
9c03dc9f 1981 }
aef9ec39
RD		 1982 }
1983}
1984
76c75b25 1985static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
aef9ec39 1986{
76c75b25 1987 struct srp_target_port *target = host_to_target(shost);
a95cadb9 1988 struct srp_rport *rport = target->rport;
509c07bc 1989 struct srp_rdma_ch *ch;
aef9ec39
RD		 1990 struct srp_request *req;
1991 struct srp_iu *iu;
1992 struct srp_cmd *cmd;
85507bcc 1993 struct ib_device *dev;
76c75b25 1994 unsigned long flags;
77f2c1a4
BVA		 1995 u32 tag;
1996 u16 idx;
d1b4289e 1997 int len, ret;
a95cadb9
BVA		 1998 const bool in_scsi_eh = !in_interrupt() && current == shost->ehandler;
1999
2000 /*
2001 * The SCSI EH thread is the only context from which srp_queuecommand()
2002 * can get invoked for blocked devices (SDEV_BLOCK /
2003 * SDEV_CREATED_BLOCK). Avoid racing with srp_reconnect_rport() by
2004 * locking the rport mutex if invoked from inside the SCSI EH.
2005 */
2006 if (in_scsi_eh)
2007 mutex_lock(&rport->mutex);
aef9ec39 2008
d1b4289e
BVA		 2009 scmnd->result = srp_chkready(target->rport);
2010 if (unlikely(scmnd->result))
2011 goto err;
2ce19e72 2012
77f2c1a4
BVA		 2013 WARN_ON_ONCE(scmnd->request->tag < 0);
2014 tag = blk_mq_unique_tag(scmnd->request);
d92c0da7 2015 ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)];
77f2c1a4
BVA		 2016 idx = blk_mq_unique_tag_to_tag(tag);
2017 WARN_ONCE(idx >= target->req_ring_size, "%s: tag %#x: idx %d >= %d\n",
2018 dev_name(&shost->shost_gendev), tag, idx,
2019 target->req_ring_size);
509c07bc
BVA		 2020
2021 spin_lock_irqsave(&ch->lock, flags);
2022 iu = __srp_get_tx_iu(ch, SRP_IU_CMD);
509c07bc 2023 spin_unlock_irqrestore(&ch->lock, flags);
aef9ec39 2024
77f2c1a4
BVA		 2025 if (!iu)
2026 goto err;
2027
2028 req = &ch->req_ring[idx];
05321937 2029 dev = target->srp_host->srp_dev->dev;
49248644 2030 ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len,
85507bcc 2031 DMA_TO_DEVICE);
aef9ec39 2032
f8b6e31e 2033 scmnd->host_scribble = (void *) req;
aef9ec39
RD		 2034
2035 cmd = iu->buf;
2036 memset(cmd, 0, sizeof *cmd);
2037
2038 cmd->opcode = SRP_CMD;
985aa495 2039 int_to_scsilun(scmnd->device->lun, &cmd->lun);
77f2c1a4 2040 cmd->tag = tag;
aef9ec39
RD		 2041 memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len);
2042
aef9ec39
RD		 2043 req->scmnd = scmnd;
2044 req->cmd = iu;
aef9ec39 2045
509c07bc 2046 len = srp_map_data(scmnd, ch, req);
aef9ec39 2047 if (len < 0) {
7aa54bd7 2048 shost_printk(KERN_ERR, target->scsi_host,
d1b4289e
BVA		 2049 PFX "Failed to map data (%d)\n", len);
2050 /*
2051 * If we ran out of memory descriptors (-ENOMEM) because an
2052 * application is queuing many requests with more than
52ede08f 2053 * max_pages_per_mr sg-list elements, tell the SCSI mid-layer
d1b4289e
BVA		 2054 * to reduce queue depth temporarily.
2055 */
2056 scmnd->result = len == -ENOMEM ?
2057 DID_OK << 16 | QUEUE_FULL << 1 : DID_ERROR << 16;
76c75b25 2058 goto err_iu;
aef9ec39
RD		 2059 }
2060
49248644 2061 ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len,
85507bcc 2062 DMA_TO_DEVICE);
aef9ec39 2063
509c07bc 2064 if (srp_post_send(ch, iu, len)) {
7aa54bd7 2065 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n");
aef9ec39
RD		 2066 goto err_unmap;
2067 }
2068
d1b4289e
BVA		 2069 ret = 0;
2070
a95cadb9
BVA		 2071unlock_rport:
2072 if (in_scsi_eh)
2073 mutex_unlock(&rport->mutex);
2074
d1b4289e 2075 return ret;
aef9ec39
RD		 2076
2077err_unmap:
509c07bc 2078 srp_unmap_data(scmnd, ch, req);
aef9ec39 2079
76c75b25 2080err_iu:
509c07bc 2081 srp_put_tx_iu(ch, iu, SRP_IU_CMD);
76c75b25 2082
024ca901
BVA		 2083 /*
2084 * Avoid that the loops that iterate over the request ring can
2085 * encounter a dangling SCSI command pointer.
2086 */
2087 req->scmnd = NULL;
2088
d1b4289e
BVA		 2089err:
2090 if (scmnd->result) {
2091 scmnd->scsi_done(scmnd);
2092 ret = 0;
2093 } else {
2094 ret = SCSI_MLQUEUE_HOST_BUSY;
2095 }
a95cadb9 2096
d1b4289e 2097 goto unlock_rport;
aef9ec39
RD		 2098}
2099
4d73f95f
BVA		 2100/*
2101 * Note: the resources allocated in this function are freed in
509c07bc 2102 * srp_free_ch_ib().
4d73f95f 2103 */
509c07bc 2104static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch)
aef9ec39 2105{
509c07bc 2106 struct srp_target_port *target = ch->target;
aef9ec39
RD		 2107 int i;
2108
509c07bc
BVA		 2109 ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring),
2110 GFP_KERNEL);
2111 if (!ch->rx_ring)
4d73f95f 2112 goto err_no_ring;
509c07bc
BVA		 2113 ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring),
2114 GFP_KERNEL);
2115 if (!ch->tx_ring)
4d73f95f
BVA		 2116 goto err_no_ring;
2117
2118 for (i = 0; i < target->queue_size; ++i) {
509c07bc
BVA		 2119 ch->rx_ring[i] = srp_alloc_iu(target->srp_host,
2120 ch->max_ti_iu_len,
2121 GFP_KERNEL, DMA_FROM_DEVICE);
2122 if (!ch->rx_ring[i])
aef9ec39
RD		 2123 goto err;
2124 }
2125
4d73f95f 2126 for (i = 0; i < target->queue_size; ++i) {
509c07bc
BVA		 2127 ch->tx_ring[i] = srp_alloc_iu(target->srp_host,
2128 target->max_iu_len,
2129 GFP_KERNEL, DMA_TO_DEVICE);
2130 if (!ch->tx_ring[i])
aef9ec39 2131 goto err;
dcb4cb85 2132
509c07bc 2133 list_add(&ch->tx_ring[i]->list, &ch->free_tx);
aef9ec39
RD		 2134 }
2135
2136 return 0;
2137
2138err:
4d73f95f 2139 for (i = 0; i < target->queue_size; ++i) {
509c07bc
BVA		 2140 srp_free_iu(target->srp_host, ch->rx_ring[i]);
2141 srp_free_iu(target->srp_host, ch->tx_ring[i]);
aef9ec39
RD		 2142 }
2143
4d73f95f
BVA		 2144
2145err_no_ring:
509c07bc
BVA		 2146 kfree(ch->tx_ring);
2147 ch->tx_ring = NULL;
2148 kfree(ch->rx_ring);
2149 ch->rx_ring = NULL;
4d73f95f 2150
aef9ec39
RD		 2151 return -ENOMEM;
2152}
2153
c9b03c1a
BVA		 2154static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask)
2155{
2156 uint64_t T_tr_ns, max_compl_time_ms;
2157 uint32_t rq_tmo_jiffies;
2158
2159 /*
2160 * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair,
2161 * table 91), both the QP timeout and the retry count have to be set
2162 * for RC QP's during the RTR to RTS transition.
2163 */
2164 WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) !=
2165 (IB_QP_TIMEOUT | IB_QP_RETRY_CNT));
2166
2167 /*
2168 * Set target->rq_tmo_jiffies to one second more than the largest time
2169 * it can take before an error completion is generated. See also
2170 * C9-140..142 in the IBTA spec for more information about how to
2171 * convert the QP Local ACK Timeout value to nanoseconds.
2172 */
2173 T_tr_ns = 4096 * (1ULL << qp_attr->timeout);
2174 max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns;
2175 do_div(max_compl_time_ms, NSEC_PER_MSEC);
2176 rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000);
2177
2178 return rq_tmo_jiffies;
2179}
2180
961e0be8 2181static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
e6300cbd 2182 const struct srp_login_rsp *lrsp,
509c07bc 2183 struct srp_rdma_ch *ch)
961e0be8 2184{
509c07bc 2185 struct srp_target_port *target = ch->target;
961e0be8
DD		 2186 struct ib_qp_attr *qp_attr = NULL;
2187 int attr_mask = 0;
2188 int ret;
2189 int i;
2190
2191 if (lrsp->opcode == SRP_LOGIN_RSP) {
509c07bc
BVA		 2192 ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
2193 ch->req_lim = be32_to_cpu(lrsp->req_lim_delta);
961e0be8
DD		 2194
2195 /*
2196 * Reserve credits for task management so we don't
2197 * bounce requests back to the SCSI mid-layer.
2198 */
2199 target->scsi_host->can_queue
509c07bc 2200 = min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE,
961e0be8 2201 target->scsi_host->can_queue);
4d73f95f
BVA		 2202 target->scsi_host->cmd_per_lun
2203 = min_t(int, target->scsi_host->can_queue,
2204 target->scsi_host->cmd_per_lun);
961e0be8
DD		 2205 } else {
2206 shost_printk(KERN_WARNING, target->scsi_host,
2207 PFX "Unhandled RSP opcode %#x\n", lrsp->opcode);
2208 ret = -ECONNRESET;
2209 goto error;
2210 }
2211
509c07bc
BVA		 2212 if (!ch->rx_ring) {
2213 ret = srp_alloc_iu_bufs(ch);
961e0be8
DD		 2214 if (ret)
2215 goto error;
2216 }
2217
2218 ret = -ENOMEM;
2219 qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
2220 if (!qp_attr)
2221 goto error;
2222
2223 qp_attr->qp_state = IB_QPS_RTR;
2224 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2225 if (ret)
2226 goto error_free;
2227
509c07bc 2228 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
961e0be8
DD		 2229 if (ret)
2230 goto error_free;
2231
4d73f95f 2232 for (i = 0; i < target->queue_size; i++) {
509c07bc
BVA		 2233 struct srp_iu *iu = ch->rx_ring[i];
2234
2235 ret = srp_post_recv(ch, iu);
961e0be8
DD		 2236 if (ret)
2237 goto error_free;
2238 }
2239
2240 qp_attr->qp_state = IB_QPS_RTS;
2241 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2242 if (ret)
2243 goto error_free;
2244
c9b03c1a
BVA		 2245 target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);
2246
509c07bc 2247 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
961e0be8
DD		 2248 if (ret)
2249 goto error_free;
2250
2251 ret = ib_send_cm_rtu(cm_id, NULL, 0);
2252
2253error_free:
2254 kfree(qp_attr);
2255
2256error:
509c07bc 2257 ch->status = ret;
961e0be8
DD		 2258}
2259
aef9ec39
RD		 2260static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
2261 struct ib_cm_event *event,
509c07bc 2262 struct srp_rdma_ch *ch)
aef9ec39 2263{
509c07bc 2264 struct srp_target_port *target = ch->target;
7aa54bd7 2265 struct Scsi_Host *shost = target->scsi_host;
aef9ec39
RD		 2266 struct ib_class_port_info *cpi;
2267 int opcode;
2268
2269 switch (event->param.rej_rcvd.reason) {
2270 case IB_CM_REJ_PORT_CM_REDIRECT:
2271 cpi = event->param.rej_rcvd.ari;
509c07bc
BVA		 2272 ch->path.dlid = cpi->redirect_lid;
2273 ch->path.pkey = cpi->redirect_pkey;
aef9ec39 2274 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
509c07bc 2275 memcpy(ch->path.dgid.raw, cpi->redirect_gid, 16);
aef9ec39 2276
509c07bc 2277 ch->status = ch->path.dlid ?
aef9ec39
RD		 2278 SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
2279 break;
2280
2281 case IB_CM_REJ_PORT_REDIRECT:
5d7cbfd6 2282 if (srp_target_is_topspin(target)) {
aef9ec39
RD		 2283 /*
2284 * Topspin/Cisco SRP gateways incorrectly send
2285 * reject reason code 25 when they mean 24
2286 * (port redirect).
2287 */
509c07bc 2288 memcpy(ch->path.dgid.raw,
aef9ec39
RD		 2289 event->param.rej_rcvd.ari, 16);
2290
7aa54bd7
DD		 2291 shost_printk(KERN_DEBUG, shost,
2292 PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
509c07bc
BVA		 2293 be64_to_cpu(ch->path.dgid.global.subnet_prefix),
2294 be64_to_cpu(ch->path.dgid.global.interface_id));
aef9ec39 2295
509c07bc 2296 ch->status = SRP_PORT_REDIRECT;
aef9ec39 2297 } else {
7aa54bd7
DD		 2298 shost_printk(KERN_WARNING, shost,
2299 " REJ reason: IB_CM_REJ_PORT_REDIRECT\n");
509c07bc 2300 ch->status = -ECONNRESET;
aef9ec39
RD		 2301 }
2302 break;
2303
2304 case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
7aa54bd7
DD		 2305 shost_printk(KERN_WARNING, shost,
2306 " REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
509c07bc 2307 ch->status = -ECONNRESET;
aef9ec39
RD		 2308 break;
2309
2310 case IB_CM_REJ_CONSUMER_DEFINED:
2311 opcode = *(u8 *) event->private_data;
2312 if (opcode == SRP_LOGIN_REJ) {
2313 struct srp_login_rej *rej = event->private_data;
2314 u32 reason = be32_to_cpu(rej->reason);
2315
2316 if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
7aa54bd7
DD		 2317 shost_printk(KERN_WARNING, shost,
2318 PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
aef9ec39 2319 else
e7ffde01
BVA		 2320 shost_printk(KERN_WARNING, shost, PFX
2321 "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n",
747fe000
BVA		 2322 target->sgid.raw,
2323 target->orig_dgid.raw, reason);
aef9ec39 2324 } else
7aa54bd7
DD		 2325 shost_printk(KERN_WARNING, shost,
2326 " REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
2327 " opcode 0x%02x\n", opcode);
509c07bc 2328 ch->status = -ECONNRESET;
aef9ec39
RD		 2329 break;
2330
9fe4bcf4
DD		 2331 case IB_CM_REJ_STALE_CONN:
2332 shost_printk(KERN_WARNING, shost, " REJ reason: stale connection\n");
509c07bc 2333 ch->status = SRP_STALE_CONN;
9fe4bcf4
DD		 2334 break;
2335
aef9ec39 2336 default:
7aa54bd7
DD		 2337 shost_printk(KERN_WARNING, shost, " REJ reason 0x%x\n",
2338 event->param.rej_rcvd.reason);
509c07bc 2339 ch->status = -ECONNRESET;
aef9ec39
RD		 2340 }
2341}
2342
2343static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2344{
509c07bc
BVA		 2345 struct srp_rdma_ch *ch = cm_id->context;
2346 struct srp_target_port *target = ch->target;
aef9ec39 2347 int comp = 0;
aef9ec39
RD		 2348
2349 switch (event->event) {
2350 case IB_CM_REQ_ERROR:
7aa54bd7
DD		 2351 shost_printk(KERN_DEBUG, target->scsi_host,
2352 PFX "Sending CM REQ failed\n");
aef9ec39 2353 comp = 1;
509c07bc 2354 ch->status = -ECONNRESET;
aef9ec39
RD		 2355 break;
2356
2357 case IB_CM_REP_RECEIVED:
2358 comp = 1;
509c07bc 2359 srp_cm_rep_handler(cm_id, event->private_data, ch);
aef9ec39
RD		 2360 break;
2361
2362 case IB_CM_REJ_RECEIVED:
7aa54bd7 2363 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
aef9ec39
RD		 2364 comp = 1;
2365
509c07bc 2366 srp_cm_rej_handler(cm_id, event, ch);
aef9ec39
RD		 2367 break;
2368
b7ac4ab4 2369 case IB_CM_DREQ_RECEIVED:
7aa54bd7
DD		 2370 shost_printk(KERN_WARNING, target->scsi_host,
2371 PFX "DREQ received - connection closed\n");
c014c8cd 2372 ch->connected = false;
b7ac4ab4 2373 if (ib_send_cm_drep(cm_id, NULL, 0))
7aa54bd7
DD		 2374 shost_printk(KERN_ERR, target->scsi_host,
2375 PFX "Sending CM DREP failed\n");
c1120f89 2376 queue_work(system_long_wq, &target->tl_err_work);
aef9ec39
RD		 2377 break;
2378
2379 case IB_CM_TIMEWAIT_EXIT:
7aa54bd7
DD		 2380 shost_printk(KERN_ERR, target->scsi_host,
2381 PFX "connection closed\n");
ac72d766 2382 comp = 1;
aef9ec39 2383
509c07bc 2384 ch->status = 0;
aef9ec39
RD		 2385 break;
2386
b7ac4ab4
IR		 2387 case IB_CM_MRA_RECEIVED:
2388 case IB_CM_DREQ_ERROR:
2389 case IB_CM_DREP_RECEIVED:
2390 break;
2391
aef9ec39 2392 default:
7aa54bd7
DD		 2393 shost_printk(KERN_WARNING, target->scsi_host,
2394 PFX "Unhandled CM event %d\n", event->event);
aef9ec39
RD		 2395 break;
2396 }
2397
2398 if (comp)
509c07bc 2399 complete(&ch->done);
aef9ec39 2400
aef9ec39
RD		 2401 return 0;
2402}
2403
71444b97
JW		 2404/**
2405 * srp_change_queue_depth - setting device queue depth
2406 * @sdev: scsi device struct
2407 * @qdepth: requested queue depth
71444b97
JW		 2408 *
2409 * Returns queue depth.
2410 */
2411static int
db5ed4df 2412srp_change_queue_depth(struct scsi_device *sdev, int qdepth)
71444b97 2413{
c40ecc12 2414 if (!sdev->tagged_supported)
1e6f2416 2415 qdepth = 1;
db5ed4df 2416 return scsi_change_queue_depth(sdev, qdepth);
71444b97
JW		 2417}
2418
985aa495
BVA		 2419static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag, u64 lun,
2420 u8 func)
aef9ec39 2421{
509c07bc 2422 struct srp_target_port *target = ch->target;
a95cadb9 2423 struct srp_rport *rport = target->rport;
19081f31 2424 struct ib_device *dev = target->srp_host->srp_dev->dev;
aef9ec39
RD		 2425 struct srp_iu *iu;
2426 struct srp_tsk_mgmt *tsk_mgmt;
aef9ec39 2427
c014c8cd 2428 if (!ch->connected || target->qp_in_error)
3780d1f0
BVA		 2429 return -1;
2430
509c07bc 2431 init_completion(&ch->tsk_mgmt_done);
aef9ec39 2432
a95cadb9 2433 /*
509c07bc 2434 * Lock the rport mutex to avoid that srp_create_ch_ib() is
a95cadb9
BVA		 2435 * invoked while a task management function is being sent.
2436 */
2437 mutex_lock(&rport->mutex);
509c07bc
BVA		 2438 spin_lock_irq(&ch->lock);
2439 iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT);
2440 spin_unlock_irq(&ch->lock);
76c75b25 2441
a95cadb9
BVA		 2442 if (!iu) {
2443 mutex_unlock(&rport->mutex);
2444
76c75b25 2445 return -1;
a95cadb9 2446 }
aef9ec39 2447
19081f31
DD		 2448 ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt,
2449 DMA_TO_DEVICE);
aef9ec39
RD		 2450 tsk_mgmt = iu->buf;
2451 memset(tsk_mgmt, 0, sizeof *tsk_mgmt);
2452
2453 tsk_mgmt->opcode = SRP_TSK_MGMT;
985aa495 2454 int_to_scsilun(lun, &tsk_mgmt->lun);
f8b6e31e 2455 tsk_mgmt->tag = req_tag | SRP_TAG_TSK_MGMT;
aef9ec39 2456 tsk_mgmt->tsk_mgmt_func = func;
f8b6e31e 2457 tsk_mgmt->task_tag = req_tag;
aef9ec39 2458
19081f31
DD		 2459 ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt,
2460 DMA_TO_DEVICE);
509c07bc
BVA		 2461 if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) {
2462 srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT);
a95cadb9
BVA		 2463 mutex_unlock(&rport->mutex);
2464
76c75b25
BVA		 2465 return -1;
2466 }
a95cadb9 2467 mutex_unlock(&rport->mutex);
d945e1df 2468
509c07bc 2469 if (!wait_for_completion_timeout(&ch->tsk_mgmt_done,
aef9ec39 2470 msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS)))
d945e1df 2471 return -1;
aef9ec39 2472
d945e1df 2473 return 0;
d945e1df
RD		 2474}
2475
aef9ec39
RD		 2476static int srp_abort(struct scsi_cmnd *scmnd)
2477{
d945e1df 2478 struct srp_target_port *target = host_to_target(scmnd->device->host);
f8b6e31e 2479 struct srp_request *req = (struct srp_request *) scmnd->host_scribble;
77f2c1a4 2480 u32 tag;
d92c0da7 2481 u16 ch_idx;
509c07bc 2482 struct srp_rdma_ch *ch;
086f44f5 2483 int ret;
d945e1df 2484
7aa54bd7 2485 shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n");
aef9ec39 2486
d92c0da7 2487 if (!req)
99b6697a 2488 return SUCCESS;
77f2c1a4 2489 tag = blk_mq_unique_tag(scmnd->request);
d92c0da7
BVA		 2490 ch_idx = blk_mq_unique_tag_to_hwq(tag);
2491 if (WARN_ON_ONCE(ch_idx >= target->ch_count))
2492 return SUCCESS;
2493 ch = &target->ch[ch_idx];
2494 if (!srp_claim_req(ch, req, NULL, scmnd))
2495 return SUCCESS;
2496 shost_printk(KERN_ERR, target->scsi_host,
2497 "Sending SRP abort for tag %#x\n", tag);
77f2c1a4 2498 if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun,
80d5e8a2 2499 SRP_TSK_ABORT_TASK) == 0)
086f44f5 2500 ret = SUCCESS;
ed9b2264 2501 else if (target->rport->state == SRP_RPORT_LOST)
99e1c139 2502 ret = FAST_IO_FAIL;
086f44f5
BVA		 2503 else
2504 ret = FAILED;
509c07bc 2505 srp_free_req(ch, req, scmnd, 0);
22032991 2506 scmnd->result = DID_ABORT << 16;
d8536670 2507 scmnd->scsi_done(scmnd);
d945e1df 2508
086f44f5 2509 return ret;
aef9ec39
RD		 2510}
2511
2512static int srp_reset_device(struct scsi_cmnd *scmnd)
2513{
d945e1df 2514 struct srp_target_port *target = host_to_target(scmnd->device->host);
d92c0da7 2515 struct srp_rdma_ch *ch;
536ae14e 2516 int i;
d945e1df 2517
7aa54bd7 2518 shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n");
aef9ec39 2519
d92c0da7 2520 ch = &target->ch[0];
509c07bc 2521 if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun,
f8b6e31e 2522 SRP_TSK_LUN_RESET))
d945e1df 2523 return FAILED;
509c07bc 2524 if (ch->tsk_mgmt_status)
d945e1df
RD		 2525 return FAILED;
2526
d92c0da7
BVA		 2527 for (i = 0; i < target->ch_count; i++) {
2528 ch = &target->ch[i];
2529 for (i = 0; i < target->req_ring_size; ++i) {
2530 struct srp_request *req = &ch->req_ring[i];
509c07bc 2531
d92c0da7
BVA		 2532 srp_finish_req(ch, req, scmnd->device, DID_RESET << 16);
2533 }
536ae14e 2534 }
d945e1df 2535
d945e1df 2536 return SUCCESS;
aef9ec39
RD		 2537}
2538
2539static int srp_reset_host(struct scsi_cmnd *scmnd)
2540{
2541 struct srp_target_port *target = host_to_target(scmnd->device->host);
aef9ec39 2542
7aa54bd7 2543 shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n");
aef9ec39 2544
ed9b2264 2545 return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED;
aef9ec39
RD		 2546}
2547
c9b03c1a
BVA		 2548static int srp_slave_configure(struct scsi_device *sdev)
2549{
2550 struct Scsi_Host *shost = sdev->host;
2551 struct srp_target_port *target = host_to_target(shost);
2552 struct request_queue *q = sdev->request_queue;
2553 unsigned long timeout;
2554
2555 if (sdev->type == TYPE_DISK) {
2556 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies);
2557 blk_queue_rq_timeout(q, timeout);
2558 }
2559
2560 return 0;
2561}
2562
ee959b00
TJ		 2563static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr,
2564 char *buf)
6ecb0c84 2565{
ee959b00 2566 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2567
45c37cad 2568 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->id_ext));
6ecb0c84
RD		 2569}
2570
ee959b00
TJ		 2571static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr,
2572 char *buf)
6ecb0c84 2573{
ee959b00 2574 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2575
45c37cad 2576 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid));
6ecb0c84
RD		 2577}
2578
ee959b00
TJ		 2579static ssize_t show_service_id(struct device *dev,
2580 struct device_attribute *attr, char *buf)
6ecb0c84 2581{
ee959b00 2582 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2583
45c37cad 2584 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->service_id));
6ecb0c84
RD		 2585}
2586
ee959b00
TJ		 2587static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
2588 char *buf)
6ecb0c84 2589{
ee959b00 2590 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2591
747fe000 2592 return sprintf(buf, "0x%04x\n", be16_to_cpu(target->pkey));
6ecb0c84
RD		 2593}
2594
848b3082
BVA		 2595static ssize_t show_sgid(struct device *dev, struct device_attribute *attr,
2596 char *buf)
2597{
2598 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2599
747fe000 2600 return sprintf(buf, "%pI6\n", target->sgid.raw);
848b3082
BVA		 2601}
2602
ee959b00
TJ		 2603static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
2604 char *buf)
6ecb0c84 2605{
ee959b00 2606 struct srp_target_port *target = host_to_target(class_to_shost(dev));
d92c0da7 2607 struct srp_rdma_ch *ch = &target->ch[0];
6ecb0c84 2608
509c07bc 2609 return sprintf(buf, "%pI6\n", ch->path.dgid.raw);
6ecb0c84
RD		 2610}
2611
ee959b00
TJ		 2612static ssize_t show_orig_dgid(struct device *dev,
2613 struct device_attribute *attr, char *buf)
3633b3d0 2614{
ee959b00 2615 struct srp_target_port *target = host_to_target(class_to_shost(dev));
3633b3d0 2616
747fe000 2617 return sprintf(buf, "%pI6\n", target->orig_dgid.raw);
3633b3d0
IR		 2618}
2619
89de7486
BVA		 2620static ssize_t show_req_lim(struct device *dev,
2621 struct device_attribute *attr, char *buf)
2622{
2623 struct srp_target_port *target = host_to_target(class_to_shost(dev));
d92c0da7
BVA		 2624 struct srp_rdma_ch *ch;
2625 int i, req_lim = INT_MAX;
89de7486 2626
d92c0da7
BVA		 2627 for (i = 0; i < target->ch_count; i++) {
2628 ch = &target->ch[i];
2629 req_lim = min(req_lim, ch->req_lim);
2630 }
2631 return sprintf(buf, "%d\n", req_lim);
89de7486
BVA		 2632}
2633
ee959b00
TJ		 2634static ssize_t show_zero_req_lim(struct device *dev,
2635 struct device_attribute *attr, char *buf)
6bfa24fa 2636{
ee959b00 2637 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6bfa24fa 2638
6bfa24fa
RD		 2639 return sprintf(buf, "%d\n", target->zero_req_lim);
2640}
2641
ee959b00
TJ		 2642static ssize_t show_local_ib_port(struct device *dev,
2643 struct device_attribute *attr, char *buf)
ded7f1a1 2644{
ee959b00 2645 struct srp_target_port *target = host_to_target(class_to_shost(dev));
ded7f1a1
IR		 2646
2647 return sprintf(buf, "%d\n", target->srp_host->port);
2648}
2649
ee959b00
TJ		 2650static ssize_t show_local_ib_device(struct device *dev,
2651 struct device_attribute *attr, char *buf)
ded7f1a1 2652{
ee959b00 2653 struct srp_target_port *target = host_to_target(class_to_shost(dev));
ded7f1a1 2654
05321937 2655 return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
ded7f1a1
IR		 2656}
2657
d92c0da7
BVA		 2658static ssize_t show_ch_count(struct device *dev, struct device_attribute *attr,
2659 char *buf)
2660{
2661 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2662
2663 return sprintf(buf, "%d\n", target->ch_count);
2664}
2665
4b5e5f41
BVA		 2666static ssize_t show_comp_vector(struct device *dev,
2667 struct device_attribute *attr, char *buf)
2668{
2669 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2670
2671 return sprintf(buf, "%d\n", target->comp_vector);
2672}
2673
7bb312e4
VP		 2674static ssize_t show_tl_retry_count(struct device *dev,
2675 struct device_attribute *attr, char *buf)
2676{
2677 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2678
2679 return sprintf(buf, "%d\n", target->tl_retry_count);
2680}
2681
49248644
DD		 2682static ssize_t show_cmd_sg_entries(struct device *dev,
2683 struct device_attribute *attr, char *buf)
2684{
2685 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2686
2687 return sprintf(buf, "%u\n", target->cmd_sg_cnt);
2688}
2689
c07d424d
DD		 2690static ssize_t show_allow_ext_sg(struct device *dev,
2691 struct device_attribute *attr, char *buf)
2692{
2693 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2694
2695 return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
2696}
2697
ee959b00
TJ		 2698static DEVICE_ATTR(id_ext, S_IRUGO, show_id_ext, NULL);
2699static DEVICE_ATTR(ioc_guid, S_IRUGO, show_ioc_guid, NULL);
2700static DEVICE_ATTR(service_id, S_IRUGO, show_service_id, NULL);
2701static DEVICE_ATTR(pkey, S_IRUGO, show_pkey, NULL);
848b3082 2702static DEVICE_ATTR(sgid, S_IRUGO, show_sgid, NULL);
ee959b00
TJ		 2703static DEVICE_ATTR(dgid, S_IRUGO, show_dgid, NULL);
2704static DEVICE_ATTR(orig_dgid, S_IRUGO, show_orig_dgid, NULL);
89de7486 2705static DEVICE_ATTR(req_lim, S_IRUGO, show_req_lim, NULL);
ee959b00
TJ		 2706static DEVICE_ATTR(zero_req_lim, S_IRUGO, show_zero_req_lim, NULL);
2707static DEVICE_ATTR(local_ib_port, S_IRUGO, show_local_ib_port, NULL);
2708static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
d92c0da7 2709static DEVICE_ATTR(ch_count, S_IRUGO, show_ch_count, NULL);
4b5e5f41 2710static DEVICE_ATTR(comp_vector, S_IRUGO, show_comp_vector, NULL);
7bb312e4 2711static DEVICE_ATTR(tl_retry_count, S_IRUGO, show_tl_retry_count, NULL);
49248644 2712static DEVICE_ATTR(cmd_sg_entries, S_IRUGO, show_cmd_sg_entries, NULL);
c07d424d 2713static DEVICE_ATTR(allow_ext_sg, S_IRUGO, show_allow_ext_sg, NULL);
ee959b00
TJ		 2714
2715static struct device_attribute *srp_host_attrs[] = {
2716 &dev_attr_id_ext,
2717 &dev_attr_ioc_guid,
2718 &dev_attr_service_id,
2719 &dev_attr_pkey,
848b3082 2720 &dev_attr_sgid,
ee959b00
TJ		 2721 &dev_attr_dgid,
2722 &dev_attr_orig_dgid,
89de7486 2723 &dev_attr_req_lim,
ee959b00
TJ		 2724 &dev_attr_zero_req_lim,
2725 &dev_attr_local_ib_port,
2726 &dev_attr_local_ib_device,
d92c0da7 2727 &dev_attr_ch_count,
4b5e5f41 2728 &dev_attr_comp_vector,
7bb312e4 2729 &dev_attr_tl_retry_count,
49248644 2730 &dev_attr_cmd_sg_entries,
c07d424d 2731 &dev_attr_allow_ext_sg,
6ecb0c84
RD		 2732 NULL
2733};
2734
aef9ec39
RD		 2735static struct scsi_host_template srp_template = {
2736 .module = THIS_MODULE,
b7f008fd
RD		 2737 .name = "InfiniBand SRP initiator",
2738 .proc_name = DRV_NAME,
c9b03c1a 2739 .slave_configure = srp_slave_configure,
aef9ec39
RD		 2740 .info = srp_target_info,
2741 .queuecommand = srp_queuecommand,
71444b97 2742 .change_queue_depth = srp_change_queue_depth,
aef9ec39
RD		 2743 .eh_abort_handler = srp_abort,
2744 .eh_device_reset_handler = srp_reset_device,
2745 .eh_host_reset_handler = srp_reset_host,
2742c1da 2746 .skip_settle_delay = true,
49248644 2747 .sg_tablesize = SRP_DEF_SG_TABLESIZE,
4d73f95f 2748 .can_queue = SRP_DEFAULT_CMD_SQ_SIZE,
aef9ec39 2749 .this_id = -1,
4d73f95f 2750 .cmd_per_lun = SRP_DEFAULT_CMD_SQ_SIZE,
6ecb0c84 2751 .use_clustering = ENABLE_CLUSTERING,
77f2c1a4
BVA		 2752 .shost_attrs = srp_host_attrs,
2753 .use_blk_tags = 1,
c40ecc12 2754 .track_queue_depth = 1,
aef9ec39
RD		 2755};
2756
34aa654e
BVA		 2757static int srp_sdev_count(struct Scsi_Host *host)
2758{
2759 struct scsi_device *sdev;
2760 int c = 0;
2761
2762 shost_for_each_device(sdev, host)
2763 c++;
2764
2765 return c;
2766}
2767
bc44bd1d
BVA		 2768/*
2769 * Return values:
2770 * < 0 upon failure. Caller is responsible for SRP target port cleanup.
2771 * 0 and target->state == SRP_TARGET_REMOVED if asynchronous target port
2772 * removal has been scheduled.
2773 * 0 and target->state != SRP_TARGET_REMOVED upon success.
2774 */
aef9ec39
RD		 2775static int srp_add_target(struct srp_host *host, struct srp_target_port *target)
2776{
3236822b
FT		 2777 struct srp_rport_identifiers ids;
2778 struct srp_rport *rport;
2779
34aa654e 2780 target->state = SRP_TARGET_SCANNING;
aef9ec39 2781 sprintf(target->target_name, "SRP.T10:%016llX",
45c37cad 2782 be64_to_cpu(target->id_ext));
aef9ec39 2783
05321937 2784 if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dma_device))
aef9ec39
RD		 2785 return -ENODEV;
2786
3236822b
FT		 2787 memcpy(ids.port_id, &target->id_ext, 8);
2788 memcpy(ids.port_id + 8, &target->ioc_guid, 8);
aebd5e47 2789 ids.roles = SRP_RPORT_ROLE_TARGET;
3236822b
FT		 2790 rport = srp_rport_add(target->scsi_host, &ids);
2791 if (IS_ERR(rport)) {
2792 scsi_remove_host(target->scsi_host);
2793 return PTR_ERR(rport);
2794 }
2795
dc1bdbd9 2796 rport->lld_data = target;
9dd69a60 2797 target->rport = rport;
dc1bdbd9 2798
b3589fd4 2799 spin_lock(&host->target_lock);
aef9ec39 2800 list_add_tail(&target->list, &host->target_list);
b3589fd4 2801 spin_unlock(&host->target_lock);
aef9ec39 2802
aef9ec39 2803 scsi_scan_target(&target->scsi_host->shost_gendev,
1962a4a1 2804 0, target->scsi_id, SCAN_WILD_CARD, 0);
aef9ec39 2805
c014c8cd
BVA		 2806 if (srp_connected_ch(target) < target->ch_count ||
2807 target->qp_in_error) {
34aa654e
BVA		 2808 shost_printk(KERN_INFO, target->scsi_host,
2809 PFX "SCSI scan failed - removing SCSI host\n");
2810 srp_queue_remove_work(target);
2811 goto out;
2812 }
2813
2814 pr_debug(PFX "%s: SCSI scan succeeded - detected %d LUNs\n",
2815 dev_name(&target->scsi_host->shost_gendev),
2816 srp_sdev_count(target->scsi_host));
2817
2818 spin_lock_irq(&target->lock);
2819 if (target->state == SRP_TARGET_SCANNING)
2820 target->state = SRP_TARGET_LIVE;
2821 spin_unlock_irq(&target->lock);
2822
2823out:
aef9ec39
RD		 2824 return 0;
2825}
2826
ee959b00 2827static void srp_release_dev(struct device *dev)
aef9ec39
RD		 2828{
2829 struct srp_host *host =
ee959b00 2830 container_of(dev, struct srp_host, dev);
aef9ec39
RD		 2831
2832 complete(&host->released);
2833}
2834
2835static struct class srp_class = {
2836 .name = "infiniband_srp",
ee959b00 2837 .dev_release = srp_release_dev
aef9ec39
RD		 2838};
2839
96fc248a
BVA		 2840/**
2841 * srp_conn_unique() - check whether the connection to a target is unique
af24663b
BVA		 2842 * @host: SRP host.
2843 * @target: SRP target port.
96fc248a
BVA		 2844 */
2845static bool srp_conn_unique(struct srp_host *host,
2846 struct srp_target_port *target)
2847{
2848 struct srp_target_port *t;
2849 bool ret = false;
2850
2851 if (target->state == SRP_TARGET_REMOVED)
2852 goto out;
2853
2854 ret = true;
2855
2856 spin_lock(&host->target_lock);
2857 list_for_each_entry(t, &host->target_list, list) {
2858 if (t != target &&
2859 target->id_ext == t->id_ext &&
2860 target->ioc_guid == t->ioc_guid &&
2861 target->initiator_ext == t->initiator_ext) {
2862 ret = false;
2863 break;
2864 }
2865 }
2866 spin_unlock(&host->target_lock);
2867
2868out:
2869 return ret;
2870}
2871
aef9ec39
RD		 2872/*
2873 * Target ports are added by writing
2874 *
2875 * id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>,
2876 * pkey=<P_Key>,service_id=<service ID>
2877 *
2878 * to the add_target sysfs attribute.
2879 */
2880enum {
2881 SRP_OPT_ERR = 0,
2882 SRP_OPT_ID_EXT = 1 << 0,
2883 SRP_OPT_IOC_GUID = 1 << 1,
2884 SRP_OPT_DGID = 1 << 2,
2885 SRP_OPT_PKEY = 1 << 3,
2886 SRP_OPT_SERVICE_ID = 1 << 4,
2887 SRP_OPT_MAX_SECT = 1 << 5,
52fb2b50 2888 SRP_OPT_MAX_CMD_PER_LUN = 1 << 6,
0c0450db 2889 SRP_OPT_IO_CLASS = 1 << 7,
01cb9bcb 2890 SRP_OPT_INITIATOR_EXT = 1 << 8,
49248644 2891 SRP_OPT_CMD_SG_ENTRIES = 1 << 9,
c07d424d
DD		 2892 SRP_OPT_ALLOW_EXT_SG = 1 << 10,
2893 SRP_OPT_SG_TABLESIZE = 1 << 11,
4b5e5f41 2894 SRP_OPT_COMP_VECTOR = 1 << 12,
7bb312e4 2895 SRP_OPT_TL_RETRY_COUNT = 1 << 13,
4d73f95f 2896 SRP_OPT_QUEUE_SIZE = 1 << 14,
aef9ec39
RD		 2897 SRP_OPT_ALL = (SRP_OPT_ID_EXT |
2898 SRP_OPT_IOC_GUID |
2899 SRP_OPT_DGID |
2900 SRP_OPT_PKEY |
2901 SRP_OPT_SERVICE_ID),
2902};
2903
a447c093 2904static const match_table_t srp_opt_tokens = {
52fb2b50
VP		 2905 { SRP_OPT_ID_EXT, "id_ext=%s" },
2906 { SRP_OPT_IOC_GUID, "ioc_guid=%s" },
2907 { SRP_OPT_DGID, "dgid=%s" },
2908 { SRP_OPT_PKEY, "pkey=%x" },
2909 { SRP_OPT_SERVICE_ID, "service_id=%s" },
2910 { SRP_OPT_MAX_SECT, "max_sect=%d" },
2911 { SRP_OPT_MAX_CMD_PER_LUN, "max_cmd_per_lun=%d" },
0c0450db 2912 { SRP_OPT_IO_CLASS, "io_class=%x" },
01cb9bcb 2913 { SRP_OPT_INITIATOR_EXT, "initiator_ext=%s" },
49248644 2914 { SRP_OPT_CMD_SG_ENTRIES, "cmd_sg_entries=%u" },
c07d424d
DD		 2915 { SRP_OPT_ALLOW_EXT_SG, "allow_ext_sg=%u" },
2916 { SRP_OPT_SG_TABLESIZE, "sg_tablesize=%u" },
4b5e5f41 2917 { SRP_OPT_COMP_VECTOR, "comp_vector=%u" },
7bb312e4 2918 { SRP_OPT_TL_RETRY_COUNT, "tl_retry_count=%u" },
4d73f95f 2919 { SRP_OPT_QUEUE_SIZE, "queue_size=%d" },
52fb2b50 2920 { SRP_OPT_ERR, NULL }
aef9ec39
RD		 2921};
2922
2923static int srp_parse_options(const char *buf, struct srp_target_port *target)
2924{
2925 char *options, *sep_opt;
2926 char *p;
2927 char dgid[3];
2928 substring_t args[MAX_OPT_ARGS];
2929 int opt_mask = 0;
2930 int token;
2931 int ret = -EINVAL;
2932 int i;
2933
2934 options = kstrdup(buf, GFP_KERNEL);
2935 if (!options)
2936 return -ENOMEM;
2937
2938 sep_opt = options;
7dcf9c19 2939 while ((p = strsep(&sep_opt, ",\n")) != NULL) {
aef9ec39
RD		 2940 if (!*p)
2941 continue;
2942
2943 token = match_token(p, srp_opt_tokens, args);
2944 opt_mask |= token;
2945
2946 switch (token) {
2947 case SRP_OPT_ID_EXT:
2948 p = match_strdup(args);
a20f3a6d
IR		 2949 if (!p) {
2950 ret = -ENOMEM;
2951 goto out;
2952 }
aef9ec39
RD		 2953 target->id_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
2954 kfree(p);
2955 break;
2956
2957 case SRP_OPT_IOC_GUID:
2958 p = match_strdup(args);
a20f3a6d
IR		 2959 if (!p) {
2960 ret = -ENOMEM;
2961 goto out;
2962 }
aef9ec39
RD		 2963 target->ioc_guid = cpu_to_be64(simple_strtoull(p, NULL, 16));
2964 kfree(p);
2965 break;
2966
2967 case SRP_OPT_DGID:
2968 p = match_strdup(args);
a20f3a6d
IR		 2969 if (!p) {
2970 ret = -ENOMEM;
2971 goto out;
2972 }
aef9ec39 2973 if (strlen(p) != 32) {
e0bda7d8 2974 pr_warn("bad dest GID parameter '%s'\n", p);
ce1823f0 2975 kfree(p);
aef9ec39
RD		 2976 goto out;
2977 }
2978
2979 for (i = 0; i < 16; ++i) {
747fe000
BVA		 2980 strlcpy(dgid, p + i * 2, sizeof(dgid));
2981 if (sscanf(dgid, "%hhx",
2982 &target->orig_dgid.raw[i]) < 1) {
2983 ret = -EINVAL;
2984 kfree(p);
2985 goto out;
2986 }
aef9ec39 2987 }
bf17c1c7 2988 kfree(p);
aef9ec39
RD		 2989 break;
2990
2991 case SRP_OPT_PKEY:
2992 if (match_hex(args, &token)) {
e0bda7d8 2993 pr_warn("bad P_Key parameter '%s'\n", p);
aef9ec39
RD		 2994 goto out;
2995 }
747fe000 2996 target->pkey = cpu_to_be16(token);
aef9ec39
RD		 2997 break;
2998
2999 case SRP_OPT_SERVICE_ID:
3000 p = match_strdup(args);
a20f3a6d
IR		 3001 if (!p) {
3002 ret = -ENOMEM;
3003 goto out;
3004 }
aef9ec39
RD		 3005 target->service_id = cpu_to_be64(simple_strtoull(p, NULL, 16));
3006 kfree(p);
3007 break;
3008
3009 case SRP_OPT_MAX_SECT:
3010 if (match_int(args, &token)) {
e0bda7d8 3011 pr_warn("bad max sect parameter '%s'\n", p);
aef9ec39
RD		 3012 goto out;
3013 }
3014 target->scsi_host->max_sectors = token;
3015 break;
3016
4d73f95f
BVA		 3017 case SRP_OPT_QUEUE_SIZE:
3018 if (match_int(args, &token) || token < 1) {
3019 pr_warn("bad queue_size parameter '%s'\n", p);
3020 goto out;
3021 }
3022 target->scsi_host->can_queue = token;
3023 target->queue_size = token + SRP_RSP_SQ_SIZE +
3024 SRP_TSK_MGMT_SQ_SIZE;
3025 if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3026 target->scsi_host->cmd_per_lun = token;
3027 break;
3028
52fb2b50 3029 case SRP_OPT_MAX_CMD_PER_LUN:
4d73f95f 3030 if (match_int(args, &token) || token < 1) {
e0bda7d8
BVA		 3031 pr_warn("bad max cmd_per_lun parameter '%s'\n",
3032 p);
52fb2b50
VP		 3033 goto out;
3034 }
4d73f95f 3035 target->scsi_host->cmd_per_lun = token;
52fb2b50
VP		 3036 break;
3037
0c0450db
R		 3038 case SRP_OPT_IO_CLASS:
3039 if (match_hex(args, &token)) {
e0bda7d8 3040 pr_warn("bad IO class parameter '%s'\n", p);
0c0450db
R		 3041 goto out;
3042 }
3043 if (token != SRP_REV10_IB_IO_CLASS &&
3044 token != SRP_REV16A_IB_IO_CLASS) {
e0bda7d8
BVA		 3045 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
3046 token, SRP_REV10_IB_IO_CLASS,
3047 SRP_REV16A_IB_IO_CLASS);
0c0450db
R		 3048 goto out;
3049 }
3050 target->io_class = token;
3051 break;
3052
01cb9bcb
IR		 3053 case SRP_OPT_INITIATOR_EXT:
3054 p = match_strdup(args);
a20f3a6d
IR		 3055 if (!p) {
3056 ret = -ENOMEM;
3057 goto out;
3058 }
01cb9bcb
IR		 3059 target->initiator_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
3060 kfree(p);
3061 break;
3062
49248644
DD		 3063 case SRP_OPT_CMD_SG_ENTRIES:
3064 if (match_int(args, &token) || token < 1 || token > 255) {
e0bda7d8
BVA		 3065 pr_warn("bad max cmd_sg_entries parameter '%s'\n",
3066 p);
49248644
DD		 3067 goto out;
3068 }
3069 target->cmd_sg_cnt = token;
3070 break;
3071
c07d424d
DD		 3072 case SRP_OPT_ALLOW_EXT_SG:
3073 if (match_int(args, &token)) {
e0bda7d8 3074 pr_warn("bad allow_ext_sg parameter '%s'\n", p);
c07d424d
DD		 3075 goto out;
3076 }
3077 target->allow_ext_sg = !!token;
3078 break;
3079
3080 case SRP_OPT_SG_TABLESIZE:
3081 if (match_int(args, &token) || token < 1 ||
3082 token > SCSI_MAX_SG_CHAIN_SEGMENTS) {
e0bda7d8
BVA		 3083 pr_warn("bad max sg_tablesize parameter '%s'\n",
3084 p);
c07d424d
DD		 3085 goto out;
3086 }
3087 target->sg_tablesize = token;
3088 break;
3089
4b5e5f41
BVA		 3090 case SRP_OPT_COMP_VECTOR:
3091 if (match_int(args, &token) || token < 0) {
3092 pr_warn("bad comp_vector parameter '%s'\n", p);
3093 goto out;
3094 }
3095 target->comp_vector = token;
3096 break;
3097
7bb312e4
VP		 3098 case SRP_OPT_TL_RETRY_COUNT:
3099 if (match_int(args, &token) || token < 2 || token > 7) {
3100 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
3101 p);
3102 goto out;
3103 }
3104 target->tl_retry_count = token;
3105 break;
3106
aef9ec39 3107 default:
e0bda7d8
BVA		 3108 pr_warn("unknown parameter or missing value '%s' in target creation request\n",
3109 p);
aef9ec39
RD		 3110 goto out;
3111 }
3112 }
3113
3114 if ((opt_mask & SRP_OPT_ALL) == SRP_OPT_ALL)
3115 ret = 0;
3116 else
3117 for (i = 0; i < ARRAY_SIZE(srp_opt_tokens); ++i)
3118 if ((srp_opt_tokens[i].token & SRP_OPT_ALL) &&
3119 !(srp_opt_tokens[i].token & opt_mask))
e0bda7d8
BVA		 3120 pr_warn("target creation request is missing parameter '%s'\n",
3121 srp_opt_tokens[i].pattern);
aef9ec39 3122
4d73f95f
BVA		 3123 if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue
3124 && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3125 pr_warn("cmd_per_lun = %d > queue_size = %d\n",
3126 target->scsi_host->cmd_per_lun,
3127 target->scsi_host->can_queue);
3128
aef9ec39
RD		 3129out:
3130 kfree(options);
3131 return ret;
3132}
3133
ee959b00
TJ		 3134static ssize_t srp_create_target(struct device *dev,
3135 struct device_attribute *attr,
aef9ec39
RD		 3136 const char *buf, size_t count)
3137{
3138 struct srp_host *host =
ee959b00 3139 container_of(dev, struct srp_host, dev);
aef9ec39
RD		 3140 struct Scsi_Host *target_host;
3141 struct srp_target_port *target;
509c07bc 3142 struct srp_rdma_ch *ch;
d1b4289e
BVA		 3143 struct srp_device *srp_dev = host->srp_dev;
3144 struct ib_device *ibdev = srp_dev->dev;
d92c0da7
BVA		 3145 int ret, node_idx, node, cpu, i;
3146 bool multich = false;
aef9ec39
RD		 3147
3148 target_host = scsi_host_alloc(&srp_template,
3149 sizeof (struct srp_target_port));
3150 if (!target_host)
3151 return -ENOMEM;
3152
49248644 3153 target_host->transportt = ib_srp_transport_template;
fd1b6c4a
BVA		 3154 target_host->max_channel = 0;
3155 target_host->max_id = 1;
985aa495 3156 target_host->max_lun = -1LL;
3c8edf0e 3157 target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb;
5f068992 3158
aef9ec39 3159 target = host_to_target(target_host);
aef9ec39 3160
49248644
DD		 3161 target->io_class = SRP_REV16A_IB_IO_CLASS;
3162 target->scsi_host = target_host;
3163 target->srp_host = host;
e6bf5f48 3164 target->lkey = host->srp_dev->pd->local_dma_lkey;
03f6fb93 3165 target->global_mr = host->srp_dev->global_mr;
49248644 3166 target->cmd_sg_cnt = cmd_sg_entries;
c07d424d
DD		 3167 target->sg_tablesize = indirect_sg_entries ? : cmd_sg_entries;
3168 target->allow_ext_sg = allow_ext_sg;
7bb312e4 3169 target->tl_retry_count = 7;
4d73f95f 3170 target->queue_size = SRP_DEFAULT_QUEUE_SIZE;
aef9ec39 3171
34aa654e
BVA		 3172 /*
3173 * Avoid that the SCSI host can be removed by srp_remove_target()
3174 * before this function returns.
3175 */
3176 scsi_host_get(target->scsi_host);
3177
2d7091bc
BVA		 3178 mutex_lock(&host->add_target_mutex);
3179
aef9ec39
RD		 3180 ret = srp_parse_options(buf, target);
3181 if (ret)
fb49c8bb 3182 goto out;
aef9ec39 3183
77f2c1a4
BVA		 3184 ret = scsi_init_shared_tag_map(target_host, target_host->can_queue);
3185 if (ret)
fb49c8bb 3186 goto out;
77f2c1a4 3187
4d73f95f
BVA		 3188 target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE;
3189
96fc248a
BVA		 3190 if (!srp_conn_unique(target->srp_host, target)) {
3191 shost_printk(KERN_INFO, target->scsi_host,
3192 PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
3193 be64_to_cpu(target->id_ext),
3194 be64_to_cpu(target->ioc_guid),
3195 be64_to_cpu(target->initiator_ext));
3196 ret = -EEXIST;
fb49c8bb 3197 goto out;
96fc248a
BVA		 3198 }
3199
5cfb1782 3200 if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg &&
d1b4289e 3201 target->cmd_sg_cnt < target->sg_tablesize) {
5cfb1782 3202 pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
c07d424d
DD		 3203 target->sg_tablesize = target->cmd_sg_cnt;
3204 }
3205
3206 target_host->sg_tablesize = target->sg_tablesize;
3207 target->indirect_size = target->sg_tablesize *
3208 sizeof (struct srp_direct_buf);
49248644
DD		 3209 target->max_iu_len = sizeof (struct srp_cmd) +
3210 sizeof (struct srp_indirect_buf) +
3211 target->cmd_sg_cnt * sizeof (struct srp_direct_buf);
3212
c1120f89 3213 INIT_WORK(&target->tl_err_work, srp_tl_err_work);
ef6c49d8 3214 INIT_WORK(&target->remove_work, srp_remove_work);
8f26c9ff 3215 spin_lock_init(&target->lock);
747fe000 3216 ret = ib_query_gid(ibdev, host->port, 0, &target->sgid);
2088ca66 3217 if (ret)
fb49c8bb 3218 goto out;
aef9ec39 3219
d92c0da7
BVA		 3220 ret = -ENOMEM;
3221 target->ch_count = max_t(unsigned, num_online_nodes(),
3222 min(ch_count ? :
3223 min(4 * num_online_nodes(),
3224 ibdev->num_comp_vectors),
3225 num_online_cpus()));
3226 target->ch = kcalloc(target->ch_count, sizeof(*target->ch),
3227 GFP_KERNEL);
3228 if (!target->ch)
fb49c8bb 3229 goto out;
aef9ec39 3230
d92c0da7
BVA		 3231 node_idx = 0;
3232 for_each_online_node(node) {
3233 const int ch_start = (node_idx * target->ch_count /
3234 num_online_nodes());
3235 const int ch_end = ((node_idx + 1) * target->ch_count /
3236 num_online_nodes());
3237 const int cv_start = (node_idx * ibdev->num_comp_vectors /
3238 num_online_nodes() + target->comp_vector)
3239 % ibdev->num_comp_vectors;
3240 const int cv_end = ((node_idx + 1) * ibdev->num_comp_vectors /
3241 num_online_nodes() + target->comp_vector)
3242 % ibdev->num_comp_vectors;
3243 int cpu_idx = 0;
3244
3245 for_each_online_cpu(cpu) {
3246 if (cpu_to_node(cpu) != node)
3247 continue;
3248 if (ch_start + cpu_idx >= ch_end)
3249 continue;
3250 ch = &target->ch[ch_start + cpu_idx];
3251 ch->target = target;
3252 ch->comp_vector = cv_start == cv_end ? cv_start :
3253 cv_start + cpu_idx % (cv_end - cv_start);
3254 spin_lock_init(&ch->lock);
3255 INIT_LIST_HEAD(&ch->free_tx);
3256 ret = srp_new_cm_id(ch);
3257 if (ret)
3258 goto err_disconnect;
aef9ec39 3259
d92c0da7
BVA		 3260 ret = srp_create_ch_ib(ch);
3261 if (ret)
3262 goto err_disconnect;
3263
3264 ret = srp_alloc_req_data(ch);
3265 if (ret)
3266 goto err_disconnect;
3267
3268 ret = srp_connect_ch(ch, multich);
3269 if (ret) {
3270 shost_printk(KERN_ERR, target->scsi_host,
3271 PFX "Connection %d/%d failed\n",
3272 ch_start + cpu_idx,
3273 target->ch_count);
3274 if (node_idx == 0 && cpu_idx == 0) {
3275 goto err_disconnect;
3276 } else {
3277 srp_free_ch_ib(target, ch);
3278 srp_free_req_data(target, ch);
3279 target->ch_count = ch - target->ch;
c257ea6f 3280 goto connected;
d92c0da7
BVA		 3281 }
3282 }
3283
3284 multich = true;
3285 cpu_idx++;
3286 }
3287 node_idx++;
aef9ec39
RD		 3288 }
3289
c257ea6f 3290connected:
d92c0da7
BVA		 3291 target->scsi_host->nr_hw_queues = target->ch_count;
3292
aef9ec39
RD		 3293 ret = srp_add_target(host, target);
3294 if (ret)
3295 goto err_disconnect;
3296
34aa654e
BVA		 3297 if (target->state != SRP_TARGET_REMOVED) {
3298 shost_printk(KERN_DEBUG, target->scsi_host, PFX
3299 "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n",
3300 be64_to_cpu(target->id_ext),
3301 be64_to_cpu(target->ioc_guid),
747fe000 3302 be16_to_cpu(target->pkey),
34aa654e 3303 be64_to_cpu(target->service_id),
747fe000 3304 target->sgid.raw, target->orig_dgid.raw);
34aa654e 3305 }
e7ffde01 3306
2d7091bc
BVA		 3307 ret = count;
3308
3309out:
3310 mutex_unlock(&host->add_target_mutex);
34aa654e
BVA		 3311
3312 scsi_host_put(target->scsi_host);
bc44bd1d
BVA		 3313 if (ret < 0)
3314 scsi_host_put(target->scsi_host);
34aa654e 3315
2d7091bc 3316 return ret;
aef9ec39
RD		 3317
3318err_disconnect:
3319 srp_disconnect_target(target);
3320
d92c0da7
BVA		 3321 for (i = 0; i < target->ch_count; i++) {
3322 ch = &target->ch[i];
3323 srp_free_ch_ib(target, ch);
3324 srp_free_req_data(target, ch);
3325 }
aef9ec39 3326
d92c0da7 3327 kfree(target->ch);
2d7091bc 3328 goto out;
aef9ec39
RD		 3329}
3330
ee959b00 3331static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target);
aef9ec39 3332
ee959b00
TJ		 3333static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
3334 char *buf)
aef9ec39 3335{
ee959b00 3336 struct srp_host *host = container_of(dev, struct srp_host, dev);
aef9ec39 3337
05321937 3338 return sprintf(buf, "%s\n", host->srp_dev->dev->name);
aef9ec39
RD		 3339}
3340
ee959b00 3341static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
aef9ec39 3342
ee959b00
TJ		 3343static ssize_t show_port(struct device *dev, struct device_attribute *attr,
3344 char *buf)
aef9ec39 3345{
ee959b00 3346 struct srp_host *host = container_of(dev, struct srp_host, dev);
aef9ec39
RD		 3347
3348 return sprintf(buf, "%d\n", host->port);
3349}
3350
ee959b00 3351static DEVICE_ATTR(port, S_IRUGO, show_port, NULL);
aef9ec39 3352
f5358a17 3353static struct srp_host *srp_add_port(struct srp_device *device, u8 port)
aef9ec39
RD		 3354{
3355 struct srp_host *host;
3356
3357 host = kzalloc(sizeof *host, GFP_KERNEL);
3358 if (!host)
3359 return NULL;
3360
3361 INIT_LIST_HEAD(&host->target_list);
b3589fd4 3362 spin_lock_init(&host->target_lock);
aef9ec39 3363 init_completion(&host->released);
2d7091bc 3364 mutex_init(&host->add_target_mutex);
05321937 3365 host->srp_dev = device;
aef9ec39
RD		 3366 host->port = port;
3367
ee959b00
TJ		 3368 host->dev.class = &srp_class;
3369 host->dev.parent = device->dev->dma_device;
d927e38c 3370 dev_set_name(&host->dev, "srp-%s-%d", device->dev->name, port);
aef9ec39 3371
ee959b00 3372 if (device_register(&host->dev))
f5358a17 3373 goto free_host;
ee959b00 3374 if (device_create_file(&host->dev, &dev_attr_add_target))
aef9ec39 3375 goto err_class;
ee959b00 3376 if (device_create_file(&host->dev, &dev_attr_ibdev))
aef9ec39 3377 goto err_class;
ee959b00 3378 if (device_create_file(&host->dev, &dev_attr_port))
aef9ec39
RD		 3379 goto err_class;
3380
3381 return host;
3382
3383err_class:
ee959b00 3384 device_unregister(&host->dev);
aef9ec39 3385
f5358a17 3386free_host:
aef9ec39
RD		 3387 kfree(host);
3388
3389 return NULL;
3390}
3391
3392static void srp_add_one(struct ib_device *device)
3393{
f5358a17
RD		 3394 struct srp_device *srp_dev;
3395 struct ib_device_attr *dev_attr;
aef9ec39 3396 struct srp_host *host;
4139032b 3397 int mr_page_shift, p;
52ede08f 3398 u64 max_pages_per_mr;
aef9ec39 3399
f5358a17
RD		 3400 dev_attr = kmalloc(sizeof *dev_attr, GFP_KERNEL);
3401 if (!dev_attr)
cf311cd4 3402 return;
aef9ec39 3403
f5358a17 3404 if (ib_query_device(device, dev_attr)) {
e0bda7d8 3405 pr_warn("Query device failed for %s\n", device->name);
f5358a17
RD		 3406 goto free_attr;
3407 }
3408
3409 srp_dev = kmalloc(sizeof *srp_dev, GFP_KERNEL);
3410 if (!srp_dev)
3411 goto free_attr;
3412
d1b4289e
BVA		 3413 srp_dev->has_fmr = (device->alloc_fmr && device->dealloc_fmr &&
3414 device->map_phys_fmr && device->unmap_fmr);
5cfb1782
BVA		 3415 srp_dev->has_fr = (dev_attr->device_cap_flags &
3416 IB_DEVICE_MEM_MGT_EXTENSIONS);
3417 if (!srp_dev->has_fmr && !srp_dev->has_fr)
3418 dev_warn(&device->dev, "neither FMR nor FR is supported\n");
3419
3420 srp_dev->use_fast_reg = (srp_dev->has_fr &&
3421 (!srp_dev->has_fmr || prefer_fr));
002f1567 3422 srp_dev->use_fmr = !srp_dev->use_fast_reg && srp_dev->has_fmr;
d1b4289e 3423
f5358a17
RD		 3424 /*
3425 * Use the smallest page size supported by the HCA, down to a
8f26c9ff
DD		 3426 * minimum of 4096 bytes. We're unlikely to build large sglists
3427 * out of smaller entries.
f5358a17 3428 */
52ede08f
BVA		 3429 mr_page_shift = max(12, ffs(dev_attr->page_size_cap) - 1);
3430 srp_dev->mr_page_size = 1 << mr_page_shift;
3431 srp_dev->mr_page_mask = ~((u64) srp_dev->mr_page_size - 1);
3432 max_pages_per_mr = dev_attr->max_mr_size;
3433 do_div(max_pages_per_mr, srp_dev->mr_page_size);
3434 srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
3435 max_pages_per_mr);
5cfb1782
BVA		 3436 if (srp_dev->use_fast_reg) {
3437 srp_dev->max_pages_per_mr =
3438 min_t(u32, srp_dev->max_pages_per_mr,
3439 dev_attr->max_fast_reg_page_list_len);
3440 }
52ede08f
BVA		 3441 srp_dev->mr_max_size = srp_dev->mr_page_size *
3442 srp_dev->max_pages_per_mr;
5cfb1782 3443 pr_debug("%s: mr_page_shift = %d, dev_attr->max_mr_size = %#llx, dev_attr->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
52ede08f 3444 device->name, mr_page_shift, dev_attr->max_mr_size,
5cfb1782 3445 dev_attr->max_fast_reg_page_list_len,
52ede08f 3446 srp_dev->max_pages_per_mr, srp_dev->mr_max_size);
f5358a17
RD		 3447
3448 INIT_LIST_HEAD(&srp_dev->dev_list);
3449
3450 srp_dev->dev = device;
3451 srp_dev->pd = ib_alloc_pd(device);
3452 if (IS_ERR(srp_dev->pd))
3453 goto free_dev;
3454
03f6fb93
BVA		 3455 if (!register_always || (!srp_dev->has_fmr && !srp_dev->has_fr)) {
3456 srp_dev->global_mr = ib_get_dma_mr(srp_dev->pd,
3457 IB_ACCESS_LOCAL_WRITE |
3458 IB_ACCESS_REMOTE_READ |
3459 IB_ACCESS_REMOTE_WRITE);
3460 if (IS_ERR(srp_dev->global_mr))
3461 goto err_pd;
3462 } else {
3463 srp_dev->global_mr = NULL;
3464 }
f5358a17 3465
4139032b 3466 for (p = rdma_start_port(device); p <= rdma_end_port(device); ++p) {
f5358a17 3467 host = srp_add_port(srp_dev, p);
aef9ec39 3468 if (host)
f5358a17 3469 list_add_tail(&host->list, &srp_dev->dev_list);
aef9ec39
RD		 3470 }
3471
f5358a17
RD		 3472 ib_set_client_data(device, &srp_client, srp_dev);
3473
3474 goto free_attr;
3475
3476err_pd:
3477 ib_dealloc_pd(srp_dev->pd);
3478
3479free_dev:
3480 kfree(srp_dev);
3481
3482free_attr:
3483 kfree(dev_attr);
aef9ec39
RD		 3484}
3485
7c1eb45a 3486static void srp_remove_one(struct ib_device *device, void *client_data)
aef9ec39 3487{
f5358a17 3488 struct srp_device *srp_dev;
aef9ec39 3489 struct srp_host *host, *tmp_host;
ef6c49d8 3490 struct srp_target_port *target;
aef9ec39 3491
7c1eb45a 3492 srp_dev = client_data;
1fe0cb84
DB		 3493 if (!srp_dev)
3494 return;
aef9ec39 3495
f5358a17 3496 list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
ee959b00 3497 device_unregister(&host->dev);
aef9ec39
RD		 3498 /*
3499 * Wait for the sysfs entry to go away, so that no new
3500 * target ports can be created.
3501 */
3502 wait_for_completion(&host->released);
3503
3504 /*
ef6c49d8 3505 * Remove all target ports.
aef9ec39 3506 */
b3589fd4 3507 spin_lock(&host->target_lock);
ef6c49d8
BVA		 3508 list_for_each_entry(target, &host->target_list, list)
3509 srp_queue_remove_work(target);
b3589fd4 3510 spin_unlock(&host->target_lock);
aef9ec39
RD		 3511
3512 /*
bcc05910 3513 * Wait for tl_err and target port removal tasks.
aef9ec39 3514 */
ef6c49d8 3515 flush_workqueue(system_long_wq);
bcc05910 3516 flush_workqueue(srp_remove_wq);
aef9ec39 3517
aef9ec39
RD		 3518 kfree(host);
3519 }
3520
03f6fb93
BVA		 3521 if (srp_dev->global_mr)
3522 ib_dereg_mr(srp_dev->global_mr);
f5358a17
RD		 3523 ib_dealloc_pd(srp_dev->pd);
3524
3525 kfree(srp_dev);
aef9ec39
RD		 3526}
3527
3236822b 3528static struct srp_function_template ib_srp_transport_functions = {
ed9b2264
BVA		 3529 .has_rport_state = true,
3530 .reset_timer_if_blocked = true,
a95cadb9 3531 .reconnect_delay = &srp_reconnect_delay,
ed9b2264
BVA		 3532 .fast_io_fail_tmo = &srp_fast_io_fail_tmo,
3533 .dev_loss_tmo = &srp_dev_loss_tmo,
3534 .reconnect = srp_rport_reconnect,
dc1bdbd9 3535 .rport_delete = srp_rport_delete,
ed9b2264 3536 .terminate_rport_io = srp_terminate_io,
3236822b
FT		 3537};
3538
aef9ec39
RD		 3539static int __init srp_init_module(void)
3540{
3541 int ret;
3542
dcb4cb85 3543 BUILD_BUG_ON(FIELD_SIZEOF(struct ib_wc, wr_id) < sizeof(void *));
dd5e6e38 3544
49248644 3545 if (srp_sg_tablesize) {
e0bda7d8 3546 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
49248644
DD		 3547 if (!cmd_sg_entries)
3548 cmd_sg_entries = srp_sg_tablesize;
3549 }
3550
3551 if (!cmd_sg_entries)
3552 cmd_sg_entries = SRP_DEF_SG_TABLESIZE;
3553
3554 if (cmd_sg_entries > 255) {
e0bda7d8 3555 pr_warn("Clamping cmd_sg_entries to 255\n");
49248644 3556 cmd_sg_entries = 255;
1e89a194
DD		 3557 }
3558
c07d424d
DD		 3559 if (!indirect_sg_entries)
3560 indirect_sg_entries = cmd_sg_entries;
3561 else if (indirect_sg_entries < cmd_sg_entries) {
e0bda7d8
BVA		 3562 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n",
3563 cmd_sg_entries);
c07d424d
DD		 3564 indirect_sg_entries = cmd_sg_entries;
3565 }
3566
bcc05910 3567 srp_remove_wq = create_workqueue("srp_remove");
da05be29
WY		 3568 if (!srp_remove_wq) {
3569 ret = -ENOMEM;
bcc05910
BVA		 3570 goto out;
3571 }
3572
3573 ret = -ENOMEM;
3236822b
FT		 3574 ib_srp_transport_template =
3575 srp_attach_transport(&ib_srp_transport_functions);
3576 if (!ib_srp_transport_template)
bcc05910 3577 goto destroy_wq;
3236822b 3578
aef9ec39
RD		 3579 ret = class_register(&srp_class);
3580 if (ret) {
e0bda7d8 3581 pr_err("couldn't register class infiniband_srp\n");
bcc05910 3582 goto release_tr;
aef9ec39
RD		 3583 }
3584
c1a0b23b
MT		 3585 ib_sa_register_client(&srp_sa_client);
3586
aef9ec39
RD		 3587 ret = ib_register_client(&srp_client);
3588 if (ret) {
e0bda7d8 3589 pr_err("couldn't register IB client\n");
bcc05910 3590 goto unreg_sa;
aef9ec39
RD		 3591 }
3592
bcc05910
BVA		 3593out:
3594 return ret;
3595
3596unreg_sa:
3597 ib_sa_unregister_client(&srp_sa_client);
3598 class_unregister(&srp_class);
3599
3600release_tr:
3601 srp_release_transport(ib_srp_transport_template);
3602
3603destroy_wq:
3604 destroy_workqueue(srp_remove_wq);
3605 goto out;
aef9ec39
RD		 3606}
3607
3608static void __exit srp_cleanup_module(void)
3609{
3610 ib_unregister_client(&srp_client);
c1a0b23b 3611 ib_sa_unregister_client(&srp_sa_client);
aef9ec39 3612 class_unregister(&srp_class);
3236822b 3613 srp_release_transport(ib_srp_transport_template);
bcc05910 3614 destroy_workqueue(srp_remove_wq);
aef9ec39
RD		 3615}
3616
3617module_init(srp_init_module);
3618module_exit(srp_cleanup_module);
Ubuntu Artful kernel mirror