]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/infiniband/ulp/srp/ib_srp.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
IB/srp: Fix a connection setup race
[mirror_ubuntu-artful-kernel.git] / drivers / infiniband / ulp / srp / ib_srp.c
CommitLineData
aef9ec39
RD		 1/*
2 * Copyright (c) 2005 Cisco Systems. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
aef9ec39
RD		 31 */
32
d236cd0e 33#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
e0bda7d8 34
aef9ec39
RD		 35#include <linux/module.h>
36#include <linux/init.h>
37#include <linux/slab.h>
38#include <linux/err.h>
39#include <linux/string.h>
40#include <linux/parser.h>
41#include <linux/random.h>
de25968c 42#include <linux/jiffies.h>
56b5390c 43#include <rdma/ib_cache.h>
aef9ec39 44
60063497 45#include <linux/atomic.h>
aef9ec39
RD		 46
47#include <scsi/scsi.h>
48#include <scsi/scsi_device.h>
49#include <scsi/scsi_dbg.h>
71444b97 50#include <scsi/scsi_tcq.h>
aef9ec39 51#include <scsi/srp.h>
3236822b 52#include <scsi/scsi_transport_srp.h>
aef9ec39 53
aef9ec39
RD		 54#include "ib_srp.h"
55
56#define DRV_NAME "ib_srp"
57#define PFX DRV_NAME ": "
e8ca4135
VP		 58#define DRV_VERSION "1.0"
59#define DRV_RELDATE "July 1, 2013"
aef9ec39
RD		 60
61MODULE_AUTHOR("Roland Dreier");
62MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator "
63 "v" DRV_VERSION " (" DRV_RELDATE ")");
64MODULE_LICENSE("Dual BSD/GPL");
65
49248644
DD		 66static unsigned int srp_sg_tablesize;
67static unsigned int cmd_sg_entries;
c07d424d
DD		 68static unsigned int indirect_sg_entries;
69static bool allow_ext_sg;
5cfb1782 70static bool prefer_fr;
b1b8854d 71static bool register_always;
49248644 72static int topspin_workarounds = 1;
74b0a15b 73
49248644
DD		 74module_param(srp_sg_tablesize, uint, 0444);
75MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries");
74b0a15b 76
49248644
DD		 77module_param(cmd_sg_entries, uint, 0444);
78MODULE_PARM_DESC(cmd_sg_entries,
79 "Default number of gather/scatter entries in the SRP command (default is 12, max 255)");
aef9ec39 80
c07d424d
DD		 81module_param(indirect_sg_entries, uint, 0444);
82MODULE_PARM_DESC(indirect_sg_entries,
83 "Default max number of gather/scatter entries (default is 12, max is " __stringify(SCSI_MAX_SG_CHAIN_SEGMENTS) ")");
84
85module_param(allow_ext_sg, bool, 0444);
86MODULE_PARM_DESC(allow_ext_sg,
87 "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)");
88
aef9ec39
RD		 89module_param(topspin_workarounds, int, 0444);
90MODULE_PARM_DESC(topspin_workarounds,
91 "Enable workarounds for Topspin/Cisco SRP target bugs if != 0");
92
5cfb1782
BVA		 93module_param(prefer_fr, bool, 0444);
94MODULE_PARM_DESC(prefer_fr,
95"Whether to use fast registration if both FMR and fast registration are supported");
96
b1b8854d
BVA		 97module_param(register_always, bool, 0444);
98MODULE_PARM_DESC(register_always,
99 "Use memory registration even for contiguous memory regions");
100
ed9b2264
BVA		 101static struct kernel_param_ops srp_tmo_ops;
102
a95cadb9
BVA		 103static int srp_reconnect_delay = 10;
104module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay,
105 S_IRUGO | S_IWUSR);
106MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts");
107
ed9b2264
BVA		 108static int srp_fast_io_fail_tmo = 15;
109module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo,
110 S_IRUGO | S_IWUSR);
111MODULE_PARM_DESC(fast_io_fail_tmo,
112 "Number of seconds between the observation of a transport"
113 " layer error and failing all I/O. \"off\" means that this"
114 " functionality is disabled.");
115
a95cadb9 116static int srp_dev_loss_tmo = 600;
ed9b2264
BVA		 117module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo,
118 S_IRUGO | S_IWUSR);
119MODULE_PARM_DESC(dev_loss_tmo,
120 "Maximum number of seconds that the SRP transport should"
121 " insulate transport layer errors. After this time has been"
122 " exceeded the SCSI host is removed. Should be"
123 " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT)
124 " if fast_io_fail_tmo has not been set. \"off\" means that"
125 " this functionality is disabled.");
126
d92c0da7
BVA		 127static unsigned ch_count;
128module_param(ch_count, uint, 0444);
129MODULE_PARM_DESC(ch_count,
130 "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA.");
131
aef9ec39
RD		 132static void srp_add_one(struct ib_device *device);
133static void srp_remove_one(struct ib_device *device);
509c07bc
BVA		 134static void srp_recv_completion(struct ib_cq *cq, void *ch_ptr);
135static void srp_send_completion(struct ib_cq *cq, void *ch_ptr);
aef9ec39
RD		 136static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
137
3236822b 138static struct scsi_transport_template *ib_srp_transport_template;
bcc05910 139static struct workqueue_struct *srp_remove_wq;
3236822b 140
aef9ec39
RD		 141static struct ib_client srp_client = {
142 .name = "srp",
143 .add = srp_add_one,
144 .remove = srp_remove_one
145};
146
c1a0b23b
MT		 147static struct ib_sa_client srp_sa_client;
148
ed9b2264
BVA		 149static int srp_tmo_get(char *buffer, const struct kernel_param *kp)
150{
151 int tmo = *(int *)kp->arg;
152
153 if (tmo >= 0)
154 return sprintf(buffer, "%d", tmo);
155 else
156 return sprintf(buffer, "off");
157}
158
159static int srp_tmo_set(const char *val, const struct kernel_param *kp)
160{
161 int tmo, res;
162
163 if (strncmp(val, "off", 3) != 0) {
164 res = kstrtoint(val, 0, &tmo);
165 if (res)
166 goto out;
167 } else {
168 tmo = -1;
169 }
a95cadb9
BVA		 170 if (kp->arg == &srp_reconnect_delay)
171 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo,
172 srp_dev_loss_tmo);
173 else if (kp->arg == &srp_fast_io_fail_tmo)
174 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo);
ed9b2264 175 else
a95cadb9
BVA		 176 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo,
177 tmo);
ed9b2264
BVA		 178 if (res)
179 goto out;
180 *(int *)kp->arg = tmo;
181
182out:
183 return res;
184}
185
186static struct kernel_param_ops srp_tmo_ops = {
187 .get = srp_tmo_get,
188 .set = srp_tmo_set,
189};
190
aef9ec39
RD		 191static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
192{
193 return (struct srp_target_port *) host->hostdata;
194}
195
196static const char *srp_target_info(struct Scsi_Host *host)
197{
198 return host_to_target(host)->target_name;
199}
200
5d7cbfd6
RD		 201static int srp_target_is_topspin(struct srp_target_port *target)
202{
203 static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad };
3d1ff48d 204 static const u8 cisco_oui[3] = { 0x00, 0x1b, 0x0d };
5d7cbfd6
RD		 205
206 return topspin_workarounds &&
3d1ff48d
RK		 207 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) ||
208 !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui));
5d7cbfd6
RD		 209}
210
aef9ec39
RD		 211static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size,
212 gfp_t gfp_mask,
213 enum dma_data_direction direction)
214{
215 struct srp_iu *iu;
216
217 iu = kmalloc(sizeof *iu, gfp_mask);
218 if (!iu)
219 goto out;
220
221 iu->buf = kzalloc(size, gfp_mask);
222 if (!iu->buf)
223 goto out_free_iu;
224
05321937
GKH		 225 iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size,
226 direction);
227 if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma))
aef9ec39
RD		 228 goto out_free_buf;
229
230 iu->size = size;
231 iu->direction = direction;
232
233 return iu;
234
235out_free_buf:
236 kfree(iu->buf);
237out_free_iu:
238 kfree(iu);
239out:
240 return NULL;
241}
242
243static void srp_free_iu(struct srp_host *host, struct srp_iu *iu)
244{
245 if (!iu)
246 return;
247
05321937
GKH		 248 ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size,
249 iu->direction);
aef9ec39
RD		 250 kfree(iu->buf);
251 kfree(iu);
252}
253
254static void srp_qp_event(struct ib_event *event, void *context)
255{
e0bda7d8 256 pr_debug("QP event %d\n", event->event);
aef9ec39
RD		 257}
258
259static int srp_init_qp(struct srp_target_port *target,
260 struct ib_qp *qp)
261{
262 struct ib_qp_attr *attr;
263 int ret;
264
265 attr = kmalloc(sizeof *attr, GFP_KERNEL);
266 if (!attr)
267 return -ENOMEM;
268
56b5390c
BVA		 269 ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev,
270 target->srp_host->port,
271 be16_to_cpu(target->pkey),
272 &attr->pkey_index);
aef9ec39
RD		 273 if (ret)
274 goto out;
275
276 attr->qp_state = IB_QPS_INIT;
277 attr->qp_access_flags = (IB_ACCESS_REMOTE_READ |
278 IB_ACCESS_REMOTE_WRITE);
279 attr->port_num = target->srp_host->port;
280
281 ret = ib_modify_qp(qp, attr,
282 IB_QP_STATE |
283 IB_QP_PKEY_INDEX |
284 IB_QP_ACCESS_FLAGS |
285 IB_QP_PORT);
286
287out:
288 kfree(attr);
289 return ret;
290}
291
509c07bc 292static int srp_new_cm_id(struct srp_rdma_ch *ch)
9fe4bcf4 293{
509c07bc 294 struct srp_target_port *target = ch->target;
9fe4bcf4
DD		 295 struct ib_cm_id *new_cm_id;
296
05321937 297 new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
509c07bc 298 srp_cm_handler, ch);
9fe4bcf4
DD		 299 if (IS_ERR(new_cm_id))
300 return PTR_ERR(new_cm_id);
301
509c07bc
BVA		 302 if (ch->cm_id)
303 ib_destroy_cm_id(ch->cm_id);
304 ch->cm_id = new_cm_id;
305 ch->path.sgid = target->sgid;
306 ch->path.dgid = target->orig_dgid;
307 ch->path.pkey = target->pkey;
308 ch->path.service_id = target->service_id;
9fe4bcf4
DD		 309
310 return 0;
311}
312
d1b4289e
BVA		 313static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target)
314{
315 struct srp_device *dev = target->srp_host->srp_dev;
316 struct ib_fmr_pool_param fmr_param;
317
318 memset(&fmr_param, 0, sizeof(fmr_param));
319 fmr_param.pool_size = target->scsi_host->can_queue;
320 fmr_param.dirty_watermark = fmr_param.pool_size / 4;
321 fmr_param.cache = 1;
52ede08f
BVA		 322 fmr_param.max_pages_per_fmr = dev->max_pages_per_mr;
323 fmr_param.page_shift = ilog2(dev->mr_page_size);
d1b4289e
BVA		 324 fmr_param.access = (IB_ACCESS_LOCAL_WRITE |
325 IB_ACCESS_REMOTE_WRITE |
326 IB_ACCESS_REMOTE_READ);
327
328 return ib_create_fmr_pool(dev->pd, &fmr_param);
329}
330
5cfb1782
BVA		 331/**
332 * srp_destroy_fr_pool() - free the resources owned by a pool
333 * @pool: Fast registration pool to be destroyed.
334 */
335static void srp_destroy_fr_pool(struct srp_fr_pool *pool)
336{
337 int i;
338 struct srp_fr_desc *d;
339
340 if (!pool)
341 return;
342
343 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
344 if (d->frpl)
345 ib_free_fast_reg_page_list(d->frpl);
346 if (d->mr)
347 ib_dereg_mr(d->mr);
348 }
349 kfree(pool);
350}
351
352/**
353 * srp_create_fr_pool() - allocate and initialize a pool for fast registration
354 * @device: IB device to allocate fast registration descriptors for.
355 * @pd: Protection domain associated with the FR descriptors.
356 * @pool_size: Number of descriptors to allocate.
357 * @max_page_list_len: Maximum fast registration work request page list length.
358 */
359static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
360 struct ib_pd *pd, int pool_size,
361 int max_page_list_len)
362{
363 struct srp_fr_pool *pool;
364 struct srp_fr_desc *d;
365 struct ib_mr *mr;
366 struct ib_fast_reg_page_list *frpl;
367 int i, ret = -EINVAL;
368
369 if (pool_size <= 0)
370 goto err;
371 ret = -ENOMEM;
372 pool = kzalloc(sizeof(struct srp_fr_pool) +
373 pool_size * sizeof(struct srp_fr_desc), GFP_KERNEL);
374 if (!pool)
375 goto err;
376 pool->size = pool_size;
377 pool->max_page_list_len = max_page_list_len;
378 spin_lock_init(&pool->lock);
379 INIT_LIST_HEAD(&pool->free_list);
380
381 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
382 mr = ib_alloc_fast_reg_mr(pd, max_page_list_len);
383 if (IS_ERR(mr)) {
384 ret = PTR_ERR(mr);
385 goto destroy_pool;
386 }
387 d->mr = mr;
388 frpl = ib_alloc_fast_reg_page_list(device, max_page_list_len);
389 if (IS_ERR(frpl)) {
390 ret = PTR_ERR(frpl);
391 goto destroy_pool;
392 }
393 d->frpl = frpl;
394 list_add_tail(&d->entry, &pool->free_list);
395 }
396
397out:
398 return pool;
399
400destroy_pool:
401 srp_destroy_fr_pool(pool);
402
403err:
404 pool = ERR_PTR(ret);
405 goto out;
406}
407
408/**
409 * srp_fr_pool_get() - obtain a descriptor suitable for fast registration
410 * @pool: Pool to obtain descriptor from.
411 */
412static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool)
413{
414 struct srp_fr_desc *d = NULL;
415 unsigned long flags;
416
417 spin_lock_irqsave(&pool->lock, flags);
418 if (!list_empty(&pool->free_list)) {
419 d = list_first_entry(&pool->free_list, typeof(*d), entry);
420 list_del(&d->entry);
421 }
422 spin_unlock_irqrestore(&pool->lock, flags);
423
424 return d;
425}
426
427/**
428 * srp_fr_pool_put() - put an FR descriptor back in the free list
429 * @pool: Pool the descriptor was allocated from.
430 * @desc: Pointer to an array of fast registration descriptor pointers.
431 * @n: Number of descriptors to put back.
432 *
433 * Note: The caller must already have queued an invalidation request for
434 * desc->mr->rkey before calling this function.
435 */
436static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc,
437 int n)
438{
439 unsigned long flags;
440 int i;
441
442 spin_lock_irqsave(&pool->lock, flags);
443 for (i = 0; i < n; i++)
444 list_add(&desc[i]->entry, &pool->free_list);
445 spin_unlock_irqrestore(&pool->lock, flags);
446}
447
448static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
449{
450 struct srp_device *dev = target->srp_host->srp_dev;
451
452 return srp_create_fr_pool(dev->dev, dev->pd,
453 target->scsi_host->can_queue,
454 dev->max_pages_per_mr);
455}
456
7dad6b2e
BVA		 457/**
458 * srp_destroy_qp() - destroy an RDMA queue pair
459 * @ch: SRP RDMA channel.
460 *
461 * Change a queue pair into the error state and wait until all receive
462 * completions have been processed before destroying it. This avoids that
463 * the receive completion handler can access the queue pair while it is
464 * being destroyed.
465 */
466static void srp_destroy_qp(struct srp_rdma_ch *ch)
467{
468 struct srp_target_port *target = ch->target;
469 static struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
470 static struct ib_recv_wr wr = { .wr_id = SRP_LAST_WR_ID };
471 struct ib_recv_wr *bad_wr;
472 int ret;
473
474 /* Destroying a QP and reusing ch->done is only safe if not connected */
475 WARN_ON_ONCE(target->connected);
476
477 ret = ib_modify_qp(ch->qp, &attr, IB_QP_STATE);
478 WARN_ONCE(ret, "ib_cm_init_qp_attr() returned %d\n", ret);
479 if (ret)
480 goto out;
481
482 init_completion(&ch->done);
483 ret = ib_post_recv(ch->qp, &wr, &bad_wr);
484 WARN_ONCE(ret, "ib_post_recv() returned %d\n", ret);
485 if (ret == 0)
486 wait_for_completion(&ch->done);
487
488out:
489 ib_destroy_qp(ch->qp);
490}
491
509c07bc 492static int srp_create_ch_ib(struct srp_rdma_ch *ch)
aef9ec39 493{
509c07bc 494 struct srp_target_port *target = ch->target;
62154b2e 495 struct srp_device *dev = target->srp_host->srp_dev;
aef9ec39 496 struct ib_qp_init_attr *init_attr;
73aa89ed
IR		 497 struct ib_cq *recv_cq, *send_cq;
498 struct ib_qp *qp;
d1b4289e 499 struct ib_fmr_pool *fmr_pool = NULL;
5cfb1782
BVA		 500 struct srp_fr_pool *fr_pool = NULL;
501 const int m = 1 + dev->use_fast_reg;
aef9ec39
RD		 502 int ret;
503
504 init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
505 if (!init_attr)
506 return -ENOMEM;
507
7dad6b2e 508 /* + 1 for SRP_LAST_WR_ID */
509c07bc 509 recv_cq = ib_create_cq(dev->dev, srp_recv_completion, NULL, ch,
7dad6b2e 510 target->queue_size + 1, ch->comp_vector);
73aa89ed
IR		 511 if (IS_ERR(recv_cq)) {
512 ret = PTR_ERR(recv_cq);
da9d2f07 513 goto err;
aef9ec39
RD		 514 }
515
509c07bc
BVA		 516 send_cq = ib_create_cq(dev->dev, srp_send_completion, NULL, ch,
517 m * target->queue_size, ch->comp_vector);
73aa89ed
IR		 518 if (IS_ERR(send_cq)) {
519 ret = PTR_ERR(send_cq);
da9d2f07 520 goto err_recv_cq;
9c03dc9f
BVA		 521 }
522
73aa89ed 523 ib_req_notify_cq(recv_cq, IB_CQ_NEXT_COMP);
aef9ec39
RD		 524
525 init_attr->event_handler = srp_qp_event;
5cfb1782 526 init_attr->cap.max_send_wr = m * target->queue_size;
7dad6b2e 527 init_attr->cap.max_recv_wr = target->queue_size + 1;
aef9ec39
RD		 528 init_attr->cap.max_recv_sge = 1;
529 init_attr->cap.max_send_sge = 1;
5cfb1782 530 init_attr->sq_sig_type = IB_SIGNAL_REQ_WR;
aef9ec39 531 init_attr->qp_type = IB_QPT_RC;
73aa89ed
IR		 532 init_attr->send_cq = send_cq;
533 init_attr->recv_cq = recv_cq;
aef9ec39 534
62154b2e 535 qp = ib_create_qp(dev->pd, init_attr);
73aa89ed
IR		 536 if (IS_ERR(qp)) {
537 ret = PTR_ERR(qp);
da9d2f07 538 goto err_send_cq;
aef9ec39
RD		 539 }
540
73aa89ed 541 ret = srp_init_qp(target, qp);
da9d2f07
RD		 542 if (ret)
543 goto err_qp;
aef9ec39 544
5cfb1782
BVA		 545 if (dev->use_fast_reg && dev->has_fr) {
546 fr_pool = srp_alloc_fr_pool(target);
547 if (IS_ERR(fr_pool)) {
548 ret = PTR_ERR(fr_pool);
549 shost_printk(KERN_WARNING, target->scsi_host, PFX
550 "FR pool allocation failed (%d)\n", ret);
551 goto err_qp;
552 }
509c07bc
BVA		 553 if (ch->fr_pool)
554 srp_destroy_fr_pool(ch->fr_pool);
555 ch->fr_pool = fr_pool;
5cfb1782 556 } else if (!dev->use_fast_reg && dev->has_fmr) {
d1b4289e
BVA		 557 fmr_pool = srp_alloc_fmr_pool(target);
558 if (IS_ERR(fmr_pool)) {
559 ret = PTR_ERR(fmr_pool);
560 shost_printk(KERN_WARNING, target->scsi_host, PFX
561 "FMR pool allocation failed (%d)\n", ret);
562 goto err_qp;
563 }
509c07bc
BVA		 564 if (ch->fmr_pool)
565 ib_destroy_fmr_pool(ch->fmr_pool);
566 ch->fmr_pool = fmr_pool;
d1b4289e
BVA		 567 }
568
509c07bc 569 if (ch->qp)
7dad6b2e 570 srp_destroy_qp(ch);
509c07bc
BVA		 571 if (ch->recv_cq)
572 ib_destroy_cq(ch->recv_cq);
573 if (ch->send_cq)
574 ib_destroy_cq(ch->send_cq);
73aa89ed 575
509c07bc
BVA		 576 ch->qp = qp;
577 ch->recv_cq = recv_cq;
578 ch->send_cq = send_cq;
73aa89ed 579
da9d2f07
RD		 580 kfree(init_attr);
581 return 0;
582
583err_qp:
73aa89ed 584 ib_destroy_qp(qp);
da9d2f07
RD		 585
586err_send_cq:
73aa89ed 587 ib_destroy_cq(send_cq);
da9d2f07
RD		 588
589err_recv_cq:
73aa89ed 590 ib_destroy_cq(recv_cq);
da9d2f07
RD		 591
592err:
aef9ec39
RD		 593 kfree(init_attr);
594 return ret;
595}
596
4d73f95f
BVA		 597/*
598 * Note: this function may be called without srp_alloc_iu_bufs() having been
509c07bc 599 * invoked. Hence the ch->[rt]x_ring checks.
4d73f95f 600 */
509c07bc
BVA		 601static void srp_free_ch_ib(struct srp_target_port *target,
602 struct srp_rdma_ch *ch)
aef9ec39 603{
5cfb1782 604 struct srp_device *dev = target->srp_host->srp_dev;
aef9ec39
RD		 605 int i;
606
d92c0da7
BVA		 607 if (!ch->target)
608 return;
609
509c07bc
BVA		 610 if (ch->cm_id) {
611 ib_destroy_cm_id(ch->cm_id);
612 ch->cm_id = NULL;
394c595e
BVA		 613 }
614
d92c0da7
BVA		 615 /* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */
616 if (!ch->qp)
617 return;
618
5cfb1782 619 if (dev->use_fast_reg) {
509c07bc
BVA		 620 if (ch->fr_pool)
621 srp_destroy_fr_pool(ch->fr_pool);
5cfb1782 622 } else {
509c07bc
BVA		 623 if (ch->fmr_pool)
624 ib_destroy_fmr_pool(ch->fmr_pool);
5cfb1782 625 }
7dad6b2e 626 srp_destroy_qp(ch);
509c07bc
BVA		 627 ib_destroy_cq(ch->send_cq);
628 ib_destroy_cq(ch->recv_cq);
aef9ec39 629
d92c0da7
BVA		 630 /*
631 * Avoid that the SCSI error handler tries to use this channel after
632 * it has been freed. The SCSI error handler can namely continue
633 * trying to perform recovery actions after scsi_remove_host()
634 * returned.
635 */
636 ch->target = NULL;
637
509c07bc
BVA		 638 ch->qp = NULL;
639 ch->send_cq = ch->recv_cq = NULL;
73aa89ed 640
509c07bc 641 if (ch->rx_ring) {
4d73f95f 642 for (i = 0; i < target->queue_size; ++i)
509c07bc
BVA		 643 srp_free_iu(target->srp_host, ch->rx_ring[i]);
644 kfree(ch->rx_ring);
645 ch->rx_ring = NULL;
4d73f95f 646 }
509c07bc 647 if (ch->tx_ring) {
4d73f95f 648 for (i = 0; i < target->queue_size; ++i)
509c07bc
BVA		 649 srp_free_iu(target->srp_host, ch->tx_ring[i]);
650 kfree(ch->tx_ring);
651 ch->tx_ring = NULL;
4d73f95f 652 }
aef9ec39
RD		 653}
654
655static void srp_path_rec_completion(int status,
656 struct ib_sa_path_rec *pathrec,
509c07bc 657 void *ch_ptr)
aef9ec39 658{
509c07bc
BVA		 659 struct srp_rdma_ch *ch = ch_ptr;
660 struct srp_target_port *target = ch->target;
aef9ec39 661
509c07bc 662 ch->status = status;
aef9ec39 663 if (status)
7aa54bd7
DD		 664 shost_printk(KERN_ERR, target->scsi_host,
665 PFX "Got failed path rec status %d\n", status);
aef9ec39 666 else
509c07bc
BVA		 667 ch->path = *pathrec;
668 complete(&ch->done);
aef9ec39
RD		 669}
670
509c07bc 671static int srp_lookup_path(struct srp_rdma_ch *ch)
aef9ec39 672{
509c07bc 673 struct srp_target_port *target = ch->target;
a702adce
BVA		 674 int ret;
675
509c07bc
BVA		 676 ch->path.numb_path = 1;
677
678 init_completion(&ch->done);
679
680 ch->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
681 target->srp_host->srp_dev->dev,
682 target->srp_host->port,
683 &ch->path,
684 IB_SA_PATH_REC_SERVICE_ID |
685 IB_SA_PATH_REC_DGID |
686 IB_SA_PATH_REC_SGID |
687 IB_SA_PATH_REC_NUMB_PATH |
688 IB_SA_PATH_REC_PKEY,
689 SRP_PATH_REC_TIMEOUT_MS,
690 GFP_KERNEL,
691 srp_path_rec_completion,
692 ch, &ch->path_query);
693 if (ch->path_query_id < 0)
694 return ch->path_query_id;
695
696 ret = wait_for_completion_interruptible(&ch->done);
a702adce
BVA		 697 if (ret < 0)
698 return ret;
aef9ec39 699
509c07bc 700 if (ch->status < 0)
7aa54bd7
DD		 701 shost_printk(KERN_WARNING, target->scsi_host,
702 PFX "Path record query failed\n");
aef9ec39 703
509c07bc 704 return ch->status;
aef9ec39
RD		 705}
706
d92c0da7 707static int srp_send_req(struct srp_rdma_ch *ch, bool multich)
aef9ec39 708{
509c07bc 709 struct srp_target_port *target = ch->target;
aef9ec39
RD		 710 struct {
711 struct ib_cm_req_param param;
712 struct srp_login_req priv;
713 } *req = NULL;
714 int status;
715
716 req = kzalloc(sizeof *req, GFP_KERNEL);
717 if (!req)
718 return -ENOMEM;
719
509c07bc 720 req->param.primary_path = &ch->path;
aef9ec39
RD		 721 req->param.alternate_path = NULL;
722 req->param.service_id = target->service_id;
509c07bc
BVA		 723 req->param.qp_num = ch->qp->qp_num;
724 req->param.qp_type = ch->qp->qp_type;
aef9ec39
RD		 725 req->param.private_data = &req->priv;
726 req->param.private_data_len = sizeof req->priv;
727 req->param.flow_control = 1;
728
729 get_random_bytes(&req->param.starting_psn, 4);
730 req->param.starting_psn &= 0xffffff;
731
732 /*
733 * Pick some arbitrary defaults here; we could make these
734 * module parameters if anyone cared about setting them.
735 */
736 req->param.responder_resources = 4;
737 req->param.remote_cm_response_timeout = 20;
738 req->param.local_cm_response_timeout = 20;
7bb312e4 739 req->param.retry_count = target->tl_retry_count;
aef9ec39
RD		 740 req->param.rnr_retry_count = 7;
741 req->param.max_cm_retries = 15;
742
743 req->priv.opcode = SRP_LOGIN_REQ;
744 req->priv.tag = 0;
49248644 745 req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len);
aef9ec39
RD		 746 req->priv.req_buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
747 SRP_BUF_FORMAT_INDIRECT);
d92c0da7
BVA		 748 req->priv.req_flags = (multich ? SRP_MULTICHAN_MULTI :
749 SRP_MULTICHAN_SINGLE);
0c0450db 750 /*
3cd96564 751 * In the published SRP specification (draft rev. 16a), the
0c0450db
R		 752 * port identifier format is 8 bytes of ID extension followed
753 * by 8 bytes of GUID. Older drafts put the two halves in the
754 * opposite order, so that the GUID comes first.
755 *
756 * Targets conforming to these obsolete drafts can be
757 * recognized by the I/O Class they report.
758 */
759 if (target->io_class == SRP_REV10_IB_IO_CLASS) {
760 memcpy(req->priv.initiator_port_id,
747fe000 761 &target->sgid.global.interface_id, 8);
0c0450db 762 memcpy(req->priv.initiator_port_id + 8,
01cb9bcb 763 &target->initiator_ext, 8);
0c0450db
R		 764 memcpy(req->priv.target_port_id, &target->ioc_guid, 8);
765 memcpy(req->priv.target_port_id + 8, &target->id_ext, 8);
766 } else {
767 memcpy(req->priv.initiator_port_id,
01cb9bcb
IR		 768 &target->initiator_ext, 8);
769 memcpy(req->priv.initiator_port_id + 8,
747fe000 770 &target->sgid.global.interface_id, 8);
0c0450db
R		 771 memcpy(req->priv.target_port_id, &target->id_ext, 8);
772 memcpy(req->priv.target_port_id + 8, &target->ioc_guid, 8);
773 }
774
aef9ec39
RD		 775 /*
776 * Topspin/Cisco SRP targets will reject our login unless we
01cb9bcb
IR		 777 * zero out the first 8 bytes of our initiator port ID and set
778 * the second 8 bytes to the local node GUID.
aef9ec39 779 */
5d7cbfd6 780 if (srp_target_is_topspin(target)) {
7aa54bd7
DD		 781 shost_printk(KERN_DEBUG, target->scsi_host,
782 PFX "Topspin/Cisco initiator port ID workaround "
783 "activated for target GUID %016llx\n",
784 (unsigned long long) be64_to_cpu(target->ioc_guid));
aef9ec39 785 memset(req->priv.initiator_port_id, 0, 8);
01cb9bcb 786 memcpy(req->priv.initiator_port_id + 8,
05321937 787 &target->srp_host->srp_dev->dev->node_guid, 8);
aef9ec39 788 }
aef9ec39 789
509c07bc 790 status = ib_send_cm_req(ch->cm_id, &req->param);
aef9ec39
RD		 791
792 kfree(req);
793
794 return status;
795}
796
ef6c49d8
BVA		 797static bool srp_queue_remove_work(struct srp_target_port *target)
798{
799 bool changed = false;
800
801 spin_lock_irq(&target->lock);
802 if (target->state != SRP_TARGET_REMOVED) {
803 target->state = SRP_TARGET_REMOVED;
804 changed = true;
805 }
806 spin_unlock_irq(&target->lock);
807
808 if (changed)
bcc05910 809 queue_work(srp_remove_wq, &target->remove_work);
ef6c49d8
BVA		 810
811 return changed;
812}
813
294c875a
BVA		 814static bool srp_change_conn_state(struct srp_target_port *target,
815 bool connected)
816{
817 bool changed = false;
818
819 spin_lock_irq(&target->lock);
820 if (target->connected != connected) {
821 target->connected = connected;
822 changed = true;
823 }
824 spin_unlock_irq(&target->lock);
825
826 return changed;
827}
828
aef9ec39
RD		 829static void srp_disconnect_target(struct srp_target_port *target)
830{
d92c0da7
BVA		 831 struct srp_rdma_ch *ch;
832 int i;
509c07bc 833
294c875a
BVA		 834 if (srp_change_conn_state(target, false)) {
835 /* XXX should send SRP_I_LOGOUT request */
aef9ec39 836
d92c0da7
BVA		 837 for (i = 0; i < target->ch_count; i++) {
838 ch = &target->ch[i];
839 if (ch->cm_id && ib_send_cm_dreq(ch->cm_id, NULL, 0)) {
840 shost_printk(KERN_DEBUG, target->scsi_host,
841 PFX "Sending CM DREQ failed\n");
842 }
294c875a 843 }
e6581056 844 }
aef9ec39
RD		 845}
846
509c07bc
BVA		 847static void srp_free_req_data(struct srp_target_port *target,
848 struct srp_rdma_ch *ch)
8f26c9ff 849{
5cfb1782
BVA		 850 struct srp_device *dev = target->srp_host->srp_dev;
851 struct ib_device *ibdev = dev->dev;
8f26c9ff
DD		 852 struct srp_request *req;
853 int i;
854
d92c0da7 855 if (!ch->target || !ch->req_ring)
4d73f95f
BVA		 856 return;
857
858 for (i = 0; i < target->req_ring_size; ++i) {
509c07bc 859 req = &ch->req_ring[i];
5cfb1782
BVA		 860 if (dev->use_fast_reg)
861 kfree(req->fr_list);
862 else
863 kfree(req->fmr_list);
8f26c9ff 864 kfree(req->map_page);
c07d424d
DD		 865 if (req->indirect_dma_addr) {
866 ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
867 target->indirect_size,
868 DMA_TO_DEVICE);
869 }
870 kfree(req->indirect_desc);
8f26c9ff 871 }
4d73f95f 872
509c07bc
BVA		 873 kfree(ch->req_ring);
874 ch->req_ring = NULL;
8f26c9ff
DD		 875}
876
509c07bc 877static int srp_alloc_req_data(struct srp_rdma_ch *ch)
b81d00bd 878{
509c07bc 879 struct srp_target_port *target = ch->target;
b81d00bd
BVA		 880 struct srp_device *srp_dev = target->srp_host->srp_dev;
881 struct ib_device *ibdev = srp_dev->dev;
882 struct srp_request *req;
5cfb1782 883 void *mr_list;
b81d00bd
BVA		 884 dma_addr_t dma_addr;
885 int i, ret = -ENOMEM;
886
509c07bc
BVA		 887 ch->req_ring = kcalloc(target->req_ring_size, sizeof(*ch->req_ring),
888 GFP_KERNEL);
889 if (!ch->req_ring)
4d73f95f
BVA		 890 goto out;
891
892 for (i = 0; i < target->req_ring_size; ++i) {
509c07bc 893 req = &ch->req_ring[i];
5cfb1782
BVA		 894 mr_list = kmalloc(target->cmd_sg_cnt * sizeof(void *),
895 GFP_KERNEL);
896 if (!mr_list)
897 goto out;
898 if (srp_dev->use_fast_reg)
899 req->fr_list = mr_list;
900 else
901 req->fmr_list = mr_list;
52ede08f 902 req->map_page = kmalloc(srp_dev->max_pages_per_mr *
d1b4289e 903 sizeof(void *), GFP_KERNEL);
5cfb1782
BVA		 904 if (!req->map_page)
905 goto out;
b81d00bd 906 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
5cfb1782 907 if (!req->indirect_desc)
b81d00bd
BVA		 908 goto out;
909
910 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
911 target->indirect_size,
912 DMA_TO_DEVICE);
913 if (ib_dma_mapping_error(ibdev, dma_addr))
914 goto out;
915
916 req->indirect_dma_addr = dma_addr;
b81d00bd
BVA		 917 }
918 ret = 0;
919
920out:
921 return ret;
922}
923
683b159a
BVA		 924/**
925 * srp_del_scsi_host_attr() - Remove attributes defined in the host template.
926 * @shost: SCSI host whose attributes to remove from sysfs.
927 *
928 * Note: Any attributes defined in the host template and that did not exist
929 * before invocation of this function will be ignored.
930 */
931static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
932{
933 struct device_attribute **attr;
934
935 for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr)
936 device_remove_file(&shost->shost_dev, *attr);
937}
938
ee12d6a8
BVA		 939static void srp_remove_target(struct srp_target_port *target)
940{
d92c0da7
BVA		 941 struct srp_rdma_ch *ch;
942 int i;
509c07bc 943
ef6c49d8
BVA		 944 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
945
ee12d6a8 946 srp_del_scsi_host_attr(target->scsi_host);
9dd69a60 947 srp_rport_get(target->rport);
ee12d6a8
BVA		 948 srp_remove_host(target->scsi_host);
949 scsi_remove_host(target->scsi_host);
93079162 950 srp_stop_rport_timers(target->rport);
ef6c49d8 951 srp_disconnect_target(target);
d92c0da7
BVA		 952 for (i = 0; i < target->ch_count; i++) {
953 ch = &target->ch[i];
954 srp_free_ch_ib(target, ch);
955 }
c1120f89 956 cancel_work_sync(&target->tl_err_work);
9dd69a60 957 srp_rport_put(target->rport);
d92c0da7
BVA		 958 for (i = 0; i < target->ch_count; i++) {
959 ch = &target->ch[i];
960 srp_free_req_data(target, ch);
961 }
962 kfree(target->ch);
963 target->ch = NULL;
65d7dd2f
VP		 964
965 spin_lock(&target->srp_host->target_lock);
966 list_del(&target->list);
967 spin_unlock(&target->srp_host->target_lock);
968
ee12d6a8
BVA		 969 scsi_host_put(target->scsi_host);
970}
971
c4028958 972static void srp_remove_work(struct work_struct *work)
aef9ec39 973{
c4028958 974 struct srp_target_port *target =
ef6c49d8 975 container_of(work, struct srp_target_port, remove_work);
aef9ec39 976
ef6c49d8 977 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
aef9ec39 978
96fc248a 979 srp_remove_target(target);
aef9ec39
RD		 980}
981
dc1bdbd9
BVA		 982static void srp_rport_delete(struct srp_rport *rport)
983{
984 struct srp_target_port *target = rport->lld_data;
985
986 srp_queue_remove_work(target);
987}
988
d92c0da7 989static int srp_connect_ch(struct srp_rdma_ch *ch, bool multich)
aef9ec39 990{
509c07bc 991 struct srp_target_port *target = ch->target;
aef9ec39
RD		 992 int ret;
993
d92c0da7 994 WARN_ON_ONCE(!multich && target->connected);
294c875a 995
509c07bc 996 ret = srp_lookup_path(ch);
aef9ec39
RD		 997 if (ret)
998 return ret;
999
1000 while (1) {
509c07bc 1001 init_completion(&ch->done);
d92c0da7 1002 ret = srp_send_req(ch, multich);
aef9ec39
RD		 1003 if (ret)
1004 return ret;
509c07bc 1005 ret = wait_for_completion_interruptible(&ch->done);
a702adce
BVA		 1006 if (ret < 0)
1007 return ret;
aef9ec39
RD		 1008
1009 /*
1010 * The CM event handling code will set status to
1011 * SRP_PORT_REDIRECT if we get a port redirect REJ
1012 * back, or SRP_DLID_REDIRECT if we get a lid/qp
1013 * redirect REJ back.
1014 */
509c07bc 1015 switch (ch->status) {
aef9ec39 1016 case 0:
294c875a 1017 srp_change_conn_state(target, true);
aef9ec39
RD		 1018 return 0;
1019
1020 case SRP_PORT_REDIRECT:
509c07bc 1021 ret = srp_lookup_path(ch);
aef9ec39
RD		 1022 if (ret)
1023 return ret;
1024 break;
1025
1026 case SRP_DLID_REDIRECT:
1027 break;
1028
9fe4bcf4 1029 case SRP_STALE_CONN:
9fe4bcf4 1030 shost_printk(KERN_ERR, target->scsi_host, PFX
205619f2 1031 "giving up on stale connection\n");
509c07bc
BVA		 1032 ch->status = -ECONNRESET;
1033 return ch->status;
9fe4bcf4 1034
aef9ec39 1035 default:
509c07bc 1036 return ch->status;
aef9ec39
RD		 1037 }
1038 }
1039}
1040
509c07bc 1041static int srp_inv_rkey(struct srp_rdma_ch *ch, u32 rkey)
5cfb1782
BVA		 1042{
1043 struct ib_send_wr *bad_wr;
1044 struct ib_send_wr wr = {
1045 .opcode = IB_WR_LOCAL_INV,
1046 .wr_id = LOCAL_INV_WR_ID_MASK,
1047 .next = NULL,
1048 .num_sge = 0,
1049 .send_flags = 0,
1050 .ex.invalidate_rkey = rkey,
1051 };
1052
509c07bc 1053 return ib_post_send(ch->qp, &wr, &bad_wr);
5cfb1782
BVA		 1054}
1055
d945e1df 1056static void srp_unmap_data(struct scsi_cmnd *scmnd,
509c07bc 1057 struct srp_rdma_ch *ch,
d945e1df
RD		 1058 struct srp_request *req)
1059{
509c07bc 1060 struct srp_target_port *target = ch->target;
5cfb1782
BVA		 1061 struct srp_device *dev = target->srp_host->srp_dev;
1062 struct ib_device *ibdev = dev->dev;
1063 int i, res;
8f26c9ff 1064
bb350d1d 1065 if (!scsi_sglist(scmnd) ||
d945e1df
RD		 1066 (scmnd->sc_data_direction != DMA_TO_DEVICE &&
1067 scmnd->sc_data_direction != DMA_FROM_DEVICE))
1068 return;
1069
5cfb1782
BVA		 1070 if (dev->use_fast_reg) {
1071 struct srp_fr_desc **pfr;
1072
1073 for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) {
509c07bc 1074 res = srp_inv_rkey(ch, (*pfr)->mr->rkey);
5cfb1782
BVA		 1075 if (res < 0) {
1076 shost_printk(KERN_ERR, target->scsi_host, PFX
1077 "Queueing INV WR for rkey %#x failed (%d)\n",
1078 (*pfr)->mr->rkey, res);
1079 queue_work(system_long_wq,
1080 &target->tl_err_work);
1081 }
1082 }
1083 if (req->nmdesc)
509c07bc 1084 srp_fr_pool_put(ch->fr_pool, req->fr_list,
5cfb1782
BVA		 1085 req->nmdesc);
1086 } else {
1087 struct ib_pool_fmr **pfmr;
1088
1089 for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++)
1090 ib_fmr_pool_unmap(*pfmr);
1091 }
f5358a17 1092
8f26c9ff
DD		 1093 ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
1094 scmnd->sc_data_direction);
d945e1df
RD		 1095}
1096
22032991
BVA		 1097/**
1098 * srp_claim_req - Take ownership of the scmnd associated with a request.
509c07bc 1099 * @ch: SRP RDMA channel.
22032991 1100 * @req: SRP request.
b3fe628d 1101 * @sdev: If not NULL, only take ownership for this SCSI device.
22032991
BVA		 1102 * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take
1103 * ownership of @req->scmnd if it equals @scmnd.
1104 *
1105 * Return value:
1106 * Either NULL or a pointer to the SCSI command the caller became owner of.
1107 */
509c07bc 1108static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch,
22032991 1109 struct srp_request *req,
b3fe628d 1110 struct scsi_device *sdev,
22032991
BVA		 1111 struct scsi_cmnd *scmnd)
1112{
1113 unsigned long flags;
1114
509c07bc 1115 spin_lock_irqsave(&ch->lock, flags);
b3fe628d
BVA		 1116 if (req->scmnd &&
1117 (!sdev || req->scmnd->device == sdev) &&
1118 (!scmnd || req->scmnd == scmnd)) {
22032991
BVA		 1119 scmnd = req->scmnd;
1120 req->scmnd = NULL;
22032991
BVA		 1121 } else {
1122 scmnd = NULL;
1123 }
509c07bc 1124 spin_unlock_irqrestore(&ch->lock, flags);
22032991
BVA		 1125
1126 return scmnd;
1127}
1128
1129/**
1130 * srp_free_req() - Unmap data and add request to the free request list.
509c07bc 1131 * @ch: SRP RDMA channel.
af24663b
BVA		 1132 * @req: Request to be freed.
1133 * @scmnd: SCSI command associated with @req.
1134 * @req_lim_delta: Amount to be added to @target->req_lim.
22032991 1135 */
509c07bc
BVA		 1136static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req,
1137 struct scsi_cmnd *scmnd, s32 req_lim_delta)
526b4caa 1138{
94a9174c
BVA		 1139 unsigned long flags;
1140
509c07bc 1141 srp_unmap_data(scmnd, ch, req);
22032991 1142
509c07bc
BVA		 1143 spin_lock_irqsave(&ch->lock, flags);
1144 ch->req_lim += req_lim_delta;
509c07bc 1145 spin_unlock_irqrestore(&ch->lock, flags);
526b4caa
IR		 1146}
1147
509c07bc
BVA		 1148static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req,
1149 struct scsi_device *sdev, int result)
526b4caa 1150{
509c07bc 1151 struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL);
22032991
BVA		 1152
1153 if (scmnd) {
509c07bc 1154 srp_free_req(ch, req, scmnd, 0);
ed9b2264 1155 scmnd->result = result;
22032991 1156 scmnd->scsi_done(scmnd);
22032991 1157 }
526b4caa
IR		 1158}
1159
ed9b2264 1160static void srp_terminate_io(struct srp_rport *rport)
aef9ec39 1161{
ed9b2264 1162 struct srp_target_port *target = rport->lld_data;
d92c0da7 1163 struct srp_rdma_ch *ch;
b3fe628d
BVA		 1164 struct Scsi_Host *shost = target->scsi_host;
1165 struct scsi_device *sdev;
d92c0da7 1166 int i, j;
ed9b2264 1167
b3fe628d
BVA		 1168 /*
1169 * Invoking srp_terminate_io() while srp_queuecommand() is running
1170 * is not safe. Hence the warning statement below.
1171 */
1172 shost_for_each_device(sdev, shost)
1173 WARN_ON_ONCE(sdev->request_queue->request_fn_active);
1174
d92c0da7
BVA		 1175 for (i = 0; i < target->ch_count; i++) {
1176 ch = &target->ch[i];
509c07bc 1177
d92c0da7
BVA		 1178 for (j = 0; j < target->req_ring_size; ++j) {
1179 struct srp_request *req = &ch->req_ring[j];
1180
1181 srp_finish_req(ch, req, NULL,
1182 DID_TRANSPORT_FAILFAST << 16);
1183 }
ed9b2264
BVA		 1184 }
1185}
aef9ec39 1186
ed9b2264
BVA		 1187/*
1188 * It is up to the caller to ensure that srp_rport_reconnect() calls are
1189 * serialized and that no concurrent srp_queuecommand(), srp_abort(),
1190 * srp_reset_device() or srp_reset_host() calls will occur while this function
1191 * is in progress. One way to realize that is not to call this function
1192 * directly but to call srp_reconnect_rport() instead since that last function
1193 * serializes calls of this function via rport->mutex and also blocks
1194 * srp_queuecommand() calls before invoking this function.
1195 */
1196static int srp_rport_reconnect(struct srp_rport *rport)
1197{
1198 struct srp_target_port *target = rport->lld_data;
d92c0da7
BVA		 1199 struct srp_rdma_ch *ch;
1200 int i, j, ret = 0;
1201 bool multich = false;
09be70a2 1202
aef9ec39 1203 srp_disconnect_target(target);
34aa654e
BVA		 1204
1205 if (target->state == SRP_TARGET_SCANNING)
1206 return -ENODEV;
1207
aef9ec39 1208 /*
c7c4e7ff
BVA		 1209 * Now get a new local CM ID so that we avoid confusing the target in
1210 * case things are really fouled up. Doing so also ensures that all CM
1211 * callbacks will have finished before a new QP is allocated.
aef9ec39 1212 */
d92c0da7
BVA		 1213 for (i = 0; i < target->ch_count; i++) {
1214 ch = &target->ch[i];
1215 if (!ch->target)
1216 break;
1217 ret += srp_new_cm_id(ch);
536ae14e 1218 }
d92c0da7
BVA		 1219 for (i = 0; i < target->ch_count; i++) {
1220 ch = &target->ch[i];
1221 if (!ch->target)
1222 break;
1223 for (j = 0; j < target->req_ring_size; ++j) {
1224 struct srp_request *req = &ch->req_ring[j];
aef9ec39 1225
d92c0da7
BVA		 1226 srp_finish_req(ch, req, NULL, DID_RESET << 16);
1227 }
1228 }
1229 for (i = 0; i < target->ch_count; i++) {
1230 ch = &target->ch[i];
1231 if (!ch->target)
1232 break;
1233 /*
1234 * Whether or not creating a new CM ID succeeded, create a new
1235 * QP. This guarantees that all completion callback function
1236 * invocations have finished before request resetting starts.
1237 */
1238 ret += srp_create_ch_ib(ch);
aef9ec39 1239
d92c0da7
BVA		 1240 INIT_LIST_HEAD(&ch->free_tx);
1241 for (j = 0; j < target->queue_size; ++j)
1242 list_add(&ch->tx_ring[j]->list, &ch->free_tx);
1243 }
8de9fe3a
BVA		 1244
1245 target->qp_in_error = false;
1246
d92c0da7
BVA		 1247 for (i = 0; i < target->ch_count; i++) {
1248 ch = &target->ch[i];
1249 if (ret || !ch->target) {
1250 if (i > 1)
1251 ret = 0;
1252 break;
1253 }
1254 ret = srp_connect_ch(ch, multich);
1255 multich = true;
1256 }
09be70a2 1257
ed9b2264
BVA		 1258 if (ret == 0)
1259 shost_printk(KERN_INFO, target->scsi_host,
1260 PFX "reconnect succeeded\n");
aef9ec39
RD		 1261
1262 return ret;
1263}
1264
8f26c9ff
DD		 1265static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
1266 unsigned int dma_len, u32 rkey)
f5358a17 1267{
8f26c9ff 1268 struct srp_direct_buf *desc = state->desc;
f5358a17 1269
8f26c9ff
DD		 1270 desc->va = cpu_to_be64(dma_addr);
1271 desc->key = cpu_to_be32(rkey);
1272 desc->len = cpu_to_be32(dma_len);
f5358a17 1273
8f26c9ff
DD		 1274 state->total_len += dma_len;
1275 state->desc++;
1276 state->ndesc++;
1277}
559ce8f1 1278
8f26c9ff 1279static int srp_map_finish_fmr(struct srp_map_state *state,
509c07bc 1280 struct srp_rdma_ch *ch)
8f26c9ff 1281{
8f26c9ff
DD		 1282 struct ib_pool_fmr *fmr;
1283 u64 io_addr = 0;
85507bcc 1284
509c07bc 1285 fmr = ib_fmr_pool_map_phys(ch->fmr_pool, state->pages,
8f26c9ff
DD		 1286 state->npages, io_addr);
1287 if (IS_ERR(fmr))
1288 return PTR_ERR(fmr);
f5358a17 1289
8f26c9ff 1290 *state->next_fmr++ = fmr;
52ede08f 1291 state->nmdesc++;
f5358a17 1292
52ede08f 1293 srp_map_desc(state, 0, state->dma_len, fmr->fmr->rkey);
539dde6f 1294
8f26c9ff
DD		 1295 return 0;
1296}
1297
5cfb1782 1298static int srp_map_finish_fr(struct srp_map_state *state,
509c07bc 1299 struct srp_rdma_ch *ch)
5cfb1782 1300{
509c07bc 1301 struct srp_target_port *target = ch->target;
5cfb1782
BVA		 1302 struct srp_device *dev = target->srp_host->srp_dev;
1303 struct ib_send_wr *bad_wr;
1304 struct ib_send_wr wr;
1305 struct srp_fr_desc *desc;
1306 u32 rkey;
1307
509c07bc 1308 desc = srp_fr_pool_get(ch->fr_pool);
5cfb1782
BVA		 1309 if (!desc)
1310 return -ENOMEM;
1311
1312 rkey = ib_inc_rkey(desc->mr->rkey);
1313 ib_update_fast_reg_key(desc->mr, rkey);
1314
1315 memcpy(desc->frpl->page_list, state->pages,
1316 sizeof(state->pages[0]) * state->npages);
1317
1318 memset(&wr, 0, sizeof(wr));
1319 wr.opcode = IB_WR_FAST_REG_MR;
1320 wr.wr_id = FAST_REG_WR_ID_MASK;
1321 wr.wr.fast_reg.iova_start = state->base_dma_addr;
1322 wr.wr.fast_reg.page_list = desc->frpl;
1323 wr.wr.fast_reg.page_list_len = state->npages;
1324 wr.wr.fast_reg.page_shift = ilog2(dev->mr_page_size);
1325 wr.wr.fast_reg.length = state->dma_len;
1326 wr.wr.fast_reg.access_flags = (IB_ACCESS_LOCAL_WRITE |
1327 IB_ACCESS_REMOTE_READ |
1328 IB_ACCESS_REMOTE_WRITE);
1329 wr.wr.fast_reg.rkey = desc->mr->lkey;
1330
1331 *state->next_fr++ = desc;
1332 state->nmdesc++;
1333
1334 srp_map_desc(state, state->base_dma_addr, state->dma_len,
1335 desc->mr->rkey);
1336
509c07bc 1337 return ib_post_send(ch->qp, &wr, &bad_wr);
5cfb1782
BVA		 1338}
1339
539dde6f 1340static int srp_finish_mapping(struct srp_map_state *state,
509c07bc 1341 struct srp_rdma_ch *ch)
539dde6f 1342{
509c07bc 1343 struct srp_target_port *target = ch->target;
539dde6f
BVA		 1344 int ret = 0;
1345
1346 if (state->npages == 0)
1347 return 0;
1348
b1b8854d 1349 if (state->npages == 1 && !register_always)
52ede08f 1350 srp_map_desc(state, state->base_dma_addr, state->dma_len,
539dde6f
BVA		 1351 target->rkey);
1352 else
5cfb1782 1353 ret = target->srp_host->srp_dev->use_fast_reg ?
509c07bc
BVA		 1354 srp_map_finish_fr(state, ch) :
1355 srp_map_finish_fmr(state, ch);
539dde6f
BVA		 1356
1357 if (ret == 0) {
1358 state->npages = 0;
52ede08f 1359 state->dma_len = 0;
539dde6f
BVA		 1360 }
1361
1362 return ret;
1363}
1364
8f26c9ff
DD		 1365static void srp_map_update_start(struct srp_map_state *state,
1366 struct scatterlist *sg, int sg_index,
1367 dma_addr_t dma_addr)
1368{
1369 state->unmapped_sg = sg;
1370 state->unmapped_index = sg_index;
1371 state->unmapped_addr = dma_addr;
1372}
85507bcc 1373
8f26c9ff 1374static int srp_map_sg_entry(struct srp_map_state *state,
509c07bc 1375 struct srp_rdma_ch *ch,
8f26c9ff 1376 struct scatterlist *sg, int sg_index,
5cfb1782 1377 bool use_mr)
8f26c9ff 1378{
509c07bc 1379 struct srp_target_port *target = ch->target;
8f26c9ff
DD		 1380 struct srp_device *dev = target->srp_host->srp_dev;
1381 struct ib_device *ibdev = dev->dev;
1382 dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg);
1383 unsigned int dma_len = ib_sg_dma_len(ibdev, sg);
1384 unsigned int len;
1385 int ret;
1386
1387 if (!dma_len)
1388 return 0;
1389
5cfb1782
BVA		 1390 if (!use_mr) {
1391 /*
1392 * Once we're in direct map mode for a request, we don't
1393 * go back to FMR or FR mode, so no need to update anything
8f26c9ff
DD		 1394 * other than the descriptor.
1395 */
1396 srp_map_desc(state, dma_addr, dma_len, target->rkey);
1397 return 0;
85507bcc 1398 }
f5358a17 1399
5cfb1782
BVA		 1400 /*
1401 * Since not all RDMA HW drivers support non-zero page offsets for
1402 * FMR, if we start at an offset into a page, don't merge into the
1403 * current FMR mapping. Finish it out, and use the kernel's MR for
1404 * this sg entry.
8f26c9ff 1405 */
5cfb1782
BVA		 1406 if ((!dev->use_fast_reg && dma_addr & ~dev->mr_page_mask) ||
1407 dma_len > dev->mr_max_size) {
509c07bc 1408 ret = srp_finish_mapping(state, ch);
8f26c9ff
DD		 1409 if (ret)
1410 return ret;
1411
1412 srp_map_desc(state, dma_addr, dma_len, target->rkey);
1413 srp_map_update_start(state, NULL, 0, 0);
1414 return 0;
f5358a17
RD		 1415 }
1416
5cfb1782
BVA		 1417 /*
1418 * If this is the first sg that will be mapped via FMR or via FR, save
1419 * our position. We need to know the first unmapped entry, its index,
1420 * and the first unmapped address within that entry to be able to
1421 * restart mapping after an error.
8f26c9ff
DD		 1422 */
1423 if (!state->unmapped_sg)
1424 srp_map_update_start(state, sg, sg_index, dma_addr);
f5358a17 1425
8f26c9ff 1426 while (dma_len) {
5cfb1782
BVA		 1427 unsigned offset = dma_addr & ~dev->mr_page_mask;
1428 if (state->npages == dev->max_pages_per_mr || offset != 0) {
509c07bc 1429 ret = srp_finish_mapping(state, ch);
8f26c9ff
DD		 1430 if (ret)
1431 return ret;
f5358a17 1432
8f26c9ff
DD		 1433 srp_map_update_start(state, sg, sg_index, dma_addr);
1434 }
1435
5cfb1782 1436 len = min_t(unsigned int, dma_len, dev->mr_page_size - offset);
f5358a17 1437
8f26c9ff
DD		 1438 if (!state->npages)
1439 state->base_dma_addr = dma_addr;
5cfb1782 1440 state->pages[state->npages++] = dma_addr & dev->mr_page_mask;
52ede08f 1441 state->dma_len += len;
8f26c9ff
DD		 1442 dma_addr += len;
1443 dma_len -= len;
1444 }
1445
5cfb1782
BVA		 1446 /*
1447 * If the last entry of the MR wasn't a full page, then we need to
8f26c9ff
DD		 1448 * close it out and start a new one -- we can only merge at page
1449 * boundries.
1450 */
1451 ret = 0;
52ede08f 1452 if (len != dev->mr_page_size) {
509c07bc 1453 ret = srp_finish_mapping(state, ch);
8f26c9ff
DD		 1454 if (!ret)
1455 srp_map_update_start(state, NULL, 0, 0);
1456 }
f5358a17
RD		 1457 return ret;
1458}
1459
509c07bc
BVA		 1460static int srp_map_sg(struct srp_map_state *state, struct srp_rdma_ch *ch,
1461 struct srp_request *req, struct scatterlist *scat,
1462 int count)
76bc1e1d 1463{
509c07bc 1464 struct srp_target_port *target = ch->target;
76bc1e1d
BVA		 1465 struct srp_device *dev = target->srp_host->srp_dev;
1466 struct ib_device *ibdev = dev->dev;
1467 struct scatterlist *sg;
5cfb1782
BVA		 1468 int i;
1469 bool use_mr;
76bc1e1d
BVA		 1470
1471 state->desc = req->indirect_desc;
1472 state->pages = req->map_page;
5cfb1782
BVA		 1473 if (dev->use_fast_reg) {
1474 state->next_fr = req->fr_list;
509c07bc 1475 use_mr = !!ch->fr_pool;
5cfb1782
BVA		 1476 } else {
1477 state->next_fmr = req->fmr_list;
509c07bc 1478 use_mr = !!ch->fmr_pool;
5cfb1782 1479 }
76bc1e1d
BVA		 1480
1481 for_each_sg(scat, sg, count, i) {
509c07bc 1482 if (srp_map_sg_entry(state, ch, sg, i, use_mr)) {
5cfb1782
BVA		 1483 /*
1484 * Memory registration failed, so backtrack to the
1485 * first unmapped entry and continue on without using
1486 * memory registration.
76bc1e1d
BVA		 1487 */
1488 dma_addr_t dma_addr;
1489 unsigned int dma_len;
1490
1491backtrack:
1492 sg = state->unmapped_sg;
1493 i = state->unmapped_index;
1494
1495 dma_addr = ib_sg_dma_address(ibdev, sg);
1496 dma_len = ib_sg_dma_len(ibdev, sg);
1497 dma_len -= (state->unmapped_addr - dma_addr);
1498 dma_addr = state->unmapped_addr;
5cfb1782 1499 use_mr = false;
76bc1e1d
BVA		 1500 srp_map_desc(state, dma_addr, dma_len, target->rkey);
1501 }
1502 }
1503
509c07bc 1504 if (use_mr && srp_finish_mapping(state, ch))
76bc1e1d
BVA		 1505 goto backtrack;
1506
52ede08f 1507 req->nmdesc = state->nmdesc;
5cfb1782
BVA		 1508
1509 return 0;
76bc1e1d
BVA		 1510}
1511
509c07bc 1512static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
aef9ec39
RD		 1513 struct srp_request *req)
1514{
509c07bc 1515 struct srp_target_port *target = ch->target;
76bc1e1d 1516 struct scatterlist *scat;
aef9ec39 1517 struct srp_cmd *cmd = req->cmd->buf;
76bc1e1d 1518 int len, nents, count;
85507bcc
RC		 1519 struct srp_device *dev;
1520 struct ib_device *ibdev;
8f26c9ff
DD		 1521 struct srp_map_state state;
1522 struct srp_indirect_buf *indirect_hdr;
8f26c9ff
DD		 1523 u32 table_len;
1524 u8 fmt;
aef9ec39 1525
bb350d1d 1526 if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE)
aef9ec39
RD		 1527 return sizeof (struct srp_cmd);
1528
1529 if (scmnd->sc_data_direction != DMA_FROM_DEVICE &&
1530 scmnd->sc_data_direction != DMA_TO_DEVICE) {
7aa54bd7
DD		 1531 shost_printk(KERN_WARNING, target->scsi_host,
1532 PFX "Unhandled data direction %d\n",
1533 scmnd->sc_data_direction);
aef9ec39
RD		 1534 return -EINVAL;
1535 }
1536
bb350d1d
FT		 1537 nents = scsi_sg_count(scmnd);
1538 scat = scsi_sglist(scmnd);
aef9ec39 1539
05321937 1540 dev = target->srp_host->srp_dev;
85507bcc
RC		 1541 ibdev = dev->dev;
1542
1543 count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction);
8f26c9ff
DD		 1544 if (unlikely(count == 0))
1545 return -EIO;
f5358a17
RD		 1546
1547 fmt = SRP_DATA_DESC_DIRECT;
1548 len = sizeof (struct srp_cmd) + sizeof (struct srp_direct_buf);
aef9ec39 1549
b1b8854d 1550 if (count == 1 && !register_always) {
f5358a17
RD		 1551 /*
1552 * The midlayer only generated a single gather/scatter
1553 * entry, or DMA mapping coalesced everything to a
1554 * single entry. So a direct descriptor along with
1555 * the DMA MR suffices.
1556 */
cf368713 1557 struct srp_direct_buf *buf = (void *) cmd->add_data;
aef9ec39 1558
85507bcc 1559 buf->va = cpu_to_be64(ib_sg_dma_address(ibdev, scat));
9af76271 1560 buf->key = cpu_to_be32(target->rkey);
85507bcc 1561 buf->len = cpu_to_be32(ib_sg_dma_len(ibdev, scat));
8f26c9ff 1562
52ede08f 1563 req->nmdesc = 0;
8f26c9ff
DD		 1564 goto map_complete;
1565 }
1566
5cfb1782
BVA		 1567 /*
1568 * We have more than one scatter/gather entry, so build our indirect
1569 * descriptor table, trying to merge as many entries as we can.
8f26c9ff
DD		 1570 */
1571 indirect_hdr = (void *) cmd->add_data;
1572
c07d424d
DD		 1573 ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr,
1574 target->indirect_size, DMA_TO_DEVICE);
1575
8f26c9ff 1576 memset(&state, 0, sizeof(state));
509c07bc 1577 srp_map_sg(&state, ch, req, scat, count);
cf368713 1578
c07d424d
DD		 1579 /* We've mapped the request, now pull as much of the indirect
1580 * descriptor table as we can into the command buffer. If this
1581 * target is not using an external indirect table, we are
1582 * guaranteed to fit into the command, as the SCSI layer won't
1583 * give us more S/G entries than we allow.
8f26c9ff 1584 */
8f26c9ff 1585 if (state.ndesc == 1) {
5cfb1782
BVA		 1586 /*
1587 * Memory registration collapsed the sg-list into one entry,
8f26c9ff
DD		 1588 * so use a direct descriptor.
1589 */
1590 struct srp_direct_buf *buf = (void *) cmd->add_data;
cf368713 1591
c07d424d 1592 *buf = req->indirect_desc[0];
8f26c9ff 1593 goto map_complete;
aef9ec39
RD		 1594 }
1595
c07d424d
DD		 1596 if (unlikely(target->cmd_sg_cnt < state.ndesc &&
1597 !target->allow_ext_sg)) {
1598 shost_printk(KERN_ERR, target->scsi_host,
1599 "Could not fit S/G list into SRP_CMD\n");
1600 return -EIO;
1601 }
1602
1603 count = min(state.ndesc, target->cmd_sg_cnt);
8f26c9ff
DD		 1604 table_len = state.ndesc * sizeof (struct srp_direct_buf);
1605
1606 fmt = SRP_DATA_DESC_INDIRECT;
1607 len = sizeof(struct srp_cmd) + sizeof (struct srp_indirect_buf);
c07d424d 1608 len += count * sizeof (struct srp_direct_buf);
8f26c9ff 1609
c07d424d
DD		 1610 memcpy(indirect_hdr->desc_list, req->indirect_desc,
1611 count * sizeof (struct srp_direct_buf));
8f26c9ff 1612
c07d424d 1613 indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr);
8f26c9ff
DD		 1614 indirect_hdr->table_desc.key = cpu_to_be32(target->rkey);
1615 indirect_hdr->table_desc.len = cpu_to_be32(table_len);
1616 indirect_hdr->len = cpu_to_be32(state.total_len);
1617
1618 if (scmnd->sc_data_direction == DMA_TO_DEVICE)
c07d424d 1619 cmd->data_out_desc_cnt = count;
8f26c9ff 1620 else
c07d424d
DD		 1621 cmd->data_in_desc_cnt = count;
1622
1623 ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len,
1624 DMA_TO_DEVICE);
8f26c9ff
DD		 1625
1626map_complete:
aef9ec39
RD		 1627 if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1628 cmd->buf_fmt = fmt << 4;
1629 else
1630 cmd->buf_fmt = fmt;
1631
aef9ec39
RD		 1632 return len;
1633}
1634
76c75b25
BVA		 1635/*
1636 * Return an IU and possible credit to the free pool
1637 */
509c07bc 1638static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu,
76c75b25
BVA		 1639 enum srp_iu_type iu_type)
1640{
1641 unsigned long flags;
1642
509c07bc
BVA		 1643 spin_lock_irqsave(&ch->lock, flags);
1644 list_add(&iu->list, &ch->free_tx);
76c75b25 1645 if (iu_type != SRP_IU_RSP)
509c07bc
BVA		 1646 ++ch->req_lim;
1647 spin_unlock_irqrestore(&ch->lock, flags);
76c75b25
BVA		 1648}
1649
05a1d750 1650/*
509c07bc 1651 * Must be called with ch->lock held to protect req_lim and free_tx.
e9684678 1652 * If IU is not sent, it must be returned using srp_put_tx_iu().
05a1d750
DD		 1653 *
1654 * Note:
1655 * An upper limit for the number of allocated information units for each
1656 * request type is:
1657 * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues
1658 * more than Scsi_Host.can_queue requests.
1659 * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE.
1660 * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than
1661 * one unanswered SRP request to an initiator.
1662 */
509c07bc 1663static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch,
05a1d750
DD		 1664 enum srp_iu_type iu_type)
1665{
509c07bc 1666 struct srp_target_port *target = ch->target;
05a1d750
DD		 1667 s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
1668 struct srp_iu *iu;
1669
509c07bc 1670 srp_send_completion(ch->send_cq, ch);
05a1d750 1671
509c07bc 1672 if (list_empty(&ch->free_tx))
05a1d750
DD		 1673 return NULL;
1674
1675 /* Initiator responses to target requests do not consume credits */
76c75b25 1676 if (iu_type != SRP_IU_RSP) {
509c07bc 1677 if (ch->req_lim <= rsv) {
76c75b25
BVA		 1678 ++target->zero_req_lim;
1679 return NULL;
1680 }
1681
509c07bc 1682 --ch->req_lim;
05a1d750
DD		 1683 }
1684
509c07bc 1685 iu = list_first_entry(&ch->free_tx, struct srp_iu, list);
76c75b25 1686 list_del(&iu->list);
05a1d750
DD		 1687 return iu;
1688}
1689
509c07bc 1690static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len)
05a1d750 1691{
509c07bc 1692 struct srp_target_port *target = ch->target;
05a1d750
DD		 1693 struct ib_sge list;
1694 struct ib_send_wr wr, *bad_wr;
05a1d750
DD		 1695
1696 list.addr = iu->dma;
1697 list.length = len;
9af76271 1698 list.lkey = target->lkey;
05a1d750
DD		 1699
1700 wr.next = NULL;
dcb4cb85 1701 wr.wr_id = (uintptr_t) iu;
05a1d750
DD		 1702 wr.sg_list = &list;
1703 wr.num_sge = 1;
1704 wr.opcode = IB_WR_SEND;
1705 wr.send_flags = IB_SEND_SIGNALED;
1706
509c07bc 1707 return ib_post_send(ch->qp, &wr, &bad_wr);
05a1d750
DD		 1708}
1709
509c07bc 1710static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu)
c996bb47 1711{
509c07bc 1712 struct srp_target_port *target = ch->target;
c996bb47 1713 struct ib_recv_wr wr, *bad_wr;
dcb4cb85 1714 struct ib_sge list;
c996bb47
BVA		 1715
1716 list.addr = iu->dma;
1717 list.length = iu->size;
9af76271 1718 list.lkey = target->lkey;
c996bb47
BVA		 1719
1720 wr.next = NULL;
dcb4cb85 1721 wr.wr_id = (uintptr_t) iu;
c996bb47
BVA		 1722 wr.sg_list = &list;
1723 wr.num_sge = 1;
1724
509c07bc 1725 return ib_post_recv(ch->qp, &wr, &bad_wr);
c996bb47
BVA		 1726}
1727
509c07bc 1728static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp)
aef9ec39 1729{
509c07bc 1730 struct srp_target_port *target = ch->target;
aef9ec39
RD		 1731 struct srp_request *req;
1732 struct scsi_cmnd *scmnd;
1733 unsigned long flags;
aef9ec39 1734
aef9ec39 1735 if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) {
509c07bc
BVA		 1736 spin_lock_irqsave(&ch->lock, flags);
1737 ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1738 spin_unlock_irqrestore(&ch->lock, flags);
94a9174c 1739
509c07bc 1740 ch->tsk_mgmt_status = -1;
f8b6e31e 1741 if (be32_to_cpu(rsp->resp_data_len) >= 4)
509c07bc
BVA		 1742 ch->tsk_mgmt_status = rsp->data[3];
1743 complete(&ch->tsk_mgmt_done);
aef9ec39 1744 } else {
77f2c1a4
BVA		 1745 scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag);
1746 if (scmnd) {
1747 req = (void *)scmnd->host_scribble;
1748 scmnd = srp_claim_req(ch, req, NULL, scmnd);
1749 }
22032991 1750 if (!scmnd) {
7aa54bd7 1751 shost_printk(KERN_ERR, target->scsi_host,
d92c0da7
BVA		 1752 "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n",
1753 rsp->tag, ch - target->ch, ch->qp->qp_num);
22032991 1754
509c07bc
BVA		 1755 spin_lock_irqsave(&ch->lock, flags);
1756 ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1757 spin_unlock_irqrestore(&ch->lock, flags);
22032991
BVA		 1758
1759 return;
1760 }
aef9ec39
RD		 1761 scmnd->result = rsp->status;
1762
1763 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) {
1764 memcpy(scmnd->sense_buffer, rsp->data +
1765 be32_to_cpu(rsp->resp_data_len),
1766 min_t(int, be32_to_cpu(rsp->sense_data_len),
1767 SCSI_SENSE_BUFFERSIZE));
1768 }
1769
e714531a 1770 if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER))
bb350d1d 1771 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt));
e714531a
BVA		 1772 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER))
1773 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt));
1774 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER))
1775 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt));
1776 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER))
1777 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt));
aef9ec39 1778
509c07bc 1779 srp_free_req(ch, req, scmnd,
22032991
BVA		 1780 be32_to_cpu(rsp->req_lim_delta));
1781
f8b6e31e
DD		 1782 scmnd->host_scribble = NULL;
1783 scmnd->scsi_done(scmnd);
aef9ec39 1784 }
aef9ec39
RD		 1785}
1786
509c07bc 1787static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta,
bb12588a
DD		 1788 void *rsp, int len)
1789{
509c07bc 1790 struct srp_target_port *target = ch->target;
76c75b25 1791 struct ib_device *dev = target->srp_host->srp_dev->dev;
bb12588a
DD		 1792 unsigned long flags;
1793 struct srp_iu *iu;
76c75b25 1794 int err;
bb12588a 1795
509c07bc
BVA		 1796 spin_lock_irqsave(&ch->lock, flags);
1797 ch->req_lim += req_delta;
1798 iu = __srp_get_tx_iu(ch, SRP_IU_RSP);
1799 spin_unlock_irqrestore(&ch->lock, flags);
76c75b25 1800
bb12588a
DD		 1801 if (!iu) {
1802 shost_printk(KERN_ERR, target->scsi_host, PFX
1803 "no IU available to send response\n");
76c75b25 1804 return 1;
bb12588a
DD		 1805 }
1806
1807 ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE);
1808 memcpy(iu->buf, rsp, len);
1809 ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE);
1810
509c07bc 1811 err = srp_post_send(ch, iu, len);
76c75b25 1812 if (err) {
bb12588a
DD		 1813 shost_printk(KERN_ERR, target->scsi_host, PFX
1814 "unable to post response: %d\n", err);
509c07bc 1815 srp_put_tx_iu(ch, iu, SRP_IU_RSP);
76c75b25 1816 }
bb12588a 1817
bb12588a
DD		 1818 return err;
1819}
1820
509c07bc 1821static void srp_process_cred_req(struct srp_rdma_ch *ch,
bb12588a
DD		 1822 struct srp_cred_req *req)
1823{
1824 struct srp_cred_rsp rsp = {
1825 .opcode = SRP_CRED_RSP,
1826 .tag = req->tag,
1827 };
1828 s32 delta = be32_to_cpu(req->req_lim_delta);
1829
509c07bc
BVA		 1830 if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
1831 shost_printk(KERN_ERR, ch->target->scsi_host, PFX
bb12588a
DD		 1832 "problems processing SRP_CRED_REQ\n");
1833}
1834
509c07bc 1835static void srp_process_aer_req(struct srp_rdma_ch *ch,
bb12588a
DD		 1836 struct srp_aer_req *req)
1837{
509c07bc 1838 struct srp_target_port *target = ch->target;
bb12588a
DD		 1839 struct srp_aer_rsp rsp = {
1840 .opcode = SRP_AER_RSP,
1841 .tag = req->tag,
1842 };
1843 s32 delta = be32_to_cpu(req->req_lim_delta);
1844
1845 shost_printk(KERN_ERR, target->scsi_host, PFX
1846 "ignoring AER for LUN %llu\n", be64_to_cpu(req->lun));
1847
509c07bc 1848 if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
bb12588a
DD		 1849 shost_printk(KERN_ERR, target->scsi_host, PFX
1850 "problems processing SRP_AER_REQ\n");
1851}
1852
509c07bc 1853static void srp_handle_recv(struct srp_rdma_ch *ch, struct ib_wc *wc)
aef9ec39 1854{
509c07bc 1855 struct srp_target_port *target = ch->target;
dcb4cb85 1856 struct ib_device *dev = target->srp_host->srp_dev->dev;
737b94eb 1857 struct srp_iu *iu = (struct srp_iu *) (uintptr_t) wc->wr_id;
c996bb47 1858 int res;
aef9ec39
RD		 1859 u8 opcode;
1860
509c07bc 1861 ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len,
85507bcc 1862 DMA_FROM_DEVICE);
aef9ec39
RD		 1863
1864 opcode = *(u8 *) iu->buf;
1865
1866 if (0) {
7aa54bd7
DD		 1867 shost_printk(KERN_ERR, target->scsi_host,
1868 PFX "recv completion, opcode 0x%02x\n", opcode);
7a700811
BVA		 1869 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1,
1870 iu->buf, wc->byte_len, true);
aef9ec39
RD		 1871 }
1872
1873 switch (opcode) {
1874 case SRP_RSP:
509c07bc 1875 srp_process_rsp(ch, iu->buf);
aef9ec39
RD		 1876 break;
1877
bb12588a 1878 case SRP_CRED_REQ:
509c07bc 1879 srp_process_cred_req(ch, iu->buf);
bb12588a
DD		 1880 break;
1881
1882 case SRP_AER_REQ:
509c07bc 1883 srp_process_aer_req(ch, iu->buf);
bb12588a
DD		 1884 break;
1885
aef9ec39
RD		 1886 case SRP_T_LOGOUT:
1887 /* XXX Handle target logout */
7aa54bd7
DD		 1888 shost_printk(KERN_WARNING, target->scsi_host,
1889 PFX "Got target logout request\n");
aef9ec39
RD		 1890 break;
1891
1892 default:
7aa54bd7
DD		 1893 shost_printk(KERN_WARNING, target->scsi_host,
1894 PFX "Unhandled SRP opcode 0x%02x\n", opcode);
aef9ec39
RD		 1895 break;
1896 }
1897
509c07bc 1898 ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len,
85507bcc 1899 DMA_FROM_DEVICE);
c996bb47 1900
509c07bc 1901 res = srp_post_recv(ch, iu);
c996bb47
BVA		 1902 if (res != 0)
1903 shost_printk(KERN_ERR, target->scsi_host,
1904 PFX "Recv failed with error code %d\n", res);
aef9ec39
RD		 1905}
1906
c1120f89
BVA		 1907/**
1908 * srp_tl_err_work() - handle a transport layer error
af24663b 1909 * @work: Work structure embedded in an SRP target port.
c1120f89
BVA		 1910 *
1911 * Note: This function may get invoked before the rport has been created,
1912 * hence the target->rport test.
1913 */
1914static void srp_tl_err_work(struct work_struct *work)
1915{
1916 struct srp_target_port *target;
1917
1918 target = container_of(work, struct srp_target_port, tl_err_work);
1919 if (target->rport)
1920 srp_start_tl_fail_timers(target->rport);
1921}
1922
5cfb1782 1923static void srp_handle_qp_err(u64 wr_id, enum ib_wc_status wc_status,
7dad6b2e 1924 bool send_err, struct srp_rdma_ch *ch)
948d1e88 1925{
7dad6b2e
BVA		 1926 struct srp_target_port *target = ch->target;
1927
1928 if (wr_id == SRP_LAST_WR_ID) {
1929 complete(&ch->done);
1930 return;
1931 }
1932
294c875a 1933 if (target->connected && !target->qp_in_error) {
5cfb1782
BVA		 1934 if (wr_id & LOCAL_INV_WR_ID_MASK) {
1935 shost_printk(KERN_ERR, target->scsi_host, PFX
1936 "LOCAL_INV failed with status %d\n",
1937 wc_status);
1938 } else if (wr_id & FAST_REG_WR_ID_MASK) {
1939 shost_printk(KERN_ERR, target->scsi_host, PFX
1940 "FAST_REG_MR failed status %d\n",
1941 wc_status);
1942 } else {
1943 shost_printk(KERN_ERR, target->scsi_host,
1944 PFX "failed %s status %d for iu %p\n",
1945 send_err ? "send" : "receive",
1946 wc_status, (void *)(uintptr_t)wr_id);
1947 }
c1120f89 1948 queue_work(system_long_wq, &target->tl_err_work);
4f0af697 1949 }
948d1e88
BVA		 1950 target->qp_in_error = true;
1951}
1952
509c07bc 1953static void srp_recv_completion(struct ib_cq *cq, void *ch_ptr)
aef9ec39 1954{
509c07bc 1955 struct srp_rdma_ch *ch = ch_ptr;
aef9ec39 1956 struct ib_wc wc;
aef9ec39
RD		 1957
1958 ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
1959 while (ib_poll_cq(cq, 1, &wc) > 0) {
948d1e88 1960 if (likely(wc.status == IB_WC_SUCCESS)) {
509c07bc 1961 srp_handle_recv(ch, &wc);
948d1e88 1962 } else {
7dad6b2e 1963 srp_handle_qp_err(wc.wr_id, wc.status, false, ch);
aef9ec39 1964 }
9c03dc9f
BVA		 1965 }
1966}
1967
509c07bc 1968static void srp_send_completion(struct ib_cq *cq, void *ch_ptr)
9c03dc9f 1969{
509c07bc 1970 struct srp_rdma_ch *ch = ch_ptr;
9c03dc9f 1971 struct ib_wc wc;
dcb4cb85 1972 struct srp_iu *iu;
9c03dc9f
BVA		 1973
1974 while (ib_poll_cq(cq, 1, &wc) > 0) {
948d1e88
BVA		 1975 if (likely(wc.status == IB_WC_SUCCESS)) {
1976 iu = (struct srp_iu *) (uintptr_t) wc.wr_id;
509c07bc 1977 list_add(&iu->list, &ch->free_tx);
948d1e88 1978 } else {
7dad6b2e 1979 srp_handle_qp_err(wc.wr_id, wc.status, true, ch);
9c03dc9f 1980 }
aef9ec39
RD		 1981 }
1982}
1983
76c75b25 1984static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
aef9ec39 1985{
76c75b25 1986 struct srp_target_port *target = host_to_target(shost);
a95cadb9 1987 struct srp_rport *rport = target->rport;
509c07bc 1988 struct srp_rdma_ch *ch;
aef9ec39
RD		 1989 struct srp_request *req;
1990 struct srp_iu *iu;
1991 struct srp_cmd *cmd;
85507bcc 1992 struct ib_device *dev;
76c75b25 1993 unsigned long flags;
77f2c1a4
BVA		 1994 u32 tag;
1995 u16 idx;
d1b4289e 1996 int len, ret;
a95cadb9
BVA		 1997 const bool in_scsi_eh = !in_interrupt() && current == shost->ehandler;
1998
1999 /*
2000 * The SCSI EH thread is the only context from which srp_queuecommand()
2001 * can get invoked for blocked devices (SDEV_BLOCK /
2002 * SDEV_CREATED_BLOCK). Avoid racing with srp_reconnect_rport() by
2003 * locking the rport mutex if invoked from inside the SCSI EH.
2004 */
2005 if (in_scsi_eh)
2006 mutex_lock(&rport->mutex);
aef9ec39 2007
d1b4289e
BVA		 2008 scmnd->result = srp_chkready(target->rport);
2009 if (unlikely(scmnd->result))
2010 goto err;
2ce19e72 2011
77f2c1a4
BVA		 2012 WARN_ON_ONCE(scmnd->request->tag < 0);
2013 tag = blk_mq_unique_tag(scmnd->request);
d92c0da7 2014 ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)];
77f2c1a4
BVA		 2015 idx = blk_mq_unique_tag_to_tag(tag);
2016 WARN_ONCE(idx >= target->req_ring_size, "%s: tag %#x: idx %d >= %d\n",
2017 dev_name(&shost->shost_gendev), tag, idx,
2018 target->req_ring_size);
509c07bc
BVA		 2019
2020 spin_lock_irqsave(&ch->lock, flags);
2021 iu = __srp_get_tx_iu(ch, SRP_IU_CMD);
509c07bc 2022 spin_unlock_irqrestore(&ch->lock, flags);
aef9ec39 2023
77f2c1a4
BVA		 2024 if (!iu)
2025 goto err;
2026
2027 req = &ch->req_ring[idx];
05321937 2028 dev = target->srp_host->srp_dev->dev;
49248644 2029 ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len,
85507bcc 2030 DMA_TO_DEVICE);
aef9ec39 2031
f8b6e31e 2032 scmnd->host_scribble = (void *) req;
aef9ec39
RD		 2033
2034 cmd = iu->buf;
2035 memset(cmd, 0, sizeof *cmd);
2036
2037 cmd->opcode = SRP_CMD;
2038 cmd->lun = cpu_to_be64((u64) scmnd->device->lun << 48);
77f2c1a4 2039 cmd->tag = tag;
aef9ec39
RD		 2040 memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len);
2041
aef9ec39
RD		 2042 req->scmnd = scmnd;
2043 req->cmd = iu;
aef9ec39 2044
509c07bc 2045 len = srp_map_data(scmnd, ch, req);
aef9ec39 2046 if (len < 0) {
7aa54bd7 2047 shost_printk(KERN_ERR, target->scsi_host,
d1b4289e
BVA		 2048 PFX "Failed to map data (%d)\n", len);
2049 /*
2050 * If we ran out of memory descriptors (-ENOMEM) because an
2051 * application is queuing many requests with more than
52ede08f 2052 * max_pages_per_mr sg-list elements, tell the SCSI mid-layer
d1b4289e
BVA		 2053 * to reduce queue depth temporarily.
2054 */
2055 scmnd->result = len == -ENOMEM ?
2056 DID_OK << 16 | QUEUE_FULL << 1 : DID_ERROR << 16;
76c75b25 2057 goto err_iu;
aef9ec39
RD		 2058 }
2059
49248644 2060 ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len,
85507bcc 2061 DMA_TO_DEVICE);
aef9ec39 2062
509c07bc 2063 if (srp_post_send(ch, iu, len)) {
7aa54bd7 2064 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n");
aef9ec39
RD		 2065 goto err_unmap;
2066 }
2067
d1b4289e
BVA		 2068 ret = 0;
2069
a95cadb9
BVA		 2070unlock_rport:
2071 if (in_scsi_eh)
2072 mutex_unlock(&rport->mutex);
2073
d1b4289e 2074 return ret;
aef9ec39
RD		 2075
2076err_unmap:
509c07bc 2077 srp_unmap_data(scmnd, ch, req);
aef9ec39 2078
76c75b25 2079err_iu:
509c07bc 2080 srp_put_tx_iu(ch, iu, SRP_IU_CMD);
76c75b25 2081
024ca901
BVA		 2082 /*
2083 * Avoid that the loops that iterate over the request ring can
2084 * encounter a dangling SCSI command pointer.
2085 */
2086 req->scmnd = NULL;
2087
d1b4289e
BVA		 2088err:
2089 if (scmnd->result) {
2090 scmnd->scsi_done(scmnd);
2091 ret = 0;
2092 } else {
2093 ret = SCSI_MLQUEUE_HOST_BUSY;
2094 }
a95cadb9 2095
d1b4289e 2096 goto unlock_rport;
aef9ec39
RD		 2097}
2098
4d73f95f
BVA		 2099/*
2100 * Note: the resources allocated in this function are freed in
509c07bc 2101 * srp_free_ch_ib().
4d73f95f 2102 */
509c07bc 2103static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch)
aef9ec39 2104{
509c07bc 2105 struct srp_target_port *target = ch->target;
aef9ec39
RD		 2106 int i;
2107
509c07bc
BVA		 2108 ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring),
2109 GFP_KERNEL);
2110 if (!ch->rx_ring)
4d73f95f 2111 goto err_no_ring;
509c07bc
BVA		 2112 ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring),
2113 GFP_KERNEL);
2114 if (!ch->tx_ring)
4d73f95f
BVA		 2115 goto err_no_ring;
2116
2117 for (i = 0; i < target->queue_size; ++i) {
509c07bc
BVA		 2118 ch->rx_ring[i] = srp_alloc_iu(target->srp_host,
2119 ch->max_ti_iu_len,
2120 GFP_KERNEL, DMA_FROM_DEVICE);
2121 if (!ch->rx_ring[i])
aef9ec39
RD		 2122 goto err;
2123 }
2124
4d73f95f 2125 for (i = 0; i < target->queue_size; ++i) {
509c07bc
BVA		 2126 ch->tx_ring[i] = srp_alloc_iu(target->srp_host,
2127 target->max_iu_len,
2128 GFP_KERNEL, DMA_TO_DEVICE);
2129 if (!ch->tx_ring[i])
aef9ec39 2130 goto err;
dcb4cb85 2131
509c07bc 2132 list_add(&ch->tx_ring[i]->list, &ch->free_tx);
aef9ec39
RD		 2133 }
2134
2135 return 0;
2136
2137err:
4d73f95f 2138 for (i = 0; i < target->queue_size; ++i) {
509c07bc
BVA		 2139 srp_free_iu(target->srp_host, ch->rx_ring[i]);
2140 srp_free_iu(target->srp_host, ch->tx_ring[i]);
aef9ec39
RD		 2141 }
2142
4d73f95f
BVA		 2143
2144err_no_ring:
509c07bc
BVA		 2145 kfree(ch->tx_ring);
2146 ch->tx_ring = NULL;
2147 kfree(ch->rx_ring);
2148 ch->rx_ring = NULL;
4d73f95f 2149
aef9ec39
RD		 2150 return -ENOMEM;
2151}
2152
c9b03c1a
BVA		 2153static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask)
2154{
2155 uint64_t T_tr_ns, max_compl_time_ms;
2156 uint32_t rq_tmo_jiffies;
2157
2158 /*
2159 * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair,
2160 * table 91), both the QP timeout and the retry count have to be set
2161 * for RC QP's during the RTR to RTS transition.
2162 */
2163 WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) !=
2164 (IB_QP_TIMEOUT | IB_QP_RETRY_CNT));
2165
2166 /*
2167 * Set target->rq_tmo_jiffies to one second more than the largest time
2168 * it can take before an error completion is generated. See also
2169 * C9-140..142 in the IBTA spec for more information about how to
2170 * convert the QP Local ACK Timeout value to nanoseconds.
2171 */
2172 T_tr_ns = 4096 * (1ULL << qp_attr->timeout);
2173 max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns;
2174 do_div(max_compl_time_ms, NSEC_PER_MSEC);
2175 rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000);
2176
2177 return rq_tmo_jiffies;
2178}
2179
961e0be8
DD		 2180static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
2181 struct srp_login_rsp *lrsp,
509c07bc 2182 struct srp_rdma_ch *ch)
961e0be8 2183{
509c07bc 2184 struct srp_target_port *target = ch->target;
961e0be8
DD		 2185 struct ib_qp_attr *qp_attr = NULL;
2186 int attr_mask = 0;
2187 int ret;
2188 int i;
2189
2190 if (lrsp->opcode == SRP_LOGIN_RSP) {
509c07bc
BVA		 2191 ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
2192 ch->req_lim = be32_to_cpu(lrsp->req_lim_delta);
961e0be8
DD		 2193
2194 /*
2195 * Reserve credits for task management so we don't
2196 * bounce requests back to the SCSI mid-layer.
2197 */
2198 target->scsi_host->can_queue
509c07bc 2199 = min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE,
961e0be8 2200 target->scsi_host->can_queue);
4d73f95f
BVA		 2201 target->scsi_host->cmd_per_lun
2202 = min_t(int, target->scsi_host->can_queue,
2203 target->scsi_host->cmd_per_lun);
961e0be8
DD		 2204 } else {
2205 shost_printk(KERN_WARNING, target->scsi_host,
2206 PFX "Unhandled RSP opcode %#x\n", lrsp->opcode);
2207 ret = -ECONNRESET;
2208 goto error;
2209 }
2210
509c07bc
BVA		 2211 if (!ch->rx_ring) {
2212 ret = srp_alloc_iu_bufs(ch);
961e0be8
DD		 2213 if (ret)
2214 goto error;
2215 }
2216
2217 ret = -ENOMEM;
2218 qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
2219 if (!qp_attr)
2220 goto error;
2221
2222 qp_attr->qp_state = IB_QPS_RTR;
2223 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2224 if (ret)
2225 goto error_free;
2226
509c07bc 2227 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
961e0be8
DD		 2228 if (ret)
2229 goto error_free;
2230
4d73f95f 2231 for (i = 0; i < target->queue_size; i++) {
509c07bc
BVA		 2232 struct srp_iu *iu = ch->rx_ring[i];
2233
2234 ret = srp_post_recv(ch, iu);
961e0be8
DD		 2235 if (ret)
2236 goto error_free;
2237 }
2238
2239 qp_attr->qp_state = IB_QPS_RTS;
2240 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2241 if (ret)
2242 goto error_free;
2243
c9b03c1a
BVA		 2244 target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);
2245
509c07bc 2246 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
961e0be8
DD		 2247 if (ret)
2248 goto error_free;
2249
2250 ret = ib_send_cm_rtu(cm_id, NULL, 0);
2251
2252error_free:
2253 kfree(qp_attr);
2254
2255error:
509c07bc 2256 ch->status = ret;
961e0be8
DD		 2257}
2258
aef9ec39
RD		 2259static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
2260 struct ib_cm_event *event,
509c07bc 2261 struct srp_rdma_ch *ch)
aef9ec39 2262{
509c07bc 2263 struct srp_target_port *target = ch->target;
7aa54bd7 2264 struct Scsi_Host *shost = target->scsi_host;
aef9ec39
RD		 2265 struct ib_class_port_info *cpi;
2266 int opcode;
2267
2268 switch (event->param.rej_rcvd.reason) {
2269 case IB_CM_REJ_PORT_CM_REDIRECT:
2270 cpi = event->param.rej_rcvd.ari;
509c07bc
BVA		 2271 ch->path.dlid = cpi->redirect_lid;
2272 ch->path.pkey = cpi->redirect_pkey;
aef9ec39 2273 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
509c07bc 2274 memcpy(ch->path.dgid.raw, cpi->redirect_gid, 16);
aef9ec39 2275
509c07bc 2276 ch->status = ch->path.dlid ?
aef9ec39
RD		 2277 SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
2278 break;
2279
2280 case IB_CM_REJ_PORT_REDIRECT:
5d7cbfd6 2281 if (srp_target_is_topspin(target)) {
aef9ec39
RD		 2282 /*
2283 * Topspin/Cisco SRP gateways incorrectly send
2284 * reject reason code 25 when they mean 24
2285 * (port redirect).
2286 */
509c07bc 2287 memcpy(ch->path.dgid.raw,
aef9ec39
RD		 2288 event->param.rej_rcvd.ari, 16);
2289
7aa54bd7
DD		 2290 shost_printk(KERN_DEBUG, shost,
2291 PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
509c07bc
BVA		 2292 be64_to_cpu(ch->path.dgid.global.subnet_prefix),
2293 be64_to_cpu(ch->path.dgid.global.interface_id));
aef9ec39 2294
509c07bc 2295 ch->status = SRP_PORT_REDIRECT;
aef9ec39 2296 } else {
7aa54bd7
DD		 2297 shost_printk(KERN_WARNING, shost,
2298 " REJ reason: IB_CM_REJ_PORT_REDIRECT\n");
509c07bc 2299 ch->status = -ECONNRESET;
aef9ec39
RD		 2300 }
2301 break;
2302
2303 case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
7aa54bd7
DD		 2304 shost_printk(KERN_WARNING, shost,
2305 " REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
509c07bc 2306 ch->status = -ECONNRESET;
aef9ec39
RD		 2307 break;
2308
2309 case IB_CM_REJ_CONSUMER_DEFINED:
2310 opcode = *(u8 *) event->private_data;
2311 if (opcode == SRP_LOGIN_REJ) {
2312 struct srp_login_rej *rej = event->private_data;
2313 u32 reason = be32_to_cpu(rej->reason);
2314
2315 if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
7aa54bd7
DD		 2316 shost_printk(KERN_WARNING, shost,
2317 PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
aef9ec39 2318 else
e7ffde01
BVA		 2319 shost_printk(KERN_WARNING, shost, PFX
2320 "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n",
747fe000
BVA		 2321 target->sgid.raw,
2322 target->orig_dgid.raw, reason);
aef9ec39 2323 } else
7aa54bd7
DD		 2324 shost_printk(KERN_WARNING, shost,
2325 " REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
2326 " opcode 0x%02x\n", opcode);
509c07bc 2327 ch->status = -ECONNRESET;
aef9ec39
RD		 2328 break;
2329
9fe4bcf4
DD		 2330 case IB_CM_REJ_STALE_CONN:
2331 shost_printk(KERN_WARNING, shost, " REJ reason: stale connection\n");
509c07bc 2332 ch->status = SRP_STALE_CONN;
9fe4bcf4
DD		 2333 break;
2334
aef9ec39 2335 default:
7aa54bd7
DD		 2336 shost_printk(KERN_WARNING, shost, " REJ reason 0x%x\n",
2337 event->param.rej_rcvd.reason);
509c07bc 2338 ch->status = -ECONNRESET;
aef9ec39
RD		 2339 }
2340}
2341
2342static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2343{
509c07bc
BVA		 2344 struct srp_rdma_ch *ch = cm_id->context;
2345 struct srp_target_port *target = ch->target;
aef9ec39 2346 int comp = 0;
aef9ec39
RD		 2347
2348 switch (event->event) {
2349 case IB_CM_REQ_ERROR:
7aa54bd7
DD		 2350 shost_printk(KERN_DEBUG, target->scsi_host,
2351 PFX "Sending CM REQ failed\n");
aef9ec39 2352 comp = 1;
509c07bc 2353 ch->status = -ECONNRESET;
aef9ec39
RD		 2354 break;
2355
2356 case IB_CM_REP_RECEIVED:
2357 comp = 1;
509c07bc 2358 srp_cm_rep_handler(cm_id, event->private_data, ch);
aef9ec39
RD		 2359 break;
2360
2361 case IB_CM_REJ_RECEIVED:
7aa54bd7 2362 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
aef9ec39
RD		 2363 comp = 1;
2364
509c07bc 2365 srp_cm_rej_handler(cm_id, event, ch);
aef9ec39
RD		 2366 break;
2367
b7ac4ab4 2368 case IB_CM_DREQ_RECEIVED:
7aa54bd7
DD		 2369 shost_printk(KERN_WARNING, target->scsi_host,
2370 PFX "DREQ received - connection closed\n");
294c875a 2371 srp_change_conn_state(target, false);
b7ac4ab4 2372 if (ib_send_cm_drep(cm_id, NULL, 0))
7aa54bd7
DD		 2373 shost_printk(KERN_ERR, target->scsi_host,
2374 PFX "Sending CM DREP failed\n");
c1120f89 2375 queue_work(system_long_wq, &target->tl_err_work);
aef9ec39
RD		 2376 break;
2377
2378 case IB_CM_TIMEWAIT_EXIT:
7aa54bd7
DD		 2379 shost_printk(KERN_ERR, target->scsi_host,
2380 PFX "connection closed\n");
ac72d766 2381 comp = 1;
aef9ec39 2382
509c07bc 2383 ch->status = 0;
aef9ec39
RD		 2384 break;
2385
b7ac4ab4
IR		 2386 case IB_CM_MRA_RECEIVED:
2387 case IB_CM_DREQ_ERROR:
2388 case IB_CM_DREP_RECEIVED:
2389 break;
2390
aef9ec39 2391 default:
7aa54bd7
DD		 2392 shost_printk(KERN_WARNING, target->scsi_host,
2393 PFX "Unhandled CM event %d\n", event->event);
aef9ec39
RD		 2394 break;
2395 }
2396
2397 if (comp)
509c07bc 2398 complete(&ch->done);
aef9ec39 2399
aef9ec39
RD		 2400 return 0;
2401}
2402
71444b97
JW		 2403/**
2404 * srp_change_queue_depth - setting device queue depth
2405 * @sdev: scsi device struct
2406 * @qdepth: requested queue depth
71444b97
JW		 2407 *
2408 * Returns queue depth.
2409 */
2410static int
db5ed4df 2411srp_change_queue_depth(struct scsi_device *sdev, int qdepth)
71444b97 2412{
c40ecc12 2413 if (!sdev->tagged_supported)
1e6f2416 2414 qdepth = 1;
db5ed4df 2415 return scsi_change_queue_depth(sdev, qdepth);
71444b97
JW		 2416}
2417
509c07bc
BVA		 2418static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag,
2419 unsigned int lun, u8 func)
aef9ec39 2420{
509c07bc 2421 struct srp_target_port *target = ch->target;
a95cadb9 2422 struct srp_rport *rport = target->rport;
19081f31 2423 struct ib_device *dev = target->srp_host->srp_dev->dev;
aef9ec39
RD		 2424 struct srp_iu *iu;
2425 struct srp_tsk_mgmt *tsk_mgmt;
aef9ec39 2426
3780d1f0
BVA		 2427 if (!target->connected || target->qp_in_error)
2428 return -1;
2429
509c07bc 2430 init_completion(&ch->tsk_mgmt_done);
aef9ec39 2431
a95cadb9 2432 /*
509c07bc 2433 * Lock the rport mutex to avoid that srp_create_ch_ib() is
a95cadb9
BVA		 2434 * invoked while a task management function is being sent.
2435 */
2436 mutex_lock(&rport->mutex);
509c07bc
BVA		 2437 spin_lock_irq(&ch->lock);
2438 iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT);
2439 spin_unlock_irq(&ch->lock);
76c75b25 2440
a95cadb9
BVA		 2441 if (!iu) {
2442 mutex_unlock(&rport->mutex);
2443
76c75b25 2444 return -1;
a95cadb9 2445 }
aef9ec39 2446
19081f31
DD		 2447 ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt,
2448 DMA_TO_DEVICE);
aef9ec39
RD		 2449 tsk_mgmt = iu->buf;
2450 memset(tsk_mgmt, 0, sizeof *tsk_mgmt);
2451
2452 tsk_mgmt->opcode = SRP_TSK_MGMT;
f8b6e31e
DD		 2453 tsk_mgmt->lun = cpu_to_be64((u64) lun << 48);
2454 tsk_mgmt->tag = req_tag | SRP_TAG_TSK_MGMT;
aef9ec39 2455 tsk_mgmt->tsk_mgmt_func = func;
f8b6e31e 2456 tsk_mgmt->task_tag = req_tag;
aef9ec39 2457
19081f31
DD		 2458 ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt,
2459 DMA_TO_DEVICE);
509c07bc
BVA		 2460 if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) {
2461 srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT);
a95cadb9
BVA		 2462 mutex_unlock(&rport->mutex);
2463
76c75b25
BVA		 2464 return -1;
2465 }
a95cadb9 2466 mutex_unlock(&rport->mutex);
d945e1df 2467
509c07bc 2468 if (!wait_for_completion_timeout(&ch->tsk_mgmt_done,
aef9ec39 2469 msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS)))
d945e1df 2470 return -1;
aef9ec39 2471
d945e1df 2472 return 0;
d945e1df
RD		 2473}
2474
aef9ec39
RD		 2475static int srp_abort(struct scsi_cmnd *scmnd)
2476{
d945e1df 2477 struct srp_target_port *target = host_to_target(scmnd->device->host);
f8b6e31e 2478 struct srp_request *req = (struct srp_request *) scmnd->host_scribble;
77f2c1a4 2479 u32 tag;
d92c0da7 2480 u16 ch_idx;
509c07bc 2481 struct srp_rdma_ch *ch;
086f44f5 2482 int ret;
d945e1df 2483
7aa54bd7 2484 shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n");
aef9ec39 2485
d92c0da7 2486 if (!req)
99b6697a 2487 return SUCCESS;
77f2c1a4 2488 tag = blk_mq_unique_tag(scmnd->request);
d92c0da7
BVA		 2489 ch_idx = blk_mq_unique_tag_to_hwq(tag);
2490 if (WARN_ON_ONCE(ch_idx >= target->ch_count))
2491 return SUCCESS;
2492 ch = &target->ch[ch_idx];
2493 if (!srp_claim_req(ch, req, NULL, scmnd))
2494 return SUCCESS;
2495 shost_printk(KERN_ERR, target->scsi_host,
2496 "Sending SRP abort for tag %#x\n", tag);
77f2c1a4 2497 if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun,
80d5e8a2 2498 SRP_TSK_ABORT_TASK) == 0)
086f44f5 2499 ret = SUCCESS;
ed9b2264 2500 else if (target->rport->state == SRP_RPORT_LOST)
99e1c139 2501 ret = FAST_IO_FAIL;
086f44f5
BVA		 2502 else
2503 ret = FAILED;
509c07bc 2504 srp_free_req(ch, req, scmnd, 0);
22032991 2505 scmnd->result = DID_ABORT << 16;
d8536670 2506 scmnd->scsi_done(scmnd);
d945e1df 2507
086f44f5 2508 return ret;
aef9ec39
RD		 2509}
2510
2511static int srp_reset_device(struct scsi_cmnd *scmnd)
2512{
d945e1df 2513 struct srp_target_port *target = host_to_target(scmnd->device->host);
d92c0da7 2514 struct srp_rdma_ch *ch;
536ae14e 2515 int i;
d945e1df 2516
7aa54bd7 2517 shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n");
aef9ec39 2518
d92c0da7 2519 ch = &target->ch[0];
509c07bc 2520 if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun,
f8b6e31e 2521 SRP_TSK_LUN_RESET))
d945e1df 2522 return FAILED;
509c07bc 2523 if (ch->tsk_mgmt_status)
d945e1df
RD		 2524 return FAILED;
2525
d92c0da7
BVA		 2526 for (i = 0; i < target->ch_count; i++) {
2527 ch = &target->ch[i];
2528 for (i = 0; i < target->req_ring_size; ++i) {
2529 struct srp_request *req = &ch->req_ring[i];
509c07bc 2530
d92c0da7
BVA		 2531 srp_finish_req(ch, req, scmnd->device, DID_RESET << 16);
2532 }
536ae14e 2533 }
d945e1df 2534
d945e1df 2535 return SUCCESS;
aef9ec39
RD		 2536}
2537
2538static int srp_reset_host(struct scsi_cmnd *scmnd)
2539{
2540 struct srp_target_port *target = host_to_target(scmnd->device->host);
aef9ec39 2541
7aa54bd7 2542 shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n");
aef9ec39 2543
ed9b2264 2544 return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED;
aef9ec39
RD		 2545}
2546
c9b03c1a
BVA		 2547static int srp_slave_configure(struct scsi_device *sdev)
2548{
2549 struct Scsi_Host *shost = sdev->host;
2550 struct srp_target_port *target = host_to_target(shost);
2551 struct request_queue *q = sdev->request_queue;
2552 unsigned long timeout;
2553
2554 if (sdev->type == TYPE_DISK) {
2555 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies);
2556 blk_queue_rq_timeout(q, timeout);
2557 }
2558
2559 return 0;
2560}
2561
ee959b00
TJ		 2562static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr,
2563 char *buf)
6ecb0c84 2564{
ee959b00 2565 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2566
6ecb0c84
RD		 2567 return sprintf(buf, "0x%016llx\n",
2568 (unsigned long long) be64_to_cpu(target->id_ext));
2569}
2570
ee959b00
TJ		 2571static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr,
2572 char *buf)
6ecb0c84 2573{
ee959b00 2574 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2575
6ecb0c84
RD		 2576 return sprintf(buf, "0x%016llx\n",
2577 (unsigned long long) be64_to_cpu(target->ioc_guid));
2578}
2579
ee959b00
TJ		 2580static ssize_t show_service_id(struct device *dev,
2581 struct device_attribute *attr, char *buf)
6ecb0c84 2582{
ee959b00 2583 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2584
6ecb0c84
RD		 2585 return sprintf(buf, "0x%016llx\n",
2586 (unsigned long long) be64_to_cpu(target->service_id));
2587}
2588
ee959b00
TJ		 2589static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
2590 char *buf)
6ecb0c84 2591{
ee959b00 2592 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2593
747fe000 2594 return sprintf(buf, "0x%04x\n", be16_to_cpu(target->pkey));
6ecb0c84
RD		 2595}
2596
848b3082
BVA		 2597static ssize_t show_sgid(struct device *dev, struct device_attribute *attr,
2598 char *buf)
2599{
2600 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2601
747fe000 2602 return sprintf(buf, "%pI6\n", target->sgid.raw);
848b3082
BVA		 2603}
2604
ee959b00
TJ		 2605static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
2606 char *buf)
6ecb0c84 2607{
ee959b00 2608 struct srp_target_port *target = host_to_target(class_to_shost(dev));
d92c0da7 2609 struct srp_rdma_ch *ch = &target->ch[0];
6ecb0c84 2610
509c07bc 2611 return sprintf(buf, "%pI6\n", ch->path.dgid.raw);
6ecb0c84
RD		 2612}
2613
ee959b00
TJ		 2614static ssize_t show_orig_dgid(struct device *dev,
2615 struct device_attribute *attr, char *buf)
3633b3d0 2616{
ee959b00 2617 struct srp_target_port *target = host_to_target(class_to_shost(dev));
3633b3d0 2618
747fe000 2619 return sprintf(buf, "%pI6\n", target->orig_dgid.raw);
3633b3d0
IR		 2620}
2621
89de7486
BVA		 2622static ssize_t show_req_lim(struct device *dev,
2623 struct device_attribute *attr, char *buf)
2624{
2625 struct srp_target_port *target = host_to_target(class_to_shost(dev));
d92c0da7
BVA		 2626 struct srp_rdma_ch *ch;
2627 int i, req_lim = INT_MAX;
89de7486 2628
d92c0da7
BVA		 2629 for (i = 0; i < target->ch_count; i++) {
2630 ch = &target->ch[i];
2631 req_lim = min(req_lim, ch->req_lim);
2632 }
2633 return sprintf(buf, "%d\n", req_lim);
89de7486
BVA		 2634}
2635
ee959b00
TJ		 2636static ssize_t show_zero_req_lim(struct device *dev,
2637 struct device_attribute *attr, char *buf)
6bfa24fa 2638{
ee959b00 2639 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6bfa24fa 2640
6bfa24fa
RD		 2641 return sprintf(buf, "%d\n", target->zero_req_lim);
2642}
2643
ee959b00
TJ		 2644static ssize_t show_local_ib_port(struct device *dev,
2645 struct device_attribute *attr, char *buf)
ded7f1a1 2646{
ee959b00 2647 struct srp_target_port *target = host_to_target(class_to_shost(dev));
ded7f1a1
IR		 2648
2649 return sprintf(buf, "%d\n", target->srp_host->port);
2650}
2651
ee959b00
TJ		 2652static ssize_t show_local_ib_device(struct device *dev,
2653 struct device_attribute *attr, char *buf)
ded7f1a1 2654{
ee959b00 2655 struct srp_target_port *target = host_to_target(class_to_shost(dev));
ded7f1a1 2656
05321937 2657 return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
ded7f1a1
IR		 2658}
2659
d92c0da7
BVA		 2660static ssize_t show_ch_count(struct device *dev, struct device_attribute *attr,
2661 char *buf)
2662{
2663 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2664
2665 return sprintf(buf, "%d\n", target->ch_count);
2666}
2667
4b5e5f41
BVA		 2668static ssize_t show_comp_vector(struct device *dev,
2669 struct device_attribute *attr, char *buf)
2670{
2671 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2672
2673 return sprintf(buf, "%d\n", target->comp_vector);
2674}
2675
7bb312e4
VP		 2676static ssize_t show_tl_retry_count(struct device *dev,
2677 struct device_attribute *attr, char *buf)
2678{
2679 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2680
2681 return sprintf(buf, "%d\n", target->tl_retry_count);
2682}
2683
49248644
DD		 2684static ssize_t show_cmd_sg_entries(struct device *dev,
2685 struct device_attribute *attr, char *buf)
2686{
2687 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2688
2689 return sprintf(buf, "%u\n", target->cmd_sg_cnt);
2690}
2691
c07d424d
DD		 2692static ssize_t show_allow_ext_sg(struct device *dev,
2693 struct device_attribute *attr, char *buf)
2694{
2695 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2696
2697 return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
2698}
2699
ee959b00
TJ		 2700static DEVICE_ATTR(id_ext, S_IRUGO, show_id_ext, NULL);
2701static DEVICE_ATTR(ioc_guid, S_IRUGO, show_ioc_guid, NULL);
2702static DEVICE_ATTR(service_id, S_IRUGO, show_service_id, NULL);
2703static DEVICE_ATTR(pkey, S_IRUGO, show_pkey, NULL);
848b3082 2704static DEVICE_ATTR(sgid, S_IRUGO, show_sgid, NULL);
ee959b00
TJ		 2705static DEVICE_ATTR(dgid, S_IRUGO, show_dgid, NULL);
2706static DEVICE_ATTR(orig_dgid, S_IRUGO, show_orig_dgid, NULL);
89de7486 2707static DEVICE_ATTR(req_lim, S_IRUGO, show_req_lim, NULL);
ee959b00
TJ		 2708static DEVICE_ATTR(zero_req_lim, S_IRUGO, show_zero_req_lim, NULL);
2709static DEVICE_ATTR(local_ib_port, S_IRUGO, show_local_ib_port, NULL);
2710static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
d92c0da7 2711static DEVICE_ATTR(ch_count, S_IRUGO, show_ch_count, NULL);
4b5e5f41 2712static DEVICE_ATTR(comp_vector, S_IRUGO, show_comp_vector, NULL);
7bb312e4 2713static DEVICE_ATTR(tl_retry_count, S_IRUGO, show_tl_retry_count, NULL);
49248644 2714static DEVICE_ATTR(cmd_sg_entries, S_IRUGO, show_cmd_sg_entries, NULL);
c07d424d 2715static DEVICE_ATTR(allow_ext_sg, S_IRUGO, show_allow_ext_sg, NULL);
ee959b00
TJ		 2716
2717static struct device_attribute *srp_host_attrs[] = {
2718 &dev_attr_id_ext,
2719 &dev_attr_ioc_guid,
2720 &dev_attr_service_id,
2721 &dev_attr_pkey,
848b3082 2722 &dev_attr_sgid,
ee959b00
TJ		 2723 &dev_attr_dgid,
2724 &dev_attr_orig_dgid,
89de7486 2725 &dev_attr_req_lim,
ee959b00
TJ		 2726 &dev_attr_zero_req_lim,
2727 &dev_attr_local_ib_port,
2728 &dev_attr_local_ib_device,
d92c0da7 2729 &dev_attr_ch_count,
4b5e5f41 2730 &dev_attr_comp_vector,
7bb312e4 2731 &dev_attr_tl_retry_count,
49248644 2732 &dev_attr_cmd_sg_entries,
c07d424d 2733 &dev_attr_allow_ext_sg,
6ecb0c84
RD		 2734 NULL
2735};
2736
aef9ec39
RD		 2737static struct scsi_host_template srp_template = {
2738 .module = THIS_MODULE,
b7f008fd
RD		 2739 .name = "InfiniBand SRP initiator",
2740 .proc_name = DRV_NAME,
c9b03c1a 2741 .slave_configure = srp_slave_configure,
aef9ec39
RD		 2742 .info = srp_target_info,
2743 .queuecommand = srp_queuecommand,
71444b97 2744 .change_queue_depth = srp_change_queue_depth,
aef9ec39
RD		 2745 .eh_abort_handler = srp_abort,
2746 .eh_device_reset_handler = srp_reset_device,
2747 .eh_host_reset_handler = srp_reset_host,
2742c1da 2748 .skip_settle_delay = true,
49248644 2749 .sg_tablesize = SRP_DEF_SG_TABLESIZE,
4d73f95f 2750 .can_queue = SRP_DEFAULT_CMD_SQ_SIZE,
aef9ec39 2751 .this_id = -1,
4d73f95f 2752 .cmd_per_lun = SRP_DEFAULT_CMD_SQ_SIZE,
6ecb0c84 2753 .use_clustering = ENABLE_CLUSTERING,
77f2c1a4
BVA		 2754 .shost_attrs = srp_host_attrs,
2755 .use_blk_tags = 1,
c40ecc12 2756 .track_queue_depth = 1,
aef9ec39
RD		 2757};
2758
34aa654e
BVA		 2759static int srp_sdev_count(struct Scsi_Host *host)
2760{
2761 struct scsi_device *sdev;
2762 int c = 0;
2763
2764 shost_for_each_device(sdev, host)
2765 c++;
2766
2767 return c;
2768}
2769
aef9ec39
RD		 2770static int srp_add_target(struct srp_host *host, struct srp_target_port *target)
2771{
3236822b
FT		 2772 struct srp_rport_identifiers ids;
2773 struct srp_rport *rport;
2774
34aa654e 2775 target->state = SRP_TARGET_SCANNING;
aef9ec39
RD		 2776 sprintf(target->target_name, "SRP.T10:%016llX",
2777 (unsigned long long) be64_to_cpu(target->id_ext));
2778
05321937 2779 if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dma_device))
aef9ec39
RD		 2780 return -ENODEV;
2781
3236822b
FT		 2782 memcpy(ids.port_id, &target->id_ext, 8);
2783 memcpy(ids.port_id + 8, &target->ioc_guid, 8);
aebd5e47 2784 ids.roles = SRP_RPORT_ROLE_TARGET;
3236822b
FT		 2785 rport = srp_rport_add(target->scsi_host, &ids);
2786 if (IS_ERR(rport)) {
2787 scsi_remove_host(target->scsi_host);
2788 return PTR_ERR(rport);
2789 }
2790
dc1bdbd9 2791 rport->lld_data = target;
9dd69a60 2792 target->rport = rport;
dc1bdbd9 2793
b3589fd4 2794 spin_lock(&host->target_lock);
aef9ec39 2795 list_add_tail(&target->list, &host->target_list);
b3589fd4 2796 spin_unlock(&host->target_lock);
aef9ec39 2797
aef9ec39 2798 scsi_scan_target(&target->scsi_host->shost_gendev,
1962a4a1 2799 0, target->scsi_id, SCAN_WILD_CARD, 0);
aef9ec39 2800
34aa654e
BVA		 2801 if (!target->connected || target->qp_in_error) {
2802 shost_printk(KERN_INFO, target->scsi_host,
2803 PFX "SCSI scan failed - removing SCSI host\n");
2804 srp_queue_remove_work(target);
2805 goto out;
2806 }
2807
2808 pr_debug(PFX "%s: SCSI scan succeeded - detected %d LUNs\n",
2809 dev_name(&target->scsi_host->shost_gendev),
2810 srp_sdev_count(target->scsi_host));
2811
2812 spin_lock_irq(&target->lock);
2813 if (target->state == SRP_TARGET_SCANNING)
2814 target->state = SRP_TARGET_LIVE;
2815 spin_unlock_irq(&target->lock);
2816
2817out:
aef9ec39
RD		 2818 return 0;
2819}
2820
ee959b00 2821static void srp_release_dev(struct device *dev)
aef9ec39
RD		 2822{
2823 struct srp_host *host =
ee959b00 2824 container_of(dev, struct srp_host, dev);
aef9ec39
RD		 2825
2826 complete(&host->released);
2827}
2828
2829static struct class srp_class = {
2830 .name = "infiniband_srp",
ee959b00 2831 .dev_release = srp_release_dev
aef9ec39
RD		 2832};
2833
96fc248a
BVA		 2834/**
2835 * srp_conn_unique() - check whether the connection to a target is unique
af24663b
BVA		 2836 * @host: SRP host.
2837 * @target: SRP target port.
96fc248a
BVA		 2838 */
2839static bool srp_conn_unique(struct srp_host *host,
2840 struct srp_target_port *target)
2841{
2842 struct srp_target_port *t;
2843 bool ret = false;
2844
2845 if (target->state == SRP_TARGET_REMOVED)
2846 goto out;
2847
2848 ret = true;
2849
2850 spin_lock(&host->target_lock);
2851 list_for_each_entry(t, &host->target_list, list) {
2852 if (t != target &&
2853 target->id_ext == t->id_ext &&
2854 target->ioc_guid == t->ioc_guid &&
2855 target->initiator_ext == t->initiator_ext) {
2856 ret = false;
2857 break;
2858 }
2859 }
2860 spin_unlock(&host->target_lock);
2861
2862out:
2863 return ret;
2864}
2865
aef9ec39
RD		 2866/*
2867 * Target ports are added by writing
2868 *
2869 * id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>,
2870 * pkey=<P_Key>,service_id=<service ID>
2871 *
2872 * to the add_target sysfs attribute.
2873 */
2874enum {
2875 SRP_OPT_ERR = 0,
2876 SRP_OPT_ID_EXT = 1 << 0,
2877 SRP_OPT_IOC_GUID = 1 << 1,
2878 SRP_OPT_DGID = 1 << 2,
2879 SRP_OPT_PKEY = 1 << 3,
2880 SRP_OPT_SERVICE_ID = 1 << 4,
2881 SRP_OPT_MAX_SECT = 1 << 5,
52fb2b50 2882 SRP_OPT_MAX_CMD_PER_LUN = 1 << 6,
0c0450db 2883 SRP_OPT_IO_CLASS = 1 << 7,
01cb9bcb 2884 SRP_OPT_INITIATOR_EXT = 1 << 8,
49248644 2885 SRP_OPT_CMD_SG_ENTRIES = 1 << 9,
c07d424d
DD		 2886 SRP_OPT_ALLOW_EXT_SG = 1 << 10,
2887 SRP_OPT_SG_TABLESIZE = 1 << 11,
4b5e5f41 2888 SRP_OPT_COMP_VECTOR = 1 << 12,
7bb312e4 2889 SRP_OPT_TL_RETRY_COUNT = 1 << 13,
4d73f95f 2890 SRP_OPT_QUEUE_SIZE = 1 << 14,
aef9ec39
RD		 2891 SRP_OPT_ALL = (SRP_OPT_ID_EXT |
2892 SRP_OPT_IOC_GUID |
2893 SRP_OPT_DGID |
2894 SRP_OPT_PKEY |
2895 SRP_OPT_SERVICE_ID),
2896};
2897
a447c093 2898static const match_table_t srp_opt_tokens = {
52fb2b50
VP		 2899 { SRP_OPT_ID_EXT, "id_ext=%s" },
2900 { SRP_OPT_IOC_GUID, "ioc_guid=%s" },
2901 { SRP_OPT_DGID, "dgid=%s" },
2902 { SRP_OPT_PKEY, "pkey=%x" },
2903 { SRP_OPT_SERVICE_ID, "service_id=%s" },
2904 { SRP_OPT_MAX_SECT, "max_sect=%d" },
2905 { SRP_OPT_MAX_CMD_PER_LUN, "max_cmd_per_lun=%d" },
0c0450db 2906 { SRP_OPT_IO_CLASS, "io_class=%x" },
01cb9bcb 2907 { SRP_OPT_INITIATOR_EXT, "initiator_ext=%s" },
49248644 2908 { SRP_OPT_CMD_SG_ENTRIES, "cmd_sg_entries=%u" },
c07d424d
DD		 2909 { SRP_OPT_ALLOW_EXT_SG, "allow_ext_sg=%u" },
2910 { SRP_OPT_SG_TABLESIZE, "sg_tablesize=%u" },
4b5e5f41 2911 { SRP_OPT_COMP_VECTOR, "comp_vector=%u" },
7bb312e4 2912 { SRP_OPT_TL_RETRY_COUNT, "tl_retry_count=%u" },
4d73f95f 2913 { SRP_OPT_QUEUE_SIZE, "queue_size=%d" },
52fb2b50 2914 { SRP_OPT_ERR, NULL }
aef9ec39
RD		 2915};
2916
2917static int srp_parse_options(const char *buf, struct srp_target_port *target)
2918{
2919 char *options, *sep_opt;
2920 char *p;
2921 char dgid[3];
2922 substring_t args[MAX_OPT_ARGS];
2923 int opt_mask = 0;
2924 int token;
2925 int ret = -EINVAL;
2926 int i;
2927
2928 options = kstrdup(buf, GFP_KERNEL);
2929 if (!options)
2930 return -ENOMEM;
2931
2932 sep_opt = options;
7dcf9c19 2933 while ((p = strsep(&sep_opt, ",\n")) != NULL) {
aef9ec39
RD		 2934 if (!*p)
2935 continue;
2936
2937 token = match_token(p, srp_opt_tokens, args);
2938 opt_mask |= token;
2939
2940 switch (token) {
2941 case SRP_OPT_ID_EXT:
2942 p = match_strdup(args);
a20f3a6d
IR		 2943 if (!p) {
2944 ret = -ENOMEM;
2945 goto out;
2946 }
aef9ec39
RD		 2947 target->id_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
2948 kfree(p);
2949 break;
2950
2951 case SRP_OPT_IOC_GUID:
2952 p = match_strdup(args);
a20f3a6d
IR		 2953 if (!p) {
2954 ret = -ENOMEM;
2955 goto out;
2956 }
aef9ec39
RD		 2957 target->ioc_guid = cpu_to_be64(simple_strtoull(p, NULL, 16));
2958 kfree(p);
2959 break;
2960
2961 case SRP_OPT_DGID:
2962 p = match_strdup(args);
a20f3a6d
IR		 2963 if (!p) {
2964 ret = -ENOMEM;
2965 goto out;
2966 }
aef9ec39 2967 if (strlen(p) != 32) {
e0bda7d8 2968 pr_warn("bad dest GID parameter '%s'\n", p);
ce1823f0 2969 kfree(p);
aef9ec39
RD		 2970 goto out;
2971 }
2972
2973 for (i = 0; i < 16; ++i) {
747fe000
BVA		 2974 strlcpy(dgid, p + i * 2, sizeof(dgid));
2975 if (sscanf(dgid, "%hhx",
2976 &target->orig_dgid.raw[i]) < 1) {
2977 ret = -EINVAL;
2978 kfree(p);
2979 goto out;
2980 }
aef9ec39 2981 }
bf17c1c7 2982 kfree(p);
aef9ec39
RD		 2983 break;
2984
2985 case SRP_OPT_PKEY:
2986 if (match_hex(args, &token)) {
e0bda7d8 2987 pr_warn("bad P_Key parameter '%s'\n", p);
aef9ec39
RD		 2988 goto out;
2989 }
747fe000 2990 target->pkey = cpu_to_be16(token);
aef9ec39
RD		 2991 break;
2992
2993 case SRP_OPT_SERVICE_ID:
2994 p = match_strdup(args);
a20f3a6d
IR		 2995 if (!p) {
2996 ret = -ENOMEM;
2997 goto out;
2998 }
aef9ec39
RD		 2999 target->service_id = cpu_to_be64(simple_strtoull(p, NULL, 16));
3000 kfree(p);
3001 break;
3002
3003 case SRP_OPT_MAX_SECT:
3004 if (match_int(args, &token)) {
e0bda7d8 3005 pr_warn("bad max sect parameter '%s'\n", p);
aef9ec39
RD		 3006 goto out;
3007 }
3008 target->scsi_host->max_sectors = token;
3009 break;
3010
4d73f95f
BVA		 3011 case SRP_OPT_QUEUE_SIZE:
3012 if (match_int(args, &token) || token < 1) {
3013 pr_warn("bad queue_size parameter '%s'\n", p);
3014 goto out;
3015 }
3016 target->scsi_host->can_queue = token;
3017 target->queue_size = token + SRP_RSP_SQ_SIZE +
3018 SRP_TSK_MGMT_SQ_SIZE;
3019 if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3020 target->scsi_host->cmd_per_lun = token;
3021 break;
3022
52fb2b50 3023 case SRP_OPT_MAX_CMD_PER_LUN:
4d73f95f 3024 if (match_int(args, &token) || token < 1) {
e0bda7d8
BVA		 3025 pr_warn("bad max cmd_per_lun parameter '%s'\n",
3026 p);
52fb2b50
VP		 3027 goto out;
3028 }
4d73f95f 3029 target->scsi_host->cmd_per_lun = token;
52fb2b50
VP		 3030 break;
3031
0c0450db
R		 3032 case SRP_OPT_IO_CLASS:
3033 if (match_hex(args, &token)) {
e0bda7d8 3034 pr_warn("bad IO class parameter '%s'\n", p);
0c0450db
R		 3035 goto out;
3036 }
3037 if (token != SRP_REV10_IB_IO_CLASS &&
3038 token != SRP_REV16A_IB_IO_CLASS) {
e0bda7d8
BVA		 3039 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
3040 token, SRP_REV10_IB_IO_CLASS,
3041 SRP_REV16A_IB_IO_CLASS);
0c0450db
R		 3042 goto out;
3043 }
3044 target->io_class = token;
3045 break;
3046
01cb9bcb
IR		 3047 case SRP_OPT_INITIATOR_EXT:
3048 p = match_strdup(args);
a20f3a6d
IR		 3049 if (!p) {
3050 ret = -ENOMEM;
3051 goto out;
3052 }
01cb9bcb
IR		 3053 target->initiator_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
3054 kfree(p);
3055 break;
3056
49248644
DD		 3057 case SRP_OPT_CMD_SG_ENTRIES:
3058 if (match_int(args, &token) || token < 1 || token > 255) {
e0bda7d8
BVA		 3059 pr_warn("bad max cmd_sg_entries parameter '%s'\n",
3060 p);
49248644
DD		 3061 goto out;
3062 }
3063 target->cmd_sg_cnt = token;
3064 break;
3065
c07d424d
DD		 3066 case SRP_OPT_ALLOW_EXT_SG:
3067 if (match_int(args, &token)) {
e0bda7d8 3068 pr_warn("bad allow_ext_sg parameter '%s'\n", p);
c07d424d
DD		 3069 goto out;
3070 }
3071 target->allow_ext_sg = !!token;
3072 break;
3073
3074 case SRP_OPT_SG_TABLESIZE:
3075 if (match_int(args, &token) || token < 1 ||
3076 token > SCSI_MAX_SG_CHAIN_SEGMENTS) {
e0bda7d8
BVA		 3077 pr_warn("bad max sg_tablesize parameter '%s'\n",
3078 p);
c07d424d
DD		 3079 goto out;
3080 }
3081 target->sg_tablesize = token;
3082 break;
3083
4b5e5f41
BVA		 3084 case SRP_OPT_COMP_VECTOR:
3085 if (match_int(args, &token) || token < 0) {
3086 pr_warn("bad comp_vector parameter '%s'\n", p);
3087 goto out;
3088 }
3089 target->comp_vector = token;
3090 break;
3091
7bb312e4
VP		 3092 case SRP_OPT_TL_RETRY_COUNT:
3093 if (match_int(args, &token) || token < 2 || token > 7) {
3094 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
3095 p);
3096 goto out;
3097 }
3098 target->tl_retry_count = token;
3099 break;
3100
aef9ec39 3101 default:
e0bda7d8
BVA		 3102 pr_warn("unknown parameter or missing value '%s' in target creation request\n",
3103 p);
aef9ec39
RD		 3104 goto out;
3105 }
3106 }
3107
3108 if ((opt_mask & SRP_OPT_ALL) == SRP_OPT_ALL)
3109 ret = 0;
3110 else
3111 for (i = 0; i < ARRAY_SIZE(srp_opt_tokens); ++i)
3112 if ((srp_opt_tokens[i].token & SRP_OPT_ALL) &&
3113 !(srp_opt_tokens[i].token & opt_mask))
e0bda7d8
BVA		 3114 pr_warn("target creation request is missing parameter '%s'\n",
3115 srp_opt_tokens[i].pattern);
aef9ec39 3116
4d73f95f
BVA		 3117 if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue
3118 && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3119 pr_warn("cmd_per_lun = %d > queue_size = %d\n",
3120 target->scsi_host->cmd_per_lun,
3121 target->scsi_host->can_queue);
3122
aef9ec39
RD		 3123out:
3124 kfree(options);
3125 return ret;
3126}
3127
ee959b00
TJ		 3128static ssize_t srp_create_target(struct device *dev,
3129 struct device_attribute *attr,
aef9ec39
RD		 3130 const char *buf, size_t count)
3131{
3132 struct srp_host *host =
ee959b00 3133 container_of(dev, struct srp_host, dev);
aef9ec39
RD		 3134 struct Scsi_Host *target_host;
3135 struct srp_target_port *target;
509c07bc 3136 struct srp_rdma_ch *ch;
d1b4289e
BVA		 3137 struct srp_device *srp_dev = host->srp_dev;
3138 struct ib_device *ibdev = srp_dev->dev;
d92c0da7
BVA		 3139 int ret, node_idx, node, cpu, i;
3140 bool multich = false;
aef9ec39
RD		 3141
3142 target_host = scsi_host_alloc(&srp_template,
3143 sizeof (struct srp_target_port));
3144 if (!target_host)
3145 return -ENOMEM;
3146
49248644 3147 target_host->transportt = ib_srp_transport_template;
fd1b6c4a
BVA		 3148 target_host->max_channel = 0;
3149 target_host->max_id = 1;
3c8edf0e
AR		 3150 target_host->max_lun = SRP_MAX_LUN;
3151 target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb;
5f068992 3152
aef9ec39 3153 target = host_to_target(target_host);
aef9ec39 3154
49248644
DD		 3155 target->io_class = SRP_REV16A_IB_IO_CLASS;
3156 target->scsi_host = target_host;
3157 target->srp_host = host;
3158 target->lkey = host->srp_dev->mr->lkey;
3159 target->rkey = host->srp_dev->mr->rkey;
3160 target->cmd_sg_cnt = cmd_sg_entries;
c07d424d
DD		 3161 target->sg_tablesize = indirect_sg_entries ? : cmd_sg_entries;
3162 target->allow_ext_sg = allow_ext_sg;
7bb312e4 3163 target->tl_retry_count = 7;
4d73f95f 3164 target->queue_size = SRP_DEFAULT_QUEUE_SIZE;
aef9ec39 3165
34aa654e
BVA		 3166 /*
3167 * Avoid that the SCSI host can be removed by srp_remove_target()
3168 * before this function returns.
3169 */
3170 scsi_host_get(target->scsi_host);
3171
2d7091bc
BVA		 3172 mutex_lock(&host->add_target_mutex);
3173
aef9ec39
RD		 3174 ret = srp_parse_options(buf, target);
3175 if (ret)
fb49c8bb 3176 goto out;
aef9ec39 3177
77f2c1a4
BVA		 3178 ret = scsi_init_shared_tag_map(target_host, target_host->can_queue);
3179 if (ret)
fb49c8bb 3180 goto out;
77f2c1a4 3181
4d73f95f
BVA		 3182 target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE;
3183
96fc248a
BVA		 3184 if (!srp_conn_unique(target->srp_host, target)) {
3185 shost_printk(KERN_INFO, target->scsi_host,
3186 PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
3187 be64_to_cpu(target->id_ext),
3188 be64_to_cpu(target->ioc_guid),
3189 be64_to_cpu(target->initiator_ext));
3190 ret = -EEXIST;
fb49c8bb 3191 goto out;
96fc248a
BVA		 3192 }
3193
5cfb1782 3194 if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg &&
d1b4289e 3195 target->cmd_sg_cnt < target->sg_tablesize) {
5cfb1782 3196 pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
c07d424d
DD		 3197 target->sg_tablesize = target->cmd_sg_cnt;
3198 }
3199
3200 target_host->sg_tablesize = target->sg_tablesize;
3201 target->indirect_size = target->sg_tablesize *
3202 sizeof (struct srp_direct_buf);
49248644
DD		 3203 target->max_iu_len = sizeof (struct srp_cmd) +
3204 sizeof (struct srp_indirect_buf) +
3205 target->cmd_sg_cnt * sizeof (struct srp_direct_buf);
3206
c1120f89 3207 INIT_WORK(&target->tl_err_work, srp_tl_err_work);
ef6c49d8 3208 INIT_WORK(&target->remove_work, srp_remove_work);
8f26c9ff 3209 spin_lock_init(&target->lock);
747fe000 3210 ret = ib_query_gid(ibdev, host->port, 0, &target->sgid);
2088ca66 3211 if (ret)
fb49c8bb 3212 goto out;
aef9ec39 3213
d92c0da7
BVA		 3214 ret = -ENOMEM;
3215 target->ch_count = max_t(unsigned, num_online_nodes(),
3216 min(ch_count ? :
3217 min(4 * num_online_nodes(),
3218 ibdev->num_comp_vectors),
3219 num_online_cpus()));
3220 target->ch = kcalloc(target->ch_count, sizeof(*target->ch),
3221 GFP_KERNEL);
3222 if (!target->ch)
fb49c8bb 3223 goto out;
aef9ec39 3224
d92c0da7
BVA		 3225 node_idx = 0;
3226 for_each_online_node(node) {
3227 const int ch_start = (node_idx * target->ch_count /
3228 num_online_nodes());
3229 const int ch_end = ((node_idx + 1) * target->ch_count /
3230 num_online_nodes());
3231 const int cv_start = (node_idx * ibdev->num_comp_vectors /
3232 num_online_nodes() + target->comp_vector)
3233 % ibdev->num_comp_vectors;
3234 const int cv_end = ((node_idx + 1) * ibdev->num_comp_vectors /
3235 num_online_nodes() + target->comp_vector)
3236 % ibdev->num_comp_vectors;
3237 int cpu_idx = 0;
3238
3239 for_each_online_cpu(cpu) {
3240 if (cpu_to_node(cpu) != node)
3241 continue;
3242 if (ch_start + cpu_idx >= ch_end)
3243 continue;
3244 ch = &target->ch[ch_start + cpu_idx];
3245 ch->target = target;
3246 ch->comp_vector = cv_start == cv_end ? cv_start :
3247 cv_start + cpu_idx % (cv_end - cv_start);
3248 spin_lock_init(&ch->lock);
3249 INIT_LIST_HEAD(&ch->free_tx);
3250 ret = srp_new_cm_id(ch);
3251 if (ret)
3252 goto err_disconnect;
aef9ec39 3253
d92c0da7
BVA		 3254 ret = srp_create_ch_ib(ch);
3255 if (ret)
3256 goto err_disconnect;
3257
3258 ret = srp_alloc_req_data(ch);
3259 if (ret)
3260 goto err_disconnect;
3261
3262 ret = srp_connect_ch(ch, multich);
3263 if (ret) {
3264 shost_printk(KERN_ERR, target->scsi_host,
3265 PFX "Connection %d/%d failed\n",
3266 ch_start + cpu_idx,
3267 target->ch_count);
3268 if (node_idx == 0 && cpu_idx == 0) {
3269 goto err_disconnect;
3270 } else {
3271 srp_free_ch_ib(target, ch);
3272 srp_free_req_data(target, ch);
3273 target->ch_count = ch - target->ch;
3274 break;
3275 }
3276 }
3277
3278 multich = true;
3279 cpu_idx++;
3280 }
3281 node_idx++;
aef9ec39
RD		 3282 }
3283
d92c0da7
BVA		 3284 target->scsi_host->nr_hw_queues = target->ch_count;
3285
aef9ec39
RD		 3286 ret = srp_add_target(host, target);
3287 if (ret)
3288 goto err_disconnect;
3289
34aa654e
BVA		 3290 if (target->state != SRP_TARGET_REMOVED) {
3291 shost_printk(KERN_DEBUG, target->scsi_host, PFX
3292 "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n",
3293 be64_to_cpu(target->id_ext),
3294 be64_to_cpu(target->ioc_guid),
747fe000 3295 be16_to_cpu(target->pkey),
34aa654e 3296 be64_to_cpu(target->service_id),
747fe000 3297 target->sgid.raw, target->orig_dgid.raw);
34aa654e 3298 }
e7ffde01 3299
2d7091bc
BVA		 3300 ret = count;
3301
3302out:
3303 mutex_unlock(&host->add_target_mutex);
34aa654e
BVA		 3304
3305 scsi_host_put(target->scsi_host);
3306
2d7091bc 3307 return ret;
aef9ec39
RD		 3308
3309err_disconnect:
3310 srp_disconnect_target(target);
3311
d92c0da7
BVA		 3312 for (i = 0; i < target->ch_count; i++) {
3313 ch = &target->ch[i];
3314 srp_free_ch_ib(target, ch);
3315 srp_free_req_data(target, ch);
3316 }
aef9ec39 3317
d92c0da7 3318 kfree(target->ch);
2d7091bc 3319 goto out;
aef9ec39
RD		 3320}
3321
ee959b00 3322static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target);
aef9ec39 3323
ee959b00
TJ		 3324static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
3325 char *buf)
aef9ec39 3326{
ee959b00 3327 struct srp_host *host = container_of(dev, struct srp_host, dev);
aef9ec39 3328
05321937 3329 return sprintf(buf, "%s\n", host->srp_dev->dev->name);
aef9ec39
RD		 3330}
3331
ee959b00 3332static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
aef9ec39 3333
ee959b00
TJ		 3334static ssize_t show_port(struct device *dev, struct device_attribute *attr,
3335 char *buf)
aef9ec39 3336{
ee959b00 3337 struct srp_host *host = container_of(dev, struct srp_host, dev);
aef9ec39
RD		 3338
3339 return sprintf(buf, "%d\n", host->port);
3340}
3341
ee959b00 3342static DEVICE_ATTR(port, S_IRUGO, show_port, NULL);
aef9ec39 3343
f5358a17 3344static struct srp_host *srp_add_port(struct srp_device *device, u8 port)
aef9ec39
RD		 3345{
3346 struct srp_host *host;
3347
3348 host = kzalloc(sizeof *host, GFP_KERNEL);
3349 if (!host)
3350 return NULL;
3351
3352 INIT_LIST_HEAD(&host->target_list);
b3589fd4 3353 spin_lock_init(&host->target_lock);
aef9ec39 3354 init_completion(&host->released);
2d7091bc 3355 mutex_init(&host->add_target_mutex);
05321937 3356 host->srp_dev = device;
aef9ec39
RD		 3357 host->port = port;
3358
ee959b00
TJ		 3359 host->dev.class = &srp_class;
3360 host->dev.parent = device->dev->dma_device;
d927e38c 3361 dev_set_name(&host->dev, "srp-%s-%d", device->dev->name, port);
aef9ec39 3362
ee959b00 3363 if (device_register(&host->dev))
f5358a17 3364 goto free_host;
ee959b00 3365 if (device_create_file(&host->dev, &dev_attr_add_target))
aef9ec39 3366 goto err_class;
ee959b00 3367 if (device_create_file(&host->dev, &dev_attr_ibdev))
aef9ec39 3368 goto err_class;
ee959b00 3369 if (device_create_file(&host->dev, &dev_attr_port))
aef9ec39
RD		 3370 goto err_class;
3371
3372 return host;
3373
3374err_class:
ee959b00 3375 device_unregister(&host->dev);
aef9ec39 3376
f5358a17 3377free_host:
aef9ec39
RD		 3378 kfree(host);
3379
3380 return NULL;
3381}
3382
3383static void srp_add_one(struct ib_device *device)
3384{
f5358a17
RD		 3385 struct srp_device *srp_dev;
3386 struct ib_device_attr *dev_attr;
aef9ec39 3387 struct srp_host *host;
52ede08f
BVA		 3388 int mr_page_shift, s, e, p;
3389 u64 max_pages_per_mr;
aef9ec39 3390
f5358a17
RD		 3391 dev_attr = kmalloc(sizeof *dev_attr, GFP_KERNEL);
3392 if (!dev_attr)
cf311cd4 3393 return;
aef9ec39 3394
f5358a17 3395 if (ib_query_device(device, dev_attr)) {
e0bda7d8 3396 pr_warn("Query device failed for %s\n", device->name);
f5358a17
RD		 3397 goto free_attr;
3398 }
3399
3400 srp_dev = kmalloc(sizeof *srp_dev, GFP_KERNEL);
3401 if (!srp_dev)
3402 goto free_attr;
3403
d1b4289e
BVA		 3404 srp_dev->has_fmr = (device->alloc_fmr && device->dealloc_fmr &&
3405 device->map_phys_fmr && device->unmap_fmr);
5cfb1782
BVA		 3406 srp_dev->has_fr = (dev_attr->device_cap_flags &
3407 IB_DEVICE_MEM_MGT_EXTENSIONS);
3408 if (!srp_dev->has_fmr && !srp_dev->has_fr)
3409 dev_warn(&device->dev, "neither FMR nor FR is supported\n");
3410
3411 srp_dev->use_fast_reg = (srp_dev->has_fr &&
3412 (!srp_dev->has_fmr || prefer_fr));
d1b4289e 3413
f5358a17
RD		 3414 /*
3415 * Use the smallest page size supported by the HCA, down to a
8f26c9ff
DD		 3416 * minimum of 4096 bytes. We're unlikely to build large sglists
3417 * out of smaller entries.
f5358a17 3418 */
52ede08f
BVA		 3419 mr_page_shift = max(12, ffs(dev_attr->page_size_cap) - 1);
3420 srp_dev->mr_page_size = 1 << mr_page_shift;
3421 srp_dev->mr_page_mask = ~((u64) srp_dev->mr_page_size - 1);
3422 max_pages_per_mr = dev_attr->max_mr_size;
3423 do_div(max_pages_per_mr, srp_dev->mr_page_size);
3424 srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
3425 max_pages_per_mr);
5cfb1782
BVA		 3426 if (srp_dev->use_fast_reg) {
3427 srp_dev->max_pages_per_mr =
3428 min_t(u32, srp_dev->max_pages_per_mr,
3429 dev_attr->max_fast_reg_page_list_len);
3430 }
52ede08f
BVA		 3431 srp_dev->mr_max_size = srp_dev->mr_page_size *
3432 srp_dev->max_pages_per_mr;
5cfb1782 3433 pr_debug("%s: mr_page_shift = %d, dev_attr->max_mr_size = %#llx, dev_attr->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
52ede08f 3434 device->name, mr_page_shift, dev_attr->max_mr_size,
5cfb1782 3435 dev_attr->max_fast_reg_page_list_len,
52ede08f 3436 srp_dev->max_pages_per_mr, srp_dev->mr_max_size);
f5358a17
RD		 3437
3438 INIT_LIST_HEAD(&srp_dev->dev_list);
3439
3440 srp_dev->dev = device;
3441 srp_dev->pd = ib_alloc_pd(device);
3442 if (IS_ERR(srp_dev->pd))
3443 goto free_dev;
3444
3445 srp_dev->mr = ib_get_dma_mr(srp_dev->pd,
3446 IB_ACCESS_LOCAL_WRITE |
3447 IB_ACCESS_REMOTE_READ |
3448 IB_ACCESS_REMOTE_WRITE);
3449 if (IS_ERR(srp_dev->mr))
3450 goto err_pd;
3451
07ebafba 3452 if (device->node_type == RDMA_NODE_IB_SWITCH) {
aef9ec39
RD		 3453 s = 0;
3454 e = 0;
3455 } else {
3456 s = 1;
3457 e = device->phys_port_cnt;
3458 }
3459
3460 for (p = s; p <= e; ++p) {
f5358a17 3461 host = srp_add_port(srp_dev, p);
aef9ec39 3462 if (host)
f5358a17 3463 list_add_tail(&host->list, &srp_dev->dev_list);
aef9ec39
RD		 3464 }
3465
f5358a17
RD		 3466 ib_set_client_data(device, &srp_client, srp_dev);
3467
3468 goto free_attr;
3469
3470err_pd:
3471 ib_dealloc_pd(srp_dev->pd);
3472
3473free_dev:
3474 kfree(srp_dev);
3475
3476free_attr:
3477 kfree(dev_attr);
aef9ec39
RD		 3478}
3479
3480static void srp_remove_one(struct ib_device *device)
3481{
f5358a17 3482 struct srp_device *srp_dev;
aef9ec39 3483 struct srp_host *host, *tmp_host;
ef6c49d8 3484 struct srp_target_port *target;
aef9ec39 3485
f5358a17 3486 srp_dev = ib_get_client_data(device, &srp_client);
1fe0cb84
DB		 3487 if (!srp_dev)
3488 return;
aef9ec39 3489
f5358a17 3490 list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
ee959b00 3491 device_unregister(&host->dev);
aef9ec39
RD		 3492 /*
3493 * Wait for the sysfs entry to go away, so that no new
3494 * target ports can be created.
3495 */
3496 wait_for_completion(&host->released);
3497
3498 /*
ef6c49d8 3499 * Remove all target ports.
aef9ec39 3500 */
b3589fd4 3501 spin_lock(&host->target_lock);
ef6c49d8
BVA		 3502 list_for_each_entry(target, &host->target_list, list)
3503 srp_queue_remove_work(target);
b3589fd4 3504 spin_unlock(&host->target_lock);
aef9ec39
RD		 3505
3506 /*
bcc05910 3507 * Wait for tl_err and target port removal tasks.
aef9ec39 3508 */
ef6c49d8 3509 flush_workqueue(system_long_wq);
bcc05910 3510 flush_workqueue(srp_remove_wq);
aef9ec39 3511
aef9ec39
RD		 3512 kfree(host);
3513 }
3514
f5358a17
RD		 3515 ib_dereg_mr(srp_dev->mr);
3516 ib_dealloc_pd(srp_dev->pd);
3517
3518 kfree(srp_dev);
aef9ec39
RD		 3519}
3520
3236822b 3521static struct srp_function_template ib_srp_transport_functions = {
ed9b2264
BVA		 3522 .has_rport_state = true,
3523 .reset_timer_if_blocked = true,
a95cadb9 3524 .reconnect_delay = &srp_reconnect_delay,
ed9b2264
BVA		 3525 .fast_io_fail_tmo = &srp_fast_io_fail_tmo,
3526 .dev_loss_tmo = &srp_dev_loss_tmo,
3527 .reconnect = srp_rport_reconnect,
dc1bdbd9 3528 .rport_delete = srp_rport_delete,
ed9b2264 3529 .terminate_rport_io = srp_terminate_io,
3236822b
FT		 3530};
3531
aef9ec39
RD		 3532static int __init srp_init_module(void)
3533{
3534 int ret;
3535
dcb4cb85 3536 BUILD_BUG_ON(FIELD_SIZEOF(struct ib_wc, wr_id) < sizeof(void *));
dd5e6e38 3537
49248644 3538 if (srp_sg_tablesize) {
e0bda7d8 3539 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
49248644
DD		 3540 if (!cmd_sg_entries)
3541 cmd_sg_entries = srp_sg_tablesize;
3542 }
3543
3544 if (!cmd_sg_entries)
3545 cmd_sg_entries = SRP_DEF_SG_TABLESIZE;
3546
3547 if (cmd_sg_entries > 255) {
e0bda7d8 3548 pr_warn("Clamping cmd_sg_entries to 255\n");
49248644 3549 cmd_sg_entries = 255;
1e89a194
DD		 3550 }
3551
c07d424d
DD		 3552 if (!indirect_sg_entries)
3553 indirect_sg_entries = cmd_sg_entries;
3554 else if (indirect_sg_entries < cmd_sg_entries) {
e0bda7d8
BVA		 3555 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n",
3556 cmd_sg_entries);
c07d424d
DD		 3557 indirect_sg_entries = cmd_sg_entries;
3558 }
3559
bcc05910 3560 srp_remove_wq = create_workqueue("srp_remove");
da05be29
WY		 3561 if (!srp_remove_wq) {
3562 ret = -ENOMEM;
bcc05910
BVA		 3563 goto out;
3564 }
3565
3566 ret = -ENOMEM;
3236822b
FT		 3567 ib_srp_transport_template =
3568 srp_attach_transport(&ib_srp_transport_functions);
3569 if (!ib_srp_transport_template)
bcc05910 3570 goto destroy_wq;
3236822b 3571
aef9ec39
RD		 3572 ret = class_register(&srp_class);
3573 if (ret) {
e0bda7d8 3574 pr_err("couldn't register class infiniband_srp\n");
bcc05910 3575 goto release_tr;
aef9ec39
RD		 3576 }
3577
c1a0b23b
MT		 3578 ib_sa_register_client(&srp_sa_client);
3579
aef9ec39
RD		 3580 ret = ib_register_client(&srp_client);
3581 if (ret) {
e0bda7d8 3582 pr_err("couldn't register IB client\n");
bcc05910 3583 goto unreg_sa;
aef9ec39
RD		 3584 }
3585
bcc05910
BVA		 3586out:
3587 return ret;
3588
3589unreg_sa:
3590 ib_sa_unregister_client(&srp_sa_client);
3591 class_unregister(&srp_class);
3592
3593release_tr:
3594 srp_release_transport(ib_srp_transport_template);
3595
3596destroy_wq:
3597 destroy_workqueue(srp_remove_wq);
3598 goto out;
aef9ec39
RD		 3599}
3600
3601static void __exit srp_cleanup_module(void)
3602{
3603 ib_unregister_client(&srp_client);
c1a0b23b 3604 ib_sa_unregister_client(&srp_sa_client);
aef9ec39 3605 class_unregister(&srp_class);
3236822b 3606 srp_release_transport(ib_srp_transport_template);
bcc05910 3607 destroy_workqueue(srp_remove_wq);
aef9ec39
RD		 3608}
3609
3610module_init(srp_init_module);
3611module_exit(srp_cleanup_module);
Ubuntu Artful kernel mirror