]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - drivers/infiniband/ulp/srp/ib_srp.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
IB/srp: Align to generic logging helpers
[mirror_ubuntu-bionic-kernel.git] / drivers / infiniband / ulp / srp / ib_srp.c
CommitLineData
aef9ec39
RD		 1/*
2 * Copyright (c) 2005 Cisco Systems. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
aef9ec39
RD		 31 */
32
d236cd0e 33#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
e0bda7d8 34
aef9ec39
RD		 35#include <linux/module.h>
36#include <linux/init.h>
37#include <linux/slab.h>
38#include <linux/err.h>
39#include <linux/string.h>
40#include <linux/parser.h>
41#include <linux/random.h>
de25968c 42#include <linux/jiffies.h>
56b5390c 43#include <rdma/ib_cache.h>
aef9ec39 44
60063497 45#include <linux/atomic.h>
aef9ec39
RD		 46
47#include <scsi/scsi.h>
48#include <scsi/scsi_device.h>
49#include <scsi/scsi_dbg.h>
71444b97 50#include <scsi/scsi_tcq.h>
aef9ec39 51#include <scsi/srp.h>
3236822b 52#include <scsi/scsi_transport_srp.h>
aef9ec39 53
aef9ec39
RD		 54#include "ib_srp.h"
55
56#define DRV_NAME "ib_srp"
57#define PFX DRV_NAME ": "
e8ca4135
VP		 58#define DRV_VERSION "1.0"
59#define DRV_RELDATE "July 1, 2013"
aef9ec39
RD		 60
61MODULE_AUTHOR("Roland Dreier");
62MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator "
63 "v" DRV_VERSION " (" DRV_RELDATE ")");
64MODULE_LICENSE("Dual BSD/GPL");
65
49248644
DD		 66static unsigned int srp_sg_tablesize;
67static unsigned int cmd_sg_entries;
c07d424d
DD		 68static unsigned int indirect_sg_entries;
69static bool allow_ext_sg;
5cfb1782 70static bool prefer_fr;
b1b8854d 71static bool register_always;
49248644 72static int topspin_workarounds = 1;
74b0a15b 73
49248644
DD		 74module_param(srp_sg_tablesize, uint, 0444);
75MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries");
74b0a15b 76
49248644
DD		 77module_param(cmd_sg_entries, uint, 0444);
78MODULE_PARM_DESC(cmd_sg_entries,
79 "Default number of gather/scatter entries in the SRP command (default is 12, max 255)");
aef9ec39 80
c07d424d
DD		 81module_param(indirect_sg_entries, uint, 0444);
82MODULE_PARM_DESC(indirect_sg_entries,
83 "Default max number of gather/scatter entries (default is 12, max is " __stringify(SCSI_MAX_SG_CHAIN_SEGMENTS) ")");
84
85module_param(allow_ext_sg, bool, 0444);
86MODULE_PARM_DESC(allow_ext_sg,
87 "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)");
88
aef9ec39
RD		 89module_param(topspin_workarounds, int, 0444);
90MODULE_PARM_DESC(topspin_workarounds,
91 "Enable workarounds for Topspin/Cisco SRP target bugs if != 0");
92
5cfb1782
BVA		 93module_param(prefer_fr, bool, 0444);
94MODULE_PARM_DESC(prefer_fr,
95"Whether to use fast registration if both FMR and fast registration are supported");
96
b1b8854d
BVA		 97module_param(register_always, bool, 0444);
98MODULE_PARM_DESC(register_always,
99 "Use memory registration even for contiguous memory regions");
100
ed9b2264
BVA		 101static struct kernel_param_ops srp_tmo_ops;
102
a95cadb9
BVA		 103static int srp_reconnect_delay = 10;
104module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay,
105 S_IRUGO | S_IWUSR);
106MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts");
107
ed9b2264
BVA		 108static int srp_fast_io_fail_tmo = 15;
109module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo,
110 S_IRUGO | S_IWUSR);
111MODULE_PARM_DESC(fast_io_fail_tmo,
112 "Number of seconds between the observation of a transport"
113 " layer error and failing all I/O. \"off\" means that this"
114 " functionality is disabled.");
115
a95cadb9 116static int srp_dev_loss_tmo = 600;
ed9b2264
BVA		 117module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo,
118 S_IRUGO | S_IWUSR);
119MODULE_PARM_DESC(dev_loss_tmo,
120 "Maximum number of seconds that the SRP transport should"
121 " insulate transport layer errors. After this time has been"
122 " exceeded the SCSI host is removed. Should be"
123 " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT)
124 " if fast_io_fail_tmo has not been set. \"off\" means that"
125 " this functionality is disabled.");
126
d92c0da7
BVA		 127static unsigned ch_count;
128module_param(ch_count, uint, 0444);
129MODULE_PARM_DESC(ch_count,
130 "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA.");
131
aef9ec39
RD		 132static void srp_add_one(struct ib_device *device);
133static void srp_remove_one(struct ib_device *device);
509c07bc
BVA		 134static void srp_recv_completion(struct ib_cq *cq, void *ch_ptr);
135static void srp_send_completion(struct ib_cq *cq, void *ch_ptr);
aef9ec39
RD		 136static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
137
3236822b 138static struct scsi_transport_template *ib_srp_transport_template;
bcc05910 139static struct workqueue_struct *srp_remove_wq;
3236822b 140
aef9ec39
RD		 141static struct ib_client srp_client = {
142 .name = "srp",
143 .add = srp_add_one,
144 .remove = srp_remove_one
145};
146
c1a0b23b
MT		 147static struct ib_sa_client srp_sa_client;
148
ed9b2264
BVA		 149static int srp_tmo_get(char *buffer, const struct kernel_param *kp)
150{
151 int tmo = *(int *)kp->arg;
152
153 if (tmo >= 0)
154 return sprintf(buffer, "%d", tmo);
155 else
156 return sprintf(buffer, "off");
157}
158
159static int srp_tmo_set(const char *val, const struct kernel_param *kp)
160{
161 int tmo, res;
162
163 if (strncmp(val, "off", 3) != 0) {
164 res = kstrtoint(val, 0, &tmo);
165 if (res)
166 goto out;
167 } else {
168 tmo = -1;
169 }
a95cadb9
BVA		 170 if (kp->arg == &srp_reconnect_delay)
171 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo,
172 srp_dev_loss_tmo);
173 else if (kp->arg == &srp_fast_io_fail_tmo)
174 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo);
ed9b2264 175 else
a95cadb9
BVA		 176 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo,
177 tmo);
ed9b2264
BVA		 178 if (res)
179 goto out;
180 *(int *)kp->arg = tmo;
181
182out:
183 return res;
184}
185
186static struct kernel_param_ops srp_tmo_ops = {
187 .get = srp_tmo_get,
188 .set = srp_tmo_set,
189};
190
aef9ec39
RD		 191static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
192{
193 return (struct srp_target_port *) host->hostdata;
194}
195
196static const char *srp_target_info(struct Scsi_Host *host)
197{
198 return host_to_target(host)->target_name;
199}
200
5d7cbfd6
RD		 201static int srp_target_is_topspin(struct srp_target_port *target)
202{
203 static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad };
3d1ff48d 204 static const u8 cisco_oui[3] = { 0x00, 0x1b, 0x0d };
5d7cbfd6
RD		 205
206 return topspin_workarounds &&
3d1ff48d
RK		 207 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) ||
208 !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui));
5d7cbfd6
RD		 209}
210
aef9ec39
RD		 211static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size,
212 gfp_t gfp_mask,
213 enum dma_data_direction direction)
214{
215 struct srp_iu *iu;
216
217 iu = kmalloc(sizeof *iu, gfp_mask);
218 if (!iu)
219 goto out;
220
221 iu->buf = kzalloc(size, gfp_mask);
222 if (!iu->buf)
223 goto out_free_iu;
224
05321937
GKH		 225 iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size,
226 direction);
227 if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma))
aef9ec39
RD		 228 goto out_free_buf;
229
230 iu->size = size;
231 iu->direction = direction;
232
233 return iu;
234
235out_free_buf:
236 kfree(iu->buf);
237out_free_iu:
238 kfree(iu);
239out:
240 return NULL;
241}
242
243static void srp_free_iu(struct srp_host *host, struct srp_iu *iu)
244{
245 if (!iu)
246 return;
247
05321937
GKH		 248 ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size,
249 iu->direction);
aef9ec39
RD		 250 kfree(iu->buf);
251 kfree(iu);
252}
253
254static void srp_qp_event(struct ib_event *event, void *context)
255{
57363d98
SG		 256 pr_debug("QP event %s (%d)\n",
257 ib_event_msg(event->event), event->event);
aef9ec39
RD		 258}
259
260static int srp_init_qp(struct srp_target_port *target,
261 struct ib_qp *qp)
262{
263 struct ib_qp_attr *attr;
264 int ret;
265
266 attr = kmalloc(sizeof *attr, GFP_KERNEL);
267 if (!attr)
268 return -ENOMEM;
269
56b5390c
BVA		 270 ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev,
271 target->srp_host->port,
272 be16_to_cpu(target->pkey),
273 &attr->pkey_index);
aef9ec39
RD		 274 if (ret)
275 goto out;
276
277 attr->qp_state = IB_QPS_INIT;
278 attr->qp_access_flags = (IB_ACCESS_REMOTE_READ |
279 IB_ACCESS_REMOTE_WRITE);
280 attr->port_num = target->srp_host->port;
281
282 ret = ib_modify_qp(qp, attr,
283 IB_QP_STATE |
284 IB_QP_PKEY_INDEX |
285 IB_QP_ACCESS_FLAGS |
286 IB_QP_PORT);
287
288out:
289 kfree(attr);
290 return ret;
291}
292
509c07bc 293static int srp_new_cm_id(struct srp_rdma_ch *ch)
9fe4bcf4 294{
509c07bc 295 struct srp_target_port *target = ch->target;
9fe4bcf4
DD		 296 struct ib_cm_id *new_cm_id;
297
05321937 298 new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
509c07bc 299 srp_cm_handler, ch);
9fe4bcf4
DD		 300 if (IS_ERR(new_cm_id))
301 return PTR_ERR(new_cm_id);
302
509c07bc
BVA		 303 if (ch->cm_id)
304 ib_destroy_cm_id(ch->cm_id);
305 ch->cm_id = new_cm_id;
306 ch->path.sgid = target->sgid;
307 ch->path.dgid = target->orig_dgid;
308 ch->path.pkey = target->pkey;
309 ch->path.service_id = target->service_id;
9fe4bcf4
DD		 310
311 return 0;
312}
313
d1b4289e
BVA		 314static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target)
315{
316 struct srp_device *dev = target->srp_host->srp_dev;
317 struct ib_fmr_pool_param fmr_param;
318
319 memset(&fmr_param, 0, sizeof(fmr_param));
320 fmr_param.pool_size = target->scsi_host->can_queue;
321 fmr_param.dirty_watermark = fmr_param.pool_size / 4;
322 fmr_param.cache = 1;
52ede08f
BVA		 323 fmr_param.max_pages_per_fmr = dev->max_pages_per_mr;
324 fmr_param.page_shift = ilog2(dev->mr_page_size);
d1b4289e
BVA		 325 fmr_param.access = (IB_ACCESS_LOCAL_WRITE |
326 IB_ACCESS_REMOTE_WRITE |
327 IB_ACCESS_REMOTE_READ);
328
329 return ib_create_fmr_pool(dev->pd, &fmr_param);
330}
331
5cfb1782
BVA		 332/**
333 * srp_destroy_fr_pool() - free the resources owned by a pool
334 * @pool: Fast registration pool to be destroyed.
335 */
336static void srp_destroy_fr_pool(struct srp_fr_pool *pool)
337{
338 int i;
339 struct srp_fr_desc *d;
340
341 if (!pool)
342 return;
343
344 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
345 if (d->frpl)
346 ib_free_fast_reg_page_list(d->frpl);
347 if (d->mr)
348 ib_dereg_mr(d->mr);
349 }
350 kfree(pool);
351}
352
353/**
354 * srp_create_fr_pool() - allocate and initialize a pool for fast registration
355 * @device: IB device to allocate fast registration descriptors for.
356 * @pd: Protection domain associated with the FR descriptors.
357 * @pool_size: Number of descriptors to allocate.
358 * @max_page_list_len: Maximum fast registration work request page list length.
359 */
360static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
361 struct ib_pd *pd, int pool_size,
362 int max_page_list_len)
363{
364 struct srp_fr_pool *pool;
365 struct srp_fr_desc *d;
366 struct ib_mr *mr;
367 struct ib_fast_reg_page_list *frpl;
368 int i, ret = -EINVAL;
369
370 if (pool_size <= 0)
371 goto err;
372 ret = -ENOMEM;
373 pool = kzalloc(sizeof(struct srp_fr_pool) +
374 pool_size * sizeof(struct srp_fr_desc), GFP_KERNEL);
375 if (!pool)
376 goto err;
377 pool->size = pool_size;
378 pool->max_page_list_len = max_page_list_len;
379 spin_lock_init(&pool->lock);
380 INIT_LIST_HEAD(&pool->free_list);
381
382 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
383 mr = ib_alloc_fast_reg_mr(pd, max_page_list_len);
384 if (IS_ERR(mr)) {
385 ret = PTR_ERR(mr);
386 goto destroy_pool;
387 }
388 d->mr = mr;
389 frpl = ib_alloc_fast_reg_page_list(device, max_page_list_len);
390 if (IS_ERR(frpl)) {
391 ret = PTR_ERR(frpl);
392 goto destroy_pool;
393 }
394 d->frpl = frpl;
395 list_add_tail(&d->entry, &pool->free_list);
396 }
397
398out:
399 return pool;
400
401destroy_pool:
402 srp_destroy_fr_pool(pool);
403
404err:
405 pool = ERR_PTR(ret);
406 goto out;
407}
408
409/**
410 * srp_fr_pool_get() - obtain a descriptor suitable for fast registration
411 * @pool: Pool to obtain descriptor from.
412 */
413static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool)
414{
415 struct srp_fr_desc *d = NULL;
416 unsigned long flags;
417
418 spin_lock_irqsave(&pool->lock, flags);
419 if (!list_empty(&pool->free_list)) {
420 d = list_first_entry(&pool->free_list, typeof(*d), entry);
421 list_del(&d->entry);
422 }
423 spin_unlock_irqrestore(&pool->lock, flags);
424
425 return d;
426}
427
428/**
429 * srp_fr_pool_put() - put an FR descriptor back in the free list
430 * @pool: Pool the descriptor was allocated from.
431 * @desc: Pointer to an array of fast registration descriptor pointers.
432 * @n: Number of descriptors to put back.
433 *
434 * Note: The caller must already have queued an invalidation request for
435 * desc->mr->rkey before calling this function.
436 */
437static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc,
438 int n)
439{
440 unsigned long flags;
441 int i;
442
443 spin_lock_irqsave(&pool->lock, flags);
444 for (i = 0; i < n; i++)
445 list_add(&desc[i]->entry, &pool->free_list);
446 spin_unlock_irqrestore(&pool->lock, flags);
447}
448
449static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
450{
451 struct srp_device *dev = target->srp_host->srp_dev;
452
453 return srp_create_fr_pool(dev->dev, dev->pd,
454 target->scsi_host->can_queue,
455 dev->max_pages_per_mr);
456}
457
7dad6b2e
BVA		 458/**
459 * srp_destroy_qp() - destroy an RDMA queue pair
460 * @ch: SRP RDMA channel.
461 *
462 * Change a queue pair into the error state and wait until all receive
463 * completions have been processed before destroying it. This avoids that
464 * the receive completion handler can access the queue pair while it is
465 * being destroyed.
466 */
467static void srp_destroy_qp(struct srp_rdma_ch *ch)
468{
469 struct srp_target_port *target = ch->target;
470 static struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
471 static struct ib_recv_wr wr = { .wr_id = SRP_LAST_WR_ID };
472 struct ib_recv_wr *bad_wr;
473 int ret;
474
475 /* Destroying a QP and reusing ch->done is only safe if not connected */
476 WARN_ON_ONCE(target->connected);
477
478 ret = ib_modify_qp(ch->qp, &attr, IB_QP_STATE);
479 WARN_ONCE(ret, "ib_cm_init_qp_attr() returned %d\n", ret);
480 if (ret)
481 goto out;
482
483 init_completion(&ch->done);
484 ret = ib_post_recv(ch->qp, &wr, &bad_wr);
485 WARN_ONCE(ret, "ib_post_recv() returned %d\n", ret);
486 if (ret == 0)
487 wait_for_completion(&ch->done);
488
489out:
490 ib_destroy_qp(ch->qp);
491}
492
509c07bc 493static int srp_create_ch_ib(struct srp_rdma_ch *ch)
aef9ec39 494{
509c07bc 495 struct srp_target_port *target = ch->target;
62154b2e 496 struct srp_device *dev = target->srp_host->srp_dev;
aef9ec39 497 struct ib_qp_init_attr *init_attr;
73aa89ed
IR		 498 struct ib_cq *recv_cq, *send_cq;
499 struct ib_qp *qp;
d1b4289e 500 struct ib_fmr_pool *fmr_pool = NULL;
5cfb1782
BVA		 501 struct srp_fr_pool *fr_pool = NULL;
502 const int m = 1 + dev->use_fast_reg;
aef9ec39
RD		 503 int ret;
504
505 init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
506 if (!init_attr)
507 return -ENOMEM;
508
7dad6b2e 509 /* + 1 for SRP_LAST_WR_ID */
509c07bc 510 recv_cq = ib_create_cq(dev->dev, srp_recv_completion, NULL, ch,
7dad6b2e 511 target->queue_size + 1, ch->comp_vector);
73aa89ed
IR		 512 if (IS_ERR(recv_cq)) {
513 ret = PTR_ERR(recv_cq);
da9d2f07 514 goto err;
aef9ec39
RD		 515 }
516
509c07bc
BVA		 517 send_cq = ib_create_cq(dev->dev, srp_send_completion, NULL, ch,
518 m * target->queue_size, ch->comp_vector);
73aa89ed
IR		 519 if (IS_ERR(send_cq)) {
520 ret = PTR_ERR(send_cq);
da9d2f07 521 goto err_recv_cq;
9c03dc9f
BVA		 522 }
523
73aa89ed 524 ib_req_notify_cq(recv_cq, IB_CQ_NEXT_COMP);
aef9ec39
RD		 525
526 init_attr->event_handler = srp_qp_event;
5cfb1782 527 init_attr->cap.max_send_wr = m * target->queue_size;
7dad6b2e 528 init_attr->cap.max_recv_wr = target->queue_size + 1;
aef9ec39
RD		 529 init_attr->cap.max_recv_sge = 1;
530 init_attr->cap.max_send_sge = 1;
5cfb1782 531 init_attr->sq_sig_type = IB_SIGNAL_REQ_WR;
aef9ec39 532 init_attr->qp_type = IB_QPT_RC;
73aa89ed
IR		 533 init_attr->send_cq = send_cq;
534 init_attr->recv_cq = recv_cq;
aef9ec39 535
62154b2e 536 qp = ib_create_qp(dev->pd, init_attr);
73aa89ed
IR		 537 if (IS_ERR(qp)) {
538 ret = PTR_ERR(qp);
da9d2f07 539 goto err_send_cq;
aef9ec39
RD		 540 }
541
73aa89ed 542 ret = srp_init_qp(target, qp);
da9d2f07
RD		 543 if (ret)
544 goto err_qp;
aef9ec39 545
5cfb1782
BVA		 546 if (dev->use_fast_reg && dev->has_fr) {
547 fr_pool = srp_alloc_fr_pool(target);
548 if (IS_ERR(fr_pool)) {
549 ret = PTR_ERR(fr_pool);
550 shost_printk(KERN_WARNING, target->scsi_host, PFX
551 "FR pool allocation failed (%d)\n", ret);
552 goto err_qp;
553 }
509c07bc
BVA		 554 if (ch->fr_pool)
555 srp_destroy_fr_pool(ch->fr_pool);
556 ch->fr_pool = fr_pool;
5cfb1782 557 } else if (!dev->use_fast_reg && dev->has_fmr) {
d1b4289e
BVA		 558 fmr_pool = srp_alloc_fmr_pool(target);
559 if (IS_ERR(fmr_pool)) {
560 ret = PTR_ERR(fmr_pool);
561 shost_printk(KERN_WARNING, target->scsi_host, PFX
562 "FMR pool allocation failed (%d)\n", ret);
563 goto err_qp;
564 }
509c07bc
BVA		 565 if (ch->fmr_pool)
566 ib_destroy_fmr_pool(ch->fmr_pool);
567 ch->fmr_pool = fmr_pool;
d1b4289e
BVA		 568 }
569
509c07bc 570 if (ch->qp)
7dad6b2e 571 srp_destroy_qp(ch);
509c07bc
BVA		 572 if (ch->recv_cq)
573 ib_destroy_cq(ch->recv_cq);
574 if (ch->send_cq)
575 ib_destroy_cq(ch->send_cq);
73aa89ed 576
509c07bc
BVA		 577 ch->qp = qp;
578 ch->recv_cq = recv_cq;
579 ch->send_cq = send_cq;
73aa89ed 580
da9d2f07
RD		 581 kfree(init_attr);
582 return 0;
583
584err_qp:
73aa89ed 585 ib_destroy_qp(qp);
da9d2f07
RD		 586
587err_send_cq:
73aa89ed 588 ib_destroy_cq(send_cq);
da9d2f07
RD		 589
590err_recv_cq:
73aa89ed 591 ib_destroy_cq(recv_cq);
da9d2f07
RD		 592
593err:
aef9ec39
RD		 594 kfree(init_attr);
595 return ret;
596}
597
4d73f95f
BVA		 598/*
599 * Note: this function may be called without srp_alloc_iu_bufs() having been
509c07bc 600 * invoked. Hence the ch->[rt]x_ring checks.
4d73f95f 601 */
509c07bc
BVA		 602static void srp_free_ch_ib(struct srp_target_port *target,
603 struct srp_rdma_ch *ch)
aef9ec39 604{
5cfb1782 605 struct srp_device *dev = target->srp_host->srp_dev;
aef9ec39
RD		 606 int i;
607
d92c0da7
BVA		 608 if (!ch->target)
609 return;
610
509c07bc
BVA		 611 if (ch->cm_id) {
612 ib_destroy_cm_id(ch->cm_id);
613 ch->cm_id = NULL;
394c595e
BVA		 614 }
615
d92c0da7
BVA		 616 /* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */
617 if (!ch->qp)
618 return;
619
5cfb1782 620 if (dev->use_fast_reg) {
509c07bc
BVA		 621 if (ch->fr_pool)
622 srp_destroy_fr_pool(ch->fr_pool);
5cfb1782 623 } else {
509c07bc
BVA		 624 if (ch->fmr_pool)
625 ib_destroy_fmr_pool(ch->fmr_pool);
5cfb1782 626 }
7dad6b2e 627 srp_destroy_qp(ch);
509c07bc
BVA		 628 ib_destroy_cq(ch->send_cq);
629 ib_destroy_cq(ch->recv_cq);
aef9ec39 630
d92c0da7
BVA		 631 /*
632 * Avoid that the SCSI error handler tries to use this channel after
633 * it has been freed. The SCSI error handler can namely continue
634 * trying to perform recovery actions after scsi_remove_host()
635 * returned.
636 */
637 ch->target = NULL;
638
509c07bc
BVA		 639 ch->qp = NULL;
640 ch->send_cq = ch->recv_cq = NULL;
73aa89ed 641
509c07bc 642 if (ch->rx_ring) {
4d73f95f 643 for (i = 0; i < target->queue_size; ++i)
509c07bc
BVA		 644 srp_free_iu(target->srp_host, ch->rx_ring[i]);
645 kfree(ch->rx_ring);
646 ch->rx_ring = NULL;
4d73f95f 647 }
509c07bc 648 if (ch->tx_ring) {
4d73f95f 649 for (i = 0; i < target->queue_size; ++i)
509c07bc
BVA		 650 srp_free_iu(target->srp_host, ch->tx_ring[i]);
651 kfree(ch->tx_ring);
652 ch->tx_ring = NULL;
4d73f95f 653 }
aef9ec39
RD		 654}
655
656static void srp_path_rec_completion(int status,
657 struct ib_sa_path_rec *pathrec,
509c07bc 658 void *ch_ptr)
aef9ec39 659{
509c07bc
BVA		 660 struct srp_rdma_ch *ch = ch_ptr;
661 struct srp_target_port *target = ch->target;
aef9ec39 662
509c07bc 663 ch->status = status;
aef9ec39 664 if (status)
7aa54bd7
DD		 665 shost_printk(KERN_ERR, target->scsi_host,
666 PFX "Got failed path rec status %d\n", status);
aef9ec39 667 else
509c07bc
BVA		 668 ch->path = *pathrec;
669 complete(&ch->done);
aef9ec39
RD		 670}
671
509c07bc 672static int srp_lookup_path(struct srp_rdma_ch *ch)
aef9ec39 673{
509c07bc 674 struct srp_target_port *target = ch->target;
a702adce
BVA		 675 int ret;
676
509c07bc
BVA		 677 ch->path.numb_path = 1;
678
679 init_completion(&ch->done);
680
681 ch->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
682 target->srp_host->srp_dev->dev,
683 target->srp_host->port,
684 &ch->path,
685 IB_SA_PATH_REC_SERVICE_ID |
686 IB_SA_PATH_REC_DGID |
687 IB_SA_PATH_REC_SGID |
688 IB_SA_PATH_REC_NUMB_PATH |
689 IB_SA_PATH_REC_PKEY,
690 SRP_PATH_REC_TIMEOUT_MS,
691 GFP_KERNEL,
692 srp_path_rec_completion,
693 ch, &ch->path_query);
694 if (ch->path_query_id < 0)
695 return ch->path_query_id;
696
697 ret = wait_for_completion_interruptible(&ch->done);
a702adce
BVA		 698 if (ret < 0)
699 return ret;
aef9ec39 700
509c07bc 701 if (ch->status < 0)
7aa54bd7
DD		 702 shost_printk(KERN_WARNING, target->scsi_host,
703 PFX "Path record query failed\n");
aef9ec39 704
509c07bc 705 return ch->status;
aef9ec39
RD		 706}
707
d92c0da7 708static int srp_send_req(struct srp_rdma_ch *ch, bool multich)
aef9ec39 709{
509c07bc 710 struct srp_target_port *target = ch->target;
aef9ec39
RD		 711 struct {
712 struct ib_cm_req_param param;
713 struct srp_login_req priv;
714 } *req = NULL;
715 int status;
716
717 req = kzalloc(sizeof *req, GFP_KERNEL);
718 if (!req)
719 return -ENOMEM;
720
509c07bc 721 req->param.primary_path = &ch->path;
aef9ec39
RD		 722 req->param.alternate_path = NULL;
723 req->param.service_id = target->service_id;
509c07bc
BVA		 724 req->param.qp_num = ch->qp->qp_num;
725 req->param.qp_type = ch->qp->qp_type;
aef9ec39
RD		 726 req->param.private_data = &req->priv;
727 req->param.private_data_len = sizeof req->priv;
728 req->param.flow_control = 1;
729
730 get_random_bytes(&req->param.starting_psn, 4);
731 req->param.starting_psn &= 0xffffff;
732
733 /*
734 * Pick some arbitrary defaults here; we could make these
735 * module parameters if anyone cared about setting them.
736 */
737 req->param.responder_resources = 4;
738 req->param.remote_cm_response_timeout = 20;
739 req->param.local_cm_response_timeout = 20;
7bb312e4 740 req->param.retry_count = target->tl_retry_count;
aef9ec39
RD		 741 req->param.rnr_retry_count = 7;
742 req->param.max_cm_retries = 15;
743
744 req->priv.opcode = SRP_LOGIN_REQ;
745 req->priv.tag = 0;
49248644 746 req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len);
aef9ec39
RD		 747 req->priv.req_buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
748 SRP_BUF_FORMAT_INDIRECT);
d92c0da7
BVA		 749 req->priv.req_flags = (multich ? SRP_MULTICHAN_MULTI :
750 SRP_MULTICHAN_SINGLE);
0c0450db 751 /*
3cd96564 752 * In the published SRP specification (draft rev. 16a), the
0c0450db
R		 753 * port identifier format is 8 bytes of ID extension followed
754 * by 8 bytes of GUID. Older drafts put the two halves in the
755 * opposite order, so that the GUID comes first.
756 *
757 * Targets conforming to these obsolete drafts can be
758 * recognized by the I/O Class they report.
759 */
760 if (target->io_class == SRP_REV10_IB_IO_CLASS) {
761 memcpy(req->priv.initiator_port_id,
747fe000 762 &target->sgid.global.interface_id, 8);
0c0450db 763 memcpy(req->priv.initiator_port_id + 8,
01cb9bcb 764 &target->initiator_ext, 8);
0c0450db
R		 765 memcpy(req->priv.target_port_id, &target->ioc_guid, 8);
766 memcpy(req->priv.target_port_id + 8, &target->id_ext, 8);
767 } else {
768 memcpy(req->priv.initiator_port_id,
01cb9bcb
IR		 769 &target->initiator_ext, 8);
770 memcpy(req->priv.initiator_port_id + 8,
747fe000 771 &target->sgid.global.interface_id, 8);
0c0450db
R		 772 memcpy(req->priv.target_port_id, &target->id_ext, 8);
773 memcpy(req->priv.target_port_id + 8, &target->ioc_guid, 8);
774 }
775
aef9ec39
RD		 776 /*
777 * Topspin/Cisco SRP targets will reject our login unless we
01cb9bcb
IR		 778 * zero out the first 8 bytes of our initiator port ID and set
779 * the second 8 bytes to the local node GUID.
aef9ec39 780 */
5d7cbfd6 781 if (srp_target_is_topspin(target)) {
7aa54bd7
DD		 782 shost_printk(KERN_DEBUG, target->scsi_host,
783 PFX "Topspin/Cisco initiator port ID workaround "
784 "activated for target GUID %016llx\n",
785 (unsigned long long) be64_to_cpu(target->ioc_guid));
aef9ec39 786 memset(req->priv.initiator_port_id, 0, 8);
01cb9bcb 787 memcpy(req->priv.initiator_port_id + 8,
05321937 788 &target->srp_host->srp_dev->dev->node_guid, 8);
aef9ec39 789 }
aef9ec39 790
509c07bc 791 status = ib_send_cm_req(ch->cm_id, &req->param);
aef9ec39
RD		 792
793 kfree(req);
794
795 return status;
796}
797
ef6c49d8
BVA		 798static bool srp_queue_remove_work(struct srp_target_port *target)
799{
800 bool changed = false;
801
802 spin_lock_irq(&target->lock);
803 if (target->state != SRP_TARGET_REMOVED) {
804 target->state = SRP_TARGET_REMOVED;
805 changed = true;
806 }
807 spin_unlock_irq(&target->lock);
808
809 if (changed)
bcc05910 810 queue_work(srp_remove_wq, &target->remove_work);
ef6c49d8
BVA		 811
812 return changed;
813}
814
294c875a
BVA		 815static bool srp_change_conn_state(struct srp_target_port *target,
816 bool connected)
817{
818 bool changed = false;
819
820 spin_lock_irq(&target->lock);
821 if (target->connected != connected) {
822 target->connected = connected;
823 changed = true;
824 }
825 spin_unlock_irq(&target->lock);
826
827 return changed;
828}
829
aef9ec39
RD		 830static void srp_disconnect_target(struct srp_target_port *target)
831{
d92c0da7
BVA		 832 struct srp_rdma_ch *ch;
833 int i;
509c07bc 834
294c875a
BVA		 835 if (srp_change_conn_state(target, false)) {
836 /* XXX should send SRP_I_LOGOUT request */
aef9ec39 837
d92c0da7
BVA		 838 for (i = 0; i < target->ch_count; i++) {
839 ch = &target->ch[i];
840 if (ch->cm_id && ib_send_cm_dreq(ch->cm_id, NULL, 0)) {
841 shost_printk(KERN_DEBUG, target->scsi_host,
842 PFX "Sending CM DREQ failed\n");
843 }
294c875a 844 }
e6581056 845 }
aef9ec39
RD		 846}
847
509c07bc
BVA		 848static void srp_free_req_data(struct srp_target_port *target,
849 struct srp_rdma_ch *ch)
8f26c9ff 850{
5cfb1782
BVA		 851 struct srp_device *dev = target->srp_host->srp_dev;
852 struct ib_device *ibdev = dev->dev;
8f26c9ff
DD		 853 struct srp_request *req;
854 int i;
855
d92c0da7 856 if (!ch->target || !ch->req_ring)
4d73f95f
BVA		 857 return;
858
859 for (i = 0; i < target->req_ring_size; ++i) {
509c07bc 860 req = &ch->req_ring[i];
5cfb1782
BVA		 861 if (dev->use_fast_reg)
862 kfree(req->fr_list);
863 else
864 kfree(req->fmr_list);
8f26c9ff 865 kfree(req->map_page);
c07d424d
DD		 866 if (req->indirect_dma_addr) {
867 ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
868 target->indirect_size,
869 DMA_TO_DEVICE);
870 }
871 kfree(req->indirect_desc);
8f26c9ff 872 }
4d73f95f 873
509c07bc
BVA		 874 kfree(ch->req_ring);
875 ch->req_ring = NULL;
8f26c9ff
DD		 876}
877
509c07bc 878static int srp_alloc_req_data(struct srp_rdma_ch *ch)
b81d00bd 879{
509c07bc 880 struct srp_target_port *target = ch->target;
b81d00bd
BVA		 881 struct srp_device *srp_dev = target->srp_host->srp_dev;
882 struct ib_device *ibdev = srp_dev->dev;
883 struct srp_request *req;
5cfb1782 884 void *mr_list;
b81d00bd
BVA		 885 dma_addr_t dma_addr;
886 int i, ret = -ENOMEM;
887
509c07bc
BVA		 888 ch->req_ring = kcalloc(target->req_ring_size, sizeof(*ch->req_ring),
889 GFP_KERNEL);
890 if (!ch->req_ring)
4d73f95f
BVA		 891 goto out;
892
893 for (i = 0; i < target->req_ring_size; ++i) {
509c07bc 894 req = &ch->req_ring[i];
5cfb1782
BVA		 895 mr_list = kmalloc(target->cmd_sg_cnt * sizeof(void *),
896 GFP_KERNEL);
897 if (!mr_list)
898 goto out;
899 if (srp_dev->use_fast_reg)
900 req->fr_list = mr_list;
901 else
902 req->fmr_list = mr_list;
52ede08f 903 req->map_page = kmalloc(srp_dev->max_pages_per_mr *
d1b4289e 904 sizeof(void *), GFP_KERNEL);
5cfb1782
BVA		 905 if (!req->map_page)
906 goto out;
b81d00bd 907 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
5cfb1782 908 if (!req->indirect_desc)
b81d00bd
BVA		 909 goto out;
910
911 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
912 target->indirect_size,
913 DMA_TO_DEVICE);
914 if (ib_dma_mapping_error(ibdev, dma_addr))
915 goto out;
916
917 req->indirect_dma_addr = dma_addr;
b81d00bd
BVA		 918 }
919 ret = 0;
920
921out:
922 return ret;
923}
924
683b159a
BVA		 925/**
926 * srp_del_scsi_host_attr() - Remove attributes defined in the host template.
927 * @shost: SCSI host whose attributes to remove from sysfs.
928 *
929 * Note: Any attributes defined in the host template and that did not exist
930 * before invocation of this function will be ignored.
931 */
932static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
933{
934 struct device_attribute **attr;
935
936 for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr)
937 device_remove_file(&shost->shost_dev, *attr);
938}
939
ee12d6a8
BVA		 940static void srp_remove_target(struct srp_target_port *target)
941{
d92c0da7
BVA		 942 struct srp_rdma_ch *ch;
943 int i;
509c07bc 944
ef6c49d8
BVA		 945 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
946
ee12d6a8 947 srp_del_scsi_host_attr(target->scsi_host);
9dd69a60 948 srp_rport_get(target->rport);
ee12d6a8
BVA		 949 srp_remove_host(target->scsi_host);
950 scsi_remove_host(target->scsi_host);
93079162 951 srp_stop_rport_timers(target->rport);
ef6c49d8 952 srp_disconnect_target(target);
d92c0da7
BVA		 953 for (i = 0; i < target->ch_count; i++) {
954 ch = &target->ch[i];
955 srp_free_ch_ib(target, ch);
956 }
c1120f89 957 cancel_work_sync(&target->tl_err_work);
9dd69a60 958 srp_rport_put(target->rport);
d92c0da7
BVA		 959 for (i = 0; i < target->ch_count; i++) {
960 ch = &target->ch[i];
961 srp_free_req_data(target, ch);
962 }
963 kfree(target->ch);
964 target->ch = NULL;
65d7dd2f
VP		 965
966 spin_lock(&target->srp_host->target_lock);
967 list_del(&target->list);
968 spin_unlock(&target->srp_host->target_lock);
969
ee12d6a8
BVA		 970 scsi_host_put(target->scsi_host);
971}
972
c4028958 973static void srp_remove_work(struct work_struct *work)
aef9ec39 974{
c4028958 975 struct srp_target_port *target =
ef6c49d8 976 container_of(work, struct srp_target_port, remove_work);
aef9ec39 977
ef6c49d8 978 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
aef9ec39 979
96fc248a 980 srp_remove_target(target);
aef9ec39
RD		 981}
982
dc1bdbd9
BVA		 983static void srp_rport_delete(struct srp_rport *rport)
984{
985 struct srp_target_port *target = rport->lld_data;
986
987 srp_queue_remove_work(target);
988}
989
d92c0da7 990static int srp_connect_ch(struct srp_rdma_ch *ch, bool multich)
aef9ec39 991{
509c07bc 992 struct srp_target_port *target = ch->target;
aef9ec39
RD		 993 int ret;
994
d92c0da7 995 WARN_ON_ONCE(!multich && target->connected);
294c875a 996
948d1e88
BVA		 997 target->qp_in_error = false;
998
509c07bc 999 ret = srp_lookup_path(ch);
aef9ec39
RD		 1000 if (ret)
1001 return ret;
1002
1003 while (1) {
509c07bc 1004 init_completion(&ch->done);
d92c0da7 1005 ret = srp_send_req(ch, multich);
aef9ec39
RD		 1006 if (ret)
1007 return ret;
509c07bc 1008 ret = wait_for_completion_interruptible(&ch->done);
a702adce
BVA		 1009 if (ret < 0)
1010 return ret;
aef9ec39
RD		 1011
1012 /*
1013 * The CM event handling code will set status to
1014 * SRP_PORT_REDIRECT if we get a port redirect REJ
1015 * back, or SRP_DLID_REDIRECT if we get a lid/qp
1016 * redirect REJ back.
1017 */
509c07bc 1018 switch (ch->status) {
aef9ec39 1019 case 0:
294c875a 1020 srp_change_conn_state(target, true);
aef9ec39
RD		 1021 return 0;
1022
1023 case SRP_PORT_REDIRECT:
509c07bc 1024 ret = srp_lookup_path(ch);
aef9ec39
RD		 1025 if (ret)
1026 return ret;
1027 break;
1028
1029 case SRP_DLID_REDIRECT:
1030 break;
1031
9fe4bcf4 1032 case SRP_STALE_CONN:
9fe4bcf4 1033 shost_printk(KERN_ERR, target->scsi_host, PFX
205619f2 1034 "giving up on stale connection\n");
509c07bc
BVA		 1035 ch->status = -ECONNRESET;
1036 return ch->status;
9fe4bcf4 1037
aef9ec39 1038 default:
509c07bc 1039 return ch->status;
aef9ec39
RD		 1040 }
1041 }
1042}
1043
509c07bc 1044static int srp_inv_rkey(struct srp_rdma_ch *ch, u32 rkey)
5cfb1782
BVA		 1045{
1046 struct ib_send_wr *bad_wr;
1047 struct ib_send_wr wr = {
1048 .opcode = IB_WR_LOCAL_INV,
1049 .wr_id = LOCAL_INV_WR_ID_MASK,
1050 .next = NULL,
1051 .num_sge = 0,
1052 .send_flags = 0,
1053 .ex.invalidate_rkey = rkey,
1054 };
1055
509c07bc 1056 return ib_post_send(ch->qp, &wr, &bad_wr);
5cfb1782
BVA		 1057}
1058
d945e1df 1059static void srp_unmap_data(struct scsi_cmnd *scmnd,
509c07bc 1060 struct srp_rdma_ch *ch,
d945e1df
RD		 1061 struct srp_request *req)
1062{
509c07bc 1063 struct srp_target_port *target = ch->target;
5cfb1782
BVA		 1064 struct srp_device *dev = target->srp_host->srp_dev;
1065 struct ib_device *ibdev = dev->dev;
1066 int i, res;
8f26c9ff 1067
bb350d1d 1068 if (!scsi_sglist(scmnd) ||
d945e1df
RD		 1069 (scmnd->sc_data_direction != DMA_TO_DEVICE &&
1070 scmnd->sc_data_direction != DMA_FROM_DEVICE))
1071 return;
1072
5cfb1782
BVA		 1073 if (dev->use_fast_reg) {
1074 struct srp_fr_desc **pfr;
1075
1076 for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) {
509c07bc 1077 res = srp_inv_rkey(ch, (*pfr)->mr->rkey);
5cfb1782
BVA		 1078 if (res < 0) {
1079 shost_printk(KERN_ERR, target->scsi_host, PFX
1080 "Queueing INV WR for rkey %#x failed (%d)\n",
1081 (*pfr)->mr->rkey, res);
1082 queue_work(system_long_wq,
1083 &target->tl_err_work);
1084 }
1085 }
1086 if (req->nmdesc)
509c07bc 1087 srp_fr_pool_put(ch->fr_pool, req->fr_list,
5cfb1782
BVA		 1088 req->nmdesc);
1089 } else {
1090 struct ib_pool_fmr **pfmr;
1091
1092 for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++)
1093 ib_fmr_pool_unmap(*pfmr);
1094 }
f5358a17 1095
8f26c9ff
DD		 1096 ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
1097 scmnd->sc_data_direction);
d945e1df
RD		 1098}
1099
22032991
BVA		 1100/**
1101 * srp_claim_req - Take ownership of the scmnd associated with a request.
509c07bc 1102 * @ch: SRP RDMA channel.
22032991 1103 * @req: SRP request.
b3fe628d 1104 * @sdev: If not NULL, only take ownership for this SCSI device.
22032991
BVA		 1105 * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take
1106 * ownership of @req->scmnd if it equals @scmnd.
1107 *
1108 * Return value:
1109 * Either NULL or a pointer to the SCSI command the caller became owner of.
1110 */
509c07bc 1111static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch,
22032991 1112 struct srp_request *req,
b3fe628d 1113 struct scsi_device *sdev,
22032991
BVA		 1114 struct scsi_cmnd *scmnd)
1115{
1116 unsigned long flags;
1117
509c07bc 1118 spin_lock_irqsave(&ch->lock, flags);
b3fe628d
BVA		 1119 if (req->scmnd &&
1120 (!sdev || req->scmnd->device == sdev) &&
1121 (!scmnd || req->scmnd == scmnd)) {
22032991
BVA		 1122 scmnd = req->scmnd;
1123 req->scmnd = NULL;
22032991
BVA		 1124 } else {
1125 scmnd = NULL;
1126 }
509c07bc 1127 spin_unlock_irqrestore(&ch->lock, flags);
22032991
BVA		 1128
1129 return scmnd;
1130}
1131
1132/**
1133 * srp_free_req() - Unmap data and add request to the free request list.
509c07bc 1134 * @ch: SRP RDMA channel.
af24663b
BVA		 1135 * @req: Request to be freed.
1136 * @scmnd: SCSI command associated with @req.
1137 * @req_lim_delta: Amount to be added to @target->req_lim.
22032991 1138 */
509c07bc
BVA		 1139static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req,
1140 struct scsi_cmnd *scmnd, s32 req_lim_delta)
526b4caa 1141{
94a9174c
BVA		 1142 unsigned long flags;
1143
509c07bc 1144 srp_unmap_data(scmnd, ch, req);
22032991 1145
509c07bc
BVA		 1146 spin_lock_irqsave(&ch->lock, flags);
1147 ch->req_lim += req_lim_delta;
509c07bc 1148 spin_unlock_irqrestore(&ch->lock, flags);
526b4caa
IR		 1149}
1150
509c07bc
BVA		 1151static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req,
1152 struct scsi_device *sdev, int result)
526b4caa 1153{
509c07bc 1154 struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL);
22032991
BVA		 1155
1156 if (scmnd) {
509c07bc 1157 srp_free_req(ch, req, scmnd, 0);
ed9b2264 1158 scmnd->result = result;
22032991 1159 scmnd->scsi_done(scmnd);
22032991 1160 }
526b4caa
IR		 1161}
1162
ed9b2264 1163static void srp_terminate_io(struct srp_rport *rport)
aef9ec39 1164{
ed9b2264 1165 struct srp_target_port *target = rport->lld_data;
d92c0da7 1166 struct srp_rdma_ch *ch;
b3fe628d
BVA		 1167 struct Scsi_Host *shost = target->scsi_host;
1168 struct scsi_device *sdev;
d92c0da7 1169 int i, j;
ed9b2264 1170
b3fe628d
BVA		 1171 /*
1172 * Invoking srp_terminate_io() while srp_queuecommand() is running
1173 * is not safe. Hence the warning statement below.
1174 */
1175 shost_for_each_device(sdev, shost)
1176 WARN_ON_ONCE(sdev->request_queue->request_fn_active);
1177
d92c0da7
BVA		 1178 for (i = 0; i < target->ch_count; i++) {
1179 ch = &target->ch[i];
509c07bc 1180
d92c0da7
BVA		 1181 for (j = 0; j < target->req_ring_size; ++j) {
1182 struct srp_request *req = &ch->req_ring[j];
1183
1184 srp_finish_req(ch, req, NULL,
1185 DID_TRANSPORT_FAILFAST << 16);
1186 }
ed9b2264
BVA		 1187 }
1188}
aef9ec39 1189
ed9b2264
BVA		 1190/*
1191 * It is up to the caller to ensure that srp_rport_reconnect() calls are
1192 * serialized and that no concurrent srp_queuecommand(), srp_abort(),
1193 * srp_reset_device() or srp_reset_host() calls will occur while this function
1194 * is in progress. One way to realize that is not to call this function
1195 * directly but to call srp_reconnect_rport() instead since that last function
1196 * serializes calls of this function via rport->mutex and also blocks
1197 * srp_queuecommand() calls before invoking this function.
1198 */
1199static int srp_rport_reconnect(struct srp_rport *rport)
1200{
1201 struct srp_target_port *target = rport->lld_data;
d92c0da7
BVA		 1202 struct srp_rdma_ch *ch;
1203 int i, j, ret = 0;
1204 bool multich = false;
09be70a2 1205
aef9ec39 1206 srp_disconnect_target(target);
34aa654e
BVA		 1207
1208 if (target->state == SRP_TARGET_SCANNING)
1209 return -ENODEV;
1210
aef9ec39 1211 /*
c7c4e7ff
BVA		 1212 * Now get a new local CM ID so that we avoid confusing the target in
1213 * case things are really fouled up. Doing so also ensures that all CM
1214 * callbacks will have finished before a new QP is allocated.
aef9ec39 1215 */
d92c0da7
BVA		 1216 for (i = 0; i < target->ch_count; i++) {
1217 ch = &target->ch[i];
1218 if (!ch->target)
1219 break;
1220 ret += srp_new_cm_id(ch);
536ae14e 1221 }
d92c0da7
BVA		 1222 for (i = 0; i < target->ch_count; i++) {
1223 ch = &target->ch[i];
1224 if (!ch->target)
1225 break;
1226 for (j = 0; j < target->req_ring_size; ++j) {
1227 struct srp_request *req = &ch->req_ring[j];
aef9ec39 1228
d92c0da7
BVA		 1229 srp_finish_req(ch, req, NULL, DID_RESET << 16);
1230 }
1231 }
1232 for (i = 0; i < target->ch_count; i++) {
1233 ch = &target->ch[i];
1234 if (!ch->target)
1235 break;
1236 /*
1237 * Whether or not creating a new CM ID succeeded, create a new
1238 * QP. This guarantees that all completion callback function
1239 * invocations have finished before request resetting starts.
1240 */
1241 ret += srp_create_ch_ib(ch);
aef9ec39 1242
d92c0da7
BVA		 1243 INIT_LIST_HEAD(&ch->free_tx);
1244 for (j = 0; j < target->queue_size; ++j)
1245 list_add(&ch->tx_ring[j]->list, &ch->free_tx);
1246 }
1247 for (i = 0; i < target->ch_count; i++) {
1248 ch = &target->ch[i];
1249 if (ret || !ch->target) {
1250 if (i > 1)
1251 ret = 0;
1252 break;
1253 }
1254 ret = srp_connect_ch(ch, multich);
1255 multich = true;
1256 }
09be70a2 1257
ed9b2264
BVA		 1258 if (ret == 0)
1259 shost_printk(KERN_INFO, target->scsi_host,
1260 PFX "reconnect succeeded\n");
aef9ec39
RD		 1261
1262 return ret;
1263}
1264
8f26c9ff
DD		 1265static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
1266 unsigned int dma_len, u32 rkey)
f5358a17 1267{
8f26c9ff 1268 struct srp_direct_buf *desc = state->desc;
f5358a17 1269
8f26c9ff
DD		 1270 desc->va = cpu_to_be64(dma_addr);
1271 desc->key = cpu_to_be32(rkey);
1272 desc->len = cpu_to_be32(dma_len);
f5358a17 1273
8f26c9ff
DD		 1274 state->total_len += dma_len;
1275 state->desc++;
1276 state->ndesc++;
1277}
559ce8f1 1278
8f26c9ff 1279static int srp_map_finish_fmr(struct srp_map_state *state,
509c07bc 1280 struct srp_rdma_ch *ch)
8f26c9ff 1281{
8f26c9ff
DD		 1282 struct ib_pool_fmr *fmr;
1283 u64 io_addr = 0;
85507bcc 1284
509c07bc 1285 fmr = ib_fmr_pool_map_phys(ch->fmr_pool, state->pages,
8f26c9ff
DD		 1286 state->npages, io_addr);
1287 if (IS_ERR(fmr))
1288 return PTR_ERR(fmr);
f5358a17 1289
8f26c9ff 1290 *state->next_fmr++ = fmr;
52ede08f 1291 state->nmdesc++;
f5358a17 1292
52ede08f 1293 srp_map_desc(state, 0, state->dma_len, fmr->fmr->rkey);
539dde6f 1294
8f26c9ff
DD		 1295 return 0;
1296}
1297
5cfb1782 1298static int srp_map_finish_fr(struct srp_map_state *state,
509c07bc 1299 struct srp_rdma_ch *ch)
5cfb1782 1300{
509c07bc 1301 struct srp_target_port *target = ch->target;
5cfb1782
BVA		 1302 struct srp_device *dev = target->srp_host->srp_dev;
1303 struct ib_send_wr *bad_wr;
1304 struct ib_send_wr wr;
1305 struct srp_fr_desc *desc;
1306 u32 rkey;
1307
509c07bc 1308 desc = srp_fr_pool_get(ch->fr_pool);
5cfb1782
BVA		 1309 if (!desc)
1310 return -ENOMEM;
1311
1312 rkey = ib_inc_rkey(desc->mr->rkey);
1313 ib_update_fast_reg_key(desc->mr, rkey);
1314
1315 memcpy(desc->frpl->page_list, state->pages,
1316 sizeof(state->pages[0]) * state->npages);
1317
1318 memset(&wr, 0, sizeof(wr));
1319 wr.opcode = IB_WR_FAST_REG_MR;
1320 wr.wr_id = FAST_REG_WR_ID_MASK;
1321 wr.wr.fast_reg.iova_start = state->base_dma_addr;
1322 wr.wr.fast_reg.page_list = desc->frpl;
1323 wr.wr.fast_reg.page_list_len = state->npages;
1324 wr.wr.fast_reg.page_shift = ilog2(dev->mr_page_size);
1325 wr.wr.fast_reg.length = state->dma_len;
1326 wr.wr.fast_reg.access_flags = (IB_ACCESS_LOCAL_WRITE |
1327 IB_ACCESS_REMOTE_READ |
1328 IB_ACCESS_REMOTE_WRITE);
1329 wr.wr.fast_reg.rkey = desc->mr->lkey;
1330
1331 *state->next_fr++ = desc;
1332 state->nmdesc++;
1333
1334 srp_map_desc(state, state->base_dma_addr, state->dma_len,
1335 desc->mr->rkey);
1336
509c07bc 1337 return ib_post_send(ch->qp, &wr, &bad_wr);
5cfb1782
BVA		 1338}
1339
539dde6f 1340static int srp_finish_mapping(struct srp_map_state *state,
509c07bc 1341 struct srp_rdma_ch *ch)
539dde6f 1342{
509c07bc 1343 struct srp_target_port *target = ch->target;
539dde6f
BVA		 1344 int ret = 0;
1345
1346 if (state->npages == 0)
1347 return 0;
1348
b1b8854d 1349 if (state->npages == 1 && !register_always)
52ede08f 1350 srp_map_desc(state, state->base_dma_addr, state->dma_len,
539dde6f
BVA		 1351 target->rkey);
1352 else
5cfb1782 1353 ret = target->srp_host->srp_dev->use_fast_reg ?
509c07bc
BVA		 1354 srp_map_finish_fr(state, ch) :
1355 srp_map_finish_fmr(state, ch);
539dde6f
BVA		 1356
1357 if (ret == 0) {
1358 state->npages = 0;
52ede08f 1359 state->dma_len = 0;
539dde6f
BVA		 1360 }
1361
1362 return ret;
1363}
1364
8f26c9ff
DD		 1365static void srp_map_update_start(struct srp_map_state *state,
1366 struct scatterlist *sg, int sg_index,
1367 dma_addr_t dma_addr)
1368{
1369 state->unmapped_sg = sg;
1370 state->unmapped_index = sg_index;
1371 state->unmapped_addr = dma_addr;
1372}
85507bcc 1373
8f26c9ff 1374static int srp_map_sg_entry(struct srp_map_state *state,
509c07bc 1375 struct srp_rdma_ch *ch,
8f26c9ff 1376 struct scatterlist *sg, int sg_index,
5cfb1782 1377 bool use_mr)
8f26c9ff 1378{
509c07bc 1379 struct srp_target_port *target = ch->target;
8f26c9ff
DD		 1380 struct srp_device *dev = target->srp_host->srp_dev;
1381 struct ib_device *ibdev = dev->dev;
1382 dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg);
1383 unsigned int dma_len = ib_sg_dma_len(ibdev, sg);
1384 unsigned int len;
1385 int ret;
1386
1387 if (!dma_len)
1388 return 0;
1389
5cfb1782
BVA		 1390 if (!use_mr) {
1391 /*
1392 * Once we're in direct map mode for a request, we don't
1393 * go back to FMR or FR mode, so no need to update anything
8f26c9ff
DD		 1394 * other than the descriptor.
1395 */
1396 srp_map_desc(state, dma_addr, dma_len, target->rkey);
1397 return 0;
85507bcc 1398 }
f5358a17 1399
5cfb1782
BVA		 1400 /*
1401 * Since not all RDMA HW drivers support non-zero page offsets for
1402 * FMR, if we start at an offset into a page, don't merge into the
1403 * current FMR mapping. Finish it out, and use the kernel's MR for
1404 * this sg entry.
8f26c9ff 1405 */
5cfb1782
BVA		 1406 if ((!dev->use_fast_reg && dma_addr & ~dev->mr_page_mask) ||
1407 dma_len > dev->mr_max_size) {
509c07bc 1408 ret = srp_finish_mapping(state, ch);
8f26c9ff
DD		 1409 if (ret)
1410 return ret;
1411
1412 srp_map_desc(state, dma_addr, dma_len, target->rkey);
1413 srp_map_update_start(state, NULL, 0, 0);
1414 return 0;
f5358a17
RD		 1415 }
1416
5cfb1782
BVA		 1417 /*
1418 * If this is the first sg that will be mapped via FMR or via FR, save
1419 * our position. We need to know the first unmapped entry, its index,
1420 * and the first unmapped address within that entry to be able to
1421 * restart mapping after an error.
8f26c9ff
DD		 1422 */
1423 if (!state->unmapped_sg)
1424 srp_map_update_start(state, sg, sg_index, dma_addr);
f5358a17 1425
8f26c9ff 1426 while (dma_len) {
5cfb1782
BVA		 1427 unsigned offset = dma_addr & ~dev->mr_page_mask;
1428 if (state->npages == dev->max_pages_per_mr || offset != 0) {
509c07bc 1429 ret = srp_finish_mapping(state, ch);
8f26c9ff
DD		 1430 if (ret)
1431 return ret;
f5358a17 1432
8f26c9ff
DD		 1433 srp_map_update_start(state, sg, sg_index, dma_addr);
1434 }
1435
5cfb1782 1436 len = min_t(unsigned int, dma_len, dev->mr_page_size - offset);
f5358a17 1437
8f26c9ff
DD		 1438 if (!state->npages)
1439 state->base_dma_addr = dma_addr;
5cfb1782 1440 state->pages[state->npages++] = dma_addr & dev->mr_page_mask;
52ede08f 1441 state->dma_len += len;
8f26c9ff
DD		 1442 dma_addr += len;
1443 dma_len -= len;
1444 }
1445
5cfb1782
BVA		 1446 /*
1447 * If the last entry of the MR wasn't a full page, then we need to
8f26c9ff
DD		 1448 * close it out and start a new one -- we can only merge at page
1449 * boundries.
1450 */
1451 ret = 0;
52ede08f 1452 if (len != dev->mr_page_size) {
509c07bc 1453 ret = srp_finish_mapping(state, ch);
8f26c9ff
DD		 1454 if (!ret)
1455 srp_map_update_start(state, NULL, 0, 0);
1456 }
f5358a17
RD		 1457 return ret;
1458}
1459
509c07bc
BVA		 1460static int srp_map_sg(struct srp_map_state *state, struct srp_rdma_ch *ch,
1461 struct srp_request *req, struct scatterlist *scat,
1462 int count)
76bc1e1d 1463{
509c07bc 1464 struct srp_target_port *target = ch->target;
76bc1e1d
BVA		 1465 struct srp_device *dev = target->srp_host->srp_dev;
1466 struct ib_device *ibdev = dev->dev;
1467 struct scatterlist *sg;
5cfb1782
BVA		 1468 int i;
1469 bool use_mr;
76bc1e1d
BVA		 1470
1471 state->desc = req->indirect_desc;
1472 state->pages = req->map_page;
5cfb1782
BVA		 1473 if (dev->use_fast_reg) {
1474 state->next_fr = req->fr_list;
509c07bc 1475 use_mr = !!ch->fr_pool;
5cfb1782
BVA		 1476 } else {
1477 state->next_fmr = req->fmr_list;
509c07bc 1478 use_mr = !!ch->fmr_pool;
5cfb1782 1479 }
76bc1e1d
BVA		 1480
1481 for_each_sg(scat, sg, count, i) {
509c07bc 1482 if (srp_map_sg_entry(state, ch, sg, i, use_mr)) {
5cfb1782
BVA		 1483 /*
1484 * Memory registration failed, so backtrack to the
1485 * first unmapped entry and continue on without using
1486 * memory registration.
76bc1e1d
BVA		 1487 */
1488 dma_addr_t dma_addr;
1489 unsigned int dma_len;
1490
1491backtrack:
1492 sg = state->unmapped_sg;
1493 i = state->unmapped_index;
1494
1495 dma_addr = ib_sg_dma_address(ibdev, sg);
1496 dma_len = ib_sg_dma_len(ibdev, sg);
1497 dma_len -= (state->unmapped_addr - dma_addr);
1498 dma_addr = state->unmapped_addr;
5cfb1782 1499 use_mr = false;
76bc1e1d
BVA		 1500 srp_map_desc(state, dma_addr, dma_len, target->rkey);
1501 }
1502 }
1503
509c07bc 1504 if (use_mr && srp_finish_mapping(state, ch))
76bc1e1d
BVA		 1505 goto backtrack;
1506
52ede08f 1507 req->nmdesc = state->nmdesc;
5cfb1782
BVA		 1508
1509 return 0;
76bc1e1d
BVA		 1510}
1511
509c07bc 1512static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
aef9ec39
RD		 1513 struct srp_request *req)
1514{
509c07bc 1515 struct srp_target_port *target = ch->target;
76bc1e1d 1516 struct scatterlist *scat;
aef9ec39 1517 struct srp_cmd *cmd = req->cmd->buf;
76bc1e1d 1518 int len, nents, count;
85507bcc
RC		 1519 struct srp_device *dev;
1520 struct ib_device *ibdev;
8f26c9ff
DD		 1521 struct srp_map_state state;
1522 struct srp_indirect_buf *indirect_hdr;
8f26c9ff
DD		 1523 u32 table_len;
1524 u8 fmt;
aef9ec39 1525
bb350d1d 1526 if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE)
aef9ec39
RD		 1527 return sizeof (struct srp_cmd);
1528
1529 if (scmnd->sc_data_direction != DMA_FROM_DEVICE &&
1530 scmnd->sc_data_direction != DMA_TO_DEVICE) {
7aa54bd7
DD		 1531 shost_printk(KERN_WARNING, target->scsi_host,
1532 PFX "Unhandled data direction %d\n",
1533 scmnd->sc_data_direction);
aef9ec39
RD		 1534 return -EINVAL;
1535 }
1536
bb350d1d
FT		 1537 nents = scsi_sg_count(scmnd);
1538 scat = scsi_sglist(scmnd);
aef9ec39 1539
05321937 1540 dev = target->srp_host->srp_dev;
85507bcc
RC		 1541 ibdev = dev->dev;
1542
1543 count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction);
8f26c9ff
DD		 1544 if (unlikely(count == 0))
1545 return -EIO;
f5358a17
RD		 1546
1547 fmt = SRP_DATA_DESC_DIRECT;
1548 len = sizeof (struct srp_cmd) + sizeof (struct srp_direct_buf);
aef9ec39 1549
b1b8854d 1550 if (count == 1 && !register_always) {
f5358a17
RD		 1551 /*
1552 * The midlayer only generated a single gather/scatter
1553 * entry, or DMA mapping coalesced everything to a
1554 * single entry. So a direct descriptor along with
1555 * the DMA MR suffices.
1556 */
cf368713 1557 struct srp_direct_buf *buf = (void *) cmd->add_data;
aef9ec39 1558
85507bcc 1559 buf->va = cpu_to_be64(ib_sg_dma_address(ibdev, scat));
9af76271 1560 buf->key = cpu_to_be32(target->rkey);
85507bcc 1561 buf->len = cpu_to_be32(ib_sg_dma_len(ibdev, scat));
8f26c9ff 1562
52ede08f 1563 req->nmdesc = 0;
8f26c9ff
DD		 1564 goto map_complete;
1565 }
1566
5cfb1782
BVA		 1567 /*
1568 * We have more than one scatter/gather entry, so build our indirect
1569 * descriptor table, trying to merge as many entries as we can.
8f26c9ff
DD		 1570 */
1571 indirect_hdr = (void *) cmd->add_data;
1572
c07d424d
DD		 1573 ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr,
1574 target->indirect_size, DMA_TO_DEVICE);
1575
8f26c9ff 1576 memset(&state, 0, sizeof(state));
509c07bc 1577 srp_map_sg(&state, ch, req, scat, count);
cf368713 1578
c07d424d
DD		 1579 /* We've mapped the request, now pull as much of the indirect
1580 * descriptor table as we can into the command buffer. If this
1581 * target is not using an external indirect table, we are
1582 * guaranteed to fit into the command, as the SCSI layer won't
1583 * give us more S/G entries than we allow.
8f26c9ff 1584 */
8f26c9ff 1585 if (state.ndesc == 1) {
5cfb1782
BVA		 1586 /*
1587 * Memory registration collapsed the sg-list into one entry,
8f26c9ff
DD		 1588 * so use a direct descriptor.
1589 */
1590 struct srp_direct_buf *buf = (void *) cmd->add_data;
cf368713 1591
c07d424d 1592 *buf = req->indirect_desc[0];
8f26c9ff 1593 goto map_complete;
aef9ec39
RD		 1594 }
1595
c07d424d
DD		 1596 if (unlikely(target->cmd_sg_cnt < state.ndesc &&
1597 !target->allow_ext_sg)) {
1598 shost_printk(KERN_ERR, target->scsi_host,
1599 "Could not fit S/G list into SRP_CMD\n");
1600 return -EIO;
1601 }
1602
1603 count = min(state.ndesc, target->cmd_sg_cnt);
8f26c9ff
DD		 1604 table_len = state.ndesc * sizeof (struct srp_direct_buf);
1605
1606 fmt = SRP_DATA_DESC_INDIRECT;
1607 len = sizeof(struct srp_cmd) + sizeof (struct srp_indirect_buf);
c07d424d 1608 len += count * sizeof (struct srp_direct_buf);
8f26c9ff 1609
c07d424d
DD		 1610 memcpy(indirect_hdr->desc_list, req->indirect_desc,
1611 count * sizeof (struct srp_direct_buf));
8f26c9ff 1612
c07d424d 1613 indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr);
8f26c9ff
DD		 1614 indirect_hdr->table_desc.key = cpu_to_be32(target->rkey);
1615 indirect_hdr->table_desc.len = cpu_to_be32(table_len);
1616 indirect_hdr->len = cpu_to_be32(state.total_len);
1617
1618 if (scmnd->sc_data_direction == DMA_TO_DEVICE)
c07d424d 1619 cmd->data_out_desc_cnt = count;
8f26c9ff 1620 else
c07d424d
DD		 1621 cmd->data_in_desc_cnt = count;
1622
1623 ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len,
1624 DMA_TO_DEVICE);
8f26c9ff
DD		 1625
1626map_complete:
aef9ec39
RD		 1627 if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1628 cmd->buf_fmt = fmt << 4;
1629 else
1630 cmd->buf_fmt = fmt;
1631
aef9ec39
RD		 1632 return len;
1633}
1634
76c75b25
BVA		 1635/*
1636 * Return an IU and possible credit to the free pool
1637 */
509c07bc 1638static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu,
76c75b25
BVA		 1639 enum srp_iu_type iu_type)
1640{
1641 unsigned long flags;
1642
509c07bc
BVA		 1643 spin_lock_irqsave(&ch->lock, flags);
1644 list_add(&iu->list, &ch->free_tx);
76c75b25 1645 if (iu_type != SRP_IU_RSP)
509c07bc
BVA		 1646 ++ch->req_lim;
1647 spin_unlock_irqrestore(&ch->lock, flags);
76c75b25
BVA		 1648}
1649
05a1d750 1650/*
509c07bc 1651 * Must be called with ch->lock held to protect req_lim and free_tx.
e9684678 1652 * If IU is not sent, it must be returned using srp_put_tx_iu().
05a1d750
DD		 1653 *
1654 * Note:
1655 * An upper limit for the number of allocated information units for each
1656 * request type is:
1657 * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues
1658 * more than Scsi_Host.can_queue requests.
1659 * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE.
1660 * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than
1661 * one unanswered SRP request to an initiator.
1662 */
509c07bc 1663static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch,
05a1d750
DD		 1664 enum srp_iu_type iu_type)
1665{
509c07bc 1666 struct srp_target_port *target = ch->target;
05a1d750
DD		 1667 s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
1668 struct srp_iu *iu;
1669
509c07bc 1670 srp_send_completion(ch->send_cq, ch);
05a1d750 1671
509c07bc 1672 if (list_empty(&ch->free_tx))
05a1d750
DD		 1673 return NULL;
1674
1675 /* Initiator responses to target requests do not consume credits */
76c75b25 1676 if (iu_type != SRP_IU_RSP) {
509c07bc 1677 if (ch->req_lim <= rsv) {
76c75b25
BVA		 1678 ++target->zero_req_lim;
1679 return NULL;
1680 }
1681
509c07bc 1682 --ch->req_lim;
05a1d750
DD		 1683 }
1684
509c07bc 1685 iu = list_first_entry(&ch->free_tx, struct srp_iu, list);
76c75b25 1686 list_del(&iu->list);
05a1d750
DD		 1687 return iu;
1688}
1689
509c07bc 1690static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len)
05a1d750 1691{
509c07bc 1692 struct srp_target_port *target = ch->target;
05a1d750
DD		 1693 struct ib_sge list;
1694 struct ib_send_wr wr, *bad_wr;
05a1d750
DD		 1695
1696 list.addr = iu->dma;
1697 list.length = len;
9af76271 1698 list.lkey = target->lkey;
05a1d750
DD		 1699
1700 wr.next = NULL;
dcb4cb85 1701 wr.wr_id = (uintptr_t) iu;
05a1d750
DD		 1702 wr.sg_list = &list;
1703 wr.num_sge = 1;
1704 wr.opcode = IB_WR_SEND;
1705 wr.send_flags = IB_SEND_SIGNALED;
1706
509c07bc 1707 return ib_post_send(ch->qp, &wr, &bad_wr);
05a1d750
DD		 1708}
1709
509c07bc 1710static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu)
c996bb47 1711{
509c07bc 1712 struct srp_target_port *target = ch->target;
c996bb47 1713 struct ib_recv_wr wr, *bad_wr;
dcb4cb85 1714 struct ib_sge list;
c996bb47
BVA		 1715
1716 list.addr = iu->dma;
1717 list.length = iu->size;
9af76271 1718 list.lkey = target->lkey;
c996bb47
BVA		 1719
1720 wr.next = NULL;
dcb4cb85 1721 wr.wr_id = (uintptr_t) iu;
c996bb47
BVA		 1722 wr.sg_list = &list;
1723 wr.num_sge = 1;
1724
509c07bc 1725 return ib_post_recv(ch->qp, &wr, &bad_wr);
c996bb47
BVA		 1726}
1727
509c07bc 1728static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp)
aef9ec39 1729{
509c07bc 1730 struct srp_target_port *target = ch->target;
aef9ec39
RD		 1731 struct srp_request *req;
1732 struct scsi_cmnd *scmnd;
1733 unsigned long flags;
aef9ec39 1734
aef9ec39 1735 if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) {
509c07bc
BVA		 1736 spin_lock_irqsave(&ch->lock, flags);
1737 ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1738 spin_unlock_irqrestore(&ch->lock, flags);
94a9174c 1739
509c07bc 1740 ch->tsk_mgmt_status = -1;
f8b6e31e 1741 if (be32_to_cpu(rsp->resp_data_len) >= 4)
509c07bc
BVA		 1742 ch->tsk_mgmt_status = rsp->data[3];
1743 complete(&ch->tsk_mgmt_done);
aef9ec39 1744 } else {
77f2c1a4
BVA		 1745 scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag);
1746 if (scmnd) {
1747 req = (void *)scmnd->host_scribble;
1748 scmnd = srp_claim_req(ch, req, NULL, scmnd);
1749 }
22032991 1750 if (!scmnd) {
7aa54bd7 1751 shost_printk(KERN_ERR, target->scsi_host,
d92c0da7
BVA		 1752 "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n",
1753 rsp->tag, ch - target->ch, ch->qp->qp_num);
22032991 1754
509c07bc
BVA		 1755 spin_lock_irqsave(&ch->lock, flags);
1756 ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1757 spin_unlock_irqrestore(&ch->lock, flags);
22032991
BVA		 1758
1759 return;
1760 }
aef9ec39
RD		 1761 scmnd->result = rsp->status;
1762
1763 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) {
1764 memcpy(scmnd->sense_buffer, rsp->data +
1765 be32_to_cpu(rsp->resp_data_len),
1766 min_t(int, be32_to_cpu(rsp->sense_data_len),
1767 SCSI_SENSE_BUFFERSIZE));
1768 }
1769
e714531a 1770 if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER))
bb350d1d 1771 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt));
e714531a
BVA		 1772 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER))
1773 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt));
1774 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER))
1775 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt));
1776 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER))
1777 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt));
aef9ec39 1778
509c07bc 1779 srp_free_req(ch, req, scmnd,
22032991
BVA		 1780 be32_to_cpu(rsp->req_lim_delta));
1781
f8b6e31e
DD		 1782 scmnd->host_scribble = NULL;
1783 scmnd->scsi_done(scmnd);
aef9ec39 1784 }
aef9ec39
RD		 1785}
1786
509c07bc 1787static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta,
bb12588a
DD		 1788 void *rsp, int len)
1789{
509c07bc 1790 struct srp_target_port *target = ch->target;
76c75b25 1791 struct ib_device *dev = target->srp_host->srp_dev->dev;
bb12588a
DD		 1792 unsigned long flags;
1793 struct srp_iu *iu;
76c75b25 1794 int err;
bb12588a 1795
509c07bc
BVA		 1796 spin_lock_irqsave(&ch->lock, flags);
1797 ch->req_lim += req_delta;
1798 iu = __srp_get_tx_iu(ch, SRP_IU_RSP);
1799 spin_unlock_irqrestore(&ch->lock, flags);
76c75b25 1800
bb12588a
DD		 1801 if (!iu) {
1802 shost_printk(KERN_ERR, target->scsi_host, PFX
1803 "no IU available to send response\n");
76c75b25 1804 return 1;
bb12588a
DD		 1805 }
1806
1807 ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE);
1808 memcpy(iu->buf, rsp, len);
1809 ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE);
1810
509c07bc 1811 err = srp_post_send(ch, iu, len);
76c75b25 1812 if (err) {
bb12588a
DD		 1813 shost_printk(KERN_ERR, target->scsi_host, PFX
1814 "unable to post response: %d\n", err);
509c07bc 1815 srp_put_tx_iu(ch, iu, SRP_IU_RSP);
76c75b25 1816 }
bb12588a 1817
bb12588a
DD		 1818 return err;
1819}
1820
509c07bc 1821static void srp_process_cred_req(struct srp_rdma_ch *ch,
bb12588a
DD		 1822 struct srp_cred_req *req)
1823{
1824 struct srp_cred_rsp rsp = {
1825 .opcode = SRP_CRED_RSP,
1826 .tag = req->tag,
1827 };
1828 s32 delta = be32_to_cpu(req->req_lim_delta);
1829
509c07bc
BVA		 1830 if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
1831 shost_printk(KERN_ERR, ch->target->scsi_host, PFX
bb12588a
DD		 1832 "problems processing SRP_CRED_REQ\n");
1833}
1834
509c07bc 1835static void srp_process_aer_req(struct srp_rdma_ch *ch,
bb12588a
DD		 1836 struct srp_aer_req *req)
1837{
509c07bc 1838 struct srp_target_port *target = ch->target;
bb12588a
DD		 1839 struct srp_aer_rsp rsp = {
1840 .opcode = SRP_AER_RSP,
1841 .tag = req->tag,
1842 };
1843 s32 delta = be32_to_cpu(req->req_lim_delta);
1844
1845 shost_printk(KERN_ERR, target->scsi_host, PFX
1846 "ignoring AER for LUN %llu\n", be64_to_cpu(req->lun));
1847
509c07bc 1848 if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
bb12588a
DD		 1849 shost_printk(KERN_ERR, target->scsi_host, PFX
1850 "problems processing SRP_AER_REQ\n");
1851}
1852
509c07bc 1853static void srp_handle_recv(struct srp_rdma_ch *ch, struct ib_wc *wc)
aef9ec39 1854{
509c07bc 1855 struct srp_target_port *target = ch->target;
dcb4cb85 1856 struct ib_device *dev = target->srp_host->srp_dev->dev;
737b94eb 1857 struct srp_iu *iu = (struct srp_iu *) (uintptr_t) wc->wr_id;
c996bb47 1858 int res;
aef9ec39
RD		 1859 u8 opcode;
1860
509c07bc 1861 ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len,
85507bcc 1862 DMA_FROM_DEVICE);
aef9ec39
RD		 1863
1864 opcode = *(u8 *) iu->buf;
1865
1866 if (0) {
7aa54bd7
DD		 1867 shost_printk(KERN_ERR, target->scsi_host,
1868 PFX "recv completion, opcode 0x%02x\n", opcode);
7a700811
BVA		 1869 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1,
1870 iu->buf, wc->byte_len, true);
aef9ec39
RD		 1871 }
1872
1873 switch (opcode) {
1874 case SRP_RSP:
509c07bc 1875 srp_process_rsp(ch, iu->buf);
aef9ec39
RD		 1876 break;
1877
bb12588a 1878 case SRP_CRED_REQ:
509c07bc 1879 srp_process_cred_req(ch, iu->buf);
bb12588a
DD		 1880 break;
1881
1882 case SRP_AER_REQ:
509c07bc 1883 srp_process_aer_req(ch, iu->buf);
bb12588a
DD		 1884 break;
1885
aef9ec39
RD		 1886 case SRP_T_LOGOUT:
1887 /* XXX Handle target logout */
7aa54bd7
DD		 1888 shost_printk(KERN_WARNING, target->scsi_host,
1889 PFX "Got target logout request\n");
aef9ec39
RD		 1890 break;
1891
1892 default:
7aa54bd7
DD		 1893 shost_printk(KERN_WARNING, target->scsi_host,
1894 PFX "Unhandled SRP opcode 0x%02x\n", opcode);
aef9ec39
RD		 1895 break;
1896 }
1897
509c07bc 1898 ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len,
85507bcc 1899 DMA_FROM_DEVICE);
c996bb47 1900
509c07bc 1901 res = srp_post_recv(ch, iu);
c996bb47
BVA		 1902 if (res != 0)
1903 shost_printk(KERN_ERR, target->scsi_host,
1904 PFX "Recv failed with error code %d\n", res);
aef9ec39
RD		 1905}
1906
c1120f89
BVA		 1907/**
1908 * srp_tl_err_work() - handle a transport layer error
af24663b 1909 * @work: Work structure embedded in an SRP target port.
c1120f89
BVA		 1910 *
1911 * Note: This function may get invoked before the rport has been created,
1912 * hence the target->rport test.
1913 */
1914static void srp_tl_err_work(struct work_struct *work)
1915{
1916 struct srp_target_port *target;
1917
1918 target = container_of(work, struct srp_target_port, tl_err_work);
1919 if (target->rport)
1920 srp_start_tl_fail_timers(target->rport);
1921}
1922
5cfb1782 1923static void srp_handle_qp_err(u64 wr_id, enum ib_wc_status wc_status,
7dad6b2e 1924 bool send_err, struct srp_rdma_ch *ch)
948d1e88 1925{
7dad6b2e
BVA		 1926 struct srp_target_port *target = ch->target;
1927
1928 if (wr_id == SRP_LAST_WR_ID) {
1929 complete(&ch->done);
1930 return;
1931 }
1932
294c875a 1933 if (target->connected && !target->qp_in_error) {
5cfb1782
BVA		 1934 if (wr_id & LOCAL_INV_WR_ID_MASK) {
1935 shost_printk(KERN_ERR, target->scsi_host, PFX
57363d98
SG		 1936 "LOCAL_INV failed with status %s (%d)\n",
1937 ib_wc_status_msg(wc_status), wc_status);
5cfb1782
BVA		 1938 } else if (wr_id & FAST_REG_WR_ID_MASK) {
1939 shost_printk(KERN_ERR, target->scsi_host, PFX
57363d98
SG		 1940 "FAST_REG_MR failed status %s (%d)\n",
1941 ib_wc_status_msg(wc_status), wc_status);
5cfb1782
BVA		 1942 } else {
1943 shost_printk(KERN_ERR, target->scsi_host,
57363d98 1944 PFX "failed %s status %s (%d) for iu %p\n",
5cfb1782 1945 send_err ? "send" : "receive",
57363d98
SG		 1946 ib_wc_status_msg(wc_status), wc_status,
1947 (void *)(uintptr_t)wr_id);
5cfb1782 1948 }
c1120f89 1949 queue_work(system_long_wq, &target->tl_err_work);
4f0af697 1950 }
948d1e88
BVA		 1951 target->qp_in_error = true;
1952}
1953
509c07bc 1954static void srp_recv_completion(struct ib_cq *cq, void *ch_ptr)
aef9ec39 1955{
509c07bc 1956 struct srp_rdma_ch *ch = ch_ptr;
aef9ec39 1957 struct ib_wc wc;
aef9ec39
RD		 1958
1959 ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
1960 while (ib_poll_cq(cq, 1, &wc) > 0) {
948d1e88 1961 if (likely(wc.status == IB_WC_SUCCESS)) {
509c07bc 1962 srp_handle_recv(ch, &wc);
948d1e88 1963 } else {
7dad6b2e 1964 srp_handle_qp_err(wc.wr_id, wc.status, false, ch);
aef9ec39 1965 }
9c03dc9f
BVA		 1966 }
1967}
1968
509c07bc 1969static void srp_send_completion(struct ib_cq *cq, void *ch_ptr)
9c03dc9f 1970{
509c07bc 1971 struct srp_rdma_ch *ch = ch_ptr;
9c03dc9f 1972 struct ib_wc wc;
dcb4cb85 1973 struct srp_iu *iu;
9c03dc9f
BVA		 1974
1975 while (ib_poll_cq(cq, 1, &wc) > 0) {
948d1e88
BVA		 1976 if (likely(wc.status == IB_WC_SUCCESS)) {
1977 iu = (struct srp_iu *) (uintptr_t) wc.wr_id;
509c07bc 1978 list_add(&iu->list, &ch->free_tx);
948d1e88 1979 } else {
7dad6b2e 1980 srp_handle_qp_err(wc.wr_id, wc.status, true, ch);
9c03dc9f 1981 }
aef9ec39
RD		 1982 }
1983}
1984
76c75b25 1985static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
aef9ec39 1986{
76c75b25 1987 struct srp_target_port *target = host_to_target(shost);
a95cadb9 1988 struct srp_rport *rport = target->rport;
509c07bc 1989 struct srp_rdma_ch *ch;
aef9ec39
RD		 1990 struct srp_request *req;
1991 struct srp_iu *iu;
1992 struct srp_cmd *cmd;
85507bcc 1993 struct ib_device *dev;
76c75b25 1994 unsigned long flags;
77f2c1a4
BVA		 1995 u32 tag;
1996 u16 idx;
d1b4289e 1997 int len, ret;
a95cadb9
BVA		 1998 const bool in_scsi_eh = !in_interrupt() && current == shost->ehandler;
1999
2000 /*
2001 * The SCSI EH thread is the only context from which srp_queuecommand()
2002 * can get invoked for blocked devices (SDEV_BLOCK /
2003 * SDEV_CREATED_BLOCK). Avoid racing with srp_reconnect_rport() by
2004 * locking the rport mutex if invoked from inside the SCSI EH.
2005 */
2006 if (in_scsi_eh)
2007 mutex_lock(&rport->mutex);
aef9ec39 2008
d1b4289e
BVA		 2009 scmnd->result = srp_chkready(target->rport);
2010 if (unlikely(scmnd->result))
2011 goto err;
2ce19e72 2012
77f2c1a4
BVA		 2013 WARN_ON_ONCE(scmnd->request->tag < 0);
2014 tag = blk_mq_unique_tag(scmnd->request);
d92c0da7 2015 ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)];
77f2c1a4
BVA		 2016 idx = blk_mq_unique_tag_to_tag(tag);
2017 WARN_ONCE(idx >= target->req_ring_size, "%s: tag %#x: idx %d >= %d\n",
2018 dev_name(&shost->shost_gendev), tag, idx,
2019 target->req_ring_size);
509c07bc
BVA		 2020
2021 spin_lock_irqsave(&ch->lock, flags);
2022 iu = __srp_get_tx_iu(ch, SRP_IU_CMD);
509c07bc 2023 spin_unlock_irqrestore(&ch->lock, flags);
aef9ec39 2024
77f2c1a4
BVA		 2025 if (!iu)
2026 goto err;
2027
2028 req = &ch->req_ring[idx];
05321937 2029 dev = target->srp_host->srp_dev->dev;
49248644 2030 ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len,
85507bcc 2031 DMA_TO_DEVICE);
aef9ec39 2032
f8b6e31e 2033 scmnd->host_scribble = (void *) req;
aef9ec39
RD		 2034
2035 cmd = iu->buf;
2036 memset(cmd, 0, sizeof *cmd);
2037
2038 cmd->opcode = SRP_CMD;
2039 cmd->lun = cpu_to_be64((u64) scmnd->device->lun << 48);
77f2c1a4 2040 cmd->tag = tag;
aef9ec39
RD		 2041 memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len);
2042
aef9ec39
RD		 2043 req->scmnd = scmnd;
2044 req->cmd = iu;
aef9ec39 2045
509c07bc 2046 len = srp_map_data(scmnd, ch, req);
aef9ec39 2047 if (len < 0) {
7aa54bd7 2048 shost_printk(KERN_ERR, target->scsi_host,
d1b4289e
BVA		 2049 PFX "Failed to map data (%d)\n", len);
2050 /*
2051 * If we ran out of memory descriptors (-ENOMEM) because an
2052 * application is queuing many requests with more than
52ede08f 2053 * max_pages_per_mr sg-list elements, tell the SCSI mid-layer
d1b4289e
BVA		 2054 * to reduce queue depth temporarily.
2055 */
2056 scmnd->result = len == -ENOMEM ?
2057 DID_OK << 16 | QUEUE_FULL << 1 : DID_ERROR << 16;
76c75b25 2058 goto err_iu;
aef9ec39
RD		 2059 }
2060
49248644 2061 ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len,
85507bcc 2062 DMA_TO_DEVICE);
aef9ec39 2063
509c07bc 2064 if (srp_post_send(ch, iu, len)) {
7aa54bd7 2065 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n");
aef9ec39
RD		 2066 goto err_unmap;
2067 }
2068
d1b4289e
BVA		 2069 ret = 0;
2070
a95cadb9
BVA		 2071unlock_rport:
2072 if (in_scsi_eh)
2073 mutex_unlock(&rport->mutex);
2074
d1b4289e 2075 return ret;
aef9ec39
RD		 2076
2077err_unmap:
509c07bc 2078 srp_unmap_data(scmnd, ch, req);
aef9ec39 2079
76c75b25 2080err_iu:
509c07bc 2081 srp_put_tx_iu(ch, iu, SRP_IU_CMD);
76c75b25 2082
024ca901
BVA		 2083 /*
2084 * Avoid that the loops that iterate over the request ring can
2085 * encounter a dangling SCSI command pointer.
2086 */
2087 req->scmnd = NULL;
2088
d1b4289e
BVA		 2089err:
2090 if (scmnd->result) {
2091 scmnd->scsi_done(scmnd);
2092 ret = 0;
2093 } else {
2094 ret = SCSI_MLQUEUE_HOST_BUSY;
2095 }
a95cadb9 2096
d1b4289e 2097 goto unlock_rport;
aef9ec39
RD		 2098}
2099
4d73f95f
BVA		 2100/*
2101 * Note: the resources allocated in this function are freed in
509c07bc 2102 * srp_free_ch_ib().
4d73f95f 2103 */
509c07bc 2104static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch)
aef9ec39 2105{
509c07bc 2106 struct srp_target_port *target = ch->target;
aef9ec39
RD		 2107 int i;
2108
509c07bc
BVA		 2109 ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring),
2110 GFP_KERNEL);
2111 if (!ch->rx_ring)
4d73f95f 2112 goto err_no_ring;
509c07bc
BVA		 2113 ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring),
2114 GFP_KERNEL);
2115 if (!ch->tx_ring)
4d73f95f
BVA		 2116 goto err_no_ring;
2117
2118 for (i = 0; i < target->queue_size; ++i) {
509c07bc
BVA		 2119 ch->rx_ring[i] = srp_alloc_iu(target->srp_host,
2120 ch->max_ti_iu_len,
2121 GFP_KERNEL, DMA_FROM_DEVICE);
2122 if (!ch->rx_ring[i])
aef9ec39
RD		 2123 goto err;
2124 }
2125
4d73f95f 2126 for (i = 0; i < target->queue_size; ++i) {
509c07bc
BVA		 2127 ch->tx_ring[i] = srp_alloc_iu(target->srp_host,
2128 target->max_iu_len,
2129 GFP_KERNEL, DMA_TO_DEVICE);
2130 if (!ch->tx_ring[i])
aef9ec39 2131 goto err;
dcb4cb85 2132
509c07bc 2133 list_add(&ch->tx_ring[i]->list, &ch->free_tx);
aef9ec39
RD		 2134 }
2135
2136 return 0;
2137
2138err:
4d73f95f 2139 for (i = 0; i < target->queue_size; ++i) {
509c07bc
BVA		 2140 srp_free_iu(target->srp_host, ch->rx_ring[i]);
2141 srp_free_iu(target->srp_host, ch->tx_ring[i]);
aef9ec39
RD		 2142 }
2143
4d73f95f
BVA		 2144
2145err_no_ring:
509c07bc
BVA		 2146 kfree(ch->tx_ring);
2147 ch->tx_ring = NULL;
2148 kfree(ch->rx_ring);
2149 ch->rx_ring = NULL;
4d73f95f 2150
aef9ec39
RD		 2151 return -ENOMEM;
2152}
2153
c9b03c1a
BVA		 2154static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask)
2155{
2156 uint64_t T_tr_ns, max_compl_time_ms;
2157 uint32_t rq_tmo_jiffies;
2158
2159 /*
2160 * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair,
2161 * table 91), both the QP timeout and the retry count have to be set
2162 * for RC QP's during the RTR to RTS transition.
2163 */
2164 WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) !=
2165 (IB_QP_TIMEOUT | IB_QP_RETRY_CNT));
2166
2167 /*
2168 * Set target->rq_tmo_jiffies to one second more than the largest time
2169 * it can take before an error completion is generated. See also
2170 * C9-140..142 in the IBTA spec for more information about how to
2171 * convert the QP Local ACK Timeout value to nanoseconds.
2172 */
2173 T_tr_ns = 4096 * (1ULL << qp_attr->timeout);
2174 max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns;
2175 do_div(max_compl_time_ms, NSEC_PER_MSEC);
2176 rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000);
2177
2178 return rq_tmo_jiffies;
2179}
2180
961e0be8
DD		 2181static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
2182 struct srp_login_rsp *lrsp,
509c07bc 2183 struct srp_rdma_ch *ch)
961e0be8 2184{
509c07bc 2185 struct srp_target_port *target = ch->target;
961e0be8
DD		 2186 struct ib_qp_attr *qp_attr = NULL;
2187 int attr_mask = 0;
2188 int ret;
2189 int i;
2190
2191 if (lrsp->opcode == SRP_LOGIN_RSP) {
509c07bc
BVA		 2192 ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
2193 ch->req_lim = be32_to_cpu(lrsp->req_lim_delta);
961e0be8
DD		 2194
2195 /*
2196 * Reserve credits for task management so we don't
2197 * bounce requests back to the SCSI mid-layer.
2198 */
2199 target->scsi_host->can_queue
509c07bc 2200 = min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE,
961e0be8 2201 target->scsi_host->can_queue);
4d73f95f
BVA		 2202 target->scsi_host->cmd_per_lun
2203 = min_t(int, target->scsi_host->can_queue,
2204 target->scsi_host->cmd_per_lun);
961e0be8
DD		 2205 } else {
2206 shost_printk(KERN_WARNING, target->scsi_host,
2207 PFX "Unhandled RSP opcode %#x\n", lrsp->opcode);
2208 ret = -ECONNRESET;
2209 goto error;
2210 }
2211
509c07bc
BVA		 2212 if (!ch->rx_ring) {
2213 ret = srp_alloc_iu_bufs(ch);
961e0be8
DD		 2214 if (ret)
2215 goto error;
2216 }
2217
2218 ret = -ENOMEM;
2219 qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
2220 if (!qp_attr)
2221 goto error;
2222
2223 qp_attr->qp_state = IB_QPS_RTR;
2224 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2225 if (ret)
2226 goto error_free;
2227
509c07bc 2228 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
961e0be8
DD		 2229 if (ret)
2230 goto error_free;
2231
4d73f95f 2232 for (i = 0; i < target->queue_size; i++) {
509c07bc
BVA		 2233 struct srp_iu *iu = ch->rx_ring[i];
2234
2235 ret = srp_post_recv(ch, iu);
961e0be8
DD		 2236 if (ret)
2237 goto error_free;
2238 }
2239
2240 qp_attr->qp_state = IB_QPS_RTS;
2241 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2242 if (ret)
2243 goto error_free;
2244
c9b03c1a
BVA		 2245 target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);
2246
509c07bc 2247 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
961e0be8
DD		 2248 if (ret)
2249 goto error_free;
2250
2251 ret = ib_send_cm_rtu(cm_id, NULL, 0);
2252
2253error_free:
2254 kfree(qp_attr);
2255
2256error:
509c07bc 2257 ch->status = ret;
961e0be8
DD		 2258}
2259
aef9ec39
RD		 2260static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
2261 struct ib_cm_event *event,
509c07bc 2262 struct srp_rdma_ch *ch)
aef9ec39 2263{
509c07bc 2264 struct srp_target_port *target = ch->target;
7aa54bd7 2265 struct Scsi_Host *shost = target->scsi_host;
aef9ec39
RD		 2266 struct ib_class_port_info *cpi;
2267 int opcode;
2268
2269 switch (event->param.rej_rcvd.reason) {
2270 case IB_CM_REJ_PORT_CM_REDIRECT:
2271 cpi = event->param.rej_rcvd.ari;
509c07bc
BVA		 2272 ch->path.dlid = cpi->redirect_lid;
2273 ch->path.pkey = cpi->redirect_pkey;
aef9ec39 2274 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
509c07bc 2275 memcpy(ch->path.dgid.raw, cpi->redirect_gid, 16);
aef9ec39 2276
509c07bc 2277 ch->status = ch->path.dlid ?
aef9ec39
RD		 2278 SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
2279 break;
2280
2281 case IB_CM_REJ_PORT_REDIRECT:
5d7cbfd6 2282 if (srp_target_is_topspin(target)) {
aef9ec39
RD		 2283 /*
2284 * Topspin/Cisco SRP gateways incorrectly send
2285 * reject reason code 25 when they mean 24
2286 * (port redirect).
2287 */
509c07bc 2288 memcpy(ch->path.dgid.raw,
aef9ec39
RD		 2289 event->param.rej_rcvd.ari, 16);
2290
7aa54bd7
DD		 2291 shost_printk(KERN_DEBUG, shost,
2292 PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
509c07bc
BVA		 2293 be64_to_cpu(ch->path.dgid.global.subnet_prefix),
2294 be64_to_cpu(ch->path.dgid.global.interface_id));
aef9ec39 2295
509c07bc 2296 ch->status = SRP_PORT_REDIRECT;
aef9ec39 2297 } else {
7aa54bd7
DD		 2298 shost_printk(KERN_WARNING, shost,
2299 " REJ reason: IB_CM_REJ_PORT_REDIRECT\n");
509c07bc 2300 ch->status = -ECONNRESET;
aef9ec39
RD		 2301 }
2302 break;
2303
2304 case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
7aa54bd7
DD		 2305 shost_printk(KERN_WARNING, shost,
2306 " REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
509c07bc 2307 ch->status = -ECONNRESET;
aef9ec39
RD		 2308 break;
2309
2310 case IB_CM_REJ_CONSUMER_DEFINED:
2311 opcode = *(u8 *) event->private_data;
2312 if (opcode == SRP_LOGIN_REJ) {
2313 struct srp_login_rej *rej = event->private_data;
2314 u32 reason = be32_to_cpu(rej->reason);
2315
2316 if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
7aa54bd7
DD		 2317 shost_printk(KERN_WARNING, shost,
2318 PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
aef9ec39 2319 else
e7ffde01
BVA		 2320 shost_printk(KERN_WARNING, shost, PFX
2321 "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n",
747fe000
BVA		 2322 target->sgid.raw,
2323 target->orig_dgid.raw, reason);
aef9ec39 2324 } else
7aa54bd7
DD		 2325 shost_printk(KERN_WARNING, shost,
2326 " REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
2327 " opcode 0x%02x\n", opcode);
509c07bc 2328 ch->status = -ECONNRESET;
aef9ec39
RD		 2329 break;
2330
9fe4bcf4
DD		 2331 case IB_CM_REJ_STALE_CONN:
2332 shost_printk(KERN_WARNING, shost, " REJ reason: stale connection\n");
509c07bc 2333 ch->status = SRP_STALE_CONN;
9fe4bcf4
DD		 2334 break;
2335
aef9ec39 2336 default:
7aa54bd7
DD		 2337 shost_printk(KERN_WARNING, shost, " REJ reason 0x%x\n",
2338 event->param.rej_rcvd.reason);
509c07bc 2339 ch->status = -ECONNRESET;
aef9ec39
RD		 2340 }
2341}
2342
2343static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2344{
509c07bc
BVA		 2345 struct srp_rdma_ch *ch = cm_id->context;
2346 struct srp_target_port *target = ch->target;
aef9ec39 2347 int comp = 0;
aef9ec39
RD		 2348
2349 switch (event->event) {
2350 case IB_CM_REQ_ERROR:
7aa54bd7
DD		 2351 shost_printk(KERN_DEBUG, target->scsi_host,
2352 PFX "Sending CM REQ failed\n");
aef9ec39 2353 comp = 1;
509c07bc 2354 ch->status = -ECONNRESET;
aef9ec39
RD		 2355 break;
2356
2357 case IB_CM_REP_RECEIVED:
2358 comp = 1;
509c07bc 2359 srp_cm_rep_handler(cm_id, event->private_data, ch);
aef9ec39
RD		 2360 break;
2361
2362 case IB_CM_REJ_RECEIVED:
7aa54bd7 2363 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
aef9ec39
RD		 2364 comp = 1;
2365
509c07bc 2366 srp_cm_rej_handler(cm_id, event, ch);
aef9ec39
RD		 2367 break;
2368
b7ac4ab4 2369 case IB_CM_DREQ_RECEIVED:
7aa54bd7
DD		 2370 shost_printk(KERN_WARNING, target->scsi_host,
2371 PFX "DREQ received - connection closed\n");
294c875a 2372 srp_change_conn_state(target, false);
b7ac4ab4 2373 if (ib_send_cm_drep(cm_id, NULL, 0))
7aa54bd7
DD		 2374 shost_printk(KERN_ERR, target->scsi_host,
2375 PFX "Sending CM DREP failed\n");
c1120f89 2376 queue_work(system_long_wq, &target->tl_err_work);
aef9ec39
RD		 2377 break;
2378
2379 case IB_CM_TIMEWAIT_EXIT:
7aa54bd7
DD		 2380 shost_printk(KERN_ERR, target->scsi_host,
2381 PFX "connection closed\n");
ac72d766 2382 comp = 1;
aef9ec39 2383
509c07bc 2384 ch->status = 0;
aef9ec39
RD		 2385 break;
2386
b7ac4ab4
IR		 2387 case IB_CM_MRA_RECEIVED:
2388 case IB_CM_DREQ_ERROR:
2389 case IB_CM_DREP_RECEIVED:
2390 break;
2391
aef9ec39 2392 default:
7aa54bd7
DD		 2393 shost_printk(KERN_WARNING, target->scsi_host,
2394 PFX "Unhandled CM event %d\n", event->event);
aef9ec39
RD		 2395 break;
2396 }
2397
2398 if (comp)
509c07bc 2399 complete(&ch->done);
aef9ec39 2400
aef9ec39
RD		 2401 return 0;
2402}
2403
71444b97
JW		 2404/**
2405 * srp_change_queue_depth - setting device queue depth
2406 * @sdev: scsi device struct
2407 * @qdepth: requested queue depth
71444b97
JW		 2408 *
2409 * Returns queue depth.
2410 */
2411static int
db5ed4df 2412srp_change_queue_depth(struct scsi_device *sdev, int qdepth)
71444b97 2413{
c40ecc12 2414 if (!sdev->tagged_supported)
1e6f2416 2415 qdepth = 1;
db5ed4df 2416 return scsi_change_queue_depth(sdev, qdepth);
71444b97
JW		 2417}
2418
509c07bc
BVA		 2419static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag,
2420 unsigned int lun, u8 func)
aef9ec39 2421{
509c07bc 2422 struct srp_target_port *target = ch->target;
a95cadb9 2423 struct srp_rport *rport = target->rport;
19081f31 2424 struct ib_device *dev = target->srp_host->srp_dev->dev;
aef9ec39
RD		 2425 struct srp_iu *iu;
2426 struct srp_tsk_mgmt *tsk_mgmt;
aef9ec39 2427
3780d1f0
BVA		 2428 if (!target->connected || target->qp_in_error)
2429 return -1;
2430
509c07bc 2431 init_completion(&ch->tsk_mgmt_done);
aef9ec39 2432
a95cadb9 2433 /*
509c07bc 2434 * Lock the rport mutex to avoid that srp_create_ch_ib() is
a95cadb9
BVA		 2435 * invoked while a task management function is being sent.
2436 */
2437 mutex_lock(&rport->mutex);
509c07bc
BVA		 2438 spin_lock_irq(&ch->lock);
2439 iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT);
2440 spin_unlock_irq(&ch->lock);
76c75b25 2441
a95cadb9
BVA		 2442 if (!iu) {
2443 mutex_unlock(&rport->mutex);
2444
76c75b25 2445 return -1;
a95cadb9 2446 }
aef9ec39 2447
19081f31
DD		 2448 ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt,
2449 DMA_TO_DEVICE);
aef9ec39
RD		 2450 tsk_mgmt = iu->buf;
2451 memset(tsk_mgmt, 0, sizeof *tsk_mgmt);
2452
2453 tsk_mgmt->opcode = SRP_TSK_MGMT;
f8b6e31e
DD		 2454 tsk_mgmt->lun = cpu_to_be64((u64) lun << 48);
2455 tsk_mgmt->tag = req_tag | SRP_TAG_TSK_MGMT;
aef9ec39 2456 tsk_mgmt->tsk_mgmt_func = func;
f8b6e31e 2457 tsk_mgmt->task_tag = req_tag;
aef9ec39 2458
19081f31
DD		 2459 ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt,
2460 DMA_TO_DEVICE);
509c07bc
BVA		 2461 if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) {
2462 srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT);
a95cadb9
BVA		 2463 mutex_unlock(&rport->mutex);
2464
76c75b25
BVA		 2465 return -1;
2466 }
a95cadb9 2467 mutex_unlock(&rport->mutex);
d945e1df 2468
509c07bc 2469 if (!wait_for_completion_timeout(&ch->tsk_mgmt_done,
aef9ec39 2470 msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS)))
d945e1df 2471 return -1;
aef9ec39 2472
d945e1df 2473 return 0;
d945e1df
RD		 2474}
2475
aef9ec39
RD		 2476static int srp_abort(struct scsi_cmnd *scmnd)
2477{
d945e1df 2478 struct srp_target_port *target = host_to_target(scmnd->device->host);
f8b6e31e 2479 struct srp_request *req = (struct srp_request *) scmnd->host_scribble;
77f2c1a4 2480 u32 tag;
d92c0da7 2481 u16 ch_idx;
509c07bc 2482 struct srp_rdma_ch *ch;
086f44f5 2483 int ret;
d945e1df 2484
7aa54bd7 2485 shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n");
aef9ec39 2486
d92c0da7 2487 if (!req)
99b6697a 2488 return SUCCESS;
77f2c1a4 2489 tag = blk_mq_unique_tag(scmnd->request);
d92c0da7
BVA		 2490 ch_idx = blk_mq_unique_tag_to_hwq(tag);
2491 if (WARN_ON_ONCE(ch_idx >= target->ch_count))
2492 return SUCCESS;
2493 ch = &target->ch[ch_idx];
2494 if (!srp_claim_req(ch, req, NULL, scmnd))
2495 return SUCCESS;
2496 shost_printk(KERN_ERR, target->scsi_host,
2497 "Sending SRP abort for tag %#x\n", tag);
77f2c1a4 2498 if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun,
80d5e8a2 2499 SRP_TSK_ABORT_TASK) == 0)
086f44f5 2500 ret = SUCCESS;
ed9b2264 2501 else if (target->rport->state == SRP_RPORT_LOST)
99e1c139 2502 ret = FAST_IO_FAIL;
086f44f5
BVA		 2503 else
2504 ret = FAILED;
509c07bc 2505 srp_free_req(ch, req, scmnd, 0);
22032991 2506 scmnd->result = DID_ABORT << 16;
d8536670 2507 scmnd->scsi_done(scmnd);
d945e1df 2508
086f44f5 2509 return ret;
aef9ec39
RD		 2510}
2511
2512static int srp_reset_device(struct scsi_cmnd *scmnd)
2513{
d945e1df 2514 struct srp_target_port *target = host_to_target(scmnd->device->host);
d92c0da7 2515 struct srp_rdma_ch *ch;
536ae14e 2516 int i;
d945e1df 2517
7aa54bd7 2518 shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n");
aef9ec39 2519
d92c0da7 2520 ch = &target->ch[0];
509c07bc 2521 if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun,
f8b6e31e 2522 SRP_TSK_LUN_RESET))
d945e1df 2523 return FAILED;
509c07bc 2524 if (ch->tsk_mgmt_status)
d945e1df
RD		 2525 return FAILED;
2526
d92c0da7
BVA		 2527 for (i = 0; i < target->ch_count; i++) {
2528 ch = &target->ch[i];
2529 for (i = 0; i < target->req_ring_size; ++i) {
2530 struct srp_request *req = &ch->req_ring[i];
509c07bc 2531
d92c0da7
BVA		 2532 srp_finish_req(ch, req, scmnd->device, DID_RESET << 16);
2533 }
536ae14e 2534 }
d945e1df 2535
d945e1df 2536 return SUCCESS;
aef9ec39
RD		 2537}
2538
2539static int srp_reset_host(struct scsi_cmnd *scmnd)
2540{
2541 struct srp_target_port *target = host_to_target(scmnd->device->host);
aef9ec39 2542
7aa54bd7 2543 shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n");
aef9ec39 2544
ed9b2264 2545 return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED;
aef9ec39
RD		 2546}
2547
c9b03c1a
BVA		 2548static int srp_slave_configure(struct scsi_device *sdev)
2549{
2550 struct Scsi_Host *shost = sdev->host;
2551 struct srp_target_port *target = host_to_target(shost);
2552 struct request_queue *q = sdev->request_queue;
2553 unsigned long timeout;
2554
2555 if (sdev->type == TYPE_DISK) {
2556 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies);
2557 blk_queue_rq_timeout(q, timeout);
2558 }
2559
2560 return 0;
2561}
2562
ee959b00
TJ		 2563static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr,
2564 char *buf)
6ecb0c84 2565{
ee959b00 2566 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2567
6ecb0c84
RD		 2568 return sprintf(buf, "0x%016llx\n",
2569 (unsigned long long) be64_to_cpu(target->id_ext));
2570}
2571
ee959b00
TJ		 2572static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr,
2573 char *buf)
6ecb0c84 2574{
ee959b00 2575 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2576
6ecb0c84
RD		 2577 return sprintf(buf, "0x%016llx\n",
2578 (unsigned long long) be64_to_cpu(target->ioc_guid));
2579}
2580
ee959b00
TJ		 2581static ssize_t show_service_id(struct device *dev,
2582 struct device_attribute *attr, char *buf)
6ecb0c84 2583{
ee959b00 2584 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2585
6ecb0c84
RD		 2586 return sprintf(buf, "0x%016llx\n",
2587 (unsigned long long) be64_to_cpu(target->service_id));
2588}
2589
ee959b00
TJ		 2590static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
2591 char *buf)
6ecb0c84 2592{
ee959b00 2593 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2594
747fe000 2595 return sprintf(buf, "0x%04x\n", be16_to_cpu(target->pkey));
6ecb0c84
RD		 2596}
2597
848b3082
BVA		 2598static ssize_t show_sgid(struct device *dev, struct device_attribute *attr,
2599 char *buf)
2600{
2601 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2602
747fe000 2603 return sprintf(buf, "%pI6\n", target->sgid.raw);
848b3082
BVA		 2604}
2605
ee959b00
TJ		 2606static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
2607 char *buf)
6ecb0c84 2608{
ee959b00 2609 struct srp_target_port *target = host_to_target(class_to_shost(dev));
d92c0da7 2610 struct srp_rdma_ch *ch = &target->ch[0];
6ecb0c84 2611
509c07bc 2612 return sprintf(buf, "%pI6\n", ch->path.dgid.raw);
6ecb0c84
RD		 2613}
2614
ee959b00
TJ		 2615static ssize_t show_orig_dgid(struct device *dev,
2616 struct device_attribute *attr, char *buf)
3633b3d0 2617{
ee959b00 2618 struct srp_target_port *target = host_to_target(class_to_shost(dev));
3633b3d0 2619
747fe000 2620 return sprintf(buf, "%pI6\n", target->orig_dgid.raw);
3633b3d0
IR		 2621}
2622
89de7486
BVA		 2623static ssize_t show_req_lim(struct device *dev,
2624 struct device_attribute *attr, char *buf)
2625{
2626 struct srp_target_port *target = host_to_target(class_to_shost(dev));
d92c0da7
BVA		 2627 struct srp_rdma_ch *ch;
2628 int i, req_lim = INT_MAX;
89de7486 2629
d92c0da7
BVA		 2630 for (i = 0; i < target->ch_count; i++) {
2631 ch = &target->ch[i];
2632 req_lim = min(req_lim, ch->req_lim);
2633 }
2634 return sprintf(buf, "%d\n", req_lim);
89de7486
BVA		 2635}
2636
ee959b00
TJ		 2637static ssize_t show_zero_req_lim(struct device *dev,
2638 struct device_attribute *attr, char *buf)
6bfa24fa 2639{
ee959b00 2640 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6bfa24fa 2641
6bfa24fa
RD		 2642 return sprintf(buf, "%d\n", target->zero_req_lim);
2643}
2644
ee959b00
TJ		 2645static ssize_t show_local_ib_port(struct device *dev,
2646 struct device_attribute *attr, char *buf)
ded7f1a1 2647{
ee959b00 2648 struct srp_target_port *target = host_to_target(class_to_shost(dev));
ded7f1a1
IR		 2649
2650 return sprintf(buf, "%d\n", target->srp_host->port);
2651}
2652
ee959b00
TJ		 2653static ssize_t show_local_ib_device(struct device *dev,
2654 struct device_attribute *attr, char *buf)
ded7f1a1 2655{
ee959b00 2656 struct srp_target_port *target = host_to_target(class_to_shost(dev));
ded7f1a1 2657
05321937 2658 return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
ded7f1a1
IR		 2659}
2660
d92c0da7
BVA		 2661static ssize_t show_ch_count(struct device *dev, struct device_attribute *attr,
2662 char *buf)
2663{
2664 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2665
2666 return sprintf(buf, "%d\n", target->ch_count);
2667}
2668
4b5e5f41
BVA		 2669static ssize_t show_comp_vector(struct device *dev,
2670 struct device_attribute *attr, char *buf)
2671{
2672 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2673
2674 return sprintf(buf, "%d\n", target->comp_vector);
2675}
2676
7bb312e4
VP		 2677static ssize_t show_tl_retry_count(struct device *dev,
2678 struct device_attribute *attr, char *buf)
2679{
2680 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2681
2682 return sprintf(buf, "%d\n", target->tl_retry_count);
2683}
2684
49248644
DD		 2685static ssize_t show_cmd_sg_entries(struct device *dev,
2686 struct device_attribute *attr, char *buf)
2687{
2688 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2689
2690 return sprintf(buf, "%u\n", target->cmd_sg_cnt);
2691}
2692
c07d424d
DD		 2693static ssize_t show_allow_ext_sg(struct device *dev,
2694 struct device_attribute *attr, char *buf)
2695{
2696 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2697
2698 return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
2699}
2700
ee959b00
TJ		 2701static DEVICE_ATTR(id_ext, S_IRUGO, show_id_ext, NULL);
2702static DEVICE_ATTR(ioc_guid, S_IRUGO, show_ioc_guid, NULL);
2703static DEVICE_ATTR(service_id, S_IRUGO, show_service_id, NULL);
2704static DEVICE_ATTR(pkey, S_IRUGO, show_pkey, NULL);
848b3082 2705static DEVICE_ATTR(sgid, S_IRUGO, show_sgid, NULL);
ee959b00
TJ		 2706static DEVICE_ATTR(dgid, S_IRUGO, show_dgid, NULL);
2707static DEVICE_ATTR(orig_dgid, S_IRUGO, show_orig_dgid, NULL);
89de7486 2708static DEVICE_ATTR(req_lim, S_IRUGO, show_req_lim, NULL);
ee959b00
TJ		 2709static DEVICE_ATTR(zero_req_lim, S_IRUGO, show_zero_req_lim, NULL);
2710static DEVICE_ATTR(local_ib_port, S_IRUGO, show_local_ib_port, NULL);
2711static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
d92c0da7 2712static DEVICE_ATTR(ch_count, S_IRUGO, show_ch_count, NULL);
4b5e5f41 2713static DEVICE_ATTR(comp_vector, S_IRUGO, show_comp_vector, NULL);
7bb312e4 2714static DEVICE_ATTR(tl_retry_count, S_IRUGO, show_tl_retry_count, NULL);
49248644 2715static DEVICE_ATTR(cmd_sg_entries, S_IRUGO, show_cmd_sg_entries, NULL);
c07d424d 2716static DEVICE_ATTR(allow_ext_sg, S_IRUGO, show_allow_ext_sg, NULL);
ee959b00
TJ		 2717
2718static struct device_attribute *srp_host_attrs[] = {
2719 &dev_attr_id_ext,
2720 &dev_attr_ioc_guid,
2721 &dev_attr_service_id,
2722 &dev_attr_pkey,
848b3082 2723 &dev_attr_sgid,
ee959b00
TJ		 2724 &dev_attr_dgid,
2725 &dev_attr_orig_dgid,
89de7486 2726 &dev_attr_req_lim,
ee959b00
TJ		 2727 &dev_attr_zero_req_lim,
2728 &dev_attr_local_ib_port,
2729 &dev_attr_local_ib_device,
d92c0da7 2730 &dev_attr_ch_count,
4b5e5f41 2731 &dev_attr_comp_vector,
7bb312e4 2732 &dev_attr_tl_retry_count,
49248644 2733 &dev_attr_cmd_sg_entries,
c07d424d 2734 &dev_attr_allow_ext_sg,
6ecb0c84
RD		 2735 NULL
2736};
2737
aef9ec39
RD		 2738static struct scsi_host_template srp_template = {
2739 .module = THIS_MODULE,
b7f008fd
RD		 2740 .name = "InfiniBand SRP initiator",
2741 .proc_name = DRV_NAME,
c9b03c1a 2742 .slave_configure = srp_slave_configure,
aef9ec39
RD		 2743 .info = srp_target_info,
2744 .queuecommand = srp_queuecommand,
71444b97 2745 .change_queue_depth = srp_change_queue_depth,
aef9ec39
RD		 2746 .eh_abort_handler = srp_abort,
2747 .eh_device_reset_handler = srp_reset_device,
2748 .eh_host_reset_handler = srp_reset_host,
2742c1da 2749 .skip_settle_delay = true,
49248644 2750 .sg_tablesize = SRP_DEF_SG_TABLESIZE,
4d73f95f 2751 .can_queue = SRP_DEFAULT_CMD_SQ_SIZE,
aef9ec39 2752 .this_id = -1,
4d73f95f 2753 .cmd_per_lun = SRP_DEFAULT_CMD_SQ_SIZE,
6ecb0c84 2754 .use_clustering = ENABLE_CLUSTERING,
77f2c1a4
BVA		 2755 .shost_attrs = srp_host_attrs,
2756 .use_blk_tags = 1,
c40ecc12 2757 .track_queue_depth = 1,
aef9ec39
RD		 2758};
2759
34aa654e
BVA		 2760static int srp_sdev_count(struct Scsi_Host *host)
2761{
2762 struct scsi_device *sdev;
2763 int c = 0;
2764
2765 shost_for_each_device(sdev, host)
2766 c++;
2767
2768 return c;
2769}
2770
aef9ec39
RD		 2771static int srp_add_target(struct srp_host *host, struct srp_target_port *target)
2772{
3236822b
FT		 2773 struct srp_rport_identifiers ids;
2774 struct srp_rport *rport;
2775
34aa654e 2776 target->state = SRP_TARGET_SCANNING;
aef9ec39
RD		 2777 sprintf(target->target_name, "SRP.T10:%016llX",
2778 (unsigned long long) be64_to_cpu(target->id_ext));
2779
05321937 2780 if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dma_device))
aef9ec39
RD		 2781 return -ENODEV;
2782
3236822b
FT		 2783 memcpy(ids.port_id, &target->id_ext, 8);
2784 memcpy(ids.port_id + 8, &target->ioc_guid, 8);
aebd5e47 2785 ids.roles = SRP_RPORT_ROLE_TARGET;
3236822b
FT		 2786 rport = srp_rport_add(target->scsi_host, &ids);
2787 if (IS_ERR(rport)) {
2788 scsi_remove_host(target->scsi_host);
2789 return PTR_ERR(rport);
2790 }
2791
dc1bdbd9 2792 rport->lld_data = target;
9dd69a60 2793 target->rport = rport;
dc1bdbd9 2794
b3589fd4 2795 spin_lock(&host->target_lock);
aef9ec39 2796 list_add_tail(&target->list, &host->target_list);
b3589fd4 2797 spin_unlock(&host->target_lock);
aef9ec39 2798
aef9ec39 2799 scsi_scan_target(&target->scsi_host->shost_gendev,
1962a4a1 2800 0, target->scsi_id, SCAN_WILD_CARD, 0);
aef9ec39 2801
34aa654e
BVA		 2802 if (!target->connected || target->qp_in_error) {
2803 shost_printk(KERN_INFO, target->scsi_host,
2804 PFX "SCSI scan failed - removing SCSI host\n");
2805 srp_queue_remove_work(target);
2806 goto out;
2807 }
2808
2809 pr_debug(PFX "%s: SCSI scan succeeded - detected %d LUNs\n",
2810 dev_name(&target->scsi_host->shost_gendev),
2811 srp_sdev_count(target->scsi_host));
2812
2813 spin_lock_irq(&target->lock);
2814 if (target->state == SRP_TARGET_SCANNING)
2815 target->state = SRP_TARGET_LIVE;
2816 spin_unlock_irq(&target->lock);
2817
2818out:
aef9ec39
RD		 2819 return 0;
2820}
2821
ee959b00 2822static void srp_release_dev(struct device *dev)
aef9ec39
RD		 2823{
2824 struct srp_host *host =
ee959b00 2825 container_of(dev, struct srp_host, dev);
aef9ec39
RD		 2826
2827 complete(&host->released);
2828}
2829
2830static struct class srp_class = {
2831 .name = "infiniband_srp",
ee959b00 2832 .dev_release = srp_release_dev
aef9ec39
RD		 2833};
2834
96fc248a
BVA		 2835/**
2836 * srp_conn_unique() - check whether the connection to a target is unique
af24663b
BVA		 2837 * @host: SRP host.
2838 * @target: SRP target port.
96fc248a
BVA		 2839 */
2840static bool srp_conn_unique(struct srp_host *host,
2841 struct srp_target_port *target)
2842{
2843 struct srp_target_port *t;
2844 bool ret = false;
2845
2846 if (target->state == SRP_TARGET_REMOVED)
2847 goto out;
2848
2849 ret = true;
2850
2851 spin_lock(&host->target_lock);
2852 list_for_each_entry(t, &host->target_list, list) {
2853 if (t != target &&
2854 target->id_ext == t->id_ext &&
2855 target->ioc_guid == t->ioc_guid &&
2856 target->initiator_ext == t->initiator_ext) {
2857 ret = false;
2858 break;
2859 }
2860 }
2861 spin_unlock(&host->target_lock);
2862
2863out:
2864 return ret;
2865}
2866
aef9ec39
RD		 2867/*
2868 * Target ports are added by writing
2869 *
2870 * id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>,
2871 * pkey=<P_Key>,service_id=<service ID>
2872 *
2873 * to the add_target sysfs attribute.
2874 */
2875enum {
2876 SRP_OPT_ERR = 0,
2877 SRP_OPT_ID_EXT = 1 << 0,
2878 SRP_OPT_IOC_GUID = 1 << 1,
2879 SRP_OPT_DGID = 1 << 2,
2880 SRP_OPT_PKEY = 1 << 3,
2881 SRP_OPT_SERVICE_ID = 1 << 4,
2882 SRP_OPT_MAX_SECT = 1 << 5,
52fb2b50 2883 SRP_OPT_MAX_CMD_PER_LUN = 1 << 6,
0c0450db 2884 SRP_OPT_IO_CLASS = 1 << 7,
01cb9bcb 2885 SRP_OPT_INITIATOR_EXT = 1 << 8,
49248644 2886 SRP_OPT_CMD_SG_ENTRIES = 1 << 9,
c07d424d
DD		 2887 SRP_OPT_ALLOW_EXT_SG = 1 << 10,
2888 SRP_OPT_SG_TABLESIZE = 1 << 11,
4b5e5f41 2889 SRP_OPT_COMP_VECTOR = 1 << 12,
7bb312e4 2890 SRP_OPT_TL_RETRY_COUNT = 1 << 13,
4d73f95f 2891 SRP_OPT_QUEUE_SIZE = 1 << 14,
aef9ec39
RD		 2892 SRP_OPT_ALL = (SRP_OPT_ID_EXT |
2893 SRP_OPT_IOC_GUID |
2894 SRP_OPT_DGID |
2895 SRP_OPT_PKEY |
2896 SRP_OPT_SERVICE_ID),
2897};
2898
a447c093 2899static const match_table_t srp_opt_tokens = {
52fb2b50
VP		 2900 { SRP_OPT_ID_EXT, "id_ext=%s" },
2901 { SRP_OPT_IOC_GUID, "ioc_guid=%s" },
2902 { SRP_OPT_DGID, "dgid=%s" },
2903 { SRP_OPT_PKEY, "pkey=%x" },
2904 { SRP_OPT_SERVICE_ID, "service_id=%s" },
2905 { SRP_OPT_MAX_SECT, "max_sect=%d" },
2906 { SRP_OPT_MAX_CMD_PER_LUN, "max_cmd_per_lun=%d" },
0c0450db 2907 { SRP_OPT_IO_CLASS, "io_class=%x" },
01cb9bcb 2908 { SRP_OPT_INITIATOR_EXT, "initiator_ext=%s" },
49248644 2909 { SRP_OPT_CMD_SG_ENTRIES, "cmd_sg_entries=%u" },
c07d424d
DD		 2910 { SRP_OPT_ALLOW_EXT_SG, "allow_ext_sg=%u" },
2911 { SRP_OPT_SG_TABLESIZE, "sg_tablesize=%u" },
4b5e5f41 2912 { SRP_OPT_COMP_VECTOR, "comp_vector=%u" },
7bb312e4 2913 { SRP_OPT_TL_RETRY_COUNT, "tl_retry_count=%u" },
4d73f95f 2914 { SRP_OPT_QUEUE_SIZE, "queue_size=%d" },
52fb2b50 2915 { SRP_OPT_ERR, NULL }
aef9ec39
RD		 2916};
2917
2918static int srp_parse_options(const char *buf, struct srp_target_port *target)
2919{
2920 char *options, *sep_opt;
2921 char *p;
2922 char dgid[3];
2923 substring_t args[MAX_OPT_ARGS];
2924 int opt_mask = 0;
2925 int token;
2926 int ret = -EINVAL;
2927 int i;
2928
2929 options = kstrdup(buf, GFP_KERNEL);
2930 if (!options)
2931 return -ENOMEM;
2932
2933 sep_opt = options;
7dcf9c19 2934 while ((p = strsep(&sep_opt, ",\n")) != NULL) {
aef9ec39
RD		 2935 if (!*p)
2936 continue;
2937
2938 token = match_token(p, srp_opt_tokens, args);
2939 opt_mask |= token;
2940
2941 switch (token) {
2942 case SRP_OPT_ID_EXT:
2943 p = match_strdup(args);
a20f3a6d
IR		 2944 if (!p) {
2945 ret = -ENOMEM;
2946 goto out;
2947 }
aef9ec39
RD		 2948 target->id_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
2949 kfree(p);
2950 break;
2951
2952 case SRP_OPT_IOC_GUID:
2953 p = match_strdup(args);
a20f3a6d
IR		 2954 if (!p) {
2955 ret = -ENOMEM;
2956 goto out;
2957 }
aef9ec39
RD		 2958 target->ioc_guid = cpu_to_be64(simple_strtoull(p, NULL, 16));
2959 kfree(p);
2960 break;
2961
2962 case SRP_OPT_DGID:
2963 p = match_strdup(args);
a20f3a6d
IR		 2964 if (!p) {
2965 ret = -ENOMEM;
2966 goto out;
2967 }
aef9ec39 2968 if (strlen(p) != 32) {
e0bda7d8 2969 pr_warn("bad dest GID parameter '%s'\n", p);
ce1823f0 2970 kfree(p);
aef9ec39
RD		 2971 goto out;
2972 }
2973
2974 for (i = 0; i < 16; ++i) {
747fe000
BVA		 2975 strlcpy(dgid, p + i * 2, sizeof(dgid));
2976 if (sscanf(dgid, "%hhx",
2977 &target->orig_dgid.raw[i]) < 1) {
2978 ret = -EINVAL;
2979 kfree(p);
2980 goto out;
2981 }
aef9ec39 2982 }
bf17c1c7 2983 kfree(p);
aef9ec39
RD		 2984 break;
2985
2986 case SRP_OPT_PKEY:
2987 if (match_hex(args, &token)) {
e0bda7d8 2988 pr_warn("bad P_Key parameter '%s'\n", p);
aef9ec39
RD		 2989 goto out;
2990 }
747fe000 2991 target->pkey = cpu_to_be16(token);
aef9ec39
RD		 2992 break;
2993
2994 case SRP_OPT_SERVICE_ID:
2995 p = match_strdup(args);
a20f3a6d
IR		 2996 if (!p) {
2997 ret = -ENOMEM;
2998 goto out;
2999 }
aef9ec39
RD		 3000 target->service_id = cpu_to_be64(simple_strtoull(p, NULL, 16));
3001 kfree(p);
3002 break;
3003
3004 case SRP_OPT_MAX_SECT:
3005 if (match_int(args, &token)) {
e0bda7d8 3006 pr_warn("bad max sect parameter '%s'\n", p);
aef9ec39
RD		 3007 goto out;
3008 }
3009 target->scsi_host->max_sectors = token;
3010 break;
3011
4d73f95f
BVA		 3012 case SRP_OPT_QUEUE_SIZE:
3013 if (match_int(args, &token) || token < 1) {
3014 pr_warn("bad queue_size parameter '%s'\n", p);
3015 goto out;
3016 }
3017 target->scsi_host->can_queue = token;
3018 target->queue_size = token + SRP_RSP_SQ_SIZE +
3019 SRP_TSK_MGMT_SQ_SIZE;
3020 if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3021 target->scsi_host->cmd_per_lun = token;
3022 break;
3023
52fb2b50 3024 case SRP_OPT_MAX_CMD_PER_LUN:
4d73f95f 3025 if (match_int(args, &token) || token < 1) {
e0bda7d8
BVA		 3026 pr_warn("bad max cmd_per_lun parameter '%s'\n",
3027 p);
52fb2b50
VP		 3028 goto out;
3029 }
4d73f95f 3030 target->scsi_host->cmd_per_lun = token;
52fb2b50
VP		 3031 break;
3032
0c0450db
R		 3033 case SRP_OPT_IO_CLASS:
3034 if (match_hex(args, &token)) {
e0bda7d8 3035 pr_warn("bad IO class parameter '%s'\n", p);
0c0450db
R		 3036 goto out;
3037 }
3038 if (token != SRP_REV10_IB_IO_CLASS &&
3039 token != SRP_REV16A_IB_IO_CLASS) {
e0bda7d8
BVA		 3040 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
3041 token, SRP_REV10_IB_IO_CLASS,
3042 SRP_REV16A_IB_IO_CLASS);
0c0450db
R		 3043 goto out;
3044 }
3045 target->io_class = token;
3046 break;
3047
01cb9bcb
IR		 3048 case SRP_OPT_INITIATOR_EXT:
3049 p = match_strdup(args);
a20f3a6d
IR		 3050 if (!p) {
3051 ret = -ENOMEM;
3052 goto out;
3053 }
01cb9bcb
IR		 3054 target->initiator_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
3055 kfree(p);
3056 break;
3057
49248644
DD		 3058 case SRP_OPT_CMD_SG_ENTRIES:
3059 if (match_int(args, &token) || token < 1 || token > 255) {
e0bda7d8
BVA		 3060 pr_warn("bad max cmd_sg_entries parameter '%s'\n",
3061 p);
49248644
DD		 3062 goto out;
3063 }
3064 target->cmd_sg_cnt = token;
3065 break;
3066
c07d424d
DD		 3067 case SRP_OPT_ALLOW_EXT_SG:
3068 if (match_int(args, &token)) {
e0bda7d8 3069 pr_warn("bad allow_ext_sg parameter '%s'\n", p);
c07d424d
DD		 3070 goto out;
3071 }
3072 target->allow_ext_sg = !!token;
3073 break;
3074
3075 case SRP_OPT_SG_TABLESIZE:
3076 if (match_int(args, &token) || token < 1 ||
3077 token > SCSI_MAX_SG_CHAIN_SEGMENTS) {
e0bda7d8
BVA		 3078 pr_warn("bad max sg_tablesize parameter '%s'\n",
3079 p);
c07d424d
DD		 3080 goto out;
3081 }
3082 target->sg_tablesize = token;
3083 break;
3084
4b5e5f41
BVA		 3085 case SRP_OPT_COMP_VECTOR:
3086 if (match_int(args, &token) || token < 0) {
3087 pr_warn("bad comp_vector parameter '%s'\n", p);
3088 goto out;
3089 }
3090 target->comp_vector = token;
3091 break;
3092
7bb312e4
VP		 3093 case SRP_OPT_TL_RETRY_COUNT:
3094 if (match_int(args, &token) || token < 2 || token > 7) {
3095 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
3096 p);
3097 goto out;
3098 }
3099 target->tl_retry_count = token;
3100 break;
3101
aef9ec39 3102 default:
e0bda7d8
BVA		 3103 pr_warn("unknown parameter or missing value '%s' in target creation request\n",
3104 p);
aef9ec39
RD		 3105 goto out;
3106 }
3107 }
3108
3109 if ((opt_mask & SRP_OPT_ALL) == SRP_OPT_ALL)
3110 ret = 0;
3111 else
3112 for (i = 0; i < ARRAY_SIZE(srp_opt_tokens); ++i)
3113 if ((srp_opt_tokens[i].token & SRP_OPT_ALL) &&
3114 !(srp_opt_tokens[i].token & opt_mask))
e0bda7d8
BVA		 3115 pr_warn("target creation request is missing parameter '%s'\n",
3116 srp_opt_tokens[i].pattern);
aef9ec39 3117
4d73f95f
BVA		 3118 if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue
3119 && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3120 pr_warn("cmd_per_lun = %d > queue_size = %d\n",
3121 target->scsi_host->cmd_per_lun,
3122 target->scsi_host->can_queue);
3123
aef9ec39
RD		 3124out:
3125 kfree(options);
3126 return ret;
3127}
3128
ee959b00
TJ		 3129static ssize_t srp_create_target(struct device *dev,
3130 struct device_attribute *attr,
aef9ec39
RD		 3131 const char *buf, size_t count)
3132{
3133 struct srp_host *host =
ee959b00 3134 container_of(dev, struct srp_host, dev);
aef9ec39
RD		 3135 struct Scsi_Host *target_host;
3136 struct srp_target_port *target;
509c07bc 3137 struct srp_rdma_ch *ch;
d1b4289e
BVA		 3138 struct srp_device *srp_dev = host->srp_dev;
3139 struct ib_device *ibdev = srp_dev->dev;
d92c0da7
BVA		 3140 int ret, node_idx, node, cpu, i;
3141 bool multich = false;
aef9ec39
RD		 3142
3143 target_host = scsi_host_alloc(&srp_template,
3144 sizeof (struct srp_target_port));
3145 if (!target_host)
3146 return -ENOMEM;
3147
49248644 3148 target_host->transportt = ib_srp_transport_template;
fd1b6c4a
BVA		 3149 target_host->max_channel = 0;
3150 target_host->max_id = 1;
3c8edf0e
AR		 3151 target_host->max_lun = SRP_MAX_LUN;
3152 target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb;
5f068992 3153
aef9ec39 3154 target = host_to_target(target_host);
aef9ec39 3155
49248644
DD		 3156 target->io_class = SRP_REV16A_IB_IO_CLASS;
3157 target->scsi_host = target_host;
3158 target->srp_host = host;
3159 target->lkey = host->srp_dev->mr->lkey;
3160 target->rkey = host->srp_dev->mr->rkey;
3161 target->cmd_sg_cnt = cmd_sg_entries;
c07d424d
DD		 3162 target->sg_tablesize = indirect_sg_entries ? : cmd_sg_entries;
3163 target->allow_ext_sg = allow_ext_sg;
7bb312e4 3164 target->tl_retry_count = 7;
4d73f95f 3165 target->queue_size = SRP_DEFAULT_QUEUE_SIZE;
aef9ec39 3166
34aa654e
BVA		 3167 /*
3168 * Avoid that the SCSI host can be removed by srp_remove_target()
3169 * before this function returns.
3170 */
3171 scsi_host_get(target->scsi_host);
3172
2d7091bc
BVA		 3173 mutex_lock(&host->add_target_mutex);
3174
aef9ec39
RD		 3175 ret = srp_parse_options(buf, target);
3176 if (ret)
3177 goto err;
3178
77f2c1a4
BVA		 3179 ret = scsi_init_shared_tag_map(target_host, target_host->can_queue);
3180 if (ret)
3181 goto err;
3182
4d73f95f
BVA		 3183 target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE;
3184
96fc248a
BVA		 3185 if (!srp_conn_unique(target->srp_host, target)) {
3186 shost_printk(KERN_INFO, target->scsi_host,
3187 PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
3188 be64_to_cpu(target->id_ext),
3189 be64_to_cpu(target->ioc_guid),
3190 be64_to_cpu(target->initiator_ext));
3191 ret = -EEXIST;
3192 goto err;
3193 }
3194
5cfb1782 3195 if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg &&
d1b4289e 3196 target->cmd_sg_cnt < target->sg_tablesize) {
5cfb1782 3197 pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
c07d424d
DD		 3198 target->sg_tablesize = target->cmd_sg_cnt;
3199 }
3200
3201 target_host->sg_tablesize = target->sg_tablesize;
3202 target->indirect_size = target->sg_tablesize *
3203 sizeof (struct srp_direct_buf);
49248644
DD		 3204 target->max_iu_len = sizeof (struct srp_cmd) +
3205 sizeof (struct srp_indirect_buf) +
3206 target->cmd_sg_cnt * sizeof (struct srp_direct_buf);
3207
c1120f89 3208 INIT_WORK(&target->tl_err_work, srp_tl_err_work);
ef6c49d8 3209 INIT_WORK(&target->remove_work, srp_remove_work);
8f26c9ff 3210 spin_lock_init(&target->lock);
747fe000 3211 ret = ib_query_gid(ibdev, host->port, 0, &target->sgid);
2088ca66 3212 if (ret)
d92c0da7 3213 goto err;
aef9ec39 3214
d92c0da7
BVA		 3215 ret = -ENOMEM;
3216 target->ch_count = max_t(unsigned, num_online_nodes(),
3217 min(ch_count ? :
3218 min(4 * num_online_nodes(),
3219 ibdev->num_comp_vectors),
3220 num_online_cpus()));
3221 target->ch = kcalloc(target->ch_count, sizeof(*target->ch),
3222 GFP_KERNEL);
3223 if (!target->ch)
3224 goto err;
aef9ec39 3225
d92c0da7
BVA		 3226 node_idx = 0;
3227 for_each_online_node(node) {
3228 const int ch_start = (node_idx * target->ch_count /
3229 num_online_nodes());
3230 const int ch_end = ((node_idx + 1) * target->ch_count /
3231 num_online_nodes());
3232 const int cv_start = (node_idx * ibdev->num_comp_vectors /
3233 num_online_nodes() + target->comp_vector)
3234 % ibdev->num_comp_vectors;
3235 const int cv_end = ((node_idx + 1) * ibdev->num_comp_vectors /
3236 num_online_nodes() + target->comp_vector)
3237 % ibdev->num_comp_vectors;
3238 int cpu_idx = 0;
3239
3240 for_each_online_cpu(cpu) {
3241 if (cpu_to_node(cpu) != node)
3242 continue;
3243 if (ch_start + cpu_idx >= ch_end)
3244 continue;
3245 ch = &target->ch[ch_start + cpu_idx];
3246 ch->target = target;
3247 ch->comp_vector = cv_start == cv_end ? cv_start :
3248 cv_start + cpu_idx % (cv_end - cv_start);
3249 spin_lock_init(&ch->lock);
3250 INIT_LIST_HEAD(&ch->free_tx);
3251 ret = srp_new_cm_id(ch);
3252 if (ret)
3253 goto err_disconnect;
aef9ec39 3254
d92c0da7
BVA		 3255 ret = srp_create_ch_ib(ch);
3256 if (ret)
3257 goto err_disconnect;
3258
3259 ret = srp_alloc_req_data(ch);
3260 if (ret)
3261 goto err_disconnect;
3262
3263 ret = srp_connect_ch(ch, multich);
3264 if (ret) {
3265 shost_printk(KERN_ERR, target->scsi_host,
3266 PFX "Connection %d/%d failed\n",
3267 ch_start + cpu_idx,
3268 target->ch_count);
3269 if (node_idx == 0 && cpu_idx == 0) {
3270 goto err_disconnect;
3271 } else {
3272 srp_free_ch_ib(target, ch);
3273 srp_free_req_data(target, ch);
3274 target->ch_count = ch - target->ch;
3275 break;
3276 }
3277 }
3278
3279 multich = true;
3280 cpu_idx++;
3281 }
3282 node_idx++;
aef9ec39
RD		 3283 }
3284
d92c0da7
BVA		 3285 target->scsi_host->nr_hw_queues = target->ch_count;
3286
aef9ec39
RD		 3287 ret = srp_add_target(host, target);
3288 if (ret)
3289 goto err_disconnect;
3290
34aa654e
BVA		 3291 if (target->state != SRP_TARGET_REMOVED) {
3292 shost_printk(KERN_DEBUG, target->scsi_host, PFX
3293 "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n",
3294 be64_to_cpu(target->id_ext),
3295 be64_to_cpu(target->ioc_guid),
747fe000 3296 be16_to_cpu(target->pkey),
34aa654e 3297 be64_to_cpu(target->service_id),
747fe000 3298 target->sgid.raw, target->orig_dgid.raw);
34aa654e 3299 }
e7ffde01 3300
2d7091bc
BVA		 3301 ret = count;
3302
3303out:
3304 mutex_unlock(&host->add_target_mutex);
34aa654e
BVA		 3305
3306 scsi_host_put(target->scsi_host);
3307
2d7091bc 3308 return ret;
aef9ec39
RD		 3309
3310err_disconnect:
3311 srp_disconnect_target(target);
3312
d92c0da7
BVA		 3313 for (i = 0; i < target->ch_count; i++) {
3314 ch = &target->ch[i];
3315 srp_free_ch_ib(target, ch);
3316 srp_free_req_data(target, ch);
3317 }
aef9ec39 3318
d92c0da7 3319 kfree(target->ch);
8f26c9ff 3320
aef9ec39
RD		 3321err:
3322 scsi_host_put(target_host);
2d7091bc 3323 goto out;
aef9ec39
RD		 3324}
3325
ee959b00 3326static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target);
aef9ec39 3327
ee959b00
TJ		 3328static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
3329 char *buf)
aef9ec39 3330{
ee959b00 3331 struct srp_host *host = container_of(dev, struct srp_host, dev);
aef9ec39 3332
05321937 3333 return sprintf(buf, "%s\n", host->srp_dev->dev->name);
aef9ec39
RD		 3334}
3335
ee959b00 3336static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
aef9ec39 3337
ee959b00
TJ		 3338static ssize_t show_port(struct device *dev, struct device_attribute *attr,
3339 char *buf)
aef9ec39 3340{
ee959b00 3341 struct srp_host *host = container_of(dev, struct srp_host, dev);
aef9ec39
RD		 3342
3343 return sprintf(buf, "%d\n", host->port);
3344}
3345
ee959b00 3346static DEVICE_ATTR(port, S_IRUGO, show_port, NULL);
aef9ec39 3347
f5358a17 3348static struct srp_host *srp_add_port(struct srp_device *device, u8 port)
aef9ec39
RD		 3349{
3350 struct srp_host *host;
3351
3352 host = kzalloc(sizeof *host, GFP_KERNEL);
3353 if (!host)
3354 return NULL;
3355
3356 INIT_LIST_HEAD(&host->target_list);
b3589fd4 3357 spin_lock_init(&host->target_lock);
aef9ec39 3358 init_completion(&host->released);
2d7091bc 3359 mutex_init(&host->add_target_mutex);
05321937 3360 host->srp_dev = device;
aef9ec39
RD		 3361 host->port = port;
3362
ee959b00
TJ		 3363 host->dev.class = &srp_class;
3364 host->dev.parent = device->dev->dma_device;
d927e38c 3365 dev_set_name(&host->dev, "srp-%s-%d", device->dev->name, port);
aef9ec39 3366
ee959b00 3367 if (device_register(&host->dev))
f5358a17 3368 goto free_host;
ee959b00 3369 if (device_create_file(&host->dev, &dev_attr_add_target))
aef9ec39 3370 goto err_class;
ee959b00 3371 if (device_create_file(&host->dev, &dev_attr_ibdev))
aef9ec39 3372 goto err_class;
ee959b00 3373 if (device_create_file(&host->dev, &dev_attr_port))
aef9ec39
RD		 3374 goto err_class;
3375
3376 return host;
3377
3378err_class:
ee959b00 3379 device_unregister(&host->dev);
aef9ec39 3380
f5358a17 3381free_host:
aef9ec39
RD		 3382 kfree(host);
3383
3384 return NULL;
3385}
3386
3387static void srp_add_one(struct ib_device *device)
3388{
f5358a17
RD		 3389 struct srp_device *srp_dev;
3390 struct ib_device_attr *dev_attr;
aef9ec39 3391 struct srp_host *host;
52ede08f
BVA		 3392 int mr_page_shift, s, e, p;
3393 u64 max_pages_per_mr;
aef9ec39 3394
f5358a17
RD		 3395 dev_attr = kmalloc(sizeof *dev_attr, GFP_KERNEL);
3396 if (!dev_attr)
cf311cd4 3397 return;
aef9ec39 3398
f5358a17 3399 if (ib_query_device(device, dev_attr)) {
e0bda7d8 3400 pr_warn("Query device failed for %s\n", device->name);
f5358a17
RD		 3401 goto free_attr;
3402 }
3403
3404 srp_dev = kmalloc(sizeof *srp_dev, GFP_KERNEL);
3405 if (!srp_dev)
3406 goto free_attr;
3407
d1b4289e
BVA		 3408 srp_dev->has_fmr = (device->alloc_fmr && device->dealloc_fmr &&
3409 device->map_phys_fmr && device->unmap_fmr);
5cfb1782
BVA		 3410 srp_dev->has_fr = (dev_attr->device_cap_flags &
3411 IB_DEVICE_MEM_MGT_EXTENSIONS);
3412 if (!srp_dev->has_fmr && !srp_dev->has_fr)
3413 dev_warn(&device->dev, "neither FMR nor FR is supported\n");
3414
3415 srp_dev->use_fast_reg = (srp_dev->has_fr &&
3416 (!srp_dev->has_fmr || prefer_fr));
d1b4289e 3417
f5358a17
RD		 3418 /*
3419 * Use the smallest page size supported by the HCA, down to a
8f26c9ff
DD		 3420 * minimum of 4096 bytes. We're unlikely to build large sglists
3421 * out of smaller entries.
f5358a17 3422 */
52ede08f
BVA		 3423 mr_page_shift = max(12, ffs(dev_attr->page_size_cap) - 1);
3424 srp_dev->mr_page_size = 1 << mr_page_shift;
3425 srp_dev->mr_page_mask = ~((u64) srp_dev->mr_page_size - 1);
3426 max_pages_per_mr = dev_attr->max_mr_size;
3427 do_div(max_pages_per_mr, srp_dev->mr_page_size);
3428 srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
3429 max_pages_per_mr);
5cfb1782
BVA		 3430 if (srp_dev->use_fast_reg) {
3431 srp_dev->max_pages_per_mr =
3432 min_t(u32, srp_dev->max_pages_per_mr,
3433 dev_attr->max_fast_reg_page_list_len);
3434 }
52ede08f
BVA		 3435 srp_dev->mr_max_size = srp_dev->mr_page_size *
3436 srp_dev->max_pages_per_mr;
5cfb1782 3437 pr_debug("%s: mr_page_shift = %d, dev_attr->max_mr_size = %#llx, dev_attr->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
52ede08f 3438 device->name, mr_page_shift, dev_attr->max_mr_size,
5cfb1782 3439 dev_attr->max_fast_reg_page_list_len,
52ede08f 3440 srp_dev->max_pages_per_mr, srp_dev->mr_max_size);
f5358a17
RD		 3441
3442 INIT_LIST_HEAD(&srp_dev->dev_list);
3443
3444 srp_dev->dev = device;
3445 srp_dev->pd = ib_alloc_pd(device);
3446 if (IS_ERR(srp_dev->pd))
3447 goto free_dev;
3448
3449 srp_dev->mr = ib_get_dma_mr(srp_dev->pd,
3450 IB_ACCESS_LOCAL_WRITE |
3451 IB_ACCESS_REMOTE_READ |
3452 IB_ACCESS_REMOTE_WRITE);
3453 if (IS_ERR(srp_dev->mr))
3454 goto err_pd;
3455
07ebafba 3456 if (device->node_type == RDMA_NODE_IB_SWITCH) {
aef9ec39
RD		 3457 s = 0;
3458 e = 0;
3459 } else {
3460 s = 1;
3461 e = device->phys_port_cnt;
3462 }
3463
3464 for (p = s; p <= e; ++p) {
f5358a17 3465 host = srp_add_port(srp_dev, p);
aef9ec39 3466 if (host)
f5358a17 3467 list_add_tail(&host->list, &srp_dev->dev_list);
aef9ec39
RD		 3468 }
3469
f5358a17
RD		 3470 ib_set_client_data(device, &srp_client, srp_dev);
3471
3472 goto free_attr;
3473
3474err_pd:
3475 ib_dealloc_pd(srp_dev->pd);
3476
3477free_dev:
3478 kfree(srp_dev);
3479
3480free_attr:
3481 kfree(dev_attr);
aef9ec39
RD		 3482}
3483
3484static void srp_remove_one(struct ib_device *device)
3485{
f5358a17 3486 struct srp_device *srp_dev;
aef9ec39 3487 struct srp_host *host, *tmp_host;
ef6c49d8 3488 struct srp_target_port *target;
aef9ec39 3489
f5358a17 3490 srp_dev = ib_get_client_data(device, &srp_client);
1fe0cb84
DB		 3491 if (!srp_dev)
3492 return;
aef9ec39 3493
f5358a17 3494 list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
ee959b00 3495 device_unregister(&host->dev);
aef9ec39
RD		 3496 /*
3497 * Wait for the sysfs entry to go away, so that no new
3498 * target ports can be created.
3499 */
3500 wait_for_completion(&host->released);
3501
3502 /*
ef6c49d8 3503 * Remove all target ports.
aef9ec39 3504 */
b3589fd4 3505 spin_lock(&host->target_lock);
ef6c49d8
BVA		 3506 list_for_each_entry(target, &host->target_list, list)
3507 srp_queue_remove_work(target);
b3589fd4 3508 spin_unlock(&host->target_lock);
aef9ec39
RD		 3509
3510 /*
bcc05910 3511 * Wait for tl_err and target port removal tasks.
aef9ec39 3512 */
ef6c49d8 3513 flush_workqueue(system_long_wq);
bcc05910 3514 flush_workqueue(srp_remove_wq);
aef9ec39 3515
aef9ec39
RD		 3516 kfree(host);
3517 }
3518
f5358a17
RD		 3519 ib_dereg_mr(srp_dev->mr);
3520 ib_dealloc_pd(srp_dev->pd);
3521
3522 kfree(srp_dev);
aef9ec39
RD		 3523}
3524
3236822b 3525static struct srp_function_template ib_srp_transport_functions = {
ed9b2264
BVA		 3526 .has_rport_state = true,
3527 .reset_timer_if_blocked = true,
a95cadb9 3528 .reconnect_delay = &srp_reconnect_delay,
ed9b2264
BVA		 3529 .fast_io_fail_tmo = &srp_fast_io_fail_tmo,
3530 .dev_loss_tmo = &srp_dev_loss_tmo,
3531 .reconnect = srp_rport_reconnect,
dc1bdbd9 3532 .rport_delete = srp_rport_delete,
ed9b2264 3533 .terminate_rport_io = srp_terminate_io,
3236822b
FT		 3534};
3535
aef9ec39
RD		 3536static int __init srp_init_module(void)
3537{
3538 int ret;
3539
dcb4cb85 3540 BUILD_BUG_ON(FIELD_SIZEOF(struct ib_wc, wr_id) < sizeof(void *));
dd5e6e38 3541
49248644 3542 if (srp_sg_tablesize) {
e0bda7d8 3543 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
49248644
DD		 3544 if (!cmd_sg_entries)
3545 cmd_sg_entries = srp_sg_tablesize;
3546 }
3547
3548 if (!cmd_sg_entries)
3549 cmd_sg_entries = SRP_DEF_SG_TABLESIZE;
3550
3551 if (cmd_sg_entries > 255) {
e0bda7d8 3552 pr_warn("Clamping cmd_sg_entries to 255\n");
49248644 3553 cmd_sg_entries = 255;
1e89a194
DD		 3554 }
3555
c07d424d
DD		 3556 if (!indirect_sg_entries)
3557 indirect_sg_entries = cmd_sg_entries;
3558 else if (indirect_sg_entries < cmd_sg_entries) {
e0bda7d8
BVA		 3559 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n",
3560 cmd_sg_entries);
c07d424d
DD		 3561 indirect_sg_entries = cmd_sg_entries;
3562 }
3563
bcc05910 3564 srp_remove_wq = create_workqueue("srp_remove");
da05be29
WY		 3565 if (!srp_remove_wq) {
3566 ret = -ENOMEM;
bcc05910
BVA		 3567 goto out;
3568 }
3569
3570 ret = -ENOMEM;
3236822b
FT		 3571 ib_srp_transport_template =
3572 srp_attach_transport(&ib_srp_transport_functions);
3573 if (!ib_srp_transport_template)
bcc05910 3574 goto destroy_wq;
3236822b 3575
aef9ec39
RD		 3576 ret = class_register(&srp_class);
3577 if (ret) {
e0bda7d8 3578 pr_err("couldn't register class infiniband_srp\n");
bcc05910 3579 goto release_tr;
aef9ec39
RD		 3580 }
3581
c1a0b23b
MT		 3582 ib_sa_register_client(&srp_sa_client);
3583
aef9ec39
RD		 3584 ret = ib_register_client(&srp_client);
3585 if (ret) {
e0bda7d8 3586 pr_err("couldn't register IB client\n");
bcc05910 3587 goto unreg_sa;
aef9ec39
RD		 3588 }
3589
bcc05910
BVA		 3590out:
3591 return ret;
3592
3593unreg_sa:
3594 ib_sa_unregister_client(&srp_sa_client);
3595 class_unregister(&srp_class);
3596
3597release_tr:
3598 srp_release_transport(ib_srp_transport_template);
3599
3600destroy_wq:
3601 destroy_workqueue(srp_remove_wq);
3602 goto out;
aef9ec39
RD		 3603}
3604
3605static void __exit srp_cleanup_module(void)
3606{
3607 ib_unregister_client(&srp_client);
c1a0b23b 3608 ib_sa_unregister_client(&srp_sa_client);
aef9ec39 3609 class_unregister(&srp_class);
3236822b 3610 srp_release_transport(ib_srp_transport_template);
bcc05910 3611 destroy_workqueue(srp_remove_wq);
aef9ec39
RD		 3612}
3613
3614module_init(srp_init_module);
3615module_exit(srp_cleanup_module);
ubuntu-bionic-kernel mirror