]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/infiniband/ulp/srp/ib_srp.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
IB/srp: Remove the memory registration backtracking code
[mirror_ubuntu-artful-kernel.git] / drivers / infiniband / ulp / srp / ib_srp.c
CommitLineData
aef9ec39
RD		 1/*
2 * Copyright (c) 2005 Cisco Systems. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
aef9ec39
RD		 31 */
32
d236cd0e 33#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
e0bda7d8 34
aef9ec39
RD		 35#include <linux/module.h>
36#include <linux/init.h>
37#include <linux/slab.h>
38#include <linux/err.h>
39#include <linux/string.h>
40#include <linux/parser.h>
41#include <linux/random.h>
de25968c 42#include <linux/jiffies.h>
56b5390c 43#include <rdma/ib_cache.h>
aef9ec39 44
60063497 45#include <linux/atomic.h>
aef9ec39
RD		 46
47#include <scsi/scsi.h>
48#include <scsi/scsi_device.h>
49#include <scsi/scsi_dbg.h>
71444b97 50#include <scsi/scsi_tcq.h>
aef9ec39 51#include <scsi/srp.h>
3236822b 52#include <scsi/scsi_transport_srp.h>
aef9ec39 53
aef9ec39
RD		 54#include "ib_srp.h"
55
56#define DRV_NAME "ib_srp"
57#define PFX DRV_NAME ": "
713ef24e
BVA		 58#define DRV_VERSION "2.0"
59#define DRV_RELDATE "July 26, 2015"
aef9ec39
RD		 60
61MODULE_AUTHOR("Roland Dreier");
33ab3e5b 62MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator");
aef9ec39 63MODULE_LICENSE("Dual BSD/GPL");
33ab3e5b
BVA		 64MODULE_VERSION(DRV_VERSION);
65MODULE_INFO(release_date, DRV_RELDATE);
aef9ec39 66
49248644
DD		 67static unsigned int srp_sg_tablesize;
68static unsigned int cmd_sg_entries;
c07d424d
DD		 69static unsigned int indirect_sg_entries;
70static bool allow_ext_sg;
5cfb1782 71static bool prefer_fr;
b1b8854d 72static bool register_always;
49248644 73static int topspin_workarounds = 1;
74b0a15b 74
49248644
DD		 75module_param(srp_sg_tablesize, uint, 0444);
76MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries");
74b0a15b 77
49248644
DD		 78module_param(cmd_sg_entries, uint, 0444);
79MODULE_PARM_DESC(cmd_sg_entries,
80 "Default number of gather/scatter entries in the SRP command (default is 12, max 255)");
aef9ec39 81
c07d424d
DD		 82module_param(indirect_sg_entries, uint, 0444);
83MODULE_PARM_DESC(indirect_sg_entries,
84 "Default max number of gather/scatter entries (default is 12, max is " __stringify(SCSI_MAX_SG_CHAIN_SEGMENTS) ")");
85
86module_param(allow_ext_sg, bool, 0444);
87MODULE_PARM_DESC(allow_ext_sg,
88 "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)");
89
aef9ec39
RD		 90module_param(topspin_workarounds, int, 0444);
91MODULE_PARM_DESC(topspin_workarounds,
92 "Enable workarounds for Topspin/Cisco SRP target bugs if != 0");
93
5cfb1782
BVA		 94module_param(prefer_fr, bool, 0444);
95MODULE_PARM_DESC(prefer_fr,
96"Whether to use fast registration if both FMR and fast registration are supported");
97
b1b8854d
BVA		 98module_param(register_always, bool, 0444);
99MODULE_PARM_DESC(register_always,
100 "Use memory registration even for contiguous memory regions");
101
9c27847d 102static const struct kernel_param_ops srp_tmo_ops;
ed9b2264 103
a95cadb9
BVA		 104static int srp_reconnect_delay = 10;
105module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay,
106 S_IRUGO | S_IWUSR);
107MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts");
108
ed9b2264
BVA		 109static int srp_fast_io_fail_tmo = 15;
110module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo,
111 S_IRUGO | S_IWUSR);
112MODULE_PARM_DESC(fast_io_fail_tmo,
113 "Number of seconds between the observation of a transport"
114 " layer error and failing all I/O. \"off\" means that this"
115 " functionality is disabled.");
116
a95cadb9 117static int srp_dev_loss_tmo = 600;
ed9b2264
BVA		 118module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo,
119 S_IRUGO | S_IWUSR);
120MODULE_PARM_DESC(dev_loss_tmo,
121 "Maximum number of seconds that the SRP transport should"
122 " insulate transport layer errors. After this time has been"
123 " exceeded the SCSI host is removed. Should be"
124 " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT)
125 " if fast_io_fail_tmo has not been set. \"off\" means that"
126 " this functionality is disabled.");
127
d92c0da7
BVA		 128static unsigned ch_count;
129module_param(ch_count, uint, 0444);
130MODULE_PARM_DESC(ch_count,
131 "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA.");
132
aef9ec39 133static void srp_add_one(struct ib_device *device);
7c1eb45a 134static void srp_remove_one(struct ib_device *device, void *client_data);
509c07bc
BVA		 135static void srp_recv_completion(struct ib_cq *cq, void *ch_ptr);
136static void srp_send_completion(struct ib_cq *cq, void *ch_ptr);
aef9ec39
RD		 137static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
138
3236822b 139static struct scsi_transport_template *ib_srp_transport_template;
bcc05910 140static struct workqueue_struct *srp_remove_wq;
3236822b 141
aef9ec39
RD		 142static struct ib_client srp_client = {
143 .name = "srp",
144 .add = srp_add_one,
145 .remove = srp_remove_one
146};
147
c1a0b23b
MT		 148static struct ib_sa_client srp_sa_client;
149
ed9b2264
BVA		 150static int srp_tmo_get(char *buffer, const struct kernel_param *kp)
151{
152 int tmo = *(int *)kp->arg;
153
154 if (tmo >= 0)
155 return sprintf(buffer, "%d", tmo);
156 else
157 return sprintf(buffer, "off");
158}
159
160static int srp_tmo_set(const char *val, const struct kernel_param *kp)
161{
162 int tmo, res;
163
3fdf70ac
SG		 164 res = srp_parse_tmo(&tmo, val);
165 if (res)
166 goto out;
167
a95cadb9
BVA		 168 if (kp->arg == &srp_reconnect_delay)
169 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo,
170 srp_dev_loss_tmo);
171 else if (kp->arg == &srp_fast_io_fail_tmo)
172 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo);
ed9b2264 173 else
a95cadb9
BVA		 174 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo,
175 tmo);
ed9b2264
BVA		 176 if (res)
177 goto out;
178 *(int *)kp->arg = tmo;
179
180out:
181 return res;
182}
183
9c27847d 184static const struct kernel_param_ops srp_tmo_ops = {
ed9b2264
BVA		 185 .get = srp_tmo_get,
186 .set = srp_tmo_set,
187};
188
aef9ec39
RD		 189static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
190{
191 return (struct srp_target_port *) host->hostdata;
192}
193
194static const char *srp_target_info(struct Scsi_Host *host)
195{
196 return host_to_target(host)->target_name;
197}
198
5d7cbfd6
RD		 199static int srp_target_is_topspin(struct srp_target_port *target)
200{
201 static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad };
3d1ff48d 202 static const u8 cisco_oui[3] = { 0x00, 0x1b, 0x0d };
5d7cbfd6
RD		 203
204 return topspin_workarounds &&
3d1ff48d
RK		 205 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) ||
206 !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui));
5d7cbfd6
RD		 207}
208
aef9ec39
RD		 209static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size,
210 gfp_t gfp_mask,
211 enum dma_data_direction direction)
212{
213 struct srp_iu *iu;
214
215 iu = kmalloc(sizeof *iu, gfp_mask);
216 if (!iu)
217 goto out;
218
219 iu->buf = kzalloc(size, gfp_mask);
220 if (!iu->buf)
221 goto out_free_iu;
222
05321937
GKH		 223 iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size,
224 direction);
225 if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma))
aef9ec39
RD		 226 goto out_free_buf;
227
228 iu->size = size;
229 iu->direction = direction;
230
231 return iu;
232
233out_free_buf:
234 kfree(iu->buf);
235out_free_iu:
236 kfree(iu);
237out:
238 return NULL;
239}
240
241static void srp_free_iu(struct srp_host *host, struct srp_iu *iu)
242{
243 if (!iu)
244 return;
245
05321937
GKH		 246 ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size,
247 iu->direction);
aef9ec39
RD		 248 kfree(iu->buf);
249 kfree(iu);
250}
251
252static void srp_qp_event(struct ib_event *event, void *context)
253{
57363d98
SG		 254 pr_debug("QP event %s (%d)\n",
255 ib_event_msg(event->event), event->event);
aef9ec39
RD		 256}
257
258static int srp_init_qp(struct srp_target_port *target,
259 struct ib_qp *qp)
260{
261 struct ib_qp_attr *attr;
262 int ret;
263
264 attr = kmalloc(sizeof *attr, GFP_KERNEL);
265 if (!attr)
266 return -ENOMEM;
267
56b5390c
BVA		 268 ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev,
269 target->srp_host->port,
270 be16_to_cpu(target->pkey),
271 &attr->pkey_index);
aef9ec39
RD		 272 if (ret)
273 goto out;
274
275 attr->qp_state = IB_QPS_INIT;
276 attr->qp_access_flags = (IB_ACCESS_REMOTE_READ |
277 IB_ACCESS_REMOTE_WRITE);
278 attr->port_num = target->srp_host->port;
279
280 ret = ib_modify_qp(qp, attr,
281 IB_QP_STATE |
282 IB_QP_PKEY_INDEX |
283 IB_QP_ACCESS_FLAGS |
284 IB_QP_PORT);
285
286out:
287 kfree(attr);
288 return ret;
289}
290
509c07bc 291static int srp_new_cm_id(struct srp_rdma_ch *ch)
9fe4bcf4 292{
509c07bc 293 struct srp_target_port *target = ch->target;
9fe4bcf4
DD		 294 struct ib_cm_id *new_cm_id;
295
05321937 296 new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
509c07bc 297 srp_cm_handler, ch);
9fe4bcf4
DD		 298 if (IS_ERR(new_cm_id))
299 return PTR_ERR(new_cm_id);
300
509c07bc
BVA		 301 if (ch->cm_id)
302 ib_destroy_cm_id(ch->cm_id);
303 ch->cm_id = new_cm_id;
304 ch->path.sgid = target->sgid;
305 ch->path.dgid = target->orig_dgid;
306 ch->path.pkey = target->pkey;
307 ch->path.service_id = target->service_id;
9fe4bcf4
DD		 308
309 return 0;
310}
311
d1b4289e
BVA		 312static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target)
313{
314 struct srp_device *dev = target->srp_host->srp_dev;
315 struct ib_fmr_pool_param fmr_param;
316
317 memset(&fmr_param, 0, sizeof(fmr_param));
318 fmr_param.pool_size = target->scsi_host->can_queue;
319 fmr_param.dirty_watermark = fmr_param.pool_size / 4;
320 fmr_param.cache = 1;
52ede08f
BVA		 321 fmr_param.max_pages_per_fmr = dev->max_pages_per_mr;
322 fmr_param.page_shift = ilog2(dev->mr_page_size);
d1b4289e
BVA		 323 fmr_param.access = (IB_ACCESS_LOCAL_WRITE |
324 IB_ACCESS_REMOTE_WRITE |
325 IB_ACCESS_REMOTE_READ);
326
327 return ib_create_fmr_pool(dev->pd, &fmr_param);
328}
329
5cfb1782
BVA		 330/**
331 * srp_destroy_fr_pool() - free the resources owned by a pool
332 * @pool: Fast registration pool to be destroyed.
333 */
334static void srp_destroy_fr_pool(struct srp_fr_pool *pool)
335{
336 int i;
337 struct srp_fr_desc *d;
338
339 if (!pool)
340 return;
341
342 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
343 if (d->frpl)
344 ib_free_fast_reg_page_list(d->frpl);
345 if (d->mr)
346 ib_dereg_mr(d->mr);
347 }
348 kfree(pool);
349}
350
351/**
352 * srp_create_fr_pool() - allocate and initialize a pool for fast registration
353 * @device: IB device to allocate fast registration descriptors for.
354 * @pd: Protection domain associated with the FR descriptors.
355 * @pool_size: Number of descriptors to allocate.
356 * @max_page_list_len: Maximum fast registration work request page list length.
357 */
358static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
359 struct ib_pd *pd, int pool_size,
360 int max_page_list_len)
361{
362 struct srp_fr_pool *pool;
363 struct srp_fr_desc *d;
364 struct ib_mr *mr;
365 struct ib_fast_reg_page_list *frpl;
366 int i, ret = -EINVAL;
367
368 if (pool_size <= 0)
369 goto err;
370 ret = -ENOMEM;
371 pool = kzalloc(sizeof(struct srp_fr_pool) +
372 pool_size * sizeof(struct srp_fr_desc), GFP_KERNEL);
373 if (!pool)
374 goto err;
375 pool->size = pool_size;
376 pool->max_page_list_len = max_page_list_len;
377 spin_lock_init(&pool->lock);
378 INIT_LIST_HEAD(&pool->free_list);
379
380 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
563b67c5
SG		 381 mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG,
382 max_page_list_len);
5cfb1782
BVA		 383 if (IS_ERR(mr)) {
384 ret = PTR_ERR(mr);
385 goto destroy_pool;
386 }
387 d->mr = mr;
388 frpl = ib_alloc_fast_reg_page_list(device, max_page_list_len);
389 if (IS_ERR(frpl)) {
390 ret = PTR_ERR(frpl);
391 goto destroy_pool;
392 }
393 d->frpl = frpl;
394 list_add_tail(&d->entry, &pool->free_list);
395 }
396
397out:
398 return pool;
399
400destroy_pool:
401 srp_destroy_fr_pool(pool);
402
403err:
404 pool = ERR_PTR(ret);
405 goto out;
406}
407
408/**
409 * srp_fr_pool_get() - obtain a descriptor suitable for fast registration
410 * @pool: Pool to obtain descriptor from.
411 */
412static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool)
413{
414 struct srp_fr_desc *d = NULL;
415 unsigned long flags;
416
417 spin_lock_irqsave(&pool->lock, flags);
418 if (!list_empty(&pool->free_list)) {
419 d = list_first_entry(&pool->free_list, typeof(*d), entry);
420 list_del(&d->entry);
421 }
422 spin_unlock_irqrestore(&pool->lock, flags);
423
424 return d;
425}
426
427/**
428 * srp_fr_pool_put() - put an FR descriptor back in the free list
429 * @pool: Pool the descriptor was allocated from.
430 * @desc: Pointer to an array of fast registration descriptor pointers.
431 * @n: Number of descriptors to put back.
432 *
433 * Note: The caller must already have queued an invalidation request for
434 * desc->mr->rkey before calling this function.
435 */
436static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc,
437 int n)
438{
439 unsigned long flags;
440 int i;
441
442 spin_lock_irqsave(&pool->lock, flags);
443 for (i = 0; i < n; i++)
444 list_add(&desc[i]->entry, &pool->free_list);
445 spin_unlock_irqrestore(&pool->lock, flags);
446}
447
448static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
449{
450 struct srp_device *dev = target->srp_host->srp_dev;
451
452 return srp_create_fr_pool(dev->dev, dev->pd,
453 target->scsi_host->can_queue,
454 dev->max_pages_per_mr);
455}
456
7dad6b2e
BVA		 457/**
458 * srp_destroy_qp() - destroy an RDMA queue pair
459 * @ch: SRP RDMA channel.
460 *
461 * Change a queue pair into the error state and wait until all receive
462 * completions have been processed before destroying it. This avoids that
463 * the receive completion handler can access the queue pair while it is
464 * being destroyed.
465 */
466static void srp_destroy_qp(struct srp_rdma_ch *ch)
467{
7dad6b2e
BVA		 468 static struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
469 static struct ib_recv_wr wr = { .wr_id = SRP_LAST_WR_ID };
470 struct ib_recv_wr *bad_wr;
471 int ret;
472
473 /* Destroying a QP and reusing ch->done is only safe if not connected */
c014c8cd 474 WARN_ON_ONCE(ch->connected);
7dad6b2e
BVA		 475
476 ret = ib_modify_qp(ch->qp, &attr, IB_QP_STATE);
477 WARN_ONCE(ret, "ib_cm_init_qp_attr() returned %d\n", ret);
478 if (ret)
479 goto out;
480
481 init_completion(&ch->done);
482 ret = ib_post_recv(ch->qp, &wr, &bad_wr);
483 WARN_ONCE(ret, "ib_post_recv() returned %d\n", ret);
484 if (ret == 0)
485 wait_for_completion(&ch->done);
486
487out:
488 ib_destroy_qp(ch->qp);
489}
490
509c07bc 491static int srp_create_ch_ib(struct srp_rdma_ch *ch)
aef9ec39 492{
509c07bc 493 struct srp_target_port *target = ch->target;
62154b2e 494 struct srp_device *dev = target->srp_host->srp_dev;
aef9ec39 495 struct ib_qp_init_attr *init_attr;
73aa89ed
IR		 496 struct ib_cq *recv_cq, *send_cq;
497 struct ib_qp *qp;
d1b4289e 498 struct ib_fmr_pool *fmr_pool = NULL;
5cfb1782
BVA		 499 struct srp_fr_pool *fr_pool = NULL;
500 const int m = 1 + dev->use_fast_reg;
8e37210b 501 struct ib_cq_init_attr cq_attr = {};
aef9ec39
RD		 502 int ret;
503
504 init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
505 if (!init_attr)
506 return -ENOMEM;
507
7dad6b2e 508 /* + 1 for SRP_LAST_WR_ID */
8e37210b
MB		 509 cq_attr.cqe = target->queue_size + 1;
510 cq_attr.comp_vector = ch->comp_vector;
509c07bc 511 recv_cq = ib_create_cq(dev->dev, srp_recv_completion, NULL, ch,
8e37210b 512 &cq_attr);
73aa89ed
IR		 513 if (IS_ERR(recv_cq)) {
514 ret = PTR_ERR(recv_cq);
da9d2f07 515 goto err;
aef9ec39
RD		 516 }
517
8e37210b
MB		 518 cq_attr.cqe = m * target->queue_size;
519 cq_attr.comp_vector = ch->comp_vector;
509c07bc 520 send_cq = ib_create_cq(dev->dev, srp_send_completion, NULL, ch,
8e37210b 521 &cq_attr);
73aa89ed
IR		 522 if (IS_ERR(send_cq)) {
523 ret = PTR_ERR(send_cq);
da9d2f07 524 goto err_recv_cq;
9c03dc9f
BVA		 525 }
526
73aa89ed 527 ib_req_notify_cq(recv_cq, IB_CQ_NEXT_COMP);
aef9ec39
RD		 528
529 init_attr->event_handler = srp_qp_event;
5cfb1782 530 init_attr->cap.max_send_wr = m * target->queue_size;
7dad6b2e 531 init_attr->cap.max_recv_wr = target->queue_size + 1;
aef9ec39
RD		 532 init_attr->cap.max_recv_sge = 1;
533 init_attr->cap.max_send_sge = 1;
5cfb1782 534 init_attr->sq_sig_type = IB_SIGNAL_REQ_WR;
aef9ec39 535 init_attr->qp_type = IB_QPT_RC;
73aa89ed
IR		 536 init_attr->send_cq = send_cq;
537 init_attr->recv_cq = recv_cq;
aef9ec39 538
62154b2e 539 qp = ib_create_qp(dev->pd, init_attr);
73aa89ed
IR		 540 if (IS_ERR(qp)) {
541 ret = PTR_ERR(qp);
da9d2f07 542 goto err_send_cq;
aef9ec39
RD		 543 }
544
73aa89ed 545 ret = srp_init_qp(target, qp);
da9d2f07
RD		 546 if (ret)
547 goto err_qp;
aef9ec39 548
5cfb1782
BVA		 549 if (dev->use_fast_reg && dev->has_fr) {
550 fr_pool = srp_alloc_fr_pool(target);
551 if (IS_ERR(fr_pool)) {
552 ret = PTR_ERR(fr_pool);
553 shost_printk(KERN_WARNING, target->scsi_host, PFX
554 "FR pool allocation failed (%d)\n", ret);
555 goto err_qp;
556 }
509c07bc
BVA		 557 if (ch->fr_pool)
558 srp_destroy_fr_pool(ch->fr_pool);
559 ch->fr_pool = fr_pool;
5cfb1782 560 } else if (!dev->use_fast_reg && dev->has_fmr) {
d1b4289e
BVA		 561 fmr_pool = srp_alloc_fmr_pool(target);
562 if (IS_ERR(fmr_pool)) {
563 ret = PTR_ERR(fmr_pool);
564 shost_printk(KERN_WARNING, target->scsi_host, PFX
565 "FMR pool allocation failed (%d)\n", ret);
566 goto err_qp;
567 }
509c07bc
BVA		 568 if (ch->fmr_pool)
569 ib_destroy_fmr_pool(ch->fmr_pool);
570 ch->fmr_pool = fmr_pool;
d1b4289e
BVA		 571 }
572
509c07bc 573 if (ch->qp)
7dad6b2e 574 srp_destroy_qp(ch);
509c07bc
BVA		 575 if (ch->recv_cq)
576 ib_destroy_cq(ch->recv_cq);
577 if (ch->send_cq)
578 ib_destroy_cq(ch->send_cq);
73aa89ed 579
509c07bc
BVA		 580 ch->qp = qp;
581 ch->recv_cq = recv_cq;
582 ch->send_cq = send_cq;
73aa89ed 583
da9d2f07
RD		 584 kfree(init_attr);
585 return 0;
586
587err_qp:
73aa89ed 588 ib_destroy_qp(qp);
da9d2f07
RD		 589
590err_send_cq:
73aa89ed 591 ib_destroy_cq(send_cq);
da9d2f07
RD		 592
593err_recv_cq:
73aa89ed 594 ib_destroy_cq(recv_cq);
da9d2f07
RD		 595
596err:
aef9ec39
RD		 597 kfree(init_attr);
598 return ret;
599}
600
4d73f95f
BVA		 601/*
602 * Note: this function may be called without srp_alloc_iu_bufs() having been
509c07bc 603 * invoked. Hence the ch->[rt]x_ring checks.
4d73f95f 604 */
509c07bc
BVA		 605static void srp_free_ch_ib(struct srp_target_port *target,
606 struct srp_rdma_ch *ch)
aef9ec39 607{
5cfb1782 608 struct srp_device *dev = target->srp_host->srp_dev;
aef9ec39
RD		 609 int i;
610
d92c0da7
BVA		 611 if (!ch->target)
612 return;
613
509c07bc
BVA		 614 if (ch->cm_id) {
615 ib_destroy_cm_id(ch->cm_id);
616 ch->cm_id = NULL;
394c595e
BVA		 617 }
618
d92c0da7
BVA		 619 /* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */
620 if (!ch->qp)
621 return;
622
5cfb1782 623 if (dev->use_fast_reg) {
509c07bc
BVA		 624 if (ch->fr_pool)
625 srp_destroy_fr_pool(ch->fr_pool);
5cfb1782 626 } else {
509c07bc
BVA		 627 if (ch->fmr_pool)
628 ib_destroy_fmr_pool(ch->fmr_pool);
5cfb1782 629 }
7dad6b2e 630 srp_destroy_qp(ch);
509c07bc
BVA		 631 ib_destroy_cq(ch->send_cq);
632 ib_destroy_cq(ch->recv_cq);
aef9ec39 633
d92c0da7
BVA		 634 /*
635 * Avoid that the SCSI error handler tries to use this channel after
636 * it has been freed. The SCSI error handler can namely continue
637 * trying to perform recovery actions after scsi_remove_host()
638 * returned.
639 */
640 ch->target = NULL;
641
509c07bc
BVA		 642 ch->qp = NULL;
643 ch->send_cq = ch->recv_cq = NULL;
73aa89ed 644
509c07bc 645 if (ch->rx_ring) {
4d73f95f 646 for (i = 0; i < target->queue_size; ++i)
509c07bc
BVA		 647 srp_free_iu(target->srp_host, ch->rx_ring[i]);
648 kfree(ch->rx_ring);
649 ch->rx_ring = NULL;
4d73f95f 650 }
509c07bc 651 if (ch->tx_ring) {
4d73f95f 652 for (i = 0; i < target->queue_size; ++i)
509c07bc
BVA		 653 srp_free_iu(target->srp_host, ch->tx_ring[i]);
654 kfree(ch->tx_ring);
655 ch->tx_ring = NULL;
4d73f95f 656 }
aef9ec39
RD		 657}
658
659static void srp_path_rec_completion(int status,
660 struct ib_sa_path_rec *pathrec,
509c07bc 661 void *ch_ptr)
aef9ec39 662{
509c07bc
BVA		 663 struct srp_rdma_ch *ch = ch_ptr;
664 struct srp_target_port *target = ch->target;
aef9ec39 665
509c07bc 666 ch->status = status;
aef9ec39 667 if (status)
7aa54bd7
DD		 668 shost_printk(KERN_ERR, target->scsi_host,
669 PFX "Got failed path rec status %d\n", status);
aef9ec39 670 else
509c07bc
BVA		 671 ch->path = *pathrec;
672 complete(&ch->done);
aef9ec39
RD		 673}
674
509c07bc 675static int srp_lookup_path(struct srp_rdma_ch *ch)
aef9ec39 676{
509c07bc 677 struct srp_target_port *target = ch->target;
a702adce
BVA		 678 int ret;
679
509c07bc
BVA		 680 ch->path.numb_path = 1;
681
682 init_completion(&ch->done);
683
684 ch->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
685 target->srp_host->srp_dev->dev,
686 target->srp_host->port,
687 &ch->path,
688 IB_SA_PATH_REC_SERVICE_ID |
689 IB_SA_PATH_REC_DGID |
690 IB_SA_PATH_REC_SGID |
691 IB_SA_PATH_REC_NUMB_PATH |
692 IB_SA_PATH_REC_PKEY,
693 SRP_PATH_REC_TIMEOUT_MS,
694 GFP_KERNEL,
695 srp_path_rec_completion,
696 ch, &ch->path_query);
697 if (ch->path_query_id < 0)
698 return ch->path_query_id;
699
700 ret = wait_for_completion_interruptible(&ch->done);
a702adce
BVA		 701 if (ret < 0)
702 return ret;
aef9ec39 703
509c07bc 704 if (ch->status < 0)
7aa54bd7
DD		 705 shost_printk(KERN_WARNING, target->scsi_host,
706 PFX "Path record query failed\n");
aef9ec39 707
509c07bc 708 return ch->status;
aef9ec39
RD		 709}
710
d92c0da7 711static int srp_send_req(struct srp_rdma_ch *ch, bool multich)
aef9ec39 712{
509c07bc 713 struct srp_target_port *target = ch->target;
aef9ec39
RD		 714 struct {
715 struct ib_cm_req_param param;
716 struct srp_login_req priv;
717 } *req = NULL;
718 int status;
719
720 req = kzalloc(sizeof *req, GFP_KERNEL);
721 if (!req)
722 return -ENOMEM;
723
509c07bc 724 req->param.primary_path = &ch->path;
aef9ec39
RD		 725 req->param.alternate_path = NULL;
726 req->param.service_id = target->service_id;
509c07bc
BVA		 727 req->param.qp_num = ch->qp->qp_num;
728 req->param.qp_type = ch->qp->qp_type;
aef9ec39
RD		 729 req->param.private_data = &req->priv;
730 req->param.private_data_len = sizeof req->priv;
731 req->param.flow_control = 1;
732
733 get_random_bytes(&req->param.starting_psn, 4);
734 req->param.starting_psn &= 0xffffff;
735
736 /*
737 * Pick some arbitrary defaults here; we could make these
738 * module parameters if anyone cared about setting them.
739 */
740 req->param.responder_resources = 4;
741 req->param.remote_cm_response_timeout = 20;
742 req->param.local_cm_response_timeout = 20;
7bb312e4 743 req->param.retry_count = target->tl_retry_count;
aef9ec39
RD		 744 req->param.rnr_retry_count = 7;
745 req->param.max_cm_retries = 15;
746
747 req->priv.opcode = SRP_LOGIN_REQ;
748 req->priv.tag = 0;
49248644 749 req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len);
aef9ec39
RD		 750 req->priv.req_buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
751 SRP_BUF_FORMAT_INDIRECT);
d92c0da7
BVA		 752 req->priv.req_flags = (multich ? SRP_MULTICHAN_MULTI :
753 SRP_MULTICHAN_SINGLE);
0c0450db 754 /*
3cd96564 755 * In the published SRP specification (draft rev. 16a), the
0c0450db
R		 756 * port identifier format is 8 bytes of ID extension followed
757 * by 8 bytes of GUID. Older drafts put the two halves in the
758 * opposite order, so that the GUID comes first.
759 *
760 * Targets conforming to these obsolete drafts can be
761 * recognized by the I/O Class they report.
762 */
763 if (target->io_class == SRP_REV10_IB_IO_CLASS) {
764 memcpy(req->priv.initiator_port_id,
747fe000 765 &target->sgid.global.interface_id, 8);
0c0450db 766 memcpy(req->priv.initiator_port_id + 8,
01cb9bcb 767 &target->initiator_ext, 8);
0c0450db
R		 768 memcpy(req->priv.target_port_id, &target->ioc_guid, 8);
769 memcpy(req->priv.target_port_id + 8, &target->id_ext, 8);
770 } else {
771 memcpy(req->priv.initiator_port_id,
01cb9bcb
IR		 772 &target->initiator_ext, 8);
773 memcpy(req->priv.initiator_port_id + 8,
747fe000 774 &target->sgid.global.interface_id, 8);
0c0450db
R		 775 memcpy(req->priv.target_port_id, &target->id_ext, 8);
776 memcpy(req->priv.target_port_id + 8, &target->ioc_guid, 8);
777 }
778
aef9ec39
RD		 779 /*
780 * Topspin/Cisco SRP targets will reject our login unless we
01cb9bcb
IR		 781 * zero out the first 8 bytes of our initiator port ID and set
782 * the second 8 bytes to the local node GUID.
aef9ec39 783 */
5d7cbfd6 784 if (srp_target_is_topspin(target)) {
7aa54bd7
DD		 785 shost_printk(KERN_DEBUG, target->scsi_host,
786 PFX "Topspin/Cisco initiator port ID workaround "
787 "activated for target GUID %016llx\n",
45c37cad 788 be64_to_cpu(target->ioc_guid));
aef9ec39 789 memset(req->priv.initiator_port_id, 0, 8);
01cb9bcb 790 memcpy(req->priv.initiator_port_id + 8,
05321937 791 &target->srp_host->srp_dev->dev->node_guid, 8);
aef9ec39 792 }
aef9ec39 793
509c07bc 794 status = ib_send_cm_req(ch->cm_id, &req->param);
aef9ec39
RD		 795
796 kfree(req);
797
798 return status;
799}
800
ef6c49d8
BVA		 801static bool srp_queue_remove_work(struct srp_target_port *target)
802{
803 bool changed = false;
804
805 spin_lock_irq(&target->lock);
806 if (target->state != SRP_TARGET_REMOVED) {
807 target->state = SRP_TARGET_REMOVED;
808 changed = true;
809 }
810 spin_unlock_irq(&target->lock);
811
812 if (changed)
bcc05910 813 queue_work(srp_remove_wq, &target->remove_work);
ef6c49d8
BVA		 814
815 return changed;
816}
817
aef9ec39
RD		 818static void srp_disconnect_target(struct srp_target_port *target)
819{
d92c0da7
BVA		 820 struct srp_rdma_ch *ch;
821 int i;
509c07bc 822
c014c8cd 823 /* XXX should send SRP_I_LOGOUT request */
aef9ec39 824
c014c8cd
BVA		 825 for (i = 0; i < target->ch_count; i++) {
826 ch = &target->ch[i];
827 ch->connected = false;
828 if (ch->cm_id && ib_send_cm_dreq(ch->cm_id, NULL, 0)) {
829 shost_printk(KERN_DEBUG, target->scsi_host,
830 PFX "Sending CM DREQ failed\n");
294c875a 831 }
e6581056 832 }
aef9ec39
RD		 833}
834
509c07bc
BVA		 835static void srp_free_req_data(struct srp_target_port *target,
836 struct srp_rdma_ch *ch)
8f26c9ff 837{
5cfb1782
BVA		 838 struct srp_device *dev = target->srp_host->srp_dev;
839 struct ib_device *ibdev = dev->dev;
8f26c9ff
DD		 840 struct srp_request *req;
841 int i;
842
47513cf4 843 if (!ch->req_ring)
4d73f95f
BVA		 844 return;
845
846 for (i = 0; i < target->req_ring_size; ++i) {
509c07bc 847 req = &ch->req_ring[i];
5cfb1782
BVA		 848 if (dev->use_fast_reg)
849 kfree(req->fr_list);
850 else
851 kfree(req->fmr_list);
8f26c9ff 852 kfree(req->map_page);
c07d424d
DD		 853 if (req->indirect_dma_addr) {
854 ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
855 target->indirect_size,
856 DMA_TO_DEVICE);
857 }
858 kfree(req->indirect_desc);
8f26c9ff 859 }
4d73f95f 860
509c07bc
BVA		 861 kfree(ch->req_ring);
862 ch->req_ring = NULL;
8f26c9ff
DD		 863}
864
509c07bc 865static int srp_alloc_req_data(struct srp_rdma_ch *ch)
b81d00bd 866{
509c07bc 867 struct srp_target_port *target = ch->target;
b81d00bd
BVA		 868 struct srp_device *srp_dev = target->srp_host->srp_dev;
869 struct ib_device *ibdev = srp_dev->dev;
870 struct srp_request *req;
5cfb1782 871 void *mr_list;
b81d00bd
BVA		 872 dma_addr_t dma_addr;
873 int i, ret = -ENOMEM;
874
509c07bc
BVA		 875 ch->req_ring = kcalloc(target->req_ring_size, sizeof(*ch->req_ring),
876 GFP_KERNEL);
877 if (!ch->req_ring)
4d73f95f
BVA		 878 goto out;
879
880 for (i = 0; i < target->req_ring_size; ++i) {
509c07bc 881 req = &ch->req_ring[i];
5cfb1782
BVA		 882 mr_list = kmalloc(target->cmd_sg_cnt * sizeof(void *),
883 GFP_KERNEL);
884 if (!mr_list)
885 goto out;
886 if (srp_dev->use_fast_reg)
887 req->fr_list = mr_list;
888 else
889 req->fmr_list = mr_list;
52ede08f 890 req->map_page = kmalloc(srp_dev->max_pages_per_mr *
d1b4289e 891 sizeof(void *), GFP_KERNEL);
5cfb1782
BVA		 892 if (!req->map_page)
893 goto out;
b81d00bd 894 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
5cfb1782 895 if (!req->indirect_desc)
b81d00bd
BVA		 896 goto out;
897
898 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
899 target->indirect_size,
900 DMA_TO_DEVICE);
901 if (ib_dma_mapping_error(ibdev, dma_addr))
902 goto out;
903
904 req->indirect_dma_addr = dma_addr;
b81d00bd
BVA		 905 }
906 ret = 0;
907
908out:
909 return ret;
910}
911
683b159a
BVA		 912/**
913 * srp_del_scsi_host_attr() - Remove attributes defined in the host template.
914 * @shost: SCSI host whose attributes to remove from sysfs.
915 *
916 * Note: Any attributes defined in the host template and that did not exist
917 * before invocation of this function will be ignored.
918 */
919static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
920{
921 struct device_attribute **attr;
922
923 for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr)
924 device_remove_file(&shost->shost_dev, *attr);
925}
926
ee12d6a8
BVA		 927static void srp_remove_target(struct srp_target_port *target)
928{
d92c0da7
BVA		 929 struct srp_rdma_ch *ch;
930 int i;
509c07bc 931
ef6c49d8
BVA		 932 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
933
ee12d6a8 934 srp_del_scsi_host_attr(target->scsi_host);
9dd69a60 935 srp_rport_get(target->rport);
ee12d6a8
BVA		 936 srp_remove_host(target->scsi_host);
937 scsi_remove_host(target->scsi_host);
93079162 938 srp_stop_rport_timers(target->rport);
ef6c49d8 939 srp_disconnect_target(target);
d92c0da7
BVA		 940 for (i = 0; i < target->ch_count; i++) {
941 ch = &target->ch[i];
942 srp_free_ch_ib(target, ch);
943 }
c1120f89 944 cancel_work_sync(&target->tl_err_work);
9dd69a60 945 srp_rport_put(target->rport);
d92c0da7
BVA		 946 for (i = 0; i < target->ch_count; i++) {
947 ch = &target->ch[i];
948 srp_free_req_data(target, ch);
949 }
950 kfree(target->ch);
951 target->ch = NULL;
65d7dd2f
VP		 952
953 spin_lock(&target->srp_host->target_lock);
954 list_del(&target->list);
955 spin_unlock(&target->srp_host->target_lock);
956
ee12d6a8
BVA		 957 scsi_host_put(target->scsi_host);
958}
959
c4028958 960static void srp_remove_work(struct work_struct *work)
aef9ec39 961{
c4028958 962 struct srp_target_port *target =
ef6c49d8 963 container_of(work, struct srp_target_port, remove_work);
aef9ec39 964
ef6c49d8 965 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
aef9ec39 966
96fc248a 967 srp_remove_target(target);
aef9ec39
RD		 968}
969
dc1bdbd9
BVA		 970static void srp_rport_delete(struct srp_rport *rport)
971{
972 struct srp_target_port *target = rport->lld_data;
973
974 srp_queue_remove_work(target);
975}
976
c014c8cd
BVA		 977/**
978 * srp_connected_ch() - number of connected channels
979 * @target: SRP target port.
980 */
981static int srp_connected_ch(struct srp_target_port *target)
982{
983 int i, c = 0;
984
985 for (i = 0; i < target->ch_count; i++)
986 c += target->ch[i].connected;
987
988 return c;
989}
990
d92c0da7 991static int srp_connect_ch(struct srp_rdma_ch *ch, bool multich)
aef9ec39 992{
509c07bc 993 struct srp_target_port *target = ch->target;
aef9ec39
RD		 994 int ret;
995
c014c8cd 996 WARN_ON_ONCE(!multich && srp_connected_ch(target) > 0);
294c875a 997
509c07bc 998 ret = srp_lookup_path(ch);
aef9ec39
RD		 999 if (ret)
1000 return ret;
1001
1002 while (1) {
509c07bc 1003 init_completion(&ch->done);
d92c0da7 1004 ret = srp_send_req(ch, multich);
aef9ec39
RD		 1005 if (ret)
1006 return ret;
509c07bc 1007 ret = wait_for_completion_interruptible(&ch->done);
a702adce
BVA		 1008 if (ret < 0)
1009 return ret;
aef9ec39
RD		 1010
1011 /*
1012 * The CM event handling code will set status to
1013 * SRP_PORT_REDIRECT if we get a port redirect REJ
1014 * back, or SRP_DLID_REDIRECT if we get a lid/qp
1015 * redirect REJ back.
1016 */
509c07bc 1017 switch (ch->status) {
aef9ec39 1018 case 0:
c014c8cd 1019 ch->connected = true;
aef9ec39
RD		 1020 return 0;
1021
1022 case SRP_PORT_REDIRECT:
509c07bc 1023 ret = srp_lookup_path(ch);
aef9ec39
RD		 1024 if (ret)
1025 return ret;
1026 break;
1027
1028 case SRP_DLID_REDIRECT:
1029 break;
1030
9fe4bcf4 1031 case SRP_STALE_CONN:
9fe4bcf4 1032 shost_printk(KERN_ERR, target->scsi_host, PFX
205619f2 1033 "giving up on stale connection\n");
509c07bc
BVA		 1034 ch->status = -ECONNRESET;
1035 return ch->status;
9fe4bcf4 1036
aef9ec39 1037 default:
509c07bc 1038 return ch->status;
aef9ec39
RD		 1039 }
1040 }
1041}
1042
509c07bc 1043static int srp_inv_rkey(struct srp_rdma_ch *ch, u32 rkey)
5cfb1782
BVA		 1044{
1045 struct ib_send_wr *bad_wr;
1046 struct ib_send_wr wr = {
1047 .opcode = IB_WR_LOCAL_INV,
1048 .wr_id = LOCAL_INV_WR_ID_MASK,
1049 .next = NULL,
1050 .num_sge = 0,
1051 .send_flags = 0,
1052 .ex.invalidate_rkey = rkey,
1053 };
1054
509c07bc 1055 return ib_post_send(ch->qp, &wr, &bad_wr);
5cfb1782
BVA		 1056}
1057
d945e1df 1058static void srp_unmap_data(struct scsi_cmnd *scmnd,
509c07bc 1059 struct srp_rdma_ch *ch,
d945e1df
RD		 1060 struct srp_request *req)
1061{
509c07bc 1062 struct srp_target_port *target = ch->target;
5cfb1782
BVA		 1063 struct srp_device *dev = target->srp_host->srp_dev;
1064 struct ib_device *ibdev = dev->dev;
1065 int i, res;
8f26c9ff 1066
bb350d1d 1067 if (!scsi_sglist(scmnd) ||
d945e1df
RD		 1068 (scmnd->sc_data_direction != DMA_TO_DEVICE &&
1069 scmnd->sc_data_direction != DMA_FROM_DEVICE))
1070 return;
1071
5cfb1782
BVA		 1072 if (dev->use_fast_reg) {
1073 struct srp_fr_desc **pfr;
1074
1075 for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) {
509c07bc 1076 res = srp_inv_rkey(ch, (*pfr)->mr->rkey);
5cfb1782
BVA		 1077 if (res < 0) {
1078 shost_printk(KERN_ERR, target->scsi_host, PFX
1079 "Queueing INV WR for rkey %#x failed (%d)\n",
1080 (*pfr)->mr->rkey, res);
1081 queue_work(system_long_wq,
1082 &target->tl_err_work);
1083 }
1084 }
1085 if (req->nmdesc)
509c07bc 1086 srp_fr_pool_put(ch->fr_pool, req->fr_list,
5cfb1782
BVA		 1087 req->nmdesc);
1088 } else {
1089 struct ib_pool_fmr **pfmr;
1090
1091 for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++)
1092 ib_fmr_pool_unmap(*pfmr);
1093 }
f5358a17 1094
8f26c9ff
DD		 1095 ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
1096 scmnd->sc_data_direction);
d945e1df
RD		 1097}
1098
22032991
BVA		 1099/**
1100 * srp_claim_req - Take ownership of the scmnd associated with a request.
509c07bc 1101 * @ch: SRP RDMA channel.
22032991 1102 * @req: SRP request.
b3fe628d 1103 * @sdev: If not NULL, only take ownership for this SCSI device.
22032991
BVA		 1104 * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take
1105 * ownership of @req->scmnd if it equals @scmnd.
1106 *
1107 * Return value:
1108 * Either NULL or a pointer to the SCSI command the caller became owner of.
1109 */
509c07bc 1110static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch,
22032991 1111 struct srp_request *req,
b3fe628d 1112 struct scsi_device *sdev,
22032991
BVA		 1113 struct scsi_cmnd *scmnd)
1114{
1115 unsigned long flags;
1116
509c07bc 1117 spin_lock_irqsave(&ch->lock, flags);
b3fe628d
BVA		 1118 if (req->scmnd &&
1119 (!sdev || req->scmnd->device == sdev) &&
1120 (!scmnd || req->scmnd == scmnd)) {
22032991
BVA		 1121 scmnd = req->scmnd;
1122 req->scmnd = NULL;
22032991
BVA		 1123 } else {
1124 scmnd = NULL;
1125 }
509c07bc 1126 spin_unlock_irqrestore(&ch->lock, flags);
22032991
BVA		 1127
1128 return scmnd;
1129}
1130
1131/**
1132 * srp_free_req() - Unmap data and add request to the free request list.
509c07bc 1133 * @ch: SRP RDMA channel.
af24663b
BVA		 1134 * @req: Request to be freed.
1135 * @scmnd: SCSI command associated with @req.
1136 * @req_lim_delta: Amount to be added to @target->req_lim.
22032991 1137 */
509c07bc
BVA		 1138static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req,
1139 struct scsi_cmnd *scmnd, s32 req_lim_delta)
526b4caa 1140{
94a9174c
BVA		 1141 unsigned long flags;
1142
509c07bc 1143 srp_unmap_data(scmnd, ch, req);
22032991 1144
509c07bc
BVA		 1145 spin_lock_irqsave(&ch->lock, flags);
1146 ch->req_lim += req_lim_delta;
509c07bc 1147 spin_unlock_irqrestore(&ch->lock, flags);
526b4caa
IR		 1148}
1149
509c07bc
BVA		 1150static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req,
1151 struct scsi_device *sdev, int result)
526b4caa 1152{
509c07bc 1153 struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL);
22032991
BVA		 1154
1155 if (scmnd) {
509c07bc 1156 srp_free_req(ch, req, scmnd, 0);
ed9b2264 1157 scmnd->result = result;
22032991 1158 scmnd->scsi_done(scmnd);
22032991 1159 }
526b4caa
IR		 1160}
1161
ed9b2264 1162static void srp_terminate_io(struct srp_rport *rport)
aef9ec39 1163{
ed9b2264 1164 struct srp_target_port *target = rport->lld_data;
d92c0da7 1165 struct srp_rdma_ch *ch;
b3fe628d
BVA		 1166 struct Scsi_Host *shost = target->scsi_host;
1167 struct scsi_device *sdev;
d92c0da7 1168 int i, j;
ed9b2264 1169
b3fe628d
BVA		 1170 /*
1171 * Invoking srp_terminate_io() while srp_queuecommand() is running
1172 * is not safe. Hence the warning statement below.
1173 */
1174 shost_for_each_device(sdev, shost)
1175 WARN_ON_ONCE(sdev->request_queue->request_fn_active);
1176
d92c0da7
BVA		 1177 for (i = 0; i < target->ch_count; i++) {
1178 ch = &target->ch[i];
509c07bc 1179
d92c0da7
BVA		 1180 for (j = 0; j < target->req_ring_size; ++j) {
1181 struct srp_request *req = &ch->req_ring[j];
1182
1183 srp_finish_req(ch, req, NULL,
1184 DID_TRANSPORT_FAILFAST << 16);
1185 }
ed9b2264
BVA		 1186 }
1187}
aef9ec39 1188
ed9b2264
BVA		 1189/*
1190 * It is up to the caller to ensure that srp_rport_reconnect() calls are
1191 * serialized and that no concurrent srp_queuecommand(), srp_abort(),
1192 * srp_reset_device() or srp_reset_host() calls will occur while this function
1193 * is in progress. One way to realize that is not to call this function
1194 * directly but to call srp_reconnect_rport() instead since that last function
1195 * serializes calls of this function via rport->mutex and also blocks
1196 * srp_queuecommand() calls before invoking this function.
1197 */
1198static int srp_rport_reconnect(struct srp_rport *rport)
1199{
1200 struct srp_target_port *target = rport->lld_data;
d92c0da7
BVA		 1201 struct srp_rdma_ch *ch;
1202 int i, j, ret = 0;
1203 bool multich = false;
09be70a2 1204
aef9ec39 1205 srp_disconnect_target(target);
34aa654e
BVA		 1206
1207 if (target->state == SRP_TARGET_SCANNING)
1208 return -ENODEV;
1209
aef9ec39 1210 /*
c7c4e7ff
BVA		 1211 * Now get a new local CM ID so that we avoid confusing the target in
1212 * case things are really fouled up. Doing so also ensures that all CM
1213 * callbacks will have finished before a new QP is allocated.
aef9ec39 1214 */
d92c0da7
BVA		 1215 for (i = 0; i < target->ch_count; i++) {
1216 ch = &target->ch[i];
d92c0da7 1217 ret += srp_new_cm_id(ch);
536ae14e 1218 }
d92c0da7
BVA		 1219 for (i = 0; i < target->ch_count; i++) {
1220 ch = &target->ch[i];
d92c0da7
BVA		 1221 for (j = 0; j < target->req_ring_size; ++j) {
1222 struct srp_request *req = &ch->req_ring[j];
aef9ec39 1223
d92c0da7
BVA		 1224 srp_finish_req(ch, req, NULL, DID_RESET << 16);
1225 }
1226 }
1227 for (i = 0; i < target->ch_count; i++) {
1228 ch = &target->ch[i];
d92c0da7
BVA		 1229 /*
1230 * Whether or not creating a new CM ID succeeded, create a new
1231 * QP. This guarantees that all completion callback function
1232 * invocations have finished before request resetting starts.
1233 */
1234 ret += srp_create_ch_ib(ch);
aef9ec39 1235
d92c0da7
BVA		 1236 INIT_LIST_HEAD(&ch->free_tx);
1237 for (j = 0; j < target->queue_size; ++j)
1238 list_add(&ch->tx_ring[j]->list, &ch->free_tx);
1239 }
8de9fe3a
BVA		 1240
1241 target->qp_in_error = false;
1242
d92c0da7
BVA		 1243 for (i = 0; i < target->ch_count; i++) {
1244 ch = &target->ch[i];
bbac5ccf 1245 if (ret)
d92c0da7 1246 break;
d92c0da7
BVA		 1247 ret = srp_connect_ch(ch, multich);
1248 multich = true;
1249 }
09be70a2 1250
ed9b2264
BVA		 1251 if (ret == 0)
1252 shost_printk(KERN_INFO, target->scsi_host,
1253 PFX "reconnect succeeded\n");
aef9ec39
RD		 1254
1255 return ret;
1256}
1257
8f26c9ff
DD		 1258static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
1259 unsigned int dma_len, u32 rkey)
f5358a17 1260{
8f26c9ff 1261 struct srp_direct_buf *desc = state->desc;
f5358a17 1262
8f26c9ff
DD		 1263 desc->va = cpu_to_be64(dma_addr);
1264 desc->key = cpu_to_be32(rkey);
1265 desc->len = cpu_to_be32(dma_len);
f5358a17 1266
8f26c9ff
DD		 1267 state->total_len += dma_len;
1268 state->desc++;
1269 state->ndesc++;
1270}
559ce8f1 1271
8f26c9ff 1272static int srp_map_finish_fmr(struct srp_map_state *state,
509c07bc 1273 struct srp_rdma_ch *ch)
8f26c9ff 1274{
186fbc66
BVA		 1275 struct srp_target_port *target = ch->target;
1276 struct srp_device *dev = target->srp_host->srp_dev;
8f26c9ff
DD		 1277 struct ib_pool_fmr *fmr;
1278 u64 io_addr = 0;
85507bcc 1279
f731ed62
BVA		 1280 if (state->fmr.next >= state->fmr.end)
1281 return -ENOMEM;
1282
509c07bc 1283 fmr = ib_fmr_pool_map_phys(ch->fmr_pool, state->pages,
8f26c9ff
DD		 1284 state->npages, io_addr);
1285 if (IS_ERR(fmr))
1286 return PTR_ERR(fmr);
f5358a17 1287
f731ed62 1288 *state->fmr.next++ = fmr;
52ede08f 1289 state->nmdesc++;
f5358a17 1290
186fbc66
BVA		 1291 srp_map_desc(state, state->base_dma_addr & ~dev->mr_page_mask,
1292 state->dma_len, fmr->fmr->rkey);
539dde6f 1293
8f26c9ff
DD		 1294 return 0;
1295}
1296
5cfb1782 1297static int srp_map_finish_fr(struct srp_map_state *state,
509c07bc 1298 struct srp_rdma_ch *ch)
5cfb1782 1299{
509c07bc 1300 struct srp_target_port *target = ch->target;
5cfb1782
BVA		 1301 struct srp_device *dev = target->srp_host->srp_dev;
1302 struct ib_send_wr *bad_wr;
1303 struct ib_send_wr wr;
1304 struct srp_fr_desc *desc;
1305 u32 rkey;
1306
f731ed62
BVA		 1307 if (state->fr.next >= state->fr.end)
1308 return -ENOMEM;
1309
509c07bc 1310 desc = srp_fr_pool_get(ch->fr_pool);
5cfb1782
BVA		 1311 if (!desc)
1312 return -ENOMEM;
1313
1314 rkey = ib_inc_rkey(desc->mr->rkey);
1315 ib_update_fast_reg_key(desc->mr, rkey);
1316
1317 memcpy(desc->frpl->page_list, state->pages,
1318 sizeof(state->pages[0]) * state->npages);
1319
1320 memset(&wr, 0, sizeof(wr));
1321 wr.opcode = IB_WR_FAST_REG_MR;
1322 wr.wr_id = FAST_REG_WR_ID_MASK;
1323 wr.wr.fast_reg.iova_start = state->base_dma_addr;
1324 wr.wr.fast_reg.page_list = desc->frpl;
1325 wr.wr.fast_reg.page_list_len = state->npages;
1326 wr.wr.fast_reg.page_shift = ilog2(dev->mr_page_size);
1327 wr.wr.fast_reg.length = state->dma_len;
1328 wr.wr.fast_reg.access_flags = (IB_ACCESS_LOCAL_WRITE |
1329 IB_ACCESS_REMOTE_READ |
1330 IB_ACCESS_REMOTE_WRITE);
1331 wr.wr.fast_reg.rkey = desc->mr->lkey;
1332
f731ed62 1333 *state->fr.next++ = desc;
5cfb1782
BVA		 1334 state->nmdesc++;
1335
1336 srp_map_desc(state, state->base_dma_addr, state->dma_len,
1337 desc->mr->rkey);
1338
509c07bc 1339 return ib_post_send(ch->qp, &wr, &bad_wr);
5cfb1782
BVA		 1340}
1341
539dde6f 1342static int srp_finish_mapping(struct srp_map_state *state,
509c07bc 1343 struct srp_rdma_ch *ch)
539dde6f 1344{
509c07bc 1345 struct srp_target_port *target = ch->target;
539dde6f
BVA		 1346 int ret = 0;
1347
1348 if (state->npages == 0)
1349 return 0;
1350
b1b8854d 1351 if (state->npages == 1 && !register_always)
52ede08f 1352 srp_map_desc(state, state->base_dma_addr, state->dma_len,
539dde6f
BVA		 1353 target->rkey);
1354 else
5cfb1782 1355 ret = target->srp_host->srp_dev->use_fast_reg ?
509c07bc
BVA		 1356 srp_map_finish_fr(state, ch) :
1357 srp_map_finish_fmr(state, ch);
539dde6f
BVA		 1358
1359 if (ret == 0) {
1360 state->npages = 0;
52ede08f 1361 state->dma_len = 0;
539dde6f
BVA		 1362 }
1363
1364 return ret;
1365}
1366
8f26c9ff 1367static int srp_map_sg_entry(struct srp_map_state *state,
509c07bc 1368 struct srp_rdma_ch *ch,
8f26c9ff 1369 struct scatterlist *sg, int sg_index,
5cfb1782 1370 bool use_mr)
8f26c9ff 1371{
509c07bc 1372 struct srp_target_port *target = ch->target;
8f26c9ff
DD		 1373 struct srp_device *dev = target->srp_host->srp_dev;
1374 struct ib_device *ibdev = dev->dev;
1375 dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg);
1376 unsigned int dma_len = ib_sg_dma_len(ibdev, sg);
1377 unsigned int len;
1378 int ret;
1379
1380 if (!dma_len)
1381 return 0;
1382
5cfb1782
BVA		 1383 if (!use_mr) {
1384 /*
1385 * Once we're in direct map mode for a request, we don't
1386 * go back to FMR or FR mode, so no need to update anything
8f26c9ff
DD		 1387 * other than the descriptor.
1388 */
1389 srp_map_desc(state, dma_addr, dma_len, target->rkey);
1390 return 0;
85507bcc 1391 }
f5358a17 1392
8f26c9ff 1393 while (dma_len) {
5cfb1782
BVA		 1394 unsigned offset = dma_addr & ~dev->mr_page_mask;
1395 if (state->npages == dev->max_pages_per_mr || offset != 0) {
509c07bc 1396 ret = srp_finish_mapping(state, ch);
8f26c9ff
DD		 1397 if (ret)
1398 return ret;
8f26c9ff
DD		 1399 }
1400
5cfb1782 1401 len = min_t(unsigned int, dma_len, dev->mr_page_size - offset);
f5358a17 1402
8f26c9ff
DD		 1403 if (!state->npages)
1404 state->base_dma_addr = dma_addr;
5cfb1782 1405 state->pages[state->npages++] = dma_addr & dev->mr_page_mask;
52ede08f 1406 state->dma_len += len;
8f26c9ff
DD		 1407 dma_addr += len;
1408 dma_len -= len;
1409 }
1410
5cfb1782
BVA		 1411 /*
1412 * If the last entry of the MR wasn't a full page, then we need to
8f26c9ff
DD		 1413 * close it out and start a new one -- we can only merge at page
1414 * boundries.
1415 */
1416 ret = 0;
0e0d3a48 1417 if (len != dev->mr_page_size)
509c07bc 1418 ret = srp_finish_mapping(state, ch);
f5358a17
RD		 1419 return ret;
1420}
1421
509c07bc
BVA		 1422static int srp_map_sg(struct srp_map_state *state, struct srp_rdma_ch *ch,
1423 struct srp_request *req, struct scatterlist *scat,
1424 int count)
76bc1e1d 1425{
509c07bc 1426 struct srp_target_port *target = ch->target;
76bc1e1d 1427 struct srp_device *dev = target->srp_host->srp_dev;
76bc1e1d 1428 struct scatterlist *sg;
0e0d3a48 1429 int i, ret;
5cfb1782 1430 bool use_mr;
76bc1e1d
BVA		 1431
1432 state->desc = req->indirect_desc;
1433 state->pages = req->map_page;
5cfb1782 1434 if (dev->use_fast_reg) {
f731ed62
BVA		 1435 state->fr.next = req->fr_list;
1436 state->fr.end = req->fr_list + target->cmd_sg_cnt;
509c07bc 1437 use_mr = !!ch->fr_pool;
5cfb1782 1438 } else {
f731ed62
BVA		 1439 state->fmr.next = req->fmr_list;
1440 state->fmr.end = req->fmr_list + target->cmd_sg_cnt;
509c07bc 1441 use_mr = !!ch->fmr_pool;
5cfb1782 1442 }
76bc1e1d
BVA		 1443
1444 for_each_sg(scat, sg, count, i) {
0e0d3a48
BVA		 1445 ret = srp_map_sg_entry(state, ch, sg, i, use_mr);
1446 if (ret)
1447 goto out;
76bc1e1d
BVA		 1448 }
1449
0e0d3a48
BVA		 1450 if (use_mr) {
1451 ret = srp_finish_mapping(state, ch);
1452 if (ret)
1453 goto out;
1454 }
76bc1e1d 1455
52ede08f 1456 req->nmdesc = state->nmdesc;
0e0d3a48 1457 ret = 0;
5cfb1782 1458
0e0d3a48
BVA		 1459out:
1460 return ret;
76bc1e1d
BVA		 1461}
1462
509c07bc 1463static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
aef9ec39
RD		 1464 struct srp_request *req)
1465{
509c07bc 1466 struct srp_target_port *target = ch->target;
76bc1e1d 1467 struct scatterlist *scat;
aef9ec39 1468 struct srp_cmd *cmd = req->cmd->buf;
76bc1e1d 1469 int len, nents, count;
85507bcc
RC		 1470 struct srp_device *dev;
1471 struct ib_device *ibdev;
8f26c9ff
DD		 1472 struct srp_map_state state;
1473 struct srp_indirect_buf *indirect_hdr;
8f26c9ff
DD		 1474 u32 table_len;
1475 u8 fmt;
aef9ec39 1476
bb350d1d 1477 if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE)
aef9ec39
RD		 1478 return sizeof (struct srp_cmd);
1479
1480 if (scmnd->sc_data_direction != DMA_FROM_DEVICE &&
1481 scmnd->sc_data_direction != DMA_TO_DEVICE) {
7aa54bd7
DD		 1482 shost_printk(KERN_WARNING, target->scsi_host,
1483 PFX "Unhandled data direction %d\n",
1484 scmnd->sc_data_direction);
aef9ec39
RD		 1485 return -EINVAL;
1486 }
1487
bb350d1d
FT		 1488 nents = scsi_sg_count(scmnd);
1489 scat = scsi_sglist(scmnd);
aef9ec39 1490
05321937 1491 dev = target->srp_host->srp_dev;
85507bcc
RC		 1492 ibdev = dev->dev;
1493
1494 count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction);
8f26c9ff
DD		 1495 if (unlikely(count == 0))
1496 return -EIO;
f5358a17
RD		 1497
1498 fmt = SRP_DATA_DESC_DIRECT;
1499 len = sizeof (struct srp_cmd) + sizeof (struct srp_direct_buf);
aef9ec39 1500
b1b8854d 1501 if (count == 1 && !register_always) {
f5358a17
RD		 1502 /*
1503 * The midlayer only generated a single gather/scatter
1504 * entry, or DMA mapping coalesced everything to a
1505 * single entry. So a direct descriptor along with
1506 * the DMA MR suffices.
1507 */
cf368713 1508 struct srp_direct_buf *buf = (void *) cmd->add_data;
aef9ec39 1509
85507bcc 1510 buf->va = cpu_to_be64(ib_sg_dma_address(ibdev, scat));
9af76271 1511 buf->key = cpu_to_be32(target->rkey);
85507bcc 1512 buf->len = cpu_to_be32(ib_sg_dma_len(ibdev, scat));
8f26c9ff 1513
52ede08f 1514 req->nmdesc = 0;
8f26c9ff
DD		 1515 goto map_complete;
1516 }
1517
5cfb1782
BVA		 1518 /*
1519 * We have more than one scatter/gather entry, so build our indirect
1520 * descriptor table, trying to merge as many entries as we can.
8f26c9ff
DD		 1521 */
1522 indirect_hdr = (void *) cmd->add_data;
1523
c07d424d
DD		 1524 ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr,
1525 target->indirect_size, DMA_TO_DEVICE);
1526
8f26c9ff 1527 memset(&state, 0, sizeof(state));
509c07bc 1528 srp_map_sg(&state, ch, req, scat, count);
cf368713 1529
c07d424d
DD		 1530 /* We've mapped the request, now pull as much of the indirect
1531 * descriptor table as we can into the command buffer. If this
1532 * target is not using an external indirect table, we are
1533 * guaranteed to fit into the command, as the SCSI layer won't
1534 * give us more S/G entries than we allow.
8f26c9ff 1535 */
8f26c9ff 1536 if (state.ndesc == 1) {
5cfb1782
BVA		 1537 /*
1538 * Memory registration collapsed the sg-list into one entry,
8f26c9ff
DD		 1539 * so use a direct descriptor.
1540 */
1541 struct srp_direct_buf *buf = (void *) cmd->add_data;
cf368713 1542
c07d424d 1543 *buf = req->indirect_desc[0];
8f26c9ff 1544 goto map_complete;
aef9ec39
RD		 1545 }
1546
c07d424d
DD		 1547 if (unlikely(target->cmd_sg_cnt < state.ndesc &&
1548 !target->allow_ext_sg)) {
1549 shost_printk(KERN_ERR, target->scsi_host,
1550 "Could not fit S/G list into SRP_CMD\n");
1551 return -EIO;
1552 }
1553
1554 count = min(state.ndesc, target->cmd_sg_cnt);
8f26c9ff
DD		 1555 table_len = state.ndesc * sizeof (struct srp_direct_buf);
1556
1557 fmt = SRP_DATA_DESC_INDIRECT;
1558 len = sizeof(struct srp_cmd) + sizeof (struct srp_indirect_buf);
c07d424d 1559 len += count * sizeof (struct srp_direct_buf);
8f26c9ff 1560
c07d424d
DD		 1561 memcpy(indirect_hdr->desc_list, req->indirect_desc,
1562 count * sizeof (struct srp_direct_buf));
8f26c9ff 1563
c07d424d 1564 indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr);
8f26c9ff
DD		 1565 indirect_hdr->table_desc.key = cpu_to_be32(target->rkey);
1566 indirect_hdr->table_desc.len = cpu_to_be32(table_len);
1567 indirect_hdr->len = cpu_to_be32(state.total_len);
1568
1569 if (scmnd->sc_data_direction == DMA_TO_DEVICE)
c07d424d 1570 cmd->data_out_desc_cnt = count;
8f26c9ff 1571 else
c07d424d
DD		 1572 cmd->data_in_desc_cnt = count;
1573
1574 ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len,
1575 DMA_TO_DEVICE);
8f26c9ff
DD		 1576
1577map_complete:
aef9ec39
RD		 1578 if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1579 cmd->buf_fmt = fmt << 4;
1580 else
1581 cmd->buf_fmt = fmt;
1582
aef9ec39
RD		 1583 return len;
1584}
1585
76c75b25
BVA		 1586/*
1587 * Return an IU and possible credit to the free pool
1588 */
509c07bc 1589static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu,
76c75b25
BVA		 1590 enum srp_iu_type iu_type)
1591{
1592 unsigned long flags;
1593
509c07bc
BVA		 1594 spin_lock_irqsave(&ch->lock, flags);
1595 list_add(&iu->list, &ch->free_tx);
76c75b25 1596 if (iu_type != SRP_IU_RSP)
509c07bc
BVA		 1597 ++ch->req_lim;
1598 spin_unlock_irqrestore(&ch->lock, flags);
76c75b25
BVA		 1599}
1600
05a1d750 1601/*
509c07bc 1602 * Must be called with ch->lock held to protect req_lim and free_tx.
e9684678 1603 * If IU is not sent, it must be returned using srp_put_tx_iu().
05a1d750
DD		 1604 *
1605 * Note:
1606 * An upper limit for the number of allocated information units for each
1607 * request type is:
1608 * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues
1609 * more than Scsi_Host.can_queue requests.
1610 * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE.
1611 * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than
1612 * one unanswered SRP request to an initiator.
1613 */
509c07bc 1614static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch,
05a1d750
DD		 1615 enum srp_iu_type iu_type)
1616{
509c07bc 1617 struct srp_target_port *target = ch->target;
05a1d750
DD		 1618 s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
1619 struct srp_iu *iu;
1620
509c07bc 1621 srp_send_completion(ch->send_cq, ch);
05a1d750 1622
509c07bc 1623 if (list_empty(&ch->free_tx))
05a1d750
DD		 1624 return NULL;
1625
1626 /* Initiator responses to target requests do not consume credits */
76c75b25 1627 if (iu_type != SRP_IU_RSP) {
509c07bc 1628 if (ch->req_lim <= rsv) {
76c75b25
BVA		 1629 ++target->zero_req_lim;
1630 return NULL;
1631 }
1632
509c07bc 1633 --ch->req_lim;
05a1d750
DD		 1634 }
1635
509c07bc 1636 iu = list_first_entry(&ch->free_tx, struct srp_iu, list);
76c75b25 1637 list_del(&iu->list);
05a1d750
DD		 1638 return iu;
1639}
1640
509c07bc 1641static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len)
05a1d750 1642{
509c07bc 1643 struct srp_target_port *target = ch->target;
05a1d750
DD		 1644 struct ib_sge list;
1645 struct ib_send_wr wr, *bad_wr;
05a1d750
DD		 1646
1647 list.addr = iu->dma;
1648 list.length = len;
9af76271 1649 list.lkey = target->lkey;
05a1d750
DD		 1650
1651 wr.next = NULL;
dcb4cb85 1652 wr.wr_id = (uintptr_t) iu;
05a1d750
DD		 1653 wr.sg_list = &list;
1654 wr.num_sge = 1;
1655 wr.opcode = IB_WR_SEND;
1656 wr.send_flags = IB_SEND_SIGNALED;
1657
509c07bc 1658 return ib_post_send(ch->qp, &wr, &bad_wr);
05a1d750
DD		 1659}
1660
509c07bc 1661static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu)
c996bb47 1662{
509c07bc 1663 struct srp_target_port *target = ch->target;
c996bb47 1664 struct ib_recv_wr wr, *bad_wr;
dcb4cb85 1665 struct ib_sge list;
c996bb47
BVA		 1666
1667 list.addr = iu->dma;
1668 list.length = iu->size;
9af76271 1669 list.lkey = target->lkey;
c996bb47
BVA		 1670
1671 wr.next = NULL;
dcb4cb85 1672 wr.wr_id = (uintptr_t) iu;
c996bb47
BVA		 1673 wr.sg_list = &list;
1674 wr.num_sge = 1;
1675
509c07bc 1676 return ib_post_recv(ch->qp, &wr, &bad_wr);
c996bb47
BVA		 1677}
1678
509c07bc 1679static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp)
aef9ec39 1680{
509c07bc 1681 struct srp_target_port *target = ch->target;
aef9ec39
RD		 1682 struct srp_request *req;
1683 struct scsi_cmnd *scmnd;
1684 unsigned long flags;
aef9ec39 1685
aef9ec39 1686 if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) {
509c07bc
BVA		 1687 spin_lock_irqsave(&ch->lock, flags);
1688 ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1689 spin_unlock_irqrestore(&ch->lock, flags);
94a9174c 1690
509c07bc 1691 ch->tsk_mgmt_status = -1;
f8b6e31e 1692 if (be32_to_cpu(rsp->resp_data_len) >= 4)
509c07bc
BVA		 1693 ch->tsk_mgmt_status = rsp->data[3];
1694 complete(&ch->tsk_mgmt_done);
aef9ec39 1695 } else {
77f2c1a4
BVA		 1696 scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag);
1697 if (scmnd) {
1698 req = (void *)scmnd->host_scribble;
1699 scmnd = srp_claim_req(ch, req, NULL, scmnd);
1700 }
22032991 1701 if (!scmnd) {
7aa54bd7 1702 shost_printk(KERN_ERR, target->scsi_host,
d92c0da7
BVA		 1703 "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n",
1704 rsp->tag, ch - target->ch, ch->qp->qp_num);
22032991 1705
509c07bc
BVA		 1706 spin_lock_irqsave(&ch->lock, flags);
1707 ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1708 spin_unlock_irqrestore(&ch->lock, flags);
22032991
BVA		 1709
1710 return;
1711 }
aef9ec39
RD		 1712 scmnd->result = rsp->status;
1713
1714 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) {
1715 memcpy(scmnd->sense_buffer, rsp->data +
1716 be32_to_cpu(rsp->resp_data_len),
1717 min_t(int, be32_to_cpu(rsp->sense_data_len),
1718 SCSI_SENSE_BUFFERSIZE));
1719 }
1720
e714531a 1721 if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER))
bb350d1d 1722 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt));
e714531a
BVA		 1723 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER))
1724 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt));
1725 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER))
1726 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt));
1727 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER))
1728 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt));
aef9ec39 1729
509c07bc 1730 srp_free_req(ch, req, scmnd,
22032991
BVA		 1731 be32_to_cpu(rsp->req_lim_delta));
1732
f8b6e31e
DD		 1733 scmnd->host_scribble = NULL;
1734 scmnd->scsi_done(scmnd);
aef9ec39 1735 }
aef9ec39
RD		 1736}
1737
509c07bc 1738static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta,
bb12588a
DD		 1739 void *rsp, int len)
1740{
509c07bc 1741 struct srp_target_port *target = ch->target;
76c75b25 1742 struct ib_device *dev = target->srp_host->srp_dev->dev;
bb12588a
DD		 1743 unsigned long flags;
1744 struct srp_iu *iu;
76c75b25 1745 int err;
bb12588a 1746
509c07bc
BVA		 1747 spin_lock_irqsave(&ch->lock, flags);
1748 ch->req_lim += req_delta;
1749 iu = __srp_get_tx_iu(ch, SRP_IU_RSP);
1750 spin_unlock_irqrestore(&ch->lock, flags);
76c75b25 1751
bb12588a
DD		 1752 if (!iu) {
1753 shost_printk(KERN_ERR, target->scsi_host, PFX
1754 "no IU available to send response\n");
76c75b25 1755 return 1;
bb12588a
DD		 1756 }
1757
1758 ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE);
1759 memcpy(iu->buf, rsp, len);
1760 ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE);
1761
509c07bc 1762 err = srp_post_send(ch, iu, len);
76c75b25 1763 if (err) {
bb12588a
DD		 1764 shost_printk(KERN_ERR, target->scsi_host, PFX
1765 "unable to post response: %d\n", err);
509c07bc 1766 srp_put_tx_iu(ch, iu, SRP_IU_RSP);
76c75b25 1767 }
bb12588a 1768
bb12588a
DD		 1769 return err;
1770}
1771
509c07bc 1772static void srp_process_cred_req(struct srp_rdma_ch *ch,
bb12588a
DD		 1773 struct srp_cred_req *req)
1774{
1775 struct srp_cred_rsp rsp = {
1776 .opcode = SRP_CRED_RSP,
1777 .tag = req->tag,
1778 };
1779 s32 delta = be32_to_cpu(req->req_lim_delta);
1780
509c07bc
BVA		 1781 if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
1782 shost_printk(KERN_ERR, ch->target->scsi_host, PFX
bb12588a
DD		 1783 "problems processing SRP_CRED_REQ\n");
1784}
1785
509c07bc 1786static void srp_process_aer_req(struct srp_rdma_ch *ch,
bb12588a
DD		 1787 struct srp_aer_req *req)
1788{
509c07bc 1789 struct srp_target_port *target = ch->target;
bb12588a
DD		 1790 struct srp_aer_rsp rsp = {
1791 .opcode = SRP_AER_RSP,
1792 .tag = req->tag,
1793 };
1794 s32 delta = be32_to_cpu(req->req_lim_delta);
1795
1796 shost_printk(KERN_ERR, target->scsi_host, PFX
985aa495 1797 "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun));
bb12588a 1798
509c07bc 1799 if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
bb12588a
DD		 1800 shost_printk(KERN_ERR, target->scsi_host, PFX
1801 "problems processing SRP_AER_REQ\n");
1802}
1803
509c07bc 1804static void srp_handle_recv(struct srp_rdma_ch *ch, struct ib_wc *wc)
aef9ec39 1805{
509c07bc 1806 struct srp_target_port *target = ch->target;
dcb4cb85 1807 struct ib_device *dev = target->srp_host->srp_dev->dev;
737b94eb 1808 struct srp_iu *iu = (struct srp_iu *) (uintptr_t) wc->wr_id;
c996bb47 1809 int res;
aef9ec39
RD		 1810 u8 opcode;
1811
509c07bc 1812 ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len,
85507bcc 1813 DMA_FROM_DEVICE);
aef9ec39
RD		 1814
1815 opcode = *(u8 *) iu->buf;
1816
1817 if (0) {
7aa54bd7
DD		 1818 shost_printk(KERN_ERR, target->scsi_host,
1819 PFX "recv completion, opcode 0x%02x\n", opcode);
7a700811
BVA		 1820 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1,
1821 iu->buf, wc->byte_len, true);
aef9ec39
RD		 1822 }
1823
1824 switch (opcode) {
1825 case SRP_RSP:
509c07bc 1826 srp_process_rsp(ch, iu->buf);
aef9ec39
RD		 1827 break;
1828
bb12588a 1829 case SRP_CRED_REQ:
509c07bc 1830 srp_process_cred_req(ch, iu->buf);
bb12588a
DD		 1831 break;
1832
1833 case SRP_AER_REQ:
509c07bc 1834 srp_process_aer_req(ch, iu->buf);
bb12588a
DD		 1835 break;
1836
aef9ec39
RD		 1837 case SRP_T_LOGOUT:
1838 /* XXX Handle target logout */
7aa54bd7
DD		 1839 shost_printk(KERN_WARNING, target->scsi_host,
1840 PFX "Got target logout request\n");
aef9ec39
RD		 1841 break;
1842
1843 default:
7aa54bd7
DD		 1844 shost_printk(KERN_WARNING, target->scsi_host,
1845 PFX "Unhandled SRP opcode 0x%02x\n", opcode);
aef9ec39
RD		 1846 break;
1847 }
1848
509c07bc 1849 ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len,
85507bcc 1850 DMA_FROM_DEVICE);
c996bb47 1851
509c07bc 1852 res = srp_post_recv(ch, iu);
c996bb47
BVA		 1853 if (res != 0)
1854 shost_printk(KERN_ERR, target->scsi_host,
1855 PFX "Recv failed with error code %d\n", res);
aef9ec39
RD		 1856}
1857
c1120f89
BVA		 1858/**
1859 * srp_tl_err_work() - handle a transport layer error
af24663b 1860 * @work: Work structure embedded in an SRP target port.
c1120f89
BVA		 1861 *
1862 * Note: This function may get invoked before the rport has been created,
1863 * hence the target->rport test.
1864 */
1865static void srp_tl_err_work(struct work_struct *work)
1866{
1867 struct srp_target_port *target;
1868
1869 target = container_of(work, struct srp_target_port, tl_err_work);
1870 if (target->rport)
1871 srp_start_tl_fail_timers(target->rport);
1872}
1873
5cfb1782 1874static void srp_handle_qp_err(u64 wr_id, enum ib_wc_status wc_status,
7dad6b2e 1875 bool send_err, struct srp_rdma_ch *ch)
948d1e88 1876{
7dad6b2e
BVA		 1877 struct srp_target_port *target = ch->target;
1878
1879 if (wr_id == SRP_LAST_WR_ID) {
1880 complete(&ch->done);
1881 return;
1882 }
1883
c014c8cd 1884 if (ch->connected && !target->qp_in_error) {
5cfb1782
BVA		 1885 if (wr_id & LOCAL_INV_WR_ID_MASK) {
1886 shost_printk(KERN_ERR, target->scsi_host, PFX
57363d98
SG		 1887 "LOCAL_INV failed with status %s (%d)\n",
1888 ib_wc_status_msg(wc_status), wc_status);
5cfb1782
BVA		 1889 } else if (wr_id & FAST_REG_WR_ID_MASK) {
1890 shost_printk(KERN_ERR, target->scsi_host, PFX
57363d98
SG		 1891 "FAST_REG_MR failed status %s (%d)\n",
1892 ib_wc_status_msg(wc_status), wc_status);
5cfb1782
BVA		 1893 } else {
1894 shost_printk(KERN_ERR, target->scsi_host,
57363d98 1895 PFX "failed %s status %s (%d) for iu %p\n",
5cfb1782 1896 send_err ? "send" : "receive",
57363d98
SG		 1897 ib_wc_status_msg(wc_status), wc_status,
1898 (void *)(uintptr_t)wr_id);
5cfb1782 1899 }
c1120f89 1900 queue_work(system_long_wq, &target->tl_err_work);
4f0af697 1901 }
948d1e88
BVA		 1902 target->qp_in_error = true;
1903}
1904
509c07bc 1905static void srp_recv_completion(struct ib_cq *cq, void *ch_ptr)
aef9ec39 1906{
509c07bc 1907 struct srp_rdma_ch *ch = ch_ptr;
aef9ec39 1908 struct ib_wc wc;
aef9ec39
RD		 1909
1910 ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
1911 while (ib_poll_cq(cq, 1, &wc) > 0) {
948d1e88 1912 if (likely(wc.status == IB_WC_SUCCESS)) {
509c07bc 1913 srp_handle_recv(ch, &wc);
948d1e88 1914 } else {
7dad6b2e 1915 srp_handle_qp_err(wc.wr_id, wc.status, false, ch);
aef9ec39 1916 }
9c03dc9f
BVA		 1917 }
1918}
1919
509c07bc 1920static void srp_send_completion(struct ib_cq *cq, void *ch_ptr)
9c03dc9f 1921{
509c07bc 1922 struct srp_rdma_ch *ch = ch_ptr;
9c03dc9f 1923 struct ib_wc wc;
dcb4cb85 1924 struct srp_iu *iu;
9c03dc9f
BVA		 1925
1926 while (ib_poll_cq(cq, 1, &wc) > 0) {
948d1e88
BVA		 1927 if (likely(wc.status == IB_WC_SUCCESS)) {
1928 iu = (struct srp_iu *) (uintptr_t) wc.wr_id;
509c07bc 1929 list_add(&iu->list, &ch->free_tx);
948d1e88 1930 } else {
7dad6b2e 1931 srp_handle_qp_err(wc.wr_id, wc.status, true, ch);
9c03dc9f 1932 }
aef9ec39
RD		 1933 }
1934}
1935
76c75b25 1936static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
aef9ec39 1937{
76c75b25 1938 struct srp_target_port *target = host_to_target(shost);
a95cadb9 1939 struct srp_rport *rport = target->rport;
509c07bc 1940 struct srp_rdma_ch *ch;
aef9ec39
RD		 1941 struct srp_request *req;
1942 struct srp_iu *iu;
1943 struct srp_cmd *cmd;
85507bcc 1944 struct ib_device *dev;
76c75b25 1945 unsigned long flags;
77f2c1a4
BVA		 1946 u32 tag;
1947 u16 idx;
d1b4289e 1948 int len, ret;
a95cadb9
BVA		 1949 const bool in_scsi_eh = !in_interrupt() && current == shost->ehandler;
1950
1951 /*
1952 * The SCSI EH thread is the only context from which srp_queuecommand()
1953 * can get invoked for blocked devices (SDEV_BLOCK /
1954 * SDEV_CREATED_BLOCK). Avoid racing with srp_reconnect_rport() by
1955 * locking the rport mutex if invoked from inside the SCSI EH.
1956 */
1957 if (in_scsi_eh)
1958 mutex_lock(&rport->mutex);
aef9ec39 1959
d1b4289e
BVA		 1960 scmnd->result = srp_chkready(target->rport);
1961 if (unlikely(scmnd->result))
1962 goto err;
2ce19e72 1963
77f2c1a4
BVA		 1964 WARN_ON_ONCE(scmnd->request->tag < 0);
1965 tag = blk_mq_unique_tag(scmnd->request);
d92c0da7 1966 ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)];
77f2c1a4
BVA		 1967 idx = blk_mq_unique_tag_to_tag(tag);
1968 WARN_ONCE(idx >= target->req_ring_size, "%s: tag %#x: idx %d >= %d\n",
1969 dev_name(&shost->shost_gendev), tag, idx,
1970 target->req_ring_size);
509c07bc
BVA		 1971
1972 spin_lock_irqsave(&ch->lock, flags);
1973 iu = __srp_get_tx_iu(ch, SRP_IU_CMD);
509c07bc 1974 spin_unlock_irqrestore(&ch->lock, flags);
aef9ec39 1975
77f2c1a4
BVA		 1976 if (!iu)
1977 goto err;
1978
1979 req = &ch->req_ring[idx];
05321937 1980 dev = target->srp_host->srp_dev->dev;
49248644 1981 ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len,
85507bcc 1982 DMA_TO_DEVICE);
aef9ec39 1983
f8b6e31e 1984 scmnd->host_scribble = (void *) req;
aef9ec39
RD		 1985
1986 cmd = iu->buf;
1987 memset(cmd, 0, sizeof *cmd);
1988
1989 cmd->opcode = SRP_CMD;
985aa495 1990 int_to_scsilun(scmnd->device->lun, &cmd->lun);
77f2c1a4 1991 cmd->tag = tag;
aef9ec39
RD		 1992 memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len);
1993
aef9ec39
RD		 1994 req->scmnd = scmnd;
1995 req->cmd = iu;
aef9ec39 1996
509c07bc 1997 len = srp_map_data(scmnd, ch, req);
aef9ec39 1998 if (len < 0) {
7aa54bd7 1999 shost_printk(KERN_ERR, target->scsi_host,
d1b4289e
BVA		 2000 PFX "Failed to map data (%d)\n", len);
2001 /*
2002 * If we ran out of memory descriptors (-ENOMEM) because an
2003 * application is queuing many requests with more than
52ede08f 2004 * max_pages_per_mr sg-list elements, tell the SCSI mid-layer
d1b4289e
BVA		 2005 * to reduce queue depth temporarily.
2006 */
2007 scmnd->result = len == -ENOMEM ?
2008 DID_OK << 16 | QUEUE_FULL << 1 : DID_ERROR << 16;
76c75b25 2009 goto err_iu;
aef9ec39
RD		 2010 }
2011
49248644 2012 ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len,
85507bcc 2013 DMA_TO_DEVICE);
aef9ec39 2014
509c07bc 2015 if (srp_post_send(ch, iu, len)) {
7aa54bd7 2016 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n");
aef9ec39
RD		 2017 goto err_unmap;
2018 }
2019
d1b4289e
BVA		 2020 ret = 0;
2021
a95cadb9
BVA		 2022unlock_rport:
2023 if (in_scsi_eh)
2024 mutex_unlock(&rport->mutex);
2025
d1b4289e 2026 return ret;
aef9ec39
RD		 2027
2028err_unmap:
509c07bc 2029 srp_unmap_data(scmnd, ch, req);
aef9ec39 2030
76c75b25 2031err_iu:
509c07bc 2032 srp_put_tx_iu(ch, iu, SRP_IU_CMD);
76c75b25 2033
024ca901
BVA		 2034 /*
2035 * Avoid that the loops that iterate over the request ring can
2036 * encounter a dangling SCSI command pointer.
2037 */
2038 req->scmnd = NULL;
2039
d1b4289e
BVA		 2040err:
2041 if (scmnd->result) {
2042 scmnd->scsi_done(scmnd);
2043 ret = 0;
2044 } else {
2045 ret = SCSI_MLQUEUE_HOST_BUSY;
2046 }
a95cadb9 2047
d1b4289e 2048 goto unlock_rport;
aef9ec39
RD		 2049}
2050
4d73f95f
BVA		 2051/*
2052 * Note: the resources allocated in this function are freed in
509c07bc 2053 * srp_free_ch_ib().
4d73f95f 2054 */
509c07bc 2055static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch)
aef9ec39 2056{
509c07bc 2057 struct srp_target_port *target = ch->target;
aef9ec39
RD		 2058 int i;
2059
509c07bc
BVA		 2060 ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring),
2061 GFP_KERNEL);
2062 if (!ch->rx_ring)
4d73f95f 2063 goto err_no_ring;
509c07bc
BVA		 2064 ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring),
2065 GFP_KERNEL);
2066 if (!ch->tx_ring)
4d73f95f
BVA		 2067 goto err_no_ring;
2068
2069 for (i = 0; i < target->queue_size; ++i) {
509c07bc
BVA		 2070 ch->rx_ring[i] = srp_alloc_iu(target->srp_host,
2071 ch->max_ti_iu_len,
2072 GFP_KERNEL, DMA_FROM_DEVICE);
2073 if (!ch->rx_ring[i])
aef9ec39
RD		 2074 goto err;
2075 }
2076
4d73f95f 2077 for (i = 0; i < target->queue_size; ++i) {
509c07bc
BVA		 2078 ch->tx_ring[i] = srp_alloc_iu(target->srp_host,
2079 target->max_iu_len,
2080 GFP_KERNEL, DMA_TO_DEVICE);
2081 if (!ch->tx_ring[i])
aef9ec39 2082 goto err;
dcb4cb85 2083
509c07bc 2084 list_add(&ch->tx_ring[i]->list, &ch->free_tx);
aef9ec39
RD		 2085 }
2086
2087 return 0;
2088
2089err:
4d73f95f 2090 for (i = 0; i < target->queue_size; ++i) {
509c07bc
BVA		 2091 srp_free_iu(target->srp_host, ch->rx_ring[i]);
2092 srp_free_iu(target->srp_host, ch->tx_ring[i]);
aef9ec39
RD		 2093 }
2094
4d73f95f
BVA		 2095
2096err_no_ring:
509c07bc
BVA		 2097 kfree(ch->tx_ring);
2098 ch->tx_ring = NULL;
2099 kfree(ch->rx_ring);
2100 ch->rx_ring = NULL;
4d73f95f 2101
aef9ec39
RD		 2102 return -ENOMEM;
2103}
2104
c9b03c1a
BVA		 2105static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask)
2106{
2107 uint64_t T_tr_ns, max_compl_time_ms;
2108 uint32_t rq_tmo_jiffies;
2109
2110 /*
2111 * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair,
2112 * table 91), both the QP timeout and the retry count have to be set
2113 * for RC QP's during the RTR to RTS transition.
2114 */
2115 WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) !=
2116 (IB_QP_TIMEOUT | IB_QP_RETRY_CNT));
2117
2118 /*
2119 * Set target->rq_tmo_jiffies to one second more than the largest time
2120 * it can take before an error completion is generated. See also
2121 * C9-140..142 in the IBTA spec for more information about how to
2122 * convert the QP Local ACK Timeout value to nanoseconds.
2123 */
2124 T_tr_ns = 4096 * (1ULL << qp_attr->timeout);
2125 max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns;
2126 do_div(max_compl_time_ms, NSEC_PER_MSEC);
2127 rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000);
2128
2129 return rq_tmo_jiffies;
2130}
2131
961e0be8 2132static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
e6300cbd 2133 const struct srp_login_rsp *lrsp,
509c07bc 2134 struct srp_rdma_ch *ch)
961e0be8 2135{
509c07bc 2136 struct srp_target_port *target = ch->target;
961e0be8
DD		 2137 struct ib_qp_attr *qp_attr = NULL;
2138 int attr_mask = 0;
2139 int ret;
2140 int i;
2141
2142 if (lrsp->opcode == SRP_LOGIN_RSP) {
509c07bc
BVA		 2143 ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
2144 ch->req_lim = be32_to_cpu(lrsp->req_lim_delta);
961e0be8
DD		 2145
2146 /*
2147 * Reserve credits for task management so we don't
2148 * bounce requests back to the SCSI mid-layer.
2149 */
2150 target->scsi_host->can_queue
509c07bc 2151 = min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE,
961e0be8 2152 target->scsi_host->can_queue);
4d73f95f
BVA		 2153 target->scsi_host->cmd_per_lun
2154 = min_t(int, target->scsi_host->can_queue,
2155 target->scsi_host->cmd_per_lun);
961e0be8
DD		 2156 } else {
2157 shost_printk(KERN_WARNING, target->scsi_host,
2158 PFX "Unhandled RSP opcode %#x\n", lrsp->opcode);
2159 ret = -ECONNRESET;
2160 goto error;
2161 }
2162
509c07bc
BVA		 2163 if (!ch->rx_ring) {
2164 ret = srp_alloc_iu_bufs(ch);
961e0be8
DD		 2165 if (ret)
2166 goto error;
2167 }
2168
2169 ret = -ENOMEM;
2170 qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
2171 if (!qp_attr)
2172 goto error;
2173
2174 qp_attr->qp_state = IB_QPS_RTR;
2175 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2176 if (ret)
2177 goto error_free;
2178
509c07bc 2179 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
961e0be8
DD		 2180 if (ret)
2181 goto error_free;
2182
4d73f95f 2183 for (i = 0; i < target->queue_size; i++) {
509c07bc
BVA		 2184 struct srp_iu *iu = ch->rx_ring[i];
2185
2186 ret = srp_post_recv(ch, iu);
961e0be8
DD		 2187 if (ret)
2188 goto error_free;
2189 }
2190
2191 qp_attr->qp_state = IB_QPS_RTS;
2192 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2193 if (ret)
2194 goto error_free;
2195
c9b03c1a
BVA		 2196 target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);
2197
509c07bc 2198 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
961e0be8
DD		 2199 if (ret)
2200 goto error_free;
2201
2202 ret = ib_send_cm_rtu(cm_id, NULL, 0);
2203
2204error_free:
2205 kfree(qp_attr);
2206
2207error:
509c07bc 2208 ch->status = ret;
961e0be8
DD		 2209}
2210
aef9ec39
RD		 2211static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
2212 struct ib_cm_event *event,
509c07bc 2213 struct srp_rdma_ch *ch)
aef9ec39 2214{
509c07bc 2215 struct srp_target_port *target = ch->target;
7aa54bd7 2216 struct Scsi_Host *shost = target->scsi_host;
aef9ec39
RD		 2217 struct ib_class_port_info *cpi;
2218 int opcode;
2219
2220 switch (event->param.rej_rcvd.reason) {
2221 case IB_CM_REJ_PORT_CM_REDIRECT:
2222 cpi = event->param.rej_rcvd.ari;
509c07bc
BVA		 2223 ch->path.dlid = cpi->redirect_lid;
2224 ch->path.pkey = cpi->redirect_pkey;
aef9ec39 2225 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
509c07bc 2226 memcpy(ch->path.dgid.raw, cpi->redirect_gid, 16);
aef9ec39 2227
509c07bc 2228 ch->status = ch->path.dlid ?
aef9ec39
RD		 2229 SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
2230 break;
2231
2232 case IB_CM_REJ_PORT_REDIRECT:
5d7cbfd6 2233 if (srp_target_is_topspin(target)) {
aef9ec39
RD		 2234 /*
2235 * Topspin/Cisco SRP gateways incorrectly send
2236 * reject reason code 25 when they mean 24
2237 * (port redirect).
2238 */
509c07bc 2239 memcpy(ch->path.dgid.raw,
aef9ec39
RD		 2240 event->param.rej_rcvd.ari, 16);
2241
7aa54bd7
DD		 2242 shost_printk(KERN_DEBUG, shost,
2243 PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
509c07bc
BVA		 2244 be64_to_cpu(ch->path.dgid.global.subnet_prefix),
2245 be64_to_cpu(ch->path.dgid.global.interface_id));
aef9ec39 2246
509c07bc 2247 ch->status = SRP_PORT_REDIRECT;
aef9ec39 2248 } else {
7aa54bd7
DD		 2249 shost_printk(KERN_WARNING, shost,
2250 " REJ reason: IB_CM_REJ_PORT_REDIRECT\n");
509c07bc 2251 ch->status = -ECONNRESET;
aef9ec39
RD		 2252 }
2253 break;
2254
2255 case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
7aa54bd7
DD		 2256 shost_printk(KERN_WARNING, shost,
2257 " REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
509c07bc 2258 ch->status = -ECONNRESET;
aef9ec39
RD		 2259 break;
2260
2261 case IB_CM_REJ_CONSUMER_DEFINED:
2262 opcode = *(u8 *) event->private_data;
2263 if (opcode == SRP_LOGIN_REJ) {
2264 struct srp_login_rej *rej = event->private_data;
2265 u32 reason = be32_to_cpu(rej->reason);
2266
2267 if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
7aa54bd7
DD		 2268 shost_printk(KERN_WARNING, shost,
2269 PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
aef9ec39 2270 else
e7ffde01
BVA		 2271 shost_printk(KERN_WARNING, shost, PFX
2272 "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n",
747fe000
BVA		 2273 target->sgid.raw,
2274 target->orig_dgid.raw, reason);
aef9ec39 2275 } else
7aa54bd7
DD		 2276 shost_printk(KERN_WARNING, shost,
2277 " REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
2278 " opcode 0x%02x\n", opcode);
509c07bc 2279 ch->status = -ECONNRESET;
aef9ec39
RD		 2280 break;
2281
9fe4bcf4
DD		 2282 case IB_CM_REJ_STALE_CONN:
2283 shost_printk(KERN_WARNING, shost, " REJ reason: stale connection\n");
509c07bc 2284 ch->status = SRP_STALE_CONN;
9fe4bcf4
DD		 2285 break;
2286
aef9ec39 2287 default:
7aa54bd7
DD		 2288 shost_printk(KERN_WARNING, shost, " REJ reason 0x%x\n",
2289 event->param.rej_rcvd.reason);
509c07bc 2290 ch->status = -ECONNRESET;
aef9ec39
RD		 2291 }
2292}
2293
2294static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2295{
509c07bc
BVA		 2296 struct srp_rdma_ch *ch = cm_id->context;
2297 struct srp_target_port *target = ch->target;
aef9ec39 2298 int comp = 0;
aef9ec39
RD		 2299
2300 switch (event->event) {
2301 case IB_CM_REQ_ERROR:
7aa54bd7
DD		 2302 shost_printk(KERN_DEBUG, target->scsi_host,
2303 PFX "Sending CM REQ failed\n");
aef9ec39 2304 comp = 1;
509c07bc 2305 ch->status = -ECONNRESET;
aef9ec39
RD		 2306 break;
2307
2308 case IB_CM_REP_RECEIVED:
2309 comp = 1;
509c07bc 2310 srp_cm_rep_handler(cm_id, event->private_data, ch);
aef9ec39
RD		 2311 break;
2312
2313 case IB_CM_REJ_RECEIVED:
7aa54bd7 2314 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
aef9ec39
RD		 2315 comp = 1;
2316
509c07bc 2317 srp_cm_rej_handler(cm_id, event, ch);
aef9ec39
RD		 2318 break;
2319
b7ac4ab4 2320 case IB_CM_DREQ_RECEIVED:
7aa54bd7
DD		 2321 shost_printk(KERN_WARNING, target->scsi_host,
2322 PFX "DREQ received - connection closed\n");
c014c8cd 2323 ch->connected = false;
b7ac4ab4 2324 if (ib_send_cm_drep(cm_id, NULL, 0))
7aa54bd7
DD		 2325 shost_printk(KERN_ERR, target->scsi_host,
2326 PFX "Sending CM DREP failed\n");
c1120f89 2327 queue_work(system_long_wq, &target->tl_err_work);
aef9ec39
RD		 2328 break;
2329
2330 case IB_CM_TIMEWAIT_EXIT:
7aa54bd7
DD		 2331 shost_printk(KERN_ERR, target->scsi_host,
2332 PFX "connection closed\n");
ac72d766 2333 comp = 1;
aef9ec39 2334
509c07bc 2335 ch->status = 0;
aef9ec39
RD		 2336 break;
2337
b7ac4ab4
IR		 2338 case IB_CM_MRA_RECEIVED:
2339 case IB_CM_DREQ_ERROR:
2340 case IB_CM_DREP_RECEIVED:
2341 break;
2342
aef9ec39 2343 default:
7aa54bd7
DD		 2344 shost_printk(KERN_WARNING, target->scsi_host,
2345 PFX "Unhandled CM event %d\n", event->event);
aef9ec39
RD		 2346 break;
2347 }
2348
2349 if (comp)
509c07bc 2350 complete(&ch->done);
aef9ec39 2351
aef9ec39
RD		 2352 return 0;
2353}
2354
71444b97
JW		 2355/**
2356 * srp_change_queue_depth - setting device queue depth
2357 * @sdev: scsi device struct
2358 * @qdepth: requested queue depth
71444b97
JW		 2359 *
2360 * Returns queue depth.
2361 */
2362static int
db5ed4df 2363srp_change_queue_depth(struct scsi_device *sdev, int qdepth)
71444b97 2364{
c40ecc12 2365 if (!sdev->tagged_supported)
1e6f2416 2366 qdepth = 1;
db5ed4df 2367 return scsi_change_queue_depth(sdev, qdepth);
71444b97
JW		 2368}
2369
985aa495
BVA		 2370static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag, u64 lun,
2371 u8 func)
aef9ec39 2372{
509c07bc 2373 struct srp_target_port *target = ch->target;
a95cadb9 2374 struct srp_rport *rport = target->rport;
19081f31 2375 struct ib_device *dev = target->srp_host->srp_dev->dev;
aef9ec39
RD		 2376 struct srp_iu *iu;
2377 struct srp_tsk_mgmt *tsk_mgmt;
aef9ec39 2378
c014c8cd 2379 if (!ch->connected || target->qp_in_error)
3780d1f0
BVA		 2380 return -1;
2381
509c07bc 2382 init_completion(&ch->tsk_mgmt_done);
aef9ec39 2383
a95cadb9 2384 /*
509c07bc 2385 * Lock the rport mutex to avoid that srp_create_ch_ib() is
a95cadb9
BVA		 2386 * invoked while a task management function is being sent.
2387 */
2388 mutex_lock(&rport->mutex);
509c07bc
BVA		 2389 spin_lock_irq(&ch->lock);
2390 iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT);
2391 spin_unlock_irq(&ch->lock);
76c75b25 2392
a95cadb9
BVA		 2393 if (!iu) {
2394 mutex_unlock(&rport->mutex);
2395
76c75b25 2396 return -1;
a95cadb9 2397 }
aef9ec39 2398
19081f31
DD		 2399 ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt,
2400 DMA_TO_DEVICE);
aef9ec39
RD		 2401 tsk_mgmt = iu->buf;
2402 memset(tsk_mgmt, 0, sizeof *tsk_mgmt);
2403
2404 tsk_mgmt->opcode = SRP_TSK_MGMT;
985aa495 2405 int_to_scsilun(lun, &tsk_mgmt->lun);
f8b6e31e 2406 tsk_mgmt->tag = req_tag | SRP_TAG_TSK_MGMT;
aef9ec39 2407 tsk_mgmt->tsk_mgmt_func = func;
f8b6e31e 2408 tsk_mgmt->task_tag = req_tag;
aef9ec39 2409
19081f31
DD		 2410 ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt,
2411 DMA_TO_DEVICE);
509c07bc
BVA		 2412 if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) {
2413 srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT);
a95cadb9
BVA		 2414 mutex_unlock(&rport->mutex);
2415
76c75b25
BVA		 2416 return -1;
2417 }
a95cadb9 2418 mutex_unlock(&rport->mutex);
d945e1df 2419
509c07bc 2420 if (!wait_for_completion_timeout(&ch->tsk_mgmt_done,
aef9ec39 2421 msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS)))
d945e1df 2422 return -1;
aef9ec39 2423
d945e1df 2424 return 0;
d945e1df
RD		 2425}
2426
aef9ec39
RD		 2427static int srp_abort(struct scsi_cmnd *scmnd)
2428{
d945e1df 2429 struct srp_target_port *target = host_to_target(scmnd->device->host);
f8b6e31e 2430 struct srp_request *req = (struct srp_request *) scmnd->host_scribble;
77f2c1a4 2431 u32 tag;
d92c0da7 2432 u16 ch_idx;
509c07bc 2433 struct srp_rdma_ch *ch;
086f44f5 2434 int ret;
d945e1df 2435
7aa54bd7 2436 shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n");
aef9ec39 2437
d92c0da7 2438 if (!req)
99b6697a 2439 return SUCCESS;
77f2c1a4 2440 tag = blk_mq_unique_tag(scmnd->request);
d92c0da7
BVA		 2441 ch_idx = blk_mq_unique_tag_to_hwq(tag);
2442 if (WARN_ON_ONCE(ch_idx >= target->ch_count))
2443 return SUCCESS;
2444 ch = &target->ch[ch_idx];
2445 if (!srp_claim_req(ch, req, NULL, scmnd))
2446 return SUCCESS;
2447 shost_printk(KERN_ERR, target->scsi_host,
2448 "Sending SRP abort for tag %#x\n", tag);
77f2c1a4 2449 if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun,
80d5e8a2 2450 SRP_TSK_ABORT_TASK) == 0)
086f44f5 2451 ret = SUCCESS;
ed9b2264 2452 else if (target->rport->state == SRP_RPORT_LOST)
99e1c139 2453 ret = FAST_IO_FAIL;
086f44f5
BVA		 2454 else
2455 ret = FAILED;
509c07bc 2456 srp_free_req(ch, req, scmnd, 0);
22032991 2457 scmnd->result = DID_ABORT << 16;
d8536670 2458 scmnd->scsi_done(scmnd);
d945e1df 2459
086f44f5 2460 return ret;
aef9ec39
RD		 2461}
2462
2463static int srp_reset_device(struct scsi_cmnd *scmnd)
2464{
d945e1df 2465 struct srp_target_port *target = host_to_target(scmnd->device->host);
d92c0da7 2466 struct srp_rdma_ch *ch;
536ae14e 2467 int i;
d945e1df 2468
7aa54bd7 2469 shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n");
aef9ec39 2470
d92c0da7 2471 ch = &target->ch[0];
509c07bc 2472 if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun,
f8b6e31e 2473 SRP_TSK_LUN_RESET))
d945e1df 2474 return FAILED;
509c07bc 2475 if (ch->tsk_mgmt_status)
d945e1df
RD		 2476 return FAILED;
2477
d92c0da7
BVA		 2478 for (i = 0; i < target->ch_count; i++) {
2479 ch = &target->ch[i];
2480 for (i = 0; i < target->req_ring_size; ++i) {
2481 struct srp_request *req = &ch->req_ring[i];
509c07bc 2482
d92c0da7
BVA		 2483 srp_finish_req(ch, req, scmnd->device, DID_RESET << 16);
2484 }
536ae14e 2485 }
d945e1df 2486
d945e1df 2487 return SUCCESS;
aef9ec39
RD		 2488}
2489
2490static int srp_reset_host(struct scsi_cmnd *scmnd)
2491{
2492 struct srp_target_port *target = host_to_target(scmnd->device->host);
aef9ec39 2493
7aa54bd7 2494 shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n");
aef9ec39 2495
ed9b2264 2496 return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED;
aef9ec39
RD		 2497}
2498
c9b03c1a
BVA		 2499static int srp_slave_configure(struct scsi_device *sdev)
2500{
2501 struct Scsi_Host *shost = sdev->host;
2502 struct srp_target_port *target = host_to_target(shost);
2503 struct request_queue *q = sdev->request_queue;
2504 unsigned long timeout;
2505
2506 if (sdev->type == TYPE_DISK) {
2507 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies);
2508 blk_queue_rq_timeout(q, timeout);
2509 }
2510
2511 return 0;
2512}
2513
ee959b00
TJ		 2514static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr,
2515 char *buf)
6ecb0c84 2516{
ee959b00 2517 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2518
45c37cad 2519 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->id_ext));
6ecb0c84
RD		 2520}
2521
ee959b00
TJ		 2522static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr,
2523 char *buf)
6ecb0c84 2524{
ee959b00 2525 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2526
45c37cad 2527 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid));
6ecb0c84
RD		 2528}
2529
ee959b00
TJ		 2530static ssize_t show_service_id(struct device *dev,
2531 struct device_attribute *attr, char *buf)
6ecb0c84 2532{
ee959b00 2533 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2534
45c37cad 2535 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->service_id));
6ecb0c84
RD		 2536}
2537
ee959b00
TJ		 2538static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
2539 char *buf)
6ecb0c84 2540{
ee959b00 2541 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2542
747fe000 2543 return sprintf(buf, "0x%04x\n", be16_to_cpu(target->pkey));
6ecb0c84
RD		 2544}
2545
848b3082
BVA		 2546static ssize_t show_sgid(struct device *dev, struct device_attribute *attr,
2547 char *buf)
2548{
2549 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2550
747fe000 2551 return sprintf(buf, "%pI6\n", target->sgid.raw);
848b3082
BVA		 2552}
2553
ee959b00
TJ		 2554static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
2555 char *buf)
6ecb0c84 2556{
ee959b00 2557 struct srp_target_port *target = host_to_target(class_to_shost(dev));
d92c0da7 2558 struct srp_rdma_ch *ch = &target->ch[0];
6ecb0c84 2559
509c07bc 2560 return sprintf(buf, "%pI6\n", ch->path.dgid.raw);
6ecb0c84
RD		 2561}
2562
ee959b00
TJ		 2563static ssize_t show_orig_dgid(struct device *dev,
2564 struct device_attribute *attr, char *buf)
3633b3d0 2565{
ee959b00 2566 struct srp_target_port *target = host_to_target(class_to_shost(dev));
3633b3d0 2567
747fe000 2568 return sprintf(buf, "%pI6\n", target->orig_dgid.raw);
3633b3d0
IR		 2569}
2570
89de7486
BVA		 2571static ssize_t show_req_lim(struct device *dev,
2572 struct device_attribute *attr, char *buf)
2573{
2574 struct srp_target_port *target = host_to_target(class_to_shost(dev));
d92c0da7
BVA		 2575 struct srp_rdma_ch *ch;
2576 int i, req_lim = INT_MAX;
89de7486 2577
d92c0da7
BVA		 2578 for (i = 0; i < target->ch_count; i++) {
2579 ch = &target->ch[i];
2580 req_lim = min(req_lim, ch->req_lim);
2581 }
2582 return sprintf(buf, "%d\n", req_lim);
89de7486
BVA		 2583}
2584
ee959b00
TJ		 2585static ssize_t show_zero_req_lim(struct device *dev,
2586 struct device_attribute *attr, char *buf)
6bfa24fa 2587{
ee959b00 2588 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6bfa24fa 2589
6bfa24fa
RD		 2590 return sprintf(buf, "%d\n", target->zero_req_lim);
2591}
2592
ee959b00
TJ		 2593static ssize_t show_local_ib_port(struct device *dev,
2594 struct device_attribute *attr, char *buf)
ded7f1a1 2595{
ee959b00 2596 struct srp_target_port *target = host_to_target(class_to_shost(dev));
ded7f1a1
IR		 2597
2598 return sprintf(buf, "%d\n", target->srp_host->port);
2599}
2600
ee959b00
TJ		 2601static ssize_t show_local_ib_device(struct device *dev,
2602 struct device_attribute *attr, char *buf)
ded7f1a1 2603{
ee959b00 2604 struct srp_target_port *target = host_to_target(class_to_shost(dev));
ded7f1a1 2605
05321937 2606 return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
ded7f1a1
IR		 2607}
2608
d92c0da7
BVA		 2609static ssize_t show_ch_count(struct device *dev, struct device_attribute *attr,
2610 char *buf)
2611{
2612 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2613
2614 return sprintf(buf, "%d\n", target->ch_count);
2615}
2616
4b5e5f41
BVA		 2617static ssize_t show_comp_vector(struct device *dev,
2618 struct device_attribute *attr, char *buf)
2619{
2620 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2621
2622 return sprintf(buf, "%d\n", target->comp_vector);
2623}
2624
7bb312e4
VP		 2625static ssize_t show_tl_retry_count(struct device *dev,
2626 struct device_attribute *attr, char *buf)
2627{
2628 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2629
2630 return sprintf(buf, "%d\n", target->tl_retry_count);
2631}
2632
49248644
DD		 2633static ssize_t show_cmd_sg_entries(struct device *dev,
2634 struct device_attribute *attr, char *buf)
2635{
2636 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2637
2638 return sprintf(buf, "%u\n", target->cmd_sg_cnt);
2639}
2640
c07d424d
DD		 2641static ssize_t show_allow_ext_sg(struct device *dev,
2642 struct device_attribute *attr, char *buf)
2643{
2644 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2645
2646 return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
2647}
2648
ee959b00
TJ		 2649static DEVICE_ATTR(id_ext, S_IRUGO, show_id_ext, NULL);
2650static DEVICE_ATTR(ioc_guid, S_IRUGO, show_ioc_guid, NULL);
2651static DEVICE_ATTR(service_id, S_IRUGO, show_service_id, NULL);
2652static DEVICE_ATTR(pkey, S_IRUGO, show_pkey, NULL);
848b3082 2653static DEVICE_ATTR(sgid, S_IRUGO, show_sgid, NULL);
ee959b00
TJ		 2654static DEVICE_ATTR(dgid, S_IRUGO, show_dgid, NULL);
2655static DEVICE_ATTR(orig_dgid, S_IRUGO, show_orig_dgid, NULL);
89de7486 2656static DEVICE_ATTR(req_lim, S_IRUGO, show_req_lim, NULL);
ee959b00
TJ		 2657static DEVICE_ATTR(zero_req_lim, S_IRUGO, show_zero_req_lim, NULL);
2658static DEVICE_ATTR(local_ib_port, S_IRUGO, show_local_ib_port, NULL);
2659static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
d92c0da7 2660static DEVICE_ATTR(ch_count, S_IRUGO, show_ch_count, NULL);
4b5e5f41 2661static DEVICE_ATTR(comp_vector, S_IRUGO, show_comp_vector, NULL);
7bb312e4 2662static DEVICE_ATTR(tl_retry_count, S_IRUGO, show_tl_retry_count, NULL);
49248644 2663static DEVICE_ATTR(cmd_sg_entries, S_IRUGO, show_cmd_sg_entries, NULL);
c07d424d 2664static DEVICE_ATTR(allow_ext_sg, S_IRUGO, show_allow_ext_sg, NULL);
ee959b00
TJ		 2665
2666static struct device_attribute *srp_host_attrs[] = {
2667 &dev_attr_id_ext,
2668 &dev_attr_ioc_guid,
2669 &dev_attr_service_id,
2670 &dev_attr_pkey,
848b3082 2671 &dev_attr_sgid,
ee959b00
TJ		 2672 &dev_attr_dgid,
2673 &dev_attr_orig_dgid,
89de7486 2674 &dev_attr_req_lim,
ee959b00
TJ		 2675 &dev_attr_zero_req_lim,
2676 &dev_attr_local_ib_port,
2677 &dev_attr_local_ib_device,
d92c0da7 2678 &dev_attr_ch_count,
4b5e5f41 2679 &dev_attr_comp_vector,
7bb312e4 2680 &dev_attr_tl_retry_count,
49248644 2681 &dev_attr_cmd_sg_entries,
c07d424d 2682 &dev_attr_allow_ext_sg,
6ecb0c84
RD		 2683 NULL
2684};
2685
aef9ec39
RD		 2686static struct scsi_host_template srp_template = {
2687 .module = THIS_MODULE,
b7f008fd
RD		 2688 .name = "InfiniBand SRP initiator",
2689 .proc_name = DRV_NAME,
c9b03c1a 2690 .slave_configure = srp_slave_configure,
aef9ec39
RD		 2691 .info = srp_target_info,
2692 .queuecommand = srp_queuecommand,
71444b97 2693 .change_queue_depth = srp_change_queue_depth,
aef9ec39
RD		 2694 .eh_abort_handler = srp_abort,
2695 .eh_device_reset_handler = srp_reset_device,
2696 .eh_host_reset_handler = srp_reset_host,
2742c1da 2697 .skip_settle_delay = true,
49248644 2698 .sg_tablesize = SRP_DEF_SG_TABLESIZE,
4d73f95f 2699 .can_queue = SRP_DEFAULT_CMD_SQ_SIZE,
aef9ec39 2700 .this_id = -1,
4d73f95f 2701 .cmd_per_lun = SRP_DEFAULT_CMD_SQ_SIZE,
6ecb0c84 2702 .use_clustering = ENABLE_CLUSTERING,
77f2c1a4
BVA		 2703 .shost_attrs = srp_host_attrs,
2704 .use_blk_tags = 1,
c40ecc12 2705 .track_queue_depth = 1,
aef9ec39
RD		 2706};
2707
34aa654e
BVA		 2708static int srp_sdev_count(struct Scsi_Host *host)
2709{
2710 struct scsi_device *sdev;
2711 int c = 0;
2712
2713 shost_for_each_device(sdev, host)
2714 c++;
2715
2716 return c;
2717}
2718
bc44bd1d
BVA		 2719/*
2720 * Return values:
2721 * < 0 upon failure. Caller is responsible for SRP target port cleanup.
2722 * 0 and target->state == SRP_TARGET_REMOVED if asynchronous target port
2723 * removal has been scheduled.
2724 * 0 and target->state != SRP_TARGET_REMOVED upon success.
2725 */
aef9ec39
RD		 2726static int srp_add_target(struct srp_host *host, struct srp_target_port *target)
2727{
3236822b
FT		 2728 struct srp_rport_identifiers ids;
2729 struct srp_rport *rport;
2730
34aa654e 2731 target->state = SRP_TARGET_SCANNING;
aef9ec39 2732 sprintf(target->target_name, "SRP.T10:%016llX",
45c37cad 2733 be64_to_cpu(target->id_ext));
aef9ec39 2734
05321937 2735 if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dma_device))
aef9ec39
RD		 2736 return -ENODEV;
2737
3236822b
FT		 2738 memcpy(ids.port_id, &target->id_ext, 8);
2739 memcpy(ids.port_id + 8, &target->ioc_guid, 8);
aebd5e47 2740 ids.roles = SRP_RPORT_ROLE_TARGET;
3236822b
FT		 2741 rport = srp_rport_add(target->scsi_host, &ids);
2742 if (IS_ERR(rport)) {
2743 scsi_remove_host(target->scsi_host);
2744 return PTR_ERR(rport);
2745 }
2746
dc1bdbd9 2747 rport->lld_data = target;
9dd69a60 2748 target->rport = rport;
dc1bdbd9 2749
b3589fd4 2750 spin_lock(&host->target_lock);
aef9ec39 2751 list_add_tail(&target->list, &host->target_list);
b3589fd4 2752 spin_unlock(&host->target_lock);
aef9ec39 2753
aef9ec39 2754 scsi_scan_target(&target->scsi_host->shost_gendev,
1962a4a1 2755 0, target->scsi_id, SCAN_WILD_CARD, 0);
aef9ec39 2756
c014c8cd
BVA		 2757 if (srp_connected_ch(target) < target->ch_count ||
2758 target->qp_in_error) {
34aa654e
BVA		 2759 shost_printk(KERN_INFO, target->scsi_host,
2760 PFX "SCSI scan failed - removing SCSI host\n");
2761 srp_queue_remove_work(target);
2762 goto out;
2763 }
2764
2765 pr_debug(PFX "%s: SCSI scan succeeded - detected %d LUNs\n",
2766 dev_name(&target->scsi_host->shost_gendev),
2767 srp_sdev_count(target->scsi_host));
2768
2769 spin_lock_irq(&target->lock);
2770 if (target->state == SRP_TARGET_SCANNING)
2771 target->state = SRP_TARGET_LIVE;
2772 spin_unlock_irq(&target->lock);
2773
2774out:
aef9ec39
RD		 2775 return 0;
2776}
2777
ee959b00 2778static void srp_release_dev(struct device *dev)
aef9ec39
RD		 2779{
2780 struct srp_host *host =
ee959b00 2781 container_of(dev, struct srp_host, dev);
aef9ec39
RD		 2782
2783 complete(&host->released);
2784}
2785
2786static struct class srp_class = {
2787 .name = "infiniband_srp",
ee959b00 2788 .dev_release = srp_release_dev
aef9ec39
RD		 2789};
2790
96fc248a
BVA		 2791/**
2792 * srp_conn_unique() - check whether the connection to a target is unique
af24663b
BVA		 2793 * @host: SRP host.
2794 * @target: SRP target port.
96fc248a
BVA		 2795 */
2796static bool srp_conn_unique(struct srp_host *host,
2797 struct srp_target_port *target)
2798{
2799 struct srp_target_port *t;
2800 bool ret = false;
2801
2802 if (target->state == SRP_TARGET_REMOVED)
2803 goto out;
2804
2805 ret = true;
2806
2807 spin_lock(&host->target_lock);
2808 list_for_each_entry(t, &host->target_list, list) {
2809 if (t != target &&
2810 target->id_ext == t->id_ext &&
2811 target->ioc_guid == t->ioc_guid &&
2812 target->initiator_ext == t->initiator_ext) {
2813 ret = false;
2814 break;
2815 }
2816 }
2817 spin_unlock(&host->target_lock);
2818
2819out:
2820 return ret;
2821}
2822
aef9ec39
RD		 2823/*
2824 * Target ports are added by writing
2825 *
2826 * id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>,
2827 * pkey=<P_Key>,service_id=<service ID>
2828 *
2829 * to the add_target sysfs attribute.
2830 */
2831enum {
2832 SRP_OPT_ERR = 0,
2833 SRP_OPT_ID_EXT = 1 << 0,
2834 SRP_OPT_IOC_GUID = 1 << 1,
2835 SRP_OPT_DGID = 1 << 2,
2836 SRP_OPT_PKEY = 1 << 3,
2837 SRP_OPT_SERVICE_ID = 1 << 4,
2838 SRP_OPT_MAX_SECT = 1 << 5,
52fb2b50 2839 SRP_OPT_MAX_CMD_PER_LUN = 1 << 6,
0c0450db 2840 SRP_OPT_IO_CLASS = 1 << 7,
01cb9bcb 2841 SRP_OPT_INITIATOR_EXT = 1 << 8,
49248644 2842 SRP_OPT_CMD_SG_ENTRIES = 1 << 9,
c07d424d
DD		 2843 SRP_OPT_ALLOW_EXT_SG = 1 << 10,
2844 SRP_OPT_SG_TABLESIZE = 1 << 11,
4b5e5f41 2845 SRP_OPT_COMP_VECTOR = 1 << 12,
7bb312e4 2846 SRP_OPT_TL_RETRY_COUNT = 1 << 13,
4d73f95f 2847 SRP_OPT_QUEUE_SIZE = 1 << 14,
aef9ec39
RD		 2848 SRP_OPT_ALL = (SRP_OPT_ID_EXT |
2849 SRP_OPT_IOC_GUID |
2850 SRP_OPT_DGID |
2851 SRP_OPT_PKEY |
2852 SRP_OPT_SERVICE_ID),
2853};
2854
a447c093 2855static const match_table_t srp_opt_tokens = {
52fb2b50
VP		 2856 { SRP_OPT_ID_EXT, "id_ext=%s" },
2857 { SRP_OPT_IOC_GUID, "ioc_guid=%s" },
2858 { SRP_OPT_DGID, "dgid=%s" },
2859 { SRP_OPT_PKEY, "pkey=%x" },
2860 { SRP_OPT_SERVICE_ID, "service_id=%s" },
2861 { SRP_OPT_MAX_SECT, "max_sect=%d" },
2862 { SRP_OPT_MAX_CMD_PER_LUN, "max_cmd_per_lun=%d" },
0c0450db 2863 { SRP_OPT_IO_CLASS, "io_class=%x" },
01cb9bcb 2864 { SRP_OPT_INITIATOR_EXT, "initiator_ext=%s" },
49248644 2865 { SRP_OPT_CMD_SG_ENTRIES, "cmd_sg_entries=%u" },
c07d424d
DD		 2866 { SRP_OPT_ALLOW_EXT_SG, "allow_ext_sg=%u" },
2867 { SRP_OPT_SG_TABLESIZE, "sg_tablesize=%u" },
4b5e5f41 2868 { SRP_OPT_COMP_VECTOR, "comp_vector=%u" },
7bb312e4 2869 { SRP_OPT_TL_RETRY_COUNT, "tl_retry_count=%u" },
4d73f95f 2870 { SRP_OPT_QUEUE_SIZE, "queue_size=%d" },
52fb2b50 2871 { SRP_OPT_ERR, NULL }
aef9ec39
RD		 2872};
2873
2874static int srp_parse_options(const char *buf, struct srp_target_port *target)
2875{
2876 char *options, *sep_opt;
2877 char *p;
2878 char dgid[3];
2879 substring_t args[MAX_OPT_ARGS];
2880 int opt_mask = 0;
2881 int token;
2882 int ret = -EINVAL;
2883 int i;
2884
2885 options = kstrdup(buf, GFP_KERNEL);
2886 if (!options)
2887 return -ENOMEM;
2888
2889 sep_opt = options;
7dcf9c19 2890 while ((p = strsep(&sep_opt, ",\n")) != NULL) {
aef9ec39
RD		 2891 if (!*p)
2892 continue;
2893
2894 token = match_token(p, srp_opt_tokens, args);
2895 opt_mask |= token;
2896
2897 switch (token) {
2898 case SRP_OPT_ID_EXT:
2899 p = match_strdup(args);
a20f3a6d
IR		 2900 if (!p) {
2901 ret = -ENOMEM;
2902 goto out;
2903 }
aef9ec39
RD		 2904 target->id_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
2905 kfree(p);
2906 break;
2907
2908 case SRP_OPT_IOC_GUID:
2909 p = match_strdup(args);
a20f3a6d
IR		 2910 if (!p) {
2911 ret = -ENOMEM;
2912 goto out;
2913 }
aef9ec39
RD		 2914 target->ioc_guid = cpu_to_be64(simple_strtoull(p, NULL, 16));
2915 kfree(p);
2916 break;
2917
2918 case SRP_OPT_DGID:
2919 p = match_strdup(args);
a20f3a6d
IR		 2920 if (!p) {
2921 ret = -ENOMEM;
2922 goto out;
2923 }
aef9ec39 2924 if (strlen(p) != 32) {
e0bda7d8 2925 pr_warn("bad dest GID parameter '%s'\n", p);
ce1823f0 2926 kfree(p);
aef9ec39
RD		 2927 goto out;
2928 }
2929
2930 for (i = 0; i < 16; ++i) {
747fe000
BVA		 2931 strlcpy(dgid, p + i * 2, sizeof(dgid));
2932 if (sscanf(dgid, "%hhx",
2933 &target->orig_dgid.raw[i]) < 1) {
2934 ret = -EINVAL;
2935 kfree(p);
2936 goto out;
2937 }
aef9ec39 2938 }
bf17c1c7 2939 kfree(p);
aef9ec39
RD		 2940 break;
2941
2942 case SRP_OPT_PKEY:
2943 if (match_hex(args, &token)) {
e0bda7d8 2944 pr_warn("bad P_Key parameter '%s'\n", p);
aef9ec39
RD		 2945 goto out;
2946 }
747fe000 2947 target->pkey = cpu_to_be16(token);
aef9ec39
RD		 2948 break;
2949
2950 case SRP_OPT_SERVICE_ID:
2951 p = match_strdup(args);
a20f3a6d
IR		 2952 if (!p) {
2953 ret = -ENOMEM;
2954 goto out;
2955 }
aef9ec39
RD		 2956 target->service_id = cpu_to_be64(simple_strtoull(p, NULL, 16));
2957 kfree(p);
2958 break;
2959
2960 case SRP_OPT_MAX_SECT:
2961 if (match_int(args, &token)) {
e0bda7d8 2962 pr_warn("bad max sect parameter '%s'\n", p);
aef9ec39
RD		 2963 goto out;
2964 }
2965 target->scsi_host->max_sectors = token;
2966 break;
2967
4d73f95f
BVA		 2968 case SRP_OPT_QUEUE_SIZE:
2969 if (match_int(args, &token) || token < 1) {
2970 pr_warn("bad queue_size parameter '%s'\n", p);
2971 goto out;
2972 }
2973 target->scsi_host->can_queue = token;
2974 target->queue_size = token + SRP_RSP_SQ_SIZE +
2975 SRP_TSK_MGMT_SQ_SIZE;
2976 if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
2977 target->scsi_host->cmd_per_lun = token;
2978 break;
2979
52fb2b50 2980 case SRP_OPT_MAX_CMD_PER_LUN:
4d73f95f 2981 if (match_int(args, &token) || token < 1) {
e0bda7d8
BVA		 2982 pr_warn("bad max cmd_per_lun parameter '%s'\n",
2983 p);
52fb2b50
VP		 2984 goto out;
2985 }
4d73f95f 2986 target->scsi_host->cmd_per_lun = token;
52fb2b50
VP		 2987 break;
2988
0c0450db
R		 2989 case SRP_OPT_IO_CLASS:
2990 if (match_hex(args, &token)) {
e0bda7d8 2991 pr_warn("bad IO class parameter '%s'\n", p);
0c0450db
R		 2992 goto out;
2993 }
2994 if (token != SRP_REV10_IB_IO_CLASS &&
2995 token != SRP_REV16A_IB_IO_CLASS) {
e0bda7d8
BVA		 2996 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
2997 token, SRP_REV10_IB_IO_CLASS,
2998 SRP_REV16A_IB_IO_CLASS);
0c0450db
R		 2999 goto out;
3000 }
3001 target->io_class = token;
3002 break;
3003
01cb9bcb
IR		 3004 case SRP_OPT_INITIATOR_EXT:
3005 p = match_strdup(args);
a20f3a6d
IR		 3006 if (!p) {
3007 ret = -ENOMEM;
3008 goto out;
3009 }
01cb9bcb
IR		 3010 target->initiator_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
3011 kfree(p);
3012 break;
3013
49248644
DD		 3014 case SRP_OPT_CMD_SG_ENTRIES:
3015 if (match_int(args, &token) || token < 1 || token > 255) {
e0bda7d8
BVA		 3016 pr_warn("bad max cmd_sg_entries parameter '%s'\n",
3017 p);
49248644
DD		 3018 goto out;
3019 }
3020 target->cmd_sg_cnt = token;
3021 break;
3022
c07d424d
DD		 3023 case SRP_OPT_ALLOW_EXT_SG:
3024 if (match_int(args, &token)) {
e0bda7d8 3025 pr_warn("bad allow_ext_sg parameter '%s'\n", p);
c07d424d
DD		 3026 goto out;
3027 }
3028 target->allow_ext_sg = !!token;
3029 break;
3030
3031 case SRP_OPT_SG_TABLESIZE:
3032 if (match_int(args, &token) || token < 1 ||
3033 token > SCSI_MAX_SG_CHAIN_SEGMENTS) {
e0bda7d8
BVA		 3034 pr_warn("bad max sg_tablesize parameter '%s'\n",
3035 p);
c07d424d
DD		 3036 goto out;
3037 }
3038 target->sg_tablesize = token;
3039 break;
3040
4b5e5f41
BVA		 3041 case SRP_OPT_COMP_VECTOR:
3042 if (match_int(args, &token) || token < 0) {
3043 pr_warn("bad comp_vector parameter '%s'\n", p);
3044 goto out;
3045 }
3046 target->comp_vector = token;
3047 break;
3048
7bb312e4
VP		 3049 case SRP_OPT_TL_RETRY_COUNT:
3050 if (match_int(args, &token) || token < 2 || token > 7) {
3051 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
3052 p);
3053 goto out;
3054 }
3055 target->tl_retry_count = token;
3056 break;
3057
aef9ec39 3058 default:
e0bda7d8
BVA		 3059 pr_warn("unknown parameter or missing value '%s' in target creation request\n",
3060 p);
aef9ec39
RD		 3061 goto out;
3062 }
3063 }
3064
3065 if ((opt_mask & SRP_OPT_ALL) == SRP_OPT_ALL)
3066 ret = 0;
3067 else
3068 for (i = 0; i < ARRAY_SIZE(srp_opt_tokens); ++i)
3069 if ((srp_opt_tokens[i].token & SRP_OPT_ALL) &&
3070 !(srp_opt_tokens[i].token & opt_mask))
e0bda7d8
BVA		 3071 pr_warn("target creation request is missing parameter '%s'\n",
3072 srp_opt_tokens[i].pattern);
aef9ec39 3073
4d73f95f
BVA		 3074 if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue
3075 && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3076 pr_warn("cmd_per_lun = %d > queue_size = %d\n",
3077 target->scsi_host->cmd_per_lun,
3078 target->scsi_host->can_queue);
3079
aef9ec39
RD		 3080out:
3081 kfree(options);
3082 return ret;
3083}
3084
ee959b00
TJ		 3085static ssize_t srp_create_target(struct device *dev,
3086 struct device_attribute *attr,
aef9ec39
RD		 3087 const char *buf, size_t count)
3088{
3089 struct srp_host *host =
ee959b00 3090 container_of(dev, struct srp_host, dev);
aef9ec39
RD		 3091 struct Scsi_Host *target_host;
3092 struct srp_target_port *target;
509c07bc 3093 struct srp_rdma_ch *ch;
d1b4289e
BVA		 3094 struct srp_device *srp_dev = host->srp_dev;
3095 struct ib_device *ibdev = srp_dev->dev;
d92c0da7
BVA		 3096 int ret, node_idx, node, cpu, i;
3097 bool multich = false;
aef9ec39
RD		 3098
3099 target_host = scsi_host_alloc(&srp_template,
3100 sizeof (struct srp_target_port));
3101 if (!target_host)
3102 return -ENOMEM;
3103
49248644 3104 target_host->transportt = ib_srp_transport_template;
fd1b6c4a
BVA		 3105 target_host->max_channel = 0;
3106 target_host->max_id = 1;
985aa495 3107 target_host->max_lun = -1LL;
3c8edf0e 3108 target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb;
5f068992 3109
aef9ec39 3110 target = host_to_target(target_host);
aef9ec39 3111
49248644
DD		 3112 target->io_class = SRP_REV16A_IB_IO_CLASS;
3113 target->scsi_host = target_host;
3114 target->srp_host = host;
e6bf5f48 3115 target->lkey = host->srp_dev->pd->local_dma_lkey;
49248644
DD		 3116 target->rkey = host->srp_dev->mr->rkey;
3117 target->cmd_sg_cnt = cmd_sg_entries;
c07d424d
DD		 3118 target->sg_tablesize = indirect_sg_entries ? : cmd_sg_entries;
3119 target->allow_ext_sg = allow_ext_sg;
7bb312e4 3120 target->tl_retry_count = 7;
4d73f95f 3121 target->queue_size = SRP_DEFAULT_QUEUE_SIZE;
aef9ec39 3122
34aa654e
BVA		 3123 /*
3124 * Avoid that the SCSI host can be removed by srp_remove_target()
3125 * before this function returns.
3126 */
3127 scsi_host_get(target->scsi_host);
3128
2d7091bc
BVA		 3129 mutex_lock(&host->add_target_mutex);
3130
aef9ec39
RD		 3131 ret = srp_parse_options(buf, target);
3132 if (ret)
fb49c8bb 3133 goto out;
aef9ec39 3134
77f2c1a4
BVA		 3135 ret = scsi_init_shared_tag_map(target_host, target_host->can_queue);
3136 if (ret)
fb49c8bb 3137 goto out;
77f2c1a4 3138
4d73f95f
BVA		 3139 target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE;
3140
96fc248a
BVA		 3141 if (!srp_conn_unique(target->srp_host, target)) {
3142 shost_printk(KERN_INFO, target->scsi_host,
3143 PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
3144 be64_to_cpu(target->id_ext),
3145 be64_to_cpu(target->ioc_guid),
3146 be64_to_cpu(target->initiator_ext));
3147 ret = -EEXIST;
fb49c8bb 3148 goto out;
96fc248a
BVA		 3149 }
3150
5cfb1782 3151 if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg &&
d1b4289e 3152 target->cmd_sg_cnt < target->sg_tablesize) {
5cfb1782 3153 pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
c07d424d
DD		 3154 target->sg_tablesize = target->cmd_sg_cnt;
3155 }
3156
3157 target_host->sg_tablesize = target->sg_tablesize;
3158 target->indirect_size = target->sg_tablesize *
3159 sizeof (struct srp_direct_buf);
49248644
DD		 3160 target->max_iu_len = sizeof (struct srp_cmd) +
3161 sizeof (struct srp_indirect_buf) +
3162 target->cmd_sg_cnt * sizeof (struct srp_direct_buf);
3163
c1120f89 3164 INIT_WORK(&target->tl_err_work, srp_tl_err_work);
ef6c49d8 3165 INIT_WORK(&target->remove_work, srp_remove_work);
8f26c9ff 3166 spin_lock_init(&target->lock);
747fe000 3167 ret = ib_query_gid(ibdev, host->port, 0, &target->sgid);
2088ca66 3168 if (ret)
fb49c8bb 3169 goto out;
aef9ec39 3170
d92c0da7
BVA		 3171 ret = -ENOMEM;
3172 target->ch_count = max_t(unsigned, num_online_nodes(),
3173 min(ch_count ? :
3174 min(4 * num_online_nodes(),
3175 ibdev->num_comp_vectors),
3176 num_online_cpus()));
3177 target->ch = kcalloc(target->ch_count, sizeof(*target->ch),
3178 GFP_KERNEL);
3179 if (!target->ch)
fb49c8bb 3180 goto out;
aef9ec39 3181
d92c0da7
BVA		 3182 node_idx = 0;
3183 for_each_online_node(node) {
3184 const int ch_start = (node_idx * target->ch_count /
3185 num_online_nodes());
3186 const int ch_end = ((node_idx + 1) * target->ch_count /
3187 num_online_nodes());
3188 const int cv_start = (node_idx * ibdev->num_comp_vectors /
3189 num_online_nodes() + target->comp_vector)
3190 % ibdev->num_comp_vectors;
3191 const int cv_end = ((node_idx + 1) * ibdev->num_comp_vectors /
3192 num_online_nodes() + target->comp_vector)
3193 % ibdev->num_comp_vectors;
3194 int cpu_idx = 0;
3195
3196 for_each_online_cpu(cpu) {
3197 if (cpu_to_node(cpu) != node)
3198 continue;
3199 if (ch_start + cpu_idx >= ch_end)
3200 continue;
3201 ch = &target->ch[ch_start + cpu_idx];
3202 ch->target = target;
3203 ch->comp_vector = cv_start == cv_end ? cv_start :
3204 cv_start + cpu_idx % (cv_end - cv_start);
3205 spin_lock_init(&ch->lock);
3206 INIT_LIST_HEAD(&ch->free_tx);
3207 ret = srp_new_cm_id(ch);
3208 if (ret)
3209 goto err_disconnect;
aef9ec39 3210
d92c0da7
BVA		 3211 ret = srp_create_ch_ib(ch);
3212 if (ret)
3213 goto err_disconnect;
3214
3215 ret = srp_alloc_req_data(ch);
3216 if (ret)
3217 goto err_disconnect;
3218
3219 ret = srp_connect_ch(ch, multich);
3220 if (ret) {
3221 shost_printk(KERN_ERR, target->scsi_host,
3222 PFX "Connection %d/%d failed\n",
3223 ch_start + cpu_idx,
3224 target->ch_count);
3225 if (node_idx == 0 && cpu_idx == 0) {
3226 goto err_disconnect;
3227 } else {
3228 srp_free_ch_ib(target, ch);
3229 srp_free_req_data(target, ch);
3230 target->ch_count = ch - target->ch;
c257ea6f 3231 goto connected;
d92c0da7
BVA		 3232 }
3233 }
3234
3235 multich = true;
3236 cpu_idx++;
3237 }
3238 node_idx++;
aef9ec39
RD		 3239 }
3240
c257ea6f 3241connected:
d92c0da7
BVA		 3242 target->scsi_host->nr_hw_queues = target->ch_count;
3243
aef9ec39
RD		 3244 ret = srp_add_target(host, target);
3245 if (ret)
3246 goto err_disconnect;
3247
34aa654e
BVA		 3248 if (target->state != SRP_TARGET_REMOVED) {
3249 shost_printk(KERN_DEBUG, target->scsi_host, PFX
3250 "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n",
3251 be64_to_cpu(target->id_ext),
3252 be64_to_cpu(target->ioc_guid),
747fe000 3253 be16_to_cpu(target->pkey),
34aa654e 3254 be64_to_cpu(target->service_id),
747fe000 3255 target->sgid.raw, target->orig_dgid.raw);
34aa654e 3256 }
e7ffde01 3257
2d7091bc
BVA		 3258 ret = count;
3259
3260out:
3261 mutex_unlock(&host->add_target_mutex);
34aa654e
BVA		 3262
3263 scsi_host_put(target->scsi_host);
bc44bd1d
BVA		 3264 if (ret < 0)
3265 scsi_host_put(target->scsi_host);
34aa654e 3266
2d7091bc 3267 return ret;
aef9ec39
RD		 3268
3269err_disconnect:
3270 srp_disconnect_target(target);
3271
d92c0da7
BVA		 3272 for (i = 0; i < target->ch_count; i++) {
3273 ch = &target->ch[i];
3274 srp_free_ch_ib(target, ch);
3275 srp_free_req_data(target, ch);
3276 }
aef9ec39 3277
d92c0da7 3278 kfree(target->ch);
2d7091bc 3279 goto out;
aef9ec39
RD		 3280}
3281
ee959b00 3282static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target);
aef9ec39 3283
ee959b00
TJ		 3284static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
3285 char *buf)
aef9ec39 3286{
ee959b00 3287 struct srp_host *host = container_of(dev, struct srp_host, dev);
aef9ec39 3288
05321937 3289 return sprintf(buf, "%s\n", host->srp_dev->dev->name);
aef9ec39
RD		 3290}
3291
ee959b00 3292static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
aef9ec39 3293
ee959b00
TJ		 3294static ssize_t show_port(struct device *dev, struct device_attribute *attr,
3295 char *buf)
aef9ec39 3296{
ee959b00 3297 struct srp_host *host = container_of(dev, struct srp_host, dev);
aef9ec39
RD		 3298
3299 return sprintf(buf, "%d\n", host->port);
3300}
3301
ee959b00 3302static DEVICE_ATTR(port, S_IRUGO, show_port, NULL);
aef9ec39 3303
f5358a17 3304static struct srp_host *srp_add_port(struct srp_device *device, u8 port)
aef9ec39
RD		 3305{
3306 struct srp_host *host;
3307
3308 host = kzalloc(sizeof *host, GFP_KERNEL);
3309 if (!host)
3310 return NULL;
3311
3312 INIT_LIST_HEAD(&host->target_list);
b3589fd4 3313 spin_lock_init(&host->target_lock);
aef9ec39 3314 init_completion(&host->released);
2d7091bc 3315 mutex_init(&host->add_target_mutex);
05321937 3316 host->srp_dev = device;
aef9ec39
RD		 3317 host->port = port;
3318
ee959b00
TJ		 3319 host->dev.class = &srp_class;
3320 host->dev.parent = device->dev->dma_device;
d927e38c 3321 dev_set_name(&host->dev, "srp-%s-%d", device->dev->name, port);
aef9ec39 3322
ee959b00 3323 if (device_register(&host->dev))
f5358a17 3324 goto free_host;
ee959b00 3325 if (device_create_file(&host->dev, &dev_attr_add_target))
aef9ec39 3326 goto err_class;
ee959b00 3327 if (device_create_file(&host->dev, &dev_attr_ibdev))
aef9ec39 3328 goto err_class;
ee959b00 3329 if (device_create_file(&host->dev, &dev_attr_port))
aef9ec39
RD		 3330 goto err_class;
3331
3332 return host;
3333
3334err_class:
ee959b00 3335 device_unregister(&host->dev);
aef9ec39 3336
f5358a17 3337free_host:
aef9ec39
RD		 3338 kfree(host);
3339
3340 return NULL;
3341}
3342
3343static void srp_add_one(struct ib_device *device)
3344{
f5358a17
RD		 3345 struct srp_device *srp_dev;
3346 struct ib_device_attr *dev_attr;
aef9ec39 3347 struct srp_host *host;
4139032b 3348 int mr_page_shift, p;
52ede08f 3349 u64 max_pages_per_mr;
aef9ec39 3350
f5358a17
RD		 3351 dev_attr = kmalloc(sizeof *dev_attr, GFP_KERNEL);
3352 if (!dev_attr)
cf311cd4 3353 return;
aef9ec39 3354
f5358a17 3355 if (ib_query_device(device, dev_attr)) {
e0bda7d8 3356 pr_warn("Query device failed for %s\n", device->name);
f5358a17
RD		 3357 goto free_attr;
3358 }
3359
3360 srp_dev = kmalloc(sizeof *srp_dev, GFP_KERNEL);
3361 if (!srp_dev)
3362 goto free_attr;
3363
d1b4289e
BVA		 3364 srp_dev->has_fmr = (device->alloc_fmr && device->dealloc_fmr &&
3365 device->map_phys_fmr && device->unmap_fmr);
5cfb1782
BVA		 3366 srp_dev->has_fr = (dev_attr->device_cap_flags &
3367 IB_DEVICE_MEM_MGT_EXTENSIONS);
3368 if (!srp_dev->has_fmr && !srp_dev->has_fr)
3369 dev_warn(&device->dev, "neither FMR nor FR is supported\n");
3370
3371 srp_dev->use_fast_reg = (srp_dev->has_fr &&
3372 (!srp_dev->has_fmr || prefer_fr));
d1b4289e 3373
f5358a17
RD		 3374 /*
3375 * Use the smallest page size supported by the HCA, down to a
8f26c9ff
DD		 3376 * minimum of 4096 bytes. We're unlikely to build large sglists
3377 * out of smaller entries.
f5358a17 3378 */
52ede08f
BVA		 3379 mr_page_shift = max(12, ffs(dev_attr->page_size_cap) - 1);
3380 srp_dev->mr_page_size = 1 << mr_page_shift;
3381 srp_dev->mr_page_mask = ~((u64) srp_dev->mr_page_size - 1);
3382 max_pages_per_mr = dev_attr->max_mr_size;
3383 do_div(max_pages_per_mr, srp_dev->mr_page_size);
3384 srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
3385 max_pages_per_mr);
5cfb1782
BVA		 3386 if (srp_dev->use_fast_reg) {
3387 srp_dev->max_pages_per_mr =
3388 min_t(u32, srp_dev->max_pages_per_mr,
3389 dev_attr->max_fast_reg_page_list_len);
3390 }
52ede08f
BVA		 3391 srp_dev->mr_max_size = srp_dev->mr_page_size *
3392 srp_dev->max_pages_per_mr;
5cfb1782 3393 pr_debug("%s: mr_page_shift = %d, dev_attr->max_mr_size = %#llx, dev_attr->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
52ede08f 3394 device->name, mr_page_shift, dev_attr->max_mr_size,
5cfb1782 3395 dev_attr->max_fast_reg_page_list_len,
52ede08f 3396 srp_dev->max_pages_per_mr, srp_dev->mr_max_size);
f5358a17
RD		 3397
3398 INIT_LIST_HEAD(&srp_dev->dev_list);
3399
3400 srp_dev->dev = device;
3401 srp_dev->pd = ib_alloc_pd(device);
3402 if (IS_ERR(srp_dev->pd))
3403 goto free_dev;
3404
3405 srp_dev->mr = ib_get_dma_mr(srp_dev->pd,
3406 IB_ACCESS_LOCAL_WRITE |
3407 IB_ACCESS_REMOTE_READ |
3408 IB_ACCESS_REMOTE_WRITE);
3409 if (IS_ERR(srp_dev->mr))
3410 goto err_pd;
3411
4139032b 3412 for (p = rdma_start_port(device); p <= rdma_end_port(device); ++p) {
f5358a17 3413 host = srp_add_port(srp_dev, p);
aef9ec39 3414 if (host)
f5358a17 3415 list_add_tail(&host->list, &srp_dev->dev_list);
aef9ec39
RD		 3416 }
3417
f5358a17
RD		 3418 ib_set_client_data(device, &srp_client, srp_dev);
3419
3420 goto free_attr;
3421
3422err_pd:
3423 ib_dealloc_pd(srp_dev->pd);
3424
3425free_dev:
3426 kfree(srp_dev);
3427
3428free_attr:
3429 kfree(dev_attr);
aef9ec39
RD		 3430}
3431
7c1eb45a 3432static void srp_remove_one(struct ib_device *device, void *client_data)
aef9ec39 3433{
f5358a17 3434 struct srp_device *srp_dev;
aef9ec39 3435 struct srp_host *host, *tmp_host;
ef6c49d8 3436 struct srp_target_port *target;
aef9ec39 3437
7c1eb45a 3438 srp_dev = client_data;
1fe0cb84
DB		 3439 if (!srp_dev)
3440 return;
aef9ec39 3441
f5358a17 3442 list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
ee959b00 3443 device_unregister(&host->dev);
aef9ec39
RD		 3444 /*
3445 * Wait for the sysfs entry to go away, so that no new
3446 * target ports can be created.
3447 */
3448 wait_for_completion(&host->released);
3449
3450 /*
ef6c49d8 3451 * Remove all target ports.
aef9ec39 3452 */
b3589fd4 3453 spin_lock(&host->target_lock);
ef6c49d8
BVA		 3454 list_for_each_entry(target, &host->target_list, list)
3455 srp_queue_remove_work(target);
b3589fd4 3456 spin_unlock(&host->target_lock);
aef9ec39
RD		 3457
3458 /*
bcc05910 3459 * Wait for tl_err and target port removal tasks.
aef9ec39 3460 */
ef6c49d8 3461 flush_workqueue(system_long_wq);
bcc05910 3462 flush_workqueue(srp_remove_wq);
aef9ec39 3463
aef9ec39
RD		 3464 kfree(host);
3465 }
3466
f5358a17
RD		 3467 ib_dereg_mr(srp_dev->mr);
3468 ib_dealloc_pd(srp_dev->pd);
3469
3470 kfree(srp_dev);
aef9ec39
RD		 3471}
3472
3236822b 3473static struct srp_function_template ib_srp_transport_functions = {
ed9b2264
BVA		 3474 .has_rport_state = true,
3475 .reset_timer_if_blocked = true,
a95cadb9 3476 .reconnect_delay = &srp_reconnect_delay,
ed9b2264
BVA		 3477 .fast_io_fail_tmo = &srp_fast_io_fail_tmo,
3478 .dev_loss_tmo = &srp_dev_loss_tmo,
3479 .reconnect = srp_rport_reconnect,
dc1bdbd9 3480 .rport_delete = srp_rport_delete,
ed9b2264 3481 .terminate_rport_io = srp_terminate_io,
3236822b
FT		 3482};
3483
aef9ec39
RD		 3484static int __init srp_init_module(void)
3485{
3486 int ret;
3487
dcb4cb85 3488 BUILD_BUG_ON(FIELD_SIZEOF(struct ib_wc, wr_id) < sizeof(void *));
dd5e6e38 3489
49248644 3490 if (srp_sg_tablesize) {
e0bda7d8 3491 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
49248644
DD		 3492 if (!cmd_sg_entries)
3493 cmd_sg_entries = srp_sg_tablesize;
3494 }
3495
3496 if (!cmd_sg_entries)
3497 cmd_sg_entries = SRP_DEF_SG_TABLESIZE;
3498
3499 if (cmd_sg_entries > 255) {
e0bda7d8 3500 pr_warn("Clamping cmd_sg_entries to 255\n");
49248644 3501 cmd_sg_entries = 255;
1e89a194
DD		 3502 }
3503
c07d424d
DD		 3504 if (!indirect_sg_entries)
3505 indirect_sg_entries = cmd_sg_entries;
3506 else if (indirect_sg_entries < cmd_sg_entries) {
e0bda7d8
BVA		 3507 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n",
3508 cmd_sg_entries);
c07d424d
DD		 3509 indirect_sg_entries = cmd_sg_entries;
3510 }
3511
bcc05910 3512 srp_remove_wq = create_workqueue("srp_remove");
da05be29
WY		 3513 if (!srp_remove_wq) {
3514 ret = -ENOMEM;
bcc05910
BVA		 3515 goto out;
3516 }
3517
3518 ret = -ENOMEM;
3236822b
FT		 3519 ib_srp_transport_template =
3520 srp_attach_transport(&ib_srp_transport_functions);
3521 if (!ib_srp_transport_template)
bcc05910 3522 goto destroy_wq;
3236822b 3523
aef9ec39
RD		 3524 ret = class_register(&srp_class);
3525 if (ret) {
e0bda7d8 3526 pr_err("couldn't register class infiniband_srp\n");
bcc05910 3527 goto release_tr;
aef9ec39
RD		 3528 }
3529
c1a0b23b
MT		 3530 ib_sa_register_client(&srp_sa_client);
3531
aef9ec39
RD		 3532 ret = ib_register_client(&srp_client);
3533 if (ret) {
e0bda7d8 3534 pr_err("couldn't register IB client\n");
bcc05910 3535 goto unreg_sa;
aef9ec39
RD		 3536 }
3537
bcc05910
BVA		 3538out:
3539 return ret;
3540
3541unreg_sa:
3542 ib_sa_unregister_client(&srp_sa_client);
3543 class_unregister(&srp_class);
3544
3545release_tr:
3546 srp_release_transport(ib_srp_transport_template);
3547
3548destroy_wq:
3549 destroy_workqueue(srp_remove_wq);
3550 goto out;
aef9ec39
RD		 3551}
3552
3553static void __exit srp_cleanup_module(void)
3554{
3555 ib_unregister_client(&srp_client);
c1a0b23b 3556 ib_sa_unregister_client(&srp_sa_client);
aef9ec39 3557 class_unregister(&srp_class);
3236822b 3558 srp_release_transport(ib_srp_transport_template);
bcc05910 3559 destroy_workqueue(srp_remove_wq);
aef9ec39
RD		 3560}
3561
3562module_init(srp_init_module);
3563module_exit(srp_cleanup_module);
Ubuntu Artful kernel mirror