]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blob - drivers/infiniband/ulp/srp/ib_srp.c
projects / mirror_ubuntu-artful-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'media/v4.10-2' of git://git.kernel.org/pub/scm/linux/kernel/git/mchehab...
[mirror_ubuntu-artful-kernel.git] / drivers / infiniband / ulp / srp / ib_srp.c
1 /*
2 * Copyright (c) 2005 Cisco Systems. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
34
35 #include <linux/module.h>
36 #include <linux/init.h>
37 #include <linux/slab.h>
38 #include <linux/err.h>
39 #include <linux/string.h>
40 #include <linux/parser.h>
41 #include <linux/random.h>
42 #include <linux/jiffies.h>
43 #include <rdma/ib_cache.h>
44
45 #include <linux/atomic.h>
46
47 #include <scsi/scsi.h>
48 #include <scsi/scsi_device.h>
49 #include <scsi/scsi_dbg.h>
50 #include <scsi/scsi_tcq.h>
51 #include <scsi/srp.h>
52 #include <scsi/scsi_transport_srp.h>
53
54 #include "ib_srp.h"
55
56 #define DRV_NAME "ib_srp"
57 #define PFX DRV_NAME ": "
58 #define DRV_VERSION "2.0"
59 #define DRV_RELDATE "July 26, 2015"
60
61 MODULE_AUTHOR("Roland Dreier");
62 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator");
63 MODULE_LICENSE("Dual BSD/GPL");
64 MODULE_VERSION(DRV_VERSION);
65 MODULE_INFO(release_date, DRV_RELDATE);
66
67 #if !defined(CONFIG_DYNAMIC_DEBUG)
68 #define DEFINE_DYNAMIC_DEBUG_METADATA(name, fmt)
69 #define DYNAMIC_DEBUG_BRANCH(descriptor) false
70 #endif
71
72 static unsigned int srp_sg_tablesize;
73 static unsigned int cmd_sg_entries;
74 static unsigned int indirect_sg_entries;
75 static bool allow_ext_sg;
76 static bool prefer_fr = true;
77 static bool register_always = true;
78 static bool never_register;
79 static int topspin_workarounds = 1;
80
81 module_param(srp_sg_tablesize, uint, 0444);
82 MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries");
83
84 module_param(cmd_sg_entries, uint, 0444);
85 MODULE_PARM_DESC(cmd_sg_entries,
86 "Default number of gather/scatter entries in the SRP command (default is 12, max 255)");
87
88 module_param(indirect_sg_entries, uint, 0444);
89 MODULE_PARM_DESC(indirect_sg_entries,
90 "Default max number of gather/scatter entries (default is 12, max is " __stringify(SG_MAX_SEGMENTS) ")");
91
92 module_param(allow_ext_sg, bool, 0444);
93 MODULE_PARM_DESC(allow_ext_sg,
94 "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)");
95
96 module_param(topspin_workarounds, int, 0444);
97 MODULE_PARM_DESC(topspin_workarounds,
98 "Enable workarounds for Topspin/Cisco SRP target bugs if != 0");
99
100 module_param(prefer_fr, bool, 0444);
101 MODULE_PARM_DESC(prefer_fr,
102 "Whether to use fast registration if both FMR and fast registration are supported");
103
104 module_param(register_always, bool, 0444);
105 MODULE_PARM_DESC(register_always,
106 "Use memory registration even for contiguous memory regions");
107
108 module_param(never_register, bool, 0444);
109 MODULE_PARM_DESC(never_register, "Never register memory");
110
111 static const struct kernel_param_ops srp_tmo_ops;
112
113 static int srp_reconnect_delay = 10;
114 module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay,
115 S_IRUGO | S_IWUSR);
116 MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts");
117
118 static int srp_fast_io_fail_tmo = 15;
119 module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo,
120 S_IRUGO | S_IWUSR);
121 MODULE_PARM_DESC(fast_io_fail_tmo,
122 "Number of seconds between the observation of a transport"
123 " layer error and failing all I/O. \"off\" means that this"
124 " functionality is disabled.");
125
126 static int srp_dev_loss_tmo = 600;
127 module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo,
128 S_IRUGO | S_IWUSR);
129 MODULE_PARM_DESC(dev_loss_tmo,
130 "Maximum number of seconds that the SRP transport should"
131 " insulate transport layer errors. After this time has been"
132 " exceeded the SCSI host is removed. Should be"
133 " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT)
134 " if fast_io_fail_tmo has not been set. \"off\" means that"
135 " this functionality is disabled.");
136
137 static unsigned ch_count;
138 module_param(ch_count, uint, 0444);
139 MODULE_PARM_DESC(ch_count,
140 "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA.");
141
142 static void srp_add_one(struct ib_device *device);
143 static void srp_remove_one(struct ib_device *device, void *client_data);
144 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc);
145 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
146 const char *opname);
147 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
148
149 static struct scsi_transport_template *ib_srp_transport_template;
150 static struct workqueue_struct *srp_remove_wq;
151
152 static struct ib_client srp_client = {
153 .name = "srp",
154 .add = srp_add_one,
155 .remove = srp_remove_one
156 };
157
158 static struct ib_sa_client srp_sa_client;
159
160 static int srp_tmo_get(char *buffer, const struct kernel_param *kp)
161 {
162 int tmo = *(int *)kp->arg;
163
164 if (tmo >= 0)
165 return sprintf(buffer, "%d", tmo);
166 else
167 return sprintf(buffer, "off");
168 }
169
170 static int srp_tmo_set(const char *val, const struct kernel_param *kp)
171 {
172 int tmo, res;
173
174 res = srp_parse_tmo(&tmo, val);
175 if (res)
176 goto out;
177
178 if (kp->arg == &srp_reconnect_delay)
179 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo,
180 srp_dev_loss_tmo);
181 else if (kp->arg == &srp_fast_io_fail_tmo)
182 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo);
183 else
184 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo,
185 tmo);
186 if (res)
187 goto out;
188 *(int *)kp->arg = tmo;
189
190 out:
191 return res;
192 }
193
194 static const struct kernel_param_ops srp_tmo_ops = {
195 .get = srp_tmo_get,
196 .set = srp_tmo_set,
197 };
198
199 static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
200 {
201 return (struct srp_target_port *) host->hostdata;
202 }
203
204 static const char *srp_target_info(struct Scsi_Host *host)
205 {
206 return host_to_target(host)->target_name;
207 }
208
209 static int srp_target_is_topspin(struct srp_target_port *target)
210 {
211 static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad };
212 static const u8 cisco_oui[3] = { 0x00, 0x1b, 0x0d };
213
214 return topspin_workarounds &&
215 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) ||
216 !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui));
217 }
218
219 static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size,
220 gfp_t gfp_mask,
221 enum dma_data_direction direction)
222 {
223 struct srp_iu *iu;
224
225 iu = kmalloc(sizeof *iu, gfp_mask);
226 if (!iu)
227 goto out;
228
229 iu->buf = kzalloc(size, gfp_mask);
230 if (!iu->buf)
231 goto out_free_iu;
232
233 iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size,
234 direction);
235 if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma))
236 goto out_free_buf;
237
238 iu->size = size;
239 iu->direction = direction;
240
241 return iu;
242
243 out_free_buf:
244 kfree(iu->buf);
245 out_free_iu:
246 kfree(iu);
247 out:
248 return NULL;
249 }
250
251 static void srp_free_iu(struct srp_host *host, struct srp_iu *iu)
252 {
253 if (!iu)
254 return;
255
256 ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size,
257 iu->direction);
258 kfree(iu->buf);
259 kfree(iu);
260 }
261
262 static void srp_qp_event(struct ib_event *event, void *context)
263 {
264 pr_debug("QP event %s (%d)\n",
265 ib_event_msg(event->event), event->event);
266 }
267
268 static int srp_init_qp(struct srp_target_port *target,
269 struct ib_qp *qp)
270 {
271 struct ib_qp_attr *attr;
272 int ret;
273
274 attr = kmalloc(sizeof *attr, GFP_KERNEL);
275 if (!attr)
276 return -ENOMEM;
277
278 ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev,
279 target->srp_host->port,
280 be16_to_cpu(target->pkey),
281 &attr->pkey_index);
282 if (ret)
283 goto out;
284
285 attr->qp_state = IB_QPS_INIT;
286 attr->qp_access_flags = (IB_ACCESS_REMOTE_READ |
287 IB_ACCESS_REMOTE_WRITE);
288 attr->port_num = target->srp_host->port;
289
290 ret = ib_modify_qp(qp, attr,
291 IB_QP_STATE |
292 IB_QP_PKEY_INDEX |
293 IB_QP_ACCESS_FLAGS |
294 IB_QP_PORT);
295
296 out:
297 kfree(attr);
298 return ret;
299 }
300
301 static int srp_new_cm_id(struct srp_rdma_ch *ch)
302 {
303 struct srp_target_port *target = ch->target;
304 struct ib_cm_id *new_cm_id;
305
306 new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
307 srp_cm_handler, ch);
308 if (IS_ERR(new_cm_id))
309 return PTR_ERR(new_cm_id);
310
311 if (ch->cm_id)
312 ib_destroy_cm_id(ch->cm_id);
313 ch->cm_id = new_cm_id;
314 ch->path.sgid = target->sgid;
315 ch->path.dgid = target->orig_dgid;
316 ch->path.pkey = target->pkey;
317 ch->path.service_id = target->service_id;
318
319 return 0;
320 }
321
322 static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target)
323 {
324 struct srp_device *dev = target->srp_host->srp_dev;
325 struct ib_fmr_pool_param fmr_param;
326
327 memset(&fmr_param, 0, sizeof(fmr_param));
328 fmr_param.pool_size = target->mr_pool_size;
329 fmr_param.dirty_watermark = fmr_param.pool_size / 4;
330 fmr_param.cache = 1;
331 fmr_param.max_pages_per_fmr = dev->max_pages_per_mr;
332 fmr_param.page_shift = ilog2(dev->mr_page_size);
333 fmr_param.access = (IB_ACCESS_LOCAL_WRITE |
334 IB_ACCESS_REMOTE_WRITE |
335 IB_ACCESS_REMOTE_READ);
336
337 return ib_create_fmr_pool(dev->pd, &fmr_param);
338 }
339
340 /**
341 * srp_destroy_fr_pool() - free the resources owned by a pool
342 * @pool: Fast registration pool to be destroyed.
343 */
344 static void srp_destroy_fr_pool(struct srp_fr_pool *pool)
345 {
346 int i;
347 struct srp_fr_desc *d;
348
349 if (!pool)
350 return;
351
352 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
353 if (d->mr)
354 ib_dereg_mr(d->mr);
355 }
356 kfree(pool);
357 }
358
359 /**
360 * srp_create_fr_pool() - allocate and initialize a pool for fast registration
361 * @device: IB device to allocate fast registration descriptors for.
362 * @pd: Protection domain associated with the FR descriptors.
363 * @pool_size: Number of descriptors to allocate.
364 * @max_page_list_len: Maximum fast registration work request page list length.
365 */
366 static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
367 struct ib_pd *pd, int pool_size,
368 int max_page_list_len)
369 {
370 struct srp_fr_pool *pool;
371 struct srp_fr_desc *d;
372 struct ib_mr *mr;
373 int i, ret = -EINVAL;
374
375 if (pool_size <= 0)
376 goto err;
377 ret = -ENOMEM;
378 pool = kzalloc(sizeof(struct srp_fr_pool) +
379 pool_size * sizeof(struct srp_fr_desc), GFP_KERNEL);
380 if (!pool)
381 goto err;
382 pool->size = pool_size;
383 pool->max_page_list_len = max_page_list_len;
384 spin_lock_init(&pool->lock);
385 INIT_LIST_HEAD(&pool->free_list);
386
387 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
388 mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG,
389 max_page_list_len);
390 if (IS_ERR(mr)) {
391 ret = PTR_ERR(mr);
392 if (ret == -ENOMEM)
393 pr_info("%s: ib_alloc_mr() failed. Try to reduce max_cmd_per_lun, max_sect or ch_count\n",
394 dev_name(&device->dev));
395 goto destroy_pool;
396 }
397 d->mr = mr;
398 list_add_tail(&d->entry, &pool->free_list);
399 }
400
401 out:
402 return pool;
403
404 destroy_pool:
405 srp_destroy_fr_pool(pool);
406
407 err:
408 pool = ERR_PTR(ret);
409 goto out;
410 }
411
412 /**
413 * srp_fr_pool_get() - obtain a descriptor suitable for fast registration
414 * @pool: Pool to obtain descriptor from.
415 */
416 static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool)
417 {
418 struct srp_fr_desc *d = NULL;
419 unsigned long flags;
420
421 spin_lock_irqsave(&pool->lock, flags);
422 if (!list_empty(&pool->free_list)) {
423 d = list_first_entry(&pool->free_list, typeof(*d), entry);
424 list_del(&d->entry);
425 }
426 spin_unlock_irqrestore(&pool->lock, flags);
427
428 return d;
429 }
430
431 /**
432 * srp_fr_pool_put() - put an FR descriptor back in the free list
433 * @pool: Pool the descriptor was allocated from.
434 * @desc: Pointer to an array of fast registration descriptor pointers.
435 * @n: Number of descriptors to put back.
436 *
437 * Note: The caller must already have queued an invalidation request for
438 * desc->mr->rkey before calling this function.
439 */
440 static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc,
441 int n)
442 {
443 unsigned long flags;
444 int i;
445
446 spin_lock_irqsave(&pool->lock, flags);
447 for (i = 0; i < n; i++)
448 list_add(&desc[i]->entry, &pool->free_list);
449 spin_unlock_irqrestore(&pool->lock, flags);
450 }
451
452 static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
453 {
454 struct srp_device *dev = target->srp_host->srp_dev;
455
456 return srp_create_fr_pool(dev->dev, dev->pd, target->mr_pool_size,
457 dev->max_pages_per_mr);
458 }
459
460 /**
461 * srp_destroy_qp() - destroy an RDMA queue pair
462 * @qp: RDMA queue pair.
463 *
464 * Drain the qp before destroying it. This avoids that the receive
465 * completion handler can access the queue pair while it is
466 * being destroyed.
467 */
468 static void srp_destroy_qp(struct ib_qp *qp)
469 {
470 ib_drain_rq(qp);
471 ib_destroy_qp(qp);
472 }
473
474 static int srp_create_ch_ib(struct srp_rdma_ch *ch)
475 {
476 struct srp_target_port *target = ch->target;
477 struct srp_device *dev = target->srp_host->srp_dev;
478 struct ib_qp_init_attr *init_attr;
479 struct ib_cq *recv_cq, *send_cq;
480 struct ib_qp *qp;
481 struct ib_fmr_pool *fmr_pool = NULL;
482 struct srp_fr_pool *fr_pool = NULL;
483 const int m = 1 + dev->use_fast_reg * target->mr_per_cmd * 2;
484 int ret;
485
486 init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
487 if (!init_attr)
488 return -ENOMEM;
489
490 /* queue_size + 1 for ib_drain_rq() */
491 recv_cq = ib_alloc_cq(dev->dev, ch, target->queue_size + 1,
492 ch->comp_vector, IB_POLL_SOFTIRQ);
493 if (IS_ERR(recv_cq)) {
494 ret = PTR_ERR(recv_cq);
495 goto err;
496 }
497
498 send_cq = ib_alloc_cq(dev->dev, ch, m * target->queue_size,
499 ch->comp_vector, IB_POLL_DIRECT);
500 if (IS_ERR(send_cq)) {
501 ret = PTR_ERR(send_cq);
502 goto err_recv_cq;
503 }
504
505 init_attr->event_handler = srp_qp_event;
506 init_attr->cap.max_send_wr = m * target->queue_size;
507 init_attr->cap.max_recv_wr = target->queue_size + 1;
508 init_attr->cap.max_recv_sge = 1;
509 init_attr->cap.max_send_sge = 1;
510 init_attr->sq_sig_type = IB_SIGNAL_REQ_WR;
511 init_attr->qp_type = IB_QPT_RC;
512 init_attr->send_cq = send_cq;
513 init_attr->recv_cq = recv_cq;
514
515 qp = ib_create_qp(dev->pd, init_attr);
516 if (IS_ERR(qp)) {
517 ret = PTR_ERR(qp);
518 goto err_send_cq;
519 }
520
521 ret = srp_init_qp(target, qp);
522 if (ret)
523 goto err_qp;
524
525 if (dev->use_fast_reg) {
526 fr_pool = srp_alloc_fr_pool(target);
527 if (IS_ERR(fr_pool)) {
528 ret = PTR_ERR(fr_pool);
529 shost_printk(KERN_WARNING, target->scsi_host, PFX
530 "FR pool allocation failed (%d)\n", ret);
531 goto err_qp;
532 }
533 } else if (dev->use_fmr) {
534 fmr_pool = srp_alloc_fmr_pool(target);
535 if (IS_ERR(fmr_pool)) {
536 ret = PTR_ERR(fmr_pool);
537 shost_printk(KERN_WARNING, target->scsi_host, PFX
538 "FMR pool allocation failed (%d)\n", ret);
539 goto err_qp;
540 }
541 }
542
543 if (ch->qp)
544 srp_destroy_qp(ch->qp);
545 if (ch->recv_cq)
546 ib_free_cq(ch->recv_cq);
547 if (ch->send_cq)
548 ib_free_cq(ch->send_cq);
549
550 ch->qp = qp;
551 ch->recv_cq = recv_cq;
552 ch->send_cq = send_cq;
553
554 if (dev->use_fast_reg) {
555 if (ch->fr_pool)
556 srp_destroy_fr_pool(ch->fr_pool);
557 ch->fr_pool = fr_pool;
558 } else if (dev->use_fmr) {
559 if (ch->fmr_pool)
560 ib_destroy_fmr_pool(ch->fmr_pool);
561 ch->fmr_pool = fmr_pool;
562 }
563
564 kfree(init_attr);
565 return 0;
566
567 err_qp:
568 srp_destroy_qp(qp);
569
570 err_send_cq:
571 ib_free_cq(send_cq);
572
573 err_recv_cq:
574 ib_free_cq(recv_cq);
575
576 err:
577 kfree(init_attr);
578 return ret;
579 }
580
581 /*
582 * Note: this function may be called without srp_alloc_iu_bufs() having been
583 * invoked. Hence the ch->[rt]x_ring checks.
584 */
585 static void srp_free_ch_ib(struct srp_target_port *target,
586 struct srp_rdma_ch *ch)
587 {
588 struct srp_device *dev = target->srp_host->srp_dev;
589 int i;
590
591 if (!ch->target)
592 return;
593
594 if (ch->cm_id) {
595 ib_destroy_cm_id(ch->cm_id);
596 ch->cm_id = NULL;
597 }
598
599 /* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */
600 if (!ch->qp)
601 return;
602
603 if (dev->use_fast_reg) {
604 if (ch->fr_pool)
605 srp_destroy_fr_pool(ch->fr_pool);
606 } else if (dev->use_fmr) {
607 if (ch->fmr_pool)
608 ib_destroy_fmr_pool(ch->fmr_pool);
609 }
610
611 srp_destroy_qp(ch->qp);
612 ib_free_cq(ch->send_cq);
613 ib_free_cq(ch->recv_cq);
614
615 /*
616 * Avoid that the SCSI error handler tries to use this channel after
617 * it has been freed. The SCSI error handler can namely continue
618 * trying to perform recovery actions after scsi_remove_host()
619 * returned.
620 */
621 ch->target = NULL;
622
623 ch->qp = NULL;
624 ch->send_cq = ch->recv_cq = NULL;
625
626 if (ch->rx_ring) {
627 for (i = 0; i < target->queue_size; ++i)
628 srp_free_iu(target->srp_host, ch->rx_ring[i]);
629 kfree(ch->rx_ring);
630 ch->rx_ring = NULL;
631 }
632 if (ch->tx_ring) {
633 for (i = 0; i < target->queue_size; ++i)
634 srp_free_iu(target->srp_host, ch->tx_ring[i]);
635 kfree(ch->tx_ring);
636 ch->tx_ring = NULL;
637 }
638 }
639
640 static void srp_path_rec_completion(int status,
641 struct ib_sa_path_rec *pathrec,
642 void *ch_ptr)
643 {
644 struct srp_rdma_ch *ch = ch_ptr;
645 struct srp_target_port *target = ch->target;
646
647 ch->status = status;
648 if (status)
649 shost_printk(KERN_ERR, target->scsi_host,
650 PFX "Got failed path rec status %d\n", status);
651 else
652 ch->path = *pathrec;
653 complete(&ch->done);
654 }
655
656 static int srp_lookup_path(struct srp_rdma_ch *ch)
657 {
658 struct srp_target_port *target = ch->target;
659 int ret;
660
661 ch->path.numb_path = 1;
662
663 init_completion(&ch->done);
664
665 ch->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
666 target->srp_host->srp_dev->dev,
667 target->srp_host->port,
668 &ch->path,
669 IB_SA_PATH_REC_SERVICE_ID |
670 IB_SA_PATH_REC_DGID |
671 IB_SA_PATH_REC_SGID |
672 IB_SA_PATH_REC_NUMB_PATH |
673 IB_SA_PATH_REC_PKEY,
674 SRP_PATH_REC_TIMEOUT_MS,
675 GFP_KERNEL,
676 srp_path_rec_completion,
677 ch, &ch->path_query);
678 if (ch->path_query_id < 0)
679 return ch->path_query_id;
680
681 ret = wait_for_completion_interruptible(&ch->done);
682 if (ret < 0)
683 return ret;
684
685 if (ch->status < 0)
686 shost_printk(KERN_WARNING, target->scsi_host,
687 PFX "Path record query failed\n");
688
689 return ch->status;
690 }
691
692 static int srp_send_req(struct srp_rdma_ch *ch, bool multich)
693 {
694 struct srp_target_port *target = ch->target;
695 struct {
696 struct ib_cm_req_param param;
697 struct srp_login_req priv;
698 } *req = NULL;
699 int status;
700
701 req = kzalloc(sizeof *req, GFP_KERNEL);
702 if (!req)
703 return -ENOMEM;
704
705 req->param.primary_path = &ch->path;
706 req->param.alternate_path = NULL;
707 req->param.service_id = target->service_id;
708 req->param.qp_num = ch->qp->qp_num;
709 req->param.qp_type = ch->qp->qp_type;
710 req->param.private_data = &req->priv;
711 req->param.private_data_len = sizeof req->priv;
712 req->param.flow_control = 1;
713
714 get_random_bytes(&req->param.starting_psn, 4);
715 req->param.starting_psn &= 0xffffff;
716
717 /*
718 * Pick some arbitrary defaults here; we could make these
719 * module parameters if anyone cared about setting them.
720 */
721 req->param.responder_resources = 4;
722 req->param.remote_cm_response_timeout = 20;
723 req->param.local_cm_response_timeout = 20;
724 req->param.retry_count = target->tl_retry_count;
725 req->param.rnr_retry_count = 7;
726 req->param.max_cm_retries = 15;
727
728 req->priv.opcode = SRP_LOGIN_REQ;
729 req->priv.tag = 0;
730 req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len);
731 req->priv.req_buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
732 SRP_BUF_FORMAT_INDIRECT);
733 req->priv.req_flags = (multich ? SRP_MULTICHAN_MULTI :
734 SRP_MULTICHAN_SINGLE);
735 /*
736 * In the published SRP specification (draft rev. 16a), the
737 * port identifier format is 8 bytes of ID extension followed
738 * by 8 bytes of GUID. Older drafts put the two halves in the
739 * opposite order, so that the GUID comes first.
740 *
741 * Targets conforming to these obsolete drafts can be
742 * recognized by the I/O Class they report.
743 */
744 if (target->io_class == SRP_REV10_IB_IO_CLASS) {
745 memcpy(req->priv.initiator_port_id,
746 &target->sgid.global.interface_id, 8);
747 memcpy(req->priv.initiator_port_id + 8,
748 &target->initiator_ext, 8);
749 memcpy(req->priv.target_port_id, &target->ioc_guid, 8);
750 memcpy(req->priv.target_port_id + 8, &target->id_ext, 8);
751 } else {
752 memcpy(req->priv.initiator_port_id,
753 &target->initiator_ext, 8);
754 memcpy(req->priv.initiator_port_id + 8,
755 &target->sgid.global.interface_id, 8);
756 memcpy(req->priv.target_port_id, &target->id_ext, 8);
757 memcpy(req->priv.target_port_id + 8, &target->ioc_guid, 8);
758 }
759
760 /*
761 * Topspin/Cisco SRP targets will reject our login unless we
762 * zero out the first 8 bytes of our initiator port ID and set
763 * the second 8 bytes to the local node GUID.
764 */
765 if (srp_target_is_topspin(target)) {
766 shost_printk(KERN_DEBUG, target->scsi_host,
767 PFX "Topspin/Cisco initiator port ID workaround "
768 "activated for target GUID %016llx\n",
769 be64_to_cpu(target->ioc_guid));
770 memset(req->priv.initiator_port_id, 0, 8);
771 memcpy(req->priv.initiator_port_id + 8,
772 &target->srp_host->srp_dev->dev->node_guid, 8);
773 }
774
775 status = ib_send_cm_req(ch->cm_id, &req->param);
776
777 kfree(req);
778
779 return status;
780 }
781
782 static bool srp_queue_remove_work(struct srp_target_port *target)
783 {
784 bool changed = false;
785
786 spin_lock_irq(&target->lock);
787 if (target->state != SRP_TARGET_REMOVED) {
788 target->state = SRP_TARGET_REMOVED;
789 changed = true;
790 }
791 spin_unlock_irq(&target->lock);
792
793 if (changed)
794 queue_work(srp_remove_wq, &target->remove_work);
795
796 return changed;
797 }
798
799 static void srp_disconnect_target(struct srp_target_port *target)
800 {
801 struct srp_rdma_ch *ch;
802 int i;
803
804 /* XXX should send SRP_I_LOGOUT request */
805
806 for (i = 0; i < target->ch_count; i++) {
807 ch = &target->ch[i];
808 ch->connected = false;
809 if (ch->cm_id && ib_send_cm_dreq(ch->cm_id, NULL, 0)) {
810 shost_printk(KERN_DEBUG, target->scsi_host,
811 PFX "Sending CM DREQ failed\n");
812 }
813 }
814 }
815
816 static void srp_free_req_data(struct srp_target_port *target,
817 struct srp_rdma_ch *ch)
818 {
819 struct srp_device *dev = target->srp_host->srp_dev;
820 struct ib_device *ibdev = dev->dev;
821 struct srp_request *req;
822 int i;
823
824 if (!ch->req_ring)
825 return;
826
827 for (i = 0; i < target->req_ring_size; ++i) {
828 req = &ch->req_ring[i];
829 if (dev->use_fast_reg) {
830 kfree(req->fr_list);
831 } else {
832 kfree(req->fmr_list);
833 kfree(req->map_page);
834 }
835 if (req->indirect_dma_addr) {
836 ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
837 target->indirect_size,
838 DMA_TO_DEVICE);
839 }
840 kfree(req->indirect_desc);
841 }
842
843 kfree(ch->req_ring);
844 ch->req_ring = NULL;
845 }
846
847 static int srp_alloc_req_data(struct srp_rdma_ch *ch)
848 {
849 struct srp_target_port *target = ch->target;
850 struct srp_device *srp_dev = target->srp_host->srp_dev;
851 struct ib_device *ibdev = srp_dev->dev;
852 struct srp_request *req;
853 void *mr_list;
854 dma_addr_t dma_addr;
855 int i, ret = -ENOMEM;
856
857 ch->req_ring = kcalloc(target->req_ring_size, sizeof(*ch->req_ring),
858 GFP_KERNEL);
859 if (!ch->req_ring)
860 goto out;
861
862 for (i = 0; i < target->req_ring_size; ++i) {
863 req = &ch->req_ring[i];
864 mr_list = kmalloc(target->mr_per_cmd * sizeof(void *),
865 GFP_KERNEL);
866 if (!mr_list)
867 goto out;
868 if (srp_dev->use_fast_reg) {
869 req->fr_list = mr_list;
870 } else {
871 req->fmr_list = mr_list;
872 req->map_page = kmalloc(srp_dev->max_pages_per_mr *
873 sizeof(void *), GFP_KERNEL);
874 if (!req->map_page)
875 goto out;
876 }
877 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
878 if (!req->indirect_desc)
879 goto out;
880
881 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
882 target->indirect_size,
883 DMA_TO_DEVICE);
884 if (ib_dma_mapping_error(ibdev, dma_addr))
885 goto out;
886
887 req->indirect_dma_addr = dma_addr;
888 }
889 ret = 0;
890
891 out:
892 return ret;
893 }
894
895 /**
896 * srp_del_scsi_host_attr() - Remove attributes defined in the host template.
897 * @shost: SCSI host whose attributes to remove from sysfs.
898 *
899 * Note: Any attributes defined in the host template and that did not exist
900 * before invocation of this function will be ignored.
901 */
902 static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
903 {
904 struct device_attribute **attr;
905
906 for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr)
907 device_remove_file(&shost->shost_dev, *attr);
908 }
909
910 static void srp_remove_target(struct srp_target_port *target)
911 {
912 struct srp_rdma_ch *ch;
913 int i;
914
915 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
916
917 srp_del_scsi_host_attr(target->scsi_host);
918 srp_rport_get(target->rport);
919 srp_remove_host(target->scsi_host);
920 scsi_remove_host(target->scsi_host);
921 srp_stop_rport_timers(target->rport);
922 srp_disconnect_target(target);
923 for (i = 0; i < target->ch_count; i++) {
924 ch = &target->ch[i];
925 srp_free_ch_ib(target, ch);
926 }
927 cancel_work_sync(&target->tl_err_work);
928 srp_rport_put(target->rport);
929 for (i = 0; i < target->ch_count; i++) {
930 ch = &target->ch[i];
931 srp_free_req_data(target, ch);
932 }
933 kfree(target->ch);
934 target->ch = NULL;
935
936 spin_lock(&target->srp_host->target_lock);
937 list_del(&target->list);
938 spin_unlock(&target->srp_host->target_lock);
939
940 scsi_host_put(target->scsi_host);
941 }
942
943 static void srp_remove_work(struct work_struct *work)
944 {
945 struct srp_target_port *target =
946 container_of(work, struct srp_target_port, remove_work);
947
948 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
949
950 srp_remove_target(target);
951 }
952
953 static void srp_rport_delete(struct srp_rport *rport)
954 {
955 struct srp_target_port *target = rport->lld_data;
956
957 srp_queue_remove_work(target);
958 }
959
960 /**
961 * srp_connected_ch() - number of connected channels
962 * @target: SRP target port.
963 */
964 static int srp_connected_ch(struct srp_target_port *target)
965 {
966 int i, c = 0;
967
968 for (i = 0; i < target->ch_count; i++)
969 c += target->ch[i].connected;
970
971 return c;
972 }
973
974 static int srp_connect_ch(struct srp_rdma_ch *ch, bool multich)
975 {
976 struct srp_target_port *target = ch->target;
977 int ret;
978
979 WARN_ON_ONCE(!multich && srp_connected_ch(target) > 0);
980
981 ret = srp_lookup_path(ch);
982 if (ret)
983 goto out;
984
985 while (1) {
986 init_completion(&ch->done);
987 ret = srp_send_req(ch, multich);
988 if (ret)
989 goto out;
990 ret = wait_for_completion_interruptible(&ch->done);
991 if (ret < 0)
992 goto out;
993
994 /*
995 * The CM event handling code will set status to
996 * SRP_PORT_REDIRECT if we get a port redirect REJ
997 * back, or SRP_DLID_REDIRECT if we get a lid/qp
998 * redirect REJ back.
999 */
1000 ret = ch->status;
1001 switch (ret) {
1002 case 0:
1003 ch->connected = true;
1004 goto out;
1005
1006 case SRP_PORT_REDIRECT:
1007 ret = srp_lookup_path(ch);
1008 if (ret)
1009 goto out;
1010 break;
1011
1012 case SRP_DLID_REDIRECT:
1013 break;
1014
1015 case SRP_STALE_CONN:
1016 shost_printk(KERN_ERR, target->scsi_host, PFX
1017 "giving up on stale connection\n");
1018 ret = -ECONNRESET;
1019 goto out;
1020
1021 default:
1022 goto out;
1023 }
1024 }
1025
1026 out:
1027 return ret <= 0 ? ret : -ENODEV;
1028 }
1029
1030 static void srp_inv_rkey_err_done(struct ib_cq *cq, struct ib_wc *wc)
1031 {
1032 srp_handle_qp_err(cq, wc, "INV RKEY");
1033 }
1034
1035 static int srp_inv_rkey(struct srp_request *req, struct srp_rdma_ch *ch,
1036 u32 rkey)
1037 {
1038 struct ib_send_wr *bad_wr;
1039 struct ib_send_wr wr = {
1040 .opcode = IB_WR_LOCAL_INV,
1041 .next = NULL,
1042 .num_sge = 0,
1043 .send_flags = 0,
1044 .ex.invalidate_rkey = rkey,
1045 };
1046
1047 wr.wr_cqe = &req->reg_cqe;
1048 req->reg_cqe.done = srp_inv_rkey_err_done;
1049 return ib_post_send(ch->qp, &wr, &bad_wr);
1050 }
1051
1052 static void srp_unmap_data(struct scsi_cmnd *scmnd,
1053 struct srp_rdma_ch *ch,
1054 struct srp_request *req)
1055 {
1056 struct srp_target_port *target = ch->target;
1057 struct srp_device *dev = target->srp_host->srp_dev;
1058 struct ib_device *ibdev = dev->dev;
1059 int i, res;
1060
1061 if (!scsi_sglist(scmnd) ||
1062 (scmnd->sc_data_direction != DMA_TO_DEVICE &&
1063 scmnd->sc_data_direction != DMA_FROM_DEVICE))
1064 return;
1065
1066 if (dev->use_fast_reg) {
1067 struct srp_fr_desc **pfr;
1068
1069 for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) {
1070 res = srp_inv_rkey(req, ch, (*pfr)->mr->rkey);
1071 if (res < 0) {
1072 shost_printk(KERN_ERR, target->scsi_host, PFX
1073 "Queueing INV WR for rkey %#x failed (%d)\n",
1074 (*pfr)->mr->rkey, res);
1075 queue_work(system_long_wq,
1076 &target->tl_err_work);
1077 }
1078 }
1079 if (req->nmdesc)
1080 srp_fr_pool_put(ch->fr_pool, req->fr_list,
1081 req->nmdesc);
1082 } else if (dev->use_fmr) {
1083 struct ib_pool_fmr **pfmr;
1084
1085 for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++)
1086 ib_fmr_pool_unmap(*pfmr);
1087 }
1088
1089 ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
1090 scmnd->sc_data_direction);
1091 }
1092
1093 /**
1094 * srp_claim_req - Take ownership of the scmnd associated with a request.
1095 * @ch: SRP RDMA channel.
1096 * @req: SRP request.
1097 * @sdev: If not NULL, only take ownership for this SCSI device.
1098 * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take
1099 * ownership of @req->scmnd if it equals @scmnd.
1100 *
1101 * Return value:
1102 * Either NULL or a pointer to the SCSI command the caller became owner of.
1103 */
1104 static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch,
1105 struct srp_request *req,
1106 struct scsi_device *sdev,
1107 struct scsi_cmnd *scmnd)
1108 {
1109 unsigned long flags;
1110
1111 spin_lock_irqsave(&ch->lock, flags);
1112 if (req->scmnd &&
1113 (!sdev || req->scmnd->device == sdev) &&
1114 (!scmnd || req->scmnd == scmnd)) {
1115 scmnd = req->scmnd;
1116 req->scmnd = NULL;
1117 } else {
1118 scmnd = NULL;
1119 }
1120 spin_unlock_irqrestore(&ch->lock, flags);
1121
1122 return scmnd;
1123 }
1124
1125 /**
1126 * srp_free_req() - Unmap data and adjust ch->req_lim.
1127 * @ch: SRP RDMA channel.
1128 * @req: Request to be freed.
1129 * @scmnd: SCSI command associated with @req.
1130 * @req_lim_delta: Amount to be added to @target->req_lim.
1131 */
1132 static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req,
1133 struct scsi_cmnd *scmnd, s32 req_lim_delta)
1134 {
1135 unsigned long flags;
1136
1137 srp_unmap_data(scmnd, ch, req);
1138
1139 spin_lock_irqsave(&ch->lock, flags);
1140 ch->req_lim += req_lim_delta;
1141 spin_unlock_irqrestore(&ch->lock, flags);
1142 }
1143
1144 static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req,
1145 struct scsi_device *sdev, int result)
1146 {
1147 struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL);
1148
1149 if (scmnd) {
1150 srp_free_req(ch, req, scmnd, 0);
1151 scmnd->result = result;
1152 scmnd->scsi_done(scmnd);
1153 }
1154 }
1155
1156 static void srp_terminate_io(struct srp_rport *rport)
1157 {
1158 struct srp_target_port *target = rport->lld_data;
1159 struct srp_rdma_ch *ch;
1160 struct Scsi_Host *shost = target->scsi_host;
1161 struct scsi_device *sdev;
1162 int i, j;
1163
1164 /*
1165 * Invoking srp_terminate_io() while srp_queuecommand() is running
1166 * is not safe. Hence the warning statement below.
1167 */
1168 shost_for_each_device(sdev, shost)
1169 WARN_ON_ONCE(sdev->request_queue->request_fn_active);
1170
1171 for (i = 0; i < target->ch_count; i++) {
1172 ch = &target->ch[i];
1173
1174 for (j = 0; j < target->req_ring_size; ++j) {
1175 struct srp_request *req = &ch->req_ring[j];
1176
1177 srp_finish_req(ch, req, NULL,
1178 DID_TRANSPORT_FAILFAST << 16);
1179 }
1180 }
1181 }
1182
1183 /*
1184 * It is up to the caller to ensure that srp_rport_reconnect() calls are
1185 * serialized and that no concurrent srp_queuecommand(), srp_abort(),
1186 * srp_reset_device() or srp_reset_host() calls will occur while this function
1187 * is in progress. One way to realize that is not to call this function
1188 * directly but to call srp_reconnect_rport() instead since that last function
1189 * serializes calls of this function via rport->mutex and also blocks
1190 * srp_queuecommand() calls before invoking this function.
1191 */
1192 static int srp_rport_reconnect(struct srp_rport *rport)
1193 {
1194 struct srp_target_port *target = rport->lld_data;
1195 struct srp_rdma_ch *ch;
1196 int i, j, ret = 0;
1197 bool multich = false;
1198
1199 srp_disconnect_target(target);
1200
1201 if (target->state == SRP_TARGET_SCANNING)
1202 return -ENODEV;
1203
1204 /*
1205 * Now get a new local CM ID so that we avoid confusing the target in
1206 * case things are really fouled up. Doing so also ensures that all CM
1207 * callbacks will have finished before a new QP is allocated.
1208 */
1209 for (i = 0; i < target->ch_count; i++) {
1210 ch = &target->ch[i];
1211 ret += srp_new_cm_id(ch);
1212 }
1213 for (i = 0; i < target->ch_count; i++) {
1214 ch = &target->ch[i];
1215 for (j = 0; j < target->req_ring_size; ++j) {
1216 struct srp_request *req = &ch->req_ring[j];
1217
1218 srp_finish_req(ch, req, NULL, DID_RESET << 16);
1219 }
1220 }
1221 for (i = 0; i < target->ch_count; i++) {
1222 ch = &target->ch[i];
1223 /*
1224 * Whether or not creating a new CM ID succeeded, create a new
1225 * QP. This guarantees that all completion callback function
1226 * invocations have finished before request resetting starts.
1227 */
1228 ret += srp_create_ch_ib(ch);
1229
1230 INIT_LIST_HEAD(&ch->free_tx);
1231 for (j = 0; j < target->queue_size; ++j)
1232 list_add(&ch->tx_ring[j]->list, &ch->free_tx);
1233 }
1234
1235 target->qp_in_error = false;
1236
1237 for (i = 0; i < target->ch_count; i++) {
1238 ch = &target->ch[i];
1239 if (ret)
1240 break;
1241 ret = srp_connect_ch(ch, multich);
1242 multich = true;
1243 }
1244
1245 if (ret == 0)
1246 shost_printk(KERN_INFO, target->scsi_host,
1247 PFX "reconnect succeeded\n");
1248
1249 return ret;
1250 }
1251
1252 static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
1253 unsigned int dma_len, u32 rkey)
1254 {
1255 struct srp_direct_buf *desc = state->desc;
1256
1257 WARN_ON_ONCE(!dma_len);
1258
1259 desc->va = cpu_to_be64(dma_addr);
1260 desc->key = cpu_to_be32(rkey);
1261 desc->len = cpu_to_be32(dma_len);
1262
1263 state->total_len += dma_len;
1264 state->desc++;
1265 state->ndesc++;
1266 }
1267
1268 static int srp_map_finish_fmr(struct srp_map_state *state,
1269 struct srp_rdma_ch *ch)
1270 {
1271 struct srp_target_port *target = ch->target;
1272 struct srp_device *dev = target->srp_host->srp_dev;
1273 struct ib_pd *pd = target->pd;
1274 struct ib_pool_fmr *fmr;
1275 u64 io_addr = 0;
1276
1277 if (state->fmr.next >= state->fmr.end) {
1278 shost_printk(KERN_ERR, ch->target->scsi_host,
1279 PFX "Out of MRs (mr_per_cmd = %d)\n",
1280 ch->target->mr_per_cmd);
1281 return -ENOMEM;
1282 }
1283
1284 WARN_ON_ONCE(!dev->use_fmr);
1285
1286 if (state->npages == 0)
1287 return 0;
1288
1289 if (state->npages == 1 && (pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)) {
1290 srp_map_desc(state, state->base_dma_addr, state->dma_len,
1291 pd->unsafe_global_rkey);
1292 goto reset_state;
1293 }
1294
1295 fmr = ib_fmr_pool_map_phys(ch->fmr_pool, state->pages,
1296 state->npages, io_addr);
1297 if (IS_ERR(fmr))
1298 return PTR_ERR(fmr);
1299
1300 *state->fmr.next++ = fmr;
1301 state->nmdesc++;
1302
1303 srp_map_desc(state, state->base_dma_addr & ~dev->mr_page_mask,
1304 state->dma_len, fmr->fmr->rkey);
1305
1306 reset_state:
1307 state->npages = 0;
1308 state->dma_len = 0;
1309
1310 return 0;
1311 }
1312
1313 static void srp_reg_mr_err_done(struct ib_cq *cq, struct ib_wc *wc)
1314 {
1315 srp_handle_qp_err(cq, wc, "FAST REG");
1316 }
1317
1318 /*
1319 * Map up to sg_nents elements of state->sg where *sg_offset_p is the offset
1320 * where to start in the first element. If sg_offset_p != NULL then
1321 * *sg_offset_p is updated to the offset in state->sg[retval] of the first
1322 * byte that has not yet been mapped.
1323 */
1324 static int srp_map_finish_fr(struct srp_map_state *state,
1325 struct srp_request *req,
1326 struct srp_rdma_ch *ch, int sg_nents,
1327 unsigned int *sg_offset_p)
1328 {
1329 struct srp_target_port *target = ch->target;
1330 struct srp_device *dev = target->srp_host->srp_dev;
1331 struct ib_pd *pd = target->pd;
1332 struct ib_send_wr *bad_wr;
1333 struct ib_reg_wr wr;
1334 struct srp_fr_desc *desc;
1335 u32 rkey;
1336 int n, err;
1337
1338 if (state->fr.next >= state->fr.end) {
1339 shost_printk(KERN_ERR, ch->target->scsi_host,
1340 PFX "Out of MRs (mr_per_cmd = %d)\n",
1341 ch->target->mr_per_cmd);
1342 return -ENOMEM;
1343 }
1344
1345 WARN_ON_ONCE(!dev->use_fast_reg);
1346
1347 if (sg_nents == 1 && (pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)) {
1348 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;
1349
1350 srp_map_desc(state, sg_dma_address(state->sg) + sg_offset,
1351 sg_dma_len(state->sg) - sg_offset,
1352 pd->unsafe_global_rkey);
1353 if (sg_offset_p)
1354 *sg_offset_p = 0;
1355 return 1;
1356 }
1357
1358 desc = srp_fr_pool_get(ch->fr_pool);
1359 if (!desc)
1360 return -ENOMEM;
1361
1362 rkey = ib_inc_rkey(desc->mr->rkey);
1363 ib_update_fast_reg_key(desc->mr, rkey);
1364
1365 n = ib_map_mr_sg(desc->mr, state->sg, sg_nents, sg_offset_p,
1366 dev->mr_page_size);
1367 if (unlikely(n < 0)) {
1368 srp_fr_pool_put(ch->fr_pool, &desc, 1);
1369 pr_debug("%s: ib_map_mr_sg(%d, %d) returned %d.\n",
1370 dev_name(&req->scmnd->device->sdev_gendev), sg_nents,
1371 sg_offset_p ? *sg_offset_p : -1, n);
1372 return n;
1373 }
1374
1375 WARN_ON_ONCE(desc->mr->length == 0);
1376
1377 req->reg_cqe.done = srp_reg_mr_err_done;
1378
1379 wr.wr.next = NULL;
1380 wr.wr.opcode = IB_WR_REG_MR;
1381 wr.wr.wr_cqe = &req->reg_cqe;
1382 wr.wr.num_sge = 0;
1383 wr.wr.send_flags = 0;
1384 wr.mr = desc->mr;
1385 wr.key = desc->mr->rkey;
1386 wr.access = (IB_ACCESS_LOCAL_WRITE |
1387 IB_ACCESS_REMOTE_READ |
1388 IB_ACCESS_REMOTE_WRITE);
1389
1390 *state->fr.next++ = desc;
1391 state->nmdesc++;
1392
1393 srp_map_desc(state, desc->mr->iova,
1394 desc->mr->length, desc->mr->rkey);
1395
1396 err = ib_post_send(ch->qp, &wr.wr, &bad_wr);
1397 if (unlikely(err)) {
1398 WARN_ON_ONCE(err == -ENOMEM);
1399 return err;
1400 }
1401
1402 return n;
1403 }
1404
1405 static int srp_map_sg_entry(struct srp_map_state *state,
1406 struct srp_rdma_ch *ch,
1407 struct scatterlist *sg)
1408 {
1409 struct srp_target_port *target = ch->target;
1410 struct srp_device *dev = target->srp_host->srp_dev;
1411 struct ib_device *ibdev = dev->dev;
1412 dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg);
1413 unsigned int dma_len = ib_sg_dma_len(ibdev, sg);
1414 unsigned int len = 0;
1415 int ret;
1416
1417 WARN_ON_ONCE(!dma_len);
1418
1419 while (dma_len) {
1420 unsigned offset = dma_addr & ~dev->mr_page_mask;
1421
1422 if (state->npages == dev->max_pages_per_mr ||
1423 (state->npages > 0 && offset != 0)) {
1424 ret = srp_map_finish_fmr(state, ch);
1425 if (ret)
1426 return ret;
1427 }
1428
1429 len = min_t(unsigned int, dma_len, dev->mr_page_size - offset);
1430
1431 if (!state->npages)
1432 state->base_dma_addr = dma_addr;
1433 state->pages[state->npages++] = dma_addr & dev->mr_page_mask;
1434 state->dma_len += len;
1435 dma_addr += len;
1436 dma_len -= len;
1437 }
1438
1439 /*
1440 * If the end of the MR is not on a page boundary then we need to
1441 * close it out and start a new one -- we can only merge at page
1442 * boundaries.
1443 */
1444 ret = 0;
1445 if ((dma_addr & ~dev->mr_page_mask) != 0)
1446 ret = srp_map_finish_fmr(state, ch);
1447 return ret;
1448 }
1449
1450 static int srp_map_sg_fmr(struct srp_map_state *state, struct srp_rdma_ch *ch,
1451 struct srp_request *req, struct scatterlist *scat,
1452 int count)
1453 {
1454 struct scatterlist *sg;
1455 int i, ret;
1456
1457 state->pages = req->map_page;
1458 state->fmr.next = req->fmr_list;
1459 state->fmr.end = req->fmr_list + ch->target->mr_per_cmd;
1460
1461 for_each_sg(scat, sg, count, i) {
1462 ret = srp_map_sg_entry(state, ch, sg);
1463 if (ret)
1464 return ret;
1465 }
1466
1467 ret = srp_map_finish_fmr(state, ch);
1468 if (ret)
1469 return ret;
1470
1471 return 0;
1472 }
1473
1474 static int srp_map_sg_fr(struct srp_map_state *state, struct srp_rdma_ch *ch,
1475 struct srp_request *req, struct scatterlist *scat,
1476 int count)
1477 {
1478 unsigned int sg_offset = 0;
1479
1480 state->fr.next = req->fr_list;
1481 state->fr.end = req->fr_list + ch->target->mr_per_cmd;
1482 state->sg = scat;
1483
1484 if (count == 0)
1485 return 0;
1486
1487 while (count) {
1488 int i, n;
1489
1490 n = srp_map_finish_fr(state, req, ch, count, &sg_offset);
1491 if (unlikely(n < 0))
1492 return n;
1493
1494 count -= n;
1495 for (i = 0; i < n; i++)
1496 state->sg = sg_next(state->sg);
1497 }
1498
1499 return 0;
1500 }
1501
1502 static int srp_map_sg_dma(struct srp_map_state *state, struct srp_rdma_ch *ch,
1503 struct srp_request *req, struct scatterlist *scat,
1504 int count)
1505 {
1506 struct srp_target_port *target = ch->target;
1507 struct srp_device *dev = target->srp_host->srp_dev;
1508 struct scatterlist *sg;
1509 int i;
1510
1511 for_each_sg(scat, sg, count, i) {
1512 srp_map_desc(state, ib_sg_dma_address(dev->dev, sg),
1513 ib_sg_dma_len(dev->dev, sg),
1514 target->pd->unsafe_global_rkey);
1515 }
1516
1517 return 0;
1518 }
1519
1520 /*
1521 * Register the indirect data buffer descriptor with the HCA.
1522 *
1523 * Note: since the indirect data buffer descriptor has been allocated with
1524 * kmalloc() it is guaranteed that this buffer is a physically contiguous
1525 * memory buffer.
1526 */
1527 static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req,
1528 void **next_mr, void **end_mr, u32 idb_len,
1529 __be32 *idb_rkey)
1530 {
1531 struct srp_target_port *target = ch->target;
1532 struct srp_device *dev = target->srp_host->srp_dev;
1533 struct srp_map_state state;
1534 struct srp_direct_buf idb_desc;
1535 u64 idb_pages[1];
1536 struct scatterlist idb_sg[1];
1537 int ret;
1538
1539 memset(&state, 0, sizeof(state));
1540 memset(&idb_desc, 0, sizeof(idb_desc));
1541 state.gen.next = next_mr;
1542 state.gen.end = end_mr;
1543 state.desc = &idb_desc;
1544 state.base_dma_addr = req->indirect_dma_addr;
1545 state.dma_len = idb_len;
1546
1547 if (dev->use_fast_reg) {
1548 state.sg = idb_sg;
1549 sg_init_one(idb_sg, req->indirect_desc, idb_len);
1550 idb_sg->dma_address = req->indirect_dma_addr; /* hack! */
1551 #ifdef CONFIG_NEED_SG_DMA_LENGTH
1552 idb_sg->dma_length = idb_sg->length; /* hack^2 */
1553 #endif
1554 ret = srp_map_finish_fr(&state, req, ch, 1, NULL);
1555 if (ret < 0)
1556 return ret;
1557 WARN_ON_ONCE(ret < 1);
1558 } else if (dev->use_fmr) {
1559 state.pages = idb_pages;
1560 state.pages[0] = (req->indirect_dma_addr &
1561 dev->mr_page_mask);
1562 state.npages = 1;
1563 ret = srp_map_finish_fmr(&state, ch);
1564 if (ret < 0)
1565 return ret;
1566 } else {
1567 return -EINVAL;
1568 }
1569
1570 *idb_rkey = idb_desc.key;
1571
1572 return 0;
1573 }
1574
1575 static void srp_check_mapping(struct srp_map_state *state,
1576 struct srp_rdma_ch *ch, struct srp_request *req,
1577 struct scatterlist *scat, int count)
1578 {
1579 struct srp_device *dev = ch->target->srp_host->srp_dev;
1580 struct srp_fr_desc **pfr;
1581 u64 desc_len = 0, mr_len = 0;
1582 int i;
1583
1584 for (i = 0; i < state->ndesc; i++)
1585 desc_len += be32_to_cpu(req->indirect_desc[i].len);
1586 if (dev->use_fast_reg)
1587 for (i = 0, pfr = req->fr_list; i < state->nmdesc; i++, pfr++)
1588 mr_len += (*pfr)->mr->length;
1589 else if (dev->use_fmr)
1590 for (i = 0; i < state->nmdesc; i++)
1591 mr_len += be32_to_cpu(req->indirect_desc[i].len);
1592 if (desc_len != scsi_bufflen(req->scmnd) ||
1593 mr_len > scsi_bufflen(req->scmnd))
1594 pr_err("Inconsistent: scsi len %d <> desc len %lld <> mr len %lld; ndesc %d; nmdesc = %d\n",
1595 scsi_bufflen(req->scmnd), desc_len, mr_len,
1596 state->ndesc, state->nmdesc);
1597 }
1598
1599 /**
1600 * srp_map_data() - map SCSI data buffer onto an SRP request
1601 * @scmnd: SCSI command to map
1602 * @ch: SRP RDMA channel
1603 * @req: SRP request
1604 *
1605 * Returns the length in bytes of the SRP_CMD IU or a negative value if
1606 * mapping failed.
1607 */
1608 static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
1609 struct srp_request *req)
1610 {
1611 struct srp_target_port *target = ch->target;
1612 struct ib_pd *pd = target->pd;
1613 struct scatterlist *scat;
1614 struct srp_cmd *cmd = req->cmd->buf;
1615 int len, nents, count, ret;
1616 struct srp_device *dev;
1617 struct ib_device *ibdev;
1618 struct srp_map_state state;
1619 struct srp_indirect_buf *indirect_hdr;
1620 u32 idb_len, table_len;
1621 __be32 idb_rkey;
1622 u8 fmt;
1623
1624 if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE)
1625 return sizeof (struct srp_cmd);
1626
1627 if (scmnd->sc_data_direction != DMA_FROM_DEVICE &&
1628 scmnd->sc_data_direction != DMA_TO_DEVICE) {
1629 shost_printk(KERN_WARNING, target->scsi_host,
1630 PFX "Unhandled data direction %d\n",
1631 scmnd->sc_data_direction);
1632 return -EINVAL;
1633 }
1634
1635 nents = scsi_sg_count(scmnd);
1636 scat = scsi_sglist(scmnd);
1637
1638 dev = target->srp_host->srp_dev;
1639 ibdev = dev->dev;
1640
1641 count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction);
1642 if (unlikely(count == 0))
1643 return -EIO;
1644
1645 fmt = SRP_DATA_DESC_DIRECT;
1646 len = sizeof (struct srp_cmd) + sizeof (struct srp_direct_buf);
1647
1648 if (count == 1 && (pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)) {
1649 /*
1650 * The midlayer only generated a single gather/scatter
1651 * entry, or DMA mapping coalesced everything to a
1652 * single entry. So a direct descriptor along with
1653 * the DMA MR suffices.
1654 */
1655 struct srp_direct_buf *buf = (void *) cmd->add_data;
1656
1657 buf->va = cpu_to_be64(ib_sg_dma_address(ibdev, scat));
1658 buf->key = cpu_to_be32(pd->unsafe_global_rkey);
1659 buf->len = cpu_to_be32(ib_sg_dma_len(ibdev, scat));
1660
1661 req->nmdesc = 0;
1662 goto map_complete;
1663 }
1664
1665 /*
1666 * We have more than one scatter/gather entry, so build our indirect
1667 * descriptor table, trying to merge as many entries as we can.
1668 */
1669 indirect_hdr = (void *) cmd->add_data;
1670
1671 ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr,
1672 target->indirect_size, DMA_TO_DEVICE);
1673
1674 memset(&state, 0, sizeof(state));
1675 state.desc = req->indirect_desc;
1676 if (dev->use_fast_reg)
1677 ret = srp_map_sg_fr(&state, ch, req, scat, count);
1678 else if (dev->use_fmr)
1679 ret = srp_map_sg_fmr(&state, ch, req, scat, count);
1680 else
1681 ret = srp_map_sg_dma(&state, ch, req, scat, count);
1682 req->nmdesc = state.nmdesc;
1683 if (ret < 0)
1684 goto unmap;
1685
1686 {
1687 DEFINE_DYNAMIC_DEBUG_METADATA(ddm,
1688 "Memory mapping consistency check");
1689 if (DYNAMIC_DEBUG_BRANCH(ddm))
1690 srp_check_mapping(&state, ch, req, scat, count);
1691 }
1692
1693 /* We've mapped the request, now pull as much of the indirect
1694 * descriptor table as we can into the command buffer. If this
1695 * target is not using an external indirect table, we are
1696 * guaranteed to fit into the command, as the SCSI layer won't
1697 * give us more S/G entries than we allow.
1698 */
1699 if (state.ndesc == 1) {
1700 /*
1701 * Memory registration collapsed the sg-list into one entry,
1702 * so use a direct descriptor.
1703 */
1704 struct srp_direct_buf *buf = (void *) cmd->add_data;
1705
1706 *buf = req->indirect_desc[0];
1707 goto map_complete;
1708 }
1709
1710 if (unlikely(target->cmd_sg_cnt < state.ndesc &&
1711 !target->allow_ext_sg)) {
1712 shost_printk(KERN_ERR, target->scsi_host,
1713 "Could not fit S/G list into SRP_CMD\n");
1714 ret = -EIO;
1715 goto unmap;
1716 }
1717
1718 count = min(state.ndesc, target->cmd_sg_cnt);
1719 table_len = state.ndesc * sizeof (struct srp_direct_buf);
1720 idb_len = sizeof(struct srp_indirect_buf) + table_len;
1721
1722 fmt = SRP_DATA_DESC_INDIRECT;
1723 len = sizeof(struct srp_cmd) + sizeof (struct srp_indirect_buf);
1724 len += count * sizeof (struct srp_direct_buf);
1725
1726 memcpy(indirect_hdr->desc_list, req->indirect_desc,
1727 count * sizeof (struct srp_direct_buf));
1728
1729 if (!(pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)) {
1730 ret = srp_map_idb(ch, req, state.gen.next, state.gen.end,
1731 idb_len, &idb_rkey);
1732 if (ret < 0)
1733 goto unmap;
1734 req->nmdesc++;
1735 } else {
1736 idb_rkey = cpu_to_be32(pd->unsafe_global_rkey);
1737 }
1738
1739 indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr);
1740 indirect_hdr->table_desc.key = idb_rkey;
1741 indirect_hdr->table_desc.len = cpu_to_be32(table_len);
1742 indirect_hdr->len = cpu_to_be32(state.total_len);
1743
1744 if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1745 cmd->data_out_desc_cnt = count;
1746 else
1747 cmd->data_in_desc_cnt = count;
1748
1749 ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len,
1750 DMA_TO_DEVICE);
1751
1752 map_complete:
1753 if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1754 cmd->buf_fmt = fmt << 4;
1755 else
1756 cmd->buf_fmt = fmt;
1757
1758 return len;
1759
1760 unmap:
1761 srp_unmap_data(scmnd, ch, req);
1762 if (ret == -ENOMEM && req->nmdesc >= target->mr_pool_size)
1763 ret = -E2BIG;
1764 return ret;
1765 }
1766
1767 /*
1768 * Return an IU and possible credit to the free pool
1769 */
1770 static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu,
1771 enum srp_iu_type iu_type)
1772 {
1773 unsigned long flags;
1774
1775 spin_lock_irqsave(&ch->lock, flags);
1776 list_add(&iu->list, &ch->free_tx);
1777 if (iu_type != SRP_IU_RSP)
1778 ++ch->req_lim;
1779 spin_unlock_irqrestore(&ch->lock, flags);
1780 }
1781
1782 /*
1783 * Must be called with ch->lock held to protect req_lim and free_tx.
1784 * If IU is not sent, it must be returned using srp_put_tx_iu().
1785 *
1786 * Note:
1787 * An upper limit for the number of allocated information units for each
1788 * request type is:
1789 * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues
1790 * more than Scsi_Host.can_queue requests.
1791 * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE.
1792 * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than
1793 * one unanswered SRP request to an initiator.
1794 */
1795 static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch,
1796 enum srp_iu_type iu_type)
1797 {
1798 struct srp_target_port *target = ch->target;
1799 s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
1800 struct srp_iu *iu;
1801
1802 ib_process_cq_direct(ch->send_cq, -1);
1803
1804 if (list_empty(&ch->free_tx))
1805 return NULL;
1806
1807 /* Initiator responses to target requests do not consume credits */
1808 if (iu_type != SRP_IU_RSP) {
1809 if (ch->req_lim <= rsv) {
1810 ++target->zero_req_lim;
1811 return NULL;
1812 }
1813
1814 --ch->req_lim;
1815 }
1816
1817 iu = list_first_entry(&ch->free_tx, struct srp_iu, list);
1818 list_del(&iu->list);
1819 return iu;
1820 }
1821
1822 static void srp_send_done(struct ib_cq *cq, struct ib_wc *wc)
1823 {
1824 struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
1825 struct srp_rdma_ch *ch = cq->cq_context;
1826
1827 if (unlikely(wc->status != IB_WC_SUCCESS)) {
1828 srp_handle_qp_err(cq, wc, "SEND");
1829 return;
1830 }
1831
1832 list_add(&iu->list, &ch->free_tx);
1833 }
1834
1835 static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len)
1836 {
1837 struct srp_target_port *target = ch->target;
1838 struct ib_sge list;
1839 struct ib_send_wr wr, *bad_wr;
1840
1841 list.addr = iu->dma;
1842 list.length = len;
1843 list.lkey = target->lkey;
1844
1845 iu->cqe.done = srp_send_done;
1846
1847 wr.next = NULL;
1848 wr.wr_cqe = &iu->cqe;
1849 wr.sg_list = &list;
1850 wr.num_sge = 1;
1851 wr.opcode = IB_WR_SEND;
1852 wr.send_flags = IB_SEND_SIGNALED;
1853
1854 return ib_post_send(ch->qp, &wr, &bad_wr);
1855 }
1856
1857 static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu)
1858 {
1859 struct srp_target_port *target = ch->target;
1860 struct ib_recv_wr wr, *bad_wr;
1861 struct ib_sge list;
1862
1863 list.addr = iu->dma;
1864 list.length = iu->size;
1865 list.lkey = target->lkey;
1866
1867 iu->cqe.done = srp_recv_done;
1868
1869 wr.next = NULL;
1870 wr.wr_cqe = &iu->cqe;
1871 wr.sg_list = &list;
1872 wr.num_sge = 1;
1873
1874 return ib_post_recv(ch->qp, &wr, &bad_wr);
1875 }
1876
1877 static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp)
1878 {
1879 struct srp_target_port *target = ch->target;
1880 struct srp_request *req;
1881 struct scsi_cmnd *scmnd;
1882 unsigned long flags;
1883
1884 if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) {
1885 spin_lock_irqsave(&ch->lock, flags);
1886 ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1887 spin_unlock_irqrestore(&ch->lock, flags);
1888
1889 ch->tsk_mgmt_status = -1;
1890 if (be32_to_cpu(rsp->resp_data_len) >= 4)
1891 ch->tsk_mgmt_status = rsp->data[3];
1892 complete(&ch->tsk_mgmt_done);
1893 } else {
1894 scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag);
1895 if (scmnd) {
1896 req = (void *)scmnd->host_scribble;
1897 scmnd = srp_claim_req(ch, req, NULL, scmnd);
1898 }
1899 if (!scmnd) {
1900 shost_printk(KERN_ERR, target->scsi_host,
1901 "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n",
1902 rsp->tag, ch - target->ch, ch->qp->qp_num);
1903
1904 spin_lock_irqsave(&ch->lock, flags);
1905 ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1906 spin_unlock_irqrestore(&ch->lock, flags);
1907
1908 return;
1909 }
1910 scmnd->result = rsp->status;
1911
1912 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) {
1913 memcpy(scmnd->sense_buffer, rsp->data +
1914 be32_to_cpu(rsp->resp_data_len),
1915 min_t(int, be32_to_cpu(rsp->sense_data_len),
1916 SCSI_SENSE_BUFFERSIZE));
1917 }
1918
1919 if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER))
1920 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt));
1921 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER))
1922 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt));
1923 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER))
1924 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt));
1925 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER))
1926 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt));
1927
1928 srp_free_req(ch, req, scmnd,
1929 be32_to_cpu(rsp->req_lim_delta));
1930
1931 scmnd->host_scribble = NULL;
1932 scmnd->scsi_done(scmnd);
1933 }
1934 }
1935
1936 static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta,
1937 void *rsp, int len)
1938 {
1939 struct srp_target_port *target = ch->target;
1940 struct ib_device *dev = target->srp_host->srp_dev->dev;
1941 unsigned long flags;
1942 struct srp_iu *iu;
1943 int err;
1944
1945 spin_lock_irqsave(&ch->lock, flags);
1946 ch->req_lim += req_delta;
1947 iu = __srp_get_tx_iu(ch, SRP_IU_RSP);
1948 spin_unlock_irqrestore(&ch->lock, flags);
1949
1950 if (!iu) {
1951 shost_printk(KERN_ERR, target->scsi_host, PFX
1952 "no IU available to send response\n");
1953 return 1;
1954 }
1955
1956 ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE);
1957 memcpy(iu->buf, rsp, len);
1958 ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE);
1959
1960 err = srp_post_send(ch, iu, len);
1961 if (err) {
1962 shost_printk(KERN_ERR, target->scsi_host, PFX
1963 "unable to post response: %d\n", err);
1964 srp_put_tx_iu(ch, iu, SRP_IU_RSP);
1965 }
1966
1967 return err;
1968 }
1969
1970 static void srp_process_cred_req(struct srp_rdma_ch *ch,
1971 struct srp_cred_req *req)
1972 {
1973 struct srp_cred_rsp rsp = {
1974 .opcode = SRP_CRED_RSP,
1975 .tag = req->tag,
1976 };
1977 s32 delta = be32_to_cpu(req->req_lim_delta);
1978
1979 if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
1980 shost_printk(KERN_ERR, ch->target->scsi_host, PFX
1981 "problems processing SRP_CRED_REQ\n");
1982 }
1983
1984 static void srp_process_aer_req(struct srp_rdma_ch *ch,
1985 struct srp_aer_req *req)
1986 {
1987 struct srp_target_port *target = ch->target;
1988 struct srp_aer_rsp rsp = {
1989 .opcode = SRP_AER_RSP,
1990 .tag = req->tag,
1991 };
1992 s32 delta = be32_to_cpu(req->req_lim_delta);
1993
1994 shost_printk(KERN_ERR, target->scsi_host, PFX
1995 "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun));
1996
1997 if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
1998 shost_printk(KERN_ERR, target->scsi_host, PFX
1999 "problems processing SRP_AER_REQ\n");
2000 }
2001
2002 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc)
2003 {
2004 struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
2005 struct srp_rdma_ch *ch = cq->cq_context;
2006 struct srp_target_port *target = ch->target;
2007 struct ib_device *dev = target->srp_host->srp_dev->dev;
2008 int res;
2009 u8 opcode;
2010
2011 if (unlikely(wc->status != IB_WC_SUCCESS)) {
2012 srp_handle_qp_err(cq, wc, "RECV");
2013 return;
2014 }
2015
2016 ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len,
2017 DMA_FROM_DEVICE);
2018
2019 opcode = *(u8 *) iu->buf;
2020
2021 if (0) {
2022 shost_printk(KERN_ERR, target->scsi_host,
2023 PFX "recv completion, opcode 0x%02x\n", opcode);
2024 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1,
2025 iu->buf, wc->byte_len, true);
2026 }
2027
2028 switch (opcode) {
2029 case SRP_RSP:
2030 srp_process_rsp(ch, iu->buf);
2031 break;
2032
2033 case SRP_CRED_REQ:
2034 srp_process_cred_req(ch, iu->buf);
2035 break;
2036
2037 case SRP_AER_REQ:
2038 srp_process_aer_req(ch, iu->buf);
2039 break;
2040
2041 case SRP_T_LOGOUT:
2042 /* XXX Handle target logout */
2043 shost_printk(KERN_WARNING, target->scsi_host,
2044 PFX "Got target logout request\n");
2045 break;
2046
2047 default:
2048 shost_printk(KERN_WARNING, target->scsi_host,
2049 PFX "Unhandled SRP opcode 0x%02x\n", opcode);
2050 break;
2051 }
2052
2053 ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len,
2054 DMA_FROM_DEVICE);
2055
2056 res = srp_post_recv(ch, iu);
2057 if (res != 0)
2058 shost_printk(KERN_ERR, target->scsi_host,
2059 PFX "Recv failed with error code %d\n", res);
2060 }
2061
2062 /**
2063 * srp_tl_err_work() - handle a transport layer error
2064 * @work: Work structure embedded in an SRP target port.
2065 *
2066 * Note: This function may get invoked before the rport has been created,
2067 * hence the target->rport test.
2068 */
2069 static void srp_tl_err_work(struct work_struct *work)
2070 {
2071 struct srp_target_port *target;
2072
2073 target = container_of(work, struct srp_target_port, tl_err_work);
2074 if (target->rport)
2075 srp_start_tl_fail_timers(target->rport);
2076 }
2077
2078 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
2079 const char *opname)
2080 {
2081 struct srp_rdma_ch *ch = cq->cq_context;
2082 struct srp_target_port *target = ch->target;
2083
2084 if (ch->connected && !target->qp_in_error) {
2085 shost_printk(KERN_ERR, target->scsi_host,
2086 PFX "failed %s status %s (%d) for CQE %p\n",
2087 opname, ib_wc_status_msg(wc->status), wc->status,
2088 wc->wr_cqe);
2089 queue_work(system_long_wq, &target->tl_err_work);
2090 }
2091 target->qp_in_error = true;
2092 }
2093
2094 static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
2095 {
2096 struct srp_target_port *target = host_to_target(shost);
2097 struct srp_rport *rport = target->rport;
2098 struct srp_rdma_ch *ch;
2099 struct srp_request *req;
2100 struct srp_iu *iu;
2101 struct srp_cmd *cmd;
2102 struct ib_device *dev;
2103 unsigned long flags;
2104 u32 tag;
2105 u16 idx;
2106 int len, ret;
2107 const bool in_scsi_eh = !in_interrupt() && current == shost->ehandler;
2108
2109 /*
2110 * The SCSI EH thread is the only context from which srp_queuecommand()
2111 * can get invoked for blocked devices (SDEV_BLOCK /
2112 * SDEV_CREATED_BLOCK). Avoid racing with srp_reconnect_rport() by
2113 * locking the rport mutex if invoked from inside the SCSI EH.
2114 */
2115 if (in_scsi_eh)
2116 mutex_lock(&rport->mutex);
2117
2118 scmnd->result = srp_chkready(target->rport);
2119 if (unlikely(scmnd->result))
2120 goto err;
2121
2122 WARN_ON_ONCE(scmnd->request->tag < 0);
2123 tag = blk_mq_unique_tag(scmnd->request);
2124 ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)];
2125 idx = blk_mq_unique_tag_to_tag(tag);
2126 WARN_ONCE(idx >= target->req_ring_size, "%s: tag %#x: idx %d >= %d\n",
2127 dev_name(&shost->shost_gendev), tag, idx,
2128 target->req_ring_size);
2129
2130 spin_lock_irqsave(&ch->lock, flags);
2131 iu = __srp_get_tx_iu(ch, SRP_IU_CMD);
2132 spin_unlock_irqrestore(&ch->lock, flags);
2133
2134 if (!iu)
2135 goto err;
2136
2137 req = &ch->req_ring[idx];
2138 dev = target->srp_host->srp_dev->dev;
2139 ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len,
2140 DMA_TO_DEVICE);
2141
2142 scmnd->host_scribble = (void *) req;
2143
2144 cmd = iu->buf;
2145 memset(cmd, 0, sizeof *cmd);
2146
2147 cmd->opcode = SRP_CMD;
2148 int_to_scsilun(scmnd->device->lun, &cmd->lun);
2149 cmd->tag = tag;
2150 memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len);
2151
2152 req->scmnd = scmnd;
2153 req->cmd = iu;
2154
2155 len = srp_map_data(scmnd, ch, req);
2156 if (len < 0) {
2157 shost_printk(KERN_ERR, target->scsi_host,
2158 PFX "Failed to map data (%d)\n", len);
2159 /*
2160 * If we ran out of memory descriptors (-ENOMEM) because an
2161 * application is queuing many requests with more than
2162 * max_pages_per_mr sg-list elements, tell the SCSI mid-layer
2163 * to reduce queue depth temporarily.
2164 */
2165 scmnd->result = len == -ENOMEM ?
2166 DID_OK << 16 | QUEUE_FULL << 1 : DID_ERROR << 16;
2167 goto err_iu;
2168 }
2169
2170 ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len,
2171 DMA_TO_DEVICE);
2172
2173 if (srp_post_send(ch, iu, len)) {
2174 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n");
2175 goto err_unmap;
2176 }
2177
2178 ret = 0;
2179
2180 unlock_rport:
2181 if (in_scsi_eh)
2182 mutex_unlock(&rport->mutex);
2183
2184 return ret;
2185
2186 err_unmap:
2187 srp_unmap_data(scmnd, ch, req);
2188
2189 err_iu:
2190 srp_put_tx_iu(ch, iu, SRP_IU_CMD);
2191
2192 /*
2193 * Avoid that the loops that iterate over the request ring can
2194 * encounter a dangling SCSI command pointer.
2195 */
2196 req->scmnd = NULL;
2197
2198 err:
2199 if (scmnd->result) {
2200 scmnd->scsi_done(scmnd);
2201 ret = 0;
2202 } else {
2203 ret = SCSI_MLQUEUE_HOST_BUSY;
2204 }
2205
2206 goto unlock_rport;
2207 }
2208
2209 /*
2210 * Note: the resources allocated in this function are freed in
2211 * srp_free_ch_ib().
2212 */
2213 static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch)
2214 {
2215 struct srp_target_port *target = ch->target;
2216 int i;
2217
2218 ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring),
2219 GFP_KERNEL);
2220 if (!ch->rx_ring)
2221 goto err_no_ring;
2222 ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring),
2223 GFP_KERNEL);
2224 if (!ch->tx_ring)
2225 goto err_no_ring;
2226
2227 for (i = 0; i < target->queue_size; ++i) {
2228 ch->rx_ring[i] = srp_alloc_iu(target->srp_host,
2229 ch->max_ti_iu_len,
2230 GFP_KERNEL, DMA_FROM_DEVICE);
2231 if (!ch->rx_ring[i])
2232 goto err;
2233 }
2234
2235 for (i = 0; i < target->queue_size; ++i) {
2236 ch->tx_ring[i] = srp_alloc_iu(target->srp_host,
2237 target->max_iu_len,
2238 GFP_KERNEL, DMA_TO_DEVICE);
2239 if (!ch->tx_ring[i])
2240 goto err;
2241
2242 list_add(&ch->tx_ring[i]->list, &ch->free_tx);
2243 }
2244
2245 return 0;
2246
2247 err:
2248 for (i = 0; i < target->queue_size; ++i) {
2249 srp_free_iu(target->srp_host, ch->rx_ring[i]);
2250 srp_free_iu(target->srp_host, ch->tx_ring[i]);
2251 }
2252
2253
2254 err_no_ring:
2255 kfree(ch->tx_ring);
2256 ch->tx_ring = NULL;
2257 kfree(ch->rx_ring);
2258 ch->rx_ring = NULL;
2259
2260 return -ENOMEM;
2261 }
2262
2263 static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask)
2264 {
2265 uint64_t T_tr_ns, max_compl_time_ms;
2266 uint32_t rq_tmo_jiffies;
2267
2268 /*
2269 * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair,
2270 * table 91), both the QP timeout and the retry count have to be set
2271 * for RC QP's during the RTR to RTS transition.
2272 */
2273 WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) !=
2274 (IB_QP_TIMEOUT | IB_QP_RETRY_CNT));
2275
2276 /*
2277 * Set target->rq_tmo_jiffies to one second more than the largest time
2278 * it can take before an error completion is generated. See also
2279 * C9-140..142 in the IBTA spec for more information about how to
2280 * convert the QP Local ACK Timeout value to nanoseconds.
2281 */
2282 T_tr_ns = 4096 * (1ULL << qp_attr->timeout);
2283 max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns;
2284 do_div(max_compl_time_ms, NSEC_PER_MSEC);
2285 rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000);
2286
2287 return rq_tmo_jiffies;
2288 }
2289
2290 static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
2291 const struct srp_login_rsp *lrsp,
2292 struct srp_rdma_ch *ch)
2293 {
2294 struct srp_target_port *target = ch->target;
2295 struct ib_qp_attr *qp_attr = NULL;
2296 int attr_mask = 0;
2297 int ret;
2298 int i;
2299
2300 if (lrsp->opcode == SRP_LOGIN_RSP) {
2301 ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
2302 ch->req_lim = be32_to_cpu(lrsp->req_lim_delta);
2303
2304 /*
2305 * Reserve credits for task management so we don't
2306 * bounce requests back to the SCSI mid-layer.
2307 */
2308 target->scsi_host->can_queue
2309 = min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE,
2310 target->scsi_host->can_queue);
2311 target->scsi_host->cmd_per_lun
2312 = min_t(int, target->scsi_host->can_queue,
2313 target->scsi_host->cmd_per_lun);
2314 } else {
2315 shost_printk(KERN_WARNING, target->scsi_host,
2316 PFX "Unhandled RSP opcode %#x\n", lrsp->opcode);
2317 ret = -ECONNRESET;
2318 goto error;
2319 }
2320
2321 if (!ch->rx_ring) {
2322 ret = srp_alloc_iu_bufs(ch);
2323 if (ret)
2324 goto error;
2325 }
2326
2327 ret = -ENOMEM;
2328 qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
2329 if (!qp_attr)
2330 goto error;
2331
2332 qp_attr->qp_state = IB_QPS_RTR;
2333 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2334 if (ret)
2335 goto error_free;
2336
2337 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
2338 if (ret)
2339 goto error_free;
2340
2341 for (i = 0; i < target->queue_size; i++) {
2342 struct srp_iu *iu = ch->rx_ring[i];
2343
2344 ret = srp_post_recv(ch, iu);
2345 if (ret)
2346 goto error_free;
2347 }
2348
2349 qp_attr->qp_state = IB_QPS_RTS;
2350 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2351 if (ret)
2352 goto error_free;
2353
2354 target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);
2355
2356 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
2357 if (ret)
2358 goto error_free;
2359
2360 ret = ib_send_cm_rtu(cm_id, NULL, 0);
2361
2362 error_free:
2363 kfree(qp_attr);
2364
2365 error:
2366 ch->status = ret;
2367 }
2368
2369 static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
2370 struct ib_cm_event *event,
2371 struct srp_rdma_ch *ch)
2372 {
2373 struct srp_target_port *target = ch->target;
2374 struct Scsi_Host *shost = target->scsi_host;
2375 struct ib_class_port_info *cpi;
2376 int opcode;
2377
2378 switch (event->param.rej_rcvd.reason) {
2379 case IB_CM_REJ_PORT_CM_REDIRECT:
2380 cpi = event->param.rej_rcvd.ari;
2381 ch->path.dlid = cpi->redirect_lid;
2382 ch->path.pkey = cpi->redirect_pkey;
2383 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
2384 memcpy(ch->path.dgid.raw, cpi->redirect_gid, 16);
2385
2386 ch->status = ch->path.dlid ?
2387 SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
2388 break;
2389
2390 case IB_CM_REJ_PORT_REDIRECT:
2391 if (srp_target_is_topspin(target)) {
2392 /*
2393 * Topspin/Cisco SRP gateways incorrectly send
2394 * reject reason code 25 when they mean 24
2395 * (port redirect).
2396 */
2397 memcpy(ch->path.dgid.raw,
2398 event->param.rej_rcvd.ari, 16);
2399
2400 shost_printk(KERN_DEBUG, shost,
2401 PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
2402 be64_to_cpu(ch->path.dgid.global.subnet_prefix),
2403 be64_to_cpu(ch->path.dgid.global.interface_id));
2404
2405 ch->status = SRP_PORT_REDIRECT;
2406 } else {
2407 shost_printk(KERN_WARNING, shost,
2408 " REJ reason: IB_CM_REJ_PORT_REDIRECT\n");
2409 ch->status = -ECONNRESET;
2410 }
2411 break;
2412
2413 case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
2414 shost_printk(KERN_WARNING, shost,
2415 " REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
2416 ch->status = -ECONNRESET;
2417 break;
2418
2419 case IB_CM_REJ_CONSUMER_DEFINED:
2420 opcode = *(u8 *) event->private_data;
2421 if (opcode == SRP_LOGIN_REJ) {
2422 struct srp_login_rej *rej = event->private_data;
2423 u32 reason = be32_to_cpu(rej->reason);
2424
2425 if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
2426 shost_printk(KERN_WARNING, shost,
2427 PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
2428 else
2429 shost_printk(KERN_WARNING, shost, PFX
2430 "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n",
2431 target->sgid.raw,
2432 target->orig_dgid.raw, reason);
2433 } else
2434 shost_printk(KERN_WARNING, shost,
2435 " REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
2436 " opcode 0x%02x\n", opcode);
2437 ch->status = -ECONNRESET;
2438 break;
2439
2440 case IB_CM_REJ_STALE_CONN:
2441 shost_printk(KERN_WARNING, shost, " REJ reason: stale connection\n");
2442 ch->status = SRP_STALE_CONN;
2443 break;
2444
2445 default:
2446 shost_printk(KERN_WARNING, shost, " REJ reason 0x%x\n",
2447 event->param.rej_rcvd.reason);
2448 ch->status = -ECONNRESET;
2449 }
2450 }
2451
2452 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2453 {
2454 struct srp_rdma_ch *ch = cm_id->context;
2455 struct srp_target_port *target = ch->target;
2456 int comp = 0;
2457
2458 switch (event->event) {
2459 case IB_CM_REQ_ERROR:
2460 shost_printk(KERN_DEBUG, target->scsi_host,
2461 PFX "Sending CM REQ failed\n");
2462 comp = 1;
2463 ch->status = -ECONNRESET;
2464 break;
2465
2466 case IB_CM_REP_RECEIVED:
2467 comp = 1;
2468 srp_cm_rep_handler(cm_id, event->private_data, ch);
2469 break;
2470
2471 case IB_CM_REJ_RECEIVED:
2472 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
2473 comp = 1;
2474
2475 srp_cm_rej_handler(cm_id, event, ch);
2476 break;
2477
2478 case IB_CM_DREQ_RECEIVED:
2479 shost_printk(KERN_WARNING, target->scsi_host,
2480 PFX "DREQ received - connection closed\n");
2481 ch->connected = false;
2482 if (ib_send_cm_drep(cm_id, NULL, 0))
2483 shost_printk(KERN_ERR, target->scsi_host,
2484 PFX "Sending CM DREP failed\n");
2485 queue_work(system_long_wq, &target->tl_err_work);
2486 break;
2487
2488 case IB_CM_TIMEWAIT_EXIT:
2489 shost_printk(KERN_ERR, target->scsi_host,
2490 PFX "connection closed\n");
2491 comp = 1;
2492
2493 ch->status = 0;
2494 break;
2495
2496 case IB_CM_MRA_RECEIVED:
2497 case IB_CM_DREQ_ERROR:
2498 case IB_CM_DREP_RECEIVED:
2499 break;
2500
2501 default:
2502 shost_printk(KERN_WARNING, target->scsi_host,
2503 PFX "Unhandled CM event %d\n", event->event);
2504 break;
2505 }
2506
2507 if (comp)
2508 complete(&ch->done);
2509
2510 return 0;
2511 }
2512
2513 /**
2514 * srp_change_queue_depth - setting device queue depth
2515 * @sdev: scsi device struct
2516 * @qdepth: requested queue depth
2517 *
2518 * Returns queue depth.
2519 */
2520 static int
2521 srp_change_queue_depth(struct scsi_device *sdev, int qdepth)
2522 {
2523 if (!sdev->tagged_supported)
2524 qdepth = 1;
2525 return scsi_change_queue_depth(sdev, qdepth);
2526 }
2527
2528 static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag, u64 lun,
2529 u8 func)
2530 {
2531 struct srp_target_port *target = ch->target;
2532 struct srp_rport *rport = target->rport;
2533 struct ib_device *dev = target->srp_host->srp_dev->dev;
2534 struct srp_iu *iu;
2535 struct srp_tsk_mgmt *tsk_mgmt;
2536
2537 if (!ch->connected || target->qp_in_error)
2538 return -1;
2539
2540 init_completion(&ch->tsk_mgmt_done);
2541
2542 /*
2543 * Lock the rport mutex to avoid that srp_create_ch_ib() is
2544 * invoked while a task management function is being sent.
2545 */
2546 mutex_lock(&rport->mutex);
2547 spin_lock_irq(&ch->lock);
2548 iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT);
2549 spin_unlock_irq(&ch->lock);
2550
2551 if (!iu) {
2552 mutex_unlock(&rport->mutex);
2553
2554 return -1;
2555 }
2556
2557 ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt,
2558 DMA_TO_DEVICE);
2559 tsk_mgmt = iu->buf;
2560 memset(tsk_mgmt, 0, sizeof *tsk_mgmt);
2561
2562 tsk_mgmt->opcode = SRP_TSK_MGMT;
2563 int_to_scsilun(lun, &tsk_mgmt->lun);
2564 tsk_mgmt->tag = req_tag | SRP_TAG_TSK_MGMT;
2565 tsk_mgmt->tsk_mgmt_func = func;
2566 tsk_mgmt->task_tag = req_tag;
2567
2568 ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt,
2569 DMA_TO_DEVICE);
2570 if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) {
2571 srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT);
2572 mutex_unlock(&rport->mutex);
2573
2574 return -1;
2575 }
2576 mutex_unlock(&rport->mutex);
2577
2578 if (!wait_for_completion_timeout(&ch->tsk_mgmt_done,
2579 msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS)))
2580 return -1;
2581
2582 return 0;
2583 }
2584
2585 static int srp_abort(struct scsi_cmnd *scmnd)
2586 {
2587 struct srp_target_port *target = host_to_target(scmnd->device->host);
2588 struct srp_request *req = (struct srp_request *) scmnd->host_scribble;
2589 u32 tag;
2590 u16 ch_idx;
2591 struct srp_rdma_ch *ch;
2592 int ret;
2593
2594 shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n");
2595
2596 if (!req)
2597 return SUCCESS;
2598 tag = blk_mq_unique_tag(scmnd->request);
2599 ch_idx = blk_mq_unique_tag_to_hwq(tag);
2600 if (WARN_ON_ONCE(ch_idx >= target->ch_count))
2601 return SUCCESS;
2602 ch = &target->ch[ch_idx];
2603 if (!srp_claim_req(ch, req, NULL, scmnd))
2604 return SUCCESS;
2605 shost_printk(KERN_ERR, target->scsi_host,
2606 "Sending SRP abort for tag %#x\n", tag);
2607 if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun,
2608 SRP_TSK_ABORT_TASK) == 0)
2609 ret = SUCCESS;
2610 else if (target->rport->state == SRP_RPORT_LOST)
2611 ret = FAST_IO_FAIL;
2612 else
2613 ret = FAILED;
2614 srp_free_req(ch, req, scmnd, 0);
2615 scmnd->result = DID_ABORT << 16;
2616 scmnd->scsi_done(scmnd);
2617
2618 return ret;
2619 }
2620
2621 static int srp_reset_device(struct scsi_cmnd *scmnd)
2622 {
2623 struct srp_target_port *target = host_to_target(scmnd->device->host);
2624 struct srp_rdma_ch *ch;
2625 int i;
2626
2627 shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n");
2628
2629 ch = &target->ch[0];
2630 if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun,
2631 SRP_TSK_LUN_RESET))
2632 return FAILED;
2633 if (ch->tsk_mgmt_status)
2634 return FAILED;
2635
2636 for (i = 0; i < target->ch_count; i++) {
2637 ch = &target->ch[i];
2638 for (i = 0; i < target->req_ring_size; ++i) {
2639 struct srp_request *req = &ch->req_ring[i];
2640
2641 srp_finish_req(ch, req, scmnd->device, DID_RESET << 16);
2642 }
2643 }
2644
2645 return SUCCESS;
2646 }
2647
2648 static int srp_reset_host(struct scsi_cmnd *scmnd)
2649 {
2650 struct srp_target_port *target = host_to_target(scmnd->device->host);
2651
2652 shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n");
2653
2654 return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED;
2655 }
2656
2657 static int srp_slave_alloc(struct scsi_device *sdev)
2658 {
2659 struct Scsi_Host *shost = sdev->host;
2660 struct srp_target_port *target = host_to_target(shost);
2661 struct srp_device *srp_dev = target->srp_host->srp_dev;
2662 struct ib_device *ibdev = srp_dev->dev;
2663
2664 if (!(ibdev->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG))
2665 blk_queue_virt_boundary(sdev->request_queue,
2666 ~srp_dev->mr_page_mask);
2667
2668 return 0;
2669 }
2670
2671 static int srp_slave_configure(struct scsi_device *sdev)
2672 {
2673 struct Scsi_Host *shost = sdev->host;
2674 struct srp_target_port *target = host_to_target(shost);
2675 struct request_queue *q = sdev->request_queue;
2676 unsigned long timeout;
2677
2678 if (sdev->type == TYPE_DISK) {
2679 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies);
2680 blk_queue_rq_timeout(q, timeout);
2681 }
2682
2683 return 0;
2684 }
2685
2686 static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr,
2687 char *buf)
2688 {
2689 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2690
2691 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->id_ext));
2692 }
2693
2694 static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr,
2695 char *buf)
2696 {
2697 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2698
2699 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid));
2700 }
2701
2702 static ssize_t show_service_id(struct device *dev,
2703 struct device_attribute *attr, char *buf)
2704 {
2705 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2706
2707 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->service_id));
2708 }
2709
2710 static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
2711 char *buf)
2712 {
2713 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2714
2715 return sprintf(buf, "0x%04x\n", be16_to_cpu(target->pkey));
2716 }
2717
2718 static ssize_t show_sgid(struct device *dev, struct device_attribute *attr,
2719 char *buf)
2720 {
2721 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2722
2723 return sprintf(buf, "%pI6\n", target->sgid.raw);
2724 }
2725
2726 static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
2727 char *buf)
2728 {
2729 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2730 struct srp_rdma_ch *ch = &target->ch[0];
2731
2732 return sprintf(buf, "%pI6\n", ch->path.dgid.raw);
2733 }
2734
2735 static ssize_t show_orig_dgid(struct device *dev,
2736 struct device_attribute *attr, char *buf)
2737 {
2738 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2739
2740 return sprintf(buf, "%pI6\n", target->orig_dgid.raw);
2741 }
2742
2743 static ssize_t show_req_lim(struct device *dev,
2744 struct device_attribute *attr, char *buf)
2745 {
2746 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2747 struct srp_rdma_ch *ch;
2748 int i, req_lim = INT_MAX;
2749
2750 for (i = 0; i < target->ch_count; i++) {
2751 ch = &target->ch[i];
2752 req_lim = min(req_lim, ch->req_lim);
2753 }
2754 return sprintf(buf, "%d\n", req_lim);
2755 }
2756
2757 static ssize_t show_zero_req_lim(struct device *dev,
2758 struct device_attribute *attr, char *buf)
2759 {
2760 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2761
2762 return sprintf(buf, "%d\n", target->zero_req_lim);
2763 }
2764
2765 static ssize_t show_local_ib_port(struct device *dev,
2766 struct device_attribute *attr, char *buf)
2767 {
2768 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2769
2770 return sprintf(buf, "%d\n", target->srp_host->port);
2771 }
2772
2773 static ssize_t show_local_ib_device(struct device *dev,
2774 struct device_attribute *attr, char *buf)
2775 {
2776 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2777
2778 return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
2779 }
2780
2781 static ssize_t show_ch_count(struct device *dev, struct device_attribute *attr,
2782 char *buf)
2783 {
2784 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2785
2786 return sprintf(buf, "%d\n", target->ch_count);
2787 }
2788
2789 static ssize_t show_comp_vector(struct device *dev,
2790 struct device_attribute *attr, char *buf)
2791 {
2792 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2793
2794 return sprintf(buf, "%d\n", target->comp_vector);
2795 }
2796
2797 static ssize_t show_tl_retry_count(struct device *dev,
2798 struct device_attribute *attr, char *buf)
2799 {
2800 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2801
2802 return sprintf(buf, "%d\n", target->tl_retry_count);
2803 }
2804
2805 static ssize_t show_cmd_sg_entries(struct device *dev,
2806 struct device_attribute *attr, char *buf)
2807 {
2808 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2809
2810 return sprintf(buf, "%u\n", target->cmd_sg_cnt);
2811 }
2812
2813 static ssize_t show_allow_ext_sg(struct device *dev,
2814 struct device_attribute *attr, char *buf)
2815 {
2816 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2817
2818 return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
2819 }
2820
2821 static DEVICE_ATTR(id_ext, S_IRUGO, show_id_ext, NULL);
2822 static DEVICE_ATTR(ioc_guid, S_IRUGO, show_ioc_guid, NULL);
2823 static DEVICE_ATTR(service_id, S_IRUGO, show_service_id, NULL);
2824 static DEVICE_ATTR(pkey, S_IRUGO, show_pkey, NULL);
2825 static DEVICE_ATTR(sgid, S_IRUGO, show_sgid, NULL);
2826 static DEVICE_ATTR(dgid, S_IRUGO, show_dgid, NULL);
2827 static DEVICE_ATTR(orig_dgid, S_IRUGO, show_orig_dgid, NULL);
2828 static DEVICE_ATTR(req_lim, S_IRUGO, show_req_lim, NULL);
2829 static DEVICE_ATTR(zero_req_lim, S_IRUGO, show_zero_req_lim, NULL);
2830 static DEVICE_ATTR(local_ib_port, S_IRUGO, show_local_ib_port, NULL);
2831 static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
2832 static DEVICE_ATTR(ch_count, S_IRUGO, show_ch_count, NULL);
2833 static DEVICE_ATTR(comp_vector, S_IRUGO, show_comp_vector, NULL);
2834 static DEVICE_ATTR(tl_retry_count, S_IRUGO, show_tl_retry_count, NULL);
2835 static DEVICE_ATTR(cmd_sg_entries, S_IRUGO, show_cmd_sg_entries, NULL);
2836 static DEVICE_ATTR(allow_ext_sg, S_IRUGO, show_allow_ext_sg, NULL);
2837
2838 static struct device_attribute *srp_host_attrs[] = {
2839 &dev_attr_id_ext,
2840 &dev_attr_ioc_guid,
2841 &dev_attr_service_id,
2842 &dev_attr_pkey,
2843 &dev_attr_sgid,
2844 &dev_attr_dgid,
2845 &dev_attr_orig_dgid,
2846 &dev_attr_req_lim,
2847 &dev_attr_zero_req_lim,
2848 &dev_attr_local_ib_port,
2849 &dev_attr_local_ib_device,
2850 &dev_attr_ch_count,
2851 &dev_attr_comp_vector,
2852 &dev_attr_tl_retry_count,
2853 &dev_attr_cmd_sg_entries,
2854 &dev_attr_allow_ext_sg,
2855 NULL
2856 };
2857
2858 static struct scsi_host_template srp_template = {
2859 .module = THIS_MODULE,
2860 .name = "InfiniBand SRP initiator",
2861 .proc_name = DRV_NAME,
2862 .slave_alloc = srp_slave_alloc,
2863 .slave_configure = srp_slave_configure,
2864 .info = srp_target_info,
2865 .queuecommand = srp_queuecommand,
2866 .change_queue_depth = srp_change_queue_depth,
2867 .eh_abort_handler = srp_abort,
2868 .eh_device_reset_handler = srp_reset_device,
2869 .eh_host_reset_handler = srp_reset_host,
2870 .skip_settle_delay = true,
2871 .sg_tablesize = SRP_DEF_SG_TABLESIZE,
2872 .can_queue = SRP_DEFAULT_CMD_SQ_SIZE,
2873 .this_id = -1,
2874 .cmd_per_lun = SRP_DEFAULT_CMD_SQ_SIZE,
2875 .use_clustering = ENABLE_CLUSTERING,
2876 .shost_attrs = srp_host_attrs,
2877 .track_queue_depth = 1,
2878 };
2879
2880 static int srp_sdev_count(struct Scsi_Host *host)
2881 {
2882 struct scsi_device *sdev;
2883 int c = 0;
2884
2885 shost_for_each_device(sdev, host)
2886 c++;
2887
2888 return c;
2889 }
2890
2891 /*
2892 * Return values:
2893 * < 0 upon failure. Caller is responsible for SRP target port cleanup.
2894 * 0 and target->state == SRP_TARGET_REMOVED if asynchronous target port
2895 * removal has been scheduled.
2896 * 0 and target->state != SRP_TARGET_REMOVED upon success.
2897 */
2898 static int srp_add_target(struct srp_host *host, struct srp_target_port *target)
2899 {
2900 struct srp_rport_identifiers ids;
2901 struct srp_rport *rport;
2902
2903 target->state = SRP_TARGET_SCANNING;
2904 sprintf(target->target_name, "SRP.T10:%016llX",
2905 be64_to_cpu(target->id_ext));
2906
2907 if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dma_device))
2908 return -ENODEV;
2909
2910 memcpy(ids.port_id, &target->id_ext, 8);
2911 memcpy(ids.port_id + 8, &target->ioc_guid, 8);
2912 ids.roles = SRP_RPORT_ROLE_TARGET;
2913 rport = srp_rport_add(target->scsi_host, &ids);
2914 if (IS_ERR(rport)) {
2915 scsi_remove_host(target->scsi_host);
2916 return PTR_ERR(rport);
2917 }
2918
2919 rport->lld_data = target;
2920 target->rport = rport;
2921
2922 spin_lock(&host->target_lock);
2923 list_add_tail(&target->list, &host->target_list);
2924 spin_unlock(&host->target_lock);
2925
2926 scsi_scan_target(&target->scsi_host->shost_gendev,
2927 0, target->scsi_id, SCAN_WILD_CARD, SCSI_SCAN_INITIAL);
2928
2929 if (srp_connected_ch(target) < target->ch_count ||
2930 target->qp_in_error) {
2931 shost_printk(KERN_INFO, target->scsi_host,
2932 PFX "SCSI scan failed - removing SCSI host\n");
2933 srp_queue_remove_work(target);
2934 goto out;
2935 }
2936
2937 pr_debug("%s: SCSI scan succeeded - detected %d LUNs\n",
2938 dev_name(&target->scsi_host->shost_gendev),
2939 srp_sdev_count(target->scsi_host));
2940
2941 spin_lock_irq(&target->lock);
2942 if (target->state == SRP_TARGET_SCANNING)
2943 target->state = SRP_TARGET_LIVE;
2944 spin_unlock_irq(&target->lock);
2945
2946 out:
2947 return 0;
2948 }
2949
2950 static void srp_release_dev(struct device *dev)
2951 {
2952 struct srp_host *host =
2953 container_of(dev, struct srp_host, dev);
2954
2955 complete(&host->released);
2956 }
2957
2958 static struct class srp_class = {
2959 .name = "infiniband_srp",
2960 .dev_release = srp_release_dev
2961 };
2962
2963 /**
2964 * srp_conn_unique() - check whether the connection to a target is unique
2965 * @host: SRP host.
2966 * @target: SRP target port.
2967 */
2968 static bool srp_conn_unique(struct srp_host *host,
2969 struct srp_target_port *target)
2970 {
2971 struct srp_target_port *t;
2972 bool ret = false;
2973
2974 if (target->state == SRP_TARGET_REMOVED)
2975 goto out;
2976
2977 ret = true;
2978
2979 spin_lock(&host->target_lock);
2980 list_for_each_entry(t, &host->target_list, list) {
2981 if (t != target &&
2982 target->id_ext == t->id_ext &&
2983 target->ioc_guid == t->ioc_guid &&
2984 target->initiator_ext == t->initiator_ext) {
2985 ret = false;
2986 break;
2987 }
2988 }
2989 spin_unlock(&host->target_lock);
2990
2991 out:
2992 return ret;
2993 }
2994
2995 /*
2996 * Target ports are added by writing
2997 *
2998 * id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>,
2999 * pkey=<P_Key>,service_id=<service ID>
3000 *
3001 * to the add_target sysfs attribute.
3002 */
3003 enum {
3004 SRP_OPT_ERR = 0,
3005 SRP_OPT_ID_EXT = 1 << 0,
3006 SRP_OPT_IOC_GUID = 1 << 1,
3007 SRP_OPT_DGID = 1 << 2,
3008 SRP_OPT_PKEY = 1 << 3,
3009 SRP_OPT_SERVICE_ID = 1 << 4,
3010 SRP_OPT_MAX_SECT = 1 << 5,
3011 SRP_OPT_MAX_CMD_PER_LUN = 1 << 6,
3012 SRP_OPT_IO_CLASS = 1 << 7,
3013 SRP_OPT_INITIATOR_EXT = 1 << 8,
3014 SRP_OPT_CMD_SG_ENTRIES = 1 << 9,
3015 SRP_OPT_ALLOW_EXT_SG = 1 << 10,
3016 SRP_OPT_SG_TABLESIZE = 1 << 11,
3017 SRP_OPT_COMP_VECTOR = 1 << 12,
3018 SRP_OPT_TL_RETRY_COUNT = 1 << 13,
3019 SRP_OPT_QUEUE_SIZE = 1 << 14,
3020 SRP_OPT_ALL = (SRP_OPT_ID_EXT |
3021 SRP_OPT_IOC_GUID |
3022 SRP_OPT_DGID |
3023 SRP_OPT_PKEY |
3024 SRP_OPT_SERVICE_ID),
3025 };
3026
3027 static const match_table_t srp_opt_tokens = {
3028 { SRP_OPT_ID_EXT, "id_ext=%s" },
3029 { SRP_OPT_IOC_GUID, "ioc_guid=%s" },
3030 { SRP_OPT_DGID, "dgid=%s" },
3031 { SRP_OPT_PKEY, "pkey=%x" },
3032 { SRP_OPT_SERVICE_ID, "service_id=%s" },
3033 { SRP_OPT_MAX_SECT, "max_sect=%d" },
3034 { SRP_OPT_MAX_CMD_PER_LUN, "max_cmd_per_lun=%d" },
3035 { SRP_OPT_IO_CLASS, "io_class=%x" },
3036 { SRP_OPT_INITIATOR_EXT, "initiator_ext=%s" },
3037 { SRP_OPT_CMD_SG_ENTRIES, "cmd_sg_entries=%u" },
3038 { SRP_OPT_ALLOW_EXT_SG, "allow_ext_sg=%u" },
3039 { SRP_OPT_SG_TABLESIZE, "sg_tablesize=%u" },
3040 { SRP_OPT_COMP_VECTOR, "comp_vector=%u" },
3041 { SRP_OPT_TL_RETRY_COUNT, "tl_retry_count=%u" },
3042 { SRP_OPT_QUEUE_SIZE, "queue_size=%d" },
3043 { SRP_OPT_ERR, NULL }
3044 };
3045
3046 static int srp_parse_options(const char *buf, struct srp_target_port *target)
3047 {
3048 char *options, *sep_opt;
3049 char *p;
3050 char dgid[3];
3051 substring_t args[MAX_OPT_ARGS];
3052 int opt_mask = 0;
3053 int token;
3054 int ret = -EINVAL;
3055 int i;
3056
3057 options = kstrdup(buf, GFP_KERNEL);
3058 if (!options)
3059 return -ENOMEM;
3060
3061 sep_opt = options;
3062 while ((p = strsep(&sep_opt, ",\n")) != NULL) {
3063 if (!*p)
3064 continue;
3065
3066 token = match_token(p, srp_opt_tokens, args);
3067 opt_mask |= token;
3068
3069 switch (token) {
3070 case SRP_OPT_ID_EXT:
3071 p = match_strdup(args);
3072 if (!p) {
3073 ret = -ENOMEM;
3074 goto out;
3075 }
3076 target->id_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
3077 kfree(p);
3078 break;
3079
3080 case SRP_OPT_IOC_GUID:
3081 p = match_strdup(args);
3082 if (!p) {
3083 ret = -ENOMEM;
3084 goto out;
3085 }
3086 target->ioc_guid = cpu_to_be64(simple_strtoull(p, NULL, 16));
3087 kfree(p);
3088 break;
3089
3090 case SRP_OPT_DGID:
3091 p = match_strdup(args);
3092 if (!p) {
3093 ret = -ENOMEM;
3094 goto out;
3095 }
3096 if (strlen(p) != 32) {
3097 pr_warn("bad dest GID parameter '%s'\n", p);
3098 kfree(p);
3099 goto out;
3100 }
3101
3102 for (i = 0; i < 16; ++i) {
3103 strlcpy(dgid, p + i * 2, sizeof(dgid));
3104 if (sscanf(dgid, "%hhx",
3105 &target->orig_dgid.raw[i]) < 1) {
3106 ret = -EINVAL;
3107 kfree(p);
3108 goto out;
3109 }
3110 }
3111 kfree(p);
3112 break;
3113
3114 case SRP_OPT_PKEY:
3115 if (match_hex(args, &token)) {
3116 pr_warn("bad P_Key parameter '%s'\n", p);
3117 goto out;
3118 }
3119 target->pkey = cpu_to_be16(token);
3120 break;
3121
3122 case SRP_OPT_SERVICE_ID:
3123 p = match_strdup(args);
3124 if (!p) {
3125 ret = -ENOMEM;
3126 goto out;
3127 }
3128 target->service_id = cpu_to_be64(simple_strtoull(p, NULL, 16));
3129 kfree(p);
3130 break;
3131
3132 case SRP_OPT_MAX_SECT:
3133 if (match_int(args, &token)) {
3134 pr_warn("bad max sect parameter '%s'\n", p);
3135 goto out;
3136 }
3137 target->scsi_host->max_sectors = token;
3138 break;
3139
3140 case SRP_OPT_QUEUE_SIZE:
3141 if (match_int(args, &token) || token < 1) {
3142 pr_warn("bad queue_size parameter '%s'\n", p);
3143 goto out;
3144 }
3145 target->scsi_host->can_queue = token;
3146 target->queue_size = token + SRP_RSP_SQ_SIZE +
3147 SRP_TSK_MGMT_SQ_SIZE;
3148 if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3149 target->scsi_host->cmd_per_lun = token;
3150 break;
3151
3152 case SRP_OPT_MAX_CMD_PER_LUN:
3153 if (match_int(args, &token) || token < 1) {
3154 pr_warn("bad max cmd_per_lun parameter '%s'\n",
3155 p);
3156 goto out;
3157 }
3158 target->scsi_host->cmd_per_lun = token;
3159 break;
3160
3161 case SRP_OPT_IO_CLASS:
3162 if (match_hex(args, &token)) {
3163 pr_warn("bad IO class parameter '%s'\n", p);
3164 goto out;
3165 }
3166 if (token != SRP_REV10_IB_IO_CLASS &&
3167 token != SRP_REV16A_IB_IO_CLASS) {
3168 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
3169 token, SRP_REV10_IB_IO_CLASS,
3170 SRP_REV16A_IB_IO_CLASS);
3171 goto out;
3172 }
3173 target->io_class = token;
3174 break;
3175
3176 case SRP_OPT_INITIATOR_EXT:
3177 p = match_strdup(args);
3178 if (!p) {
3179 ret = -ENOMEM;
3180 goto out;
3181 }
3182 target->initiator_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
3183 kfree(p);
3184 break;
3185
3186 case SRP_OPT_CMD_SG_ENTRIES:
3187 if (match_int(args, &token) || token < 1 || token > 255) {
3188 pr_warn("bad max cmd_sg_entries parameter '%s'\n",
3189 p);
3190 goto out;
3191 }
3192 target->cmd_sg_cnt = token;
3193 break;
3194
3195 case SRP_OPT_ALLOW_EXT_SG:
3196 if (match_int(args, &token)) {
3197 pr_warn("bad allow_ext_sg parameter '%s'\n", p);
3198 goto out;
3199 }
3200 target->allow_ext_sg = !!token;
3201 break;
3202
3203 case SRP_OPT_SG_TABLESIZE:
3204 if (match_int(args, &token) || token < 1 ||
3205 token > SG_MAX_SEGMENTS) {
3206 pr_warn("bad max sg_tablesize parameter '%s'\n",
3207 p);
3208 goto out;
3209 }
3210 target->sg_tablesize = token;
3211 break;
3212
3213 case SRP_OPT_COMP_VECTOR:
3214 if (match_int(args, &token) || token < 0) {
3215 pr_warn("bad comp_vector parameter '%s'\n", p);
3216 goto out;
3217 }
3218 target->comp_vector = token;
3219 break;
3220
3221 case SRP_OPT_TL_RETRY_COUNT:
3222 if (match_int(args, &token) || token < 2 || token > 7) {
3223 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
3224 p);
3225 goto out;
3226 }
3227 target->tl_retry_count = token;
3228 break;
3229
3230 default:
3231 pr_warn("unknown parameter or missing value '%s' in target creation request\n",
3232 p);
3233 goto out;
3234 }
3235 }
3236
3237 if ((opt_mask & SRP_OPT_ALL) == SRP_OPT_ALL)
3238 ret = 0;
3239 else
3240 for (i = 0; i < ARRAY_SIZE(srp_opt_tokens); ++i)
3241 if ((srp_opt_tokens[i].token & SRP_OPT_ALL) &&
3242 !(srp_opt_tokens[i].token & opt_mask))
3243 pr_warn("target creation request is missing parameter '%s'\n",
3244 srp_opt_tokens[i].pattern);
3245
3246 if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue
3247 && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3248 pr_warn("cmd_per_lun = %d > queue_size = %d\n",
3249 target->scsi_host->cmd_per_lun,
3250 target->scsi_host->can_queue);
3251
3252 out:
3253 kfree(options);
3254 return ret;
3255 }
3256
3257 static ssize_t srp_create_target(struct device *dev,
3258 struct device_attribute *attr,
3259 const char *buf, size_t count)
3260 {
3261 struct srp_host *host =
3262 container_of(dev, struct srp_host, dev);
3263 struct Scsi_Host *target_host;
3264 struct srp_target_port *target;
3265 struct srp_rdma_ch *ch;
3266 struct srp_device *srp_dev = host->srp_dev;
3267 struct ib_device *ibdev = srp_dev->dev;
3268 int ret, node_idx, node, cpu, i;
3269 unsigned int max_sectors_per_mr, mr_per_cmd = 0;
3270 bool multich = false;
3271
3272 target_host = scsi_host_alloc(&srp_template,
3273 sizeof (struct srp_target_port));
3274 if (!target_host)
3275 return -ENOMEM;
3276
3277 target_host->transportt = ib_srp_transport_template;
3278 target_host->max_channel = 0;
3279 target_host->max_id = 1;
3280 target_host->max_lun = -1LL;
3281 target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb;
3282
3283 target = host_to_target(target_host);
3284
3285 target->io_class = SRP_REV16A_IB_IO_CLASS;
3286 target->scsi_host = target_host;
3287 target->srp_host = host;
3288 target->pd = host->srp_dev->pd;
3289 target->lkey = host->srp_dev->pd->local_dma_lkey;
3290 target->cmd_sg_cnt = cmd_sg_entries;
3291 target->sg_tablesize = indirect_sg_entries ? : cmd_sg_entries;
3292 target->allow_ext_sg = allow_ext_sg;
3293 target->tl_retry_count = 7;
3294 target->queue_size = SRP_DEFAULT_QUEUE_SIZE;
3295
3296 /*
3297 * Avoid that the SCSI host can be removed by srp_remove_target()
3298 * before this function returns.
3299 */
3300 scsi_host_get(target->scsi_host);
3301
3302 ret = mutex_lock_interruptible(&host->add_target_mutex);
3303 if (ret < 0)
3304 goto put;
3305
3306 ret = srp_parse_options(buf, target);
3307 if (ret)
3308 goto out;
3309
3310 target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE;
3311
3312 if (!srp_conn_unique(target->srp_host, target)) {
3313 shost_printk(KERN_INFO, target->scsi_host,
3314 PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
3315 be64_to_cpu(target->id_ext),
3316 be64_to_cpu(target->ioc_guid),
3317 be64_to_cpu(target->initiator_ext));
3318 ret = -EEXIST;
3319 goto out;
3320 }
3321
3322 if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg &&
3323 target->cmd_sg_cnt < target->sg_tablesize) {
3324 pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
3325 target->sg_tablesize = target->cmd_sg_cnt;
3326 }
3327
3328 if (srp_dev->use_fast_reg || srp_dev->use_fmr) {
3329 /*
3330 * FR and FMR can only map one HCA page per entry. If the
3331 * start address is not aligned on a HCA page boundary two
3332 * entries will be used for the head and the tail although
3333 * these two entries combined contain at most one HCA page of
3334 * data. Hence the "+ 1" in the calculation below.
3335 *
3336 * The indirect data buffer descriptor is contiguous so the
3337 * memory for that buffer will only be registered if
3338 * register_always is true. Hence add one to mr_per_cmd if
3339 * register_always has been set.
3340 */
3341 max_sectors_per_mr = srp_dev->max_pages_per_mr <<
3342 (ilog2(srp_dev->mr_page_size) - 9);
3343 mr_per_cmd = register_always +
3344 (target->scsi_host->max_sectors + 1 +
3345 max_sectors_per_mr - 1) / max_sectors_per_mr;
3346 pr_debug("max_sectors = %u; max_pages_per_mr = %u; mr_page_size = %u; max_sectors_per_mr = %u; mr_per_cmd = %u\n",
3347 target->scsi_host->max_sectors,
3348 srp_dev->max_pages_per_mr, srp_dev->mr_page_size,
3349 max_sectors_per_mr, mr_per_cmd);
3350 }
3351
3352 target_host->sg_tablesize = target->sg_tablesize;
3353 target->mr_pool_size = target->scsi_host->can_queue * mr_per_cmd;
3354 target->mr_per_cmd = mr_per_cmd;
3355 target->indirect_size = target->sg_tablesize *
3356 sizeof (struct srp_direct_buf);
3357 target->max_iu_len = sizeof (struct srp_cmd) +
3358 sizeof (struct srp_indirect_buf) +
3359 target->cmd_sg_cnt * sizeof (struct srp_direct_buf);
3360
3361 INIT_WORK(&target->tl_err_work, srp_tl_err_work);
3362 INIT_WORK(&target->remove_work, srp_remove_work);
3363 spin_lock_init(&target->lock);
3364 ret = ib_query_gid(ibdev, host->port, 0, &target->sgid, NULL);
3365 if (ret)
3366 goto out;
3367
3368 ret = -ENOMEM;
3369 target->ch_count = max_t(unsigned, num_online_nodes(),
3370 min(ch_count ? :
3371 min(4 * num_online_nodes(),
3372 ibdev->num_comp_vectors),
3373 num_online_cpus()));
3374 target->ch = kcalloc(target->ch_count, sizeof(*target->ch),
3375 GFP_KERNEL);
3376 if (!target->ch)
3377 goto out;
3378
3379 node_idx = 0;
3380 for_each_online_node(node) {
3381 const int ch_start = (node_idx * target->ch_count /
3382 num_online_nodes());
3383 const int ch_end = ((node_idx + 1) * target->ch_count /
3384 num_online_nodes());
3385 const int cv_start = (node_idx * ibdev->num_comp_vectors /
3386 num_online_nodes() + target->comp_vector)
3387 % ibdev->num_comp_vectors;
3388 const int cv_end = ((node_idx + 1) * ibdev->num_comp_vectors /
3389 num_online_nodes() + target->comp_vector)
3390 % ibdev->num_comp_vectors;
3391 int cpu_idx = 0;
3392
3393 for_each_online_cpu(cpu) {
3394 if (cpu_to_node(cpu) != node)
3395 continue;
3396 if (ch_start + cpu_idx >= ch_end)
3397 continue;
3398 ch = &target->ch[ch_start + cpu_idx];
3399 ch->target = target;
3400 ch->comp_vector = cv_start == cv_end ? cv_start :
3401 cv_start + cpu_idx % (cv_end - cv_start);
3402 spin_lock_init(&ch->lock);
3403 INIT_LIST_HEAD(&ch->free_tx);
3404 ret = srp_new_cm_id(ch);
3405 if (ret)
3406 goto err_disconnect;
3407
3408 ret = srp_create_ch_ib(ch);
3409 if (ret)
3410 goto err_disconnect;
3411
3412 ret = srp_alloc_req_data(ch);
3413 if (ret)
3414 goto err_disconnect;
3415
3416 ret = srp_connect_ch(ch, multich);
3417 if (ret) {
3418 shost_printk(KERN_ERR, target->scsi_host,
3419 PFX "Connection %d/%d failed\n",
3420 ch_start + cpu_idx,
3421 target->ch_count);
3422 if (node_idx == 0 && cpu_idx == 0) {
3423 goto err_disconnect;
3424 } else {
3425 srp_free_ch_ib(target, ch);
3426 srp_free_req_data(target, ch);
3427 target->ch_count = ch - target->ch;
3428 goto connected;
3429 }
3430 }
3431
3432 multich = true;
3433 cpu_idx++;
3434 }
3435 node_idx++;
3436 }
3437
3438 connected:
3439 target->scsi_host->nr_hw_queues = target->ch_count;
3440
3441 ret = srp_add_target(host, target);
3442 if (ret)
3443 goto err_disconnect;
3444
3445 if (target->state != SRP_TARGET_REMOVED) {
3446 shost_printk(KERN_DEBUG, target->scsi_host, PFX
3447 "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n",
3448 be64_to_cpu(target->id_ext),
3449 be64_to_cpu(target->ioc_guid),
3450 be16_to_cpu(target->pkey),
3451 be64_to_cpu(target->service_id),
3452 target->sgid.raw, target->orig_dgid.raw);
3453 }
3454
3455 ret = count;
3456
3457 out:
3458 mutex_unlock(&host->add_target_mutex);
3459
3460 put:
3461 scsi_host_put(target->scsi_host);
3462 if (ret < 0)
3463 scsi_host_put(target->scsi_host);
3464
3465 return ret;
3466
3467 err_disconnect:
3468 srp_disconnect_target(target);
3469
3470 for (i = 0; i < target->ch_count; i++) {
3471 ch = &target->ch[i];
3472 srp_free_ch_ib(target, ch);
3473 srp_free_req_data(target, ch);
3474 }
3475
3476 kfree(target->ch);
3477 goto out;
3478 }
3479
3480 static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target);
3481
3482 static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
3483 char *buf)
3484 {
3485 struct srp_host *host = container_of(dev, struct srp_host, dev);
3486
3487 return sprintf(buf, "%s\n", host->srp_dev->dev->name);
3488 }
3489
3490 static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
3491
3492 static ssize_t show_port(struct device *dev, struct device_attribute *attr,
3493 char *buf)
3494 {
3495 struct srp_host *host = container_of(dev, struct srp_host, dev);
3496
3497 return sprintf(buf, "%d\n", host->port);
3498 }
3499
3500 static DEVICE_ATTR(port, S_IRUGO, show_port, NULL);
3501
3502 static struct srp_host *srp_add_port(struct srp_device *device, u8 port)
3503 {
3504 struct srp_host *host;
3505
3506 host = kzalloc(sizeof *host, GFP_KERNEL);
3507 if (!host)
3508 return NULL;
3509
3510 INIT_LIST_HEAD(&host->target_list);
3511 spin_lock_init(&host->target_lock);
3512 init_completion(&host->released);
3513 mutex_init(&host->add_target_mutex);
3514 host->srp_dev = device;
3515 host->port = port;
3516
3517 host->dev.class = &srp_class;
3518 host->dev.parent = device->dev->dma_device;
3519 dev_set_name(&host->dev, "srp-%s-%d", device->dev->name, port);
3520
3521 if (device_register(&host->dev))
3522 goto free_host;
3523 if (device_create_file(&host->dev, &dev_attr_add_target))
3524 goto err_class;
3525 if (device_create_file(&host->dev, &dev_attr_ibdev))
3526 goto err_class;
3527 if (device_create_file(&host->dev, &dev_attr_port))
3528 goto err_class;
3529
3530 return host;
3531
3532 err_class:
3533 device_unregister(&host->dev);
3534
3535 free_host:
3536 kfree(host);
3537
3538 return NULL;
3539 }
3540
3541 static void srp_add_one(struct ib_device *device)
3542 {
3543 struct srp_device *srp_dev;
3544 struct ib_device_attr *attr = &device->attrs;
3545 struct srp_host *host;
3546 int mr_page_shift, p;
3547 u64 max_pages_per_mr;
3548 unsigned int flags = 0;
3549
3550 srp_dev = kzalloc(sizeof(*srp_dev), GFP_KERNEL);
3551 if (!srp_dev)
3552 return;
3553
3554 /*
3555 * Use the smallest page size supported by the HCA, down to a
3556 * minimum of 4096 bytes. We're unlikely to build large sglists
3557 * out of smaller entries.
3558 */
3559 mr_page_shift = max(12, ffs(attr->page_size_cap) - 1);
3560 srp_dev->mr_page_size = 1 << mr_page_shift;
3561 srp_dev->mr_page_mask = ~((u64) srp_dev->mr_page_size - 1);
3562 max_pages_per_mr = attr->max_mr_size;
3563 do_div(max_pages_per_mr, srp_dev->mr_page_size);
3564 pr_debug("%s: %llu / %u = %llu <> %u\n", __func__,
3565 attr->max_mr_size, srp_dev->mr_page_size,
3566 max_pages_per_mr, SRP_MAX_PAGES_PER_MR);
3567 srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
3568 max_pages_per_mr);
3569
3570 srp_dev->has_fmr = (device->alloc_fmr && device->dealloc_fmr &&
3571 device->map_phys_fmr && device->unmap_fmr);
3572 srp_dev->has_fr = (attr->device_cap_flags &
3573 IB_DEVICE_MEM_MGT_EXTENSIONS);
3574 if (!never_register && !srp_dev->has_fmr && !srp_dev->has_fr) {
3575 dev_warn(&device->dev, "neither FMR nor FR is supported\n");
3576 } else if (!never_register &&
3577 attr->max_mr_size >= 2 * srp_dev->mr_page_size) {
3578 srp_dev->use_fast_reg = (srp_dev->has_fr &&
3579 (!srp_dev->has_fmr || prefer_fr));
3580 srp_dev->use_fmr = !srp_dev->use_fast_reg && srp_dev->has_fmr;
3581 }
3582
3583 if (never_register || !register_always ||
3584 (!srp_dev->has_fmr && !srp_dev->has_fr))
3585 flags |= IB_PD_UNSAFE_GLOBAL_RKEY;
3586
3587 if (srp_dev->use_fast_reg) {
3588 srp_dev->max_pages_per_mr =
3589 min_t(u32, srp_dev->max_pages_per_mr,
3590 attr->max_fast_reg_page_list_len);
3591 }
3592 srp_dev->mr_max_size = srp_dev->mr_page_size *
3593 srp_dev->max_pages_per_mr;
3594 pr_debug("%s: mr_page_shift = %d, device->max_mr_size = %#llx, device->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
3595 device->name, mr_page_shift, attr->max_mr_size,
3596 attr->max_fast_reg_page_list_len,
3597 srp_dev->max_pages_per_mr, srp_dev->mr_max_size);
3598
3599 INIT_LIST_HEAD(&srp_dev->dev_list);
3600
3601 srp_dev->dev = device;
3602 srp_dev->pd = ib_alloc_pd(device, flags);
3603 if (IS_ERR(srp_dev->pd))
3604 goto free_dev;
3605
3606
3607 for (p = rdma_start_port(device); p <= rdma_end_port(device); ++p) {
3608 host = srp_add_port(srp_dev, p);
3609 if (host)
3610 list_add_tail(&host->list, &srp_dev->dev_list);
3611 }
3612
3613 ib_set_client_data(device, &srp_client, srp_dev);
3614 return;
3615
3616 free_dev:
3617 kfree(srp_dev);
3618 }
3619
3620 static void srp_remove_one(struct ib_device *device, void *client_data)
3621 {
3622 struct srp_device *srp_dev;
3623 struct srp_host *host, *tmp_host;
3624 struct srp_target_port *target;
3625
3626 srp_dev = client_data;
3627 if (!srp_dev)
3628 return;
3629
3630 list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
3631 device_unregister(&host->dev);
3632 /*
3633 * Wait for the sysfs entry to go away, so that no new
3634 * target ports can be created.
3635 */
3636 wait_for_completion(&host->released);
3637
3638 /*
3639 * Remove all target ports.
3640 */
3641 spin_lock(&host->target_lock);
3642 list_for_each_entry(target, &host->target_list, list)
3643 srp_queue_remove_work(target);
3644 spin_unlock(&host->target_lock);
3645
3646 /*
3647 * Wait for tl_err and target port removal tasks.
3648 */
3649 flush_workqueue(system_long_wq);
3650 flush_workqueue(srp_remove_wq);
3651
3652 kfree(host);
3653 }
3654
3655 ib_dealloc_pd(srp_dev->pd);
3656
3657 kfree(srp_dev);
3658 }
3659
3660 static struct srp_function_template ib_srp_transport_functions = {
3661 .has_rport_state = true,
3662 .reset_timer_if_blocked = true,
3663 .reconnect_delay = &srp_reconnect_delay,
3664 .fast_io_fail_tmo = &srp_fast_io_fail_tmo,
3665 .dev_loss_tmo = &srp_dev_loss_tmo,
3666 .reconnect = srp_rport_reconnect,
3667 .rport_delete = srp_rport_delete,
3668 .terminate_rport_io = srp_terminate_io,
3669 };
3670
3671 static int __init srp_init_module(void)
3672 {
3673 int ret;
3674
3675 if (srp_sg_tablesize) {
3676 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
3677 if (!cmd_sg_entries)
3678 cmd_sg_entries = srp_sg_tablesize;
3679 }
3680
3681 if (!cmd_sg_entries)
3682 cmd_sg_entries = SRP_DEF_SG_TABLESIZE;
3683
3684 if (cmd_sg_entries > 255) {
3685 pr_warn("Clamping cmd_sg_entries to 255\n");
3686 cmd_sg_entries = 255;
3687 }
3688
3689 if (!indirect_sg_entries)
3690 indirect_sg_entries = cmd_sg_entries;
3691 else if (indirect_sg_entries < cmd_sg_entries) {
3692 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n",
3693 cmd_sg_entries);
3694 indirect_sg_entries = cmd_sg_entries;
3695 }
3696
3697 srp_remove_wq = create_workqueue("srp_remove");
3698 if (!srp_remove_wq) {
3699 ret = -ENOMEM;
3700 goto out;
3701 }
3702
3703 ret = -ENOMEM;
3704 ib_srp_transport_template =
3705 srp_attach_transport(&ib_srp_transport_functions);
3706 if (!ib_srp_transport_template)
3707 goto destroy_wq;
3708
3709 ret = class_register(&srp_class);
3710 if (ret) {
3711 pr_err("couldn't register class infiniband_srp\n");
3712 goto release_tr;
3713 }
3714
3715 ib_sa_register_client(&srp_sa_client);
3716
3717 ret = ib_register_client(&srp_client);
3718 if (ret) {
3719 pr_err("couldn't register IB client\n");
3720 goto unreg_sa;
3721 }
3722
3723 out:
3724 return ret;
3725
3726 unreg_sa:
3727 ib_sa_unregister_client(&srp_sa_client);
3728 class_unregister(&srp_class);
3729
3730 release_tr:
3731 srp_release_transport(ib_srp_transport_template);
3732
3733 destroy_wq:
3734 destroy_workqueue(srp_remove_wq);
3735 goto out;
3736 }
3737
3738 static void __exit srp_cleanup_module(void)
3739 {
3740 ib_unregister_client(&srp_client);
3741 ib_sa_unregister_client(&srp_sa_client);
3742 class_unregister(&srp_class);
3743 srp_release_transport(ib_srp_transport_template);
3744 destroy_workqueue(srp_remove_wq);
3745 }
3746
3747 module_init(srp_init_module);
3748 module_exit(srp_cleanup_module);
Ubuntu Artful kernel mirror