]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/infiniband/ulp/srpt/ib_srpt.c
IB/srpt: Eliminate srpt_find_channel()
[mirror_ubuntu-artful-kernel.git] / drivers / infiniband / ulp / srpt / ib_srpt.c
CommitLineData
a42d985b
BVA
1/*
2 * Copyright (c) 2006 - 2009 Mellanox Technology Inc. All rights reserved.
3 * Copyright (C) 2008 - 2011 Bart Van Assche <bvanassche@acm.org>.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 *
33 */
34
35#include <linux/module.h>
36#include <linux/init.h>
37#include <linux/slab.h>
38#include <linux/err.h>
39#include <linux/ctype.h>
40#include <linux/kthread.h>
41#include <linux/string.h>
42#include <linux/delay.h>
43#include <linux/atomic.h>
ba929992 44#include <scsi/scsi_proto.h>
a42d985b 45#include <scsi/scsi_tcq.h>
a42d985b 46#include <target/target_core_base.h>
a42d985b 47#include <target/target_core_fabric.h>
a42d985b
BVA
48#include "ib_srpt.h"
49
50/* Name of this kernel module. */
51#define DRV_NAME "ib_srpt"
52#define DRV_VERSION "2.0.0"
53#define DRV_RELDATE "2011-02-14"
54
55#define SRPT_ID_STRING "Linux SRP target"
56
57#undef pr_fmt
58#define pr_fmt(fmt) DRV_NAME " " fmt
59
60MODULE_AUTHOR("Vu Pham and Bart Van Assche");
61MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol target "
62 "v" DRV_VERSION " (" DRV_RELDATE ")");
63MODULE_LICENSE("Dual BSD/GPL");
64
65/*
66 * Global Variables
67 */
68
69static u64 srpt_service_guid;
486d8b9f
RD
70static DEFINE_SPINLOCK(srpt_dev_lock); /* Protects srpt_dev_list. */
71static LIST_HEAD(srpt_dev_list); /* List of srpt_device structures. */
a42d985b
BVA
72
73static unsigned srp_max_req_size = DEFAULT_MAX_REQ_SIZE;
74module_param(srp_max_req_size, int, 0444);
75MODULE_PARM_DESC(srp_max_req_size,
76 "Maximum size of SRP request messages in bytes.");
77
78static int srpt_srq_size = DEFAULT_SRPT_SRQ_SIZE;
79module_param(srpt_srq_size, int, 0444);
80MODULE_PARM_DESC(srpt_srq_size,
81 "Shared receive queue (SRQ) size.");
82
83static int srpt_get_u64_x(char *buffer, struct kernel_param *kp)
84{
85 return sprintf(buffer, "0x%016llx", *(u64 *)kp->arg);
86}
87module_param_call(srpt_service_guid, NULL, srpt_get_u64_x, &srpt_service_guid,
88 0444);
89MODULE_PARM_DESC(srpt_service_guid,
90 "Using this value for ioc_guid, id_ext, and cm_listen_id"
91 " instead of using the node_guid of the first HCA.");
92
93static struct ib_client srpt_client;
2c7f37ff 94static void srpt_release_cmd(struct se_cmd *se_cmd);
a42d985b
BVA
95static void srpt_release_channel(struct srpt_rdma_ch *ch);
96static int srpt_queue_status(struct se_cmd *cmd);
59fae4de
CH
97static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc);
98static void srpt_send_done(struct ib_cq *cq, struct ib_wc *wc);
a42d985b 99
f130c220
BVA
100/*
101 * The only allowed channel state changes are those that change the channel
102 * state into a state with a higher numerical value. Hence the new > prev test.
a42d985b 103 */
f130c220 104static bool srpt_set_ch_state(struct srpt_rdma_ch *ch, enum rdma_ch_state new)
a42d985b
BVA
105{
106 unsigned long flags;
107 enum rdma_ch_state prev;
f130c220 108 bool changed = false;
a42d985b
BVA
109
110 spin_lock_irqsave(&ch->spinlock, flags);
111 prev = ch->state;
f130c220 112 if (new > prev) {
a42d985b 113 ch->state = new;
f130c220
BVA
114 changed = true;
115 }
a42d985b 116 spin_unlock_irqrestore(&ch->spinlock, flags);
f130c220
BVA
117
118 return changed;
a42d985b
BVA
119}
120
121/**
122 * srpt_event_handler() - Asynchronous IB event callback function.
123 *
124 * Callback function called by the InfiniBand core when an asynchronous IB
125 * event occurs. This callback may occur in interrupt context. See also
126 * section 11.5.2, Set Asynchronous Event Handler in the InfiniBand
127 * Architecture Specification.
128 */
129static void srpt_event_handler(struct ib_event_handler *handler,
130 struct ib_event *event)
131{
132 struct srpt_device *sdev;
133 struct srpt_port *sport;
134
135 sdev = ib_get_client_data(event->device, &srpt_client);
136 if (!sdev || sdev->device != event->device)
137 return;
138
139 pr_debug("ASYNC event= %d on device= %s\n", event->event,
f68cba4e 140 sdev->device->name);
a42d985b
BVA
141
142 switch (event->event) {
143 case IB_EVENT_PORT_ERR:
144 if (event->element.port_num <= sdev->device->phys_port_cnt) {
145 sport = &sdev->port[event->element.port_num - 1];
146 sport->lid = 0;
147 sport->sm_lid = 0;
148 }
149 break;
150 case IB_EVENT_PORT_ACTIVE:
151 case IB_EVENT_LID_CHANGE:
152 case IB_EVENT_PKEY_CHANGE:
153 case IB_EVENT_SM_CHANGE:
154 case IB_EVENT_CLIENT_REREGISTER:
2aa1cf64 155 case IB_EVENT_GID_CHANGE:
a42d985b
BVA
156 /* Refresh port data asynchronously. */
157 if (event->element.port_num <= sdev->device->phys_port_cnt) {
158 sport = &sdev->port[event->element.port_num - 1];
159 if (!sport->lid && !sport->sm_lid)
160 schedule_work(&sport->work);
161 }
162 break;
163 default:
9f5d32af 164 pr_err("received unrecognized IB event %d\n",
a42d985b
BVA
165 event->event);
166 break;
167 }
168}
169
170/**
171 * srpt_srq_event() - SRQ event callback function.
172 */
173static void srpt_srq_event(struct ib_event *event, void *ctx)
174{
9f5d32af 175 pr_info("SRQ event %d\n", event->event);
a42d985b
BVA
176}
177
178/**
179 * srpt_qp_event() - QP event callback function.
180 */
181static void srpt_qp_event(struct ib_event *event, struct srpt_rdma_ch *ch)
182{
183 pr_debug("QP event %d on cm_id=%p sess_name=%s state=%d\n",
33912d73 184 event->event, ch->cm_id, ch->sess_name, ch->state);
a42d985b
BVA
185
186 switch (event->event) {
187 case IB_EVENT_COMM_EST:
188 ib_cm_notify(ch->cm_id, event->event);
189 break;
190 case IB_EVENT_QP_LAST_WQE_REACHED:
f130c220 191 if (srpt_set_ch_state(ch, CH_RELEASING))
a42d985b
BVA
192 srpt_release_channel(ch);
193 else
194 pr_debug("%s: state %d - ignored LAST_WQE.\n",
33912d73 195 ch->sess_name, ch->state);
a42d985b
BVA
196 break;
197 default:
9f5d32af 198 pr_err("received unrecognized IB QP event %d\n", event->event);
a42d985b
BVA
199 break;
200 }
201}
202
203/**
204 * srpt_set_ioc() - Helper function for initializing an IOUnitInfo structure.
205 *
206 * @slot: one-based slot number.
207 * @value: four-bit value.
208 *
209 * Copies the lowest four bits of value in element slot of the array of four
210 * bit elements called c_list (controller list). The index slot is one-based.
211 */
212static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value)
213{
214 u16 id;
215 u8 tmp;
216
217 id = (slot - 1) / 2;
218 if (slot & 0x1) {
219 tmp = c_list[id] & 0xf;
220 c_list[id] = (value << 4) | tmp;
221 } else {
222 tmp = c_list[id] & 0xf0;
223 c_list[id] = (value & 0xf) | tmp;
224 }
225}
226
227/**
228 * srpt_get_class_port_info() - Copy ClassPortInfo to a management datagram.
229 *
230 * See also section 16.3.3.1 ClassPortInfo in the InfiniBand Architecture
231 * Specification.
232 */
233static void srpt_get_class_port_info(struct ib_dm_mad *mad)
234{
235 struct ib_class_port_info *cif;
236
237 cif = (struct ib_class_port_info *)mad->data;
9d2aa2b4 238 memset(cif, 0, sizeof(*cif));
a42d985b
BVA
239 cif->base_version = 1;
240 cif->class_version = 1;
241 cif->resp_time_value = 20;
242
243 mad->mad_hdr.status = 0;
244}
245
246/**
247 * srpt_get_iou() - Write IOUnitInfo to a management datagram.
248 *
249 * See also section 16.3.3.3 IOUnitInfo in the InfiniBand Architecture
250 * Specification. See also section B.7, table B.6 in the SRP r16a document.
251 */
252static void srpt_get_iou(struct ib_dm_mad *mad)
253{
254 struct ib_dm_iou_info *ioui;
255 u8 slot;
256 int i;
257
258 ioui = (struct ib_dm_iou_info *)mad->data;
b356c1c1 259 ioui->change_id = cpu_to_be16(1);
a42d985b
BVA
260 ioui->max_controllers = 16;
261
262 /* set present for slot 1 and empty for the rest */
263 srpt_set_ioc(ioui->controller_list, 1, 1);
264 for (i = 1, slot = 2; i < 16; i++, slot++)
265 srpt_set_ioc(ioui->controller_list, slot, 0);
266
267 mad->mad_hdr.status = 0;
268}
269
270/**
271 * srpt_get_ioc() - Write IOControllerprofile to a management datagram.
272 *
273 * See also section 16.3.3.4 IOControllerProfile in the InfiniBand
274 * Architecture Specification. See also section B.7, table B.7 in the SRP
275 * r16a document.
276 */
277static void srpt_get_ioc(struct srpt_port *sport, u32 slot,
278 struct ib_dm_mad *mad)
279{
280 struct srpt_device *sdev = sport->sdev;
281 struct ib_dm_ioc_profile *iocp;
282
283 iocp = (struct ib_dm_ioc_profile *)mad->data;
284
285 if (!slot || slot > 16) {
286 mad->mad_hdr.status
b356c1c1 287 = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
a42d985b
BVA
288 return;
289 }
290
291 if (slot > 2) {
292 mad->mad_hdr.status
b356c1c1 293 = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
a42d985b
BVA
294 return;
295 }
296
9d2aa2b4 297 memset(iocp, 0, sizeof(*iocp));
a42d985b
BVA
298 strcpy(iocp->id_string, SRPT_ID_STRING);
299 iocp->guid = cpu_to_be64(srpt_service_guid);
4a061b28
OG
300 iocp->vendor_id = cpu_to_be32(sdev->device->attrs.vendor_id);
301 iocp->device_id = cpu_to_be32(sdev->device->attrs.vendor_part_id);
302 iocp->device_version = cpu_to_be16(sdev->device->attrs.hw_ver);
303 iocp->subsys_vendor_id = cpu_to_be32(sdev->device->attrs.vendor_id);
a42d985b 304 iocp->subsys_device_id = 0x0;
b356c1c1
VT
305 iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS);
306 iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS);
307 iocp->protocol = cpu_to_be16(SRP_PROTOCOL);
308 iocp->protocol_version = cpu_to_be16(SRP_PROTOCOL_VERSION);
a42d985b
BVA
309 iocp->send_queue_depth = cpu_to_be16(sdev->srq_size);
310 iocp->rdma_read_depth = 4;
311 iocp->send_size = cpu_to_be32(srp_max_req_size);
312 iocp->rdma_size = cpu_to_be32(min(sport->port_attrib.srp_max_rdma_size,
313 1U << 24));
314 iocp->num_svc_entries = 1;
315 iocp->op_cap_mask = SRP_SEND_TO_IOC | SRP_SEND_FROM_IOC |
316 SRP_RDMA_READ_FROM_IOC | SRP_RDMA_WRITE_FROM_IOC;
317
318 mad->mad_hdr.status = 0;
319}
320
321/**
322 * srpt_get_svc_entries() - Write ServiceEntries to a management datagram.
323 *
324 * See also section 16.3.3.5 ServiceEntries in the InfiniBand Architecture
325 * Specification. See also section B.7, table B.8 in the SRP r16a document.
326 */
327static void srpt_get_svc_entries(u64 ioc_guid,
328 u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad)
329{
330 struct ib_dm_svc_entries *svc_entries;
331
332 WARN_ON(!ioc_guid);
333
334 if (!slot || slot > 16) {
335 mad->mad_hdr.status
b356c1c1 336 = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
a42d985b
BVA
337 return;
338 }
339
340 if (slot > 2 || lo > hi || hi > 1) {
341 mad->mad_hdr.status
b356c1c1 342 = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
a42d985b
BVA
343 return;
344 }
345
346 svc_entries = (struct ib_dm_svc_entries *)mad->data;
9d2aa2b4 347 memset(svc_entries, 0, sizeof(*svc_entries));
a42d985b
BVA
348 svc_entries->service_entries[0].id = cpu_to_be64(ioc_guid);
349 snprintf(svc_entries->service_entries[0].name,
350 sizeof(svc_entries->service_entries[0].name),
351 "%s%016llx",
352 SRP_SERVICE_NAME_PREFIX,
353 ioc_guid);
354
355 mad->mad_hdr.status = 0;
356}
357
358/**
359 * srpt_mgmt_method_get() - Process a received management datagram.
360 * @sp: source port through which the MAD has been received.
361 * @rq_mad: received MAD.
362 * @rsp_mad: response MAD.
363 */
364static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
365 struct ib_dm_mad *rsp_mad)
366{
367 u16 attr_id;
368 u32 slot;
369 u8 hi, lo;
370
371 attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id);
372 switch (attr_id) {
373 case DM_ATTR_CLASS_PORT_INFO:
374 srpt_get_class_port_info(rsp_mad);
375 break;
376 case DM_ATTR_IOU_INFO:
377 srpt_get_iou(rsp_mad);
378 break;
379 case DM_ATTR_IOC_PROFILE:
380 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
381 srpt_get_ioc(sp, slot, rsp_mad);
382 break;
383 case DM_ATTR_SVC_ENTRIES:
384 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
385 hi = (u8) ((slot >> 8) & 0xff);
386 lo = (u8) (slot & 0xff);
387 slot = (u16) ((slot >> 16) & 0xffff);
388 srpt_get_svc_entries(srpt_service_guid,
389 slot, hi, lo, rsp_mad);
390 break;
391 default:
392 rsp_mad->mad_hdr.status =
b356c1c1 393 cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
a42d985b
BVA
394 break;
395 }
396}
397
398/**
399 * srpt_mad_send_handler() - Post MAD-send callback function.
400 */
401static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
402 struct ib_mad_send_wc *mad_wc)
403{
404 ib_destroy_ah(mad_wc->send_buf->ah);
405 ib_free_send_mad(mad_wc->send_buf);
406}
407
408/**
409 * srpt_mad_recv_handler() - MAD reception callback function.
410 */
411static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
ca281265 412 struct ib_mad_send_buf *send_buf,
a42d985b
BVA
413 struct ib_mad_recv_wc *mad_wc)
414{
415 struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
416 struct ib_ah *ah;
417 struct ib_mad_send_buf *rsp;
418 struct ib_dm_mad *dm_mad;
419
420 if (!mad_wc || !mad_wc->recv_buf.mad)
421 return;
422
423 ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
424 mad_wc->recv_buf.grh, mad_agent->port_num);
425 if (IS_ERR(ah))
426 goto err;
427
428 BUILD_BUG_ON(offsetof(struct ib_dm_mad, data) != IB_MGMT_DEVICE_HDR);
429
430 rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
431 mad_wc->wc->pkey_index, 0,
432 IB_MGMT_DEVICE_HDR, IB_MGMT_DEVICE_DATA,
da2dfaa3
IW
433 GFP_KERNEL,
434 IB_MGMT_BASE_VERSION);
a42d985b
BVA
435 if (IS_ERR(rsp))
436 goto err_rsp;
437
438 rsp->ah = ah;
439
440 dm_mad = rsp->mad;
9d2aa2b4 441 memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof(*dm_mad));
a42d985b
BVA
442 dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
443 dm_mad->mad_hdr.status = 0;
444
445 switch (mad_wc->recv_buf.mad->mad_hdr.method) {
446 case IB_MGMT_METHOD_GET:
447 srpt_mgmt_method_get(sport, mad_wc->recv_buf.mad, dm_mad);
448 break;
449 case IB_MGMT_METHOD_SET:
450 dm_mad->mad_hdr.status =
b356c1c1 451 cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
a42d985b
BVA
452 break;
453 default:
454 dm_mad->mad_hdr.status =
b356c1c1 455 cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD);
a42d985b
BVA
456 break;
457 }
458
459 if (!ib_post_send_mad(rsp, NULL)) {
460 ib_free_recv_mad(mad_wc);
461 /* will destroy_ah & free_send_mad in send completion */
462 return;
463 }
464
465 ib_free_send_mad(rsp);
466
467err_rsp:
468 ib_destroy_ah(ah);
469err:
470 ib_free_recv_mad(mad_wc);
471}
472
473/**
474 * srpt_refresh_port() - Configure a HCA port.
475 *
476 * Enable InfiniBand management datagram processing, update the cached sm_lid,
477 * lid and gid values, and register a callback function for processing MADs
478 * on the specified port.
479 *
480 * Note: It is safe to call this function more than once for the same port.
481 */
482static int srpt_refresh_port(struct srpt_port *sport)
483{
484 struct ib_mad_reg_req reg_req;
485 struct ib_port_modify port_modify;
486 struct ib_port_attr port_attr;
487 int ret;
488
9d2aa2b4 489 memset(&port_modify, 0, sizeof(port_modify));
a42d985b
BVA
490 port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
491 port_modify.clr_port_cap_mask = 0;
492
493 ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
494 if (ret)
495 goto err_mod_port;
496
497 ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);
498 if (ret)
499 goto err_query_port;
500
501 sport->sm_lid = port_attr.sm_lid;
502 sport->lid = port_attr.lid;
503
55ee3ab2
MB
504 ret = ib_query_gid(sport->sdev->device, sport->port, 0, &sport->gid,
505 NULL);
a42d985b
BVA
506 if (ret)
507 goto err_query_port;
508
509 if (!sport->mad_agent) {
9d2aa2b4 510 memset(&reg_req, 0, sizeof(reg_req));
a42d985b
BVA
511 reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
512 reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION;
513 set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
514 set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
515
516 sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
517 sport->port,
518 IB_QPT_GSI,
519 &reg_req, 0,
520 srpt_mad_send_handler,
521 srpt_mad_recv_handler,
0f29b46d 522 sport, 0);
a42d985b
BVA
523 if (IS_ERR(sport->mad_agent)) {
524 ret = PTR_ERR(sport->mad_agent);
525 sport->mad_agent = NULL;
526 goto err_query_port;
527 }
528 }
529
530 return 0;
531
532err_query_port:
533
534 port_modify.set_port_cap_mask = 0;
535 port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
536 ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
537
538err_mod_port:
539
540 return ret;
541}
542
543/**
544 * srpt_unregister_mad_agent() - Unregister MAD callback functions.
545 *
546 * Note: It is safe to call this function more than once for the same device.
547 */
548static void srpt_unregister_mad_agent(struct srpt_device *sdev)
549{
550 struct ib_port_modify port_modify = {
551 .clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP,
552 };
553 struct srpt_port *sport;
554 int i;
555
556 for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
557 sport = &sdev->port[i - 1];
558 WARN_ON(sport->port != i);
559 if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)
9f5d32af 560 pr_err("disabling MAD processing failed.\n");
a42d985b
BVA
561 if (sport->mad_agent) {
562 ib_unregister_mad_agent(sport->mad_agent);
563 sport->mad_agent = NULL;
564 }
565 }
566}
567
568/**
569 * srpt_alloc_ioctx() - Allocate an SRPT I/O context structure.
570 */
571static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev,
572 int ioctx_size, int dma_size,
573 enum dma_data_direction dir)
574{
575 struct srpt_ioctx *ioctx;
576
577 ioctx = kmalloc(ioctx_size, GFP_KERNEL);
578 if (!ioctx)
579 goto err;
580
581 ioctx->buf = kmalloc(dma_size, GFP_KERNEL);
582 if (!ioctx->buf)
583 goto err_free_ioctx;
584
585 ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf, dma_size, dir);
586 if (ib_dma_mapping_error(sdev->device, ioctx->dma))
587 goto err_free_buf;
588
589 return ioctx;
590
591err_free_buf:
592 kfree(ioctx->buf);
593err_free_ioctx:
594 kfree(ioctx);
595err:
596 return NULL;
597}
598
599/**
600 * srpt_free_ioctx() - Free an SRPT I/O context structure.
601 */
602static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx,
603 int dma_size, enum dma_data_direction dir)
604{
605 if (!ioctx)
606 return;
607
608 ib_dma_unmap_single(sdev->device, ioctx->dma, dma_size, dir);
609 kfree(ioctx->buf);
610 kfree(ioctx);
611}
612
613/**
614 * srpt_alloc_ioctx_ring() - Allocate a ring of SRPT I/O context structures.
615 * @sdev: Device to allocate the I/O context ring for.
616 * @ring_size: Number of elements in the I/O context ring.
617 * @ioctx_size: I/O context size.
618 * @dma_size: DMA buffer size.
619 * @dir: DMA data direction.
620 */
621static struct srpt_ioctx **srpt_alloc_ioctx_ring(struct srpt_device *sdev,
622 int ring_size, int ioctx_size,
623 int dma_size, enum dma_data_direction dir)
624{
625 struct srpt_ioctx **ring;
626 int i;
627
628 WARN_ON(ioctx_size != sizeof(struct srpt_recv_ioctx)
629 && ioctx_size != sizeof(struct srpt_send_ioctx));
630
631 ring = kmalloc(ring_size * sizeof(ring[0]), GFP_KERNEL);
632 if (!ring)
633 goto out;
634 for (i = 0; i < ring_size; ++i) {
635 ring[i] = srpt_alloc_ioctx(sdev, ioctx_size, dma_size, dir);
636 if (!ring[i])
637 goto err;
638 ring[i]->index = i;
639 }
640 goto out;
641
642err:
643 while (--i >= 0)
644 srpt_free_ioctx(sdev, ring[i], dma_size, dir);
645 kfree(ring);
715252d4 646 ring = NULL;
a42d985b
BVA
647out:
648 return ring;
649}
650
651/**
652 * srpt_free_ioctx_ring() - Free the ring of SRPT I/O context structures.
653 */
654static void srpt_free_ioctx_ring(struct srpt_ioctx **ioctx_ring,
655 struct srpt_device *sdev, int ring_size,
656 int dma_size, enum dma_data_direction dir)
657{
658 int i;
659
660 for (i = 0; i < ring_size; ++i)
661 srpt_free_ioctx(sdev, ioctx_ring[i], dma_size, dir);
662 kfree(ioctx_ring);
663}
664
665/**
666 * srpt_get_cmd_state() - Get the state of a SCSI command.
667 */
668static enum srpt_command_state srpt_get_cmd_state(struct srpt_send_ioctx *ioctx)
669{
670 enum srpt_command_state state;
671 unsigned long flags;
672
673 BUG_ON(!ioctx);
674
675 spin_lock_irqsave(&ioctx->spinlock, flags);
676 state = ioctx->state;
677 spin_unlock_irqrestore(&ioctx->spinlock, flags);
678 return state;
679}
680
681/**
682 * srpt_set_cmd_state() - Set the state of a SCSI command.
683 *
684 * Does not modify the state of aborted commands. Returns the previous command
685 * state.
686 */
687static enum srpt_command_state srpt_set_cmd_state(struct srpt_send_ioctx *ioctx,
688 enum srpt_command_state new)
689{
690 enum srpt_command_state previous;
691 unsigned long flags;
692
693 BUG_ON(!ioctx);
694
695 spin_lock_irqsave(&ioctx->spinlock, flags);
696 previous = ioctx->state;
697 if (previous != SRPT_STATE_DONE)
698 ioctx->state = new;
699 spin_unlock_irqrestore(&ioctx->spinlock, flags);
700
701 return previous;
702}
703
704/**
705 * srpt_test_and_set_cmd_state() - Test and set the state of a command.
706 *
707 * Returns true if and only if the previous command state was equal to 'old'.
708 */
709static bool srpt_test_and_set_cmd_state(struct srpt_send_ioctx *ioctx,
710 enum srpt_command_state old,
711 enum srpt_command_state new)
712{
713 enum srpt_command_state previous;
714 unsigned long flags;
715
716 WARN_ON(!ioctx);
717 WARN_ON(old == SRPT_STATE_DONE);
718 WARN_ON(new == SRPT_STATE_NEW);
719
720 spin_lock_irqsave(&ioctx->spinlock, flags);
721 previous = ioctx->state;
722 if (previous == old)
723 ioctx->state = new;
724 spin_unlock_irqrestore(&ioctx->spinlock, flags);
725 return previous == old;
726}
727
728/**
729 * srpt_post_recv() - Post an IB receive request.
730 */
731static int srpt_post_recv(struct srpt_device *sdev,
732 struct srpt_recv_ioctx *ioctx)
733{
734 struct ib_sge list;
735 struct ib_recv_wr wr, *bad_wr;
736
737 BUG_ON(!sdev);
a42d985b
BVA
738 list.addr = ioctx->ioctx.dma;
739 list.length = srp_max_req_size;
5a783956 740 list.lkey = sdev->pd->local_dma_lkey;
a42d985b 741
59fae4de
CH
742 ioctx->ioctx.cqe.done = srpt_recv_done;
743 wr.wr_cqe = &ioctx->ioctx.cqe;
a42d985b
BVA
744 wr.next = NULL;
745 wr.sg_list = &list;
746 wr.num_sge = 1;
747
748 return ib_post_srq_recv(sdev->srq, &wr, &bad_wr);
749}
750
751/**
752 * srpt_post_send() - Post an IB send request.
753 *
754 * Returns zero upon success and a non-zero value upon failure.
755 */
756static int srpt_post_send(struct srpt_rdma_ch *ch,
757 struct srpt_send_ioctx *ioctx, int len)
758{
759 struct ib_sge list;
760 struct ib_send_wr wr, *bad_wr;
761 struct srpt_device *sdev = ch->sport->sdev;
762 int ret;
763
764 atomic_inc(&ch->req_lim);
765
766 ret = -ENOMEM;
767 if (unlikely(atomic_dec_return(&ch->sq_wr_avail) < 0)) {
9f5d32af 768 pr_warn("IB send queue full (needed 1)\n");
a42d985b
BVA
769 goto out;
770 }
771
772 ib_dma_sync_single_for_device(sdev->device, ioctx->ioctx.dma, len,
773 DMA_TO_DEVICE);
774
775 list.addr = ioctx->ioctx.dma;
776 list.length = len;
5a783956 777 list.lkey = sdev->pd->local_dma_lkey;
a42d985b 778
59fae4de 779 ioctx->ioctx.cqe.done = srpt_send_done;
a42d985b 780 wr.next = NULL;
59fae4de 781 wr.wr_cqe = &ioctx->ioctx.cqe;
a42d985b
BVA
782 wr.sg_list = &list;
783 wr.num_sge = 1;
784 wr.opcode = IB_WR_SEND;
785 wr.send_flags = IB_SEND_SIGNALED;
786
787 ret = ib_post_send(ch->qp, &wr, &bad_wr);
788
789out:
790 if (ret < 0) {
791 atomic_inc(&ch->sq_wr_avail);
792 atomic_dec(&ch->req_lim);
793 }
794 return ret;
795}
796
797/**
798 * srpt_get_desc_tbl() - Parse the data descriptors of an SRP_CMD request.
799 * @ioctx: Pointer to the I/O context associated with the request.
800 * @srp_cmd: Pointer to the SRP_CMD request data.
801 * @dir: Pointer to the variable to which the transfer direction will be
802 * written.
803 * @data_len: Pointer to the variable to which the total data length of all
804 * descriptors in the SRP_CMD request will be written.
805 *
806 * This function initializes ioctx->nrbuf and ioctx->r_bufs.
807 *
808 * Returns -EINVAL when the SRP_CMD request contains inconsistent descriptors;
809 * -ENOMEM when memory allocation fails and zero upon success.
810 */
811static int srpt_get_desc_tbl(struct srpt_send_ioctx *ioctx,
812 struct srp_cmd *srp_cmd,
813 enum dma_data_direction *dir, u64 *data_len)
814{
815 struct srp_indirect_buf *idb;
816 struct srp_direct_buf *db;
817 unsigned add_cdb_offset;
818 int ret;
819
820 /*
821 * The pointer computations below will only be compiled correctly
822 * if srp_cmd::add_data is declared as s8*, u8*, s8[] or u8[], so check
823 * whether srp_cmd::add_data has been declared as a byte pointer.
824 */
825 BUILD_BUG_ON(!__same_type(srp_cmd->add_data[0], (s8)0)
826 && !__same_type(srp_cmd->add_data[0], (u8)0));
827
828 BUG_ON(!dir);
829 BUG_ON(!data_len);
830
831 ret = 0;
832 *data_len = 0;
833
834 /*
835 * The lower four bits of the buffer format field contain the DATA-IN
836 * buffer descriptor format, and the highest four bits contain the
837 * DATA-OUT buffer descriptor format.
838 */
839 *dir = DMA_NONE;
840 if (srp_cmd->buf_fmt & 0xf)
841 /* DATA-IN: transfer data from target to initiator (read). */
842 *dir = DMA_FROM_DEVICE;
843 else if (srp_cmd->buf_fmt >> 4)
844 /* DATA-OUT: transfer data from initiator to target (write). */
845 *dir = DMA_TO_DEVICE;
846
847 /*
848 * According to the SRP spec, the lower two bits of the 'ADDITIONAL
849 * CDB LENGTH' field are reserved and the size in bytes of this field
850 * is four times the value specified in bits 3..7. Hence the "& ~3".
851 */
852 add_cdb_offset = srp_cmd->add_cdb_len & ~3;
853 if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) ||
854 ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
855 ioctx->n_rbuf = 1;
856 ioctx->rbufs = &ioctx->single_rbuf;
857
858 db = (struct srp_direct_buf *)(srp_cmd->add_data
859 + add_cdb_offset);
9d2aa2b4 860 memcpy(ioctx->rbufs, db, sizeof(*db));
a42d985b
BVA
861 *data_len = be32_to_cpu(db->len);
862 } else if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_INDIRECT) ||
863 ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_INDIRECT)) {
864 idb = (struct srp_indirect_buf *)(srp_cmd->add_data
865 + add_cdb_offset);
866
9d2aa2b4 867 ioctx->n_rbuf = be32_to_cpu(idb->table_desc.len) / sizeof(*db);
a42d985b
BVA
868
869 if (ioctx->n_rbuf >
870 (srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
9f5d32af 871 pr_err("received unsupported SRP_CMD request"
a42d985b
BVA
872 " type (%u out + %u in != %u / %zu)\n",
873 srp_cmd->data_out_desc_cnt,
874 srp_cmd->data_in_desc_cnt,
875 be32_to_cpu(idb->table_desc.len),
876 sizeof(*db));
877 ioctx->n_rbuf = 0;
878 ret = -EINVAL;
879 goto out;
880 }
881
882 if (ioctx->n_rbuf == 1)
883 ioctx->rbufs = &ioctx->single_rbuf;
884 else {
885 ioctx->rbufs =
9d2aa2b4 886 kmalloc(ioctx->n_rbuf * sizeof(*db), GFP_ATOMIC);
a42d985b
BVA
887 if (!ioctx->rbufs) {
888 ioctx->n_rbuf = 0;
889 ret = -ENOMEM;
890 goto out;
891 }
892 }
893
894 db = idb->desc_list;
9d2aa2b4 895 memcpy(ioctx->rbufs, db, ioctx->n_rbuf * sizeof(*db));
a42d985b
BVA
896 *data_len = be32_to_cpu(idb->len);
897 }
898out:
899 return ret;
900}
901
902/**
903 * srpt_init_ch_qp() - Initialize queue pair attributes.
904 *
905 * Initialized the attributes of queue pair 'qp' by allowing local write,
906 * remote read and remote write. Also transitions 'qp' to state IB_QPS_INIT.
907 */
908static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
909{
910 struct ib_qp_attr *attr;
911 int ret;
912
9d2aa2b4 913 attr = kzalloc(sizeof(*attr), GFP_KERNEL);
a42d985b
BVA
914 if (!attr)
915 return -ENOMEM;
916
917 attr->qp_state = IB_QPS_INIT;
918 attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ |
919 IB_ACCESS_REMOTE_WRITE;
920 attr->port_num = ch->sport->port;
921 attr->pkey_index = 0;
922
923 ret = ib_modify_qp(qp, attr,
924 IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT |
925 IB_QP_PKEY_INDEX);
926
927 kfree(attr);
928 return ret;
929}
930
931/**
932 * srpt_ch_qp_rtr() - Change the state of a channel to 'ready to receive' (RTR).
933 * @ch: channel of the queue pair.
934 * @qp: queue pair to change the state of.
935 *
936 * Returns zero upon success and a negative value upon failure.
937 *
938 * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
939 * If this structure ever becomes larger, it might be necessary to allocate
940 * it dynamically instead of on the stack.
941 */
942static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp)
943{
944 struct ib_qp_attr qp_attr;
945 int attr_mask;
946 int ret;
947
948 qp_attr.qp_state = IB_QPS_RTR;
949 ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
950 if (ret)
951 goto out;
952
953 qp_attr.max_dest_rd_atomic = 4;
954
955 ret = ib_modify_qp(qp, &qp_attr, attr_mask);
956
957out:
958 return ret;
959}
960
961/**
962 * srpt_ch_qp_rts() - Change the state of a channel to 'ready to send' (RTS).
963 * @ch: channel of the queue pair.
964 * @qp: queue pair to change the state of.
965 *
966 * Returns zero upon success and a negative value upon failure.
967 *
968 * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
969 * If this structure ever becomes larger, it might be necessary to allocate
970 * it dynamically instead of on the stack.
971 */
972static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp)
973{
974 struct ib_qp_attr qp_attr;
975 int attr_mask;
976 int ret;
977
978 qp_attr.qp_state = IB_QPS_RTS;
979 ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
980 if (ret)
981 goto out;
982
983 qp_attr.max_rd_atomic = 4;
984
985 ret = ib_modify_qp(qp, &qp_attr, attr_mask);
986
987out:
988 return ret;
989}
990
991/**
992 * srpt_ch_qp_err() - Set the channel queue pair state to 'error'.
993 */
994static int srpt_ch_qp_err(struct srpt_rdma_ch *ch)
995{
996 struct ib_qp_attr qp_attr;
997
998 qp_attr.qp_state = IB_QPS_ERR;
999 return ib_modify_qp(ch->qp, &qp_attr, IB_QP_STATE);
1000}
1001
1002/**
1003 * srpt_unmap_sg_to_ib_sge() - Unmap an IB SGE list.
1004 */
1005static void srpt_unmap_sg_to_ib_sge(struct srpt_rdma_ch *ch,
1006 struct srpt_send_ioctx *ioctx)
1007{
1008 struct scatterlist *sg;
1009 enum dma_data_direction dir;
1010
1011 BUG_ON(!ch);
1012 BUG_ON(!ioctx);
59fae4de 1013 BUG_ON(ioctx->n_rdma && !ioctx->rdma_wrs);
a42d985b
BVA
1014
1015 while (ioctx->n_rdma)
59fae4de 1016 kfree(ioctx->rdma_wrs[--ioctx->n_rdma].wr.sg_list);
a42d985b 1017
59fae4de
CH
1018 kfree(ioctx->rdma_wrs);
1019 ioctx->rdma_wrs = NULL;
a42d985b
BVA
1020
1021 if (ioctx->mapped_sg_count) {
1022 sg = ioctx->sg;
1023 WARN_ON(!sg);
1024 dir = ioctx->cmd.data_direction;
1025 BUG_ON(dir == DMA_NONE);
1026 ib_dma_unmap_sg(ch->sport->sdev->device, sg, ioctx->sg_cnt,
671ec1b2 1027 target_reverse_dma_direction(&ioctx->cmd));
a42d985b
BVA
1028 ioctx->mapped_sg_count = 0;
1029 }
1030}
1031
1032/**
1033 * srpt_map_sg_to_ib_sge() - Map an SG list to an IB SGE list.
1034 */
1035static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch,
1036 struct srpt_send_ioctx *ioctx)
1037{
b0768080 1038 struct ib_device *dev = ch->sport->sdev->device;
a42d985b
BVA
1039 struct se_cmd *cmd;
1040 struct scatterlist *sg, *sg_orig;
1041 int sg_cnt;
1042 enum dma_data_direction dir;
59fae4de 1043 struct ib_rdma_wr *riu;
a42d985b
BVA
1044 struct srp_direct_buf *db;
1045 dma_addr_t dma_addr;
1046 struct ib_sge *sge;
1047 u64 raddr;
1048 u32 rsize;
1049 u32 tsize;
1050 u32 dma_len;
1051 int count, nrdma;
1052 int i, j, k;
1053
1054 BUG_ON(!ch);
1055 BUG_ON(!ioctx);
1056 cmd = &ioctx->cmd;
1057 dir = cmd->data_direction;
1058 BUG_ON(dir == DMA_NONE);
1059
6f9e7f01
RD
1060 ioctx->sg = sg = sg_orig = cmd->t_data_sg;
1061 ioctx->sg_cnt = sg_cnt = cmd->t_data_nents;
a42d985b
BVA
1062
1063 count = ib_dma_map_sg(ch->sport->sdev->device, sg, sg_cnt,
671ec1b2 1064 target_reverse_dma_direction(cmd));
a42d985b
BVA
1065 if (unlikely(!count))
1066 return -EAGAIN;
1067
1068 ioctx->mapped_sg_count = count;
1069
59fae4de
CH
1070 if (ioctx->rdma_wrs && ioctx->n_rdma_wrs)
1071 nrdma = ioctx->n_rdma_wrs;
a42d985b
BVA
1072 else {
1073 nrdma = (count + SRPT_DEF_SG_PER_WQE - 1) / SRPT_DEF_SG_PER_WQE
1074 + ioctx->n_rbuf;
1075
59fae4de
CH
1076 ioctx->rdma_wrs = kcalloc(nrdma, sizeof(*ioctx->rdma_wrs),
1077 GFP_KERNEL);
1078 if (!ioctx->rdma_wrs)
a42d985b
BVA
1079 goto free_mem;
1080
59fae4de 1081 ioctx->n_rdma_wrs = nrdma;
a42d985b
BVA
1082 }
1083
1084 db = ioctx->rbufs;
1085 tsize = cmd->data_length;
b0768080 1086 dma_len = ib_sg_dma_len(dev, &sg[0]);
59fae4de 1087 riu = ioctx->rdma_wrs;
a42d985b
BVA
1088
1089 /*
1090 * For each remote desc - calculate the #ib_sge.
1091 * If #ib_sge < SRPT_DEF_SG_PER_WQE per rdma operation then
1092 * each remote desc rdma_iu is required a rdma wr;
1093 * else
1094 * we need to allocate extra rdma_iu to carry extra #ib_sge in
1095 * another rdma wr
1096 */
1097 for (i = 0, j = 0;
1098 j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
1099 rsize = be32_to_cpu(db->len);
1100 raddr = be64_to_cpu(db->va);
59fae4de 1101 riu->remote_addr = raddr;
a42d985b 1102 riu->rkey = be32_to_cpu(db->key);
59fae4de 1103 riu->wr.num_sge = 0;
a42d985b
BVA
1104
1105 /* calculate how many sge required for this remote_buf */
1106 while (rsize > 0 && tsize > 0) {
1107
1108 if (rsize >= dma_len) {
1109 tsize -= dma_len;
1110 rsize -= dma_len;
1111 raddr += dma_len;
1112
1113 if (tsize > 0) {
1114 ++j;
1115 if (j < count) {
1116 sg = sg_next(sg);
b0768080
MM
1117 dma_len = ib_sg_dma_len(
1118 dev, sg);
a42d985b
BVA
1119 }
1120 }
1121 } else {
1122 tsize -= rsize;
1123 dma_len -= rsize;
1124 rsize = 0;
1125 }
1126
59fae4de 1127 ++riu->wr.num_sge;
a42d985b 1128
59fae4de
CH
1129 if (rsize > 0 &&
1130 riu->wr.num_sge == SRPT_DEF_SG_PER_WQE) {
a42d985b 1131 ++ioctx->n_rdma;
59fae4de
CH
1132 riu->wr.sg_list = kmalloc_array(riu->wr.num_sge,
1133 sizeof(*riu->wr.sg_list),
1134 GFP_KERNEL);
1135 if (!riu->wr.sg_list)
a42d985b
BVA
1136 goto free_mem;
1137
1138 ++riu;
59fae4de
CH
1139 riu->wr.num_sge = 0;
1140 riu->remote_addr = raddr;
a42d985b
BVA
1141 riu->rkey = be32_to_cpu(db->key);
1142 }
1143 }
1144
1145 ++ioctx->n_rdma;
59fae4de
CH
1146 riu->wr.sg_list = kmalloc_array(riu->wr.num_sge,
1147 sizeof(*riu->wr.sg_list),
1148 GFP_KERNEL);
1149 if (!riu->wr.sg_list)
a42d985b
BVA
1150 goto free_mem;
1151 }
1152
1153 db = ioctx->rbufs;
1154 tsize = cmd->data_length;
59fae4de 1155 riu = ioctx->rdma_wrs;
a42d985b 1156 sg = sg_orig;
b0768080
MM
1157 dma_len = ib_sg_dma_len(dev, &sg[0]);
1158 dma_addr = ib_sg_dma_address(dev, &sg[0]);
a42d985b
BVA
1159
1160 /* this second loop is really mapped sg_addres to rdma_iu->ib_sge */
1161 for (i = 0, j = 0;
1162 j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
1163 rsize = be32_to_cpu(db->len);
59fae4de 1164 sge = riu->wr.sg_list;
a42d985b
BVA
1165 k = 0;
1166
1167 while (rsize > 0 && tsize > 0) {
1168 sge->addr = dma_addr;
5a783956 1169 sge->lkey = ch->sport->sdev->pd->local_dma_lkey;
a42d985b
BVA
1170
1171 if (rsize >= dma_len) {
1172 sge->length =
1173 (tsize < dma_len) ? tsize : dma_len;
1174 tsize -= dma_len;
1175 rsize -= dma_len;
1176
1177 if (tsize > 0) {
1178 ++j;
1179 if (j < count) {
1180 sg = sg_next(sg);
b0768080
MM
1181 dma_len = ib_sg_dma_len(
1182 dev, sg);
1183 dma_addr = ib_sg_dma_address(
1184 dev, sg);
a42d985b
BVA
1185 }
1186 }
1187 } else {
1188 sge->length = (tsize < rsize) ? tsize : rsize;
1189 tsize -= rsize;
1190 dma_len -= rsize;
1191 dma_addr += rsize;
1192 rsize = 0;
1193 }
1194
1195 ++k;
59fae4de 1196 if (k == riu->wr.num_sge && rsize > 0 && tsize > 0) {
a42d985b 1197 ++riu;
59fae4de 1198 sge = riu->wr.sg_list;
a42d985b
BVA
1199 k = 0;
1200 } else if (rsize > 0 && tsize > 0)
1201 ++sge;
1202 }
1203 }
1204
1205 return 0;
1206
1207free_mem:
1208 srpt_unmap_sg_to_ib_sge(ch, ioctx);
1209
1210 return -ENOMEM;
1211}
1212
1213/**
1214 * srpt_get_send_ioctx() - Obtain an I/O context for sending to the initiator.
1215 */
1216static struct srpt_send_ioctx *srpt_get_send_ioctx(struct srpt_rdma_ch *ch)
1217{
1218 struct srpt_send_ioctx *ioctx;
1219 unsigned long flags;
1220
1221 BUG_ON(!ch);
1222
1223 ioctx = NULL;
1224 spin_lock_irqsave(&ch->spinlock, flags);
1225 if (!list_empty(&ch->free_list)) {
1226 ioctx = list_first_entry(&ch->free_list,
1227 struct srpt_send_ioctx, free_list);
1228 list_del(&ioctx->free_list);
1229 }
1230 spin_unlock_irqrestore(&ch->spinlock, flags);
1231
1232 if (!ioctx)
1233 return ioctx;
1234
1235 BUG_ON(ioctx->ch != ch);
a42d985b
BVA
1236 spin_lock_init(&ioctx->spinlock);
1237 ioctx->state = SRPT_STATE_NEW;
1238 ioctx->n_rbuf = 0;
1239 ioctx->rbufs = NULL;
1240 ioctx->n_rdma = 0;
59fae4de
CH
1241 ioctx->n_rdma_wrs = 0;
1242 ioctx->rdma_wrs = NULL;
a42d985b
BVA
1243 ioctx->mapped_sg_count = 0;
1244 init_completion(&ioctx->tx_done);
1245 ioctx->queue_status_only = false;
1246 /*
1247 * transport_init_se_cmd() does not initialize all fields, so do it
1248 * here.
1249 */
1250 memset(&ioctx->cmd, 0, sizeof(ioctx->cmd));
1251 memset(&ioctx->sense_data, 0, sizeof(ioctx->sense_data));
1252
1253 return ioctx;
1254}
1255
a42d985b
BVA
1256/**
1257 * srpt_abort_cmd() - Abort a SCSI command.
1258 * @ioctx: I/O context associated with the SCSI command.
1259 * @context: Preferred execution context.
1260 */
1261static int srpt_abort_cmd(struct srpt_send_ioctx *ioctx)
1262{
1263 enum srpt_command_state state;
1264 unsigned long flags;
1265
1266 BUG_ON(!ioctx);
1267
1268 /*
1269 * If the command is in a state where the target core is waiting for
49f40163 1270 * the ib_srpt driver, change the state to the next state.
a42d985b
BVA
1271 */
1272
1273 spin_lock_irqsave(&ioctx->spinlock, flags);
1274 state = ioctx->state;
1275 switch (state) {
1276 case SRPT_STATE_NEED_DATA:
1277 ioctx->state = SRPT_STATE_DATA_IN;
1278 break;
a42d985b
BVA
1279 case SRPT_STATE_CMD_RSP_SENT:
1280 case SRPT_STATE_MGMT_RSP_SENT:
1281 ioctx->state = SRPT_STATE_DONE;
1282 break;
1283 default:
49f40163
BVA
1284 WARN_ONCE(true, "%s: unexpected I/O context state %d\n",
1285 __func__, state);
a42d985b
BVA
1286 break;
1287 }
1288 spin_unlock_irqrestore(&ioctx->spinlock, flags);
1289
a42d985b 1290 pr_debug("Aborting cmd with state %d and tag %lld\n", state,
649ee054 1291 ioctx->cmd.tag);
a42d985b
BVA
1292
1293 switch (state) {
1294 case SRPT_STATE_NEW:
1295 case SRPT_STATE_DATA_IN:
1296 case SRPT_STATE_MGMT:
49f40163 1297 case SRPT_STATE_DONE:
a42d985b
BVA
1298 /*
1299 * Do nothing - defer abort processing until
1300 * srpt_queue_response() is invoked.
1301 */
a42d985b
BVA
1302 break;
1303 case SRPT_STATE_NEED_DATA:
49f40163
BVA
1304 pr_debug("tag %#llx: RDMA read error\n", ioctx->cmd.tag);
1305 transport_generic_request_failure(&ioctx->cmd,
1306 TCM_CHECK_CONDITION_ABORT_CMD);
a42d985b
BVA
1307 break;
1308 case SRPT_STATE_CMD_RSP_SENT:
1309 /*
1310 * SRP_RSP sending failed or the SRP_RSP send completion has
1311 * not been received in time.
1312 */
1313 srpt_unmap_sg_to_ib_sge(ioctx->ch, ioctx);
49f40163 1314 transport_generic_free_cmd(&ioctx->cmd, 0);
a42d985b
BVA
1315 break;
1316 case SRPT_STATE_MGMT_RSP_SENT:
49f40163 1317 transport_generic_free_cmd(&ioctx->cmd, 0);
a42d985b
BVA
1318 break;
1319 default:
532ec6f1 1320 WARN(1, "Unexpected command state (%d)", state);
a42d985b
BVA
1321 break;
1322 }
1323
a42d985b
BVA
1324 return state;
1325}
1326
1327/**
e672a47f
CH
1328 * XXX: what is now target_execute_cmd used to be asynchronous, and unmapping
1329 * the data that has been transferred via IB RDMA had to be postponed until the
142ad5db 1330 * check_stop_free() callback. None of this is necessary anymore and needs to
e672a47f 1331 * be cleaned up.
a42d985b 1332 */
59fae4de 1333static void srpt_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc)
a42d985b 1334{
59fae4de
CH
1335 struct srpt_rdma_ch *ch = cq->cq_context;
1336 struct srpt_send_ioctx *ioctx =
19f57298 1337 container_of(wc->wr_cqe, struct srpt_send_ioctx, rdma_cqe);
59fae4de 1338
a42d985b
BVA
1339 WARN_ON(ioctx->n_rdma <= 0);
1340 atomic_add(ioctx->n_rdma, &ch->sq_wr_avail);
1341
59fae4de
CH
1342 if (unlikely(wc->status != IB_WC_SUCCESS)) {
1343 pr_info("RDMA_READ for ioctx 0x%p failed with status %d\n",
1344 ioctx, wc->status);
1345 srpt_abort_cmd(ioctx);
1346 return;
a42d985b 1347 }
59fae4de
CH
1348
1349 if (srpt_test_and_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA,
1350 SRPT_STATE_DATA_IN))
1351 target_execute_cmd(&ioctx->cmd);
1352 else
1353 pr_err("%s[%d]: wrong state = %d\n", __func__,
1354 __LINE__, srpt_get_cmd_state(ioctx));
a42d985b
BVA
1355}
1356
59fae4de 1357static void srpt_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc)
a42d985b 1358{
59fae4de 1359 struct srpt_send_ioctx *ioctx =
19f57298 1360 container_of(wc->wr_cqe, struct srpt_send_ioctx, rdma_cqe);
a42d985b 1361
59fae4de 1362 if (unlikely(wc->status != IB_WC_SUCCESS)) {
49f40163
BVA
1363 /*
1364 * Note: if an RDMA write error completion is received that
1365 * means that a SEND also has been posted. Defer further
1366 * processing of the associated command until the send error
1367 * completion has been received.
1368 */
59fae4de
CH
1369 pr_info("RDMA_WRITE for ioctx 0x%p failed with status %d\n",
1370 ioctx, wc->status);
a42d985b
BVA
1371 }
1372}
1373
1374/**
1375 * srpt_build_cmd_rsp() - Build an SRP_RSP response.
1376 * @ch: RDMA channel through which the request has been received.
1377 * @ioctx: I/O context associated with the SRP_CMD request. The response will
1378 * be built in the buffer ioctx->buf points at and hence this function will
1379 * overwrite the request data.
1380 * @tag: tag of the request for which this response is being generated.
1381 * @status: value for the STATUS field of the SRP_RSP information unit.
1382 *
1383 * Returns the size in bytes of the SRP_RSP response.
1384 *
1385 * An SRP_RSP response contains a SCSI status or service response. See also
1386 * section 6.9 in the SRP r16a document for the format of an SRP_RSP
1387 * response. See also SPC-2 for more information about sense data.
1388 */
1389static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
1390 struct srpt_send_ioctx *ioctx, u64 tag,
1391 int status)
1392{
1393 struct srp_rsp *srp_rsp;
1394 const u8 *sense_data;
1395 int sense_data_len, max_sense_len;
1396
1397 /*
1398 * The lowest bit of all SAM-3 status codes is zero (see also
1399 * paragraph 5.3 in SAM-3).
1400 */
1401 WARN_ON(status & 1);
1402
1403 srp_rsp = ioctx->ioctx.buf;
1404 BUG_ON(!srp_rsp);
1405
1406 sense_data = ioctx->sense_data;
1407 sense_data_len = ioctx->cmd.scsi_sense_length;
1408 WARN_ON(sense_data_len > sizeof(ioctx->sense_data));
1409
9d2aa2b4 1410 memset(srp_rsp, 0, sizeof(*srp_rsp));
a42d985b
BVA
1411 srp_rsp->opcode = SRP_RSP;
1412 srp_rsp->req_lim_delta =
b356c1c1 1413 cpu_to_be32(1 + atomic_xchg(&ch->req_lim_delta, 0));
a42d985b
BVA
1414 srp_rsp->tag = tag;
1415 srp_rsp->status = status;
1416
1417 if (sense_data_len) {
1418 BUILD_BUG_ON(MIN_MAX_RSP_SIZE <= sizeof(*srp_rsp));
1419 max_sense_len = ch->max_ti_iu_len - sizeof(*srp_rsp);
1420 if (sense_data_len > max_sense_len) {
9f5d32af
DL
1421 pr_warn("truncated sense data from %d to %d"
1422 " bytes\n", sense_data_len, max_sense_len);
a42d985b
BVA
1423 sense_data_len = max_sense_len;
1424 }
1425
1426 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
1427 srp_rsp->sense_data_len = cpu_to_be32(sense_data_len);
1428 memcpy(srp_rsp + 1, sense_data, sense_data_len);
1429 }
1430
1431 return sizeof(*srp_rsp) + sense_data_len;
1432}
1433
1434/**
1435 * srpt_build_tskmgmt_rsp() - Build a task management response.
1436 * @ch: RDMA channel through which the request has been received.
1437 * @ioctx: I/O context in which the SRP_RSP response will be built.
1438 * @rsp_code: RSP_CODE that will be stored in the response.
1439 * @tag: Tag of the request for which this response is being generated.
1440 *
1441 * Returns the size in bytes of the SRP_RSP response.
1442 *
1443 * An SRP_RSP response contains a SCSI status or service response. See also
1444 * section 6.9 in the SRP r16a document for the format of an SRP_RSP
1445 * response.
1446 */
1447static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch,
1448 struct srpt_send_ioctx *ioctx,
1449 u8 rsp_code, u64 tag)
1450{
1451 struct srp_rsp *srp_rsp;
1452 int resp_data_len;
1453 int resp_len;
1454
c807f643 1455 resp_data_len = 4;
a42d985b
BVA
1456 resp_len = sizeof(*srp_rsp) + resp_data_len;
1457
1458 srp_rsp = ioctx->ioctx.buf;
1459 BUG_ON(!srp_rsp);
9d2aa2b4 1460 memset(srp_rsp, 0, sizeof(*srp_rsp));
a42d985b
BVA
1461
1462 srp_rsp->opcode = SRP_RSP;
b356c1c1
VT
1463 srp_rsp->req_lim_delta =
1464 cpu_to_be32(1 + atomic_xchg(&ch->req_lim_delta, 0));
a42d985b
BVA
1465 srp_rsp->tag = tag;
1466
c807f643
JW
1467 srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID;
1468 srp_rsp->resp_data_len = cpu_to_be32(resp_data_len);
1469 srp_rsp->data[3] = rsp_code;
a42d985b
BVA
1470
1471 return resp_len;
1472}
1473
a42d985b
BVA
1474static int srpt_check_stop_free(struct se_cmd *cmd)
1475{
9474b043
NB
1476 struct srpt_send_ioctx *ioctx = container_of(cmd,
1477 struct srpt_send_ioctx, cmd);
a42d985b 1478
afc16604 1479 return target_put_sess_cmd(&ioctx->cmd);
a42d985b
BVA
1480}
1481
1482/**
1483 * srpt_handle_cmd() - Process SRP_CMD.
1484 */
2c7f37ff
BVA
1485static void srpt_handle_cmd(struct srpt_rdma_ch *ch,
1486 struct srpt_recv_ioctx *recv_ioctx,
1487 struct srpt_send_ioctx *send_ioctx)
a42d985b
BVA
1488{
1489 struct se_cmd *cmd;
1490 struct srp_cmd *srp_cmd;
a42d985b
BVA
1491 u64 data_len;
1492 enum dma_data_direction dir;
9474b043 1493 int rc;
a42d985b
BVA
1494
1495 BUG_ON(!send_ioctx);
1496
1497 srp_cmd = recv_ioctx->ioctx.buf;
a42d985b 1498 cmd = &send_ioctx->cmd;
649ee054 1499 cmd->tag = srp_cmd->tag;
a42d985b
BVA
1500
1501 switch (srp_cmd->task_attr) {
1502 case SRP_CMD_SIMPLE_Q:
68d81f40 1503 cmd->sam_task_attr = TCM_SIMPLE_TAG;
a42d985b
BVA
1504 break;
1505 case SRP_CMD_ORDERED_Q:
1506 default:
68d81f40 1507 cmd->sam_task_attr = TCM_ORDERED_TAG;
a42d985b
BVA
1508 break;
1509 case SRP_CMD_HEAD_OF_Q:
68d81f40 1510 cmd->sam_task_attr = TCM_HEAD_TAG;
a42d985b
BVA
1511 break;
1512 case SRP_CMD_ACA:
68d81f40 1513 cmd->sam_task_attr = TCM_ACA_TAG;
a42d985b
BVA
1514 break;
1515 }
1516
de103c93 1517 if (srpt_get_desc_tbl(send_ioctx, srp_cmd, &dir, &data_len)) {
9f5d32af 1518 pr_err("0x%llx: parsing SRP descriptor table failed.\n",
a42d985b 1519 srp_cmd->tag);
2c7f37ff 1520 goto release_ioctx;
a42d985b
BVA
1521 }
1522
9474b043 1523 rc = target_submit_cmd(cmd, ch->sess, srp_cmd->cdb,
e1dd413c
BVA
1524 &send_ioctx->sense_data[0],
1525 scsilun_to_int(&srp_cmd->lun), data_len,
1526 TCM_SIMPLE_TAG, dir, TARGET_SCF_ACK_KREF);
9474b043 1527 if (rc != 0) {
2c7f37ff
BVA
1528 pr_debug("target_submit_cmd() returned %d for tag %#llx\n", rc,
1529 srp_cmd->tag);
1530 goto release_ioctx;
187e70a5 1531 }
2c7f37ff 1532 return;
a42d985b 1533
2c7f37ff
BVA
1534release_ioctx:
1535 send_ioctx->state = SRPT_STATE_DONE;
1536 srpt_release_cmd(cmd);
a42d985b
BVA
1537}
1538
a42d985b
BVA
1539static int srp_tmr_to_tcm(int fn)
1540{
1541 switch (fn) {
1542 case SRP_TSK_ABORT_TASK:
1543 return TMR_ABORT_TASK;
1544 case SRP_TSK_ABORT_TASK_SET:
1545 return TMR_ABORT_TASK_SET;
1546 case SRP_TSK_CLEAR_TASK_SET:
1547 return TMR_CLEAR_TASK_SET;
1548 case SRP_TSK_LUN_RESET:
1549 return TMR_LUN_RESET;
1550 case SRP_TSK_CLEAR_ACA:
1551 return TMR_CLEAR_ACA;
1552 default:
1553 return -1;
1554 }
1555}
1556
1557/**
1558 * srpt_handle_tsk_mgmt() - Process an SRP_TSK_MGMT information unit.
1559 *
1560 * Returns 0 if and only if the request will be processed by the target core.
1561 *
1562 * For more information about SRP_TSK_MGMT information units, see also section
1563 * 6.7 in the SRP r16a document.
1564 */
1565static void srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
1566 struct srpt_recv_ioctx *recv_ioctx,
1567 struct srpt_send_ioctx *send_ioctx)
1568{
1569 struct srp_tsk_mgmt *srp_tsk;
1570 struct se_cmd *cmd;
3e4f5748 1571 struct se_session *sess = ch->sess;
a42d985b 1572 int tcm_tmr;
3e4f5748 1573 int rc;
a42d985b
BVA
1574
1575 BUG_ON(!send_ioctx);
1576
1577 srp_tsk = recv_ioctx->ioctx.buf;
1578 cmd = &send_ioctx->cmd;
1579
1580 pr_debug("recv tsk_mgmt fn %d for task_tag %lld and cmd tag %lld"
1581 " cm_id %p sess %p\n", srp_tsk->tsk_mgmt_func,
1582 srp_tsk->task_tag, srp_tsk->tag, ch->cm_id, ch->sess);
1583
1584 srpt_set_cmd_state(send_ioctx, SRPT_STATE_MGMT);
649ee054 1585 send_ioctx->cmd.tag = srp_tsk->tag;
a42d985b 1586 tcm_tmr = srp_tmr_to_tcm(srp_tsk->tsk_mgmt_func);
e1dd413c
BVA
1587 rc = target_submit_tmr(&send_ioctx->cmd, sess, NULL,
1588 scsilun_to_int(&srp_tsk->lun), srp_tsk, tcm_tmr,
1589 GFP_KERNEL, srp_tsk->task_tag,
1590 TARGET_SCF_ACK_KREF);
3e4f5748
NB
1591 if (rc != 0) {
1592 send_ioctx->cmd.se_tmr_req->response = TMR_FUNCTION_REJECTED;
de103c93 1593 goto fail;
a42d985b 1594 }
de103c93
CH
1595 return;
1596fail:
de103c93 1597 transport_send_check_condition_and_sense(cmd, 0, 0); // XXX:
a42d985b
BVA
1598}
1599
1600/**
1601 * srpt_handle_new_iu() - Process a newly received information unit.
1602 * @ch: RDMA channel through which the information unit has been received.
1603 * @ioctx: SRPT I/O context associated with the information unit.
1604 */
1605static void srpt_handle_new_iu(struct srpt_rdma_ch *ch,
1606 struct srpt_recv_ioctx *recv_ioctx,
1607 struct srpt_send_ioctx *send_ioctx)
1608{
1609 struct srp_cmd *srp_cmd;
a42d985b
BVA
1610
1611 BUG_ON(!ch);
1612 BUG_ON(!recv_ioctx);
1613
1614 ib_dma_sync_single_for_cpu(ch->sport->sdev->device,
1615 recv_ioctx->ioctx.dma, srp_max_req_size,
1616 DMA_FROM_DEVICE);
1617
33912d73 1618 if (unlikely(ch->state == CH_CONNECTING)) {
a42d985b
BVA
1619 list_add_tail(&recv_ioctx->wait_list, &ch->cmd_wait_list);
1620 goto out;
1621 }
1622
33912d73 1623 if (unlikely(ch->state != CH_LIVE))
a42d985b
BVA
1624 goto out;
1625
1626 srp_cmd = recv_ioctx->ioctx.buf;
1627 if (srp_cmd->opcode == SRP_CMD || srp_cmd->opcode == SRP_TSK_MGMT) {
1628 if (!send_ioctx)
1629 send_ioctx = srpt_get_send_ioctx(ch);
1630 if (unlikely(!send_ioctx)) {
1631 list_add_tail(&recv_ioctx->wait_list,
1632 &ch->cmd_wait_list);
1633 goto out;
1634 }
1635 }
1636
a42d985b
BVA
1637 switch (srp_cmd->opcode) {
1638 case SRP_CMD:
1639 srpt_handle_cmd(ch, recv_ioctx, send_ioctx);
1640 break;
1641 case SRP_TSK_MGMT:
1642 srpt_handle_tsk_mgmt(ch, recv_ioctx, send_ioctx);
1643 break;
1644 case SRP_I_LOGOUT:
9f5d32af 1645 pr_err("Not yet implemented: SRP_I_LOGOUT\n");
a42d985b
BVA
1646 break;
1647 case SRP_CRED_RSP:
1648 pr_debug("received SRP_CRED_RSP\n");
1649 break;
1650 case SRP_AER_RSP:
1651 pr_debug("received SRP_AER_RSP\n");
1652 break;
1653 case SRP_RSP:
9f5d32af 1654 pr_err("Received SRP_RSP\n");
a42d985b
BVA
1655 break;
1656 default:
9f5d32af 1657 pr_err("received IU with unknown opcode 0x%x\n",
a42d985b
BVA
1658 srp_cmd->opcode);
1659 break;
1660 }
1661
1662 srpt_post_recv(ch->sport->sdev, recv_ioctx);
1663out:
1664 return;
1665}
1666
59fae4de 1667static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc)
a42d985b 1668{
59fae4de
CH
1669 struct srpt_rdma_ch *ch = cq->cq_context;
1670 struct srpt_recv_ioctx *ioctx =
1671 container_of(wc->wr_cqe, struct srpt_recv_ioctx, ioctx.cqe);
a42d985b 1672
a42d985b
BVA
1673 if (wc->status == IB_WC_SUCCESS) {
1674 int req_lim;
1675
1676 req_lim = atomic_dec_return(&ch->req_lim);
1677 if (unlikely(req_lim < 0))
9f5d32af 1678 pr_err("req_lim = %d < 0\n", req_lim);
a42d985b
BVA
1679 srpt_handle_new_iu(ch, ioctx, NULL);
1680 } else {
59fae4de
CH
1681 pr_info("receiving failed for ioctx %p with status %d\n",
1682 ioctx, wc->status);
a42d985b
BVA
1683 }
1684}
1685
1686/**
a42d985b
BVA
1687 * Note: Although this has not yet been observed during tests, at least in
1688 * theory it is possible that the srpt_get_send_ioctx() call invoked by
1689 * srpt_handle_new_iu() fails. This is possible because the req_lim_delta
1690 * value in each response is set to one, and it is possible that this response
1691 * makes the initiator send a new request before the send completion for that
1692 * response has been processed. This could e.g. happen if the call to
1693 * srpt_put_send_iotcx() is delayed because of a higher priority interrupt or
1694 * if IB retransmission causes generation of the send completion to be
1695 * delayed. Incoming information units for which srpt_get_send_ioctx() fails
1696 * are queued on cmd_wait_list. The code below processes these delayed
1697 * requests one at a time.
1698 */
59fae4de 1699static void srpt_send_done(struct ib_cq *cq, struct ib_wc *wc)
a42d985b 1700{
59fae4de
CH
1701 struct srpt_rdma_ch *ch = cq->cq_context;
1702 struct srpt_send_ioctx *ioctx =
1703 container_of(wc->wr_cqe, struct srpt_send_ioctx, ioctx.cqe);
1704 enum srpt_command_state state;
a42d985b 1705
59fae4de
CH
1706 state = srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);
1707
1708 WARN_ON(state != SRPT_STATE_CMD_RSP_SENT &&
1709 state != SRPT_STATE_MGMT_RSP_SENT);
1710
1711 atomic_inc(&ch->sq_wr_avail);
1712
49f40163 1713 if (wc->status != IB_WC_SUCCESS)
59fae4de
CH
1714 pr_info("sending response for ioctx 0x%p failed"
1715 " with status %d\n", ioctx, wc->status);
1716
59fae4de
CH
1717 if (state != SRPT_STATE_DONE) {
1718 srpt_unmap_sg_to_ib_sge(ch, ioctx);
1719 transport_generic_free_cmd(&ioctx->cmd, 0);
a42d985b 1720 } else {
59fae4de
CH
1721 pr_err("IB completion has been received too late for"
1722 " wr_id = %u.\n", ioctx->ioctx.index);
a42d985b
BVA
1723 }
1724
59fae4de 1725 while (!list_empty(&ch->cmd_wait_list) &&
33912d73 1726 ch->state == CH_LIVE &&
59fae4de 1727 (ioctx = srpt_get_send_ioctx(ch)) != NULL) {
a42d985b
BVA
1728 struct srpt_recv_ioctx *recv_ioctx;
1729
1730 recv_ioctx = list_first_entry(&ch->cmd_wait_list,
1731 struct srpt_recv_ioctx,
1732 wait_list);
1733 list_del(&recv_ioctx->wait_list);
59fae4de 1734 srpt_handle_new_iu(ch, recv_ioctx, ioctx);
a42d985b
BVA
1735 }
1736}
1737
a42d985b
BVA
1738/**
1739 * srpt_create_ch_ib() - Create receive and send completion queues.
1740 */
1741static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
1742{
1743 struct ib_qp_init_attr *qp_init;
1744 struct srpt_port *sport = ch->sport;
1745 struct srpt_device *sdev = sport->sdev;
1746 u32 srp_sq_size = sport->port_attrib.srp_sq_size;
1747 int ret;
1748
1749 WARN_ON(ch->rq_size < 1);
1750
1751 ret = -ENOMEM;
9d2aa2b4 1752 qp_init = kzalloc(sizeof(*qp_init), GFP_KERNEL);
a42d985b
BVA
1753 if (!qp_init)
1754 goto out;
1755
ab477c1f 1756retry:
59fae4de
CH
1757 ch->cq = ib_alloc_cq(sdev->device, ch, ch->rq_size + srp_sq_size,
1758 0 /* XXX: spread CQs */, IB_POLL_WORKQUEUE);
a42d985b
BVA
1759 if (IS_ERR(ch->cq)) {
1760 ret = PTR_ERR(ch->cq);
9f5d32af 1761 pr_err("failed to create CQ cqe= %d ret= %d\n",
a42d985b
BVA
1762 ch->rq_size + srp_sq_size, ret);
1763 goto out;
1764 }
1765
1766 qp_init->qp_context = (void *)ch;
1767 qp_init->event_handler
1768 = (void(*)(struct ib_event *, void*))srpt_qp_event;
1769 qp_init->send_cq = ch->cq;
1770 qp_init->recv_cq = ch->cq;
1771 qp_init->srq = sdev->srq;
1772 qp_init->sq_sig_type = IB_SIGNAL_REQ_WR;
1773 qp_init->qp_type = IB_QPT_RC;
1774 qp_init->cap.max_send_wr = srp_sq_size;
1775 qp_init->cap.max_send_sge = SRPT_DEF_SG_PER_WQE;
1776
1777 ch->qp = ib_create_qp(sdev->pd, qp_init);
1778 if (IS_ERR(ch->qp)) {
1779 ret = PTR_ERR(ch->qp);
ab477c1f
BVA
1780 if (ret == -ENOMEM) {
1781 srp_sq_size /= 2;
1782 if (srp_sq_size >= MIN_SRPT_SQ_SIZE) {
1783 ib_destroy_cq(ch->cq);
1784 goto retry;
1785 }
1786 }
9f5d32af 1787 pr_err("failed to create_qp ret= %d\n", ret);
a42d985b
BVA
1788 goto err_destroy_cq;
1789 }
1790
1791 atomic_set(&ch->sq_wr_avail, qp_init->cap.max_send_wr);
1792
1793 pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d cm_id= %p\n",
1794 __func__, ch->cq->cqe, qp_init->cap.max_send_sge,
1795 qp_init->cap.max_send_wr, ch->cm_id);
1796
1797 ret = srpt_init_ch_qp(ch, ch->qp);
1798 if (ret)
1799 goto err_destroy_qp;
1800
a42d985b
BVA
1801out:
1802 kfree(qp_init);
1803 return ret;
1804
1805err_destroy_qp:
1806 ib_destroy_qp(ch->qp);
1807err_destroy_cq:
59fae4de 1808 ib_free_cq(ch->cq);
a42d985b
BVA
1809 goto out;
1810}
1811
1812static void srpt_destroy_ch_ib(struct srpt_rdma_ch *ch)
1813{
a42d985b 1814 ib_destroy_qp(ch->qp);
59fae4de 1815 ib_free_cq(ch->cq);
a42d985b
BVA
1816}
1817
1818/**
1819 * __srpt_close_ch() - Close an RDMA channel by setting the QP error state.
1820 *
1821 * Reset the QP and make sure all resources associated with the channel will
1822 * be deallocated at an appropriate time.
1823 *
1824 * Note: The caller must hold ch->sport->sdev->spinlock.
1825 */
1826static void __srpt_close_ch(struct srpt_rdma_ch *ch)
1827{
a42d985b
BVA
1828 enum rdma_ch_state prev_state;
1829 unsigned long flags;
1830
a42d985b
BVA
1831 spin_lock_irqsave(&ch->spinlock, flags);
1832 prev_state = ch->state;
1833 switch (prev_state) {
1834 case CH_CONNECTING:
1835 case CH_LIVE:
1836 ch->state = CH_DISCONNECTING;
1837 break;
1838 default:
1839 break;
1840 }
1841 spin_unlock_irqrestore(&ch->spinlock, flags);
1842
1843 switch (prev_state) {
1844 case CH_CONNECTING:
1845 ib_send_cm_rej(ch->cm_id, IB_CM_REJ_NO_RESOURCES, NULL, 0,
1846 NULL, 0);
1847 /* fall through */
1848 case CH_LIVE:
1849 if (ib_send_cm_dreq(ch->cm_id, NULL, 0) < 0)
9f5d32af 1850 pr_err("sending CM DREQ failed.\n");
a42d985b
BVA
1851 break;
1852 case CH_DISCONNECTING:
1853 break;
1854 case CH_DRAINING:
1855 case CH_RELEASING:
1856 break;
1857 }
1858}
1859
1860/**
1861 * srpt_close_ch() - Close an RDMA channel.
1862 */
1863static void srpt_close_ch(struct srpt_rdma_ch *ch)
1864{
1865 struct srpt_device *sdev;
1866
1867 sdev = ch->sport->sdev;
1868 spin_lock_irq(&sdev->spinlock);
1869 __srpt_close_ch(ch);
1870 spin_unlock_irq(&sdev->spinlock);
1871}
1872
1d19f780
NB
1873/**
1874 * srpt_shutdown_session() - Whether or not a session may be shut down.
1875 */
1876static int srpt_shutdown_session(struct se_session *se_sess)
1877{
88936259 1878 return 1;
1d19f780
NB
1879}
1880
a42d985b
BVA
1881/**
1882 * srpt_drain_channel() - Drain a channel by resetting the IB queue pair.
1883 * @cm_id: Pointer to the CM ID of the channel to be drained.
1884 *
1885 * Note: Must be called from inside srpt_cm_handler to avoid a race between
1886 * accessing sdev->spinlock and the call to kfree(sdev) in srpt_remove_one()
1887 * (the caller of srpt_cm_handler holds the cm_id spinlock; srpt_remove_one()
1888 * waits until all target sessions for the associated IB device have been
1889 * unregistered and target session registration involves a call to
1890 * ib_destroy_cm_id(), which locks the cm_id spinlock and hence waits until
1891 * this function has finished).
1892 */
2739b592 1893static void srpt_drain_channel(struct srpt_rdma_ch *ch)
a42d985b 1894{
a42d985b
BVA
1895 int ret;
1896 bool do_reset = false;
1897
1898 WARN_ON_ONCE(irqs_disabled());
1899
2739b592 1900 do_reset = srpt_set_ch_state(ch, CH_DRAINING);
a42d985b
BVA
1901
1902 if (do_reset) {
1d19f780
NB
1903 if (ch->sess)
1904 srpt_shutdown_session(ch->sess);
1905
a42d985b
BVA
1906 ret = srpt_ch_qp_err(ch);
1907 if (ret < 0)
9f5d32af 1908 pr_err("Setting queue pair in error state"
a42d985b
BVA
1909 " failed: %d\n", ret);
1910 }
1911}
1912
a42d985b
BVA
1913/**
1914 * srpt_release_channel() - Release channel resources.
1915 *
1916 * Schedules the actual release because:
1917 * - Calling the ib_destroy_cm_id() call from inside an IB CM callback would
1918 * trigger a deadlock.
1919 * - It is not safe to call TCM transport_* functions from interrupt context.
1920 */
1921static void srpt_release_channel(struct srpt_rdma_ch *ch)
1922{
1923 schedule_work(&ch->release_work);
1924}
1925
1926static void srpt_release_channel_work(struct work_struct *w)
1927{
1928 struct srpt_rdma_ch *ch;
1929 struct srpt_device *sdev;
9474b043 1930 struct se_session *se_sess;
a42d985b
BVA
1931
1932 ch = container_of(w, struct srpt_rdma_ch, release_work);
f108f0f6
BVA
1933 pr_debug("%s: %s-%d; release_done = %p\n", __func__, ch->sess_name,
1934 ch->qp->qp_num, ch->release_done);
a42d985b
BVA
1935
1936 sdev = ch->sport->sdev;
1937 BUG_ON(!sdev);
1938
9474b043
NB
1939 se_sess = ch->sess;
1940 BUG_ON(!se_sess);
1941
88936259 1942 target_sess_cmd_list_set_waiting(se_sess);
be646c2d 1943 target_wait_for_sess_cmds(se_sess);
9474b043
NB
1944
1945 transport_deregister_session_configfs(se_sess);
1946 transport_deregister_session(se_sess);
a42d985b
BVA
1947 ch->sess = NULL;
1948
0b41d6ca
NB
1949 ib_destroy_cm_id(ch->cm_id);
1950
a42d985b
BVA
1951 srpt_destroy_ch_ib(ch);
1952
1953 srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring,
1954 ch->sport->sdev, ch->rq_size,
1955 ch->rsp_size, DMA_TO_DEVICE);
1956
1957 spin_lock_irq(&sdev->spinlock);
f108f0f6 1958 list_del_init(&ch->list);
a42d985b
BVA
1959 if (ch->release_done)
1960 complete(ch->release_done);
f108f0f6 1961 spin_unlock_irq(&sdev->spinlock);
a42d985b
BVA
1962
1963 wake_up(&sdev->ch_releaseQ);
1964
1965 kfree(ch);
1966}
1967
a42d985b
BVA
1968/**
1969 * srpt_cm_req_recv() - Process the event IB_CM_REQ_RECEIVED.
1970 *
1971 * Ownership of the cm_id is transferred to the target session if this
1972 * functions returns zero. Otherwise the caller remains the owner of cm_id.
1973 */
1974static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
1975 struct ib_cm_req_event_param *param,
1976 void *private_data)
1977{
1978 struct srpt_device *sdev = cm_id->context;
1979 struct srpt_port *sport = &sdev->port[param->port - 1];
1980 struct srp_login_req *req;
1981 struct srp_login_rsp *rsp;
1982 struct srp_login_rej *rej;
1983 struct ib_cm_rep_param *rep_param;
1984 struct srpt_rdma_ch *ch, *tmp_ch;
f246c941 1985 struct se_node_acl *se_acl;
a42d985b 1986 u32 it_iu_len;
f246c941
NB
1987 int i, ret = 0;
1988 unsigned char *p;
a42d985b
BVA
1989
1990 WARN_ON_ONCE(irqs_disabled());
1991
1992 if (WARN_ON(!sdev || !private_data))
1993 return -EINVAL;
1994
1995 req = (struct srp_login_req *)private_data;
1996
1997 it_iu_len = be32_to_cpu(req->req_it_iu_len);
1998
9f5d32af
DL
1999 pr_info("Received SRP_LOGIN_REQ with i_port_id 0x%llx:0x%llx,"
2000 " t_port_id 0x%llx:0x%llx and it_iu_len %d on port %d"
2001 " (guid=0x%llx:0x%llx)\n",
2002 be64_to_cpu(*(__be64 *)&req->initiator_port_id[0]),
2003 be64_to_cpu(*(__be64 *)&req->initiator_port_id[8]),
2004 be64_to_cpu(*(__be64 *)&req->target_port_id[0]),
2005 be64_to_cpu(*(__be64 *)&req->target_port_id[8]),
2006 it_iu_len,
2007 param->port,
2008 be64_to_cpu(*(__be64 *)&sdev->port[param->port - 1].gid.raw[0]),
2009 be64_to_cpu(*(__be64 *)&sdev->port[param->port - 1].gid.raw[8]));
a42d985b 2010
9d2aa2b4
BVA
2011 rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
2012 rej = kzalloc(sizeof(*rej), GFP_KERNEL);
2013 rep_param = kzalloc(sizeof(*rep_param), GFP_KERNEL);
a42d985b
BVA
2014
2015 if (!rsp || !rej || !rep_param) {
2016 ret = -ENOMEM;
2017 goto out;
2018 }
2019
2020 if (it_iu_len > srp_max_req_size || it_iu_len < 64) {
b356c1c1
VT
2021 rej->reason = cpu_to_be32(
2022 SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
a42d985b 2023 ret = -EINVAL;
9f5d32af 2024 pr_err("rejected SRP_LOGIN_REQ because its"
a42d985b
BVA
2025 " length (%d bytes) is out of range (%d .. %d)\n",
2026 it_iu_len, 64, srp_max_req_size);
2027 goto reject;
2028 }
2029
2030 if (!sport->enabled) {
b356c1c1
VT
2031 rej->reason = cpu_to_be32(
2032 SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
a42d985b 2033 ret = -EINVAL;
9f5d32af 2034 pr_err("rejected SRP_LOGIN_REQ because the target port"
a42d985b
BVA
2035 " has not yet been enabled\n");
2036 goto reject;
2037 }
2038
2039 if ((req->req_flags & SRP_MTCH_ACTION) == SRP_MULTICHAN_SINGLE) {
2040 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
2041
2042 spin_lock_irq(&sdev->spinlock);
2043
2044 list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
2045 if (!memcmp(ch->i_port_id, req->initiator_port_id, 16)
2046 && !memcmp(ch->t_port_id, req->target_port_id, 16)
2047 && param->port == ch->sport->port
2048 && param->listen_id == ch->sport->sdev->cm_id
2049 && ch->cm_id) {
33912d73
BVA
2050 if (ch->state != CH_CONNECTING
2051 && ch->state != CH_LIVE)
a42d985b
BVA
2052 continue;
2053
2054 /* found an existing channel */
2055 pr_debug("Found existing channel %s"
2056 " cm_id= %p state= %d\n",
33912d73 2057 ch->sess_name, ch->cm_id, ch->state);
a42d985b
BVA
2058
2059 __srpt_close_ch(ch);
2060
2061 rsp->rsp_flags =
2062 SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
2063 }
2064 }
2065
2066 spin_unlock_irq(&sdev->spinlock);
2067
2068 } else
2069 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
2070
2071 if (*(__be64 *)req->target_port_id != cpu_to_be64(srpt_service_guid)
2072 || *(__be64 *)(req->target_port_id + 8) !=
2073 cpu_to_be64(srpt_service_guid)) {
b356c1c1
VT
2074 rej->reason = cpu_to_be32(
2075 SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
a42d985b 2076 ret = -ENOMEM;
9f5d32af 2077 pr_err("rejected SRP_LOGIN_REQ because it"
a42d985b
BVA
2078 " has an invalid target port identifier.\n");
2079 goto reject;
2080 }
2081
9d2aa2b4 2082 ch = kzalloc(sizeof(*ch), GFP_KERNEL);
a42d985b 2083 if (!ch) {
b356c1c1
VT
2084 rej->reason = cpu_to_be32(
2085 SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
9f5d32af 2086 pr_err("rejected SRP_LOGIN_REQ because no memory.\n");
a42d985b
BVA
2087 ret = -ENOMEM;
2088 goto reject;
2089 }
2090
2091 INIT_WORK(&ch->release_work, srpt_release_channel_work);
2092 memcpy(ch->i_port_id, req->initiator_port_id, 16);
2093 memcpy(ch->t_port_id, req->target_port_id, 16);
2094 ch->sport = &sdev->port[param->port - 1];
2095 ch->cm_id = cm_id;
2739b592 2096 cm_id->context = ch;
a42d985b
BVA
2097 /*
2098 * Avoid QUEUE_FULL conditions by limiting the number of buffers used
2099 * for the SRP protocol to the command queue size.
2100 */
2101 ch->rq_size = SRPT_RQ_SIZE;
2102 spin_lock_init(&ch->spinlock);
2103 ch->state = CH_CONNECTING;
2104 INIT_LIST_HEAD(&ch->cmd_wait_list);
2105 ch->rsp_size = ch->sport->port_attrib.srp_max_rsp_size;
2106
2107 ch->ioctx_ring = (struct srpt_send_ioctx **)
2108 srpt_alloc_ioctx_ring(ch->sport->sdev, ch->rq_size,
2109 sizeof(*ch->ioctx_ring[0]),
2110 ch->rsp_size, DMA_TO_DEVICE);
2111 if (!ch->ioctx_ring)
2112 goto free_ch;
2113
2114 INIT_LIST_HEAD(&ch->free_list);
2115 for (i = 0; i < ch->rq_size; i++) {
2116 ch->ioctx_ring[i]->ch = ch;
2117 list_add_tail(&ch->ioctx_ring[i]->free_list, &ch->free_list);
2118 }
2119
2120 ret = srpt_create_ch_ib(ch);
2121 if (ret) {
b356c1c1
VT
2122 rej->reason = cpu_to_be32(
2123 SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
9f5d32af 2124 pr_err("rejected SRP_LOGIN_REQ because creating"
a42d985b
BVA
2125 " a new RDMA channel failed.\n");
2126 goto free_ring;
2127 }
2128
2129 ret = srpt_ch_qp_rtr(ch, ch->qp);
2130 if (ret) {
b356c1c1 2131 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
9f5d32af 2132 pr_err("rejected SRP_LOGIN_REQ because enabling"
a42d985b
BVA
2133 " RTR failed (error code = %d)\n", ret);
2134 goto destroy_ib;
2135 }
f246c941 2136
a42d985b 2137 /*
f246c941
NB
2138 * Use the initator port identifier as the session name, when
2139 * checking against se_node_acl->initiatorname[] this can be
2140 * with or without preceeding '0x'.
a42d985b
BVA
2141 */
2142 snprintf(ch->sess_name, sizeof(ch->sess_name), "0x%016llx%016llx",
2143 be64_to_cpu(*(__be64 *)ch->i_port_id),
2144 be64_to_cpu(*(__be64 *)(ch->i_port_id + 8)));
2145
2146 pr_debug("registering session %s\n", ch->sess_name);
f246c941 2147 p = &ch->sess_name[0];
a42d985b 2148
f246c941
NB
2149 ch->sess = transport_init_session(TARGET_PROT_NORMAL);
2150 if (IS_ERR(ch->sess)) {
b356c1c1 2151 rej->reason = cpu_to_be32(
f246c941
NB
2152 SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2153 pr_debug("Failed to create session\n");
a42d985b
BVA
2154 goto destroy_ib;
2155 }
2156
f246c941
NB
2157try_again:
2158 se_acl = core_tpg_get_initiator_node_acl(&sport->port_tpg_1, p);
2159 if (!se_acl) {
2160 pr_info("Rejected login because no ACL has been"
2161 " configured yet for initiator %s.\n", ch->sess_name);
2162 /*
2163 * XXX: Hack to retry of ch->i_port_id without leading '0x'
2164 */
2165 if (p == &ch->sess_name[0]) {
2166 p += 2;
2167 goto try_again;
2168 }
b356c1c1 2169 rej->reason = cpu_to_be32(
f246c941
NB
2170 SRP_LOGIN_REJ_CHANNEL_LIMIT_REACHED);
2171 transport_free_session(ch->sess);
2172 goto destroy_ib;
a42d985b 2173 }
f246c941
NB
2174 ch->sess->se_node_acl = se_acl;
2175
2176 transport_register_session(&sport->port_tpg_1, se_acl, ch->sess, ch);
a42d985b
BVA
2177
2178 pr_debug("Establish connection sess=%p name=%s cm_id=%p\n", ch->sess,
2179 ch->sess_name, ch->cm_id);
2180
2181 /* create srp_login_response */
2182 rsp->opcode = SRP_LOGIN_RSP;
2183 rsp->tag = req->tag;
2184 rsp->max_it_iu_len = req->req_it_iu_len;
2185 rsp->max_ti_iu_len = req->req_it_iu_len;
2186 ch->max_ti_iu_len = it_iu_len;
b356c1c1
VT
2187 rsp->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT
2188 | SRP_BUF_FORMAT_INDIRECT);
a42d985b
BVA
2189 rsp->req_lim_delta = cpu_to_be32(ch->rq_size);
2190 atomic_set(&ch->req_lim, ch->rq_size);
2191 atomic_set(&ch->req_lim_delta, 0);
2192
2193 /* create cm reply */
2194 rep_param->qp_num = ch->qp->qp_num;
2195 rep_param->private_data = (void *)rsp;
9d2aa2b4 2196 rep_param->private_data_len = sizeof(*rsp);
a42d985b
BVA
2197 rep_param->rnr_retry_count = 7;
2198 rep_param->flow_control = 1;
2199 rep_param->failover_accepted = 0;
2200 rep_param->srq = 1;
2201 rep_param->responder_resources = 4;
2202 rep_param->initiator_depth = 4;
2203
2204 ret = ib_send_cm_rep(cm_id, rep_param);
2205 if (ret) {
9f5d32af 2206 pr_err("sending SRP_LOGIN_REQ response failed"
a42d985b
BVA
2207 " (error code = %d)\n", ret);
2208 goto release_channel;
2209 }
2210
2211 spin_lock_irq(&sdev->spinlock);
2212 list_add_tail(&ch->list, &sdev->rch_list);
2213 spin_unlock_irq(&sdev->spinlock);
2214
2215 goto out;
2216
2217release_channel:
2218 srpt_set_ch_state(ch, CH_RELEASING);
2219 transport_deregister_session_configfs(ch->sess);
a42d985b
BVA
2220 transport_deregister_session(ch->sess);
2221 ch->sess = NULL;
2222
2223destroy_ib:
2224 srpt_destroy_ch_ib(ch);
2225
2226free_ring:
2227 srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring,
2228 ch->sport->sdev, ch->rq_size,
2229 ch->rsp_size, DMA_TO_DEVICE);
2230free_ch:
2231 kfree(ch);
2232
2233reject:
2234 rej->opcode = SRP_LOGIN_REJ;
2235 rej->tag = req->tag;
b356c1c1
VT
2236 rej->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT
2237 | SRP_BUF_FORMAT_INDIRECT);
a42d985b
BVA
2238
2239 ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
9d2aa2b4 2240 (void *)rej, sizeof(*rej));
a42d985b
BVA
2241
2242out:
2243 kfree(rep_param);
2244 kfree(rsp);
2245 kfree(rej);
2246
2247 return ret;
2248}
2249
2739b592
BVA
2250static void srpt_cm_rej_recv(struct srpt_rdma_ch *ch,
2251 enum ib_cm_rej_reason reason,
2252 const u8 *private_data,
2253 u8 private_data_len)
a42d985b 2254{
2739b592
BVA
2255 pr_info("Received CM REJ for ch %s-%d; reason %d.\n",
2256 ch->sess_name, ch->qp->qp_num, reason);
2257 srpt_drain_channel(ch);
a42d985b
BVA
2258}
2259
2260/**
2261 * srpt_cm_rtu_recv() - Process an IB_CM_RTU_RECEIVED or USER_ESTABLISHED event.
2262 *
2263 * An IB_CM_RTU_RECEIVED message indicates that the connection is established
2264 * and that the recipient may begin transmitting (RTU = ready to use).
2265 */
2739b592 2266static void srpt_cm_rtu_recv(struct srpt_rdma_ch *ch)
a42d985b 2267{
a42d985b
BVA
2268 int ret;
2269
f130c220 2270 if (srpt_set_ch_state(ch, CH_LIVE)) {
a42d985b
BVA
2271 struct srpt_recv_ioctx *ioctx, *ioctx_tmp;
2272
2273 ret = srpt_ch_qp_rts(ch, ch->qp);
2274
2275 list_for_each_entry_safe(ioctx, ioctx_tmp, &ch->cmd_wait_list,
2276 wait_list) {
2277 list_del(&ioctx->wait_list);
2278 srpt_handle_new_iu(ch, ioctx, NULL);
2279 }
2280 if (ret)
2281 srpt_close_ch(ch);
2282 }
2283}
2284
a42d985b
BVA
2285/**
2286 * srpt_cm_dreq_recv() - Process reception of a DREQ message.
2287 */
2739b592 2288static void srpt_cm_dreq_recv(struct srpt_rdma_ch *ch)
a42d985b 2289{
a42d985b
BVA
2290 unsigned long flags;
2291 bool send_drep = false;
2292
2739b592
BVA
2293 pr_debug("ch %s-%d state %d\n", ch->sess_name, ch->qp->qp_num,
2294 ch->state);
a42d985b
BVA
2295
2296 spin_lock_irqsave(&ch->spinlock, flags);
2297 switch (ch->state) {
2298 case CH_CONNECTING:
2299 case CH_LIVE:
2300 send_drep = true;
2301 ch->state = CH_DISCONNECTING;
2302 break;
2303 case CH_DISCONNECTING:
2304 case CH_DRAINING:
2305 case CH_RELEASING:
2306 WARN(true, "unexpected channel state %d\n", ch->state);
2307 break;
2308 }
2309 spin_unlock_irqrestore(&ch->spinlock, flags);
2310
2311 if (send_drep) {
2312 if (ib_send_cm_drep(ch->cm_id, NULL, 0) < 0)
9f5d32af
DL
2313 pr_err("Sending IB DREP failed.\n");
2314 pr_info("Received DREQ and sent DREP for session %s.\n",
2315 ch->sess_name);
a42d985b
BVA
2316 }
2317}
2318
a42d985b
BVA
2319/**
2320 * srpt_cm_handler() - IB connection manager callback function.
2321 *
2322 * A non-zero return value will cause the caller destroy the CM ID.
2323 *
2324 * Note: srpt_cm_handler() must only return a non-zero value when transferring
2325 * ownership of the cm_id to a channel by srpt_cm_req_recv() failed. Returning
2326 * a non-zero value in any other case will trigger a race with the
2327 * ib_destroy_cm_id() call in srpt_release_channel().
2328 */
2329static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2330{
2739b592 2331 struct srpt_rdma_ch *ch = cm_id->context;
a42d985b
BVA
2332 int ret;
2333
2334 ret = 0;
2335 switch (event->event) {
2336 case IB_CM_REQ_RECEIVED:
2337 ret = srpt_cm_req_recv(cm_id, &event->param.req_rcvd,
2338 event->private_data);
2339 break;
2340 case IB_CM_REJ_RECEIVED:
2739b592
BVA
2341 srpt_cm_rej_recv(ch, event->param.rej_rcvd.reason,
2342 event->private_data,
2343 IB_CM_REJ_PRIVATE_DATA_SIZE);
a42d985b
BVA
2344 break;
2345 case IB_CM_RTU_RECEIVED:
2346 case IB_CM_USER_ESTABLISHED:
2739b592 2347 srpt_cm_rtu_recv(ch);
a42d985b
BVA
2348 break;
2349 case IB_CM_DREQ_RECEIVED:
2739b592 2350 srpt_cm_dreq_recv(ch);
a42d985b
BVA
2351 break;
2352 case IB_CM_DREP_RECEIVED:
2739b592
BVA
2353 pr_info("Received CM DREP message for ch %s-%d.\n",
2354 ch->sess_name, ch->qp->qp_num);
2355 srpt_drain_channel(ch);
a42d985b
BVA
2356 break;
2357 case IB_CM_TIMEWAIT_EXIT:
2739b592
BVA
2358 pr_info("Received CM TimeWait exit for ch %s-%d.\n",
2359 ch->sess_name, ch->qp->qp_num);
2360 srpt_drain_channel(ch);
a42d985b
BVA
2361 break;
2362 case IB_CM_REP_ERROR:
2739b592
BVA
2363 pr_info("Received CM REP error for ch %s-%d.\n", ch->sess_name,
2364 ch->qp->qp_num);
2365 srpt_drain_channel(ch);
a42d985b
BVA
2366 break;
2367 case IB_CM_DREQ_ERROR:
1e20a2a5 2368 pr_info("Received CM DREQ ERROR event.\n");
a42d985b
BVA
2369 break;
2370 case IB_CM_MRA_RECEIVED:
1e20a2a5 2371 pr_info("Received CM MRA event\n");
a42d985b
BVA
2372 break;
2373 default:
1e20a2a5 2374 pr_err("received unrecognized CM event %d\n", event->event);
a42d985b
BVA
2375 break;
2376 }
2377
2378 return ret;
2379}
2380
2381/**
2382 * srpt_perform_rdmas() - Perform IB RDMA.
2383 *
2384 * Returns zero upon success or a negative number upon failure.
2385 */
2386static int srpt_perform_rdmas(struct srpt_rdma_ch *ch,
2387 struct srpt_send_ioctx *ioctx)
2388{
a42d985b 2389 struct ib_send_wr *bad_wr;
59fae4de 2390 int sq_wr_avail, ret, i;
a42d985b
BVA
2391 enum dma_data_direction dir;
2392 const int n_rdma = ioctx->n_rdma;
2393
2394 dir = ioctx->cmd.data_direction;
2395 if (dir == DMA_TO_DEVICE) {
2396 /* write */
2397 ret = -ENOMEM;
2398 sq_wr_avail = atomic_sub_return(n_rdma, &ch->sq_wr_avail);
2399 if (sq_wr_avail < 0) {
9f5d32af
DL
2400 pr_warn("IB send queue full (needed %d)\n",
2401 n_rdma);
a42d985b
BVA
2402 goto out;
2403 }
2404 }
2405
59fae4de
CH
2406 for (i = 0; i < n_rdma; i++) {
2407 struct ib_send_wr *wr = &ioctx->rdma_wrs[i].wr;
a42d985b 2408
59fae4de
CH
2409 wr->opcode = (dir == DMA_FROM_DEVICE) ?
2410 IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;
a42d985b 2411
59fae4de
CH
2412 if (i == n_rdma - 1) {
2413 /* only get completion event for the last rdma read */
2414 if (dir == DMA_TO_DEVICE) {
2415 wr->send_flags = IB_SEND_SIGNALED;
2416 ioctx->rdma_cqe.done = srpt_rdma_read_done;
2417 } else {
2418 ioctx->rdma_cqe.done = srpt_rdma_write_done;
2419 }
2420 wr->wr_cqe = &ioctx->rdma_cqe;
2421 wr->next = NULL;
2422 } else {
2423 wr->wr_cqe = NULL;
2424 wr->next = &ioctx->rdma_wrs[i + 1].wr;
2425 }
a42d985b
BVA
2426 }
2427
59fae4de 2428 ret = ib_post_send(ch->qp, &ioctx->rdma_wrs->wr, &bad_wr);
a42d985b 2429 if (ret)
9f5d32af 2430 pr_err("%s[%d]: ib_post_send() returned %d for %d/%d\n",
a42d985b 2431 __func__, __LINE__, ret, i, n_rdma);
a42d985b
BVA
2432out:
2433 if (unlikely(dir == DMA_TO_DEVICE && ret < 0))
2434 atomic_add(n_rdma, &ch->sq_wr_avail);
2435 return ret;
2436}
2437
2438/**
2439 * srpt_xfer_data() - Start data transfer from initiator to target.
2440 */
2441static int srpt_xfer_data(struct srpt_rdma_ch *ch,
2442 struct srpt_send_ioctx *ioctx)
2443{
2444 int ret;
2445
2446 ret = srpt_map_sg_to_ib_sge(ch, ioctx);
2447 if (ret) {
9f5d32af 2448 pr_err("%s[%d] ret=%d\n", __func__, __LINE__, ret);
a42d985b
BVA
2449 goto out;
2450 }
2451
2452 ret = srpt_perform_rdmas(ch, ioctx);
2453 if (ret) {
2454 if (ret == -EAGAIN || ret == -ENOMEM)
9f5d32af
DL
2455 pr_info("%s[%d] queue full -- ret=%d\n",
2456 __func__, __LINE__, ret);
a42d985b 2457 else
9f5d32af 2458 pr_err("%s[%d] fatal error -- ret=%d\n",
a42d985b
BVA
2459 __func__, __LINE__, ret);
2460 goto out_unmap;
2461 }
2462
2463out:
2464 return ret;
2465out_unmap:
2466 srpt_unmap_sg_to_ib_sge(ch, ioctx);
2467 goto out;
2468}
2469
2470static int srpt_write_pending_status(struct se_cmd *se_cmd)
2471{
2472 struct srpt_send_ioctx *ioctx;
2473
2474 ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);
2475 return srpt_get_cmd_state(ioctx) == SRPT_STATE_NEED_DATA;
2476}
2477
2478/*
2479 * srpt_write_pending() - Start data transfer from initiator to target (write).
2480 */
2481static int srpt_write_pending(struct se_cmd *se_cmd)
2482{
2483 struct srpt_rdma_ch *ch;
2484 struct srpt_send_ioctx *ioctx;
2485 enum srpt_command_state new_state;
a42d985b
BVA
2486 int ret;
2487
2488 ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);
2489
2490 new_state = srpt_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA);
2491 WARN_ON(new_state == SRPT_STATE_DONE);
2492
2493 ch = ioctx->ch;
2494 BUG_ON(!ch);
2495
33912d73 2496 switch (ch->state) {
a42d985b 2497 case CH_CONNECTING:
33912d73 2498 WARN(true, "unexpected channel state %d\n", ch->state);
a42d985b
BVA
2499 ret = -EINVAL;
2500 goto out;
2501 case CH_LIVE:
2502 break;
2503 case CH_DISCONNECTING:
2504 case CH_DRAINING:
2505 case CH_RELEASING:
2506 pr_debug("cmd with tag %lld: channel disconnecting\n",
649ee054 2507 ioctx->cmd.tag);
a42d985b
BVA
2508 srpt_set_cmd_state(ioctx, SRPT_STATE_DATA_IN);
2509 ret = -EINVAL;
2510 goto out;
2511 }
2512 ret = srpt_xfer_data(ch, ioctx);
2513
2514out:
2515 return ret;
2516}
2517
2518static u8 tcm_to_srp_tsk_mgmt_status(const int tcm_mgmt_status)
2519{
2520 switch (tcm_mgmt_status) {
2521 case TMR_FUNCTION_COMPLETE:
2522 return SRP_TSK_MGMT_SUCCESS;
2523 case TMR_FUNCTION_REJECTED:
2524 return SRP_TSK_MGMT_FUNC_NOT_SUPP;
2525 }
2526 return SRP_TSK_MGMT_FAILED;
2527}
2528
2529/**
2530 * srpt_queue_response() - Transmits the response to a SCSI command.
2531 *
2532 * Callback function called by the TCM core. Must not block since it can be
2533 * invoked on the context of the IB completion handler.
2534 */
b79fafac 2535static void srpt_queue_response(struct se_cmd *cmd)
a42d985b
BVA
2536{
2537 struct srpt_rdma_ch *ch;
2538 struct srpt_send_ioctx *ioctx;
2539 enum srpt_command_state state;
2540 unsigned long flags;
2541 int ret;
2542 enum dma_data_direction dir;
2543 int resp_len;
2544 u8 srp_tm_status;
2545
a42d985b
BVA
2546 ioctx = container_of(cmd, struct srpt_send_ioctx, cmd);
2547 ch = ioctx->ch;
2548 BUG_ON(!ch);
2549
2550 spin_lock_irqsave(&ioctx->spinlock, flags);
2551 state = ioctx->state;
2552 switch (state) {
2553 case SRPT_STATE_NEW:
2554 case SRPT_STATE_DATA_IN:
2555 ioctx->state = SRPT_STATE_CMD_RSP_SENT;
2556 break;
2557 case SRPT_STATE_MGMT:
2558 ioctx->state = SRPT_STATE_MGMT_RSP_SENT;
2559 break;
2560 default:
2561 WARN(true, "ch %p; cmd %d: unexpected command state %d\n",
2562 ch, ioctx->ioctx.index, ioctx->state);
2563 break;
2564 }
2565 spin_unlock_irqrestore(&ioctx->spinlock, flags);
2566
2567 if (unlikely(transport_check_aborted_status(&ioctx->cmd, false)
2568 || WARN_ON_ONCE(state == SRPT_STATE_CMD_RSP_SENT))) {
2569 atomic_inc(&ch->req_lim_delta);
2570 srpt_abort_cmd(ioctx);
b79fafac 2571 return;
a42d985b
BVA
2572 }
2573
2574 dir = ioctx->cmd.data_direction;
2575
2576 /* For read commands, transfer the data to the initiator. */
2577 if (dir == DMA_FROM_DEVICE && ioctx->cmd.data_length &&
2578 !ioctx->queue_status_only) {
2579 ret = srpt_xfer_data(ch, ioctx);
2580 if (ret) {
9f5d32af 2581 pr_err("xfer_data failed for tag %llu\n",
649ee054 2582 ioctx->cmd.tag);
b79fafac 2583 return;
a42d985b
BVA
2584 }
2585 }
2586
2587 if (state != SRPT_STATE_MGMT)
649ee054 2588 resp_len = srpt_build_cmd_rsp(ch, ioctx, ioctx->cmd.tag,
a42d985b
BVA
2589 cmd->scsi_status);
2590 else {
2591 srp_tm_status
2592 = tcm_to_srp_tsk_mgmt_status(cmd->se_tmr_req->response);
2593 resp_len = srpt_build_tskmgmt_rsp(ch, ioctx, srp_tm_status,
649ee054 2594 ioctx->cmd.tag);
a42d985b
BVA
2595 }
2596 ret = srpt_post_send(ch, ioctx, resp_len);
2597 if (ret) {
9f5d32af 2598 pr_err("sending cmd response failed for tag %llu\n",
649ee054 2599 ioctx->cmd.tag);
a42d985b
BVA
2600 srpt_unmap_sg_to_ib_sge(ch, ioctx);
2601 srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);
afc16604 2602 target_put_sess_cmd(&ioctx->cmd);
a42d985b 2603 }
b79fafac 2604}
a42d985b 2605
b79fafac
JE
2606static int srpt_queue_data_in(struct se_cmd *cmd)
2607{
2608 srpt_queue_response(cmd);
2609 return 0;
2610}
2611
2612static void srpt_queue_tm_rsp(struct se_cmd *cmd)
2613{
2614 srpt_queue_response(cmd);
a42d985b
BVA
2615}
2616
131e6abc
NB
2617static void srpt_aborted_task(struct se_cmd *cmd)
2618{
2619 struct srpt_send_ioctx *ioctx = container_of(cmd,
2620 struct srpt_send_ioctx, cmd);
2621
2622 srpt_unmap_sg_to_ib_sge(ioctx->ch, ioctx);
2623}
2624
a42d985b
BVA
2625static int srpt_queue_status(struct se_cmd *cmd)
2626{
2627 struct srpt_send_ioctx *ioctx;
2628
2629 ioctx = container_of(cmd, struct srpt_send_ioctx, cmd);
2630 BUG_ON(ioctx->sense_data != cmd->sense_buffer);
2631 if (cmd->se_cmd_flags &
2632 (SCF_TRANSPORT_TASK_SENSE | SCF_EMULATED_TASK_SENSE))
2633 WARN_ON(cmd->scsi_status != SAM_STAT_CHECK_CONDITION);
2634 ioctx->queue_status_only = true;
b79fafac
JE
2635 srpt_queue_response(cmd);
2636 return 0;
a42d985b
BVA
2637}
2638
2639static void srpt_refresh_port_work(struct work_struct *work)
2640{
2641 struct srpt_port *sport = container_of(work, struct srpt_port, work);
2642
2643 srpt_refresh_port(sport);
2644}
2645
2646static int srpt_ch_list_empty(struct srpt_device *sdev)
2647{
2648 int res;
2649
2650 spin_lock_irq(&sdev->spinlock);
2651 res = list_empty(&sdev->rch_list);
2652 spin_unlock_irq(&sdev->spinlock);
2653
2654 return res;
2655}
2656
2657/**
2658 * srpt_release_sdev() - Free the channel resources associated with a target.
2659 */
2660static int srpt_release_sdev(struct srpt_device *sdev)
2661{
2662 struct srpt_rdma_ch *ch, *tmp_ch;
2663 int res;
2664
2665 WARN_ON_ONCE(irqs_disabled());
2666
2667 BUG_ON(!sdev);
2668
2669 spin_lock_irq(&sdev->spinlock);
2670 list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list)
2671 __srpt_close_ch(ch);
2672 spin_unlock_irq(&sdev->spinlock);
2673
2674 res = wait_event_interruptible(sdev->ch_releaseQ,
2675 srpt_ch_list_empty(sdev));
2676 if (res)
9f5d32af 2677 pr_err("%s: interrupted.\n", __func__);
a42d985b
BVA
2678
2679 return 0;
2680}
2681
2682static struct srpt_port *__srpt_lookup_port(const char *name)
2683{
2684 struct ib_device *dev;
2685 struct srpt_device *sdev;
2686 struct srpt_port *sport;
2687 int i;
2688
2689 list_for_each_entry(sdev, &srpt_dev_list, list) {
2690 dev = sdev->device;
2691 if (!dev)
2692 continue;
2693
2694 for (i = 0; i < dev->phys_port_cnt; i++) {
2695 sport = &sdev->port[i];
2696
2697 if (!strcmp(sport->port_guid, name))
2698 return sport;
2699 }
2700 }
2701
2702 return NULL;
2703}
2704
2705static struct srpt_port *srpt_lookup_port(const char *name)
2706{
2707 struct srpt_port *sport;
2708
2709 spin_lock(&srpt_dev_lock);
2710 sport = __srpt_lookup_port(name);
2711 spin_unlock(&srpt_dev_lock);
2712
2713 return sport;
2714}
2715
2716/**
2717 * srpt_add_one() - Infiniband device addition callback function.
2718 */
2719static void srpt_add_one(struct ib_device *device)
2720{
2721 struct srpt_device *sdev;
2722 struct srpt_port *sport;
2723 struct ib_srq_init_attr srq_attr;
2724 int i;
2725
2726 pr_debug("device = %p, device->dma_ops = %p\n", device,
2727 device->dma_ops);
2728
9d2aa2b4 2729 sdev = kzalloc(sizeof(*sdev), GFP_KERNEL);
a42d985b
BVA
2730 if (!sdev)
2731 goto err;
2732
2733 sdev->device = device;
2734 INIT_LIST_HEAD(&sdev->rch_list);
2735 init_waitqueue_head(&sdev->ch_releaseQ);
2736 spin_lock_init(&sdev->spinlock);
2737
a42d985b
BVA
2738 sdev->pd = ib_alloc_pd(device);
2739 if (IS_ERR(sdev->pd))
2740 goto free_dev;
2741
4a061b28 2742 sdev->srq_size = min(srpt_srq_size, sdev->device->attrs.max_srq_wr);
a42d985b
BVA
2743
2744 srq_attr.event_handler = srpt_srq_event;
2745 srq_attr.srq_context = (void *)sdev;
2746 srq_attr.attr.max_wr = sdev->srq_size;
2747 srq_attr.attr.max_sge = 1;
2748 srq_attr.attr.srq_limit = 0;
6f360336 2749 srq_attr.srq_type = IB_SRQT_BASIC;
a42d985b
BVA
2750
2751 sdev->srq = ib_create_srq(sdev->pd, &srq_attr);
2752 if (IS_ERR(sdev->srq))
5a783956 2753 goto err_pd;
a42d985b
BVA
2754
2755 pr_debug("%s: create SRQ #wr= %d max_allow=%d dev= %s\n",
4a061b28 2756 __func__, sdev->srq_size, sdev->device->attrs.max_srq_wr,
a42d985b
BVA
2757 device->name);
2758
2759 if (!srpt_service_guid)
2760 srpt_service_guid = be64_to_cpu(device->node_guid);
2761
2762 sdev->cm_id = ib_create_cm_id(device, srpt_cm_handler, sdev);
2763 if (IS_ERR(sdev->cm_id))
2764 goto err_srq;
2765
2766 /* print out target login information */
2767 pr_debug("Target login info: id_ext=%016llx,ioc_guid=%016llx,"
2768 "pkey=ffff,service_id=%016llx\n", srpt_service_guid,
2769 srpt_service_guid, srpt_service_guid);
2770
2771 /*
2772 * We do not have a consistent service_id (ie. also id_ext of target_id)
2773 * to identify this target. We currently use the guid of the first HCA
2774 * in the system as service_id; therefore, the target_id will change
2775 * if this HCA is gone bad and replaced by different HCA
2776 */
73fec7fd 2777 if (ib_cm_listen(sdev->cm_id, cpu_to_be64(srpt_service_guid), 0))
a42d985b
BVA
2778 goto err_cm;
2779
2780 INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device,
2781 srpt_event_handler);
2782 if (ib_register_event_handler(&sdev->event_handler))
2783 goto err_cm;
2784
2785 sdev->ioctx_ring = (struct srpt_recv_ioctx **)
2786 srpt_alloc_ioctx_ring(sdev, sdev->srq_size,
2787 sizeof(*sdev->ioctx_ring[0]),
2788 srp_max_req_size, DMA_FROM_DEVICE);
2789 if (!sdev->ioctx_ring)
2790 goto err_event;
2791
2792 for (i = 0; i < sdev->srq_size; ++i)
2793 srpt_post_recv(sdev, sdev->ioctx_ring[i]);
2794
f225066b 2795 WARN_ON(sdev->device->phys_port_cnt > ARRAY_SIZE(sdev->port));
a42d985b
BVA
2796
2797 for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
2798 sport = &sdev->port[i - 1];
2799 sport->sdev = sdev;
2800 sport->port = i;
2801 sport->port_attrib.srp_max_rdma_size = DEFAULT_MAX_RDMA_SIZE;
2802 sport->port_attrib.srp_max_rsp_size = DEFAULT_MAX_RSP_SIZE;
2803 sport->port_attrib.srp_sq_size = DEF_SRPT_SQ_SIZE;
2804 INIT_WORK(&sport->work, srpt_refresh_port_work);
a42d985b
BVA
2805
2806 if (srpt_refresh_port(sport)) {
9f5d32af 2807 pr_err("MAD registration failed for %s-%d.\n",
f68cba4e 2808 sdev->device->name, i);
a42d985b
BVA
2809 goto err_ring;
2810 }
2811 snprintf(sport->port_guid, sizeof(sport->port_guid),
2812 "0x%016llx%016llx",
2813 be64_to_cpu(sport->gid.global.subnet_prefix),
2814 be64_to_cpu(sport->gid.global.interface_id));
2815 }
2816
2817 spin_lock(&srpt_dev_lock);
2818 list_add_tail(&sdev->list, &srpt_dev_list);
2819 spin_unlock(&srpt_dev_lock);
2820
2821out:
2822 ib_set_client_data(device, &srpt_client, sdev);
2823 pr_debug("added %s.\n", device->name);
2824 return;
2825
2826err_ring:
2827 srpt_free_ioctx_ring((struct srpt_ioctx **)sdev->ioctx_ring, sdev,
2828 sdev->srq_size, srp_max_req_size,
2829 DMA_FROM_DEVICE);
2830err_event:
2831 ib_unregister_event_handler(&sdev->event_handler);
2832err_cm:
2833 ib_destroy_cm_id(sdev->cm_id);
2834err_srq:
2835 ib_destroy_srq(sdev->srq);
a42d985b
BVA
2836err_pd:
2837 ib_dealloc_pd(sdev->pd);
2838free_dev:
2839 kfree(sdev);
2840err:
2841 sdev = NULL;
9f5d32af 2842 pr_info("%s(%s) failed.\n", __func__, device->name);
a42d985b
BVA
2843 goto out;
2844}
2845
2846/**
2847 * srpt_remove_one() - InfiniBand device removal callback function.
2848 */
7c1eb45a 2849static void srpt_remove_one(struct ib_device *device, void *client_data)
a42d985b 2850{
7c1eb45a 2851 struct srpt_device *sdev = client_data;
a42d985b
BVA
2852 int i;
2853
a42d985b 2854 if (!sdev) {
9f5d32af 2855 pr_info("%s(%s): nothing to do.\n", __func__, device->name);
a42d985b
BVA
2856 return;
2857 }
2858
2859 srpt_unregister_mad_agent(sdev);
2860
2861 ib_unregister_event_handler(&sdev->event_handler);
2862
2863 /* Cancel any work queued by the just unregistered IB event handler. */
2864 for (i = 0; i < sdev->device->phys_port_cnt; i++)
2865 cancel_work_sync(&sdev->port[i].work);
2866
2867 ib_destroy_cm_id(sdev->cm_id);
2868
2869 /*
2870 * Unregistering a target must happen after destroying sdev->cm_id
2871 * such that no new SRP_LOGIN_REQ information units can arrive while
2872 * destroying the target.
2873 */
2874 spin_lock(&srpt_dev_lock);
2875 list_del(&sdev->list);
2876 spin_unlock(&srpt_dev_lock);
2877 srpt_release_sdev(sdev);
2878
2879 ib_destroy_srq(sdev->srq);
a42d985b
BVA
2880 ib_dealloc_pd(sdev->pd);
2881
2882 srpt_free_ioctx_ring((struct srpt_ioctx **)sdev->ioctx_ring, sdev,
2883 sdev->srq_size, srp_max_req_size, DMA_FROM_DEVICE);
2884 sdev->ioctx_ring = NULL;
2885 kfree(sdev);
2886}
2887
2888static struct ib_client srpt_client = {
2889 .name = DRV_NAME,
2890 .add = srpt_add_one,
2891 .remove = srpt_remove_one
2892};
2893
2894static int srpt_check_true(struct se_portal_group *se_tpg)
2895{
2896 return 1;
2897}
2898
2899static int srpt_check_false(struct se_portal_group *se_tpg)
2900{
2901 return 0;
2902}
2903
2904static char *srpt_get_fabric_name(void)
2905{
2906 return "srpt";
2907}
2908
a42d985b
BVA
2909static char *srpt_get_fabric_wwn(struct se_portal_group *tpg)
2910{
2911 struct srpt_port *sport = container_of(tpg, struct srpt_port, port_tpg_1);
2912
2913 return sport->port_guid;
2914}
2915
2916static u16 srpt_get_tag(struct se_portal_group *tpg)
2917{
2918 return 1;
2919}
2920
a42d985b
BVA
2921static u32 srpt_tpg_get_inst_index(struct se_portal_group *se_tpg)
2922{
2923 return 1;
2924}
2925
2926static void srpt_release_cmd(struct se_cmd *se_cmd)
2927{
9474b043
NB
2928 struct srpt_send_ioctx *ioctx = container_of(se_cmd,
2929 struct srpt_send_ioctx, cmd);
2930 struct srpt_rdma_ch *ch = ioctx->ch;
2931 unsigned long flags;
2932
2933 WARN_ON(ioctx->state != SRPT_STATE_DONE);
2934 WARN_ON(ioctx->mapped_sg_count != 0);
2935
2936 if (ioctx->n_rbuf > 1) {
2937 kfree(ioctx->rbufs);
2938 ioctx->rbufs = NULL;
2939 ioctx->n_rbuf = 0;
2940 }
2941
2942 spin_lock_irqsave(&ch->spinlock, flags);
2943 list_add(&ioctx->free_list, &ch->free_list);
2944 spin_unlock_irqrestore(&ch->spinlock, flags);
a42d985b
BVA
2945}
2946
a42d985b
BVA
2947/**
2948 * srpt_close_session() - Forcibly close a session.
2949 *
2950 * Callback function invoked by the TCM core to clean up sessions associated
2951 * with a node ACL when the user invokes
2952 * rmdir /sys/kernel/config/target/$driver/$port/$tpg/acls/$i_port_id
2953 */
2954static void srpt_close_session(struct se_session *se_sess)
2955{
2956 DECLARE_COMPLETION_ONSTACK(release_done);
f108f0f6
BVA
2957 struct srpt_rdma_ch *ch = se_sess->fabric_sess_ptr;
2958 struct srpt_device *sdev = ch->sport->sdev;
2959 bool wait;
a42d985b 2960
f108f0f6
BVA
2961 pr_debug("ch %s-%d state %d\n", ch->sess_name, ch->qp->qp_num,
2962 ch->state);
a42d985b 2963
a42d985b
BVA
2964 spin_lock_irq(&sdev->spinlock);
2965 BUG_ON(ch->release_done);
2966 ch->release_done = &release_done;
f108f0f6 2967 wait = !list_empty(&ch->list);
a42d985b
BVA
2968 __srpt_close_ch(ch);
2969 spin_unlock_irq(&sdev->spinlock);
2970
f108f0f6
BVA
2971 if (!wait)
2972 return;
2973
2974 while (wait_for_completion_timeout(&release_done, 180 * HZ) == 0)
2975 pr_info("%s(%s-%d state %d): still waiting ...\n", __func__,
2976 ch->sess_name, ch->qp->qp_num, ch->state);
a42d985b
BVA
2977}
2978
a42d985b
BVA
2979/**
2980 * srpt_sess_get_index() - Return the value of scsiAttIntrPortIndex (SCSI-MIB).
2981 *
2982 * A quote from RFC 4455 (SCSI-MIB) about this MIB object:
2983 * This object represents an arbitrary integer used to uniquely identify a
2984 * particular attached remote initiator port to a particular SCSI target port
2985 * within a particular SCSI target device within a particular SCSI instance.
2986 */
2987static u32 srpt_sess_get_index(struct se_session *se_sess)
2988{
2989 return 0;
2990}
2991
2992static void srpt_set_default_node_attrs(struct se_node_acl *nacl)
2993{
2994}
2995
a42d985b
BVA
2996/* Note: only used from inside debug printk's by the TCM core. */
2997static int srpt_get_tcm_cmd_state(struct se_cmd *se_cmd)
2998{
2999 struct srpt_send_ioctx *ioctx;
3000
3001 ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);
3002 return srpt_get_cmd_state(ioctx);
3003}
3004
a42d985b
BVA
3005/**
3006 * srpt_parse_i_port_id() - Parse an initiator port ID.
3007 * @name: ASCII representation of a 128-bit initiator port ID.
3008 * @i_port_id: Binary 128-bit port ID.
3009 */
3010static int srpt_parse_i_port_id(u8 i_port_id[16], const char *name)
3011{
3012 const char *p;
3013 unsigned len, count, leading_zero_bytes;
3014 int ret, rc;
3015
3016 p = name;
b60459f0 3017 if (strncasecmp(p, "0x", 2) == 0)
a42d985b
BVA
3018 p += 2;
3019 ret = -EINVAL;
3020 len = strlen(p);
3021 if (len % 2)
3022 goto out;
3023 count = min(len / 2, 16U);
3024 leading_zero_bytes = 16 - count;
3025 memset(i_port_id, 0, leading_zero_bytes);
3026 rc = hex2bin(i_port_id + leading_zero_bytes, p, count);
3027 if (rc < 0)
3028 pr_debug("hex2bin failed for srpt_parse_i_port_id: %d\n", rc);
3029 ret = 0;
3030out:
3031 return ret;
3032}
3033
3034/*
3035 * configfs callback function invoked for
3036 * mkdir /sys/kernel/config/target/$driver/$port/$tpg/acls/$i_port_id
3037 */
c7d6a803 3038static int srpt_init_nodeacl(struct se_node_acl *se_nacl, const char *name)
a42d985b 3039{
a42d985b
BVA
3040 u8 i_port_id[16];
3041
3042 if (srpt_parse_i_port_id(i_port_id, name) < 0) {
9f5d32af 3043 pr_err("invalid initiator port ID %s\n", name);
c7d6a803 3044 return -EINVAL;
a42d985b 3045 }
c7d6a803 3046 return 0;
a42d985b
BVA
3047}
3048
2eafd729
CH
3049static ssize_t srpt_tpg_attrib_srp_max_rdma_size_show(struct config_item *item,
3050 char *page)
a42d985b 3051{
2eafd729 3052 struct se_portal_group *se_tpg = attrib_to_tpg(item);
a42d985b
BVA
3053 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
3054
3055 return sprintf(page, "%u\n", sport->port_attrib.srp_max_rdma_size);
3056}
3057
2eafd729
CH
3058static ssize_t srpt_tpg_attrib_srp_max_rdma_size_store(struct config_item *item,
3059 const char *page, size_t count)
a42d985b 3060{
2eafd729 3061 struct se_portal_group *se_tpg = attrib_to_tpg(item);
a42d985b
BVA
3062 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
3063 unsigned long val;
3064 int ret;
3065
9d8abf45 3066 ret = kstrtoul(page, 0, &val);
a42d985b 3067 if (ret < 0) {
9d8abf45 3068 pr_err("kstrtoul() failed with ret: %d\n", ret);
a42d985b
BVA
3069 return -EINVAL;
3070 }
3071 if (val > MAX_SRPT_RDMA_SIZE) {
3072 pr_err("val: %lu exceeds MAX_SRPT_RDMA_SIZE: %d\n", val,
3073 MAX_SRPT_RDMA_SIZE);
3074 return -EINVAL;
3075 }
3076 if (val < DEFAULT_MAX_RDMA_SIZE) {
3077 pr_err("val: %lu smaller than DEFAULT_MAX_RDMA_SIZE: %d\n",
3078 val, DEFAULT_MAX_RDMA_SIZE);
3079 return -EINVAL;
3080 }
3081 sport->port_attrib.srp_max_rdma_size = val;
3082
3083 return count;
3084}
3085
2eafd729
CH
3086static ssize_t srpt_tpg_attrib_srp_max_rsp_size_show(struct config_item *item,
3087 char *page)
a42d985b 3088{
2eafd729 3089 struct se_portal_group *se_tpg = attrib_to_tpg(item);
a42d985b
BVA
3090 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
3091
3092 return sprintf(page, "%u\n", sport->port_attrib.srp_max_rsp_size);
3093}
3094
2eafd729
CH
3095static ssize_t srpt_tpg_attrib_srp_max_rsp_size_store(struct config_item *item,
3096 const char *page, size_t count)
a42d985b 3097{
2eafd729 3098 struct se_portal_group *se_tpg = attrib_to_tpg(item);
a42d985b
BVA
3099 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
3100 unsigned long val;
3101 int ret;
3102
9d8abf45 3103 ret = kstrtoul(page, 0, &val);
a42d985b 3104 if (ret < 0) {
9d8abf45 3105 pr_err("kstrtoul() failed with ret: %d\n", ret);
a42d985b
BVA
3106 return -EINVAL;
3107 }
3108 if (val > MAX_SRPT_RSP_SIZE) {
3109 pr_err("val: %lu exceeds MAX_SRPT_RSP_SIZE: %d\n", val,
3110 MAX_SRPT_RSP_SIZE);
3111 return -EINVAL;
3112 }
3113 if (val < MIN_MAX_RSP_SIZE) {
3114 pr_err("val: %lu smaller than MIN_MAX_RSP_SIZE: %d\n", val,
3115 MIN_MAX_RSP_SIZE);
3116 return -EINVAL;
3117 }
3118 sport->port_attrib.srp_max_rsp_size = val;
3119
3120 return count;
3121}
3122
2eafd729
CH
3123static ssize_t srpt_tpg_attrib_srp_sq_size_show(struct config_item *item,
3124 char *page)
a42d985b 3125{
2eafd729 3126 struct se_portal_group *se_tpg = attrib_to_tpg(item);
a42d985b
BVA
3127 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
3128
3129 return sprintf(page, "%u\n", sport->port_attrib.srp_sq_size);
3130}
3131
2eafd729
CH
3132static ssize_t srpt_tpg_attrib_srp_sq_size_store(struct config_item *item,
3133 const char *page, size_t count)
a42d985b 3134{
2eafd729 3135 struct se_portal_group *se_tpg = attrib_to_tpg(item);
a42d985b
BVA
3136 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
3137 unsigned long val;
3138 int ret;
3139
9d8abf45 3140 ret = kstrtoul(page, 0, &val);
a42d985b 3141 if (ret < 0) {
9d8abf45 3142 pr_err("kstrtoul() failed with ret: %d\n", ret);
a42d985b
BVA
3143 return -EINVAL;
3144 }
3145 if (val > MAX_SRPT_SRQ_SIZE) {
3146 pr_err("val: %lu exceeds MAX_SRPT_SRQ_SIZE: %d\n", val,
3147 MAX_SRPT_SRQ_SIZE);
3148 return -EINVAL;
3149 }
3150 if (val < MIN_SRPT_SRQ_SIZE) {
3151 pr_err("val: %lu smaller than MIN_SRPT_SRQ_SIZE: %d\n", val,
3152 MIN_SRPT_SRQ_SIZE);
3153 return -EINVAL;
3154 }
3155 sport->port_attrib.srp_sq_size = val;
3156
3157 return count;
3158}
3159
2eafd729
CH
3160CONFIGFS_ATTR(srpt_tpg_attrib_, srp_max_rdma_size);
3161CONFIGFS_ATTR(srpt_tpg_attrib_, srp_max_rsp_size);
3162CONFIGFS_ATTR(srpt_tpg_attrib_, srp_sq_size);
a42d985b
BVA
3163
3164static struct configfs_attribute *srpt_tpg_attrib_attrs[] = {
2eafd729
CH
3165 &srpt_tpg_attrib_attr_srp_max_rdma_size,
3166 &srpt_tpg_attrib_attr_srp_max_rsp_size,
3167 &srpt_tpg_attrib_attr_srp_sq_size,
a42d985b
BVA
3168 NULL,
3169};
3170
2eafd729 3171static ssize_t srpt_tpg_enable_show(struct config_item *item, char *page)
a42d985b 3172{
2eafd729 3173 struct se_portal_group *se_tpg = to_tpg(item);
a42d985b
BVA
3174 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
3175
3176 return snprintf(page, PAGE_SIZE, "%d\n", (sport->enabled) ? 1: 0);
3177}
3178
2eafd729
CH
3179static ssize_t srpt_tpg_enable_store(struct config_item *item,
3180 const char *page, size_t count)
a42d985b 3181{
2eafd729 3182 struct se_portal_group *se_tpg = to_tpg(item);
a42d985b
BVA
3183 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
3184 unsigned long tmp;
3185 int ret;
3186
9d8abf45 3187 ret = kstrtoul(page, 0, &tmp);
a42d985b 3188 if (ret < 0) {
9f5d32af 3189 pr_err("Unable to extract srpt_tpg_store_enable\n");
a42d985b
BVA
3190 return -EINVAL;
3191 }
3192
3193 if ((tmp != 0) && (tmp != 1)) {
9f5d32af 3194 pr_err("Illegal value for srpt_tpg_store_enable: %lu\n", tmp);
a42d985b
BVA
3195 return -EINVAL;
3196 }
3197 if (tmp == 1)
3198 sport->enabled = true;
3199 else
3200 sport->enabled = false;
3201
3202 return count;
3203}
3204
2eafd729 3205CONFIGFS_ATTR(srpt_tpg_, enable);
a42d985b
BVA
3206
3207static struct configfs_attribute *srpt_tpg_attrs[] = {
2eafd729 3208 &srpt_tpg_attr_enable,
a42d985b
BVA
3209 NULL,
3210};
3211
3212/**
3213 * configfs callback invoked for
3214 * mkdir /sys/kernel/config/target/$driver/$port/$tpg
3215 */
3216static struct se_portal_group *srpt_make_tpg(struct se_wwn *wwn,
3217 struct config_group *group,
3218 const char *name)
3219{
3220 struct srpt_port *sport = container_of(wwn, struct srpt_port, port_wwn);
3221 int res;
3222
3223 /* Initialize sport->port_wwn and sport->port_tpg_1 */
bc0c94b1 3224 res = core_tpg_register(&sport->port_wwn, &sport->port_tpg_1, SCSI_PROTOCOL_SRP);
a42d985b
BVA
3225 if (res)
3226 return ERR_PTR(res);
3227
3228 return &sport->port_tpg_1;
3229}
3230
3231/**
3232 * configfs callback invoked for
3233 * rmdir /sys/kernel/config/target/$driver/$port/$tpg
3234 */
3235static void srpt_drop_tpg(struct se_portal_group *tpg)
3236{
3237 struct srpt_port *sport = container_of(tpg,
3238 struct srpt_port, port_tpg_1);
3239
3240 sport->enabled = false;
3241 core_tpg_deregister(&sport->port_tpg_1);
3242}
3243
3244/**
3245 * configfs callback invoked for
3246 * mkdir /sys/kernel/config/target/$driver/$port
3247 */
3248static struct se_wwn *srpt_make_tport(struct target_fabric_configfs *tf,
3249 struct config_group *group,
3250 const char *name)
3251{
3252 struct srpt_port *sport;
3253 int ret;
3254
3255 sport = srpt_lookup_port(name);
3256 pr_debug("make_tport(%s)\n", name);
3257 ret = -EINVAL;
3258 if (!sport)
3259 goto err;
3260
3261 return &sport->port_wwn;
3262
3263err:
3264 return ERR_PTR(ret);
3265}
3266
3267/**
3268 * configfs callback invoked for
3269 * rmdir /sys/kernel/config/target/$driver/$port
3270 */
3271static void srpt_drop_tport(struct se_wwn *wwn)
3272{
3273 struct srpt_port *sport = container_of(wwn, struct srpt_port, port_wwn);
3274
3275 pr_debug("drop_tport(%s\n", config_item_name(&sport->port_wwn.wwn_group.cg_item));
3276}
3277
2eafd729 3278static ssize_t srpt_wwn_version_show(struct config_item *item, char *buf)
a42d985b
BVA
3279{
3280 return scnprintf(buf, PAGE_SIZE, "%s\n", DRV_VERSION);
3281}
3282
2eafd729 3283CONFIGFS_ATTR_RO(srpt_wwn_, version);
a42d985b
BVA
3284
3285static struct configfs_attribute *srpt_wwn_attrs[] = {
2eafd729 3286 &srpt_wwn_attr_version,
a42d985b
BVA
3287 NULL,
3288};
3289
9ac8928e
CH
3290static const struct target_core_fabric_ops srpt_template = {
3291 .module = THIS_MODULE,
3292 .name = "srpt",
a42d985b 3293 .get_fabric_name = srpt_get_fabric_name,
a42d985b
BVA
3294 .tpg_get_wwn = srpt_get_fabric_wwn,
3295 .tpg_get_tag = srpt_get_tag,
a42d985b
BVA
3296 .tpg_check_demo_mode = srpt_check_false,
3297 .tpg_check_demo_mode_cache = srpt_check_true,
3298 .tpg_check_demo_mode_write_protect = srpt_check_true,
3299 .tpg_check_prod_mode_write_protect = srpt_check_false,
a42d985b
BVA
3300 .tpg_get_inst_index = srpt_tpg_get_inst_index,
3301 .release_cmd = srpt_release_cmd,
3302 .check_stop_free = srpt_check_stop_free,
3303 .shutdown_session = srpt_shutdown_session,
3304 .close_session = srpt_close_session,
a42d985b
BVA
3305 .sess_get_index = srpt_sess_get_index,
3306 .sess_get_initiator_sid = NULL,
3307 .write_pending = srpt_write_pending,
3308 .write_pending_status = srpt_write_pending_status,
3309 .set_default_node_attributes = srpt_set_default_node_attrs,
a42d985b 3310 .get_cmd_state = srpt_get_tcm_cmd_state,
b79fafac 3311 .queue_data_in = srpt_queue_data_in,
a42d985b 3312 .queue_status = srpt_queue_status,
b79fafac 3313 .queue_tm_rsp = srpt_queue_tm_rsp,
131e6abc 3314 .aborted_task = srpt_aborted_task,
a42d985b
BVA
3315 /*
3316 * Setup function pointers for generic logic in
3317 * target_core_fabric_configfs.c
3318 */
3319 .fabric_make_wwn = srpt_make_tport,
3320 .fabric_drop_wwn = srpt_drop_tport,
3321 .fabric_make_tpg = srpt_make_tpg,
3322 .fabric_drop_tpg = srpt_drop_tpg,
c7d6a803 3323 .fabric_init_nodeacl = srpt_init_nodeacl,
9ac8928e
CH
3324
3325 .tfc_wwn_attrs = srpt_wwn_attrs,
3326 .tfc_tpg_base_attrs = srpt_tpg_attrs,
3327 .tfc_tpg_attrib_attrs = srpt_tpg_attrib_attrs,
a42d985b
BVA
3328};
3329
3330/**
3331 * srpt_init_module() - Kernel module initialization.
3332 *
3333 * Note: Since ib_register_client() registers callback functions, and since at
3334 * least one of these callback functions (srpt_add_one()) calls target core
3335 * functions, this driver must be registered with the target core before
3336 * ib_register_client() is called.
3337 */
3338static int __init srpt_init_module(void)
3339{
3340 int ret;
3341
3342 ret = -EINVAL;
3343 if (srp_max_req_size < MIN_MAX_REQ_SIZE) {
9f5d32af 3344 pr_err("invalid value %d for kernel module parameter"
a42d985b
BVA
3345 " srp_max_req_size -- must be at least %d.\n",
3346 srp_max_req_size, MIN_MAX_REQ_SIZE);
3347 goto out;
3348 }
3349
3350 if (srpt_srq_size < MIN_SRPT_SRQ_SIZE
3351 || srpt_srq_size > MAX_SRPT_SRQ_SIZE) {
9f5d32af 3352 pr_err("invalid value %d for kernel module parameter"
a42d985b
BVA
3353 " srpt_srq_size -- must be in the range [%d..%d].\n",
3354 srpt_srq_size, MIN_SRPT_SRQ_SIZE, MAX_SRPT_SRQ_SIZE);
3355 goto out;
3356 }
3357
9ac8928e
CH
3358 ret = target_register_template(&srpt_template);
3359 if (ret)
a42d985b 3360 goto out;
a42d985b
BVA
3361
3362 ret = ib_register_client(&srpt_client);
3363 if (ret) {
9f5d32af 3364 pr_err("couldn't register IB client\n");
a42d985b
BVA
3365 goto out_unregister_target;
3366 }
3367
3368 return 0;
3369
3370out_unregister_target:
9ac8928e 3371 target_unregister_template(&srpt_template);
a42d985b
BVA
3372out:
3373 return ret;
3374}
3375
3376static void __exit srpt_cleanup_module(void)
3377{
3378 ib_unregister_client(&srpt_client);
9ac8928e 3379 target_unregister_template(&srpt_template);
a42d985b
BVA
3380}
3381
3382module_init(srpt_init_module);
3383module_exit(srpt_cleanup_module);