]>
Commit | Line | Data |
---|---|---|
e85b24b5 OG |
1 | /* |
2 | * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved. | |
28f292e8 | 3 | * Copyright (c) 2013 Mellanox Technologies. All rights reserved. |
e85b24b5 OG |
4 | * |
5 | * This software is available to you under a choice of one of two | |
6 | * licenses. You may choose to be licensed under the terms of the GNU | |
7 | * General Public License (GPL) Version 2, available from the file | |
8 | * COPYING in the main directory of this source tree, or the | |
9 | * OpenIB.org BSD license below: | |
10 | * | |
11 | * Redistribution and use in source and binary forms, with or | |
12 | * without modification, are permitted provided that the following | |
13 | * conditions are met: | |
14 | * | |
15 | * - Redistributions of source code must retain the above | |
16 | * copyright notice, this list of conditions and the following | |
17 | * disclaimer. | |
18 | * | |
19 | * - Redistributions in binary form must reproduce the above | |
20 | * copyright notice, this list of conditions and the following | |
21 | * disclaimer in the documentation and/or other materials | |
22 | * provided with the distribution. | |
23 | * | |
24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
31 | * SOFTWARE. | |
e85b24b5 OG |
32 | */ |
33 | #include <linux/kernel.h> | |
34 | #include <linux/slab.h> | |
35 | #include <linux/mm.h> | |
e85b24b5 OG |
36 | #include <linux/scatterlist.h> |
37 | #include <linux/kfifo.h> | |
38 | #include <scsi/scsi_cmnd.h> | |
39 | #include <scsi/scsi_host.h> | |
40 | ||
41 | #include "iscsi_iser.h" | |
42 | ||
e85b24b5 OG |
43 | /* Register user buffer memory and initialize passive rdma |
44 | * dto descriptor. Total data size is stored in | |
2261ec3d | 45 | * iser_task->data[ISER_DIR_IN].data_len |
e85b24b5 | 46 | */ |
2261ec3d | 47 | static int iser_prepare_read_cmd(struct iscsi_task *task, |
e85b24b5 OG |
48 | unsigned int edtl) |
49 | ||
50 | { | |
2261ec3d | 51 | struct iscsi_iser_task *iser_task = task->dd_data; |
e85b24b5 OG |
52 | struct iser_regd_buf *regd_buf; |
53 | int err; | |
2261ec3d MC |
54 | struct iser_hdr *hdr = &iser_task->desc.iser_header; |
55 | struct iser_data_buf *buf_in = &iser_task->data[ISER_DIR_IN]; | |
e85b24b5 | 56 | |
2261ec3d | 57 | err = iser_dma_map_task_data(iser_task, |
e85b24b5 OG |
58 | buf_in, |
59 | ISER_DIR_IN, | |
60 | DMA_FROM_DEVICE); | |
61 | if (err) | |
62 | return err; | |
63 | ||
2261ec3d | 64 | if (edtl > iser_task->data[ISER_DIR_IN].data_len) { |
e85b24b5 OG |
65 | iser_err("Total data length: %ld, less than EDTL: " |
66 | "%d, in READ cmd BHS itt: %d, conn: 0x%p\n", | |
2261ec3d MC |
67 | iser_task->data[ISER_DIR_IN].data_len, edtl, |
68 | task->itt, iser_task->iser_conn); | |
e85b24b5 OG |
69 | return -EINVAL; |
70 | } | |
71 | ||
2261ec3d | 72 | err = iser_reg_rdma_mem(iser_task,ISER_DIR_IN); |
e85b24b5 OG |
73 | if (err) { |
74 | iser_err("Failed to set up Data-IN RDMA\n"); | |
75 | return err; | |
76 | } | |
2261ec3d | 77 | regd_buf = &iser_task->rdma_regd[ISER_DIR_IN]; |
e85b24b5 OG |
78 | |
79 | hdr->flags |= ISER_RSV; | |
80 | hdr->read_stag = cpu_to_be32(regd_buf->reg.rkey); | |
81 | hdr->read_va = cpu_to_be64(regd_buf->reg.va); | |
82 | ||
83 | iser_dbg("Cmd itt:%d READ tags RKEY:%#.4X VA:%#llX\n", | |
2261ec3d | 84 | task->itt, regd_buf->reg.rkey, |
e85b24b5 OG |
85 | (unsigned long long)regd_buf->reg.va); |
86 | ||
87 | return 0; | |
88 | } | |
89 | ||
90 | /* Register user buffer memory and initialize passive rdma | |
91 | * dto descriptor. Total data size is stored in | |
2261ec3d | 92 | * task->data[ISER_DIR_OUT].data_len |
e85b24b5 OG |
93 | */ |
94 | static int | |
2261ec3d | 95 | iser_prepare_write_cmd(struct iscsi_task *task, |
e85b24b5 OG |
96 | unsigned int imm_sz, |
97 | unsigned int unsol_sz, | |
98 | unsigned int edtl) | |
99 | { | |
2261ec3d | 100 | struct iscsi_iser_task *iser_task = task->dd_data; |
e85b24b5 OG |
101 | struct iser_regd_buf *regd_buf; |
102 | int err; | |
2261ec3d MC |
103 | struct iser_hdr *hdr = &iser_task->desc.iser_header; |
104 | struct iser_data_buf *buf_out = &iser_task->data[ISER_DIR_OUT]; | |
f19624aa | 105 | struct ib_sge *tx_dsg = &iser_task->desc.tx_sg[1]; |
e85b24b5 | 106 | |
2261ec3d | 107 | err = iser_dma_map_task_data(iser_task, |
e85b24b5 OG |
108 | buf_out, |
109 | ISER_DIR_OUT, | |
110 | DMA_TO_DEVICE); | |
111 | if (err) | |
112 | return err; | |
113 | ||
2261ec3d | 114 | if (edtl > iser_task->data[ISER_DIR_OUT].data_len) { |
e85b24b5 OG |
115 | iser_err("Total data length: %ld, less than EDTL: %d, " |
116 | "in WRITE cmd BHS itt: %d, conn: 0x%p\n", | |
2261ec3d MC |
117 | iser_task->data[ISER_DIR_OUT].data_len, |
118 | edtl, task->itt, task->conn); | |
e85b24b5 OG |
119 | return -EINVAL; |
120 | } | |
121 | ||
2261ec3d | 122 | err = iser_reg_rdma_mem(iser_task,ISER_DIR_OUT); |
e85b24b5 OG |
123 | if (err != 0) { |
124 | iser_err("Failed to register write cmd RDMA mem\n"); | |
125 | return err; | |
126 | } | |
127 | ||
2261ec3d | 128 | regd_buf = &iser_task->rdma_regd[ISER_DIR_OUT]; |
e85b24b5 OG |
129 | |
130 | if (unsol_sz < edtl) { | |
131 | hdr->flags |= ISER_WSV; | |
132 | hdr->write_stag = cpu_to_be32(regd_buf->reg.rkey); | |
133 | hdr->write_va = cpu_to_be64(regd_buf->reg.va + unsol_sz); | |
134 | ||
135 | iser_dbg("Cmd itt:%d, WRITE tags, RKEY:%#.4X " | |
136 | "VA:%#llX + unsol:%d\n", | |
2261ec3d | 137 | task->itt, regd_buf->reg.rkey, |
e85b24b5 OG |
138 | (unsigned long long)regd_buf->reg.va, unsol_sz); |
139 | } | |
140 | ||
141 | if (imm_sz > 0) { | |
142 | iser_dbg("Cmd itt:%d, WRITE, adding imm.data sz: %d\n", | |
2261ec3d | 143 | task->itt, imm_sz); |
f19624aa OG |
144 | tx_dsg->addr = regd_buf->reg.va; |
145 | tx_dsg->length = imm_sz; | |
146 | tx_dsg->lkey = regd_buf->reg.lkey; | |
147 | iser_task->desc.num_sge = 2; | |
e85b24b5 OG |
148 | } |
149 | ||
150 | return 0; | |
151 | } | |
152 | ||
e85b24b5 | 153 | /* creates a new tx descriptor and adds header regd buffer */ |
f19624aa OG |
154 | static void iser_create_send_desc(struct iser_conn *ib_conn, |
155 | struct iser_tx_desc *tx_desc) | |
e85b24b5 | 156 | { |
f19624aa | 157 | struct iser_device *device = ib_conn->device; |
e85b24b5 | 158 | |
f19624aa OG |
159 | ib_dma_sync_single_for_cpu(device->ib_device, |
160 | tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE); | |
e85b24b5 OG |
161 | |
162 | memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); | |
163 | tx_desc->iser_header.flags = ISER_VER; | |
164 | ||
f19624aa OG |
165 | tx_desc->num_sge = 1; |
166 | ||
167 | if (tx_desc->tx_sg[0].lkey != device->mr->lkey) { | |
168 | tx_desc->tx_sg[0].lkey = device->mr->lkey; | |
169 | iser_dbg("sdesc %p lkey mismatch, fixing\n", tx_desc); | |
170 | } | |
e85b24b5 OG |
171 | } |
172 | ||
f19624aa | 173 | |
89e984e2 | 174 | int iser_alloc_rx_descriptors(struct iser_conn *ib_conn) |
bcc60c38 OG |
175 | { |
176 | int i, j; | |
177 | u64 dma_addr; | |
178 | struct iser_rx_desc *rx_desc; | |
179 | struct ib_sge *rx_sg; | |
180 | struct iser_device *device = ib_conn->device; | |
181 | ||
182 | ib_conn->rx_descs = kmalloc(ISER_QP_MAX_RECV_DTOS * | |
183 | sizeof(struct iser_rx_desc), GFP_KERNEL); | |
184 | if (!ib_conn->rx_descs) | |
185 | goto rx_desc_alloc_fail; | |
186 | ||
187 | rx_desc = ib_conn->rx_descs; | |
188 | ||
189 | for (i = 0; i < ISER_QP_MAX_RECV_DTOS; i++, rx_desc++) { | |
190 | dma_addr = ib_dma_map_single(device->ib_device, (void *)rx_desc, | |
191 | ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); | |
192 | if (ib_dma_mapping_error(device->ib_device, dma_addr)) | |
193 | goto rx_desc_dma_map_failed; | |
194 | ||
195 | rx_desc->dma_addr = dma_addr; | |
196 | ||
197 | rx_sg = &rx_desc->rx_sg; | |
198 | rx_sg->addr = rx_desc->dma_addr; | |
199 | rx_sg->length = ISER_RX_PAYLOAD_SIZE; | |
200 | rx_sg->lkey = device->mr->lkey; | |
201 | } | |
202 | ||
203 | ib_conn->rx_desc_head = 0; | |
204 | return 0; | |
205 | ||
206 | rx_desc_dma_map_failed: | |
207 | rx_desc = ib_conn->rx_descs; | |
208 | for (j = 0; j < i; j++, rx_desc++) | |
209 | ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, | |
210 | ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); | |
211 | kfree(ib_conn->rx_descs); | |
212 | ib_conn->rx_descs = NULL; | |
213 | rx_desc_alloc_fail: | |
214 | iser_err("failed allocating rx descriptors / data buffers\n"); | |
215 | return -ENOMEM; | |
216 | } | |
217 | ||
218 | void iser_free_rx_descriptors(struct iser_conn *ib_conn) | |
219 | { | |
220 | int i; | |
221 | struct iser_rx_desc *rx_desc; | |
222 | struct iser_device *device = ib_conn->device; | |
223 | ||
bcc60c38 OG |
224 | if (!ib_conn->rx_descs) |
225 | return; | |
226 | ||
227 | rx_desc = ib_conn->rx_descs; | |
228 | for (i = 0; i < ISER_QP_MAX_RECV_DTOS; i++, rx_desc++) | |
229 | ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, | |
230 | ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); | |
231 | kfree(ib_conn->rx_descs); | |
232 | } | |
233 | ||
89e984e2 | 234 | static int iser_post_rx_bufs(struct iscsi_conn *conn, struct iscsi_hdr *req) |
e85b24b5 OG |
235 | { |
236 | struct iscsi_iser_conn *iser_conn = conn->dd_data; | |
6a06a4b8 | 237 | struct iscsi_session *session = conn->session; |
e85b24b5 | 238 | |
89e984e2 OG |
239 | iser_dbg("req op %x flags %x\n", req->opcode, req->flags); |
240 | /* check if this is the last login - going to full feature phase */ | |
241 | if ((req->flags & ISCSI_FULL_FEATURE_PHASE) != ISCSI_FULL_FEATURE_PHASE) | |
242 | return 0; | |
e85b24b5 | 243 | |
89e984e2 OG |
244 | /* |
245 | * Check that there is one posted recv buffer (for the last login | |
246 | * response) and no posted send buffers left - they must have been | |
247 | * consumed during previous login phases. | |
248 | */ | |
249 | WARN_ON(iser_conn->ib_conn->post_recv_buf_count != 1); | |
250 | WARN_ON(atomic_read(&iser_conn->ib_conn->post_send_buf_count) != 0); | |
bcc60c38 | 251 | |
6a06a4b8 OG |
252 | if (session->discovery_sess) { |
253 | iser_info("Discovery session, re-using login RX buffer\n"); | |
254 | return 0; | |
255 | } else | |
256 | iser_info("Normal session, posting batch of RX %d buffers\n", | |
257 | ISER_MIN_POSTED_RX); | |
258 | ||
e85b24b5 | 259 | /* Initial post receive buffers */ |
bcc60c38 OG |
260 | if (iser_post_recvm(iser_conn->ib_conn, ISER_MIN_POSTED_RX)) |
261 | return -ENOMEM; | |
262 | ||
e85b24b5 OG |
263 | return 0; |
264 | } | |
265 | ||
e85b24b5 OG |
266 | /** |
267 | * iser_send_command - send command PDU | |
268 | */ | |
2747fdb2 | 269 | int iser_send_command(struct iscsi_conn *conn, |
2261ec3d | 270 | struct iscsi_task *task) |
e85b24b5 OG |
271 | { |
272 | struct iscsi_iser_conn *iser_conn = conn->dd_data; | |
2261ec3d | 273 | struct iscsi_iser_task *iser_task = task->dd_data; |
e85b24b5 | 274 | unsigned long edtl; |
bcc60c38 | 275 | int err; |
e85b24b5 | 276 | struct iser_data_buf *data_buf; |
12352183 | 277 | struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)task->hdr; |
2261ec3d | 278 | struct scsi_cmnd *sc = task->sc; |
f19624aa | 279 | struct iser_tx_desc *tx_desc = &iser_task->desc; |
e85b24b5 | 280 | |
e85b24b5 OG |
281 | edtl = ntohl(hdr->data_length); |
282 | ||
283 | /* build the tx desc regd header and add it to the tx desc dto */ | |
f19624aa OG |
284 | tx_desc->type = ISCSI_TX_SCSI_COMMAND; |
285 | iser_create_send_desc(iser_conn->ib_conn, tx_desc); | |
e85b24b5 OG |
286 | |
287 | if (hdr->flags & ISCSI_FLAG_CMD_READ) | |
2261ec3d | 288 | data_buf = &iser_task->data[ISER_DIR_IN]; |
e85b24b5 | 289 | else |
2261ec3d | 290 | data_buf = &iser_task->data[ISER_DIR_OUT]; |
e85b24b5 | 291 | |
da9c0c77 FT |
292 | if (scsi_sg_count(sc)) { /* using a scatter list */ |
293 | data_buf->buf = scsi_sglist(sc); | |
294 | data_buf->size = scsi_sg_count(sc); | |
e85b24b5 OG |
295 | } |
296 | ||
da9c0c77 | 297 | data_buf->data_len = scsi_bufflen(sc); |
e85b24b5 OG |
298 | |
299 | if (hdr->flags & ISCSI_FLAG_CMD_READ) { | |
2261ec3d | 300 | err = iser_prepare_read_cmd(task, edtl); |
e85b24b5 OG |
301 | if (err) |
302 | goto send_command_error; | |
303 | } | |
304 | if (hdr->flags & ISCSI_FLAG_CMD_WRITE) { | |
2261ec3d MC |
305 | err = iser_prepare_write_cmd(task, |
306 | task->imm_count, | |
307 | task->imm_count + | |
0f9c7449 | 308 | task->unsol_r2t.data_length, |
e85b24b5 OG |
309 | edtl); |
310 | if (err) | |
311 | goto send_command_error; | |
312 | } | |
313 | ||
2261ec3d | 314 | iser_task->status = ISER_TASK_STATUS_STARTED; |
e85b24b5 | 315 | |
f19624aa | 316 | err = iser_post_send(iser_conn->ib_conn, tx_desc); |
e85b24b5 OG |
317 | if (!err) |
318 | return 0; | |
319 | ||
320 | send_command_error: | |
2261ec3d | 321 | iser_err("conn %p failed task->itt %d err %d\n",conn, task->itt, err); |
e85b24b5 OG |
322 | return err; |
323 | } | |
324 | ||
325 | /** | |
326 | * iser_send_data_out - send data out PDU | |
327 | */ | |
2747fdb2 | 328 | int iser_send_data_out(struct iscsi_conn *conn, |
2261ec3d | 329 | struct iscsi_task *task, |
e85b24b5 OG |
330 | struct iscsi_data *hdr) |
331 | { | |
332 | struct iscsi_iser_conn *iser_conn = conn->dd_data; | |
2261ec3d | 333 | struct iscsi_iser_task *iser_task = task->dd_data; |
f19624aa OG |
334 | struct iser_tx_desc *tx_desc = NULL; |
335 | struct iser_regd_buf *regd_buf; | |
e85b24b5 OG |
336 | unsigned long buf_offset; |
337 | unsigned long data_seg_len; | |
0a22ab92 | 338 | uint32_t itt; |
e85b24b5 | 339 | int err = 0; |
f19624aa OG |
340 | struct ib_sge *tx_dsg; |
341 | ||
0a22ab92 | 342 | itt = (__force uint32_t)hdr->itt; |
e85b24b5 OG |
343 | data_seg_len = ntoh24(hdr->dlength); |
344 | buf_offset = ntohl(hdr->offset); | |
345 | ||
346 | iser_dbg("%s itt %d dseg_len %d offset %d\n", | |
347 | __func__,(int)itt,(int)data_seg_len,(int)buf_offset); | |
348 | ||
528f4e8c | 349 | tx_desc = kmem_cache_zalloc(ig.desc_cache, GFP_ATOMIC); |
e85b24b5 OG |
350 | if (tx_desc == NULL) { |
351 | iser_err("Failed to alloc desc for post dataout\n"); | |
352 | return -ENOMEM; | |
353 | } | |
354 | ||
355 | tx_desc->type = ISCSI_TX_DATAOUT; | |
f19624aa | 356 | tx_desc->iser_header.flags = ISER_VER; |
e85b24b5 OG |
357 | memcpy(&tx_desc->iscsi_header, hdr, sizeof(struct iscsi_hdr)); |
358 | ||
f19624aa OG |
359 | /* build the tx desc */ |
360 | iser_initialize_task_headers(task, tx_desc); | |
e85b24b5 | 361 | |
f19624aa OG |
362 | regd_buf = &iser_task->rdma_regd[ISER_DIR_OUT]; |
363 | tx_dsg = &tx_desc->tx_sg[1]; | |
364 | tx_dsg->addr = regd_buf->reg.va + buf_offset; | |
365 | tx_dsg->length = data_seg_len; | |
366 | tx_dsg->lkey = regd_buf->reg.lkey; | |
367 | tx_desc->num_sge = 2; | |
e85b24b5 | 368 | |
2261ec3d | 369 | if (buf_offset + data_seg_len > iser_task->data[ISER_DIR_OUT].data_len) { |
e85b24b5 OG |
370 | iser_err("Offset:%ld & DSL:%ld in Data-Out " |
371 | "inconsistent with total len:%ld, itt:%d\n", | |
372 | buf_offset, data_seg_len, | |
2261ec3d | 373 | iser_task->data[ISER_DIR_OUT].data_len, itt); |
e85b24b5 OG |
374 | err = -EINVAL; |
375 | goto send_data_out_error; | |
376 | } | |
377 | iser_dbg("data-out itt: %d, offset: %ld, sz: %ld\n", | |
378 | itt, buf_offset, data_seg_len); | |
379 | ||
380 | ||
f19624aa | 381 | err = iser_post_send(iser_conn->ib_conn, tx_desc); |
e85b24b5 OG |
382 | if (!err) |
383 | return 0; | |
384 | ||
385 | send_data_out_error: | |
e85b24b5 OG |
386 | kmem_cache_free(ig.desc_cache, tx_desc); |
387 | iser_err("conn %p failed err %d\n",conn, err); | |
388 | return err; | |
389 | } | |
390 | ||
391 | int iser_send_control(struct iscsi_conn *conn, | |
2261ec3d | 392 | struct iscsi_task *task) |
e85b24b5 OG |
393 | { |
394 | struct iscsi_iser_conn *iser_conn = conn->dd_data; | |
2261ec3d | 395 | struct iscsi_iser_task *iser_task = task->dd_data; |
f19624aa | 396 | struct iser_tx_desc *mdesc = &iser_task->desc; |
e85b24b5 | 397 | unsigned long data_seg_len; |
f19624aa | 398 | int err = 0; |
e85b24b5 | 399 | struct iser_device *device; |
2c4ce609 | 400 | struct iser_conn *ib_conn = iser_conn->ib_conn; |
e85b24b5 | 401 | |
e85b24b5 OG |
402 | /* build the tx desc regd header and add it to the tx desc dto */ |
403 | mdesc->type = ISCSI_TX_CONTROL; | |
f19624aa | 404 | iser_create_send_desc(iser_conn->ib_conn, mdesc); |
e85b24b5 OG |
405 | |
406 | device = iser_conn->ib_conn->device; | |
407 | ||
2261ec3d | 408 | data_seg_len = ntoh24(task->hdr->dlength); |
e85b24b5 OG |
409 | |
410 | if (data_seg_len > 0) { | |
f19624aa OG |
411 | struct ib_sge *tx_dsg = &mdesc->tx_sg[1]; |
412 | if (task != conn->login_task) { | |
413 | iser_err("data present on non login task!!!\n"); | |
414 | goto send_control_error; | |
415 | } | |
2c4ce609 OG |
416 | |
417 | ib_dma_sync_single_for_cpu(device->ib_device, | |
418 | ib_conn->login_req_dma, task->data_count, | |
419 | DMA_TO_DEVICE); | |
420 | ||
421 | memcpy(iser_conn->ib_conn->login_req_buf, task->data, | |
f19624aa | 422 | task->data_count); |
2c4ce609 OG |
423 | |
424 | ib_dma_sync_single_for_device(device->ib_device, | |
425 | ib_conn->login_req_dma, task->data_count, | |
426 | DMA_TO_DEVICE); | |
427 | ||
428 | tx_dsg->addr = iser_conn->ib_conn->login_req_dma; | |
200ae1a0 | 429 | tx_dsg->length = task->data_count; |
f19624aa OG |
430 | tx_dsg->lkey = device->mr->lkey; |
431 | mdesc->num_sge = 2; | |
e85b24b5 OG |
432 | } |
433 | ||
bcc60c38 | 434 | if (task == conn->login_task) { |
6a06a4b8 OG |
435 | iser_dbg("op %x dsl %lx, posting login rx buffer\n", |
436 | task->hdr->opcode, data_seg_len); | |
bcc60c38 OG |
437 | err = iser_post_recvl(iser_conn->ib_conn); |
438 | if (err) | |
439 | goto send_control_error; | |
89e984e2 OG |
440 | err = iser_post_rx_bufs(conn, task->hdr); |
441 | if (err) | |
442 | goto send_control_error; | |
e85b24b5 OG |
443 | } |
444 | ||
f19624aa | 445 | err = iser_post_send(iser_conn->ib_conn, mdesc); |
e85b24b5 OG |
446 | if (!err) |
447 | return 0; | |
448 | ||
449 | send_control_error: | |
e85b24b5 OG |
450 | iser_err("conn %p failed err %d\n",conn, err); |
451 | return err; | |
452 | } | |
453 | ||
454 | /** | |
455 | * iser_rcv_dto_completion - recv DTO completion | |
456 | */ | |
bcc60c38 OG |
457 | void iser_rcv_completion(struct iser_rx_desc *rx_desc, |
458 | unsigned long rx_xfer_len, | |
459 | struct iser_conn *ib_conn) | |
e85b24b5 | 460 | { |
bcc60c38 | 461 | struct iscsi_iser_conn *conn = ib_conn->iser_conn; |
e85b24b5 | 462 | struct iscsi_hdr *hdr; |
bcc60c38 OG |
463 | u64 rx_dma; |
464 | int rx_buflen, outstanding, count, err; | |
465 | ||
466 | /* differentiate between login to all other PDUs */ | |
2c4ce609 OG |
467 | if ((char *)rx_desc == ib_conn->login_resp_buf) { |
468 | rx_dma = ib_conn->login_resp_dma; | |
bcc60c38 OG |
469 | rx_buflen = ISER_RX_LOGIN_SIZE; |
470 | } else { | |
471 | rx_dma = rx_desc->dma_addr; | |
472 | rx_buflen = ISER_RX_PAYLOAD_SIZE; | |
473 | } | |
e85b24b5 | 474 | |
bcc60c38 OG |
475 | ib_dma_sync_single_for_cpu(ib_conn->device->ib_device, rx_dma, |
476 | rx_buflen, DMA_FROM_DEVICE); | |
e85b24b5 | 477 | |
bcc60c38 | 478 | hdr = &rx_desc->iscsi_header; |
e85b24b5 | 479 | |
bcc60c38 OG |
480 | iser_dbg("op 0x%x itt 0x%x dlen %d\n", hdr->opcode, |
481 | hdr->itt, (int)(rx_xfer_len - ISER_HEADERS_LEN)); | |
e85b24b5 | 482 | |
bcc60c38 OG |
483 | iscsi_iser_recv(conn->iscsi_conn, hdr, |
484 | rx_desc->data, rx_xfer_len - ISER_HEADERS_LEN); | |
e85b24b5 | 485 | |
bcc60c38 OG |
486 | ib_dma_sync_single_for_device(ib_conn->device->ib_device, rx_dma, |
487 | rx_buflen, DMA_FROM_DEVICE); | |
e85b24b5 OG |
488 | |
489 | /* decrementing conn->post_recv_buf_count only --after-- freeing the * | |
490 | * task eliminates the need to worry on tasks which are completed in * | |
491 | * parallel to the execution of iser_conn_term. So the code that waits * | |
492 | * for the posted rx bufs refcount to become zero handles everything */ | |
704315f0 | 493 | conn->ib_conn->post_recv_buf_count--; |
bcc60c38 | 494 | |
2c4ce609 | 495 | if (rx_dma == ib_conn->login_resp_dma) |
bcc60c38 OG |
496 | return; |
497 | ||
704315f0 | 498 | outstanding = ib_conn->post_recv_buf_count; |
bcc60c38 OG |
499 | if (outstanding + ISER_MIN_POSTED_RX <= ISER_QP_MAX_RECV_DTOS) { |
500 | count = min(ISER_QP_MAX_RECV_DTOS - outstanding, | |
501 | ISER_MIN_POSTED_RX); | |
502 | err = iser_post_recvm(ib_conn, count); | |
503 | if (err) | |
504 | iser_err("posting %d rx bufs err %d\n", count, err); | |
505 | } | |
e85b24b5 OG |
506 | } |
507 | ||
f19624aa OG |
508 | void iser_snd_completion(struct iser_tx_desc *tx_desc, |
509 | struct iser_conn *ib_conn) | |
e85b24b5 | 510 | { |
2261ec3d | 511 | struct iscsi_task *task; |
f19624aa | 512 | struct iser_device *device = ib_conn->device; |
e85b24b5 | 513 | |
f19624aa OG |
514 | if (tx_desc->type == ISCSI_TX_DATAOUT) { |
515 | ib_dma_unmap_single(device->ib_device, tx_desc->dma_addr, | |
516 | ISER_HEADERS_LEN, DMA_TO_DEVICE); | |
e85b24b5 | 517 | kmem_cache_free(ig.desc_cache, tx_desc); |
f19624aa | 518 | } |
e85b24b5 | 519 | |
87e8df7a | 520 | atomic_dec(&ib_conn->post_send_buf_count); |
e85b24b5 | 521 | |
e85b24b5 OG |
522 | if (tx_desc->type == ISCSI_TX_CONTROL) { |
523 | /* this arithmetic is legal by libiscsi dd_data allocation */ | |
2261ec3d MC |
524 | task = (void *) ((long)(void *)tx_desc - |
525 | sizeof(struct iscsi_task)); | |
526 | if (task->hdr->itt == RESERVED_ITT) | |
527 | iscsi_put_task(task); | |
e85b24b5 OG |
528 | } |
529 | } | |
530 | ||
2261ec3d | 531 | void iser_task_rdma_init(struct iscsi_iser_task *iser_task) |
e85b24b5 OG |
532 | |
533 | { | |
2261ec3d | 534 | iser_task->status = ISER_TASK_STATUS_INIT; |
e85b24b5 | 535 | |
2261ec3d MC |
536 | iser_task->dir[ISER_DIR_IN] = 0; |
537 | iser_task->dir[ISER_DIR_OUT] = 0; | |
e85b24b5 | 538 | |
2261ec3d MC |
539 | iser_task->data[ISER_DIR_IN].data_len = 0; |
540 | iser_task->data[ISER_DIR_OUT].data_len = 0; | |
e85b24b5 | 541 | |
2261ec3d | 542 | memset(&iser_task->rdma_regd[ISER_DIR_IN], 0, |
e85b24b5 | 543 | sizeof(struct iser_regd_buf)); |
2261ec3d | 544 | memset(&iser_task->rdma_regd[ISER_DIR_OUT], 0, |
e85b24b5 OG |
545 | sizeof(struct iser_regd_buf)); |
546 | } | |
547 | ||
2261ec3d | 548 | void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task) |
e85b24b5 | 549 | { |
74a20780 | 550 | int is_rdma_aligned = 1; |
3104a217 | 551 | struct iser_regd_buf *regd; |
e85b24b5 OG |
552 | |
553 | /* if we were reading, copy back to unaligned sglist, | |
554 | * anyway dma_unmap and free the copy | |
555 | */ | |
2261ec3d | 556 | if (iser_task->data_copy[ISER_DIR_IN].copy_buf != NULL) { |
74a20780 | 557 | is_rdma_aligned = 0; |
2261ec3d | 558 | iser_finalize_rdma_unaligned_sg(iser_task, ISER_DIR_IN); |
74a20780 | 559 | } |
2261ec3d | 560 | if (iser_task->data_copy[ISER_DIR_OUT].copy_buf != NULL) { |
74a20780 | 561 | is_rdma_aligned = 0; |
2261ec3d | 562 | iser_finalize_rdma_unaligned_sg(iser_task, ISER_DIR_OUT); |
74a20780 | 563 | } |
e85b24b5 | 564 | |
2261ec3d MC |
565 | if (iser_task->dir[ISER_DIR_IN]) { |
566 | regd = &iser_task->rdma_regd[ISER_DIR_IN]; | |
f19624aa OG |
567 | if (regd->reg.is_fmr) |
568 | iser_unreg_mem(®d->reg); | |
e85b24b5 OG |
569 | } |
570 | ||
2261ec3d MC |
571 | if (iser_task->dir[ISER_DIR_OUT]) { |
572 | regd = &iser_task->rdma_regd[ISER_DIR_OUT]; | |
f19624aa OG |
573 | if (regd->reg.is_fmr) |
574 | iser_unreg_mem(®d->reg); | |
e85b24b5 OG |
575 | } |
576 | ||
74a20780 EZ |
577 | /* if the data was unaligned, it was already unmapped and then copied */ |
578 | if (is_rdma_aligned) | |
2261ec3d | 579 | iser_dma_unmap_task_data(iser_task); |
e85b24b5 | 580 | } |