]>
Commit | Line | Data |
---|---|---|
6461f64a OG |
1 | /* |
2 | * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved. | |
3ee07d27 | 3 | * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved. |
6461f64a OG |
4 | * |
5 | * This software is available to you under a choice of one of two | |
6 | * licenses. You may choose to be licensed under the terms of the GNU | |
7 | * General Public License (GPL) Version 2, available from the file | |
8 | * COPYING in the main directory of this source tree, or the | |
9 | * OpenIB.org BSD license below: | |
10 | * | |
11 | * Redistribution and use in source and binary forms, with or | |
12 | * without modification, are permitted provided that the following | |
13 | * conditions are met: | |
14 | * | |
15 | * - Redistributions of source code must retain the above | |
16 | * copyright notice, this list of conditions and the following | |
17 | * disclaimer. | |
18 | * | |
19 | * - Redistributions in binary form must reproduce the above | |
20 | * copyright notice, this list of conditions and the following | |
21 | * disclaimer in the documentation and/or other materials | |
22 | * provided with the distribution. | |
23 | * | |
24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
31 | * SOFTWARE. | |
6461f64a OG |
32 | */ |
33 | #include <linux/module.h> | |
34 | #include <linux/kernel.h> | |
35 | #include <linux/slab.h> | |
36 | #include <linux/mm.h> | |
a1f8e7f7 | 37 | #include <linux/highmem.h> |
6461f64a OG |
38 | #include <linux/scatterlist.h> |
39 | ||
40 | #include "iscsi_iser.h" | |
41 | ||
42 | #define ISER_KMALLOC_THRESHOLD 0x20000 /* 128K - kmalloc limit */ | |
8dfa0876 | 43 | |
6461f64a OG |
44 | /** |
45 | * iser_start_rdma_unaligned_sg | |
46 | */ | |
2261ec3d | 47 | static int iser_start_rdma_unaligned_sg(struct iscsi_iser_task *iser_task, |
5f588e3d SG |
48 | struct iser_data_buf *data, |
49 | struct iser_data_buf *data_copy, | |
41179e2d | 50 | enum iser_data_dir cmd_dir) |
6461f64a | 51 | { |
a4ee3539 | 52 | struct ib_device *dev = iser_task->iser_conn->ib_conn.device->ib_device; |
5f588e3d SG |
53 | struct scatterlist *sgl = (struct scatterlist *)data->buf; |
54 | struct scatterlist *sg; | |
6461f64a | 55 | char *mem = NULL; |
5f588e3d SG |
56 | unsigned long cmd_data_len = 0; |
57 | int dma_nents, i; | |
58 | ||
59 | for_each_sg(sgl, sg, data->size, i) | |
60 | cmd_data_len += ib_sg_dma_len(dev, sg); | |
6461f64a OG |
61 | |
62 | if (cmd_data_len > ISER_KMALLOC_THRESHOLD) | |
528f4e8c | 63 | mem = (void *)__get_free_pages(GFP_ATOMIC, |
f0d1b0b3 | 64 | ilog2(roundup_pow_of_two(cmd_data_len)) - PAGE_SHIFT); |
6461f64a | 65 | else |
528f4e8c | 66 | mem = kmalloc(cmd_data_len, GFP_ATOMIC); |
6461f64a OG |
67 | |
68 | if (mem == NULL) { | |
69 | iser_err("Failed to allocate mem size %d %d for copying sglist\n", | |
5f588e3d | 70 | data->size, (int)cmd_data_len); |
6461f64a OG |
71 | return -ENOMEM; |
72 | } | |
73 | ||
74 | if (cmd_dir == ISER_DIR_OUT) { | |
75 | /* copy the unaligned sg the buffer which is used for RDMA */ | |
6461f64a OG |
76 | char *p, *from; |
77 | ||
5f588e3d | 78 | sgl = (struct scatterlist *)data->buf; |
53d412fc JA |
79 | p = mem; |
80 | for_each_sg(sgl, sg, data->size, i) { | |
2a156d09 | 81 | from = kmap_atomic(sg_page(sg)); |
6461f64a | 82 | memcpy(p, |
53d412fc JA |
83 | from + sg->offset, |
84 | sg->length); | |
2a156d09 | 85 | kunmap_atomic(from); |
53d412fc | 86 | p += sg->length; |
6461f64a OG |
87 | } |
88 | } | |
89 | ||
5f588e3d SG |
90 | sg_init_one(&data_copy->sg_single, mem, cmd_data_len); |
91 | data_copy->buf = &data_copy->sg_single; | |
92 | data_copy->size = 1; | |
93 | data_copy->copy_buf = mem; | |
6461f64a | 94 | |
5f588e3d | 95 | dma_nents = ib_dma_map_sg(dev, &data_copy->sg_single, 1, |
5180311f RC |
96 | (cmd_dir == ISER_DIR_OUT) ? |
97 | DMA_TO_DEVICE : DMA_FROM_DEVICE); | |
6461f64a OG |
98 | BUG_ON(dma_nents == 0); |
99 | ||
5f588e3d SG |
100 | data_copy->dma_nents = dma_nents; |
101 | data_copy->data_len = cmd_data_len; | |
102 | ||
6461f64a OG |
103 | return 0; |
104 | } | |
105 | ||
106 | /** | |
107 | * iser_finalize_rdma_unaligned_sg | |
108 | */ | |
9a8b08fa | 109 | |
2261ec3d | 110 | void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task, |
9a8b08fa SG |
111 | struct iser_data_buf *data, |
112 | struct iser_data_buf *data_copy, | |
113 | enum iser_data_dir cmd_dir) | |
6461f64a | 114 | { |
5180311f | 115 | struct ib_device *dev; |
6461f64a OG |
116 | unsigned long cmd_data_len; |
117 | ||
a4ee3539 | 118 | dev = iser_task->iser_conn->ib_conn.device->ib_device; |
6461f64a | 119 | |
9a8b08fa | 120 | ib_dma_unmap_sg(dev, &data_copy->sg_single, 1, |
5180311f RC |
121 | (cmd_dir == ISER_DIR_OUT) ? |
122 | DMA_TO_DEVICE : DMA_FROM_DEVICE); | |
6461f64a OG |
123 | |
124 | if (cmd_dir == ISER_DIR_IN) { | |
125 | char *mem; | |
53d412fc | 126 | struct scatterlist *sgl, *sg; |
6461f64a OG |
127 | unsigned char *p, *to; |
128 | unsigned int sg_size; | |
129 | int i; | |
130 | ||
131 | /* copy back read RDMA to unaligned sg */ | |
9a8b08fa | 132 | mem = data_copy->copy_buf; |
6461f64a | 133 | |
9a8b08fa SG |
134 | sgl = (struct scatterlist *)data->buf; |
135 | sg_size = data->size; | |
6461f64a | 136 | |
53d412fc JA |
137 | p = mem; |
138 | for_each_sg(sgl, sg, sg_size, i) { | |
2a156d09 | 139 | to = kmap_atomic(sg_page(sg)); |
53d412fc | 140 | memcpy(to + sg->offset, |
6461f64a | 141 | p, |
53d412fc | 142 | sg->length); |
2a156d09 | 143 | kunmap_atomic(to); |
53d412fc | 144 | p += sg->length; |
6461f64a OG |
145 | } |
146 | } | |
147 | ||
9a8b08fa | 148 | cmd_data_len = data->data_len; |
6461f64a OG |
149 | |
150 | if (cmd_data_len > ISER_KMALLOC_THRESHOLD) | |
9a8b08fa | 151 | free_pages((unsigned long)data_copy->copy_buf, |
f0d1b0b3 | 152 | ilog2(roundup_pow_of_two(cmd_data_len)) - PAGE_SHIFT); |
6461f64a | 153 | else |
9a8b08fa | 154 | kfree(data_copy->copy_buf); |
6461f64a | 155 | |
9a8b08fa | 156 | data_copy->copy_buf = NULL; |
6461f64a OG |
157 | } |
158 | ||
c1ccaf24 OG |
159 | #define IS_4K_ALIGNED(addr) ((((unsigned long)addr) & ~MASK_4K) == 0) |
160 | ||
6461f64a OG |
161 | /** |
162 | * iser_sg_to_page_vec - Translates scatterlist entries to physical addresses | |
163 | * and returns the length of resulting physical address array (may be less than | |
164 | * the original due to possible compaction). | |
165 | * | |
166 | * we build a "page vec" under the assumption that the SG meets the RDMA | |
167 | * alignment requirements. Other then the first and last SG elements, all | |
168 | * the "internal" elements can be compacted into a list whose elements are | |
169 | * dma addresses of physical pages. The code supports also the weird case | |
170 | * where --few fragments of the same page-- are present in the SG as | |
171 | * consecutive elements. Also, it handles one entry SG. | |
172 | */ | |
c1ccaf24 | 173 | |
6461f64a | 174 | static int iser_sg_to_page_vec(struct iser_data_buf *data, |
919fc274 SG |
175 | struct ib_device *ibdev, u64 *pages, |
176 | int *offset, int *data_size) | |
6461f64a | 177 | { |
c1ccaf24 OG |
178 | struct scatterlist *sg, *sgl = (struct scatterlist *)data->buf; |
179 | u64 start_addr, end_addr, page, chunk_start = 0; | |
6461f64a | 180 | unsigned long total_sz = 0; |
c1ccaf24 OG |
181 | unsigned int dma_len; |
182 | int i, new_chunk, cur_page, last_ent = data->dma_nents - 1; | |
6461f64a OG |
183 | |
184 | /* compute the offset of first element */ | |
919fc274 | 185 | *offset = (u64) sgl[0].offset & ~MASK_4K; |
6461f64a | 186 | |
c1ccaf24 OG |
187 | new_chunk = 1; |
188 | cur_page = 0; | |
53d412fc | 189 | for_each_sg(sgl, sg, data->dma_nents, i) { |
c1ccaf24 OG |
190 | start_addr = ib_sg_dma_address(ibdev, sg); |
191 | if (new_chunk) | |
192 | chunk_start = start_addr; | |
193 | dma_len = ib_sg_dma_len(ibdev, sg); | |
194 | end_addr = start_addr + dma_len; | |
5180311f | 195 | total_sz += dma_len; |
6461f64a | 196 | |
c1ccaf24 OG |
197 | /* collect page fragments until aligned or end of SG list */ |
198 | if (!IS_4K_ALIGNED(end_addr) && i < last_ent) { | |
199 | new_chunk = 0; | |
200 | continue; | |
6461f64a | 201 | } |
c1ccaf24 OG |
202 | new_chunk = 1; |
203 | ||
204 | /* address of the first page in the contiguous chunk; | |
205 | masking relevant for the very first SG entry, | |
206 | which might be unaligned */ | |
207 | page = chunk_start & MASK_4K; | |
208 | do { | |
919fc274 | 209 | pages[cur_page++] = page; |
8dfa0876 | 210 | page += SIZE_4K; |
c1ccaf24 | 211 | } while (page < end_addr); |
6461f64a | 212 | } |
c1ccaf24 | 213 | |
919fc274 SG |
214 | *data_size = total_sz; |
215 | iser_dbg("page_vec->data_size:%d cur_page %d\n", | |
216 | *data_size, cur_page); | |
6461f64a OG |
217 | return cur_page; |
218 | } | |
219 | ||
6461f64a OG |
220 | |
221 | /** | |
222 | * iser_data_buf_aligned_len - Tries to determine the maximal correctly aligned | |
223 | * for RDMA sub-list of a scatter-gather list of memory buffers, and returns | |
224 | * the number of entries which are aligned correctly. Supports the case where | |
225 | * consecutive SG elements are actually fragments of the same physcial page. | |
226 | */ | |
c1ccaf24 OG |
227 | static int iser_data_buf_aligned_len(struct iser_data_buf *data, |
228 | struct ib_device *ibdev) | |
6461f64a | 229 | { |
c1ccaf24 OG |
230 | struct scatterlist *sgl, *sg, *next_sg = NULL; |
231 | u64 start_addr, end_addr; | |
232 | int i, ret_len, start_check = 0; | |
233 | ||
234 | if (data->dma_nents == 1) | |
235 | return 1; | |
6461f64a | 236 | |
53d412fc | 237 | sgl = (struct scatterlist *)data->buf; |
c1ccaf24 | 238 | start_addr = ib_sg_dma_address(ibdev, sgl); |
6461f64a | 239 | |
53d412fc | 240 | for_each_sg(sgl, sg, data->dma_nents, i) { |
c1ccaf24 OG |
241 | if (start_check && !IS_4K_ALIGNED(start_addr)) |
242 | break; | |
243 | ||
244 | next_sg = sg_next(sg); | |
245 | if (!next_sg) | |
246 | break; | |
247 | ||
248 | end_addr = start_addr + ib_sg_dma_len(ibdev, sg); | |
249 | start_addr = ib_sg_dma_address(ibdev, next_sg); | |
250 | ||
251 | if (end_addr == start_addr) { | |
252 | start_check = 0; | |
253 | continue; | |
254 | } else | |
255 | start_check = 1; | |
256 | ||
257 | if (!IS_4K_ALIGNED(end_addr)) | |
258 | break; | |
6461f64a | 259 | } |
c1ccaf24 | 260 | ret_len = (next_sg) ? i : i+1; |
6461f64a OG |
261 | iser_dbg("Found %d aligned entries out of %d in sg:0x%p\n", |
262 | ret_len, data->dma_nents, data); | |
263 | return ret_len; | |
264 | } | |
265 | ||
5180311f RC |
266 | static void iser_data_buf_dump(struct iser_data_buf *data, |
267 | struct ib_device *ibdev) | |
6461f64a | 268 | { |
53d412fc JA |
269 | struct scatterlist *sgl = (struct scatterlist *)data->buf; |
270 | struct scatterlist *sg; | |
6461f64a OG |
271 | int i; |
272 | ||
53d412fc | 273 | for_each_sg(sgl, sg, data->dma_nents, i) |
f91424cf | 274 | iser_dbg("sg[%d] dma_addr:0x%lX page:0x%p " |
e981f1d4 | 275 | "off:0x%x sz:0x%x dma_len:0x%x\n", |
53d412fc | 276 | i, (unsigned long)ib_sg_dma_address(ibdev, sg), |
45711f1a | 277 | sg_page(sg), sg->offset, |
53d412fc | 278 | sg->length, ib_sg_dma_len(ibdev, sg)); |
6461f64a OG |
279 | } |
280 | ||
281 | static void iser_dump_page_vec(struct iser_page_vec *page_vec) | |
282 | { | |
283 | int i; | |
284 | ||
285 | iser_err("page vec length %d data size %d\n", | |
286 | page_vec->length, page_vec->data_size); | |
287 | for (i = 0; i < page_vec->length; i++) | |
288 | iser_err("%d %lx\n",i,(unsigned long)page_vec->pages[i]); | |
289 | } | |
290 | ||
291 | static void iser_page_vec_build(struct iser_data_buf *data, | |
5180311f RC |
292 | struct iser_page_vec *page_vec, |
293 | struct ib_device *ibdev) | |
6461f64a OG |
294 | { |
295 | int page_vec_len = 0; | |
296 | ||
297 | page_vec->length = 0; | |
298 | page_vec->offset = 0; | |
299 | ||
300 | iser_dbg("Translating sg sz: %d\n", data->dma_nents); | |
919fc274 SG |
301 | page_vec_len = iser_sg_to_page_vec(data, ibdev, page_vec->pages, |
302 | &page_vec->offset, | |
303 | &page_vec->data_size); | |
304 | iser_dbg("sg len %d page_vec_len %d\n", data->dma_nents, page_vec_len); | |
6461f64a OG |
305 | |
306 | page_vec->length = page_vec_len; | |
307 | ||
8dfa0876 | 308 | if (page_vec_len * SIZE_4K < page_vec->data_size) { |
6461f64a | 309 | iser_err("page_vec too short to hold this SG\n"); |
5180311f | 310 | iser_data_buf_dump(data, ibdev); |
6461f64a OG |
311 | iser_dump_page_vec(page_vec); |
312 | BUG(); | |
313 | } | |
314 | } | |
315 | ||
2261ec3d MC |
316 | int iser_dma_map_task_data(struct iscsi_iser_task *iser_task, |
317 | struct iser_data_buf *data, | |
318 | enum iser_data_dir iser_dir, | |
319 | enum dma_data_direction dma_dir) | |
74a20780 | 320 | { |
5180311f | 321 | struct ib_device *dev; |
74a20780 | 322 | |
2261ec3d | 323 | iser_task->dir[iser_dir] = 1; |
a4ee3539 | 324 | dev = iser_task->iser_conn->ib_conn.device->ib_device; |
74a20780 | 325 | |
5180311f | 326 | data->dma_nents = ib_dma_map_sg(dev, data->buf, data->size, dma_dir); |
74a20780 EZ |
327 | if (data->dma_nents == 0) { |
328 | iser_err("dma_map_sg failed!!!\n"); | |
329 | return -EINVAL; | |
330 | } | |
331 | return 0; | |
332 | } | |
333 | ||
9a8b08fa SG |
334 | void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task, |
335 | struct iser_data_buf *data) | |
74a20780 | 336 | { |
5180311f | 337 | struct ib_device *dev; |
74a20780 | 338 | |
a4ee3539 | 339 | dev = iser_task->iser_conn->ib_conn.device->ib_device; |
9a8b08fa | 340 | ib_dma_unmap_sg(dev, data->buf, data->size, DMA_FROM_DEVICE); |
74a20780 EZ |
341 | } |
342 | ||
919fc274 SG |
343 | static int fall_to_bounce_buf(struct iscsi_iser_task *iser_task, |
344 | struct ib_device *ibdev, | |
5f588e3d SG |
345 | struct iser_data_buf *mem, |
346 | struct iser_data_buf *mem_copy, | |
919fc274 SG |
347 | enum iser_data_dir cmd_dir, |
348 | int aligned_len) | |
349 | { | |
5716af6e | 350 | struct iscsi_conn *iscsi_conn = iser_task->iser_conn->iscsi_conn; |
919fc274 SG |
351 | |
352 | iscsi_conn->fmr_unalign_cnt++; | |
353 | iser_warn("rdma alignment violation (%d/%d aligned) or FMR not supported\n", | |
354 | aligned_len, mem->size); | |
355 | ||
356 | if (iser_debug_level > 0) | |
357 | iser_data_buf_dump(mem, ibdev); | |
358 | ||
359 | /* unmap the command data before accessing it */ | |
5f588e3d | 360 | iser_dma_unmap_task_data(iser_task, mem); |
919fc274 SG |
361 | |
362 | /* allocate copy buf, if we are writing, copy the */ | |
363 | /* unaligned scatterlist, dma map the copy */ | |
5f588e3d SG |
364 | if (iser_start_rdma_unaligned_sg(iser_task, mem, mem_copy, cmd_dir) != 0) |
365 | return -ENOMEM; | |
919fc274 SG |
366 | |
367 | return 0; | |
368 | } | |
369 | ||
6461f64a | 370 | /** |
e657571b SG |
371 | * iser_reg_rdma_mem_fmr - Registers memory intended for RDMA, |
372 | * using FMR (if possible) obtaining rkey and va | |
6461f64a OG |
373 | * |
374 | * returns 0 on success, errno code on failure | |
375 | */ | |
e657571b SG |
376 | int iser_reg_rdma_mem_fmr(struct iscsi_iser_task *iser_task, |
377 | enum iser_data_dir cmd_dir) | |
6461f64a | 378 | { |
a4ee3539 SG |
379 | struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; |
380 | struct iser_device *device = ib_conn->device; | |
5180311f | 381 | struct ib_device *ibdev = device->ib_device; |
2261ec3d | 382 | struct iser_data_buf *mem = &iser_task->data[cmd_dir]; |
6461f64a OG |
383 | struct iser_regd_buf *regd_buf; |
384 | int aligned_len; | |
385 | int err; | |
e981f1d4 | 386 | int i; |
d8111028 | 387 | struct scatterlist *sg; |
6461f64a | 388 | |
2261ec3d | 389 | regd_buf = &iser_task->rdma_regd[cmd_dir]; |
6461f64a | 390 | |
5180311f | 391 | aligned_len = iser_data_buf_aligned_len(mem, ibdev); |
5587856c | 392 | if (aligned_len != mem->dma_nents) { |
5f588e3d SG |
393 | err = fall_to_bounce_buf(iser_task, ibdev, mem, |
394 | &iser_task->data_copy[cmd_dir], | |
919fc274 SG |
395 | cmd_dir, aligned_len); |
396 | if (err) { | |
397 | iser_err("failed to allocate bounce buffer\n"); | |
398 | return err; | |
399 | } | |
2261ec3d | 400 | mem = &iser_task->data_copy[cmd_dir]; |
6461f64a OG |
401 | } |
402 | ||
d8111028 EZ |
403 | /* if there a single dma entry, FMR is not needed */ |
404 | if (mem->dma_nents == 1) { | |
405 | sg = (struct scatterlist *)mem->buf; | |
406 | ||
407 | regd_buf->reg.lkey = device->mr->lkey; | |
408 | regd_buf->reg.rkey = device->mr->rkey; | |
5180311f RC |
409 | regd_buf->reg.len = ib_sg_dma_len(ibdev, &sg[0]); |
410 | regd_buf->reg.va = ib_sg_dma_address(ibdev, &sg[0]); | |
5587856c | 411 | regd_buf->reg.is_mr = 0; |
d8111028 EZ |
412 | |
413 | iser_dbg("PHYSICAL Mem.register: lkey: 0x%08X rkey: 0x%08X " | |
414 | "va: 0x%08lX sz: %ld]\n", | |
415 | (unsigned int)regd_buf->reg.lkey, | |
416 | (unsigned int)regd_buf->reg.rkey, | |
417 | (unsigned long)regd_buf->reg.va, | |
418 | (unsigned long)regd_buf->reg.len); | |
419 | } else { /* use FMR for multiple dma entries */ | |
a4ee3539 SG |
420 | iser_page_vec_build(mem, ib_conn->fmr.page_vec, ibdev); |
421 | err = iser_reg_page_vec(ib_conn, ib_conn->fmr.page_vec, | |
e657571b | 422 | ®d_buf->reg); |
819a0873 | 423 | if (err && err != -EAGAIN) { |
5180311f | 424 | iser_data_buf_dump(mem, ibdev); |
2261ec3d MC |
425 | iser_err("mem->dma_nents = %d (dlength = 0x%x)\n", |
426 | mem->dma_nents, | |
427 | ntoh24(iser_task->desc.iscsi_header.dlength)); | |
d8111028 | 428 | iser_err("page_vec: data_size = 0x%x, length = %d, offset = 0x%x\n", |
a4ee3539 SG |
429 | ib_conn->fmr.page_vec->data_size, |
430 | ib_conn->fmr.page_vec->length, | |
431 | ib_conn->fmr.page_vec->offset); | |
432 | for (i = 0; i < ib_conn->fmr.page_vec->length; i++) | |
d8111028 | 433 | iser_err("page_vec[%d] = 0x%llx\n", i, |
a4ee3539 | 434 | (unsigned long long)ib_conn->fmr.page_vec->pages[i]); |
e981f1d4 | 435 | } |
450d1e40 OG |
436 | if (err) |
437 | return err; | |
e981f1d4 | 438 | } |
6461f64a OG |
439 | return 0; |
440 | } | |
5587856c | 441 | |
92792c0a SG |
442 | static inline void |
443 | iser_set_dif_domain(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs, | |
444 | struct ib_sig_domain *domain) | |
445 | { | |
78eda2bb | 446 | domain->sig_type = IB_SIG_TYPE_T10_DIF; |
92792c0a SG |
447 | domain->sig.dif.pi_interval = sc->device->sector_size; |
448 | domain->sig.dif.ref_tag = scsi_get_lba(sc) & 0xffffffff; | |
78eda2bb SG |
449 | /* |
450 | * At the moment we hard code those, but in the future | |
451 | * we will take them from sc. | |
452 | */ | |
453 | domain->sig.dif.apptag_check_mask = 0xffff; | |
454 | domain->sig.dif.app_escape = true; | |
455 | domain->sig.dif.ref_escape = true; | |
456 | if (scsi_get_prot_type(sc) == SCSI_PROT_DIF_TYPE1 || | |
457 | scsi_get_prot_type(sc) == SCSI_PROT_DIF_TYPE2) | |
458 | domain->sig.dif.ref_remap = true; | |
92792c0a | 459 | }; |
177e31bd SG |
460 | |
461 | static int | |
462 | iser_set_sig_attrs(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs) | |
463 | { | |
177e31bd SG |
464 | switch (scsi_get_prot_op(sc)) { |
465 | case SCSI_PROT_WRITE_INSERT: | |
466 | case SCSI_PROT_READ_STRIP: | |
78eda2bb | 467 | sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; |
92792c0a | 468 | iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire); |
177e31bd | 469 | sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; |
177e31bd SG |
470 | break; |
471 | case SCSI_PROT_READ_INSERT: | |
472 | case SCSI_PROT_WRITE_STRIP: | |
78eda2bb | 473 | sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; |
92792c0a | 474 | iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem); |
78eda2bb SG |
475 | /* |
476 | * At the moment we use this modparam to tell what is | |
477 | * the memory bg_type, in the future we will take it | |
478 | * from sc. | |
479 | */ | |
92792c0a SG |
480 | sig_attrs->mem.sig.dif.bg_type = iser_pi_guard ? IB_T10DIF_CSUM : |
481 | IB_T10DIF_CRC; | |
177e31bd SG |
482 | break; |
483 | case SCSI_PROT_READ_PASS: | |
484 | case SCSI_PROT_WRITE_PASS: | |
92792c0a | 485 | iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire); |
177e31bd | 486 | sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; |
92792c0a | 487 | iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem); |
78eda2bb SG |
488 | /* |
489 | * At the moment we use this modparam to tell what is | |
490 | * the memory bg_type, in the future we will take it | |
491 | * from sc. | |
492 | */ | |
92792c0a SG |
493 | sig_attrs->mem.sig.dif.bg_type = iser_pi_guard ? IB_T10DIF_CSUM : |
494 | IB_T10DIF_CRC; | |
177e31bd SG |
495 | break; |
496 | default: | |
497 | iser_err("Unsupported PI operation %d\n", | |
498 | scsi_get_prot_op(sc)); | |
499 | return -EINVAL; | |
500 | } | |
78eda2bb | 501 | |
177e31bd SG |
502 | return 0; |
503 | } | |
504 | ||
177e31bd SG |
505 | static int |
506 | iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask) | |
507 | { | |
508 | switch (scsi_get_prot_type(sc)) { | |
509 | case SCSI_PROT_DIF_TYPE0: | |
177e31bd SG |
510 | break; |
511 | case SCSI_PROT_DIF_TYPE1: | |
512 | case SCSI_PROT_DIF_TYPE2: | |
513 | *mask = ISER_CHECK_GUARD | ISER_CHECK_REFTAG; | |
514 | break; | |
515 | case SCSI_PROT_DIF_TYPE3: | |
516 | *mask = ISER_CHECK_GUARD; | |
517 | break; | |
518 | default: | |
519 | iser_err("Unsupported protection type %d\n", | |
520 | scsi_get_prot_type(sc)); | |
521 | return -EINVAL; | |
522 | } | |
523 | ||
524 | return 0; | |
525 | } | |
526 | ||
527 | static int | |
528 | iser_reg_sig_mr(struct iscsi_iser_task *iser_task, | |
529 | struct fast_reg_descriptor *desc, struct ib_sge *data_sge, | |
530 | struct ib_sge *prot_sge, struct ib_sge *sig_sge) | |
531 | { | |
a4ee3539 | 532 | struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; |
177e31bd SG |
533 | struct iser_pi_context *pi_ctx = desc->pi_ctx; |
534 | struct ib_send_wr sig_wr, inv_wr; | |
535 | struct ib_send_wr *bad_wr, *wr = NULL; | |
536 | struct ib_sig_attrs sig_attrs; | |
537 | int ret; | |
538 | u32 key; | |
539 | ||
540 | memset(&sig_attrs, 0, sizeof(sig_attrs)); | |
541 | ret = iser_set_sig_attrs(iser_task->sc, &sig_attrs); | |
542 | if (ret) | |
543 | goto err; | |
544 | ||
545 | ret = iser_set_prot_checks(iser_task->sc, &sig_attrs.check_mask); | |
546 | if (ret) | |
547 | goto err; | |
548 | ||
549 | if (!(desc->reg_indicators & ISER_SIG_KEY_VALID)) { | |
550 | memset(&inv_wr, 0, sizeof(inv_wr)); | |
551 | inv_wr.opcode = IB_WR_LOCAL_INV; | |
552 | inv_wr.wr_id = ISER_FASTREG_LI_WRID; | |
553 | inv_wr.ex.invalidate_rkey = pi_ctx->sig_mr->rkey; | |
554 | wr = &inv_wr; | |
555 | /* Bump the key */ | |
556 | key = (u8)(pi_ctx->sig_mr->rkey & 0x000000FF); | |
557 | ib_update_fast_reg_key(pi_ctx->sig_mr, ++key); | |
558 | } | |
559 | ||
560 | memset(&sig_wr, 0, sizeof(sig_wr)); | |
561 | sig_wr.opcode = IB_WR_REG_SIG_MR; | |
562 | sig_wr.wr_id = ISER_FASTREG_LI_WRID; | |
563 | sig_wr.sg_list = data_sge; | |
564 | sig_wr.num_sge = 1; | |
565 | sig_wr.wr.sig_handover.sig_attrs = &sig_attrs; | |
566 | sig_wr.wr.sig_handover.sig_mr = pi_ctx->sig_mr; | |
567 | if (scsi_prot_sg_count(iser_task->sc)) | |
568 | sig_wr.wr.sig_handover.prot = prot_sge; | |
569 | sig_wr.wr.sig_handover.access_flags = IB_ACCESS_LOCAL_WRITE | | |
570 | IB_ACCESS_REMOTE_READ | | |
571 | IB_ACCESS_REMOTE_WRITE; | |
572 | ||
573 | if (!wr) | |
574 | wr = &sig_wr; | |
575 | else | |
576 | wr->next = &sig_wr; | |
577 | ||
a4ee3539 | 578 | ret = ib_post_send(ib_conn->qp, wr, &bad_wr); |
177e31bd SG |
579 | if (ret) { |
580 | iser_err("reg_sig_mr failed, ret:%d\n", ret); | |
581 | goto err; | |
582 | } | |
583 | desc->reg_indicators &= ~ISER_SIG_KEY_VALID; | |
584 | ||
585 | sig_sge->lkey = pi_ctx->sig_mr->lkey; | |
586 | sig_sge->addr = 0; | |
587 | sig_sge->length = data_sge->length + prot_sge->length; | |
588 | if (scsi_get_prot_op(iser_task->sc) == SCSI_PROT_WRITE_INSERT || | |
589 | scsi_get_prot_op(iser_task->sc) == SCSI_PROT_READ_STRIP) { | |
590 | sig_sge->length += (data_sge->length / | |
591 | iser_task->sc->device->sector_size) * 8; | |
592 | } | |
593 | ||
594 | iser_dbg("sig_sge: addr: 0x%llx length: %u lkey: 0x%x\n", | |
595 | sig_sge->addr, sig_sge->length, | |
596 | sig_sge->lkey); | |
597 | err: | |
598 | return ret; | |
599 | } | |
600 | ||
d11ec4ec | 601 | static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task, |
5587856c | 602 | struct iser_regd_buf *regd_buf, |
d11ec4ec | 603 | struct iser_data_buf *mem, |
177e31bd | 604 | enum iser_reg_indicator ind, |
d11ec4ec | 605 | struct ib_sge *sge) |
5587856c | 606 | { |
d11ec4ec | 607 | struct fast_reg_descriptor *desc = regd_buf->reg.mem_h; |
a4ee3539 SG |
608 | struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; |
609 | struct iser_device *device = ib_conn->device; | |
d11ec4ec | 610 | struct ib_device *ibdev = device->ib_device; |
177e31bd SG |
611 | struct ib_mr *mr; |
612 | struct ib_fast_reg_page_list *frpl; | |
5587856c SG |
613 | struct ib_send_wr fastreg_wr, inv_wr; |
614 | struct ib_send_wr *bad_wr, *wr = NULL; | |
615 | u8 key; | |
d11ec4ec SG |
616 | int ret, offset, size, plen; |
617 | ||
618 | /* if there a single dma entry, dma mr suffices */ | |
619 | if (mem->dma_nents == 1) { | |
620 | struct scatterlist *sg = (struct scatterlist *)mem->buf; | |
621 | ||
622 | sge->lkey = device->mr->lkey; | |
623 | sge->addr = ib_sg_dma_address(ibdev, &sg[0]); | |
624 | sge->length = ib_sg_dma_len(ibdev, &sg[0]); | |
625 | ||
626 | iser_dbg("Single DMA entry: lkey=0x%x, addr=0x%llx, length=0x%x\n", | |
627 | sge->lkey, sge->addr, sge->length); | |
628 | return 0; | |
629 | } | |
630 | ||
177e31bd SG |
631 | if (ind == ISER_DATA_KEY_VALID) { |
632 | mr = desc->data_mr; | |
633 | frpl = desc->data_frpl; | |
634 | } else { | |
635 | mr = desc->pi_ctx->prot_mr; | |
636 | frpl = desc->pi_ctx->prot_frpl; | |
637 | } | |
638 | ||
639 | plen = iser_sg_to_page_vec(mem, device->ib_device, frpl->page_list, | |
d11ec4ec SG |
640 | &offset, &size); |
641 | if (plen * SIZE_4K < size) { | |
642 | iser_err("fast reg page_list too short to hold this SG\n"); | |
643 | return -EINVAL; | |
644 | } | |
5587856c | 645 | |
177e31bd | 646 | if (!(desc->reg_indicators & ind)) { |
5587856c | 647 | memset(&inv_wr, 0, sizeof(inv_wr)); |
7306b8fa | 648 | inv_wr.wr_id = ISER_FASTREG_LI_WRID; |
5587856c | 649 | inv_wr.opcode = IB_WR_LOCAL_INV; |
177e31bd | 650 | inv_wr.ex.invalidate_rkey = mr->rkey; |
5587856c SG |
651 | wr = &inv_wr; |
652 | /* Bump the key */ | |
177e31bd SG |
653 | key = (u8)(mr->rkey & 0x000000FF); |
654 | ib_update_fast_reg_key(mr, ++key); | |
5587856c SG |
655 | } |
656 | ||
657 | /* Prepare FASTREG WR */ | |
658 | memset(&fastreg_wr, 0, sizeof(fastreg_wr)); | |
7306b8fa | 659 | fastreg_wr.wr_id = ISER_FASTREG_LI_WRID; |
5587856c | 660 | fastreg_wr.opcode = IB_WR_FAST_REG_MR; |
177e31bd SG |
661 | fastreg_wr.wr.fast_reg.iova_start = frpl->page_list[0] + offset; |
662 | fastreg_wr.wr.fast_reg.page_list = frpl; | |
d11ec4ec | 663 | fastreg_wr.wr.fast_reg.page_list_len = plen; |
5587856c | 664 | fastreg_wr.wr.fast_reg.page_shift = SHIFT_4K; |
d11ec4ec | 665 | fastreg_wr.wr.fast_reg.length = size; |
177e31bd | 666 | fastreg_wr.wr.fast_reg.rkey = mr->rkey; |
5587856c SG |
667 | fastreg_wr.wr.fast_reg.access_flags = (IB_ACCESS_LOCAL_WRITE | |
668 | IB_ACCESS_REMOTE_WRITE | | |
669 | IB_ACCESS_REMOTE_READ); | |
670 | ||
db523b8d | 671 | if (!wr) |
5587856c | 672 | wr = &fastreg_wr; |
db523b8d | 673 | else |
5587856c | 674 | wr->next = &fastreg_wr; |
5587856c | 675 | |
a4ee3539 | 676 | ret = ib_post_send(ib_conn->qp, wr, &bad_wr); |
5587856c | 677 | if (ret) { |
5587856c SG |
678 | iser_err("fast registration failed, ret:%d\n", ret); |
679 | return ret; | |
680 | } | |
177e31bd | 681 | desc->reg_indicators &= ~ind; |
5587856c | 682 | |
177e31bd SG |
683 | sge->lkey = mr->lkey; |
684 | sge->addr = frpl->page_list[0] + offset; | |
d11ec4ec | 685 | sge->length = size; |
5587856c SG |
686 | |
687 | return ret; | |
688 | } | |
689 | ||
690 | /** | |
7306b8fa | 691 | * iser_reg_rdma_mem_fastreg - Registers memory intended for RDMA, |
5587856c SG |
692 | * using Fast Registration WR (if possible) obtaining rkey and va |
693 | * | |
694 | * returns 0 on success, errno code on failure | |
695 | */ | |
7306b8fa SG |
696 | int iser_reg_rdma_mem_fastreg(struct iscsi_iser_task *iser_task, |
697 | enum iser_data_dir cmd_dir) | |
5587856c | 698 | { |
a4ee3539 SG |
699 | struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; |
700 | struct iser_device *device = ib_conn->device; | |
5587856c SG |
701 | struct ib_device *ibdev = device->ib_device; |
702 | struct iser_data_buf *mem = &iser_task->data[cmd_dir]; | |
703 | struct iser_regd_buf *regd_buf = &iser_task->rdma_regd[cmd_dir]; | |
d11ec4ec SG |
704 | struct fast_reg_descriptor *desc = NULL; |
705 | struct ib_sge data_sge; | |
5587856c SG |
706 | int err, aligned_len; |
707 | unsigned long flags; | |
5587856c SG |
708 | |
709 | aligned_len = iser_data_buf_aligned_len(mem, ibdev); | |
710 | if (aligned_len != mem->dma_nents) { | |
5f588e3d SG |
711 | err = fall_to_bounce_buf(iser_task, ibdev, mem, |
712 | &iser_task->data_copy[cmd_dir], | |
5587856c SG |
713 | cmd_dir, aligned_len); |
714 | if (err) { | |
715 | iser_err("failed to allocate bounce buffer\n"); | |
716 | return err; | |
717 | } | |
718 | mem = &iser_task->data_copy[cmd_dir]; | |
719 | } | |
720 | ||
177e31bd SG |
721 | if (mem->dma_nents != 1 || |
722 | scsi_get_prot_op(iser_task->sc) != SCSI_PROT_NORMAL) { | |
a4ee3539 SG |
723 | spin_lock_irqsave(&ib_conn->lock, flags); |
724 | desc = list_first_entry(&ib_conn->fastreg.pool, | |
5587856c SG |
725 | struct fast_reg_descriptor, list); |
726 | list_del(&desc->list); | |
a4ee3539 | 727 | spin_unlock_irqrestore(&ib_conn->lock, flags); |
d11ec4ec SG |
728 | regd_buf->reg.mem_h = desc; |
729 | } | |
5587856c | 730 | |
177e31bd SG |
731 | err = iser_fast_reg_mr(iser_task, regd_buf, mem, |
732 | ISER_DATA_KEY_VALID, &data_sge); | |
d11ec4ec SG |
733 | if (err) |
734 | goto err_reg; | |
735 | ||
177e31bd SG |
736 | if (scsi_get_prot_op(iser_task->sc) != SCSI_PROT_NORMAL) { |
737 | struct ib_sge prot_sge, sig_sge; | |
738 | ||
739 | memset(&prot_sge, 0, sizeof(prot_sge)); | |
740 | if (scsi_prot_sg_count(iser_task->sc)) { | |
741 | mem = &iser_task->prot[cmd_dir]; | |
742 | aligned_len = iser_data_buf_aligned_len(mem, ibdev); | |
743 | if (aligned_len != mem->dma_nents) { | |
744 | err = fall_to_bounce_buf(iser_task, ibdev, mem, | |
745 | &iser_task->prot_copy[cmd_dir], | |
746 | cmd_dir, aligned_len); | |
747 | if (err) { | |
748 | iser_err("failed to allocate bounce buffer\n"); | |
749 | return err; | |
750 | } | |
751 | mem = &iser_task->prot_copy[cmd_dir]; | |
752 | } | |
753 | ||
754 | err = iser_fast_reg_mr(iser_task, regd_buf, mem, | |
755 | ISER_PROT_KEY_VALID, &prot_sge); | |
756 | if (err) | |
757 | goto err_reg; | |
758 | } | |
759 | ||
760 | err = iser_reg_sig_mr(iser_task, desc, &data_sge, | |
761 | &prot_sge, &sig_sge); | |
762 | if (err) { | |
763 | iser_err("Failed to register signature mr\n"); | |
764 | return err; | |
765 | } | |
766 | desc->reg_indicators |= ISER_FASTREG_PROTECTED; | |
767 | ||
768 | regd_buf->reg.lkey = sig_sge.lkey; | |
769 | regd_buf->reg.rkey = desc->pi_ctx->sig_mr->rkey; | |
770 | regd_buf->reg.va = sig_sge.addr; | |
771 | regd_buf->reg.len = sig_sge.length; | |
d11ec4ec SG |
772 | regd_buf->reg.is_mr = 1; |
773 | } else { | |
177e31bd SG |
774 | if (desc) { |
775 | regd_buf->reg.rkey = desc->data_mr->rkey; | |
776 | regd_buf->reg.is_mr = 1; | |
777 | } else { | |
778 | regd_buf->reg.rkey = device->mr->rkey; | |
779 | regd_buf->reg.is_mr = 0; | |
780 | } | |
5587856c | 781 | |
177e31bd SG |
782 | regd_buf->reg.lkey = data_sge.lkey; |
783 | regd_buf->reg.va = data_sge.addr; | |
784 | regd_buf->reg.len = data_sge.length; | |
785 | } | |
d11ec4ec | 786 | |
5587856c SG |
787 | return 0; |
788 | err_reg: | |
d11ec4ec | 789 | if (desc) { |
a4ee3539 SG |
790 | spin_lock_irqsave(&ib_conn->lock, flags); |
791 | list_add_tail(&desc->list, &ib_conn->fastreg.pool); | |
792 | spin_unlock_irqrestore(&ib_conn->lock, flags); | |
d11ec4ec SG |
793 | } |
794 | ||
5587856c SG |
795 | return err; |
796 | } |