]>
git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - drivers/infiniband/hw/cxgb4/mem.c
6791fd16272c46f1d23efa3f585994aa0fd7a554
2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/module.h>
34 #include <linux/moduleparam.h>
35 #include <rdma/ib_umem.h>
36 #include <linux/atomic.h>
41 module_param(use_dsgl
, int, 0644);
42 MODULE_PARM_DESC(use_dsgl
, "Use DSGL for PBL/FastReg (default=0)");
44 #define T4_ULPTX_MIN_IO 32
45 #define C4IW_MAX_INLINE_SIZE 96
46 #define T4_ULPTX_MAX_DMA 1024
47 #define C4IW_INLINE_THRESHOLD 128
49 static int inline_threshold
= C4IW_INLINE_THRESHOLD
;
50 module_param(inline_threshold
, int, 0644);
51 MODULE_PARM_DESC(inline_threshold
, "inline vs dsgl threshold (default=128)");
53 static int mr_exceeds_hw_limits(struct c4iw_dev
*dev
, u64 length
)
55 return (is_t4(dev
->rdev
.lldi
.adapter_type
) ||
56 is_t5(dev
->rdev
.lldi
.adapter_type
)) &&
57 length
>= 8*1024*1024*1024ULL;
60 static int _c4iw_write_mem_dma_aligned(struct c4iw_rdev
*rdev
, u32 addr
,
61 u32 len
, dma_addr_t data
, int wait
)
64 struct ulp_mem_io
*req
;
65 struct ulptx_sgl
*sgl
;
68 struct c4iw_wr_wait wr_wait
;
73 c4iw_init_wr_wait(&wr_wait
);
74 wr_len
= roundup(sizeof(*req
) + sizeof(*sgl
), 16);
76 skb
= alloc_skb(wr_len
, GFP_KERNEL
| __GFP_NOFAIL
);
79 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, 0);
81 req
= (struct ulp_mem_io
*)__skb_put(skb
, wr_len
);
82 memset(req
, 0, wr_len
);
83 INIT_ULPTX_WR(req
, wr_len
, 0, 0);
84 req
->wr
.wr_hi
= cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR
) |
85 (wait
? FW_WR_COMPL_F
: 0));
86 req
->wr
.wr_lo
= wait
? (__force __be64
)(unsigned long) &wr_wait
: 0L;
87 req
->wr
.wr_mid
= cpu_to_be32(FW_WR_LEN16_V(DIV_ROUND_UP(wr_len
, 16)));
88 req
->cmd
= cpu_to_be32(ULPTX_CMD_V(ULP_TX_MEM_WRITE
));
89 req
->cmd
|= cpu_to_be32(T5_ULP_MEMIO_ORDER_V(1));
90 req
->dlen
= cpu_to_be32(ULP_MEMIO_DATA_LEN_V(len
>>5));
91 req
->len16
= cpu_to_be32(DIV_ROUND_UP(wr_len
-sizeof(req
->wr
), 16));
92 req
->lock_addr
= cpu_to_be32(ULP_MEMIO_ADDR_V(addr
));
94 sgl
= (struct ulptx_sgl
*)(req
+ 1);
95 sgl
->cmd_nsge
= cpu_to_be32(ULPTX_CMD_V(ULP_TX_SC_DSGL
) |
97 sgl
->len0
= cpu_to_be32(len
);
98 sgl
->addr0
= cpu_to_be64(data
);
100 ret
= c4iw_ofld_send(rdev
, skb
);
104 ret
= c4iw_wait_for_reply(rdev
, &wr_wait
, 0, 0, __func__
);
108 static int _c4iw_write_mem_inline(struct c4iw_rdev
*rdev
, u32 addr
, u32 len
,
112 struct ulp_mem_io
*req
;
113 struct ulptx_idata
*sc
;
114 u8 wr_len
, *to_dp
, *from_dp
;
115 int copy_len
, num_wqe
, i
, ret
= 0;
116 struct c4iw_wr_wait wr_wait
;
117 __be32 cmd
= cpu_to_be32(ULPTX_CMD_V(ULP_TX_MEM_WRITE
));
119 if (is_t4(rdev
->lldi
.adapter_type
))
120 cmd
|= cpu_to_be32(ULP_MEMIO_ORDER_F
);
122 cmd
|= cpu_to_be32(T5_ULP_MEMIO_IMM_F
);
125 PDBG("%s addr 0x%x len %u\n", __func__
, addr
, len
);
126 num_wqe
= DIV_ROUND_UP(len
, C4IW_MAX_INLINE_SIZE
);
127 c4iw_init_wr_wait(&wr_wait
);
128 for (i
= 0; i
< num_wqe
; i
++) {
130 copy_len
= len
> C4IW_MAX_INLINE_SIZE
? C4IW_MAX_INLINE_SIZE
:
132 wr_len
= roundup(sizeof *req
+ sizeof *sc
+
133 roundup(copy_len
, T4_ULPTX_MIN_IO
), 16);
135 skb
= alloc_skb(wr_len
, GFP_KERNEL
);
138 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, 0);
140 req
= (struct ulp_mem_io
*)__skb_put(skb
, wr_len
);
141 memset(req
, 0, wr_len
);
142 INIT_ULPTX_WR(req
, wr_len
, 0, 0);
144 if (i
== (num_wqe
-1)) {
145 req
->wr
.wr_hi
= cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR
) |
147 req
->wr
.wr_lo
= (__force __be64
)(unsigned long) &wr_wait
;
149 req
->wr
.wr_hi
= cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR
));
150 req
->wr
.wr_mid
= cpu_to_be32(
151 FW_WR_LEN16_V(DIV_ROUND_UP(wr_len
, 16)));
154 req
->dlen
= cpu_to_be32(ULP_MEMIO_DATA_LEN_V(
155 DIV_ROUND_UP(copy_len
, T4_ULPTX_MIN_IO
)));
156 req
->len16
= cpu_to_be32(DIV_ROUND_UP(wr_len
-sizeof(req
->wr
),
158 req
->lock_addr
= cpu_to_be32(ULP_MEMIO_ADDR_V(addr
+ i
* 3));
160 sc
= (struct ulptx_idata
*)(req
+ 1);
161 sc
->cmd_more
= cpu_to_be32(ULPTX_CMD_V(ULP_TX_SC_IMM
));
162 sc
->len
= cpu_to_be32(roundup(copy_len
, T4_ULPTX_MIN_IO
));
164 to_dp
= (u8
*)(sc
+ 1);
165 from_dp
= (u8
*)data
+ i
* C4IW_MAX_INLINE_SIZE
;
167 memcpy(to_dp
, from_dp
, copy_len
);
169 memset(to_dp
, 0, copy_len
);
170 if (copy_len
% T4_ULPTX_MIN_IO
)
171 memset(to_dp
+ copy_len
, 0, T4_ULPTX_MIN_IO
-
172 (copy_len
% T4_ULPTX_MIN_IO
));
173 ret
= c4iw_ofld_send(rdev
, skb
);
176 len
-= C4IW_MAX_INLINE_SIZE
;
179 ret
= c4iw_wait_for_reply(rdev
, &wr_wait
, 0, 0, __func__
);
183 static int _c4iw_write_mem_dma(struct c4iw_rdev
*rdev
, u32 addr
, u32 len
, void *data
)
191 daddr
= dma_map_single(&rdev
->lldi
.pdev
->dev
, data
, len
, DMA_TO_DEVICE
);
192 if (dma_mapping_error(&rdev
->lldi
.pdev
->dev
, daddr
))
196 while (remain
> inline_threshold
) {
197 if (remain
< T4_ULPTX_MAX_DMA
) {
198 if (remain
& ~T4_ULPTX_MIN_IO
)
199 dmalen
= remain
& ~(T4_ULPTX_MIN_IO
-1);
203 dmalen
= T4_ULPTX_MAX_DMA
;
205 ret
= _c4iw_write_mem_dma_aligned(rdev
, addr
, dmalen
, daddr
,
214 ret
= _c4iw_write_mem_inline(rdev
, addr
, remain
, data
);
216 dma_unmap_single(&rdev
->lldi
.pdev
->dev
, save
, len
, DMA_TO_DEVICE
);
221 * write len bytes of data into addr (32B aligned address)
222 * If data is NULL, clear len byte of memory to zero.
224 static int write_adapter_mem(struct c4iw_rdev
*rdev
, u32 addr
, u32 len
,
227 if (is_t5(rdev
->lldi
.adapter_type
) && use_dsgl
) {
228 if (len
> inline_threshold
) {
229 if (_c4iw_write_mem_dma(rdev
, addr
, len
, data
)) {
230 printk_ratelimited(KERN_WARNING
232 " failure (non fatal)\n",
233 pci_name(rdev
->lldi
.pdev
));
234 return _c4iw_write_mem_inline(rdev
, addr
, len
,
239 return _c4iw_write_mem_inline(rdev
, addr
, len
, data
);
241 return _c4iw_write_mem_inline(rdev
, addr
, len
, data
);
245 * Build and write a TPT entry.
246 * IN: stag key, pdid, perm, bind_enabled, zbva, to, len, page_size,
247 * pbl_size and pbl_addr
250 static int write_tpt_entry(struct c4iw_rdev
*rdev
, u32 reset_tpt_entry
,
251 u32
*stag
, u8 stag_state
, u32 pdid
,
252 enum fw_ri_stag_type type
, enum fw_ri_mem_perms perm
,
253 int bind_enabled
, u32 zbva
, u64 to
,
254 u64 len
, u8 page_size
, u32 pbl_size
, u32 pbl_addr
)
257 struct fw_ri_tpte tpt
;
261 if (c4iw_fatal_error(rdev
))
264 stag_state
= stag_state
> 0;
265 stag_idx
= (*stag
) >> 8;
267 if ((!reset_tpt_entry
) && (*stag
== T4_STAG_UNSET
)) {
268 stag_idx
= c4iw_get_resource(&rdev
->resource
.tpt_table
);
270 mutex_lock(&rdev
->stats
.lock
);
271 rdev
->stats
.stag
.fail
++;
272 mutex_unlock(&rdev
->stats
.lock
);
275 mutex_lock(&rdev
->stats
.lock
);
276 rdev
->stats
.stag
.cur
+= 32;
277 if (rdev
->stats
.stag
.cur
> rdev
->stats
.stag
.max
)
278 rdev
->stats
.stag
.max
= rdev
->stats
.stag
.cur
;
279 mutex_unlock(&rdev
->stats
.lock
);
280 *stag
= (stag_idx
<< 8) | (atomic_inc_return(&key
) & 0xff);
282 PDBG("%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x\n",
283 __func__
, stag_state
, type
, pdid
, stag_idx
);
285 /* write TPT entry */
287 memset(&tpt
, 0, sizeof(tpt
));
289 tpt
.valid_to_pdid
= cpu_to_be32(FW_RI_TPTE_VALID_F
|
290 FW_RI_TPTE_STAGKEY_V((*stag
& FW_RI_TPTE_STAGKEY_M
)) |
291 FW_RI_TPTE_STAGSTATE_V(stag_state
) |
292 FW_RI_TPTE_STAGTYPE_V(type
) | FW_RI_TPTE_PDID_V(pdid
));
293 tpt
.locread_to_qpid
= cpu_to_be32(FW_RI_TPTE_PERM_V(perm
) |
294 (bind_enabled
? FW_RI_TPTE_MWBINDEN_F
: 0) |
295 FW_RI_TPTE_ADDRTYPE_V((zbva
? FW_RI_ZERO_BASED_TO
:
297 FW_RI_TPTE_PS_V(page_size
));
298 tpt
.nosnoop_pbladdr
= !pbl_size
? 0 : cpu_to_be32(
299 FW_RI_TPTE_PBLADDR_V(PBL_OFF(rdev
, pbl_addr
)>>3));
300 tpt
.len_lo
= cpu_to_be32((u32
)(len
& 0xffffffffUL
));
301 tpt
.va_hi
= cpu_to_be32((u32
)(to
>> 32));
302 tpt
.va_lo_fbo
= cpu_to_be32((u32
)(to
& 0xffffffffUL
));
303 tpt
.dca_mwbcnt_pstag
= cpu_to_be32(0);
304 tpt
.len_hi
= cpu_to_be32((u32
)(len
>> 32));
306 err
= write_adapter_mem(rdev
, stag_idx
+
307 (rdev
->lldi
.vr
->stag
.start
>> 5),
310 if (reset_tpt_entry
) {
311 c4iw_put_resource(&rdev
->resource
.tpt_table
, stag_idx
);
312 mutex_lock(&rdev
->stats
.lock
);
313 rdev
->stats
.stag
.cur
-= 32;
314 mutex_unlock(&rdev
->stats
.lock
);
319 static int write_pbl(struct c4iw_rdev
*rdev
, __be64
*pbl
,
320 u32 pbl_addr
, u32 pbl_size
)
324 PDBG("%s *pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d\n",
325 __func__
, pbl_addr
, rdev
->lldi
.vr
->pbl
.start
,
328 err
= write_adapter_mem(rdev
, pbl_addr
>> 5, pbl_size
<< 3, pbl
);
332 static int dereg_mem(struct c4iw_rdev
*rdev
, u32 stag
, u32 pbl_size
,
335 return write_tpt_entry(rdev
, 1, &stag
, 0, 0, 0, 0, 0, 0, 0UL, 0, 0,
339 static int allocate_window(struct c4iw_rdev
*rdev
, u32
* stag
, u32 pdid
)
341 *stag
= T4_STAG_UNSET
;
342 return write_tpt_entry(rdev
, 0, stag
, 0, pdid
, FW_RI_STAG_MW
, 0, 0, 0,
346 static int deallocate_window(struct c4iw_rdev
*rdev
, u32 stag
)
348 return write_tpt_entry(rdev
, 1, &stag
, 0, 0, 0, 0, 0, 0, 0UL, 0, 0, 0,
352 static int allocate_stag(struct c4iw_rdev
*rdev
, u32
*stag
, u32 pdid
,
353 u32 pbl_size
, u32 pbl_addr
)
355 *stag
= T4_STAG_UNSET
;
356 return write_tpt_entry(rdev
, 0, stag
, 0, pdid
, FW_RI_STAG_NSMR
, 0, 0, 0,
357 0UL, 0, 0, pbl_size
, pbl_addr
);
360 static int finish_mem_reg(struct c4iw_mr
*mhp
, u32 stag
)
365 mhp
->attr
.stag
= stag
;
367 mhp
->ibmr
.rkey
= mhp
->ibmr
.lkey
= stag
;
368 PDBG("%s mmid 0x%x mhp %p\n", __func__
, mmid
, mhp
);
369 return insert_handle(mhp
->rhp
, &mhp
->rhp
->mmidr
, mhp
, mmid
);
372 static int register_mem(struct c4iw_dev
*rhp
, struct c4iw_pd
*php
,
373 struct c4iw_mr
*mhp
, int shift
)
375 u32 stag
= T4_STAG_UNSET
;
378 ret
= write_tpt_entry(&rhp
->rdev
, 0, &stag
, 1, mhp
->attr
.pdid
,
379 FW_RI_STAG_NSMR
, mhp
->attr
.len
?
381 mhp
->attr
.mw_bind_enable
, mhp
->attr
.zbva
,
382 mhp
->attr
.va_fbo
, mhp
->attr
.len
?
383 mhp
->attr
.len
: -1, shift
- 12,
384 mhp
->attr
.pbl_size
, mhp
->attr
.pbl_addr
);
388 ret
= finish_mem_reg(mhp
, stag
);
390 dereg_mem(&rhp
->rdev
, mhp
->attr
.stag
, mhp
->attr
.pbl_size
,
395 static int reregister_mem(struct c4iw_dev
*rhp
, struct c4iw_pd
*php
,
396 struct c4iw_mr
*mhp
, int shift
, int npages
)
401 if (npages
> mhp
->attr
.pbl_size
)
404 stag
= mhp
->attr
.stag
;
405 ret
= write_tpt_entry(&rhp
->rdev
, 0, &stag
, 1, mhp
->attr
.pdid
,
406 FW_RI_STAG_NSMR
, mhp
->attr
.perms
,
407 mhp
->attr
.mw_bind_enable
, mhp
->attr
.zbva
,
408 mhp
->attr
.va_fbo
, mhp
->attr
.len
, shift
- 12,
409 mhp
->attr
.pbl_size
, mhp
->attr
.pbl_addr
);
413 ret
= finish_mem_reg(mhp
, stag
);
415 dereg_mem(&rhp
->rdev
, mhp
->attr
.stag
, mhp
->attr
.pbl_size
,
421 static int alloc_pbl(struct c4iw_mr
*mhp
, int npages
)
423 mhp
->attr
.pbl_addr
= c4iw_pblpool_alloc(&mhp
->rhp
->rdev
,
426 if (!mhp
->attr
.pbl_addr
)
429 mhp
->attr
.pbl_size
= npages
;
434 static int build_phys_page_list(struct ib_phys_buf
*buffer_list
,
435 int num_phys_buf
, u64
*iova_start
,
436 u64
*total_size
, int *npages
,
437 int *shift
, __be64
**page_list
)
444 for (i
= 0; i
< num_phys_buf
; ++i
) {
445 if (i
!= 0 && buffer_list
[i
].addr
& ~PAGE_MASK
)
447 if (i
!= 0 && i
!= num_phys_buf
- 1 &&
448 (buffer_list
[i
].size
& ~PAGE_MASK
))
450 *total_size
+= buffer_list
[i
].size
;
452 mask
|= buffer_list
[i
].addr
;
454 mask
|= buffer_list
[i
].addr
& PAGE_MASK
;
455 if (i
!= num_phys_buf
- 1)
456 mask
|= buffer_list
[i
].addr
+ buffer_list
[i
].size
;
458 mask
|= (buffer_list
[i
].addr
+ buffer_list
[i
].size
+
459 PAGE_SIZE
- 1) & PAGE_MASK
;
462 if (*total_size
> 0xFFFFFFFFULL
)
465 /* Find largest page shift we can use to cover buffers */
466 for (*shift
= PAGE_SHIFT
; *shift
< 27; ++(*shift
))
467 if ((1ULL << *shift
) & mask
)
470 buffer_list
[0].size
+= buffer_list
[0].addr
& ((1ULL << *shift
) - 1);
471 buffer_list
[0].addr
&= ~0ull << *shift
;
474 for (i
= 0; i
< num_phys_buf
; ++i
)
475 *npages
+= (buffer_list
[i
].size
+
476 (1ULL << *shift
) - 1) >> *shift
;
481 *page_list
= kmalloc(sizeof(u64
) * *npages
, GFP_KERNEL
);
486 for (i
= 0; i
< num_phys_buf
; ++i
)
488 j
< (buffer_list
[i
].size
+ (1ULL << *shift
) - 1) >> *shift
;
490 (*page_list
)[n
++] = cpu_to_be64(buffer_list
[i
].addr
+
491 ((u64
) j
<< *shift
));
493 PDBG("%s va 0x%llx mask 0x%llx shift %d len %lld pbl_size %d\n",
494 __func__
, (unsigned long long)*iova_start
,
495 (unsigned long long)mask
, *shift
, (unsigned long long)*total_size
,
502 int c4iw_reregister_phys_mem(struct ib_mr
*mr
, int mr_rereg_mask
,
503 struct ib_pd
*pd
, struct ib_phys_buf
*buffer_list
,
504 int num_phys_buf
, int acc
, u64
*iova_start
)
507 struct c4iw_mr mh
, *mhp
;
509 struct c4iw_dev
*rhp
;
510 __be64
*page_list
= NULL
;
516 PDBG("%s ib_mr %p ib_pd %p\n", __func__
, mr
, pd
);
518 /* There can be no memory windows */
519 if (atomic_read(&mr
->usecnt
))
522 mhp
= to_c4iw_mr(mr
);
524 php
= to_c4iw_pd(mr
->pd
);
526 /* make sure we are on the same adapter */
530 memcpy(&mh
, mhp
, sizeof *mhp
);
532 if (mr_rereg_mask
& IB_MR_REREG_PD
)
533 php
= to_c4iw_pd(pd
);
534 if (mr_rereg_mask
& IB_MR_REREG_ACCESS
) {
535 mh
.attr
.perms
= c4iw_ib_to_tpt_access(acc
);
536 mh
.attr
.mw_bind_enable
= (acc
& IB_ACCESS_MW_BIND
) ==
539 if (mr_rereg_mask
& IB_MR_REREG_TRANS
) {
540 ret
= build_phys_page_list(buffer_list
, num_phys_buf
,
542 &total_size
, &npages
,
548 if (mr_exceeds_hw_limits(rhp
, total_size
)) {
553 ret
= reregister_mem(rhp
, php
, &mh
, shift
, npages
);
557 if (mr_rereg_mask
& IB_MR_REREG_PD
)
558 mhp
->attr
.pdid
= php
->pdid
;
559 if (mr_rereg_mask
& IB_MR_REREG_ACCESS
)
560 mhp
->attr
.perms
= c4iw_ib_to_tpt_access(acc
);
561 if (mr_rereg_mask
& IB_MR_REREG_TRANS
) {
563 mhp
->attr
.va_fbo
= *iova_start
;
564 mhp
->attr
.page_size
= shift
- 12;
565 mhp
->attr
.len
= (u32
) total_size
;
566 mhp
->attr
.pbl_size
= npages
;
572 struct ib_mr
*c4iw_register_phys_mem(struct ib_pd
*pd
,
573 struct ib_phys_buf
*buffer_list
,
574 int num_phys_buf
, int acc
, u64
*iova_start
)
580 struct c4iw_dev
*rhp
;
585 PDBG("%s ib_pd %p\n", __func__
, pd
);
586 php
= to_c4iw_pd(pd
);
589 mhp
= kzalloc(sizeof(*mhp
), GFP_KERNEL
);
591 return ERR_PTR(-ENOMEM
);
595 /* First check that we have enough alignment */
596 if ((*iova_start
& ~PAGE_MASK
) != (buffer_list
[0].addr
& ~PAGE_MASK
)) {
601 if (num_phys_buf
> 1 &&
602 ((buffer_list
[0].addr
+ buffer_list
[0].size
) & ~PAGE_MASK
)) {
607 ret
= build_phys_page_list(buffer_list
, num_phys_buf
, iova_start
,
608 &total_size
, &npages
, &shift
,
613 if (mr_exceeds_hw_limits(rhp
, total_size
)) {
619 ret
= alloc_pbl(mhp
, npages
);
625 ret
= write_pbl(&mhp
->rhp
->rdev
, page_list
, mhp
->attr
.pbl_addr
,
631 mhp
->attr
.pdid
= php
->pdid
;
634 mhp
->attr
.perms
= c4iw_ib_to_tpt_access(acc
);
635 mhp
->attr
.va_fbo
= *iova_start
;
636 mhp
->attr
.page_size
= shift
- 12;
638 mhp
->attr
.len
= (u32
) total_size
;
639 mhp
->attr
.pbl_size
= npages
;
640 ret
= register_mem(rhp
, php
, mhp
, shift
);
647 c4iw_pblpool_free(&mhp
->rhp
->rdev
, mhp
->attr
.pbl_addr
,
648 mhp
->attr
.pbl_size
<< 3);
656 struct ib_mr
*c4iw_get_dma_mr(struct ib_pd
*pd
, int acc
)
658 struct c4iw_dev
*rhp
;
662 u32 stag
= T4_STAG_UNSET
;
664 PDBG("%s ib_pd %p\n", __func__
, pd
);
665 php
= to_c4iw_pd(pd
);
668 mhp
= kzalloc(sizeof(*mhp
), GFP_KERNEL
);
670 return ERR_PTR(-ENOMEM
);
673 mhp
->attr
.pdid
= php
->pdid
;
674 mhp
->attr
.perms
= c4iw_ib_to_tpt_access(acc
);
675 mhp
->attr
.mw_bind_enable
= (acc
&IB_ACCESS_MW_BIND
) == IB_ACCESS_MW_BIND
;
677 mhp
->attr
.va_fbo
= 0;
678 mhp
->attr
.page_size
= 0;
679 mhp
->attr
.len
= ~0UL;
680 mhp
->attr
.pbl_size
= 0;
682 ret
= write_tpt_entry(&rhp
->rdev
, 0, &stag
, 1, php
->pdid
,
683 FW_RI_STAG_NSMR
, mhp
->attr
.perms
,
684 mhp
->attr
.mw_bind_enable
, 0, 0, ~0UL, 0, 0, 0);
688 ret
= finish_mem_reg(mhp
, stag
);
693 dereg_mem(&rhp
->rdev
, mhp
->attr
.stag
, mhp
->attr
.pbl_size
,
700 struct ib_mr
*c4iw_reg_user_mr(struct ib_pd
*pd
, u64 start
, u64 length
,
701 u64 virt
, int acc
, struct ib_udata
*udata
)
707 struct scatterlist
*sg
;
708 struct c4iw_dev
*rhp
;
712 PDBG("%s ib_pd %p\n", __func__
, pd
);
715 return ERR_PTR(-EINVAL
);
717 if ((length
+ start
) < start
)
718 return ERR_PTR(-EINVAL
);
720 php
= to_c4iw_pd(pd
);
723 if (mr_exceeds_hw_limits(rhp
, length
))
724 return ERR_PTR(-EINVAL
);
726 mhp
= kzalloc(sizeof(*mhp
), GFP_KERNEL
);
728 return ERR_PTR(-ENOMEM
);
732 mhp
->umem
= ib_umem_get(pd
->uobject
->context
, start
, length
, acc
, 0);
733 if (IS_ERR(mhp
->umem
)) {
734 err
= PTR_ERR(mhp
->umem
);
739 shift
= ffs(mhp
->umem
->page_size
) - 1;
742 err
= alloc_pbl(mhp
, n
);
746 pages
= (__be64
*) __get_free_page(GFP_KERNEL
);
754 for_each_sg(mhp
->umem
->sg_head
.sgl
, sg
, mhp
->umem
->nmap
, entry
) {
755 len
= sg_dma_len(sg
) >> shift
;
756 for (k
= 0; k
< len
; ++k
) {
757 pages
[i
++] = cpu_to_be64(sg_dma_address(sg
) +
758 mhp
->umem
->page_size
* k
);
759 if (i
== PAGE_SIZE
/ sizeof *pages
) {
760 err
= write_pbl(&mhp
->rhp
->rdev
,
762 mhp
->attr
.pbl_addr
+ (n
<< 3), i
);
772 err
= write_pbl(&mhp
->rhp
->rdev
, pages
,
773 mhp
->attr
.pbl_addr
+ (n
<< 3), i
);
776 free_page((unsigned long) pages
);
780 mhp
->attr
.pdid
= php
->pdid
;
782 mhp
->attr
.perms
= c4iw_ib_to_tpt_access(acc
);
783 mhp
->attr
.va_fbo
= virt
;
784 mhp
->attr
.page_size
= shift
- 12;
785 mhp
->attr
.len
= length
;
787 err
= register_mem(rhp
, php
, mhp
, shift
);
794 c4iw_pblpool_free(&mhp
->rhp
->rdev
, mhp
->attr
.pbl_addr
,
795 mhp
->attr
.pbl_size
<< 3);
798 ib_umem_release(mhp
->umem
);
803 struct ib_mw
*c4iw_alloc_mw(struct ib_pd
*pd
, enum ib_mw_type type
)
805 struct c4iw_dev
*rhp
;
812 if (type
!= IB_MW_TYPE_1
)
813 return ERR_PTR(-EINVAL
);
815 php
= to_c4iw_pd(pd
);
817 mhp
= kzalloc(sizeof(*mhp
), GFP_KERNEL
);
819 return ERR_PTR(-ENOMEM
);
820 ret
= allocate_window(&rhp
->rdev
, &stag
, php
->pdid
);
826 mhp
->attr
.pdid
= php
->pdid
;
827 mhp
->attr
.type
= FW_RI_STAG_MW
;
828 mhp
->attr
.stag
= stag
;
830 mhp
->ibmw
.rkey
= stag
;
831 if (insert_handle(rhp
, &rhp
->mmidr
, mhp
, mmid
)) {
832 deallocate_window(&rhp
->rdev
, mhp
->attr
.stag
);
834 return ERR_PTR(-ENOMEM
);
836 PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__
, mmid
, mhp
, stag
);
840 int c4iw_dealloc_mw(struct ib_mw
*mw
)
842 struct c4iw_dev
*rhp
;
846 mhp
= to_c4iw_mw(mw
);
848 mmid
= (mw
->rkey
) >> 8;
849 remove_handle(rhp
, &rhp
->mmidr
, mmid
);
850 deallocate_window(&rhp
->rdev
, mhp
->attr
.stag
);
852 PDBG("%s ib_mw %p mmid 0x%x ptr %p\n", __func__
, mw
, mmid
, mhp
);
856 struct ib_mr
*c4iw_alloc_fast_reg_mr(struct ib_pd
*pd
, int pbl_depth
)
858 struct c4iw_dev
*rhp
;
865 php
= to_c4iw_pd(pd
);
867 mhp
= kzalloc(sizeof(*mhp
), GFP_KERNEL
);
874 ret
= alloc_pbl(mhp
, pbl_depth
);
877 mhp
->attr
.pbl_size
= pbl_depth
;
878 ret
= allocate_stag(&rhp
->rdev
, &stag
, php
->pdid
,
879 mhp
->attr
.pbl_size
, mhp
->attr
.pbl_addr
);
882 mhp
->attr
.pdid
= php
->pdid
;
883 mhp
->attr
.type
= FW_RI_STAG_NSMR
;
884 mhp
->attr
.stag
= stag
;
887 mhp
->ibmr
.rkey
= mhp
->ibmr
.lkey
= stag
;
888 if (insert_handle(rhp
, &rhp
->mmidr
, mhp
, mmid
)) {
893 PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__
, mmid
, mhp
, stag
);
896 dereg_mem(&rhp
->rdev
, stag
, mhp
->attr
.pbl_size
,
899 c4iw_pblpool_free(&mhp
->rhp
->rdev
, mhp
->attr
.pbl_addr
,
900 mhp
->attr
.pbl_size
<< 3);
907 struct ib_fast_reg_page_list
*c4iw_alloc_fastreg_pbl(struct ib_device
*device
,
910 struct c4iw_fr_page_list
*c4pl
;
911 struct c4iw_dev
*dev
= to_c4iw_dev(device
);
913 int pll_len
= roundup(page_list_len
* sizeof(u64
), 32);
915 c4pl
= kmalloc(sizeof(*c4pl
), GFP_KERNEL
);
917 return ERR_PTR(-ENOMEM
);
919 c4pl
->ibpl
.page_list
= dma_alloc_coherent(&dev
->rdev
.lldi
.pdev
->dev
,
922 if (!c4pl
->ibpl
.page_list
) {
924 return ERR_PTR(-ENOMEM
);
926 dma_unmap_addr_set(c4pl
, mapping
, dma_addr
);
927 c4pl
->dma_addr
= dma_addr
;
929 c4pl
->pll_len
= pll_len
;
931 PDBG("%s c4pl %p pll_len %u page_list %p dma_addr %pad\n",
932 __func__
, c4pl
, c4pl
->pll_len
, c4pl
->ibpl
.page_list
,
938 void c4iw_free_fastreg_pbl(struct ib_fast_reg_page_list
*ibpl
)
940 struct c4iw_fr_page_list
*c4pl
= to_c4iw_fr_page_list(ibpl
);
942 PDBG("%s c4pl %p pll_len %u page_list %p dma_addr %pad\n",
943 __func__
, c4pl
, c4pl
->pll_len
, c4pl
->ibpl
.page_list
,
946 dma_free_coherent(&c4pl
->dev
->rdev
.lldi
.pdev
->dev
,
948 c4pl
->ibpl
.page_list
, dma_unmap_addr(c4pl
, mapping
));
952 int c4iw_dereg_mr(struct ib_mr
*ib_mr
)
954 struct c4iw_dev
*rhp
;
958 PDBG("%s ib_mr %p\n", __func__
, ib_mr
);
959 /* There can be no memory windows */
960 if (atomic_read(&ib_mr
->usecnt
))
963 mhp
= to_c4iw_mr(ib_mr
);
965 mmid
= mhp
->attr
.stag
>> 8;
966 remove_handle(rhp
, &rhp
->mmidr
, mmid
);
967 dereg_mem(&rhp
->rdev
, mhp
->attr
.stag
, mhp
->attr
.pbl_size
,
969 if (mhp
->attr
.pbl_size
)
970 c4iw_pblpool_free(&mhp
->rhp
->rdev
, mhp
->attr
.pbl_addr
,
971 mhp
->attr
.pbl_size
<< 3);
973 kfree((void *) (unsigned long) mhp
->kva
);
975 ib_umem_release(mhp
->umem
);
976 PDBG("%s mmid 0x%x ptr %p\n", __func__
, mmid
, mhp
);