2 * IBM eServer eHCA Infiniband device driver for Linux on POWER
6 * Authors: Dietmar Decker <ddecker@de.ibm.com>
7 * Christoph Raisch <raisch@de.ibm.com>
9 * Copyright (c) 2005 IBM Corporation
11 * All rights reserved.
13 * This source code is distributed under a dual license of GPL v2.0 and OpenIB
18 * Redistribution and use in source and binary forms, with or without
19 * modification, are permitted provided that the following conditions are met:
21 * Redistributions of source code must retain the above copyright notice, this
22 * list of conditions and the following disclaimer.
24 * Redistributions in binary form must reproduce the above copyright notice,
25 * this list of conditions and the following disclaimer in the documentation
26 * and/or other materials
27 * provided with the distribution.
29 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
30 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
33 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
34 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
35 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
36 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
37 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
38 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
39 * POSSIBILITY OF SUCH DAMAGE.
42 #include <rdma/ib_umem.h>
44 #include <asm/current.h>
46 #include "ehca_iverbs.h"
47 #include "ehca_mrmw.h"
51 #define NUM_CHUNKS(length, chunk_size) \
52 (((length) + (chunk_size - 1)) / (chunk_size))
53 /* max number of rpages (per hcall register_rpages) */
54 #define MAX_RPAGES 512
56 static struct kmem_cache
*mr_cache
;
57 static struct kmem_cache
*mw_cache
;
59 static struct ehca_mr
*ehca_mr_new(void)
63 me
= kmem_cache_zalloc(mr_cache
, GFP_KERNEL
);
65 spin_lock_init(&me
->mrlock
);
67 ehca_gen_err("alloc failed");
72 static void ehca_mr_delete(struct ehca_mr
*me
)
74 kmem_cache_free(mr_cache
, me
);
77 static struct ehca_mw
*ehca_mw_new(void)
81 me
= kmem_cache_zalloc(mw_cache
, GFP_KERNEL
);
83 spin_lock_init(&me
->mwlock
);
85 ehca_gen_err("alloc failed");
90 static void ehca_mw_delete(struct ehca_mw
*me
)
92 kmem_cache_free(mw_cache
, me
);
95 /*----------------------------------------------------------------------*/
97 struct ib_mr
*ehca_get_dma_mr(struct ib_pd
*pd
, int mr_access_flags
)
101 struct ehca_mr
*e_maxmr
;
102 struct ehca_pd
*e_pd
= container_of(pd
, struct ehca_pd
, ib_pd
);
103 struct ehca_shca
*shca
=
104 container_of(pd
->device
, struct ehca_shca
, ib_device
);
107 e_maxmr
= ehca_mr_new();
109 ehca_err(&shca
->ib_device
, "out of memory");
110 ib_mr
= ERR_PTR(-ENOMEM
);
111 goto get_dma_mr_exit0
;
114 ret
= ehca_reg_maxmr(shca
, e_maxmr
, (u64
*)KERNELBASE
,
115 mr_access_flags
, e_pd
,
116 &e_maxmr
->ib
.ib_mr
.lkey
,
117 &e_maxmr
->ib
.ib_mr
.rkey
);
119 ehca_mr_delete(e_maxmr
);
120 ib_mr
= ERR_PTR(ret
);
121 goto get_dma_mr_exit0
;
123 ib_mr
= &e_maxmr
->ib
.ib_mr
;
125 ehca_err(&shca
->ib_device
, "no internal max-MR exist!");
126 ib_mr
= ERR_PTR(-EINVAL
);
127 goto get_dma_mr_exit0
;
132 ehca_err(&shca
->ib_device
, "rc=%lx pd=%p mr_access_flags=%x ",
133 PTR_ERR(ib_mr
), pd
, mr_access_flags
);
135 } /* end ehca_get_dma_mr() */
137 /*----------------------------------------------------------------------*/
139 struct ib_mr
*ehca_reg_phys_mr(struct ib_pd
*pd
,
140 struct ib_phys_buf
*phys_buf_array
,
147 struct ehca_mr
*e_mr
;
148 struct ehca_shca
*shca
=
149 container_of(pd
->device
, struct ehca_shca
, ib_device
);
150 struct ehca_pd
*e_pd
= container_of(pd
, struct ehca_pd
, ib_pd
);
154 if ((num_phys_buf
<= 0) || !phys_buf_array
) {
155 ehca_err(pd
->device
, "bad input values: num_phys_buf=%x "
156 "phys_buf_array=%p", num_phys_buf
, phys_buf_array
);
157 ib_mr
= ERR_PTR(-EINVAL
);
158 goto reg_phys_mr_exit0
;
160 if (((mr_access_flags
& IB_ACCESS_REMOTE_WRITE
) &&
161 !(mr_access_flags
& IB_ACCESS_LOCAL_WRITE
)) ||
162 ((mr_access_flags
& IB_ACCESS_REMOTE_ATOMIC
) &&
163 !(mr_access_flags
& IB_ACCESS_LOCAL_WRITE
))) {
165 * Remote Write Access requires Local Write Access
166 * Remote Atomic Access requires Local Write Access
168 ehca_err(pd
->device
, "bad input values: mr_access_flags=%x",
170 ib_mr
= ERR_PTR(-EINVAL
);
171 goto reg_phys_mr_exit0
;
174 /* check physical buffer list and calculate size */
175 ret
= ehca_mr_chk_buf_and_calc_size(phys_buf_array
, num_phys_buf
,
178 ib_mr
= ERR_PTR(ret
);
179 goto reg_phys_mr_exit0
;
182 (((u64
)iova_start
+ size
) < (u64
)iova_start
)) {
183 ehca_err(pd
->device
, "bad input values: size=%lx iova_start=%p",
185 ib_mr
= ERR_PTR(-EINVAL
);
186 goto reg_phys_mr_exit0
;
189 e_mr
= ehca_mr_new();
191 ehca_err(pd
->device
, "out of memory");
192 ib_mr
= ERR_PTR(-ENOMEM
);
193 goto reg_phys_mr_exit0
;
196 /* register MR on HCA */
197 if (ehca_mr_is_maxmr(size
, iova_start
)) {
198 e_mr
->flags
|= EHCA_MR_FLAG_MAXMR
;
199 ret
= ehca_reg_maxmr(shca
, e_mr
, iova_start
, mr_access_flags
,
200 e_pd
, &e_mr
->ib
.ib_mr
.lkey
,
201 &e_mr
->ib
.ib_mr
.rkey
);
203 ib_mr
= ERR_PTR(ret
);
204 goto reg_phys_mr_exit1
;
207 struct ehca_mr_pginfo pginfo
;
211 num_kpages
= NUM_CHUNKS(((u64
)iova_start
% PAGE_SIZE
) + size
,
213 num_hwpages
= NUM_CHUNKS(((u64
)iova_start
% EHCA_PAGESIZE
) +
214 size
, EHCA_PAGESIZE
);
215 memset(&pginfo
, 0, sizeof(pginfo
));
216 pginfo
.type
= EHCA_MR_PGI_PHYS
;
217 pginfo
.num_kpages
= num_kpages
;
218 pginfo
.num_hwpages
= num_hwpages
;
219 pginfo
.u
.phy
.num_phys_buf
= num_phys_buf
;
220 pginfo
.u
.phy
.phys_buf_array
= phys_buf_array
;
221 pginfo
.next_hwpage
= (((u64
)iova_start
& ~PAGE_MASK
) /
224 ret
= ehca_reg_mr(shca
, e_mr
, iova_start
, size
, mr_access_flags
,
225 e_pd
, &pginfo
, &e_mr
->ib
.ib_mr
.lkey
,
226 &e_mr
->ib
.ib_mr
.rkey
);
228 ib_mr
= ERR_PTR(ret
);
229 goto reg_phys_mr_exit1
;
233 /* successful registration of all pages */
234 return &e_mr
->ib
.ib_mr
;
237 ehca_mr_delete(e_mr
);
240 ehca_err(pd
->device
, "rc=%lx pd=%p phys_buf_array=%p "
241 "num_phys_buf=%x mr_access_flags=%x iova_start=%p",
242 PTR_ERR(ib_mr
), pd
, phys_buf_array
,
243 num_phys_buf
, mr_access_flags
, iova_start
);
245 } /* end ehca_reg_phys_mr() */
247 /*----------------------------------------------------------------------*/
249 struct ib_mr
*ehca_reg_user_mr(struct ib_pd
*pd
, u64 start
, u64 length
,
250 u64 virt
, int mr_access_flags
,
251 struct ib_udata
*udata
)
254 struct ehca_mr
*e_mr
;
255 struct ehca_shca
*shca
=
256 container_of(pd
->device
, struct ehca_shca
, ib_device
);
257 struct ehca_pd
*e_pd
= container_of(pd
, struct ehca_pd
, ib_pd
);
258 struct ehca_mr_pginfo pginfo
;
264 ehca_gen_err("bad pd=%p", pd
);
265 return ERR_PTR(-EFAULT
);
268 if (((mr_access_flags
& IB_ACCESS_REMOTE_WRITE
) &&
269 !(mr_access_flags
& IB_ACCESS_LOCAL_WRITE
)) ||
270 ((mr_access_flags
& IB_ACCESS_REMOTE_ATOMIC
) &&
271 !(mr_access_flags
& IB_ACCESS_LOCAL_WRITE
))) {
273 * Remote Write Access requires Local Write Access
274 * Remote Atomic Access requires Local Write Access
276 ehca_err(pd
->device
, "bad input values: mr_access_flags=%x",
278 ib_mr
= ERR_PTR(-EINVAL
);
279 goto reg_user_mr_exit0
;
282 if (length
== 0 || virt
+ length
< virt
) {
283 ehca_err(pd
->device
, "bad input values: length=%lx "
284 "virt_base=%lx", length
, virt
);
285 ib_mr
= ERR_PTR(-EINVAL
);
286 goto reg_user_mr_exit0
;
289 e_mr
= ehca_mr_new();
291 ehca_err(pd
->device
, "out of memory");
292 ib_mr
= ERR_PTR(-ENOMEM
);
293 goto reg_user_mr_exit0
;
296 e_mr
->umem
= ib_umem_get(pd
->uobject
->context
, start
, length
,
298 if (IS_ERR(e_mr
->umem
)) {
299 ib_mr
= (void *)e_mr
->umem
;
300 goto reg_user_mr_exit1
;
303 if (e_mr
->umem
->page_size
!= PAGE_SIZE
) {
304 ehca_err(pd
->device
, "page size not supported, "
305 "e_mr->umem->page_size=%x", e_mr
->umem
->page_size
);
306 ib_mr
= ERR_PTR(-EINVAL
);
307 goto reg_user_mr_exit2
;
310 /* determine number of MR pages */
311 num_kpages
= NUM_CHUNKS((virt
% PAGE_SIZE
) + length
, PAGE_SIZE
);
312 num_hwpages
= NUM_CHUNKS((virt
% EHCA_PAGESIZE
) + length
,
315 /* register MR on HCA */
316 memset(&pginfo
, 0, sizeof(pginfo
));
317 pginfo
.type
= EHCA_MR_PGI_USER
;
318 pginfo
.num_kpages
= num_kpages
;
319 pginfo
.num_hwpages
= num_hwpages
;
320 pginfo
.u
.usr
.region
= e_mr
->umem
;
321 pginfo
.next_hwpage
= e_mr
->umem
->offset
/ EHCA_PAGESIZE
;
322 pginfo
.u
.usr
.next_chunk
= list_prepare_entry(pginfo
.u
.usr
.next_chunk
,
323 (&e_mr
->umem
->chunk_list
),
326 ret
= ehca_reg_mr(shca
, e_mr
, (u64
*)virt
, length
, mr_access_flags
,
327 e_pd
, &pginfo
, &e_mr
->ib
.ib_mr
.lkey
,
328 &e_mr
->ib
.ib_mr
.rkey
);
330 ib_mr
= ERR_PTR(ret
);
331 goto reg_user_mr_exit2
;
334 /* successful registration of all pages */
335 return &e_mr
->ib
.ib_mr
;
338 ib_umem_release(e_mr
->umem
);
340 ehca_mr_delete(e_mr
);
343 ehca_err(pd
->device
, "rc=%lx pd=%p mr_access_flags=%x"
345 PTR_ERR(ib_mr
), pd
, mr_access_flags
, udata
);
347 } /* end ehca_reg_user_mr() */
349 /*----------------------------------------------------------------------*/
351 int ehca_rereg_phys_mr(struct ib_mr
*mr
,
354 struct ib_phys_buf
*phys_buf_array
,
361 struct ehca_shca
*shca
=
362 container_of(mr
->device
, struct ehca_shca
, ib_device
);
363 struct ehca_mr
*e_mr
= container_of(mr
, struct ehca_mr
, ib
.ib_mr
);
364 struct ehca_pd
*my_pd
= container_of(mr
->pd
, struct ehca_pd
, ib_pd
);
368 struct ehca_pd
*new_pd
;
369 u32 tmp_lkey
, tmp_rkey
;
370 unsigned long sl_flags
;
373 struct ehca_mr_pginfo pginfo
;
374 u32 cur_pid
= current
->tgid
;
376 if (my_pd
->ib_pd
.uobject
&& my_pd
->ib_pd
.uobject
->context
&&
377 (my_pd
->ownpid
!= cur_pid
)) {
378 ehca_err(mr
->device
, "Invalid caller pid=%x ownpid=%x",
379 cur_pid
, my_pd
->ownpid
);
381 goto rereg_phys_mr_exit0
;
384 if (!(mr_rereg_mask
& IB_MR_REREG_TRANS
)) {
385 /* TODO not supported, because PHYP rereg hCall needs pages */
386 ehca_err(mr
->device
, "rereg without IB_MR_REREG_TRANS not "
387 "supported yet, mr_rereg_mask=%x", mr_rereg_mask
);
389 goto rereg_phys_mr_exit0
;
392 if (mr_rereg_mask
& IB_MR_REREG_PD
) {
394 ehca_err(mr
->device
, "rereg with bad pd, pd=%p "
395 "mr_rereg_mask=%x", pd
, mr_rereg_mask
);
397 goto rereg_phys_mr_exit0
;
402 ~(IB_MR_REREG_TRANS
| IB_MR_REREG_PD
| IB_MR_REREG_ACCESS
)) ||
403 (mr_rereg_mask
== 0)) {
405 goto rereg_phys_mr_exit0
;
408 /* check other parameters */
409 if (e_mr
== shca
->maxmr
) {
410 /* should be impossible, however reject to be sure */
411 ehca_err(mr
->device
, "rereg internal max-MR impossible, mr=%p "
412 "shca->maxmr=%p mr->lkey=%x",
413 mr
, shca
->maxmr
, mr
->lkey
);
415 goto rereg_phys_mr_exit0
;
417 if (mr_rereg_mask
& IB_MR_REREG_TRANS
) { /* transl., i.e. addr/size */
418 if (e_mr
->flags
& EHCA_MR_FLAG_FMR
) {
419 ehca_err(mr
->device
, "not supported for FMR, mr=%p "
420 "flags=%x", mr
, e_mr
->flags
);
422 goto rereg_phys_mr_exit0
;
424 if (!phys_buf_array
|| num_phys_buf
<= 0) {
425 ehca_err(mr
->device
, "bad input values mr_rereg_mask=%x"
426 " phys_buf_array=%p num_phys_buf=%x",
427 mr_rereg_mask
, phys_buf_array
, num_phys_buf
);
429 goto rereg_phys_mr_exit0
;
432 if ((mr_rereg_mask
& IB_MR_REREG_ACCESS
) && /* change ACL */
433 (((mr_access_flags
& IB_ACCESS_REMOTE_WRITE
) &&
434 !(mr_access_flags
& IB_ACCESS_LOCAL_WRITE
)) ||
435 ((mr_access_flags
& IB_ACCESS_REMOTE_ATOMIC
) &&
436 !(mr_access_flags
& IB_ACCESS_LOCAL_WRITE
)))) {
438 * Remote Write Access requires Local Write Access
439 * Remote Atomic Access requires Local Write Access
441 ehca_err(mr
->device
, "bad input values: mr_rereg_mask=%x "
442 "mr_access_flags=%x", mr_rereg_mask
, mr_access_flags
);
444 goto rereg_phys_mr_exit0
;
447 /* set requested values dependent on rereg request */
448 spin_lock_irqsave(&e_mr
->mrlock
, sl_flags
);
449 new_start
= e_mr
->start
;
450 new_size
= e_mr
->size
;
452 new_pd
= container_of(mr
->pd
, struct ehca_pd
, ib_pd
);
454 if (mr_rereg_mask
& IB_MR_REREG_TRANS
) {
455 new_start
= iova_start
; /* change address */
456 /* check physical buffer list and calculate size */
457 ret
= ehca_mr_chk_buf_and_calc_size(phys_buf_array
,
458 num_phys_buf
, iova_start
,
461 goto rereg_phys_mr_exit1
;
462 if ((new_size
== 0) ||
463 (((u64
)iova_start
+ new_size
) < (u64
)iova_start
)) {
464 ehca_err(mr
->device
, "bad input values: new_size=%lx "
465 "iova_start=%p", new_size
, iova_start
);
467 goto rereg_phys_mr_exit1
;
469 num_kpages
= NUM_CHUNKS(((u64
)new_start
% PAGE_SIZE
) +
470 new_size
, PAGE_SIZE
);
471 num_hwpages
= NUM_CHUNKS(((u64
)new_start
% EHCA_PAGESIZE
) +
472 new_size
, EHCA_PAGESIZE
);
473 memset(&pginfo
, 0, sizeof(pginfo
));
474 pginfo
.type
= EHCA_MR_PGI_PHYS
;
475 pginfo
.num_kpages
= num_kpages
;
476 pginfo
.num_hwpages
= num_hwpages
;
477 pginfo
.u
.phy
.num_phys_buf
= num_phys_buf
;
478 pginfo
.u
.phy
.phys_buf_array
= phys_buf_array
;
479 pginfo
.next_hwpage
= (((u64
)iova_start
& ~PAGE_MASK
) /
482 if (mr_rereg_mask
& IB_MR_REREG_ACCESS
)
483 new_acl
= mr_access_flags
;
484 if (mr_rereg_mask
& IB_MR_REREG_PD
)
485 new_pd
= container_of(pd
, struct ehca_pd
, ib_pd
);
487 ret
= ehca_rereg_mr(shca
, e_mr
, new_start
, new_size
, new_acl
,
488 new_pd
, &pginfo
, &tmp_lkey
, &tmp_rkey
);
490 goto rereg_phys_mr_exit1
;
492 /* successful reregistration */
493 if (mr_rereg_mask
& IB_MR_REREG_PD
)
499 spin_unlock_irqrestore(&e_mr
->mrlock
, sl_flags
);
502 ehca_err(mr
->device
, "ret=%x mr=%p mr_rereg_mask=%x pd=%p "
503 "phys_buf_array=%p num_phys_buf=%x mr_access_flags=%x "
505 ret
, mr
, mr_rereg_mask
, pd
, phys_buf_array
,
506 num_phys_buf
, mr_access_flags
, iova_start
);
508 } /* end ehca_rereg_phys_mr() */
510 /*----------------------------------------------------------------------*/
512 int ehca_query_mr(struct ib_mr
*mr
, struct ib_mr_attr
*mr_attr
)
516 struct ehca_shca
*shca
=
517 container_of(mr
->device
, struct ehca_shca
, ib_device
);
518 struct ehca_mr
*e_mr
= container_of(mr
, struct ehca_mr
, ib
.ib_mr
);
519 struct ehca_pd
*my_pd
= container_of(mr
->pd
, struct ehca_pd
, ib_pd
);
520 u32 cur_pid
= current
->tgid
;
521 unsigned long sl_flags
;
522 struct ehca_mr_hipzout_parms hipzout
;
524 if (my_pd
->ib_pd
.uobject
&& my_pd
->ib_pd
.uobject
->context
&&
525 (my_pd
->ownpid
!= cur_pid
)) {
526 ehca_err(mr
->device
, "Invalid caller pid=%x ownpid=%x",
527 cur_pid
, my_pd
->ownpid
);
532 if ((e_mr
->flags
& EHCA_MR_FLAG_FMR
)) {
533 ehca_err(mr
->device
, "not supported for FMR, mr=%p e_mr=%p "
534 "e_mr->flags=%x", mr
, e_mr
, e_mr
->flags
);
539 memset(mr_attr
, 0, sizeof(struct ib_mr_attr
));
540 spin_lock_irqsave(&e_mr
->mrlock
, sl_flags
);
542 h_ret
= hipz_h_query_mr(shca
->ipz_hca_handle
, e_mr
, &hipzout
);
543 if (h_ret
!= H_SUCCESS
) {
544 ehca_err(mr
->device
, "hipz_mr_query failed, h_ret=%lx mr=%p "
545 "hca_hndl=%lx mr_hndl=%lx lkey=%x",
546 h_ret
, mr
, shca
->ipz_hca_handle
.handle
,
547 e_mr
->ipz_mr_handle
.handle
, mr
->lkey
);
548 ret
= ehca2ib_return_code(h_ret
);
551 mr_attr
->pd
= mr
->pd
;
552 mr_attr
->device_virt_addr
= hipzout
.vaddr
;
553 mr_attr
->size
= hipzout
.len
;
554 mr_attr
->lkey
= hipzout
.lkey
;
555 mr_attr
->rkey
= hipzout
.rkey
;
556 ehca_mrmw_reverse_map_acl(&hipzout
.acl
, &mr_attr
->mr_access_flags
);
559 spin_unlock_irqrestore(&e_mr
->mrlock
, sl_flags
);
562 ehca_err(mr
->device
, "ret=%x mr=%p mr_attr=%p",
565 } /* end ehca_query_mr() */
567 /*----------------------------------------------------------------------*/
569 int ehca_dereg_mr(struct ib_mr
*mr
)
573 struct ehca_shca
*shca
=
574 container_of(mr
->device
, struct ehca_shca
, ib_device
);
575 struct ehca_mr
*e_mr
= container_of(mr
, struct ehca_mr
, ib
.ib_mr
);
576 struct ehca_pd
*my_pd
= container_of(mr
->pd
, struct ehca_pd
, ib_pd
);
577 u32 cur_pid
= current
->tgid
;
579 if (my_pd
->ib_pd
.uobject
&& my_pd
->ib_pd
.uobject
->context
&&
580 (my_pd
->ownpid
!= cur_pid
)) {
581 ehca_err(mr
->device
, "Invalid caller pid=%x ownpid=%x",
582 cur_pid
, my_pd
->ownpid
);
587 if ((e_mr
->flags
& EHCA_MR_FLAG_FMR
)) {
588 ehca_err(mr
->device
, "not supported for FMR, mr=%p e_mr=%p "
589 "e_mr->flags=%x", mr
, e_mr
, e_mr
->flags
);
592 } else if (e_mr
== shca
->maxmr
) {
593 /* should be impossible, however reject to be sure */
594 ehca_err(mr
->device
, "dereg internal max-MR impossible, mr=%p "
595 "shca->maxmr=%p mr->lkey=%x",
596 mr
, shca
->maxmr
, mr
->lkey
);
601 /* TODO: BUSY: MR still has bound window(s) */
602 h_ret
= hipz_h_free_resource_mr(shca
->ipz_hca_handle
, e_mr
);
603 if (h_ret
!= H_SUCCESS
) {
604 ehca_err(mr
->device
, "hipz_free_mr failed, h_ret=%lx shca=%p "
605 "e_mr=%p hca_hndl=%lx mr_hndl=%lx mr->lkey=%x",
606 h_ret
, shca
, e_mr
, shca
->ipz_hca_handle
.handle
,
607 e_mr
->ipz_mr_handle
.handle
, mr
->lkey
);
608 ret
= ehca2ib_return_code(h_ret
);
613 ib_umem_release(e_mr
->umem
);
615 /* successful deregistration */
616 ehca_mr_delete(e_mr
);
620 ehca_err(mr
->device
, "ret=%x mr=%p", ret
, mr
);
622 } /* end ehca_dereg_mr() */
624 /*----------------------------------------------------------------------*/
626 struct ib_mw
*ehca_alloc_mw(struct ib_pd
*pd
)
630 struct ehca_mw
*e_mw
;
631 struct ehca_pd
*e_pd
= container_of(pd
, struct ehca_pd
, ib_pd
);
632 struct ehca_shca
*shca
=
633 container_of(pd
->device
, struct ehca_shca
, ib_device
);
634 struct ehca_mw_hipzout_parms hipzout
;
636 e_mw
= ehca_mw_new();
638 ib_mw
= ERR_PTR(-ENOMEM
);
642 h_ret
= hipz_h_alloc_resource_mw(shca
->ipz_hca_handle
, e_mw
,
643 e_pd
->fw_pd
, &hipzout
);
644 if (h_ret
!= H_SUCCESS
) {
645 ehca_err(pd
->device
, "hipz_mw_allocate failed, h_ret=%lx "
646 "shca=%p hca_hndl=%lx mw=%p",
647 h_ret
, shca
, shca
->ipz_hca_handle
.handle
, e_mw
);
648 ib_mw
= ERR_PTR(ehca2ib_return_code(h_ret
));
651 /* successful MW allocation */
652 e_mw
->ipz_mw_handle
= hipzout
.handle
;
653 e_mw
->ib_mw
.rkey
= hipzout
.rkey
;
657 ehca_mw_delete(e_mw
);
660 ehca_err(pd
->device
, "rc=%lx pd=%p", PTR_ERR(ib_mw
), pd
);
662 } /* end ehca_alloc_mw() */
664 /*----------------------------------------------------------------------*/
666 int ehca_bind_mw(struct ib_qp
*qp
,
668 struct ib_mw_bind
*mw_bind
)
670 /* TODO: not supported up to now */
671 ehca_gen_err("bind MW currently not supported by HCAD");
674 } /* end ehca_bind_mw() */
676 /*----------------------------------------------------------------------*/
678 int ehca_dealloc_mw(struct ib_mw
*mw
)
681 struct ehca_shca
*shca
=
682 container_of(mw
->device
, struct ehca_shca
, ib_device
);
683 struct ehca_mw
*e_mw
= container_of(mw
, struct ehca_mw
, ib_mw
);
685 h_ret
= hipz_h_free_resource_mw(shca
->ipz_hca_handle
, e_mw
);
686 if (h_ret
!= H_SUCCESS
) {
687 ehca_err(mw
->device
, "hipz_free_mw failed, h_ret=%lx shca=%p "
688 "mw=%p rkey=%x hca_hndl=%lx mw_hndl=%lx",
689 h_ret
, shca
, mw
, mw
->rkey
, shca
->ipz_hca_handle
.handle
,
690 e_mw
->ipz_mw_handle
.handle
);
691 return ehca2ib_return_code(h_ret
);
693 /* successful deallocation */
694 ehca_mw_delete(e_mw
);
696 } /* end ehca_dealloc_mw() */
698 /*----------------------------------------------------------------------*/
700 struct ib_fmr
*ehca_alloc_fmr(struct ib_pd
*pd
,
702 struct ib_fmr_attr
*fmr_attr
)
704 struct ib_fmr
*ib_fmr
;
705 struct ehca_shca
*shca
=
706 container_of(pd
->device
, struct ehca_shca
, ib_device
);
707 struct ehca_pd
*e_pd
= container_of(pd
, struct ehca_pd
, ib_pd
);
708 struct ehca_mr
*e_fmr
;
710 u32 tmp_lkey
, tmp_rkey
;
711 struct ehca_mr_pginfo pginfo
;
713 /* check other parameters */
714 if (((mr_access_flags
& IB_ACCESS_REMOTE_WRITE
) &&
715 !(mr_access_flags
& IB_ACCESS_LOCAL_WRITE
)) ||
716 ((mr_access_flags
& IB_ACCESS_REMOTE_ATOMIC
) &&
717 !(mr_access_flags
& IB_ACCESS_LOCAL_WRITE
))) {
719 * Remote Write Access requires Local Write Access
720 * Remote Atomic Access requires Local Write Access
722 ehca_err(pd
->device
, "bad input values: mr_access_flags=%x",
724 ib_fmr
= ERR_PTR(-EINVAL
);
725 goto alloc_fmr_exit0
;
727 if (mr_access_flags
& IB_ACCESS_MW_BIND
) {
728 ehca_err(pd
->device
, "bad input values: mr_access_flags=%x",
730 ib_fmr
= ERR_PTR(-EINVAL
);
731 goto alloc_fmr_exit0
;
733 if ((fmr_attr
->max_pages
== 0) || (fmr_attr
->max_maps
== 0)) {
734 ehca_err(pd
->device
, "bad input values: fmr_attr->max_pages=%x "
735 "fmr_attr->max_maps=%x fmr_attr->page_shift=%x",
736 fmr_attr
->max_pages
, fmr_attr
->max_maps
,
737 fmr_attr
->page_shift
);
738 ib_fmr
= ERR_PTR(-EINVAL
);
739 goto alloc_fmr_exit0
;
741 if (((1 << fmr_attr
->page_shift
) != EHCA_PAGESIZE
) &&
742 ((1 << fmr_attr
->page_shift
) != PAGE_SIZE
)) {
743 ehca_err(pd
->device
, "unsupported fmr_attr->page_shift=%x",
744 fmr_attr
->page_shift
);
745 ib_fmr
= ERR_PTR(-EINVAL
);
746 goto alloc_fmr_exit0
;
749 e_fmr
= ehca_mr_new();
751 ib_fmr
= ERR_PTR(-ENOMEM
);
752 goto alloc_fmr_exit0
;
754 e_fmr
->flags
|= EHCA_MR_FLAG_FMR
;
756 /* register MR on HCA */
757 memset(&pginfo
, 0, sizeof(pginfo
));
758 ret
= ehca_reg_mr(shca
, e_fmr
, NULL
,
759 fmr_attr
->max_pages
* (1 << fmr_attr
->page_shift
),
760 mr_access_flags
, e_pd
, &pginfo
,
761 &tmp_lkey
, &tmp_rkey
);
763 ib_fmr
= ERR_PTR(ret
);
764 goto alloc_fmr_exit1
;
768 e_fmr
->fmr_page_size
= 1 << fmr_attr
->page_shift
;
769 e_fmr
->fmr_max_pages
= fmr_attr
->max_pages
;
770 e_fmr
->fmr_max_maps
= fmr_attr
->max_maps
;
771 e_fmr
->fmr_map_cnt
= 0;
772 return &e_fmr
->ib
.ib_fmr
;
775 ehca_mr_delete(e_fmr
);
778 ehca_err(pd
->device
, "rc=%lx pd=%p mr_access_flags=%x "
779 "fmr_attr=%p", PTR_ERR(ib_fmr
), pd
,
780 mr_access_flags
, fmr_attr
);
782 } /* end ehca_alloc_fmr() */
784 /*----------------------------------------------------------------------*/
786 int ehca_map_phys_fmr(struct ib_fmr
*fmr
,
792 struct ehca_shca
*shca
=
793 container_of(fmr
->device
, struct ehca_shca
, ib_device
);
794 struct ehca_mr
*e_fmr
= container_of(fmr
, struct ehca_mr
, ib
.ib_fmr
);
795 struct ehca_pd
*e_pd
= container_of(fmr
->pd
, struct ehca_pd
, ib_pd
);
796 struct ehca_mr_pginfo pginfo
;
797 u32 tmp_lkey
, tmp_rkey
;
799 if (!(e_fmr
->flags
& EHCA_MR_FLAG_FMR
)) {
800 ehca_err(fmr
->device
, "not a FMR, e_fmr=%p e_fmr->flags=%x",
801 e_fmr
, e_fmr
->flags
);
803 goto map_phys_fmr_exit0
;
805 ret
= ehca_fmr_check_page_list(e_fmr
, page_list
, list_len
);
807 goto map_phys_fmr_exit0
;
808 if (iova
% e_fmr
->fmr_page_size
) {
809 /* only whole-numbered pages */
810 ehca_err(fmr
->device
, "bad iova, iova=%lx fmr_page_size=%x",
811 iova
, e_fmr
->fmr_page_size
);
813 goto map_phys_fmr_exit0
;
815 if (e_fmr
->fmr_map_cnt
>= e_fmr
->fmr_max_maps
) {
816 /* HCAD does not limit the maps, however trace this anyway */
817 ehca_info(fmr
->device
, "map limit exceeded, fmr=%p "
818 "e_fmr->fmr_map_cnt=%x e_fmr->fmr_max_maps=%x",
819 fmr
, e_fmr
->fmr_map_cnt
, e_fmr
->fmr_max_maps
);
822 memset(&pginfo
, 0, sizeof(pginfo
));
823 pginfo
.type
= EHCA_MR_PGI_FMR
;
824 pginfo
.num_kpages
= list_len
;
825 pginfo
.num_hwpages
= list_len
* (e_fmr
->fmr_page_size
/ EHCA_PAGESIZE
);
826 pginfo
.u
.fmr
.page_list
= page_list
;
827 pginfo
.next_hwpage
= ((iova
& (e_fmr
->fmr_page_size
-1)) /
829 pginfo
.u
.fmr
.fmr_pgsize
= e_fmr
->fmr_page_size
;
831 ret
= ehca_rereg_mr(shca
, e_fmr
, (u64
*)iova
,
832 list_len
* e_fmr
->fmr_page_size
,
833 e_fmr
->acl
, e_pd
, &pginfo
, &tmp_lkey
, &tmp_rkey
);
835 goto map_phys_fmr_exit0
;
837 /* successful reregistration */
838 e_fmr
->fmr_map_cnt
++;
839 e_fmr
->ib
.ib_fmr
.lkey
= tmp_lkey
;
840 e_fmr
->ib
.ib_fmr
.rkey
= tmp_rkey
;
845 ehca_err(fmr
->device
, "ret=%x fmr=%p page_list=%p list_len=%x "
846 "iova=%lx", ret
, fmr
, page_list
, list_len
, iova
);
848 } /* end ehca_map_phys_fmr() */
850 /*----------------------------------------------------------------------*/
852 int ehca_unmap_fmr(struct list_head
*fmr_list
)
855 struct ib_fmr
*ib_fmr
;
856 struct ehca_shca
*shca
= NULL
;
857 struct ehca_shca
*prev_shca
;
858 struct ehca_mr
*e_fmr
;
860 u32 unmap_fmr_cnt
= 0;
862 /* check all FMR belong to same SHCA, and check internal flag */
863 list_for_each_entry(ib_fmr
, fmr_list
, list
) {
866 ehca_gen_err("bad fmr=%p in list", ib_fmr
);
868 goto unmap_fmr_exit0
;
870 shca
= container_of(ib_fmr
->device
, struct ehca_shca
,
872 e_fmr
= container_of(ib_fmr
, struct ehca_mr
, ib
.ib_fmr
);
873 if ((shca
!= prev_shca
) && prev_shca
) {
874 ehca_err(&shca
->ib_device
, "SHCA mismatch, shca=%p "
875 "prev_shca=%p e_fmr=%p",
876 shca
, prev_shca
, e_fmr
);
878 goto unmap_fmr_exit0
;
880 if (!(e_fmr
->flags
& EHCA_MR_FLAG_FMR
)) {
881 ehca_err(&shca
->ib_device
, "not a FMR, e_fmr=%p "
882 "e_fmr->flags=%x", e_fmr
, e_fmr
->flags
);
884 goto unmap_fmr_exit0
;
889 /* loop over all FMRs to unmap */
890 list_for_each_entry(ib_fmr
, fmr_list
, list
) {
892 e_fmr
= container_of(ib_fmr
, struct ehca_mr
, ib
.ib_fmr
);
893 shca
= container_of(ib_fmr
->device
, struct ehca_shca
,
895 ret
= ehca_unmap_one_fmr(shca
, e_fmr
);
897 /* unmap failed, stop unmapping of rest of FMRs */
898 ehca_err(&shca
->ib_device
, "unmap of one FMR failed, "
899 "stop rest, e_fmr=%p num_fmr=%x "
900 "unmap_fmr_cnt=%x lkey=%x", e_fmr
, num_fmr
,
901 unmap_fmr_cnt
, e_fmr
->ib
.ib_fmr
.lkey
);
902 goto unmap_fmr_exit0
;
908 ehca_gen_err("ret=%x fmr_list=%p num_fmr=%x unmap_fmr_cnt=%x",
909 ret
, fmr_list
, num_fmr
, unmap_fmr_cnt
);
911 } /* end ehca_unmap_fmr() */
913 /*----------------------------------------------------------------------*/
915 int ehca_dealloc_fmr(struct ib_fmr
*fmr
)
919 struct ehca_shca
*shca
=
920 container_of(fmr
->device
, struct ehca_shca
, ib_device
);
921 struct ehca_mr
*e_fmr
= container_of(fmr
, struct ehca_mr
, ib
.ib_fmr
);
923 if (!(e_fmr
->flags
& EHCA_MR_FLAG_FMR
)) {
924 ehca_err(fmr
->device
, "not a FMR, e_fmr=%p e_fmr->flags=%x",
925 e_fmr
, e_fmr
->flags
);
930 h_ret
= hipz_h_free_resource_mr(shca
->ipz_hca_handle
, e_fmr
);
931 if (h_ret
!= H_SUCCESS
) {
932 ehca_err(fmr
->device
, "hipz_free_mr failed, h_ret=%lx e_fmr=%p "
933 "hca_hndl=%lx fmr_hndl=%lx fmr->lkey=%x",
934 h_ret
, e_fmr
, shca
->ipz_hca_handle
.handle
,
935 e_fmr
->ipz_mr_handle
.handle
, fmr
->lkey
);
936 ret
= ehca2ib_return_code(h_ret
);
939 /* successful deregistration */
940 ehca_mr_delete(e_fmr
);
945 ehca_err(&shca
->ib_device
, "ret=%x fmr=%p", ret
, fmr
);
947 } /* end ehca_dealloc_fmr() */
949 /*----------------------------------------------------------------------*/
951 int ehca_reg_mr(struct ehca_shca
*shca
,
952 struct ehca_mr
*e_mr
,
956 struct ehca_pd
*e_pd
,
957 struct ehca_mr_pginfo
*pginfo
,
964 struct ehca_mr_hipzout_parms hipzout
;
966 ehca_mrmw_map_acl(acl
, &hipz_acl
);
967 ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl
);
968 if (ehca_use_hp_mr
== 1)
969 hipz_acl
|= 0x00000001;
971 h_ret
= hipz_h_alloc_resource_mr(shca
->ipz_hca_handle
, e_mr
,
972 (u64
)iova_start
, size
, hipz_acl
,
973 e_pd
->fw_pd
, &hipzout
);
974 if (h_ret
!= H_SUCCESS
) {
975 ehca_err(&shca
->ib_device
, "hipz_alloc_mr failed, h_ret=%lx "
976 "hca_hndl=%lx", h_ret
, shca
->ipz_hca_handle
.handle
);
977 ret
= ehca2ib_return_code(h_ret
);
978 goto ehca_reg_mr_exit0
;
981 e_mr
->ipz_mr_handle
= hipzout
.handle
;
983 ret
= ehca_reg_mr_rpages(shca
, e_mr
, pginfo
);
985 goto ehca_reg_mr_exit1
;
987 /* successful registration */
988 e_mr
->num_kpages
= pginfo
->num_kpages
;
989 e_mr
->num_hwpages
= pginfo
->num_hwpages
;
990 e_mr
->start
= iova_start
;
993 *lkey
= hipzout
.lkey
;
994 *rkey
= hipzout
.rkey
;
998 h_ret
= hipz_h_free_resource_mr(shca
->ipz_hca_handle
, e_mr
);
999 if (h_ret
!= H_SUCCESS
) {
1000 ehca_err(&shca
->ib_device
, "h_ret=%lx shca=%p e_mr=%p "
1001 "iova_start=%p size=%lx acl=%x e_pd=%p lkey=%x "
1002 "pginfo=%p num_kpages=%lx num_hwpages=%lx ret=%x",
1003 h_ret
, shca
, e_mr
, iova_start
, size
, acl
, e_pd
,
1004 hipzout
.lkey
, pginfo
, pginfo
->num_kpages
,
1005 pginfo
->num_hwpages
, ret
);
1006 ehca_err(&shca
->ib_device
, "internal error in ehca_reg_mr, "
1011 ehca_err(&shca
->ib_device
, "ret=%x shca=%p e_mr=%p "
1012 "iova_start=%p size=%lx acl=%x e_pd=%p pginfo=%p "
1013 "num_kpages=%lx num_hwpages=%lx",
1014 ret
, shca
, e_mr
, iova_start
, size
, acl
, e_pd
, pginfo
,
1015 pginfo
->num_kpages
, pginfo
->num_hwpages
);
1017 } /* end ehca_reg_mr() */
1019 /*----------------------------------------------------------------------*/
1021 int ehca_reg_mr_rpages(struct ehca_shca
*shca
,
1022 struct ehca_mr
*e_mr
,
1023 struct ehca_mr_pginfo
*pginfo
)
1032 kpage
= ehca_alloc_fw_ctrlblock(GFP_KERNEL
);
1034 ehca_err(&shca
->ib_device
, "kpage alloc failed");
1036 goto ehca_reg_mr_rpages_exit0
;
1039 /* max 512 pages per shot */
1040 for (i
= 0; i
< NUM_CHUNKS(pginfo
->num_hwpages
, MAX_RPAGES
); i
++) {
1042 if (i
== NUM_CHUNKS(pginfo
->num_hwpages
, MAX_RPAGES
) - 1) {
1043 rnum
= pginfo
->num_hwpages
% MAX_RPAGES
; /* last shot */
1045 rnum
= MAX_RPAGES
; /* last shot is full */
1049 ret
= ehca_set_pagebuf(pginfo
, rnum
, kpage
);
1051 ehca_err(&shca
->ib_device
, "ehca_set_pagebuf "
1052 "bad rc, ret=%x rnum=%x kpage=%p",
1054 goto ehca_reg_mr_rpages_exit1
;
1058 rpage
= virt_to_abs(kpage
);
1060 ehca_err(&shca
->ib_device
, "kpage=%p i=%x",
1063 goto ehca_reg_mr_rpages_exit1
;
1068 h_ret
= hipz_h_register_rpage_mr(shca
->ipz_hca_handle
, e_mr
,
1069 0, /* pagesize 4k */
1072 if (i
== NUM_CHUNKS(pginfo
->num_hwpages
, MAX_RPAGES
) - 1) {
1074 * check for 'registration complete'==H_SUCCESS
1075 * and for 'page registered'==H_PAGE_REGISTERED
1077 if (h_ret
!= H_SUCCESS
) {
1078 ehca_err(&shca
->ib_device
, "last "
1079 "hipz_reg_rpage_mr failed, h_ret=%lx "
1080 "e_mr=%p i=%x hca_hndl=%lx mr_hndl=%lx"
1081 " lkey=%x", h_ret
, e_mr
, i
,
1082 shca
->ipz_hca_handle
.handle
,
1083 e_mr
->ipz_mr_handle
.handle
,
1084 e_mr
->ib
.ib_mr
.lkey
);
1085 ret
= ehca2ib_return_code(h_ret
);
1089 } else if (h_ret
!= H_PAGE_REGISTERED
) {
1090 ehca_err(&shca
->ib_device
, "hipz_reg_rpage_mr failed, "
1091 "h_ret=%lx e_mr=%p i=%x lkey=%x hca_hndl=%lx "
1092 "mr_hndl=%lx", h_ret
, e_mr
, i
,
1093 e_mr
->ib
.ib_mr
.lkey
,
1094 shca
->ipz_hca_handle
.handle
,
1095 e_mr
->ipz_mr_handle
.handle
);
1096 ret
= ehca2ib_return_code(h_ret
);
1103 ehca_reg_mr_rpages_exit1
:
1104 ehca_free_fw_ctrlblock(kpage
);
1105 ehca_reg_mr_rpages_exit0
:
1107 ehca_err(&shca
->ib_device
, "ret=%x shca=%p e_mr=%p pginfo=%p "
1108 "num_kpages=%lx num_hwpages=%lx", ret
, shca
, e_mr
,
1109 pginfo
, pginfo
->num_kpages
, pginfo
->num_hwpages
);
1111 } /* end ehca_reg_mr_rpages() */
1113 /*----------------------------------------------------------------------*/
1115 inline int ehca_rereg_mr_rereg1(struct ehca_shca
*shca
,
1116 struct ehca_mr
*e_mr
,
1120 struct ehca_pd
*e_pd
,
1121 struct ehca_mr_pginfo
*pginfo
,
1130 struct ehca_mr_pginfo pginfo_save
;
1131 struct ehca_mr_hipzout_parms hipzout
;
1133 ehca_mrmw_map_acl(acl
, &hipz_acl
);
1134 ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl
);
1136 kpage
= ehca_alloc_fw_ctrlblock(GFP_KERNEL
);
1138 ehca_err(&shca
->ib_device
, "kpage alloc failed");
1140 goto ehca_rereg_mr_rereg1_exit0
;
1143 pginfo_save
= *pginfo
;
1144 ret
= ehca_set_pagebuf(pginfo
, pginfo
->num_hwpages
, kpage
);
1146 ehca_err(&shca
->ib_device
, "set pagebuf failed, e_mr=%p "
1147 "pginfo=%p type=%x num_kpages=%lx num_hwpages=%lx "
1148 "kpage=%p", e_mr
, pginfo
, pginfo
->type
,
1149 pginfo
->num_kpages
, pginfo
->num_hwpages
, kpage
);
1150 goto ehca_rereg_mr_rereg1_exit1
;
1152 rpage
= virt_to_abs(kpage
);
1154 ehca_err(&shca
->ib_device
, "kpage=%p", kpage
);
1156 goto ehca_rereg_mr_rereg1_exit1
;
1158 h_ret
= hipz_h_reregister_pmr(shca
->ipz_hca_handle
, e_mr
,
1159 (u64
)iova_start
, size
, hipz_acl
,
1160 e_pd
->fw_pd
, rpage
, &hipzout
);
1161 if (h_ret
!= H_SUCCESS
) {
1163 * reregistration unsuccessful, try it again with the 3 hCalls,
1164 * e.g. this is required in case H_MR_CONDITION
1165 * (MW bound or MR is shared)
1167 ehca_warn(&shca
->ib_device
, "hipz_h_reregister_pmr failed "
1168 "(Rereg1), h_ret=%lx e_mr=%p", h_ret
, e_mr
);
1169 *pginfo
= pginfo_save
;
1171 } else if ((u64
*)hipzout
.vaddr
!= iova_start
) {
1172 ehca_err(&shca
->ib_device
, "PHYP changed iova_start in "
1173 "rereg_pmr, iova_start=%p iova_start_out=%lx e_mr=%p "
1174 "mr_handle=%lx lkey=%x lkey_out=%x", iova_start
,
1175 hipzout
.vaddr
, e_mr
, e_mr
->ipz_mr_handle
.handle
,
1176 e_mr
->ib
.ib_mr
.lkey
, hipzout
.lkey
);
1180 * successful reregistration
1181 * note: start and start_out are identical for eServer HCAs
1183 e_mr
->num_kpages
= pginfo
->num_kpages
;
1184 e_mr
->num_hwpages
= pginfo
->num_hwpages
;
1185 e_mr
->start
= iova_start
;
1188 *lkey
= hipzout
.lkey
;
1189 *rkey
= hipzout
.rkey
;
1192 ehca_rereg_mr_rereg1_exit1
:
1193 ehca_free_fw_ctrlblock(kpage
);
1194 ehca_rereg_mr_rereg1_exit0
:
1195 if ( ret
&& (ret
!= -EAGAIN
) )
1196 ehca_err(&shca
->ib_device
, "ret=%x lkey=%x rkey=%x "
1197 "pginfo=%p num_kpages=%lx num_hwpages=%lx",
1198 ret
, *lkey
, *rkey
, pginfo
, pginfo
->num_kpages
,
1199 pginfo
->num_hwpages
);
1201 } /* end ehca_rereg_mr_rereg1() */
1203 /*----------------------------------------------------------------------*/
1205 int ehca_rereg_mr(struct ehca_shca
*shca
,
1206 struct ehca_mr
*e_mr
,
1210 struct ehca_pd
*e_pd
,
1211 struct ehca_mr_pginfo
*pginfo
,
1217 int rereg_1_hcall
= 1; /* 1: use hipz_h_reregister_pmr directly */
1218 int rereg_3_hcall
= 0; /* 1: use 3 hipz calls for reregistration */
1220 /* first determine reregistration hCall(s) */
1221 if ((pginfo
->num_hwpages
> MAX_RPAGES
) ||
1222 (e_mr
->num_hwpages
> MAX_RPAGES
) ||
1223 (pginfo
->num_hwpages
> e_mr
->num_hwpages
)) {
1224 ehca_dbg(&shca
->ib_device
, "Rereg3 case, "
1225 "pginfo->num_hwpages=%lx e_mr->num_hwpages=%x",
1226 pginfo
->num_hwpages
, e_mr
->num_hwpages
);
1231 if (e_mr
->flags
& EHCA_MR_FLAG_MAXMR
) { /* check for max-MR */
1234 e_mr
->flags
&= ~EHCA_MR_FLAG_MAXMR
;
1235 ehca_err(&shca
->ib_device
, "Rereg MR for max-MR! e_mr=%p",
1239 if (rereg_1_hcall
) {
1240 ret
= ehca_rereg_mr_rereg1(shca
, e_mr
, iova_start
, size
,
1241 acl
, e_pd
, pginfo
, lkey
, rkey
);
1246 goto ehca_rereg_mr_exit0
;
1250 if (rereg_3_hcall
) {
1251 struct ehca_mr save_mr
;
1253 /* first deregister old MR */
1254 h_ret
= hipz_h_free_resource_mr(shca
->ipz_hca_handle
, e_mr
);
1255 if (h_ret
!= H_SUCCESS
) {
1256 ehca_err(&shca
->ib_device
, "hipz_free_mr failed, "
1257 "h_ret=%lx e_mr=%p hca_hndl=%lx mr_hndl=%lx "
1259 h_ret
, e_mr
, shca
->ipz_hca_handle
.handle
,
1260 e_mr
->ipz_mr_handle
.handle
,
1261 e_mr
->ib
.ib_mr
.lkey
);
1262 ret
= ehca2ib_return_code(h_ret
);
1263 goto ehca_rereg_mr_exit0
;
1265 /* clean ehca_mr_t, without changing struct ib_mr and lock */
1267 ehca_mr_deletenew(e_mr
);
1269 /* set some MR values */
1270 e_mr
->flags
= save_mr
.flags
;
1271 e_mr
->fmr_page_size
= save_mr
.fmr_page_size
;
1272 e_mr
->fmr_max_pages
= save_mr
.fmr_max_pages
;
1273 e_mr
->fmr_max_maps
= save_mr
.fmr_max_maps
;
1274 e_mr
->fmr_map_cnt
= save_mr
.fmr_map_cnt
;
1276 ret
= ehca_reg_mr(shca
, e_mr
, iova_start
, size
, acl
,
1277 e_pd
, pginfo
, lkey
, rkey
);
1279 u32 offset
= (u64
)(&e_mr
->flags
) - (u64
)e_mr
;
1280 memcpy(&e_mr
->flags
, &(save_mr
.flags
),
1281 sizeof(struct ehca_mr
) - offset
);
1282 goto ehca_rereg_mr_exit0
;
1286 ehca_rereg_mr_exit0
:
1288 ehca_err(&shca
->ib_device
, "ret=%x shca=%p e_mr=%p "
1289 "iova_start=%p size=%lx acl=%x e_pd=%p pginfo=%p "
1290 "num_kpages=%lx lkey=%x rkey=%x rereg_1_hcall=%x "
1291 "rereg_3_hcall=%x", ret
, shca
, e_mr
, iova_start
, size
,
1292 acl
, e_pd
, pginfo
, pginfo
->num_kpages
, *lkey
, *rkey
,
1293 rereg_1_hcall
, rereg_3_hcall
);
1295 } /* end ehca_rereg_mr() */
1297 /*----------------------------------------------------------------------*/
1299 int ehca_unmap_one_fmr(struct ehca_shca
*shca
,
1300 struct ehca_mr
*e_fmr
)
1304 struct ehca_pd
*e_pd
=
1305 container_of(e_fmr
->ib
.ib_fmr
.pd
, struct ehca_pd
, ib_pd
);
1306 struct ehca_mr save_fmr
;
1307 u32 tmp_lkey
, tmp_rkey
;
1308 struct ehca_mr_pginfo pginfo
;
1309 struct ehca_mr_hipzout_parms hipzout
;
1310 struct ehca_mr save_mr
;
1312 if (e_fmr
->fmr_max_pages
<= MAX_RPAGES
) {
1314 * note: after using rereg hcall with len=0,
1315 * rereg hcall must be used again for registering pages
1317 h_ret
= hipz_h_reregister_pmr(shca
->ipz_hca_handle
, e_fmr
, 0,
1318 0, 0, e_pd
->fw_pd
, 0, &hipzout
);
1319 if (h_ret
== H_SUCCESS
) {
1320 /* successful reregistration */
1321 e_fmr
->start
= NULL
;
1323 tmp_lkey
= hipzout
.lkey
;
1324 tmp_rkey
= hipzout
.rkey
;
1328 * should not happen, because length checked above,
1329 * FMRs are not shared and no MW bound to FMRs
1331 ehca_err(&shca
->ib_device
, "hipz_reregister_pmr failed "
1332 "(Rereg1), h_ret=%lx e_fmr=%p hca_hndl=%lx "
1333 "mr_hndl=%lx lkey=%x lkey_out=%x",
1334 h_ret
, e_fmr
, shca
->ipz_hca_handle
.handle
,
1335 e_fmr
->ipz_mr_handle
.handle
,
1336 e_fmr
->ib
.ib_fmr
.lkey
, hipzout
.lkey
);
1337 /* try free and rereg */
1340 /* first free old FMR */
1341 h_ret
= hipz_h_free_resource_mr(shca
->ipz_hca_handle
, e_fmr
);
1342 if (h_ret
!= H_SUCCESS
) {
1343 ehca_err(&shca
->ib_device
, "hipz_free_mr failed, "
1344 "h_ret=%lx e_fmr=%p hca_hndl=%lx mr_hndl=%lx "
1346 h_ret
, e_fmr
, shca
->ipz_hca_handle
.handle
,
1347 e_fmr
->ipz_mr_handle
.handle
,
1348 e_fmr
->ib
.ib_fmr
.lkey
);
1349 ret
= ehca2ib_return_code(h_ret
);
1350 goto ehca_unmap_one_fmr_exit0
;
1352 /* clean ehca_mr_t, without changing lock */
1354 ehca_mr_deletenew(e_fmr
);
1356 /* set some MR values */
1357 e_fmr
->flags
= save_fmr
.flags
;
1358 e_fmr
->fmr_page_size
= save_fmr
.fmr_page_size
;
1359 e_fmr
->fmr_max_pages
= save_fmr
.fmr_max_pages
;
1360 e_fmr
->fmr_max_maps
= save_fmr
.fmr_max_maps
;
1361 e_fmr
->fmr_map_cnt
= save_fmr
.fmr_map_cnt
;
1362 e_fmr
->acl
= save_fmr
.acl
;
1364 memset(&pginfo
, 0, sizeof(pginfo
));
1365 pginfo
.type
= EHCA_MR_PGI_FMR
;
1366 pginfo
.num_kpages
= 0;
1367 pginfo
.num_hwpages
= 0;
1368 ret
= ehca_reg_mr(shca
, e_fmr
, NULL
,
1369 (e_fmr
->fmr_max_pages
* e_fmr
->fmr_page_size
),
1370 e_fmr
->acl
, e_pd
, &pginfo
, &tmp_lkey
,
1373 u32 offset
= (u64
)(&e_fmr
->flags
) - (u64
)e_fmr
;
1374 memcpy(&e_fmr
->flags
, &(save_mr
.flags
),
1375 sizeof(struct ehca_mr
) - offset
);
1376 goto ehca_unmap_one_fmr_exit0
;
1379 ehca_unmap_one_fmr_exit0
:
1381 ehca_err(&shca
->ib_device
, "ret=%x tmp_lkey=%x tmp_rkey=%x "
1383 ret
, tmp_lkey
, tmp_rkey
, e_fmr
->fmr_max_pages
);
1385 } /* end ehca_unmap_one_fmr() */
1387 /*----------------------------------------------------------------------*/
1389 int ehca_reg_smr(struct ehca_shca
*shca
,
1390 struct ehca_mr
*e_origmr
,
1391 struct ehca_mr
*e_newmr
,
1394 struct ehca_pd
*e_pd
,
1401 struct ehca_mr_hipzout_parms hipzout
;
1403 ehca_mrmw_map_acl(acl
, &hipz_acl
);
1404 ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl
);
1406 h_ret
= hipz_h_register_smr(shca
->ipz_hca_handle
, e_newmr
, e_origmr
,
1407 (u64
)iova_start
, hipz_acl
, e_pd
->fw_pd
,
1409 if (h_ret
!= H_SUCCESS
) {
1410 ehca_err(&shca
->ib_device
, "hipz_reg_smr failed, h_ret=%lx "
1411 "shca=%p e_origmr=%p e_newmr=%p iova_start=%p acl=%x "
1412 "e_pd=%p hca_hndl=%lx mr_hndl=%lx lkey=%x",
1413 h_ret
, shca
, e_origmr
, e_newmr
, iova_start
, acl
, e_pd
,
1414 shca
->ipz_hca_handle
.handle
,
1415 e_origmr
->ipz_mr_handle
.handle
,
1416 e_origmr
->ib
.ib_mr
.lkey
);
1417 ret
= ehca2ib_return_code(h_ret
);
1418 goto ehca_reg_smr_exit0
;
1420 /* successful registration */
1421 e_newmr
->num_kpages
= e_origmr
->num_kpages
;
1422 e_newmr
->num_hwpages
= e_origmr
->num_hwpages
;
1423 e_newmr
->start
= iova_start
;
1424 e_newmr
->size
= e_origmr
->size
;
1426 e_newmr
->ipz_mr_handle
= hipzout
.handle
;
1427 *lkey
= hipzout
.lkey
;
1428 *rkey
= hipzout
.rkey
;
1433 ehca_err(&shca
->ib_device
, "ret=%x shca=%p e_origmr=%p "
1434 "e_newmr=%p iova_start=%p acl=%x e_pd=%p",
1435 ret
, shca
, e_origmr
, e_newmr
, iova_start
, acl
, e_pd
);
1437 } /* end ehca_reg_smr() */
1439 /*----------------------------------------------------------------------*/
1441 /* register internal max-MR to internal SHCA */
1442 int ehca_reg_internal_maxmr(
1443 struct ehca_shca
*shca
,
1444 struct ehca_pd
*e_pd
,
1445 struct ehca_mr
**e_maxmr
) /*OUT*/
1448 struct ehca_mr
*e_mr
;
1451 struct ehca_mr_pginfo pginfo
;
1452 struct ib_phys_buf ib_pbuf
;
1456 e_mr
= ehca_mr_new();
1458 ehca_err(&shca
->ib_device
, "out of memory");
1460 goto ehca_reg_internal_maxmr_exit0
;
1462 e_mr
->flags
|= EHCA_MR_FLAG_MAXMR
;
1464 /* register internal max-MR on HCA */
1465 size_maxmr
= (u64
)high_memory
- PAGE_OFFSET
;
1466 iova_start
= (u64
*)KERNELBASE
;
1468 ib_pbuf
.size
= size_maxmr
;
1469 num_kpages
= NUM_CHUNKS(((u64
)iova_start
% PAGE_SIZE
) + size_maxmr
,
1471 num_hwpages
= NUM_CHUNKS(((u64
)iova_start
% EHCA_PAGESIZE
) + size_maxmr
,
1474 memset(&pginfo
, 0, sizeof(pginfo
));
1475 pginfo
.type
= EHCA_MR_PGI_PHYS
;
1476 pginfo
.num_kpages
= num_kpages
;
1477 pginfo
.num_hwpages
= num_hwpages
;
1478 pginfo
.u
.phy
.num_phys_buf
= 1;
1479 pginfo
.u
.phy
.phys_buf_array
= &ib_pbuf
;
1481 ret
= ehca_reg_mr(shca
, e_mr
, iova_start
, size_maxmr
, 0, e_pd
,
1482 &pginfo
, &e_mr
->ib
.ib_mr
.lkey
,
1483 &e_mr
->ib
.ib_mr
.rkey
);
1485 ehca_err(&shca
->ib_device
, "reg of internal max MR failed, "
1486 "e_mr=%p iova_start=%p size_maxmr=%lx num_kpages=%x "
1487 "num_hwpages=%x", e_mr
, iova_start
, size_maxmr
,
1488 num_kpages
, num_hwpages
);
1489 goto ehca_reg_internal_maxmr_exit1
;
1492 /* successful registration of all pages */
1493 e_mr
->ib
.ib_mr
.device
= e_pd
->ib_pd
.device
;
1494 e_mr
->ib
.ib_mr
.pd
= &e_pd
->ib_pd
;
1495 e_mr
->ib
.ib_mr
.uobject
= NULL
;
1496 atomic_inc(&(e_pd
->ib_pd
.usecnt
));
1497 atomic_set(&(e_mr
->ib
.ib_mr
.usecnt
), 0);
1501 ehca_reg_internal_maxmr_exit1
:
1502 ehca_mr_delete(e_mr
);
1503 ehca_reg_internal_maxmr_exit0
:
1505 ehca_err(&shca
->ib_device
, "ret=%x shca=%p e_pd=%p e_maxmr=%p",
1506 ret
, shca
, e_pd
, e_maxmr
);
1508 } /* end ehca_reg_internal_maxmr() */
1510 /*----------------------------------------------------------------------*/
1512 int ehca_reg_maxmr(struct ehca_shca
*shca
,
1513 struct ehca_mr
*e_newmr
,
1516 struct ehca_pd
*e_pd
,
1521 struct ehca_mr
*e_origmr
= shca
->maxmr
;
1523 struct ehca_mr_hipzout_parms hipzout
;
1525 ehca_mrmw_map_acl(acl
, &hipz_acl
);
1526 ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl
);
1528 h_ret
= hipz_h_register_smr(shca
->ipz_hca_handle
, e_newmr
, e_origmr
,
1529 (u64
)iova_start
, hipz_acl
, e_pd
->fw_pd
,
1531 if (h_ret
!= H_SUCCESS
) {
1532 ehca_err(&shca
->ib_device
, "hipz_reg_smr failed, h_ret=%lx "
1533 "e_origmr=%p hca_hndl=%lx mr_hndl=%lx lkey=%x",
1534 h_ret
, e_origmr
, shca
->ipz_hca_handle
.handle
,
1535 e_origmr
->ipz_mr_handle
.handle
,
1536 e_origmr
->ib
.ib_mr
.lkey
);
1537 return ehca2ib_return_code(h_ret
);
1539 /* successful registration */
1540 e_newmr
->num_kpages
= e_origmr
->num_kpages
;
1541 e_newmr
->num_hwpages
= e_origmr
->num_hwpages
;
1542 e_newmr
->start
= iova_start
;
1543 e_newmr
->size
= e_origmr
->size
;
1545 e_newmr
->ipz_mr_handle
= hipzout
.handle
;
1546 *lkey
= hipzout
.lkey
;
1547 *rkey
= hipzout
.rkey
;
1549 } /* end ehca_reg_maxmr() */
1551 /*----------------------------------------------------------------------*/
1553 int ehca_dereg_internal_maxmr(struct ehca_shca
*shca
)
1556 struct ehca_mr
*e_maxmr
;
1557 struct ib_pd
*ib_pd
;
1560 ehca_err(&shca
->ib_device
, "bad call, shca=%p", shca
);
1562 goto ehca_dereg_internal_maxmr_exit0
;
1565 e_maxmr
= shca
->maxmr
;
1566 ib_pd
= e_maxmr
->ib
.ib_mr
.pd
;
1567 shca
->maxmr
= NULL
; /* remove internal max-MR indication from SHCA */
1569 ret
= ehca_dereg_mr(&e_maxmr
->ib
.ib_mr
);
1571 ehca_err(&shca
->ib_device
, "dereg internal max-MR failed, "
1572 "ret=%x e_maxmr=%p shca=%p lkey=%x",
1573 ret
, e_maxmr
, shca
, e_maxmr
->ib
.ib_mr
.lkey
);
1574 shca
->maxmr
= e_maxmr
;
1575 goto ehca_dereg_internal_maxmr_exit0
;
1578 atomic_dec(&ib_pd
->usecnt
);
1580 ehca_dereg_internal_maxmr_exit0
:
1582 ehca_err(&shca
->ib_device
, "ret=%x shca=%p shca->maxmr=%p",
1583 ret
, shca
, shca
->maxmr
);
1585 } /* end ehca_dereg_internal_maxmr() */
1587 /*----------------------------------------------------------------------*/
1590 * check physical buffer array of MR verbs for validness and
1591 * calculates MR size
1593 int ehca_mr_chk_buf_and_calc_size(struct ib_phys_buf
*phys_buf_array
,
1598 struct ib_phys_buf
*pbuf
= phys_buf_array
;
1602 if (num_phys_buf
== 0) {
1603 ehca_gen_err("bad phys buf array len, num_phys_buf=0");
1606 /* check first buffer */
1607 if (((u64
)iova_start
& ~PAGE_MASK
) != (pbuf
->addr
& ~PAGE_MASK
)) {
1608 ehca_gen_err("iova_start/addr mismatch, iova_start=%p "
1609 "pbuf->addr=%lx pbuf->size=%lx",
1610 iova_start
, pbuf
->addr
, pbuf
->size
);
1613 if (((pbuf
->addr
+ pbuf
->size
) % PAGE_SIZE
) &&
1614 (num_phys_buf
> 1)) {
1615 ehca_gen_err("addr/size mismatch in 1st buf, pbuf->addr=%lx "
1616 "pbuf->size=%lx", pbuf
->addr
, pbuf
->size
);
1620 for (i
= 0; i
< num_phys_buf
; i
++) {
1621 if ((i
> 0) && (pbuf
->addr
% PAGE_SIZE
)) {
1622 ehca_gen_err("bad address, i=%x pbuf->addr=%lx "
1624 i
, pbuf
->addr
, pbuf
->size
);
1627 if (((i
> 0) && /* not 1st */
1628 (i
< (num_phys_buf
- 1)) && /* not last */
1629 (pbuf
->size
% PAGE_SIZE
)) || (pbuf
->size
== 0)) {
1630 ehca_gen_err("bad size, i=%x pbuf->size=%lx",
1634 size_count
+= pbuf
->size
;
1640 } /* end ehca_mr_chk_buf_and_calc_size() */
1642 /*----------------------------------------------------------------------*/
1644 /* check page list of map FMR verb for validness */
1645 int ehca_fmr_check_page_list(struct ehca_mr
*e_fmr
,
1652 if ((list_len
== 0) || (list_len
> e_fmr
->fmr_max_pages
)) {
1653 ehca_gen_err("bad list_len, list_len=%x "
1654 "e_fmr->fmr_max_pages=%x fmr=%p",
1655 list_len
, e_fmr
->fmr_max_pages
, e_fmr
);
1659 /* each page must be aligned */
1661 for (i
= 0; i
< list_len
; i
++) {
1662 if (*page
% e_fmr
->fmr_page_size
) {
1663 ehca_gen_err("bad page, i=%x *page=%lx page=%p fmr=%p "
1664 "fmr_page_size=%x", i
, *page
, page
, e_fmr
,
1665 e_fmr
->fmr_page_size
);
1672 } /* end ehca_fmr_check_page_list() */
1674 /*----------------------------------------------------------------------*/
1676 /* PAGE_SIZE >= pginfo->hwpage_size */
1677 static int ehca_set_pagebuf_user1(struct ehca_mr_pginfo
*pginfo
,
1682 struct ib_umem_chunk
*prev_chunk
;
1683 struct ib_umem_chunk
*chunk
;
1688 /* loop over desired chunk entries */
1689 chunk
= pginfo
->u
.usr
.next_chunk
;
1690 prev_chunk
= pginfo
->u
.usr
.next_chunk
;
1691 list_for_each_entry_continue(
1692 chunk
, (&(pginfo
->u
.usr
.region
->chunk_list
)), list
) {
1693 for (i
= pginfo
->u
.usr
.next_nmap
; i
< chunk
->nmap
; ) {
1694 pgaddr
= page_to_pfn(chunk
->page_list
[i
].page
)
1696 *kpage
= phys_to_abs(pgaddr
+
1697 (pginfo
->next_hwpage
*
1700 ehca_gen_err("pgaddr=%lx "
1701 "chunk->page_list[i]=%lx "
1702 "i=%x next_hwpage=%lx",
1703 pgaddr
, (u64
)sg_dma_address(
1704 &chunk
->page_list
[i
]),
1705 i
, pginfo
->next_hwpage
);
1708 (pginfo
->hwpage_cnt
)++;
1709 (pginfo
->next_hwpage
)++;
1711 if (pginfo
->next_hwpage
%
1712 (PAGE_SIZE
/ EHCA_PAGESIZE
) == 0) {
1713 (pginfo
->kpage_cnt
)++;
1714 (pginfo
->u
.usr
.next_nmap
)++;
1715 pginfo
->next_hwpage
= 0;
1719 if (j
>= number
) break;
1721 if ((pginfo
->u
.usr
.next_nmap
>= chunk
->nmap
) &&
1723 pginfo
->u
.usr
.next_nmap
= 0;
1726 } else if (pginfo
->u
.usr
.next_nmap
>= chunk
->nmap
) {
1727 pginfo
->u
.usr
.next_nmap
= 0;
1729 } else if (j
>= number
)
1734 pginfo
->u
.usr
.next_chunk
=
1735 list_prepare_entry(prev_chunk
,
1736 (&(pginfo
->u
.usr
.region
->chunk_list
)),
1741 int ehca_set_pagebuf_phys(struct ehca_mr_pginfo
*pginfo
,
1746 struct ib_phys_buf
*pbuf
;
1747 u64 num_hw
, offs_hw
;
1750 /* loop over desired phys_buf_array entries */
1751 while (i
< number
) {
1752 pbuf
= pginfo
->u
.phy
.phys_buf_array
+ pginfo
->u
.phy
.next_buf
;
1753 num_hw
= NUM_CHUNKS((pbuf
->addr
% EHCA_PAGESIZE
) +
1754 pbuf
->size
, EHCA_PAGESIZE
);
1755 offs_hw
= (pbuf
->addr
& ~PAGE_MASK
) / EHCA_PAGESIZE
;
1756 while (pginfo
->next_hwpage
< offs_hw
+ num_hw
) {
1758 if ((pginfo
->kpage_cnt
>= pginfo
->num_kpages
) ||
1759 (pginfo
->hwpage_cnt
>= pginfo
->num_hwpages
)) {
1760 ehca_gen_err("kpage_cnt >= num_kpages, "
1761 "kpage_cnt=%lx num_kpages=%lx "
1763 "num_hwpages=%lx i=%x",
1767 pginfo
->num_hwpages
, i
);
1770 *kpage
= phys_to_abs(
1771 (pbuf
->addr
& EHCA_PAGEMASK
)
1772 + (pginfo
->next_hwpage
* EHCA_PAGESIZE
));
1773 if ( !(*kpage
) && pbuf
->addr
) {
1774 ehca_gen_err("pbuf->addr=%lx "
1776 "next_hwpage=%lx", pbuf
->addr
,
1778 pginfo
->next_hwpage
);
1781 (pginfo
->hwpage_cnt
)++;
1782 (pginfo
->next_hwpage
)++;
1783 if (pginfo
->next_hwpage
%
1784 (PAGE_SIZE
/ EHCA_PAGESIZE
) == 0)
1785 (pginfo
->kpage_cnt
)++;
1788 if (i
>= number
) break;
1790 if (pginfo
->next_hwpage
>= offs_hw
+ num_hw
) {
1791 (pginfo
->u
.phy
.next_buf
)++;
1792 pginfo
->next_hwpage
= 0;
1798 int ehca_set_pagebuf_fmr(struct ehca_mr_pginfo
*pginfo
,
1806 /* loop over desired page_list entries */
1807 fmrlist
= pginfo
->u
.fmr
.page_list
+ pginfo
->u
.fmr
.next_listelem
;
1808 for (i
= 0; i
< number
; i
++) {
1809 *kpage
= phys_to_abs((*fmrlist
& EHCA_PAGEMASK
) +
1810 pginfo
->next_hwpage
* EHCA_PAGESIZE
);
1812 ehca_gen_err("*fmrlist=%lx fmrlist=%p "
1813 "next_listelem=%lx next_hwpage=%lx",
1815 pginfo
->u
.fmr
.next_listelem
,
1816 pginfo
->next_hwpage
);
1819 (pginfo
->hwpage_cnt
)++;
1820 (pginfo
->next_hwpage
)++;
1822 if (pginfo
->next_hwpage
%
1823 (pginfo
->u
.fmr
.fmr_pgsize
/ EHCA_PAGESIZE
) == 0) {
1824 (pginfo
->kpage_cnt
)++;
1825 (pginfo
->u
.fmr
.next_listelem
)++;
1827 pginfo
->next_hwpage
= 0;
1833 /* setup page buffer from page info */
1834 int ehca_set_pagebuf(struct ehca_mr_pginfo
*pginfo
,
1840 switch (pginfo
->type
) {
1841 case EHCA_MR_PGI_PHYS
:
1842 ret
= ehca_set_pagebuf_phys(pginfo
, number
, kpage
);
1844 case EHCA_MR_PGI_USER
:
1845 ret
= ehca_set_pagebuf_user1(pginfo
, number
, kpage
);
1847 case EHCA_MR_PGI_FMR
:
1848 ret
= ehca_set_pagebuf_fmr(pginfo
, number
, kpage
);
1851 ehca_gen_err("bad pginfo->type=%x", pginfo
->type
);
1856 } /* end ehca_set_pagebuf() */
1858 /*----------------------------------------------------------------------*/
1861 * check MR if it is a max-MR, i.e. uses whole memory
1862 * in case it's a max-MR 1 is returned, else 0
1864 int ehca_mr_is_maxmr(u64 size
,
1867 /* a MR is treated as max-MR only if it fits following: */
1868 if ((size
== ((u64
)high_memory
- PAGE_OFFSET
)) &&
1869 (iova_start
== (void *)KERNELBASE
)) {
1870 ehca_gen_dbg("this is a max-MR");
1874 } /* end ehca_mr_is_maxmr() */
1876 /*----------------------------------------------------------------------*/
1878 /* map access control for MR/MW. This routine is used for MR and MW. */
1879 void ehca_mrmw_map_acl(int ib_acl
,
1883 if (ib_acl
& IB_ACCESS_REMOTE_READ
)
1884 *hipz_acl
|= HIPZ_ACCESSCTRL_R_READ
;
1885 if (ib_acl
& IB_ACCESS_REMOTE_WRITE
)
1886 *hipz_acl
|= HIPZ_ACCESSCTRL_R_WRITE
;
1887 if (ib_acl
& IB_ACCESS_REMOTE_ATOMIC
)
1888 *hipz_acl
|= HIPZ_ACCESSCTRL_R_ATOMIC
;
1889 if (ib_acl
& IB_ACCESS_LOCAL_WRITE
)
1890 *hipz_acl
|= HIPZ_ACCESSCTRL_L_WRITE
;
1891 if (ib_acl
& IB_ACCESS_MW_BIND
)
1892 *hipz_acl
|= HIPZ_ACCESSCTRL_MW_BIND
;
1893 } /* end ehca_mrmw_map_acl() */
1895 /*----------------------------------------------------------------------*/
1897 /* sets page size in hipz access control for MR/MW. */
1898 void ehca_mrmw_set_pgsize_hipz_acl(u32
*hipz_acl
) /*INOUT*/
1900 return; /* HCA supports only 4k */
1901 } /* end ehca_mrmw_set_pgsize_hipz_acl() */
1903 /*----------------------------------------------------------------------*/
1906 * reverse map access control for MR/MW.
1907 * This routine is used for MR and MW.
1909 void ehca_mrmw_reverse_map_acl(const u32
*hipz_acl
,
1910 int *ib_acl
) /*OUT*/
1913 if (*hipz_acl
& HIPZ_ACCESSCTRL_R_READ
)
1914 *ib_acl
|= IB_ACCESS_REMOTE_READ
;
1915 if (*hipz_acl
& HIPZ_ACCESSCTRL_R_WRITE
)
1916 *ib_acl
|= IB_ACCESS_REMOTE_WRITE
;
1917 if (*hipz_acl
& HIPZ_ACCESSCTRL_R_ATOMIC
)
1918 *ib_acl
|= IB_ACCESS_REMOTE_ATOMIC
;
1919 if (*hipz_acl
& HIPZ_ACCESSCTRL_L_WRITE
)
1920 *ib_acl
|= IB_ACCESS_LOCAL_WRITE
;
1921 if (*hipz_acl
& HIPZ_ACCESSCTRL_MW_BIND
)
1922 *ib_acl
|= IB_ACCESS_MW_BIND
;
1923 } /* end ehca_mrmw_reverse_map_acl() */
1926 /*----------------------------------------------------------------------*/
1929 * MR destructor and constructor
1930 * used in Reregister MR verb, sets all fields in ehca_mr_t to 0,
1931 * except struct ib_mr and spinlock
1933 void ehca_mr_deletenew(struct ehca_mr
*mr
)
1937 mr
->num_hwpages
= 0;
1940 mr
->fmr_page_size
= 0;
1941 mr
->fmr_max_pages
= 0;
1942 mr
->fmr_max_maps
= 0;
1943 mr
->fmr_map_cnt
= 0;
1944 memset(&mr
->ipz_mr_handle
, 0, sizeof(mr
->ipz_mr_handle
));
1945 memset(&mr
->galpas
, 0, sizeof(mr
->galpas
));
1946 } /* end ehca_mr_deletenew() */
1948 int ehca_init_mrmw_cache(void)
1950 mr_cache
= kmem_cache_create("ehca_cache_mr",
1951 sizeof(struct ehca_mr
), 0,
1956 mw_cache
= kmem_cache_create("ehca_cache_mw",
1957 sizeof(struct ehca_mw
), 0,
1961 kmem_cache_destroy(mr_cache
);
1968 void ehca_cleanup_mrmw_cache(void)
1971 kmem_cache_destroy(mr_cache
);
1973 kmem_cache_destroy(mw_cache
);