2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <rdma/ib_umem.h>
34 #include <rdma/ib_umem_odp.h>
39 #define MAX_PREFETCH_LEN (4*1024*1024U)
41 /* Timeout in ms to wait for an active mmu notifier to complete when handling
43 #define MMU_NOTIFIER_TIMEOUT 1000
45 #define MLX5_IMR_MTT_BITS (30 - PAGE_SHIFT)
46 #define MLX5_IMR_MTT_SHIFT (MLX5_IMR_MTT_BITS + PAGE_SHIFT)
47 #define MLX5_IMR_MTT_ENTRIES BIT_ULL(MLX5_IMR_MTT_BITS)
48 #define MLX5_IMR_MTT_SIZE BIT_ULL(MLX5_IMR_MTT_SHIFT)
49 #define MLX5_IMR_MTT_MASK (~(MLX5_IMR_MTT_SIZE - 1))
51 #define MLX5_KSM_PAGE_SHIFT MLX5_IMR_MTT_SHIFT
53 static u64 mlx5_imr_ksm_entries
;
55 static int check_parent(struct ib_umem_odp
*odp
,
56 struct mlx5_ib_mr
*parent
)
58 struct mlx5_ib_mr
*mr
= odp
->private;
60 return mr
&& mr
->parent
== parent
&& !odp
->dying
;
63 static struct ib_umem_odp
*odp_next(struct ib_umem_odp
*odp
)
65 struct mlx5_ib_mr
*mr
= odp
->private, *parent
= mr
->parent
;
66 struct ib_ucontext
*ctx
= odp
->umem
->context
;
69 down_read(&ctx
->umem_rwsem
);
71 rb
= rb_next(&odp
->interval_tree
.rb
);
74 odp
= rb_entry(rb
, struct ib_umem_odp
, interval_tree
.rb
);
75 if (check_parent(odp
, parent
))
81 up_read(&ctx
->umem_rwsem
);
85 static struct ib_umem_odp
*odp_lookup(struct ib_ucontext
*ctx
,
86 u64 start
, u64 length
,
87 struct mlx5_ib_mr
*parent
)
89 struct ib_umem_odp
*odp
;
92 down_read(&ctx
->umem_rwsem
);
93 odp
= rbt_ib_umem_lookup(&ctx
->umem_tree
, start
, length
);
98 if (check_parent(odp
, parent
))
100 rb
= rb_next(&odp
->interval_tree
.rb
);
103 odp
= rb_entry(rb
, struct ib_umem_odp
, interval_tree
.rb
);
104 if (ib_umem_start(odp
->umem
) > start
+ length
)
110 up_read(&ctx
->umem_rwsem
);
114 void mlx5_odp_populate_klm(struct mlx5_klm
*pklm
, size_t offset
,
115 size_t nentries
, struct mlx5_ib_mr
*mr
, int flags
)
117 struct ib_pd
*pd
= mr
->ibmr
.pd
;
118 struct ib_ucontext
*ctx
= pd
->uobject
->context
;
119 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
120 struct ib_umem_odp
*odp
;
124 if (flags
& MLX5_IB_UPD_XLT_ZAP
) {
125 for (i
= 0; i
< nentries
; i
++, pklm
++) {
126 pklm
->bcount
= cpu_to_be32(MLX5_IMR_MTT_SIZE
);
127 pklm
->key
= cpu_to_be32(dev
->null_mkey
);
133 odp
= odp_lookup(ctx
, offset
* MLX5_IMR_MTT_SIZE
,
134 nentries
* MLX5_IMR_MTT_SIZE
, mr
);
136 for (i
= 0; i
< nentries
; i
++, pklm
++) {
137 pklm
->bcount
= cpu_to_be32(MLX5_IMR_MTT_SIZE
);
138 va
= (offset
+ i
) * MLX5_IMR_MTT_SIZE
;
139 if (odp
&& odp
->umem
->address
== va
) {
140 struct mlx5_ib_mr
*mtt
= odp
->private;
142 pklm
->key
= cpu_to_be32(mtt
->ibmr
.lkey
);
145 pklm
->key
= cpu_to_be32(dev
->null_mkey
);
147 mlx5_ib_dbg(dev
, "[%d] va %lx key %x\n",
148 i
, va
, be32_to_cpu(pklm
->key
));
152 static void mr_leaf_free_action(struct work_struct
*work
)
154 struct ib_umem_odp
*odp
= container_of(work
, struct ib_umem_odp
, work
);
155 int idx
= ib_umem_start(odp
->umem
) >> MLX5_IMR_MTT_SHIFT
;
156 struct mlx5_ib_mr
*mr
= odp
->private, *imr
= mr
->parent
;
159 synchronize_srcu(&mr
->dev
->mr_srcu
);
161 ib_umem_release(odp
->umem
);
163 mlx5_ib_update_xlt(imr
, idx
, 1, 0,
164 MLX5_IB_UPD_XLT_INDIRECT
|
165 MLX5_IB_UPD_XLT_ATOMIC
);
166 mlx5_mr_cache_free(mr
->dev
, mr
);
168 if (atomic_dec_and_test(&imr
->num_leaf_free
))
169 wake_up(&imr
->q_leaf_free
);
172 void mlx5_ib_invalidate_range(struct ib_umem
*umem
, unsigned long start
,
175 struct mlx5_ib_mr
*mr
;
176 const u64 umr_block_mask
= (MLX5_UMR_MTT_ALIGNMENT
/
177 sizeof(struct mlx5_mtt
)) - 1;
178 u64 idx
= 0, blk_start_idx
= 0;
182 if (!umem
|| !umem
->odp_data
) {
183 pr_err("invalidation called on NULL umem or non-ODP umem\n");
187 mr
= umem
->odp_data
->private;
189 if (!mr
|| !mr
->ibmr
.pd
)
192 start
= max_t(u64
, ib_umem_start(umem
), start
);
193 end
= min_t(u64
, ib_umem_end(umem
), end
);
196 * Iteration one - zap the HW's MTTs. The notifiers_count ensures that
197 * while we are doing the invalidation, no page fault will attempt to
198 * overwrite the same MTTs. Concurent invalidations might race us,
199 * but they will write 0s as well, so no difference in the end result.
202 for (addr
= start
; addr
< end
; addr
+= BIT(umem
->page_shift
)) {
203 idx
= (addr
- ib_umem_start(umem
)) >> umem
->page_shift
;
205 * Strive to write the MTTs in chunks, but avoid overwriting
206 * non-existing MTTs. The huristic here can be improved to
207 * estimate the cost of another UMR vs. the cost of bigger
210 if (umem
->odp_data
->dma_list
[idx
] &
211 (ODP_READ_ALLOWED_BIT
| ODP_WRITE_ALLOWED_BIT
)) {
217 u64 umr_offset
= idx
& umr_block_mask
;
219 if (in_block
&& umr_offset
== 0) {
220 mlx5_ib_update_xlt(mr
, blk_start_idx
,
221 idx
- blk_start_idx
, 0,
222 MLX5_IB_UPD_XLT_ZAP
|
223 MLX5_IB_UPD_XLT_ATOMIC
);
229 mlx5_ib_update_xlt(mr
, blk_start_idx
,
230 idx
- blk_start_idx
+ 1, 0,
231 MLX5_IB_UPD_XLT_ZAP
|
232 MLX5_IB_UPD_XLT_ATOMIC
);
234 * We are now sure that the device will not access the
235 * memory. We can safely unmap it, and mark it as dirty if
239 ib_umem_odp_unmap_dma_pages(umem
, start
, end
);
241 if (unlikely(!umem
->npages
&& mr
->parent
&&
242 !umem
->odp_data
->dying
)) {
243 WRITE_ONCE(umem
->odp_data
->dying
, 1);
244 atomic_inc(&mr
->parent
->num_leaf_free
);
245 schedule_work(&umem
->odp_data
->work
);
249 void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev
*dev
)
251 struct ib_odp_caps
*caps
= &dev
->odp_caps
;
253 memset(caps
, 0, sizeof(*caps
));
255 if (!MLX5_CAP_GEN(dev
->mdev
, pg
))
258 caps
->general_caps
= IB_ODP_SUPPORT
;
260 if (MLX5_CAP_GEN(dev
->mdev
, umr_extended_translation_offset
))
261 dev
->odp_max_size
= U64_MAX
;
263 dev
->odp_max_size
= BIT_ULL(MLX5_MAX_UMR_SHIFT
+ PAGE_SHIFT
);
265 if (MLX5_CAP_ODP(dev
->mdev
, ud_odp_caps
.send
))
266 caps
->per_transport_caps
.ud_odp_caps
|= IB_ODP_SUPPORT_SEND
;
268 if (MLX5_CAP_ODP(dev
->mdev
, rc_odp_caps
.send
))
269 caps
->per_transport_caps
.rc_odp_caps
|= IB_ODP_SUPPORT_SEND
;
271 if (MLX5_CAP_ODP(dev
->mdev
, rc_odp_caps
.receive
))
272 caps
->per_transport_caps
.rc_odp_caps
|= IB_ODP_SUPPORT_RECV
;
274 if (MLX5_CAP_ODP(dev
->mdev
, rc_odp_caps
.write
))
275 caps
->per_transport_caps
.rc_odp_caps
|= IB_ODP_SUPPORT_WRITE
;
277 if (MLX5_CAP_ODP(dev
->mdev
, rc_odp_caps
.read
))
278 caps
->per_transport_caps
.rc_odp_caps
|= IB_ODP_SUPPORT_READ
;
280 if (MLX5_CAP_ODP(dev
->mdev
, rc_odp_caps
.atomic
))
281 caps
->per_transport_caps
.rc_odp_caps
|= IB_ODP_SUPPORT_ATOMIC
;
283 if (MLX5_CAP_GEN(dev
->mdev
, fixed_buffer_size
) &&
284 MLX5_CAP_GEN(dev
->mdev
, null_mkey
) &&
285 MLX5_CAP_GEN(dev
->mdev
, umr_extended_translation_offset
))
286 caps
->general_caps
|= IB_ODP_SUPPORT_IMPLICIT
;
291 static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev
*dev
,
292 struct mlx5_pagefault
*pfault
,
295 int wq_num
= pfault
->event_subtype
== MLX5_PFAULT_SUBTYPE_WQE
?
296 pfault
->wqe
.wq_num
: pfault
->token
;
297 int ret
= mlx5_core_page_fault_resume(dev
->mdev
,
303 mlx5_ib_err(dev
, "Failed to resolve the page fault on WQ 0x%x\n",
307 static struct mlx5_ib_mr
*implicit_mr_alloc(struct ib_pd
*pd
,
308 struct ib_umem
*umem
,
309 bool ksm
, int access_flags
)
311 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
312 struct mlx5_ib_mr
*mr
;
315 mr
= mlx5_mr_cache_alloc(dev
, ksm
? MLX5_IMR_KSM_CACHE_ENTRY
:
316 MLX5_IMR_MTT_CACHE_ENTRY
);
324 mr
->access_flags
= access_flags
;
329 err
= mlx5_ib_update_xlt(mr
, 0,
330 mlx5_imr_ksm_entries
,
332 MLX5_IB_UPD_XLT_INDIRECT
|
333 MLX5_IB_UPD_XLT_ZAP
|
334 MLX5_IB_UPD_XLT_ENABLE
);
337 err
= mlx5_ib_update_xlt(mr
, 0,
338 MLX5_IMR_MTT_ENTRIES
,
340 MLX5_IB_UPD_XLT_ZAP
|
341 MLX5_IB_UPD_XLT_ENABLE
|
342 MLX5_IB_UPD_XLT_ATOMIC
);
348 mr
->ibmr
.lkey
= mr
->mmkey
.key
;
349 mr
->ibmr
.rkey
= mr
->mmkey
.key
;
353 mlx5_ib_dbg(dev
, "key %x dev %p mr %p\n",
354 mr
->mmkey
.key
, dev
->mdev
, mr
);
359 mlx5_ib_err(dev
, "Failed to register MKEY %d\n", err
);
360 mlx5_mr_cache_free(dev
, mr
);
365 static struct ib_umem_odp
*implicit_mr_get_data(struct mlx5_ib_mr
*mr
,
366 u64 io_virt
, size_t bcnt
)
368 struct ib_ucontext
*ctx
= mr
->ibmr
.pd
->uobject
->context
;
369 struct mlx5_ib_dev
*dev
= to_mdev(mr
->ibmr
.pd
->device
);
370 struct ib_umem_odp
*odp
, *result
= NULL
;
371 u64 addr
= io_virt
& MLX5_IMR_MTT_MASK
;
372 int nentries
= 0, start_idx
= 0, ret
;
373 struct mlx5_ib_mr
*mtt
;
374 struct ib_umem
*umem
;
376 mutex_lock(&mr
->umem
->odp_data
->umem_mutex
);
377 odp
= odp_lookup(ctx
, addr
, 1, mr
);
379 mlx5_ib_dbg(dev
, "io_virt:%llx bcnt:%zx addr:%llx odp:%p\n",
380 io_virt
, bcnt
, addr
, odp
);
387 umem
= ib_alloc_odp_umem(ctx
, addr
, MLX5_IMR_MTT_SIZE
);
389 mutex_unlock(&mr
->umem
->odp_data
->umem_mutex
);
390 return ERR_CAST(umem
);
393 mtt
= implicit_mr_alloc(mr
->ibmr
.pd
, umem
, 0, mr
->access_flags
);
395 mutex_unlock(&mr
->umem
->odp_data
->umem_mutex
);
396 ib_umem_release(umem
);
397 return ERR_CAST(mtt
);
400 odp
= umem
->odp_data
;
403 mtt
->mmkey
.iova
= addr
;
405 INIT_WORK(&odp
->work
, mr_leaf_free_action
);
408 start_idx
= addr
>> MLX5_IMR_MTT_SHIFT
;
412 /* Return first odp if region not covered by single one */
416 addr
+= MLX5_IMR_MTT_SIZE
;
417 if (unlikely(addr
< io_virt
+ bcnt
)) {
419 if (odp
&& odp
->umem
->address
!= addr
)
424 if (unlikely(nentries
)) {
425 ret
= mlx5_ib_update_xlt(mr
, start_idx
, nentries
, 0,
426 MLX5_IB_UPD_XLT_INDIRECT
|
427 MLX5_IB_UPD_XLT_ATOMIC
);
429 mlx5_ib_err(dev
, "Failed to update PAS\n");
430 result
= ERR_PTR(ret
);
434 mutex_unlock(&mr
->umem
->odp_data
->umem_mutex
);
438 struct mlx5_ib_mr
*mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd
*pd
,
441 struct ib_ucontext
*ctx
= pd
->ibpd
.uobject
->context
;
442 struct mlx5_ib_mr
*imr
;
443 struct ib_umem
*umem
;
445 umem
= ib_umem_get(ctx
, 0, 0, IB_ACCESS_ON_DEMAND
, 0);
447 return ERR_CAST(umem
);
449 imr
= implicit_mr_alloc(&pd
->ibpd
, umem
, 1, access_flags
);
451 ib_umem_release(umem
);
452 return ERR_CAST(imr
);
456 init_waitqueue_head(&imr
->q_leaf_free
);
457 atomic_set(&imr
->num_leaf_free
, 0);
462 static int mr_leaf_free(struct ib_umem
*umem
, u64 start
,
463 u64 end
, void *cookie
)
465 struct mlx5_ib_mr
*mr
= umem
->odp_data
->private, *imr
= cookie
;
467 if (mr
->parent
!= imr
)
470 ib_umem_odp_unmap_dma_pages(umem
,
474 if (umem
->odp_data
->dying
)
477 WRITE_ONCE(umem
->odp_data
->dying
, 1);
478 atomic_inc(&imr
->num_leaf_free
);
479 schedule_work(&umem
->odp_data
->work
);
484 void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr
*imr
)
486 struct ib_ucontext
*ctx
= imr
->ibmr
.pd
->uobject
->context
;
488 down_read(&ctx
->umem_rwsem
);
489 rbt_ib_umem_for_each_in_range(&ctx
->umem_tree
, 0, ULLONG_MAX
,
491 up_read(&ctx
->umem_rwsem
);
493 wait_event(imr
->q_leaf_free
, !atomic_read(&imr
->num_leaf_free
));
496 static int pagefault_mr(struct mlx5_ib_dev
*dev
, struct mlx5_ib_mr
*mr
,
497 u64 io_virt
, size_t bcnt
, u32
*bytes_mapped
)
499 u64 access_mask
= ODP_READ_ALLOWED_BIT
;
500 int npages
= 0, page_shift
, np
;
501 u64 start_idx
, page_mask
;
502 struct ib_umem_odp
*odp
;
507 if (!mr
->umem
->odp_data
->page_list
) {
508 odp
= implicit_mr_get_data(mr
, io_virt
, bcnt
);
515 odp
= mr
->umem
->odp_data
;
519 size
= min_t(size_t, bcnt
, ib_umem_end(odp
->umem
) - io_virt
);
521 page_shift
= mr
->umem
->page_shift
;
522 page_mask
= ~(BIT(page_shift
) - 1);
523 start_idx
= (io_virt
- (mr
->mmkey
.iova
& page_mask
)) >> page_shift
;
525 if (mr
->umem
->writable
)
526 access_mask
|= ODP_WRITE_ALLOWED_BIT
;
528 current_seq
= READ_ONCE(odp
->notifiers_seq
);
530 * Ensure the sequence number is valid for some time before we call
535 ret
= ib_umem_odp_map_dma_pages(mr
->umem
, io_virt
, size
,
536 access_mask
, current_seq
);
543 mutex_lock(&odp
->umem_mutex
);
544 if (!ib_umem_mmu_notifier_retry(mr
->umem
, current_seq
)) {
546 * No need to check whether the MTTs really belong to
547 * this MR, since ib_umem_odp_map_dma_pages already
550 ret
= mlx5_ib_update_xlt(mr
, start_idx
, np
,
551 page_shift
, MLX5_IB_UPD_XLT_ATOMIC
);
555 mutex_unlock(&odp
->umem_mutex
);
559 mlx5_ib_err(dev
, "Failed to update mkey page tables\n");
564 u32 new_mappings
= (np
<< page_shift
) -
565 (io_virt
- round_down(io_virt
, 1 << page_shift
));
566 *bytes_mapped
+= min_t(u32
, new_mappings
, size
);
569 npages
+= np
<< (page_shift
- PAGE_SHIFT
);
572 if (unlikely(bcnt
)) {
573 struct ib_umem_odp
*next
;
576 next
= odp_next(odp
);
577 if (unlikely(!next
|| next
->umem
->address
!= io_virt
)) {
578 mlx5_ib_dbg(dev
, "next implicit leaf removed at 0x%llx. got %p\n",
590 if (ret
== -EAGAIN
) {
591 if (mr
->parent
|| !odp
->dying
) {
592 unsigned long timeout
=
593 msecs_to_jiffies(MMU_NOTIFIER_TIMEOUT
);
595 if (!wait_for_completion_timeout(
596 &odp
->notifier_completion
,
598 mlx5_ib_warn(dev
, "timeout waiting for mmu notifier. seq %d against %d\n",
599 current_seq
, odp
->notifiers_seq
);
602 /* The MR is being killed, kill the QP as well. */
611 struct pf_frame
*next
;
619 * Handle a single data segment in a page-fault WQE or RDMA region.
621 * Returns number of OS pages retrieved on success. The caller may continue to
622 * the next data segment.
623 * Can return the following error codes:
624 * -EAGAIN to designate a temporary error. The caller will abort handling the
625 * page fault and resolve it.
626 * -EFAULT when there's an error mapping the requested pages. The caller will
627 * abort the page fault handling.
629 static int pagefault_single_data_segment(struct mlx5_ib_dev
*dev
,
630 u32 key
, u64 io_virt
, size_t bcnt
,
631 u32
*bytes_committed
,
634 int npages
= 0, srcu_key
, ret
, i
, outlen
, cur_outlen
= 0, depth
= 0;
635 struct pf_frame
*head
= NULL
, *frame
;
636 struct mlx5_core_mkey
*mmkey
;
637 struct mlx5_ib_mw
*mw
;
638 struct mlx5_ib_mr
*mr
;
639 struct mlx5_klm
*pklm
;
643 srcu_key
= srcu_read_lock(&dev
->mr_srcu
);
645 io_virt
+= *bytes_committed
;
646 bcnt
-= *bytes_committed
;
649 mmkey
= __mlx5_mr_lookup(dev
->mdev
, mlx5_base_mkey(key
));
650 if (!mmkey
|| mmkey
->key
!= key
) {
651 mlx5_ib_dbg(dev
, "failed to find mkey %x\n", key
);
656 switch (mmkey
->type
) {
658 mr
= container_of(mmkey
, struct mlx5_ib_mr
, mmkey
);
659 if (!mr
->live
|| !mr
->ibmr
.pd
) {
660 mlx5_ib_dbg(dev
, "got dead MR\n");
665 ret
= pagefault_mr(dev
, mr
, io_virt
, bcnt
, bytes_mapped
);
674 mw
= container_of(mmkey
, struct mlx5_ib_mw
, mmkey
);
676 if (depth
>= MLX5_CAP_GEN(dev
->mdev
, max_indirection
)) {
677 mlx5_ib_dbg(dev
, "indirection level exceeded\n");
682 outlen
= MLX5_ST_SZ_BYTES(query_mkey_out
) +
683 sizeof(*pklm
) * (mw
->ndescs
- 2);
685 if (outlen
> cur_outlen
) {
687 out
= kzalloc(outlen
, GFP_KERNEL
);
695 pklm
= (struct mlx5_klm
*)MLX5_ADDR_OF(query_mkey_out
, out
,
696 bsf0_klm0_pas_mtt0_1
);
698 ret
= mlx5_core_query_mkey(dev
->mdev
, &mw
->mmkey
, out
, outlen
);
702 offset
= io_virt
- MLX5_GET64(query_mkey_out
, out
,
703 memory_key_mkey_entry
.start_addr
);
705 for (i
= 0; bcnt
&& i
< mw
->ndescs
; i
++, pklm
++) {
706 if (offset
>= be32_to_cpu(pklm
->bcount
)) {
707 offset
-= be32_to_cpu(pklm
->bcount
);
711 frame
= kzalloc(sizeof(*frame
), GFP_KERNEL
);
717 frame
->key
= be32_to_cpu(pklm
->key
);
718 frame
->io_virt
= be64_to_cpu(pklm
->va
) + offset
;
719 frame
->bcnt
= min_t(size_t, bcnt
,
720 be32_to_cpu(pklm
->bcount
) - offset
);
721 frame
->depth
= depth
+ 1;
730 mlx5_ib_dbg(dev
, "wrong mkey type %d\n", mmkey
->type
);
740 io_virt
= frame
->io_virt
;
742 depth
= frame
->depth
;
756 srcu_read_unlock(&dev
->mr_srcu
, srcu_key
);
757 *bytes_committed
= 0;
758 return ret
? ret
: npages
;
762 * Parse a series of data segments for page fault handling.
764 * @qp the QP on which the fault occurred.
765 * @pfault contains page fault information.
766 * @wqe points at the first data segment in the WQE.
767 * @wqe_end points after the end of the WQE.
768 * @bytes_mapped receives the number of bytes that the function was able to
769 * map. This allows the caller to decide intelligently whether
770 * enough memory was mapped to resolve the page fault
771 * successfully (e.g. enough for the next MTU, or the entire
773 * @total_wqe_bytes receives the total data size of this WQE in bytes (minus
774 * the committed bytes).
776 * Returns the number of pages loaded if positive, zero for an empty WQE, or a
777 * negative error code.
779 static int pagefault_data_segments(struct mlx5_ib_dev
*dev
,
780 struct mlx5_pagefault
*pfault
,
781 struct mlx5_ib_qp
*qp
, void *wqe
,
782 void *wqe_end
, u32
*bytes_mapped
,
783 u32
*total_wqe_bytes
, int receive_queue
)
785 int ret
= 0, npages
= 0;
792 /* Skip SRQ next-WQE segment. */
793 if (receive_queue
&& qp
->ibqp
.srq
)
794 wqe
+= sizeof(struct mlx5_wqe_srq_next_seg
);
799 *total_wqe_bytes
= 0;
801 while (wqe
< wqe_end
) {
802 struct mlx5_wqe_data_seg
*dseg
= wqe
;
804 io_virt
= be64_to_cpu(dseg
->addr
);
805 key
= be32_to_cpu(dseg
->lkey
);
806 byte_count
= be32_to_cpu(dseg
->byte_count
);
807 inline_segment
= !!(byte_count
& MLX5_INLINE_SEG
);
808 bcnt
= byte_count
& ~MLX5_INLINE_SEG
;
810 if (inline_segment
) {
811 bcnt
= bcnt
& MLX5_WQE_INLINE_SEG_BYTE_COUNT_MASK
;
812 wqe
+= ALIGN(sizeof(struct mlx5_wqe_inline_seg
) + bcnt
,
815 wqe
+= sizeof(*dseg
);
818 /* receive WQE end of sg list. */
819 if (receive_queue
&& bcnt
== 0 && key
== MLX5_INVALID_LKEY
&&
823 if (!inline_segment
&& total_wqe_bytes
) {
824 *total_wqe_bytes
+= bcnt
- min_t(size_t, bcnt
,
825 pfault
->bytes_committed
);
828 /* A zero length data segment designates a length of 2GB. */
832 if (inline_segment
|| bcnt
<= pfault
->bytes_committed
) {
833 pfault
->bytes_committed
-=
835 pfault
->bytes_committed
);
839 ret
= pagefault_single_data_segment(dev
, key
, io_virt
, bcnt
,
840 &pfault
->bytes_committed
,
847 return ret
< 0 ? ret
: npages
;
850 static const u32 mlx5_ib_odp_opcode_cap
[] = {
851 [MLX5_OPCODE_SEND
] = IB_ODP_SUPPORT_SEND
,
852 [MLX5_OPCODE_SEND_IMM
] = IB_ODP_SUPPORT_SEND
,
853 [MLX5_OPCODE_SEND_INVAL
] = IB_ODP_SUPPORT_SEND
,
854 [MLX5_OPCODE_RDMA_WRITE
] = IB_ODP_SUPPORT_WRITE
,
855 [MLX5_OPCODE_RDMA_WRITE_IMM
] = IB_ODP_SUPPORT_WRITE
,
856 [MLX5_OPCODE_RDMA_READ
] = IB_ODP_SUPPORT_READ
,
857 [MLX5_OPCODE_ATOMIC_CS
] = IB_ODP_SUPPORT_ATOMIC
,
858 [MLX5_OPCODE_ATOMIC_FA
] = IB_ODP_SUPPORT_ATOMIC
,
862 * Parse initiator WQE. Advances the wqe pointer to point at the
863 * scatter-gather list, and set wqe_end to the end of the WQE.
865 static int mlx5_ib_mr_initiator_pfault_handler(
866 struct mlx5_ib_dev
*dev
, struct mlx5_pagefault
*pfault
,
867 struct mlx5_ib_qp
*qp
, void **wqe
, void **wqe_end
, int wqe_length
)
869 struct mlx5_wqe_ctrl_seg
*ctrl
= *wqe
;
870 u16 wqe_index
= pfault
->wqe
.wqe_index
;
872 struct mlx5_base_av
*av
;
875 u32 ctrl_wqe_index
, ctrl_qpn
;
877 u32 qpn
= qp
->trans_qp
.base
.mqp
.qpn
;
879 ds
= be32_to_cpu(ctrl
->qpn_ds
) & MLX5_WQE_CTRL_DS_MASK
;
880 if (ds
* MLX5_WQE_DS_UNITS
> wqe_length
) {
881 mlx5_ib_err(dev
, "Unable to read the complete WQE. ds = 0x%x, ret = 0x%x\n",
887 mlx5_ib_err(dev
, "Got WQE with zero DS. wqe_index=%x, qpn=%x\n",
893 ctrl_wqe_index
= (be32_to_cpu(ctrl
->opmod_idx_opcode
) &
894 MLX5_WQE_CTRL_WQE_INDEX_MASK
) >>
895 MLX5_WQE_CTRL_WQE_INDEX_SHIFT
;
896 if (wqe_index
!= ctrl_wqe_index
) {
897 mlx5_ib_err(dev
, "Got WQE with invalid wqe_index. wqe_index=0x%x, qpn=0x%x ctrl->wqe_index=0x%x\n",
903 ctrl_qpn
= (be32_to_cpu(ctrl
->qpn_ds
) & MLX5_WQE_CTRL_QPN_MASK
) >>
904 MLX5_WQE_CTRL_QPN_SHIFT
;
905 if (qpn
!= ctrl_qpn
) {
906 mlx5_ib_err(dev
, "Got WQE with incorrect QP number. wqe_index=0x%x, qpn=0x%x ctrl->qpn=0x%x\n",
913 *wqe_end
= *wqe
+ ds
* MLX5_WQE_DS_UNITS
;
914 *wqe
+= sizeof(*ctrl
);
916 opcode
= be32_to_cpu(ctrl
->opmod_idx_opcode
) &
917 MLX5_WQE_CTRL_OPCODE_MASK
;
919 switch (qp
->ibqp
.qp_type
) {
921 transport_caps
= dev
->odp_caps
.per_transport_caps
.rc_odp_caps
;
924 transport_caps
= dev
->odp_caps
.per_transport_caps
.ud_odp_caps
;
927 mlx5_ib_err(dev
, "ODP fault on QP of an unsupported transport 0x%x\n",
932 if (unlikely(opcode
>= sizeof(mlx5_ib_odp_opcode_cap
) /
933 sizeof(mlx5_ib_odp_opcode_cap
[0]) ||
934 !(transport_caps
& mlx5_ib_odp_opcode_cap
[opcode
]))) {
935 mlx5_ib_err(dev
, "ODP fault on QP of an unsupported opcode 0x%x\n",
940 if (qp
->ibqp
.qp_type
!= IB_QPT_RC
) {
942 if (av
->dqp_dct
& be32_to_cpu(MLX5_WQE_AV_EXT
))
943 *wqe
+= sizeof(struct mlx5_av
);
945 *wqe
+= sizeof(struct mlx5_base_av
);
949 case MLX5_OPCODE_RDMA_WRITE
:
950 case MLX5_OPCODE_RDMA_WRITE_IMM
:
951 case MLX5_OPCODE_RDMA_READ
:
952 *wqe
+= sizeof(struct mlx5_wqe_raddr_seg
);
954 case MLX5_OPCODE_ATOMIC_CS
:
955 case MLX5_OPCODE_ATOMIC_FA
:
956 *wqe
+= sizeof(struct mlx5_wqe_raddr_seg
);
957 *wqe
+= sizeof(struct mlx5_wqe_atomic_seg
);
965 * Parse responder WQE. Advances the wqe pointer to point at the
966 * scatter-gather list, and set wqe_end to the end of the WQE.
968 static int mlx5_ib_mr_responder_pfault_handler(
969 struct mlx5_ib_dev
*dev
, struct mlx5_pagefault
*pfault
,
970 struct mlx5_ib_qp
*qp
, void **wqe
, void **wqe_end
, int wqe_length
)
972 struct mlx5_ib_wq
*wq
= &qp
->rq
;
973 int wqe_size
= 1 << wq
->wqe_shift
;
976 mlx5_ib_err(dev
, "ODP fault on SRQ is not supported\n");
981 mlx5_ib_err(dev
, "ODP fault with WQE signatures is not supported\n");
985 if (wqe_size
> wqe_length
) {
986 mlx5_ib_err(dev
, "Couldn't read all of the receive WQE's content\n");
990 switch (qp
->ibqp
.qp_type
) {
992 if (!(dev
->odp_caps
.per_transport_caps
.rc_odp_caps
&
993 IB_ODP_SUPPORT_RECV
))
994 goto invalid_transport_or_opcode
;
997 invalid_transport_or_opcode
:
998 mlx5_ib_err(dev
, "ODP fault on QP of an unsupported transport. transport: 0x%x\n",
1003 *wqe_end
= *wqe
+ wqe_size
;
1008 static struct mlx5_ib_qp
*mlx5_ib_odp_find_qp(struct mlx5_ib_dev
*dev
,
1011 struct mlx5_core_qp
*mqp
= __mlx5_qp_lookup(dev
->mdev
, wq_num
);
1014 mlx5_ib_err(dev
, "QPN 0x%6x not found\n", wq_num
);
1018 return to_mibqp(mqp
);
1021 static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev
*dev
,
1022 struct mlx5_pagefault
*pfault
)
1025 void *wqe
, *wqe_end
;
1026 u32 bytes_mapped
, total_wqe_bytes
;
1027 char *buffer
= NULL
;
1028 int resume_with_error
= 1;
1029 u16 wqe_index
= pfault
->wqe
.wqe_index
;
1030 int requestor
= pfault
->type
& MLX5_PFAULT_REQUESTOR
;
1031 struct mlx5_ib_qp
*qp
;
1033 buffer
= (char *)__get_free_page(GFP_KERNEL
);
1035 mlx5_ib_err(dev
, "Error allocating memory for IO page fault handling.\n");
1036 goto resolve_page_fault
;
1039 qp
= mlx5_ib_odp_find_qp(dev
, pfault
->wqe
.wq_num
);
1041 goto resolve_page_fault
;
1043 ret
= mlx5_ib_read_user_wqe(qp
, requestor
, wqe_index
, buffer
,
1044 PAGE_SIZE
, &qp
->trans_qp
.base
);
1046 mlx5_ib_err(dev
, "Failed reading a WQE following page fault, error=%d, wqe_index=%x, qpn=%x\n",
1047 ret
, wqe_index
, pfault
->token
);
1048 goto resolve_page_fault
;
1053 ret
= mlx5_ib_mr_initiator_pfault_handler(dev
, pfault
, qp
, &wqe
,
1056 ret
= mlx5_ib_mr_responder_pfault_handler(dev
, pfault
, qp
, &wqe
,
1059 goto resolve_page_fault
;
1061 if (wqe
>= wqe_end
) {
1062 mlx5_ib_err(dev
, "ODP fault on invalid WQE.\n");
1063 goto resolve_page_fault
;
1066 ret
= pagefault_data_segments(dev
, pfault
, qp
, wqe
, wqe_end
,
1067 &bytes_mapped
, &total_wqe_bytes
,
1069 if (ret
== -EAGAIN
) {
1070 resume_with_error
= 0;
1071 goto resolve_page_fault
;
1072 } else if (ret
< 0 || total_wqe_bytes
> bytes_mapped
) {
1073 goto resolve_page_fault
;
1076 resume_with_error
= 0;
1078 mlx5_ib_page_fault_resume(dev
, pfault
, resume_with_error
);
1079 mlx5_ib_dbg(dev
, "PAGE FAULT completed. QP 0x%x resume_with_error=%d, type: 0x%x\n",
1080 pfault
->wqe
.wq_num
, resume_with_error
,
1082 free_page((unsigned long)buffer
);
1085 static int pages_in_range(u64 address
, u32 length
)
1087 return (ALIGN(address
+ length
, PAGE_SIZE
) -
1088 (address
& PAGE_MASK
)) >> PAGE_SHIFT
;
1091 static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev
*dev
,
1092 struct mlx5_pagefault
*pfault
)
1096 u32 prefetch_len
= pfault
->bytes_committed
;
1097 int prefetch_activated
= 0;
1098 u32 rkey
= pfault
->rdma
.r_key
;
1101 /* The RDMA responder handler handles the page fault in two parts.
1102 * First it brings the necessary pages for the current packet
1103 * (and uses the pfault context), and then (after resuming the QP)
1104 * prefetches more pages. The second operation cannot use the pfault
1105 * context and therefore uses the dummy_pfault context allocated on
1107 pfault
->rdma
.rdma_va
+= pfault
->bytes_committed
;
1108 pfault
->rdma
.rdma_op_len
-= min(pfault
->bytes_committed
,
1109 pfault
->rdma
.rdma_op_len
);
1110 pfault
->bytes_committed
= 0;
1112 address
= pfault
->rdma
.rdma_va
;
1113 length
= pfault
->rdma
.rdma_op_len
;
1115 /* For some operations, the hardware cannot tell the exact message
1116 * length, and in those cases it reports zero. Use prefetch
1119 prefetch_activated
= 1;
1120 length
= pfault
->rdma
.packet_size
;
1121 prefetch_len
= min(MAX_PREFETCH_LEN
, prefetch_len
);
1124 ret
= pagefault_single_data_segment(dev
, rkey
, address
, length
,
1125 &pfault
->bytes_committed
, NULL
);
1126 if (ret
== -EAGAIN
) {
1127 /* We're racing with an invalidation, don't prefetch */
1128 prefetch_activated
= 0;
1129 } else if (ret
< 0 || pages_in_range(address
, length
) > ret
) {
1130 mlx5_ib_page_fault_resume(dev
, pfault
, 1);
1132 mlx5_ib_dbg(dev
, "PAGE FAULT error %d. QP 0x%x, type: 0x%x\n",
1133 ret
, pfault
->token
, pfault
->type
);
1137 mlx5_ib_page_fault_resume(dev
, pfault
, 0);
1138 mlx5_ib_dbg(dev
, "PAGE FAULT completed. QP 0x%x, type: 0x%x, prefetch_activated: %d\n",
1139 pfault
->token
, pfault
->type
,
1140 prefetch_activated
);
1142 /* At this point, there might be a new pagefault already arriving in
1143 * the eq, switch to the dummy pagefault for the rest of the
1144 * processing. We're still OK with the objects being alive as the
1145 * work-queue is being fenced. */
1147 if (prefetch_activated
) {
1148 u32 bytes_committed
= 0;
1150 ret
= pagefault_single_data_segment(dev
, rkey
, address
,
1152 &bytes_committed
, NULL
);
1153 if (ret
< 0 && ret
!= -EAGAIN
) {
1154 mlx5_ib_dbg(dev
, "Prefetch failed. ret: %d, QP 0x%x, address: 0x%.16llx, length = 0x%.16x\n",
1155 ret
, pfault
->token
, address
, prefetch_len
);
1160 void mlx5_ib_pfault(struct mlx5_core_dev
*mdev
, void *context
,
1161 struct mlx5_pagefault
*pfault
)
1163 struct mlx5_ib_dev
*dev
= context
;
1164 u8 event_subtype
= pfault
->event_subtype
;
1166 switch (event_subtype
) {
1167 case MLX5_PFAULT_SUBTYPE_WQE
:
1168 mlx5_ib_mr_wqe_pfault_handler(dev
, pfault
);
1170 case MLX5_PFAULT_SUBTYPE_RDMA
:
1171 mlx5_ib_mr_rdma_pfault_handler(dev
, pfault
);
1174 mlx5_ib_err(dev
, "Invalid page fault event subtype: 0x%x\n",
1176 mlx5_ib_page_fault_resume(dev
, pfault
, 1);
1180 void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent
*ent
)
1182 if (!(ent
->dev
->odp_caps
.general_caps
& IB_ODP_SUPPORT_IMPLICIT
))
1185 switch (ent
->order
- 2) {
1186 case MLX5_IMR_MTT_CACHE_ENTRY
:
1187 ent
->page
= PAGE_SHIFT
;
1188 ent
->xlt
= MLX5_IMR_MTT_ENTRIES
*
1189 sizeof(struct mlx5_mtt
) /
1190 MLX5_IB_UMR_OCTOWORD
;
1191 ent
->access_mode
= MLX5_MKC_ACCESS_MODE_MTT
;
1195 case MLX5_IMR_KSM_CACHE_ENTRY
:
1196 ent
->page
= MLX5_KSM_PAGE_SHIFT
;
1197 ent
->xlt
= mlx5_imr_ksm_entries
*
1198 sizeof(struct mlx5_klm
) /
1199 MLX5_IB_UMR_OCTOWORD
;
1200 ent
->access_mode
= MLX5_MKC_ACCESS_MODE_KSM
;
1206 int mlx5_ib_odp_init_one(struct mlx5_ib_dev
*dev
)
1210 ret
= init_srcu_struct(&dev
->mr_srcu
);
1214 if (dev
->odp_caps
.general_caps
& IB_ODP_SUPPORT_IMPLICIT
) {
1215 ret
= mlx5_cmd_null_mkey(dev
->mdev
, &dev
->null_mkey
);
1217 mlx5_ib_err(dev
, "Error getting null_mkey %d\n", ret
);
1225 void mlx5_ib_odp_remove_one(struct mlx5_ib_dev
*dev
)
1227 cleanup_srcu_struct(&dev
->mr_srcu
);
1230 int mlx5_ib_odp_init(void)
1232 mlx5_imr_ksm_entries
= BIT_ULL(get_order(TASK_SIZE
) -