2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <rdma/ib_umem.h>
34 #include <rdma/ib_umem_odp.h>
35 #include <linux/kernel.h>
40 #include <linux/mlx5/eq.h>
42 /* Contains the details of a pagefault. */
43 struct mlx5_pagefault
{
49 /* Initiator or send message responder pagefault details. */
51 /* Received packet size, only valid for responders. */
54 * Number of resource holding WQE, depends on type.
58 * WQE index. Refers to either the send queue or
59 * receive queue, according to event_subtype.
63 /* RDMA responder pagefault details */
67 * Received packet size, minimal size page fault
68 * resolution required for forward progress.
76 struct mlx5_ib_pf_eq
*eq
;
77 struct work_struct work
;
80 #define MAX_PREFETCH_LEN (4*1024*1024U)
82 /* Timeout in ms to wait for an active mmu notifier to complete when handling
84 #define MMU_NOTIFIER_TIMEOUT 1000
86 #define MLX5_IMR_MTT_BITS (30 - PAGE_SHIFT)
87 #define MLX5_IMR_MTT_SHIFT (MLX5_IMR_MTT_BITS + PAGE_SHIFT)
88 #define MLX5_IMR_MTT_ENTRIES BIT_ULL(MLX5_IMR_MTT_BITS)
89 #define MLX5_IMR_MTT_SIZE BIT_ULL(MLX5_IMR_MTT_SHIFT)
90 #define MLX5_IMR_MTT_MASK (~(MLX5_IMR_MTT_SIZE - 1))
92 #define MLX5_KSM_PAGE_SHIFT MLX5_IMR_MTT_SHIFT
94 static u64 mlx5_imr_ksm_entries
;
96 static int check_parent(struct ib_umem_odp
*odp
,
97 struct mlx5_ib_mr
*parent
)
99 struct mlx5_ib_mr
*mr
= odp
->private;
101 return mr
&& mr
->parent
== parent
&& !odp
->dying
;
104 static struct ib_ucontext_per_mm
*mr_to_per_mm(struct mlx5_ib_mr
*mr
)
106 if (WARN_ON(!mr
|| !is_odp_mr(mr
)))
109 return to_ib_umem_odp(mr
->umem
)->per_mm
;
112 static struct ib_umem_odp
*odp_next(struct ib_umem_odp
*odp
)
114 struct mlx5_ib_mr
*mr
= odp
->private, *parent
= mr
->parent
;
115 struct ib_ucontext_per_mm
*per_mm
= odp
->per_mm
;
118 down_read(&per_mm
->umem_rwsem
);
120 rb
= rb_next(&odp
->interval_tree
.rb
);
123 odp
= rb_entry(rb
, struct ib_umem_odp
, interval_tree
.rb
);
124 if (check_parent(odp
, parent
))
130 up_read(&per_mm
->umem_rwsem
);
134 static struct ib_umem_odp
*odp_lookup(u64 start
, u64 length
,
135 struct mlx5_ib_mr
*parent
)
137 struct ib_ucontext_per_mm
*per_mm
= mr_to_per_mm(parent
);
138 struct ib_umem_odp
*odp
;
141 down_read(&per_mm
->umem_rwsem
);
142 odp
= rbt_ib_umem_lookup(&per_mm
->umem_tree
, start
, length
);
147 if (check_parent(odp
, parent
))
149 rb
= rb_next(&odp
->interval_tree
.rb
);
152 odp
= rb_entry(rb
, struct ib_umem_odp
, interval_tree
.rb
);
153 if (ib_umem_start(odp
) > start
+ length
)
159 up_read(&per_mm
->umem_rwsem
);
163 void mlx5_odp_populate_klm(struct mlx5_klm
*pklm
, size_t offset
,
164 size_t nentries
, struct mlx5_ib_mr
*mr
, int flags
)
166 struct ib_pd
*pd
= mr
->ibmr
.pd
;
167 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
168 struct ib_umem_odp
*odp
;
172 if (flags
& MLX5_IB_UPD_XLT_ZAP
) {
173 for (i
= 0; i
< nentries
; i
++, pklm
++) {
174 pklm
->bcount
= cpu_to_be32(MLX5_IMR_MTT_SIZE
);
175 pklm
->key
= cpu_to_be32(dev
->null_mkey
);
182 * The locking here is pretty subtle. Ideally the implicit children
183 * list would be protected by the umem_mutex, however that is not
184 * possible. Instead this uses a weaker update-then-lock pattern:
187 * <change children list>
188 * mutex_lock(umem_mutex)
189 * mlx5_ib_update_xlt()
190 * mutex_unlock(umem_mutex)
193 * ie any change the children list must be followed by the locked
194 * update_xlt before destroying.
196 * The umem_mutex provides the acquire/release semantic needed to make
197 * the children list visible to a racing thread. While SRCU is not
198 * technically required, using it gives consistent use of the SRCU
199 * locking around the children list.
201 lockdep_assert_held(&to_ib_umem_odp(mr
->umem
)->umem_mutex
);
202 lockdep_assert_held(&mr
->dev
->mr_srcu
);
204 odp
= odp_lookup(offset
* MLX5_IMR_MTT_SIZE
,
205 nentries
* MLX5_IMR_MTT_SIZE
, mr
);
207 for (i
= 0; i
< nentries
; i
++, pklm
++) {
208 pklm
->bcount
= cpu_to_be32(MLX5_IMR_MTT_SIZE
);
209 va
= (offset
+ i
) * MLX5_IMR_MTT_SIZE
;
210 if (odp
&& ib_umem_start(odp
) == va
) {
211 struct mlx5_ib_mr
*mtt
= odp
->private;
213 pklm
->key
= cpu_to_be32(mtt
->ibmr
.lkey
);
216 pklm
->key
= cpu_to_be32(dev
->null_mkey
);
218 mlx5_ib_dbg(dev
, "[%d] va %lx key %x\n",
219 i
, va
, be32_to_cpu(pklm
->key
));
223 static void mr_leaf_free_action(struct work_struct
*work
)
225 struct ib_umem_odp
*odp
= container_of(work
, struct ib_umem_odp
, work
);
226 int idx
= ib_umem_start(odp
) >> MLX5_IMR_MTT_SHIFT
;
227 struct mlx5_ib_mr
*mr
= odp
->private, *imr
= mr
->parent
;
228 struct ib_umem_odp
*odp_imr
= to_ib_umem_odp(imr
->umem
);
232 synchronize_srcu(&mr
->dev
->mr_srcu
);
234 if (smp_load_acquire(&imr
->live
)) {
235 srcu_key
= srcu_read_lock(&mr
->dev
->mr_srcu
);
236 mutex_lock(&odp_imr
->umem_mutex
);
237 mlx5_ib_update_xlt(imr
, idx
, 1, 0,
238 MLX5_IB_UPD_XLT_INDIRECT
|
239 MLX5_IB_UPD_XLT_ATOMIC
);
240 mutex_unlock(&odp_imr
->umem_mutex
);
241 srcu_read_unlock(&mr
->dev
->mr_srcu
, srcu_key
);
243 ib_umem_odp_release(odp
);
244 mlx5_mr_cache_free(mr
->dev
, mr
);
246 if (atomic_dec_and_test(&imr
->num_leaf_free
))
247 wake_up(&imr
->q_leaf_free
);
250 void mlx5_ib_invalidate_range(struct ib_umem_odp
*umem_odp
, unsigned long start
,
253 struct mlx5_ib_mr
*mr
;
254 const u64 umr_block_mask
= (MLX5_UMR_MTT_ALIGNMENT
/
255 sizeof(struct mlx5_mtt
)) - 1;
256 u64 idx
= 0, blk_start_idx
= 0;
261 pr_err("invalidation called on NULL umem or non-ODP umem\n");
265 mr
= umem_odp
->private;
267 if (!mr
|| !mr
->ibmr
.pd
)
270 start
= max_t(u64
, ib_umem_start(umem_odp
), start
);
271 end
= min_t(u64
, ib_umem_end(umem_odp
), end
);
274 * Iteration one - zap the HW's MTTs. The notifiers_count ensures that
275 * while we are doing the invalidation, no page fault will attempt to
276 * overwrite the same MTTs. Concurent invalidations might race us,
277 * but they will write 0s as well, so no difference in the end result.
279 mutex_lock(&umem_odp
->umem_mutex
);
280 for (addr
= start
; addr
< end
; addr
+= BIT(umem_odp
->page_shift
)) {
281 idx
= (addr
- ib_umem_start(umem_odp
)) >> umem_odp
->page_shift
;
283 * Strive to write the MTTs in chunks, but avoid overwriting
284 * non-existing MTTs. The huristic here can be improved to
285 * estimate the cost of another UMR vs. the cost of bigger
288 if (umem_odp
->dma_list
[idx
] &
289 (ODP_READ_ALLOWED_BIT
| ODP_WRITE_ALLOWED_BIT
)) {
295 u64 umr_offset
= idx
& umr_block_mask
;
297 if (in_block
&& umr_offset
== 0) {
298 mlx5_ib_update_xlt(mr
, blk_start_idx
,
299 idx
- blk_start_idx
, 0,
300 MLX5_IB_UPD_XLT_ZAP
|
301 MLX5_IB_UPD_XLT_ATOMIC
);
307 mlx5_ib_update_xlt(mr
, blk_start_idx
,
308 idx
- blk_start_idx
+ 1, 0,
309 MLX5_IB_UPD_XLT_ZAP
|
310 MLX5_IB_UPD_XLT_ATOMIC
);
312 * We are now sure that the device will not access the
313 * memory. We can safely unmap it, and mark it as dirty if
317 ib_umem_odp_unmap_dma_pages(umem_odp
, start
, end
);
319 if (unlikely(!umem_odp
->npages
&& mr
->parent
&&
321 WRITE_ONCE(mr
->live
, 0);
323 atomic_inc(&mr
->parent
->num_leaf_free
);
324 schedule_work(&umem_odp
->work
);
326 mutex_unlock(&umem_odp
->umem_mutex
);
329 void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev
*dev
)
331 struct ib_odp_caps
*caps
= &dev
->odp_caps
;
333 memset(caps
, 0, sizeof(*caps
));
335 if (!MLX5_CAP_GEN(dev
->mdev
, pg
) ||
336 !mlx5_ib_can_use_umr(dev
, true))
339 caps
->general_caps
= IB_ODP_SUPPORT
;
341 if (MLX5_CAP_GEN(dev
->mdev
, umr_extended_translation_offset
))
342 dev
->odp_max_size
= U64_MAX
;
344 dev
->odp_max_size
= BIT_ULL(MLX5_MAX_UMR_SHIFT
+ PAGE_SHIFT
);
346 if (MLX5_CAP_ODP(dev
->mdev
, ud_odp_caps
.send
))
347 caps
->per_transport_caps
.ud_odp_caps
|= IB_ODP_SUPPORT_SEND
;
349 if (MLX5_CAP_ODP(dev
->mdev
, ud_odp_caps
.srq_receive
))
350 caps
->per_transport_caps
.ud_odp_caps
|= IB_ODP_SUPPORT_SRQ_RECV
;
352 if (MLX5_CAP_ODP(dev
->mdev
, rc_odp_caps
.send
))
353 caps
->per_transport_caps
.rc_odp_caps
|= IB_ODP_SUPPORT_SEND
;
355 if (MLX5_CAP_ODP(dev
->mdev
, rc_odp_caps
.receive
))
356 caps
->per_transport_caps
.rc_odp_caps
|= IB_ODP_SUPPORT_RECV
;
358 if (MLX5_CAP_ODP(dev
->mdev
, rc_odp_caps
.write
))
359 caps
->per_transport_caps
.rc_odp_caps
|= IB_ODP_SUPPORT_WRITE
;
361 if (MLX5_CAP_ODP(dev
->mdev
, rc_odp_caps
.read
))
362 caps
->per_transport_caps
.rc_odp_caps
|= IB_ODP_SUPPORT_READ
;
364 if (MLX5_CAP_ODP(dev
->mdev
, rc_odp_caps
.atomic
))
365 caps
->per_transport_caps
.rc_odp_caps
|= IB_ODP_SUPPORT_ATOMIC
;
367 if (MLX5_CAP_ODP(dev
->mdev
, rc_odp_caps
.srq_receive
))
368 caps
->per_transport_caps
.rc_odp_caps
|= IB_ODP_SUPPORT_SRQ_RECV
;
370 if (MLX5_CAP_ODP(dev
->mdev
, xrc_odp_caps
.send
))
371 caps
->per_transport_caps
.xrc_odp_caps
|= IB_ODP_SUPPORT_SEND
;
373 if (MLX5_CAP_ODP(dev
->mdev
, xrc_odp_caps
.receive
))
374 caps
->per_transport_caps
.xrc_odp_caps
|= IB_ODP_SUPPORT_RECV
;
376 if (MLX5_CAP_ODP(dev
->mdev
, xrc_odp_caps
.write
))
377 caps
->per_transport_caps
.xrc_odp_caps
|= IB_ODP_SUPPORT_WRITE
;
379 if (MLX5_CAP_ODP(dev
->mdev
, xrc_odp_caps
.read
))
380 caps
->per_transport_caps
.xrc_odp_caps
|= IB_ODP_SUPPORT_READ
;
382 if (MLX5_CAP_ODP(dev
->mdev
, xrc_odp_caps
.atomic
))
383 caps
->per_transport_caps
.xrc_odp_caps
|= IB_ODP_SUPPORT_ATOMIC
;
385 if (MLX5_CAP_ODP(dev
->mdev
, xrc_odp_caps
.srq_receive
))
386 caps
->per_transport_caps
.xrc_odp_caps
|= IB_ODP_SUPPORT_SRQ_RECV
;
388 if (MLX5_CAP_GEN(dev
->mdev
, fixed_buffer_size
) &&
389 MLX5_CAP_GEN(dev
->mdev
, null_mkey
) &&
390 MLX5_CAP_GEN(dev
->mdev
, umr_extended_translation_offset
) &&
391 !MLX5_CAP_GEN(dev
->mdev
, umr_indirect_mkey_disabled
))
392 caps
->general_caps
|= IB_ODP_SUPPORT_IMPLICIT
;
397 static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev
*dev
,
398 struct mlx5_pagefault
*pfault
,
401 int wq_num
= pfault
->event_subtype
== MLX5_PFAULT_SUBTYPE_WQE
?
402 pfault
->wqe
.wq_num
: pfault
->token
;
403 u32 out
[MLX5_ST_SZ_DW(page_fault_resume_out
)] = { };
404 u32 in
[MLX5_ST_SZ_DW(page_fault_resume_in
)] = { };
407 MLX5_SET(page_fault_resume_in
, in
, opcode
, MLX5_CMD_OP_PAGE_FAULT_RESUME
);
408 MLX5_SET(page_fault_resume_in
, in
, page_fault_type
, pfault
->type
);
409 MLX5_SET(page_fault_resume_in
, in
, token
, pfault
->token
);
410 MLX5_SET(page_fault_resume_in
, in
, wq_number
, wq_num
);
411 MLX5_SET(page_fault_resume_in
, in
, error
, !!error
);
413 err
= mlx5_cmd_exec(dev
->mdev
, in
, sizeof(in
), out
, sizeof(out
));
415 mlx5_ib_err(dev
, "Failed to resolve the page fault on WQ 0x%x err %d\n",
419 static struct mlx5_ib_mr
*implicit_mr_alloc(struct ib_pd
*pd
,
420 struct ib_umem_odp
*umem_odp
,
421 bool ksm
, int access_flags
)
423 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
424 struct mlx5_ib_mr
*mr
;
427 mr
= mlx5_mr_cache_alloc(dev
, ksm
? MLX5_IMR_KSM_CACHE_ENTRY
:
428 MLX5_IMR_MTT_CACHE_ENTRY
);
436 mr
->access_flags
= access_flags
;
438 mr
->umem
= &umem_odp
->umem
;
441 err
= mlx5_ib_update_xlt(mr
, 0,
442 mlx5_imr_ksm_entries
,
444 MLX5_IB_UPD_XLT_INDIRECT
|
445 MLX5_IB_UPD_XLT_ZAP
|
446 MLX5_IB_UPD_XLT_ENABLE
);
449 err
= mlx5_ib_update_xlt(mr
, 0,
450 MLX5_IMR_MTT_ENTRIES
,
452 MLX5_IB_UPD_XLT_ZAP
|
453 MLX5_IB_UPD_XLT_ENABLE
|
454 MLX5_IB_UPD_XLT_ATOMIC
);
460 mr
->ibmr
.lkey
= mr
->mmkey
.key
;
461 mr
->ibmr
.rkey
= mr
->mmkey
.key
;
463 mlx5_ib_dbg(dev
, "key %x dev %p mr %p\n",
464 mr
->mmkey
.key
, dev
->mdev
, mr
);
469 mlx5_ib_err(dev
, "Failed to register MKEY %d\n", err
);
470 mlx5_mr_cache_free(dev
, mr
);
475 static struct ib_umem_odp
*implicit_mr_get_data(struct mlx5_ib_mr
*mr
,
476 u64 io_virt
, size_t bcnt
)
478 struct mlx5_ib_dev
*dev
= to_mdev(mr
->ibmr
.pd
->device
);
479 struct ib_umem_odp
*odp
, *result
= NULL
;
480 struct ib_umem_odp
*odp_mr
= to_ib_umem_odp(mr
->umem
);
481 u64 addr
= io_virt
& MLX5_IMR_MTT_MASK
;
482 int nentries
= 0, start_idx
= 0, ret
;
483 struct mlx5_ib_mr
*mtt
;
485 mutex_lock(&odp_mr
->umem_mutex
);
486 odp
= odp_lookup(addr
, 1, mr
);
488 mlx5_ib_dbg(dev
, "io_virt:%llx bcnt:%zx addr:%llx odp:%p\n",
489 io_virt
, bcnt
, addr
, odp
);
496 odp
= ib_umem_odp_alloc_child(odp_mr
, addr
, MLX5_IMR_MTT_SIZE
);
498 mutex_unlock(&odp_mr
->umem_mutex
);
499 return ERR_CAST(odp
);
502 mtt
= implicit_mr_alloc(mr
->ibmr
.pd
, odp
, 0,
505 mutex_unlock(&odp_mr
->umem_mutex
);
506 ib_umem_odp_release(odp
);
507 return ERR_CAST(mtt
);
511 mtt
->umem
= &odp
->umem
;
512 mtt
->mmkey
.iova
= addr
;
514 INIT_WORK(&odp
->work
, mr_leaf_free_action
);
516 smp_store_release(&mtt
->live
, 1);
519 start_idx
= addr
>> MLX5_IMR_MTT_SHIFT
;
523 /* Return first odp if region not covered by single one */
527 addr
+= MLX5_IMR_MTT_SIZE
;
528 if (unlikely(addr
< io_virt
+ bcnt
)) {
530 if (odp
&& ib_umem_start(odp
) != addr
)
535 if (unlikely(nentries
)) {
536 ret
= mlx5_ib_update_xlt(mr
, start_idx
, nentries
, 0,
537 MLX5_IB_UPD_XLT_INDIRECT
|
538 MLX5_IB_UPD_XLT_ATOMIC
);
540 mlx5_ib_err(dev
, "Failed to update PAS\n");
541 result
= ERR_PTR(ret
);
545 mutex_unlock(&odp_mr
->umem_mutex
);
549 struct mlx5_ib_mr
*mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd
*pd
,
550 struct ib_udata
*udata
,
553 struct mlx5_ib_mr
*imr
;
554 struct ib_umem_odp
*umem_odp
;
556 umem_odp
= ib_umem_odp_alloc_implicit(udata
, access_flags
);
557 if (IS_ERR(umem_odp
))
558 return ERR_CAST(umem_odp
);
560 imr
= implicit_mr_alloc(&pd
->ibpd
, umem_odp
, 1, access_flags
);
562 ib_umem_odp_release(umem_odp
);
563 return ERR_CAST(imr
);
566 imr
->umem
= &umem_odp
->umem
;
567 init_waitqueue_head(&imr
->q_leaf_free
);
568 atomic_set(&imr
->num_leaf_free
, 0);
569 atomic_set(&imr
->num_pending_prefetch
, 0);
570 smp_store_release(&imr
->live
, 1);
575 void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr
*imr
)
577 struct ib_ucontext_per_mm
*per_mm
= mr_to_per_mm(imr
);
578 struct rb_node
*node
;
580 down_read(&per_mm
->umem_rwsem
);
581 for (node
= rb_first_cached(&per_mm
->umem_tree
); node
;
582 node
= rb_next(node
)) {
583 struct ib_umem_odp
*umem_odp
=
584 rb_entry(node
, struct ib_umem_odp
, interval_tree
.rb
);
585 struct mlx5_ib_mr
*mr
= umem_odp
->private;
587 if (mr
->parent
!= imr
)
590 mutex_lock(&umem_odp
->umem_mutex
);
591 ib_umem_odp_unmap_dma_pages(umem_odp
, ib_umem_start(umem_odp
),
592 ib_umem_end(umem_odp
));
594 if (umem_odp
->dying
) {
595 mutex_unlock(&umem_odp
->umem_mutex
);
600 atomic_inc(&imr
->num_leaf_free
);
601 schedule_work(&umem_odp
->work
);
602 mutex_unlock(&umem_odp
->umem_mutex
);
604 up_read(&per_mm
->umem_rwsem
);
606 wait_event(imr
->q_leaf_free
, !atomic_read(&imr
->num_leaf_free
));
609 #define MLX5_PF_FLAGS_PREFETCH BIT(0)
610 #define MLX5_PF_FLAGS_DOWNGRADE BIT(1)
611 static int pagefault_mr(struct mlx5_ib_dev
*dev
, struct mlx5_ib_mr
*mr
,
612 u64 io_virt
, size_t bcnt
, u32
*bytes_mapped
,
615 int npages
= 0, current_seq
, page_shift
, ret
, np
;
616 struct ib_umem_odp
*odp_mr
= to_ib_umem_odp(mr
->umem
);
617 bool downgrade
= flags
& MLX5_PF_FLAGS_DOWNGRADE
;
618 bool prefetch
= flags
& MLX5_PF_FLAGS_PREFETCH
;
620 u64 start_idx
, page_mask
;
621 struct ib_umem_odp
*odp
;
624 if (odp_mr
->is_implicit_odp
) {
625 odp
= implicit_mr_get_data(mr
, io_virt
, bcnt
);
635 size
= min_t(size_t, bcnt
, ib_umem_end(odp
) - io_virt
);
637 page_shift
= odp
->page_shift
;
638 page_mask
= ~(BIT(page_shift
) - 1);
639 start_idx
= (io_virt
- (mr
->mmkey
.iova
& page_mask
)) >> page_shift
;
640 access_mask
= ODP_READ_ALLOWED_BIT
;
642 if (prefetch
&& !downgrade
&& !odp
->umem
.writable
) {
643 /* prefetch with write-access must
644 * be supported by the MR
650 if (odp
->umem
.writable
&& !downgrade
)
651 access_mask
|= ODP_WRITE_ALLOWED_BIT
;
653 current_seq
= READ_ONCE(odp
->notifiers_seq
);
655 * Ensure the sequence number is valid for some time before we call
660 ret
= ib_umem_odp_map_dma_pages(odp
, io_virt
, size
, access_mask
,
668 mutex_lock(&odp
->umem_mutex
);
669 if (!ib_umem_mmu_notifier_retry(odp
, current_seq
)) {
671 * No need to check whether the MTTs really belong to
672 * this MR, since ib_umem_odp_map_dma_pages already
675 ret
= mlx5_ib_update_xlt(mr
, start_idx
, np
,
676 page_shift
, MLX5_IB_UPD_XLT_ATOMIC
);
680 mutex_unlock(&odp
->umem_mutex
);
684 mlx5_ib_err(dev
, "Failed to update mkey page tables\n");
689 u32 new_mappings
= (np
<< page_shift
) -
690 (io_virt
- round_down(io_virt
, 1 << page_shift
));
691 *bytes_mapped
+= min_t(u32
, new_mappings
, size
);
694 npages
+= np
<< (page_shift
- PAGE_SHIFT
);
697 if (unlikely(bcnt
)) {
698 struct ib_umem_odp
*next
;
701 next
= odp_next(odp
);
702 if (unlikely(!next
|| ib_umem_start(next
) != io_virt
)) {
703 mlx5_ib_dbg(dev
, "next implicit leaf removed at 0x%llx. got %p\n",
715 if (ret
== -EAGAIN
) {
716 unsigned long timeout
= msecs_to_jiffies(MMU_NOTIFIER_TIMEOUT
);
718 if (!wait_for_completion_timeout(&odp
->notifier_completion
,
722 "timeout waiting for mmu notifier. seq %d against %d. notifiers_count=%d\n",
723 current_seq
, odp
->notifiers_seq
,
724 odp
->notifiers_count
);
732 struct pf_frame
*next
;
739 static bool mkey_is_eq(struct mlx5_core_mkey
*mmkey
, u32 key
)
743 if (mmkey
->type
== MLX5_MKEY_MW
)
744 return mlx5_base_mkey(mmkey
->key
) == mlx5_base_mkey(key
);
745 return mmkey
->key
== key
;
748 static int get_indirect_num_descs(struct mlx5_core_mkey
*mmkey
)
750 struct mlx5_ib_mw
*mw
;
751 struct mlx5_ib_devx_mr
*devx_mr
;
753 if (mmkey
->type
== MLX5_MKEY_MW
) {
754 mw
= container_of(mmkey
, struct mlx5_ib_mw
, mmkey
);
758 devx_mr
= container_of(mmkey
, struct mlx5_ib_devx_mr
,
760 return devx_mr
->ndescs
;
764 * Handle a single data segment in a page-fault WQE or RDMA region.
766 * Returns number of OS pages retrieved on success. The caller may continue to
767 * the next data segment.
768 * Can return the following error codes:
769 * -EAGAIN to designate a temporary error. The caller will abort handling the
770 * page fault and resolve it.
771 * -EFAULT when there's an error mapping the requested pages. The caller will
772 * abort the page fault handling.
774 static int pagefault_single_data_segment(struct mlx5_ib_dev
*dev
,
775 struct ib_pd
*pd
, u32 key
,
776 u64 io_virt
, size_t bcnt
,
777 u32
*bytes_committed
,
778 u32
*bytes_mapped
, u32 flags
)
780 int npages
= 0, srcu_key
, ret
, i
, outlen
, cur_outlen
= 0, depth
= 0;
781 bool prefetch
= flags
& MLX5_PF_FLAGS_PREFETCH
;
782 struct pf_frame
*head
= NULL
, *frame
;
783 struct mlx5_core_mkey
*mmkey
;
784 struct mlx5_ib_mr
*mr
;
785 struct mlx5_klm
*pklm
;
790 srcu_key
= srcu_read_lock(&dev
->mr_srcu
);
792 io_virt
+= *bytes_committed
;
793 bcnt
-= *bytes_committed
;
796 mmkey
= xa_load(&dev
->mdev
->priv
.mkey_table
, mlx5_base_mkey(key
));
797 if (!mkey_is_eq(mmkey
, key
)) {
798 mlx5_ib_dbg(dev
, "failed to find mkey %x\n", key
);
803 if (prefetch
&& mmkey
->type
!= MLX5_MKEY_MR
) {
804 mlx5_ib_dbg(dev
, "prefetch is allowed only for MR\n");
809 switch (mmkey
->type
) {
811 mr
= container_of(mmkey
, struct mlx5_ib_mr
, mmkey
);
812 if (!smp_load_acquire(&mr
->live
) || !mr
->ibmr
.pd
) {
813 mlx5_ib_dbg(dev
, "got dead MR\n");
819 if (!is_odp_mr(mr
) ||
821 mlx5_ib_dbg(dev
, "Invalid prefetch request: %s\n",
822 is_odp_mr(mr
) ? "MR is not ODP" :
823 "PD is not of the MR");
829 if (!is_odp_mr(mr
)) {
830 mlx5_ib_dbg(dev
, "skipping non ODP MR (lkey=0x%06x) in page fault handler.\n",
833 *bytes_mapped
+= bcnt
;
838 ret
= pagefault_mr(dev
, mr
, io_virt
, bcnt
, bytes_mapped
, flags
);
847 case MLX5_MKEY_INDIRECT_DEVX
:
848 ndescs
= get_indirect_num_descs(mmkey
);
850 if (depth
>= MLX5_CAP_GEN(dev
->mdev
, max_indirection
)) {
851 mlx5_ib_dbg(dev
, "indirection level exceeded\n");
856 outlen
= MLX5_ST_SZ_BYTES(query_mkey_out
) +
857 sizeof(*pklm
) * (ndescs
- 2);
859 if (outlen
> cur_outlen
) {
861 out
= kzalloc(outlen
, GFP_KERNEL
);
869 pklm
= (struct mlx5_klm
*)MLX5_ADDR_OF(query_mkey_out
, out
,
870 bsf0_klm0_pas_mtt0_1
);
872 ret
= mlx5_core_query_mkey(dev
->mdev
, mmkey
, out
, outlen
);
876 offset
= io_virt
- MLX5_GET64(query_mkey_out
, out
,
877 memory_key_mkey_entry
.start_addr
);
879 for (i
= 0; bcnt
&& i
< ndescs
; i
++, pklm
++) {
880 if (offset
>= be32_to_cpu(pklm
->bcount
)) {
881 offset
-= be32_to_cpu(pklm
->bcount
);
885 frame
= kzalloc(sizeof(*frame
), GFP_KERNEL
);
891 frame
->key
= be32_to_cpu(pklm
->key
);
892 frame
->io_virt
= be64_to_cpu(pklm
->va
) + offset
;
893 frame
->bcnt
= min_t(size_t, bcnt
,
894 be32_to_cpu(pklm
->bcount
) - offset
);
895 frame
->depth
= depth
+ 1;
905 mlx5_ib_dbg(dev
, "wrong mkey type %d\n", mmkey
->type
);
915 io_virt
= frame
->io_virt
;
917 depth
= frame
->depth
;
931 srcu_read_unlock(&dev
->mr_srcu
, srcu_key
);
932 *bytes_committed
= 0;
933 return ret
? ret
: npages
;
937 * Parse a series of data segments for page fault handling.
939 * @pfault contains page fault information.
940 * @wqe points at the first data segment in the WQE.
941 * @wqe_end points after the end of the WQE.
942 * @bytes_mapped receives the number of bytes that the function was able to
943 * map. This allows the caller to decide intelligently whether
944 * enough memory was mapped to resolve the page fault
945 * successfully (e.g. enough for the next MTU, or the entire
947 * @total_wqe_bytes receives the total data size of this WQE in bytes (minus
948 * the committed bytes).
950 * Returns the number of pages loaded if positive, zero for an empty WQE, or a
951 * negative error code.
953 static int pagefault_data_segments(struct mlx5_ib_dev
*dev
,
954 struct mlx5_pagefault
*pfault
,
956 void *wqe_end
, u32
*bytes_mapped
,
957 u32
*total_wqe_bytes
, bool receive_queue
)
959 int ret
= 0, npages
= 0;
969 *total_wqe_bytes
= 0;
971 while (wqe
< wqe_end
) {
972 struct mlx5_wqe_data_seg
*dseg
= wqe
;
974 io_virt
= be64_to_cpu(dseg
->addr
);
975 key
= be32_to_cpu(dseg
->lkey
);
976 byte_count
= be32_to_cpu(dseg
->byte_count
);
977 inline_segment
= !!(byte_count
& MLX5_INLINE_SEG
);
978 bcnt
= byte_count
& ~MLX5_INLINE_SEG
;
980 if (inline_segment
) {
981 bcnt
= bcnt
& MLX5_WQE_INLINE_SEG_BYTE_COUNT_MASK
;
982 wqe
+= ALIGN(sizeof(struct mlx5_wqe_inline_seg
) + bcnt
,
985 wqe
+= sizeof(*dseg
);
988 /* receive WQE end of sg list. */
989 if (receive_queue
&& bcnt
== 0 && key
== MLX5_INVALID_LKEY
&&
993 if (!inline_segment
&& total_wqe_bytes
) {
994 *total_wqe_bytes
+= bcnt
- min_t(size_t, bcnt
,
995 pfault
->bytes_committed
);
998 /* A zero length data segment designates a length of 2GB. */
1002 if (inline_segment
|| bcnt
<= pfault
->bytes_committed
) {
1003 pfault
->bytes_committed
-=
1005 pfault
->bytes_committed
);
1009 ret
= pagefault_single_data_segment(dev
, NULL
, key
,
1011 &pfault
->bytes_committed
,
1018 return ret
< 0 ? ret
: npages
;
1022 * Parse initiator WQE. Advances the wqe pointer to point at the
1023 * scatter-gather list, and set wqe_end to the end of the WQE.
1025 static int mlx5_ib_mr_initiator_pfault_handler(
1026 struct mlx5_ib_dev
*dev
, struct mlx5_pagefault
*pfault
,
1027 struct mlx5_ib_qp
*qp
, void **wqe
, void **wqe_end
, int wqe_length
)
1029 struct mlx5_wqe_ctrl_seg
*ctrl
= *wqe
;
1030 u16 wqe_index
= pfault
->wqe
.wqe_index
;
1031 struct mlx5_base_av
*av
;
1032 unsigned ds
, opcode
;
1033 u32 qpn
= qp
->trans_qp
.base
.mqp
.qpn
;
1035 ds
= be32_to_cpu(ctrl
->qpn_ds
) & MLX5_WQE_CTRL_DS_MASK
;
1036 if (ds
* MLX5_WQE_DS_UNITS
> wqe_length
) {
1037 mlx5_ib_err(dev
, "Unable to read the complete WQE. ds = 0x%x, ret = 0x%x\n",
1043 mlx5_ib_err(dev
, "Got WQE with zero DS. wqe_index=%x, qpn=%x\n",
1048 *wqe_end
= *wqe
+ ds
* MLX5_WQE_DS_UNITS
;
1049 *wqe
+= sizeof(*ctrl
);
1051 opcode
= be32_to_cpu(ctrl
->opmod_idx_opcode
) &
1052 MLX5_WQE_CTRL_OPCODE_MASK
;
1054 if (qp
->ibqp
.qp_type
== IB_QPT_XRC_INI
)
1055 *wqe
+= sizeof(struct mlx5_wqe_xrc_seg
);
1057 if (qp
->ibqp
.qp_type
== IB_QPT_UD
||
1058 qp
->qp_sub_type
== MLX5_IB_QPT_DCI
) {
1060 if (av
->dqp_dct
& cpu_to_be32(MLX5_EXTENDED_UD_AV
))
1061 *wqe
+= sizeof(struct mlx5_av
);
1063 *wqe
+= sizeof(struct mlx5_base_av
);
1067 case MLX5_OPCODE_RDMA_WRITE
:
1068 case MLX5_OPCODE_RDMA_WRITE_IMM
:
1069 case MLX5_OPCODE_RDMA_READ
:
1070 *wqe
+= sizeof(struct mlx5_wqe_raddr_seg
);
1072 case MLX5_OPCODE_ATOMIC_CS
:
1073 case MLX5_OPCODE_ATOMIC_FA
:
1074 *wqe
+= sizeof(struct mlx5_wqe_raddr_seg
);
1075 *wqe
+= sizeof(struct mlx5_wqe_atomic_seg
);
1083 * Parse responder WQE and set wqe_end to the end of the WQE.
1085 static int mlx5_ib_mr_responder_pfault_handler_srq(struct mlx5_ib_dev
*dev
,
1086 struct mlx5_ib_srq
*srq
,
1087 void **wqe
, void **wqe_end
,
1090 int wqe_size
= 1 << srq
->msrq
.wqe_shift
;
1092 if (wqe_size
> wqe_length
) {
1093 mlx5_ib_err(dev
, "Couldn't read all of the receive WQE's content\n");
1097 *wqe_end
= *wqe
+ wqe_size
;
1098 *wqe
+= sizeof(struct mlx5_wqe_srq_next_seg
);
1103 static int mlx5_ib_mr_responder_pfault_handler_rq(struct mlx5_ib_dev
*dev
,
1104 struct mlx5_ib_qp
*qp
,
1105 void *wqe
, void **wqe_end
,
1108 struct mlx5_ib_wq
*wq
= &qp
->rq
;
1109 int wqe_size
= 1 << wq
->wqe_shift
;
1112 mlx5_ib_err(dev
, "ODP fault with WQE signatures is not supported\n");
1116 if (wqe_size
> wqe_length
) {
1117 mlx5_ib_err(dev
, "Couldn't read all of the receive WQE's content\n");
1121 *wqe_end
= wqe
+ wqe_size
;
1126 static inline struct mlx5_core_rsc_common
*odp_get_rsc(struct mlx5_ib_dev
*dev
,
1127 u32 wq_num
, int pf_type
)
1129 struct mlx5_core_rsc_common
*common
= NULL
;
1130 struct mlx5_core_srq
*srq
;
1133 case MLX5_WQE_PF_TYPE_RMP
:
1134 srq
= mlx5_cmd_get_srq(dev
, wq_num
);
1136 common
= &srq
->common
;
1138 case MLX5_WQE_PF_TYPE_REQ_SEND_OR_WRITE
:
1139 case MLX5_WQE_PF_TYPE_RESP
:
1140 case MLX5_WQE_PF_TYPE_REQ_READ_OR_ATOMIC
:
1141 common
= mlx5_core_res_hold(dev
->mdev
, wq_num
, MLX5_RES_QP
);
1150 static inline struct mlx5_ib_qp
*res_to_qp(struct mlx5_core_rsc_common
*res
)
1152 struct mlx5_core_qp
*mqp
= (struct mlx5_core_qp
*)res
;
1154 return to_mibqp(mqp
);
1157 static inline struct mlx5_ib_srq
*res_to_srq(struct mlx5_core_rsc_common
*res
)
1159 struct mlx5_core_srq
*msrq
=
1160 container_of(res
, struct mlx5_core_srq
, common
);
1162 return to_mibsrq(msrq
);
1165 static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev
*dev
,
1166 struct mlx5_pagefault
*pfault
)
1168 bool sq
= pfault
->type
& MLX5_PFAULT_REQUESTOR
;
1169 u16 wqe_index
= pfault
->wqe
.wqe_index
;
1170 void *wqe
, *wqe_start
= NULL
, *wqe_end
= NULL
;
1171 u32 bytes_mapped
, total_wqe_bytes
;
1172 struct mlx5_core_rsc_common
*res
;
1173 int resume_with_error
= 1;
1174 struct mlx5_ib_qp
*qp
;
1175 size_t bytes_copied
;
1178 res
= odp_get_rsc(dev
, pfault
->wqe
.wq_num
, pfault
->type
);
1180 mlx5_ib_dbg(dev
, "wqe page fault for missing resource %d\n", pfault
->wqe
.wq_num
);
1184 if (res
->res
!= MLX5_RES_QP
&& res
->res
!= MLX5_RES_SRQ
&&
1185 res
->res
!= MLX5_RES_XSRQ
) {
1186 mlx5_ib_err(dev
, "wqe page fault for unsupported type %d\n",
1188 goto resolve_page_fault
;
1191 wqe_start
= (void *)__get_free_page(GFP_KERNEL
);
1193 mlx5_ib_err(dev
, "Error allocating memory for IO page fault handling.\n");
1194 goto resolve_page_fault
;
1198 qp
= (res
->res
== MLX5_RES_QP
) ? res_to_qp(res
) : NULL
;
1200 ret
= mlx5_ib_read_user_wqe_sq(qp
, wqe_index
, wqe
, PAGE_SIZE
,
1204 ret
= mlx5_ib_mr_initiator_pfault_handler(
1205 dev
, pfault
, qp
, &wqe
, &wqe_end
, bytes_copied
);
1206 } else if (qp
&& !sq
) {
1207 ret
= mlx5_ib_read_user_wqe_rq(qp
, wqe_index
, wqe
, PAGE_SIZE
,
1211 ret
= mlx5_ib_mr_responder_pfault_handler_rq(
1212 dev
, qp
, wqe
, &wqe_end
, bytes_copied
);
1214 struct mlx5_ib_srq
*srq
= res_to_srq(res
);
1216 ret
= mlx5_ib_read_user_wqe_srq(srq
, wqe_index
, wqe
, PAGE_SIZE
,
1220 ret
= mlx5_ib_mr_responder_pfault_handler_srq(
1221 dev
, srq
, &wqe
, &wqe_end
, bytes_copied
);
1224 if (ret
< 0 || wqe
>= wqe_end
)
1225 goto resolve_page_fault
;
1227 ret
= pagefault_data_segments(dev
, pfault
, wqe
, wqe_end
, &bytes_mapped
,
1228 &total_wqe_bytes
, !sq
);
1232 if (ret
< 0 || total_wqe_bytes
> bytes_mapped
)
1233 goto resolve_page_fault
;
1237 resume_with_error
= 0;
1243 "Failed reading a WQE following page fault, error %d, wqe_index %x, qpn %x\n",
1244 ret
, wqe_index
, pfault
->token
);
1247 mlx5_ib_page_fault_resume(dev
, pfault
, resume_with_error
);
1248 mlx5_ib_dbg(dev
, "PAGE FAULT completed. QP 0x%x resume_with_error=%d, type: 0x%x\n",
1249 pfault
->wqe
.wq_num
, resume_with_error
,
1251 mlx5_core_res_put(res
);
1252 free_page((unsigned long)wqe_start
);
1255 static int pages_in_range(u64 address
, u32 length
)
1257 return (ALIGN(address
+ length
, PAGE_SIZE
) -
1258 (address
& PAGE_MASK
)) >> PAGE_SHIFT
;
1261 static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev
*dev
,
1262 struct mlx5_pagefault
*pfault
)
1266 u32 prefetch_len
= pfault
->bytes_committed
;
1267 int prefetch_activated
= 0;
1268 u32 rkey
= pfault
->rdma
.r_key
;
1271 /* The RDMA responder handler handles the page fault in two parts.
1272 * First it brings the necessary pages for the current packet
1273 * (and uses the pfault context), and then (after resuming the QP)
1274 * prefetches more pages. The second operation cannot use the pfault
1275 * context and therefore uses the dummy_pfault context allocated on
1277 pfault
->rdma
.rdma_va
+= pfault
->bytes_committed
;
1278 pfault
->rdma
.rdma_op_len
-= min(pfault
->bytes_committed
,
1279 pfault
->rdma
.rdma_op_len
);
1280 pfault
->bytes_committed
= 0;
1282 address
= pfault
->rdma
.rdma_va
;
1283 length
= pfault
->rdma
.rdma_op_len
;
1285 /* For some operations, the hardware cannot tell the exact message
1286 * length, and in those cases it reports zero. Use prefetch
1289 prefetch_activated
= 1;
1290 length
= pfault
->rdma
.packet_size
;
1291 prefetch_len
= min(MAX_PREFETCH_LEN
, prefetch_len
);
1294 ret
= pagefault_single_data_segment(dev
, NULL
, rkey
, address
, length
,
1295 &pfault
->bytes_committed
, NULL
,
1297 if (ret
== -EAGAIN
) {
1298 /* We're racing with an invalidation, don't prefetch */
1299 prefetch_activated
= 0;
1300 } else if (ret
< 0 || pages_in_range(address
, length
) > ret
) {
1301 mlx5_ib_page_fault_resume(dev
, pfault
, 1);
1303 mlx5_ib_dbg(dev
, "PAGE FAULT error %d. QP 0x%x, type: 0x%x\n",
1304 ret
, pfault
->token
, pfault
->type
);
1308 mlx5_ib_page_fault_resume(dev
, pfault
, 0);
1309 mlx5_ib_dbg(dev
, "PAGE FAULT completed. QP 0x%x, type: 0x%x, prefetch_activated: %d\n",
1310 pfault
->token
, pfault
->type
,
1311 prefetch_activated
);
1313 /* At this point, there might be a new pagefault already arriving in
1314 * the eq, switch to the dummy pagefault for the rest of the
1315 * processing. We're still OK with the objects being alive as the
1316 * work-queue is being fenced. */
1318 if (prefetch_activated
) {
1319 u32 bytes_committed
= 0;
1321 ret
= pagefault_single_data_segment(dev
, NULL
, rkey
, address
,
1323 &bytes_committed
, NULL
,
1325 if (ret
< 0 && ret
!= -EAGAIN
) {
1326 mlx5_ib_dbg(dev
, "Prefetch failed. ret: %d, QP 0x%x, address: 0x%.16llx, length = 0x%.16x\n",
1327 ret
, pfault
->token
, address
, prefetch_len
);
1332 static void mlx5_ib_pfault(struct mlx5_ib_dev
*dev
, struct mlx5_pagefault
*pfault
)
1334 u8 event_subtype
= pfault
->event_subtype
;
1336 switch (event_subtype
) {
1337 case MLX5_PFAULT_SUBTYPE_WQE
:
1338 mlx5_ib_mr_wqe_pfault_handler(dev
, pfault
);
1340 case MLX5_PFAULT_SUBTYPE_RDMA
:
1341 mlx5_ib_mr_rdma_pfault_handler(dev
, pfault
);
1344 mlx5_ib_err(dev
, "Invalid page fault event subtype: 0x%x\n",
1346 mlx5_ib_page_fault_resume(dev
, pfault
, 1);
1350 static void mlx5_ib_eqe_pf_action(struct work_struct
*work
)
1352 struct mlx5_pagefault
*pfault
= container_of(work
,
1353 struct mlx5_pagefault
,
1355 struct mlx5_ib_pf_eq
*eq
= pfault
->eq
;
1357 mlx5_ib_pfault(eq
->dev
, pfault
);
1358 mempool_free(pfault
, eq
->pool
);
1361 static void mlx5_ib_eq_pf_process(struct mlx5_ib_pf_eq
*eq
)
1363 struct mlx5_eqe_page_fault
*pf_eqe
;
1364 struct mlx5_pagefault
*pfault
;
1365 struct mlx5_eqe
*eqe
;
1368 while ((eqe
= mlx5_eq_get_eqe(eq
->core
, cc
))) {
1369 pfault
= mempool_alloc(eq
->pool
, GFP_ATOMIC
);
1371 schedule_work(&eq
->work
);
1375 pf_eqe
= &eqe
->data
.page_fault
;
1376 pfault
->event_subtype
= eqe
->sub_type
;
1377 pfault
->bytes_committed
= be32_to_cpu(pf_eqe
->bytes_committed
);
1379 mlx5_ib_dbg(eq
->dev
,
1380 "PAGE_FAULT: subtype: 0x%02x, bytes_committed: 0x%06x\n",
1381 eqe
->sub_type
, pfault
->bytes_committed
);
1383 switch (eqe
->sub_type
) {
1384 case MLX5_PFAULT_SUBTYPE_RDMA
:
1385 /* RDMA based event */
1387 be32_to_cpu(pf_eqe
->rdma
.pftype_token
) >> 24;
1389 be32_to_cpu(pf_eqe
->rdma
.pftype_token
) &
1391 pfault
->rdma
.r_key
=
1392 be32_to_cpu(pf_eqe
->rdma
.r_key
);
1393 pfault
->rdma
.packet_size
=
1394 be16_to_cpu(pf_eqe
->rdma
.packet_length
);
1395 pfault
->rdma
.rdma_op_len
=
1396 be32_to_cpu(pf_eqe
->rdma
.rdma_op_len
);
1397 pfault
->rdma
.rdma_va
=
1398 be64_to_cpu(pf_eqe
->rdma
.rdma_va
);
1399 mlx5_ib_dbg(eq
->dev
,
1400 "PAGE_FAULT: type:0x%x, token: 0x%06x, r_key: 0x%08x\n",
1401 pfault
->type
, pfault
->token
,
1402 pfault
->rdma
.r_key
);
1403 mlx5_ib_dbg(eq
->dev
,
1404 "PAGE_FAULT: rdma_op_len: 0x%08x, rdma_va: 0x%016llx\n",
1405 pfault
->rdma
.rdma_op_len
,
1406 pfault
->rdma
.rdma_va
);
1409 case MLX5_PFAULT_SUBTYPE_WQE
:
1410 /* WQE based event */
1412 (be32_to_cpu(pf_eqe
->wqe
.pftype_wq
) >> 24) & 0x7;
1414 be32_to_cpu(pf_eqe
->wqe
.token
);
1415 pfault
->wqe
.wq_num
=
1416 be32_to_cpu(pf_eqe
->wqe
.pftype_wq
) &
1418 pfault
->wqe
.wqe_index
=
1419 be16_to_cpu(pf_eqe
->wqe
.wqe_index
);
1420 pfault
->wqe
.packet_size
=
1421 be16_to_cpu(pf_eqe
->wqe
.packet_length
);
1422 mlx5_ib_dbg(eq
->dev
,
1423 "PAGE_FAULT: type:0x%x, token: 0x%06x, wq_num: 0x%06x, wqe_index: 0x%04x\n",
1424 pfault
->type
, pfault
->token
,
1426 pfault
->wqe
.wqe_index
);
1430 mlx5_ib_warn(eq
->dev
,
1431 "Unsupported page fault event sub-type: 0x%02hhx\n",
1433 /* Unsupported page faults should still be
1434 * resolved by the page fault handler
1439 INIT_WORK(&pfault
->work
, mlx5_ib_eqe_pf_action
);
1440 queue_work(eq
->wq
, &pfault
->work
);
1442 cc
= mlx5_eq_update_cc(eq
->core
, ++cc
);
1445 mlx5_eq_update_ci(eq
->core
, cc
, 1);
1448 static int mlx5_ib_eq_pf_int(struct notifier_block
*nb
, unsigned long type
,
1451 struct mlx5_ib_pf_eq
*eq
=
1452 container_of(nb
, struct mlx5_ib_pf_eq
, irq_nb
);
1453 unsigned long flags
;
1455 if (spin_trylock_irqsave(&eq
->lock
, flags
)) {
1456 mlx5_ib_eq_pf_process(eq
);
1457 spin_unlock_irqrestore(&eq
->lock
, flags
);
1459 schedule_work(&eq
->work
);
1465 /* mempool_refill() was proposed but unfortunately wasn't accepted
1466 * http://lkml.iu.edu/hypermail/linux/kernel/1512.1/05073.html
1469 static void mempool_refill(mempool_t
*pool
)
1471 while (pool
->curr_nr
< pool
->min_nr
)
1472 mempool_free(mempool_alloc(pool
, GFP_KERNEL
), pool
);
1475 static void mlx5_ib_eq_pf_action(struct work_struct
*work
)
1477 struct mlx5_ib_pf_eq
*eq
=
1478 container_of(work
, struct mlx5_ib_pf_eq
, work
);
1480 mempool_refill(eq
->pool
);
1482 spin_lock_irq(&eq
->lock
);
1483 mlx5_ib_eq_pf_process(eq
);
1484 spin_unlock_irq(&eq
->lock
);
1488 MLX5_IB_NUM_PF_EQE
= 0x1000,
1489 MLX5_IB_NUM_PF_DRAIN
= 64,
1493 mlx5_ib_create_pf_eq(struct mlx5_ib_dev
*dev
, struct mlx5_ib_pf_eq
*eq
)
1495 struct mlx5_eq_param param
= {};
1498 INIT_WORK(&eq
->work
, mlx5_ib_eq_pf_action
);
1499 spin_lock_init(&eq
->lock
);
1502 eq
->pool
= mempool_create_kmalloc_pool(MLX5_IB_NUM_PF_DRAIN
,
1503 sizeof(struct mlx5_pagefault
));
1507 eq
->wq
= alloc_workqueue("mlx5_ib_page_fault",
1508 WQ_HIGHPRI
| WQ_UNBOUND
| WQ_MEM_RECLAIM
,
1515 eq
->irq_nb
.notifier_call
= mlx5_ib_eq_pf_int
;
1516 param
= (struct mlx5_eq_param
) {
1518 .nent
= MLX5_IB_NUM_PF_EQE
,
1520 param
.mask
[0] = 1ull << MLX5_EVENT_TYPE_PAGE_FAULT
;
1521 eq
->core
= mlx5_eq_create_generic(dev
->mdev
, ¶m
);
1522 if (IS_ERR(eq
->core
)) {
1523 err
= PTR_ERR(eq
->core
);
1526 err
= mlx5_eq_enable(dev
->mdev
, eq
->core
, &eq
->irq_nb
);
1528 mlx5_ib_err(dev
, "failed to enable odp EQ %d\n", err
);
1534 mlx5_eq_destroy_generic(dev
->mdev
, eq
->core
);
1536 destroy_workqueue(eq
->wq
);
1538 mempool_destroy(eq
->pool
);
1543 mlx5_ib_destroy_pf_eq(struct mlx5_ib_dev
*dev
, struct mlx5_ib_pf_eq
*eq
)
1547 mlx5_eq_disable(dev
->mdev
, eq
->core
, &eq
->irq_nb
);
1548 err
= mlx5_eq_destroy_generic(dev
->mdev
, eq
->core
);
1549 cancel_work_sync(&eq
->work
);
1550 destroy_workqueue(eq
->wq
);
1551 mempool_destroy(eq
->pool
);
1556 void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent
*ent
)
1558 if (!(ent
->dev
->odp_caps
.general_caps
& IB_ODP_SUPPORT_IMPLICIT
))
1561 switch (ent
->order
- 2) {
1562 case MLX5_IMR_MTT_CACHE_ENTRY
:
1563 ent
->page
= PAGE_SHIFT
;
1564 ent
->xlt
= MLX5_IMR_MTT_ENTRIES
*
1565 sizeof(struct mlx5_mtt
) /
1566 MLX5_IB_UMR_OCTOWORD
;
1567 ent
->access_mode
= MLX5_MKC_ACCESS_MODE_MTT
;
1571 case MLX5_IMR_KSM_CACHE_ENTRY
:
1572 ent
->page
= MLX5_KSM_PAGE_SHIFT
;
1573 ent
->xlt
= mlx5_imr_ksm_entries
*
1574 sizeof(struct mlx5_klm
) /
1575 MLX5_IB_UMR_OCTOWORD
;
1576 ent
->access_mode
= MLX5_MKC_ACCESS_MODE_KSM
;
1582 static const struct ib_device_ops mlx5_ib_dev_odp_ops
= {
1583 .advise_mr
= mlx5_ib_advise_mr
,
1584 .invalidate_range
= mlx5_ib_invalidate_range
,
1587 int mlx5_ib_odp_init_one(struct mlx5_ib_dev
*dev
)
1591 if (!(dev
->odp_caps
.general_caps
& IB_ODP_SUPPORT
))
1594 ib_set_device_ops(&dev
->ib_dev
, &mlx5_ib_dev_odp_ops
);
1596 if (dev
->odp_caps
.general_caps
& IB_ODP_SUPPORT_IMPLICIT
) {
1597 ret
= mlx5_cmd_null_mkey(dev
->mdev
, &dev
->null_mkey
);
1599 mlx5_ib_err(dev
, "Error getting null_mkey %d\n", ret
);
1604 ret
= mlx5_ib_create_pf_eq(dev
, &dev
->odp_pf_eq
);
1609 void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev
*dev
)
1611 if (!(dev
->odp_caps
.general_caps
& IB_ODP_SUPPORT
))
1614 mlx5_ib_destroy_pf_eq(dev
, &dev
->odp_pf_eq
);
1617 int mlx5_ib_odp_init(void)
1619 mlx5_imr_ksm_entries
= BIT_ULL(get_order(TASK_SIZE
) -
1625 struct prefetch_mr_work
{
1626 struct work_struct work
;
1630 struct ib_sge sg_list
[0];
1633 static void num_pending_prefetch_dec(struct mlx5_ib_dev
*dev
,
1634 struct ib_sge
*sg_list
, u32 num_sge
,
1640 srcu_key
= srcu_read_lock(&dev
->mr_srcu
);
1642 for (i
= from
; i
< num_sge
; ++i
) {
1643 struct mlx5_core_mkey
*mmkey
;
1644 struct mlx5_ib_mr
*mr
;
1646 mmkey
= xa_load(&dev
->mdev
->priv
.mkey_table
,
1647 mlx5_base_mkey(sg_list
[i
].lkey
));
1648 mr
= container_of(mmkey
, struct mlx5_ib_mr
, mmkey
);
1649 atomic_dec(&mr
->num_pending_prefetch
);
1652 srcu_read_unlock(&dev
->mr_srcu
, srcu_key
);
1655 static bool num_pending_prefetch_inc(struct ib_pd
*pd
,
1656 struct ib_sge
*sg_list
, u32 num_sge
)
1658 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
1662 for (i
= 0; i
< num_sge
; ++i
) {
1663 struct mlx5_core_mkey
*mmkey
;
1664 struct mlx5_ib_mr
*mr
;
1666 mmkey
= xa_load(&dev
->mdev
->priv
.mkey_table
,
1667 mlx5_base_mkey(sg_list
[i
].lkey
));
1668 if (!mmkey
|| mmkey
->key
!= sg_list
[i
].lkey
) {
1673 if (mmkey
->type
!= MLX5_MKEY_MR
) {
1678 mr
= container_of(mmkey
, struct mlx5_ib_mr
, mmkey
);
1680 if (!smp_load_acquire(&mr
->live
)) {
1685 if (mr
->ibmr
.pd
!= pd
) {
1690 atomic_inc(&mr
->num_pending_prefetch
);
1694 num_pending_prefetch_dec(dev
, sg_list
, i
, 0);
1699 static int mlx5_ib_prefetch_sg_list(struct ib_pd
*pd
, u32 pf_flags
,
1700 struct ib_sge
*sg_list
, u32 num_sge
)
1704 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
1706 for (i
= 0; i
< num_sge
; ++i
) {
1707 struct ib_sge
*sg
= &sg_list
[i
];
1708 int bytes_committed
= 0;
1710 ret
= pagefault_single_data_segment(dev
, pd
, sg
->lkey
, sg
->addr
,
1712 &bytes_committed
, NULL
,
1718 return ret
< 0 ? ret
: 0;
1721 static void mlx5_ib_prefetch_mr_work(struct work_struct
*work
)
1723 struct prefetch_mr_work
*w
=
1724 container_of(work
, struct prefetch_mr_work
, work
);
1726 if (ib_device_try_get(w
->pd
->device
)) {
1727 mlx5_ib_prefetch_sg_list(w
->pd
, w
->pf_flags
, w
->sg_list
,
1729 ib_device_put(w
->pd
->device
);
1732 num_pending_prefetch_dec(to_mdev(w
->pd
->device
), w
->sg_list
,
1737 int mlx5_ib_advise_mr_prefetch(struct ib_pd
*pd
,
1738 enum ib_uverbs_advise_mr_advice advice
,
1739 u32 flags
, struct ib_sge
*sg_list
, u32 num_sge
)
1741 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
1742 u32 pf_flags
= MLX5_PF_FLAGS_PREFETCH
;
1743 struct prefetch_mr_work
*work
;
1747 if (advice
== IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH
)
1748 pf_flags
|= MLX5_PF_FLAGS_DOWNGRADE
;
1750 if (flags
& IB_UVERBS_ADVISE_MR_FLAG_FLUSH
)
1751 return mlx5_ib_prefetch_sg_list(pd
, pf_flags
, sg_list
,
1754 work
= kvzalloc(struct_size(work
, sg_list
, num_sge
), GFP_KERNEL
);
1758 memcpy(work
->sg_list
, sg_list
, num_sge
* sizeof(struct ib_sge
));
1760 /* It is guaranteed that the pd when work is executed is the pd when
1761 * work was queued since pd can't be destroyed while it holds MRs and
1762 * destroying a MR leads to flushing the workquque
1765 work
->pf_flags
= pf_flags
;
1766 work
->num_sge
= num_sge
;
1768 INIT_WORK(&work
->work
, mlx5_ib_prefetch_mr_work
);
1770 srcu_key
= srcu_read_lock(&dev
->mr_srcu
);
1772 valid_req
= num_pending_prefetch_inc(pd
, sg_list
, num_sge
);
1774 queue_work(system_unbound_wq
, &work
->work
);
1778 srcu_read_unlock(&dev
->mr_srcu
, srcu_key
);
1780 return valid_req
? 0 : -EINVAL
;