2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/kref.h>
35 #include <linux/random.h>
36 #include <linux/debugfs.h>
37 #include <linux/export.h>
38 #include <linux/delay.h>
39 #include <rdma/ib_umem.h>
40 #include <rdma/ib_umem_odp.h>
41 #include <rdma/ib_verbs.h>
46 MAX_PENDING_REG_MR
= 8,
49 #define MLX5_UMR_ALIGN 2048
50 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
51 static __be64 mlx5_ib_update_mtt_emergency_buffer
[
52 MLX5_UMR_MTT_MIN_CHUNK_SIZE
/sizeof(__be64
)]
53 __aligned(MLX5_UMR_ALIGN
);
54 static DEFINE_MUTEX(mlx5_ib_update_mtt_emergency_buffer_mutex
);
57 static int clean_mr(struct mlx5_ib_mr
*mr
);
59 static int destroy_mkey(struct mlx5_ib_dev
*dev
, struct mlx5_ib_mr
*mr
)
61 int err
= mlx5_core_destroy_mkey(dev
->mdev
, &mr
->mmkey
);
63 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
64 /* Wait until all page fault handlers using the mr complete. */
65 synchronize_srcu(&dev
->mr_srcu
);
71 static int order2idx(struct mlx5_ib_dev
*dev
, int order
)
73 struct mlx5_mr_cache
*cache
= &dev
->cache
;
75 if (order
< cache
->ent
[0].order
)
78 return order
- cache
->ent
[0].order
;
81 static bool use_umr_mtt_update(struct mlx5_ib_mr
*mr
, u64 start
, u64 length
)
83 return ((u64
)1 << mr
->order
) * MLX5_ADAPTER_PAGE_SIZE
>=
84 length
+ (start
& (MLX5_ADAPTER_PAGE_SIZE
- 1));
87 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
88 static void update_odp_mr(struct mlx5_ib_mr
*mr
)
90 if (mr
->umem
->odp_data
) {
92 * This barrier prevents the compiler from moving the
93 * setting of umem->odp_data->private to point to our
94 * MR, before reg_umr finished, to ensure that the MR
95 * initialization have finished before starting to
96 * handle invalidations.
99 mr
->umem
->odp_data
->private = mr
;
101 * Make sure we will see the new
102 * umem->odp_data->private value in the invalidation
103 * routines, before we can get page faults on the
104 * MR. Page faults can happen once we put the MR in
105 * the tree, below this line. Without the barrier,
106 * there can be a fault handling and an invalidation
107 * before umem->odp_data->private == mr is visible to
108 * the invalidation handler.
115 static void reg_mr_callback(int status
, void *context
)
117 struct mlx5_ib_mr
*mr
= context
;
118 struct mlx5_ib_dev
*dev
= mr
->dev
;
119 struct mlx5_mr_cache
*cache
= &dev
->cache
;
120 int c
= order2idx(dev
, mr
->order
);
121 struct mlx5_cache_ent
*ent
= &cache
->ent
[c
];
124 struct mlx5_mkey_table
*table
= &dev
->mdev
->priv
.mkey_table
;
127 spin_lock_irqsave(&ent
->lock
, flags
);
129 spin_unlock_irqrestore(&ent
->lock
, flags
);
131 mlx5_ib_warn(dev
, "async reg mr failed. status %d\n", status
);
134 mod_timer(&dev
->delay_timer
, jiffies
+ HZ
);
138 spin_lock_irqsave(&dev
->mdev
->priv
.mkey_lock
, flags
);
139 key
= dev
->mdev
->priv
.mkey_key
++;
140 spin_unlock_irqrestore(&dev
->mdev
->priv
.mkey_lock
, flags
);
141 mr
->mmkey
.key
= mlx5_idx_to_mkey(MLX5_GET(create_mkey_out
, mr
->out
, mkey_index
)) | key
;
143 cache
->last_add
= jiffies
;
145 spin_lock_irqsave(&ent
->lock
, flags
);
146 list_add_tail(&mr
->list
, &ent
->head
);
149 spin_unlock_irqrestore(&ent
->lock
, flags
);
151 write_lock_irqsave(&table
->lock
, flags
);
152 err
= radix_tree_insert(&table
->tree
, mlx5_base_mkey(mr
->mmkey
.key
),
155 pr_err("Error inserting to mkey tree. 0x%x\n", -err
);
156 write_unlock_irqrestore(&table
->lock
, flags
);
159 static int add_keys(struct mlx5_ib_dev
*dev
, int c
, int num
)
161 struct mlx5_mr_cache
*cache
= &dev
->cache
;
162 struct mlx5_cache_ent
*ent
= &cache
->ent
[c
];
163 int inlen
= MLX5_ST_SZ_BYTES(create_mkey_in
);
164 struct mlx5_ib_mr
*mr
;
165 int npages
= 1 << ent
->order
;
171 in
= kzalloc(inlen
, GFP_KERNEL
);
175 mkc
= MLX5_ADDR_OF(create_mkey_in
, in
, memory_key_mkey_entry
);
176 for (i
= 0; i
< num
; i
++) {
177 if (ent
->pending
>= MAX_PENDING_REG_MR
) {
182 mr
= kzalloc(sizeof(*mr
), GFP_KERNEL
);
187 mr
->order
= ent
->order
;
191 MLX5_SET(mkc
, mkc
, free
, 1);
192 MLX5_SET(mkc
, mkc
, umr_en
, 1);
193 MLX5_SET(mkc
, mkc
, access_mode
, MLX5_MKC_ACCESS_MODE_MTT
);
195 MLX5_SET(mkc
, mkc
, qpn
, 0xffffff);
196 MLX5_SET(mkc
, mkc
, translations_octword_size
, (npages
+ 1) / 2);
197 MLX5_SET(mkc
, mkc
, log_page_size
, 12);
199 spin_lock_irq(&ent
->lock
);
201 spin_unlock_irq(&ent
->lock
);
202 err
= mlx5_core_create_mkey_cb(dev
->mdev
, &mr
->mmkey
,
204 mr
->out
, sizeof(mr
->out
),
205 reg_mr_callback
, mr
);
207 spin_lock_irq(&ent
->lock
);
209 spin_unlock_irq(&ent
->lock
);
210 mlx5_ib_warn(dev
, "create mkey failed %d\n", err
);
220 static void remove_keys(struct mlx5_ib_dev
*dev
, int c
, int num
)
222 struct mlx5_mr_cache
*cache
= &dev
->cache
;
223 struct mlx5_cache_ent
*ent
= &cache
->ent
[c
];
224 struct mlx5_ib_mr
*mr
;
228 for (i
= 0; i
< num
; i
++) {
229 spin_lock_irq(&ent
->lock
);
230 if (list_empty(&ent
->head
)) {
231 spin_unlock_irq(&ent
->lock
);
234 mr
= list_first_entry(&ent
->head
, struct mlx5_ib_mr
, list
);
238 spin_unlock_irq(&ent
->lock
);
239 err
= destroy_mkey(dev
, mr
);
241 mlx5_ib_warn(dev
, "failed destroy mkey\n");
247 static ssize_t
size_write(struct file
*filp
, const char __user
*buf
,
248 size_t count
, loff_t
*pos
)
250 struct mlx5_cache_ent
*ent
= filp
->private_data
;
251 struct mlx5_ib_dev
*dev
= ent
->dev
;
257 if (copy_from_user(lbuf
, buf
, sizeof(lbuf
)))
260 c
= order2idx(dev
, ent
->order
);
261 lbuf
[sizeof(lbuf
) - 1] = 0;
263 if (sscanf(lbuf
, "%u", &var
) != 1)
266 if (var
< ent
->limit
)
269 if (var
> ent
->size
) {
271 err
= add_keys(dev
, c
, var
- ent
->size
);
272 if (err
&& err
!= -EAGAIN
)
275 usleep_range(3000, 5000);
277 } else if (var
< ent
->size
) {
278 remove_keys(dev
, c
, ent
->size
- var
);
284 static ssize_t
size_read(struct file
*filp
, char __user
*buf
, size_t count
,
287 struct mlx5_cache_ent
*ent
= filp
->private_data
;
294 err
= snprintf(lbuf
, sizeof(lbuf
), "%d\n", ent
->size
);
298 if (copy_to_user(buf
, lbuf
, err
))
306 static const struct file_operations size_fops
= {
307 .owner
= THIS_MODULE
,
313 static ssize_t
limit_write(struct file
*filp
, const char __user
*buf
,
314 size_t count
, loff_t
*pos
)
316 struct mlx5_cache_ent
*ent
= filp
->private_data
;
317 struct mlx5_ib_dev
*dev
= ent
->dev
;
323 if (copy_from_user(lbuf
, buf
, sizeof(lbuf
)))
326 c
= order2idx(dev
, ent
->order
);
327 lbuf
[sizeof(lbuf
) - 1] = 0;
329 if (sscanf(lbuf
, "%u", &var
) != 1)
337 if (ent
->cur
< ent
->limit
) {
338 err
= add_keys(dev
, c
, 2 * ent
->limit
- ent
->cur
);
346 static ssize_t
limit_read(struct file
*filp
, char __user
*buf
, size_t count
,
349 struct mlx5_cache_ent
*ent
= filp
->private_data
;
356 err
= snprintf(lbuf
, sizeof(lbuf
), "%d\n", ent
->limit
);
360 if (copy_to_user(buf
, lbuf
, err
))
368 static const struct file_operations limit_fops
= {
369 .owner
= THIS_MODULE
,
371 .write
= limit_write
,
375 static int someone_adding(struct mlx5_mr_cache
*cache
)
379 for (i
= 0; i
< MAX_MR_CACHE_ENTRIES
; i
++) {
380 if (cache
->ent
[i
].cur
< cache
->ent
[i
].limit
)
387 static void __cache_work_func(struct mlx5_cache_ent
*ent
)
389 struct mlx5_ib_dev
*dev
= ent
->dev
;
390 struct mlx5_mr_cache
*cache
= &dev
->cache
;
391 int i
= order2idx(dev
, ent
->order
);
397 ent
= &dev
->cache
.ent
[i
];
398 if (ent
->cur
< 2 * ent
->limit
&& !dev
->fill_delay
) {
399 err
= add_keys(dev
, i
, 1);
400 if (ent
->cur
< 2 * ent
->limit
) {
401 if (err
== -EAGAIN
) {
402 mlx5_ib_dbg(dev
, "returned eagain, order %d\n",
404 queue_delayed_work(cache
->wq
, &ent
->dwork
,
405 msecs_to_jiffies(3));
407 mlx5_ib_warn(dev
, "command failed order %d, err %d\n",
409 queue_delayed_work(cache
->wq
, &ent
->dwork
,
410 msecs_to_jiffies(1000));
412 queue_work(cache
->wq
, &ent
->work
);
415 } else if (ent
->cur
> 2 * ent
->limit
) {
417 * The remove_keys() logic is performed as garbage collection
418 * task. Such task is intended to be run when no other active
419 * processes are running.
421 * The need_resched() will return TRUE if there are user tasks
422 * to be activated in near future.
424 * In such case, we don't execute remove_keys() and postpone
425 * the garbage collection work to try to run in next cycle,
426 * in order to free CPU resources to other tasks.
428 if (!need_resched() && !someone_adding(cache
) &&
429 time_after(jiffies
, cache
->last_add
+ 300 * HZ
)) {
430 remove_keys(dev
, i
, 1);
431 if (ent
->cur
> ent
->limit
)
432 queue_work(cache
->wq
, &ent
->work
);
434 queue_delayed_work(cache
->wq
, &ent
->dwork
, 300 * HZ
);
439 static void delayed_cache_work_func(struct work_struct
*work
)
441 struct mlx5_cache_ent
*ent
;
443 ent
= container_of(work
, struct mlx5_cache_ent
, dwork
.work
);
444 __cache_work_func(ent
);
447 static void cache_work_func(struct work_struct
*work
)
449 struct mlx5_cache_ent
*ent
;
451 ent
= container_of(work
, struct mlx5_cache_ent
, work
);
452 __cache_work_func(ent
);
455 static struct mlx5_ib_mr
*alloc_cached_mr(struct mlx5_ib_dev
*dev
, int order
)
457 struct mlx5_mr_cache
*cache
= &dev
->cache
;
458 struct mlx5_ib_mr
*mr
= NULL
;
459 struct mlx5_cache_ent
*ent
;
463 c
= order2idx(dev
, order
);
464 if (c
< 0 || c
>= MAX_MR_CACHE_ENTRIES
) {
465 mlx5_ib_warn(dev
, "order %d, cache index %d\n", order
, c
);
469 for (i
= c
; i
< MAX_MR_CACHE_ENTRIES
; i
++) {
470 ent
= &cache
->ent
[i
];
472 mlx5_ib_dbg(dev
, "order %d, cache index %d\n", ent
->order
, i
);
474 spin_lock_irq(&ent
->lock
);
475 if (!list_empty(&ent
->head
)) {
476 mr
= list_first_entry(&ent
->head
, struct mlx5_ib_mr
,
480 spin_unlock_irq(&ent
->lock
);
481 if (ent
->cur
< ent
->limit
)
482 queue_work(cache
->wq
, &ent
->work
);
485 spin_unlock_irq(&ent
->lock
);
487 queue_work(cache
->wq
, &ent
->work
);
491 cache
->ent
[c
].miss
++;
496 static void free_cached_mr(struct mlx5_ib_dev
*dev
, struct mlx5_ib_mr
*mr
)
498 struct mlx5_mr_cache
*cache
= &dev
->cache
;
499 struct mlx5_cache_ent
*ent
;
503 c
= order2idx(dev
, mr
->order
);
504 if (c
< 0 || c
>= MAX_MR_CACHE_ENTRIES
) {
505 mlx5_ib_warn(dev
, "order %d, cache index %d\n", mr
->order
, c
);
508 ent
= &cache
->ent
[c
];
509 spin_lock_irq(&ent
->lock
);
510 list_add_tail(&mr
->list
, &ent
->head
);
512 if (ent
->cur
> 2 * ent
->limit
)
514 spin_unlock_irq(&ent
->lock
);
517 queue_work(cache
->wq
, &ent
->work
);
520 static void clean_keys(struct mlx5_ib_dev
*dev
, int c
)
522 struct mlx5_mr_cache
*cache
= &dev
->cache
;
523 struct mlx5_cache_ent
*ent
= &cache
->ent
[c
];
524 struct mlx5_ib_mr
*mr
;
527 cancel_delayed_work(&ent
->dwork
);
529 spin_lock_irq(&ent
->lock
);
530 if (list_empty(&ent
->head
)) {
531 spin_unlock_irq(&ent
->lock
);
534 mr
= list_first_entry(&ent
->head
, struct mlx5_ib_mr
, list
);
538 spin_unlock_irq(&ent
->lock
);
539 err
= destroy_mkey(dev
, mr
);
541 mlx5_ib_warn(dev
, "failed destroy mkey\n");
547 static int mlx5_mr_cache_debugfs_init(struct mlx5_ib_dev
*dev
)
549 struct mlx5_mr_cache
*cache
= &dev
->cache
;
550 struct mlx5_cache_ent
*ent
;
553 if (!mlx5_debugfs_root
)
556 cache
->root
= debugfs_create_dir("mr_cache", dev
->mdev
->priv
.dbg_root
);
560 for (i
= 0; i
< MAX_MR_CACHE_ENTRIES
; i
++) {
561 ent
= &cache
->ent
[i
];
562 sprintf(ent
->name
, "%d", ent
->order
);
563 ent
->dir
= debugfs_create_dir(ent
->name
, cache
->root
);
567 ent
->fsize
= debugfs_create_file("size", 0600, ent
->dir
, ent
,
572 ent
->flimit
= debugfs_create_file("limit", 0600, ent
->dir
, ent
,
577 ent
->fcur
= debugfs_create_u32("cur", 0400, ent
->dir
,
582 ent
->fmiss
= debugfs_create_u32("miss", 0600, ent
->dir
,
591 static void mlx5_mr_cache_debugfs_cleanup(struct mlx5_ib_dev
*dev
)
593 if (!mlx5_debugfs_root
)
596 debugfs_remove_recursive(dev
->cache
.root
);
599 static void delay_time_func(unsigned long ctx
)
601 struct mlx5_ib_dev
*dev
= (struct mlx5_ib_dev
*)ctx
;
606 int mlx5_mr_cache_init(struct mlx5_ib_dev
*dev
)
608 struct mlx5_mr_cache
*cache
= &dev
->cache
;
609 struct mlx5_cache_ent
*ent
;
614 cache
->wq
= create_singlethread_workqueue("mkey_cache");
616 mlx5_ib_warn(dev
, "failed to create work queue\n");
620 setup_timer(&dev
->delay_timer
, delay_time_func
, (unsigned long)dev
);
621 for (i
= 0; i
< MAX_MR_CACHE_ENTRIES
; i
++) {
622 INIT_LIST_HEAD(&cache
->ent
[i
].head
);
623 spin_lock_init(&cache
->ent
[i
].lock
);
625 ent
= &cache
->ent
[i
];
626 INIT_LIST_HEAD(&ent
->head
);
627 spin_lock_init(&ent
->lock
);
631 if (dev
->mdev
->profile
->mask
& MLX5_PROF_MASK_MR_CACHE
)
632 limit
= dev
->mdev
->profile
->mr_cache
[i
].limit
;
636 INIT_WORK(&ent
->work
, cache_work_func
);
637 INIT_DELAYED_WORK(&ent
->dwork
, delayed_cache_work_func
);
639 queue_work(cache
->wq
, &ent
->work
);
642 err
= mlx5_mr_cache_debugfs_init(dev
);
644 mlx5_ib_warn(dev
, "cache debugfs failure\n");
649 int mlx5_mr_cache_cleanup(struct mlx5_ib_dev
*dev
)
653 dev
->cache
.stopped
= 1;
654 flush_workqueue(dev
->cache
.wq
);
656 mlx5_mr_cache_debugfs_cleanup(dev
);
658 for (i
= 0; i
< MAX_MR_CACHE_ENTRIES
; i
++)
661 destroy_workqueue(dev
->cache
.wq
);
662 del_timer_sync(&dev
->delay_timer
);
667 struct ib_mr
*mlx5_ib_get_dma_mr(struct ib_pd
*pd
, int acc
)
669 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
670 int inlen
= MLX5_ST_SZ_BYTES(create_mkey_in
);
671 struct mlx5_core_dev
*mdev
= dev
->mdev
;
672 struct mlx5_ib_mr
*mr
;
677 mr
= kzalloc(sizeof(*mr
), GFP_KERNEL
);
679 return ERR_PTR(-ENOMEM
);
681 in
= kzalloc(inlen
, GFP_KERNEL
);
687 mkc
= MLX5_ADDR_OF(create_mkey_in
, in
, memory_key_mkey_entry
);
689 MLX5_SET(mkc
, mkc
, access_mode
, MLX5_MKC_ACCESS_MODE_PA
);
690 MLX5_SET(mkc
, mkc
, a
, !!(acc
& IB_ACCESS_REMOTE_ATOMIC
));
691 MLX5_SET(mkc
, mkc
, rw
, !!(acc
& IB_ACCESS_REMOTE_WRITE
));
692 MLX5_SET(mkc
, mkc
, rr
, !!(acc
& IB_ACCESS_REMOTE_READ
));
693 MLX5_SET(mkc
, mkc
, lw
, !!(acc
& IB_ACCESS_LOCAL_WRITE
));
694 MLX5_SET(mkc
, mkc
, lr
, 1);
696 MLX5_SET(mkc
, mkc
, length64
, 1);
697 MLX5_SET(mkc
, mkc
, pd
, to_mpd(pd
)->pdn
);
698 MLX5_SET(mkc
, mkc
, qpn
, 0xffffff);
699 MLX5_SET64(mkc
, mkc
, start_addr
, 0);
701 err
= mlx5_core_create_mkey(mdev
, &mr
->mmkey
, in
, inlen
);
706 mr
->ibmr
.lkey
= mr
->mmkey
.key
;
707 mr
->ibmr
.rkey
= mr
->mmkey
.key
;
721 static int get_octo_len(u64 addr
, u64 len
, int page_size
)
726 offset
= addr
& (page_size
- 1);
727 npages
= ALIGN(len
+ offset
, page_size
) >> ilog2(page_size
);
728 return (npages
+ 1) / 2;
731 static int use_umr(int order
)
733 return order
<= MLX5_MAX_UMR_SHIFT
;
736 static int dma_map_mr_pas(struct mlx5_ib_dev
*dev
, struct ib_umem
*umem
,
737 int npages
, int page_shift
, int *size
,
738 __be64
**mr_pas
, dma_addr_t
*dma
)
741 struct device
*ddev
= dev
->ib_dev
.dma_device
;
744 * UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes.
745 * To avoid copying garbage after the pas array, we allocate
748 *size
= ALIGN(sizeof(u64
) * npages
, MLX5_UMR_MTT_ALIGNMENT
);
749 *mr_pas
= kmalloc(*size
+ MLX5_UMR_ALIGN
- 1, GFP_KERNEL
);
753 pas
= PTR_ALIGN(*mr_pas
, MLX5_UMR_ALIGN
);
754 mlx5_ib_populate_pas(dev
, umem
, page_shift
, pas
, MLX5_IB_MTT_PRESENT
);
755 /* Clear padding after the actual pages. */
756 memset(pas
+ npages
, 0, *size
- npages
* sizeof(u64
));
758 *dma
= dma_map_single(ddev
, pas
, *size
, DMA_TO_DEVICE
);
759 if (dma_mapping_error(ddev
, *dma
)) {
767 static void prep_umr_wqe_common(struct ib_pd
*pd
, struct ib_send_wr
*wr
,
768 struct ib_sge
*sg
, u64 dma
, int n
, u32 key
,
771 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
772 struct mlx5_umr_wr
*umrwr
= umr_wr(wr
);
775 sg
->length
= ALIGN(sizeof(u64
) * n
, 64);
776 sg
->lkey
= dev
->umrc
.pd
->local_dma_lkey
;
785 wr
->opcode
= MLX5_IB_WR_UMR
;
788 umrwr
->page_shift
= page_shift
;
792 static void prep_umr_reg_wqe(struct ib_pd
*pd
, struct ib_send_wr
*wr
,
793 struct ib_sge
*sg
, u64 dma
, int n
, u32 key
,
794 int page_shift
, u64 virt_addr
, u64 len
,
797 struct mlx5_umr_wr
*umrwr
= umr_wr(wr
);
799 prep_umr_wqe_common(pd
, wr
, sg
, dma
, n
, key
, page_shift
);
803 umrwr
->target
.virt_addr
= virt_addr
;
805 umrwr
->access_flags
= access_flags
;
809 static void prep_umr_unreg_wqe(struct mlx5_ib_dev
*dev
,
810 struct ib_send_wr
*wr
, u32 key
)
812 struct mlx5_umr_wr
*umrwr
= umr_wr(wr
);
814 wr
->send_flags
= MLX5_IB_SEND_UMR_UNREG
| MLX5_IB_SEND_UMR_FAIL_IF_FREE
;
815 wr
->opcode
= MLX5_IB_WR_UMR
;
819 static struct ib_umem
*mr_umem_get(struct ib_pd
*pd
, u64 start
, u64 length
,
820 int access_flags
, int *npages
,
821 int *page_shift
, int *ncont
, int *order
)
823 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
824 struct ib_umem
*umem
= ib_umem_get(pd
->uobject
->context
, start
, length
,
827 mlx5_ib_err(dev
, "umem get failed (%ld)\n", PTR_ERR(umem
));
831 mlx5_ib_cont_pages(umem
, start
, npages
, page_shift
, ncont
, order
);
833 mlx5_ib_warn(dev
, "avoid zero region\n");
834 ib_umem_release(umem
);
835 return ERR_PTR(-EINVAL
);
838 mlx5_ib_dbg(dev
, "npages %d, ncont %d, order %d, page_shift %d\n",
839 *npages
, *ncont
, *order
, *page_shift
);
844 static void mlx5_ib_umr_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
846 struct mlx5_ib_umr_context
*context
=
847 container_of(wc
->wr_cqe
, struct mlx5_ib_umr_context
, cqe
);
849 context
->status
= wc
->status
;
850 complete(&context
->done
);
853 static inline void mlx5_ib_init_umr_context(struct mlx5_ib_umr_context
*context
)
855 context
->cqe
.done
= mlx5_ib_umr_done
;
856 context
->status
= -1;
857 init_completion(&context
->done
);
860 static struct mlx5_ib_mr
*reg_umr(struct ib_pd
*pd
, struct ib_umem
*umem
,
861 u64 virt_addr
, u64 len
, int npages
,
862 int page_shift
, int order
, int access_flags
)
864 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
865 struct device
*ddev
= dev
->ib_dev
.dma_device
;
866 struct umr_common
*umrc
= &dev
->umrc
;
867 struct mlx5_ib_umr_context umr_context
;
868 struct mlx5_umr_wr umrwr
= {};
869 struct ib_send_wr
*bad
;
870 struct mlx5_ib_mr
*mr
;
878 for (i
= 0; i
< 1; i
++) {
879 mr
= alloc_cached_mr(dev
, order
);
883 err
= add_keys(dev
, order2idx(dev
, order
), 1);
884 if (err
&& err
!= -EAGAIN
) {
885 mlx5_ib_warn(dev
, "add_keys failed, err %d\n", err
);
891 return ERR_PTR(-EAGAIN
);
893 err
= dma_map_mr_pas(dev
, umem
, npages
, page_shift
, &size
, &mr_pas
,
898 mlx5_ib_init_umr_context(&umr_context
);
900 umrwr
.wr
.wr_cqe
= &umr_context
.cqe
;
901 prep_umr_reg_wqe(pd
, &umrwr
.wr
, &sg
, dma
, npages
, mr
->mmkey
.key
,
902 page_shift
, virt_addr
, len
, access_flags
);
905 err
= ib_post_send(umrc
->qp
, &umrwr
.wr
, &bad
);
907 mlx5_ib_warn(dev
, "post send failed, err %d\n", err
);
910 wait_for_completion(&umr_context
.done
);
911 if (umr_context
.status
!= IB_WC_SUCCESS
) {
912 mlx5_ib_warn(dev
, "reg umr failed\n");
917 mr
->mmkey
.iova
= virt_addr
;
918 mr
->mmkey
.size
= len
;
919 mr
->mmkey
.pd
= to_mpd(pd
)->pdn
;
925 dma_unmap_single(ddev
, dma
, size
, DMA_TO_DEVICE
);
931 free_cached_mr(dev
, mr
);
938 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
939 int mlx5_ib_update_mtt(struct mlx5_ib_mr
*mr
, u64 start_page_index
, int npages
,
942 struct mlx5_ib_dev
*dev
= mr
->dev
;
943 struct device
*ddev
= dev
->ib_dev
.dma_device
;
944 struct umr_common
*umrc
= &dev
->umrc
;
945 struct mlx5_ib_umr_context umr_context
;
946 struct ib_umem
*umem
= mr
->umem
;
950 struct ib_send_wr
*bad
;
951 struct mlx5_umr_wr wr
;
954 const int page_index_alignment
= MLX5_UMR_MTT_ALIGNMENT
/ sizeof(u64
);
955 const int page_index_mask
= page_index_alignment
- 1;
956 size_t pages_mapped
= 0;
957 size_t pages_to_map
= 0;
958 size_t pages_iter
= 0;
959 int use_emergency_buf
= 0;
961 /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes,
962 * so we need to align the offset and length accordingly */
963 if (start_page_index
& page_index_mask
) {
964 npages
+= start_page_index
& page_index_mask
;
965 start_page_index
&= ~page_index_mask
;
968 pages_to_map
= ALIGN(npages
, page_index_alignment
);
970 if (start_page_index
+ pages_to_map
> MLX5_MAX_UMR_PAGES
)
973 size
= sizeof(u64
) * pages_to_map
;
974 size
= min_t(int, PAGE_SIZE
, size
);
975 /* We allocate with GFP_ATOMIC to avoid recursion into page-reclaim
976 * code, when we are called from an invalidation. The pas buffer must
977 * be 2k-aligned for Connect-IB. */
978 pas
= (__be64
*)get_zeroed_page(GFP_ATOMIC
);
980 mlx5_ib_warn(dev
, "unable to allocate memory during MTT update, falling back to slower chunked mechanism.\n");
981 pas
= mlx5_ib_update_mtt_emergency_buffer
;
982 size
= MLX5_UMR_MTT_MIN_CHUNK_SIZE
;
983 use_emergency_buf
= 1;
984 mutex_lock(&mlx5_ib_update_mtt_emergency_buffer_mutex
);
985 memset(pas
, 0, size
);
987 pages_iter
= size
/ sizeof(u64
);
988 dma
= dma_map_single(ddev
, pas
, size
, DMA_TO_DEVICE
);
989 if (dma_mapping_error(ddev
, dma
)) {
990 mlx5_ib_err(dev
, "unable to map DMA during MTT update.\n");
995 for (pages_mapped
= 0;
996 pages_mapped
< pages_to_map
&& !err
;
997 pages_mapped
+= pages_iter
, start_page_index
+= pages_iter
) {
998 dma_sync_single_for_cpu(ddev
, dma
, size
, DMA_TO_DEVICE
);
1000 npages
= min_t(size_t,
1002 ib_umem_num_pages(umem
) - start_page_index
);
1005 __mlx5_ib_populate_pas(dev
, umem
, PAGE_SHIFT
,
1006 start_page_index
, npages
, pas
,
1007 MLX5_IB_MTT_PRESENT
);
1008 /* Clear padding after the pages brought from the
1010 memset(pas
+ npages
, 0, size
- npages
* sizeof(u64
));
1013 dma_sync_single_for_device(ddev
, dma
, size
, DMA_TO_DEVICE
);
1015 mlx5_ib_init_umr_context(&umr_context
);
1017 memset(&wr
, 0, sizeof(wr
));
1018 wr
.wr
.wr_cqe
= &umr_context
.cqe
;
1021 sg
.length
= ALIGN(npages
* sizeof(u64
),
1022 MLX5_UMR_MTT_ALIGNMENT
);
1023 sg
.lkey
= dev
->umrc
.pd
->local_dma_lkey
;
1025 wr
.wr
.send_flags
= MLX5_IB_SEND_UMR_FAIL_IF_FREE
|
1026 MLX5_IB_SEND_UMR_UPDATE_MTT
;
1027 wr
.wr
.sg_list
= &sg
;
1029 wr
.wr
.opcode
= MLX5_IB_WR_UMR
;
1030 wr
.npages
= sg
.length
/ sizeof(u64
);
1031 wr
.page_shift
= PAGE_SHIFT
;
1032 wr
.mkey
= mr
->mmkey
.key
;
1033 wr
.target
.offset
= start_page_index
;
1036 err
= ib_post_send(umrc
->qp
, &wr
.wr
, &bad
);
1038 mlx5_ib_err(dev
, "UMR post send failed, err %d\n", err
);
1040 wait_for_completion(&umr_context
.done
);
1041 if (umr_context
.status
!= IB_WC_SUCCESS
) {
1042 mlx5_ib_err(dev
, "UMR completion failed, code %d\n",
1043 umr_context
.status
);
1049 dma_unmap_single(ddev
, dma
, size
, DMA_TO_DEVICE
);
1052 if (!use_emergency_buf
)
1053 free_page((unsigned long)pas
);
1055 mutex_unlock(&mlx5_ib_update_mtt_emergency_buffer_mutex
);
1062 * If ibmr is NULL it will be allocated by reg_create.
1063 * Else, the given ibmr will be used.
1065 static struct mlx5_ib_mr
*reg_create(struct ib_mr
*ibmr
, struct ib_pd
*pd
,
1066 u64 virt_addr
, u64 length
,
1067 struct ib_umem
*umem
, int npages
,
1068 int page_shift
, int access_flags
)
1070 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
1071 struct mlx5_ib_mr
*mr
;
1077 bool pg_cap
= !!(MLX5_CAP_GEN(dev
->mdev
, pg
));
1079 mr
= ibmr
? to_mmr(ibmr
) : kzalloc(sizeof(*mr
), GFP_KERNEL
);
1081 return ERR_PTR(-ENOMEM
);
1083 inlen
= MLX5_ST_SZ_BYTES(create_mkey_in
) +
1084 sizeof(*pas
) * ((npages
+ 1) / 2) * 2;
1085 in
= mlx5_vzalloc(inlen
);
1090 pas
= (__be64
*)MLX5_ADDR_OF(create_mkey_in
, in
, klm_pas_mtt
);
1091 mlx5_ib_populate_pas(dev
, umem
, page_shift
, pas
,
1092 pg_cap
? MLX5_IB_MTT_PRESENT
: 0);
1094 /* The pg_access bit allows setting the access flags
1095 * in the page list submitted with the command. */
1096 MLX5_SET(create_mkey_in
, in
, pg_access
, !!(pg_cap
));
1098 mkc
= MLX5_ADDR_OF(create_mkey_in
, in
, memory_key_mkey_entry
);
1099 MLX5_SET(mkc
, mkc
, access_mode
, MLX5_MKC_ACCESS_MODE_MTT
);
1100 MLX5_SET(mkc
, mkc
, a
, !!(access_flags
& IB_ACCESS_REMOTE_ATOMIC
));
1101 MLX5_SET(mkc
, mkc
, rw
, !!(access_flags
& IB_ACCESS_REMOTE_WRITE
));
1102 MLX5_SET(mkc
, mkc
, rr
, !!(access_flags
& IB_ACCESS_REMOTE_READ
));
1103 MLX5_SET(mkc
, mkc
, lw
, !!(access_flags
& IB_ACCESS_LOCAL_WRITE
));
1104 MLX5_SET(mkc
, mkc
, lr
, 1);
1106 MLX5_SET64(mkc
, mkc
, start_addr
, virt_addr
);
1107 MLX5_SET64(mkc
, mkc
, len
, length
);
1108 MLX5_SET(mkc
, mkc
, pd
, to_mpd(pd
)->pdn
);
1109 MLX5_SET(mkc
, mkc
, bsf_octword_size
, 0);
1110 MLX5_SET(mkc
, mkc
, translations_octword_size
,
1111 get_octo_len(virt_addr
, length
, 1 << page_shift
));
1112 MLX5_SET(mkc
, mkc
, log_page_size
, page_shift
);
1113 MLX5_SET(mkc
, mkc
, qpn
, 0xffffff);
1114 MLX5_SET(create_mkey_in
, in
, translations_octword_actual_size
,
1115 get_octo_len(virt_addr
, length
, 1 << page_shift
));
1117 err
= mlx5_core_create_mkey(dev
->mdev
, &mr
->mmkey
, in
, inlen
);
1119 mlx5_ib_warn(dev
, "create mkey failed\n");
1127 mlx5_ib_dbg(dev
, "mkey = 0x%x\n", mr
->mmkey
.key
);
1138 return ERR_PTR(err
);
1141 static void set_mr_fileds(struct mlx5_ib_dev
*dev
, struct mlx5_ib_mr
*mr
,
1142 int npages
, u64 length
, int access_flags
)
1144 mr
->npages
= npages
;
1145 atomic_add(npages
, &dev
->mdev
->priv
.reg_pages
);
1146 mr
->ibmr
.lkey
= mr
->mmkey
.key
;
1147 mr
->ibmr
.rkey
= mr
->mmkey
.key
;
1148 mr
->ibmr
.length
= length
;
1149 mr
->access_flags
= access_flags
;
1152 struct ib_mr
*mlx5_ib_reg_user_mr(struct ib_pd
*pd
, u64 start
, u64 length
,
1153 u64 virt_addr
, int access_flags
,
1154 struct ib_udata
*udata
)
1156 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
1157 struct mlx5_ib_mr
*mr
= NULL
;
1158 struct ib_umem
*umem
;
1165 mlx5_ib_dbg(dev
, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
1166 start
, virt_addr
, length
, access_flags
);
1167 umem
= mr_umem_get(pd
, start
, length
, access_flags
, &npages
,
1168 &page_shift
, &ncont
, &order
);
1171 return (void *)umem
;
1173 if (use_umr(order
)) {
1174 mr
= reg_umr(pd
, umem
, virt_addr
, length
, ncont
, page_shift
,
1175 order
, access_flags
);
1176 if (PTR_ERR(mr
) == -EAGAIN
) {
1177 mlx5_ib_dbg(dev
, "cache empty for order %d", order
);
1180 } else if (access_flags
& IB_ACCESS_ON_DEMAND
) {
1182 pr_err("Got MR registration for ODP MR > 512MB, not supported for Connect-IB");
1187 mr
= reg_create(NULL
, pd
, virt_addr
, length
, umem
, ncont
,
1188 page_shift
, access_flags
);
1195 mlx5_ib_dbg(dev
, "mkey 0x%x\n", mr
->mmkey
.key
);
1198 set_mr_fileds(dev
, mr
, npages
, length
, access_flags
);
1200 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1207 ib_umem_release(umem
);
1208 return ERR_PTR(err
);
1211 static int unreg_umr(struct mlx5_ib_dev
*dev
, struct mlx5_ib_mr
*mr
)
1213 struct mlx5_core_dev
*mdev
= dev
->mdev
;
1214 struct umr_common
*umrc
= &dev
->umrc
;
1215 struct mlx5_ib_umr_context umr_context
;
1216 struct mlx5_umr_wr umrwr
= {};
1217 struct ib_send_wr
*bad
;
1220 if (mdev
->state
== MLX5_DEVICE_STATE_INTERNAL_ERROR
)
1223 mlx5_ib_init_umr_context(&umr_context
);
1225 umrwr
.wr
.wr_cqe
= &umr_context
.cqe
;
1226 prep_umr_unreg_wqe(dev
, &umrwr
.wr
, mr
->mmkey
.key
);
1229 err
= ib_post_send(umrc
->qp
, &umrwr
.wr
, &bad
);
1232 mlx5_ib_dbg(dev
, "err %d\n", err
);
1235 wait_for_completion(&umr_context
.done
);
1238 if (umr_context
.status
!= IB_WC_SUCCESS
) {
1239 mlx5_ib_warn(dev
, "unreg umr failed\n");
1249 static int rereg_umr(struct ib_pd
*pd
, struct mlx5_ib_mr
*mr
, u64 virt_addr
,
1250 u64 length
, int npages
, int page_shift
, int order
,
1251 int access_flags
, int flags
)
1253 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
1254 struct device
*ddev
= dev
->ib_dev
.dma_device
;
1255 struct mlx5_ib_umr_context umr_context
;
1256 struct ib_send_wr
*bad
;
1257 struct mlx5_umr_wr umrwr
= {};
1259 struct umr_common
*umrc
= &dev
->umrc
;
1261 __be64
*mr_pas
= NULL
;
1265 mlx5_ib_init_umr_context(&umr_context
);
1267 umrwr
.wr
.wr_cqe
= &umr_context
.cqe
;
1268 umrwr
.wr
.send_flags
= MLX5_IB_SEND_UMR_FAIL_IF_FREE
;
1270 if (flags
& IB_MR_REREG_TRANS
) {
1271 err
= dma_map_mr_pas(dev
, mr
->umem
, npages
, page_shift
, &size
,
1276 umrwr
.target
.virt_addr
= virt_addr
;
1277 umrwr
.length
= length
;
1278 umrwr
.wr
.send_flags
|= MLX5_IB_SEND_UMR_UPDATE_TRANSLATION
;
1281 prep_umr_wqe_common(pd
, &umrwr
.wr
, &sg
, dma
, npages
, mr
->mmkey
.key
,
1284 if (flags
& IB_MR_REREG_PD
) {
1286 umrwr
.wr
.send_flags
|= MLX5_IB_SEND_UMR_UPDATE_PD
;
1289 if (flags
& IB_MR_REREG_ACCESS
) {
1290 umrwr
.access_flags
= access_flags
;
1291 umrwr
.wr
.send_flags
|= MLX5_IB_SEND_UMR_UPDATE_ACCESS
;
1294 /* post send request to UMR QP */
1296 err
= ib_post_send(umrc
->qp
, &umrwr
.wr
, &bad
);
1299 mlx5_ib_warn(dev
, "post send failed, err %d\n", err
);
1301 wait_for_completion(&umr_context
.done
);
1302 if (umr_context
.status
!= IB_WC_SUCCESS
) {
1303 mlx5_ib_warn(dev
, "reg umr failed (%u)\n",
1304 umr_context
.status
);
1310 if (flags
& IB_MR_REREG_TRANS
) {
1311 dma_unmap_single(ddev
, dma
, size
, DMA_TO_DEVICE
);
1317 int mlx5_ib_rereg_user_mr(struct ib_mr
*ib_mr
, int flags
, u64 start
,
1318 u64 length
, u64 virt_addr
, int new_access_flags
,
1319 struct ib_pd
*new_pd
, struct ib_udata
*udata
)
1321 struct mlx5_ib_dev
*dev
= to_mdev(ib_mr
->device
);
1322 struct mlx5_ib_mr
*mr
= to_mmr(ib_mr
);
1323 struct ib_pd
*pd
= (flags
& IB_MR_REREG_PD
) ? new_pd
: ib_mr
->pd
;
1324 int access_flags
= flags
& IB_MR_REREG_ACCESS
?
1327 u64 addr
= (flags
& IB_MR_REREG_TRANS
) ? virt_addr
: mr
->umem
->address
;
1328 u64 len
= (flags
& IB_MR_REREG_TRANS
) ? length
: mr
->umem
->length
;
1335 mlx5_ib_dbg(dev
, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
1336 start
, virt_addr
, length
, access_flags
);
1338 if (flags
!= IB_MR_REREG_PD
) {
1340 * Replace umem. This needs to be done whether or not UMR is
1343 flags
|= IB_MR_REREG_TRANS
;
1344 ib_umem_release(mr
->umem
);
1345 mr
->umem
= mr_umem_get(pd
, addr
, len
, access_flags
, &npages
,
1346 &page_shift
, &ncont
, &order
);
1347 if (IS_ERR(mr
->umem
)) {
1348 err
= PTR_ERR(mr
->umem
);
1354 if (flags
& IB_MR_REREG_TRANS
&& !use_umr_mtt_update(mr
, addr
, len
)) {
1356 * UMR can't be used - MKey needs to be replaced.
1359 err
= unreg_umr(dev
, mr
);
1361 mlx5_ib_warn(dev
, "Failed to unregister MR\n");
1363 err
= destroy_mkey(dev
, mr
);
1365 mlx5_ib_warn(dev
, "Failed to destroy MKey\n");
1370 mr
= reg_create(ib_mr
, pd
, addr
, len
, mr
->umem
, ncont
,
1371 page_shift
, access_flags
);
1381 err
= rereg_umr(pd
, mr
, addr
, len
, npages
, page_shift
,
1382 order
, access_flags
, flags
);
1384 mlx5_ib_warn(dev
, "Failed to rereg UMR\n");
1389 if (flags
& IB_MR_REREG_PD
) {
1391 mr
->mmkey
.pd
= to_mpd(pd
)->pdn
;
1394 if (flags
& IB_MR_REREG_ACCESS
)
1395 mr
->access_flags
= access_flags
;
1397 if (flags
& IB_MR_REREG_TRANS
) {
1398 atomic_sub(mr
->npages
, &dev
->mdev
->priv
.reg_pages
);
1399 set_mr_fileds(dev
, mr
, npages
, len
, access_flags
);
1400 mr
->mmkey
.iova
= addr
;
1401 mr
->mmkey
.size
= len
;
1403 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1411 mlx5_alloc_priv_descs(struct ib_device
*device
,
1412 struct mlx5_ib_mr
*mr
,
1416 int size
= ndescs
* desc_size
;
1420 add_size
= max_t(int, MLX5_UMR_ALIGN
- ARCH_KMALLOC_MINALIGN
, 0);
1422 mr
->descs_alloc
= kzalloc(size
+ add_size
, GFP_KERNEL
);
1423 if (!mr
->descs_alloc
)
1426 mr
->descs
= PTR_ALIGN(mr
->descs_alloc
, MLX5_UMR_ALIGN
);
1428 mr
->desc_map
= dma_map_single(device
->dma_device
, mr
->descs
,
1429 size
, DMA_TO_DEVICE
);
1430 if (dma_mapping_error(device
->dma_device
, mr
->desc_map
)) {
1437 kfree(mr
->descs_alloc
);
1443 mlx5_free_priv_descs(struct mlx5_ib_mr
*mr
)
1446 struct ib_device
*device
= mr
->ibmr
.device
;
1447 int size
= mr
->max_descs
* mr
->desc_size
;
1449 dma_unmap_single(device
->dma_device
, mr
->desc_map
,
1450 size
, DMA_TO_DEVICE
);
1451 kfree(mr
->descs_alloc
);
1456 static int clean_mr(struct mlx5_ib_mr
*mr
)
1458 struct mlx5_ib_dev
*dev
= to_mdev(mr
->ibmr
.device
);
1459 int umred
= mr
->umred
;
1463 if (mlx5_core_destroy_psv(dev
->mdev
,
1464 mr
->sig
->psv_memory
.psv_idx
))
1465 mlx5_ib_warn(dev
, "failed to destroy mem psv %d\n",
1466 mr
->sig
->psv_memory
.psv_idx
);
1467 if (mlx5_core_destroy_psv(dev
->mdev
,
1468 mr
->sig
->psv_wire
.psv_idx
))
1469 mlx5_ib_warn(dev
, "failed to destroy wire psv %d\n",
1470 mr
->sig
->psv_wire
.psv_idx
);
1475 mlx5_free_priv_descs(mr
);
1478 err
= destroy_mkey(dev
, mr
);
1480 mlx5_ib_warn(dev
, "failed to destroy mkey 0x%x (%d)\n",
1481 mr
->mmkey
.key
, err
);
1485 err
= unreg_umr(dev
, mr
);
1487 mlx5_ib_warn(dev
, "failed unregister\n");
1490 free_cached_mr(dev
, mr
);
1499 int mlx5_ib_dereg_mr(struct ib_mr
*ibmr
)
1501 struct mlx5_ib_dev
*dev
= to_mdev(ibmr
->device
);
1502 struct mlx5_ib_mr
*mr
= to_mmr(ibmr
);
1503 int npages
= mr
->npages
;
1504 struct ib_umem
*umem
= mr
->umem
;
1506 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1507 if (umem
&& umem
->odp_data
) {
1508 /* Prevent new page faults from succeeding */
1510 /* Wait for all running page-fault handlers to finish. */
1511 synchronize_srcu(&dev
->mr_srcu
);
1512 /* Destroy all page mappings */
1513 mlx5_ib_invalidate_range(umem
, ib_umem_start(umem
),
1516 * We kill the umem before the MR for ODP,
1517 * so that there will not be any invalidations in
1518 * flight, looking at the *mr struct.
1520 ib_umem_release(umem
);
1521 atomic_sub(npages
, &dev
->mdev
->priv
.reg_pages
);
1523 /* Avoid double-freeing the umem. */
1531 ib_umem_release(umem
);
1532 atomic_sub(npages
, &dev
->mdev
->priv
.reg_pages
);
1538 struct ib_mr
*mlx5_ib_alloc_mr(struct ib_pd
*pd
,
1539 enum ib_mr_type mr_type
,
1542 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
1543 int inlen
= MLX5_ST_SZ_BYTES(create_mkey_in
);
1544 int ndescs
= ALIGN(max_num_sg
, 4);
1545 struct mlx5_ib_mr
*mr
;
1550 mr
= kzalloc(sizeof(*mr
), GFP_KERNEL
);
1552 return ERR_PTR(-ENOMEM
);
1554 in
= kzalloc(inlen
, GFP_KERNEL
);
1560 mkc
= MLX5_ADDR_OF(create_mkey_in
, in
, memory_key_mkey_entry
);
1561 MLX5_SET(mkc
, mkc
, free
, 1);
1562 MLX5_SET(mkc
, mkc
, translations_octword_size
, ndescs
);
1563 MLX5_SET(mkc
, mkc
, qpn
, 0xffffff);
1564 MLX5_SET(mkc
, mkc
, pd
, to_mpd(pd
)->pdn
);
1566 if (mr_type
== IB_MR_TYPE_MEM_REG
) {
1567 mr
->access_mode
= MLX5_MKC_ACCESS_MODE_MTT
;
1568 MLX5_SET(mkc
, mkc
, log_page_size
, PAGE_SHIFT
);
1569 err
= mlx5_alloc_priv_descs(pd
->device
, mr
,
1570 ndescs
, sizeof(u64
));
1574 mr
->desc_size
= sizeof(u64
);
1575 mr
->max_descs
= ndescs
;
1576 } else if (mr_type
== IB_MR_TYPE_SG_GAPS
) {
1577 mr
->access_mode
= MLX5_MKC_ACCESS_MODE_KLMS
;
1579 err
= mlx5_alloc_priv_descs(pd
->device
, mr
,
1580 ndescs
, sizeof(struct mlx5_klm
));
1583 mr
->desc_size
= sizeof(struct mlx5_klm
);
1584 mr
->max_descs
= ndescs
;
1585 } else if (mr_type
== IB_MR_TYPE_SIGNATURE
) {
1588 MLX5_SET(mkc
, mkc
, bsf_en
, 1);
1589 MLX5_SET(mkc
, mkc
, bsf_octword_size
, MLX5_MKEY_BSF_OCTO_SIZE
);
1590 mr
->sig
= kzalloc(sizeof(*mr
->sig
), GFP_KERNEL
);
1596 /* create mem & wire PSVs */
1597 err
= mlx5_core_create_psv(dev
->mdev
, to_mpd(pd
)->pdn
,
1602 mr
->access_mode
= MLX5_MKC_ACCESS_MODE_KLMS
;
1603 mr
->sig
->psv_memory
.psv_idx
= psv_index
[0];
1604 mr
->sig
->psv_wire
.psv_idx
= psv_index
[1];
1606 mr
->sig
->sig_status_checked
= true;
1607 mr
->sig
->sig_err_exists
= false;
1608 /* Next UMR, Arm SIGERR */
1609 ++mr
->sig
->sigerr_count
;
1611 mlx5_ib_warn(dev
, "Invalid mr type %d\n", mr_type
);
1616 MLX5_SET(mkc
, mkc
, access_mode
, mr
->access_mode
);
1617 MLX5_SET(mkc
, mkc
, umr_en
, 1);
1619 err
= mlx5_core_create_mkey(dev
->mdev
, &mr
->mmkey
, in
, inlen
);
1621 goto err_destroy_psv
;
1623 mr
->ibmr
.lkey
= mr
->mmkey
.key
;
1624 mr
->ibmr
.rkey
= mr
->mmkey
.key
;
1632 if (mlx5_core_destroy_psv(dev
->mdev
,
1633 mr
->sig
->psv_memory
.psv_idx
))
1634 mlx5_ib_warn(dev
, "failed to destroy mem psv %d\n",
1635 mr
->sig
->psv_memory
.psv_idx
);
1636 if (mlx5_core_destroy_psv(dev
->mdev
,
1637 mr
->sig
->psv_wire
.psv_idx
))
1638 mlx5_ib_warn(dev
, "failed to destroy wire psv %d\n",
1639 mr
->sig
->psv_wire
.psv_idx
);
1641 mlx5_free_priv_descs(mr
);
1648 return ERR_PTR(err
);
1651 struct ib_mw
*mlx5_ib_alloc_mw(struct ib_pd
*pd
, enum ib_mw_type type
,
1652 struct ib_udata
*udata
)
1654 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
1655 int inlen
= MLX5_ST_SZ_BYTES(create_mkey_in
);
1656 struct mlx5_ib_mw
*mw
= NULL
;
1661 struct mlx5_ib_alloc_mw req
= {};
1664 __u32 response_length
;
1667 err
= ib_copy_from_udata(&req
, udata
, min(udata
->inlen
, sizeof(req
)));
1669 return ERR_PTR(err
);
1671 if (req
.comp_mask
|| req
.reserved1
|| req
.reserved2
)
1672 return ERR_PTR(-EOPNOTSUPP
);
1674 if (udata
->inlen
> sizeof(req
) &&
1675 !ib_is_udata_cleared(udata
, sizeof(req
),
1676 udata
->inlen
- sizeof(req
)))
1677 return ERR_PTR(-EOPNOTSUPP
);
1679 ndescs
= req
.num_klms
? roundup(req
.num_klms
, 4) : roundup(1, 4);
1681 mw
= kzalloc(sizeof(*mw
), GFP_KERNEL
);
1682 in
= kzalloc(inlen
, GFP_KERNEL
);
1688 mkc
= MLX5_ADDR_OF(create_mkey_in
, in
, memory_key_mkey_entry
);
1690 MLX5_SET(mkc
, mkc
, free
, 1);
1691 MLX5_SET(mkc
, mkc
, translations_octword_size
, ndescs
);
1692 MLX5_SET(mkc
, mkc
, pd
, to_mpd(pd
)->pdn
);
1693 MLX5_SET(mkc
, mkc
, umr_en
, 1);
1694 MLX5_SET(mkc
, mkc
, lr
, 1);
1695 MLX5_SET(mkc
, mkc
, access_mode
, MLX5_MKC_ACCESS_MODE_KLMS
);
1696 MLX5_SET(mkc
, mkc
, en_rinval
, !!((type
== IB_MW_TYPE_2
)));
1697 MLX5_SET(mkc
, mkc
, qpn
, 0xffffff);
1699 err
= mlx5_core_create_mkey(dev
->mdev
, &mw
->mmkey
, in
, inlen
);
1703 mw
->ibmw
.rkey
= mw
->mmkey
.key
;
1705 resp
.response_length
= min(offsetof(typeof(resp
), response_length
) +
1706 sizeof(resp
.response_length
), udata
->outlen
);
1707 if (resp
.response_length
) {
1708 err
= ib_copy_to_udata(udata
, &resp
, resp
.response_length
);
1710 mlx5_core_destroy_mkey(dev
->mdev
, &mw
->mmkey
);
1721 return ERR_PTR(err
);
1724 int mlx5_ib_dealloc_mw(struct ib_mw
*mw
)
1726 struct mlx5_ib_mw
*mmw
= to_mmw(mw
);
1729 err
= mlx5_core_destroy_mkey((to_mdev(mw
->device
))->mdev
,
1736 int mlx5_ib_check_mr_status(struct ib_mr
*ibmr
, u32 check_mask
,
1737 struct ib_mr_status
*mr_status
)
1739 struct mlx5_ib_mr
*mmr
= to_mmr(ibmr
);
1742 if (check_mask
& ~IB_MR_CHECK_SIG_STATUS
) {
1743 pr_err("Invalid status check mask\n");
1748 mr_status
->fail_status
= 0;
1749 if (check_mask
& IB_MR_CHECK_SIG_STATUS
) {
1752 pr_err("signature status check requested on a non-signature enabled MR\n");
1756 mmr
->sig
->sig_status_checked
= true;
1757 if (!mmr
->sig
->sig_err_exists
)
1760 if (ibmr
->lkey
== mmr
->sig
->err_item
.key
)
1761 memcpy(&mr_status
->sig_err
, &mmr
->sig
->err_item
,
1762 sizeof(mr_status
->sig_err
));
1764 mr_status
->sig_err
.err_type
= IB_SIG_BAD_GUARD
;
1765 mr_status
->sig_err
.sig_err_offset
= 0;
1766 mr_status
->sig_err
.key
= mmr
->sig
->err_item
.key
;
1769 mmr
->sig
->sig_err_exists
= false;
1770 mr_status
->fail_status
|= IB_MR_CHECK_SIG_STATUS
;
1778 mlx5_ib_sg_to_klms(struct mlx5_ib_mr
*mr
,
1779 struct scatterlist
*sgl
,
1780 unsigned short sg_nents
,
1781 unsigned int *sg_offset_p
)
1783 struct scatterlist
*sg
= sgl
;
1784 struct mlx5_klm
*klms
= mr
->descs
;
1785 unsigned int sg_offset
= sg_offset_p
? *sg_offset_p
: 0;
1786 u32 lkey
= mr
->ibmr
.pd
->local_dma_lkey
;
1789 mr
->ibmr
.iova
= sg_dma_address(sg
) + sg_offset
;
1790 mr
->ibmr
.length
= 0;
1791 mr
->ndescs
= sg_nents
;
1793 for_each_sg(sgl
, sg
, sg_nents
, i
) {
1794 if (unlikely(i
> mr
->max_descs
))
1796 klms
[i
].va
= cpu_to_be64(sg_dma_address(sg
) + sg_offset
);
1797 klms
[i
].bcount
= cpu_to_be32(sg_dma_len(sg
) - sg_offset
);
1798 klms
[i
].key
= cpu_to_be32(lkey
);
1799 mr
->ibmr
.length
+= sg_dma_len(sg
);
1805 *sg_offset_p
= sg_offset
;
1810 static int mlx5_set_page(struct ib_mr
*ibmr
, u64 addr
)
1812 struct mlx5_ib_mr
*mr
= to_mmr(ibmr
);
1815 if (unlikely(mr
->ndescs
== mr
->max_descs
))
1819 descs
[mr
->ndescs
++] = cpu_to_be64(addr
| MLX5_EN_RD
| MLX5_EN_WR
);
1824 int mlx5_ib_map_mr_sg(struct ib_mr
*ibmr
, struct scatterlist
*sg
, int sg_nents
,
1825 unsigned int *sg_offset
)
1827 struct mlx5_ib_mr
*mr
= to_mmr(ibmr
);
1832 ib_dma_sync_single_for_cpu(ibmr
->device
, mr
->desc_map
,
1833 mr
->desc_size
* mr
->max_descs
,
1836 if (mr
->access_mode
== MLX5_MKC_ACCESS_MODE_KLMS
)
1837 n
= mlx5_ib_sg_to_klms(mr
, sg
, sg_nents
, sg_offset
);
1839 n
= ib_sg_to_pages(ibmr
, sg
, sg_nents
, sg_offset
,
1842 ib_dma_sync_single_for_device(ibmr
->device
, mr
->desc_map
,
1843 mr
->desc_size
* mr
->max_descs
,