]> git.proxmox.com Git - mirror_ubuntu-kernels.git/blob - drivers/infiniband/hw/mlx5/mr.c
projects / mirror_ubuntu-kernels.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
HID: logitech-dj: fix spelling in printk
[mirror_ubuntu-kernels.git] / drivers / infiniband / hw / mlx5 / mr.c
1 /*
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33
34 #include <linux/kref.h>
35 #include <linux/random.h>
36 #include <linux/debugfs.h>
37 #include <linux/export.h>
38 #include <linux/delay.h>
39 #include <rdma/ib_umem.h>
40 #include <rdma/ib_umem_odp.h>
41 #include <rdma/ib_verbs.h>
42 #include "mlx5_ib.h"
43
44 enum {
45 MAX_PENDING_REG_MR = 8,
46 };
47
48 #define MLX5_UMR_ALIGN 2048
49
50 static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
51 static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
52 static int mr_cache_max_order(struct mlx5_ib_dev *dev);
53 static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
54 static bool umr_can_modify_entity_size(struct mlx5_ib_dev *dev)
55 {
56 return !MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled);
57 }
58
59 static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev)
60 {
61 return !MLX5_CAP_GEN(dev->mdev, umr_indirect_mkey_disabled);
62 }
63
64 static bool use_umr(struct mlx5_ib_dev *dev, int order)
65 {
66 return order <= mr_cache_max_order(dev) &&
67 umr_can_modify_entity_size(dev);
68 }
69
70 static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
71 {
72 int err = mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
73
74 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
75 /* Wait until all page fault handlers using the mr complete. */
76 synchronize_srcu(&dev->mr_srcu);
77 #endif
78
79 return err;
80 }
81
82 static int order2idx(struct mlx5_ib_dev *dev, int order)
83 {
84 struct mlx5_mr_cache *cache = &dev->cache;
85
86 if (order < cache->ent[0].order)
87 return 0;
88 else
89 return order - cache->ent[0].order;
90 }
91
92 static bool use_umr_mtt_update(struct mlx5_ib_mr *mr, u64 start, u64 length)
93 {
94 return ((u64)1 << mr->order) * MLX5_ADAPTER_PAGE_SIZE >=
95 length + (start & (MLX5_ADAPTER_PAGE_SIZE - 1));
96 }
97
98 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
99 static void update_odp_mr(struct mlx5_ib_mr *mr)
100 {
101 if (mr->umem->is_odp) {
102 /*
103 * This barrier prevents the compiler from moving the
104 * setting of umem->odp_data->private to point to our
105 * MR, before reg_umr finished, to ensure that the MR
106 * initialization have finished before starting to
107 * handle invalidations.
108 */
109 smp_wmb();
110 to_ib_umem_odp(mr->umem)->private = mr;
111 /*
112 * Make sure we will see the new
113 * umem->odp_data->private value in the invalidation
114 * routines, before we can get page faults on the
115 * MR. Page faults can happen once we put the MR in
116 * the tree, below this line. Without the barrier,
117 * there can be a fault handling and an invalidation
118 * before umem->odp_data->private == mr is visible to
119 * the invalidation handler.
120 */
121 smp_wmb();
122 }
123 }
124 #endif
125
126 static void reg_mr_callback(int status, struct mlx5_async_work *context)
127 {
128 struct mlx5_ib_mr *mr =
129 container_of(context, struct mlx5_ib_mr, cb_work);
130 struct mlx5_ib_dev *dev = mr->dev;
131 struct mlx5_mr_cache *cache = &dev->cache;
132 int c = order2idx(dev, mr->order);
133 struct mlx5_cache_ent *ent = &cache->ent[c];
134 u8 key;
135 unsigned long flags;
136 struct mlx5_mkey_table *table = &dev->mdev->priv.mkey_table;
137 int err;
138
139 spin_lock_irqsave(&ent->lock, flags);
140 ent->pending--;
141 spin_unlock_irqrestore(&ent->lock, flags);
142 if (status) {
143 mlx5_ib_warn(dev, "async reg mr failed. status %d\n", status);
144 kfree(mr);
145 dev->fill_delay = 1;
146 mod_timer(&dev->delay_timer, jiffies + HZ);
147 return;
148 }
149
150 mr->mmkey.type = MLX5_MKEY_MR;
151 spin_lock_irqsave(&dev->mdev->priv.mkey_lock, flags);
152 key = dev->mdev->priv.mkey_key++;
153 spin_unlock_irqrestore(&dev->mdev->priv.mkey_lock, flags);
154 mr->mmkey.key = mlx5_idx_to_mkey(MLX5_GET(create_mkey_out, mr->out, mkey_index)) | key;
155
156 cache->last_add = jiffies;
157
158 spin_lock_irqsave(&ent->lock, flags);
159 list_add_tail(&mr->list, &ent->head);
160 ent->cur++;
161 ent->size++;
162 spin_unlock_irqrestore(&ent->lock, flags);
163
164 write_lock_irqsave(&table->lock, flags);
165 err = radix_tree_insert(&table->tree, mlx5_base_mkey(mr->mmkey.key),
166 &mr->mmkey);
167 if (err)
168 pr_err("Error inserting to mkey tree. 0x%x\n", -err);
169 write_unlock_irqrestore(&table->lock, flags);
170
171 if (!completion_done(&ent->compl))
172 complete(&ent->compl);
173 }
174
175 static int add_keys(struct mlx5_ib_dev *dev, int c, int num)
176 {
177 struct mlx5_mr_cache *cache = &dev->cache;
178 struct mlx5_cache_ent *ent = &cache->ent[c];
179 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
180 struct mlx5_ib_mr *mr;
181 void *mkc;
182 u32 *in;
183 int err = 0;
184 int i;
185
186 in = kzalloc(inlen, GFP_KERNEL);
187 if (!in)
188 return -ENOMEM;
189
190 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
191 for (i = 0; i < num; i++) {
192 if (ent->pending >= MAX_PENDING_REG_MR) {
193 err = -EAGAIN;
194 break;
195 }
196
197 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
198 if (!mr) {
199 err = -ENOMEM;
200 break;
201 }
202 mr->order = ent->order;
203 mr->allocated_from_cache = 1;
204 mr->dev = dev;
205
206 MLX5_SET(mkc, mkc, free, 1);
207 MLX5_SET(mkc, mkc, umr_en, 1);
208 MLX5_SET(mkc, mkc, access_mode_1_0, ent->access_mode & 0x3);
209 MLX5_SET(mkc, mkc, access_mode_4_2,
210 (ent->access_mode >> 2) & 0x7);
211
212 MLX5_SET(mkc, mkc, qpn, 0xffffff);
213 MLX5_SET(mkc, mkc, translations_octword_size, ent->xlt);
214 MLX5_SET(mkc, mkc, log_page_size, ent->page);
215
216 spin_lock_irq(&ent->lock);
217 ent->pending++;
218 spin_unlock_irq(&ent->lock);
219 err = mlx5_core_create_mkey_cb(dev->mdev, &mr->mmkey,
220 &dev->async_ctx, in, inlen,
221 mr->out, sizeof(mr->out),
222 reg_mr_callback, &mr->cb_work);
223 if (err) {
224 spin_lock_irq(&ent->lock);
225 ent->pending--;
226 spin_unlock_irq(&ent->lock);
227 mlx5_ib_warn(dev, "create mkey failed %d\n", err);
228 kfree(mr);
229 break;
230 }
231 }
232
233 kfree(in);
234 return err;
235 }
236
237 static void remove_keys(struct mlx5_ib_dev *dev, int c, int num)
238 {
239 struct mlx5_mr_cache *cache = &dev->cache;
240 struct mlx5_cache_ent *ent = &cache->ent[c];
241 struct mlx5_ib_mr *tmp_mr;
242 struct mlx5_ib_mr *mr;
243 LIST_HEAD(del_list);
244 int i;
245
246 for (i = 0; i < num; i++) {
247 spin_lock_irq(&ent->lock);
248 if (list_empty(&ent->head)) {
249 spin_unlock_irq(&ent->lock);
250 break;
251 }
252 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
253 list_move(&mr->list, &del_list);
254 ent->cur--;
255 ent->size--;
256 spin_unlock_irq(&ent->lock);
257 mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
258 }
259
260 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
261 synchronize_srcu(&dev->mr_srcu);
262 #endif
263
264 list_for_each_entry_safe(mr, tmp_mr, &del_list, list) {
265 list_del(&mr->list);
266 kfree(mr);
267 }
268 }
269
270 static ssize_t size_write(struct file *filp, const char __user *buf,
271 size_t count, loff_t *pos)
272 {
273 struct mlx5_cache_ent *ent = filp->private_data;
274 struct mlx5_ib_dev *dev = ent->dev;
275 char lbuf[20] = {0};
276 u32 var;
277 int err;
278 int c;
279
280 count = min(count, sizeof(lbuf) - 1);
281 if (copy_from_user(lbuf, buf, count))
282 return -EFAULT;
283
284 c = order2idx(dev, ent->order);
285
286 if (sscanf(lbuf, "%u", &var) != 1)
287 return -EINVAL;
288
289 if (var < ent->limit)
290 return -EINVAL;
291
292 if (var > ent->size) {
293 do {
294 err = add_keys(dev, c, var - ent->size);
295 if (err && err != -EAGAIN)
296 return err;
297
298 usleep_range(3000, 5000);
299 } while (err);
300 } else if (var < ent->size) {
301 remove_keys(dev, c, ent->size - var);
302 }
303
304 return count;
305 }
306
307 static ssize_t size_read(struct file *filp, char __user *buf, size_t count,
308 loff_t *pos)
309 {
310 struct mlx5_cache_ent *ent = filp->private_data;
311 char lbuf[20];
312 int err;
313
314 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->size);
315 if (err < 0)
316 return err;
317
318 return simple_read_from_buffer(buf, count, pos, lbuf, err);
319 }
320
321 static const struct file_operations size_fops = {
322 .owner = THIS_MODULE,
323 .open = simple_open,
324 .write = size_write,
325 .read = size_read,
326 };
327
328 static ssize_t limit_write(struct file *filp, const char __user *buf,
329 size_t count, loff_t *pos)
330 {
331 struct mlx5_cache_ent *ent = filp->private_data;
332 struct mlx5_ib_dev *dev = ent->dev;
333 char lbuf[20] = {0};
334 u32 var;
335 int err;
336 int c;
337
338 count = min(count, sizeof(lbuf) - 1);
339 if (copy_from_user(lbuf, buf, count))
340 return -EFAULT;
341
342 c = order2idx(dev, ent->order);
343
344 if (sscanf(lbuf, "%u", &var) != 1)
345 return -EINVAL;
346
347 if (var > ent->size)
348 return -EINVAL;
349
350 ent->limit = var;
351
352 if (ent->cur < ent->limit) {
353 err = add_keys(dev, c, 2 * ent->limit - ent->cur);
354 if (err)
355 return err;
356 }
357
358 return count;
359 }
360
361 static ssize_t limit_read(struct file *filp, char __user *buf, size_t count,
362 loff_t *pos)
363 {
364 struct mlx5_cache_ent *ent = filp->private_data;
365 char lbuf[20];
366 int err;
367
368 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->limit);
369 if (err < 0)
370 return err;
371
372 return simple_read_from_buffer(buf, count, pos, lbuf, err);
373 }
374
375 static const struct file_operations limit_fops = {
376 .owner = THIS_MODULE,
377 .open = simple_open,
378 .write = limit_write,
379 .read = limit_read,
380 };
381
382 static int someone_adding(struct mlx5_mr_cache *cache)
383 {
384 int i;
385
386 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
387 if (cache->ent[i].cur < cache->ent[i].limit)
388 return 1;
389 }
390
391 return 0;
392 }
393
394 static void __cache_work_func(struct mlx5_cache_ent *ent)
395 {
396 struct mlx5_ib_dev *dev = ent->dev;
397 struct mlx5_mr_cache *cache = &dev->cache;
398 int i = order2idx(dev, ent->order);
399 int err;
400
401 if (cache->stopped)
402 return;
403
404 ent = &dev->cache.ent[i];
405 if (ent->cur < 2 * ent->limit && !dev->fill_delay) {
406 err = add_keys(dev, i, 1);
407 if (ent->cur < 2 * ent->limit) {
408 if (err == -EAGAIN) {
409 mlx5_ib_dbg(dev, "returned eagain, order %d\n",
410 i + 2);
411 queue_delayed_work(cache->wq, &ent->dwork,
412 msecs_to_jiffies(3));
413 } else if (err) {
414 mlx5_ib_warn(dev, "command failed order %d, err %d\n",
415 i + 2, err);
416 queue_delayed_work(cache->wq, &ent->dwork,
417 msecs_to_jiffies(1000));
418 } else {
419 queue_work(cache->wq, &ent->work);
420 }
421 }
422 } else if (ent->cur > 2 * ent->limit) {
423 /*
424 * The remove_keys() logic is performed as garbage collection
425 * task. Such task is intended to be run when no other active
426 * processes are running.
427 *
428 * The need_resched() will return TRUE if there are user tasks
429 * to be activated in near future.
430 *
431 * In such case, we don't execute remove_keys() and postpone
432 * the garbage collection work to try to run in next cycle,
433 * in order to free CPU resources to other tasks.
434 */
435 if (!need_resched() && !someone_adding(cache) &&
436 time_after(jiffies, cache->last_add + 300 * HZ)) {
437 remove_keys(dev, i, 1);
438 if (ent->cur > ent->limit)
439 queue_work(cache->wq, &ent->work);
440 } else {
441 queue_delayed_work(cache->wq, &ent->dwork, 300 * HZ);
442 }
443 }
444 }
445
446 static void delayed_cache_work_func(struct work_struct *work)
447 {
448 struct mlx5_cache_ent *ent;
449
450 ent = container_of(work, struct mlx5_cache_ent, dwork.work);
451 __cache_work_func(ent);
452 }
453
454 static void cache_work_func(struct work_struct *work)
455 {
456 struct mlx5_cache_ent *ent;
457
458 ent = container_of(work, struct mlx5_cache_ent, work);
459 __cache_work_func(ent);
460 }
461
462 struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, int entry)
463 {
464 struct mlx5_mr_cache *cache = &dev->cache;
465 struct mlx5_cache_ent *ent;
466 struct mlx5_ib_mr *mr;
467 int err;
468
469 if (entry < 0 || entry >= MAX_MR_CACHE_ENTRIES) {
470 mlx5_ib_err(dev, "cache entry %d is out of range\n", entry);
471 return NULL;
472 }
473
474 ent = &cache->ent[entry];
475 while (1) {
476 spin_lock_irq(&ent->lock);
477 if (list_empty(&ent->head)) {
478 spin_unlock_irq(&ent->lock);
479
480 err = add_keys(dev, entry, 1);
481 if (err && err != -EAGAIN)
482 return ERR_PTR(err);
483
484 wait_for_completion(&ent->compl);
485 } else {
486 mr = list_first_entry(&ent->head, struct mlx5_ib_mr,
487 list);
488 list_del(&mr->list);
489 ent->cur--;
490 spin_unlock_irq(&ent->lock);
491 if (ent->cur < ent->limit)
492 queue_work(cache->wq, &ent->work);
493 return mr;
494 }
495 }
496 }
497
498 static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_ib_dev *dev, int order)
499 {
500 struct mlx5_mr_cache *cache = &dev->cache;
501 struct mlx5_ib_mr *mr = NULL;
502 struct mlx5_cache_ent *ent;
503 int last_umr_cache_entry;
504 int c;
505 int i;
506
507 c = order2idx(dev, order);
508 last_umr_cache_entry = order2idx(dev, mr_cache_max_order(dev));
509 if (c < 0 || c > last_umr_cache_entry) {
510 mlx5_ib_warn(dev, "order %d, cache index %d\n", order, c);
511 return NULL;
512 }
513
514 for (i = c; i <= last_umr_cache_entry; i++) {
515 ent = &cache->ent[i];
516
517 mlx5_ib_dbg(dev, "order %d, cache index %d\n", ent->order, i);
518
519 spin_lock_irq(&ent->lock);
520 if (!list_empty(&ent->head)) {
521 mr = list_first_entry(&ent->head, struct mlx5_ib_mr,
522 list);
523 list_del(&mr->list);
524 ent->cur--;
525 spin_unlock_irq(&ent->lock);
526 if (ent->cur < ent->limit)
527 queue_work(cache->wq, &ent->work);
528 break;
529 }
530 spin_unlock_irq(&ent->lock);
531
532 queue_work(cache->wq, &ent->work);
533 }
534
535 if (!mr)
536 cache->ent[c].miss++;
537
538 return mr;
539 }
540
541 void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
542 {
543 struct mlx5_mr_cache *cache = &dev->cache;
544 struct mlx5_cache_ent *ent;
545 int shrink = 0;
546 int c;
547
548 if (!mr->allocated_from_cache)
549 return;
550
551 c = order2idx(dev, mr->order);
552 if (c < 0 || c >= MAX_MR_CACHE_ENTRIES) {
553 mlx5_ib_warn(dev, "order %d, cache index %d\n", mr->order, c);
554 return;
555 }
556
557 if (unreg_umr(dev, mr))
558 return;
559
560 ent = &cache->ent[c];
561 spin_lock_irq(&ent->lock);
562 list_add_tail(&mr->list, &ent->head);
563 ent->cur++;
564 if (ent->cur > 2 * ent->limit)
565 shrink = 1;
566 spin_unlock_irq(&ent->lock);
567
568 if (shrink)
569 queue_work(cache->wq, &ent->work);
570 }
571
572 static void clean_keys(struct mlx5_ib_dev *dev, int c)
573 {
574 struct mlx5_mr_cache *cache = &dev->cache;
575 struct mlx5_cache_ent *ent = &cache->ent[c];
576 struct mlx5_ib_mr *tmp_mr;
577 struct mlx5_ib_mr *mr;
578 LIST_HEAD(del_list);
579
580 cancel_delayed_work(&ent->dwork);
581 while (1) {
582 spin_lock_irq(&ent->lock);
583 if (list_empty(&ent->head)) {
584 spin_unlock_irq(&ent->lock);
585 break;
586 }
587 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
588 list_move(&mr->list, &del_list);
589 ent->cur--;
590 ent->size--;
591 spin_unlock_irq(&ent->lock);
592 mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
593 }
594
595 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
596 synchronize_srcu(&dev->mr_srcu);
597 #endif
598
599 list_for_each_entry_safe(mr, tmp_mr, &del_list, list) {
600 list_del(&mr->list);
601 kfree(mr);
602 }
603 }
604
605 static void mlx5_mr_cache_debugfs_cleanup(struct mlx5_ib_dev *dev)
606 {
607 if (!mlx5_debugfs_root || dev->rep)
608 return;
609
610 debugfs_remove_recursive(dev->cache.root);
611 dev->cache.root = NULL;
612 }
613
614 static int mlx5_mr_cache_debugfs_init(struct mlx5_ib_dev *dev)
615 {
616 struct mlx5_mr_cache *cache = &dev->cache;
617 struct mlx5_cache_ent *ent;
618 int i;
619
620 if (!mlx5_debugfs_root || dev->rep)
621 return 0;
622
623 cache->root = debugfs_create_dir("mr_cache", dev->mdev->priv.dbg_root);
624 if (!cache->root)
625 return -ENOMEM;
626
627 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
628 ent = &cache->ent[i];
629 sprintf(ent->name, "%d", ent->order);
630 ent->dir = debugfs_create_dir(ent->name, cache->root);
631 if (!ent->dir)
632 goto err;
633
634 ent->fsize = debugfs_create_file("size", 0600, ent->dir, ent,
635 &size_fops);
636 if (!ent->fsize)
637 goto err;
638
639 ent->flimit = debugfs_create_file("limit", 0600, ent->dir, ent,
640 &limit_fops);
641 if (!ent->flimit)
642 goto err;
643
644 ent->fcur = debugfs_create_u32("cur", 0400, ent->dir,
645 &ent->cur);
646 if (!ent->fcur)
647 goto err;
648
649 ent->fmiss = debugfs_create_u32("miss", 0600, ent->dir,
650 &ent->miss);
651 if (!ent->fmiss)
652 goto err;
653 }
654
655 return 0;
656 err:
657 mlx5_mr_cache_debugfs_cleanup(dev);
658
659 return -ENOMEM;
660 }
661
662 static void delay_time_func(struct timer_list *t)
663 {
664 struct mlx5_ib_dev *dev = from_timer(dev, t, delay_timer);
665
666 dev->fill_delay = 0;
667 }
668
669 int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
670 {
671 struct mlx5_mr_cache *cache = &dev->cache;
672 struct mlx5_cache_ent *ent;
673 int err;
674 int i;
675
676 mutex_init(&dev->slow_path_mutex);
677 cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM);
678 if (!cache->wq) {
679 mlx5_ib_warn(dev, "failed to create work queue\n");
680 return -ENOMEM;
681 }
682
683 mlx5_cmd_init_async_ctx(dev->mdev, &dev->async_ctx);
684 timer_setup(&dev->delay_timer, delay_time_func, 0);
685 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
686 ent = &cache->ent[i];
687 INIT_LIST_HEAD(&ent->head);
688 spin_lock_init(&ent->lock);
689 ent->order = i + 2;
690 ent->dev = dev;
691 ent->limit = 0;
692
693 init_completion(&ent->compl);
694 INIT_WORK(&ent->work, cache_work_func);
695 INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);
696
697 if (i > MR_CACHE_LAST_STD_ENTRY) {
698 mlx5_odp_init_mr_cache_entry(ent);
699 continue;
700 }
701
702 if (ent->order > mr_cache_max_order(dev))
703 continue;
704
705 ent->page = PAGE_SHIFT;
706 ent->xlt = (1 << ent->order) * sizeof(struct mlx5_mtt) /
707 MLX5_IB_UMR_OCTOWORD;
708 ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
709 if ((dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) &&
710 !dev->rep &&
711 mlx5_core_is_pf(dev->mdev))
712 ent->limit = dev->mdev->profile->mr_cache[i].limit;
713 else
714 ent->limit = 0;
715 queue_work(cache->wq, &ent->work);
716 }
717
718 err = mlx5_mr_cache_debugfs_init(dev);
719 if (err)
720 mlx5_ib_warn(dev, "cache debugfs failure\n");
721
722 /*
723 * We don't want to fail driver if debugfs failed to initialize,
724 * so we are not forwarding error to the user.
725 */
726
727 return 0;
728 }
729
730 int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev)
731 {
732 int i;
733
734 if (!dev->cache.wq)
735 return 0;
736
737 dev->cache.stopped = 1;
738 flush_workqueue(dev->cache.wq);
739
740 mlx5_mr_cache_debugfs_cleanup(dev);
741 mlx5_cmd_cleanup_async_ctx(&dev->async_ctx);
742
743 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++)
744 clean_keys(dev, i);
745
746 destroy_workqueue(dev->cache.wq);
747 del_timer_sync(&dev->delay_timer);
748
749 return 0;
750 }
751
752 struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc)
753 {
754 struct mlx5_ib_dev *dev = to_mdev(pd->device);
755 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
756 struct mlx5_core_dev *mdev = dev->mdev;
757 struct mlx5_ib_mr *mr;
758 void *mkc;
759 u32 *in;
760 int err;
761
762 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
763 if (!mr)
764 return ERR_PTR(-ENOMEM);
765
766 in = kzalloc(inlen, GFP_KERNEL);
767 if (!in) {
768 err = -ENOMEM;
769 goto err_free;
770 }
771
772 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
773
774 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
775 MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC));
776 MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE));
777 MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ));
778 MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE));
779 MLX5_SET(mkc, mkc, lr, 1);
780
781 MLX5_SET(mkc, mkc, length64, 1);
782 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
783 MLX5_SET(mkc, mkc, qpn, 0xffffff);
784 MLX5_SET64(mkc, mkc, start_addr, 0);
785
786 err = mlx5_core_create_mkey(mdev, &mr->mmkey, in, inlen);
787 if (err)
788 goto err_in;
789
790 kfree(in);
791 mr->mmkey.type = MLX5_MKEY_MR;
792 mr->ibmr.lkey = mr->mmkey.key;
793 mr->ibmr.rkey = mr->mmkey.key;
794 mr->umem = NULL;
795
796 return &mr->ibmr;
797
798 err_in:
799 kfree(in);
800
801 err_free:
802 kfree(mr);
803
804 return ERR_PTR(err);
805 }
806
807 static int get_octo_len(u64 addr, u64 len, int page_shift)
808 {
809 u64 page_size = 1ULL << page_shift;
810 u64 offset;
811 int npages;
812
813 offset = addr & (page_size - 1);
814 npages = ALIGN(len + offset, page_size) >> page_shift;
815 return (npages + 1) / 2;
816 }
817
818 static int mr_cache_max_order(struct mlx5_ib_dev *dev)
819 {
820 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset))
821 return MR_CACHE_LAST_STD_ENTRY + 2;
822 return MLX5_MAX_UMR_SHIFT;
823 }
824
825 static int mr_umem_get(struct ib_pd *pd, u64 start, u64 length,
826 int access_flags, struct ib_umem **umem,
827 int *npages, int *page_shift, int *ncont,
828 int *order)
829 {
830 struct mlx5_ib_dev *dev = to_mdev(pd->device);
831 struct ib_umem *u;
832 int err;
833
834 *umem = NULL;
835
836 u = ib_umem_get(pd->uobject->context, start, length, access_flags, 0);
837 err = PTR_ERR_OR_ZERO(u);
838 if (err) {
839 mlx5_ib_dbg(dev, "umem get failed (%d)\n", err);
840 return err;
841 }
842
843 mlx5_ib_cont_pages(u, start, MLX5_MKEY_PAGE_SHIFT_MASK, npages,
844 page_shift, ncont, order);
845 if (!*npages) {
846 mlx5_ib_warn(dev, "avoid zero region\n");
847 ib_umem_release(u);
848 return -EINVAL;
849 }
850
851 *umem = u;
852
853 mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n",
854 *npages, *ncont, *order, *page_shift);
855
856 return 0;
857 }
858
859 static void mlx5_ib_umr_done(struct ib_cq *cq, struct ib_wc *wc)
860 {
861 struct mlx5_ib_umr_context *context =
862 container_of(wc->wr_cqe, struct mlx5_ib_umr_context, cqe);
863
864 context->status = wc->status;
865 complete(&context->done);
866 }
867
868 static inline void mlx5_ib_init_umr_context(struct mlx5_ib_umr_context *context)
869 {
870 context->cqe.done = mlx5_ib_umr_done;
871 context->status = -1;
872 init_completion(&context->done);
873 }
874
875 static int mlx5_ib_post_send_wait(struct mlx5_ib_dev *dev,
876 struct mlx5_umr_wr *umrwr)
877 {
878 struct umr_common *umrc = &dev->umrc;
879 const struct ib_send_wr *bad;
880 int err;
881 struct mlx5_ib_umr_context umr_context;
882
883 mlx5_ib_init_umr_context(&umr_context);
884 umrwr->wr.wr_cqe = &umr_context.cqe;
885
886 down(&umrc->sem);
887 err = ib_post_send(umrc->qp, &umrwr->wr, &bad);
888 if (err) {
889 mlx5_ib_warn(dev, "UMR post send failed, err %d\n", err);
890 } else {
891 wait_for_completion(&umr_context.done);
892 if (umr_context.status != IB_WC_SUCCESS) {
893 mlx5_ib_warn(dev, "reg umr failed (%u)\n",
894 umr_context.status);
895 err = -EFAULT;
896 }
897 }
898 up(&umrc->sem);
899 return err;
900 }
901
902 static struct mlx5_ib_mr *alloc_mr_from_cache(
903 struct ib_pd *pd, struct ib_umem *umem,
904 u64 virt_addr, u64 len, int npages,
905 int page_shift, int order, int access_flags)
906 {
907 struct mlx5_ib_dev *dev = to_mdev(pd->device);
908 struct mlx5_ib_mr *mr;
909 int err = 0;
910 int i;
911
912 for (i = 0; i < 1; i++) {
913 mr = alloc_cached_mr(dev, order);
914 if (mr)
915 break;
916
917 err = add_keys(dev, order2idx(dev, order), 1);
918 if (err && err != -EAGAIN) {
919 mlx5_ib_warn(dev, "add_keys failed, err %d\n", err);
920 break;
921 }
922 }
923
924 if (!mr)
925 return ERR_PTR(-EAGAIN);
926
927 mr->ibmr.pd = pd;
928 mr->umem = umem;
929 mr->access_flags = access_flags;
930 mr->desc_size = sizeof(struct mlx5_mtt);
931 mr->mmkey.iova = virt_addr;
932 mr->mmkey.size = len;
933 mr->mmkey.pd = to_mpd(pd)->pdn;
934
935 return mr;
936 }
937
938 static inline int populate_xlt(struct mlx5_ib_mr *mr, int idx, int npages,
939 void *xlt, int page_shift, size_t size,
940 int flags)
941 {
942 struct mlx5_ib_dev *dev = mr->dev;
943 struct ib_umem *umem = mr->umem;
944
945 if (flags & MLX5_IB_UPD_XLT_INDIRECT) {
946 if (!umr_can_use_indirect_mkey(dev))
947 return -EPERM;
948 mlx5_odp_populate_klm(xlt, idx, npages, mr, flags);
949 return npages;
950 }
951
952 npages = min_t(size_t, npages, ib_umem_num_pages(umem) - idx);
953
954 if (!(flags & MLX5_IB_UPD_XLT_ZAP)) {
955 __mlx5_ib_populate_pas(dev, umem, page_shift,
956 idx, npages, xlt,
957 MLX5_IB_MTT_PRESENT);
958 /* Clear padding after the pages
959 * brought from the umem.
960 */
961 memset(xlt + (npages * sizeof(struct mlx5_mtt)), 0,
962 size - npages * sizeof(struct mlx5_mtt));
963 }
964
965 return npages;
966 }
967
968 #define MLX5_MAX_UMR_CHUNK ((1 << (MLX5_MAX_UMR_SHIFT + 4)) - \
969 MLX5_UMR_MTT_ALIGNMENT)
970 #define MLX5_SPARE_UMR_CHUNK 0x10000
971
972 int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
973 int page_shift, int flags)
974 {
975 struct mlx5_ib_dev *dev = mr->dev;
976 struct device *ddev = dev->ib_dev.dev.parent;
977 int size;
978 void *xlt;
979 dma_addr_t dma;
980 struct mlx5_umr_wr wr;
981 struct ib_sge sg;
982 int err = 0;
983 int desc_size = (flags & MLX5_IB_UPD_XLT_INDIRECT)
984 ? sizeof(struct mlx5_klm)
985 : sizeof(struct mlx5_mtt);
986 const int page_align = MLX5_UMR_MTT_ALIGNMENT / desc_size;
987 const int page_mask = page_align - 1;
988 size_t pages_mapped = 0;
989 size_t pages_to_map = 0;
990 size_t pages_iter = 0;
991 gfp_t gfp;
992 bool use_emergency_page = false;
993
994 if ((flags & MLX5_IB_UPD_XLT_INDIRECT) &&
995 !umr_can_use_indirect_mkey(dev))
996 return -EPERM;
997
998 /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes,
999 * so we need to align the offset and length accordingly
1000 */
1001 if (idx & page_mask) {
1002 npages += idx & page_mask;
1003 idx &= ~page_mask;
1004 }
1005
1006 gfp = flags & MLX5_IB_UPD_XLT_ATOMIC ? GFP_ATOMIC : GFP_KERNEL;
1007 gfp |= __GFP_ZERO | __GFP_NOWARN;
1008
1009 pages_to_map = ALIGN(npages, page_align);
1010 size = desc_size * pages_to_map;
1011 size = min_t(int, size, MLX5_MAX_UMR_CHUNK);
1012
1013 xlt = (void *)__get_free_pages(gfp, get_order(size));
1014 if (!xlt && size > MLX5_SPARE_UMR_CHUNK) {
1015 mlx5_ib_dbg(dev, "Failed to allocate %d bytes of order %d. fallback to spare UMR allocation od %d bytes\n",
1016 size, get_order(size), MLX5_SPARE_UMR_CHUNK);
1017
1018 size = MLX5_SPARE_UMR_CHUNK;
1019 xlt = (void *)__get_free_pages(gfp, get_order(size));
1020 }
1021
1022 if (!xlt) {
1023 mlx5_ib_warn(dev, "Using XLT emergency buffer\n");
1024 xlt = (void *)mlx5_ib_get_xlt_emergency_page();
1025 size = PAGE_SIZE;
1026 memset(xlt, 0, size);
1027 use_emergency_page = true;
1028 }
1029 pages_iter = size / desc_size;
1030 dma = dma_map_single(ddev, xlt, size, DMA_TO_DEVICE);
1031 if (dma_mapping_error(ddev, dma)) {
1032 mlx5_ib_err(dev, "unable to map DMA during XLT update.\n");
1033 err = -ENOMEM;
1034 goto free_xlt;
1035 }
1036
1037 sg.addr = dma;
1038 sg.lkey = dev->umrc.pd->local_dma_lkey;
1039
1040 memset(&wr, 0, sizeof(wr));
1041 wr.wr.send_flags = MLX5_IB_SEND_UMR_UPDATE_XLT;
1042 if (!(flags & MLX5_IB_UPD_XLT_ENABLE))
1043 wr.wr.send_flags |= MLX5_IB_SEND_UMR_FAIL_IF_FREE;
1044 wr.wr.sg_list = &sg;
1045 wr.wr.num_sge = 1;
1046 wr.wr.opcode = MLX5_IB_WR_UMR;
1047
1048 wr.pd = mr->ibmr.pd;
1049 wr.mkey = mr->mmkey.key;
1050 wr.length = mr->mmkey.size;
1051 wr.virt_addr = mr->mmkey.iova;
1052 wr.access_flags = mr->access_flags;
1053 wr.page_shift = page_shift;
1054
1055 for (pages_mapped = 0;
1056 pages_mapped < pages_to_map && !err;
1057 pages_mapped += pages_iter, idx += pages_iter) {
1058 npages = min_t(int, pages_iter, pages_to_map - pages_mapped);
1059 dma_sync_single_for_cpu(ddev, dma, size, DMA_TO_DEVICE);
1060 npages = populate_xlt(mr, idx, npages, xlt,
1061 page_shift, size, flags);
1062
1063 dma_sync_single_for_device(ddev, dma, size, DMA_TO_DEVICE);
1064
1065 sg.length = ALIGN(npages * desc_size,
1066 MLX5_UMR_MTT_ALIGNMENT);
1067
1068 if (pages_mapped + pages_iter >= pages_to_map) {
1069 if (flags & MLX5_IB_UPD_XLT_ENABLE)
1070 wr.wr.send_flags |=
1071 MLX5_IB_SEND_UMR_ENABLE_MR |
1072 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS |
1073 MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;
1074 if (flags & MLX5_IB_UPD_XLT_PD ||
1075 flags & MLX5_IB_UPD_XLT_ACCESS)
1076 wr.wr.send_flags |=
1077 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
1078 if (flags & MLX5_IB_UPD_XLT_ADDR)
1079 wr.wr.send_flags |=
1080 MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;
1081 }
1082
1083 wr.offset = idx * desc_size;
1084 wr.xlt_size = sg.length;
1085
1086 err = mlx5_ib_post_send_wait(dev, &wr);
1087 }
1088 dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE);
1089
1090 free_xlt:
1091 if (use_emergency_page)
1092 mlx5_ib_put_xlt_emergency_page();
1093 else
1094 free_pages((unsigned long)xlt, get_order(size));
1095
1096 return err;
1097 }
1098
1099 /*
1100 * If ibmr is NULL it will be allocated by reg_create.
1101 * Else, the given ibmr will be used.
1102 */
1103 static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
1104 u64 virt_addr, u64 length,
1105 struct ib_umem *umem, int npages,
1106 int page_shift, int access_flags,
1107 bool populate)
1108 {
1109 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1110 struct mlx5_ib_mr *mr;
1111 __be64 *pas;
1112 void *mkc;
1113 int inlen;
1114 u32 *in;
1115 int err;
1116 bool pg_cap = !!(MLX5_CAP_GEN(dev->mdev, pg));
1117
1118 mr = ibmr ? to_mmr(ibmr) : kzalloc(sizeof(*mr), GFP_KERNEL);
1119 if (!mr)
1120 return ERR_PTR(-ENOMEM);
1121
1122 mr->ibmr.pd = pd;
1123 mr->access_flags = access_flags;
1124
1125 inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1126 if (populate)
1127 inlen += sizeof(*pas) * roundup(npages, 2);
1128 in = kvzalloc(inlen, GFP_KERNEL);
1129 if (!in) {
1130 err = -ENOMEM;
1131 goto err_1;
1132 }
1133 pas = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
1134 if (populate && !(access_flags & IB_ACCESS_ON_DEMAND))
1135 mlx5_ib_populate_pas(dev, umem, page_shift, pas,
1136 pg_cap ? MLX5_IB_MTT_PRESENT : 0);
1137
1138 /* The pg_access bit allows setting the access flags
1139 * in the page list submitted with the command. */
1140 MLX5_SET(create_mkey_in, in, pg_access, !!(pg_cap));
1141
1142 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1143 MLX5_SET(mkc, mkc, free, !populate);
1144 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
1145 MLX5_SET(mkc, mkc, a, !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
1146 MLX5_SET(mkc, mkc, rw, !!(access_flags & IB_ACCESS_REMOTE_WRITE));
1147 MLX5_SET(mkc, mkc, rr, !!(access_flags & IB_ACCESS_REMOTE_READ));
1148 MLX5_SET(mkc, mkc, lw, !!(access_flags & IB_ACCESS_LOCAL_WRITE));
1149 MLX5_SET(mkc, mkc, lr, 1);
1150 MLX5_SET(mkc, mkc, umr_en, 1);
1151
1152 MLX5_SET64(mkc, mkc, start_addr, virt_addr);
1153 MLX5_SET64(mkc, mkc, len, length);
1154 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
1155 MLX5_SET(mkc, mkc, bsf_octword_size, 0);
1156 MLX5_SET(mkc, mkc, translations_octword_size,
1157 get_octo_len(virt_addr, length, page_shift));
1158 MLX5_SET(mkc, mkc, log_page_size, page_shift);
1159 MLX5_SET(mkc, mkc, qpn, 0xffffff);
1160 if (populate) {
1161 MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
1162 get_octo_len(virt_addr, length, page_shift));
1163 }
1164
1165 err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen);
1166 if (err) {
1167 mlx5_ib_warn(dev, "create mkey failed\n");
1168 goto err_2;
1169 }
1170 mr->mmkey.type = MLX5_MKEY_MR;
1171 mr->desc_size = sizeof(struct mlx5_mtt);
1172 mr->dev = dev;
1173 kvfree(in);
1174
1175 mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmkey.key);
1176
1177 return mr;
1178
1179 err_2:
1180 kvfree(in);
1181
1182 err_1:
1183 if (!ibmr)
1184 kfree(mr);
1185
1186 return ERR_PTR(err);
1187 }
1188
1189 static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,
1190 int npages, u64 length, int access_flags)
1191 {
1192 mr->npages = npages;
1193 atomic_add(npages, &dev->mdev->priv.reg_pages);
1194 mr->ibmr.lkey = mr->mmkey.key;
1195 mr->ibmr.rkey = mr->mmkey.key;
1196 mr->ibmr.length = length;
1197 mr->access_flags = access_flags;
1198 }
1199
1200 static struct ib_mr *mlx5_ib_get_memic_mr(struct ib_pd *pd, u64 memic_addr,
1201 u64 length, int acc)
1202 {
1203 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1204 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1205 struct mlx5_core_dev *mdev = dev->mdev;
1206 struct mlx5_ib_mr *mr;
1207 void *mkc;
1208 u32 *in;
1209 int err;
1210
1211 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1212 if (!mr)
1213 return ERR_PTR(-ENOMEM);
1214
1215 in = kzalloc(inlen, GFP_KERNEL);
1216 if (!in) {
1217 err = -ENOMEM;
1218 goto err_free;
1219 }
1220
1221 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1222
1223 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MEMIC & 0x3);
1224 MLX5_SET(mkc, mkc, access_mode_4_2,
1225 (MLX5_MKC_ACCESS_MODE_MEMIC >> 2) & 0x7);
1226 MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC));
1227 MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE));
1228 MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ));
1229 MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE));
1230 MLX5_SET(mkc, mkc, lr, 1);
1231
1232 MLX5_SET64(mkc, mkc, len, length);
1233 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
1234 MLX5_SET(mkc, mkc, qpn, 0xffffff);
1235 MLX5_SET64(mkc, mkc, start_addr,
1236 memic_addr - pci_resource_start(dev->mdev->pdev, 0));
1237
1238 err = mlx5_core_create_mkey(mdev, &mr->mmkey, in, inlen);
1239 if (err)
1240 goto err_in;
1241
1242 kfree(in);
1243
1244 mr->umem = NULL;
1245 set_mr_fields(dev, mr, 0, length, acc);
1246
1247 return &mr->ibmr;
1248
1249 err_in:
1250 kfree(in);
1251
1252 err_free:
1253 kfree(mr);
1254
1255 return ERR_PTR(err);
1256 }
1257
1258 int mlx5_ib_advise_mr(struct ib_pd *pd,
1259 enum ib_uverbs_advise_mr_advice advice,
1260 u32 flags,
1261 struct ib_sge *sg_list,
1262 u32 num_sge,
1263 struct uverbs_attr_bundle *attrs)
1264 {
1265 if (advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH &&
1266 advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE)
1267 return -EOPNOTSUPP;
1268
1269 return mlx5_ib_advise_mr_prefetch(pd, advice, flags,
1270 sg_list, num_sge);
1271 }
1272
1273 struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm,
1274 struct ib_dm_mr_attr *attr,
1275 struct uverbs_attr_bundle *attrs)
1276 {
1277 struct mlx5_ib_dm *mdm = to_mdm(dm);
1278 u64 memic_addr;
1279
1280 if (attr->access_flags & ~MLX5_IB_DM_ALLOWED_ACCESS)
1281 return ERR_PTR(-EINVAL);
1282
1283 memic_addr = mdm->dev_addr + attr->offset;
1284
1285 return mlx5_ib_get_memic_mr(pd, memic_addr, attr->length,
1286 attr->access_flags);
1287 }
1288
1289 struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
1290 u64 virt_addr, int access_flags,
1291 struct ib_udata *udata)
1292 {
1293 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1294 struct mlx5_ib_mr *mr = NULL;
1295 bool populate_mtts = false;
1296 struct ib_umem *umem;
1297 int page_shift;
1298 int npages;
1299 int ncont;
1300 int order;
1301 int err;
1302
1303 if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM))
1304 return ERR_PTR(-EOPNOTSUPP);
1305
1306 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
1307 start, virt_addr, length, access_flags);
1308
1309 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1310 if (!start && length == U64_MAX) {
1311 if (!(access_flags & IB_ACCESS_ON_DEMAND) ||
1312 !(dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT))
1313 return ERR_PTR(-EINVAL);
1314
1315 mr = mlx5_ib_alloc_implicit_mr(to_mpd(pd), access_flags);
1316 if (IS_ERR(mr))
1317 return ERR_CAST(mr);
1318 return &mr->ibmr;
1319 }
1320 #endif
1321
1322 err = mr_umem_get(pd, start, length, access_flags, &umem, &npages,
1323 &page_shift, &ncont, &order);
1324
1325 if (err < 0)
1326 return ERR_PTR(err);
1327
1328 if (use_umr(dev, order)) {
1329 mr = alloc_mr_from_cache(pd, umem, virt_addr, length, ncont,
1330 page_shift, order, access_flags);
1331 if (PTR_ERR(mr) == -EAGAIN) {
1332 mlx5_ib_dbg(dev, "cache empty for order %d\n", order);
1333 mr = NULL;
1334 }
1335 populate_mtts = false;
1336 } else if (!MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) {
1337 if (access_flags & IB_ACCESS_ON_DEMAND) {
1338 err = -EINVAL;
1339 pr_err("Got MR registration for ODP MR > 512MB, not supported for Connect-IB\n");
1340 goto error;
1341 }
1342 populate_mtts = true;
1343 }
1344
1345 if (!mr) {
1346 if (!umr_can_modify_entity_size(dev))
1347 populate_mtts = true;
1348 mutex_lock(&dev->slow_path_mutex);
1349 mr = reg_create(NULL, pd, virt_addr, length, umem, ncont,
1350 page_shift, access_flags, populate_mtts);
1351 mutex_unlock(&dev->slow_path_mutex);
1352 }
1353
1354 if (IS_ERR(mr)) {
1355 err = PTR_ERR(mr);
1356 goto error;
1357 }
1358
1359 mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key);
1360
1361 mr->umem = umem;
1362 set_mr_fields(dev, mr, npages, length, access_flags);
1363
1364 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1365 update_odp_mr(mr);
1366 #endif
1367
1368 if (!populate_mtts) {
1369 int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE;
1370
1371 if (access_flags & IB_ACCESS_ON_DEMAND)
1372 update_xlt_flags |= MLX5_IB_UPD_XLT_ZAP;
1373
1374 err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift,
1375 update_xlt_flags);
1376
1377 if (err) {
1378 dereg_mr(dev, mr);
1379 return ERR_PTR(err);
1380 }
1381 }
1382
1383 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1384 mr->live = 1;
1385 #endif
1386 return &mr->ibmr;
1387 error:
1388 ib_umem_release(umem);
1389 return ERR_PTR(err);
1390 }
1391
1392 static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
1393 {
1394 struct mlx5_core_dev *mdev = dev->mdev;
1395 struct mlx5_umr_wr umrwr = {};
1396
1397 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
1398 return 0;
1399
1400 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_DISABLE_MR |
1401 MLX5_IB_SEND_UMR_FAIL_IF_FREE;
1402 umrwr.wr.opcode = MLX5_IB_WR_UMR;
1403 umrwr.mkey = mr->mmkey.key;
1404
1405 return mlx5_ib_post_send_wait(dev, &umrwr);
1406 }
1407
1408 static int rereg_umr(struct ib_pd *pd, struct mlx5_ib_mr *mr,
1409 int access_flags, int flags)
1410 {
1411 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1412 struct mlx5_umr_wr umrwr = {};
1413 int err;
1414
1415 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_FAIL_IF_FREE;
1416
1417 umrwr.wr.opcode = MLX5_IB_WR_UMR;
1418 umrwr.mkey = mr->mmkey.key;
1419
1420 if (flags & IB_MR_REREG_PD || flags & IB_MR_REREG_ACCESS) {
1421 umrwr.pd = pd;
1422 umrwr.access_flags = access_flags;
1423 umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
1424 }
1425
1426 err = mlx5_ib_post_send_wait(dev, &umrwr);
1427
1428 return err;
1429 }
1430
1431 int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
1432 u64 length, u64 virt_addr, int new_access_flags,
1433 struct ib_pd *new_pd, struct ib_udata *udata)
1434 {
1435 struct mlx5_ib_dev *dev = to_mdev(ib_mr->device);
1436 struct mlx5_ib_mr *mr = to_mmr(ib_mr);
1437 struct ib_pd *pd = (flags & IB_MR_REREG_PD) ? new_pd : ib_mr->pd;
1438 int access_flags = flags & IB_MR_REREG_ACCESS ?
1439 new_access_flags :
1440 mr->access_flags;
1441 int page_shift = 0;
1442 int upd_flags = 0;
1443 int npages = 0;
1444 int ncont = 0;
1445 int order = 0;
1446 u64 addr, len;
1447 int err;
1448
1449 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
1450 start, virt_addr, length, access_flags);
1451
1452 atomic_sub(mr->npages, &dev->mdev->priv.reg_pages);
1453
1454 if (!mr->umem)
1455 return -EINVAL;
1456
1457 if (flags & IB_MR_REREG_TRANS) {
1458 addr = virt_addr;
1459 len = length;
1460 } else {
1461 addr = mr->umem->address;
1462 len = mr->umem->length;
1463 }
1464
1465 if (flags != IB_MR_REREG_PD) {
1466 /*
1467 * Replace umem. This needs to be done whether or not UMR is
1468 * used.
1469 */
1470 flags |= IB_MR_REREG_TRANS;
1471 ib_umem_release(mr->umem);
1472 mr->umem = NULL;
1473 err = mr_umem_get(pd, addr, len, access_flags, &mr->umem,
1474 &npages, &page_shift, &ncont, &order);
1475 if (err)
1476 goto err;
1477 }
1478
1479 if (flags & IB_MR_REREG_TRANS && !use_umr_mtt_update(mr, addr, len)) {
1480 /*
1481 * UMR can't be used - MKey needs to be replaced.
1482 */
1483 if (mr->allocated_from_cache)
1484 err = unreg_umr(dev, mr);
1485 else
1486 err = destroy_mkey(dev, mr);
1487 if (err)
1488 goto err;
1489
1490 mr = reg_create(ib_mr, pd, addr, len, mr->umem, ncont,
1491 page_shift, access_flags, true);
1492
1493 if (IS_ERR(mr)) {
1494 err = PTR_ERR(mr);
1495 mr = to_mmr(ib_mr);
1496 goto err;
1497 }
1498
1499 mr->allocated_from_cache = 0;
1500 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1501 mr->live = 1;
1502 #endif
1503 } else {
1504 /*
1505 * Send a UMR WQE
1506 */
1507 mr->ibmr.pd = pd;
1508 mr->access_flags = access_flags;
1509 mr->mmkey.iova = addr;
1510 mr->mmkey.size = len;
1511 mr->mmkey.pd = to_mpd(pd)->pdn;
1512
1513 if (flags & IB_MR_REREG_TRANS) {
1514 upd_flags = MLX5_IB_UPD_XLT_ADDR;
1515 if (flags & IB_MR_REREG_PD)
1516 upd_flags |= MLX5_IB_UPD_XLT_PD;
1517 if (flags & IB_MR_REREG_ACCESS)
1518 upd_flags |= MLX5_IB_UPD_XLT_ACCESS;
1519 err = mlx5_ib_update_xlt(mr, 0, npages, page_shift,
1520 upd_flags);
1521 } else {
1522 err = rereg_umr(pd, mr, access_flags, flags);
1523 }
1524
1525 if (err)
1526 goto err;
1527 }
1528
1529 set_mr_fields(dev, mr, npages, len, access_flags);
1530
1531 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1532 update_odp_mr(mr);
1533 #endif
1534 return 0;
1535
1536 err:
1537 if (mr->umem) {
1538 ib_umem_release(mr->umem);
1539 mr->umem = NULL;
1540 }
1541 clean_mr(dev, mr);
1542 return err;
1543 }
1544
1545 static int
1546 mlx5_alloc_priv_descs(struct ib_device *device,
1547 struct mlx5_ib_mr *mr,
1548 int ndescs,
1549 int desc_size)
1550 {
1551 int size = ndescs * desc_size;
1552 int add_size;
1553 int ret;
1554
1555 add_size = max_t(int, MLX5_UMR_ALIGN - ARCH_KMALLOC_MINALIGN, 0);
1556
1557 mr->descs_alloc = kzalloc(size + add_size, GFP_KERNEL);
1558 if (!mr->descs_alloc)
1559 return -ENOMEM;
1560
1561 mr->descs = PTR_ALIGN(mr->descs_alloc, MLX5_UMR_ALIGN);
1562
1563 mr->desc_map = dma_map_single(device->dev.parent, mr->descs,
1564 size, DMA_TO_DEVICE);
1565 if (dma_mapping_error(device->dev.parent, mr->desc_map)) {
1566 ret = -ENOMEM;
1567 goto err;
1568 }
1569
1570 return 0;
1571 err:
1572 kfree(mr->descs_alloc);
1573
1574 return ret;
1575 }
1576
1577 static void
1578 mlx5_free_priv_descs(struct mlx5_ib_mr *mr)
1579 {
1580 if (mr->descs) {
1581 struct ib_device *device = mr->ibmr.device;
1582 int size = mr->max_descs * mr->desc_size;
1583
1584 dma_unmap_single(device->dev.parent, mr->desc_map,
1585 size, DMA_TO_DEVICE);
1586 kfree(mr->descs_alloc);
1587 mr->descs = NULL;
1588 }
1589 }
1590
1591 static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
1592 {
1593 int allocated_from_cache = mr->allocated_from_cache;
1594
1595 if (mr->sig) {
1596 if (mlx5_core_destroy_psv(dev->mdev,
1597 mr->sig->psv_memory.psv_idx))
1598 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",
1599 mr->sig->psv_memory.psv_idx);
1600 if (mlx5_core_destroy_psv(dev->mdev,
1601 mr->sig->psv_wire.psv_idx))
1602 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",
1603 mr->sig->psv_wire.psv_idx);
1604 kfree(mr->sig);
1605 mr->sig = NULL;
1606 }
1607
1608 mlx5_free_priv_descs(mr);
1609
1610 if (!allocated_from_cache)
1611 destroy_mkey(dev, mr);
1612 }
1613
1614 static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
1615 {
1616 int npages = mr->npages;
1617 struct ib_umem *umem = mr->umem;
1618
1619 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1620 if (umem && umem->is_odp) {
1621 struct ib_umem_odp *umem_odp = to_ib_umem_odp(umem);
1622
1623 /* Prevent new page faults from succeeding */
1624 mr->live = 0;
1625 /* Wait for all running page-fault handlers to finish. */
1626 synchronize_srcu(&dev->mr_srcu);
1627 /* Destroy all page mappings */
1628 if (umem_odp->page_list)
1629 mlx5_ib_invalidate_range(umem_odp, ib_umem_start(umem),
1630 ib_umem_end(umem));
1631 else
1632 mlx5_ib_free_implicit_mr(mr);
1633 /*
1634 * We kill the umem before the MR for ODP,
1635 * so that there will not be any invalidations in
1636 * flight, looking at the *mr struct.
1637 */
1638 ib_umem_release(umem);
1639 atomic_sub(npages, &dev->mdev->priv.reg_pages);
1640
1641 /* Avoid double-freeing the umem. */
1642 umem = NULL;
1643 }
1644 #endif
1645 clean_mr(dev, mr);
1646
1647 /*
1648 * We should unregister the DMA address from the HCA before
1649 * remove the DMA mapping.
1650 */
1651 mlx5_mr_cache_free(dev, mr);
1652 if (umem) {
1653 ib_umem_release(umem);
1654 atomic_sub(npages, &dev->mdev->priv.reg_pages);
1655 }
1656 if (!mr->allocated_from_cache)
1657 kfree(mr);
1658 }
1659
1660 int mlx5_ib_dereg_mr(struct ib_mr *ibmr)
1661 {
1662 dereg_mr(to_mdev(ibmr->device), to_mmr(ibmr));
1663 return 0;
1664 }
1665
1666 struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
1667 enum ib_mr_type mr_type,
1668 u32 max_num_sg)
1669 {
1670 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1671 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1672 int ndescs = ALIGN(max_num_sg, 4);
1673 struct mlx5_ib_mr *mr;
1674 void *mkc;
1675 u32 *in;
1676 int err;
1677
1678 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1679 if (!mr)
1680 return ERR_PTR(-ENOMEM);
1681
1682 in = kzalloc(inlen, GFP_KERNEL);
1683 if (!in) {
1684 err = -ENOMEM;
1685 goto err_free;
1686 }
1687
1688 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1689 MLX5_SET(mkc, mkc, free, 1);
1690 MLX5_SET(mkc, mkc, translations_octword_size, ndescs);
1691 MLX5_SET(mkc, mkc, qpn, 0xffffff);
1692 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
1693
1694 if (mr_type == IB_MR_TYPE_MEM_REG) {
1695 mr->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
1696 MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
1697 err = mlx5_alloc_priv_descs(pd->device, mr,
1698 ndescs, sizeof(struct mlx5_mtt));
1699 if (err)
1700 goto err_free_in;
1701
1702 mr->desc_size = sizeof(struct mlx5_mtt);
1703 mr->max_descs = ndescs;
1704 } else if (mr_type == IB_MR_TYPE_SG_GAPS) {
1705 mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS;
1706
1707 err = mlx5_alloc_priv_descs(pd->device, mr,
1708 ndescs, sizeof(struct mlx5_klm));
1709 if (err)
1710 goto err_free_in;
1711 mr->desc_size = sizeof(struct mlx5_klm);
1712 mr->max_descs = ndescs;
1713 } else if (mr_type == IB_MR_TYPE_SIGNATURE) {
1714 u32 psv_index[2];
1715
1716 MLX5_SET(mkc, mkc, bsf_en, 1);
1717 MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE);
1718 mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL);
1719 if (!mr->sig) {
1720 err = -ENOMEM;
1721 goto err_free_in;
1722 }
1723
1724 /* create mem & wire PSVs */
1725 err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn,
1726 2, psv_index);
1727 if (err)
1728 goto err_free_sig;
1729
1730 mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS;
1731 mr->sig->psv_memory.psv_idx = psv_index[0];
1732 mr->sig->psv_wire.psv_idx = psv_index[1];
1733
1734 mr->sig->sig_status_checked = true;
1735 mr->sig->sig_err_exists = false;
1736 /* Next UMR, Arm SIGERR */
1737 ++mr->sig->sigerr_count;
1738 } else {
1739 mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type);
1740 err = -EINVAL;
1741 goto err_free_in;
1742 }
1743
1744 MLX5_SET(mkc, mkc, access_mode_1_0, mr->access_mode & 0x3);
1745 MLX5_SET(mkc, mkc, access_mode_4_2, (mr->access_mode >> 2) & 0x7);
1746 MLX5_SET(mkc, mkc, umr_en, 1);
1747
1748 mr->ibmr.device = pd->device;
1749 err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen);
1750 if (err)
1751 goto err_destroy_psv;
1752
1753 mr->mmkey.type = MLX5_MKEY_MR;
1754 mr->ibmr.lkey = mr->mmkey.key;
1755 mr->ibmr.rkey = mr->mmkey.key;
1756 mr->umem = NULL;
1757 kfree(in);
1758
1759 return &mr->ibmr;
1760
1761 err_destroy_psv:
1762 if (mr->sig) {
1763 if (mlx5_core_destroy_psv(dev->mdev,
1764 mr->sig->psv_memory.psv_idx))
1765 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",
1766 mr->sig->psv_memory.psv_idx);
1767 if (mlx5_core_destroy_psv(dev->mdev,
1768 mr->sig->psv_wire.psv_idx))
1769 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",
1770 mr->sig->psv_wire.psv_idx);
1771 }
1772 mlx5_free_priv_descs(mr);
1773 err_free_sig:
1774 kfree(mr->sig);
1775 err_free_in:
1776 kfree(in);
1777 err_free:
1778 kfree(mr);
1779 return ERR_PTR(err);
1780 }
1781
1782 struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
1783 struct ib_udata *udata)
1784 {
1785 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1786 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1787 struct mlx5_ib_mw *mw = NULL;
1788 u32 *in = NULL;
1789 void *mkc;
1790 int ndescs;
1791 int err;
1792 struct mlx5_ib_alloc_mw req = {};
1793 struct {
1794 __u32 comp_mask;
1795 __u32 response_length;
1796 } resp = {};
1797
1798 err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req)));
1799 if (err)
1800 return ERR_PTR(err);
1801
1802 if (req.comp_mask || req.reserved1 || req.reserved2)
1803 return ERR_PTR(-EOPNOTSUPP);
1804
1805 if (udata->inlen > sizeof(req) &&
1806 !ib_is_udata_cleared(udata, sizeof(req),
1807 udata->inlen - sizeof(req)))
1808 return ERR_PTR(-EOPNOTSUPP);
1809
1810 ndescs = req.num_klms ? roundup(req.num_klms, 4) : roundup(1, 4);
1811
1812 mw = kzalloc(sizeof(*mw), GFP_KERNEL);
1813 in = kzalloc(inlen, GFP_KERNEL);
1814 if (!mw || !in) {
1815 err = -ENOMEM;
1816 goto free;
1817 }
1818
1819 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1820
1821 MLX5_SET(mkc, mkc, free, 1);
1822 MLX5_SET(mkc, mkc, translations_octword_size, ndescs);
1823 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
1824 MLX5_SET(mkc, mkc, umr_en, 1);
1825 MLX5_SET(mkc, mkc, lr, 1);
1826 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS);
1827 MLX5_SET(mkc, mkc, en_rinval, !!((type == IB_MW_TYPE_2)));
1828 MLX5_SET(mkc, mkc, qpn, 0xffffff);
1829
1830 err = mlx5_core_create_mkey(dev->mdev, &mw->mmkey, in, inlen);
1831 if (err)
1832 goto free;
1833
1834 mw->mmkey.type = MLX5_MKEY_MW;
1835 mw->ibmw.rkey = mw->mmkey.key;
1836 mw->ndescs = ndescs;
1837
1838 resp.response_length = min(offsetof(typeof(resp), response_length) +
1839 sizeof(resp.response_length), udata->outlen);
1840 if (resp.response_length) {
1841 err = ib_copy_to_udata(udata, &resp, resp.response_length);
1842 if (err) {
1843 mlx5_core_destroy_mkey(dev->mdev, &mw->mmkey);
1844 goto free;
1845 }
1846 }
1847
1848 kfree(in);
1849 return &mw->ibmw;
1850
1851 free:
1852 kfree(mw);
1853 kfree(in);
1854 return ERR_PTR(err);
1855 }
1856
1857 int mlx5_ib_dealloc_mw(struct ib_mw *mw)
1858 {
1859 struct mlx5_ib_mw *mmw = to_mmw(mw);
1860 int err;
1861
1862 err = mlx5_core_destroy_mkey((to_mdev(mw->device))->mdev,
1863 &mmw->mmkey);
1864 if (!err)
1865 kfree(mmw);
1866 return err;
1867 }
1868
1869 int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
1870 struct ib_mr_status *mr_status)
1871 {
1872 struct mlx5_ib_mr *mmr = to_mmr(ibmr);
1873 int ret = 0;
1874
1875 if (check_mask & ~IB_MR_CHECK_SIG_STATUS) {
1876 pr_err("Invalid status check mask\n");
1877 ret = -EINVAL;
1878 goto done;
1879 }
1880
1881 mr_status->fail_status = 0;
1882 if (check_mask & IB_MR_CHECK_SIG_STATUS) {
1883 if (!mmr->sig) {
1884 ret = -EINVAL;
1885 pr_err("signature status check requested on a non-signature enabled MR\n");
1886 goto done;
1887 }
1888
1889 mmr->sig->sig_status_checked = true;
1890 if (!mmr->sig->sig_err_exists)
1891 goto done;
1892
1893 if (ibmr->lkey == mmr->sig->err_item.key)
1894 memcpy(&mr_status->sig_err, &mmr->sig->err_item,
1895 sizeof(mr_status->sig_err));
1896 else {
1897 mr_status->sig_err.err_type = IB_SIG_BAD_GUARD;
1898 mr_status->sig_err.sig_err_offset = 0;
1899 mr_status->sig_err.key = mmr->sig->err_item.key;
1900 }
1901
1902 mmr->sig->sig_err_exists = false;
1903 mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS;
1904 }
1905
1906 done:
1907 return ret;
1908 }
1909
1910 static int
1911 mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr,
1912 struct scatterlist *sgl,
1913 unsigned short sg_nents,
1914 unsigned int *sg_offset_p)
1915 {
1916 struct scatterlist *sg = sgl;
1917 struct mlx5_klm *klms = mr->descs;
1918 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;
1919 u32 lkey = mr->ibmr.pd->local_dma_lkey;
1920 int i;
1921
1922 mr->ibmr.iova = sg_dma_address(sg) + sg_offset;
1923 mr->ibmr.length = 0;
1924
1925 for_each_sg(sgl, sg, sg_nents, i) {
1926 if (unlikely(i >= mr->max_descs))
1927 break;
1928 klms[i].va = cpu_to_be64(sg_dma_address(sg) + sg_offset);
1929 klms[i].bcount = cpu_to_be32(sg_dma_len(sg) - sg_offset);
1930 klms[i].key = cpu_to_be32(lkey);
1931 mr->ibmr.length += sg_dma_len(sg) - sg_offset;
1932
1933 sg_offset = 0;
1934 }
1935 mr->ndescs = i;
1936
1937 if (sg_offset_p)
1938 *sg_offset_p = sg_offset;
1939
1940 return i;
1941 }
1942
1943 static int mlx5_set_page(struct ib_mr *ibmr, u64 addr)
1944 {
1945 struct mlx5_ib_mr *mr = to_mmr(ibmr);
1946 __be64 *descs;
1947
1948 if (unlikely(mr->ndescs == mr->max_descs))
1949 return -ENOMEM;
1950
1951 descs = mr->descs;
1952 descs[mr->ndescs++] = cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR);
1953
1954 return 0;
1955 }
1956
1957 int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
1958 unsigned int *sg_offset)
1959 {
1960 struct mlx5_ib_mr *mr = to_mmr(ibmr);
1961 int n;
1962
1963 mr->ndescs = 0;
1964
1965 ib_dma_sync_single_for_cpu(ibmr->device, mr->desc_map,
1966 mr->desc_size * mr->max_descs,
1967 DMA_TO_DEVICE);
1968
1969 if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
1970 n = mlx5_ib_sg_to_klms(mr, sg, sg_nents, sg_offset);
1971 else
1972 n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset,
1973 mlx5_set_page);
1974
1975 ib_dma_sync_single_for_device(ibmr->device, mr->desc_map,
1976 mr->desc_size * mr->max_descs,
1977 DMA_TO_DEVICE);
1978
1979 return n;
1980 }
Ubuntu combined Linux kernel mirror