]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - drivers/gpu/drm/ttm/ttm_bo.c
projects / mirror_ubuntu-jammy-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
drm/ttm: replace last move_notify with delete_mem_notify
[mirror_ubuntu-jammy-kernel.git] / drivers / gpu / drm / ttm / ttm_bo.c
1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
2 /**************************************************************************
3 *
4 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
5 * All Rights Reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
25 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
27 **************************************************************************/
28 /*
29 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
30 */
31
32 #define pr_fmt(fmt) "[TTM] " fmt
33
34 #include <drm/ttm/ttm_module.h>
35 #include <drm/ttm/ttm_bo_driver.h>
36 #include <drm/ttm/ttm_placement.h>
37 #include <linux/jiffies.h>
38 #include <linux/slab.h>
39 #include <linux/sched.h>
40 #include <linux/mm.h>
41 #include <linux/file.h>
42 #include <linux/module.h>
43 #include <linux/atomic.h>
44 #include <linux/dma-resv.h>
45
46 static void ttm_bo_global_kobj_release(struct kobject *kobj);
47
48 /**
49 * ttm_global_mutex - protecting the global BO state
50 */
51 DEFINE_MUTEX(ttm_global_mutex);
52 unsigned ttm_bo_glob_use_count;
53 struct ttm_bo_global ttm_bo_glob;
54 EXPORT_SYMBOL(ttm_bo_glob);
55
56 static struct attribute ttm_bo_count = {
57 .name = "bo_count",
58 .mode = S_IRUGO
59 };
60
61 /* default destructor */
62 static void ttm_bo_default_destroy(struct ttm_buffer_object *bo)
63 {
64 kfree(bo);
65 }
66
67 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
68 struct ttm_placement *placement)
69 {
70 struct drm_printer p = drm_debug_printer(TTM_PFX);
71 struct ttm_resource_manager *man;
72 int i, mem_type;
73
74 drm_printf(&p, "No space for %p (%lu pages, %luK, %luM)\n",
75 bo, bo->mem.num_pages, bo->mem.size >> 10,
76 bo->mem.size >> 20);
77 for (i = 0; i < placement->num_placement; i++) {
78 mem_type = placement->placement[i].mem_type;
79 drm_printf(&p, " placement[%d]=0x%08X (%d)\n",
80 i, placement->placement[i].flags, mem_type);
81 man = ttm_manager_type(bo->bdev, mem_type);
82 ttm_resource_manager_debug(man, &p);
83 }
84 }
85
86 static ssize_t ttm_bo_global_show(struct kobject *kobj,
87 struct attribute *attr,
88 char *buffer)
89 {
90 struct ttm_bo_global *glob =
91 container_of(kobj, struct ttm_bo_global, kobj);
92
93 return snprintf(buffer, PAGE_SIZE, "%d\n",
94 atomic_read(&glob->bo_count));
95 }
96
97 static struct attribute *ttm_bo_global_attrs[] = {
98 &ttm_bo_count,
99 NULL
100 };
101
102 static const struct sysfs_ops ttm_bo_global_ops = {
103 .show = &ttm_bo_global_show
104 };
105
106 static struct kobj_type ttm_bo_glob_kobj_type = {
107 .release = &ttm_bo_global_kobj_release,
108 .sysfs_ops = &ttm_bo_global_ops,
109 .default_attrs = ttm_bo_global_attrs
110 };
111
112 static void ttm_bo_add_mem_to_lru(struct ttm_buffer_object *bo,
113 struct ttm_resource *mem)
114 {
115 struct ttm_bo_device *bdev = bo->bdev;
116 struct ttm_resource_manager *man;
117
118 if (!list_empty(&bo->lru) || bo->pin_count)
119 return;
120
121 man = ttm_manager_type(bdev, mem->mem_type);
122 list_add_tail(&bo->lru, &man->lru[bo->priority]);
123
124 if (man->use_tt && bo->ttm &&
125 !(bo->ttm->page_flags & (TTM_PAGE_FLAG_SG |
126 TTM_PAGE_FLAG_SWAPPED))) {
127 list_add_tail(&bo->swap, &ttm_bo_glob.swap_lru[bo->priority]);
128 }
129 }
130
131 static void ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
132 {
133 struct ttm_bo_device *bdev = bo->bdev;
134 bool notify = false;
135
136 if (!list_empty(&bo->swap)) {
137 list_del_init(&bo->swap);
138 notify = true;
139 }
140 if (!list_empty(&bo->lru)) {
141 list_del_init(&bo->lru);
142 notify = true;
143 }
144
145 if (notify && bdev->driver->del_from_lru_notify)
146 bdev->driver->del_from_lru_notify(bo);
147 }
148
149 static void ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos,
150 struct ttm_buffer_object *bo)
151 {
152 if (!pos->first)
153 pos->first = bo;
154 pos->last = bo;
155 }
156
157 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo,
158 struct ttm_lru_bulk_move *bulk)
159 {
160 dma_resv_assert_held(bo->base.resv);
161
162 ttm_bo_del_from_lru(bo);
163 ttm_bo_add_mem_to_lru(bo, &bo->mem);
164
165 if (bulk && !bo->pin_count) {
166 switch (bo->mem.mem_type) {
167 case TTM_PL_TT:
168 ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo);
169 break;
170
171 case TTM_PL_VRAM:
172 ttm_bo_bulk_move_set_pos(&bulk->vram[bo->priority], bo);
173 break;
174 }
175 if (bo->ttm && !(bo->ttm->page_flags &
176 (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED)))
177 ttm_bo_bulk_move_set_pos(&bulk->swap[bo->priority], bo);
178 }
179 }
180 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
181
182 void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk)
183 {
184 unsigned i;
185
186 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
187 struct ttm_lru_bulk_move_pos *pos = &bulk->tt[i];
188 struct ttm_resource_manager *man;
189
190 if (!pos->first)
191 continue;
192
193 dma_resv_assert_held(pos->first->base.resv);
194 dma_resv_assert_held(pos->last->base.resv);
195
196 man = ttm_manager_type(pos->first->bdev, TTM_PL_TT);
197 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
198 &pos->last->lru);
199 }
200
201 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
202 struct ttm_lru_bulk_move_pos *pos = &bulk->vram[i];
203 struct ttm_resource_manager *man;
204
205 if (!pos->first)
206 continue;
207
208 dma_resv_assert_held(pos->first->base.resv);
209 dma_resv_assert_held(pos->last->base.resv);
210
211 man = ttm_manager_type(pos->first->bdev, TTM_PL_VRAM);
212 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
213 &pos->last->lru);
214 }
215
216 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
217 struct ttm_lru_bulk_move_pos *pos = &bulk->swap[i];
218 struct list_head *lru;
219
220 if (!pos->first)
221 continue;
222
223 dma_resv_assert_held(pos->first->base.resv);
224 dma_resv_assert_held(pos->last->base.resv);
225
226 lru = &ttm_bo_glob.swap_lru[i];
227 list_bulk_move_tail(lru, &pos->first->swap, &pos->last->swap);
228 }
229 }
230 EXPORT_SYMBOL(ttm_bo_bulk_move_lru_tail);
231
232 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
233 struct ttm_resource *mem, bool evict,
234 struct ttm_operation_ctx *ctx)
235 {
236 struct ttm_bo_device *bdev = bo->bdev;
237 struct ttm_resource_manager *old_man = ttm_manager_type(bdev, bo->mem.mem_type);
238 struct ttm_resource_manager *new_man = ttm_manager_type(bdev, mem->mem_type);
239 int ret;
240
241 ttm_bo_unmap_virtual(bo);
242
243 /*
244 * Create and bind a ttm if required.
245 */
246
247 if (new_man->use_tt) {
248 /* Zero init the new TTM structure if the old location should
249 * have used one as well.
250 */
251 ret = ttm_tt_create(bo, old_man->use_tt);
252 if (ret)
253 goto out_err;
254
255 if (mem->mem_type != TTM_PL_SYSTEM) {
256 ret = ttm_tt_populate(bo->bdev, bo->ttm, ctx);
257 if (ret)
258 goto out_err;
259 }
260 }
261
262 ret = bdev->driver->move(bo, evict, ctx, mem);
263 if (ret)
264 goto out_err;
265
266 ctx->bytes_moved += bo->num_pages << PAGE_SHIFT;
267 return 0;
268
269 out_err:
270 new_man = ttm_manager_type(bdev, bo->mem.mem_type);
271 if (!new_man->use_tt)
272 ttm_bo_tt_destroy(bo);
273
274 return ret;
275 }
276
277 /**
278 * Call bo::reserved.
279 * Will release GPU memory type usage on destruction.
280 * This is the place to put in driver specific hooks to release
281 * driver private resources.
282 * Will release the bo::reserved lock.
283 */
284
285 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
286 {
287 if (bo->bdev->driver->delete_mem_notify)
288 bo->bdev->driver->delete_mem_notify(bo);
289
290 ttm_bo_tt_destroy(bo);
291 ttm_resource_free(bo, &bo->mem);
292 }
293
294 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
295 {
296 int r;
297
298 if (bo->base.resv == &bo->base._resv)
299 return 0;
300
301 BUG_ON(!dma_resv_trylock(&bo->base._resv));
302
303 r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv);
304 dma_resv_unlock(&bo->base._resv);
305 if (r)
306 return r;
307
308 if (bo->type != ttm_bo_type_sg) {
309 /* This works because the BO is about to be destroyed and nobody
310 * reference it any more. The only tricky case is the trylock on
311 * the resv object while holding the lru_lock.
312 */
313 spin_lock(&ttm_bo_glob.lru_lock);
314 bo->base.resv = &bo->base._resv;
315 spin_unlock(&ttm_bo_glob.lru_lock);
316 }
317
318 return r;
319 }
320
321 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
322 {
323 struct dma_resv *resv = &bo->base._resv;
324 struct dma_resv_list *fobj;
325 struct dma_fence *fence;
326 int i;
327
328 rcu_read_lock();
329 fobj = rcu_dereference(resv->fence);
330 fence = rcu_dereference(resv->fence_excl);
331 if (fence && !fence->ops->signaled)
332 dma_fence_enable_sw_signaling(fence);
333
334 for (i = 0; fobj && i < fobj->shared_count; ++i) {
335 fence = rcu_dereference(fobj->shared[i]);
336
337 if (!fence->ops->signaled)
338 dma_fence_enable_sw_signaling(fence);
339 }
340 rcu_read_unlock();
341 }
342
343 /**
344 * function ttm_bo_cleanup_refs
345 * If bo idle, remove from lru lists, and unref.
346 * If not idle, block if possible.
347 *
348 * Must be called with lru_lock and reservation held, this function
349 * will drop the lru lock and optionally the reservation lock before returning.
350 *
351 * @interruptible Any sleeps should occur interruptibly.
352 * @no_wait_gpu Never wait for gpu. Return -EBUSY instead.
353 * @unlock_resv Unlock the reservation lock as well.
354 */
355
356 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
357 bool interruptible, bool no_wait_gpu,
358 bool unlock_resv)
359 {
360 struct dma_resv *resv = &bo->base._resv;
361 int ret;
362
363 if (dma_resv_test_signaled_rcu(resv, true))
364 ret = 0;
365 else
366 ret = -EBUSY;
367
368 if (ret && !no_wait_gpu) {
369 long lret;
370
371 if (unlock_resv)
372 dma_resv_unlock(bo->base.resv);
373 spin_unlock(&ttm_bo_glob.lru_lock);
374
375 lret = dma_resv_wait_timeout_rcu(resv, true, interruptible,
376 30 * HZ);
377
378 if (lret < 0)
379 return lret;
380 else if (lret == 0)
381 return -EBUSY;
382
383 spin_lock(&ttm_bo_glob.lru_lock);
384 if (unlock_resv && !dma_resv_trylock(bo->base.resv)) {
385 /*
386 * We raced, and lost, someone else holds the reservation now,
387 * and is probably busy in ttm_bo_cleanup_memtype_use.
388 *
389 * Even if it's not the case, because we finished waiting any
390 * delayed destruction would succeed, so just return success
391 * here.
392 */
393 spin_unlock(&ttm_bo_glob.lru_lock);
394 return 0;
395 }
396 ret = 0;
397 }
398
399 if (ret || unlikely(list_empty(&bo->ddestroy))) {
400 if (unlock_resv)
401 dma_resv_unlock(bo->base.resv);
402 spin_unlock(&ttm_bo_glob.lru_lock);
403 return ret;
404 }
405
406 ttm_bo_del_from_lru(bo);
407 list_del_init(&bo->ddestroy);
408 spin_unlock(&ttm_bo_glob.lru_lock);
409 ttm_bo_cleanup_memtype_use(bo);
410
411 if (unlock_resv)
412 dma_resv_unlock(bo->base.resv);
413
414 ttm_bo_put(bo);
415
416 return 0;
417 }
418
419 /**
420 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
421 * encountered buffers.
422 */
423 static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
424 {
425 struct ttm_bo_global *glob = &ttm_bo_glob;
426 struct list_head removed;
427 bool empty;
428
429 INIT_LIST_HEAD(&removed);
430
431 spin_lock(&glob->lru_lock);
432 while (!list_empty(&bdev->ddestroy)) {
433 struct ttm_buffer_object *bo;
434
435 bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object,
436 ddestroy);
437 list_move_tail(&bo->ddestroy, &removed);
438 if (!ttm_bo_get_unless_zero(bo))
439 continue;
440
441 if (remove_all || bo->base.resv != &bo->base._resv) {
442 spin_unlock(&glob->lru_lock);
443 dma_resv_lock(bo->base.resv, NULL);
444
445 spin_lock(&glob->lru_lock);
446 ttm_bo_cleanup_refs(bo, false, !remove_all, true);
447
448 } else if (dma_resv_trylock(bo->base.resv)) {
449 ttm_bo_cleanup_refs(bo, false, !remove_all, true);
450 } else {
451 spin_unlock(&glob->lru_lock);
452 }
453
454 ttm_bo_put(bo);
455 spin_lock(&glob->lru_lock);
456 }
457 list_splice_tail(&removed, &bdev->ddestroy);
458 empty = list_empty(&bdev->ddestroy);
459 spin_unlock(&glob->lru_lock);
460
461 return empty;
462 }
463
464 static void ttm_bo_delayed_workqueue(struct work_struct *work)
465 {
466 struct ttm_bo_device *bdev =
467 container_of(work, struct ttm_bo_device, wq.work);
468
469 if (!ttm_bo_delayed_delete(bdev, false))
470 schedule_delayed_work(&bdev->wq,
471 ((HZ / 100) < 1) ? 1 : HZ / 100);
472 }
473
474 static void ttm_bo_release(struct kref *kref)
475 {
476 struct ttm_buffer_object *bo =
477 container_of(kref, struct ttm_buffer_object, kref);
478 struct ttm_bo_device *bdev = bo->bdev;
479 size_t acc_size = bo->acc_size;
480 int ret;
481
482 if (!bo->deleted) {
483 ret = ttm_bo_individualize_resv(bo);
484 if (ret) {
485 /* Last resort, if we fail to allocate memory for the
486 * fences block for the BO to become idle
487 */
488 dma_resv_wait_timeout_rcu(bo->base.resv, true, false,
489 30 * HZ);
490 }
491
492 if (bo->bdev->driver->release_notify)
493 bo->bdev->driver->release_notify(bo);
494
495 drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node);
496 ttm_mem_io_free(bdev, &bo->mem);
497 }
498
499 if (!dma_resv_test_signaled_rcu(bo->base.resv, true) ||
500 !dma_resv_trylock(bo->base.resv)) {
501 /* The BO is not idle, resurrect it for delayed destroy */
502 ttm_bo_flush_all_fences(bo);
503 bo->deleted = true;
504
505 spin_lock(&ttm_bo_glob.lru_lock);
506
507 /*
508 * Make pinned bos immediately available to
509 * shrinkers, now that they are queued for
510 * destruction.
511 */
512 if (bo->pin_count) {
513 bo->pin_count = 0;
514 ttm_bo_del_from_lru(bo);
515 ttm_bo_add_mem_to_lru(bo, &bo->mem);
516 }
517
518 kref_init(&bo->kref);
519 list_add_tail(&bo->ddestroy, &bdev->ddestroy);
520 spin_unlock(&ttm_bo_glob.lru_lock);
521
522 schedule_delayed_work(&bdev->wq,
523 ((HZ / 100) < 1) ? 1 : HZ / 100);
524 return;
525 }
526
527 spin_lock(&ttm_bo_glob.lru_lock);
528 ttm_bo_del_from_lru(bo);
529 list_del(&bo->ddestroy);
530 spin_unlock(&ttm_bo_glob.lru_lock);
531
532 ttm_bo_cleanup_memtype_use(bo);
533 dma_resv_unlock(bo->base.resv);
534
535 atomic_dec(&ttm_bo_glob.bo_count);
536 dma_fence_put(bo->moving);
537 if (!ttm_bo_uses_embedded_gem_object(bo))
538 dma_resv_fini(&bo->base._resv);
539 bo->destroy(bo);
540 ttm_mem_global_free(&ttm_mem_glob, acc_size);
541 }
542
543 void ttm_bo_put(struct ttm_buffer_object *bo)
544 {
545 kref_put(&bo->kref, ttm_bo_release);
546 }
547 EXPORT_SYMBOL(ttm_bo_put);
548
549 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
550 {
551 return cancel_delayed_work_sync(&bdev->wq);
552 }
553 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);
554
555 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
556 {
557 if (resched)
558 schedule_delayed_work(&bdev->wq,
559 ((HZ / 100) < 1) ? 1 : HZ / 100);
560 }
561 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
562
563 static int ttm_bo_evict(struct ttm_buffer_object *bo,
564 struct ttm_operation_ctx *ctx)
565 {
566 struct ttm_bo_device *bdev = bo->bdev;
567 struct ttm_resource evict_mem;
568 struct ttm_placement placement;
569 int ret = 0;
570
571 dma_resv_assert_held(bo->base.resv);
572
573 placement.num_placement = 0;
574 placement.num_busy_placement = 0;
575 bdev->driver->evict_flags(bo, &placement);
576
577 if (!placement.num_placement && !placement.num_busy_placement) {
578 ttm_bo_wait(bo, false, false);
579
580 ttm_bo_cleanup_memtype_use(bo);
581 return ttm_tt_create(bo, false);
582 }
583
584 evict_mem = bo->mem;
585 evict_mem.mm_node = NULL;
586 evict_mem.bus.offset = 0;
587 evict_mem.bus.addr = NULL;
588
589 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
590 if (ret) {
591 if (ret != -ERESTARTSYS) {
592 pr_err("Failed to find memory space for buffer 0x%p eviction\n",
593 bo);
594 ttm_bo_mem_space_debug(bo, &placement);
595 }
596 goto out;
597 }
598
599 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx);
600 if (unlikely(ret)) {
601 if (ret != -ERESTARTSYS)
602 pr_err("Buffer eviction failed\n");
603 ttm_resource_free(bo, &evict_mem);
604 }
605 out:
606 return ret;
607 }
608
609 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
610 const struct ttm_place *place)
611 {
612 /* Don't evict this BO if it's outside of the
613 * requested placement range
614 */
615 if (place->fpfn >= (bo->mem.start + bo->mem.size) ||
616 (place->lpfn && place->lpfn <= bo->mem.start))
617 return false;
618
619 return true;
620 }
621 EXPORT_SYMBOL(ttm_bo_eviction_valuable);
622
623 /**
624 * Check the target bo is allowable to be evicted or swapout, including cases:
625 *
626 * a. if share same reservation object with ctx->resv, have assumption
627 * reservation objects should already be locked, so not lock again and
628 * return true directly when either the opreation allow_reserved_eviction
629 * or the target bo already is in delayed free list;
630 *
631 * b. Otherwise, trylock it.
632 */
633 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
634 struct ttm_operation_ctx *ctx, bool *locked, bool *busy)
635 {
636 bool ret = false;
637
638 if (bo->base.resv == ctx->resv) {
639 dma_resv_assert_held(bo->base.resv);
640 if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT)
641 ret = true;
642 *locked = false;
643 if (busy)
644 *busy = false;
645 } else {
646 ret = dma_resv_trylock(bo->base.resv);
647 *locked = ret;
648 if (busy)
649 *busy = !ret;
650 }
651
652 return ret;
653 }
654
655 /**
656 * ttm_mem_evict_wait_busy - wait for a busy BO to become available
657 *
658 * @busy_bo: BO which couldn't be locked with trylock
659 * @ctx: operation context
660 * @ticket: acquire ticket
661 *
662 * Try to lock a busy buffer object to avoid failing eviction.
663 */
664 static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo,
665 struct ttm_operation_ctx *ctx,
666 struct ww_acquire_ctx *ticket)
667 {
668 int r;
669
670 if (!busy_bo || !ticket)
671 return -EBUSY;
672
673 if (ctx->interruptible)
674 r = dma_resv_lock_interruptible(busy_bo->base.resv,
675 ticket);
676 else
677 r = dma_resv_lock(busy_bo->base.resv, ticket);
678
679 /*
680 * TODO: It would be better to keep the BO locked until allocation is at
681 * least tried one more time, but that would mean a much larger rework
682 * of TTM.
683 */
684 if (!r)
685 dma_resv_unlock(busy_bo->base.resv);
686
687 return r == -EDEADLK ? -EBUSY : r;
688 }
689
690 int ttm_mem_evict_first(struct ttm_bo_device *bdev,
691 struct ttm_resource_manager *man,
692 const struct ttm_place *place,
693 struct ttm_operation_ctx *ctx,
694 struct ww_acquire_ctx *ticket)
695 {
696 struct ttm_buffer_object *bo = NULL, *busy_bo = NULL;
697 bool locked = false;
698 unsigned i;
699 int ret;
700
701 spin_lock(&ttm_bo_glob.lru_lock);
702 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
703 list_for_each_entry(bo, &man->lru[i], lru) {
704 bool busy;
705
706 if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked,
707 &busy)) {
708 if (busy && !busy_bo && ticket !=
709 dma_resv_locking_ctx(bo->base.resv))
710 busy_bo = bo;
711 continue;
712 }
713
714 if (place && !bdev->driver->eviction_valuable(bo,
715 place)) {
716 if (locked)
717 dma_resv_unlock(bo->base.resv);
718 continue;
719 }
720 if (!ttm_bo_get_unless_zero(bo)) {
721 if (locked)
722 dma_resv_unlock(bo->base.resv);
723 continue;
724 }
725 break;
726 }
727
728 /* If the inner loop terminated early, we have our candidate */
729 if (&bo->lru != &man->lru[i])
730 break;
731
732 bo = NULL;
733 }
734
735 if (!bo) {
736 if (busy_bo && !ttm_bo_get_unless_zero(busy_bo))
737 busy_bo = NULL;
738 spin_unlock(&ttm_bo_glob.lru_lock);
739 ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket);
740 if (busy_bo)
741 ttm_bo_put(busy_bo);
742 return ret;
743 }
744
745 if (bo->deleted) {
746 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
747 ctx->no_wait_gpu, locked);
748 ttm_bo_put(bo);
749 return ret;
750 }
751
752 spin_unlock(&ttm_bo_glob.lru_lock);
753
754 ret = ttm_bo_evict(bo, ctx);
755 if (locked)
756 ttm_bo_unreserve(bo);
757
758 ttm_bo_put(bo);
759 return ret;
760 }
761
762 /**
763 * Add the last move fence to the BO and reserve a new shared slot.
764 */
765 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
766 struct ttm_resource_manager *man,
767 struct ttm_resource *mem,
768 bool no_wait_gpu)
769 {
770 struct dma_fence *fence;
771 int ret;
772
773 spin_lock(&man->move_lock);
774 fence = dma_fence_get(man->move);
775 spin_unlock(&man->move_lock);
776
777 if (!fence)
778 return 0;
779
780 if (no_wait_gpu) {
781 dma_fence_put(fence);
782 return -EBUSY;
783 }
784
785 dma_resv_add_shared_fence(bo->base.resv, fence);
786
787 ret = dma_resv_reserve_shared(bo->base.resv, 1);
788 if (unlikely(ret)) {
789 dma_fence_put(fence);
790 return ret;
791 }
792
793 dma_fence_put(bo->moving);
794 bo->moving = fence;
795 return 0;
796 }
797
798 /**
799 * Repeatedly evict memory from the LRU for @mem_type until we create enough
800 * space, or we've evicted everything and there isn't enough space.
801 */
802 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
803 const struct ttm_place *place,
804 struct ttm_resource *mem,
805 struct ttm_operation_ctx *ctx)
806 {
807 struct ttm_bo_device *bdev = bo->bdev;
808 struct ttm_resource_manager *man = ttm_manager_type(bdev, mem->mem_type);
809 struct ww_acquire_ctx *ticket;
810 int ret;
811
812 ticket = dma_resv_locking_ctx(bo->base.resv);
813 do {
814 ret = ttm_resource_alloc(bo, place, mem);
815 if (likely(!ret))
816 break;
817 if (unlikely(ret != -ENOSPC))
818 return ret;
819 ret = ttm_mem_evict_first(bdev, man, place, ctx,
820 ticket);
821 if (unlikely(ret != 0))
822 return ret;
823 } while (1);
824
825 return ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu);
826 }
827
828 /**
829 * ttm_bo_mem_placement - check if placement is compatible
830 * @bo: BO to find memory for
831 * @place: where to search
832 * @mem: the memory object to fill in
833 *
834 * Check if placement is compatible and fill in mem structure.
835 * Returns -EBUSY if placement won't work or negative error code.
836 * 0 when placement can be used.
837 */
838 static int ttm_bo_mem_placement(struct ttm_buffer_object *bo,
839 const struct ttm_place *place,
840 struct ttm_resource *mem)
841 {
842 struct ttm_bo_device *bdev = bo->bdev;
843 struct ttm_resource_manager *man;
844
845 man = ttm_manager_type(bdev, place->mem_type);
846 if (!man || !ttm_resource_manager_used(man))
847 return -EBUSY;
848
849 mem->mem_type = place->mem_type;
850 mem->placement = place->flags;
851
852 spin_lock(&ttm_bo_glob.lru_lock);
853 ttm_bo_del_from_lru(bo);
854 ttm_bo_add_mem_to_lru(bo, mem);
855 spin_unlock(&ttm_bo_glob.lru_lock);
856
857 return 0;
858 }
859
860 /**
861 * Creates space for memory region @mem according to its type.
862 *
863 * This function first searches for free space in compatible memory types in
864 * the priority order defined by the driver. If free space isn't found, then
865 * ttm_bo_mem_force_space is attempted in priority order to evict and find
866 * space.
867 */
868 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
869 struct ttm_placement *placement,
870 struct ttm_resource *mem,
871 struct ttm_operation_ctx *ctx)
872 {
873 struct ttm_bo_device *bdev = bo->bdev;
874 bool type_found = false;
875 int i, ret;
876
877 ret = dma_resv_reserve_shared(bo->base.resv, 1);
878 if (unlikely(ret))
879 return ret;
880
881 for (i = 0; i < placement->num_placement; ++i) {
882 const struct ttm_place *place = &placement->placement[i];
883 struct ttm_resource_manager *man;
884
885 ret = ttm_bo_mem_placement(bo, place, mem);
886 if (ret)
887 continue;
888
889 type_found = true;
890 ret = ttm_resource_alloc(bo, place, mem);
891 if (ret == -ENOSPC)
892 continue;
893 if (unlikely(ret))
894 goto error;
895
896 man = ttm_manager_type(bdev, mem->mem_type);
897 ret = ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu);
898 if (unlikely(ret)) {
899 ttm_resource_free(bo, mem);
900 if (ret == -EBUSY)
901 continue;
902
903 goto error;
904 }
905 return 0;
906 }
907
908 for (i = 0; i < placement->num_busy_placement; ++i) {
909 const struct ttm_place *place = &placement->busy_placement[i];
910
911 ret = ttm_bo_mem_placement(bo, place, mem);
912 if (ret)
913 continue;
914
915 type_found = true;
916 ret = ttm_bo_mem_force_space(bo, place, mem, ctx);
917 if (likely(!ret))
918 return 0;
919
920 if (ret && ret != -EBUSY)
921 goto error;
922 }
923
924 ret = -ENOMEM;
925 if (!type_found) {
926 pr_err(TTM_PFX "No compatible memory type found\n");
927 ret = -EINVAL;
928 }
929
930 error:
931 if (bo->mem.mem_type == TTM_PL_SYSTEM && !list_empty(&bo->lru)) {
932 ttm_bo_move_to_lru_tail_unlocked(bo);
933 }
934
935 return ret;
936 }
937 EXPORT_SYMBOL(ttm_bo_mem_space);
938
939 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
940 struct ttm_placement *placement,
941 struct ttm_operation_ctx *ctx)
942 {
943 int ret = 0;
944 struct ttm_resource mem;
945
946 dma_resv_assert_held(bo->base.resv);
947
948 mem.num_pages = bo->num_pages;
949 mem.size = mem.num_pages << PAGE_SHIFT;
950 mem.page_alignment = bo->mem.page_alignment;
951 mem.bus.offset = 0;
952 mem.bus.addr = NULL;
953 mem.mm_node = NULL;
954
955 /*
956 * Determine where to move the buffer.
957 */
958 ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
959 if (ret)
960 goto out_unlock;
961 ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx);
962 out_unlock:
963 if (ret)
964 ttm_resource_free(bo, &mem);
965 return ret;
966 }
967
968 static bool ttm_bo_places_compat(const struct ttm_place *places,
969 unsigned num_placement,
970 struct ttm_resource *mem,
971 uint32_t *new_flags)
972 {
973 unsigned i;
974
975 for (i = 0; i < num_placement; i++) {
976 const struct ttm_place *heap = &places[i];
977
978 if ((mem->start < heap->fpfn ||
979 (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn)))
980 continue;
981
982 *new_flags = heap->flags;
983 if ((mem->mem_type == heap->mem_type) &&
984 (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) ||
985 (mem->placement & TTM_PL_FLAG_CONTIGUOUS)))
986 return true;
987 }
988 return false;
989 }
990
991 bool ttm_bo_mem_compat(struct ttm_placement *placement,
992 struct ttm_resource *mem,
993 uint32_t *new_flags)
994 {
995 if (ttm_bo_places_compat(placement->placement, placement->num_placement,
996 mem, new_flags))
997 return true;
998
999 if ((placement->busy_placement != placement->placement ||
1000 placement->num_busy_placement > placement->num_placement) &&
1001 ttm_bo_places_compat(placement->busy_placement,
1002 placement->num_busy_placement,
1003 mem, new_flags))
1004 return true;
1005
1006 return false;
1007 }
1008 EXPORT_SYMBOL(ttm_bo_mem_compat);
1009
1010 int ttm_bo_validate(struct ttm_buffer_object *bo,
1011 struct ttm_placement *placement,
1012 struct ttm_operation_ctx *ctx)
1013 {
1014 int ret;
1015 uint32_t new_flags;
1016
1017 dma_resv_assert_held(bo->base.resv);
1018
1019 /*
1020 * Remove the backing store if no placement is given.
1021 */
1022 if (!placement->num_placement && !placement->num_busy_placement) {
1023 ret = ttm_bo_pipeline_gutting(bo);
1024 if (ret)
1025 return ret;
1026
1027 return ttm_tt_create(bo, false);
1028 }
1029
1030 /*
1031 * Check whether we need to move buffer.
1032 */
1033 if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) {
1034 ret = ttm_bo_move_buffer(bo, placement, ctx);
1035 if (ret)
1036 return ret;
1037 }
1038 /*
1039 * We might need to add a TTM.
1040 */
1041 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
1042 ret = ttm_tt_create(bo, true);
1043 if (ret)
1044 return ret;
1045 }
1046 return 0;
1047 }
1048 EXPORT_SYMBOL(ttm_bo_validate);
1049
1050 int ttm_bo_init_reserved(struct ttm_bo_device *bdev,
1051 struct ttm_buffer_object *bo,
1052 unsigned long size,
1053 enum ttm_bo_type type,
1054 struct ttm_placement *placement,
1055 uint32_t page_alignment,
1056 struct ttm_operation_ctx *ctx,
1057 size_t acc_size,
1058 struct sg_table *sg,
1059 struct dma_resv *resv,
1060 void (*destroy) (struct ttm_buffer_object *))
1061 {
1062 struct ttm_mem_global *mem_glob = &ttm_mem_glob;
1063 int ret = 0;
1064 unsigned long num_pages;
1065 bool locked;
1066
1067 ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx);
1068 if (ret) {
1069 pr_err("Out of kernel memory\n");
1070 if (destroy)
1071 (*destroy)(bo);
1072 else
1073 kfree(bo);
1074 return -ENOMEM;
1075 }
1076
1077 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1078 if (num_pages == 0) {
1079 pr_err("Illegal buffer object size\n");
1080 if (destroy)
1081 (*destroy)(bo);
1082 else
1083 kfree(bo);
1084 ttm_mem_global_free(mem_glob, acc_size);
1085 return -EINVAL;
1086 }
1087 bo->destroy = destroy ? destroy : ttm_bo_default_destroy;
1088
1089 kref_init(&bo->kref);
1090 INIT_LIST_HEAD(&bo->lru);
1091 INIT_LIST_HEAD(&bo->ddestroy);
1092 INIT_LIST_HEAD(&bo->swap);
1093 bo->bdev = bdev;
1094 bo->type = type;
1095 bo->num_pages = num_pages;
1096 bo->mem.size = num_pages << PAGE_SHIFT;
1097 bo->mem.mem_type = TTM_PL_SYSTEM;
1098 bo->mem.num_pages = bo->num_pages;
1099 bo->mem.mm_node = NULL;
1100 bo->mem.page_alignment = page_alignment;
1101 bo->mem.bus.offset = 0;
1102 bo->mem.bus.addr = NULL;
1103 bo->moving = NULL;
1104 bo->mem.placement = 0;
1105 bo->acc_size = acc_size;
1106 bo->pin_count = 0;
1107 bo->sg = sg;
1108 if (resv) {
1109 bo->base.resv = resv;
1110 dma_resv_assert_held(bo->base.resv);
1111 } else {
1112 bo->base.resv = &bo->base._resv;
1113 }
1114 if (!ttm_bo_uses_embedded_gem_object(bo)) {
1115 /*
1116 * bo.gem is not initialized, so we have to setup the
1117 * struct elements we want use regardless.
1118 */
1119 dma_resv_init(&bo->base._resv);
1120 drm_vma_node_reset(&bo->base.vma_node);
1121 }
1122 atomic_inc(&ttm_bo_glob.bo_count);
1123
1124 /*
1125 * For ttm_bo_type_device buffers, allocate
1126 * address space from the device.
1127 */
1128 if (bo->type == ttm_bo_type_device ||
1129 bo->type == ttm_bo_type_sg)
1130 ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node,
1131 bo->mem.num_pages);
1132
1133 /* passed reservation objects should already be locked,
1134 * since otherwise lockdep will be angered in radeon.
1135 */
1136 if (!resv) {
1137 locked = dma_resv_trylock(bo->base.resv);
1138 WARN_ON(!locked);
1139 }
1140
1141 if (likely(!ret))
1142 ret = ttm_bo_validate(bo, placement, ctx);
1143
1144 if (unlikely(ret)) {
1145 if (!resv)
1146 ttm_bo_unreserve(bo);
1147
1148 ttm_bo_put(bo);
1149 return ret;
1150 }
1151
1152 ttm_bo_move_to_lru_tail_unlocked(bo);
1153
1154 return ret;
1155 }
1156 EXPORT_SYMBOL(ttm_bo_init_reserved);
1157
1158 int ttm_bo_init(struct ttm_bo_device *bdev,
1159 struct ttm_buffer_object *bo,
1160 unsigned long size,
1161 enum ttm_bo_type type,
1162 struct ttm_placement *placement,
1163 uint32_t page_alignment,
1164 bool interruptible,
1165 size_t acc_size,
1166 struct sg_table *sg,
1167 struct dma_resv *resv,
1168 void (*destroy) (struct ttm_buffer_object *))
1169 {
1170 struct ttm_operation_ctx ctx = { interruptible, false };
1171 int ret;
1172
1173 ret = ttm_bo_init_reserved(bdev, bo, size, type, placement,
1174 page_alignment, &ctx, acc_size,
1175 sg, resv, destroy);
1176 if (ret)
1177 return ret;
1178
1179 if (!resv)
1180 ttm_bo_unreserve(bo);
1181
1182 return 0;
1183 }
1184 EXPORT_SYMBOL(ttm_bo_init);
1185
1186 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev,
1187 unsigned long bo_size,
1188 unsigned struct_size)
1189 {
1190 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1191 size_t size = 0;
1192
1193 size += ttm_round_pot(struct_size);
1194 size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t)));
1195 size += ttm_round_pot(sizeof(struct ttm_dma_tt));
1196 return size;
1197 }
1198 EXPORT_SYMBOL(ttm_bo_dma_acc_size);
1199
1200 static void ttm_bo_global_kobj_release(struct kobject *kobj)
1201 {
1202 struct ttm_bo_global *glob =
1203 container_of(kobj, struct ttm_bo_global, kobj);
1204
1205 __free_page(glob->dummy_read_page);
1206 }
1207
1208 static void ttm_bo_global_release(void)
1209 {
1210 struct ttm_bo_global *glob = &ttm_bo_glob;
1211
1212 mutex_lock(&ttm_global_mutex);
1213 if (--ttm_bo_glob_use_count > 0)
1214 goto out;
1215
1216 kobject_del(&glob->kobj);
1217 kobject_put(&glob->kobj);
1218 ttm_mem_global_release(&ttm_mem_glob);
1219 memset(glob, 0, sizeof(*glob));
1220 out:
1221 mutex_unlock(&ttm_global_mutex);
1222 }
1223
1224 static int ttm_bo_global_init(void)
1225 {
1226 struct ttm_bo_global *glob = &ttm_bo_glob;
1227 int ret = 0;
1228 unsigned i;
1229
1230 mutex_lock(&ttm_global_mutex);
1231 if (++ttm_bo_glob_use_count > 1)
1232 goto out;
1233
1234 ret = ttm_mem_global_init(&ttm_mem_glob);
1235 if (ret)
1236 goto out;
1237
1238 spin_lock_init(&glob->lru_lock);
1239 glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32);
1240
1241 if (unlikely(glob->dummy_read_page == NULL)) {
1242 ret = -ENOMEM;
1243 goto out;
1244 }
1245
1246 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1247 INIT_LIST_HEAD(&glob->swap_lru[i]);
1248 INIT_LIST_HEAD(&glob->device_list);
1249 atomic_set(&glob->bo_count, 0);
1250
1251 ret = kobject_init_and_add(
1252 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects");
1253 if (unlikely(ret != 0))
1254 kobject_put(&glob->kobj);
1255 out:
1256 mutex_unlock(&ttm_global_mutex);
1257 return ret;
1258 }
1259
1260 int ttm_bo_device_release(struct ttm_bo_device *bdev)
1261 {
1262 struct ttm_bo_global *glob = &ttm_bo_glob;
1263 int ret = 0;
1264 unsigned i;
1265 struct ttm_resource_manager *man;
1266
1267 man = ttm_manager_type(bdev, TTM_PL_SYSTEM);
1268 ttm_resource_manager_set_used(man, false);
1269 ttm_set_driver_manager(bdev, TTM_PL_SYSTEM, NULL);
1270
1271 mutex_lock(&ttm_global_mutex);
1272 list_del(&bdev->device_list);
1273 mutex_unlock(&ttm_global_mutex);
1274
1275 cancel_delayed_work_sync(&bdev->wq);
1276
1277 if (ttm_bo_delayed_delete(bdev, true))
1278 pr_debug("Delayed destroy list was clean\n");
1279
1280 spin_lock(&glob->lru_lock);
1281 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1282 if (list_empty(&man->lru[0]))
1283 pr_debug("Swap list %d was clean\n", i);
1284 spin_unlock(&glob->lru_lock);
1285
1286 if (!ret)
1287 ttm_bo_global_release();
1288
1289 return ret;
1290 }
1291 EXPORT_SYMBOL(ttm_bo_device_release);
1292
1293 static void ttm_bo_init_sysman(struct ttm_bo_device *bdev)
1294 {
1295 struct ttm_resource_manager *man = &bdev->sysman;
1296
1297 /*
1298 * Initialize the system memory buffer type.
1299 * Other types need to be driver / IOCTL initialized.
1300 */
1301 man->use_tt = true;
1302
1303 ttm_resource_manager_init(man, 0);
1304 ttm_set_driver_manager(bdev, TTM_PL_SYSTEM, man);
1305 ttm_resource_manager_set_used(man, true);
1306 }
1307
1308 int ttm_bo_device_init(struct ttm_bo_device *bdev,
1309 struct ttm_bo_driver *driver,
1310 struct address_space *mapping,
1311 struct drm_vma_offset_manager *vma_manager,
1312 bool need_dma32)
1313 {
1314 struct ttm_bo_global *glob = &ttm_bo_glob;
1315 int ret;
1316
1317 if (WARN_ON(vma_manager == NULL))
1318 return -EINVAL;
1319
1320 ret = ttm_bo_global_init();
1321 if (ret)
1322 return ret;
1323
1324 bdev->driver = driver;
1325
1326 ttm_bo_init_sysman(bdev);
1327
1328 bdev->vma_manager = vma_manager;
1329 INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
1330 INIT_LIST_HEAD(&bdev->ddestroy);
1331 bdev->dev_mapping = mapping;
1332 bdev->need_dma32 = need_dma32;
1333 mutex_lock(&ttm_global_mutex);
1334 list_add_tail(&bdev->device_list, &glob->device_list);
1335 mutex_unlock(&ttm_global_mutex);
1336
1337 return 0;
1338 }
1339 EXPORT_SYMBOL(ttm_bo_device_init);
1340
1341 /*
1342 * buffer object vm functions.
1343 */
1344
1345 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1346 {
1347 struct ttm_bo_device *bdev = bo->bdev;
1348
1349 drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping);
1350 ttm_mem_io_free(bdev, &bo->mem);
1351 }
1352 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1353
1354 int ttm_bo_wait(struct ttm_buffer_object *bo,
1355 bool interruptible, bool no_wait)
1356 {
1357 long timeout = 15 * HZ;
1358
1359 if (no_wait) {
1360 if (dma_resv_test_signaled_rcu(bo->base.resv, true))
1361 return 0;
1362 else
1363 return -EBUSY;
1364 }
1365
1366 timeout = dma_resv_wait_timeout_rcu(bo->base.resv, true,
1367 interruptible, timeout);
1368 if (timeout < 0)
1369 return timeout;
1370
1371 if (timeout == 0)
1372 return -EBUSY;
1373
1374 dma_resv_add_excl_fence(bo->base.resv, NULL);
1375 return 0;
1376 }
1377 EXPORT_SYMBOL(ttm_bo_wait);
1378
1379 /**
1380 * A buffer object shrink method that tries to swap out the first
1381 * buffer object on the bo_global::swap_lru list.
1382 */
1383 int ttm_bo_swapout(struct ttm_operation_ctx *ctx)
1384 {
1385 struct ttm_bo_global *glob = &ttm_bo_glob;
1386 struct ttm_buffer_object *bo;
1387 int ret = -EBUSY;
1388 bool locked;
1389 unsigned i;
1390
1391 spin_lock(&glob->lru_lock);
1392 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
1393 list_for_each_entry(bo, &glob->swap_lru[i], swap) {
1394 if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked,
1395 NULL))
1396 continue;
1397
1398 if (!ttm_bo_get_unless_zero(bo)) {
1399 if (locked)
1400 dma_resv_unlock(bo->base.resv);
1401 continue;
1402 }
1403
1404 ret = 0;
1405 break;
1406 }
1407 if (!ret)
1408 break;
1409 }
1410
1411 if (ret) {
1412 spin_unlock(&glob->lru_lock);
1413 return ret;
1414 }
1415
1416 if (bo->deleted) {
1417 ret = ttm_bo_cleanup_refs(bo, false, false, locked);
1418 ttm_bo_put(bo);
1419 return ret;
1420 }
1421
1422 ttm_bo_del_from_lru(bo);
1423 spin_unlock(&glob->lru_lock);
1424
1425 /**
1426 * Move to system cached
1427 */
1428
1429 if (bo->mem.mem_type != TTM_PL_SYSTEM) {
1430 struct ttm_operation_ctx ctx = { false, false };
1431 struct ttm_resource evict_mem;
1432
1433 evict_mem = bo->mem;
1434 evict_mem.mm_node = NULL;
1435 evict_mem.placement = 0;
1436 evict_mem.mem_type = TTM_PL_SYSTEM;
1437
1438 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx);
1439 if (unlikely(ret != 0))
1440 goto out;
1441 }
1442
1443 /**
1444 * Make sure BO is idle.
1445 */
1446
1447 ret = ttm_bo_wait(bo, false, false);
1448 if (unlikely(ret != 0))
1449 goto out;
1450
1451 ttm_bo_unmap_virtual(bo);
1452
1453 /**
1454 * Swap out. Buffer will be swapped in again as soon as
1455 * anyone tries to access a ttm page.
1456 */
1457
1458 if (bo->bdev->driver->swap_notify)
1459 bo->bdev->driver->swap_notify(bo);
1460
1461 ret = ttm_tt_swapout(bo->bdev, bo->ttm);
1462 out:
1463
1464 /**
1465 *
1466 * Unreserve without putting on LRU to avoid swapping out an
1467 * already swapped buffer.
1468 */
1469 if (locked)
1470 dma_resv_unlock(bo->base.resv);
1471 ttm_bo_put(bo);
1472 return ret;
1473 }
1474 EXPORT_SYMBOL(ttm_bo_swapout);
1475
1476 void ttm_bo_tt_destroy(struct ttm_buffer_object *bo)
1477 {
1478 if (bo->ttm == NULL)
1479 return;
1480
1481 ttm_tt_destroy(bo->bdev, bo->ttm);
1482 bo->ttm = NULL;
1483 }
1484
Ubuntu Jammy Linux kernel mirror