2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
24 * Authors: Dave Airlie
28 #include <linux/dma-fence-array.h>
29 #include <linux/interval_tree_generic.h>
30 #include <linux/idr.h>
32 #include <drm/amdgpu_drm.h>
34 #include "amdgpu_trace.h"
35 #include "amdgpu_amdkfd.h"
36 #include "amdgpu_gmc.h"
41 * GPUVM is similar to the legacy gart on older asics, however
42 * rather than there being a single global gart table
43 * for the entire GPU, there are multiple VM page tables active
44 * at any given time. The VM page tables can contain a mix
45 * vram pages and system memory pages and system memory pages
46 * can be mapped as snooped (cached system pages) or unsnooped
47 * (uncached system pages).
48 * Each VM has an ID associated with it and there is a page table
49 * associated with each VMID. When execting a command buffer,
50 * the kernel tells the the ring what VMID to use for that command
51 * buffer. VMIDs are allocated dynamically as commands are submitted.
52 * The userspace drivers maintain their own address space and the kernel
53 * sets up their pages tables accordingly when they submit their
54 * command buffers and a VMID is assigned.
55 * Cayman/Trinity support up to 8 active VMs at any given time;
59 #define START(node) ((node)->start)
60 #define LAST(node) ((node)->last)
62 INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping
, rb
, uint64_t, __subtree_last
,
63 START
, LAST
, static, amdgpu_vm_it
)
69 * struct amdgpu_pte_update_params - Local structure
71 * Encapsulate some VM table update parameters to reduce
72 * the number of function parameters
75 struct amdgpu_pte_update_params
{
78 * @adev: amdgpu device we do this update for
80 struct amdgpu_device
*adev
;
83 * @vm: optional amdgpu_vm we do this update for
88 * @src: address where to copy page table entries from
93 * @ib: indirect buffer to fill with commands
98 * @func: Function which actually does the update
100 void (*func
)(struct amdgpu_pte_update_params
*params
,
101 struct amdgpu_bo
*bo
, uint64_t pe
,
102 uint64_t addr
, unsigned count
, uint32_t incr
,
107 * DMA addresses to use for mapping, used during VM update by CPU
109 dma_addr_t
*pages_addr
;
113 * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback
115 struct amdgpu_prt_cb
{
118 * @adev: amdgpu device
120 struct amdgpu_device
*adev
;
125 struct dma_fence_cb cb
;
129 * amdgpu_vm_level_shift - return the addr shift for each level
131 * @adev: amdgpu_device pointer
135 * The number of bits the pfn needs to be right shifted for a level.
137 static unsigned amdgpu_vm_level_shift(struct amdgpu_device
*adev
,
140 unsigned shift
= 0xff;
146 shift
= 9 * (AMDGPU_VM_PDB0
- level
) +
147 adev
->vm_manager
.block_size
;
153 dev_err(adev
->dev
, "the level%d isn't supported.\n", level
);
160 * amdgpu_vm_num_entries - return the number of entries in a PD/PT
162 * @adev: amdgpu_device pointer
166 * The number of entries in a page directory or page table.
168 static unsigned amdgpu_vm_num_entries(struct amdgpu_device
*adev
,
171 unsigned shift
= amdgpu_vm_level_shift(adev
,
172 adev
->vm_manager
.root_level
);
174 if (level
== adev
->vm_manager
.root_level
)
175 /* For the root directory */
176 return round_up(adev
->vm_manager
.max_pfn
, 1ULL << shift
) >> shift
;
177 else if (level
!= AMDGPU_VM_PTB
)
178 /* Everything in between */
181 /* For the page tables on the leaves */
182 return AMDGPU_VM_PTE_COUNT(adev
);
186 * amdgpu_vm_entries_mask - the mask to get the entry number of a PD/PT
188 * @adev: amdgpu_device pointer
192 * The mask to extract the entry number of a PD/PT from an address.
194 static uint32_t amdgpu_vm_entries_mask(struct amdgpu_device
*adev
,
197 if (level
<= adev
->vm_manager
.root_level
)
199 else if (level
!= AMDGPU_VM_PTB
)
202 return AMDGPU_VM_PTE_COUNT(adev
) - 1;
206 * amdgpu_vm_bo_size - returns the size of the BOs in bytes
208 * @adev: amdgpu_device pointer
212 * The size of the BO for a page directory or page table in bytes.
214 static unsigned amdgpu_vm_bo_size(struct amdgpu_device
*adev
, unsigned level
)
216 return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev
, level
) * 8);
220 * amdgpu_vm_bo_evicted - vm_bo is evicted
222 * @vm_bo: vm_bo which is evicted
224 * State for PDs/PTs and per VM BOs which are not at the location they should
227 static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base
*vm_bo
)
229 struct amdgpu_vm
*vm
= vm_bo
->vm
;
230 struct amdgpu_bo
*bo
= vm_bo
->bo
;
233 if (bo
->tbo
.type
== ttm_bo_type_kernel
)
234 list_move(&vm_bo
->vm_status
, &vm
->evicted
);
236 list_move_tail(&vm_bo
->vm_status
, &vm
->evicted
);
240 * amdgpu_vm_bo_relocated - vm_bo is reloacted
242 * @vm_bo: vm_bo which is relocated
244 * State for PDs/PTs which needs to update their parent PD.
246 static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base
*vm_bo
)
248 list_move(&vm_bo
->vm_status
, &vm_bo
->vm
->relocated
);
252 * amdgpu_vm_bo_moved - vm_bo is moved
254 * @vm_bo: vm_bo which is moved
256 * State for per VM BOs which are moved, but that change is not yet reflected
257 * in the page tables.
259 static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base
*vm_bo
)
261 list_move(&vm_bo
->vm_status
, &vm_bo
->vm
->moved
);
265 * amdgpu_vm_bo_idle - vm_bo is idle
267 * @vm_bo: vm_bo which is now idle
269 * State for PDs/PTs and per VM BOs which have gone through the state machine
272 static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base
*vm_bo
)
274 list_move(&vm_bo
->vm_status
, &vm_bo
->vm
->idle
);
275 vm_bo
->moved
= false;
279 * amdgpu_vm_bo_invalidated - vm_bo is invalidated
281 * @vm_bo: vm_bo which is now invalidated
283 * State for normal BOs which are invalidated and that change not yet reflected
286 static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base
*vm_bo
)
288 spin_lock(&vm_bo
->vm
->invalidated_lock
);
289 list_move(&vm_bo
->vm_status
, &vm_bo
->vm
->invalidated
);
290 spin_unlock(&vm_bo
->vm
->invalidated_lock
);
294 * amdgpu_vm_bo_done - vm_bo is done
296 * @vm_bo: vm_bo which is now done
298 * State for normal BOs which are invalidated and that change has been updated
301 static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base
*vm_bo
)
303 spin_lock(&vm_bo
->vm
->invalidated_lock
);
304 list_del_init(&vm_bo
->vm_status
);
305 spin_unlock(&vm_bo
->vm
->invalidated_lock
);
309 * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm
311 * @base: base structure for tracking BO usage in a VM
312 * @vm: vm to which bo is to be added
313 * @bo: amdgpu buffer object
315 * Initialize a bo_va_base structure and add it to the appropriate lists
318 static void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base
*base
,
319 struct amdgpu_vm
*vm
,
320 struct amdgpu_bo
*bo
)
325 INIT_LIST_HEAD(&base
->vm_status
);
329 base
->next
= bo
->vm_bo
;
332 if (bo
->tbo
.resv
!= vm
->root
.base
.bo
->tbo
.resv
)
335 vm
->bulk_moveable
= false;
336 if (bo
->tbo
.type
== ttm_bo_type_kernel
)
337 amdgpu_vm_bo_relocated(base
);
339 amdgpu_vm_bo_idle(base
);
341 if (bo
->preferred_domains
&
342 amdgpu_mem_type_to_domain(bo
->tbo
.mem
.mem_type
))
346 * we checked all the prerequisites, but it looks like this per vm bo
347 * is currently evicted. add the bo to the evicted list to make sure it
348 * is validated on next vm use to avoid fault.
350 amdgpu_vm_bo_evicted(base
);
354 * amdgpu_vm_pt_parent - get the parent page directory
356 * @pt: child page table
358 * Helper to get the parent entry for the child page table. NULL if we are at
359 * the root page directory.
361 static struct amdgpu_vm_pt
*amdgpu_vm_pt_parent(struct amdgpu_vm_pt
*pt
)
363 struct amdgpu_bo
*parent
= pt
->base
.bo
->parent
;
368 return container_of(parent
->vm_bo
, struct amdgpu_vm_pt
, base
);
372 * amdgpu_vm_pt_cursor - state for for_each_amdgpu_vm_pt
374 struct amdgpu_vm_pt_cursor
{
376 struct amdgpu_vm_pt
*parent
;
377 struct amdgpu_vm_pt
*entry
;
382 * amdgpu_vm_pt_start - start PD/PT walk
384 * @adev: amdgpu_device pointer
385 * @vm: amdgpu_vm structure
386 * @start: start address of the walk
387 * @cursor: state to initialize
389 * Initialize a amdgpu_vm_pt_cursor to start a walk.
391 static void amdgpu_vm_pt_start(struct amdgpu_device
*adev
,
392 struct amdgpu_vm
*vm
, uint64_t start
,
393 struct amdgpu_vm_pt_cursor
*cursor
)
396 cursor
->parent
= NULL
;
397 cursor
->entry
= &vm
->root
;
398 cursor
->level
= adev
->vm_manager
.root_level
;
402 * amdgpu_vm_pt_descendant - go to child node
404 * @adev: amdgpu_device pointer
405 * @cursor: current state
407 * Walk to the child node of the current node.
409 * True if the walk was possible, false otherwise.
411 static bool amdgpu_vm_pt_descendant(struct amdgpu_device
*adev
,
412 struct amdgpu_vm_pt_cursor
*cursor
)
414 unsigned mask
, shift
, idx
;
416 if (!cursor
->entry
->entries
)
419 BUG_ON(!cursor
->entry
->base
.bo
);
420 mask
= amdgpu_vm_entries_mask(adev
, cursor
->level
);
421 shift
= amdgpu_vm_level_shift(adev
, cursor
->level
);
424 idx
= (cursor
->pfn
>> shift
) & mask
;
425 cursor
->parent
= cursor
->entry
;
426 cursor
->entry
= &cursor
->entry
->entries
[idx
];
431 * amdgpu_vm_pt_sibling - go to sibling node
433 * @adev: amdgpu_device pointer
434 * @cursor: current state
436 * Walk to the sibling node of the current node.
438 * True if the walk was possible, false otherwise.
440 static bool amdgpu_vm_pt_sibling(struct amdgpu_device
*adev
,
441 struct amdgpu_vm_pt_cursor
*cursor
)
443 unsigned shift
, num_entries
;
445 /* Root doesn't have a sibling */
449 /* Go to our parents and see if we got a sibling */
450 shift
= amdgpu_vm_level_shift(adev
, cursor
->level
- 1);
451 num_entries
= amdgpu_vm_num_entries(adev
, cursor
->level
- 1);
453 if (cursor
->entry
== &cursor
->parent
->entries
[num_entries
- 1])
456 cursor
->pfn
+= 1ULL << shift
;
457 cursor
->pfn
&= ~((1ULL << shift
) - 1);
463 * amdgpu_vm_pt_ancestor - go to parent node
465 * @cursor: current state
467 * Walk to the parent node of the current node.
469 * True if the walk was possible, false otherwise.
471 static bool amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor
*cursor
)
477 cursor
->entry
= cursor
->parent
;
478 cursor
->parent
= amdgpu_vm_pt_parent(cursor
->parent
);
483 * amdgpu_vm_pt_next - get next PD/PT in hieratchy
485 * @adev: amdgpu_device pointer
486 * @cursor: current state
488 * Walk the PD/PT tree to the next node.
490 static void amdgpu_vm_pt_next(struct amdgpu_device
*adev
,
491 struct amdgpu_vm_pt_cursor
*cursor
)
493 /* First try a newborn child */
494 if (amdgpu_vm_pt_descendant(adev
, cursor
))
497 /* If that didn't worked try to find a sibling */
498 while (!amdgpu_vm_pt_sibling(adev
, cursor
)) {
499 /* No sibling, go to our parents and grandparents */
500 if (!amdgpu_vm_pt_ancestor(cursor
)) {
508 * amdgpu_vm_pt_first_leaf - get first leaf PD/PT
510 * @adev: amdgpu_device pointer
511 * @vm: amdgpu_vm structure
512 * @start: start addr of the walk
513 * @cursor: state to initialize
515 * Start a walk and go directly to the leaf node.
517 static void amdgpu_vm_pt_first_leaf(struct amdgpu_device
*adev
,
518 struct amdgpu_vm
*vm
, uint64_t start
,
519 struct amdgpu_vm_pt_cursor
*cursor
)
521 amdgpu_vm_pt_start(adev
, vm
, start
, cursor
);
522 while (amdgpu_vm_pt_descendant(adev
, cursor
));
526 * amdgpu_vm_pt_next_leaf - get next leaf PD/PT
528 * @adev: amdgpu_device pointer
529 * @cursor: current state
531 * Walk the PD/PT tree to the next leaf node.
533 static void amdgpu_vm_pt_next_leaf(struct amdgpu_device
*adev
,
534 struct amdgpu_vm_pt_cursor
*cursor
)
536 amdgpu_vm_pt_next(adev
, cursor
);
537 if (cursor
->pfn
!= ~0ll)
538 while (amdgpu_vm_pt_descendant(adev
, cursor
));
542 * for_each_amdgpu_vm_pt_leaf - walk over all leaf PDs/PTs in the hierarchy
544 #define for_each_amdgpu_vm_pt_leaf(adev, vm, start, end, cursor) \
545 for (amdgpu_vm_pt_first_leaf((adev), (vm), (start), &(cursor)); \
546 (cursor).pfn <= end; amdgpu_vm_pt_next_leaf((adev), &(cursor)))
549 * amdgpu_vm_pt_first_dfs - start a deep first search
551 * @adev: amdgpu_device structure
552 * @vm: amdgpu_vm structure
553 * @cursor: state to initialize
555 * Starts a deep first traversal of the PD/PT tree.
557 static void amdgpu_vm_pt_first_dfs(struct amdgpu_device
*adev
,
558 struct amdgpu_vm
*vm
,
559 struct amdgpu_vm_pt_cursor
*cursor
)
561 amdgpu_vm_pt_start(adev
, vm
, 0, cursor
);
562 while (amdgpu_vm_pt_descendant(adev
, cursor
));
566 * amdgpu_vm_pt_next_dfs - get the next node for a deep first search
568 * @adev: amdgpu_device structure
569 * @cursor: current state
571 * Move the cursor to the next node in a deep first search.
573 static void amdgpu_vm_pt_next_dfs(struct amdgpu_device
*adev
,
574 struct amdgpu_vm_pt_cursor
*cursor
)
580 cursor
->entry
= NULL
;
581 else if (amdgpu_vm_pt_sibling(adev
, cursor
))
582 while (amdgpu_vm_pt_descendant(adev
, cursor
));
584 amdgpu_vm_pt_ancestor(cursor
);
588 * for_each_amdgpu_vm_pt_dfs_safe - safe deep first search of all PDs/PTs
590 #define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, cursor, entry) \
591 for (amdgpu_vm_pt_first_dfs((adev), (vm), &(cursor)), \
592 (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor));\
593 (entry); (entry) = (cursor).entry, \
594 amdgpu_vm_pt_next_dfs((adev), &(cursor)))
597 * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
599 * @vm: vm providing the BOs
600 * @validated: head of validation list
601 * @entry: entry to add
603 * Add the page directory to the list of BOs to
604 * validate for command submission.
606 void amdgpu_vm_get_pd_bo(struct amdgpu_vm
*vm
,
607 struct list_head
*validated
,
608 struct amdgpu_bo_list_entry
*entry
)
611 entry
->tv
.bo
= &vm
->root
.base
.bo
->tbo
;
612 /* One for the VM updates, one for TTM and one for the CS job */
613 entry
->tv
.num_shared
= 3;
614 entry
->user_pages
= NULL
;
615 list_add(&entry
->tv
.head
, validated
);
618 void amdgpu_vm_del_from_lru_notify(struct ttm_buffer_object
*bo
)
620 struct amdgpu_bo
*abo
;
621 struct amdgpu_vm_bo_base
*bo_base
;
623 if (!amdgpu_bo_is_amdgpu_bo(bo
))
626 if (bo
->mem
.placement
& TTM_PL_FLAG_NO_EVICT
)
629 abo
= ttm_to_amdgpu_bo(bo
);
632 for (bo_base
= abo
->vm_bo
; bo_base
; bo_base
= bo_base
->next
) {
633 struct amdgpu_vm
*vm
= bo_base
->vm
;
635 if (abo
->tbo
.resv
== vm
->root
.base
.bo
->tbo
.resv
)
636 vm
->bulk_moveable
= false;
641 * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU
643 * @adev: amdgpu device pointer
644 * @vm: vm providing the BOs
646 * Move all BOs to the end of LRU and remember their positions to put them
649 void amdgpu_vm_move_to_lru_tail(struct amdgpu_device
*adev
,
650 struct amdgpu_vm
*vm
)
652 struct ttm_bo_global
*glob
= adev
->mman
.bdev
.glob
;
653 struct amdgpu_vm_bo_base
*bo_base
;
656 if (vm
->bulk_moveable
) {
657 spin_lock(&glob
->lru_lock
);
658 ttm_bo_bulk_move_lru_tail(&vm
->lru_bulk_move
);
659 spin_unlock(&glob
->lru_lock
);
664 memset(&vm
->lru_bulk_move
, 0, sizeof(vm
->lru_bulk_move
));
666 spin_lock(&glob
->lru_lock
);
667 list_for_each_entry(bo_base
, &vm
->idle
, vm_status
) {
668 struct amdgpu_bo
*bo
= bo_base
->bo
;
673 ttm_bo_move_to_lru_tail(&bo
->tbo
, &vm
->lru_bulk_move
);
675 ttm_bo_move_to_lru_tail(&bo
->shadow
->tbo
,
678 spin_unlock(&glob
->lru_lock
);
680 vm
->bulk_moveable
= true;
684 * amdgpu_vm_validate_pt_bos - validate the page table BOs
686 * @adev: amdgpu device pointer
687 * @vm: vm providing the BOs
688 * @validate: callback to do the validation
689 * @param: parameter for the validation callback
691 * Validate the page table BOs on command submission if neccessary.
696 int amdgpu_vm_validate_pt_bos(struct amdgpu_device
*adev
, struct amdgpu_vm
*vm
,
697 int (*validate
)(void *p
, struct amdgpu_bo
*bo
),
700 struct amdgpu_vm_bo_base
*bo_base
, *tmp
;
703 vm
->bulk_moveable
&= list_empty(&vm
->evicted
);
705 list_for_each_entry_safe(bo_base
, tmp
, &vm
->evicted
, vm_status
) {
706 struct amdgpu_bo
*bo
= bo_base
->bo
;
708 r
= validate(param
, bo
);
712 if (bo
->tbo
.type
!= ttm_bo_type_kernel
) {
713 amdgpu_vm_bo_moved(bo_base
);
715 if (vm
->use_cpu_for_update
)
716 r
= amdgpu_bo_kmap(bo
, NULL
);
718 r
= amdgpu_ttm_alloc_gart(&bo
->tbo
);
722 r
= amdgpu_ttm_alloc_gart(&bo
->shadow
->tbo
);
726 amdgpu_vm_bo_relocated(bo_base
);
734 * amdgpu_vm_ready - check VM is ready for updates
738 * Check if all VM PDs/PTs are ready for updates
741 * True if eviction list is empty.
743 bool amdgpu_vm_ready(struct amdgpu_vm
*vm
)
745 return list_empty(&vm
->evicted
);
749 * amdgpu_vm_clear_bo - initially clear the PDs/PTs
751 * @adev: amdgpu_device pointer
752 * @vm: VM to clear BO from
754 * @level: level this BO is at
755 * @pte_support_ats: indicate ATS support from PTE
757 * Root PD needs to be reserved when calling this.
760 * 0 on success, errno otherwise.
762 static int amdgpu_vm_clear_bo(struct amdgpu_device
*adev
,
763 struct amdgpu_vm
*vm
, struct amdgpu_bo
*bo
,
764 unsigned level
, bool pte_support_ats
)
766 struct ttm_operation_ctx ctx
= { true, false };
767 struct dma_fence
*fence
= NULL
;
768 unsigned entries
, ats_entries
;
769 struct amdgpu_ring
*ring
;
770 struct amdgpu_job
*job
;
774 entries
= amdgpu_bo_size(bo
) / 8;
776 if (pte_support_ats
) {
777 if (level
== adev
->vm_manager
.root_level
) {
778 ats_entries
= amdgpu_vm_level_shift(adev
, level
);
779 ats_entries
+= AMDGPU_GPU_PAGE_SHIFT
;
780 ats_entries
= AMDGPU_GMC_HOLE_START
>> ats_entries
;
781 ats_entries
= min(ats_entries
, entries
);
782 entries
-= ats_entries
;
784 ats_entries
= entries
;
791 ring
= container_of(vm
->entity
.rq
->sched
, struct amdgpu_ring
, sched
);
793 r
= ttm_bo_validate(&bo
->tbo
, &bo
->placement
, &ctx
);
797 r
= amdgpu_ttm_alloc_gart(&bo
->tbo
);
801 r
= amdgpu_job_alloc_with_ib(adev
, 64, &job
);
805 addr
= amdgpu_bo_gpu_offset(bo
);
809 ats_value
= AMDGPU_PTE_DEFAULT_ATC
;
810 if (level
!= AMDGPU_VM_PTB
)
811 ats_value
|= AMDGPU_PDE_PTE
;
813 amdgpu_vm_set_pte_pde(adev
, &job
->ibs
[0], addr
, 0,
814 ats_entries
, 0, ats_value
);
815 addr
+= ats_entries
* 8;
821 /* Workaround for fault priority problem on GMC9 */
822 if (level
== AMDGPU_VM_PTB
&& adev
->asic_type
>= CHIP_VEGA10
)
823 value
= AMDGPU_PTE_EXECUTABLE
;
825 amdgpu_vm_set_pte_pde(adev
, &job
->ibs
[0], addr
, 0,
829 amdgpu_ring_pad_ib(ring
, &job
->ibs
[0]);
831 WARN_ON(job
->ibs
[0].length_dw
> 64);
832 r
= amdgpu_sync_resv(adev
, &job
->sync
, bo
->tbo
.resv
,
833 AMDGPU_FENCE_OWNER_KFD
, false);
837 r
= amdgpu_job_submit(job
, &vm
->entity
, AMDGPU_FENCE_OWNER_UNDEFINED
,
842 amdgpu_bo_fence(bo
, fence
, true);
843 dma_fence_put(fence
);
846 return amdgpu_vm_clear_bo(adev
, vm
, bo
->shadow
,
847 level
, pte_support_ats
);
852 amdgpu_job_free(job
);
859 * amdgpu_vm_bo_param - fill in parameters for PD/PT allocation
861 * @adev: amdgpu_device pointer
863 * @bp: resulting BO allocation parameters
865 static void amdgpu_vm_bo_param(struct amdgpu_device
*adev
, struct amdgpu_vm
*vm
,
866 int level
, struct amdgpu_bo_param
*bp
)
868 memset(bp
, 0, sizeof(*bp
));
870 bp
->size
= amdgpu_vm_bo_size(adev
, level
);
871 bp
->byte_align
= AMDGPU_GPU_PAGE_SIZE
;
872 bp
->domain
= AMDGPU_GEM_DOMAIN_VRAM
;
873 bp
->domain
= amdgpu_bo_get_preferred_pin_domain(adev
, bp
->domain
);
874 bp
->flags
= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS
|
875 AMDGPU_GEM_CREATE_CPU_GTT_USWC
;
876 if (vm
->use_cpu_for_update
)
877 bp
->flags
|= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
;
878 else if (!vm
->root
.base
.bo
|| vm
->root
.base
.bo
->shadow
)
879 bp
->flags
|= AMDGPU_GEM_CREATE_SHADOW
;
880 bp
->type
= ttm_bo_type_kernel
;
881 if (vm
->root
.base
.bo
)
882 bp
->resv
= vm
->root
.base
.bo
->tbo
.resv
;
886 * amdgpu_vm_alloc_pts - Allocate page tables.
888 * @adev: amdgpu_device pointer
889 * @vm: VM to allocate page tables for
890 * @saddr: Start address which needs to be allocated
891 * @size: Size from start address we need.
893 * Make sure the page directories and page tables are allocated
896 * 0 on success, errno otherwise.
898 int amdgpu_vm_alloc_pts(struct amdgpu_device
*adev
,
899 struct amdgpu_vm
*vm
,
900 uint64_t saddr
, uint64_t size
)
902 struct amdgpu_vm_pt_cursor cursor
;
903 struct amdgpu_bo
*pt
;
908 /* validate the parameters */
909 if (saddr
& AMDGPU_GPU_PAGE_MASK
|| size
& AMDGPU_GPU_PAGE_MASK
)
912 eaddr
= saddr
+ size
- 1;
914 if (vm
->pte_support_ats
)
915 ats
= saddr
< AMDGPU_GMC_HOLE_START
;
917 saddr
/= AMDGPU_GPU_PAGE_SIZE
;
918 eaddr
/= AMDGPU_GPU_PAGE_SIZE
;
920 if (eaddr
>= adev
->vm_manager
.max_pfn
) {
921 dev_err(adev
->dev
, "va above limit (0x%08llX >= 0x%08llX)\n",
922 eaddr
, adev
->vm_manager
.max_pfn
);
926 for_each_amdgpu_vm_pt_leaf(adev
, vm
, saddr
, eaddr
, cursor
) {
927 struct amdgpu_vm_pt
*entry
= cursor
.entry
;
928 struct amdgpu_bo_param bp
;
930 if (cursor
.level
< AMDGPU_VM_PTB
) {
931 unsigned num_entries
;
933 num_entries
= amdgpu_vm_num_entries(adev
, cursor
.level
);
934 entry
->entries
= kvmalloc_array(num_entries
,
935 sizeof(*entry
->entries
),
946 amdgpu_vm_bo_param(adev
, vm
, cursor
.level
, &bp
);
948 r
= amdgpu_bo_create(adev
, &bp
, &pt
);
952 if (vm
->use_cpu_for_update
) {
953 r
= amdgpu_bo_kmap(pt
, NULL
);
958 /* Keep a reference to the root directory to avoid
959 * freeing them up in the wrong order.
961 pt
->parent
= amdgpu_bo_ref(cursor
.parent
->base
.bo
);
963 amdgpu_vm_bo_base_init(&entry
->base
, vm
, pt
);
965 r
= amdgpu_vm_clear_bo(adev
, vm
, pt
, cursor
.level
, ats
);
973 amdgpu_bo_unref(&pt
->shadow
);
974 amdgpu_bo_unref(&pt
);
979 * amdgpu_vm_free_pts - free PD/PT levels
981 * @adev: amdgpu device structure
982 * @vm: amdgpu vm structure
984 * Free the page directory or page table level and all sub levels.
986 static void amdgpu_vm_free_pts(struct amdgpu_device
*adev
,
987 struct amdgpu_vm
*vm
)
989 struct amdgpu_vm_pt_cursor cursor
;
990 struct amdgpu_vm_pt
*entry
;
992 for_each_amdgpu_vm_pt_dfs_safe(adev
, vm
, cursor
, entry
) {
994 if (entry
->base
.bo
) {
995 entry
->base
.bo
->vm_bo
= NULL
;
996 list_del(&entry
->base
.vm_status
);
997 amdgpu_bo_unref(&entry
->base
.bo
->shadow
);
998 amdgpu_bo_unref(&entry
->base
.bo
);
1000 kvfree(entry
->entries
);
1003 BUG_ON(vm
->root
.base
.bo
);
1007 * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug
1009 * @adev: amdgpu_device pointer
1011 void amdgpu_vm_check_compute_bug(struct amdgpu_device
*adev
)
1013 const struct amdgpu_ip_block
*ip_block
;
1014 bool has_compute_vm_bug
;
1015 struct amdgpu_ring
*ring
;
1018 has_compute_vm_bug
= false;
1020 ip_block
= amdgpu_device_ip_get_ip_block(adev
, AMD_IP_BLOCK_TYPE_GFX
);
1022 /* Compute has a VM bug for GFX version < 7.
1023 Compute has a VM bug for GFX 8 MEC firmware version < 673.*/
1024 if (ip_block
->version
->major
<= 7)
1025 has_compute_vm_bug
= true;
1026 else if (ip_block
->version
->major
== 8)
1027 if (adev
->gfx
.mec_fw_version
< 673)
1028 has_compute_vm_bug
= true;
1031 for (i
= 0; i
< adev
->num_rings
; i
++) {
1032 ring
= adev
->rings
[i
];
1033 if (ring
->funcs
->type
== AMDGPU_RING_TYPE_COMPUTE
)
1034 /* only compute rings */
1035 ring
->has_compute_vm_bug
= has_compute_vm_bug
;
1037 ring
->has_compute_vm_bug
= false;
1042 * amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job.
1044 * @ring: ring on which the job will be submitted
1045 * @job: job to submit
1048 * True if sync is needed.
1050 bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring
*ring
,
1051 struct amdgpu_job
*job
)
1053 struct amdgpu_device
*adev
= ring
->adev
;
1054 unsigned vmhub
= ring
->funcs
->vmhub
;
1055 struct amdgpu_vmid_mgr
*id_mgr
= &adev
->vm_manager
.id_mgr
[vmhub
];
1056 struct amdgpu_vmid
*id
;
1057 bool gds_switch_needed
;
1058 bool vm_flush_needed
= job
->vm_needs_flush
|| ring
->has_compute_vm_bug
;
1062 id
= &id_mgr
->ids
[job
->vmid
];
1063 gds_switch_needed
= ring
->funcs
->emit_gds_switch
&& (
1064 id
->gds_base
!= job
->gds_base
||
1065 id
->gds_size
!= job
->gds_size
||
1066 id
->gws_base
!= job
->gws_base
||
1067 id
->gws_size
!= job
->gws_size
||
1068 id
->oa_base
!= job
->oa_base
||
1069 id
->oa_size
!= job
->oa_size
);
1071 if (amdgpu_vmid_had_gpu_reset(adev
, id
))
1074 return vm_flush_needed
|| gds_switch_needed
;
1078 * amdgpu_vm_flush - hardware flush the vm
1080 * @ring: ring to use for flush
1082 * @need_pipe_sync: is pipe sync needed
1084 * Emit a VM flush when it is necessary.
1087 * 0 on success, errno otherwise.
1089 int amdgpu_vm_flush(struct amdgpu_ring
*ring
, struct amdgpu_job
*job
, bool need_pipe_sync
)
1091 struct amdgpu_device
*adev
= ring
->adev
;
1092 unsigned vmhub
= ring
->funcs
->vmhub
;
1093 struct amdgpu_vmid_mgr
*id_mgr
= &adev
->vm_manager
.id_mgr
[vmhub
];
1094 struct amdgpu_vmid
*id
= &id_mgr
->ids
[job
->vmid
];
1095 bool gds_switch_needed
= ring
->funcs
->emit_gds_switch
&& (
1096 id
->gds_base
!= job
->gds_base
||
1097 id
->gds_size
!= job
->gds_size
||
1098 id
->gws_base
!= job
->gws_base
||
1099 id
->gws_size
!= job
->gws_size
||
1100 id
->oa_base
!= job
->oa_base
||
1101 id
->oa_size
!= job
->oa_size
);
1102 bool vm_flush_needed
= job
->vm_needs_flush
;
1103 bool pasid_mapping_needed
= id
->pasid
!= job
->pasid
||
1104 !id
->pasid_mapping
||
1105 !dma_fence_is_signaled(id
->pasid_mapping
);
1106 struct dma_fence
*fence
= NULL
;
1107 unsigned patch_offset
= 0;
1110 if (amdgpu_vmid_had_gpu_reset(adev
, id
)) {
1111 gds_switch_needed
= true;
1112 vm_flush_needed
= true;
1113 pasid_mapping_needed
= true;
1116 gds_switch_needed
&= !!ring
->funcs
->emit_gds_switch
;
1117 vm_flush_needed
&= !!ring
->funcs
->emit_vm_flush
&&
1118 job
->vm_pd_addr
!= AMDGPU_BO_INVALID_OFFSET
;
1119 pasid_mapping_needed
&= adev
->gmc
.gmc_funcs
->emit_pasid_mapping
&&
1120 ring
->funcs
->emit_wreg
;
1122 if (!vm_flush_needed
&& !gds_switch_needed
&& !need_pipe_sync
)
1125 if (ring
->funcs
->init_cond_exec
)
1126 patch_offset
= amdgpu_ring_init_cond_exec(ring
);
1129 amdgpu_ring_emit_pipeline_sync(ring
);
1131 if (vm_flush_needed
) {
1132 trace_amdgpu_vm_flush(ring
, job
->vmid
, job
->vm_pd_addr
);
1133 amdgpu_ring_emit_vm_flush(ring
, job
->vmid
, job
->vm_pd_addr
);
1136 if (pasid_mapping_needed
)
1137 amdgpu_gmc_emit_pasid_mapping(ring
, job
->vmid
, job
->pasid
);
1139 if (vm_flush_needed
|| pasid_mapping_needed
) {
1140 r
= amdgpu_fence_emit(ring
, &fence
, 0);
1145 if (vm_flush_needed
) {
1146 mutex_lock(&id_mgr
->lock
);
1147 dma_fence_put(id
->last_flush
);
1148 id
->last_flush
= dma_fence_get(fence
);
1149 id
->current_gpu_reset_count
=
1150 atomic_read(&adev
->gpu_reset_counter
);
1151 mutex_unlock(&id_mgr
->lock
);
1154 if (pasid_mapping_needed
) {
1155 id
->pasid
= job
->pasid
;
1156 dma_fence_put(id
->pasid_mapping
);
1157 id
->pasid_mapping
= dma_fence_get(fence
);
1159 dma_fence_put(fence
);
1161 if (ring
->funcs
->emit_gds_switch
&& gds_switch_needed
) {
1162 id
->gds_base
= job
->gds_base
;
1163 id
->gds_size
= job
->gds_size
;
1164 id
->gws_base
= job
->gws_base
;
1165 id
->gws_size
= job
->gws_size
;
1166 id
->oa_base
= job
->oa_base
;
1167 id
->oa_size
= job
->oa_size
;
1168 amdgpu_ring_emit_gds_switch(ring
, job
->vmid
, job
->gds_base
,
1169 job
->gds_size
, job
->gws_base
,
1170 job
->gws_size
, job
->oa_base
,
1174 if (ring
->funcs
->patch_cond_exec
)
1175 amdgpu_ring_patch_cond_exec(ring
, patch_offset
);
1177 /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
1178 if (ring
->funcs
->emit_switch_buffer
) {
1179 amdgpu_ring_emit_switch_buffer(ring
);
1180 amdgpu_ring_emit_switch_buffer(ring
);
1186 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
1189 * @bo: requested buffer object
1191 * Find @bo inside the requested vm.
1192 * Search inside the @bos vm list for the requested vm
1193 * Returns the found bo_va or NULL if none is found
1195 * Object has to be reserved!
1198 * Found bo_va or NULL.
1200 struct amdgpu_bo_va
*amdgpu_vm_bo_find(struct amdgpu_vm
*vm
,
1201 struct amdgpu_bo
*bo
)
1203 struct amdgpu_vm_bo_base
*base
;
1205 for (base
= bo
->vm_bo
; base
; base
= base
->next
) {
1209 return container_of(base
, struct amdgpu_bo_va
, base
);
1215 * amdgpu_vm_do_set_ptes - helper to call the right asic function
1217 * @params: see amdgpu_pte_update_params definition
1218 * @bo: PD/PT to update
1219 * @pe: addr of the page entry
1220 * @addr: dst addr to write into pe
1221 * @count: number of page entries to update
1222 * @incr: increase next addr by incr bytes
1223 * @flags: hw access flags
1225 * Traces the parameters and calls the right asic functions
1226 * to setup the page table using the DMA.
1228 static void amdgpu_vm_do_set_ptes(struct amdgpu_pte_update_params
*params
,
1229 struct amdgpu_bo
*bo
,
1230 uint64_t pe
, uint64_t addr
,
1231 unsigned count
, uint32_t incr
,
1234 pe
+= amdgpu_bo_gpu_offset(bo
);
1235 trace_amdgpu_vm_set_ptes(pe
, addr
, count
, incr
, flags
);
1238 amdgpu_vm_write_pte(params
->adev
, params
->ib
, pe
,
1239 addr
| flags
, count
, incr
);
1242 amdgpu_vm_set_pte_pde(params
->adev
, params
->ib
, pe
, addr
,
1243 count
, incr
, flags
);
1248 * amdgpu_vm_do_copy_ptes - copy the PTEs from the GART
1250 * @params: see amdgpu_pte_update_params definition
1251 * @bo: PD/PT to update
1252 * @pe: addr of the page entry
1253 * @addr: dst addr to write into pe
1254 * @count: number of page entries to update
1255 * @incr: increase next addr by incr bytes
1256 * @flags: hw access flags
1258 * Traces the parameters and calls the DMA function to copy the PTEs.
1260 static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params
*params
,
1261 struct amdgpu_bo
*bo
,
1262 uint64_t pe
, uint64_t addr
,
1263 unsigned count
, uint32_t incr
,
1266 uint64_t src
= (params
->src
+ (addr
>> 12) * 8);
1268 pe
+= amdgpu_bo_gpu_offset(bo
);
1269 trace_amdgpu_vm_copy_ptes(pe
, src
, count
);
1271 amdgpu_vm_copy_pte(params
->adev
, params
->ib
, pe
, src
, count
);
1275 * amdgpu_vm_map_gart - Resolve gart mapping of addr
1277 * @pages_addr: optional DMA address to use for lookup
1278 * @addr: the unmapped addr
1280 * Look up the physical address of the page that the pte resolves
1284 * The pointer for the page table entry.
1286 static uint64_t amdgpu_vm_map_gart(const dma_addr_t
*pages_addr
, uint64_t addr
)
1290 /* page table offset */
1291 result
= pages_addr
[addr
>> PAGE_SHIFT
];
1293 /* in case cpu page size != gpu page size*/
1294 result
|= addr
& (~PAGE_MASK
);
1296 result
&= 0xFFFFFFFFFFFFF000ULL
;
1302 * amdgpu_vm_cpu_set_ptes - helper to update page tables via CPU
1304 * @params: see amdgpu_pte_update_params definition
1305 * @bo: PD/PT to update
1306 * @pe: kmap addr of the page entry
1307 * @addr: dst addr to write into pe
1308 * @count: number of page entries to update
1309 * @incr: increase next addr by incr bytes
1310 * @flags: hw access flags
1312 * Write count number of PT/PD entries directly.
1314 static void amdgpu_vm_cpu_set_ptes(struct amdgpu_pte_update_params
*params
,
1315 struct amdgpu_bo
*bo
,
1316 uint64_t pe
, uint64_t addr
,
1317 unsigned count
, uint32_t incr
,
1323 pe
+= (unsigned long)amdgpu_bo_kptr(bo
);
1325 trace_amdgpu_vm_set_ptes(pe
, addr
, count
, incr
, flags
);
1327 for (i
= 0; i
< count
; i
++) {
1328 value
= params
->pages_addr
?
1329 amdgpu_vm_map_gart(params
->pages_addr
, addr
) :
1331 amdgpu_gmc_set_pte_pde(params
->adev
, (void *)(uintptr_t)pe
,
1338 * amdgpu_vm_update_func - helper to call update function
1340 * Calls the update function for both the given BO as well as its shadow.
1342 static void amdgpu_vm_update_func(struct amdgpu_pte_update_params
*params
,
1343 struct amdgpu_bo
*bo
,
1344 uint64_t pe
, uint64_t addr
,
1345 unsigned count
, uint32_t incr
,
1349 params
->func(params
, bo
->shadow
, pe
, addr
, count
, incr
, flags
);
1350 params
->func(params
, bo
, pe
, addr
, count
, incr
, flags
);
1354 * amdgpu_vm_update_pde - update a single level in the hierarchy
1356 * @param: parameters for the update
1358 * @parent: parent directory
1359 * @entry: entry to update
1361 * Makes sure the requested entry in parent is up to date.
1363 static void amdgpu_vm_update_pde(struct amdgpu_pte_update_params
*params
,
1364 struct amdgpu_vm
*vm
,
1365 struct amdgpu_vm_pt
*parent
,
1366 struct amdgpu_vm_pt
*entry
)
1368 struct amdgpu_bo
*bo
= parent
->base
.bo
, *pbo
;
1369 uint64_t pde
, pt
, flags
;
1372 /* Don't update huge pages here */
1376 for (level
= 0, pbo
= bo
->parent
; pbo
; ++level
)
1379 level
+= params
->adev
->vm_manager
.root_level
;
1380 amdgpu_gmc_get_pde_for_bo(entry
->base
.bo
, level
, &pt
, &flags
);
1381 pde
= (entry
- parent
->entries
) * 8;
1382 amdgpu_vm_update_func(params
, bo
, pde
, pt
, 1, 0, flags
);
1386 * amdgpu_vm_invalidate_pds - mark all PDs as invalid
1388 * @adev: amdgpu_device pointer
1391 * Mark all PD level as invalid after an error.
1393 static void amdgpu_vm_invalidate_pds(struct amdgpu_device
*adev
,
1394 struct amdgpu_vm
*vm
)
1396 struct amdgpu_vm_pt_cursor cursor
;
1397 struct amdgpu_vm_pt
*entry
;
1399 for_each_amdgpu_vm_pt_dfs_safe(adev
, vm
, cursor
, entry
)
1400 if (entry
->base
.bo
&& !entry
->base
.moved
)
1401 amdgpu_vm_bo_relocated(&entry
->base
);
1405 * amdgpu_vm_update_directories - make sure that all directories are valid
1407 * @adev: amdgpu_device pointer
1410 * Makes sure all directories are up to date.
1413 * 0 for success, error for failure.
1415 int amdgpu_vm_update_directories(struct amdgpu_device
*adev
,
1416 struct amdgpu_vm
*vm
)
1418 struct amdgpu_pte_update_params params
;
1419 struct amdgpu_job
*job
;
1423 if (list_empty(&vm
->relocated
))
1427 memset(¶ms
, 0, sizeof(params
));
1430 if (vm
->use_cpu_for_update
) {
1431 r
= amdgpu_bo_sync_wait(vm
->root
.base
.bo
,
1432 AMDGPU_FENCE_OWNER_VM
, true);
1436 params
.func
= amdgpu_vm_cpu_set_ptes
;
1439 r
= amdgpu_job_alloc_with_ib(adev
, ndw
* 4, &job
);
1443 params
.ib
= &job
->ibs
[0];
1444 params
.func
= amdgpu_vm_do_set_ptes
;
1447 while (!list_empty(&vm
->relocated
)) {
1448 struct amdgpu_vm_pt
*pt
, *entry
;
1450 entry
= list_first_entry(&vm
->relocated
, struct amdgpu_vm_pt
,
1452 amdgpu_vm_bo_idle(&entry
->base
);
1454 pt
= amdgpu_vm_pt_parent(entry
);
1458 amdgpu_vm_update_pde(¶ms
, vm
, pt
, entry
);
1460 if (!vm
->use_cpu_for_update
&&
1461 (ndw
- params
.ib
->length_dw
) < 32)
1465 if (vm
->use_cpu_for_update
) {
1468 amdgpu_asic_flush_hdp(adev
, NULL
);
1469 } else if (params
.ib
->length_dw
== 0) {
1470 amdgpu_job_free(job
);
1472 struct amdgpu_bo
*root
= vm
->root
.base
.bo
;
1473 struct amdgpu_ring
*ring
;
1474 struct dma_fence
*fence
;
1476 ring
= container_of(vm
->entity
.rq
->sched
, struct amdgpu_ring
,
1479 amdgpu_ring_pad_ib(ring
, params
.ib
);
1480 amdgpu_sync_resv(adev
, &job
->sync
, root
->tbo
.resv
,
1481 AMDGPU_FENCE_OWNER_VM
, false);
1482 WARN_ON(params
.ib
->length_dw
> ndw
);
1483 r
= amdgpu_job_submit(job
, &vm
->entity
, AMDGPU_FENCE_OWNER_VM
,
1488 amdgpu_bo_fence(root
, fence
, true);
1489 dma_fence_put(vm
->last_update
);
1490 vm
->last_update
= fence
;
1493 if (!list_empty(&vm
->relocated
))
1499 amdgpu_vm_invalidate_pds(adev
, vm
);
1500 amdgpu_job_free(job
);
1505 * amdgpu_vm_update_flags - figure out flags for PTE updates
1507 * Make sure to set the right flags for the PTEs at the desired level.
1509 static void amdgpu_vm_update_flags(struct amdgpu_pte_update_params
*params
,
1510 struct amdgpu_bo
*bo
, unsigned level
,
1511 uint64_t pe
, uint64_t addr
,
1512 unsigned count
, uint32_t incr
,
1516 if (level
!= AMDGPU_VM_PTB
) {
1517 flags
|= AMDGPU_PDE_PTE
;
1518 amdgpu_gmc_get_vm_pde(params
->adev
, level
, &addr
, &flags
);
1520 } else if (params
->adev
->asic_type
>= CHIP_VEGA10
&&
1521 !(flags
& AMDGPU_PTE_VALID
) &&
1522 !(flags
& AMDGPU_PTE_PRT
)) {
1524 /* Workaround for fault priority problem on GMC9 */
1525 flags
|= AMDGPU_PTE_EXECUTABLE
;
1528 amdgpu_vm_update_func(params
, bo
, pe
, addr
, count
, incr
, flags
);
1532 * amdgpu_vm_fragment - get fragment for PTEs
1534 * @params: see amdgpu_pte_update_params definition
1535 * @start: first PTE to handle
1536 * @end: last PTE to handle
1537 * @flags: hw mapping flags
1538 * @frag: resulting fragment size
1539 * @frag_end: end of this fragment
1541 * Returns the first possible fragment for the start and end address.
1543 static void amdgpu_vm_fragment(struct amdgpu_pte_update_params
*params
,
1544 uint64_t start
, uint64_t end
, uint64_t flags
,
1545 unsigned int *frag
, uint64_t *frag_end
)
1548 * The MC L1 TLB supports variable sized pages, based on a fragment
1549 * field in the PTE. When this field is set to a non-zero value, page
1550 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
1551 * flags are considered valid for all PTEs within the fragment range
1552 * and corresponding mappings are assumed to be physically contiguous.
1554 * The L1 TLB can store a single PTE for the whole fragment,
1555 * significantly increasing the space available for translation
1556 * caching. This leads to large improvements in throughput when the
1557 * TLB is under pressure.
1559 * The L2 TLB distributes small and large fragments into two
1560 * asymmetric partitions. The large fragment cache is significantly
1561 * larger. Thus, we try to use large fragments wherever possible.
1562 * Userspace can support this by aligning virtual base address and
1563 * allocation size to the fragment size.
1565 * Starting with Vega10 the fragment size only controls the L1. The L2
1566 * is now directly feed with small/huge/giant pages from the walker.
1570 if (params
->adev
->asic_type
< CHIP_VEGA10
)
1571 max_frag
= params
->adev
->vm_manager
.fragment_size
;
1575 /* system pages are non continuously */
1582 /* This intentionally wraps around if no bit is set */
1583 *frag
= min((unsigned)ffs(start
) - 1, (unsigned)fls64(end
- start
) - 1);
1584 if (*frag
>= max_frag
) {
1586 *frag_end
= end
& ~((1ULL << max_frag
) - 1);
1588 *frag_end
= start
+ (1 << *frag
);
1593 * amdgpu_vm_update_ptes - make sure that page tables are valid
1595 * @params: see amdgpu_pte_update_params definition
1596 * @start: start of GPU address range
1597 * @end: end of GPU address range
1598 * @dst: destination address to map to, the next dst inside the function
1599 * @flags: mapping flags
1601 * Update the page tables in the range @start - @end.
1604 * 0 for success, -EINVAL for failure.
1606 static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params
*params
,
1607 uint64_t start
, uint64_t end
,
1608 uint64_t dst
, uint64_t flags
)
1610 struct amdgpu_device
*adev
= params
->adev
;
1611 struct amdgpu_vm_pt_cursor cursor
;
1612 uint64_t frag_start
= start
, frag_end
;
1615 /* figure out the initial fragment */
1616 amdgpu_vm_fragment(params
, frag_start
, end
, flags
, &frag
, &frag_end
);
1618 /* walk over the address space and update the PTs */
1619 amdgpu_vm_pt_start(adev
, params
->vm
, start
, &cursor
);
1620 while (cursor
.pfn
< end
) {
1621 struct amdgpu_bo
*pt
= cursor
.entry
->base
.bo
;
1622 unsigned shift
, parent_shift
, mask
;
1623 uint64_t incr
, entry_end
, pe_start
;
1628 /* The root level can't be a huge page */
1629 if (cursor
.level
== adev
->vm_manager
.root_level
) {
1630 if (!amdgpu_vm_pt_descendant(adev
, &cursor
))
1635 /* If it isn't already handled it can't be a huge page */
1636 if (cursor
.entry
->huge
) {
1637 /* Add the entry to the relocated list to update it. */
1638 cursor
.entry
->huge
= false;
1639 amdgpu_vm_bo_relocated(&cursor
.entry
->base
);
1642 shift
= amdgpu_vm_level_shift(adev
, cursor
.level
);
1643 parent_shift
= amdgpu_vm_level_shift(adev
, cursor
.level
- 1);
1644 if (adev
->asic_type
< CHIP_VEGA10
) {
1645 /* No huge page support before GMC v9 */
1646 if (cursor
.level
!= AMDGPU_VM_PTB
) {
1647 if (!amdgpu_vm_pt_descendant(adev
, &cursor
))
1651 } else if (frag
< shift
) {
1652 /* We can't use this level when the fragment size is
1653 * smaller than the address shift. Go to the next
1654 * child entry and try again.
1656 if (!amdgpu_vm_pt_descendant(adev
, &cursor
))
1659 } else if (frag
>= parent_shift
&&
1660 cursor
.level
- 1 != adev
->vm_manager
.root_level
) {
1661 /* If the fragment size is even larger than the parent
1662 * shift we should go up one level and check it again
1663 * unless one level up is the root level.
1665 if (!amdgpu_vm_pt_ancestor(&cursor
))
1670 /* Looks good so far, calculate parameters for the update */
1671 incr
= (uint64_t)AMDGPU_GPU_PAGE_SIZE
<< shift
;
1672 mask
= amdgpu_vm_entries_mask(adev
, cursor
.level
);
1673 pe_start
= ((cursor
.pfn
>> shift
) & mask
) * 8;
1674 entry_end
= (uint64_t)(mask
+ 1) << shift
;
1675 entry_end
+= cursor
.pfn
& ~(entry_end
- 1);
1676 entry_end
= min(entry_end
, end
);
1679 uint64_t upd_end
= min(entry_end
, frag_end
);
1680 unsigned nptes
= (upd_end
- frag_start
) >> shift
;
1682 amdgpu_vm_update_flags(params
, pt
, cursor
.level
,
1683 pe_start
, dst
, nptes
, incr
,
1684 flags
| AMDGPU_PTE_FRAG(frag
));
1686 pe_start
+= nptes
* 8;
1687 dst
+= (uint64_t)nptes
* AMDGPU_GPU_PAGE_SIZE
<< shift
;
1689 frag_start
= upd_end
;
1690 if (frag_start
>= frag_end
) {
1691 /* figure out the next fragment */
1692 amdgpu_vm_fragment(params
, frag_start
, end
,
1693 flags
, &frag
, &frag_end
);
1697 } while (frag_start
< entry_end
);
1699 if (amdgpu_vm_pt_descendant(adev
, &cursor
)) {
1700 /* Mark all child entries as huge */
1701 while (cursor
.pfn
< frag_start
) {
1702 cursor
.entry
->huge
= true;
1703 amdgpu_vm_pt_next(adev
, &cursor
);
1706 } else if (frag
>= shift
) {
1707 /* or just move on to the next on the same level. */
1708 amdgpu_vm_pt_next(adev
, &cursor
);
1716 * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
1718 * @adev: amdgpu_device pointer
1719 * @exclusive: fence we need to sync to
1720 * @pages_addr: DMA addresses to use for mapping
1722 * @start: start of mapped range
1723 * @last: last mapped entry
1724 * @flags: flags for the entries
1725 * @addr: addr to set the area to
1726 * @fence: optional resulting fence
1728 * Fill in the page table entries between @start and @last.
1731 * 0 for success, -EINVAL for failure.
1733 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device
*adev
,
1734 struct dma_fence
*exclusive
,
1735 dma_addr_t
*pages_addr
,
1736 struct amdgpu_vm
*vm
,
1737 uint64_t start
, uint64_t last
,
1738 uint64_t flags
, uint64_t addr
,
1739 struct dma_fence
**fence
)
1741 struct amdgpu_ring
*ring
;
1742 void *owner
= AMDGPU_FENCE_OWNER_VM
;
1743 unsigned nptes
, ncmds
, ndw
;
1744 struct amdgpu_job
*job
;
1745 struct amdgpu_pte_update_params params
;
1746 struct dma_fence
*f
= NULL
;
1749 memset(¶ms
, 0, sizeof(params
));
1753 /* sync to everything except eviction fences on unmapping */
1754 if (!(flags
& AMDGPU_PTE_VALID
))
1755 owner
= AMDGPU_FENCE_OWNER_KFD
;
1757 if (vm
->use_cpu_for_update
) {
1758 /* params.src is used as flag to indicate system Memory */
1762 /* Wait for PT BOs to be idle. PTs share the same resv. object
1765 r
= amdgpu_bo_sync_wait(vm
->root
.base
.bo
, owner
, true);
1769 /* Wait for any BO move to be completed */
1771 r
= dma_fence_wait(exclusive
, true);
1776 params
.func
= amdgpu_vm_cpu_set_ptes
;
1777 params
.pages_addr
= pages_addr
;
1778 return amdgpu_vm_update_ptes(¶ms
, start
, last
+ 1,
1782 ring
= container_of(vm
->entity
.rq
->sched
, struct amdgpu_ring
, sched
);
1784 nptes
= last
- start
+ 1;
1787 * reserve space for two commands every (1 << BLOCK_SIZE)
1788 * entries or 2k dwords (whatever is smaller)
1790 ncmds
= ((nptes
>> min(adev
->vm_manager
.block_size
, 11u)) + 1);
1792 /* The second command is for the shadow pagetables. */
1793 if (vm
->root
.base
.bo
->shadow
)
1800 /* copy commands needed */
1801 ndw
+= ncmds
* adev
->vm_manager
.vm_pte_funcs
->copy_pte_num_dw
;
1806 params
.func
= amdgpu_vm_do_copy_ptes
;
1809 /* set page commands needed */
1812 /* extra commands for begin/end fragments */
1813 ncmds
= 2 * adev
->vm_manager
.fragment_size
;
1814 if (vm
->root
.base
.bo
->shadow
)
1819 params
.func
= amdgpu_vm_do_set_ptes
;
1822 r
= amdgpu_job_alloc_with_ib(adev
, ndw
* 4, &job
);
1826 params
.ib
= &job
->ibs
[0];
1832 /* Put the PTEs at the end of the IB. */
1833 i
= ndw
- nptes
* 2;
1834 pte
= (uint64_t *)&(job
->ibs
->ptr
[i
]);
1835 params
.src
= job
->ibs
->gpu_addr
+ i
* 4;
1837 for (i
= 0; i
< nptes
; ++i
) {
1838 pte
[i
] = amdgpu_vm_map_gart(pages_addr
, addr
+ i
*
1839 AMDGPU_GPU_PAGE_SIZE
);
1845 r
= amdgpu_sync_fence(adev
, &job
->sync
, exclusive
, false);
1849 r
= amdgpu_sync_resv(adev
, &job
->sync
, vm
->root
.base
.bo
->tbo
.resv
,
1854 r
= amdgpu_vm_update_ptes(¶ms
, start
, last
+ 1, addr
, flags
);
1858 amdgpu_ring_pad_ib(ring
, params
.ib
);
1859 WARN_ON(params
.ib
->length_dw
> ndw
);
1860 r
= amdgpu_job_submit(job
, &vm
->entity
, AMDGPU_FENCE_OWNER_VM
, &f
);
1864 amdgpu_bo_fence(vm
->root
.base
.bo
, f
, true);
1865 dma_fence_put(*fence
);
1870 amdgpu_job_free(job
);
1875 * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
1877 * @adev: amdgpu_device pointer
1878 * @exclusive: fence we need to sync to
1879 * @pages_addr: DMA addresses to use for mapping
1881 * @mapping: mapped range and flags to use for the update
1882 * @flags: HW flags for the mapping
1883 * @nodes: array of drm_mm_nodes with the MC addresses
1884 * @fence: optional resulting fence
1886 * Split the mapping into smaller chunks so that each update fits
1890 * 0 for success, -EINVAL for failure.
1892 static int amdgpu_vm_bo_split_mapping(struct amdgpu_device
*adev
,
1893 struct dma_fence
*exclusive
,
1894 dma_addr_t
*pages_addr
,
1895 struct amdgpu_vm
*vm
,
1896 struct amdgpu_bo_va_mapping
*mapping
,
1898 struct drm_mm_node
*nodes
,
1899 struct dma_fence
**fence
)
1901 unsigned min_linear_pages
= 1 << adev
->vm_manager
.fragment_size
;
1902 uint64_t pfn
, start
= mapping
->start
;
1905 /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
1906 * but in case of something, we filter the flags in first place
1908 if (!(mapping
->flags
& AMDGPU_PTE_READABLE
))
1909 flags
&= ~AMDGPU_PTE_READABLE
;
1910 if (!(mapping
->flags
& AMDGPU_PTE_WRITEABLE
))
1911 flags
&= ~AMDGPU_PTE_WRITEABLE
;
1913 flags
&= ~AMDGPU_PTE_EXECUTABLE
;
1914 flags
|= mapping
->flags
& AMDGPU_PTE_EXECUTABLE
;
1916 flags
&= ~AMDGPU_PTE_MTYPE_MASK
;
1917 flags
|= (mapping
->flags
& AMDGPU_PTE_MTYPE_MASK
);
1919 if ((mapping
->flags
& AMDGPU_PTE_PRT
) &&
1920 (adev
->asic_type
>= CHIP_VEGA10
)) {
1921 flags
|= AMDGPU_PTE_PRT
;
1922 flags
&= ~AMDGPU_PTE_VALID
;
1925 trace_amdgpu_vm_bo_update(mapping
);
1927 pfn
= mapping
->offset
>> PAGE_SHIFT
;
1929 while (pfn
>= nodes
->size
) {
1936 dma_addr_t
*dma_addr
= NULL
;
1937 uint64_t max_entries
;
1938 uint64_t addr
, last
;
1941 addr
= nodes
->start
<< PAGE_SHIFT
;
1942 max_entries
= (nodes
->size
- pfn
) *
1943 AMDGPU_GPU_PAGES_IN_CPU_PAGE
;
1946 max_entries
= S64_MAX
;
1952 max_entries
= min(max_entries
, 16ull * 1024ull);
1954 count
< max_entries
/ AMDGPU_GPU_PAGES_IN_CPU_PAGE
;
1956 uint64_t idx
= pfn
+ count
;
1958 if (pages_addr
[idx
] !=
1959 (pages_addr
[idx
- 1] + PAGE_SIZE
))
1963 if (count
< min_linear_pages
) {
1964 addr
= pfn
<< PAGE_SHIFT
;
1965 dma_addr
= pages_addr
;
1967 addr
= pages_addr
[pfn
];
1968 max_entries
= count
* AMDGPU_GPU_PAGES_IN_CPU_PAGE
;
1971 } else if (flags
& AMDGPU_PTE_VALID
) {
1972 addr
+= adev
->vm_manager
.vram_base_offset
;
1973 addr
+= pfn
<< PAGE_SHIFT
;
1976 last
= min((uint64_t)mapping
->last
, start
+ max_entries
- 1);
1977 r
= amdgpu_vm_bo_update_mapping(adev
, exclusive
, dma_addr
, vm
,
1978 start
, last
, flags
, addr
,
1983 pfn
+= (last
- start
+ 1) / AMDGPU_GPU_PAGES_IN_CPU_PAGE
;
1984 if (nodes
&& nodes
->size
== pfn
) {
1990 } while (unlikely(start
!= mapping
->last
+ 1));
1996 * amdgpu_vm_bo_update - update all BO mappings in the vm page table
1998 * @adev: amdgpu_device pointer
1999 * @bo_va: requested BO and VM object
2000 * @clear: if true clear the entries
2002 * Fill in the page table entries for @bo_va.
2005 * 0 for success, -EINVAL for failure.
2007 int amdgpu_vm_bo_update(struct amdgpu_device
*adev
,
2008 struct amdgpu_bo_va
*bo_va
,
2011 struct amdgpu_bo
*bo
= bo_va
->base
.bo
;
2012 struct amdgpu_vm
*vm
= bo_va
->base
.vm
;
2013 struct amdgpu_bo_va_mapping
*mapping
;
2014 dma_addr_t
*pages_addr
= NULL
;
2015 struct ttm_mem_reg
*mem
;
2016 struct drm_mm_node
*nodes
;
2017 struct dma_fence
*exclusive
, **last_update
;
2026 struct ttm_dma_tt
*ttm
;
2029 nodes
= mem
->mm_node
;
2030 if (mem
->mem_type
== TTM_PL_TT
) {
2031 ttm
= container_of(bo
->tbo
.ttm
, struct ttm_dma_tt
, ttm
);
2032 pages_addr
= ttm
->dma_address
;
2034 exclusive
= reservation_object_get_excl(bo
->tbo
.resv
);
2038 flags
= amdgpu_ttm_tt_pte_flags(adev
, bo
->tbo
.ttm
, mem
);
2042 if (clear
|| (bo
&& bo
->tbo
.resv
== vm
->root
.base
.bo
->tbo
.resv
))
2043 last_update
= &vm
->last_update
;
2045 last_update
= &bo_va
->last_pt_update
;
2047 if (!clear
&& bo_va
->base
.moved
) {
2048 bo_va
->base
.moved
= false;
2049 list_splice_init(&bo_va
->valids
, &bo_va
->invalids
);
2051 } else if (bo_va
->cleared
!= clear
) {
2052 list_splice_init(&bo_va
->valids
, &bo_va
->invalids
);
2055 list_for_each_entry(mapping
, &bo_va
->invalids
, list
) {
2056 r
= amdgpu_vm_bo_split_mapping(adev
, exclusive
, pages_addr
, vm
,
2057 mapping
, flags
, nodes
,
2063 if (vm
->use_cpu_for_update
) {
2066 amdgpu_asic_flush_hdp(adev
, NULL
);
2069 /* If the BO is not in its preferred location add it back to
2070 * the evicted list so that it gets validated again on the
2071 * next command submission.
2073 if (bo
&& bo
->tbo
.resv
== vm
->root
.base
.bo
->tbo
.resv
) {
2074 uint32_t mem_type
= bo
->tbo
.mem
.mem_type
;
2076 if (!(bo
->preferred_domains
& amdgpu_mem_type_to_domain(mem_type
)))
2077 amdgpu_vm_bo_evicted(&bo_va
->base
);
2079 amdgpu_vm_bo_idle(&bo_va
->base
);
2081 amdgpu_vm_bo_done(&bo_va
->base
);
2084 list_splice_init(&bo_va
->invalids
, &bo_va
->valids
);
2085 bo_va
->cleared
= clear
;
2087 if (trace_amdgpu_vm_bo_mapping_enabled()) {
2088 list_for_each_entry(mapping
, &bo_va
->valids
, list
)
2089 trace_amdgpu_vm_bo_mapping(mapping
);
2096 * amdgpu_vm_update_prt_state - update the global PRT state
2098 * @adev: amdgpu_device pointer
2100 static void amdgpu_vm_update_prt_state(struct amdgpu_device
*adev
)
2102 unsigned long flags
;
2105 spin_lock_irqsave(&adev
->vm_manager
.prt_lock
, flags
);
2106 enable
= !!atomic_read(&adev
->vm_manager
.num_prt_users
);
2107 adev
->gmc
.gmc_funcs
->set_prt(adev
, enable
);
2108 spin_unlock_irqrestore(&adev
->vm_manager
.prt_lock
, flags
);
2112 * amdgpu_vm_prt_get - add a PRT user
2114 * @adev: amdgpu_device pointer
2116 static void amdgpu_vm_prt_get(struct amdgpu_device
*adev
)
2118 if (!adev
->gmc
.gmc_funcs
->set_prt
)
2121 if (atomic_inc_return(&adev
->vm_manager
.num_prt_users
) == 1)
2122 amdgpu_vm_update_prt_state(adev
);
2126 * amdgpu_vm_prt_put - drop a PRT user
2128 * @adev: amdgpu_device pointer
2130 static void amdgpu_vm_prt_put(struct amdgpu_device
*adev
)
2132 if (atomic_dec_return(&adev
->vm_manager
.num_prt_users
) == 0)
2133 amdgpu_vm_update_prt_state(adev
);
2137 * amdgpu_vm_prt_cb - callback for updating the PRT status
2139 * @fence: fence for the callback
2140 * @_cb: the callback function
2142 static void amdgpu_vm_prt_cb(struct dma_fence
*fence
, struct dma_fence_cb
*_cb
)
2144 struct amdgpu_prt_cb
*cb
= container_of(_cb
, struct amdgpu_prt_cb
, cb
);
2146 amdgpu_vm_prt_put(cb
->adev
);
2151 * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
2153 * @adev: amdgpu_device pointer
2154 * @fence: fence for the callback
2156 static void amdgpu_vm_add_prt_cb(struct amdgpu_device
*adev
,
2157 struct dma_fence
*fence
)
2159 struct amdgpu_prt_cb
*cb
;
2161 if (!adev
->gmc
.gmc_funcs
->set_prt
)
2164 cb
= kmalloc(sizeof(struct amdgpu_prt_cb
), GFP_KERNEL
);
2166 /* Last resort when we are OOM */
2168 dma_fence_wait(fence
, false);
2170 amdgpu_vm_prt_put(adev
);
2173 if (!fence
|| dma_fence_add_callback(fence
, &cb
->cb
,
2175 amdgpu_vm_prt_cb(fence
, &cb
->cb
);
2180 * amdgpu_vm_free_mapping - free a mapping
2182 * @adev: amdgpu_device pointer
2184 * @mapping: mapping to be freed
2185 * @fence: fence of the unmap operation
2187 * Free a mapping and make sure we decrease the PRT usage count if applicable.
2189 static void amdgpu_vm_free_mapping(struct amdgpu_device
*adev
,
2190 struct amdgpu_vm
*vm
,
2191 struct amdgpu_bo_va_mapping
*mapping
,
2192 struct dma_fence
*fence
)
2194 if (mapping
->flags
& AMDGPU_PTE_PRT
)
2195 amdgpu_vm_add_prt_cb(adev
, fence
);
2200 * amdgpu_vm_prt_fini - finish all prt mappings
2202 * @adev: amdgpu_device pointer
2205 * Register a cleanup callback to disable PRT support after VM dies.
2207 static void amdgpu_vm_prt_fini(struct amdgpu_device
*adev
, struct amdgpu_vm
*vm
)
2209 struct reservation_object
*resv
= vm
->root
.base
.bo
->tbo
.resv
;
2210 struct dma_fence
*excl
, **shared
;
2211 unsigned i
, shared_count
;
2214 r
= reservation_object_get_fences_rcu(resv
, &excl
,
2215 &shared_count
, &shared
);
2217 /* Not enough memory to grab the fence list, as last resort
2218 * block for all the fences to complete.
2220 reservation_object_wait_timeout_rcu(resv
, true, false,
2221 MAX_SCHEDULE_TIMEOUT
);
2225 /* Add a callback for each fence in the reservation object */
2226 amdgpu_vm_prt_get(adev
);
2227 amdgpu_vm_add_prt_cb(adev
, excl
);
2229 for (i
= 0; i
< shared_count
; ++i
) {
2230 amdgpu_vm_prt_get(adev
);
2231 amdgpu_vm_add_prt_cb(adev
, shared
[i
]);
2238 * amdgpu_vm_clear_freed - clear freed BOs in the PT
2240 * @adev: amdgpu_device pointer
2242 * @fence: optional resulting fence (unchanged if no work needed to be done
2243 * or if an error occurred)
2245 * Make sure all freed BOs are cleared in the PT.
2246 * PTs have to be reserved and mutex must be locked!
2252 int amdgpu_vm_clear_freed(struct amdgpu_device
*adev
,
2253 struct amdgpu_vm
*vm
,
2254 struct dma_fence
**fence
)
2256 struct amdgpu_bo_va_mapping
*mapping
;
2257 uint64_t init_pte_value
= 0;
2258 struct dma_fence
*f
= NULL
;
2261 while (!list_empty(&vm
->freed
)) {
2262 mapping
= list_first_entry(&vm
->freed
,
2263 struct amdgpu_bo_va_mapping
, list
);
2264 list_del(&mapping
->list
);
2266 if (vm
->pte_support_ats
&&
2267 mapping
->start
< AMDGPU_GMC_HOLE_START
)
2268 init_pte_value
= AMDGPU_PTE_DEFAULT_ATC
;
2270 r
= amdgpu_vm_bo_update_mapping(adev
, NULL
, NULL
, vm
,
2271 mapping
->start
, mapping
->last
,
2272 init_pte_value
, 0, &f
);
2273 amdgpu_vm_free_mapping(adev
, vm
, mapping
, f
);
2281 dma_fence_put(*fence
);
2292 * amdgpu_vm_handle_moved - handle moved BOs in the PT
2294 * @adev: amdgpu_device pointer
2297 * Make sure all BOs which are moved are updated in the PTs.
2302 * PTs have to be reserved!
2304 int amdgpu_vm_handle_moved(struct amdgpu_device
*adev
,
2305 struct amdgpu_vm
*vm
)
2307 struct amdgpu_bo_va
*bo_va
, *tmp
;
2308 struct reservation_object
*resv
;
2312 list_for_each_entry_safe(bo_va
, tmp
, &vm
->moved
, base
.vm_status
) {
2313 /* Per VM BOs never need to bo cleared in the page tables */
2314 r
= amdgpu_vm_bo_update(adev
, bo_va
, false);
2319 spin_lock(&vm
->invalidated_lock
);
2320 while (!list_empty(&vm
->invalidated
)) {
2321 bo_va
= list_first_entry(&vm
->invalidated
, struct amdgpu_bo_va
,
2323 resv
= bo_va
->base
.bo
->tbo
.resv
;
2324 spin_unlock(&vm
->invalidated_lock
);
2326 /* Try to reserve the BO to avoid clearing its ptes */
2327 if (!amdgpu_vm_debug
&& reservation_object_trylock(resv
))
2329 /* Somebody else is using the BO right now */
2333 r
= amdgpu_vm_bo_update(adev
, bo_va
, clear
);
2338 reservation_object_unlock(resv
);
2339 spin_lock(&vm
->invalidated_lock
);
2341 spin_unlock(&vm
->invalidated_lock
);
2347 * amdgpu_vm_bo_add - add a bo to a specific vm
2349 * @adev: amdgpu_device pointer
2351 * @bo: amdgpu buffer object
2353 * Add @bo into the requested vm.
2354 * Add @bo to the list of bos associated with the vm
2357 * Newly added bo_va or NULL for failure
2359 * Object has to be reserved!
2361 struct amdgpu_bo_va
*amdgpu_vm_bo_add(struct amdgpu_device
*adev
,
2362 struct amdgpu_vm
*vm
,
2363 struct amdgpu_bo
*bo
)
2365 struct amdgpu_bo_va
*bo_va
;
2367 bo_va
= kzalloc(sizeof(struct amdgpu_bo_va
), GFP_KERNEL
);
2368 if (bo_va
== NULL
) {
2371 amdgpu_vm_bo_base_init(&bo_va
->base
, vm
, bo
);
2373 bo_va
->ref_count
= 1;
2374 INIT_LIST_HEAD(&bo_va
->valids
);
2375 INIT_LIST_HEAD(&bo_va
->invalids
);
2382 * amdgpu_vm_bo_insert_mapping - insert a new mapping
2384 * @adev: amdgpu_device pointer
2385 * @bo_va: bo_va to store the address
2386 * @mapping: the mapping to insert
2388 * Insert a new mapping into all structures.
2390 static void amdgpu_vm_bo_insert_map(struct amdgpu_device
*adev
,
2391 struct amdgpu_bo_va
*bo_va
,
2392 struct amdgpu_bo_va_mapping
*mapping
)
2394 struct amdgpu_vm
*vm
= bo_va
->base
.vm
;
2395 struct amdgpu_bo
*bo
= bo_va
->base
.bo
;
2397 mapping
->bo_va
= bo_va
;
2398 list_add(&mapping
->list
, &bo_va
->invalids
);
2399 amdgpu_vm_it_insert(mapping
, &vm
->va
);
2401 if (mapping
->flags
& AMDGPU_PTE_PRT
)
2402 amdgpu_vm_prt_get(adev
);
2404 if (bo
&& bo
->tbo
.resv
== vm
->root
.base
.bo
->tbo
.resv
&&
2405 !bo_va
->base
.moved
) {
2406 list_move(&bo_va
->base
.vm_status
, &vm
->moved
);
2408 trace_amdgpu_vm_bo_map(bo_va
, mapping
);
2412 * amdgpu_vm_bo_map - map bo inside a vm
2414 * @adev: amdgpu_device pointer
2415 * @bo_va: bo_va to store the address
2416 * @saddr: where to map the BO
2417 * @offset: requested offset in the BO
2418 * @size: BO size in bytes
2419 * @flags: attributes of pages (read/write/valid/etc.)
2421 * Add a mapping of the BO at the specefied addr into the VM.
2424 * 0 for success, error for failure.
2426 * Object has to be reserved and unreserved outside!
2428 int amdgpu_vm_bo_map(struct amdgpu_device
*adev
,
2429 struct amdgpu_bo_va
*bo_va
,
2430 uint64_t saddr
, uint64_t offset
,
2431 uint64_t size
, uint64_t flags
)
2433 struct amdgpu_bo_va_mapping
*mapping
, *tmp
;
2434 struct amdgpu_bo
*bo
= bo_va
->base
.bo
;
2435 struct amdgpu_vm
*vm
= bo_va
->base
.vm
;
2438 /* validate the parameters */
2439 if (saddr
& AMDGPU_GPU_PAGE_MASK
|| offset
& AMDGPU_GPU_PAGE_MASK
||
2440 size
== 0 || size
& AMDGPU_GPU_PAGE_MASK
)
2443 /* make sure object fit at this offset */
2444 eaddr
= saddr
+ size
- 1;
2445 if (saddr
>= eaddr
||
2446 (bo
&& offset
+ size
> amdgpu_bo_size(bo
)))
2449 saddr
/= AMDGPU_GPU_PAGE_SIZE
;
2450 eaddr
/= AMDGPU_GPU_PAGE_SIZE
;
2452 tmp
= amdgpu_vm_it_iter_first(&vm
->va
, saddr
, eaddr
);
2454 /* bo and tmp overlap, invalid addr */
2455 dev_err(adev
->dev
, "bo %p va 0x%010Lx-0x%010Lx conflict with "
2456 "0x%010Lx-0x%010Lx\n", bo
, saddr
, eaddr
,
2457 tmp
->start
, tmp
->last
+ 1);
2461 mapping
= kmalloc(sizeof(*mapping
), GFP_KERNEL
);
2465 mapping
->start
= saddr
;
2466 mapping
->last
= eaddr
;
2467 mapping
->offset
= offset
;
2468 mapping
->flags
= flags
;
2470 amdgpu_vm_bo_insert_map(adev
, bo_va
, mapping
);
2476 * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
2478 * @adev: amdgpu_device pointer
2479 * @bo_va: bo_va to store the address
2480 * @saddr: where to map the BO
2481 * @offset: requested offset in the BO
2482 * @size: BO size in bytes
2483 * @flags: attributes of pages (read/write/valid/etc.)
2485 * Add a mapping of the BO at the specefied addr into the VM. Replace existing
2486 * mappings as we do so.
2489 * 0 for success, error for failure.
2491 * Object has to be reserved and unreserved outside!
2493 int amdgpu_vm_bo_replace_map(struct amdgpu_device
*adev
,
2494 struct amdgpu_bo_va
*bo_va
,
2495 uint64_t saddr
, uint64_t offset
,
2496 uint64_t size
, uint64_t flags
)
2498 struct amdgpu_bo_va_mapping
*mapping
;
2499 struct amdgpu_bo
*bo
= bo_va
->base
.bo
;
2503 /* validate the parameters */
2504 if (saddr
& AMDGPU_GPU_PAGE_MASK
|| offset
& AMDGPU_GPU_PAGE_MASK
||
2505 size
== 0 || size
& AMDGPU_GPU_PAGE_MASK
)
2508 /* make sure object fit at this offset */
2509 eaddr
= saddr
+ size
- 1;
2510 if (saddr
>= eaddr
||
2511 (bo
&& offset
+ size
> amdgpu_bo_size(bo
)))
2514 /* Allocate all the needed memory */
2515 mapping
= kmalloc(sizeof(*mapping
), GFP_KERNEL
);
2519 r
= amdgpu_vm_bo_clear_mappings(adev
, bo_va
->base
.vm
, saddr
, size
);
2525 saddr
/= AMDGPU_GPU_PAGE_SIZE
;
2526 eaddr
/= AMDGPU_GPU_PAGE_SIZE
;
2528 mapping
->start
= saddr
;
2529 mapping
->last
= eaddr
;
2530 mapping
->offset
= offset
;
2531 mapping
->flags
= flags
;
2533 amdgpu_vm_bo_insert_map(adev
, bo_va
, mapping
);
2539 * amdgpu_vm_bo_unmap - remove bo mapping from vm
2541 * @adev: amdgpu_device pointer
2542 * @bo_va: bo_va to remove the address from
2543 * @saddr: where to the BO is mapped
2545 * Remove a mapping of the BO at the specefied addr from the VM.
2548 * 0 for success, error for failure.
2550 * Object has to be reserved and unreserved outside!
2552 int amdgpu_vm_bo_unmap(struct amdgpu_device
*adev
,
2553 struct amdgpu_bo_va
*bo_va
,
2556 struct amdgpu_bo_va_mapping
*mapping
;
2557 struct amdgpu_vm
*vm
= bo_va
->base
.vm
;
2560 saddr
/= AMDGPU_GPU_PAGE_SIZE
;
2562 list_for_each_entry(mapping
, &bo_va
->valids
, list
) {
2563 if (mapping
->start
== saddr
)
2567 if (&mapping
->list
== &bo_va
->valids
) {
2570 list_for_each_entry(mapping
, &bo_va
->invalids
, list
) {
2571 if (mapping
->start
== saddr
)
2575 if (&mapping
->list
== &bo_va
->invalids
)
2579 list_del(&mapping
->list
);
2580 amdgpu_vm_it_remove(mapping
, &vm
->va
);
2581 mapping
->bo_va
= NULL
;
2582 trace_amdgpu_vm_bo_unmap(bo_va
, mapping
);
2585 list_add(&mapping
->list
, &vm
->freed
);
2587 amdgpu_vm_free_mapping(adev
, vm
, mapping
,
2588 bo_va
->last_pt_update
);
2594 * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
2596 * @adev: amdgpu_device pointer
2597 * @vm: VM structure to use
2598 * @saddr: start of the range
2599 * @size: size of the range
2601 * Remove all mappings in a range, split them as appropriate.
2604 * 0 for success, error for failure.
2606 int amdgpu_vm_bo_clear_mappings(struct amdgpu_device
*adev
,
2607 struct amdgpu_vm
*vm
,
2608 uint64_t saddr
, uint64_t size
)
2610 struct amdgpu_bo_va_mapping
*before
, *after
, *tmp
, *next
;
2614 eaddr
= saddr
+ size
- 1;
2615 saddr
/= AMDGPU_GPU_PAGE_SIZE
;
2616 eaddr
/= AMDGPU_GPU_PAGE_SIZE
;
2618 /* Allocate all the needed memory */
2619 before
= kzalloc(sizeof(*before
), GFP_KERNEL
);
2622 INIT_LIST_HEAD(&before
->list
);
2624 after
= kzalloc(sizeof(*after
), GFP_KERNEL
);
2629 INIT_LIST_HEAD(&after
->list
);
2631 /* Now gather all removed mappings */
2632 tmp
= amdgpu_vm_it_iter_first(&vm
->va
, saddr
, eaddr
);
2634 /* Remember mapping split at the start */
2635 if (tmp
->start
< saddr
) {
2636 before
->start
= tmp
->start
;
2637 before
->last
= saddr
- 1;
2638 before
->offset
= tmp
->offset
;
2639 before
->flags
= tmp
->flags
;
2640 before
->bo_va
= tmp
->bo_va
;
2641 list_add(&before
->list
, &tmp
->bo_va
->invalids
);
2644 /* Remember mapping split at the end */
2645 if (tmp
->last
> eaddr
) {
2646 after
->start
= eaddr
+ 1;
2647 after
->last
= tmp
->last
;
2648 after
->offset
= tmp
->offset
;
2649 after
->offset
+= after
->start
- tmp
->start
;
2650 after
->flags
= tmp
->flags
;
2651 after
->bo_va
= tmp
->bo_va
;
2652 list_add(&after
->list
, &tmp
->bo_va
->invalids
);
2655 list_del(&tmp
->list
);
2656 list_add(&tmp
->list
, &removed
);
2658 tmp
= amdgpu_vm_it_iter_next(tmp
, saddr
, eaddr
);
2661 /* And free them up */
2662 list_for_each_entry_safe(tmp
, next
, &removed
, list
) {
2663 amdgpu_vm_it_remove(tmp
, &vm
->va
);
2664 list_del(&tmp
->list
);
2666 if (tmp
->start
< saddr
)
2668 if (tmp
->last
> eaddr
)
2672 list_add(&tmp
->list
, &vm
->freed
);
2673 trace_amdgpu_vm_bo_unmap(NULL
, tmp
);
2676 /* Insert partial mapping before the range */
2677 if (!list_empty(&before
->list
)) {
2678 amdgpu_vm_it_insert(before
, &vm
->va
);
2679 if (before
->flags
& AMDGPU_PTE_PRT
)
2680 amdgpu_vm_prt_get(adev
);
2685 /* Insert partial mapping after the range */
2686 if (!list_empty(&after
->list
)) {
2687 amdgpu_vm_it_insert(after
, &vm
->va
);
2688 if (after
->flags
& AMDGPU_PTE_PRT
)
2689 amdgpu_vm_prt_get(adev
);
2698 * amdgpu_vm_bo_lookup_mapping - find mapping by address
2700 * @vm: the requested VM
2701 * @addr: the address
2703 * Find a mapping by it's address.
2706 * The amdgpu_bo_va_mapping matching for addr or NULL
2709 struct amdgpu_bo_va_mapping
*amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm
*vm
,
2712 return amdgpu_vm_it_iter_first(&vm
->va
, addr
, addr
);
2716 * amdgpu_vm_bo_trace_cs - trace all reserved mappings
2718 * @vm: the requested vm
2719 * @ticket: CS ticket
2721 * Trace all mappings of BOs reserved during a command submission.
2723 void amdgpu_vm_bo_trace_cs(struct amdgpu_vm
*vm
, struct ww_acquire_ctx
*ticket
)
2725 struct amdgpu_bo_va_mapping
*mapping
;
2727 if (!trace_amdgpu_vm_bo_cs_enabled())
2730 for (mapping
= amdgpu_vm_it_iter_first(&vm
->va
, 0, U64_MAX
); mapping
;
2731 mapping
= amdgpu_vm_it_iter_next(mapping
, 0, U64_MAX
)) {
2732 if (mapping
->bo_va
&& mapping
->bo_va
->base
.bo
) {
2733 struct amdgpu_bo
*bo
;
2735 bo
= mapping
->bo_va
->base
.bo
;
2736 if (READ_ONCE(bo
->tbo
.resv
->lock
.ctx
) != ticket
)
2740 trace_amdgpu_vm_bo_cs(mapping
);
2745 * amdgpu_vm_bo_rmv - remove a bo to a specific vm
2747 * @adev: amdgpu_device pointer
2748 * @bo_va: requested bo_va
2750 * Remove @bo_va->bo from the requested vm.
2752 * Object have to be reserved!
2754 void amdgpu_vm_bo_rmv(struct amdgpu_device
*adev
,
2755 struct amdgpu_bo_va
*bo_va
)
2757 struct amdgpu_bo_va_mapping
*mapping
, *next
;
2758 struct amdgpu_bo
*bo
= bo_va
->base
.bo
;
2759 struct amdgpu_vm
*vm
= bo_va
->base
.vm
;
2760 struct amdgpu_vm_bo_base
**base
;
2763 if (bo
->tbo
.resv
== vm
->root
.base
.bo
->tbo
.resv
)
2764 vm
->bulk_moveable
= false;
2766 for (base
= &bo_va
->base
.bo
->vm_bo
; *base
;
2767 base
= &(*base
)->next
) {
2768 if (*base
!= &bo_va
->base
)
2771 *base
= bo_va
->base
.next
;
2776 spin_lock(&vm
->invalidated_lock
);
2777 list_del(&bo_va
->base
.vm_status
);
2778 spin_unlock(&vm
->invalidated_lock
);
2780 list_for_each_entry_safe(mapping
, next
, &bo_va
->valids
, list
) {
2781 list_del(&mapping
->list
);
2782 amdgpu_vm_it_remove(mapping
, &vm
->va
);
2783 mapping
->bo_va
= NULL
;
2784 trace_amdgpu_vm_bo_unmap(bo_va
, mapping
);
2785 list_add(&mapping
->list
, &vm
->freed
);
2787 list_for_each_entry_safe(mapping
, next
, &bo_va
->invalids
, list
) {
2788 list_del(&mapping
->list
);
2789 amdgpu_vm_it_remove(mapping
, &vm
->va
);
2790 amdgpu_vm_free_mapping(adev
, vm
, mapping
,
2791 bo_va
->last_pt_update
);
2794 dma_fence_put(bo_va
->last_pt_update
);
2799 * amdgpu_vm_bo_invalidate - mark the bo as invalid
2801 * @adev: amdgpu_device pointer
2802 * @bo: amdgpu buffer object
2803 * @evicted: is the BO evicted
2805 * Mark @bo as invalid.
2807 void amdgpu_vm_bo_invalidate(struct amdgpu_device
*adev
,
2808 struct amdgpu_bo
*bo
, bool evicted
)
2810 struct amdgpu_vm_bo_base
*bo_base
;
2812 /* shadow bo doesn't have bo base, its validation needs its parent */
2813 if (bo
->parent
&& bo
->parent
->shadow
== bo
)
2816 for (bo_base
= bo
->vm_bo
; bo_base
; bo_base
= bo_base
->next
) {
2817 struct amdgpu_vm
*vm
= bo_base
->vm
;
2819 if (evicted
&& bo
->tbo
.resv
== vm
->root
.base
.bo
->tbo
.resv
) {
2820 amdgpu_vm_bo_evicted(bo_base
);
2826 bo_base
->moved
= true;
2828 if (bo
->tbo
.type
== ttm_bo_type_kernel
)
2829 amdgpu_vm_bo_relocated(bo_base
);
2830 else if (bo
->tbo
.resv
== vm
->root
.base
.bo
->tbo
.resv
)
2831 amdgpu_vm_bo_moved(bo_base
);
2833 amdgpu_vm_bo_invalidated(bo_base
);
2838 * amdgpu_vm_get_block_size - calculate VM page table size as power of two
2843 * VM page table as power of two
2845 static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size
)
2847 /* Total bits covered by PD + PTs */
2848 unsigned bits
= ilog2(vm_size
) + 18;
2850 /* Make sure the PD is 4K in size up to 8GB address space.
2851 Above that split equal between PD and PTs */
2855 return ((bits
+ 3) / 2);
2859 * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
2861 * @adev: amdgpu_device pointer
2862 * @min_vm_size: the minimum vm size in GB if it's set auto
2863 * @fragment_size_default: Default PTE fragment size
2864 * @max_level: max VMPT level
2865 * @max_bits: max address space size in bits
2868 void amdgpu_vm_adjust_size(struct amdgpu_device
*adev
, uint32_t min_vm_size
,
2869 uint32_t fragment_size_default
, unsigned max_level
,
2872 unsigned int max_size
= 1 << (max_bits
- 30);
2873 unsigned int vm_size
;
2876 /* adjust vm size first */
2877 if (amdgpu_vm_size
!= -1) {
2878 vm_size
= amdgpu_vm_size
;
2879 if (vm_size
> max_size
) {
2880 dev_warn(adev
->dev
, "VM size (%d) too large, max is %u GB\n",
2881 amdgpu_vm_size
, max_size
);
2886 unsigned int phys_ram_gb
;
2888 /* Optimal VM size depends on the amount of physical
2889 * RAM available. Underlying requirements and
2892 * - Need to map system memory and VRAM from all GPUs
2893 * - VRAM from other GPUs not known here
2894 * - Assume VRAM <= system memory
2895 * - On GFX8 and older, VM space can be segmented for
2897 * - Need to allow room for fragmentation, guard pages etc.
2899 * This adds up to a rough guess of system memory x3.
2900 * Round up to power of two to maximize the available
2901 * VM size with the given page table size.
2904 phys_ram_gb
= ((uint64_t)si
.totalram
* si
.mem_unit
+
2905 (1 << 30) - 1) >> 30;
2906 vm_size
= roundup_pow_of_two(
2907 min(max(phys_ram_gb
* 3, min_vm_size
), max_size
));
2910 adev
->vm_manager
.max_pfn
= (uint64_t)vm_size
<< 18;
2912 tmp
= roundup_pow_of_two(adev
->vm_manager
.max_pfn
);
2913 if (amdgpu_vm_block_size
!= -1)
2914 tmp
>>= amdgpu_vm_block_size
- 9;
2915 tmp
= DIV_ROUND_UP(fls64(tmp
) - 1, 9) - 1;
2916 adev
->vm_manager
.num_level
= min(max_level
, (unsigned)tmp
);
2917 switch (adev
->vm_manager
.num_level
) {
2919 adev
->vm_manager
.root_level
= AMDGPU_VM_PDB2
;
2922 adev
->vm_manager
.root_level
= AMDGPU_VM_PDB1
;
2925 adev
->vm_manager
.root_level
= AMDGPU_VM_PDB0
;
2928 dev_err(adev
->dev
, "VMPT only supports 2~4+1 levels\n");
2930 /* block size depends on vm size and hw setup*/
2931 if (amdgpu_vm_block_size
!= -1)
2932 adev
->vm_manager
.block_size
=
2933 min((unsigned)amdgpu_vm_block_size
, max_bits
2934 - AMDGPU_GPU_PAGE_SHIFT
2935 - 9 * adev
->vm_manager
.num_level
);
2936 else if (adev
->vm_manager
.num_level
> 1)
2937 adev
->vm_manager
.block_size
= 9;
2939 adev
->vm_manager
.block_size
= amdgpu_vm_get_block_size(tmp
);
2941 if (amdgpu_vm_fragment_size
== -1)
2942 adev
->vm_manager
.fragment_size
= fragment_size_default
;
2944 adev
->vm_manager
.fragment_size
= amdgpu_vm_fragment_size
;
2946 DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n",
2947 vm_size
, adev
->vm_manager
.num_level
+ 1,
2948 adev
->vm_manager
.block_size
,
2949 adev
->vm_manager
.fragment_size
);
2952 static struct amdgpu_retryfault_hashtable
*init_fault_hash(void)
2954 struct amdgpu_retryfault_hashtable
*fault_hash
;
2956 fault_hash
= kmalloc(sizeof(*fault_hash
), GFP_KERNEL
);
2960 INIT_CHASH_TABLE(fault_hash
->hash
,
2961 AMDGPU_PAGEFAULT_HASH_BITS
, 8, 0);
2962 spin_lock_init(&fault_hash
->lock
);
2963 fault_hash
->count
= 0;
2969 * amdgpu_vm_init - initialize a vm instance
2971 * @adev: amdgpu_device pointer
2973 * @vm_context: Indicates if it GFX or Compute context
2974 * @pasid: Process address space identifier
2979 * 0 for success, error for failure.
2981 int amdgpu_vm_init(struct amdgpu_device
*adev
, struct amdgpu_vm
*vm
,
2982 int vm_context
, unsigned int pasid
)
2984 struct amdgpu_bo_param bp
;
2985 struct amdgpu_bo
*root
;
2988 vm
->va
= RB_ROOT_CACHED
;
2989 for (i
= 0; i
< AMDGPU_MAX_VMHUBS
; i
++)
2990 vm
->reserved_vmid
[i
] = NULL
;
2991 INIT_LIST_HEAD(&vm
->evicted
);
2992 INIT_LIST_HEAD(&vm
->relocated
);
2993 INIT_LIST_HEAD(&vm
->moved
);
2994 INIT_LIST_HEAD(&vm
->idle
);
2995 INIT_LIST_HEAD(&vm
->invalidated
);
2996 spin_lock_init(&vm
->invalidated_lock
);
2997 INIT_LIST_HEAD(&vm
->freed
);
2999 /* create scheduler entity for page table updates */
3000 r
= drm_sched_entity_init(&vm
->entity
, adev
->vm_manager
.vm_pte_rqs
,
3001 adev
->vm_manager
.vm_pte_num_rqs
, NULL
);
3005 vm
->pte_support_ats
= false;
3007 if (vm_context
== AMDGPU_VM_CONTEXT_COMPUTE
) {
3008 vm
->use_cpu_for_update
= !!(adev
->vm_manager
.vm_update_mode
&
3009 AMDGPU_VM_USE_CPU_FOR_COMPUTE
);
3011 if (adev
->asic_type
== CHIP_RAVEN
)
3012 vm
->pte_support_ats
= true;
3014 vm
->use_cpu_for_update
= !!(adev
->vm_manager
.vm_update_mode
&
3015 AMDGPU_VM_USE_CPU_FOR_GFX
);
3017 DRM_DEBUG_DRIVER("VM update mode is %s\n",
3018 vm
->use_cpu_for_update
? "CPU" : "SDMA");
3019 WARN_ONCE((vm
->use_cpu_for_update
&& !amdgpu_gmc_vram_full_visible(&adev
->gmc
)),
3020 "CPU update of VM recommended only for large BAR system\n");
3021 vm
->last_update
= NULL
;
3023 amdgpu_vm_bo_param(adev
, vm
, adev
->vm_manager
.root_level
, &bp
);
3024 if (vm_context
== AMDGPU_VM_CONTEXT_COMPUTE
)
3025 bp
.flags
&= ~AMDGPU_GEM_CREATE_SHADOW
;
3026 r
= amdgpu_bo_create(adev
, &bp
, &root
);
3028 goto error_free_sched_entity
;
3030 r
= amdgpu_bo_reserve(root
, true);
3032 goto error_free_root
;
3034 r
= reservation_object_reserve_shared(root
->tbo
.resv
, 1);
3036 goto error_unreserve
;
3038 amdgpu_vm_bo_base_init(&vm
->root
.base
, vm
, root
);
3040 r
= amdgpu_vm_clear_bo(adev
, vm
, root
,
3041 adev
->vm_manager
.root_level
,
3042 vm
->pte_support_ats
);
3044 goto error_unreserve
;
3046 amdgpu_bo_unreserve(vm
->root
.base
.bo
);
3049 unsigned long flags
;
3051 spin_lock_irqsave(&adev
->vm_manager
.pasid_lock
, flags
);
3052 r
= idr_alloc(&adev
->vm_manager
.pasid_idr
, vm
, pasid
, pasid
+ 1,
3054 spin_unlock_irqrestore(&adev
->vm_manager
.pasid_lock
, flags
);
3056 goto error_free_root
;
3061 vm
->fault_hash
= init_fault_hash();
3062 if (!vm
->fault_hash
) {
3064 goto error_free_root
;
3067 INIT_KFIFO(vm
->faults
);
3072 amdgpu_bo_unreserve(vm
->root
.base
.bo
);
3075 amdgpu_bo_unref(&vm
->root
.base
.bo
->shadow
);
3076 amdgpu_bo_unref(&vm
->root
.base
.bo
);
3077 vm
->root
.base
.bo
= NULL
;
3079 error_free_sched_entity
:
3080 drm_sched_entity_destroy(&vm
->entity
);
3086 * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM
3088 * @adev: amdgpu_device pointer
3091 * This only works on GFX VMs that don't have any BOs added and no
3092 * page tables allocated yet.
3094 * Changes the following VM parameters:
3095 * - use_cpu_for_update
3096 * - pte_supports_ats
3097 * - pasid (old PASID is released, because compute manages its own PASIDs)
3099 * Reinitializes the page directory to reflect the changed ATS
3103 * 0 for success, -errno for errors.
3105 int amdgpu_vm_make_compute(struct amdgpu_device
*adev
, struct amdgpu_vm
*vm
, unsigned int pasid
)
3107 bool pte_support_ats
= (adev
->asic_type
== CHIP_RAVEN
);
3110 r
= amdgpu_bo_reserve(vm
->root
.base
.bo
, true);
3115 if (!RB_EMPTY_ROOT(&vm
->va
.rb_root
) || vm
->root
.entries
) {
3121 unsigned long flags
;
3123 spin_lock_irqsave(&adev
->vm_manager
.pasid_lock
, flags
);
3124 r
= idr_alloc(&adev
->vm_manager
.pasid_idr
, vm
, pasid
, pasid
+ 1,
3126 spin_unlock_irqrestore(&adev
->vm_manager
.pasid_lock
, flags
);
3133 /* Check if PD needs to be reinitialized and do it before
3134 * changing any other state, in case it fails.
3136 if (pte_support_ats
!= vm
->pte_support_ats
) {
3137 r
= amdgpu_vm_clear_bo(adev
, vm
, vm
->root
.base
.bo
,
3138 adev
->vm_manager
.root_level
,
3144 /* Update VM state */
3145 vm
->use_cpu_for_update
= !!(adev
->vm_manager
.vm_update_mode
&
3146 AMDGPU_VM_USE_CPU_FOR_COMPUTE
);
3147 vm
->pte_support_ats
= pte_support_ats
;
3148 DRM_DEBUG_DRIVER("VM update mode is %s\n",
3149 vm
->use_cpu_for_update
? "CPU" : "SDMA");
3150 WARN_ONCE((vm
->use_cpu_for_update
&& !amdgpu_gmc_vram_full_visible(&adev
->gmc
)),
3151 "CPU update of VM recommended only for large BAR system\n");
3154 unsigned long flags
;
3156 spin_lock_irqsave(&adev
->vm_manager
.pasid_lock
, flags
);
3157 idr_remove(&adev
->vm_manager
.pasid_idr
, vm
->pasid
);
3158 spin_unlock_irqrestore(&adev
->vm_manager
.pasid_lock
, flags
);
3160 /* Free the original amdgpu allocated pasid
3161 * Will be replaced with kfd allocated pasid
3163 amdgpu_pasid_free(vm
->pasid
);
3167 /* Free the shadow bo for compute VM */
3168 amdgpu_bo_unref(&vm
->root
.base
.bo
->shadow
);
3177 unsigned long flags
;
3179 spin_lock_irqsave(&adev
->vm_manager
.pasid_lock
, flags
);
3180 idr_remove(&adev
->vm_manager
.pasid_idr
, pasid
);
3181 spin_unlock_irqrestore(&adev
->vm_manager
.pasid_lock
, flags
);
3184 amdgpu_bo_unreserve(vm
->root
.base
.bo
);
3189 * amdgpu_vm_release_compute - release a compute vm
3190 * @adev: amdgpu_device pointer
3191 * @vm: a vm turned into compute vm by calling amdgpu_vm_make_compute
3193 * This is a correspondant of amdgpu_vm_make_compute. It decouples compute
3194 * pasid from vm. Compute should stop use of vm after this call.
3196 void amdgpu_vm_release_compute(struct amdgpu_device
*adev
, struct amdgpu_vm
*vm
)
3199 unsigned long flags
;
3201 spin_lock_irqsave(&adev
->vm_manager
.pasid_lock
, flags
);
3202 idr_remove(&adev
->vm_manager
.pasid_idr
, vm
->pasid
);
3203 spin_unlock_irqrestore(&adev
->vm_manager
.pasid_lock
, flags
);
3209 * amdgpu_vm_fini - tear down a vm instance
3211 * @adev: amdgpu_device pointer
3215 * Unbind the VM and remove all bos from the vm bo list
3217 void amdgpu_vm_fini(struct amdgpu_device
*adev
, struct amdgpu_vm
*vm
)
3219 struct amdgpu_bo_va_mapping
*mapping
, *tmp
;
3220 bool prt_fini_needed
= !!adev
->gmc
.gmc_funcs
->set_prt
;
3221 struct amdgpu_bo
*root
;
3225 amdgpu_amdkfd_gpuvm_destroy_cb(adev
, vm
);
3227 /* Clear pending page faults from IH when the VM is destroyed */
3228 while (kfifo_get(&vm
->faults
, &fault
))
3229 amdgpu_vm_clear_fault(vm
->fault_hash
, fault
);
3232 unsigned long flags
;
3234 spin_lock_irqsave(&adev
->vm_manager
.pasid_lock
, flags
);
3235 idr_remove(&adev
->vm_manager
.pasid_idr
, vm
->pasid
);
3236 spin_unlock_irqrestore(&adev
->vm_manager
.pasid_lock
, flags
);
3239 kfree(vm
->fault_hash
);
3240 vm
->fault_hash
= NULL
;
3242 drm_sched_entity_destroy(&vm
->entity
);
3244 if (!RB_EMPTY_ROOT(&vm
->va
.rb_root
)) {
3245 dev_err(adev
->dev
, "still active bo inside vm\n");
3247 rbtree_postorder_for_each_entry_safe(mapping
, tmp
,
3248 &vm
->va
.rb_root
, rb
) {
3249 /* Don't remove the mapping here, we don't want to trigger a
3250 * rebalance and the tree is about to be destroyed anyway.
3252 list_del(&mapping
->list
);
3255 list_for_each_entry_safe(mapping
, tmp
, &vm
->freed
, list
) {
3256 if (mapping
->flags
& AMDGPU_PTE_PRT
&& prt_fini_needed
) {
3257 amdgpu_vm_prt_fini(adev
, vm
);
3258 prt_fini_needed
= false;
3261 list_del(&mapping
->list
);
3262 amdgpu_vm_free_mapping(adev
, vm
, mapping
, NULL
);
3265 root
= amdgpu_bo_ref(vm
->root
.base
.bo
);
3266 r
= amdgpu_bo_reserve(root
, true);
3268 dev_err(adev
->dev
, "Leaking page tables because BO reservation failed\n");
3270 amdgpu_vm_free_pts(adev
, vm
);
3271 amdgpu_bo_unreserve(root
);
3273 amdgpu_bo_unref(&root
);
3274 dma_fence_put(vm
->last_update
);
3275 for (i
= 0; i
< AMDGPU_MAX_VMHUBS
; i
++)
3276 amdgpu_vmid_free_reserved(adev
, vm
, i
);
3280 * amdgpu_vm_manager_init - init the VM manager
3282 * @adev: amdgpu_device pointer
3284 * Initialize the VM manager structures
3286 void amdgpu_vm_manager_init(struct amdgpu_device
*adev
)
3290 amdgpu_vmid_mgr_init(adev
);
3292 adev
->vm_manager
.fence_context
=
3293 dma_fence_context_alloc(AMDGPU_MAX_RINGS
);
3294 for (i
= 0; i
< AMDGPU_MAX_RINGS
; ++i
)
3295 adev
->vm_manager
.seqno
[i
] = 0;
3297 spin_lock_init(&adev
->vm_manager
.prt_lock
);
3298 atomic_set(&adev
->vm_manager
.num_prt_users
, 0);
3300 /* If not overridden by the user, by default, only in large BAR systems
3301 * Compute VM tables will be updated by CPU
3303 #ifdef CONFIG_X86_64
3304 if (amdgpu_vm_update_mode
== -1) {
3305 if (amdgpu_gmc_vram_full_visible(&adev
->gmc
))
3306 adev
->vm_manager
.vm_update_mode
=
3307 AMDGPU_VM_USE_CPU_FOR_COMPUTE
;
3309 adev
->vm_manager
.vm_update_mode
= 0;
3311 adev
->vm_manager
.vm_update_mode
= amdgpu_vm_update_mode
;
3313 adev
->vm_manager
.vm_update_mode
= 0;
3316 idr_init(&adev
->vm_manager
.pasid_idr
);
3317 spin_lock_init(&adev
->vm_manager
.pasid_lock
);
3321 * amdgpu_vm_manager_fini - cleanup VM manager
3323 * @adev: amdgpu_device pointer
3325 * Cleanup the VM manager and free resources.
3327 void amdgpu_vm_manager_fini(struct amdgpu_device
*adev
)
3329 WARN_ON(!idr_is_empty(&adev
->vm_manager
.pasid_idr
));
3330 idr_destroy(&adev
->vm_manager
.pasid_idr
);
3332 amdgpu_vmid_mgr_fini(adev
);
3336 * amdgpu_vm_ioctl - Manages VMID reservation for vm hubs.
3338 * @dev: drm device pointer
3339 * @data: drm_amdgpu_vm
3340 * @filp: drm file pointer
3343 * 0 for success, -errno for errors.
3345 int amdgpu_vm_ioctl(struct drm_device
*dev
, void *data
, struct drm_file
*filp
)
3347 union drm_amdgpu_vm
*args
= data
;
3348 struct amdgpu_device
*adev
= dev
->dev_private
;
3349 struct amdgpu_fpriv
*fpriv
= filp
->driver_priv
;
3352 switch (args
->in
.op
) {
3353 case AMDGPU_VM_OP_RESERVE_VMID
:
3354 /* current, we only have requirement to reserve vmid from gfxhub */
3355 r
= amdgpu_vmid_alloc_reserved(adev
, &fpriv
->vm
, AMDGPU_GFXHUB
);
3359 case AMDGPU_VM_OP_UNRESERVE_VMID
:
3360 amdgpu_vmid_free_reserved(adev
, &fpriv
->vm
, AMDGPU_GFXHUB
);
3370 * amdgpu_vm_get_task_info - Extracts task info for a PASID.
3372 * @adev: drm device pointer
3373 * @pasid: PASID identifier for VM
3374 * @task_info: task_info to fill.
3376 void amdgpu_vm_get_task_info(struct amdgpu_device
*adev
, unsigned int pasid
,
3377 struct amdgpu_task_info
*task_info
)
3379 struct amdgpu_vm
*vm
;
3380 unsigned long flags
;
3382 spin_lock_irqsave(&adev
->vm_manager
.pasid_lock
, flags
);
3384 vm
= idr_find(&adev
->vm_manager
.pasid_idr
, pasid
);
3386 *task_info
= vm
->task_info
;
3388 spin_unlock_irqrestore(&adev
->vm_manager
.pasid_lock
, flags
);
3392 * amdgpu_vm_set_task_info - Sets VMs task info.
3394 * @vm: vm for which to set the info
3396 void amdgpu_vm_set_task_info(struct amdgpu_vm
*vm
)
3398 if (!vm
->task_info
.pid
) {
3399 vm
->task_info
.pid
= current
->pid
;
3400 get_task_comm(vm
->task_info
.task_name
, current
);
3402 if (current
->group_leader
->mm
== current
->mm
) {
3403 vm
->task_info
.tgid
= current
->group_leader
->pid
;
3404 get_task_comm(vm
->task_info
.process_name
, current
->group_leader
);
3410 * amdgpu_vm_add_fault - Add a page fault record to fault hash table
3412 * @fault_hash: fault hash table
3413 * @key: 64-bit encoding of PASID and address
3415 * This should be called when a retry page fault interrupt is
3416 * received. If this is a new page fault, it will be added to a hash
3417 * table. The return value indicates whether this is a new fault, or
3418 * a fault that was already known and is already being handled.
3420 * If there are too many pending page faults, this will fail. Retry
3421 * interrupts should be ignored in this case until there is enough
3424 * Returns 0 if the fault was added, 1 if the fault was already known,
3425 * -ENOSPC if there are too many pending faults.
3427 int amdgpu_vm_add_fault(struct amdgpu_retryfault_hashtable
*fault_hash
, u64 key
)
3429 unsigned long flags
;
3432 if (WARN_ON_ONCE(!fault_hash
))
3433 /* Should be allocated in amdgpu_vm_init
3437 spin_lock_irqsave(&fault_hash
->lock
, flags
);
3439 /* Only let the hash table fill up to 50% for best performance */
3440 if (fault_hash
->count
>= (1 << (AMDGPU_PAGEFAULT_HASH_BITS
-1)))
3443 r
= chash_table_copy_in(&fault_hash
->hash
, key
, NULL
);
3445 fault_hash
->count
++;
3447 /* chash_table_copy_in should never fail unless we're losing count */
3448 WARN_ON_ONCE(r
< 0);
3451 spin_unlock_irqrestore(&fault_hash
->lock
, flags
);
3456 * amdgpu_vm_clear_fault - Remove a page fault record
3458 * @fault_hash: fault hash table
3459 * @key: 64-bit encoding of PASID and address
3461 * This should be called when a page fault has been handled. Any
3462 * future interrupt with this key will be processed as a new
3465 void amdgpu_vm_clear_fault(struct amdgpu_retryfault_hashtable
*fault_hash
, u64 key
)
3467 unsigned long flags
;
3473 spin_lock_irqsave(&fault_hash
->lock
, flags
);
3475 r
= chash_table_remove(&fault_hash
->hash
, key
, NULL
);
3476 if (!WARN_ON_ONCE(r
< 0)) {
3477 fault_hash
->count
--;
3478 WARN_ON_ONCE(fault_hash
->count
< 0);
3481 spin_unlock_irqrestore(&fault_hash
->lock
, flags
);