2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
24 * Authors: Dave Airlie
28 #include <linux/dma-fence-array.h>
30 #include <drm/amdgpu_drm.h>
32 #include "amdgpu_trace.h"
36 * GPUVM is similar to the legacy gart on older asics, however
37 * rather than there being a single global gart table
38 * for the entire GPU, there are multiple VM page tables active
39 * at any given time. The VM page tables can contain a mix
40 * vram pages and system memory pages and system memory pages
41 * can be mapped as snooped (cached system pages) or unsnooped
42 * (uncached system pages).
43 * Each VM has an ID associated with it and there is a page table
44 * associated with each VMID. When execting a command buffer,
45 * the kernel tells the the ring what VMID to use for that command
46 * buffer. VMIDs are allocated dynamically as commands are submitted.
47 * The userspace drivers maintain their own address space and the kernel
48 * sets up their pages tables accordingly when they submit their
49 * command buffers and a VMID is assigned.
50 * Cayman/Trinity support up to 8 active VMs at any given time;
54 /* Local structure. Encapsulate some VM table update parameters to reduce
55 * the number of function parameters
57 struct amdgpu_pte_update_params
{
58 /* amdgpu device we do this update for */
59 struct amdgpu_device
*adev
;
60 /* address where to copy page table entries from */
62 /* indirect buffer to fill with commands */
64 /* Function which actually does the update */
65 void (*func
)(struct amdgpu_pte_update_params
*params
, uint64_t pe
,
66 uint64_t addr
, unsigned count
, uint32_t incr
,
68 /* indicate update pt or its shadow */
73 * amdgpu_vm_num_pde - return the number of page directory entries
75 * @adev: amdgpu_device pointer
77 * Calculate the number of page directory entries.
79 static unsigned amdgpu_vm_num_pdes(struct amdgpu_device
*adev
)
81 return adev
->vm_manager
.max_pfn
>> amdgpu_vm_block_size
;
85 * amdgpu_vm_directory_size - returns the size of the page directory in bytes
87 * @adev: amdgpu_device pointer
89 * Calculate the size of the page directory in bytes.
91 static unsigned amdgpu_vm_directory_size(struct amdgpu_device
*adev
)
93 return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_pdes(adev
) * 8);
97 * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
99 * @vm: vm providing the BOs
100 * @validated: head of validation list
101 * @entry: entry to add
103 * Add the page directory to the list of BOs to
104 * validate for command submission.
106 void amdgpu_vm_get_pd_bo(struct amdgpu_vm
*vm
,
107 struct list_head
*validated
,
108 struct amdgpu_bo_list_entry
*entry
)
110 entry
->robj
= vm
->page_directory
;
112 entry
->tv
.bo
= &vm
->page_directory
->tbo
;
113 entry
->tv
.shared
= true;
114 entry
->user_pages
= NULL
;
115 list_add(&entry
->tv
.head
, validated
);
119 * amdgpu_vm_validate_pt_bos - validate the page table BOs
121 * @adev: amdgpu device pointer
122 * @vm: vm providing the BOs
123 * @validate: callback to do the validation
124 * @param: parameter for the validation callback
126 * Validate the page table BOs on command submission if neccessary.
128 int amdgpu_vm_validate_pt_bos(struct amdgpu_device
*adev
, struct amdgpu_vm
*vm
,
129 int (*validate
)(void *p
, struct amdgpu_bo
*bo
),
132 uint64_t num_evictions
;
136 /* We only need to validate the page tables
137 * if they aren't already valid.
139 num_evictions
= atomic64_read(&adev
->num_evictions
);
140 if (num_evictions
== vm
->last_eviction_counter
)
143 /* add the vm page table to the list */
144 for (i
= 0; i
<= vm
->max_pde_used
; ++i
) {
145 struct amdgpu_bo
*bo
= vm
->page_tables
[i
].bo
;
150 r
= validate(param
, bo
);
159 * amdgpu_vm_move_pt_bos_in_lru - move the PT BOs to the LRU tail
161 * @adev: amdgpu device instance
162 * @vm: vm providing the BOs
164 * Move the PT BOs to the tail of the LRU.
166 void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device
*adev
,
167 struct amdgpu_vm
*vm
)
169 struct ttm_bo_global
*glob
= adev
->mman
.bdev
.glob
;
172 spin_lock(&glob
->lru_lock
);
173 for (i
= 0; i
<= vm
->max_pde_used
; ++i
) {
174 struct amdgpu_bo
*bo
= vm
->page_tables
[i
].bo
;
179 ttm_bo_move_to_lru_tail(&bo
->tbo
);
181 spin_unlock(&glob
->lru_lock
);
184 static bool amdgpu_vm_is_gpu_reset(struct amdgpu_device
*adev
,
185 struct amdgpu_vm_id
*id
)
187 return id
->current_gpu_reset_count
!=
188 atomic_read(&adev
->gpu_reset_counter
) ? true : false;
192 * amdgpu_vm_grab_id - allocate the next free VMID
194 * @vm: vm to allocate id for
195 * @ring: ring we want to submit job to
196 * @sync: sync object where we add dependencies
197 * @fence: fence protecting ID from reuse
199 * Allocate an id for the vm, adding fences to the sync obj as necessary.
201 int amdgpu_vm_grab_id(struct amdgpu_vm
*vm
, struct amdgpu_ring
*ring
,
202 struct amdgpu_sync
*sync
, struct dma_fence
*fence
,
203 struct amdgpu_job
*job
)
205 struct amdgpu_device
*adev
= ring
->adev
;
206 uint64_t fence_context
= adev
->fence_context
+ ring
->idx
;
207 struct dma_fence
*updates
= sync
->last_vm_update
;
208 struct amdgpu_vm_id
*id
, *idle
;
209 struct dma_fence
**fences
;
213 fences
= kmalloc_array(sizeof(void *), adev
->vm_manager
.num_ids
,
218 mutex_lock(&adev
->vm_manager
.lock
);
220 /* Check if we have an idle VMID */
222 list_for_each_entry(idle
, &adev
->vm_manager
.ids_lru
, list
) {
223 fences
[i
] = amdgpu_sync_peek_fence(&idle
->active
, ring
);
229 /* If we can't find a idle VMID to use, wait till one becomes available */
230 if (&idle
->list
== &adev
->vm_manager
.ids_lru
) {
231 u64 fence_context
= adev
->vm_manager
.fence_context
+ ring
->idx
;
232 unsigned seqno
= ++adev
->vm_manager
.seqno
[ring
->idx
];
233 struct dma_fence_array
*array
;
236 for (j
= 0; j
< i
; ++j
)
237 dma_fence_get(fences
[j
]);
239 array
= dma_fence_array_create(i
, fences
, fence_context
,
242 for (j
= 0; j
< i
; ++j
)
243 dma_fence_put(fences
[j
]);
250 r
= amdgpu_sync_fence(ring
->adev
, sync
, &array
->base
);
251 dma_fence_put(&array
->base
);
255 mutex_unlock(&adev
->vm_manager
.lock
);
261 job
->vm_needs_flush
= true;
262 /* Check if we can use a VMID already assigned to this VM */
265 struct dma_fence
*flushed
;
268 if (i
== AMDGPU_MAX_RINGS
)
271 /* Check all the prerequisites to using this VMID */
274 if (amdgpu_vm_is_gpu_reset(adev
, id
))
277 if (atomic64_read(&id
->owner
) != vm
->client_id
)
280 if (job
->vm_pd_addr
!= id
->pd_gpu_addr
)
286 if (id
->last_flush
->context
!= fence_context
&&
287 !dma_fence_is_signaled(id
->last_flush
))
290 flushed
= id
->flushed_updates
;
292 (!flushed
|| dma_fence_is_later(updates
, flushed
)))
295 /* Good we can use this VMID. Remember this submission as
298 r
= amdgpu_sync_fence(ring
->adev
, &id
->active
, fence
);
302 id
->current_gpu_reset_count
= atomic_read(&adev
->gpu_reset_counter
);
303 list_move_tail(&id
->list
, &adev
->vm_manager
.ids_lru
);
304 vm
->ids
[ring
->idx
] = id
;
306 job
->vm_id
= id
- adev
->vm_manager
.ids
;
307 job
->vm_needs_flush
= false;
308 trace_amdgpu_vm_grab_id(vm
, ring
->idx
, job
);
310 mutex_unlock(&adev
->vm_manager
.lock
);
313 } while (i
!= ring
->idx
);
315 /* Still no ID to use? Then use the idle one found earlier */
318 /* Remember this submission as user of the VMID */
319 r
= amdgpu_sync_fence(ring
->adev
, &id
->active
, fence
);
323 dma_fence_put(id
->first
);
324 id
->first
= dma_fence_get(fence
);
326 dma_fence_put(id
->last_flush
);
327 id
->last_flush
= NULL
;
329 dma_fence_put(id
->flushed_updates
);
330 id
->flushed_updates
= dma_fence_get(updates
);
332 id
->pd_gpu_addr
= job
->vm_pd_addr
;
333 id
->current_gpu_reset_count
= atomic_read(&adev
->gpu_reset_counter
);
334 list_move_tail(&id
->list
, &adev
->vm_manager
.ids_lru
);
335 atomic64_set(&id
->owner
, vm
->client_id
);
336 vm
->ids
[ring
->idx
] = id
;
338 job
->vm_id
= id
- adev
->vm_manager
.ids
;
339 trace_amdgpu_vm_grab_id(vm
, ring
->idx
, job
);
342 mutex_unlock(&adev
->vm_manager
.lock
);
346 static bool amdgpu_vm_ring_has_compute_vm_bug(struct amdgpu_ring
*ring
)
348 struct amdgpu_device
*adev
= ring
->adev
;
349 const struct amdgpu_ip_block
*ip_block
;
351 if (ring
->funcs
->type
!= AMDGPU_RING_TYPE_COMPUTE
)
352 /* only compute rings */
355 ip_block
= amdgpu_get_ip_block(adev
, AMD_IP_BLOCK_TYPE_GFX
);
359 if (ip_block
->version
->major
<= 7) {
360 /* gfx7 has no workaround */
362 } else if (ip_block
->version
->major
== 8) {
363 if (adev
->gfx
.mec_fw_version
>= 673)
364 /* gfx8 is fixed in MEC firmware 673 */
373 * amdgpu_vm_flush - hardware flush the vm
375 * @ring: ring to use for flush
376 * @vm_id: vmid number to use
377 * @pd_addr: address of the page directory
379 * Emit a VM flush when it is necessary.
381 int amdgpu_vm_flush(struct amdgpu_ring
*ring
, struct amdgpu_job
*job
)
383 struct amdgpu_device
*adev
= ring
->adev
;
384 struct amdgpu_vm_id
*id
= &adev
->vm_manager
.ids
[job
->vm_id
];
385 bool gds_switch_needed
= ring
->funcs
->emit_gds_switch
&& (
386 id
->gds_base
!= job
->gds_base
||
387 id
->gds_size
!= job
->gds_size
||
388 id
->gws_base
!= job
->gws_base
||
389 id
->gws_size
!= job
->gws_size
||
390 id
->oa_base
!= job
->oa_base
||
391 id
->oa_size
!= job
->oa_size
);
394 if (ring
->funcs
->emit_pipeline_sync
&& (
395 job
->vm_needs_flush
|| gds_switch_needed
||
396 amdgpu_vm_ring_has_compute_vm_bug(ring
)))
397 amdgpu_ring_emit_pipeline_sync(ring
);
399 if (ring
->funcs
->emit_vm_flush
&& (job
->vm_needs_flush
||
400 amdgpu_vm_is_gpu_reset(adev
, id
))) {
401 struct dma_fence
*fence
;
403 trace_amdgpu_vm_flush(job
->vm_pd_addr
, ring
->idx
, job
->vm_id
);
404 amdgpu_ring_emit_vm_flush(ring
, job
->vm_id
, job
->vm_pd_addr
);
406 r
= amdgpu_fence_emit(ring
, &fence
);
410 mutex_lock(&adev
->vm_manager
.lock
);
411 dma_fence_put(id
->last_flush
);
412 id
->last_flush
= fence
;
413 mutex_unlock(&adev
->vm_manager
.lock
);
416 if (gds_switch_needed
) {
417 id
->gds_base
= job
->gds_base
;
418 id
->gds_size
= job
->gds_size
;
419 id
->gws_base
= job
->gws_base
;
420 id
->gws_size
= job
->gws_size
;
421 id
->oa_base
= job
->oa_base
;
422 id
->oa_size
= job
->oa_size
;
423 amdgpu_ring_emit_gds_switch(ring
, job
->vm_id
,
424 job
->gds_base
, job
->gds_size
,
425 job
->gws_base
, job
->gws_size
,
426 job
->oa_base
, job
->oa_size
);
433 * amdgpu_vm_reset_id - reset VMID to zero
435 * @adev: amdgpu device structure
436 * @vm_id: vmid number to use
438 * Reset saved GDW, GWS and OA to force switch on next flush.
440 void amdgpu_vm_reset_id(struct amdgpu_device
*adev
, unsigned vm_id
)
442 struct amdgpu_vm_id
*id
= &adev
->vm_manager
.ids
[vm_id
];
453 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
456 * @bo: requested buffer object
458 * Find @bo inside the requested vm.
459 * Search inside the @bos vm list for the requested vm
460 * Returns the found bo_va or NULL if none is found
462 * Object has to be reserved!
464 struct amdgpu_bo_va
*amdgpu_vm_bo_find(struct amdgpu_vm
*vm
,
465 struct amdgpu_bo
*bo
)
467 struct amdgpu_bo_va
*bo_va
;
469 list_for_each_entry(bo_va
, &bo
->va
, bo_list
) {
470 if (bo_va
->vm
== vm
) {
478 * amdgpu_vm_do_set_ptes - helper to call the right asic function
480 * @params: see amdgpu_pte_update_params definition
481 * @pe: addr of the page entry
482 * @addr: dst addr to write into pe
483 * @count: number of page entries to update
484 * @incr: increase next addr by incr bytes
485 * @flags: hw access flags
487 * Traces the parameters and calls the right asic functions
488 * to setup the page table using the DMA.
490 static void amdgpu_vm_do_set_ptes(struct amdgpu_pte_update_params
*params
,
491 uint64_t pe
, uint64_t addr
,
492 unsigned count
, uint32_t incr
,
495 trace_amdgpu_vm_set_ptes(pe
, addr
, count
, incr
, flags
);
498 amdgpu_vm_write_pte(params
->adev
, params
->ib
, pe
,
499 addr
| flags
, count
, incr
);
502 amdgpu_vm_set_pte_pde(params
->adev
, params
->ib
, pe
, addr
,
508 * amdgpu_vm_do_copy_ptes - copy the PTEs from the GART
510 * @params: see amdgpu_pte_update_params definition
511 * @pe: addr of the page entry
512 * @addr: dst addr to write into pe
513 * @count: number of page entries to update
514 * @incr: increase next addr by incr bytes
515 * @flags: hw access flags
517 * Traces the parameters and calls the DMA function to copy the PTEs.
519 static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params
*params
,
520 uint64_t pe
, uint64_t addr
,
521 unsigned count
, uint32_t incr
,
524 uint64_t src
= (params
->src
+ (addr
>> 12) * 8);
527 trace_amdgpu_vm_copy_ptes(pe
, src
, count
);
529 amdgpu_vm_copy_pte(params
->adev
, params
->ib
, pe
, src
, count
);
533 * amdgpu_vm_map_gart - Resolve gart mapping of addr
535 * @pages_addr: optional DMA address to use for lookup
536 * @addr: the unmapped addr
538 * Look up the physical address of the page that the pte resolves
539 * to and return the pointer for the page table entry.
541 static uint64_t amdgpu_vm_map_gart(const dma_addr_t
*pages_addr
, uint64_t addr
)
545 /* page table offset */
546 result
= pages_addr
[addr
>> PAGE_SHIFT
];
548 /* in case cpu page size != gpu page size*/
549 result
|= addr
& (~PAGE_MASK
);
551 result
&= 0xFFFFFFFFFFFFF000ULL
;
557 * amdgpu_vm_update_pdes - make sure that page directory is valid
559 * @adev: amdgpu_device pointer
561 * @start: start of GPU address range
562 * @end: end of GPU address range
564 * Allocates new page tables if necessary
565 * and updates the page directory.
566 * Returns 0 for success, error for failure.
568 int amdgpu_vm_update_page_directory(struct amdgpu_device
*adev
,
569 struct amdgpu_vm
*vm
)
571 struct amdgpu_bo
*shadow
;
572 struct amdgpu_ring
*ring
;
573 uint64_t pd_addr
, shadow_addr
;
574 uint32_t incr
= AMDGPU_VM_PTE_COUNT
* 8;
575 uint64_t last_pde
= ~0, last_pt
= ~0, last_shadow
= ~0;
576 unsigned count
= 0, pt_idx
, ndw
;
577 struct amdgpu_job
*job
;
578 struct amdgpu_pte_update_params params
;
579 struct dma_fence
*fence
= NULL
;
583 ring
= container_of(vm
->entity
.sched
, struct amdgpu_ring
, sched
);
584 shadow
= vm
->page_directory
->shadow
;
589 /* assume the worst case */
590 ndw
+= vm
->max_pde_used
* 6;
592 pd_addr
= amdgpu_bo_gpu_offset(vm
->page_directory
);
594 r
= amdgpu_ttm_bind(&shadow
->tbo
, &shadow
->tbo
.mem
);
597 shadow_addr
= amdgpu_bo_gpu_offset(shadow
);
603 r
= amdgpu_job_alloc_with_ib(adev
, ndw
* 4, &job
);
607 memset(¶ms
, 0, sizeof(params
));
609 params
.ib
= &job
->ibs
[0];
611 /* walk over the address space and update the page directory */
612 for (pt_idx
= 0; pt_idx
<= vm
->max_pde_used
; ++pt_idx
) {
613 struct amdgpu_bo
*bo
= vm
->page_tables
[pt_idx
].bo
;
620 struct amdgpu_bo
*pt_shadow
= bo
->shadow
;
622 r
= amdgpu_ttm_bind(&pt_shadow
->tbo
,
623 &pt_shadow
->tbo
.mem
);
628 pt
= amdgpu_bo_gpu_offset(bo
);
629 if (vm
->page_tables
[pt_idx
].addr
== pt
)
632 vm
->page_tables
[pt_idx
].addr
= pt
;
634 pde
= pd_addr
+ pt_idx
* 8;
635 if (((last_pde
+ 8 * count
) != pde
) ||
636 ((last_pt
+ incr
* count
) != pt
) ||
637 (count
== AMDGPU_VM_MAX_UPDATE_SIZE
)) {
641 amdgpu_vm_do_set_ptes(¶ms
,
647 amdgpu_vm_do_set_ptes(¶ms
, last_pde
,
648 last_pt
, count
, incr
,
654 last_shadow
= shadow_addr
+ pt_idx
* 8;
662 if (vm
->page_directory
->shadow
)
663 amdgpu_vm_do_set_ptes(¶ms
, last_shadow
, last_pt
,
664 count
, incr
, AMDGPU_PTE_VALID
);
666 amdgpu_vm_do_set_ptes(¶ms
, last_pde
, last_pt
,
667 count
, incr
, AMDGPU_PTE_VALID
);
670 if (params
.ib
->length_dw
== 0) {
671 amdgpu_job_free(job
);
675 amdgpu_ring_pad_ib(ring
, params
.ib
);
676 amdgpu_sync_resv(adev
, &job
->sync
, vm
->page_directory
->tbo
.resv
,
677 AMDGPU_FENCE_OWNER_VM
);
679 amdgpu_sync_resv(adev
, &job
->sync
, shadow
->tbo
.resv
,
680 AMDGPU_FENCE_OWNER_VM
);
682 WARN_ON(params
.ib
->length_dw
> ndw
);
683 r
= amdgpu_job_submit(job
, ring
, &vm
->entity
,
684 AMDGPU_FENCE_OWNER_VM
, &fence
);
688 amdgpu_bo_fence(vm
->page_directory
, fence
, true);
689 dma_fence_put(vm
->page_directory_fence
);
690 vm
->page_directory_fence
= dma_fence_get(fence
);
691 dma_fence_put(fence
);
696 amdgpu_job_free(job
);
701 * amdgpu_vm_update_ptes - make sure that page tables are valid
703 * @params: see amdgpu_pte_update_params definition
705 * @start: start of GPU address range
706 * @end: end of GPU address range
707 * @dst: destination address to map to, the next dst inside the function
708 * @flags: mapping flags
710 * Update the page tables in the range @start - @end.
712 static void amdgpu_vm_update_ptes(struct amdgpu_pte_update_params
*params
,
713 struct amdgpu_vm
*vm
,
714 uint64_t start
, uint64_t end
,
715 uint64_t dst
, uint32_t flags
)
717 const uint64_t mask
= AMDGPU_VM_PTE_COUNT
- 1;
719 uint64_t cur_pe_start
, cur_nptes
, cur_dst
;
720 uint64_t addr
; /* next GPU address to be updated */
722 struct amdgpu_bo
*pt
;
723 unsigned nptes
; /* next number of ptes to be updated */
724 uint64_t next_pe_start
;
726 /* initialize the variables */
728 pt_idx
= addr
>> amdgpu_vm_block_size
;
729 pt
= vm
->page_tables
[pt_idx
].bo
;
730 if (params
->shadow
) {
735 if ((addr
& ~mask
) == (end
& ~mask
))
738 nptes
= AMDGPU_VM_PTE_COUNT
- (addr
& mask
);
740 cur_pe_start
= amdgpu_bo_gpu_offset(pt
);
741 cur_pe_start
+= (addr
& mask
) * 8;
747 dst
+= nptes
* AMDGPU_GPU_PAGE_SIZE
;
749 /* walk over the address space and update the page tables */
751 pt_idx
= addr
>> amdgpu_vm_block_size
;
752 pt
= vm
->page_tables
[pt_idx
].bo
;
753 if (params
->shadow
) {
759 if ((addr
& ~mask
) == (end
& ~mask
))
762 nptes
= AMDGPU_VM_PTE_COUNT
- (addr
& mask
);
764 next_pe_start
= amdgpu_bo_gpu_offset(pt
);
765 next_pe_start
+= (addr
& mask
) * 8;
767 if ((cur_pe_start
+ 8 * cur_nptes
) == next_pe_start
&&
768 ((cur_nptes
+ nptes
) <= AMDGPU_VM_MAX_UPDATE_SIZE
)) {
769 /* The next ptb is consecutive to current ptb.
770 * Don't call the update function now.
771 * Will update two ptbs together in future.
775 params
->func(params
, cur_pe_start
, cur_dst
, cur_nptes
,
776 AMDGPU_GPU_PAGE_SIZE
, flags
);
778 cur_pe_start
= next_pe_start
;
785 dst
+= nptes
* AMDGPU_GPU_PAGE_SIZE
;
788 params
->func(params
, cur_pe_start
, cur_dst
, cur_nptes
,
789 AMDGPU_GPU_PAGE_SIZE
, flags
);
793 * amdgpu_vm_frag_ptes - add fragment information to PTEs
795 * @params: see amdgpu_pte_update_params definition
797 * @start: first PTE to handle
798 * @end: last PTE to handle
799 * @dst: addr those PTEs should point to
800 * @flags: hw mapping flags
802 static void amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params
*params
,
803 struct amdgpu_vm
*vm
,
804 uint64_t start
, uint64_t end
,
805 uint64_t dst
, uint32_t flags
)
808 * The MC L1 TLB supports variable sized pages, based on a fragment
809 * field in the PTE. When this field is set to a non-zero value, page
810 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
811 * flags are considered valid for all PTEs within the fragment range
812 * and corresponding mappings are assumed to be physically contiguous.
814 * The L1 TLB can store a single PTE for the whole fragment,
815 * significantly increasing the space available for translation
816 * caching. This leads to large improvements in throughput when the
817 * TLB is under pressure.
819 * The L2 TLB distributes small and large fragments into two
820 * asymmetric partitions. The large fragment cache is significantly
821 * larger. Thus, we try to use large fragments wherever possible.
822 * Userspace can support this by aligning virtual base address and
823 * allocation size to the fragment size.
826 /* SI and newer are optimized for 64KB */
827 uint64_t frag_flags
= AMDGPU_PTE_FRAG(AMDGPU_LOG2_PAGES_PER_FRAG
);
828 uint64_t frag_align
= 1 << AMDGPU_LOG2_PAGES_PER_FRAG
;
830 uint64_t frag_start
= ALIGN(start
, frag_align
);
831 uint64_t frag_end
= end
& ~(frag_align
- 1);
833 /* system pages are non continuously */
834 if (params
->src
|| !(flags
& AMDGPU_PTE_VALID
) ||
835 (frag_start
>= frag_end
)) {
837 amdgpu_vm_update_ptes(params
, vm
, start
, end
, dst
, flags
);
841 /* handle the 4K area at the beginning */
842 if (start
!= frag_start
) {
843 amdgpu_vm_update_ptes(params
, vm
, start
, frag_start
,
845 dst
+= (frag_start
- start
) * AMDGPU_GPU_PAGE_SIZE
;
848 /* handle the area in the middle */
849 amdgpu_vm_update_ptes(params
, vm
, frag_start
, frag_end
, dst
,
852 /* handle the 4K area at the end */
853 if (frag_end
!= end
) {
854 dst
+= (frag_end
- frag_start
) * AMDGPU_GPU_PAGE_SIZE
;
855 amdgpu_vm_update_ptes(params
, vm
, frag_end
, end
, dst
, flags
);
860 * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
862 * @adev: amdgpu_device pointer
863 * @exclusive: fence we need to sync to
864 * @src: address where to copy page table entries from
865 * @pages_addr: DMA addresses to use for mapping
867 * @start: start of mapped range
868 * @last: last mapped entry
869 * @flags: flags for the entries
870 * @addr: addr to set the area to
871 * @fence: optional resulting fence
873 * Fill in the page table entries between @start and @last.
874 * Returns 0 for success, -EINVAL for failure.
876 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device
*adev
,
877 struct dma_fence
*exclusive
,
879 dma_addr_t
*pages_addr
,
880 struct amdgpu_vm
*vm
,
881 uint64_t start
, uint64_t last
,
882 uint32_t flags
, uint64_t addr
,
883 struct dma_fence
**fence
)
885 struct amdgpu_ring
*ring
;
886 void *owner
= AMDGPU_FENCE_OWNER_VM
;
887 unsigned nptes
, ncmds
, ndw
;
888 struct amdgpu_job
*job
;
889 struct amdgpu_pte_update_params params
;
890 struct dma_fence
*f
= NULL
;
893 memset(¶ms
, 0, sizeof(params
));
897 ring
= container_of(vm
->entity
.sched
, struct amdgpu_ring
, sched
);
899 memset(¶ms
, 0, sizeof(params
));
903 /* sync to everything on unmapping */
904 if (!(flags
& AMDGPU_PTE_VALID
))
905 owner
= AMDGPU_FENCE_OWNER_UNDEFINED
;
907 nptes
= last
- start
+ 1;
910 * reserve space for one command every (1 << BLOCK_SIZE)
911 * entries or 2k dwords (whatever is smaller)
913 ncmds
= (nptes
>> min(amdgpu_vm_block_size
, 11)) + 1;
919 /* only copy commands needed */
922 params
.func
= amdgpu_vm_do_copy_ptes
;
924 } else if (pages_addr
) {
925 /* copy commands needed */
931 params
.func
= amdgpu_vm_do_copy_ptes
;
934 /* set page commands needed */
937 /* two extra commands for begin/end of fragment */
940 params
.func
= amdgpu_vm_do_set_ptes
;
943 r
= amdgpu_job_alloc_with_ib(adev
, ndw
* 4, &job
);
947 params
.ib
= &job
->ibs
[0];
949 if (!src
&& pages_addr
) {
953 /* Put the PTEs at the end of the IB. */
955 pte
= (uint64_t *)&(job
->ibs
->ptr
[i
]);
956 params
.src
= job
->ibs
->gpu_addr
+ i
* 4;
958 for (i
= 0; i
< nptes
; ++i
) {
959 pte
[i
] = amdgpu_vm_map_gart(pages_addr
, addr
+ i
*
960 AMDGPU_GPU_PAGE_SIZE
);
966 r
= amdgpu_sync_fence(adev
, &job
->sync
, exclusive
);
970 r
= amdgpu_sync_resv(adev
, &job
->sync
, vm
->page_directory
->tbo
.resv
,
975 r
= reservation_object_reserve_shared(vm
->page_directory
->tbo
.resv
);
979 params
.shadow
= true;
980 amdgpu_vm_frag_ptes(¶ms
, vm
, start
, last
+ 1, addr
, flags
);
981 params
.shadow
= false;
982 amdgpu_vm_frag_ptes(¶ms
, vm
, start
, last
+ 1, addr
, flags
);
984 amdgpu_ring_pad_ib(ring
, params
.ib
);
985 WARN_ON(params
.ib
->length_dw
> ndw
);
986 r
= amdgpu_job_submit(job
, ring
, &vm
->entity
,
987 AMDGPU_FENCE_OWNER_VM
, &f
);
991 amdgpu_bo_fence(vm
->page_directory
, f
, true);
993 dma_fence_put(*fence
);
994 *fence
= dma_fence_get(f
);
1000 amdgpu_job_free(job
);
1005 * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
1007 * @adev: amdgpu_device pointer
1008 * @exclusive: fence we need to sync to
1009 * @gtt_flags: flags as they are used for GTT
1010 * @pages_addr: DMA addresses to use for mapping
1012 * @mapping: mapped range and flags to use for the update
1013 * @flags: HW flags for the mapping
1014 * @nodes: array of drm_mm_nodes with the MC addresses
1015 * @fence: optional resulting fence
1017 * Split the mapping into smaller chunks so that each update fits
1019 * Returns 0 for success, -EINVAL for failure.
1021 static int amdgpu_vm_bo_split_mapping(struct amdgpu_device
*adev
,
1022 struct dma_fence
*exclusive
,
1024 dma_addr_t
*pages_addr
,
1025 struct amdgpu_vm
*vm
,
1026 struct amdgpu_bo_va_mapping
*mapping
,
1028 struct drm_mm_node
*nodes
,
1029 struct dma_fence
**fence
)
1031 uint64_t pfn
, src
= 0, start
= mapping
->it
.start
;
1034 /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
1035 * but in case of something, we filter the flags in first place
1037 if (!(mapping
->flags
& AMDGPU_PTE_READABLE
))
1038 flags
&= ~AMDGPU_PTE_READABLE
;
1039 if (!(mapping
->flags
& AMDGPU_PTE_WRITEABLE
))
1040 flags
&= ~AMDGPU_PTE_WRITEABLE
;
1042 trace_amdgpu_vm_bo_update(mapping
);
1044 pfn
= mapping
->offset
>> PAGE_SHIFT
;
1046 while (pfn
>= nodes
->size
) {
1053 uint64_t max_entries
;
1054 uint64_t addr
, last
;
1057 addr
= nodes
->start
<< PAGE_SHIFT
;
1058 max_entries
= (nodes
->size
- pfn
) *
1059 (PAGE_SIZE
/ AMDGPU_GPU_PAGE_SIZE
);
1062 max_entries
= S64_MAX
;
1066 if (flags
== gtt_flags
)
1067 src
= adev
->gart
.table_addr
+
1068 (addr
>> AMDGPU_GPU_PAGE_SHIFT
) * 8;
1070 max_entries
= min(max_entries
, 16ull * 1024ull);
1072 } else if (flags
& AMDGPU_PTE_VALID
) {
1073 addr
+= adev
->vm_manager
.vram_base_offset
;
1075 addr
+= pfn
<< PAGE_SHIFT
;
1077 last
= min((uint64_t)mapping
->it
.last
, start
+ max_entries
- 1);
1078 r
= amdgpu_vm_bo_update_mapping(adev
, exclusive
,
1079 src
, pages_addr
, vm
,
1080 start
, last
, flags
, addr
,
1085 pfn
+= last
- start
+ 1;
1086 if (nodes
&& nodes
->size
== pfn
) {
1092 } while (unlikely(start
!= mapping
->it
.last
+ 1));
1098 * amdgpu_vm_bo_update - update all BO mappings in the vm page table
1100 * @adev: amdgpu_device pointer
1101 * @bo_va: requested BO and VM object
1102 * @clear: if true clear the entries
1104 * Fill in the page table entries for @bo_va.
1105 * Returns 0 for success, -EINVAL for failure.
1107 int amdgpu_vm_bo_update(struct amdgpu_device
*adev
,
1108 struct amdgpu_bo_va
*bo_va
,
1111 struct amdgpu_vm
*vm
= bo_va
->vm
;
1112 struct amdgpu_bo_va_mapping
*mapping
;
1113 dma_addr_t
*pages_addr
= NULL
;
1114 uint32_t gtt_flags
, flags
;
1115 struct ttm_mem_reg
*mem
;
1116 struct drm_mm_node
*nodes
;
1117 struct dma_fence
*exclusive
;
1125 struct ttm_dma_tt
*ttm
;
1127 mem
= &bo_va
->bo
->tbo
.mem
;
1128 nodes
= mem
->mm_node
;
1129 if (mem
->mem_type
== TTM_PL_TT
) {
1130 ttm
= container_of(bo_va
->bo
->tbo
.ttm
, struct
1132 pages_addr
= ttm
->dma_address
;
1134 exclusive
= reservation_object_get_excl(bo_va
->bo
->tbo
.resv
);
1137 flags
= amdgpu_ttm_tt_pte_flags(adev
, bo_va
->bo
->tbo
.ttm
, mem
);
1138 gtt_flags
= (amdgpu_ttm_is_bound(bo_va
->bo
->tbo
.ttm
) &&
1139 adev
== amdgpu_ttm_adev(bo_va
->bo
->tbo
.bdev
)) ? flags
: 0;
1141 spin_lock(&vm
->status_lock
);
1142 if (!list_empty(&bo_va
->vm_status
))
1143 list_splice_init(&bo_va
->valids
, &bo_va
->invalids
);
1144 spin_unlock(&vm
->status_lock
);
1146 list_for_each_entry(mapping
, &bo_va
->invalids
, list
) {
1147 r
= amdgpu_vm_bo_split_mapping(adev
, exclusive
,
1148 gtt_flags
, pages_addr
, vm
,
1149 mapping
, flags
, nodes
,
1150 &bo_va
->last_pt_update
);
1155 if (trace_amdgpu_vm_bo_mapping_enabled()) {
1156 list_for_each_entry(mapping
, &bo_va
->valids
, list
)
1157 trace_amdgpu_vm_bo_mapping(mapping
);
1159 list_for_each_entry(mapping
, &bo_va
->invalids
, list
)
1160 trace_amdgpu_vm_bo_mapping(mapping
);
1163 spin_lock(&vm
->status_lock
);
1164 list_splice_init(&bo_va
->invalids
, &bo_va
->valids
);
1165 list_del_init(&bo_va
->vm_status
);
1167 list_add(&bo_va
->vm_status
, &vm
->cleared
);
1168 spin_unlock(&vm
->status_lock
);
1174 * amdgpu_vm_clear_freed - clear freed BOs in the PT
1176 * @adev: amdgpu_device pointer
1179 * Make sure all freed BOs are cleared in the PT.
1180 * Returns 0 for success.
1182 * PTs have to be reserved and mutex must be locked!
1184 int amdgpu_vm_clear_freed(struct amdgpu_device
*adev
,
1185 struct amdgpu_vm
*vm
)
1187 struct amdgpu_bo_va_mapping
*mapping
;
1190 while (!list_empty(&vm
->freed
)) {
1191 mapping
= list_first_entry(&vm
->freed
,
1192 struct amdgpu_bo_va_mapping
, list
);
1193 list_del(&mapping
->list
);
1195 r
= amdgpu_vm_bo_split_mapping(adev
, NULL
, 0, NULL
, vm
, mapping
,
1207 * amdgpu_vm_clear_invalids - clear invalidated BOs in the PT
1209 * @adev: amdgpu_device pointer
1212 * Make sure all invalidated BOs are cleared in the PT.
1213 * Returns 0 for success.
1215 * PTs have to be reserved and mutex must be locked!
1217 int amdgpu_vm_clear_invalids(struct amdgpu_device
*adev
,
1218 struct amdgpu_vm
*vm
, struct amdgpu_sync
*sync
)
1220 struct amdgpu_bo_va
*bo_va
= NULL
;
1223 spin_lock(&vm
->status_lock
);
1224 while (!list_empty(&vm
->invalidated
)) {
1225 bo_va
= list_first_entry(&vm
->invalidated
,
1226 struct amdgpu_bo_va
, vm_status
);
1227 spin_unlock(&vm
->status_lock
);
1229 r
= amdgpu_vm_bo_update(adev
, bo_va
, true);
1233 spin_lock(&vm
->status_lock
);
1235 spin_unlock(&vm
->status_lock
);
1238 r
= amdgpu_sync_fence(adev
, sync
, bo_va
->last_pt_update
);
1244 * amdgpu_vm_bo_add - add a bo to a specific vm
1246 * @adev: amdgpu_device pointer
1248 * @bo: amdgpu buffer object
1250 * Add @bo into the requested vm.
1251 * Add @bo to the list of bos associated with the vm
1252 * Returns newly added bo_va or NULL for failure
1254 * Object has to be reserved!
1256 struct amdgpu_bo_va
*amdgpu_vm_bo_add(struct amdgpu_device
*adev
,
1257 struct amdgpu_vm
*vm
,
1258 struct amdgpu_bo
*bo
)
1260 struct amdgpu_bo_va
*bo_va
;
1262 bo_va
= kzalloc(sizeof(struct amdgpu_bo_va
), GFP_KERNEL
);
1263 if (bo_va
== NULL
) {
1268 bo_va
->ref_count
= 1;
1269 INIT_LIST_HEAD(&bo_va
->bo_list
);
1270 INIT_LIST_HEAD(&bo_va
->valids
);
1271 INIT_LIST_HEAD(&bo_va
->invalids
);
1272 INIT_LIST_HEAD(&bo_va
->vm_status
);
1274 list_add_tail(&bo_va
->bo_list
, &bo
->va
);
1280 * amdgpu_vm_bo_map - map bo inside a vm
1282 * @adev: amdgpu_device pointer
1283 * @bo_va: bo_va to store the address
1284 * @saddr: where to map the BO
1285 * @offset: requested offset in the BO
1286 * @flags: attributes of pages (read/write/valid/etc.)
1288 * Add a mapping of the BO at the specefied addr into the VM.
1289 * Returns 0 for success, error for failure.
1291 * Object has to be reserved and unreserved outside!
1293 int amdgpu_vm_bo_map(struct amdgpu_device
*adev
,
1294 struct amdgpu_bo_va
*bo_va
,
1295 uint64_t saddr
, uint64_t offset
,
1296 uint64_t size
, uint32_t flags
)
1298 struct amdgpu_bo_va_mapping
*mapping
;
1299 struct amdgpu_vm
*vm
= bo_va
->vm
;
1300 struct interval_tree_node
*it
;
1301 unsigned last_pfn
, pt_idx
;
1305 /* validate the parameters */
1306 if (saddr
& AMDGPU_GPU_PAGE_MASK
|| offset
& AMDGPU_GPU_PAGE_MASK
||
1307 size
== 0 || size
& AMDGPU_GPU_PAGE_MASK
)
1310 /* make sure object fit at this offset */
1311 eaddr
= saddr
+ size
- 1;
1312 if ((saddr
>= eaddr
) || (offset
+ size
> amdgpu_bo_size(bo_va
->bo
)))
1315 last_pfn
= eaddr
/ AMDGPU_GPU_PAGE_SIZE
;
1316 if (last_pfn
>= adev
->vm_manager
.max_pfn
) {
1317 dev_err(adev
->dev
, "va above limit (0x%08X >= 0x%08X)\n",
1318 last_pfn
, adev
->vm_manager
.max_pfn
);
1322 saddr
/= AMDGPU_GPU_PAGE_SIZE
;
1323 eaddr
/= AMDGPU_GPU_PAGE_SIZE
;
1325 it
= interval_tree_iter_first(&vm
->va
, saddr
, eaddr
);
1327 struct amdgpu_bo_va_mapping
*tmp
;
1328 tmp
= container_of(it
, struct amdgpu_bo_va_mapping
, it
);
1329 /* bo and tmp overlap, invalid addr */
1330 dev_err(adev
->dev
, "bo %p va 0x%010Lx-0x%010Lx conflict with "
1331 "0x%010lx-0x%010lx\n", bo_va
->bo
, saddr
, eaddr
,
1332 tmp
->it
.start
, tmp
->it
.last
+ 1);
1337 mapping
= kmalloc(sizeof(*mapping
), GFP_KERNEL
);
1343 INIT_LIST_HEAD(&mapping
->list
);
1344 mapping
->it
.start
= saddr
;
1345 mapping
->it
.last
= eaddr
;
1346 mapping
->offset
= offset
;
1347 mapping
->flags
= flags
;
1349 list_add(&mapping
->list
, &bo_va
->invalids
);
1350 interval_tree_insert(&mapping
->it
, &vm
->va
);
1352 /* Make sure the page tables are allocated */
1353 saddr
>>= amdgpu_vm_block_size
;
1354 eaddr
>>= amdgpu_vm_block_size
;
1356 BUG_ON(eaddr
>= amdgpu_vm_num_pdes(adev
));
1358 if (eaddr
> vm
->max_pde_used
)
1359 vm
->max_pde_used
= eaddr
;
1361 /* walk over the address space and allocate the page tables */
1362 for (pt_idx
= saddr
; pt_idx
<= eaddr
; ++pt_idx
) {
1363 struct reservation_object
*resv
= vm
->page_directory
->tbo
.resv
;
1364 struct amdgpu_bo
*pt
;
1366 if (vm
->page_tables
[pt_idx
].bo
)
1369 r
= amdgpu_bo_create(adev
, AMDGPU_VM_PTE_COUNT
* 8,
1370 AMDGPU_GPU_PAGE_SIZE
, true,
1371 AMDGPU_GEM_DOMAIN_VRAM
,
1372 AMDGPU_GEM_CREATE_NO_CPU_ACCESS
|
1373 AMDGPU_GEM_CREATE_SHADOW
|
1374 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS
|
1375 AMDGPU_GEM_CREATE_VRAM_CLEARED
,
1380 /* Keep a reference to the page table to avoid freeing
1381 * them up in the wrong order.
1383 pt
->parent
= amdgpu_bo_ref(vm
->page_directory
);
1385 vm
->page_tables
[pt_idx
].bo
= pt
;
1386 vm
->page_tables
[pt_idx
].addr
= 0;
1392 list_del(&mapping
->list
);
1393 interval_tree_remove(&mapping
->it
, &vm
->va
);
1394 trace_amdgpu_vm_bo_unmap(bo_va
, mapping
);
1402 * amdgpu_vm_bo_unmap - remove bo mapping from vm
1404 * @adev: amdgpu_device pointer
1405 * @bo_va: bo_va to remove the address from
1406 * @saddr: where to the BO is mapped
1408 * Remove a mapping of the BO at the specefied addr from the VM.
1409 * Returns 0 for success, error for failure.
1411 * Object has to be reserved and unreserved outside!
1413 int amdgpu_vm_bo_unmap(struct amdgpu_device
*adev
,
1414 struct amdgpu_bo_va
*bo_va
,
1417 struct amdgpu_bo_va_mapping
*mapping
;
1418 struct amdgpu_vm
*vm
= bo_va
->vm
;
1421 saddr
/= AMDGPU_GPU_PAGE_SIZE
;
1423 list_for_each_entry(mapping
, &bo_va
->valids
, list
) {
1424 if (mapping
->it
.start
== saddr
)
1428 if (&mapping
->list
== &bo_va
->valids
) {
1431 list_for_each_entry(mapping
, &bo_va
->invalids
, list
) {
1432 if (mapping
->it
.start
== saddr
)
1436 if (&mapping
->list
== &bo_va
->invalids
)
1440 list_del(&mapping
->list
);
1441 interval_tree_remove(&mapping
->it
, &vm
->va
);
1442 trace_amdgpu_vm_bo_unmap(bo_va
, mapping
);
1445 list_add(&mapping
->list
, &vm
->freed
);
1453 * amdgpu_vm_bo_rmv - remove a bo to a specific vm
1455 * @adev: amdgpu_device pointer
1456 * @bo_va: requested bo_va
1458 * Remove @bo_va->bo from the requested vm.
1460 * Object have to be reserved!
1462 void amdgpu_vm_bo_rmv(struct amdgpu_device
*adev
,
1463 struct amdgpu_bo_va
*bo_va
)
1465 struct amdgpu_bo_va_mapping
*mapping
, *next
;
1466 struct amdgpu_vm
*vm
= bo_va
->vm
;
1468 list_del(&bo_va
->bo_list
);
1470 spin_lock(&vm
->status_lock
);
1471 list_del(&bo_va
->vm_status
);
1472 spin_unlock(&vm
->status_lock
);
1474 list_for_each_entry_safe(mapping
, next
, &bo_va
->valids
, list
) {
1475 list_del(&mapping
->list
);
1476 interval_tree_remove(&mapping
->it
, &vm
->va
);
1477 trace_amdgpu_vm_bo_unmap(bo_va
, mapping
);
1478 list_add(&mapping
->list
, &vm
->freed
);
1480 list_for_each_entry_safe(mapping
, next
, &bo_va
->invalids
, list
) {
1481 list_del(&mapping
->list
);
1482 interval_tree_remove(&mapping
->it
, &vm
->va
);
1486 dma_fence_put(bo_va
->last_pt_update
);
1491 * amdgpu_vm_bo_invalidate - mark the bo as invalid
1493 * @adev: amdgpu_device pointer
1495 * @bo: amdgpu buffer object
1497 * Mark @bo as invalid.
1499 void amdgpu_vm_bo_invalidate(struct amdgpu_device
*adev
,
1500 struct amdgpu_bo
*bo
)
1502 struct amdgpu_bo_va
*bo_va
;
1504 list_for_each_entry(bo_va
, &bo
->va
, bo_list
) {
1505 spin_lock(&bo_va
->vm
->status_lock
);
1506 if (list_empty(&bo_va
->vm_status
))
1507 list_add(&bo_va
->vm_status
, &bo_va
->vm
->invalidated
);
1508 spin_unlock(&bo_va
->vm
->status_lock
);
1513 * amdgpu_vm_init - initialize a vm instance
1515 * @adev: amdgpu_device pointer
1520 int amdgpu_vm_init(struct amdgpu_device
*adev
, struct amdgpu_vm
*vm
)
1522 const unsigned align
= min(AMDGPU_VM_PTB_ALIGN_SIZE
,
1523 AMDGPU_VM_PTE_COUNT
* 8);
1524 unsigned pd_size
, pd_entries
;
1525 unsigned ring_instance
;
1526 struct amdgpu_ring
*ring
;
1527 struct amd_sched_rq
*rq
;
1530 for (i
= 0; i
< AMDGPU_MAX_RINGS
; ++i
)
1533 vm
->client_id
= atomic64_inc_return(&adev
->vm_manager
.client_counter
);
1534 spin_lock_init(&vm
->status_lock
);
1535 INIT_LIST_HEAD(&vm
->invalidated
);
1536 INIT_LIST_HEAD(&vm
->cleared
);
1537 INIT_LIST_HEAD(&vm
->freed
);
1539 pd_size
= amdgpu_vm_directory_size(adev
);
1540 pd_entries
= amdgpu_vm_num_pdes(adev
);
1542 /* allocate page table array */
1543 vm
->page_tables
= drm_calloc_large(pd_entries
, sizeof(struct amdgpu_vm_pt
));
1544 if (vm
->page_tables
== NULL
) {
1545 DRM_ERROR("Cannot allocate memory for page table array\n");
1549 /* create scheduler entity for page table updates */
1551 ring_instance
= atomic_inc_return(&adev
->vm_manager
.vm_pte_next_ring
);
1552 ring_instance
%= adev
->vm_manager
.vm_pte_num_rings
;
1553 ring
= adev
->vm_manager
.vm_pte_rings
[ring_instance
];
1554 rq
= &ring
->sched
.sched_rq
[AMD_SCHED_PRIORITY_KERNEL
];
1555 r
= amd_sched_entity_init(&ring
->sched
, &vm
->entity
,
1556 rq
, amdgpu_sched_jobs
);
1560 vm
->page_directory_fence
= NULL
;
1562 r
= amdgpu_bo_create(adev
, pd_size
, align
, true,
1563 AMDGPU_GEM_DOMAIN_VRAM
,
1564 AMDGPU_GEM_CREATE_NO_CPU_ACCESS
|
1565 AMDGPU_GEM_CREATE_SHADOW
|
1566 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS
|
1567 AMDGPU_GEM_CREATE_VRAM_CLEARED
,
1568 NULL
, NULL
, &vm
->page_directory
);
1570 goto error_free_sched_entity
;
1572 r
= amdgpu_bo_reserve(vm
->page_directory
, false);
1574 goto error_free_page_directory
;
1576 vm
->last_eviction_counter
= atomic64_read(&adev
->num_evictions
);
1577 amdgpu_bo_unreserve(vm
->page_directory
);
1581 error_free_page_directory
:
1582 amdgpu_bo_unref(&vm
->page_directory
->shadow
);
1583 amdgpu_bo_unref(&vm
->page_directory
);
1584 vm
->page_directory
= NULL
;
1586 error_free_sched_entity
:
1587 amd_sched_entity_fini(&ring
->sched
, &vm
->entity
);
1590 drm_free_large(vm
->page_tables
);
1596 * amdgpu_vm_fini - tear down a vm instance
1598 * @adev: amdgpu_device pointer
1602 * Unbind the VM and remove all bos from the vm bo list
1604 void amdgpu_vm_fini(struct amdgpu_device
*adev
, struct amdgpu_vm
*vm
)
1606 struct amdgpu_bo_va_mapping
*mapping
, *tmp
;
1609 amd_sched_entity_fini(vm
->entity
.sched
, &vm
->entity
);
1611 if (!RB_EMPTY_ROOT(&vm
->va
)) {
1612 dev_err(adev
->dev
, "still active bo inside vm\n");
1614 rbtree_postorder_for_each_entry_safe(mapping
, tmp
, &vm
->va
, it
.rb
) {
1615 list_del(&mapping
->list
);
1616 interval_tree_remove(&mapping
->it
, &vm
->va
);
1619 list_for_each_entry_safe(mapping
, tmp
, &vm
->freed
, list
) {
1620 list_del(&mapping
->list
);
1624 for (i
= 0; i
< amdgpu_vm_num_pdes(adev
); i
++) {
1625 struct amdgpu_bo
*pt
= vm
->page_tables
[i
].bo
;
1630 amdgpu_bo_unref(&pt
->shadow
);
1631 amdgpu_bo_unref(&pt
);
1633 drm_free_large(vm
->page_tables
);
1635 amdgpu_bo_unref(&vm
->page_directory
->shadow
);
1636 amdgpu_bo_unref(&vm
->page_directory
);
1637 dma_fence_put(vm
->page_directory_fence
);
1641 * amdgpu_vm_manager_init - init the VM manager
1643 * @adev: amdgpu_device pointer
1645 * Initialize the VM manager structures
1647 void amdgpu_vm_manager_init(struct amdgpu_device
*adev
)
1651 INIT_LIST_HEAD(&adev
->vm_manager
.ids_lru
);
1653 /* skip over VMID 0, since it is the system VM */
1654 for (i
= 1; i
< adev
->vm_manager
.num_ids
; ++i
) {
1655 amdgpu_vm_reset_id(adev
, i
);
1656 amdgpu_sync_create(&adev
->vm_manager
.ids
[i
].active
);
1657 list_add_tail(&adev
->vm_manager
.ids
[i
].list
,
1658 &adev
->vm_manager
.ids_lru
);
1661 adev
->vm_manager
.fence_context
=
1662 dma_fence_context_alloc(AMDGPU_MAX_RINGS
);
1663 for (i
= 0; i
< AMDGPU_MAX_RINGS
; ++i
)
1664 adev
->vm_manager
.seqno
[i
] = 0;
1666 atomic_set(&adev
->vm_manager
.vm_pte_next_ring
, 0);
1667 atomic64_set(&adev
->vm_manager
.client_counter
, 0);
1671 * amdgpu_vm_manager_fini - cleanup VM manager
1673 * @adev: amdgpu_device pointer
1675 * Cleanup the VM manager and free resources.
1677 void amdgpu_vm_manager_fini(struct amdgpu_device
*adev
)
1681 for (i
= 0; i
< AMDGPU_NUM_VM
; ++i
) {
1682 struct amdgpu_vm_id
*id
= &adev
->vm_manager
.ids
[i
];
1684 dma_fence_put(adev
->vm_manager
.ids
[i
].first
);
1685 amdgpu_sync_free(&adev
->vm_manager
.ids
[i
].active
);
1686 dma_fence_put(id
->flushed_updates
);
1687 dma_fence_put(id
->last_flush
);