2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
24 * Authors: Dave Airlie
28 #include <linux/dma-fence-array.h>
29 #include <linux/interval_tree_generic.h>
31 #include <drm/amdgpu_drm.h>
33 #include "amdgpu_trace.h"
37 * GPUVM is similar to the legacy gart on older asics, however
38 * rather than there being a single global gart table
39 * for the entire GPU, there are multiple VM page tables active
40 * at any given time. The VM page tables can contain a mix
41 * vram pages and system memory pages and system memory pages
42 * can be mapped as snooped (cached system pages) or unsnooped
43 * (uncached system pages).
44 * Each VM has an ID associated with it and there is a page table
45 * associated with each VMID. When execting a command buffer,
46 * the kernel tells the the ring what VMID to use for that command
47 * buffer. VMIDs are allocated dynamically as commands are submitted.
48 * The userspace drivers maintain their own address space and the kernel
49 * sets up their pages tables accordingly when they submit their
50 * command buffers and a VMID is assigned.
51 * Cayman/Trinity support up to 8 active VMs at any given time;
55 #define START(node) ((node)->start)
56 #define LAST(node) ((node)->last)
58 INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping
, rb
, uint64_t, __subtree_last
,
59 START
, LAST
, static, amdgpu_vm_it
)
64 /* Local structure. Encapsulate some VM table update parameters to reduce
65 * the number of function parameters
67 struct amdgpu_pte_update_params
{
68 /* amdgpu device we do this update for */
69 struct amdgpu_device
*adev
;
70 /* optional amdgpu_vm we do this update for */
72 /* address where to copy page table entries from */
74 /* indirect buffer to fill with commands */
76 /* Function which actually does the update */
77 void (*func
)(struct amdgpu_pte_update_params
*params
, uint64_t pe
,
78 uint64_t addr
, unsigned count
, uint32_t incr
,
80 /* indicate update pt or its shadow */
84 /* Helper to disable partial resident texture feature from a fence callback */
85 struct amdgpu_prt_cb
{
86 struct amdgpu_device
*adev
;
87 struct dma_fence_cb cb
;
91 * amdgpu_vm_num_entries - return the number of entries in a PD/PT
93 * @adev: amdgpu_device pointer
95 * Calculate the number of entries in a page directory or page table.
97 static unsigned amdgpu_vm_num_entries(struct amdgpu_device
*adev
,
101 /* For the root directory */
102 return adev
->vm_manager
.max_pfn
>>
103 (adev
->vm_manager
.block_size
*
104 adev
->vm_manager
.num_level
);
105 else if (level
== adev
->vm_manager
.num_level
)
106 /* For the page tables on the leaves */
107 return AMDGPU_VM_PTE_COUNT(adev
);
109 /* Everything in between */
110 return 1 << adev
->vm_manager
.block_size
;
114 * amdgpu_vm_bo_size - returns the size of the BOs in bytes
116 * @adev: amdgpu_device pointer
118 * Calculate the size of the BO for a page directory or page table in bytes.
120 static unsigned amdgpu_vm_bo_size(struct amdgpu_device
*adev
, unsigned level
)
122 return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev
, level
) * 8);
126 * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
128 * @vm: vm providing the BOs
129 * @validated: head of validation list
130 * @entry: entry to add
132 * Add the page directory to the list of BOs to
133 * validate for command submission.
135 void amdgpu_vm_get_pd_bo(struct amdgpu_vm
*vm
,
136 struct list_head
*validated
,
137 struct amdgpu_bo_list_entry
*entry
)
139 entry
->robj
= vm
->root
.bo
;
141 entry
->tv
.bo
= &entry
->robj
->tbo
;
142 entry
->tv
.shared
= true;
143 entry
->user_pages
= NULL
;
144 list_add(&entry
->tv
.head
, validated
);
148 * amdgpu_vm_validate_layer - validate a single page table level
150 * @parent: parent page table level
151 * @validate: callback to do the validation
152 * @param: parameter for the validation callback
154 * Validate the page table BOs on command submission if neccessary.
156 static int amdgpu_vm_validate_level(struct amdgpu_vm_pt
*parent
,
157 int (*validate
)(void *, struct amdgpu_bo
*),
163 if (!parent
->entries
)
166 for (i
= 0; i
<= parent
->last_entry_used
; ++i
) {
167 struct amdgpu_vm_pt
*entry
= &parent
->entries
[i
];
172 r
= validate(param
, entry
->bo
);
177 * Recurse into the sub directory. This is harmless because we
178 * have only a maximum of 5 layers.
180 r
= amdgpu_vm_validate_level(entry
, validate
, param
);
189 * amdgpu_vm_validate_pt_bos - validate the page table BOs
191 * @adev: amdgpu device pointer
192 * @vm: vm providing the BOs
193 * @validate: callback to do the validation
194 * @param: parameter for the validation callback
196 * Validate the page table BOs on command submission if neccessary.
198 int amdgpu_vm_validate_pt_bos(struct amdgpu_device
*adev
, struct amdgpu_vm
*vm
,
199 int (*validate
)(void *p
, struct amdgpu_bo
*bo
),
202 uint64_t num_evictions
;
204 /* We only need to validate the page tables
205 * if they aren't already valid.
207 num_evictions
= atomic64_read(&adev
->num_evictions
);
208 if (num_evictions
== vm
->last_eviction_counter
)
211 return amdgpu_vm_validate_level(&vm
->root
, validate
, param
);
215 * amdgpu_vm_move_level_in_lru - move one level of PT BOs to the LRU tail
217 * @adev: amdgpu device instance
218 * @vm: vm providing the BOs
220 * Move the PT BOs to the tail of the LRU.
222 static void amdgpu_vm_move_level_in_lru(struct amdgpu_vm_pt
*parent
)
226 if (!parent
->entries
)
229 for (i
= 0; i
<= parent
->last_entry_used
; ++i
) {
230 struct amdgpu_vm_pt
*entry
= &parent
->entries
[i
];
235 ttm_bo_move_to_lru_tail(&entry
->bo
->tbo
);
236 amdgpu_vm_move_level_in_lru(entry
);
241 * amdgpu_vm_move_pt_bos_in_lru - move the PT BOs to the LRU tail
243 * @adev: amdgpu device instance
244 * @vm: vm providing the BOs
246 * Move the PT BOs to the tail of the LRU.
248 void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device
*adev
,
249 struct amdgpu_vm
*vm
)
251 struct ttm_bo_global
*glob
= adev
->mman
.bdev
.glob
;
253 spin_lock(&glob
->lru_lock
);
254 amdgpu_vm_move_level_in_lru(&vm
->root
);
255 spin_unlock(&glob
->lru_lock
);
259 * amdgpu_vm_alloc_levels - allocate the PD/PT levels
261 * @adev: amdgpu_device pointer
263 * @saddr: start of the address range
264 * @eaddr: end of the address range
266 * Make sure the page directories and page tables are allocated
268 static int amdgpu_vm_alloc_levels(struct amdgpu_device
*adev
,
269 struct amdgpu_vm
*vm
,
270 struct amdgpu_vm_pt
*parent
,
271 uint64_t saddr
, uint64_t eaddr
,
274 unsigned shift
= (adev
->vm_manager
.num_level
- level
) *
275 adev
->vm_manager
.block_size
;
276 unsigned pt_idx
, from
, to
;
279 if (!parent
->entries
) {
280 unsigned num_entries
= amdgpu_vm_num_entries(adev
, level
);
282 parent
->entries
= drm_calloc_large(num_entries
,
283 sizeof(struct amdgpu_vm_pt
));
284 if (!parent
->entries
)
286 memset(parent
->entries
, 0 , sizeof(struct amdgpu_vm_pt
));
289 from
= saddr
>> shift
;
291 if (from
>= amdgpu_vm_num_entries(adev
, level
) ||
292 to
>= amdgpu_vm_num_entries(adev
, level
))
295 if (to
> parent
->last_entry_used
)
296 parent
->last_entry_used
= to
;
299 saddr
= saddr
& ((1 << shift
) - 1);
300 eaddr
= eaddr
& ((1 << shift
) - 1);
302 /* walk over the address space and allocate the page tables */
303 for (pt_idx
= from
; pt_idx
<= to
; ++pt_idx
) {
304 struct reservation_object
*resv
= vm
->root
.bo
->tbo
.resv
;
305 struct amdgpu_vm_pt
*entry
= &parent
->entries
[pt_idx
];
306 struct amdgpu_bo
*pt
;
309 r
= amdgpu_bo_create(adev
,
310 amdgpu_vm_bo_size(adev
, level
),
311 AMDGPU_GPU_PAGE_SIZE
, true,
312 AMDGPU_GEM_DOMAIN_VRAM
,
313 AMDGPU_GEM_CREATE_NO_CPU_ACCESS
|
314 AMDGPU_GEM_CREATE_SHADOW
|
315 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS
|
316 AMDGPU_GEM_CREATE_VRAM_CLEARED
,
321 /* Keep a reference to the root directory to avoid
322 * freeing them up in the wrong order.
324 pt
->parent
= amdgpu_bo_ref(vm
->root
.bo
);
330 if (level
< adev
->vm_manager
.num_level
) {
331 uint64_t sub_saddr
= (pt_idx
== from
) ? saddr
: 0;
332 uint64_t sub_eaddr
= (pt_idx
== to
) ? eaddr
:
334 r
= amdgpu_vm_alloc_levels(adev
, vm
, entry
, sub_saddr
,
345 * amdgpu_vm_alloc_pts - Allocate page tables.
347 * @adev: amdgpu_device pointer
348 * @vm: VM to allocate page tables for
349 * @saddr: Start address which needs to be allocated
350 * @size: Size from start address we need.
352 * Make sure the page tables are allocated.
354 int amdgpu_vm_alloc_pts(struct amdgpu_device
*adev
,
355 struct amdgpu_vm
*vm
,
356 uint64_t saddr
, uint64_t size
)
361 /* validate the parameters */
362 if (saddr
& AMDGPU_GPU_PAGE_MASK
|| size
& AMDGPU_GPU_PAGE_MASK
)
365 eaddr
= saddr
+ size
- 1;
366 last_pfn
= eaddr
/ AMDGPU_GPU_PAGE_SIZE
;
367 if (last_pfn
>= adev
->vm_manager
.max_pfn
) {
368 dev_err(adev
->dev
, "va above limit (0x%08llX >= 0x%08llX)\n",
369 last_pfn
, adev
->vm_manager
.max_pfn
);
373 saddr
/= AMDGPU_GPU_PAGE_SIZE
;
374 eaddr
/= AMDGPU_GPU_PAGE_SIZE
;
376 return amdgpu_vm_alloc_levels(adev
, vm
, &vm
->root
, saddr
, eaddr
, 0);
380 * amdgpu_vm_had_gpu_reset - check if reset occured since last use
382 * @adev: amdgpu_device pointer
383 * @id: VMID structure
385 * Check if GPU reset occured since last use of the VMID.
387 static bool amdgpu_vm_had_gpu_reset(struct amdgpu_device
*adev
,
388 struct amdgpu_vm_id
*id
)
390 return id
->current_gpu_reset_count
!=
391 atomic_read(&adev
->gpu_reset_counter
);
395 * amdgpu_vm_grab_id - allocate the next free VMID
397 * @vm: vm to allocate id for
398 * @ring: ring we want to submit job to
399 * @sync: sync object where we add dependencies
400 * @fence: fence protecting ID from reuse
402 * Allocate an id for the vm, adding fences to the sync obj as necessary.
404 int amdgpu_vm_grab_id(struct amdgpu_vm
*vm
, struct amdgpu_ring
*ring
,
405 struct amdgpu_sync
*sync
, struct dma_fence
*fence
,
406 struct amdgpu_job
*job
)
408 struct amdgpu_device
*adev
= ring
->adev
;
409 uint64_t fence_context
= adev
->fence_context
+ ring
->idx
;
410 struct dma_fence
*updates
= sync
->last_vm_update
;
411 struct amdgpu_vm_id
*id
, *idle
;
412 struct dma_fence
**fences
;
416 fences
= kmalloc_array(sizeof(void *), adev
->vm_manager
.num_ids
,
421 mutex_lock(&adev
->vm_manager
.lock
);
423 /* Check if we have an idle VMID */
425 list_for_each_entry(idle
, &adev
->vm_manager
.ids_lru
, list
) {
426 fences
[i
] = amdgpu_sync_peek_fence(&idle
->active
, ring
);
432 /* If we can't find a idle VMID to use, wait till one becomes available */
433 if (&idle
->list
== &adev
->vm_manager
.ids_lru
) {
434 u64 fence_context
= adev
->vm_manager
.fence_context
+ ring
->idx
;
435 unsigned seqno
= ++adev
->vm_manager
.seqno
[ring
->idx
];
436 struct dma_fence_array
*array
;
439 for (j
= 0; j
< i
; ++j
)
440 dma_fence_get(fences
[j
]);
442 array
= dma_fence_array_create(i
, fences
, fence_context
,
445 for (j
= 0; j
< i
; ++j
)
446 dma_fence_put(fences
[j
]);
453 r
= amdgpu_sync_fence(ring
->adev
, sync
, &array
->base
);
454 dma_fence_put(&array
->base
);
458 mutex_unlock(&adev
->vm_manager
.lock
);
464 job
->vm_needs_flush
= true;
465 /* Check if we can use a VMID already assigned to this VM */
468 struct dma_fence
*flushed
;
471 if (i
== AMDGPU_MAX_RINGS
)
474 /* Check all the prerequisites to using this VMID */
477 if (amdgpu_vm_had_gpu_reset(adev
, id
))
480 if (atomic64_read(&id
->owner
) != vm
->client_id
)
483 if (job
->vm_pd_addr
!= id
->pd_gpu_addr
)
489 if (id
->last_flush
->context
!= fence_context
&&
490 !dma_fence_is_signaled(id
->last_flush
))
493 flushed
= id
->flushed_updates
;
495 (!flushed
|| dma_fence_is_later(updates
, flushed
)))
498 /* Good we can use this VMID. Remember this submission as
501 r
= amdgpu_sync_fence(ring
->adev
, &id
->active
, fence
);
505 list_move_tail(&id
->list
, &adev
->vm_manager
.ids_lru
);
506 vm
->ids
[ring
->idx
] = id
;
508 job
->vm_id
= id
- adev
->vm_manager
.ids
;
509 job
->vm_needs_flush
= false;
510 trace_amdgpu_vm_grab_id(vm
, ring
->idx
, job
);
512 mutex_unlock(&adev
->vm_manager
.lock
);
515 } while (i
!= ring
->idx
);
517 /* Still no ID to use? Then use the idle one found earlier */
520 /* Remember this submission as user of the VMID */
521 r
= amdgpu_sync_fence(ring
->adev
, &id
->active
, fence
);
525 dma_fence_put(id
->last_flush
);
526 id
->last_flush
= NULL
;
528 dma_fence_put(id
->flushed_updates
);
529 id
->flushed_updates
= dma_fence_get(updates
);
531 id
->pd_gpu_addr
= job
->vm_pd_addr
;
532 id
->current_gpu_reset_count
= atomic_read(&adev
->gpu_reset_counter
);
533 list_move_tail(&id
->list
, &adev
->vm_manager
.ids_lru
);
534 atomic64_set(&id
->owner
, vm
->client_id
);
535 vm
->ids
[ring
->idx
] = id
;
537 job
->vm_id
= id
- adev
->vm_manager
.ids
;
538 trace_amdgpu_vm_grab_id(vm
, ring
->idx
, job
);
541 mutex_unlock(&adev
->vm_manager
.lock
);
545 static bool amdgpu_vm_ring_has_compute_vm_bug(struct amdgpu_ring
*ring
)
547 struct amdgpu_device
*adev
= ring
->adev
;
548 const struct amdgpu_ip_block
*ip_block
;
550 if (ring
->funcs
->type
!= AMDGPU_RING_TYPE_COMPUTE
)
551 /* only compute rings */
554 ip_block
= amdgpu_get_ip_block(adev
, AMD_IP_BLOCK_TYPE_GFX
);
558 if (ip_block
->version
->major
<= 7) {
559 /* gfx7 has no workaround */
561 } else if (ip_block
->version
->major
== 8) {
562 if (adev
->gfx
.mec_fw_version
>= 673)
563 /* gfx8 is fixed in MEC firmware 673 */
571 static u64
amdgpu_vm_adjust_mc_addr(struct amdgpu_device
*adev
, u64 mc_addr
)
575 if (adev
->gart
.gart_funcs
->adjust_mc_addr
)
576 addr
= adev
->gart
.gart_funcs
->adjust_mc_addr(adev
, addr
);
582 * amdgpu_vm_flush - hardware flush the vm
584 * @ring: ring to use for flush
585 * @vm_id: vmid number to use
586 * @pd_addr: address of the page directory
588 * Emit a VM flush when it is necessary.
590 int amdgpu_vm_flush(struct amdgpu_ring
*ring
, struct amdgpu_job
*job
)
592 struct amdgpu_device
*adev
= ring
->adev
;
593 struct amdgpu_vm_id
*id
= &adev
->vm_manager
.ids
[job
->vm_id
];
594 bool gds_switch_needed
= ring
->funcs
->emit_gds_switch
&& (
595 id
->gds_base
!= job
->gds_base
||
596 id
->gds_size
!= job
->gds_size
||
597 id
->gws_base
!= job
->gws_base
||
598 id
->gws_size
!= job
->gws_size
||
599 id
->oa_base
!= job
->oa_base
||
600 id
->oa_size
!= job
->oa_size
);
601 bool vm_flush_needed
= job
->vm_needs_flush
||
602 amdgpu_vm_ring_has_compute_vm_bug(ring
);
603 unsigned patch_offset
= 0;
606 if (amdgpu_vm_had_gpu_reset(adev
, id
)) {
607 gds_switch_needed
= true;
608 vm_flush_needed
= true;
611 if (!vm_flush_needed
&& !gds_switch_needed
)
614 if (ring
->funcs
->init_cond_exec
)
615 patch_offset
= amdgpu_ring_init_cond_exec(ring
);
617 if (ring
->funcs
->emit_pipeline_sync
)
618 amdgpu_ring_emit_pipeline_sync(ring
);
620 if (ring
->funcs
->emit_vm_flush
&& vm_flush_needed
) {
621 u64 pd_addr
= amdgpu_vm_adjust_mc_addr(adev
, job
->vm_pd_addr
);
622 struct dma_fence
*fence
;
624 trace_amdgpu_vm_flush(pd_addr
, ring
->idx
, job
->vm_id
);
625 amdgpu_ring_emit_vm_flush(ring
, job
->vm_id
, pd_addr
);
627 r
= amdgpu_fence_emit(ring
, &fence
);
631 mutex_lock(&adev
->vm_manager
.lock
);
632 dma_fence_put(id
->last_flush
);
633 id
->last_flush
= fence
;
634 mutex_unlock(&adev
->vm_manager
.lock
);
637 if (gds_switch_needed
) {
638 id
->gds_base
= job
->gds_base
;
639 id
->gds_size
= job
->gds_size
;
640 id
->gws_base
= job
->gws_base
;
641 id
->gws_size
= job
->gws_size
;
642 id
->oa_base
= job
->oa_base
;
643 id
->oa_size
= job
->oa_size
;
644 amdgpu_ring_emit_gds_switch(ring
, job
->vm_id
, job
->gds_base
,
645 job
->gds_size
, job
->gws_base
,
646 job
->gws_size
, job
->oa_base
,
650 if (ring
->funcs
->patch_cond_exec
)
651 amdgpu_ring_patch_cond_exec(ring
, patch_offset
);
653 /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
654 if (ring
->funcs
->emit_switch_buffer
) {
655 amdgpu_ring_emit_switch_buffer(ring
);
656 amdgpu_ring_emit_switch_buffer(ring
);
662 * amdgpu_vm_reset_id - reset VMID to zero
664 * @adev: amdgpu device structure
665 * @vm_id: vmid number to use
667 * Reset saved GDW, GWS and OA to force switch on next flush.
669 void amdgpu_vm_reset_id(struct amdgpu_device
*adev
, unsigned vm_id
)
671 struct amdgpu_vm_id
*id
= &adev
->vm_manager
.ids
[vm_id
];
682 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
685 * @bo: requested buffer object
687 * Find @bo inside the requested vm.
688 * Search inside the @bos vm list for the requested vm
689 * Returns the found bo_va or NULL if none is found
691 * Object has to be reserved!
693 struct amdgpu_bo_va
*amdgpu_vm_bo_find(struct amdgpu_vm
*vm
,
694 struct amdgpu_bo
*bo
)
696 struct amdgpu_bo_va
*bo_va
;
698 list_for_each_entry(bo_va
, &bo
->va
, bo_list
) {
699 if (bo_va
->vm
== vm
) {
707 * amdgpu_vm_do_set_ptes - helper to call the right asic function
709 * @params: see amdgpu_pte_update_params definition
710 * @pe: addr of the page entry
711 * @addr: dst addr to write into pe
712 * @count: number of page entries to update
713 * @incr: increase next addr by incr bytes
714 * @flags: hw access flags
716 * Traces the parameters and calls the right asic functions
717 * to setup the page table using the DMA.
719 static void amdgpu_vm_do_set_ptes(struct amdgpu_pte_update_params
*params
,
720 uint64_t pe
, uint64_t addr
,
721 unsigned count
, uint32_t incr
,
724 trace_amdgpu_vm_set_ptes(pe
, addr
, count
, incr
, flags
);
727 amdgpu_vm_write_pte(params
->adev
, params
->ib
, pe
,
728 addr
| flags
, count
, incr
);
731 amdgpu_vm_set_pte_pde(params
->adev
, params
->ib
, pe
, addr
,
737 * amdgpu_vm_do_copy_ptes - copy the PTEs from the GART
739 * @params: see amdgpu_pte_update_params definition
740 * @pe: addr of the page entry
741 * @addr: dst addr to write into pe
742 * @count: number of page entries to update
743 * @incr: increase next addr by incr bytes
744 * @flags: hw access flags
746 * Traces the parameters and calls the DMA function to copy the PTEs.
748 static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params
*params
,
749 uint64_t pe
, uint64_t addr
,
750 unsigned count
, uint32_t incr
,
753 uint64_t src
= (params
->src
+ (addr
>> 12) * 8);
756 trace_amdgpu_vm_copy_ptes(pe
, src
, count
);
758 amdgpu_vm_copy_pte(params
->adev
, params
->ib
, pe
, src
, count
);
762 * amdgpu_vm_map_gart - Resolve gart mapping of addr
764 * @pages_addr: optional DMA address to use for lookup
765 * @addr: the unmapped addr
767 * Look up the physical address of the page that the pte resolves
768 * to and return the pointer for the page table entry.
770 static uint64_t amdgpu_vm_map_gart(const dma_addr_t
*pages_addr
, uint64_t addr
)
774 /* page table offset */
775 result
= pages_addr
[addr
>> PAGE_SHIFT
];
777 /* in case cpu page size != gpu page size*/
778 result
|= addr
& (~PAGE_MASK
);
780 result
&= 0xFFFFFFFFFFFFF000ULL
;
786 * amdgpu_vm_update_level - update a single level in the hierarchy
788 * @adev: amdgpu_device pointer
790 * @parent: parent directory
792 * Makes sure all entries in @parent are up to date.
793 * Returns 0 for success, error for failure.
795 static int amdgpu_vm_update_level(struct amdgpu_device
*adev
,
796 struct amdgpu_vm
*vm
,
797 struct amdgpu_vm_pt
*parent
,
800 struct amdgpu_bo
*shadow
;
801 struct amdgpu_ring
*ring
;
802 uint64_t pd_addr
, shadow_addr
;
803 uint32_t incr
= amdgpu_vm_bo_size(adev
, level
+ 1);
804 uint64_t last_pde
= ~0, last_pt
= ~0, last_shadow
= ~0;
805 unsigned count
= 0, pt_idx
, ndw
;
806 struct amdgpu_job
*job
;
807 struct amdgpu_pte_update_params params
;
808 struct dma_fence
*fence
= NULL
;
812 if (!parent
->entries
)
814 ring
= container_of(vm
->entity
.sched
, struct amdgpu_ring
, sched
);
819 /* assume the worst case */
820 ndw
+= parent
->last_entry_used
* 6;
822 pd_addr
= amdgpu_bo_gpu_offset(parent
->bo
);
824 shadow
= parent
->bo
->shadow
;
826 r
= amdgpu_ttm_bind(&shadow
->tbo
, &shadow
->tbo
.mem
);
829 shadow_addr
= amdgpu_bo_gpu_offset(shadow
);
835 r
= amdgpu_job_alloc_with_ib(adev
, ndw
* 4, &job
);
839 memset(¶ms
, 0, sizeof(params
));
841 params
.ib
= &job
->ibs
[0];
843 /* walk over the address space and update the directory */
844 for (pt_idx
= 0; pt_idx
<= parent
->last_entry_used
; ++pt_idx
) {
845 struct amdgpu_bo
*bo
= parent
->entries
[pt_idx
].bo
;
852 struct amdgpu_bo
*pt_shadow
= bo
->shadow
;
854 r
= amdgpu_ttm_bind(&pt_shadow
->tbo
,
855 &pt_shadow
->tbo
.mem
);
860 pt
= amdgpu_bo_gpu_offset(bo
);
861 if (parent
->entries
[pt_idx
].addr
== pt
)
864 parent
->entries
[pt_idx
].addr
= pt
;
866 pde
= pd_addr
+ pt_idx
* 8;
867 if (((last_pde
+ 8 * count
) != pde
) ||
868 ((last_pt
+ incr
* count
) != pt
) ||
869 (count
== AMDGPU_VM_MAX_UPDATE_SIZE
)) {
873 amdgpu_vm_adjust_mc_addr(adev
, last_pt
);
876 amdgpu_vm_do_set_ptes(¶ms
,
882 amdgpu_vm_do_set_ptes(¶ms
, last_pde
,
883 pt_addr
, count
, incr
,
889 last_shadow
= shadow_addr
+ pt_idx
* 8;
897 uint64_t pt_addr
= amdgpu_vm_adjust_mc_addr(adev
, last_pt
);
899 if (vm
->root
.bo
->shadow
)
900 amdgpu_vm_do_set_ptes(¶ms
, last_shadow
, pt_addr
,
901 count
, incr
, AMDGPU_PTE_VALID
);
903 amdgpu_vm_do_set_ptes(¶ms
, last_pde
, pt_addr
,
904 count
, incr
, AMDGPU_PTE_VALID
);
907 if (params
.ib
->length_dw
== 0) {
908 amdgpu_job_free(job
);
910 amdgpu_ring_pad_ib(ring
, params
.ib
);
911 amdgpu_sync_resv(adev
, &job
->sync
, parent
->bo
->tbo
.resv
,
912 AMDGPU_FENCE_OWNER_VM
);
914 amdgpu_sync_resv(adev
, &job
->sync
, shadow
->tbo
.resv
,
915 AMDGPU_FENCE_OWNER_VM
);
917 WARN_ON(params
.ib
->length_dw
> ndw
);
918 r
= amdgpu_job_submit(job
, ring
, &vm
->entity
,
919 AMDGPU_FENCE_OWNER_VM
, &fence
);
923 amdgpu_bo_fence(parent
->bo
, fence
, true);
924 dma_fence_put(vm
->last_dir_update
);
925 vm
->last_dir_update
= dma_fence_get(fence
);
926 dma_fence_put(fence
);
929 * Recurse into the subdirectories. This recursion is harmless because
930 * we only have a maximum of 5 layers.
932 for (pt_idx
= 0; pt_idx
<= parent
->last_entry_used
; ++pt_idx
) {
933 struct amdgpu_vm_pt
*entry
= &parent
->entries
[pt_idx
];
938 r
= amdgpu_vm_update_level(adev
, vm
, entry
, level
+ 1);
946 amdgpu_job_free(job
);
951 * amdgpu_vm_update_directories - make sure that all directories are valid
953 * @adev: amdgpu_device pointer
956 * Makes sure all directories are up to date.
957 * Returns 0 for success, error for failure.
959 int amdgpu_vm_update_directories(struct amdgpu_device
*adev
,
960 struct amdgpu_vm
*vm
)
962 return amdgpu_vm_update_level(adev
, vm
, &vm
->root
, 0);
966 * amdgpu_vm_find_pt - find the page table for an address
968 * @p: see amdgpu_pte_update_params definition
969 * @addr: virtual address in question
971 * Find the page table BO for a virtual address, return NULL when none found.
973 static struct amdgpu_bo
*amdgpu_vm_get_pt(struct amdgpu_pte_update_params
*p
,
976 struct amdgpu_vm_pt
*entry
= &p
->vm
->root
;
977 unsigned idx
, level
= p
->adev
->vm_manager
.num_level
;
979 while (entry
->entries
) {
980 idx
= addr
>> (p
->adev
->vm_manager
.block_size
* level
--);
981 idx
%= amdgpu_bo_size(entry
->bo
) / 8;
982 entry
= &entry
->entries
[idx
];
992 * amdgpu_vm_update_ptes - make sure that page tables are valid
994 * @params: see amdgpu_pte_update_params definition
996 * @start: start of GPU address range
997 * @end: end of GPU address range
998 * @dst: destination address to map to, the next dst inside the function
999 * @flags: mapping flags
1001 * Update the page tables in the range @start - @end.
1003 static void amdgpu_vm_update_ptes(struct amdgpu_pte_update_params
*params
,
1004 uint64_t start
, uint64_t end
,
1005 uint64_t dst
, uint64_t flags
)
1007 struct amdgpu_device
*adev
= params
->adev
;
1008 const uint64_t mask
= AMDGPU_VM_PTE_COUNT(adev
) - 1;
1010 uint64_t cur_pe_start
, cur_nptes
, cur_dst
;
1011 uint64_t addr
; /* next GPU address to be updated */
1012 struct amdgpu_bo
*pt
;
1013 unsigned nptes
; /* next number of ptes to be updated */
1014 uint64_t next_pe_start
;
1016 /* initialize the variables */
1018 pt
= amdgpu_vm_get_pt(params
, addr
);
1020 pr_err("PT not found, aborting update_ptes\n");
1024 if (params
->shadow
) {
1029 if ((addr
& ~mask
) == (end
& ~mask
))
1032 nptes
= AMDGPU_VM_PTE_COUNT(adev
) - (addr
& mask
);
1034 cur_pe_start
= amdgpu_bo_gpu_offset(pt
);
1035 cur_pe_start
+= (addr
& mask
) * 8;
1041 dst
+= nptes
* AMDGPU_GPU_PAGE_SIZE
;
1043 /* walk over the address space and update the page tables */
1044 while (addr
< end
) {
1045 pt
= amdgpu_vm_get_pt(params
, addr
);
1047 pr_err("PT not found, aborting update_ptes\n");
1051 if (params
->shadow
) {
1057 if ((addr
& ~mask
) == (end
& ~mask
))
1060 nptes
= AMDGPU_VM_PTE_COUNT(adev
) - (addr
& mask
);
1062 next_pe_start
= amdgpu_bo_gpu_offset(pt
);
1063 next_pe_start
+= (addr
& mask
) * 8;
1065 if ((cur_pe_start
+ 8 * cur_nptes
) == next_pe_start
&&
1066 ((cur_nptes
+ nptes
) <= AMDGPU_VM_MAX_UPDATE_SIZE
)) {
1067 /* The next ptb is consecutive to current ptb.
1068 * Don't call the update function now.
1069 * Will update two ptbs together in future.
1073 params
->func(params
, cur_pe_start
, cur_dst
, cur_nptes
,
1074 AMDGPU_GPU_PAGE_SIZE
, flags
);
1076 cur_pe_start
= next_pe_start
;
1083 dst
+= nptes
* AMDGPU_GPU_PAGE_SIZE
;
1086 params
->func(params
, cur_pe_start
, cur_dst
, cur_nptes
,
1087 AMDGPU_GPU_PAGE_SIZE
, flags
);
1091 * amdgpu_vm_frag_ptes - add fragment information to PTEs
1093 * @params: see amdgpu_pte_update_params definition
1095 * @start: first PTE to handle
1096 * @end: last PTE to handle
1097 * @dst: addr those PTEs should point to
1098 * @flags: hw mapping flags
1100 static void amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params
*params
,
1101 uint64_t start
, uint64_t end
,
1102 uint64_t dst
, uint64_t flags
)
1105 * The MC L1 TLB supports variable sized pages, based on a fragment
1106 * field in the PTE. When this field is set to a non-zero value, page
1107 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
1108 * flags are considered valid for all PTEs within the fragment range
1109 * and corresponding mappings are assumed to be physically contiguous.
1111 * The L1 TLB can store a single PTE for the whole fragment,
1112 * significantly increasing the space available for translation
1113 * caching. This leads to large improvements in throughput when the
1114 * TLB is under pressure.
1116 * The L2 TLB distributes small and large fragments into two
1117 * asymmetric partitions. The large fragment cache is significantly
1118 * larger. Thus, we try to use large fragments wherever possible.
1119 * Userspace can support this by aligning virtual base address and
1120 * allocation size to the fragment size.
1123 /* SI and newer are optimized for 64KB */
1124 uint64_t frag_flags
= AMDGPU_PTE_FRAG(AMDGPU_LOG2_PAGES_PER_FRAG
);
1125 uint64_t frag_align
= 1 << AMDGPU_LOG2_PAGES_PER_FRAG
;
1127 uint64_t frag_start
= ALIGN(start
, frag_align
);
1128 uint64_t frag_end
= end
& ~(frag_align
- 1);
1130 /* system pages are non continuously */
1131 if (params
->src
|| !(flags
& AMDGPU_PTE_VALID
) ||
1132 (frag_start
>= frag_end
)) {
1134 amdgpu_vm_update_ptes(params
, start
, end
, dst
, flags
);
1138 /* handle the 4K area at the beginning */
1139 if (start
!= frag_start
) {
1140 amdgpu_vm_update_ptes(params
, start
, frag_start
,
1142 dst
+= (frag_start
- start
) * AMDGPU_GPU_PAGE_SIZE
;
1145 /* handle the area in the middle */
1146 amdgpu_vm_update_ptes(params
, frag_start
, frag_end
, dst
,
1147 flags
| frag_flags
);
1149 /* handle the 4K area at the end */
1150 if (frag_end
!= end
) {
1151 dst
+= (frag_end
- frag_start
) * AMDGPU_GPU_PAGE_SIZE
;
1152 amdgpu_vm_update_ptes(params
, frag_end
, end
, dst
, flags
);
1157 * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
1159 * @adev: amdgpu_device pointer
1160 * @exclusive: fence we need to sync to
1161 * @src: address where to copy page table entries from
1162 * @pages_addr: DMA addresses to use for mapping
1164 * @start: start of mapped range
1165 * @last: last mapped entry
1166 * @flags: flags for the entries
1167 * @addr: addr to set the area to
1168 * @fence: optional resulting fence
1170 * Fill in the page table entries between @start and @last.
1171 * Returns 0 for success, -EINVAL for failure.
1173 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device
*adev
,
1174 struct dma_fence
*exclusive
,
1176 dma_addr_t
*pages_addr
,
1177 struct amdgpu_vm
*vm
,
1178 uint64_t start
, uint64_t last
,
1179 uint64_t flags
, uint64_t addr
,
1180 struct dma_fence
**fence
)
1182 struct amdgpu_ring
*ring
;
1183 void *owner
= AMDGPU_FENCE_OWNER_VM
;
1184 unsigned nptes
, ncmds
, ndw
;
1185 struct amdgpu_job
*job
;
1186 struct amdgpu_pte_update_params params
;
1187 struct dma_fence
*f
= NULL
;
1190 memset(¶ms
, 0, sizeof(params
));
1195 ring
= container_of(vm
->entity
.sched
, struct amdgpu_ring
, sched
);
1197 /* sync to everything on unmapping */
1198 if (!(flags
& AMDGPU_PTE_VALID
))
1199 owner
= AMDGPU_FENCE_OWNER_UNDEFINED
;
1201 nptes
= last
- start
+ 1;
1204 * reserve space for one command every (1 << BLOCK_SIZE)
1205 * entries or 2k dwords (whatever is smaller)
1207 ncmds
= (nptes
>> min(adev
->vm_manager
.block_size
, 11u)) + 1;
1213 /* only copy commands needed */
1216 params
.func
= amdgpu_vm_do_copy_ptes
;
1218 } else if (pages_addr
) {
1219 /* copy commands needed */
1225 params
.func
= amdgpu_vm_do_copy_ptes
;
1228 /* set page commands needed */
1231 /* two extra commands for begin/end of fragment */
1234 params
.func
= amdgpu_vm_do_set_ptes
;
1237 r
= amdgpu_job_alloc_with_ib(adev
, ndw
* 4, &job
);
1241 params
.ib
= &job
->ibs
[0];
1243 if (!src
&& pages_addr
) {
1247 /* Put the PTEs at the end of the IB. */
1248 i
= ndw
- nptes
* 2;
1249 pte
= (uint64_t *)&(job
->ibs
->ptr
[i
]);
1250 params
.src
= job
->ibs
->gpu_addr
+ i
* 4;
1252 for (i
= 0; i
< nptes
; ++i
) {
1253 pte
[i
] = amdgpu_vm_map_gart(pages_addr
, addr
+ i
*
1254 AMDGPU_GPU_PAGE_SIZE
);
1260 r
= amdgpu_sync_fence(adev
, &job
->sync
, exclusive
);
1264 r
= amdgpu_sync_resv(adev
, &job
->sync
, vm
->root
.bo
->tbo
.resv
,
1269 r
= reservation_object_reserve_shared(vm
->root
.bo
->tbo
.resv
);
1273 params
.shadow
= true;
1274 amdgpu_vm_frag_ptes(¶ms
, start
, last
+ 1, addr
, flags
);
1275 params
.shadow
= false;
1276 amdgpu_vm_frag_ptes(¶ms
, start
, last
+ 1, addr
, flags
);
1278 amdgpu_ring_pad_ib(ring
, params
.ib
);
1279 WARN_ON(params
.ib
->length_dw
> ndw
);
1280 r
= amdgpu_job_submit(job
, ring
, &vm
->entity
,
1281 AMDGPU_FENCE_OWNER_VM
, &f
);
1285 amdgpu_bo_fence(vm
->root
.bo
, f
, true);
1286 dma_fence_put(*fence
);
1291 amdgpu_job_free(job
);
1296 * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
1298 * @adev: amdgpu_device pointer
1299 * @exclusive: fence we need to sync to
1300 * @gtt_flags: flags as they are used for GTT
1301 * @pages_addr: DMA addresses to use for mapping
1303 * @mapping: mapped range and flags to use for the update
1304 * @flags: HW flags for the mapping
1305 * @nodes: array of drm_mm_nodes with the MC addresses
1306 * @fence: optional resulting fence
1308 * Split the mapping into smaller chunks so that each update fits
1310 * Returns 0 for success, -EINVAL for failure.
1312 static int amdgpu_vm_bo_split_mapping(struct amdgpu_device
*adev
,
1313 struct dma_fence
*exclusive
,
1315 dma_addr_t
*pages_addr
,
1316 struct amdgpu_vm
*vm
,
1317 struct amdgpu_bo_va_mapping
*mapping
,
1319 struct drm_mm_node
*nodes
,
1320 struct dma_fence
**fence
)
1322 uint64_t pfn
, src
= 0, start
= mapping
->start
;
1325 /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
1326 * but in case of something, we filter the flags in first place
1328 if (!(mapping
->flags
& AMDGPU_PTE_READABLE
))
1329 flags
&= ~AMDGPU_PTE_READABLE
;
1330 if (!(mapping
->flags
& AMDGPU_PTE_WRITEABLE
))
1331 flags
&= ~AMDGPU_PTE_WRITEABLE
;
1333 flags
&= ~AMDGPU_PTE_EXECUTABLE
;
1334 flags
|= mapping
->flags
& AMDGPU_PTE_EXECUTABLE
;
1336 flags
&= ~AMDGPU_PTE_MTYPE_MASK
;
1337 flags
|= (mapping
->flags
& AMDGPU_PTE_MTYPE_MASK
);
1339 trace_amdgpu_vm_bo_update(mapping
);
1341 pfn
= mapping
->offset
>> PAGE_SHIFT
;
1343 while (pfn
>= nodes
->size
) {
1350 uint64_t max_entries
;
1351 uint64_t addr
, last
;
1354 addr
= nodes
->start
<< PAGE_SHIFT
;
1355 max_entries
= (nodes
->size
- pfn
) *
1356 (PAGE_SIZE
/ AMDGPU_GPU_PAGE_SIZE
);
1359 max_entries
= S64_MAX
;
1363 if (flags
== gtt_flags
)
1364 src
= adev
->gart
.table_addr
+
1365 (addr
>> AMDGPU_GPU_PAGE_SHIFT
) * 8;
1367 max_entries
= min(max_entries
, 16ull * 1024ull);
1369 } else if (flags
& AMDGPU_PTE_VALID
) {
1370 addr
+= adev
->vm_manager
.vram_base_offset
;
1372 addr
+= pfn
<< PAGE_SHIFT
;
1374 last
= min((uint64_t)mapping
->last
, start
+ max_entries
- 1);
1375 r
= amdgpu_vm_bo_update_mapping(adev
, exclusive
,
1376 src
, pages_addr
, vm
,
1377 start
, last
, flags
, addr
,
1382 pfn
+= last
- start
+ 1;
1383 if (nodes
&& nodes
->size
== pfn
) {
1389 } while (unlikely(start
!= mapping
->last
+ 1));
1395 * amdgpu_vm_bo_update - update all BO mappings in the vm page table
1397 * @adev: amdgpu_device pointer
1398 * @bo_va: requested BO and VM object
1399 * @clear: if true clear the entries
1401 * Fill in the page table entries for @bo_va.
1402 * Returns 0 for success, -EINVAL for failure.
1404 int amdgpu_vm_bo_update(struct amdgpu_device
*adev
,
1405 struct amdgpu_bo_va
*bo_va
,
1408 struct amdgpu_vm
*vm
= bo_va
->vm
;
1409 struct amdgpu_bo_va_mapping
*mapping
;
1410 dma_addr_t
*pages_addr
= NULL
;
1411 uint64_t gtt_flags
, flags
;
1412 struct ttm_mem_reg
*mem
;
1413 struct drm_mm_node
*nodes
;
1414 struct dma_fence
*exclusive
;
1417 if (clear
|| !bo_va
->bo
) {
1422 struct ttm_dma_tt
*ttm
;
1424 mem
= &bo_va
->bo
->tbo
.mem
;
1425 nodes
= mem
->mm_node
;
1426 if (mem
->mem_type
== TTM_PL_TT
) {
1427 ttm
= container_of(bo_va
->bo
->tbo
.ttm
, struct
1429 pages_addr
= ttm
->dma_address
;
1431 exclusive
= reservation_object_get_excl(bo_va
->bo
->tbo
.resv
);
1435 flags
= amdgpu_ttm_tt_pte_flags(adev
, bo_va
->bo
->tbo
.ttm
, mem
);
1436 gtt_flags
= (amdgpu_ttm_is_bound(bo_va
->bo
->tbo
.ttm
) &&
1437 adev
== amdgpu_ttm_adev(bo_va
->bo
->tbo
.bdev
)) ?
1444 spin_lock(&vm
->status_lock
);
1445 if (!list_empty(&bo_va
->vm_status
))
1446 list_splice_init(&bo_va
->valids
, &bo_va
->invalids
);
1447 spin_unlock(&vm
->status_lock
);
1449 list_for_each_entry(mapping
, &bo_va
->invalids
, list
) {
1450 r
= amdgpu_vm_bo_split_mapping(adev
, exclusive
,
1451 gtt_flags
, pages_addr
, vm
,
1452 mapping
, flags
, nodes
,
1453 &bo_va
->last_pt_update
);
1458 if (trace_amdgpu_vm_bo_mapping_enabled()) {
1459 list_for_each_entry(mapping
, &bo_va
->valids
, list
)
1460 trace_amdgpu_vm_bo_mapping(mapping
);
1462 list_for_each_entry(mapping
, &bo_va
->invalids
, list
)
1463 trace_amdgpu_vm_bo_mapping(mapping
);
1466 spin_lock(&vm
->status_lock
);
1467 list_splice_init(&bo_va
->invalids
, &bo_va
->valids
);
1468 list_del_init(&bo_va
->vm_status
);
1470 list_add(&bo_va
->vm_status
, &vm
->cleared
);
1471 spin_unlock(&vm
->status_lock
);
1477 * amdgpu_vm_update_prt_state - update the global PRT state
1479 static void amdgpu_vm_update_prt_state(struct amdgpu_device
*adev
)
1481 unsigned long flags
;
1484 spin_lock_irqsave(&adev
->vm_manager
.prt_lock
, flags
);
1485 enable
= !!atomic_read(&adev
->vm_manager
.num_prt_users
);
1486 adev
->gart
.gart_funcs
->set_prt(adev
, enable
);
1487 spin_unlock_irqrestore(&adev
->vm_manager
.prt_lock
, flags
);
1491 * amdgpu_vm_prt_get - add a PRT user
1493 static void amdgpu_vm_prt_get(struct amdgpu_device
*adev
)
1495 if (!adev
->gart
.gart_funcs
->set_prt
)
1498 if (atomic_inc_return(&adev
->vm_manager
.num_prt_users
) == 1)
1499 amdgpu_vm_update_prt_state(adev
);
1503 * amdgpu_vm_prt_put - drop a PRT user
1505 static void amdgpu_vm_prt_put(struct amdgpu_device
*adev
)
1507 if (atomic_dec_return(&adev
->vm_manager
.num_prt_users
) == 0)
1508 amdgpu_vm_update_prt_state(adev
);
1512 * amdgpu_vm_prt_cb - callback for updating the PRT status
1514 static void amdgpu_vm_prt_cb(struct dma_fence
*fence
, struct dma_fence_cb
*_cb
)
1516 struct amdgpu_prt_cb
*cb
= container_of(_cb
, struct amdgpu_prt_cb
, cb
);
1518 amdgpu_vm_prt_put(cb
->adev
);
1523 * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
1525 static void amdgpu_vm_add_prt_cb(struct amdgpu_device
*adev
,
1526 struct dma_fence
*fence
)
1528 struct amdgpu_prt_cb
*cb
;
1530 if (!adev
->gart
.gart_funcs
->set_prt
)
1533 cb
= kmalloc(sizeof(struct amdgpu_prt_cb
), GFP_KERNEL
);
1535 /* Last resort when we are OOM */
1537 dma_fence_wait(fence
, false);
1539 amdgpu_vm_prt_put(adev
);
1542 if (!fence
|| dma_fence_add_callback(fence
, &cb
->cb
,
1544 amdgpu_vm_prt_cb(fence
, &cb
->cb
);
1549 * amdgpu_vm_free_mapping - free a mapping
1551 * @adev: amdgpu_device pointer
1553 * @mapping: mapping to be freed
1554 * @fence: fence of the unmap operation
1556 * Free a mapping and make sure we decrease the PRT usage count if applicable.
1558 static void amdgpu_vm_free_mapping(struct amdgpu_device
*adev
,
1559 struct amdgpu_vm
*vm
,
1560 struct amdgpu_bo_va_mapping
*mapping
,
1561 struct dma_fence
*fence
)
1563 if (mapping
->flags
& AMDGPU_PTE_PRT
)
1564 amdgpu_vm_add_prt_cb(adev
, fence
);
1569 * amdgpu_vm_prt_fini - finish all prt mappings
1571 * @adev: amdgpu_device pointer
1574 * Register a cleanup callback to disable PRT support after VM dies.
1576 static void amdgpu_vm_prt_fini(struct amdgpu_device
*adev
, struct amdgpu_vm
*vm
)
1578 struct reservation_object
*resv
= vm
->root
.bo
->tbo
.resv
;
1579 struct dma_fence
*excl
, **shared
;
1580 unsigned i
, shared_count
;
1583 r
= reservation_object_get_fences_rcu(resv
, &excl
,
1584 &shared_count
, &shared
);
1586 /* Not enough memory to grab the fence list, as last resort
1587 * block for all the fences to complete.
1589 reservation_object_wait_timeout_rcu(resv
, true, false,
1590 MAX_SCHEDULE_TIMEOUT
);
1594 /* Add a callback for each fence in the reservation object */
1595 amdgpu_vm_prt_get(adev
);
1596 amdgpu_vm_add_prt_cb(adev
, excl
);
1598 for (i
= 0; i
< shared_count
; ++i
) {
1599 amdgpu_vm_prt_get(adev
);
1600 amdgpu_vm_add_prt_cb(adev
, shared
[i
]);
1607 * amdgpu_vm_clear_freed - clear freed BOs in the PT
1609 * @adev: amdgpu_device pointer
1611 * @fence: optional resulting fence (unchanged if no work needed to be done
1612 * or if an error occurred)
1614 * Make sure all freed BOs are cleared in the PT.
1615 * Returns 0 for success.
1617 * PTs have to be reserved and mutex must be locked!
1619 int amdgpu_vm_clear_freed(struct amdgpu_device
*adev
,
1620 struct amdgpu_vm
*vm
,
1621 struct dma_fence
**fence
)
1623 struct amdgpu_bo_va_mapping
*mapping
;
1624 struct dma_fence
*f
= NULL
;
1627 while (!list_empty(&vm
->freed
)) {
1628 mapping
= list_first_entry(&vm
->freed
,
1629 struct amdgpu_bo_va_mapping
, list
);
1630 list_del(&mapping
->list
);
1632 r
= amdgpu_vm_bo_split_mapping(adev
, NULL
, 0, NULL
, vm
, mapping
,
1634 amdgpu_vm_free_mapping(adev
, vm
, mapping
, f
);
1642 dma_fence_put(*fence
);
1653 * amdgpu_vm_clear_invalids - clear invalidated BOs in the PT
1655 * @adev: amdgpu_device pointer
1658 * Make sure all invalidated BOs are cleared in the PT.
1659 * Returns 0 for success.
1661 * PTs have to be reserved and mutex must be locked!
1663 int amdgpu_vm_clear_invalids(struct amdgpu_device
*adev
,
1664 struct amdgpu_vm
*vm
, struct amdgpu_sync
*sync
)
1666 struct amdgpu_bo_va
*bo_va
= NULL
;
1669 spin_lock(&vm
->status_lock
);
1670 while (!list_empty(&vm
->invalidated
)) {
1671 bo_va
= list_first_entry(&vm
->invalidated
,
1672 struct amdgpu_bo_va
, vm_status
);
1673 spin_unlock(&vm
->status_lock
);
1675 r
= amdgpu_vm_bo_update(adev
, bo_va
, true);
1679 spin_lock(&vm
->status_lock
);
1681 spin_unlock(&vm
->status_lock
);
1684 r
= amdgpu_sync_fence(adev
, sync
, bo_va
->last_pt_update
);
1690 * amdgpu_vm_bo_add - add a bo to a specific vm
1692 * @adev: amdgpu_device pointer
1694 * @bo: amdgpu buffer object
1696 * Add @bo into the requested vm.
1697 * Add @bo to the list of bos associated with the vm
1698 * Returns newly added bo_va or NULL for failure
1700 * Object has to be reserved!
1702 struct amdgpu_bo_va
*amdgpu_vm_bo_add(struct amdgpu_device
*adev
,
1703 struct amdgpu_vm
*vm
,
1704 struct amdgpu_bo
*bo
)
1706 struct amdgpu_bo_va
*bo_va
;
1708 bo_va
= kzalloc(sizeof(struct amdgpu_bo_va
), GFP_KERNEL
);
1709 if (bo_va
== NULL
) {
1714 bo_va
->ref_count
= 1;
1715 INIT_LIST_HEAD(&bo_va
->bo_list
);
1716 INIT_LIST_HEAD(&bo_va
->valids
);
1717 INIT_LIST_HEAD(&bo_va
->invalids
);
1718 INIT_LIST_HEAD(&bo_va
->vm_status
);
1721 list_add_tail(&bo_va
->bo_list
, &bo
->va
);
1727 * amdgpu_vm_bo_map - map bo inside a vm
1729 * @adev: amdgpu_device pointer
1730 * @bo_va: bo_va to store the address
1731 * @saddr: where to map the BO
1732 * @offset: requested offset in the BO
1733 * @flags: attributes of pages (read/write/valid/etc.)
1735 * Add a mapping of the BO at the specefied addr into the VM.
1736 * Returns 0 for success, error for failure.
1738 * Object has to be reserved and unreserved outside!
1740 int amdgpu_vm_bo_map(struct amdgpu_device
*adev
,
1741 struct amdgpu_bo_va
*bo_va
,
1742 uint64_t saddr
, uint64_t offset
,
1743 uint64_t size
, uint64_t flags
)
1745 struct amdgpu_bo_va_mapping
*mapping
, *tmp
;
1746 struct amdgpu_vm
*vm
= bo_va
->vm
;
1749 /* validate the parameters */
1750 if (saddr
& AMDGPU_GPU_PAGE_MASK
|| offset
& AMDGPU_GPU_PAGE_MASK
||
1751 size
== 0 || size
& AMDGPU_GPU_PAGE_MASK
)
1754 /* make sure object fit at this offset */
1755 eaddr
= saddr
+ size
- 1;
1756 if (saddr
>= eaddr
||
1757 (bo_va
->bo
&& offset
+ size
> amdgpu_bo_size(bo_va
->bo
)))
1760 saddr
/= AMDGPU_GPU_PAGE_SIZE
;
1761 eaddr
/= AMDGPU_GPU_PAGE_SIZE
;
1763 tmp
= amdgpu_vm_it_iter_first(&vm
->va
, saddr
, eaddr
);
1765 /* bo and tmp overlap, invalid addr */
1766 dev_err(adev
->dev
, "bo %p va 0x%010Lx-0x%010Lx conflict with "
1767 "0x%010Lx-0x%010Lx\n", bo_va
->bo
, saddr
, eaddr
,
1768 tmp
->start
, tmp
->last
+ 1);
1772 mapping
= kmalloc(sizeof(*mapping
), GFP_KERNEL
);
1776 INIT_LIST_HEAD(&mapping
->list
);
1777 mapping
->start
= saddr
;
1778 mapping
->last
= eaddr
;
1779 mapping
->offset
= offset
;
1780 mapping
->flags
= flags
;
1782 list_add(&mapping
->list
, &bo_va
->invalids
);
1783 amdgpu_vm_it_insert(mapping
, &vm
->va
);
1785 if (flags
& AMDGPU_PTE_PRT
)
1786 amdgpu_vm_prt_get(adev
);
1792 * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
1794 * @adev: amdgpu_device pointer
1795 * @bo_va: bo_va to store the address
1796 * @saddr: where to map the BO
1797 * @offset: requested offset in the BO
1798 * @flags: attributes of pages (read/write/valid/etc.)
1800 * Add a mapping of the BO at the specefied addr into the VM. Replace existing
1801 * mappings as we do so.
1802 * Returns 0 for success, error for failure.
1804 * Object has to be reserved and unreserved outside!
1806 int amdgpu_vm_bo_replace_map(struct amdgpu_device
*adev
,
1807 struct amdgpu_bo_va
*bo_va
,
1808 uint64_t saddr
, uint64_t offset
,
1809 uint64_t size
, uint64_t flags
)
1811 struct amdgpu_bo_va_mapping
*mapping
;
1812 struct amdgpu_vm
*vm
= bo_va
->vm
;
1816 /* validate the parameters */
1817 if (saddr
& AMDGPU_GPU_PAGE_MASK
|| offset
& AMDGPU_GPU_PAGE_MASK
||
1818 size
== 0 || size
& AMDGPU_GPU_PAGE_MASK
)
1821 /* make sure object fit at this offset */
1822 eaddr
= saddr
+ size
- 1;
1823 if (saddr
>= eaddr
||
1824 (bo_va
->bo
&& offset
+ size
> amdgpu_bo_size(bo_va
->bo
)))
1827 /* Allocate all the needed memory */
1828 mapping
= kmalloc(sizeof(*mapping
), GFP_KERNEL
);
1832 r
= amdgpu_vm_bo_clear_mappings(adev
, bo_va
->vm
, saddr
, size
);
1838 saddr
/= AMDGPU_GPU_PAGE_SIZE
;
1839 eaddr
/= AMDGPU_GPU_PAGE_SIZE
;
1841 mapping
->start
= saddr
;
1842 mapping
->last
= eaddr
;
1843 mapping
->offset
= offset
;
1844 mapping
->flags
= flags
;
1846 list_add(&mapping
->list
, &bo_va
->invalids
);
1847 amdgpu_vm_it_insert(mapping
, &vm
->va
);
1849 if (flags
& AMDGPU_PTE_PRT
)
1850 amdgpu_vm_prt_get(adev
);
1856 * amdgpu_vm_bo_unmap - remove bo mapping from vm
1858 * @adev: amdgpu_device pointer
1859 * @bo_va: bo_va to remove the address from
1860 * @saddr: where to the BO is mapped
1862 * Remove a mapping of the BO at the specefied addr from the VM.
1863 * Returns 0 for success, error for failure.
1865 * Object has to be reserved and unreserved outside!
1867 int amdgpu_vm_bo_unmap(struct amdgpu_device
*adev
,
1868 struct amdgpu_bo_va
*bo_va
,
1871 struct amdgpu_bo_va_mapping
*mapping
;
1872 struct amdgpu_vm
*vm
= bo_va
->vm
;
1875 saddr
/= AMDGPU_GPU_PAGE_SIZE
;
1877 list_for_each_entry(mapping
, &bo_va
->valids
, list
) {
1878 if (mapping
->start
== saddr
)
1882 if (&mapping
->list
== &bo_va
->valids
) {
1885 list_for_each_entry(mapping
, &bo_va
->invalids
, list
) {
1886 if (mapping
->start
== saddr
)
1890 if (&mapping
->list
== &bo_va
->invalids
)
1894 list_del(&mapping
->list
);
1895 amdgpu_vm_it_remove(mapping
, &vm
->va
);
1896 trace_amdgpu_vm_bo_unmap(bo_va
, mapping
);
1899 list_add(&mapping
->list
, &vm
->freed
);
1901 amdgpu_vm_free_mapping(adev
, vm
, mapping
,
1902 bo_va
->last_pt_update
);
1908 * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
1910 * @adev: amdgpu_device pointer
1911 * @vm: VM structure to use
1912 * @saddr: start of the range
1913 * @size: size of the range
1915 * Remove all mappings in a range, split them as appropriate.
1916 * Returns 0 for success, error for failure.
1918 int amdgpu_vm_bo_clear_mappings(struct amdgpu_device
*adev
,
1919 struct amdgpu_vm
*vm
,
1920 uint64_t saddr
, uint64_t size
)
1922 struct amdgpu_bo_va_mapping
*before
, *after
, *tmp
, *next
;
1926 eaddr
= saddr
+ size
- 1;
1927 saddr
/= AMDGPU_GPU_PAGE_SIZE
;
1928 eaddr
/= AMDGPU_GPU_PAGE_SIZE
;
1930 /* Allocate all the needed memory */
1931 before
= kzalloc(sizeof(*before
), GFP_KERNEL
);
1934 INIT_LIST_HEAD(&before
->list
);
1936 after
= kzalloc(sizeof(*after
), GFP_KERNEL
);
1941 INIT_LIST_HEAD(&after
->list
);
1943 /* Now gather all removed mappings */
1944 tmp
= amdgpu_vm_it_iter_first(&vm
->va
, saddr
, eaddr
);
1946 /* Remember mapping split at the start */
1947 if (tmp
->start
< saddr
) {
1948 before
->start
= tmp
->start
;
1949 before
->last
= saddr
- 1;
1950 before
->offset
= tmp
->offset
;
1951 before
->flags
= tmp
->flags
;
1952 list_add(&before
->list
, &tmp
->list
);
1955 /* Remember mapping split at the end */
1956 if (tmp
->last
> eaddr
) {
1957 after
->start
= eaddr
+ 1;
1958 after
->last
= tmp
->last
;
1959 after
->offset
= tmp
->offset
;
1960 after
->offset
+= after
->start
- tmp
->start
;
1961 after
->flags
= tmp
->flags
;
1962 list_add(&after
->list
, &tmp
->list
);
1965 list_del(&tmp
->list
);
1966 list_add(&tmp
->list
, &removed
);
1968 tmp
= amdgpu_vm_it_iter_next(tmp
, saddr
, eaddr
);
1971 /* And free them up */
1972 list_for_each_entry_safe(tmp
, next
, &removed
, list
) {
1973 amdgpu_vm_it_remove(tmp
, &vm
->va
);
1974 list_del(&tmp
->list
);
1976 if (tmp
->start
< saddr
)
1978 if (tmp
->last
> eaddr
)
1981 list_add(&tmp
->list
, &vm
->freed
);
1982 trace_amdgpu_vm_bo_unmap(NULL
, tmp
);
1985 /* Insert partial mapping before the range */
1986 if (!list_empty(&before
->list
)) {
1987 amdgpu_vm_it_insert(before
, &vm
->va
);
1988 if (before
->flags
& AMDGPU_PTE_PRT
)
1989 amdgpu_vm_prt_get(adev
);
1994 /* Insert partial mapping after the range */
1995 if (!list_empty(&after
->list
)) {
1996 amdgpu_vm_it_insert(after
, &vm
->va
);
1997 if (after
->flags
& AMDGPU_PTE_PRT
)
1998 amdgpu_vm_prt_get(adev
);
2007 * amdgpu_vm_bo_rmv - remove a bo to a specific vm
2009 * @adev: amdgpu_device pointer
2010 * @bo_va: requested bo_va
2012 * Remove @bo_va->bo from the requested vm.
2014 * Object have to be reserved!
2016 void amdgpu_vm_bo_rmv(struct amdgpu_device
*adev
,
2017 struct amdgpu_bo_va
*bo_va
)
2019 struct amdgpu_bo_va_mapping
*mapping
, *next
;
2020 struct amdgpu_vm
*vm
= bo_va
->vm
;
2022 list_del(&bo_va
->bo_list
);
2024 spin_lock(&vm
->status_lock
);
2025 list_del(&bo_va
->vm_status
);
2026 spin_unlock(&vm
->status_lock
);
2028 list_for_each_entry_safe(mapping
, next
, &bo_va
->valids
, list
) {
2029 list_del(&mapping
->list
);
2030 amdgpu_vm_it_remove(mapping
, &vm
->va
);
2031 trace_amdgpu_vm_bo_unmap(bo_va
, mapping
);
2032 list_add(&mapping
->list
, &vm
->freed
);
2034 list_for_each_entry_safe(mapping
, next
, &bo_va
->invalids
, list
) {
2035 list_del(&mapping
->list
);
2036 amdgpu_vm_it_remove(mapping
, &vm
->va
);
2037 amdgpu_vm_free_mapping(adev
, vm
, mapping
,
2038 bo_va
->last_pt_update
);
2041 dma_fence_put(bo_va
->last_pt_update
);
2046 * amdgpu_vm_bo_invalidate - mark the bo as invalid
2048 * @adev: amdgpu_device pointer
2050 * @bo: amdgpu buffer object
2052 * Mark @bo as invalid.
2054 void amdgpu_vm_bo_invalidate(struct amdgpu_device
*adev
,
2055 struct amdgpu_bo
*bo
)
2057 struct amdgpu_bo_va
*bo_va
;
2059 list_for_each_entry(bo_va
, &bo
->va
, bo_list
) {
2060 spin_lock(&bo_va
->vm
->status_lock
);
2061 if (list_empty(&bo_va
->vm_status
))
2062 list_add(&bo_va
->vm_status
, &bo_va
->vm
->invalidated
);
2063 spin_unlock(&bo_va
->vm
->status_lock
);
2067 static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size
)
2069 /* Total bits covered by PD + PTs */
2070 unsigned bits
= ilog2(vm_size
) + 18;
2072 /* Make sure the PD is 4K in size up to 8GB address space.
2073 Above that split equal between PD and PTs */
2077 return ((bits
+ 3) / 2);
2081 * amdgpu_vm_adjust_size - adjust vm size and block size
2083 * @adev: amdgpu_device pointer
2084 * @vm_size: the default vm size if it's set auto
2086 void amdgpu_vm_adjust_size(struct amdgpu_device
*adev
, uint64_t vm_size
)
2088 /* adjust vm size firstly */
2089 if (amdgpu_vm_size
== -1)
2090 adev
->vm_manager
.vm_size
= vm_size
;
2092 adev
->vm_manager
.vm_size
= amdgpu_vm_size
;
2094 /* block size depends on vm size */
2095 if (amdgpu_vm_block_size
== -1)
2096 adev
->vm_manager
.block_size
=
2097 amdgpu_vm_get_block_size(adev
->vm_manager
.vm_size
);
2099 adev
->vm_manager
.block_size
= amdgpu_vm_block_size
;
2101 DRM_INFO("vm size is %llu GB, block size is %u-bit\n",
2102 adev
->vm_manager
.vm_size
, adev
->vm_manager
.block_size
);
2106 * amdgpu_vm_init - initialize a vm instance
2108 * @adev: amdgpu_device pointer
2113 int amdgpu_vm_init(struct amdgpu_device
*adev
, struct amdgpu_vm
*vm
)
2115 const unsigned align
= min(AMDGPU_VM_PTB_ALIGN_SIZE
,
2116 AMDGPU_VM_PTE_COUNT(adev
) * 8);
2117 unsigned ring_instance
;
2118 struct amdgpu_ring
*ring
;
2119 struct amd_sched_rq
*rq
;
2122 for (i
= 0; i
< AMDGPU_MAX_RINGS
; ++i
)
2125 vm
->client_id
= atomic64_inc_return(&adev
->vm_manager
.client_counter
);
2126 spin_lock_init(&vm
->status_lock
);
2127 INIT_LIST_HEAD(&vm
->invalidated
);
2128 INIT_LIST_HEAD(&vm
->cleared
);
2129 INIT_LIST_HEAD(&vm
->freed
);
2131 /* create scheduler entity for page table updates */
2133 ring_instance
= atomic_inc_return(&adev
->vm_manager
.vm_pte_next_ring
);
2134 ring_instance
%= adev
->vm_manager
.vm_pte_num_rings
;
2135 ring
= adev
->vm_manager
.vm_pte_rings
[ring_instance
];
2136 rq
= &ring
->sched
.sched_rq
[AMD_SCHED_PRIORITY_KERNEL
];
2137 r
= amd_sched_entity_init(&ring
->sched
, &vm
->entity
,
2138 rq
, amdgpu_sched_jobs
);
2142 vm
->last_dir_update
= NULL
;
2144 r
= amdgpu_bo_create(adev
, amdgpu_vm_bo_size(adev
, 0), align
, true,
2145 AMDGPU_GEM_DOMAIN_VRAM
,
2146 AMDGPU_GEM_CREATE_NO_CPU_ACCESS
|
2147 AMDGPU_GEM_CREATE_SHADOW
|
2148 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS
|
2149 AMDGPU_GEM_CREATE_VRAM_CLEARED
,
2150 NULL
, NULL
, &vm
->root
.bo
);
2152 goto error_free_sched_entity
;
2154 r
= amdgpu_bo_reserve(vm
->root
.bo
, false);
2156 goto error_free_root
;
2158 vm
->last_eviction_counter
= atomic64_read(&adev
->num_evictions
);
2159 amdgpu_bo_unreserve(vm
->root
.bo
);
2164 amdgpu_bo_unref(&vm
->root
.bo
->shadow
);
2165 amdgpu_bo_unref(&vm
->root
.bo
);
2168 error_free_sched_entity
:
2169 amd_sched_entity_fini(&ring
->sched
, &vm
->entity
);
2175 * amdgpu_vm_free_levels - free PD/PT levels
2177 * @level: PD/PT starting level to free
2179 * Free the page directory or page table level and all sub levels.
2181 static void amdgpu_vm_free_levels(struct amdgpu_vm_pt
*level
)
2186 amdgpu_bo_unref(&level
->bo
->shadow
);
2187 amdgpu_bo_unref(&level
->bo
);
2191 for (i
= 0; i
<= level
->last_entry_used
; i
++)
2192 amdgpu_vm_free_levels(&level
->entries
[i
]);
2194 drm_free_large(level
->entries
);
2198 * amdgpu_vm_fini - tear down a vm instance
2200 * @adev: amdgpu_device pointer
2204 * Unbind the VM and remove all bos from the vm bo list
2206 void amdgpu_vm_fini(struct amdgpu_device
*adev
, struct amdgpu_vm
*vm
)
2208 struct amdgpu_bo_va_mapping
*mapping
, *tmp
;
2209 bool prt_fini_needed
= !!adev
->gart
.gart_funcs
->set_prt
;
2211 amd_sched_entity_fini(vm
->entity
.sched
, &vm
->entity
);
2213 if (!RB_EMPTY_ROOT(&vm
->va
)) {
2214 dev_err(adev
->dev
, "still active bo inside vm\n");
2216 rbtree_postorder_for_each_entry_safe(mapping
, tmp
, &vm
->va
, rb
) {
2217 list_del(&mapping
->list
);
2218 amdgpu_vm_it_remove(mapping
, &vm
->va
);
2221 list_for_each_entry_safe(mapping
, tmp
, &vm
->freed
, list
) {
2222 if (mapping
->flags
& AMDGPU_PTE_PRT
&& prt_fini_needed
) {
2223 amdgpu_vm_prt_fini(adev
, vm
);
2224 prt_fini_needed
= false;
2227 list_del(&mapping
->list
);
2228 amdgpu_vm_free_mapping(adev
, vm
, mapping
, NULL
);
2231 amdgpu_vm_free_levels(&vm
->root
);
2232 dma_fence_put(vm
->last_dir_update
);
2236 * amdgpu_vm_manager_init - init the VM manager
2238 * @adev: amdgpu_device pointer
2240 * Initialize the VM manager structures
2242 void amdgpu_vm_manager_init(struct amdgpu_device
*adev
)
2246 INIT_LIST_HEAD(&adev
->vm_manager
.ids_lru
);
2248 /* skip over VMID 0, since it is the system VM */
2249 for (i
= 1; i
< adev
->vm_manager
.num_ids
; ++i
) {
2250 amdgpu_vm_reset_id(adev
, i
);
2251 amdgpu_sync_create(&adev
->vm_manager
.ids
[i
].active
);
2252 list_add_tail(&adev
->vm_manager
.ids
[i
].list
,
2253 &adev
->vm_manager
.ids_lru
);
2256 adev
->vm_manager
.fence_context
=
2257 dma_fence_context_alloc(AMDGPU_MAX_RINGS
);
2258 for (i
= 0; i
< AMDGPU_MAX_RINGS
; ++i
)
2259 adev
->vm_manager
.seqno
[i
] = 0;
2261 atomic_set(&adev
->vm_manager
.vm_pte_next_ring
, 0);
2262 atomic64_set(&adev
->vm_manager
.client_counter
, 0);
2263 spin_lock_init(&adev
->vm_manager
.prt_lock
);
2264 atomic_set(&adev
->vm_manager
.num_prt_users
, 0);
2268 * amdgpu_vm_manager_fini - cleanup VM manager
2270 * @adev: amdgpu_device pointer
2272 * Cleanup the VM manager and free resources.
2274 void amdgpu_vm_manager_fini(struct amdgpu_device
*adev
)
2278 for (i
= 0; i
< AMDGPU_NUM_VM
; ++i
) {
2279 struct amdgpu_vm_id
*id
= &adev
->vm_manager
.ids
[i
];
2281 amdgpu_sync_free(&adev
->vm_manager
.ids
[i
].active
);
2282 dma_fence_put(id
->flushed_updates
);
2283 dma_fence_put(id
->last_flush
);