2 * Copyright 2014-2018 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
23 #define pr_fmt(fmt) "kfd2kgd: " fmt
25 #include <linux/dma-buf.h>
26 #include <linux/list.h>
27 #include <linux/pagemap.h>
28 #include <linux/sched/mm.h>
29 #include <linux/sched/task.h>
31 #include "amdgpu_object.h"
32 #include "amdgpu_vm.h"
33 #include "amdgpu_amdkfd.h"
34 #include "amdgpu_dma_buf.h"
36 /* Special VM and GART address alignment needed for VI pre-Fiji due to
39 #define VI_BO_SIZE_ALIGN (0x8000)
41 /* BO flag to indicate a KFD userptr BO */
42 #define AMDGPU_AMDKFD_USERPTR_BO (1ULL << 63)
44 /* Userptr restore delay, just long enough to allow consecutive VM
45 * changes to accumulate
47 #define AMDGPU_USERPTR_RESTORE_DELAY_MS 1
49 /* Impose limit on how much memory KFD can use */
51 uint64_t max_system_mem_limit
;
52 uint64_t max_ttm_mem_limit
;
53 int64_t system_mem_used
;
55 spinlock_t mem_limit_lock
;
58 /* Struct used for amdgpu_amdkfd_bo_validate */
59 struct amdgpu_vm_parser
{
64 static const char * const domain_bit_to_string
[] = {
73 #define domain_string(domain) domain_bit_to_string[ffs(domain)-1]
75 static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct
*work
);
78 static inline struct amdgpu_device
*get_amdgpu_device(struct kgd_dev
*kgd
)
80 return (struct amdgpu_device
*)kgd
;
83 static bool check_if_add_bo_to_vm(struct amdgpu_vm
*avm
,
86 struct kfd_bo_va_list
*entry
;
88 list_for_each_entry(entry
, &mem
->bo_va_list
, bo_list
)
89 if (entry
->bo_va
->base
.vm
== avm
)
95 /* Set memory usage limits. Current, limits are
96 * System (TTM + userptr) memory - 3/4th System RAM
97 * TTM memory - 3/8th System RAM
99 void amdgpu_amdkfd_gpuvm_init_mem_limits(void)
105 mem
= si
.totalram
- si
.totalhigh
;
108 spin_lock_init(&kfd_mem_limit
.mem_limit_lock
);
109 kfd_mem_limit
.max_system_mem_limit
= (mem
>> 1) + (mem
>> 2);
110 kfd_mem_limit
.max_ttm_mem_limit
= (mem
>> 1) - (mem
>> 3);
111 pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n",
112 (kfd_mem_limit
.max_system_mem_limit
>> 20),
113 (kfd_mem_limit
.max_ttm_mem_limit
>> 20));
116 static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device
*adev
,
117 uint64_t size
, u32 domain
, bool sg
)
119 size_t acc_size
, system_mem_needed
, ttm_mem_needed
, vram_needed
;
120 uint64_t reserved_for_pt
= amdgpu_amdkfd_total_mem_size
>> 9;
123 acc_size
= ttm_bo_dma_acc_size(&adev
->mman
.bdev
, size
,
124 sizeof(struct amdgpu_bo
));
127 if (domain
== AMDGPU_GEM_DOMAIN_GTT
) {
129 system_mem_needed
= acc_size
+ size
;
130 ttm_mem_needed
= acc_size
+ size
;
131 } else if (domain
== AMDGPU_GEM_DOMAIN_CPU
&& !sg
) {
133 system_mem_needed
= acc_size
+ size
;
134 ttm_mem_needed
= acc_size
;
137 system_mem_needed
= acc_size
;
138 ttm_mem_needed
= acc_size
;
139 if (domain
== AMDGPU_GEM_DOMAIN_VRAM
)
143 spin_lock(&kfd_mem_limit
.mem_limit_lock
);
145 if ((kfd_mem_limit
.system_mem_used
+ system_mem_needed
>
146 kfd_mem_limit
.max_system_mem_limit
) ||
147 (kfd_mem_limit
.ttm_mem_used
+ ttm_mem_needed
>
148 kfd_mem_limit
.max_ttm_mem_limit
) ||
149 (adev
->kfd
.vram_used
+ vram_needed
>
150 adev
->gmc
.real_vram_size
- reserved_for_pt
)) {
153 kfd_mem_limit
.system_mem_used
+= system_mem_needed
;
154 kfd_mem_limit
.ttm_mem_used
+= ttm_mem_needed
;
155 adev
->kfd
.vram_used
+= vram_needed
;
158 spin_unlock(&kfd_mem_limit
.mem_limit_lock
);
162 static void unreserve_mem_limit(struct amdgpu_device
*adev
,
163 uint64_t size
, u32 domain
, bool sg
)
167 acc_size
= ttm_bo_dma_acc_size(&adev
->mman
.bdev
, size
,
168 sizeof(struct amdgpu_bo
));
170 spin_lock(&kfd_mem_limit
.mem_limit_lock
);
171 if (domain
== AMDGPU_GEM_DOMAIN_GTT
) {
172 kfd_mem_limit
.system_mem_used
-= (acc_size
+ size
);
173 kfd_mem_limit
.ttm_mem_used
-= (acc_size
+ size
);
174 } else if (domain
== AMDGPU_GEM_DOMAIN_CPU
&& !sg
) {
175 kfd_mem_limit
.system_mem_used
-= (acc_size
+ size
);
176 kfd_mem_limit
.ttm_mem_used
-= acc_size
;
178 kfd_mem_limit
.system_mem_used
-= acc_size
;
179 kfd_mem_limit
.ttm_mem_used
-= acc_size
;
180 if (domain
== AMDGPU_GEM_DOMAIN_VRAM
) {
181 adev
->kfd
.vram_used
-= size
;
182 WARN_ONCE(adev
->kfd
.vram_used
< 0,
183 "kfd VRAM memory accounting unbalanced");
186 WARN_ONCE(kfd_mem_limit
.system_mem_used
< 0,
187 "kfd system memory accounting unbalanced");
188 WARN_ONCE(kfd_mem_limit
.ttm_mem_used
< 0,
189 "kfd TTM memory accounting unbalanced");
191 spin_unlock(&kfd_mem_limit
.mem_limit_lock
);
194 void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo
*bo
)
196 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->tbo
.bdev
);
197 u32 domain
= bo
->preferred_domains
;
198 bool sg
= (bo
->preferred_domains
== AMDGPU_GEM_DOMAIN_CPU
);
200 if (bo
->flags
& AMDGPU_AMDKFD_USERPTR_BO
) {
201 domain
= AMDGPU_GEM_DOMAIN_CPU
;
205 unreserve_mem_limit(adev
, amdgpu_bo_size(bo
), domain
, sg
);
209 /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
210 * reservation object.
212 * @bo: [IN] Remove eviction fence(s) from this BO
213 * @ef: [IN] This eviction fence is removed if it
214 * is present in the shared list.
216 * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held.
218 static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo
*bo
,
219 struct amdgpu_amdkfd_fence
*ef
)
221 struct reservation_object
*resv
= bo
->tbo
.resv
;
222 struct reservation_object_list
*old
, *new;
223 unsigned int i
, j
, k
;
228 old
= reservation_object_get_list(resv
);
232 new = kmalloc(offsetof(typeof(*new), shared
[old
->shared_max
]),
237 /* Go through all the shared fences in the resevation object and sort
238 * the interesting ones to the end of the list.
240 for (i
= 0, j
= old
->shared_count
, k
= 0; i
< old
->shared_count
; ++i
) {
243 f
= rcu_dereference_protected(old
->shared
[i
],
244 reservation_object_held(resv
));
246 if (f
->context
== ef
->base
.context
)
247 RCU_INIT_POINTER(new->shared
[--j
], f
);
249 RCU_INIT_POINTER(new->shared
[k
++], f
);
251 new->shared_max
= old
->shared_max
;
252 new->shared_count
= k
;
254 /* Install the new fence list, seqcount provides the barriers */
256 write_seqcount_begin(&resv
->seq
);
257 RCU_INIT_POINTER(resv
->fence
, new);
258 write_seqcount_end(&resv
->seq
);
261 /* Drop the references to the removed fences or move them to ef_list */
262 for (i
= j
, k
= 0; i
< old
->shared_count
; ++i
) {
265 f
= rcu_dereference_protected(new->shared
[i
],
266 reservation_object_held(resv
));
274 static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo
*bo
, uint32_t domain
,
277 struct ttm_operation_ctx ctx
= { false, false };
280 if (WARN(amdgpu_ttm_tt_get_usermm(bo
->tbo
.ttm
),
281 "Called with userptr BO"))
284 amdgpu_bo_placement_from_domain(bo
, domain
);
286 ret
= ttm_bo_validate(&bo
->tbo
, &bo
->placement
, &ctx
);
290 amdgpu_bo_sync_wait(bo
, AMDGPU_FENCE_OWNER_KFD
, false);
296 static int amdgpu_amdkfd_validate(void *param
, struct amdgpu_bo
*bo
)
298 struct amdgpu_vm_parser
*p
= param
;
300 return amdgpu_amdkfd_bo_validate(bo
, p
->domain
, p
->wait
);
303 /* vm_validate_pt_pd_bos - Validate page table and directory BOs
305 * Page directories are not updated here because huge page handling
306 * during page table updates can invalidate page directory entries
307 * again. Page directories are only updated after updating page
310 static int vm_validate_pt_pd_bos(struct amdgpu_vm
*vm
)
312 struct amdgpu_bo
*pd
= vm
->root
.base
.bo
;
313 struct amdgpu_device
*adev
= amdgpu_ttm_adev(pd
->tbo
.bdev
);
314 struct amdgpu_vm_parser param
;
317 param
.domain
= AMDGPU_GEM_DOMAIN_VRAM
;
320 ret
= amdgpu_vm_validate_pt_bos(adev
, vm
, amdgpu_amdkfd_validate
,
323 pr_err("amdgpu: failed to validate PT BOs\n");
327 ret
= amdgpu_amdkfd_validate(¶m
, pd
);
329 pr_err("amdgpu: failed to validate PD\n");
333 vm
->pd_phys_addr
= amdgpu_gmc_pd_addr(vm
->root
.base
.bo
);
335 if (vm
->use_cpu_for_update
) {
336 ret
= amdgpu_bo_kmap(pd
, NULL
);
338 pr_err("amdgpu: failed to kmap PD, ret=%d\n", ret
);
346 static int vm_update_pds(struct amdgpu_vm
*vm
, struct amdgpu_sync
*sync
)
348 struct amdgpu_bo
*pd
= vm
->root
.base
.bo
;
349 struct amdgpu_device
*adev
= amdgpu_ttm_adev(pd
->tbo
.bdev
);
352 ret
= amdgpu_vm_update_directories(adev
, vm
);
356 return amdgpu_sync_fence(NULL
, sync
, vm
->last_update
, false);
359 /* add_bo_to_vm - Add a BO to a VM
361 * Everything that needs to bo done only once when a BO is first added
362 * to a VM. It can later be mapped and unmapped many times without
363 * repeating these steps.
365 * 1. Allocate and initialize BO VA entry data structure
366 * 2. Add BO to the VM
367 * 3. Determine ASIC-specific PTE flags
368 * 4. Alloc page tables and directories if needed
369 * 4a. Validate new page tables and directories
371 static int add_bo_to_vm(struct amdgpu_device
*adev
, struct kgd_mem
*mem
,
372 struct amdgpu_vm
*vm
, bool is_aql
,
373 struct kfd_bo_va_list
**p_bo_va_entry
)
376 struct kfd_bo_va_list
*bo_va_entry
;
377 struct amdgpu_bo
*bo
= mem
->bo
;
378 uint64_t va
= mem
->va
;
379 struct list_head
*list_bo_va
= &mem
->bo_va_list
;
380 unsigned long bo_size
= bo
->tbo
.mem
.size
;
383 pr_err("Invalid VA when adding BO to VM\n");
390 bo_va_entry
= kzalloc(sizeof(*bo_va_entry
), GFP_KERNEL
);
394 pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va
,
397 /* Add BO to VM internal data structures*/
398 bo_va_entry
->bo_va
= amdgpu_vm_bo_add(adev
, vm
, bo
);
399 if (!bo_va_entry
->bo_va
) {
401 pr_err("Failed to add BO object to VM. ret == %d\n",
406 bo_va_entry
->va
= va
;
407 bo_va_entry
->pte_flags
= amdgpu_gmc_get_pte_flags(adev
,
409 bo_va_entry
->kgd_dev
= (void *)adev
;
410 list_add(&bo_va_entry
->bo_list
, list_bo_va
);
413 *p_bo_va_entry
= bo_va_entry
;
415 /* Allocate validate page tables if needed */
416 ret
= vm_validate_pt_pd_bos(vm
);
418 pr_err("validate_pt_pd_bos() failed\n");
425 amdgpu_vm_bo_rmv(adev
, bo_va_entry
->bo_va
);
426 list_del(&bo_va_entry
->bo_list
);
432 static void remove_bo_from_vm(struct amdgpu_device
*adev
,
433 struct kfd_bo_va_list
*entry
, unsigned long size
)
435 pr_debug("\t remove VA 0x%llx - 0x%llx in entry %p\n",
437 entry
->va
+ size
, entry
);
438 amdgpu_vm_bo_rmv(adev
, entry
->bo_va
);
439 list_del(&entry
->bo_list
);
443 static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem
*mem
,
444 struct amdkfd_process_info
*process_info
,
447 struct ttm_validate_buffer
*entry
= &mem
->validate_list
;
448 struct amdgpu_bo
*bo
= mem
->bo
;
450 INIT_LIST_HEAD(&entry
->head
);
451 entry
->num_shared
= 1;
452 entry
->bo
= &bo
->tbo
;
453 mutex_lock(&process_info
->lock
);
455 list_add_tail(&entry
->head
, &process_info
->userptr_valid_list
);
457 list_add_tail(&entry
->head
, &process_info
->kfd_bo_list
);
458 mutex_unlock(&process_info
->lock
);
461 static void remove_kgd_mem_from_kfd_bo_list(struct kgd_mem
*mem
,
462 struct amdkfd_process_info
*process_info
)
464 struct ttm_validate_buffer
*bo_list_entry
;
466 bo_list_entry
= &mem
->validate_list
;
467 mutex_lock(&process_info
->lock
);
468 list_del(&bo_list_entry
->head
);
469 mutex_unlock(&process_info
->lock
);
472 /* Initializes user pages. It registers the MMU notifier and validates
473 * the userptr BO in the GTT domain.
475 * The BO must already be on the userptr_valid_list. Otherwise an
476 * eviction and restore may happen that leaves the new BO unmapped
477 * with the user mode queues running.
479 * Takes the process_info->lock to protect against concurrent restore
482 * Returns 0 for success, negative errno for errors.
484 static int init_user_pages(struct kgd_mem
*mem
, struct mm_struct
*mm
,
487 struct amdkfd_process_info
*process_info
= mem
->process_info
;
488 struct amdgpu_bo
*bo
= mem
->bo
;
489 struct ttm_operation_ctx ctx
= { true, false };
492 mutex_lock(&process_info
->lock
);
494 ret
= amdgpu_ttm_tt_set_userptr(bo
->tbo
.ttm
, user_addr
, 0);
496 pr_err("%s: Failed to set userptr: %d\n", __func__
, ret
);
500 ret
= amdgpu_mn_register(bo
, user_addr
);
502 pr_err("%s: Failed to register MMU notifier: %d\n",
507 ret
= amdgpu_ttm_tt_get_user_pages(bo
, bo
->tbo
.ttm
->pages
);
509 pr_err("%s: Failed to get user pages: %d\n", __func__
, ret
);
513 ret
= amdgpu_bo_reserve(bo
, true);
515 pr_err("%s: Failed to reserve BO\n", __func__
);
518 amdgpu_bo_placement_from_domain(bo
, mem
->domain
);
519 ret
= ttm_bo_validate(&bo
->tbo
, &bo
->placement
, &ctx
);
521 pr_err("%s: failed to validate BO\n", __func__
);
522 amdgpu_bo_unreserve(bo
);
525 amdgpu_ttm_tt_get_user_pages_done(bo
->tbo
.ttm
);
528 amdgpu_mn_unregister(bo
);
530 mutex_unlock(&process_info
->lock
);
534 /* Reserving a BO and its page table BOs must happen atomically to
535 * avoid deadlocks. Some operations update multiple VMs at once. Track
536 * all the reservation info in a context structure. Optionally a sync
537 * object can track VM updates.
539 struct bo_vm_reservation_context
{
540 struct amdgpu_bo_list_entry kfd_bo
; /* BO list entry for the KFD BO */
541 unsigned int n_vms
; /* Number of VMs reserved */
542 struct amdgpu_bo_list_entry
*vm_pd
; /* Array of VM BO list entries */
543 struct ww_acquire_ctx ticket
; /* Reservation ticket */
544 struct list_head list
, duplicates
; /* BO lists */
545 struct amdgpu_sync
*sync
; /* Pointer to sync object */
546 bool reserved
; /* Whether BOs are reserved */
550 BO_VM_NOT_MAPPED
= 0, /* Match VMs where a BO is not mapped */
551 BO_VM_MAPPED
, /* Match VMs where a BO is mapped */
552 BO_VM_ALL
, /* Match all VMs a BO was added to */
556 * reserve_bo_and_vm - reserve a BO and a VM unconditionally.
557 * @mem: KFD BO structure.
558 * @vm: the VM to reserve.
559 * @ctx: the struct that will be used in unreserve_bo_and_vms().
561 static int reserve_bo_and_vm(struct kgd_mem
*mem
,
562 struct amdgpu_vm
*vm
,
563 struct bo_vm_reservation_context
*ctx
)
565 struct amdgpu_bo
*bo
= mem
->bo
;
570 ctx
->reserved
= false;
572 ctx
->sync
= &mem
->sync
;
574 INIT_LIST_HEAD(&ctx
->list
);
575 INIT_LIST_HEAD(&ctx
->duplicates
);
577 ctx
->vm_pd
= kcalloc(ctx
->n_vms
, sizeof(*ctx
->vm_pd
), GFP_KERNEL
);
581 ctx
->kfd_bo
.priority
= 0;
582 ctx
->kfd_bo
.tv
.bo
= &bo
->tbo
;
583 ctx
->kfd_bo
.tv
.num_shared
= 1;
584 list_add(&ctx
->kfd_bo
.tv
.head
, &ctx
->list
);
586 amdgpu_vm_get_pd_bo(vm
, &ctx
->list
, &ctx
->vm_pd
[0]);
588 ret
= ttm_eu_reserve_buffers(&ctx
->ticket
, &ctx
->list
,
589 false, &ctx
->duplicates
, true);
591 ctx
->reserved
= true;
593 pr_err("Failed to reserve buffers in ttm\n");
602 * reserve_bo_and_cond_vms - reserve a BO and some VMs conditionally
603 * @mem: KFD BO structure.
604 * @vm: the VM to reserve. If NULL, then all VMs associated with the BO
605 * is used. Otherwise, a single VM associated with the BO.
606 * @map_type: the mapping status that will be used to filter the VMs.
607 * @ctx: the struct that will be used in unreserve_bo_and_vms().
609 * Returns 0 for success, negative for failure.
611 static int reserve_bo_and_cond_vms(struct kgd_mem
*mem
,
612 struct amdgpu_vm
*vm
, enum bo_vm_match map_type
,
613 struct bo_vm_reservation_context
*ctx
)
615 struct amdgpu_bo
*bo
= mem
->bo
;
616 struct kfd_bo_va_list
*entry
;
620 ctx
->reserved
= false;
623 ctx
->sync
= &mem
->sync
;
625 INIT_LIST_HEAD(&ctx
->list
);
626 INIT_LIST_HEAD(&ctx
->duplicates
);
628 list_for_each_entry(entry
, &mem
->bo_va_list
, bo_list
) {
629 if ((vm
&& vm
!= entry
->bo_va
->base
.vm
) ||
630 (entry
->is_mapped
!= map_type
631 && map_type
!= BO_VM_ALL
))
637 if (ctx
->n_vms
!= 0) {
638 ctx
->vm_pd
= kcalloc(ctx
->n_vms
, sizeof(*ctx
->vm_pd
),
644 ctx
->kfd_bo
.priority
= 0;
645 ctx
->kfd_bo
.tv
.bo
= &bo
->tbo
;
646 ctx
->kfd_bo
.tv
.num_shared
= 1;
647 list_add(&ctx
->kfd_bo
.tv
.head
, &ctx
->list
);
650 list_for_each_entry(entry
, &mem
->bo_va_list
, bo_list
) {
651 if ((vm
&& vm
!= entry
->bo_va
->base
.vm
) ||
652 (entry
->is_mapped
!= map_type
653 && map_type
!= BO_VM_ALL
))
656 amdgpu_vm_get_pd_bo(entry
->bo_va
->base
.vm
, &ctx
->list
,
661 ret
= ttm_eu_reserve_buffers(&ctx
->ticket
, &ctx
->list
,
662 false, &ctx
->duplicates
, true);
664 ctx
->reserved
= true;
666 pr_err("Failed to reserve buffers in ttm.\n");
677 * unreserve_bo_and_vms - Unreserve BO and VMs from a reservation context
678 * @ctx: Reservation context to unreserve
679 * @wait: Optionally wait for a sync object representing pending VM updates
680 * @intr: Whether the wait is interruptible
682 * Also frees any resources allocated in
683 * reserve_bo_and_(cond_)vm(s). Returns the status from
686 static int unreserve_bo_and_vms(struct bo_vm_reservation_context
*ctx
,
687 bool wait
, bool intr
)
692 ret
= amdgpu_sync_wait(ctx
->sync
, intr
);
695 ttm_eu_backoff_reservation(&ctx
->ticket
, &ctx
->list
);
700 ctx
->reserved
= false;
706 static int unmap_bo_from_gpuvm(struct amdgpu_device
*adev
,
707 struct kfd_bo_va_list
*entry
,
708 struct amdgpu_sync
*sync
)
710 struct amdgpu_bo_va
*bo_va
= entry
->bo_va
;
711 struct amdgpu_vm
*vm
= bo_va
->base
.vm
;
713 amdgpu_vm_bo_unmap(adev
, bo_va
, entry
->va
);
715 amdgpu_vm_clear_freed(adev
, vm
, &bo_va
->last_pt_update
);
717 amdgpu_sync_fence(NULL
, sync
, bo_va
->last_pt_update
, false);
722 static int update_gpuvm_pte(struct amdgpu_device
*adev
,
723 struct kfd_bo_va_list
*entry
,
724 struct amdgpu_sync
*sync
)
727 struct amdgpu_bo_va
*bo_va
= entry
->bo_va
;
729 /* Update the page tables */
730 ret
= amdgpu_vm_bo_update(adev
, bo_va
, false);
732 pr_err("amdgpu_vm_bo_update failed\n");
736 return amdgpu_sync_fence(NULL
, sync
, bo_va
->last_pt_update
, false);
739 static int map_bo_to_gpuvm(struct amdgpu_device
*adev
,
740 struct kfd_bo_va_list
*entry
, struct amdgpu_sync
*sync
,
745 /* Set virtual address for the allocation */
746 ret
= amdgpu_vm_bo_map(adev
, entry
->bo_va
, entry
->va
, 0,
747 amdgpu_bo_size(entry
->bo_va
->base
.bo
),
750 pr_err("Failed to map VA 0x%llx in vm. ret %d\n",
758 ret
= update_gpuvm_pte(adev
, entry
, sync
);
760 pr_err("update_gpuvm_pte() failed\n");
761 goto update_gpuvm_pte_failed
;
766 update_gpuvm_pte_failed
:
767 unmap_bo_from_gpuvm(adev
, entry
, sync
);
771 static struct sg_table
*create_doorbell_sg(uint64_t addr
, uint32_t size
)
773 struct sg_table
*sg
= kmalloc(sizeof(*sg
), GFP_KERNEL
);
777 if (sg_alloc_table(sg
, 1, GFP_KERNEL
)) {
781 sg
->sgl
->dma_address
= addr
;
782 sg
->sgl
->length
= size
;
783 #ifdef CONFIG_NEED_SG_DMA_LENGTH
784 sg
->sgl
->dma_length
= size
;
789 static int process_validate_vms(struct amdkfd_process_info
*process_info
)
791 struct amdgpu_vm
*peer_vm
;
794 list_for_each_entry(peer_vm
, &process_info
->vm_list_head
,
796 ret
= vm_validate_pt_pd_bos(peer_vm
);
804 static int process_sync_pds_resv(struct amdkfd_process_info
*process_info
,
805 struct amdgpu_sync
*sync
)
807 struct amdgpu_vm
*peer_vm
;
810 list_for_each_entry(peer_vm
, &process_info
->vm_list_head
,
812 struct amdgpu_bo
*pd
= peer_vm
->root
.base
.bo
;
814 ret
= amdgpu_sync_resv(NULL
,
816 AMDGPU_FENCE_OWNER_KFD
, false);
824 static int process_update_pds(struct amdkfd_process_info
*process_info
,
825 struct amdgpu_sync
*sync
)
827 struct amdgpu_vm
*peer_vm
;
830 list_for_each_entry(peer_vm
, &process_info
->vm_list_head
,
832 ret
= vm_update_pds(peer_vm
, sync
);
840 static int init_kfd_vm(struct amdgpu_vm
*vm
, void **process_info
,
841 struct dma_fence
**ef
)
843 struct amdkfd_process_info
*info
= NULL
;
846 if (!*process_info
) {
847 info
= kzalloc(sizeof(*info
), GFP_KERNEL
);
851 mutex_init(&info
->lock
);
852 INIT_LIST_HEAD(&info
->vm_list_head
);
853 INIT_LIST_HEAD(&info
->kfd_bo_list
);
854 INIT_LIST_HEAD(&info
->userptr_valid_list
);
855 INIT_LIST_HEAD(&info
->userptr_inval_list
);
857 info
->eviction_fence
=
858 amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1),
860 if (!info
->eviction_fence
) {
861 pr_err("Failed to create eviction fence\n");
863 goto create_evict_fence_fail
;
866 info
->pid
= get_task_pid(current
->group_leader
, PIDTYPE_PID
);
867 atomic_set(&info
->evicted_bos
, 0);
868 INIT_DELAYED_WORK(&info
->restore_userptr_work
,
869 amdgpu_amdkfd_restore_userptr_worker
);
871 *process_info
= info
;
872 *ef
= dma_fence_get(&info
->eviction_fence
->base
);
875 vm
->process_info
= *process_info
;
877 /* Validate page directory and attach eviction fence */
878 ret
= amdgpu_bo_reserve(vm
->root
.base
.bo
, true);
880 goto reserve_pd_fail
;
881 ret
= vm_validate_pt_pd_bos(vm
);
883 pr_err("validate_pt_pd_bos() failed\n");
884 goto validate_pd_fail
;
886 ret
= amdgpu_bo_sync_wait(vm
->root
.base
.bo
,
887 AMDGPU_FENCE_OWNER_KFD
, false);
890 ret
= reservation_object_reserve_shared(vm
->root
.base
.bo
->tbo
.resv
, 1);
892 goto reserve_shared_fail
;
893 amdgpu_bo_fence(vm
->root
.base
.bo
,
894 &vm
->process_info
->eviction_fence
->base
, true);
895 amdgpu_bo_unreserve(vm
->root
.base
.bo
);
897 /* Update process info */
898 mutex_lock(&vm
->process_info
->lock
);
899 list_add_tail(&vm
->vm_list_node
,
900 &(vm
->process_info
->vm_list_head
));
901 vm
->process_info
->n_vms
++;
902 mutex_unlock(&vm
->process_info
->lock
);
909 amdgpu_bo_unreserve(vm
->root
.base
.bo
);
911 vm
->process_info
= NULL
;
913 /* Two fence references: one in info and one in *ef */
914 dma_fence_put(&info
->eviction_fence
->base
);
917 *process_info
= NULL
;
919 create_evict_fence_fail
:
920 mutex_destroy(&info
->lock
);
926 int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev
*kgd
, unsigned int pasid
,
927 void **vm
, void **process_info
,
928 struct dma_fence
**ef
)
930 struct amdgpu_device
*adev
= get_amdgpu_device(kgd
);
931 struct amdgpu_vm
*new_vm
;
934 new_vm
= kzalloc(sizeof(*new_vm
), GFP_KERNEL
);
938 /* Initialize AMDGPU part of the VM */
939 ret
= amdgpu_vm_init(adev
, new_vm
, AMDGPU_VM_CONTEXT_COMPUTE
, pasid
);
941 pr_err("Failed init vm ret %d\n", ret
);
942 goto amdgpu_vm_init_fail
;
945 /* Initialize KFD part of the VM and process info */
946 ret
= init_kfd_vm(new_vm
, process_info
, ef
);
948 goto init_kfd_vm_fail
;
950 *vm
= (void *) new_vm
;
955 amdgpu_vm_fini(adev
, new_vm
);
961 int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev
*kgd
,
962 struct file
*filp
, unsigned int pasid
,
963 void **vm
, void **process_info
,
964 struct dma_fence
**ef
)
966 struct amdgpu_device
*adev
= get_amdgpu_device(kgd
);
967 struct drm_file
*drm_priv
= filp
->private_data
;
968 struct amdgpu_fpriv
*drv_priv
= drm_priv
->driver_priv
;
969 struct amdgpu_vm
*avm
= &drv_priv
->vm
;
972 /* Already a compute VM? */
973 if (avm
->process_info
)
976 /* Convert VM into a compute VM */
977 ret
= amdgpu_vm_make_compute(adev
, avm
, pasid
);
981 /* Initialize KFD part of the VM and process info */
982 ret
= init_kfd_vm(avm
, process_info
, ef
);
991 void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device
*adev
,
992 struct amdgpu_vm
*vm
)
994 struct amdkfd_process_info
*process_info
= vm
->process_info
;
995 struct amdgpu_bo
*pd
= vm
->root
.base
.bo
;
1000 /* Release eviction fence from PD */
1001 amdgpu_bo_reserve(pd
, false);
1002 amdgpu_bo_fence(pd
, NULL
, false);
1003 amdgpu_bo_unreserve(pd
);
1005 /* Update process info */
1006 mutex_lock(&process_info
->lock
);
1007 process_info
->n_vms
--;
1008 list_del(&vm
->vm_list_node
);
1009 mutex_unlock(&process_info
->lock
);
1011 /* Release per-process resources when last compute VM is destroyed */
1012 if (!process_info
->n_vms
) {
1013 WARN_ON(!list_empty(&process_info
->kfd_bo_list
));
1014 WARN_ON(!list_empty(&process_info
->userptr_valid_list
));
1015 WARN_ON(!list_empty(&process_info
->userptr_inval_list
));
1017 dma_fence_put(&process_info
->eviction_fence
->base
);
1018 cancel_delayed_work_sync(&process_info
->restore_userptr_work
);
1019 put_pid(process_info
->pid
);
1020 mutex_destroy(&process_info
->lock
);
1021 kfree(process_info
);
1025 void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev
*kgd
, void *vm
)
1027 struct amdgpu_device
*adev
= get_amdgpu_device(kgd
);
1028 struct amdgpu_vm
*avm
= (struct amdgpu_vm
*)vm
;
1030 if (WARN_ON(!kgd
|| !vm
))
1033 pr_debug("Destroying process vm %p\n", vm
);
1035 /* Release the VM context */
1036 amdgpu_vm_fini(adev
, avm
);
1040 void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev
*kgd
, void *vm
)
1042 struct amdgpu_device
*adev
= get_amdgpu_device(kgd
);
1043 struct amdgpu_vm
*avm
= (struct amdgpu_vm
*)vm
;
1045 if (WARN_ON(!kgd
|| !vm
))
1048 pr_debug("Releasing process vm %p\n", vm
);
1050 /* The original pasid of amdgpu vm has already been
1051 * released during making a amdgpu vm to a compute vm
1052 * The current pasid is managed by kfd and will be
1053 * released on kfd process destroy. Set amdgpu pasid
1054 * to 0 to avoid duplicate release.
1056 amdgpu_vm_release_compute(adev
, avm
);
1059 uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm
)
1061 struct amdgpu_vm
*avm
= (struct amdgpu_vm
*)vm
;
1062 struct amdgpu_bo
*pd
= avm
->root
.base
.bo
;
1063 struct amdgpu_device
*adev
= amdgpu_ttm_adev(pd
->tbo
.bdev
);
1065 if (adev
->asic_type
< CHIP_VEGA10
)
1066 return avm
->pd_phys_addr
>> AMDGPU_GPU_PAGE_SHIFT
;
1067 return avm
->pd_phys_addr
;
1070 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1071 struct kgd_dev
*kgd
, uint64_t va
, uint64_t size
,
1072 void *vm
, struct kgd_mem
**mem
,
1073 uint64_t *offset
, uint32_t flags
)
1075 struct amdgpu_device
*adev
= get_amdgpu_device(kgd
);
1076 struct amdgpu_vm
*avm
= (struct amdgpu_vm
*)vm
;
1077 enum ttm_bo_type bo_type
= ttm_bo_type_device
;
1078 struct sg_table
*sg
= NULL
;
1079 uint64_t user_addr
= 0;
1080 struct amdgpu_bo
*bo
;
1081 struct amdgpu_bo_param bp
;
1083 u32 domain
, alloc_domain
;
1085 uint32_t mapping_flags
;
1089 * Check on which domain to allocate BO
1091 if (flags
& ALLOC_MEM_FLAGS_VRAM
) {
1092 domain
= alloc_domain
= AMDGPU_GEM_DOMAIN_VRAM
;
1093 alloc_flags
= AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE
;
1094 alloc_flags
|= (flags
& ALLOC_MEM_FLAGS_PUBLIC
) ?
1095 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
:
1096 AMDGPU_GEM_CREATE_NO_CPU_ACCESS
;
1097 } else if (flags
& ALLOC_MEM_FLAGS_GTT
) {
1098 domain
= alloc_domain
= AMDGPU_GEM_DOMAIN_GTT
;
1100 } else if (flags
& ALLOC_MEM_FLAGS_USERPTR
) {
1101 domain
= AMDGPU_GEM_DOMAIN_GTT
;
1102 alloc_domain
= AMDGPU_GEM_DOMAIN_CPU
;
1104 if (!offset
|| !*offset
)
1106 user_addr
= *offset
;
1107 } else if (flags
& (ALLOC_MEM_FLAGS_DOORBELL
|
1108 ALLOC_MEM_FLAGS_MMIO_REMAP
)) {
1109 domain
= AMDGPU_GEM_DOMAIN_GTT
;
1110 alloc_domain
= AMDGPU_GEM_DOMAIN_CPU
;
1111 bo_type
= ttm_bo_type_sg
;
1113 if (size
> UINT_MAX
)
1115 sg
= create_doorbell_sg(*offset
, size
);
1122 *mem
= kzalloc(sizeof(struct kgd_mem
), GFP_KERNEL
);
1127 INIT_LIST_HEAD(&(*mem
)->bo_va_list
);
1128 mutex_init(&(*mem
)->lock
);
1129 (*mem
)->aql_queue
= !!(flags
& ALLOC_MEM_FLAGS_AQL_QUEUE_MEM
);
1131 /* Workaround for AQL queue wraparound bug. Map the same
1132 * memory twice. That means we only actually allocate half
1135 if ((*mem
)->aql_queue
)
1138 /* Workaround for TLB bug on older VI chips */
1139 byte_align
= (adev
->family
== AMDGPU_FAMILY_VI
&&
1140 adev
->asic_type
!= CHIP_FIJI
&&
1141 adev
->asic_type
!= CHIP_POLARIS10
&&
1142 adev
->asic_type
!= CHIP_POLARIS11
&&
1143 adev
->asic_type
!= CHIP_POLARIS12
&&
1144 adev
->asic_type
!= CHIP_VEGAM
) ?
1145 VI_BO_SIZE_ALIGN
: 1;
1147 mapping_flags
= AMDGPU_VM_PAGE_READABLE
;
1148 if (flags
& ALLOC_MEM_FLAGS_WRITABLE
)
1149 mapping_flags
|= AMDGPU_VM_PAGE_WRITEABLE
;
1150 if (flags
& ALLOC_MEM_FLAGS_EXECUTABLE
)
1151 mapping_flags
|= AMDGPU_VM_PAGE_EXECUTABLE
;
1152 if (flags
& ALLOC_MEM_FLAGS_COHERENT
)
1153 mapping_flags
|= AMDGPU_VM_MTYPE_UC
;
1155 mapping_flags
|= AMDGPU_VM_MTYPE_NC
;
1156 (*mem
)->mapping_flags
= mapping_flags
;
1158 amdgpu_sync_create(&(*mem
)->sync
);
1160 ret
= amdgpu_amdkfd_reserve_mem_limit(adev
, size
, alloc_domain
, !!sg
);
1162 pr_debug("Insufficient system memory\n");
1163 goto err_reserve_limit
;
1166 pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s\n",
1167 va
, size
, domain_string(alloc_domain
));
1169 memset(&bp
, 0, sizeof(bp
));
1171 bp
.byte_align
= byte_align
;
1172 bp
.domain
= alloc_domain
;
1173 bp
.flags
= alloc_flags
;
1176 ret
= amdgpu_bo_create(adev
, &bp
, &bo
);
1178 pr_debug("Failed to create BO on domain %s. ret %d\n",
1179 domain_string(alloc_domain
), ret
);
1182 if (bo_type
== ttm_bo_type_sg
) {
1184 bo
->tbo
.ttm
->sg
= sg
;
1189 bo
->flags
|= AMDGPU_AMDKFD_USERPTR_BO
;
1192 (*mem
)->domain
= domain
;
1193 (*mem
)->mapped_to_gpu_memory
= 0;
1194 (*mem
)->process_info
= avm
->process_info
;
1195 add_kgd_mem_to_kfd_bo_list(*mem
, avm
->process_info
, user_addr
);
1198 ret
= init_user_pages(*mem
, current
->mm
, user_addr
);
1200 goto allocate_init_user_pages_failed
;
1204 *offset
= amdgpu_bo_mmap_offset(bo
);
1208 allocate_init_user_pages_failed
:
1209 remove_kgd_mem_from_kfd_bo_list(*mem
, avm
->process_info
);
1210 amdgpu_bo_unref(&bo
);
1211 /* Don't unreserve system mem limit twice */
1212 goto err_reserve_limit
;
1214 unreserve_mem_limit(adev
, size
, alloc_domain
, !!sg
);
1216 mutex_destroy(&(*mem
)->lock
);
1226 int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
1227 struct kgd_dev
*kgd
, struct kgd_mem
*mem
)
1229 struct amdkfd_process_info
*process_info
= mem
->process_info
;
1230 unsigned long bo_size
= mem
->bo
->tbo
.mem
.size
;
1231 struct kfd_bo_va_list
*entry
, *tmp
;
1232 struct bo_vm_reservation_context ctx
;
1233 struct ttm_validate_buffer
*bo_list_entry
;
1236 mutex_lock(&mem
->lock
);
1238 if (mem
->mapped_to_gpu_memory
> 0) {
1239 pr_debug("BO VA 0x%llx size 0x%lx is still mapped.\n",
1241 mutex_unlock(&mem
->lock
);
1245 mutex_unlock(&mem
->lock
);
1246 /* lock is not needed after this, since mem is unused and will
1250 /* No more MMU notifiers */
1251 amdgpu_mn_unregister(mem
->bo
);
1253 /* Make sure restore workers don't access the BO any more */
1254 bo_list_entry
= &mem
->validate_list
;
1255 mutex_lock(&process_info
->lock
);
1256 list_del(&bo_list_entry
->head
);
1257 mutex_unlock(&process_info
->lock
);
1259 ret
= reserve_bo_and_cond_vms(mem
, NULL
, BO_VM_ALL
, &ctx
);
1263 /* The eviction fence should be removed by the last unmap.
1264 * TODO: Log an error condition if the bo still has the eviction fence
1267 amdgpu_amdkfd_remove_eviction_fence(mem
->bo
,
1268 process_info
->eviction_fence
);
1269 pr_debug("Release VA 0x%llx - 0x%llx\n", mem
->va
,
1270 mem
->va
+ bo_size
* (1 + mem
->aql_queue
));
1272 /* Remove from VM internal data structures */
1273 list_for_each_entry_safe(entry
, tmp
, &mem
->bo_va_list
, bo_list
)
1274 remove_bo_from_vm((struct amdgpu_device
*)entry
->kgd_dev
,
1277 ret
= unreserve_bo_and_vms(&ctx
, false, false);
1279 /* Free the sync object */
1280 amdgpu_sync_free(&mem
->sync
);
1282 /* If the SG is not NULL, it's one we created for a doorbell or mmio
1283 * remap BO. We need to free it.
1285 if (mem
->bo
->tbo
.sg
) {
1286 sg_free_table(mem
->bo
->tbo
.sg
);
1287 kfree(mem
->bo
->tbo
.sg
);
1291 amdgpu_bo_unref(&mem
->bo
);
1292 mutex_destroy(&mem
->lock
);
1298 int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1299 struct kgd_dev
*kgd
, struct kgd_mem
*mem
, void *vm
)
1301 struct amdgpu_device
*adev
= get_amdgpu_device(kgd
);
1302 struct amdgpu_vm
*avm
= (struct amdgpu_vm
*)vm
;
1304 struct amdgpu_bo
*bo
;
1306 struct kfd_bo_va_list
*entry
;
1307 struct bo_vm_reservation_context ctx
;
1308 struct kfd_bo_va_list
*bo_va_entry
= NULL
;
1309 struct kfd_bo_va_list
*bo_va_entry_aql
= NULL
;
1310 unsigned long bo_size
;
1311 bool is_invalid_userptr
= false;
1315 pr_err("Invalid BO when mapping memory to GPU\n");
1319 /* Make sure restore is not running concurrently. Since we
1320 * don't map invalid userptr BOs, we rely on the next restore
1321 * worker to do the mapping
1323 mutex_lock(&mem
->process_info
->lock
);
1325 /* Lock mmap-sem. If we find an invalid userptr BO, we can be
1326 * sure that the MMU notifier is no longer running
1327 * concurrently and the queues are actually stopped
1329 if (amdgpu_ttm_tt_get_usermm(bo
->tbo
.ttm
)) {
1330 down_write(¤t
->mm
->mmap_sem
);
1331 is_invalid_userptr
= atomic_read(&mem
->invalid
);
1332 up_write(¤t
->mm
->mmap_sem
);
1335 mutex_lock(&mem
->lock
);
1337 domain
= mem
->domain
;
1338 bo_size
= bo
->tbo
.mem
.size
;
1340 pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n",
1342 mem
->va
+ bo_size
* (1 + mem
->aql_queue
),
1343 vm
, domain_string(domain
));
1345 ret
= reserve_bo_and_vm(mem
, vm
, &ctx
);
1349 /* Userptr can be marked as "not invalid", but not actually be
1350 * validated yet (still in the system domain). In that case
1351 * the queues are still stopped and we can leave mapping for
1352 * the next restore worker
1354 if (amdgpu_ttm_tt_get_usermm(bo
->tbo
.ttm
) &&
1355 bo
->tbo
.mem
.mem_type
== TTM_PL_SYSTEM
)
1356 is_invalid_userptr
= true;
1358 if (check_if_add_bo_to_vm(avm
, mem
)) {
1359 ret
= add_bo_to_vm(adev
, mem
, avm
, false,
1362 goto add_bo_to_vm_failed
;
1363 if (mem
->aql_queue
) {
1364 ret
= add_bo_to_vm(adev
, mem
, avm
,
1365 true, &bo_va_entry_aql
);
1367 goto add_bo_to_vm_failed_aql
;
1370 ret
= vm_validate_pt_pd_bos(avm
);
1372 goto add_bo_to_vm_failed
;
1375 if (mem
->mapped_to_gpu_memory
== 0 &&
1376 !amdgpu_ttm_tt_get_usermm(bo
->tbo
.ttm
)) {
1377 /* Validate BO only once. The eviction fence gets added to BO
1378 * the first time it is mapped. Validate will wait for all
1379 * background evictions to complete.
1381 ret
= amdgpu_amdkfd_bo_validate(bo
, domain
, true);
1383 pr_debug("Validate failed\n");
1384 goto map_bo_to_gpuvm_failed
;
1388 list_for_each_entry(entry
, &mem
->bo_va_list
, bo_list
) {
1389 if (entry
->bo_va
->base
.vm
== vm
&& !entry
->is_mapped
) {
1390 pr_debug("\t map VA 0x%llx - 0x%llx in entry %p\n",
1391 entry
->va
, entry
->va
+ bo_size
,
1394 ret
= map_bo_to_gpuvm(adev
, entry
, ctx
.sync
,
1395 is_invalid_userptr
);
1397 pr_err("Failed to map bo to gpuvm\n");
1398 goto map_bo_to_gpuvm_failed
;
1401 ret
= vm_update_pds(vm
, ctx
.sync
);
1403 pr_err("Failed to update page directories\n");
1404 goto map_bo_to_gpuvm_failed
;
1407 entry
->is_mapped
= true;
1408 mem
->mapped_to_gpu_memory
++;
1409 pr_debug("\t INC mapping count %d\n",
1410 mem
->mapped_to_gpu_memory
);
1414 if (!amdgpu_ttm_tt_get_usermm(bo
->tbo
.ttm
) && !bo
->pin_count
)
1416 &avm
->process_info
->eviction_fence
->base
,
1418 ret
= unreserve_bo_and_vms(&ctx
, false, false);
1422 map_bo_to_gpuvm_failed
:
1423 if (bo_va_entry_aql
)
1424 remove_bo_from_vm(adev
, bo_va_entry_aql
, bo_size
);
1425 add_bo_to_vm_failed_aql
:
1427 remove_bo_from_vm(adev
, bo_va_entry
, bo_size
);
1428 add_bo_to_vm_failed
:
1429 unreserve_bo_and_vms(&ctx
, false, false);
1431 mutex_unlock(&mem
->process_info
->lock
);
1432 mutex_unlock(&mem
->lock
);
1436 int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1437 struct kgd_dev
*kgd
, struct kgd_mem
*mem
, void *vm
)
1439 struct amdgpu_device
*adev
= get_amdgpu_device(kgd
);
1440 struct amdkfd_process_info
*process_info
=
1441 ((struct amdgpu_vm
*)vm
)->process_info
;
1442 unsigned long bo_size
= mem
->bo
->tbo
.mem
.size
;
1443 struct kfd_bo_va_list
*entry
;
1444 struct bo_vm_reservation_context ctx
;
1447 mutex_lock(&mem
->lock
);
1449 ret
= reserve_bo_and_cond_vms(mem
, vm
, BO_VM_MAPPED
, &ctx
);
1452 /* If no VMs were reserved, it means the BO wasn't actually mapped */
1453 if (ctx
.n_vms
== 0) {
1458 ret
= vm_validate_pt_pd_bos((struct amdgpu_vm
*)vm
);
1462 pr_debug("Unmap VA 0x%llx - 0x%llx from vm %p\n",
1464 mem
->va
+ bo_size
* (1 + mem
->aql_queue
),
1467 list_for_each_entry(entry
, &mem
->bo_va_list
, bo_list
) {
1468 if (entry
->bo_va
->base
.vm
== vm
&& entry
->is_mapped
) {
1469 pr_debug("\t unmap VA 0x%llx - 0x%llx from entry %p\n",
1471 entry
->va
+ bo_size
,
1474 ret
= unmap_bo_from_gpuvm(adev
, entry
, ctx
.sync
);
1476 entry
->is_mapped
= false;
1478 pr_err("failed to unmap VA 0x%llx\n",
1483 mem
->mapped_to_gpu_memory
--;
1484 pr_debug("\t DEC mapping count %d\n",
1485 mem
->mapped_to_gpu_memory
);
1489 /* If BO is unmapped from all VMs, unfence it. It can be evicted if
1492 if (mem
->mapped_to_gpu_memory
== 0 &&
1493 !amdgpu_ttm_tt_get_usermm(mem
->bo
->tbo
.ttm
) && !mem
->bo
->pin_count
)
1494 amdgpu_amdkfd_remove_eviction_fence(mem
->bo
,
1495 process_info
->eviction_fence
);
1498 unreserve_bo_and_vms(&ctx
, false, false);
1500 mutex_unlock(&mem
->lock
);
1504 int amdgpu_amdkfd_gpuvm_sync_memory(
1505 struct kgd_dev
*kgd
, struct kgd_mem
*mem
, bool intr
)
1507 struct amdgpu_sync sync
;
1510 amdgpu_sync_create(&sync
);
1512 mutex_lock(&mem
->lock
);
1513 amdgpu_sync_clone(&mem
->sync
, &sync
);
1514 mutex_unlock(&mem
->lock
);
1516 ret
= amdgpu_sync_wait(&sync
, intr
);
1517 amdgpu_sync_free(&sync
);
1521 int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev
*kgd
,
1522 struct kgd_mem
*mem
, void **kptr
, uint64_t *size
)
1525 struct amdgpu_bo
*bo
= mem
->bo
;
1527 if (amdgpu_ttm_tt_get_usermm(bo
->tbo
.ttm
)) {
1528 pr_err("userptr can't be mapped to kernel\n");
1532 /* delete kgd_mem from kfd_bo_list to avoid re-validating
1533 * this BO in BO's restoring after eviction.
1535 mutex_lock(&mem
->process_info
->lock
);
1537 ret
= amdgpu_bo_reserve(bo
, true);
1539 pr_err("Failed to reserve bo. ret %d\n", ret
);
1540 goto bo_reserve_failed
;
1543 ret
= amdgpu_bo_pin(bo
, AMDGPU_GEM_DOMAIN_GTT
);
1545 pr_err("Failed to pin bo. ret %d\n", ret
);
1549 ret
= amdgpu_bo_kmap(bo
, kptr
);
1551 pr_err("Failed to map bo to kernel. ret %d\n", ret
);
1555 amdgpu_amdkfd_remove_eviction_fence(
1556 bo
, mem
->process_info
->eviction_fence
);
1557 list_del_init(&mem
->validate_list
.head
);
1560 *size
= amdgpu_bo_size(bo
);
1562 amdgpu_bo_unreserve(bo
);
1564 mutex_unlock(&mem
->process_info
->lock
);
1568 amdgpu_bo_unpin(bo
);
1570 amdgpu_bo_unreserve(bo
);
1572 mutex_unlock(&mem
->process_info
->lock
);
1577 int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev
*kgd
,
1578 struct kfd_vm_fault_info
*mem
)
1580 struct amdgpu_device
*adev
;
1582 adev
= (struct amdgpu_device
*)kgd
;
1583 if (atomic_read(&adev
->gmc
.vm_fault_info_updated
) == 1) {
1584 *mem
= *adev
->gmc
.vm_fault_info
;
1586 atomic_set(&adev
->gmc
.vm_fault_info_updated
, 0);
1591 int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev
*kgd
,
1592 struct dma_buf
*dma_buf
,
1593 uint64_t va
, void *vm
,
1594 struct kgd_mem
**mem
, uint64_t *size
,
1595 uint64_t *mmap_offset
)
1597 struct amdgpu_device
*adev
= (struct amdgpu_device
*)kgd
;
1598 struct drm_gem_object
*obj
;
1599 struct amdgpu_bo
*bo
;
1600 struct amdgpu_vm
*avm
= (struct amdgpu_vm
*)vm
;
1602 if (dma_buf
->ops
!= &amdgpu_dmabuf_ops
)
1603 /* Can't handle non-graphics buffers */
1606 obj
= dma_buf
->priv
;
1607 if (obj
->dev
->dev_private
!= adev
)
1608 /* Can't handle buffers from other devices */
1611 bo
= gem_to_amdgpu_bo(obj
);
1612 if (!(bo
->preferred_domains
& (AMDGPU_GEM_DOMAIN_VRAM
|
1613 AMDGPU_GEM_DOMAIN_GTT
)))
1614 /* Only VRAM and GTT BOs are supported */
1617 *mem
= kzalloc(sizeof(struct kgd_mem
), GFP_KERNEL
);
1622 *size
= amdgpu_bo_size(bo
);
1625 *mmap_offset
= amdgpu_bo_mmap_offset(bo
);
1627 INIT_LIST_HEAD(&(*mem
)->bo_va_list
);
1628 mutex_init(&(*mem
)->lock
);
1629 (*mem
)->mapping_flags
=
1630 AMDGPU_VM_PAGE_READABLE
| AMDGPU_VM_PAGE_WRITEABLE
|
1631 AMDGPU_VM_PAGE_EXECUTABLE
| AMDGPU_VM_MTYPE_NC
;
1633 (*mem
)->bo
= amdgpu_bo_ref(bo
);
1635 (*mem
)->domain
= (bo
->preferred_domains
& AMDGPU_GEM_DOMAIN_VRAM
) ?
1636 AMDGPU_GEM_DOMAIN_VRAM
: AMDGPU_GEM_DOMAIN_GTT
;
1637 (*mem
)->mapped_to_gpu_memory
= 0;
1638 (*mem
)->process_info
= avm
->process_info
;
1639 add_kgd_mem_to_kfd_bo_list(*mem
, avm
->process_info
, false);
1640 amdgpu_sync_create(&(*mem
)->sync
);
1645 /* Evict a userptr BO by stopping the queues if necessary
1647 * Runs in MMU notifier, may be in RECLAIM_FS context. This means it
1648 * cannot do any memory allocations, and cannot take any locks that
1649 * are held elsewhere while allocating memory. Therefore this is as
1650 * simple as possible, using atomic counters.
1652 * It doesn't do anything to the BO itself. The real work happens in
1653 * restore, where we get updated page addresses. This function only
1654 * ensures that GPU access to the BO is stopped.
1656 int amdgpu_amdkfd_evict_userptr(struct kgd_mem
*mem
,
1657 struct mm_struct
*mm
)
1659 struct amdkfd_process_info
*process_info
= mem
->process_info
;
1660 int invalid
, evicted_bos
;
1663 invalid
= atomic_inc_return(&mem
->invalid
);
1664 evicted_bos
= atomic_inc_return(&process_info
->evicted_bos
);
1665 if (evicted_bos
== 1) {
1666 /* First eviction, stop the queues */
1667 r
= kgd2kfd_quiesce_mm(mm
);
1669 pr_err("Failed to quiesce KFD\n");
1670 schedule_delayed_work(&process_info
->restore_userptr_work
,
1671 msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS
));
1677 /* Update invalid userptr BOs
1679 * Moves invalidated (evicted) userptr BOs from userptr_valid_list to
1680 * userptr_inval_list and updates user pages for all BOs that have
1681 * been invalidated since their last update.
1683 static int update_invalid_user_pages(struct amdkfd_process_info
*process_info
,
1684 struct mm_struct
*mm
)
1686 struct kgd_mem
*mem
, *tmp_mem
;
1687 struct amdgpu_bo
*bo
;
1688 struct ttm_operation_ctx ctx
= { false, false };
1691 /* Move all invalidated BOs to the userptr_inval_list and
1692 * release their user pages by migration to the CPU domain
1694 list_for_each_entry_safe(mem
, tmp_mem
,
1695 &process_info
->userptr_valid_list
,
1696 validate_list
.head
) {
1697 if (!atomic_read(&mem
->invalid
))
1698 continue; /* BO is still valid */
1702 if (amdgpu_bo_reserve(bo
, true))
1704 amdgpu_bo_placement_from_domain(bo
, AMDGPU_GEM_DOMAIN_CPU
);
1705 ret
= ttm_bo_validate(&bo
->tbo
, &bo
->placement
, &ctx
);
1706 amdgpu_bo_unreserve(bo
);
1708 pr_err("%s: Failed to invalidate userptr BO\n",
1713 list_move_tail(&mem
->validate_list
.head
,
1714 &process_info
->userptr_inval_list
);
1717 if (list_empty(&process_info
->userptr_inval_list
))
1718 return 0; /* All evicted userptr BOs were freed */
1720 /* Go through userptr_inval_list and update any invalid user_pages */
1721 list_for_each_entry(mem
, &process_info
->userptr_inval_list
,
1722 validate_list
.head
) {
1723 invalid
= atomic_read(&mem
->invalid
);
1725 /* BO hasn't been invalidated since the last
1726 * revalidation attempt. Keep its BO list.
1732 /* Get updated user pages */
1733 ret
= amdgpu_ttm_tt_get_user_pages(bo
, bo
->tbo
.ttm
->pages
);
1735 pr_debug("%s: Failed to get user pages: %d\n",
1738 /* Return error -EBUSY or -ENOMEM, retry restore */
1742 amdgpu_ttm_tt_get_user_pages_done(bo
->tbo
.ttm
);
1744 /* Mark the BO as valid unless it was invalidated
1745 * again concurrently.
1747 if (atomic_cmpxchg(&mem
->invalid
, invalid
, 0) != invalid
)
1754 /* Validate invalid userptr BOs
1756 * Validates BOs on the userptr_inval_list, and moves them back to the
1757 * userptr_valid_list. Also updates GPUVM page tables with new page
1758 * addresses and waits for the page table updates to complete.
1760 static int validate_invalid_user_pages(struct amdkfd_process_info
*process_info
)
1762 struct amdgpu_bo_list_entry
*pd_bo_list_entries
;
1763 struct list_head resv_list
, duplicates
;
1764 struct ww_acquire_ctx ticket
;
1765 struct amdgpu_sync sync
;
1767 struct amdgpu_vm
*peer_vm
;
1768 struct kgd_mem
*mem
, *tmp_mem
;
1769 struct amdgpu_bo
*bo
;
1770 struct ttm_operation_ctx ctx
= { false, false };
1773 pd_bo_list_entries
= kcalloc(process_info
->n_vms
,
1774 sizeof(struct amdgpu_bo_list_entry
),
1776 if (!pd_bo_list_entries
) {
1777 pr_err("%s: Failed to allocate PD BO list entries\n", __func__
);
1782 INIT_LIST_HEAD(&resv_list
);
1783 INIT_LIST_HEAD(&duplicates
);
1785 /* Get all the page directory BOs that need to be reserved */
1787 list_for_each_entry(peer_vm
, &process_info
->vm_list_head
,
1789 amdgpu_vm_get_pd_bo(peer_vm
, &resv_list
,
1790 &pd_bo_list_entries
[i
++]);
1791 /* Add the userptr_inval_list entries to resv_list */
1792 list_for_each_entry(mem
, &process_info
->userptr_inval_list
,
1793 validate_list
.head
) {
1794 list_add_tail(&mem
->resv_list
.head
, &resv_list
);
1795 mem
->resv_list
.bo
= mem
->validate_list
.bo
;
1796 mem
->resv_list
.num_shared
= mem
->validate_list
.num_shared
;
1799 /* Reserve all BOs and page tables for validation */
1800 ret
= ttm_eu_reserve_buffers(&ticket
, &resv_list
, false, &duplicates
,
1802 WARN(!list_empty(&duplicates
), "Duplicates should be empty");
1806 amdgpu_sync_create(&sync
);
1808 ret
= process_validate_vms(process_info
);
1812 /* Validate BOs and update GPUVM page tables */
1813 list_for_each_entry_safe(mem
, tmp_mem
,
1814 &process_info
->userptr_inval_list
,
1815 validate_list
.head
) {
1816 struct kfd_bo_va_list
*bo_va_entry
;
1820 /* Validate the BO if we got user pages */
1821 if (bo
->tbo
.ttm
->pages
[0]) {
1822 amdgpu_bo_placement_from_domain(bo
, mem
->domain
);
1823 ret
= ttm_bo_validate(&bo
->tbo
, &bo
->placement
, &ctx
);
1825 pr_err("%s: failed to validate BO\n", __func__
);
1830 list_move_tail(&mem
->validate_list
.head
,
1831 &process_info
->userptr_valid_list
);
1833 /* Update mapping. If the BO was not validated
1834 * (because we couldn't get user pages), this will
1835 * clear the page table entries, which will result in
1836 * VM faults if the GPU tries to access the invalid
1839 list_for_each_entry(bo_va_entry
, &mem
->bo_va_list
, bo_list
) {
1840 if (!bo_va_entry
->is_mapped
)
1843 ret
= update_gpuvm_pte((struct amdgpu_device
*)
1844 bo_va_entry
->kgd_dev
,
1845 bo_va_entry
, &sync
);
1847 pr_err("%s: update PTE failed\n", __func__
);
1848 /* make sure this gets validated again */
1849 atomic_inc(&mem
->invalid
);
1855 /* Update page directories */
1856 ret
= process_update_pds(process_info
, &sync
);
1859 ttm_eu_backoff_reservation(&ticket
, &resv_list
);
1860 amdgpu_sync_wait(&sync
, false);
1861 amdgpu_sync_free(&sync
);
1863 kfree(pd_bo_list_entries
);
1869 /* Worker callback to restore evicted userptr BOs
1871 * Tries to update and validate all userptr BOs. If successful and no
1872 * concurrent evictions happened, the queues are restarted. Otherwise,
1873 * reschedule for another attempt later.
1875 static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct
*work
)
1877 struct delayed_work
*dwork
= to_delayed_work(work
);
1878 struct amdkfd_process_info
*process_info
=
1879 container_of(dwork
, struct amdkfd_process_info
,
1880 restore_userptr_work
);
1881 struct task_struct
*usertask
;
1882 struct mm_struct
*mm
;
1885 evicted_bos
= atomic_read(&process_info
->evicted_bos
);
1889 /* Reference task and mm in case of concurrent process termination */
1890 usertask
= get_pid_task(process_info
->pid
, PIDTYPE_PID
);
1893 mm
= get_task_mm(usertask
);
1895 put_task_struct(usertask
);
1899 mutex_lock(&process_info
->lock
);
1901 if (update_invalid_user_pages(process_info
, mm
))
1903 /* userptr_inval_list can be empty if all evicted userptr BOs
1904 * have been freed. In that case there is nothing to validate
1905 * and we can just restart the queues.
1907 if (!list_empty(&process_info
->userptr_inval_list
)) {
1908 if (atomic_read(&process_info
->evicted_bos
) != evicted_bos
)
1909 goto unlock_out
; /* Concurrent eviction, try again */
1911 if (validate_invalid_user_pages(process_info
))
1914 /* Final check for concurrent evicton and atomic update. If
1915 * another eviction happens after successful update, it will
1916 * be a first eviction that calls quiesce_mm. The eviction
1917 * reference counting inside KFD will handle this case.
1919 if (atomic_cmpxchg(&process_info
->evicted_bos
, evicted_bos
, 0) !=
1923 if (kgd2kfd_resume_mm(mm
)) {
1924 pr_err("%s: Failed to resume KFD\n", __func__
);
1925 /* No recovery from this failure. Probably the CP is
1926 * hanging. No point trying again.
1931 mutex_unlock(&process_info
->lock
);
1933 put_task_struct(usertask
);
1935 /* If validation failed, reschedule another attempt */
1937 schedule_delayed_work(&process_info
->restore_userptr_work
,
1938 msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS
));
1941 /** amdgpu_amdkfd_gpuvm_restore_process_bos - Restore all BOs for the given
1942 * KFD process identified by process_info
1944 * @process_info: amdkfd_process_info of the KFD process
1946 * After memory eviction, restore thread calls this function. The function
1947 * should be called when the Process is still valid. BO restore involves -
1949 * 1. Release old eviction fence and create new one
1950 * 2. Get two copies of PD BO list from all the VMs. Keep one copy as pd_list.
1951 * 3 Use the second PD list and kfd_bo_list to create a list (ctx.list) of
1952 * BOs that need to be reserved.
1953 * 4. Reserve all the BOs
1954 * 5. Validate of PD and PT BOs.
1955 * 6. Validate all KFD BOs using kfd_bo_list and Map them and add new fence
1956 * 7. Add fence to all PD and PT BOs.
1957 * 8. Unreserve all BOs
1959 int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info
, struct dma_fence
**ef
)
1961 struct amdgpu_bo_list_entry
*pd_bo_list
;
1962 struct amdkfd_process_info
*process_info
= info
;
1963 struct amdgpu_vm
*peer_vm
;
1964 struct kgd_mem
*mem
;
1965 struct bo_vm_reservation_context ctx
;
1966 struct amdgpu_amdkfd_fence
*new_fence
;
1968 struct list_head duplicate_save
;
1969 struct amdgpu_sync sync_obj
;
1971 INIT_LIST_HEAD(&duplicate_save
);
1972 INIT_LIST_HEAD(&ctx
.list
);
1973 INIT_LIST_HEAD(&ctx
.duplicates
);
1975 pd_bo_list
= kcalloc(process_info
->n_vms
,
1976 sizeof(struct amdgpu_bo_list_entry
),
1982 mutex_lock(&process_info
->lock
);
1983 list_for_each_entry(peer_vm
, &process_info
->vm_list_head
,
1985 amdgpu_vm_get_pd_bo(peer_vm
, &ctx
.list
, &pd_bo_list
[i
++]);
1987 /* Reserve all BOs and page tables/directory. Add all BOs from
1988 * kfd_bo_list to ctx.list
1990 list_for_each_entry(mem
, &process_info
->kfd_bo_list
,
1991 validate_list
.head
) {
1993 list_add_tail(&mem
->resv_list
.head
, &ctx
.list
);
1994 mem
->resv_list
.bo
= mem
->validate_list
.bo
;
1995 mem
->resv_list
.num_shared
= mem
->validate_list
.num_shared
;
1998 ret
= ttm_eu_reserve_buffers(&ctx
.ticket
, &ctx
.list
,
1999 false, &duplicate_save
, true);
2001 pr_debug("Memory eviction: TTM Reserve Failed. Try again\n");
2002 goto ttm_reserve_fail
;
2005 amdgpu_sync_create(&sync_obj
);
2007 /* Validate PDs and PTs */
2008 ret
= process_validate_vms(process_info
);
2010 goto validate_map_fail
;
2012 ret
= process_sync_pds_resv(process_info
, &sync_obj
);
2014 pr_debug("Memory eviction: Failed to sync to PD BO moving fence. Try again\n");
2015 goto validate_map_fail
;
2018 /* Validate BOs and map them to GPUVM (update VM page tables). */
2019 list_for_each_entry(mem
, &process_info
->kfd_bo_list
,
2020 validate_list
.head
) {
2022 struct amdgpu_bo
*bo
= mem
->bo
;
2023 uint32_t domain
= mem
->domain
;
2024 struct kfd_bo_va_list
*bo_va_entry
;
2026 ret
= amdgpu_amdkfd_bo_validate(bo
, domain
, false);
2028 pr_debug("Memory eviction: Validate BOs failed. Try again\n");
2029 goto validate_map_fail
;
2031 ret
= amdgpu_sync_fence(NULL
, &sync_obj
, bo
->tbo
.moving
, false);
2033 pr_debug("Memory eviction: Sync BO fence failed. Try again\n");
2034 goto validate_map_fail
;
2036 list_for_each_entry(bo_va_entry
, &mem
->bo_va_list
,
2038 ret
= update_gpuvm_pte((struct amdgpu_device
*)
2039 bo_va_entry
->kgd_dev
,
2043 pr_debug("Memory eviction: update PTE failed. Try again\n");
2044 goto validate_map_fail
;
2049 /* Update page directories */
2050 ret
= process_update_pds(process_info
, &sync_obj
);
2052 pr_debug("Memory eviction: update PDs failed. Try again\n");
2053 goto validate_map_fail
;
2056 /* Wait for validate and PT updates to finish */
2057 amdgpu_sync_wait(&sync_obj
, false);
2059 /* Release old eviction fence and create new one, because fence only
2060 * goes from unsignaled to signaled, fence cannot be reused.
2061 * Use context and mm from the old fence.
2063 new_fence
= amdgpu_amdkfd_fence_create(
2064 process_info
->eviction_fence
->base
.context
,
2065 process_info
->eviction_fence
->mm
);
2067 pr_err("Failed to create eviction fence\n");
2069 goto validate_map_fail
;
2071 dma_fence_put(&process_info
->eviction_fence
->base
);
2072 process_info
->eviction_fence
= new_fence
;
2073 *ef
= dma_fence_get(&new_fence
->base
);
2075 /* Attach new eviction fence to all BOs */
2076 list_for_each_entry(mem
, &process_info
->kfd_bo_list
,
2078 amdgpu_bo_fence(mem
->bo
,
2079 &process_info
->eviction_fence
->base
, true);
2081 /* Attach eviction fence to PD / PT BOs */
2082 list_for_each_entry(peer_vm
, &process_info
->vm_list_head
,
2084 struct amdgpu_bo
*bo
= peer_vm
->root
.base
.bo
;
2086 amdgpu_bo_fence(bo
, &process_info
->eviction_fence
->base
, true);
2090 ttm_eu_backoff_reservation(&ctx
.ticket
, &ctx
.list
);
2091 amdgpu_sync_free(&sync_obj
);
2093 mutex_unlock(&process_info
->lock
);
2098 int amdgpu_amdkfd_add_gws_to_process(void *info
, void *gws
, struct kgd_mem
**mem
)
2100 struct amdkfd_process_info
*process_info
= (struct amdkfd_process_info
*)info
;
2101 struct amdgpu_bo
*gws_bo
= (struct amdgpu_bo
*)gws
;
2107 *mem
= kzalloc(sizeof(struct kgd_mem
), GFP_KERNEL
);
2111 mutex_init(&(*mem
)->lock
);
2112 (*mem
)->bo
= amdgpu_bo_ref(gws_bo
);
2113 (*mem
)->domain
= AMDGPU_GEM_DOMAIN_GWS
;
2114 (*mem
)->process_info
= process_info
;
2115 add_kgd_mem_to_kfd_bo_list(*mem
, process_info
, false);
2116 amdgpu_sync_create(&(*mem
)->sync
);
2119 /* Validate gws bo the first time it is added to process */
2120 mutex_lock(&(*mem
)->process_info
->lock
);
2121 ret
= amdgpu_bo_reserve(gws_bo
, false);
2122 if (unlikely(ret
)) {
2123 pr_err("Reserve gws bo failed %d\n", ret
);
2124 goto bo_reservation_failure
;
2127 ret
= amdgpu_amdkfd_bo_validate(gws_bo
, AMDGPU_GEM_DOMAIN_GWS
, true);
2129 pr_err("GWS BO validate failed %d\n", ret
);
2130 goto bo_validation_failure
;
2132 /* GWS resource is shared b/t amdgpu and amdkfd
2133 * Add process eviction fence to bo so they can
2136 ret
= reservation_object_reserve_shared(gws_bo
->tbo
.resv
, 1);
2138 goto reserve_shared_fail
;
2139 amdgpu_bo_fence(gws_bo
, &process_info
->eviction_fence
->base
, true);
2140 amdgpu_bo_unreserve(gws_bo
);
2141 mutex_unlock(&(*mem
)->process_info
->lock
);
2145 reserve_shared_fail
:
2146 bo_validation_failure
:
2147 amdgpu_bo_unreserve(gws_bo
);
2148 bo_reservation_failure
:
2149 mutex_unlock(&(*mem
)->process_info
->lock
);
2150 amdgpu_sync_free(&(*mem
)->sync
);
2151 remove_kgd_mem_from_kfd_bo_list(*mem
, process_info
);
2152 amdgpu_bo_unref(&gws_bo
);
2153 mutex_destroy(&(*mem
)->lock
);
2159 int amdgpu_amdkfd_remove_gws_from_process(void *info
, void *mem
)
2162 struct amdkfd_process_info
*process_info
= (struct amdkfd_process_info
*)info
;
2163 struct kgd_mem
*kgd_mem
= (struct kgd_mem
*)mem
;
2164 struct amdgpu_bo
*gws_bo
= kgd_mem
->bo
;
2166 /* Remove BO from process's validate list so restore worker won't touch
2169 remove_kgd_mem_from_kfd_bo_list(kgd_mem
, process_info
);
2171 ret
= amdgpu_bo_reserve(gws_bo
, false);
2172 if (unlikely(ret
)) {
2173 pr_err("Reserve gws bo failed %d\n", ret
);
2174 //TODO add BO back to validate_list?
2177 amdgpu_amdkfd_remove_eviction_fence(gws_bo
,
2178 process_info
->eviction_fence
);
2179 amdgpu_bo_unreserve(gws_bo
);
2180 amdgpu_sync_free(&kgd_mem
->sync
);
2181 amdgpu_bo_unref(&gws_bo
);
2182 mutex_destroy(&kgd_mem
->lock
);