*/
static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm)
{
- struct amdgpu_bo *pd = vm->root.base.bo;
+ struct amdgpu_bo *pd = vm->root.bo;
struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
int ret;
return ret;
}
- vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.base.bo);
+ vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.bo);
if (vm->use_cpu_for_update) {
ret = amdgpu_bo_kmap(pd, NULL);
static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync)
{
- struct amdgpu_bo *pd = vm->root.base.bo;
+ struct amdgpu_bo *pd = vm->root.bo;
struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
int ret;
list_for_each_entry(peer_vm, &process_info->vm_list_head,
vm_list_node) {
- struct amdgpu_bo *pd = peer_vm->root.base.bo;
+ struct amdgpu_bo *pd = peer_vm->root.bo;
ret = amdgpu_sync_resv(NULL, sync, pd->tbo.base.resv,
AMDGPU_SYNC_NE_OWNER,
vm->process_info = *process_info;
/* Validate page directory and attach eviction fence */
- ret = amdgpu_bo_reserve(vm->root.base.bo, true);
+ ret = amdgpu_bo_reserve(vm->root.bo, true);
if (ret)
goto reserve_pd_fail;
ret = vm_validate_pt_pd_bos(vm);
pr_err("validate_pt_pd_bos() failed\n");
goto validate_pd_fail;
}
- ret = amdgpu_bo_sync_wait(vm->root.base.bo,
+ ret = amdgpu_bo_sync_wait(vm->root.bo,
AMDGPU_FENCE_OWNER_KFD, false);
if (ret)
goto wait_pd_fail;
- ret = dma_resv_reserve_shared(vm->root.base.bo->tbo.base.resv, 1);
+ ret = dma_resv_reserve_shared(vm->root.bo->tbo.base.resv, 1);
if (ret)
goto reserve_shared_fail;
- amdgpu_bo_fence(vm->root.base.bo,
+ amdgpu_bo_fence(vm->root.bo,
&vm->process_info->eviction_fence->base, true);
- amdgpu_bo_unreserve(vm->root.base.bo);
+ amdgpu_bo_unreserve(vm->root.bo);
/* Update process info */
mutex_lock(&vm->process_info->lock);
reserve_shared_fail:
wait_pd_fail:
validate_pd_fail:
- amdgpu_bo_unreserve(vm->root.base.bo);
+ amdgpu_bo_unreserve(vm->root.bo);
reserve_pd_fail:
vm->process_info = NULL;
if (info) {
struct amdgpu_vm *vm)
{
struct amdkfd_process_info *process_info = vm->process_info;
- struct amdgpu_bo *pd = vm->root.base.bo;
+ struct amdgpu_bo *pd = vm->root.bo;
if (!process_info)
return;
uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv)
{
struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
- struct amdgpu_bo *pd = avm->root.base.bo;
+ struct amdgpu_bo *pd = avm->root.bo;
struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
if (adev->asic_type < CHIP_VEGA10)
/* Attach eviction fence to PD / PT BOs */
list_for_each_entry(peer_vm, &process_info->vm_list_head,
vm_list_node) {
- struct amdgpu_bo *bo = peer_vm->root.base.bo;
+ struct amdgpu_bo *bo = peer_vm->root.bo;
amdgpu_bo_fence(bo, &process_info->eviction_fence->base, true);
}
base->next = bo->vm_bo;
bo->vm_bo = base;
- if (bo->tbo.base.resv != vm->root.base.bo->tbo.base.resv)
+ if (bo->tbo.base.resv != vm->root.bo->tbo.base.resv)
return;
vm->bulk_moveable = false;
* Helper to get the parent entry for the child page table. NULL if we are at
* the root page directory.
*/
-static struct amdgpu_vm_pt *amdgpu_vm_pt_parent(struct amdgpu_vm_pt *pt)
+static struct amdgpu_vm_bo_base *amdgpu_vm_pt_parent(struct amdgpu_vm_bo_base *pt)
{
- struct amdgpu_bo *parent = pt->base.bo->parent;
+ struct amdgpu_bo *parent = pt->bo->parent;
if (!parent)
return NULL;
- return container_of(parent->vm_bo, struct amdgpu_vm_pt, base);
+ return parent->vm_bo;
}
/*
*/
struct amdgpu_vm_pt_cursor {
uint64_t pfn;
- struct amdgpu_vm_pt *parent;
- struct amdgpu_vm_pt *entry;
+ struct amdgpu_vm_bo_base *parent;
+ struct amdgpu_vm_bo_base *entry;
unsigned level;
};
{
unsigned mask, shift, idx;
- if (!cursor->entry->entries)
+ if ((cursor->level == AMDGPU_VM_PTB) || !cursor->entry ||
+ !cursor->entry->bo)
return false;
- BUG_ON(!cursor->entry->base.bo);
mask = amdgpu_vm_entries_mask(adev, cursor->level);
shift = amdgpu_vm_level_shift(adev, cursor->level);
++cursor->level;
idx = (cursor->pfn >> shift) & mask;
cursor->parent = cursor->entry;
- cursor->entry = &cursor->entry->entries[idx];
+ cursor->entry = &to_amdgpu_bo_vm(cursor->entry->bo)->entries[idx];
return true;
}
shift = amdgpu_vm_level_shift(adev, cursor->level - 1);
num_entries = amdgpu_vm_num_entries(adev, cursor->level - 1);
- if (cursor->entry == &cursor->parent->entries[num_entries - 1])
+ if (cursor->entry == &to_amdgpu_bo_vm(cursor->parent->bo)->entries[num_entries - 1])
return false;
cursor->pfn += 1ULL << shift;
* True when the search should continue, false otherwise.
*/
static bool amdgpu_vm_pt_continue_dfs(struct amdgpu_vm_pt_cursor *start,
- struct amdgpu_vm_pt *entry)
+ struct amdgpu_vm_bo_base *entry)
{
return entry && (!start || entry != start->entry);
}
struct amdgpu_bo_list_entry *entry)
{
entry->priority = 0;
- entry->tv.bo = &vm->root.base.bo->tbo;
+ entry->tv.bo = &vm->root.bo->tbo;
/* Two for VM updates, one for TTM and one for the CS job */
entry->tv.num_shared = 4;
entry->user_pages = NULL;
for (bo_base = abo->vm_bo; bo_base; bo_base = bo_base->next) {
struct amdgpu_vm *vm = bo_base->vm;
- if (abo->tbo.base.resv == vm->root.base.bo->tbo.base.resv)
+ if (abo->tbo.base.resv == vm->root.bo->tbo.base.resv)
vm->bulk_moveable = false;
}
entries -= ats_entries;
} else {
- struct amdgpu_vm_pt *pt;
+ struct amdgpu_vm_bo_base *pt;
- pt = container_of(ancestor->vm_bo, struct amdgpu_vm_pt, base);
+ pt = ancestor->vm_bo;
ats_entries = amdgpu_vm_num_ats_entries(adev);
- if ((pt - vm->root.entries) >= ats_entries) {
+ if ((pt - to_amdgpu_bo_vm(vm->root.bo)->entries) >= ats_entries) {
ats_entries = 0;
} else {
ats_entries = entries;
bp.type = ttm_bo_type_kernel;
bp.no_wait_gpu = immediate;
- if (vm->root.base.bo)
- bp.resv = vm->root.base.bo->tbo.base.resv;
+ if (vm->root.bo)
+ bp.resv = vm->root.bo->tbo.base.resv;
r = amdgpu_bo_create_vm(adev, &bp, vmbo);
if (r)
struct amdgpu_vm_pt_cursor *cursor,
bool immediate)
{
- struct amdgpu_vm_pt *entry = cursor->entry;
+ struct amdgpu_vm_bo_base *entry = cursor->entry;
struct amdgpu_bo *pt_bo;
struct amdgpu_bo_vm *pt;
int r;
- if (entry->base.bo) {
- if (cursor->level < AMDGPU_VM_PTB)
- entry->entries =
- to_amdgpu_bo_vm(entry->base.bo)->entries;
- else
- entry->entries = NULL;
+ if (entry->bo)
return 0;
- }
r = amdgpu_vm_pt_create(adev, vm, cursor->level, immediate, &pt);
if (r)
* freeing them up in the wrong order.
*/
pt_bo = &pt->bo;
- pt_bo->parent = amdgpu_bo_ref(cursor->parent->base.bo);
- amdgpu_vm_bo_base_init(&entry->base, vm, pt_bo);
- if (cursor->level < AMDGPU_VM_PTB)
- entry->entries = pt->entries;
- else
- entry->entries = NULL;
-
+ pt_bo->parent = amdgpu_bo_ref(cursor->parent->bo);
+ amdgpu_vm_bo_base_init(entry, vm, pt_bo);
r = amdgpu_vm_clear_bo(adev, vm, pt, immediate);
if (r)
goto error_free_pt;
*
* @entry: PDE to free
*/
-static void amdgpu_vm_free_table(struct amdgpu_vm_pt *entry)
+static void amdgpu_vm_free_table(struct amdgpu_vm_bo_base *entry)
{
struct amdgpu_bo *shadow;
- if (entry->base.bo) {
- shadow = amdgpu_bo_shadowed(entry->base.bo);
- entry->base.bo->vm_bo = NULL;
- list_del(&entry->base.vm_status);
- amdgpu_bo_unref(&shadow);
- amdgpu_bo_unref(&entry->base.bo);
- }
- entry->entries = NULL;
+ if (!entry->bo)
+ return;
+ shadow = amdgpu_bo_shadowed(entry->bo);
+ entry->bo->vm_bo = NULL;
+ list_del(&entry->vm_status);
+ amdgpu_bo_unref(&shadow);
+ amdgpu_bo_unref(&entry->bo);
}
/**
struct amdgpu_vm_pt_cursor *start)
{
struct amdgpu_vm_pt_cursor cursor;
- struct amdgpu_vm_pt *entry;
+ struct amdgpu_vm_bo_base *entry;
vm->bulk_moveable = false;
*/
static int amdgpu_vm_update_pde(struct amdgpu_vm_update_params *params,
struct amdgpu_vm *vm,
- struct amdgpu_vm_pt *entry)
+ struct amdgpu_vm_bo_base *entry)
{
- struct amdgpu_vm_pt *parent = amdgpu_vm_pt_parent(entry);
- struct amdgpu_bo *bo = parent->base.bo, *pbo;
+ struct amdgpu_vm_bo_base *parent = amdgpu_vm_pt_parent(entry);
+ struct amdgpu_bo *bo = parent->bo, *pbo;
uint64_t pde, pt, flags;
unsigned level;
pbo = pbo->parent;
level += params->adev->vm_manager.root_level;
- amdgpu_gmc_get_pde_for_bo(entry->base.bo, level, &pt, &flags);
- pde = (entry - parent->entries) * 8;
+ amdgpu_gmc_get_pde_for_bo(entry->bo, level, &pt, &flags);
+ pde = (entry - to_amdgpu_bo_vm(parent->bo)->entries) * 8;
return vm->update_funcs->update(params, to_amdgpu_bo_vm(bo), pde, pt,
1, 0, flags);
}
struct amdgpu_vm *vm)
{
struct amdgpu_vm_pt_cursor cursor;
- struct amdgpu_vm_pt *entry;
+ struct amdgpu_vm_bo_base *entry;
for_each_amdgpu_vm_pt_dfs_safe(adev, vm, NULL, cursor, entry)
- if (entry->base.bo && !entry->base.moved)
- amdgpu_vm_bo_relocated(&entry->base);
+ if (entry->bo && !entry->moved)
+ amdgpu_vm_bo_relocated(entry);
}
/**
return r;
while (!list_empty(&vm->relocated)) {
- struct amdgpu_vm_pt *entry;
+ struct amdgpu_vm_bo_base *entry;
- entry = list_first_entry(&vm->relocated, struct amdgpu_vm_pt,
- base.vm_status);
- amdgpu_vm_bo_idle(&entry->base);
+ entry = list_first_entry(&vm->relocated,
+ struct amdgpu_vm_bo_base,
+ vm_status);
+ amdgpu_vm_bo_idle(entry);
r = amdgpu_vm_update_pde(¶ms, vm, entry);
if (r)
continue;
}
- pt = cursor.entry->base.bo;
+ pt = cursor.entry->bo;
if (!pt) {
/* We need all PDs and PTs for mapping something, */
if (flags & AMDGPU_PTE_VALID)
if (!amdgpu_vm_pt_ancestor(&cursor))
return -EINVAL;
- pt = cursor.entry->base.bo;
+ pt = cursor.entry->bo;
shift = parent_shift;
frag_end = max(frag_end, ALIGN(frag_start + 1,
1ULL << shift));
*/
while (cursor.pfn < frag_start) {
/* Make sure previous mapping is freed */
- if (cursor.entry->base.bo) {
+ if (cursor.entry->bo) {
params->table_freed = true;
amdgpu_vm_free_pts(adev, params->vm, &cursor);
}
if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) {
struct dma_fence *tmp = dma_fence_get_stub();
- amdgpu_bo_fence(vm->root.base.bo, vm->last_unlocked, true);
+ amdgpu_bo_fence(vm->root.bo, vm->last_unlocked, true);
swap(vm->last_unlocked, tmp);
dma_fence_put(tmp);
}
if (clear || !bo) {
mem = NULL;
- resv = vm->root.base.bo->tbo.base.resv;
+ resv = vm->root.bo->tbo.base.resv;
} else {
struct drm_gem_object *obj = &bo->tbo.base;
}
if (clear || (bo && bo->tbo.base.resv ==
- vm->root.base.bo->tbo.base.resv))
+ vm->root.bo->tbo.base.resv))
last_update = &vm->last_update;
else
last_update = &bo_va->last_pt_update;
* the evicted list so that it gets validated again on the
* next command submission.
*/
- if (bo && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) {
+ if (bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv) {
uint32_t mem_type = bo->tbo.resource->mem_type;
if (!(bo->preferred_domains &
*/
static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
{
- struct dma_resv *resv = vm->root.base.bo->tbo.base.resv;
+ struct dma_resv *resv = vm->root.bo->tbo.base.resv;
struct dma_fence *excl, **shared;
unsigned i, shared_count;
int r;
struct amdgpu_vm *vm,
struct dma_fence **fence)
{
- struct dma_resv *resv = vm->root.base.bo->tbo.base.resv;
+ struct dma_resv *resv = vm->root.bo->tbo.base.resv;
struct amdgpu_bo_va_mapping *mapping;
uint64_t init_pte_value = 0;
struct dma_fence *f = NULL;
if (mapping->flags & AMDGPU_PTE_PRT)
amdgpu_vm_prt_get(adev);
- if (bo && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv &&
+ if (bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv &&
!bo_va->base.moved) {
list_move(&bo_va->base.vm_status, &vm->moved);
}
struct amdgpu_vm_bo_base **base;
if (bo) {
- if (bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv)
+ if (bo->tbo.base.resv == vm->root.bo->tbo.base.resv)
vm->bulk_moveable = false;
for (base = &bo_va->base.bo->vm_bo; *base;
for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
struct amdgpu_vm *vm = bo_base->vm;
- if (evicted && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) {
+ if (evicted && bo->tbo.base.resv == vm->root.bo->tbo.base.resv) {
amdgpu_vm_bo_evicted(bo_base);
continue;
}
if (bo->tbo.type == ttm_bo_type_kernel)
amdgpu_vm_bo_relocated(bo_base);
- else if (bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv)
+ else if (bo->tbo.base.resv == vm->root.bo->tbo.base.resv)
amdgpu_vm_bo_moved(bo_base);
else
amdgpu_vm_bo_invalidated(bo_base);
*/
long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout)
{
- timeout = dma_resv_wait_timeout(vm->root.base.bo->tbo.base.resv, true,
+ timeout = dma_resv_wait_timeout(vm->root.bo->tbo.base.resv, true,
true, timeout);
if (timeout <= 0)
return timeout;
if (r)
goto error_unreserve;
- amdgpu_vm_bo_base_init(&vm->root.base, vm, root_bo);
+ amdgpu_vm_bo_base_init(&vm->root, vm, root_bo);
r = amdgpu_vm_clear_bo(adev, vm, root, false);
if (r)
goto error_unreserve;
- amdgpu_bo_unreserve(vm->root.base.bo);
+ amdgpu_bo_unreserve(vm->root.bo);
if (pasid) {
unsigned long flags;
return 0;
error_unreserve:
- amdgpu_bo_unreserve(vm->root.base.bo);
+ amdgpu_bo_unreserve(vm->root.bo);
error_free_root:
amdgpu_bo_unref(&root->shadow);
amdgpu_bo_unref(&root_bo);
- vm->root.base.bo = NULL;
+ vm->root.bo = NULL;
error_free_delayed:
dma_fence_put(vm->last_unlocked);
* 0 if this VM is clean
*/
static int amdgpu_vm_check_clean_reserved(struct amdgpu_device *adev,
- struct amdgpu_vm *vm)
+ struct amdgpu_vm *vm)
{
enum amdgpu_vm_level root = adev->vm_manager.root_level;
unsigned int entries = amdgpu_vm_num_entries(adev, root);
unsigned int i = 0;
- if (!(vm->root.entries))
- return 0;
-
for (i = 0; i < entries; i++) {
- if (vm->root.entries[i].base.bo)
+ if (to_amdgpu_bo_vm(vm->root.bo)->entries[i].bo)
return -EINVAL;
}
bool pte_support_ats = (adev->asic_type == CHIP_RAVEN);
int r;
- r = amdgpu_bo_reserve(vm->root.base.bo, true);
+ r = amdgpu_bo_reserve(vm->root.bo, true);
if (r)
return r;
if (pte_support_ats != vm->pte_support_ats) {
vm->pte_support_ats = pte_support_ats;
r = amdgpu_vm_clear_bo(adev, vm,
- to_amdgpu_bo_vm(vm->root.base.bo),
+ to_amdgpu_bo_vm(vm->root.bo),
false);
if (r)
goto free_idr;
if (vm->use_cpu_for_update) {
/* Sync with last SDMA update/clear before switching to CPU */
- r = amdgpu_bo_sync_wait(vm->root.base.bo,
+ r = amdgpu_bo_sync_wait(vm->root.bo,
AMDGPU_FENCE_OWNER_UNDEFINED, true);
if (r)
goto free_idr;
}
/* Free the shadow bo for compute VM */
- amdgpu_bo_unref(&to_amdgpu_bo_vm(vm->root.base.bo)->shadow);
+ amdgpu_bo_unref(&to_amdgpu_bo_vm(vm->root.bo)->shadow);
if (pasid)
vm->pasid = pasid;
spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
}
unreserve_bo:
- amdgpu_bo_unreserve(vm->root.base.bo);
+ amdgpu_bo_unreserve(vm->root.bo);
return r;
}
amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);
- root = amdgpu_bo_ref(vm->root.base.bo);
+ root = amdgpu_bo_ref(vm->root.bo);
amdgpu_bo_reserve(root, true);
if (vm->pasid) {
unsigned long flags;
amdgpu_vm_free_pts(adev, vm, NULL);
amdgpu_bo_unreserve(root);
amdgpu_bo_unref(&root);
- WARN_ON(vm->root.base.bo);
+ WARN_ON(vm->root.bo);
drm_sched_entity_destroy(&vm->immediate);
drm_sched_entity_destroy(&vm->delayed);
/* Wait vm idle to make sure the vmid set in SPM_VMID is
* not referenced anymore.
*/
- r = amdgpu_bo_reserve(fpriv->vm.root.base.bo, true);
+ r = amdgpu_bo_reserve(fpriv->vm.root.bo, true);
if (r)
return r;
if (r < 0)
return r;
- amdgpu_bo_unreserve(fpriv->vm.root.base.bo);
+ amdgpu_bo_unreserve(fpriv->vm.root.bo);
amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB_0);
break;
default:
spin_lock(&adev->vm_manager.pasid_lock);
vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
if (vm) {
- root = amdgpu_bo_ref(vm->root.base.bo);
+ root = amdgpu_bo_ref(vm->root.bo);
is_compute_context = vm->is_compute_context;
} else {
root = NULL;
/* Double check that the VM still exists */
spin_lock(&adev->vm_manager.pasid_lock);
vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
- if (vm && vm->root.base.bo != root)
+ if (vm && vm->root.bo != root)
vm = NULL;
spin_unlock(&adev->vm_manager.pasid_lock);
if (!vm)