2 * Copyright 2008 Jerome Glisse.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
25 * Jerome Glisse <glisse@freedesktop.org>
27 #include <linux/pagemap.h>
28 #include <linux/sync_file.h>
30 #include <drm/amdgpu_drm.h>
31 #include <drm/drm_syncobj.h>
33 #include "amdgpu_trace.h"
34 #include "amdgpu_gmc.h"
35 #include "amdgpu_gem.h"
37 static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser
*p
,
38 struct drm_amdgpu_cs_chunk_fence
*data
,
41 struct drm_gem_object
*gobj
;
46 gobj
= drm_gem_object_lookup(p
->filp
, data
->handle
);
50 bo
= amdgpu_bo_ref(gem_to_amdgpu_bo(gobj
));
51 p
->uf_entry
.priority
= 0;
52 p
->uf_entry
.tv
.bo
= &bo
->tbo
;
53 /* One for TTM and one for the CS job */
54 p
->uf_entry
.tv
.num_shared
= 2;
55 p
->uf_entry
.user_pages
= NULL
;
57 drm_gem_object_put_unlocked(gobj
);
59 size
= amdgpu_bo_size(bo
);
60 if (size
!= PAGE_SIZE
|| (data
->offset
+ 8) > size
) {
65 if (amdgpu_ttm_tt_get_usermm(bo
->tbo
.ttm
)) {
70 *offset
= data
->offset
;
79 static int amdgpu_cs_bo_handles_chunk(struct amdgpu_cs_parser
*p
,
80 struct drm_amdgpu_bo_list_in
*data
)
83 struct drm_amdgpu_bo_list_entry
*info
= NULL
;
85 r
= amdgpu_bo_create_list_entry_array(data
, &info
);
89 r
= amdgpu_bo_list_create(p
->adev
, p
->filp
, info
, data
->bo_number
,
104 static int amdgpu_cs_parser_init(struct amdgpu_cs_parser
*p
, union drm_amdgpu_cs
*cs
)
106 struct amdgpu_fpriv
*fpriv
= p
->filp
->driver_priv
;
107 struct amdgpu_vm
*vm
= &fpriv
->vm
;
108 uint64_t *chunk_array_user
;
109 uint64_t *chunk_array
;
110 unsigned size
, num_ibs
= 0;
111 uint32_t uf_offset
= 0;
115 if (cs
->in
.num_chunks
== 0)
118 chunk_array
= kmalloc_array(cs
->in
.num_chunks
, sizeof(uint64_t), GFP_KERNEL
);
122 p
->ctx
= amdgpu_ctx_get(fpriv
, cs
->in
.ctx_id
);
128 mutex_lock(&p
->ctx
->lock
);
130 /* skip guilty context job */
131 if (atomic_read(&p
->ctx
->guilty
) == 1) {
137 chunk_array_user
= u64_to_user_ptr(cs
->in
.chunks
);
138 if (copy_from_user(chunk_array
, chunk_array_user
,
139 sizeof(uint64_t)*cs
->in
.num_chunks
)) {
144 p
->nchunks
= cs
->in
.num_chunks
;
145 p
->chunks
= kmalloc_array(p
->nchunks
, sizeof(struct amdgpu_cs_chunk
),
152 for (i
= 0; i
< p
->nchunks
; i
++) {
153 struct drm_amdgpu_cs_chunk __user
**chunk_ptr
= NULL
;
154 struct drm_amdgpu_cs_chunk user_chunk
;
155 uint32_t __user
*cdata
;
157 chunk_ptr
= u64_to_user_ptr(chunk_array
[i
]);
158 if (copy_from_user(&user_chunk
, chunk_ptr
,
159 sizeof(struct drm_amdgpu_cs_chunk
))) {
162 goto free_partial_kdata
;
164 p
->chunks
[i
].chunk_id
= user_chunk
.chunk_id
;
165 p
->chunks
[i
].length_dw
= user_chunk
.length_dw
;
167 size
= p
->chunks
[i
].length_dw
;
168 cdata
= u64_to_user_ptr(user_chunk
.chunk_data
);
170 p
->chunks
[i
].kdata
= kvmalloc_array(size
, sizeof(uint32_t), GFP_KERNEL
);
171 if (p
->chunks
[i
].kdata
== NULL
) {
174 goto free_partial_kdata
;
176 size
*= sizeof(uint32_t);
177 if (copy_from_user(p
->chunks
[i
].kdata
, cdata
, size
)) {
179 goto free_partial_kdata
;
182 switch (p
->chunks
[i
].chunk_id
) {
183 case AMDGPU_CHUNK_ID_IB
:
187 case AMDGPU_CHUNK_ID_FENCE
:
188 size
= sizeof(struct drm_amdgpu_cs_chunk_fence
);
189 if (p
->chunks
[i
].length_dw
* sizeof(uint32_t) < size
) {
191 goto free_partial_kdata
;
194 ret
= amdgpu_cs_user_fence_chunk(p
, p
->chunks
[i
].kdata
,
197 goto free_partial_kdata
;
201 case AMDGPU_CHUNK_ID_BO_HANDLES
:
202 size
= sizeof(struct drm_amdgpu_bo_list_in
);
203 if (p
->chunks
[i
].length_dw
* sizeof(uint32_t) < size
) {
205 goto free_partial_kdata
;
208 ret
= amdgpu_cs_bo_handles_chunk(p
, p
->chunks
[i
].kdata
);
210 goto free_partial_kdata
;
214 case AMDGPU_CHUNK_ID_DEPENDENCIES
:
215 case AMDGPU_CHUNK_ID_SYNCOBJ_IN
:
216 case AMDGPU_CHUNK_ID_SYNCOBJ_OUT
:
221 goto free_partial_kdata
;
225 ret
= amdgpu_job_alloc(p
->adev
, num_ibs
, &p
->job
, vm
);
229 if (p
->ctx
->vram_lost_counter
!= p
->job
->vram_lost_counter
) {
234 if (p
->uf_entry
.tv
.bo
)
235 p
->job
->uf_addr
= uf_offset
;
238 /* Use this opportunity to fill in task info for the vm */
239 amdgpu_vm_set_task_info(vm
);
247 kvfree(p
->chunks
[i
].kdata
);
257 /* Convert microseconds to bytes. */
258 static u64
us_to_bytes(struct amdgpu_device
*adev
, s64 us
)
260 if (us
<= 0 || !adev
->mm_stats
.log2_max_MBps
)
263 /* Since accum_us is incremented by a million per second, just
264 * multiply it by the number of MB/s to get the number of bytes.
266 return us
<< adev
->mm_stats
.log2_max_MBps
;
269 static s64
bytes_to_us(struct amdgpu_device
*adev
, u64 bytes
)
271 if (!adev
->mm_stats
.log2_max_MBps
)
274 return bytes
>> adev
->mm_stats
.log2_max_MBps
;
277 /* Returns how many bytes TTM can move right now. If no bytes can be moved,
278 * it returns 0. If it returns non-zero, it's OK to move at least one buffer,
279 * which means it can go over the threshold once. If that happens, the driver
280 * will be in debt and no other buffer migrations can be done until that debt
283 * This approach allows moving a buffer of any size (it's important to allow
286 * The currency is simply time in microseconds and it increases as the clock
287 * ticks. The accumulated microseconds (us) are converted to bytes and
290 static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device
*adev
,
294 s64 time_us
, increment_us
;
295 u64 free_vram
, total_vram
, used_vram
;
297 /* Allow a maximum of 200 accumulated ms. This is basically per-IB
300 * It means that in order to get full max MBps, at least 5 IBs per
301 * second must be submitted and not more than 200ms apart from each
304 const s64 us_upper_bound
= 200000;
306 if (!adev
->mm_stats
.log2_max_MBps
) {
312 total_vram
= adev
->gmc
.real_vram_size
- atomic64_read(&adev
->vram_pin_size
);
313 used_vram
= amdgpu_vram_mgr_usage(&adev
->mman
.bdev
.man
[TTM_PL_VRAM
]);
314 free_vram
= used_vram
>= total_vram
? 0 : total_vram
- used_vram
;
316 spin_lock(&adev
->mm_stats
.lock
);
318 /* Increase the amount of accumulated us. */
319 time_us
= ktime_to_us(ktime_get());
320 increment_us
= time_us
- adev
->mm_stats
.last_update_us
;
321 adev
->mm_stats
.last_update_us
= time_us
;
322 adev
->mm_stats
.accum_us
= min(adev
->mm_stats
.accum_us
+ increment_us
,
325 /* This prevents the short period of low performance when the VRAM
326 * usage is low and the driver is in debt or doesn't have enough
327 * accumulated us to fill VRAM quickly.
329 * The situation can occur in these cases:
330 * - a lot of VRAM is freed by userspace
331 * - the presence of a big buffer causes a lot of evictions
332 * (solution: split buffers into smaller ones)
334 * If 128 MB or 1/8th of VRAM is free, start filling it now by setting
335 * accum_us to a positive number.
337 if (free_vram
>= 128 * 1024 * 1024 || free_vram
>= total_vram
/ 8) {
340 /* Be more aggresive on dGPUs. Try to fill a portion of free
343 if (!(adev
->flags
& AMD_IS_APU
))
344 min_us
= bytes_to_us(adev
, free_vram
/ 4);
346 min_us
= 0; /* Reset accum_us on APUs. */
348 adev
->mm_stats
.accum_us
= max(min_us
, adev
->mm_stats
.accum_us
);
351 /* This is set to 0 if the driver is in debt to disallow (optional)
354 *max_bytes
= us_to_bytes(adev
, adev
->mm_stats
.accum_us
);
356 /* Do the same for visible VRAM if half of it is free */
357 if (!amdgpu_gmc_vram_full_visible(&adev
->gmc
)) {
358 u64 total_vis_vram
= adev
->gmc
.visible_vram_size
;
360 amdgpu_vram_mgr_vis_usage(&adev
->mman
.bdev
.man
[TTM_PL_VRAM
]);
362 if (used_vis_vram
< total_vis_vram
) {
363 u64 free_vis_vram
= total_vis_vram
- used_vis_vram
;
364 adev
->mm_stats
.accum_us_vis
= min(adev
->mm_stats
.accum_us_vis
+
365 increment_us
, us_upper_bound
);
367 if (free_vis_vram
>= total_vis_vram
/ 2)
368 adev
->mm_stats
.accum_us_vis
=
369 max(bytes_to_us(adev
, free_vis_vram
/ 2),
370 adev
->mm_stats
.accum_us_vis
);
373 *max_vis_bytes
= us_to_bytes(adev
, adev
->mm_stats
.accum_us_vis
);
378 spin_unlock(&adev
->mm_stats
.lock
);
381 /* Report how many bytes have really been moved for the last command
382 * submission. This can result in a debt that can stop buffer migrations
385 void amdgpu_cs_report_moved_bytes(struct amdgpu_device
*adev
, u64 num_bytes
,
388 spin_lock(&adev
->mm_stats
.lock
);
389 adev
->mm_stats
.accum_us
-= bytes_to_us(adev
, num_bytes
);
390 adev
->mm_stats
.accum_us_vis
-= bytes_to_us(adev
, num_vis_bytes
);
391 spin_unlock(&adev
->mm_stats
.lock
);
394 static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser
*p
,
395 struct amdgpu_bo
*bo
)
397 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->tbo
.bdev
);
398 struct ttm_operation_ctx ctx
= {
399 .interruptible
= true,
400 .no_wait_gpu
= false,
401 .resv
= bo
->tbo
.resv
,
410 /* Don't move this buffer if we have depleted our allowance
411 * to move it. Don't move anything if the threshold is zero.
413 if (p
->bytes_moved
< p
->bytes_moved_threshold
) {
414 if (!amdgpu_gmc_vram_full_visible(&adev
->gmc
) &&
415 (bo
->flags
& AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
)) {
416 /* And don't move a CPU_ACCESS_REQUIRED BO to limited
417 * visible VRAM if we've depleted our allowance to do
420 if (p
->bytes_moved_vis
< p
->bytes_moved_vis_threshold
)
421 domain
= bo
->preferred_domains
;
423 domain
= bo
->allowed_domains
;
425 domain
= bo
->preferred_domains
;
428 domain
= bo
->allowed_domains
;
432 amdgpu_bo_placement_from_domain(bo
, domain
);
433 r
= ttm_bo_validate(&bo
->tbo
, &bo
->placement
, &ctx
);
435 p
->bytes_moved
+= ctx
.bytes_moved
;
436 if (!amdgpu_gmc_vram_full_visible(&adev
->gmc
) &&
437 amdgpu_bo_in_cpu_visible_vram(bo
))
438 p
->bytes_moved_vis
+= ctx
.bytes_moved
;
440 if (unlikely(r
== -ENOMEM
) && domain
!= bo
->allowed_domains
) {
441 domain
= bo
->allowed_domains
;
448 /* Last resort, try to evict something from the current working set */
449 static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser
*p
,
450 struct amdgpu_bo
*validated
)
452 uint32_t domain
= validated
->allowed_domains
;
453 struct ttm_operation_ctx ctx
= { true, false };
459 for (;&p
->evictable
->tv
.head
!= &p
->validated
;
460 p
->evictable
= list_prev_entry(p
->evictable
, tv
.head
)) {
462 struct amdgpu_bo_list_entry
*candidate
= p
->evictable
;
463 struct amdgpu_bo
*bo
= ttm_to_amdgpu_bo(candidate
->tv
.bo
);
464 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->tbo
.bdev
);
465 bool update_bytes_moved_vis
;
468 /* If we reached our current BO we can forget it */
472 /* We can't move pinned BOs here */
476 other
= amdgpu_mem_type_to_domain(bo
->tbo
.mem
.mem_type
);
478 /* Check if this BO is in one of the domains we need space for */
479 if (!(other
& domain
))
482 /* Check if we can move this BO somewhere else */
483 other
= bo
->allowed_domains
& ~domain
;
487 /* Good we can try to move this BO somewhere else */
488 update_bytes_moved_vis
=
489 !amdgpu_gmc_vram_full_visible(&adev
->gmc
) &&
490 amdgpu_bo_in_cpu_visible_vram(bo
);
491 amdgpu_bo_placement_from_domain(bo
, other
);
492 r
= ttm_bo_validate(&bo
->tbo
, &bo
->placement
, &ctx
);
493 p
->bytes_moved
+= ctx
.bytes_moved
;
494 if (update_bytes_moved_vis
)
495 p
->bytes_moved_vis
+= ctx
.bytes_moved
;
500 p
->evictable
= list_prev_entry(p
->evictable
, tv
.head
);
501 list_move(&candidate
->tv
.head
, &p
->validated
);
509 static int amdgpu_cs_validate(void *param
, struct amdgpu_bo
*bo
)
511 struct amdgpu_cs_parser
*p
= param
;
515 r
= amdgpu_cs_bo_validate(p
, bo
);
516 } while (r
== -ENOMEM
&& amdgpu_cs_try_evict(p
, bo
));
521 r
= amdgpu_cs_bo_validate(p
, bo
->shadow
);
526 static int amdgpu_cs_list_validate(struct amdgpu_cs_parser
*p
,
527 struct list_head
*validated
)
529 struct ttm_operation_ctx ctx
= { true, false };
530 struct amdgpu_bo_list_entry
*lobj
;
533 list_for_each_entry(lobj
, validated
, tv
.head
) {
534 struct amdgpu_bo
*bo
= ttm_to_amdgpu_bo(lobj
->tv
.bo
);
535 bool binding_userptr
= false;
536 struct mm_struct
*usermm
;
538 usermm
= amdgpu_ttm_tt_get_usermm(bo
->tbo
.ttm
);
539 if (usermm
&& usermm
!= current
->mm
)
542 /* Check if we have user pages and nobody bound the BO already */
543 if (amdgpu_ttm_tt_userptr_needs_pages(bo
->tbo
.ttm
) &&
545 amdgpu_bo_placement_from_domain(bo
,
546 AMDGPU_GEM_DOMAIN_CPU
);
547 r
= ttm_bo_validate(&bo
->tbo
, &bo
->placement
, &ctx
);
550 amdgpu_ttm_tt_set_user_pages(bo
->tbo
.ttm
,
552 binding_userptr
= true;
555 if (p
->evictable
== lobj
)
558 r
= amdgpu_cs_validate(p
, bo
);
562 if (binding_userptr
) {
563 kvfree(lobj
->user_pages
);
564 lobj
->user_pages
= NULL
;
570 static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser
*p
,
571 union drm_amdgpu_cs
*cs
)
573 struct amdgpu_fpriv
*fpriv
= p
->filp
->driver_priv
;
574 struct amdgpu_vm
*vm
= &fpriv
->vm
;
575 struct amdgpu_bo_list_entry
*e
;
576 struct list_head duplicates
;
577 struct amdgpu_bo
*gds
;
578 struct amdgpu_bo
*gws
;
579 struct amdgpu_bo
*oa
;
583 INIT_LIST_HEAD(&p
->validated
);
585 /* p->bo_list could already be assigned if AMDGPU_CHUNK_ID_BO_HANDLES is present */
586 if (cs
->in
.bo_list_handle
) {
590 r
= amdgpu_bo_list_get(fpriv
, cs
->in
.bo_list_handle
,
594 } else if (!p
->bo_list
) {
595 /* Create a empty bo_list when no handle is provided */
596 r
= amdgpu_bo_list_create(p
->adev
, p
->filp
, NULL
, 0,
602 /* One for TTM and one for the CS job */
603 amdgpu_bo_list_for_each_entry(e
, p
->bo_list
)
604 e
->tv
.num_shared
= 2;
606 amdgpu_bo_list_get_list(p
->bo_list
, &p
->validated
);
607 if (p
->bo_list
->first_userptr
!= p
->bo_list
->num_entries
)
608 p
->mn
= amdgpu_mn_get(p
->adev
, AMDGPU_MN_TYPE_GFX
);
610 INIT_LIST_HEAD(&duplicates
);
611 amdgpu_vm_get_pd_bo(&fpriv
->vm
, &p
->validated
, &p
->vm_pd
);
613 if (p
->uf_entry
.tv
.bo
&& !ttm_to_amdgpu_bo(p
->uf_entry
.tv
.bo
)->parent
)
614 list_add(&p
->uf_entry
.tv
.head
, &p
->validated
);
617 struct list_head need_pages
;
619 r
= ttm_eu_reserve_buffers(&p
->ticket
, &p
->validated
, true,
621 if (unlikely(r
!= 0)) {
622 if (r
!= -ERESTARTSYS
)
623 DRM_ERROR("ttm_eu_reserve_buffers failed.\n");
624 goto error_free_pages
;
627 INIT_LIST_HEAD(&need_pages
);
628 amdgpu_bo_list_for_each_userptr_entry(e
, p
->bo_list
) {
629 struct amdgpu_bo
*bo
= ttm_to_amdgpu_bo(e
->tv
.bo
);
631 if (amdgpu_ttm_tt_userptr_invalidated(bo
->tbo
.ttm
,
632 &e
->user_invalidated
) && e
->user_pages
) {
634 /* We acquired a page array, but somebody
635 * invalidated it. Free it and try again
637 release_pages(e
->user_pages
,
638 bo
->tbo
.ttm
->num_pages
);
639 kvfree(e
->user_pages
);
640 e
->user_pages
= NULL
;
643 if (amdgpu_ttm_tt_userptr_needs_pages(bo
->tbo
.ttm
) &&
645 list_del(&e
->tv
.head
);
646 list_add(&e
->tv
.head
, &need_pages
);
648 amdgpu_bo_unreserve(bo
);
652 if (list_empty(&need_pages
))
655 /* Unreserve everything again. */
656 ttm_eu_backoff_reservation(&p
->ticket
, &p
->validated
);
658 /* We tried too many times, just abort */
661 DRM_ERROR("deadlock in %s\n", __func__
);
662 goto error_free_pages
;
665 /* Fill the page arrays for all userptrs. */
666 list_for_each_entry(e
, &need_pages
, tv
.head
) {
667 struct ttm_tt
*ttm
= e
->tv
.bo
->ttm
;
669 e
->user_pages
= kvmalloc_array(ttm
->num_pages
,
670 sizeof(struct page
*),
671 GFP_KERNEL
| __GFP_ZERO
);
672 if (!e
->user_pages
) {
674 DRM_ERROR("calloc failure in %s\n", __func__
);
675 goto error_free_pages
;
678 r
= amdgpu_ttm_tt_get_user_pages(ttm
, e
->user_pages
);
680 DRM_ERROR("amdgpu_ttm_tt_get_user_pages failed.\n");
681 kvfree(e
->user_pages
);
682 e
->user_pages
= NULL
;
683 goto error_free_pages
;
688 list_splice(&need_pages
, &p
->validated
);
691 amdgpu_cs_get_threshold_for_moves(p
->adev
, &p
->bytes_moved_threshold
,
692 &p
->bytes_moved_vis_threshold
);
694 p
->bytes_moved_vis
= 0;
695 p
->evictable
= list_last_entry(&p
->validated
,
696 struct amdgpu_bo_list_entry
,
699 r
= amdgpu_vm_validate_pt_bos(p
->adev
, &fpriv
->vm
,
700 amdgpu_cs_validate
, p
);
702 DRM_ERROR("amdgpu_vm_validate_pt_bos() failed.\n");
706 r
= amdgpu_cs_list_validate(p
, &duplicates
);
708 DRM_ERROR("amdgpu_cs_list_validate(duplicates) failed.\n");
712 r
= amdgpu_cs_list_validate(p
, &p
->validated
);
714 DRM_ERROR("amdgpu_cs_list_validate(validated) failed.\n");
718 amdgpu_cs_report_moved_bytes(p
->adev
, p
->bytes_moved
,
721 gds
= p
->bo_list
->gds_obj
;
722 gws
= p
->bo_list
->gws_obj
;
723 oa
= p
->bo_list
->oa_obj
;
725 amdgpu_bo_list_for_each_entry(e
, p
->bo_list
) {
726 struct amdgpu_bo
*bo
= ttm_to_amdgpu_bo(e
->tv
.bo
);
728 /* Make sure we use the exclusive slot for shared BOs */
729 if (bo
->prime_shared_count
)
730 e
->tv
.num_shared
= 0;
731 e
->bo_va
= amdgpu_vm_bo_find(vm
, bo
);
735 p
->job
->gds_base
= amdgpu_bo_gpu_offset(gds
) >> PAGE_SHIFT
;
736 p
->job
->gds_size
= amdgpu_bo_size(gds
) >> PAGE_SHIFT
;
739 p
->job
->gws_base
= amdgpu_bo_gpu_offset(gws
) >> PAGE_SHIFT
;
740 p
->job
->gws_size
= amdgpu_bo_size(gws
) >> PAGE_SHIFT
;
743 p
->job
->oa_base
= amdgpu_bo_gpu_offset(oa
) >> PAGE_SHIFT
;
744 p
->job
->oa_size
= amdgpu_bo_size(oa
) >> PAGE_SHIFT
;
747 if (!r
&& p
->uf_entry
.tv
.bo
) {
748 struct amdgpu_bo
*uf
= ttm_to_amdgpu_bo(p
->uf_entry
.tv
.bo
);
750 r
= amdgpu_ttm_alloc_gart(&uf
->tbo
);
751 p
->job
->uf_addr
+= amdgpu_bo_gpu_offset(uf
);
756 ttm_eu_backoff_reservation(&p
->ticket
, &p
->validated
);
760 amdgpu_bo_list_for_each_userptr_entry(e
, p
->bo_list
) {
764 release_pages(e
->user_pages
, e
->tv
.bo
->ttm
->num_pages
);
765 kvfree(e
->user_pages
);
771 static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser
*p
)
773 struct amdgpu_bo_list_entry
*e
;
776 list_for_each_entry(e
, &p
->validated
, tv
.head
) {
777 struct amdgpu_bo
*bo
= ttm_to_amdgpu_bo(e
->tv
.bo
);
778 struct reservation_object
*resv
= bo
->tbo
.resv
;
780 r
= amdgpu_sync_resv(p
->adev
, &p
->job
->sync
, resv
, p
->filp
,
781 amdgpu_bo_explicit_sync(bo
));
790 * cs_parser_fini() - clean parser states
791 * @parser: parser structure holding parsing context.
792 * @error: error number
794 * If error is set than unvalidate buffer, otherwise just free memory
795 * used by parsing context.
797 static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser
*parser
, int error
,
802 if (error
&& backoff
)
803 ttm_eu_backoff_reservation(&parser
->ticket
,
806 for (i
= 0; i
< parser
->num_post_dep_syncobjs
; i
++)
807 drm_syncobj_put(parser
->post_dep_syncobjs
[i
]);
808 kfree(parser
->post_dep_syncobjs
);
810 dma_fence_put(parser
->fence
);
813 mutex_unlock(&parser
->ctx
->lock
);
814 amdgpu_ctx_put(parser
->ctx
);
817 amdgpu_bo_list_put(parser
->bo_list
);
819 for (i
= 0; i
< parser
->nchunks
; i
++)
820 kvfree(parser
->chunks
[i
].kdata
);
821 kfree(parser
->chunks
);
823 amdgpu_job_free(parser
->job
);
824 if (parser
->uf_entry
.tv
.bo
) {
825 struct amdgpu_bo
*uf
= ttm_to_amdgpu_bo(parser
->uf_entry
.tv
.bo
);
827 amdgpu_bo_unref(&uf
);
831 static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser
*p
)
833 struct amdgpu_ring
*ring
= to_amdgpu_ring(p
->entity
->rq
->sched
);
834 struct amdgpu_fpriv
*fpriv
= p
->filp
->driver_priv
;
835 struct amdgpu_device
*adev
= p
->adev
;
836 struct amdgpu_vm
*vm
= &fpriv
->vm
;
837 struct amdgpu_bo_list_entry
*e
;
838 struct amdgpu_bo_va
*bo_va
;
839 struct amdgpu_bo
*bo
;
842 /* Only for UVD/VCE VM emulation */
843 if (ring
->funcs
->parse_cs
|| ring
->funcs
->patch_cs_in_place
) {
846 for (i
= 0, j
= 0; i
< p
->nchunks
&& j
< p
->job
->num_ibs
; i
++) {
847 struct drm_amdgpu_cs_chunk_ib
*chunk_ib
;
848 struct amdgpu_bo_va_mapping
*m
;
849 struct amdgpu_bo
*aobj
= NULL
;
850 struct amdgpu_cs_chunk
*chunk
;
851 uint64_t offset
, va_start
;
852 struct amdgpu_ib
*ib
;
855 chunk
= &p
->chunks
[i
];
856 ib
= &p
->job
->ibs
[j
];
857 chunk_ib
= chunk
->kdata
;
859 if (chunk
->chunk_id
!= AMDGPU_CHUNK_ID_IB
)
862 va_start
= chunk_ib
->va_start
& AMDGPU_GMC_HOLE_MASK
;
863 r
= amdgpu_cs_find_mapping(p
, va_start
, &aobj
, &m
);
865 DRM_ERROR("IB va_start is invalid\n");
869 if ((va_start
+ chunk_ib
->ib_bytes
) >
870 (m
->last
+ 1) * AMDGPU_GPU_PAGE_SIZE
) {
871 DRM_ERROR("IB va_start+ib_bytes is invalid\n");
875 /* the IB should be reserved at this point */
876 r
= amdgpu_bo_kmap(aobj
, (void **)&kptr
);
881 offset
= m
->start
* AMDGPU_GPU_PAGE_SIZE
;
882 kptr
+= va_start
- offset
;
884 if (ring
->funcs
->parse_cs
) {
885 memcpy(ib
->ptr
, kptr
, chunk_ib
->ib_bytes
);
886 amdgpu_bo_kunmap(aobj
);
888 r
= amdgpu_ring_parse_cs(ring
, p
, j
);
892 ib
->ptr
= (uint32_t *)kptr
;
893 r
= amdgpu_ring_patch_cs_in_place(ring
, p
, j
);
894 amdgpu_bo_kunmap(aobj
);
904 return amdgpu_cs_sync_rings(p
);
907 r
= amdgpu_vm_clear_freed(adev
, vm
, NULL
);
911 r
= amdgpu_vm_bo_update(adev
, fpriv
->prt_va
, false);
915 r
= amdgpu_sync_fence(adev
, &p
->job
->sync
,
916 fpriv
->prt_va
->last_pt_update
, false);
920 if (amdgpu_sriov_vf(adev
)) {
923 bo_va
= fpriv
->csa_va
;
925 r
= amdgpu_vm_bo_update(adev
, bo_va
, false);
929 f
= bo_va
->last_pt_update
;
930 r
= amdgpu_sync_fence(adev
, &p
->job
->sync
, f
, false);
935 amdgpu_bo_list_for_each_entry(e
, p
->bo_list
) {
938 /* ignore duplicates */
939 bo
= ttm_to_amdgpu_bo(e
->tv
.bo
);
947 r
= amdgpu_vm_bo_update(adev
, bo_va
, false);
951 f
= bo_va
->last_pt_update
;
952 r
= amdgpu_sync_fence(adev
, &p
->job
->sync
, f
, false);
957 r
= amdgpu_vm_handle_moved(adev
, vm
);
961 r
= amdgpu_vm_update_directories(adev
, vm
);
965 r
= amdgpu_sync_fence(adev
, &p
->job
->sync
, vm
->last_update
, false);
969 p
->job
->vm_pd_addr
= amdgpu_gmc_pd_addr(vm
->root
.base
.bo
);
971 if (amdgpu_vm_debug
) {
972 /* Invalidate all BOs to test for userspace bugs */
973 amdgpu_bo_list_for_each_entry(e
, p
->bo_list
) {
974 struct amdgpu_bo
*bo
= ttm_to_amdgpu_bo(e
->tv
.bo
);
976 /* ignore duplicates */
980 amdgpu_vm_bo_invalidate(adev
, bo
, false);
984 return amdgpu_cs_sync_rings(p
);
987 static int amdgpu_cs_ib_fill(struct amdgpu_device
*adev
,
988 struct amdgpu_cs_parser
*parser
)
990 struct amdgpu_fpriv
*fpriv
= parser
->filp
->driver_priv
;
991 struct amdgpu_vm
*vm
= &fpriv
->vm
;
992 int r
, ce_preempt
= 0, de_preempt
= 0;
993 struct amdgpu_ring
*ring
;
996 for (i
= 0, j
= 0; i
< parser
->nchunks
&& j
< parser
->job
->num_ibs
; i
++) {
997 struct amdgpu_cs_chunk
*chunk
;
998 struct amdgpu_ib
*ib
;
999 struct drm_amdgpu_cs_chunk_ib
*chunk_ib
;
1000 struct drm_sched_entity
*entity
;
1002 chunk
= &parser
->chunks
[i
];
1003 ib
= &parser
->job
->ibs
[j
];
1004 chunk_ib
= (struct drm_amdgpu_cs_chunk_ib
*)chunk
->kdata
;
1006 if (chunk
->chunk_id
!= AMDGPU_CHUNK_ID_IB
)
1009 if (chunk_ib
->ip_type
== AMDGPU_HW_IP_GFX
&& amdgpu_sriov_vf(adev
)) {
1010 if (chunk_ib
->flags
& AMDGPU_IB_FLAG_PREEMPT
) {
1011 if (chunk_ib
->flags
& AMDGPU_IB_FLAG_CE
)
1017 /* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */
1018 if (ce_preempt
> 1 || de_preempt
> 1)
1022 r
= amdgpu_ctx_get_entity(parser
->ctx
, chunk_ib
->ip_type
,
1023 chunk_ib
->ip_instance
, chunk_ib
->ring
,
1028 if (chunk_ib
->flags
& AMDGPU_IB_FLAG_PREAMBLE
)
1029 parser
->job
->preamble_status
|=
1030 AMDGPU_PREAMBLE_IB_PRESENT
;
1032 if (parser
->entity
&& parser
->entity
!= entity
)
1035 parser
->entity
= entity
;
1037 ring
= to_amdgpu_ring(entity
->rq
->sched
);
1038 r
= amdgpu_ib_get(adev
, vm
, ring
->funcs
->parse_cs
?
1039 chunk_ib
->ib_bytes
: 0, ib
);
1041 DRM_ERROR("Failed to get ib !\n");
1045 ib
->gpu_addr
= chunk_ib
->va_start
;
1046 ib
->length_dw
= chunk_ib
->ib_bytes
/ 4;
1047 ib
->flags
= chunk_ib
->flags
;
1052 /* UVD & VCE fw doesn't support user fences */
1053 ring
= to_amdgpu_ring(parser
->entity
->rq
->sched
);
1054 if (parser
->job
->uf_addr
&& (
1055 ring
->funcs
->type
== AMDGPU_RING_TYPE_UVD
||
1056 ring
->funcs
->type
== AMDGPU_RING_TYPE_VCE
))
1059 return amdgpu_ctx_wait_prev_fence(parser
->ctx
, parser
->entity
);
1062 static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser
*p
,
1063 struct amdgpu_cs_chunk
*chunk
)
1065 struct amdgpu_fpriv
*fpriv
= p
->filp
->driver_priv
;
1068 struct drm_amdgpu_cs_chunk_dep
*deps
;
1070 deps
= (struct drm_amdgpu_cs_chunk_dep
*)chunk
->kdata
;
1071 num_deps
= chunk
->length_dw
* 4 /
1072 sizeof(struct drm_amdgpu_cs_chunk_dep
);
1074 for (i
= 0; i
< num_deps
; ++i
) {
1075 struct amdgpu_ctx
*ctx
;
1076 struct drm_sched_entity
*entity
;
1077 struct dma_fence
*fence
;
1079 ctx
= amdgpu_ctx_get(fpriv
, deps
[i
].ctx_id
);
1083 r
= amdgpu_ctx_get_entity(ctx
, deps
[i
].ip_type
,
1084 deps
[i
].ip_instance
,
1085 deps
[i
].ring
, &entity
);
1087 amdgpu_ctx_put(ctx
);
1091 fence
= amdgpu_ctx_get_fence(ctx
, entity
,
1093 if (IS_ERR(fence
)) {
1095 amdgpu_ctx_put(ctx
);
1098 r
= amdgpu_sync_fence(p
->adev
, &p
->job
->sync
, fence
,
1100 dma_fence_put(fence
);
1101 amdgpu_ctx_put(ctx
);
1109 static int amdgpu_syncobj_lookup_and_add_to_sync(struct amdgpu_cs_parser
*p
,
1113 struct dma_fence
*fence
;
1114 r
= drm_syncobj_find_fence(p
->filp
, handle
, 0, 0, &fence
);
1118 r
= amdgpu_sync_fence(p
->adev
, &p
->job
->sync
, fence
, true);
1119 dma_fence_put(fence
);
1124 static int amdgpu_cs_process_syncobj_in_dep(struct amdgpu_cs_parser
*p
,
1125 struct amdgpu_cs_chunk
*chunk
)
1129 struct drm_amdgpu_cs_chunk_sem
*deps
;
1131 deps
= (struct drm_amdgpu_cs_chunk_sem
*)chunk
->kdata
;
1132 num_deps
= chunk
->length_dw
* 4 /
1133 sizeof(struct drm_amdgpu_cs_chunk_sem
);
1135 for (i
= 0; i
< num_deps
; ++i
) {
1136 r
= amdgpu_syncobj_lookup_and_add_to_sync(p
, deps
[i
].handle
);
1143 static int amdgpu_cs_process_syncobj_out_dep(struct amdgpu_cs_parser
*p
,
1144 struct amdgpu_cs_chunk
*chunk
)
1148 struct drm_amdgpu_cs_chunk_sem
*deps
;
1149 deps
= (struct drm_amdgpu_cs_chunk_sem
*)chunk
->kdata
;
1150 num_deps
= chunk
->length_dw
* 4 /
1151 sizeof(struct drm_amdgpu_cs_chunk_sem
);
1153 p
->post_dep_syncobjs
= kmalloc_array(num_deps
,
1154 sizeof(struct drm_syncobj
*),
1156 p
->num_post_dep_syncobjs
= 0;
1158 if (!p
->post_dep_syncobjs
)
1161 for (i
= 0; i
< num_deps
; ++i
) {
1162 p
->post_dep_syncobjs
[i
] = drm_syncobj_find(p
->filp
, deps
[i
].handle
);
1163 if (!p
->post_dep_syncobjs
[i
])
1165 p
->num_post_dep_syncobjs
++;
1170 static int amdgpu_cs_dependencies(struct amdgpu_device
*adev
,
1171 struct amdgpu_cs_parser
*p
)
1175 for (i
= 0; i
< p
->nchunks
; ++i
) {
1176 struct amdgpu_cs_chunk
*chunk
;
1178 chunk
= &p
->chunks
[i
];
1180 if (chunk
->chunk_id
== AMDGPU_CHUNK_ID_DEPENDENCIES
) {
1181 r
= amdgpu_cs_process_fence_dep(p
, chunk
);
1184 } else if (chunk
->chunk_id
== AMDGPU_CHUNK_ID_SYNCOBJ_IN
) {
1185 r
= amdgpu_cs_process_syncobj_in_dep(p
, chunk
);
1188 } else if (chunk
->chunk_id
== AMDGPU_CHUNK_ID_SYNCOBJ_OUT
) {
1189 r
= amdgpu_cs_process_syncobj_out_dep(p
, chunk
);
1198 static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser
*p
)
1202 for (i
= 0; i
< p
->num_post_dep_syncobjs
; ++i
)
1203 drm_syncobj_replace_fence(p
->post_dep_syncobjs
[i
], p
->fence
);
1206 static int amdgpu_cs_submit(struct amdgpu_cs_parser
*p
,
1207 union drm_amdgpu_cs
*cs
)
1209 struct amdgpu_fpriv
*fpriv
= p
->filp
->driver_priv
;
1210 struct drm_sched_entity
*entity
= p
->entity
;
1211 enum drm_sched_priority priority
;
1212 struct amdgpu_ring
*ring
;
1213 struct amdgpu_bo_list_entry
*e
;
1214 struct amdgpu_job
*job
;
1222 r
= drm_sched_job_init(&job
->base
, entity
, p
->filp
);
1226 /* No memory allocation is allowed while holding the mn lock */
1227 amdgpu_mn_lock(p
->mn
);
1228 amdgpu_bo_list_for_each_userptr_entry(e
, p
->bo_list
) {
1229 struct amdgpu_bo
*bo
= ttm_to_amdgpu_bo(e
->tv
.bo
);
1231 if (amdgpu_ttm_tt_userptr_needs_pages(bo
->tbo
.ttm
)) {
1237 job
->owner
= p
->filp
;
1238 p
->fence
= dma_fence_get(&job
->base
.s_fence
->finished
);
1240 amdgpu_ctx_add_fence(p
->ctx
, entity
, p
->fence
, &seq
);
1241 amdgpu_cs_post_dependencies(p
);
1243 if ((job
->preamble_status
& AMDGPU_PREAMBLE_IB_PRESENT
) &&
1244 !p
->ctx
->preamble_presented
) {
1245 job
->preamble_status
|= AMDGPU_PREAMBLE_IB_PRESENT_FIRST
;
1246 p
->ctx
->preamble_presented
= true;
1249 cs
->out
.handle
= seq
;
1250 job
->uf_sequence
= seq
;
1252 amdgpu_job_free_resources(job
);
1254 trace_amdgpu_cs_ioctl(job
);
1255 amdgpu_vm_bo_trace_cs(&fpriv
->vm
, &p
->ticket
);
1256 priority
= job
->base
.s_priority
;
1257 drm_sched_entity_push_job(&job
->base
, entity
);
1259 ring
= to_amdgpu_ring(entity
->rq
->sched
);
1260 amdgpu_ring_priority_get(ring
, priority
);
1262 amdgpu_vm_move_to_lru_tail(p
->adev
, &fpriv
->vm
);
1264 ttm_eu_fence_buffer_objects(&p
->ticket
, &p
->validated
, p
->fence
);
1265 amdgpu_mn_unlock(p
->mn
);
1270 drm_sched_job_cleanup(&job
->base
);
1271 amdgpu_mn_unlock(p
->mn
);
1274 amdgpu_job_free(job
);
1278 int amdgpu_cs_ioctl(struct drm_device
*dev
, void *data
, struct drm_file
*filp
)
1280 struct amdgpu_device
*adev
= dev
->dev_private
;
1281 union drm_amdgpu_cs
*cs
= data
;
1282 struct amdgpu_cs_parser parser
= {};
1283 bool reserved_buffers
= false;
1286 if (!adev
->accel_working
)
1292 r
= amdgpu_cs_parser_init(&parser
, data
);
1294 DRM_ERROR("Failed to initialize parser %d!\n", r
);
1298 r
= amdgpu_cs_ib_fill(adev
, &parser
);
1302 r
= amdgpu_cs_dependencies(adev
, &parser
);
1304 DRM_ERROR("Failed in the dependencies handling %d!\n", r
);
1308 r
= amdgpu_cs_parser_bos(&parser
, data
);
1311 DRM_ERROR("Not enough memory for command submission!\n");
1312 else if (r
!= -ERESTARTSYS
)
1313 DRM_ERROR("Failed to process the buffer list %d!\n", r
);
1317 reserved_buffers
= true;
1319 for (i
= 0; i
< parser
.job
->num_ibs
; i
++)
1320 trace_amdgpu_cs(&parser
, i
);
1322 r
= amdgpu_cs_vm_handling(&parser
);
1326 r
= amdgpu_cs_submit(&parser
, cs
);
1329 amdgpu_cs_parser_fini(&parser
, r
, reserved_buffers
);
1334 * amdgpu_cs_wait_ioctl - wait for a command submission to finish
1337 * @data: data from userspace
1338 * @filp: file private
1340 * Wait for the command submission identified by handle to finish.
1342 int amdgpu_cs_wait_ioctl(struct drm_device
*dev
, void *data
,
1343 struct drm_file
*filp
)
1345 union drm_amdgpu_wait_cs
*wait
= data
;
1346 unsigned long timeout
= amdgpu_gem_timeout(wait
->in
.timeout
);
1347 struct drm_sched_entity
*entity
;
1348 struct amdgpu_ctx
*ctx
;
1349 struct dma_fence
*fence
;
1352 ctx
= amdgpu_ctx_get(filp
->driver_priv
, wait
->in
.ctx_id
);
1356 r
= amdgpu_ctx_get_entity(ctx
, wait
->in
.ip_type
, wait
->in
.ip_instance
,
1357 wait
->in
.ring
, &entity
);
1359 amdgpu_ctx_put(ctx
);
1363 fence
= amdgpu_ctx_get_fence(ctx
, entity
, wait
->in
.handle
);
1367 r
= dma_fence_wait_timeout(fence
, true, timeout
);
1368 if (r
> 0 && fence
->error
)
1370 dma_fence_put(fence
);
1374 amdgpu_ctx_put(ctx
);
1378 memset(wait
, 0, sizeof(*wait
));
1379 wait
->out
.status
= (r
== 0);
1385 * amdgpu_cs_get_fence - helper to get fence from drm_amdgpu_fence
1387 * @adev: amdgpu device
1388 * @filp: file private
1389 * @user: drm_amdgpu_fence copied from user space
1391 static struct dma_fence
*amdgpu_cs_get_fence(struct amdgpu_device
*adev
,
1392 struct drm_file
*filp
,
1393 struct drm_amdgpu_fence
*user
)
1395 struct drm_sched_entity
*entity
;
1396 struct amdgpu_ctx
*ctx
;
1397 struct dma_fence
*fence
;
1400 ctx
= amdgpu_ctx_get(filp
->driver_priv
, user
->ctx_id
);
1402 return ERR_PTR(-EINVAL
);
1404 r
= amdgpu_ctx_get_entity(ctx
, user
->ip_type
, user
->ip_instance
,
1405 user
->ring
, &entity
);
1407 amdgpu_ctx_put(ctx
);
1411 fence
= amdgpu_ctx_get_fence(ctx
, entity
, user
->seq_no
);
1412 amdgpu_ctx_put(ctx
);
1417 int amdgpu_cs_fence_to_handle_ioctl(struct drm_device
*dev
, void *data
,
1418 struct drm_file
*filp
)
1420 struct amdgpu_device
*adev
= dev
->dev_private
;
1421 union drm_amdgpu_fence_to_handle
*info
= data
;
1422 struct dma_fence
*fence
;
1423 struct drm_syncobj
*syncobj
;
1424 struct sync_file
*sync_file
;
1427 fence
= amdgpu_cs_get_fence(adev
, filp
, &info
->in
.fence
);
1429 return PTR_ERR(fence
);
1431 switch (info
->in
.what
) {
1432 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ
:
1433 r
= drm_syncobj_create(&syncobj
, 0, fence
);
1434 dma_fence_put(fence
);
1437 r
= drm_syncobj_get_handle(filp
, syncobj
, &info
->out
.handle
);
1438 drm_syncobj_put(syncobj
);
1441 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD
:
1442 r
= drm_syncobj_create(&syncobj
, 0, fence
);
1443 dma_fence_put(fence
);
1446 r
= drm_syncobj_get_fd(syncobj
, (int*)&info
->out
.handle
);
1447 drm_syncobj_put(syncobj
);
1450 case AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD
:
1451 fd
= get_unused_fd_flags(O_CLOEXEC
);
1453 dma_fence_put(fence
);
1457 sync_file
= sync_file_create(fence
);
1458 dma_fence_put(fence
);
1464 fd_install(fd
, sync_file
->file
);
1465 info
->out
.handle
= fd
;
1474 * amdgpu_cs_wait_all_fence - wait on all fences to signal
1476 * @adev: amdgpu device
1477 * @filp: file private
1478 * @wait: wait parameters
1479 * @fences: array of drm_amdgpu_fence
1481 static int amdgpu_cs_wait_all_fences(struct amdgpu_device
*adev
,
1482 struct drm_file
*filp
,
1483 union drm_amdgpu_wait_fences
*wait
,
1484 struct drm_amdgpu_fence
*fences
)
1486 uint32_t fence_count
= wait
->in
.fence_count
;
1490 for (i
= 0; i
< fence_count
; i
++) {
1491 struct dma_fence
*fence
;
1492 unsigned long timeout
= amdgpu_gem_timeout(wait
->in
.timeout_ns
);
1494 fence
= amdgpu_cs_get_fence(adev
, filp
, &fences
[i
]);
1496 return PTR_ERR(fence
);
1500 r
= dma_fence_wait_timeout(fence
, true, timeout
);
1501 dma_fence_put(fence
);
1509 return fence
->error
;
1512 memset(wait
, 0, sizeof(*wait
));
1513 wait
->out
.status
= (r
> 0);
1519 * amdgpu_cs_wait_any_fence - wait on any fence to signal
1521 * @adev: amdgpu device
1522 * @filp: file private
1523 * @wait: wait parameters
1524 * @fences: array of drm_amdgpu_fence
1526 static int amdgpu_cs_wait_any_fence(struct amdgpu_device
*adev
,
1527 struct drm_file
*filp
,
1528 union drm_amdgpu_wait_fences
*wait
,
1529 struct drm_amdgpu_fence
*fences
)
1531 unsigned long timeout
= amdgpu_gem_timeout(wait
->in
.timeout_ns
);
1532 uint32_t fence_count
= wait
->in
.fence_count
;
1533 uint32_t first
= ~0;
1534 struct dma_fence
**array
;
1538 /* Prepare the fence array */
1539 array
= kcalloc(fence_count
, sizeof(struct dma_fence
*), GFP_KERNEL
);
1544 for (i
= 0; i
< fence_count
; i
++) {
1545 struct dma_fence
*fence
;
1547 fence
= amdgpu_cs_get_fence(adev
, filp
, &fences
[i
]);
1548 if (IS_ERR(fence
)) {
1550 goto err_free_fence_array
;
1553 } else { /* NULL, the fence has been already signaled */
1560 r
= dma_fence_wait_any_timeout(array
, fence_count
, true, timeout
,
1563 goto err_free_fence_array
;
1566 memset(wait
, 0, sizeof(*wait
));
1567 wait
->out
.status
= (r
> 0);
1568 wait
->out
.first_signaled
= first
;
1570 if (first
< fence_count
&& array
[first
])
1571 r
= array
[first
]->error
;
1575 err_free_fence_array
:
1576 for (i
= 0; i
< fence_count
; i
++)
1577 dma_fence_put(array
[i
]);
1584 * amdgpu_cs_wait_fences_ioctl - wait for multiple command submissions to finish
1587 * @data: data from userspace
1588 * @filp: file private
1590 int amdgpu_cs_wait_fences_ioctl(struct drm_device
*dev
, void *data
,
1591 struct drm_file
*filp
)
1593 struct amdgpu_device
*adev
= dev
->dev_private
;
1594 union drm_amdgpu_wait_fences
*wait
= data
;
1595 uint32_t fence_count
= wait
->in
.fence_count
;
1596 struct drm_amdgpu_fence
*fences_user
;
1597 struct drm_amdgpu_fence
*fences
;
1600 /* Get the fences from userspace */
1601 fences
= kmalloc_array(fence_count
, sizeof(struct drm_amdgpu_fence
),
1606 fences_user
= u64_to_user_ptr(wait
->in
.fences
);
1607 if (copy_from_user(fences
, fences_user
,
1608 sizeof(struct drm_amdgpu_fence
) * fence_count
)) {
1610 goto err_free_fences
;
1613 if (wait
->in
.wait_all
)
1614 r
= amdgpu_cs_wait_all_fences(adev
, filp
, wait
, fences
);
1616 r
= amdgpu_cs_wait_any_fence(adev
, filp
, wait
, fences
);
1625 * amdgpu_cs_find_bo_va - find bo_va for VM address
1627 * @parser: command submission parser context
1629 * @bo: resulting BO of the mapping found
1631 * Search the buffer objects in the command submission context for a certain
1632 * virtual memory address. Returns allocation structure when found, NULL
1635 int amdgpu_cs_find_mapping(struct amdgpu_cs_parser
*parser
,
1636 uint64_t addr
, struct amdgpu_bo
**bo
,
1637 struct amdgpu_bo_va_mapping
**map
)
1639 struct amdgpu_fpriv
*fpriv
= parser
->filp
->driver_priv
;
1640 struct ttm_operation_ctx ctx
= { false, false };
1641 struct amdgpu_vm
*vm
= &fpriv
->vm
;
1642 struct amdgpu_bo_va_mapping
*mapping
;
1645 addr
/= AMDGPU_GPU_PAGE_SIZE
;
1647 mapping
= amdgpu_vm_bo_lookup_mapping(vm
, addr
);
1648 if (!mapping
|| !mapping
->bo_va
|| !mapping
->bo_va
->base
.bo
)
1651 *bo
= mapping
->bo_va
->base
.bo
;
1654 /* Double check that the BO is reserved by this CS */
1655 if (READ_ONCE((*bo
)->tbo
.resv
->lock
.ctx
) != &parser
->ticket
)
1658 if (!((*bo
)->flags
& AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS
)) {
1659 (*bo
)->flags
|= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS
;
1660 amdgpu_bo_placement_from_domain(*bo
, (*bo
)->allowed_domains
);
1661 r
= ttm_bo_validate(&(*bo
)->tbo
, &(*bo
)->placement
, &ctx
);
1666 return amdgpu_ttm_alloc_gart(&(*bo
)->tbo
);