2 * Copyright 2008 Jerome Glisse.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
25 * Jerome Glisse <glisse@freedesktop.org>
28 #include <linux/file.h>
29 #include <linux/pagemap.h>
30 #include <linux/sync_file.h>
32 #include <drm/amdgpu_drm.h>
33 #include <drm/drm_syncobj.h>
35 #include "amdgpu_trace.h"
36 #include "amdgpu_gmc.h"
37 #include "amdgpu_gem.h"
39 static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser
*p
,
40 struct drm_amdgpu_cs_chunk_fence
*data
,
43 struct drm_gem_object
*gobj
;
48 gobj
= drm_gem_object_lookup(p
->filp
, data
->handle
);
52 bo
= amdgpu_bo_ref(gem_to_amdgpu_bo(gobj
));
53 p
->uf_entry
.priority
= 0;
54 p
->uf_entry
.tv
.bo
= &bo
->tbo
;
55 /* One for TTM and one for the CS job */
56 p
->uf_entry
.tv
.num_shared
= 2;
58 drm_gem_object_put_unlocked(gobj
);
60 size
= amdgpu_bo_size(bo
);
61 if (size
!= PAGE_SIZE
|| (data
->offset
+ 8) > size
) {
66 if (amdgpu_ttm_tt_get_usermm(bo
->tbo
.ttm
)) {
71 *offset
= data
->offset
;
80 static int amdgpu_cs_bo_handles_chunk(struct amdgpu_cs_parser
*p
,
81 struct drm_amdgpu_bo_list_in
*data
)
84 struct drm_amdgpu_bo_list_entry
*info
= NULL
;
86 r
= amdgpu_bo_create_list_entry_array(data
, &info
);
90 r
= amdgpu_bo_list_create(p
->adev
, p
->filp
, info
, data
->bo_number
,
105 static int amdgpu_cs_parser_init(struct amdgpu_cs_parser
*p
, union drm_amdgpu_cs
*cs
)
107 struct amdgpu_fpriv
*fpriv
= p
->filp
->driver_priv
;
108 struct amdgpu_vm
*vm
= &fpriv
->vm
;
109 uint64_t *chunk_array_user
;
110 uint64_t *chunk_array
;
111 unsigned size
, num_ibs
= 0;
112 uint32_t uf_offset
= 0;
116 if (cs
->in
.num_chunks
== 0)
119 chunk_array
= kmalloc_array(cs
->in
.num_chunks
, sizeof(uint64_t), GFP_KERNEL
);
123 p
->ctx
= amdgpu_ctx_get(fpriv
, cs
->in
.ctx_id
);
129 mutex_lock(&p
->ctx
->lock
);
131 /* skip guilty context job */
132 if (atomic_read(&p
->ctx
->guilty
) == 1) {
138 chunk_array_user
= u64_to_user_ptr(cs
->in
.chunks
);
139 if (copy_from_user(chunk_array
, chunk_array_user
,
140 sizeof(uint64_t)*cs
->in
.num_chunks
)) {
145 p
->nchunks
= cs
->in
.num_chunks
;
146 p
->chunks
= kmalloc_array(p
->nchunks
, sizeof(struct amdgpu_cs_chunk
),
153 for (i
= 0; i
< p
->nchunks
; i
++) {
154 struct drm_amdgpu_cs_chunk __user
**chunk_ptr
= NULL
;
155 struct drm_amdgpu_cs_chunk user_chunk
;
156 uint32_t __user
*cdata
;
158 chunk_ptr
= u64_to_user_ptr(chunk_array
[i
]);
159 if (copy_from_user(&user_chunk
, chunk_ptr
,
160 sizeof(struct drm_amdgpu_cs_chunk
))) {
163 goto free_partial_kdata
;
165 p
->chunks
[i
].chunk_id
= user_chunk
.chunk_id
;
166 p
->chunks
[i
].length_dw
= user_chunk
.length_dw
;
168 size
= p
->chunks
[i
].length_dw
;
169 cdata
= u64_to_user_ptr(user_chunk
.chunk_data
);
171 p
->chunks
[i
].kdata
= kvmalloc_array(size
, sizeof(uint32_t), GFP_KERNEL
);
172 if (p
->chunks
[i
].kdata
== NULL
) {
175 goto free_partial_kdata
;
177 size
*= sizeof(uint32_t);
178 if (copy_from_user(p
->chunks
[i
].kdata
, cdata
, size
)) {
180 goto free_partial_kdata
;
183 switch (p
->chunks
[i
].chunk_id
) {
184 case AMDGPU_CHUNK_ID_IB
:
188 case AMDGPU_CHUNK_ID_FENCE
:
189 size
= sizeof(struct drm_amdgpu_cs_chunk_fence
);
190 if (p
->chunks
[i
].length_dw
* sizeof(uint32_t) < size
) {
192 goto free_partial_kdata
;
195 ret
= amdgpu_cs_user_fence_chunk(p
, p
->chunks
[i
].kdata
,
198 goto free_partial_kdata
;
202 case AMDGPU_CHUNK_ID_BO_HANDLES
:
203 size
= sizeof(struct drm_amdgpu_bo_list_in
);
204 if (p
->chunks
[i
].length_dw
* sizeof(uint32_t) < size
) {
206 goto free_partial_kdata
;
209 ret
= amdgpu_cs_bo_handles_chunk(p
, p
->chunks
[i
].kdata
);
211 goto free_partial_kdata
;
215 case AMDGPU_CHUNK_ID_DEPENDENCIES
:
216 case AMDGPU_CHUNK_ID_SYNCOBJ_IN
:
217 case AMDGPU_CHUNK_ID_SYNCOBJ_OUT
:
218 case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES
:
219 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT
:
220 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL
:
225 goto free_partial_kdata
;
229 ret
= amdgpu_job_alloc(p
->adev
, num_ibs
, &p
->job
, vm
);
233 if (p
->ctx
->vram_lost_counter
!= p
->job
->vram_lost_counter
) {
238 if (p
->uf_entry
.tv
.bo
)
239 p
->job
->uf_addr
= uf_offset
;
242 /* Use this opportunity to fill in task info for the vm */
243 amdgpu_vm_set_task_info(vm
);
251 kvfree(p
->chunks
[i
].kdata
);
261 /* Convert microseconds to bytes. */
262 static u64
us_to_bytes(struct amdgpu_device
*adev
, s64 us
)
264 if (us
<= 0 || !adev
->mm_stats
.log2_max_MBps
)
267 /* Since accum_us is incremented by a million per second, just
268 * multiply it by the number of MB/s to get the number of bytes.
270 return us
<< adev
->mm_stats
.log2_max_MBps
;
273 static s64
bytes_to_us(struct amdgpu_device
*adev
, u64 bytes
)
275 if (!adev
->mm_stats
.log2_max_MBps
)
278 return bytes
>> adev
->mm_stats
.log2_max_MBps
;
281 /* Returns how many bytes TTM can move right now. If no bytes can be moved,
282 * it returns 0. If it returns non-zero, it's OK to move at least one buffer,
283 * which means it can go over the threshold once. If that happens, the driver
284 * will be in debt and no other buffer migrations can be done until that debt
287 * This approach allows moving a buffer of any size (it's important to allow
290 * The currency is simply time in microseconds and it increases as the clock
291 * ticks. The accumulated microseconds (us) are converted to bytes and
294 static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device
*adev
,
298 s64 time_us
, increment_us
;
299 u64 free_vram
, total_vram
, used_vram
;
301 /* Allow a maximum of 200 accumulated ms. This is basically per-IB
304 * It means that in order to get full max MBps, at least 5 IBs per
305 * second must be submitted and not more than 200ms apart from each
308 const s64 us_upper_bound
= 200000;
310 if (!adev
->mm_stats
.log2_max_MBps
) {
316 total_vram
= adev
->gmc
.real_vram_size
- atomic64_read(&adev
->vram_pin_size
);
317 used_vram
= amdgpu_vram_mgr_usage(&adev
->mman
.bdev
.man
[TTM_PL_VRAM
]);
318 free_vram
= used_vram
>= total_vram
? 0 : total_vram
- used_vram
;
320 spin_lock(&adev
->mm_stats
.lock
);
322 /* Increase the amount of accumulated us. */
323 time_us
= ktime_to_us(ktime_get());
324 increment_us
= time_us
- adev
->mm_stats
.last_update_us
;
325 adev
->mm_stats
.last_update_us
= time_us
;
326 adev
->mm_stats
.accum_us
= min(adev
->mm_stats
.accum_us
+ increment_us
,
329 /* This prevents the short period of low performance when the VRAM
330 * usage is low and the driver is in debt or doesn't have enough
331 * accumulated us to fill VRAM quickly.
333 * The situation can occur in these cases:
334 * - a lot of VRAM is freed by userspace
335 * - the presence of a big buffer causes a lot of evictions
336 * (solution: split buffers into smaller ones)
338 * If 128 MB or 1/8th of VRAM is free, start filling it now by setting
339 * accum_us to a positive number.
341 if (free_vram
>= 128 * 1024 * 1024 || free_vram
>= total_vram
/ 8) {
344 /* Be more aggresive on dGPUs. Try to fill a portion of free
347 if (!(adev
->flags
& AMD_IS_APU
))
348 min_us
= bytes_to_us(adev
, free_vram
/ 4);
350 min_us
= 0; /* Reset accum_us on APUs. */
352 adev
->mm_stats
.accum_us
= max(min_us
, adev
->mm_stats
.accum_us
);
355 /* This is set to 0 if the driver is in debt to disallow (optional)
358 *max_bytes
= us_to_bytes(adev
, adev
->mm_stats
.accum_us
);
360 /* Do the same for visible VRAM if half of it is free */
361 if (!amdgpu_gmc_vram_full_visible(&adev
->gmc
)) {
362 u64 total_vis_vram
= adev
->gmc
.visible_vram_size
;
364 amdgpu_vram_mgr_vis_usage(&adev
->mman
.bdev
.man
[TTM_PL_VRAM
]);
366 if (used_vis_vram
< total_vis_vram
) {
367 u64 free_vis_vram
= total_vis_vram
- used_vis_vram
;
368 adev
->mm_stats
.accum_us_vis
= min(adev
->mm_stats
.accum_us_vis
+
369 increment_us
, us_upper_bound
);
371 if (free_vis_vram
>= total_vis_vram
/ 2)
372 adev
->mm_stats
.accum_us_vis
=
373 max(bytes_to_us(adev
, free_vis_vram
/ 2),
374 adev
->mm_stats
.accum_us_vis
);
377 *max_vis_bytes
= us_to_bytes(adev
, adev
->mm_stats
.accum_us_vis
);
382 spin_unlock(&adev
->mm_stats
.lock
);
385 /* Report how many bytes have really been moved for the last command
386 * submission. This can result in a debt that can stop buffer migrations
389 void amdgpu_cs_report_moved_bytes(struct amdgpu_device
*adev
, u64 num_bytes
,
392 spin_lock(&adev
->mm_stats
.lock
);
393 adev
->mm_stats
.accum_us
-= bytes_to_us(adev
, num_bytes
);
394 adev
->mm_stats
.accum_us_vis
-= bytes_to_us(adev
, num_vis_bytes
);
395 spin_unlock(&adev
->mm_stats
.lock
);
398 static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser
*p
,
399 struct amdgpu_bo
*bo
)
401 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->tbo
.bdev
);
402 struct ttm_operation_ctx ctx
= {
403 .interruptible
= true,
404 .no_wait_gpu
= false,
405 .resv
= bo
->tbo
.resv
,
414 /* Don't move this buffer if we have depleted our allowance
415 * to move it. Don't move anything if the threshold is zero.
417 if (p
->bytes_moved
< p
->bytes_moved_threshold
) {
418 if (!amdgpu_gmc_vram_full_visible(&adev
->gmc
) &&
419 (bo
->flags
& AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
)) {
420 /* And don't move a CPU_ACCESS_REQUIRED BO to limited
421 * visible VRAM if we've depleted our allowance to do
424 if (p
->bytes_moved_vis
< p
->bytes_moved_vis_threshold
)
425 domain
= bo
->preferred_domains
;
427 domain
= bo
->allowed_domains
;
429 domain
= bo
->preferred_domains
;
432 domain
= bo
->allowed_domains
;
436 amdgpu_bo_placement_from_domain(bo
, domain
);
437 r
= ttm_bo_validate(&bo
->tbo
, &bo
->placement
, &ctx
);
439 p
->bytes_moved
+= ctx
.bytes_moved
;
440 if (!amdgpu_gmc_vram_full_visible(&adev
->gmc
) &&
441 amdgpu_bo_in_cpu_visible_vram(bo
))
442 p
->bytes_moved_vis
+= ctx
.bytes_moved
;
444 if (unlikely(r
== -ENOMEM
) && domain
!= bo
->allowed_domains
) {
445 domain
= bo
->allowed_domains
;
452 /* Last resort, try to evict something from the current working set */
453 static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser
*p
,
454 struct amdgpu_bo
*validated
)
456 uint32_t domain
= validated
->allowed_domains
;
457 struct ttm_operation_ctx ctx
= { true, false };
463 for (;&p
->evictable
->tv
.head
!= &p
->validated
;
464 p
->evictable
= list_prev_entry(p
->evictable
, tv
.head
)) {
466 struct amdgpu_bo_list_entry
*candidate
= p
->evictable
;
467 struct amdgpu_bo
*bo
= ttm_to_amdgpu_bo(candidate
->tv
.bo
);
468 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->tbo
.bdev
);
469 bool update_bytes_moved_vis
;
472 /* If we reached our current BO we can forget it */
476 /* We can't move pinned BOs here */
480 other
= amdgpu_mem_type_to_domain(bo
->tbo
.mem
.mem_type
);
482 /* Check if this BO is in one of the domains we need space for */
483 if (!(other
& domain
))
486 /* Check if we can move this BO somewhere else */
487 other
= bo
->allowed_domains
& ~domain
;
491 /* Good we can try to move this BO somewhere else */
492 update_bytes_moved_vis
=
493 !amdgpu_gmc_vram_full_visible(&adev
->gmc
) &&
494 amdgpu_bo_in_cpu_visible_vram(bo
);
495 amdgpu_bo_placement_from_domain(bo
, other
);
496 r
= ttm_bo_validate(&bo
->tbo
, &bo
->placement
, &ctx
);
497 p
->bytes_moved
+= ctx
.bytes_moved
;
498 if (update_bytes_moved_vis
)
499 p
->bytes_moved_vis
+= ctx
.bytes_moved
;
504 p
->evictable
= list_prev_entry(p
->evictable
, tv
.head
);
505 list_move(&candidate
->tv
.head
, &p
->validated
);
513 static int amdgpu_cs_validate(void *param
, struct amdgpu_bo
*bo
)
515 struct amdgpu_cs_parser
*p
= param
;
519 r
= amdgpu_cs_bo_validate(p
, bo
);
520 } while (r
== -ENOMEM
&& amdgpu_cs_try_evict(p
, bo
));
525 r
= amdgpu_cs_bo_validate(p
, bo
->shadow
);
530 static int amdgpu_cs_list_validate(struct amdgpu_cs_parser
*p
,
531 struct list_head
*validated
)
533 struct ttm_operation_ctx ctx
= { true, false };
534 struct amdgpu_bo_list_entry
*lobj
;
537 list_for_each_entry(lobj
, validated
, tv
.head
) {
538 struct amdgpu_bo
*bo
= ttm_to_amdgpu_bo(lobj
->tv
.bo
);
539 bool binding_userptr
= false;
540 struct mm_struct
*usermm
;
542 usermm
= amdgpu_ttm_tt_get_usermm(bo
->tbo
.ttm
);
543 if (usermm
&& usermm
!= current
->mm
)
546 if (amdgpu_ttm_tt_is_userptr(bo
->tbo
.ttm
) &&
547 lobj
->user_invalidated
&& lobj
->user_pages
) {
548 amdgpu_bo_placement_from_domain(bo
,
549 AMDGPU_GEM_DOMAIN_CPU
);
550 r
= ttm_bo_validate(&bo
->tbo
, &bo
->placement
, &ctx
);
554 amdgpu_ttm_tt_set_user_pages(bo
->tbo
.ttm
,
556 binding_userptr
= true;
559 if (p
->evictable
== lobj
)
562 r
= amdgpu_cs_validate(p
, bo
);
566 if (binding_userptr
) {
567 kvfree(lobj
->user_pages
);
568 lobj
->user_pages
= NULL
;
574 static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser
*p
,
575 union drm_amdgpu_cs
*cs
)
577 struct amdgpu_fpriv
*fpriv
= p
->filp
->driver_priv
;
578 struct amdgpu_vm
*vm
= &fpriv
->vm
;
579 struct amdgpu_bo_list_entry
*e
;
580 struct list_head duplicates
;
581 struct amdgpu_bo
*gds
;
582 struct amdgpu_bo
*gws
;
583 struct amdgpu_bo
*oa
;
586 INIT_LIST_HEAD(&p
->validated
);
588 /* p->bo_list could already be assigned if AMDGPU_CHUNK_ID_BO_HANDLES is present */
589 if (cs
->in
.bo_list_handle
) {
593 r
= amdgpu_bo_list_get(fpriv
, cs
->in
.bo_list_handle
,
597 } else if (!p
->bo_list
) {
598 /* Create a empty bo_list when no handle is provided */
599 r
= amdgpu_bo_list_create(p
->adev
, p
->filp
, NULL
, 0,
605 /* One for TTM and one for the CS job */
606 amdgpu_bo_list_for_each_entry(e
, p
->bo_list
)
607 e
->tv
.num_shared
= 2;
609 amdgpu_bo_list_get_list(p
->bo_list
, &p
->validated
);
610 if (p
->bo_list
->first_userptr
!= p
->bo_list
->num_entries
)
611 p
->mn
= amdgpu_mn_get(p
->adev
, AMDGPU_MN_TYPE_GFX
);
613 INIT_LIST_HEAD(&duplicates
);
614 amdgpu_vm_get_pd_bo(&fpriv
->vm
, &p
->validated
, &p
->vm_pd
);
616 if (p
->uf_entry
.tv
.bo
&& !ttm_to_amdgpu_bo(p
->uf_entry
.tv
.bo
)->parent
)
617 list_add(&p
->uf_entry
.tv
.head
, &p
->validated
);
619 /* Get userptr backing pages. If pages are updated after registered
620 * in amdgpu_gem_userptr_ioctl(), amdgpu_cs_list_validate() will do
621 * amdgpu_ttm_backend_bind() to flush and invalidate new pages
623 amdgpu_bo_list_for_each_userptr_entry(e
, p
->bo_list
) {
624 struct amdgpu_bo
*bo
= ttm_to_amdgpu_bo(e
->tv
.bo
);
625 bool userpage_invalidated
= false;
628 e
->user_pages
= kvmalloc_array(bo
->tbo
.ttm
->num_pages
,
629 sizeof(struct page
*),
630 GFP_KERNEL
| __GFP_ZERO
);
631 if (!e
->user_pages
) {
632 DRM_ERROR("calloc failure\n");
636 r
= amdgpu_ttm_tt_get_user_pages(bo
->tbo
.ttm
, e
->user_pages
);
638 kvfree(e
->user_pages
);
639 e
->user_pages
= NULL
;
643 for (i
= 0; i
< bo
->tbo
.ttm
->num_pages
; i
++) {
644 if (bo
->tbo
.ttm
->pages
[i
] != e
->user_pages
[i
]) {
645 userpage_invalidated
= true;
649 e
->user_invalidated
= userpage_invalidated
;
652 r
= ttm_eu_reserve_buffers(&p
->ticket
, &p
->validated
, true,
654 if (unlikely(r
!= 0)) {
655 if (r
!= -ERESTARTSYS
)
656 DRM_ERROR("ttm_eu_reserve_buffers failed.\n");
660 amdgpu_cs_get_threshold_for_moves(p
->adev
, &p
->bytes_moved_threshold
,
661 &p
->bytes_moved_vis_threshold
);
663 p
->bytes_moved_vis
= 0;
664 p
->evictable
= list_last_entry(&p
->validated
,
665 struct amdgpu_bo_list_entry
,
668 r
= amdgpu_vm_validate_pt_bos(p
->adev
, &fpriv
->vm
,
669 amdgpu_cs_validate
, p
);
671 DRM_ERROR("amdgpu_vm_validate_pt_bos() failed.\n");
675 r
= amdgpu_cs_list_validate(p
, &duplicates
);
679 r
= amdgpu_cs_list_validate(p
, &p
->validated
);
683 amdgpu_cs_report_moved_bytes(p
->adev
, p
->bytes_moved
,
686 gds
= p
->bo_list
->gds_obj
;
687 gws
= p
->bo_list
->gws_obj
;
688 oa
= p
->bo_list
->oa_obj
;
690 amdgpu_bo_list_for_each_entry(e
, p
->bo_list
) {
691 struct amdgpu_bo
*bo
= ttm_to_amdgpu_bo(e
->tv
.bo
);
693 /* Make sure we use the exclusive slot for shared BOs */
694 if (bo
->prime_shared_count
)
695 e
->tv
.num_shared
= 0;
696 e
->bo_va
= amdgpu_vm_bo_find(vm
, bo
);
700 p
->job
->gds_base
= amdgpu_bo_gpu_offset(gds
) >> PAGE_SHIFT
;
701 p
->job
->gds_size
= amdgpu_bo_size(gds
) >> PAGE_SHIFT
;
704 p
->job
->gws_base
= amdgpu_bo_gpu_offset(gws
) >> PAGE_SHIFT
;
705 p
->job
->gws_size
= amdgpu_bo_size(gws
) >> PAGE_SHIFT
;
708 p
->job
->oa_base
= amdgpu_bo_gpu_offset(oa
) >> PAGE_SHIFT
;
709 p
->job
->oa_size
= amdgpu_bo_size(oa
) >> PAGE_SHIFT
;
712 if (!r
&& p
->uf_entry
.tv
.bo
) {
713 struct amdgpu_bo
*uf
= ttm_to_amdgpu_bo(p
->uf_entry
.tv
.bo
);
715 r
= amdgpu_ttm_alloc_gart(&uf
->tbo
);
716 p
->job
->uf_addr
+= amdgpu_bo_gpu_offset(uf
);
721 ttm_eu_backoff_reservation(&p
->ticket
, &p
->validated
);
726 static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser
*p
)
728 struct amdgpu_bo_list_entry
*e
;
731 list_for_each_entry(e
, &p
->validated
, tv
.head
) {
732 struct amdgpu_bo
*bo
= ttm_to_amdgpu_bo(e
->tv
.bo
);
733 struct reservation_object
*resv
= bo
->tbo
.resv
;
735 r
= amdgpu_sync_resv(p
->adev
, &p
->job
->sync
, resv
, p
->filp
,
736 amdgpu_bo_explicit_sync(bo
));
745 * cs_parser_fini() - clean parser states
746 * @parser: parser structure holding parsing context.
747 * @error: error number
749 * If error is set than unvalidate buffer, otherwise just free memory
750 * used by parsing context.
752 static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser
*parser
, int error
,
757 if (error
&& backoff
)
758 ttm_eu_backoff_reservation(&parser
->ticket
,
761 for (i
= 0; i
< parser
->num_post_deps
; i
++) {
762 drm_syncobj_put(parser
->post_deps
[i
].syncobj
);
763 kfree(parser
->post_deps
[i
].chain
);
765 kfree(parser
->post_deps
);
767 dma_fence_put(parser
->fence
);
770 mutex_unlock(&parser
->ctx
->lock
);
771 amdgpu_ctx_put(parser
->ctx
);
774 amdgpu_bo_list_put(parser
->bo_list
);
776 for (i
= 0; i
< parser
->nchunks
; i
++)
777 kvfree(parser
->chunks
[i
].kdata
);
778 kfree(parser
->chunks
);
780 amdgpu_job_free(parser
->job
);
781 if (parser
->uf_entry
.tv
.bo
) {
782 struct amdgpu_bo
*uf
= ttm_to_amdgpu_bo(parser
->uf_entry
.tv
.bo
);
784 amdgpu_bo_unref(&uf
);
788 static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser
*p
)
790 struct amdgpu_ring
*ring
= to_amdgpu_ring(p
->entity
->rq
->sched
);
791 struct amdgpu_fpriv
*fpriv
= p
->filp
->driver_priv
;
792 struct amdgpu_device
*adev
= p
->adev
;
793 struct amdgpu_vm
*vm
= &fpriv
->vm
;
794 struct amdgpu_bo_list_entry
*e
;
795 struct amdgpu_bo_va
*bo_va
;
796 struct amdgpu_bo
*bo
;
799 /* Only for UVD/VCE VM emulation */
800 if (ring
->funcs
->parse_cs
|| ring
->funcs
->patch_cs_in_place
) {
803 for (i
= 0, j
= 0; i
< p
->nchunks
&& j
< p
->job
->num_ibs
; i
++) {
804 struct drm_amdgpu_cs_chunk_ib
*chunk_ib
;
805 struct amdgpu_bo_va_mapping
*m
;
806 struct amdgpu_bo
*aobj
= NULL
;
807 struct amdgpu_cs_chunk
*chunk
;
808 uint64_t offset
, va_start
;
809 struct amdgpu_ib
*ib
;
812 chunk
= &p
->chunks
[i
];
813 ib
= &p
->job
->ibs
[j
];
814 chunk_ib
= chunk
->kdata
;
816 if (chunk
->chunk_id
!= AMDGPU_CHUNK_ID_IB
)
819 va_start
= chunk_ib
->va_start
& AMDGPU_GMC_HOLE_MASK
;
820 r
= amdgpu_cs_find_mapping(p
, va_start
, &aobj
, &m
);
822 DRM_ERROR("IB va_start is invalid\n");
826 if ((va_start
+ chunk_ib
->ib_bytes
) >
827 (m
->last
+ 1) * AMDGPU_GPU_PAGE_SIZE
) {
828 DRM_ERROR("IB va_start+ib_bytes is invalid\n");
832 /* the IB should be reserved at this point */
833 r
= amdgpu_bo_kmap(aobj
, (void **)&kptr
);
838 offset
= m
->start
* AMDGPU_GPU_PAGE_SIZE
;
839 kptr
+= va_start
- offset
;
841 if (ring
->funcs
->parse_cs
) {
842 memcpy(ib
->ptr
, kptr
, chunk_ib
->ib_bytes
);
843 amdgpu_bo_kunmap(aobj
);
845 r
= amdgpu_ring_parse_cs(ring
, p
, j
);
849 ib
->ptr
= (uint32_t *)kptr
;
850 r
= amdgpu_ring_patch_cs_in_place(ring
, p
, j
);
851 amdgpu_bo_kunmap(aobj
);
861 return amdgpu_cs_sync_rings(p
);
864 r
= amdgpu_vm_clear_freed(adev
, vm
, NULL
);
868 r
= amdgpu_vm_bo_update(adev
, fpriv
->prt_va
, false);
872 r
= amdgpu_sync_fence(adev
, &p
->job
->sync
,
873 fpriv
->prt_va
->last_pt_update
, false);
877 if (amdgpu_mcbp
|| amdgpu_sriov_vf(adev
)) {
880 bo_va
= fpriv
->csa_va
;
882 r
= amdgpu_vm_bo_update(adev
, bo_va
, false);
886 f
= bo_va
->last_pt_update
;
887 r
= amdgpu_sync_fence(adev
, &p
->job
->sync
, f
, false);
892 amdgpu_bo_list_for_each_entry(e
, p
->bo_list
) {
895 /* ignore duplicates */
896 bo
= ttm_to_amdgpu_bo(e
->tv
.bo
);
904 r
= amdgpu_vm_bo_update(adev
, bo_va
, false);
908 f
= bo_va
->last_pt_update
;
909 r
= amdgpu_sync_fence(adev
, &p
->job
->sync
, f
, false);
914 r
= amdgpu_vm_handle_moved(adev
, vm
);
918 r
= amdgpu_vm_update_directories(adev
, vm
);
922 r
= amdgpu_sync_fence(adev
, &p
->job
->sync
, vm
->last_update
, false);
926 p
->job
->vm_pd_addr
= amdgpu_gmc_pd_addr(vm
->root
.base
.bo
);
928 if (amdgpu_vm_debug
) {
929 /* Invalidate all BOs to test for userspace bugs */
930 amdgpu_bo_list_for_each_entry(e
, p
->bo_list
) {
931 struct amdgpu_bo
*bo
= ttm_to_amdgpu_bo(e
->tv
.bo
);
933 /* ignore duplicates */
937 amdgpu_vm_bo_invalidate(adev
, bo
, false);
941 return amdgpu_cs_sync_rings(p
);
944 static int amdgpu_cs_ib_fill(struct amdgpu_device
*adev
,
945 struct amdgpu_cs_parser
*parser
)
947 struct amdgpu_fpriv
*fpriv
= parser
->filp
->driver_priv
;
948 struct amdgpu_vm
*vm
= &fpriv
->vm
;
949 int r
, ce_preempt
= 0, de_preempt
= 0;
950 struct amdgpu_ring
*ring
;
953 for (i
= 0, j
= 0; i
< parser
->nchunks
&& j
< parser
->job
->num_ibs
; i
++) {
954 struct amdgpu_cs_chunk
*chunk
;
955 struct amdgpu_ib
*ib
;
956 struct drm_amdgpu_cs_chunk_ib
*chunk_ib
;
957 struct drm_sched_entity
*entity
;
959 chunk
= &parser
->chunks
[i
];
960 ib
= &parser
->job
->ibs
[j
];
961 chunk_ib
= (struct drm_amdgpu_cs_chunk_ib
*)chunk
->kdata
;
963 if (chunk
->chunk_id
!= AMDGPU_CHUNK_ID_IB
)
966 if (chunk_ib
->ip_type
== AMDGPU_HW_IP_GFX
&&
967 (amdgpu_mcbp
|| amdgpu_sriov_vf(adev
))) {
968 if (chunk_ib
->flags
& AMDGPU_IB_FLAG_PREEMPT
) {
969 if (chunk_ib
->flags
& AMDGPU_IB_FLAG_CE
)
975 /* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */
976 if (ce_preempt
> 1 || de_preempt
> 1)
980 r
= amdgpu_ctx_get_entity(parser
->ctx
, chunk_ib
->ip_type
,
981 chunk_ib
->ip_instance
, chunk_ib
->ring
,
986 if (chunk_ib
->flags
& AMDGPU_IB_FLAG_PREAMBLE
)
987 parser
->job
->preamble_status
|=
988 AMDGPU_PREAMBLE_IB_PRESENT
;
990 if (parser
->entity
&& parser
->entity
!= entity
)
993 parser
->entity
= entity
;
995 ring
= to_amdgpu_ring(entity
->rq
->sched
);
996 r
= amdgpu_ib_get(adev
, vm
, ring
->funcs
->parse_cs
?
997 chunk_ib
->ib_bytes
: 0, ib
);
999 DRM_ERROR("Failed to get ib !\n");
1003 ib
->gpu_addr
= chunk_ib
->va_start
;
1004 ib
->length_dw
= chunk_ib
->ib_bytes
/ 4;
1005 ib
->flags
= chunk_ib
->flags
;
1010 /* MM engine doesn't support user fences */
1011 ring
= to_amdgpu_ring(parser
->entity
->rq
->sched
);
1012 if (parser
->job
->uf_addr
&& ring
->funcs
->no_user_fence
)
1015 return amdgpu_ctx_wait_prev_fence(parser
->ctx
, parser
->entity
);
1018 static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser
*p
,
1019 struct amdgpu_cs_chunk
*chunk
)
1021 struct amdgpu_fpriv
*fpriv
= p
->filp
->driver_priv
;
1024 struct drm_amdgpu_cs_chunk_dep
*deps
;
1026 deps
= (struct drm_amdgpu_cs_chunk_dep
*)chunk
->kdata
;
1027 num_deps
= chunk
->length_dw
* 4 /
1028 sizeof(struct drm_amdgpu_cs_chunk_dep
);
1030 for (i
= 0; i
< num_deps
; ++i
) {
1031 struct amdgpu_ctx
*ctx
;
1032 struct drm_sched_entity
*entity
;
1033 struct dma_fence
*fence
;
1035 ctx
= amdgpu_ctx_get(fpriv
, deps
[i
].ctx_id
);
1039 r
= amdgpu_ctx_get_entity(ctx
, deps
[i
].ip_type
,
1040 deps
[i
].ip_instance
,
1041 deps
[i
].ring
, &entity
);
1043 amdgpu_ctx_put(ctx
);
1047 fence
= amdgpu_ctx_get_fence(ctx
, entity
,
1050 if (chunk
->chunk_id
== AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES
) {
1051 struct drm_sched_fence
*s_fence
= to_drm_sched_fence(fence
);
1052 struct dma_fence
*old
= fence
;
1054 fence
= dma_fence_get(&s_fence
->scheduled
);
1058 if (IS_ERR(fence
)) {
1060 amdgpu_ctx_put(ctx
);
1063 r
= amdgpu_sync_fence(p
->adev
, &p
->job
->sync
, fence
,
1065 dma_fence_put(fence
);
1066 amdgpu_ctx_put(ctx
);
1074 static int amdgpu_syncobj_lookup_and_add_to_sync(struct amdgpu_cs_parser
*p
,
1075 uint32_t handle
, u64 point
,
1078 struct dma_fence
*fence
;
1081 r
= drm_syncobj_find_fence(p
->filp
, handle
, point
, flags
, &fence
);
1083 DRM_ERROR("syncobj %u failed to find fence @ %llu (%d)!\n",
1088 r
= amdgpu_sync_fence(p
->adev
, &p
->job
->sync
, fence
, true);
1089 dma_fence_put(fence
);
1094 static int amdgpu_cs_process_syncobj_in_dep(struct amdgpu_cs_parser
*p
,
1095 struct amdgpu_cs_chunk
*chunk
)
1097 struct drm_amdgpu_cs_chunk_sem
*deps
;
1101 deps
= (struct drm_amdgpu_cs_chunk_sem
*)chunk
->kdata
;
1102 num_deps
= chunk
->length_dw
* 4 /
1103 sizeof(struct drm_amdgpu_cs_chunk_sem
);
1104 for (i
= 0; i
< num_deps
; ++i
) {
1105 r
= amdgpu_syncobj_lookup_and_add_to_sync(p
, deps
[i
].handle
,
1115 static int amdgpu_cs_process_syncobj_timeline_in_dep(struct amdgpu_cs_parser
*p
,
1116 struct amdgpu_cs_chunk
*chunk
)
1118 struct drm_amdgpu_cs_chunk_syncobj
*syncobj_deps
;
1122 syncobj_deps
= (struct drm_amdgpu_cs_chunk_syncobj
*)chunk
->kdata
;
1123 num_deps
= chunk
->length_dw
* 4 /
1124 sizeof(struct drm_amdgpu_cs_chunk_syncobj
);
1125 for (i
= 0; i
< num_deps
; ++i
) {
1126 r
= amdgpu_syncobj_lookup_and_add_to_sync(p
,
1127 syncobj_deps
[i
].handle
,
1128 syncobj_deps
[i
].point
,
1129 syncobj_deps
[i
].flags
);
1137 static int amdgpu_cs_process_syncobj_out_dep(struct amdgpu_cs_parser
*p
,
1138 struct amdgpu_cs_chunk
*chunk
)
1140 struct drm_amdgpu_cs_chunk_sem
*deps
;
1144 deps
= (struct drm_amdgpu_cs_chunk_sem
*)chunk
->kdata
;
1145 num_deps
= chunk
->length_dw
* 4 /
1146 sizeof(struct drm_amdgpu_cs_chunk_sem
);
1148 p
->post_deps
= kmalloc_array(num_deps
, sizeof(*p
->post_deps
),
1150 p
->num_post_deps
= 0;
1156 for (i
= 0; i
< num_deps
; ++i
) {
1157 p
->post_deps
[i
].syncobj
=
1158 drm_syncobj_find(p
->filp
, deps
[i
].handle
);
1159 if (!p
->post_deps
[i
].syncobj
)
1161 p
->post_deps
[i
].chain
= NULL
;
1162 p
->post_deps
[i
].point
= 0;
1170 static int amdgpu_cs_process_syncobj_timeline_out_dep(struct amdgpu_cs_parser
*p
,
1171 struct amdgpu_cs_chunk
1174 struct drm_amdgpu_cs_chunk_syncobj
*syncobj_deps
;
1178 syncobj_deps
= (struct drm_amdgpu_cs_chunk_syncobj
*)chunk
->kdata
;
1179 num_deps
= chunk
->length_dw
* 4 /
1180 sizeof(struct drm_amdgpu_cs_chunk_syncobj
);
1182 p
->post_deps
= kmalloc_array(num_deps
, sizeof(*p
->post_deps
),
1184 p
->num_post_deps
= 0;
1189 for (i
= 0; i
< num_deps
; ++i
) {
1190 struct amdgpu_cs_post_dep
*dep
= &p
->post_deps
[i
];
1193 if (syncobj_deps
[i
].point
) {
1194 dep
->chain
= kmalloc(sizeof(*dep
->chain
), GFP_KERNEL
);
1199 dep
->syncobj
= drm_syncobj_find(p
->filp
,
1200 syncobj_deps
[i
].handle
);
1201 if (!dep
->syncobj
) {
1205 dep
->point
= syncobj_deps
[i
].point
;
1212 static int amdgpu_cs_dependencies(struct amdgpu_device
*adev
,
1213 struct amdgpu_cs_parser
*p
)
1217 for (i
= 0; i
< p
->nchunks
; ++i
) {
1218 struct amdgpu_cs_chunk
*chunk
;
1220 chunk
= &p
->chunks
[i
];
1222 switch (chunk
->chunk_id
) {
1223 case AMDGPU_CHUNK_ID_DEPENDENCIES
:
1224 case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES
:
1225 r
= amdgpu_cs_process_fence_dep(p
, chunk
);
1229 case AMDGPU_CHUNK_ID_SYNCOBJ_IN
:
1230 r
= amdgpu_cs_process_syncobj_in_dep(p
, chunk
);
1234 case AMDGPU_CHUNK_ID_SYNCOBJ_OUT
:
1235 r
= amdgpu_cs_process_syncobj_out_dep(p
, chunk
);
1239 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT
:
1240 r
= amdgpu_cs_process_syncobj_timeline_in_dep(p
, chunk
);
1244 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL
:
1245 r
= amdgpu_cs_process_syncobj_timeline_out_dep(p
, chunk
);
1255 static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser
*p
)
1259 for (i
= 0; i
< p
->num_post_deps
; ++i
) {
1260 if (p
->post_deps
[i
].chain
&& p
->post_deps
[i
].point
) {
1261 drm_syncobj_add_point(p
->post_deps
[i
].syncobj
,
1262 p
->post_deps
[i
].chain
,
1263 p
->fence
, p
->post_deps
[i
].point
);
1264 p
->post_deps
[i
].chain
= NULL
;
1266 drm_syncobj_replace_fence(p
->post_deps
[i
].syncobj
,
1272 static int amdgpu_cs_submit(struct amdgpu_cs_parser
*p
,
1273 union drm_amdgpu_cs
*cs
)
1275 struct amdgpu_fpriv
*fpriv
= p
->filp
->driver_priv
;
1276 struct drm_sched_entity
*entity
= p
->entity
;
1277 enum drm_sched_priority priority
;
1278 struct amdgpu_ring
*ring
;
1279 struct amdgpu_bo_list_entry
*e
;
1280 struct amdgpu_job
*job
;
1287 r
= drm_sched_job_init(&job
->base
, entity
, p
->filp
);
1291 /* No memory allocation is allowed while holding the mn lock.
1292 * p->mn is hold until amdgpu_cs_submit is finished and fence is added
1295 amdgpu_mn_lock(p
->mn
);
1297 /* If userptr are invalidated after amdgpu_cs_parser_bos(), return
1298 * -EAGAIN, drmIoctl in libdrm will restart the amdgpu_cs_ioctl.
1300 amdgpu_bo_list_for_each_userptr_entry(e
, p
->bo_list
) {
1301 struct amdgpu_bo
*bo
= ttm_to_amdgpu_bo(e
->tv
.bo
);
1303 r
|= !amdgpu_ttm_tt_get_user_pages_done(bo
->tbo
.ttm
);
1310 job
->owner
= p
->filp
;
1311 p
->fence
= dma_fence_get(&job
->base
.s_fence
->finished
);
1313 amdgpu_ctx_add_fence(p
->ctx
, entity
, p
->fence
, &seq
);
1314 amdgpu_cs_post_dependencies(p
);
1316 if ((job
->preamble_status
& AMDGPU_PREAMBLE_IB_PRESENT
) &&
1317 !p
->ctx
->preamble_presented
) {
1318 job
->preamble_status
|= AMDGPU_PREAMBLE_IB_PRESENT_FIRST
;
1319 p
->ctx
->preamble_presented
= true;
1322 cs
->out
.handle
= seq
;
1323 job
->uf_sequence
= seq
;
1325 amdgpu_job_free_resources(job
);
1327 trace_amdgpu_cs_ioctl(job
);
1328 amdgpu_vm_bo_trace_cs(&fpriv
->vm
, &p
->ticket
);
1329 priority
= job
->base
.s_priority
;
1330 drm_sched_entity_push_job(&job
->base
, entity
);
1332 ring
= to_amdgpu_ring(entity
->rq
->sched
);
1333 amdgpu_ring_priority_get(ring
, priority
);
1335 amdgpu_vm_move_to_lru_tail(p
->adev
, &fpriv
->vm
);
1337 ttm_eu_fence_buffer_objects(&p
->ticket
, &p
->validated
, p
->fence
);
1338 amdgpu_mn_unlock(p
->mn
);
1343 drm_sched_job_cleanup(&job
->base
);
1344 amdgpu_mn_unlock(p
->mn
);
1347 amdgpu_job_free(job
);
1351 int amdgpu_cs_ioctl(struct drm_device
*dev
, void *data
, struct drm_file
*filp
)
1353 struct amdgpu_device
*adev
= dev
->dev_private
;
1354 union drm_amdgpu_cs
*cs
= data
;
1355 struct amdgpu_cs_parser parser
= {};
1356 bool reserved_buffers
= false;
1359 if (!adev
->accel_working
)
1365 r
= amdgpu_cs_parser_init(&parser
, data
);
1367 DRM_ERROR("Failed to initialize parser %d!\n", r
);
1371 r
= amdgpu_cs_ib_fill(adev
, &parser
);
1375 r
= amdgpu_cs_dependencies(adev
, &parser
);
1377 DRM_ERROR("Failed in the dependencies handling %d!\n", r
);
1381 r
= amdgpu_cs_parser_bos(&parser
, data
);
1384 DRM_ERROR("Not enough memory for command submission!\n");
1385 else if (r
!= -ERESTARTSYS
&& r
!= -EAGAIN
)
1386 DRM_ERROR("Failed to process the buffer list %d!\n", r
);
1390 reserved_buffers
= true;
1392 for (i
= 0; i
< parser
.job
->num_ibs
; i
++)
1393 trace_amdgpu_cs(&parser
, i
);
1395 r
= amdgpu_cs_vm_handling(&parser
);
1399 r
= amdgpu_cs_submit(&parser
, cs
);
1402 amdgpu_cs_parser_fini(&parser
, r
, reserved_buffers
);
1408 * amdgpu_cs_wait_ioctl - wait for a command submission to finish
1411 * @data: data from userspace
1412 * @filp: file private
1414 * Wait for the command submission identified by handle to finish.
1416 int amdgpu_cs_wait_ioctl(struct drm_device
*dev
, void *data
,
1417 struct drm_file
*filp
)
1419 union drm_amdgpu_wait_cs
*wait
= data
;
1420 unsigned long timeout
= amdgpu_gem_timeout(wait
->in
.timeout
);
1421 struct drm_sched_entity
*entity
;
1422 struct amdgpu_ctx
*ctx
;
1423 struct dma_fence
*fence
;
1426 ctx
= amdgpu_ctx_get(filp
->driver_priv
, wait
->in
.ctx_id
);
1430 r
= amdgpu_ctx_get_entity(ctx
, wait
->in
.ip_type
, wait
->in
.ip_instance
,
1431 wait
->in
.ring
, &entity
);
1433 amdgpu_ctx_put(ctx
);
1437 fence
= amdgpu_ctx_get_fence(ctx
, entity
, wait
->in
.handle
);
1441 r
= dma_fence_wait_timeout(fence
, true, timeout
);
1442 if (r
> 0 && fence
->error
)
1444 dma_fence_put(fence
);
1448 amdgpu_ctx_put(ctx
);
1452 memset(wait
, 0, sizeof(*wait
));
1453 wait
->out
.status
= (r
== 0);
1459 * amdgpu_cs_get_fence - helper to get fence from drm_amdgpu_fence
1461 * @adev: amdgpu device
1462 * @filp: file private
1463 * @user: drm_amdgpu_fence copied from user space
1465 static struct dma_fence
*amdgpu_cs_get_fence(struct amdgpu_device
*adev
,
1466 struct drm_file
*filp
,
1467 struct drm_amdgpu_fence
*user
)
1469 struct drm_sched_entity
*entity
;
1470 struct amdgpu_ctx
*ctx
;
1471 struct dma_fence
*fence
;
1474 ctx
= amdgpu_ctx_get(filp
->driver_priv
, user
->ctx_id
);
1476 return ERR_PTR(-EINVAL
);
1478 r
= amdgpu_ctx_get_entity(ctx
, user
->ip_type
, user
->ip_instance
,
1479 user
->ring
, &entity
);
1481 amdgpu_ctx_put(ctx
);
1485 fence
= amdgpu_ctx_get_fence(ctx
, entity
, user
->seq_no
);
1486 amdgpu_ctx_put(ctx
);
1491 int amdgpu_cs_fence_to_handle_ioctl(struct drm_device
*dev
, void *data
,
1492 struct drm_file
*filp
)
1494 struct amdgpu_device
*adev
= dev
->dev_private
;
1495 union drm_amdgpu_fence_to_handle
*info
= data
;
1496 struct dma_fence
*fence
;
1497 struct drm_syncobj
*syncobj
;
1498 struct sync_file
*sync_file
;
1501 fence
= amdgpu_cs_get_fence(adev
, filp
, &info
->in
.fence
);
1503 return PTR_ERR(fence
);
1506 fence
= dma_fence_get_stub();
1508 switch (info
->in
.what
) {
1509 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ
:
1510 r
= drm_syncobj_create(&syncobj
, 0, fence
);
1511 dma_fence_put(fence
);
1514 r
= drm_syncobj_get_handle(filp
, syncobj
, &info
->out
.handle
);
1515 drm_syncobj_put(syncobj
);
1518 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD
:
1519 r
= drm_syncobj_create(&syncobj
, 0, fence
);
1520 dma_fence_put(fence
);
1523 r
= drm_syncobj_get_fd(syncobj
, (int*)&info
->out
.handle
);
1524 drm_syncobj_put(syncobj
);
1527 case AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD
:
1528 fd
= get_unused_fd_flags(O_CLOEXEC
);
1530 dma_fence_put(fence
);
1534 sync_file
= sync_file_create(fence
);
1535 dma_fence_put(fence
);
1541 fd_install(fd
, sync_file
->file
);
1542 info
->out
.handle
= fd
;
1551 * amdgpu_cs_wait_all_fence - wait on all fences to signal
1553 * @adev: amdgpu device
1554 * @filp: file private
1555 * @wait: wait parameters
1556 * @fences: array of drm_amdgpu_fence
1558 static int amdgpu_cs_wait_all_fences(struct amdgpu_device
*adev
,
1559 struct drm_file
*filp
,
1560 union drm_amdgpu_wait_fences
*wait
,
1561 struct drm_amdgpu_fence
*fences
)
1563 uint32_t fence_count
= wait
->in
.fence_count
;
1567 for (i
= 0; i
< fence_count
; i
++) {
1568 struct dma_fence
*fence
;
1569 unsigned long timeout
= amdgpu_gem_timeout(wait
->in
.timeout_ns
);
1571 fence
= amdgpu_cs_get_fence(adev
, filp
, &fences
[i
]);
1573 return PTR_ERR(fence
);
1577 r
= dma_fence_wait_timeout(fence
, true, timeout
);
1578 dma_fence_put(fence
);
1586 return fence
->error
;
1589 memset(wait
, 0, sizeof(*wait
));
1590 wait
->out
.status
= (r
> 0);
1596 * amdgpu_cs_wait_any_fence - wait on any fence to signal
1598 * @adev: amdgpu device
1599 * @filp: file private
1600 * @wait: wait parameters
1601 * @fences: array of drm_amdgpu_fence
1603 static int amdgpu_cs_wait_any_fence(struct amdgpu_device
*adev
,
1604 struct drm_file
*filp
,
1605 union drm_amdgpu_wait_fences
*wait
,
1606 struct drm_amdgpu_fence
*fences
)
1608 unsigned long timeout
= amdgpu_gem_timeout(wait
->in
.timeout_ns
);
1609 uint32_t fence_count
= wait
->in
.fence_count
;
1610 uint32_t first
= ~0;
1611 struct dma_fence
**array
;
1615 /* Prepare the fence array */
1616 array
= kcalloc(fence_count
, sizeof(struct dma_fence
*), GFP_KERNEL
);
1621 for (i
= 0; i
< fence_count
; i
++) {
1622 struct dma_fence
*fence
;
1624 fence
= amdgpu_cs_get_fence(adev
, filp
, &fences
[i
]);
1625 if (IS_ERR(fence
)) {
1627 goto err_free_fence_array
;
1630 } else { /* NULL, the fence has been already signaled */
1637 r
= dma_fence_wait_any_timeout(array
, fence_count
, true, timeout
,
1640 goto err_free_fence_array
;
1643 memset(wait
, 0, sizeof(*wait
));
1644 wait
->out
.status
= (r
> 0);
1645 wait
->out
.first_signaled
= first
;
1647 if (first
< fence_count
&& array
[first
])
1648 r
= array
[first
]->error
;
1652 err_free_fence_array
:
1653 for (i
= 0; i
< fence_count
; i
++)
1654 dma_fence_put(array
[i
]);
1661 * amdgpu_cs_wait_fences_ioctl - wait for multiple command submissions to finish
1664 * @data: data from userspace
1665 * @filp: file private
1667 int amdgpu_cs_wait_fences_ioctl(struct drm_device
*dev
, void *data
,
1668 struct drm_file
*filp
)
1670 struct amdgpu_device
*adev
= dev
->dev_private
;
1671 union drm_amdgpu_wait_fences
*wait
= data
;
1672 uint32_t fence_count
= wait
->in
.fence_count
;
1673 struct drm_amdgpu_fence
*fences_user
;
1674 struct drm_amdgpu_fence
*fences
;
1677 /* Get the fences from userspace */
1678 fences
= kmalloc_array(fence_count
, sizeof(struct drm_amdgpu_fence
),
1683 fences_user
= u64_to_user_ptr(wait
->in
.fences
);
1684 if (copy_from_user(fences
, fences_user
,
1685 sizeof(struct drm_amdgpu_fence
) * fence_count
)) {
1687 goto err_free_fences
;
1690 if (wait
->in
.wait_all
)
1691 r
= amdgpu_cs_wait_all_fences(adev
, filp
, wait
, fences
);
1693 r
= amdgpu_cs_wait_any_fence(adev
, filp
, wait
, fences
);
1702 * amdgpu_cs_find_bo_va - find bo_va for VM address
1704 * @parser: command submission parser context
1706 * @bo: resulting BO of the mapping found
1708 * Search the buffer objects in the command submission context for a certain
1709 * virtual memory address. Returns allocation structure when found, NULL
1712 int amdgpu_cs_find_mapping(struct amdgpu_cs_parser
*parser
,
1713 uint64_t addr
, struct amdgpu_bo
**bo
,
1714 struct amdgpu_bo_va_mapping
**map
)
1716 struct amdgpu_fpriv
*fpriv
= parser
->filp
->driver_priv
;
1717 struct ttm_operation_ctx ctx
= { false, false };
1718 struct amdgpu_vm
*vm
= &fpriv
->vm
;
1719 struct amdgpu_bo_va_mapping
*mapping
;
1722 addr
/= AMDGPU_GPU_PAGE_SIZE
;
1724 mapping
= amdgpu_vm_bo_lookup_mapping(vm
, addr
);
1725 if (!mapping
|| !mapping
->bo_va
|| !mapping
->bo_va
->base
.bo
)
1728 *bo
= mapping
->bo_va
->base
.bo
;
1731 /* Double check that the BO is reserved by this CS */
1732 if (READ_ONCE((*bo
)->tbo
.resv
->lock
.ctx
) != &parser
->ticket
)
1735 if (!((*bo
)->flags
& AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS
)) {
1736 (*bo
)->flags
|= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS
;
1737 amdgpu_bo_placement_from_domain(*bo
, (*bo
)->allowed_domains
);
1738 r
= ttm_bo_validate(&(*bo
)->tbo
, &(*bo
)->placement
, &ctx
);
1743 return amdgpu_ttm_alloc_gart(&(*bo
)->tbo
);