2 * Copyright 2008 Jerome Glisse.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
25 * Jerome Glisse <glisse@freedesktop.org>
27 #include <linux/pagemap.h>
29 #include <drm/amdgpu_drm.h>
31 #include "amdgpu_trace.h"
33 int amdgpu_cs_get_ring(struct amdgpu_device
*adev
, u32 ip_type
,
34 u32 ip_instance
, u32 ring
,
35 struct amdgpu_ring
**out_ring
)
37 /* Right now all IPs have only one instance - multiple rings. */
38 if (ip_instance
!= 0) {
39 DRM_ERROR("invalid ip instance: %d\n", ip_instance
);
45 DRM_ERROR("unknown ip type: %d\n", ip_type
);
47 case AMDGPU_HW_IP_GFX
:
48 if (ring
< adev
->gfx
.num_gfx_rings
) {
49 *out_ring
= &adev
->gfx
.gfx_ring
[ring
];
51 DRM_ERROR("only %d gfx rings are supported now\n",
52 adev
->gfx
.num_gfx_rings
);
56 case AMDGPU_HW_IP_COMPUTE
:
57 if (ring
< adev
->gfx
.num_compute_rings
) {
58 *out_ring
= &adev
->gfx
.compute_ring
[ring
];
60 DRM_ERROR("only %d compute rings are supported now\n",
61 adev
->gfx
.num_compute_rings
);
65 case AMDGPU_HW_IP_DMA
:
66 if (ring
< adev
->sdma
.num_instances
) {
67 *out_ring
= &adev
->sdma
.instance
[ring
].ring
;
69 DRM_ERROR("only %d SDMA rings are supported\n",
70 adev
->sdma
.num_instances
);
74 case AMDGPU_HW_IP_UVD
:
75 *out_ring
= &adev
->uvd
.ring
;
77 case AMDGPU_HW_IP_VCE
:
78 if (ring
< adev
->vce
.num_rings
){
79 *out_ring
= &adev
->vce
.ring
[ring
];
81 DRM_ERROR("only %d VCE rings are supported\n", adev
->vce
.num_rings
);
85 case AMDGPU_HW_IP_UVD_ENC
:
86 if (ring
< adev
->uvd
.num_enc_rings
){
87 *out_ring
= &adev
->uvd
.ring_enc
[ring
];
89 DRM_ERROR("only %d UVD ENC rings are supported\n",
90 adev
->uvd
.num_enc_rings
);
96 if (!(*out_ring
&& (*out_ring
)->adev
)) {
97 DRM_ERROR("Ring %d is not initialized on IP %d\n",
105 static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser
*p
,
106 struct drm_amdgpu_cs_chunk_fence
*data
,
109 struct drm_gem_object
*gobj
;
112 gobj
= drm_gem_object_lookup(p
->filp
, data
->handle
);
116 p
->uf_entry
.robj
= amdgpu_bo_ref(gem_to_amdgpu_bo(gobj
));
117 p
->uf_entry
.priority
= 0;
118 p
->uf_entry
.tv
.bo
= &p
->uf_entry
.robj
->tbo
;
119 p
->uf_entry
.tv
.shared
= true;
120 p
->uf_entry
.user_pages
= NULL
;
122 size
= amdgpu_bo_size(p
->uf_entry
.robj
);
123 if (size
!= PAGE_SIZE
|| (data
->offset
+ 8) > size
)
126 *offset
= data
->offset
;
128 drm_gem_object_unreference_unlocked(gobj
);
130 if (amdgpu_ttm_tt_get_usermm(p
->uf_entry
.robj
->tbo
.ttm
)) {
131 amdgpu_bo_unref(&p
->uf_entry
.robj
);
138 int amdgpu_cs_parser_init(struct amdgpu_cs_parser
*p
, void *data
)
140 struct amdgpu_fpriv
*fpriv
= p
->filp
->driver_priv
;
141 struct amdgpu_vm
*vm
= &fpriv
->vm
;
142 union drm_amdgpu_cs
*cs
= data
;
143 uint64_t *chunk_array_user
;
144 uint64_t *chunk_array
;
145 unsigned size
, num_ibs
= 0;
146 uint32_t uf_offset
= 0;
150 if (cs
->in
.num_chunks
== 0)
153 chunk_array
= kmalloc_array(cs
->in
.num_chunks
, sizeof(uint64_t), GFP_KERNEL
);
157 p
->ctx
= amdgpu_ctx_get(fpriv
, cs
->in
.ctx_id
);
164 chunk_array_user
= (uint64_t __user
*)(uintptr_t)(cs
->in
.chunks
);
165 if (copy_from_user(chunk_array
, chunk_array_user
,
166 sizeof(uint64_t)*cs
->in
.num_chunks
)) {
171 p
->nchunks
= cs
->in
.num_chunks
;
172 p
->chunks
= kmalloc_array(p
->nchunks
, sizeof(struct amdgpu_cs_chunk
),
179 for (i
= 0; i
< p
->nchunks
; i
++) {
180 struct drm_amdgpu_cs_chunk __user
**chunk_ptr
= NULL
;
181 struct drm_amdgpu_cs_chunk user_chunk
;
182 uint32_t __user
*cdata
;
184 chunk_ptr
= (void __user
*)(uintptr_t)chunk_array
[i
];
185 if (copy_from_user(&user_chunk
, chunk_ptr
,
186 sizeof(struct drm_amdgpu_cs_chunk
))) {
189 goto free_partial_kdata
;
191 p
->chunks
[i
].chunk_id
= user_chunk
.chunk_id
;
192 p
->chunks
[i
].length_dw
= user_chunk
.length_dw
;
194 size
= p
->chunks
[i
].length_dw
;
195 cdata
= (void __user
*)(uintptr_t)user_chunk
.chunk_data
;
197 p
->chunks
[i
].kdata
= drm_malloc_ab(size
, sizeof(uint32_t));
198 if (p
->chunks
[i
].kdata
== NULL
) {
201 goto free_partial_kdata
;
203 size
*= sizeof(uint32_t);
204 if (copy_from_user(p
->chunks
[i
].kdata
, cdata
, size
)) {
206 goto free_partial_kdata
;
209 switch (p
->chunks
[i
].chunk_id
) {
210 case AMDGPU_CHUNK_ID_IB
:
214 case AMDGPU_CHUNK_ID_FENCE
:
215 size
= sizeof(struct drm_amdgpu_cs_chunk_fence
);
216 if (p
->chunks
[i
].length_dw
* sizeof(uint32_t) < size
) {
218 goto free_partial_kdata
;
221 ret
= amdgpu_cs_user_fence_chunk(p
, p
->chunks
[i
].kdata
,
224 goto free_partial_kdata
;
228 case AMDGPU_CHUNK_ID_DEPENDENCIES
:
233 goto free_partial_kdata
;
237 ret
= amdgpu_job_alloc(p
->adev
, num_ibs
, &p
->job
, vm
);
241 if (p
->uf_entry
.robj
)
242 p
->job
->uf_addr
= uf_offset
;
250 drm_free_large(p
->chunks
[i
].kdata
);
255 amdgpu_ctx_put(p
->ctx
);
262 /* Convert microseconds to bytes. */
263 static u64
us_to_bytes(struct amdgpu_device
*adev
, s64 us
)
265 if (us
<= 0 || !adev
->mm_stats
.log2_max_MBps
)
268 /* Since accum_us is incremented by a million per second, just
269 * multiply it by the number of MB/s to get the number of bytes.
271 return us
<< adev
->mm_stats
.log2_max_MBps
;
274 static s64
bytes_to_us(struct amdgpu_device
*adev
, u64 bytes
)
276 if (!adev
->mm_stats
.log2_max_MBps
)
279 return bytes
>> adev
->mm_stats
.log2_max_MBps
;
282 /* Returns how many bytes TTM can move right now. If no bytes can be moved,
283 * it returns 0. If it returns non-zero, it's OK to move at least one buffer,
284 * which means it can go over the threshold once. If that happens, the driver
285 * will be in debt and no other buffer migrations can be done until that debt
288 * This approach allows moving a buffer of any size (it's important to allow
291 * The currency is simply time in microseconds and it increases as the clock
292 * ticks. The accumulated microseconds (us) are converted to bytes and
295 static u64
amdgpu_cs_get_threshold_for_moves(struct amdgpu_device
*adev
)
297 s64 time_us
, increment_us
;
299 u64 free_vram
, total_vram
, used_vram
;
301 /* Allow a maximum of 200 accumulated ms. This is basically per-IB
304 * It means that in order to get full max MBps, at least 5 IBs per
305 * second must be submitted and not more than 200ms apart from each
308 const s64 us_upper_bound
= 200000;
310 if (!adev
->mm_stats
.log2_max_MBps
)
313 total_vram
= adev
->mc
.real_vram_size
- adev
->vram_pin_size
;
314 used_vram
= atomic64_read(&adev
->vram_usage
);
315 free_vram
= used_vram
>= total_vram
? 0 : total_vram
- used_vram
;
317 spin_lock(&adev
->mm_stats
.lock
);
319 /* Increase the amount of accumulated us. */
320 time_us
= ktime_to_us(ktime_get());
321 increment_us
= time_us
- adev
->mm_stats
.last_update_us
;
322 adev
->mm_stats
.last_update_us
= time_us
;
323 adev
->mm_stats
.accum_us
= min(adev
->mm_stats
.accum_us
+ increment_us
,
326 /* This prevents the short period of low performance when the VRAM
327 * usage is low and the driver is in debt or doesn't have enough
328 * accumulated us to fill VRAM quickly.
330 * The situation can occur in these cases:
331 * - a lot of VRAM is freed by userspace
332 * - the presence of a big buffer causes a lot of evictions
333 * (solution: split buffers into smaller ones)
335 * If 128 MB or 1/8th of VRAM is free, start filling it now by setting
336 * accum_us to a positive number.
338 if (free_vram
>= 128 * 1024 * 1024 || free_vram
>= total_vram
/ 8) {
341 /* Be more aggresive on dGPUs. Try to fill a portion of free
344 if (!(adev
->flags
& AMD_IS_APU
))
345 min_us
= bytes_to_us(adev
, free_vram
/ 4);
347 min_us
= 0; /* Reset accum_us on APUs. */
349 adev
->mm_stats
.accum_us
= max(min_us
, adev
->mm_stats
.accum_us
);
352 /* This returns 0 if the driver is in debt to disallow (optional)
355 max_bytes
= us_to_bytes(adev
, adev
->mm_stats
.accum_us
);
357 spin_unlock(&adev
->mm_stats
.lock
);
361 /* Report how many bytes have really been moved for the last command
362 * submission. This can result in a debt that can stop buffer migrations
365 void amdgpu_cs_report_moved_bytes(struct amdgpu_device
*adev
, u64 num_bytes
)
367 spin_lock(&adev
->mm_stats
.lock
);
368 adev
->mm_stats
.accum_us
-= bytes_to_us(adev
, num_bytes
);
369 spin_unlock(&adev
->mm_stats
.lock
);
372 static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser
*p
,
373 struct amdgpu_bo
*bo
)
375 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->tbo
.bdev
);
376 u64 initial_bytes_moved
;
383 /* Don't move this buffer if we have depleted our allowance
384 * to move it. Don't move anything if the threshold is zero.
386 if (p
->bytes_moved
< p
->bytes_moved_threshold
)
387 domain
= bo
->prefered_domains
;
389 domain
= bo
->allowed_domains
;
392 amdgpu_ttm_placement_from_domain(bo
, domain
);
393 initial_bytes_moved
= atomic64_read(&adev
->num_bytes_moved
);
394 r
= ttm_bo_validate(&bo
->tbo
, &bo
->placement
, true, false);
395 p
->bytes_moved
+= atomic64_read(&adev
->num_bytes_moved
) -
398 if (unlikely(r
== -ENOMEM
) && domain
!= bo
->allowed_domains
) {
399 domain
= bo
->allowed_domains
;
406 /* Last resort, try to evict something from the current working set */
407 static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser
*p
,
408 struct amdgpu_bo
*validated
)
410 uint32_t domain
= validated
->allowed_domains
;
416 for (;&p
->evictable
->tv
.head
!= &p
->validated
;
417 p
->evictable
= list_prev_entry(p
->evictable
, tv
.head
)) {
419 struct amdgpu_bo_list_entry
*candidate
= p
->evictable
;
420 struct amdgpu_bo
*bo
= candidate
->robj
;
421 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->tbo
.bdev
);
422 u64 initial_bytes_moved
;
425 /* If we reached our current BO we can forget it */
426 if (candidate
->robj
== validated
)
429 other
= amdgpu_mem_type_to_domain(bo
->tbo
.mem
.mem_type
);
431 /* Check if this BO is in one of the domains we need space for */
432 if (!(other
& domain
))
435 /* Check if we can move this BO somewhere else */
436 other
= bo
->allowed_domains
& ~domain
;
440 /* Good we can try to move this BO somewhere else */
441 amdgpu_ttm_placement_from_domain(bo
, other
);
442 initial_bytes_moved
= atomic64_read(&adev
->num_bytes_moved
);
443 r
= ttm_bo_validate(&bo
->tbo
, &bo
->placement
, true, false);
444 p
->bytes_moved
+= atomic64_read(&adev
->num_bytes_moved
) -
450 p
->evictable
= list_prev_entry(p
->evictable
, tv
.head
);
451 list_move(&candidate
->tv
.head
, &p
->validated
);
459 static int amdgpu_cs_validate(void *param
, struct amdgpu_bo
*bo
)
461 struct amdgpu_cs_parser
*p
= param
;
465 r
= amdgpu_cs_bo_validate(p
, bo
);
466 } while (r
== -ENOMEM
&& amdgpu_cs_try_evict(p
, bo
));
471 r
= amdgpu_cs_bo_validate(p
, bo
->shadow
);
476 static int amdgpu_cs_list_validate(struct amdgpu_cs_parser
*p
,
477 struct list_head
*validated
)
479 struct amdgpu_bo_list_entry
*lobj
;
482 list_for_each_entry(lobj
, validated
, tv
.head
) {
483 struct amdgpu_bo
*bo
= lobj
->robj
;
484 bool binding_userptr
= false;
485 struct mm_struct
*usermm
;
487 usermm
= amdgpu_ttm_tt_get_usermm(bo
->tbo
.ttm
);
488 if (usermm
&& usermm
!= current
->mm
)
491 /* Check if we have user pages and nobody bound the BO already */
492 if (lobj
->user_pages
&& bo
->tbo
.ttm
->state
!= tt_bound
) {
493 size_t size
= sizeof(struct page
*);
495 size
*= bo
->tbo
.ttm
->num_pages
;
496 memcpy(bo
->tbo
.ttm
->pages
, lobj
->user_pages
, size
);
497 binding_userptr
= true;
500 if (p
->evictable
== lobj
)
503 r
= amdgpu_cs_validate(p
, bo
);
507 if (binding_userptr
) {
508 drm_free_large(lobj
->user_pages
);
509 lobj
->user_pages
= NULL
;
515 static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser
*p
,
516 union drm_amdgpu_cs
*cs
)
518 struct amdgpu_fpriv
*fpriv
= p
->filp
->driver_priv
;
519 struct amdgpu_bo_list_entry
*e
;
520 struct list_head duplicates
;
521 bool need_mmap_lock
= false;
522 unsigned i
, tries
= 10;
525 INIT_LIST_HEAD(&p
->validated
);
527 p
->bo_list
= amdgpu_bo_list_get(fpriv
, cs
->in
.bo_list_handle
);
529 need_mmap_lock
= p
->bo_list
->first_userptr
!=
530 p
->bo_list
->num_entries
;
531 amdgpu_bo_list_get_list(p
->bo_list
, &p
->validated
);
534 INIT_LIST_HEAD(&duplicates
);
535 amdgpu_vm_get_pd_bo(&fpriv
->vm
, &p
->validated
, &p
->vm_pd
);
537 if (p
->uf_entry
.robj
)
538 list_add(&p
->uf_entry
.tv
.head
, &p
->validated
);
541 down_read(¤t
->mm
->mmap_sem
);
544 struct list_head need_pages
;
547 r
= ttm_eu_reserve_buffers(&p
->ticket
, &p
->validated
, true,
549 if (unlikely(r
!= 0)) {
550 if (r
!= -ERESTARTSYS
)
551 DRM_ERROR("ttm_eu_reserve_buffers failed.\n");
552 goto error_free_pages
;
555 /* Without a BO list we don't have userptr BOs */
559 INIT_LIST_HEAD(&need_pages
);
560 for (i
= p
->bo_list
->first_userptr
;
561 i
< p
->bo_list
->num_entries
; ++i
) {
563 e
= &p
->bo_list
->array
[i
];
565 if (amdgpu_ttm_tt_userptr_invalidated(e
->robj
->tbo
.ttm
,
566 &e
->user_invalidated
) && e
->user_pages
) {
568 /* We acquired a page array, but somebody
569 * invalidated it. Free it an try again
571 release_pages(e
->user_pages
,
572 e
->robj
->tbo
.ttm
->num_pages
,
574 drm_free_large(e
->user_pages
);
575 e
->user_pages
= NULL
;
578 if (e
->robj
->tbo
.ttm
->state
!= tt_bound
&&
580 list_del(&e
->tv
.head
);
581 list_add(&e
->tv
.head
, &need_pages
);
583 amdgpu_bo_unreserve(e
->robj
);
587 if (list_empty(&need_pages
))
590 /* Unreserve everything again. */
591 ttm_eu_backoff_reservation(&p
->ticket
, &p
->validated
);
593 /* We tried too many times, just abort */
596 DRM_ERROR("deadlock in %s\n", __func__
);
597 goto error_free_pages
;
600 /* Fill the page arrays for all useptrs. */
601 list_for_each_entry(e
, &need_pages
, tv
.head
) {
602 struct ttm_tt
*ttm
= e
->robj
->tbo
.ttm
;
604 e
->user_pages
= drm_calloc_large(ttm
->num_pages
,
605 sizeof(struct page
*));
606 if (!e
->user_pages
) {
608 DRM_ERROR("calloc failure in %s\n", __func__
);
609 goto error_free_pages
;
612 r
= amdgpu_ttm_tt_get_user_pages(ttm
, e
->user_pages
);
614 DRM_ERROR("amdgpu_ttm_tt_get_user_pages failed.\n");
615 drm_free_large(e
->user_pages
);
616 e
->user_pages
= NULL
;
617 goto error_free_pages
;
622 list_splice(&need_pages
, &p
->validated
);
625 p
->bytes_moved_threshold
= amdgpu_cs_get_threshold_for_moves(p
->adev
);
627 p
->evictable
= list_last_entry(&p
->validated
,
628 struct amdgpu_bo_list_entry
,
631 r
= amdgpu_vm_validate_pt_bos(p
->adev
, &fpriv
->vm
,
632 amdgpu_cs_validate
, p
);
634 DRM_ERROR("amdgpu_vm_validate_pt_bos() failed.\n");
638 r
= amdgpu_cs_list_validate(p
, &duplicates
);
640 DRM_ERROR("amdgpu_cs_list_validate(duplicates) failed.\n");
644 r
= amdgpu_cs_list_validate(p
, &p
->validated
);
646 DRM_ERROR("amdgpu_cs_list_validate(validated) failed.\n");
650 amdgpu_cs_report_moved_bytes(p
->adev
, p
->bytes_moved
);
652 fpriv
->vm
.last_eviction_counter
=
653 atomic64_read(&p
->adev
->num_evictions
);
656 struct amdgpu_bo
*gds
= p
->bo_list
->gds_obj
;
657 struct amdgpu_bo
*gws
= p
->bo_list
->gws_obj
;
658 struct amdgpu_bo
*oa
= p
->bo_list
->oa_obj
;
659 struct amdgpu_vm
*vm
= &fpriv
->vm
;
662 for (i
= 0; i
< p
->bo_list
->num_entries
; i
++) {
663 struct amdgpu_bo
*bo
= p
->bo_list
->array
[i
].robj
;
665 p
->bo_list
->array
[i
].bo_va
= amdgpu_vm_bo_find(vm
, bo
);
669 p
->job
->gds_base
= amdgpu_bo_gpu_offset(gds
);
670 p
->job
->gds_size
= amdgpu_bo_size(gds
);
673 p
->job
->gws_base
= amdgpu_bo_gpu_offset(gws
);
674 p
->job
->gws_size
= amdgpu_bo_size(gws
);
677 p
->job
->oa_base
= amdgpu_bo_gpu_offset(oa
);
678 p
->job
->oa_size
= amdgpu_bo_size(oa
);
682 if (!r
&& p
->uf_entry
.robj
) {
683 struct amdgpu_bo
*uf
= p
->uf_entry
.robj
;
685 r
= amdgpu_ttm_bind(&uf
->tbo
, &uf
->tbo
.mem
);
686 p
->job
->uf_addr
+= amdgpu_bo_gpu_offset(uf
);
691 amdgpu_vm_move_pt_bos_in_lru(p
->adev
, &fpriv
->vm
);
692 ttm_eu_backoff_reservation(&p
->ticket
, &p
->validated
);
698 up_read(¤t
->mm
->mmap_sem
);
701 for (i
= p
->bo_list
->first_userptr
;
702 i
< p
->bo_list
->num_entries
; ++i
) {
703 e
= &p
->bo_list
->array
[i
];
708 release_pages(e
->user_pages
,
709 e
->robj
->tbo
.ttm
->num_pages
,
711 drm_free_large(e
->user_pages
);
718 static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser
*p
)
720 struct amdgpu_bo_list_entry
*e
;
723 list_for_each_entry(e
, &p
->validated
, tv
.head
) {
724 struct reservation_object
*resv
= e
->robj
->tbo
.resv
;
725 r
= amdgpu_sync_resv(p
->adev
, &p
->job
->sync
, resv
, p
->filp
);
734 * cs_parser_fini() - clean parser states
735 * @parser: parser structure holding parsing context.
736 * @error: error number
738 * If error is set than unvalidate buffer, otherwise just free memory
739 * used by parsing context.
741 static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser
*parser
, int error
, bool backoff
)
743 struct amdgpu_fpriv
*fpriv
= parser
->filp
->driver_priv
;
747 amdgpu_vm_move_pt_bos_in_lru(parser
->adev
, &fpriv
->vm
);
749 ttm_eu_fence_buffer_objects(&parser
->ticket
,
752 } else if (backoff
) {
753 ttm_eu_backoff_reservation(&parser
->ticket
,
756 dma_fence_put(parser
->fence
);
759 amdgpu_ctx_put(parser
->ctx
);
761 amdgpu_bo_list_put(parser
->bo_list
);
763 for (i
= 0; i
< parser
->nchunks
; i
++)
764 drm_free_large(parser
->chunks
[i
].kdata
);
765 kfree(parser
->chunks
);
767 amdgpu_job_free(parser
->job
);
768 amdgpu_bo_unref(&parser
->uf_entry
.robj
);
771 static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser
*p
)
773 struct amdgpu_device
*adev
= p
->adev
;
774 struct amdgpu_fpriv
*fpriv
= p
->filp
->driver_priv
;
775 struct amdgpu_vm
*vm
= &fpriv
->vm
;
776 struct amdgpu_bo_va
*bo_va
;
777 struct amdgpu_bo
*bo
;
780 r
= amdgpu_vm_update_directories(adev
, vm
);
784 r
= amdgpu_sync_fence(adev
, &p
->job
->sync
, vm
->last_dir_update
);
788 r
= amdgpu_vm_clear_freed(adev
, vm
, NULL
);
792 r
= amdgpu_vm_bo_update(adev
, fpriv
->prt_va
, false);
796 r
= amdgpu_sync_fence(adev
, &p
->job
->sync
,
797 fpriv
->prt_va
->last_pt_update
);
801 if (amdgpu_sriov_vf(adev
)) {
803 bo_va
= vm
->csa_bo_va
;
805 r
= amdgpu_vm_bo_update(adev
, bo_va
, false);
809 f
= bo_va
->last_pt_update
;
810 r
= amdgpu_sync_fence(adev
, &p
->job
->sync
, f
);
816 for (i
= 0; i
< p
->bo_list
->num_entries
; i
++) {
819 /* ignore duplicates */
820 bo
= p
->bo_list
->array
[i
].robj
;
824 bo_va
= p
->bo_list
->array
[i
].bo_va
;
828 r
= amdgpu_vm_bo_update(adev
, bo_va
, false);
832 f
= bo_va
->last_pt_update
;
833 r
= amdgpu_sync_fence(adev
, &p
->job
->sync
, f
);
840 r
= amdgpu_vm_clear_invalids(adev
, vm
, &p
->job
->sync
);
842 if (amdgpu_vm_debug
&& p
->bo_list
) {
843 /* Invalidate all BOs to test for userspace bugs */
844 for (i
= 0; i
< p
->bo_list
->num_entries
; i
++) {
845 /* ignore duplicates */
846 bo
= p
->bo_list
->array
[i
].robj
;
850 amdgpu_vm_bo_invalidate(adev
, bo
);
857 static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device
*adev
,
858 struct amdgpu_cs_parser
*p
)
860 struct amdgpu_fpriv
*fpriv
= p
->filp
->driver_priv
;
861 struct amdgpu_vm
*vm
= &fpriv
->vm
;
862 struct amdgpu_ring
*ring
= p
->job
->ring
;
865 /* Only for UVD/VCE VM emulation */
866 if (ring
->funcs
->parse_cs
) {
867 for (i
= 0; i
< p
->job
->num_ibs
; i
++) {
868 r
= amdgpu_ring_parse_cs(ring
, p
, i
);
875 p
->job
->vm_pd_addr
= amdgpu_bo_gpu_offset(vm
->root
.bo
);
877 r
= amdgpu_bo_vm_update_pte(p
);
882 return amdgpu_cs_sync_rings(p
);
885 static int amdgpu_cs_ib_fill(struct amdgpu_device
*adev
,
886 struct amdgpu_cs_parser
*parser
)
888 struct amdgpu_fpriv
*fpriv
= parser
->filp
->driver_priv
;
889 struct amdgpu_vm
*vm
= &fpriv
->vm
;
891 int r
, ce_preempt
= 0, de_preempt
= 0;
893 for (i
= 0, j
= 0; i
< parser
->nchunks
&& j
< parser
->job
->num_ibs
; i
++) {
894 struct amdgpu_cs_chunk
*chunk
;
895 struct amdgpu_ib
*ib
;
896 struct drm_amdgpu_cs_chunk_ib
*chunk_ib
;
897 struct amdgpu_ring
*ring
;
899 chunk
= &parser
->chunks
[i
];
900 ib
= &parser
->job
->ibs
[j
];
901 chunk_ib
= (struct drm_amdgpu_cs_chunk_ib
*)chunk
->kdata
;
903 if (chunk
->chunk_id
!= AMDGPU_CHUNK_ID_IB
)
906 if (chunk_ib
->ip_type
== AMDGPU_HW_IP_GFX
&& amdgpu_sriov_vf(adev
)) {
907 if (chunk_ib
->flags
& AMDGPU_IB_FLAG_PREEMPT
) {
908 if (chunk_ib
->flags
& AMDGPU_IB_FLAG_CE
)
914 /* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */
915 if (ce_preempt
> 1 || de_preempt
> 1)
919 r
= amdgpu_cs_get_ring(adev
, chunk_ib
->ip_type
,
920 chunk_ib
->ip_instance
, chunk_ib
->ring
,
925 if (chunk_ib
->flags
& AMDGPU_IB_FLAG_PREAMBLE
) {
926 parser
->job
->preamble_status
|= AMDGPU_PREAMBLE_IB_PRESENT
;
927 if (!parser
->ctx
->preamble_presented
) {
928 parser
->job
->preamble_status
|= AMDGPU_PREAMBLE_IB_PRESENT_FIRST
;
929 parser
->ctx
->preamble_presented
= true;
933 if (parser
->job
->ring
&& parser
->job
->ring
!= ring
)
936 parser
->job
->ring
= ring
;
938 if (ring
->funcs
->parse_cs
) {
939 struct amdgpu_bo_va_mapping
*m
;
940 struct amdgpu_bo
*aobj
= NULL
;
944 m
= amdgpu_cs_find_mapping(parser
, chunk_ib
->va_start
,
947 DRM_ERROR("IB va_start is invalid\n");
951 if ((chunk_ib
->va_start
+ chunk_ib
->ib_bytes
) >
952 (m
->last
+ 1) * AMDGPU_GPU_PAGE_SIZE
) {
953 DRM_ERROR("IB va_start+ib_bytes is invalid\n");
957 /* the IB should be reserved at this point */
958 r
= amdgpu_bo_kmap(aobj
, (void **)&kptr
);
963 offset
= m
->start
* AMDGPU_GPU_PAGE_SIZE
;
964 kptr
+= chunk_ib
->va_start
- offset
;
966 r
= amdgpu_ib_get(adev
, vm
, chunk_ib
->ib_bytes
, ib
);
968 DRM_ERROR("Failed to get ib !\n");
972 memcpy(ib
->ptr
, kptr
, chunk_ib
->ib_bytes
);
973 amdgpu_bo_kunmap(aobj
);
975 r
= amdgpu_ib_get(adev
, vm
, 0, ib
);
977 DRM_ERROR("Failed to get ib !\n");
983 ib
->gpu_addr
= chunk_ib
->va_start
;
984 ib
->length_dw
= chunk_ib
->ib_bytes
/ 4;
985 ib
->flags
= chunk_ib
->flags
;
989 /* UVD & VCE fw doesn't support user fences */
990 if (parser
->job
->uf_addr
&& (
991 parser
->job
->ring
->funcs
->type
== AMDGPU_RING_TYPE_UVD
||
992 parser
->job
->ring
->funcs
->type
== AMDGPU_RING_TYPE_VCE
))
998 static int amdgpu_cs_dependencies(struct amdgpu_device
*adev
,
999 struct amdgpu_cs_parser
*p
)
1001 struct amdgpu_fpriv
*fpriv
= p
->filp
->driver_priv
;
1004 for (i
= 0; i
< p
->nchunks
; ++i
) {
1005 struct drm_amdgpu_cs_chunk_dep
*deps
;
1006 struct amdgpu_cs_chunk
*chunk
;
1009 chunk
= &p
->chunks
[i
];
1011 if (chunk
->chunk_id
!= AMDGPU_CHUNK_ID_DEPENDENCIES
)
1014 deps
= (struct drm_amdgpu_cs_chunk_dep
*)chunk
->kdata
;
1015 num_deps
= chunk
->length_dw
* 4 /
1016 sizeof(struct drm_amdgpu_cs_chunk_dep
);
1018 for (j
= 0; j
< num_deps
; ++j
) {
1019 struct amdgpu_ring
*ring
;
1020 struct amdgpu_ctx
*ctx
;
1021 struct dma_fence
*fence
;
1023 r
= amdgpu_cs_get_ring(adev
, deps
[j
].ip_type
,
1024 deps
[j
].ip_instance
,
1025 deps
[j
].ring
, &ring
);
1029 ctx
= amdgpu_ctx_get(fpriv
, deps
[j
].ctx_id
);
1033 fence
= amdgpu_ctx_get_fence(ctx
, ring
,
1035 if (IS_ERR(fence
)) {
1037 amdgpu_ctx_put(ctx
);
1041 r
= amdgpu_sync_fence(adev
, &p
->job
->sync
,
1043 dma_fence_put(fence
);
1044 amdgpu_ctx_put(ctx
);
1054 static int amdgpu_cs_submit(struct amdgpu_cs_parser
*p
,
1055 union drm_amdgpu_cs
*cs
)
1057 struct amdgpu_ring
*ring
= p
->job
->ring
;
1058 struct amd_sched_entity
*entity
= &p
->ctx
->rings
[ring
->idx
].entity
;
1059 struct amdgpu_job
*job
;
1065 r
= amd_sched_job_init(&job
->base
, &ring
->sched
, entity
, p
->filp
);
1067 amdgpu_job_free(job
);
1071 job
->owner
= p
->filp
;
1072 job
->fence_ctx
= entity
->fence_context
;
1073 p
->fence
= dma_fence_get(&job
->base
.s_fence
->finished
);
1074 cs
->out
.handle
= amdgpu_ctx_add_fence(p
->ctx
, ring
, p
->fence
);
1075 job
->uf_sequence
= cs
->out
.handle
;
1076 amdgpu_job_free_resources(job
);
1078 trace_amdgpu_cs_ioctl(job
);
1079 amd_sched_entity_push_job(&job
->base
);
1084 int amdgpu_cs_ioctl(struct drm_device
*dev
, void *data
, struct drm_file
*filp
)
1086 struct amdgpu_device
*adev
= dev
->dev_private
;
1087 union drm_amdgpu_cs
*cs
= data
;
1088 struct amdgpu_cs_parser parser
= {};
1089 bool reserved_buffers
= false;
1092 if (!adev
->accel_working
)
1098 r
= amdgpu_cs_parser_init(&parser
, data
);
1100 DRM_ERROR("Failed to initialize parser !\n");
1104 r
= amdgpu_cs_parser_bos(&parser
, data
);
1107 DRM_ERROR("Not enough memory for command submission!\n");
1108 else if (r
!= -ERESTARTSYS
)
1109 DRM_ERROR("Failed to process the buffer list %d!\n", r
);
1113 reserved_buffers
= true;
1114 r
= amdgpu_cs_ib_fill(adev
, &parser
);
1118 r
= amdgpu_cs_dependencies(adev
, &parser
);
1120 DRM_ERROR("Failed in the dependencies handling %d!\n", r
);
1124 for (i
= 0; i
< parser
.job
->num_ibs
; i
++)
1125 trace_amdgpu_cs(&parser
, i
);
1127 r
= amdgpu_cs_ib_vm_chunk(adev
, &parser
);
1131 r
= amdgpu_cs_submit(&parser
, cs
);
1134 amdgpu_cs_parser_fini(&parser
, r
, reserved_buffers
);
1139 * amdgpu_cs_wait_ioctl - wait for a command submission to finish
1142 * @data: data from userspace
1143 * @filp: file private
1145 * Wait for the command submission identified by handle to finish.
1147 int amdgpu_cs_wait_ioctl(struct drm_device
*dev
, void *data
,
1148 struct drm_file
*filp
)
1150 union drm_amdgpu_wait_cs
*wait
= data
;
1151 struct amdgpu_device
*adev
= dev
->dev_private
;
1152 unsigned long timeout
= amdgpu_gem_timeout(wait
->in
.timeout
);
1153 struct amdgpu_ring
*ring
= NULL
;
1154 struct amdgpu_ctx
*ctx
;
1155 struct dma_fence
*fence
;
1158 r
= amdgpu_cs_get_ring(adev
, wait
->in
.ip_type
, wait
->in
.ip_instance
,
1159 wait
->in
.ring
, &ring
);
1163 ctx
= amdgpu_ctx_get(filp
->driver_priv
, wait
->in
.ctx_id
);
1167 fence
= amdgpu_ctx_get_fence(ctx
, ring
, wait
->in
.handle
);
1171 r
= dma_fence_wait_timeout(fence
, true, timeout
);
1172 dma_fence_put(fence
);
1176 amdgpu_ctx_put(ctx
);
1180 memset(wait
, 0, sizeof(*wait
));
1181 wait
->out
.status
= (r
== 0);
1187 * amdgpu_cs_get_fence - helper to get fence from drm_amdgpu_fence
1189 * @adev: amdgpu device
1190 * @filp: file private
1191 * @user: drm_amdgpu_fence copied from user space
1193 static struct dma_fence
*amdgpu_cs_get_fence(struct amdgpu_device
*adev
,
1194 struct drm_file
*filp
,
1195 struct drm_amdgpu_fence
*user
)
1197 struct amdgpu_ring
*ring
;
1198 struct amdgpu_ctx
*ctx
;
1199 struct dma_fence
*fence
;
1202 r
= amdgpu_cs_get_ring(adev
, user
->ip_type
, user
->ip_instance
,
1207 ctx
= amdgpu_ctx_get(filp
->driver_priv
, user
->ctx_id
);
1209 return ERR_PTR(-EINVAL
);
1211 fence
= amdgpu_ctx_get_fence(ctx
, ring
, user
->seq_no
);
1212 amdgpu_ctx_put(ctx
);
1218 * amdgpu_cs_wait_all_fence - wait on all fences to signal
1220 * @adev: amdgpu device
1221 * @filp: file private
1222 * @wait: wait parameters
1223 * @fences: array of drm_amdgpu_fence
1225 static int amdgpu_cs_wait_all_fences(struct amdgpu_device
*adev
,
1226 struct drm_file
*filp
,
1227 union drm_amdgpu_wait_fences
*wait
,
1228 struct drm_amdgpu_fence
*fences
)
1230 uint32_t fence_count
= wait
->in
.fence_count
;
1234 for (i
= 0; i
< fence_count
; i
++) {
1235 struct dma_fence
*fence
;
1236 unsigned long timeout
= amdgpu_gem_timeout(wait
->in
.timeout_ns
);
1238 fence
= amdgpu_cs_get_fence(adev
, filp
, &fences
[i
]);
1240 return PTR_ERR(fence
);
1244 r
= dma_fence_wait_timeout(fence
, true, timeout
);
1245 dma_fence_put(fence
);
1253 memset(wait
, 0, sizeof(*wait
));
1254 wait
->out
.status
= (r
> 0);
1260 * amdgpu_cs_wait_any_fence - wait on any fence to signal
1262 * @adev: amdgpu device
1263 * @filp: file private
1264 * @wait: wait parameters
1265 * @fences: array of drm_amdgpu_fence
1267 static int amdgpu_cs_wait_any_fence(struct amdgpu_device
*adev
,
1268 struct drm_file
*filp
,
1269 union drm_amdgpu_wait_fences
*wait
,
1270 struct drm_amdgpu_fence
*fences
)
1272 unsigned long timeout
= amdgpu_gem_timeout(wait
->in
.timeout_ns
);
1273 uint32_t fence_count
= wait
->in
.fence_count
;
1274 uint32_t first
= ~0;
1275 struct dma_fence
**array
;
1279 /* Prepare the fence array */
1280 array
= kcalloc(fence_count
, sizeof(struct dma_fence
*), GFP_KERNEL
);
1285 for (i
= 0; i
< fence_count
; i
++) {
1286 struct dma_fence
*fence
;
1288 fence
= amdgpu_cs_get_fence(adev
, filp
, &fences
[i
]);
1289 if (IS_ERR(fence
)) {
1291 goto err_free_fence_array
;
1294 } else { /* NULL, the fence has been already signaled */
1300 r
= dma_fence_wait_any_timeout(array
, fence_count
, true, timeout
,
1303 goto err_free_fence_array
;
1306 memset(wait
, 0, sizeof(*wait
));
1307 wait
->out
.status
= (r
> 0);
1308 wait
->out
.first_signaled
= first
;
1309 /* set return value 0 to indicate success */
1312 err_free_fence_array
:
1313 for (i
= 0; i
< fence_count
; i
++)
1314 dma_fence_put(array
[i
]);
1321 * amdgpu_cs_wait_fences_ioctl - wait for multiple command submissions to finish
1324 * @data: data from userspace
1325 * @filp: file private
1327 int amdgpu_cs_wait_fences_ioctl(struct drm_device
*dev
, void *data
,
1328 struct drm_file
*filp
)
1330 struct amdgpu_device
*adev
= dev
->dev_private
;
1331 union drm_amdgpu_wait_fences
*wait
= data
;
1332 uint32_t fence_count
= wait
->in
.fence_count
;
1333 struct drm_amdgpu_fence
*fences_user
;
1334 struct drm_amdgpu_fence
*fences
;
1337 /* Get the fences from userspace */
1338 fences
= kmalloc_array(fence_count
, sizeof(struct drm_amdgpu_fence
),
1343 fences_user
= (void __user
*)(uintptr_t)(wait
->in
.fences
);
1344 if (copy_from_user(fences
, fences_user
,
1345 sizeof(struct drm_amdgpu_fence
) * fence_count
)) {
1347 goto err_free_fences
;
1350 if (wait
->in
.wait_all
)
1351 r
= amdgpu_cs_wait_all_fences(adev
, filp
, wait
, fences
);
1353 r
= amdgpu_cs_wait_any_fence(adev
, filp
, wait
, fences
);
1362 * amdgpu_cs_find_bo_va - find bo_va for VM address
1364 * @parser: command submission parser context
1366 * @bo: resulting BO of the mapping found
1368 * Search the buffer objects in the command submission context for a certain
1369 * virtual memory address. Returns allocation structure when found, NULL
1372 struct amdgpu_bo_va_mapping
*
1373 amdgpu_cs_find_mapping(struct amdgpu_cs_parser
*parser
,
1374 uint64_t addr
, struct amdgpu_bo
**bo
)
1376 struct amdgpu_bo_va_mapping
*mapping
;
1379 if (!parser
->bo_list
)
1382 addr
/= AMDGPU_GPU_PAGE_SIZE
;
1384 for (i
= 0; i
< parser
->bo_list
->num_entries
; i
++) {
1385 struct amdgpu_bo_list_entry
*lobj
;
1387 lobj
= &parser
->bo_list
->array
[i
];
1391 list_for_each_entry(mapping
, &lobj
->bo_va
->valids
, list
) {
1392 if (mapping
->start
> addr
||
1393 addr
> mapping
->last
)
1396 *bo
= lobj
->bo_va
->bo
;
1400 list_for_each_entry(mapping
, &lobj
->bo_va
->invalids
, list
) {
1401 if (mapping
->start
> addr
||
1402 addr
> mapping
->last
)
1405 *bo
= lobj
->bo_va
->bo
;
1414 * amdgpu_cs_sysvm_access_required - make BOs accessible by the system VM
1416 * @parser: command submission parser context
1418 * Helper for UVD/VCE VM emulation, make sure BOs are accessible by the system VM.
1420 int amdgpu_cs_sysvm_access_required(struct amdgpu_cs_parser
*parser
)
1425 if (!parser
->bo_list
)
1428 for (i
= 0; i
< parser
->bo_list
->num_entries
; i
++) {
1429 struct amdgpu_bo
*bo
= parser
->bo_list
->array
[i
].robj
;
1431 r
= amdgpu_ttm_bind(&bo
->tbo
, &bo
->tbo
.mem
);
1435 if (bo
->flags
& AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS
)
1438 bo
->flags
|= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS
;
1439 amdgpu_ttm_placement_from_domain(bo
, bo
->allowed_domains
);
1440 r
= ttm_bo_validate(&bo
->tbo
, &bo
->placement
, false, false);