2 * Copyright 2009 Jerome Glisse.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
28 * Jerome Glisse <glisse@freedesktop.org>
29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
32 #include <linux/list.h>
33 #include <linux/slab.h>
34 #include <linux/dma-buf.h>
36 #include <drm/amdgpu_drm.h>
37 #include <drm/drm_cache.h>
39 #include "amdgpu_trace.h"
40 #include "amdgpu_amdkfd.h"
45 * This defines the interfaces to operate on an &amdgpu_bo buffer object which
46 * represents memory used by driver (VRAM, system memory, etc.). The driver
47 * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces
48 * to create/destroy/set buffer object which are then managed by the kernel TTM
50 * The interfaces are also used internally by kernel clients, including gfx,
51 * uvd, etc. for kernel managed allocations used by the GPU.
55 static void amdgpu_bo_destroy(struct ttm_buffer_object
*tbo
)
57 struct amdgpu_bo
*bo
= ttm_to_amdgpu_bo(tbo
);
61 if (bo
->tbo
.base
.import_attach
)
62 drm_prime_gem_destroy(&bo
->tbo
.base
, bo
->tbo
.sg
);
63 drm_gem_object_release(&bo
->tbo
.base
);
64 amdgpu_bo_unref(&bo
->parent
);
68 static void amdgpu_bo_user_destroy(struct ttm_buffer_object
*tbo
)
70 struct amdgpu_bo
*bo
= ttm_to_amdgpu_bo(tbo
);
71 struct amdgpu_bo_user
*ubo
;
73 ubo
= to_amdgpu_bo_user(bo
);
75 amdgpu_bo_destroy(tbo
);
78 static void amdgpu_bo_vm_destroy(struct ttm_buffer_object
*tbo
)
80 struct amdgpu_device
*adev
= amdgpu_ttm_adev(tbo
->bdev
);
81 struct amdgpu_bo
*bo
= ttm_to_amdgpu_bo(tbo
);
82 struct amdgpu_bo_vm
*vmbo
;
84 vmbo
= to_amdgpu_bo_vm(bo
);
85 /* in case amdgpu_device_recover_vram got NULL of bo->parent */
86 if (!list_empty(&vmbo
->shadow_list
)) {
87 mutex_lock(&adev
->shadow_list_lock
);
88 list_del_init(&vmbo
->shadow_list
);
89 mutex_unlock(&adev
->shadow_list_lock
);
92 amdgpu_bo_destroy(tbo
);
96 * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo
97 * @bo: buffer object to be checked
99 * Uses destroy function associated with the object to determine if this is
103 * true if the object belongs to &amdgpu_bo, false if not.
105 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object
*bo
)
107 if (bo
->destroy
== &amdgpu_bo_destroy
||
108 bo
->destroy
== &amdgpu_bo_user_destroy
||
109 bo
->destroy
== &amdgpu_bo_vm_destroy
)
116 * amdgpu_bo_placement_from_domain - set buffer's placement
117 * @abo: &amdgpu_bo buffer object whose placement is to be set
118 * @domain: requested domain
120 * Sets buffer's placement according to requested domain and the buffer's
123 void amdgpu_bo_placement_from_domain(struct amdgpu_bo
*abo
, u32 domain
)
125 struct amdgpu_device
*adev
= amdgpu_ttm_adev(abo
->tbo
.bdev
);
126 struct ttm_placement
*placement
= &abo
->placement
;
127 struct ttm_place
*places
= abo
->placements
;
128 u64 flags
= abo
->flags
;
131 if (domain
& AMDGPU_GEM_DOMAIN_VRAM
) {
132 unsigned visible_pfn
= adev
->gmc
.visible_vram_size
>> PAGE_SHIFT
;
136 places
[c
].mem_type
= TTM_PL_VRAM
;
139 if (flags
& AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
)
140 places
[c
].lpfn
= visible_pfn
;
142 places
[c
].flags
|= TTM_PL_FLAG_TOPDOWN
;
144 if (flags
& AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS
)
145 places
[c
].flags
|= TTM_PL_FLAG_CONTIGUOUS
;
149 if (domain
& AMDGPU_GEM_DOMAIN_GTT
) {
153 abo
->flags
& AMDGPU_GEM_CREATE_PREEMPTIBLE
?
154 AMDGPU_PL_PREEMPT
: TTM_PL_TT
;
159 if (domain
& AMDGPU_GEM_DOMAIN_CPU
) {
162 places
[c
].mem_type
= TTM_PL_SYSTEM
;
167 if (domain
& AMDGPU_GEM_DOMAIN_GDS
) {
170 places
[c
].mem_type
= AMDGPU_PL_GDS
;
175 if (domain
& AMDGPU_GEM_DOMAIN_GWS
) {
178 places
[c
].mem_type
= AMDGPU_PL_GWS
;
183 if (domain
& AMDGPU_GEM_DOMAIN_OA
) {
186 places
[c
].mem_type
= AMDGPU_PL_OA
;
194 places
[c
].mem_type
= TTM_PL_SYSTEM
;
199 BUG_ON(c
> AMDGPU_BO_MAX_PLACEMENTS
);
201 placement
->num_placement
= c
;
202 placement
->placement
= places
;
204 placement
->num_busy_placement
= c
;
205 placement
->busy_placement
= places
;
209 * amdgpu_bo_create_reserved - create reserved BO for kernel use
211 * @adev: amdgpu device object
212 * @size: size for the new BO
213 * @align: alignment for the new BO
214 * @domain: where to place it
215 * @bo_ptr: used to initialize BOs in structures
216 * @gpu_addr: GPU addr of the pinned BO
217 * @cpu_addr: optional CPU address mapping
219 * Allocates and pins a BO for kernel internal use, and returns it still
222 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
225 * 0 on success, negative error code otherwise.
227 int amdgpu_bo_create_reserved(struct amdgpu_device
*adev
,
228 unsigned long size
, int align
,
229 u32 domain
, struct amdgpu_bo
**bo_ptr
,
230 u64
*gpu_addr
, void **cpu_addr
)
232 struct amdgpu_bo_param bp
;
237 amdgpu_bo_unref(bo_ptr
);
241 memset(&bp
, 0, sizeof(bp
));
243 bp
.byte_align
= align
;
245 bp
.flags
= cpu_addr
? AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
246 : AMDGPU_GEM_CREATE_NO_CPU_ACCESS
;
247 bp
.flags
|= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS
;
248 bp
.type
= ttm_bo_type_kernel
;
250 bp
.bo_ptr_size
= sizeof(struct amdgpu_bo
);
253 r
= amdgpu_bo_create(adev
, &bp
, bo_ptr
);
255 dev_err(adev
->dev
, "(%d) failed to allocate kernel bo\n",
262 r
= amdgpu_bo_reserve(*bo_ptr
, false);
264 dev_err(adev
->dev
, "(%d) failed to reserve kernel bo\n", r
);
268 r
= amdgpu_bo_pin(*bo_ptr
, domain
);
270 dev_err(adev
->dev
, "(%d) kernel bo pin failed\n", r
);
271 goto error_unreserve
;
274 r
= amdgpu_ttm_alloc_gart(&(*bo_ptr
)->tbo
);
276 dev_err(adev
->dev
, "%p bind failed\n", *bo_ptr
);
281 *gpu_addr
= amdgpu_bo_gpu_offset(*bo_ptr
);
284 r
= amdgpu_bo_kmap(*bo_ptr
, cpu_addr
);
286 dev_err(adev
->dev
, "(%d) kernel bo map failed\n", r
);
294 amdgpu_bo_unpin(*bo_ptr
);
296 amdgpu_bo_unreserve(*bo_ptr
);
300 amdgpu_bo_unref(bo_ptr
);
306 * amdgpu_bo_create_kernel - create BO for kernel use
308 * @adev: amdgpu device object
309 * @size: size for the new BO
310 * @align: alignment for the new BO
311 * @domain: where to place it
312 * @bo_ptr: used to initialize BOs in structures
313 * @gpu_addr: GPU addr of the pinned BO
314 * @cpu_addr: optional CPU address mapping
316 * Allocates and pins a BO for kernel internal use.
318 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
321 * 0 on success, negative error code otherwise.
323 int amdgpu_bo_create_kernel(struct amdgpu_device
*adev
,
324 unsigned long size
, int align
,
325 u32 domain
, struct amdgpu_bo
**bo_ptr
,
326 u64
*gpu_addr
, void **cpu_addr
)
330 r
= amdgpu_bo_create_reserved(adev
, size
, align
, domain
, bo_ptr
,
337 amdgpu_bo_unreserve(*bo_ptr
);
343 * amdgpu_bo_create_kernel_at - create BO for kernel use at specific location
345 * @adev: amdgpu device object
346 * @offset: offset of the BO
347 * @size: size of the BO
348 * @domain: where to place it
349 * @bo_ptr: used to initialize BOs in structures
350 * @cpu_addr: optional CPU address mapping
352 * Creates a kernel BO at a specific offset in the address space of the domain.
355 * 0 on success, negative error code otherwise.
357 int amdgpu_bo_create_kernel_at(struct amdgpu_device
*adev
,
358 uint64_t offset
, uint64_t size
, uint32_t domain
,
359 struct amdgpu_bo
**bo_ptr
, void **cpu_addr
)
361 struct ttm_operation_ctx ctx
= { false, false };
366 size
= ALIGN(size
, PAGE_SIZE
);
368 r
= amdgpu_bo_create_reserved(adev
, size
, PAGE_SIZE
, domain
, bo_ptr
,
373 if ((*bo_ptr
) == NULL
)
377 * Remove the original mem node and create a new one at the request
381 amdgpu_bo_kunmap(*bo_ptr
);
383 ttm_resource_free(&(*bo_ptr
)->tbo
, &(*bo_ptr
)->tbo
.resource
);
385 for (i
= 0; i
< (*bo_ptr
)->placement
.num_placement
; ++i
) {
386 (*bo_ptr
)->placements
[i
].fpfn
= offset
>> PAGE_SHIFT
;
387 (*bo_ptr
)->placements
[i
].lpfn
= (offset
+ size
) >> PAGE_SHIFT
;
389 r
= ttm_bo_mem_space(&(*bo_ptr
)->tbo
, &(*bo_ptr
)->placement
,
390 &(*bo_ptr
)->tbo
.resource
, &ctx
);
395 r
= amdgpu_bo_kmap(*bo_ptr
, cpu_addr
);
400 amdgpu_bo_unreserve(*bo_ptr
);
404 amdgpu_bo_unreserve(*bo_ptr
);
405 amdgpu_bo_unref(bo_ptr
);
410 * amdgpu_bo_free_kernel - free BO for kernel use
412 * @bo: amdgpu BO to free
413 * @gpu_addr: pointer to where the BO's GPU memory space address was stored
414 * @cpu_addr: pointer to where the BO's CPU memory space address was stored
416 * unmaps and unpin a BO for kernel internal use.
418 void amdgpu_bo_free_kernel(struct amdgpu_bo
**bo
, u64
*gpu_addr
,
424 if (likely(amdgpu_bo_reserve(*bo
, true) == 0)) {
426 amdgpu_bo_kunmap(*bo
);
428 amdgpu_bo_unpin(*bo
);
429 amdgpu_bo_unreserve(*bo
);
440 /* Validate bo size is bit bigger then the request domain */
441 static bool amdgpu_bo_validate_size(struct amdgpu_device
*adev
,
442 unsigned long size
, u32 domain
)
444 struct ttm_resource_manager
*man
= NULL
;
447 * If GTT is part of requested domains the check must succeed to
448 * allow fall back to GTT
450 if (domain
& AMDGPU_GEM_DOMAIN_GTT
) {
451 man
= ttm_manager_type(&adev
->mman
.bdev
, TTM_PL_TT
);
453 if (size
< (man
->size
<< PAGE_SHIFT
))
459 if (domain
& AMDGPU_GEM_DOMAIN_VRAM
) {
460 man
= ttm_manager_type(&adev
->mman
.bdev
, TTM_PL_VRAM
);
462 if (size
< (man
->size
<< PAGE_SHIFT
))
469 /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU */
473 DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size
,
474 man
->size
<< PAGE_SHIFT
);
478 bool amdgpu_bo_support_uswc(u64 bo_flags
)
482 /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit
483 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627
486 #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT)
487 /* Don't try to enable write-combining when it can't work, or things
489 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758
492 #ifndef CONFIG_COMPILE_TEST
493 #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \
494 thanks to write-combining
497 if (bo_flags
& AMDGPU_GEM_CREATE_CPU_GTT_USWC
)
498 DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for "
499 "better performance thanks to write-combining\n");
502 /* For architectures that don't support WC memory,
503 * mask out the WC flag from the BO
505 if (!drm_arch_can_wc_memory())
513 * amdgpu_bo_create - create an &amdgpu_bo buffer object
514 * @adev: amdgpu device object
515 * @bp: parameters to be used for the buffer object
516 * @bo_ptr: pointer to the buffer object pointer
518 * Creates an &amdgpu_bo buffer object.
521 * 0 for success or a negative error code on failure.
523 int amdgpu_bo_create(struct amdgpu_device
*adev
,
524 struct amdgpu_bo_param
*bp
,
525 struct amdgpu_bo
**bo_ptr
)
527 struct ttm_operation_ctx ctx
= {
528 .interruptible
= (bp
->type
!= ttm_bo_type_kernel
),
529 .no_wait_gpu
= bp
->no_wait_gpu
,
530 /* We opt to avoid OOM on system pages allocations */
531 .gfp_retry_mayfail
= true,
532 .allow_res_evict
= bp
->type
!= ttm_bo_type_kernel
,
535 struct amdgpu_bo
*bo
;
536 unsigned long page_align
, size
= bp
->size
;
539 /* Note that GDS/GWS/OA allocates 1 page per byte/resource. */
540 if (bp
->domain
& (AMDGPU_GEM_DOMAIN_GWS
| AMDGPU_GEM_DOMAIN_OA
)) {
541 /* GWS and OA don't need any alignment. */
542 page_align
= bp
->byte_align
;
544 } else if (bp
->domain
& AMDGPU_GEM_DOMAIN_GDS
) {
545 /* Both size and alignment must be a multiple of 4. */
546 page_align
= ALIGN(bp
->byte_align
, 4);
547 size
= ALIGN(size
, 4) << PAGE_SHIFT
;
549 /* Memory should be aligned at least to a page size. */
550 page_align
= ALIGN(bp
->byte_align
, PAGE_SIZE
) >> PAGE_SHIFT
;
551 size
= ALIGN(size
, PAGE_SIZE
);
554 if (!amdgpu_bo_validate_size(adev
, size
, bp
->domain
))
557 BUG_ON(bp
->bo_ptr_size
< sizeof(struct amdgpu_bo
));
560 bo
= kvzalloc(bp
->bo_ptr_size
, GFP_KERNEL
);
563 drm_gem_private_object_init(adev_to_drm(adev
), &bo
->tbo
.base
, size
);
565 bo
->preferred_domains
= bp
->preferred_domain
? bp
->preferred_domain
:
567 bo
->allowed_domains
= bo
->preferred_domains
;
568 if (bp
->type
!= ttm_bo_type_kernel
&&
569 bo
->allowed_domains
== AMDGPU_GEM_DOMAIN_VRAM
)
570 bo
->allowed_domains
|= AMDGPU_GEM_DOMAIN_GTT
;
572 bo
->flags
= bp
->flags
;
574 if (!amdgpu_bo_support_uswc(bo
->flags
))
575 bo
->flags
&= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC
;
577 bo
->tbo
.bdev
= &adev
->mman
.bdev
;
578 if (bp
->domain
& (AMDGPU_GEM_DOMAIN_GWS
| AMDGPU_GEM_DOMAIN_OA
|
579 AMDGPU_GEM_DOMAIN_GDS
))
580 amdgpu_bo_placement_from_domain(bo
, AMDGPU_GEM_DOMAIN_CPU
);
582 amdgpu_bo_placement_from_domain(bo
, bp
->domain
);
583 if (bp
->type
== ttm_bo_type_kernel
)
584 bo
->tbo
.priority
= 1;
587 bp
->destroy
= &amdgpu_bo_destroy
;
589 r
= ttm_bo_init_reserved(&adev
->mman
.bdev
, &bo
->tbo
, size
, bp
->type
,
590 &bo
->placement
, page_align
, &ctx
, NULL
,
591 bp
->resv
, bp
->destroy
);
592 if (unlikely(r
!= 0))
595 if (!amdgpu_gmc_vram_full_visible(&adev
->gmc
) &&
596 bo
->tbo
.resource
->mem_type
== TTM_PL_VRAM
&&
597 bo
->tbo
.resource
->start
< adev
->gmc
.visible_vram_size
>> PAGE_SHIFT
)
598 amdgpu_cs_report_moved_bytes(adev
, ctx
.bytes_moved
,
601 amdgpu_cs_report_moved_bytes(adev
, ctx
.bytes_moved
, 0);
603 if (bp
->flags
& AMDGPU_GEM_CREATE_VRAM_CLEARED
&&
604 bo
->tbo
.resource
->mem_type
== TTM_PL_VRAM
) {
605 struct dma_fence
*fence
;
607 r
= amdgpu_fill_buffer(bo
, 0, bo
->tbo
.base
.resv
, &fence
);
611 amdgpu_bo_fence(bo
, fence
, false);
612 dma_fence_put(bo
->tbo
.moving
);
613 bo
->tbo
.moving
= dma_fence_get(fence
);
614 dma_fence_put(fence
);
617 amdgpu_bo_unreserve(bo
);
620 trace_amdgpu_bo_create(bo
);
622 /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */
623 if (bp
->type
== ttm_bo_type_device
)
624 bo
->flags
&= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
;
630 dma_resv_unlock(bo
->tbo
.base
.resv
);
631 amdgpu_bo_unref(&bo
);
636 * amdgpu_bo_create_user - create an &amdgpu_bo_user buffer object
637 * @adev: amdgpu device object
638 * @bp: parameters to be used for the buffer object
639 * @ubo_ptr: pointer to the buffer object pointer
641 * Create a BO to be used by user application;
644 * 0 for success or a negative error code on failure.
647 int amdgpu_bo_create_user(struct amdgpu_device
*adev
,
648 struct amdgpu_bo_param
*bp
,
649 struct amdgpu_bo_user
**ubo_ptr
)
651 struct amdgpu_bo
*bo_ptr
;
654 bp
->bo_ptr_size
= sizeof(struct amdgpu_bo_user
);
655 bp
->destroy
= &amdgpu_bo_user_destroy
;
656 r
= amdgpu_bo_create(adev
, bp
, &bo_ptr
);
660 *ubo_ptr
= to_amdgpu_bo_user(bo_ptr
);
665 * amdgpu_bo_create_vm - create an &amdgpu_bo_vm buffer object
666 * @adev: amdgpu device object
667 * @bp: parameters to be used for the buffer object
668 * @vmbo_ptr: pointer to the buffer object pointer
670 * Create a BO to be for GPUVM.
673 * 0 for success or a negative error code on failure.
676 int amdgpu_bo_create_vm(struct amdgpu_device
*adev
,
677 struct amdgpu_bo_param
*bp
,
678 struct amdgpu_bo_vm
**vmbo_ptr
)
680 struct amdgpu_bo
*bo_ptr
;
683 /* bo_ptr_size will be determined by the caller and it depends on
684 * num of amdgpu_vm_pt entries.
686 BUG_ON(bp
->bo_ptr_size
< sizeof(struct amdgpu_bo_vm
));
687 bp
->destroy
= &amdgpu_bo_vm_destroy
;
688 r
= amdgpu_bo_create(adev
, bp
, &bo_ptr
);
692 *vmbo_ptr
= to_amdgpu_bo_vm(bo_ptr
);
693 INIT_LIST_HEAD(&(*vmbo_ptr
)->shadow_list
);
698 * amdgpu_bo_validate - validate an &amdgpu_bo buffer object
699 * @bo: pointer to the buffer object
701 * Sets placement according to domain; and changes placement and caching
702 * policy of the buffer object according to the placement.
703 * This is used for validating shadow bos. It calls ttm_bo_validate() to
704 * make sure the buffer is resident where it needs to be.
707 * 0 for success or a negative error code on failure.
709 int amdgpu_bo_validate(struct amdgpu_bo
*bo
)
711 struct ttm_operation_ctx ctx
= { false, false };
715 if (bo
->tbo
.pin_count
)
718 domain
= bo
->preferred_domains
;
721 amdgpu_bo_placement_from_domain(bo
, domain
);
722 r
= ttm_bo_validate(&bo
->tbo
, &bo
->placement
, &ctx
);
723 if (unlikely(r
== -ENOMEM
) && domain
!= bo
->allowed_domains
) {
724 domain
= bo
->allowed_domains
;
732 * amdgpu_bo_add_to_shadow_list - add a BO to the shadow list
734 * @vmbo: BO that will be inserted into the shadow list
736 * Insert a BO to the shadow list.
738 void amdgpu_bo_add_to_shadow_list(struct amdgpu_bo_vm
*vmbo
)
740 struct amdgpu_device
*adev
= amdgpu_ttm_adev(vmbo
->bo
.tbo
.bdev
);
742 mutex_lock(&adev
->shadow_list_lock
);
743 list_add_tail(&vmbo
->shadow_list
, &adev
->shadow_list
);
744 mutex_unlock(&adev
->shadow_list_lock
);
748 * amdgpu_bo_restore_shadow - restore an &amdgpu_bo shadow
750 * @shadow: &amdgpu_bo shadow to be restored
751 * @fence: dma_fence associated with the operation
753 * Copies a buffer object's shadow content back to the object.
754 * This is used for recovering a buffer from its shadow in case of a gpu
755 * reset where vram context may be lost.
758 * 0 for success or a negative error code on failure.
760 int amdgpu_bo_restore_shadow(struct amdgpu_bo
*shadow
, struct dma_fence
**fence
)
763 struct amdgpu_device
*adev
= amdgpu_ttm_adev(shadow
->tbo
.bdev
);
764 struct amdgpu_ring
*ring
= adev
->mman
.buffer_funcs_ring
;
765 uint64_t shadow_addr
, parent_addr
;
767 shadow_addr
= amdgpu_bo_gpu_offset(shadow
);
768 parent_addr
= amdgpu_bo_gpu_offset(shadow
->parent
);
770 return amdgpu_copy_buffer(ring
, shadow_addr
, parent_addr
,
771 amdgpu_bo_size(shadow
), NULL
, fence
,
776 * amdgpu_bo_kmap - map an &amdgpu_bo buffer object
777 * @bo: &amdgpu_bo buffer object to be mapped
778 * @ptr: kernel virtual address to be returned
780 * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls
781 * amdgpu_bo_kptr() to get the kernel virtual address.
784 * 0 for success or a negative error code on failure.
786 int amdgpu_bo_kmap(struct amdgpu_bo
*bo
, void **ptr
)
791 if (bo
->flags
& AMDGPU_GEM_CREATE_NO_CPU_ACCESS
)
794 kptr
= amdgpu_bo_kptr(bo
);
801 r
= dma_resv_wait_timeout(bo
->tbo
.base
.resv
, false, false,
802 MAX_SCHEDULE_TIMEOUT
);
806 r
= ttm_bo_kmap(&bo
->tbo
, 0, bo
->tbo
.resource
->num_pages
, &bo
->kmap
);
811 *ptr
= amdgpu_bo_kptr(bo
);
817 * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object
818 * @bo: &amdgpu_bo buffer object
820 * Calls ttm_kmap_obj_virtual() to get the kernel virtual address
823 * the virtual address of a buffer object area.
825 void *amdgpu_bo_kptr(struct amdgpu_bo
*bo
)
829 return ttm_kmap_obj_virtual(&bo
->kmap
, &is_iomem
);
833 * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object
834 * @bo: &amdgpu_bo buffer object to be unmapped
836 * Unmaps a kernel map set up by amdgpu_bo_kmap().
838 void amdgpu_bo_kunmap(struct amdgpu_bo
*bo
)
841 ttm_bo_kunmap(&bo
->kmap
);
845 * amdgpu_bo_ref - reference an &amdgpu_bo buffer object
846 * @bo: &amdgpu_bo buffer object
848 * References the contained &ttm_buffer_object.
851 * a refcounted pointer to the &amdgpu_bo buffer object.
853 struct amdgpu_bo
*amdgpu_bo_ref(struct amdgpu_bo
*bo
)
858 ttm_bo_get(&bo
->tbo
);
863 * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object
864 * @bo: &amdgpu_bo buffer object
866 * Unreferences the contained &ttm_buffer_object and clear the pointer
868 void amdgpu_bo_unref(struct amdgpu_bo
**bo
)
870 struct ttm_buffer_object
*tbo
;
881 * amdgpu_bo_pin_restricted - pin an &amdgpu_bo buffer object
882 * @bo: &amdgpu_bo buffer object to be pinned
883 * @domain: domain to be pinned to
884 * @min_offset: the start of requested address range
885 * @max_offset: the end of requested address range
887 * Pins the buffer object according to requested domain and address range. If
888 * the memory is unbound gart memory, binds the pages into gart table. Adjusts
889 * pin_count and pin_size accordingly.
891 * Pinning means to lock pages in memory along with keeping them at a fixed
892 * offset. It is required when a buffer can not be moved, for example, when
893 * a display buffer is being scanned out.
895 * Compared with amdgpu_bo_pin(), this function gives more flexibility on
896 * where to pin a buffer if there are specific restrictions on where a buffer
900 * 0 for success or a negative error code on failure.
902 int amdgpu_bo_pin_restricted(struct amdgpu_bo
*bo
, u32 domain
,
903 u64 min_offset
, u64 max_offset
)
905 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->tbo
.bdev
);
906 struct ttm_operation_ctx ctx
= { false, false };
909 if (amdgpu_ttm_tt_get_usermm(bo
->tbo
.ttm
))
912 if (WARN_ON_ONCE(min_offset
> max_offset
))
915 /* A shared bo cannot be migrated to VRAM */
916 if (bo
->tbo
.base
.import_attach
) {
917 if (domain
& AMDGPU_GEM_DOMAIN_GTT
)
918 domain
= AMDGPU_GEM_DOMAIN_GTT
;
923 if (bo
->tbo
.pin_count
) {
924 uint32_t mem_type
= bo
->tbo
.resource
->mem_type
;
925 uint32_t mem_flags
= bo
->tbo
.resource
->placement
;
927 if (!(domain
& amdgpu_mem_type_to_domain(mem_type
)))
930 if ((mem_type
== TTM_PL_VRAM
) &&
931 (bo
->flags
& AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS
) &&
932 !(mem_flags
& TTM_PL_FLAG_CONTIGUOUS
))
935 ttm_bo_pin(&bo
->tbo
);
937 if (max_offset
!= 0) {
938 u64 domain_start
= amdgpu_ttm_domain_start(adev
,
940 WARN_ON_ONCE(max_offset
<
941 (amdgpu_bo_gpu_offset(bo
) - domain_start
));
947 /* This assumes only APU display buffers are pinned with (VRAM|GTT).
948 * See function amdgpu_display_supported_domains()
950 domain
= amdgpu_bo_get_preferred_domain(adev
, domain
);
952 if (bo
->tbo
.base
.import_attach
)
953 dma_buf_pin(bo
->tbo
.base
.import_attach
);
955 /* force to pin into visible video ram */
956 if (!(bo
->flags
& AMDGPU_GEM_CREATE_NO_CPU_ACCESS
))
957 bo
->flags
|= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
;
958 amdgpu_bo_placement_from_domain(bo
, domain
);
959 for (i
= 0; i
< bo
->placement
.num_placement
; i
++) {
962 fpfn
= min_offset
>> PAGE_SHIFT
;
963 lpfn
= max_offset
>> PAGE_SHIFT
;
965 if (fpfn
> bo
->placements
[i
].fpfn
)
966 bo
->placements
[i
].fpfn
= fpfn
;
967 if (!bo
->placements
[i
].lpfn
||
968 (lpfn
&& lpfn
< bo
->placements
[i
].lpfn
))
969 bo
->placements
[i
].lpfn
= lpfn
;
972 r
= ttm_bo_validate(&bo
->tbo
, &bo
->placement
, &ctx
);
974 dev_err(adev
->dev
, "%p pin failed\n", bo
);
978 ttm_bo_pin(&bo
->tbo
);
980 domain
= amdgpu_mem_type_to_domain(bo
->tbo
.resource
->mem_type
);
981 if (domain
== AMDGPU_GEM_DOMAIN_VRAM
) {
982 atomic64_add(amdgpu_bo_size(bo
), &adev
->vram_pin_size
);
983 atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo
),
984 &adev
->visible_pin_size
);
985 } else if (domain
== AMDGPU_GEM_DOMAIN_GTT
) {
986 atomic64_add(amdgpu_bo_size(bo
), &adev
->gart_pin_size
);
994 * amdgpu_bo_pin - pin an &amdgpu_bo buffer object
995 * @bo: &amdgpu_bo buffer object to be pinned
996 * @domain: domain to be pinned to
998 * A simple wrapper to amdgpu_bo_pin_restricted().
999 * Provides a simpler API for buffers that do not have any strict restrictions
1000 * on where a buffer must be located.
1003 * 0 for success or a negative error code on failure.
1005 int amdgpu_bo_pin(struct amdgpu_bo
*bo
, u32 domain
)
1007 bo
->flags
|= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS
;
1008 return amdgpu_bo_pin_restricted(bo
, domain
, 0, 0);
1012 * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object
1013 * @bo: &amdgpu_bo buffer object to be unpinned
1015 * Decreases the pin_count, and clears the flags if pin_count reaches 0.
1016 * Changes placement and pin size accordingly.
1019 * 0 for success or a negative error code on failure.
1021 void amdgpu_bo_unpin(struct amdgpu_bo
*bo
)
1023 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->tbo
.bdev
);
1025 ttm_bo_unpin(&bo
->tbo
);
1026 if (bo
->tbo
.pin_count
)
1029 if (bo
->tbo
.base
.import_attach
)
1030 dma_buf_unpin(bo
->tbo
.base
.import_attach
);
1032 if (bo
->tbo
.resource
->mem_type
== TTM_PL_VRAM
) {
1033 atomic64_sub(amdgpu_bo_size(bo
), &adev
->vram_pin_size
);
1034 atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo
),
1035 &adev
->visible_pin_size
);
1036 } else if (bo
->tbo
.resource
->mem_type
== TTM_PL_TT
) {
1037 atomic64_sub(amdgpu_bo_size(bo
), &adev
->gart_pin_size
);
1041 static const char *amdgpu_vram_names
[] = {
1056 * amdgpu_bo_init - initialize memory manager
1057 * @adev: amdgpu device object
1059 * Calls amdgpu_ttm_init() to initialize amdgpu memory manager.
1062 * 0 for success or a negative error code on failure.
1064 int amdgpu_bo_init(struct amdgpu_device
*adev
)
1066 /* On A+A platform, VRAM can be mapped as WB */
1067 if (!adev
->gmc
.xgmi
.connected_to_cpu
) {
1068 /* reserve PAT memory space to WC for VRAM */
1069 arch_io_reserve_memtype_wc(adev
->gmc
.aper_base
,
1070 adev
->gmc
.aper_size
);
1072 /* Add an MTRR for the VRAM */
1073 adev
->gmc
.vram_mtrr
= arch_phys_wc_add(adev
->gmc
.aper_base
,
1074 adev
->gmc
.aper_size
);
1077 DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
1078 adev
->gmc
.mc_vram_size
>> 20,
1079 (unsigned long long)adev
->gmc
.aper_size
>> 20);
1080 DRM_INFO("RAM width %dbits %s\n",
1081 adev
->gmc
.vram_width
, amdgpu_vram_names
[adev
->gmc
.vram_type
]);
1082 return amdgpu_ttm_init(adev
);
1086 * amdgpu_bo_fini - tear down memory manager
1087 * @adev: amdgpu device object
1089 * Reverses amdgpu_bo_init() to tear down memory manager.
1091 void amdgpu_bo_fini(struct amdgpu_device
*adev
)
1093 amdgpu_ttm_fini(adev
);
1097 * amdgpu_bo_set_tiling_flags - set tiling flags
1098 * @bo: &amdgpu_bo buffer object
1099 * @tiling_flags: new flags
1101 * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or
1102 * kernel driver to set the tiling flags on a buffer.
1105 * 0 for success or a negative error code on failure.
1107 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo
*bo
, u64 tiling_flags
)
1109 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->tbo
.bdev
);
1110 struct amdgpu_bo_user
*ubo
;
1112 BUG_ON(bo
->tbo
.type
== ttm_bo_type_kernel
);
1113 if (adev
->family
<= AMDGPU_FAMILY_CZ
&&
1114 AMDGPU_TILING_GET(tiling_flags
, TILE_SPLIT
) > 6)
1117 ubo
= to_amdgpu_bo_user(bo
);
1118 ubo
->tiling_flags
= tiling_flags
;
1123 * amdgpu_bo_get_tiling_flags - get tiling flags
1124 * @bo: &amdgpu_bo buffer object
1125 * @tiling_flags: returned flags
1127 * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to
1128 * set the tiling flags on a buffer.
1130 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo
*bo
, u64
*tiling_flags
)
1132 struct amdgpu_bo_user
*ubo
;
1134 BUG_ON(bo
->tbo
.type
== ttm_bo_type_kernel
);
1135 dma_resv_assert_held(bo
->tbo
.base
.resv
);
1136 ubo
= to_amdgpu_bo_user(bo
);
1139 *tiling_flags
= ubo
->tiling_flags
;
1143 * amdgpu_bo_set_metadata - set metadata
1144 * @bo: &amdgpu_bo buffer object
1145 * @metadata: new metadata
1146 * @metadata_size: size of the new metadata
1147 * @flags: flags of the new metadata
1149 * Sets buffer object's metadata, its size and flags.
1150 * Used via GEM ioctl.
1153 * 0 for success or a negative error code on failure.
1155 int amdgpu_bo_set_metadata (struct amdgpu_bo
*bo
, void *metadata
,
1156 uint32_t metadata_size
, uint64_t flags
)
1158 struct amdgpu_bo_user
*ubo
;
1161 BUG_ON(bo
->tbo
.type
== ttm_bo_type_kernel
);
1162 ubo
= to_amdgpu_bo_user(bo
);
1163 if (!metadata_size
) {
1164 if (ubo
->metadata_size
) {
1165 kfree(ubo
->metadata
);
1166 ubo
->metadata
= NULL
;
1167 ubo
->metadata_size
= 0;
1172 if (metadata
== NULL
)
1175 buffer
= kmemdup(metadata
, metadata_size
, GFP_KERNEL
);
1179 kfree(ubo
->metadata
);
1180 ubo
->metadata_flags
= flags
;
1181 ubo
->metadata
= buffer
;
1182 ubo
->metadata_size
= metadata_size
;
1188 * amdgpu_bo_get_metadata - get metadata
1189 * @bo: &amdgpu_bo buffer object
1190 * @buffer: returned metadata
1191 * @buffer_size: size of the buffer
1192 * @metadata_size: size of the returned metadata
1193 * @flags: flags of the returned metadata
1195 * Gets buffer object's metadata, its size and flags. buffer_size shall not be
1196 * less than metadata_size.
1197 * Used via GEM ioctl.
1200 * 0 for success or a negative error code on failure.
1202 int amdgpu_bo_get_metadata(struct amdgpu_bo
*bo
, void *buffer
,
1203 size_t buffer_size
, uint32_t *metadata_size
,
1206 struct amdgpu_bo_user
*ubo
;
1208 if (!buffer
&& !metadata_size
)
1211 BUG_ON(bo
->tbo
.type
== ttm_bo_type_kernel
);
1212 ubo
= to_amdgpu_bo_user(bo
);
1214 *metadata_size
= ubo
->metadata_size
;
1217 if (buffer_size
< ubo
->metadata_size
)
1220 if (ubo
->metadata_size
)
1221 memcpy(buffer
, ubo
->metadata
, ubo
->metadata_size
);
1225 *flags
= ubo
->metadata_flags
;
1231 * amdgpu_bo_move_notify - notification about a memory move
1232 * @bo: pointer to a buffer object
1233 * @evict: if this move is evicting the buffer from the graphics address space
1234 * @new_mem: new information of the bufer object
1236 * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs
1238 * TTM driver callback which is called when ttm moves a buffer.
1240 void amdgpu_bo_move_notify(struct ttm_buffer_object
*bo
,
1242 struct ttm_resource
*new_mem
)
1244 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->bdev
);
1245 struct amdgpu_bo
*abo
;
1246 struct ttm_resource
*old_mem
= bo
->resource
;
1248 if (!amdgpu_bo_is_amdgpu_bo(bo
))
1251 abo
= ttm_to_amdgpu_bo(bo
);
1252 amdgpu_vm_bo_invalidate(adev
, abo
, evict
);
1254 amdgpu_bo_kunmap(abo
);
1256 if (abo
->tbo
.base
.dma_buf
&& !abo
->tbo
.base
.import_attach
&&
1257 bo
->resource
->mem_type
!= TTM_PL_SYSTEM
)
1258 dma_buf_move_notify(abo
->tbo
.base
.dma_buf
);
1260 /* remember the eviction */
1262 atomic64_inc(&adev
->num_evictions
);
1264 /* update statistics */
1268 /* move_notify is called before move happens */
1269 trace_amdgpu_bo_move(abo
, new_mem
->mem_type
, old_mem
->mem_type
);
1272 void amdgpu_bo_get_memory(struct amdgpu_bo
*bo
, uint64_t *vram_mem
,
1273 uint64_t *gtt_mem
, uint64_t *cpu_mem
)
1275 unsigned int domain
;
1277 domain
= amdgpu_mem_type_to_domain(bo
->tbo
.resource
->mem_type
);
1279 case AMDGPU_GEM_DOMAIN_VRAM
:
1280 *vram_mem
+= amdgpu_bo_size(bo
);
1282 case AMDGPU_GEM_DOMAIN_GTT
:
1283 *gtt_mem
+= amdgpu_bo_size(bo
);
1285 case AMDGPU_GEM_DOMAIN_CPU
:
1287 *cpu_mem
+= amdgpu_bo_size(bo
);
1293 * amdgpu_bo_release_notify - notification about a BO being released
1294 * @bo: pointer to a buffer object
1296 * Wipes VRAM buffers whose contents should not be leaked before the
1297 * memory is released.
1299 void amdgpu_bo_release_notify(struct ttm_buffer_object
*bo
)
1301 struct dma_fence
*fence
= NULL
;
1302 struct amdgpu_bo
*abo
;
1305 if (!amdgpu_bo_is_amdgpu_bo(bo
))
1308 abo
= ttm_to_amdgpu_bo(bo
);
1311 amdgpu_amdkfd_unreserve_memory_limit(abo
);
1313 /* We only remove the fence if the resv has individualized. */
1314 WARN_ON_ONCE(bo
->type
== ttm_bo_type_kernel
1315 && bo
->base
.resv
!= &bo
->base
._resv
);
1316 if (bo
->base
.resv
== &bo
->base
._resv
)
1317 amdgpu_amdkfd_remove_fence_on_pt_pd_bos(abo
);
1319 if (bo
->resource
->mem_type
!= TTM_PL_VRAM
||
1320 !(abo
->flags
& AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE
))
1323 if (WARN_ON_ONCE(!dma_resv_trylock(bo
->base
.resv
)))
1326 r
= amdgpu_fill_buffer(abo
, AMDGPU_POISON
, bo
->base
.resv
, &fence
);
1328 amdgpu_bo_fence(abo
, fence
, false);
1329 dma_fence_put(fence
);
1332 dma_resv_unlock(bo
->base
.resv
);
1336 * amdgpu_bo_fault_reserve_notify - notification about a memory fault
1337 * @bo: pointer to a buffer object
1339 * Notifies the driver we are taking a fault on this BO and have reserved it,
1340 * also performs bookkeeping.
1341 * TTM driver callback for dealing with vm faults.
1344 * 0 for success or a negative error code on failure.
1346 vm_fault_t
amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object
*bo
)
1348 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->bdev
);
1349 struct ttm_operation_ctx ctx
= { false, false };
1350 struct amdgpu_bo
*abo
= ttm_to_amdgpu_bo(bo
);
1351 unsigned long offset
;
1354 /* Remember that this BO was accessed by the CPU */
1355 abo
->flags
|= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
;
1357 if (bo
->resource
->mem_type
!= TTM_PL_VRAM
)
1360 offset
= bo
->resource
->start
<< PAGE_SHIFT
;
1361 if ((offset
+ bo
->base
.size
) <= adev
->gmc
.visible_vram_size
)
1364 /* Can't move a pinned BO to visible VRAM */
1365 if (abo
->tbo
.pin_count
> 0)
1366 return VM_FAULT_SIGBUS
;
1368 /* hurrah the memory is not visible ! */
1369 atomic64_inc(&adev
->num_vram_cpu_page_faults
);
1370 amdgpu_bo_placement_from_domain(abo
, AMDGPU_GEM_DOMAIN_VRAM
|
1371 AMDGPU_GEM_DOMAIN_GTT
);
1373 /* Avoid costly evictions; only set GTT as a busy placement */
1374 abo
->placement
.num_busy_placement
= 1;
1375 abo
->placement
.busy_placement
= &abo
->placements
[1];
1377 r
= ttm_bo_validate(bo
, &abo
->placement
, &ctx
);
1378 if (unlikely(r
== -EBUSY
|| r
== -ERESTARTSYS
))
1379 return VM_FAULT_NOPAGE
;
1380 else if (unlikely(r
))
1381 return VM_FAULT_SIGBUS
;
1383 offset
= bo
->resource
->start
<< PAGE_SHIFT
;
1384 /* this should never happen */
1385 if (bo
->resource
->mem_type
== TTM_PL_VRAM
&&
1386 (offset
+ bo
->base
.size
) > adev
->gmc
.visible_vram_size
)
1387 return VM_FAULT_SIGBUS
;
1389 ttm_bo_move_to_lru_tail_unlocked(bo
);
1394 * amdgpu_bo_fence - add fence to buffer object
1396 * @bo: buffer object in question
1397 * @fence: fence to add
1398 * @shared: true if fence should be added shared
1401 void amdgpu_bo_fence(struct amdgpu_bo
*bo
, struct dma_fence
*fence
,
1404 struct dma_resv
*resv
= bo
->tbo
.base
.resv
;
1407 dma_resv_add_shared_fence(resv
, fence
);
1409 dma_resv_add_excl_fence(resv
, fence
);
1413 * amdgpu_bo_sync_wait_resv - Wait for BO reservation fences
1415 * @adev: amdgpu device pointer
1416 * @resv: reservation object to sync to
1417 * @sync_mode: synchronization mode
1418 * @owner: fence owner
1419 * @intr: Whether the wait is interruptible
1421 * Extract the fences from the reservation object and waits for them to finish.
1424 * 0 on success, errno otherwise.
1426 int amdgpu_bo_sync_wait_resv(struct amdgpu_device
*adev
, struct dma_resv
*resv
,
1427 enum amdgpu_sync_mode sync_mode
, void *owner
,
1430 struct amdgpu_sync sync
;
1433 amdgpu_sync_create(&sync
);
1434 amdgpu_sync_resv(adev
, &sync
, resv
, sync_mode
, owner
);
1435 r
= amdgpu_sync_wait(&sync
, intr
);
1436 amdgpu_sync_free(&sync
);
1441 * amdgpu_bo_sync_wait - Wrapper for amdgpu_bo_sync_wait_resv
1442 * @bo: buffer object to wait for
1443 * @owner: fence owner
1444 * @intr: Whether the wait is interruptible
1446 * Wrapper to wait for fences in a BO.
1448 * 0 on success, errno otherwise.
1450 int amdgpu_bo_sync_wait(struct amdgpu_bo
*bo
, void *owner
, bool intr
)
1452 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->tbo
.bdev
);
1454 return amdgpu_bo_sync_wait_resv(adev
, bo
->tbo
.base
.resv
,
1455 AMDGPU_SYNC_NE_OWNER
, owner
, intr
);
1459 * amdgpu_bo_gpu_offset - return GPU offset of bo
1460 * @bo: amdgpu object for which we query the offset
1462 * Note: object should either be pinned or reserved when calling this
1463 * function, it might be useful to add check for this for debugging.
1466 * current GPU offset of the object.
1468 u64
amdgpu_bo_gpu_offset(struct amdgpu_bo
*bo
)
1470 WARN_ON_ONCE(bo
->tbo
.resource
->mem_type
== TTM_PL_SYSTEM
);
1471 WARN_ON_ONCE(!dma_resv_is_locked(bo
->tbo
.base
.resv
) &&
1472 !bo
->tbo
.pin_count
&& bo
->tbo
.type
!= ttm_bo_type_kernel
);
1473 WARN_ON_ONCE(bo
->tbo
.resource
->start
== AMDGPU_BO_INVALID_OFFSET
);
1474 WARN_ON_ONCE(bo
->tbo
.resource
->mem_type
== TTM_PL_VRAM
&&
1475 !(bo
->flags
& AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS
));
1477 return amdgpu_bo_gpu_offset_no_check(bo
);
1481 * amdgpu_bo_gpu_offset_no_check - return GPU offset of bo
1482 * @bo: amdgpu object for which we query the offset
1485 * current GPU offset of the object without raising warnings.
1487 u64
amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo
*bo
)
1489 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->tbo
.bdev
);
1492 offset
= (bo
->tbo
.resource
->start
<< PAGE_SHIFT
) +
1493 amdgpu_ttm_domain_start(adev
, bo
->tbo
.resource
->mem_type
);
1495 return amdgpu_gmc_sign_extend(offset
);
1499 * amdgpu_bo_get_preferred_domain - get preferred domain
1500 * @adev: amdgpu device object
1501 * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>`
1504 * Which of the allowed domains is preferred for allocating the BO.
1506 uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device
*adev
,
1509 if (domain
== (AMDGPU_GEM_DOMAIN_VRAM
| AMDGPU_GEM_DOMAIN_GTT
)) {
1510 domain
= AMDGPU_GEM_DOMAIN_VRAM
;
1511 if (adev
->gmc
.real_vram_size
<= AMDGPU_SG_THRESHOLD
)
1512 domain
= AMDGPU_GEM_DOMAIN_GTT
;
1517 #if defined(CONFIG_DEBUG_FS)
1518 #define amdgpu_bo_print_flag(m, bo, flag) \
1520 if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) { \
1521 seq_printf((m), " " #flag); \
1526 * amdgpu_bo_print_info - print BO info in debugfs file
1528 * @id: Index or Id of the BO
1529 * @bo: Requested BO for printing info
1532 * Print BO information in debugfs file
1535 * Size of the BO in bytes.
1537 u64
amdgpu_bo_print_info(int id
, struct amdgpu_bo
*bo
, struct seq_file
*m
)
1539 struct dma_buf_attachment
*attachment
;
1540 struct dma_buf
*dma_buf
;
1541 unsigned int domain
;
1542 const char *placement
;
1543 unsigned int pin_count
;
1546 domain
= amdgpu_mem_type_to_domain(bo
->tbo
.resource
->mem_type
);
1548 case AMDGPU_GEM_DOMAIN_VRAM
:
1551 case AMDGPU_GEM_DOMAIN_GTT
:
1554 case AMDGPU_GEM_DOMAIN_CPU
:
1560 size
= amdgpu_bo_size(bo
);
1561 seq_printf(m
, "\t\t0x%08x: %12lld byte %s",
1562 id
, size
, placement
);
1564 pin_count
= READ_ONCE(bo
->tbo
.pin_count
);
1566 seq_printf(m
, " pin count %d", pin_count
);
1568 dma_buf
= READ_ONCE(bo
->tbo
.base
.dma_buf
);
1569 attachment
= READ_ONCE(bo
->tbo
.base
.import_attach
);
1572 seq_printf(m
, " imported from %p", dma_buf
);
1574 seq_printf(m
, " exported as %p", dma_buf
);
1576 amdgpu_bo_print_flag(m
, bo
, CPU_ACCESS_REQUIRED
);
1577 amdgpu_bo_print_flag(m
, bo
, NO_CPU_ACCESS
);
1578 amdgpu_bo_print_flag(m
, bo
, CPU_GTT_USWC
);
1579 amdgpu_bo_print_flag(m
, bo
, VRAM_CLEARED
);
1580 amdgpu_bo_print_flag(m
, bo
, VRAM_CONTIGUOUS
);
1581 amdgpu_bo_print_flag(m
, bo
, VM_ALWAYS_VALID
);
1582 amdgpu_bo_print_flag(m
, bo
, EXPLICIT_SYNC
);