2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
24 * Authors: Dave Airlie
28 #ifndef __AMDGPU_OBJECT_H__
29 #define __AMDGPU_OBJECT_H__
31 #include <drm/amdgpu_drm.h>
33 #include "amdgpu_res_cursor.h"
35 #ifdef CONFIG_MMU_NOTIFIER
36 #include <linux/mmu_notifier.h>
39 #define AMDGPU_BO_INVALID_OFFSET LONG_MAX
40 #define AMDGPU_BO_MAX_PLACEMENTS 3
42 /* BO flag to indicate a KFD userptr BO */
43 #define AMDGPU_AMDKFD_CREATE_USERPTR_BO (1ULL << 63)
44 #define AMDGPU_AMDKFD_CREATE_SVM_BO (1ULL << 62)
46 #define to_amdgpu_bo_user(abo) container_of((abo), struct amdgpu_bo_user, bo)
47 #define to_amdgpu_bo_vm(abo) container_of((abo), struct amdgpu_bo_vm, bo)
49 struct amdgpu_bo_param
{
56 enum ttm_bo_type type
;
58 struct dma_resv
*resv
;
59 void (*destroy
)(struct ttm_buffer_object
*bo
);
62 /* bo virtual addresses in a vm */
63 struct amdgpu_bo_va_mapping
{
64 struct amdgpu_bo_va
*bo_va
;
65 struct list_head list
;
69 uint64_t __subtree_last
;
74 /* User space allocated BO in a VM */
76 struct amdgpu_vm_bo_base base
;
78 /* protected by bo being reserved */
81 /* all other members protected by the VM PD being reserved */
82 struct dma_fence
*last_pt_update
;
84 /* mappings for this bo_va */
85 struct list_head invalids
;
86 struct list_head valids
;
88 /* If the mappings are cleared or filled */
95 /* Protected by tbo.reserved */
96 u32 preferred_domains
;
98 struct ttm_place placements
[AMDGPU_BO_MAX_PLACEMENTS
];
99 struct ttm_placement placement
;
100 struct ttm_buffer_object tbo
;
101 struct ttm_bo_kmap_obj kmap
;
103 /* per VM structure for page tables and with virtual addresses */
104 struct amdgpu_vm_bo_base
*vm_bo
;
105 /* Constant after initialization */
106 struct amdgpu_bo
*parent
;
108 #ifdef CONFIG_MMU_NOTIFIER
109 struct mmu_interval_notifier notifier
;
111 struct kgd_mem
*kfd_bo
;
114 struct amdgpu_bo_user
{
123 struct amdgpu_bo_vm
{
125 struct amdgpu_bo
*shadow
;
126 struct list_head shadow_list
;
127 struct amdgpu_vm_bo_base entries
[];
130 static inline struct amdgpu_bo
*ttm_to_amdgpu_bo(struct ttm_buffer_object
*tbo
)
132 return container_of(tbo
, struct amdgpu_bo
, tbo
);
136 * amdgpu_mem_type_to_domain - return domain corresponding to mem_type
137 * @mem_type: ttm memory type
139 * Returns corresponding domain of the ttm mem_type
141 static inline unsigned amdgpu_mem_type_to_domain(u32 mem_type
)
145 return AMDGPU_GEM_DOMAIN_VRAM
;
147 return AMDGPU_GEM_DOMAIN_GTT
;
149 return AMDGPU_GEM_DOMAIN_CPU
;
151 return AMDGPU_GEM_DOMAIN_GDS
;
153 return AMDGPU_GEM_DOMAIN_GWS
;
155 return AMDGPU_GEM_DOMAIN_OA
;
163 * amdgpu_bo_reserve - reserve bo
165 * @no_intr: don't return -ERESTARTSYS on pending signal
168 * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by
169 * a signal. Release all buffer reservations and return to user-space.
171 static inline int amdgpu_bo_reserve(struct amdgpu_bo
*bo
, bool no_intr
)
173 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->tbo
.bdev
);
176 r
= ttm_bo_reserve(&bo
->tbo
, !no_intr
, false, NULL
);
177 if (unlikely(r
!= 0)) {
178 if (r
!= -ERESTARTSYS
)
179 dev_err(adev
->dev
, "%p reserve failed\n", bo
);
185 static inline void amdgpu_bo_unreserve(struct amdgpu_bo
*bo
)
187 ttm_bo_unreserve(&bo
->tbo
);
190 static inline unsigned long amdgpu_bo_size(struct amdgpu_bo
*bo
)
192 return bo
->tbo
.base
.size
;
195 static inline unsigned amdgpu_bo_ngpu_pages(struct amdgpu_bo
*bo
)
197 return bo
->tbo
.base
.size
/ AMDGPU_GPU_PAGE_SIZE
;
200 static inline unsigned amdgpu_bo_gpu_page_alignment(struct amdgpu_bo
*bo
)
202 return (bo
->tbo
.page_alignment
<< PAGE_SHIFT
) / AMDGPU_GPU_PAGE_SIZE
;
206 * amdgpu_bo_mmap_offset - return mmap offset of bo
207 * @bo: amdgpu object for which we query the offset
209 * Returns mmap offset of the object.
211 static inline u64
amdgpu_bo_mmap_offset(struct amdgpu_bo
*bo
)
213 return drm_vma_node_offset_addr(&bo
->tbo
.base
.vma_node
);
217 * amdgpu_bo_in_cpu_visible_vram - check if BO is (partly) in visible VRAM
219 static inline bool amdgpu_bo_in_cpu_visible_vram(struct amdgpu_bo
*bo
)
221 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->tbo
.bdev
);
222 struct amdgpu_res_cursor cursor
;
224 if (bo
->tbo
.resource
->mem_type
!= TTM_PL_VRAM
)
227 amdgpu_res_first(bo
->tbo
.resource
, 0, amdgpu_bo_size(bo
), &cursor
);
228 while (cursor
.remaining
) {
229 if (cursor
.start
< adev
->gmc
.visible_vram_size
)
232 amdgpu_res_next(&cursor
, cursor
.size
);
239 * amdgpu_bo_explicit_sync - return whether the bo is explicitly synced
241 static inline bool amdgpu_bo_explicit_sync(struct amdgpu_bo
*bo
)
243 return bo
->flags
& AMDGPU_GEM_CREATE_EXPLICIT_SYNC
;
247 * amdgpu_bo_encrypted - test if the BO is encrypted
248 * @bo: pointer to a buffer object
250 * Return true if the buffer object is encrypted, false otherwise.
252 static inline bool amdgpu_bo_encrypted(struct amdgpu_bo
*bo
)
254 return bo
->flags
& AMDGPU_GEM_CREATE_ENCRYPTED
;
258 * amdgpu_bo_shadowed - check if the BO is shadowed
260 * @bo: BO to be tested.
263 * NULL if not shadowed or else return a BO pointer.
265 static inline struct amdgpu_bo
*amdgpu_bo_shadowed(struct amdgpu_bo
*bo
)
267 if (bo
->tbo
.type
== ttm_bo_type_kernel
)
268 return to_amdgpu_bo_vm(bo
)->shadow
;
273 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object
*bo
);
274 void amdgpu_bo_placement_from_domain(struct amdgpu_bo
*abo
, u32 domain
);
276 int amdgpu_bo_create(struct amdgpu_device
*adev
,
277 struct amdgpu_bo_param
*bp
,
278 struct amdgpu_bo
**bo_ptr
);
279 int amdgpu_bo_create_reserved(struct amdgpu_device
*adev
,
280 unsigned long size
, int align
,
281 u32 domain
, struct amdgpu_bo
**bo_ptr
,
282 u64
*gpu_addr
, void **cpu_addr
);
283 int amdgpu_bo_create_kernel(struct amdgpu_device
*adev
,
284 unsigned long size
, int align
,
285 u32 domain
, struct amdgpu_bo
**bo_ptr
,
286 u64
*gpu_addr
, void **cpu_addr
);
287 int amdgpu_bo_create_kernel_at(struct amdgpu_device
*adev
,
288 uint64_t offset
, uint64_t size
, uint32_t domain
,
289 struct amdgpu_bo
**bo_ptr
, void **cpu_addr
);
290 int amdgpu_bo_create_user(struct amdgpu_device
*adev
,
291 struct amdgpu_bo_param
*bp
,
292 struct amdgpu_bo_user
**ubo_ptr
);
293 int amdgpu_bo_create_vm(struct amdgpu_device
*adev
,
294 struct amdgpu_bo_param
*bp
,
295 struct amdgpu_bo_vm
**ubo_ptr
);
296 void amdgpu_bo_free_kernel(struct amdgpu_bo
**bo
, u64
*gpu_addr
,
298 int amdgpu_bo_kmap(struct amdgpu_bo
*bo
, void **ptr
);
299 void *amdgpu_bo_kptr(struct amdgpu_bo
*bo
);
300 void amdgpu_bo_kunmap(struct amdgpu_bo
*bo
);
301 struct amdgpu_bo
*amdgpu_bo_ref(struct amdgpu_bo
*bo
);
302 void amdgpu_bo_unref(struct amdgpu_bo
**bo
);
303 int amdgpu_bo_pin(struct amdgpu_bo
*bo
, u32 domain
);
304 int amdgpu_bo_pin_restricted(struct amdgpu_bo
*bo
, u32 domain
,
305 u64 min_offset
, u64 max_offset
);
306 void amdgpu_bo_unpin(struct amdgpu_bo
*bo
);
307 int amdgpu_bo_init(struct amdgpu_device
*adev
);
308 void amdgpu_bo_fini(struct amdgpu_device
*adev
);
309 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo
*bo
, u64 tiling_flags
);
310 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo
*bo
, u64
*tiling_flags
);
311 int amdgpu_bo_set_metadata (struct amdgpu_bo
*bo
, void *metadata
,
312 uint32_t metadata_size
, uint64_t flags
);
313 int amdgpu_bo_get_metadata(struct amdgpu_bo
*bo
, void *buffer
,
314 size_t buffer_size
, uint32_t *metadata_size
,
316 void amdgpu_bo_move_notify(struct ttm_buffer_object
*bo
,
318 struct ttm_resource
*new_mem
);
319 void amdgpu_bo_release_notify(struct ttm_buffer_object
*bo
);
320 vm_fault_t
amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object
*bo
);
321 void amdgpu_bo_fence(struct amdgpu_bo
*bo
, struct dma_fence
*fence
,
323 int amdgpu_bo_sync_wait_resv(struct amdgpu_device
*adev
, struct dma_resv
*resv
,
324 enum amdgpu_sync_mode sync_mode
, void *owner
,
326 int amdgpu_bo_sync_wait(struct amdgpu_bo
*bo
, void *owner
, bool intr
);
327 u64
amdgpu_bo_gpu_offset(struct amdgpu_bo
*bo
);
328 u64
amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo
*bo
);
329 int amdgpu_bo_validate(struct amdgpu_bo
*bo
);
330 void amdgpu_bo_get_memory(struct amdgpu_bo
*bo
, uint64_t *vram_mem
,
331 uint64_t *gtt_mem
, uint64_t *cpu_mem
);
332 void amdgpu_bo_add_to_shadow_list(struct amdgpu_bo_vm
*vmbo
);
333 int amdgpu_bo_restore_shadow(struct amdgpu_bo
*shadow
,
334 struct dma_fence
**fence
);
335 uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device
*adev
,
342 static inline uint64_t amdgpu_sa_bo_gpu_addr(struct amdgpu_sa_bo
*sa_bo
)
344 return sa_bo
->manager
->gpu_addr
+ sa_bo
->soffset
;
347 static inline void * amdgpu_sa_bo_cpu_addr(struct amdgpu_sa_bo
*sa_bo
)
349 return sa_bo
->manager
->cpu_ptr
+ sa_bo
->soffset
;
352 int amdgpu_sa_bo_manager_init(struct amdgpu_device
*adev
,
353 struct amdgpu_sa_manager
*sa_manager
,
354 unsigned size
, u32 align
, u32 domain
);
355 void amdgpu_sa_bo_manager_fini(struct amdgpu_device
*adev
,
356 struct amdgpu_sa_manager
*sa_manager
);
357 int amdgpu_sa_bo_manager_start(struct amdgpu_device
*adev
,
358 struct amdgpu_sa_manager
*sa_manager
);
359 int amdgpu_sa_bo_new(struct amdgpu_sa_manager
*sa_manager
,
360 struct amdgpu_sa_bo
**sa_bo
,
361 unsigned size
, unsigned align
);
362 void amdgpu_sa_bo_free(struct amdgpu_device
*adev
,
363 struct amdgpu_sa_bo
**sa_bo
,
364 struct dma_fence
*fence
);
365 #if defined(CONFIG_DEBUG_FS)
366 void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager
*sa_manager
,
368 u64
amdgpu_bo_print_info(int id
, struct amdgpu_bo
*bo
, struct seq_file
*m
);
370 void amdgpu_debugfs_sa_init(struct amdgpu_device
*adev
);
372 bool amdgpu_bo_support_uswc(u64 bo_flags
);