2 * Copyright 2016 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Christian König
24 #ifndef __AMDGPU_RING_H__
25 #define __AMDGPU_RING_H__
27 #include "gpu_scheduler.h"
29 /* max number of rings */
30 #define AMDGPU_MAX_RINGS 18
31 #define AMDGPU_MAX_GFX_RINGS 1
32 #define AMDGPU_MAX_COMPUTE_RINGS 8
33 #define AMDGPU_MAX_VCE_RINGS 3
34 #define AMDGPU_MAX_UVD_ENC_RINGS 2
36 /* some special values for the owner field */
37 #define AMDGPU_FENCE_OWNER_UNDEFINED ((void*)0ul)
38 #define AMDGPU_FENCE_OWNER_VM ((void*)1ul)
40 #define AMDGPU_FENCE_FLAG_64BIT (1 << 0)
41 #define AMDGPU_FENCE_FLAG_INT (1 << 1)
43 enum amdgpu_ring_type
{
45 AMDGPU_RING_TYPE_COMPUTE
,
46 AMDGPU_RING_TYPE_SDMA
,
50 AMDGPU_RING_TYPE_UVD_ENC
,
51 AMDGPU_RING_TYPE_VCN_DEC
,
52 AMDGPU_RING_TYPE_VCN_ENC
58 struct amdgpu_cs_parser
;
63 struct amdgpu_fence_driver
{
65 volatile uint32_t *cpu_addr
;
66 /* sync_seq is protected by ring emission lock */
70 struct amdgpu_irq_src
*irq_src
;
72 struct timer_list fallback_timer
;
73 unsigned num_fences_mask
;
75 struct dma_fence
**fences
;
78 int amdgpu_fence_driver_init(struct amdgpu_device
*adev
);
79 void amdgpu_fence_driver_fini(struct amdgpu_device
*adev
);
80 void amdgpu_fence_driver_force_completion(struct amdgpu_device
*adev
);
81 void amdgpu_fence_driver_force_completion_ring(struct amdgpu_ring
*ring
);
83 int amdgpu_fence_driver_init_ring(struct amdgpu_ring
*ring
,
84 unsigned num_hw_submission
);
85 int amdgpu_fence_driver_start_ring(struct amdgpu_ring
*ring
,
86 struct amdgpu_irq_src
*irq_src
,
88 void amdgpu_fence_driver_suspend(struct amdgpu_device
*adev
);
89 void amdgpu_fence_driver_resume(struct amdgpu_device
*adev
);
90 int amdgpu_fence_emit(struct amdgpu_ring
*ring
, struct dma_fence
**fence
);
91 void amdgpu_fence_process(struct amdgpu_ring
*ring
);
92 int amdgpu_fence_wait_empty(struct amdgpu_ring
*ring
);
93 unsigned amdgpu_fence_count_emitted(struct amdgpu_ring
*ring
);
99 /* provided by hw blocks that expose a ring buffer for commands */
100 struct amdgpu_ring_funcs
{
101 enum amdgpu_ring_type type
;
104 bool support_64bit_ptrs
;
107 /* ring read/write ptr handling */
108 u64 (*get_rptr
)(struct amdgpu_ring
*ring
);
109 u64 (*get_wptr
)(struct amdgpu_ring
*ring
);
110 void (*set_wptr
)(struct amdgpu_ring
*ring
);
111 /* validating and patching of IBs */
112 int (*parse_cs
)(struct amdgpu_cs_parser
*p
, uint32_t ib_idx
);
113 /* constants to calculate how many DW are needed for an emit */
114 unsigned emit_frame_size
;
115 unsigned emit_ib_size
;
116 /* command emit functions */
117 void (*emit_ib
)(struct amdgpu_ring
*ring
,
118 struct amdgpu_ib
*ib
,
119 unsigned vm_id
, bool ctx_switch
);
120 void (*emit_fence
)(struct amdgpu_ring
*ring
, uint64_t addr
,
121 uint64_t seq
, unsigned flags
);
122 void (*emit_pipeline_sync
)(struct amdgpu_ring
*ring
);
123 void (*emit_vm_flush
)(struct amdgpu_ring
*ring
, unsigned vm_id
,
125 void (*emit_hdp_flush
)(struct amdgpu_ring
*ring
);
126 void (*emit_hdp_invalidate
)(struct amdgpu_ring
*ring
);
127 void (*emit_gds_switch
)(struct amdgpu_ring
*ring
, uint32_t vmid
,
128 uint32_t gds_base
, uint32_t gds_size
,
129 uint32_t gws_base
, uint32_t gws_size
,
130 uint32_t oa_base
, uint32_t oa_size
);
131 /* testing functions */
132 int (*test_ring
)(struct amdgpu_ring
*ring
);
133 int (*test_ib
)(struct amdgpu_ring
*ring
, long timeout
);
134 /* insert NOP packets */
135 void (*insert_nop
)(struct amdgpu_ring
*ring
, uint32_t count
);
136 void (*insert_start
)(struct amdgpu_ring
*ring
);
137 void (*insert_end
)(struct amdgpu_ring
*ring
);
138 /* pad the indirect buffer to the necessary number of dw */
139 void (*pad_ib
)(struct amdgpu_ring
*ring
, struct amdgpu_ib
*ib
);
140 unsigned (*init_cond_exec
)(struct amdgpu_ring
*ring
);
141 void (*patch_cond_exec
)(struct amdgpu_ring
*ring
, unsigned offset
);
142 /* note usage for clock and power gating */
143 void (*begin_use
)(struct amdgpu_ring
*ring
);
144 void (*end_use
)(struct amdgpu_ring
*ring
);
145 void (*emit_switch_buffer
) (struct amdgpu_ring
*ring
);
146 void (*emit_cntxcntl
) (struct amdgpu_ring
*ring
, uint32_t flags
);
147 void (*emit_rreg
)(struct amdgpu_ring
*ring
, uint32_t reg
);
148 void (*emit_wreg
)(struct amdgpu_ring
*ring
, uint32_t reg
, uint32_t val
);
149 void (*emit_tmz
)(struct amdgpu_ring
*ring
, bool start
);
153 struct amdgpu_device
*adev
;
154 const struct amdgpu_ring_funcs
*funcs
;
155 struct amdgpu_fence_driver fence_drv
;
156 struct amd_gpu_scheduler sched
;
157 struct list_head lru_list
;
159 struct amdgpu_bo
*ring_obj
;
160 volatile uint32_t *ring
;
175 struct amdgpu_bo
*mqd_obj
;
176 uint64_t mqd_gpu_addr
;
178 uint64_t eop_gpu_addr
;
183 uint64_t current_ctx
;
185 unsigned cond_exe_offs
;
186 u64 cond_exe_gpu_addr
;
187 volatile u32
*cond_exe_cpu_addr
;
189 bool has_compute_vm_bug
;
190 #if defined(CONFIG_DEBUG_FS)
195 int amdgpu_ring_alloc(struct amdgpu_ring
*ring
, unsigned ndw
);
196 void amdgpu_ring_insert_nop(struct amdgpu_ring
*ring
, uint32_t count
);
197 void amdgpu_ring_generic_pad_ib(struct amdgpu_ring
*ring
, struct amdgpu_ib
*ib
);
198 void amdgpu_ring_commit(struct amdgpu_ring
*ring
);
199 void amdgpu_ring_undo(struct amdgpu_ring
*ring
);
200 int amdgpu_ring_init(struct amdgpu_device
*adev
, struct amdgpu_ring
*ring
,
201 unsigned ring_size
, struct amdgpu_irq_src
*irq_src
,
203 void amdgpu_ring_fini(struct amdgpu_ring
*ring
);
204 int amdgpu_ring_lru_get(struct amdgpu_device
*adev
, int type
, int *blacklist
,
205 int num_blacklist
, struct amdgpu_ring
**ring
);
206 void amdgpu_ring_lru_touch(struct amdgpu_device
*adev
, struct amdgpu_ring
*ring
);
207 static inline void amdgpu_ring_clear_ring(struct amdgpu_ring
*ring
)
210 while (i
<= ring
->buf_mask
)
211 ring
->ring
[i
++] = ring
->funcs
->nop
;
215 static inline void amdgpu_ring_write(struct amdgpu_ring
*ring
, uint32_t v
)
217 if (ring
->count_dw
<= 0)
218 DRM_ERROR("amdgpu: writing more dwords to the ring than expected!\n");
219 ring
->ring
[ring
->wptr
++ & ring
->buf_mask
] = v
;
220 ring
->wptr
&= ring
->ptr_mask
;
224 static inline void amdgpu_ring_write_multiple(struct amdgpu_ring
*ring
,
225 void *src
, int count_dw
)
227 unsigned occupied
, chunk1
, chunk2
;
230 if (unlikely(ring
->count_dw
< count_dw
))
231 DRM_ERROR("amdgpu: writing more dwords to the ring than expected!\n");
233 occupied
= ring
->wptr
& ring
->buf_mask
;
234 dst
= (void *)&ring
->ring
[occupied
];
235 chunk1
= ring
->buf_mask
+ 1 - occupied
;
236 chunk1
= (chunk1
>= count_dw
) ? count_dw
: chunk1
;
237 chunk2
= count_dw
- chunk1
;
242 memcpy(dst
, src
, chunk1
);
246 dst
= (void *)ring
->ring
;
247 memcpy(dst
, src
, chunk2
);
250 ring
->wptr
+= count_dw
;
251 ring
->wptr
&= ring
->ptr_mask
;
252 ring
->count_dw
-= count_dw
;