2 * Copyright 2016 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Christian König
24 #ifndef __AMDGPU_RING_H__
25 #define __AMDGPU_RING_H__
27 #include <drm/amdgpu_drm.h>
28 #include <drm/gpu_scheduler.h>
29 #include <drm/drm_print.h>
31 /* max number of rings */
32 #define AMDGPU_MAX_RINGS 28
33 #define AMDGPU_MAX_HWIP_RINGS 8
34 #define AMDGPU_MAX_GFX_RINGS 2
35 #define AMDGPU_MAX_COMPUTE_RINGS 8
36 #define AMDGPU_MAX_VCE_RINGS 3
37 #define AMDGPU_MAX_UVD_ENC_RINGS 2
39 #define AMDGPU_RING_PRIO_DEFAULT 1
40 #define AMDGPU_RING_PRIO_MAX AMDGPU_GFX_PIPE_PRIO_MAX
42 /* some special values for the owner field */
43 #define AMDGPU_FENCE_OWNER_UNDEFINED ((void *)0ul)
44 #define AMDGPU_FENCE_OWNER_VM ((void *)1ul)
45 #define AMDGPU_FENCE_OWNER_KFD ((void *)2ul)
47 #define AMDGPU_FENCE_FLAG_64BIT (1 << 0)
48 #define AMDGPU_FENCE_FLAG_INT (1 << 1)
49 #define AMDGPU_FENCE_FLAG_TC_WB_ONLY (1 << 2)
51 #define to_amdgpu_ring(s) container_of((s), struct amdgpu_ring, sched)
53 #define AMDGPU_IB_POOL_SIZE (1024 * 1024)
55 enum amdgpu_ring_type
{
56 AMDGPU_RING_TYPE_GFX
= AMDGPU_HW_IP_GFX
,
57 AMDGPU_RING_TYPE_COMPUTE
= AMDGPU_HW_IP_COMPUTE
,
58 AMDGPU_RING_TYPE_SDMA
= AMDGPU_HW_IP_DMA
,
59 AMDGPU_RING_TYPE_UVD
= AMDGPU_HW_IP_UVD
,
60 AMDGPU_RING_TYPE_VCE
= AMDGPU_HW_IP_VCE
,
61 AMDGPU_RING_TYPE_UVD_ENC
= AMDGPU_HW_IP_UVD_ENC
,
62 AMDGPU_RING_TYPE_VCN_DEC
= AMDGPU_HW_IP_VCN_DEC
,
63 AMDGPU_RING_TYPE_VCN_ENC
= AMDGPU_HW_IP_VCN_ENC
,
64 AMDGPU_RING_TYPE_VCN_JPEG
= AMDGPU_HW_IP_VCN_JPEG
,
68 enum amdgpu_ib_pool_type
{
69 /* Normal submissions to the top of the pipeline. */
70 AMDGPU_IB_POOL_DELAYED
,
71 /* Immediate submissions to the bottom of the pipeline. */
72 AMDGPU_IB_POOL_IMMEDIATE
,
73 /* Direct submission to the ring buffer during init and reset. */
74 AMDGPU_IB_POOL_DIRECT
,
82 struct amdgpu_cs_parser
;
87 struct drm_gpu_scheduler
*sched
[AMDGPU_MAX_HWIP_RINGS
];
93 struct amdgpu_fence_driver
{
95 volatile uint32_t *cpu_addr
;
96 /* sync_seq is protected by ring emission lock */
100 struct amdgpu_irq_src
*irq_src
;
102 struct timer_list fallback_timer
;
103 unsigned num_fences_mask
;
105 struct dma_fence
**fences
;
108 int amdgpu_fence_driver_init(struct amdgpu_device
*adev
);
109 void amdgpu_fence_driver_fini(struct amdgpu_device
*adev
);
110 void amdgpu_fence_driver_force_completion(struct amdgpu_ring
*ring
);
112 int amdgpu_fence_driver_init_ring(struct amdgpu_ring
*ring
,
113 unsigned num_hw_submission
);
114 int amdgpu_fence_driver_start_ring(struct amdgpu_ring
*ring
,
115 struct amdgpu_irq_src
*irq_src
,
117 void amdgpu_fence_driver_suspend(struct amdgpu_device
*adev
);
118 void amdgpu_fence_driver_resume(struct amdgpu_device
*adev
);
119 int amdgpu_fence_emit(struct amdgpu_ring
*ring
, struct dma_fence
**fence
,
121 int amdgpu_fence_emit_polling(struct amdgpu_ring
*ring
, uint32_t *s
,
123 bool amdgpu_fence_process(struct amdgpu_ring
*ring
);
124 int amdgpu_fence_wait_empty(struct amdgpu_ring
*ring
);
125 signed long amdgpu_fence_wait_polling(struct amdgpu_ring
*ring
,
127 signed long timeout
);
128 unsigned amdgpu_fence_count_emitted(struct amdgpu_ring
*ring
);
134 /* provided by hw blocks that expose a ring buffer for commands */
135 struct amdgpu_ring_funcs
{
136 enum amdgpu_ring_type type
;
139 bool support_64bit_ptrs
;
144 /* ring read/write ptr handling */
145 u64 (*get_rptr
)(struct amdgpu_ring
*ring
);
146 u64 (*get_wptr
)(struct amdgpu_ring
*ring
);
147 void (*set_wptr
)(struct amdgpu_ring
*ring
);
148 /* validating and patching of IBs */
149 int (*parse_cs
)(struct amdgpu_cs_parser
*p
, uint32_t ib_idx
);
150 int (*patch_cs_in_place
)(struct amdgpu_cs_parser
*p
, uint32_t ib_idx
);
151 /* constants to calculate how many DW are needed for an emit */
152 unsigned emit_frame_size
;
153 unsigned emit_ib_size
;
154 /* command emit functions */
155 void (*emit_ib
)(struct amdgpu_ring
*ring
,
156 struct amdgpu_job
*job
,
157 struct amdgpu_ib
*ib
,
159 void (*emit_fence
)(struct amdgpu_ring
*ring
, uint64_t addr
,
160 uint64_t seq
, unsigned flags
);
161 void (*emit_pipeline_sync
)(struct amdgpu_ring
*ring
);
162 void (*emit_vm_flush
)(struct amdgpu_ring
*ring
, unsigned vmid
,
164 void (*emit_hdp_flush
)(struct amdgpu_ring
*ring
);
165 void (*emit_gds_switch
)(struct amdgpu_ring
*ring
, uint32_t vmid
,
166 uint32_t gds_base
, uint32_t gds_size
,
167 uint32_t gws_base
, uint32_t gws_size
,
168 uint32_t oa_base
, uint32_t oa_size
);
169 /* testing functions */
170 int (*test_ring
)(struct amdgpu_ring
*ring
);
171 int (*test_ib
)(struct amdgpu_ring
*ring
, long timeout
);
172 /* insert NOP packets */
173 void (*insert_nop
)(struct amdgpu_ring
*ring
, uint32_t count
);
174 void (*insert_start
)(struct amdgpu_ring
*ring
);
175 void (*insert_end
)(struct amdgpu_ring
*ring
);
176 /* pad the indirect buffer to the necessary number of dw */
177 void (*pad_ib
)(struct amdgpu_ring
*ring
, struct amdgpu_ib
*ib
);
178 unsigned (*init_cond_exec
)(struct amdgpu_ring
*ring
);
179 void (*patch_cond_exec
)(struct amdgpu_ring
*ring
, unsigned offset
);
180 /* note usage for clock and power gating */
181 void (*begin_use
)(struct amdgpu_ring
*ring
);
182 void (*end_use
)(struct amdgpu_ring
*ring
);
183 void (*emit_switch_buffer
) (struct amdgpu_ring
*ring
);
184 void (*emit_cntxcntl
) (struct amdgpu_ring
*ring
, uint32_t flags
);
185 void (*emit_rreg
)(struct amdgpu_ring
*ring
, uint32_t reg
,
186 uint32_t reg_val_offs
);
187 void (*emit_wreg
)(struct amdgpu_ring
*ring
, uint32_t reg
, uint32_t val
);
188 void (*emit_reg_wait
)(struct amdgpu_ring
*ring
, uint32_t reg
,
189 uint32_t val
, uint32_t mask
);
190 void (*emit_reg_write_reg_wait
)(struct amdgpu_ring
*ring
,
191 uint32_t reg0
, uint32_t reg1
,
192 uint32_t ref
, uint32_t mask
);
193 void (*emit_frame_cntl
)(struct amdgpu_ring
*ring
, bool start
,
195 /* Try to soft recover the ring to make the fence signal */
196 void (*soft_recovery
)(struct amdgpu_ring
*ring
, unsigned vmid
);
197 int (*preempt_ib
)(struct amdgpu_ring
*ring
);
198 void (*emit_mem_sync
)(struct amdgpu_ring
*ring
);
202 struct amdgpu_device
*adev
;
203 const struct amdgpu_ring_funcs
*funcs
;
204 struct amdgpu_fence_driver fence_drv
;
205 struct drm_gpu_scheduler sched
;
207 struct amdgpu_bo
*ring_obj
;
208 volatile uint32_t *ring
;
222 struct amdgpu_bo
*mqd_obj
;
223 uint64_t mqd_gpu_addr
;
225 uint64_t eop_gpu_addr
;
231 uint64_t current_ctx
;
234 unsigned trail_fence_offs
;
235 u64 trail_fence_gpu_addr
;
236 volatile u32
*trail_fence_cpu_addr
;
237 unsigned cond_exe_offs
;
238 u64 cond_exe_gpu_addr
;
239 volatile u32
*cond_exe_cpu_addr
;
241 struct dma_fence
*vmid_wait
;
242 bool has_compute_vm_bug
;
245 atomic_t num_jobs
[DRM_SCHED_PRIORITY_MAX
];
246 struct mutex priority_mutex
;
247 /* protected by priority_mutex */
250 #if defined(CONFIG_DEBUG_FS)
255 #define amdgpu_ring_parse_cs(r, p, ib) ((r)->funcs->parse_cs((p), (ib)))
256 #define amdgpu_ring_patch_cs_in_place(r, p, ib) ((r)->funcs->patch_cs_in_place((p), (ib)))
257 #define amdgpu_ring_test_ring(r) (r)->funcs->test_ring((r))
258 #define amdgpu_ring_test_ib(r, t) (r)->funcs->test_ib((r), (t))
259 #define amdgpu_ring_get_rptr(r) (r)->funcs->get_rptr((r))
260 #define amdgpu_ring_get_wptr(r) (r)->funcs->get_wptr((r))
261 #define amdgpu_ring_set_wptr(r) (r)->funcs->set_wptr((r))
262 #define amdgpu_ring_emit_ib(r, job, ib, flags) ((r)->funcs->emit_ib((r), (job), (ib), (flags)))
263 #define amdgpu_ring_emit_pipeline_sync(r) (r)->funcs->emit_pipeline_sync((r))
264 #define amdgpu_ring_emit_vm_flush(r, vmid, addr) (r)->funcs->emit_vm_flush((r), (vmid), (addr))
265 #define amdgpu_ring_emit_fence(r, addr, seq, flags) (r)->funcs->emit_fence((r), (addr), (seq), (flags))
266 #define amdgpu_ring_emit_gds_switch(r, v, db, ds, wb, ws, ab, as) (r)->funcs->emit_gds_switch((r), (v), (db), (ds), (wb), (ws), (ab), (as))
267 #define amdgpu_ring_emit_hdp_flush(r) (r)->funcs->emit_hdp_flush((r))
268 #define amdgpu_ring_emit_switch_buffer(r) (r)->funcs->emit_switch_buffer((r))
269 #define amdgpu_ring_emit_cntxcntl(r, d) (r)->funcs->emit_cntxcntl((r), (d))
270 #define amdgpu_ring_emit_rreg(r, d, o) (r)->funcs->emit_rreg((r), (d), (o))
271 #define amdgpu_ring_emit_wreg(r, d, v) (r)->funcs->emit_wreg((r), (d), (v))
272 #define amdgpu_ring_emit_reg_wait(r, d, v, m) (r)->funcs->emit_reg_wait((r), (d), (v), (m))
273 #define amdgpu_ring_emit_reg_write_reg_wait(r, d0, d1, v, m) (r)->funcs->emit_reg_write_reg_wait((r), (d0), (d1), (v), (m))
274 #define amdgpu_ring_emit_frame_cntl(r, b, s) (r)->funcs->emit_frame_cntl((r), (b), (s))
275 #define amdgpu_ring_pad_ib(r, ib) ((r)->funcs->pad_ib((r), (ib)))
276 #define amdgpu_ring_init_cond_exec(r) (r)->funcs->init_cond_exec((r))
277 #define amdgpu_ring_patch_cond_exec(r,o) (r)->funcs->patch_cond_exec((r),(o))
278 #define amdgpu_ring_preempt_ib(r) (r)->funcs->preempt_ib(r)
280 int amdgpu_ring_alloc(struct amdgpu_ring
*ring
, unsigned ndw
);
281 void amdgpu_ring_insert_nop(struct amdgpu_ring
*ring
, uint32_t count
);
282 void amdgpu_ring_generic_pad_ib(struct amdgpu_ring
*ring
, struct amdgpu_ib
*ib
);
283 void amdgpu_ring_commit(struct amdgpu_ring
*ring
);
284 void amdgpu_ring_undo(struct amdgpu_ring
*ring
);
285 int amdgpu_ring_init(struct amdgpu_device
*adev
, struct amdgpu_ring
*ring
,
286 unsigned int ring_size
, struct amdgpu_irq_src
*irq_src
,
287 unsigned int irq_type
, unsigned int prio
);
288 void amdgpu_ring_fini(struct amdgpu_ring
*ring
);
289 void amdgpu_ring_emit_reg_write_reg_wait_helper(struct amdgpu_ring
*ring
,
290 uint32_t reg0
, uint32_t val0
,
291 uint32_t reg1
, uint32_t val1
);
292 bool amdgpu_ring_soft_recovery(struct amdgpu_ring
*ring
, unsigned int vmid
,
293 struct dma_fence
*fence
);
295 static inline void amdgpu_ring_set_preempt_cond_exec(struct amdgpu_ring
*ring
,
298 *ring
->cond_exe_cpu_addr
= cond_exec
;
301 static inline void amdgpu_ring_clear_ring(struct amdgpu_ring
*ring
)
304 while (i
<= ring
->buf_mask
)
305 ring
->ring
[i
++] = ring
->funcs
->nop
;
309 static inline void amdgpu_ring_write(struct amdgpu_ring
*ring
, uint32_t v
)
311 if (ring
->count_dw
<= 0)
312 DRM_ERROR("amdgpu: writing more dwords to the ring than expected!\n");
313 ring
->ring
[ring
->wptr
++ & ring
->buf_mask
] = v
;
314 ring
->wptr
&= ring
->ptr_mask
;
318 static inline void amdgpu_ring_write_multiple(struct amdgpu_ring
*ring
,
319 void *src
, int count_dw
)
321 unsigned occupied
, chunk1
, chunk2
;
324 if (unlikely(ring
->count_dw
< count_dw
))
325 DRM_ERROR("amdgpu: writing more dwords to the ring than expected!\n");
327 occupied
= ring
->wptr
& ring
->buf_mask
;
328 dst
= (void *)&ring
->ring
[occupied
];
329 chunk1
= ring
->buf_mask
+ 1 - occupied
;
330 chunk1
= (chunk1
>= count_dw
) ? count_dw
: chunk1
;
331 chunk2
= count_dw
- chunk1
;
336 memcpy(dst
, src
, chunk1
);
340 dst
= (void *)ring
->ring
;
341 memcpy(dst
, src
, chunk2
);
344 ring
->wptr
+= count_dw
;
345 ring
->wptr
&= ring
->ptr_mask
;
346 ring
->count_dw
-= count_dw
;
349 int amdgpu_ring_test_helper(struct amdgpu_ring
*ring
);
351 int amdgpu_debugfs_ring_init(struct amdgpu_device
*adev
,
352 struct amdgpu_ring
*ring
);
353 void amdgpu_debugfs_ring_fini(struct amdgpu_ring
*ring
);