2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
24 #include <linux/printk.h>
25 #include <linux/slab.h>
26 #include <linux/mm_types.h>
29 #include "kfd_mqd_manager.h"
31 #include "cik_structs.h"
32 #include "oss/oss_2_4_sh_mask.h"
34 static inline struct cik_mqd
*get_mqd(void *mqd
)
36 return (struct cik_mqd
*)mqd
;
39 static int init_mqd(struct mqd_manager
*mm
, void **mqd
,
40 struct kfd_mem_obj
**mqd_mem_obj
, uint64_t *gart_addr
,
41 struct queue_properties
*q
)
47 BUG_ON(!mm
|| !q
|| !mqd
);
49 pr_debug("kfd: In func %s\n", __func__
);
51 retval
= kfd_gtt_sa_allocate(mm
->dev
, sizeof(struct cik_mqd
),
57 m
= (struct cik_mqd
*) (*mqd_mem_obj
)->cpu_ptr
;
58 addr
= (*mqd_mem_obj
)->gpu_addr
;
60 memset(m
, 0, ALIGN(sizeof(struct cik_mqd
), 256));
62 m
->header
= 0xC0310800;
63 m
->compute_pipelinestat_enable
= 1;
64 m
->compute_static_thread_mgmt_se0
= 0xFFFFFFFF;
65 m
->compute_static_thread_mgmt_se1
= 0xFFFFFFFF;
66 m
->compute_static_thread_mgmt_se2
= 0xFFFFFFFF;
67 m
->compute_static_thread_mgmt_se3
= 0xFFFFFFFF;
70 * Make sure to use the last queue state saved on mqd when the cp
71 * reassigns the queue, so when queue is switched on/off (e.g over
72 * subscription or quantum timeout) the context will be consistent
74 m
->cp_hqd_persistent_state
=
75 DEFAULT_CP_HQD_PERSISTENT_STATE
| PRELOAD_REQ
;
77 m
->cp_mqd_control
= MQD_CONTROL_PRIV_STATE_EN
;
78 m
->cp_mqd_base_addr_lo
= lower_32_bits(addr
);
79 m
->cp_mqd_base_addr_hi
= upper_32_bits(addr
);
81 m
->cp_hqd_ib_control
= DEFAULT_MIN_IB_AVAIL_SIZE
| IB_ATC_EN
;
82 /* Although WinKFD writes this, I suspect it should not be necessary */
83 m
->cp_hqd_ib_control
= IB_ATC_EN
| DEFAULT_MIN_IB_AVAIL_SIZE
;
85 m
->cp_hqd_quantum
= QUANTUM_EN
| QUANTUM_SCALE_1MS
|
90 * Identifies the pipe relative priority when this queue is connected
91 * to the pipeline. The pipe priority is against the GFX pipe and HP3D.
92 * In KFD we are using a fixed pipe priority set to CS_MEDIUM.
93 * 0 = CS_LOW (typically below GFX)
94 * 1 = CS_MEDIUM (typically between HP3D and GFX
95 * 2 = CS_HIGH (typically above HP3D)
97 m
->cp_hqd_pipe_priority
= 1;
98 m
->cp_hqd_queue_priority
= 15;
100 if (q
->format
== KFD_QUEUE_FORMAT_AQL
)
101 m
->cp_hqd_iq_rptr
= AQL_ENABLE
;
104 if (gart_addr
!= NULL
)
106 retval
= mm
->update_mqd(mm
, m
, q
);
111 static int init_mqd_sdma(struct mqd_manager
*mm
, void **mqd
,
112 struct kfd_mem_obj
**mqd_mem_obj
, uint64_t *gart_addr
,
113 struct queue_properties
*q
)
116 struct cik_sdma_rlc_registers
*m
;
118 BUG_ON(!mm
|| !mqd
|| !mqd_mem_obj
);
120 retval
= kfd_gtt_sa_allocate(mm
->dev
,
121 sizeof(struct cik_sdma_rlc_registers
),
127 m
= (struct cik_sdma_rlc_registers
*) (*mqd_mem_obj
)->cpu_ptr
;
129 memset(m
, 0, sizeof(struct cik_sdma_rlc_registers
));
132 if (gart_addr
!= NULL
)
133 *gart_addr
= (*mqd_mem_obj
)->gpu_addr
;
135 retval
= mm
->update_mqd(mm
, m
, q
);
140 static void uninit_mqd(struct mqd_manager
*mm
, void *mqd
,
141 struct kfd_mem_obj
*mqd_mem_obj
)
144 kfd_gtt_sa_free(mm
->dev
, mqd_mem_obj
);
147 static void uninit_mqd_sdma(struct mqd_manager
*mm
, void *mqd
,
148 struct kfd_mem_obj
*mqd_mem_obj
)
151 kfd_gtt_sa_free(mm
->dev
, mqd_mem_obj
);
154 static int load_mqd(struct mqd_manager
*mm
, void *mqd
, uint32_t pipe_id
,
155 uint32_t queue_id
, uint32_t __user
*wptr
)
157 return mm
->dev
->kfd2kgd
->hqd_load
158 (mm
->dev
->kgd
, mqd
, pipe_id
, queue_id
, wptr
);
161 static int load_mqd_sdma(struct mqd_manager
*mm
, void *mqd
,
162 uint32_t pipe_id
, uint32_t queue_id
,
163 uint32_t __user
*wptr
)
165 return mm
->dev
->kfd2kgd
->hqd_sdma_load(mm
->dev
->kgd
, mqd
);
168 static int update_mqd(struct mqd_manager
*mm
, void *mqd
,
169 struct queue_properties
*q
)
173 BUG_ON(!mm
|| !q
|| !mqd
);
175 pr_debug("kfd: In func %s\n", __func__
);
178 m
->cp_hqd_pq_control
= DEFAULT_RPTR_BLOCK_SIZE
|
179 DEFAULT_MIN_AVAIL_SIZE
| PQ_ATC_EN
;
182 * Calculating queue size which is log base 2 of actual queue size -1
183 * dwords and another -1 for ffs
185 m
->cp_hqd_pq_control
|= ffs(q
->queue_size
/ sizeof(unsigned int))
187 m
->cp_hqd_pq_base_lo
= lower_32_bits((uint64_t)q
->queue_address
>> 8);
188 m
->cp_hqd_pq_base_hi
= upper_32_bits((uint64_t)q
->queue_address
>> 8);
189 m
->cp_hqd_pq_rptr_report_addr_lo
= lower_32_bits((uint64_t)q
->read_ptr
);
190 m
->cp_hqd_pq_rptr_report_addr_hi
= upper_32_bits((uint64_t)q
->read_ptr
);
191 m
->cp_hqd_pq_doorbell_control
= DOORBELL_EN
|
192 DOORBELL_OFFSET(q
->doorbell_off
);
194 m
->cp_hqd_vmid
= q
->vmid
;
196 if (q
->format
== KFD_QUEUE_FORMAT_AQL
) {
197 m
->cp_hqd_pq_control
|= NO_UPDATE_RPTR
;
200 m
->cp_hqd_active
= 0;
201 q
->is_active
= false;
202 if (q
->queue_size
> 0 &&
203 q
->queue_address
!= 0 &&
204 q
->queue_percent
> 0) {
205 m
->cp_hqd_active
= 1;
212 static int update_mqd_sdma(struct mqd_manager
*mm
, void *mqd
,
213 struct queue_properties
*q
)
215 struct cik_sdma_rlc_registers
*m
;
217 BUG_ON(!mm
|| !mqd
|| !q
);
219 m
= get_sdma_mqd(mqd
);
220 m
->sdma_rlc_rb_cntl
= ffs(q
->queue_size
/ sizeof(unsigned int)) <<
221 SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT
|
222 q
->vmid
<< SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT
|
223 1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT
|
224 6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT
;
226 m
->sdma_rlc_rb_base
= lower_32_bits(q
->queue_address
>> 8);
227 m
->sdma_rlc_rb_base_hi
= upper_32_bits(q
->queue_address
>> 8);
228 m
->sdma_rlc_rb_rptr_addr_lo
= lower_32_bits((uint64_t)q
->read_ptr
);
229 m
->sdma_rlc_rb_rptr_addr_hi
= upper_32_bits((uint64_t)q
->read_ptr
);
230 m
->sdma_rlc_doorbell
= q
->doorbell_off
<<
231 SDMA0_RLC0_DOORBELL__OFFSET__SHIFT
|
232 1 << SDMA0_RLC0_DOORBELL__ENABLE__SHIFT
;
234 m
->sdma_rlc_virtual_addr
= q
->sdma_vm_addr
;
236 m
->sdma_engine_id
= q
->sdma_engine_id
;
237 m
->sdma_queue_id
= q
->sdma_queue_id
;
239 q
->is_active
= false;
240 if (q
->queue_size
> 0 &&
241 q
->queue_address
!= 0 &&
242 q
->queue_percent
> 0) {
243 m
->sdma_rlc_rb_cntl
|=
244 1 << SDMA0_RLC0_RB_CNTL__RB_ENABLE__SHIFT
;
252 static int destroy_mqd(struct mqd_manager
*mm
, void *mqd
,
253 enum kfd_preempt_type type
,
254 unsigned int timeout
, uint32_t pipe_id
,
257 return mm
->dev
->kfd2kgd
->hqd_destroy(mm
->dev
->kgd
, type
, timeout
,
262 * preempt type here is ignored because there is only one way
263 * to preempt sdma queue
265 static int destroy_mqd_sdma(struct mqd_manager
*mm
, void *mqd
,
266 enum kfd_preempt_type type
,
267 unsigned int timeout
, uint32_t pipe_id
,
270 return mm
->dev
->kfd2kgd
->hqd_sdma_destroy(mm
->dev
->kgd
, mqd
, timeout
);
273 static bool is_occupied(struct mqd_manager
*mm
, void *mqd
,
274 uint64_t queue_address
, uint32_t pipe_id
,
278 return mm
->dev
->kfd2kgd
->hqd_is_occupied(mm
->dev
->kgd
, queue_address
,
283 static bool is_occupied_sdma(struct mqd_manager
*mm
, void *mqd
,
284 uint64_t queue_address
, uint32_t pipe_id
,
287 return mm
->dev
->kfd2kgd
->hqd_sdma_is_occupied(mm
->dev
->kgd
, mqd
);
291 * HIQ MQD Implementation, concrete implementation for HIQ MQD implementation.
292 * The HIQ queue in Kaveri is using the same MQD structure as all the user mode
293 * queues but with different initial values.
296 static int init_mqd_hiq(struct mqd_manager
*mm
, void **mqd
,
297 struct kfd_mem_obj
**mqd_mem_obj
, uint64_t *gart_addr
,
298 struct queue_properties
*q
)
304 BUG_ON(!mm
|| !q
|| !mqd
|| !mqd_mem_obj
);
306 pr_debug("kfd: In func %s\n", __func__
);
308 retval
= kfd_gtt_sa_allocate(mm
->dev
, sizeof(struct cik_mqd
),
314 m
= (struct cik_mqd
*) (*mqd_mem_obj
)->cpu_ptr
;
315 addr
= (*mqd_mem_obj
)->gpu_addr
;
317 memset(m
, 0, ALIGN(sizeof(struct cik_mqd
), 256));
319 m
->header
= 0xC0310800;
320 m
->compute_pipelinestat_enable
= 1;
321 m
->compute_static_thread_mgmt_se0
= 0xFFFFFFFF;
322 m
->compute_static_thread_mgmt_se1
= 0xFFFFFFFF;
323 m
->compute_static_thread_mgmt_se2
= 0xFFFFFFFF;
324 m
->compute_static_thread_mgmt_se3
= 0xFFFFFFFF;
326 m
->cp_hqd_persistent_state
= DEFAULT_CP_HQD_PERSISTENT_STATE
|
328 m
->cp_hqd_quantum
= QUANTUM_EN
| QUANTUM_SCALE_1MS
|
329 QUANTUM_DURATION(10);
331 m
->cp_mqd_control
= MQD_CONTROL_PRIV_STATE_EN
;
332 m
->cp_mqd_base_addr_lo
= lower_32_bits(addr
);
333 m
->cp_mqd_base_addr_hi
= upper_32_bits(addr
);
335 m
->cp_hqd_ib_control
= DEFAULT_MIN_IB_AVAIL_SIZE
;
339 * Identifies the pipe relative priority when this queue is connected
340 * to the pipeline. The pipe priority is against the GFX pipe and HP3D.
341 * In KFD we are using a fixed pipe priority set to CS_MEDIUM.
342 * 0 = CS_LOW (typically below GFX)
343 * 1 = CS_MEDIUM (typically between HP3D and GFX
344 * 2 = CS_HIGH (typically above HP3D)
346 m
->cp_hqd_pipe_priority
= 1;
347 m
->cp_hqd_queue_priority
= 15;
352 retval
= mm
->update_mqd(mm
, m
, q
);
357 static int update_mqd_hiq(struct mqd_manager
*mm
, void *mqd
,
358 struct queue_properties
*q
)
362 BUG_ON(!mm
|| !q
|| !mqd
);
364 pr_debug("kfd: In func %s\n", __func__
);
367 m
->cp_hqd_pq_control
= DEFAULT_RPTR_BLOCK_SIZE
|
368 DEFAULT_MIN_AVAIL_SIZE
|
373 * Calculating queue size which is log base 2 of actual queue
376 m
->cp_hqd_pq_control
|= ffs(q
->queue_size
/ sizeof(unsigned int))
378 m
->cp_hqd_pq_base_lo
= lower_32_bits((uint64_t)q
->queue_address
>> 8);
379 m
->cp_hqd_pq_base_hi
= upper_32_bits((uint64_t)q
->queue_address
>> 8);
380 m
->cp_hqd_pq_rptr_report_addr_lo
= lower_32_bits((uint64_t)q
->read_ptr
);
381 m
->cp_hqd_pq_rptr_report_addr_hi
= upper_32_bits((uint64_t)q
->read_ptr
);
382 m
->cp_hqd_pq_doorbell_control
= DOORBELL_EN
|
383 DOORBELL_OFFSET(q
->doorbell_off
);
385 m
->cp_hqd_vmid
= q
->vmid
;
387 m
->cp_hqd_active
= 0;
388 q
->is_active
= false;
389 if (q
->queue_size
> 0 &&
390 q
->queue_address
!= 0 &&
391 q
->queue_percent
> 0) {
392 m
->cp_hqd_active
= 1;
399 struct cik_sdma_rlc_registers
*get_sdma_mqd(void *mqd
)
401 struct cik_sdma_rlc_registers
*m
;
405 m
= (struct cik_sdma_rlc_registers
*)mqd
;
410 struct mqd_manager
*mqd_manager_init_cik(enum KFD_MQD_TYPE type
,
413 struct mqd_manager
*mqd
;
416 BUG_ON(type
>= KFD_MQD_TYPE_MAX
);
418 pr_debug("kfd: In func %s\n", __func__
);
420 mqd
= kzalloc(sizeof(struct mqd_manager
), GFP_KERNEL
);
427 case KFD_MQD_TYPE_CP
:
428 case KFD_MQD_TYPE_COMPUTE
:
429 mqd
->init_mqd
= init_mqd
;
430 mqd
->uninit_mqd
= uninit_mqd
;
431 mqd
->load_mqd
= load_mqd
;
432 mqd
->update_mqd
= update_mqd
;
433 mqd
->destroy_mqd
= destroy_mqd
;
434 mqd
->is_occupied
= is_occupied
;
436 case KFD_MQD_TYPE_HIQ
:
437 mqd
->init_mqd
= init_mqd_hiq
;
438 mqd
->uninit_mqd
= uninit_mqd
;
439 mqd
->load_mqd
= load_mqd
;
440 mqd
->update_mqd
= update_mqd_hiq
;
441 mqd
->destroy_mqd
= destroy_mqd
;
442 mqd
->is_occupied
= is_occupied
;
444 case KFD_MQD_TYPE_SDMA
:
445 mqd
->init_mqd
= init_mqd_sdma
;
446 mqd
->uninit_mqd
= uninit_mqd_sdma
;
447 mqd
->load_mqd
= load_mqd_sdma
;
448 mqd
->update_mqd
= update_mqd_sdma
;
449 mqd
->destroy_mqd
= destroy_mqd_sdma
;
450 mqd
->is_occupied
= is_occupied_sdma
;