2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
24 #include <linux/printk.h>
25 #include <linux/slab.h>
26 #include <linux/mm_types.h>
29 #include "kfd_mqd_manager.h"
31 #include "cik_structs.h"
32 #include "oss/oss_2_4_sh_mask.h"
34 static inline struct cik_mqd
*get_mqd(void *mqd
)
36 return (struct cik_mqd
*)mqd
;
39 static int init_mqd(struct mqd_manager
*mm
, void **mqd
,
40 struct kfd_mem_obj
**mqd_mem_obj
, uint64_t *gart_addr
,
41 struct queue_properties
*q
)
47 retval
= kfd_gtt_sa_allocate(mm
->dev
, sizeof(struct cik_mqd
),
53 m
= (struct cik_mqd
*) (*mqd_mem_obj
)->cpu_ptr
;
54 addr
= (*mqd_mem_obj
)->gpu_addr
;
56 memset(m
, 0, ALIGN(sizeof(struct cik_mqd
), 256));
58 m
->header
= 0xC0310800;
59 m
->compute_pipelinestat_enable
= 1;
60 m
->compute_static_thread_mgmt_se0
= 0xFFFFFFFF;
61 m
->compute_static_thread_mgmt_se1
= 0xFFFFFFFF;
62 m
->compute_static_thread_mgmt_se2
= 0xFFFFFFFF;
63 m
->compute_static_thread_mgmt_se3
= 0xFFFFFFFF;
66 * Make sure to use the last queue state saved on mqd when the cp
67 * reassigns the queue, so when queue is switched on/off (e.g over
68 * subscription or quantum timeout) the context will be consistent
70 m
->cp_hqd_persistent_state
=
71 DEFAULT_CP_HQD_PERSISTENT_STATE
| PRELOAD_REQ
;
73 m
->cp_mqd_control
= MQD_CONTROL_PRIV_STATE_EN
;
74 m
->cp_mqd_base_addr_lo
= lower_32_bits(addr
);
75 m
->cp_mqd_base_addr_hi
= upper_32_bits(addr
);
77 m
->cp_hqd_ib_control
= DEFAULT_MIN_IB_AVAIL_SIZE
| IB_ATC_EN
;
78 /* Although WinKFD writes this, I suspect it should not be necessary */
79 m
->cp_hqd_ib_control
= IB_ATC_EN
| DEFAULT_MIN_IB_AVAIL_SIZE
;
81 m
->cp_hqd_quantum
= QUANTUM_EN
| QUANTUM_SCALE_1MS
|
86 * Identifies the pipe relative priority when this queue is connected
87 * to the pipeline. The pipe priority is against the GFX pipe and HP3D.
88 * In KFD we are using a fixed pipe priority set to CS_MEDIUM.
89 * 0 = CS_LOW (typically below GFX)
90 * 1 = CS_MEDIUM (typically between HP3D and GFX
91 * 2 = CS_HIGH (typically above HP3D)
93 m
->cp_hqd_pipe_priority
= 1;
94 m
->cp_hqd_queue_priority
= 15;
96 if (q
->format
== KFD_QUEUE_FORMAT_AQL
)
97 m
->cp_hqd_iq_rptr
= AQL_ENABLE
;
102 retval
= mm
->update_mqd(mm
, m
, q
);
107 static int init_mqd_sdma(struct mqd_manager
*mm
, void **mqd
,
108 struct kfd_mem_obj
**mqd_mem_obj
, uint64_t *gart_addr
,
109 struct queue_properties
*q
)
112 struct cik_sdma_rlc_registers
*m
;
114 retval
= kfd_gtt_sa_allocate(mm
->dev
,
115 sizeof(struct cik_sdma_rlc_registers
),
121 m
= (struct cik_sdma_rlc_registers
*) (*mqd_mem_obj
)->cpu_ptr
;
123 memset(m
, 0, sizeof(struct cik_sdma_rlc_registers
));
127 *gart_addr
= (*mqd_mem_obj
)->gpu_addr
;
129 retval
= mm
->update_mqd(mm
, m
, q
);
134 static void uninit_mqd(struct mqd_manager
*mm
, void *mqd
,
135 struct kfd_mem_obj
*mqd_mem_obj
)
137 kfd_gtt_sa_free(mm
->dev
, mqd_mem_obj
);
140 static void uninit_mqd_sdma(struct mqd_manager
*mm
, void *mqd
,
141 struct kfd_mem_obj
*mqd_mem_obj
)
143 kfd_gtt_sa_free(mm
->dev
, mqd_mem_obj
);
146 static int load_mqd(struct mqd_manager
*mm
, void *mqd
, uint32_t pipe_id
,
147 uint32_t queue_id
, struct queue_properties
*p
,
148 struct mm_struct
*mms
)
150 /* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
151 uint32_t wptr_shift
= (p
->format
== KFD_QUEUE_FORMAT_AQL
? 4 : 0);
152 uint32_t wptr_mask
= (uint32_t)((p
->queue_size
/ sizeof(uint32_t)) - 1);
154 return mm
->dev
->kfd2kgd
->hqd_load(mm
->dev
->kgd
, mqd
, pipe_id
, queue_id
,
155 (uint32_t __user
*)p
->write_ptr
,
156 wptr_shift
, wptr_mask
, mms
);
159 static int load_mqd_sdma(struct mqd_manager
*mm
, void *mqd
,
160 uint32_t pipe_id
, uint32_t queue_id
,
161 struct queue_properties
*p
, struct mm_struct
*mms
)
163 return mm
->dev
->kfd2kgd
->hqd_sdma_load(mm
->dev
->kgd
, mqd
);
166 static int update_mqd(struct mqd_manager
*mm
, void *mqd
,
167 struct queue_properties
*q
)
172 m
->cp_hqd_pq_control
= DEFAULT_RPTR_BLOCK_SIZE
|
173 DEFAULT_MIN_AVAIL_SIZE
| PQ_ATC_EN
;
176 * Calculating queue size which is log base 2 of actual queue size -1
177 * dwords and another -1 for ffs
179 m
->cp_hqd_pq_control
|= ffs(q
->queue_size
/ sizeof(unsigned int))
181 m
->cp_hqd_pq_base_lo
= lower_32_bits((uint64_t)q
->queue_address
>> 8);
182 m
->cp_hqd_pq_base_hi
= upper_32_bits((uint64_t)q
->queue_address
>> 8);
183 m
->cp_hqd_pq_rptr_report_addr_lo
= lower_32_bits((uint64_t)q
->read_ptr
);
184 m
->cp_hqd_pq_rptr_report_addr_hi
= upper_32_bits((uint64_t)q
->read_ptr
);
185 m
->cp_hqd_pq_doorbell_control
= DOORBELL_OFFSET(q
->doorbell_off
);
187 m
->cp_hqd_vmid
= q
->vmid
;
189 if (q
->format
== KFD_QUEUE_FORMAT_AQL
)
190 m
->cp_hqd_pq_control
|= NO_UPDATE_RPTR
;
192 q
->is_active
= (q
->queue_size
> 0 &&
193 q
->queue_address
!= 0 &&
194 q
->queue_percent
> 0);
199 static int update_mqd_sdma(struct mqd_manager
*mm
, void *mqd
,
200 struct queue_properties
*q
)
202 struct cik_sdma_rlc_registers
*m
;
204 m
= get_sdma_mqd(mqd
);
205 m
->sdma_rlc_rb_cntl
= ffs(q
->queue_size
/ sizeof(unsigned int)) <<
206 SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT
|
207 q
->vmid
<< SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT
|
208 1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT
|
209 6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT
;
211 m
->sdma_rlc_rb_base
= lower_32_bits(q
->queue_address
>> 8);
212 m
->sdma_rlc_rb_base_hi
= upper_32_bits(q
->queue_address
>> 8);
213 m
->sdma_rlc_rb_rptr_addr_lo
= lower_32_bits((uint64_t)q
->read_ptr
);
214 m
->sdma_rlc_rb_rptr_addr_hi
= upper_32_bits((uint64_t)q
->read_ptr
);
215 m
->sdma_rlc_doorbell
=
216 q
->doorbell_off
<< SDMA0_RLC0_DOORBELL__OFFSET__SHIFT
;
218 m
->sdma_rlc_virtual_addr
= q
->sdma_vm_addr
;
220 m
->sdma_engine_id
= q
->sdma_engine_id
;
221 m
->sdma_queue_id
= q
->sdma_queue_id
;
223 q
->is_active
= (q
->queue_size
> 0 &&
224 q
->queue_address
!= 0 &&
225 q
->queue_percent
> 0);
230 static int destroy_mqd(struct mqd_manager
*mm
, void *mqd
,
231 enum kfd_preempt_type type
,
232 unsigned int timeout
, uint32_t pipe_id
,
235 return mm
->dev
->kfd2kgd
->hqd_destroy(mm
->dev
->kgd
, mqd
, type
, timeout
,
240 * preempt type here is ignored because there is only one way
241 * to preempt sdma queue
243 static int destroy_mqd_sdma(struct mqd_manager
*mm
, void *mqd
,
244 enum kfd_preempt_type type
,
245 unsigned int timeout
, uint32_t pipe_id
,
248 return mm
->dev
->kfd2kgd
->hqd_sdma_destroy(mm
->dev
->kgd
, mqd
, timeout
);
251 static bool is_occupied(struct mqd_manager
*mm
, void *mqd
,
252 uint64_t queue_address
, uint32_t pipe_id
,
256 return mm
->dev
->kfd2kgd
->hqd_is_occupied(mm
->dev
->kgd
, queue_address
,
261 static bool is_occupied_sdma(struct mqd_manager
*mm
, void *mqd
,
262 uint64_t queue_address
, uint32_t pipe_id
,
265 return mm
->dev
->kfd2kgd
->hqd_sdma_is_occupied(mm
->dev
->kgd
, mqd
);
269 * HIQ MQD Implementation, concrete implementation for HIQ MQD implementation.
270 * The HIQ queue in Kaveri is using the same MQD structure as all the user mode
271 * queues but with different initial values.
274 static int init_mqd_hiq(struct mqd_manager
*mm
, void **mqd
,
275 struct kfd_mem_obj
**mqd_mem_obj
, uint64_t *gart_addr
,
276 struct queue_properties
*q
)
282 retval
= kfd_gtt_sa_allocate(mm
->dev
, sizeof(struct cik_mqd
),
288 m
= (struct cik_mqd
*) (*mqd_mem_obj
)->cpu_ptr
;
289 addr
= (*mqd_mem_obj
)->gpu_addr
;
291 memset(m
, 0, ALIGN(sizeof(struct cik_mqd
), 256));
293 m
->header
= 0xC0310800;
294 m
->compute_pipelinestat_enable
= 1;
295 m
->compute_static_thread_mgmt_se0
= 0xFFFFFFFF;
296 m
->compute_static_thread_mgmt_se1
= 0xFFFFFFFF;
297 m
->compute_static_thread_mgmt_se2
= 0xFFFFFFFF;
298 m
->compute_static_thread_mgmt_se3
= 0xFFFFFFFF;
300 m
->cp_hqd_persistent_state
= DEFAULT_CP_HQD_PERSISTENT_STATE
|
302 m
->cp_hqd_quantum
= QUANTUM_EN
| QUANTUM_SCALE_1MS
|
303 QUANTUM_DURATION(10);
305 m
->cp_mqd_control
= MQD_CONTROL_PRIV_STATE_EN
;
306 m
->cp_mqd_base_addr_lo
= lower_32_bits(addr
);
307 m
->cp_mqd_base_addr_hi
= upper_32_bits(addr
);
309 m
->cp_hqd_ib_control
= DEFAULT_MIN_IB_AVAIL_SIZE
;
313 * Identifies the pipe relative priority when this queue is connected
314 * to the pipeline. The pipe priority is against the GFX pipe and HP3D.
315 * In KFD we are using a fixed pipe priority set to CS_MEDIUM.
316 * 0 = CS_LOW (typically below GFX)
317 * 1 = CS_MEDIUM (typically between HP3D and GFX
318 * 2 = CS_HIGH (typically above HP3D)
320 m
->cp_hqd_pipe_priority
= 1;
321 m
->cp_hqd_queue_priority
= 15;
326 retval
= mm
->update_mqd(mm
, m
, q
);
331 static int update_mqd_hiq(struct mqd_manager
*mm
, void *mqd
,
332 struct queue_properties
*q
)
337 m
->cp_hqd_pq_control
= DEFAULT_RPTR_BLOCK_SIZE
|
338 DEFAULT_MIN_AVAIL_SIZE
|
343 * Calculating queue size which is log base 2 of actual queue
346 m
->cp_hqd_pq_control
|= ffs(q
->queue_size
/ sizeof(unsigned int))
348 m
->cp_hqd_pq_base_lo
= lower_32_bits((uint64_t)q
->queue_address
>> 8);
349 m
->cp_hqd_pq_base_hi
= upper_32_bits((uint64_t)q
->queue_address
>> 8);
350 m
->cp_hqd_pq_rptr_report_addr_lo
= lower_32_bits((uint64_t)q
->read_ptr
);
351 m
->cp_hqd_pq_rptr_report_addr_hi
= upper_32_bits((uint64_t)q
->read_ptr
);
352 m
->cp_hqd_pq_doorbell_control
= DOORBELL_OFFSET(q
->doorbell_off
);
354 m
->cp_hqd_vmid
= q
->vmid
;
356 q
->is_active
= (q
->queue_size
> 0 &&
357 q
->queue_address
!= 0 &&
358 q
->queue_percent
> 0);
363 struct cik_sdma_rlc_registers
*get_sdma_mqd(void *mqd
)
365 struct cik_sdma_rlc_registers
*m
;
367 m
= (struct cik_sdma_rlc_registers
*)mqd
;
372 struct mqd_manager
*mqd_manager_init_cik(enum KFD_MQD_TYPE type
,
375 struct mqd_manager
*mqd
;
377 if (WARN_ON(type
>= KFD_MQD_TYPE_MAX
))
380 mqd
= kzalloc(sizeof(*mqd
), GFP_KERNEL
);
387 case KFD_MQD_TYPE_CP
:
388 case KFD_MQD_TYPE_COMPUTE
:
389 mqd
->init_mqd
= init_mqd
;
390 mqd
->uninit_mqd
= uninit_mqd
;
391 mqd
->load_mqd
= load_mqd
;
392 mqd
->update_mqd
= update_mqd
;
393 mqd
->destroy_mqd
= destroy_mqd
;
394 mqd
->is_occupied
= is_occupied
;
396 case KFD_MQD_TYPE_HIQ
:
397 mqd
->init_mqd
= init_mqd_hiq
;
398 mqd
->uninit_mqd
= uninit_mqd
;
399 mqd
->load_mqd
= load_mqd
;
400 mqd
->update_mqd
= update_mqd_hiq
;
401 mqd
->destroy_mqd
= destroy_mqd
;
402 mqd
->is_occupied
= is_occupied
;
404 case KFD_MQD_TYPE_SDMA
:
405 mqd
->init_mqd
= init_mqd_sdma
;
406 mqd
->uninit_mqd
= uninit_mqd_sdma
;
407 mqd
->load_mqd
= load_mqd_sdma
;
408 mqd
->update_mqd
= update_mqd_sdma
;
409 mqd
->destroy_mqd
= destroy_mqd_sdma
;
410 mqd
->is_occupied
= is_occupied_sdma
;