2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
24 #include <linux/printk.h>
25 #include <linux/slab.h>
26 #include <linux/mm_types.h>
29 #include "kfd_mqd_manager.h"
30 #include "vi_structs.h"
31 #include "gca/gfx_8_0_sh_mask.h"
32 #include "gca/gfx_8_0_enum.h"
34 #define CP_MQD_CONTROL__PRIV_STATE__SHIFT 0x8
36 static inline struct vi_mqd
*get_mqd(void *mqd
)
38 return (struct vi_mqd
*)mqd
;
41 static int init_mqd(struct mqd_manager
*mm
, void **mqd
,
42 struct kfd_mem_obj
**mqd_mem_obj
, uint64_t *gart_addr
,
43 struct queue_properties
*q
)
49 retval
= kfd_gtt_sa_allocate(mm
->dev
, sizeof(struct vi_mqd
),
54 m
= (struct vi_mqd
*) (*mqd_mem_obj
)->cpu_ptr
;
55 addr
= (*mqd_mem_obj
)->gpu_addr
;
57 memset(m
, 0, sizeof(struct vi_mqd
));
59 m
->header
= 0xC0310800;
60 m
->compute_pipelinestat_enable
= 1;
61 m
->compute_static_thread_mgmt_se0
= 0xFFFFFFFF;
62 m
->compute_static_thread_mgmt_se1
= 0xFFFFFFFF;
63 m
->compute_static_thread_mgmt_se2
= 0xFFFFFFFF;
64 m
->compute_static_thread_mgmt_se3
= 0xFFFFFFFF;
66 m
->cp_hqd_persistent_state
= CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK
|
67 0x53 << CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT
;
69 m
->cp_mqd_control
= 1 << CP_MQD_CONTROL__PRIV_STATE__SHIFT
|
70 MTYPE_UC
<< CP_MQD_CONTROL__MTYPE__SHIFT
;
72 m
->cp_mqd_base_addr_lo
= lower_32_bits(addr
);
73 m
->cp_mqd_base_addr_hi
= upper_32_bits(addr
);
75 m
->cp_hqd_quantum
= 1 << CP_HQD_QUANTUM__QUANTUM_EN__SHIFT
|
76 1 << CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT
|
77 10 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT
;
79 m
->cp_hqd_pipe_priority
= 1;
80 m
->cp_hqd_queue_priority
= 15;
82 m
->cp_hqd_eop_rptr
= 1 << CP_HQD_EOP_RPTR__INIT_FETCHER__SHIFT
;
84 if (q
->format
== KFD_QUEUE_FORMAT_AQL
)
85 m
->cp_hqd_iq_rptr
= 1;
88 if (gart_addr
!= NULL
)
90 retval
= mm
->update_mqd(mm
, m
, q
);
95 static int load_mqd(struct mqd_manager
*mm
, void *mqd
,
96 uint32_t pipe_id
, uint32_t queue_id
,
97 uint32_t __user
*wptr
)
99 return mm
->dev
->kfd2kgd
->hqd_load
100 (mm
->dev
->kgd
, mqd
, pipe_id
, queue_id
, wptr
);
103 static int __update_mqd(struct mqd_manager
*mm
, void *mqd
,
104 struct queue_properties
*q
, unsigned int mtype
,
105 unsigned int atc_bit
)
109 BUG_ON(!mm
|| !q
|| !mqd
);
111 pr_debug("kfd: In func %s\n", __func__
);
115 m
->cp_hqd_pq_control
= 5 << CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT
|
116 atc_bit
<< CP_HQD_PQ_CONTROL__PQ_ATC__SHIFT
|
117 mtype
<< CP_HQD_PQ_CONTROL__MTYPE__SHIFT
;
118 m
->cp_hqd_pq_control
|=
119 ffs(q
->queue_size
/ sizeof(unsigned int)) - 1 - 1;
120 pr_debug("kfd: cp_hqd_pq_control 0x%x\n", m
->cp_hqd_pq_control
);
122 m
->cp_hqd_pq_base_lo
= lower_32_bits((uint64_t)q
->queue_address
>> 8);
123 m
->cp_hqd_pq_base_hi
= upper_32_bits((uint64_t)q
->queue_address
>> 8);
125 m
->cp_hqd_pq_rptr_report_addr_lo
= lower_32_bits((uint64_t)q
->read_ptr
);
126 m
->cp_hqd_pq_rptr_report_addr_hi
= upper_32_bits((uint64_t)q
->read_ptr
);
128 m
->cp_hqd_pq_doorbell_control
=
129 1 << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN__SHIFT
|
131 CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT
;
132 pr_debug("kfd: cp_hqd_pq_doorbell_control 0x%x\n",
133 m
->cp_hqd_pq_doorbell_control
);
135 m
->cp_hqd_eop_control
= atc_bit
<< CP_HQD_EOP_CONTROL__EOP_ATC__SHIFT
|
136 mtype
<< CP_HQD_EOP_CONTROL__MTYPE__SHIFT
;
138 m
->cp_hqd_ib_control
= atc_bit
<< CP_HQD_IB_CONTROL__IB_ATC__SHIFT
|
139 3 << CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT
|
140 mtype
<< CP_HQD_IB_CONTROL__MTYPE__SHIFT
;
142 m
->cp_hqd_eop_control
|=
143 ffs(q
->eop_ring_buffer_size
/ sizeof(unsigned int)) - 1 - 1;
144 m
->cp_hqd_eop_base_addr_lo
=
145 lower_32_bits(q
->eop_ring_buffer_address
>> 8);
146 m
->cp_hqd_eop_base_addr_hi
=
147 upper_32_bits(q
->eop_ring_buffer_address
>> 8);
149 m
->cp_hqd_iq_timer
= atc_bit
<< CP_HQD_IQ_TIMER__IQ_ATC__SHIFT
|
150 mtype
<< CP_HQD_IQ_TIMER__MTYPE__SHIFT
;
152 m
->cp_hqd_vmid
= q
->vmid
;
154 if (q
->format
== KFD_QUEUE_FORMAT_AQL
) {
155 m
->cp_hqd_pq_control
|= CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK
|
156 2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT
;
159 m
->cp_hqd_active
= 0;
160 q
->is_active
= false;
161 if (q
->queue_size
> 0 &&
162 q
->queue_address
!= 0 &&
163 q
->queue_percent
> 0) {
164 m
->cp_hqd_active
= 1;
172 static int update_mqd(struct mqd_manager
*mm
, void *mqd
,
173 struct queue_properties
*q
)
175 return __update_mqd(mm
, mqd
, q
, MTYPE_CC
, 1);
178 static int destroy_mqd(struct mqd_manager
*mm
, void *mqd
,
179 enum kfd_preempt_type type
,
180 unsigned int timeout
, uint32_t pipe_id
,
183 return mm
->dev
->kfd2kgd
->hqd_destroy
184 (mm
->dev
->kgd
, type
, timeout
,
188 static void uninit_mqd(struct mqd_manager
*mm
, void *mqd
,
189 struct kfd_mem_obj
*mqd_mem_obj
)
192 kfd_gtt_sa_free(mm
->dev
, mqd_mem_obj
);
195 static bool is_occupied(struct mqd_manager
*mm
, void *mqd
,
196 uint64_t queue_address
, uint32_t pipe_id
,
199 return mm
->dev
->kfd2kgd
->hqd_is_occupied(
200 mm
->dev
->kgd
, queue_address
,
204 static int init_mqd_hiq(struct mqd_manager
*mm
, void **mqd
,
205 struct kfd_mem_obj
**mqd_mem_obj
, uint64_t *gart_addr
,
206 struct queue_properties
*q
)
209 int retval
= init_mqd(mm
, mqd
, mqd_mem_obj
, gart_addr
, q
);
216 m
->cp_hqd_pq_control
|= 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT
|
217 1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT
;
222 static int update_mqd_hiq(struct mqd_manager
*mm
, void *mqd
,
223 struct queue_properties
*q
)
226 int retval
= __update_mqd(mm
, mqd
, q
, MTYPE_UC
, 0);
232 m
->cp_hqd_vmid
= q
->vmid
;
236 struct mqd_manager
*mqd_manager_init_vi(enum KFD_MQD_TYPE type
,
239 struct mqd_manager
*mqd
;
242 BUG_ON(type
>= KFD_MQD_TYPE_MAX
);
244 pr_debug("kfd: In func %s\n", __func__
);
246 mqd
= kzalloc(sizeof(struct mqd_manager
), GFP_KERNEL
);
253 case KFD_MQD_TYPE_CP
:
254 case KFD_MQD_TYPE_COMPUTE
:
255 mqd
->init_mqd
= init_mqd
;
256 mqd
->uninit_mqd
= uninit_mqd
;
257 mqd
->load_mqd
= load_mqd
;
258 mqd
->update_mqd
= update_mqd
;
259 mqd
->destroy_mqd
= destroy_mqd
;
260 mqd
->is_occupied
= is_occupied
;
262 case KFD_MQD_TYPE_HIQ
:
263 mqd
->init_mqd
= init_mqd_hiq
;
264 mqd
->uninit_mqd
= uninit_mqd
;
265 mqd
->load_mqd
= load_mqd
;
266 mqd
->update_mqd
= update_mqd_hiq
;
267 mqd
->destroy_mqd
= destroy_mqd
;
268 mqd
->is_occupied
= is_occupied
;
270 case KFD_MQD_TYPE_SDMA
: