2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
24 #include <linux/types.h>
25 #include <linux/mutex.h>
26 #include <linux/slab.h>
27 #include <linux/printk.h>
28 #include <linux/sched.h>
29 #include "kfd_kernel_queue.h"
31 #include "kfd_device_queue_manager.h"
32 #include "kfd_pm4_headers.h"
33 #include "kfd_pm4_opcodes.h"
35 #define PM4_COUNT_ZERO (((1 << 15) - 1) << 16)
37 static bool initialize(struct kernel_queue
*kq
, struct kfd_dev
*dev
,
38 enum kfd_queue_type type
, unsigned int queue_size
)
40 struct queue_properties prop
;
42 union PM4_MES_TYPE_3_HEADER nop
;
44 if (WARN_ON(type
!= KFD_QUEUE_TYPE_DIQ
&& type
!= KFD_QUEUE_TYPE_HIQ
))
47 pr_debug("Initializing queue type %d size %d\n", KFD_QUEUE_TYPE_HIQ
,
50 memset(&prop
, 0, sizeof(prop
));
51 memset(&nop
, 0, sizeof(nop
));
54 nop
.type
= PM4_TYPE_3
;
55 nop
.u32all
|= PM4_COUNT_ZERO
;
58 kq
->nop_packet
= nop
.u32all
;
60 case KFD_QUEUE_TYPE_DIQ
:
61 case KFD_QUEUE_TYPE_HIQ
:
62 kq
->mqd
= dev
->dqm
->ops
.get_mqd_manager(dev
->dqm
,
66 pr_err("Invalid queue type %d\n", type
);
73 prop
.doorbell_ptr
= kfd_get_kernel_doorbell(dev
, &prop
.doorbell_off
);
75 if (!prop
.doorbell_ptr
) {
76 pr_err("Failed to initialize doorbell");
77 goto err_get_kernel_doorbell
;
80 retval
= kfd_gtt_sa_allocate(dev
, queue_size
, &kq
->pq
);
82 pr_err("Failed to init pq queues size %d\n", queue_size
);
83 goto err_pq_allocate_vidmem
;
86 kq
->pq_kernel_addr
= kq
->pq
->cpu_ptr
;
87 kq
->pq_gpu_addr
= kq
->pq
->gpu_addr
;
89 retval
= kq
->ops_asic_specific
.initialize(kq
, dev
, type
, queue_size
);
91 goto err_eop_allocate_vidmem
;
93 retval
= kfd_gtt_sa_allocate(dev
, sizeof(*kq
->rptr_kernel
),
97 goto err_rptr_allocate_vidmem
;
99 kq
->rptr_kernel
= kq
->rptr_mem
->cpu_ptr
;
100 kq
->rptr_gpu_addr
= kq
->rptr_mem
->gpu_addr
;
102 retval
= kfd_gtt_sa_allocate(dev
, sizeof(*kq
->wptr_kernel
),
106 goto err_wptr_allocate_vidmem
;
108 kq
->wptr_kernel
= kq
->wptr_mem
->cpu_ptr
;
109 kq
->wptr_gpu_addr
= kq
->wptr_mem
->gpu_addr
;
111 memset(kq
->pq_kernel_addr
, 0, queue_size
);
112 memset(kq
->rptr_kernel
, 0, sizeof(*kq
->rptr_kernel
));
113 memset(kq
->wptr_kernel
, 0, sizeof(*kq
->wptr_kernel
));
115 prop
.queue_size
= queue_size
;
116 prop
.is_interop
= false;
118 prop
.queue_percent
= 100;
121 prop
.queue_address
= kq
->pq_gpu_addr
;
122 prop
.read_ptr
= (uint32_t *) kq
->rptr_gpu_addr
;
123 prop
.write_ptr
= (uint32_t *) kq
->wptr_gpu_addr
;
124 prop
.eop_ring_buffer_address
= kq
->eop_gpu_addr
;
125 prop
.eop_ring_buffer_size
= PAGE_SIZE
;
127 if (init_queue(&kq
->queue
, &prop
) != 0)
130 kq
->queue
->device
= dev
;
131 kq
->queue
->process
= kfd_get_process(current
);
133 retval
= kq
->mqd
->init_mqd(kq
->mqd
, &kq
->queue
->mqd
,
134 &kq
->queue
->mqd_mem_obj
,
135 &kq
->queue
->gart_mqd_addr
,
136 &kq
->queue
->properties
);
140 /* assign HIQ to HQD */
141 if (type
== KFD_QUEUE_TYPE_HIQ
) {
142 pr_debug("Assigning hiq to hqd\n");
143 kq
->queue
->pipe
= KFD_CIK_HIQ_PIPE
;
144 kq
->queue
->queue
= KFD_CIK_HIQ_QUEUE
;
145 kq
->mqd
->load_mqd(kq
->mqd
, kq
->queue
->mqd
, kq
->queue
->pipe
,
146 kq
->queue
->queue
, &kq
->queue
->properties
,
149 /* allocate fence for DIQ */
151 retval
= kfd_gtt_sa_allocate(dev
, sizeof(uint32_t),
155 goto err_alloc_fence
;
157 kq
->fence_kernel_address
= kq
->fence_mem_obj
->cpu_ptr
;
158 kq
->fence_gpu_addr
= kq
->fence_mem_obj
->gpu_addr
;
161 print_queue(kq
->queue
);
166 uninit_queue(kq
->queue
);
168 kfd_gtt_sa_free(dev
, kq
->wptr_mem
);
169 err_wptr_allocate_vidmem
:
170 kfd_gtt_sa_free(dev
, kq
->rptr_mem
);
171 err_rptr_allocate_vidmem
:
172 kfd_gtt_sa_free(dev
, kq
->eop_mem
);
173 err_eop_allocate_vidmem
:
174 kfd_gtt_sa_free(dev
, kq
->pq
);
175 err_pq_allocate_vidmem
:
176 kfd_release_kernel_doorbell(dev
, prop
.doorbell_ptr
);
177 err_get_kernel_doorbell
:
182 static void uninitialize(struct kernel_queue
*kq
)
184 if (kq
->queue
->properties
.type
== KFD_QUEUE_TYPE_HIQ
)
185 kq
->mqd
->destroy_mqd(kq
->mqd
,
187 KFD_PREEMPT_TYPE_WAVEFRONT_RESET
,
188 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS
,
191 else if (kq
->queue
->properties
.type
== KFD_QUEUE_TYPE_DIQ
)
192 kfd_gtt_sa_free(kq
->dev
, kq
->fence_mem_obj
);
194 kq
->mqd
->uninit_mqd(kq
->mqd
, kq
->queue
->mqd
, kq
->queue
->mqd_mem_obj
);
196 kfd_gtt_sa_free(kq
->dev
, kq
->rptr_mem
);
197 kfd_gtt_sa_free(kq
->dev
, kq
->wptr_mem
);
198 kq
->ops_asic_specific
.uninitialize(kq
);
199 kfd_gtt_sa_free(kq
->dev
, kq
->pq
);
200 kfd_release_kernel_doorbell(kq
->dev
,
201 kq
->queue
->properties
.doorbell_ptr
);
202 uninit_queue(kq
->queue
);
205 static int acquire_packet_buffer(struct kernel_queue
*kq
,
206 size_t packet_size_in_dwords
, unsigned int **buffer_ptr
)
208 size_t available_size
;
209 size_t queue_size_dwords
;
211 unsigned int *queue_address
;
213 /* When rptr == wptr, the buffer is empty.
214 * When rptr == wptr + 1, the buffer is full.
215 * It is always rptr that advances to the position of wptr, rather than
216 * the opposite. So we can only use up to queue_size_dwords - 1 dwords.
218 rptr
= *kq
->rptr_kernel
;
219 wptr
= *kq
->wptr_kernel
;
220 queue_address
= (unsigned int *)kq
->pq_kernel_addr
;
221 queue_size_dwords
= kq
->queue
->properties
.queue_size
/ sizeof(uint32_t);
223 pr_debug("rptr: %d\n", rptr
);
224 pr_debug("wptr: %d\n", wptr
);
225 pr_debug("queue_address 0x%p\n", queue_address
);
227 available_size
= (rptr
+ queue_size_dwords
- 1 - wptr
) %
230 if (packet_size_in_dwords
> available_size
) {
232 * make sure calling functions know
233 * acquire_packet_buffer() failed
239 if (wptr
+ packet_size_in_dwords
>= queue_size_dwords
) {
240 /* make sure after rolling back to position 0, there is
241 * still enough space.
243 if (packet_size_in_dwords
>= rptr
) {
247 /* fill nops, roll back and start at position 0 */
249 queue_address
[wptr
] = kq
->nop_packet
;
250 wptr
= (wptr
+ 1) % queue_size_dwords
;
254 *buffer_ptr
= &queue_address
[wptr
];
255 kq
->pending_wptr
= wptr
+ packet_size_in_dwords
;
260 static void submit_packet(struct kernel_queue
*kq
)
265 for (i
= *kq
->wptr_kernel
; i
< kq
->pending_wptr
; i
++) {
266 pr_debug("0x%2X ", kq
->pq_kernel_addr
[i
]);
273 *kq
->wptr_kernel
= kq
->pending_wptr
;
274 write_kernel_doorbell(kq
->queue
->properties
.doorbell_ptr
,
278 static void rollback_packet(struct kernel_queue
*kq
)
280 kq
->pending_wptr
= *kq
->queue
->properties
.write_ptr
;
283 struct kernel_queue
*kernel_queue_init(struct kfd_dev
*dev
,
284 enum kfd_queue_type type
)
286 struct kernel_queue
*kq
;
288 kq
= kzalloc(sizeof(*kq
), GFP_KERNEL
);
292 kq
->ops
.initialize
= initialize
;
293 kq
->ops
.uninitialize
= uninitialize
;
294 kq
->ops
.acquire_packet_buffer
= acquire_packet_buffer
;
295 kq
->ops
.submit_packet
= submit_packet
;
296 kq
->ops
.rollback_packet
= rollback_packet
;
298 switch (dev
->device_info
->asic_family
) {
300 kernel_queue_init_vi(&kq
->ops_asic_specific
);
304 kernel_queue_init_cik(&kq
->ops_asic_specific
);
308 if (!kq
->ops
.initialize(kq
, dev
, type
, KFD_KERNEL_QUEUE_SIZE
)) {
309 pr_err("Failed to init kernel queue\n");
316 void kernel_queue_uninit(struct kernel_queue
*kq
)
318 kq
->ops
.uninitialize(kq
);
322 /* FIXME: Can this test be removed? */
323 static __attribute__((unused
)) void test_kq(struct kfd_dev
*dev
)
325 struct kernel_queue
*kq
;
329 pr_err("Starting kernel queue test\n");
331 kq
= kernel_queue_init(dev
, KFD_QUEUE_TYPE_HIQ
);
333 pr_err(" Failed to initialize HIQ\n");
334 pr_err("Kernel queue test failed\n");
338 retval
= kq
->ops
.acquire_packet_buffer(kq
, 5, &buffer
);
339 if (unlikely(retval
!= 0)) {
340 pr_err(" Failed to acquire packet buffer\n");
341 pr_err("Kernel queue test failed\n");
344 for (i
= 0; i
< 5; i
++)
345 buffer
[i
] = kq
->nop_packet
;
346 kq
->ops
.submit_packet(kq
);
348 pr_err("Ending kernel queue test\n");