2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
24 #include <linux/slab.h>
25 #include <linux/list.h>
26 #include "kfd_device_queue_manager.h"
28 #include "kfd_kernel_queue.h"
30 static inline struct process_queue_node
*get_queue_by_qid(
31 struct process_queue_manager
*pqm
, unsigned int qid
)
33 struct process_queue_node
*pqn
;
35 list_for_each_entry(pqn
, &pqm
->queues
, process_queue_list
) {
36 if ((pqn
->q
&& pqn
->q
->properties
.queue_id
== qid
) ||
37 (pqn
->kq
&& pqn
->kq
->queue
->properties
.queue_id
== qid
))
44 static int find_available_queue_slot(struct process_queue_manager
*pqm
,
49 found
= find_first_zero_bit(pqm
->queue_slot_bitmap
,
50 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS
);
52 pr_debug("The new slot id %lu\n", found
);
54 if (found
>= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS
) {
55 pr_info("Cannot open more queues for process with pasid %d\n",
60 set_bit(found
, pqm
->queue_slot_bitmap
);
66 int pqm_init(struct process_queue_manager
*pqm
, struct kfd_process
*p
)
68 INIT_LIST_HEAD(&pqm
->queues
);
69 pqm
->queue_slot_bitmap
=
70 kzalloc(DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS
,
71 BITS_PER_BYTE
), GFP_KERNEL
);
72 if (!pqm
->queue_slot_bitmap
)
79 void pqm_uninit(struct process_queue_manager
*pqm
)
82 struct process_queue_node
*pqn
, *next
;
84 list_for_each_entry_safe(pqn
, next
, &pqm
->queues
, process_queue_list
) {
85 retval
= pqm_destroy_queue(
88 pqn
->q
->properties
.queue_id
:
89 pqn
->kq
->queue
->properties
.queue_id
);
92 pr_err("failed to destroy queue\n");
96 kfree(pqm
->queue_slot_bitmap
);
97 pqm
->queue_slot_bitmap
= NULL
;
100 static int create_cp_queue(struct process_queue_manager
*pqm
,
101 struct kfd_dev
*dev
, struct queue
**q
,
102 struct queue_properties
*q_properties
,
103 struct file
*f
, unsigned int qid
)
107 /* Doorbell initialized in user space*/
108 q_properties
->doorbell_ptr
= NULL
;
110 q_properties
->doorbell_off
=
111 kfd_queue_id_to_doorbell(dev
, pqm
->process
, qid
);
113 /* let DQM handle it*/
114 q_properties
->vmid
= 0;
115 q_properties
->queue_id
= qid
;
117 retval
= init_queue(q
, q_properties
);
122 (*q
)->process
= pqm
->process
;
124 pr_debug("PQM After init queue");
129 int pqm_create_queue(struct process_queue_manager
*pqm
,
132 struct queue_properties
*properties
,
134 enum kfd_queue_type type
,
138 struct kfd_process_device
*pdd
;
139 struct queue_properties q_properties
;
141 struct process_queue_node
*pqn
;
142 struct kernel_queue
*kq
;
146 memset(&q_properties
, 0, sizeof(struct queue_properties
));
147 memcpy(&q_properties
, properties
, sizeof(struct queue_properties
));
151 pdd
= kfd_get_process_device_data(dev
, pqm
->process
);
153 pr_err("Process device data doesn't exist\n");
158 * for debug process, verify that it is within the static queues limit
159 * currently limit is set to half of the total avail HQD slots
160 * If we are just about to create DIQ, the is_debug flag is not set yet
161 * Hence we also check the type as well
163 if ((pdd
->qpd
.is_debug
) ||
164 (type
== KFD_QUEUE_TYPE_DIQ
)) {
165 list_for_each_entry(cur
, &pdd
->qpd
.queues_list
, list
)
167 if (num_queues
>= dev
->device_info
->max_no_of_hqd
/2)
171 retval
= find_available_queue_slot(pqm
, qid
);
175 if (list_empty(&pqm
->queues
)) {
177 dev
->dqm
->ops
.register_process(dev
->dqm
, &pdd
->qpd
);
180 pqn
= kzalloc(sizeof(*pqn
), GFP_KERNEL
);
183 goto err_allocate_pqn
;
187 case KFD_QUEUE_TYPE_SDMA
:
188 case KFD_QUEUE_TYPE_COMPUTE
:
189 /* check if there is over subscription */
190 if ((sched_policy
== KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION
) &&
191 ((dev
->dqm
->processes_count
>= VMID_PER_DEVICE
) ||
192 (dev
->dqm
->queue_count
>= get_queues_num(dev
->dqm
)))) {
193 pr_err("Over-subscription is not allowed in radeon_kfd.sched_policy == 1\n");
195 goto err_create_queue
;
198 retval
= create_cp_queue(pqm
, dev
, &q
, &q_properties
, f
, *qid
);
200 goto err_create_queue
;
203 retval
= dev
->dqm
->ops
.create_queue(dev
->dqm
, q
, &pdd
->qpd
,
204 &q
->properties
.vmid
);
205 pr_debug("DQM returned %d for create_queue\n", retval
);
208 case KFD_QUEUE_TYPE_DIQ
:
209 kq
= kernel_queue_init(dev
, KFD_QUEUE_TYPE_DIQ
);
212 goto err_create_queue
;
214 kq
->queue
->properties
.queue_id
= *qid
;
217 retval
= dev
->dqm
->ops
.create_kernel_queue(dev
->dqm
,
221 WARN(1, "Invalid queue type %d", type
);
226 pr_err("DQM create queue failed\n");
227 goto err_create_queue
;
230 pr_debug("PQM After DQM create queue\n");
232 list_add(&pqn
->process_queue_list
, &pqm
->queues
);
235 *properties
= q
->properties
;
236 pr_debug("PQM done creating queue\n");
237 print_queue_properties(properties
);
245 /* check if queues list is empty unregister process from device */
246 clear_bit(*qid
, pqm
->queue_slot_bitmap
);
247 if (list_empty(&pqm
->queues
))
248 dev
->dqm
->ops
.unregister_process(dev
->dqm
, &pdd
->qpd
);
252 int pqm_destroy_queue(struct process_queue_manager
*pqm
, unsigned int qid
)
254 struct process_queue_node
*pqn
;
255 struct kfd_process_device
*pdd
;
256 struct device_queue_manager
*dqm
;
264 pqn
= get_queue_by_qid(pqm
, qid
);
266 pr_err("Queue id does not match any known queue\n");
274 dev
= pqn
->q
->device
;
278 pdd
= kfd_get_process_device_data(dev
, pqm
->process
);
280 pr_err("Process device data doesn't exist\n");
285 /* destroy kernel queue (DIQ) */
286 dqm
= pqn
->kq
->dev
->dqm
;
287 dqm
->ops
.destroy_kernel_queue(dqm
, pqn
->kq
, &pdd
->qpd
);
288 kernel_queue_uninit(pqn
->kq
);
292 dqm
= pqn
->q
->device
->dqm
;
293 retval
= dqm
->ops
.destroy_queue(dqm
, &pdd
->qpd
, pqn
->q
);
297 uninit_queue(pqn
->q
);
300 list_del(&pqn
->process_queue_list
);
302 clear_bit(qid
, pqm
->queue_slot_bitmap
);
304 if (list_empty(&pqm
->queues
))
305 dqm
->ops
.unregister_process(dqm
, &pdd
->qpd
);
310 int pqm_update_queue(struct process_queue_manager
*pqm
, unsigned int qid
,
311 struct queue_properties
*p
)
314 struct process_queue_node
*pqn
;
316 pqn
= get_queue_by_qid(pqm
, qid
);
318 pr_debug("No queue %d exists for update operation\n", qid
);
322 pqn
->q
->properties
.queue_address
= p
->queue_address
;
323 pqn
->q
->properties
.queue_size
= p
->queue_size
;
324 pqn
->q
->properties
.queue_percent
= p
->queue_percent
;
325 pqn
->q
->properties
.priority
= p
->priority
;
327 retval
= pqn
->q
->device
->dqm
->ops
.update_queue(pqn
->q
->device
->dqm
,
335 struct kernel_queue
*pqm_get_kernel_queue(
336 struct process_queue_manager
*pqm
,
339 struct process_queue_node
*pqn
;
341 pqn
= get_queue_by_qid(pqm
, qid
);