2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
24 #include <linux/slab.h>
25 #include <linux/list.h>
26 #include "kfd_device_queue_manager.h"
28 #include "kfd_kernel_queue.h"
30 static inline struct process_queue_node
*get_queue_by_qid(
31 struct process_queue_manager
*pqm
, unsigned int qid
)
33 struct process_queue_node
*pqn
;
35 list_for_each_entry(pqn
, &pqm
->queues
, process_queue_list
) {
36 if ((pqn
->q
&& pqn
->q
->properties
.queue_id
== qid
) ||
37 (pqn
->kq
&& pqn
->kq
->queue
->properties
.queue_id
== qid
))
44 static int find_available_queue_slot(struct process_queue_manager
*pqm
,
49 found
= find_first_zero_bit(pqm
->queue_slot_bitmap
,
50 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS
);
52 pr_debug("The new slot id %lu\n", found
);
54 if (found
>= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS
) {
55 pr_info("Cannot open more queues for process with pasid %d\n",
60 set_bit(found
, pqm
->queue_slot_bitmap
);
66 void kfd_process_dequeue_from_device(struct kfd_process_device
*pdd
)
68 struct kfd_dev
*dev
= pdd
->dev
;
70 if (pdd
->already_dequeued
)
73 dev
->dqm
->ops
.process_termination(dev
->dqm
, &pdd
->qpd
);
74 pdd
->already_dequeued
= true;
77 void kfd_process_dequeue_from_all_devices(struct kfd_process
*p
)
79 struct kfd_process_device
*pdd
;
81 list_for_each_entry(pdd
, &p
->per_device_data
, per_device_list
)
82 kfd_process_dequeue_from_device(pdd
);
85 int pqm_init(struct process_queue_manager
*pqm
, struct kfd_process
*p
)
87 INIT_LIST_HEAD(&pqm
->queues
);
88 pqm
->queue_slot_bitmap
=
89 kzalloc(DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS
,
90 BITS_PER_BYTE
), GFP_KERNEL
);
91 if (!pqm
->queue_slot_bitmap
)
98 void pqm_uninit(struct process_queue_manager
*pqm
)
100 struct process_queue_node
*pqn
, *next
;
102 list_for_each_entry_safe(pqn
, next
, &pqm
->queues
, process_queue_list
) {
103 uninit_queue(pqn
->q
);
104 list_del(&pqn
->process_queue_list
);
108 kfree(pqm
->queue_slot_bitmap
);
109 pqm
->queue_slot_bitmap
= NULL
;
112 static int create_cp_queue(struct process_queue_manager
*pqm
,
113 struct kfd_dev
*dev
, struct queue
**q
,
114 struct queue_properties
*q_properties
,
115 struct file
*f
, unsigned int qid
)
119 /* Doorbell initialized in user space*/
120 q_properties
->doorbell_ptr
= NULL
;
122 q_properties
->doorbell_off
=
123 kfd_queue_id_to_doorbell(dev
, pqm
->process
, qid
);
125 /* let DQM handle it*/
126 q_properties
->vmid
= 0;
127 q_properties
->queue_id
= qid
;
129 retval
= init_queue(q
, q_properties
);
134 (*q
)->process
= pqm
->process
;
136 pr_debug("PQM After init queue");
141 int pqm_create_queue(struct process_queue_manager
*pqm
,
144 struct queue_properties
*properties
,
148 struct kfd_process_device
*pdd
;
150 struct process_queue_node
*pqn
;
151 struct kernel_queue
*kq
;
152 enum kfd_queue_type type
= properties
->type
;
153 unsigned int max_queues
= 127; /* HWS limit */
158 pdd
= kfd_get_process_device_data(dev
, pqm
->process
);
160 pr_err("Process device data doesn't exist\n");
165 * for debug process, verify that it is within the static queues limit
166 * currently limit is set to half of the total avail HQD slots
167 * If we are just about to create DIQ, the is_debug flag is not set yet
168 * Hence we also check the type as well
170 if ((pdd
->qpd
.is_debug
) || (type
== KFD_QUEUE_TYPE_DIQ
))
171 max_queues
= dev
->device_info
->max_no_of_hqd
/2;
173 if (pdd
->qpd
.queue_count
>= max_queues
)
176 retval
= find_available_queue_slot(pqm
, qid
);
180 if (list_empty(&pdd
->qpd
.queues_list
) &&
181 list_empty(&pdd
->qpd
.priv_queue_list
)) {
183 dev
->dqm
->ops
.register_process(dev
->dqm
, &pdd
->qpd
);
186 pqn
= kzalloc(sizeof(*pqn
), GFP_KERNEL
);
189 goto err_allocate_pqn
;
193 case KFD_QUEUE_TYPE_SDMA
:
194 case KFD_QUEUE_TYPE_COMPUTE
:
195 /* check if there is over subscription */
196 if ((sched_policy
== KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION
) &&
197 ((dev
->dqm
->processes_count
>= dev
->vm_info
.vmid_num_kfd
) ||
198 (dev
->dqm
->queue_count
>= get_queues_num(dev
->dqm
)))) {
199 pr_err("Over-subscription is not allowed in radeon_kfd.sched_policy == 1\n");
201 goto err_create_queue
;
204 retval
= create_cp_queue(pqm
, dev
, &q
, properties
, f
, *qid
);
206 goto err_create_queue
;
209 retval
= dev
->dqm
->ops
.create_queue(dev
->dqm
, q
, &pdd
->qpd
,
210 &q
->properties
.vmid
);
211 pr_debug("DQM returned %d for create_queue\n", retval
);
214 case KFD_QUEUE_TYPE_DIQ
:
215 kq
= kernel_queue_init(dev
, KFD_QUEUE_TYPE_DIQ
);
218 goto err_create_queue
;
220 kq
->queue
->properties
.queue_id
= *qid
;
223 retval
= dev
->dqm
->ops
.create_kernel_queue(dev
->dqm
,
227 WARN(1, "Invalid queue type %d", type
);
232 pr_err("DQM create queue failed\n");
233 goto err_create_queue
;
236 pr_debug("PQM After DQM create queue\n");
238 list_add(&pqn
->process_queue_list
, &pqm
->queues
);
241 pr_debug("PQM done creating queue\n");
242 print_queue_properties(&q
->properties
);
250 /* check if queues list is empty unregister process from device */
251 clear_bit(*qid
, pqm
->queue_slot_bitmap
);
252 if (list_empty(&pdd
->qpd
.queues_list
) &&
253 list_empty(&pdd
->qpd
.priv_queue_list
))
254 dev
->dqm
->ops
.unregister_process(dev
->dqm
, &pdd
->qpd
);
258 int pqm_destroy_queue(struct process_queue_manager
*pqm
, unsigned int qid
)
260 struct process_queue_node
*pqn
;
261 struct kfd_process_device
*pdd
;
262 struct device_queue_manager
*dqm
;
270 pqn
= get_queue_by_qid(pqm
, qid
);
272 pr_err("Queue id does not match any known queue\n");
280 dev
= pqn
->q
->device
;
284 pdd
= kfd_get_process_device_data(dev
, pqm
->process
);
286 pr_err("Process device data doesn't exist\n");
291 /* destroy kernel queue (DIQ) */
292 dqm
= pqn
->kq
->dev
->dqm
;
293 dqm
->ops
.destroy_kernel_queue(dqm
, pqn
->kq
, &pdd
->qpd
);
294 kernel_queue_uninit(pqn
->kq
);
298 dqm
= pqn
->q
->device
->dqm
;
299 retval
= dqm
->ops
.destroy_queue(dqm
, &pdd
->qpd
, pqn
->q
);
300 uninit_queue(pqn
->q
);
303 list_del(&pqn
->process_queue_list
);
305 clear_bit(qid
, pqm
->queue_slot_bitmap
);
307 if (list_empty(&pdd
->qpd
.queues_list
) &&
308 list_empty(&pdd
->qpd
.priv_queue_list
))
309 dqm
->ops
.unregister_process(dqm
, &pdd
->qpd
);
314 int pqm_update_queue(struct process_queue_manager
*pqm
, unsigned int qid
,
315 struct queue_properties
*p
)
318 struct process_queue_node
*pqn
;
320 pqn
= get_queue_by_qid(pqm
, qid
);
322 pr_debug("No queue %d exists for update operation\n", qid
);
326 pqn
->q
->properties
.queue_address
= p
->queue_address
;
327 pqn
->q
->properties
.queue_size
= p
->queue_size
;
328 pqn
->q
->properties
.queue_percent
= p
->queue_percent
;
329 pqn
->q
->properties
.priority
= p
->priority
;
331 retval
= pqn
->q
->device
->dqm
->ops
.update_queue(pqn
->q
->device
->dqm
,
339 struct kernel_queue
*pqm_get_kernel_queue(
340 struct process_queue_manager
*pqm
,
343 struct process_queue_node
*pqn
;
345 pqn
= get_queue_by_qid(pqm
, qid
);