2 * Copyright 2015 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
24 #include <linux/kthread.h>
25 #include <linux/wait.h>
26 #include <linux/sched.h>
27 #include <uapi/linux/sched/types.h>
29 #include "gpu_scheduler.h"
31 #define CREATE_TRACE_POINTS
32 #include "gpu_sched_trace.h"
34 static bool amd_sched_entity_is_ready(struct amd_sched_entity
*entity
);
35 static void amd_sched_wakeup(struct amd_gpu_scheduler
*sched
);
36 static void amd_sched_process_job(struct dma_fence
*f
, struct dma_fence_cb
*cb
);
38 /* Initialize a given run queue struct */
39 static void amd_sched_rq_init(struct amd_sched_rq
*rq
)
41 spin_lock_init(&rq
->lock
);
42 INIT_LIST_HEAD(&rq
->entities
);
43 rq
->current_entity
= NULL
;
46 static void amd_sched_rq_add_entity(struct amd_sched_rq
*rq
,
47 struct amd_sched_entity
*entity
)
49 if (!list_empty(&entity
->list
))
52 list_add_tail(&entity
->list
, &rq
->entities
);
53 spin_unlock(&rq
->lock
);
56 static void amd_sched_rq_remove_entity(struct amd_sched_rq
*rq
,
57 struct amd_sched_entity
*entity
)
59 if (list_empty(&entity
->list
))
62 list_del_init(&entity
->list
);
63 if (rq
->current_entity
== entity
)
64 rq
->current_entity
= NULL
;
65 spin_unlock(&rq
->lock
);
69 * Select an entity which could provide a job to run
71 * @rq The run queue to check.
73 * Try to find a ready entity, returns NULL if none found.
75 static struct amd_sched_entity
*
76 amd_sched_rq_select_entity(struct amd_sched_rq
*rq
)
78 struct amd_sched_entity
*entity
;
82 entity
= rq
->current_entity
;
84 list_for_each_entry_continue(entity
, &rq
->entities
, list
) {
85 if (amd_sched_entity_is_ready(entity
)) {
86 rq
->current_entity
= entity
;
87 spin_unlock(&rq
->lock
);
93 list_for_each_entry(entity
, &rq
->entities
, list
) {
95 if (amd_sched_entity_is_ready(entity
)) {
96 rq
->current_entity
= entity
;
97 spin_unlock(&rq
->lock
);
101 if (entity
== rq
->current_entity
)
105 spin_unlock(&rq
->lock
);
111 * Init a context entity used by scheduler when submit to HW ring.
113 * @sched The pointer to the scheduler
114 * @entity The pointer to a valid amd_sched_entity
115 * @rq The run queue this entity belongs
116 * @kernel If this is an entity for the kernel
117 * @jobs The max number of jobs in the job queue
119 * return 0 if succeed. negative error code on failure
121 int amd_sched_entity_init(struct amd_gpu_scheduler
*sched
,
122 struct amd_sched_entity
*entity
,
123 struct amd_sched_rq
*rq
,
128 if (!(sched
&& entity
&& rq
))
131 memset(entity
, 0, sizeof(struct amd_sched_entity
));
132 INIT_LIST_HEAD(&entity
->list
);
134 entity
->sched
= sched
;
136 spin_lock_init(&entity
->queue_lock
);
137 r
= kfifo_alloc(&entity
->job_queue
, jobs
* sizeof(void *), GFP_KERNEL
);
141 atomic_set(&entity
->fence_seq
, 0);
142 entity
->fence_context
= dma_fence_context_alloc(2);
148 * Query if entity is initialized
150 * @sched Pointer to scheduler instance
151 * @entity The pointer to a valid scheduler entity
153 * return true if entity is initialized, false otherwise
155 static bool amd_sched_entity_is_initialized(struct amd_gpu_scheduler
*sched
,
156 struct amd_sched_entity
*entity
)
158 return entity
->sched
== sched
&&
163 * Check if entity is idle
165 * @entity The pointer to a valid scheduler entity
167 * Return true if entity don't has any unscheduled jobs.
169 static bool amd_sched_entity_is_idle(struct amd_sched_entity
*entity
)
172 if (kfifo_is_empty(&entity
->job_queue
))
179 * Check if entity is ready
181 * @entity The pointer to a valid scheduler entity
183 * Return true if entity could provide a job.
185 static bool amd_sched_entity_is_ready(struct amd_sched_entity
*entity
)
187 if (kfifo_is_empty(&entity
->job_queue
))
190 if (ACCESS_ONCE(entity
->dependency
))
197 * Destroy a context entity
199 * @sched Pointer to scheduler instance
200 * @entity The pointer to a valid scheduler entity
202 * Cleanup and free the allocated resources.
204 void amd_sched_entity_fini(struct amd_gpu_scheduler
*sched
,
205 struct amd_sched_entity
*entity
)
207 struct amd_sched_rq
*rq
= entity
->rq
;
209 if (!amd_sched_entity_is_initialized(sched
, entity
))
213 * The client will not queue more IBs during this fini, consume existing
216 wait_event(sched
->job_scheduled
, amd_sched_entity_is_idle(entity
));
218 amd_sched_rq_remove_entity(rq
, entity
);
219 kfifo_free(&entity
->job_queue
);
222 static void amd_sched_entity_wakeup(struct dma_fence
*f
, struct dma_fence_cb
*cb
)
224 struct amd_sched_entity
*entity
=
225 container_of(cb
, struct amd_sched_entity
, cb
);
226 entity
->dependency
= NULL
;
228 amd_sched_wakeup(entity
->sched
);
231 static void amd_sched_entity_clear_dep(struct dma_fence
*f
, struct dma_fence_cb
*cb
)
233 struct amd_sched_entity
*entity
=
234 container_of(cb
, struct amd_sched_entity
, cb
);
235 entity
->dependency
= NULL
;
239 static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity
*entity
)
241 struct amd_gpu_scheduler
*sched
= entity
->sched
;
242 struct dma_fence
* fence
= entity
->dependency
;
243 struct amd_sched_fence
*s_fence
;
245 if (fence
->context
== entity
->fence_context
) {
246 /* We can ignore fences from ourself */
247 dma_fence_put(entity
->dependency
);
251 s_fence
= to_amd_sched_fence(fence
);
252 if (s_fence
&& s_fence
->sched
== sched
) {
255 * Fence is from the same scheduler, only need to wait for
258 fence
= dma_fence_get(&s_fence
->scheduled
);
259 dma_fence_put(entity
->dependency
);
260 entity
->dependency
= fence
;
261 if (!dma_fence_add_callback(fence
, &entity
->cb
,
262 amd_sched_entity_clear_dep
))
265 /* Ignore it when it is already scheduled */
266 dma_fence_put(fence
);
270 if (!dma_fence_add_callback(entity
->dependency
, &entity
->cb
,
271 amd_sched_entity_wakeup
))
274 dma_fence_put(entity
->dependency
);
278 static struct amd_sched_job
*
279 amd_sched_entity_pop_job(struct amd_sched_entity
*entity
)
281 struct amd_gpu_scheduler
*sched
= entity
->sched
;
282 struct amd_sched_job
*sched_job
;
284 if (!kfifo_out_peek(&entity
->job_queue
, &sched_job
, sizeof(sched_job
)))
287 while ((entity
->dependency
= sched
->ops
->dependency(sched_job
)))
288 if (amd_sched_entity_add_dependency_cb(entity
))
295 * Helper to submit a job to the job queue
297 * @sched_job The pointer to job required to submit
299 * Returns true if we could submit the job.
301 static bool amd_sched_entity_in(struct amd_sched_job
*sched_job
)
303 struct amd_gpu_scheduler
*sched
= sched_job
->sched
;
304 struct amd_sched_entity
*entity
= sched_job
->s_entity
;
305 bool added
, first
= false;
307 spin_lock(&entity
->queue_lock
);
308 added
= kfifo_in(&entity
->job_queue
, &sched_job
,
309 sizeof(sched_job
)) == sizeof(sched_job
);
311 if (added
&& kfifo_len(&entity
->job_queue
) == sizeof(sched_job
))
314 spin_unlock(&entity
->queue_lock
);
316 /* first job wakes up scheduler */
318 /* Add the entity to the run queue */
319 amd_sched_rq_add_entity(entity
->rq
, entity
);
320 amd_sched_wakeup(sched
);
325 /* job_finish is called after hw fence signaled, and
326 * the job had already been deleted from ring_mirror_list
328 static void amd_sched_job_finish(struct work_struct
*work
)
330 struct amd_sched_job
*s_job
= container_of(work
, struct amd_sched_job
,
332 struct amd_gpu_scheduler
*sched
= s_job
->sched
;
334 /* remove job from ring_mirror_list */
335 spin_lock(&sched
->job_list_lock
);
336 list_del_init(&s_job
->node
);
337 if (sched
->timeout
!= MAX_SCHEDULE_TIMEOUT
) {
338 struct amd_sched_job
*next
;
340 spin_unlock(&sched
->job_list_lock
);
341 cancel_delayed_work_sync(&s_job
->work_tdr
);
342 spin_lock(&sched
->job_list_lock
);
344 /* queue TDR for next job */
345 next
= list_first_entry_or_null(&sched
->ring_mirror_list
,
346 struct amd_sched_job
, node
);
349 schedule_delayed_work(&next
->work_tdr
, sched
->timeout
);
351 spin_unlock(&sched
->job_list_lock
);
352 sched
->ops
->free_job(s_job
);
355 static void amd_sched_job_finish_cb(struct dma_fence
*f
,
356 struct dma_fence_cb
*cb
)
358 struct amd_sched_job
*job
= container_of(cb
, struct amd_sched_job
,
360 schedule_work(&job
->finish_work
);
363 static void amd_sched_job_begin(struct amd_sched_job
*s_job
)
365 struct amd_gpu_scheduler
*sched
= s_job
->sched
;
367 spin_lock(&sched
->job_list_lock
);
368 list_add_tail(&s_job
->node
, &sched
->ring_mirror_list
);
369 if (sched
->timeout
!= MAX_SCHEDULE_TIMEOUT
&&
370 list_first_entry_or_null(&sched
->ring_mirror_list
,
371 struct amd_sched_job
, node
) == s_job
)
372 schedule_delayed_work(&s_job
->work_tdr
, sched
->timeout
);
373 spin_unlock(&sched
->job_list_lock
);
376 static void amd_sched_job_timedout(struct work_struct
*work
)
378 struct amd_sched_job
*job
= container_of(work
, struct amd_sched_job
,
381 job
->sched
->ops
->timedout_job(job
);
384 void amd_sched_hw_job_reset(struct amd_gpu_scheduler
*sched
)
386 struct amd_sched_job
*s_job
;
388 spin_lock(&sched
->job_list_lock
);
389 list_for_each_entry_reverse(s_job
, &sched
->ring_mirror_list
, node
) {
390 if (dma_fence_remove_callback(s_job
->s_fence
->parent
, &s_job
->s_fence
->cb
)) {
391 dma_fence_put(s_job
->s_fence
->parent
);
392 s_job
->s_fence
->parent
= NULL
;
395 atomic_set(&sched
->hw_rq_count
, 0);
396 spin_unlock(&sched
->job_list_lock
);
399 void amd_sched_job_recovery(struct amd_gpu_scheduler
*sched
)
401 struct amd_sched_job
*s_job
, *tmp
;
404 spin_lock(&sched
->job_list_lock
);
405 s_job
= list_first_entry_or_null(&sched
->ring_mirror_list
,
406 struct amd_sched_job
, node
);
407 if (s_job
&& sched
->timeout
!= MAX_SCHEDULE_TIMEOUT
)
408 schedule_delayed_work(&s_job
->work_tdr
, sched
->timeout
);
410 list_for_each_entry_safe(s_job
, tmp
, &sched
->ring_mirror_list
, node
) {
411 struct amd_sched_fence
*s_fence
= s_job
->s_fence
;
412 struct dma_fence
*fence
;
414 spin_unlock(&sched
->job_list_lock
);
415 fence
= sched
->ops
->run_job(s_job
);
416 atomic_inc(&sched
->hw_rq_count
);
418 s_fence
->parent
= dma_fence_get(fence
);
419 r
= dma_fence_add_callback(fence
, &s_fence
->cb
,
420 amd_sched_process_job
);
422 amd_sched_process_job(fence
, &s_fence
->cb
);
424 DRM_ERROR("fence add callback failed (%d)\n",
426 dma_fence_put(fence
);
428 DRM_ERROR("Failed to run job!\n");
429 amd_sched_process_job(NULL
, &s_fence
->cb
);
431 spin_lock(&sched
->job_list_lock
);
433 spin_unlock(&sched
->job_list_lock
);
437 * Submit a job to the job queue
439 * @sched_job The pointer to job required to submit
441 * Returns 0 for success, negative error code otherwise.
443 void amd_sched_entity_push_job(struct amd_sched_job
*sched_job
)
445 struct amd_sched_entity
*entity
= sched_job
->s_entity
;
447 trace_amd_sched_job(sched_job
);
448 dma_fence_add_callback(&sched_job
->s_fence
->finished
, &sched_job
->finish_cb
,
449 amd_sched_job_finish_cb
);
450 wait_event(entity
->sched
->job_scheduled
,
451 amd_sched_entity_in(sched_job
));
454 /* init a sched_job with basic field */
455 int amd_sched_job_init(struct amd_sched_job
*job
,
456 struct amd_gpu_scheduler
*sched
,
457 struct amd_sched_entity
*entity
,
461 job
->s_entity
= entity
;
462 job
->s_fence
= amd_sched_fence_create(entity
, owner
);
466 INIT_WORK(&job
->finish_work
, amd_sched_job_finish
);
467 INIT_LIST_HEAD(&job
->node
);
468 INIT_DELAYED_WORK(&job
->work_tdr
, amd_sched_job_timedout
);
474 * Return ture if we can push more jobs to the hw.
476 static bool amd_sched_ready(struct amd_gpu_scheduler
*sched
)
478 return atomic_read(&sched
->hw_rq_count
) <
479 sched
->hw_submission_limit
;
483 * Wake up the scheduler when it is ready
485 static void amd_sched_wakeup(struct amd_gpu_scheduler
*sched
)
487 if (amd_sched_ready(sched
))
488 wake_up_interruptible(&sched
->wake_up_worker
);
492 * Select next entity to process
494 static struct amd_sched_entity
*
495 amd_sched_select_entity(struct amd_gpu_scheduler
*sched
)
497 struct amd_sched_entity
*entity
;
500 if (!amd_sched_ready(sched
))
503 /* Kernel run queue has higher priority than normal run queue*/
504 for (i
= 0; i
< AMD_SCHED_MAX_PRIORITY
; i
++) {
505 entity
= amd_sched_rq_select_entity(&sched
->sched_rq
[i
]);
513 static void amd_sched_process_job(struct dma_fence
*f
, struct dma_fence_cb
*cb
)
515 struct amd_sched_fence
*s_fence
=
516 container_of(cb
, struct amd_sched_fence
, cb
);
517 struct amd_gpu_scheduler
*sched
= s_fence
->sched
;
519 atomic_dec(&sched
->hw_rq_count
);
520 amd_sched_fence_finished(s_fence
);
522 trace_amd_sched_process_job(s_fence
);
523 dma_fence_put(&s_fence
->finished
);
524 wake_up_interruptible(&sched
->wake_up_worker
);
527 static bool amd_sched_blocked(struct amd_gpu_scheduler
*sched
)
529 if (kthread_should_park()) {
537 static int amd_sched_main(void *param
)
539 struct sched_param sparam
= {.sched_priority
= 1};
540 struct amd_gpu_scheduler
*sched
= (struct amd_gpu_scheduler
*)param
;
543 sched_setscheduler(current
, SCHED_FIFO
, &sparam
);
545 while (!kthread_should_stop()) {
546 struct amd_sched_entity
*entity
= NULL
;
547 struct amd_sched_fence
*s_fence
;
548 struct amd_sched_job
*sched_job
;
549 struct dma_fence
*fence
;
551 wait_event_interruptible(sched
->wake_up_worker
,
552 (!amd_sched_blocked(sched
) &&
553 (entity
= amd_sched_select_entity(sched
))) ||
554 kthread_should_stop());
559 sched_job
= amd_sched_entity_pop_job(entity
);
563 s_fence
= sched_job
->s_fence
;
565 atomic_inc(&sched
->hw_rq_count
);
566 amd_sched_job_begin(sched_job
);
568 fence
= sched
->ops
->run_job(sched_job
);
569 amd_sched_fence_scheduled(s_fence
);
571 s_fence
->parent
= dma_fence_get(fence
);
572 r
= dma_fence_add_callback(fence
, &s_fence
->cb
,
573 amd_sched_process_job
);
575 amd_sched_process_job(fence
, &s_fence
->cb
);
577 DRM_ERROR("fence add callback failed (%d)\n",
579 dma_fence_put(fence
);
581 DRM_ERROR("Failed to run job!\n");
582 amd_sched_process_job(NULL
, &s_fence
->cb
);
585 count
= kfifo_out(&entity
->job_queue
, &sched_job
,
587 WARN_ON(count
!= sizeof(sched_job
));
588 wake_up(&sched
->job_scheduled
);
594 * Init a gpu scheduler instance
596 * @sched The pointer to the scheduler
597 * @ops The backend operations for this scheduler.
598 * @hw_submissions Number of hw submissions to do.
599 * @name Name used for debugging
601 * Return 0 on success, otherwise error code.
603 int amd_sched_init(struct amd_gpu_scheduler
*sched
,
604 const struct amd_sched_backend_ops
*ops
,
605 unsigned hw_submission
, long timeout
, const char *name
)
609 sched
->hw_submission_limit
= hw_submission
;
611 sched
->timeout
= timeout
;
612 for (i
= 0; i
< AMD_SCHED_MAX_PRIORITY
; i
++)
613 amd_sched_rq_init(&sched
->sched_rq
[i
]);
615 init_waitqueue_head(&sched
->wake_up_worker
);
616 init_waitqueue_head(&sched
->job_scheduled
);
617 INIT_LIST_HEAD(&sched
->ring_mirror_list
);
618 spin_lock_init(&sched
->job_list_lock
);
619 atomic_set(&sched
->hw_rq_count
, 0);
621 /* Each scheduler will run on a seperate kernel thread */
622 sched
->thread
= kthread_run(amd_sched_main
, sched
, sched
->name
);
623 if (IS_ERR(sched
->thread
)) {
624 DRM_ERROR("Failed to create scheduler for %s.\n", name
);
625 return PTR_ERR(sched
->thread
);
632 * Destroy a gpu scheduler
634 * @sched The pointer to the scheduler
636 void amd_sched_fini(struct amd_gpu_scheduler
*sched
)
639 kthread_stop(sched
->thread
);