2 * Copyright 2015 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
24 #include <linux/kthread.h>
25 #include <linux/wait.h>
26 #include <linux/sched.h>
27 #include <uapi/linux/sched/types.h>
29 #include "gpu_scheduler.h"
31 #define CREATE_TRACE_POINTS
32 #include "gpu_sched_trace.h"
34 static bool amd_sched_entity_is_ready(struct amd_sched_entity
*entity
);
35 static void amd_sched_wakeup(struct amd_gpu_scheduler
*sched
);
36 static void amd_sched_process_job(struct dma_fence
*f
, struct dma_fence_cb
*cb
);
38 /* Initialize a given run queue struct */
39 static void amd_sched_rq_init(struct amd_sched_rq
*rq
)
41 spin_lock_init(&rq
->lock
);
42 INIT_LIST_HEAD(&rq
->entities
);
43 rq
->current_entity
= NULL
;
46 static void amd_sched_rq_add_entity(struct amd_sched_rq
*rq
,
47 struct amd_sched_entity
*entity
)
49 if (!list_empty(&entity
->list
))
52 list_add_tail(&entity
->list
, &rq
->entities
);
53 spin_unlock(&rq
->lock
);
56 static void amd_sched_rq_remove_entity(struct amd_sched_rq
*rq
,
57 struct amd_sched_entity
*entity
)
59 if (list_empty(&entity
->list
))
62 list_del_init(&entity
->list
);
63 if (rq
->current_entity
== entity
)
64 rq
->current_entity
= NULL
;
65 spin_unlock(&rq
->lock
);
69 * Select an entity which could provide a job to run
71 * @rq The run queue to check.
73 * Try to find a ready entity, returns NULL if none found.
75 static struct amd_sched_entity
*
76 amd_sched_rq_select_entity(struct amd_sched_rq
*rq
)
78 struct amd_sched_entity
*entity
;
82 entity
= rq
->current_entity
;
84 list_for_each_entry_continue(entity
, &rq
->entities
, list
) {
85 if (amd_sched_entity_is_ready(entity
)) {
86 rq
->current_entity
= entity
;
87 spin_unlock(&rq
->lock
);
93 list_for_each_entry(entity
, &rq
->entities
, list
) {
95 if (amd_sched_entity_is_ready(entity
)) {
96 rq
->current_entity
= entity
;
97 spin_unlock(&rq
->lock
);
101 if (entity
== rq
->current_entity
)
105 spin_unlock(&rq
->lock
);
111 * Init a context entity used by scheduler when submit to HW ring.
113 * @sched The pointer to the scheduler
114 * @entity The pointer to a valid amd_sched_entity
115 * @rq The run queue this entity belongs
116 * @kernel If this is an entity for the kernel
117 * @jobs The max number of jobs in the job queue
119 * return 0 if succeed. negative error code on failure
121 int amd_sched_entity_init(struct amd_gpu_scheduler
*sched
,
122 struct amd_sched_entity
*entity
,
123 struct amd_sched_rq
*rq
,
128 if (!(sched
&& entity
&& rq
))
131 memset(entity
, 0, sizeof(struct amd_sched_entity
));
132 INIT_LIST_HEAD(&entity
->list
);
134 entity
->sched
= sched
;
136 spin_lock_init(&entity
->queue_lock
);
137 r
= kfifo_alloc(&entity
->job_queue
, jobs
* sizeof(void *), GFP_KERNEL
);
141 atomic_set(&entity
->fence_seq
, 0);
142 entity
->fence_context
= dma_fence_context_alloc(2);
148 * Query if entity is initialized
150 * @sched Pointer to scheduler instance
151 * @entity The pointer to a valid scheduler entity
153 * return true if entity is initialized, false otherwise
155 static bool amd_sched_entity_is_initialized(struct amd_gpu_scheduler
*sched
,
156 struct amd_sched_entity
*entity
)
158 return entity
->sched
== sched
&&
163 * Check if entity is idle
165 * @entity The pointer to a valid scheduler entity
167 * Return true if entity don't has any unscheduled jobs.
169 static bool amd_sched_entity_is_idle(struct amd_sched_entity
*entity
)
172 if (kfifo_is_empty(&entity
->job_queue
))
179 * Check if entity is ready
181 * @entity The pointer to a valid scheduler entity
183 * Return true if entity could provide a job.
185 static bool amd_sched_entity_is_ready(struct amd_sched_entity
*entity
)
187 if (kfifo_is_empty(&entity
->job_queue
))
190 if (ACCESS_ONCE(entity
->dependency
))
197 * Destroy a context entity
199 * @sched Pointer to scheduler instance
200 * @entity The pointer to a valid scheduler entity
202 * Cleanup and free the allocated resources.
204 void amd_sched_entity_fini(struct amd_gpu_scheduler
*sched
,
205 struct amd_sched_entity
*entity
)
207 struct amd_sched_rq
*rq
= entity
->rq
;
210 if (!amd_sched_entity_is_initialized(sched
, entity
))
213 * The client will not queue more IBs during this fini, consume existing
214 * queued IBs or discard them on SIGKILL
216 if ((current
->flags
& PF_SIGNALED
) && current
->exit_code
== SIGKILL
)
219 r
= wait_event_killable(sched
->job_scheduled
,
220 amd_sched_entity_is_idle(entity
));
221 amd_sched_rq_remove_entity(rq
, entity
);
223 struct amd_sched_job
*job
;
225 /* Park the kernel for a moment to make sure it isn't processing
228 kthread_park(sched
->thread
);
229 kthread_unpark(sched
->thread
);
230 while (kfifo_out(&entity
->job_queue
, &job
, sizeof(job
))) {
231 struct amd_sched_fence
*s_fence
= job
->s_fence
;
232 amd_sched_fence_scheduled(s_fence
);
233 dma_fence_set_error(&s_fence
->finished
, -ESRCH
);
234 amd_sched_fence_finished(s_fence
);
235 dma_fence_put(&s_fence
->finished
);
236 sched
->ops
->free_job(job
);
240 kfifo_free(&entity
->job_queue
);
243 static void amd_sched_entity_wakeup(struct dma_fence
*f
, struct dma_fence_cb
*cb
)
245 struct amd_sched_entity
*entity
=
246 container_of(cb
, struct amd_sched_entity
, cb
);
247 entity
->dependency
= NULL
;
249 amd_sched_wakeup(entity
->sched
);
252 static void amd_sched_entity_clear_dep(struct dma_fence
*f
, struct dma_fence_cb
*cb
)
254 struct amd_sched_entity
*entity
=
255 container_of(cb
, struct amd_sched_entity
, cb
);
256 entity
->dependency
= NULL
;
260 bool amd_sched_dependency_optimized(struct dma_fence
* fence
,
261 struct amd_sched_entity
*entity
)
263 struct amd_gpu_scheduler
*sched
= entity
->sched
;
264 struct amd_sched_fence
*s_fence
;
266 if (!fence
|| dma_fence_is_signaled(fence
))
268 if (fence
->context
== entity
->fence_context
)
270 s_fence
= to_amd_sched_fence(fence
);
271 if (s_fence
&& s_fence
->sched
== sched
)
277 static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity
*entity
)
279 struct amd_gpu_scheduler
*sched
= entity
->sched
;
280 struct dma_fence
* fence
= entity
->dependency
;
281 struct amd_sched_fence
*s_fence
;
283 if (fence
->context
== entity
->fence_context
) {
284 /* We can ignore fences from ourself */
285 dma_fence_put(entity
->dependency
);
289 s_fence
= to_amd_sched_fence(fence
);
290 if (s_fence
&& s_fence
->sched
== sched
) {
293 * Fence is from the same scheduler, only need to wait for
296 fence
= dma_fence_get(&s_fence
->scheduled
);
297 dma_fence_put(entity
->dependency
);
298 entity
->dependency
= fence
;
299 if (!dma_fence_add_callback(fence
, &entity
->cb
,
300 amd_sched_entity_clear_dep
))
303 /* Ignore it when it is already scheduled */
304 dma_fence_put(fence
);
308 if (!dma_fence_add_callback(entity
->dependency
, &entity
->cb
,
309 amd_sched_entity_wakeup
))
312 dma_fence_put(entity
->dependency
);
316 static struct amd_sched_job
*
317 amd_sched_entity_peek_job(struct amd_sched_entity
*entity
)
319 struct amd_gpu_scheduler
*sched
= entity
->sched
;
320 struct amd_sched_job
*sched_job
;
322 if (!kfifo_out_peek(&entity
->job_queue
, &sched_job
, sizeof(sched_job
)))
325 while ((entity
->dependency
= sched
->ops
->dependency(sched_job
)))
326 if (amd_sched_entity_add_dependency_cb(entity
))
333 * Helper to submit a job to the job queue
335 * @sched_job The pointer to job required to submit
337 * Returns true if we could submit the job.
339 static bool amd_sched_entity_in(struct amd_sched_job
*sched_job
)
341 struct amd_gpu_scheduler
*sched
= sched_job
->sched
;
342 struct amd_sched_entity
*entity
= sched_job
->s_entity
;
343 bool added
, first
= false;
345 spin_lock(&entity
->queue_lock
);
346 added
= kfifo_in(&entity
->job_queue
, &sched_job
,
347 sizeof(sched_job
)) == sizeof(sched_job
);
349 if (added
&& kfifo_len(&entity
->job_queue
) == sizeof(sched_job
))
352 spin_unlock(&entity
->queue_lock
);
354 /* first job wakes up scheduler */
356 /* Add the entity to the run queue */
357 amd_sched_rq_add_entity(entity
->rq
, entity
);
358 amd_sched_wakeup(sched
);
363 /* job_finish is called after hw fence signaled
365 static void amd_sched_job_finish(struct work_struct
*work
)
367 struct amd_sched_job
*s_job
= container_of(work
, struct amd_sched_job
,
369 struct amd_gpu_scheduler
*sched
= s_job
->sched
;
371 /* remove job from ring_mirror_list */
372 spin_lock(&sched
->job_list_lock
);
373 list_del_init(&s_job
->node
);
374 if (sched
->timeout
!= MAX_SCHEDULE_TIMEOUT
) {
375 struct amd_sched_job
*next
;
377 spin_unlock(&sched
->job_list_lock
);
378 cancel_delayed_work_sync(&s_job
->work_tdr
);
379 spin_lock(&sched
->job_list_lock
);
381 /* queue TDR for next job */
382 next
= list_first_entry_or_null(&sched
->ring_mirror_list
,
383 struct amd_sched_job
, node
);
386 schedule_delayed_work(&next
->work_tdr
, sched
->timeout
);
388 spin_unlock(&sched
->job_list_lock
);
389 sched
->ops
->free_job(s_job
);
392 static void amd_sched_job_finish_cb(struct dma_fence
*f
,
393 struct dma_fence_cb
*cb
)
395 struct amd_sched_job
*job
= container_of(cb
, struct amd_sched_job
,
397 schedule_work(&job
->finish_work
);
400 static void amd_sched_job_begin(struct amd_sched_job
*s_job
)
402 struct amd_gpu_scheduler
*sched
= s_job
->sched
;
404 dma_fence_add_callback(&s_job
->s_fence
->finished
, &s_job
->finish_cb
,
405 amd_sched_job_finish_cb
);
407 spin_lock(&sched
->job_list_lock
);
408 list_add_tail(&s_job
->node
, &sched
->ring_mirror_list
);
409 if (sched
->timeout
!= MAX_SCHEDULE_TIMEOUT
&&
410 list_first_entry_or_null(&sched
->ring_mirror_list
,
411 struct amd_sched_job
, node
) == s_job
)
412 schedule_delayed_work(&s_job
->work_tdr
, sched
->timeout
);
413 spin_unlock(&sched
->job_list_lock
);
416 static void amd_sched_job_timedout(struct work_struct
*work
)
418 struct amd_sched_job
*job
= container_of(work
, struct amd_sched_job
,
421 job
->sched
->ops
->timedout_job(job
);
424 void amd_sched_hw_job_reset(struct amd_gpu_scheduler
*sched
)
426 struct amd_sched_job
*s_job
;
428 spin_lock(&sched
->job_list_lock
);
429 list_for_each_entry_reverse(s_job
, &sched
->ring_mirror_list
, node
) {
430 if (s_job
->s_fence
->parent
&&
431 dma_fence_remove_callback(s_job
->s_fence
->parent
,
432 &s_job
->s_fence
->cb
)) {
433 dma_fence_put(s_job
->s_fence
->parent
);
434 s_job
->s_fence
->parent
= NULL
;
435 atomic_dec(&sched
->hw_rq_count
);
438 spin_unlock(&sched
->job_list_lock
);
441 void amd_sched_job_kickout(struct amd_sched_job
*s_job
)
443 struct amd_gpu_scheduler
*sched
= s_job
->sched
;
445 spin_lock(&sched
->job_list_lock
);
446 list_del_init(&s_job
->node
);
447 spin_unlock(&sched
->job_list_lock
);
450 void amd_sched_job_recovery(struct amd_gpu_scheduler
*sched
)
452 struct amd_sched_job
*s_job
, *tmp
;
455 spin_lock(&sched
->job_list_lock
);
456 s_job
= list_first_entry_or_null(&sched
->ring_mirror_list
,
457 struct amd_sched_job
, node
);
458 if (s_job
&& sched
->timeout
!= MAX_SCHEDULE_TIMEOUT
)
459 schedule_delayed_work(&s_job
->work_tdr
, sched
->timeout
);
461 list_for_each_entry_safe(s_job
, tmp
, &sched
->ring_mirror_list
, node
) {
462 struct amd_sched_fence
*s_fence
= s_job
->s_fence
;
463 struct dma_fence
*fence
;
465 spin_unlock(&sched
->job_list_lock
);
466 fence
= sched
->ops
->run_job(s_job
);
467 atomic_inc(&sched
->hw_rq_count
);
469 s_fence
->parent
= dma_fence_get(fence
);
470 r
= dma_fence_add_callback(fence
, &s_fence
->cb
,
471 amd_sched_process_job
);
473 amd_sched_process_job(fence
, &s_fence
->cb
);
475 DRM_ERROR("fence add callback failed (%d)\n",
477 dma_fence_put(fence
);
479 DRM_ERROR("Failed to run job!\n");
480 amd_sched_process_job(NULL
, &s_fence
->cb
);
482 spin_lock(&sched
->job_list_lock
);
484 spin_unlock(&sched
->job_list_lock
);
488 * Submit a job to the job queue
490 * @sched_job The pointer to job required to submit
492 * Returns 0 for success, negative error code otherwise.
494 void amd_sched_entity_push_job(struct amd_sched_job
*sched_job
)
496 struct amd_sched_entity
*entity
= sched_job
->s_entity
;
498 trace_amd_sched_job(sched_job
);
499 wait_event(entity
->sched
->job_scheduled
,
500 amd_sched_entity_in(sched_job
));
503 /* init a sched_job with basic field */
504 int amd_sched_job_init(struct amd_sched_job
*job
,
505 struct amd_gpu_scheduler
*sched
,
506 struct amd_sched_entity
*entity
,
510 job
->s_entity
= entity
;
511 job
->s_fence
= amd_sched_fence_create(entity
, owner
);
514 job
->id
= atomic64_inc_return(&sched
->job_id_count
);
516 INIT_WORK(&job
->finish_work
, amd_sched_job_finish
);
517 INIT_LIST_HEAD(&job
->node
);
518 INIT_DELAYED_WORK(&job
->work_tdr
, amd_sched_job_timedout
);
524 * Return ture if we can push more jobs to the hw.
526 static bool amd_sched_ready(struct amd_gpu_scheduler
*sched
)
528 return atomic_read(&sched
->hw_rq_count
) <
529 sched
->hw_submission_limit
;
533 * Wake up the scheduler when it is ready
535 static void amd_sched_wakeup(struct amd_gpu_scheduler
*sched
)
537 if (amd_sched_ready(sched
))
538 wake_up_interruptible(&sched
->wake_up_worker
);
542 * Select next entity to process
544 static struct amd_sched_entity
*
545 amd_sched_select_entity(struct amd_gpu_scheduler
*sched
)
547 struct amd_sched_entity
*entity
;
550 if (!amd_sched_ready(sched
))
553 /* Kernel run queue has higher priority than normal run queue*/
554 for (i
= AMD_SCHED_PRIORITY_MAX
- 1; i
>= AMD_SCHED_PRIORITY_MIN
; i
--) {
555 entity
= amd_sched_rq_select_entity(&sched
->sched_rq
[i
]);
563 static void amd_sched_process_job(struct dma_fence
*f
, struct dma_fence_cb
*cb
)
565 struct amd_sched_fence
*s_fence
=
566 container_of(cb
, struct amd_sched_fence
, cb
);
567 struct amd_gpu_scheduler
*sched
= s_fence
->sched
;
569 atomic_dec(&sched
->hw_rq_count
);
570 amd_sched_fence_finished(s_fence
);
572 trace_amd_sched_process_job(s_fence
);
573 dma_fence_put(&s_fence
->finished
);
574 wake_up_interruptible(&sched
->wake_up_worker
);
577 static bool amd_sched_blocked(struct amd_gpu_scheduler
*sched
)
579 if (kthread_should_park()) {
587 static int amd_sched_main(void *param
)
589 struct sched_param sparam
= {.sched_priority
= 1};
590 struct amd_gpu_scheduler
*sched
= (struct amd_gpu_scheduler
*)param
;
593 sched_setscheduler(current
, SCHED_FIFO
, &sparam
);
595 while (!kthread_should_stop()) {
596 struct amd_sched_entity
*entity
= NULL
;
597 struct amd_sched_fence
*s_fence
;
598 struct amd_sched_job
*sched_job
;
599 struct dma_fence
*fence
;
601 wait_event_interruptible(sched
->wake_up_worker
,
602 (!amd_sched_blocked(sched
) &&
603 (entity
= amd_sched_select_entity(sched
))) ||
604 kthread_should_stop());
609 sched_job
= amd_sched_entity_peek_job(entity
);
613 s_fence
= sched_job
->s_fence
;
615 atomic_inc(&sched
->hw_rq_count
);
616 amd_sched_job_begin(sched_job
);
618 fence
= sched
->ops
->run_job(sched_job
);
619 amd_sched_fence_scheduled(s_fence
);
621 /* amd_sched_process_job drops the job's reference of the fence. */
622 sched_job
->s_fence
= NULL
;
625 s_fence
->parent
= dma_fence_get(fence
);
626 r
= dma_fence_add_callback(fence
, &s_fence
->cb
,
627 amd_sched_process_job
);
629 amd_sched_process_job(fence
, &s_fence
->cb
);
631 DRM_ERROR("fence add callback failed (%d)\n",
633 dma_fence_put(fence
);
635 DRM_ERROR("Failed to run job!\n");
636 amd_sched_process_job(NULL
, &s_fence
->cb
);
639 count
= kfifo_out(&entity
->job_queue
, &sched_job
,
641 WARN_ON(count
!= sizeof(sched_job
));
642 wake_up(&sched
->job_scheduled
);
648 * Init a gpu scheduler instance
650 * @sched The pointer to the scheduler
651 * @ops The backend operations for this scheduler.
652 * @hw_submissions Number of hw submissions to do.
653 * @name Name used for debugging
655 * Return 0 on success, otherwise error code.
657 int amd_sched_init(struct amd_gpu_scheduler
*sched
,
658 const struct amd_sched_backend_ops
*ops
,
659 unsigned hw_submission
, long timeout
, const char *name
)
663 sched
->hw_submission_limit
= hw_submission
;
665 sched
->timeout
= timeout
;
666 for (i
= AMD_SCHED_PRIORITY_MIN
; i
< AMD_SCHED_PRIORITY_MAX
; i
++)
667 amd_sched_rq_init(&sched
->sched_rq
[i
]);
669 init_waitqueue_head(&sched
->wake_up_worker
);
670 init_waitqueue_head(&sched
->job_scheduled
);
671 INIT_LIST_HEAD(&sched
->ring_mirror_list
);
672 spin_lock_init(&sched
->job_list_lock
);
673 atomic_set(&sched
->hw_rq_count
, 0);
674 atomic64_set(&sched
->job_id_count
, 0);
676 /* Each scheduler will run on a seperate kernel thread */
677 sched
->thread
= kthread_run(amd_sched_main
, sched
, sched
->name
);
678 if (IS_ERR(sched
->thread
)) {
679 DRM_ERROR("Failed to create scheduler for %s.\n", name
);
680 return PTR_ERR(sched
->thread
);
687 * Destroy a gpu scheduler
689 * @sched The pointer to the scheduler
691 void amd_sched_fini(struct amd_gpu_scheduler
*sched
)
694 kthread_stop(sched
->thread
);