2 * Copyright 2015 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
24 #include <linux/kthread.h>
25 #include <linux/wait.h>
26 #include <linux/sched.h>
27 #include <uapi/linux/sched/types.h>
29 #include "gpu_scheduler.h"
31 #define CREATE_TRACE_POINTS
32 #include "gpu_sched_trace.h"
34 static bool amd_sched_entity_is_ready(struct amd_sched_entity
*entity
);
35 static void amd_sched_wakeup(struct amd_gpu_scheduler
*sched
);
36 static void amd_sched_process_job(struct dma_fence
*f
, struct dma_fence_cb
*cb
);
38 /* Initialize a given run queue struct */
39 static void amd_sched_rq_init(struct amd_sched_rq
*rq
)
41 spin_lock_init(&rq
->lock
);
42 INIT_LIST_HEAD(&rq
->entities
);
43 rq
->current_entity
= NULL
;
46 static void amd_sched_rq_add_entity(struct amd_sched_rq
*rq
,
47 struct amd_sched_entity
*entity
)
49 if (!list_empty(&entity
->list
))
52 list_add_tail(&entity
->list
, &rq
->entities
);
53 spin_unlock(&rq
->lock
);
56 static void amd_sched_rq_remove_entity(struct amd_sched_rq
*rq
,
57 struct amd_sched_entity
*entity
)
59 if (list_empty(&entity
->list
))
62 list_del_init(&entity
->list
);
63 if (rq
->current_entity
== entity
)
64 rq
->current_entity
= NULL
;
65 spin_unlock(&rq
->lock
);
69 * Select an entity which could provide a job to run
71 * @rq The run queue to check.
73 * Try to find a ready entity, returns NULL if none found.
75 static struct amd_sched_entity
*
76 amd_sched_rq_select_entity(struct amd_sched_rq
*rq
)
78 struct amd_sched_entity
*entity
;
82 entity
= rq
->current_entity
;
84 list_for_each_entry_continue(entity
, &rq
->entities
, list
) {
85 if (amd_sched_entity_is_ready(entity
)) {
86 rq
->current_entity
= entity
;
87 spin_unlock(&rq
->lock
);
93 list_for_each_entry(entity
, &rq
->entities
, list
) {
95 if (amd_sched_entity_is_ready(entity
)) {
96 rq
->current_entity
= entity
;
97 spin_unlock(&rq
->lock
);
101 if (entity
== rq
->current_entity
)
105 spin_unlock(&rq
->lock
);
111 * Init a context entity used by scheduler when submit to HW ring.
113 * @sched The pointer to the scheduler
114 * @entity The pointer to a valid amd_sched_entity
115 * @rq The run queue this entity belongs
116 * @kernel If this is an entity for the kernel
117 * @jobs The max number of jobs in the job queue
119 * return 0 if succeed. negative error code on failure
121 int amd_sched_entity_init(struct amd_gpu_scheduler
*sched
,
122 struct amd_sched_entity
*entity
,
123 struct amd_sched_rq
*rq
,
128 if (!(sched
&& entity
&& rq
))
131 memset(entity
, 0, sizeof(struct amd_sched_entity
));
132 INIT_LIST_HEAD(&entity
->list
);
134 entity
->sched
= sched
;
136 spin_lock_init(&entity
->queue_lock
);
137 r
= kfifo_alloc(&entity
->job_queue
, jobs
* sizeof(void *), GFP_KERNEL
);
141 atomic_set(&entity
->fence_seq
, 0);
142 entity
->fence_context
= dma_fence_context_alloc(2);
148 * Query if entity is initialized
150 * @sched Pointer to scheduler instance
151 * @entity The pointer to a valid scheduler entity
153 * return true if entity is initialized, false otherwise
155 static bool amd_sched_entity_is_initialized(struct amd_gpu_scheduler
*sched
,
156 struct amd_sched_entity
*entity
)
158 return entity
->sched
== sched
&&
163 * Check if entity is idle
165 * @entity The pointer to a valid scheduler entity
167 * Return true if entity don't has any unscheduled jobs.
169 static bool amd_sched_entity_is_idle(struct amd_sched_entity
*entity
)
172 if (kfifo_is_empty(&entity
->job_queue
))
179 * Check if entity is ready
181 * @entity The pointer to a valid scheduler entity
183 * Return true if entity could provide a job.
185 static bool amd_sched_entity_is_ready(struct amd_sched_entity
*entity
)
187 if (kfifo_is_empty(&entity
->job_queue
))
190 if (READ_ONCE(entity
->dependency
))
197 * Destroy a context entity
199 * @sched Pointer to scheduler instance
200 * @entity The pointer to a valid scheduler entity
202 * Cleanup and free the allocated resources.
204 void amd_sched_entity_fini(struct amd_gpu_scheduler
*sched
,
205 struct amd_sched_entity
*entity
)
207 struct amd_sched_rq
*rq
= entity
->rq
;
209 if (!amd_sched_entity_is_initialized(sched
, entity
))
213 * The client will not queue more IBs during this fini, consume existing
216 wait_event(sched
->job_scheduled
, amd_sched_entity_is_idle(entity
));
218 amd_sched_rq_remove_entity(rq
, entity
);
219 kfifo_free(&entity
->job_queue
);
222 static void amd_sched_entity_wakeup(struct dma_fence
*f
, struct dma_fence_cb
*cb
)
224 struct amd_sched_entity
*entity
=
225 container_of(cb
, struct amd_sched_entity
, cb
);
226 entity
->dependency
= NULL
;
228 amd_sched_wakeup(entity
->sched
);
231 static void amd_sched_entity_clear_dep(struct dma_fence
*f
, struct dma_fence_cb
*cb
)
233 struct amd_sched_entity
*entity
=
234 container_of(cb
, struct amd_sched_entity
, cb
);
235 entity
->dependency
= NULL
;
239 bool amd_sched_dependency_optimized(struct dma_fence
* fence
,
240 struct amd_sched_entity
*entity
)
242 struct amd_gpu_scheduler
*sched
= entity
->sched
;
243 struct amd_sched_fence
*s_fence
;
245 if (!fence
|| dma_fence_is_signaled(fence
))
247 if (fence
->context
== entity
->fence_context
)
249 s_fence
= to_amd_sched_fence(fence
);
250 if (s_fence
&& s_fence
->sched
== sched
)
256 static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity
*entity
)
258 struct amd_gpu_scheduler
*sched
= entity
->sched
;
259 struct dma_fence
* fence
= entity
->dependency
;
260 struct amd_sched_fence
*s_fence
;
262 if (fence
->context
== entity
->fence_context
) {
263 /* We can ignore fences from ourself */
264 dma_fence_put(entity
->dependency
);
268 s_fence
= to_amd_sched_fence(fence
);
269 if (s_fence
&& s_fence
->sched
== sched
) {
272 * Fence is from the same scheduler, only need to wait for
275 fence
= dma_fence_get(&s_fence
->scheduled
);
276 dma_fence_put(entity
->dependency
);
277 entity
->dependency
= fence
;
278 if (!dma_fence_add_callback(fence
, &entity
->cb
,
279 amd_sched_entity_clear_dep
))
282 /* Ignore it when it is already scheduled */
283 dma_fence_put(fence
);
287 if (!dma_fence_add_callback(entity
->dependency
, &entity
->cb
,
288 amd_sched_entity_wakeup
))
291 dma_fence_put(entity
->dependency
);
295 static struct amd_sched_job
*
296 amd_sched_entity_pop_job(struct amd_sched_entity
*entity
)
298 struct amd_gpu_scheduler
*sched
= entity
->sched
;
299 struct amd_sched_job
*sched_job
;
301 if (!kfifo_out_peek(&entity
->job_queue
, &sched_job
, sizeof(sched_job
)))
304 while ((entity
->dependency
= sched
->ops
->dependency(sched_job
)))
305 if (amd_sched_entity_add_dependency_cb(entity
))
312 * Helper to submit a job to the job queue
314 * @sched_job The pointer to job required to submit
316 * Returns true if we could submit the job.
318 static bool amd_sched_entity_in(struct amd_sched_job
*sched_job
)
320 struct amd_gpu_scheduler
*sched
= sched_job
->sched
;
321 struct amd_sched_entity
*entity
= sched_job
->s_entity
;
322 bool added
, first
= false;
324 spin_lock(&entity
->queue_lock
);
325 added
= kfifo_in(&entity
->job_queue
, &sched_job
,
326 sizeof(sched_job
)) == sizeof(sched_job
);
328 if (added
&& kfifo_len(&entity
->job_queue
) == sizeof(sched_job
))
331 spin_unlock(&entity
->queue_lock
);
333 /* first job wakes up scheduler */
335 /* Add the entity to the run queue */
336 amd_sched_rq_add_entity(entity
->rq
, entity
);
337 amd_sched_wakeup(sched
);
342 /* job_finish is called after hw fence signaled, and
343 * the job had already been deleted from ring_mirror_list
345 static void amd_sched_job_finish(struct work_struct
*work
)
347 struct amd_sched_job
*s_job
= container_of(work
, struct amd_sched_job
,
349 struct amd_gpu_scheduler
*sched
= s_job
->sched
;
351 /* remove job from ring_mirror_list */
352 spin_lock(&sched
->job_list_lock
);
353 list_del_init(&s_job
->node
);
354 if (sched
->timeout
!= MAX_SCHEDULE_TIMEOUT
) {
355 struct amd_sched_job
*next
;
357 spin_unlock(&sched
->job_list_lock
);
358 cancel_delayed_work_sync(&s_job
->work_tdr
);
359 spin_lock(&sched
->job_list_lock
);
361 /* queue TDR for next job */
362 next
= list_first_entry_or_null(&sched
->ring_mirror_list
,
363 struct amd_sched_job
, node
);
366 schedule_delayed_work(&next
->work_tdr
, sched
->timeout
);
368 spin_unlock(&sched
->job_list_lock
);
369 sched
->ops
->free_job(s_job
);
372 static void amd_sched_job_finish_cb(struct dma_fence
*f
,
373 struct dma_fence_cb
*cb
)
375 struct amd_sched_job
*job
= container_of(cb
, struct amd_sched_job
,
377 schedule_work(&job
->finish_work
);
380 static void amd_sched_job_begin(struct amd_sched_job
*s_job
)
382 struct amd_gpu_scheduler
*sched
= s_job
->sched
;
384 spin_lock(&sched
->job_list_lock
);
385 list_add_tail(&s_job
->node
, &sched
->ring_mirror_list
);
386 if (sched
->timeout
!= MAX_SCHEDULE_TIMEOUT
&&
387 list_first_entry_or_null(&sched
->ring_mirror_list
,
388 struct amd_sched_job
, node
) == s_job
)
389 schedule_delayed_work(&s_job
->work_tdr
, sched
->timeout
);
390 spin_unlock(&sched
->job_list_lock
);
393 static void amd_sched_job_timedout(struct work_struct
*work
)
395 struct amd_sched_job
*job
= container_of(work
, struct amd_sched_job
,
398 job
->sched
->ops
->timedout_job(job
);
401 void amd_sched_hw_job_reset(struct amd_gpu_scheduler
*sched
)
403 struct amd_sched_job
*s_job
;
405 spin_lock(&sched
->job_list_lock
);
406 list_for_each_entry_reverse(s_job
, &sched
->ring_mirror_list
, node
) {
407 if (s_job
->s_fence
->parent
&&
408 dma_fence_remove_callback(s_job
->s_fence
->parent
,
409 &s_job
->s_fence
->cb
)) {
410 dma_fence_put(s_job
->s_fence
->parent
);
411 s_job
->s_fence
->parent
= NULL
;
412 atomic_dec(&sched
->hw_rq_count
);
415 spin_unlock(&sched
->job_list_lock
);
418 void amd_sched_job_kickout(struct amd_sched_job
*s_job
)
420 struct amd_gpu_scheduler
*sched
= s_job
->sched
;
422 spin_lock(&sched
->job_list_lock
);
423 list_del_init(&s_job
->node
);
424 spin_unlock(&sched
->job_list_lock
);
427 void amd_sched_job_recovery(struct amd_gpu_scheduler
*sched
)
429 struct amd_sched_job
*s_job
, *tmp
;
432 spin_lock(&sched
->job_list_lock
);
433 s_job
= list_first_entry_or_null(&sched
->ring_mirror_list
,
434 struct amd_sched_job
, node
);
435 if (s_job
&& sched
->timeout
!= MAX_SCHEDULE_TIMEOUT
)
436 schedule_delayed_work(&s_job
->work_tdr
, sched
->timeout
);
438 list_for_each_entry_safe(s_job
, tmp
, &sched
->ring_mirror_list
, node
) {
439 struct amd_sched_fence
*s_fence
= s_job
->s_fence
;
440 struct dma_fence
*fence
;
442 spin_unlock(&sched
->job_list_lock
);
443 fence
= sched
->ops
->run_job(s_job
);
444 atomic_inc(&sched
->hw_rq_count
);
446 s_fence
->parent
= dma_fence_get(fence
);
447 r
= dma_fence_add_callback(fence
, &s_fence
->cb
,
448 amd_sched_process_job
);
450 amd_sched_process_job(fence
, &s_fence
->cb
);
452 DRM_ERROR("fence add callback failed (%d)\n",
454 dma_fence_put(fence
);
456 DRM_ERROR("Failed to run job!\n");
457 amd_sched_process_job(NULL
, &s_fence
->cb
);
459 spin_lock(&sched
->job_list_lock
);
461 spin_unlock(&sched
->job_list_lock
);
465 * Submit a job to the job queue
467 * @sched_job The pointer to job required to submit
469 * Returns 0 for success, negative error code otherwise.
471 void amd_sched_entity_push_job(struct amd_sched_job
*sched_job
)
473 struct amd_sched_entity
*entity
= sched_job
->s_entity
;
475 trace_amd_sched_job(sched_job
);
476 dma_fence_add_callback(&sched_job
->s_fence
->finished
, &sched_job
->finish_cb
,
477 amd_sched_job_finish_cb
);
478 wait_event(entity
->sched
->job_scheduled
,
479 amd_sched_entity_in(sched_job
));
482 /* init a sched_job with basic field */
483 int amd_sched_job_init(struct amd_sched_job
*job
,
484 struct amd_gpu_scheduler
*sched
,
485 struct amd_sched_entity
*entity
,
489 job
->s_entity
= entity
;
490 job
->s_fence
= amd_sched_fence_create(entity
, owner
);
493 job
->id
= atomic64_inc_return(&sched
->job_id_count
);
495 INIT_WORK(&job
->finish_work
, amd_sched_job_finish
);
496 INIT_LIST_HEAD(&job
->node
);
497 INIT_DELAYED_WORK(&job
->work_tdr
, amd_sched_job_timedout
);
503 * Return ture if we can push more jobs to the hw.
505 static bool amd_sched_ready(struct amd_gpu_scheduler
*sched
)
507 return atomic_read(&sched
->hw_rq_count
) <
508 sched
->hw_submission_limit
;
512 * Wake up the scheduler when it is ready
514 static void amd_sched_wakeup(struct amd_gpu_scheduler
*sched
)
516 if (amd_sched_ready(sched
))
517 wake_up_interruptible(&sched
->wake_up_worker
);
521 * Select next entity to process
523 static struct amd_sched_entity
*
524 amd_sched_select_entity(struct amd_gpu_scheduler
*sched
)
526 struct amd_sched_entity
*entity
;
529 if (!amd_sched_ready(sched
))
532 /* Kernel run queue has higher priority than normal run queue*/
533 for (i
= AMD_SCHED_PRIORITY_MAX
- 1; i
>= AMD_SCHED_PRIORITY_MIN
; i
--) {
534 entity
= amd_sched_rq_select_entity(&sched
->sched_rq
[i
]);
542 static void amd_sched_process_job(struct dma_fence
*f
, struct dma_fence_cb
*cb
)
544 struct amd_sched_fence
*s_fence
=
545 container_of(cb
, struct amd_sched_fence
, cb
);
546 struct amd_gpu_scheduler
*sched
= s_fence
->sched
;
548 atomic_dec(&sched
->hw_rq_count
);
549 amd_sched_fence_finished(s_fence
);
551 trace_amd_sched_process_job(s_fence
);
552 dma_fence_put(&s_fence
->finished
);
553 wake_up_interruptible(&sched
->wake_up_worker
);
556 static bool amd_sched_blocked(struct amd_gpu_scheduler
*sched
)
558 if (kthread_should_park()) {
566 static int amd_sched_main(void *param
)
568 struct sched_param sparam
= {.sched_priority
= 1};
569 struct amd_gpu_scheduler
*sched
= (struct amd_gpu_scheduler
*)param
;
572 sched_setscheduler(current
, SCHED_FIFO
, &sparam
);
574 while (!kthread_should_stop()) {
575 struct amd_sched_entity
*entity
= NULL
;
576 struct amd_sched_fence
*s_fence
;
577 struct amd_sched_job
*sched_job
;
578 struct dma_fence
*fence
;
580 wait_event_interruptible(sched
->wake_up_worker
,
581 (!amd_sched_blocked(sched
) &&
582 (entity
= amd_sched_select_entity(sched
))) ||
583 kthread_should_stop());
588 sched_job
= amd_sched_entity_pop_job(entity
);
592 s_fence
= sched_job
->s_fence
;
594 atomic_inc(&sched
->hw_rq_count
);
595 amd_sched_job_begin(sched_job
);
597 fence
= sched
->ops
->run_job(sched_job
);
598 amd_sched_fence_scheduled(s_fence
);
600 s_fence
->parent
= dma_fence_get(fence
);
601 r
= dma_fence_add_callback(fence
, &s_fence
->cb
,
602 amd_sched_process_job
);
604 amd_sched_process_job(fence
, &s_fence
->cb
);
606 DRM_ERROR("fence add callback failed (%d)\n",
608 dma_fence_put(fence
);
610 DRM_ERROR("Failed to run job!\n");
611 amd_sched_process_job(NULL
, &s_fence
->cb
);
614 count
= kfifo_out(&entity
->job_queue
, &sched_job
,
616 WARN_ON(count
!= sizeof(sched_job
));
617 wake_up(&sched
->job_scheduled
);
623 * Init a gpu scheduler instance
625 * @sched The pointer to the scheduler
626 * @ops The backend operations for this scheduler.
627 * @hw_submissions Number of hw submissions to do.
628 * @name Name used for debugging
630 * Return 0 on success, otherwise error code.
632 int amd_sched_init(struct amd_gpu_scheduler
*sched
,
633 const struct amd_sched_backend_ops
*ops
,
634 unsigned hw_submission
, long timeout
, const char *name
)
638 sched
->hw_submission_limit
= hw_submission
;
640 sched
->timeout
= timeout
;
641 for (i
= AMD_SCHED_PRIORITY_MIN
; i
< AMD_SCHED_PRIORITY_MAX
; i
++)
642 amd_sched_rq_init(&sched
->sched_rq
[i
]);
644 init_waitqueue_head(&sched
->wake_up_worker
);
645 init_waitqueue_head(&sched
->job_scheduled
);
646 INIT_LIST_HEAD(&sched
->ring_mirror_list
);
647 spin_lock_init(&sched
->job_list_lock
);
648 atomic_set(&sched
->hw_rq_count
, 0);
649 atomic64_set(&sched
->job_id_count
, 0);
651 /* Each scheduler will run on a seperate kernel thread */
652 sched
->thread
= kthread_run(amd_sched_main
, sched
, sched
->name
);
653 if (IS_ERR(sched
->thread
)) {
654 DRM_ERROR("Failed to create scheduler for %s.\n", name
);
655 return PTR_ERR(sched
->thread
);
662 * Destroy a gpu scheduler
664 * @sched The pointer to the scheduler
666 void amd_sched_fini(struct amd_gpu_scheduler
*sched
)
669 kthread_stop(sched
->thread
);