2 * Copyright 2015 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
24 #include <linux/kthread.h>
25 #include <linux/wait.h>
26 #include <linux/sched.h>
27 #include <uapi/linux/sched/types.h>
29 #include "gpu_scheduler.h"
31 #define CREATE_TRACE_POINTS
32 #include "gpu_sched_trace.h"
34 static bool amd_sched_entity_is_ready(struct amd_sched_entity
*entity
);
35 static void amd_sched_wakeup(struct amd_gpu_scheduler
*sched
);
36 static void amd_sched_process_job(struct dma_fence
*f
, struct dma_fence_cb
*cb
);
38 /* Initialize a given run queue struct */
39 static void amd_sched_rq_init(struct amd_sched_rq
*rq
)
41 spin_lock_init(&rq
->lock
);
42 INIT_LIST_HEAD(&rq
->entities
);
43 rq
->current_entity
= NULL
;
46 static void amd_sched_rq_add_entity(struct amd_sched_rq
*rq
,
47 struct amd_sched_entity
*entity
)
49 if (!list_empty(&entity
->list
))
52 list_add_tail(&entity
->list
, &rq
->entities
);
53 spin_unlock(&rq
->lock
);
56 static void amd_sched_rq_remove_entity(struct amd_sched_rq
*rq
,
57 struct amd_sched_entity
*entity
)
59 if (list_empty(&entity
->list
))
62 list_del_init(&entity
->list
);
63 if (rq
->current_entity
== entity
)
64 rq
->current_entity
= NULL
;
65 spin_unlock(&rq
->lock
);
69 * Select an entity which could provide a job to run
71 * @rq The run queue to check.
73 * Try to find a ready entity, returns NULL if none found.
75 static struct amd_sched_entity
*
76 amd_sched_rq_select_entity(struct amd_sched_rq
*rq
)
78 struct amd_sched_entity
*entity
;
82 entity
= rq
->current_entity
;
84 list_for_each_entry_continue(entity
, &rq
->entities
, list
) {
85 if (amd_sched_entity_is_ready(entity
)) {
86 rq
->current_entity
= entity
;
87 spin_unlock(&rq
->lock
);
93 list_for_each_entry(entity
, &rq
->entities
, list
) {
95 if (amd_sched_entity_is_ready(entity
)) {
96 rq
->current_entity
= entity
;
97 spin_unlock(&rq
->lock
);
101 if (entity
== rq
->current_entity
)
105 spin_unlock(&rq
->lock
);
111 * Init a context entity used by scheduler when submit to HW ring.
113 * @sched The pointer to the scheduler
114 * @entity The pointer to a valid amd_sched_entity
115 * @rq The run queue this entity belongs
116 * @kernel If this is an entity for the kernel
117 * @jobs The max number of jobs in the job queue
119 * return 0 if succeed. negative error code on failure
121 int amd_sched_entity_init(struct amd_gpu_scheduler
*sched
,
122 struct amd_sched_entity
*entity
,
123 struct amd_sched_rq
*rq
,
128 if (!(sched
&& entity
&& rq
))
131 memset(entity
, 0, sizeof(struct amd_sched_entity
));
132 INIT_LIST_HEAD(&entity
->list
);
134 entity
->sched
= sched
;
136 spin_lock_init(&entity
->queue_lock
);
137 r
= kfifo_alloc(&entity
->job_queue
, jobs
* sizeof(void *), GFP_KERNEL
);
141 atomic_set(&entity
->fence_seq
, 0);
142 entity
->fence_context
= dma_fence_context_alloc(2);
148 * Query if entity is initialized
150 * @sched Pointer to scheduler instance
151 * @entity The pointer to a valid scheduler entity
153 * return true if entity is initialized, false otherwise
155 static bool amd_sched_entity_is_initialized(struct amd_gpu_scheduler
*sched
,
156 struct amd_sched_entity
*entity
)
158 return entity
->sched
== sched
&&
163 * Check if entity is idle
165 * @entity The pointer to a valid scheduler entity
167 * Return true if entity don't has any unscheduled jobs.
169 static bool amd_sched_entity_is_idle(struct amd_sched_entity
*entity
)
172 if (kfifo_is_empty(&entity
->job_queue
))
179 * Check if entity is ready
181 * @entity The pointer to a valid scheduler entity
183 * Return true if entity could provide a job.
185 static bool amd_sched_entity_is_ready(struct amd_sched_entity
*entity
)
187 if (kfifo_is_empty(&entity
->job_queue
))
190 if (ACCESS_ONCE(entity
->dependency
))
197 * Destroy a context entity
199 * @sched Pointer to scheduler instance
200 * @entity The pointer to a valid scheduler entity
202 * Cleanup and free the allocated resources.
204 void amd_sched_entity_fini(struct amd_gpu_scheduler
*sched
,
205 struct amd_sched_entity
*entity
)
207 struct amd_sched_rq
*rq
= entity
->rq
;
210 if (!amd_sched_entity_is_initialized(sched
, entity
))
213 * The client will not queue more IBs during this fini, consume existing
214 * queued IBs or discard them on SIGKILL
216 if ((current
->flags
& PF_SIGNALED
) && current
->exit_code
== SIGKILL
)
219 r
= wait_event_killable(sched
->job_scheduled
,
220 amd_sched_entity_is_idle(entity
));
221 amd_sched_rq_remove_entity(rq
, entity
);
223 struct amd_sched_job
*job
;
225 /* Park the kernel for a moment to make sure it isn't processing
228 kthread_park(sched
->thread
);
229 kthread_unpark(sched
->thread
);
230 while (kfifo_out(&entity
->job_queue
, &job
, sizeof(job
)))
231 sched
->ops
->free_job(job
);
234 kfifo_free(&entity
->job_queue
);
237 static void amd_sched_entity_wakeup(struct dma_fence
*f
, struct dma_fence_cb
*cb
)
239 struct amd_sched_entity
*entity
=
240 container_of(cb
, struct amd_sched_entity
, cb
);
241 entity
->dependency
= NULL
;
243 amd_sched_wakeup(entity
->sched
);
246 static void amd_sched_entity_clear_dep(struct dma_fence
*f
, struct dma_fence_cb
*cb
)
248 struct amd_sched_entity
*entity
=
249 container_of(cb
, struct amd_sched_entity
, cb
);
250 entity
->dependency
= NULL
;
254 bool amd_sched_dependency_optimized(struct dma_fence
* fence
,
255 struct amd_sched_entity
*entity
)
257 struct amd_gpu_scheduler
*sched
= entity
->sched
;
258 struct amd_sched_fence
*s_fence
;
260 if (!fence
|| dma_fence_is_signaled(fence
))
262 if (fence
->context
== entity
->fence_context
)
264 s_fence
= to_amd_sched_fence(fence
);
265 if (s_fence
&& s_fence
->sched
== sched
)
271 static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity
*entity
)
273 struct amd_gpu_scheduler
*sched
= entity
->sched
;
274 struct dma_fence
* fence
= entity
->dependency
;
275 struct amd_sched_fence
*s_fence
;
277 if (fence
->context
== entity
->fence_context
) {
278 /* We can ignore fences from ourself */
279 dma_fence_put(entity
->dependency
);
283 s_fence
= to_amd_sched_fence(fence
);
284 if (s_fence
&& s_fence
->sched
== sched
) {
287 * Fence is from the same scheduler, only need to wait for
290 fence
= dma_fence_get(&s_fence
->scheduled
);
291 dma_fence_put(entity
->dependency
);
292 entity
->dependency
= fence
;
293 if (!dma_fence_add_callback(fence
, &entity
->cb
,
294 amd_sched_entity_clear_dep
))
297 /* Ignore it when it is already scheduled */
298 dma_fence_put(fence
);
302 if (!dma_fence_add_callback(entity
->dependency
, &entity
->cb
,
303 amd_sched_entity_wakeup
))
306 dma_fence_put(entity
->dependency
);
310 static struct amd_sched_job
*
311 amd_sched_entity_pop_job(struct amd_sched_entity
*entity
)
313 struct amd_gpu_scheduler
*sched
= entity
->sched
;
314 struct amd_sched_job
*sched_job
;
316 if (!kfifo_out_peek(&entity
->job_queue
, &sched_job
, sizeof(sched_job
)))
319 while ((entity
->dependency
= sched
->ops
->dependency(sched_job
)))
320 if (amd_sched_entity_add_dependency_cb(entity
))
327 * Helper to submit a job to the job queue
329 * @sched_job The pointer to job required to submit
331 * Returns true if we could submit the job.
333 static bool amd_sched_entity_in(struct amd_sched_job
*sched_job
)
335 struct amd_gpu_scheduler
*sched
= sched_job
->sched
;
336 struct amd_sched_entity
*entity
= sched_job
->s_entity
;
337 bool added
, first
= false;
339 spin_lock(&entity
->queue_lock
);
340 added
= kfifo_in(&entity
->job_queue
, &sched_job
,
341 sizeof(sched_job
)) == sizeof(sched_job
);
343 if (added
&& kfifo_len(&entity
->job_queue
) == sizeof(sched_job
))
346 spin_unlock(&entity
->queue_lock
);
348 /* first job wakes up scheduler */
350 /* Add the entity to the run queue */
351 amd_sched_rq_add_entity(entity
->rq
, entity
);
352 amd_sched_wakeup(sched
);
357 /* job_finish is called after hw fence signaled, and
358 * the job had already been deleted from ring_mirror_list
360 static void amd_sched_job_finish(struct work_struct
*work
)
362 struct amd_sched_job
*s_job
= container_of(work
, struct amd_sched_job
,
364 struct amd_gpu_scheduler
*sched
= s_job
->sched
;
366 /* remove job from ring_mirror_list */
367 spin_lock(&sched
->job_list_lock
);
368 list_del_init(&s_job
->node
);
369 if (sched
->timeout
!= MAX_SCHEDULE_TIMEOUT
) {
370 struct amd_sched_job
*next
;
372 spin_unlock(&sched
->job_list_lock
);
373 cancel_delayed_work_sync(&s_job
->work_tdr
);
374 spin_lock(&sched
->job_list_lock
);
376 /* queue TDR for next job */
377 next
= list_first_entry_or_null(&sched
->ring_mirror_list
,
378 struct amd_sched_job
, node
);
381 schedule_delayed_work(&next
->work_tdr
, sched
->timeout
);
383 spin_unlock(&sched
->job_list_lock
);
384 sched
->ops
->free_job(s_job
);
387 static void amd_sched_job_finish_cb(struct dma_fence
*f
,
388 struct dma_fence_cb
*cb
)
390 struct amd_sched_job
*job
= container_of(cb
, struct amd_sched_job
,
392 schedule_work(&job
->finish_work
);
395 static void amd_sched_job_begin(struct amd_sched_job
*s_job
)
397 struct amd_gpu_scheduler
*sched
= s_job
->sched
;
399 spin_lock(&sched
->job_list_lock
);
400 list_add_tail(&s_job
->node
, &sched
->ring_mirror_list
);
401 if (sched
->timeout
!= MAX_SCHEDULE_TIMEOUT
&&
402 list_first_entry_or_null(&sched
->ring_mirror_list
,
403 struct amd_sched_job
, node
) == s_job
)
404 schedule_delayed_work(&s_job
->work_tdr
, sched
->timeout
);
405 spin_unlock(&sched
->job_list_lock
);
408 static void amd_sched_job_timedout(struct work_struct
*work
)
410 struct amd_sched_job
*job
= container_of(work
, struct amd_sched_job
,
413 job
->sched
->ops
->timedout_job(job
);
416 void amd_sched_hw_job_reset(struct amd_gpu_scheduler
*sched
)
418 struct amd_sched_job
*s_job
;
420 spin_lock(&sched
->job_list_lock
);
421 list_for_each_entry_reverse(s_job
, &sched
->ring_mirror_list
, node
) {
422 if (s_job
->s_fence
->parent
&&
423 dma_fence_remove_callback(s_job
->s_fence
->parent
,
424 &s_job
->s_fence
->cb
)) {
425 dma_fence_put(s_job
->s_fence
->parent
);
426 s_job
->s_fence
->parent
= NULL
;
427 atomic_dec(&sched
->hw_rq_count
);
430 spin_unlock(&sched
->job_list_lock
);
433 void amd_sched_job_kickout(struct amd_sched_job
*s_job
)
435 struct amd_gpu_scheduler
*sched
= s_job
->sched
;
437 spin_lock(&sched
->job_list_lock
);
438 list_del_init(&s_job
->node
);
439 spin_unlock(&sched
->job_list_lock
);
442 void amd_sched_job_recovery(struct amd_gpu_scheduler
*sched
)
444 struct amd_sched_job
*s_job
, *tmp
;
447 spin_lock(&sched
->job_list_lock
);
448 s_job
= list_first_entry_or_null(&sched
->ring_mirror_list
,
449 struct amd_sched_job
, node
);
450 if (s_job
&& sched
->timeout
!= MAX_SCHEDULE_TIMEOUT
)
451 schedule_delayed_work(&s_job
->work_tdr
, sched
->timeout
);
453 list_for_each_entry_safe(s_job
, tmp
, &sched
->ring_mirror_list
, node
) {
454 struct amd_sched_fence
*s_fence
= s_job
->s_fence
;
455 struct dma_fence
*fence
;
457 spin_unlock(&sched
->job_list_lock
);
458 fence
= sched
->ops
->run_job(s_job
);
459 atomic_inc(&sched
->hw_rq_count
);
461 s_fence
->parent
= dma_fence_get(fence
);
462 r
= dma_fence_add_callback(fence
, &s_fence
->cb
,
463 amd_sched_process_job
);
465 amd_sched_process_job(fence
, &s_fence
->cb
);
467 DRM_ERROR("fence add callback failed (%d)\n",
469 dma_fence_put(fence
);
471 DRM_ERROR("Failed to run job!\n");
472 amd_sched_process_job(NULL
, &s_fence
->cb
);
474 spin_lock(&sched
->job_list_lock
);
476 spin_unlock(&sched
->job_list_lock
);
480 * Submit a job to the job queue
482 * @sched_job The pointer to job required to submit
484 * Returns 0 for success, negative error code otherwise.
486 void amd_sched_entity_push_job(struct amd_sched_job
*sched_job
)
488 struct amd_sched_entity
*entity
= sched_job
->s_entity
;
490 trace_amd_sched_job(sched_job
);
491 dma_fence_add_callback(&sched_job
->s_fence
->finished
, &sched_job
->finish_cb
,
492 amd_sched_job_finish_cb
);
493 wait_event(entity
->sched
->job_scheduled
,
494 amd_sched_entity_in(sched_job
));
497 /* init a sched_job with basic field */
498 int amd_sched_job_init(struct amd_sched_job
*job
,
499 struct amd_gpu_scheduler
*sched
,
500 struct amd_sched_entity
*entity
,
504 job
->s_entity
= entity
;
505 job
->s_fence
= amd_sched_fence_create(entity
, owner
);
508 job
->id
= atomic64_inc_return(&sched
->job_id_count
);
510 INIT_WORK(&job
->finish_work
, amd_sched_job_finish
);
511 INIT_LIST_HEAD(&job
->node
);
512 INIT_DELAYED_WORK(&job
->work_tdr
, amd_sched_job_timedout
);
518 * Return ture if we can push more jobs to the hw.
520 static bool amd_sched_ready(struct amd_gpu_scheduler
*sched
)
522 return atomic_read(&sched
->hw_rq_count
) <
523 sched
->hw_submission_limit
;
527 * Wake up the scheduler when it is ready
529 static void amd_sched_wakeup(struct amd_gpu_scheduler
*sched
)
531 if (amd_sched_ready(sched
))
532 wake_up_interruptible(&sched
->wake_up_worker
);
536 * Select next entity to process
538 static struct amd_sched_entity
*
539 amd_sched_select_entity(struct amd_gpu_scheduler
*sched
)
541 struct amd_sched_entity
*entity
;
544 if (!amd_sched_ready(sched
))
547 /* Kernel run queue has higher priority than normal run queue*/
548 for (i
= AMD_SCHED_PRIORITY_MAX
- 1; i
>= AMD_SCHED_PRIORITY_MIN
; i
--) {
549 entity
= amd_sched_rq_select_entity(&sched
->sched_rq
[i
]);
557 static void amd_sched_process_job(struct dma_fence
*f
, struct dma_fence_cb
*cb
)
559 struct amd_sched_fence
*s_fence
=
560 container_of(cb
, struct amd_sched_fence
, cb
);
561 struct amd_gpu_scheduler
*sched
= s_fence
->sched
;
563 atomic_dec(&sched
->hw_rq_count
);
564 amd_sched_fence_finished(s_fence
);
566 trace_amd_sched_process_job(s_fence
);
567 dma_fence_put(&s_fence
->finished
);
568 wake_up_interruptible(&sched
->wake_up_worker
);
571 static bool amd_sched_blocked(struct amd_gpu_scheduler
*sched
)
573 if (kthread_should_park()) {
581 static int amd_sched_main(void *param
)
583 struct sched_param sparam
= {.sched_priority
= 1};
584 struct amd_gpu_scheduler
*sched
= (struct amd_gpu_scheduler
*)param
;
587 sched_setscheduler(current
, SCHED_FIFO
, &sparam
);
589 while (!kthread_should_stop()) {
590 struct amd_sched_entity
*entity
= NULL
;
591 struct amd_sched_fence
*s_fence
;
592 struct amd_sched_job
*sched_job
;
593 struct dma_fence
*fence
;
595 wait_event_interruptible(sched
->wake_up_worker
,
596 (!amd_sched_blocked(sched
) &&
597 (entity
= amd_sched_select_entity(sched
))) ||
598 kthread_should_stop());
603 sched_job
= amd_sched_entity_pop_job(entity
);
607 s_fence
= sched_job
->s_fence
;
609 atomic_inc(&sched
->hw_rq_count
);
610 amd_sched_job_begin(sched_job
);
612 fence
= sched
->ops
->run_job(sched_job
);
613 amd_sched_fence_scheduled(s_fence
);
615 s_fence
->parent
= dma_fence_get(fence
);
616 r
= dma_fence_add_callback(fence
, &s_fence
->cb
,
617 amd_sched_process_job
);
619 amd_sched_process_job(fence
, &s_fence
->cb
);
621 DRM_ERROR("fence add callback failed (%d)\n",
623 dma_fence_put(fence
);
625 DRM_ERROR("Failed to run job!\n");
626 amd_sched_process_job(NULL
, &s_fence
->cb
);
629 count
= kfifo_out(&entity
->job_queue
, &sched_job
,
631 WARN_ON(count
!= sizeof(sched_job
));
632 wake_up(&sched
->job_scheduled
);
638 * Init a gpu scheduler instance
640 * @sched The pointer to the scheduler
641 * @ops The backend operations for this scheduler.
642 * @hw_submissions Number of hw submissions to do.
643 * @name Name used for debugging
645 * Return 0 on success, otherwise error code.
647 int amd_sched_init(struct amd_gpu_scheduler
*sched
,
648 const struct amd_sched_backend_ops
*ops
,
649 unsigned hw_submission
, long timeout
, const char *name
)
653 sched
->hw_submission_limit
= hw_submission
;
655 sched
->timeout
= timeout
;
656 for (i
= AMD_SCHED_PRIORITY_MIN
; i
< AMD_SCHED_PRIORITY_MAX
; i
++)
657 amd_sched_rq_init(&sched
->sched_rq
[i
]);
659 init_waitqueue_head(&sched
->wake_up_worker
);
660 init_waitqueue_head(&sched
->job_scheduled
);
661 INIT_LIST_HEAD(&sched
->ring_mirror_list
);
662 spin_lock_init(&sched
->job_list_lock
);
663 atomic_set(&sched
->hw_rq_count
, 0);
664 atomic64_set(&sched
->job_id_count
, 0);
666 /* Each scheduler will run on a seperate kernel thread */
667 sched
->thread
= kthread_run(amd_sched_main
, sched
, sched
->name
);
668 if (IS_ERR(sched
->thread
)) {
669 DRM_ERROR("Failed to create scheduler for %s.\n", name
);
670 return PTR_ERR(sched
->thread
);
677 * Destroy a gpu scheduler
679 * @sched The pointer to the scheduler
681 void amd_sched_fini(struct amd_gpu_scheduler
*sched
)
684 kthread_stop(sched
->thread
);