[mirror_ubuntu-focal-kernel.git] / drivers / gpu / drm / amd / scheduler / gpu_scheduler.c

/*
 * Copyright 2015 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 *
 */
#include <linux/kthread.h>
#include <linux/wait.h>
#include <linux/sched.h>
#include <drm/drmP.h>
#include "gpu_scheduler.h"

/* Initialize a given run queue struct */
static void amd_sched_rq_init(struct amd_sched_rq *rq)
{
	spin_lock_init(&rq->lock);
	INIT_LIST_HEAD(&rq->entities);
	rq->current_entity = NULL;
}

static void amd_sched_rq_add_entity(struct amd_sched_rq *rq,
				    struct amd_sched_entity *entity)
{
	spin_lock(&rq->lock);
	list_add_tail(&entity->list, &rq->entities);
	spin_unlock(&rq->lock);
}

static void amd_sched_rq_remove_entity(struct amd_sched_rq *rq,
				       struct amd_sched_entity *entity)
{
	spin_lock(&rq->lock);
	list_del_init(&entity->list);
	if (rq->current_entity == entity)
		rq->current_entity = NULL;
	spin_unlock(&rq->lock);
}

/**
 * Select next entity from a specified run queue with round robin policy.
 * It could return the same entity as current one if current is the only
 * available one in the queue. Return NULL if nothing available.
 */
static struct amd_sched_entity *
amd_sched_rq_select_entity(struct amd_sched_rq *rq)
{
	struct amd_sched_entity *entity;

	spin_lock(&rq->lock);

	entity = rq->current_entity;
	if (entity) {
		list_for_each_entry_continue(entity, &rq->entities, list) {
			if (!kfifo_is_empty(&entity->job_queue)) {
				rq->current_entity = entity;
				spin_unlock(&rq->lock);
				return rq->current_entity;
			}
		}
	}

	list_for_each_entry(entity, &rq->entities, list) {

		if (!kfifo_is_empty(&entity->job_queue)) {
			rq->current_entity = entity;
			spin_unlock(&rq->lock);
			return rq->current_entity;
		}

		if (entity == rq->current_entity)
			break;
	}

	spin_unlock(&rq->lock);

	return NULL;
}

/**
 * Return ture if we can push more jobs to the hw.
 */
static bool amd_sched_ready(struct amd_gpu_scheduler *sched)
{
	return atomic_read(&sched->hw_rq_count) <
		sched->hw_submission_limit;
}

/**
 * Select next entity containing real IB submissions
*/
static struct amd_sched_entity *
amd_sched_select_context(struct amd_gpu_scheduler *sched)
{
	struct amd_sched_entity *wake_entity = NULL;
	struct amd_sched_entity *tmp;

	if (!amd_sched_ready(sched))
		return NULL;

	/* Kernel run queue has higher priority than normal run queue*/
	tmp = amd_sched_rq_select_entity(&sched->kernel_rq);
	if (tmp == NULL)
		tmp = amd_sched_rq_select_entity(&sched->sched_rq);

	if (sched->current_entity && (sched->current_entity != tmp))
		wake_entity = sched->current_entity;
	sched->current_entity = tmp;
	if (wake_entity && wake_entity->need_wakeup)
		wake_up(&wake_entity->wait_queue);
	return tmp;
}

/**
 * Init a context entity used by scheduler when submit to HW ring.
 *
 * @sched	The pointer to the scheduler
 * @entity	The pointer to a valid amd_sched_entity
 * @rq		The run queue this entity belongs
 * @kernel	If this is an entity for the kernel
 * @jobs	The max number of jobs in the job queue
 *
 * return 0 if succeed. negative error code on failure
*/
int amd_sched_entity_init(struct amd_gpu_scheduler *sched,
			  struct amd_sched_entity *entity,
			  struct amd_sched_rq *rq,
			  uint32_t jobs)
{
	char name[20];

	if (!(sched && entity && rq))
		return -EINVAL;

	memset(entity, 0, sizeof(struct amd_sched_entity));
	entity->belongto_rq = rq;
	entity->scheduler = sched;
	init_waitqueue_head(&entity->wait_queue);
	entity->fence_context = fence_context_alloc(1);
	snprintf(name, sizeof(name), "c_entity[%llu]", entity->fence_context);
	memcpy(entity->name, name, 20);
	entity->need_wakeup = false;
	if(kfifo_alloc(&entity->job_queue,
		       jobs * sizeof(void *),
		       GFP_KERNEL))
		return -EINVAL;

	spin_lock_init(&entity->queue_lock);
	atomic_set(&entity->fence_seq, 0);

	/* Add the entity to the run queue */
	amd_sched_rq_add_entity(rq, entity);
	return 0;
}

/**
 * Query if entity is initialized
 *
 * @sched       Pointer to scheduler instance
 * @entity	The pointer to a valid scheduler entity
 *
 * return true if entity is initialized, false otherwise
*/
static bool is_context_entity_initialized(struct amd_gpu_scheduler *sched,
					  struct amd_sched_entity *entity)
{
	return entity->scheduler == sched &&
		entity->belongto_rq != NULL;
}

static bool is_context_entity_idle(struct amd_gpu_scheduler *sched,
				   struct amd_sched_entity *entity)
{
	/**
	 * Idle means no pending IBs, and the entity is not
	 * currently being used.
	*/
	barrier();
	if ((sched->current_entity != entity) &&
	    kfifo_is_empty(&entity->job_queue))
		return true;

	return false;
}

/**
 * Destroy a context entity
 *
 * @sched       Pointer to scheduler instance
 * @entity	The pointer to a valid scheduler entity
 *
 * return 0 if succeed. negative error code on failure
 */
int amd_sched_entity_fini(struct amd_gpu_scheduler *sched,
			    struct amd_sched_entity *entity)
{
	int r = 0;
	struct amd_sched_rq *rq = entity->belongto_rq;

	if (!is_context_entity_initialized(sched, entity))
		return 0;
	entity->need_wakeup = true;
	/**
	 * The client will not queue more IBs during this fini, consume existing
	 * queued IBs
	*/
	r = wait_event_timeout(
		entity->wait_queue,
		is_context_entity_idle(sched, entity),
		msecs_to_jiffies(AMD_GPU_WAIT_IDLE_TIMEOUT_IN_MS)
		) ?  0 : -1;

	if (r)
		DRM_INFO("Entity %p is in waiting state during fini\n",
			 entity);

	amd_sched_rq_remove_entity(rq, entity);
	kfifo_free(&entity->job_queue);
	return r;
}

/**
 * Submit a normal job to the job queue
 *
 * @sched	The pointer to the scheduler
 * @c_entity    The pointer to amd_sched_entity
 * @job		The pointer to job required to submit
 * return 0 if succeed. -1 if failed.
 *        -2 indicate queue is full for this client, client should wait untill
 *	     scheduler consum some queued command.
 *	  -1 other fail.
*/
int amd_sched_push_job(struct amd_sched_job *sched_job)
{
	struct amd_sched_fence 	*fence =
		amd_sched_fence_create(sched_job->s_entity);
	if (!fence)
		return -EINVAL;
	fence_get(&fence->base);
	sched_job->s_fence = fence;
	while (kfifo_in_spinlocked(&sched_job->s_entity->job_queue,
				   &sched_job, sizeof(void *),
				   &sched_job->s_entity->queue_lock) !=
	       sizeof(void *)) {
		/**
		 * Current context used up all its IB slots
		 * wait here, or need to check whether GPU is hung
		*/
		schedule();
	}
	/* first job wake up scheduler */
	if ((kfifo_len(&sched_job->s_entity->job_queue) / sizeof(void *)) == 1)
		wake_up_interruptible(&sched_job->sched->wait_queue);
	return 0;
}

static void amd_sched_process_job(struct fence *f, struct fence_cb *cb)
{
	struct amd_sched_job *sched_job =
		container_of(cb, struct amd_sched_job, cb);
	struct amd_gpu_scheduler *sched;

	sched = sched_job->sched;
	amd_sched_fence_signal(sched_job->s_fence);
	atomic_dec(&sched->hw_rq_count);
	fence_put(&sched_job->s_fence->base);
	sched->ops->process_job(sched, sched_job);
	wake_up_interruptible(&sched->wait_queue);
}

static int amd_sched_main(void *param)
{
	struct sched_param sparam = {.sched_priority = 1};
	struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param;
	int r;

	sched_setscheduler(current, SCHED_FIFO, &sparam);

	while (!kthread_should_stop()) {
		struct amd_sched_entity *c_entity = NULL;
		struct amd_sched_job *job;
		struct fence *fence;

		wait_event_interruptible(sched->wait_queue,
			kthread_should_stop() ||
			(c_entity = amd_sched_select_context(sched)));

		if (!c_entity)
			continue;

		r = kfifo_out(&c_entity->job_queue, &job, sizeof(void *));
		if (r != sizeof(void *))
			continue;
		r = 0;
		if (sched->ops->prepare_job)
			r = sched->ops->prepare_job(sched, c_entity, job);
		if (!r) {
			atomic_inc(&sched->hw_rq_count);
		}
		mutex_lock(&sched->sched_lock);
		fence = sched->ops->run_job(sched, c_entity, job);
		if (fence) {
			r = fence_add_callback(fence, &job->cb,
					       amd_sched_process_job);
			if (r == -ENOENT)
				amd_sched_process_job(fence, &job->cb);
			else if (r)
				DRM_ERROR("fence add callback failed (%d)\n", r);
			fence_put(fence);
		}
		mutex_unlock(&sched->sched_lock);
	}
	return 0;
}

/**
 * Create a gpu scheduler
 *
 * @device	The device context for this scheduler
 * @ops		The backend operations for this scheduler.
 * @id	        The scheduler is per ring, here is ring id.
 * @granularity	The minumum ms unit the scheduler will scheduled.
 * @preemption  Indicate whether this ring support preemption, 0 is no.
 *
 * return the pointer to scheduler for success, otherwise return NULL
*/
struct amd_gpu_scheduler *amd_sched_create(void *device,
					   struct amd_sched_backend_ops *ops,
					   unsigned ring,
					   unsigned granularity,
					   unsigned preemption,
					   unsigned hw_submission)
{
	struct amd_gpu_scheduler *sched;
	char name[20];

	sched = kzalloc(sizeof(struct amd_gpu_scheduler), GFP_KERNEL);
	if (!sched)
		return NULL;

	sched->device = device;
	sched->ops = ops;
	sched->granularity = granularity;
	sched->ring_id = ring;
	sched->preemption = preemption;
	sched->hw_submission_limit = hw_submission;
	snprintf(name, sizeof(name), "gpu_sched[%d]", ring);
	mutex_init(&sched->sched_lock);
	amd_sched_rq_init(&sched->sched_rq);
	amd_sched_rq_init(&sched->kernel_rq);

	init_waitqueue_head(&sched->wait_queue);
	atomic_set(&sched->hw_rq_count, 0);
	/* Each scheduler will run on a seperate kernel thread */
	sched->thread = kthread_create(amd_sched_main, sched, name);
	if (sched->thread) {
		wake_up_process(sched->thread);
		return sched;
	}

	DRM_ERROR("Failed to create scheduler for id %d.\n", ring);
	kfree(sched);
	return NULL;
}

/**
 * Destroy a gpu scheduler
 *
 * @sched	The pointer to the scheduler
 *
 * return 0 if succeed. -1 if failed.
 */
int amd_sched_destroy(struct amd_gpu_scheduler *sched)
{
	kthread_stop(sched->thread);
	kfree(sched);
	return  0;
}
Commit	Line	Data
a72ce6f8 JZ	1	/*
	2	* Copyright 2015 Advanced Micro Devices, Inc.
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice shall be included in
	12	* all copies or substantial portions of the Software.
	13	*
	14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	20	* OTHER DEALINGS IN THE SOFTWARE.
	21	*
	22	*
	23	*/
	24	#include <linux/kthread.h>
	25	#include <linux/wait.h>
	26	#include <linux/sched.h>
	27	#include <drm/drmP.h>
	28	#include "gpu_scheduler.h"
	29
	30	/* Initialize a given run queue struct */
432a4ff8	31	static void amd_sched_rq_init(struct amd_sched_rq *rq)
a72ce6f8	32	{
2b184d8d	33	spin_lock_init(&rq->lock);
432a4ff8	34	INIT_LIST_HEAD(&rq->entities);
432a4ff8	35	rq->current_entity = NULL;
a72ce6f8 JZ	36	}
a72ce6f8 JZ	37
432a4ff8 CK	38	static void amd_sched_rq_add_entity(struct amd_sched_rq *rq,
432a4ff8 CK	39	struct amd_sched_entity *entity)
a72ce6f8	40	{
2b184d8d	41	spin_lock(&rq->lock);
432a4ff8	42	list_add_tail(&entity->list, &rq->entities);
2b184d8d	43	spin_unlock(&rq->lock);
a72ce6f8 JZ	44	}
a72ce6f8 JZ	45
432a4ff8 CK	46	static void amd_sched_rq_remove_entity(struct amd_sched_rq *rq,
432a4ff8 CK	47	struct amd_sched_entity *entity)
a72ce6f8	48	{
2b184d8d	49	spin_lock(&rq->lock);
432a4ff8 CK	50	list_del_init(&entity->list);
	51	if (rq->current_entity == entity)
	52	rq->current_entity = NULL;
2b184d8d	53	spin_unlock(&rq->lock);
a72ce6f8 JZ	54	}
	55
	56	/**
	57	* Select next entity from a specified run queue with round robin policy.
	58	* It could return the same entity as current one if current is the only
	59	* available one in the queue. Return NULL if nothing available.
	60	*/
432a4ff8 CK	61	static struct amd_sched_entity *
432a4ff8 CK	62	amd_sched_rq_select_entity(struct amd_sched_rq *rq)
a72ce6f8	63	{
2b184d8d	64	struct amd_sched_entity *entity;
432a4ff8	65
2b184d8d CK	66	spin_lock(&rq->lock);
	67
	68	entity = rq->current_entity;
432a4ff8 CK	69	if (entity) {
	70	list_for_each_entry_continue(entity, &rq->entities, list) {
	71	if (!kfifo_is_empty(&entity->job_queue)) {
	72	rq->current_entity = entity;
2b184d8d	73	spin_unlock(&rq->lock);
432a4ff8 CK	74	return rq->current_entity;
432a4ff8 CK	75	}
a72ce6f8	76	}
a72ce6f8	77	}
a72ce6f8	78
432a4ff8	79	list_for_each_entry(entity, &rq->entities, list) {
a72ce6f8	80
432a4ff8 CK	81	if (!kfifo_is_empty(&entity->job_queue)) {
432a4ff8 CK	82	rq->current_entity = entity;
2b184d8d	83	spin_unlock(&rq->lock);
432a4ff8 CK	84	return rq->current_entity;
432a4ff8 CK	85	}
a72ce6f8	86
432a4ff8 CK	87	if (entity == rq->current_entity)
	88	break;
	89	}
a72ce6f8	90
2b184d8d CK	91	spin_unlock(&rq->lock);
2b184d8d CK	92
432a4ff8	93	return NULL;
a72ce6f8 JZ	94	}
	95
	96	/**
c746ba22 CK	97	* Return ture if we can push more jobs to the hw.
	98	*/
	99	static bool amd_sched_ready(struct amd_gpu_scheduler *sched)
a72ce6f8	100	{
c746ba22 CK	101	return atomic_read(&sched->hw_rq_count) <
c746ba22 CK	102	sched->hw_submission_limit;
a72ce6f8 JZ	103	}
a72ce6f8 JZ	104
a72ce6f8 JZ	105	/**
	106	* Select next entity containing real IB submissions
	107	*/
91404fb2	108	static struct amd_sched_entity *
f85a6dd9	109	amd_sched_select_context(struct amd_gpu_scheduler *sched)
a72ce6f8	110	{
91404fb2 CK	111	struct amd_sched_entity *wake_entity = NULL;
91404fb2 CK	112	struct amd_sched_entity *tmp;
a72ce6f8	113
c746ba22	114	if (!amd_sched_ready(sched))
a72ce6f8 JZ	115	return NULL;
	116
	117	/* Kernel run queue has higher priority than normal run queue*/
2b184d8d CK	118	tmp = amd_sched_rq_select_entity(&sched->kernel_rq);
	119	if (tmp == NULL)
	120	tmp = amd_sched_rq_select_entity(&sched->sched_rq);
	121
a72ce6f8 JZ	122	if (sched->current_entity && (sched->current_entity != tmp))
	123	wake_entity = sched->current_entity;
	124	sched->current_entity = tmp;
1c8f805a	125	if (wake_entity && wake_entity->need_wakeup)
a72ce6f8 JZ	126	wake_up(&wake_entity->wait_queue);
	127	return tmp;
	128	}
	129
	130	/**
	131	* Init a context entity used by scheduler when submit to HW ring.
	132	*
	133	* @sched The pointer to the scheduler
91404fb2	134	* @entity The pointer to a valid amd_sched_entity
a72ce6f8	135	* @rq The run queue this entity belongs
0e89d0c1	136	* @kernel If this is an entity for the kernel
1333f723	137	* @jobs The max number of jobs in the job queue
a72ce6f8 JZ	138	*
	139	* return 0 if succeed. negative error code on failure
	140	*/
91404fb2	141	int amd_sched_entity_init(struct amd_gpu_scheduler *sched,
6f0e54a9	142	struct amd_sched_entity *entity,
432a4ff8	143	struct amd_sched_rq *rq,
6f0e54a9	144	uint32_t jobs)
a72ce6f8	145	{
f556cb0c	146	char name[20];
a72ce6f8 JZ	147
	148	if (!(sched && entity && rq))
	149	return -EINVAL;
	150
91404fb2	151	memset(entity, 0, sizeof(struct amd_sched_entity));
91404fb2	152	entity->belongto_rq = rq;
a72ce6f8 JZ	153	entity->scheduler = sched;
a72ce6f8 JZ	154	init_waitqueue_head(&entity->wait_queue);
f556cb0c CZ	155	entity->fence_context = fence_context_alloc(1);
	156	snprintf(name, sizeof(name), "c_entity[%llu]", entity->fence_context);
	157	memcpy(entity->name, name, 20);
1c8f805a	158	entity->need_wakeup = false;
a72ce6f8	159	if(kfifo_alloc(&entity->job_queue,
1333f723	160	jobs * sizeof(void *),
a72ce6f8 JZ	161	GFP_KERNEL))
	162	return -EINVAL;
	163
	164	spin_lock_init(&entity->queue_lock);
ce882e6d	165	atomic_set(&entity->fence_seq, 0);
a72ce6f8 JZ	166
a72ce6f8 JZ	167	/* Add the entity to the run queue */
432a4ff8	168	amd_sched_rq_add_entity(rq, entity);
a72ce6f8 JZ	169	return 0;
	170	}
	171
	172	/**
	173	* Query if entity is initialized
	174	*
	175	* @sched Pointer to scheduler instance
	176	* @entity The pointer to a valid scheduler entity
	177	*
	178	* return true if entity is initialized, false otherwise
	179	*/
	180	static bool is_context_entity_initialized(struct amd_gpu_scheduler *sched,
91404fb2	181	struct amd_sched_entity *entity)
a72ce6f8 JZ	182	{
a72ce6f8 JZ	183	return entity->scheduler == sched &&
91404fb2	184	entity->belongto_rq != NULL;
a72ce6f8 JZ	185	}
	186
	187	static bool is_context_entity_idle(struct amd_gpu_scheduler *sched,
91404fb2	188	struct amd_sched_entity *entity)
a72ce6f8 JZ	189	{
	190	/**
	191	* Idle means no pending IBs, and the entity is not
	192	* currently being used.
	193	*/
	194	barrier();
	195	if ((sched->current_entity != entity) &&
	196	kfifo_is_empty(&entity->job_queue))
	197	return true;
	198
	199	return false;
	200	}
	201
	202	/**
	203	* Destroy a context entity
	204	*
	205	* @sched Pointer to scheduler instance
	206	* @entity The pointer to a valid scheduler entity
	207	*
	208	* return 0 if succeed. negative error code on failure
	209	*/
91404fb2 CK	210	int amd_sched_entity_fini(struct amd_gpu_scheduler *sched,
91404fb2 CK	211	struct amd_sched_entity *entity)
a72ce6f8 JZ	212	{
a72ce6f8 JZ	213	int r = 0;
432a4ff8	214	struct amd_sched_rq *rq = entity->belongto_rq;
a72ce6f8 JZ	215
	216	if (!is_context_entity_initialized(sched, entity))
	217	return 0;
1c8f805a	218	entity->need_wakeup = true;
a72ce6f8 JZ	219	/**
	220	* The client will not queue more IBs during this fini, consume existing
	221	* queued IBs
	222	*/
	223	r = wait_event_timeout(
	224	entity->wait_queue,
	225	is_context_entity_idle(sched, entity),
	226	msecs_to_jiffies(AMD_GPU_WAIT_IDLE_TIMEOUT_IN_MS)
	227	) ? 0 : -1;
	228
9788ec40 CK	229	if (r)
	230	DRM_INFO("Entity %p is in waiting state during fini\n",
	231	entity);
a72ce6f8	232
432a4ff8	233	amd_sched_rq_remove_entity(rq, entity);
a72ce6f8 JZ	234	kfifo_free(&entity->job_queue);
	235	return r;
	236	}
	237
	238	/**
	239	* Submit a normal job to the job queue
	240	*
	241	* @sched The pointer to the scheduler
91404fb2	242	* @c_entity The pointer to amd_sched_entity
a72ce6f8	243	* @job The pointer to job required to submit
80de5913 CZ	244	* return 0 if succeed. -1 if failed.
	245	* -2 indicate queue is full for this client, client should wait untill
	246	* scheduler consum some queued command.
	247	* -1 other fail.
a72ce6f8	248	*/
bb977d37	249	int amd_sched_push_job(struct amd_sched_job *sched_job)
a72ce6f8	250	{
bb977d37 CZ	251	struct amd_sched_fence *fence =
bb977d37 CZ	252	amd_sched_fence_create(sched_job->s_entity);
f556cb0c CZ	253	if (!fence)
f556cb0c CZ	254	return -EINVAL;
bb977d37 CZ	255	fence_get(&fence->base);
	256	sched_job->s_fence = fence;
	257	while (kfifo_in_spinlocked(&sched_job->s_entity->job_queue,
	258	&sched_job, sizeof(void *),
	259	&sched_job->s_entity->queue_lock) !=
	260	sizeof(void *)) {
a72ce6f8 JZ	261	/**
	262	* Current context used up all its IB slots
	263	* wait here, or need to check whether GPU is hung
	264	*/
	265	schedule();
	266	}
1c8f805a	267	/* first job wake up scheduler */
bb977d37 CZ	268	if ((kfifo_len(&sched_job->s_entity->job_queue) / sizeof(void *)) == 1)
bb977d37 CZ	269	wake_up_interruptible(&sched_job->sched->wait_queue);
80de5913	270	return 0;
a72ce6f8 JZ	271	}
a72ce6f8 JZ	272
6f0e54a9 CK	273	static void amd_sched_process_job(struct fence f, struct fence_cb cb)
	274	{
	275	struct amd_sched_job *sched_job =
	276	container_of(cb, struct amd_sched_job, cb);
	277	struct amd_gpu_scheduler *sched;
6f0e54a9 CK	278
6f0e54a9 CK	279	sched = sched_job->sched;
f556cb0c	280	amd_sched_fence_signal(sched_job->s_fence);
c746ba22	281	atomic_dec(&sched->hw_rq_count);
f556cb0c	282	fence_put(&sched_job->s_fence->base);
bb977d37	283	sched->ops->process_job(sched, sched_job);
6f0e54a9 CK	284	wake_up_interruptible(&sched->wait_queue);
	285	}
	286
a72ce6f8 JZ	287	static int amd_sched_main(void *param)
a72ce6f8 JZ	288	{
a72ce6f8	289	struct sched_param sparam = {.sched_priority = 1};
a72ce6f8	290	struct amd_gpu_scheduler sched = (struct amd_gpu_scheduler )param;
f85a6dd9	291	int r;
a72ce6f8 JZ	292
	293	sched_setscheduler(current, SCHED_FIFO, &sparam);
	294
	295	while (!kthread_should_stop()) {
f85a6dd9 CK	296	struct amd_sched_entity *c_entity = NULL;
f85a6dd9 CK	297	struct amd_sched_job *job;
6f0e54a9 CK	298	struct fence *fence;
6f0e54a9 CK	299
a72ce6f8	300	wait_event_interruptible(sched->wait_queue,
f85a6dd9 CK	301	kthread_should_stop() \|\|
	302	(c_entity = amd_sched_select_context(sched)));
	303
	304	if (!c_entity)
	305	continue;
	306
a72ce6f8 JZ	307	r = kfifo_out(&c_entity->job_queue, &job, sizeof(void *));
	308	if (r != sizeof(void *))
	309	continue;
bb977d37 CZ	310	r = 0;
	311	if (sched->ops->prepare_job)
	312	r = sched->ops->prepare_job(sched, c_entity, job);
4cef9267	313	if (!r) {
c746ba22	314	atomic_inc(&sched->hw_rq_count);
4cef9267	315	}
a72ce6f8	316	mutex_lock(&sched->sched_lock);
953e8fd4	317	fence = sched->ops->run_job(sched, c_entity, job);
6f0e54a9	318	if (fence) {
953e8fd4	319	r = fence_add_callback(fence, &job->cb,
6f0e54a9 CK	320	amd_sched_process_job);
6f0e54a9 CK	321	if (r == -ENOENT)
953e8fd4	322	amd_sched_process_job(fence, &job->cb);
6f0e54a9 CK	323	else if (r)
	324	DRM_ERROR("fence add callback failed (%d)\n", r);
	325	fence_put(fence);
	326	}
a72ce6f8 JZ	327	mutex_unlock(&sched->sched_lock);
	328	}
	329	return 0;
	330	}
	331
a72ce6f8 JZ	332	/**
	333	* Create a gpu scheduler
	334	*
	335	* @device The device context for this scheduler
	336	* @ops The backend operations for this scheduler.
	337	* @id The scheduler is per ring, here is ring id.
	338	* @granularity The minumum ms unit the scheduler will scheduled.
	339	* @preemption Indicate whether this ring support preemption, 0 is no.
	340	*
	341	* return the pointer to scheduler for success, otherwise return NULL
	342	*/
	343	struct amd_gpu_scheduler amd_sched_create(void device,
	344	struct amd_sched_backend_ops *ops,
	345	unsigned ring,
	346	unsigned granularity,
4afcb303 JZ	347	unsigned preemption,
4afcb303 JZ	348	unsigned hw_submission)
a72ce6f8 JZ	349	{
a72ce6f8 JZ	350	struct amd_gpu_scheduler *sched;
4cd7f42c	351	char name[20];
a72ce6f8 JZ	352
	353	sched = kzalloc(sizeof(struct amd_gpu_scheduler), GFP_KERNEL);
	354	if (!sched)
	355	return NULL;
	356
	357	sched->device = device;
	358	sched->ops = ops;
	359	sched->granularity = granularity;
	360	sched->ring_id = ring;
	361	sched->preemption = preemption;
4cef9267	362	sched->hw_submission_limit = hw_submission;
a72ce6f8 JZ	363	snprintf(name, sizeof(name), "gpu_sched[%d]", ring);
a72ce6f8 JZ	364	mutex_init(&sched->sched_lock);
432a4ff8 CK	365	amd_sched_rq_init(&sched->sched_rq);
432a4ff8 CK	366	amd_sched_rq_init(&sched->kernel_rq);
a72ce6f8 JZ	367
a72ce6f8 JZ	368	init_waitqueue_head(&sched->wait_queue);
c746ba22	369	atomic_set(&sched->hw_rq_count, 0);
a72ce6f8 JZ	370	/* Each scheduler will run on a seperate kernel thread */
	371	sched->thread = kthread_create(amd_sched_main, sched, name);
	372	if (sched->thread) {
	373	wake_up_process(sched->thread);
a72ce6f8 JZ	374	return sched;
	375	}
	376
	377	DRM_ERROR("Failed to create scheduler for id %d.\n", ring);
a72ce6f8 JZ	378	kfree(sched);
	379	return NULL;
	380	}
	381
	382	/**
	383	* Destroy a gpu scheduler
	384	*
	385	* @sched The pointer to the scheduler
	386	*
	387	* return 0 if succeed. -1 if failed.
	388	*/
	389	int amd_sched_destroy(struct amd_gpu_scheduler *sched)
	390	{
	391	kthread_stop(sched->thread);
a72ce6f8 JZ	392	kfree(sched);
	393	return 0;
	394	}