]>
Commit | Line | Data |
---|---|---|
a72ce6f8 JZ |
1 | /* |
2 | * Copyright 2015 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | * | |
23 | */ | |
24 | #include <linux/kthread.h> | |
25 | #include <linux/wait.h> | |
26 | #include <linux/sched.h> | |
27 | #include <drm/drmP.h> | |
28 | #include "gpu_scheduler.h" | |
29 | ||
353da3c5 CZ |
30 | #define CREATE_TRACE_POINTS |
31 | #include "gpu_sched_trace.h" | |
32 | ||
69bd5bf1 CK |
33 | static struct amd_sched_job * |
34 | amd_sched_entity_pop_job(struct amd_sched_entity *entity); | |
88079006 CK |
35 | static void amd_sched_wakeup(struct amd_gpu_scheduler *sched); |
36 | ||
a72ce6f8 | 37 | /* Initialize a given run queue struct */ |
432a4ff8 | 38 | static void amd_sched_rq_init(struct amd_sched_rq *rq) |
a72ce6f8 | 39 | { |
2b184d8d | 40 | spin_lock_init(&rq->lock); |
432a4ff8 | 41 | INIT_LIST_HEAD(&rq->entities); |
432a4ff8 | 42 | rq->current_entity = NULL; |
a72ce6f8 JZ |
43 | } |
44 | ||
432a4ff8 CK |
45 | static void amd_sched_rq_add_entity(struct amd_sched_rq *rq, |
46 | struct amd_sched_entity *entity) | |
a72ce6f8 | 47 | { |
2b184d8d | 48 | spin_lock(&rq->lock); |
432a4ff8 | 49 | list_add_tail(&entity->list, &rq->entities); |
2b184d8d | 50 | spin_unlock(&rq->lock); |
a72ce6f8 JZ |
51 | } |
52 | ||
432a4ff8 CK |
53 | static void amd_sched_rq_remove_entity(struct amd_sched_rq *rq, |
54 | struct amd_sched_entity *entity) | |
a72ce6f8 | 55 | { |
2b184d8d | 56 | spin_lock(&rq->lock); |
432a4ff8 CK |
57 | list_del_init(&entity->list); |
58 | if (rq->current_entity == entity) | |
59 | rq->current_entity = NULL; | |
2b184d8d | 60 | spin_unlock(&rq->lock); |
a72ce6f8 JZ |
61 | } |
62 | ||
63 | /** | |
69bd5bf1 CK |
64 | * Select next job from a specified run queue with round robin policy. |
65 | * Return NULL if nothing available. | |
a72ce6f8 | 66 | */ |
69bd5bf1 CK |
67 | static struct amd_sched_job * |
68 | amd_sched_rq_select_job(struct amd_sched_rq *rq) | |
a72ce6f8 | 69 | { |
2b184d8d | 70 | struct amd_sched_entity *entity; |
69bd5bf1 | 71 | struct amd_sched_job *job; |
432a4ff8 | 72 | |
2b184d8d CK |
73 | spin_lock(&rq->lock); |
74 | ||
75 | entity = rq->current_entity; | |
432a4ff8 CK |
76 | if (entity) { |
77 | list_for_each_entry_continue(entity, &rq->entities, list) { | |
69bd5bf1 CK |
78 | job = amd_sched_entity_pop_job(entity); |
79 | if (job) { | |
432a4ff8 | 80 | rq->current_entity = entity; |
2b184d8d | 81 | spin_unlock(&rq->lock); |
69bd5bf1 | 82 | return job; |
432a4ff8 | 83 | } |
a72ce6f8 | 84 | } |
a72ce6f8 | 85 | } |
a72ce6f8 | 86 | |
432a4ff8 | 87 | list_for_each_entry(entity, &rq->entities, list) { |
a72ce6f8 | 88 | |
69bd5bf1 CK |
89 | job = amd_sched_entity_pop_job(entity); |
90 | if (job) { | |
432a4ff8 | 91 | rq->current_entity = entity; |
2b184d8d | 92 | spin_unlock(&rq->lock); |
69bd5bf1 | 93 | return job; |
432a4ff8 | 94 | } |
a72ce6f8 | 95 | |
432a4ff8 CK |
96 | if (entity == rq->current_entity) |
97 | break; | |
98 | } | |
a72ce6f8 | 99 | |
2b184d8d CK |
100 | spin_unlock(&rq->lock); |
101 | ||
432a4ff8 | 102 | return NULL; |
a72ce6f8 JZ |
103 | } |
104 | ||
a72ce6f8 JZ |
105 | /** |
106 | * Init a context entity used by scheduler when submit to HW ring. | |
107 | * | |
108 | * @sched The pointer to the scheduler | |
91404fb2 | 109 | * @entity The pointer to a valid amd_sched_entity |
a72ce6f8 | 110 | * @rq The run queue this entity belongs |
0e89d0c1 | 111 | * @kernel If this is an entity for the kernel |
1333f723 | 112 | * @jobs The max number of jobs in the job queue |
a72ce6f8 JZ |
113 | * |
114 | * return 0 if succeed. negative error code on failure | |
115 | */ | |
91404fb2 | 116 | int amd_sched_entity_init(struct amd_gpu_scheduler *sched, |
6f0e54a9 | 117 | struct amd_sched_entity *entity, |
432a4ff8 | 118 | struct amd_sched_rq *rq, |
6f0e54a9 | 119 | uint32_t jobs) |
a72ce6f8 | 120 | { |
a72ce6f8 JZ |
121 | if (!(sched && entity && rq)) |
122 | return -EINVAL; | |
123 | ||
91404fb2 | 124 | memset(entity, 0, sizeof(struct amd_sched_entity)); |
91404fb2 | 125 | entity->belongto_rq = rq; |
a72ce6f8 | 126 | entity->scheduler = sched; |
f556cb0c | 127 | entity->fence_context = fence_context_alloc(1); |
a72ce6f8 | 128 | if(kfifo_alloc(&entity->job_queue, |
1333f723 | 129 | jobs * sizeof(void *), |
a72ce6f8 JZ |
130 | GFP_KERNEL)) |
131 | return -EINVAL; | |
132 | ||
133 | spin_lock_init(&entity->queue_lock); | |
ce882e6d | 134 | atomic_set(&entity->fence_seq, 0); |
a72ce6f8 JZ |
135 | |
136 | /* Add the entity to the run queue */ | |
432a4ff8 | 137 | amd_sched_rq_add_entity(rq, entity); |
a72ce6f8 JZ |
138 | return 0; |
139 | } | |
140 | ||
141 | /** | |
142 | * Query if entity is initialized | |
143 | * | |
144 | * @sched Pointer to scheduler instance | |
145 | * @entity The pointer to a valid scheduler entity | |
146 | * | |
147 | * return true if entity is initialized, false otherwise | |
148 | */ | |
d54fdb94 CK |
149 | static bool amd_sched_entity_is_initialized(struct amd_gpu_scheduler *sched, |
150 | struct amd_sched_entity *entity) | |
a72ce6f8 JZ |
151 | { |
152 | return entity->scheduler == sched && | |
91404fb2 | 153 | entity->belongto_rq != NULL; |
a72ce6f8 JZ |
154 | } |
155 | ||
aef4852e CK |
156 | /** |
157 | * Check if entity is idle | |
158 | * | |
159 | * @entity The pointer to a valid scheduler entity | |
160 | * | |
161 | * Return true if entity don't has any unscheduled jobs. | |
162 | */ | |
163 | static bool amd_sched_entity_is_idle(struct amd_sched_entity *entity) | |
a72ce6f8 | 164 | { |
aef4852e CK |
165 | rmb(); |
166 | if (kfifo_is_empty(&entity->job_queue)) | |
a72ce6f8 JZ |
167 | return true; |
168 | ||
169 | return false; | |
170 | } | |
171 | ||
172 | /** | |
173 | * Destroy a context entity | |
174 | * | |
175 | * @sched Pointer to scheduler instance | |
176 | * @entity The pointer to a valid scheduler entity | |
177 | * | |
062c7fb3 | 178 | * Cleanup and free the allocated resources. |
a72ce6f8 | 179 | */ |
062c7fb3 CK |
180 | void amd_sched_entity_fini(struct amd_gpu_scheduler *sched, |
181 | struct amd_sched_entity *entity) | |
a72ce6f8 | 182 | { |
432a4ff8 | 183 | struct amd_sched_rq *rq = entity->belongto_rq; |
a72ce6f8 | 184 | |
d54fdb94 | 185 | if (!amd_sched_entity_is_initialized(sched, entity)) |
062c7fb3 | 186 | return; |
6c859274 | 187 | |
a72ce6f8 JZ |
188 | /** |
189 | * The client will not queue more IBs during this fini, consume existing | |
190 | * queued IBs | |
191 | */ | |
c2b6bd7e | 192 | wait_event(sched->job_scheduled, amd_sched_entity_is_idle(entity)); |
a72ce6f8 | 193 | |
432a4ff8 | 194 | amd_sched_rq_remove_entity(rq, entity); |
a72ce6f8 | 195 | kfifo_free(&entity->job_queue); |
a72ce6f8 JZ |
196 | } |
197 | ||
e61235db CK |
198 | static void amd_sched_entity_wakeup(struct fence *f, struct fence_cb *cb) |
199 | { | |
200 | struct amd_sched_entity *entity = | |
201 | container_of(cb, struct amd_sched_entity, cb); | |
202 | entity->dependency = NULL; | |
203 | fence_put(f); | |
204 | amd_sched_wakeup(entity->scheduler); | |
205 | } | |
206 | ||
69bd5bf1 CK |
207 | static struct amd_sched_job * |
208 | amd_sched_entity_pop_job(struct amd_sched_entity *entity) | |
209 | { | |
e61235db | 210 | struct amd_gpu_scheduler *sched = entity->scheduler; |
69bd5bf1 CK |
211 | struct amd_sched_job *job; |
212 | ||
e61235db CK |
213 | if (ACCESS_ONCE(entity->dependency)) |
214 | return NULL; | |
215 | ||
69bd5bf1 CK |
216 | if (!kfifo_out_peek(&entity->job_queue, &job, sizeof(job))) |
217 | return NULL; | |
218 | ||
e61235db CK |
219 | while ((entity->dependency = sched->ops->dependency(job))) { |
220 | ||
221 | if (fence_add_callback(entity->dependency, &entity->cb, | |
222 | amd_sched_entity_wakeup)) | |
223 | fence_put(entity->dependency); | |
224 | else | |
225 | return NULL; | |
226 | } | |
227 | ||
69bd5bf1 CK |
228 | return job; |
229 | } | |
230 | ||
a72ce6f8 | 231 | /** |
6c859274 | 232 | * Helper to submit a job to the job queue |
a72ce6f8 | 233 | * |
a72ce6f8 | 234 | * @job The pointer to job required to submit |
6c859274 CK |
235 | * |
236 | * Returns true if we could submit the job. | |
237 | */ | |
238 | static bool amd_sched_entity_in(struct amd_sched_job *job) | |
a72ce6f8 | 239 | { |
6c859274 CK |
240 | struct amd_sched_entity *entity = job->s_entity; |
241 | bool added, first = false; | |
242 | ||
243 | spin_lock(&entity->queue_lock); | |
244 | added = kfifo_in(&entity->job_queue, &job, sizeof(job)) == sizeof(job); | |
245 | ||
246 | if (added && kfifo_len(&entity->job_queue) == sizeof(job)) | |
247 | first = true; | |
248 | ||
249 | spin_unlock(&entity->queue_lock); | |
250 | ||
251 | /* first job wakes up scheduler */ | |
252 | if (first) | |
88079006 | 253 | amd_sched_wakeup(job->sched); |
6c859274 CK |
254 | |
255 | return added; | |
256 | } | |
257 | ||
258 | /** | |
259 | * Submit a job to the job queue | |
260 | * | |
261 | * @job The pointer to job required to submit | |
262 | * | |
263 | * Returns 0 for success, negative error code otherwise. | |
264 | */ | |
265 | int amd_sched_entity_push_job(struct amd_sched_job *sched_job) | |
266 | { | |
267 | struct amd_sched_entity *entity = sched_job->s_entity; | |
84f76ea6 CZ |
268 | struct amd_sched_fence *fence = amd_sched_fence_create( |
269 | entity, sched_job->owner); | |
6c859274 | 270 | |
f556cb0c | 271 | if (!fence) |
6c859274 CK |
272 | return -ENOMEM; |
273 | ||
bb977d37 CZ |
274 | fence_get(&fence->base); |
275 | sched_job->s_fence = fence; | |
6c859274 | 276 | |
c9f0fe5e CZ |
277 | wait_event(entity->scheduler->job_scheduled, |
278 | amd_sched_entity_in(sched_job)); | |
353da3c5 | 279 | trace_amd_sched_job(sched_job); |
c9f0fe5e | 280 | return 0; |
a72ce6f8 JZ |
281 | } |
282 | ||
e688b728 CK |
283 | /** |
284 | * Return ture if we can push more jobs to the hw. | |
285 | */ | |
286 | static bool amd_sched_ready(struct amd_gpu_scheduler *sched) | |
287 | { | |
288 | return atomic_read(&sched->hw_rq_count) < | |
289 | sched->hw_submission_limit; | |
290 | } | |
291 | ||
88079006 CK |
292 | /** |
293 | * Wake up the scheduler when it is ready | |
294 | */ | |
295 | static void amd_sched_wakeup(struct amd_gpu_scheduler *sched) | |
296 | { | |
297 | if (amd_sched_ready(sched)) | |
c2b6bd7e | 298 | wake_up_interruptible(&sched->wake_up_worker); |
88079006 CK |
299 | } |
300 | ||
e688b728 | 301 | /** |
69bd5bf1 | 302 | * Select next to run |
e688b728 | 303 | */ |
69bd5bf1 CK |
304 | static struct amd_sched_job * |
305 | amd_sched_select_job(struct amd_gpu_scheduler *sched) | |
e688b728 | 306 | { |
69bd5bf1 | 307 | struct amd_sched_job *job; |
e688b728 CK |
308 | |
309 | if (!amd_sched_ready(sched)) | |
310 | return NULL; | |
311 | ||
312 | /* Kernel run queue has higher priority than normal run queue*/ | |
69bd5bf1 CK |
313 | job = amd_sched_rq_select_job(&sched->kernel_rq); |
314 | if (job == NULL) | |
315 | job = amd_sched_rq_select_job(&sched->sched_rq); | |
e688b728 | 316 | |
69bd5bf1 | 317 | return job; |
e688b728 CK |
318 | } |
319 | ||
6f0e54a9 CK |
320 | static void amd_sched_process_job(struct fence *f, struct fence_cb *cb) |
321 | { | |
322 | struct amd_sched_job *sched_job = | |
323 | container_of(cb, struct amd_sched_job, cb); | |
324 | struct amd_gpu_scheduler *sched; | |
6f0e54a9 CK |
325 | |
326 | sched = sched_job->sched; | |
f556cb0c | 327 | amd_sched_fence_signal(sched_job->s_fence); |
c746ba22 | 328 | atomic_dec(&sched->hw_rq_count); |
f556cb0c | 329 | fence_put(&sched_job->s_fence->base); |
bd755d08 | 330 | sched->ops->process_job(sched_job); |
c2b6bd7e | 331 | wake_up_interruptible(&sched->wake_up_worker); |
6f0e54a9 CK |
332 | } |
333 | ||
a72ce6f8 JZ |
334 | static int amd_sched_main(void *param) |
335 | { | |
a72ce6f8 | 336 | struct sched_param sparam = {.sched_priority = 1}; |
a72ce6f8 | 337 | struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param; |
5134e999 | 338 | int r, count; |
a72ce6f8 JZ |
339 | |
340 | sched_setscheduler(current, SCHED_FIFO, &sparam); | |
341 | ||
342 | while (!kthread_should_stop()) { | |
69bd5bf1 | 343 | struct amd_sched_entity *entity; |
f85a6dd9 | 344 | struct amd_sched_job *job; |
6f0e54a9 CK |
345 | struct fence *fence; |
346 | ||
c2b6bd7e | 347 | wait_event_interruptible(sched->wake_up_worker, |
f85a6dd9 | 348 | kthread_should_stop() || |
69bd5bf1 | 349 | (job = amd_sched_select_job(sched))); |
f85a6dd9 | 350 | |
69bd5bf1 | 351 | if (!job) |
f85a6dd9 CK |
352 | continue; |
353 | ||
69bd5bf1 | 354 | entity = job->s_entity; |
b034b572 | 355 | atomic_inc(&sched->hw_rq_count); |
bd755d08 | 356 | fence = sched->ops->run_job(job); |
6f0e54a9 | 357 | if (fence) { |
953e8fd4 | 358 | r = fence_add_callback(fence, &job->cb, |
6f0e54a9 CK |
359 | amd_sched_process_job); |
360 | if (r == -ENOENT) | |
953e8fd4 | 361 | amd_sched_process_job(fence, &job->cb); |
6f0e54a9 CK |
362 | else if (r) |
363 | DRM_ERROR("fence add callback failed (%d)\n", r); | |
364 | fence_put(fence); | |
27439fca CK |
365 | } else { |
366 | DRM_ERROR("Failed to run job!\n"); | |
367 | amd_sched_process_job(NULL, &job->cb); | |
6f0e54a9 | 368 | } |
aef4852e | 369 | |
5134e999 AD |
370 | count = kfifo_out(&entity->job_queue, &job, sizeof(job)); |
371 | WARN_ON(count != sizeof(job)); | |
c2b6bd7e | 372 | wake_up(&sched->job_scheduled); |
a72ce6f8 JZ |
373 | } |
374 | return 0; | |
375 | } | |
376 | ||
a72ce6f8 JZ |
377 | /** |
378 | * Create a gpu scheduler | |
379 | * | |
69f7dd65 CK |
380 | * @ops The backend operations for this scheduler. |
381 | * @ring The the ring id for the scheduler. | |
382 | * @hw_submissions Number of hw submissions to do. | |
a72ce6f8 | 383 | * |
69f7dd65 | 384 | * Return the pointer to scheduler for success, otherwise return NULL |
a72ce6f8 | 385 | */ |
69f7dd65 | 386 | struct amd_gpu_scheduler *amd_sched_create(struct amd_sched_backend_ops *ops, |
f38fdfdd CZ |
387 | unsigned ring, unsigned hw_submission, |
388 | void *priv) | |
a72ce6f8 JZ |
389 | { |
390 | struct amd_gpu_scheduler *sched; | |
a72ce6f8 JZ |
391 | |
392 | sched = kzalloc(sizeof(struct amd_gpu_scheduler), GFP_KERNEL); | |
393 | if (!sched) | |
394 | return NULL; | |
395 | ||
a72ce6f8 | 396 | sched->ops = ops; |
a72ce6f8 | 397 | sched->ring_id = ring; |
4cef9267 | 398 | sched->hw_submission_limit = hw_submission; |
f38fdfdd | 399 | sched->priv = priv; |
c14692f0 | 400 | snprintf(sched->name, sizeof(sched->name), "amdgpu[%d]", ring); |
432a4ff8 CK |
401 | amd_sched_rq_init(&sched->sched_rq); |
402 | amd_sched_rq_init(&sched->kernel_rq); | |
a72ce6f8 | 403 | |
c2b6bd7e CK |
404 | init_waitqueue_head(&sched->wake_up_worker); |
405 | init_waitqueue_head(&sched->job_scheduled); | |
c746ba22 | 406 | atomic_set(&sched->hw_rq_count, 0); |
a72ce6f8 | 407 | /* Each scheduler will run on a seperate kernel thread */ |
c14692f0 | 408 | sched->thread = kthread_run(amd_sched_main, sched, sched->name); |
f4956598 CK |
409 | if (IS_ERR(sched->thread)) { |
410 | DRM_ERROR("Failed to create scheduler for id %d.\n", ring); | |
411 | kfree(sched); | |
412 | return NULL; | |
a72ce6f8 JZ |
413 | } |
414 | ||
f4956598 | 415 | return sched; |
a72ce6f8 JZ |
416 | } |
417 | ||
418 | /** | |
419 | * Destroy a gpu scheduler | |
420 | * | |
421 | * @sched The pointer to the scheduler | |
422 | * | |
423 | * return 0 if succeed. -1 if failed. | |
424 | */ | |
425 | int amd_sched_destroy(struct amd_gpu_scheduler *sched) | |
426 | { | |
427 | kthread_stop(sched->thread); | |
a72ce6f8 JZ |
428 | kfree(sched); |
429 | return 0; | |
430 | } |