drivers/gpu/drm/amd/scheduler/gpu_scheduler.c

   1 /*
   2  * Copyright 2015 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  *
  23  */
  24 #include <linux/kthread.h>
  25 #include <linux/wait.h>
  26 #include <linux/sched.h>
  27 #include <drm/drmP.h>
  28 #include "gpu_scheduler.h"
  29
  30 #define CREATE_TRACE_POINTS
  31 #include "gpu_sched_trace.h"
  32
  33 static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity);
  34 static void amd_sched_wakeup(struct amd_gpu_scheduler *sched);
  35
  36 struct kmem_cache *sched_fence_slab;
  37 atomic_t sched_fence_slab_ref = ATOMIC_INIT(0);
  38
  39 /* Initialize a given run queue struct */
  40 static void amd_sched_rq_init(struct amd_sched_rq *rq)
  41 {
  42         spin_lock_init(&rq->lock);
  43         INIT_LIST_HEAD(&rq->entities);
  44         rq->current_entity = NULL;
  45 }
  46
  47 static void amd_sched_rq_add_entity(struct amd_sched_rq *rq,
  48                                     struct amd_sched_entity *entity)
  49 {
  50         spin_lock(&rq->lock);
  51         list_add_tail(&entity->list, &rq->entities);
  52         spin_unlock(&rq->lock);
  53 }
  54
  55 static void amd_sched_rq_remove_entity(struct amd_sched_rq *rq,
  56                                        struct amd_sched_entity *entity)
  57 {
  58         spin_lock(&rq->lock);
  59         list_del_init(&entity->list);
  60         if (rq->current_entity == entity)
  61                 rq->current_entity = NULL;
  62         spin_unlock(&rq->lock);
  63 }
  64
  65 /**
  66  * Select an entity which could provide a job to run
  67  *
  68  * @rq          The run queue to check.
  69  *
  70  * Try to find a ready entity, returns NULL if none found.
  71  */
  72 static struct amd_sched_entity *
  73 amd_sched_rq_select_entity(struct amd_sched_rq *rq)
  74 {
  75         struct amd_sched_entity *entity;
  76
  77         spin_lock(&rq->lock);
  78
  79         entity = rq->current_entity;
  80         if (entity) {
  81                 list_for_each_entry_continue(entity, &rq->entities, list) {
  82                         if (amd_sched_entity_is_ready(entity)) {
  83                                 rq->current_entity = entity;
  84                                 spin_unlock(&rq->lock);
  85                                 return entity;
  86                         }
  87                 }
  88         }
  89
  90         list_for_each_entry(entity, &rq->entities, list) {
  91
  92                 if (amd_sched_entity_is_ready(entity)) {
  93                         rq->current_entity = entity;
  94                         spin_unlock(&rq->lock);
  95                         return entity;
  96                 }
  97
  98                 if (entity == rq->current_entity)
  99                         break;
 100         }
 101
 102         spin_unlock(&rq->lock);
 103
 104         return NULL;
 105 }
 106
 107 /**
 108  * Init a context entity used by scheduler when submit to HW ring.
 109  *
 110  * @sched       The pointer to the scheduler
 111  * @entity      The pointer to a valid amd_sched_entity
 112  * @rq          The run queue this entity belongs
 113  * @kernel      If this is an entity for the kernel
 114  * @jobs        The max number of jobs in the job queue
 115  *
 116  * return 0 if succeed. negative error code on failure
 117 */
 118 int amd_sched_entity_init(struct amd_gpu_scheduler *sched,
 119                           struct amd_sched_entity *entity,
 120                           struct amd_sched_rq *rq,
 121                           uint32_t jobs)
 122 {
 123         int r;
 124
 125         if (!(sched && entity && rq))
 126                 return -EINVAL;
 127
 128         memset(entity, 0, sizeof(struct amd_sched_entity));
 129         INIT_LIST_HEAD(&entity->list);
 130         entity->rq = rq;
 131         entity->sched = sched;
 132
 133         spin_lock_init(&entity->queue_lock);
 134         r = kfifo_alloc(&entity->job_queue, jobs * sizeof(void *), GFP_KERNEL);
 135         if (r)
 136                 return r;
 137
 138         atomic_set(&entity->fence_seq, 0);
 139         entity->fence_context = fence_context_alloc(1);
 140
 141         /* Add the entity to the run queue */
 142         amd_sched_rq_add_entity(rq, entity);
 143
 144         return 0;
 145 }
 146
 147 /**
 148  * Query if entity is initialized
 149  *
 150  * @sched       Pointer to scheduler instance
 151  * @entity      The pointer to a valid scheduler entity
 152  *
 153  * return true if entity is initialized, false otherwise
 154 */
 155 static bool amd_sched_entity_is_initialized(struct amd_gpu_scheduler *sched,
 156                                             struct amd_sched_entity *entity)
 157 {
 158         return entity->sched == sched &&
 159                 entity->rq != NULL;
 160 }
 161
 162 /**
 163  * Check if entity is idle
 164  *
 165  * @entity      The pointer to a valid scheduler entity
 166  *
 167  * Return true if entity don't has any unscheduled jobs.
 168  */
 169 static bool amd_sched_entity_is_idle(struct amd_sched_entity *entity)
 170 {
 171         rmb();
 172         if (kfifo_is_empty(&entity->job_queue))
 173                 return true;
 174
 175         return false;
 176 }
 177
 178 /**
 179  * Check if entity is ready
 180  *
 181  * @entity      The pointer to a valid scheduler entity
 182  *
 183  * Return true if entity could provide a job.
 184  */
 185 static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity)
 186 {
 187         if (kfifo_is_empty(&entity->job_queue))
 188                 return false;
 189
 190         if (ACCESS_ONCE(entity->dependency))
 191                 return false;
 192
 193         return true;
 194 }
 195
 196 /**
 197  * Destroy a context entity
 198  *
 199  * @sched       Pointer to scheduler instance
 200  * @entity      The pointer to a valid scheduler entity
 201  *
 202  * Cleanup and free the allocated resources.
 203  */
 204 void amd_sched_entity_fini(struct amd_gpu_scheduler *sched,
 205                            struct amd_sched_entity *entity)
 206 {
 207         struct amd_sched_rq *rq = entity->rq;
 208
 209         if (!amd_sched_entity_is_initialized(sched, entity))
 210                 return;
 211
 212         /**
 213          * The client will not queue more IBs during this fini, consume existing
 214          * queued IBs
 215         */
 216         wait_event(sched->job_scheduled, amd_sched_entity_is_idle(entity));
 217
 218         amd_sched_rq_remove_entity(rq, entity);
 219         kfifo_free(&entity->job_queue);
 220 }
 221
 222 static void amd_sched_entity_wakeup(struct fence *f, struct fence_cb *cb)
 223 {
 224         struct amd_sched_entity *entity =
 225                 container_of(cb, struct amd_sched_entity, cb);
 226         entity->dependency = NULL;
 227         fence_put(f);
 228         amd_sched_wakeup(entity->sched);
 229 }
 230
 231 static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity *entity)
 232 {
 233         struct amd_gpu_scheduler *sched = entity->sched;
 234         struct fence * fence = entity->dependency;
 235         struct amd_sched_fence *s_fence;
 236
 237         if (fence->context == entity->fence_context) {
 238                 /* We can ignore fences from ourself */
 239                 fence_put(entity->dependency);
 240                 return false;
 241         }
 242
 243         s_fence = to_amd_sched_fence(fence);
 244         if (s_fence && s_fence->sched == sched) {
 245                 /* Fence is from the same scheduler */
 246                 if (test_bit(AMD_SCHED_FENCE_SCHEDULED_BIT, &fence->flags)) {
 247                         /* Ignore it when it is already scheduled */
 248                         fence_put(entity->dependency);
 249                         return false;
 250                 }
 251
 252                 /* Wait for fence to be scheduled */
 253                 entity->cb.func = amd_sched_entity_wakeup;
 254                 list_add_tail(&entity->cb.node, &s_fence->scheduled_cb);
 255                 return true;
 256         }
 257
 258         if (!fence_add_callback(entity->dependency, &entity->cb,
 259                                 amd_sched_entity_wakeup))
 260                 return true;
 261
 262         fence_put(entity->dependency);
 263         return false;
 264 }
 265
 266 static struct amd_sched_job *
 267 amd_sched_entity_pop_job(struct amd_sched_entity *entity)
 268 {
 269         struct amd_gpu_scheduler *sched = entity->sched;
 270         struct amd_sched_job *sched_job;
 271
 272         if (!kfifo_out_peek(&entity->job_queue, &sched_job, sizeof(sched_job)))
 273                 return NULL;
 274
 275         while ((entity->dependency = sched->ops->dependency(sched_job)))
 276                 if (amd_sched_entity_add_dependency_cb(entity))
 277                         return NULL;
 278
 279         return sched_job;
 280 }
 281
 282 /**
 283  * Helper to submit a job to the job queue
 284  *
 285  * @sched_job           The pointer to job required to submit
 286  *
 287  * Returns true if we could submit the job.
 288  */
 289 static bool amd_sched_entity_in(struct amd_sched_job *sched_job)
 290 {
 291         struct amd_sched_entity *entity = sched_job->s_entity;
 292         bool added, first = false;
 293
 294         spin_lock(&entity->queue_lock);
 295         added = kfifo_in(&entity->job_queue, &sched_job,
 296                         sizeof(sched_job)) == sizeof(sched_job);
 297
 298         if (added && kfifo_len(&entity->job_queue) == sizeof(sched_job))
 299                 first = true;
 300
 301         spin_unlock(&entity->queue_lock);
 302
 303         /* first job wakes up scheduler */
 304         if (first)
 305                 amd_sched_wakeup(sched_job->sched);
 306
 307         return added;
 308 }
 309
 310 /**
 311  * Submit a job to the job queue
 312  *
 313  * @sched_job           The pointer to job required to submit
 314  *
 315  * Returns 0 for success, negative error code otherwise.
 316  */
 317 void amd_sched_entity_push_job(struct amd_sched_job *sched_job)
 318 {
 319         struct amd_sched_entity *entity = sched_job->s_entity;
 320
 321         wait_event(entity->sched->job_scheduled,
 322                    amd_sched_entity_in(sched_job));
 323         trace_amd_sched_job(sched_job);
 324 }
 325
 326 /**
 327  * Return ture if we can push more jobs to the hw.
 328  */
 329 static bool amd_sched_ready(struct amd_gpu_scheduler *sched)
 330 {
 331         return atomic_read(&sched->hw_rq_count) <
 332                 sched->hw_submission_limit;
 333 }
 334
 335 /**
 336  * Wake up the scheduler when it is ready
 337  */
 338 static void amd_sched_wakeup(struct amd_gpu_scheduler *sched)
 339 {
 340         if (amd_sched_ready(sched))
 341                 wake_up_interruptible(&sched->wake_up_worker);
 342 }
 343
 344 /**
 345  * Select next entity to process
 346 */
 347 static struct amd_sched_entity *
 348 amd_sched_select_entity(struct amd_gpu_scheduler *sched)
 349 {
 350         struct amd_sched_entity *entity;
 351
 352         if (!amd_sched_ready(sched))
 353                 return NULL;
 354
 355         /* Kernel run queue has higher priority than normal run queue*/
 356         entity = amd_sched_rq_select_entity(&sched->kernel_rq);
 357         if (entity == NULL)
 358                 entity = amd_sched_rq_select_entity(&sched->sched_rq);
 359
 360         return entity;
 361 }
 362
 363 static void amd_sched_process_job(struct fence *f, struct fence_cb *cb)
 364 {
 365         struct amd_sched_fence *s_fence =
 366                 container_of(cb, struct amd_sched_fence, cb);
 367         struct amd_gpu_scheduler *sched = s_fence->sched;
 368         unsigned long flags;
 369
 370         atomic_dec(&sched->hw_rq_count);
 371         amd_sched_fence_signal(s_fence);
 372         if (sched->timeout != MAX_SCHEDULE_TIMEOUT) {
 373                 cancel_delayed_work(&s_fence->dwork);
 374                 spin_lock_irqsave(&sched->fence_list_lock, flags);
 375                 list_del_init(&s_fence->list);
 376                 spin_unlock_irqrestore(&sched->fence_list_lock, flags);
 377         }
 378         trace_amd_sched_process_job(s_fence);
 379         fence_put(&s_fence->base);
 380         wake_up_interruptible(&sched->wake_up_worker);
 381 }
 382
 383 static void amd_sched_fence_work_func(struct work_struct *work)
 384 {
 385         struct amd_sched_fence *s_fence =
 386                 container_of(work, struct amd_sched_fence, dwork.work);
 387         struct amd_gpu_scheduler *sched = s_fence->sched;
 388         struct amd_sched_fence *entity, *tmp;
 389         unsigned long flags;
 390
 391         DRM_ERROR("[%s] scheduler is timeout!\n", sched->name);
 392
 393         /* Clean all pending fences */
 394         spin_lock_irqsave(&sched->fence_list_lock, flags);
 395         list_for_each_entry_safe(entity, tmp, &sched->fence_list, list) {
 396                 DRM_ERROR("  fence no %d\n", entity->base.seqno);
 397                 cancel_delayed_work(&entity->dwork);
 398                 list_del_init(&entity->list);
 399                 fence_put(&entity->base);
 400         }
 401         spin_unlock_irqrestore(&sched->fence_list_lock, flags);
 402 }
 403
 404 static int amd_sched_main(void *param)
 405 {
 406         struct sched_param sparam = {.sched_priority = 1};
 407         struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param;
 408         int r, count;
 409
 410         spin_lock_init(&sched->fence_list_lock);
 411         INIT_LIST_HEAD(&sched->fence_list);
 412         sched_setscheduler(current, SCHED_FIFO, &sparam);
 413
 414         while (!kthread_should_stop()) {
 415                 struct amd_sched_entity *entity;
 416                 struct amd_sched_fence *s_fence;
 417                 struct amd_sched_job *sched_job;
 418                 struct fence *fence;
 419                 unsigned long flags;
 420
 421                 wait_event_interruptible(sched->wake_up_worker,
 422                         (entity = amd_sched_select_entity(sched)) ||
 423                         kthread_should_stop());
 424
 425                 if (!entity)
 426                         continue;
 427
 428                 sched_job = amd_sched_entity_pop_job(entity);
 429                 if (!sched_job)
 430                         continue;
 431
 432                 s_fence = sched_job->s_fence;
 433
 434                 if (sched->timeout != MAX_SCHEDULE_TIMEOUT) {
 435                         INIT_DELAYED_WORK(&s_fence->dwork, amd_sched_fence_work_func);
 436                         schedule_delayed_work(&s_fence->dwork, sched->timeout);
 437                         spin_lock_irqsave(&sched->fence_list_lock, flags);
 438                         list_add_tail(&s_fence->list, &sched->fence_list);
 439                         spin_unlock_irqrestore(&sched->fence_list_lock, flags);
 440                 }
 441
 442                 atomic_inc(&sched->hw_rq_count);
 443                 fence = sched->ops->run_job(sched_job);
 444                 amd_sched_fence_scheduled(s_fence);
 445                 if (fence) {
 446                         r = fence_add_callback(fence, &s_fence->cb,
 447                                                amd_sched_process_job);
 448                         if (r == -ENOENT)
 449                                 amd_sched_process_job(fence, &s_fence->cb);
 450                         else if (r)
 451                                 DRM_ERROR("fence add callback failed (%d)\n", r);
 452                         fence_put(fence);
 453                 } else {
 454                         DRM_ERROR("Failed to run job!\n");
 455                         amd_sched_process_job(NULL, &s_fence->cb);
 456                 }
 457
 458                 count = kfifo_out(&entity->job_queue, &sched_job,
 459                                 sizeof(sched_job));
 460                 WARN_ON(count != sizeof(sched_job));
 461                 wake_up(&sched->job_scheduled);
 462         }
 463         return 0;
 464 }
 465
 466 /**
 467  * Init a gpu scheduler instance
 468  *
 469  * @sched               The pointer to the scheduler
 470  * @ops                 The backend operations for this scheduler.
 471  * @hw_submissions      Number of hw submissions to do.
 472  * @name                Name used for debugging
 473  *
 474  * Return 0 on success, otherwise error code.
 475 */
 476 int amd_sched_init(struct amd_gpu_scheduler *sched,
 477                    struct amd_sched_backend_ops *ops,
 478                    unsigned hw_submission, long timeout, const char *name)
 479 {
 480         sched->ops = ops;
 481         sched->hw_submission_limit = hw_submission;
 482         sched->name = name;
 483         sched->timeout = timeout;
 484         amd_sched_rq_init(&sched->sched_rq);
 485         amd_sched_rq_init(&sched->kernel_rq);
 486
 487         init_waitqueue_head(&sched->wake_up_worker);
 488         init_waitqueue_head(&sched->job_scheduled);
 489         atomic_set(&sched->hw_rq_count, 0);
 490         if (atomic_inc_return(&sched_fence_slab_ref) == 1) {
 491                 sched_fence_slab = kmem_cache_create(
 492                         "amd_sched_fence", sizeof(struct amd_sched_fence), 0,
 493                         SLAB_HWCACHE_ALIGN, NULL);
 494                 if (!sched_fence_slab)
 495                         return -ENOMEM;
 496         }
 497
 498         /* Each scheduler will run on a seperate kernel thread */
 499         sched->thread = kthread_run(amd_sched_main, sched, sched->name);
 500         if (IS_ERR(sched->thread)) {
 501                 DRM_ERROR("Failed to create scheduler for %s.\n", name);
 502                 return PTR_ERR(sched->thread);
 503         }
 504
 505         return 0;
 506 }
 507
 508 /**
 509  * Destroy a gpu scheduler
 510  *
 511  * @sched       The pointer to the scheduler
 512  */
 513 void amd_sched_fini(struct amd_gpu_scheduler *sched)
 514 {
 515         if (sched->thread)
 516                 kthread_stop(sched->thread);
 517         if (atomic_dec_and_test(&sched_fence_slab_ref))
 518                 kmem_cache_destroy(sched_fence_slab);
 519 }