drivers/gpu/drm/amd/amdkfd/kfd_process.c

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 #include <linux/mutex.h>
  24 #include <linux/log2.h>
  25 #include <linux/sched.h>
  26 #include <linux/sched/mm.h>
  27 #include <linux/slab.h>
  28 #include <linux/amd-iommu.h>
  29 #include <linux/notifier.h>
  30 #include <linux/compat.h>
  31
  32 struct mm_struct;
  33
  34 #include "kfd_priv.h"
  35 #include "kfd_dbgmgr.h"
  36
  37 /*
  38  * Initial size for the array of queues.
  39  * The allocated size is doubled each time
  40  * it is exceeded up to MAX_PROCESS_QUEUES.
  41  */
  42 #define INITIAL_QUEUE_ARRAY_SIZE 16
  43
  44 /*
  45  * List of struct kfd_process (field kfd_process).
  46  * Unique/indexed by mm_struct*
  47  */
  48 #define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
  49 static DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
  50 static DEFINE_MUTEX(kfd_processes_mutex);
  51
  52 DEFINE_STATIC_SRCU(kfd_processes_srcu);
  53
  54 static struct workqueue_struct *kfd_process_wq;
  55
  56 struct kfd_process_release_work {
  57         struct work_struct kfd_work;
  58         struct kfd_process *p;
  59 };
  60
  61 static struct kfd_process *find_process(const struct task_struct *thread);
  62 static struct kfd_process *create_process(const struct task_struct *thread);
  63
  64 void kfd_process_create_wq(void)
  65 {
  66         if (!kfd_process_wq)
  67                 kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);
  68 }
  69
  70 void kfd_process_destroy_wq(void)
  71 {
  72         if (kfd_process_wq) {
  73                 destroy_workqueue(kfd_process_wq);
  74                 kfd_process_wq = NULL;
  75         }
  76 }
  77
  78 struct kfd_process *kfd_create_process(const struct task_struct *thread)
  79 {
  80         struct kfd_process *process;
  81
  82         BUG_ON(!kfd_process_wq);
  83
  84         if (thread->mm == NULL)
  85                 return ERR_PTR(-EINVAL);
  86
  87         /* Only the pthreads threading model is supported. */
  88         if (thread->group_leader->mm != thread->mm)
  89                 return ERR_PTR(-EINVAL);
  90
  91         /* Take mmap_sem because we call __mmu_notifier_register inside */
  92         down_write(&thread->mm->mmap_sem);
  93
  94         /*
  95          * take kfd processes mutex before starting of process creation
  96          * so there won't be a case where two threads of the same process
  97          * create two kfd_process structures
  98          */
  99         mutex_lock(&kfd_processes_mutex);
 100
 101         /* A prior open of /dev/kfd could have already created the process. */
 102         process = find_process(thread);
 103         if (process)
 104                 pr_debug("kfd: process already found\n");
 105
 106         if (!process)
 107                 process = create_process(thread);
 108
 109         mutex_unlock(&kfd_processes_mutex);
 110
 111         up_write(&thread->mm->mmap_sem);
 112
 113         return process;
 114 }
 115
 116 struct kfd_process *kfd_get_process(const struct task_struct *thread)
 117 {
 118         struct kfd_process *process;
 119
 120         if (thread->mm == NULL)
 121                 return ERR_PTR(-EINVAL);
 122
 123         /* Only the pthreads threading model is supported. */
 124         if (thread->group_leader->mm != thread->mm)
 125                 return ERR_PTR(-EINVAL);
 126
 127         process = find_process(thread);
 128
 129         return process;
 130 }
 131
 132 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
 133 {
 134         struct kfd_process *process;
 135
 136         hash_for_each_possible_rcu(kfd_processes_table, process,
 137                                         kfd_processes, (uintptr_t)mm)
 138                 if (process->mm == mm)
 139                         return process;
 140
 141         return NULL;
 142 }
 143
 144 static struct kfd_process *find_process(const struct task_struct *thread)
 145 {
 146         struct kfd_process *p;
 147         int idx;
 148
 149         idx = srcu_read_lock(&kfd_processes_srcu);
 150         p = find_process_by_mm(thread->mm);
 151         srcu_read_unlock(&kfd_processes_srcu, idx);
 152
 153         return p;
 154 }
 155
 156 static void kfd_process_wq_release(struct work_struct *work)
 157 {
 158         struct kfd_process_release_work *my_work;
 159         struct kfd_process_device *pdd, *temp;
 160         struct kfd_process *p;
 161
 162         my_work = (struct kfd_process_release_work *) work;
 163
 164         p = my_work->p;
 165
 166         pr_debug("Releasing process (pasid %d) in workqueue\n",
 167                         p->pasid);
 168
 169         mutex_lock(&p->mutex);
 170
 171         list_for_each_entry_safe(pdd, temp, &p->per_device_data,
 172                                                         per_device_list) {
 173                 pr_debug("Releasing pdd (topology id %d) for process (pasid %d) in workqueue\n",
 174                                 pdd->dev->id, p->pasid);
 175
 176                 if (pdd->reset_wavefronts)
 177                         dbgdev_wave_reset_wavefronts(pdd->dev, p);
 178
 179                 amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
 180                 list_del(&pdd->per_device_list);
 181
 182                 kfree(pdd);
 183         }
 184
 185         kfd_event_free_process(p);
 186
 187         kfd_pasid_free(p->pasid);
 188
 189         mutex_unlock(&p->mutex);
 190
 191         mutex_destroy(&p->mutex);
 192
 193         kfree(p->queues);
 194
 195         kfree(p);
 196
 197         kfree(work);
 198 }
 199
 200 static void kfd_process_destroy_delayed(struct rcu_head *rcu)
 201 {
 202         struct kfd_process_release_work *work;
 203         struct kfd_process *p;
 204
 205         BUG_ON(!kfd_process_wq);
 206
 207         p = container_of(rcu, struct kfd_process, rcu);
 208         BUG_ON(atomic_read(&p->mm->mm_count) <= 0);
 209
 210         mmdrop(p->mm);
 211
 212         work = kmalloc(sizeof(struct kfd_process_release_work), GFP_ATOMIC);
 213
 214         if (work) {
 215                 INIT_WORK((struct work_struct *) work, kfd_process_wq_release);
 216                 work->p = p;
 217                 queue_work(kfd_process_wq, (struct work_struct *) work);
 218         }
 219 }
 220
 221 static void kfd_process_notifier_release(struct mmu_notifier *mn,
 222                                         struct mm_struct *mm)
 223 {
 224         struct kfd_process *p;
 225         struct kfd_process_device *pdd = NULL;
 226
 227         /*
 228          * The kfd_process structure can not be free because the
 229          * mmu_notifier srcu is read locked
 230          */
 231         p = container_of(mn, struct kfd_process, mmu_notifier);
 232         BUG_ON(p->mm != mm);
 233
 234         mutex_lock(&kfd_processes_mutex);
 235         hash_del_rcu(&p->kfd_processes);
 236         mutex_unlock(&kfd_processes_mutex);
 237         synchronize_srcu(&kfd_processes_srcu);
 238
 239         mutex_lock(&p->mutex);
 240
 241         /* In case our notifier is called before IOMMU notifier */
 242         pqm_uninit(&p->pqm);
 243
 244         /* Iterate over all process device data structure and check
 245          * if we should delete debug managers and reset all wavefronts
 246          */
 247         list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
 248                 if ((pdd->dev->dbgmgr) &&
 249                                 (pdd->dev->dbgmgr->pasid == p->pasid))
 250                         kfd_dbgmgr_destroy(pdd->dev->dbgmgr);
 251
 252                 if (pdd->reset_wavefronts) {
 253                         pr_warn("amdkfd: Resetting all wave fronts\n");
 254                         dbgdev_wave_reset_wavefronts(pdd->dev, p);
 255                         pdd->reset_wavefronts = false;
 256                 }
 257         }
 258
 259         mutex_unlock(&p->mutex);
 260
 261         /*
 262          * Because we drop mm_count inside kfd_process_destroy_delayed
 263          * and because the mmu_notifier_unregister function also drop
 264          * mm_count we need to take an extra count here.
 265          */
 266         mmgrab(p->mm);
 267         mmu_notifier_unregister_no_release(&p->mmu_notifier, p->mm);
 268         mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed);
 269 }
 270
 271 static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
 272         .release = kfd_process_notifier_release,
 273 };
 274
 275 static struct kfd_process *create_process(const struct task_struct *thread)
 276 {
 277         struct kfd_process *process;
 278         int err = -ENOMEM;
 279
 280         process = kzalloc(sizeof(*process), GFP_KERNEL);
 281
 282         if (!process)
 283                 goto err_alloc_process;
 284
 285         process->queues = kmalloc_array(INITIAL_QUEUE_ARRAY_SIZE,
 286                                         sizeof(process->queues[0]), GFP_KERNEL);
 287         if (!process->queues)
 288                 goto err_alloc_queues;
 289
 290         process->pasid = kfd_pasid_alloc();
 291         if (process->pasid == 0)
 292                 goto err_alloc_pasid;
 293
 294         mutex_init(&process->mutex);
 295
 296         process->mm = thread->mm;
 297
 298         /* register notifier */
 299         process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
 300         err = __mmu_notifier_register(&process->mmu_notifier, process->mm);
 301         if (err)
 302                 goto err_mmu_notifier;
 303
 304         hash_add_rcu(kfd_processes_table, &process->kfd_processes,
 305                         (uintptr_t)process->mm);
 306
 307         process->lead_thread = thread->group_leader;
 308
 309         process->queue_array_size = INITIAL_QUEUE_ARRAY_SIZE;
 310
 311         INIT_LIST_HEAD(&process->per_device_data);
 312
 313         kfd_event_init_process(process);
 314
 315         err = pqm_init(&process->pqm, process);
 316         if (err != 0)
 317                 goto err_process_pqm_init;
 318
 319         /* init process apertures*/
 320         process->is_32bit_user_mode = in_compat_syscall();
 321         if (kfd_init_apertures(process) != 0)
 322                 goto err_init_apretures;
 323
 324         return process;
 325
 326 err_init_apretures:
 327         pqm_uninit(&process->pqm);
 328 err_process_pqm_init:
 329         hash_del_rcu(&process->kfd_processes);
 330         synchronize_rcu();
 331         mmu_notifier_unregister_no_release(&process->mmu_notifier, process->mm);
 332 err_mmu_notifier:
 333         mutex_destroy(&process->mutex);
 334         kfd_pasid_free(process->pasid);
 335 err_alloc_pasid:
 336         kfree(process->queues);
 337 err_alloc_queues:
 338         kfree(process);
 339 err_alloc_process:
 340         return ERR_PTR(err);
 341 }
 342
 343 struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
 344                                                         struct kfd_process *p)
 345 {
 346         struct kfd_process_device *pdd = NULL;
 347
 348         list_for_each_entry(pdd, &p->per_device_data, per_device_list)
 349                 if (pdd->dev == dev)
 350                         break;
 351
 352         return pdd;
 353 }
 354
 355 struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
 356                                                         struct kfd_process *p)
 357 {
 358         struct kfd_process_device *pdd = NULL;
 359
 360         pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
 361         if (pdd != NULL) {
 362                 pdd->dev = dev;
 363                 INIT_LIST_HEAD(&pdd->qpd.queues_list);
 364                 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
 365                 pdd->qpd.dqm = dev->dqm;
 366                 pdd->reset_wavefronts = false;
 367                 list_add(&pdd->per_device_list, &p->per_device_data);
 368         }
 369
 370         return pdd;
 371 }
 372
 373 /*
 374  * Direct the IOMMU to bind the process (specifically the pasid->mm)
 375  * to the device.
 376  * Unbinding occurs when the process dies or the device is removed.
 377  *
 378  * Assumes that the process lock is held.
 379  */
 380 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
 381                                                         struct kfd_process *p)
 382 {
 383         struct kfd_process_device *pdd;
 384         int err;
 385
 386         pdd = kfd_get_process_device_data(dev, p);
 387         if (!pdd) {
 388                 pr_err("Process device data doesn't exist\n");
 389                 return ERR_PTR(-ENOMEM);
 390         }
 391
 392         if (pdd->bound)
 393                 return pdd;
 394
 395         err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
 396         if (err < 0)
 397                 return ERR_PTR(err);
 398
 399         pdd->bound = true;
 400
 401         return pdd;
 402 }
 403
 404 void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
 405 {
 406         struct kfd_process *p;
 407         struct kfd_process_device *pdd;
 408
 409         BUG_ON(dev == NULL);
 410
 411         /*
 412          * Look for the process that matches the pasid. If there is no such
 413          * process, we either released it in amdkfd's own notifier, or there
 414          * is a bug. Unfortunately, there is no way to tell...
 415          */
 416         p = kfd_lookup_process_by_pasid(pasid);
 417         if (!p)
 418                 return;
 419
 420         pr_debug("Unbinding process %d from IOMMU\n", pasid);
 421
 422         if ((dev->dbgmgr) && (dev->dbgmgr->pasid == p->pasid))
 423                 kfd_dbgmgr_destroy(dev->dbgmgr);
 424
 425         pqm_uninit(&p->pqm);
 426
 427         pdd = kfd_get_process_device_data(dev, p);
 428
 429         if (!pdd) {
 430                 mutex_unlock(&p->mutex);
 431                 return;
 432         }
 433
 434         if (pdd->reset_wavefronts) {
 435                 dbgdev_wave_reset_wavefronts(pdd->dev, p);
 436                 pdd->reset_wavefronts = false;
 437         }
 438
 439         /*
 440          * Just mark pdd as unbound, because we still need it
 441          * to call amd_iommu_unbind_pasid() in when the
 442          * process exits.
 443          * We don't call amd_iommu_unbind_pasid() here
 444          * because the IOMMU called us.
 445          */
 446         pdd->bound = false;
 447
 448         mutex_unlock(&p->mutex);
 449 }
 450
 451 struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p)
 452 {
 453         return list_first_entry(&p->per_device_data,
 454                                 struct kfd_process_device,
 455                                 per_device_list);
 456 }
 457
 458 struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p,
 459                                                 struct kfd_process_device *pdd)
 460 {
 461         if (list_is_last(&pdd->per_device_list, &p->per_device_data))
 462                 return NULL;
 463         return list_next_entry(pdd, per_device_list);
 464 }
 465
 466 bool kfd_has_process_device_data(struct kfd_process *p)
 467 {
 468         return !(list_empty(&p->per_device_data));
 469 }
 470
 471 /* This returns with process->mutex locked. */
 472 struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid)
 473 {
 474         struct kfd_process *p;
 475         unsigned int temp;
 476
 477         int idx = srcu_read_lock(&kfd_processes_srcu);
 478
 479         hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
 480                 if (p->pasid == pasid) {
 481                         mutex_lock(&p->mutex);
 482                         break;
 483                 }
 484         }
 485
 486         srcu_read_unlock(&kfd_processes_srcu, idx);
 487
 488         return p;
 489 }