drivers/staging/android/ion/ion.c

   1 /*
   2  *
   3  * drivers/staging/android/ion/ion.c
   4  *
   5  * Copyright (C) 2011 Google, Inc.
   6  *
   7  * This software is licensed under the terms of the GNU General Public
   8  * License version 2, as published by the Free Software Foundation, and
   9  * may be copied, distributed, and modified under those terms.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  */
  17
  18 #include <linux/device.h>
  19 #include <linux/err.h>
  20 #include <linux/file.h>
  21 #include <linux/freezer.h>
  22 #include <linux/fs.h>
  23 #include <linux/anon_inodes.h>
  24 #include <linux/kthread.h>
  25 #include <linux/list.h>
  26 #include <linux/memblock.h>
  27 #include <linux/miscdevice.h>
  28 #include <linux/export.h>
  29 #include <linux/mm.h>
  30 #include <linux/mm_types.h>
  31 #include <linux/rbtree.h>
  32 #include <linux/slab.h>
  33 #include <linux/seq_file.h>
  34 #include <linux/uaccess.h>
  35 #include <linux/vmalloc.h>
  36 #include <linux/debugfs.h>
  37 #include <linux/dma-buf.h>
  38 #include <linux/idr.h>
  39 #include <linux/sched/task.h>
  40
  41 #include "ion.h"
  42
  43 static struct ion_device *internal_dev;
  44 static int heap_id;
  45
  46 bool ion_buffer_cached(struct ion_buffer *buffer)
  47 {
  48         return !!(buffer->flags & ION_FLAG_CACHED);
  49 }
  50
  51 /* this function should only be called while dev->lock is held */
  52 static void ion_buffer_add(struct ion_device *dev,
  53                            struct ion_buffer *buffer)
  54 {
  55         struct rb_node **p = &dev->buffers.rb_node;
  56         struct rb_node *parent = NULL;
  57         struct ion_buffer *entry;
  58
  59         while (*p) {
  60                 parent = *p;
  61                 entry = rb_entry(parent, struct ion_buffer, node);
  62
  63                 if (buffer < entry) {
  64                         p = &(*p)->rb_left;
  65                 } else if (buffer > entry) {
  66                         p = &(*p)->rb_right;
  67                 } else {
  68                         pr_err("%s: buffer already found.", __func__);
  69                         BUG();
  70                 }
  71         }
  72
  73         rb_link_node(&buffer->node, parent, p);
  74         rb_insert_color(&buffer->node, &dev->buffers);
  75 }
  76
  77 /* this function should only be called while dev->lock is held */
  78 static struct ion_buffer *ion_buffer_create(struct ion_heap *heap,
  79                                             struct ion_device *dev,
  80                                             unsigned long len,
  81                                             unsigned long flags)
  82 {
  83         struct ion_buffer *buffer;
  84         int ret;
  85
  86         buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
  87         if (!buffer)
  88                 return ERR_PTR(-ENOMEM);
  89
  90         buffer->heap = heap;
  91         buffer->flags = flags;
  92
  93         ret = heap->ops->allocate(heap, buffer, len, flags);
  94
  95         if (ret) {
  96                 if (!(heap->flags & ION_HEAP_FLAG_DEFER_FREE))
  97                         goto err2;
  98
  99                 ion_heap_freelist_drain(heap, 0);
 100                 ret = heap->ops->allocate(heap, buffer, len, flags);
 101                 if (ret)
 102                         goto err2;
 103         }
 104
 105         if (!buffer->sg_table) {
 106                 WARN_ONCE(1, "This heap needs to set the sgtable");
 107                 ret = -EINVAL;
 108                 goto err1;
 109         }
 110
 111         buffer->dev = dev;
 112         buffer->size = len;
 113
 114         buffer->dev = dev;
 115         buffer->size = len;
 116         INIT_LIST_HEAD(&buffer->attachments);
 117         mutex_init(&buffer->lock);
 118         mutex_lock(&dev->buffer_lock);
 119         ion_buffer_add(dev, buffer);
 120         mutex_unlock(&dev->buffer_lock);
 121         return buffer;
 122
 123 err1:
 124         heap->ops->free(buffer);
 125 err2:
 126         kfree(buffer);
 127         return ERR_PTR(ret);
 128 }
 129
 130 void ion_buffer_destroy(struct ion_buffer *buffer)
 131 {
 132         if (WARN_ON(buffer->kmap_cnt > 0))
 133                 buffer->heap->ops->unmap_kernel(buffer->heap, buffer);
 134         buffer->heap->ops->free(buffer);
 135         kfree(buffer);
 136 }
 137
 138 static void _ion_buffer_destroy(struct ion_buffer *buffer)
 139 {
 140         struct ion_heap *heap = buffer->heap;
 141         struct ion_device *dev = buffer->dev;
 142
 143         mutex_lock(&dev->buffer_lock);
 144         rb_erase(&buffer->node, &dev->buffers);
 145         mutex_unlock(&dev->buffer_lock);
 146
 147         if (heap->flags & ION_HEAP_FLAG_DEFER_FREE)
 148                 ion_heap_freelist_add(heap, buffer);
 149         else
 150                 ion_buffer_destroy(buffer);
 151 }
 152
 153 static void *ion_buffer_kmap_get(struct ion_buffer *buffer)
 154 {
 155         void *vaddr;
 156
 157         if (buffer->kmap_cnt) {
 158                 buffer->kmap_cnt++;
 159                 return buffer->vaddr;
 160         }
 161         vaddr = buffer->heap->ops->map_kernel(buffer->heap, buffer);
 162         if (WARN_ONCE(!vaddr,
 163                       "heap->ops->map_kernel should return ERR_PTR on error"))
 164                 return ERR_PTR(-EINVAL);
 165         if (IS_ERR(vaddr))
 166                 return vaddr;
 167         buffer->vaddr = vaddr;
 168         buffer->kmap_cnt++;
 169         return vaddr;
 170 }
 171
 172 static void ion_buffer_kmap_put(struct ion_buffer *buffer)
 173 {
 174         buffer->kmap_cnt--;
 175         if (!buffer->kmap_cnt) {
 176                 buffer->heap->ops->unmap_kernel(buffer->heap, buffer);
 177                 buffer->vaddr = NULL;
 178         }
 179 }
 180
 181 static struct sg_table *dup_sg_table(struct sg_table *table)
 182 {
 183         struct sg_table *new_table;
 184         int ret, i;
 185         struct scatterlist *sg, *new_sg;
 186
 187         new_table = kzalloc(sizeof(*new_table), GFP_KERNEL);
 188         if (!new_table)
 189                 return ERR_PTR(-ENOMEM);
 190
 191         ret = sg_alloc_table(new_table, table->nents, GFP_KERNEL);
 192         if (ret) {
 193                 kfree(new_table);
 194                 return ERR_PTR(-ENOMEM);
 195         }
 196
 197         new_sg = new_table->sgl;
 198         for_each_sg(table->sgl, sg, table->nents, i) {
 199                 memcpy(new_sg, sg, sizeof(*sg));
 200                 sg->dma_address = 0;
 201                 new_sg = sg_next(new_sg);
 202         }
 203
 204         return new_table;
 205 }
 206
 207 static void free_duped_table(struct sg_table *table)
 208 {
 209         sg_free_table(table);
 210         kfree(table);
 211 }
 212
 213 struct ion_dma_buf_attachment {
 214         struct device *dev;
 215         struct sg_table *table;
 216         struct list_head list;
 217 };
 218
 219 static int ion_dma_buf_attach(struct dma_buf *dmabuf, struct device *dev,
 220                               struct dma_buf_attachment *attachment)
 221 {
 222         struct ion_dma_buf_attachment *a;
 223         struct sg_table *table;
 224         struct ion_buffer *buffer = dmabuf->priv;
 225
 226         a = kzalloc(sizeof(*a), GFP_KERNEL);
 227         if (!a)
 228                 return -ENOMEM;
 229
 230         table = dup_sg_table(buffer->sg_table);
 231         if (IS_ERR(table)) {
 232                 kfree(a);
 233                 return -ENOMEM;
 234         }
 235
 236         a->table = table;
 237         a->dev = dev;
 238         INIT_LIST_HEAD(&a->list);
 239
 240         attachment->priv = a;
 241
 242         mutex_lock(&buffer->lock);
 243         list_add(&a->list, &buffer->attachments);
 244         mutex_unlock(&buffer->lock);
 245
 246         return 0;
 247 }
 248
 249 static void ion_dma_buf_detatch(struct dma_buf *dmabuf,
 250                                 struct dma_buf_attachment *attachment)
 251 {
 252         struct ion_dma_buf_attachment *a = attachment->priv;
 253         struct ion_buffer *buffer = dmabuf->priv;
 254
 255         free_duped_table(a->table);
 256         mutex_lock(&buffer->lock);
 257         list_del(&a->list);
 258         mutex_unlock(&buffer->lock);
 259
 260         kfree(a);
 261 }
 262
 263 static struct sg_table *ion_map_dma_buf(struct dma_buf_attachment *attachment,
 264                                         enum dma_data_direction direction)
 265 {
 266         struct ion_dma_buf_attachment *a = attachment->priv;
 267         struct sg_table *table;
 268
 269         table = a->table;
 270
 271         if (!dma_map_sg(attachment->dev, table->sgl, table->nents,
 272                         direction))
 273                 return ERR_PTR(-ENOMEM);
 274
 275         return table;
 276 }
 277
 278 static void ion_unmap_dma_buf(struct dma_buf_attachment *attachment,
 279                               struct sg_table *table,
 280                               enum dma_data_direction direction)
 281 {
 282         dma_unmap_sg(attachment->dev, table->sgl, table->nents, direction);
 283 }
 284
 285 static int ion_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
 286 {
 287         struct ion_buffer *buffer = dmabuf->priv;
 288         int ret = 0;
 289
 290         if (!buffer->heap->ops->map_user) {
 291                 pr_err("%s: this heap does not define a method for mapping to userspace\n",
 292                        __func__);
 293                 return -EINVAL;
 294         }
 295
 296         if (!(buffer->flags & ION_FLAG_CACHED))
 297                 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
 298
 299         mutex_lock(&buffer->lock);
 300         /* now map it to userspace */
 301         ret = buffer->heap->ops->map_user(buffer->heap, buffer, vma);
 302         mutex_unlock(&buffer->lock);
 303
 304         if (ret)
 305                 pr_err("%s: failure mapping buffer to userspace\n",
 306                        __func__);
 307
 308         return ret;
 309 }
 310
 311 static void ion_dma_buf_release(struct dma_buf *dmabuf)
 312 {
 313         struct ion_buffer *buffer = dmabuf->priv;
 314
 315         _ion_buffer_destroy(buffer);
 316 }
 317
 318 static void *ion_dma_buf_kmap(struct dma_buf *dmabuf, unsigned long offset)
 319 {
 320         struct ion_buffer *buffer = dmabuf->priv;
 321
 322         return buffer->vaddr + offset * PAGE_SIZE;
 323 }
 324
 325 static void ion_dma_buf_kunmap(struct dma_buf *dmabuf, unsigned long offset,
 326                                void *ptr)
 327 {
 328 }
 329
 330 static int ion_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
 331                                         enum dma_data_direction direction)
 332 {
 333         struct ion_buffer *buffer = dmabuf->priv;
 334         void *vaddr;
 335         struct ion_dma_buf_attachment *a;
 336
 337         /*
 338          * TODO: Move this elsewhere because we don't always need a vaddr
 339          */
 340         if (buffer->heap->ops->map_kernel) {
 341                 mutex_lock(&buffer->lock);
 342                 vaddr = ion_buffer_kmap_get(buffer);
 343                 mutex_unlock(&buffer->lock);
 344         }
 345
 346         mutex_lock(&buffer->lock);
 347         list_for_each_entry(a, &buffer->attachments, list) {
 348                 dma_sync_sg_for_cpu(a->dev, a->table->sgl, a->table->nents,
 349                                     direction);
 350         }
 351         mutex_unlock(&buffer->lock);
 352
 353         return 0;
 354 }
 355
 356 static int ion_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
 357                                       enum dma_data_direction direction)
 358 {
 359         struct ion_buffer *buffer = dmabuf->priv;
 360         struct ion_dma_buf_attachment *a;
 361
 362         if (buffer->heap->ops->map_kernel) {
 363                 mutex_lock(&buffer->lock);
 364                 ion_buffer_kmap_put(buffer);
 365                 mutex_unlock(&buffer->lock);
 366         }
 367
 368         mutex_lock(&buffer->lock);
 369         list_for_each_entry(a, &buffer->attachments, list) {
 370                 dma_sync_sg_for_device(a->dev, a->table->sgl, a->table->nents,
 371                                        direction);
 372         }
 373         mutex_unlock(&buffer->lock);
 374
 375         return 0;
 376 }
 377
 378 static const struct dma_buf_ops dma_buf_ops = {
 379         .map_dma_buf = ion_map_dma_buf,
 380         .unmap_dma_buf = ion_unmap_dma_buf,
 381         .mmap = ion_mmap,
 382         .release = ion_dma_buf_release,
 383         .attach = ion_dma_buf_attach,
 384         .detach = ion_dma_buf_detatch,
 385         .begin_cpu_access = ion_dma_buf_begin_cpu_access,
 386         .end_cpu_access = ion_dma_buf_end_cpu_access,
 387         .map_atomic = ion_dma_buf_kmap,
 388         .unmap_atomic = ion_dma_buf_kunmap,
 389         .map = ion_dma_buf_kmap,
 390         .unmap = ion_dma_buf_kunmap,
 391 };
 392
 393 int ion_alloc(size_t len, unsigned int heap_id_mask, unsigned int flags)
 394 {
 395         struct ion_device *dev = internal_dev;
 396         struct ion_buffer *buffer = NULL;
 397         struct ion_heap *heap;
 398         DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
 399         int fd;
 400         struct dma_buf *dmabuf;
 401
 402         pr_debug("%s: len %zu heap_id_mask %u flags %x\n", __func__,
 403                  len, heap_id_mask, flags);
 404         /*
 405          * traverse the list of heaps available in this system in priority
 406          * order.  If the heap type is supported by the client, and matches the
 407          * request of the caller allocate from it.  Repeat until allocate has
 408          * succeeded or all heaps have been tried
 409          */
 410         len = PAGE_ALIGN(len);
 411
 412         if (!len)
 413                 return -EINVAL;
 414
 415         down_read(&dev->lock);
 416         plist_for_each_entry(heap, &dev->heaps, node) {
 417                 /* if the caller didn't specify this heap id */
 418                 if (!((1 << heap->id) & heap_id_mask))
 419                         continue;
 420                 buffer = ion_buffer_create(heap, dev, len, flags);
 421                 if (!IS_ERR(buffer))
 422                         break;
 423         }
 424         up_read(&dev->lock);
 425
 426         if (!buffer)
 427                 return -ENODEV;
 428
 429         if (IS_ERR(buffer))
 430                 return PTR_ERR(buffer);
 431
 432         exp_info.ops = &dma_buf_ops;
 433         exp_info.size = buffer->size;
 434         exp_info.flags = O_RDWR;
 435         exp_info.priv = buffer;
 436
 437         dmabuf = dma_buf_export(&exp_info);
 438         if (IS_ERR(dmabuf)) {
 439                 _ion_buffer_destroy(buffer);
 440                 return PTR_ERR(dmabuf);
 441         }
 442
 443         fd = dma_buf_fd(dmabuf, O_CLOEXEC);
 444         if (fd < 0)
 445                 dma_buf_put(dmabuf);
 446
 447         return fd;
 448 }
 449
 450 int ion_query_heaps(struct ion_heap_query *query)
 451 {
 452         struct ion_device *dev = internal_dev;
 453         struct ion_heap_data __user *buffer = u64_to_user_ptr(query->heaps);
 454         int ret = -EINVAL, cnt = 0, max_cnt;
 455         struct ion_heap *heap;
 456         struct ion_heap_data hdata;
 457
 458         memset(&hdata, 0, sizeof(hdata));
 459
 460         down_read(&dev->lock);
 461         if (!buffer) {
 462                 query->cnt = dev->heap_cnt;
 463                 ret = 0;
 464                 goto out;
 465         }
 466
 467         if (query->cnt <= 0)
 468                 goto out;
 469
 470         max_cnt = query->cnt;
 471
 472         plist_for_each_entry(heap, &dev->heaps, node) {
 473                 strncpy(hdata.name, heap->name, MAX_HEAP_NAME);
 474                 hdata.name[sizeof(hdata.name) - 1] = '\0';
 475                 hdata.type = heap->type;
 476                 hdata.heap_id = heap->id;
 477
 478                 if (copy_to_user(&buffer[cnt], &hdata, sizeof(hdata))) {
 479                         ret = -EFAULT;
 480                         goto out;
 481                 }
 482
 483                 cnt++;
 484                 if (cnt >= max_cnt)
 485                         break;
 486         }
 487
 488         query->cnt = cnt;
 489         ret = 0;
 490 out:
 491         up_read(&dev->lock);
 492         return ret;
 493 }
 494
 495 static const struct file_operations ion_fops = {
 496         .owner          = THIS_MODULE,
 497         .unlocked_ioctl = ion_ioctl,
 498 #ifdef CONFIG_COMPAT
 499         .compat_ioctl   = ion_ioctl,
 500 #endif
 501 };
 502
 503 static int debug_shrink_set(void *data, u64 val)
 504 {
 505         struct ion_heap *heap = data;
 506         struct shrink_control sc;
 507         int objs;
 508
 509         sc.gfp_mask = GFP_HIGHUSER;
 510         sc.nr_to_scan = val;
 511
 512         if (!val) {
 513                 objs = heap->shrinker.count_objects(&heap->shrinker, &sc);
 514                 sc.nr_to_scan = objs;
 515         }
 516
 517         heap->shrinker.scan_objects(&heap->shrinker, &sc);
 518         return 0;
 519 }
 520
 521 static int debug_shrink_get(void *data, u64 *val)
 522 {
 523         struct ion_heap *heap = data;
 524         struct shrink_control sc;
 525         int objs;
 526
 527         sc.gfp_mask = GFP_HIGHUSER;
 528         sc.nr_to_scan = 0;
 529
 530         objs = heap->shrinker.count_objects(&heap->shrinker, &sc);
 531         *val = objs;
 532         return 0;
 533 }
 534
 535 DEFINE_SIMPLE_ATTRIBUTE(debug_shrink_fops, debug_shrink_get,
 536                         debug_shrink_set, "%llu\n");
 537
 538 void ion_device_add_heap(struct ion_heap *heap)
 539 {
 540         struct dentry *debug_file;
 541         struct ion_device *dev = internal_dev;
 542
 543         if (!heap->ops->allocate || !heap->ops->free)
 544                 pr_err("%s: can not add heap with invalid ops struct.\n",
 545                        __func__);
 546
 547         spin_lock_init(&heap->free_lock);
 548         heap->free_list_size = 0;
 549
 550         if (heap->flags & ION_HEAP_FLAG_DEFER_FREE)
 551                 ion_heap_init_deferred_free(heap);
 552
 553         if ((heap->flags & ION_HEAP_FLAG_DEFER_FREE) || heap->ops->shrink)
 554                 ion_heap_init_shrinker(heap);
 555
 556         heap->dev = dev;
 557         down_write(&dev->lock);
 558         heap->id = heap_id++;
 559         /*
 560          * use negative heap->id to reverse the priority -- when traversing
 561          * the list later attempt higher id numbers first
 562          */
 563         plist_node_init(&heap->node, -heap->id);
 564         plist_add(&heap->node, &dev->heaps);
 565
 566         if (heap->shrinker.count_objects && heap->shrinker.scan_objects) {
 567                 char debug_name[64];
 568
 569                 snprintf(debug_name, 64, "%s_shrink", heap->name);
 570                 debug_file = debugfs_create_file(
 571                         debug_name, 0644, dev->debug_root, heap,
 572                         &debug_shrink_fops);
 573                 if (!debug_file) {
 574                         char buf[256], *path;
 575
 576                         path = dentry_path(dev->debug_root, buf, 256);
 577                         pr_err("Failed to create heap shrinker debugfs at %s/%s\n",
 578                                path, debug_name);
 579                 }
 580         }
 581
 582         dev->heap_cnt++;
 583         up_write(&dev->lock);
 584 }
 585 EXPORT_SYMBOL(ion_device_add_heap);
 586
 587 static int ion_device_create(void)
 588 {
 589         struct ion_device *idev;
 590         int ret;
 591
 592         idev = kzalloc(sizeof(*idev), GFP_KERNEL);
 593         if (!idev)
 594                 return -ENOMEM;
 595
 596         idev->dev.minor = MISC_DYNAMIC_MINOR;
 597         idev->dev.name = "ion";
 598         idev->dev.fops = &ion_fops;
 599         idev->dev.parent = NULL;
 600         ret = misc_register(&idev->dev);
 601         if (ret) {
 602                 pr_err("ion: failed to register misc device.\n");
 603                 kfree(idev);
 604                 return ret;
 605         }
 606
 607         idev->debug_root = debugfs_create_dir("ion", NULL);
 608         if (!idev->debug_root) {
 609                 pr_err("ion: failed to create debugfs root directory.\n");
 610                 goto debugfs_done;
 611         }
 612
 613 debugfs_done:
 614         idev->buffers = RB_ROOT;
 615         mutex_init(&idev->buffer_lock);
 616         init_rwsem(&idev->lock);
 617         plist_head_init(&idev->heaps);
 618         internal_dev = idev;
 619         return 0;
 620 }
 621 subsys_initcall(ion_device_create);