3 * drivers/staging/android/ion/ion.c
5 * Copyright (C) 2011 Google, Inc.
7 * This software is licensed under the terms of the GNU General Public
8 * License version 2, as published by the Free Software Foundation, and
9 * may be copied, distributed, and modified under those terms.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
18 #include <linux/device.h>
19 #include <linux/err.h>
20 #include <linux/file.h>
21 #include <linux/freezer.h>
23 #include <linux/anon_inodes.h>
24 #include <linux/kthread.h>
25 #include <linux/list.h>
26 #include <linux/memblock.h>
27 #include <linux/miscdevice.h>
28 #include <linux/export.h>
30 #include <linux/mm_types.h>
31 #include <linux/rbtree.h>
32 #include <linux/slab.h>
33 #include <linux/seq_file.h>
34 #include <linux/uaccess.h>
35 #include <linux/vmalloc.h>
36 #include <linux/debugfs.h>
37 #include <linux/dma-buf.h>
38 #include <linux/idr.h>
39 #include <linux/sched/task.h>
43 static struct ion_device
*internal_dev
;
46 bool ion_buffer_cached(struct ion_buffer
*buffer
)
48 return !!(buffer
->flags
& ION_FLAG_CACHED
);
51 /* this function should only be called while dev->lock is held */
52 static void ion_buffer_add(struct ion_device
*dev
,
53 struct ion_buffer
*buffer
)
55 struct rb_node
**p
= &dev
->buffers
.rb_node
;
56 struct rb_node
*parent
= NULL
;
57 struct ion_buffer
*entry
;
61 entry
= rb_entry(parent
, struct ion_buffer
, node
);
65 } else if (buffer
> entry
) {
68 pr_err("%s: buffer already found.", __func__
);
73 rb_link_node(&buffer
->node
, parent
, p
);
74 rb_insert_color(&buffer
->node
, &dev
->buffers
);
77 /* this function should only be called while dev->lock is held */
78 static struct ion_buffer
*ion_buffer_create(struct ion_heap
*heap
,
79 struct ion_device
*dev
,
83 struct ion_buffer
*buffer
;
86 buffer
= kzalloc(sizeof(*buffer
), GFP_KERNEL
);
88 return ERR_PTR(-ENOMEM
);
91 buffer
->flags
= flags
;
93 ret
= heap
->ops
->allocate(heap
, buffer
, len
, flags
);
96 if (!(heap
->flags
& ION_HEAP_FLAG_DEFER_FREE
))
99 ion_heap_freelist_drain(heap
, 0);
100 ret
= heap
->ops
->allocate(heap
, buffer
, len
, flags
);
105 if (!buffer
->sg_table
) {
106 WARN_ONCE(1, "This heap needs to set the sgtable");
116 INIT_LIST_HEAD(&buffer
->attachments
);
117 mutex_init(&buffer
->lock
);
118 mutex_lock(&dev
->buffer_lock
);
119 ion_buffer_add(dev
, buffer
);
120 mutex_unlock(&dev
->buffer_lock
);
124 heap
->ops
->free(buffer
);
130 void ion_buffer_destroy(struct ion_buffer
*buffer
)
132 if (WARN_ON(buffer
->kmap_cnt
> 0))
133 buffer
->heap
->ops
->unmap_kernel(buffer
->heap
, buffer
);
134 buffer
->heap
->ops
->free(buffer
);
138 static void _ion_buffer_destroy(struct ion_buffer
*buffer
)
140 struct ion_heap
*heap
= buffer
->heap
;
141 struct ion_device
*dev
= buffer
->dev
;
143 mutex_lock(&dev
->buffer_lock
);
144 rb_erase(&buffer
->node
, &dev
->buffers
);
145 mutex_unlock(&dev
->buffer_lock
);
147 if (heap
->flags
& ION_HEAP_FLAG_DEFER_FREE
)
148 ion_heap_freelist_add(heap
, buffer
);
150 ion_buffer_destroy(buffer
);
153 static void *ion_buffer_kmap_get(struct ion_buffer
*buffer
)
157 if (buffer
->kmap_cnt
) {
159 return buffer
->vaddr
;
161 vaddr
= buffer
->heap
->ops
->map_kernel(buffer
->heap
, buffer
);
162 if (WARN_ONCE(!vaddr
,
163 "heap->ops->map_kernel should return ERR_PTR on error"))
164 return ERR_PTR(-EINVAL
);
167 buffer
->vaddr
= vaddr
;
172 static void ion_buffer_kmap_put(struct ion_buffer
*buffer
)
175 if (!buffer
->kmap_cnt
) {
176 buffer
->heap
->ops
->unmap_kernel(buffer
->heap
, buffer
);
177 buffer
->vaddr
= NULL
;
181 static struct sg_table
*dup_sg_table(struct sg_table
*table
)
183 struct sg_table
*new_table
;
185 struct scatterlist
*sg
, *new_sg
;
187 new_table
= kzalloc(sizeof(*new_table
), GFP_KERNEL
);
189 return ERR_PTR(-ENOMEM
);
191 ret
= sg_alloc_table(new_table
, table
->nents
, GFP_KERNEL
);
194 return ERR_PTR(-ENOMEM
);
197 new_sg
= new_table
->sgl
;
198 for_each_sg(table
->sgl
, sg
, table
->nents
, i
) {
199 memcpy(new_sg
, sg
, sizeof(*sg
));
201 new_sg
= sg_next(new_sg
);
207 static void free_duped_table(struct sg_table
*table
)
209 sg_free_table(table
);
213 struct ion_dma_buf_attachment
{
215 struct sg_table
*table
;
216 struct list_head list
;
219 static int ion_dma_buf_attach(struct dma_buf
*dmabuf
, struct device
*dev
,
220 struct dma_buf_attachment
*attachment
)
222 struct ion_dma_buf_attachment
*a
;
223 struct sg_table
*table
;
224 struct ion_buffer
*buffer
= dmabuf
->priv
;
226 a
= kzalloc(sizeof(*a
), GFP_KERNEL
);
230 table
= dup_sg_table(buffer
->sg_table
);
238 INIT_LIST_HEAD(&a
->list
);
240 attachment
->priv
= a
;
242 mutex_lock(&buffer
->lock
);
243 list_add(&a
->list
, &buffer
->attachments
);
244 mutex_unlock(&buffer
->lock
);
249 static void ion_dma_buf_detatch(struct dma_buf
*dmabuf
,
250 struct dma_buf_attachment
*attachment
)
252 struct ion_dma_buf_attachment
*a
= attachment
->priv
;
253 struct ion_buffer
*buffer
= dmabuf
->priv
;
255 free_duped_table(a
->table
);
256 mutex_lock(&buffer
->lock
);
258 mutex_unlock(&buffer
->lock
);
263 static struct sg_table
*ion_map_dma_buf(struct dma_buf_attachment
*attachment
,
264 enum dma_data_direction direction
)
266 struct ion_dma_buf_attachment
*a
= attachment
->priv
;
267 struct sg_table
*table
;
271 if (!dma_map_sg(attachment
->dev
, table
->sgl
, table
->nents
,
273 return ERR_PTR(-ENOMEM
);
278 static void ion_unmap_dma_buf(struct dma_buf_attachment
*attachment
,
279 struct sg_table
*table
,
280 enum dma_data_direction direction
)
282 dma_unmap_sg(attachment
->dev
, table
->sgl
, table
->nents
, direction
);
285 static int ion_mmap(struct dma_buf
*dmabuf
, struct vm_area_struct
*vma
)
287 struct ion_buffer
*buffer
= dmabuf
->priv
;
290 if (!buffer
->heap
->ops
->map_user
) {
291 pr_err("%s: this heap does not define a method for mapping to userspace\n",
296 if (!(buffer
->flags
& ION_FLAG_CACHED
))
297 vma
->vm_page_prot
= pgprot_writecombine(vma
->vm_page_prot
);
299 mutex_lock(&buffer
->lock
);
300 /* now map it to userspace */
301 ret
= buffer
->heap
->ops
->map_user(buffer
->heap
, buffer
, vma
);
302 mutex_unlock(&buffer
->lock
);
305 pr_err("%s: failure mapping buffer to userspace\n",
311 static void ion_dma_buf_release(struct dma_buf
*dmabuf
)
313 struct ion_buffer
*buffer
= dmabuf
->priv
;
315 _ion_buffer_destroy(buffer
);
318 static void *ion_dma_buf_kmap(struct dma_buf
*dmabuf
, unsigned long offset
)
320 struct ion_buffer
*buffer
= dmabuf
->priv
;
322 return buffer
->vaddr
+ offset
* PAGE_SIZE
;
325 static void ion_dma_buf_kunmap(struct dma_buf
*dmabuf
, unsigned long offset
,
330 static int ion_dma_buf_begin_cpu_access(struct dma_buf
*dmabuf
,
331 enum dma_data_direction direction
)
333 struct ion_buffer
*buffer
= dmabuf
->priv
;
335 struct ion_dma_buf_attachment
*a
;
338 * TODO: Move this elsewhere because we don't always need a vaddr
340 if (buffer
->heap
->ops
->map_kernel
) {
341 mutex_lock(&buffer
->lock
);
342 vaddr
= ion_buffer_kmap_get(buffer
);
343 mutex_unlock(&buffer
->lock
);
346 mutex_lock(&buffer
->lock
);
347 list_for_each_entry(a
, &buffer
->attachments
, list
) {
348 dma_sync_sg_for_cpu(a
->dev
, a
->table
->sgl
, a
->table
->nents
,
351 mutex_unlock(&buffer
->lock
);
356 static int ion_dma_buf_end_cpu_access(struct dma_buf
*dmabuf
,
357 enum dma_data_direction direction
)
359 struct ion_buffer
*buffer
= dmabuf
->priv
;
360 struct ion_dma_buf_attachment
*a
;
362 if (buffer
->heap
->ops
->map_kernel
) {
363 mutex_lock(&buffer
->lock
);
364 ion_buffer_kmap_put(buffer
);
365 mutex_unlock(&buffer
->lock
);
368 mutex_lock(&buffer
->lock
);
369 list_for_each_entry(a
, &buffer
->attachments
, list
) {
370 dma_sync_sg_for_device(a
->dev
, a
->table
->sgl
, a
->table
->nents
,
373 mutex_unlock(&buffer
->lock
);
378 static const struct dma_buf_ops dma_buf_ops
= {
379 .map_dma_buf
= ion_map_dma_buf
,
380 .unmap_dma_buf
= ion_unmap_dma_buf
,
382 .release
= ion_dma_buf_release
,
383 .attach
= ion_dma_buf_attach
,
384 .detach
= ion_dma_buf_detatch
,
385 .begin_cpu_access
= ion_dma_buf_begin_cpu_access
,
386 .end_cpu_access
= ion_dma_buf_end_cpu_access
,
387 .map_atomic
= ion_dma_buf_kmap
,
388 .unmap_atomic
= ion_dma_buf_kunmap
,
389 .map
= ion_dma_buf_kmap
,
390 .unmap
= ion_dma_buf_kunmap
,
393 int ion_alloc(size_t len
, unsigned int heap_id_mask
, unsigned int flags
)
395 struct ion_device
*dev
= internal_dev
;
396 struct ion_buffer
*buffer
= NULL
;
397 struct ion_heap
*heap
;
398 DEFINE_DMA_BUF_EXPORT_INFO(exp_info
);
400 struct dma_buf
*dmabuf
;
402 pr_debug("%s: len %zu heap_id_mask %u flags %x\n", __func__
,
403 len
, heap_id_mask
, flags
);
405 * traverse the list of heaps available in this system in priority
406 * order. If the heap type is supported by the client, and matches the
407 * request of the caller allocate from it. Repeat until allocate has
408 * succeeded or all heaps have been tried
410 len
= PAGE_ALIGN(len
);
415 down_read(&dev
->lock
);
416 plist_for_each_entry(heap
, &dev
->heaps
, node
) {
417 /* if the caller didn't specify this heap id */
418 if (!((1 << heap
->id
) & heap_id_mask
))
420 buffer
= ion_buffer_create(heap
, dev
, len
, flags
);
430 return PTR_ERR(buffer
);
432 exp_info
.ops
= &dma_buf_ops
;
433 exp_info
.size
= buffer
->size
;
434 exp_info
.flags
= O_RDWR
;
435 exp_info
.priv
= buffer
;
437 dmabuf
= dma_buf_export(&exp_info
);
438 if (IS_ERR(dmabuf
)) {
439 _ion_buffer_destroy(buffer
);
440 return PTR_ERR(dmabuf
);
443 fd
= dma_buf_fd(dmabuf
, O_CLOEXEC
);
450 int ion_query_heaps(struct ion_heap_query
*query
)
452 struct ion_device
*dev
= internal_dev
;
453 struct ion_heap_data __user
*buffer
= u64_to_user_ptr(query
->heaps
);
454 int ret
= -EINVAL
, cnt
= 0, max_cnt
;
455 struct ion_heap
*heap
;
456 struct ion_heap_data hdata
;
458 memset(&hdata
, 0, sizeof(hdata
));
460 down_read(&dev
->lock
);
462 query
->cnt
= dev
->heap_cnt
;
470 max_cnt
= query
->cnt
;
472 plist_for_each_entry(heap
, &dev
->heaps
, node
) {
473 strncpy(hdata
.name
, heap
->name
, MAX_HEAP_NAME
);
474 hdata
.name
[sizeof(hdata
.name
) - 1] = '\0';
475 hdata
.type
= heap
->type
;
476 hdata
.heap_id
= heap
->id
;
478 if (copy_to_user(&buffer
[cnt
], &hdata
, sizeof(hdata
))) {
495 static const struct file_operations ion_fops
= {
496 .owner
= THIS_MODULE
,
497 .unlocked_ioctl
= ion_ioctl
,
499 .compat_ioctl
= ion_ioctl
,
503 static int debug_shrink_set(void *data
, u64 val
)
505 struct ion_heap
*heap
= data
;
506 struct shrink_control sc
;
509 sc
.gfp_mask
= GFP_HIGHUSER
;
513 objs
= heap
->shrinker
.count_objects(&heap
->shrinker
, &sc
);
514 sc
.nr_to_scan
= objs
;
517 heap
->shrinker
.scan_objects(&heap
->shrinker
, &sc
);
521 static int debug_shrink_get(void *data
, u64
*val
)
523 struct ion_heap
*heap
= data
;
524 struct shrink_control sc
;
527 sc
.gfp_mask
= GFP_HIGHUSER
;
530 objs
= heap
->shrinker
.count_objects(&heap
->shrinker
, &sc
);
535 DEFINE_SIMPLE_ATTRIBUTE(debug_shrink_fops
, debug_shrink_get
,
536 debug_shrink_set
, "%llu\n");
538 void ion_device_add_heap(struct ion_heap
*heap
)
540 struct dentry
*debug_file
;
541 struct ion_device
*dev
= internal_dev
;
543 if (!heap
->ops
->allocate
|| !heap
->ops
->free
)
544 pr_err("%s: can not add heap with invalid ops struct.\n",
547 spin_lock_init(&heap
->free_lock
);
548 heap
->free_list_size
= 0;
550 if (heap
->flags
& ION_HEAP_FLAG_DEFER_FREE
)
551 ion_heap_init_deferred_free(heap
);
553 if ((heap
->flags
& ION_HEAP_FLAG_DEFER_FREE
) || heap
->ops
->shrink
)
554 ion_heap_init_shrinker(heap
);
557 down_write(&dev
->lock
);
558 heap
->id
= heap_id
++;
560 * use negative heap->id to reverse the priority -- when traversing
561 * the list later attempt higher id numbers first
563 plist_node_init(&heap
->node
, -heap
->id
);
564 plist_add(&heap
->node
, &dev
->heaps
);
566 if (heap
->shrinker
.count_objects
&& heap
->shrinker
.scan_objects
) {
569 snprintf(debug_name
, 64, "%s_shrink", heap
->name
);
570 debug_file
= debugfs_create_file(
571 debug_name
, 0644, dev
->debug_root
, heap
,
574 char buf
[256], *path
;
576 path
= dentry_path(dev
->debug_root
, buf
, 256);
577 pr_err("Failed to create heap shrinker debugfs at %s/%s\n",
583 up_write(&dev
->lock
);
585 EXPORT_SYMBOL(ion_device_add_heap
);
587 static int ion_device_create(void)
589 struct ion_device
*idev
;
592 idev
= kzalloc(sizeof(*idev
), GFP_KERNEL
);
596 idev
->dev
.minor
= MISC_DYNAMIC_MINOR
;
597 idev
->dev
.name
= "ion";
598 idev
->dev
.fops
= &ion_fops
;
599 idev
->dev
.parent
= NULL
;
600 ret
= misc_register(&idev
->dev
);
602 pr_err("ion: failed to register misc device.\n");
607 idev
->debug_root
= debugfs_create_dir("ion", NULL
);
608 if (!idev
->debug_root
) {
609 pr_err("ion: failed to create debugfs root directory.\n");
614 idev
->buffers
= RB_ROOT
;
615 mutex_init(&idev
->buffer_lock
);
616 init_rwsem(&idev
->lock
);
617 plist_head_init(&idev
->heaps
);
621 subsys_initcall(ion_device_create
);