2 * videobuf2-dma-contig.c - DMA contig memory allocator for videobuf2
4 * Copyright (C) 2010 Samsung Electronics
6 * Author: Pawel Osciak <pawel@osciak.com>
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation.
13 #include <linux/dma-buf.h>
14 #include <linux/module.h>
15 #include <linux/scatterlist.h>
16 #include <linux/sched.h>
17 #include <linux/slab.h>
18 #include <linux/dma-mapping.h>
20 #include <media/videobuf2-v4l2.h>
21 #include <media/videobuf2-dma-contig.h>
22 #include <media/videobuf2-memops.h>
34 struct dma_attrs attrs
;
35 enum dma_data_direction dma_dir
;
36 struct sg_table
*dma_sgt
;
37 struct frame_vector
*vec
;
40 struct vb2_vmarea_handler handler
;
42 struct sg_table
*sgt_base
;
45 struct dma_buf_attachment
*db_attach
;
48 /*********************************************/
49 /* scatterlist table functions */
50 /*********************************************/
52 static unsigned long vb2_dc_get_contiguous_size(struct sg_table
*sgt
)
54 struct scatterlist
*s
;
55 dma_addr_t expected
= sg_dma_address(sgt
->sgl
);
57 unsigned long size
= 0;
59 for_each_sg(sgt
->sgl
, s
, sgt
->nents
, i
) {
60 if (sg_dma_address(s
) != expected
)
62 expected
= sg_dma_address(s
) + sg_dma_len(s
);
63 size
+= sg_dma_len(s
);
68 /*********************************************/
69 /* callbacks for all buffers */
70 /*********************************************/
72 static void *vb2_dc_cookie(void *buf_priv
)
74 struct vb2_dc_buf
*buf
= buf_priv
;
76 return &buf
->dma_addr
;
79 static void *vb2_dc_vaddr(void *buf_priv
)
81 struct vb2_dc_buf
*buf
= buf_priv
;
83 if (!buf
->vaddr
&& buf
->db_attach
)
84 buf
->vaddr
= dma_buf_vmap(buf
->db_attach
->dmabuf
);
89 static unsigned int vb2_dc_num_users(void *buf_priv
)
91 struct vb2_dc_buf
*buf
= buf_priv
;
93 return atomic_read(&buf
->refcount
);
96 static void vb2_dc_prepare(void *buf_priv
)
98 struct vb2_dc_buf
*buf
= buf_priv
;
99 struct sg_table
*sgt
= buf
->dma_sgt
;
101 /* DMABUF exporter will flush the cache for us */
102 if (!sgt
|| buf
->db_attach
)
105 dma_sync_sg_for_device(buf
->dev
, sgt
->sgl
, sgt
->orig_nents
,
109 static void vb2_dc_finish(void *buf_priv
)
111 struct vb2_dc_buf
*buf
= buf_priv
;
112 struct sg_table
*sgt
= buf
->dma_sgt
;
114 /* DMABUF exporter will flush the cache for us */
115 if (!sgt
|| buf
->db_attach
)
118 dma_sync_sg_for_cpu(buf
->dev
, sgt
->sgl
, sgt
->orig_nents
, buf
->dma_dir
);
121 /*********************************************/
122 /* callbacks for MMAP buffers */
123 /*********************************************/
125 static void vb2_dc_put(void *buf_priv
)
127 struct vb2_dc_buf
*buf
= buf_priv
;
129 if (!atomic_dec_and_test(&buf
->refcount
))
133 sg_free_table(buf
->sgt_base
);
134 kfree(buf
->sgt_base
);
136 dma_free_attrs(buf
->dev
, buf
->size
, buf
->cookie
, buf
->dma_addr
,
138 put_device(buf
->dev
);
142 static void *vb2_dc_alloc(void *alloc_ctx
, const struct dma_attrs
*attrs
,
143 unsigned long size
, enum dma_data_direction dma_dir
,
146 struct vb2_dc_conf
*conf
= alloc_ctx
;
147 struct device
*dev
= conf
->dev
;
148 struct vb2_dc_buf
*buf
;
150 buf
= kzalloc(sizeof *buf
, GFP_KERNEL
);
152 return ERR_PTR(-ENOMEM
);
156 buf
->cookie
= dma_alloc_attrs(dev
, size
, &buf
->dma_addr
,
157 GFP_KERNEL
| gfp_flags
, &buf
->attrs
);
159 dev_err(dev
, "dma_alloc_coherent of size %ld failed\n", size
);
161 return ERR_PTR(-ENOMEM
);
164 if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING
, &buf
->attrs
))
165 buf
->vaddr
= buf
->cookie
;
167 /* Prevent the device from being released while the buffer is used */
168 buf
->dev
= get_device(dev
);
170 buf
->dma_dir
= dma_dir
;
172 buf
->handler
.refcount
= &buf
->refcount
;
173 buf
->handler
.put
= vb2_dc_put
;
174 buf
->handler
.arg
= buf
;
176 atomic_inc(&buf
->refcount
);
181 static int vb2_dc_mmap(void *buf_priv
, struct vm_area_struct
*vma
)
183 struct vb2_dc_buf
*buf
= buf_priv
;
187 printk(KERN_ERR
"No buffer to map\n");
192 * dma_mmap_* uses vm_pgoff as in-buffer offset, but we want to
197 ret
= dma_mmap_attrs(buf
->dev
, vma
, buf
->cookie
,
198 buf
->dma_addr
, buf
->size
, &buf
->attrs
);
201 pr_err("Remapping memory failed, error: %d\n", ret
);
205 vma
->vm_flags
|= VM_DONTEXPAND
| VM_DONTDUMP
;
206 vma
->vm_private_data
= &buf
->handler
;
207 vma
->vm_ops
= &vb2_common_vm_ops
;
209 vma
->vm_ops
->open(vma
);
211 pr_debug("%s: mapped dma addr 0x%08lx at 0x%08lx, size %ld\n",
212 __func__
, (unsigned long)buf
->dma_addr
, vma
->vm_start
,
218 /*********************************************/
219 /* DMABUF ops for exporters */
220 /*********************************************/
222 struct vb2_dc_attachment
{
224 enum dma_data_direction dma_dir
;
227 static int vb2_dc_dmabuf_ops_attach(struct dma_buf
*dbuf
, struct device
*dev
,
228 struct dma_buf_attachment
*dbuf_attach
)
230 struct vb2_dc_attachment
*attach
;
232 struct scatterlist
*rd
, *wr
;
233 struct sg_table
*sgt
;
234 struct vb2_dc_buf
*buf
= dbuf
->priv
;
237 attach
= kzalloc(sizeof(*attach
), GFP_KERNEL
);
242 /* Copy the buf->base_sgt scatter list to the attachment, as we can't
243 * map the same scatter list to multiple attachments at the same time.
245 ret
= sg_alloc_table(sgt
, buf
->sgt_base
->orig_nents
, GFP_KERNEL
);
251 rd
= buf
->sgt_base
->sgl
;
253 for (i
= 0; i
< sgt
->orig_nents
; ++i
) {
254 sg_set_page(wr
, sg_page(rd
), rd
->length
, rd
->offset
);
259 attach
->dma_dir
= DMA_NONE
;
260 dbuf_attach
->priv
= attach
;
265 static void vb2_dc_dmabuf_ops_detach(struct dma_buf
*dbuf
,
266 struct dma_buf_attachment
*db_attach
)
268 struct vb2_dc_attachment
*attach
= db_attach
->priv
;
269 struct sg_table
*sgt
;
276 /* release the scatterlist cache */
277 if (attach
->dma_dir
!= DMA_NONE
)
278 dma_unmap_sg(db_attach
->dev
, sgt
->sgl
, sgt
->orig_nents
,
282 db_attach
->priv
= NULL
;
285 static struct sg_table
*vb2_dc_dmabuf_ops_map(
286 struct dma_buf_attachment
*db_attach
, enum dma_data_direction dma_dir
)
288 struct vb2_dc_attachment
*attach
= db_attach
->priv
;
289 /* stealing dmabuf mutex to serialize map/unmap operations */
290 struct mutex
*lock
= &db_attach
->dmabuf
->lock
;
291 struct sg_table
*sgt
;
296 /* return previously mapped sg table */
297 if (attach
->dma_dir
== dma_dir
) {
302 /* release any previous cache */
303 if (attach
->dma_dir
!= DMA_NONE
) {
304 dma_unmap_sg(db_attach
->dev
, sgt
->sgl
, sgt
->orig_nents
,
306 attach
->dma_dir
= DMA_NONE
;
309 /* mapping to the client with new direction */
310 sgt
->nents
= dma_map_sg(db_attach
->dev
, sgt
->sgl
, sgt
->orig_nents
,
313 pr_err("failed to map scatterlist\n");
315 return ERR_PTR(-EIO
);
318 attach
->dma_dir
= dma_dir
;
325 static void vb2_dc_dmabuf_ops_unmap(struct dma_buf_attachment
*db_attach
,
326 struct sg_table
*sgt
, enum dma_data_direction dma_dir
)
328 /* nothing to be done here */
331 static void vb2_dc_dmabuf_ops_release(struct dma_buf
*dbuf
)
333 /* drop reference obtained in vb2_dc_get_dmabuf */
334 vb2_dc_put(dbuf
->priv
);
337 static void *vb2_dc_dmabuf_ops_kmap(struct dma_buf
*dbuf
, unsigned long pgnum
)
339 struct vb2_dc_buf
*buf
= dbuf
->priv
;
341 return buf
->vaddr
? buf
->vaddr
+ pgnum
* PAGE_SIZE
: NULL
;
344 static void *vb2_dc_dmabuf_ops_vmap(struct dma_buf
*dbuf
)
346 struct vb2_dc_buf
*buf
= dbuf
->priv
;
351 static int vb2_dc_dmabuf_ops_mmap(struct dma_buf
*dbuf
,
352 struct vm_area_struct
*vma
)
354 return vb2_dc_mmap(dbuf
->priv
, vma
);
357 static struct dma_buf_ops vb2_dc_dmabuf_ops
= {
358 .attach
= vb2_dc_dmabuf_ops_attach
,
359 .detach
= vb2_dc_dmabuf_ops_detach
,
360 .map_dma_buf
= vb2_dc_dmabuf_ops_map
,
361 .unmap_dma_buf
= vb2_dc_dmabuf_ops_unmap
,
362 .kmap
= vb2_dc_dmabuf_ops_kmap
,
363 .kmap_atomic
= vb2_dc_dmabuf_ops_kmap
,
364 .vmap
= vb2_dc_dmabuf_ops_vmap
,
365 .mmap
= vb2_dc_dmabuf_ops_mmap
,
366 .release
= vb2_dc_dmabuf_ops_release
,
369 static struct sg_table
*vb2_dc_get_base_sgt(struct vb2_dc_buf
*buf
)
372 struct sg_table
*sgt
;
374 sgt
= kmalloc(sizeof(*sgt
), GFP_KERNEL
);
376 dev_err(buf
->dev
, "failed to alloc sg table\n");
380 ret
= dma_get_sgtable_attrs(buf
->dev
, sgt
, buf
->cookie
, buf
->dma_addr
,
381 buf
->size
, &buf
->attrs
);
383 dev_err(buf
->dev
, "failed to get scatterlist from DMA API\n");
391 static struct dma_buf
*vb2_dc_get_dmabuf(void *buf_priv
, unsigned long flags
)
393 struct vb2_dc_buf
*buf
= buf_priv
;
394 struct dma_buf
*dbuf
;
395 DEFINE_DMA_BUF_EXPORT_INFO(exp_info
);
397 exp_info
.ops
= &vb2_dc_dmabuf_ops
;
398 exp_info
.size
= buf
->size
;
399 exp_info
.flags
= flags
;
403 buf
->sgt_base
= vb2_dc_get_base_sgt(buf
);
405 if (WARN_ON(!buf
->sgt_base
))
408 dbuf
= dma_buf_export(&exp_info
);
412 /* dmabuf keeps reference to vb2 buffer */
413 atomic_inc(&buf
->refcount
);
418 /*********************************************/
419 /* callbacks for USERPTR buffers */
420 /*********************************************/
422 static void vb2_dc_put_userptr(void *buf_priv
)
424 struct vb2_dc_buf
*buf
= buf_priv
;
425 struct sg_table
*sgt
= buf
->dma_sgt
;
430 DEFINE_DMA_ATTRS(attrs
);
432 dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC
, &attrs
);
434 * No need to sync to CPU, it's already synced to the CPU
435 * since the finish() memop will have been called before this.
437 dma_unmap_sg_attrs(buf
->dev
, sgt
->sgl
, sgt
->orig_nents
,
438 buf
->dma_dir
, &attrs
);
439 pages
= frame_vector_pages(buf
->vec
);
440 /* sgt should exist only if vector contains pages... */
441 BUG_ON(IS_ERR(pages
));
442 for (i
= 0; i
< frame_vector_count(buf
->vec
); i
++)
443 set_page_dirty_lock(pages
[i
]);
447 vb2_destroy_framevec(buf
->vec
);
452 * For some kind of reserved memory there might be no struct page available,
453 * so all that can be done to support such 'pages' is to try to convert
454 * pfn to dma address or at the last resort just assume that
455 * dma address == physical address (like it has been assumed in earlier version
456 * of videobuf2-dma-contig
459 #ifdef __arch_pfn_to_dma
460 static inline dma_addr_t
vb2_dc_pfn_to_dma(struct device
*dev
, unsigned long pfn
)
462 return (dma_addr_t
)__arch_pfn_to_dma(dev
, pfn
);
464 #elif defined(__pfn_to_bus)
465 static inline dma_addr_t
vb2_dc_pfn_to_dma(struct device
*dev
, unsigned long pfn
)
467 return (dma_addr_t
)__pfn_to_bus(pfn
);
469 #elif defined(__pfn_to_phys)
470 static inline dma_addr_t
vb2_dc_pfn_to_dma(struct device
*dev
, unsigned long pfn
)
472 return (dma_addr_t
)__pfn_to_phys(pfn
);
475 static inline dma_addr_t
vb2_dc_pfn_to_dma(struct device
*dev
, unsigned long pfn
)
477 /* really, we cannot do anything better at this point */
478 return (dma_addr_t
)(pfn
) << PAGE_SHIFT
;
482 static void *vb2_dc_get_userptr(void *alloc_ctx
, unsigned long vaddr
,
483 unsigned long size
, enum dma_data_direction dma_dir
)
485 struct vb2_dc_conf
*conf
= alloc_ctx
;
486 struct vb2_dc_buf
*buf
;
487 struct frame_vector
*vec
;
488 unsigned long offset
;
491 struct sg_table
*sgt
;
492 unsigned long contig_size
;
493 unsigned long dma_align
= dma_get_cache_alignment();
494 DEFINE_DMA_ATTRS(attrs
);
496 dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC
, &attrs
);
498 /* Only cache aligned DMA transfers are reliable */
499 if (!IS_ALIGNED(vaddr
| size
, dma_align
)) {
500 pr_debug("user data must be aligned to %lu bytes\n", dma_align
);
501 return ERR_PTR(-EINVAL
);
505 pr_debug("size is zero\n");
506 return ERR_PTR(-EINVAL
);
509 buf
= kzalloc(sizeof *buf
, GFP_KERNEL
);
511 return ERR_PTR(-ENOMEM
);
513 buf
->dev
= conf
->dev
;
514 buf
->dma_dir
= dma_dir
;
516 offset
= vaddr
& ~PAGE_MASK
;
517 vec
= vb2_create_framevec(vaddr
, size
, dma_dir
== DMA_FROM_DEVICE
);
523 n_pages
= frame_vector_count(vec
);
524 ret
= frame_vector_to_pages(vec
);
526 unsigned long *nums
= frame_vector_pfns(vec
);
529 * Failed to convert to pages... Check the memory is physically
530 * contiguous and use direct mapping
532 for (i
= 1; i
< n_pages
; i
++)
533 if (nums
[i
-1] + 1 != nums
[i
])
535 buf
->dma_addr
= vb2_dc_pfn_to_dma(buf
->dev
, nums
[0]);
539 sgt
= kzalloc(sizeof(*sgt
), GFP_KERNEL
);
541 pr_err("failed to allocate sg table\n");
546 ret
= sg_alloc_table_from_pages(sgt
, frame_vector_pages(vec
), n_pages
,
547 offset
, size
, GFP_KERNEL
);
549 pr_err("failed to initialize sg table\n");
554 * No need to sync to the device, this will happen later when the
555 * prepare() memop is called.
557 sgt
->nents
= dma_map_sg_attrs(buf
->dev
, sgt
->sgl
, sgt
->orig_nents
,
558 buf
->dma_dir
, &attrs
);
559 if (sgt
->nents
<= 0) {
560 pr_err("failed to map scatterlist\n");
565 contig_size
= vb2_dc_get_contiguous_size(sgt
);
566 if (contig_size
< size
) {
567 pr_err("contiguous mapping is too small %lu/%lu\n",
573 buf
->dma_addr
= sg_dma_address(sgt
->sgl
);
581 dma_unmap_sg_attrs(buf
->dev
, sgt
->sgl
, sgt
->orig_nents
,
582 buf
->dma_dir
, &attrs
);
591 vb2_destroy_framevec(vec
);
599 /*********************************************/
600 /* callbacks for DMABUF buffers */
601 /*********************************************/
603 static int vb2_dc_map_dmabuf(void *mem_priv
)
605 struct vb2_dc_buf
*buf
= mem_priv
;
606 struct sg_table
*sgt
;
607 unsigned long contig_size
;
609 if (WARN_ON(!buf
->db_attach
)) {
610 pr_err("trying to pin a non attached buffer\n");
614 if (WARN_ON(buf
->dma_sgt
)) {
615 pr_err("dmabuf buffer is already pinned\n");
619 /* get the associated scatterlist for this buffer */
620 sgt
= dma_buf_map_attachment(buf
->db_attach
, buf
->dma_dir
);
622 pr_err("Error getting dmabuf scatterlist\n");
626 /* checking if dmabuf is big enough to store contiguous chunk */
627 contig_size
= vb2_dc_get_contiguous_size(sgt
);
628 if (contig_size
< buf
->size
) {
629 pr_err("contiguous chunk is too small %lu/%lu b\n",
630 contig_size
, buf
->size
);
631 dma_buf_unmap_attachment(buf
->db_attach
, sgt
, buf
->dma_dir
);
635 buf
->dma_addr
= sg_dma_address(sgt
->sgl
);
642 static void vb2_dc_unmap_dmabuf(void *mem_priv
)
644 struct vb2_dc_buf
*buf
= mem_priv
;
645 struct sg_table
*sgt
= buf
->dma_sgt
;
647 if (WARN_ON(!buf
->db_attach
)) {
648 pr_err("trying to unpin a not attached buffer\n");
653 pr_err("dmabuf buffer is already unpinned\n");
658 dma_buf_vunmap(buf
->db_attach
->dmabuf
, buf
->vaddr
);
661 dma_buf_unmap_attachment(buf
->db_attach
, sgt
, buf
->dma_dir
);
667 static void vb2_dc_detach_dmabuf(void *mem_priv
)
669 struct vb2_dc_buf
*buf
= mem_priv
;
671 /* if vb2 works correctly you should never detach mapped buffer */
672 if (WARN_ON(buf
->dma_addr
))
673 vb2_dc_unmap_dmabuf(buf
);
675 /* detach this attachment */
676 dma_buf_detach(buf
->db_attach
->dmabuf
, buf
->db_attach
);
680 static void *vb2_dc_attach_dmabuf(void *alloc_ctx
, struct dma_buf
*dbuf
,
681 unsigned long size
, enum dma_data_direction dma_dir
)
683 struct vb2_dc_conf
*conf
= alloc_ctx
;
684 struct vb2_dc_buf
*buf
;
685 struct dma_buf_attachment
*dba
;
687 if (dbuf
->size
< size
)
688 return ERR_PTR(-EFAULT
);
690 buf
= kzalloc(sizeof(*buf
), GFP_KERNEL
);
692 return ERR_PTR(-ENOMEM
);
694 buf
->dev
= conf
->dev
;
695 /* create attachment for the dmabuf with the user device */
696 dba
= dma_buf_attach(dbuf
, buf
->dev
);
698 pr_err("failed to attach dmabuf\n");
703 buf
->dma_dir
= dma_dir
;
705 buf
->db_attach
= dba
;
710 /*********************************************/
711 /* DMA CONTIG exported functions */
712 /*********************************************/
714 const struct vb2_mem_ops vb2_dma_contig_memops
= {
715 .alloc
= vb2_dc_alloc
,
717 .get_dmabuf
= vb2_dc_get_dmabuf
,
718 .cookie
= vb2_dc_cookie
,
719 .vaddr
= vb2_dc_vaddr
,
721 .get_userptr
= vb2_dc_get_userptr
,
722 .put_userptr
= vb2_dc_put_userptr
,
723 .prepare
= vb2_dc_prepare
,
724 .finish
= vb2_dc_finish
,
725 .map_dmabuf
= vb2_dc_map_dmabuf
,
726 .unmap_dmabuf
= vb2_dc_unmap_dmabuf
,
727 .attach_dmabuf
= vb2_dc_attach_dmabuf
,
728 .detach_dmabuf
= vb2_dc_detach_dmabuf
,
729 .num_users
= vb2_dc_num_users
,
731 EXPORT_SYMBOL_GPL(vb2_dma_contig_memops
);
733 void *vb2_dma_contig_init_ctx(struct device
*dev
)
735 struct vb2_dc_conf
*conf
;
737 conf
= kzalloc(sizeof *conf
, GFP_KERNEL
);
739 return ERR_PTR(-ENOMEM
);
745 EXPORT_SYMBOL_GPL(vb2_dma_contig_init_ctx
);
747 void vb2_dma_contig_cleanup_ctx(void *alloc_ctx
)
749 if (!IS_ERR_OR_NULL(alloc_ctx
))
752 EXPORT_SYMBOL_GPL(vb2_dma_contig_cleanup_ctx
);
755 * vb2_dma_contig_set_max_seg_size() - configure DMA max segment size
756 * @dev: device for configuring DMA parameters
757 * @size: size of DMA max segment size to set
759 * To allow mapping the scatter-list into a single chunk in the DMA
760 * address space, the device is required to have the DMA max segment
761 * size parameter set to a value larger than the buffer size. Otherwise,
762 * the DMA-mapping subsystem will split the mapping into max segment
763 * size chunks. This function sets the DMA max segment size
764 * parameter to let DMA-mapping map a buffer as a single chunk in DMA
766 * This code assumes that the DMA-mapping subsystem will merge all
767 * scatterlist segments if this is really possible (for example when
768 * an IOMMU is available and enabled).
769 * Ideally, this parameter should be set by the generic bus code, but it
770 * is left with the default 64KiB value due to historical litmiations in
771 * other subsystems (like limited USB host drivers) and there no good
772 * place to set it to the proper value.
773 * This function should be called from the drivers, which are known to
774 * operate on platforms with IOMMU and provide access to shared buffers
775 * (either USERPTR or DMABUF). This should be done before initializing
778 int vb2_dma_contig_set_max_seg_size(struct device
*dev
, unsigned int size
)
780 if (!dev
->dma_parms
) {
781 dev
->dma_parms
= kzalloc(sizeof(dev
->dma_parms
), GFP_KERNEL
);
785 if (dma_get_max_seg_size(dev
) < size
)
786 return dma_set_max_seg_size(dev
, size
);
790 EXPORT_SYMBOL_GPL(vb2_dma_contig_set_max_seg_size
);
793 * vb2_dma_contig_clear_max_seg_size() - release resources for DMA parameters
794 * @dev: device for configuring DMA parameters
796 * This function releases resources allocated to configure DMA parameters
797 * (see vb2_dma_contig_set_max_seg_size() function). It should be called from
798 * device drivers on driver remove.
800 void vb2_dma_contig_clear_max_seg_size(struct device
*dev
)
802 kfree(dev
->dma_parms
);
803 dev
->dma_parms
= NULL
;
805 EXPORT_SYMBOL_GPL(vb2_dma_contig_clear_max_seg_size
);
807 MODULE_DESCRIPTION("DMA-contig memory handling routines for videobuf2");
808 MODULE_AUTHOR("Pawel Osciak <pawel@osciak.com>");
809 MODULE_LICENSE("GPL");