[mirror_ubuntu-artful-kernel.git] / drivers / media / v4l2-core / videobuf2-dma-sg.c

/*
 * videobuf2-dma-sg.c - dma scatter/gather memory allocator for videobuf2
 *
 * Copyright (C) 2010 Samsung Electronics
 *
 * Author: Andrzej Pietrasiewicz <andrzej.p@samsung.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation.
 */

#include <linux/module.h>
#include <linux/mm.h>
#include <linux/scatterlist.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>

#include <media/videobuf2-v4l2.h>
#include <media/videobuf2-memops.h>
#include <media/videobuf2-dma-sg.h>

static int debug;
module_param(debug, int, 0644);

#define dprintk(level, fmt, arg...)					\
	do {								\
		if (debug >= level)					\
			printk(KERN_DEBUG "vb2-dma-sg: " fmt, ## arg);	\
	} while (0)

struct vb2_dma_sg_conf {
	struct device		*dev;
};

struct vb2_dma_sg_buf {
	struct device			*dev;
	void				*vaddr;
	struct page			**pages;
	struct frame_vector		*vec;
	int				offset;
	enum dma_data_direction		dma_dir;
	struct sg_table			sg_table;
	/*
	 * This will point to sg_table when used with the MMAP or USERPTR
	 * memory model, and to the dma_buf sglist when used with the
	 * DMABUF memory model.
	 */
	struct sg_table			*dma_sgt;
	size_t				size;
	unsigned int			num_pages;
	atomic_t			refcount;
	struct vb2_vmarea_handler	handler;

	struct dma_buf_attachment	*db_attach;
};

static void vb2_dma_sg_put(void *buf_priv);

static int vb2_dma_sg_alloc_compacted(struct vb2_dma_sg_buf *buf,
		gfp_t gfp_flags)
{
	unsigned int last_page = 0;
	int size = buf->size;

	while (size > 0) {
		struct page *pages;
		int order;
		int i;

		order = get_order(size);
		/* Dont over allocate*/
		if ((PAGE_SIZE << order) > size)
			order--;

		pages = NULL;
		while (!pages) {
			pages = alloc_pages(GFP_KERNEL | __GFP_ZERO |
					__GFP_NOWARN | gfp_flags, order);
			if (pages)
				break;

			if (order == 0) {
				while (last_page--)
					__free_page(buf->pages[last_page]);
				return -ENOMEM;
			}
			order--;
		}

		split_page(pages, order);
		for (i = 0; i < (1 << order); i++)
			buf->pages[last_page++] = &pages[i];

		size -= PAGE_SIZE << order;
	}

	return 0;
}

static void *vb2_dma_sg_alloc(void *alloc_ctx, unsigned long size,
			      enum dma_data_direction dma_dir, gfp_t gfp_flags)
{
	struct vb2_dma_sg_conf *conf = alloc_ctx;
	struct vb2_dma_sg_buf *buf;
	struct sg_table *sgt;
	int ret;
	int num_pages;
	DEFINE_DMA_ATTRS(attrs);

	dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs);

	if (WARN_ON(alloc_ctx == NULL))
		return NULL;
	buf = kzalloc(sizeof *buf, GFP_KERNEL);
	if (!buf)
		return NULL;

	buf->vaddr = NULL;
	buf->dma_dir = dma_dir;
	buf->offset = 0;
	buf->size = size;
	/* size is already page aligned */
	buf->num_pages = size >> PAGE_SHIFT;
	buf->dma_sgt = &buf->sg_table;

	buf->pages = kzalloc(buf->num_pages * sizeof(struct page *),
			     GFP_KERNEL);
	if (!buf->pages)
		goto fail_pages_array_alloc;

	ret = vb2_dma_sg_alloc_compacted(buf, gfp_flags);
	if (ret)
		goto fail_pages_alloc;

	ret = sg_alloc_table_from_pages(buf->dma_sgt, buf->pages,
			buf->num_pages, 0, size, GFP_KERNEL);
	if (ret)
		goto fail_table_alloc;

	/* Prevent the device from being released while the buffer is used */
	buf->dev = get_device(conf->dev);

	sgt = &buf->sg_table;
	/*
	 * No need to sync to the device, this will happen later when the
	 * prepare() memop is called.
	 */
	sgt->nents = dma_map_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,
				      buf->dma_dir, &attrs);
	if (!sgt->nents)
		goto fail_map;

	buf->handler.refcount = &buf->refcount;
	buf->handler.put = vb2_dma_sg_put;
	buf->handler.arg = buf;

	atomic_inc(&buf->refcount);

	dprintk(1, "%s: Allocated buffer of %d pages\n",
		__func__, buf->num_pages);
	return buf;

fail_map:
	put_device(buf->dev);
	sg_free_table(buf->dma_sgt);
fail_table_alloc:
	num_pages = buf->num_pages;
	while (num_pages--)
		__free_page(buf->pages[num_pages]);
fail_pages_alloc:
	kfree(buf->pages);
fail_pages_array_alloc:
	kfree(buf);
	return NULL;
}

static void vb2_dma_sg_put(void *buf_priv)
{
	struct vb2_dma_sg_buf *buf = buf_priv;
	struct sg_table *sgt = &buf->sg_table;
	int i = buf->num_pages;

	if (atomic_dec_and_test(&buf->refcount)) {
		DEFINE_DMA_ATTRS(attrs);

		dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs);
		dprintk(1, "%s: Freeing buffer of %d pages\n", __func__,
			buf->num_pages);
		dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,
				   buf->dma_dir, &attrs);
		if (buf->vaddr)
			vm_unmap_ram(buf->vaddr, buf->num_pages);
		sg_free_table(buf->dma_sgt);
		while (--i >= 0)
			__free_page(buf->pages[i]);
		kfree(buf->pages);
		put_device(buf->dev);
		kfree(buf);
	}
}

static void vb2_dma_sg_prepare(void *buf_priv)
{
	struct vb2_dma_sg_buf *buf = buf_priv;
	struct sg_table *sgt = buf->dma_sgt;

	/* DMABUF exporter will flush the cache for us */
	if (buf->db_attach)
		return;

	dma_sync_sg_for_device(buf->dev, sgt->sgl, sgt->nents, buf->dma_dir);
}

static void vb2_dma_sg_finish(void *buf_priv)
{
	struct vb2_dma_sg_buf *buf = buf_priv;
	struct sg_table *sgt = buf->dma_sgt;

	/* DMABUF exporter will flush the cache for us */
	if (buf->db_attach)
		return;

	dma_sync_sg_for_cpu(buf->dev, sgt->sgl, sgt->nents, buf->dma_dir);
}

static void *vb2_dma_sg_get_userptr(void *alloc_ctx, unsigned long vaddr,
				    unsigned long size,
				    enum dma_data_direction dma_dir)
{
	struct vb2_dma_sg_conf *conf = alloc_ctx;
	struct vb2_dma_sg_buf *buf;
	struct sg_table *sgt;
	DEFINE_DMA_ATTRS(attrs);
	struct frame_vector *vec;

	dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs);
	buf = kzalloc(sizeof *buf, GFP_KERNEL);
	if (!buf)
		return NULL;

	buf->vaddr = NULL;
	buf->dev = conf->dev;
	buf->dma_dir = dma_dir;
	buf->offset = vaddr & ~PAGE_MASK;
	buf->size = size;
	buf->dma_sgt = &buf->sg_table;
	vec = vb2_create_framevec(vaddr, size, buf->dma_dir == DMA_FROM_DEVICE);
	if (IS_ERR(vec))
		goto userptr_fail_pfnvec;
	buf->vec = vec;

	buf->pages = frame_vector_pages(vec);
	if (IS_ERR(buf->pages))
		goto userptr_fail_sgtable;
	buf->num_pages = frame_vector_count(vec);

	if (sg_alloc_table_from_pages(buf->dma_sgt, buf->pages,
			buf->num_pages, buf->offset, size, 0))
		goto userptr_fail_sgtable;

	sgt = &buf->sg_table;
	/*
	 * No need to sync to the device, this will happen later when the
	 * prepare() memop is called.
	 */
	sgt->nents = dma_map_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,
				      buf->dma_dir, &attrs);
	if (!sgt->nents)
		goto userptr_fail_map;

	return buf;

userptr_fail_map:
	sg_free_table(&buf->sg_table);
userptr_fail_sgtable:
	vb2_destroy_framevec(vec);
userptr_fail_pfnvec:
	kfree(buf);
	return NULL;
}

/*
 * @put_userptr: inform the allocator that a USERPTR buffer will no longer
 *		 be used
 */
static void vb2_dma_sg_put_userptr(void *buf_priv)
{
	struct vb2_dma_sg_buf *buf = buf_priv;
	struct sg_table *sgt = &buf->sg_table;
	int i = buf->num_pages;
	DEFINE_DMA_ATTRS(attrs);

	dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs);

	dprintk(1, "%s: Releasing userspace buffer of %d pages\n",
	       __func__, buf->num_pages);
	dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents, buf->dma_dir,
			   &attrs);
	if (buf->vaddr)
		vm_unmap_ram(buf->vaddr, buf->num_pages);
	sg_free_table(buf->dma_sgt);
	while (--i >= 0) {
		if (buf->dma_dir == DMA_FROM_DEVICE)
			set_page_dirty_lock(buf->pages[i]);
	}
	vb2_destroy_framevec(buf->vec);
	kfree(buf);
}

static void *vb2_dma_sg_vaddr(void *buf_priv)
{
	struct vb2_dma_sg_buf *buf = buf_priv;

	BUG_ON(!buf);

	if (!buf->vaddr) {
		if (buf->db_attach)
			buf->vaddr = dma_buf_vmap(buf->db_attach->dmabuf);
		else
			buf->vaddr = vm_map_ram(buf->pages,
					buf->num_pages, -1, PAGE_KERNEL);
	}

	/* add offset in case userptr is not page-aligned */
	return buf->vaddr ? buf->vaddr + buf->offset : NULL;
}

static unsigned int vb2_dma_sg_num_users(void *buf_priv)
{
	struct vb2_dma_sg_buf *buf = buf_priv;

	return atomic_read(&buf->refcount);
}

static int vb2_dma_sg_mmap(void *buf_priv, struct vm_area_struct *vma)
{
	struct vb2_dma_sg_buf *buf = buf_priv;
	unsigned long uaddr = vma->vm_start;
	unsigned long usize = vma->vm_end - vma->vm_start;
	int i = 0;

	if (!buf) {
		printk(KERN_ERR "No memory to map\n");
		return -EINVAL;
	}

	do {
		int ret;

		ret = vm_insert_page(vma, uaddr, buf->pages[i++]);
		if (ret) {
			printk(KERN_ERR "Remapping memory, error: %d\n", ret);
			return ret;
		}

		uaddr += PAGE_SIZE;
		usize -= PAGE_SIZE;
	} while (usize > 0);


	/*
	 * Use common vm_area operations to track buffer refcount.
	 */
	vma->vm_private_data	= &buf->handler;
	vma->vm_ops		= &vb2_common_vm_ops;

	vma->vm_ops->open(vma);

	return 0;
}

/*********************************************/
/*         DMABUF ops for exporters          */
/*********************************************/

struct vb2_dma_sg_attachment {
	struct sg_table sgt;
	enum dma_data_direction dma_dir;
};

static int vb2_dma_sg_dmabuf_ops_attach(struct dma_buf *dbuf, struct device *dev,
	struct dma_buf_attachment *dbuf_attach)
{
	struct vb2_dma_sg_attachment *attach;
	unsigned int i;
	struct scatterlist *rd, *wr;
	struct sg_table *sgt;
	struct vb2_dma_sg_buf *buf = dbuf->priv;
	int ret;

	attach = kzalloc(sizeof(*attach), GFP_KERNEL);
	if (!attach)
		return -ENOMEM;

	sgt = &attach->sgt;
	/* Copy the buf->base_sgt scatter list to the attachment, as we can't
	 * map the same scatter list to multiple attachments at the same time.
	 */
	ret = sg_alloc_table(sgt, buf->dma_sgt->orig_nents, GFP_KERNEL);
	if (ret) {
		kfree(attach);
		return -ENOMEM;
	}

	rd = buf->dma_sgt->sgl;
	wr = sgt->sgl;
	for (i = 0; i < sgt->orig_nents; ++i) {
		sg_set_page(wr, sg_page(rd), rd->length, rd->offset);
		rd = sg_next(rd);
		wr = sg_next(wr);
	}

	attach->dma_dir = DMA_NONE;
	dbuf_attach->priv = attach;

	return 0;
}

static void vb2_dma_sg_dmabuf_ops_detach(struct dma_buf *dbuf,
	struct dma_buf_attachment *db_attach)
{
	struct vb2_dma_sg_attachment *attach = db_attach->priv;
	struct sg_table *sgt;

	if (!attach)
		return;

	sgt = &attach->sgt;

	/* release the scatterlist cache */
	if (attach->dma_dir != DMA_NONE)
		dma_unmap_sg(db_attach->dev, sgt->sgl, sgt->orig_nents,
			attach->dma_dir);
	sg_free_table(sgt);
	kfree(attach);
	db_attach->priv = NULL;
}

static struct sg_table *vb2_dma_sg_dmabuf_ops_map(
	struct dma_buf_attachment *db_attach, enum dma_data_direction dma_dir)
{
	struct vb2_dma_sg_attachment *attach = db_attach->priv;
	/* stealing dmabuf mutex to serialize map/unmap operations */
	struct mutex *lock = &db_attach->dmabuf->lock;
	struct sg_table *sgt;

	mutex_lock(lock);

	sgt = &attach->sgt;
	/* return previously mapped sg table */
	if (attach->dma_dir == dma_dir) {
		mutex_unlock(lock);
		return sgt;
	}

	/* release any previous cache */
	if (attach->dma_dir != DMA_NONE) {
		dma_unmap_sg(db_attach->dev, sgt->sgl, sgt->orig_nents,
			attach->dma_dir);
		attach->dma_dir = DMA_NONE;
	}

	/* mapping to the client with new direction */
	sgt->nents = dma_map_sg(db_attach->dev, sgt->sgl, sgt->orig_nents,
				dma_dir);
	if (!sgt->nents) {
		pr_err("failed to map scatterlist\n");
		mutex_unlock(lock);
		return ERR_PTR(-EIO);
	}

	attach->dma_dir = dma_dir;

	mutex_unlock(lock);

	return sgt;
}

static void vb2_dma_sg_dmabuf_ops_unmap(struct dma_buf_attachment *db_attach,
	struct sg_table *sgt, enum dma_data_direction dma_dir)
{
	/* nothing to be done here */
}

static void vb2_dma_sg_dmabuf_ops_release(struct dma_buf *dbuf)
{
	/* drop reference obtained in vb2_dma_sg_get_dmabuf */
	vb2_dma_sg_put(dbuf->priv);
}

static void *vb2_dma_sg_dmabuf_ops_kmap(struct dma_buf *dbuf, unsigned long pgnum)
{
	struct vb2_dma_sg_buf *buf = dbuf->priv;

	return buf->vaddr ? buf->vaddr + pgnum * PAGE_SIZE : NULL;
}

static void *vb2_dma_sg_dmabuf_ops_vmap(struct dma_buf *dbuf)
{
	struct vb2_dma_sg_buf *buf = dbuf->priv;

	return vb2_dma_sg_vaddr(buf);
}

static int vb2_dma_sg_dmabuf_ops_mmap(struct dma_buf *dbuf,
	struct vm_area_struct *vma)
{
	return vb2_dma_sg_mmap(dbuf->priv, vma);
}

static struct dma_buf_ops vb2_dma_sg_dmabuf_ops = {
	.attach = vb2_dma_sg_dmabuf_ops_attach,
	.detach = vb2_dma_sg_dmabuf_ops_detach,
	.map_dma_buf = vb2_dma_sg_dmabuf_ops_map,
	.unmap_dma_buf = vb2_dma_sg_dmabuf_ops_unmap,
	.kmap = vb2_dma_sg_dmabuf_ops_kmap,
	.kmap_atomic = vb2_dma_sg_dmabuf_ops_kmap,
	.vmap = vb2_dma_sg_dmabuf_ops_vmap,
	.mmap = vb2_dma_sg_dmabuf_ops_mmap,
	.release = vb2_dma_sg_dmabuf_ops_release,
};

static struct dma_buf *vb2_dma_sg_get_dmabuf(void *buf_priv, unsigned long flags)
{
	struct vb2_dma_sg_buf *buf = buf_priv;
	struct dma_buf *dbuf;
	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);

	exp_info.ops = &vb2_dma_sg_dmabuf_ops;
	exp_info.size = buf->size;
	exp_info.flags = flags;
	exp_info.priv = buf;

	if (WARN_ON(!buf->dma_sgt))
		return NULL;

	dbuf = dma_buf_export(&exp_info);
	if (IS_ERR(dbuf))
		return NULL;

	/* dmabuf keeps reference to vb2 buffer */
	atomic_inc(&buf->refcount);

	return dbuf;
}

/*********************************************/
/*       callbacks for DMABUF buffers        */
/*********************************************/

static int vb2_dma_sg_map_dmabuf(void *mem_priv)
{
	struct vb2_dma_sg_buf *buf = mem_priv;
	struct sg_table *sgt;

	if (WARN_ON(!buf->db_attach)) {
		pr_err("trying to pin a non attached buffer\n");
		return -EINVAL;
	}

	if (WARN_ON(buf->dma_sgt)) {
		pr_err("dmabuf buffer is already pinned\n");
		return 0;
	}

	/* get the associated scatterlist for this buffer */
	sgt = dma_buf_map_attachment(buf->db_attach, buf->dma_dir);
	if (IS_ERR(sgt)) {
		pr_err("Error getting dmabuf scatterlist\n");
		return -EINVAL;
	}

	buf->dma_sgt = sgt;
	buf->vaddr = NULL;

	return 0;
}

static void vb2_dma_sg_unmap_dmabuf(void *mem_priv)
{
	struct vb2_dma_sg_buf *buf = mem_priv;
	struct sg_table *sgt = buf->dma_sgt;

	if (WARN_ON(!buf->db_attach)) {
		pr_err("trying to unpin a not attached buffer\n");
		return;
	}

	if (WARN_ON(!sgt)) {
		pr_err("dmabuf buffer is already unpinned\n");
		return;
	}

	if (buf->vaddr) {
		dma_buf_vunmap(buf->db_attach->dmabuf, buf->vaddr);
		buf->vaddr = NULL;
	}
	dma_buf_unmap_attachment(buf->db_attach, sgt, buf->dma_dir);

	buf->dma_sgt = NULL;
}

static void vb2_dma_sg_detach_dmabuf(void *mem_priv)
{
	struct vb2_dma_sg_buf *buf = mem_priv;

	/* if vb2 works correctly you should never detach mapped buffer */
	if (WARN_ON(buf->dma_sgt))
		vb2_dma_sg_unmap_dmabuf(buf);

	/* detach this attachment */
	dma_buf_detach(buf->db_attach->dmabuf, buf->db_attach);
	kfree(buf);
}

static void *vb2_dma_sg_attach_dmabuf(void *alloc_ctx, struct dma_buf *dbuf,
	unsigned long size, enum dma_data_direction dma_dir)
{
	struct vb2_dma_sg_conf *conf = alloc_ctx;
	struct vb2_dma_sg_buf *buf;
	struct dma_buf_attachment *dba;

	if (dbuf->size < size)
		return ERR_PTR(-EFAULT);

	buf = kzalloc(sizeof(*buf), GFP_KERNEL);
	if (!buf)
		return ERR_PTR(-ENOMEM);

	buf->dev = conf->dev;
	/* create attachment for the dmabuf with the user device */
	dba = dma_buf_attach(dbuf, buf->dev);
	if (IS_ERR(dba)) {
		pr_err("failed to attach dmabuf\n");
		kfree(buf);
		return dba;
	}

	buf->dma_dir = dma_dir;
	buf->size = size;
	buf->db_attach = dba;

	return buf;
}

static void *vb2_dma_sg_cookie(void *buf_priv)
{
	struct vb2_dma_sg_buf *buf = buf_priv;

	return buf->dma_sgt;
}

const struct vb2_mem_ops vb2_dma_sg_memops = {
	.alloc		= vb2_dma_sg_alloc,
	.put		= vb2_dma_sg_put,
	.get_userptr	= vb2_dma_sg_get_userptr,
	.put_userptr	= vb2_dma_sg_put_userptr,
	.prepare	= vb2_dma_sg_prepare,
	.finish		= vb2_dma_sg_finish,
	.vaddr		= vb2_dma_sg_vaddr,
	.mmap		= vb2_dma_sg_mmap,
	.num_users	= vb2_dma_sg_num_users,
	.get_dmabuf	= vb2_dma_sg_get_dmabuf,
	.map_dmabuf	= vb2_dma_sg_map_dmabuf,
	.unmap_dmabuf	= vb2_dma_sg_unmap_dmabuf,
	.attach_dmabuf	= vb2_dma_sg_attach_dmabuf,
	.detach_dmabuf	= vb2_dma_sg_detach_dmabuf,
	.cookie		= vb2_dma_sg_cookie,
};
EXPORT_SYMBOL_GPL(vb2_dma_sg_memops);

void *vb2_dma_sg_init_ctx(struct device *dev)
{
	struct vb2_dma_sg_conf *conf;

	conf = kzalloc(sizeof(*conf), GFP_KERNEL);
	if (!conf)
		return ERR_PTR(-ENOMEM);

	conf->dev = dev;

	return conf;
}
EXPORT_SYMBOL_GPL(vb2_dma_sg_init_ctx);

void vb2_dma_sg_cleanup_ctx(void *alloc_ctx)
{
	if (!IS_ERR_OR_NULL(alloc_ctx))
		kfree(alloc_ctx);
}
EXPORT_SYMBOL_GPL(vb2_dma_sg_cleanup_ctx);

MODULE_DESCRIPTION("dma scatter/gather memory handling routines for videobuf2");
MODULE_AUTHOR("Andrzej Pietrasiewicz");
MODULE_LICENSE("GPL");
Commit	Line	Data
5ba3f757 AP	1	/*
	2	* videobuf2-dma-sg.c - dma scatter/gather memory allocator for videobuf2
	3	*
	4	* Copyright (C) 2010 Samsung Electronics
	5	*
	6	* Author: Andrzej Pietrasiewicz <andrzej.p@samsung.com>
	7	*
	8	* This program is free software; you can redistribute it and/or modify
	9	* it under the terms of the GNU General Public License as published by
	10	* the Free Software Foundation.
	11	*/
	12
	13	#include <linux/module.h>
	14	#include <linux/mm.h>
	15	#include <linux/scatterlist.h>
	16	#include <linux/sched.h>
	17	#include <linux/slab.h>
	18	#include <linux/vmalloc.h>
	19
c139990e	20	#include <media/videobuf2-v4l2.h>
5ba3f757 AP	21	#include <media/videobuf2-memops.h>
	22	#include <media/videobuf2-dma-sg.h>
	23
ffdc78ef HV	24	static int debug;
	25	module_param(debug, int, 0644);
	26
	27	#define dprintk(level, fmt, arg...) \
	28	do { \
	29	if (debug >= level) \
	30	printk(KERN_DEBUG "vb2-dma-sg: " fmt, ## arg); \
	31	} while (0)
	32
0c3a14c1 HV	33	struct vb2_dma_sg_conf {
	34	struct device *dev;
	35	};
	36
5ba3f757	37	struct vb2_dma_sg_buf {
0c3a14c1	38	struct device *dev;
5ba3f757 AP	39	void *vaddr;
5ba3f757 AP	40	struct page **pages;
3336c24f	41	struct frame_vector *vec;
5ba3f757	42	int offset;
cd474037	43	enum dma_data_direction dma_dir;
22301247	44	struct sg_table sg_table;
e078b79d HV	45	/*
	46	* This will point to sg_table when used with the MMAP or USERPTR
	47	* memory model, and to the dma_buf sglist when used with the
	48	* DMABUF memory model.
	49	*/
	50	struct sg_table *dma_sgt;
22301247 RRD	51	size_t size;
22301247 RRD	52	unsigned int num_pages;
5ba3f757 AP	53	atomic_t refcount;
5ba3f757 AP	54	struct vb2_vmarea_handler handler;
e078b79d HV	55
e078b79d HV	56	struct dma_buf_attachment *db_attach;
5ba3f757 AP	57	};
	58
	59	static void vb2_dma_sg_put(void *buf_priv);
	60
df237281 RRD	61	static int vb2_dma_sg_alloc_compacted(struct vb2_dma_sg_buf *buf,
	62	gfp_t gfp_flags)
	63	{
	64	unsigned int last_page = 0;
22301247	65	int size = buf->size;
df237281 RRD	66
	67	while (size > 0) {
	68	struct page *pages;
	69	int order;
	70	int i;
	71
	72	order = get_order(size);
	73	/* Dont over allocate*/
	74	if ((PAGE_SIZE << order) > size)
	75	order--;
	76
	77	pages = NULL;
	78	while (!pages) {
	79	pages = alloc_pages(GFP_KERNEL \| __GFP_ZERO \|
	80	__GFP_NOWARN \| gfp_flags, order);
	81	if (pages)
	82	break;
	83
	84	if (order == 0) {
	85	while (last_page--)
	86	__free_page(buf->pages[last_page]);
	87	return -ENOMEM;
	88	}
	89	order--;
	90	}
	91
	92	split_page(pages, order);
22301247 RRD	93	for (i = 0; i < (1 << order); i++)
22301247 RRD	94	buf->pages[last_page++] = &pages[i];
df237281 RRD	95
	96	size -= PAGE_SIZE << order;
	97	}
	98
	99	return 0;
	100	}
	101
d935c57e HV	102	static void vb2_dma_sg_alloc(void alloc_ctx, unsigned long size,
d935c57e HV	103	enum dma_data_direction dma_dir, gfp_t gfp_flags)
5ba3f757	104	{
0c3a14c1	105	struct vb2_dma_sg_conf *conf = alloc_ctx;
5ba3f757	106	struct vb2_dma_sg_buf *buf;
d790b7ed	107	struct sg_table *sgt;
df237281	108	int ret;
22301247	109	int num_pages;
251a79f8 HV	110	DEFINE_DMA_ATTRS(attrs);
	111
	112	dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs);
5ba3f757	113
0c3a14c1 HV	114	if (WARN_ON(alloc_ctx == NULL))
0c3a14c1 HV	115	return NULL;
5ba3f757 AP	116	buf = kzalloc(sizeof *buf, GFP_KERNEL);
	117	if (!buf)
	118	return NULL;
	119
	120	buf->vaddr = NULL;
d935c57e	121	buf->dma_dir = dma_dir;
5ba3f757	122	buf->offset = 0;
22301247	123	buf->size = size;
7f841459	124	/* size is already page aligned */
22301247	125	buf->num_pages = size >> PAGE_SHIFT;
e078b79d	126	buf->dma_sgt = &buf->sg_table;
5ba3f757	127
22301247	128	buf->pages = kzalloc(buf->num_pages * sizeof(struct page *),
5ba3f757 AP	129	GFP_KERNEL);
	130	if (!buf->pages)
	131	goto fail_pages_array_alloc;
	132
df237281 RRD	133	ret = vb2_dma_sg_alloc_compacted(buf, gfp_flags);
	134	if (ret)
	135	goto fail_pages_alloc;
5ba3f757	136
e078b79d	137	ret = sg_alloc_table_from_pages(buf->dma_sgt, buf->pages,
47bc59c5	138	buf->num_pages, 0, size, GFP_KERNEL);
22301247 RRD	139	if (ret)
	140	goto fail_table_alloc;
	141
0c3a14c1 HV	142	/* Prevent the device from being released while the buffer is used */
0c3a14c1 HV	143	buf->dev = get_device(conf->dev);
d790b7ed HV	144
d790b7ed HV	145	sgt = &buf->sg_table;
251a79f8 HV	146	/*
	147	* No need to sync to the device, this will happen later when the
	148	* prepare() memop is called.
	149	*/
6a5d77cb RRD	150	sgt->nents = dma_map_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,
	151	buf->dma_dir, &attrs);
	152	if (!sgt->nents)
d790b7ed	153	goto fail_map;
d790b7ed	154
5ba3f757 AP	155	buf->handler.refcount = &buf->refcount;
	156	buf->handler.put = vb2_dma_sg_put;
	157	buf->handler.arg = buf;
	158
	159	atomic_inc(&buf->refcount);
	160
ffdc78ef	161	dprintk(1, "%s: Allocated buffer of %d pages\n",
22301247	162	__func__, buf->num_pages);
5ba3f757 AP	163	return buf;
5ba3f757 AP	164
d790b7ed HV	165	fail_map:
d790b7ed HV	166	put_device(buf->dev);
e078b79d	167	sg_free_table(buf->dma_sgt);
22301247 RRD	168	fail_table_alloc:
	169	num_pages = buf->num_pages;
	170	while (num_pages--)
	171	__free_page(buf->pages[num_pages]);
5ba3f757	172	fail_pages_alloc:
a9bb36aa	173	kfree(buf->pages);
5ba3f757	174	fail_pages_array_alloc:
5ba3f757 AP	175	kfree(buf);
	176	return NULL;
	177	}
	178
	179	static void vb2_dma_sg_put(void *buf_priv)
	180	{
	181	struct vb2_dma_sg_buf *buf = buf_priv;
d790b7ed	182	struct sg_table *sgt = &buf->sg_table;
22301247	183	int i = buf->num_pages;
5ba3f757 AP	184
5ba3f757 AP	185	if (atomic_dec_and_test(&buf->refcount)) {
251a79f8 HV	186	DEFINE_DMA_ATTRS(attrs);
	187
	188	dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs);
ffdc78ef	189	dprintk(1, "%s: Freeing buffer of %d pages\n", __func__,
22301247	190	buf->num_pages);
6a5d77cb	191	dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,
251a79f8	192	buf->dma_dir, &attrs);
5ba3f757	193	if (buf->vaddr)
22301247	194	vm_unmap_ram(buf->vaddr, buf->num_pages);
e078b79d	195	sg_free_table(buf->dma_sgt);
5ba3f757 AP	196	while (--i >= 0)
	197	__free_page(buf->pages[i]);
	198	kfree(buf->pages);
0c3a14c1	199	put_device(buf->dev);
5ba3f757 AP	200	kfree(buf);
	201	}
	202	}
	203
d790b7ed HV	204	static void vb2_dma_sg_prepare(void *buf_priv)
	205	{
	206	struct vb2_dma_sg_buf *buf = buf_priv;
e078b79d HV	207	struct sg_table *sgt = buf->dma_sgt;
	208
	209	/* DMABUF exporter will flush the cache for us */
	210	if (buf->db_attach)
	211	return;
d790b7ed HV	212
	213	dma_sync_sg_for_device(buf->dev, sgt->sgl, sgt->nents, buf->dma_dir);
	214	}
	215
	216	static void vb2_dma_sg_finish(void *buf_priv)
	217	{
	218	struct vb2_dma_sg_buf *buf = buf_priv;
e078b79d HV	219	struct sg_table *sgt = buf->dma_sgt;
	220
	221	/* DMABUF exporter will flush the cache for us */
	222	if (buf->db_attach)
	223	return;
d790b7ed HV	224
	225	dma_sync_sg_for_cpu(buf->dev, sgt->sgl, sgt->nents, buf->dma_dir);
	226	}
	227
5ba3f757	228	static void vb2_dma_sg_get_userptr(void alloc_ctx, unsigned long vaddr,
cd474037 HV	229	unsigned long size,
cd474037 HV	230	enum dma_data_direction dma_dir)
5ba3f757	231	{
d790b7ed	232	struct vb2_dma_sg_conf *conf = alloc_ctx;
5ba3f757	233	struct vb2_dma_sg_buf *buf;
d790b7ed	234	struct sg_table *sgt;
251a79f8	235	DEFINE_DMA_ATTRS(attrs);
3336c24f	236	struct frame_vector *vec;
251a79f8 HV	237
251a79f8 HV	238	dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs);
5ba3f757 AP	239	buf = kzalloc(sizeof *buf, GFP_KERNEL);
	240	if (!buf)
	241	return NULL;
	242
	243	buf->vaddr = NULL;
d790b7ed	244	buf->dev = conf->dev;
cd474037	245	buf->dma_dir = dma_dir;
5ba3f757	246	buf->offset = vaddr & ~PAGE_MASK;
22301247	247	buf->size = size;
e078b79d	248	buf->dma_sgt = &buf->sg_table;
3336c24f JK	249	vec = vb2_create_framevec(vaddr, size, buf->dma_dir == DMA_FROM_DEVICE);
	250	if (IS_ERR(vec))
	251	goto userptr_fail_pfnvec;
	252	buf->vec = vec;
5ba3f757	253
3336c24f JK	254	buf->pages = frame_vector_pages(vec);
	255	if (IS_ERR(buf->pages))
	256	goto userptr_fail_sgtable;
	257	buf->num_pages = frame_vector_count(vec);
5ba3f757	258
e078b79d	259	if (sg_alloc_table_from_pages(buf->dma_sgt, buf->pages,
22301247	260	buf->num_pages, buf->offset, size, 0))
3336c24f	261	goto userptr_fail_sgtable;
22301247	262
d790b7ed	263	sgt = &buf->sg_table;
251a79f8 HV	264	/*
	265	* No need to sync to the device, this will happen later when the
	266	* prepare() memop is called.
	267	*/
6a5d77cb RRD	268	sgt->nents = dma_map_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,
	269	buf->dma_dir, &attrs);
	270	if (!sgt->nents)
d790b7ed	271	goto userptr_fail_map;
6a5d77cb	272
5ba3f757 AP	273	return buf;
5ba3f757 AP	274
d790b7ed HV	275	userptr_fail_map:
d790b7ed HV	276	sg_free_table(&buf->sg_table);
3336c24f JK	277	userptr_fail_sgtable:
	278	vb2_destroy_framevec(vec);
	279	userptr_fail_pfnvec:
5ba3f757 AP	280	kfree(buf);
	281	return NULL;
	282	}
	283
	284	/*
	285	* @put_userptr: inform the allocator that a USERPTR buffer will no longer
	286	* be used
	287	*/
	288	static void vb2_dma_sg_put_userptr(void *buf_priv)
	289	{
	290	struct vb2_dma_sg_buf *buf = buf_priv;
d790b7ed	291	struct sg_table *sgt = &buf->sg_table;
22301247	292	int i = buf->num_pages;
251a79f8 HV	293	DEFINE_DMA_ATTRS(attrs);
	294
	295	dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs);
5ba3f757	296
ffdc78ef	297	dprintk(1, "%s: Releasing userspace buffer of %d pages\n",
22301247	298	__func__, buf->num_pages);
6a5d77cb RRD	299	dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents, buf->dma_dir,
6a5d77cb RRD	300	&attrs);
5ba3f757	301	if (buf->vaddr)
22301247	302	vm_unmap_ram(buf->vaddr, buf->num_pages);
e078b79d	303	sg_free_table(buf->dma_sgt);
5ba3f757	304	while (--i >= 0) {
cd474037	305	if (buf->dma_dir == DMA_FROM_DEVICE)
5ba3f757	306	set_page_dirty_lock(buf->pages[i]);
5ba3f757	307	}
3336c24f	308	vb2_destroy_framevec(buf->vec);
5ba3f757 AP	309	kfree(buf);
	310	}
	311
	312	static void vb2_dma_sg_vaddr(void buf_priv)
	313	{
	314	struct vb2_dma_sg_buf *buf = buf_priv;
	315
	316	BUG_ON(!buf);
	317
e078b79d HV	318	if (!buf->vaddr) {
	319	if (buf->db_attach)
	320	buf->vaddr = dma_buf_vmap(buf->db_attach->dmabuf);
	321	else
	322	buf->vaddr = vm_map_ram(buf->pages,
	323	buf->num_pages, -1, PAGE_KERNEL);
	324	}
5ba3f757 AP	325
5ba3f757 AP	326	/* add offset in case userptr is not page-aligned */
e078b79d	327	return buf->vaddr ? buf->vaddr + buf->offset : NULL;
5ba3f757 AP	328	}
	329
	330	static unsigned int vb2_dma_sg_num_users(void *buf_priv)
	331	{
	332	struct vb2_dma_sg_buf *buf = buf_priv;
	333
	334	return atomic_read(&buf->refcount);
	335	}
	336
	337	static int vb2_dma_sg_mmap(void buf_priv, struct vm_area_struct vma)
	338	{
	339	struct vb2_dma_sg_buf *buf = buf_priv;
	340	unsigned long uaddr = vma->vm_start;
	341	unsigned long usize = vma->vm_end - vma->vm_start;
	342	int i = 0;
	343
	344	if (!buf) {
	345	printk(KERN_ERR "No memory to map\n");
	346	return -EINVAL;
	347	}
	348
	349	do {
	350	int ret;
	351
	352	ret = vm_insert_page(vma, uaddr, buf->pages[i++]);
	353	if (ret) {
	354	printk(KERN_ERR "Remapping memory, error: %d\n", ret);
	355	return ret;
	356	}
	357
	358	uaddr += PAGE_SIZE;
	359	usize -= PAGE_SIZE;
	360	} while (usize > 0);
	361
	362
	363	/*
	364	* Use common vm_area operations to track buffer refcount.
	365	*/
	366	vma->vm_private_data = &buf->handler;
	367	vma->vm_ops = &vb2_common_vm_ops;
	368
	369	vma->vm_ops->open(vma);
	370
	371	return 0;
	372	}
	373
041c7b6a HV	374	/*********************************************/
	375	/* DMABUF ops for exporters */
	376	/*********************************************/
	377
	378	struct vb2_dma_sg_attachment {
	379	struct sg_table sgt;
	380	enum dma_data_direction dma_dir;
	381	};
	382
	383	static int vb2_dma_sg_dmabuf_ops_attach(struct dma_buf dbuf, struct device dev,
	384	struct dma_buf_attachment *dbuf_attach)
	385	{
	386	struct vb2_dma_sg_attachment *attach;
	387	unsigned int i;
	388	struct scatterlist rd, wr;
	389	struct sg_table *sgt;
	390	struct vb2_dma_sg_buf *buf = dbuf->priv;
	391	int ret;
	392
	393	attach = kzalloc(sizeof(*attach), GFP_KERNEL);
	394	if (!attach)
	395	return -ENOMEM;
	396
	397	sgt = &attach->sgt;
	398	/* Copy the buf->base_sgt scatter list to the attachment, as we can't
	399	* map the same scatter list to multiple attachments at the same time.
	400	*/
	401	ret = sg_alloc_table(sgt, buf->dma_sgt->orig_nents, GFP_KERNEL);
	402	if (ret) {
	403	kfree(attach);
	404	return -ENOMEM;
	405	}
	406
	407	rd = buf->dma_sgt->sgl;
	408	wr = sgt->sgl;
	409	for (i = 0; i < sgt->orig_nents; ++i) {
	410	sg_set_page(wr, sg_page(rd), rd->length, rd->offset);
	411	rd = sg_next(rd);
	412	wr = sg_next(wr);
	413	}
	414
	415	attach->dma_dir = DMA_NONE;
	416	dbuf_attach->priv = attach;
	417
	418	return 0;
	419	}
	420
	421	static void vb2_dma_sg_dmabuf_ops_detach(struct dma_buf *dbuf,
	422	struct dma_buf_attachment *db_attach)
	423	{
	424	struct vb2_dma_sg_attachment *attach = db_attach->priv;
	425	struct sg_table *sgt;
	426
	427	if (!attach)
	428	return;
	429
	430	sgt = &attach->sgt;
	431
	432	/* release the scatterlist cache */
	433	if (attach->dma_dir != DMA_NONE)
	434	dma_unmap_sg(db_attach->dev, sgt->sgl, sgt->orig_nents,
	435	attach->dma_dir);
	436	sg_free_table(sgt);
	437	kfree(attach);
438	db_attach->priv = NULL;
439	}
440
441	static struct sg_table *vb2_dma_sg_dmabuf_ops_map(
442	struct dma_buf_attachment *db_attach, enum dma_data_direction dma_dir)
443	{
444	struct vb2_dma_sg_attachment *attach = db_attach->priv;
445	/* stealing dmabuf mutex to serialize map/unmap operations */
446	struct mutex *lock = &db_attach->dmabuf->lock;
447	struct sg_table *sgt;
041c7b6a HV	448
	449	mutex_lock(lock);
	450
	451	sgt = &attach->sgt;
	452	/* return previously mapped sg table */
	453	if (attach->dma_dir == dma_dir) {
	454	mutex_unlock(lock);
	455	return sgt;
	456	}
	457
	458	/* release any previous cache */
	459	if (attach->dma_dir != DMA_NONE) {
	460	dma_unmap_sg(db_attach->dev, sgt->sgl, sgt->orig_nents,
	461	attach->dma_dir);
	462	attach->dma_dir = DMA_NONE;
	463	}
	464
	465	/* mapping to the client with new direction */
6a5d77cb RRD	466	sgt->nents = dma_map_sg(db_attach->dev, sgt->sgl, sgt->orig_nents,
	467	dma_dir);
	468	if (!sgt->nents) {
041c7b6a HV	469	pr_err("failed to map scatterlist\n");
	470	mutex_unlock(lock);
	471	return ERR_PTR(-EIO);
	472	}
	473
	474	attach->dma_dir = dma_dir;
	475
	476	mutex_unlock(lock);
	477
	478	return sgt;
	479	}
	480
	481	static void vb2_dma_sg_dmabuf_ops_unmap(struct dma_buf_attachment *db_attach,
	482	struct sg_table *sgt, enum dma_data_direction dma_dir)
	483	{
	484	/* nothing to be done here */
	485	}
	486
	487	static void vb2_dma_sg_dmabuf_ops_release(struct dma_buf *dbuf)
	488	{
	489	/* drop reference obtained in vb2_dma_sg_get_dmabuf */
	490	vb2_dma_sg_put(dbuf->priv);
	491	}
	492
	493	static void vb2_dma_sg_dmabuf_ops_kmap(struct dma_buf dbuf, unsigned long pgnum)
	494	{
	495	struct vb2_dma_sg_buf *buf = dbuf->priv;
	496
	497	return buf->vaddr ? buf->vaddr + pgnum * PAGE_SIZE : NULL;
	498	}
	499
	500	static void vb2_dma_sg_dmabuf_ops_vmap(struct dma_buf dbuf)
	501	{
	502	struct vb2_dma_sg_buf *buf = dbuf->priv;
	503
	504	return vb2_dma_sg_vaddr(buf);
	505	}
	506
	507	static int vb2_dma_sg_dmabuf_ops_mmap(struct dma_buf *dbuf,
	508	struct vm_area_struct *vma)
	509	{
	510	return vb2_dma_sg_mmap(dbuf->priv, vma);
	511	}
	512
	513	static struct dma_buf_ops vb2_dma_sg_dmabuf_ops = {
	514	.attach = vb2_dma_sg_dmabuf_ops_attach,
	515	.detach = vb2_dma_sg_dmabuf_ops_detach,
	516	.map_dma_buf = vb2_dma_sg_dmabuf_ops_map,
	517	.unmap_dma_buf = vb2_dma_sg_dmabuf_ops_unmap,
	518	.kmap = vb2_dma_sg_dmabuf_ops_kmap,
	519	.kmap_atomic = vb2_dma_sg_dmabuf_ops_kmap,
	520	.vmap = vb2_dma_sg_dmabuf_ops_vmap,
	521	.mmap = vb2_dma_sg_dmabuf_ops_mmap,
	522	.release = vb2_dma_sg_dmabuf_ops_release,
	523	};
	524
	525	static struct dma_buf vb2_dma_sg_get_dmabuf(void buf_priv, unsigned long flags)
	526	{
	527	struct vb2_dma_sg_buf *buf = buf_priv;
	528	struct dma_buf *dbuf;
d8fbe341 SS	529	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
	530
	531	exp_info.ops = &vb2_dma_sg_dmabuf_ops;
	532	exp_info.size = buf->size;
	533	exp_info.flags = flags;
	534	exp_info.priv = buf;
041c7b6a HV	535
	536	if (WARN_ON(!buf->dma_sgt))
	537	return NULL;
	538
d8fbe341	539	dbuf = dma_buf_export(&exp_info);
041c7b6a HV	540	if (IS_ERR(dbuf))
	541	return NULL;
	542
	543	/* dmabuf keeps reference to vb2 buffer */
	544	atomic_inc(&buf->refcount);
	545
	546	return dbuf;
	547	}
	548
e078b79d HV	549	/*********************************************/
	550	/* callbacks for DMABUF buffers */
	551	/*********************************************/
	552
	553	static int vb2_dma_sg_map_dmabuf(void *mem_priv)
	554	{
	555	struct vb2_dma_sg_buf *buf = mem_priv;
	556	struct sg_table *sgt;
	557
	558	if (WARN_ON(!buf->db_attach)) {
	559	pr_err("trying to pin a non attached buffer\n");
	560	return -EINVAL;
	561	}
	562
	563	if (WARN_ON(buf->dma_sgt)) {
	564	pr_err("dmabuf buffer is already pinned\n");
	565	return 0;
	566	}
	567
	568	/* get the associated scatterlist for this buffer */
	569	sgt = dma_buf_map_attachment(buf->db_attach, buf->dma_dir);
	570	if (IS_ERR(sgt)) {
	571	pr_err("Error getting dmabuf scatterlist\n");
	572	return -EINVAL;
	573	}
	574
	575	buf->dma_sgt = sgt;
	576	buf->vaddr = NULL;
	577
	578	return 0;
	579	}
	580
	581	static void vb2_dma_sg_unmap_dmabuf(void *mem_priv)
	582	{
	583	struct vb2_dma_sg_buf *buf = mem_priv;
	584	struct sg_table *sgt = buf->dma_sgt;
	585
	586	if (WARN_ON(!buf->db_attach)) {
	587	pr_err("trying to unpin a not attached buffer\n");
	588	return;
	589	}
	590
	591	if (WARN_ON(!sgt)) {
	592	pr_err("dmabuf buffer is already unpinned\n");
	593	return;
	594	}
	595
	596	if (buf->vaddr) {
	597	dma_buf_vunmap(buf->db_attach->dmabuf, buf->vaddr);
	598	buf->vaddr = NULL;
	599	}
	600	dma_buf_unmap_attachment(buf->db_attach, sgt, buf->dma_dir);
	601
	602	buf->dma_sgt = NULL;
	603	}
	604
	605	static void vb2_dma_sg_detach_dmabuf(void *mem_priv)
	606	{
	607	struct vb2_dma_sg_buf *buf = mem_priv;
	608
	609	/* if vb2 works correctly you should never detach mapped buffer */
	610	if (WARN_ON(buf->dma_sgt))
	611	vb2_dma_sg_unmap_dmabuf(buf);
	612
613	/* detach this attachment */
614	dma_buf_detach(buf->db_attach->dmabuf, buf->db_attach);
615	kfree(buf);
616	}
617
618	static void vb2_dma_sg_attach_dmabuf(void alloc_ctx, struct dma_buf *dbuf,
619	unsigned long size, enum dma_data_direction dma_dir)
620	{
621	struct vb2_dma_sg_conf *conf = alloc_ctx;
622	struct vb2_dma_sg_buf *buf;
623	struct dma_buf_attachment *dba;
624
625	if (dbuf->size < size)
626	return ERR_PTR(-EFAULT);
627
628	buf = kzalloc(sizeof(*buf), GFP_KERNEL);
629	if (!buf)
630	return ERR_PTR(-ENOMEM);
631
632	buf->dev = conf->dev;
633	/* create attachment for the dmabuf with the user device */
634	dba = dma_buf_attach(dbuf, buf->dev);
635	if (IS_ERR(dba)) {
636	pr_err("failed to attach dmabuf\n");
637	kfree(buf);
638	return dba;
639	}
640
641	buf->dma_dir = dma_dir;
642	buf->size = size;
643	buf->db_attach = dba;
644
645	return buf;
646	}
647
5ba3f757 AP	648	static void vb2_dma_sg_cookie(void buf_priv)
	649	{
	650	struct vb2_dma_sg_buf *buf = buf_priv;
	651
e078b79d	652	return buf->dma_sgt;
5ba3f757 AP	653	}
	654
	655	const struct vb2_mem_ops vb2_dma_sg_memops = {
	656	.alloc = vb2_dma_sg_alloc,
	657	.put = vb2_dma_sg_put,
	658	.get_userptr = vb2_dma_sg_get_userptr,
	659	.put_userptr = vb2_dma_sg_put_userptr,
d790b7ed HV	660	.prepare = vb2_dma_sg_prepare,
d790b7ed HV	661	.finish = vb2_dma_sg_finish,
5ba3f757 AP	662	.vaddr = vb2_dma_sg_vaddr,
	663	.mmap = vb2_dma_sg_mmap,
	664	.num_users = vb2_dma_sg_num_users,
041c7b6a	665	.get_dmabuf = vb2_dma_sg_get_dmabuf,
e078b79d HV	666	.map_dmabuf = vb2_dma_sg_map_dmabuf,
	667	.unmap_dmabuf = vb2_dma_sg_unmap_dmabuf,
	668	.attach_dmabuf = vb2_dma_sg_attach_dmabuf,
	669	.detach_dmabuf = vb2_dma_sg_detach_dmabuf,
5ba3f757 AP	670	.cookie = vb2_dma_sg_cookie,
	671	};
	672	EXPORT_SYMBOL_GPL(vb2_dma_sg_memops);
	673
0c3a14c1 HV	674	void vb2_dma_sg_init_ctx(struct device dev)
	675	{
	676	struct vb2_dma_sg_conf *conf;
	677
	678	conf = kzalloc(sizeof(*conf), GFP_KERNEL);
	679	if (!conf)
	680	return ERR_PTR(-ENOMEM);
	681
	682	conf->dev = dev;
	683
	684	return conf;
	685	}
	686	EXPORT_SYMBOL_GPL(vb2_dma_sg_init_ctx);
	687
	688	void vb2_dma_sg_cleanup_ctx(void *alloc_ctx)
	689	{
	690	if (!IS_ERR_OR_NULL(alloc_ctx))
	691	kfree(alloc_ctx);
	692	}
	693	EXPORT_SYMBOL_GPL(vb2_dma_sg_cleanup_ctx);
	694
5ba3f757 AP	695	MODULE_DESCRIPTION("dma scatter/gather memory handling routines for videobuf2");
	696	MODULE_AUTHOR("Andrzej Pietrasiewicz");
	697	MODULE_LICENSE("GPL");